From 095fa75f56833259cb4cd6c8fb3cf1d4f6c17602 Mon Sep 17 00:00:00 2001 From: Aljoscha Krettek Date: Wed, 3 Feb 2016 11:07:38 +0100 Subject: [PATCH 1/3] [hotfix] Fix typos in Trigger.java --- .../plantranslate/JobGraphGenerator.java | 2 +- .../flink/runtime/jobgraph/JobGraph.java | 48 +++++++++++++++---- .../ExecutionGraphConstructionTest.java | 5 +- .../flink/runtime/jobgraph/JobGraphTest.java | 26 ++++++---- .../jobgraph/jsonplan/JsonGeneratorTest.java | 9 ++-- .../ScheduleOrUpdateConsumersTest.java | 9 ++-- .../runtime/jobmanager/JobManagerITCase.scala | 12 +++-- .../jobmanager/SlotSharingITCase.scala | 5 +- .../api/graph/StreamingJobGraphGenerator.java | 2 +- 9 files changed, 83 insertions(+), 35 deletions(-) diff --git a/flink-optimizer/src/main/java/org/apache/flink/optimizer/plantranslate/JobGraphGenerator.java b/flink-optimizer/src/main/java/org/apache/flink/optimizer/plantranslate/JobGraphGenerator.java index 159a94ae4ecb30..165943dee60e15 100644 --- a/flink-optimizer/src/main/java/org/apache/flink/optimizer/plantranslate/JobGraphGenerator.java +++ b/flink-optimizer/src/main/java/org/apache/flink/optimizer/plantranslate/JobGraphGenerator.java @@ -217,7 +217,7 @@ public JobGraph compileJobGraph(OptimizedPlan program, JobID jobId) { // ----------- finalize the job graph ----------- // create the job graph object - JobGraph graph = new JobGraph(jobId, program.getJobName()); + JobGraph graph = new JobGraph(jobId, program.getJobName(), program.getOriginalPlan().getExecutionConfig()); graph.setNumberOfExecutionRetries(program.getOriginalPlan().getNumberOfExecutionRetries()); graph.setExecutionRetryDelay(program.getOriginalPlan().getExecutionRetryDelay()); graph.setAllowQueuedScheduling(false); diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/jobgraph/JobGraph.java b/flink-runtime/src/main/java/org/apache/flink/runtime/jobgraph/JobGraph.java index 403ad67c0c7ecf..fc2ed2f1238320 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/jobgraph/JobGraph.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/jobgraph/JobGraph.java @@ -18,6 +18,7 @@ package org.apache.flink.runtime.jobgraph; +import org.apache.flink.api.common.ExecutionConfig; import org.apache.flink.api.common.InvalidProgramException; import org.apache.flink.api.common.JobID; import org.apache.flink.configuration.Configuration; @@ -33,13 +34,14 @@ import java.net.InetSocketAddress; import java.net.URL; import java.util.ArrayList; -import java.util.Collections; -import java.util.Iterator; import java.util.LinkedHashMap; -import java.util.LinkedHashSet; import java.util.List; import java.util.Map; +import java.util.Collections; +import java.util.Arrays; import java.util.Set; +import java.util.LinkedHashSet; +import java.util.Iterator; /** * The JobGraph represents a Flink dataflow program, at the low level that the JobManager accepts. @@ -78,6 +80,8 @@ public class JobGraph implements Serializable { /** Name of this job. */ private final String jobName; + private final ExecutionConfig executionConfig; + /** The number of times that failed tasks should be re-executed */ private int numExecutionRetries; @@ -114,7 +118,20 @@ public JobGraph() { * @param jobName The name of the job */ public JobGraph(String jobName) { - this(null, jobName); + this(null, jobName, (ExecutionConfig) null); + } + + public JobGraph(JobID jobID, String jobName) { + this(jobID, jobName, (ExecutionConfig) null); + } + + /** + * Constructs a new job graph with the given name, a random job ID. + * + * @param jobName The name of the job + */ + public JobGraph(String jobName, ExecutionConfig config) { + this(null, jobName, config); } /** @@ -123,9 +140,22 @@ public JobGraph(String jobName) { * @param jobId The id of the job. A random ID is generated, if {@code null} is passed. * @param jobName The name of the job. */ - public JobGraph(JobID jobId, String jobName) { + public JobGraph(JobID jobId, String jobName, ExecutionConfig config) { this.jobID = jobId == null ? new JobID() : jobId; this.jobName = jobName == null ? "(unnamed job)" : jobName; + this.executionConfig = config; + } + + public JobGraph(String jobName, JobVertex vertex) { + this(jobName, Collections.singletonList(vertex)); + } + + public JobGraph(String jobName, JobVertex vertex1, JobVertex vertex2) { + this(jobName, Arrays.asList(vertex1, vertex2)); + } + + public JobGraph(JobVertex vertex) { + this(null, Collections.singletonList(vertex)); } /** @@ -133,7 +163,7 @@ public JobGraph(JobID jobId, String jobName) { * * @param vertices The vertices to add to the graph. */ - public JobGraph(JobVertex... vertices) { + public JobGraph(List vertices) { this(null, vertices); } @@ -143,7 +173,7 @@ public JobGraph(JobVertex... vertices) { * @param jobName The name of the job. * @param vertices The vertices to add to the graph. */ - public JobGraph(String jobName, JobVertex... vertices) { + public JobGraph(String jobName, List vertices) { this(null, jobName, vertices); } @@ -154,8 +184,8 @@ public JobGraph(String jobName, JobVertex... vertices) { * @param jobName The name of the job. * @param vertices The vertices to add to the graph. */ - public JobGraph(JobID jobId, String jobName, JobVertex... vertices) { - this(jobId, jobName); + public JobGraph(JobID jobId, String jobName, List vertices) { + this(jobId, jobName, (ExecutionConfig) null); for (JobVertex vertex : vertices) { addVertex(vertex); diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/ExecutionGraphConstructionTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/ExecutionGraphConstructionTest.java index a47ea7769b880b..34f67aadb8ca66 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/ExecutionGraphConstructionTest.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/ExecutionGraphConstructionTest.java @@ -688,8 +688,9 @@ public void testCoLocationConstraintCreation() { // isolated vertex JobVertex v8 = new JobVertex("vertex8"); v8.setParallelism(2); - - JobGraph jg = new JobGraph(jobId, jobName, v1, v2, v3, v4, v5, v6, v7, v8); + + List vertices = Arrays.asList(v1, v2, v3, v4, v5, v6, v7, v8); + JobGraph jg = new JobGraph(jobId, jobName, vertices); ExecutionGraph eg = new ExecutionGraph( TestingUtils.defaultExecutionContext(), diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/jobgraph/JobGraphTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/jobgraph/JobGraphTest.java index ca047e8a603011..1272c3a30c10fc 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/jobgraph/JobGraphTest.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/jobgraph/JobGraphTest.java @@ -20,6 +20,7 @@ import static org.junit.Assert.*; +import java.util.Arrays; import java.util.List; import org.apache.flink.api.common.InvalidProgramException; @@ -89,8 +90,9 @@ public void testTopologicalSort1() { target2.connectNewDataSetAsInput(intermediate2, DistributionPattern.POINTWISE); intermediate2.connectNewDataSetAsInput(intermediate1, DistributionPattern.POINTWISE); intermediate1.connectNewDataSetAsInput(source2, DistributionPattern.POINTWISE); - - JobGraph graph = new JobGraph("TestGraph", source1, source2, intermediate1, intermediate2, target1, target2); + + List vertices = Arrays.asList(source1, source2, intermediate1, intermediate2, target1, target2); + JobGraph graph = new JobGraph("TestGraph", vertices); List sorted = graph.getVerticesSortedTopologicallyFromSources(); assertEquals(6, sorted.size()); @@ -133,8 +135,9 @@ public void testTopologicalSort2() { l12.connectNewDataSetAsInput(source2, DistributionPattern.POINTWISE); l13.connectNewDataSetAsInput(source2, DistributionPattern.POINTWISE); - - JobGraph graph = new JobGraph("TestGraph", source1, source2, root, l11, l13, l12, l2); + + List vertices = Arrays.asList(source1, source2, root, l11, l13, l12, l2); + JobGraph graph = new JobGraph("TestGraph", vertices); List sorted = graph.getVerticesSortedTopologicallyFromSources(); assertEquals(7, sorted.size()); @@ -179,8 +182,9 @@ public void testTopologicalSort3() { op2.connectNewDataSetAsInput(op1, DistributionPattern.POINTWISE); op2.connectNewDataSetAsInput(source, DistributionPattern.POINTWISE); op3.connectNewDataSetAsInput(op2, DistributionPattern.POINTWISE); - - JobGraph graph = new JobGraph("TestGraph", source, op1, op2, op3); + + List vertices = Arrays.asList(source, op1, op2, op3); + JobGraph graph = new JobGraph("TestGraph", vertices); List sorted = graph.getVerticesSortedTopologicallyFromSources(); assertEquals(4, sorted.size()); @@ -208,8 +212,9 @@ public void testTopoSortCyclicGraphNoSources() { v2.connectNewDataSetAsInput(v1, DistributionPattern.POINTWISE); v3.connectNewDataSetAsInput(v2, DistributionPattern.POINTWISE); v4.connectNewDataSetAsInput(v3, DistributionPattern.POINTWISE); - - JobGraph jg = new JobGraph("Cyclic Graph", v1, v2, v3, v4); + + List vertices = Arrays.asList(v1, v2, v3, v4); + JobGraph jg = new JobGraph("Cyclic Graph", vertices); try { jg.getVerticesSortedTopologicallyFromSources(); fail("Failed to raise error on topologically sorting cyclic graph."); @@ -240,8 +245,9 @@ public void testTopoSortCyclicGraphIntermediateCycle() { v3.connectNewDataSetAsInput(v2, DistributionPattern.POINTWISE); v4.connectNewDataSetAsInput(v3, DistributionPattern.POINTWISE); target.connectNewDataSetAsInput(v3, DistributionPattern.POINTWISE); - - JobGraph jg = new JobGraph("Cyclic Graph", v1, v2, v3, v4, source, target); + + List vertices = Arrays.asList(v1, v2, v3, v4, source, target); + JobGraph jg = new JobGraph("Cyclic Graph", vertices); try { jg.getVerticesSortedTopologicallyFromSources(); fail("Failed to raise error on topologically sorting cyclic graph."); diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/jobgraph/jsonplan/JsonGeneratorTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/jobgraph/jsonplan/JsonGeneratorTest.java index 46fb694545a948..98e8e6e5446e5a 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/jobgraph/jsonplan/JsonGeneratorTest.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/jobgraph/jsonplan/JsonGeneratorTest.java @@ -29,7 +29,9 @@ import org.apache.flink.runtime.operators.testutils.DummyInvokable; import org.junit.Test; +import java.util.Arrays; import java.util.Iterator; +import java.util.List; import static org.junit.Assert.*; @@ -65,9 +67,10 @@ public void testGeneratorWithoutAnyAttachements() { sink1.connectNewDataSetAsInput(join2, DistributionPattern.POINTWISE); sink2.connectNewDataSetAsInput(join1, DistributionPattern.ALL_TO_ALL); - - JobGraph jg = new JobGraph("my job", source1, source2, source3, - intermediate1, intermediate2, join1, join2, sink1, sink2); + + List vertices = Arrays.asList(source1, source2, source3, + intermediate1, intermediate2, join1, join2, sink1, sink2); + JobGraph jg = new JobGraph("my job", vertices); String plan = JsonPlanGenerator.generatePlan(jg); assertNotNull(plan); diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/jobmanager/scheduler/ScheduleOrUpdateConsumersTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/jobmanager/scheduler/ScheduleOrUpdateConsumersTest.java index eb4d96fb297593..e345310d441626 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/jobmanager/scheduler/ScheduleOrUpdateConsumersTest.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/jobmanager/scheduler/ScheduleOrUpdateConsumersTest.java @@ -35,6 +35,7 @@ import org.junit.BeforeClass; import org.junit.Test; +import java.util.Arrays; import java.util.List; import static org.apache.flink.runtime.jobmanager.SlotCountExceedingParallelismTest.SubtaskIndexReceiver.CONFIG_KEY; @@ -110,11 +111,13 @@ public void testMixedPipelinedAndBlockingResults() throws Exception { pipelinedReceiver.setSlotSharingGroup(slotSharingGroup); blockingReceiver.setSlotSharingGroup(slotSharingGroup); + List vertices = Arrays.asList(sender, + pipelinedReceiver, + blockingReceiver); + final JobGraph jobGraph = new JobGraph( "Mixed pipelined and blocking result", - sender, - pipelinedReceiver, - blockingReceiver); + vertices); flink.submitJobAndWait(jobGraph, false, TestingUtils.TESTING_DURATION()); } diff --git a/flink-runtime/src/test/scala/org/apache/flink/runtime/jobmanager/JobManagerITCase.scala b/flink-runtime/src/test/scala/org/apache/flink/runtime/jobmanager/JobManagerITCase.scala index ec54b7e3f4f0c6..b398732877e8e7 100644 --- a/flink-runtime/src/test/scala/org/apache/flink/runtime/jobmanager/JobManagerITCase.scala +++ b/flink-runtime/src/test/scala/org/apache/flink/runtime/jobmanager/JobManagerITCase.scala @@ -33,7 +33,6 @@ import org.apache.flink.runtime.jobmanager.scheduler.{NoResourceAvailableExcepti import org.apache.flink.runtime.messages.JobManagerMessages._ import org.apache.flink.runtime.testingUtils.TestingJobManagerMessages._ import org.apache.flink.runtime.testingUtils.{ScalaTestingUtils, TestingUtils} -import org.apache.flink.runtime.testutils.JobManagerActorTestUtils import org.junit.runner.RunWith import org.mockito.Mockito._ import org.scalatest.junit.JUnitRunner @@ -44,6 +43,8 @@ import scala.concurrent.duration._ import scala.language.postfixOps import scala.util.Random +import scala.collection.JavaConverters._ + @RunWith(classOf[JUnitRunner]) class JobManagerITCase(_system: ActorSystem) extends TestKit(_system) @@ -253,7 +254,8 @@ class JobManagerITCase(_system: ActorSystem) receiver.connectNewDataSetAsInput(sender1, DistributionPattern.POINTWISE) receiver.connectNewDataSetAsInput(sender2, DistributionPattern.ALL_TO_ALL) - val jobGraph = new JobGraph("Bipartite Job", sender1, receiver, sender2) + val vertices = (List(sender1, receiver, sender2)).asJava + val jobGraph = new JobGraph("Bipartite Job", vertices) val cluster = TestingUtils.startTestingCluster(6 * num_tasks) val jmGateway = cluster.getLeaderGateway(1 seconds) @@ -298,7 +300,8 @@ class JobManagerITCase(_system: ActorSystem) receiver.connectNewDataSetAsInput(sender1, DistributionPattern.POINTWISE) receiver.connectNewDataSetAsInput(sender2, DistributionPattern.ALL_TO_ALL) - val jobGraph = new JobGraph("Bipartite Job", sender1, receiver, sender2) + val vertices = (List(sender1, receiver, sender2)).asJava + val jobGraph = new JobGraph("Bipartite Job", vertices) val cluster = TestingUtils.startTestingCluster(6 * num_tasks) val jmGateway = cluster.getLeaderGateway(1 seconds) @@ -340,7 +343,8 @@ class JobManagerITCase(_system: ActorSystem) forwarder.connectNewDataSetAsInput(sender, DistributionPattern.ALL_TO_ALL) receiver.connectNewDataSetAsInput(forwarder, DistributionPattern.ALL_TO_ALL) - val jobGraph = new JobGraph("Forwarding Job", sender, forwarder, receiver) + val vertices = (List(sender, forwarder, receiver)).asJava + val jobGraph = new JobGraph("Forwarding Job", vertices) jobGraph.setScheduleMode(ScheduleMode.ALL) diff --git a/flink-runtime/src/test/scala/org/apache/flink/runtime/jobmanager/SlotSharingITCase.scala b/flink-runtime/src/test/scala/org/apache/flink/runtime/jobmanager/SlotSharingITCase.scala index a6d60dd2f45dad..3e3c6733e67e60 100644 --- a/flink-runtime/src/test/scala/org/apache/flink/runtime/jobmanager/SlotSharingITCase.scala +++ b/flink-runtime/src/test/scala/org/apache/flink/runtime/jobmanager/SlotSharingITCase.scala @@ -19,7 +19,6 @@ package org.apache.flink.runtime.jobmanager import akka.actor.ActorSystem -import akka.actor.Status.Success import akka.testkit.{ImplicitSender, TestKit} import org.apache.flink.runtime.akka.ListeningBehaviour import org.apache.flink.runtime.jobgraph.{JobVertex, DistributionPattern, JobGraph} @@ -31,6 +30,7 @@ import org.junit.runner.RunWith import org.scalatest.junit.JUnitRunner import org.scalatest.{BeforeAndAfterAll, Matchers, WordSpecLike} import scala.concurrent.duration._ +import scala.collection.JavaConverters._ @RunWith(classOf[JUnitRunner]) class SlotSharingITCase(_system: ActorSystem) @@ -109,7 +109,8 @@ class SlotSharingITCase(_system: ActorSystem) receiver.connectNewDataSetAsInput(sender1, DistributionPattern.POINTWISE) receiver.connectNewDataSetAsInput(sender2, DistributionPattern.ALL_TO_ALL) - val jobGraph = new JobGraph("Bipartite job", sender1, sender2, receiver) + val vertices = (List(sender1, sender2, receiver)).asJava + val jobGraph = new JobGraph("Bipartite job", vertices) val cluster = TestingUtils.startTestingCluster(num_tasks) val jmGateway = cluster.getLeaderGateway(1 seconds) diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/graph/StreamingJobGraphGenerator.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/graph/StreamingJobGraphGenerator.java index 04b1917437417f..0dbc146d467068 100644 --- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/graph/StreamingJobGraphGenerator.java +++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/graph/StreamingJobGraphGenerator.java @@ -102,7 +102,7 @@ private void init() { } public JobGraph createJobGraph(String jobName) { - jobGraph = new JobGraph(streamGraph.getJobName()); + jobGraph = new JobGraph(streamGraph.getJobName(), streamGraph.getExecutionConfig()); // make sure that all vertices start immediately jobGraph.setScheduleMode(ScheduleMode.ALL); From aa6ca2b7e2c9f3837a0e1e0add2c4cd0ebba3e65 Mon Sep 17 00:00:00 2001 From: Kostas Kloudas Date: Wed, 3 Feb 2016 13:58:12 +0100 Subject: [PATCH 2/3] FLINK-3327: Attaches the ExecutionConfig to the JobGraph and propagates it to the Task itself. --- .../flink/client/FlinkYarnSessionCli.java | 2 +- .../flink/api/common/ExecutionConfig.java | 3 - .../org/apache/flink/api/common/Plan.java | 19 --- .../plantranslate/JobGraphGenerator.java | 14 --- .../BackPressureStatsTrackerITCase.java | 3 +- .../deployment/TaskDeploymentDescriptor.java | 38 ++++-- .../flink/runtime/execution/Environment.java | 8 ++ .../executiongraph/ExecutionGraph.java | 17 ++- .../executiongraph/ExecutionVertex.java | 4 +- .../flink/runtime/jobgraph/JobGraph.java | 112 +++++++----------- .../jobgraph/tasks/AbstractInvokable.java | 27 +---- .../flink/runtime/operators/DataSinkTask.java | 20 +--- .../runtime/operators/DataSourceTask.java | 18 +-- .../taskmanager/RuntimeEnvironment.java | 11 +- .../flink/runtime/taskmanager/Task.java | 9 +- .../flink/runtime/jobmanager/JobManager.scala | 7 +- .../checkpoint/CoordinatorShutdownTest.java | 5 +- ...ecutionGraphCheckpointCoordinatorTest.java | 2 + .../client/JobClientActorRecoveryITCase.java | 3 +- .../runtime/client/JobClientActorTest.java | 3 +- .../TaskDeploymentDescriptorTest.java | 12 +- .../ExecutionGraphConstructionTest.java | 4 +- .../ExecutionGraphRestartTest.java | 15 +-- .../executiongraph/LocalInputSplitsTest.java | 5 +- .../VertexLocationConstraintTest.java | 13 +- .../PartialConsumePipelinedResultTest.java | 3 +- .../flink/runtime/jobgraph/JobGraphTest.java | 21 ++-- .../jobgraph/jsonplan/JsonGeneratorTest.java | 8 +- ...nagerSubmittedJobGraphsRecoveryITCase.java | 3 +- .../runtime/jobmanager/JobManagerTest.java | 3 +- .../runtime/jobmanager/JobSubmitTest.java | 5 +- .../SlotCountExceedingParallelismTest.java | 3 +- .../StandaloneSubmittedJobGraphStoreTest.java | 3 +- ...ooKeeperSubmittedJobGraphsStoreITCase.java | 3 +- .../ScheduleOrUpdateConsumersTest.java | 11 +- .../LeaderChangeStateCleanupTest.java | 3 +- .../operators/testutils/DummyEnvironment.java | 7 ++ .../operators/testutils/MockEnvironment.java | 11 +- .../taskmanager/TaskAsyncCallTest.java | 3 +- .../runtime/taskmanager/TaskCancelTest.java | 3 +- .../runtime/taskmanager/TaskManagerTest.java | 37 ++++-- .../flink/runtime/taskmanager/TaskTest.java | 3 +- .../TaskManagerLossFailsTasksTest.scala | 4 +- .../CoLocationConstraintITCase.scala | 3 +- .../runtime/jobmanager/JobManagerITCase.scala | 54 ++++----- .../runtime/jobmanager/RecoveryITCase.scala | 19 ++- .../jobmanager/SlotSharingITCase.scala | 9 +- ...askManagerFailsWithSlotSharingITCase.scala | 6 +- .../api/graph/StreamingJobGraphGenerator.java | 24 ---- .../graph/StreamingJobGraphGeneratorTest.java | 28 +++-- .../partitioner/RescalePartitionerTest.java | 3 +- .../runtime/tasks/StreamMockEnvironment.java | 18 ++- .../tasks/StreamTaskAsyncCheckpointTest.java | 6 + .../runtime/tasks/StreamTaskTest.java | 3 +- .../runtime/tasks/StreamTaskTestHarness.java | 9 +- .../JobSubmissionFailsITCase.java | 7 +- .../test/javaApiOperators/MapITCase.java | 33 ++++++ .../JobManagerCheckpointRecoveryITCase.java | 3 +- .../runtime/NetworkStackThroughputITCase.java | 3 +- .../ZooKeeperLeaderElectionITCase.java | 3 +- .../jobmanager/JobManagerFailsITCase.scala | 5 +- .../JobManagerLeaderSessionIDITSuite.scala | 3 +- .../taskmanager/TaskManagerFailsITCase.scala | 11 +- .../org/apache/flink/yarn/YarnTestBase.java | 2 +- 64 files changed, 384 insertions(+), 376 deletions(-) diff --git a/flink-clients/src/main/java/org/apache/flink/client/FlinkYarnSessionCli.java b/flink-clients/src/main/java/org/apache/flink/client/FlinkYarnSessionCli.java index 4f540a65abb8b2..84d4f9a1f22090 100644 --- a/flink-clients/src/main/java/org/apache/flink/client/FlinkYarnSessionCli.java +++ b/flink-clients/src/main/java/org/apache/flink/client/FlinkYarnSessionCli.java @@ -393,7 +393,7 @@ public int run(String[] args) { printUsage(); return 1; } - + // Query cluster for metrics if (cmd.hasOption(QUERY.getOpt())) { AbstractFlinkYarnClient flinkYarnClient = getFlinkYarnClient(); diff --git a/flink-core/src/main/java/org/apache/flink/api/common/ExecutionConfig.java b/flink-core/src/main/java/org/apache/flink/api/common/ExecutionConfig.java index 8d5211bba349f9..24742a3381485d 100644 --- a/flink-core/src/main/java/org/apache/flink/api/common/ExecutionConfig.java +++ b/flink-core/src/main/java/org/apache/flink/api/common/ExecutionConfig.java @@ -58,9 +58,6 @@ public class ExecutionConfig implements Serializable { private static final long serialVersionUID = 1L; - // Key for storing it in the Job Configuration - public static final String CONFIG_KEY = "runtime.config"; - /** * The constant to use for the parallelism, if the system should use the number * of currently available slots. diff --git a/flink-core/src/main/java/org/apache/flink/api/common/Plan.java b/flink-core/src/main/java/org/apache/flink/api/common/Plan.java index 899b9d6c8962f7..97a06acc2aa7c8 100644 --- a/flink-core/src/main/java/org/apache/flink/api/common/Plan.java +++ b/flink-core/src/main/java/org/apache/flink/api/common/Plan.java @@ -293,25 +293,6 @@ public void setDefaultParallelism(int defaultParallelism) { this.defaultParallelism = defaultParallelism; } - /** - * Gets the number of times the system will try to re-execute failed tasks. A value - * of {@code -1} indicates that the system default value (as defined in the configuration) - * should be used. - * - * @return The number of times the system will try to re-execute failed tasks. - */ - public int getNumberOfExecutionRetries() { - return getExecutionConfig().getNumberOfExecutionRetries(); - } - - /** - * Gets the delay between retry failed task. - * @return The delay the system will wait to retry. - */ - public long getExecutionRetryDelay() { - return getExecutionConfig().getExecutionRetryDelay(); - } - /** * Gets the optimizer post-pass class for this job. The post-pass typically creates utility classes * for data types and is specific to a particular data model (record, tuple, Scala, ...) diff --git a/flink-optimizer/src/main/java/org/apache/flink/optimizer/plantranslate/JobGraphGenerator.java b/flink-optimizer/src/main/java/org/apache/flink/optimizer/plantranslate/JobGraphGenerator.java index 165943dee60e15..c7aaa7d0d856d1 100644 --- a/flink-optimizer/src/main/java/org/apache/flink/optimizer/plantranslate/JobGraphGenerator.java +++ b/flink-optimizer/src/main/java/org/apache/flink/optimizer/plantranslate/JobGraphGenerator.java @@ -20,7 +20,6 @@ import com.fasterxml.jackson.core.JsonFactory; -import org.apache.flink.api.common.ExecutionConfig; import org.apache.flink.api.common.JobID; import org.apache.flink.api.common.aggregators.AggregatorRegistry; import org.apache.flink.api.common.aggregators.AggregatorWithName; @@ -81,11 +80,9 @@ import org.apache.flink.runtime.operators.shipping.ShipStrategyType; import org.apache.flink.runtime.operators.util.LocalStrategy; import org.apache.flink.runtime.operators.util.TaskConfig; -import org.apache.flink.util.InstantiationUtil; import org.apache.flink.util.StringUtils; import org.apache.flink.util.Visitor; -import java.io.IOException; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; @@ -218,8 +215,6 @@ public JobGraph compileJobGraph(OptimizedPlan program, JobID jobId) { // create the job graph object JobGraph graph = new JobGraph(jobId, program.getJobName(), program.getOriginalPlan().getExecutionConfig()); - graph.setNumberOfExecutionRetries(program.getOriginalPlan().getNumberOfExecutionRetries()); - graph.setExecutionRetryDelay(program.getOriginalPlan().getExecutionRetryDelay()); graph.setAllowQueuedScheduling(false); graph.setSessionTimeout(program.getOriginalPlan().getSessionTimeout()); @@ -238,15 +233,6 @@ public JobGraph compileJobGraph(OptimizedPlan program, JobID jobId) { DistributedCache.writeFileInfoToConfig(e.getKey(), e.getValue(), graph.getJobConfiguration()); } - try { - InstantiationUtil.writeObjectToConfig( - program.getOriginalPlan().getExecutionConfig(), - graph.getJobConfiguration(), - ExecutionConfig.CONFIG_KEY); - } catch (IOException e) { - throw new RuntimeException("Config object could not be written to Job Configuration: " + e); - } - // release all references again this.vertices = null; this.chainedTasks = null; diff --git a/flink-runtime-web/src/test/java/org/apache/flink/runtime/webmonitor/BackPressureStatsTrackerITCase.java b/flink-runtime-web/src/test/java/org/apache/flink/runtime/webmonitor/BackPressureStatsTrackerITCase.java index 52b0794d5ce23b..d6e4ee56eb72aa 100644 --- a/flink-runtime-web/src/test/java/org/apache/flink/runtime/webmonitor/BackPressureStatsTrackerITCase.java +++ b/flink-runtime-web/src/test/java/org/apache/flink/runtime/webmonitor/BackPressureStatsTrackerITCase.java @@ -20,6 +20,7 @@ import akka.actor.ActorSystem; import akka.testkit.JavaTestKit; +import org.apache.flink.api.common.ExecutionConfig; import org.apache.flink.configuration.ConfigConstants; import org.apache.flink.configuration.Configuration; import org.apache.flink.core.memory.MemoryType; @@ -90,7 +91,7 @@ public void testBackPressuredProducer() throws Exception { final FiniteDuration deadline = new FiniteDuration(60, TimeUnit.SECONDS); // The JobGraph - final JobGraph jobGraph = new JobGraph(); + final JobGraph jobGraph = new JobGraph(new ExecutionConfig()); final int parallelism = 4; final JobVertex task = new JobVertex("Task"); diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/deployment/TaskDeploymentDescriptor.java b/flink-runtime/src/main/java/org/apache/flink/runtime/deployment/TaskDeploymentDescriptor.java index 912a0cea13ea62..e154390e0ae2e1 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/deployment/TaskDeploymentDescriptor.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/deployment/TaskDeploymentDescriptor.java @@ -19,6 +19,7 @@ package org.apache.flink.runtime.deployment; import org.apache.flink.api.common.ApplicationID; +import org.apache.flink.api.common.ExecutionConfig; import org.apache.flink.api.common.JobID; import org.apache.flink.api.common.TaskInfo; import org.apache.flink.configuration.Configuration; @@ -92,6 +93,9 @@ public final class TaskDeploymentDescriptor implements Serializable { private final SerializedValue> operatorState; + /** The execution configuration (see {@link ExecutionConfig}) related to the specific job. */ + private final ExecutionConfig executionConfig; + private long recoveryTimestamp; /** @@ -99,9 +103,9 @@ public final class TaskDeploymentDescriptor implements Serializable { */ public TaskDeploymentDescriptor( ApplicationID appId, JobID jobID, JobVertexID vertexID, ExecutionAttemptID executionId, - String taskName, int indexInSubtaskGroup, int numberOfSubtasks, int attemptNumber, - Configuration jobConfiguration, Configuration taskConfiguration, String invokableClassName, - List producedPartitions, + ExecutionConfig executionConfig, String taskName, int indexInSubtaskGroup, int numberOfSubtasks, + int attemptNumber, Configuration jobConfiguration, Configuration taskConfiguration, + String invokableClassName, List producedPartitions, List inputGates, List requiredJarFiles, List requiredClasspaths, int targetSlotNumber, SerializedValue> operatorState, @@ -116,6 +120,7 @@ public TaskDeploymentDescriptor( this.jobID = checkNotNull(jobID); this.vertexID = checkNotNull(vertexID); this.executionId = checkNotNull(executionId); + this.executionConfig = checkNotNull(executionConfig); this.taskName = checkNotNull(taskName); this.indexInSubtaskGroup = indexInSubtaskGroup; this.numberOfSubtasks = numberOfSubtasks; @@ -133,17 +138,24 @@ public TaskDeploymentDescriptor( } public TaskDeploymentDescriptor( - ApplicationID appId, JobID jobID, JobVertexID vertexID, ExecutionAttemptID executionId, - String taskName, int indexInSubtaskGroup, int numberOfSubtasks, int attemptNumber, - Configuration jobConfiguration, Configuration taskConfiguration, String invokableClassName, - List producedPartitions, - List inputGates, - List requiredJarFiles, List requiredClasspaths, - int targetSlotNumber) { + ApplicationID appId, JobID jobID, JobVertexID vertexID, ExecutionAttemptID executionId, + ExecutionConfig executionConfig, String taskName, int indexInSubtaskGroup, int numberOfSubtasks, + int attemptNumber, Configuration jobConfiguration, Configuration taskConfiguration, + String invokableClassName, List producedPartitions, + List inputGates, + List requiredJarFiles, List requiredClasspaths, + int targetSlotNumber) { + + this(appId, jobID, vertexID, executionId, executionConfig, taskName, indexInSubtaskGroup, + numberOfSubtasks, attemptNumber, jobConfiguration, taskConfiguration, invokableClassName, + producedPartitions, inputGates, requiredJarFiles, requiredClasspaths, targetSlotNumber, null, -1); + } - this(appId, jobID, vertexID, executionId, taskName, indexInSubtaskGroup, numberOfSubtasks, attemptNumber, - jobConfiguration, taskConfiguration, invokableClassName, producedPartitions, - inputGates, requiredJarFiles, requiredClasspaths, targetSlotNumber, null, -1); + /** + * Returns the execution configuration (see {@link ExecutionConfig}) related to the specific job. + */ + public ExecutionConfig getExecutionConfig() { + return executionConfig; } /** diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/execution/Environment.java b/flink-runtime/src/main/java/org/apache/flink/runtime/execution/Environment.java index fef7d0e96a4e47..b7c8e02a7b0810 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/execution/Environment.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/execution/Environment.java @@ -18,6 +18,7 @@ package org.apache.flink.runtime.execution; +import org.apache.flink.api.common.ExecutionConfig; import org.apache.flink.api.common.TaskInfo; import org.apache.flink.api.common.ApplicationID; import org.apache.flink.configuration.Configuration; @@ -46,6 +47,13 @@ */ public interface Environment { + /** + * Returns the job specific {@link ExecutionConfig}. + * + * @return The execution configuration associated with the current job. + * */ + ExecutionConfig getExecutionConfig(); + /** * Returns the ID of the application the task belongs to. * diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ExecutionGraph.java b/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ExecutionGraph.java index c3bfbb045e1afb..4aabdbf028e240 100755 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ExecutionGraph.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ExecutionGraph.java @@ -20,6 +20,7 @@ import akka.actor.ActorSystem; +import com.google.common.base.Preconditions; import org.apache.flink.api.common.ApplicationID; import org.apache.flink.api.common.ExecutionConfig; import org.apache.flink.api.common.accumulators.Accumulator; @@ -59,7 +60,6 @@ import org.apache.flink.runtime.util.SerializedThrowable; import org.apache.flink.util.SerializedValue; import org.apache.flink.util.ExceptionUtils; -import org.apache.flink.util.InstantiationUtil; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -183,6 +183,9 @@ public class ExecutionGraph implements Serializable { // ------ Configuration of the Execution ------- + /** The execution configuration (see {@link ExecutionConfig}) related to this specific job. */ + private ExecutionConfig executionConfig; + /** The number of times failed executions should be retried. */ private int numberOfRetriesLeft; @@ -239,7 +242,6 @@ public class ExecutionGraph implements Serializable { private ExecutionContext executionContext; // ------ Fields that are only relevant for archived execution graphs ------------ - private ExecutionConfig executionConfig; private String jsonPlan; @@ -261,6 +263,7 @@ public class ExecutionGraph implements Serializable { jobId, jobName, jobConfig, + new ExecutionConfig(), timeout, new ArrayList(), new ArrayList(), @@ -273,6 +276,7 @@ public ExecutionGraph( JobID jobId, String jobName, Configuration jobConfig, + ExecutionConfig config, FiniteDuration timeout, List requiredJarFiles, List requiredClasspaths, @@ -302,7 +306,7 @@ public ExecutionGraph( this.requiredJarFiles = requiredJarFiles; this.requiredClasspaths = requiredClasspaths; - + this.executionConfig = Preconditions.checkNotNull(config); this.timeout = timeout; } @@ -939,12 +943,7 @@ public void prepareForArchiving() { if (!state.isTerminalState()) { throw new IllegalStateException("Can only archive the job from a terminal state"); } - // "unpack" execution config before we throw away the usercode classloader. - try { - executionConfig = (ExecutionConfig) InstantiationUtil.readObjectFromConfig(jobConfiguration, ExecutionConfig.CONFIG_KEY,userClassLoader); - } catch (Exception e) { - LOG.warn("Error deserializing the execution config while archiving the execution graph", e); - } + // clear the non-serializable fields userClassLoader = null; scheduler = null; diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ExecutionVertex.java b/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ExecutionVertex.java index 165dce4641fd4e..5fa5f6f8752ad5 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ExecutionVertex.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ExecutionVertex.java @@ -19,6 +19,7 @@ package org.apache.flink.runtime.executiongraph; import org.apache.flink.api.common.ApplicationID; +import org.apache.flink.api.common.ExecutionConfig; import org.apache.flink.runtime.JobException; import org.apache.flink.runtime.blob.BlobKey; import org.apache.flink.runtime.deployment.InputChannelDeploymentDescriptor; @@ -667,10 +668,11 @@ TaskDeploymentDescriptor createDeploymentDescriptor( consumedPartitions.add(new InputGateDeploymentDescriptor(resultId, queueToRequest, partitions)); } + ExecutionConfig config = getExecutionGraph().getExecutionConfig(); List jarFiles = getExecutionGraph().getRequiredJarFiles(); List classpaths = getExecutionGraph().getRequiredClasspaths(); - return new TaskDeploymentDescriptor(getApplicationId(), getJobId(), getJobvertexId(), executionId, getTaskName(), + return new TaskDeploymentDescriptor(getApplicationId(), getJobId(), getJobvertexId(), executionId, config, getTaskName(), subTaskIndex, getTotalNumberOfParallelSubtasks(), attemptNumber, getExecutionGraph().getJobConfiguration(), jobVertex.getJobVertex().getConfiguration(), jobVertex.getJobVertex().getInvokableClassName(), producedPartitions, consumedPartitions, jarFiles, classpaths, targetSlot.getRoot().getSlotNumber(), diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/jobgraph/JobGraph.java b/flink-runtime/src/main/java/org/apache/flink/runtime/jobgraph/JobGraph.java index fc2ed2f1238320..6fb7f1b00d5e14 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/jobgraph/JobGraph.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/jobgraph/JobGraph.java @@ -18,6 +18,7 @@ package org.apache.flink.runtime.jobgraph; +import com.google.common.base.Preconditions; import org.apache.flink.api.common.ExecutionConfig; import org.apache.flink.api.common.InvalidProgramException; import org.apache.flink.api.common.JobID; @@ -38,7 +39,6 @@ import java.util.List; import java.util.Map; import java.util.Collections; -import java.util.Arrays; import java.util.Set; import java.util.LinkedHashSet; import java.util.Iterator; @@ -82,11 +82,6 @@ public class JobGraph implements Serializable { private final ExecutionConfig executionConfig; - /** The number of times that failed tasks should be re-executed */ - private int numExecutionRetries; - - private long executionRetryDelay; - /** The number of seconds after which the corresponding ExecutionGraph is removed at the * job manager after it has been executed. */ private long sessionTimeout = 0; @@ -105,87 +100,76 @@ public class JobGraph implements Serializable { // -------------------------------------------------------------------------------------------- - /** - * Constructs a new job graph with no name and a random job ID. - */ - public JobGraph() { - this((String) null); - } /** - * Constructs a new job graph with the given name, a random job ID. + * Constructs a new job graph with no name, a random job ID, and the given + * {@link ExecutionConfig}. * - * @param jobName The name of the job + * @param config The {@link ExecutionConfig} for the job. */ - public JobGraph(String jobName) { - this(null, jobName, (ExecutionConfig) null); - } - - public JobGraph(JobID jobID, String jobName) { - this(jobID, jobName, (ExecutionConfig) null); + public JobGraph(ExecutionConfig config) { + this((String) null, config); } /** - * Constructs a new job graph with the given name, a random job ID. + * Constructs a new job graph with the given name, the given {@link ExecutionConfig}, + * and a random job ID. * - * @param jobName The name of the job + * @param jobName The name of the job. + * @param config The execution configuration of the job. */ public JobGraph(String jobName, ExecutionConfig config) { this(null, jobName, config); } /** - * Constructs a new job graph with the given name and a random job ID if null supplied as an id. + * Constructs a new job graph with the given job ID (or a random ID, if {@code null} is passed), + * the given name and the given execution configuration (see {@link ExecutionConfig}). * * @param jobId The id of the job. A random ID is generated, if {@code null} is passed. * @param jobName The name of the job. + * @param config The execution configuration of the job. */ public JobGraph(JobID jobId, String jobName, ExecutionConfig config) { this.jobID = jobId == null ? new JobID() : jobId; this.jobName = jobName == null ? "(unnamed job)" : jobName; - this.executionConfig = config; - } - - public JobGraph(String jobName, JobVertex vertex) { - this(jobName, Collections.singletonList(vertex)); - } - - public JobGraph(String jobName, JobVertex vertex1, JobVertex vertex2) { - this(jobName, Arrays.asList(vertex1, vertex2)); - } - - public JobGraph(JobVertex vertex) { - this(null, Collections.singletonList(vertex)); + this.executionConfig = Preconditions.checkNotNull(config); } /** - * Constructs a new job graph with no name and a random job ID if null supplied as an id. + * Constructs a new job graph with no name, a random job ID, the given {@link ExecutionConfig}, and + * the given job vertices. * + * @param config The execution configuration of the job. * @param vertices The vertices to add to the graph. */ - public JobGraph(List vertices) { - this(null, vertices); + public JobGraph(ExecutionConfig config, JobVertex... vertices) { + this(null, config, vertices); } /** - * Constructs a new job graph with the given name and a random job ID. + * Constructs a new job graph with the given name, the given {@link ExecutionConfig}, a random job ID, + * and the given job vertices. * * @param jobName The name of the job. + * @param config The execution configuration of the job. * @param vertices The vertices to add to the graph. */ - public JobGraph(String jobName, List vertices) { - this(null, jobName, vertices); + public JobGraph(String jobName, ExecutionConfig config, JobVertex... vertices) { + this(null, jobName, config, vertices); } /** - * Constructs a new job graph with the given name and a random job ID if null supplied as an id. + * Constructs a new job graph with the given name, the given {@link ExecutionConfig}, + * the given jobId or a random one if null supplied, and the given job vertices. * * @param jobId The id of the job. A random ID is generated, if {@code null} is passed. * @param jobName The name of the job. + * @param config The execution configuration of the job. * @param vertices The vertices to add to the graph. */ - public JobGraph(JobID jobId, String jobName, List vertices) { - this(jobId, jobName, (ExecutionConfig) null); + public JobGraph(JobID jobId, String jobName, ExecutionConfig config, JobVertex... vertices) { + this(jobId, jobName, config); for (JobVertex vertex : vertices) { addVertex(vertex); @@ -222,19 +206,8 @@ public Configuration getJobConfiguration() { return this.jobConfiguration; } - /** - * Sets the number of times that failed tasks are re-executed. A value of zero - * effectively disables fault tolerance. A value of {@code -1} indicates that the system - * default value (as defined in the configuration) should be used. - * - * @param numberOfExecutionRetries The number of times the system will try to re-execute failed tasks. - */ - public void setNumberOfExecutionRetries(int numberOfExecutionRetries) { - if (numberOfExecutionRetries < -1) { - throw new IllegalArgumentException( - "The number of execution retries must be non-negative, or -1 (use system default)"); - } - this.numExecutionRetries = numberOfExecutionRetries; + public ExecutionConfig getExecutionConfig() { + return this.executionConfig; } /** @@ -245,7 +218,12 @@ public void setNumberOfExecutionRetries(int numberOfExecutionRetries) { * @return The number of times the system will try to re-execute failed tasks. */ public int getNumberOfExecutionRetries() { - return numExecutionRetries; + int retries = executionConfig.getNumberOfExecutionRetries(); + if (retries < -1) { + throw new IllegalArgumentException( + "The number of execution retries must be non-negative, or -1 (use system default)"); + } + return retries; } /** @@ -255,22 +233,12 @@ public int getNumberOfExecutionRetries() { * @return The delay of time in milliseconds the system will try to re-execute failed tasks. */ public long getExecutionRetryDelay() { - return executionRetryDelay; - } - - /** - * Sets the delay that failed tasks are re-executed. A value of zero - * effectively disables fault tolerance. A value of {@code -1} indicates that the system - * default value (as defined in the configuration) should be used. - * - * @param executionRetryDelay The delay of time the system will wait to re-execute failed tasks. - */ - public void setExecutionRetryDelay(long executionRetryDelay){ - if (executionRetryDelay < -1) { + long retryDelay = executionConfig.getExecutionRetryDelay(); + if (retryDelay < -1) { throw new IllegalArgumentException( "The delay between reties must be non-negative, or -1 (use system default)"); } - this.executionRetryDelay = executionRetryDelay; + return retryDelay; } /** diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/jobgraph/tasks/AbstractInvokable.java b/flink-runtime/src/main/java/org/apache/flink/runtime/jobgraph/tasks/AbstractInvokable.java index 30f32a11614fa0..d7dfaf5874b53f 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/jobgraph/tasks/AbstractInvokable.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/jobgraph/tasks/AbstractInvokable.java @@ -22,7 +22,6 @@ import org.apache.flink.configuration.Configuration; import org.apache.flink.runtime.execution.Environment; import org.apache.flink.runtime.operators.BatchTask; -import org.apache.flink.util.InstantiationUtil; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -42,9 +41,6 @@ public abstract class AbstractInvokable { /** The environment assigned to this invokable. */ private Environment environment; - /** The execution config, cached from the deserialization from the JobConfiguration */ - private ExecutionConfig executionConfig; - /** * Starts the execution. * @@ -125,29 +121,10 @@ public Configuration getJobConfiguration() { } /** - * Returns the global ExecutionConfig, obtained from the job configuration. + * Returns the global ExecutionConfig. */ public ExecutionConfig getExecutionConfig() { - if (executionConfig != null) { - return executionConfig; - } - - try { - executionConfig = (ExecutionConfig) InstantiationUtil.readObjectFromConfig( - getJobConfiguration(), - ExecutionConfig.CONFIG_KEY, - getUserCodeClassLoader()); - - if (executionConfig == null) { - LOG.warn("Environment did not contain an ExecutionConfig - using a default config."); - executionConfig = new ExecutionConfig(); - } - return executionConfig; - } - catch (Exception e) { - LOG.warn("Could not load ExecutionConfig from Environment, returning default ExecutionConfig", e); - return new ExecutionConfig(); - } + return this.environment.getExecutionConfig(); } /** diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/operators/DataSinkTask.java b/flink-runtime/src/main/java/org/apache/flink/runtime/operators/DataSinkTask.java index 0645668ac3b828..07393b1bf8d45f 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/operators/DataSinkTask.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/operators/DataSinkTask.java @@ -41,13 +41,10 @@ import org.apache.flink.runtime.operators.util.ReaderIterator; import org.apache.flink.runtime.operators.util.TaskConfig; import org.apache.flink.runtime.plugable.DeserializationDelegate; -import org.apache.flink.util.InstantiationUtil; import org.apache.flink.util.MutableObjectIterator; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.IOException; - /** * DataSinkTask which is executed by a task manager. The task hands the data to an output format. * @@ -112,21 +109,8 @@ public void invoke() throws Exception { LOG.debug(getLogString("Rich Sink detected. Initializing runtime context.")); } - ExecutionConfig executionConfig; - try { - ExecutionConfig c = (ExecutionConfig) InstantiationUtil.readObjectFromConfig( - getJobConfiguration(), - ExecutionConfig.CONFIG_KEY, - getUserCodeClassLoader()); - if (c != null) { - executionConfig = c; - } else { - LOG.warn("The execution config returned by the configuration was null"); - executionConfig = new ExecutionConfig(); - } - } catch (IOException | ClassNotFoundException e) { - throw new RuntimeException("Could not load ExecutionConfig from Job Configuration: " + e); - } + ExecutionConfig executionConfig = getExecutionConfig(); + boolean objectReuseEnabled = executionConfig.isObjectReuseEnabled(); try { diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/operators/DataSourceTask.java b/flink-runtime/src/main/java/org/apache/flink/runtime/operators/DataSourceTask.java index 93a4f9c03b1f4d..960faf74f05e0f 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/operators/DataSourceTask.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/operators/DataSourceTask.java @@ -36,11 +36,9 @@ import org.apache.flink.runtime.operators.util.DistributedRuntimeUDFContext; import org.apache.flink.runtime.operators.util.TaskConfig; import org.apache.flink.util.Collector; -import org.apache.flink.util.InstantiationUtil; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.IOException; import java.util.ArrayList; import java.util.Iterator; import java.util.List; @@ -104,21 +102,7 @@ public void invoke() throws Exception { LOG.debug(getLogString("Rich Source detected. Initializing runtime context.")); } - ExecutionConfig executionConfig; - try { - ExecutionConfig c = (ExecutionConfig) InstantiationUtil.readObjectFromConfig( - getJobConfiguration(), - ExecutionConfig.CONFIG_KEY, - getUserCodeClassLoader()); - if (c != null) { - executionConfig = c; - } else { - LOG.warn("ExecutionConfig from job configuration is null. Creating empty config"); - executionConfig = new ExecutionConfig(); - } - } catch (IOException | ClassNotFoundException e) { - throw new RuntimeException("Could not load ExecutionConfig from Job Configuration: ", e); - } + ExecutionConfig executionConfig = getExecutionConfig(); boolean objectReuseEnabled = executionConfig.isObjectReuseEnabled(); diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/taskmanager/RuntimeEnvironment.java b/flink-runtime/src/main/java/org/apache/flink/runtime/taskmanager/RuntimeEnvironment.java index 0b33942e659d8e..c3acd2488ef902 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/taskmanager/RuntimeEnvironment.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/taskmanager/RuntimeEnvironment.java @@ -19,6 +19,7 @@ package org.apache.flink.runtime.taskmanager; import org.apache.flink.api.common.ApplicationID; +import org.apache.flink.api.common.ExecutionConfig; import org.apache.flink.api.common.JobID; import org.apache.flink.api.common.TaskInfo; import org.apache.flink.configuration.Configuration; @@ -57,7 +58,8 @@ public class RuntimeEnvironment implements Environment { private final Configuration jobConfiguration; private final Configuration taskConfiguration; - + private final ExecutionConfig executionConfig; + private final ClassLoader userCodeClassLoader; private final MemoryManager memManager; @@ -83,6 +85,7 @@ public RuntimeEnvironment( JobID jobId, JobVertexID jobVertexId, ExecutionAttemptID executionId, + ExecutionConfig executionConfig, TaskInfo taskInfo, Configuration jobConfiguration, Configuration taskConfiguration, @@ -103,6 +106,7 @@ public RuntimeEnvironment( this.jobVertexId = checkNotNull(jobVertexId); this.executionId = checkNotNull(executionId); this.taskInfo = checkNotNull(taskInfo); + this.executionConfig = checkNotNull(executionConfig); this.jobConfiguration = checkNotNull(jobConfiguration); this.taskConfiguration = checkNotNull(taskConfiguration); this.userCodeClassLoader = checkNotNull(userCodeClassLoader); @@ -120,6 +124,11 @@ public RuntimeEnvironment( // ------------------------------------------------------------------------ + @Override + public ExecutionConfig getExecutionConfig() { + return this.executionConfig; + } + @Override public ApplicationID getApplicationID() { return appId; diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/taskmanager/Task.java b/flink-runtime/src/main/java/org/apache/flink/runtime/taskmanager/Task.java index 78327207ea20f5..b2e99b2ae26e10 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/taskmanager/Task.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/taskmanager/Task.java @@ -20,6 +20,7 @@ import org.apache.flink.api.common.ApplicationID; +import org.apache.flink.api.common.ExecutionConfig; import org.apache.flink.api.common.JobID; import org.apache.flink.api.common.TaskInfo; import org.apache.flink.api.common.cache.DistributedCache; @@ -219,6 +220,9 @@ public class Task implements Runnable { private volatile long recoveryTs; + /** The job specific execution configuration (see {@link ExecutionConfig}). */ + private final ExecutionConfig executionConfig; + /** *

IMPORTANT: This constructor may not start any work that would need to * be undone in the case of a failing task deployment.

@@ -248,6 +252,7 @@ public Task(TaskDeploymentDescriptor tdd, this.nameOfInvokableClass = checkNotNull(tdd.getInvokableClassName()); this.operatorState = tdd.getOperatorState(); this.recoveryTs = tdd.getRecoveryTimestamp(); + this.executionConfig = checkNotNull(tdd.getExecutionConfig()); this.memoryManager = checkNotNull(memManager); this.ioManager = checkNotNull(ioManager); @@ -496,8 +501,8 @@ else if (current == ExecutionState.CANCELING) { TaskInputSplitProvider splitProvider = new TaskInputSplitProvider(jobManager, jobId, vertexId, executionId, userCodeClassLoader, actorAskTimeout); - Environment env = new RuntimeEnvironment(appId, jobId, vertexId, executionId, taskInfo, - jobConfiguration, taskConfiguration, + Environment env = new RuntimeEnvironment(appId, jobId, vertexId, executionId, + executionConfig, taskInfo, jobConfiguration, taskConfiguration, userCodeClassLoader, memoryManager, ioManager, broadcastVariableManager, accumulatorRegistry, splitProvider, distributedCacheEntries, diff --git a/flink-runtime/src/main/scala/org/apache/flink/runtime/jobmanager/JobManager.scala b/flink-runtime/src/main/scala/org/apache/flink/runtime/jobmanager/JobManager.scala index 78612c0910f170..04caf6295254f4 100644 --- a/flink-runtime/src/main/scala/org/apache/flink/runtime/jobmanager/JobManager.scala +++ b/flink-runtime/src/main/scala/org/apache/flink/runtime/jobmanager/JobManager.scala @@ -914,6 +914,7 @@ class JobManager( jobGraph.getJobID, jobGraph.getName, jobGraph.getJobConfiguration, + jobGraph.getExecutionConfig, timeout, jobGraph.getUserJarBlobKeys, jobGraph.getClasspaths, @@ -1051,7 +1052,8 @@ class JobManager( } catch { case t: Throwable => - log.error(s"Failed to submit job $jobId ($jobName)", t) + val message = t.getMessage + log.error(s"Failed to submit job $jobId ($jobName): $message", t) libraryCacheManager.unregisterJob(jobId) currentJobs.remove(jobId) @@ -1063,7 +1065,8 @@ class JobManager( val rt: Throwable = if (t.isInstanceOf[JobExecutionException]) { t } else { - new JobExecutionException(jobId, s"Failed to submit job $jobId ($jobName)", t) + val message = t.getMessage + new JobExecutionException(jobId, s"Failed to submit job $jobId ($jobName): $message", t) } jobInfo.client ! decorateMessage(JobResultFailure(new SerializedThrowable(rt))) diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/CoordinatorShutdownTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/CoordinatorShutdownTest.java index 1c666e5e5fb13c..03ff83d536b9a5 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/CoordinatorShutdownTest.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/CoordinatorShutdownTest.java @@ -18,6 +18,7 @@ package org.apache.flink.runtime.checkpoint; +import org.apache.flink.api.common.ExecutionConfig; import org.apache.flink.configuration.ConfigConstants; import org.apache.flink.configuration.Configuration; import org.apache.flink.runtime.akka.ListeningBehaviour; @@ -60,7 +61,7 @@ public void testCoordinatorShutsDownOnFailure() { vertex.setInvokableClass(Tasks.NoOpInvokable.class); List vertexIdList = Collections.singletonList(vertex.getID()); - JobGraph testGraph = new JobGraph("test job", vertex); + JobGraph testGraph = new JobGraph("test job", new ExecutionConfig(), vertex); testGraph.setSnapshotSettings(new JobSnapshottingSettings(vertexIdList, vertexIdList, vertexIdList, 5000, 60000, 0L, Integer.MAX_VALUE)); @@ -112,7 +113,7 @@ public void testCoordinatorShutsDownOnSuccess() { vertex.setInvokableClass(Tasks.NoOpInvokable.class); List vertexIdList = Collections.singletonList(vertex.getID()); - JobGraph testGraph = new JobGraph("test job", vertex); + JobGraph testGraph = new JobGraph("test job", new ExecutionConfig(), vertex); testGraph.setSnapshotSettings(new JobSnapshottingSettings(vertexIdList, vertexIdList, vertexIdList, 5000, 60000, 0L, Integer.MAX_VALUE)); diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/ExecutionGraphCheckpointCoordinatorTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/ExecutionGraphCheckpointCoordinatorTest.java index e921e92f534854..275d0d5f7286ff 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/ExecutionGraphCheckpointCoordinatorTest.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/ExecutionGraphCheckpointCoordinatorTest.java @@ -19,6 +19,7 @@ package org.apache.flink.runtime.checkpoint; import akka.actor.ActorSystem; +import org.apache.flink.api.common.ExecutionConfig; import org.apache.flink.api.common.JobID; import org.apache.flink.configuration.Configuration; import org.apache.flink.runtime.akka.AkkaUtils; @@ -47,6 +48,7 @@ public void testCheckpointAndSavepointCoordinatorShareCheckpointIDCounter() thro new JobID(), "test", new Configuration(), + new ExecutionConfig(), new FiniteDuration(1, TimeUnit.DAYS), Collections.emptyList(), Collections.emptyList(), diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/client/JobClientActorRecoveryITCase.java b/flink-runtime/src/test/java/org/apache/flink/runtime/client/JobClientActorRecoveryITCase.java index aeb521caaf2e91..865760e686c5de 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/client/JobClientActorRecoveryITCase.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/client/JobClientActorRecoveryITCase.java @@ -20,6 +20,7 @@ import akka.actor.PoisonPill; import org.apache.curator.test.TestingServer; +import org.apache.flink.api.common.ExecutionConfig; import org.apache.flink.api.common.JobExecutionResult; import org.apache.flink.configuration.ConfigConstants; import org.apache.flink.configuration.Configuration; @@ -94,7 +95,7 @@ public void testJobClientRecovery() throws Exception { JobVertex blockingVertex = new JobVertex("Blocking Vertex"); blockingVertex.setInvokableClass(BlockingTask.class); blockingVertex.setParallelism(1); - final JobGraph jobGraph = new JobGraph("Blocking Test Job", blockingVertex); + final JobGraph jobGraph = new JobGraph("Blocking Test Job", new ExecutionConfig(), blockingVertex); final Promise promise = new scala.concurrent.impl.Promise.DefaultPromise<>(); Deadline deadline = new FiniteDuration(2, TimeUnit.MINUTES).fromNow(); diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/client/JobClientActorTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/client/JobClientActorTest.java index 00ad632b343e77..ee1fd603533026 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/client/JobClientActorTest.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/client/JobClientActorTest.java @@ -25,6 +25,7 @@ import akka.pattern.Patterns; import akka.testkit.JavaTestKit; import akka.util.Timeout; +import org.apache.flink.api.common.ExecutionConfig; import org.apache.flink.configuration.Configuration; import org.apache.flink.runtime.akka.AkkaUtils; import org.apache.flink.runtime.akka.FlinkUntypedActor; @@ -47,7 +48,7 @@ public class JobClientActorTest extends TestLogger { private static ActorSystem system; - private static JobGraph testJobGraph = new JobGraph("Test Job"); + private static JobGraph testJobGraph = new JobGraph("Test Job", new ExecutionConfig()); @BeforeClass public static void setup() { diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/deployment/TaskDeploymentDescriptorTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/deployment/TaskDeploymentDescriptorTest.java index 5eeb150fdaa595..09c32682953d35 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/deployment/TaskDeploymentDescriptorTest.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/deployment/TaskDeploymentDescriptorTest.java @@ -27,6 +27,7 @@ import java.util.List; import org.apache.flink.api.common.ApplicationID; +import org.apache.flink.api.common.ExecutionConfig; import org.apache.flink.configuration.Configuration; import org.apache.flink.runtime.blob.BlobKey; import org.apache.flink.runtime.executiongraph.ExecutionAttemptID; @@ -56,10 +57,12 @@ public void testSerialization() { final List inputGates = new ArrayList(0); final List requiredJars = new ArrayList(0); final List requiredClasspaths = new ArrayList(0); - - final TaskDeploymentDescriptor orig = new TaskDeploymentDescriptor(appId, jobID, vertexID, execId, taskName, - indexInSubtaskGroup, currentNumberOfSubtasks, attemptNumber, jobConfiguration, taskConfiguration, - invokableClass.getName(), producedResults, inputGates, requiredJars, requiredClasspaths, 47); + final ExecutionConfig executionConfig = new ExecutionConfig(); + + final TaskDeploymentDescriptor orig = new TaskDeploymentDescriptor(appId, jobID, vertexID, execId, + executionConfig, taskName, indexInSubtaskGroup, currentNumberOfSubtasks, attemptNumber, + jobConfiguration, taskConfiguration, invokableClass.getName(), producedResults, inputGates, + requiredJars, requiredClasspaths, 47); final TaskDeploymentDescriptor copy = CommonTestUtils.createCopySerializable(orig); @@ -78,6 +81,7 @@ public void testSerialization() { assertEquals(orig.getAttemptNumber(), copy.getAttemptNumber()); assertEquals(orig.getProducedPartitions(), copy.getProducedPartitions()); assertEquals(orig.getInputGates(), copy.getInputGates()); + assertEquals(orig.getExecutionConfig(), copy.getExecutionConfig()); assertEquals(orig.getRequiredJarFiles(), copy.getRequiredJarFiles()); assertEquals(orig.getRequiredClasspaths(), copy.getRequiredClasspaths()); diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/ExecutionGraphConstructionTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/ExecutionGraphConstructionTest.java index 34f67aadb8ca66..9b8c98f0570129 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/ExecutionGraphConstructionTest.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/ExecutionGraphConstructionTest.java @@ -26,6 +26,7 @@ import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; +import org.apache.flink.api.common.ExecutionConfig; import org.apache.flink.runtime.akka.AkkaUtils; import org.apache.flink.runtime.io.network.partition.ResultPartitionType; import org.apache.flink.runtime.testingUtils.TestingUtils; @@ -689,8 +690,7 @@ public void testCoLocationConstraintCreation() { JobVertex v8 = new JobVertex("vertex8"); v8.setParallelism(2); - List vertices = Arrays.asList(v1, v2, v3, v4, v5, v6, v7, v8); - JobGraph jg = new JobGraph(jobId, jobName, vertices); + JobGraph jg = new JobGraph(jobId, jobName, new ExecutionConfig(), v1, v2, v3, v4, v5, v6, v7, v8); ExecutionGraph eg = new ExecutionGraph( TestingUtils.defaultExecutionContext(), diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/ExecutionGraphRestartTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/ExecutionGraphRestartTest.java index 47a48a299dca9d..98c0cceb604973 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/ExecutionGraphRestartTest.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/ExecutionGraphRestartTest.java @@ -18,6 +18,7 @@ package org.apache.flink.runtime.executiongraph; +import org.apache.flink.api.common.ExecutionConfig; import org.apache.flink.api.common.JobID; import org.apache.flink.configuration.Configuration; import org.apache.flink.runtime.akka.AkkaUtils; @@ -67,7 +68,7 @@ public void testNoManualRestart() throws Exception { sender.setInvokableClass(Tasks.NoOpInvokable.class); sender.setParallelism(NUM_TASKS); - JobGraph jobGraph = new JobGraph("Pointwise job", sender); + JobGraph jobGraph = new JobGraph("Pointwise job", new ExecutionConfig(), sender); ExecutionGraph eg = new ExecutionGraph( TestingUtils.defaultExecutionContext(), @@ -122,7 +123,7 @@ public void testConstraintsAfterRestart() throws Exception { groupVertex.setStrictlyCoLocatedWith(groupVertex2); //initiate and schedule job - JobGraph jobGraph = new JobGraph("Pointwise job", groupVertex, groupVertex2); + JobGraph jobGraph = new JobGraph("Pointwise job", new ExecutionConfig(), groupVertex, groupVertex2); ExecutionGraph eg = new ExecutionGraph( TestingUtils.defaultExecutionContext(), new JobID(), @@ -176,7 +177,7 @@ public void testRestartAutomatically() throws Exception { sender.setInvokableClass(Tasks.NoOpInvokable.class); sender.setParallelism(NUM_TASKS); - JobGraph jobGraph = new JobGraph("Pointwise job", sender); + JobGraph jobGraph = new JobGraph("Pointwise job", new ExecutionConfig(), sender); ExecutionGraph eg = new ExecutionGraph( TestingUtils.defaultExecutionContext(), @@ -217,7 +218,7 @@ public void testCancelWhileRestarting() throws Exception { jobVertex.setInvokableClass(Tasks.NoOpInvokable.class); jobVertex.setParallelism(NUM_TASKS); - JobGraph jobGraph = new JobGraph("TestJob", jobVertex); + JobGraph jobGraph = new JobGraph("TestJob", new ExecutionConfig(), jobVertex); // We want to manually control the restart and delay executionGraph.setNumberOfRetriesLeft(Integer.MAX_VALUE); @@ -283,7 +284,7 @@ public void testCancelWhileFailing() throws Exception { jobVertex.setInvokableClass(Tasks.NoOpInvokable.class); jobVertex.setParallelism(NUM_TASKS); - JobGraph jobGraph = new JobGraph("TestJob", jobVertex); + JobGraph jobGraph = new JobGraph("TestJob", new ExecutionConfig(), jobVertex); // We want to manually control the restart and delay executionGraph.setNumberOfRetriesLeft(Integer.MAX_VALUE); @@ -345,7 +346,7 @@ public void testNoRestartOnUnrecoverableException() throws Exception { sender.setInvokableClass(Tasks.NoOpInvokable.class); sender.setParallelism(NUM_TASKS); - JobGraph jobGraph = new JobGraph("Pointwise job", sender); + JobGraph jobGraph = new JobGraph("Pointwise job", new ExecutionConfig(), sender); ExecutionGraph eg = spy(new ExecutionGraph( TestingUtils.defaultExecutionContext(), @@ -410,7 +411,7 @@ public void testFailingExecutionAfterRestart() throws Exception { receiver.setInvokableClass(Tasks.NoOpInvokable.class); receiver.setParallelism(1); - JobGraph jobGraph = new JobGraph("Pointwise job", sender, receiver); + JobGraph jobGraph = new JobGraph("Pointwise job", new ExecutionConfig(), sender, receiver); ExecutionGraph eg = new ExecutionGraph( TestingUtils.defaultExecutionContext(), diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/LocalInputSplitsTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/LocalInputSplitsTest.java index 2530a53d73ef78..11666576874c9d 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/LocalInputSplitsTest.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/LocalInputSplitsTest.java @@ -25,6 +25,7 @@ import java.net.InetAddress; import java.util.concurrent.TimeUnit; +import org.apache.flink.api.common.ExecutionConfig; import org.apache.flink.api.common.io.StrictlyLocalAssignment; import org.apache.flink.core.io.InputSplitAssigner; import org.apache.flink.core.io.InputSplitSource; @@ -263,7 +264,7 @@ public void testMultipleInstancesPerHost() { vertex.setInvokableClass(DummyInvokable.class); vertex.setInputSplitSource(new TestInputSplitSource(splits)); - JobGraph jobGraph = new JobGraph("test job", vertex); + JobGraph jobGraph = new JobGraph("test job", new ExecutionConfig(), vertex); ExecutionGraph eg = new ExecutionGraph( TestingUtils.defaultExecutionContext(), @@ -326,7 +327,7 @@ private static String[] runTests(int numHosts, int slotsPerHost, int parallelism vertex.setInvokableClass(DummyInvokable.class); vertex.setInputSplitSource(new TestInputSplitSource(splits)); - JobGraph jobGraph = new JobGraph("test job", vertex); + JobGraph jobGraph = new JobGraph("test job", new ExecutionConfig(), vertex); ExecutionGraph eg = new ExecutionGraph( TestingUtils.defaultExecutionContext(), diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/VertexLocationConstraintTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/VertexLocationConstraintTest.java index 8604b63a111b76..f456524a719002 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/VertexLocationConstraintTest.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/VertexLocationConstraintTest.java @@ -26,6 +26,7 @@ import java.util.Collections; import java.util.concurrent.TimeUnit; +import org.apache.flink.api.common.ExecutionConfig; import org.apache.flink.runtime.instance.DummyActorGateway; import org.apache.flink.runtime.instance.HardwareDescription; import org.apache.flink.runtime.instance.Instance; @@ -73,7 +74,7 @@ public void testScheduleWithConstraint1() { JobVertex jobVertex = new JobVertex("test vertex", new JobVertexID()); jobVertex.setInvokableClass(DummyInvokable.class); jobVertex.setParallelism(2); - JobGraph jg = new JobGraph("test job", jobVertex); + JobGraph jg = new JobGraph("test job", new ExecutionConfig(), jobVertex); ExecutionGraph eg = new ExecutionGraph( TestingUtils.defaultExecutionContext(), @@ -144,7 +145,7 @@ public void testScheduleWithConstraint2() { JobVertex jobVertex = new JobVertex("test vertex", new JobVertexID()); jobVertex.setInvokableClass(DummyInvokable.class); jobVertex.setParallelism(2); - JobGraph jg = new JobGraph("test job", jobVertex); + JobGraph jg = new JobGraph("test job", new ExecutionConfig(), jobVertex); ExecutionGraph eg = new ExecutionGraph( TestingUtils.defaultExecutionContext(), @@ -219,7 +220,7 @@ public void testScheduleWithConstraintAndSlotSharing() { jobVertex1.setSlotSharingGroup(sharingGroup); jobVertex2.setSlotSharingGroup(sharingGroup); - JobGraph jg = new JobGraph("test job", jobVertex1, jobVertex2); + JobGraph jg = new JobGraph("test job", new ExecutionConfig(), jobVertex1, jobVertex2); ExecutionGraph eg = new ExecutionGraph( TestingUtils.defaultExecutionContext(), @@ -285,7 +286,7 @@ public void testScheduleWithUnfulfillableConstraint() { JobVertex jobVertex = new JobVertex("test vertex", new JobVertexID()); jobVertex.setInvokableClass(DummyInvokable.class); jobVertex.setParallelism(1); - JobGraph jg = new JobGraph("test job", jobVertex); + JobGraph jg = new JobGraph("test job", new ExecutionConfig(), jobVertex); ExecutionGraph eg = new ExecutionGraph( TestingUtils.defaultExecutionContext(), @@ -349,7 +350,7 @@ public void testScheduleWithUnfulfillableConstraintInSharingGroup() { jobVertex1.setParallelism(1); jobVertex2.setParallelism(1); - JobGraph jg = new JobGraph("test job", jobVertex1, jobVertex2); + JobGraph jg = new JobGraph("test job", new ExecutionConfig(), jobVertex1, jobVertex2); SlotSharingGroup sharingGroup = new SlotSharingGroup(); jobVertex1.setSlotSharingGroup(sharingGroup); @@ -391,7 +392,7 @@ public void testScheduleWithUnfulfillableConstraintInSharingGroup() { public void testArchivingClearsFields() { try { JobVertex vertex = new JobVertex("test vertex", new JobVertexID()); - JobGraph jg = new JobGraph("test job", vertex); + JobGraph jg = new JobGraph("test job", new ExecutionConfig(), vertex); ExecutionGraph eg = new ExecutionGraph( TestingUtils.defaultExecutionContext(), diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/PartialConsumePipelinedResultTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/PartialConsumePipelinedResultTest.java index 854be5f2a2aea0..317eed7edf47d4 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/PartialConsumePipelinedResultTest.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/PartialConsumePipelinedResultTest.java @@ -18,6 +18,7 @@ package org.apache.flink.runtime.io.network.partition; +import org.apache.flink.api.common.ExecutionConfig; import org.apache.flink.configuration.ConfigConstants; import org.apache.flink.configuration.Configuration; import org.apache.flink.runtime.io.network.api.reader.BufferReader; @@ -89,7 +90,7 @@ public void testPartialConsumePipelinedResultReceiver() throws Exception { sender, DistributionPattern.POINTWISE, ResultPartitionType.PIPELINED); final JobGraph jobGraph = new JobGraph( - "Partial Consume of Pipelined Result", sender, receiver); + "Partial Consume of Pipelined Result", new ExecutionConfig(), sender, receiver); final SlotSharingGroup slotSharingGroup = new SlotSharingGroup( sender.getID(), receiver.getID()); diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/jobgraph/JobGraphTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/jobgraph/JobGraphTest.java index 1272c3a30c10fc..68b05b209a01c1 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/jobgraph/JobGraphTest.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/jobgraph/JobGraphTest.java @@ -20,9 +20,9 @@ import static org.junit.Assert.*; -import java.util.Arrays; import java.util.List; +import org.apache.flink.api.common.ExecutionConfig; import org.apache.flink.api.common.InvalidProgramException; import org.apache.flink.core.testutils.CommonTestUtils; import org.junit.Test; @@ -32,7 +32,7 @@ public class JobGraphTest { @Test public void testSerialization() { try { - JobGraph jg = new JobGraph("The graph"); + JobGraph jg = new JobGraph("The graph", new ExecutionConfig()); // add some configuration values { @@ -91,8 +91,8 @@ public void testTopologicalSort1() { intermediate2.connectNewDataSetAsInput(intermediate1, DistributionPattern.POINTWISE); intermediate1.connectNewDataSetAsInput(source2, DistributionPattern.POINTWISE); - List vertices = Arrays.asList(source1, source2, intermediate1, intermediate2, target1, target2); - JobGraph graph = new JobGraph("TestGraph", vertices); + JobGraph graph = new JobGraph("TestGraph", new ExecutionConfig(), + source1, source2, intermediate1, intermediate2, target1, target2); List sorted = graph.getVerticesSortedTopologicallyFromSources(); assertEquals(6, sorted.size()); @@ -136,8 +136,8 @@ public void testTopologicalSort2() { l13.connectNewDataSetAsInput(source2, DistributionPattern.POINTWISE); - List vertices = Arrays.asList(source1, source2, root, l11, l13, l12, l2); - JobGraph graph = new JobGraph("TestGraph", vertices); + JobGraph graph = new JobGraph("TestGraph", new ExecutionConfig(), + source1, source2, root, l11, l13, l12, l2); List sorted = graph.getVerticesSortedTopologicallyFromSources(); assertEquals(7, sorted.size()); @@ -183,8 +183,7 @@ public void testTopologicalSort3() { op2.connectNewDataSetAsInput(source, DistributionPattern.POINTWISE); op3.connectNewDataSetAsInput(op2, DistributionPattern.POINTWISE); - List vertices = Arrays.asList(source, op1, op2, op3); - JobGraph graph = new JobGraph("TestGraph", vertices); + JobGraph graph = new JobGraph("TestGraph", new ExecutionConfig(), source, op1, op2, op3); List sorted = graph.getVerticesSortedTopologicallyFromSources(); assertEquals(4, sorted.size()); @@ -213,8 +212,7 @@ public void testTopoSortCyclicGraphNoSources() { v3.connectNewDataSetAsInput(v2, DistributionPattern.POINTWISE); v4.connectNewDataSetAsInput(v3, DistributionPattern.POINTWISE); - List vertices = Arrays.asList(v1, v2, v3, v4); - JobGraph jg = new JobGraph("Cyclic Graph", vertices); + JobGraph jg = new JobGraph("Cyclic Graph", new ExecutionConfig(), v1, v2, v3, v4); try { jg.getVerticesSortedTopologicallyFromSources(); fail("Failed to raise error on topologically sorting cyclic graph."); @@ -246,8 +244,7 @@ public void testTopoSortCyclicGraphIntermediateCycle() { v4.connectNewDataSetAsInput(v3, DistributionPattern.POINTWISE); target.connectNewDataSetAsInput(v3, DistributionPattern.POINTWISE); - List vertices = Arrays.asList(v1, v2, v3, v4, source, target); - JobGraph jg = new JobGraph("Cyclic Graph", vertices); + JobGraph jg = new JobGraph("Cyclic Graph", new ExecutionConfig(), v1, v2, v3, v4, source, target); try { jg.getVerticesSortedTopologicallyFromSources(); fail("Failed to raise error on topologically sorting cyclic graph."); diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/jobgraph/jsonplan/JsonGeneratorTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/jobgraph/jsonplan/JsonGeneratorTest.java index 98e8e6e5446e5a..612f64f9c9f257 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/jobgraph/jsonplan/JsonGeneratorTest.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/jobgraph/jsonplan/JsonGeneratorTest.java @@ -21,6 +21,7 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.node.TextNode; +import org.apache.flink.api.common.ExecutionConfig; import org.apache.flink.runtime.io.network.partition.ResultPartitionType; import org.apache.flink.runtime.jobgraph.DistributionPattern; import org.apache.flink.runtime.jobgraph.JobGraph; @@ -29,9 +30,7 @@ import org.apache.flink.runtime.operators.testutils.DummyInvokable; import org.junit.Test; -import java.util.Arrays; import java.util.Iterator; -import java.util.List; import static org.junit.Assert.*; @@ -68,9 +67,8 @@ public void testGeneratorWithoutAnyAttachements() { sink1.connectNewDataSetAsInput(join2, DistributionPattern.POINTWISE); sink2.connectNewDataSetAsInput(join1, DistributionPattern.ALL_TO_ALL); - List vertices = Arrays.asList(source1, source2, source3, - intermediate1, intermediate2, join1, join2, sink1, sink2); - JobGraph jg = new JobGraph("my job", vertices); + JobGraph jg = new JobGraph("my job", new ExecutionConfig(), source1, source2, source3, + intermediate1, intermediate2, join1, join2, sink1, sink2); String plan = JsonPlanGenerator.generatePlan(jg); assertNotNull(plan); diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/jobmanager/JobManagerSubmittedJobGraphsRecoveryITCase.java b/flink-runtime/src/test/java/org/apache/flink/runtime/jobmanager/JobManagerSubmittedJobGraphsRecoveryITCase.java index 59c7c39e621f75..6bf2fe2938d18d 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/jobmanager/JobManagerSubmittedJobGraphsRecoveryITCase.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/jobmanager/JobManagerSubmittedJobGraphsRecoveryITCase.java @@ -25,6 +25,7 @@ import akka.testkit.TestActorRef; import org.apache.commons.io.FileUtils; import org.apache.commons.io.filefilter.TrueFileFilter; +import org.apache.flink.api.common.ExecutionConfig; import org.apache.flink.configuration.ConfigConstants; import org.apache.flink.configuration.Configuration; import org.apache.flink.runtime.akka.AkkaUtils; @@ -421,7 +422,7 @@ public void awaitJobResult(long timeout) throws InterruptedException { * Creates a simple blocking JobGraph. */ private static JobGraph createBlockingJobGraph() { - JobGraph jobGraph = new JobGraph("Blocking program"); + JobGraph jobGraph = new JobGraph("Blocking program", new ExecutionConfig()); JobVertex jobVertex = new JobVertex("Blocking Vertex"); jobVertex.setInvokableClass(Tasks.BlockingNoOpInvokable.class); diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/jobmanager/JobManagerTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/jobmanager/JobManagerTest.java index db6df75de1891b..06ff5c5286de3a 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/jobmanager/JobManagerTest.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/jobmanager/JobManagerTest.java @@ -23,6 +23,7 @@ import com.typesafe.config.Config; +import org.apache.flink.api.common.ExecutionConfig; import org.apache.flink.api.common.JobID; import org.apache.flink.configuration.Configuration; import org.apache.flink.runtime.akka.AkkaUtils; @@ -118,7 +119,7 @@ public void testRequestPartitionState() throws Exception { sender.setInvokableClass(Tasks.BlockingNoOpInvokable.class); // just block sender.createAndAddResultDataSet(rid, PIPELINED); - final JobGraph jobGraph = new JobGraph("Blocking test job", sender); + final JobGraph jobGraph = new JobGraph("Blocking test job", new ExecutionConfig(), sender); final JobID jid = jobGraph.getJobID(); final ActorGateway jobManagerGateway = cluster.getLeaderGateway( diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/jobmanager/JobSubmitTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/jobmanager/JobSubmitTest.java index adaff29040dc58..1cd01ffb09411a 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/jobmanager/JobSubmitTest.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/jobmanager/JobSubmitTest.java @@ -20,6 +20,7 @@ import akka.actor.ActorRef; import akka.actor.ActorSystem; +import org.apache.flink.api.common.ExecutionConfig; import org.apache.flink.configuration.ConfigConstants; import org.apache.flink.configuration.Configuration; import org.apache.flink.runtime.akka.AkkaUtils; @@ -105,7 +106,7 @@ public void testFailureWhenJarBlobsMissing() { // create a simple job graph JobVertex jobVertex = new JobVertex("Test Vertex"); jobVertex.setInvokableClass(Tasks.NoOpInvokable.class); - JobGraph jg = new JobGraph("test job", jobVertex); + JobGraph jg = new JobGraph("test job", new ExecutionConfig(), jobVertex); // request the blob port from the job manager Future future = jmGateway.ask(JobManagerMessages.getRequestBlobManagerPort(), timeout); @@ -169,7 +170,7 @@ public void initializeOnMaster(ClassLoader loader) throws Exception { }; jobVertex.setInvokableClass(Tasks.NoOpInvokable.class); - JobGraph jg = new JobGraph("test job", jobVertex); + JobGraph jg = new JobGraph("test job", new ExecutionConfig(), jobVertex); // submit the job Future submitFuture = jmGateway.ask( diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/jobmanager/SlotCountExceedingParallelismTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/jobmanager/SlotCountExceedingParallelismTest.java index 17334064605a2f..dfb0b91d11eec6 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/jobmanager/SlotCountExceedingParallelismTest.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/jobmanager/SlotCountExceedingParallelismTest.java @@ -18,6 +18,7 @@ package org.apache.flink.runtime.jobmanager; +import org.apache.flink.api.common.ExecutionConfig; import org.apache.flink.runtime.client.JobExecutionException; import org.apache.flink.runtime.io.network.api.reader.RecordReader; import org.apache.flink.runtime.io.network.api.writer.RecordWriter; @@ -103,7 +104,7 @@ private JobGraph createTestJobGraph( DistributionPattern.ALL_TO_ALL, ResultPartitionType.BLOCKING); - final JobGraph jobGraph = new JobGraph(jobName, sender, receiver); + final JobGraph jobGraph = new JobGraph(jobName, new ExecutionConfig(), sender, receiver); // We need to allow queued scheduling, because there are not enough slots available // to run all tasks at once. We queue tasks and then let them finish/consume the blocking diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/jobmanager/StandaloneSubmittedJobGraphStoreTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/jobmanager/StandaloneSubmittedJobGraphStoreTest.java index 753e7be2a00807..ca2ecf515ec549 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/jobmanager/StandaloneSubmittedJobGraphStoreTest.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/jobmanager/StandaloneSubmittedJobGraphStoreTest.java @@ -19,6 +19,7 @@ package org.apache.flink.runtime.jobmanager; import akka.actor.ActorRef; +import org.apache.flink.api.common.ExecutionConfig; import org.apache.flink.api.common.JobID; import org.apache.flink.runtime.akka.ListeningBehaviour; import org.apache.flink.runtime.jobgraph.JobGraph; @@ -37,7 +38,7 @@ public void testNoOps() throws Exception { StandaloneSubmittedJobGraphStore jobGraphs = new StandaloneSubmittedJobGraphStore(); SubmittedJobGraph jobGraph = new SubmittedJobGraph( - new JobGraph("testNoOps"), + new JobGraph("testNoOps", new ExecutionConfig()), new JobInfo(ActorRef.noSender(), ListeningBehaviour.DETACHED, 0, Integer.MAX_VALUE)); assertEquals(0, jobGraphs.recoverJobGraphs().size()); diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/jobmanager/ZooKeeperSubmittedJobGraphsStoreITCase.java b/flink-runtime/src/test/java/org/apache/flink/runtime/jobmanager/ZooKeeperSubmittedJobGraphsStoreITCase.java index 356ba3687ac81b..5e53596fab28f6 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/jobmanager/ZooKeeperSubmittedJobGraphsStoreITCase.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/jobmanager/ZooKeeperSubmittedJobGraphsStoreITCase.java @@ -19,6 +19,7 @@ package org.apache.flink.runtime.jobmanager; import akka.actor.ActorRef; +import org.apache.flink.api.common.ExecutionConfig; import org.apache.flink.api.common.JobID; import org.apache.flink.runtime.akka.ListeningBehaviour; import org.apache.flink.runtime.jobgraph.JobGraph; @@ -260,7 +261,7 @@ public void testUpdateJobGraphYouDidNotGetOrAdd() throws Exception { // --------------------------------------------------------------------------------------------- private SubmittedJobGraph createSubmittedJobGraph(JobID jobId, long start) { - final JobGraph jobGraph = new JobGraph(jobId, "Test JobGraph"); + final JobGraph jobGraph = new JobGraph(jobId, "Test JobGraph", new ExecutionConfig()); final JobVertex jobVertex = new JobVertex("Test JobVertex"); jobVertex.setParallelism(1); diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/jobmanager/scheduler/ScheduleOrUpdateConsumersTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/jobmanager/scheduler/ScheduleOrUpdateConsumersTest.java index e345310d441626..07fc2c550e1edb 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/jobmanager/scheduler/ScheduleOrUpdateConsumersTest.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/jobmanager/scheduler/ScheduleOrUpdateConsumersTest.java @@ -20,6 +20,7 @@ import com.google.common.collect.Lists; +import org.apache.flink.api.common.ExecutionConfig; import org.apache.flink.runtime.io.network.api.writer.RecordWriter; import org.apache.flink.runtime.io.network.partition.ResultPartitionType; import org.apache.flink.runtime.jobgraph.JobVertex; @@ -35,7 +36,6 @@ import org.junit.BeforeClass; import org.junit.Test; -import java.util.Arrays; import java.util.List; import static org.apache.flink.runtime.jobmanager.SlotCountExceedingParallelismTest.SubtaskIndexReceiver.CONFIG_KEY; @@ -111,13 +111,12 @@ public void testMixedPipelinedAndBlockingResults() throws Exception { pipelinedReceiver.setSlotSharingGroup(slotSharingGroup); blockingReceiver.setSlotSharingGroup(slotSharingGroup); - List vertices = Arrays.asList(sender, - pipelinedReceiver, - blockingReceiver); - final JobGraph jobGraph = new JobGraph( "Mixed pipelined and blocking result", - vertices); + new ExecutionConfig(), + sender, + pipelinedReceiver, + blockingReceiver); flink.submitJobAndWait(jobGraph, false, TestingUtils.TESTING_DURATION()); } diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/leaderelection/LeaderChangeStateCleanupTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/leaderelection/LeaderChangeStateCleanupTest.java index c490a640c3269c..f14d62f34bc4e5 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/leaderelection/LeaderChangeStateCleanupTest.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/leaderelection/LeaderChangeStateCleanupTest.java @@ -18,6 +18,7 @@ package org.apache.flink.runtime.leaderelection; +import org.apache.flink.api.common.ExecutionConfig; import org.apache.flink.configuration.ConfigConstants; import org.apache.flink.configuration.Configuration; import org.apache.flink.runtime.instance.ActorGateway; @@ -268,6 +269,6 @@ public JobGraph createBlockingJob(int parallelism) { sender.setSlotSharingGroup(slotSharingGroup); receiver.setSlotSharingGroup(slotSharingGroup); - return new JobGraph("Blocking test job", sender, receiver); + return new JobGraph("Blocking test job", new ExecutionConfig(), sender, receiver); } } diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/operators/testutils/DummyEnvironment.java b/flink-runtime/src/test/java/org/apache/flink/runtime/operators/testutils/DummyEnvironment.java index 85e8603606aba2..a60199d2870575 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/operators/testutils/DummyEnvironment.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/operators/testutils/DummyEnvironment.java @@ -22,6 +22,7 @@ import java.util.concurrent.Future; import org.apache.flink.api.common.ApplicationID; +import org.apache.flink.api.common.ExecutionConfig; import org.apache.flink.api.common.JobID; import org.apache.flink.api.common.TaskInfo; import org.apache.flink.configuration.Configuration; @@ -45,11 +46,17 @@ public class DummyEnvironment implements Environment { private final ApplicationID appId = new ApplicationID(); private final JobID jobId = new JobID(); private final JobVertexID jobVertexId = new JobVertexID(); + private final ExecutionConfig executionConfig = new ExecutionConfig(); public DummyEnvironment(String taskName, int numSubTasks, int subTaskIndex) { this.taskInfo = new TaskInfo(taskName, subTaskIndex, numSubTasks, 0); } + @Override + public ExecutionConfig getExecutionConfig() { + return executionConfig; + } + @Override public ApplicationID getApplicationID() { return appId; diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/operators/testutils/MockEnvironment.java b/flink-runtime/src/test/java/org/apache/flink/runtime/operators/testutils/MockEnvironment.java index 0ff03c3b63f450..5a56505f61f259 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/operators/testutils/MockEnvironment.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/operators/testutils/MockEnvironment.java @@ -19,6 +19,7 @@ package org.apache.flink.runtime.operators.testutils; import org.apache.flink.api.common.ApplicationID; +import org.apache.flink.api.common.ExecutionConfig; import org.apache.flink.api.common.TaskInfo; import org.apache.flink.configuration.Configuration; import org.apache.flink.configuration.UnmodifiableConfiguration; @@ -65,7 +66,9 @@ public class MockEnvironment implements Environment { private final TaskInfo taskInfo; - + + private final ExecutionConfig executionConfig; + private final MemoryManager memManager; private final IOManager ioManager; @@ -99,6 +102,7 @@ public MockEnvironment(String taskName, long memorySize, MockInputSplitProvider this.memManager = new MemoryManager(memorySize, 1); this.ioManager = new IOManagerAsync(); + this.executionConfig = new ExecutionConfig(); this.inputSplitProvider = inputSplitProvider; this.bufferSize = bufferSize; @@ -188,6 +192,11 @@ public IOManager getIOManager() { return this.ioManager; } + @Override + public ExecutionConfig getExecutionConfig() { + return this.executionConfig; + } + @Override public ApplicationID getApplicationID() { return this.appId; diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/taskmanager/TaskAsyncCallTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/taskmanager/TaskAsyncCallTest.java index 54723ffa505e5f..f7f9bd5e0adfdf 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/taskmanager/TaskAsyncCallTest.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/taskmanager/TaskAsyncCallTest.java @@ -19,6 +19,7 @@ package org.apache.flink.runtime.taskmanager; import org.apache.flink.api.common.ApplicationID; +import org.apache.flink.api.common.ExecutionConfig; import org.apache.flink.api.common.JobID; import org.apache.flink.configuration.Configuration; import org.apache.flink.runtime.blob.BlobKey; @@ -149,7 +150,7 @@ private static Task createTask() { TaskDeploymentDescriptor tdd = new TaskDeploymentDescriptor( new ApplicationID(), new JobID(), new JobVertexID(), new ExecutionAttemptID(), - "Test Task", 0, 1, 0, + new ExecutionConfig(), "Test Task", 0, 1, 0, new Configuration(), new Configuration(), CheckpointsInOrderInvokable.class.getName(), Collections.emptyList(), diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/taskmanager/TaskCancelTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/taskmanager/TaskCancelTest.java index a3d18831c49893..e7f4c5c4a2bee1 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/taskmanager/TaskCancelTest.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/taskmanager/TaskCancelTest.java @@ -18,6 +18,7 @@ package org.apache.flink.runtime.taskmanager; +import org.apache.flink.api.common.ExecutionConfig; import org.apache.flink.api.common.JobID; import org.apache.flink.configuration.ConfigConstants; import org.apache.flink.configuration.Configuration; @@ -73,7 +74,7 @@ public void testCancelUnion() throws Exception { flink.start(); // Setup - final JobGraph jobGraph = new JobGraph("Cancel Big Union"); + final JobGraph jobGraph = new JobGraph("Cancel Big Union", new ExecutionConfig()); JobVertex[] sources = new JobVertex[numberOfSources]; SlotSharingGroup group = new SlotSharingGroup(); diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/taskmanager/TaskManagerTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/taskmanager/TaskManagerTest.java index f5271dfa6d3a90..35c2a79fc8ab09 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/taskmanager/TaskManagerTest.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/taskmanager/TaskManagerTest.java @@ -24,6 +24,7 @@ import akka.japi.Creator; import akka.testkit.JavaTestKit; import org.apache.flink.api.common.ApplicationID; +import org.apache.flink.api.common.ExecutionConfig; import org.apache.flink.api.common.JobID; import org.apache.flink.configuration.ConfigConstants; import org.apache.flink.configuration.Configuration; @@ -157,9 +158,11 @@ protected void run() { final JobID jid = new JobID(); final JobVertexID vid = new JobVertexID(); final ExecutionAttemptID eid = new ExecutionAttemptID(); + final ExecutionConfig executionConfig = new ExecutionConfig(); - final TaskDeploymentDescriptor tdd = new TaskDeploymentDescriptor(appId, jid, vid, eid, "TestTask", 2, 7, 0, - new Configuration(), new Configuration(), TestInvokableCorrect.class.getName(), + final TaskDeploymentDescriptor tdd = new TaskDeploymentDescriptor(appId, jid, vid, eid, executionConfig, + "TestTask", 2, 7, 0, new Configuration(), new Configuration(), + TestInvokableCorrect.class.getName(), Collections.emptyList(), Collections.emptyList(), new ArrayList(), Collections.emptyList(), 0); @@ -258,13 +261,15 @@ public void testJobSubmissionAndCanceling() { final ExecutionAttemptID eid1 = new ExecutionAttemptID(); final ExecutionAttemptID eid2 = new ExecutionAttemptID(); - final TaskDeploymentDescriptor tdd1 = new TaskDeploymentDescriptor(appId, jid1, vid1, eid1, "TestTask1", 1, 5, 0, + final TaskDeploymentDescriptor tdd1 = new TaskDeploymentDescriptor(appId, jid1, vid1, eid1, + new ExecutionConfig(), "TestTask1", 1, 5, 0, new Configuration(), new Configuration(), TestInvokableBlockingCancelable.class.getName(), Collections.emptyList(), Collections.emptyList(), new ArrayList(), Collections.emptyList(), 0); - final TaskDeploymentDescriptor tdd2 = new TaskDeploymentDescriptor(appId, jid2, vid2, eid2, "TestTask2", 2, 7, 0, + final TaskDeploymentDescriptor tdd2 = new TaskDeploymentDescriptor(appId, jid2, vid2, eid2, + new ExecutionConfig(), "TestTask2", 2, 7, 0, new Configuration(), new Configuration(), TestInvokableBlockingCancelable.class.getName(), Collections.emptyList(), Collections.emptyList(), @@ -391,13 +396,15 @@ public void testGateChannelEdgeMismatch() { final ExecutionAttemptID eid1 = new ExecutionAttemptID(); final ExecutionAttemptID eid2 = new ExecutionAttemptID(); - final TaskDeploymentDescriptor tdd1 = new TaskDeploymentDescriptor(appId, jid, vid1, eid1, "Sender", 0, 1, 0, + final TaskDeploymentDescriptor tdd1 = new TaskDeploymentDescriptor(appId, jid, vid1, eid1, + new ExecutionConfig(), "Sender", 0, 1, 0, new Configuration(), new Configuration(), Tasks.Sender.class.getName(), Collections.emptyList(), Collections.emptyList(), new ArrayList(), Collections.emptyList(), 0); - final TaskDeploymentDescriptor tdd2 = new TaskDeploymentDescriptor(appId, jid, vid2, eid2, "Receiver", 2, 7, 0, + final TaskDeploymentDescriptor tdd2 = new TaskDeploymentDescriptor(appId, jid, vid2, eid2, + new ExecutionConfig(), "Receiver", 2, 7, 0, new Configuration(), new Configuration(), Tasks.Receiver.class.getName(), Collections.emptyList(), Collections.emptyList(), @@ -490,12 +497,14 @@ public void testRunJobWithForwardChannel() { } ); - final TaskDeploymentDescriptor tdd1 = new TaskDeploymentDescriptor(appId, jid, vid1, eid1, "Sender", 0, 1, 0, + final TaskDeploymentDescriptor tdd1 = new TaskDeploymentDescriptor(appId, jid, vid1, eid1, + new ExecutionConfig(), "Sender", 0, 1, 0, new Configuration(), new Configuration(), Tasks.Sender.class.getName(), irpdd, Collections.emptyList(), new ArrayList(), Collections.emptyList(), 0); - final TaskDeploymentDescriptor tdd2 = new TaskDeploymentDescriptor(appId, jid, vid2, eid2, "Receiver", 2, 7, 0, + final TaskDeploymentDescriptor tdd2 = new TaskDeploymentDescriptor(appId, jid, vid2, eid2, + new ExecutionConfig(), "Receiver", 2, 7, 0, new Configuration(), new Configuration(), Tasks.Receiver.class.getName(), Collections.emptyList(), Collections.singletonList(ircdd), @@ -629,12 +638,14 @@ public void testCancellingDependentAndStateUpdateFails() { } ); - final TaskDeploymentDescriptor tdd1 = new TaskDeploymentDescriptor(appId, jid, vid1, eid1, "Sender", 0, 1, 0, + final TaskDeploymentDescriptor tdd1 = new TaskDeploymentDescriptor(appId, jid, vid1, eid1, + new ExecutionConfig(), "Sender", 0, 1, 0, new Configuration(), new Configuration(), Tasks.Sender.class.getName(), irpdd, Collections.emptyList(), new ArrayList(), Collections.emptyList(), 0); - final TaskDeploymentDescriptor tdd2 = new TaskDeploymentDescriptor(appId, jid, vid2, eid2, "Receiver", 2, 7, 0, + final TaskDeploymentDescriptor tdd2 = new TaskDeploymentDescriptor(appId, jid, vid2, eid2, + new ExecutionConfig(), "Receiver", 2, 7, 0, new Configuration(), new Configuration(), Tasks.BlockingReceiver.class.getName(), Collections.emptyList(), Collections.singletonList(ircdd), @@ -772,7 +783,8 @@ public void testRemotePartitionNotFound() throws Exception { new InputGateDeploymentDescriptor(resultId, 0, icdd); final TaskDeploymentDescriptor tdd = new TaskDeploymentDescriptor( - appId, jid, vid, eid, "Receiver", 0, 1, 0, + appId, jid, vid, eid, + new ExecutionConfig(), "Receiver", 0, 1, 0, new Configuration(), new Configuration(), Tasks.AgnosticReceiver.class.getName(), Collections.emptyList(), @@ -865,7 +877,7 @@ public void testLocalPartitionNotFound() throws Exception { new InputGateDeploymentDescriptor(resultId, 0, icdd); final TaskDeploymentDescriptor tdd = new TaskDeploymentDescriptor( - appId, jid, vid, eid, "Receiver", 0, 1, 0, + appId, jid, vid, eid, new ExecutionConfig(), "Receiver", 0, 1, 0, new Configuration(), new Configuration(), Tasks.AgnosticReceiver.class.getName(), Collections.emptyList(), @@ -953,6 +965,7 @@ protected void run() { new JobID(), new JobVertexID(), new ExecutionAttemptID(), + new ExecutionConfig(), "Task", 0, 1, diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/taskmanager/TaskTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/taskmanager/TaskTest.java index f7c0ae0a39ffbf..d3fe8475a134f4 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/taskmanager/TaskTest.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/taskmanager/TaskTest.java @@ -21,6 +21,7 @@ import com.google.common.collect.Maps; import org.apache.flink.api.common.ApplicationID; +import org.apache.flink.api.common.ExecutionConfig; import org.apache.flink.configuration.Configuration; import org.apache.flink.runtime.blob.BlobKey; import org.apache.flink.runtime.broadcast.BroadcastVariableManager; @@ -628,7 +629,7 @@ private Task createTask(Class invokable, private TaskDeploymentDescriptor createTaskDeploymentDescriptor(Class invokable) { return new TaskDeploymentDescriptor( new ApplicationID(), new JobID(), new JobVertexID(), new ExecutionAttemptID(), - "Test Task", 0, 1, 0, + new ExecutionConfig(), "Test Task", 0, 1, 0, new Configuration(), new Configuration(), invokable.getName(), Collections.emptyList(), diff --git a/flink-runtime/src/test/scala/org/apache/flink/runtime/executiongraph/TaskManagerLossFailsTasksTest.scala b/flink-runtime/src/test/scala/org/apache/flink/runtime/executiongraph/TaskManagerLossFailsTasksTest.scala index 177dc851ec19d2..273cf107cb88b2 100644 --- a/flink-runtime/src/test/scala/org/apache/flink/runtime/executiongraph/TaskManagerLossFailsTasksTest.scala +++ b/flink-runtime/src/test/scala/org/apache/flink/runtime/executiongraph/TaskManagerLossFailsTasksTest.scala @@ -18,7 +18,7 @@ package org.apache.flink.runtime.executiongraph -import org.apache.flink.api.common.JobID +import org.apache.flink.api.common.{ExecutionConfig, JobID} import org.apache.flink.configuration.Configuration import org.apache.flink.runtime.akka.AkkaUtils import org.apache.flink.runtime.executiongraph.ExecutionGraphTestUtils.SimpleActorGateway @@ -49,7 +49,7 @@ class TaskManagerLossFailsTasksTest extends WordSpecLike with Matchers { sender.setInvokableClass(classOf[Tasks.NoOpInvokable]) sender.setParallelism(20) - val jobGraph = new JobGraph("Pointwise job", sender) + val jobGraph = new JobGraph("Pointwise job", new ExecutionConfig(), sender) val eg = new ExecutionGraph( TestingUtils.defaultExecutionContext, diff --git a/flink-runtime/src/test/scala/org/apache/flink/runtime/jobmanager/CoLocationConstraintITCase.scala b/flink-runtime/src/test/scala/org/apache/flink/runtime/jobmanager/CoLocationConstraintITCase.scala index a0c144a785eb32..f52d37e5ee8543 100644 --- a/flink-runtime/src/test/scala/org/apache/flink/runtime/jobmanager/CoLocationConstraintITCase.scala +++ b/flink-runtime/src/test/scala/org/apache/flink/runtime/jobmanager/CoLocationConstraintITCase.scala @@ -21,6 +21,7 @@ package org.apache.flink.runtime.jobmanager import akka.actor.ActorSystem import akka.actor.Status.Success import akka.testkit.{ImplicitSender, TestKit} +import org.apache.flink.api.common.ExecutionConfig import org.apache.flink.runtime.akka.ListeningBehaviour import org.apache.flink.runtime.jobgraph.{JobGraph, DistributionPattern, JobVertex} import org.apache.flink.runtime.jobmanager.Tasks.{Receiver, Sender} @@ -67,7 +68,7 @@ class CoLocationConstraintITCase(_system: ActorSystem) receiver.setStrictlyCoLocatedWith(sender) - val jobGraph = new JobGraph("Pointwise job", sender, receiver) + val jobGraph = new JobGraph("Pointwise job", new ExecutionConfig(), sender, receiver) val cluster = TestingUtils.startTestingCluster(num_tasks) val jmGateway = cluster.getLeaderGateway(1 seconds) diff --git a/flink-runtime/src/test/scala/org/apache/flink/runtime/jobmanager/JobManagerITCase.scala b/flink-runtime/src/test/scala/org/apache/flink/runtime/jobmanager/JobManagerITCase.scala index b398732877e8e7..894ba389ab366a 100644 --- a/flink-runtime/src/test/scala/org/apache/flink/runtime/jobmanager/JobManagerITCase.scala +++ b/flink-runtime/src/test/scala/org/apache/flink/runtime/jobmanager/JobManagerITCase.scala @@ -22,7 +22,7 @@ package org.apache.flink.runtime.jobmanager import akka.actor.ActorSystem import akka.testkit.{ImplicitSender, TestKit} import akka.util.Timeout -import org.apache.flink.api.common.JobID +import org.apache.flink.api.common.{ExecutionConfig, JobID} import org.apache.flink.runtime.akka.ListeningBehaviour import org.apache.flink.runtime.checkpoint.{CheckpointCoordinator, SavepointCoordinator} import org.apache.flink.runtime.client.JobExecutionException @@ -33,6 +33,7 @@ import org.apache.flink.runtime.jobmanager.scheduler.{NoResourceAvailableExcepti import org.apache.flink.runtime.messages.JobManagerMessages._ import org.apache.flink.runtime.testingUtils.TestingJobManagerMessages._ import org.apache.flink.runtime.testingUtils.{ScalaTestingUtils, TestingUtils} +import org.apache.flink.runtime.testutils.JobManagerActorTestUtils import org.junit.runner.RunWith import org.mockito.Mockito._ import org.scalatest.junit.JUnitRunner @@ -43,8 +44,6 @@ import scala.concurrent.duration._ import scala.language.postfixOps import scala.util.Random -import scala.collection.JavaConverters._ - @RunWith(classOf[JUnitRunner]) class JobManagerITCase(_system: ActorSystem) extends TestKit(_system) @@ -69,7 +68,7 @@ class JobManagerITCase(_system: ActorSystem) vertex.setParallelism(2) vertex.setInvokableClass(classOf[BlockingNoOpInvokable]) - val jobGraph = new JobGraph("Test Job", vertex) + val jobGraph = new JobGraph("Test Job", new ExecutionConfig(), vertex) val cluster = TestingUtils.startTestingCluster(1) val jmGateway = cluster.getLeaderGateway(1 seconds) @@ -111,7 +110,7 @@ class JobManagerITCase(_system: ActorSystem) vertex.setParallelism(num_tasks) vertex.setInvokableClass(classOf[NoOpInvokable]) - val jobGraph = new JobGraph("Test Job", vertex) + val jobGraph = new JobGraph("Test Job", new ExecutionConfig(), vertex) val cluster = TestingUtils.startTestingCluster(num_tasks) val jmGateway = cluster.getLeaderGateway(1 seconds) @@ -146,7 +145,7 @@ class JobManagerITCase(_system: ActorSystem) vertex.setParallelism(num_tasks) vertex.setInvokableClass(classOf[NoOpInvokable]) - val jobGraph = new JobGraph("Test job", vertex) + val jobGraph = new JobGraph("Test job", new ExecutionConfig(), vertex) jobGraph.setAllowQueuedScheduling(true) val cluster = TestingUtils.startTestingCluster(10) @@ -182,7 +181,7 @@ class JobManagerITCase(_system: ActorSystem) receiver.connectNewDataSetAsInput(sender, DistributionPattern.POINTWISE) - val jobGraph = new JobGraph("Pointwise Job", sender, receiver) + val jobGraph = new JobGraph("Pointwise Job", new ExecutionConfig(), sender, receiver) val cluster = TestingUtils.startTestingCluster(2 * num_tasks) val jmGateway = cluster.getLeaderGateway(1 seconds) @@ -217,7 +216,7 @@ class JobManagerITCase(_system: ActorSystem) receiver.connectNewDataSetAsInput(sender, DistributionPattern.POINTWISE) - val jobGraph = new JobGraph("Bipartite Job", sender, receiver) + val jobGraph = new JobGraph("Bipartite Job", new ExecutionConfig(), sender, receiver) val cluster = TestingUtils.startTestingCluster(2 * num_tasks) val jmGateway = cluster.getLeaderGateway(1 seconds) @@ -254,8 +253,8 @@ class JobManagerITCase(_system: ActorSystem) receiver.connectNewDataSetAsInput(sender1, DistributionPattern.POINTWISE) receiver.connectNewDataSetAsInput(sender2, DistributionPattern.ALL_TO_ALL) - val vertices = (List(sender1, receiver, sender2)).asJava - val jobGraph = new JobGraph("Bipartite Job", vertices) + val jobGraph = new JobGraph("Bipartite Job", new ExecutionConfig(), + sender1, receiver, sender2) val cluster = TestingUtils.startTestingCluster(6 * num_tasks) val jmGateway = cluster.getLeaderGateway(1 seconds) @@ -300,8 +299,8 @@ class JobManagerITCase(_system: ActorSystem) receiver.connectNewDataSetAsInput(sender1, DistributionPattern.POINTWISE) receiver.connectNewDataSetAsInput(sender2, DistributionPattern.ALL_TO_ALL) - val vertices = (List(sender1, receiver, sender2)).asJava - val jobGraph = new JobGraph("Bipartite Job", vertices) + val jobGraph = new JobGraph("Bipartite Job", new ExecutionConfig(), + sender1, receiver, sender2) val cluster = TestingUtils.startTestingCluster(6 * num_tasks) val jmGateway = cluster.getLeaderGateway(1 seconds) @@ -343,8 +342,8 @@ class JobManagerITCase(_system: ActorSystem) forwarder.connectNewDataSetAsInput(sender, DistributionPattern.ALL_TO_ALL) receiver.connectNewDataSetAsInput(forwarder, DistributionPattern.ALL_TO_ALL) - val vertices = (List(sender, forwarder, receiver)).asJava - val jobGraph = new JobGraph("Forwarding Job", vertices) + val jobGraph = new JobGraph("Forwarding Job", new ExecutionConfig(), + sender, forwarder, receiver) jobGraph.setScheduleMode(ScheduleMode.ALL) @@ -380,7 +379,7 @@ class JobManagerITCase(_system: ActorSystem) receiver.connectNewDataSetAsInput(sender, DistributionPattern.POINTWISE) - val jobGraph = new JobGraph("Pointwise Job", sender, receiver) + val jobGraph = new JobGraph("Pointwise Job", new ExecutionConfig(), sender, receiver) val cluster = TestingUtils.startTestingCluster(num_tasks) val jmGateway = cluster.getLeaderGateway(1 seconds) @@ -428,7 +427,7 @@ class JobManagerITCase(_system: ActorSystem) receiver.connectNewDataSetAsInput(sender, DistributionPattern.POINTWISE) - val jobGraph = new JobGraph("Pointwise Job", sender, receiver) + val jobGraph = new JobGraph("Pointwise Job", new ExecutionConfig(), sender, receiver) val cluster = TestingUtils.startTestingCluster(num_tasks) val jmGateway = cluster.getLeaderGateway(1 seconds) @@ -473,7 +472,7 @@ class JobManagerITCase(_system: ActorSystem) receiver.connectNewDataSetAsInput(sender, DistributionPattern.POINTWISE) - val jobGraph = new JobGraph("Pointwise job", sender, receiver) + val jobGraph = new JobGraph("Pointwise job", new ExecutionConfig(), sender, receiver) val cluster = TestingUtils.startTestingCluster(2 * num_tasks) val jmGateway = cluster.getLeaderGateway(1 seconds) @@ -513,7 +512,7 @@ class JobManagerITCase(_system: ActorSystem) receiver.connectNewDataSetAsInput(sender, DistributionPattern.POINTWISE) - val jobGraph = new JobGraph("Pointwise job", sender, receiver) + val jobGraph = new JobGraph("Pointwise job", new ExecutionConfig(), sender, receiver) val cluster = TestingUtils.startTestingCluster(num_tasks) val jmGateway = cluster.getLeaderGateway(1 seconds) @@ -561,7 +560,7 @@ class JobManagerITCase(_system: ActorSystem) receiver.connectNewDataSetAsInput(sender, DistributionPattern.POINTWISE) - val jobGraph = new JobGraph("Pointwise job", sender, receiver) + val jobGraph = new JobGraph("Pointwise job", new ExecutionConfig(), sender, receiver) val cluster = TestingUtils.startTestingCluster(num_tasks) val jmGateway = cluster.getLeaderGateway(1 seconds) @@ -604,7 +603,8 @@ class JobManagerITCase(_system: ActorSystem) source.setParallelism(num_tasks) sink.setParallelism(num_tasks) - val jobGraph = new JobGraph("SubtaskInFinalStateRaceCondition", source, sink) + val jobGraph = new JobGraph("SubtaskInFinalStateRaceCondition", + new ExecutionConfig(), source, sink) val cluster = TestingUtils.startTestingCluster(2*num_tasks) val jmGateway = cluster.getLeaderGateway(1 seconds) @@ -630,12 +630,12 @@ class JobManagerITCase(_system: ActorSystem) val vertex = new JobVertex("Test Vertex") vertex.setInvokableClass(classOf[NoOpInvokable]) - val jobGraph1 = new JobGraph("Test Job", vertex) + val jobGraph1 = new JobGraph("Test Job", new ExecutionConfig(), vertex) val slowVertex = new WaitingOnFinalizeJobVertex("Long running Vertex", 2000) slowVertex.setInvokableClass(classOf[NoOpInvokable]) - val jobGraph2 = new JobGraph("Long running Job", slowVertex) + val jobGraph2 = new JobGraph("Long running Job", new ExecutionConfig(), slowVertex) val cluster = TestingUtils.startTestingCluster(1) val jm = cluster.getLeaderGateway(1 seconds) @@ -684,7 +684,7 @@ class JobManagerITCase(_system: ActorSystem) vertex.setParallelism(1) vertex.setInvokableClass(classOf[NoOpInvokable]) - val jobGraph = new JobGraph("Test Job", vertex) + val jobGraph = new JobGraph("Test Job", new ExecutionConfig(), vertex) val cluster = TestingUtils.startTestingCluster(1) val jm = cluster.getLeaderGateway(1 seconds) @@ -782,7 +782,7 @@ class JobManagerITCase(_system: ActorSystem) val jobVertex = new JobVertex("Blocking vertex") jobVertex.setInvokableClass(classOf[BlockingNoOpInvokable]) - val jobGraph = new JobGraph(jobVertex) + val jobGraph = new JobGraph(new ExecutionConfig(), jobVertex) // Submit job w/o checkpointing configured jobManager.tell(SubmitJob(jobGraph, ListeningBehaviour.DETACHED), testActor) @@ -815,7 +815,7 @@ class JobManagerITCase(_system: ActorSystem) val jobVertex = new JobVertex("Blocking vertex") jobVertex.setInvokableClass(classOf[BlockingNoOpInvokable]) - val jobGraph = new JobGraph(jobVertex) + val jobGraph = new JobGraph(new ExecutionConfig(), jobVertex) jobGraph.setSnapshotSettings(new JobSnapshottingSettings( java.util.Collections.emptyList(), java.util.Collections.emptyList(), @@ -868,7 +868,7 @@ class JobManagerITCase(_system: ActorSystem) val jobVertex = new JobVertex("Blocking vertex") jobVertex.setInvokableClass(classOf[BlockingNoOpInvokable]) - val jobGraph = new JobGraph(jobVertex) + val jobGraph = new JobGraph(new ExecutionConfig(), jobVertex) jobGraph.setSnapshotSettings(new JobSnapshottingSettings( java.util.Collections.emptyList(), java.util.Collections.emptyList(), @@ -926,7 +926,7 @@ class JobManagerITCase(_system: ActorSystem) val jobVertex = new JobVertex("Blocking vertex") jobVertex.setInvokableClass(classOf[BlockingNoOpInvokable]) - val jobGraph = new JobGraph(jobVertex) + val jobGraph = new JobGraph(new ExecutionConfig(), jobVertex) jobGraph.setSnapshotSettings(new JobSnapshottingSettings( java.util.Collections.emptyList(), java.util.Collections.emptyList(), diff --git a/flink-runtime/src/test/scala/org/apache/flink/runtime/jobmanager/RecoveryITCase.scala b/flink-runtime/src/test/scala/org/apache/flink/runtime/jobmanager/RecoveryITCase.scala index 68923fe11c4b65..db7e85c2961a96 100644 --- a/flink-runtime/src/test/scala/org/apache/flink/runtime/jobmanager/RecoveryITCase.scala +++ b/flink-runtime/src/test/scala/org/apache/flink/runtime/jobmanager/RecoveryITCase.scala @@ -20,6 +20,7 @@ package org.apache.flink.runtime.jobmanager import akka.actor.{PoisonPill, ActorSystem} import akka.testkit.{ImplicitSender, TestKit} +import org.apache.flink.api.common.ExecutionConfig import org.apache.flink.configuration.{ConfigConstants, Configuration} import org.apache.flink.runtime.akka.ListeningBehaviour import org.apache.flink.runtime.jobgraph.{JobStatus, JobGraph, DistributionPattern, JobVertex} @@ -78,8 +79,10 @@ class RecoveryITCase(_system: ActorSystem) receiver.connectNewDataSetAsInput(sender, DistributionPattern.POINTWISE) - val jobGraph = new JobGraph("Pointwise job", sender, receiver) - jobGraph.setNumberOfExecutionRetries(1) + val executionConfig = new ExecutionConfig() + executionConfig.setNumberOfExecutionRetries(1); + + val jobGraph = new JobGraph("Pointwise job", executionConfig, sender, receiver) val cluster = createTestClusterWithHeartbeatTimeout(2 * NUM_TASKS, 1, "2 s") cluster.start() @@ -123,8 +126,10 @@ class RecoveryITCase(_system: ActorSystem) sender.setSlotSharingGroup(sharingGroup) receiver.setSlotSharingGroup(sharingGroup) - val jobGraph = new JobGraph("Pointwise job", sender, receiver) - jobGraph.setNumberOfExecutionRetries(1) + val executionConfig = new ExecutionConfig() + executionConfig.setNumberOfExecutionRetries(1); + + val jobGraph = new JobGraph("Pointwise job", executionConfig, sender, receiver) val cluster = createTestClusterWithHeartbeatTimeout(NUM_TASKS, 1, "2 s") cluster.start() @@ -168,8 +173,10 @@ class RecoveryITCase(_system: ActorSystem) sender.setSlotSharingGroup(sharingGroup) receiver.setSlotSharingGroup(sharingGroup) - val jobGraph = new JobGraph("Pointwise job", sender, receiver) - jobGraph.setNumberOfExecutionRetries(1) + val executionConfig = new ExecutionConfig() + executionConfig.setNumberOfExecutionRetries(1); + + val jobGraph = new JobGraph("Pointwise job", executionConfig, sender, receiver) val cluster = createTestClusterWithHeartbeatTimeout(NUM_TASKS, 2, "2 s") cluster.start() diff --git a/flink-runtime/src/test/scala/org/apache/flink/runtime/jobmanager/SlotSharingITCase.scala b/flink-runtime/src/test/scala/org/apache/flink/runtime/jobmanager/SlotSharingITCase.scala index 3e3c6733e67e60..4d320eabdadcbf 100644 --- a/flink-runtime/src/test/scala/org/apache/flink/runtime/jobmanager/SlotSharingITCase.scala +++ b/flink-runtime/src/test/scala/org/apache/flink/runtime/jobmanager/SlotSharingITCase.scala @@ -19,7 +19,9 @@ package org.apache.flink.runtime.jobmanager import akka.actor.ActorSystem +import akka.actor.Status.Success import akka.testkit.{ImplicitSender, TestKit} +import org.apache.flink.api.common.ExecutionConfig import org.apache.flink.runtime.akka.ListeningBehaviour import org.apache.flink.runtime.jobgraph.{JobVertex, DistributionPattern, JobGraph} import org.apache.flink.runtime.jobmanager.Tasks.{Sender, AgnosticBinaryReceiver, Receiver} @@ -30,7 +32,6 @@ import org.junit.runner.RunWith import org.scalatest.junit.JUnitRunner import org.scalatest.{BeforeAndAfterAll, Matchers, WordSpecLike} import scala.concurrent.duration._ -import scala.collection.JavaConverters._ @RunWith(classOf[JUnitRunner]) class SlotSharingITCase(_system: ActorSystem) @@ -65,7 +66,7 @@ class SlotSharingITCase(_system: ActorSystem) sender.setSlotSharingGroup(sharingGroup) receiver.setSlotSharingGroup(sharingGroup) - val jobGraph = new JobGraph("Pointwise Job", sender, receiver) + val jobGraph = new JobGraph("Pointwise Job", new ExecutionConfig(), sender, receiver) val cluster = TestingUtils.startTestingCluster(num_tasks) val jmGateway = cluster.getLeaderGateway(1 seconds) @@ -109,8 +110,8 @@ class SlotSharingITCase(_system: ActorSystem) receiver.connectNewDataSetAsInput(sender1, DistributionPattern.POINTWISE) receiver.connectNewDataSetAsInput(sender2, DistributionPattern.ALL_TO_ALL) - val vertices = (List(sender1, sender2, receiver)).asJava - val jobGraph = new JobGraph("Bipartite job", vertices) + val jobGraph = new JobGraph("Bipartite job", new ExecutionConfig(), + sender1, sender2, receiver) val cluster = TestingUtils.startTestingCluster(num_tasks) val jmGateway = cluster.getLeaderGateway(1 seconds) diff --git a/flink-runtime/src/test/scala/org/apache/flink/runtime/jobmanager/TaskManagerFailsWithSlotSharingITCase.scala b/flink-runtime/src/test/scala/org/apache/flink/runtime/jobmanager/TaskManagerFailsWithSlotSharingITCase.scala index 49a1c9535caa4f..c108596ef2280d 100644 --- a/flink-runtime/src/test/scala/org/apache/flink/runtime/jobmanager/TaskManagerFailsWithSlotSharingITCase.scala +++ b/flink-runtime/src/test/scala/org/apache/flink/runtime/jobmanager/TaskManagerFailsWithSlotSharingITCase.scala @@ -20,6 +20,7 @@ package org.apache.flink.runtime.jobmanager import akka.actor.{Kill, ActorSystem, PoisonPill} import akka.testkit.{ImplicitSender, TestKit} +import org.apache.flink.api.common.ExecutionConfig import org.apache.flink.runtime.akka.ListeningBehaviour import org.apache.flink.runtime.client.JobExecutionException import org.apache.flink.runtime.jobgraph.{JobVertex, DistributionPattern, JobGraph} @@ -28,7 +29,6 @@ import org.apache.flink.runtime.jobmanager.scheduler.SlotSharingGroup import org.apache.flink.runtime.messages.JobManagerMessages.{JobResultFailure, JobSubmitSuccess, SubmitJob} import org.apache.flink.runtime.testingUtils.TestingJobManagerMessages._ import org.apache.flink.runtime.testingUtils.{ScalaTestingUtils, TestingUtils} -import org.apache.flink.runtime.util.SerializedThrowable import org.junit.runner.RunWith import org.scalatest.junit.JUnitRunner import org.scalatest.{BeforeAndAfterAll, Matchers, WordSpecLike} @@ -67,7 +67,7 @@ class TaskManagerFailsWithSlotSharingITCase(_system: ActorSystem) sender.setSlotSharingGroup(sharingGroup) receiver.setSlotSharingGroup(sharingGroup) - val jobGraph = new JobGraph("Pointwise Job", sender, receiver) + val jobGraph = new JobGraph("Pointwise Job", new ExecutionConfig(), sender, receiver) val jobID = jobGraph.getJobID val cluster = TestingUtils.startTestingCluster(num_tasks/2, 2) @@ -116,7 +116,7 @@ class TaskManagerFailsWithSlotSharingITCase(_system: ActorSystem) sender.setSlotSharingGroup(sharingGroup) receiver.setSlotSharingGroup(sharingGroup) - val jobGraph = new JobGraph("Pointwise Job", sender, receiver) + val jobGraph = new JobGraph("Pointwise Job", new ExecutionConfig(), sender, receiver) val jobID = jobGraph.getJobID val cluster = TestingUtils.startTestingCluster(num_tasks/2, 2) diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/graph/StreamingJobGraphGenerator.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/graph/StreamingJobGraphGenerator.java index 0dbc146d467068..41f39010487391 100644 --- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/graph/StreamingJobGraphGenerator.java +++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/graph/StreamingJobGraphGenerator.java @@ -22,7 +22,6 @@ import com.google.common.hash.Hashing; import org.apache.commons.lang3.StringUtils; import org.apache.flink.annotation.Internal; -import org.apache.flink.api.common.ExecutionConfig; import org.apache.flink.api.common.operators.util.UserCodeObjectWrapper; import org.apache.flink.api.java.tuple.Tuple2; import org.apache.flink.configuration.Configuration; @@ -49,11 +48,9 @@ import org.apache.flink.streaming.runtime.partitioner.StreamPartitioner; import org.apache.flink.streaming.runtime.tasks.StreamIterationHead; import org.apache.flink.streaming.runtime.tasks.StreamIterationTail; -import org.apache.flink.util.InstantiationUtil; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.IOException; import java.nio.charset.Charset; import java.util.ArrayDeque; import java.util.ArrayList; @@ -120,17 +117,6 @@ public JobGraph createJobGraph(String jobName) { setSlotSharing(); configureCheckpointing(); - - configureExecutionRetries(); - - configureExecutionRetryDelay(); - - try { - InstantiationUtil.writeObjectToConfig(this.streamGraph.getExecutionConfig(), this.jobGraph.getJobConfiguration(), ExecutionConfig.CONFIG_KEY); - } catch (IOException e) { - throw new RuntimeException("Config object could not be written to Job Configuration: ", e); - } - return jobGraph; } @@ -485,16 +471,6 @@ private void configureCheckpointing() { } } - private void configureExecutionRetries() { - int executionRetries = streamGraph.getExecutionConfig().getNumberOfExecutionRetries(); - jobGraph.setNumberOfExecutionRetries(executionRetries); - } - - private void configureExecutionRetryDelay() { - long executionRetryDelay = streamGraph.getExecutionConfig().getExecutionRetryDelay(); - jobGraph.setExecutionRetryDelay(executionRetryDelay); - } - // ------------------------------------------------------------------------ /** diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/graph/StreamingJobGraphGeneratorTest.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/graph/StreamingJobGraphGeneratorTest.java index b5f1e20eae3cdb..a8808ad324130e 100644 --- a/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/graph/StreamingJobGraphGeneratorTest.java +++ b/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/graph/StreamingJobGraphGeneratorTest.java @@ -17,9 +17,6 @@ package org.apache.flink.streaming.api.graph; -import java.io.IOException; -import java.util.Random; - import org.apache.flink.api.common.ExecutionConfig; import org.apache.flink.api.common.functions.MapFunction; import org.apache.flink.api.java.tuple.Tuple2; @@ -27,10 +24,13 @@ import org.apache.flink.streaming.api.datastream.DataStream; import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; import org.apache.flink.streaming.api.functions.sink.SinkFunction; -import org.apache.flink.util.InstantiationUtil; +import org.apache.flink.util.InstantiationUtil; import org.junit.Test; +import java.io.IOException; +import java.util.Random; + import static org.junit.Assert.*; public class StreamingJobGraphGeneratorTest { @@ -44,10 +44,10 @@ public void testExecutionConfigSerialization() throws IOException, ClassNotFound StreamGraph streamingJob = new StreamGraph(env); StreamingJobGraphGenerator compiler = new StreamingJobGraphGenerator(streamingJob); - + boolean closureCleanerEnabled = r.nextBoolean(), forceAvroEnabled = r.nextBoolean(), forceKryoEnabled = r.nextBoolean(), objectReuseEnabled = r.nextBoolean(), sysoutLoggingEnabled = r.nextBoolean(); int dop = 1 + r.nextInt(10); - + ExecutionConfig config = streamingJob.getExecutionConfig(); if(closureCleanerEnabled) { config.enableClosureCleaner(); @@ -75,16 +75,22 @@ public void testExecutionConfigSerialization() throws IOException, ClassNotFound config.disableSysoutLogging(); } config.setParallelism(dop); - + JobGraph jobGraph = compiler.createJobGraph("test"); - + + final String exec_config_key = "runtime.config"; + + InstantiationUtil.writeObjectToConfig(jobGraph.getExecutionConfig(), + jobGraph.getJobConfiguration(), + exec_config_key); + ExecutionConfig executionConfig = InstantiationUtil.readObjectFromConfig( jobGraph.getJobConfiguration(), - ExecutionConfig.CONFIG_KEY, + exec_config_key, Thread.currentThread().getContextClassLoader()); - + assertNotNull(executionConfig); - + assertEquals(closureCleanerEnabled, executionConfig.isClosureCleanerEnabled()); assertEquals(forceAvroEnabled, executionConfig.isForceAvroEnabled()); assertEquals(forceKryoEnabled, executionConfig.isForceKryoEnabled()); diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/partitioner/RescalePartitionerTest.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/partitioner/RescalePartitionerTest.java index bac7fa5edfb8b9..a812cb043c43e4 100644 --- a/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/partitioner/RescalePartitionerTest.java +++ b/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/partitioner/RescalePartitionerTest.java @@ -17,6 +17,7 @@ package org.apache.flink.streaming.runtime.partitioner; +import org.apache.flink.api.common.ExecutionConfig; import org.apache.flink.api.common.JobID; import org.apache.flink.api.common.functions.FlatMapFunction; import org.apache.flink.api.java.tuple.Tuple; @@ -132,7 +133,7 @@ public void flatMap(String value, TestingUtils.defaultExecutionContext(), jobId, jobName, - cfg, + cfg, new ExecutionConfig(), AkkaUtils.getDefaultTimeout(),new ArrayList(), new ArrayList(), ExecutionGraph.class.getClassLoader()); try { eg.attachJobGraph(jobVertices); diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/tasks/StreamMockEnvironment.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/tasks/StreamMockEnvironment.java index dcf6166d81240a..31c05becd2d53a 100644 --- a/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/tasks/StreamMockEnvironment.java +++ b/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/tasks/StreamMockEnvironment.java @@ -19,6 +19,7 @@ package org.apache.flink.streaming.runtime.tasks; import org.apache.flink.api.common.ApplicationID; +import org.apache.flink.api.common.ExecutionConfig; import org.apache.flink.api.common.JobID; import org.apache.flink.api.common.TaskInfo; import org.apache.flink.api.common.typeutils.TypeSerializer; @@ -93,8 +94,10 @@ public class StreamMockEnvironment implements Environment { private final int bufferSize; - public StreamMockEnvironment(Configuration jobConfig, Configuration taskConfig, long memorySize, - MockInputSplitProvider inputSplitProvider, int bufferSize) { + private final ExecutionConfig executionConfig; + + public StreamMockEnvironment(Configuration jobConfig, Configuration taskConfig, ExecutionConfig executionConfig, + long memorySize, MockInputSplitProvider inputSplitProvider, int bufferSize) { this.taskInfo = new TaskInfo("", 0, 1, 0); this.jobConfiguration = jobConfig; this.taskConfiguration = taskConfig; @@ -106,9 +109,15 @@ public StreamMockEnvironment(Configuration jobConfig, Configuration taskConfig, this.inputSplitProvider = inputSplitProvider; this.bufferSize = bufferSize; + this.executionConfig = executionConfig; this.accumulatorRegistry = new AccumulatorRegistry(jobID, getExecutionId()); } + public StreamMockEnvironment(Configuration jobConfig, Configuration taskConfig, long memorySize, + MockInputSplitProvider inputSplitProvider, int bufferSize) { + this(jobConfig, taskConfig, null, memorySize, inputSplitProvider, bufferSize); + } + public void addInputGate(InputGate gate) { inputs.add(gate); } @@ -208,6 +217,11 @@ public IOManager getIOManager() { return this.ioManager; } + @Override + public ExecutionConfig getExecutionConfig() { + return this.executionConfig; + } + @Override public ApplicationID getApplicationID() { return this.appId; diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/tasks/StreamTaskAsyncCheckpointTest.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/tasks/StreamTaskAsyncCheckpointTest.java index 1830054cfa4d1c..ed1dd60cb08585 100644 --- a/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/tasks/StreamTaskAsyncCheckpointTest.java +++ b/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/tasks/StreamTaskAsyncCheckpointTest.java @@ -18,6 +18,7 @@ package org.apache.flink.streaming.runtime.tasks; +import org.apache.flink.api.common.ExecutionConfig; import org.apache.flink.api.common.functions.MapFunction; import org.apache.flink.api.common.typeinfo.BasicTypeInfo; import org.apache.flink.runtime.io.network.api.writer.ResultPartitionWriter; @@ -76,6 +77,11 @@ public void testAsyncCheckpoints() throws Exception { new MockInputSplitProvider(), testHarness.bufferSize) { + @Override + public ExecutionConfig getExecutionConfig() { + return testHarness.executionConfig; + } + @Override public void acknowledgeCheckpoint(long checkpointId) { super.acknowledgeCheckpoint(checkpointId); diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/tasks/StreamTaskTest.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/tasks/StreamTaskTest.java index c18d150b1af80a..51532b0712e06c 100644 --- a/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/tasks/StreamTaskTest.java +++ b/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/tasks/StreamTaskTest.java @@ -21,6 +21,7 @@ import akka.actor.ActorRef; import org.apache.flink.api.common.ApplicationID; +import org.apache.flink.api.common.ExecutionConfig; import org.apache.flink.api.common.JobID; import org.apache.flink.configuration.Configuration; import org.apache.flink.runtime.blob.BlobKey; @@ -135,7 +136,7 @@ private Task createTask(Class invokable, StreamConf TaskDeploymentDescriptor tdd = new TaskDeploymentDescriptor( new ApplicationID(), new JobID(), new JobVertexID(), new ExecutionAttemptID(), - "Test Task", 0, 1, 0, + new ExecutionConfig(), "Test Task", 0, 1, 0, new Configuration(), taskConfig.getConfiguration(), invokable.getName(), diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/tasks/StreamTaskTestHarness.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/tasks/StreamTaskTestHarness.java index e32b3046f6c7c4..9768cf6181a6f4 100644 --- a/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/tasks/StreamTaskTestHarness.java +++ b/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/tasks/StreamTaskTestHarness.java @@ -36,7 +36,6 @@ import org.apache.flink.streaming.runtime.partitioner.BroadcastPartitioner; import org.apache.flink.streaming.runtime.streamrecord.MultiplexingStreamRecordSerializer; import org.apache.flink.streaming.runtime.streamrecord.StreamElement; -import org.apache.flink.util.InstantiationUtil; import org.junit.Assert; import java.io.IOException; @@ -104,11 +103,6 @@ public StreamTaskTestHarness(AbstractInvokable task, TypeInformation output this.taskConfig = new Configuration(); this.executionConfig = new ExecutionConfig(); executionConfig.enableTimestamps(); - try { - InstantiationUtil.writeObjectToConfig(executionConfig, this.jobConfig, ExecutionConfig.CONFIG_KEY); - } catch (IOException e) { - throw new RuntimeException(e); - } streamConfig = new StreamConfig(taskConfig); streamConfig.setChainStart(); @@ -154,7 +148,8 @@ private void initializeOutput() { * Task thread to finish running. */ public void invoke() throws Exception { - mockEnv = new StreamMockEnvironment(jobConfig, taskConfig, memorySize, new MockInputSplitProvider(), bufferSize); + mockEnv = new StreamMockEnvironment(jobConfig, taskConfig, executionConfig, + memorySize, new MockInputSplitProvider(), bufferSize); task.setEnvironment(mockEnv); initializeInputs(); diff --git a/flink-tests/src/test/java/org/apache/flink/test/failingPrograms/JobSubmissionFailsITCase.java b/flink-tests/src/test/java/org/apache/flink/test/failingPrograms/JobSubmissionFailsITCase.java index 28c2e5837fc954..f6c22d439f958a 100644 --- a/flink-tests/src/test/java/org/apache/flink/test/failingPrograms/JobSubmissionFailsITCase.java +++ b/flink-tests/src/test/java/org/apache/flink/test/failingPrograms/JobSubmissionFailsITCase.java @@ -18,6 +18,7 @@ package org.apache.flink.test.failingPrograms; +import org.apache.flink.api.common.ExecutionConfig; import org.apache.flink.api.common.JobExecutionResult; import org.apache.flink.configuration.ConfigConstants; import org.apache.flink.configuration.Configuration; @@ -64,7 +65,7 @@ public static void setup() { final JobVertex jobVertex = new JobVertex("Working job vertex."); jobVertex.setInvokableClass(Tasks.NoOpInvokable.class); - workingJobGraph = new JobGraph("Working testing job", jobVertex); + workingJobGraph = new JobGraph("Working testing job", new ExecutionConfig(), jobVertex); } catch (Exception e) { e.printStackTrace(); @@ -115,7 +116,7 @@ public void testExceptionInInitializeOnMaster() { final JobVertex failingJobVertex = new FailingJobVertex("Failing job vertex"); failingJobVertex.setInvokableClass(Tasks.NoOpInvokable.class); - final JobGraph failingJobGraph = new JobGraph("Failing testing job", failingJobVertex); + final JobGraph failingJobGraph = new JobGraph("Failing testing job", new ExecutionConfig(), failingJobVertex); try { submitJob(failingJobGraph); @@ -140,7 +141,7 @@ public void testExceptionInInitializeOnMaster() { @Test public void testSubmitEmptyJobGraph() { try { - final JobGraph jobGraph = new JobGraph("Testing job"); + final JobGraph jobGraph = new JobGraph("Testing job", new ExecutionConfig()); try { submitJob(jobGraph); diff --git a/flink-tests/src/test/java/org/apache/flink/test/javaApiOperators/MapITCase.java b/flink-tests/src/test/java/org/apache/flink/test/javaApiOperators/MapITCase.java index e5a494b2f02724..c1bd5e2fdd8ef5 100644 --- a/flink-tests/src/test/java/org/apache/flink/test/javaApiOperators/MapITCase.java +++ b/flink-tests/src/test/java/org/apache/flink/test/javaApiOperators/MapITCase.java @@ -68,6 +68,39 @@ public void testIdentityMapWithBasicType() throws Exception { compareResultAsText(result, expected); } + @Test + public void testRuntimeContextAndExecutionConfigParams() throws Exception { + /* + * Test identity map with basic type + */ + + final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment(); + env.getConfig().setNumberOfExecutionRetries(1000); + + DataSet ds = CollectionDataSets.getStringDataSet(env); + DataSet identityMapDs = ds. + map(new RichMapFunction() { + @Override + public String map(String value) throws Exception { + Assert.assertTrue(1000 == getRuntimeContext().getExecutionConfig().getNumberOfExecutionRetries()); + return value; + } + }); + + List result = identityMapDs.collect(); + + String expected = "Hi\n" + + "Hello\n" + + "Hello world\n" + + "Hello world, how are you?\n" + + "I am fine.\n" + + "Luke Skywalker\n" + + "Random comment\n" + + "LOL\n"; + + compareResultAsText(result, expected); + } + public static class Mapper1 implements MapFunction { private static final long serialVersionUID = 1L; diff --git a/flink-tests/src/test/java/org/apache/flink/test/recovery/JobManagerCheckpointRecoveryITCase.java b/flink-tests/src/test/java/org/apache/flink/test/recovery/JobManagerCheckpointRecoveryITCase.java index ea30c58e3ae42d..bc7a66a982e7eb 100644 --- a/flink-tests/src/test/java/org/apache/flink/test/recovery/JobManagerCheckpointRecoveryITCase.java +++ b/flink-tests/src/test/java/org/apache/flink/test/recovery/JobManagerCheckpointRecoveryITCase.java @@ -21,6 +21,7 @@ import akka.actor.ActorRef; import akka.actor.ActorSystem; import org.apache.commons.io.FileUtils; +import org.apache.flink.api.common.ExecutionConfig; import org.apache.flink.configuration.ConfigConstants; import org.apache.flink.configuration.Configuration; import org.apache.flink.runtime.akka.AkkaUtils; @@ -366,7 +367,7 @@ public void testCheckpointRecoveryFailure() throws Exception { // BLocking JobGraph JobVertex blockingVertex = new JobVertex("Blocking vertex"); blockingVertex.setInvokableClass(Tasks.BlockingNoOpInvokable.class); - JobGraph jobGraph = new JobGraph(blockingVertex); + JobGraph jobGraph = new JobGraph(new ExecutionConfig(), blockingVertex); // Submit the job in detached mode leader.tell(new SubmitJob(jobGraph, ListeningBehaviour.DETACHED)); diff --git a/flink-tests/src/test/java/org/apache/flink/test/runtime/NetworkStackThroughputITCase.java b/flink-tests/src/test/java/org/apache/flink/test/runtime/NetworkStackThroughputITCase.java index aada3647ced63c..a87057845b7dac 100644 --- a/flink-tests/src/test/java/org/apache/flink/test/runtime/NetworkStackThroughputITCase.java +++ b/flink-tests/src/test/java/org/apache/flink/test/runtime/NetworkStackThroughputITCase.java @@ -18,6 +18,7 @@ package org.apache.flink.test.runtime; +import org.apache.flink.api.common.ExecutionConfig; import org.apache.flink.configuration.Configuration; import org.apache.flink.core.io.IOReadableWritable; import org.apache.flink.core.memory.DataInputView; @@ -96,7 +97,7 @@ protected JobGraph getJobGraph() throws Exception { private JobGraph createJobGraph(int dataVolumeGb, boolean useForwarder, boolean isSlowSender, boolean isSlowReceiver, int numSubtasks) { - JobGraph jobGraph = new JobGraph("Speed Test"); + JobGraph jobGraph = new JobGraph("Speed Test", new ExecutionConfig()); SlotSharingGroup sharingGroup = new SlotSharingGroup(); JobVertex producer = new JobVertex("Speed Test Producer"); diff --git a/flink-tests/src/test/java/org/apache/flink/test/runtime/leaderelection/ZooKeeperLeaderElectionITCase.java b/flink-tests/src/test/java/org/apache/flink/test/runtime/leaderelection/ZooKeeperLeaderElectionITCase.java index 68b1d5aaba57c4..a92cc3ac419467 100644 --- a/flink-tests/src/test/java/org/apache/flink/test/runtime/leaderelection/ZooKeeperLeaderElectionITCase.java +++ b/flink-tests/src/test/java/org/apache/flink/test/runtime/leaderelection/ZooKeeperLeaderElectionITCase.java @@ -22,6 +22,7 @@ import akka.actor.Kill; import akka.actor.PoisonPill; import org.apache.commons.io.FileUtils; +import org.apache.flink.api.common.ExecutionConfig; import org.apache.flink.api.common.JobExecutionResult; import org.apache.flink.configuration.ConfigConstants; import org.apache.flink.configuration.Configuration; @@ -167,7 +168,7 @@ public void testJobExecutionOnClusterWithLeaderReelection() throws Exception { sender.setSlotSharingGroup(slotSharingGroup); receiver.setSlotSharingGroup(slotSharingGroup); - final JobGraph graph = new JobGraph("Blocking test job", sender, receiver); + final JobGraph graph = new JobGraph("Blocking test job", new ExecutionConfig(), sender, receiver); final ForkableFlinkMiniCluster cluster = new ForkableFlinkMiniCluster(configuration); diff --git a/flink-tests/src/test/scala/org/apache/flink/api/scala/runtime/jobmanager/JobManagerFailsITCase.scala b/flink-tests/src/test/scala/org/apache/flink/api/scala/runtime/jobmanager/JobManagerFailsITCase.scala index 6aea26cef7ebd5..f8512ced5cdf7c 100644 --- a/flink-tests/src/test/scala/org/apache/flink/api/scala/runtime/jobmanager/JobManagerFailsITCase.scala +++ b/flink-tests/src/test/scala/org/apache/flink/api/scala/runtime/jobmanager/JobManagerFailsITCase.scala @@ -20,6 +20,7 @@ package org.apache.flink.api.scala.runtime.jobmanager import akka.actor.{ActorSystem, PoisonPill} import akka.testkit.{ImplicitSender, TestKit} +import org.apache.flink.api.common.ExecutionConfig import org.junit.runner.RunWith @@ -94,12 +95,12 @@ class JobManagerFailsITCase(_system: ActorSystem) val sender = new JobVertex("BlockingSender") sender.setParallelism(num_slots) sender.setInvokableClass(classOf[BlockingNoOpInvokable]) - val jobGraph = new JobGraph("Blocking Testjob", sender) + val jobGraph = new JobGraph("Blocking Testjob", new ExecutionConfig(), sender) val noOp = new JobVertex("NoOpInvokable") noOp.setParallelism(num_slots) noOp.setInvokableClass(classOf[NoOpInvokable]) - val jobGraph2 = new JobGraph("NoOp Testjob", noOp) + val jobGraph2 = new JobGraph("NoOp Testjob", new ExecutionConfig(), noOp) val cluster = startDeathwatchCluster(num_slots / 2, 2) diff --git a/flink-tests/src/test/scala/org/apache/flink/api/scala/runtime/jobmanager/JobManagerLeaderSessionIDITSuite.scala b/flink-tests/src/test/scala/org/apache/flink/api/scala/runtime/jobmanager/JobManagerLeaderSessionIDITSuite.scala index 09af4307bcfb29..9aa1e944682a42 100644 --- a/flink-tests/src/test/scala/org/apache/flink/api/scala/runtime/jobmanager/JobManagerLeaderSessionIDITSuite.scala +++ b/flink-tests/src/test/scala/org/apache/flink/api/scala/runtime/jobmanager/JobManagerLeaderSessionIDITSuite.scala @@ -23,6 +23,7 @@ import java.util.UUID import akka.actor.ActorSystem import akka.actor.Status.Success import akka.testkit.{ImplicitSender, TestKit} +import org.apache.flink.api.common.ExecutionConfig import org.apache.flink.runtime.akka.{ListeningBehaviour, AkkaUtils} import org.apache.flink.runtime.jobgraph.tasks.AbstractInvokable import org.apache.flink.runtime.jobgraph.{JobGraph, JobVertex} @@ -64,7 +65,7 @@ class JobManagerLeaderSessionIDITSuite(_system: ActorSystem) val sender = new JobVertex("BlockingSender"); sender.setParallelism(numSlots) sender.setInvokableClass(classOf[BlockingUntilSignalNoOpInvokable]) - val jobGraph = new JobGraph("TestJob", sender) + val jobGraph = new JobGraph("TestJob", new ExecutionConfig(), sender) val oldSessionID = UUID.randomUUID() diff --git a/flink-tests/src/test/scala/org/apache/flink/api/scala/runtime/taskmanager/TaskManagerFailsITCase.scala b/flink-tests/src/test/scala/org/apache/flink/api/scala/runtime/taskmanager/TaskManagerFailsITCase.scala index 869af82ed36df1..88d760d17ac9a3 100644 --- a/flink-tests/src/test/scala/org/apache/flink/api/scala/runtime/taskmanager/TaskManagerFailsITCase.scala +++ b/flink-tests/src/test/scala/org/apache/flink/api/scala/runtime/taskmanager/TaskManagerFailsITCase.scala @@ -18,9 +18,9 @@ package org.apache.flink.api.scala.runtime.taskmanager -import akka.actor.Status.{Failure, Success} import akka.actor.{ActorSystem, Kill, PoisonPill} import akka.testkit.{ImplicitSender, TestKit} +import org.apache.flink.api.common.ExecutionConfig import org.apache.flink.configuration.ConfigConstants import org.apache.flink.configuration.Configuration @@ -33,7 +33,6 @@ import org.apache.flink.runtime.messages.TaskManagerMessages.{RegisteredAtJobMan import org.apache.flink.runtime.testingUtils.TestingJobManagerMessages._ import org.apache.flink.runtime.testingUtils.TestingMessages.DisableDisconnect import org.apache.flink.runtime.testingUtils.{ScalaTestingUtils, TestingUtils} -import org.apache.flink.runtime.util.SerializedThrowable import org.apache.flink.test.util.ForkableFlinkMiniCluster import org.junit.runner.RunWith @@ -100,7 +99,7 @@ class TaskManagerFailsITCase(_system: ActorSystem) receiver.setParallelism(num_tasks) receiver.connectNewDataSetAsInput(sender, DistributionPattern.POINTWISE) - val jobGraph = new JobGraph("Pointwise Job", sender, receiver) + val jobGraph = new JobGraph("Pointwise Job", new ExecutionConfig(), sender, receiver) val jobID = jobGraph.getJobID val cluster = ForkableFlinkMiniCluster.startCluster(num_tasks, 2) @@ -152,7 +151,7 @@ class TaskManagerFailsITCase(_system: ActorSystem) receiver.setParallelism(num_tasks) receiver.connectNewDataSetAsInput(sender, DistributionPattern.POINTWISE) - val jobGraph = new JobGraph("Pointwise Job", sender, receiver) + val jobGraph = new JobGraph("Pointwise Job", new ExecutionConfig(), sender, receiver) val jobID = jobGraph.getJobID val cluster = ForkableFlinkMiniCluster.startCluster(num_tasks, 2) @@ -191,12 +190,12 @@ class TaskManagerFailsITCase(_system: ActorSystem) val sender = new JobVertex("BlockingSender") sender.setParallelism(num_slots) sender.setInvokableClass(classOf[BlockingNoOpInvokable]) - val jobGraph = new JobGraph("Blocking Testjob", sender) + val jobGraph = new JobGraph("Blocking Testjob", new ExecutionConfig(), sender) val noOp = new JobVertex("NoOpInvokable") noOp.setParallelism(num_slots) noOp.setInvokableClass(classOf[NoOpInvokable]) - val jobGraph2 = new JobGraph("NoOp Testjob", noOp) + val jobGraph2 = new JobGraph("NoOp Testjob", new ExecutionConfig(), noOp) val cluster = createDeathwatchCluster(num_slots/2, 2) diff --git a/flink-yarn-tests/src/main/java/org/apache/flink/yarn/YarnTestBase.java b/flink-yarn-tests/src/main/java/org/apache/flink/yarn/YarnTestBase.java index d3132d7717b375..2a759ec1fba2bd 100644 --- a/flink-yarn-tests/src/main/java/org/apache/flink/yarn/YarnTestBase.java +++ b/flink-yarn-tests/src/main/java/org/apache/flink/yarn/YarnTestBase.java @@ -112,7 +112,7 @@ public abstract class YarnTestBase extends TestLogger { * lib/ folder of the flink distribution. */ protected static File flinkLibFolder; - + static { yarnConfiguration = new YarnConfiguration(); yarnConfiguration.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, 512); From 43dc592a7ac5393021b7fd0409a2b332853a2aa8 Mon Sep 17 00:00:00 2001 From: Kostas Kloudas Date: Tue, 9 Feb 2016 15:48:00 +0100 Subject: [PATCH 3/3] FLINK-2523: Makes the task cancellation interval configurable through the ExecutionConfig. --- docs/apis/batch/index.md | 2 +- .../flink/api/common/ExecutionConfig.java | 31 +++++++++++++++++-- .../flink/configuration/ConfigConstants.java | 5 +++ .../flink/runtime/taskmanager/Task.java | 13 +++++--- .../test/javaApiOperators/MapITCase.java | 2 ++ 5 files changed, 46 insertions(+), 7 deletions(-) diff --git a/docs/apis/batch/index.md b/docs/apis/batch/index.md index 28184905d70192..71694db9ddf59a 100644 --- a/docs/apis/batch/index.md +++ b/docs/apis/batch/index.md @@ -2123,7 +2123,7 @@ Note that types registered with `registerKryoType()` are not available to Flink' - `disableAutoTypeRegistration()` Automatic type registration is enabled by default. The automatic type registration is registering all types (including sub-types) used by usercode with Kryo and the POJO serializer. - +- `setTaskCancellationInterval(long interval)` Sets the the interval (in milliseconds) to wait between consecutive attempts to cancel a running task. When a task is canceled, the cancellation calls periodically `interrupt()` on the task thread, if the task thread does not terminate within a certain time. This parameter refers to the time between consecutive calls to `interrupt()` and is set by default to **30000** milliseconds, or **30 seconds**. The `RuntimeContext` which is accessible in `Rich*` functions through the `getRuntimeContext()` method also allows to access the `ExecutionConfig` in all user defined functions. diff --git a/flink-core/src/main/java/org/apache/flink/api/common/ExecutionConfig.java b/flink-core/src/main/java/org/apache/flink/api/common/ExecutionConfig.java index 24742a3381485d..ccdf4770266ed3 100644 --- a/flink-core/src/main/java/org/apache/flink/api/common/ExecutionConfig.java +++ b/flink-core/src/main/java/org/apache/flink/api/common/ExecutionConfig.java @@ -21,6 +21,7 @@ import com.esotericsoftware.kryo.Serializer; import org.apache.flink.annotation.PublicEvolving; import org.apache.flink.annotation.Public; +import org.apache.flink.configuration.ConfigConstants; import java.io.Serializable; import java.util.LinkedHashMap; @@ -96,6 +97,8 @@ public class ExecutionConfig implements Serializable { private long executionRetryDelay = -1; + private long taskCancellationIntervalMillis = ConfigConstants.DEFAULT_TASK_CANCELLATION_INTERVAL_MILLIS; + // Serializers and types registered with Kryo and the PojoSerializer // we store them in linked maps/sets to ensure they are registered in order in all kryo instances. @@ -241,6 +244,28 @@ public ExecutionConfig setParallelism(int parallelism) { return this; } + /** + * Gets the interval (in milliseconds) between consecutive attempts to cancel a running task. + */ + public long getTaskCancellationInterval() { + return this.taskCancellationIntervalMillis; + } + + /** + * Sets the configuration parameter specifying the interval (in milliseconds) + * between consecutive attempts to cancel a running task. + * @param interval the interval (in milliseconds). + */ + public ExecutionConfig setTaskCancellationInterval(long interval) { + if(interval < 0) { + throw new IllegalArgumentException( + "The task cancellation interval cannot be negative." + ); + } + this.taskCancellationIntervalMillis = interval; + return this; + } + /** * Gets the number of times the system will try to re-execute failed tasks. A value * of {@code -1} indicates that the system default value (as defined in the configuration) @@ -624,7 +649,8 @@ public boolean equals(Object obj) { registeredTypesWithKryoSerializerClasses.equals(other.registeredTypesWithKryoSerializerClasses) && defaultKryoSerializerClasses.equals(other.defaultKryoSerializerClasses) && registeredKryoTypes.equals(other.registeredKryoTypes) && - registeredPojoTypes.equals(other.registeredPojoTypes); + registeredPojoTypes.equals(other.registeredPojoTypes) && + taskCancellationIntervalMillis == other.taskCancellationIntervalMillis; } else { return false; @@ -650,7 +676,8 @@ public int hashCode() { registeredTypesWithKryoSerializerClasses, defaultKryoSerializerClasses, registeredKryoTypes, - registeredPojoTypes); + registeredPojoTypes, + taskCancellationIntervalMillis); } public boolean canEqual(Object obj) { diff --git a/flink-core/src/main/java/org/apache/flink/configuration/ConfigConstants.java b/flink-core/src/main/java/org/apache/flink/configuration/ConfigConstants.java index 2b756449d53d45..259ae377795e8c 100644 --- a/flink-core/src/main/java/org/apache/flink/configuration/ConfigConstants.java +++ b/flink-core/src/main/java/org/apache/flink/configuration/ConfigConstants.java @@ -612,6 +612,11 @@ public final class ConfigConstants { */ public static final boolean DEFAULT_TASK_MANAGER_MEMORY_PRE_ALLOCATE = false; + /** + * The default interval (in milliseconds) to wait between consecutive task cancellation attempts (= 30000 msec). + * */ + public static final long DEFAULT_TASK_CANCELLATION_INTERVAL_MILLIS = 30000; + // ------------------------ Runtime Algorithms ------------------------ /** diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/taskmanager/Task.java b/flink-runtime/src/main/java/org/apache/flink/runtime/taskmanager/Task.java index b2e99b2ae26e10..df2d78a126171e 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/taskmanager/Task.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/taskmanager/Task.java @@ -810,7 +810,9 @@ else if (current == ExecutionState.RUNNING) { // because the canceling may block on user code, we cancel from a separate thread // we do not reuse the async call handler, because that one may be blocked, in which // case the canceling could not continue - Runnable canceler = new TaskCanceler(LOG, invokable, executingThread, taskNameWithSubtask); + long taskCancellationInterval = this.executionConfig.getTaskCancellationInterval(); + Runnable canceler = new TaskCanceler(LOG, invokable, executingThread, taskNameWithSubtask, + taskCancellationInterval); Thread cancelThread = new Thread(executingThread.getThreadGroup(), canceler, "Canceler for " + taskNameWithSubtask); cancelThread.setDaemon(true); @@ -1054,12 +1056,15 @@ private static class TaskCanceler implements Runnable { private final AbstractInvokable invokable; private final Thread executer; private final String taskName; + private final long taskCancellationIntervalMillis; - public TaskCanceler(Logger logger, AbstractInvokable invokable, Thread executer, String taskName) { + public TaskCanceler(Logger logger, AbstractInvokable invokable, + Thread executer, String taskName, long cancelationInterval) { this.logger = logger; this.invokable = invokable; this.executer = executer; this.taskName = taskName; + this.taskCancellationIntervalMillis = cancelationInterval; } @Override @@ -1077,7 +1082,7 @@ public void run() { // interrupt the running thread initially executer.interrupt(); try { - executer.join(30000); + executer.join(taskCancellationIntervalMillis); } catch (InterruptedException e) { // we can ignore this @@ -1100,7 +1105,7 @@ public void run() { executer.interrupt(); try { - executer.join(30000); + executer.join(taskCancellationIntervalMillis); } catch (InterruptedException e) { // we can ignore this diff --git a/flink-tests/src/test/java/org/apache/flink/test/javaApiOperators/MapITCase.java b/flink-tests/src/test/java/org/apache/flink/test/javaApiOperators/MapITCase.java index c1bd5e2fdd8ef5..fb3e5890ae7cc3 100644 --- a/flink-tests/src/test/java/org/apache/flink/test/javaApiOperators/MapITCase.java +++ b/flink-tests/src/test/java/org/apache/flink/test/javaApiOperators/MapITCase.java @@ -76,6 +76,7 @@ public void testRuntimeContextAndExecutionConfigParams() throws Exception { final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment(); env.getConfig().setNumberOfExecutionRetries(1000); + env.getConfig().setTaskCancellationInterval(50000); DataSet ds = CollectionDataSets.getStringDataSet(env); DataSet identityMapDs = ds. @@ -83,6 +84,7 @@ public void testRuntimeContextAndExecutionConfigParams() throws Exception { @Override public String map(String value) throws Exception { Assert.assertTrue(1000 == getRuntimeContext().getExecutionConfig().getNumberOfExecutionRetries()); + Assert.assertTrue(50000 == getRuntimeContext().getExecutionConfig().getTaskCancellationInterval()); return value; } });