diff --git a/client-spark/spark-2/src/main/java/org/apache/spark/shuffle/celeborn/SparkShuffleManager.java b/client-spark/spark-2/src/main/java/org/apache/spark/shuffle/celeborn/SparkShuffleManager.java index 96d28acf4f8..52f383905b1 100644 --- a/client-spark/spark-2/src/main/java/org/apache/spark/shuffle/celeborn/SparkShuffleManager.java +++ b/client-spark/spark-2/src/main/java/org/apache/spark/shuffle/celeborn/SparkShuffleManager.java @@ -117,15 +117,16 @@ public ShuffleHandle registerShuffle( String appId = SparkUtils.appUniqueId(dependency.rdd().context()); initializeLifecycleManager(appId); - lifecycleManager.registerAppShuffleDeterminate( - shuffleId, - !DeterministicLevel.INDETERMINATE().equals(dependency.rdd().getOutputDeterministicLevel())); - if (fallbackPolicyRunner.applyFallbackPolicies(dependency, lifecycleManager)) { logger.warn("Fallback to SortShuffleManager!"); sortShuffleIds.add(shuffleId); return sortShuffleManager().registerShuffle(shuffleId, numMaps, dependency); } else { + lifecycleManager.registerAppShuffleDeterminate( + shuffleId, + !DeterministicLevel.INDETERMINATE() + .equals(dependency.rdd().getOutputDeterministicLevel())); + return new CelebornShuffleHandle<>( appUniqueId, lifecycleManager.getHost(),