Skip to content
Switch branches/tags

Name already in use

A tag already exists with the provided branch name. Many Git commands accept both tag and branch names, so creating this branch may cause unexpected behavior. Are you sure you want to create this branch?
Go to file
Cannot retrieve contributors at this time
package com.madhukaraphatak.spark.barrier
import org.apache.spark.sql.SparkSession
object BarrierExceptionExample {
def main(args: Array[String]): Unit = {
val sparkSession = SparkSession.builder.
val df = sparkSession.range(0,100).repartition(4)
val barrierRdd = df.rdd.barrier()
//fails running as it needs minimum four cores for four partitions
val count = barrierRdd.mapPartitions(v => v).count()
println("count is " + count)