Skip to content

Commit

Permalink
[GAE-Java] GRAPE JDK support Spark local mode for graphx (#2216)
Browse files Browse the repository at this point in the history
  • Loading branch information
zhanglei1949 committed Nov 10, 2022
1 parent ed33b08 commit f21754f
Showing 1 changed file with 20 additions and 1 deletion.
Expand Up @@ -42,9 +42,10 @@ object ExecutorInfoHelper extends Logging {
val executorDataMap = getExecutorDataMapReflect(field, coarseGrainedSchedulerBackend)
executorDatMapToId2Host(executorDataMap)
}
// For local mode, there will be only one executor, i.e. driver & executor.
case local: LocalSchedulerBackend => {
val res = new mutable.HashMap[String, String]()
res.+=((InetAddress.getLocalHost.getHostName, "0"))
res.+=(("0", InetAddress.getLocalHost.getHostName))
res
}
}
Expand All @@ -66,6 +67,13 @@ object ExecutorInfoHelper extends Logging {
executorDataMapToHost2Id(executorDataMap)
}

// For local mode, there will be only one executor, i.e. driver & executor.
case local: LocalSchedulerBackend => {
val res = new mutable.HashMap[String, ArrayBuffer[String]]()
res.+=((InetAddress.getLocalHost.getHostName, new ArrayBuffer[String].+=("driver")))
res
}

case _ => throw new IllegalStateException("Unsupported backend")
}
}
Expand All @@ -87,6 +95,13 @@ object ExecutorInfoHelper extends Logging {
executorDataMap.map(t => (t._1, t._2.freeCores))
}

case localSchedulerBackend: LocalSchedulerBackend => {
val numCores = getNumCoresFromLocalBackend(localSchedulerBackend)
val res = new mutable.HashMap[String, Int]
res.+=(("driver", numCores))
res
}

case _ => throw new IllegalStateException("Unsupported backend")
}
}
Expand All @@ -107,6 +122,10 @@ object ExecutorInfoHelper extends Logging {
field
}

def getNumCoresFromLocalBackend(backend: LocalSchedulerBackend): Int = {
backend.totalCores
}

def getExecutorDataMapReflect(
field: Field,
castedBackend: CoarseGrainedSchedulerBackend
Expand Down

0 comments on commit f21754f

Please sign in to comment.