Skip to content

Commit

Permalink
Restore "null" as default for RDD name
Browse files Browse the repository at this point in the history
Instead, RDDInfo's name field will default to the RDD class name
if the RDD's name is not defined. This avoids changing RDD#toString
behavior.
  • Loading branch information
Andrew Or committed May 4, 2015
1 parent 5fdd89d commit 3ffe566
Show file tree
Hide file tree
Showing 4 changed files with 5 additions and 5 deletions.
2 changes: 1 addition & 1 deletion core/src/main/scala/org/apache/spark/SparkContext.scala
Original file line number Diff line number Diff line change
Expand Up @@ -898,7 +898,7 @@ class SparkContext(config: SparkConf) extends Logging with ExecutorAllocationCli
inputFormatClass,
keyClass,
valueClass,
minPartitions).setName(s"HadoopRDD[$path]")
minPartitions).setName(path)
}

/**
Expand Down
2 changes: 1 addition & 1 deletion core/src/main/scala/org/apache/spark/rdd/RDD.scala
Original file line number Diff line number Diff line change
Expand Up @@ -140,7 +140,7 @@ abstract class RDD[T: ClassTag](
val id: Int = sc.newRddId()

/** A friendly name for this RDD */
@transient var name: String = Utils.getFormattedClassName(this)
@transient var name: String = null

/** Assign a name to this RDD */
def setName(_name: String): this.type = {
Expand Down
2 changes: 1 addition & 1 deletion core/src/main/scala/org/apache/spark/storage/RDDInfo.scala
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,7 @@ class RDDInfo(

private[spark] object RDDInfo {
def fromRdd(rdd: RDD[_]): RDDInfo = {
val rddName = Option(rdd.name).getOrElse(rdd.id.toString)
val rddName = Option(rdd.name).getOrElse(Utils.getFormattedClassName(rdd))
val parentIds = rdd.dependencies.map(_.rdd.id)
new RDDInfo(rdd.id, rddName, rdd.partitions.length, rdd.getStorageLevel, parentIds, rdd.scope)
}
Expand Down
4 changes: 2 additions & 2 deletions python/pyspark/sql/dataframe.py
Original file line number Diff line number Diff line change
Expand Up @@ -244,8 +244,8 @@ def explain(self, extended=False):
:param extended: boolean, default ``False``. If ``False``, prints only the physical plan.
>>> df.explain()
PhysicalRDD [age#0,name#1], MapPartitionsRDD MapPartitionsRDD[7] at applySchemaToPythonRDD\
at NativeMethodAccessorImpl.java:...
PhysicalRDD [age#0,name#1], MapPartitionsRDD[...] at applySchemaToPythonRDD at\
NativeMethodAccessorImpl.java:...
>>> df.explain(True)
== Parsed Logical Plan ==
Expand Down

0 comments on commit 3ffe566

Please sign in to comment.