Skip to content

Commit

Permalink
Add some docs
Browse files Browse the repository at this point in the history
  • Loading branch information
zsxwing committed Feb 3, 2015
1 parent a0e8509 commit 0b9017f
Show file tree
Hide file tree
Showing 2 changed files with 7 additions and 0 deletions.
1 change: 1 addition & 0 deletions core/src/main/scala/org/apache/spark/SparkContext.scala
Original file line number Diff line number Diff line change
Expand Up @@ -1747,6 +1747,7 @@ object SparkContext extends Logging {
rdd: RDD[(K, V)]) = {
val kf = implicitly[K => Writable]
val vf = implicitly[V => Writable]
// Set the Writable class to null and `SequenceFileRDDFunctions` will use Reflection to get it
implicit val keyWritableFactory = new WritableFactory[K](_ => null, kf)
implicit val valueWritableFactory = new WritableFactory[V](_ => null, vf)
RDD.rddToSequenceFileRDDFunctions(rdd)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -45,13 +45,15 @@ class SequenceFileRDDFunctions[K <% Writable: ClassTag, V <% Writable : ClassTag

private val keyWritableClass =
if (_keyWritableClass == null) {
// pre 1.3.0, we need to use Reflection to get the Writable class
getWritableClass[K]()
} else {
_keyWritableClass
}

private val valueWritableClass =
if (_valueWritableClass == null) {
// pre 1.3.0, we need to use Reflection to get the Writable class
getWritableClass[V]()
} else {
_valueWritableClass
Expand Down Expand Up @@ -86,6 +88,10 @@ class SequenceFileRDDFunctions[K <% Writable: ClassTag, V <% Writable : ClassTag
def saveAsSequenceFile(path: String, codec: Option[Class[_ <: CompressionCodec]] = None) {
def anyToWritable[U <% Writable](u: U): Writable = u

// TODO We cannot force the return type of `anyToWritable` be same as keyWritableClass and
// valueWritableClass at the compile time. To implement that, we need to add type parameters to
// SequenceFileRDDFunctions. however, SequenceFileRDDFunctions is a public class so it will be a
// breaking change.
val convertKey = self.keyClass != keyWritableClass
val convertValue = self.valueClass != valueWritableClass

Expand Down

0 comments on commit 0b9017f

Please sign in to comment.