remove unneeded assert, cache the converter

twitter · May 14, 2015 · 925291c · 925291c
1 parent a83b9ff
commit 925291c
Show file tree

Hide file tree

Showing 2 changed files with 12 additions and 4 deletions.
diff --git a/scalding-commons/src/main/scala/com/twitter/scalding/commons/source/LzoGenericScheme.scala b/scalding-commons/src/main/scala/com/twitter/scalding/commons/source/LzoGenericScheme.scala
@@ -64,12 +64,20 @@ private[source] object ConfigBinaryConverterProvider {
  */
 private[source] class ConfigBinaryConverterProvider[M] extends BinaryConverterProvider[M] {
   import ConfigBinaryConverterProvider._
+
+  private[this] var cached: Option[(String, BinaryConverter[M])] = None
+
   override def getConverter(conf: Configuration): BinaryConverter[M] = {
     val data = conf.get(ProviderConfKey)
     require(data != null, s"$ProviderConfKey is not set in configuration")
-
-    val extern = ExternalizerSerializer.inj.invert(data).get
-    extern.get.asInstanceOf[BinaryConverter[M]]
+    cached match {
+      case Some((data, conv)) => conv
+      case _ =>
+        val extern = ExternalizerSerializer.inj.invert(data).get
+        val conv = extern.get.asInstanceOf[BinaryConverter[M]]
+        cached = Some((data, conv))
+        conv
+    }
   }
 }
 

diff --git a/scalding-commons/src/main/scala/com/twitter/scalding/commons/source/LzoGenericSource.scala b/scalding-commons/src/main/scala/com/twitter/scalding/commons/source/LzoGenericSource.scala
@@ -37,6 +37,6 @@ object LzoGenericSource {
     new LzoGenericSource[T] {
       override val conv: BinaryConverter[T] = passedConv
       override val hdfsPaths = paths
-      override val localPaths = { assert(paths.size == 1, "Cannot use multiple input files on local mode"); paths }
+      override val localPaths = paths
     }
 }