From 925291cf3c75b4aa4eecdaa53809d8a8c87f6c68 Mon Sep 17 00:00:00 2001 From: Ruban Monu Date: Thu, 14 May 2015 11:40:59 -0700 Subject: [PATCH] remove unneeded assert, cache the converter --- .../scalding/commons/source/LzoGenericScheme.scala | 14 +++++++++++--- .../scalding/commons/source/LzoGenericSource.scala | 2 +- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/scalding-commons/src/main/scala/com/twitter/scalding/commons/source/LzoGenericScheme.scala b/scalding-commons/src/main/scala/com/twitter/scalding/commons/source/LzoGenericScheme.scala index 2e3d882d6b..1ed2c6ad59 100644 --- a/scalding-commons/src/main/scala/com/twitter/scalding/commons/source/LzoGenericScheme.scala +++ b/scalding-commons/src/main/scala/com/twitter/scalding/commons/source/LzoGenericScheme.scala @@ -64,12 +64,20 @@ private[source] object ConfigBinaryConverterProvider { */ private[source] class ConfigBinaryConverterProvider[M] extends BinaryConverterProvider[M] { import ConfigBinaryConverterProvider._ + + private[this] var cached: Option[(String, BinaryConverter[M])] = None + override def getConverter(conf: Configuration): BinaryConverter[M] = { val data = conf.get(ProviderConfKey) require(data != null, s"$ProviderConfKey is not set in configuration") - - val extern = ExternalizerSerializer.inj.invert(data).get - extern.get.asInstanceOf[BinaryConverter[M]] + cached match { + case Some((data, conv)) => conv + case _ => + val extern = ExternalizerSerializer.inj.invert(data).get + val conv = extern.get.asInstanceOf[BinaryConverter[M]] + cached = Some((data, conv)) + conv + } } } diff --git a/scalding-commons/src/main/scala/com/twitter/scalding/commons/source/LzoGenericSource.scala b/scalding-commons/src/main/scala/com/twitter/scalding/commons/source/LzoGenericSource.scala index 49eb70cea2..439726d1c4 100644 --- a/scalding-commons/src/main/scala/com/twitter/scalding/commons/source/LzoGenericSource.scala +++ b/scalding-commons/src/main/scala/com/twitter/scalding/commons/source/LzoGenericSource.scala @@ -37,6 +37,6 @@ object LzoGenericSource { new LzoGenericSource[T] { override val conv: BinaryConverter[T] = passedConv override val hdfsPaths = paths - override val localPaths = { assert(paths.size == 1, "Cannot use multiple input files on local mode"); paths } + override val localPaths = paths } }