Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Browse files

Create a multi source tap that deals with identifier issues.

  • Loading branch information...
commit 1d0b3dadc6d6cc3bdb91609670e3634b3401c936 1 parent 36f2d32
Argyris Zymnis azymnis authored
Showing with 8 additions and 4 deletions.
  1. +8 −4 src/main/scala/com/twitter/scalding/FileSource.scala
12 src/main/scala/com/twitter/scalding/FileSource.scala
View
@@ -16,9 +16,7 @@ limitations under the License.
package com.twitter.scalding
import java.io.File
-import java.util.TimeZone
-import java.util.Calendar
-import java.util.{Map => JMap}
+import java.util.{Calendar, TimeZone, UUID, Map => JMap}
import cascading.flow.hadoop.HadoopFlowProcess
import cascading.flow.{FlowProcess, FlowDef}
@@ -133,11 +131,17 @@ abstract class FileSource extends Source {
castHfsTap(new Hfs(hdfsScheme, hdfsPaths.head, SinkMode.KEEP))
}
case 1 => taps.head
- case _ => new MultiSourceTap( taps : _*)
+ case _ => new ScaldingMultiSourceTap(taps)
}
}
}
+class ScaldingMultiSourceTap(taps : Seq[Tap[JobConf, RecordReader[_,_], OutputCollector[_,_]]])
+ extends MultiSourceTap[Tap[JobConf, RecordReader[_,_], OutputCollector[_,_]], JobConf, RecordReader[_,_]](taps : _*) {
+ private final val randomId = UUID.randomUUID.toString
+ override def getIdentifier() = randomId
+}
+
/**
* The fields here are ('offset, 'line)
*/
Please sign in to comment.
Something went wrong with that request. Please try again.