Skip to content
Browse files

Updates to cascading wip-234

  • Loading branch information...
1 parent 6bb2b69 commit cf9166c626a5b9c0ab536591cca240cba8b107fb @johnynek johnynek committed Feb 24, 2012
Showing with 8 additions and 9 deletions.
  1. +4 −4 build.sbt
  2. +4 −5 src/main/scala/com/twitter/scalding/Source.scala
View
8 build.sbt
@@ -10,15 +10,15 @@ scalaVersion := "2.8.1"
resolvers += "Concurrent Maven Repo" at "http://conjars.org/repo"
-libraryDependencies += "cascading" % "cascading-core" % "2.0.0-wip-227"
+libraryDependencies += "cascading" % "cascading-core" % "2.0.0-wip-234"
-libraryDependencies += "cascading" % "cascading-local" % "2.0.0-wip-227"
+libraryDependencies += "cascading" % "cascading-local" % "2.0.0-wip-234"
-libraryDependencies += "cascading" % "cascading-hadoop" % "2.0.0-wip-227"
+libraryDependencies += "cascading" % "cascading-hadoop" % "2.0.0-wip-234"
libraryDependencies += "cascading.kryo" % "cascading.kryo" % "0.2.1"
-libraryDependencies += "com.twitter" % "meat-locker" % "0.1.5"
+libraryDependencies += "com.twitter" % "meat-locker" % "0.1.6"
libraryDependencies += "commons-lang" % "commons-lang" % "2.4"
View
9 src/main/scala/com/twitter/scalding/Source.scala
@@ -151,7 +151,7 @@ abstract class Source extends java.io.Serializable {
}
protected def createHadoopTestReadTap(buffer : Iterable[Tuple]) :
- Tap[HadoopFlowProcess, JobConf, RecordReader[_,_], OutputCollector[_,_]] = {
+ Tap[HadoopFlowProcess, JobConf, RecordReader[_,_], _] = {
new MemorySourceTap(buffer.toList.asJava, hdfsScheme.getSourceFields())
}
@@ -205,7 +205,7 @@ abstract class Source extends java.io.Serializable {
}
protected def createHdfsReadTap(hdfsMode : Hdfs) :
- Tap[HadoopFlowProcess, JobConf, RecordReader[_,_], OutputCollector[_,_]] = {
+ Tap[HadoopFlowProcess, JobConf, RecordReader[_,_], _] = {
val goodPaths = if (hdfsMode.sourceStrictness) {
//we check later that all the paths are good
hdfsPaths
@@ -223,12 +223,11 @@ abstract class Source extends java.io.Serializable {
new Hfs(hdfsScheme, hdfsPaths.head, SinkMode.KEEP)
}
case 1 => taps.head
- case _ => new MultiSourceTap[HadoopFlowProcess,
- JobConf, RecordReader[_,_], OutputCollector[_,_]](taps.toSeq : _*)
+ case _ => new MultiSourceTap[Hfs, HadoopFlowProcess, JobConf, RecordReader[_,_]]( taps.toSeq : _*)
}
}
protected def createHdfsWriteTap(hdfsMode : Hdfs) :
- Tap[HadoopFlowProcess, JobConf, RecordReader[_,_], OutputCollector[_,_]] = {
+ Tap[HadoopFlowProcess, JobConf, _, OutputCollector[_,_]] = {
new Hfs(hdfsScheme, hdfsWritePath, SinkMode.REPLACE)
}

0 comments on commit cf9166c

Please sign in to comment.
Something went wrong with that request. Please try again.