Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Browse files

Merge branch 'release/0.5.3'

  • Loading branch information...
commit b33631992f3069e1701872916b11db336b79a837 2 parents 2391e97 + 3333e75
@azymnis azymnis authored
View
3  CHANGES.md
@@ -1,5 +1,8 @@
# Scalding #
+### Version 0.5.3 ###
+* ISSUE 77: Add a scalding multi source tap that has a proper unique identifier
+
### Version 0.5.2 ###
* ISSUE 74: Upgrade cascading to wip-291 and maple to 0.1.7
View
4 README.md
@@ -4,7 +4,7 @@ Scalding is a Scala library that makes it easy to write MapReduce jobs in Hadoop
Scalding is built on top of [Cascading](http://www.cascading.org/), a Java library that abstracts away much of the complexity of Hadoop.
-Current version: 0.5.2
+Current version: 0.5.3
## Word Count
@@ -49,7 +49,7 @@ You can find more example code under [examples/](https://github.com/twitter/scal
We use [Travis CI](http://travis-ci.org/) to verify the build:
[![Build Status](https://secure.travis-ci.org/twitter/scalding.png)](http://travis-ci.org/twitter/scalding)
-The current version is 0.5.2 and is available from maven central: org="com.twitter",
+The current version is 0.5.3 and is available from maven central: org="com.twitter",
artifact="scalding_2.8.1" or artifact="scalding_2.9.1".
## Contact
View
2  build.sbt
@@ -2,7 +2,7 @@ import AssemblyKeys._
name := "scalding"
-version := "0.5.2"
+version := "0.5.3"
organization := "com.twitter"
View
2  scripts/scald.rb
@@ -2,7 +2,7 @@
require 'fileutils'
require 'thread'
-SCALDING_VERSION="0.5.2"
+SCALDING_VERSION="0.5.3"
#Usage : scald.rb [--hdfs|--local|--print] job <job args>
# --hdfs: if job ends in ".scala" or ".java" and the file exists, link it against JARFILE (below) and then run it on HOST.
View
12 src/main/scala/com/twitter/scalding/FileSource.scala
@@ -16,9 +16,7 @@ limitations under the License.
package com.twitter.scalding
import java.io.File
-import java.util.TimeZone
-import java.util.Calendar
-import java.util.{Map => JMap}
+import java.util.{Calendar, TimeZone, UUID, Map => JMap}
import cascading.flow.hadoop.HadoopFlowProcess
import cascading.flow.{FlowProcess, FlowDef}
@@ -133,11 +131,17 @@ abstract class FileSource extends Source {
castHfsTap(new Hfs(hdfsScheme, hdfsPaths.head, SinkMode.KEEP))
}
case 1 => taps.head
- case _ => new MultiSourceTap( taps : _*)
+ case _ => new ScaldingMultiSourceTap(taps)
}
}
}
+class ScaldingMultiSourceTap(taps : Seq[Tap[JobConf, RecordReader[_,_], OutputCollector[_,_]]])
+ extends MultiSourceTap[Tap[JobConf, RecordReader[_,_], OutputCollector[_,_]], JobConf, RecordReader[_,_]](taps : _*) {
+ private final val randomId = UUID.randomUUID.toString
+ override def getIdentifier() = randomId
+}
+
/**
* The fields here are ('offset, 'line)
*/
Please sign in to comment.
Something went wrong with that request. Please try again.