Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with
or
.
Download ZIP
Browse files

Fixes a bug with nulls in IntegralComparator

  • Loading branch information...
commit 64da82eb0f4bb258e0aaaf8ab21f4f4292f6a382 1 parent 4990bae
@johnynek johnynek authored
View
2  README.md
@@ -41,6 +41,8 @@ recommendations:
We use Travis-ci.org to verify the build:
[![Build Status](https://secure.travis-ci.org/twitter/scalding.png)](http://travis-ci.org/twitter/scalding)
+The current version is 0.3.4 and available from maven central: org="com.twitter", artifact="scalding_2.8.1".
+
## Comparison to Scrunch/Scoobi
Scalding comes with an executable tutorial set that does not require a Hadoop
cluster. If you're curious about scalding, why not invest a bit of time and run the tutorial
View
8 build.sbt
@@ -2,7 +2,7 @@ import AssemblyKeys._
name := "scalding"
-version := "0.3.3"
+version := "0.3.4"
organization := "com.twitter"
@@ -10,11 +10,11 @@ scalaVersion := "2.8.1"
resolvers += "Concurrent Maven Repo" at "http://conjars.org/repo"
-libraryDependencies += "cascading" % "cascading-core" % "2.0.0-wip-236"
+libraryDependencies += "cascading" % "cascading-core" % "2.0.0-wip-238"
-libraryDependencies += "cascading" % "cascading-local" % "2.0.0-wip-236"
+libraryDependencies += "cascading" % "cascading-local" % "2.0.0-wip-238"
-libraryDependencies += "cascading" % "cascading-hadoop" % "2.0.0-wip-236"
+libraryDependencies += "cascading" % "cascading-hadoop" % "2.0.0-wip-238"
libraryDependencies += "cascading.kryo" % "cascading.kryo" % "0.2.1"
View
2  scripts/scald.rb
@@ -2,7 +2,7 @@
require 'fileutils'
require 'thread'
-SCALDING_VERSION="0.3.3"
+SCALDING_VERSION="0.3.4"
#Usage : scald.rb [--hdfs|--local|--print] job <job args>
# --hdfs: if job ends in ".scala" or ".java" and the file exists, link it against JARFILE (below) and then run it on HOST.
View
42 src/main/scala/com/twitter/scalding/IntegralComparator.scala
@@ -26,27 +26,21 @@ import java.util.Comparator;
*/
class IntegralComparator extends Comparator[AnyRef] with Hasher[AnyRef] with Serializable {
- def isIntegral(boxed : AnyRef) = {
- val bclass = boxed.getClass
- if (bclass == classOf[java.lang.Long]) {
- true
- }
- else if (bclass == classOf[java.lang.Integer]) {
- true
- }
- else if (bclass == classOf[java.lang.Short]) {
- true
- }
- else if (bclass == classOf[java.lang.Byte]) {
- true
- }
- else {
- false
- }
- }
+ val integralTypes : Set[Class[_]] = Set(classOf[java.lang.Long],
+ classOf[java.lang.Integer],
+ classOf[java.lang.Short],
+ classOf[java.lang.Byte])
+
+ def isIntegral(boxed : AnyRef) = integralTypes(boxed.getClass)
override def compare(a1: AnyRef, a2: AnyRef) : Int = {
- if (isIntegral(a1) && isIntegral(a2)) {
+ val a1IsNull = if (null == a1) 1 else 0
+ val a2IsNull = if (null == a2) 1 else 0
+ if (a1IsNull + a2IsNull > 0) {
+ //if a2IsNull, but a1IsNot, a2 is less:
+ a2IsNull - a1IsNull
+ }
+ else if (isIntegral(a1) && isIntegral(a2)) {
val long1 = a1.asInstanceOf[Number].longValue
val long2 = a2.asInstanceOf[Number].longValue
if (long1 < long2)
@@ -61,9 +55,13 @@ class IntegralComparator extends Comparator[AnyRef] with Hasher[AnyRef] with Ser
}
override def hashCode(obj : AnyRef) : Int = {
- if (isIntegral(obj)) {
- val longv = obj.asInstanceOf[Number].longValue
- (longv ^ (longv >>> 32)).toInt
+ if (null == obj) {
+ 0
+ }
+ else if (isIntegral(obj)) {
+ obj.asInstanceOf[Number]
+ .longValue
+ .hashCode
}
else {
//Use the default:
View
52 src/test/scala/com/twitter/scalding/IntegralCompTest.scala
@@ -0,0 +1,52 @@
+package com.twitter.scalding
+import org.specs._
+
+class IntegralCompTest extends Specification {
+ def box[T](t : T) = t.asInstanceOf[AnyRef]
+
+ "IntegralComparator" should {
+ val intComp = new IntegralComparator
+ "recognize integral types" in {
+ intComp.isIntegral(box(1)) must beTrue
+ intComp.isIntegral(box(1L)) must beTrue
+ intComp.isIntegral(box(1.asInstanceOf[Short])) must beTrue
+ //Boxed
+ intComp.isIntegral(new java.lang.Long(2)) must beTrue
+ intComp.isIntegral(new java.lang.Integer(2)) must beTrue
+ intComp.isIntegral(new java.lang.Short(2.asInstanceOf[Short])) must beTrue
+ intComp.isIntegral(new java.lang.Long(2)) must beTrue
+ intComp.isIntegral(new java.lang.Long(2)) must beTrue
+ //These are not integrals
+ intComp.isIntegral(box(0.0)) must beFalse
+ intComp.isIntegral(box("hey")) must beFalse
+ intComp.isIntegral(box(Nil)) must beFalse
+ intComp.isIntegral(box(None)) must beFalse
+ }
+ "handle null inputs" in {
+ intComp.hashCode(null) must be_==(0)
+ List(box(1),box("hey"),box(2L),box(0.0)).foreach { x =>
+ intComp.compare(null, x) must be_<(0)
+ intComp.compare(x,null) must be_>(0)
+ intComp.compare(x, x) must be_==(0)
+ }
+ intComp.compare(null,null) must be_==(0)
+ }
+ "have consistent hashcode" in {
+ List( (box(1),box(1L)), (box(2),box(2L)), (box(3),box(3L)) )
+ .foreach { pair =>
+ intComp.compare(pair._1, pair._2) must be_==(0)
+ intComp.hashCode(pair._1) must be_==(intComp.hashCode(pair._2))
+ }
+ List( (box(1),box(2L)), (box(2),box(3L)), (box(3),box(4L)) )
+ .foreach { pair =>
+ intComp.compare(pair._1, pair._2) must be_<(0)
+ intComp.compare(pair._2, pair._1) must be_>(0)
+ }
+ }
+ "Compare strings properly" in {
+ intComp.compare("hey","you") must be_==("hey".compareTo("you"))
+ intComp.compare("hey","hey") must be_==("hey".compareTo("hey"))
+ intComp.compare("you","hey") must be_==("you".compareTo("hey"))
+ }
+ }
+}
Please sign in to comment.
Something went wrong with that request. Please try again.