sort: fixed bug in (Float64Slice) Less; NaN less than anything else

Previously comparisons with NaN led to contradictory results if it was
compared to anything not NaN, since Less always returned false, thus
breaking monotonicity of ordering.
This fix makes NaN less than anything else and adds NaN and (+-)Inf to
testcases.

Fixes #2092.

R=golang-dev, r, rsc, r
CC=golang-dev
https://golang.org/cl/4805051

src/pkg/sort/sort.go

@@ -6,6 +6,8 @@
 // collections.
 package sort
 
+import "math"
+
 // A type, typically a collection, that satisfies sort.Interface can be
 // sorted by the routines in this package.  The methods require that the
 // elements of the collection be enumerated by an integer index.
@@ -167,7 +169,7 @@ func (p IntSlice) Sort() { Sort(p) }
 type Float64Slice []float64
 
 func (p Float64Slice) Len() int           { return len(p) }
-func (p Float64Slice) Less(i, j int) bool { return p[i] < p[j] }
+func (p Float64Slice) Less(i, j int) bool { return p[i] < p[j] || math.IsNaN(p[i]) && !math.IsNaN(p[j]) }
 func (p Float64Slice) Swap(i, j int)      { p[i], p[j] = p[j], p[i] }
 
 // Sort is a convenience method.

src/pkg/sort/sort_test.go

@@ -6,14 +6,15 @@ package sort_test
 
 import (
 	"fmt"
+	"math"
 	"rand"
 	. "sort"
 	"strconv"
 	"testing"
 )
 
 var ints = [...]int{74, 59, 238, -784, 9845, 959, 905, 0, 0, 42, 7586, -5467984, 7586}
-var float64s = [...]float64{74.3, 59.0, 238.2, -784.0, 2.3, 9845.768, -959.7485, 905, 7.8, 7.8}
+var float64s = [...]float64{74.3, 59.0, math.Inf(1), 238.2, -784.0, 2.3, math.NaN(), math.NaN(), math.Inf(-1), 9845.768, -959.7485, 905, 7.8, 7.8}
 var strings = [...]string{"", "Hello", "foo", "bar", "foo", "f00", "%*&^*&^&", "***"}
 
 func TestSortIntSlice(t *testing.T) {