math/big: allow all values for GCD

Allow the inputs a and b to be zero or negative to GCD
with the following definitions.

If x or y are not nil, GCD sets their value such that z = a*x + b*y.
Regardless of the signs of a and b, z is always >= 0.
If a == b == 0, GCD sets z = x = y = 0.
If a == 0 and b != 0, GCD sets z = |b|, x = 0, y = sign(b) * 1.
If a != 0 and b == 0, GCD sets z = |a|, x = sign(a) * 1, y = 0.

Fixes #28878

Change-Id: Ia83fce66912a96545c95cd8df0549bfd852652f3
Reviewed-on: https://go-review.googlesource.com/c/go/+/164972
Run-TryBot: Brian Kessler <brian.m.kessler@gmail.com>
TryBot-Result: Gobot Gobot <gobot@golang.org>
Reviewed-by: Robert Griesemer <gri@golang.org>

src/math/big/int.go

@@ -502,18 +502,36 @@ func (z *Int) Exp(x, y, m *Int) *Int {
 	return z
 }
 
-// GCD sets z to the greatest common divisor of a and b, which both must
-// be > 0, and returns z.
+// GCD sets z to the greatest common divisor of a and b and returns z.
 // If x or y are not nil, GCD sets their value such that z = a*x + b*y.
-// If either a or b is <= 0, GCD sets z = x = y = 0.
+// Regardless of the signs of a and b, z is always >= 0.
+// If a == b == 0, GCD sets z = x = y = 0.
+// If a == 0 and b != 0, GCD sets z = |b|, x = 0, y = sign(b) * 1.
+// If a != 0 and b == 0, GCD sets z = |a|, x = sign(a) * 1, y = 0.
 func (z *Int) GCD(x, y, a, b *Int) *Int {
-	if a.Sign() <= 0 || b.Sign() <= 0 {
-		z.SetInt64(0)
+	if len(a.abs) == 0 || len(b.abs) == 0 {
+		lenA, lenB, negA, negB := len(a.abs), len(b.abs), a.neg, b.neg
+		if lenA == 0 {
+			z.Set(b)
+		} else {
+			z.Set(a)
+		}
+		z.neg = false
 		if x != nil {
-			x.SetInt64(0)
+			if lenA == 0 {
+				x.SetUint64(0)
+			} else {
+				x.SetUint64(1)
+				x.neg = negA
+			}
 		}
 		if y != nil {
-			y.SetInt64(0)
+			if lenB == 0 {
+				y.SetUint64(0)
+			} else {
+				y.SetUint64(1)
+				y.neg = negB
+			}
 		}
 		return z
 	}
@@ -621,7 +639,7 @@ func euclidUpdate(A, B, Ua, Ub, q, r, s, t *Int, extended bool) {
 }
 
 // lehmerGCD sets z to the greatest common divisor of a and b,
-// which both must be > 0, and returns z.
+// which both must be != 0, and returns z.
 // If x or y are not nil, their values are set such that z = a*x + b*y.
 // See Knuth, The Art of Computer Programming, Vol. 2, Section 4.5.2, Algorithm L.
 // This implementation uses the improved condition by Collins requiring only one
@@ -633,8 +651,8 @@ func euclidUpdate(A, B, Ua, Ub, q, r, s, t *Int, extended bool) {
 func (z *Int) lehmerGCD(x, y, a, b *Int) *Int {
 	var A, B, Ua, Ub *Int
 
-	A = new(Int).Set(a)
-	B = new(Int).Set(b)
+	A = new(Int).Abs(a)
+	B = new(Int).Abs(b)
 
 	extended := x != nil || y != nil
 
@@ -720,7 +738,7 @@ func (z *Int) lehmerGCD(x, y, a, b *Int) *Int {
 			A.abs[0] = aWord
 		}
 	}
-
+	negA := a.neg
 	if y != nil {
 		// avoid aliasing b needed in the division below
 		if y == b {
@@ -730,12 +748,18 @@ func (z *Int) lehmerGCD(x, y, a, b *Int) *Int {
 		}
 		// y = (z - a*x)/b
 		y.Mul(a, Ua) // y can safely alias a
+		if negA {
+			y.neg = !y.neg
+		}
 		y.Sub(A, y)
 		y.Div(y, B)
 	}
 
 	if x != nil {
 		*x = *Ua
+		if negA {
+			x.neg = !x.neg
+		}
 	}
 
 	*z = *A

src/math/big/int_test.go

@@ -757,11 +757,13 @@ var gcdTests = []struct {
 }{
 	// a <= 0 || b <= 0
 	{"0", "0", "0", "0", "0"},
-	{"0", "0", "0", "0", "7"},
-	{"0", "0", "0", "11", "0"},
-	{"0", "0", "0", "-77", "35"},
-	{"0", "0", "0", "64515", "-24310"},
-	{"0", "0", "0", "-64515", "-24310"},
+	{"7", "0", "1", "0", "7"},
+	{"7", "0", "-1", "0", "-7"},
+	{"11", "1", "0", "11", "0"},
+	{"7", "-1", "-2", "-77", "35"},
+	{"935", "-3", "8", "64515", "24310"},
+	{"935", "-3", "-8", "64515", "-24310"},
+	{"935", "3", "-8", "-64515", "-24310"},
 
 	{"1", "-9", "47", "120", "23"},
 	{"7", "1", "-2", "77", "35"},