crypto/rsa: fix salt length calculation with PSSSaltLengthAuto

When PSSSaltLength is set, the maximum salt length must equal:

    (modulus_key_size - 1 + 7)/8 - hash_length - 2
and for example, with a 4096 bit modulus key, and a SHA-1 hash,
it should be:

     (4096 -1 + 7)/8 - 20 - 2 = 490
Previously we'd encounter this error:

     crypto/rsa: key size too small for PSS signature

Fixes #42741

Change-Id: I18bb82c41c511d564b3f4c443f4b3a38ab010ac5
Reviewed-on: https://go-review.googlesource.com/c/go/+/302230
Reviewed-by: Emmanuel Odeke <emmanuel@orijtech.com>
Reviewed-by: Filippo Valsorda <filippo@golang.org>
Trust: Emmanuel Odeke <emmanuel@orijtech.com>
Run-TryBot: Emmanuel Odeke <emmanuel@orijtech.com>
TryBot-Result: Go Bot <gobot@golang.org>

src/crypto/rsa/pss.go

@@ -269,7 +269,7 @@ func SignPSS(rand io.Reader, priv *PrivateKey, hash crypto.Hash, digest []byte,
 	saltLength := opts.saltLength()
 	switch saltLength {
 	case PSSSaltLengthAuto:
-		saltLength = priv.Size() - 2 - hash.Size()
+		saltLength = (priv.N.BitLen()-1+7)/8 - 2 - hash.Size()
 	case PSSSaltLengthEqualsHash:
 		saltLength = hash.Size()
 	}

src/crypto/rsa/pss_test.go

@@ -12,7 +12,7 @@ import (
 	_ "crypto/md5"
 	"crypto/rand"
 	"crypto/sha1"
-	_ "crypto/sha256"
+	"crypto/sha256"
 	"encoding/hex"
 	"math/big"
 	"os"
@@ -233,6 +233,24 @@ func TestPSSSigning(t *testing.T) {
 	}
 }
 
+func TestSignWithPSSSaltLengthAuto(t *testing.T) {
+	key, err := GenerateKey(rand.Reader, 513)
+	if err != nil {
+		t.Fatal(err)
+	}
+	digest := sha256.Sum256([]byte("message"))
+	signature, err := key.Sign(rand.Reader, digest[:], &PSSOptions{
+		SaltLength: PSSSaltLengthAuto,
+		Hash:       crypto.SHA256,
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(signature) == 0 {
+		t.Fatal("empty signature returned")
+	}
+}
+
 func bigFromHex(hex string) *big.Int {
 	n, ok := new(big.Int).SetString(hex, 16)
 	if !ok {