Increase number of attempts before failure in ECDSA-DER (#33933)

Of the four test keys, 3 have a 75% chance of being smaller than max on each call.
 (1 - 0.75) ^ 10 = 9.5e-7, so failure happens slightly more rarely than 1 in a million.

The fourth test key only has a 43.75% chance (1 - (.75 * .75)) of being smaller,
and (1 - .4375) ^ 10 = 3.1e-3, so slightly more likely than 3 per thousand.

Raising the iteration count to 36 brings the fourth key odds of failure to about 1 in a billion (1.01e-9).
36 iterations makes the other 3 fail at 1e-22; but 1e-9 is attained at 15 iterations.

src/libraries/Common/tests/System/Security/Cryptography/AlgorithmImplementations/ECDsa/ECDsaSignatureFormatTests.cs

@@ -331,15 +331,17 @@ public void Rfc23279TrySignHashUnderMax()
             ECDsa key = (ECDsa)keyDescription.Key;
 
             const DSASignatureFormat SignatureFormat = DSASignatureFormat.Rfc3279DerSequence;
-            const int RetryCount = 10;
+            // Make secp521r1 (7/16 chance of being smaller) and mod-8 keys (3/4 chance of being smaller)
+            // have the same 1-in-a-billion chance of failure.
+            int retryCount = keyDescription.FieldSizeInBits % 8 == 1 ? 36 : 15;
             byte[] hash = { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20 };
 
             int expectedSize = GetExpectedSize(keyDescription.FieldSizeInBits);
             int maxSize = key.GetMaxSignatureSize(DSASignatureFormat.Rfc3279DerSequence);
             Assert.True(expectedSize < maxSize, "expectedSize < maxSize");
             byte[] signature = new byte[expectedSize];
 
-            for (int i = 0; i < RetryCount; i++)
+            for (int i = 0; i < retryCount; i++)
             {
                 if (key.TrySignHash(hash, signature, SignatureFormat, out int written))
                 {
@@ -359,15 +361,17 @@ public void Rfc23279TrySignDataUnderMax()
             ECDsa key = (ECDsa)keyDescription.Key;
 
             const DSASignatureFormat SignatureFormat = DSASignatureFormat.Rfc3279DerSequence;
-            const int RetryCount = 10;
+            // Make secp521r1 (7/16 chance of being smaller) and mod-8 keys (3/4 chance of being smaller)
+            // have the same 1-in-a-billion chance of failure.
+            int retryCount = keyDescription.FieldSizeInBits % 8 == 1 ? 36 : 15;
             HashAlgorithmName hashAlgorithm = HashAlgorithmName.SHA1;
 
             int expectedSize = GetExpectedSize(keyDescription.FieldSizeInBits);
             int maxSize = key.GetMaxSignatureSize(DSASignatureFormat.Rfc3279DerSequence);
             Assert.True(expectedSize < maxSize, "expectedSize < maxSize");
             byte[] signature = new byte[expectedSize];
 
-            for (int i = 0; i < RetryCount; i++)
+            for (int i = 0; i < retryCount; i++)
             {
                 if (key.TrySignData(Array.Empty<byte>(), signature, hashAlgorithm, SignatureFormat, out int written))
                 {