Merge bitcoin#808: Exhaustive test improvements + exhaustive schnorrs…

…ig tests

8b7dcdd Add exhaustive test for extrakeys and schnorrsig (Pieter Wuille)
08d7d89 Make pubkey parsing test whether points are in the correct subgroup (Pieter Wuille)
87af00b Abstract out challenge computation in schnorrsig (Pieter Wuille)
63e1b2a Disable output buffering in tests_exhaustive.c (Pieter Wuille)
39f67dd Support splitting exhaustive tests across cores (Pieter Wuille)
e99b26f Give exhaustive_tests count and seed cmdline inputs (Pieter Wuille)
49e6630 refactor: move RNG seeding to testrand (Pieter Wuille)
b110c10 Change exhaustive test groups so they have a point with X=1 (Pieter Wuille)
cec7b18 Select exhaustive lambda in function of order (Pieter Wuille)
78f6cdf Make the curve B constant a secp256k1_fe (Pieter Wuille)
d7f39ae Delete gej_is_valid_var: unused outside tests (Pieter Wuille)
8bcd78c Make secp256k1_scalar_b32 detect overflow in scalar_low (Pieter Wuille)
c498366 Move exhaustive tests for recovery to module (Pieter Wuille)
be31791 Make group order purely compile-time in exhaustive tests (Pieter Wuille)

Pull request description:

  A few miscellaneous improvements:
  * Just use EXHAUSTIVE_TEST_ORDER as order everywhere, rather than a variable
  * Move exhaustive tests for recovery module to the recovery module directory
  * Make `secp256k1_scalar_set_b32` detect overflow correctly for scalar_low (a comment in the recovery exhaustive test indicated why this was the case, but this looks incorrect).
  * Change the small test groups so that they include a point with X coordinate 1.
  * Initialize the RNG seed, allowing configurating from the cmdline, and report it.
  * Permit changing the number of iterations (re-randomizing for each).
  * Support splitting the work across cores from the cmdline.

  And a big one:
  * Add exhaustive tests for schnorrsig module (and limited ones for extrakeys).

ACKs for top commit:
  real-or-random:
    ACK 8b7dcdd
  jonasnick:
    ACK 8b7dcdd

Tree-SHA512: 18d7f362402085238faaced164c0ca34079717a477001fc0b13448b3529ea2ad705793a13b7a36f34bf12e9231fee11070f88cc51bfc2a83ca82aa13f7aaae71

sage/gen_exhaustive_groups.sage

@@ -0,0 +1,129 @@
+# Define field size and field
+P = 2^256 - 2^32 - 977
+F = GF(P)
+BETA = F(0x7ae96a2b657c07106e64479eac3434e99cf0497512f58995c1396c28719501ee)
+
+assert(BETA != F(1) and BETA^3 == F(1))
+
+orders_done = set()
+results = {}
+first = True
+for b in range(1, P):
+    # There are only 6 curves (up to isomorphism) of the form y^2=x^3+B. Stop once we have tried all.
+    if len(orders_done) == 6:
+        break
+
+    E = EllipticCurve(F, [0, b])
+    print("Analyzing curve y^2 = x^3 + %i" % b)
+    n = E.order()
+    # Skip curves with an order we've already tried
+    if n in orders_done:
+        print("- Isomorphic to earlier curve")
+        continue
+    orders_done.add(n)
+    # Skip curves isomorphic to the real secp256k1
+    if n.is_pseudoprime():
+        print(" - Isomorphic to secp256k1")
+        continue
+
+    print("- Finding subgroups")
+
+    # Find what prime subgroups exist
+    for f, _ in n.factor():
+        print("- Analyzing subgroup of order %i" % f)
+        # Skip subgroups of order >1000
+        if f < 4 or f > 1000:
+            print("  - Bad size")
+            continue
+
+        # Iterate over X coordinates until we find one that is on the curve, has order f,
+        # and for which curve isomorphism exists that maps it to X coordinate 1.
+        for x in range(1, P):
+            # Skip X coordinates not on the curve, and construct the full point otherwise.
+            if not E.is_x_coord(x):
+                continue
+            G = E.lift_x(F(x))
+
+            print("  - Analyzing (multiples of) point with X=%i" % x)
+
+            # Skip points whose order is not a multiple of f. Project the point to have
+            # order f otherwise.
+            if (G.order() % f):
+                print("    - Bad order")
+                continue
+            G = G * (G.order() // f)
+
+            # Find lambda for endomorphism. Skip if none can be found.
+            lam = None
+            for l in Integers(f)(1).nth_root(3, all=True):
+                if int(l)*G == E(BETA*G[0], G[1]):
+                    lam = int(l)
+                    break
+            if lam is None:
+                print("    - No endomorphism for this subgroup")
+                break
+
+            # Now look for an isomorphism of the curve that gives this point an X
+            # coordinate equal to 1.
+            # If (x,y) is on y^2 = x^3 + b, then (a^2*x, a^3*y) is on y^2 = x^3 + a^6*b.
+            # So look for m=a^2=1/x.
+            m = F(1)/G[0]
+            if not m.is_square():
+                print("    - No curve isomorphism maps it to a point with X=1")
+                continue
+            a = m.sqrt()
+            rb = a^6*b
+            RE = EllipticCurve(F, [0, rb])
+
+            # Use as generator twice the image of G under the above isormorphism.
+            # This means that generator*(1/2 mod f) will have X coordinate 1.
+            RG = RE(1, a^3*G[1]) * 2
+            # And even Y coordinate.
+            if int(RG[1]) % 2:
+                RG = -RG
+            assert(RG.order() == f)
+            assert(lam*RG == RE(BETA*RG[0], RG[1]))
+
+            # We have found curve RE:y^2=x^3+rb with generator RG of order f. Remember it
+            results[f] = {"b": rb, "G": RG, "lambda": lam}
+            print("    - Found solution")
+            break
+
+    print("")
+
+print("")
+print("")
+print("/* To be put in src/group_impl.h: */")
+first = True
+for f in sorted(results.keys()):
+    b = results[f]["b"]
+    G = results[f]["G"]
+    print("#  %s EXHAUSTIVE_TEST_ORDER == %i" % ("if" if first else "elif", f))
+    first = False
+    print("static const secp256k1_ge secp256k1_ge_const_g = SECP256K1_GE_CONST(")
+    print("    0x%08x, 0x%08x, 0x%08x, 0x%08x," % tuple((int(G[0]) >> (32 * (7 - i))) & 0xffffffff for i in range(4)))
+    print("    0x%08x, 0x%08x, 0x%08x, 0x%08x," % tuple((int(G[0]) >> (32 * (7 - i))) & 0xffffffff for i in range(4, 8)))
+    print("    0x%08x, 0x%08x, 0x%08x, 0x%08x," % tuple((int(G[1]) >> (32 * (7 - i))) & 0xffffffff for i in range(4)))
+    print("    0x%08x, 0x%08x, 0x%08x, 0x%08x" % tuple((int(G[1]) >> (32 * (7 - i))) & 0xffffffff for i in range(4, 8)))
+    print(");")
+    print("static const secp256k1_fe secp256k1_fe_const_b = SECP256K1_FE_CONST(")
+    print("    0x%08x, 0x%08x, 0x%08x, 0x%08x," % tuple((int(b) >> (32 * (7 - i))) & 0xffffffff for i in range(4)))
+    print("    0x%08x, 0x%08x, 0x%08x, 0x%08x" % tuple((int(b) >> (32 * (7 - i))) & 0xffffffff for i in range(4, 8)))
+    print(");")
+print("#  else")
+print("#    error No known generator for the specified exhaustive test group order.")
+print("#  endif")
+
+print("")
+print("")
+print("/* To be put in src/scalar_impl.h: */")
+first = True
+for f in sorted(results.keys()):
+    lam = results[f]["lambda"]
+    print("#  %s EXHAUSTIVE_TEST_ORDER == %i" % ("if" if first else "elif", f))
+    first = False
+    print("#    define EXHAUSTIVE_TEST_LAMBDA %i" % lam)
+print("#  else")
+print("#    error No known lambda for the specified exhaustive test group order.")
+print("#  endif")
+print("")

src/group.h

@@ -139,4 +139,15 @@ static void secp256k1_ge_storage_cmov(secp256k1_ge_storage *r, const secp256k1_g
 /** Rescale a jacobian point by b which must be non-zero. Constant-time. */
 static void secp256k1_gej_rescale(secp256k1_gej *r, const secp256k1_fe *b);
 
+/** Determine if a point (which is assumed to be on the curve) is in the correct (sub)group of the curve.
+ *
+ * In normal mode, the used group is secp256k1, which has cofactor=1 meaning that every point on the curve is in the
+ * group, and this function returns always true.
+ *
+ * When compiling in exhaustive test mode, a slightly different curve equation is used, leading to a group with a
+ * (very) small subgroup, and that subgroup is what is used for all cryptographic operations. In that mode, this
+ * function checks whether a point that is on the curve is in fact also in that subgroup.
+ */
+static int secp256k1_ge_is_in_correct_subgroup(const secp256k1_ge* ge);
+
 #endif /* SECP256K1_GROUP_H */

src/group_impl.h

@@ -11,49 +11,38 @@
 #include "field.h"
 #include "group.h"
 
-/* These points can be generated in sage as follows:
+/* These exhaustive group test orders and generators are chosen such that:
+ * - The field size is equal to that of secp256k1, so field code is the same.
+ * - The curve equation is of the form y^2=x^3+B for some constant B.
+ * - The subgroup has a generator 2*P, where P.x=1.
+ * - The subgroup has size less than 1000 to permit exhaustive testing.
+ * - The subgroup admits an endomorphism of the form lambda*(x,y) == (beta*x,y).
  *
- * 0. Setup a worksheet with the following parameters.
- *   b = 4  # whatever CURVE_B will be set to
- *   F = FiniteField (0xFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFEFFFFFC2F)
- *   C = EllipticCurve ([F (0), F (b)])
- *
- * 1. Determine all the small orders available to you. (If there are
- *    no satisfactory ones, go back and change b.)
- *   print C.order().factor(limit=1000)
- *
- * 2. Choose an order as one of the prime factors listed in the above step.
- *    (You can also multiply some to get a composite order, though the
- *    tests will crash trying to invert scalars during signing.) We take a
- *    random point and scale it to drop its order to the desired value.
- *    There is some probability this won't work; just try again.
- *   order = 199
- *   P = C.random_point()
- *   P = (int(P.order()) / int(order)) * P
- *   assert(P.order() == order)
- *
- * 3. Print the values. You'll need to use a vim macro or something to
- *    split the hex output into 4-byte chunks.
- *   print "%x %x" % P.xy()
+ * These parameters are generated using sage/gen_exhaustive_groups.sage.
  */
 #if defined(EXHAUSTIVE_TEST_ORDER)
-#  if EXHAUSTIVE_TEST_ORDER == 199
+#  if EXHAUSTIVE_TEST_ORDER == 13
 static const secp256k1_ge secp256k1_ge_const_g = SECP256K1_GE_CONST(
-    0xFA7CC9A7, 0x0737F2DB, 0xA749DD39, 0x2B4FB069,
-    0x3B017A7D, 0xA808C2F1, 0xFB12940C, 0x9EA66C18,
-    0x78AC123A, 0x5ED8AEF3, 0x8732BC91, 0x1F3A2868,
-    0x48DF246C, 0x808DAE72, 0xCFE52572, 0x7F0501ED
+    0xc3459c3d, 0x35326167, 0xcd86cce8, 0x07a2417f,
+    0x5b8bd567, 0xde8538ee, 0x0d507b0c, 0xd128f5bb,
+    0x8e467fec, 0xcd30000a, 0x6cc1184e, 0x25d382c2,
+    0xa2f4494e, 0x2fbe9abc, 0x8b64abac, 0xd005fb24
 );
-
-static const int CURVE_B = 4;
-#  elif EXHAUSTIVE_TEST_ORDER == 13
+static const secp256k1_fe secp256k1_fe_const_b = SECP256K1_FE_CONST(
+    0x3d3486b2, 0x159a9ca5, 0xc75638be, 0xb23a69bc,
+    0x946a45ab, 0x24801247, 0xb4ed2b8e, 0x26b6a417
+);
+#  elif EXHAUSTIVE_TEST_ORDER == 199
 static const secp256k1_ge secp256k1_ge_const_g = SECP256K1_GE_CONST(
-    0xedc60018, 0xa51a786b, 0x2ea91f4d, 0x4c9416c0,
-    0x9de54c3b, 0xa1316554, 0x6cf4345c, 0x7277ef15,
-    0x54cb1b6b, 0xdc8c1273, 0x087844ea, 0x43f4603e,
-    0x0eaf9a43, 0xf6effe55, 0x939f806d, 0x37adf8ac
+    0x226e653f, 0xc8df7744, 0x9bacbf12, 0x7d1dcbf9,
+    0x87f05b2a, 0xe7edbd28, 0x1f564575, 0xc48dcf18,
+    0xa13872c2, 0xe933bb17, 0x5d9ffd5b, 0xb5b6e10c,
+    0x57fe3c00, 0xbaaaa15a, 0xe003ec3e, 0x9c269bae
+);
+static const secp256k1_fe secp256k1_fe_const_b = SECP256K1_FE_CONST(
+    0x2cca28fa, 0xfc614b80, 0x2a3db42b, 0x00ba00b1,
+    0xbea8d943, 0xdace9ab2, 0x9536daea, 0x0074defb
 );
-static const int CURVE_B = 2;
 #  else
 #    error No known generator for the specified exhaustive test group order.
 #  endif
@@ -68,7 +57,7 @@ static const secp256k1_ge secp256k1_ge_const_g = SECP256K1_GE_CONST(
     0xFD17B448UL, 0xA6855419UL, 0x9C47D08FUL, 0xFB10D4B8UL
 );
 
-static const int CURVE_B = 7;
+static const secp256k1_fe secp256k1_fe_const_b = SECP256K1_FE_CONST(0, 0, 0, 0, 0, 0, 0, 7);
 #endif
 
 static void secp256k1_ge_set_gej_zinv(secp256k1_ge *r, const secp256k1_gej *a, const secp256k1_fe *zi) {
@@ -219,14 +208,13 @@ static void secp256k1_ge_clear(secp256k1_ge *r) {
 }
 
 static int secp256k1_ge_set_xquad(secp256k1_ge *r, const secp256k1_fe *x) {
-    secp256k1_fe x2, x3, c;
+    secp256k1_fe x2, x3;
     r->x = *x;
     secp256k1_fe_sqr(&x2, x);
     secp256k1_fe_mul(&x3, x, &x2);
     r->infinity = 0;
-    secp256k1_fe_set_int(&c, CURVE_B);
-    secp256k1_fe_add(&c, &x3);
-    return secp256k1_fe_sqrt(&r->y, &c);
+    secp256k1_fe_add(&x3, &secp256k1_fe_const_b);
+    return secp256k1_fe_sqrt(&r->y, &x3);
 }
 
 static int secp256k1_ge_set_xo_var(secp256k1_ge *r, const secp256k1_fe *x, int odd) {
@@ -269,36 +257,15 @@ static int secp256k1_gej_is_infinity(const secp256k1_gej *a) {
     return a->infinity;
 }
 
-static int secp256k1_gej_is_valid_var(const secp256k1_gej *a) {
-    secp256k1_fe y2, x3, z2, z6;
-    if (a->infinity) {
-        return 0;
-    }
-    /** y^2 = x^3 + 7
-     *  (Y/Z^3)^2 = (X/Z^2)^3 + 7
-     *  Y^2 / Z^6 = X^3 / Z^6 + 7
-     *  Y^2 = X^3 + 7*Z^6
-     */
-    secp256k1_fe_sqr(&y2, &a->y);
-    secp256k1_fe_sqr(&x3, &a->x); secp256k1_fe_mul(&x3, &x3, &a->x);
-    secp256k1_fe_sqr(&z2, &a->z);
-    secp256k1_fe_sqr(&z6, &z2); secp256k1_fe_mul(&z6, &z6, &z2);
-    secp256k1_fe_mul_int(&z6, CURVE_B);
-    secp256k1_fe_add(&x3, &z6);
-    secp256k1_fe_normalize_weak(&x3);
-    return secp256k1_fe_equal_var(&y2, &x3);
-}
-
 static int secp256k1_ge_is_valid_var(const secp256k1_ge *a) {
-    secp256k1_fe y2, x3, c;
+    secp256k1_fe y2, x3;
     if (a->infinity) {
         return 0;
     }
     /* y^2 = x^3 + 7 */
     secp256k1_fe_sqr(&y2, &a->y);
     secp256k1_fe_sqr(&x3, &a->x); secp256k1_fe_mul(&x3, &x3, &a->x);
-    secp256k1_fe_set_int(&c, CURVE_B);
-    secp256k1_fe_add(&x3, &c);
+    secp256k1_fe_add(&x3, &secp256k1_fe_const_b);
     secp256k1_fe_normalize_weak(&x3);
     return secp256k1_fe_equal_var(&y2, &x3);
 }
@@ -704,4 +671,25 @@ static int secp256k1_gej_has_quad_y_var(const secp256k1_gej *a) {
     return secp256k1_fe_is_quad_var(&yz);
 }
 
+static int secp256k1_ge_is_in_correct_subgroup(const secp256k1_ge* ge) {
+#ifdef EXHAUSTIVE_TEST_ORDER
+    secp256k1_gej out;
+    int i;
+
+    /* A very simple EC multiplication ladder that avoids a dependecy on ecmult. */
+    secp256k1_gej_set_infinity(&out);
+    for (i = 0; i < 32; ++i) {
+        secp256k1_gej_double_var(&out, &out, NULL);
+        if ((((uint32_t)EXHAUSTIVE_TEST_ORDER) >> (31 - i)) & 1) {
+            secp256k1_gej_add_ge_var(&out, &out, ge, NULL);
+        }
+    }
+    return secp256k1_gej_is_infinity(&out);
+#else
+    (void)ge;
+    /* The real secp256k1 group has cofactor 1, so the subgroup is the entire curve. */
+    return 1;
+#endif
+}
+
 #endif /* SECP256K1_GROUP_IMPL_H */

src/modules/extrakeys/Makefile.am.include

@@ -1,3 +1,4 @@
 include_HEADERS += include/secp256k1_extrakeys.h
 noinst_HEADERS += src/modules/extrakeys/tests_impl.h
+noinst_HEADERS += src/modules/extrakeys/tests_exhaustive_impl.h
 noinst_HEADERS += src/modules/extrakeys/main_impl.h

src/modules/extrakeys/main_impl.h

@@ -33,6 +33,9 @@ int secp256k1_xonly_pubkey_parse(const secp256k1_context* ctx, secp256k1_xonly_p
     if (!secp256k1_ge_set_xo_var(&pk, &x, 0)) {
         return 0;
     }
+    if (!secp256k1_ge_is_in_correct_subgroup(&pk)) {
+        return 0;
+    }
     secp256k1_xonly_pubkey_save(pubkey, &pk);
     return 1;
 }

src/modules/extrakeys/tests_exhaustive_impl.h

@@ -0,0 +1,68 @@
+/**********************************************************************
+ * Copyright (c) 2020 Pieter Wuille                                   *
+ * Distributed under the MIT software license, see the accompanying   *
+ * file COPYING or http://www.opensource.org/licenses/mit-license.php.*
+ **********************************************************************/
+
+#ifndef _SECP256K1_MODULE_EXTRAKEYS_TESTS_EXHAUSTIVE_
+#define _SECP256K1_MODULE_EXTRAKEYS_TESTS_EXHAUSTIVE_
+
+#include "src/modules/extrakeys/main_impl.h"
+#include "include/secp256k1_extrakeys.h"
+
+static void test_exhaustive_extrakeys(const secp256k1_context *ctx, const secp256k1_ge* group) {
+    secp256k1_keypair keypair[EXHAUSTIVE_TEST_ORDER - 1];
+    secp256k1_pubkey pubkey[EXHAUSTIVE_TEST_ORDER - 1];
+    secp256k1_xonly_pubkey xonly_pubkey[EXHAUSTIVE_TEST_ORDER - 1];
+    int parities[EXHAUSTIVE_TEST_ORDER - 1];
+    unsigned char xonly_pubkey_bytes[EXHAUSTIVE_TEST_ORDER - 1][32];
+    int i;
+
+    for (i = 1; i < EXHAUSTIVE_TEST_ORDER; i++) {
+        secp256k1_fe fe;
+        secp256k1_scalar scalar_i;
+        unsigned char buf[33];
+        int parity;
+
+        secp256k1_scalar_set_int(&scalar_i, i);
+        secp256k1_scalar_get_b32(buf, &scalar_i);
+
+        /* Construct pubkey and keypair. */
+        CHECK(secp256k1_keypair_create(ctx, &keypair[i - 1], buf));
+        CHECK(secp256k1_ec_pubkey_create(ctx, &pubkey[i - 1], buf));
+
+        /* Construct serialized xonly_pubkey from keypair. */
+        CHECK(secp256k1_keypair_xonly_pub(ctx, &xonly_pubkey[i - 1], &parities[i - 1], &keypair[i - 1]));
+        CHECK(secp256k1_xonly_pubkey_serialize(ctx, xonly_pubkey_bytes[i - 1], &xonly_pubkey[i - 1]));
+
+        /* Parse the xonly_pubkey back and verify it matches the previously serialized value. */
+        CHECK(secp256k1_xonly_pubkey_parse(ctx, &xonly_pubkey[i - 1], xonly_pubkey_bytes[i - 1]));
+        CHECK(secp256k1_xonly_pubkey_serialize(ctx, buf, &xonly_pubkey[i - 1]));
+        CHECK(memcmp(xonly_pubkey_bytes[i - 1], buf, 32) == 0);
+
+        /* Construct the xonly_pubkey from the pubkey, and verify it matches the same. */
+        CHECK(secp256k1_xonly_pubkey_from_pubkey(ctx, &xonly_pubkey[i - 1], &parity, &pubkey[i - 1]));
+        CHECK(parity == parities[i - 1]);
+        CHECK(secp256k1_xonly_pubkey_serialize(ctx, buf, &xonly_pubkey[i - 1]));
+        CHECK(memcmp(xonly_pubkey_bytes[i - 1], buf, 32) == 0);
+
+        /* Compare the xonly_pubkey bytes against the precomputed group. */
+        secp256k1_fe_set_b32(&fe, xonly_pubkey_bytes[i - 1]);
+        CHECK(secp256k1_fe_equal_var(&fe, &group[i].x));
+
+        /* Check the parity against the precomputed group. */
+        fe = group[i].y;
+        secp256k1_fe_normalize_var(&fe);
+        CHECK(secp256k1_fe_is_odd(&fe) == parities[i - 1]);
+
+        /* Verify that the higher half is identical to the lower half mirrored. */
+        if (i > EXHAUSTIVE_TEST_ORDER / 2) {
+            CHECK(memcmp(xonly_pubkey_bytes[i - 1], xonly_pubkey_bytes[EXHAUSTIVE_TEST_ORDER - i - 1], 32) == 0);
+            CHECK(parities[i - 1] == 1 - parities[EXHAUSTIVE_TEST_ORDER - i - 1]);
+        }
+    }
+
+    /* TODO: keypair/xonly_pubkey tweak tests */
+}
+
+#endif