bn/bn_lib.c: add BN_FLG_FIXED_TOP flag.

The new flag marks vectors that were not treated with bn_correct_top,
in other words such vectors are permitted to be zero padded. For now
it's BN_DEBUG-only flag, as initial use case for zero-padded vectors
would be controlled Montgomery multiplication/exponentiation, not
general purpose. For general purpose use another type might be more
appropriate. Advantage of this suggestion is that it's possible to
back-port it...

bn/bn_div.c: fix memory sanitizer problem.
bn/bn_sqr.c: harmonize with BN_mul.

Reviewed-by: Rich Salz <rsalz@openssl.org>
(Merged from #6810)

(cherry picked from commit 305b68f)

Resolved conflicts:
	crypto/bn/bn_lcl.h
	crypto/bn/bn_lib.c

crypto/bn/bn.h

@@ -824,6 +824,16 @@ BIGNUM *bn_dup_expand(const BIGNUM *a, int words); /* unused */
 /* We only need assert() when debugging */
 #  include <assert.h>
 
+/*
+ * The new BN_FLG_FIXED_TOP flag marks vectors that were not treated with
+ * bn_correct_top, in other words such vectors are permitted to have zeros
+ * in most significant limbs. Such vectors are used internally to achieve
+ * execution time invariance for critical operations with private keys.
+ * It's BN_DEBUG-only flag, because user application is not supposed to
+ * observe it anyway. Moreover, optimizing compiler would actually remove
+ * all operations manipulating the bit in question in non-BN_DEBUG build.
+ */
+#  define BN_FLG_FIXED_TOP 0x10000
 #  ifdef BN_DEBUG_RAND
 /* To avoid "make update" cvs wars due to BN_DEBUG, use some tricks */
 #   ifndef RAND_pseudo_bytes
@@ -856,8 +866,10 @@ int RAND_pseudo_bytes(unsigned char *buf, int num);
         do { \
                 const BIGNUM *_bnum2 = (a); \
                 if (_bnum2 != NULL) { \
-                        assert((_bnum2->top == 0) || \
-                                (_bnum2->d[_bnum2->top - 1] != 0)); \
+                        int _top = _bnum2->top; \
+                        assert((_top == 0) || \
+                               (_bnum2->flags & BN_FLG_FIXED_TOP) || \
+                               (_bnum2->d[_top - 1] != 0)); \
                         bn_pollute(_bnum2); \
                 } \
         } while(0)
@@ -875,6 +887,7 @@ int RAND_pseudo_bytes(unsigned char *buf, int num);
 
 # else                          /* !BN_DEBUG */
 
+#  define BN_FLG_FIXED_TOP 0
 #  define bn_pollute(a)
 #  define bn_check_top(a)
 #  define bn_fix_top(a)           bn_correct_top(a)

crypto/bn/bn_div.c

@@ -290,6 +290,7 @@ int BN_div(BIGNUM *dv, BIGNUM *rm, const BIGNUM *num, const BIGNUM *divisor,
     wnum.neg = 0;
     wnum.d = &(snum->d[loop]);
     wnum.top = div_n;
+    wnum.flags = BN_FLG_STATIC_DATA;
     /*
      * only needed when BN_ucmp messes up the values between top and max
      */

crypto/bn/bn_lib.c

@@ -492,14 +492,16 @@ BIGNUM *BN_copy(BIGNUM *a, const BIGNUM *b)
     memcpy(a->d, b->d, sizeof(b->d[0]) * b->top);
 #endif
 
-    a->top = b->top;
     a->neg = b->neg;
+    a->top = b->top;
+    a->flags |= b->flags & BN_FLG_FIXED_TOP;
     bn_check_top(a);
     return (a);
 }
 
 #define FLAGS_DATA(flags) ((flags) & (BN_FLG_STATIC_DATA \
-                                    | BN_FLG_CONSTTIME))
+                                    | BN_FLG_CONSTTIME   \
+                                    | BN_FLG_FIXED_TOP))
 #define FLAGS_STRUCT(flags) ((flags) & (BN_FLG_MALLOCED))
 
 void BN_swap(BIGNUM *a, BIGNUM *b)
@@ -542,6 +544,7 @@ void BN_clear(BIGNUM *a)
         OPENSSL_cleanse(a->d, a->dmax * sizeof(a->d[0]));
     a->top = 0;
     a->neg = 0;
+    a->flags &= ~BN_FLG_FIXED_TOP;
 }
 
 BN_ULONG BN_get_word(const BIGNUM *a)
@@ -562,6 +565,7 @@ int BN_set_word(BIGNUM *a, BN_ULONG w)
     a->neg = 0;
     a->d[0] = w;
     a->top = (w ? 1 : 0);
+    a->flags &= ~BN_FLG_FIXED_TOP;
     bn_check_top(a);
     return (1);
 }
@@ -708,6 +712,7 @@ int BN_set_bit(BIGNUM *a, int n)
         for (k = a->top; k < i + 1; k++)
             a->d[k] = 0;
         a->top = i + 1;
+        a->flags &= ~BN_FLG_FIXED_TOP;
     }
 
     a->d[i] |= (((BN_ULONG)1) << j);

crypto/bn/bn_sqr.c

@@ -135,14 +135,8 @@ int BN_sqr(BIGNUM *r, const BIGNUM *a, BN_CTX *ctx)
     }
 
     rr->neg = 0;
-    /*
-     * If the most-significant half of the top word of 'a' is zero, then the
-     * square of 'a' will max-1 words.
-     */
-    if (a->d[al - 1] == (a->d[al - 1] & BN_MASK2l))
-        rr->top = max - 1;
-    else
-        rr->top = max;
+    rr->top = max;
+    bn_correct_top(rr);
     if (r != rr && BN_copy(r, rr) == NULL)
         goto err;