bn: check -1 return from BIGNUM functions

Although the manpage says that BIGNUM functions return 0 on error, OpenSSL versions before 1.0.2n and current LibreSSL versions may return -1 instead. Note that the implementation of OpenSSL::BN#mod_inverse is extracted from BIGNUM_2c() macro as it didn't really share the same function signature with others.
ruby · Feb 18, 2021 · 9b59f34 · 9b59f34
1 parent b3972a7
commit 9b59f34
Showing 1 changed file with 22 additions and 12 deletions.
diff --git a/ext/openssl/ossl_bn.c b/ext/openssl/ossl_bn.c
@@ -397,7 +397,7 @@ ossl_bn_is_negative(VALUE self)
 	if (!(result = BN_new())) {			\
 	    ossl_raise(eBNError, NULL);			\
 	}						\
-	if (!BN_##func(result, bn, ossl_bn_ctx)) {	\
+	if (BN_##func(result, bn, ossl_bn_ctx) <= 0) {	\
 	    BN_free(result);				\
 	    ossl_raise(eBNError, NULL);			\
 	}						\
@@ -423,7 +423,7 @@ BIGNUM_1c(sqr)
 	if (!(result = BN_new())) {			\
 	    ossl_raise(eBNError, NULL);			\
 	}						\
-	if (!BN_##func(result, bn1, bn2)) {		\
+	if (BN_##func(result, bn1, bn2) <= 0) {		\
 	    BN_free(result);				\
 	    ossl_raise(eBNError, NULL);			\
 	}						\
@@ -456,7 +456,7 @@ BIGNUM_2(sub)
 	if (!(result = BN_new())) {				\
 	    ossl_raise(eBNError, NULL);				\
 	}							\
-	if (!BN_##func(result, bn1, bn2, ossl_bn_ctx)) {	\
+	if (BN_##func(result, bn1, bn2, ossl_bn_ctx) <= 0) {	\
 	    BN_free(result);					\
 	    ossl_raise(eBNError, NULL);				\
 	}							\
@@ -500,11 +500,21 @@ BIGNUM_2c(gcd)
 BIGNUM_2c(mod_sqr)
 
 /*
- * Document-method: OpenSSL::BN#mod_inverse
  * call-seq:
- *   bn.mod_inverse(bn2) => aBN
+ *    bn.mod_inverse(bn2) => aBN
  */
-BIGNUM_2c(mod_inverse)
+static VALUE
+ossl_bn_mod_inverse(VALUE self, VALUE other)
+{
+    BIGNUM *bn1, *bn2 = GetBNPtr(other), *result;
+    VALUE obj;
+    GetBN(self, bn1);
+    obj = NewBN(rb_obj_class(self));
+    if (!(result = BN_mod_inverse(NULL, bn1, bn2, ossl_bn_ctx)))
+        ossl_raise(eBNError, "BN_mod_inverse");
+    SetBN(obj, result);
+    return obj;
+}
 
 /*
  * call-seq:
@@ -553,7 +563,7 @@ ossl_bn_div(VALUE self, VALUE other)
 	if (!(result = BN_new())) {				\
 	    ossl_raise(eBNError, NULL);				\
 	}							\
-	if (!BN_##func(result, bn1, bn2, bn3, ossl_bn_ctx)) {	\
+	if (BN_##func(result, bn1, bn2, bn3, ossl_bn_ctx) <= 0) { \
 	    BN_free(result);					\
 	    ossl_raise(eBNError, NULL);				\
 	}							\
@@ -595,7 +605,7 @@ BIGNUM_3c(mod_exp)
     {							\
 	BIGNUM *bn;					\
 	GetBN(self, bn);				\
-	if (!BN_##func(bn, NUM2INT(bit))) {		\
+	if (BN_##func(bn, NUM2INT(bit)) <= 0) {		\
 	    ossl_raise(eBNError, NULL);			\
 	}						\
 	return self;					\
@@ -655,7 +665,7 @@ ossl_bn_is_bit_set(VALUE self, VALUE bit)
 	if (!(result = BN_new())) {			\
 		ossl_raise(eBNError, NULL);		\
 	}						\
-	if (!BN_##func(result, bn, b)) {		\
+	if (BN_##func(result, bn, b) <= 0) {		\
 		BN_free(result);			\
 		ossl_raise(eBNError, NULL);		\
 	}						\
@@ -685,7 +695,7 @@ BIGNUM_SHIFT(rshift)
 	int b;						\
 	b = NUM2INT(bits);				\
 	GetBN(self, bn);				\
-	if (!BN_##func(bn, bn, b))			\
+	if (BN_##func(bn, bn, b) <= 0)			\
 		ossl_raise(eBNError, NULL);		\
 	return self;					\
     }
@@ -724,7 +734,7 @@ BIGNUM_SELF_SHIFT(rshift)
 	if (!(result = BN_new())) {				\
 	    ossl_raise(eBNError, NULL);				\
 	}							\
-	if (!BN_##func(result, b, top, bottom)) {		\
+	if (BN_##func(result, b, top, bottom) <= 0) {		\
 	    BN_free(result);					\
 	    ossl_raise(eBNError, NULL);				\
 	}							\
@@ -753,7 +763,7 @@ BIGNUM_RAND(pseudo_rand)
 	if (!(result = BN_new())) {				\
 	    ossl_raise(eBNError, NULL);				\
 	}							\
-	if (!BN_##func##_range(result, bn)) {			\
+	if (BN_##func##_range(result, bn) <= 0) {		\
 	    BN_free(result);					\
 	    ossl_raise(eBNError, NULL);				\
 	}							\