[ruby/openssl] Add OpenSSL::BN#set_flags and #get_flags

Also, OpenSSL::BN::CONSTTIME is added.

OpenSSL itself had a feature that was vulnerable against a side-channel
attack. The OpenSSL authors determined that it was not a security issue,
and they have already fixed the issue by using BN_set_flags.

openssl/openssl#13888

If a Ruby OpenSSL user was faced with a similar issue, they couldn't
prevent the issue because Ruby OpenSSL lacks a wrapper to BN_set_flags.
For the case, this change introduces the wrapper.

ruby/openssl@1e565eba89

ext/openssl/ossl_bn.c

@@ -1190,6 +1190,42 @@ ossl_bn_is_prime_fasttest(int argc, VALUE *argv, VALUE self)
     return Qnil;
 }
 
+/*
+ * call-seq:
+ *    bn.get_flags(flags) => flags
+ *
+ * Returns the flags on the BN object.
+ * The argument is used as a bit mask.
+ *
+ * === Parameters
+ * * _flags_ - integer
+ */
+static VALUE
+ossl_bn_get_flags(VALUE self, VALUE arg)
+{
+    BIGNUM *bn;
+    GetBN(self, bn);
+
+    return INT2NUM(BN_get_flags(bn, NUM2INT(arg)));
+}
+
+/*
+ * call-seq:
+ *    bn.set_flags(flags) => nil
+ *
+ * Enables the flags on the BN object.
+ * Currently, the flags argument can contain zero of OpenSSL::BN::CONSTTIME.
+ */
+static VALUE
+ossl_bn_set_flags(VALUE self, VALUE arg)
+{
+    BIGNUM *bn;
+    GetBN(self, bn);
+
+    BN_set_flags(bn, NUM2INT(arg));
+    return Qnil;
+}
+
 /*
  * INIT
  * (NOTE: ordering of methods is the same as in 'man bn')
@@ -1289,6 +1325,23 @@ Init_ossl_bn(void)
     /* lshift1 - DON'T IMPL. */
     /* rshift1 - DON'T IMPL. */
 
+    rb_define_method(cBN, "get_flags", ossl_bn_get_flags, 1);
+    rb_define_method(cBN, "set_flags", ossl_bn_set_flags, 1);
+
+#ifdef BN_FLG_CONSTTIME
+    rb_define_const(cBN, "CONSTTIME", INT2NUM(BN_FLG_CONSTTIME));
+#endif
+    /* BN_FLG_MALLOCED and BN_FLG_STATIC_DATA seems for C programming.
+     * Allowing them leads to memory leak.
+     * So, for now, they are not exported
+#ifdef BN_FLG_MALLOCED
+    rb_define_const(cBN, "MALLOCED", INT2NUM(BN_FLG_MALLOCED));
+#endif
+#ifdef BN_FLG_STATIC_DATA
+    rb_define_const(cBN, "STATIC_DATA", INT2NUM(BN_FLG_STATIC_DATA));
+#endif
+    */
+
     /*
      * bn2bin
      * bin2bn

test/openssl/test_bn.rb

@@ -307,6 +307,29 @@ def test_argument_error
     bug15760 = '[ruby-core:92231] [Bug #15760]'
     assert_raise(ArgumentError, bug15760) { OpenSSL::BN.new(nil, 2) }
   end
+
+  def test_get_flags_and_set_flags
+    e = OpenSSL::BN.new(999)
+
+    assert_equal(0, e.get_flags(OpenSSL::BN::CONSTTIME))
+
+    e.set_flags(OpenSSL::BN::CONSTTIME)
+    assert_equal(OpenSSL::BN::CONSTTIME, e.get_flags(OpenSSL::BN::CONSTTIME))
+
+    b = OpenSSL::BN.new(2)
+    m = OpenSSL::BN.new(99)
+    assert_equal("17", b.mod_exp(e, m).to_s)
+
+    # mod_exp fails when m is even and any argument has CONSTTIME flag
+    m = OpenSSL::BN.new(98)
+    assert_raise(OpenSSL::BNError) do
+      b.mod_exp(e, m)
+    end
+
+    # It looks like flags cannot be removed once enabled
+    e.set_flags(0)
+    assert_equal(4, e.get_flags(OpenSSL::BN::CONSTTIME))
+  end
 end
 
 end