Fix test failures with TLS 1.3-capable OpenSSL

Rakefile

@@ -58,11 +58,12 @@ namespace :sync do
 
     paths = [
       ["ext/openssl/", "ext/openssl/"],
+      ["lib/", "ext/openssl/lib/"],
+      ["sample/", "sample/openssl/"],
+      ["test/fixtures/", "test/openssl/fixtures/"],
       ["test/utils.rb", "test/openssl/"],
       ["test/ut_eof.rb", "test/openssl/"],
       ["test/test_*", "test/openssl/"],
-      ["lib/", "ext/openssl/lib/"],
-      ["sample/", "sample/openssl/"],
       ["History.md", "ext/openssl/"],
     ]
     paths.each do |src, dst|

appveyor.yml

@@ -17,9 +17,9 @@ install:
   - openssl version
   - rake install_dependencies
 build_script:
-  - rake -rdevkit compile -- --with-openssl-dir=%openssl_dir%
+  - rake -rdevkit compile -- --with-openssl-dir=%openssl_dir% --enable-debug
 test_script:
-  - rake test
+  - rake test OSSL_MDEBUG=1
 deploy: off
 environment:
   matrix:

docker-compose.yml

@@ -3,11 +3,10 @@ compile: &defaults
   environment:
     RUBY_VERSION:
     OPENSSL_VERSION:
-    MDEBUG:
   command: rake compile
 test:
   <<: *defaults
-  command: rake compile test
+  command: rake compile test OSSL_MDEBUG=1 -- --enable-debug
 debug:
   <<: *defaults
   command: /bin/bash

ext/openssl/ossl.c

@@ -414,6 +414,72 @@ ossl_fips_mode_set(VALUE self, VALUE enabled)
 #endif
 }
 
+#if defined(OSSL_DEBUG)
+#if !defined(LIBRESSL_VERSION_NUMBER) && \
+    (OPENSSL_VERSION_NUMBER >= 0x10100000 && !defined(OPENSSL_NO_CRYPTO_MDEBUG) || \
+     defined(CRYPTO_malloc_debug_init))
+/*
+ * call-seq:
+ *   OpenSSL.mem_check_start -> nil
+ *
+ * Calls CRYPTO_mem_ctrl(CRYPTO_MEM_CHECK_ON). Starts tracking memory
+ * allocations. See also OpenSSL.print_mem_leaks.
+ *
+ * This is available only when built with a capable OpenSSL and --enable-debug
+ * configure option.
+ */
+static VALUE
+mem_check_start(VALUE self)
+{
+	CRYPTO_mem_ctrl(CRYPTO_MEM_CHECK_ON);
+	return Qnil;
+}
+
+/*
+ * call-seq:
+ *   OpenSSL.print_mem_leaks -> true | false
+ *
+ * For debugging the Ruby/OpenSSL library. Calls CRYPTO_mem_leaks_fp(stderr).
+ * Prints detected memory leaks to standard error. This cleans the global state
+ * up thus you cannot use any methods of the library after calling this.
+ *
+ * Returns true if leaks detected, false otherwise.
+ *
+ * This is available only when built with a capable OpenSSL and --enable-debug
+ * configure option.
+ *
+ * === Example
+ *   OpenSSL.mem_check_start
+ *   NOT_GCED = OpenSSL::PKey::RSA.new(256)
+ *
+ *   END {
+ *     GC.start
+ *     OpenSSL.print_mem_leaks # will print the leakage
+ *   }
+ */
+static VALUE
+print_mem_leaks(VALUE self)
+{
+#if OPENSSL_VERSION_NUMBER >= 0x10100000
+    int ret;
+#endif
+
+    BN_CTX_free(ossl_bn_ctx);
+    ossl_bn_ctx = NULL;
+
+#if OPENSSL_VERSION_NUMBER >= 0x10100000
+    ret = CRYPTO_mem_leaks_fp(stderr);
+    if (ret < 0)
+	ossl_raise(eOSSLError, "CRYPTO_mem_leaks_fp");
+    return ret ? Qfalse : Qtrue;
+#else
+    CRYPTO_mem_leaks_fp(stderr);
+    return Qnil;
+#endif
+}
+#endif
+#endif
+
 #if !defined(HAVE_OPENSSL_110_THREADING_API)
 /**
  * Stores locks needed for OpenSSL thread safety
@@ -1114,15 +1180,40 @@ Init_openssl(void)
     Init_ossl_ocsp();
     Init_ossl_engine();
     Init_ossl_asn1();
-}
 
 #if defined(OSSL_DEBUG)
-/*
- * Check if all symbols are OK with 'make LDSHARED=gcc all'
- */
-int
-main(int argc, char *argv[])
-{
-    return 0;
+    /*
+     * For debugging Ruby/OpenSSL. Enable only when built with --enable-debug
+     */
+#if !defined(LIBRESSL_VERSION_NUMBER) && \
+    (OPENSSL_VERSION_NUMBER >= 0x10100000 && !defined(OPENSSL_NO_CRYPTO_MDEBUG) || \
+     defined(CRYPTO_malloc_debug_init))
+    rb_define_module_function(mOSSL, "mem_check_start", mem_check_start, 0);
+    rb_define_module_function(mOSSL, "print_mem_leaks", print_mem_leaks, 0);
+
+#if defined(CRYPTO_malloc_debug_init) /* <= 1.0.2 */
+    CRYPTO_malloc_debug_init();
+#endif
+
+#if defined(V_CRYPTO_MDEBUG_ALL) /* <= 1.0.2 */
+    CRYPTO_set_mem_debug_options(V_CRYPTO_MDEBUG_ALL);
+#endif
+
+#if OPENSSL_VERSION_NUMBER < 0x10100000 /* <= 1.0.2 */
+    {
+	int i;
+	/*
+	 * See crypto/ex_data.c; call def_get_class() immediately to avoid
+	 * allocations. 15 is the maximum number that is used as the class index
+	 * in OpenSSL 1.0.2.
+	 */
+	for (i = 0; i <= 15; i++) {
+	    if (CRYPTO_get_ex_new_index(i, 0, (void *)"ossl-mdebug-dummy", 0, 0, 0) < 0)
+		rb_raise(rb_eRuntimeError, "CRYPTO_get_ex_new_index for "
+			 "class index %d failed", i);
+	}
+    }
+#endif
+#endif
+#endif
 }
-#endif /* OSSL_DEBUG */

test/fixtures/pkey/dh1024.pem

@@ -0,0 +1,5 @@
+-----BEGIN DH PARAMETERS-----
+MIGHAoGBAKnKQ8MNK6nYZzLrrcuTsLxuiJGXoOO5gT+tljOTbHBuiktdMTITzIY0
+pFxIvjG05D7HoBZQfrR0c92NGWPkAiCkhQKB8JCbPVzwNLDy6DZ0pmofDKrEsYHG
+AQjjxMXhwULlmuR/K+WwlaZPiLIBYalLAZQ7ZbOPeVkJ8ePao0eLAgEC
+-----END DH PARAMETERS-----

test/fixtures/pkey/dsa1024.pem

@@ -0,0 +1,12 @@
+-----BEGIN DSA PRIVATE KEY-----
+MIIBugIBAAKBgQCH9aAoXvWWThIjkA6D+nI1F9ksF9iDq594rkiGNOT9sPDOdB+n
+D+qeeeeloRlj19ymCSADPI0ZLRgkchkAEnY2RnqnhHOjVf/roGgRbW+iQDMbQ9wa
+/pvc6/fAbsu1goE1hBYjm98/sZEeXavj8tR56IXnjF1b6Nx0+sgeUKFKEQIVAMiz
+4BJUFeTtddyM4uadBM7HKLPRAoGAZdLBSYNGiij7vAjesF5mGUKTIgPd+JKuBEDx
+OaBclsgfdoyoF/TMOkIty+PVlYD+//Vl2xnoUEIRaMXHwHfm0r2xUX++oeRaSScg
+YizJdUxe5jvBuBszGPRc/mGpb9YvP0sB+FL1KmuxYmdODfCe51zl8uM/CVhouJ3w
+DjmRGscCgYAuFlfC7p+e8huCKydfcv/beftqjewiOPpQ3u5uI6KPCtCJPpDhs3+4
+IihH2cPsAlqwGF4tlibW1+/z/OZ1AZinPK3y7b2jSJASEaPeEltVzB92hcd1khk2
+jTYcmSsV4VddplOPK9czytR/GbbibxsrhhgZUbd8LPbvIgaiadJ1PgIUBnJ/5vN2
+CVArsEzlPUCbohPvZnE=
+-----END DSA PRIVATE KEY-----

test/fixtures/pkey/dsa256.pem

@@ -0,0 +1,8 @@
+-----BEGIN DSA PRIVATE KEY-----
+MIH3AgEAAkEAhk2libbY2a8y2Pt21+YPYGZeW6wzaW2yfj5oiClXro9XMR7XWLkE
+9B7XxLNFCS2gmCCdMsMW1HulaHtLFQmB2wIVAM43JZrcgpu6ajZ01VkLc93gu/Ed
+AkAOhujZrrKV5CzBKutKLb0GVyVWmdC7InoNSMZEeGU72rT96IjM59YzoqmD0pGM
+3I1o4cGqg1D1DfM1rQlnN1eSAkBq6xXfEDwJ1mLNxF6q8Zm/ugFYWR5xcX/3wFiT
+b4+EjHP/DbNh9Vm5wcfnDBJ1zKvrMEf2xqngYdrV/3CiGJeKAhRvL57QvJZcQGvn
+ISNX5cMzFHRW3Q==
+-----END DSA PRIVATE KEY-----

test/fixtures/pkey/dsa512.pem

@@ -0,0 +1,8 @@
+-----BEGIN DSA PRIVATE KEY-----
+MIH4AgEAAkEA5lB4GvEwjrsMlGDqGsxrbqeFRh6o9OWt6FgTYiEEHaOYhkIxv0Ok
+RZPDNwOG997mDjBnvDJ1i56OmS3MbTnovwIVAJgub/aDrSDB4DZGH7UyarcaGy6D
+AkB9HdFw/3td8K4l1FZHv7TCZeJ3ZLb7dF3TWoGUP003RCqoji3/lHdKoVdTQNuR
+S/m6DlCwhjRjiQ/lBRgCLCcaAkEAjN891JBjzpMj4bWgsACmMggFf57DS0Ti+5++
+Q1VB8qkJN7rA7/2HrCR3gTsWNb1YhAsnFsoeRscC+LxXoXi9OAIUBG98h4tilg6S
+55jreJD3Se3slps=
+-----END DSA PRIVATE KEY-----

test/fixtures/pkey/p256.pem

@@ -0,0 +1,5 @@
+-----BEGIN EC PRIVATE KEY-----
+MHcCAQEEIID49FDqcf1O1eO8saTgG70UbXQw9Fqwseliit2aWhH1oAoGCCqGSM49
+AwEHoUQDQgAEFglk2c+oVUIKQ64eZG9bhLNPWB7lSZ/ArK41eGy5wAzU/0G51Xtt
+CeBUl+MahZtn9fO1JKdF4qJmS39dXnpENg==
+-----END EC PRIVATE KEY-----

test/fixtures/pkey/rsa1024.pem

@@ -0,0 +1,15 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIICXgIBAAKBgQDLwsSw1ECnPtT+PkOgHhcGA71nwC2/nL85VBGnRqDxOqjVh7Cx
+aKPERYHsk4BPCkE3brtThPWc9kjHEQQ7uf9Y1rbCz0layNqHyywQEVLFmp1cpIt/
+Q3geLv8ZD9pihowKJDyMDiN6ArYUmZczvW4976MU3+l54E6lF/JfFEU5hwIDAQAB
+AoGBAKSl/MQarye1yOysqX6P8fDFQt68VvtXkNmlSiKOGuzyho0M+UVSFcs6k1L0
+maDE25AMZUiGzuWHyaU55d7RXDgeskDMakD1v6ZejYtxJkSXbETOTLDwUWTn618T
+gnb17tU1jktUtU67xK/08i/XodlgnQhs6VoHTuCh3Hu77O6RAkEA7+gxqBuZR572
+74/akiW/SuXm0SXPEviyO1MuSRwtI87B02D0qgV8D1UHRm4AhMnJ8MCs1809kMQE
+JiQUCrp9mQJBANlt2ngBO14us6NnhuAseFDTBzCHXwUUu1YKHpMMmxpnGqaldGgX
+sOZB3lgJsT9VlGf3YGYdkLTNVbogQKlKpB8CQQDiSwkb4vyQfDe8/NpU5Not0fII
+8jsDUCb+opWUTMmfbxWRR3FBNu8wnym/m19N4fFj8LqYzHX4KY0oVPu6qvJxAkEA
+wa5snNekFcqONLIE4G5cosrIrb74sqL8GbGb+KuTAprzj5z1K8Bm0UW9lTjVDjDi
+qRYgZfZSL+x1P/54+xTFSwJAY1FxA/N3QPCXCjPh5YqFxAMQs2VVYTfg+t0MEcJD
+dPMQD5JX6g5HKnHFg2mZtoXQrWmJSn7p8GJK8yNTopEErA==
+-----END RSA PRIVATE KEY-----

test/fixtures/pkey/rsa2048.pem

@@ -0,0 +1,27 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIEpAIBAAKCAQEAuV9ht9J7k4NBs38jOXvvTKY9gW8nLICSno5EETR1cuF7i4pN
+s9I1QJGAFAX0BEO4KbzXmuOvfCpD3CU+Slp1enenfzq/t/e/1IRW0wkJUJUFQign
+4CtrkJL+P07yx18UjyPlBXb81ApEmAB5mrJVSrWmqbjs07JbuS4QQGGXLc+Su96D
+kYKmSNVjBiLxVVSpyZfAY3hD37d60uG+X8xdW5v68JkRFIhdGlb6JL8fllf/A/bl
+NwdJOhVr9mESHhwGjwfSeTDPfd8ZLE027E5lyAVX9KZYcU00mOX+fdxOSnGqS/8J
+DRh0EPHDL15RcJjV2J6vZjPb0rOYGDoMcH+94wIDAQABAoIBAAzsamqfYQAqwXTb
+I0CJtGg6msUgU7HVkOM+9d3hM2L791oGHV6xBAdpXW2H8LgvZHJ8eOeSghR8+dgq
+PIqAffo4x1Oma+FOg3A0fb0evyiACyrOk+EcBdbBeLo/LcvahBtqnDfiUMQTpy6V
+seSoFCwuN91TSCeGIsDpRjbG1vxZgtx+uI+oH5+ytqJOmfCksRDCkMglGkzyfcl0
+Xc5CUhIJ0my53xijEUQl19rtWdMnNnnkdbG8PT3LZlOta5Do86BElzUYka0C6dUc
+VsBDQ0Nup0P6rEQgy7tephHoRlUGTYamsajGJaAo1F3IQVIrRSuagi7+YpSpCqsW
+wORqorkCgYEA7RdX6MDVrbw7LePnhyuaqTiMK+055/R1TqhB1JvvxJ1CXk2rDL6G
+0TLHQ7oGofd5LYiemg4ZVtWdJe43BPZlVgT6lvL/iGo8JnrncB9Da6L7nrq/+Rvj
+XGjf1qODCK+LmreZWEsaLPURIoR/Ewwxb9J2zd0CaMjeTwafJo1CZvcCgYEAyCgb
+aqoWvUecX8VvARfuA593Lsi50t4MEArnOXXcd1RnXoZWhbx5rgO8/ATKfXr0BK/n
+h2GF9PfKzHFm/4V6e82OL7gu/kLy2u9bXN74vOvWFL5NOrOKPM7Kg+9I131kNYOw
+Ivnr/VtHE5s0dY7JChYWE1F3vArrOw3T00a4CXUCgYEA0SqY+dS2LvIzW4cHCe9k
+IQqsT0yYm5TFsUEr4sA3xcPfe4cV8sZb9k/QEGYb1+SWWZ+AHPV3UW5fl8kTbSNb
+v4ng8i8rVVQ0ANbJO9e5CUrepein2MPL0AkOATR8M7t7dGGpvYV0cFk8ZrFx0oId
+U0PgYDotF/iueBWlbsOM430CgYEAqYI95dFyPI5/AiSkY5queeb8+mQH62sdcCCr
+vd/w/CZA/K5sbAo4SoTj8dLk4evU6HtIa0DOP63y071eaxvRpTNqLUOgmLh+D6gS
+Cc7TfLuFrD+WDBatBd5jZ+SoHccVrLR/4L8jeodo5FPW05A+9gnKXEXsTxY4LOUC
+9bS4e1kCgYAqVXZh63JsMwoaxCYmQ66eJojKa47VNrOeIZDZvd2BPVf30glBOT41
+gBoDG3WMPZoQj9pb7uMcrnvs4APj2FIhMU8U15LcPAj59cD6S6rWnAxO8NFK7HQG
+4Jxg3JNNf8ErQoCHb1B3oVdXJkmbJkARoDpBKmTCgKtP8ADYLmVPQw==
+-----END RSA PRIVATE KEY-----

test/test_asn1.rb

@@ -1,10 +1,12 @@
 # frozen_string_literal: false
 require_relative 'utils'
 
+if defined?(OpenSSL::TestUtils)
+
 class  OpenSSL::TestASN1 < OpenSSL::TestCase
   def test_decode
     subj = OpenSSL::X509::Name.parse("/DC=org/DC=ruby-lang/CN=TestCA")
-    key = OpenSSL::TestUtils::TEST_KEY_RSA1024
+    key = Fixtures.pkey("rsa1024")
     now = Time.at(Time.now.to_i) # suppress usec
     s = 0xdeadbeafdeadbeafdeadbeafdeadbeaf
     exts = [
@@ -582,5 +584,6 @@ def assert_universal(tag, asn1)
     end
     assert_equal(:UNIVERSAL, asn1.tag_class)
   end
+end
 
-end if defined?(OpenSSL::TestUtils)
+end

test/test_buffering.rb

@@ -1,9 +1,9 @@
 # frozen_string_literal: false
 require_relative 'utils'
-require 'stringio'
 
-class OpenSSL::TestBuffering < OpenSSL::TestCase
+if defined?(OpenSSL::TestUtils)
 
+class OpenSSL::TestBuffering < OpenSSL::TestCase
   class IO
     include OpenSSL::Buffering
 
@@ -85,5 +85,6 @@ def test_each_byte
     end
     assert_equal([97, 98, 99], res)
   end
+end
 
-end if defined?(OpenSSL::TestUtils)
+end

test/test_config.rb

@@ -1,6 +1,8 @@
 # frozen_string_literal: false
 require_relative 'utils'
 
+if defined?(OpenSSL::TestUtils)
+
 class OpenSSL::TestConfig < OpenSSL::TestCase
   def setup
     super
@@ -171,7 +173,7 @@ def test_get_value_ENV
 
   def test_value
     # suppress deprecation warnings
-    OpenSSL::TestUtils.silent do
+    EnvUtil.suppress_warning do
       assert_equal('CA_default', @it.value('ca', 'default_ca'))
       assert_equal(nil, @it.value('ca', 'no such key'))
       assert_equal(nil, @it.value('no such section', 'no such key'))
@@ -184,7 +186,7 @@ def test_value
   end
 
   def test_value_ENV
-    OpenSSL::TestUtils.silent do
+    EnvUtil.suppress_warning do
       key = ENV.keys.first
       assert_not_nil(key) # make sure we have at least one ENV var.
       assert_equal(ENV[key], @it.value('ENV', key))
@@ -199,7 +201,7 @@ def test_aref
   end
 
   def test_section
-    OpenSSL::TestUtils.silent do
+    EnvUtil.suppress_warning do
       assert_equal({'HOME' => '.'}, @it.section('default'))
       assert_equal({'dir' => './demoCA', 'certs' => './certs'}, @it.section('CA_default'))
       assert_equal({}, @it.section('no_such_section'))
@@ -297,4 +299,6 @@ def test_clone
     @it['newsection'] = {'a' => 'b'}
     assert_not_equal(@it.sections.sort, c.sections.sort)
   end
-end if defined?(OpenSSL::TestUtils)
+end
+
+end