Remove OSSL_DEBUG compile-time option

Remove the OSSL_DEBUG flag and OpenSSL.mem_check_start which is only
compiled when the flag is given. They are meant purely for development
of Ruby/OpenSSL.

OpenSSL.mem_check_start helped us find memory leak bugs in past, but
it is no longer working with the recent OpenSSL versions. Let's just
remove it now.

Author: rhenium

.github/workflows/test.yml

@@ -58,10 +58,10 @@ jobs:
         if: runner.os == 'Windows' && matrix.ruby == '3.2'
 
       - name: compile
-        run:  rake compile -- --enable-debug
+        run:  rake compile
 
       - name: test
-        run:  rake test TESTOPTS="-v --no-show-detail-immediately" OSSL_MDEBUG=1
+        run:  rake test TESTOPTS="-v --no-show-detail-immediately"
         timeout-minutes: 5
 
   test-openssls:
@@ -169,10 +169,10 @@ jobs:
         if: ${{ !matrix.skip-warnings }}
 
       - name: compile
-        run:  rake compile -- --enable-debug --with-openssl-dir=$HOME/.openssl/${{ matrix.openssl }}
+        run:  rake compile -- --with-openssl-dir=$HOME/.openssl/${{ matrix.openssl }}
 
       - name: test
-        run:  rake test TESTOPTS="-v --no-show-detail-immediately" OSSL_MDEBUG=1
+        run:  rake test TESTOPTS="-v --no-show-detail-immediately"
         timeout-minutes: 5
         if: ${{ !matrix.fips-enabled }}
 

ext/openssl/extconf.rb

@@ -45,13 +45,6 @@
 
 Logging::message "=== OpenSSL for Ruby configurator ===\n"
 
-##
-# Adds -DOSSL_DEBUG for compilation and some more targets when GCC is used
-# To turn it on, use: --with-debug or --enable-debug
-#
-if with_config("debug") or enable_config("debug")
-  $defs.push("-DOSSL_DEBUG")
-end
 $defs.push("-D""OPENSSL_SUPPRESS_DEPRECATED")
 
 have_func("rb_io_descriptor")

ext/openssl/ossl.c

@@ -463,75 +463,6 @@ ossl_fips_mode_set(VALUE self, VALUE enabled)
 #endif
 }
 
-#if defined(OSSL_DEBUG)
-#if !defined(LIBRESSL_VERSION_NUMBER) && \
-    (OPENSSL_VERSION_NUMBER >= 0x10100000 && !defined(OPENSSL_NO_CRYPTO_MDEBUG) || \
-     defined(CRYPTO_malloc_debug_init))
-/*
- * call-seq:
- *   OpenSSL.mem_check_start -> nil
- *
- * Calls CRYPTO_mem_ctrl(CRYPTO_MEM_CHECK_ON). Starts tracking memory
- * allocations. See also OpenSSL.print_mem_leaks.
- *
- * This is available only when built with a capable OpenSSL and --enable-debug
- * configure option.
- */
-static VALUE
-mem_check_start(VALUE self)
-{
-	CRYPTO_mem_ctrl(CRYPTO_MEM_CHECK_ON);
-	return Qnil;
-}
-
-/*
- * call-seq:
- *   OpenSSL.print_mem_leaks -> true | false
- *
- * For debugging the Ruby/OpenSSL library. Calls CRYPTO_mem_leaks_fp(stderr).
- * Prints detected memory leaks to standard error. This cleans the global state
- * up thus you cannot use any methods of the library after calling this.
- *
- * Returns +true+ if leaks detected, +false+ otherwise.
- *
- * This is available only when built with a capable OpenSSL and --enable-debug
- * configure option.
- *
- * === Example
- *   OpenSSL.mem_check_start
- *   NOT_GCED = OpenSSL::PKey::RSA.new(256)
- *
- *   END {
- *     GC.start
- *     OpenSSL.print_mem_leaks # will print the leakage
- *   }
- */
-static VALUE
-print_mem_leaks(VALUE self)
-{
-#if OPENSSL_VERSION_NUMBER >= 0x10100000
-    int ret;
-#endif
-
-#ifndef HAVE_RB_EXT_RACTOR_SAFE
-    // for Ruby 2.x
-    void ossl_bn_ctx_free(void); // ossl_bn.c
-    ossl_bn_ctx_free();
-#endif
-
-#if OPENSSL_VERSION_NUMBER >= 0x10100000
-    ret = CRYPTO_mem_leaks_fp(stderr);
-    if (ret < 0)
-	ossl_raise(eOSSLError, "CRYPTO_mem_leaks_fp");
-    return ret ? Qfalse : Qtrue;
-#else
-    CRYPTO_mem_leaks_fp(stderr);
-    return Qnil;
-#endif
-}
-#endif
-#endif
-
 #if !defined(HAVE_OPENSSL_110_THREADING_API)
 /**
  * Stores locks needed for OpenSSL thread safety
@@ -1239,40 +1170,4 @@ Init_openssl(void)
     Init_ossl_provider();
     Init_ossl_asn1();
     Init_ossl_kdf();
-
-#if defined(OSSL_DEBUG)
-    /*
-     * For debugging Ruby/OpenSSL. Enable only when built with --enable-debug
-     */
-#if !defined(LIBRESSL_VERSION_NUMBER) && \
-    (OPENSSL_VERSION_NUMBER >= 0x10100000 && !defined(OPENSSL_NO_CRYPTO_MDEBUG) || \
-     defined(CRYPTO_malloc_debug_init))
-    rb_define_module_function(mOSSL, "mem_check_start", mem_check_start, 0);
-    rb_define_module_function(mOSSL, "print_mem_leaks", print_mem_leaks, 0);
-
-#if defined(CRYPTO_malloc_debug_init) /* <= 1.0.2 */
-    CRYPTO_malloc_debug_init();
-#endif
-
-#if defined(V_CRYPTO_MDEBUG_ALL) /* <= 1.0.2 */
-    CRYPTO_set_mem_debug_options(V_CRYPTO_MDEBUG_ALL);
-#endif
-
-#if OPENSSL_VERSION_NUMBER < 0x10100000 /* <= 1.0.2 */
-    {
-	int i;
-	/*
-	 * See crypto/ex_data.c; call def_get_class() immediately to avoid
-	 * allocations. 15 is the maximum number that is used as the class index
-	 * in OpenSSL 1.0.2.
-	 */
-	for (i = 0; i <= 15; i++) {
-	    if (CRYPTO_get_ex_new_index(i, 0, (void *)"ossl-mdebug-dummy", 0, 0, 0) < 0)
-		rb_raise(rb_eRuntimeError, "CRYPTO_get_ex_new_index for "
-			 "class index %d failed", i);
-	}
-    }
-#endif
-#endif
-#endif
 }

test/openssl/test_engine.rb

@@ -82,7 +82,7 @@ def test_openssl_engine_cipher_rc4
 
   # this is required because OpenSSL::Engine methods change global state
   def with_openssl(code, **opts)
-    assert_separately([{ "OSSL_MDEBUG" => nil }, "-ropenssl"], <<~"end;", **opts)
+    assert_separately(["-ropenssl"], <<~"end;", **opts)
       #{code}
     end;
   end

test/openssl/test_fips.rb

@@ -9,7 +9,7 @@ def test_fips_mode_get_is_true_on_fips_mode_enabled
       omit "Only for FIPS mode environment"
     end
 
-    assert_separately([{ "OSSL_MDEBUG" => nil }, "-ropenssl"], <<~"end;")
+    assert_separately(["-ropenssl"], <<~"end;")
       assert OpenSSL.fips_mode == true, ".fips_mode should return true on FIPS mode enabled"
     end;
   end
@@ -19,7 +19,7 @@ def test_fips_mode_get_is_false_on_fips_mode_disabled
       omit "Only for non-FIPS mode environment"
     end
 
-    assert_separately([{ "OSSL_MDEBUG" => nil }, "-ropenssl"], <<~"end;")
+    assert_separately(["-ropenssl"], <<~"end;")
       message = ".fips_mode should return false on FIPS mode disabled. " \
                 "If you run the test on FIPS mode, please set " \
                 "TEST_RUBY_OPENSSL_FIPS_ENABLED=true"
@@ -35,7 +35,7 @@ def test_fips_mode_is_reentrant
   def test_fips_mode_get_with_fips_mode_set
     omit('OpenSSL is not FIPS-capable') unless OpenSSL::OPENSSL_FIPS
 
-    assert_separately([{ "OSSL_MDEBUG" => nil }, "-ropenssl"], <<~"end;")
+    assert_separately(["-ropenssl"], <<~"end;")
       begin
         OpenSSL.fips_mode = true
         assert OpenSSL.fips_mode == true, ".fips_mode should return true when .fips_mode=true"

test/openssl/test_provider.rb

@@ -58,7 +58,7 @@ def test_openssl_legacy_provider
 
   # this is required because OpenSSL::Provider methods change global state
   def with_openssl(code, **opts)
-    assert_separately([{ "OSSL_MDEBUG" => nil }, "-ropenssl"], <<~"end;", **opts)
+    assert_separately(["-ropenssl"], <<~"end;", **opts)
       #{code}
     end;
   end

test/openssl/utils.rb

@@ -4,26 +4,6 @@
 rescue LoadError
 end
 
-# Compile OpenSSL with crypto-mdebug and run this test suite with OSSL_MDEBUG=1
-# environment variable to enable memory leak check.
-if ENV["OSSL_MDEBUG"] == "1"
-  if OpenSSL.respond_to?(:print_mem_leaks)
-    OpenSSL.mem_check_start
-
-    END {
-      GC.start
-      case OpenSSL.print_mem_leaks
-      when nil
-        warn "mdebug: check what is printed"
-      when true
-        raise "mdebug: memory leaks detected"
-      end
-    }
-  else
-    warn "OSSL_MDEBUG=1 is specified but OpenSSL is not built with crypto-mdebug"
-  end
-end
-
 require "test/unit"
 require "core_assertions"
 require "tempfile"