s2n_rand_cleanup: be sure to unregister s2n RAND engine from libcrypto (

#3966)

`RAND_bytes()` would fail if called after `s2n_rand_cleanup()`. `libcrypto`
internally keeps an RAND "engine table". Calling `RAND_set_rand_engine(NULL)`
alone isn't enough to remove s2n's RAND engine from the engine table. It needs
to be explicitly unregistered from there by calling `ENGINE_unregister_RAND()`.
This patch adds a call to `ENGINE_unregister_RAND()`, so that after
`s2n_rand_cleanup()` libcrypto's RAND points back to its builtin RAND engine.

Signed-off-by: Rivers Zhang <hzhang320@bloomberg.net>
Co-authored-by: Lindsay Stewart <slindsay@amazon.com>

tests/unit/s2n_random_test.c

@@ -24,6 +24,7 @@
 
 #include "utils/s2n_random.h"
 
+#include <openssl/rand.h>
 #include <pthread.h>
 #include <stdlib.h>
 #include <sys/wait.h>
@@ -727,6 +728,18 @@ static int s2n_random_test_case_failure_cb(struct random_test_case *test_case)
     return EXIT_SUCCESS;
 }
 
+static int s2n_random_rand_bytes_after_cleanup_cb(struct random_test_case *test_case)
+{
+    s2n_disable_atexit();
+    EXPECT_SUCCESS(s2n_init());
+    EXPECT_SUCCESS(s2n_cleanup());
+
+    unsigned char rndbytes[16];
+    EXPECT_EQUAL(RAND_bytes(rndbytes, sizeof(rndbytes)), 1);
+
+    return S2N_SUCCESS;
+}
+
 struct random_test_case random_test_cases[] = {
     { "Random API.", s2n_random_test_case_default_cb, CLONE_TEST_DETERMINE_AT_RUNTIME, EXIT_SUCCESS },
     { "Random API without prediction resistance.", s2n_random_test_case_without_pr_cb, CLONE_TEST_DETERMINE_AT_RUNTIME, EXIT_SUCCESS },
@@ -737,6 +750,7 @@ struct random_test_case random_test_cases[] = {
      * to use 1 below and in s2n_random_test_case_failure_cb().
      */
     { "Test failure.", s2n_random_test_case_failure_cb, CLONE_TEST_DETERMINE_AT_RUNTIME, 1 },
+    { "Test libcrypto's RAND engine is reset correctly after manual s2n_cleanup()", s2n_random_rand_bytes_after_cleanup_cb, CLONE_TEST_DETERMINE_AT_RUNTIME, EXIT_SUCCESS },
 };
 
 int main(int argc, char **argv)

utils/s2n_random.c

@@ -440,6 +440,7 @@ S2N_RESULT s2n_rand_cleanup(void)
     if (rand_engine) {
         ENGINE_remove(rand_engine);
         ENGINE_finish(rand_engine);
+        ENGINE_unregister_RAND(rand_engine);
         ENGINE_free(rand_engine);
         ENGINE_cleanup();
         RAND_set_rand_engine(NULL);