RUBY-3154 Test crypt_shared with older server versions (#2705)

* RUBY-3154 run all encryption tests against both mongocryptd and crypt_shared

* allow fork_reconnect_spec to run on a mac

* let's try disabling these tests on jruby...

* no need to overcomplicate the without_crypt_shared_lib_path helper

* lock older server versions to recent crypt_shared

Author: jamis

.evergreen/config.yml

@@ -106,6 +106,7 @@ functions:
             export COMPRESSOR=${COMPRESSOR}
             export RVM_RUBY="${RVM_RUBY}"
             export MONGODB_VERSION=${MONGODB_VERSION}
+            export CRYPT_SHARED_VERSION=${CRYPT_SHARED_VERSION}
             export FCV=${FCV}
             export MONGO_RUBY_DRIVER_LINT=${LINT}
             export RETRY_READS=${RETRY_READS}
@@ -616,18 +617,22 @@ axes:
         display_name: "5.3"
         variables:
           MONGODB_VERSION: "5.3"
+          CRYPT_SHARED_VERSION: "6.0.5"
       - id: "5.0"
         display_name: "5.0"
         variables:
           MONGODB_VERSION: "5.0"
+          CRYPT_SHARED_VERSION: "6.0.5"
       - id: "4.4"
         display_name: "4.4"
         variables:
           MONGODB_VERSION: "4.4"
+          CRYPT_SHARED_VERSION: "6.0.5"
       - id: "4.2"
         display_name: "4.2"
         variables:
           MONGODB_VERSION: "4.2"
+          CRYPT_SHARED_VERSION: "6.0.5"
       - id: "4.0"
         display_name: "4.0"
         variables:

.evergreen/config/axes.yml.erb

@@ -25,18 +25,22 @@ axes:
         display_name: "5.3"
         variables:
           MONGODB_VERSION: "5.3"
+          CRYPT_SHARED_VERSION: "6.0.5"
       - id: "5.0"
         display_name: "5.0"
         variables:
           MONGODB_VERSION: "5.0"
+          CRYPT_SHARED_VERSION: "6.0.5"
       - id: "4.4"
         display_name: "4.4"
         variables:
           MONGODB_VERSION: "4.4"
+          CRYPT_SHARED_VERSION: "6.0.5"
       - id: "4.2"
         display_name: "4.2"
         variables:
           MONGODB_VERSION: "4.2"
+          CRYPT_SHARED_VERSION: "6.0.5"
       - id: "4.0"
         display_name: "4.0"
         variables:

.evergreen/config/common.yml.erb

@@ -104,6 +104,7 @@ functions:
             export COMPRESSOR=${COMPRESSOR}
             export RVM_RUBY="${RVM_RUBY}"
             export MONGODB_VERSION=${MONGODB_VERSION}
+            export CRYPT_SHARED_VERSION=${CRYPT_SHARED_VERSION}
             export FCV=${FCV}
             export MONGO_RUBY_DRIVER_LINT=${LINT}
             export RETRY_READS=${RETRY_READS}

.evergreen/run-tests.sh

@@ -165,7 +165,7 @@ fi
 if test -n "$FLE"; then
   # Downloading crypt shared lib
   if [ -z "$MONGO_CRYPT_SHARED_DOWNLOAD_URL" ]; then
-    crypt_shared_version=$("${BINDIR}"/mongod --version | grep -oP 'db version v\K.*')
+    crypt_shared_version=${CRYPT_SHARED_VERSION:-$("${BINDIR}"/mongod --version | grep -oP 'db version v\K.*')}
     python3 -u .evergreen/mongodl.py --component crypt_shared -V ${crypt_shared_version} --out $(pwd)/csfle_lib  --target $(host_distro) || true
     if test -f $(pwd)/csfle_lib/lib/mongo_crypt_v1.so
     then

spec/integration/fork_reconnect_spec.rb

@@ -42,7 +42,7 @@
         child_socket.object_id.should_not == socket.object_id
 
         # Exec so that we do not close any clients etc. in the child.
-        exec('/bin/true')
+        exec(Utils::BIN_TRUE)
       end
 
       # Connection should remain serviceable in the parent.

spec/spec_tests/client_side_encryption_spec.rb

@@ -10,5 +10,21 @@
   require_libmongocrypt
   require_enterprise
 
-  define_transactions_spec_tests(CLIENT_SIDE_ENCRYPTION_TESTS, expectations_bson_types: true)
+  context 'with mongocryptd' do
+    SpecConfig.instance.without_crypt_shared_lib_path do
+      define_transactions_spec_tests(CLIENT_SIDE_ENCRYPTION_TESTS, expectations_bson_types: true)
+    end
+  end
+
+  context 'with crypt_shared' do
+    # Under JRuby+Evergreen, these specs complain about the crypt_shared
+    # library not loading; however, crypt_shared appears to load for other
+    # specs that require it (see the client_side_encryption_unified_spec and
+    # mongocryptd_prose_spec tests).
+    fails_on_jruby
+
+    SpecConfig.instance.require_crypt_shared do
+      define_transactions_spec_tests(CLIENT_SIDE_ENCRYPTION_TESTS, expectations_bson_types: true)
+    end
+  end
 end

spec/spec_tests/client_side_encryption_unified_spec.rb

@@ -12,5 +12,15 @@
   require_libmongocrypt
   require_enterprise
 
-  define_unified_spec_tests(base, CLIENT_SIDE_ENCRYPTION_UNIFIED_TESTS)
+  context 'with mongocryptd' do
+    SpecConfig.instance.without_crypt_shared_lib_path do
+      define_unified_spec_tests(base, CLIENT_SIDE_ENCRYPTION_UNIFIED_TESTS)
+    end
+  end
+
+  context 'with crypt_shared' do
+    SpecConfig.instance.require_crypt_shared do
+      define_unified_spec_tests(base, CLIENT_SIDE_ENCRYPTION_UNIFIED_TESTS)
+    end
+  end
 end

spec/support/spec_config.rb

@@ -489,7 +489,27 @@ def mongocryptd_port
   end
 
   def crypt_shared_lib_path
-    ENV['MONGO_RUBY_DRIVER_CRYPT_SHARED_LIB_PATH']
+    if @without_crypt_shared_lib_path
+      nil
+    else
+      ENV['MONGO_RUBY_DRIVER_CRYPT_SHARED_LIB_PATH']
+    end
+  end
+
+  def without_crypt_shared_lib_path
+    saved, @without_crypt_shared_lib_path = @without_crypt_shared_lib_path, true
+    yield
+  ensure
+    @without_crypt_shared_lib_path = saved
+  end
+
+  attr_accessor :crypt_shared_lib_required
+
+  def require_crypt_shared
+    saved, self.crypt_shared_lib_required = crypt_shared_lib_required, true
+    yield
+  ensure
+    self.crypt_shared_lib_required = saved
   end
 
   def auth?

spec/support/utils.rb

@@ -138,16 +138,8 @@ def convert_client_options(spec_test_options)
     mapper = Mongo::URI::OptionsMapper.new
     spec_test_options.reduce({}) do |opts, (name, value)|
       if name == 'autoEncryptOpts'
-        opts.merge!(
-          auto_encryption_options: convert_auto_encryption_client_options(value)
-            .merge(
-              # Spawn mongocryptd on non-default port for sharded cluster tests
-              extra_options: {
-                mongocryptd_spawn_args: ["--port=#{SpecConfig.instance.mongocryptd_port}"],
-                mongocryptd_uri: "mongodb://localhost:#{SpecConfig.instance.mongocryptd_port}",
-              }
-            )
-        )
+        auto_encryption_options = convert_auto_encryption_client_options(value)
+        opts.merge!(auto_encryption_options: auto_encryption_options)
       else
         mapper.add_uri_option(name, value.to_s, opts)
       end
@@ -206,10 +198,30 @@ def convert_client_options(spec_test_options)
       auto_encrypt_opts[:encrypted_fields_map] = BSON::ExtJSON.parse_obj(opts['encryptedFieldsMap'])
     end
 
-    auto_encrypt_opts
+    auto_encrypt_opts.merge!(extra_options: convert_auto_encryption_extra_options(auto_encrypt_opts))
   end
   module_function :convert_auto_encryption_client_options
 
+  private def convert_auto_encryption_extra_options(opts)
+    # Spawn mongocryptd on non-default port for sharded cluster tests
+    extra_options = {
+      mongocryptd_spawn_args: [ "--port=#{SpecConfig.instance.mongocryptd_port}" ],
+      mongocryptd_uri: "mongodb://localhost:#{SpecConfig.instance.mongocryptd_port}"
+    }.merge(opts[:extra_options] || {})
+
+    # if bypass_query_analysis has been explicitly specified, then we ignore
+    # any requirement to use the shared library, as the two are not
+    # compatible.
+    if SpecConfig.instance.crypt_shared_lib_required && !opts[:bypass_query_analysis]
+      extra_options[:crypt_shared_lib_required] = SpecConfig.instance.crypt_shared_lib_required
+      extra_options[:crypt_shared_lib_path] = SpecConfig.instance.crypt_shared_lib_path
+      extra_options[:mongocryptd_uri] = "mongodb://localhost:27777"
+    end
+
+    extra_options
+  end
+  module_function :convert_auto_encryption_extra_options
+
   def order_hash(hash)
     Hash[hash.to_a.sort]
   end
@@ -539,15 +551,18 @@ def match_with_type?(expected, actual)
     end
   end
 
+  BIN_FALSE = File.executable?("/bin/false") ? "/bin/false" : "false"
+  BIN_TRUE  = File.executable?("/bin/true") ? "/bin/true" : "true"
+
   module_function def wrap_forked_child
     begin
       yield
     rescue => e
       STDERR.puts "Failing process #{Process.pid} due to #{e.class}: #{e}"
-      exec('/bin/false')
+      exec(BIN_FALSE)
     else
       # Exec so that we do not close any clients etc. in the child.
-      exec('/bin/true')
+      exec(BIN_TRUE)
     end
   end