RUBY-2135 improve efficiency of testing on docker, add more configura…

…tions (#1706)

* implement version locator in ruby, add 2.6 support via full catalog

* 2.6 server support with mlaunch

* use python in ruby toolchain

* preload ruby toolchain for faster builds

* extract download url determinator

* preload server option

* preinstall gems when preloading ruby

* install git

* support ubuntu1204 on docker

* ubuntu1404 support

* rhel support

* need python toolchain when testing ruby-head

* use curl instead of wget

* need bzip2 for ruby-head

* exclude ruby-head from preloading

* switch the order of server and ruby preloading to cache the server always

* do not require python (2) presence

* check ruby version directly in the shell

* add documentation

Co-authored-by: Oleg Pudeyev <p@users.noreply.github.com>

.evergreen/Dockerfile.erb

@@ -1,17 +1,22 @@
-# erb spec/Dockerfile.erb >spec/Dockerfile &&  cat spec/Dockerfile && docker build -t test-driver  -f spec/Dockerfile .&& docker run -ti --tmpfs /tmpfs:exec test-driver
-
 # Python toolchain as of this writing is available on rhel62, debian92 and
 # ubuntu1604.
+#
+# To run rhel62 in docker, host system must be configured to emulate syscalls:
+# https://github.com/CentOS/sig-cloud-instance-images/issues/103
 
 <%
 
-toolchain_url = "https://s3.amazonaws.com//mciuploads/mongo-ruby-toolchain/#{distro}/f11598d091441ffc8d746aacfdc6c26741a3e629/mongo_ruby_driver_toolchain_#{distro.gsub('-', '_')}_f11598d091441ffc8d746aacfdc6c26741a3e629_20_02_01_23_51_34.tar.gz"
 python_toolchain_url = "https://s3.amazonaws.com//mciuploads/mongo-python-driver-toolchain/#{distro}/ba92de2700c04ee2d4f82c3ffdfc33105140cb04/mongo_python_driver_toolchain_#{distro.gsub('-', '_')}_ba92de2700c04ee2d4f82c3ffdfc33105140cb04_19_11_14_15_33_33.tar.gz"
 server_version = '4.3.3'
 server_url = "http://downloads.10gen.com/linux/mongodb-linux-x86_64-enterprise-#{distro}-#{server_version}.tgz"
 server_archive_basename = File.basename(server_url)
 server_extracted_dir = server_archive_basename.sub(/\.(tar\.gz|tgz)$/, '')
 
+toolchain_upper='f11598d091441ffc8d746aacfdc6c26741a3e629' 
+toolchain_lower='5e46f2793e8e866f36eda2c5_20_02_14_19_18_18' 
+
+ruby_toolchain_url = "https://s3.amazonaws.com//mciuploads/mongo-ruby-toolchain/#{distro}/#{toolchain_upper}/mongo_ruby_driver_toolchain_#{distro.gsub('-', '_')}_patch_#{toolchain_upper}_#{toolchain_lower}.tar.gz"
+
 %>
 
 FROM <%= base_image %>
@@ -26,16 +31,76 @@ FROM <%= base_image %>
 
 <% end %>
 
-RUN curl --retry 3 -fL <%= python_toolchain_url %> -o python-toolchain.tar.gz
-RUN tar -xC /opt -zf python-toolchain.tar.gz
+<% if ruby_head? %>
+
+  RUN curl --retry 3 -fL <%= python_toolchain_url %> -o python-toolchain.tar.gz
+  RUN tar -xC /opt -zf python-toolchain.tar.gz
+
+<% end %>
+
+<% if debian? %>
+
+  # Ruby runtime dependencies: libyaml-0-2
+  # Compiling ruby libraries: gcc make
+  # JRuby: openjdk-8-jre
+  # Server dependencies: libsnmp30 libcurl3
+  # Determining OS we are running on: lsb-release
+  # Kerberos testing: krb5-user
+  # Installing mlaunch from git: git
+  # ruby-head archive: bzip2
+
+  # ubuntu1204, ubuntu1404 only have openjdk-7-jre
+  <% unless %w(ubuntu1204 ubuntu1404).include?(distro) %>
+    RUN apt-get install -y openjdk-8-jre
+  <% end %>
+
+  <% if %w(ubuntu1204).include?(distro) %>
+    RUN apt-get install -y libsnmp15
+  <% else %>
+    RUN apt-get install -y libsnmp30
+  <% end %>
+
+  # ubuntu1204, ubuntu1404, ubuntu1604: libcurl3
+  RUN apt-get install -y libyaml-0-2 gcc make git lsb-release libcurl3 krb5-user bzip2
 
-RUN apt-get install -y libyaml-0-2 gcc make openjdk-8-jre
+<% else %>
 
-# ubuntu1604: libcurl3
-RUN apt-get install -y lsb-release python libsnmp30 libcurl3 krb5-user
+  RUN yum install -y redhat-lsb-core which
+
+<% end %>
+
+<% if preload? %>
+
+  WORKDIR /app
+
+  RUN curl --retry 3 -fL <%= server_download_url %> -o <%= File.basename(server_download_url) %>
+  RUN tar xfz <%= File.basename(server_download_url) %>
+  RUN mv mongo*/ /opt/mongodb
+  ENV USE_OPT_MONGODB=1
+
+  <% unless ruby_head? %>
+
+    RUN curl --retry 3 -fL <%= ruby_toolchain_url %> -o ruby-toolchain.tar.gz
+    RUN tar -xC /opt -zf ruby-toolchain.tar.gz
+    ENV PATH=/opt/rubies/<%= ruby %>/bin:/opt/rubies/python/3/bin:$PATH
+    ENV USE_OPT_TOOLCHAIN=1
+
+  <% end %>
+
+<% end %>
 
 WORKDIR /app
 
+<% if preload? && !ruby_head? %>
+
+  COPY Gemfile .
+  COPY gemfiles gemfiles
+  COPY *.gemspec .
+  COPY lib/mongo/version.rb lib/mongo/version.rb
+  RUN bundle install
+
+<% end %>
+
 COPY . .
 
 ENV MONGO_ORCHESTRATION_HOME=/tmpfs

.evergreen/README.md

@@ -21,3 +21,18 @@ By default the entire test suite is run (using mlaunch to launch the server);
 to specify another script, use `-s` option:
 
     ./.evergreen/run-on-docker -s .evergreen/run-enterprise-auth-tests.sh
+
+### Toolchain and Server Preloading
+
+The docker test runner supports preloading Ruby interpreters and server
+binaries in the docker image, which reduces the runtime of subsequent
+test runs. To turn on preloading, use `-p` option:
+
+    ./.evergreen/run-on-docker -p
+
+### rhel62
+
+To run rhel62 distro in docker, host system must be configured to [emulate
+syscalls](https://github.com/CentOS/sig-cloud-instance-images/issues/103).
+Note that this defeats one of the patches for the Spectre set of processor
+vulnerabilities.

.evergreen/functions.sh

@@ -39,6 +39,9 @@ _detect_arch() {
     elif lsb_release -i |grep -q RedHat; then
       release=`lsb_release -r |awk '{print $2}' |tr -d .`
       arch="rhel$release"
+    elif lsb_release -i |grep -q CentOS; then
+      release=`lsb_release -r |awk '{print $2}' |cut -c 1 |sed -e s/7/70/ -e s/6/62/`
+      arch="rhel$release"
     else
       echo 'Unknown RHEL flavor' 1>&2
       return 1
@@ -138,21 +141,30 @@ setup_ruby() {
   fi
 
   if [ "$RVM_RUBY" == "ruby-head" ]; then
+    # When we use ruby-head, we do not install the Ruby toolchain.
+    # But we still need Python 3.6+ to run mlaunch.
+    # Since the ruby-head tests are run on ubuntu1604, we can use the
+    # globally installed Python toolchain.
+    export PATH=/opt/python/3.7/bin:$PATH
+
     # 12.04, 14.04 and 16.04 are good
-    wget -O ruby-head.tar.bz2 http://rubies.travis-ci.org/ubuntu/`lsb_release -rs`/x86_64/ruby-head.tar.bz2
+    curl -fLo ruby-head.tar.bz2 http://rubies.travis-ci.org/ubuntu/`lsb_release -rs`/x86_64/ruby-head.tar.bz2
     tar xf ruby-head.tar.bz2
     export PATH=`pwd`/ruby-head/bin:`pwd`/ruby-head/lib/ruby/gems/2.6.0/bin:$PATH
     ruby --version
     ruby --version |grep dev
 
     #rvm reinstall $RVM_RUBY
   else
-    if true; then
+    if test "$USE_OPT_TOOLCHAIN" = 1; then
+      # nothing, also PATH is already set
+      :
+    elif true; then
 
     # For testing toolchains:
-    toolchain_url=https://s3.amazonaws.com//mciuploads/mongo-ruby-toolchain/`host_arch`/f11598d091441ffc8d746aacfdc6c26741a3e629/mongo_ruby_driver_toolchain_`host_arch |tr - _`_f11598d091441ffc8d746aacfdc6c26741a3e629_20_02_01_23_51_34.tar.gz
+    toolchain_url=https://s3.amazonaws.com//mciuploads/mongo-ruby-toolchain/`host_arch`/f11598d091441ffc8d746aacfdc6c26741a3e629/mongo_ruby_driver_toolchain_`host_arch |tr - _`_patch_f11598d091441ffc8d746aacfdc6c26741a3e629_5e46f2793e8e866f36eda2c5_20_02_14_19_18_18.tar.gz
     curl --retry 3 -fL $toolchain_url |tar zxf -
-    export PATH=`pwd`/rubies/$RVM_RUBY/bin:$PATH
+    export PATH=`pwd`/rubies/$RVM_RUBY/bin:`pwd`/rubies/python/3/bin:$PATH
 
     # Attempt to get bundler to report all errors - so far unsuccessful
     #curl -o bundler-openssl.diff https://github.com/bundler/bundler/compare/v2.0.1...p-mongo:report-errors.diff
@@ -178,12 +190,11 @@ setup_ruby() {
     ruby --version
 
     # Ensure we're using the right ruby
-    python - <<EOH
-ruby = "${RVM_RUBY}".split("-")[0]
-version = "${RVM_RUBY}".split("-")[1]
-assert(ruby in "`ruby --version`")
-assert(version in "`ruby --version`")
-EOH
+    ruby_name=`echo $RVM_RUBY |awk -F- '{print $1}'`
+    ruby_version=`echo $RVM_RUBY |awk -F- '{print $2}' |cut -c 1-3`
+
+    ruby -v |fgrep $ruby_name
+    ruby -v |fgrep $ruby_version
 
     # We shouldn't need to update rubygems, and there is value in
     # testing on whatever rubygems came with each supported ruby version
@@ -225,9 +236,24 @@ kill_jruby() {
 
 prepare_server() {
   arch=$1
-  version=$2
+
+  if test -n "$USE_OPT_MONGODB"; then
+    export BINDIR=/opt/mongodb/bin
+    export PATH=$BINDIR:$PATH
+    return
+  fi
+
+  if test "$MONGODB_VERSION" = 2.6; then
+    # The only OS which has Python toolchain for Python 3.6+ and
+    # which has MongoDB 2.6 server builds for it is rhel62.
+    # Unfortunately running it in Docker on a Debian 10 host crashes.
+    # Try the generic Linux binary since we aren't using any enterprise
+    # features pre FLE which requires 4.2 server.
+    url=https://fastdl.mongodb.org/linux/mongodb-linux-x86_64-2.6.12.tgz
+  else
+    url=`$(dirname $0)/get-mongodb-download-url $MONGODB_VERSION $arch`
+  fi
 
-  url=http://downloads.10gen.com/linux/mongodb-linux-x86_64-enterprise-$arch-$version.tgz
   prepare_server_from_url $url
 }
 
@@ -241,13 +267,26 @@ prepare_server_from_url() {
   export PATH="$BINDIR":$PATH
 }
 
-install_mlaunch() {
-  export PATH=/opt/python/3.7/bin:$PATH
-  python -V
+install_mlaunch_virtualenv() {
+  #export PATH=/opt/python/3.7/bin:$PATH
+  python -V || true
   python3 -V
   #pip3 install --user virtualenv
   venvpath="$MONGO_ORCHESTRATION_HOME"/venv
   virtualenv -p python3 $venvpath
   . $venvpath/bin/activate
   pip install 'mtools[mlaunch]'
 }
+
+install_mlaunch_pip() {
+  python -V || true
+  python3 -V
+  pythonpath="$MONGO_ORCHESTRATION_HOME"/python
+  # The scripts in a python installation have shebangs pointing to the
+  # prefix, which doesn't work for us because we unpack toolchain to a
+  # different directory than prefix used for building. Work around this by
+  # explicitly running pip3 with python.
+  python3 `which pip3` install -t "$pythonpath" 'mtools[mlaunch]'
+  export PATH="$pythonpath/bin":$PATH
+  export PYTHONPATH="$pythonpath"
+}

.evergreen/get-mongodb-download-url

@@ -0,0 +1,11 @@
+#!/usr/bin/env ruby
+
+desired_version, arch = ARGV
+if arch.nil?
+  STDERR.puts "Usage: get-mongodb-download-url desired-version arch"
+  exit 1
+end
+
+load File.join(File.dirname(__FILE__), 'tools.rb')
+
+puts ServerVersionRegistry.new(desired_version, arch).download_url

.evergreen/run-fle-tests.sh

@@ -5,7 +5,7 @@ set -o errexit  # Exit the script with error if any of the commands fail
 
 . `dirname "$0"`/functions.sh
 
-wget "https://s3.amazonaws.com/mciuploads/libmongocrypt/all/master/latest/libmongocrypt-all.tar.gz"
+curl -fLo libmongocrypt-all.tar.gz "https://s3.amazonaws.com/mciuploads/libmongocrypt/all/master/latest/libmongocrypt-all.tar.gz"
 tar -xvf libmongocrypt-all.tar.gz
 
 export LIBMONGOCRYPT_PATH=`pwd`/rhel-70-64-bit/nocrypto/lib64/libmongocrypt.so

.evergreen/run-local-tls-tests.sh

@@ -27,7 +27,7 @@ arch=`host_arch`
 version=4.0.16
 prepare_server $arch $version
 
-install_mlaunch
+install_mlaunch_pip
 
 # Launching mongod under $MONGO_ORCHESTRATION_HOME
 # makes its log available through log collecting machinery

.evergreen/run-tests-mlaunch.sh

@@ -22,22 +22,9 @@ setup_ruby
 
 arch=`host_arch`
 
-desired_version=$MONGODB_VERSION
-prog=`cat <<-EOT
-import urllib, json
-url = 'http://downloads.mongodb.org/current.json'
-info = json.load(urllib.urlopen(url))
-info = [i for i in info['versions'] if i['version'].startswith('$MONGODB_VERSION')][0]
-info = [i for i in info['downloads'] if i['archive']['url'].find('enterprise-$arch') > 0][0]
-url = info['archive']['url']
-print(url)
-EOT`
+prepare_server $arch
 
-url=`python -c "$prog"`
-
-prepare_server_from_url $url
-
-install_mlaunch
+install_mlaunch_pip
 
 # Launching mongod under $MONGO_ORCHESTRATION_HOME
 # makes its log available through log collecting machinery
@@ -46,7 +33,9 @@ export dbdir="$MONGO_ORCHESTRATION_HOME"/db
 mkdir -p "$dbdir"
 
 args="--setParameter enableTestCommands=1"
-args="$args --setParameter diagnosticDataCollectionEnabled=false"
+if ! test "$MONGODB_VERSION" = 2.6 && ! test "$MONGODB_VERSION" = 3.0; then
+  args="$args --setParameter diagnosticDataCollectionEnabled=false"
+fi
 uri_options=
 if test "$TOPOLOGY" = replica_set; then
   args="$args --replicaset --name ruby-driver-rs"

.evergreen/run-x509-tests.sh

@@ -13,10 +13,9 @@ setup_ruby
 install_deps
 
 arch=`host_arch`
-version=4.2.3
-prepare_server $arch $version
+prepare_server $arch
 
-install_mlaunch
+install_mlaunch_pip
 
 export dbdir="$MONGO_ORCHESTRATION_HOME"/db
 mkdir -p "$dbdir"

.evergreen/test-on-docker

@@ -3,6 +3,8 @@
 require 'optparse'
 require 'erb'
 
+load File.join(File.dirname(__FILE__), 'tools.rb')
+
 class Runner
   def run
     process_arguments
@@ -22,6 +24,10 @@ class Runner
         @options[:distro] = v
       end
 
+      opts.on('-p', '--preload', 'Preload Ruby toolchain and server binaries in docker') do |v|
+        @options[:preload] = v
+      end
+
       opts.on('-s', '--script=SCRIPT', 'Test script to invoke') do |v|
         @options[:script] = v
       end
@@ -73,6 +79,7 @@ class Runner
   BASE_IMAGES = {
     'debian81' => 'debian:jessie',
     'debian92' => 'debian:stretch',
+    'ubuntu1204' => 'ubuntu:precise',
     'ubuntu1404' => 'ubuntu:trusty',
     'ubuntu1604' => 'ubuntu:xenial',
     'ubuntu1804' => 'ubuntu:bionic',
@@ -88,6 +95,10 @@ class Runner
     @env['RVM_RUBY']
   end
 
+  def ruby_head?
+    ruby == 'ruby-head'
+  end
+
   def server_version
     @env['MONGODB_VERSION']
   end
@@ -99,6 +110,14 @@ class Runner
   def debian?
     distro =~ /debian|ubuntu/
   end
+
+  def preload?
+    !!@options[:preload]
+  end
+
+  def server_download_url
+    @server_download_url ||= ServerVersionRegistry.new(server_version, distro).download_url
+  end
 end
 
 Runner.new.run