From 357874f86c764a85cd0d29c1d005b053f01d42a2 Mon Sep 17 00:00:00 2001
From: Oleg Pudeyev <oleg@bsdpower.com>
Date: Fri, 14 Feb 2020 00:10:19 -0500
Subject: [PATCH 01/19] implement version locator in ruby, add 2.6 support via
 full catalog

---
 .evergreen/get-mongodb-download-url | 34 +++++++++++++++++++++++++++++
 .evergreen/run-tests-mlaunch.sh     | 13 +----------
 2 files changed, 35 insertions(+), 12 deletions(-)
 create mode 100755 .evergreen/get-mongodb-download-url

diff --git a/.evergreen/get-mongodb-download-url b/.evergreen/get-mongodb-download-url
new file mode 100755
index 0000000000..a99486723b
--- /dev/null
+++ b/.evergreen/get-mongodb-download-url
@@ -0,0 +1,34 @@
+#!/usr/bin/env ruby
+
+require 'json'
+require 'open-uri'
+
+desired_version, arch = ARGV
+if arch.nil?
+  STDERR.puts "Usage: get-mongodb-download-url desired-version arch"
+  exit 1
+end
+
+info = JSON.load(open('http://downloads.mongodb.org/current.json').read)
+version = info['versions'].detect { |version| version['version'].start_with?(desired_version) }
+if version.nil?
+  info = JSON.load(URI.open('http://downloads.mongodb.org/full.json').read)
+  versions = info['versions'].select { |version| version['version'].start_with?(desired_version) }
+  # Get rid of rc, beta etc. versions.
+  versions.delete_if { |version| version['version'].include?('-') }
+  # Versions are ordered with newest first, take the first one i.e. the most
+  # recent one.
+  version = versions.first
+  if version.nil?
+    STDERR.puts "Error: no version #{desired_version}"
+    exit 2
+  end
+end
+dl = version['downloads'].detect { |dl| dl['archive']['url'].index("enterprise-#{arch}") }
+unless dl
+  STDERR.puts "Error: no download for #{arch} for #{version['version']}"
+  exit 2
+end
+url = dl['archive']['url']
+
+puts url
diff --git a/.evergreen/run-tests-mlaunch.sh b/.evergreen/run-tests-mlaunch.sh
index af0ad9c92c..954cc2a10c 100755
--- a/.evergreen/run-tests-mlaunch.sh
+++ b/.evergreen/run-tests-mlaunch.sh
@@ -22,18 +22,7 @@ setup_ruby
 
 arch=`host_arch`
 
-desired_version=$MONGODB_VERSION
-prog=`cat <<-EOT
-import urllib, json
-url = 'http://downloads.mongodb.org/current.json'
-info = json.load(urllib.urlopen(url))
-info = [i for i in info['versions'] if i['version'].startswith('$MONGODB_VERSION')][0]
-info = [i for i in info['downloads'] if i['archive']['url'].find('enterprise-$arch') > 0][0]
-url = info['archive']['url']
-print(url)
-EOT`
-
-url=`python -c "$prog"`
+url=`$(dirname $0)/get-mongodb-download-url $MONGODB_VERSION $arch`
 
 prepare_server_from_url $url
 

From 2442dca8de359ebef5424a70a78fa9d8ebc46317 Mon Sep 17 00:00:00 2001
From: Oleg Pudeyev <oleg@bsdpower.com>
Date: Fri, 14 Feb 2020 00:35:09 -0500
Subject: [PATCH 02/19] 2.6 server support with mlaunch

---
 .evergreen/run-tests-mlaunch.sh | 15 +++++++++++++--
 1 file changed, 13 insertions(+), 2 deletions(-)

diff --git a/.evergreen/run-tests-mlaunch.sh b/.evergreen/run-tests-mlaunch.sh
index 954cc2a10c..42089168e0 100755
--- a/.evergreen/run-tests-mlaunch.sh
+++ b/.evergreen/run-tests-mlaunch.sh
@@ -22,7 +22,16 @@ setup_ruby
 
 arch=`host_arch`
 
-url=`$(dirname $0)/get-mongodb-download-url $MONGODB_VERSION $arch`
+if test "$MONGODB_VERSION" = 2.6; then
+  # The only OS which has Python toolchain for Python 3.6+ and
+  # which has MongoDB 2.6 server builds for it is rhel62.
+  # Unfortunately running it in Docker on a Debian 10 host crashes.
+  # Try the generic Linux binary since we aren't using any enterprise
+  # features pre FLE which requires 4.2 server.
+  url=https://fastdl.mongodb.org/linux/mongodb-linux-x86_64-2.6.12.tgz
+else
+  url=`$(dirname $0)/get-mongodb-download-url $MONGODB_VERSION $arch`
+fi
 
 prepare_server_from_url $url
 
@@ -35,7 +44,9 @@ export dbdir="$MONGO_ORCHESTRATION_HOME"/db
 mkdir -p "$dbdir"
 
 args="--setParameter enableTestCommands=1"
-args="$args --setParameter diagnosticDataCollectionEnabled=false"
+if ! test "$MONGODB_VERSION" = 2.6 && ! test "$MONGODB_VERSION" = 3.0; then
+  args="$args --setParameter diagnosticDataCollectionEnabled=false"
+fi
 uri_options=
 if test "$TOPOLOGY" = replica_set; then
   args="$args --replicaset --name ruby-driver-rs"

From 9897673f26747933406e09aff8aeff39405e6e7f Mon Sep 17 00:00:00 2001
From: Oleg Pudeyev <oleg@bsdpower.com>
Date: Fri, 14 Feb 2020 15:39:58 -0500
Subject: [PATCH 03/19] use python in ruby toolchain

---
 .evergreen/Dockerfile.erb         |  5 ++---
 .evergreen/functions.sh           | 21 +++++++++++++++++----
 .evergreen/run-local-tls-tests.sh |  2 +-
 .evergreen/run-tests-mlaunch.sh   |  2 +-
 .evergreen/run-x509-tests.sh      |  2 +-
 5 files changed, 22 insertions(+), 10 deletions(-)

diff --git a/.evergreen/Dockerfile.erb b/.evergreen/Dockerfile.erb
index a40ea65e4c..c1b7b31939 100644
--- a/.evergreen/Dockerfile.erb
+++ b/.evergreen/Dockerfile.erb
@@ -5,7 +5,6 @@
 
 <%
 
-toolchain_url = "https://s3.amazonaws.com//mciuploads/mongo-ruby-toolchain/#{distro}/f11598d091441ffc8d746aacfdc6c26741a3e629/mongo_ruby_driver_toolchain_#{distro.gsub('-', '_')}_f11598d091441ffc8d746aacfdc6c26741a3e629_20_02_01_23_51_34.tar.gz"
 python_toolchain_url = "https://s3.amazonaws.com//mciuploads/mongo-python-driver-toolchain/#{distro}/ba92de2700c04ee2d4f82c3ffdfc33105140cb04/mongo_python_driver_toolchain_#{distro.gsub('-', '_')}_ba92de2700c04ee2d4f82c3ffdfc33105140cb04_19_11_14_15_33_33.tar.gz"
 server_version = '4.3.3'
 server_url = "http://downloads.10gen.com/linux/mongodb-linux-x86_64-enterprise-#{distro}-#{server_version}.tgz"
@@ -26,8 +25,8 @@ FROM <%= base_image %>
 
 <% end %>
 
-RUN curl --retry 3 -fL <%= python_toolchain_url %> -o python-toolchain.tar.gz
-RUN tar -xC /opt -zf python-toolchain.tar.gz
+#RUN curl --retry 3 -fL <%= python_toolchain_url %> -o python-toolchain.tar.gz
+#RUN tar -xC /opt -zf python-toolchain.tar.gz
 
 RUN apt-get install -y libyaml-0-2 gcc make openjdk-8-jre
 
diff --git a/.evergreen/functions.sh b/.evergreen/functions.sh
index 5e578fc688..b62079eebc 100644
--- a/.evergreen/functions.sh
+++ b/.evergreen/functions.sh
@@ -150,9 +150,9 @@ setup_ruby() {
     if true; then
 
     # For testing toolchains:
-    toolchain_url=https://s3.amazonaws.com//mciuploads/mongo-ruby-toolchain/`host_arch`/f11598d091441ffc8d746aacfdc6c26741a3e629/mongo_ruby_driver_toolchain_`host_arch |tr - _`_f11598d091441ffc8d746aacfdc6c26741a3e629_20_02_01_23_51_34.tar.gz
+    toolchain_url=https://s3.amazonaws.com//mciuploads/mongo-ruby-toolchain/`host_arch`/f11598d091441ffc8d746aacfdc6c26741a3e629/mongo_ruby_driver_toolchain_`host_arch |tr - _`_patch_f11598d091441ffc8d746aacfdc6c26741a3e629_5e46f2793e8e866f36eda2c5_20_02_14_19_18_18.tar.gz
     curl --retry 3 -fL $toolchain_url |tar zxf -
-    export PATH=`pwd`/rubies/$RVM_RUBY/bin:$PATH
+    export PATH=`pwd`/rubies/$RVM_RUBY/bin:`pwd`/rubies/python/3/bin:$PATH
 
     # Attempt to get bundler to report all errors - so far unsuccessful
     #curl -o bundler-openssl.diff https://github.com/bundler/bundler/compare/v2.0.1...p-mongo:report-errors.diff
@@ -241,8 +241,8 @@ prepare_server_from_url() {
   export PATH="$BINDIR":$PATH
 }
 
-install_mlaunch() {
-  export PATH=/opt/python/3.7/bin:$PATH
+install_mlaunch_virtualenv() {
+  #export PATH=/opt/python/3.7/bin:$PATH
   python -V
   python3 -V
   #pip3 install --user virtualenv
@@ -251,3 +251,16 @@ install_mlaunch() {
   . $venvpath/bin/activate
   pip install 'mtools[mlaunch]'
 }
+
+install_mlaunch_pip() {
+  python -V
+  python3 -V
+  pythonpath="$MONGO_ORCHESTRATION_HOME"/python
+  # The scripts in a python installation have shebangs pointing to the
+  # prefix, which doesn't work for us because we unpack toolchain to a
+  # different directory than prefix used for building. Work around this by
+  # explicitly running pip3 with python.
+  python3 `which pip3` install -t "$pythonpath" 'mtools[mlaunch]'
+  export PATH="$pythonpath/bin":$PATH
+  export PYTHONPATH="$pythonpath"
+}
diff --git a/.evergreen/run-local-tls-tests.sh b/.evergreen/run-local-tls-tests.sh
index 334a9aa373..6af898e132 100755
--- a/.evergreen/run-local-tls-tests.sh
+++ b/.evergreen/run-local-tls-tests.sh
@@ -27,7 +27,7 @@ arch=`host_arch`
 version=4.0.16
 prepare_server $arch $version
 
-install_mlaunch
+install_mlaunch_pip
 
 # Launching mongod under $MONGO_ORCHESTRATION_HOME
 # makes its log available through log collecting machinery
diff --git a/.evergreen/run-tests-mlaunch.sh b/.evergreen/run-tests-mlaunch.sh
index 42089168e0..210209779c 100755
--- a/.evergreen/run-tests-mlaunch.sh
+++ b/.evergreen/run-tests-mlaunch.sh
@@ -35,7 +35,7 @@ fi
 
 prepare_server_from_url $url
 
-install_mlaunch
+install_mlaunch_pip
 
 # Launching mongod under $MONGO_ORCHESTRATION_HOME
 # makes its log available through log collecting machinery
diff --git a/.evergreen/run-x509-tests.sh b/.evergreen/run-x509-tests.sh
index 15a872083e..66e9859a9c 100755
--- a/.evergreen/run-x509-tests.sh
+++ b/.evergreen/run-x509-tests.sh
@@ -16,7 +16,7 @@ arch=`host_arch`
 version=4.2.3
 prepare_server $arch $version
 
-install_mlaunch
+install_mlaunch_pip
 
 export dbdir="$MONGO_ORCHESTRATION_HOME"/db
 mkdir -p "$dbdir"

From 5d4946964dbf50f466b91a541f1de02d12bba650 Mon Sep 17 00:00:00 2001
From: Oleg Pudeyev <oleg@bsdpower.com>
Date: Sun, 16 Feb 2020 16:43:19 -0500
Subject: [PATCH 04/19] extract download url determinator

---
 .evergreen/get-mongodb-download-url | 27 ++----------------
 .evergreen/tools.rb                 | 44 +++++++++++++++++++++++++++++
 2 files changed, 46 insertions(+), 25 deletions(-)
 create mode 100644 .evergreen/tools.rb

diff --git a/.evergreen/get-mongodb-download-url b/.evergreen/get-mongodb-download-url
index a99486723b..9872fd0071 100755
--- a/.evergreen/get-mongodb-download-url
+++ b/.evergreen/get-mongodb-download-url
@@ -1,34 +1,11 @@
 #!/usr/bin/env ruby
 
-require 'json'
-require 'open-uri'
-
 desired_version, arch = ARGV
 if arch.nil?
   STDERR.puts "Usage: get-mongodb-download-url desired-version arch"
   exit 1
 end
 
-info = JSON.load(open('http://downloads.mongodb.org/current.json').read)
-version = info['versions'].detect { |version| version['version'].start_with?(desired_version) }
-if version.nil?
-  info = JSON.load(URI.open('http://downloads.mongodb.org/full.json').read)
-  versions = info['versions'].select { |version| version['version'].start_with?(desired_version) }
-  # Get rid of rc, beta etc. versions.
-  versions.delete_if { |version| version['version'].include?('-') }
-  # Versions are ordered with newest first, take the first one i.e. the most
-  # recent one.
-  version = versions.first
-  if version.nil?
-    STDERR.puts "Error: no version #{desired_version}"
-    exit 2
-  end
-end
-dl = version['downloads'].detect { |dl| dl['archive']['url'].index("enterprise-#{arch}") }
-unless dl
-  STDERR.puts "Error: no download for #{arch} for #{version['version']}"
-  exit 2
-end
-url = dl['archive']['url']
+load File.join(File.dirname(__FILE__), 'tools.rb')
 
-puts url
+puts ServerVersionRegistry.new(desired_version, arch).download_url
diff --git a/.evergreen/tools.rb b/.evergreen/tools.rb
new file mode 100644
index 0000000000..0dc99a4210
--- /dev/null
+++ b/.evergreen/tools.rb
@@ -0,0 +1,44 @@
+require 'json'
+require 'open-uri'
+
+class ServerVersionRegistry
+  def initialize(desired_version, arch)
+    @desired_version, @arch = desired_version, arch
+  end
+
+  attr_reader :desired_version, :arch
+
+  def download_url
+    @download_url ||= begin
+      info = JSON.load(uri_open('http://downloads.mongodb.org/current.json').read)
+      version = info['versions'].detect { |version| version['version'].start_with?(desired_version) }
+      if version.nil?
+        info = JSON.load(URI.open('http://downloads.mongodb.org/full.json').read)
+        versions = info['versions'].select { |version| version['version'].start_with?(desired_version) }
+        # Get rid of rc, beta etc. versions.
+        versions.delete_if { |version| version['version'].include?('-') }
+        # Versions are ordered with newest first, take the first one i.e. the most
+        # recent one.
+        version = versions.first
+        if version.nil?
+          STDERR.puts "Error: no version #{desired_version}"
+          exit 2
+        end
+      end
+      dl = version['downloads'].detect { |dl| dl['archive']['url'].index("enterprise-#{arch}") }
+      unless dl
+        STDERR.puts "Error: no download for #{arch} for #{version['version']}"
+        exit 2
+      end
+      url = dl['archive']['url']
+    end
+  end
+
+  def uri_open(*args)
+    if RUBY_VERSION < '2.5'
+      open(*args)
+    else
+      URI.open(*args)
+    end
+  end
+end

From 8f9135a86723bd0687f62a7ec639b334617d69c8 Mon Sep 17 00:00:00 2001
From: Oleg Pudeyev <oleg@bsdpower.com>
Date: Sun, 16 Feb 2020 16:36:13 -0500
Subject: [PATCH 05/19] preload ruby toolchain for faster builds

---
 .evergreen/Dockerfile.erb | 15 +++++++++++++++
 .evergreen/functions.sh   |  5 ++++-
 .evergreen/test-on-docker |  8 ++++++++
 3 files changed, 27 insertions(+), 1 deletion(-)

diff --git a/.evergreen/Dockerfile.erb b/.evergreen/Dockerfile.erb
index c1b7b31939..e8cceb41c8 100644
--- a/.evergreen/Dockerfile.erb
+++ b/.evergreen/Dockerfile.erb
@@ -11,6 +11,11 @@ server_url = "http://downloads.10gen.com/linux/mongodb-linux-x86_64-enterprise-#
 server_archive_basename = File.basename(server_url)
 server_extracted_dir = server_archive_basename.sub(/\.(tar\.gz|tgz)$/, '')
 
+toolchain_upper='f11598d091441ffc8d746aacfdc6c26741a3e629' 
+toolchain_lower='5e46f2793e8e866f36eda2c5_20_02_14_19_18_18' 
+
+ruby_toolchain_url = "https://s3.amazonaws.com//mciuploads/mongo-ruby-toolchain/#{distro}/#{toolchain_upper}/mongo_ruby_driver_toolchain_#{distro.gsub('-', '_')}_patch_#{toolchain_upper}_#{toolchain_lower}.tar.gz"
+
 %>
 
 FROM <%= base_image %>
@@ -33,6 +38,16 @@ RUN apt-get install -y libyaml-0-2 gcc make openjdk-8-jre
 # ubuntu1604: libcurl3
 RUN apt-get install -y lsb-release python libsnmp30 libcurl3 krb5-user
 
+<% if preload? %>
+
+  WORKDIR /app
+  RUN curl --retry 3 -fL <%= ruby_toolchain_url %> -o ruby-toolchain.tar.gz
+  RUN tar -xC /opt -zf ruby-toolchain.tar.gz
+  ENV PATH=/opt/rubies/<%= ruby %>/bin:/opt/rubies/python/3/bin:$PATH
+  ENV USE_OPT_TOOLCHAIN=1
+
+<% end %>
+
 WORKDIR /app
 
 COPY . .
diff --git a/.evergreen/functions.sh b/.evergreen/functions.sh
index b62079eebc..822020d7d8 100644
--- a/.evergreen/functions.sh
+++ b/.evergreen/functions.sh
@@ -147,7 +147,10 @@ setup_ruby() {
 
     #rvm reinstall $RVM_RUBY
   else
-    if true; then
+    if test "$USE_OPT_TOOLCHAIN" = 1; then
+      # nothing, also PATH is already set
+      :
+    elif true; then
 
     # For testing toolchains:
     toolchain_url=https://s3.amazonaws.com//mciuploads/mongo-ruby-toolchain/`host_arch`/f11598d091441ffc8d746aacfdc6c26741a3e629/mongo_ruby_driver_toolchain_`host_arch |tr - _`_patch_f11598d091441ffc8d746aacfdc6c26741a3e629_5e46f2793e8e866f36eda2c5_20_02_14_19_18_18.tar.gz
diff --git a/.evergreen/test-on-docker b/.evergreen/test-on-docker
index 0c5691a5ce..3717c2fc0d 100755
--- a/.evergreen/test-on-docker
+++ b/.evergreen/test-on-docker
@@ -22,6 +22,10 @@ class Runner
         @options[:distro] = v
       end
 
+      opts.on('-p', '--preload', 'Preload Ruby toolchain and server binaries in docker') do |v|
+        @options[:preload] = v
+      end
+
       opts.on('-s', '--script=SCRIPT', 'Test script to invoke') do |v|
         @options[:script] = v
       end
@@ -99,6 +103,10 @@ class Runner
   def debian?
     distro =~ /debian|ubuntu/
   end
+
+  def preload?
+    !!@options[:preload]
+  end
 end
 
 Runner.new.run

From 1cd01757f528bc55be034dc86c67011bf8ed2a0e Mon Sep 17 00:00:00 2001
From: Oleg Pudeyev <oleg@bsdpower.com>
Date: Sun, 16 Feb 2020 16:59:23 -0500
Subject: [PATCH 06/19] preload server option

---
 .evergreen/Dockerfile.erb       |  5 +++++
 .evergreen/functions.sh         | 19 +++++++++++++++++--
 .evergreen/run-tests-mlaunch.sh | 13 +------------
 .evergreen/run-x509-tests.sh    |  3 +--
 .evergreen/test-on-docker       |  6 ++++++
 5 files changed, 30 insertions(+), 16 deletions(-)

diff --git a/.evergreen/Dockerfile.erb b/.evergreen/Dockerfile.erb
index e8cceb41c8..188e2fafdf 100644
--- a/.evergreen/Dockerfile.erb
+++ b/.evergreen/Dockerfile.erb
@@ -46,6 +46,11 @@ RUN apt-get install -y lsb-release python libsnmp30 libcurl3 krb5-user
   ENV PATH=/opt/rubies/<%= ruby %>/bin:/opt/rubies/python/3/bin:$PATH
   ENV USE_OPT_TOOLCHAIN=1
 
+  RUN curl --retry 3 -fL <%= server_download_url %> -o <%= File.basename(server_download_url) %>
+  RUN tar xfz <%= File.basename(server_download_url) %>
+  RUN mv mongo*/ /opt/mongodb
+  ENV USE_OPT_MONGODB=1
+
 <% end %>
 
 WORKDIR /app
diff --git a/.evergreen/functions.sh b/.evergreen/functions.sh
index 822020d7d8..d50c082d0c 100644
--- a/.evergreen/functions.sh
+++ b/.evergreen/functions.sh
@@ -228,9 +228,24 @@ kill_jruby() {
 
 prepare_server() {
   arch=$1
-  version=$2
+  
+  if test -n "$USE_OPT_MONGODB"; then
+    export BINDIR=/opt/mongodb/bin
+    export PATH=$BINDIR:$PATH
+    return
+  fi
+
+  if test "$MONGODB_VERSION" = 2.6; then
+    # The only OS which has Python toolchain for Python 3.6+ and
+    # which has MongoDB 2.6 server builds for it is rhel62.
+    # Unfortunately running it in Docker on a Debian 10 host crashes.
+    # Try the generic Linux binary since we aren't using any enterprise
+    # features pre FLE which requires 4.2 server.
+    url=https://fastdl.mongodb.org/linux/mongodb-linux-x86_64-2.6.12.tgz
+  else
+    url=`$(dirname $0)/get-mongodb-download-url $MONGODB_VERSION $arch`
+  fi
 
-  url=http://downloads.10gen.com/linux/mongodb-linux-x86_64-enterprise-$arch-$version.tgz
   prepare_server_from_url $url
 }
 
diff --git a/.evergreen/run-tests-mlaunch.sh b/.evergreen/run-tests-mlaunch.sh
index 210209779c..8e7a7a99bb 100755
--- a/.evergreen/run-tests-mlaunch.sh
+++ b/.evergreen/run-tests-mlaunch.sh
@@ -22,18 +22,7 @@ setup_ruby
 
 arch=`host_arch`
 
-if test "$MONGODB_VERSION" = 2.6; then
-  # The only OS which has Python toolchain for Python 3.6+ and
-  # which has MongoDB 2.6 server builds for it is rhel62.
-  # Unfortunately running it in Docker on a Debian 10 host crashes.
-  # Try the generic Linux binary since we aren't using any enterprise
-  # features pre FLE which requires 4.2 server.
-  url=https://fastdl.mongodb.org/linux/mongodb-linux-x86_64-2.6.12.tgz
-else
-  url=`$(dirname $0)/get-mongodb-download-url $MONGODB_VERSION $arch`
-fi
-
-prepare_server_from_url $url
+prepare_server $arch
 
 install_mlaunch_pip
 
diff --git a/.evergreen/run-x509-tests.sh b/.evergreen/run-x509-tests.sh
index 66e9859a9c..549eb6c5eb 100755
--- a/.evergreen/run-x509-tests.sh
+++ b/.evergreen/run-x509-tests.sh
@@ -13,8 +13,7 @@ setup_ruby
 install_deps
 
 arch=`host_arch`
-version=4.2.3
-prepare_server $arch $version
+prepare_server $arch
 
 install_mlaunch_pip
 
diff --git a/.evergreen/test-on-docker b/.evergreen/test-on-docker
index 3717c2fc0d..bde50af082 100755
--- a/.evergreen/test-on-docker
+++ b/.evergreen/test-on-docker
@@ -3,6 +3,8 @@
 require 'optparse'
 require 'erb'
 
+load File.join(File.dirname(__FILE__), 'tools.rb')
+
 class Runner
   def run
     process_arguments
@@ -107,6 +109,10 @@ class Runner
   def preload?
     !!@options[:preload]
   end
+
+  def server_download_url
+    @server_download_url ||= ServerVersionRegistry.new(server_version, distro).download_url
+  end
 end
 
 Runner.new.run

From f27cfd56a92c07556d2a424a3f98cf51660da988 Mon Sep 17 00:00:00 2001
From: Oleg Pudeyev <oleg@bsdpower.com>
Date: Sun, 16 Feb 2020 17:47:30 -0500
Subject: [PATCH 07/19] preinstall gems when preloading ruby

---
 .evergreen/Dockerfile.erb | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/.evergreen/Dockerfile.erb b/.evergreen/Dockerfile.erb
index 188e2fafdf..dc9afc2853 100644
--- a/.evergreen/Dockerfile.erb
+++ b/.evergreen/Dockerfile.erb
@@ -55,6 +55,16 @@ RUN apt-get install -y lsb-release python libsnmp30 libcurl3 krb5-user
 
 WORKDIR /app
 
+<% if preload? %>
+
+  COPY Gemfile .
+  COPY gemfiles gemfiles
+  COPY *.gemspec .
+  COPY lib/mongo/version.rb lib/mongo/version.rb
+  RUN bundle install
+
+<% end %>
+
 COPY . .
 
 ENV MONGO_ORCHESTRATION_HOME=/tmpfs

From 3f13bb289ad814c7505dab5e93aa8e3bac920f53 Mon Sep 17 00:00:00 2001
From: Oleg Pudeyev <oleg@bsdpower.com>
Date: Sun, 16 Feb 2020 18:04:17 -0500
Subject: [PATCH 08/19] install git

---
 .evergreen/Dockerfile.erb | 13 ++++++++++---
 1 file changed, 10 insertions(+), 3 deletions(-)

diff --git a/.evergreen/Dockerfile.erb b/.evergreen/Dockerfile.erb
index dc9afc2853..8b6fa80003 100644
--- a/.evergreen/Dockerfile.erb
+++ b/.evergreen/Dockerfile.erb
@@ -33,10 +33,17 @@ FROM <%= base_image %>
 #RUN curl --retry 3 -fL <%= python_toolchain_url %> -o python-toolchain.tar.gz
 #RUN tar -xC /opt -zf python-toolchain.tar.gz
 
-RUN apt-get install -y libyaml-0-2 gcc make openjdk-8-jre
-
+# Ruby runtime dependencies: libyaml-0-2
+# Compiling ruby libraries: gcc make
+# JRuby: openjdk-8-jre
+# Server dependencies: libsnmp30 libcurl3
+# Determining OS we are running on: lsb-release
+# Kerberos testing: krb5-user
+# Installing mlaunch from git: git
+#
 # ubuntu1604: libcurl3
-RUN apt-get install -y lsb-release python libsnmp30 libcurl3 krb5-user
+RUN apt-get install -y libyaml-0-2 gcc make openjdk-8-jre git \
+  lsb-release libsnmp30 libcurl3 krb5-user
 
 <% if preload? %>
 

From f47f75ad246694c8fef74424e7eae534b78ac420 Mon Sep 17 00:00:00 2001
From: Oleg Pudeyev <oleg@bsdpower.com>
Date: Sun, 16 Feb 2020 21:26:09 -0500
Subject: [PATCH 09/19] support ubuntu1204 on docker

---
 .evergreen/Dockerfile.erb | 17 +++++++++++++++--
 .evergreen/test-on-docker |  1 +
 2 files changed, 16 insertions(+), 2 deletions(-)

diff --git a/.evergreen/Dockerfile.erb b/.evergreen/Dockerfile.erb
index 8b6fa80003..31c7be767f 100644
--- a/.evergreen/Dockerfile.erb
+++ b/.evergreen/Dockerfile.erb
@@ -42,8 +42,21 @@ FROM <%= base_image %>
 # Installing mlaunch from git: git
 #
 # ubuntu1604: libcurl3
-RUN apt-get install -y libyaml-0-2 gcc make openjdk-8-jre git \
-  lsb-release libsnmp30 libcurl3 krb5-user
+
+RUN apt-cache search libsnmp
+
+# ubuntu1204 only has openjdk-7-jre
+<% unless %w(ubuntu1204).include?(distro) %>
+  RUN apt-get install -y openjdk-8-jre
+<% end %>
+
+<% if %w(ubuntu1204).include?(distro) %>
+  RUN apt-get install -y libsnmp15
+<% else %>
+  RUN apt-get install -y libsnmp30
+<% end %>
+
+RUN apt-get install -y libyaml-0-2 gcc make git lsb-release libcurl3 krb5-user
 
 <% if preload? %>
 
diff --git a/.evergreen/test-on-docker b/.evergreen/test-on-docker
index bde50af082..09c058c50f 100755
--- a/.evergreen/test-on-docker
+++ b/.evergreen/test-on-docker
@@ -79,6 +79,7 @@ class Runner
   BASE_IMAGES = {
     'debian81' => 'debian:jessie',
     'debian92' => 'debian:stretch',
+    'ubuntu1204' => 'ubuntu:precise',
     'ubuntu1404' => 'ubuntu:trusty',
     'ubuntu1604' => 'ubuntu:xenial',
     'ubuntu1804' => 'ubuntu:bionic',

From 34f6cd5cbc2274798744b244cb410f25fa2ffb52 Mon Sep 17 00:00:00 2001
From: Oleg Pudeyev <oleg@bsdpower.com>
Date: Sun, 16 Feb 2020 21:42:00 -0500
Subject: [PATCH 10/19] ubuntu1404 support

---
 .evergreen/Dockerfile.erb | 9 +++------
 1 file changed, 3 insertions(+), 6 deletions(-)

diff --git a/.evergreen/Dockerfile.erb b/.evergreen/Dockerfile.erb
index 31c7be767f..24c36cc197 100644
--- a/.evergreen/Dockerfile.erb
+++ b/.evergreen/Dockerfile.erb
@@ -40,13 +40,9 @@ FROM <%= base_image %>
 # Determining OS we are running on: lsb-release
 # Kerberos testing: krb5-user
 # Installing mlaunch from git: git
-#
-# ubuntu1604: libcurl3
 
-RUN apt-cache search libsnmp
-
-# ubuntu1204 only has openjdk-7-jre
-<% unless %w(ubuntu1204).include?(distro) %>
+# ubuntu1204, ubuntu1404 only have openjdk-7-jre
+<% unless %w(ubuntu1204 ubuntu1404).include?(distro) %>
   RUN apt-get install -y openjdk-8-jre
 <% end %>
 
@@ -56,6 +52,7 @@ RUN apt-cache search libsnmp
   RUN apt-get install -y libsnmp30
 <% end %>
 
+# ubuntu1204, ubuntu1404, ubuntu1604: libcurl3
 RUN apt-get install -y libyaml-0-2 gcc make git lsb-release libcurl3 krb5-user
 
 <% if preload? %>

From 3a78d833c80c44cf760083f6c98e4781363238d8 Mon Sep 17 00:00:00 2001
From: Oleg Pudeyev <oleg@bsdpower.com>
Date: Sun, 16 Feb 2020 22:40:37 -0500
Subject: [PATCH 11/19] rhel support

---
 .evergreen/Dockerfile.erb | 49 +++++++++++++++++++++++----------------
 .evergreen/functions.sh   |  3 +++
 2 files changed, 32 insertions(+), 20 deletions(-)

diff --git a/.evergreen/Dockerfile.erb b/.evergreen/Dockerfile.erb
index 24c36cc197..c361baa4ef 100644
--- a/.evergreen/Dockerfile.erb
+++ b/.evergreen/Dockerfile.erb
@@ -1,7 +1,8 @@
-# erb spec/Dockerfile.erb >spec/Dockerfile &&  cat spec/Dockerfile && docker build -t test-driver  -f spec/Dockerfile .&& docker run -ti --tmpfs /tmpfs:exec test-driver
-
 # Python toolchain as of this writing is available on rhel62, debian92 and
 # ubuntu1604.
+#
+# To run rhel62 in docker, host system must be configured to emulate syscalls:
+# https://github.com/CentOS/sig-cloud-instance-images/issues/103
 
 <%
 
@@ -33,27 +34,35 @@ FROM <%= base_image %>
 #RUN curl --retry 3 -fL <%= python_toolchain_url %> -o python-toolchain.tar.gz
 #RUN tar -xC /opt -zf python-toolchain.tar.gz
 
-# Ruby runtime dependencies: libyaml-0-2
-# Compiling ruby libraries: gcc make
-# JRuby: openjdk-8-jre
-# Server dependencies: libsnmp30 libcurl3
-# Determining OS we are running on: lsb-release
-# Kerberos testing: krb5-user
-# Installing mlaunch from git: git
-
-# ubuntu1204, ubuntu1404 only have openjdk-7-jre
-<% unless %w(ubuntu1204 ubuntu1404).include?(distro) %>
-  RUN apt-get install -y openjdk-8-jre
-<% end %>
+<% if debian? %>
+
+  # Ruby runtime dependencies: libyaml-0-2
+  # Compiling ruby libraries: gcc make
+  # JRuby: openjdk-8-jre
+  # Server dependencies: libsnmp30 libcurl3
+  # Determining OS we are running on: lsb-release
+  # Kerberos testing: krb5-user
+  # Installing mlaunch from git: git
+
+  # ubuntu1204, ubuntu1404 only have openjdk-7-jre
+  <% unless %w(ubuntu1204 ubuntu1404).include?(distro) %>
+    RUN apt-get install -y openjdk-8-jre
+  <% end %>
+
+  <% if %w(ubuntu1204).include?(distro) %>
+    RUN apt-get install -y libsnmp15
+  <% else %>
+    RUN apt-get install -y libsnmp30
+  <% end %>
+
+  # ubuntu1204, ubuntu1404, ubuntu1604: libcurl3
+  RUN apt-get install -y libyaml-0-2 gcc make git lsb-release libcurl3 krb5-user
 
-<% if %w(ubuntu1204).include?(distro) %>
-  RUN apt-get install -y libsnmp15
 <% else %>
-  RUN apt-get install -y libsnmp30
-<% end %>
 
-# ubuntu1204, ubuntu1404, ubuntu1604: libcurl3
-RUN apt-get install -y libyaml-0-2 gcc make git lsb-release libcurl3 krb5-user
+  RUN yum install -y redhat-lsb-core which
+
+<% end %>
 
 <% if preload? %>
 
diff --git a/.evergreen/functions.sh b/.evergreen/functions.sh
index d50c082d0c..1236e18438 100644
--- a/.evergreen/functions.sh
+++ b/.evergreen/functions.sh
@@ -39,6 +39,9 @@ _detect_arch() {
     elif lsb_release -i |grep -q RedHat; then
       release=`lsb_release -r |awk '{print $2}' |tr -d .`
       arch="rhel$release"
+    elif lsb_release -i |grep -q CentOS; then
+      release=`lsb_release -r |awk '{print $2}' |cut -c 1 |sed -e s/7/70/ -e s/6/62/`
+      arch="rhel$release"
     else
       echo 'Unknown RHEL flavor' 1>&2
       return 1

From 0d49e99d2cc8a358b20767b355f6caa3dfaa7fb6 Mon Sep 17 00:00:00 2001
From: Oleg Pudeyev <oleg@bsdpower.com>
Date: Sun, 16 Feb 2020 23:40:23 -0500
Subject: [PATCH 12/19] need python toolchain when testing ruby-head

---
 .evergreen/Dockerfile.erb | 8 ++++++--
 .evergreen/functions.sh   | 6 ++++++
 .evergreen/test-on-docker | 4 ++++
 3 files changed, 16 insertions(+), 2 deletions(-)

diff --git a/.evergreen/Dockerfile.erb b/.evergreen/Dockerfile.erb
index c361baa4ef..5a13bb2b3c 100644
--- a/.evergreen/Dockerfile.erb
+++ b/.evergreen/Dockerfile.erb
@@ -31,8 +31,12 @@ FROM <%= base_image %>
 
 <% end %>
 
-#RUN curl --retry 3 -fL <%= python_toolchain_url %> -o python-toolchain.tar.gz
-#RUN tar -xC /opt -zf python-toolchain.tar.gz
+<% if ruby_head? %>
+
+  RUN curl --retry 3 -fL <%= python_toolchain_url %> -o python-toolchain.tar.gz
+  RUN tar -xC /opt -zf python-toolchain.tar.gz
+
+<% end %>
 
 <% if debian? %>
 
diff --git a/.evergreen/functions.sh b/.evergreen/functions.sh
index 1236e18438..94f162b3c2 100644
--- a/.evergreen/functions.sh
+++ b/.evergreen/functions.sh
@@ -141,6 +141,12 @@ setup_ruby() {
   fi
 
   if [ "$RVM_RUBY" == "ruby-head" ]; then
+    # When we use ruby-head, we do not install the Ruby toolchain.
+    # But we still need Python 3.6+ to run mlaunch.
+    # Since the ruby-head tests are run on ubuntu1604, we can use the
+    # globally installed Python toolchain.
+    export PATH=/opt/python/3.7/bin:$PATH
+
     # 12.04, 14.04 and 16.04 are good
     wget -O ruby-head.tar.bz2 http://rubies.travis-ci.org/ubuntu/`lsb_release -rs`/x86_64/ruby-head.tar.bz2
     tar xf ruby-head.tar.bz2
diff --git a/.evergreen/test-on-docker b/.evergreen/test-on-docker
index 09c058c50f..4583e49e6a 100755
--- a/.evergreen/test-on-docker
+++ b/.evergreen/test-on-docker
@@ -95,6 +95,10 @@ class Runner
     @env['RVM_RUBY']
   end
 
+  def ruby_head?
+    ruby == 'ruby-head'
+  end
+
   def server_version
     @env['MONGODB_VERSION']
   end

From ee3fd105a20b72e03d4cfa32c90bebc7faa501fb Mon Sep 17 00:00:00 2001
From: Oleg Pudeyev <oleg@bsdpower.com>
Date: Sun, 16 Feb 2020 23:55:00 -0500
Subject: [PATCH 13/19] use curl instead of wget

---
 .evergreen/functions.sh     | 2 +-
 .evergreen/run-fle-tests.sh | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/.evergreen/functions.sh b/.evergreen/functions.sh
index 94f162b3c2..56a4a2c3cf 100644
--- a/.evergreen/functions.sh
+++ b/.evergreen/functions.sh
@@ -148,7 +148,7 @@ setup_ruby() {
     export PATH=/opt/python/3.7/bin:$PATH
 
     # 12.04, 14.04 and 16.04 are good
-    wget -O ruby-head.tar.bz2 http://rubies.travis-ci.org/ubuntu/`lsb_release -rs`/x86_64/ruby-head.tar.bz2
+    curl -fLo ruby-head.tar.bz2 http://rubies.travis-ci.org/ubuntu/`lsb_release -rs`/x86_64/ruby-head.tar.bz2
     tar xf ruby-head.tar.bz2
     export PATH=`pwd`/ruby-head/bin:`pwd`/ruby-head/lib/ruby/gems/2.6.0/bin:$PATH
     ruby --version
diff --git a/.evergreen/run-fle-tests.sh b/.evergreen/run-fle-tests.sh
index c51edae346..7544aa083e 100755
--- a/.evergreen/run-fle-tests.sh
+++ b/.evergreen/run-fle-tests.sh
@@ -5,7 +5,7 @@ set -o errexit  # Exit the script with error if any of the commands fail
 
 . `dirname "$0"`/functions.sh
 
-wget "https://s3.amazonaws.com/mciuploads/libmongocrypt/all/master/latest/libmongocrypt-all.tar.gz"
+curl -fLo libmongocrypt-all.tar.gz "https://s3.amazonaws.com/mciuploads/libmongocrypt/all/master/latest/libmongocrypt-all.tar.gz"
 tar -xvf libmongocrypt-all.tar.gz
 
 export LIBMONGOCRYPT_PATH=`pwd`/rhel-70-64-bit/nocrypto/lib64/libmongocrypt.so

From 04ca5e3a99a9020a2f57139d0e3aa03a29713eb0 Mon Sep 17 00:00:00 2001
From: Oleg Pudeyev <oleg@bsdpower.com>
Date: Sun, 16 Feb 2020 23:57:32 -0500
Subject: [PATCH 14/19] need bzip2 for ruby-head

---
 .evergreen/Dockerfile.erb | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/.evergreen/Dockerfile.erb b/.evergreen/Dockerfile.erb
index 5a13bb2b3c..814cca6fbd 100644
--- a/.evergreen/Dockerfile.erb
+++ b/.evergreen/Dockerfile.erb
@@ -47,6 +47,7 @@ FROM <%= base_image %>
   # Determining OS we are running on: lsb-release
   # Kerberos testing: krb5-user
   # Installing mlaunch from git: git
+  # ruby-head archive: bzip2
 
   # ubuntu1204, ubuntu1404 only have openjdk-7-jre
   <% unless %w(ubuntu1204 ubuntu1404).include?(distro) %>
@@ -60,7 +61,7 @@ FROM <%= base_image %>
   <% end %>
 
   # ubuntu1204, ubuntu1404, ubuntu1604: libcurl3
-  RUN apt-get install -y libyaml-0-2 gcc make git lsb-release libcurl3 krb5-user
+  RUN apt-get install -y libyaml-0-2 gcc make git lsb-release libcurl3 krb5-user bzip2
 
 <% else %>
 

From b10643b60bbd9f015180d5e91b0cf1088db25356 Mon Sep 17 00:00:00 2001
From: Oleg Pudeyev <oleg@bsdpower.com>
Date: Sun, 16 Feb 2020 23:59:37 -0500
Subject: [PATCH 15/19] exclude ruby-head from preloading

---
 .evergreen/Dockerfile.erb | 15 ++++++++++-----
 1 file changed, 10 insertions(+), 5 deletions(-)

diff --git a/.evergreen/Dockerfile.erb b/.evergreen/Dockerfile.erb
index 814cca6fbd..7db985e473 100644
--- a/.evergreen/Dockerfile.erb
+++ b/.evergreen/Dockerfile.erb
@@ -72,10 +72,15 @@ FROM <%= base_image %>
 <% if preload? %>
 
   WORKDIR /app
-  RUN curl --retry 3 -fL <%= ruby_toolchain_url %> -o ruby-toolchain.tar.gz
-  RUN tar -xC /opt -zf ruby-toolchain.tar.gz
-  ENV PATH=/opt/rubies/<%= ruby %>/bin:/opt/rubies/python/3/bin:$PATH
-  ENV USE_OPT_TOOLCHAIN=1
+  
+  <% unless ruby_head? %>
+    
+    RUN curl --retry 3 -fL <%= ruby_toolchain_url %> -o ruby-toolchain.tar.gz
+    RUN tar -xC /opt -zf ruby-toolchain.tar.gz
+    ENV PATH=/opt/rubies/<%= ruby %>/bin:/opt/rubies/python/3/bin:$PATH
+    ENV USE_OPT_TOOLCHAIN=1
+  
+  <% end %>
 
   RUN curl --retry 3 -fL <%= server_download_url %> -o <%= File.basename(server_download_url) %>
   RUN tar xfz <%= File.basename(server_download_url) %>
@@ -86,7 +91,7 @@ FROM <%= base_image %>
 
 WORKDIR /app
 
-<% if preload? %>
+<% if preload? && !ruby_head? %>
 
   COPY Gemfile .
   COPY gemfiles gemfiles

From b95e54782f112089cb864576b89b8f86faeec3ad Mon Sep 17 00:00:00 2001
From: Oleg Pudeyev <oleg@bsdpower.com>
Date: Mon, 17 Feb 2020 00:00:58 -0500
Subject: [PATCH 16/19] switch the order of server and ruby preloading to cache
 the server always

---
 .evergreen/Dockerfile.erb | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/.evergreen/Dockerfile.erb b/.evergreen/Dockerfile.erb
index 7db985e473..3e1b124b1f 100644
--- a/.evergreen/Dockerfile.erb
+++ b/.evergreen/Dockerfile.erb
@@ -72,6 +72,11 @@ FROM <%= base_image %>
 <% if preload? %>
 
   WORKDIR /app
+
+  RUN curl --retry 3 -fL <%= server_download_url %> -o <%= File.basename(server_download_url) %>
+  RUN tar xfz <%= File.basename(server_download_url) %>
+  RUN mv mongo*/ /opt/mongodb
+  ENV USE_OPT_MONGODB=1
   
   <% unless ruby_head? %>
     
@@ -82,11 +87,6 @@ FROM <%= base_image %>
   
   <% end %>
 
-  RUN curl --retry 3 -fL <%= server_download_url %> -o <%= File.basename(server_download_url) %>
-  RUN tar xfz <%= File.basename(server_download_url) %>
-  RUN mv mongo*/ /opt/mongodb
-  ENV USE_OPT_MONGODB=1
-
 <% end %>
 
 WORKDIR /app

From fd4a8fecd40157fa2195b5eab607314b1f58ecaf Mon Sep 17 00:00:00 2001
From: Oleg Pudeyev <oleg@bsdpower.com>
Date: Mon, 17 Feb 2020 00:05:25 -0500
Subject: [PATCH 17/19] do not require python (2) presence

---
 .evergreen/functions.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.evergreen/functions.sh b/.evergreen/functions.sh
index 56a4a2c3cf..e9ffec2e5f 100644
--- a/.evergreen/functions.sh
+++ b/.evergreen/functions.sh
@@ -270,7 +270,7 @@ prepare_server_from_url() {
 
 install_mlaunch_virtualenv() {
   #export PATH=/opt/python/3.7/bin:$PATH
-  python -V
+  python -V || true
   python3 -V
   #pip3 install --user virtualenv
   venvpath="$MONGO_ORCHESTRATION_HOME"/venv
@@ -280,7 +280,7 @@ install_mlaunch_virtualenv() {
 }
 
 install_mlaunch_pip() {
-  python -V
+  python -V || true
   python3 -V
   pythonpath="$MONGO_ORCHESTRATION_HOME"/python
   # The scripts in a python installation have shebangs pointing to the

From 00b9d68968ce4b5b790aa84bf5ca6799fa652354 Mon Sep 17 00:00:00 2001
From: Oleg Pudeyev <oleg@bsdpower.com>
Date: Mon, 17 Feb 2020 00:52:38 -0500
Subject: [PATCH 18/19] check ruby version directly in the shell

---
 .evergreen/functions.sh | 11 +++++------
 1 file changed, 5 insertions(+), 6 deletions(-)

diff --git a/.evergreen/functions.sh b/.evergreen/functions.sh
index e9ffec2e5f..35206eb181 100644
--- a/.evergreen/functions.sh
+++ b/.evergreen/functions.sh
@@ -190,12 +190,11 @@ setup_ruby() {
     ruby --version
 
     # Ensure we're using the right ruby
-    python - <<EOH
-ruby = "${RVM_RUBY}".split("-")[0]
-version = "${RVM_RUBY}".split("-")[1]
-assert(ruby in "`ruby --version`")
-assert(version in "`ruby --version`")
-EOH
+    ruby_name=`echo $RVM_RUBY |awk -F- '{print $1}'`
+    ruby_version=`echo $RVM_RUBY |awk -F- '{print $2}' |cut -c 1-3`
+    
+    ruby -v |fgrep $ruby_name
+    ruby -v |fgrep $ruby_version
 
     # We shouldn't need to update rubygems, and there is value in
     # testing on whatever rubygems came with each supported ruby version

From ffb6a183a99d433a17d276ba3400f62aeeff4d55 Mon Sep 17 00:00:00 2001
From: Oleg Pudeyev <oleg@bsdpower.com>
Date: Mon, 17 Feb 2020 13:47:57 -0500
Subject: [PATCH 19/19] add documentation

---
 .evergreen/README.md | 15 +++++++++++++++
 1 file changed, 15 insertions(+)

diff --git a/.evergreen/README.md b/.evergreen/README.md
index e51c828861..9d5f3d1b43 100644
--- a/.evergreen/README.md
+++ b/.evergreen/README.md
@@ -21,3 +21,18 @@ By default the entire test suite is run (using mlaunch to launch the server);
 to specify another script, use `-s` option:
 
     ./.evergreen/run-on-docker -s .evergreen/run-enterprise-auth-tests.sh
+
+### Toolchain and Server Preloading
+
+The docker test runner supports preloading Ruby interpreters and server
+binaries in the docker image, which reduces the runtime of subsequent
+test runs. To turn on preloading, use `-p` option:
+
+    ./.evergreen/run-on-docker -p
+
+### rhel62
+
+To run rhel62 distro in docker, host system must be configured to [emulate
+syscalls](https://github.com/CentOS/sig-cloud-instance-images/issues/103).
+Note that this defeats one of the patches for the Spectre set of processor
+vulnerabilities.