diff --git a/.evergreen/Dockerfile.erb b/.evergreen/Dockerfile.erb deleted file mode 100644 index bb18722063..0000000000 --- a/.evergreen/Dockerfile.erb +++ /dev/null @@ -1,221 +0,0 @@ -# Python toolchain as of this writing is available on rhel62, debian92 and -# ubuntu1604. -# -# To run rhel62 in docker, host system must be configured to emulate syscalls: -# https://github.com/CentOS/sig-cloud-instance-images/issues/103 - -<% - -python_toolchain_url = "https://s3.amazonaws.com//mciuploads/mongo-python-driver-toolchain/#{distro}/ba92de2700c04ee2d4f82c3ffdfc33105140cb04/mongo_python_driver_toolchain_#{distro.gsub('-', '_')}_ba92de2700c04ee2d4f82c3ffdfc33105140cb04_19_11_14_15_33_33.tar.gz" -server_version = '4.3.3' -server_url = "http://downloads.10gen.com/linux/mongodb-linux-x86_64-enterprise-#{distro}-#{server_version}.tgz" -server_archive_basename = File.basename(server_url) -server_extracted_dir = server_archive_basename.sub(/\.(tar\.gz|tgz)$/, '') - -toolchain_upper='291ba4a4e8297f142796e70eee71b99f333e35e1' - -ruby_toolchain_url = "http://boxes.10gen.com/build/toolchain-drivers/mongo-ruby-driver/ruby-toolchain-#{distro}-#{toolchain_upper}.tar.xz" -#ruby_toolchain_url = "https://s3.amazonaws.com//mciuploads/mongo-ruby-toolchain/#{distro}/#{toolchain_upper}/mongo_ruby_driver_toolchain_#{distro.gsub('-', '_')}_patch_#{toolchain_upper}_#{toolchain_lower}.tar.gz" - -%> - -FROM <%= base_image %> - -<% if debian? %> - - ENV DEBIAN_FRONTEND=noninteractive - - # increment the counter to force apt-get update to run. - # zsh is not required for any scripts but it is a better interactive shell - # than bash. - RUN echo 4 && \ - apt-get update && \ - apt-get install -y curl zsh - -<% else %> - - RUN echo assumeyes=1 |tee -a /etc/yum.conf - -<% end %> - -<% if ruby_head? %> - - # To use current versions of mlaunch, Python 3.6+ is required. - # Most distros ship with older Pythons, therefore we need to install - # a newer Python from somewhere. This section installs the Python - # toolhcain which comes with recent Pythons. - # Alternatively, Ruby toolchain compiles its own copy of Python 3 but - # this is currently incomplete in that on older distros with old OpenSSL, - # the built Python has no ssl module and hence practically is unusable. - # Currently Ruby driver uses mtools-legacy which supports Python 2, - # avoiding this entire issue for the time being. - - #RUN curl --retry 3 -fL <%= python_toolchain_url %> -o python-toolchain.tar.gz - #RUN tar -xC /opt -zf python-toolchain.tar.gz - -<% end %> - -<% if debian? %> - - # Ruby runtime dependencies: libyaml-0-2 - # Compiling ruby libraries: gcc make - # Compiling pyhton packages: python2.7-dev - # JRuby: openjdk-8-jre - # Server dependencies: libsnmp30 libcurl3/libcurl4 - # Determining OS we are running on: lsb-release - # Kerberos testing: krb5-user - # Local Kerberos server: krb5-kdc krb5-admin-server - # Installing mlaunch from git: git - # ruby-head archive: bzip2 - # nio4r on JRuby: libgmp-dev - # Snappy compression: libsnappy-dev - # - # We currently use Python 2-compatible version of mtools, which - # is installable via pip (which uses Python 2). All of the MongoDB - # distros have pip installed (but none as of this writing have pip3) - # therefore install python-pip in all configurations here. - - # ubuntu1404 only has openjdk-7-jre - <% if distro !~ /ubuntu1404/ %> - RUN apt-get install -y openjdk-8-jre - <% end %> - - # ubuntu1404, ubuntu1604: libcurl3 - # ubuntu1804: libcurl4 - <% if distro =~ /ubuntu1804/ %> - RUN apt-get install -y libcurl4 - <% else %> - RUN apt-get install -y libcurl3 - <% end %> - - RUN apt-get install -y libsnmp30 libyaml-0-2 gcc make git lsb-release \ - krb5-user krb5-kdc krb5-admin-server libsasl2-dev libsasl2-modules-gssapi-mit \ - bzip2 libgmp-dev python-pip python2.7-dev python3-pip libsnappy-dev - -<% else %> - - # Enterprise server: net-snmp - # lsb_release: redhat-lsb-core - # our runner scripts: which - # Ruby dependency: libyaml - # compiling python packages: gcc python-devel - # Kerberos tests: krb5-workstation + cyrus-sasl-devel to build the - # mongo_kerberos gem + cyrus-sasl-gssapi for authentication to work - # Local Kerberos server: krb5-server - # JRuby: java-1.8.0-openjdk - # - # Note: lacking cyrus-sasl-gssapi produces a cryptic message - # "SASL(-4): no mechanism available: No worthy mechs found" - # https://github.com/farorm/python-ad/issues/10 - - RUN yum install -y redhat-lsb-core which git gcc libyaml krb5-server \ - krb5-workstation cyrus-sasl-devel cyrus-sasl-gssapi java-1.8.0-openjdk \ - net-snmp - - <% if distro =~ /rhel6/ %> - - # RHEL 6 ships with Python 2.6. - - RUN yum install -y centos-release-scl && \ - yum install -y python27-python python27-python-devel - ENV PATH=/opt/rh/python27/root/usr/bin:$PATH \ - LD_LIBRARY_PATH=/opt/rh/python27/root/usr/lib64 - - <% else %> - - RUN yum install -y python-devel - - <% end %> - -<% end %> - -<% if preload? %> - - WORKDIR /app - - RUN curl --retry 3 -fL <%= server_download_url %> |tar xzf - && \ - mv mongo*/ /opt/mongodb - ENV USE_OPT_MONGODB=1 - - <% unless ruby_head? %> - - RUN curl --retry 3 -fL <%= ruby_toolchain_url %> |tar -xC /opt -Jf - - ENV PATH=/opt/rubies/<%= ruby %>/bin:$PATH \ - USE_OPT_TOOLCHAIN=1 - #ENV PATH=/opt/rubies/python/3/bin:$PATH - - <% end %> - - <% if distro =~ /rhel|ubuntu1604/ %> - - # Ubuntu 12.04 ships pip 1.0 which is ancient and does not work. - # - # Ubuntu 16.04 apparently also ships a pip that does not work: - # https://stackoverflow.com/questions/37495375/python-pip-install-throws-typeerror-unsupported-operand-types-for-retry - # Potentially this only affects environments with less than ideal - # connectivity (or, perhaps, when python package registry is experiencing - # availability issues) when pip must retry to install packages. - # - # rhel apparently does not package pip at all in core repoitories, - # therefore install it the manual way. - # - # https://pip.pypa.io/en/stable/installing/ - RUN curl --retry 3 -fL https://raw.githubusercontent.com/pypa/get-pip/master/2.7/get-pip.py | python - - <% end %> - - RUN pip --version && \ - pip install mtools-legacy[mlaunch] - - <% if @env.fetch('MONGODB_VERSION') >= '4.4' %> - RUN python3 -mpip install asn1crypto oscrypto flask - <% end %> - -<% end %> - -WORKDIR /app - -<% if preload? && !ruby_head? %> - - COPY Gemfile . - COPY gemfiles gemfiles - COPY *.gemspec . - COPY lib/mongo/version.rb lib/mongo/version.rb - RUN bundle install - COPY .evergreen/patch-debuggers .evergreen/patch-debuggers - RUN .evergreen/patch-debuggers /opt/rubies - -<% end %> - -<% if fle? %> - RUN curl --retry 3 -fLo libmongocrypt-all.tar.gz "https://s3.amazonaws.com/mciuploads/libmongocrypt/all/master/latest/libmongocrypt-all.tar.gz" - RUN tar xf libmongocrypt-all.tar.gz - - <%= "ENV LIBMONGOCRYPT_PATH #{libmongocrypt_path}" %> -<% end %> - -ENV MONGO_ORCHESTRATION_HOME=/tmpfs \ - PROJECT_DIRECTORY=/app \ - <%= @env.map { |k, v| %Q`#{k}="#{v.gsub('$', "\\$").gsub('"', "\\\"")}"` }.join(" \\\n ") %> - -<% if interactive? %> - ENV INTERACTIVE=1 -<% end %> - -COPY . . - -<% if expose? %> - - <% ports = [] %> - - <% 0.upto(num_exposed_ports-1) do |i| %> - <% ports << 27017 + i %> - <% end %> - - <% if @env['OCSP_ALGORITHM'] %> - <% ports << 8100 %> - <% end %> - - EXPOSE <%= ports.map(&:to_s).join(' ') %> - -<% end %> diff --git a/.evergreen/get-mongodb-download-url b/.evergreen/get-mongodb-download-url index 9872fd0071..5eb58dbce0 100755 --- a/.evergreen/get-mongodb-download-url +++ b/.evergreen/get-mongodb-download-url @@ -6,6 +6,7 @@ if arch.nil? exit 1 end -load File.join(File.dirname(__FILE__), 'tools.rb') +$: << File.join(File.dirname(__FILE__), '../spec/shared/lib') +require 'mrss/server_version_registry' -puts ServerVersionRegistry.new(desired_version, arch).download_url +puts Mrss::ServerVersionRegistry.new(desired_version, arch).download_url diff --git a/.evergreen/test-on-docker b/.evergreen/test-on-docker index 0b4c38b8ea..5887f70258 100755 --- a/.evergreen/test-on-docker +++ b/.evergreen/test-on-docker @@ -1,237 +1,11 @@ #!/usr/bin/env ruby -require 'optparse' -require 'erb' -autoload :Dotenv, 'dotenv' - -load File.join(File.dirname(__FILE__), 'tools.rb') - -class Runner - def run - process_arguments - unless @options[:exec_only] - create_dockerfile - create_image - end - if @options[:mongo_only] - run_deployment - else - run_tests - end - end - - private - - def process_arguments - @options = {} - OptionParser.new do |opts| - opts.banner = "Usage: test-on-docker [-d distro] [evergreen_key=value ...]" - - opts.on("-a", "--add-env=PATH", "Load environment variables from PATH in .env format") do |path| - @options[:extra_env] ||= {} - unless File.exist?(path) - raise "-a option references nonexistent file #{path}" - end - Dotenv.parse(path).each do |k, v| - @options[:extra_env][k] = v - end - end - - opts.on("-d", "--distro=DISTRO", "Distro to use") do |v| - @options[:distro] = v - end - - opts.on('-e', '--exec-only', 'Execute tests using existing Dockerfile (for offline user)') do |v| - @options[:exec_only] = v - end - - opts.on('-m', '--mongo-only=PORT', 'Start the MongoDB deployment and expose it to host on ports starting with PORT') do |v| - @options[:mongo_only] = v.to_i - end - - opts.on('-p', '--preload', 'Preload Ruby toolchain and server binaries in docker') do |v| - @options[:preload] = v - end - - opts.on('-s', '--script=SCRIPT', 'Test script to invoke') do |v| - @options[:script] = v - end - - opts.on('-i', '--interactive', 'Interactive mode - disable per-test timeouts') do |v| - @options[:interactive] = v - end - end.parse! - - @env = Hash[ARGV.map do |arg| - arg.split('=', 2) - end] - - @env['RVM_RUBY'] ||= 'ruby-2.7' - unless ruby =~ /^j?ruby-/ - raise "RVM_RUBY option is not in expected format: #{ruby}" - end - - @env['MONGODB_VERSION'] ||= '4.4' - end - - def create_dockerfile - template_path = File.join(File.dirname(__FILE__), 'Dockerfile.erb') - result = ERB.new(File.read(template_path)).result(binding) - File.open(File.join(File.dirname(__FILE__), 'Dockerfile'), 'w') do |f| - f << result - end - end - - def create_image - run_command(%w(docker build -t test-driver -f .evergreen/Dockerfile .)) - end - - BASE_TEST_COMMAND = %w(docker run -i --tmpfs /tmpfs:exec).freeze - - def run_tests - run_command(BASE_TEST_COMMAND + tty_arg + extra_env + %w(test-driver) + - script.split(/\s+/)) - end - - def run_deployment - run_command(BASE_TEST_COMMAND + tty_arg + extra_env + [ - '-e', %q`TEST_CMD=watch -x bash -c "ps awwxu |egrep 'mongo|ocsp'"`, - '-e', 'BIND_ALL=true', - ] + port_forwards + %w(test-driver) + script.split(/\s+/)) - end - - def tty_arg - tty = File.open('/dev/stdin') do |f| - f.isatty - end - if tty - %w(-t --init) - else - [] - end - end - - def extra_env - if @options[:extra_env] - @options[:extra_env].map do |k, v| - # Here the value must not be escaped - ['-e', "#{k}=#{v}"] - end.flatten - else - [] - end - end - - def port_forwards - args = (0...num_exposed_ports).map do |i| - host_port = @options[:mongo_only] + i - container_port = 27017 + i - ['-p', "#{host_port}:#{container_port}"] - end.flatten - - if @env['OCSP_ALGORITHM'] && !@env['OCSP_VERIFIER'] - args += %w(-p 8100:8100) - end - - args - end - - def run_command(cmd) - if pid = fork - Process.wait(pid) - unless $?.exitstatus == 0 - raise "Process exited with code #{$?.exitstatus}" - end - else - exec(*cmd) - end - end - - def distro - @options[:distro] || 'ubuntu1604' - end - - BASE_IMAGES = { - 'debian81' => 'debian:jessie', - 'debian92' => 'debian:stretch', - 'ubuntu1404' => 'ubuntu:trusty', - 'ubuntu1604' => 'ubuntu:xenial', - 'ubuntu1804' => 'ubuntu:bionic', - 'rhel62' => 'centos:6', - 'rhel70' => 'centos:7', - }.freeze - - def base_image - BASE_IMAGES[distro] or raise "Unknown distro: #{distro}" - end - - def ruby - @env['RVM_RUBY'] - end - - def ruby_head? - ruby == 'ruby-head' - end - - def server_version - @env['MONGODB_VERSION'] - end - - def script - @options[:script] || 'bash -x .evergreen/run-tests.sh' - end - - def debian? - distro =~ /debian|ubuntu/ - end - - def preload? - !!@options[:preload] - end - - def interactive? - !!@options[:interactive] - end - - def server_download_url - @server_download_url ||= ServerVersionRegistry.new(server_version, distro).download_url - end - - def libmongocrypt_path - case distro - when /ubuntu1604/ - "./ubuntu1604/nocrypto/lib64/libmongocrypt.so" - when /ubuntu1804/ - "./ubuntu1804-64/nocrypto/lib64/libmongocrypt.so" - when /debian92/ - "./debian92/nocrypto/lib64/libmongocrypt.so" - else - raise "This script does not support running FLE tests on #{distro}. Use ubuntu1604, ubuntu1804 or debian92 instead" - end - end - - def expose? - !!@options[:mongo_only] - end - - def fle? - %w(1 true yes).include?(@env['FLE']&.downcase) - end - - def num_exposed_ports - case @env['TOPOLOGY'] || 'standalone' - when 'standalone' - 1 - when 'replica-set' - 3 - when 'sharded-cluster' - if @env['SINGLE_MONGOS'] - 1 - else - 2 - end - end - end -end - -Runner.new.run +$: << File.join(File.dirname(__FILE__), '../spec/shared/lib') +require 'mrss/docker_runner' + +Mrss::DockerRunner.new( + image_tag: 'ruby-driver-test', + dockerfile_path: '.evergreen/Dockerfile', + default_script: 'bash -x .evergreen/run-tests.sh', + project_lib_subdir: 'mongo', +).run diff --git a/spec/shared b/spec/shared index 2fdcf269fb..c832db3224 160000 --- a/spec/shared +++ b/spec/shared @@ -1 +1 @@ -Subproject commit 2fdcf269fb4cf78d4318e2accb2736e60cb341ae +Subproject commit c832db32247426d6fd9f2c8607d3e1b684986496