run-tests.sh

#!/bin/bash

# Note that mlaunch is executed with (and therefore installed with) Python 2.
# The reason for this is that in the past, some of the distros we tested on
# had an ancient version of Python 3 that was unusable (e.g. it couldn't
# install anything from PyPI due to outdated TLS/SSL implementation).
# It is likely that all of the current distros we use have a recent enough
# and working Python 3 implementation, such that we could use Python 3 for
# everything.
#
# Note that some distros (e.g. ubuntu2004) do not contain a `python' binary
# at all, thus python2 or python3 must be explicitly specified depending on
# the desired version.

set -e
set -o pipefail

if echo "$AUTH" |grep -q ^aws; then
  # Do not set -x as this will expose passwords in Evergreen logs
  set +x
else
  set -x
fi

MRSS_ROOT=`dirname "$0"`/../spec/shared

. $MRSS_ROOT/shlib/distro.sh
. $MRSS_ROOT/shlib/set_env.sh
. $MRSS_ROOT/shlib/server.sh
. $MRSS_ROOT/shlib/config.sh
. `dirname "$0"`/functions.sh
. `dirname "$0"`/functions-aws.sh
. `dirname "$0"`/functions-config.sh

arch=`host_distro`

show_local_instructions

set_home
set_env_vars
set_env_python
set_env_ruby

prepare_server $arch

if test "$DOCKER_PRELOAD" != 1; then
  install_mlaunch_venv
fi

# Make sure cmake is installed (in case we need to install the libmongocrypt
# helper)
if [ "$FLE" = "helper" ]; then
  install_cmake
fi

# Launching mongod under $MONGO_ORCHESTRATION_HOME
# makes its log available through log collecting machinery

export dbdir="$MONGO_ORCHESTRATION_HOME"/db
mkdir -p "$dbdir"

if test -z "$TOPOLOGY"; then
  export TOPOLOGY=standalone
fi

calculate_server_args
launch_ocsp_mock
launch_server "$dbdir"

uri_options="$URI_OPTIONS"

bundle_install

if test "$TOPOLOGY" = sharded-cluster; then
  if test -n "$SINGLE_MONGOS"; then
    # Some tests may run into https://jira.mongodb.org/browse/SERVER-16836
    # when executing against a multi-sharded mongos.
    # At the same time, due to pinning in sharded transactions,
    # it is beneficial to test a single shard to ensure that server
    # monitoring and selection are working correctly and recover the driver's
    # ability to operate in reasonable time after errors and fail points trigger
    # on a single shard
    echo Restricting to a single mongos
    hosts=localhost:27017
  else
    hosts=localhost:27017,localhost:27018
  fi
elif test "$TOPOLOGY" = replica-set; then
  # To set FCV we use mongo shell, it needs to be placed in replica set topology
  # or it can try to send the commands to secondaries.
  hosts=localhost:27017,localhost:27018
  uri_options="$uri_options&replicaSet=test-rs"
else
  hosts=localhost:27017
fi

if test "$AUTH" = auth; then
  hosts="bob:pwd123@$hosts"
elif test "$AUTH" = x509; then
  create_user_cmd="`cat <<'EOT'
    db.getSiblingDB("$external").runCommand(
      {
        createUser: "C=US,ST=New York,L=New York City,O=MongoDB,OU=x509,CN=localhost",
        roles: [
             { role: "root", db: "admin" },
        ],
        writeConcern: { w: "majority" , wtimeout: 5000 },
      }
    )
EOT
  `"

  "$BINDIR"/mongo --tls \
    --tlsCAFile spec/support/certificates/ca.crt \
    --tlsCertificateKeyFile spec/support/certificates/client-x509.pem \
    -u bootstrap -p bootstrap \
    --eval "$create_user_cmd"
elif test "$AUTH" = aws-regular; then
  clear_instance_profile

  ruby -Ilib -I.evergreen/lib -rserver_setup -e ServerSetup.new.setup_aws_auth

  hosts="`uri_escape $MONGO_RUBY_DRIVER_AWS_AUTH_ACCESS_KEY_ID`:`uri_escape $MONGO_RUBY_DRIVER_AWS_AUTH_SECRET_ACCESS_KEY`@$hosts"
elif test "$AUTH" = aws-assume-role; then
  clear_instance_profile

  ./.evergreen/aws -a "$MONGO_RUBY_DRIVER_AWS_AUTH_ACCESS_KEY_ID" \
    -s "$MONGO_RUBY_DRIVER_AWS_AUTH_SECRET_ACCESS_KEY" \
    -r us-east-1 \
    assume-role "$MONGO_RUBY_DRIVER_AWS_AUTH_ASSUME_ROLE_ARN" >.env.private.gen
  eval `cat .env.private.gen`
  export MONGO_RUBY_DRIVER_AWS_AUTH_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID
  export MONGO_RUBY_DRIVER_AWS_AUTH_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY
  export MONGO_RUBY_DRIVER_AWS_AUTH_SESSION_TOKEN=$AWS_SESSION_TOKEN
  ruby -Ilib -I.evergreen/lib -rserver_setup -e ServerSetup.new.setup_aws_auth

  export AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID
  export AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY
  export AWS_SESSION_TOKEN=$AWS_SESSION_TOKEN

  aws sts get-caller-identity

  hosts="`uri_escape $MONGO_RUBY_DRIVER_AWS_AUTH_ACCESS_KEY_ID`:`uri_escape $MONGO_RUBY_DRIVER_AWS_AUTH_SECRET_ACCESS_KEY`@$hosts"

  uri_options="$uri_options&"\
"authMechanismProperties=AWS_SESSION_TOKEN:`uri_escape $MONGO_RUBY_DRIVER_AWS_AUTH_SESSION_TOKEN`"
elif test "$AUTH" = aws-ec2; then
  ruby -Ilib -I.evergreen/lib -rserver_setup -e ServerSetup.new.setup_aws_auth

  # We need to assign an instance profile to the current instance, otherwise
  # since we don't place credentials into the environment the test suite
  # cannot connect to the MongoDB server while bootstrapping.
  # The EC2 credential retrieval tests clears the instance profile as part
  # of one of the tests.
  ruby -Ispec -Ilib -I.evergreen/lib -rec2_setup -e Ec2Setup.new.assign_instance_profile
elif test "$AUTH" = aws-ecs; then
  if test -z "$AWS_CONTAINER_CREDENTIALS_RELATIVE_URI"; then
    # drivers-evergreen-tools performs this operation in its ECS E2E tester.
    eval export `strings /proc/1/environ |grep ^AWS_CONTAINER_CREDENTIALS_RELATIVE_URI`
  fi

  ruby -Ilib -I.evergreen/lib -rserver_setup -e ServerSetup.new.setup_aws_auth
elif test "$AUTH" = aws-web-identity; then
  clear_instance_profile

  ruby -Ilib -I.evergreen/lib -rserver_setup -e ServerSetup.new.setup_aws_auth
elif test "$AUTH" = kerberos; then
  export MONGO_RUBY_DRIVER_KERBEROS=1
fi

if test -n "$FLE"; then
  # Downloading crypt shared lib
  if [ -z "$MONGO_CRYPT_SHARED_DOWNLOAD_URL" ]; then
    crypt_shared_version=${CRYPT_SHARED_VERSION:-$("${BINDIR}"/mongod --version | grep -oP 'db version v\K.*')}
    python3 -u .evergreen/mongodl.py --component crypt_shared -V ${crypt_shared_version} --out $(pwd)/csfle_lib  --target $(host_distro) || true
    if test -f $(pwd)/csfle_lib/lib/mongo_crypt_v1.so
    then
      export MONGO_RUBY_DRIVER_CRYPT_SHARED_LIB_PATH=$(pwd)/csfle_lib/lib/mongo_crypt_v1.so
    else
      echo 'Could not find crypt_shared library'
    fi
  else
    echo "Downloading crypt_shared package from $MONGO_CRYPT_SHARED_DOWNLOAD_URL"
    mkdir -p $(pwd)/csfle_lib
    cd $(pwd)/csfle_lib
    curl --retry 3 -fL $MONGO_CRYPT_SHARED_DOWNLOAD_URL | tar zxf -
    export MONGO_RUBY_DRIVER_CRYPT_SHARED_LIB_PATH=$(pwd)/lib/mongo_crypt_v1.so
    cd -
  fi

  # Start the KMS servers first so that they are launching while we are
  # fetching libmongocrypt.
  if test "$DOCKER_PRELOAD" != 1; then
    # We already have a virtualenv activated for mlaunch,
    # install kms dependencies into it.
    #. .evergreen/csfle/activate_venv.sh

    # Adjusted package versions:
    # cryptography 3.4 requires rust, see
    # https://github.com/pyca/cryptography/issues/5771.
    #pip install boto3~=1.19 cryptography~=3.4.8 pykmip~=0.10.0
    pip3 install boto3~=1.19 'cryptography<3.4' pykmip~=0.10.0 'sqlalchemy<2.0.0'
  fi
  python3 -u .evergreen/csfle/kms_http_server.py --ca_file .evergreen/x509gen/ca.pem --cert_file .evergreen/x509gen/server.pem --port 7999 &
  python3 -u .evergreen/csfle/kms_http_server.py --ca_file .evergreen/x509gen/ca.pem --cert_file .evergreen/x509gen/expired.pem --port 8000 &
  python3 -u .evergreen/csfle/kms_http_server.py --ca_file .evergreen/x509gen/ca.pem --cert_file .evergreen/x509gen/wrong-host.pem --port 8001 &
  python3 -u .evergreen/csfle/kms_http_server.py --ca_file .evergreen/x509gen/ca.pem --cert_file .evergreen/x509gen/server.pem --port 8002 --require_client_cert &
  python3 -u .evergreen/csfle/kms_kmip_server.py &
  python3 -u .evergreen/csfle/fake_azure.py &

  # Obtain temporary AWS credentials
  PYTHON=python3 . .evergreen/csfle/set-temp-creds.sh

  if test "$FLE" = helper; then
    echo "Using helper gem"
  elif test "$FLE" = path; then
    if false; then
      # We would ideally like to use the actual libmongocrypt binary here,
      # however there isn't a straightforward way to obtain a binary that
      # 1) is of a release version and 2) doesn't contain crypto.
      # These could be theoretically spelunked out of libmongocrypt's
      # evergreen tasks.
      curl --retry 3 -fLo libmongocrypt-all.tar.gz "https://s3.amazonaws.com/mciuploads/libmongocrypt/all/master/latest/libmongocrypt-all.tar.gz"
      tar xf libmongocrypt-all.tar.gz

      export LIBMONGOCRYPT_PATH=`pwd`/rhel-70-64-bit/nocrypto/lib64/libmongocrypt.so
    else
      # So, install the helper for the binary.
      gem install libmongocrypt-helper --pre

      # https://stackoverflow.com/questions/19072070/how-to-find-where-gem-files-are-installed
      path=$(find `gem env |grep INSTALLATION |awk -F: '{print $2}'` -name libmongocrypt.so |head -1 || true)
      if test -z "$path"; then
        echo Failed to find libmongocrypt.so in installed gems 1>&2
        exit 1
      fi
      cp $path .
      export LIBMONGOCRYPT_PATH=`pwd`/libmongocrypt.so

      gem uni libmongocrypt-helper
    fi
    test -f "$LIBMONGOCRYPT_PATH"
    ldd "$LIBMONGOCRYPT_PATH"
  else
    echo "Unknown FLE value: $FLE" 1>&2
    exit 1
  fi

  echo "Waiting for mock KMS servers to start..."
   wait_for_kms_server() {
      for i in $(seq 60); do
         if curl -s "localhost:$1"; test $? -ne 7; then
            return 0
         else
            sleep 1
         fi
      done
      echo "Could not detect mock KMS server on port $1"
      return 1
   }
   wait_for_kms_server 8000
   wait_for_kms_server 8001
   wait_for_kms_server 8002
   wait_for_kms_server 5698
   wait_for_kms_server 8080
   echo "Waiting for mock KMS servers to start... done."
fi

if test -n "$OCSP_CONNECTIVITY"; then
  # TODO Maybe OCSP_CONNECTIVITY=* should set SSL=ssl instead.
  uri_options="$uri_options&tls=true"
fi

if test -n "$EXTRA_URI_OPTIONS"; then
  uri_options="$uri_options&$EXTRA_URI_OPTIONS"
fi

export MONGODB_URI="mongodb://$hosts/?serverSelectionTimeoutMS=30000$uri_options"

if echo "$AUTH" |grep -q ^aws-assume-role; then
  $BINDIR/mongo "$MONGODB_URI" --eval 'db.runCommand({serverStatus: 1})' |wc
fi

set_fcv

if test "$TOPOLOGY" = replica-set && ! echo "$MONGODB_VERSION" |fgrep -q 2.6; then
  ruby -Ilib -I.evergreen/lib -rbundler/setup -rserver_setup -e ServerSetup.new.setup_tags
fi

if test "$API_VERSION_REQUIRED" = 1; then
  ruby -Ilib -I.evergreen/lib -rbundler/setup -rserver_setup -e ServerSetup.new.require_api_version
  export SERVER_API='version: "1"'
fi

if ! test "$OCSP_VERIFIER" = 1 && ! test -n "$OCSP_CONNECTIVITY"; then
  echo Preparing the test suite
  bundle exec rake spec:prepare
fi

if test "$TOPOLOGY" = sharded-cluster && test $MONGODB_VERSION = 3.6; then
  # On 3.6 server the sessions collection is not immediately available,
  # wait for it to spring into existence
  bundle exec rake spec:wait_for_sessions
fi

export MONGODB_URI="mongodb://$hosts/?appName=test-suite$uri_options"

# Compression is handled via an environment variable, convert to URI option
if test "$COMPRESSOR" = zlib && ! echo $MONGODB_URI |grep -q compressors=; then
  add_uri_option compressors=zlib
fi

if test "$COMPRESSOR" = snappy; then
  add_uri_option compressors=snappy
fi

if test "$COMPRESSOR" = zstd; then
  add_uri_option compressors=zstd
fi


echo "Running tests"
set +e
if test -n "$TEST_CMD"; then
  eval $TEST_CMD
elif test "$FORK" = 1; then
  bundle exec rspec spec/integration/fork*spec.rb spec/stress/fork*spec.rb
elif test "$STRESS" = 1; then
  bundle exec rspec spec/integration/fork*spec.rb spec/stress
elif test "$OCSP_VERIFIER" = 1; then
  bundle exec rspec spec/integration/ocsp_verifier_spec.rb
elif test -n "$OCSP_CONNECTIVITY"; then
  bundle exec rspec spec/integration/ocsp_connectivity_spec.rb
elif test "$SOLO" = 1; then
  for attempt in `seq 10`; do
    echo "Attempt $attempt"
    bundle exec rspec spec/solo/clean_exit_spec.rb 2>&1 |tee test.log
    if grep -qi 'segmentation fault' test.log; then
      echo 'Test failed - Ruby crashed' 1>&2
      exit 1
    fi
    if fgrep -i '[BUG]' test.log; then
      echo 'Test failed - Ruby complained about a bug' 1>&2
      exit 1
    fi
  done
else
  bundle exec rake spec:ci
fi

test_status=$?
echo "TEST STATUS: ${test_status}"
set -e

if test -f tmp/rspec-all.json; then
  mv tmp/rspec-all.json tmp/rspec.json
fi

kill_jruby

if test -n "$OCSP_MOCK_PID"; then
  kill "$OCSP_MOCK_PID"
fi

python3 -m mtools.mlaunch.mlaunch stop --dir "$dbdir"

if test -n "$FLE" && test "$DOCKER_PRELOAD" != 1; then
  # Terminate all kmip servers... and whatever else happens to be running
  # that is a python script.
  pkill python3
fi

exit ${test_status}