From e8cbd5b0e4fd66217399383d46d1fc116035acf3 Mon Sep 17 00:00:00 2001 From: Oleg Pudeyev Date: Wed, 10 Feb 2021 20:15:49 -0500 Subject: [PATCH 1/3] Move server installation code to spec/shared for Mongoid --- .evergreen/functions-server.sh | 125 --------------------------------- .evergreen/run-tests.sh | 2 +- spec/shared | 2 +- 3 files changed, 2 insertions(+), 127 deletions(-) delete mode 100644 .evergreen/functions-server.sh diff --git a/.evergreen/functions-server.sh b/.evergreen/functions-server.sh deleted file mode 100644 index 4a7edea44a..0000000000 --- a/.evergreen/functions-server.sh +++ /dev/null @@ -1,125 +0,0 @@ -# This file contains functions pertaining to downloading, starting and -# configuring a MongoDB server. - -set_fcv() { - if test -n "$FCV"; then - mongo --eval 'assert.commandWorked(db.adminCommand( { setFeatureCompatibilityVersion: "'"$FCV"'" } ));' "$MONGODB_URI" - mongo --quiet --eval 'db.adminCommand( { getParameter: 1, featureCompatibilityVersion: 1 } )' |grep "version.*$FCV" - fi -} - -add_uri_option() { - opt=$1 - - if ! echo $MONGODB_URI |sed -e s,//,, |grep -q /; then - MONGODB_URI="$MONGODB_URI/" - fi - - if ! echo $MONGODB_URI |grep -q '?'; then - MONGODB_URI="$MONGODB_URI?" - fi - - MONGODB_URI="$MONGODB_URI&$opt" -} - -prepare_server() { - arch=$1 - - if test -n "$USE_OPT_MONGODB"; then - export BINDIR=/opt/mongodb/bin - export PATH=$BINDIR:$PATH - return - fi - - if test "$MONGODB_VERSION" = latest; then - # Test on the most recent published 4.3 release. - # https://jira.mongodb.org/browse/RUBY-1724 - echo 'Using "latest" server is not currently implemented' 1>&2 - exit 1 - else - download_version="$MONGODB_VERSION" - fi - - url=`$(dirname $0)/get-mongodb-download-url $download_version $arch` - - prepare_server_from_url $url -} - -prepare_server_from_url() { - url=$1 - - mongodb_dir="$MONGO_ORCHESTRATION_HOME"/mdb - mkdir -p "$mongodb_dir" - curl --retry 3 $url |tar xz -C "$mongodb_dir" -f - - BINDIR="$mongodb_dir"/`basename $url |sed -e s/.tgz//`/bin - export PATH="$BINDIR":$PATH -} - -install_mlaunch_virtualenv() { - python2 -V || true - # Current virtualenv fails with - # https://github.com/pypa/virtualenv/issues/1630 - python -m pip install 'virtualenv<20' --user - venvpath="$MONGO_ORCHESTRATION_HOME"/venv - python2 -m virtualenv -p python2 $venvpath - . $venvpath/bin/activate - pip install 'mtools-legacy[mlaunch]' -} - -install_mlaunch_pip() { - if test -n "$USE_OPT_MONGODB" && which mlaunch >/dev/null 2>&1; then - # mlaunch is preinstalled in the docker image, do not install it here - return - fi - - python -V || true - python3 -V || true - pythonpath="$MONGO_ORCHESTRATION_HOME"/python - pip install -t "$pythonpath" 'mtools-legacy[mlaunch]' - export PATH="$pythonpath/bin":$PATH - export PYTHONPATH="$pythonpath" -} - -install_mlaunch_git() { - repo=$1 - branch=$2 - python -V || true - python3 -V || true - which pip || true - which pip3 || true - - if false; then - if ! virtualenv --version; then - python3 `which pip3` install --user virtualenv - export PATH=$HOME/.local/bin:$PATH - virtualenv --version - fi - - venvpath="$MONGO_ORCHESTRATION_HOME"/venv - virtualenv -p python3 $venvpath - . $venvpath/bin/activate - - pip3 install psutil pymongo - - git clone $repo mlaunch - cd mlaunch - git checkout origin/$branch - python3 setup.py install - cd .. - else - pip install --user 'virtualenv==13' - export PATH=$HOME/.local/bin:$PATH - - venvpath="$MONGO_ORCHESTRATION_HOME"/venv - virtualenv $venvpath - . $venvpath/bin/activate - - pip install psutil pymongo - - git clone $repo mlaunch - (cd mlaunch && - git checkout origin/$branch && - python setup.py install - ) - fi -} diff --git a/.evergreen/run-tests.sh b/.evergreen/run-tests.sh index dfe9bdc5ab..a58704b7d1 100755 --- a/.evergreen/run-tests.sh +++ b/.evergreen/run-tests.sh @@ -12,9 +12,9 @@ fi . `dirname "$0"`/../spec/shared/shlib/distro.sh . `dirname "$0"`/../spec/shared/shlib/set_env.sh +. `dirname "$0"`/../spec/shared/shlib/server.sh . `dirname "$0"`/functions.sh . `dirname "$0"`/functions-aws.sh -. `dirname "$0"`/functions-server.sh . `dirname "$0"`/functions-config.sh arch=`host_distro` diff --git a/spec/shared b/spec/shared index c832db3224..cfca9fdc81 160000 --- a/spec/shared +++ b/spec/shared @@ -1 +1 @@ -Subproject commit c832db32247426d6fd9f2c8607d3e1b684986496 +Subproject commit cfca9fdc81b68d3ddba19fc54d83718047913bd7 From 95a246185c558ccc341a76f0b009a5ab64cae586 Mon Sep 17 00:00:00 2001 From: Oleg Pudeyev Date: Wed, 10 Feb 2021 20:34:36 -0500 Subject: [PATCH 2/3] move server launching code into spec/shared --- .evergreen/run-tests.sh | 137 ++-------------------------------------- spec/shared | 2 +- 2 files changed, 7 insertions(+), 132 deletions(-) diff --git a/.evergreen/run-tests.sh b/.evergreen/run-tests.sh index a58704b7d1..e71d96f84e 100755 --- a/.evergreen/run-tests.sh +++ b/.evergreen/run-tests.sh @@ -35,136 +35,11 @@ install_mlaunch_virtualenv export dbdir="$MONGO_ORCHESTRATION_HOME"/db mkdir -p "$dbdir" -mongo_version=`echo $MONGODB_VERSION |tr -d .` -if test $mongo_version = latest; then - mongo_version=44 -fi - -args="--setParameter enableTestCommands=1" -# diagnosticDataCollectionEnabled is a mongod-only parameter on server 3.2, -# and mlaunch does not support specifying mongod-only parameters: -# https://github.com/rueckstiess/mtools/issues/696 -# Pass it to 3.4 and newer servers where it is accepted by all daemons. -if test $mongo_version -ge 34; then - args="$args --setParameter diagnosticDataCollectionEnabled=false" -fi -uri_options= -if test "$TOPOLOGY" = replica-set; then - args="$args --replicaset --name ruby-driver-rs --nodes 2 --arbiter" - export HAVE_ARBITER=1 -elif test "$TOPOLOGY" = sharded-cluster; then - args="$args --replicaset --nodes 1 --sharded 1 --name ruby-driver-rs" - if test -z "$SINGLE_MONGOS"; then - args="$args --mongos 2" - fi -else - args="$args --single" -fi -if test -n "$MMAPV1"; then - args="$args --storageEngine mmapv1 --smallfiles --noprealloc" - uri_options="$uri_options&retryReads=false&retryWrites=false" -fi -if test "$AUTH" = auth; then - args="$args --auth --username bob --password pwd123" -elif test "$AUTH" = x509; then - args="$args --auth --username bootstrap --password bootstrap" -elif echo "$AUTH" |grep -q ^aws; then - args="$args --auth --username bootstrap --password bootstrap" - args="$args --setParameter authenticationMechanisms=MONGODB-AWS,SCRAM-SHA-1,SCRAM-SHA-256" - uri_options="$uri_options&authMechanism=MONGODB-AWS&authSource=\$external" -fi - -if test -n "$OCSP"; then - if test -z "$OCSP_ALGORITHM"; then - echo "OCSP_ALGORITHM must be set if OCSP is set" 1>&2 - exit 1 - fi -fi - -if test "$SSL" = ssl || test -n "$OCSP_ALGORITHM"; then - if test -n "$OCSP_ALGORITHM"; then - if test "$OCSP_MUST_STAPLE" = 1; then - server_cert_path=spec/support/ocsp/$OCSP_ALGORITHM/server-mustStaple.pem - else - server_cert_path=spec/support/ocsp/$OCSP_ALGORITHM/server.pem - fi - server_ca_path=spec/support/ocsp/$OCSP_ALGORITHM/ca.crt - server_client_cert_path=spec/support/ocsp/$OCSP_ALGORITHM/server.pem - else - server_cert_path=spec/support/certificates/server-second-level-bundle.pem - server_ca_path=spec/support/certificates/ca.crt - server_client_cert_path=spec/support/certificates/client.pem - fi - - if test -n "$OCSP_ALGORITHM"; then - client_cert_path=spec/support/ocsp/$OCSP_ALGORITHM/server.pem - elif test "$AUTH" = x509; then - client_cert_path=spec/support/certificates/client-x509.pem - - uri_options="$uri_options&authMechanism=MONGODB-X509" - elif echo $RVM_RUBY |grep -q jruby; then - # JRuby does not grok chained certificate bundles - - # https://github.com/jruby/jruby-openssl/issues/181 - client_cert_path=spec/support/certificates/client.pem - else - client_cert_path=spec/support/certificates/client-second-level-bundle.pem - fi - - uri_options="$uri_options&tls=true&"\ -"tlsCAFile=$server_ca_path&"\ -"tlsCertificateKeyFile=$client_cert_path" - - args="$args --sslMode requireSSL"\ -" --sslPEMKeyFile $server_cert_path"\ -" --sslCAFile $server_ca_path"\ -" --sslClientCertificate $server_client_cert_path" -fi - -# Docker forwards ports to the external interface, not to the loopback. -# Hence we must bind to all interfaces here. -if test -n "$BIND_ALL"; then - args="$args --bind_ip_all" -fi - -# MongoDB servers pre-4.2 do not enable zlib compression by default -if test "$COMPRESSOR" = snappy; then - args="$args --networkMessageCompressors snappy" -elif test "$COMPRESSOR" = zlib; then - args="$args --networkMessageCompressors zlib" -fi - -if test -n "$OCSP_ALGORITHM" || test -n "$OCSP_VERIFIER"; then - python3 -m pip install asn1crypto oscrypto flask -fi - -ocsp_mock_pid= -if test -n "$OCSP_ALGORITHM"; then - if test -z "$server_ca_path"; then - echo "server_ca_path must have been set" 1>&2 - exit 1 - fi - ocsp_args="--ca_file $server_ca_path" - if test "$OCSP_DELEGATE" = 1; then - ocsp_args="$ocsp_args \ ---ocsp_responder_cert spec/support/ocsp/$OCSP_ALGORITHM/ocsp-responder.crt \ ---ocsp_responder_key spec/support/ocsp/$OCSP_ALGORITHM/ocsp-responder.key \ -" - else - ocsp_args="$ocsp_args \ ---ocsp_responder_cert spec/support/ocsp/$OCSP_ALGORITHM/ca.crt \ ---ocsp_responder_key spec/support/ocsp/$OCSP_ALGORITHM/ca.key \ -" - fi - if test -n "$OCSP_STATUS"; then - ocsp_args="$ocsp_args --fault $OCSP_STATUS" - fi - - # Bind to 0.0.0.0 for Docker - python3 spec/support/ocsp/ocsp_mock.py $ocsp_args -b 0.0.0.0 -p 8100 & - ocsp_mock_pid=$! -fi +calculate_server_args +launch_ocsp_mock +launch_server "$dbdir" -python -m mtools.mlaunch.mlaunch --dir "$dbdir" --binarypath "$BINDIR" $args +uri_options="$URI_OPTIONS" bundle_install @@ -347,8 +222,8 @@ fi kill_jruby -if test -n "$ocsp_mock_pid"; then - kill "$ocsp_mock_pid" +if test -n "$OCSP_MOCK_PID"; then + kill "$OCSP_MOCK_PID" fi python -m mtools.mlaunch.mlaunch stop --dir "$dbdir" diff --git a/spec/shared b/spec/shared index cfca9fdc81..45e8256129 160000 --- a/spec/shared +++ b/spec/shared @@ -1 +1 @@ -Subproject commit cfca9fdc81b68d3ddba19fc54d83718047913bd7 +Subproject commit 45e825612922ba1b4f92c33887fe424ab2657f33 From d7b504ea1a168998fd1d78e8197ac2dc740ec847 Mon Sep 17 00:00:00 2001 From: Oleg Pudeyev Date: Wed, 10 Feb 2021 20:54:48 -0500 Subject: [PATCH 3/3] sync with Mongoid changes --- spec/shared | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spec/shared b/spec/shared index 45e8256129..be6386e865 160000 --- a/spec/shared +++ b/spec/shared @@ -1 +1 @@ -Subproject commit 45e825612922ba1b4f92c33887fe424ab2657f33 +Subproject commit be6386e86540f04740e352cc8c51f9f1ffeb078f