diff --git a/.evergreen/config.yml b/.evergreen/config.yml index 27c450279c..4938dc0f81 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -97,29 +97,10 @@ functions: go version go env - LIBMONGOCRYPT_TAG="1.8.0-alpha1" - # Install libmongocrypt based on OS. + # Install libmongocrypt. + bash etc/install-libmongocrypt.sh if [ "Windows_NT" = "$OS" ]; then - mkdir -p c:/libmongocrypt/include - mkdir -p c:/libmongocrypt/bin - echo "fetching build for Windows ... begin" - mkdir libmongocrypt-all - cd libmongocrypt-all - # The following URL is published from the upload-all task in the libmongocrypt Evergreen project. - curl https://mciuploads.s3.amazonaws.com/libmongocrypt/all/$LIBMONGOCRYPT_TAG/libmongocrypt-all.tar.gz -o libmongocrypt-all.tar.gz - tar -xf libmongocrypt-all.tar.gz - cd .. - cp libmongocrypt-all/windows-test/bin/mongocrypt.dll c:/libmongocrypt/bin - cp libmongocrypt-all/windows-test/include/mongocrypt/*.h c:/libmongocrypt/include - export PATH=$PATH:/cygdrive/c/libmongocrypt/bin - rm -rf libmongocrypt-all - echo "fetching build for Windows ... end" - else - git clone https://github.com/mongodb/libmongocrypt --depth=1 --branch $LIBMONGOCRYPT_TAG - if ! ( ./libmongocrypt/.evergreen/compile.sh >| output.txt 2>&1 ); then - cat output.txt 1>&2 - exit 1 - fi + export PATH=$PATH:/cygdrive/c/libmongocrypt/bin fi cat < expansion.yml @@ -614,6 +595,17 @@ functions: MONGODB_URI="mongodb://mhuser:pencil@localhost" \ make evg-test-atlas-data-lake + run-docker-test: + - command: shell.exec + type: test + params: + shell: "bash" + working_dir: src/go.mongodb.org/mongo-driver + script: | + ${PREPARE_SHELL} + bash etc/run_docker.sh + TOPOLOGY=sharded_cluster bash etc/run_docker.sh test-short + run-valid-ocsp-server: - command: shell.exec params: @@ -1678,6 +1670,10 @@ tasks: - func: bootstrap-mongohoused - func: run-atlas-data-lake-test + - name: test-docker-runner + commands: + - func: run-docker-test + - name: test-load-balancer-noauth-nossl tags: ["load-balancer"] commands: @@ -2720,6 +2716,15 @@ buildvariants: tasks: - name: "test-atlas-data-lake" + - name: docker-runner-test + display_name: "Docker Runner Test" + run_on: + - ubuntu2204-large + expansions: + GO_DIST: "/opt/golang/go1.20" + tasks: + - name: "test-docker-runner" + - matrix_name: "tests-36-with-zlib-support" tags: ["pullrequest"] matrix_spec: { version: ["3.6"], os-ssl-32: ["windows-64-go-1-20", "rhel87-64-go-1-20"] } diff --git a/.evergreen/run-tests.sh b/.evergreen/run-tests.sh index a223f309f8..154ad9ccba 100644 --- a/.evergreen/run-tests.sh +++ b/.evergreen/run-tests.sh @@ -4,7 +4,11 @@ set -o errexit export GOPATH=$(dirname $(dirname $(dirname `pwd`))) export GOCACHE="$(pwd)/.cache" -export DRIVERS_TOOLS="$(pwd)/../drivers-tools" +export DRIVERS_TOOLS=${DRIVERS_TOOLS:-""} + +if [ -z $DRIVERS_TOOLS ]; then + export DRIVERS_TOOLS="$(pwd)/../drivers-tools" +fi if [ "Windows_NT" = "$OS" ]; then export GOPATH=$(cygpath -m $GOPATH) @@ -14,6 +18,7 @@ fi export GOROOT="${GOROOT}" export PATH="${GOROOT}/bin:${GCC_PATH}:$GOPATH/bin:$PATH" +export PATH="${MONGODB_BINARIES:-$DRIVERS_TOOLS/mongodb/bin}:$PATH" export PROJECT="${project}" export PKG_CONFIG_PATH=$(pwd)/install/libmongocrypt/lib64/pkgconfig:$(pwd)/install/mongo-c-driver/lib/pkgconfig export LD_LIBRARY_PATH=$(pwd)/install/libmongocrypt/lib64 diff --git a/.gitignore b/.gitignore index 16b52325e4..369aecd8c1 100644 --- a/.gitignore +++ b/.gitignore @@ -12,7 +12,9 @@ perf **mongocryptd.pid *.test .DS_Store +install main.so +test.suite # AWS SAM-generated files internal/test/faas/awslambda/.aws-sam diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000000..446ab1d5f2 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,41 @@ +# Dockerfile for Go Driver local development. + +# Build libmongocrypt in a separate build stage. +FROM ubuntu:20.04 as libmongocrypt + +RUN apt-get -qq update && \ + apt-get -qqy install --no-install-recommends \ + git \ + ca-certificates \ + curl \ + build-essential \ + libssl-dev \ + python + +COPY etc/install-libmongocrypt.sh /root/install-libmongocrypt.sh +RUN cd /root && bash ./install-libmongocrypt.sh + + +# Inherit from the drivers-evergreen-tools image and copy in the files +# from the libmongocrypt build stage. +FROM drivers-evergreen-tools + +RUN export DEBIAN_FRONTEND=noninteractive && \ + export TZ=Etc/UTC && \ + apt-get -qq update && \ + apt-get -qqy install --no-install-recommends \ + pkg-config \ + tzdata \ + gpg \ + apt-utils \ + make && \ + apt-add-repository ppa:longsleep/golang-backports && \ + apt-get -qq update && \ + apt-get -qqy install --no-install-recommends golang-go && \ + rm -rf /var/lib/apt/lists/* + +COPY ./etc/docker_entry.sh /root/docker_entry.sh + +COPY --from=libmongocrypt /root/install /root/install + +ENTRYPOINT ["/bin/bash", "/root/docker_entry.sh"] diff --git a/docs/CONTRIBUTING.md b/docs/CONTRIBUTING.md index 71f0df3446..8a25e6f4d9 100644 --- a/docs/CONTRIBUTING.md +++ b/docs/CONTRIBUTING.md @@ -51,6 +51,9 @@ prompt before creating a PR to the target branch. The driver tests can be run against several database configurations. The most simple configuration is a standalone mongod with no auth, no ssl, and no compression. To run these basic driver tests, make sure a standalone MongoDB server instance is running at localhost:27017. To run the tests, you can run `make` (on Windows, run `nmake`). This will run coverage, run go-lint, run go-vet, and build the examples. +You can install `libmongocrypt` locally by running `bash etc/build-libmongocrypt.sh`, which will create an `install` directory +in the repository top level directory. On Windows you will also need to add `c:/libmongocrypt/` to your `PATH`. + ### Testing Different Topologies To test a **replica set** or **sharded cluster**, set `MONGODB_URI=""` for the `make` command. @@ -129,6 +132,24 @@ The usage of host.docker.internal comes from the [Docker networking documentatio There is currently no arm64 support for the go1.x runtime, see [here](https://docs.aws.amazon.com/lambda/latest/dg/lambda-runtimes.html). Known issues running on linux/arm64 include the inability to network with the localhost from the public.ecr.aws/lambda/go Docker image. +### Testing in Docker + +We support local testing in Docker. Ensure ``docker`` is installed and running, and then run: + +```bash +bash etc/run_docker.sh +``` + +The script takes an optional argument for the ``MAKEFILE_TARGET`` and allows for some environment variable overrides. +The docker container has the required binaries, including libmongocrypt. +The entry script starts a MongoDB topology, and then executes the desired ``MAKEFILE_TARGET``. + +For example, to test against a sharded cluster, using enterprise auth, run: + +```bash +TOPOLOGY=sharded_cluster bash etc/run_docker.sh evg-test-enterprise-auth +``` + ## Talk To Us If you want to work on the driver, write documentation, or have questions/complaints, please reach out to us either via [MongoDB Community Forums](https://community.mongodb.com/tags/c/drivers-odms-connectors/7/go-driver) or by creating a Question issue in [Jira](https://jira.mongodb.org/secure/CreateIssue!default.jspa). diff --git a/etc/docker_entry.sh b/etc/docker_entry.sh new file mode 100644 index 0000000000..51f31217f1 --- /dev/null +++ b/etc/docker_entry.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env bash +# +# Entry point for Dockerfile for launching a server and running a go test. +# +set -eux + +# Start the server. +bash /root/base-entrypoint.sh +source $DRIVERS_TOOLS/.evergreen/mo-expansion.sh + +# Prep files. +cd /src +rm -f test.suite +cp -r $HOME/install ./install +export PATH="$MONGODB_BINARIES:$PATH" + +# Run the test. +bash ./.evergreen/run-tests.sh diff --git a/etc/install-libmongocrypt.sh b/etc/install-libmongocrypt.sh new file mode 100644 index 0000000000..af0a266aa9 --- /dev/null +++ b/etc/install-libmongocrypt.sh @@ -0,0 +1,33 @@ +#!/usr/bin/env bash +# install libmongocrypt +# This script installs libmongocrypt into an "install" directory. +set -eux + +LIBMONGOCRYPT_TAG="1.8.2" + +# Install libmongocrypt based on OS. +if [ "Windows_NT" = "${OS:-}" ]; then + mkdir -p c:/libmongocrypt/include + mkdir -p c:/libmongocrypt/bin + echo "fetching build for Windows ... begin" + mkdir libmongocrypt-all + cd libmongocrypt-all + # The following URL is published from the upload-all task in the libmongocrypt Evergreen project. + curl https://mciuploads.s3.amazonaws.com/libmongocrypt/all/$LIBMONGOCRYPT_TAG/libmongocrypt-all.tar.gz -o libmongocrypt-all.tar.gz + tar -xf libmongocrypt-all.tar.gz + cd .. + cp libmongocrypt-all/windows-test/bin/mongocrypt.dll c:/libmongocrypt/bin + cp libmongocrypt-all/windows-test/include/mongocrypt/*.h c:/libmongocrypt/include + + rm -rf libmongocrypt-all + echo "fetching build for Windows ... end" +else + rm -rf libmongocrypt + git clone https://github.com/mongodb/libmongocrypt --depth=1 --branch $LIBMONGOCRYPT_TAG 2> /dev/null + if ! ( ./libmongocrypt/.evergreen/compile.sh >| output.txt 2>&1 ); then + cat output.txt 1>&2 + exit 1 + fi + mv output.txt install + rm -rf libmongocrypt +fi diff --git a/etc/run_docker.sh b/etc/run_docker.sh new file mode 100644 index 0000000000..a4b400035b --- /dev/null +++ b/etc/run_docker.sh @@ -0,0 +1,40 @@ +#!/usr/bin/env bash +# +# Script to run a test suite in docker locally +set -eux + +if [ -z "$DRIVERS_TOOLS" ]; then + echo "Please set DRIVERS_TOOLS env variable." + exit 1 +fi +PLATFORM=${DOCKER_PLATFORM:-} + +pushd $DRIVERS_TOOLS/.evergreen/docker/ubuntu20.04 +docker build $PLATFORM -t drivers-evergreen-tools . +popd +docker build $PLATFORM -t go-test . + +# Handle environment variables and optional positional arg for the makefile target. + +MAKEFILE_TARGET=${1:-evg-test-versioned-api} +MONGODB_VERSION=${MONGODB_VERSION:-latest} +TOPOLOGY=${TOPOLOGY:-replica_set} +ORCHESTRATION_FILE=${ORCHESTRATION_FILE:-basic.json} +AUTH=${AUTH:-""} +SSL=${SSL:=""} +GO_BUILD_TAGS=${GO_BUILD_TAGS:-""} + +ENV="-e MONGODB_VERSION=$MONGODB_VERSION -e TOPOLOGY=$TOPOLOGY" +ENV="$ENV -e MAKEFILE_TARGET=$MAKEFILE_TARGET -e AUTH=$AUTH" +ENV="$ENV -e ORCHESTRATION_FILE=$ORCHESTRATION_FILE -e SSL=$SSL" +ENV="$ENV -e GO_BUILD_TAGS=$GO_BUILD_TAGS" + +VOL="-v `pwd`:/src" +VOL="$VOL -v $DRIVERS_TOOLS:/root/drivers-evergreen-tools" +USE_TTY="" +test -t 1 && USE_TTY="-t" + +docker run $PLATFORM --rm $VOL $ENV -i $USE_TTY go-test +if [ -f "test.suite" ]; then + tail test.suite +fi diff --git a/mongo/client_test.go b/mongo/client_test.go index e1e8f322a8..d4aeaeacd3 100644 --- a/mongo/client_test.go +++ b/mongo/client_test.go @@ -23,6 +23,7 @@ import ( "go.mongodb.org/mongo-driver/mongo/readpref" "go.mongodb.org/mongo-driver/mongo/writeconcern" "go.mongodb.org/mongo-driver/tag" + "go.mongodb.org/mongo-driver/x/mongo/driver/mongocrypt" "go.mongodb.org/mongo-driver/x/mongo/driver/session" "go.mongodb.org/mongo-driver/x/mongo/driver/topology" ) @@ -443,6 +444,9 @@ func TestClient(t *testing.T) { if cryptSharedLibPath == "" { t.Skip("CRYPT_SHARED_LIB_PATH not set, skipping") } + if len(mongocrypt.Version()) == 0 { + t.Skip("Not built with cse flag") + } testCases := []struct { description string diff --git a/mongo/integration/handshake_test.go b/mongo/integration/handshake_test.go index 3e9989158e..b2cb7562f0 100644 --- a/mongo/integration/handshake_test.go +++ b/mongo/integration/handshake_test.go @@ -25,6 +25,10 @@ import ( func TestHandshakeProse(t *testing.T) { mt := mtest.New(t) + if len(os.Getenv("DOCKER_RUNNING")) > 0 { + t.Skip("These tests gives different results when run in Docker due to extra environment data.") + } + opts := mtest.NewOptions(). CreateCollection(false). ClientType(mtest.Proxy) diff --git a/mongo/integration/sdam_prose_test.go b/mongo/integration/sdam_prose_test.go index 21b1fea4ba..4e7f7dcab0 100644 --- a/mongo/integration/sdam_prose_test.go +++ b/mongo/integration/sdam_prose_test.go @@ -8,6 +8,7 @@ package integration import ( "context" + "os" "runtime" "testing" "time" @@ -52,6 +53,9 @@ func TestSDAMProse(t *testing.T) { // sent messages. The sleep duration will be at least the specified duration but // possibly longer, which could lead to extra heartbeat messages, so account for that in // the assertions. + if len(os.Getenv("DOCKER_RUNNING")) > 0 { + mt.Skip("skipping test in docker environment") + } start := time.Now() time.Sleep(2 * time.Second) messages := mt.GetProxiedMessages()