diff --git a/.evergreen.yml b/.evergreen.yml
index 89ff220883..b7d3c04e14 100644
--- a/.evergreen.yml
+++ b/.evergreen.yml
@@ -27,7 +27,7 @@ variables:
             -e NODE_JS_VERSION \
             -e BUILD_VARIANT \
             --rm -v $PWD:/tmp/build --network host ubuntu16.04-build \
-            -c 'cd /tmp/build && npm run evergreen-release package'
+            -c 'cd /tmp/build && npm run evergreen-release package && dist/mongosh --version'
         else
           npm run evergreen-release package
           if [ `uname` == Darwin ]; then
@@ -43,7 +43,11 @@ variables:
           bash ~/release_mongosh.sh
         fi
 
-        ls -la dist/
+        source .evergreen/.setup_env
+        export MONGOSH_TEST_EXECUTABLE_PATH="$(pwd)/dist/mongosh"
+        echo "$MONGOSH_TEST_EXECUTABLE_PATH"
+        npm run test-e2e-ci
+
   - &compile_and_release_ps
     command: shell.exec
     params:
@@ -57,6 +61,11 @@ variables:
         $Env:PUPPETEER_SKIP_CHROMIUM_DOWNLOAD = "true"
         $Env:EVERGREEN_EXPANSIONS_PATH = $(Join-Path -Path '..' -ChildPath 'tmp/expansions.yaml' -Resolve)
         npm run evergreen-release package
+        .\.evergreen\DownloadAndRunMongod
+        $Env:MONGOSH_TEST_SERVER_URL = "mongodb://localhost:27018"
+        $Env:MONGOSH_TEST_EXECUTABLE_PATH = $(Join-Path -Path '.' -ChildPath 'dist/mongosh.exe' -Resolve)
+        echo "$Env:MONGOSH_TEST_EXECUTABLE_PATH"
+        npm run test-e2e-ci
 
 # Functions are any command that can be run.
 #
@@ -116,9 +125,6 @@ functions:
           set -e
           source .evergreen/.setup_env
           export EVERGREEN_EXPANSIONS_PATH="$(pwd)/../tmp/expansions.yaml"
-          export MONGOSH_TEST_EXECUTABLE_PATH="$(pwd)/dist/mongosh"
-          echo "$MONGOSH_TEST_EXECUTABLE_PATH"
-          npm run compile-exec
           npm run test-ci
   test_ps:
     - command: expansions.write
@@ -134,10 +140,7 @@ functions:
           .\.evergreen\SetupEnv
           .\.evergreen\DownloadAndRunMongod
           $Env:EVERGREEN_EXPANSIONS_PATH = $(Join-Path -Path '..' -ChildPath 'tmp/expansions.yaml' -Resolve)
-          npm run compile-exec
-          $Env:MONGOSH_TEST_EXECUTABLE_PATH = $(Join-Path -Path '.' -ChildPath 'dist/mongosh.exe' -Resolve)
           $Env:MONGOSH_TEST_SERVER_URL = "mongodb://localhost:27018"
-          echo "$Env:MONGOSH_TEST_EXECUTABLE_PATH"
           npm run test-ci
   release_macos:
     - command: expansions.write
diff --git a/package.json b/package.json
index f87bc85832..496fa9bd4c 100644
--- a/package.json
+++ b/package.json
@@ -19,6 +19,8 @@
     "check-ci": "lerna run check",
     "test": "lerna run test",
     "test-ci": "lerna run test-ci",
+    "test-e2e": "lerna run test-e2e",
+    "test-e2e-ci": "lerna run test-e2e-ci",
     "compile-ts": "lerna run compile-ts",
     "compile-cli": "lerna run compile-ts --scope @mongosh/cli-repl --include-dependencies",
     "prestart-cli": "npm run compile-cli",
diff --git a/packages/build/package.json b/packages/build/package.json
index 0237ec4800..8cfa4597c7 100644
--- a/packages/build/package.json
+++ b/packages/build/package.json
@@ -1,6 +1,7 @@
 {
   "name": "@mongosh/build",
   "version": "0.4.0",
+  "private": true,
   "description": "MongoDB Shell Build Tools",
   "main": "lib/index.js",
   "types": "lib/index.d.ts",
diff --git a/packages/cli-repl/package.json b/packages/cli-repl/package.json
index a8d8481f1b..2e5b8d3170 100644
--- a/packages/cli-repl/package.json
+++ b/packages/cli-repl/package.json
@@ -22,6 +22,7 @@
     "test-ci": "mocha -r \"../../scripts/import-expansions.js\" --timeout 60000 -r ts-node/register \"./{src,test}/**/*.spec.ts\"",
     "pretest-e2e": "npm run compile-ts",
     "test-e2e": "mocha -r \"../../scripts/import-expansions.js\" --timeout 15000 --colors -r ts-node/register \"./test/e2e.spec.ts\"",
+    "test-e2e-ci": "mocha -r \"../../scripts/import-expansions.js\" --timeout 15000 --colors -r ts-node/register \"./test/e2e.spec.ts\"",
     "lint": "eslint \"**/*.{js,ts,tsx}\"",
     "check": "npm run lint",
     "prepublish": "npm run compile-ts"
diff --git a/scripts/docker/ubuntu16.04-build.Dockerfile b/scripts/docker/ubuntu16.04-build.Dockerfile
index f574a2c283..168bf41d28 100644
--- a/scripts/docker/ubuntu16.04-build.Dockerfile
+++ b/scripts/docker/ubuntu16.04-build.Dockerfile
@@ -12,9 +12,9 @@ RUN echo "deb-src https://deb.nodesource.com/node_12.x xenial main" | tee -a /et
 RUN add-apt-repository ppa:ubuntu-toolchain-r/test
 
 RUN apt-get update
-RUN apt-get -y -qq install nodejs g++-9 rpm
+RUN apt-get -y -qq install nodejs g++-6 rpm
 
-ENV CC=gcc-9
-ENV CXX=g++-9
+ENV CC=gcc-6
+ENV CXX=g++-6
 
 ENTRYPOINT [ "bash" ]