diff --git a/.evergreen.yml b/.evergreen.yml
index 930b8886fb..1fd62372fa 100644
--- a/.evergreen.yml
+++ b/.evergreen.yml
@@ -94,6 +94,9 @@ functions:
           set -e
           source .evergreen/.setup_env
           export EVERGREEN_EXPANSIONS_PATH="$(pwd)/../tmp/expansions.yaml"
+          export MONGOSH_TEST_EXECUTABLE_PATH="$(pwd)/dist/mongosh"
+          echo "$MONGOSH_TEST_EXECUTABLE_PATH"
+          npm run compile-exec
           npm run test-ci
   test_ps:
     - command: expansions.write
@@ -106,6 +109,8 @@ functions:
         shell: powershell
         script: |
           .\.evergreen\SetupEnv
+          $Env:MONGOSH_TEST_EXECUTABLE_PATH = $(Join-Path -Path '.' -ChildPath 'dist/mongosh.exe' -Resolve)
+          echo "$Env:MONGOSH_TEST_EXECUTABLE_PATH"
           $Env:EVERGREEN_EXPANSIONS_PATH = $(Join-Path -Path '..' -ChildPath 'tmp/expansions.yaml' -Resolve)
           npm run test-ci
   release_macos:
diff --git a/packages/cli-repl/test/e2e.spec.ts b/packages/cli-repl/test/e2e.spec.ts
index cd3adda6f8..a294f199b7 100644
--- a/packages/cli-repl/test/e2e.spec.ts
+++ b/packages/cli-repl/test/e2e.spec.ts
@@ -16,12 +16,12 @@ describe('e2e', function() {
     it('shows version', async() => {
       const shell = TestShell.start({ args: [ '--version' ] });
 
-      await eventually(() => {
-        shell.assertNoErrors();
-        shell.assertContainsOutput(
-          require('../package.json').version
-        );
-      });
+      await shell.waitForExit();
+
+      shell.assertNoErrors();
+      shell.assertContainsOutput(
+        require('../package.json').version
+      );
     });
   });
 
diff --git a/packages/cli-repl/test/helpers.ts b/packages/cli-repl/test/helpers.ts
index cd9f27015a..1a864cb9f7 100644
--- a/packages/cli-repl/test/helpers.ts
+++ b/packages/cli-repl/test/helpers.ts
@@ -1,7 +1,7 @@
 export async function eventually(fn: Function, options: { frequency?: number; timeout?: number } = {}): Promise<any> {
   options = {
     frequency: 100,
-    timeout: 10000,
+    timeout: process.env.IS_CI ? 10000 : 1000,
     ...options
   };
 
diff --git a/packages/cli-repl/test/test-shell.ts b/packages/cli-repl/test/test-shell.ts
index 389c54af07..10dabf55f6 100644
--- a/packages/cli-repl/test/test-shell.ts
+++ b/packages/cli-repl/test/test-shell.ts
@@ -8,7 +8,8 @@ import stripAnsi from 'strip-ansi';
 import assert from 'assert';
 
 const PROMPT_PATTERN = /^> /m;
-const ERROR_PATTERN = /Thrown:\n([^>]*)/m;
+const ERROR_PATTERN_1 = /Thrown:\n([^>]*)/m; // node <= 12.14
+const ERROR_PATTERN_2 = /Uncaught[:\n ]+([^>]*)/m;
 
 /**
  * Test shell helper class.
@@ -16,14 +17,22 @@ const ERROR_PATTERN = /Thrown:\n([^>]*)/m;
 export class TestShell {
   private static _openShells: TestShell[] = [];
 
-  static start(options: { args: string[] } = { args: [] }): TestShell {
-    const execPath = path.resolve(__dirname, '..', 'bin', 'mongosh.js');
+  static start(options: {
+    args: string[];
+  } = { args: [] }): TestShell {
+    let shellProcess: ChildProcess;
 
-    const process = spawn('node', [execPath, ...options.args], {
-      stdio: [ 'pipe', 'pipe', 'pipe' ]
-    });
+    if (process.env.MONGOSH_TEST_EXECUTABLE_PATH) {
+      shellProcess = spawn(process.env.MONGOSH_TEST_EXECUTABLE_PATH, [...options.args], {
+        stdio: [ 'pipe', 'pipe', 'pipe' ]
+      });
+    } else {
+      shellProcess = spawn('node', [path.resolve(__dirname, '..', 'bin', 'mongosh.js'), ...options.args], {
+        stdio: [ 'pipe', 'pipe', 'pipe' ]
+      });
+    }
 
-    const shell = new TestShell(process);
+    const shell = new TestShell(shellProcess);
     TestShell._openShells.push(shell);
 
     return shell;
@@ -38,6 +47,7 @@ export class TestShell {
   private _process: ChildProcess;
 
   private _output: string;
+  private _onClose: Promise<number>;
 
   constructor(shellProcess: ChildProcess) {
     this._process = shellProcess;
@@ -54,6 +64,12 @@ export class TestShell {
     shellProcess.stderr.on('data', (chunk) => {
       this._output += stripAnsi(stderrDecoder.write(chunk));
     });
+
+    this._onClose = new Promise((resolve) => {
+      shellProcess.once('close', (code) => {
+        resolve(code);
+      });
+    });
   }
 
   get output(): string {
@@ -72,6 +88,10 @@ export class TestShell {
     });
   }
 
+  waitForExit(): Promise<number> {
+    return this._onClose;
+  }
+
   kill(): void {
     this._process.kill();
   }
@@ -132,7 +152,11 @@ export class TestShell {
   }
 
   private _getAllErrors(): string[] {
-    return [...(this._output as any).matchAll(ERROR_PATTERN)]
+    const output = (this._output as any);
+    return [
+      ...output.matchAll(ERROR_PATTERN_1),
+      ...output.matchAll(ERROR_PATTERN_2)
+    ]
       .map(m => m[1].trim());
   }
 }