diff --git a/packages/compass-e2e-tests/helpers/compass.ts b/packages/compass-e2e-tests/helpers/compass.ts
index 65a40b12229..a03dd5f2302 100644
--- a/packages/compass-e2e-tests/helpers/compass.ts
+++ b/packages/compass-e2e-tests/helpers/compass.ts
@@ -1,6 +1,6 @@
 import { inspect } from 'util';
 import { ObjectId, EJSON } from 'bson';
-import { promises as fs } from 'fs';
+import { promises as fs, rmdirSync } from 'fs';
 import type Mocha from 'mocha';
 import path from 'path';
 import os from 'os';
@@ -42,20 +42,16 @@ let j = 0;
 // For the html
 //let k = 0;
 
-type CompassOptions = {
-  userDataDir: string;
-};
-
 export class Compass {
   browser: CompassBrowser;
+  isFirstRun: boolean;
   renderLogs: any[]; // TODO
   logs: LogEntry[];
   logPath?: string;
-  userDataDir: string;
 
-  constructor(browser: CompassBrowser, options: CompassOptions) {
+  constructor(browser: CompassBrowser, { isFirstRun = false } = {}) {
     this.browser = browser;
-    this.userDataDir = options.userDataDir;
+    this.isFirstRun = isFirstRun;
     this.logs = [];
     this.renderLogs = [];
 
@@ -90,7 +86,15 @@ export class Compass {
         // first arg is usually == text, but not always
         const args = [];
         for (const arg of message.args()) {
-          args.push(await arg.jsonValue());
+          let value;
+          try {
+            value = await arg.jsonValue();
+          } catch (err) {
+            // there are still some edge cases we can't easily convert into text
+            console.error('could not convert', arg);
+            value = '¯\\_(ツ)_/¯';
+          }
+          args.push(value);
         }
 
         // uncomment to see browser logs
@@ -190,16 +194,6 @@ export class Compass {
     debug(`Writing Compass application log to ${compassLogPath}`);
     await fs.writeFile(compassLogPath, compassLog.raw);
     this.logs = compassLog.structured;
-
-    debug('Removing user data');
-    try {
-      await fs.rmdir(this.userDataDir, { recursive: true });
-    } catch (e) {
-      debug(
-        `Failed to remove temporary user data directory at ${this.userDataDir}:`
-      );
-      debug(e);
-    }
   }
 
   async capturePage(
@@ -215,23 +209,63 @@ export class Compass {
   }
 }
 
-async function startCompass(
-  testPackagedApp = ['1', 'true'].includes(process.env.TEST_PACKAGED_APP ?? ''),
-  opts = {}
-): Promise<Compass> {
-  const nowFormatted = formattedDate();
+interface StartCompassOptions {
+  firstRun?: boolean;
+}
 
-  const userDataDir = path.join(
-    os.tmpdir(),
-    `user-data-dir-${Date.now().toString(32)}-${++i}`
+let defaultUserDataDir: string | undefined;
+
+export function removeUserDataDir(): void {
+  if (!defaultUserDataDir) {
+    return;
+  }
+  debug('Removing user data');
+  try {
+    // this is sync so we can use it in cleanup() in index.ts
+    rmdirSync(defaultUserDataDir, { recursive: true });
+  } catch (e) {
+    debug(
+      `Failed to remove temporary user data directory at ${defaultUserDataDir}:`
+    );
+    debug(e);
+  }
+}
+
+async function startCompass(opts: StartCompassOptions = {}): Promise<Compass> {
+  const testPackagedApp = ['1', 'true'].includes(
+    process.env.TEST_PACKAGED_APP ?? ''
   );
+
+  const nowFormatted = formattedDate();
+
+  const isFirstRun = opts.firstRun || !defaultUserDataDir;
+
+  // If this is not the first run, but we want it to be, delete the user data
+  // dir so it will be recreated below.
+  if (defaultUserDataDir && opts.firstRun) {
+    removeUserDataDir();
+    // windows seems to be weird about us deleting and recreating this dir, so
+    // just make a new one for next time
+    defaultUserDataDir = undefined;
+  }
+
+  // Calculate the userDataDir once so it will be the same between runs. That
+  // way we can test first run vs second run experience.
+  if (!defaultUserDataDir) {
+    defaultUserDataDir = path.join(
+      os.tmpdir(),
+      `user-data-dir-${Date.now().toString(32)}-${++i}`
+    );
+  }
   const chromedriverLogPath = path.join(
     LOG_PATH,
     `chromedriver.${nowFormatted}.log`
   );
   const webdriverLogPath = path.join(LOG_PATH, 'webdriver');
 
-  await fs.mkdir(userDataDir, { recursive: true });
+  // Ensure that the user data dir exists
+  await fs.mkdir(defaultUserDataDir, { recursive: true });
+
   // Chromedriver will fail if log path doesn't exist, webdriver doesn't care,
   // for consistency let's mkdir for both of them just in case
   await fs.mkdir(path.dirname(chromedriverLogPath), { recursive: true });
@@ -252,7 +286,7 @@ async function startCompass(
   // https://peter.sh/experiments/chromium-command-line-switches/
   // https://www.electronjs.org/docs/latest/api/command-line-switches
   chromeArgs.push(
-    `--user-data-dir=${userDataDir}`,
+    `--user-data-dir=${defaultUserDataDir}`,
     // Chromecast feature that is enabled by default in some chrome versions
     // and breaks the app on Ubuntu
     '--media-router=0',
@@ -323,7 +357,7 @@ async function startCompass(
   // @ts-expect-error
   const browser = await remote(options);
 
-  const compass = new Compass(browser, { userDataDir });
+  const compass = new Compass(browser, { isFirstRun });
 
   await compass.recordLogs();
 
@@ -514,14 +548,20 @@ function augmentError(error: Error, stack: string) {
   error.stack = `${error.stack ?? ''}\nvia ${strippedLines.join('\n')}`;
 }
 
-export async function beforeTests(): Promise<Compass> {
-  const compass = await startCompass();
+export async function beforeTests(
+  opts?: StartCompassOptions
+): Promise<Compass> {
+  const compass = await startCompass(opts);
 
   const { browser } = compass;
 
   await browser.waitForConnectionScreen();
-  await browser.closeTourModal();
-  await browser.closePrivacySettingsModal();
+  if (process.env.SHOW_TOUR) {
+    await browser.closeTourModal();
+  }
+  if (compass.isFirstRun) {
+    await browser.closePrivacySettingsModal();
+  }
 
   return compass;
 }
diff --git a/packages/compass-e2e-tests/index.ts b/packages/compass-e2e-tests/index.ts
index 43470d07e5c..fa052c74052 100644
--- a/packages/compass-e2e-tests/index.ts
+++ b/packages/compass-e2e-tests/index.ts
@@ -12,6 +12,7 @@ import {
   compileCompassAssets,
   buildCompass,
   LOG_PATH,
+  removeUserDataDir,
 } from './helpers/compass';
 import { createUnlockedKeychain } from './helpers/keychain';
 import ResultLogger from './helpers/result-logger';
@@ -22,6 +23,8 @@ const keychain = createUnlockedKeychain();
 // We can't import mongodb here yet because native modules will be recompiled
 let metricsClient: MongoClient;
 
+const FIRST_TEST = 'tests/time-to-first-query.test.ts';
+
 async function setup() {
   await keychain.activate();
 
@@ -48,6 +51,8 @@ async function setup() {
 }
 
 function cleanup() {
+  removeUserDataDir();
+
   keychain.reset();
 
   const disableStartStop = process.argv.includes('--disable-start-stop');
@@ -112,10 +117,18 @@ async function main() {
     }
   }
 
-  const tests = await promisify(glob)('tests/**/*.{test,spec}.ts', {
+  const rawTests = await promisify(glob)('tests/**/*.{test,spec}.ts', {
     cwd: __dirname,
   });
 
+  // The only test file that's interested in the first-run experience (at the
+  // time of writing) is time-to-first-query.ts and that happens to be
+  // alphabetically right at the end. Which is fine, but the first test to run
+  // will also get the slow first run experience for no good reason unless it is
+  // the time-to-first-query.ts test.
+  // So yeah.. this is a bit of a micro optimisation.
+  const tests = [FIRST_TEST, ...rawTests.filter((t) => t !== FIRST_TEST)];
+
   const bail = process.argv.includes('--bail');
 
   const mocha = new Mocha({
@@ -203,6 +216,7 @@ async function run() {
     if (metricsClient) {
       await metricsClient.close();
     }
+
     cleanup();
   }
 }
diff --git a/packages/compass-e2e-tests/tests/logging.test.ts b/packages/compass-e2e-tests/tests/logging.test.ts
index 71cc74e6e4c..9ce68d5d4e3 100644
--- a/packages/compass-e2e-tests/tests/logging.test.ts
+++ b/packages/compass-e2e-tests/tests/logging.test.ts
@@ -11,7 +11,7 @@ describe('Logging and Telemetry integration', function () {
 
     before(async function () {
       telemetry = await startTelemetryServer();
-      const compass = await beforeTests();
+      const compass = await beforeTests({ firstRun: true });
       const { browser } = compass;
       try {
         await browser.connectWithConnectionString(
diff --git a/packages/compass-e2e-tests/tests/time-to-first-query.test.ts b/packages/compass-e2e-tests/tests/time-to-first-query.test.ts
index 3e6be270551..546915e54f4 100644
--- a/packages/compass-e2e-tests/tests/time-to-first-query.test.ts
+++ b/packages/compass-e2e-tests/tests/time-to-first-query.test.ts
@@ -4,9 +4,56 @@ import type { Compass } from '../helpers/compass';
 import * as Selectors from '../helpers/selectors';
 
 describe('Time to first query', function () {
-  let compass: Compass;
+  let compass: Compass | undefined;
 
-  it('can open compass, connect to a database and run a query on a collection', async function () {
+  afterEach(async function () {
+    // cleanup outside of the test so that the time it takes to run does not
+    // get added to the time it took to run the first query
+    delete process.env.SHOW_TOUR;
+    if (compass) {
+      // even though this is after (not afterEach) currentTest points to the last test
+      await afterTest(compass, this.currentTest);
+      await afterTests(compass);
+      compass = undefined;
+    }
+  });
+
+  it('can open compass, connect to a database and run a query on a collection (first run)', async function () {
+    // start compass inside the test so that the time is measured together
+    compass = await beforeTests({ firstRun: true });
+
+    const { browser } = compass;
+
+    await browser.connectWithConnectionString('mongodb://localhost:27018/test');
+
+    await browser.navigateToCollectionTab('test', 'numbers', 'Documents');
+
+    // search for the document with id == 42 and wait for just one result to appear
+    const aceCommentElement = await browser.$(
+      '#query-bar-option-input-filter .ace_scroller'
+    );
+    await aceCommentElement.click();
+
+    await browser.keys('{ i: 42 }');
+    const filterButtonElement = await browser.$(
+      Selectors.queryBarApplyFilterButton('Documents')
+    );
+    await filterButtonElement.click();
+    await browser.waitUntil(async () => {
+      // we start off with 20 results (assuming no filter) and we expect to
+      // have just one once the filter finishes
+      const result = await browser.$$('.document-list .document');
+      return result.length === 1;
+    });
+
+    const documentElementValue = await browser.$(
+      '.document-list .document .element-value-is-int32'
+    );
+    const text = await documentElementValue.getText();
+    expect(text).to.equal('42');
+  });
+
+  it('can open compass, connect to a database and run a query on a collection (second run onwards)', async function () {
     // start compass inside the test so that the time is measured together
     compass = await beforeTests();
 
@@ -41,14 +88,41 @@ describe('Time to first query', function () {
     expect(text).to.equal('42');
   });
 
-  // eslint-disable-next-line mocha/no-hooks-for-single-case
-  after(async function () {
-    // cleanup outside of the test so that the time it takes to run does not
-    // get added to the time it took to run the first query
-    if (compass) {
-      // even though this is after (not afterEach) currentTest points to the last test
-      await afterTest(compass, this.currentTest);
-      await afterTests(compass);
-    }
+  it('can open compass, connect to a database and run a query on a collection (new version)', async function () {
+    // force the tour modal which would normally only appear for new versions
+    process.env.SHOW_TOUR = 'true';
+
+    // start compass inside the test so that the time is measured together
+    compass = await beforeTests();
+
+    const { browser } = compass;
+
+    await browser.connectWithConnectionString('mongodb://localhost:27018/test');
+
+    await browser.navigateToCollectionTab('test', 'numbers', 'Documents');
+
+    // search for the document with id == 42 and wait for just one result to appear
+    const aceCommentElement = await browser.$(
+      '#query-bar-option-input-filter .ace_scroller'
+    );
+    await aceCommentElement.click();
+
+    await browser.keys('{ i: 42 }');
+    const filterButtonElement = await browser.$(
+      Selectors.queryBarApplyFilterButton('Documents')
+    );
+    await filterButtonElement.click();
+    await browser.waitUntil(async () => {
+      // we start off with 20 results (assuming no filter) and we expect to
+      // have just one once the filter finishes
+      const result = await browser.$$('.document-list .document');
+      return result.length === 1;
+    });
+
+    const documentElementValue = await browser.$(
+      '.document-list .document .element-value-is-int32'
+    );
+    const text = await documentElementValue.getText();
+    expect(text).to.equal('42');
   });
 });
diff --git a/packages/compass/src/app/index.js b/packages/compass/src/app/index.js
index 6b61ebc6735..06a8cc46f7a 100644
--- a/packages/compass/src/app/index.js
+++ b/packages/compass/src/app/index.js
@@ -72,7 +72,7 @@ ipc.once('app:launched', function() {
 
 const { log, mongoLogId, debug, track } =
   require('@mongodb-js/compass-logging').createLoggerAndTelemetry('COMPASS-APP');
-  
+
 /**
  * The top-level application singleton that brings everything together!
  */
@@ -309,8 +309,9 @@ var Application = View.extend({
       var save = false;
       if (
         semver.lt(oldVersion, currentVersion) ||
-        // So we can test the tour in any e2e environment, not only on prod
-        process.env.APP_ENV === 'webdriverio'
+        // this is so we can test the tour modal in E2E tests where the version
+        // is always the same
+        process.env.SHOW_TOUR
       ) {
         prefs.showFeatureTour = oldVersion;
         save = true;