diff --git a/packages/cli-repl/test/e2e-direct.spec.ts b/packages/cli-repl/test/e2e-direct.spec.ts
index 5b85f5fa03..2cd8013d36 100644
--- a/packages/cli-repl/test/e2e-direct.spec.ts
+++ b/packages/cli-repl/test/e2e-direct.spec.ts
@@ -230,6 +230,16 @@ describe('e2e direct connection', () => {
           });
         });
       });
+
+      describe('fail-fast connections', () => {
+        it('does not fail fast for ECONNREFUSED errors when one host is reachable', async() => {
+          const shell = TestShell.start({ args: [
+            `mongodb://${await rs1.hostport()},127.0.0.1:1/?replicaSet=${replSetId}&readPreference=secondary`
+          ] });
+          const result = await shell.waitForPromptOrExit();
+          expect(result).to.deep.equal({ state: 'prompt' });
+        });
+      });
     });
   });
 });
diff --git a/packages/cli-repl/test/e2e.spec.ts b/packages/cli-repl/test/e2e.spec.ts
index e92e4c1fef..32ac746fff 100644
--- a/packages/cli-repl/test/e2e.spec.ts
+++ b/packages/cli-repl/test/e2e.spec.ts
@@ -939,9 +939,23 @@ describe('e2e', function() {
       expect(exitCode).to.equal(1);
     });
 
-    it('fails fast for ECONNREFUSED errors', async() => {
+    it('fails fast for ECONNREFUSED errors to a single host', async() => {
       const shell = TestShell.start({ args: [
-        '--port', '0'
+        '--port', '1'
+      ] });
+      const result = await shell.waitForPromptOrExit();
+      expect(result).to.deep.equal({ state: 'exit', exitCode: 1 });
+    });
+
+    it('fails fast for ECONNREFUSED errors to multiple hosts', async function() {
+      if (process.platform === 'darwin') {
+        // On macOS, for some reason only connection that fails is the 127.0.0.1:1
+        // one, over and over. It should be fine to only skip the test there, as this
+        // isn't a shell-specific issue.
+        return this.skip();
+      }
+      const shell = TestShell.start({ args: [
+        'mongodb://127.0.0.1:1,127.0.0.2:1,127.0.0.3:1/?replicaSet=foo&readPreference=secondary'
       ] });
       const result = await shell.waitForPromptOrExit();
       expect(result).to.deep.equal({ state: 'exit', exitCode: 1 });
diff --git a/packages/service-provider-server/src/cli-service-provider.spec.ts b/packages/service-provider-server/src/cli-service-provider.spec.ts
index 7c2a5def9a..1733f580ee 100644
--- a/packages/service-provider-server/src/cli-service-provider.spec.ts
+++ b/packages/service-provider-server/src/cli-service-provider.spec.ts
@@ -41,6 +41,7 @@ describe('CliServiceProvider', () => {
       connect() {}
       db() {}
       close() {}
+      topology: any;
     }
 
     it('connects once when no AutoEncryption set', async() => {
@@ -141,9 +142,16 @@ describe('CliServiceProvider', () => {
       mClient.connect = () => new Promise((resolve, reject) => {
         rejectConnect = reject;
         setImmediate(() => {
-          mClient.emit('serverHeartbeatFailed', { failure: err });
+          mClient.emit('serverHeartbeatFailed', { failure: err, connectionId: uri });
         });
       });
+      mClient.topology = {
+        description: {
+          servers: new Map([
+            ['localhost:27017', {}]
+          ])
+        }
+      };
       try {
         await connectMongoClient(uri, {}, mClientType as any);
       } catch (e) {
diff --git a/packages/service-provider-server/src/cli-service-provider.ts b/packages/service-provider-server/src/cli-service-provider.ts
index 966c16137d..625b79912a 100644
--- a/packages/service-provider-server/src/cli-service-provider.ts
+++ b/packages/service-provider-server/src/cli-service-provider.ts
@@ -19,7 +19,9 @@ import {
   ReadPreferenceFromOptions,
   ReadPreferenceLike,
   OperationOptions,
-  ServerHeartbeatFailedEvent
+  ServerHeartbeatFailedEvent,
+  ServerHeartbeatSucceededEvent,
+  TopologyDescription
 } from 'mongodb';
 
 import {
@@ -141,21 +143,41 @@ const DEFAULT_BASE_OPTIONS: OperationOptions = Object.freeze({
  * errors.
  */
 async function connectWithFailFast(client: MongoClient): Promise<void> {
-  let failFastErr;
-  const heartbeatFailureListener = ({ failure }: ServerHeartbeatFailedEvent) => {
+  const failedConnections = new Map<string, Error>();
+  let failedEarly = false;
+
+  const heartbeatFailureListener = ({ failure, connectionId }: ServerHeartbeatFailedEvent) => {
+    const topologyDescription: TopologyDescription | undefined = (client as any).topology?.description;
+    const servers = topologyDescription?.servers;
+    if (!servers?.has(connectionId)) {
+      return; // TODO: Make a way to log this condition to the mongosh bus.
+    }
+
     if (isFastFailureConnectionError(failure)) {
-      failFastErr = failure;
-      client.close();
+      failedConnections.set(connectionId, failure);
+      if ([...servers.keys()].every(server => failedConnections.has(server))) {
+        failedEarly = true;
+        client.close();
+      }
     }
   };
 
+  const heartbeatSucceededListener = ({ connectionId }: ServerHeartbeatSucceededEvent) => {
+    failedConnections.delete(connectionId);
+  };
+
   client.addListener('serverHeartbeatFailed', heartbeatFailureListener);
+  client.addListener('serverHeartbeatSucceeded', heartbeatSucceededListener);
   try {
     await client.connect();
   } catch (err) {
-    throw failFastErr || err;
+    if (failedEarly) {
+      throw failedConnections.values().next().value; // Just use the first failure.
+    }
+    throw err;
   } finally {
     client.removeListener('serverHeartbeatFailed', heartbeatFailureListener);
+    client.removeListener('serverHeartbeatSucceeded', heartbeatSucceededListener);
   }
 }