diff --git a/src/Symfony/Component/Mailer/Tests/Transport/Smtp/Stream/SocketStreamTest.php b/src/Symfony/Component/Mailer/Tests/Transport/Smtp/Stream/SocketStreamTest.php
new file mode 100644
index 000000000000..1f532d7c107f
--- /dev/null
+++ b/src/Symfony/Component/Mailer/Tests/Transport/Smtp/Stream/SocketStreamTest.php
@@ -0,0 +1,49 @@
+<?php
+
+/*
+ * This file is part of the Symfony package.
+ *
+ * (c) Fabien Potencier <fabien@symfony.com>
+ *
+ * For the full copyright and license information, please view the LICENSE
+ * file that was distributed with this source code.
+ */
+
+namespace Symfony\Component\Mailer\Tests\Transport\Smtp\Stream;
+
+use PHPUnit\Framework\TestCase;
+use Symfony\Component\Mailer\Transport\Smtp\Stream\SocketStream;
+
+class SocketStreamTest extends TestCase
+{
+    /**
+     * @expectedException \Symfony\Component\Mailer\Exception\TransportException
+     * @expectedExceptionMessage Connection refused
+     */
+    public function testSocketErrorNoConnection()
+    {
+        $s = new SocketStream();
+        $s->setTimeout(0.1);
+        $s->setPort(9999);
+        $s->initialize();
+    }
+
+    /**
+     * @expectedException \Symfony\Component\Mailer\Exception\TransportException
+     * @expectedExceptionMessage no valid certs found cafile stream
+     */
+    public function testSocketErrorBeforeConnectError()
+    {
+        $s = new SocketStream();
+        $s->setStreamOptions([
+            'ssl' => [
+                // not a CA file :)
+                'cafile' => __FILE__,
+            ],
+        ]);
+        $s->setEncryption('ssl');
+        $s->setHost('smtp.gmail.com');
+        $s->setPort(465);
+        $s->initialize();
+    }
+}
diff --git a/src/Symfony/Component/Mailer/Transport/Smtp/Stream/SocketStream.php b/src/Symfony/Component/Mailer/Transport/Smtp/Stream/SocketStream.php
index 0d075b0b6e94..a502c85e822a 100644
--- a/src/Symfony/Component/Mailer/Transport/Smtp/Stream/SocketStream.php
+++ b/src/Symfony/Component/Mailer/Transport/Smtp/Stream/SocketStream.php
@@ -144,10 +144,16 @@ public function initialize(): void
             $options['ssl']['crypto_method'] = $options['ssl']['crypto_method'] ?? STREAM_CRYPTO_METHOD_TLS_CLIENT | STREAM_CRYPTO_METHOD_TLSv1_2_CLIENT | STREAM_CRYPTO_METHOD_TLSv1_1_CLIENT;
         }
         $streamContext = stream_context_create($options);
-        $this->stream = @stream_socket_client($this->url, $errno, $errstr, $this->timeout, STREAM_CLIENT_CONNECT, $streamContext);
-        if (false === $this->stream) {
-            throw new TransportException(sprintf('Connection could not be established with host "%s": %s (%s)', $this->url, $errstr, $errno));
+
+        set_error_handler(function ($type, $msg) {
+            throw new TransportException(sprintf('Connection could not be established with host "%s": %s.', $this->url, $msg));
+        });
+        try {
+            $this->stream = stream_socket_client($this->url, $errno, $errstr, $this->timeout, STREAM_CLIENT_CONNECT, $streamContext);
+        } finally {
+            restore_error_handler();
         }
+
         stream_set_blocking($this->stream, true);
         stream_set_timeout($this->stream, $this->timeout);
         $this->in = &$this->stream;