diff --git a/homeassistant/components/esphome/voice_assistant.py b/homeassistant/components/esphome/voice_assistant.py
index de6b521d980823..26914cae1cb4b8 100644
--- a/homeassistant/components/esphome/voice_assistant.py
+++ b/homeassistant/components/esphome/voice_assistant.py
@@ -39,7 +39,8 @@
 
 _LOGGER = logging.getLogger(__name__)
 
-UDP_PORT = 0  # Set to 0 to let the OS pick a free random port
+UDP_PORT_MIN = 10500  # Set to 0 to let the OS pick a free random port
+UDP_PORT_MAX = 10520
 UDP_MAX_PACKET_SIZE = 1024
 
 _VOICE_ASSISTANT_EVENT_TYPES: EsphomeEnumMapper[
@@ -108,7 +109,23 @@ def accept_connection() -> VoiceAssistantUDPServer:
         sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
         sock.setblocking(False)
 
-        sock.bind(("", UDP_PORT))
+        def find_port(port: int, port_max: int) -> int:
+            sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
+            while port <= port_max:
+                try:
+                    sock.bind(("", port))
+                    sock.close()
+                    return port
+                except OSError:
+                    port += 1
+            raise OSError("No free ports")
+
+        if UDP_PORT_MIN == 0:
+            udp_port = 0
+        else:
+            udp_port = find_port(UDP_PORT_MIN, UDP_PORT_MAX)
+
+        sock.bind(("", udp_port))
 
         await asyncio.get_running_loop().create_datagram_endpoint(
             accept_connection, sock=sock