Limit max ports per rpc for dhcp_ready_on_ports()

The Neutron dhcp agents reports all ready ports to the Neutron
server via the dhcp_ready_on_ports() rpc call. When the dhcp agent
gets ports ready faster than the server can process them the amount
of ports per rpc call can grow so high (e.g. 10000 Ports) that the
neutron server never has a chance of processing the request before
the rpc timeout kills the request, leading to the dhcp agent
sending the request again, resulting in an endless loop of
dhcp_ready_on_ports() calls. This happens especially on agent startup.

To mitigate this problems we now limit the number of ports sent
per dhcp_ready_on_ports() call.

Closes-bug: #1834257
Change-Id: I407e126e760ebf6aca4c31b9c3ff58dcfa55107f
(cherry picked from commit 76ccdb3)

neutron/agent/dhcp/agent.py

@@ -47,6 +47,8 @@
 
 DEFAULT_PRIORITY = 255
 
+DHCP_READY_PORTS_SYNC_MAX = 64
+
 
 def _sync_lock(f):
     """Decorator to block all operations for a global sync call."""
@@ -222,8 +224,11 @@ def _dhcp_ready_ports_loop(self):
             # this is just watching a set so we can do it really frequently
             eventlet.sleep(0.1)
             if self.dhcp_ready_ports:
-                ports_to_send = self.dhcp_ready_ports
-                self.dhcp_ready_ports = set()
+                ports_to_send = set()
+                for port_count in range(min(len(self.dhcp_ready_ports),
+                                            DHCP_READY_PORTS_SYNC_MAX)):
+                    ports_to_send.add(self.dhcp_ready_ports.pop())
+
                 try:
                     self.plugin_rpc.dhcp_ready_on_ports(ports_to_send)
                     continue

neutron/tests/unit/agent/dhcp/test_agent.py

@@ -459,6 +459,32 @@ def test__dhcp_ready_ports_loop(self):
         # should have been called with all ports again after the failure
         ready.assert_has_calls([mock.call(set(range(4)))] * 2)
 
+    def test_dhcp_ready_ports_loop_with_limit_ports_per_call(self):
+        dhcp = dhcp_agent.DhcpAgent(HOSTNAME)
+        sync_max = dhcp_agent.DHCP_READY_PORTS_SYNC_MAX
+        port_count = sync_max + 1
+        dhcp.dhcp_ready_ports = set(range(port_count))
+
+        with mock.patch.object(dhcp.plugin_rpc,
+                               'dhcp_ready_on_ports') as ready:
+            # exit after 2 iterations
+            with mock.patch.object(dhcp_agent.eventlet, 'sleep',
+                                   side_effect=[0, 0, RuntimeError]):
+                with testtools.ExpectedException(RuntimeError):
+                    dhcp._dhcp_ready_ports_loop()
+
+        # all ports should have been processed
+        self.assertEqual(set(), dhcp.dhcp_ready_ports)
+        # two calls are expected, one with DHCP_READY_PORTS_SYNC_MAX ports,
+        # second one with one port
+        self.assertEqual(2, ready.call_count)
+        self.assertEqual(sync_max, len(ready.call_args_list[0][0][0]))
+        self.assertEqual(1, len(ready.call_args_list[1][0][0]))
+        # all ports need to be ready
+        ports_ready = (ready.call_args_list[0][0][0] |
+                       ready.call_args_list[1][0][0])
+        self.assertEqual(set(range(port_count)), ports_ready)
+
     def test_dhcp_ready_ports_updates_after_enable_dhcp(self):
         dhcp = dhcp_agent.DhcpAgent(HOSTNAME)
         self.assertEqual(set(), dhcp.dhcp_ready_ports)