[ovn] Stop monitoring the SB MAC_Binding table to reduce mem footprint

The MAC_Binding table in the SB database may grow indefinitely (due
to a lack of an aging mechanism of its entries) and eventually
lead to OOM killers for neutron-server which maintains an in-memory
copy of the database.

In order to stop monitoring this table, this patch is invoking
the ovsdb-client tool to remove the entries associated to
Floating IPs that have just been detached. The execution of this
tool is really fast as it will just invoke a JSON-RPC transact command
which doesn't require downloading the database contents.

In a scale test, the memory consumption of neutron-server dropped
from 75GB to 7GB with this patch.

Closes-Bug: #1946318

Signed-off-by: Daniel Alvarez Sanchez <dalvarez@redhat.com>
Change-Id: Id84bf17953527c415d611bfc198038fb6f811de3

Author: danalsan

neutron/plugins/ml2/drivers/ovn/mech_driver/mech_driver.py

@@ -34,6 +34,7 @@
 from neutron_lib.exceptions import availability_zone as az_exc
 from neutron_lib.plugins import directory
 from neutron_lib.plugins.ml2 import api
+from oslo_concurrency import processutils
 from oslo_config import cfg
 from oslo_db import exception as os_db_exc
 from oslo_log import log
@@ -1098,10 +1099,18 @@ def set_port_status_down(self, port_id):
 
     def delete_mac_binding_entries(self, external_ip):
         """Delete all MAC_Binding entries associated to this IP address"""
-        mac_binds = self.sb_ovn.db_find_rows(
-            'MAC_Binding', ('ip', '=', external_ip)).execute() or []
-        for entry in mac_binds:
-            self.sb_ovn.db_destroy('MAC_Binding', entry.uuid).execute()
+        cmd = ['ovsdb-client', 'transact', ovn_conf.get_ovn_sb_connection()]
+
+        if ovn_conf.get_ovn_sb_private_key():
+            cmd += ['-p', ovn_conf.get_ovn_sb_private_key(), '-c',
+                    ovn_conf.get_ovn_sb_certificate(), '-C',
+                    ovn_conf.get_ovn_sb_ca_cert()]
+
+        cmd += ['["OVN_Southbound", {"op": "delete", "table": "MAC_Binding", '
+                '"where": [["ip", "==", "%s"]]}]' % external_ip]
+
+        return processutils.execute(*cmd,
+                                    log_errors=processutils.LOG_FINAL_ERROR)
 
     def update_segment_host_mapping(self, host, phy_nets):
         """Update SegmentHostMapping in DB"""

neutron/plugins/ml2/drivers/ovn/mech_driver/ovsdb/ovsdb_monitor.py

@@ -736,7 +736,6 @@ def from_server(cls, connection_string, helper, driver):
         helper.register_table('Encap')
         helper.register_table('Port_Binding')
         helper.register_table('Datapath_Binding')
-        helper.register_table('MAC_Binding')
         helper.register_table('Connection')
         helper.register_columns('SB_Global', ['external_ids'])
         return cls(driver, connection_string, helper)

neutron/tests/functional/plugins/ml2/drivers/ovn/mech_driver/ovsdb/test_ovsdb_monitor.py

@@ -15,6 +15,7 @@
 from unittest import mock
 
 import fixtures as og_fixtures
+from oslo_concurrency import processutils
 from oslo_utils import uuidutils
 
 from neutron.common.ovn import constants as ovn_const
@@ -33,17 +34,6 @@
 from ovsdbapp.backend.ovs_idl import idlutils
 
 
-class WaitForMACBindingDeleteEvent(event.WaitEvent):
-    event_name = 'WaitForMACBindingDeleteEvent'
-
-    def __init__(self, entry):
-        table = 'MAC_Binding'
-        events = (self.ROW_DELETE,)
-        conditions = (('_uuid', '=', entry),)
-        super(WaitForMACBindingDeleteEvent, self).__init__(
-            events, table, conditions, timeout=15)
-
-
 class WaitForDataPathBindingCreateEvent(event.WaitEvent):
     event_name = 'WaitForDataPathBindingCreateEvent'
 
@@ -132,6 +122,22 @@ def _create_fip(self, port, fip_address):
                 'port_id': port['id']}})
         return r1_f2
 
+    def _check_mac_binding_exists(self, macb_id):
+        cmd = ['ovsdb-client', 'transact',
+               self.mech_driver.sb_ovn.connection_string]
+
+        if self._ovsdb_protocol == 'ssl':
+            cmd += ['-p', self.ovsdb_server_mgr.private_key, '-c',
+                    self.ovsdb_server_mgr.certificate, '-C',
+                    self.ovsdb_server_mgr.ca_cert]
+
+        cmd += ['["OVN_Southbound", {"op": "select", "table": "MAC_Binding", '
+                '"where": [["_uuid", "==", ["uuid", "%s"]]]}]' % macb_id]
+
+        out, _ = processutils.execute(*cmd,
+                                      log_errors=False)
+        return str(macb_id) in out
+
     def test_floatingip_mac_bindings(self):
         """Check that MAC_Binding entries are cleared on FIP add/removal
 
@@ -146,6 +152,8 @@ def test_floatingip_mac_bindings(self):
         * Check that the MAC_Binding entry gets deleted.
         """
         net_name = 'network1'
+        self.mech_driver.sb_ovn.idl.update_tables(
+            ['MAC_Binding'], self.mech_driver.sb_schema_helper.schema_json)
         row_event = WaitForDataPathBindingCreateEvent(net_name)
         self.mech_driver.sb_ovn.idl.notify_handler.watch_event(row_event)
         self._make_network(self.fmt, net_name, True)
@@ -158,22 +166,21 @@ def test_floatingip_mac_bindings(self):
         port = self.create_port()
 
         # Ensure that the MAC_Binding entry gets deleted after creating a FIP
-        row_event = WaitForMACBindingDeleteEvent(macb_id)
-        self.mech_driver.sb_ovn.idl.notify_handler.watch_event(row_event)
         fip = self._create_fip(port, '100.0.0.21')
-        self.assertTrue(row_event.wait())
+        n_utils.wait_until_true(
+            lambda: not self._check_mac_binding_exists(macb_id),
+            timeout=15, sleep=1)
 
         # Now that the FIP is created, add a new MAC_Binding entry with the
         # same IP address
-
         macb_id = self.sb_api.db_create('MAC_Binding', datapath=dp[0]['_uuid'],
                                         ip='100.0.0.21').execute()
 
         # Ensure that the MAC_Binding entry gets deleted after deleting the FIP
-        row_event = WaitForMACBindingDeleteEvent(macb_id)
-        self.mech_driver.sb_ovn.idl.notify_handler.watch_event(row_event)
         self.l3_plugin.delete_floatingip(self.context, fip['id'])
-        self.assertTrue(row_event.wait())
+        n_utils.wait_until_true(
+            lambda: not self._check_mac_binding_exists(macb_id),
+            timeout=15, sleep=1)
 
     def _test_port_binding_and_status(self, port_id, action, status):
         # This function binds or unbinds port to chassis and

neutron/tests/unit/plugins/ml2/drivers/ovn/mech_driver/test_mech_driver.py

@@ -14,6 +14,7 @@
 
 import copy
 import datetime
+import shlex
 from unittest import mock
 import uuid
 
@@ -32,6 +33,7 @@
 from neutron_lib.plugins import directory
 from neutron_lib.tests import tools
 from neutron_lib.utils import net as n_net
+from oslo_concurrency import processutils
 from oslo_config import cfg
 from oslo_db import exception as os_db_exc
 from oslo_serialization import jsonutils
@@ -131,6 +133,29 @@ def setUp(self):
         p.start()
         self.addCleanup(p.stop)
 
+    def test_delete_mac_binding_entries(self):
+        self.config(group='ovn', ovn_sb_private_key=None)
+        expected = ('ovsdb-client transact tcp:127.0.0.1:6642 '
+                   '\'["OVN_Southbound", {"op": "delete", "table": '
+                   '"MAC_Binding", "where": [["ip", "==", "1.1.1.1"]]}]\'')
+        with mock.patch.object(processutils, 'execute') as mock_execute:
+            self.mech_driver.delete_mac_binding_entries('1.1.1.1')
+            mock_execute.assert_called_once_with(*shlex.split(expected),
+                    log_errors=processutils.LOG_FINAL_ERROR)
+
+    def test_delete_mac_binding_entries_ssl(self):
+        self.config(group='ovn', ovn_sb_private_key='pk')
+        self.config(group='ovn', ovn_sb_certificate='cert')
+        self.config(group='ovn', ovn_sb_ca_cert='ca')
+        expected = ('ovsdb-client transact tcp:127.0.0.1:6642 '
+                   '-p pk -c cert -C ca '
+                   '\'["OVN_Southbound", {"op": "delete", "table": '
+                   '"MAC_Binding", "where": [["ip", "==", "1.1.1.1"]]}]\'')
+        with mock.patch.object(processutils, 'execute') as mock_execute:
+            self.mech_driver.delete_mac_binding_entries('1.1.1.1')
+            mock_execute.assert_called_once_with(*shlex.split(expected),
+                    log_errors=processutils.LOG_FINAL_ERROR)
+
 
 class TestOVNMechanismDriver(TestOVNMechanismDriverBase):
     @mock.patch.object(ovsdb_monitor.OvnInitPGNbIdl, 'from_server')