OVN: Enable E-W Traffic, Vlan backed DVR

Background:
[1] https://mail.openvswitch.org/pipermail/ovs-dev/2018-October/353066.html
[2] https://docs.google.com/document/d/1uoQH478wM1OZ16HrxzbOUvk5LvFnfNEWbkPT6Zmm9OU/edit?usp=sharing

Key difference between an overlay logical switch and
vlan backed logical switch is that for vlan logical switches
packets are not encapsulated.

Hence, if a distributed router port is connected to vlan backed
logical switch, then router port mac as source mac could be
seen from multiple hypervisors. Same <mac,vlan> pairs coming
from multiple ports from a top of the rack switch (TOR) perspective
could be seen as a security threat and it could send alarms, drop
the packets or block the ports etc.

This patch addresses the same by introducing the concept of chassis mac.
A chassis mac is CMS provisioned unique mac per chassis. For any routed packet
(i.e source mac is router port mac) going on the wire on a vlan type
logical switch, we will replace its source mac with chassis mac.

This replacing of source mac with chassis mac will happen in table=65
of the logical switch datapath. A flow is added at priority 150, which
matches the source mac and replaces it with chassis mac if the value
is a router port mac.

Example flow:
cookie=0x0, duration=67765.830s, table=65, n_packets=0, n_bytes=0,
idle_age=65534, hard_age=65534, priority=150,reg15=0x1,metadata=0x4,
dl_src=00:00:01:01:02:03 actions=mod_dl_src:aa:bb:cc:dd:ee:ff,
mod_vlan_vid:1000,output:16

Here, 00:00:01:01:02:03 is router port mac and aa:bb:cc:dd:ee:ff
is chassis mac.

Acked-by: Numan Siddique <nusiddiq@redhat.com>
Signed-off-by: Ankur Sharma <ankur.sharma@nutanix.com>
Signed-off-by: Ben Pfaff <blp@ovn.org>

ovn/controller/binding.c

@@ -159,13 +159,11 @@ add_local_datapath__(struct ovsdb_idl_index *sbrec_datapath_binding_by_key,
                                          sbrec_port_binding_by_name,
                                          peer->datapath, false,
                                          depth + 1, local_datapaths);
-                    ld->n_peer_dps++;
-                    ld->peer_dps = xrealloc(
-                            ld->peer_dps,
-                            ld->n_peer_dps * sizeof *ld->peer_dps);
-                    ld->peer_dps[ld->n_peer_dps - 1] = datapath_lookup_by_key(
-                        sbrec_datapath_binding_by_key,
-                        peer->datapath->tunnel_key);
+                    ld->n_peer_ports++;
+                    ld->peer_ports = xrealloc(ld->peer_ports,
+                                              ld->n_peer_ports *
+                                              sizeof *ld->peer_ports);
+                    ld->peer_ports[ld->n_peer_ports - 1] = peer;
                 }
             }
         }

ovn/controller/chassis.c

@@ -23,6 +23,7 @@
 #include "lib/vswitch-idl.h"
 #include "openvswitch/dynamic-string.h"
 #include "openvswitch/vlog.h"
+#include "openvswitch/ofp-parse.h"
 #include "ovn/lib/chassis-index.h"
 #include "ovn/lib/ovn-sb-idl.h"
 #include "ovn-controller.h"
@@ -68,6 +69,12 @@ get_bridge_mappings(const struct smap *ext_ids)
     return smap_get_def(ext_ids, "ovn-bridge-mappings", "");
 }
 
+static const char *
+get_chassis_mac_mappings(const struct smap *ext_ids)
+{
+    return smap_get_def(ext_ids, "ovn-chassis-mac-mappings", "");
+}
+
 static const char *
 get_cms_options(const struct smap *ext_ids)
 {
@@ -162,6 +169,7 @@ chassis_run(struct ovsdb_idl_txn *ovnsb_idl_txn,
     const char *datapath_type =
         br_int && br_int->datapath_type ? br_int->datapath_type : "";
     const char *cms_options = get_cms_options(&cfg->external_ids);
+    const char *chassis_macs = get_chassis_mac_mappings(&cfg->external_ids);
 
     struct ds iface_types = DS_EMPTY_INITIALIZER;
     ds_put_cstr(&iface_types, "");
@@ -190,18 +198,22 @@ chassis_run(struct ovsdb_idl_txn *ovnsb_idl_txn,
             = smap_get_def(&chassis_rec->external_ids, "iface-types", "");
         const char *chassis_cms_options
             = get_cms_options(&chassis_rec->external_ids);
+        const char *chassis_mac_mappings
+            = get_chassis_mac_mappings(&chassis_rec->external_ids);
 
         /* If any of the external-ids should change, update them. */
         if (strcmp(bridge_mappings, chassis_bridge_mappings) ||
             strcmp(datapath_type, chassis_datapath_type) ||
             strcmp(iface_types_str, chassis_iface_types) ||
-            strcmp(cms_options, chassis_cms_options)) {
+            strcmp(cms_options, chassis_cms_options) ||
+            strcmp(chassis_macs, chassis_mac_mappings)) {
             struct smap new_ids;
             smap_clone(&new_ids, &chassis_rec->external_ids);
             smap_replace(&new_ids, "ovn-bridge-mappings", bridge_mappings);
             smap_replace(&new_ids, "datapath-type", datapath_type);
             smap_replace(&new_ids, "iface-types", iface_types_str);
             smap_replace(&new_ids, "ovn-cms-options", cms_options);
+            smap_replace(&new_ids, "ovn-chassis-mac-mappings", chassis_macs);
             sbrec_chassis_verify_external_ids(chassis_rec);
             sbrec_chassis_set_external_ids(chassis_rec, &new_ids);
             smap_destroy(&new_ids);
@@ -319,6 +331,51 @@ chassis_run(struct ovsdb_idl_txn *ovnsb_idl_txn,
     return chassis_rec;
 }
 
+bool
+chassis_get_mac(const struct sbrec_chassis *chassis_rec,
+                const char *bridge_mapping,
+                struct eth_addr *chassis_mac)
+{
+    const char *tokens
+        = get_chassis_mac_mappings(&chassis_rec->external_ids);
+    if (!tokens[0]) {
+       return false;
+    }
+
+    char *save_ptr = NULL;
+    bool ret = false;
+    char *tokstr = xstrdup(tokens);
+
+    /* Format for a chassis mac configuration is:
+     * ovn-chassis-mac-mappings="bridge-name1:MAC1,bridge-name2:MAC2"
+     */
+    for (char *token = strtok_r(tokstr, ",", &save_ptr);
+         token != NULL;
+         token = strtok_r(NULL, ",", &save_ptr)) {
+        char *save_ptr2 = NULL;
+        char *chassis_mac_bridge = strtok_r(token, ":", &save_ptr2);
+        char *chassis_mac_str = strtok_r(NULL, "", &save_ptr2);
+
+        if (!strcmp(chassis_mac_bridge, bridge_mapping)) {
+            struct eth_addr temp_mac;
+
+            /* Return the first chassis mac. */
+            char *err_str = str_to_mac(chassis_mac_str, &temp_mac);
+            if (err_str) {
+                free(err_str);
+                continue;
+            }
+
+            ret = true;
+            *chassis_mac = temp_mac;
+            break;
+        }
+    }
+
+    free(tokstr);
+    return ret;
+}
+
 /* Returns true if the database is all cleaned up, false if more work is
  * required. */
 bool

ovn/controller/chassis.h

@@ -26,6 +26,7 @@ struct ovsrec_open_vswitch_table;
 struct sbrec_chassis;
 struct sbrec_chassis_table;
 struct sset;
+struct eth_addr;
 
 void chassis_register_ovs_idl(struct ovsdb_idl *);
 const struct sbrec_chassis *chassis_run(
@@ -36,5 +37,8 @@ const struct sbrec_chassis *chassis_run(
     const struct sset *transport_zones);
 bool chassis_cleanup(struct ovsdb_idl_txn *ovnsb_idl_txn,
                      const struct sbrec_chassis *);
+bool chassis_get_mac(const struct sbrec_chassis *chassis,
+                     const char *bridge_mapping,
+                     struct eth_addr *chassis_mac);
 
 #endif /* ovn/chassis.h */

ovn/controller/ovn-controller.8.xml

@@ -196,6 +196,16 @@
           transport zone.
         </p>
       </dd>
+      <dt><code>external_ids:ovn-chassis-mac-mappings</code></dt>
+      <dd>
+        A list of key-value pairs that map a chassis specific mac to
+        a physical network name. An example
+        value mapping two chassis macs to two physical network names would be:
+        <code>physnet1:aa:bb:cc:dd:ee:ff,physnet2:a1:b2:c3:d4:e5:f6</code>.
+        These are the macs that ovn-controller will replace a router port
+        mac with, if packet is going from a distributed router port on
+        vlan type logical switch.
+      </dd>
     </dl>
 
     <p>

ovn/controller/ovn-controller.c

@@ -909,7 +909,7 @@ en_runtime_data_cleanup(struct engine_node *node)
     struct local_datapath *cur_node, *next_node;
     HMAP_FOR_EACH_SAFE (cur_node, next_node, hmap_node,
                         &data->local_datapaths) {
-        free(cur_node->peer_dps);
+        free(cur_node->peer_ports);
         hmap_remove(&data->local_datapaths, &cur_node->hmap_node);
         free(cur_node);
     }
@@ -939,7 +939,7 @@ en_runtime_data_run(struct engine_node *node)
     } else {
         struct local_datapath *cur_node, *next_node;
         HMAP_FOR_EACH_SAFE (cur_node, next_node, hmap_node, local_datapaths) {
-            free(cur_node->peer_dps);
+            free(cur_node->peer_ports);
             hmap_remove(local_datapaths, &cur_node->hmap_node);
             free(cur_node);
         }

ovn/controller/ovn-controller.h

@@ -59,8 +59,9 @@ struct local_datapath {
     /* True if this datapath contains an l3gateway port located on this
      * hypervisor. */
     bool has_local_l3gateway;
-    const struct sbrec_datapath_binding **peer_dps;
-    size_t n_peer_dps;
+
+    const struct sbrec_port_binding **peer_ports;
+    size_t n_peer_ports;
 };
 
 struct local_datapath *get_local_datapath(const struct hmap *,

ovn/controller/physical.c

@@ -21,6 +21,7 @@
 #include "ha-chassis.h"
 #include "lflow.h"
 #include "lport.h"
+#include "chassis.h"
 #include "lib/bundle.h"
 #include "openvswitch/poll-loop.h"
 #include "lib/uuid.h"
@@ -31,6 +32,7 @@
 #include "openvswitch/ofp-actions.h"
 #include "openvswitch/ofpbuf.h"
 #include "openvswitch/vlog.h"
+#include "openvswitch/ofp-parse.h"
 #include "ovn-controller.h"
 #include "ovn/lib/chassis-index.h"
 #include "ovn/lib/ovn-sb-idl.h"
@@ -225,6 +227,92 @@ get_zone_ids(const struct sbrec_port_binding *binding,
     return zone_ids;
 }
 
+static void
+put_replace_router_port_mac_flows(const struct
+                                  sbrec_port_binding *localnet_port,
+                                  const struct sbrec_chassis *chassis,
+                                  const struct hmap *local_datapaths,
+                                  struct ofpbuf *ofpacts_p,
+                                  ofp_port_t ofport,
+                                  struct ovn_desired_flow_table *flow_table)
+{
+    struct local_datapath *ld = get_local_datapath(local_datapaths,
+                                                   localnet_port->datapath->
+                                                   tunnel_key);
+    ovs_assert(ld);
+
+    uint32_t dp_key = localnet_port->datapath->tunnel_key;
+    uint32_t port_key = localnet_port->tunnel_key;
+    int tag = localnet_port->tag ? *localnet_port->tag : 0;
+    const char *network = smap_get(&localnet_port->options, "network_name");
+    struct eth_addr chassis_mac;
+
+    if (!network) {
+        static struct vlog_rate_limit rl = VLOG_RATE_LIMIT_INIT(1, 1);
+        VLOG_WARN_RL(&rl, "Physical network not configured for datapath:"
+                     "%"PRId64" with localnet port",
+                     localnet_port->datapath->tunnel_key);
+        return;
+    }
+
+    /* Get chassis mac */
+    if (!chassis_get_mac(chassis, network, &chassis_mac)) {
+        static struct vlog_rate_limit rl = VLOG_RATE_LIMIT_INIT(1, 1);
+        /* Keeping the log level low for backward compatibility.
+         * Chassis mac is a new configuration.
+         */
+        VLOG_DBG_RL(&rl, "Could not get chassis mac for network: %s", network);
+        return;
+    }
+
+    for (int i = 0; i < ld->n_peer_ports; i++) {
+        const struct sbrec_port_binding *rport_binding = ld->peer_ports[i];
+        struct eth_addr router_port_mac;
+        struct match match;
+        struct ofpact_mac *replace_mac;
+
+        /* Table 65, priority 150.
+         * =======================
+         *
+         * Implements output to localnet port.
+         * a. Flow replaces ingress router port mac with a chassis mac.
+         * b. Flow appends the vlan id localnet port is configured with.
+         */
+        match_init_catchall(&match);
+        ofpbuf_clear(ofpacts_p);
+
+        ovs_assert(rport_binding->n_mac == 1);
+        char *err_str = str_to_mac(rport_binding->mac[0], &router_port_mac);
+        if (err_str) {
+            /* Parsing of mac failed. */
+            VLOG_WARN("Parsing or router port mac failed for router port: %s, "
+                      "with error: %s", rport_binding->logical_port, err_str);
+            free(err_str);
+            return;
+        }
+
+        /* Replace Router mac flow */
+        match_set_metadata(&match, htonll(dp_key));
+        match_set_reg(&match, MFF_LOG_OUTPORT - MFF_REG0, port_key);
+        match_set_dl_src(&match, router_port_mac);
+
+        replace_mac = ofpact_put_SET_ETH_SRC(ofpacts_p);
+        replace_mac->mac = chassis_mac;
+
+        if (tag) {
+            struct ofpact_vlan_vid *vlan_vid;
+            vlan_vid = ofpact_put_SET_VLAN_VID(ofpacts_p);
+            vlan_vid->vlan_vid = tag;
+            vlan_vid->push_vlan_if_needed = true;
+        }
+
+        ofpact_put_OUTPUT(ofpacts_p)->port = ofport;
+
+        ofctrl_add_flow(flow_table, OFTABLE_LOG_TO_PHY, 150, 0,
+                        &match, ofpacts_p, &localnet_port->header_.uuid);
+    }
+}
+
 static void
 put_local_common_flows(uint32_t dp_key, uint32_t port_key,
                        uint32_t parent_port_key,
@@ -697,6 +785,13 @@ consider_port_binding(struct ovsdb_idl_index *sbrec_port_binding_by_name,
         }
         ofctrl_add_flow(flow_table, OFTABLE_LOG_TO_PHY, 100, 0,
                         &match, ofpacts_p, &binding->header_.uuid);
+
+        if (!strcmp(binding->type, "localnet")) {
+            put_replace_router_port_mac_flows(binding, chassis,
+                                              local_datapaths, ofpacts_p,
+                                              ofport, flow_table);
+        }
+
     } else if (!tun && !is_ha_remote) {
         /* Remote port connected by localnet port */
         /* Table 33, priority 100.

ovn/ovn-architecture.7.xml

@@ -1403,10 +1403,41 @@
     </li>
 
     <li>
-      From the router datapath, packet enters the ingress pipeline and then
-      egress pipeline of the destination localnet logical switch datapath
-      and goes out of the integration bridge to the provider bridge (
-      belonging to the destination logical switch) via the localnet port.
+      <p>
+        From the router datapath, packet enters the ingress pipeline and then
+        egress pipeline of the destination localnet logical switch datapath
+        and goes out of the integration bridge to the provider bridge (
+        belonging to the destination logical switch) via the localnet port.
+        While sending the packet to provider bridge, we also replace router
+        port MAC as source MAC with a chassis unique MAC.
+      </p>
+
+      <p>
+        This chassis unique MAC is configured as global ovs config on each
+        chassis (eg. via "<code>ovs-vsctl set open . external-ids:
+        ovn-chassis-mac-mappings="phys:aa:bb:cc:dd:ee:$i$i"</code>").  For more
+        details, see <code>ovn-controller</code>(8).
+      </p>
+
+      <p>
+        If the above is not configured, then source MAC would be the router
+        port MAC.  This could create problem if we have more than one chassis.
+        This is because, since the router port is distributed, the same
+        (MAC,VLAN) tuple will seen by physical network from other chassis as
+        well, which could cause these issues:
+      </p>
+
+      <ul>
+        <li>
+          Continuous MAC moves in top-of-rack switch (ToR).
+        </li>
+        <li>
+          ToR dropping the traffic, which is causing continuous MAC moves.
+        </li>
+        <li>
+          ToR blocking the ports from which MAC moves are happening.
+        </li>
+      </ul>
     </li>
 
     <li>

ovn/ovn-sb.xml

@@ -301,6 +301,14 @@
       See <code>ovn-controller</code>(8) for more information.
     </column>
 
+    <column name="external_ids" key="ovn-chassis-mac-mappings">
+      <code>ovn-controller</code> populates this key with the set of options
+      configured in the <ref table="Open_vSwitch"
+      column="external_ids:ovn-chassis-mac-mappings"/> column of the
+      Open_vSwitch database's <ref table="Open_vSwitch" db="Open_vSwitch"/>
+      table. See <code>ovn-controller</code>(8) for more information.
+    </column>
+
     <group title="Common Columns">
       The overall purpose of these columns is described under <code>Common
       Columns</code> at the beginning of this document.