Add support for overlay provider networks.

It is expected that a provider network logical switch has a localnet logical
switch port in order to bridge the overlay traffic to the underlay traffic.
There can be some usecases where a overlay logical switch (without
a localnet port) can act as a provider network and presently NAT doesn't
work as expected.  This patch adds this support.  A new config option
"overlay_provider_network" is added to support this feature.
This feature gets enabled for a logical switch 'P' if:
  - The above option is set to true in the Logical_Switch.other_config
    column.
  - The logical switch 'P' doesn't have any localnet ports.
  - The logical router port of a router 'R' connecting to 'P'
    is a gateway router port.
  - And the logical router 'R' has only one gateway router port.

If all the above conditions are met, ovn-northd creates a chassisredirect
port for the logical switch port (of type router) connecting to the
router 'R'.  For example, if the logical port is named as "P-R" and its
peer router port is "R-P", then chassisredirect port cr-P-R is created
along with cr-R-P.  Gateway chassis binding the cr-R-P also binds cr-P-R.
This ensures that the routing is centralized on this gateway chassis for
the traffic coming from switch "P" towards the router or vice versa.
This centralization is required in order to support NAT (both SNAT and
DNAT).  Distributed NAT (i.e if external_mac and router_port is set) is
not supported and instead the router port mac is used for such traffic.

Reported-at: https://issues.redhat.com/browse/FDP-364
Signed-off-by: Numan Siddique <numans@ovn.org>

NEWS

@@ -21,6 +21,8 @@ Post v24.03.0
     MAC addresses configured on the LSP with "unknown", are learnt via the
     OVN native FDB.
   - Add support for ovsdb-server `--config-file` option in ovn-ctl.
+  - Added Overlay provider network support to a logical switch if
+    the config "overlay_provider_network" is set to true.
 
 OVN v24.03.0 - 01 Mar 2024
 --------------------------

controller/physical.c

@@ -1587,6 +1587,10 @@ consider_port_binding(struct ovsdb_idl_index *sbrec_port_binding_by_name,
                                                     ct_zones);
             put_zones_ofpacts(&zone_ids, ofpacts_p);
 
+            /* Clear the MFF_INPORT.  Its possible that the same packet may
+             * go out from the same tunnel inport. */
+            put_load(ofp_to_u16(OFPP_NONE), MFF_IN_PORT, 0, 16, ofpacts_p);
+
             /* Resubmit to table 41. */
             put_resubmit(OFTABLE_CHECK_LOOPBACK, ofpacts_p);
         }

northd/northd.c

@@ -2098,6 +2098,53 @@ parse_lsp_addrs(struct ovn_port *op)
     }
 }
 
+static struct ovn_port *
+create_cr_port(struct ovn_port *op, struct hmap *ports,
+               struct ovs_list *both_dbs, struct ovs_list *nb_only)
+{
+    char *redirect_name = ovn_chassis_redirect_name(
+        op->nbsp ? op->nbsp->name : op->nbrp->name);
+
+    struct ovn_port *crp = ovn_port_find(ports, redirect_name);
+    if (crp && crp->sb && crp->sb->datapath == op->od->sb) {
+        ovn_port_set_nb(crp, NULL, op->nbrp);
+        ovs_list_remove(&crp->list);
+        ovs_list_push_back(both_dbs, &crp->list);
+    } else {
+        crp = ovn_port_create(ports, redirect_name,
+                              op->nbsp, op->nbrp, NULL);
+        ovs_list_push_back(nb_only, &crp->list);
+    }
+
+    crp->primary_port = op;
+    op->cr_port = crp;
+    crp->od = op->od;
+    free(redirect_name);
+
+    return crp;
+}
+
+/* Returns true if chassis resident port needs to be created for
+ * op's peer logical switch.  False otherwise.
+ *
+ * Chassis resident port needs to be created if the op's logical router
+ *   - Has only one gateway router port and
+ *   - op's peer logical switch has no localnet ports and
+ *   - op's logical switch has the option 'overlay_provider_network'
+ *     set to true.
+ */
+static bool
+needs_cr_port_creation(struct ovn_port *op)
+{
+    if (op->od->n_l3dgw_ports == 1 && op->peer && op->peer->nbsp
+        && !op->peer->od->n_localnet_ports) {
+        return smap_get_bool(&op->peer->od->nbs->other_config,
+                             "overlay_provider_network", false);
+    }
+
+    return false;
+}
+
 static void
 join_logical_ports(const struct sbrec_port_binding_table *sbrec_pb_table,
                    struct hmap *ls_datapaths, struct hmap *lr_datapaths,
@@ -2205,9 +2252,10 @@ join_logical_ports(const struct sbrec_port_binding_table *sbrec_pb_table,
             tag_alloc_add_existing_tags(tag_alloc_table, nbsp);
         }
     }
+
+    struct hmapx gw_ports = HMAPX_INITIALIZER(&gw_ports);
     HMAP_FOR_EACH (od, key_node, lr_datapaths) {
         ovs_assert(od->nbr);
-        size_t n_allocated_l3dgw_ports = 0;
         for (size_t i = 0; i < od->nbr->n_ports; i++) {
             const struct nbrec_logical_router_port *nbrp
                 = od->nbr->ports[i];
@@ -2271,10 +2319,7 @@ join_logical_ports(const struct sbrec_port_binding_table *sbrec_pb_table,
                     redirect_type && !strcasecmp(redirect_type, "bridged");
             }
 
-            if (op->nbrp->ha_chassis_group ||
-                op->nbrp->n_gateway_chassis) {
-                /* Additional "derived" ovn_port crp represents the
-                 * instance of op on the gateway chassis. */
+            if (op->nbrp->ha_chassis_group || op->nbrp->n_gateway_chassis) {
                 const char *gw_chassis = smap_get(&op->od->nbr->options,
                                                "chassis");
                 if (gw_chassis) {
@@ -2283,34 +2328,9 @@ join_logical_ports(const struct sbrec_port_binding_table *sbrec_pb_table,
                     VLOG_WARN_RL(&rl, "Bad configuration: distributed "
                                  "gateway port configured on port %s "
                                  "on L3 gateway router", nbrp->name);
-                    continue;
-                }
-
-                char *redirect_name =
-                    ovn_chassis_redirect_name(nbrp->name);
-                struct ovn_port *crp = ovn_port_find(ports, redirect_name);
-                if (crp && crp->sb && crp->sb->datapath == od->sb) {
-                    ovn_port_set_nb(crp, NULL, nbrp);
-                    ovs_list_remove(&crp->list);
-                    ovs_list_push_back(both, &crp->list);
                 } else {
-                    crp = ovn_port_create(ports, redirect_name,
-                                          NULL, nbrp, NULL);
-                    ovs_list_push_back(nb_only, &crp->list);
-                }
-                crp->primary_port = op;
-                op->cr_port = crp;
-                crp->od = od;
-                free(redirect_name);
-
-                /* Add to l3dgw_ports in od, for later use during flow
-                 * creation. */
-                if (od->n_l3dgw_ports == n_allocated_l3dgw_ports) {
-                    od->l3dgw_ports = x2nrealloc(od->l3dgw_ports,
-                                                 &n_allocated_l3dgw_ports,
-                                                 sizeof *od->l3dgw_ports);
+                    hmapx_add(&gw_ports, op);
                 }
-                od->l3dgw_ports[od->n_l3dgw_ports++] = op;
            }
         }
     }
@@ -2367,12 +2387,6 @@ join_logical_ports(const struct sbrec_port_binding_table *sbrec_pb_table,
                         arp_proxy, op->nbsp->name);
                 }
             }
-
-            /* Only used for the router type LSP whose peer is l3dgw_port */
-            if (op->peer && is_l3dgw_port(op->peer)) {
-                op->enable_router_port_acl = smap_get_bool(
-                    &op->nbsp->options, "enable_router_port_acl", false);
-            }
         } else if (op->nbrp && op->nbrp->peer && !is_cr_port(op)) {
             struct ovn_port *peer = ovn_port_find(ports, op->nbrp->peer);
             if (peer) {
@@ -2393,6 +2407,56 @@ join_logical_ports(const struct sbrec_port_binding_table *sbrec_pb_table,
         }
     }
 
+    struct hmapx_node *hmapx_node;
+    HMAPX_FOR_EACH (hmapx_node, &gw_ports) {
+        op = hmapx_node->data;
+        od = op->od;
+        ovs_assert(op->nbrp);
+        ovs_assert(op->nbrp->ha_chassis_group || op->nbrp->n_gateway_chassis);
+
+        /* Additional "derived" ovn_port crp represents the instance of op on
+         * the gateway chassis. */
+        struct ovn_port *crp = create_cr_port(op, ports, both, nb_only);
+        ovs_assert(crp);
+
+        /* Add to l3dgw_ports in od, for later use during flow creation. */
+        if (od->n_l3dgw_ports == od->n_allocated_l3dgw_ports) {
+            od->l3dgw_ports = x2nrealloc(od->l3dgw_ports,
+                                        &od->n_allocated_l3dgw_ports,
+                                        sizeof *od->l3dgw_ports);
+        }
+        od->l3dgw_ports[od->n_l3dgw_ports++] = op;
+
+        if (op->peer && op->peer->nbsp) {
+            /* Only used for the router type LSP whose peer is l3dgw_port */
+            op->peer->enable_router_port_acl = smap_get_bool(
+                    &op->peer->nbsp->options, "enable_router_port_acl", false);
+        }
+    }
+
+
+    /* Create chassisresident port for the gateway router port's peer if
+     *  - Gateway router port's router has only one gateway router port and
+     *  - Its peer is a logical switch port and
+     *  - It's peer's logical switch has no localnet ports.
+     *  - Its peer's logical switch has the option overlay_provider_network
+     *    is set to true in the other_config column.
+     *
+     * This is required to support NAT via geneve (for the overlay provider
+     * networks) and the routing coming from this logical switch destined to
+     * the router port and vice versa is centralized on the gateway chassis.
+     *
+     * Future enhancement: Support NAT via geneve if the logical router has
+     * multiple gateway ports.
+     * */
+    HMAPX_FOR_EACH (hmapx_node, &gw_ports) {
+        op = hmapx_node->data;
+        if (needs_cr_port_creation(op)) {
+            create_cr_port(op->peer, ports, both, nb_only);
+        }
+    }
+    hmapx_destroy(&gw_ports);
+
     /* Wait until all ports have been connected to add to IPAM since
      * it relies on proper peers to be set
      */
@@ -3175,16 +3239,28 @@ ovn_port_update_sbrec(struct ovsdb_idl_txn *ovnsb_txn,
              * type "l3gateway". */
             if (chassis) {
                 sbrec_port_binding_set_type(op->sb, "l3gateway");
+            } else if (is_cr_port(op)) {
+                sbrec_port_binding_set_type(op->sb, "chassisredirect");
+                ovs_assert(op->primary_port->peer);
+                ovs_assert(op->primary_port->peer->cr_port);
+                ovs_assert(op->primary_port->peer->cr_port->sb);
+                sbrec_port_binding_set_ha_chassis_group(
+                    op->sb,
+                    op->primary_port->peer->cr_port->sb->ha_chassis_group);
+
             } else {
                 sbrec_port_binding_set_type(op->sb, "patch");
             }
 
             const char *router_port = smap_get(&op->nbsp->options,
                                                "router-port");
-            if (router_port || chassis) {
+            if (router_port || chassis || is_cr_port(op)) {
                 struct smap new;
                 smap_init(&new);
-                if (router_port) {
+
+                if (is_cr_port(op)) {
+                    smap_add(&new, "distributed-port", op->nbsp->name);
+                } else if (router_port) {
                     smap_add(&new, "peer", router_port);
                 }
                 if (chassis) {
@@ -8198,9 +8274,18 @@ build_lswitch_rport_arp_req_flow(
     struct lflow_ref *lflow_ref)
 {
     struct ds match   = DS_EMPTY_INITIALIZER;
+    struct ds m       = DS_EMPTY_INITIALIZER;
     struct ds actions = DS_EMPTY_INITIALIZER;
 
-    arp_nd_ns_match(ips, addr_family, &match);
+    arp_nd_ns_match(ips, addr_family, &m);
+    ds_clone(&match, &m);
+
+    bool has_cr_port = patch_op->cr_port;
+
+    if (has_cr_port) {
+        ds_put_format(&match, " && is_chassis_resident(%s)",
+                      patch_op->cr_port->json_key);
+    }
 
     /* Send a the packet to the router pipeline.  If the switch has non-router
      * ports then flood it there as well.
@@ -8222,6 +8307,31 @@ build_lswitch_rport_arp_req_flow(
                                 lflow_ref);
     }
 
+    if (has_cr_port) {
+        ds_clear(&match);
+        ds_put_format(&match, "%s && !is_chassis_resident(%s)", ds_cstr(&m),
+                      patch_op->cr_port->json_key);
+        ds_clear(&actions);
+        if (od->n_router_ports != od->nbs->n_ports) {
+            ds_put_format(&actions, "clone {outport = %s; output; }; "
+                                    "outport = \""MC_FLOOD_L2"\"; output;",
+                        patch_op->cr_port->json_key);
+            ovn_lflow_add_with_hint(lflows, od, S_SWITCH_IN_L2_LKUP,
+                                    priority, ds_cstr(&match),
+                                    ds_cstr(&actions), stage_hint,
+                                    lflow_ref);
+        } else {
+            ds_put_format(&actions, "outport = %s; output;",
+                          patch_op->cr_port->json_key);
+            ovn_lflow_add_with_hint(lflows, od, S_SWITCH_IN_L2_LKUP,
+                                    priority, ds_cstr(&match),
+                                    ds_cstr(&actions),
+                                    stage_hint,
+                                    lflow_ref);
+        }
+    }
+
+    ds_destroy(&m);
     ds_destroy(&match);
     ds_destroy(&actions);
 }
@@ -9594,7 +9704,11 @@ build_lswitch_ip_unicast_lookup(struct ovn_port *op,
                                 struct ds *actions, struct ds *match)
 {
     ovs_assert(op->nbsp);
-    if (lsp_is_external(op->nbsp)) {
+
+    /* Note: A switch port can also have a chassis resident derived port.
+     * Check if 'op' is a chassis resident dervied port. If so, skip
+     * adding unicast lookup flows for this port. */
+    if (lsp_is_external(op->nbsp) || is_cr_port(op)) {
         return;
     }
 
@@ -9612,8 +9726,6 @@ build_lswitch_ip_unicast_lookup(struct ovn_port *op,
                            "outport = \""MC_UNKNOWN "\"; output;"
                          : "outport = %s; output;")
                          : debug_drop_action();
-    ds_clear(actions);
-    ds_put_format(actions, action, op->json_key);
 
     if (lsp_is_router(op->nbsp) && op->peer && op->peer->nbrp) {
         /* For ports connected to logical routers add flows to bypass the
@@ -9660,14 +9772,35 @@ build_lswitch_ip_unicast_lookup(struct ovn_port *op,
             if (add_chassis_resident_check) {
                 ds_put_format(match, " && is_chassis_resident(%s)", json_key);
             }
+        } else if (op->cr_port) {
+            ds_clear(actions);
+            ds_put_format(actions, action, op->cr_port->json_key);
+
+            struct ds m = DS_EMPTY_INITIALIZER;
+            ds_put_format(&m, "eth.dst == %s && !is_chassis_resident(%s)",
+                          op->peer->lrp_networks.ea_s,
+                          op->cr_port->json_key);
+
+            ovn_lflow_add_with_hint(lflows, op->od,
+                                    S_SWITCH_IN_L2_LKUP, 50,
+                                    ds_cstr(&m), ds_cstr(actions),
+                                    &op->nbsp->header_,
+                                    op->lflow_ref);
+            ds_destroy(&m);
+            ds_put_format(match, " && is_chassis_resident(%s)",
+                          op->cr_port->json_key);
         }
 
+        ds_clear(actions);
+        ds_put_format(actions, action, op->json_key);
         ovn_lflow_add_with_hint(lflows, op->od,
                                 S_SWITCH_IN_L2_LKUP, 50,
                                 ds_cstr(match), ds_cstr(actions),
                                 &op->nbsp->header_,
                                 op->lflow_ref);
     } else {
+        ds_clear(actions);
+        ds_put_format(actions, action, op->json_key);
         for (size_t i = 0; i < op->n_lsp_addrs; i++) {
             ds_clear(match);
             ds_put_format(match, "eth.dst == %s", op->lsp_addrs[i].ea_s);
@@ -11765,6 +11898,15 @@ build_lrouter_port_nat_arp_nd_flow(struct ovn_port *op,
         return;
     }
 
+    if (op->peer && op->peer->cr_port) {
+        /* We don't add the below flows if the router port's peer has
+         * a chassisresident port.  That's because routing is centralized on
+         * the gateway chassis for the traffic from the peer port to this
+         * router or from this router to the peer logical switch.
+         */
+        return;
+    }
+
     /* Mac address to use when replying to ARP/NS. */
     const char *mac_s = REG_INPORT_ETH_ADDR;
     struct eth_addr mac;
@@ -15154,6 +15296,17 @@ lrouter_check_nat_entry(const struct ovn_datapath *od,
     /* For distributed router NAT, determine whether this NAT rule
      * satisfies the conditions for distributed NAT processing. */
     *distributed = false;
+
+    /* NAT cannnot be distributed if the gateway port's peer
+     * has a chassisresident port (and the routing is centralized
+     * on the gateway chassis for the traffic from the peer
+     * to this router and traffic to the peer.)
+     */
+    struct ovn_port *l3dgw_port = *nat_l3dgw_port;
+    if (l3dgw_port && l3dgw_port->peer && l3dgw_port->peer->cr_port) {
+        return 0;
+    }
+
     if (od->n_l3dgw_ports && !strcmp(nat->type, "dnat_and_snat") &&
         nat->logical_port && nat->external_mac) {
         if (eth_addr_from_string(nat->external_mac, mac)) {

northd/northd.h

@@ -325,6 +325,7 @@ struct ovn_datapath {
      * will be NULL. */
     struct ovn_port **l3dgw_ports;
     size_t n_l3dgw_ports;
+    size_t n_allocated_l3dgw_ports;
 
     /* router datapath has a logical port with redirect-type set to bridged. */
     bool redirect_bridged;