provider networks: Provide the option to tunnel traffic.

This patch adds a global config option - 'provider_network_overlay' and
when set to true, any traffic destined to a VIF logical port of a
provider logical switch (having localnet port(s)), is tunnelled to
the destination chassis, instead of sending it out via the localnet
port.  This feature is useful for the following reasons:

1.  CMS can add both provider logical switches and overlay logical
    swithes to a logical router.  With this option set, E-W routing between
    these logical switches will be tunnelled all the time.  The router port
    mac addresses are not leaked from multiple chassis to the upstream
    switches anymore.

2.  NATting will work as expected either in the gateway chassis or on
    the source VIF chassis (if external_mac and logical_port set).

3.  With this option set, there is no need to centralize routing
    for provider logical switches ('reside-on-redirect-chassis').

4.  With the commits [1] now merged, MTU issues arising due to tunnel
    overhead will be handled gracefully.

[1] - 3faadc7 ("northd: Fix pmtud for non routed traffic.")
      221476a ("ovn: Add tunnel PMTUD support.")

Reported-at: https://issues.redhat.com/browse/FDP-209
Signed-off-by: Numan Siddique <numans@ovn.org>

controller/ovn-controller.c

@@ -3719,6 +3719,11 @@ non_vif_data_ovs_iface_handler(struct engine_node *node, void *data OVS_UNUSED)
 struct ed_type_northd_options {
     bool lb_hairpin_use_ct_mark;
     bool explicit_arp_ns_output;
+    bool provider_network_overlay; /* Indicates if the traffic to the
+                                    * logical port of a bridged logical
+                                    * switch (i.e with localnet port) should
+                                    * be tunnelled or sent via the localnet
+                                    * port.  Default value is 'false'. */
 };
 
 
@@ -3756,6 +3761,12 @@ en_northd_options_run(struct engine_node *node, void *data)
                             false)
             : false;
 
+    n_opts->provider_network_overlay =
+            sb_global
+            ? smap_get_bool(&sb_global->options, "provider_network_overlay",
+                            false)
+            : false;
+
     engine_set_node_state(node, EN_UPDATED);
 }
 
@@ -3790,6 +3801,17 @@ en_northd_options_sb_sb_global_handler(struct engine_node *node, void *data)
         engine_set_node_state(node, EN_UPDATED);
     }
 
+    bool provider_network_overlay =
+            sb_global
+            ? smap_get_bool(&sb_global->options, "provider_network_overlay",
+                            false)
+            : false;
+
+    if (provider_network_overlay != n_opts->provider_network_overlay) {
+        n_opts->provider_network_overlay = provider_network_overlay;
+        engine_set_node_state(node, EN_UPDATED);
+    }
+
     return true;
 }
 
@@ -4691,6 +4713,9 @@ static void init_physical_ctx(struct engine_node *node,
         engine_get_input_data("ct_zones", node);
     struct simap *ct_zones = &ct_zones_data->current;
 
+    struct ed_type_northd_options *n_opts =
+        engine_get_input_data("northd_options", node);
+
     parse_encap_ips(ovs_table, &p_ctx->n_encap_ips, &p_ctx->encap_ips);
     p_ctx->sbrec_port_binding_by_name = sbrec_port_binding_by_name;
     p_ctx->sbrec_port_binding_by_datapath = sbrec_port_binding_by_datapath;
@@ -4708,6 +4733,7 @@ static void init_physical_ctx(struct engine_node *node,
     p_ctx->local_bindings = &rt_data->lbinding_data.bindings;
     p_ctx->patch_ofports = &non_vif_data->patch_ofports;
     p_ctx->chassis_tunnels = &non_vif_data->chassis_tunnels;
+    p_ctx->provider_network_overlay = n_opts->provider_network_overlay;
 
     struct controller_engine_ctx *ctrl_ctx = engine_get_context()->client_ctx;
     p_ctx->if_mgr = ctrl_ctx->if_mgr;
@@ -5376,6 +5402,7 @@ main(int argc, char *argv[])
      */
     engine_add_input(&en_pflow_output, &en_non_vif_data,
                      NULL);
+    engine_add_input(&en_pflow_output, &en_northd_options, NULL);
     engine_add_input(&en_pflow_output, &en_ct_zones,
                      pflow_output_ct_zones_handler);
     engine_add_input(&en_pflow_output, &en_sb_chassis,

controller/physical.c

@@ -1488,6 +1488,7 @@ consider_port_binding(struct ovsdb_idl_index *sbrec_port_binding_by_name,
                       const struct if_status_mgr *if_mgr,
                       size_t n_encap_ips,
                       const char **encap_ips,
+                      bool provider_network_overlay,
                       struct ovn_desired_flow_table *flow_table,
                       struct ofpbuf *ofpacts_p)
 {
@@ -1921,14 +1922,19 @@ consider_port_binding(struct ovsdb_idl_index *sbrec_port_binding_by_name,
                             binding->header_.uuid.parts[0], &match,
                             ofpacts_p, &binding->header_.uuid);
         }
-    } else if (access_type == PORT_LOCALNET) {
+    } else if (access_type == PORT_LOCALNET && !provider_network_overlay) {
         /* Remote port connected by localnet port */
         /* Table 40, priority 100.
          * =======================
          *
          * Implements switching to localnet port. Each flow matches a
          * logical output port on remote hypervisor, switch the output port
          * to connected localnet port and resubmits to same table.
+         *
+         * Note: If 'provider_network_overlay' is true, then
+         * put_remote_port_redirect_overlay() called from below takes care
+         * of adding the flow in OFTABLE_REMOTE_OUTPUT table to tunnel to
+         * the destination chassis.
          */
 
         ofpbuf_clear(ofpacts_p);
@@ -2354,6 +2360,7 @@ physical_eval_port_binding(struct physical_ctx *p_ctx,
                           p_ctx->if_mgr,
                           p_ctx->n_encap_ips,
                           p_ctx->encap_ips,
+                          p_ctx->provider_network_overlay,
                           flow_table, &ofpacts);
     ofpbuf_uninit(&ofpacts);
 }
@@ -2481,6 +2488,7 @@ physical_run(struct physical_ctx *p_ctx,
                               p_ctx->if_mgr,
                               p_ctx->n_encap_ips,
                               p_ctx->encap_ips,
+                              p_ctx->provider_network_overlay,
                               flow_table, &ofpacts);
     }
 

controller/physical.h

@@ -69,6 +69,7 @@ struct physical_ctx {
     size_t n_encap_ips;
     const char **encap_ips;
     struct physical_debug debug;
+    bool provider_network_overlay;
 };
 
 void physical_register_ovs_idl(struct ovsdb_idl *);

northd/en-global-config.c

@@ -533,6 +533,11 @@ check_nb_options_out_of_sync(const struct nbrec_nb_global *nb,
         return true;
     }
 
+    if (config_out_of_sync(&nb->options, &config_data->nb_options,
+                           "provider_network_overlay", false)) {
+        return true;
+    }
+
     return false;
 }
 

ovn-nb.xml

@@ -381,6 +381,22 @@
         of SB changes would be very noticeable.
       </column>
 
+      <column name="options" key="provider_network_overlay"
+           type='{"type": "boolean"}'>
+        <p>
+          If set to true, then the traffic destined to a VIF of a provider
+          logical switch (having a localnet port) will be tunnelled instead
+          of sending it via the localnet port.  This option will be useful
+          if CMS wants to connect overlay logical switches (without
+          localnet port) and provider logical switches to a router.  Without
+          this option set, the traffic path will be a mix of tunnelling and
+          localnet ports (since routing is distributed) resulting in the
+          leakage of the router port mac address to the upstream switches
+          and undefined behavior if NATting is involed.  This option is
+          disabled by default.
+        </p>
+      </column>
+
       <group title="Options for configuring interconnection route advertisement">
         <p>
           These options control how routes are advertised between OVN

tests/multinode-macros.at

@@ -22,6 +22,25 @@ m4_define([M_NS_CHECK_EXEC],
     [ AT_CHECK([M_NS_EXEC([$1], [$2], [$3])], m4_shift(m4_shift(m4_shift($@)))) ]
 )
 
+# M_DAEMONIZE([fake_node],[command],[pidfile])
+m4_define([M_DAEMONIZE],
+    [podman exec $1 $2 & echo $! > $3
+     echo "kill \`cat $3\`" >> cleanup
+    ]
+)
+
+# M_START_TCPDUMP([fake_node], [params], [name])
+#
+# Helper to properly start tcpdump and wait for the startup.
+# The tcpdump output is available in <name>.tcpdump file.
+m4_define([M_START_TCPDUMP],
+    [
+     podman exec $1 tcpdump -l $2 >$3.tcpdump 2>$3.stderr &
+     OVS_WAIT_UNTIL([grep -q "listening" $3.stderr])
+    ]
+)
+
+
 OVS_START_SHELL_HELPERS
 
 m_as() {