ovn-controller: Add support for load balancing.

ovn-controller now supports 2 new logical actions.

1. ct_lb;
Sends the packet through the conntrack zone to NAT
packets. Packets that are part of established connection
will automatically get NATed based on the NAT arguments
supplied to conntrack when the first packet was committed.

2. ct_lb(192.168.1.2, 192.168.1.3);
   ct_lb(192.168.1.2:80, 192.168.1.3:80);
Creates an OpenFlow group with multiple buckets and equal weights
that changes the destination IP address (and port number) of the packet
statefully to one of the options provided inside the parenthesis.

Signed-off-by: Gurucharan Shetty <guru@ovn.org>
Signed-off-by: Ben Pfaff <blp@ovn.org>

ovn/controller/lflow.c

@@ -319,6 +319,7 @@ static void consider_logical_flow(const struct lport_index *lports,
                                   const struct sbrec_logical_flow *lflow,
                                   const struct hmap *local_datapaths,
                                   const struct hmap *patched_datapaths,
+                                  struct group_table *group_table,
                                   const struct simap *ct_zones,
                                   struct hmap *dhcp_opts_p,
                                   uint32_t *conj_id_ofs_p,
@@ -359,6 +360,7 @@ add_logical_flows(struct controller_ctx *ctx, const struct lport_index *lports,
                   const struct mcgroup_index *mcgroups,
                   const struct hmap *local_datapaths,
                   const struct hmap *patched_datapaths,
+                  struct group_table *group_table,
                   const struct simap *ct_zones, struct hmap *flow_table)
 {
     uint32_t conj_id_ofs = 1;
@@ -373,7 +375,7 @@ add_logical_flows(struct controller_ctx *ctx, const struct lport_index *lports,
     const struct sbrec_logical_flow *lflow;
     SBREC_LOGICAL_FLOW_FOR_EACH (lflow, ctx->ovnsb_idl) {
         consider_logical_flow(lports, mcgroups, lflow, local_datapaths,
-                              patched_datapaths, ct_zones,
+                              patched_datapaths, group_table, ct_zones,
                               &dhcp_opts, &conj_id_ofs, flow_table);
     }
 
@@ -386,6 +388,7 @@ consider_logical_flow(const struct lport_index *lports,
                       const struct sbrec_logical_flow *lflow,
                       const struct hmap *local_datapaths,
                       const struct hmap *patched_datapaths,
+                      struct group_table *group_table,
                       const struct simap *ct_zones,
                       struct hmap *dhcp_opts_p,
                       uint32_t *conj_id_ofs_p,
@@ -464,6 +467,7 @@ consider_logical_flow(const struct lport_index *lports,
         .lookup_port = lookup_port_cb,
         .aux = &aux,
         .ct_zones = ct_zones,
+        .group_table = group_table,
 
         .n_tables = LOG_PIPELINE_LEN,
         .first_ptable = first_ptable,
@@ -622,11 +626,12 @@ lflow_run(struct controller_ctx *ctx, const struct lport_index *lports,
           const struct mcgroup_index *mcgroups,
           const struct hmap *local_datapaths,
           const struct hmap *patched_datapaths,
+          struct group_table *group_table,
           const struct simap *ct_zones, struct hmap *flow_table)
 {
     update_address_sets(ctx);
     add_logical_flows(ctx, lports, mcgroups, local_datapaths,
-                      patched_datapaths, ct_zones, flow_table);
+                      patched_datapaths, group_table, ct_zones, flow_table);
     add_neighbor_flows(ctx, lports, flow_table);
 }
 

ovn/controller/lflow.h

@@ -36,6 +36,7 @@
 #include <stdint.h>
 
 struct controller_ctx;
+struct group_table;
 struct hmap;
 struct lport_index;
 struct mcgroup_index;
@@ -63,6 +64,7 @@ void lflow_run(struct controller_ctx *, const struct lport_index *,
                const struct mcgroup_index *,
                const struct hmap *local_datapaths,
                const struct hmap *patched_datapaths,
+               struct group_table *group_table,
                const struct simap *ct_zones,
                struct hmap *flow_table);
 void lflow_destroy(void);

ovn/controller/ofctrl.c

@@ -1,4 +1,4 @@
-/* Copyright (c) 2015 Nicira, Inc.
+/* Copyright (c) 2015, 2016 Nicira, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -14,6 +14,7 @@
  */
 
 #include <config.h>
+#include "bitmap.h"
 #include "byte-order.h"
 #include "dirs.h"
 #include "hash.h"
@@ -24,11 +25,13 @@
 #include "openvswitch/match.h"
 #include "openvswitch/ofp-actions.h"
 #include "openvswitch/ofp-msgs.h"
+#include "openvswitch/ofp-parse.h"
 #include "openvswitch/ofp-print.h"
 #include "openvswitch/ofp-util.h"
 #include "openvswitch/ofpbuf.h"
 #include "openvswitch/vlog.h"
 #include "ovn-controller.h"
+#include "ovn/lib/actions.h"
 #include "physical.h"
 #include "rconn.h"
 #include "socket-util.h"
@@ -63,6 +66,8 @@ static void queue_flow_mod(struct ofputil_flow_mod *);
 /* OpenFlow connection to the switch. */
 static struct rconn *swconn;
 
+static void queue_group_mod(struct ofputil_group_mod *);
+
 /* Last seen sequence number for 'swconn'.  When this differs from
  * rconn_get_connection_seqno(rconn), 'swconn' has reconnected. */
 static unsigned int seqno;
@@ -95,6 +100,9 @@ static struct rconn_packet_counter *tx_counter;
  * installed in the switch. */
 static struct hmap installed_flows;
 
+/* A reference to the group_table. */
+static struct group_table *groups;
+
 /* MFF_* field ID for our Geneve option.  In S_TLV_TABLE_MOD_SENT, this is
  * the option we requested (we don't know whether we obtained it yet).  In
  * S_CLEAR_FLOWS or S_UPDATE_FLOWS, this is really the option we have. */
@@ -103,6 +111,9 @@ static enum mf_field_id mff_ovn_geneve;
 static void ovn_flow_table_clear(struct hmap *flow_table);
 static void ovn_flow_table_destroy(struct hmap *flow_table);
 
+static void ovn_group_table_clear(struct group_table *group_table,
+                                  bool existing);
+
 static void ofctrl_recv(const struct ofp_header *, enum ofptype);
 
 void
@@ -312,9 +323,23 @@ run_S_CLEAR_FLOWS(void)
     queue_flow_mod(&fm);
     VLOG_DBG("clearing all flows");
 
+    struct ofputil_group_mod gm;
+    memset(&gm, 0, sizeof gm);
+    gm.command = OFPGC11_DELETE;
+    gm.group_id = OFPG_ALL;
+    gm.command_bucket_id = OFPG15_BUCKET_ALL;
+    ovs_list_init(&gm.buckets);
+    queue_group_mod(&gm);
+    ofputil_bucket_list_destroy(&gm.buckets);
+
     /* Clear installed_flows, to match the state of the switch. */
     ovn_flow_table_clear(&installed_flows);
 
+    /* Clear existing groups, to match the state of the switch. */
+    if (groups) {
+        ovn_group_table_clear(groups, true);
+    }
+
     state = S_UPDATE_FLOWS;
 }
 
@@ -591,16 +616,70 @@ queue_flow_mod(struct ofputil_flow_mod *fm)
     queue_msg(ofputil_encode_flow_mod(fm, OFPUTIL_P_OF13_OXM));
 }
 
+
+/* group_table. */
+
+/* Finds and returns a group_info in 'existing_groups' whose key is identical
+ * to 'target''s key, or NULL if there is none. */
+static struct group_info *
+ovn_group_lookup(struct hmap *exisiting_groups,
+                 const struct group_info *target)
+{
+    struct group_info *e;
+
+    HMAP_FOR_EACH_WITH_HASH(e, hmap_node, target->hmap_node.hash,
+                            exisiting_groups) {
+        if (e->group_id == target->group_id) {
+            return e;
+        }
+   }
+    return NULL;
+}
+
+/* Clear either desired_groups or existing_groups in group_table. */
+static void
+ovn_group_table_clear(struct group_table *group_table, bool existing)
+{
+    struct group_info *g, *next;
+    struct hmap *target_group = existing
+                                ? &group_table->existing_groups
+                                : &group_table->desired_groups;
+
+    HMAP_FOR_EACH_SAFE (g, next, hmap_node, target_group) {
+        hmap_remove(target_group, &g->hmap_node);
+        bitmap_set0(group_table->group_ids, g->group_id);
+        ds_destroy(&g->group);
+        free(g);
+    }
+}
+
+static void
+queue_group_mod(struct ofputil_group_mod *gm)
+{
+    queue_msg(ofputil_encode_group_mod(OFP13_VERSION, gm));
+}
+
+
 /* Replaces the flow table on the switch, if possible, by the flows in
  * 'flow_table', which should have been added with ofctrl_add_flow().
  * Regardless of whether the flow table is updated, this deletes all of the
  * flows from 'flow_table' and frees them.  (The hmap itself isn't
  * destroyed.)
  *
+ * Replaces the group table on the switch, if possible, by the groups in
+ * 'group_table->desired_groups'. Regardless of whether the group table
+ * is updated, this deletes all the groups from the
+ * 'group_table->desired_groups' and frees them. (The hmap itself isn't
+ * destroyed.)
+ *
  * This called be called be ofctrl_run() within the main loop. */
 void
-ofctrl_put(struct hmap *flow_table)
+ofctrl_put(struct hmap *flow_table, struct group_table *group_table)
 {
+    if (!groups) {
+        groups = group_table;
+    }
+
     /* The flow table can be updated if the connection to the switch is up and
      * in the correct state and not backlogged with existing flow_mods.  (Our
      * criteria for being backlogged appear very conservative, but the socket
@@ -610,9 +689,39 @@ ofctrl_put(struct hmap *flow_table)
     if (state != S_UPDATE_FLOWS
         || rconn_packet_counter_n_packets(tx_counter)) {
         ovn_flow_table_clear(flow_table);
+        ovn_group_table_clear(group_table, false);
         return;
     }
 
+    /* Iterate through all the desired groups. If there are new ones,
+     * add them to the switch. */
+    struct group_info *desired;
+    HMAP_FOR_EACH(desired, hmap_node, &group_table->desired_groups) {
+        if (!ovn_group_lookup(&group_table->existing_groups, desired)) {
+            /* Create and install new group. */
+            struct ofputil_group_mod gm;
+            enum ofputil_protocol usable_protocols;
+            char *error;
+            struct ds group_string = DS_EMPTY_INITIALIZER;
+            ds_put_format(&group_string, "group_id=%u,%s",
+                          desired->group_id, ds_cstr(&desired->group));
+
+            error = parse_ofp_group_mod_str(&gm, OFPGC11_ADD,
+                                            ds_cstr(&group_string),
+                                            &usable_protocols);
+            if (!error) {
+                queue_group_mod(&gm);
+            } else {
+                static struct vlog_rate_limit rl = VLOG_RATE_LIMIT_INIT(5, 1);
+                VLOG_ERR_RL(&rl, "new group %s %s", error,
+                         ds_cstr(&group_string));
+                free(error);
+            }
+            ds_destroy(&group_string);
+            ofputil_bucket_list_destroy(&gm.buckets);
+        }
+    }
+
     /* Iterate through all of the installed flows.  If any of them are no
      * longer desired, delete them; if any of them should have different
      * actions, update them. */
@@ -682,4 +791,54 @@ ofctrl_put(struct hmap *flow_table)
         hmap_remove(flow_table, &d->hmap_node);
         hmap_insert(&installed_flows, &d->hmap_node, d->hmap_node.hash);
     }
+
+    /* Iterate through the installed groups from previous runs. If they
+     * are not needed delete them. */
+    struct group_info *installed, *next_group;
+    HMAP_FOR_EACH_SAFE(installed, next_group, hmap_node,
+                       &group_table->existing_groups) {
+        if (!ovn_group_lookup(&group_table->desired_groups, installed)) {
+            /* Delete the group. */
+            struct ofputil_group_mod gm;
+            enum ofputil_protocol usable_protocols;
+            char *error;
+            struct ds group_string = DS_EMPTY_INITIALIZER;
+            ds_put_format(&group_string, "group_id=%u", installed->group_id);
+
+            error = parse_ofp_group_mod_str(&gm, OFPGC11_DELETE,
+                                            ds_cstr(&group_string),
+                                            &usable_protocols);
+            if (!error) {
+                queue_group_mod(&gm);
+            } else {
+                static struct vlog_rate_limit rl = VLOG_RATE_LIMIT_INIT(5, 1);
+                VLOG_ERR_RL(&rl, "Error deleting group %d: %s",
+                         installed->group_id, error);
+                free(error);
+            }
+            ds_destroy(&group_string);
+            ofputil_bucket_list_destroy(&gm.buckets);
+
+            /* Remove 'installed' from 'group_table->existing_groups' */
+            hmap_remove(&group_table->existing_groups, &installed->hmap_node);
+            ds_destroy(&installed->group);
+
+            /* Dealloc group_id. */
+            bitmap_set0(group_table->group_ids, installed->group_id);
+            free(installed);
+        }
+    }
+
+    /* Move the contents of desired_groups to existing_groups. */
+    HMAP_FOR_EACH_SAFE(desired, next_group, hmap_node,
+                       &group_table->desired_groups) {
+        hmap_remove(&group_table->desired_groups, &desired->hmap_node);
+        if (!ovn_group_lookup(&group_table->existing_groups, desired)) {
+            hmap_insert(&group_table->existing_groups, &desired->hmap_node,
+                        desired->hmap_node.hash);
+        } else {
+            ds_destroy(&desired->group);
+            free(desired);
+        }
+    }
 }

ovn/controller/ofctrl.h

@@ -26,11 +26,12 @@ struct hmap;
 struct match;
 struct ofpbuf;
 struct ovsrec_bridge;
+struct group_table;
 
 /* Interface for OVN main loop. */
 void ofctrl_init(void);
 enum mf_field_id ofctrl_run(const struct ovsrec_bridge *br_int);
-void ofctrl_put(struct hmap *flows);
+void ofctrl_put(struct hmap *flows, struct group_table *group_table);
 void ofctrl_wait(void);
 void ofctrl_destroy(void);
 

ovn/controller/ovn-controller.c

@@ -39,6 +39,7 @@
 #include "ofctrl.h"
 #include "openvswitch/vconn.h"
 #include "openvswitch/vlog.h"
+#include "ovn/lib/actions.h"
 #include "ovn/lib/ovn-sb-idl.h"
 #include "ovn/lib/ovn-util.h"
 #include "patch.h"
@@ -327,6 +328,13 @@ main(int argc, char *argv[])
     }
     unixctl_command_register("exit", "", 0, 0, ovn_controller_exit, &exiting);
 
+    /* Initialize group ids for loadbalancing. */
+    struct group_table group_table;
+    group_table.group_ids = bitmap_allocate(MAX_OVN_GROUPS);
+    bitmap_set1(group_table.group_ids, 0); /* Group id 0 is invalid. */
+    hmap_init(&group_table.desired_groups);
+    hmap_init(&group_table.existing_groups);
+
     daemonize_complete();
 
     ovsrec_init();
@@ -435,13 +443,14 @@ main(int argc, char *argv[])
 
             struct hmap flow_table = HMAP_INITIALIZER(&flow_table);
             lflow_run(&ctx, &lports, &mcgroups, &local_datapaths,
-                      &patched_datapaths, &ct_zones, &flow_table);
+                      &patched_datapaths, &group_table, &ct_zones,
+                      &flow_table);
             if (chassis_id) {
                 physical_run(&ctx, mff_ovn_geneve,
                              br_int, chassis_id, &ct_zones, &flow_table,
                              &local_datapaths, &patched_datapaths);
             }
-            ofctrl_put(&flow_table);
+            ofctrl_put(&flow_table, &group_table);
             hmap_destroy(&flow_table);
         }
 
@@ -501,6 +510,18 @@ main(int argc, char *argv[])
 
     simap_destroy(&ct_zones);
 
+    bitmap_free(group_table.group_ids);
+    hmap_destroy(&group_table.desired_groups);
+
+    struct group_info *installed, *next_group;
+    HMAP_FOR_EACH_SAFE(installed, next_group, hmap_node,
+                       &group_table.existing_groups) {
+        hmap_remove(&group_table.existing_groups, &installed->hmap_node);
+        ds_destroy(&installed->group);
+        free(installed);
+    }
+    hmap_destroy(&group_table.existing_groups);
+
     ovsdb_idl_loop_destroy(&ovs_idl_loop);
     ovsdb_idl_loop_destroy(&ovnsb_idl_loop);