Include hardware offload hooks

This changeset provides a generic infrastructure for configuring hardware
offloads for vrouter. This defines function prototypes for callbacks which will
be registered by a vrouter offload module, which will be called at certain
points in the vrouter.ko processing (mainly around where it does configurations
of tables via sandesh messages).

Offloaded functionality includes flow, vif, MPLS, VXLAN, mirror, nexthop, layer2
and route tables. QoS API is defined but not currently supported by offload
module (offload module will return an error if configured).

Change-Id: I00119ec244bf0c3bc8fa218aae10dd1b164b47ce
Closes-Bug: #1767107
Signed-off-by: Frik Botha <frederick.botha@netronome.com>

Makefile

@@ -53,6 +53,7 @@ ifneq ($(KERNELRELEASE), )
 	vrouter-y += linux/vrouter_mod.o linux/vhost_dev.o
 	vrouter-y += linux/vr_host_interface.o linux/vr_genetlink.o
 	vrouter-y += linux/vr_mem.o linux/vr_fragment_assembler.o
+	vrouter-y += linux/vr_offloads.o
 
 	vrouter-y += dp-core/vr_message.o dp-core/vr_sandesh.o
 	vrouter-y += dp-core/vr_queue.o dp-core/vr_index_table.o

dp-core/vr_flow.c

@@ -20,6 +20,8 @@
 #include "vr_ip_mtrie.h"
 #include "vr_bridge.h"
 
+#include "vr_offloads.h"
+
 #define VR_NUM_FLOW_TABLES          1
 
 #define VR_NUM_OFLOW_TABLES         1
@@ -182,6 +184,8 @@ vr_flow_reset_mirror(struct vrouter *router, struct vr_flow_entry *fe,
         fe->fe_sec_mirror_id = VR_MAX_MIRROR_INDICES;
         if (fe->fe_mme) {
             vr_mirror_meta_entry_del(router, fe->fe_mme);
+            /* XXX: Index corruption may result after eviction - see 505bafc1 */
+            (void)vr_offload_flow_meta_data_set(index, 0, 0, 0);
             fe->fe_mme = NULL;
         }
     }
@@ -214,6 +218,8 @@ __vr_flow_reset_entry(struct vrouter *router, struct vr_flow_entry *fe)
     fe->fe_hold_list = NULL;
     fe->fe_key.flow_key_len = 0;
 
+    (void)vr_offload_flow_del(fe);
+
     vr_flow_reset_mirror(router, fe, fe->fe_hentry.hentry_index);
     fe->fe_ecmp_nh_index = -1;
     fe->fe_src_nh_index = NH_DISCARD_ID;
@@ -1589,6 +1595,41 @@ vr_flow_fat_flow_lookup(struct vrouter *router, struct vr_packet *pkt,
     return vif_fat_flow_lookup(vif_l, l4_proto, sport, dport);
 }
 
+/*
+ * Called by offload module to update flow stats with packets which have been
+ * offloaded.  over_flow_bytes accounts for overflows which happen in firmware
+ * between updates using this function.
+ */
+int
+vr_flow_incr_stats(int fe_index, uint32_t flow_bytes, uint16_t over_flow_bytes,
+                   uint32_t flow_packets)
+{
+    struct vrouter *router = vrouter_get(0);
+    struct vr_flow_entry *fe;
+    uint32_t new_stats;
+
+    if (router == NULL)
+        return -EINVAL;
+
+    fe = vr_flow_get_entry(router, fe_index);
+    if (fe == NULL)
+        return -ENOENT;
+
+    if (!(fe->fe_flags & VR_FLOW_FLAG_ACTIVE))
+        return -ENOENT;
+
+    new_stats = __sync_add_and_fetch(&fe->fe_stats.flow_bytes, flow_bytes);
+    if (new_stats < flow_bytes)
+        ++fe->fe_stats.flow_bytes_oflow;
+    fe->fe_stats.flow_bytes_oflow += over_flow_bytes;
+
+    new_stats = __sync_add_and_fetch(&fe->fe_stats.flow_packets, flow_packets);
+    if (new_stats < flow_packets)
+        ++fe->fe_stats.flow_packets_oflow;
+
+    return 0;
+}
+
 static flow_result_t
 vr_do_flow_lookup(struct vrouter *router, struct vr_packet *pkt,
                 struct vr_forwarding_md *fmd)
@@ -1838,6 +1879,11 @@ vr_flow_set_mirror(struct vrouter *router, vr_flow_req *req,
                 req->fr_mir_sip, req->fr_mir_sport,
                 req->fr_pcap_meta_data, req->fr_pcap_meta_data_size,
                 req->fr_mir_vrf);
+        if (fe->fe_mme)
+            (void)vr_offload_flow_meta_data_set(req->fr_index,
+                                                req->fr_pcap_meta_data_size,
+                                                req->fr_pcap_meta_data,
+                                                req->fr_mir_vrf);
     }
 
     return;
@@ -2183,6 +2229,7 @@ vr_flow_set(struct vrouter *router, vr_flow_req *req,
     if (fe) {
         if (!(modified = vr_flow_start_modify(router, fe)))
             return -EBUSY;
+        fe_index = (unsigned int)(req->fr_index);
     }
 
     if ((ret = vr_flow_set_req_is_invalid(router, req, fe)))
@@ -2319,6 +2366,14 @@ vr_flow_set(struct vrouter *router, vr_flow_req *req,
 
     ret = vr_flow_schedule_transition(router, req, fe);
 
+    /*
+     * offload, no need to differentiate between add and modify. Pass the
+     * reverse flow as well if present.
+     */
+    if (!ret) {
+        (void)vr_offload_flow_set(fe, fe_index, rfe);
+    }
+
 exit_set:
     if (modified && fe) {
         vr_flow_stop_modify(router, fe);

dp-core/vr_interface.c

@@ -16,6 +16,7 @@
 #include "vr_btable.h"
 
 unsigned int vr_interfaces = VR_MAX_INTERFACES;
+#include "vr_offloads.h"
 
 volatile bool agent_alive = false;
 
@@ -1938,6 +1939,8 @@ vr_interface_delete(vr_interface_req *req, bool need_response)
     if (!vif && (ret = -ENODEV))
         goto del_fail;
 
+    (void)vr_offload_interface_del(vif);
+
     vif_delete(vif);
 
 del_fail:
@@ -2173,6 +2176,10 @@ vr_interface_add(vr_interface_req *req, bool need_response)
     vif = __vrouter_get_interface(router, req->vifr_idx);
     if (vif) {
         ret = vr_interface_change(vif, req);
+        /* notify hw offload of change, if enabled */
+        if (!ret)
+            ret = vr_offload_interface_add(vif);
+
         goto generate_resp;
     }
 
@@ -2314,6 +2321,15 @@ vr_interface_add(vr_interface_req *req, bool need_response)
     if (!ret)
         vrouter_setup_vif(vif);
 
+    /* notify hw offload of change, if enabled */
+    if (!ret) {
+        ret = vr_offload_interface_add(vif);
+        if (ret) {
+            vif_delete(vif);
+            vif = NULL;
+        }
+    }
+
 error:
     if (ret && vif)
         vif_free(vif);
@@ -2406,6 +2422,7 @@ __vr_interface_make_req(vr_interface_req *req, struct vr_interface *intf,
     req->vifr_transport = intf->vif_transport;
     req->vifr_os_idx = intf->vif_os_idx;
     req->vifr_mtu = intf->vif_mtu;
+    req->vifr_nh_id = intf->vif_nh_id;
     if (req->vifr_mac_size && req->vifr_mac)
         memcpy(req->vifr_mac, intf->vif_mac,
                 MINIMUM(req->vifr_mac_size, sizeof(intf->vif_mac)));
@@ -2774,6 +2791,10 @@ vr_interface_get(vr_interface_req *req)
     if (ret < 0)
         goto generate_response;
 
+    /* adds in stats for pkts which were offloaded on NIC and does debug
+       comparison to check if matching entry is programmed on NIC */
+    (void)vr_offload_interface_get(vif_resp);
+
     mm.vr_mm_object_type[obj_cnt] = VR_INTERFACE_OBJECT_ID;
     mm.vr_mm_object[obj_cnt] = vif_resp;
     obj_cnt++;
@@ -2850,6 +2871,10 @@ vr_interface_dump(vr_interface_req *r)
         if (vif) {
             /* zero vifr_core means to sum up all the per-core stats */
             vr_interface_make_req(resp, vif, (unsigned)(r->vifr_core - 1));
+
+            /* let hw offload fill in relevant fields */
+            (void)vr_offload_interface_get(resp);
+
             ret = vr_message_dump_object(dumper, VR_INTERFACE_OBJECT_ID, resp);
             if (ret <= 0)
                 break;

dp-core/vr_mirror.c

@@ -9,6 +9,7 @@
 #include "vr_sandesh.h"
 #include "vr_message.h"
 #include "vr_mirror.h"
+#include "vr_offloads.h"
 
 struct vr_mirror_entry *
 vrouter_get_mirror(unsigned int rid, unsigned int index)
@@ -63,6 +64,8 @@ __vr_mirror_del(struct vrouter *router, unsigned int index)
     }
     vrouter_put_nexthop(nh);
 
+    (void)vr_offload_mirror_del(index);
+
     return 0;
 }
 
@@ -128,6 +131,13 @@ vr_mirror_add(vr_mirror_req *req)
     if (old_nh)
         vrouter_put_nexthop(old_nh);
 
+    /* if offload failed, release the newly added mirror entry.
+     * vrouter_put_mirror() also drops the reference on the nhop.
+     */
+    ret = vr_offload_mirror_add(mirror, req->mirr_index);
+    if (ret)
+        __vr_mirror_del(router, req->mirr_index);
+
 generate_resp:
     vr_send_response(ret);
 
@@ -176,6 +186,7 @@ vr_mirror_dump(vr_mirror_req *r)
         mirror = router->vr_mirrors[i];
         if (mirror) {
            vr_mirror_make_req(&req, mirror, i);
+           (void)vr_offload_mirror_get(&req);
            ret = vr_message_dump_object(dumper, VR_MIRROR_OBJECT_ID, &req);
            if (ret <= 0)
                break;
@@ -207,6 +218,8 @@ vr_mirror_get(vr_mirror_req *req)
 
     if (mirror) {
         vr_mirror_make_req(req, mirror, req->mirr_index);
+        /* Debug comparison to check if matching entry is programmed on NIC */
+        (void)vr_offload_mirror_get(req);
     } else
         req = NULL;
 

dp-core/vr_mpls.c

@@ -13,6 +13,7 @@
 #include "vr_bridge.h"
 #include "vr_datapath.h"
 #include "vr_btable.h"
+#include "vr_offloads.h"
 
 unsigned int vr_mpls_labels = VR_DEF_LABELS;
 
@@ -86,6 +87,10 @@ vr_mpls_del(vr_mpls_req *req)
 
     ret =  __vr_mpls_del(router, req->mr_label);
 
+    /* notify hw offload of change, if enabled */
+    if (!ret)
+        (void)vr_offload_mpls_del(req->mr_label);
+
 generate_resp:
     vr_send_response(ret);
 
@@ -131,6 +136,13 @@ vr_mpls_add(vr_mpls_req *req)
         && nh->nh_type == NH_ENCAP && !(nh->nh_flags & NH_FLAG_MCAST))
         vrouter_host->hos_add_mpls(router, req->mr_label);
 
+    /* notify hw offload of change, if enabled */
+    if (!ret) {
+        ret = vr_offload_mpls_add(nh, req->mr_label);
+        if (ret)
+            __vr_mpls_del(router, req->mr_label);
+    }
+
 generate_resp:
     vr_send_response(ret);
 
@@ -145,6 +157,9 @@ vr_mpls_make_req(vr_mpls_req *req, struct vr_nexthop *nh,
     req->mr_nhid = nh->nh_id;
     req->mr_label = label;
 
+    /* Debug comparison to check if matching entry is programmed on NIC */
+    (void)vr_offload_mpls_get(req);
+
     return;
 }
 

dp-core/vr_nexthop.c

@@ -18,6 +18,7 @@
 #include "vr_route.h"
 #include "vr_hash.h"
 #include "vr_mirror.h"
+#include "vr_offloads.h"
 
 extern bool vr_has_to_fragment(struct vr_interface *, struct vr_packet *,
         unsigned int);
@@ -2720,6 +2721,7 @@ vr_nexthop_delete(vr_nexthop_req *req)
     if (!nh) {
         ret = -EINVAL;
     } else {
+        (void)vr_offload_nexthop_del(nh);
         vrouter_put_nexthop(nh);
         nh->nh_destructor(nh);
     }
@@ -3426,6 +3428,15 @@ vr_nexthop_add(vr_nexthop_req *req)
         nh->nh_flags |= NH_FLAG_VALID;
 
     ret = vrouter_add_nexthop(nh);
+
+    if (ret) {
+        nh->nh_destructor(nh);
+        goto generate_resp;
+    }
+    else /* notify hw offload of change, if enabled */
+        ret = vr_offload_nexthop_add(nh);
+
+    /* if offload failed, delete kernel entry for consistency */
     if (ret)
         nh->nh_destructor(nh);
 
@@ -3752,6 +3763,10 @@ vr_nexthop_get(vr_nexthop_req *req)
     } else
         ret = -ENOENT;
 
+    /* Debug comparison to check if matching entry is programmed on NIC */
+    if (!ret)
+        (void)vr_offload_nexthop_get(nh, resp);
+
 generate_response:
     vr_message_response(VR_NEXTHOP_OBJECT_ID, ret < 0 ? NULL : resp, ret, false);
     if (resp)
@@ -3790,6 +3805,10 @@ vr_nexthop_dump(vr_nexthop_req *r)
 
             resp->h_op = SANDESH_OP_DUMP;
             ret = vr_nexthop_make_req(resp, nh);
+
+            if (!ret)
+                (void)vr_offload_nexthop_get(nh, resp);
+
             if ((ret < 0) || ((ret = vr_message_dump_object(dumper,
                                 VR_NEXTHOP_OBJECT_ID, resp)) <= 0)) {
                 vr_nexthop_req_destroy(resp);
@@ -3923,3 +3942,42 @@ vr_nexthop_init(struct vrouter *router)
 {
     return nh_table_init(router);
 }
+
+/*
+ * Called by offload module to update vrfstats with packets which have been
+ * offloaded.
+ */
+int
+vr_nexthop_update_offload_vrfstats(uint32_t vrfid, uint32_t num_cntrs,
+                               uint64_t *cntrs)
+{
+    uint32_t i;
+    uint64_t *dst_cntr;
+    uint64_t *src_cntr;
+    struct vr_vrf_stats *stats;
+    uint32_t tmp[2];
+    uint64_t tmp64;
+
+    if (!vr_inet_vrf_stats)
+        return 0;
+
+    /* hw offload stats always go to CPU 0 */
+    stats = vr_inet_vrf_stats(vrfid, 0);
+    if (stats && num_cntrs <= sizeof(struct vr_vrf_stats) * sizeof(uint64_t)) {
+        dst_cntr = (uint64_t *)stats;
+        src_cntr = cntrs;
+        for (i = 0; i < num_cntrs; i++) {
+            tmp[0] = (uint32_t)(*src_cntr);
+            tmp[1] = (uint32_t)((*src_cntr) >> 32);
+            tmp[0] = ntohl(tmp[0]);
+            tmp[1] = ntohl(tmp[1]);
+            tmp64 = (uint64_t)tmp[0] |
+                    ((uint64_t)tmp[1] << 32);
+            *dst_cntr += tmp64;
+            src_cntr++;
+            dst_cntr++;
+        }
+    }
+
+    return 0;
+}