From b09e7384f19d02e19c74c9973e94db8ab8b81b91 Mon Sep 17 00:00:00 2001 From: Trevor Patrie Date: Tue, 15 Nov 2016 11:39:52 -0500 Subject: [PATCH] dp_core: Introduce offloads module Smart nics may have ability to parse and act the Vrouter tunneled packets. So, according to the Vrouter tunnels, aka MPLSoGRE, MPLSoUDP, and VXLAN, the inner packet 5-tupples and the outer packet specifications, the HW can do some actions. For example, the HW can detect MPLSoGRE packets and to do RSS on the inner packet 5-tupples or just tag the packets based on the packet HW parser. Using this HW feature the Vrouter parformance can be improved. Add a Vrouter offloads module to use the described HW ability. Partial-Bug: #1781402 Change-Id: I337dd1d25671271528be00a5fef2eebddf1b0292 Signed-off-by: matan --- Makefile | 1 + dp-core/vr_datapath.c | 2 + dp-core/vr_flow.c | 67 +++-- dp-core/vr_interface.c | 26 ++ dp-core/vr_mirror.c | 13 + dp-core/vr_mpls.c | 27 +- dp-core/vr_nexthop.c | 20 ++ dp-core/vr_offloads.c | 460 ++++++++++++++++++++++++++++++++ dp-core/vr_proto_ip.c | 23 +- dp-core/vr_proto_ip6.c | 20 +- dp-core/vr_qos.c | 33 +++ dp-core/vr_route.c | 20 +- dp-core/vr_stats.c | 7 + dp-core/vr_vrf_assign.c | 11 + dp-core/vr_vxlan.c | 42 ++- dp-core/vrouter.c | 13 +- include/ini_parser.h | 3 + include/vr_offloads.h | 350 ++++++++++++++++++++++++ include/vr_packet.h | 2 + include/vrouter.h | 8 + utils/dropstats.c | 113 +++++++- utils/ini_parser.c | 11 + windows/vRouter.vcxproj | 2 + windows/vRouter.vcxproj.filters | 6 + 24 files changed, 1221 insertions(+), 59 deletions(-) create mode 100644 dp-core/vr_offloads.c create mode 100644 include/vr_offloads.h diff --git a/Makefile b/Makefile index 2708ad627..5b4c89b87 100644 --- a/Makefile +++ b/Makefile @@ -69,6 +69,7 @@ ifneq ($(KERNELRELEASE), ) vrouter-y += dp-core/vr_vxlan.o dp-core/vr_fragment.o vrouter-y += dp-core/vr_proto_ip6.o dp-core/vr_buildinfo.o vrouter-y += dp-core/vr_bitmap.o dp-core/vr_qos.o + vrouter-y += dp-core/vr_offloads.o ccflags-y += -I$(src)/include -I$(SANDESH_HEADER_PATH)/sandesh/gen-c ccflags-y += -I$(SANDESH_EXTRA_HEADER_PATH) diff --git a/dp-core/vr_datapath.c b/dp-core/vr_datapath.c index 7d45957c0..049a27c83 100644 --- a/dp-core/vr_datapath.c +++ b/dp-core/vr_datapath.c @@ -670,6 +670,8 @@ vr_fabric_input_inline(struct vr_interface *vif, struct vr_packet *pkt, unsigned short pull_len; unsigned char *data, eth_dmac[VR_ETHER_ALEN]; + if (vr_offload_prepare) + vr_offload_prepare(pkt, fmd); fmd->fmd_vlan = vlan_id; fmd->fmd_dvrf = vif->vif_vrf; diff --git a/dp-core/vr_flow.c b/dp-core/vr_flow.c index 462ce9289..1e5748222 100644 --- a/dp-core/vr_flow.c +++ b/dp-core/vr_flow.c @@ -20,6 +20,8 @@ #include "vr_ip_mtrie.h" #include "vr_bridge.h" +#include "vr_offloads.h" + #define VR_NUM_FLOW_TABLES 1 #define VR_NUM_OFLOW_TABLES 1 @@ -190,6 +192,7 @@ vr_flow_reset_mirror(struct vrouter *router, struct vr_flow_entry *fe, if (fe->fe_mme) { vr_mirror_meta_entry_del(router, fe->fe_mme); fe->fe_mme = NULL; + vr_offload_flow_meta_data_set(index, 0, 0, 0); } } fe->fe_flags &= ~VR_FLOW_FLAG_MIRROR; @@ -221,6 +224,8 @@ __vr_flow_reset_entry(struct vrouter *router, struct vr_flow_entry *fe) fe->fe_hold_list = NULL; fe->fe_key.flow_key_len = 0; + (void)vr_offload_flow_del(fe); + vr_flow_reset_mirror(router, fe, fe->fe_hentry.hentry_index); fe->fe_ecmp_nh_index = -1; fe->fe_src_nh_index = NH_DISCARD_ID; @@ -804,6 +809,9 @@ vr_rflow_update_ecmp_index(struct vrouter *router, struct vr_flow_entry *fe, rfe->fe_ecmp_nh_index = new_ecmp_index; + /* Update hardware reverse flow. */ + (void)vr_offload_flow_set(rfe, fe->fe_rflow, NULL); + fmd->fmd_ecmp_src_nh_index = new_ecmp_index; return 0; @@ -1487,28 +1495,33 @@ vr_flow_lookup(struct vrouter *router, struct vr_flow *key, pkt->vp_flags |= VP_FLAG_FLOW_SET; - flow_e = vr_find_flow(router, key, pkt->vp_type, &fe_index); - if (!flow_e) { - if (pkt->vp_nh && - (pkt->vp_nh->nh_flags & - (NH_FLAG_RELAXED_POLICY | NH_FLAG_FLOW_LOOKUP))) - return FLOW_FORWARD; + if (!fmd->fmd_fe) { + flow_e = vr_find_flow(router, key, pkt->vp_type, &fe_index); + if (!flow_e) { + if (pkt->vp_nh && + (pkt->vp_nh->nh_flags & + (NH_FLAG_RELAXED_POLICY | NH_FLAG_FLOW_LOOKUP))) + return FLOW_FORWARD; + + if (!vr_flow_allow_new_flow(router, pkt, &drop_reason, &burst)) { + vr_pfree(pkt, drop_reason); + return FLOW_CONSUMED; + } - if (!vr_flow_allow_new_flow(router, pkt, &drop_reason, &burst)) { - vr_pfree(pkt, drop_reason); - return FLOW_CONSUMED; - } + flow_e = vr_flow_get_free_entry(router, key, pkt->vp_type, + true, &fe_index); + if (!flow_e) { + vr_pfree(pkt, VP_DROP_FLOW_TABLE_FULL); + return FLOW_CONSUMED; + } - flow_e = vr_flow_get_free_entry(router, key, pkt->vp_type, - true, &fe_index); - if (!flow_e) { - vr_pfree(pkt, VP_DROP_FLOW_TABLE_FULL); - return FLOW_CONSUMED; + flow_e->fe_vrf = fmd->fmd_dvrf; + /* mark as hold */ + vr_flow_entry_set_hold(router, flow_e, burst); } - - flow_e->fe_vrf = fmd->fmd_dvrf; - /* mark as hold */ - vr_flow_entry_set_hold(router, flow_e, burst); + } else { + flow_e = fmd->fmd_fe; + fe_index = fmd->fmd_flow_index; } if (flow_e->fe_flags & VR_FLOW_FLAG_EVICT_CANDIDATE) @@ -1865,6 +1878,13 @@ vr_flow_set_mirror(struct vrouter *router, vr_flow_req *req, req->fr_mir_sip, req->fr_mir_sport, req->fr_pcap_meta_data, req->fr_pcap_meta_data_size, req->fr_mir_vrf); + + if (fe->fe_mme) { + vr_offload_flow_meta_data_set(req->fr_index, + req->fr_pcap_meta_data_size, + req->fr_pcap_meta_data, + req->fr_mir_vrf); + } } return; @@ -2229,6 +2249,7 @@ vr_flow_set(struct vrouter *router, vr_flow_req *req, if (fe) { if (!(modified = vr_flow_start_modify(router, fe))) return -EBUSY; + fe_index = (unsigned int)(req->fr_index); } if ((ret = vr_flow_set_req_is_invalid(router, req, fe))) @@ -2365,6 +2386,14 @@ vr_flow_set(struct vrouter *router, vr_flow_req *req, ret = vr_flow_schedule_transition(router, req, fe); + /* + * offload, no need to differentiate between add and modify. Pass the + * reverse flow as well if present. + */ + if (!ret) { + vr_offload_flow_set(fe, fe_index, rfe); + } + exit_set: if (modified && fe) { vr_flow_stop_modify(router, fe); diff --git a/dp-core/vr_interface.c b/dp-core/vr_interface.c index 973c2d1f9..346a49011 100644 --- a/dp-core/vr_interface.c +++ b/dp-core/vr_interface.c @@ -16,6 +16,7 @@ #include "vr_btable.h" #include "vr_route.h" #include "vr_ip_mtrie.h" +#include "vr_offloads.h" unsigned int vr_interfaces = VR_MAX_INTERFACES; @@ -2049,6 +2050,8 @@ vr_interface_delete(vr_interface_req *req, bool need_response) if (!vif && (ret = -ENODEV)) goto del_fail; + vr_offload_interface_del(vif); + vif_delete(vif); del_fail: @@ -2284,6 +2287,10 @@ vr_interface_add(vr_interface_req *req, bool need_response) vif = __vrouter_get_interface(router, req->vifr_idx); if (vif) { ret = vr_interface_change(vif, req); + /* notify hw offload of change, if enabled */ + if (!ret) + ret = vr_offload_interface_add(vif); + goto generate_resp; } @@ -2432,6 +2439,14 @@ vr_interface_add(vr_interface_req *req, bool need_response) if (ret && vif) vif_free(vif); + /* notify hw offload of change, if enabled */ + if (!ret) { + ret = vr_offload_interface_add(vif); + if (ret) { + vif_delete(vif); + vif = NULL; + } + } generate_resp: if (need_response) vr_send_response(ret); @@ -2518,6 +2533,7 @@ __vr_interface_make_req(vr_interface_req *req, struct vr_interface *intf, req->vifr_transport = intf->vif_transport; req->vifr_os_idx = intf->vif_os_idx; req->vifr_mtu = intf->vif_mtu; + req->vifr_nh_id = intf->vif_nh_id; if (req->vifr_mac_size && req->vifr_mac) memcpy(req->vifr_mac, intf->vif_mac, MINIMUM(req->vifr_mac_size, sizeof(intf->vif_mac))); @@ -3060,6 +3076,12 @@ vr_interface_get(vr_interface_req *req) mm.vr_mm_object_type[obj_cnt] = VR_DROP_STATS_OBJECT_ID; mm.vr_mm_object[obj_cnt] = drop_resp; obj_cnt++; + + /* zero vifr_core means to sum up all the per-core stats */ + vr_interface_make_req(vif_resp, vif, (unsigned)(req->vifr_core - 1)); + /* adds in stats for pkts which were offloaded on NIC and does debug + comparison to check if matching entry is programmed on NIC */ + ret = vr_offload_interface_get(vif_resp); } generate_response: @@ -3117,6 +3139,10 @@ vr_interface_dump(vr_interface_req *r) if (vif) { /* zero vifr_core means to sum up all the per-core stats */ vr_interface_make_req(resp, vif, (unsigned)(r->vifr_core - 1)); + + /* let hw offload fill in relevant fields */ + vr_offload_interface_get(resp); + ret = vr_message_dump_object(dumper, VR_INTERFACE_OBJECT_ID, resp); if (ret <= 0) break; diff --git a/dp-core/vr_mirror.c b/dp-core/vr_mirror.c index e3447a296..6b298ecfd 100644 --- a/dp-core/vr_mirror.c +++ b/dp-core/vr_mirror.c @@ -9,6 +9,7 @@ #include "vr_sandesh.h" #include "vr_message.h" #include "vr_mirror.h" +#include "vr_offloads.h" struct vr_mirror_entry * vrouter_get_mirror(unsigned int rid, unsigned int index) @@ -63,6 +64,8 @@ __vr_mirror_del(struct vrouter *router, unsigned int index) } vrouter_put_nexthop(nh); + vr_offload_mirror_del(index); + return 0; } @@ -128,6 +131,13 @@ vr_mirror_add(vr_mirror_req *req) if (old_nh) vrouter_put_nexthop(old_nh); + /* if offload failed, release the newly added mirror entry. + * vrouter_put_mirror() also drops the reference on the nhop. + */ + ret = vr_offload_mirror_add(mirror, req->mirr_index); + if (ret) + __vr_mirror_del(router, req->mirr_index); + generate_resp: vr_send_response(ret); @@ -176,6 +186,7 @@ vr_mirror_dump(vr_mirror_req *r) mirror = router->vr_mirrors[i]; if (mirror) { vr_mirror_make_req(&req, mirror, i); + vr_offload_mirror_get(&req); ret = vr_message_dump_object(dumper, VR_MIRROR_OBJECT_ID, &req); if (ret <= 0) break; @@ -207,6 +218,8 @@ vr_mirror_get(vr_mirror_req *req) if (mirror) { vr_mirror_make_req(req, mirror, req->mirr_index); + /* Debug comparison to check if matching entry is programmed on NIC */ + vr_offload_mirror_get(req); } else req = NULL; diff --git a/dp-core/vr_mpls.c b/dp-core/vr_mpls.c index 7ccef4724..fd1693b46 100644 --- a/dp-core/vr_mpls.c +++ b/dp-core/vr_mpls.c @@ -13,6 +13,7 @@ #include "vr_bridge.h" #include "vr_datapath.h" #include "vr_btable.h" +#include "vr_offloads.h" unsigned int vr_mpls_labels = VR_DEF_LABELS; @@ -84,6 +85,9 @@ vr_mpls_del(vr_mpls_req *req) goto generate_resp; } + /* notify hw offload of change, if enabled */ + vr_offload_mpls_del(req->mr_label); + ret = __vr_mpls_del(router, req->mr_label); generate_resp: @@ -131,6 +135,13 @@ vr_mpls_add(vr_mpls_req *req) && nh->nh_type == NH_ENCAP && !(nh->nh_flags & NH_FLAG_MCAST)) vrouter_host->hos_add_mpls(router, req->mr_label); + /* notify hw offload of change, if enabled */ + if (!ret) { + ret = vr_offload_mpls_add(nh, req->mr_label); + if (ret) + __vr_mpls_del(router, req->mr_label); + } + generate_resp: vr_send_response(ret); @@ -145,6 +156,9 @@ vr_mpls_make_req(vr_mpls_req *req, struct vr_nexthop *nh, req->mr_nhid = nh->nh_id; req->mr_label = label; + /* Debug comparison to check if matching entry is programmed on NIC */ + vr_offload_mpls_get(req); + return; } @@ -337,11 +351,14 @@ vr_mpls_input(struct vrouter *router, struct vr_packet *pkt, goto dropit; } - nh = __vrouter_get_label(router, label); - if (!nh) { - drop_reason = VP_DROP_INVALID_LABEL; - goto dropit; - } + if (!fmd->fmd_fe) { + nh = __vrouter_get_label(router, label); + if (!nh) { + drop_reason = VP_DROP_INVALID_LABEL; + goto dropit; + } + } else + nh = pkt->vp_nh; /* * Mark it for GRO. Diag, L2 and multicast nexthops unmark if diff --git a/dp-core/vr_nexthop.c b/dp-core/vr_nexthop.c index 62f1557de..5fb20514a 100644 --- a/dp-core/vr_nexthop.c +++ b/dp-core/vr_nexthop.c @@ -18,6 +18,7 @@ #include "vr_route.h" #include "vr_hash.h" #include "vr_mirror.h" +#include "vr_offloads.h" extern bool vr_has_to_fragment(struct vr_interface *, struct vr_packet *, unsigned int); @@ -2840,6 +2841,7 @@ vr_nexthop_delete(vr_nexthop_req *req) if (!nh) { ret = -EINVAL; } else { + vr_offload_nexthop_del(nh); vrouter_put_nexthop(nh); nh->nh_destructor(nh); } @@ -3596,6 +3598,15 @@ vr_nexthop_add(vr_nexthop_req *req) nh->nh_flags |= NH_FLAG_VALID; ret = vrouter_add_nexthop(nh); + + if (ret) { + nh->nh_destructor(nh); + goto generate_resp; + } + else /* notify hw offload of change, if enabled */ + ret = vr_offload_nexthop_add(nh); + + /* if offload failed, delete kernel entry for consistency */ if (ret) nh->nh_destructor(nh); @@ -3932,6 +3943,10 @@ vr_nexthop_get(vr_nexthop_req *req) } else ret = -ENOENT; + /* Debug comparison to check if matching entry is programmed on NIC */ + if (!ret) + vr_offload_nexthop_get(nh, resp); + generate_response: vr_message_response(VR_NEXTHOP_OBJECT_ID, ret < 0 ? NULL : resp, ret, false); if (resp) @@ -3970,6 +3985,10 @@ vr_nexthop_dump(vr_nexthop_req *r) resp->h_op = SANDESH_OP_DUMP; ret = vr_nexthop_make_req(resp, nh); + + if (!ret) + vr_offload_nexthop_get(nh, resp); + if ((ret < 0) || ((ret = vr_message_dump_object(dumper, VR_NEXTHOP_OBJECT_ID, resp)) <= 0)) { vr_nexthop_req_destroy(resp); @@ -4103,3 +4122,4 @@ vr_nexthop_init(struct vrouter *router) { return nh_table_init(router); } + diff --git a/dp-core/vr_offloads.c b/dp-core/vr_offloads.c new file mode 100644 index 000000000..48ab2d663 --- /dev/null +++ b/dp-core/vr_offloads.c @@ -0,0 +1,460 @@ +/* + * vr_offloads.c -- datapath flow offloads management + * + * Copyright 2018 Mellanox Technologies, Ltd + */ +#include +#include +#include +#include + +extern unsigned int vr_interfaces; +extern unsigned int vr_flow_entries; +extern unsigned int vr_oflow_entries; + +struct vr_interface *pvif; +unsigned int host_ip; +struct vr_btable *offload_flows; +struct vr_btable *offload_tags; +unsigned int datapath_offloads; + +int +vr_offloads_interface_add(struct vr_interface *vif) +{ + if (!vif) + return -EINVAL; + if (vif->vif_type == VIF_TYPE_PHYSICAL) { + if (pvif && pvif != vif) + vr_printf("offload: More than one physical interface\n"); + else + pvif = vif; + } else if (vif->vif_type == VIF_TYPE_HOST) { + if (host_ip && vif->vif_ip != host_ip) + vr_printf("offload: More than one host interface\n"); + else + host_ip = vif->vif_ip; + } + return 0; +} + +int +vr_offloads_interface_del(struct vr_interface *vif) +{ + if (!vif) + return -EINVAL; + if (vif->vif_type == VIF_TYPE_PHYSICAL) { + if(vif != pvif) + vr_printf("offload: More than one physical interface\n"); + else + pvif = NULL; + } else if (vif->vif_type == VIF_TYPE_HOST) { + if (vif->vif_ip != host_ip) + vr_printf("offload: More than one host interface\n"); + else + host_ip = 0; + } + + return 0; +} + +static bool +vr_offloads_is_offloaded_nexthop(struct vr_nexthop *nh) +{ + return ((nh->nh_flags & NH_FLAG_VALID) && !(nh->nh_flags & NH_FLAG_MCAST) && + (nh->nh_flags & NH_FLAG_POLICY_ENABLED) && (nh->nh_type == NH_ENCAP) && + nh->nh_dev); +} + +int +vr_offloads_mpls_add(struct vr_nexthop *nh, int label) +{ + struct vr_offload_tag *otag; + + if (!nh) + return -EINVAL; + if (!vr_offloads_is_offloaded_nexthop(nh)) + return 0; + + otag = (struct vr_offload_tag *)vr_btable_get(offload_tags, nh->nh_dev->vif_idx); + if (!otag) { + vr_printf("offload: Invalid tag for nexthop ID %u\n",nh->nh_id); + return 0; + } + + if (nh->nh_family == AF_BRIDGE && (nh->nh_flags & NH_FLAG_L2_CONTROL_DATA)) + otag += VR_OFFLOADS_TAG_TYPE_MPLS_L2; + else if (nh->nh_family != AF_BRIDGE) + otag += VR_OFFLOADS_TAG_TYPE_MPLS_L3; + else + return 0; + + if (!otag->nh) { + otag->tag = label; + otag->nh = nh; + } else if (otag->tag != label) { + vr_printf("offload: 2 different MPLS labels (%u,%u) point to the same" + " tag type of vif %u nexthop %u\n", otag->tag, label, + nh->nh_dev->vif_idx, nh->nh_id); + } + + return 0; +} + +int +vr_offloads_vxlan_add(struct vr_nexthop * nh, int vnid) +{ + struct vr_offload_tag *otag; + + if (!nh) + return -EINVAL; + if (!vr_offloads_is_offloaded_nexthop(nh)) + return 0; + + otag = (struct vr_offload_tag *)vr_btable_get(offload_tags, nh->nh_dev->vif_idx); + if (!otag) { + vr_printf("offload: Invalid tag for nexthop ID %u\n",nh->nh_id); + return 0; + } + + otag += VR_OFFLOADS_TAG_TYPE_VXLAN; + + if (!otag->nh) { + otag->tag = vnid; + otag->nh = nh; + } else if (otag->tag != vnid) { + vr_printf("offload: 2 different VXLAN VNIs (%u,%u) point to the same" + " tag type of vif %u nexthop %u\n", otag->tag, vnid, + nh->nh_dev->vif_idx, nh->nh_id); + } + + return 0; +} + +int +vr_offloads_mpls_del(int label) +{ + struct vr_offload_tag *otag; + struct vrouter *router = vrouter_get(0); + struct vr_nexthop *nh = __vrouter_get_label(router, label); + + if (!nh) + return -EINVAL; + + if (!vr_offloads_is_offloaded_nexthop(nh)) + return 0; + + otag = (struct vr_offload_tag *)vr_btable_get(offload_tags, nh->nh_dev->vif_idx); + if (!otag) { + vr_printf("offload: Invalid tag for nexthop ID %u\n",nh->nh_id); + return 0; + } + + if (nh->nh_family == AF_BRIDGE && (nh->nh_flags & NH_FLAG_L2_CONTROL_DATA)) + otag += VR_OFFLOADS_TAG_TYPE_MPLS_L2; + else if (nh->nh_family != AF_BRIDGE) + otag += VR_OFFLOADS_TAG_TYPE_MPLS_L3; + else + return 0; + + if (otag->nh && otag->tag == label) + memset(otag, 0, sizeof(*otag)); + else + vr_printf("offload: Unexpected tag %u for MPLS label %u, nexthop %u\n", + otag->tag, label, nh->nh_id); + + return 0; +} + +int +vr_offloads_vxlan_del(int vnid) +{ + struct vr_offload_tag *otag; + struct vrouter *router = vrouter_get(0); + struct vr_nexthop *nh = (struct vr_nexthop *)vr_itable_get(router->vr_vxlan_table, vnid); + + if (!nh) + return -EINVAL; + + if (!vr_offloads_is_offloaded_nexthop(nh)) + return 0; + + otag = (struct vr_offload_tag *)vr_btable_get(offload_tags, nh->nh_dev->vif_idx); + if (!otag) { + vr_printf("offload: Invalid tag for nexthop ID %u\n",nh->nh_id); + return 0; + } + + otag += VR_OFFLOADS_TAG_TYPE_VXLAN; + + if (otag->nh && otag->tag == vnid) + memset(otag, 0, sizeof(*otag)); + else + vr_printf("offload: Unexpected tag %u for VXLAN vni %u, nexthop %u\n", + otag->tag, vnid, nh->nh_id); + + return 0; +} + +int +vr_offloads_flow_del(struct vr_flow_entry * fe) +{ + struct vr_offload_flow *oflow; + int ret = 0; + + if (!fe) + return -EINVAL; + + if (!offload_flows) + return 0; + + oflow = (struct vr_offload_flow *)vr_btable_get(offload_flows, + fe->fe_hentry.hentry_index); + if (!oflow) { + vr_printf("offload: Invalid tag for flow ID %u\n", + fe->fe_hentry.hentry_index); + return 0; + } + + if(oflow->flow_handle) { + ret = vr_offload_flow_destroy(oflow); + if (!ret) + memset(oflow, 0, sizeof(*oflow)); + else + vr_printf("offload: Failed to destroy flow ID %u\n", oflow->fe_index); + } + + return 0; +} + +int +vr_offloads_flow_set(struct vr_flow_entry * fe, unsigned int fe_index, + struct vr_flow_entry * rfe) +{ + struct vr_nexthop *nh, *snh; + struct vr_offload_flow *oflow; + struct vr_offload_tag *otag; + struct vrouter *router = vrouter_get(0); + int ret = 0; + + if (!fe) + return -EINVAL; + + if (!pvif || !host_ip) { + vr_printf("offload: Missing physical/host interface\n"); + return 0; + } + + oflow = (struct vr_offload_flow *)vr_btable_get(offload_flows, fe_index); + if (!oflow) { + vr_printf("offload: Invalid tag for flow ID %u\n", fe_index); + return 0; + } + + /* For flow change do destroy and create */ + if (oflow->flow_handle) { + ret = vr_offload_flow_destroy(oflow); + if (!ret) { + memset(oflow, 0, sizeof(*oflow)); + } else { + vr_printf("offload: Failed to change flow ID %u\n", oflow->fe_index); + return 0; + } + } + + nh = __vrouter_get_nexthop(router, fe->fe_key.flow_nh_id); + if (!nh) + return -EINVAL; + + if (!(fe->fe_flags & VR_FLOW_FLAG_ACTIVE) || fe->fe_action != VR_FLOW_ACTION_FORWARD || + !vr_offloads_is_offloaded_nexthop(nh)) + /* Not a valid flow to be offloaded */ + return 0; + + snh = __vrouter_get_nexthop(router, fe->fe_src_nh_index); + if (!snh) + return -EINVAL; + + if (snh->nh_type != NH_TUNNEL || !(snh->nh_flags & NH_FLAG_VALID)) + /* Not a valid flow to be offloaded */ + return 0; + + otag = (struct vr_offload_tag *)vr_btable_get(offload_tags, nh->nh_dev->vif_idx); + if (!otag) { + vr_printf("offload: Invalid tag for nexthop ID %u\n", nh->nh_id); + return 0; + } + + if (snh->nh_flags & NH_FLAG_TUNNEL_GRE) { + if (otag[VR_OFFLOADS_TAG_TYPE_MPLS_L2].nh && (nh->nh_dev->vif_flags & + VIF_FLAG_L2_ENABLED)) { + oflow->tunnel_type = NH_FLAG_TUNNEL_GRE; + oflow->tunnel_tag =otag[VR_OFFLOADS_TAG_TYPE_MPLS_L2].tag; + oflow->nh = otag[VR_OFFLOADS_TAG_TYPE_MPLS_L2].nh; + oflow->is_mpls_l2 = true; + } else if (otag[VR_OFFLOADS_TAG_TYPE_MPLS_L3].nh && (nh->nh_dev->vif_flags & + VIF_FLAG_L3_ENABLED)) { + oflow->tunnel_type = NH_FLAG_TUNNEL_GRE; + oflow->tunnel_tag = otag[VR_OFFLOADS_TAG_TYPE_MPLS_L3].tag; + oflow->nh = otag[VR_OFFLOADS_TAG_TYPE_MPLS_L3].nh; + oflow->is_mpls_l2 = false; + } + } else if (snh->nh_flags & NH_FLAG_TUNNEL_UDP_MPLS) { + if (otag[VR_OFFLOADS_TAG_TYPE_MPLS_L2].nh && (nh->nh_dev->vif_flags & + VIF_FLAG_L2_ENABLED)) { + oflow->tunnel_type = NH_FLAG_TUNNEL_UDP_MPLS; + oflow->tunnel_tag =otag[VR_OFFLOADS_TAG_TYPE_MPLS_L2].tag; + oflow->nh = otag[VR_OFFLOADS_TAG_TYPE_MPLS_L2].nh; + oflow->is_mpls_l2 = true; + } else if (otag[VR_OFFLOADS_TAG_TYPE_MPLS_L3].nh && (nh->nh_dev->vif_flags & + VIF_FLAG_L3_ENABLED)) { + oflow->tunnel_type = NH_FLAG_TUNNEL_UDP_MPLS; + oflow->tunnel_tag = otag[VR_OFFLOADS_TAG_TYPE_MPLS_L3].tag; + oflow->nh = otag[VR_OFFLOADS_TAG_TYPE_MPLS_L3].nh; + oflow->is_mpls_l2 = false; + } + } else if (snh->nh_flags & NH_FLAG_TUNNEL_VXLAN) { + if (otag[VR_OFFLOADS_TAG_TYPE_VXLAN].nh) { + oflow->tunnel_type = NH_FLAG_TUNNEL_VXLAN; + oflow->tunnel_tag = otag[VR_OFFLOADS_TAG_TYPE_VXLAN].tag; + oflow->nh = otag[VR_OFFLOADS_TAG_TYPE_VXLAN].nh; + } + } else { + /* Not a valid flow to be offloaded */ + return 0; + } + + if (!oflow->nh) { + vr_printf("offload: Invalid tag type for flow create\n"); + return 0; + } + + oflow->pvif = pvif; + oflow->fe = fe; + oflow->ip = host_ip; + oflow->fe_index = fe_index; + + ret = vr_offload_flow_create(oflow); + if (ret) { + vr_printf("offload: Failed to create flow ID %u\n", fe_index); + memset(oflow, 0, sizeof(*oflow)); + } + + return 0; +} + +struct vr_offload_ops vr_offload_ops = { + .voo_flow_set = vr_offloads_flow_set, + .voo_flow_del = vr_offloads_flow_del, + .voo_interface_add = vr_offloads_interface_add, + .voo_interface_del = vr_offloads_interface_del, + .voo_mpls_add = vr_offloads_mpls_add, + .voo_mpls_del = vr_offloads_mpls_del, + .voo_vxlan_add = vr_offloads_vxlan_add, + .voo_vxlan_del = vr_offloads_vxlan_del, +}; + +int +vr_offloads_init(struct vrouter *router) +{ + unsigned int entry_size; + + if (!datapath_offloads) + return 0; + + if (!vr_offload_flow_destroy || !vr_offload_flow_create || + !vr_offload_prepare) + return -ENOSYS; + + if (!offload_tags) { + /* Round up to the next divisor */ + for (entry_size = sizeof(struct vr_offload_tag) * VR_OFFLOADS_TAG_TYPE_MAX; + VR_SINGLE_ALLOC_LIMIT % entry_size; entry_size++); + offload_tags = vr_btable_alloc(vr_interfaces, entry_size); + if (!offload_tags) { + return -ENOMEM; + } + } + + if (!offload_flows) { + /* Round up to the next divisor */ + for (entry_size = sizeof(struct vr_offload_flow); + VR_SINGLE_ALLOC_LIMIT % entry_size; entry_size++); + offload_flows = vr_btable_alloc(vr_flow_entries + vr_oflow_entries, entry_size); + if (!offload_flows) { + vr_btable_free(offload_tags); + offload_tags = NULL; + return -ENOMEM; + } + } + + offload_ops = &vr_offload_ops; + + return 0; + +} + +void +vr_offloads_exit(struct vrouter *router, bool soft_reset) +{ + struct vr_offload_flow *oflow; + struct vr_offload_tag *otag; + unsigned int i; + unsigned int entry_num, entry_size; + + if (!datapath_offloads) + return; + + if (!vr_offload_flow_destroy || !vr_offload_flow_create || + !vr_offload_prepare) + return; + + if (offload_flows) { + entry_num = vr_btable_entries(offload_flows); + entry_size = vr_btable_size(offload_flows) / entry_num; + for (i = 0; i < entry_num; i++) { + oflow = (struct vr_offload_flow *)vr_btable_get(offload_flows, i); + if (!oflow) + continue; + if(oflow->flow_handle) + vr_offload_flow_destroy(oflow); + memset(oflow, 0, entry_size); + } + + if (!soft_reset) { + vr_btable_free(offload_flows); + offload_flows = NULL; + } + } + + if (offload_tags) { + entry_num = vr_btable_entries(offload_tags); + entry_size = vr_btable_size(offload_tags) / entry_num; + for (i = 0; i < entry_num; i++) { + otag = (struct vr_offload_tag *)vr_btable_get(offload_tags, i); + if (otag) + memset(otag, 0, entry_size); + } + + if (!soft_reset) { + vr_btable_free(offload_tags); + offload_tags = NULL; + } + } + + pvif = NULL; + host_ip = 0; + offload_ops = NULL; +} + +inline struct vr_offload_flow * +vr_offloads_flow_get(unsigned int index) +{ + struct vr_offload_flow *oflow = (struct vr_offload_flow *) + vr_btable_get(offload_flows, index); + + if (!oflow || !oflow->flow_handle) + /* An invalid packet flow */ + return NULL; + + return oflow; +} diff --git a/dp-core/vr_proto_ip.c b/dp-core/vr_proto_ip.c index 9105b124d..c63c1e0e4 100644 --- a/dp-core/vr_proto_ip.c +++ b/dp-core/vr_proto_ip.c @@ -1081,19 +1081,22 @@ vr_inet_flow_lookup(struct vrouter *router, struct vr_packet *pkt, return FLOW_FORWARD; } - ret = vr_inet_form_flow(router, fmd->fmd_dvrf, pkt, - fmd->fmd_vlan, flow_p, VR_FLOW_KEY_ALL, true); - if (ret < 0) { - if (!vr_ip_transport_header_valid(ip) && vr_enqueue_to_assembler) { - vr_enqueue_to_assembler(router, pkt, fmd); - } else { - /* unlikely to be hit. you can safely discount misc drops here */ - vr_pfree(pkt, VP_DROP_MISC); + if(fmd->fmd_fe) + flow_p = &fmd->fmd_fe->fe_key; + else { + ret = vr_inet_form_flow(router, fmd->fmd_dvrf, pkt, + fmd->fmd_vlan, flow_p, VR_FLOW_KEY_ALL, true); + if (ret < 0) { + if (!vr_ip_transport_header_valid(ip) && vr_enqueue_to_assembler) { + vr_enqueue_to_assembler(router, pkt, fmd); + } else { + /* unlikely to be hit. you can safely discount misc drops here */ + vr_pfree(pkt, VP_DROP_MISC); + } + return FLOW_CONSUMED; } - return FLOW_CONSUMED; } - if (vif_is_fabric(pkt->vp_if) && !fmd->fmd_outer_src_ip) { if (flow_p->flow4_proto == VR_IP_PROTO_GRE) { return FLOW_FORWARD; diff --git a/dp-core/vr_proto_ip6.c b/dp-core/vr_proto_ip6.c index 45b6ca6c4..2563cb043 100644 --- a/dp-core/vr_proto_ip6.c +++ b/dp-core/vr_proto_ip6.c @@ -462,15 +462,19 @@ vr_inet6_flow_lookup(struct vrouter *router, struct vr_packet *pkt, if (!lookup) return FLOW_FORWARD; - ret = vr_inet6_form_flow(router, fmd->fmd_dvrf, pkt, fmd->fmd_vlan, - ip6, flow_p, VR_FLOW_KEY_ALL, true, true); - if (ret < 0) { - if (!vr_ip6_transport_header_valid(ip6) && vr_enqueue_to_assembler) { - vr_enqueue_to_assembler(router, pkt, fmd); - } else { - vr_pfree(pkt, VP_DROP_MISC); + if(fmd->fmd_fe) + flow_p = &fmd->fmd_fe->fe_key; + else { + ret = vr_inet6_form_flow(router, fmd->fmd_dvrf, pkt, fmd->fmd_vlan, + ip6, flow_p, VR_FLOW_KEY_ALL, true, true); + if (ret < 0) { + if (!vr_ip6_transport_header_valid(ip6) && vr_enqueue_to_assembler) { + vr_enqueue_to_assembler(router, pkt, fmd); + } else { + vr_pfree(pkt, VP_DROP_MISC); + } + return FLOW_CONSUMED; } - return FLOW_CONSUMED; } if (pkt->vp_flags & VP_FLAG_FLOW_SET) diff --git a/dp-core/vr_qos.c b/dp-core/vr_qos.c index ed8250343..1bfe38baa 100644 --- a/dp-core/vr_qos.c +++ b/dp-core/vr_qos.c @@ -13,6 +13,7 @@ #include "vr_interface.h" #include "vr_datapath.h" #include "vr_qos.h" +#include "vr_offloads.h" unsigned int vr_qos_map_entries = VR_DEF_QOS_MAP_ENTRIES; unsigned int vr_fc_map_entries = VR_DEF_FC_MAP_ENTRIES; @@ -270,6 +271,8 @@ vr_qos_map_delete(vr_qos_map_req *req) vr_free(fc_p, VR_QOS_MAP_OBJECT); } + (void)vr_offload_qos_map_del(req); + generate_response: vr_send_response(ret); return; @@ -307,6 +310,9 @@ vr_qos_map_dump(vr_qos_map_req *req) } vr_qos_map_make_req(i, resp, fc_p); + + (void)vr_offload_qos_map_get(resp); + ret = vr_message_dump_object(dumper, VR_QOS_MAP_OBJECT_ID, resp); vr_qos_map_req_destroy(resp); if (ret <= 0) @@ -345,6 +351,10 @@ vr_qos_map_get(vr_qos_map_req *req) } vr_qos_map_make_req(req->qmr_id, resp, fc_p); + + /* Debug comparison to check if matching entry is programmed on NIC */ + (void)vr_offload_qos_map_get(resp); + vr_message_response(VR_QOS_MAP_OBJECT_ID, resp, ret, false); if (resp) { vr_qos_map_req_destroy(resp); @@ -414,6 +424,13 @@ vr_qos_map_add(vr_qos_map_req *req) fc_e->vfc_valid = 1; } + ret = vr_offload_qos_map_add(req); + if (ret) { + vr_printf("offload QoS map not supported - not configuring\n"); + vr_free(fc_p, VR_QOS_MAP_OBJECT); + goto generate_response; + } + if (need_set) { vr_qos_map_set_fc(router, req->qmr_id, fc_p); } @@ -568,6 +585,7 @@ vr_fc_map_delete(vr_fc_map_req *req) } memset(fc_p, 0, sizeof(*fc_p)); + (void)vr_offload_fc_map_del(req); vr_send_response(0); return; @@ -614,6 +632,8 @@ vr_fc_map_dump(vr_fc_map_req *req) resp->fmr_dotonep[0] = fc_p->vfc_dotonep_qos; resp->fmr_queue_id[0] = fc_p->vfc_queue_id; + (void)vr_offload_fc_map_get(resp); + ret = vr_message_dump_object(dumper, VR_FC_MAP_OBJECT_ID, resp); vr_fc_map_req_destroy(resp); if (ret <= 0) @@ -662,6 +682,9 @@ vr_fc_map_get(vr_fc_map_req *req) resp->fmr_dotonep[0] = fc_p->vfc_dotonep_qos; resp->fmr_queue_id[0] = fc_p->vfc_queue_id; + /* Debug comparison to check if matching entry is programmed on NIC */ + (void)vr_offload_fc_map_get(resp); + generate_response: vr_message_response(VR_FC_MAP_OBJECT_ID, ret < 0 ? NULL : resp, ret, false); if (resp) @@ -703,6 +726,16 @@ vr_fc_map_add(vr_fc_map_req *req) fc_p->vfc_valid = 1; } + ret = vr_offload_fc_map_add(req); + if (ret) { + vr_printf("offload FC map not supported - not configuring\n"); + for (i = 0; i < req->fmr_id_size; i++) { + fc_p = vr_fc_map_get_fc(router, req->fmr_id[i]); + if (fc_p) + memset(fc_p, 0, sizeof(*fc_p)); + } + } + generate_response: vr_send_response(ret); return; diff --git a/dp-core/vr_route.c b/dp-core/vr_route.c index d6838df50..f115f5562 100644 --- a/dp-core/vr_route.c +++ b/dp-core/vr_route.c @@ -9,6 +9,7 @@ #include #include "vr_message.h" #include "vr_sandesh.h" +#include "vr_offloads.h" unsigned int vr_vrfs = VR_DEF_VRFS; @@ -114,6 +115,10 @@ vr_route_delete(vr_route_req *req) ret = fs->route_del(fs, &vr_req); } + /* notify hw offload of change, if enabled */ + if (!ret) + vr_offload_route_del(req); + error: vr_send_response(ret); vr_send_broadcast(VR_ROUTE_OBJECT_ID, &vr_req, SANDESH_OP_DEL, ret); @@ -146,6 +151,13 @@ vr_route_add(vr_route_req *req) ret = fs->route_add(fs, &vr_req); } + /* notify hw offload of change, if enabled */ + if (!ret) { + ret = vr_offload_route_add(req); + if (ret) + fs->route_del(fs, &vr_req); + } + vr_send_response(ret); vr_send_broadcast(VR_ROUTE_OBJECT_ID, &vr_req, SANDESH_OP_ADD, ret); @@ -200,7 +212,9 @@ vr_route_get(vr_route_req *req) ret = rtable->algo_get(vr_req.rtr_req.rtr_vrf_id, &vr_req); } - + /* Allow for debug comparison to check if matching entry is programmed on NIC */ + if (!ret) + vr_offload_route_get(req); generate_response: vr_message_response(VR_ROUTE_OBJECT_ID, ret ? NULL : &vr_req, ret, false); if (mac_mem_free && vr_req.rtr_req.rtr_mac) { @@ -256,6 +270,10 @@ vr_route_dump(vr_route_req *req) ret = rtable->algo_dump(NULL, &vr_req); } + /* Allow for debug comparison to check if matching entry is programmed on NIC */ + if (!ret) + vr_offload_route_dump(&vr_req); + return ret; generate_error: diff --git a/dp-core/vr_stats.c b/dp-core/vr_stats.c index 90363aebd..e000b1694 100644 --- a/dp-core/vr_stats.c +++ b/dp-core/vr_stats.c @@ -7,6 +7,7 @@ #include #include #include +#include #include "vr_message.h" #include "vr_btable.h" @@ -128,6 +129,12 @@ vr_drop_stats_get(unsigned int rid, short core) for (cpu = 0; cpu < vr_num_cpus; cpu++) { vr_drop_stats_make_response(response, router->vr_pdrop_stats[cpu]); } + /* offload gets added to summed stats */ + vr_offload_drop_stats_get(response); + } else if (core == (unsigned)-2) { + /* this allows returning only offloaded stats without + changing the current usage of this function */ + vr_offload_drop_stats_get(response); } else if (core < vr_num_cpus) { /* stats for a specific core */ vr_drop_stats_make_response(response, router->vr_pdrop_stats[core]); diff --git a/dp-core/vr_vrf_assign.c b/dp-core/vr_vrf_assign.c index fcc241e4a..394e59dc0 100644 --- a/dp-core/vr_vrf_assign.c +++ b/dp-core/vr_vrf_assign.c @@ -8,6 +8,7 @@ #include "vr_message.h" #include "vr_sandesh.h" #include "vr_packet.h" +#include "vr_offloads.h" #include #include @@ -89,6 +90,16 @@ vr_vrf_assign_set(vr_vrf_assign_req *req) ret = vif_vrf_table_set(vif, req->var_vlan_id, req->var_vif_vrf, req->var_nh_id); + if (ret == 0) { + ret = vr_offload_vif_vrf_set(req); + if (ret) { + /* + * If the offload fails to set the entry, reset vif_vrf_table. + */ + vif_vrf_table_set(vif, req->var_vlan_id, -1, 0); + } + } + exit_set: if (vif) vrouter_put_interface(vif); diff --git a/dp-core/vr_vxlan.c b/dp-core/vr_vxlan.c index 3d7a745d1..4936b50fe 100644 --- a/dp-core/vr_vxlan.c +++ b/dp-core/vr_vxlan.c @@ -12,6 +12,7 @@ #include "vr_vxlan.h" #include "vr_bridge.h" #include "vr_datapath.h" +#include "vr_offloads.h" int vr_vxlan_input(struct vrouter *router, struct vr_packet *pkt, @@ -38,21 +39,25 @@ vr_vxlan_input(struct vrouter *router, struct vr_packet *pkt, drop_reason = VP_DROP_INVALID_VNID; goto fail; } + if (!fmd->fmd_fe) { + vnid = ntohl(vxlan->vxlan_vnid) >> VR_VXLAN_VNID_SHIFT; + if (!pkt_pull(pkt, sizeof(struct vr_vxlan))) { + drop_reason = VP_DROP_PULL; + goto fail; + } - vnid = ntohl(vxlan->vxlan_vnid) >> VR_VXLAN_VNID_SHIFT; - if (!pkt_pull(pkt, sizeof(struct vr_vxlan))) { - drop_reason = VP_DROP_PULL; - goto fail; + nh = (struct vr_nexthop *)vr_itable_get(router->vr_vxlan_table, vnid); + if (!nh) { + drop_reason = VP_DROP_INVALID_VNID; + goto fail; + } + } else { + vnid = fmd->fmd_label; + nh = pkt->vp_nh; } vr_fmd_set_label(fmd, vnid, VR_LABEL_TYPE_VXLAN_ID); - nh = (struct vr_nexthop *)vr_itable_get(router->vr_vxlan_table, vnid); - if (!nh) { - drop_reason = VP_DROP_INVALID_VNID; - goto fail; - } - fmd->fmd_vlan = VLAN_ID_INVALID; if (vr_pkt_type(pkt, 0, fmd) < 0) { @@ -90,6 +95,8 @@ vr_vxlan_make_req(vr_vxlan_req *req, struct vr_nexthop *nh, unsigned int vnid) req->vxlanr_vnid = vnid; if (nh) req->vxlanr_nhid = nh->nh_id; + /* Debug comparison to check if matching entry is programmed on NIC */ + vr_offload_vxlan_get(req); return; } @@ -170,6 +177,9 @@ vr_vxlan_del(vr_vxlan_req *req) goto generate_resp; } + /* notify hw offload of change, if enabled */ + vr_offload_vxlan_del(req->vxlanr_vnid); + nh = vr_itable_del(router->vr_vxlan_table, req->vxlanr_vnid); if (nh) vrouter_put_nexthop(nh); @@ -199,6 +209,18 @@ vr_vxlan_add(vr_vxlan_req *req) } nh_old = vr_itable_set(router->vr_vxlan_table, req->vxlanr_vnid, nh); + + ret = vr_offload_vxlan_add(nh, req->vxlanr_vnid); + if (ret) { + /* If offload add fails, restore old nexthop */ + if (nh_old && nh_old != VR_ITABLE_ERR_PTR) + vr_itable_set(router->vr_vxlan_table, req->vxlanr_vnid, nh_old); + else + vr_itable_del(router->vr_vxlan_table, req->vxlanr_vnid); + vrouter_put_nexthop(nh); + goto generate_resp; + } + if (nh_old) { if (nh_old == VR_ITABLE_ERR_PTR) { ret = -EINVAL; diff --git a/dp-core/vrouter.c b/dp-core/vrouter.c index 31261b372..1df483b30 100644 --- a/dp-core/vrouter.c +++ b/dp-core/vrouter.c @@ -23,9 +23,11 @@ #include #include #include +#include static struct vrouter router; struct host_os *vrouter_host; +struct vr_offload_ops *offload_ops; extern struct host_os *vrouter_get_host(void); extern int vr_stats_init(struct vrouter *); @@ -108,6 +110,11 @@ static struct vr_module modules[] = { .init = vr_qos_init, .exit = vr_qos_exit, }, + { + .mod_name = "Offloads", + .init = vr_offloads_init, + .exit = vr_offloads_exit, + }, }; @@ -545,8 +552,12 @@ vrouter_init(void) static int vrouter_soft_reset(void) { + int ret = 0; vrouter_exit(true); - return vrouter_init(); + ret = vrouter_init(); + if (!ret) + ret = vr_offload_soft_reset(); + return ret; } void diff --git a/include/ini_parser.h b/include/ini_parser.h index 12a687b49..e99a9a898 100644 --- a/include/ini_parser.h +++ b/include/ini_parser.h @@ -18,6 +18,8 @@ extern "C" { #define PLATFORM_KEY "platform" #define PLATFORM_DPDK "dpdk" #define PLATFORM_NIC "nic" +#define OFFLOAD_KEY "offload" +#define OFFLOAD_ENABLED "enabled" extern int read_int(const char *section, const char *key); extern const char *read_string(const char *section, const char *key); @@ -32,6 +34,7 @@ extern int get_socket_type(void); extern int get_vrouter_ip(void); extern int get_platform(void); extern const char *get_platform_str(void); +extern bool get_offload_enabled(void); extern int get_protocol(void); extern int parse_ini_file(void); diff --git a/include/vr_offloads.h b/include/vr_offloads.h new file mode 100644 index 000000000..bae02c2a8 --- /dev/null +++ b/include/vr_offloads.h @@ -0,0 +1,350 @@ +/* + * vr_offloads.h -- register callbacks for hardware offload features + * + * Copyright (c) 2016 Netronome Systems, Inc. All rights reserved. + * Copyright 2018 Mellanox Technologies, Ltd + */ + +#ifndef __VR_OFFLOADS_H__ +#define __VR_OFFLOADS_H__ +#include +#include +#include +#include +#include +#include +#include +#include + +enum vr_offloads_tag_type { + VR_OFFLOADS_TAG_TYPE_MPLS_L2, + VR_OFFLOADS_TAG_TYPE_MPLS_L3, + VR_OFFLOADS_TAG_TYPE_VXLAN, + VR_OFFLOADS_TAG_TYPE_MAX +}; + +__attribute__packed__open__ +struct vr_offload_flow { + struct vr_nexthop *nh; + struct vr_flow_entry *fe; + unsigned int fe_index; + unsigned int tunnel_tag; + unsigned int tunnel_type; + bool is_mpls_l2; + struct vr_interface *pvif; + unsigned int ip; + void *flow_handle; +} __attribute__packed__close__; + +struct vr_offload_tag { + unsigned int tag; + struct vr_nexthop *nh; +}; + +struct vr_offload_ops { + char *voo_handler_id; /* Hardware vendor identifier */ + + /* perform soft reset, including initializing tables */ + int (*voo_soft_reset)(void); + + /* flow related functions */ + int (*voo_flow_set)(struct vr_flow_entry *, unsigned int, + struct vr_flow_entry *); + int (*voo_flow_del)(struct vr_flow_entry *); + int (*voo_flow_meta_data_set)(unsigned int, unsigned int, void *, + unsigned short); + + /* Dropstats */ + int (*voo_drop_stats_get)(vr_drop_stats_req *response); + + /* Interface */ + int (*voo_interface_add)(struct vr_interface *); + int (*voo_interface_del)(struct vr_interface *); + int (*voo_interface_get)(vr_interface_req *); + + /* vif_vrf table */ + int (*voo_vif_vrf_set)(vr_vrf_assign_req *); + int (*voo_vif_vrf_get)(vr_vrf_assign_req *); + + /* MPLS (ILM) */ + int (*voo_mpls_add)(struct vr_nexthop *, int); + int (*voo_mpls_del)(int); + int (*voo_mpls_get)(vr_mpls_req *); + + /* VXLAN (VNID) */ + int (*voo_vxlan_add)(struct vr_nexthop *, int); + int (*voo_vxlan_del)(int); + int (*voo_vxlan_get)(vr_vxlan_req *); + + /* Mirror table */ + int (*voo_mirror_add)(struct vr_mirror_entry *, unsigned int); + int (*voo_mirror_del)(unsigned int); + int (*voo_mirror_get)(vr_mirror_req *); + + /* NHOP */ + int (*voo_nexthop_add)(struct vr_nexthop *); + int (*voo_nexthop_del)(struct vr_nexthop *); + int (*voo_nexthop_get)(struct vr_nexthop *, vr_nexthop_req *); + + /* route */ + int (*voo_route_add)(vr_route_req *); + int (*voo_route_del)(vr_route_req *); + int (*voo_route_get)(vr_route_req *); + int (*voo_route_dump)(struct vr_route_req *); + + /* QoS */ + int (*voo_fc_map_add)(vr_fc_map_req *); + int (*voo_fc_map_del)(vr_fc_map_req *); + int (*voo_fc_map_get)(vr_fc_map_req *); + + int (*voo_qos_map_add)(vr_qos_map_req *); + int (*voo_qos_map_del)(vr_qos_map_req *); + int (*voo_qos_map_get)(vr_qos_map_req *); +}; + +int vr_offloads_init(struct vrouter *router); +void vr_offloads_exit(struct vrouter *router, bool soft_reset); +struct vr_offload_flow *vr_offloads_flow_get(unsigned int index); + +extern struct vr_offload_ops *offload_ops; + +/* Wrappers for calling offload function with locking in place */ +static inline int vr_offload_soft_reset(void) +{ + if (offload_ops && offload_ops->voo_soft_reset) + return offload_ops->voo_soft_reset(); + return 0; +} + +/* Flow offload functions */ +static inline int vr_offload_flow_set(struct vr_flow_entry * fe, + unsigned int fe_index, + struct vr_flow_entry * rfe) +{ + if (offload_ops && offload_ops->voo_flow_set) + return offload_ops->voo_flow_set(fe, fe_index, rfe); + return 0; +} + +static inline int vr_offload_flow_del(struct vr_flow_entry * fe) +{ + if (offload_ops && offload_ops->voo_flow_del) + return offload_ops->voo_flow_del(fe); + return 0; +} + +/* + * Used both to set and reset meta data entry for a flow. + */ +static inline int vr_offload_flow_meta_data_set(unsigned int fe_index, + unsigned int meta_data_len, + void *meta_data, + unsigned short mir_vrf) +{ + if (offload_ops && offload_ops->voo_flow_meta_data_set) + return offload_ops->voo_flow_meta_data_set(fe_index, meta_data_len, + meta_data, mir_vrf); + return 0; +} + +/* Dropstats */ +static inline int vr_offload_drop_stats_get(vr_drop_stats_req *resp) +{ + if (offload_ops && offload_ops->voo_drop_stats_get) + return offload_ops->voo_drop_stats_get(resp); + return 0; +} + +/* interface offload functions */ +static inline int vr_offload_interface_add(struct vr_interface * intf) +{ + if (offload_ops && offload_ops->voo_interface_add) + return offload_ops->voo_interface_add(intf); + return 0; +} + +static inline int vr_offload_interface_get(vr_interface_req *resp) +{ + if (offload_ops && offload_ops->voo_interface_get) + return offload_ops->voo_interface_get(resp); + return 0; +} + +static inline int vr_offload_interface_del(struct vr_interface * intf) +{ + if (offload_ops && offload_ops->voo_interface_del) + return offload_ops->voo_interface_del(intf); + return 0; +} + +static inline int vr_offload_vif_vrf_set(vr_vrf_assign_req *req) +{ + if (offload_ops && offload_ops->voo_vif_vrf_set) + return offload_ops->voo_vif_vrf_set(req); + return 0; +} + +static inline int vr_offload_vif_vrf_get(vr_vrf_assign_req *resp) +{ + if (offload_ops && offload_ops->voo_vif_vrf_get) + return offload_ops->voo_vif_vrf_get(resp); + return 0; +} + +static inline int vr_offload_nexthop_add(struct vr_nexthop * nh) +{ + if (offload_ops && offload_ops->voo_nexthop_add) + return offload_ops->voo_nexthop_add(nh); + return 0; +} + +static inline int vr_offload_nexthop_del(struct vr_nexthop * nh) +{ + if (offload_ops && offload_ops->voo_nexthop_del) + return offload_ops->voo_nexthop_del(nh); + return 0; +} + +static inline int vr_offload_nexthop_get(struct vr_nexthop * nh, + vr_nexthop_req * resp) +{ + if (offload_ops && offload_ops->voo_nexthop_get) + return offload_ops->voo_nexthop_get(nh, resp); + return 0; +} + +static inline int vr_offload_mpls_add(struct vr_nexthop * nh, int label) +{ + if (offload_ops && offload_ops->voo_mpls_add) + return offload_ops->voo_mpls_add(nh, label); + return 0; +} + +static inline int vr_offload_mpls_get(vr_mpls_req * resp) +{ + if (offload_ops && offload_ops->voo_mpls_get) + return offload_ops->voo_mpls_get(resp); + return 0; +} + +static inline int vr_offload_mpls_del(int label) +{ + if (offload_ops && offload_ops->voo_mpls_del) + return offload_ops->voo_mpls_del(label); + return 0; +} + +static inline int vr_offload_vxlan_add(struct vr_nexthop * nh, int vnid) +{ + if (offload_ops && offload_ops->voo_vxlan_add) + return offload_ops->voo_vxlan_add(nh, vnid); + return 0; +} + +static inline int vr_offload_vxlan_get(vr_vxlan_req * resp) +{ + if (offload_ops && offload_ops->voo_vxlan_get) + return offload_ops->voo_vxlan_get(resp); + return 0; +} + +static inline int vr_offload_vxlan_del(int vnid) +{ + if (offload_ops && offload_ops->voo_vxlan_del) + return offload_ops->voo_vxlan_del(vnid); + return 0; +} + +static inline int vr_offload_mirror_add(struct vr_mirror_entry * mirror, + unsigned int index) +{ + if (offload_ops && offload_ops->voo_mirror_add) + return offload_ops->voo_mirror_add(mirror, index); + return 0; +} + +static inline int vr_offload_mirror_del(unsigned int index) +{ + if (offload_ops && offload_ops->voo_mirror_del) + return offload_ops->voo_mirror_del(index); + return 0; +} + +static inline int vr_offload_mirror_get(vr_mirror_req * resp) +{ + if (offload_ops && offload_ops->voo_mirror_get) + return offload_ops->voo_mirror_get(resp); + return 0; +} + +static inline int vr_offload_route_del(vr_route_req * req) +{ + if (offload_ops && offload_ops->voo_route_del) + return offload_ops->voo_route_del(req); + return 0; +} + +static inline int vr_offload_route_add(vr_route_req * req) +{ + if (offload_ops && offload_ops->voo_route_add) + return offload_ops->voo_route_add(req); + return 0; +} + +static inline int vr_offload_route_get(vr_route_req * req) +{ + if (offload_ops && offload_ops->voo_route_get) + return offload_ops->voo_route_get(req); + return 0; +} + +static inline int vr_offload_route_dump(struct vr_route_req * req) +{ + if (offload_ops && offload_ops->voo_route_dump) + return offload_ops->voo_route_dump(req); + return 0; +} + +static inline int vr_offload_fc_map_add(vr_fc_map_req * req) +{ + if (offload_ops && offload_ops->voo_fc_map_add) + return offload_ops->voo_fc_map_add(req); + return 0; +} + +static inline int vr_offload_fc_map_del(vr_fc_map_req * req) +{ + if (offload_ops && offload_ops->voo_fc_map_del) + return offload_ops->voo_fc_map_del(req); + return 0; +} + +static inline int vr_offload_fc_map_get(vr_fc_map_req * req) +{ + if (offload_ops && offload_ops->voo_fc_map_get) + return offload_ops->voo_fc_map_get(req); + return 0; +} + +static inline int vr_offload_qos_map_add(vr_qos_map_req * req) +{ + if (offload_ops && offload_ops->voo_qos_map_add) + return offload_ops->voo_qos_map_add(req); + return 0; +} + +static inline int vr_offload_qos_map_del(vr_qos_map_req * req) +{ + if (offload_ops && offload_ops->voo_qos_map_del) + return offload_ops->voo_qos_map_del(req); + return 0; +} + +static inline int vr_offload_qos_map_get(vr_qos_map_req * req) +{ + if (offload_ops && offload_ops->voo_qos_map_get) + return offload_ops->voo_qos_map_get(req); + return 0; +} + +#endif /* __VR_OFFLOADS_H__ */ diff --git a/include/vr_packet.h b/include/vr_packet.h index 0629d7c10..c0e1fbbf8 100644 --- a/include/vr_packet.h +++ b/include/vr_packet.h @@ -1019,6 +1019,7 @@ enum { #define FMD_MIRROR_INVALID_DATA 0xFFFF struct vr_forwarding_md { + struct vr_flow_entry *fmd_fe; int32_t fmd_flow_index; int32_t fmd_label; int8_t fmd_ecmp_nh_index; @@ -1040,6 +1041,7 @@ struct vr_forwarding_md { static inline void vr_init_forwarding_md(struct vr_forwarding_md *fmd) { + fmd->fmd_fe = NULL; fmd->fmd_flow_index = -1; fmd->fmd_ecmp_nh_index = -1; fmd->fmd_ecmp_src_nh_index = -1; diff --git a/include/vrouter.h b/include/vrouter.h index 1da864f54..000883ab9 100644 --- a/include/vrouter.h +++ b/include/vrouter.h @@ -23,6 +23,7 @@ extern "C" { #include "vr_mpls.h" #include "vr_index_table.h" #include "vr_mem.h" +#include "vr_offloads.h" #define VR_NATIVE_VRF 0 #define VR_UNIX_PATH_MAX 108 @@ -229,6 +230,10 @@ struct host_os { bool hos_nl_broadcast_supported; int (*hos_huge_page_config)(uint64_t *, int, int *, int); void *(*hos_huge_page_mem_get)(int); + int (*hos_offload_flow_create)(struct vr_offload_flow *oflow); + int (*hos_offload_flow_destroy)(struct vr_offload_flow *oflow); + void (*hos_offload_prepare)(struct vr_packet *pkt, struct vr_forwarding_md *fmd); + }; #define vr_printf vrouter_host->hos_printf @@ -281,6 +286,9 @@ struct host_os { #define vr_nl_broadcast_supported vrouter_host->hos_nl_broadcast_supported #define vr_huge_page_config vrouter_host->hos_huge_page_config #define vr_huge_page_mem_get vrouter_host->hos_huge_page_mem_get +#define vr_offload_flow_destroy vrouter_host->hos_offload_flow_destroy +#define vr_offload_flow_create vrouter_host->hos_offload_flow_create +#define vr_offload_prepare vrouter_host->hos_offload_prepare extern struct host_os *vrouter_host; diff --git a/utils/dropstats.c b/utils/dropstats.c index d18515d28..ea3c4554c 100644 --- a/utils/dropstats.c +++ b/utils/dropstats.c @@ -27,13 +27,91 @@ #include "ini_parser.h" static struct nl_client *cl; -static int help_set, core_set; +static int help_set, core_set, offload_set; static unsigned int core = (unsigned)-1; static void drop_stats_req_process(void *s_req) { vr_drop_stats_req *stats = (vr_drop_stats_req *)s_req; + int platform = get_platform(); + if (core == (unsigned)-2) + printf("Statistics for NIC offloads\n\n"); + else if (core != (unsigned)-1) + printf("Statistics for core %u\n\n", core); + printf("Invalid ARPs %" PRIu64 "\n", + stats->vds_invalid_arp); + printf("\n"); + + printf("Invalid IF %" PRIu64 "\n", + stats->vds_invalid_if); + printf("Trap No IF %" PRIu64 "\n", + stats->vds_trap_no_if); + printf("IF TX Discard %" PRIu64 "\n", + stats->vds_interface_tx_discard); + printf("IF Drop %" PRIu64 "\n", + stats->vds_interface_drop); + printf("IF RX Discard %" PRIu64 "\n", + stats->vds_interface_rx_discard); + printf("\n"); + + printf("Flow Unusable %" PRIu64 "\n", + stats->vds_flow_unusable); + printf("Flow No Memory %" PRIu64 "\n", + stats->vds_flow_no_memory); + printf("Flow Table Full %" PRIu64 "\n", + stats->vds_flow_table_full); + printf("Flow NAT no rflow %" PRIu64 "\n", + stats->vds_flow_nat_no_rflow); + printf("Flow Action Drop %" PRIu64 "\n", + stats->vds_flow_action_drop); + printf("Flow Action Invalid %" PRIu64 "\n", + stats->vds_flow_action_invalid); + printf("Flow Invalid Protocol %" PRIu64 "\n", + stats->vds_flow_invalid_protocol); + printf("Flow Queue Limit Exceeded %" PRIu64 "\n", + stats->vds_flow_queue_limit_exceeded); + printf("New Flow Drops %" PRIu64 "\n", + stats->vds_drop_new_flow); + printf("Flow Unusable (Eviction) %" PRIu64 "\n", + stats->vds_flow_evict); + printf("\n"); + + printf("Discards %" PRIu64 "\n", + stats->vds_discard); + printf("TTL Exceeded %" PRIu64 "\n", + stats->vds_ttl_exceeded); + printf("Mcast Clone Fail %" PRIu64 "\n", + stats->vds_mcast_clone_fail); + printf("Cloned Original %" PRIu64 "\n", + stats->vds_cloned_original); + printf("\n"); + + printf("Invalid NH %" PRIu64 "\n", + stats->vds_invalid_nh); + printf("Invalid Label %" PRIu64 "\n", + stats->vds_invalid_label); + printf("Invalid Protocol %" PRIu64 "\n", + stats->vds_invalid_protocol); + printf("Rewrite Fail %" PRIu64 "\n", + stats->vds_rewrite_fail); + printf("Invalid Mcast Source %" PRIu64 "\n", + stats->vds_invalid_mcast_source); + printf("\n"); + + printf("Push Fails %" PRIu64 "\n", + stats->vds_push); + printf("Pull Fails %" PRIu64 "\n", + stats->vds_pull); + printf("Duplicated %" PRIu64 "\n", + stats->vds_duplicated); + printf("Head Alloc Fails %" PRIu64 "\n", + stats->vds_head_alloc_fail); + printf("PCOW fails %" PRIu64 "\n", + stats->vds_pcow_fail); + printf("Invalid Packets %" PRIu64 "\n", + stats->vds_invalid_packet); + printf("\n"); vr_print_drop_stats(stats, core); return; @@ -66,6 +144,11 @@ vr_get_drop_stats(struct nl_client *cl) * conversion between those enumerating systems. The dropstats utility * increments by 1 the core number user asked for. Then it is * decremented back in vRouter. + * + * vRouter will return only the offloaded dropstats if the "core" + * is passed in as -2. This allows returning of only dropstats offloaded + * on NIC using this same mechanism. If all CPUs are requested, the + * offloaded dropstats are included. */ ret = vr_send_drop_stats_get(cl, 0, core + 1); if (ret < 0) @@ -81,21 +164,28 @@ vr_get_drop_stats(struct nl_client *cl) enum opt_index { HELP_OPT_INDEX, CORE_OPT_INDEX, + OFFL_OPT_INDEX, MAX_OPT_INDEX, }; static struct option long_options[] = { [HELP_OPT_INDEX] = {"help", no_argument, &help_set, 1}, [CORE_OPT_INDEX] = {"core", required_argument, &core_set, 1}, - [MAX_OPT_INDEX] = {NULL, 0, 0, 0}, + [OFFL_OPT_INDEX] = {"offload", no_argument, &offload_set, 1}, + [MAX_OPT_INDEX] = {"NULL", 0, 0, 0}, }; static void Usage() { printf("Usage: dropstats [--help]\n"); - printf("Usage: dropstats [--core|-c] \n\n"); + printf("Usage: dropstats [--core|-c] %s\n\n", + get_offload_enabled()?"[--offload|-o]":""); printf("--core \t Show statistics for a specified CPU core\n"); + if (get_offload_enabled()) { + printf("--offload\t\t Show statistics for pkts offloaded on NIC\n"); + printf("\t\t\t (offload stats included if no flags given)\n"); + } exit(-EINVAL); } @@ -113,7 +203,13 @@ parse_long_opts(int opt_index, char *opt_arg) Usage(); } break; - + case OFFL_OPT_INDEX: + if (!get_offload_enabled()) { + printf("Error: hardware offloads not enabled\n"); + Usage(); + } + core = -2; + break; case HELP_OPT_INDEX: default: Usage(); @@ -130,7 +226,9 @@ main(int argc, char *argv[]) dropstats_fill_nl_callbacks(); - while (((opt = getopt_long(argc, argv, "h:c:", + parse_ini_file(); + + while (((opt = getopt_long(argc, argv, "h:c:o", long_options, &option_index)) >= 0)) { switch (opt) { case 'c': @@ -138,6 +236,11 @@ main(int argc, char *argv[]) parse_long_opts(CORE_OPT_INDEX, optarg); break; + case 'o': + offload_set = 1; + parse_long_opts(OFFL_OPT_INDEX, optarg); + break; + case 0: parse_long_opts(option_index, optarg); break; diff --git a/utils/ini_parser.c b/utils/ini_parser.c index f66181c59..5b02bb280 100644 --- a/utils/ini_parser.c +++ b/utils/ini_parser.c @@ -251,3 +251,14 @@ get_platform_str(void) { return read_string(DEFAULT_SECTION, PLATFORM_KEY); } + +bool +get_offload_enabled(void) +{ + const char *offload_str = read_string(DEFAULT_SECTION, OFFLOAD_KEY); + if (offload_str) { + if (!strcmp(offload_str, OFFLOAD_ENABLED)) + return true; + } + return false; +} diff --git a/windows/vRouter.vcxproj b/windows/vRouter.vcxproj index 8276deba8..1195df061 100644 --- a/windows/vRouter.vcxproj +++ b/windows/vRouter.vcxproj @@ -126,6 +126,7 @@ + @@ -170,6 +171,7 @@ + diff --git a/windows/vRouter.vcxproj.filters b/windows/vRouter.vcxproj.filters index b1c327487..cf6103bba 100644 --- a/windows/vRouter.vcxproj.filters +++ b/windows/vRouter.vcxproj.filters @@ -100,6 +100,9 @@ Header Files + + Header Files + Header Files @@ -255,6 +258,9 @@ Source Files + + Source Files + Source Files