From b1ba5766ee7c22145831fcfe4aab07650ebdf194 Mon Sep 17 00:00:00 2001 From: Petri Savolainen Date: Thu, 8 Feb 2018 12:28:28 +0200 Subject: [PATCH 1/4] linux-gen: packet: optimize parse ipv4 and udp Optimize IPv4 and UDP parse functions. Signed-off-by: Petri Savolainen --- platform/linux-generic/odp_packet.c | 35 ++++++++++++++--------------- 1 file changed, 17 insertions(+), 18 deletions(-) diff --git a/platform/linux-generic/odp_packet.c b/platform/linux-generic/odp_packet.c index 9c3be1bccd..92a64801f3 100644 --- a/platform/linux-generic/odp_packet.c +++ b/platform/linux-generic/odp_packet.c @@ -1988,15 +1988,15 @@ static inline uint8_t parse_ipv4(packet_parser_t *prs, const uint8_t **parseptr, uint32_t *offset, uint32_t frame_len) { const _odp_ipv4hdr_t *ipv4 = (const _odp_ipv4hdr_t *)*parseptr; - uint8_t ver = _ODP_IPV4HDR_VER(ipv4->ver_ihl); - uint8_t ihl = _ODP_IPV4HDR_IHL(ipv4->ver_ihl); - uint16_t frag_offset; uint32_t dstaddr = _odp_be_to_cpu_32(ipv4->dst_addr); uint32_t l3_len = _odp_be_to_cpu_16(ipv4->tot_len); + uint16_t frag_offset = _odp_be_to_cpu_16(ipv4->frag_offset); + uint8_t ver = _ODP_IPV4HDR_VER(ipv4->ver_ihl); + uint8_t ihl = _ODP_IPV4HDR_IHL(ipv4->ver_ihl); - if (odp_unlikely(ihl < _ODP_IPV4HDR_IHL_MIN) || - odp_unlikely(ver != 4) || - (l3_len > frame_len - *offset)) { + if (odp_unlikely(ihl < _ODP_IPV4HDR_IHL_MIN || + ver != 4 || + (l3_len > frame_len - *offset))) { prs->error_flags.ip_err = 1; return 0; } @@ -2012,13 +2012,15 @@ static inline uint8_t parse_ipv4(packet_parser_t *prs, const uint8_t **parseptr, * OR * "fragment offset" field is nonzero (all fragments except the first) */ - frag_offset = _odp_be_to_cpu_16(ipv4->frag_offset); if (odp_unlikely(_ODP_IPV4HDR_IS_FRAGMENT(frag_offset))) prs->input_flags.ipfrag = 1; /* Handle IPv4 broadcast / multicast */ - prs->input_flags.ip_bcast = (dstaddr == 0xffffffff); - prs->input_flags.ip_mcast = (dstaddr >> 28) == 0xd; + if (odp_unlikely(dstaddr == 0xffffffff)) + prs->input_flags.ip_bcast = 1; + + if (odp_unlikely((dstaddr >> 28) == 0xd)) + prs->input_flags.ip_mcast = 1; return ipv4->proto; } @@ -2107,17 +2109,16 @@ static inline void parse_tcp(packet_parser_t *prs, /** * Parser helper function for UDP */ -static inline void parse_udp(packet_parser_t *prs, - const uint8_t **parseptr, uint32_t *offset) +static inline void parse_udp(packet_parser_t *prs, const uint8_t **parseptr) { const _odp_udphdr_t *udp = (const _odp_udphdr_t *)*parseptr; uint32_t udplen = _odp_be_to_cpu_16(udp->length); + uint16_t ipsec_port = _odp_cpu_to_be_16(_ODP_UDP_IPSEC_PORT); if (odp_unlikely(udplen < sizeof(_odp_udphdr_t))) prs->error_flags.udp_err = 1; - if (_odp_cpu_to_be_16(_ODP_UDP_IPSEC_PORT) == udp->dst_port && - udplen > 4) { + if (odp_unlikely(ipsec_port == udp->dst_port && udplen > 4)) { uint32_t val; memcpy(&val, udp + 1, 4); @@ -2127,8 +2128,6 @@ static inline void parse_udp(packet_parser_t *prs, } } - if (offset) - *offset += sizeof(_odp_udphdr_t); *parseptr += sizeof(_odp_udphdr_t); } @@ -2142,7 +2141,7 @@ int packet_parse_common_l3_l4(packet_parser_t *prs, const uint8_t *parseptr, prs->l3_offset = offset; - if (layer <= ODP_PROTO_LAYER_L2) + if (odp_unlikely(layer <= ODP_PROTO_LAYER_L2)) return prs->error_flags.all != 0; /* Set l3 flag only for known ethtypes */ @@ -2203,7 +2202,7 @@ int packet_parse_common_l3_l4(packet_parser_t *prs, const uint8_t *parseptr, if (odp_unlikely(offset + _ODP_UDPHDR_LEN > seg_len)) return -1; prs->input_flags.udp = 1; - parse_udp(prs, &parseptr, NULL); + parse_udp(prs, &parseptr); break; case _ODP_IPPROTO_AH: @@ -2232,7 +2231,7 @@ int packet_parse_common_l3_l4(packet_parser_t *prs, const uint8_t *parseptr, * Parse common packet headers up to given layer * * The function expects at least PACKET_PARSE_SEG_LEN bytes of data to be - * available from the ptr. + * available from the ptr. Also parse metadata must be already initialized. */ int packet_parse_common(packet_parser_t *prs, const uint8_t *ptr, uint32_t frame_len, uint32_t seg_len, From 9a955bc946f5a8d5e4c1a5309b1ca133eb37552d Mon Sep 17 00:00:00 2001 From: Petri Savolainen Date: Thu, 8 Feb 2018 13:04:30 +0200 Subject: [PATCH 2/4] linux-gen: packet: optimize tcp parse Signed-off-by: Petri Savolainen --- .../include/odp/api/plat/packet_inline_types.h | 1 - platform/linux-generic/odp_packet.c | 14 +++++--------- 2 files changed, 5 insertions(+), 10 deletions(-) diff --git a/platform/linux-generic/include/odp/api/plat/packet_inline_types.h b/platform/linux-generic/include/odp/api/plat/packet_inline_types.h index 47ec821c9c..984783d5d3 100644 --- a/platform/linux-generic/include/odp/api/plat/packet_inline_types.h +++ b/platform/linux-generic/include/odp/api/plat/packet_inline_types.h @@ -101,7 +101,6 @@ typedef union { payload */ uint64_t udp:1; /**< UDP */ uint64_t tcp:1; /**< TCP */ - uint64_t tcpopt:1; /**< TCP options present */ uint64_t sctp:1; /**< SCTP */ uint64_t icmp:1; /**< ICMP */ diff --git a/platform/linux-generic/odp_packet.c b/platform/linux-generic/odp_packet.c index 92a64801f3..480cea1949 100644 --- a/platform/linux-generic/odp_packet.c +++ b/platform/linux-generic/odp_packet.c @@ -2091,19 +2091,15 @@ static inline uint8_t parse_ipv6(packet_parser_t *prs, const uint8_t **parseptr, /** * Parser helper function for TCP */ -static inline void parse_tcp(packet_parser_t *prs, - const uint8_t **parseptr, uint32_t *offset) +static inline void parse_tcp(packet_parser_t *prs, const uint8_t **parseptr) { const _odp_tcphdr_t *tcp = (const _odp_tcphdr_t *)*parseptr; + uint32_t len = tcp->hl * 4; - if (tcp->hl < sizeof(_odp_tcphdr_t) / sizeof(uint32_t)) + if (odp_unlikely(tcp->hl < sizeof(_odp_tcphdr_t) / sizeof(uint32_t))) prs->error_flags.tcp_err = 1; - else if ((uint32_t)tcp->hl * 4 > sizeof(_odp_tcphdr_t)) - prs->input_flags.tcpopt = 1; - if (offset) - *offset += (uint32_t)tcp->hl * 4; - *parseptr += (uint32_t)tcp->hl * 4; + *parseptr += len; } /** @@ -2195,7 +2191,7 @@ int packet_parse_common_l3_l4(packet_parser_t *prs, const uint8_t *parseptr, if (odp_unlikely(offset + _ODP_TCPHDR_LEN > seg_len)) return -1; prs->input_flags.tcp = 1; - parse_tcp(prs, &parseptr, NULL); + parse_tcp(prs, &parseptr); break; case _ODP_IPPROTO_UDP: From f6377bd4b7870a89e4546d484072bea7fbb9947f Mon Sep 17 00:00:00 2001 From: Petri Savolainen Date: Wed, 7 Feb 2018 14:05:08 +0200 Subject: [PATCH 3/4] linux-gen: packet: optimize parse eth Optimize parse ethernet function. Signed-off-by: Petri Savolainen --- platform/linux-generic/odp_packet.c | 46 ++++++++++++++++------------- 1 file changed, 26 insertions(+), 20 deletions(-) diff --git a/platform/linux-generic/odp_packet.c b/platform/linux-generic/odp_packet.c index 480cea1949..b2a22903c6 100644 --- a/platform/linux-generic/odp_packet.c +++ b/platform/linux-generic/odp_packet.c @@ -1917,28 +1917,33 @@ static inline uint16_t parse_eth(packet_parser_t *prs, const uint8_t **parseptr, const _odp_ethhdr_t *eth; uint16_t macaddr0, macaddr2, macaddr4; const _odp_vlanhdr_t *vlan; + _odp_packet_input_flags_t input_flags; - /* Detect jumbo frames */ - if (frame_len > _ODP_ETH_LEN_MAX) - prs->input_flags.jumbo = 1; + input_flags.all = 0; + input_flags.l2 = 1; + input_flags.eth = 1; eth = (const _odp_ethhdr_t *)*parseptr; + /* Detect jumbo frames */ + if (odp_unlikely(frame_len > _ODP_ETH_LEN_MAX)) + input_flags.jumbo = 1; + /* Handle Ethernet broadcast/multicast addresses */ macaddr0 = _odp_be_to_cpu_16(*((const uint16_t *)(const void *)eth)); - prs->input_flags.eth_mcast = (macaddr0 & 0x0100) == 0x0100; + if (odp_unlikely((macaddr0 & 0x0100) == 0x0100)) + input_flags.eth_mcast = 1; - if (macaddr0 == 0xffff) { + if (odp_unlikely(macaddr0 == 0xffff)) { macaddr2 = _odp_be_to_cpu_16(*((const uint16_t *) (const void *)eth + 1)); macaddr4 = _odp_be_to_cpu_16(*((const uint16_t *) (const void *)eth + 2)); - prs->input_flags.eth_bcast = - (macaddr2 == 0xffff) && (macaddr4 == 0xffff); - } else { - prs->input_flags.eth_bcast = 0; + + if ((macaddr2 == 0xffff) && (macaddr4 == 0xffff)) + input_flags.eth_bcast = 1; } /* Get Ethertype */ @@ -1947,11 +1952,12 @@ static inline uint16_t parse_eth(packet_parser_t *prs, const uint8_t **parseptr, *parseptr += sizeof(*eth); /* Check for SNAP vs. DIX */ - if (ethtype < _ODP_ETH_LEN_MAX) { - prs->input_flags.snap = 1; + if (odp_unlikely(ethtype < _ODP_ETH_LEN_MAX)) { + input_flags.snap = 1; if (ethtype > frame_len - *offset) { prs->error_flags.snap_len = 1; - return 0; + ethtype = 0; + goto error; } ethtype = _odp_be_to_cpu_16(*((const uint16_t *)(uintptr_t) (parseptr + 6))); @@ -1960,9 +1966,9 @@ static inline uint16_t parse_eth(packet_parser_t *prs, const uint8_t **parseptr, } /* Parse the VLAN header(s), if present */ - if (ethtype == _ODP_ETHTYPE_VLAN_OUTER) { - prs->input_flags.vlan_qinq = 1; - prs->input_flags.vlan = 1; + if (odp_unlikely(ethtype == _ODP_ETHTYPE_VLAN_OUTER)) { + input_flags.vlan_qinq = 1; + input_flags.vlan = 1; vlan = (const _odp_vlanhdr_t *)*parseptr; ethtype = _odp_be_to_cpu_16(vlan->type); @@ -1971,13 +1977,16 @@ static inline uint16_t parse_eth(packet_parser_t *prs, const uint8_t **parseptr, } if (ethtype == _ODP_ETHTYPE_VLAN) { - prs->input_flags.vlan = 1; + input_flags.vlan = 1; vlan = (const _odp_vlanhdr_t *)*parseptr; ethtype = _odp_be_to_cpu_16(vlan->type); *offset += sizeof(_odp_vlanhdr_t); *parseptr += sizeof(_odp_vlanhdr_t); } +error: + prs->input_flags.all |= input_flags.all; + return ethtype; } @@ -2240,14 +2249,11 @@ int packet_parse_common(packet_parser_t *prs, const uint8_t *ptr, parseptr = ptr; offset = 0; - if (layer == ODP_PROTO_LAYER_NONE) + if (odp_unlikely(layer == ODP_PROTO_LAYER_NONE)) return 0; /* Assume valid L2 header, no CRC/FCS check in SW */ prs->l2_offset = offset; - prs->input_flags.l2 = 1; - /* We only support Ethernet for now */ - prs->input_flags.eth = 1; ethtype = parse_eth(prs, &parseptr, &offset, frame_len); From 5ba9c91812fcdcfd950a45519f951562f9e7eff9 Mon Sep 17 00:00:00 2001 From: Petri Savolainen Date: Thu, 8 Feb 2018 15:06:56 +0200 Subject: [PATCH 4/4] linux-gen: dpdk: optimize mbuf_to_pkt Optimize mbuf to packet functions. Read data from pktio structure only once. Prefetch data also in zero copy mode. Signed-off-by: Petri Savolainen --- platform/linux-generic/pktio/dpdk.c | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/platform/linux-generic/pktio/dpdk.c b/platform/linux-generic/pktio/dpdk.c index bd6920e6a0..643a8e9b00 100644 --- a/platform/linux-generic/pktio/dpdk.c +++ b/platform/linux-generic/pktio/dpdk.c @@ -433,6 +433,8 @@ static inline int mbuf_to_pkt(pktio_entry_t *pktio_entry, int alloc_len, num; odp_pool_t pool = pktio_entry->s.pkt_dpdk.pool; odp_pktin_config_opt_t *pktin_cfg = &pktio_entry->s.config.pktin; + odp_proto_layer_t parse_layer = pktio_entry->s.config.parser.layer; + odp_pktio_t input = pktio_entry->s.handle; /* Allocate maximum sized packets */ alloc_len = pktio_entry->s.pkt_dpdk.data_room; @@ -474,13 +476,12 @@ static inline int mbuf_to_pkt(pktio_entry_t *pktio_entry, if (_odp_packet_copy_from_mem(pkt, 0, pkt_len, data) != 0) goto fail; - pkt_hdr->input = pktio_entry->s.handle; + pkt_hdr->input = input; if (pktio_cls_enabled(pktio_entry)) copy_packet_cls_metadata(&parsed_hdr, pkt_hdr); - else if (pktio_entry->s.config.parser.layer) - packet_parse_layer(pkt_hdr, - pktio_entry->s.config.parser.layer); + else if (parse_layer != ODP_PROTO_LAYER_NONE) + packet_parse_layer(pkt_hdr, parse_layer); if (mbuf->ol_flags & PKT_RX_RSS_HASH) packet_set_flow_hash(pkt_hdr, mbuf->hash.rss); @@ -682,6 +683,8 @@ static inline int mbuf_to_pkt_zero(pktio_entry_t *pktio_entry, int nb_pkts = 0; odp_pool_t pool = pktio_entry->s.pkt_dpdk.pool; odp_pktin_config_opt_t *pktin_cfg = &pktio_entry->s.config.pktin; + odp_proto_layer_t parse_layer = pktio_entry->s.config.parser.layer; + odp_pktio_t input = pktio_entry->s.handle; for (i = 0; i < mbuf_num; i++) { odp_packet_hdr_t parsed_hdr; @@ -694,6 +697,8 @@ static inline int mbuf_to_pkt_zero(pktio_entry_t *pktio_entry, } data = rte_pktmbuf_mtod(mbuf, char *); + odp_prefetch(data); + pkt_len = rte_pktmbuf_pkt_len(mbuf); pkt = (odp_packet_t)mbuf->userdata; @@ -714,13 +719,12 @@ static inline int mbuf_to_pkt_zero(pktio_entry_t *pktio_entry, pkt_hdr->buf_hdr.seg[0].data = data; packet_init(pkt_hdr, pkt_len); - pkt_hdr->input = pktio_entry->s.handle; + pkt_hdr->input = input; if (pktio_cls_enabled(pktio_entry)) copy_packet_cls_metadata(&parsed_hdr, pkt_hdr); - else if (pktio_entry->s.config.parser.layer) - packet_parse_layer(pkt_hdr, - pktio_entry->s.config.parser.layer); + else if (parse_layer != ODP_PROTO_LAYER_NONE) + packet_parse_layer(pkt_hdr, parse_layer); if (mbuf->ol_flags & PKT_RX_RSS_HASH) packet_set_flow_hash(pkt_hdr, mbuf->hash.rss);