From 119fe2837ce2e3c2d031be8b989cab1743f93338 Mon Sep 17 00:00:00 2001 From: Brian Brooks Date: Thu, 5 Oct 2017 13:01:18 -0500 Subject: [PATCH 1/4] linux-gen: packet: remove duplicated odp_packet_hdr() function odp_packet_hdr() and packet_hdr() are both internal-only functions that do the same exact thing. Remove odp_packet_hdr() version since "odp_" prefix is reserved for top-level API identifiers. This is also consistent with other packet_xxx() internal-only functions. Signed-off-by: Brian Brooks Reviewed-by: Yi He Reviewed-by: Ola Liljedahl Reviewed-by: Honnappa Nagarahalli Reviewed-by: Kevin Wang --- .../linux-dpdk/include/odp_packet_internal.h | 5 +++++ .../include/odp_packet_internal.h | 4 ++-- platform/linux-generic/odp_crypto.c | 2 +- platform/linux-generic/odp_ipsec.c | 6 ++--- platform/linux-generic/odp_packet.c | 5 ----- platform/linux-generic/odp_packet_flags.c | 22 +++++++++---------- platform/linux-generic/odp_packet_io.c | 4 ++-- platform/linux-generic/pktio/dpdk.c | 12 +++++----- platform/linux-generic/pktio/ipc.c | 14 ++++++------ platform/linux-generic/pktio/loopback.c | 2 +- platform/linux-generic/pktio/netmap.c | 2 +- platform/linux-generic/pktio/pcap.c | 2 +- platform/linux-generic/pktio/socket.c | 2 +- platform/linux-generic/pktio/socket_mmap.c | 2 +- platform/linux-generic/pktio/tap.c | 2 +- 15 files changed, 43 insertions(+), 43 deletions(-) diff --git a/platform/linux-dpdk/include/odp_packet_internal.h b/platform/linux-dpdk/include/odp_packet_internal.h index 16a583346a..605fff9ddc 100644 --- a/platform/linux-dpdk/include/odp_packet_internal.h +++ b/platform/linux-dpdk/include/odp_packet_internal.h @@ -141,6 +141,11 @@ static inline odp_packet_hdr_t *odp_packet_hdr(odp_packet_t pkt) return (odp_packet_hdr_t *)(uintptr_t)pkt; } + static inline odp_packet_hdr_t *packet_hdr(odp_packet_t pkt) +{ + return (odp_packet_hdr_t *)(uintptr_t)pkt; +} + static inline struct rte_mbuf *pkt_to_mbuf(odp_packet_hdr_t *pkt_hdr) { return &pkt_hdr->buf_hdr.mb; diff --git a/platform/linux-generic/include/odp_packet_internal.h b/platform/linux-generic/include/odp_packet_internal.h index fed562aa34..3a86ce9284 100644 --- a/platform/linux-generic/include/odp_packet_internal.h +++ b/platform/linux-generic/include/odp_packet_internal.h @@ -162,7 +162,7 @@ typedef struct { /** * Return the packet header */ -static inline odp_packet_hdr_t *odp_packet_hdr(odp_packet_t pkt) +static inline odp_packet_hdr_t *packet_hdr(odp_packet_t pkt) { return (odp_packet_hdr_t *)(uintptr_t)pkt; } @@ -174,7 +174,7 @@ static inline odp_packet_t packet_handle(odp_packet_hdr_t *pkt_hdr) static inline odp_buffer_hdr_t *packet_to_buf_hdr(odp_packet_t pkt) { - return &odp_packet_hdr(pkt)->buf_hdr; + return &packet_hdr(pkt)->buf_hdr; } static inline odp_packet_t packet_from_buf_hdr(odp_buffer_hdr_t *buf_hdr) diff --git a/platform/linux-generic/odp_crypto.c b/platform/linux-generic/odp_crypto.c index 3174feee09..2a85124ee9 100644 --- a/platform/linux-generic/odp_crypto.c +++ b/platform/linux-generic/odp_crypto.c @@ -1127,7 +1127,7 @@ odp_event_t odp_crypto_packet_to_event(odp_packet_t pkt) static odp_crypto_packet_result_t *get_op_result_from_packet(odp_packet_t pkt) { - odp_packet_hdr_t *hdr = odp_packet_hdr(pkt); + odp_packet_hdr_t *hdr = packet_hdr(pkt); return &hdr->crypto_op_result; } diff --git a/platform/linux-generic/odp_ipsec.c b/platform/linux-generic/odp_ipsec.c index e57736c2a7..23f86f69ac 100644 --- a/platform/linux-generic/odp_ipsec.c +++ b/platform/linux-generic/odp_ipsec.c @@ -98,7 +98,7 @@ static odp_ipsec_packet_result_t *ipsec_pkt_result(odp_packet_t packet) ODP_ASSERT(ODP_EVENT_PACKET_IPSEC == odp_event_subtype(odp_packet_to_event(packet))); - return &odp_packet_hdr(packet)->ipsec_ctx; + return &packet_hdr(packet)->ipsec_ctx; } /** @@ -541,7 +541,7 @@ static ipsec_sa_t *ipsec_in_single(odp_packet_t pkt, } if (!status->error.all) { - odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); + odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); packet_parse_reset(pkt_hdr); @@ -1120,7 +1120,7 @@ int _odp_ipsec_try_inline(odp_packet_t pkt) result->status = status; result->sa = ipsec_sa->ipsec_sa_hdl; - pkt_hdr = odp_packet_hdr(pkt); + pkt_hdr = packet_hdr(pkt); pkt_hdr->p.input_flags.dst_queue = 1; pkt_hdr->dst_queue = queue_fn->from_ext(ipsec_sa->queue); diff --git a/platform/linux-generic/odp_packet.c b/platform/linux-generic/odp_packet.c index cb20eeda81..239506e10b 100644 --- a/platform/linux-generic/odp_packet.c +++ b/platform/linux-generic/odp_packet.c @@ -46,11 +46,6 @@ const _odp_packet_inline_offset_t _odp_packet_inline ODP_ALIGNED_CACHE = { #include -static inline odp_packet_hdr_t *packet_hdr(odp_packet_t pkt) -{ - return (odp_packet_hdr_t *)(uintptr_t)pkt; -} - static inline odp_buffer_t buffer_handle(odp_packet_hdr_t *pkt_hdr) { return (odp_buffer_t)pkt_hdr; diff --git a/platform/linux-generic/odp_packet_flags.c b/platform/linux-generic/odp_packet_flags.c index 2e26ad6017..5beae933d1 100644 --- a/platform/linux-generic/odp_packet_flags.c +++ b/platform/linux-generic/odp_packet_flags.c @@ -11,18 +11,18 @@ #include #define retflag(pkt, x) do { \ - odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); \ + odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); \ return pkt_hdr->p.x; \ } while (0) #define setflag(pkt, x, v) do { \ - odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); \ + odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); \ pkt_hdr->p.x = (v) & 1; \ } while (0) int odp_packet_has_error(odp_packet_t pkt) { - odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); + odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); return pkt_hdr->p.error_flags.all != 0; } @@ -31,7 +31,7 @@ int odp_packet_has_error(odp_packet_t pkt) int odp_packet_has_l2_error(odp_packet_t pkt) { - odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); + odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); /* L2 parsing is always done by default and hence no additional check is required */ return pkt_hdr->p.error_flags.frame_len @@ -46,7 +46,7 @@ int odp_packet_has_l3(odp_packet_t pkt) int odp_packet_has_l3_error(odp_packet_t pkt) { - odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); + odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); return pkt_hdr->p.error_flags.ip_err; } @@ -58,7 +58,7 @@ int odp_packet_has_l4(odp_packet_t pkt) int odp_packet_has_l4_error(odp_packet_t pkt) { - odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); + odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); return pkt_hdr->p.error_flags.tcp_err | pkt_hdr->p.error_flags.udp_err; } @@ -150,14 +150,14 @@ odp_packet_color_t odp_packet_color(odp_packet_t pkt) void odp_packet_color_set(odp_packet_t pkt, odp_packet_color_t color) { - odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); + odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); pkt_hdr->p.input_flags.color = color; } odp_bool_t odp_packet_drop_eligible(odp_packet_t pkt) { - odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); + odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); return !pkt_hdr->p.input_flags.nodrop; } @@ -174,7 +174,7 @@ int8_t odp_packet_shaper_len_adjust(odp_packet_t pkt) void odp_packet_shaper_len_adjust_set(odp_packet_t pkt, int8_t adj) { - odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); + odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); pkt_hdr->p.output_flags.shaper_len_adj = adj; } @@ -288,14 +288,14 @@ void odp_packet_has_icmp_set(odp_packet_t pkt, int val) void odp_packet_has_flow_hash_clr(odp_packet_t pkt) { - odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); + odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); pkt_hdr->p.input_flags.flow_hash = 0; } void odp_packet_has_ts_clr(odp_packet_t pkt) { - odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); + odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); pkt_hdr->p.input_flags.timestamp = 0; } diff --git a/platform/linux-generic/odp_packet_io.c b/platform/linux-generic/odp_packet_io.c index efeac02694..671422fae5 100644 --- a/platform/linux-generic/odp_packet_io.c +++ b/platform/linux-generic/odp_packet_io.c @@ -553,7 +553,7 @@ static inline int pktin_recv_buf(odp_pktin_queue_t queue, for (i = 0; i < pkts; i++) { pkt = packets[i]; - pkt_hdr = odp_packet_hdr(pkt); + pkt_hdr = packet_hdr(pkt); buf_hdr = packet_to_buf_hdr(pkt); if (pkt_hdr->p.input_flags.dst_queue) { @@ -681,7 +681,7 @@ int pktin_poll_one(int pktio_index, num_rx = 0; for (i = 0; i < num_pkts; i++) { pkt = packets[i]; - pkt_hdr = odp_packet_hdr(pkt); + pkt_hdr = packet_hdr(pkt); if (odp_unlikely(pkt_hdr->p.input_flags.dst_queue)) { queue = pkt_hdr->dst_queue; buf_hdr = packet_to_buf_hdr(pkt); diff --git a/platform/linux-generic/pktio/dpdk.c b/platform/linux-generic/pktio/dpdk.c index 7d39bb1bc4..bdd12292fc 100644 --- a/platform/linux-generic/pktio/dpdk.c +++ b/platform/linux-generic/pktio/dpdk.c @@ -244,7 +244,7 @@ static int pool_dequeue_bulk(struct rte_mempool *mp, void **obj_table, for (i = 0; i < pkts; i++) { odp_packet_t pkt = packet_tbl[i]; - odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); + odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); struct rte_mbuf *mbuf = (struct rte_mbuf *) (uintptr_t)pkt_hdr->extra; if (pkt_hdr->extra_type != PKT_EXTRA_TYPE_DPDK) @@ -428,7 +428,7 @@ static inline int mbuf_to_pkt(pktio_entry_t *pktio_entry, } pkt = pkt_table[i]; - pkt_hdr = odp_packet_hdr(pkt); + pkt_hdr = packet_hdr(pkt); pull_tail(pkt_hdr, alloc_len - pkt_len); if (odp_packet_copy_from_mem(pkt, 0, pkt_len, data) != 0) @@ -591,7 +591,7 @@ static inline int pkt_to_mbuf(pktio_entry_t *pktio_entry, if (pktout_cfg->all_bits) pkt_set_ol_tx(pktout_cfg, - odp_packet_hdr(pkt_table[i]), + packet_hdr(pkt_table[i]), mbuf_table[i], data); } return i; @@ -633,7 +633,7 @@ static inline int mbuf_to_pkt_zero(pktio_entry_t *pktio_entry, pkt_len = rte_pktmbuf_pkt_len(mbuf); pkt = (odp_packet_t)mbuf->userdata; - pkt_hdr = odp_packet_hdr(pkt); + pkt_hdr = packet_hdr(pkt); if (pktio_cls_enabled(pktio_entry)) { if (cls_classify_packet(pktio_entry, @@ -688,7 +688,7 @@ static inline int pkt_to_mbuf_zero(pktio_entry_t *pktio_entry, for (i = 0; i < num; i++) { odp_packet_t pkt = pkt_table[i]; - odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); + odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); struct rte_mbuf *mbuf = (struct rte_mbuf *) (uintptr_t)pkt_hdr->extra; uint16_t pkt_len = odp_packet_len(pkt); @@ -1481,7 +1481,7 @@ static int dpdk_send(pktio_entry_t *pktio_entry, int index, for (i = 0; i < mbufs && freed != copy_count; i++) { odp_packet_t pkt = pkt_table[i]; - odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); + odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); if (pkt_hdr->buf_hdr.segcount > 1) { if (odp_likely(i < tx_pkts)) diff --git a/platform/linux-generic/pktio/ipc.c b/platform/linux-generic/pktio/ipc.c index 9d928c72a9..64c7606dfd 100644 --- a/platform/linux-generic/pktio/ipc.c +++ b/platform/linux-generic/pktio/ipc.c @@ -532,14 +532,14 @@ static int ipc_pktio_recv_lockless(pktio_entry_t *pktio_entry, memcpy(pkt_data, rmt_data_ptr, phdr->frame_len); /* Copy packets L2, L3 parsed offsets and size */ - copy_packet_cls_metadata(phdr, odp_packet_hdr(pkt)); + copy_packet_cls_metadata(phdr, packet_hdr(pkt)); - odp_packet_hdr(pkt)->frame_len = phdr->frame_len; - odp_packet_hdr(pkt)->headroom = phdr->headroom; - odp_packet_hdr(pkt)->tailroom = phdr->tailroom; + packet_hdr(pkt)->frame_len = phdr->frame_len; + packet_hdr(pkt)->headroom = phdr->headroom; + packet_hdr(pkt)->tailroom = phdr->tailroom; /* Take classification fields */ - odp_packet_hdr(pkt)->p = phdr->p; + packet_hdr(pkt)->p = phdr->p; pkt_table[i] = pkt; } @@ -630,7 +630,7 @@ static int ipc_pktio_send_lockless(pktio_entry_t *pktio_entry, odp_packet_hdr_t *pkt_hdr; pool_t *pool; - pkt_hdr = odp_packet_hdr(pkt); + pkt_hdr = packet_hdr(pkt); pool = pkt_hdr->buf_hdr.pool_ptr; if (pool->pool_idx != ipc_pool->pool_idx || @@ -652,7 +652,7 @@ static int ipc_pktio_send_lockless(pktio_entry_t *pktio_entry, for (i = 0; i < len; i++) { uint64_t data_pool_off; odp_packet_t pkt = pkt_table_mapped[i]; - odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); + odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); odp_pool_t pool_hdl = odp_packet_pool(pkt); pool_t *pool = pool_entry_from_hdl(pool_hdl); diff --git a/platform/linux-generic/pktio/loopback.c b/platform/linux-generic/pktio/loopback.c index ff08364a2d..8ac5e28c3f 100644 --- a/platform/linux-generic/pktio/loopback.c +++ b/platform/linux-generic/pktio/loopback.c @@ -109,7 +109,7 @@ static int loopback_recv(pktio_entry_t *pktio_entry, int index ODP_UNUSED, pkt = packet_from_buf_hdr(hdr_tbl[i]); pkt_len = odp_packet_len(pkt); - pkt_hdr = odp_packet_hdr(pkt); + pkt_hdr = packet_hdr(pkt); if (pktio_cls_enabled(pktio_entry)) { odp_packet_t new_pkt; diff --git a/platform/linux-generic/pktio/netmap.c b/platform/linux-generic/pktio/netmap.c index 2631c816d9..72626503d1 100644 --- a/platform/linux-generic/pktio/netmap.c +++ b/platform/linux-generic/pktio/netmap.c @@ -676,7 +676,7 @@ static inline int netmap_pkt_to_odp(pktio_entry_t *pktio_entry, } pkt = pkt_tbl[i]; - pkt_hdr = odp_packet_hdr(pkt); + pkt_hdr = packet_hdr(pkt); pull_tail(pkt_hdr, alloc_len - len); /* For now copy the data in the mbuf, diff --git a/platform/linux-generic/pktio/pcap.c b/platform/linux-generic/pktio/pcap.c index f3f0005743..8cba047a6e 100644 --- a/platform/linux-generic/pktio/pcap.c +++ b/platform/linux-generic/pktio/pcap.c @@ -259,7 +259,7 @@ static int pcapif_recv_pkt(pktio_entry_t *pktio_entry, int index ODP_UNUSED, if (ts != NULL) ts_val = odp_time_global(); - pkt_hdr = odp_packet_hdr(pkt); + pkt_hdr = packet_hdr(pkt); if (odp_packet_copy_from_mem(pkt, 0, hdr->caplen, data) != 0) { ODP_ERR("failed to copy packet data\n"); diff --git a/platform/linux-generic/pktio/socket.c b/platform/linux-generic/pktio/socket.c index 320de973c5..7ee9649e33 100644 --- a/platform/linux-generic/pktio/socket.c +++ b/platform/linux-generic/pktio/socket.c @@ -306,7 +306,7 @@ static int sock_mmsg_recv(pktio_entry_t *pktio_entry, int index ODP_UNUSED, void *base = msgvec[i].msg_hdr.msg_iov->iov_base; struct ethhdr *eth_hdr = base; odp_packet_t pkt = pkt_table[i]; - odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); + odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); uint16_t pkt_len = msgvec[i].msg_len; int ret; diff --git a/platform/linux-generic/pktio/socket_mmap.c b/platform/linux-generic/pktio/socket_mmap.c index 26d0a2957a..43e83a9913 100644 --- a/platform/linux-generic/pktio/socket_mmap.c +++ b/platform/linux-generic/pktio/socket_mmap.c @@ -224,7 +224,7 @@ static inline unsigned pkt_mmap_v2_rx(pktio_entry_t *pktio_entry, frame_num = next_frame_num; continue; } - hdr = odp_packet_hdr(pkt_table[nb_rx]); + hdr = packet_hdr(pkt_table[nb_rx]); ret = odp_packet_copy_from_mem(pkt_table[nb_rx], 0, pkt_len, pkt_buf); if (ret != 0) { diff --git a/platform/linux-generic/pktio/tap.c b/platform/linux-generic/pktio/tap.c index bb404d9143..2f2aca6eca 100644 --- a/platform/linux-generic/pktio/tap.c +++ b/platform/linux-generic/pktio/tap.c @@ -296,7 +296,7 @@ static odp_packet_t pack_odp_pkt(pktio_entry_t *pktio_entry, const void *data, return ODP_PACKET_INVALID; } - pkt_hdr = odp_packet_hdr(pkt); + pkt_hdr = packet_hdr(pkt); if (pktio_cls_enabled(pktio_entry)) copy_packet_cls_metadata(&parsed_hdr, pkt_hdr); From 3ad5e6cd0aa6c038d0d075a757889e6e42e06ef0 Mon Sep 17 00:00:00 2001 From: Brian Brooks Date: Thu, 5 Oct 2017 13:10:31 -0500 Subject: [PATCH 2/4] linux-gen: packet: remove duplicated _odp_packet_from_buf_hdr() function _odp_packet_from_buf_hdr() and packet_from_buf_hdr() are both internal-only functions that do the exact same thing. Remove _odp_packet_from_buf_hdr() version since internal-only identifiers should not use any form of an "odp_" prefix. This is also consistent with other packet_xxx() internal-only functions. Signed-off-by: Brian Brooks Reviewed-by: Yi He Reviewed-by: Ola Liljedahl Reviewed-by: Honnappa Nagarahalli --- platform/linux-generic/include/odp_packet_internal.h | 3 --- platform/linux-generic/odp_packet.c | 5 ----- platform/linux-generic/odp_traffic_mngr.c | 2 +- 3 files changed, 1 insertion(+), 9 deletions(-) diff --git a/platform/linux-generic/include/odp_packet_internal.h b/platform/linux-generic/include/odp_packet_internal.h index 3a86ce9284..98d364a8d5 100644 --- a/platform/linux-generic/include/odp_packet_internal.h +++ b/platform/linux-generic/include/odp_packet_internal.h @@ -290,9 +290,6 @@ int packet_parse_l3_l4(odp_packet_hdr_t *pkt_hdr, /* Reset parser metadata for a new parse */ void packet_parse_reset(odp_packet_hdr_t *pkt_hdr); -/* Convert a buffer handle to a packet handle */ -odp_packet_t _odp_packet_from_buf_hdr(odp_buffer_hdr_t *buf_hdr); - static inline int packet_hdr_has_l2(odp_packet_hdr_t *pkt_hdr) { return pkt_hdr->p.input_flags.l2; diff --git a/platform/linux-generic/odp_packet.c b/platform/linux-generic/odp_packet.c index 239506e10b..bc4777cff8 100644 --- a/platform/linux-generic/odp_packet.c +++ b/platform/linux-generic/odp_packet.c @@ -56,11 +56,6 @@ static inline odp_packet_hdr_t *buf_to_packet_hdr(odp_buffer_t buf) return (odp_packet_hdr_t *)buf_hdl_to_hdr(buf); } -odp_packet_t _odp_packet_from_buf_hdr(odp_buffer_hdr_t *buf_hdr) -{ - return (odp_packet_t)buf_hdr; -} - static inline seg_entry_t *seg_entry(odp_packet_hdr_t *hdr, uint32_t seg_idx) { diff --git a/platform/linux-generic/odp_traffic_mngr.c b/platform/linux-generic/odp_traffic_mngr.c index 20af537fa2..4f4acba2bd 100644 --- a/platform/linux-generic/odp_traffic_mngr.c +++ b/platform/linux-generic/odp_traffic_mngr.c @@ -108,7 +108,7 @@ static int queue_tm_reenq(queue_t queue, odp_buffer_hdr_t *buf_hdr) odp_tm_queue_t tm_queue = MAKE_ODP_TM_QUEUE((uint8_t *)queue - offsetof(tm_queue_obj_t, tm_qentry)); - odp_packet_t pkt = _odp_packet_from_buf_hdr(buf_hdr); + odp_packet_t pkt = packet_from_buf_hdr(buf_hdr); return odp_tm_enq(tm_queue, pkt); } From 68458f978a08ff87541447113f1f8e4eb05f85ea Mon Sep 17 00:00:00 2001 From: Brian Brooks Date: Mon, 13 Nov 2017 15:39:08 -0600 Subject: [PATCH 3/4] linux-gen: packet: remove excess segmentation metadata Use a single pointer to next segment instead of an array of pointers. This reduces buffer metadata by ~140 bytes. Signed-off-by: Brian Brooks Signed-off-by: Honnappa Nagarahalli --- platform/linux-generic/buffer/generic.c | 4 +- .../include/odp_buffer_internal.h | 20 +- .../include/odp_packet_internal.h | 91 +- platform/linux-generic/odp_packet.c | 1452 +++++------------ platform/linux-generic/pktio/dpdk.c | 8 +- platform/linux-generic/pktio/ipc.c | 2 +- platform/linux-generic/pool/generic.c | 25 +- test/validation/api/packet/packet.c | 2 +- 8 files changed, 500 insertions(+), 1104 deletions(-) diff --git a/platform/linux-generic/buffer/generic.c b/platform/linux-generic/buffer/generic.c index 543cc530c0..4acca35225 100644 --- a/platform/linux-generic/buffer/generic.c +++ b/platform/linux-generic/buffer/generic.c @@ -31,7 +31,7 @@ static void *generic_buffer_addr(odp_buffer_t buf) { odp_buffer_hdr_t *hdr = buf_hdl_to_hdr(buf); - return hdr->seg[0].data; + return hdr->base_data; } static uint32_t generic_buffer_size(odp_buffer_t buf) @@ -61,7 +61,7 @@ int odp_buffer_snprint(char *str, uint32_t n, odp_buffer_t buf) " pool %" PRIu64 "\n", odp_pool_to_u64(pool->pool_hdl)); len += snprintf(&str[len], n - len, - " addr %p\n", hdr->seg[0].data); + " addr %p\n", hdr->base_data); len += snprintf(&str[len], n - len, " size %" PRIu32 "\n", hdr->size); len += snprintf(&str[len], n - len, diff --git a/platform/linux-generic/include/odp_buffer_internal.h b/platform/linux-generic/include/odp_buffer_internal.h index b8e8c1717d..0046afb3a4 100644 --- a/platform/linux-generic/include/odp_buffer_internal.h +++ b/platform/linux-generic/include/odp_buffer_internal.h @@ -35,12 +35,6 @@ extern "C" { #define BUFFER_BURST_SIZE CONFIG_BURST_SIZE -typedef struct seg_entry_t { - void *hdr; - uint8_t *data; - uint32_t len; -} seg_entry_t; - /* Common buffer header */ struct odp_buffer_hdr_t { /* Total segment count */ @@ -49,26 +43,18 @@ struct odp_buffer_hdr_t { /* Pool type */ int8_t type; - /* Number of seg[] entries used */ - uint8_t num_seg; + /* Offset used to restore base_data */ + uint8_t pristine_offset; /* Next header which continues the segment list */ void *next_seg; - /* Last header of the segment list */ - void *last_seg; - - /* Initial buffer data pointer */ + /* Pointer to start of segment */ uint8_t *base_data; /* Pool pointer */ void *pool_ptr; - /* --- 40 bytes --- */ - - /* Segments */ - seg_entry_t seg[CONFIG_PACKET_SEGS_PER_HDR]; - #ifndef ODP_SCHEDULE_SCALABLE /* Burst counts */ uint8_t burst_num; diff --git a/platform/linux-generic/include/odp_packet_internal.h b/platform/linux-generic/include/odp_packet_internal.h index 98d364a8d5..8f381f0977 100644 --- a/platform/linux-generic/include/odp_packet_internal.h +++ b/platform/linux-generic/include/odp_packet_internal.h @@ -20,6 +20,7 @@ extern "C" { #include #include #include +#include #include #include #include @@ -159,9 +160,6 @@ typedef struct { uint8_t data[0]; } odp_packet_hdr_t; -/** - * Return the packet header - */ static inline odp_packet_hdr_t *packet_hdr(odp_packet_t pkt) { return (odp_packet_hdr_t *)(uintptr_t)pkt; @@ -182,58 +180,76 @@ static inline odp_packet_t packet_from_buf_hdr(odp_buffer_hdr_t *buf_hdr) return (odp_packet_t)(odp_packet_hdr_t *)buf_hdr; } -static inline seg_entry_t *seg_entry_last(odp_packet_hdr_t *hdr) +static inline odp_packet_hdr_t *packet_last_seg(odp_packet_hdr_t *pkt_hdr) { - odp_packet_hdr_t *last; - uint8_t last_seg; + int segcount = pkt_hdr->buf_hdr.segcount; + + if (odp_unlikely(segcount > 1)) { + while (--segcount) + pkt_hdr = pkt_hdr->buf_hdr.next_seg; + } - last = hdr->buf_hdr.last_seg; - last_seg = last->buf_hdr.num_seg - 1; - return &last->buf_hdr.seg[last_seg]; + ODP_ASSERT(pkt_hdr->buf_hdr.next_seg == NULL); + + return pkt_hdr; } -/** - * Initialize packet - */ static inline void packet_init(odp_packet_hdr_t *pkt_hdr, uint32_t len) { pool_t *pool = pool_entry_from_hdl(pkt_hdr->buf_hdr.pool_hdl); - uint32_t seg_len; - int num = pkt_hdr->buf_hdr.segcount; - - if (odp_likely(CONFIG_PACKET_SEG_DISABLED || num == 1)) { - seg_len = len; - pkt_hdr->buf_hdr.seg[0].len = len; - } else { - seg_entry_t *last; + odp_packet_hdr_t *old_pkt_hdr = pkt_hdr; - seg_len = len - ((num - 1) * pool->seg_len); + pkt_hdr->frame_len = len; - /* Last segment data length */ - last = seg_entry_last(pkt_hdr); - last->len = seg_len; - } + pkt_hdr->headroom = CONFIG_PACKET_HEADROOM; + pkt_hdr->tailroom = CONFIG_PACKET_TAILROOM; + pkt_hdr->input = ODP_PKTIO_INVALID; pkt_hdr->p.input_flags.all = 0; pkt_hdr->p.output_flags.all = 0; pkt_hdr->p.error_flags.all = 0; - pkt_hdr->p.l2_offset = 0; pkt_hdr->p.l3_offset = ODP_PACKET_OFFSET_INVALID; pkt_hdr->p.l4_offset = ODP_PACKET_OFFSET_INVALID; - /* - * Packet headroom is set from the pool's headroom - * Packet tailroom is rounded up to fill the last - * segment occupied by the allocated length. - */ - pkt_hdr->frame_len = len; - pkt_hdr->headroom = CONFIG_PACKET_HEADROOM; - pkt_hdr->tailroom = pool->seg_len - seg_len + CONFIG_PACKET_TAILROOM; - - pkt_hdr->input = ODP_PKTIO_INVALID; pkt_hdr->buf_hdr.event_subtype = ODP_EVENT_PACKET_BASIC; + /* Restore each segment's base_data and size fields */ + while (len > pool->seg_len) { + pkt_hdr->buf_hdr.base_data = + &pkt_hdr->data[pkt_hdr->buf_hdr.pristine_offset]; + pkt_hdr->buf_hdr.size = pool->seg_len; + + len -= pool->seg_len; + + pkt_hdr = pkt_hdr->buf_hdr.next_seg; + } + pkt_hdr->buf_hdr.base_data = + &pkt_hdr->data[pkt_hdr->buf_hdr.pristine_offset]; + pkt_hdr->buf_hdr.size = len; + + old_pkt_hdr->tailroom += pool->seg_len - len; +} + +static inline void packet_init_segs(odp_packet_hdr_t *pkt_hdrs[], int num_seg) +{ + odp_packet_hdr_t *pkt_hdr = pkt_hdrs[0]; + + pkt_hdr->buf_hdr.segcount = num_seg; + + if (odp_unlikely(num_seg > 1)) { + for (int i = 1; i < num_seg; i++) { + pkt_hdr->buf_hdr.next_seg = pkt_hdrs[i]; + pkt_hdr = pkt_hdrs[i]; + } + } + + pkt_hdr->buf_hdr.next_seg = NULL; +} + +static inline uint32_t packet_buf_len(odp_packet_hdr_t *pkt_hdr) +{ + return pkt_hdr->buf_hdr.size + pkt_hdr->headroom + pkt_hdr->tailroom; } static inline void copy_packet_parser_metadata(odp_packet_hdr_t *src_hdr, @@ -253,11 +269,8 @@ static inline void copy_packet_cls_metadata(odp_packet_hdr_t *src_hdr, static inline void pull_tail(odp_packet_hdr_t *pkt_hdr, uint32_t len) { - seg_entry_t *last = seg_entry_last(pkt_hdr); - pkt_hdr->tailroom += len; pkt_hdr->frame_len -= len; - last->len -= len; } static inline uint32_t packet_len(odp_packet_hdr_t *pkt_hdr) diff --git a/platform/linux-generic/odp_packet.c b/platform/linux-generic/odp_packet.c index bc4777cff8..cae38fbda7 100644 --- a/platform/linux-generic/odp_packet.c +++ b/platform/linux-generic/odp_packet.c @@ -28,8 +28,8 @@ /* Fill in packet header field offsets for inline functions */ const _odp_packet_inline_offset_t _odp_packet_inline ODP_ALIGNED_CACHE = { - .data = offsetof(odp_packet_hdr_t, buf_hdr.seg[0].data), - .seg_len = offsetof(odp_packet_hdr_t, buf_hdr.seg[0].len), + .data = offsetof(odp_packet_hdr_t, buf_hdr.base_data), + .seg_len = offsetof(odp_packet_hdr_t, buf_hdr.size), .frame_len = offsetof(odp_packet_hdr_t, frame_len), .headroom = offsetof(odp_packet_hdr_t, headroom), .tailroom = offsetof(odp_packet_hdr_t, tailroom), @@ -41,275 +41,15 @@ const _odp_packet_inline_offset_t _odp_packet_inline ODP_ALIGNED_CACHE = { .flow_hash = offsetof(odp_packet_hdr_t, flow_hash), .timestamp = offsetof(odp_packet_hdr_t, timestamp), .input_flags = offsetof(odp_packet_hdr_t, p.input_flags) - }; #include -static inline odp_buffer_t buffer_handle(odp_packet_hdr_t *pkt_hdr) -{ - return (odp_buffer_t)pkt_hdr; -} - -static inline odp_packet_hdr_t *buf_to_packet_hdr(odp_buffer_t buf) +static odp_packet_hdr_t *buf_to_packet_hdr(odp_buffer_t buf) { return (odp_packet_hdr_t *)buf_hdl_to_hdr(buf); } -static inline seg_entry_t *seg_entry(odp_packet_hdr_t *hdr, - uint32_t seg_idx) -{ - uint32_t idx = 0; - uint8_t num_seg = hdr->buf_hdr.num_seg; - - while (odp_unlikely(idx + num_seg - 1 < seg_idx)) { - idx += num_seg; - hdr = hdr->buf_hdr.next_seg; - num_seg = hdr->buf_hdr.num_seg; - } - - idx = seg_idx - idx; - - return &hdr->buf_hdr.seg[idx]; -} - -static inline void seg_entry_find_idx(odp_packet_hdr_t **p_hdr, - uint8_t *p_idx, - uint32_t find_idx) -{ - odp_packet_hdr_t *hdr = *p_hdr; - uint32_t idx = 0; - uint8_t num_seg = hdr->buf_hdr.num_seg; - - while (odp_unlikely(idx + num_seg - 1 < find_idx)) { - idx += num_seg; - hdr = hdr->buf_hdr.next_seg; - num_seg = hdr->buf_hdr.num_seg; - } - - idx = find_idx - idx; - *p_hdr = hdr; - *p_idx = idx; -} - -/* Return pointer to the current segment entry and step cur_hdr / cur_idx - * forward. - */ -static inline seg_entry_t *seg_entry_next(odp_packet_hdr_t **cur_hdr, - uint8_t *cur_idx) -{ - odp_packet_hdr_t *hdr = *cur_hdr; - uint8_t idx = *cur_idx; - uint8_t num_seg = hdr->buf_hdr.num_seg; - - if (idx == num_seg - 1) { - *cur_hdr = hdr->buf_hdr.next_seg; - *cur_idx = 0; - } else { - *cur_idx = idx + 1; - } - - return &hdr->buf_hdr.seg[idx]; -} - -static inline void seg_entry_find_offset(odp_packet_hdr_t **p_hdr, - uint8_t *p_idx, - uint32_t *seg_offset, - uint32_t *seg_idx, - uint32_t offset) -{ - int i; - odp_packet_hdr_t *hdr, *cur_hdr; - uint8_t idx, cur_idx; - seg_entry_t *seg = NULL; - uint32_t seg_start = 0, seg_end = 0; - int seg_count; - - hdr = *p_hdr; - cur_hdr = hdr; - idx = 0; - cur_idx = 0; - seg_count = hdr->buf_hdr.segcount; - - for (i = 0; i < seg_count; i++) { - cur_hdr = hdr; - cur_idx = idx; - seg = seg_entry_next(&hdr, &idx); - seg_end += seg->len; - - if (odp_likely(offset < seg_end)) - break; - - seg_start = seg_end; - } - - *p_hdr = cur_hdr; - *p_idx = cur_idx; - *seg_offset = offset - seg_start; - *seg_idx = i; -} - -static inline uint32_t packet_seg_len(odp_packet_hdr_t *pkt_hdr, - uint32_t seg_idx) -{ - seg_entry_t *seg = seg_entry(pkt_hdr, seg_idx); - - return seg->len; -} - -static inline void *packet_seg_data(odp_packet_hdr_t *pkt_hdr, uint32_t seg_idx) -{ - seg_entry_t *seg = seg_entry(pkt_hdr, seg_idx); - - return seg->data; -} - -static inline uint16_t packet_last_seg(odp_packet_hdr_t *pkt_hdr) -{ - if (CONFIG_PACKET_SEG_DISABLED) - return 0; - else - return pkt_hdr->buf_hdr.segcount - 1; -} - -static inline uint32_t packet_first_seg_len(odp_packet_hdr_t *pkt_hdr) -{ - return pkt_hdr->buf_hdr.seg[0].len; -} - -static inline void *packet_data(odp_packet_hdr_t *pkt_hdr) -{ - return pkt_hdr->buf_hdr.seg[0].data; -} - -static inline void *packet_tail(odp_packet_hdr_t *pkt_hdr) -{ - seg_entry_t *last_seg = seg_entry_last(pkt_hdr); - - return last_seg->data + last_seg->len; -} - -static inline uint32_t seg_headroom(odp_packet_hdr_t *pkt_hdr, int seg_idx) -{ - seg_entry_t *seg = seg_entry(pkt_hdr, seg_idx); - odp_buffer_hdr_t *hdr = seg->hdr; - uint8_t *base = hdr->base_data; - uint8_t *head = seg->data; - - return CONFIG_PACKET_HEADROOM + (head - base); -} - -static inline uint32_t seg_tailroom(odp_packet_hdr_t *pkt_hdr, int seg_idx) -{ - seg_entry_t *seg = seg_entry(pkt_hdr, seg_idx); - - odp_buffer_hdr_t *hdr = seg->hdr; - uint8_t *tail = seg->data + seg->len; - - return hdr->buf_end - tail; -} - -static inline void push_head(odp_packet_hdr_t *pkt_hdr, uint32_t len) -{ - pkt_hdr->headroom -= len; - pkt_hdr->frame_len += len; - pkt_hdr->buf_hdr.seg[0].data -= len; - pkt_hdr->buf_hdr.seg[0].len += len; -} - -static inline void pull_head(odp_packet_hdr_t *pkt_hdr, uint32_t len) -{ - pkt_hdr->headroom += len; - pkt_hdr->frame_len -= len; - pkt_hdr->buf_hdr.seg[0].data += len; - pkt_hdr->buf_hdr.seg[0].len -= len; -} - -static inline void push_tail(odp_packet_hdr_t *pkt_hdr, uint32_t len) -{ - seg_entry_t *last_seg = seg_entry_last(pkt_hdr); - - pkt_hdr->tailroom -= len; - pkt_hdr->frame_len += len; - last_seg->len += len; -} - -/* Copy all metadata for segmentation modification. Segment data and lengths - * are not copied. */ -static inline void packet_seg_copy_md(odp_packet_hdr_t *dst, - odp_packet_hdr_t *src) -{ - dst->p = src->p; - - /* lengths are not copied: - * .frame_len - * .headroom - * .tailroom - */ - - dst->input = src->input; - dst->dst_queue = src->dst_queue; - dst->flow_hash = src->flow_hash; - dst->timestamp = src->timestamp; - - /* buffer header side packet metadata */ - dst->buf_hdr.buf_u64 = src->buf_hdr.buf_u64; - dst->buf_hdr.uarea_addr = src->buf_hdr.uarea_addr; - - /* segmentation data is not copied: - * buf_hdr.seg[] - * buf_hdr.segcount - * buf_hdr.num_seg - * buf_hdr.next_seg - * buf_hdr.last_seg - */ -} - -static inline void *packet_map(odp_packet_hdr_t *pkt_hdr, - uint32_t offset, uint32_t *seg_len, int *seg_idx) -{ - void *addr; - uint32_t len; - int seg_id = 0; - int seg_count = pkt_hdr->buf_hdr.segcount; - - if (odp_unlikely(offset >= pkt_hdr->frame_len)) - return NULL; - - if (odp_likely(CONFIG_PACKET_SEG_DISABLED || seg_count == 1)) { - addr = pkt_hdr->buf_hdr.seg[0].data + offset; - len = pkt_hdr->buf_hdr.seg[0].len - offset; - } else { - int i; - seg_entry_t *seg = NULL; - uint32_t seg_start = 0, seg_end = 0; - odp_packet_hdr_t *hdr = pkt_hdr; - uint8_t idx = 0; - - for (i = 0; i < seg_count; i++) { - seg = seg_entry_next(&hdr, &idx); - seg_end += seg->len; - - if (odp_likely(offset < seg_end)) - break; - - seg_start = seg_end; - } - - addr = seg->data + (offset - seg_start); - len = seg->len - (offset - seg_start); - seg_id = i; - } - - if (seg_len) - *seg_len = len; - - if (seg_idx) - *seg_idx = seg_id; - - return addr; -} - void packet_parse_reset(odp_packet_hdr_t *pkt_hdr) { /* Reset parser metadata before new parse */ @@ -321,103 +61,7 @@ void packet_parse_reset(odp_packet_hdr_t *pkt_hdr) pkt_hdr->p.l4_offset = ODP_PACKET_OFFSET_INVALID; } -static inline void link_segments(odp_packet_hdr_t *pkt_hdr[], int num) -{ - int cur, i; - odp_packet_hdr_t *hdr; - odp_packet_hdr_t *head = pkt_hdr[0]; - uint32_t seg_len = pool_entry_from_hdl(head->buf_hdr.pool_hdl)->seg_len; - - cur = 0; - - while (1) { - hdr = pkt_hdr[cur]; - - for (i = 0; i < CONFIG_PACKET_SEGS_PER_HDR; i++) { - odp_buffer_hdr_t *buf_hdr; - - buf_hdr = &pkt_hdr[cur]->buf_hdr; - hdr->buf_hdr.seg[i].hdr = buf_hdr; - hdr->buf_hdr.seg[i].data = buf_hdr->base_data; - hdr->buf_hdr.seg[i].len = seg_len; - - /* init_segments() handles first seg ref_cnt init */ - if (ODP_DEBUG == 1 && cur > 0) { - uint32_t prev_ref = - odp_atomic_fetch_inc_u32( - &pkt_hdr[cur]->buf_hdr.ref_cnt); - - ODP_ASSERT(prev_ref == 0); - } - - cur++; - - if (cur == num) { - /* Last segment */ - hdr->buf_hdr.num_seg = i + 1; - hdr->buf_hdr.next_seg = NULL; - head->buf_hdr.last_seg = &hdr->buf_hdr; - return; - } - } - - hdr->buf_hdr.num_seg = CONFIG_PACKET_SEGS_PER_HDR; - hdr->buf_hdr.next_seg = pkt_hdr[cur]; - } -} - -static inline void init_segments(odp_packet_hdr_t *pkt_hdr[], int num) -{ - odp_packet_hdr_t *hdr; - uint32_t seg_len; - - /* First segment is the packet descriptor */ - hdr = pkt_hdr[0]; - seg_len = pool_entry_from_hdl(hdr->buf_hdr.pool_hdl)->seg_len; - - /* Defaults for single segment packet */ - hdr->buf_hdr.seg[0].data = hdr->buf_hdr.base_data; - hdr->buf_hdr.seg[0].len = seg_len; - - if (ODP_DEBUG == 1) { - uint32_t prev_ref = - odp_atomic_fetch_inc_u32(&hdr->buf_hdr.ref_cnt); - - ODP_ASSERT(prev_ref == 0); - } - - if (!CONFIG_PACKET_SEG_DISABLED) { - hdr->buf_hdr.segcount = num; - hdr->buf_hdr.num_seg = 1; - hdr->buf_hdr.next_seg = NULL; - hdr->buf_hdr.last_seg = &hdr->buf_hdr; - - /* Link segments */ - if (odp_unlikely(num > 1)) - link_segments(pkt_hdr, num); - } -} - -static inline void reset_seg(odp_packet_hdr_t *pkt_hdr, int first, int num) -{ - odp_packet_hdr_t *hdr = pkt_hdr; - void *base; - int i; - seg_entry_t *seg; - uint32_t seg_len = pool_entry_from_hdl(hdr->buf_hdr.pool_hdl)->seg_len; - uint8_t idx; - - seg_entry_find_idx(&hdr, &idx, first); - - for (i = 0; i < num; i++) { - base = hdr->buf_hdr.base_data; - seg = seg_entry_next(&hdr, &idx); - seg->len = seg_len; - seg->data = base; - } -} - -/* Calculate the number of segments */ +/* Calculate number of segments required for a packet of len 'seg_len' bytes. */ static inline int num_segments(uint32_t len, uint32_t seg_len) { int num; @@ -437,375 +81,255 @@ static inline int num_segments(uint32_t len, uint32_t seg_len) return num; } -static inline void add_all_segs(odp_packet_hdr_t *to, odp_packet_hdr_t *from) +/* Returns a pointer to the Nth (0-based) segment */ +static odp_packet_hdr_t *get_seg(odp_packet_hdr_t *pkt_hdr, uint32_t n) { - odp_packet_hdr_t *last = to->buf_hdr.last_seg; + uint16_t segcount = pkt_hdr->buf_hdr.segcount; + + ODP_ASSERT(n < segcount); + + if (odp_likely(CONFIG_PACKET_MAX_SEGS == 1 || segcount == 1)) + return pkt_hdr; - last->buf_hdr.next_seg = from; - to->buf_hdr.last_seg = from->buf_hdr.last_seg; - to->buf_hdr.segcount += from->buf_hdr.segcount; + while (n--) { + ODP_ASSERT(pkt_hdr->buf_hdr.next_seg); + pkt_hdr = pkt_hdr->buf_hdr.next_seg; + } + + return pkt_hdr; } -static inline odp_packet_hdr_t *alloc_segments(pool_t *pool, int num) +/* + * Returns a pointer to the segment containing byte 'offset' as well as + * the number of segments and bytes skipped to get to the segment. + */ +static odp_packet_hdr_t *get_seg_at_offset(odp_packet_hdr_t *pkt_hdr, + uint32_t offset, + uint32_t *bytes_skipped, + uint32_t *segs_skipped) { - odp_packet_hdr_t *pkt_hdr[num]; - int ret; + uint32_t skipped_bytes = 0; + uint32_t skipped_segs = 0; - ret = buffer_alloc_multi(pool, (odp_buffer_hdr_t **)pkt_hdr, num); + ODP_ASSERT(offset < pkt_hdr->frame_len); - if (odp_unlikely(ret != num)) { - if (ret > 0) - buffer_free_multi((odp_buffer_hdr_t **)pkt_hdr, ret); + if (odp_unlikely(pkt_hdr->buf_hdr.segcount > 1)) { + while (offset >= pkt_hdr->buf_hdr.size) { + skipped_bytes += pkt_hdr->buf_hdr.size; + skipped_segs++; - return NULL; + offset -= pkt_hdr->buf_hdr.size; + pkt_hdr = pkt_hdr->buf_hdr.next_seg; + ODP_ASSERT(pkt_hdr); + } } - init_segments(pkt_hdr, num); + if (bytes_skipped) + *bytes_skipped = skipped_bytes; + if (segs_skipped) + *segs_skipped = skipped_segs; - return pkt_hdr[0]; + return pkt_hdr; } -static inline odp_packet_hdr_t *add_segments(odp_packet_hdr_t *pkt_hdr, - pool_t *pool, uint32_t len, - int num, int head) +/* Link two segment chains together. Adjusts segcounts. */ +static void concat_seg(odp_packet_hdr_t *seg_a, odp_packet_hdr_t *seg_b) { - odp_packet_hdr_t *new_hdr; - uint32_t seg_len, offset; - - new_hdr = alloc_segments(pool, num); - - if (new_hdr == NULL) - return NULL; - - seg_len = len - ((num - 1) * pool->seg_len); - offset = pool->seg_len - seg_len; + odp_packet_hdr_t *seg_a_last = packet_last_seg(seg_a); - if (head) { - /* add into the head*/ - add_all_segs(new_hdr, pkt_hdr); + ODP_ASSERT(seg_a_last->buf_hdr.next_seg == NULL); + seg_a_last->buf_hdr.next_seg = seg_b; - /* adjust first segment length */ - new_hdr->buf_hdr.seg[0].data += offset; - new_hdr->buf_hdr.seg[0].len = seg_len; + ODP_ASSERT(seg_a->buf_hdr.segcount > 0); + ODP_ASSERT(seg_b->buf_hdr.segcount > 0); - packet_seg_copy_md(new_hdr, pkt_hdr); - new_hdr->frame_len = pkt_hdr->frame_len + len; - new_hdr->headroom = pool->headroom + offset; - new_hdr->tailroom = pkt_hdr->tailroom; + seg_a->buf_hdr.segcount += seg_b->buf_hdr.segcount; + seg_b->buf_hdr.segcount = 0; +} - pkt_hdr = new_hdr; - } else { - seg_entry_t *last_seg; +/* + * Returns a pointer to start of packet + 'offset' bytes and stores + * the remaining length of the resulting segment in 'seg_len' and the + * segment index in 'seg_idx'. + */ +static void *packet_map(odp_packet_hdr_t *pkt_hdr, uint32_t offset, + uint32_t *seg_len, int *seg_idx) +{ + uint32_t skipped_segs = 0; - /* add into the tail */ - add_all_segs(pkt_hdr, new_hdr); + if (odp_unlikely(offset >= pkt_hdr->frame_len)) + return NULL; - /* adjust last segment length */ - last_seg = seg_entry_last(pkt_hdr); - last_seg->len = seg_len; + if (odp_unlikely(pkt_hdr->buf_hdr.segcount > 1)) { + while (offset >= pkt_hdr->buf_hdr.size) { + skipped_segs++; - pkt_hdr->frame_len += len; - pkt_hdr->tailroom = pool->tailroom + offset; + offset -= pkt_hdr->buf_hdr.size; + pkt_hdr = pkt_hdr->buf_hdr.next_seg; + } } - return pkt_hdr; -} - -static inline int seg_is_link(void *hdr) -{ - odp_packet_hdr_t *pkt_hdr = hdr; + if (seg_len) + *seg_len = pkt_hdr->buf_hdr.size - offset; + if (seg_idx) + *seg_idx = skipped_segs; - return pkt_hdr != pkt_hdr->buf_hdr.seg[0].hdr; + return pkt_hdr->buf_hdr.base_data + offset; } -static inline void buffer_ref_inc(odp_buffer_hdr_t *buf_hdr) +static void buffer_ref_inc(odp_buffer_hdr_t *buf_hdr) { - uint32_t ref_cnt = odp_atomic_load_u32(&buf_hdr->ref_cnt); - - /* First count increment after alloc */ - if (odp_likely(ref_cnt) == 0) - odp_atomic_store_u32(&buf_hdr->ref_cnt, 2); - else - odp_atomic_inc_u32(&buf_hdr->ref_cnt); + odp_atomic_inc_u32(&buf_hdr->ref_cnt); } -static inline uint32_t buffer_ref_dec(odp_buffer_hdr_t *buf_hdr) +static uint32_t buffer_ref_dec(odp_buffer_hdr_t *buf_hdr) { return odp_atomic_fetch_dec_u32(&buf_hdr->ref_cnt); } -static inline uint32_t buffer_ref(odp_buffer_hdr_t *buf_hdr) +static uint32_t buffer_ref(odp_buffer_hdr_t *buf_hdr) { return odp_atomic_load_u32(&buf_hdr->ref_cnt); } -static inline int is_multi_ref(uint32_t ref_cnt) +static int is_multi_ref(uint32_t ref_cnt) { - return (ref_cnt > 1); + return ref_cnt > 0; } -static inline void packet_ref_inc(odp_packet_hdr_t *pkt_hdr) +static void packet_ref_inc(odp_packet_hdr_t *pkt_hdr) { - seg_entry_t *seg; - int i; - int seg_count = pkt_hdr->buf_hdr.segcount; - odp_packet_hdr_t *hdr = pkt_hdr; - uint8_t idx = 0; - - for (i = 0; i < seg_count; i++) { - seg = seg_entry_next(&hdr, &idx); - buffer_ref_inc(seg->hdr); + while (pkt_hdr) { + buffer_ref_inc(&pkt_hdr->buf_hdr); + pkt_hdr = pkt_hdr->buf_hdr.next_seg; } } -static inline void packet_free_multi(odp_buffer_hdr_t *hdr[], int num) +/* Allocate 'num_pkt' packets of length 'len' bytes */ +static int packet_alloc(pool_t *pool, uint32_t len, int num_pkt, + odp_packet_t *pkts) { - int i; - uint32_t ref_cnt; - int num_ref = 0; + int segs_per_pkt = num_segments(len, pool->seg_len); + int num_buf = num_pkt * segs_per_pkt; + odp_packet_hdr_t *pkt_hdr[num_buf]; + int npkt = num_pkt; + int nbuf; - for (i = 0; i < num; i++) { - /* Zero when reference API has not been used */ - ref_cnt = buffer_ref(hdr[i]); + ODP_ASSERT(segs_per_pkt <= CONFIG_PACKET_MAX_SEGS); - if (odp_unlikely(ref_cnt)) { - ref_cnt = buffer_ref_dec(hdr[i]); + nbuf = buffer_alloc_multi(pool, (odp_buffer_hdr_t **)pkt_hdr, num_buf); - if (is_multi_ref(ref_cnt)) { - num_ref++; - continue; - } - } + /* If we did not get the total number of buffers we asked for, free any + * buffers near the end of the list that cannot be used to make a + * whole packet. */ + if (odp_unlikely(nbuf != num_buf)) { + int nfree; - /* Reset link header back to normal header */ - if (odp_unlikely(seg_is_link(hdr[i]))) - hdr[i]->seg[0].hdr = hdr[i]; + npkt = nbuf / segs_per_pkt; + nfree = nbuf - (npkt * segs_per_pkt); - /* Skip references and pack to be freed headers to array head */ - if (odp_unlikely(num_ref)) - hdr[i - num_ref] = hdr[i]; + if (nfree > 0) { + odp_buffer_hdr_t **p = + (odp_buffer_hdr_t **)&pkt_hdr[nbuf - nfree]; + buffer_free_multi(p, nfree); + } + } + + for (int i = 0; i < npkt; i++) { + packet_init_segs(&pkt_hdr[i * segs_per_pkt], segs_per_pkt); + packet_init(pkt_hdr[i * segs_per_pkt], len); + pkts[i] = (odp_packet_t)pkt_hdr[i * segs_per_pkt]; } - num -= num_ref; + for (int i = 0; i < nbuf; i++) { + ODP_ASSERT(((intptr_t)pkt_hdr[i]->buf_hdr.base_data - + (intptr_t)pkt_hdr[i]) <= + (int)(sizeof(odp_packet_hdr_t) + + pool->headroom + pool->align)); + } - if (odp_likely(num)) - buffer_free_multi(hdr, num); + return npkt; } -static inline void free_all_segments(odp_packet_hdr_t *pkt_hdr, int num) +int packet_alloc_multi(odp_pool_t pool_hdl, uint32_t len, odp_packet_t pkt[], + int max_num) { - int i; - odp_buffer_hdr_t *buf_hdr[num + 1]; - - if (odp_likely(pkt_hdr->buf_hdr.num_seg == num)) { - for (i = 0; i < num; i++) - buf_hdr[i] = pkt_hdr->buf_hdr.seg[i].hdr; + return packet_alloc(pool_entry_from_hdl(pool_hdl), + len, max_num, pkt); +} - if (odp_unlikely(seg_is_link(pkt_hdr))) { - buf_hdr[num] = &pkt_hdr->buf_hdr; - num++; - } - } else { - seg_entry_t *seg; - odp_buffer_hdr_t *link_hdr[num]; - uint8_t idx = 0; - int links = 0; - - for (i = 0; i < num; i++) { - /* Free also link headers */ - if (odp_unlikely(idx == 0 && seg_is_link(pkt_hdr))) { - link_hdr[links] = &pkt_hdr->buf_hdr; - links++; - } +/* Free 'num' segments starting from segment 'n'. */ +static void packet_free_segs(odp_packet_hdr_t *pkt, uint32_t num, uint32_t n) +{ + odp_packet_hdr_t *pkt_hdr[num]; - seg = seg_entry_next(&pkt_hdr, &idx); - buf_hdr[i] = seg->hdr; - } + ODP_ASSERT(n < pkt->buf_hdr.segcount); + ODP_ASSERT(n + num <= pkt->buf_hdr.segcount); - if (odp_unlikely(links)) - packet_free_multi(link_hdr, links); + pkt = get_seg(pkt, n); + for (unsigned i = 0; i < num; i++) { + pkt_hdr[i] = pkt; + pkt = pkt->buf_hdr.next_seg; } - packet_free_multi(buf_hdr, num); -} - -static inline odp_packet_hdr_t *free_segments(odp_packet_hdr_t *pkt_hdr, - int num, uint32_t free_len, - uint32_t pull_len, int head) -{ - seg_entry_t *seg; - int i; - int num_remain = pkt_hdr->buf_hdr.segcount - num; - odp_packet_hdr_t *hdr = pkt_hdr; - odp_packet_hdr_t *last_hdr = pkt_hdr->buf_hdr.last_seg; - uint8_t idx; - uint8_t num_seg; - odp_buffer_hdr_t *buf_hdr[num]; - odp_buffer_hdr_t *link_hdr[num]; - odp_packet_hdr_t *tmp_hdr; - int links = 0; - - if (head) { - odp_packet_hdr_t *new_hdr; - - idx = 0; - for (i = 0; i < num; i++) { - tmp_hdr = hdr; - seg = seg_entry_next(&hdr, &idx); - buf_hdr[i] = seg->hdr; - - /* Free link headers, if those become empty */ - if (odp_unlikely(idx == 0 && seg_is_link(tmp_hdr))) { - link_hdr[links] = &tmp_hdr->buf_hdr; - links++; - } - } + buffer_free_multi((odp_buffer_hdr_t **)&pkt_hdr, num); +} - /* The first remaining header is the new packet descriptor. - * Copy remaining segments from the last to-be-removed header - * to the new header. */ - new_hdr = hdr->buf_hdr.seg[idx].hdr; - num_seg = hdr->buf_hdr.num_seg - idx; +static void packet_free(odp_packet_hdr_t *pkt_hdr) +{ + int segcount = pkt_hdr->buf_hdr.segcount; + uint32_t ref_cnt; - new_hdr->buf_hdr.next_seg = hdr->buf_hdr.next_seg; + ODP_ASSERT(segcount > 0); - if (hdr == last_hdr) - new_hdr->buf_hdr.last_seg = new_hdr; - else - new_hdr->buf_hdr.last_seg = last_hdr; + if (odp_likely(segcount == 1)) { + ref_cnt = buffer_ref((odp_buffer_hdr_t *)pkt_hdr); - new_hdr->buf_hdr.num_seg = num_seg; - new_hdr->buf_hdr.segcount = num_remain; + if (odp_unlikely(ref_cnt)) { + ref_cnt = buffer_ref_dec((odp_buffer_hdr_t *)pkt_hdr); - for (i = 0; i < num_seg; i++) { - seg = seg_entry_next(&hdr, &idx); - new_hdr->buf_hdr.seg[i] = *seg; + if (is_multi_ref(ref_cnt)) + return; } - packet_seg_copy_md(new_hdr, pkt_hdr); - - /* Tailroom not changed */ - new_hdr->tailroom = pkt_hdr->tailroom; - - /* Link header does not have headroom */ - if (seg_is_link(new_hdr)) - new_hdr->headroom = 0; - else - new_hdr->headroom = seg_headroom(new_hdr, 0); - - new_hdr->frame_len = pkt_hdr->frame_len - free_len; - - pull_head(new_hdr, pull_len); - - pkt_hdr = new_hdr; - - if (odp_unlikely(links)) - packet_free_multi(link_hdr, links); - - packet_free_multi(buf_hdr, num); + buffer_free_multi((odp_buffer_hdr_t **)&pkt_hdr, 1); } else { - /* Free last 'num' bufs. - * First, find the last remaining header. */ - seg_entry_find_idx(&hdr, &idx, num_remain - 1); - last_hdr = hdr; - num_seg = idx + 1; - - seg_entry_next(&hdr, &idx); - - for (i = 0; i < num; i++) { - tmp_hdr = hdr; - seg = seg_entry_next(&hdr, &idx); - buf_hdr[i] = seg->hdr; - - /* Free link headers, if those become empty */ - if (odp_unlikely(idx == 0 && seg_is_link(tmp_hdr))) { - link_hdr[links] = &tmp_hdr->buf_hdr; - links++; - } - } - - if (odp_unlikely(links)) - packet_free_multi(link_hdr, links); - - packet_free_multi(buf_hdr, num); - - /* Head segment remains, no need to copy or update majority - * of the metadata. */ - last_hdr->buf_hdr.num_seg = num_seg; - last_hdr->buf_hdr.next_seg = NULL; - - pkt_hdr->buf_hdr.last_seg = last_hdr; - pkt_hdr->buf_hdr.segcount = num_remain; - pkt_hdr->frame_len -= free_len; - pkt_hdr->tailroom = seg_tailroom(pkt_hdr, num_remain - 1); - - pull_tail(pkt_hdr, pull_len); - } + odp_packet_hdr_t *hdr[segcount]; + int num_ref = 0; - return pkt_hdr; -} - -static inline int packet_alloc(pool_t *pool, uint32_t len, int max_pkt, - int num_seg, odp_packet_t *pkt) -{ - int num_buf, i; - int num = max_pkt; - int max_buf = max_pkt * num_seg; - odp_packet_hdr_t *pkt_hdr[max_buf]; + for (int i = 0; i < segcount; i++) { + ref_cnt = buffer_ref((odp_buffer_hdr_t *)pkt_hdr); - num_buf = buffer_alloc_multi(pool, (odp_buffer_hdr_t **)pkt_hdr, - max_buf); + if (odp_unlikely(ref_cnt)) { + ref_cnt = buffer_ref_dec( + (odp_buffer_hdr_t *)pkt_hdr); - /* Failed to allocate all segments */ - if (odp_unlikely(num_buf != max_buf)) { - int num_free; + if (is_multi_ref(ref_cnt)) { + num_ref++; - num = num_buf / num_seg; - num_free = num_buf - (num * num_seg); + pkt_hdr = pkt_hdr->buf_hdr.next_seg; + continue; + } + } - if (num_free > 0) { - odp_buffer_hdr_t **p; + hdr[i - num_ref] = pkt_hdr; - p = (odp_buffer_hdr_t **)&pkt_hdr[num_buf - num_free]; - buffer_free_multi(p, num_free); + pkt_hdr = pkt_hdr->buf_hdr.next_seg; } + ODP_ASSERT(pkt_hdr == NULL); - if (num == 0) - return 0; - } - - for (i = 0; i < num; i++) { - odp_packet_hdr_t *hdr; - - /* First buffer is the packet descriptor */ - hdr = pkt_hdr[i * num_seg]; - pkt[i] = packet_handle(hdr); - init_segments(&pkt_hdr[i * num_seg], num_seg); - - packet_init(hdr, len); + if (segcount - num_ref) + buffer_free_multi((odp_buffer_hdr_t **)&hdr, + segcount - num_ref); } - - return num; -} - -int packet_alloc_multi(odp_pool_t pool_hdl, uint32_t len, - odp_packet_t pkt[], int max_num) -{ - pool_t *pool = pool_entry_from_hdl(pool_hdl); - int num, num_seg; - - num_seg = num_segments(len, pool->seg_len); - num = packet_alloc(pool, len, max_num, num_seg, pkt); - - return num; } odp_packet_t odp_packet_alloc(odp_pool_t pool_hdl, uint32_t len) { pool_t *pool = pool_entry_from_hdl(pool_hdl); odp_packet_t pkt; - int num, num_seg; + int num; if (odp_unlikely(pool->params.type != ODP_POOL_PACKET)) { __odp_errno = EINVAL; @@ -815,8 +339,7 @@ odp_packet_t odp_packet_alloc(odp_pool_t pool_hdl, uint32_t len) if (odp_unlikely(len > pool->max_len)) return ODP_PACKET_INVALID; - num_seg = num_segments(len, pool->seg_len); - num = packet_alloc(pool, len, 1, num_seg, &pkt); + num = packet_alloc(pool, len, 1, &pkt); if (odp_unlikely(num == 0)) return ODP_PACKET_INVALID; @@ -828,7 +351,6 @@ int odp_packet_alloc_multi(odp_pool_t pool_hdl, uint32_t len, odp_packet_t pkt[], int max_num) { pool_t *pool = pool_entry_from_hdl(pool_hdl); - int num, num_seg; if (odp_unlikely(pool->params.type != ODP_POOL_PACKET)) { __odp_errno = EINVAL; @@ -838,69 +360,18 @@ int odp_packet_alloc_multi(odp_pool_t pool_hdl, uint32_t len, if (odp_unlikely(len > pool->max_len)) return -1; - num_seg = num_segments(len, pool->seg_len); - num = packet_alloc(pool, len, max_num, num_seg, pkt); - - return num; + return packet_alloc(pool, len, max_num, pkt); } void odp_packet_free(odp_packet_t pkt) { - odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); - int num_seg = pkt_hdr->buf_hdr.segcount; - - ODP_ASSERT(buffer_ref(&pkt_hdr->buf_hdr) > 0); - - if (odp_likely(CONFIG_PACKET_SEG_DISABLED || num_seg == 1)) { - odp_buffer_hdr_t *buf_hdr[2]; - int num = 1; - - buf_hdr[0] = &pkt_hdr->buf_hdr; - - if (odp_unlikely(seg_is_link(pkt_hdr))) { - num = 2; - buf_hdr[1] = pkt_hdr->buf_hdr.seg[0].hdr; - } - - packet_free_multi(buf_hdr, num); - } else { - free_all_segments(pkt_hdr, num_seg); - } + packet_free(packet_hdr(pkt)); } void odp_packet_free_multi(const odp_packet_t pkt[], int num) { - odp_buffer_hdr_t *buf_hdr[num]; - odp_buffer_hdr_t *buf_hdr2[num]; - int i; - int links = 0; - int num_freed = 0; - - for (i = 0; i < num; i++) { - odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt[i]); - int num_seg = pkt_hdr->buf_hdr.segcount; - - ODP_ASSERT(buffer_ref(&pkt_hdr->buf_hdr) > 0); - - if (odp_unlikely(num_seg > 1)) { - free_all_segments(pkt_hdr, num_seg); - num_freed++; - continue; - } - - if (odp_unlikely(seg_is_link(pkt_hdr))) { - buf_hdr2[links] = pkt_hdr->buf_hdr.seg[0].hdr; - links++; - } - - buf_hdr[i - num_freed] = &pkt_hdr->buf_hdr; - } - - if (odp_unlikely(links)) - packet_free_multi(buf_hdr2, links); - - if (odp_likely(num - num_freed)) - packet_free_multi(buf_hdr, num - num_freed); + for (int i = 0; i < num; i++) + packet_free(packet_hdr(pkt[i])); } int odp_packet_reset(odp_packet_t pkt, uint32_t len) @@ -912,8 +383,6 @@ int odp_packet_reset(odp_packet_t pkt, uint32_t len) if (odp_unlikely(len > (pool->seg_len * num))) return -1; - reset_seg(pkt_hdr, 0, num); - packet_init(pkt_hdr, len); return 0; @@ -932,28 +401,35 @@ odp_event_t odp_packet_to_event(odp_packet_t pkt) if (odp_unlikely(pkt == ODP_PACKET_INVALID)) return ODP_EVENT_INVALID; - return (odp_event_t)buffer_handle(packet_hdr(pkt)); + return (odp_event_t)packet_hdr(pkt); } -/* - * - * Pointers and lengths - * ******************************************************** - * - */ - uint32_t odp_packet_buf_len(odp_packet_t pkt) { odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); - return pkt_hdr->buf_hdr.size * pkt_hdr->buf_hdr.segcount; + return packet_buf_len(pkt_hdr) * pkt_hdr->buf_hdr.segcount; } void *odp_packet_tail(odp_packet_t pkt) { odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); + odp_packet_hdr_t *seg = packet_last_seg(pkt_hdr); + + return seg->buf_hdr.buf_end - pkt_hdr->tailroom; +} + +void *odp_packet_offset(odp_packet_t pkt, uint32_t offset, uint32_t *len, + odp_packet_seg_t *seg) +{ + int seg_idx; + void *addr = packet_map( + (odp_packet_hdr_t *)(uintptr_t)pkt, offset, len, &seg_idx); + + if (addr != NULL && seg != NULL) + *seg = _odp_packet_seg_from_ndx(seg_idx); - return packet_tail(pkt_hdr); + return addr; } void *odp_packet_push_head(odp_packet_t pkt, uint32_t len) @@ -963,225 +439,261 @@ void *odp_packet_push_head(odp_packet_t pkt, uint32_t len) if (len > pkt_hdr->headroom) return NULL; - push_head(pkt_hdr, len); - return packet_data(pkt_hdr); + pkt_hdr->buf_hdr.base_data -= len; + pkt_hdr->buf_hdr.size += len; + + pkt_hdr->headroom -= len; + pkt_hdr->frame_len += len; + + return pkt_hdr->buf_hdr.base_data; } -int odp_packet_extend_head(odp_packet_t *pkt, uint32_t len, - void **data_ptr, uint32_t *seg_len) +void *odp_packet_pull_head(odp_packet_t pkt, uint32_t len) { - odp_packet_hdr_t *pkt_hdr = packet_hdr(*pkt); - uint32_t frame_len = pkt_hdr->frame_len; - uint32_t headroom = pkt_hdr->headroom; - int ret = 0; + odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); - if (len > headroom) { - pool_t *pool = pkt_hdr->buf_hdr.pool_ptr; - int num; - void *ptr; + if (len > pkt_hdr->frame_len) + return NULL; - if (odp_unlikely((frame_len + len) > pool->max_len)) - return -1; + pkt_hdr->buf_hdr.base_data += len; + pkt_hdr->buf_hdr.size -= len; - num = num_segments(len - headroom, pool->seg_len); - push_head(pkt_hdr, headroom); - ptr = add_segments(pkt_hdr, pool, len - headroom, num, 1); + pkt_hdr->headroom += len; + pkt_hdr->frame_len -= len; - if (ptr == NULL) { - /* segment alloc failed, rollback changes */ - pull_head(pkt_hdr, headroom); - return -1; - } + return pkt_hdr->buf_hdr.base_data; +} - *pkt = packet_handle(ptr); - pkt_hdr = ptr; - } else { - push_head(pkt_hdr, len); - } +void *odp_packet_push_tail(odp_packet_t pkt, uint32_t len) +{ + odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); + odp_packet_hdr_t *seg = packet_last_seg(pkt_hdr); + void *old_tail; - if (data_ptr) - *data_ptr = packet_data(pkt_hdr); + if (len > pkt_hdr->tailroom) + return NULL; - if (seg_len) - *seg_len = packet_first_seg_len(pkt_hdr); + old_tail = seg->buf_hdr.buf_end - pkt_hdr->tailroom; + + seg->buf_hdr.size += len; - return ret; + pkt_hdr->tailroom -= len; + pkt_hdr->frame_len += len; + + return old_tail; } -void *odp_packet_pull_head(odp_packet_t pkt, uint32_t len) +void *odp_packet_pull_tail(odp_packet_t pkt, uint32_t len) { odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); + odp_packet_hdr_t *seg = packet_last_seg(pkt_hdr); - if (len > pkt_hdr->frame_len) + if (len > seg->buf_hdr.size) return NULL; - pull_head(pkt_hdr, len); - return packet_data(pkt_hdr); + seg->buf_hdr.size -= len; + + pkt_hdr->tailroom += len; + pkt_hdr->frame_len -= len; + + return seg->buf_hdr.buf_end - pkt_hdr->tailroom; } -int odp_packet_trunc_head(odp_packet_t *pkt, uint32_t len, - void **data_ptr, uint32_t *seg_len_out) +/* Copy a subset of metadata fields from one packet to another. */ +static void packet_copy_md(odp_packet_hdr_t *dst, odp_packet_hdr_t *src) +{ + dst->p = src->p; + dst->input = src->input; + dst->dst_queue = src->dst_queue; + dst->flow_hash = src->flow_hash; + dst->timestamp = src->timestamp; + dst->buf_hdr.buf_u64 = src->buf_hdr.buf_u64; + dst->buf_hdr.uarea_addr = src->buf_hdr.uarea_addr; +} + +int odp_packet_extend_head(odp_packet_t *pkt, uint32_t len, + void **data_ptr, uint32_t *seg_len) { odp_packet_hdr_t *pkt_hdr = packet_hdr(*pkt); - uint32_t seg_len = packet_first_seg_len(pkt_hdr); + int rv; - if (len > pkt_hdr->frame_len) - return -1; + if (len <= pkt_hdr->headroom) { + if (odp_packet_push_head(*pkt, len) == NULL) + return -1; - if (len < seg_len) { - pull_head(pkt_hdr, len); - } else if (!CONFIG_PACKET_SEG_DISABLED) { - int num = 0; - uint32_t pull_len = 0; + if (data_ptr) + *data_ptr = pkt_hdr->buf_hdr.base_data; + if (seg_len) + *seg_len = pkt_hdr->buf_hdr.size; - while (seg_len <= len) { - pull_len = len - seg_len; - num++; - seg_len += packet_seg_len(pkt_hdr, num); - } + rv = 0; + } else { + pool_t *pool = pkt_hdr->buf_hdr.pool_ptr; + odp_packet_t head; - pkt_hdr = free_segments(pkt_hdr, num, len - pull_len, - pull_len, 1); - *pkt = packet_handle(pkt_hdr); - } + if (odp_unlikely(pkt_hdr->frame_len + len > pool->max_len)) + return -1; - if (data_ptr) - *data_ptr = packet_data(pkt_hdr); + pkt_hdr->frame_len += pkt_hdr->headroom; + pkt_hdr->buf_hdr.base_data -= pkt_hdr->headroom; + pkt_hdr->buf_hdr.size += pkt_hdr->headroom; - if (seg_len_out) - *seg_len_out = packet_first_seg_len(pkt_hdr); + if (packet_alloc(pool, len - pkt_hdr->headroom, 1, &head) != 1) + return -1; - return 0; + concat_seg((odp_packet_hdr_t *)(uintptr_t)head, pkt_hdr); + + packet_hdr(head)->frame_len += pkt_hdr->frame_len; + packet_hdr(head)->tailroom = pkt_hdr->tailroom; + + packet_copy_md(packet_hdr(head), pkt_hdr); + + if (data_ptr) + *data_ptr = packet_hdr(head)->buf_hdr.base_data; + if (seg_len) + *seg_len = packet_hdr(head)->buf_hdr.size; + + *pkt = head; + + rv = 1; + } + return rv; } -void *odp_packet_push_tail(odp_packet_t pkt, uint32_t len) +int odp_packet_trunc_head(odp_packet_t *pkt, uint32_t len, + void **data_ptr, uint32_t *seg_len) { - odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); - void *old_tail; + odp_packet_hdr_t *pkt_hdr = packet_hdr(*pkt); + int rv = -1; - if (len > pkt_hdr->tailroom) - return NULL; + if (len > pkt_hdr->frame_len) + return rv; - ODP_ASSERT(odp_packet_has_ref(pkt) == 0); + if (len < pkt_hdr->buf_hdr.size) { + if (odp_packet_pull_head(*pkt, len) == NULL) + return -1; - old_tail = packet_tail(pkt_hdr); - push_tail(pkt_hdr, len); + if (data_ptr) + *data_ptr = pkt_hdr->buf_hdr.base_data; + if (seg_len) + *seg_len = pkt_hdr->buf_hdr.size; - return old_tail; + rv = 0; + } else { + uint32_t bytes_skipped, segs_skipped; + odp_packet_hdr_t *head = get_seg_at_offset(pkt_hdr, len, + &bytes_skipped, + &segs_skipped); + ODP_ASSERT(bytes_skipped > 0); + ODP_ASSERT(segs_skipped > 0); + + packet_copy_md(head, pkt_hdr); + + head->buf_hdr.segcount = + pkt_hdr->buf_hdr.segcount - segs_skipped; + head->frame_len = pkt_hdr->frame_len - bytes_skipped; + head->headroom = 0; + head->tailroom = pkt_hdr->tailroom; + + packet_free_segs(pkt_hdr, segs_skipped, 0); + + if (odp_packet_pull_head( + (odp_packet_t)head, len - bytes_skipped) == NULL) + return -1; + + if (data_ptr) + *data_ptr = head->buf_hdr.base_data; + if (seg_len) + *seg_len = head->buf_hdr.size; + + *pkt = (odp_packet_t)head; + + rv = 1; + } + return rv; } int odp_packet_extend_tail(odp_packet_t *pkt, uint32_t len, void **data_ptr, uint32_t *seg_len_out) { odp_packet_hdr_t *pkt_hdr = packet_hdr(*pkt); - uint32_t frame_len = pkt_hdr->frame_len; - uint32_t tailroom = pkt_hdr->tailroom; - uint32_t tail_off = frame_len; - int ret = 0; - - ODP_ASSERT(odp_packet_has_ref(*pkt) == 0); + uint32_t old_frame_len = pkt_hdr->frame_len; + uint32_t seg_len = 0; + void *offset; - if (len > tailroom) { + if (len <= pkt_hdr->tailroom) { + if (odp_packet_push_tail(*pkt, len) == NULL) + return -1; + } else { pool_t *pool = pkt_hdr->buf_hdr.pool_ptr; - int num; - void *ptr; + odp_packet_hdr_t *seg; + odp_packet_t tail; - if (odp_unlikely((frame_len + len) > pool->max_len)) + if (odp_unlikely(pkt_hdr->frame_len + len > pool->max_len)) return -1; - num = num_segments(len - tailroom, pool->seg_len); - push_tail(pkt_hdr, tailroom); - ptr = add_segments(pkt_hdr, pool, len - tailroom, num, 0); + seg = packet_last_seg(pkt_hdr); + seg->buf_hdr.size += pkt_hdr->tailroom; - if (ptr == NULL) { - /* segment alloc failed, rollback changes */ - pull_tail(pkt_hdr, tailroom); + if (packet_alloc(pool, len - pkt_hdr->tailroom, 1, &tail) != 1) return -1; - } - } else { - push_tail(pkt_hdr, len); - } - - if (data_ptr) - *data_ptr = packet_map(pkt_hdr, tail_off, seg_len_out, NULL); - - return ret; -} -void *odp_packet_pull_tail(odp_packet_t pkt, uint32_t len) -{ - odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); - seg_entry_t *last_seg = seg_entry_last(pkt_hdr); + concat_seg(pkt_hdr, packet_hdr(tail)); - ODP_ASSERT(odp_packet_has_ref(pkt) == 0); + pkt_hdr->frame_len += len; + pkt_hdr->tailroom = packet_hdr(tail)->tailroom; + } - if (len > last_seg->len) - return NULL; + if (data_ptr || seg_len_out) + offset = packet_map(pkt_hdr, old_frame_len, &seg_len, NULL); - pull_tail(pkt_hdr, len); + if (data_ptr) + *data_ptr = offset; + if (seg_len_out) + *seg_len_out = seg_len; - return packet_tail(pkt_hdr); + return 0; } int odp_packet_trunc_tail(odp_packet_t *pkt, uint32_t len, void **tail_ptr, uint32_t *tailroom) { - int last; - uint32_t seg_len; - seg_entry_t *last_seg; odp_packet_hdr_t *pkt_hdr = packet_hdr(*pkt); + odp_packet_hdr_t *pkt_hdr_last = packet_last_seg(pkt_hdr); - if (len > pkt_hdr->frame_len) - return -1; + if (len <= pkt_hdr_last->buf_hdr.size) { + if (odp_packet_pull_tail(*pkt, len) == NULL) + return -1; + } else { + uint32_t bytes_skipped, segs_skipped; + odp_packet_hdr_t *last; + int last_size; + int num; - ODP_ASSERT(odp_packet_has_ref(*pkt) == 0); + last = get_seg_at_offset(pkt_hdr, pkt_hdr->frame_len - len, + &bytes_skipped, &segs_skipped); - last = packet_last_seg(pkt_hdr); - last_seg = seg_entry_last(pkt_hdr); - seg_len = last_seg->len; + num = pkt_hdr->buf_hdr.segcount - (segs_skipped + 1); + packet_free_segs(pkt_hdr, num, segs_skipped + 1); - if (len < seg_len) { - pull_tail(pkt_hdr, len); - } else if (!CONFIG_PACKET_SEG_DISABLED) { - int num = 0; - uint32_t pull_len = 0; + pkt_hdr->buf_hdr.segcount -= num; + last->buf_hdr.next_seg = NULL; - while (seg_len <= len) { - pull_len = len - seg_len; - num++; - seg_len += packet_seg_len(pkt_hdr, last - num); - } + pkt_hdr->frame_len -= len; - free_segments(pkt_hdr, num, len - pull_len, pull_len, 0); + last_size = pkt_hdr->frame_len - bytes_skipped; + pkt_hdr->tailroom = last->buf_hdr.size - last_size; + last->buf_hdr.size = last_size; } if (tail_ptr) - *tail_ptr = packet_tail(pkt_hdr); - + *tail_ptr = odp_packet_tail(*pkt); if (tailroom) *tailroom = pkt_hdr->tailroom; - return 0; -} - -void *odp_packet_offset(odp_packet_t pkt, uint32_t offset, uint32_t *len, - odp_packet_seg_t *seg) -{ - int seg_idx; - odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); - void *addr = packet_map(pkt_hdr, offset, len, &seg_idx); - - if (addr != NULL && seg != NULL) - *seg = _odp_packet_seg_from_ndx(seg_idx); - return addr; + return 0; } -/* - * - * Meta-data - * ******************************************************** - * - */ uint32_t odp_packet_user_area_size(odp_packet_t pkt) { pool_t *pool = pool_entry_from_hdl(odp_packet_pool(pkt)); @@ -1295,13 +807,6 @@ void odp_packet_ts_set(odp_packet_t pkt, odp_time_t timestamp) pkt_hdr->p.input_flags.timestamp = 1; } -/* - * - * Segment level - * ******************************************************** - * - */ - void *odp_packet_seg_data(odp_packet_t pkt, odp_packet_seg_t seg) { odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); @@ -1310,26 +815,21 @@ void *odp_packet_seg_data(odp_packet_t pkt, odp_packet_seg_t seg) pkt_hdr->buf_hdr.segcount)) return NULL; - return packet_seg_data(pkt_hdr, _odp_packet_seg_to_ndx(seg)); + pkt_hdr = get_seg(pkt_hdr, _odp_packet_seg_to_ndx(seg)); + + return pkt_hdr->buf_hdr.base_data; } uint32_t odp_packet_seg_data_len(odp_packet_t pkt, odp_packet_seg_t seg) { odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); - if (odp_unlikely(_odp_packet_seg_to_ndx(seg) >= - pkt_hdr->buf_hdr.segcount)) - return 0; + ODP_ASSERT(_odp_packet_seg_to_ndx(seg) < pkt_hdr->buf_hdr.segcount); - return packet_seg_len(pkt_hdr, _odp_packet_seg_to_ndx(seg)); -} + pkt_hdr = get_seg(pkt_hdr, _odp_packet_seg_to_ndx(seg)); -/* - * - * Manipulation - * ******************************************************** - * - */ + return pkt_hdr->buf_hdr.size; +} int odp_packet_add_data(odp_packet_t *pkt_ptr, uint32_t offset, uint32_t len) { @@ -1403,8 +903,6 @@ int odp_packet_align(odp_packet_t *pkt, uint32_t offset, uint32_t len, if (align > ODP_CACHE_LINE_SIZE) return -1; - ODP_ASSERT(odp_packet_has_ref(*pkt) == 0); - if (seglen >= len) { misalign = align <= 1 ? 0 : ROUNDUP_ALIGN(uaddr, align) - uaddr; @@ -1442,8 +940,6 @@ int odp_packet_concat(odp_packet_t *dst, odp_packet_t src) uint32_t dst_len = dst_hdr->frame_len; uint32_t src_len = src_hdr->frame_len; - ODP_ASSERT(odp_packet_has_ref(*dst) == 0); - /* Do a copy if packets are from different pools. */ if (odp_unlikely(dst_pool != src_pool)) { if (odp_packet_extend_tail(dst, src_len, NULL, NULL) >= 0) { @@ -1458,7 +954,7 @@ int odp_packet_concat(odp_packet_t *dst, odp_packet_t src) return -1; } - add_all_segs(dst_hdr, src_hdr); + concat_seg(dst_hdr, src_hdr); dst_hdr->frame_len = dst_len + src_len; dst_hdr->tailroom = src_hdr->tailroom; @@ -1474,8 +970,6 @@ int odp_packet_split(odp_packet_t *pkt, uint32_t len, odp_packet_t *tail) if (len >= pktlen || tail == NULL) return -1; - ODP_ASSERT(odp_packet_has_ref(*pkt) == 0); - *tail = odp_packet_copy_part(*pkt, len, pktlen - len, odp_packet_pool(*pkt)); @@ -1485,13 +979,6 @@ int odp_packet_split(odp_packet_t *pkt, uint32_t len, odp_packet_t *tail) return odp_packet_trunc_tail(pkt, pktlen - len, NULL, NULL); } -/* - * - * Copy - * ******************************************************** - * - */ - odp_packet_t odp_packet_copy(odp_packet_t pkt, odp_pool_t pool) { odp_packet_hdr_t *srchdr = packet_hdr(pkt); @@ -1689,19 +1176,9 @@ int _odp_packet_cmp_data(odp_packet_t pkt, uint32_t offset, return 0; } -/* - * - * Debugging - * ******************************************************** - * - */ void odp_packet_print(odp_packet_t pkt) { - odp_packet_seg_t seg; - seg_entry_t *seg_entry; - odp_packet_hdr_t *seg_hdr; - uint8_t idx; - int max_len = 1024; + int max_len = 2048; char str[max_len]; int len = 0; int n = max_len - 1; @@ -1717,17 +1194,20 @@ void odp_packet_print(odp_packet_t pkt) len += snprintf(&str[len], n - len, " output_flags 0x%" PRIx32 "\n", hdr->p.output_flags.all); + len += snprintf(&str[len], n - len, + " uarea_addr 0x%p\n", + hdr->buf_hdr.uarea_addr); len += snprintf(&str[len], n - len, " l2_offset %" PRIu32 "\n", hdr->p.l2_offset); len += snprintf(&str[len], n - len, " l3_offset %" PRIu32 "\n", hdr->p.l3_offset); len += snprintf(&str[len], n - len, " l4_offset %" PRIu32 "\n", hdr->p.l4_offset); - len += snprintf(&str[len], n - len, - " frame_len %" PRIu32 "\n", hdr->frame_len); len += snprintf(&str[len], n - len, " input %" PRIu64 "\n", odp_pktio_to_u64(hdr->input)); + len += snprintf(&str[len], n - len, + " frame_len %" PRIu32 "\n", hdr->frame_len); len += snprintf(&str[len], n - len, " headroom %" PRIu32 "\n", odp_packet_headroom(pkt)); @@ -1737,37 +1217,23 @@ void odp_packet_print(odp_packet_t pkt) len += snprintf(&str[len], n - len, " num_segs %i\n", odp_packet_num_segs(pkt)); - seg_hdr = hdr; - idx = 0; - seg = odp_packet_first_seg(pkt); - - while (seg != ODP_PACKET_SEG_INVALID) { - odp_buffer_hdr_t *buf_hdr; - odp_packet_hdr_t *tmp_hdr; - - tmp_hdr = seg_hdr; - seg_entry = seg_entry_next(&seg_hdr, &idx); - buf_hdr = seg_entry->hdr; - + do { len += snprintf(&str[len], n - len, - " seg_len %-4" PRIu32 " seg_data %p ", - odp_packet_seg_data_len(pkt, seg), - odp_packet_seg_data(pkt, seg)); - len += snprintf(&str[len], n - len, "ref_cnt %u", - buffer_ref(buf_hdr)); - if (seg_is_link(tmp_hdr)) { - uint32_t ref; - - ref = buffer_ref(&tmp_hdr->buf_hdr); - len += snprintf(&str[len], n - len, "L(%u)\n", ref); - } else { - len += snprintf(&str[len], n - len, "\n"); - } + " %p ref_cnt=%u size=%-5u " + "base_data=%p buf_end=%p next_seg=%p\n", + hdr, + buffer_ref(&hdr->buf_hdr), + hdr->buf_hdr.size, + hdr->buf_hdr.base_data, + hdr->buf_hdr.buf_end, + hdr->buf_hdr.next_seg); - seg = odp_packet_next_seg(pkt, seg); - } + hdr = hdr->buf_hdr.next_seg; + } while (hdr); - ODP_PRINT("%s\n", str); + str[len] = '\0'; + + ODP_PRINT("\n%s\n", str); } void odp_packet_print_data(odp_packet_t pkt, uint32_t offset, @@ -1843,13 +1309,6 @@ int odp_packet_is_valid(odp_packet_t pkt) return 1; } -/* - * - * Internal Use Routines - * ******************************************************** - * - */ - int _odp_packet_copy_md_to_packet(odp_packet_t srcpkt, odp_packet_t dstpkt) { odp_packet_hdr_t *srchdr = packet_hdr(srcpkt); @@ -2222,11 +1681,11 @@ int packet_parse_common(packet_parser_t *prs, const uint8_t *ptr, int packet_parse_layer(odp_packet_hdr_t *pkt_hdr, odp_pktio_parser_layer_t layer) { - uint32_t seg_len = packet_first_seg_len(pkt_hdr); - void *base = packet_data(pkt_hdr); - - return packet_parse_common(&pkt_hdr->p, base, pkt_hdr->frame_len, - seg_len, layer); + return packet_parse_common(&pkt_hdr->p, + pkt_hdr->buf_hdr.base_data, + pkt_hdr->frame_len, + pkt_hdr->buf_hdr.size, + layer); } int packet_parse_l3_l4(odp_packet_hdr_t *pkt_hdr, @@ -2266,16 +1725,11 @@ odp_packet_t odp_packet_ref_static(odp_packet_t pkt) odp_packet_t odp_packet_ref(odp_packet_t pkt, uint32_t offset) { - odp_packet_t ref; - odp_packet_hdr_t *link_hdr; - odp_packet_hdr_t *next_hdr; odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); - odp_packet_hdr_t *hdr = pkt_hdr; - seg_entry_t *seg; - uint32_t seg_idx = 0; - uint8_t idx = 0; - uint32_t seg_offset = 0; - int i, num_copy, segcount; + odp_packet_hdr_t *seg; + odp_packet_hdr_t *ref_hdr; + odp_packet_t ref; + uint32_t bytes_skipped, segs_skipped; uint32_t len; if (offset >= pkt_hdr->frame_len) { @@ -2283,73 +1737,33 @@ odp_packet_t odp_packet_ref(odp_packet_t pkt, uint32_t offset) return ODP_PACKET_INVALID; } - /* Allocate link segment */ - if (packet_alloc(pkt_hdr->buf_hdr.pool_ptr, 0, 1, 1, &ref) != 1) { + seg = get_seg_at_offset(pkt_hdr, offset, &bytes_skipped, &segs_skipped); + + if (packet_alloc(pkt_hdr->buf_hdr.pool_ptr, 0, 1, &ref) != 1) { ODP_DBG("segment alloc failed\n"); return ODP_PACKET_INVALID; } + ref_hdr = packet_hdr(ref); - link_hdr = packet_hdr(ref); - - seg_entry_find_offset(&hdr, &idx, &seg_offset, &seg_idx, offset); - num_copy = hdr->buf_hdr.num_seg - idx; - segcount = pkt_hdr->buf_hdr.segcount; - - /* In addition to segments, update reference count of - * an existing link header. */ - if (seg_is_link(hdr)) - buffer_ref_inc((odp_buffer_hdr_t *)hdr); - - seg = seg_entry_next(&hdr, &idx); - link_hdr->buf_hdr.num_seg = 1; - link_hdr->buf_hdr.seg[0].hdr = seg->hdr; - link_hdr->buf_hdr.seg[0].data = seg->data + seg_offset; - link_hdr->buf_hdr.seg[0].len = seg->len - seg_offset; - buffer_ref_inc(seg->hdr); - - /* The 'CONFIG_PACKET_SEGS_PER_HDR > 1' condition is required to fix an - * invalid error ('array subscript is above array bounds') thrown by - * gcc (5.4.0). */ - for (i = 1; CONFIG_PACKET_SEGS_PER_HDR > 1 && i < num_copy; i++) { - /* Update link header reference count */ - if (idx == 0 && seg_is_link(hdr)) - buffer_ref_inc((odp_buffer_hdr_t *)hdr); - - seg = seg_entry_next(&hdr, &idx); - - link_hdr->buf_hdr.num_seg++; - link_hdr->buf_hdr.seg[i].hdr = seg->hdr; - link_hdr->buf_hdr.seg[i].data = seg->data; - link_hdr->buf_hdr.seg[i].len = seg->len; - buffer_ref_inc(seg->hdr); - } + len = pkt_hdr->frame_len - offset; - next_hdr = hdr; + ref_hdr->buf_hdr.segcount = + 1 + (pkt_hdr->buf_hdr.segcount - segs_skipped); + ref_hdr->buf_hdr.next_seg = seg; - /* Increment ref count for remaining segments */ - for (i = seg_idx + num_copy; i < segcount; i++) { - /* Update link header reference count */ - if (idx == 0 && seg_is_link(hdr)) - buffer_ref_inc((odp_buffer_hdr_t *)hdr); + ref_hdr->frame_len = len; - seg = seg_entry_next(&hdr, &idx); - buffer_ref_inc(seg->hdr); - } + ref_hdr->tailroom = pkt_hdr->tailroom; + ref_hdr->headroom = 0; - len = pkt_hdr->frame_len - offset; - link_hdr->buf_hdr.next_seg = next_hdr; - link_hdr->buf_hdr.last_seg = pkt_hdr->buf_hdr.last_seg; - link_hdr->buf_hdr.segcount = segcount - seg_idx; - link_hdr->frame_len = len; - link_hdr->tailroom = pkt_hdr->tailroom; + /* Bump refcnt of trailing segments. */ + do { + buffer_ref_inc(&seg->buf_hdr); - /* Link header does not have headroom, it just points to other - * buffers. Zero length headroom ensures that head of the other buffer - * is not pushed through a reference. */ - link_hdr->headroom = 0; + seg = seg->buf_hdr.next_seg; + } while (seg); return ref; - } odp_packet_t odp_packet_ref_pkt(odp_packet_t pkt, uint32_t offset, @@ -2374,26 +1788,20 @@ odp_packet_t odp_packet_ref_pkt(odp_packet_t pkt, uint32_t offset, } return hdr; + } int odp_packet_has_ref(odp_packet_t pkt) { - odp_buffer_hdr_t *buf_hdr; - seg_entry_t *seg; - int i; - uint32_t ref_cnt; - odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); - int seg_count = pkt_hdr->buf_hdr.segcount; - odp_packet_hdr_t *hdr = pkt_hdr; - uint8_t idx = 0; + odp_packet_hdr_t *pkt_hdr = (odp_packet_hdr_t *)(uintptr_t)pkt; - for (i = 0; i < seg_count; i++) { - seg = seg_entry_next(&hdr, &idx); - buf_hdr = seg->hdr; - ref_cnt = buffer_ref(buf_hdr); + while (pkt_hdr) { + uint32_t ref_cnt = buffer_ref(&pkt_hdr->buf_hdr); if (is_multi_ref(ref_cnt)) return 1; + + pkt_hdr = pkt_hdr->buf_hdr.next_seg; } return 0; diff --git a/platform/linux-generic/pktio/dpdk.c b/platform/linux-generic/pktio/dpdk.c index bdd12292fc..42df14ddd2 100644 --- a/platform/linux-generic/pktio/dpdk.c +++ b/platform/linux-generic/pktio/dpdk.c @@ -106,8 +106,7 @@ static unsigned cache_size(uint32_t num) static inline uint16_t mbuf_data_off(struct rte_mbuf *mbuf, odp_packet_hdr_t *pkt_hdr) { - return (uint64_t)pkt_hdr->buf_hdr.seg[0].data - - (uint64_t)mbuf->buf_addr; + return (uint64_t)pkt_hdr->buf_hdr.base_data - (uint64_t)mbuf->buf_addr; } /** @@ -122,8 +121,7 @@ static inline void mbuf_update(struct rte_mbuf *mbuf, odp_packet_hdr_t *pkt_hdr, mbuf->pkt_len = pkt_len; mbuf->refcnt = 1; - if (odp_unlikely(pkt_hdr->buf_hdr.base_data != - pkt_hdr->buf_hdr.seg[0].data)) + if (odp_unlikely(pkt_hdr->headroom != CONFIG_PACKET_HEADROOM)) mbuf->data_off = mbuf_data_off(mbuf, pkt_hdr); } @@ -647,7 +645,7 @@ static inline int mbuf_to_pkt_zero(pktio_entry_t *pktio_entry, /* Init buffer segments. Currently, only single segment packets * are supported. */ - pkt_hdr->buf_hdr.seg[0].data = data; + pkt_hdr->buf_hdr.base_data = data; packet_init(pkt_hdr, pkt_len); pkt_hdr->input = pktio_entry->s.handle; diff --git a/platform/linux-generic/pktio/ipc.c b/platform/linux-generic/pktio/ipc.c index 64c7606dfd..66c3dfad68 100644 --- a/platform/linux-generic/pktio/ipc.c +++ b/platform/linux-generic/pktio/ipc.c @@ -658,7 +658,7 @@ static int ipc_pktio_send_lockless(pktio_entry_t *pktio_entry, offsets[i] = (uint8_t *)pkt_hdr - (uint8_t *)odp_shm_addr(pool->shm); - data_pool_off = (uint8_t *)pkt_hdr->buf_hdr.seg[0].data - + data_pool_off = (uint8_t *)pkt_hdr->buf_hdr.base_data - (uint8_t *)odp_shm_addr(pool->shm); /* compile all function code even if ipc disabled with config */ diff --git a/platform/linux-generic/pool/generic.c b/platform/linux-generic/pool/generic.c index 4d5f594610..85a1ebe314 100644 --- a/platform/linux-generic/pool/generic.c +++ b/platform/linux-generic/pool/generic.c @@ -207,7 +207,6 @@ static void init_buffers(pool_t *pool) ring_t *ring; uint32_t mask; int type; - uint32_t seg_size; uint64_t page_size; int skipped_blocks = 0; @@ -256,33 +255,25 @@ static void init_buffers(pool_t *pool) memset(buf_hdr, 0, (uintptr_t)data - (uintptr_t)buf_hdr); - seg_size = pool->headroom + pool->seg_len + pool->tailroom; - /* Initialize buffer metadata */ - buf_hdr->size = seg_size; + buf_hdr->size = pool->seg_len; buf_hdr->type = type; buf_hdr->event_type = type; buf_hdr->event_subtype = ODP_EVENT_NO_SUBTYPE; buf_hdr->pool_hdl = pool->pool_hdl; buf_hdr->pool_ptr = pool; buf_hdr->uarea_addr = uarea; - /* Show user requested size through API */ - buf_hdr->segcount = 1; - buf_hdr->num_seg = 1; + buf_hdr->segcount = 0; buf_hdr->next_seg = NULL; - buf_hdr->last_seg = buf_hdr; - - /* Pointer to data start (of the first segment) */ - buf_hdr->seg[0].hdr = buf_hdr; - buf_hdr->seg[0].data = &data[offset]; - buf_hdr->seg[0].len = pool->seg_len; odp_atomic_init_u32(&buf_hdr->ref_cnt, 0); - /* Store base values for fast init */ - buf_hdr->base_data = buf_hdr->seg[0].data; - buf_hdr->buf_end = &data[offset + pool->seg_len + - pool->tailroom]; + ODP_ASSERT(offset <= 255); + buf_hdr->pristine_offset = offset; + + buf_hdr->base_data = &data[offset]; + buf_hdr->buf_end = + &data[offset + pool->seg_len + pool->tailroom]; /* Store buffer index into the global pool */ ring_enq(ring, mask, i); diff --git a/test/validation/api/packet/packet.c b/test/validation/api/packet/packet.c index 37550a2f55..6f1d10abf4 100644 --- a/test/validation/api/packet/packet.c +++ b/test/validation/api/packet/packet.c @@ -2193,7 +2193,7 @@ void packet_test_ref(void) odp_packet_pool(segmented_test_packet)); CU_ASSERT_FATAL(hdr_pkt[1] != ODP_PACKET_INVALID); hdr_len[1] = odp_packet_len(hdr_pkt[1]); - offset[1] = 5; + offset[1] = 0; hdr_pkt[2] = odp_packet_copy_part(test_packet, 0, odp_packet_len(test_packet) / 4, From 4de1028b88d842fc65984b67e94dafc332412371 Mon Sep 17 00:00:00 2001 From: Brian Brooks Date: Mon, 18 Dec 2017 13:57:23 -0600 Subject: [PATCH 4/4] linux-gen: packet: calculate the start of packet pointer The start of packet pointer was adjusted on every operation involving headroom manipulation. This meant that this pointer had to be restored to its pristine (aligned) offset when the buffer was recycled for reuse. Instead of maintaining a pristine offset that could be used to restore this pointer, use a (fixed) pointer to the start of the buffer i.e. start of headroom. On operations involving headroom manipulation, only modify the headroom offset value. This means that the start of packet pointer must be calculated as start of buffer + headroom. Signed-off-by: Brian Brooks Signed-off-by: Honnappa Nagarahalli --- platform/linux-generic/buffer/generic.c | 4 +- .../include/odp/api/plat/packet_inlines.h | 15 ++--- .../include/odp/api/plat/packet_types.h | 2 +- .../include/odp_buffer_internal.h | 11 ++-- .../include/odp_packet_internal.h | 12 ++-- platform/linux-generic/odp_packet.c | 59 +++++++++---------- platform/linux-generic/pktio/dpdk.c | 6 +- platform/linux-generic/pktio/ipc.c | 2 +- platform/linux-generic/pool/generic.c | 13 ++-- 9 files changed, 58 insertions(+), 66 deletions(-) diff --git a/platform/linux-generic/buffer/generic.c b/platform/linux-generic/buffer/generic.c index 4acca35225..7f0060124d 100644 --- a/platform/linux-generic/buffer/generic.c +++ b/platform/linux-generic/buffer/generic.c @@ -31,7 +31,7 @@ static void *generic_buffer_addr(odp_buffer_t buf) { odp_buffer_hdr_t *hdr = buf_hdl_to_hdr(buf); - return hdr->base_data; + return hdr->buf_start; } static uint32_t generic_buffer_size(odp_buffer_t buf) @@ -61,7 +61,7 @@ int odp_buffer_snprint(char *str, uint32_t n, odp_buffer_t buf) " pool %" PRIu64 "\n", odp_pool_to_u64(pool->pool_hdl)); len += snprintf(&str[len], n - len, - " addr %p\n", hdr->base_data); + " addr %p\n", hdr->buf_start); len += snprintf(&str[len], n - len, " size %" PRIu32 "\n", hdr->size); len += snprintf(&str[len], n - len, diff --git a/platform/linux-generic/include/odp/api/plat/packet_inlines.h b/platform/linux-generic/include/odp/api/plat/packet_inlines.h index 06bcf8557e..af7c54ea7f 100644 --- a/platform/linux-generic/include/odp/api/plat/packet_inlines.h +++ b/platform/linux-generic/include/odp/api/plat/packet_inlines.h @@ -35,12 +35,6 @@ static inline odp_packet_seg_t _odp_packet_seg_from_ndx(uint32_t ndx) } #endif -/** @internal Inline function @param pkt @return */ -static inline void *_odp_packet_data(odp_packet_t pkt) -{ - return _odp_pkt_get(pkt, void *, data); -} - /** @internal Inline function @param pkt @return */ static inline uint32_t _odp_packet_seg_len(odp_packet_t pkt) { @@ -65,6 +59,13 @@ static inline uint32_t _odp_packet_tailroom(odp_packet_t pkt) return _odp_pkt_get(pkt, uint16_t, tailroom); } +/** @internal Inline function @param pkt @return */ +static inline void *_odp_packet_data(odp_packet_t pkt) +{ + return _odp_pkt_get(pkt, uint8_t *, buf_start) + + _odp_packet_headroom(pkt); +} + /** @internal Inline function @param pkt @return */ static inline odp_pool_t _odp_packet_pool(odp_packet_t pkt) { @@ -110,7 +111,7 @@ static inline odp_time_t _odp_packet_ts(odp_packet_t pkt) /** @internal Inline function @param pkt @return */ static inline void *_odp_packet_head(odp_packet_t pkt) { - return (uint8_t *)_odp_packet_data(pkt) - _odp_packet_headroom(pkt); + return _odp_pkt_get(pkt, void *, buf_start); } /** @internal Inline function @param pkt @return */ diff --git a/platform/linux-generic/include/odp/api/plat/packet_types.h b/platform/linux-generic/include/odp/api/plat/packet_types.h index 009a3aa7c7..e3774cf0d0 100644 --- a/platform/linux-generic/include/odp/api/plat/packet_types.h +++ b/platform/linux-generic/include/odp/api/plat/packet_types.h @@ -74,7 +74,7 @@ typedef enum { /** @internal Packet header field offsets for inline functions */ typedef struct _odp_packet_inline_offset_t { /** @internal field offset */ - uint16_t data; + uint16_t buf_start; /** @internal field offset */ uint16_t seg_len; /** @internal field offset */ diff --git a/platform/linux-generic/include/odp_buffer_internal.h b/platform/linux-generic/include/odp_buffer_internal.h index 0046afb3a4..dea7d6261c 100644 --- a/platform/linux-generic/include/odp_buffer_internal.h +++ b/platform/linux-generic/include/odp_buffer_internal.h @@ -43,14 +43,13 @@ struct odp_buffer_hdr_t { /* Pool type */ int8_t type; - /* Offset used to restore base_data */ - uint8_t pristine_offset; + /* uint8_t unused0; */ /* Next header which continues the segment list */ void *next_seg; - /* Pointer to start of segment */ - uint8_t *base_data; + /* Pointer to start of aligned buffer */ + uint8_t *buf_start; /* Pool pointer */ void *pool_ptr; @@ -81,8 +80,8 @@ struct odp_buffer_hdr_t { /* Event type. Maybe different than pool type (crypto compl event) */ int8_t event_type; - /* Initial buffer tail pointer */ - uint8_t *buf_end; + /* Pointer to end of buffer */ + uint8_t *buf_end; /* User area pointer */ void *uarea_addr; diff --git a/platform/linux-generic/include/odp_packet_internal.h b/platform/linux-generic/include/odp_packet_internal.h index 8f381f0977..91f7a1425f 100644 --- a/platform/linux-generic/include/odp_packet_internal.h +++ b/platform/linux-generic/include/odp_packet_internal.h @@ -180,6 +180,11 @@ static inline odp_packet_t packet_from_buf_hdr(odp_buffer_hdr_t *buf_hdr) return (odp_packet_t)(odp_packet_hdr_t *)buf_hdr; } +static inline uint8_t *packet_base_data(odp_packet_hdr_t *pkt_hdr) +{ + return pkt_hdr->buf_hdr.buf_start + pkt_hdr->headroom; +} + static inline odp_packet_hdr_t *packet_last_seg(odp_packet_hdr_t *pkt_hdr) { int segcount = pkt_hdr->buf_hdr.segcount; @@ -214,18 +219,13 @@ static inline void packet_init(odp_packet_hdr_t *pkt_hdr, uint32_t len) pkt_hdr->buf_hdr.event_subtype = ODP_EVENT_PACKET_BASIC; - /* Restore each segment's base_data and size fields */ + /* Restore each segment's size field */ while (len > pool->seg_len) { - pkt_hdr->buf_hdr.base_data = - &pkt_hdr->data[pkt_hdr->buf_hdr.pristine_offset]; pkt_hdr->buf_hdr.size = pool->seg_len; len -= pool->seg_len; - pkt_hdr = pkt_hdr->buf_hdr.next_seg; } - pkt_hdr->buf_hdr.base_data = - &pkt_hdr->data[pkt_hdr->buf_hdr.pristine_offset]; pkt_hdr->buf_hdr.size = len; old_pkt_hdr->tailroom += pool->seg_len - len; diff --git a/platform/linux-generic/odp_packet.c b/platform/linux-generic/odp_packet.c index cae38fbda7..6c570677d0 100644 --- a/platform/linux-generic/odp_packet.c +++ b/platform/linux-generic/odp_packet.c @@ -28,7 +28,7 @@ /* Fill in packet header field offsets for inline functions */ const _odp_packet_inline_offset_t _odp_packet_inline ODP_ALIGNED_CACHE = { - .data = offsetof(odp_packet_hdr_t, buf_hdr.base_data), + .buf_start = offsetof(odp_packet_hdr_t, buf_hdr.buf_start), .seg_len = offsetof(odp_packet_hdr_t, buf_hdr.size), .frame_len = offsetof(odp_packet_hdr_t, frame_len), .headroom = offsetof(odp_packet_hdr_t, headroom), @@ -174,7 +174,7 @@ static void *packet_map(odp_packet_hdr_t *pkt_hdr, uint32_t offset, if (seg_idx) *seg_idx = skipped_segs; - return pkt_hdr->buf_hdr.base_data + offset; + return packet_base_data(pkt_hdr) + offset; } static void buffer_ref_inc(odp_buffer_hdr_t *buf_hdr) @@ -236,17 +236,11 @@ static int packet_alloc(pool_t *pool, uint32_t len, int num_pkt, } for (int i = 0; i < npkt; i++) { - packet_init_segs(&pkt_hdr[i * segs_per_pkt], segs_per_pkt); - packet_init(pkt_hdr[i * segs_per_pkt], len); + int seg = i * segs_per_pkt; - pkts[i] = (odp_packet_t)pkt_hdr[i * segs_per_pkt]; - } - - for (int i = 0; i < nbuf; i++) { - ODP_ASSERT(((intptr_t)pkt_hdr[i]->buf_hdr.base_data - - (intptr_t)pkt_hdr[i]) <= - (int)(sizeof(odp_packet_hdr_t) + - pool->headroom + pool->align)); + packet_init_segs(&pkt_hdr[seg], segs_per_pkt); + packet_init(pkt_hdr[seg], len); + pkts[i] = (odp_packet_t)pkt_hdr[seg]; } return npkt; @@ -407,8 +401,13 @@ odp_event_t odp_packet_to_event(odp_packet_t pkt) uint32_t odp_packet_buf_len(odp_packet_t pkt) { odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); + uint32_t len = 0; - return packet_buf_len(pkt_hdr) * pkt_hdr->buf_hdr.segcount; + while (pkt_hdr) { + len += packet_buf_len(pkt_hdr); + pkt_hdr = pkt_hdr->buf_hdr.next_seg; + } + return len; } void *odp_packet_tail(odp_packet_t pkt) @@ -439,13 +438,11 @@ void *odp_packet_push_head(odp_packet_t pkt, uint32_t len) if (len > pkt_hdr->headroom) return NULL; - pkt_hdr->buf_hdr.base_data -= len; pkt_hdr->buf_hdr.size += len; - pkt_hdr->headroom -= len; pkt_hdr->frame_len += len; - return pkt_hdr->buf_hdr.base_data; + return packet_base_data(pkt_hdr); } void *odp_packet_pull_head(odp_packet_t pkt, uint32_t len) @@ -455,13 +452,11 @@ void *odp_packet_pull_head(odp_packet_t pkt, uint32_t len) if (len > pkt_hdr->frame_len) return NULL; - pkt_hdr->buf_hdr.base_data += len; pkt_hdr->buf_hdr.size -= len; - pkt_hdr->headroom += len; pkt_hdr->frame_len -= len; - return pkt_hdr->buf_hdr.base_data; + return packet_base_data(pkt_hdr); } void *odp_packet_push_tail(odp_packet_t pkt, uint32_t len) @@ -476,7 +471,6 @@ void *odp_packet_push_tail(odp_packet_t pkt, uint32_t len) old_tail = seg->buf_hdr.buf_end - pkt_hdr->tailroom; seg->buf_hdr.size += len; - pkt_hdr->tailroom -= len; pkt_hdr->frame_len += len; @@ -492,7 +486,6 @@ void *odp_packet_pull_tail(odp_packet_t pkt, uint32_t len) return NULL; seg->buf_hdr.size -= len; - pkt_hdr->tailroom += len; pkt_hdr->frame_len -= len; @@ -522,7 +515,7 @@ int odp_packet_extend_head(odp_packet_t *pkt, uint32_t len, return -1; if (data_ptr) - *data_ptr = pkt_hdr->buf_hdr.base_data; + *data_ptr = packet_base_data(pkt_hdr); if (seg_len) *seg_len = pkt_hdr->buf_hdr.size; @@ -530,15 +523,17 @@ int odp_packet_extend_head(odp_packet_t *pkt, uint32_t len, } else { pool_t *pool = pkt_hdr->buf_hdr.pool_ptr; odp_packet_t head; + uint16_t headroom; if (odp_unlikely(pkt_hdr->frame_len + len > pool->max_len)) return -1; - pkt_hdr->frame_len += pkt_hdr->headroom; - pkt_hdr->buf_hdr.base_data -= pkt_hdr->headroom; - pkt_hdr->buf_hdr.size += pkt_hdr->headroom; + headroom = pkt_hdr->headroom; + pkt_hdr->buf_hdr.size += headroom; + pkt_hdr->headroom -= headroom; + pkt_hdr->frame_len += headroom; - if (packet_alloc(pool, len - pkt_hdr->headroom, 1, &head) != 1) + if (packet_alloc(pool, len - headroom, 1, &head) != 1) return -1; concat_seg((odp_packet_hdr_t *)(uintptr_t)head, pkt_hdr); @@ -549,7 +544,7 @@ int odp_packet_extend_head(odp_packet_t *pkt, uint32_t len, packet_copy_md(packet_hdr(head), pkt_hdr); if (data_ptr) - *data_ptr = packet_hdr(head)->buf_hdr.base_data; + *data_ptr = packet_base_data(packet_hdr(head)); if (seg_len) *seg_len = packet_hdr(head)->buf_hdr.size; @@ -574,7 +569,7 @@ int odp_packet_trunc_head(odp_packet_t *pkt, uint32_t len, return -1; if (data_ptr) - *data_ptr = pkt_hdr->buf_hdr.base_data; + *data_ptr = packet_base_data(pkt_hdr); if (seg_len) *seg_len = pkt_hdr->buf_hdr.size; @@ -602,7 +597,7 @@ int odp_packet_trunc_head(odp_packet_t *pkt, uint32_t len, return -1; if (data_ptr) - *data_ptr = head->buf_hdr.base_data; + *data_ptr = packet_base_data(head); if (seg_len) *seg_len = head->buf_hdr.size; @@ -817,7 +812,7 @@ void *odp_packet_seg_data(odp_packet_t pkt, odp_packet_seg_t seg) pkt_hdr = get_seg(pkt_hdr, _odp_packet_seg_to_ndx(seg)); - return pkt_hdr->buf_hdr.base_data; + return packet_base_data(pkt_hdr); } uint32_t odp_packet_seg_data_len(odp_packet_t pkt, odp_packet_seg_t seg) @@ -1224,7 +1219,7 @@ void odp_packet_print(odp_packet_t pkt) hdr, buffer_ref(&hdr->buf_hdr), hdr->buf_hdr.size, - hdr->buf_hdr.base_data, + packet_base_data(hdr), hdr->buf_hdr.buf_end, hdr->buf_hdr.next_seg); @@ -1682,7 +1677,7 @@ int packet_parse_layer(odp_packet_hdr_t *pkt_hdr, odp_pktio_parser_layer_t layer) { return packet_parse_common(&pkt_hdr->p, - pkt_hdr->buf_hdr.base_data, + packet_base_data(pkt_hdr), pkt_hdr->frame_len, pkt_hdr->buf_hdr.size, layer); diff --git a/platform/linux-generic/pktio/dpdk.c b/platform/linux-generic/pktio/dpdk.c index 42df14ddd2..9aeb4a3ccb 100644 --- a/platform/linux-generic/pktio/dpdk.c +++ b/platform/linux-generic/pktio/dpdk.c @@ -106,7 +106,7 @@ static unsigned cache_size(uint32_t num) static inline uint16_t mbuf_data_off(struct rte_mbuf *mbuf, odp_packet_hdr_t *pkt_hdr) { - return (uint64_t)pkt_hdr->buf_hdr.base_data - (uint64_t)mbuf->buf_addr; + return (uint64_t)packet_base_data(pkt_hdr) - (uint64_t)mbuf->buf_addr; } /** @@ -133,7 +133,7 @@ static inline void mbuf_update(struct rte_mbuf *mbuf, odp_packet_hdr_t *pkt_hdr, static void mbuf_init(struct rte_mempool *mp, struct rte_mbuf *mbuf, odp_packet_hdr_t *pkt_hdr) { - void *buf_addr = pkt_hdr->buf_hdr.base_data - RTE_PKTMBUF_HEADROOM; + void *buf_addr = packet_base_data(pkt_hdr) - RTE_PKTMBUF_HEADROOM; memset(mbuf, 0, sizeof(struct rte_mbuf)); @@ -645,7 +645,7 @@ static inline int mbuf_to_pkt_zero(pktio_entry_t *pktio_entry, /* Init buffer segments. Currently, only single segment packets * are supported. */ - pkt_hdr->buf_hdr.base_data = data; + pkt_hdr->buf_hdr.buf_start = data; packet_init(pkt_hdr, pkt_len); pkt_hdr->input = pktio_entry->s.handle; diff --git a/platform/linux-generic/pktio/ipc.c b/platform/linux-generic/pktio/ipc.c index 66c3dfad68..3c4699db87 100644 --- a/platform/linux-generic/pktio/ipc.c +++ b/platform/linux-generic/pktio/ipc.c @@ -658,7 +658,7 @@ static int ipc_pktio_send_lockless(pktio_entry_t *pktio_entry, offsets[i] = (uint8_t *)pkt_hdr - (uint8_t *)odp_shm_addr(pool->shm); - data_pool_off = (uint8_t *)pkt_hdr->buf_hdr.base_data - + data_pool_off = packet_base_data(pkt_hdr) - (uint8_t *)odp_shm_addr(pool->shm); /* compile all function code even if ipc disabled with config */ diff --git a/platform/linux-generic/pool/generic.c b/platform/linux-generic/pool/generic.c index 85a1ebe314..da907b9eb7 100644 --- a/platform/linux-generic/pool/generic.c +++ b/platform/linux-generic/pool/generic.c @@ -247,11 +247,11 @@ static void init_buffers(pool_t *pool) if (type == ODP_POOL_PACKET) data = pkt_hdr->data; + /* Determine amount of offset needed for desired alignment. */ offset = pool->headroom; - - /* move to correct align */ while (((uintptr_t)&data[offset]) % pool->align != 0) offset++; + offset -= pool->headroom; memset(buf_hdr, 0, (uintptr_t)data - (uintptr_t)buf_hdr); @@ -268,12 +268,9 @@ static void init_buffers(pool_t *pool) odp_atomic_init_u32(&buf_hdr->ref_cnt, 0); - ODP_ASSERT(offset <= 255); - buf_hdr->pristine_offset = offset; - - buf_hdr->base_data = &data[offset]; - buf_hdr->buf_end = - &data[offset + pool->seg_len + pool->tailroom]; + buf_hdr->buf_start = &data[offset]; + buf_hdr->buf_end = buf_hdr->buf_start + + pool->headroom + pool->seg_len + pool->tailroom; /* Store buffer index into the global pool */ ring_enq(ring, mask, i);