Skip to content

Commit

Permalink
net/hns3: support NEON Rx
Browse files Browse the repository at this point in the history 
This patch adds NEON vector instructions to optimize Rx burst process.

Signed-off-by: Chengwen Feng <fengchengwen@huawei.com>
Signed-off-by: Wei Hu (Xavier) <xavier.huwei@huawei.com>
Signed-off-by: Huisong Li <lihuisong@huawei.com>
  • Loading branch information
@xavierhw
xavierhw authored and Ferruh Yigit committed Sep 21, 2020
1 parent e31f123 commit a3d4f4d
Show file tree
Hide file tree
Showing 8 changed files with 514 additions and 8 deletions.
1 change: 1 addition & 0 deletions drivers/net/hns3/hns3_ethdev.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2352,6 +2352,7 @@ hns3_dev_configure(struct rte_eth_dev *dev)
goto cfg_err;

hns->rx_simple_allowed = true;
hns->rx_vec_allowed = true;
hns->tx_simple_allowed = true;
hns->tx_vec_allowed = true;

Expand Down
1 change: 1 addition & 0 deletions drivers/net/hns3/hns3_ethdev.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -643,6 +643,7 @@ struct hns3_adapter {
};

bool rx_simple_allowed;
bool rx_vec_allowed;
bool tx_simple_allowed;
bool tx_vec_allowed;

Expand Down
1 change: 1 addition & 0 deletions drivers/net/hns3/hns3_ethdev_vf.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -822,6 +822,7 @@ hns3vf_dev_configure(struct rte_eth_dev *dev)
goto cfg_err;

hns->rx_simple_allowed = true;
hns->rx_vec_allowed = true;
hns->tx_simple_allowed = true;
hns->tx_vec_allowed = true;

Expand Down
94 changes: 88 additions & 6 deletions drivers/net/hns3/hns3_rxtx.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -41,9 +41,19 @@ hns3_rx_queue_release_mbufs(struct hns3_rx_queue *rxq)
if (rxq->sw_ring == NULL)
return;

for (i = 0; i < rxq->nb_rx_desc; i++)
if (rxq->sw_ring[i].mbuf)
rte_pktmbuf_free_seg(rxq->sw_ring[i].mbuf);
if (rxq->rx_rearm_nb == 0) {
for (i = 0; i < rxq->nb_rx_desc; i++) {
if (rxq->sw_ring[i].mbuf != NULL)
rte_pktmbuf_free_seg(rxq->sw_ring[i].mbuf);
}
} else {
for (i = rxq->next_to_use;
i != rxq->rx_rearm_start;
i = (i + 1) % rxq->nb_rx_desc) {
if (rxq->sw_ring[i].mbuf != NULL)
rte_pktmbuf_free_seg(rxq->sw_ring[i].mbuf);
}
}

for (i = 0; i < rxq->bulk_mbuf_num; i++)
rte_pktmbuf_free_seg(rxq->bulk_mbuf[i]);
Expand Down Expand Up @@ -661,10 +671,13 @@ hns3_dev_rx_queue_start(struct hns3_adapter *hns, uint16_t idx)
}

rxq->next_to_use = 0;
rxq->rx_rearm_start = 0;
rxq->rx_free_hold = 0;
rxq->rx_rearm_nb = 0;
rxq->pkt_first_seg = NULL;
rxq->pkt_last_seg = NULL;
hns3_init_rx_queue_hw(rxq);
hns3_rxq_vec_setup(rxq);

return 0;
}
Expand All @@ -678,6 +691,8 @@ hns3_fake_rx_queue_start(struct hns3_adapter *hns, uint16_t idx)
rxq = (struct hns3_rx_queue *)hw->fkq_data.rx_queues[idx];
rxq->next_to_use = 0;
rxq->rx_free_hold = 0;
rxq->rx_rearm_start = 0;
rxq->rx_rearm_nb = 0;
hns3_init_rx_queue_hw(rxq);
}

Expand Down Expand Up @@ -860,6 +875,40 @@ hns3_stop_queues(struct hns3_adapter *hns, bool reset_queue)
return 0;
}

/*
* Iterate over all Rx Queue, and call the callback() function for each Rx
* queue.
*
* @param[in] dev
* The target eth dev.
* @param[in] callback
* The function to call for each queue.
* if callback function return nonzero will stop iterate and return it's value
* @param[in] arg
* The arguments to provide the callback function with.
*
* @return
* 0 on success, otherwise with errno set.
*/
int
hns3_rxq_iterate(struct rte_eth_dev *dev,
int (*callback)(struct hns3_rx_queue *, void *), void *arg)
{
uint32_t i;
int ret;

if (dev->data->rx_queues == NULL)
return -EINVAL;

for (i = 0; i < dev->data->nb_rx_queues; i++) {
ret = callback(dev->data->rx_queues[i], arg);
if (ret != 0)
return ret;
}

return 0;
}

static void*
hns3_alloc_rxq_and_dma_zone(struct rte_eth_dev *dev,
struct hns3_queue_info *q_info)
Expand All @@ -880,7 +929,13 @@ hns3_alloc_rxq_and_dma_zone(struct rte_eth_dev *dev,
/* Allocate rx ring hardware descriptors. */
rxq->queue_id = q_info->idx;
rxq->nb_rx_desc = q_info->nb_desc;
rx_desc = rxq->nb_rx_desc * sizeof(struct hns3_desc);

/*
* Allocate a litter more memory because rx vector functions
* don't check boundaries each time.
*/
rx_desc = (rxq->nb_rx_desc + HNS3_DEFAULT_RX_BURST) *
sizeof(struct hns3_desc);
rx_mz = rte_eth_dma_zone_reserve(dev, q_info->ring_name, q_info->idx,
rx_desc, HNS3_RING_BASE_ALIGN,
q_info->socket_id);
Expand Down Expand Up @@ -1329,7 +1384,8 @@ hns3_rx_queue_setup(struct rte_eth_dev *dev, uint16_t idx, uint16_t nb_desc,
conf->rx_free_thresh : HNS3_DEFAULT_RX_FREE_THRESH;
rxq->rx_deferred_start = conf->rx_deferred_start;

rx_entry_len = sizeof(struct hns3_entry) * rxq->nb_rx_desc;
rx_entry_len = (rxq->nb_rx_desc + HNS3_DEFAULT_RX_BURST) *
sizeof(struct hns3_entry);
rxq->sw_ring = rte_zmalloc_socket("hns3 RX sw ring", rx_entry_len,
RTE_CACHE_LINE_SIZE, socket_id);
if (rxq->sw_ring == NULL) {
Expand All @@ -1340,6 +1396,8 @@ hns3_rx_queue_setup(struct rte_eth_dev *dev, uint16_t idx, uint16_t nb_desc,

rxq->next_to_use = 0;
rxq->rx_free_hold = 0;
rxq->rx_rearm_start = 0;
rxq->rx_rearm_nb = 0;
rxq->pkt_first_seg = NULL;
rxq->pkt_last_seg = NULL;
rxq->port_id = dev->data->port_id;
Expand Down Expand Up @@ -1431,7 +1489,8 @@ hns3_dev_supported_ptypes_get(struct rte_eth_dev *dev)
};

if (dev->rx_pkt_burst == hns3_recv_pkts ||
dev->rx_pkt_burst == hns3_recv_scattered_pkts)
dev->rx_pkt_burst == hns3_recv_scattered_pkts ||
dev->rx_pkt_burst == hns3_recv_pkts_vec)
return ptypes;

return NULL;
Expand Down Expand Up @@ -1915,6 +1974,25 @@ hns3_recv_scattered_pkts(void *rx_queue,
return nb_rx;
}

void __rte_weak
hns3_rxq_vec_setup(__rte_unused struct hns3_rx_queue *rxq)
{
}

int __rte_weak
hns3_rx_check_vec_support(__rte_unused struct rte_eth_dev *dev)
{
return -ENOTSUP;
}

uint16_t __rte_weak
hns3_recv_pkts_vec(__rte_unused void *tx_queue,
__rte_unused struct rte_mbuf **tx_pkts,
__rte_unused uint16_t nb_pkts)
{
return 0;
}

int
hns3_rx_burst_mode_get(struct rte_eth_dev *dev, __rte_unused uint16_t queue_id,
struct rte_eth_burst_mode *mode)
Expand All @@ -1925,6 +2003,7 @@ hns3_rx_burst_mode_get(struct rte_eth_dev *dev, __rte_unused uint16_t queue_id,
} burst_infos[] = {
{ hns3_recv_pkts, "Scalar" },
{ hns3_recv_scattered_pkts, "Scalar Scattered" },
{ hns3_recv_pkts_vec, "Vector Neon" },
};

eth_rx_burst_t pkt_burst = dev->rx_pkt_burst;
Expand All @@ -1949,6 +2028,9 @@ hns3_get_rx_function(struct rte_eth_dev *dev)
struct hns3_adapter *hns = dev->data->dev_private;
uint64_t offloads = dev->data->dev_conf.rxmode.offloads;

if (hns->rx_vec_allowed && hns3_rx_check_vec_support(dev) == 0)
return hns3_recv_pkts_vec;

if (hns->rx_simple_allowed && !dev->data->scattered_rx &&
(offloads & DEV_RX_OFFLOAD_TCP_LRO) == 0)
return hns3_recv_pkts;
Expand Down
35 changes: 33 additions & 2 deletions drivers/net/hns3/hns3_rxtx.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,18 @@
#define HNS3_DEFAULT_TX_RS_THRESH 32
#define HNS3_TX_FAST_FREE_AHEAD 64

#define HNS3_DEFAULT_RX_BURST 32
#if (HNS3_DEFAULT_RX_BURST > 64)
#error "PMD HNS3: HNS3_DEFAULT_RX_BURST must <= 64\n"
#endif
#define HNS3_DEFAULT_DESCS_PER_LOOP 4
#define HNS3_SVE_DEFAULT_DESCS_PER_LOOP 8
#if (HNS3_DEFAULT_DESCS_PER_LOOP > HNS3_SVE_DEFAULT_DESCS_PER_LOOP)
#define HNS3_VECTOR_RX_OFFSET_TABLE_LEN HNS3_DEFAULT_DESCS_PER_LOOP
#else
#define HNS3_VECTOR_RX_OFFSET_TABLE_LEN HNS3_SVE_DEFAULT_DESCS_PER_LOOP
#endif
#define HNS3_DEFAULT_RXQ_REARM_THRESH 64
#define HNS3_UINT8_BIT 8
#define HNS3_UINT16_BIT 16
#define HNS3_UINT32_BIT 32
Expand Down Expand Up @@ -236,7 +248,13 @@ struct hns3_desc {
uint16_t ot_vlan_tag;
};
};
uint32_t bd_base_info;
union {
uint32_t bd_base_info;
struct {
uint16_t bdtype_vld_udp0;
uint16_t fe_lum_crcp_l3l4p;
};
};
} rx;
};
} __rte_packed;
Expand Down Expand Up @@ -270,7 +288,8 @@ struct hns3_rx_queue {
uint16_t rx_free_thresh;
uint16_t next_to_use; /* index of next BD to be polled */
uint16_t rx_free_hold; /* num of BDs waited to passed to hardware */

uint16_t rx_rearm_start; /* index of BD that driver re-arming from */
uint16_t rx_rearm_nb; /* number of remaining BDs to be re-armed */
/*
* port based vlan configuration state.
* value range: HNS3_PORT_BASE_VLAN_DISABLE / HNS3_PORT_BASE_VLAN_ENABLE
Expand All @@ -292,6 +311,11 @@ struct hns3_rx_queue {

struct rte_mbuf *bulk_mbuf[HNS3_BULK_ALLOC_MBUF_NUM];
uint16_t bulk_mbuf_num;

/* offset_table: used for vector, to solve execute re-order problem */
uint8_t offset_table[HNS3_VECTOR_RX_OFFSET_TABLE_LEN + 1];
uint64_t mbuf_initializer; /* value to init mbufs used with vector rx */
struct rte_mbuf fake_mbuf; /* fake mbuf used with vector rx */
};

struct hns3_tx_queue {
Expand Down Expand Up @@ -554,6 +578,8 @@ int hns3_dev_rx_queue_intr_disable(struct rte_eth_dev *dev, uint16_t queue_id);
void hns3_enable_all_queues(struct hns3_hw *hw, bool en);
int hns3_start_queues(struct hns3_adapter *hns, bool reset_queue);
int hns3_stop_queues(struct hns3_adapter *hns, bool reset_queue);
int hns3_rxq_iterate(struct rte_eth_dev *dev,
int (*callback)(struct hns3_rx_queue *, void *), void *arg);
void hns3_dev_release_mbufs(struct hns3_adapter *hns);
int hns3_rx_queue_setup(struct rte_eth_dev *dev, uint16_t idx, uint16_t nb_desc,
unsigned int socket, const struct rte_eth_rxconf *conf,
Expand All @@ -564,9 +590,12 @@ uint16_t hns3_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts,
uint16_t nb_pkts);
uint16_t hns3_recv_scattered_pkts(void *rx_queue, struct rte_mbuf **rx_pkts,
uint16_t nb_pkts);
uint16_t hns3_recv_pkts_vec(void *rx_queue, struct rte_mbuf **rx_pkts,
uint16_t nb_pkts);
int hns3_rx_burst_mode_get(struct rte_eth_dev *dev,
__rte_unused uint16_t queue_id,
struct rte_eth_burst_mode *mode);
int hns3_rx_check_vec_support(struct rte_eth_dev *dev);
uint16_t hns3_prep_pkts(__rte_unused void *tx_queue, struct rte_mbuf **tx_pkts,
uint16_t nb_pkts);
uint16_t hns3_xmit_pkts_simple(void *tx_queue, struct rte_mbuf **tx_pkts,
Expand Down Expand Up @@ -594,7 +623,9 @@ int hns3_restore_gro_conf(struct hns3_hw *hw);
void hns3_update_all_queues_pvid_state(struct hns3_hw *hw);
void hns3_rx_scattered_reset(struct rte_eth_dev *dev);
void hns3_rx_scattered_calc(struct rte_eth_dev *dev);
int hns3_rx_check_vec_support(struct rte_eth_dev *dev);
int hns3_tx_check_vec_support(struct rte_eth_dev *dev);
void hns3_rxq_vec_setup(struct hns3_rx_queue *rxq);
void hns3_rxq_info_get(struct rte_eth_dev *dev, uint16_t queue_id,
struct rte_eth_rxq_info *qinfo);
void hns3_txq_info_get(struct rte_eth_dev *dev, uint16_t queue_id,
Expand Down
Loading

0 comments on commit a3d4f4d

Please sign in to comment.