Skip to content

Commit

Permalink
net/hns3: support different numbers of Rx and Tx queues
Browse files Browse the repository at this point in the history 
Hardware does not support individually enable/disable/reset the Tx or Rx
queue in hns3 network engine, driver must enable/disable/reset Tx and Rx
queues at the same time.

Currently, hns3 PMD driver does not support the scenarios as below:
1) When calling the following function, the input parameter nb_rx_q and
   nb_tx_q are not equal.
     rte_eth_dev_configure(uint16_t port_id, uint16_t nb_rx_q,
                      uint16_t nb_tx_q,
		      const struct rte_eth_conf *dev_conf);
2) When calling the following functions to setup queues, the
   cumulatively setup Rx queues are not the same as the setup Tx queues.
     rte_eth_rx_queue_setup(uint16_t port_id, uint16_t rx_queue_id,,,);
     rte_eth_tx_queue_setup(uint16_t port_id, uint16_t tx_queue_id,,,);
However, these are common usage scenarios in some applications, such as,
l3fwd, ip_ressmbly and OVS-DPDK, etc.

This patch adds support for this usage of these functions by setup fake
Tx or Rx queues to adjust numbers of Tx/Rx queues. But these fake queues
are imperceptible, and can not be used by upper applications.

Signed-off-by: Huisong Li <lihuisong@huawei.com>
Signed-off-by: Wei Hu (Xavier) <xavier.huwei@huawei.com>
  • Loading branch information
@xavierhw
xavierhw authored and Ferruh Yigit committed Jan 17, 2020
1 parent c5f8365 commit a951c1e
Show file tree
Hide file tree
Showing 8 changed files with 734 additions and 193 deletions.
88 changes: 59 additions & 29 deletions drivers/net/hns3/hns3_dcb.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -578,17 +578,33 @@ hns3_dcb_pri_shaper_cfg(struct hns3_hw *hw)
}

void
hns3_tc_queue_mapping_cfg(struct hns3_hw *hw)
hns3_set_rss_size(struct hns3_hw *hw, uint16_t nb_rx_q)
{
uint16_t rx_qnum_per_tc;

rx_qnum_per_tc = nb_rx_q / hw->num_tc;
rx_qnum_per_tc = RTE_MIN(hw->rss_size_max, rx_qnum_per_tc);
if (hw->alloc_rss_size != rx_qnum_per_tc) {
hns3_info(hw, "rss size changes from %u to %u",
hw->alloc_rss_size, rx_qnum_per_tc);
hw->alloc_rss_size = rx_qnum_per_tc;
}
hw->used_rx_queues = hw->num_tc * hw->alloc_rss_size;
}

void
hns3_tc_queue_mapping_cfg(struct hns3_hw *hw, uint16_t nb_queue)
{
struct hns3_tc_queue_info *tc_queue;
uint8_t i;

hw->tx_qnum_per_tc = nb_queue / hw->num_tc;
for (i = 0; i < HNS3_MAX_TC_NUM; i++) {
tc_queue = &hw->tc_queue[i];
if (hw->hw_tc_map & BIT(i) && i < hw->num_tc) {
tc_queue->enable = true;
tc_queue->tqp_offset = i * hw->alloc_rss_size;
tc_queue->tqp_count = hw->alloc_rss_size;
tc_queue->tqp_offset = i * hw->tx_qnum_per_tc;
tc_queue->tqp_count = hw->tx_qnum_per_tc;
tc_queue->tc = i;
} else {
/* Set to default queue if TC is disable */
Expand All @@ -598,30 +614,22 @@ hns3_tc_queue_mapping_cfg(struct hns3_hw *hw)
tc_queue->tc = 0;
}
}
hw->used_tx_queues = hw->num_tc * hw->tx_qnum_per_tc;
}

static void
hns3_dcb_update_tc_queue_mapping(struct hns3_hw *hw, uint16_t queue_num)
hns3_dcb_update_tc_queue_mapping(struct hns3_hw *hw, uint16_t nb_rx_q,
uint16_t nb_tx_q)
{
struct hns3_adapter *hns = HNS3_DEV_HW_TO_ADAPTER(hw);
struct hns3_pf *pf = &hns->pf;
uint16_t tqpnum_per_tc;
uint16_t alloc_tqps;

alloc_tqps = RTE_MIN(hw->tqps_num, queue_num);
hw->num_tc = RTE_MIN(alloc_tqps, hw->dcb_info.num_tc);
tqpnum_per_tc = RTE_MIN(hw->rss_size_max, alloc_tqps / hw->num_tc);

if (hw->alloc_rss_size != tqpnum_per_tc) {
PMD_INIT_LOG(INFO, "rss size changes from %d to %d",
hw->alloc_rss_size, tqpnum_per_tc);
hw->alloc_rss_size = tqpnum_per_tc;
}
hw->alloc_tqps = hw->num_tc * hw->alloc_rss_size;
hw->num_tc = hw->dcb_info.num_tc;
hns3_set_rss_size(hw, nb_rx_q);
hns3_tc_queue_mapping_cfg(hw, nb_tx_q);

hns3_tc_queue_mapping_cfg(hw);

memcpy(pf->prio_tc, hw->dcb_info.prio_tc, HNS3_MAX_USER_PRIO);
if (!hns->is_vf)
memcpy(pf->prio_tc, hw->dcb_info.prio_tc, HNS3_MAX_USER_PRIO);
}

int
Expand Down Expand Up @@ -1309,20 +1317,35 @@ hns3_dcb_info_cfg(struct hns3_adapter *hns)
for (i = 0; i < HNS3_MAX_USER_PRIO; i++)
hw->dcb_info.prio_tc[i] = dcb_rx_conf->dcb_tc[i];

hns3_dcb_update_tc_queue_mapping(hw, hw->data->nb_rx_queues);
hns3_dcb_update_tc_queue_mapping(hw, hw->data->nb_rx_queues,
hw->data->nb_tx_queues);
}

static void
static int
hns3_dcb_info_update(struct hns3_adapter *hns, uint8_t num_tc)
{
struct hns3_pf *pf = &hns->pf;
struct hns3_hw *hw = &hns->hw;
uint16_t nb_rx_q = hw->data->nb_rx_queues;
uint16_t nb_tx_q = hw->data->nb_tx_queues;
uint8_t bit_map = 0;
uint8_t i;

if (pf->tx_sch_mode != HNS3_FLAG_TC_BASE_SCH_MODE &&
hw->dcb_info.num_pg != 1)
return;
return -EINVAL;

if (nb_rx_q < num_tc) {
hns3_err(hw, "number of Rx queues(%d) is less than tcs(%d).",
nb_rx_q, num_tc);
return -EINVAL;
}

if (nb_tx_q < num_tc) {
hns3_err(hw, "number of Tx queues(%d) is less than tcs(%d).",
nb_tx_q, num_tc);
return -EINVAL;
}

/* Currently not support uncontinuous tc */
hw->dcb_info.num_tc = num_tc;
Expand All @@ -1333,10 +1356,10 @@ hns3_dcb_info_update(struct hns3_adapter *hns, uint8_t num_tc)
bit_map = 1;
hw->dcb_info.num_tc = 1;
}

hw->hw_tc_map = bit_map;

hns3_dcb_info_cfg(hns);

return 0;
}

static int
Expand Down Expand Up @@ -1422,10 +1445,15 @@ hns3_dcb_configure(struct hns3_adapter *hns)

hns3_dcb_cfg_validate(hns, &num_tc, &map_changed);
if (map_changed || rte_atomic16_read(&hw->reset.resetting)) {
hns3_dcb_info_update(hns, num_tc);
ret = hns3_dcb_info_update(hns, num_tc);
if (ret) {
hns3_err(hw, "dcb info update failed: %d", ret);
return ret;
}

ret = hns3_dcb_hw_configure(hns);
if (ret) {
hns3_err(hw, "dcb sw configure fails: %d", ret);
hns3_err(hw, "dcb sw configure failed: %d", ret);
return ret;
}
}
Expand Down Expand Up @@ -1479,7 +1507,8 @@ hns3_dcb_init(struct hns3_hw *hw)
hns3_err(hw, "dcb info init failed: %d", ret);
return ret;
}
hns3_dcb_update_tc_queue_mapping(hw, hw->tqps_num);
hns3_dcb_update_tc_queue_mapping(hw, hw->tqps_num,
hw->tqps_num);
}

/*
Expand All @@ -1502,10 +1531,11 @@ static int
hns3_update_queue_map_configure(struct hns3_adapter *hns)
{
struct hns3_hw *hw = &hns->hw;
uint16_t queue_num = hw->data->nb_rx_queues;
uint16_t nb_rx_q = hw->data->nb_rx_queues;
uint16_t nb_tx_q = hw->data->nb_tx_queues;
int ret;

hns3_dcb_update_tc_queue_mapping(hw, queue_num);
hns3_dcb_update_tc_queue_mapping(hw, nb_rx_q, nb_tx_q);
ret = hns3_q_to_qs_map(hw);
if (ret) {
hns3_err(hw, "failed to map nq to qs! ret = %d", ret);
Expand Down
4 changes: 3 additions & 1 deletion drivers/net/hns3/hns3_dcb.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -159,7 +159,9 @@ hns3_fc_enable(struct rte_eth_dev *dev, struct rte_eth_fc_conf *fc_conf);
int
hns3_dcb_pfc_enable(struct rte_eth_dev *dev, struct rte_eth_pfc_conf *pfc_conf);

void hns3_tc_queue_mapping_cfg(struct hns3_hw *hw);
void hns3_set_rss_size(struct hns3_hw *hw, uint16_t nb_rx_q);

void hns3_tc_queue_mapping_cfg(struct hns3_hw *hw, uint16_t nb_queue);

int hns3_dcb_cfg_update(struct hns3_adapter *hns);

Expand Down
56 changes: 32 additions & 24 deletions drivers/net/hns3/hns3_ethdev.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2061,34 +2061,40 @@ hns3_bind_ring_with_vector(struct rte_eth_dev *dev, uint8_t vector_id,
static int
hns3_dev_configure(struct rte_eth_dev *dev)
{
struct hns3_hw *hw = HNS3_DEV_PRIVATE_TO_HW(dev->data->dev_private);
struct hns3_rss_conf *rss_cfg = &hw->rss_info;
struct hns3_adapter *hns = dev->data->dev_private;
struct rte_eth_conf *conf = &dev->data->dev_conf;
enum rte_eth_rx_mq_mode mq_mode = conf->rxmode.mq_mode;
struct hns3_hw *hw = &hns->hw;
struct hns3_rss_conf *rss_cfg = &hw->rss_info;
uint16_t nb_rx_q = dev->data->nb_rx_queues;
uint16_t nb_tx_q = dev->data->nb_tx_queues;
struct rte_eth_rss_conf rss_conf;
uint16_t mtu;
int ret;

/*
* Hardware does not support where the number of rx and tx queues is
* not equal in hip08.
* Hardware does not support individually enable/disable/reset the Tx or
* Rx queue in hns3 network engine. Driver must enable/disable/reset Tx
* and Rx queues at the same time. When the numbers of Tx queues
* allocated by upper applications are not equal to the numbers of Rx
* queues, driver needs to setup fake Tx or Rx queues to adjust numbers
* of Tx/Rx queues. otherwise, network engine can not work as usual. But
* these fake queues are imperceptible, and can not be used by upper
* applications.
*/
if (nb_rx_q != nb_tx_q) {
hns3_err(hw,
"nb_rx_queues(%u) not equal with nb_tx_queues(%u)! "
"Hardware does not support this configuration!",
nb_rx_q, nb_tx_q);
return -EINVAL;
ret = hns3_set_fake_rx_or_tx_queues(dev, nb_rx_q, nb_tx_q);
if (ret) {
hns3_err(hw, "Failed to set rx/tx fake queues: %d", ret);
return ret;
}

hw->adapter_state = HNS3_NIC_CONFIGURING;
if (conf->link_speeds & ETH_LINK_SPEED_FIXED) {
hns3_err(hw, "setting link speed/duplex not supported");
return -EINVAL;
ret = -EINVAL;
goto cfg_err;
}

hw->adapter_state = HNS3_NIC_CONFIGURING;
if ((uint32_t)mq_mode & ETH_MQ_RX_DCB_FLAG) {
ret = hns3_check_dcb_cfg(dev);
if (ret)
Expand Down Expand Up @@ -2134,7 +2140,9 @@ hns3_dev_configure(struct rte_eth_dev *dev)
return 0;

cfg_err:
(void)hns3_set_fake_rx_or_tx_queues(dev, 0, 0);
hw->adapter_state = HNS3_NIC_INITIALIZED;

return ret;
}

Expand Down Expand Up @@ -4084,18 +4092,18 @@ hns3_map_rx_interrupt(struct rte_eth_dev *dev)
/* check and configure queue intr-vector mapping */
if (rte_intr_cap_multiple(intr_handle) ||
!RTE_ETH_DEV_SRIOV(dev).active) {
intr_vector = dev->data->nb_rx_queues;
intr_vector = hw->used_rx_queues;
/* creates event fd for each intr vector when MSIX is used */
if (rte_intr_efd_enable(intr_handle, intr_vector))
return -EINVAL;
}
if (rte_intr_dp_is_en(intr_handle) && !intr_handle->intr_vec) {
intr_handle->intr_vec =
rte_zmalloc("intr_vec",
dev->data->nb_rx_queues * sizeof(int), 0);
hw->used_rx_queues * sizeof(int), 0);
if (intr_handle->intr_vec == NULL) {
hns3_err(hw, "Failed to allocate %d rx_queues"
" intr_vec", dev->data->nb_rx_queues);
" intr_vec", hw->used_rx_queues);
ret = -ENOMEM;
goto alloc_intr_vec_error;
}
Expand All @@ -4106,7 +4114,7 @@ hns3_map_rx_interrupt(struct rte_eth_dev *dev)
base = RTE_INTR_VEC_RXTX_OFFSET;
}
if (rte_intr_dp_is_en(intr_handle)) {
for (q_id = 0; q_id < dev->data->nb_rx_queues; q_id++) {
for (q_id = 0; q_id < hw->used_rx_queues; q_id++) {
ret = hns3_bind_ring_with_vector(dev, vec, true, q_id);
if (ret)
goto bind_vector_error;
Expand Down Expand Up @@ -4190,6 +4198,8 @@ hns3_unmap_rx_interrupt(struct rte_eth_dev *dev)
{
struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
struct rte_intr_handle *intr_handle = &pci_dev->intr_handle;
struct hns3_adapter *hns = dev->data->dev_private;
struct hns3_hw *hw = &hns->hw;
uint8_t base = 0;
uint8_t vec = 0;
uint16_t q_id;
Expand All @@ -4203,7 +4213,7 @@ hns3_unmap_rx_interrupt(struct rte_eth_dev *dev)
base = RTE_INTR_VEC_RXTX_OFFSET;
}
if (rte_intr_dp_is_en(intr_handle)) {
for (q_id = 0; q_id < dev->data->nb_rx_queues; q_id++) {
for (q_id = 0; q_id < hw->used_rx_queues; q_id++) {
(void)hns3_bind_ring_with_vector(dev, vec, false, q_id);
if (vec < base + intr_handle->nb_efd - 1)
vec++;
Expand Down Expand Up @@ -4446,15 +4456,13 @@ hns3_get_dcb_info(struct rte_eth_dev *dev, struct rte_eth_dcb_info *dcb_info)
for (i = 0; i < dcb_info->nb_tcs; i++)
dcb_info->tc_bws[i] = hw->dcb_info.pg_info[0].tc_dwrr[i];

for (i = 0; i < HNS3_MAX_TC_NUM; i++) {
dcb_info->tc_queue.tc_rxq[0][i].base =
hw->tc_queue[i].tqp_offset;
for (i = 0; i < hw->num_tc; i++) {
dcb_info->tc_queue.tc_rxq[0][i].base = hw->alloc_rss_size * i;
dcb_info->tc_queue.tc_txq[0][i].base =
hw->tc_queue[i].tqp_offset;
dcb_info->tc_queue.tc_rxq[0][i].nb_queue =
hw->tc_queue[i].tqp_count;
hw->tc_queue[i].tqp_offset;
dcb_info->tc_queue.tc_rxq[0][i].nb_queue = hw->alloc_rss_size;
dcb_info->tc_queue.tc_txq[0][i].nb_queue =
hw->tc_queue[i].tqp_count;
hw->tc_queue[i].tqp_count;
}
rte_spinlock_unlock(&hw->lock);

Expand Down
16 changes: 14 additions & 2 deletions drivers/net/hns3/hns3_ethdev.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -153,6 +153,12 @@ struct hns3_mac {
uint32_t link_speed; /* ETH_SPEED_NUM_ */
};

struct hns3_fake_queue_data {
void **rx_queues; /* Array of pointers to fake RX queues. */
void **tx_queues; /* Array of pointers to fake TX queues. */
uint16_t nb_fake_rx_queues; /* Number of fake RX queues. */
uint16_t nb_fake_tx_queues; /* Number of fake TX queues. */
};

/* Primary process maintains driver state in main thread.
*
Expand Down Expand Up @@ -365,8 +371,14 @@ struct hns3_hw {
struct hns3_dcb_info dcb_info;
enum hns3_fc_status current_fc_status; /* current flow control status */
struct hns3_tc_queue_info tc_queue[HNS3_MAX_TC_NUM];
uint16_t alloc_tqps;
uint16_t alloc_rss_size; /* Queue number per TC */
uint16_t used_rx_queues;
uint16_t used_tx_queues;

/* Config max queue numbers between rx and tx queues from user */
uint16_t cfg_max_queues;
struct hns3_fake_queue_data fkq_data; /* fake queue data */
uint16_t alloc_rss_size; /* RX queue number per TC */
uint16_t tx_qnum_per_tc; /* TX queue number per TC */

uint32_t flag;
/*
Expand Down

0 comments on commit a951c1e

Please sign in to comment.