Skip to content

Commit

Permalink
net/qede: fix odd number of queues usage in 100G mode
Browse files Browse the repository at this point in the history 
[ upstream commit 8de0c42 ]

As per HW design of 100Gb mode, device internally uses 2 engines
(eng0 and eng1), and both engines need to be configured symmetrically.
Based on this requirement, driver design chose an approach
to allow user to allocate only even number of queues and split
those queues on both engines equally.

This approach puts a limitation on number of queues to be allocated -
i.e. user can't configure odd number of queues on 100Gb mode.
OVS configures DPDK port with 1 rxq and 1 txq, which causes initialization
of qede port to fail.

Issue is fixed by changing the implementation of queue allocation and
assignment to hw engines only for 100Gb devices and allowing user to
configure odd number queues.

New approach works as below -
- Create 'struct qede_fastpath_cmt' to hold hw queue pair of both engines
  and provide it to rte_ethdev's Rx/Tx queues structure.
- So ethdev will see only one queue for underlying queue pair created for
  hw engine pair.
- Install separate Rx/Tx data path handlers for 100Gb mode and regular mode
- Rx/Tx handlers for 100Gb mode will split packet processing across both
  engines by providing hw queue structures from 'struct qede_fastpath_cmt'
  passed by Rx/Tx callbacks to respective engines.

Fixes: 2af14ca ("net/qede: support 100G")

Signed-off-by: Shahed Shaikh <shshaikh@marvell.com>
  • Loading branch information
@kevintraynor
Shahed Shaikh authored and kevintraynor committed Nov 22, 2019
1 parent 13b44f3 commit eabce98
Show file tree
Hide file tree
Showing 5 changed files with 219 additions and 90 deletions.
112 changes: 58 additions & 54 deletions drivers/net/qede/qede_ethdev.c
View file
Expand Up @@ -303,19 +303,20 @@ static void qede_print_adapter_info(struct qede_dev *qdev)

static void qede_reset_queue_stats(struct qede_dev *qdev, bool xstats)
{
struct rte_eth_dev *dev = (struct rte_eth_dev *)qdev->ethdev;
struct ecore_dev *edev = QEDE_INIT_EDEV(qdev);
unsigned int i = 0, j = 0, qid;
unsigned int rxq_stat_cntrs, txq_stat_cntrs;
struct qede_tx_queue *txq;

DP_VERBOSE(edev, ECORE_MSG_DEBUG, "Clearing queue stats\n");

rxq_stat_cntrs = RTE_MIN(QEDE_RSS_COUNT(qdev),
rxq_stat_cntrs = RTE_MIN(QEDE_RSS_COUNT(dev),
RTE_ETHDEV_QUEUE_STAT_CNTRS);
txq_stat_cntrs = RTE_MIN(QEDE_TSS_COUNT(qdev),
txq_stat_cntrs = RTE_MIN(QEDE_TSS_COUNT(dev),
RTE_ETHDEV_QUEUE_STAT_CNTRS);

for_each_rss(qid) {
for (qid = 0; qid < qdev->num_rx_queues; qid++) {
OSAL_MEMSET(((char *)(qdev->fp_array[qid].rxq)) +
offsetof(struct qede_rx_queue, rcv_pkts), 0,
sizeof(uint64_t));
Expand All @@ -341,7 +342,7 @@ static void qede_reset_queue_stats(struct qede_dev *qdev, bool xstats)

i = 0;

for_each_tss(qid) {
for (qid = 0; qid < qdev->num_tx_queues; qid++) {
txq = qdev->fp_array[qid].txq;

OSAL_MEMSET((uint64_t *)(uintptr_t)
Expand Down Expand Up @@ -988,7 +989,7 @@ int qede_config_rss(struct rte_eth_dev *eth_dev)
for (i = 0; i < ECORE_RSS_IND_TABLE_SIZE; i++) {
id = i / RTE_RETA_GROUP_SIZE;
pos = i % RTE_RETA_GROUP_SIZE;
q = i % QEDE_RSS_COUNT(qdev);
q = i % QEDE_RSS_COUNT(eth_dev);
reta_conf[id].reta[pos] = q;
}
if (qede_rss_reta_update(eth_dev, &reta_conf[0],
Expand Down Expand Up @@ -1162,22 +1163,6 @@ static int qede_dev_configure(struct rte_eth_dev *eth_dev)

PMD_INIT_FUNC_TRACE(edev);

/* Check requirements for 100G mode */
if (ECORE_IS_CMT(edev)) {
if (eth_dev->data->nb_rx_queues < 2 ||
eth_dev->data->nb_tx_queues < 2) {
DP_ERR(edev, "100G mode needs min. 2 RX/TX queues\n");
return -EINVAL;
}

if ((eth_dev->data->nb_rx_queues % 2 != 0) ||
(eth_dev->data->nb_tx_queues % 2 != 0)) {
DP_ERR(edev,
"100G mode needs even no. of RX/TX queues\n");
return -EINVAL;
}
}

/* We need to have min 1 RX queue.There is no min check in
* rte_eth_dev_configure(), so we are checking it here.
*/
Expand All @@ -1204,8 +1189,9 @@ static int qede_dev_configure(struct rte_eth_dev *eth_dev)
return -ENOTSUP;

qede_dealloc_fp_resc(eth_dev);
qdev->num_tx_queues = eth_dev->data->nb_tx_queues;
qdev->num_rx_queues = eth_dev->data->nb_rx_queues;
qdev->num_tx_queues = eth_dev->data->nb_tx_queues * edev->num_hwfns;
qdev->num_rx_queues = eth_dev->data->nb_rx_queues * edev->num_hwfns;

if (qede_alloc_fp_resc(qdev))
return -ENOMEM;

Expand All @@ -1230,7 +1216,12 @@ static int qede_dev_configure(struct rte_eth_dev *eth_dev)
return ret;

DP_INFO(edev, "Device configured with RSS=%d TSS=%d\n",
QEDE_RSS_COUNT(qdev), QEDE_TSS_COUNT(qdev));
QEDE_RSS_COUNT(eth_dev), QEDE_TSS_COUNT(eth_dev));

if (ECORE_IS_CMT(edev))
DP_INFO(edev, "Actual HW queues for CMT mode - RX = %d TX = %d\n",
qdev->num_rx_queues, qdev->num_tx_queues);


return 0;
}
Expand Down Expand Up @@ -1272,6 +1263,10 @@ qede_dev_info_get(struct rte_eth_dev *eth_dev,
else
dev_info->max_rx_queues = (uint16_t)RTE_MIN(
QEDE_MAX_RSS_CNT(qdev), ECORE_MAX_VF_CHAINS_PER_PF);
/* Since CMT mode internally doubles the number of queues */
if (ECORE_IS_CMT(edev))
dev_info->max_rx_queues = dev_info->max_rx_queues / 2;

dev_info->max_tx_queues = dev_info->max_rx_queues;

dev_info->max_mac_addrs = qdev->dev_info.num_mac_filters;
Expand Down Expand Up @@ -1515,18 +1510,18 @@ qede_get_stats(struct rte_eth_dev *eth_dev, struct rte_eth_stats *eth_stats)
eth_stats->oerrors = stats.common.tx_err_drop_pkts;

/* Queue stats */
rxq_stat_cntrs = RTE_MIN(QEDE_RSS_COUNT(qdev),
rxq_stat_cntrs = RTE_MIN(QEDE_RSS_COUNT(eth_dev),
RTE_ETHDEV_QUEUE_STAT_CNTRS);
txq_stat_cntrs = RTE_MIN(QEDE_TSS_COUNT(qdev),
txq_stat_cntrs = RTE_MIN(QEDE_TSS_COUNT(eth_dev),
RTE_ETHDEV_QUEUE_STAT_CNTRS);
if ((rxq_stat_cntrs != (unsigned int)QEDE_RSS_COUNT(qdev)) ||
(txq_stat_cntrs != (unsigned int)QEDE_TSS_COUNT(qdev)))
if (rxq_stat_cntrs != (unsigned int)QEDE_RSS_COUNT(eth_dev) ||
txq_stat_cntrs != (unsigned int)QEDE_TSS_COUNT(eth_dev))
DP_VERBOSE(edev, ECORE_MSG_DEBUG,
"Not all the queue stats will be displayed. Set"
" RTE_ETHDEV_QUEUE_STAT_CNTRS config param"
" appropriately and retry.\n");

for_each_rss(qid) {
for (qid = 0; qid < eth_dev->data->nb_rx_queues; qid++) {
eth_stats->q_ipackets[i] =
*(uint64_t *)(
((char *)(qdev->fp_array[qid].rxq)) +
Expand All @@ -1546,7 +1541,7 @@ qede_get_stats(struct rte_eth_dev *eth_dev, struct rte_eth_stats *eth_stats)
break;
}

for_each_tss(qid) {
for (qid = 0; qid < eth_dev->data->nb_tx_queues; qid++) {
txq = qdev->fp_array[qid].txq;
eth_stats->q_opackets[j] =
*((uint64_t *)(uintptr_t)
Expand All @@ -1563,18 +1558,18 @@ qede_get_stats(struct rte_eth_dev *eth_dev, struct rte_eth_stats *eth_stats)

static unsigned
qede_get_xstats_count(struct qede_dev *qdev) {
struct rte_eth_dev *dev = (struct rte_eth_dev *)qdev->ethdev;

if (ECORE_IS_BB(&qdev->edev))
return RTE_DIM(qede_xstats_strings) +
RTE_DIM(qede_bb_xstats_strings) +
(RTE_DIM(qede_rxq_xstats_strings) *
RTE_MIN(QEDE_RSS_COUNT(qdev),
RTE_ETHDEV_QUEUE_STAT_CNTRS));
QEDE_RSS_COUNT(dev) * qdev->edev.num_hwfns);
else
return RTE_DIM(qede_xstats_strings) +
RTE_DIM(qede_ah_xstats_strings) +
(RTE_DIM(qede_rxq_xstats_strings) *
RTE_MIN(QEDE_RSS_COUNT(qdev),
RTE_ETHDEV_QUEUE_STAT_CNTRS));
QEDE_RSS_COUNT(dev));
}

static int
Expand Down Expand Up @@ -1615,7 +1610,7 @@ qede_get_xstats_names(struct rte_eth_dev *dev,
}
}

rxq_stat_cntrs = RTE_MIN(QEDE_RSS_COUNT(qdev),
rxq_stat_cntrs = RTE_MIN(QEDE_RSS_COUNT(dev),
RTE_ETHDEV_QUEUE_STAT_CNTRS);
for (qid = 0; qid < rxq_stat_cntrs; qid++) {
for (i = 0; i < RTE_DIM(qede_rxq_xstats_strings); i++) {
Expand Down Expand Up @@ -1673,17 +1668,15 @@ qede_get_xstats(struct rte_eth_dev *dev, struct rte_eth_xstat *xstats,
}
}

rxq_stat_cntrs = RTE_MIN(QEDE_RSS_COUNT(qdev),
rxq_stat_cntrs = RTE_MIN(QEDE_RSS_COUNT(dev),
RTE_ETHDEV_QUEUE_STAT_CNTRS);
for (qid = 0; qid < rxq_stat_cntrs; qid++) {
for_each_rss(qid) {
for (i = 0; i < RTE_DIM(qede_rxq_xstats_strings); i++) {
xstats[stat_idx].value = *(uint64_t *)(
((char *)(qdev->fp_array[qid].rxq)) +
qede_rxq_xstats_strings[i].offset);
xstats[stat_idx].id = stat_idx;
stat_idx++;
}
for (i = 0; i < RTE_DIM(qede_rxq_xstats_strings); i++) {
xstats[stat_idx].value = *(uint64_t *)
(((char *)(qdev->fp_array[qid].rxq)) +
qede_rxq_xstats_strings[i].offset);
xstats[stat_idx].id = stat_idx;
stat_idx++;
}
}

Expand Down Expand Up @@ -1937,7 +1930,8 @@ qede_dev_supported_ptypes_get(struct rte_eth_dev *eth_dev)
RTE_PTYPE_UNKNOWN
};

if (eth_dev->rx_pkt_burst == qede_recv_pkts)
if (eth_dev->rx_pkt_burst == qede_recv_pkts ||
eth_dev->rx_pkt_burst == qede_recv_pkts_cmt)
return ptypes;

return NULL;
Expand Down Expand Up @@ -2004,7 +1998,7 @@ int qede_rss_hash_update(struct rte_eth_dev *eth_dev,
vport_update_params.vport_id = 0;
/* pass the L2 handles instead of qids */
for (i = 0 ; i < ECORE_RSS_IND_TABLE_SIZE ; i++) {
idx = i % QEDE_RSS_COUNT(qdev);
idx = i % QEDE_RSS_COUNT(eth_dev);
rss_params.rss_ind_table[i] = qdev->fp_array[idx].rxq->handle;
}
vport_update_params.rss_params = &rss_params;
Expand Down Expand Up @@ -2256,7 +2250,7 @@ static int qede_set_mtu(struct rte_eth_dev *dev, uint16_t mtu)
qdev->mtu = mtu;

/* Fix up RX buf size for all queues of the port */
for_each_rss(i) {
for (i = 0; i < qdev->num_rx_queues; i++) {
fp = &qdev->fp_array[i];
if (fp->rxq != NULL) {
bufsz = (uint16_t)rte_pktmbuf_data_room_size(
Expand Down Expand Up @@ -2285,9 +2279,13 @@ static int qede_set_mtu(struct rte_eth_dev *dev, uint16_t mtu)
/* update max frame size */
dev->data->dev_conf.rxmode.max_rx_pkt_len = max_rx_pkt_len;
/* Reassign back */
dev->rx_pkt_burst = qede_recv_pkts;
dev->tx_pkt_burst = qede_xmit_pkts;

if (ECORE_IS_CMT(edev)) {
dev->rx_pkt_burst = qede_recv_pkts_cmt;
dev->tx_pkt_burst = qede_xmit_pkts_cmt;
} else {
dev->rx_pkt_burst = qede_recv_pkts;
dev->tx_pkt_burst = qede_xmit_pkts;
}
return 0;
}

Expand Down Expand Up @@ -2428,10 +2426,6 @@ static int qede_common_dev_init(struct rte_eth_dev *eth_dev, bool is_vf)
pci_addr.bus, pci_addr.devid, pci_addr.function,
eth_dev->data->port_id);

eth_dev->rx_pkt_burst = qede_recv_pkts;
eth_dev->tx_pkt_burst = qede_xmit_pkts;
eth_dev->tx_pkt_prepare = qede_xmit_prep_pkts;

if (rte_eal_process_type() != RTE_PROC_PRIMARY) {
DP_ERR(edev, "Skipping device init from secondary process\n");
return 0;
Expand Down Expand Up @@ -2489,6 +2483,16 @@ static int qede_common_dev_init(struct rte_eth_dev *eth_dev, bool is_vf)
strncpy((char *)params.name, QEDE_PMD_VER_PREFIX,
QEDE_PMD_DRV_VER_STR_SIZE);

if (ECORE_IS_CMT(edev)) {
eth_dev->rx_pkt_burst = qede_recv_pkts_cmt;
eth_dev->tx_pkt_burst = qede_xmit_pkts_cmt;
} else {
eth_dev->rx_pkt_burst = qede_recv_pkts;
eth_dev->tx_pkt_burst = qede_xmit_pkts;
}

eth_dev->tx_pkt_prepare = qede_xmit_prep_pkts;

/* For CMT mode device do periodic polling for slowpath events.
* This is required since uio device uses only one MSI-x
* interrupt vector but we need one for each engine.
Expand Down
5 changes: 3 additions & 2 deletions drivers/net/qede/qede_ethdev.h
View file
Expand Up @@ -66,8 +66,8 @@
(edev)->dev_info.num_tc)

#define QEDE_QUEUE_CNT(qdev) ((qdev)->num_queues)
#define QEDE_RSS_COUNT(qdev) ((qdev)->num_rx_queues)
#define QEDE_TSS_COUNT(qdev) ((qdev)->num_tx_queues)
#define QEDE_RSS_COUNT(dev) ((dev)->data->nb_rx_queues)
#define QEDE_TSS_COUNT(dev) ((dev)->data->nb_tx_queues)

#define QEDE_DUPLEX_FULL 1
#define QEDE_DUPLEX_HALF 2
Expand Down Expand Up @@ -215,6 +215,7 @@ struct qede_dev {
struct qed_dev_eth_info dev_info;
struct ecore_sb_info *sb_array;
struct qede_fastpath *fp_array;
struct qede_fastpath_cmt *fp_array_cmt;
uint16_t mtu;
bool enable_tx_switching;
bool rss_enable;
Expand Down
5 changes: 2 additions & 3 deletions drivers/net/qede/qede_filter.c
View file
Expand Up @@ -431,7 +431,7 @@ qede_fdir_filter_add(struct rte_eth_dev *eth_dev,
return -EINVAL;
}

if (fdir->action.rx_queue >= QEDE_RSS_COUNT(qdev)) {
if (fdir->action.rx_queue >= QEDE_RSS_COUNT(eth_dev)) {
DP_ERR(edev, "invalid queue number %u\n",
fdir->action.rx_queue);
return -EINVAL;
Expand Down Expand Up @@ -1343,7 +1343,6 @@ qede_flow_parse_actions(struct rte_eth_dev *dev,
struct rte_flow_error *error,
struct rte_flow *flow)
{
struct qede_dev *qdev = QEDE_INIT_QDEV(dev);
const struct rte_flow_action_queue *queue;

if (actions == NULL) {
Expand All @@ -1358,7 +1357,7 @@ qede_flow_parse_actions(struct rte_eth_dev *dev,
case RTE_FLOW_ACTION_TYPE_QUEUE:
queue = actions->conf;

if (queue->index >= QEDE_RSS_COUNT(qdev)) {
if (queue->index >= QEDE_RSS_COUNT(dev)) {
rte_flow_error_set(error, EINVAL,
RTE_FLOW_ERROR_TYPE_ACTION,
actions,
Expand Down

0 comments on commit eabce98

Please sign in to comment.