Skip to content

Commit a34e25a

Browse files
emuslndavem330
emusln
authored and
davem330
committed
ionic: change the descriptor ring length without full reset
The original way of changing ring length was to completely tear down the lif's queue structure and then rebuild it, while running the risk of allocations that might fail in the middle and leave us with a broken driver. Instead, we can set up all the new queue and descriptor allocations first, then swap them out and delete the old allocations. If the new allocations fail, we report the error, stay with the old setup and continue running. This gives us a safer path, and a smaller window of time where we're not processing traffic. Signed-off-by: Shannon Nelson <snelson@pensando.io> Signed-off-by: David S. Miller <davem@davemloft.net>
1 parent f053e1f commit a34e25a

File tree

3 files changed

+185
-14
lines changed

3 files changed

+185
-14
lines changed

‎drivers/net/ethernet/pensando/ionic/ionic_ethtool.c‎

Lines changed: 26 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -493,18 +493,14 @@ static void ionic_get_ringparam(struct net_device *netdev,
493493
ring->rx_pending = lif->nrxq_descs;
494494
}
495495

496-
static void ionic_set_ringsize(struct ionic_lif *lif, void *arg)
497-
{
498-
struct ethtool_ringparam *ring = arg;
499-
500-
lif->ntxq_descs = ring->tx_pending;
501-
lif->nrxq_descs = ring->rx_pending;
502-
}
503-
504496
static int ionic_set_ringparam(struct net_device *netdev,
505497
struct ethtool_ringparam *ring)
506498
{
507499
struct ionic_lif *lif = netdev_priv(netdev);
500+
struct ionic_queue_params qparam;
501+
int err;
502+
503+
ionic_init_queue_params(lif, &qparam);
508504

509505
if (ring->rx_mini_pending || ring->rx_jumbo_pending) {
510506
netdev_info(netdev, "Changing jumbo or mini descriptors not supported\n");
@@ -522,7 +518,28 @@ static int ionic_set_ringparam(struct net_device *netdev,
522518
ring->rx_pending == lif->nrxq_descs)
523519
return 0;
524520

525-
return ionic_reset_queues(lif, ionic_set_ringsize, ring);
521+
if (ring->tx_pending != lif->ntxq_descs)
522+
netdev_info(netdev, "Changing Tx ring size from %d to %d\n",
523+
lif->ntxq_descs, ring->tx_pending);
524+
525+
if (ring->rx_pending != lif->nrxq_descs)
526+
netdev_info(netdev, "Changing Rx ring size from %d to %d\n",
527+
lif->nrxq_descs, ring->rx_pending);
528+
529+
/* if we're not running, just set the values and return */
530+
if (!netif_running(lif->netdev)) {
531+
lif->ntxq_descs = ring->tx_pending;
532+
lif->nrxq_descs = ring->rx_pending;
533+
return 0;
534+
}
535+
536+
qparam.ntxq_descs = ring->tx_pending;
537+
qparam.nrxq_descs = ring->rx_pending;
538+
err = ionic_reconfigure_queues(lif, &qparam);
539+
if (err)
540+
netdev_info(netdev, "Ring reconfiguration failed, changes canceled: %d\n", err);
541+
542+
return err;
526543
}
527544

528545
static void ionic_get_channels(struct net_device *netdev,

‎drivers/net/ethernet/pensando/ionic/ionic_lif.c‎

Lines changed: 141 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -333,10 +333,14 @@ static void ionic_qcq_free(struct ionic_lif *lif, struct ionic_qcq *qcq)
333333
ionic_intr_free(lif->ionic, qcq->intr.index);
334334
}
335335

336-
devm_kfree(dev, qcq->cq.info);
337-
qcq->cq.info = NULL;
338-
devm_kfree(dev, qcq->q.info);
339-
qcq->q.info = NULL;
336+
if (qcq->cq.info) {
337+
devm_kfree(dev, qcq->cq.info);
338+
qcq->cq.info = NULL;
339+
}
340+
if (qcq->q.info) {
341+
devm_kfree(dev, qcq->q.info);
342+
qcq->q.info = NULL;
343+
}
340344
devm_kfree(dev, qcq);
341345
}
342346

@@ -2047,6 +2051,139 @@ static const struct net_device_ops ionic_netdev_ops = {
20472051
.ndo_get_vf_stats = ionic_get_vf_stats,
20482052
};
20492053

2054+
static void ionic_swap_queues(struct ionic_qcq *a, struct ionic_qcq *b)
2055+
{
2056+
/* only swapping the queues, not the napi, flags, or other stuff */
2057+
swap(a->q.num_descs, b->q.num_descs);
2058+
swap(a->q.base, b->q.base);
2059+
swap(a->q.base_pa, b->q.base_pa);
2060+
swap(a->q.info, b->q.info);
2061+
swap(a->q_base, b->q_base);
2062+
swap(a->q_base_pa, b->q_base_pa);
2063+
swap(a->q_size, b->q_size);
2064+
2065+
swap(a->q.sg_base, b->q.sg_base);
2066+
swap(a->q.sg_base_pa, b->q.sg_base_pa);
2067+
swap(a->sg_base, b->sg_base);
2068+
swap(a->sg_base_pa, b->sg_base_pa);
2069+
swap(a->sg_size, b->sg_size);
2070+
2071+
swap(a->cq.num_descs, b->cq.num_descs);
2072+
swap(a->cq.base, b->cq.base);
2073+
swap(a->cq.base_pa, b->cq.base_pa);
2074+
swap(a->cq.info, b->cq.info);
2075+
swap(a->cq_base, b->cq_base);
2076+
swap(a->cq_base_pa, b->cq_base_pa);
2077+
swap(a->cq_size, b->cq_size);
2078+
}
2079+
2080+
int ionic_reconfigure_queues(struct ionic_lif *lif,
2081+
struct ionic_queue_params *qparam)
2082+
{
2083+
struct ionic_qcq **tx_qcqs = NULL;
2084+
struct ionic_qcq **rx_qcqs = NULL;
2085+
unsigned int sg_desc_sz;
2086+
unsigned int flags;
2087+
int err = -ENOMEM;
2088+
unsigned int i;
2089+
2090+
/* allocate temporary qcq arrays to hold new queue structs */
2091+
if (qparam->ntxq_descs != lif->ntxq_descs) {
2092+
tx_qcqs = devm_kcalloc(lif->ionic->dev, lif->nxqs,
2093+
sizeof(struct ionic_qcq *), GFP_KERNEL);
2094+
if (!tx_qcqs)
2095+
goto err_out;
2096+
}
2097+
if (qparam->nrxq_descs != lif->nrxq_descs) {
2098+
rx_qcqs = devm_kcalloc(lif->ionic->dev, lif->nxqs,
2099+
sizeof(struct ionic_qcq *), GFP_KERNEL);
2100+
if (!rx_qcqs)
2101+
goto err_out;
2102+
}
2103+
2104+
/* allocate new desc_info and rings with no interrupt flag */
2105+
if (lif->qtype_info[IONIC_QTYPE_TXQ].version >= 1 &&
2106+
lif->qtype_info[IONIC_QTYPE_TXQ].sg_desc_sz ==
2107+
sizeof(struct ionic_txq_sg_desc_v1))
2108+
sg_desc_sz = sizeof(struct ionic_txq_sg_desc_v1);
2109+
else
2110+
sg_desc_sz = sizeof(struct ionic_txq_sg_desc);
2111+
2112+
if (tx_qcqs) {
2113+
for (i = 0; i < lif->nxqs; i++) {
2114+
flags = lif->txqcqs[i]->flags & ~IONIC_QCQ_F_INTR;
2115+
err = ionic_qcq_alloc(lif, IONIC_QTYPE_TXQ, i, "tx", flags,
2116+
qparam->ntxq_descs,
2117+
sizeof(struct ionic_txq_desc),
2118+
sizeof(struct ionic_txq_comp),
2119+
sg_desc_sz,
2120+
lif->kern_pid, &tx_qcqs[i]);
2121+
if (err)
2122+
goto err_out;
2123+
}
2124+
}
2125+
2126+
if (rx_qcqs) {
2127+
for (i = 0; i < lif->nxqs; i++) {
2128+
flags = lif->rxqcqs[i]->flags & ~IONIC_QCQ_F_INTR;
2129+
err = ionic_qcq_alloc(lif, IONIC_QTYPE_RXQ, i, "rx", flags,
2130+
qparam->nrxq_descs,
2131+
sizeof(struct ionic_rxq_desc),
2132+
sizeof(struct ionic_rxq_comp),
2133+
sizeof(struct ionic_rxq_sg_desc),
2134+
lif->kern_pid, &rx_qcqs[i]);
2135+
if (err)
2136+
goto err_out;
2137+
}
2138+
}
2139+
2140+
/* stop and clean the queues */
2141+
ionic_stop_queues_reconfig(lif);
2142+
2143+
/* swap new desc_info and rings, keeping existing interrupt config */
2144+
if (tx_qcqs) {
2145+
lif->ntxq_descs = qparam->ntxq_descs;
2146+
for (i = 0; i < lif->nxqs; i++)
2147+
ionic_swap_queues(lif->txqcqs[i], tx_qcqs[i]);
2148+
}
2149+
2150+
if (rx_qcqs) {
2151+
lif->nrxq_descs = qparam->nrxq_descs;
2152+
for (i = 0; i < lif->nxqs; i++)
2153+
ionic_swap_queues(lif->rxqcqs[i], rx_qcqs[i]);
2154+
}
2155+
2156+
/* re-init the queues */
2157+
err = ionic_start_queues_reconfig(lif);
2158+
2159+
err_out:
2160+
/* free old allocs without cleaning intr */
2161+
for (i = 0; i < lif->nxqs; i++) {
2162+
if (tx_qcqs && tx_qcqs[i]) {
2163+
tx_qcqs[i]->flags &= ~IONIC_QCQ_F_INTR;
2164+
ionic_qcq_free(lif, tx_qcqs[i]);
2165+
tx_qcqs[i] = NULL;
2166+
}
2167+
if (rx_qcqs && rx_qcqs[i]) {
2168+
rx_qcqs[i]->flags &= ~IONIC_QCQ_F_INTR;
2169+
ionic_qcq_free(lif, rx_qcqs[i]);
2170+
rx_qcqs[i] = NULL;
2171+
}
2172+
}
2173+
2174+
/* free q array */
2175+
if (rx_qcqs) {
2176+
devm_kfree(lif->ionic->dev, rx_qcqs);
2177+
rx_qcqs = NULL;
2178+
}
2179+
if (tx_qcqs) {
2180+
devm_kfree(lif->ionic->dev, tx_qcqs);
2181+
tx_qcqs = NULL;
2182+
}
2183+
2184+
return err;
2185+
}
2186+
20502187
int ionic_reset_queues(struct ionic_lif *lif, ionic_reset_cb cb, void *arg)
20512188
{
20522189
bool running;

‎drivers/net/ethernet/pensando/ionic/ionic_lif.h‎

Lines changed: 18 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -207,6 +207,22 @@ struct ionic_lif {
207207
struct work_struct tx_timeout_work;
208208
};
209209

210+
struct ionic_queue_params {
211+
unsigned int nxqs;
212+
unsigned int ntxq_descs;
213+
unsigned int nrxq_descs;
214+
unsigned int intr_split;
215+
};
216+
217+
static inline void ionic_init_queue_params(struct ionic_lif *lif,
218+
struct ionic_queue_params *qparam)
219+
{
220+
qparam->nxqs = lif->nxqs;
221+
qparam->ntxq_descs = lif->ntxq_descs;
222+
qparam->nrxq_descs = lif->nrxq_descs;
223+
qparam->intr_split = test_bit(IONIC_LIF_F_SPLIT_INTR, lif->state);
224+
}
225+
210226
static inline u32 ionic_coal_usec_to_hw(struct ionic *ionic, u32 usecs)
211227
{
212228
u32 mult = le32_to_cpu(ionic->ident.dev.intr_coal_mult);
@@ -241,7 +257,8 @@ int ionic_lif_identify(struct ionic *ionic, u8 lif_type,
241257
int ionic_lif_size(struct ionic *ionic);
242258
int ionic_lif_rss_config(struct ionic_lif *lif, u16 types,
243259
const u8 *key, const u32 *indir);
244-
260+
int ionic_reconfigure_queues(struct ionic_lif *lif,
261+
struct ionic_queue_params *qparam);
245262
int ionic_reset_queues(struct ionic_lif *lif, ionic_reset_cb cb, void *arg);
246263

247264
static inline void debug_stats_txq_post(struct ionic_queue *q, bool dbell)

0 commit comments

Comments
 (0)