Skip to content

Commit 3034322

Browse files
ajitkhapardeajitkhaparde
committed
bnxt_en: Remove runtime interrupt vector allocation
Modified the bnxt_en code to create and pre-configure RDMA devices with the right MSI-X vector count for the ROCE driver to use. This is to align the ROCE driver to the auxiliary device model which will simply bind the driver without getting into PCI-related handling. All PCI-related logic will now be in the bnxt_en driver. Suggested-by: Leon Romanovsky <leonro@nvidia.com> Signed-off-by: Ajit Khaparde <ajit.khaparde@broadcom.com> Reviewed-by: Leon Romanovsky <leonro@nvidia.com>
1 parent a43c26f commit 3034322

File tree

4 files changed

+56
-155
lines changed

4 files changed

+56
-155
lines changed

‎drivers/infiniband/hw/bnxt_re/bnxt_re.h‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -129,7 +129,6 @@ struct bnxt_re_dev {
129129
unsigned int version, major, minor;
130130
struct bnxt_qplib_chip_ctx *chip_ctx;
131131
struct bnxt_en_dev *en_dev;
132-
struct bnxt_msix_entry msix_entries[BNXT_RE_MAX_MSIX];
133132
int num_msix;
134133

135134
int id;

‎drivers/infiniband/hw/bnxt_re/main.c‎

Lines changed: 12 additions & 36 deletions
Original file line numberDiff line numberDiff line change
@@ -262,7 +262,7 @@ static void bnxt_re_stop_irq(void *handle)
262262
static void bnxt_re_start_irq(void *handle, struct bnxt_msix_entry *ent)
263263
{
264264
struct bnxt_re_dev *rdev = (struct bnxt_re_dev *)handle;
265-
struct bnxt_msix_entry *msix_ent = rdev->msix_entries;
265+
struct bnxt_msix_entry *msix_ent = rdev->en_dev->msix_entries;
266266
struct bnxt_qplib_rcfw *rcfw = &rdev->rcfw;
267267
struct bnxt_qplib_nq *nq;
268268
int indx, rc;
@@ -281,7 +281,7 @@ static void bnxt_re_start_irq(void *handle, struct bnxt_msix_entry *ent)
281281
* in device sctructure.
282282
*/
283283
for (indx = 0; indx < rdev->num_msix; indx++)
284-
rdev->msix_entries[indx].vector = ent[indx].vector;
284+
rdev->en_dev->msix_entries[indx].vector = ent[indx].vector;
285285

286286
bnxt_qplib_rcfw_start_irq(rcfw, msix_ent[BNXT_RE_AEQ_IDX].vector,
287287
false);
@@ -315,32 +315,6 @@ static int bnxt_re_register_netdev(struct bnxt_re_dev *rdev)
315315
return rc;
316316
}
317317

318-
static int bnxt_re_request_msix(struct bnxt_re_dev *rdev)
319-
{
320-
int rc = 0, num_msix_want = BNXT_RE_MAX_MSIX, num_msix_got;
321-
struct bnxt_en_dev *en_dev;
322-
323-
en_dev = rdev->en_dev;
324-
325-
num_msix_want = min_t(u32, BNXT_RE_MAX_MSIX, num_online_cpus());
326-
327-
num_msix_got = bnxt_req_msix_vecs(en_dev,
328-
rdev->msix_entries,
329-
num_msix_want);
330-
if (num_msix_got < BNXT_RE_MIN_MSIX) {
331-
rc = -EINVAL;
332-
goto done;
333-
}
334-
if (num_msix_got != num_msix_want) {
335-
ibdev_warn(&rdev->ibdev,
336-
"Requested %d MSI-X vectors, got %d\n",
337-
num_msix_want, num_msix_got);
338-
}
339-
rdev->num_msix = num_msix_got;
340-
done:
341-
return rc;
342-
}
343-
344318
static void bnxt_re_init_hwrm_hdr(struct bnxt_re_dev *rdev, struct input *hdr,
345319
u16 opcd, u16 crid, u16 trid)
346320
{
@@ -785,7 +759,7 @@ static u32 bnxt_re_get_nqdb_offset(struct bnxt_re_dev *rdev, u16 indx)
785759
return bnxt_qplib_is_chip_gen_p5(rdev->chip_ctx) ?
786760
(rdev->is_virtfn ? BNXT_RE_GEN_P5_VF_NQ_DB :
787761
BNXT_RE_GEN_P5_PF_NQ_DB) :
788-
rdev->msix_entries[indx].db_offset;
762+
rdev->en_dev->msix_entries[indx].db_offset;
789763
}
790764

791765
static void bnxt_re_cleanup_res(struct bnxt_re_dev *rdev)
@@ -810,7 +784,7 @@ static int bnxt_re_init_res(struct bnxt_re_dev *rdev)
810784
for (i = 1; i < rdev->num_msix ; i++) {
811785
db_offt = bnxt_re_get_nqdb_offset(rdev, i);
812786
rc = bnxt_qplib_enable_nq(rdev->en_dev->pdev, &rdev->nq[i - 1],
813-
i - 1, rdev->msix_entries[i].vector,
787+
i - 1, rdev->en_dev->msix_entries[i].vector,
814788
db_offt, &bnxt_re_cqn_handler,
815789
&bnxt_re_srqn_handler);
816790
if (rc) {
@@ -897,7 +871,7 @@ static int bnxt_re_alloc_res(struct bnxt_re_dev *rdev)
897871
rattr.type = type;
898872
rattr.mode = RING_ALLOC_REQ_INT_MODE_MSIX;
899873
rattr.depth = BNXT_QPLIB_NQE_MAX_CNT - 1;
900-
rattr.lrid = rdev->msix_entries[i + 1].ring_idx;
874+
rattr.lrid = rdev->en_dev->msix_entries[i + 1].ring_idx;
901875
rc = bnxt_re_net_ring_alloc(rdev, &rattr, &nq->ring_id);
902876
if (rc) {
903877
ibdev_err(&rdev->ibdev,
@@ -1217,7 +1191,7 @@ static void bnxt_re_dev_uninit(struct bnxt_re_dev *rdev)
12171191
bnxt_qplib_free_rcfw_channel(&rdev->rcfw);
12181192
}
12191193
if (test_and_clear_bit(BNXT_RE_FLAG_GOT_MSIX, &rdev->flags))
1220-
bnxt_free_msix_vecs(rdev->en_dev);
1194+
rdev->num_msix = 0;
12211195

12221196
bnxt_re_destroy_chip_ctx(rdev);
12231197
if (test_and_clear_bit(BNXT_RE_FLAG_NETDEV_REGISTERED, &rdev->flags))
@@ -1262,13 +1236,15 @@ static int bnxt_re_dev_init(struct bnxt_re_dev *rdev, u8 wqe_mode)
12621236
/* Check whether VF or PF */
12631237
bnxt_re_get_sriov_func_type(rdev);
12641238

1265-
rc = bnxt_re_request_msix(rdev);
1266-
if (rc) {
1239+
if (!rdev->en_dev->ulp_tbl->msix_requested) {
12671240
ibdev_err(&rdev->ibdev,
12681241
"Failed to get MSI-X vectors: %#x\n", rc);
12691242
rc = -EINVAL;
12701243
goto fail;
12711244
}
1245+
ibdev_dbg(&rdev->ibdev, "Got %d MSI-X vectors\n",
1246+
rdev->en_dev->ulp_tbl->msix_requested);
1247+
rdev->num_msix = rdev->en_dev->ulp_tbl->msix_requested;
12721248
set_bit(BNXT_RE_FLAG_GOT_MSIX, &rdev->flags);
12731249

12741250
bnxt_re_query_hwrm_intf_version(rdev);
@@ -1292,14 +1268,14 @@ static int bnxt_re_dev_init(struct bnxt_re_dev *rdev, u8 wqe_mode)
12921268
rattr.type = type;
12931269
rattr.mode = RING_ALLOC_REQ_INT_MODE_MSIX;
12941270
rattr.depth = BNXT_QPLIB_CREQE_MAX_CNT - 1;
1295-
rattr.lrid = rdev->msix_entries[BNXT_RE_AEQ_IDX].ring_idx;
1271+
rattr.lrid = rdev->en_dev->msix_entries[BNXT_RE_AEQ_IDX].ring_idx;
12961272
rc = bnxt_re_net_ring_alloc(rdev, &rattr, &creq->ring_id);
12971273
if (rc) {
12981274
ibdev_err(&rdev->ibdev, "Failed to allocate CREQ: %#x\n", rc);
12991275
goto free_rcfw;
13001276
}
13011277
db_offt = bnxt_re_get_nqdb_offset(rdev, BNXT_RE_AEQ_IDX);
1302-
vid = rdev->msix_entries[BNXT_RE_AEQ_IDX].vector;
1278+
vid = rdev->en_dev->msix_entries[BNXT_RE_AEQ_IDX].vector;
13031279
rc = bnxt_qplib_enable_rcfw_channel(&rdev->rcfw,
13041280
vid, db_offt, rdev->is_virtfn,
13051281
&bnxt_re_aeq_handler);

‎drivers/net/ethernet/broadcom/bnxt/bnxt_ulp.c‎

Lines changed: 41 additions & 115 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,30 @@
2828

2929
static DEFINE_IDA(bnxt_aux_dev_ids);
3030

31+
static void bnxt_fill_msix_vecs(struct bnxt *bp, struct bnxt_msix_entry *ent)
32+
{
33+
struct bnxt_en_dev *edev = bp->edev;
34+
int num_msix, idx, i;
35+
36+
if (!edev->ulp_tbl->msix_requested) {
37+
netdev_warn(bp->dev, "Requested MSI-X vectors insufficient\n");
38+
return;
39+
}
40+
num_msix = edev->ulp_tbl->msix_requested;
41+
idx = edev->ulp_tbl->msix_base;
42+
for (i = 0; i < num_msix; i++) {
43+
ent[i].vector = bp->irq_tbl[idx + i].vector;
44+
ent[i].ring_idx = idx + i;
45+
if (bp->flags & BNXT_FLAG_CHIP_P5) {
46+
ent[i].db_offset = DB_PF_OFFSET_P5;
47+
if (BNXT_VF(bp))
48+
ent[i].db_offset = DB_VF_OFFSET_P5;
49+
} else {
50+
ent[i].db_offset = (idx + i) * 0x80;
51+
}
52+
}
53+
}
54+
3155
int bnxt_register_dev(struct bnxt_en_dev *edev,
3256
struct bnxt_ulp_ops *ulp_ops,
3357
void *handle)
@@ -42,17 +66,18 @@ int bnxt_register_dev(struct bnxt_en_dev *edev,
4266
bp->cp_nr_rings == max_stat_ctxs)
4367
return -ENOMEM;
4468

45-
ulp = kzalloc(sizeof(*ulp), GFP_KERNEL);
69+
ulp = edev->ulp_tbl;
4670
if (!ulp)
4771
return -ENOMEM;
4872

49-
edev->ulp_tbl = ulp;
5073
ulp->handle = handle;
5174
rcu_assign_pointer(ulp->ulp_ops, ulp_ops);
5275

5376
if (test_bit(BNXT_STATE_OPEN, &bp->state))
5477
bnxt_hwrm_vnic_cfg(bp, 0);
5578

79+
bnxt_fill_msix_vecs(bp, bp->edev->msix_entries);
80+
edev->flags |= BNXT_EN_FLAG_MSIX_REQUESTED;
5681
return 0;
5782
}
5883
EXPORT_SYMBOL(bnxt_register_dev);
@@ -66,7 +91,7 @@ void bnxt_unregister_dev(struct bnxt_en_dev *edev)
6691

6792
ulp = edev->ulp_tbl;
6893
if (ulp->msix_requested)
69-
bnxt_free_msix_vecs(edev);
94+
edev->flags &= ~BNXT_EN_FLAG_MSIX_REQUESTED;
7095

7196
if (ulp->max_async_event_id)
7297
bnxt_hwrm_func_drv_rgtr(bp, NULL, 0, true);
@@ -79,125 +104,17 @@ void bnxt_unregister_dev(struct bnxt_en_dev *edev)
79104
msleep(100);
80105
i++;
81106
}
82-
kfree(ulp);
83-
edev->ulp_tbl = NULL;
84107
return;
85108
}
86109
EXPORT_SYMBOL(bnxt_unregister_dev);
87110

88-
static void bnxt_fill_msix_vecs(struct bnxt *bp, struct bnxt_msix_entry *ent)
89-
{
90-
struct bnxt_en_dev *edev = bp->edev;
91-
int num_msix, idx, i;
92-
93-
num_msix = edev->ulp_tbl->msix_requested;
94-
idx = edev->ulp_tbl->msix_base;
95-
for (i = 0; i < num_msix; i++) {
96-
ent[i].vector = bp->irq_tbl[idx + i].vector;
97-
ent[i].ring_idx = idx + i;
98-
if (bp->flags & BNXT_FLAG_CHIP_P5) {
99-
ent[i].db_offset = DB_PF_OFFSET_P5;
100-
if (BNXT_VF(bp))
101-
ent[i].db_offset = DB_VF_OFFSET_P5;
102-
} else {
103-
ent[i].db_offset = (idx + i) * 0x80;
104-
}
105-
}
106-
}
107-
108-
int bnxt_req_msix_vecs(struct bnxt_en_dev *edev,
109-
struct bnxt_msix_entry *ent,
110-
int num_msix)
111-
{
112-
struct net_device *dev = edev->net;
113-
struct bnxt *bp = netdev_priv(dev);
114-
struct bnxt_hw_resc *hw_resc;
115-
int max_idx, max_cp_rings;
116-
int avail_msix, idx;
117-
int total_vecs;
118-
int rc = 0;
119-
120-
if (!(bp->flags & BNXT_FLAG_USING_MSIX))
121-
return -ENODEV;
122-
123-
if (edev->ulp_tbl->msix_requested)
124-
return -EAGAIN;
125-
126-
max_cp_rings = bnxt_get_max_func_cp_rings(bp);
127-
avail_msix = bnxt_get_avail_msix(bp, num_msix);
128-
if (!avail_msix)
129-
return -ENOMEM;
130-
if (avail_msix > num_msix)
131-
avail_msix = num_msix;
132-
133-
if (BNXT_NEW_RM(bp)) {
134-
idx = bp->cp_nr_rings;
135-
} else {
136-
max_idx = min_t(int, bp->total_irqs, max_cp_rings);
137-
idx = max_idx - avail_msix;
138-
}
139-
edev->ulp_tbl->msix_base = idx;
140-
edev->ulp_tbl->msix_requested = avail_msix;
141-
hw_resc = &bp->hw_resc;
142-
total_vecs = idx + avail_msix;
143-
rtnl_lock();
144-
if (bp->total_irqs < total_vecs ||
145-
(BNXT_NEW_RM(bp) && hw_resc->resv_irqs < total_vecs)) {
146-
if (netif_running(dev)) {
147-
bnxt_close_nic(bp, true, false);
148-
rc = bnxt_open_nic(bp, true, false);
149-
} else {
150-
rc = bnxt_reserve_rings(bp, true);
151-
}
152-
}
153-
rtnl_unlock();
154-
if (rc) {
155-
edev->ulp_tbl->msix_requested = 0;
156-
return -EAGAIN;
157-
}
158-
159-
if (BNXT_NEW_RM(bp)) {
160-
int resv_msix;
161-
162-
resv_msix = hw_resc->resv_irqs - bp->cp_nr_rings;
163-
avail_msix = min_t(int, resv_msix, avail_msix);
164-
edev->ulp_tbl->msix_requested = avail_msix;
165-
}
166-
bnxt_fill_msix_vecs(bp, ent);
167-
edev->flags |= BNXT_EN_FLAG_MSIX_REQUESTED;
168-
return avail_msix;
169-
}
170-
EXPORT_SYMBOL(bnxt_req_msix_vecs);
171-
172-
void bnxt_free_msix_vecs(struct bnxt_en_dev *edev)
173-
{
174-
struct net_device *dev = edev->net;
175-
struct bnxt *bp = netdev_priv(dev);
176-
177-
if (!(edev->flags & BNXT_EN_FLAG_MSIX_REQUESTED))
178-
return;
179-
180-
edev->ulp_tbl->msix_requested = 0;
181-
edev->flags &= ~BNXT_EN_FLAG_MSIX_REQUESTED;
182-
rtnl_lock();
183-
if (netif_running(dev) && !(edev->flags & BNXT_EN_FLAG_ULP_STOPPED)) {
184-
bnxt_close_nic(bp, true, false);
185-
bnxt_open_nic(bp, true, false);
186-
}
187-
rtnl_unlock();
188-
189-
return;
190-
}
191-
EXPORT_SYMBOL(bnxt_free_msix_vecs);
192-
193111
int bnxt_get_ulp_msix_num(struct bnxt *bp)
194112
{
195-
if (bnxt_ulp_registered(bp->edev)) {
196-
struct bnxt_en_dev *edev = bp->edev;
113+
u32 roce_msix = BNXT_VF(bp) ?
114+
BNXT_MAX_VF_ROCE_MSIX : BNXT_MAX_ROCE_MSIX;
197115

198-
return edev->ulp_tbl->msix_requested;
199-
}
200-
return 0;
116+
return ((bp->flags & BNXT_FLAG_ROCE_CAP) ?
117+
min_t(u32, roce_msix, num_online_cpus()) : 0);
201118
}
202119

203120
int bnxt_get_ulp_msix_base(struct bnxt *bp)
@@ -402,6 +319,7 @@ static void bnxt_aux_dev_release(struct device *dev)
402319
container_of(dev, struct bnxt_aux_priv, aux_dev.dev);
403320

404321
ida_free(&bnxt_aux_dev_ids, aux_priv->id);
322+
kfree(aux_priv->edev->ulp_tbl);
405323
kfree(aux_priv->edev);
406324
kfree(aux_priv);
407325
}
@@ -424,13 +342,16 @@ static void bnxt_set_edev_info(struct bnxt_en_dev *edev, struct bnxt *bp)
424342
edev->hw_ring_stats_size = bp->hw_ring_stats_size;
425343
edev->pf_port_id = bp->pf.port_id;
426344
edev->en_state = bp->state;
345+
346+
edev->ulp_tbl->msix_requested = bnxt_get_ulp_msix_num(bp);
427347
}
428348

429349
void bnxt_rdma_aux_device_init(struct bnxt *bp)
430350
{
431351
struct auxiliary_device *aux_dev;
432352
struct bnxt_aux_priv *aux_priv;
433353
struct bnxt_en_dev *edev;
354+
struct bnxt_ulp *ulp;
434355
int rc;
435356

436357
if (!(bp->flags & BNXT_FLAG_ROCE_CAP))
@@ -470,6 +391,11 @@ void bnxt_rdma_aux_device_init(struct bnxt *bp)
470391
if (!edev)
471392
goto aux_dev_uninit;
472393

394+
ulp = kzalloc(sizeof(*ulp), GFP_KERNEL);
395+
if (!ulp)
396+
goto aux_dev_uninit;
397+
398+
edev->ulp_tbl = ulp;
473399
aux_priv->edev = edev;
474400
bp->edev = edev;
475401
bnxt_set_edev_info(edev, bp);

‎drivers/net/ethernet/broadcom/bnxt/bnxt_ulp.h‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,8 @@
1515

1616
#define BNXT_MIN_ROCE_CP_RINGS 2
1717
#define BNXT_MIN_ROCE_STAT_CTXS 1
18+
#define BNXT_MAX_ROCE_MSIX 9
19+
#define BNXT_MAX_VF_ROCE_MSIX 2
1820

1921
struct hwrm_async_event_cmpl;
2022
struct bnxt;
@@ -51,6 +53,7 @@ struct bnxt_ulp {
5153
struct bnxt_en_dev {
5254
struct net_device *net;
5355
struct pci_dev *pdev;
56+
struct bnxt_msix_entry msix_entries[BNXT_MAX_ROCE_MSIX];
5457
u32 flags;
5558
#define BNXT_EN_FLAG_ROCEV1_CAP 0x1
5659
#define BNXT_EN_FLAG_ROCEV2_CAP 0x2
@@ -101,9 +104,6 @@ void bnxt_rdma_aux_device_init(struct bnxt *bp);
101104
int bnxt_register_dev(struct bnxt_en_dev *edev, struct bnxt_ulp_ops *ulp_ops,
102105
void *handle);
103106
void bnxt_unregister_dev(struct bnxt_en_dev *edev);
104-
int bnxt_req_msix_vecs(struct bnxt_en_dev *edev, struct bnxt_msix_entry *ent,
105-
int num_msix);
106-
void bnxt_free_msix_vecs(struct bnxt_en_dev *edev);
107107
int bnxt_send_msg(struct bnxt_en_dev *edev, struct bnxt_fw_msg *fw_msg);
108108
int bnxt_register_async_events(struct bnxt_en_dev *edev,
109109
unsigned long *events_bmap, u16 max_id);

0 commit comments

Comments
 (0)