Skip to content

Commit cde29af

Browse files
shinas-marvellPaolo Abeni
shinas-marvell
authored and
Paolo Abeni
committed
octeon_ep: add PF-VF mailbox communication
Implement mailbox communication between PF and VFs. PF-VF mailbox is used for all control commands from VF to PF and asynchronous notification messages from PF to VF. Signed-off-by: Shinas Rasheed <srasheed@marvell.com> Reviewed-by: Simon Horman <horms@kernel.org> Signed-off-by: Paolo Abeni <pabeni@redhat.com>
1 parent c49b292 commit cde29af

File tree

11 files changed

+713
-49
lines changed

11 files changed

+713
-49
lines changed

‎drivers/net/ethernet/marvell/octeon_ep/Makefile‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,4 +7,4 @@ obj-$(CONFIG_OCTEON_EP) += octeon_ep.o
77

88
octeon_ep-y := octep_main.o octep_cn9k_pf.o octep_tx.o octep_rx.o \
99
octep_ethtool.o octep_ctrl_mbox.o octep_ctrl_net.o \
10-
octep_cnxk_pf.o
10+
octep_pfvf_mbox.o octep_cnxk_pf.o

‎drivers/net/ethernet/marvell/octeon_ep/octep_cn9k_pf.c‎

Lines changed: 52 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -362,16 +362,55 @@ static void octep_setup_mbox_regs_cn93_pf(struct octep_device *oct, int q_no)
362362
{
363363
struct octep_mbox *mbox = oct->mbox[q_no];
364364

365-
mbox->q_no = q_no;
366-
367-
/* PF mbox interrupt reg */
368-
mbox->mbox_int_reg = oct->mmio[0].hw_addr + CN93_SDP_EPF_MBOX_RINT(0);
369-
370365
/* PF to VF DATA reg. PF writes into this reg */
371-
mbox->mbox_write_reg = oct->mmio[0].hw_addr + CN93_SDP_R_MBOX_PF_VF_DATA(q_no);
366+
mbox->pf_vf_data_reg = oct->mmio[0].hw_addr + CN93_SDP_MBOX_PF_VF_DATA(q_no);
372367

373368
/* VF to PF DATA reg. PF reads from this reg */
374-
mbox->mbox_read_reg = oct->mmio[0].hw_addr + CN93_SDP_R_MBOX_VF_PF_DATA(q_no);
369+
mbox->vf_pf_data_reg = oct->mmio[0].hw_addr + CN93_SDP_MBOX_VF_PF_DATA(q_no);
370+
}
371+
372+
/* Poll for mailbox messages from VF */
373+
static void octep_poll_pfvf_mailbox(struct octep_device *oct)
374+
{
375+
u32 vf, active_vfs, active_rings_per_vf, vf_mbox_queue;
376+
u64 reg0, reg1;
377+
378+
reg0 = octep_read_csr64(oct, CN93_SDP_EPF_MBOX_RINT(0));
379+
reg1 = octep_read_csr64(oct, CN93_SDP_EPF_MBOX_RINT(1));
380+
if (reg0 || reg1) {
381+
active_vfs = CFG_GET_ACTIVE_VFS(oct->conf);
382+
active_rings_per_vf = CFG_GET_ACTIVE_RPVF(oct->conf);
383+
for (vf = 0; vf < active_vfs; vf++) {
384+
vf_mbox_queue = vf * active_rings_per_vf;
385+
386+
if (vf_mbox_queue < 64) {
387+
if (!(reg0 & (0x1UL << vf_mbox_queue)))
388+
continue;
389+
} else {
390+
if (!(reg1 & (0x1UL << (vf_mbox_queue - 64))))
391+
continue;
392+
}
393+
394+
if (!oct->mbox[vf_mbox_queue]) {
395+
dev_err(&oct->pdev->dev, "bad mbox vf %d\n", vf);
396+
continue;
397+
}
398+
schedule_work(&oct->mbox[vf_mbox_queue]->wk.work);
399+
}
400+
if (reg0)
401+
octep_write_csr64(oct, CN93_SDP_EPF_MBOX_RINT(0), reg0);
402+
if (reg1)
403+
octep_write_csr64(oct, CN93_SDP_EPF_MBOX_RINT(1), reg1);
404+
}
405+
}
406+
407+
/* PF-VF mailbox interrupt handler */
408+
static irqreturn_t octep_pfvf_mbox_intr_handler_cn93_pf(void *dev)
409+
{
410+
struct octep_device *oct = (struct octep_device *)dev;
411+
412+
octep_poll_pfvf_mailbox(oct);
413+
return IRQ_HANDLED;
375414
}
376415

377416
/* Poll OEI events like heartbeat */
@@ -403,6 +442,7 @@ static irqreturn_t octep_oei_intr_handler_cn93_pf(void *dev)
403442
*/
404443
static void octep_poll_non_ioq_interrupts_cn93_pf(struct octep_device *oct)
405444
{
445+
octep_poll_pfvf_mailbox(oct);
406446
octep_poll_oei_cn93_pf(oct);
407447
}
408448

@@ -646,6 +686,8 @@ static void octep_enable_interrupts_cn93_pf(struct octep_device *oct)
646686

647687
octep_write_csr64(oct, CN93_SDP_EPF_MISC_RINT_ENA_W1S, intr_mask);
648688
octep_write_csr64(oct, CN93_SDP_EPF_DMA_RINT_ENA_W1S, intr_mask);
689+
octep_write_csr64(oct, CN93_SDP_EPF_MBOX_RINT_ENA_W1S(0), -1ULL);
690+
octep_write_csr64(oct, CN93_SDP_EPF_MBOX_RINT_ENA_W1S(1), -1ULL);
649691

650692
octep_write_csr64(oct, CN93_SDP_EPF_DMA_VF_RINT_ENA_W1S(0), -1ULL);
651693
octep_write_csr64(oct, CN93_SDP_EPF_PP_VF_RINT_ENA_W1S(0), -1ULL);
@@ -672,6 +714,8 @@ static void octep_disable_interrupts_cn93_pf(struct octep_device *oct)
672714

673715
octep_write_csr64(oct, CN93_SDP_EPF_MISC_RINT_ENA_W1C, intr_mask);
674716
octep_write_csr64(oct, CN93_SDP_EPF_DMA_RINT_ENA_W1C, intr_mask);
717+
octep_write_csr64(oct, CN93_SDP_EPF_MBOX_RINT_ENA_W1C(0), -1ULL);
718+
octep_write_csr64(oct, CN93_SDP_EPF_MBOX_RINT_ENA_W1C(1), -1ULL);
675719

676720
octep_write_csr64(oct, CN93_SDP_EPF_DMA_VF_RINT_ENA_W1C(0), -1ULL);
677721
octep_write_csr64(oct, CN93_SDP_EPF_PP_VF_RINT_ENA_W1C(0), -1ULL);
@@ -807,6 +851,7 @@ void octep_device_setup_cn93_pf(struct octep_device *oct)
807851
oct->hw_ops.setup_oq_regs = octep_setup_oq_regs_cn93_pf;
808852
oct->hw_ops.setup_mbox_regs = octep_setup_mbox_regs_cn93_pf;
809853

854+
oct->hw_ops.mbox_intr_handler = octep_pfvf_mbox_intr_handler_cn93_pf;
810855
oct->hw_ops.oei_intr_handler = octep_oei_intr_handler_cn93_pf;
811856
oct->hw_ops.ire_intr_handler = octep_ire_intr_handler_cn93_pf;
812857
oct->hw_ops.ore_intr_handler = octep_ore_intr_handler_cn93_pf;

‎drivers/net/ethernet/marvell/octeon_ep/octep_cnxk_pf.c‎

Lines changed: 39 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -392,16 +392,44 @@ static void octep_setup_mbox_regs_cnxk_pf(struct octep_device *oct, int q_no)
392392
{
393393
struct octep_mbox *mbox = oct->mbox[q_no];
394394

395-
mbox->q_no = q_no;
396-
397-
/* PF mbox interrupt reg */
398-
mbox->mbox_int_reg = oct->mmio[0].hw_addr + CNXK_SDP_EPF_MBOX_RINT(0);
399-
400395
/* PF to VF DATA reg. PF writes into this reg */
401-
mbox->mbox_write_reg = oct->mmio[0].hw_addr + CNXK_SDP_R_MBOX_PF_VF_DATA(q_no);
396+
mbox->pf_vf_data_reg = oct->mmio[0].hw_addr + CNXK_SDP_MBOX_PF_VF_DATA(q_no);
402397

403398
/* VF to PF DATA reg. PF reads from this reg */
404-
mbox->mbox_read_reg = oct->mmio[0].hw_addr + CNXK_SDP_R_MBOX_VF_PF_DATA(q_no);
399+
mbox->vf_pf_data_reg = oct->mmio[0].hw_addr + CNXK_SDP_MBOX_VF_PF_DATA(q_no);
400+
}
401+
402+
static void octep_poll_pfvf_mailbox_cnxk_pf(struct octep_device *oct)
403+
{
404+
u32 vf, active_vfs, active_rings_per_vf, vf_mbox_queue;
405+
u64 reg0;
406+
407+
reg0 = octep_read_csr64(oct, CNXK_SDP_EPF_MBOX_RINT(0));
408+
if (reg0) {
409+
active_vfs = CFG_GET_ACTIVE_VFS(oct->conf);
410+
active_rings_per_vf = CFG_GET_ACTIVE_RPVF(oct->conf);
411+
for (vf = 0; vf < active_vfs; vf++) {
412+
vf_mbox_queue = vf * active_rings_per_vf;
413+
if (!(reg0 & (0x1UL << vf_mbox_queue)))
414+
continue;
415+
416+
if (!oct->mbox[vf_mbox_queue]) {
417+
dev_err(&oct->pdev->dev, "bad mbox vf %d\n", vf);
418+
continue;
419+
}
420+
schedule_work(&oct->mbox[vf_mbox_queue]->wk.work);
421+
}
422+
if (reg0)
423+
octep_write_csr64(oct, CNXK_SDP_EPF_MBOX_RINT(0), reg0);
424+
}
425+
}
426+
427+
static irqreturn_t octep_pfvf_mbox_intr_handler_cnxk_pf(void *dev)
428+
{
429+
struct octep_device *oct = (struct octep_device *)dev;
430+
431+
octep_poll_pfvf_mailbox_cnxk_pf(oct);
432+
return IRQ_HANDLED;
405433
}
406434

407435
/* Poll OEI events like heartbeat */
@@ -435,6 +463,7 @@ static irqreturn_t octep_oei_intr_handler_cnxk_pf(void *dev)
435463
*/
436464
static void octep_poll_non_ioq_interrupts_cnxk_pf(struct octep_device *oct)
437465
{
466+
octep_poll_pfvf_mailbox_cnxk_pf(oct);
438467
octep_poll_oei_cnxk_pf(oct);
439468
}
440469

@@ -682,6 +711,7 @@ static void octep_enable_interrupts_cnxk_pf(struct octep_device *oct)
682711

683712
octep_write_csr64(oct, CNXK_SDP_EPF_MISC_RINT_ENA_W1S, intr_mask);
684713
octep_write_csr64(oct, CNXK_SDP_EPF_DMA_RINT_ENA_W1S, intr_mask);
714+
octep_write_csr64(oct, CNXK_SDP_EPF_MBOX_RINT_ENA_W1S(0), -1ULL);
685715

686716
octep_write_csr64(oct, CNXK_SDP_EPF_DMA_VF_RINT_ENA_W1S(0), -1ULL);
687717
octep_write_csr64(oct, CNXK_SDP_EPF_PP_VF_RINT_ENA_W1S(0), -1ULL);
@@ -708,6 +738,7 @@ static void octep_disable_interrupts_cnxk_pf(struct octep_device *oct)
708738

709739
octep_write_csr64(oct, CNXK_SDP_EPF_MISC_RINT_ENA_W1C, intr_mask);
710740
octep_write_csr64(oct, CNXK_SDP_EPF_DMA_RINT_ENA_W1C, intr_mask);
741+
octep_write_csr64(oct, CNXK_SDP_EPF_MBOX_RINT_ENA_W1C(0), -1ULL);
711742

712743
octep_write_csr64(oct, CNXK_SDP_EPF_DMA_VF_RINT_ENA_W1C(0), -1ULL);
713744
octep_write_csr64(oct, CNXK_SDP_EPF_PP_VF_RINT_ENA_W1C(0), -1ULL);
@@ -843,6 +874,7 @@ void octep_device_setup_cnxk_pf(struct octep_device *oct)
843874
oct->hw_ops.setup_oq_regs = octep_setup_oq_regs_cnxk_pf;
844875
oct->hw_ops.setup_mbox_regs = octep_setup_mbox_regs_cnxk_pf;
845876

877+
oct->hw_ops.mbox_intr_handler = octep_pfvf_mbox_intr_handler_cnxk_pf;
846878
oct->hw_ops.oei_intr_handler = octep_oei_intr_handler_cnxk_pf;
847879
oct->hw_ops.ire_intr_handler = octep_ire_intr_handler_cnxk_pf;
848880
oct->hw_ops.ore_intr_handler = octep_ore_intr_handler_cnxk_pf;

‎drivers/net/ethernet/marvell/octeon_ep/octep_ctrl_mbox.h‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,10 +16,12 @@
1616
* |reserved (4 bytes) |
1717
* |-------------------------------------------|
1818
* |host version (8 bytes) |
19+
* | low 32 bits |
1920
* |host status (8 bytes) |
2021
* |host reserved (104 bytes) |
2122
* |-------------------------------------------|
22-
* |fw version (8 bytes) |
23+
* |fw version's (8 bytes) |
24+
* | min=high 32 bits, max=low 32 bits |
2325
* |fw status (8 bytes) |
2426
* |fw reserved (104 bytes) |
2527
* |===========================================|

‎drivers/net/ethernet/marvell/octeon_ep/octep_main.c‎

Lines changed: 81 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@
1616
#include "octep_config.h"
1717
#include "octep_main.h"
1818
#include "octep_ctrl_net.h"
19+
#include "octep_pfvf_mbox.h"
1920

2021
#define OCTEP_INTR_POLL_TIME_MSECS 100
2122
struct workqueue_struct *octep_wq;
@@ -159,6 +160,21 @@ static void octep_disable_msix(struct octep_device *oct)
159160
dev_info(&oct->pdev->dev, "Disabled MSI-X\n");
160161
}
161162

163+
/**
164+
* octep_mbox_intr_handler() - common handler for pfvf mbox interrupts.
165+
*
166+
* @irq: Interrupt number.
167+
* @data: interrupt data.
168+
*
169+
* this is common handler for pfvf mbox interrupts.
170+
*/
171+
static irqreturn_t octep_mbox_intr_handler(int irq, void *data)
172+
{
173+
struct octep_device *oct = data;
174+
175+
return oct->hw_ops.mbox_intr_handler(oct);
176+
}
177+
162178
/**
163179
* octep_oei_intr_handler() - common handler for output endpoint interrupts.
164180
*
@@ -362,8 +378,12 @@ static int octep_request_irqs(struct octep_device *oct)
362378

363379
snprintf(irq_name, OCTEP_MSIX_NAME_SIZE,
364380
"%s-%s", netdev->name, non_ioq_msix_names[i]);
365-
if (!strncmp(non_ioq_msix_names[i], "epf_oei_rint",
366-
strlen("epf_oei_rint"))) {
381+
if (!strncmp(non_ioq_msix_names[i], "epf_mbox_rint", strlen("epf_mbox_rint"))) {
382+
ret = request_irq(msix_entry->vector,
383+
octep_mbox_intr_handler, 0,
384+
irq_name, oct);
385+
} else if (!strncmp(non_ioq_msix_names[i], "epf_oei_rint",
386+
strlen("epf_oei_rint"))) {
367387
ret = request_irq(msix_entry->vector,
368388
octep_oei_intr_handler, 0,
369389
irq_name, oct);
@@ -1322,6 +1342,7 @@ static void octep_device_cleanup(struct octep_device *oct)
13221342
oct->mbox[i] = NULL;
13231343
}
13241344

1345+
octep_delete_pfvf_mbox(oct);
13251346
octep_ctrl_net_uninit(oct);
13261347
cancel_delayed_work_sync(&oct->hb_task);
13271348

@@ -1419,6 +1440,12 @@ static int octep_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
14191440
goto err_octep_config;
14201441
}
14211442

1443+
err = octep_setup_pfvf_mbox(octep_dev);
1444+
if (err) {
1445+
dev_err(&pdev->dev, "PF-VF mailbox setup failed\n");
1446+
goto register_dev_err;
1447+
}
1448+
14221449
err = octep_ctrl_net_get_info(octep_dev, OCTEP_CTRL_NET_INVALID_VFID,
14231450
&octep_dev->conf->fw_info);
14241451
if (err) {
@@ -1487,6 +1514,21 @@ static int octep_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
14871514
return err;
14881515
}
14891516

1517+
static int octep_sriov_disable(struct octep_device *oct)
1518+
{
1519+
struct pci_dev *pdev = oct->pdev;
1520+
1521+
if (pci_vfs_assigned(oct->pdev)) {
1522+
dev_warn(&pdev->dev, "Can't disable SRIOV while VFs are assigned\n");
1523+
return -EPERM;
1524+
}
1525+
1526+
pci_disable_sriov(pdev);
1527+
CFG_GET_ACTIVE_VFS(oct->conf) = 0;
1528+
1529+
return 0;
1530+
}
1531+
14901532
/**
14911533
* octep_remove() - Remove Octeon PCI device from driver control.
14921534
*
@@ -1504,6 +1546,7 @@ static void octep_remove(struct pci_dev *pdev)
15041546
return;
15051547

15061548
netdev = oct->netdev;
1549+
octep_sriov_disable(oct);
15071550
if (netdev->reg_state == NETREG_REGISTERED)
15081551
unregister_netdev(netdev);
15091552

@@ -1514,11 +1557,47 @@ static void octep_remove(struct pci_dev *pdev)
15141557
pci_disable_device(pdev);
15151558
}
15161559

1560+
static int octep_sriov_enable(struct octep_device *oct, int num_vfs)
1561+
{
1562+
struct pci_dev *pdev = oct->pdev;
1563+
int err;
1564+
1565+
CFG_GET_ACTIVE_VFS(oct->conf) = num_vfs;
1566+
err = pci_enable_sriov(pdev, num_vfs);
1567+
if (err) {
1568+
dev_warn(&pdev->dev, "Failed to enable SRIOV err=%d\n", err);
1569+
CFG_GET_ACTIVE_VFS(oct->conf) = 0;
1570+
return err;
1571+
}
1572+
1573+
return num_vfs;
1574+
}
1575+
1576+
static int octep_sriov_configure(struct pci_dev *pdev, int num_vfs)
1577+
{
1578+
struct octep_device *oct = pci_get_drvdata(pdev);
1579+
int max_nvfs;
1580+
1581+
if (num_vfs == 0)
1582+
return octep_sriov_disable(oct);
1583+
1584+
max_nvfs = CFG_GET_MAX_VFS(oct->conf);
1585+
1586+
if (num_vfs > max_nvfs) {
1587+
dev_err(&pdev->dev, "Invalid VF count Max supported VFs = %d\n",
1588+
max_nvfs);
1589+
return -EINVAL;
1590+
}
1591+
1592+
return octep_sriov_enable(oct, num_vfs);
1593+
}
1594+
15171595
static struct pci_driver octep_driver = {
15181596
.name = OCTEP_DRV_NAME,
15191597
.id_table = octep_pci_id_tbl,
15201598
.probe = octep_probe,
15211599
.remove = octep_remove,
1600+
.sriov_configure = octep_sriov_configure,
15221601
};
15231602

15241603
/**

0 commit comments

Comments
 (0)