Skip to content

Commit

Permalink
bnxt_en: Add auxiliary driver support
Browse files Browse the repository at this point in the history 
Add auxiliary driver support.
An auxiliary device will be created if the hardware indicates
support for RDMA.
The bnxt_ulp_probe() function has been removed and a new
bnxt_rdma_aux_device_add() function has been added.
The bnxt_free_msix_vecs() and bnxt_req_msix_vecs() will now hold
the RTNL lock when they call the bnxt_close_nic()and bnxt_open_nic()
since the device close and open need to be protected under RTNL lock.
The operations between the bnxt_en and bnxt_re will be protected
using the en_ops_lock.
This will be used by the bnxt_re driver in a follow-on patch
to create ROCE interfaces.

Signed-off-by: Ajit Khaparde <ajit.khaparde@broadcom.com>
Reviewed-by: Andy Gospodarek <andrew.gospodarek@broadcom.com>
Reviewed-by: Selvin Xavier <selvin.xavier@broadcom.com>
Reviewed-by: Leon Romanovsky <leonro@nvidia.com>
  • Loading branch information
@ajitkhaparde @intel-lab-lkp
ajitkhaparde authored and intel-lab-lkp committed Feb 1, 2023
1 parent 074dd3b commit 896eba0
Show file tree
Hide file tree
Showing 5 changed files with 128 additions and 26 deletions.
1 change: 1 addition & 0 deletions drivers/net/ethernet/broadcom/Kconfig
View file
Expand Up @@ -213,6 +213,7 @@ config BNXT
select NET_DEVLINK
select PAGE_POOL
select DIMLIB
select AUXILIARY_BUS
help
This driver supports Broadcom NetXtreme-C/E 10/25/40/50 gigabit
Ethernet cards. To compile this driver as a module, choose M here:
Expand Down
7 changes: 5 additions & 2 deletions drivers/net/ethernet/broadcom/bnxt/bnxt.c
View file
Expand Up @@ -13181,6 +13181,8 @@ static void bnxt_remove_one(struct pci_dev *pdev)
if (BNXT_PF(bp))
bnxt_sriov_disable(bp);

bnxt_rdma_aux_device_uninit(bp);

bnxt_ptp_clear(bp);
pci_disable_pcie_error_reporting(pdev);
unregister_netdev(dev);
Expand Down Expand Up @@ -13776,11 +13778,13 @@ static int bnxt_init_one(struct pci_dev *pdev, const struct pci_device_id *ent)

bnxt_dl_fw_reporters_create(bp);

bnxt_rdma_aux_device_init(bp);

bnxt_print_device_info(bp);

pci_save_state(pdev);
return 0;

return 0;
init_err_cleanup:
bnxt_dl_unregister(bp);
init_err_dl:
Expand Down Expand Up @@ -13824,7 +13828,6 @@ static void bnxt_shutdown(struct pci_dev *pdev)
if (netif_running(dev))
dev_close(dev);

bnxt_ulp_shutdown(bp);
bnxt_clear_int_mode(bp);
pci_disable_device(pdev);

Expand Down
8 changes: 8 additions & 0 deletions drivers/net/ethernet/broadcom/bnxt/bnxt.h
View file
Expand Up @@ -24,6 +24,7 @@
#include <linux/interrupt.h>
#include <linux/rhashtable.h>
#include <linux/crash_dump.h>
#include <linux/auxiliary_bus.h>
#include <net/devlink.h>
#include <net/dst_metadata.h>
#include <net/xdp.h>
Expand Down Expand Up @@ -1631,6 +1632,12 @@ struct bnxt_fw_health {
#define BNXT_FW_IF_RETRY 10
#define BNXT_FW_SLOT_RESET_RETRY 4

struct bnxt_aux_priv {
struct auxiliary_device aux_dev;
struct bnxt_en_dev *edev;
int id;
};

enum board_idx {
BCM57301,
BCM57302,
Expand Down Expand Up @@ -1852,6 +1859,7 @@ struct bnxt {
#define BNXT_CHIP_P4_PLUS(bp) \
(BNXT_CHIP_P4(bp) || BNXT_CHIP_P5(bp))

struct bnxt_aux_priv *aux_priv;
struct bnxt_en_dev *edev;

struct bnxt_napi **bnapi;
Expand Down
135 changes: 112 additions & 23 deletions drivers/net/ethernet/broadcom/bnxt/bnxt_ulp.c
View file
Expand Up @@ -25,14 +25,15 @@
#include "bnxt_hwrm.h"
#include "bnxt_ulp.h"

static DEFINE_IDA(bnxt_aux_dev_ids);

static int bnxt_register_dev(struct bnxt_en_dev *edev, unsigned int ulp_id,
struct bnxt_ulp_ops *ulp_ops, void *handle)
{
struct net_device *dev = edev->net;
struct bnxt *bp = netdev_priv(dev);
struct bnxt_ulp *ulp;

ASSERT_RTNL();
if (ulp_id >= BNXT_MAX_ULP)
return -EINVAL;

Expand Down Expand Up @@ -69,10 +70,11 @@ static int bnxt_unregister_dev(struct bnxt_en_dev *edev, unsigned int ulp_id)
struct bnxt_ulp *ulp;
int i = 0;

ASSERT_RTNL();
if (ulp_id >= BNXT_MAX_ULP)
return -EINVAL;

edev->flags |= BNXT_EN_FLAG_ULP_STOPPED;

ulp = &edev->ulp_tbl[ulp_id];
if (!rcu_access_pointer(ulp->ulp_ops)) {
netdev_err(bp->dev, "ulp id %d not registered\n", ulp_id);
Expand Down Expand Up @@ -126,7 +128,6 @@ static int bnxt_req_msix_vecs(struct bnxt_en_dev *edev, unsigned int ulp_id,
int total_vecs;
int rc = 0;

ASSERT_RTNL();
if (ulp_id != BNXT_ROCE_ULP)
return -EINVAL;

Expand All @@ -153,6 +154,7 @@ static int bnxt_req_msix_vecs(struct bnxt_en_dev *edev, unsigned int ulp_id,
edev->ulp_tbl[ulp_id].msix_requested = avail_msix;
hw_resc = &bp->hw_resc;
total_vecs = idx + avail_msix;
rtnl_lock();
if (bp->total_irqs < total_vecs ||
(BNXT_NEW_RM(bp) && hw_resc->resv_irqs < total_vecs)) {
if (netif_running(dev)) {
Expand All @@ -162,6 +164,7 @@ static int bnxt_req_msix_vecs(struct bnxt_en_dev *edev, unsigned int ulp_id,
rc = bnxt_reserve_rings(bp, true);
}
}
rtnl_unlock();
if (rc) {
edev->ulp_tbl[ulp_id].msix_requested = 0;
return -EAGAIN;
Expand All @@ -184,7 +187,6 @@ static int bnxt_free_msix_vecs(struct bnxt_en_dev *edev, unsigned int ulp_id)
struct net_device *dev = edev->net;
struct bnxt *bp = netdev_priv(dev);

ASSERT_RTNL();
if (ulp_id != BNXT_ROCE_ULP)
return -EINVAL;

Expand All @@ -193,10 +195,13 @@ static int bnxt_free_msix_vecs(struct bnxt_en_dev *edev, unsigned int ulp_id)

edev->ulp_tbl[ulp_id].msix_requested = 0;
edev->flags &= ~BNXT_EN_FLAG_MSIX_REQUESTED;
rtnl_lock();
if (netif_running(dev) && !(edev->flags & BNXT_EN_FLAG_ULP_STOPPED)) {
bnxt_close_nic(bp, true, false);
bnxt_open_nic(bp, true, false);
}
rtnl_unlock();

return 0;
}

Expand Down Expand Up @@ -347,25 +352,6 @@ void bnxt_ulp_sriov_cfg(struct bnxt *bp, int num_vfs)
}
}

void bnxt_ulp_shutdown(struct bnxt *bp)
{
struct bnxt_en_dev *edev = bp->edev;
struct bnxt_ulp_ops *ops;
int i;

if (!edev)
return;

for (i = 0; i < BNXT_MAX_ULP; i++) {
struct bnxt_ulp *ulp = &edev->ulp_tbl[i];

ops = rtnl_dereference(ulp->ulp_ops);
if (!ops || !ops->ulp_shutdown)
continue;
ops->ulp_shutdown(ulp->handle);
}
}

void bnxt_ulp_irq_stop(struct bnxt *bp)
{
struct bnxt_en_dev *edev = bp->edev;
Expand Down Expand Up @@ -475,6 +461,109 @@ static const struct bnxt_en_ops bnxt_en_ops_tbl = {
.bnxt_register_fw_async_events = bnxt_register_async_events,
};

void bnxt_rdma_aux_device_uninit(struct bnxt *bp)
{
struct bnxt_aux_priv *aux_priv;
struct auxiliary_device *adev;

/* Skip if no auxiliary device init was done. */
if (!(bp->flags & BNXT_FLAG_ROCE_CAP))
return;

aux_priv = bp->aux_priv;
adev = &aux_priv->aux_dev;
auxiliary_device_delete(adev);
auxiliary_device_uninit(adev);
}

static void bnxt_aux_dev_release(struct device *dev)
{
struct bnxt_aux_priv *aux_priv =
container_of(dev, struct bnxt_aux_priv, aux_dev.dev);
struct bnxt *bp = netdev_priv(aux_priv->edev->net);

ida_free(&bnxt_aux_dev_ids, aux_priv->id);
kfree(aux_priv->edev);
kfree(aux_priv);
}

static void bnxt_set_edev_info(struct bnxt_en_dev *edev, struct bnxt *bp)
{
edev->en_ops = &bnxt_en_ops_tbl;
edev->net = bp->dev;
edev->pdev = bp->pdev;
edev->l2_db_size = bp->db_size;
edev->l2_db_size_nc = bp->db_size;

if (bp->flags & BNXT_FLAG_ROCEV1_CAP)
edev->flags |= BNXT_EN_FLAG_ROCEV1_CAP;
if (bp->flags & BNXT_FLAG_ROCEV2_CAP)
edev->flags |= BNXT_EN_FLAG_ROCEV2_CAP;
}

void bnxt_rdma_aux_device_init(struct bnxt *bp)
{
struct auxiliary_device *aux_dev;
struct bnxt_aux_priv *aux_priv;
struct bnxt_en_dev *edev;
int rc;

if (!(bp->flags & BNXT_FLAG_ROCE_CAP))
return;

bp->aux_priv = kzalloc(sizeof(*bp->aux_priv), GFP_KERNEL);
if (!bp->aux_priv)
goto exit;

bp->aux_priv->id = ida_alloc(&bnxt_aux_dev_ids, GFP_KERNEL);
if (bp->aux_priv->id < 0) {
netdev_warn(bp->dev,
"ida alloc failed for ROCE auxiliary device\n");
kfree(bp->aux_priv);
goto exit;
}

aux_priv = bp->aux_priv;
aux_dev = &aux_priv->aux_dev;
aux_dev->id = aux_priv->id;
aux_dev->name = "rdma";
aux_dev->dev.parent = &bp->pdev->dev;
aux_dev->dev.release = bnxt_aux_dev_release;

rc = auxiliary_device_init(aux_dev);
if (rc) {
ida_free(&bnxt_aux_dev_ids, bp->aux_priv->id);
kfree(bp->aux_priv);
goto exit;
}

/* From this point, all cleanup will happen via the .release callback &
* any error unwinding will need to include a call to
* auxiliary_device_uninit.
*/
edev = kzalloc(sizeof(*edev), GFP_KERNEL);
if (!edev)
goto aux_dev_uninit;

aux_priv->edev = edev;
bp->edev = edev;
bnxt_set_edev_info(edev, bp);

rc = auxiliary_device_add(aux_dev);
if (rc) {
netdev_warn(bp->dev,
"Failed to add auxiliary device for ROCE\n");
goto aux_dev_uninit;
}

return;

aux_dev_uninit:
auxiliary_device_uninit(aux_dev);
exit:
bp->flags &= ~BNXT_FLAG_ROCE_CAP;
}

struct bnxt_en_dev *bnxt_ulp_probe(struct net_device *dev)
{
struct bnxt *bp = netdev_priv(dev);
Expand Down
3 changes: 2 additions & 1 deletion drivers/net/ethernet/broadcom/bnxt/bnxt_ulp.h
View file
Expand Up @@ -102,10 +102,11 @@ int bnxt_get_ulp_stat_ctxs(struct bnxt *bp);
void bnxt_ulp_stop(struct bnxt *bp);
void bnxt_ulp_start(struct bnxt *bp, int err);
void bnxt_ulp_sriov_cfg(struct bnxt *bp, int num_vfs);
void bnxt_ulp_shutdown(struct bnxt *bp);
void bnxt_ulp_irq_stop(struct bnxt *bp);
void bnxt_ulp_irq_restart(struct bnxt *bp, int err);
void bnxt_ulp_async_events(struct bnxt *bp, struct hwrm_async_event_cmpl *cmpl);
void bnxt_rdma_aux_device_uninit(struct bnxt *bp);
void bnxt_rdma_aux_device_init(struct bnxt *bp);
struct bnxt_en_dev *bnxt_ulp_probe(struct net_device *dev);

#endif

0 comments on commit 896eba0

Please sign in to comment.