Skip to content

Commit

Permalink
bnxt_en: Add auxiliary driver support
Browse files Browse the repository at this point in the history 
Add auxiliary driver support.
An auxiliary device will be created if the hardware indicates
support for RDMA.
The bnxt_ulp_probe() function has been removed and a new
bnxt_rdma_aux_device_add() function has been added.
The bnxt_free_msix_vecs() and bnxt_req_msix_vecs() will now hold
the RTNL lock when they call the bnxt_close_nic()and bnxt_open_nic()
since the device close and open need to be protected under RTNL lock.
The operations between the bnxt_en and bnxt_re will be protected
using the en_ops_lock.
This will be used by the bnxt_re driver in a follow-on patch
to create ROCE interfaces.

Signed-off-by: Ajit Khaparde <ajit.khaparde@broadcom.com>
Reviewed-by: Andy Gospodarek <andrew.gospodarek@broadcom.com>
  • Loading branch information
@ajitkhaparde @intel-lab-lkp
ajitkhaparde authored and intel-lab-lkp committed Nov 4, 2022
1 parent ee6050c commit aadefb5
Show file tree
Hide file tree
Showing 4 changed files with 169 additions and 31 deletions.
8 changes: 6 additions & 2 deletions drivers/net/ethernet/broadcom/bnxt/bnxt.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13119,6 +13119,9 @@ static void bnxt_remove_one(struct pci_dev *pdev)
struct net_device *dev = pci_get_drvdata(pdev);
struct bnxt *bp = netdev_priv(dev);

bnxt_rdma_aux_device_uninit(bp->aux_dev);
bnxt_aux_dev_free(bp);

if (BNXT_PF(bp))
bnxt_sriov_disable(bp);

Expand Down Expand Up @@ -13716,11 +13719,13 @@ static int bnxt_init_one(struct pci_dev *pdev, const struct pci_device_id *ent)
devlink_port_type_eth_set(&bp->dl_port, bp->dev);
bnxt_dl_fw_reporters_create(bp);

bnxt_rdma_aux_device_init(bp);

bnxt_print_device_info(bp);

pci_save_state(pdev);
return 0;

return 0;
init_err_cleanup:
bnxt_dl_unregister(bp);
init_err_dl:
Expand Down Expand Up @@ -13764,7 +13769,6 @@ static void bnxt_shutdown(struct pci_dev *pdev)
if (netif_running(dev))
dev_close(dev);

bnxt_ulp_shutdown(bp);
bnxt_clear_int_mode(bp);
pci_disable_device(pdev);

Expand Down
8 changes: 8 additions & 0 deletions drivers/net/ethernet/broadcom/bnxt/bnxt.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@
#include <linux/interrupt.h>
#include <linux/rhashtable.h>
#include <linux/crash_dump.h>
#include <linux/auxiliary_bus.h>
#include <net/devlink.h>
#include <net/dst_metadata.h>
#include <net/xdp.h>
Expand Down Expand Up @@ -1622,6 +1623,12 @@ struct bnxt_fw_health {
#define BNXT_FW_RETRY 5
#define BNXT_FW_IF_RETRY 10

struct bnxt_aux_dev {
struct auxiliary_device aux_dev;
struct bnxt_en_dev *edev;
int id;
};

enum board_idx {
BCM57301,
BCM57302,
Expand Down Expand Up @@ -1843,6 +1850,7 @@ struct bnxt {
#define BNXT_CHIP_P4_PLUS(bp) \
(BNXT_CHIP_P4(bp) || BNXT_CHIP_P5(bp))

struct bnxt_aux_dev *aux_dev;
struct bnxt_en_dev *edev;

struct bnxt_napi **bnapi;
Expand Down
177 changes: 149 additions & 28 deletions drivers/net/ethernet/broadcom/bnxt/bnxt_ulp.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,32 +25,37 @@
#include "bnxt_hwrm.h"
#include "bnxt_ulp.h"

static DEFINE_IDA(bnxt_aux_dev_ids);

static int bnxt_register_dev(struct bnxt_en_dev *edev, unsigned int ulp_id,
struct bnxt_ulp_ops *ulp_ops, void *handle)
{
struct net_device *dev = edev->net;
struct bnxt *bp = netdev_priv(dev);
struct bnxt_ulp *ulp;
int rc = 0;

ASSERT_RTNL();
if (ulp_id >= BNXT_MAX_ULP)
return -EINVAL;

ulp = &edev->ulp_tbl[ulp_id];
if (rcu_access_pointer(ulp->ulp_ops)) {
netdev_err(bp->dev, "ulp id %d already registered\n", ulp_id);
return -EBUSY;
rc = -EBUSY;
goto exit;
}
if (ulp_id == BNXT_ROCE_ULP) {
unsigned int max_stat_ctxs;

max_stat_ctxs = bnxt_get_max_func_stat_ctxs(bp);
if (max_stat_ctxs <= BNXT_MIN_ROCE_STAT_CTXS ||
bp->cp_nr_rings == max_stat_ctxs)
return -ENOMEM;
bp->cp_nr_rings == max_stat_ctxs) {
rc = -ENOMEM;
goto exit;
}
}

atomic_set(&ulp->ref_count, 0);
atomic_set(&ulp->ref_count, 1);
ulp->handle = handle;
rcu_assign_pointer(ulp->ulp_ops, ulp_ops);

Expand All @@ -59,7 +64,8 @@ static int bnxt_register_dev(struct bnxt_en_dev *edev, unsigned int ulp_id,
bnxt_hwrm_vnic_cfg(bp, 0);
}

return 0;
exit:
return rc;
}

static int bnxt_unregister_dev(struct bnxt_en_dev *edev, unsigned int ulp_id)
Expand All @@ -69,10 +75,11 @@ static int bnxt_unregister_dev(struct bnxt_en_dev *edev, unsigned int ulp_id)
struct bnxt_ulp *ulp;
int i = 0;

ASSERT_RTNL();
if (ulp_id >= BNXT_MAX_ULP)
return -EINVAL;

edev->flags |= BNXT_EN_FLAG_ULP_STOPPED;

ulp = &edev->ulp_tbl[ulp_id];
if (!rcu_access_pointer(ulp->ulp_ops)) {
netdev_err(bp->dev, "ulp id %d not registered\n", ulp_id);
Expand Down Expand Up @@ -126,7 +133,6 @@ static int bnxt_req_msix_vecs(struct bnxt_en_dev *edev, unsigned int ulp_id,
int total_vecs;
int rc = 0;

ASSERT_RTNL();
if (ulp_id != BNXT_ROCE_ULP)
return -EINVAL;

Expand All @@ -149,15 +155,18 @@ static int bnxt_req_msix_vecs(struct bnxt_en_dev *edev, unsigned int ulp_id,
max_idx = min_t(int, bp->total_irqs, max_cp_rings);
idx = max_idx - avail_msix;
}

edev->ulp_tbl[ulp_id].msix_base = idx;
edev->ulp_tbl[ulp_id].msix_requested = avail_msix;
hw_resc = &bp->hw_resc;
total_vecs = idx + avail_msix;
if (bp->total_irqs < total_vecs ||
(BNXT_NEW_RM(bp) && hw_resc->resv_irqs < total_vecs)) {
if (netif_running(dev)) {
rtnl_lock();
bnxt_close_nic(bp, true, false);
rc = bnxt_open_nic(bp, true, false);
rtnl_unlock();
} else {
rc = bnxt_reserve_rings(bp, true);
}
Expand All @@ -184,7 +193,6 @@ static int bnxt_free_msix_vecs(struct bnxt_en_dev *edev, unsigned int ulp_id)
struct net_device *dev = edev->net;
struct bnxt *bp = netdev_priv(dev);

ASSERT_RTNL();
if (ulp_id != BNXT_ROCE_ULP)
return -EINVAL;

Expand All @@ -194,9 +202,12 @@ static int bnxt_free_msix_vecs(struct bnxt_en_dev *edev, unsigned int ulp_id)
edev->ulp_tbl[ulp_id].msix_requested = 0;
edev->flags &= ~BNXT_EN_FLAG_MSIX_REQUESTED;
if (netif_running(dev) && !(edev->flags & BNXT_EN_FLAG_ULP_STOPPED)) {
rtnl_lock();
bnxt_close_nic(bp, true, false);
bnxt_open_nic(bp, true, false);
rtnl_unlock();
}

return 0;
}

Expand Down Expand Up @@ -347,25 +358,6 @@ void bnxt_ulp_sriov_cfg(struct bnxt *bp, int num_vfs)
}
}

void bnxt_ulp_shutdown(struct bnxt *bp)
{
struct bnxt_en_dev *edev = bp->edev;
struct bnxt_ulp_ops *ops;
int i;

if (!edev)
return;

for (i = 0; i < BNXT_MAX_ULP; i++) {
struct bnxt_ulp *ulp = &edev->ulp_tbl[i];

ops = rtnl_dereference(ulp->ulp_ops);
if (!ops || !ops->ulp_shutdown)
continue;
ops->ulp_shutdown(ulp->handle);
}
}

void bnxt_ulp_irq_stop(struct bnxt *bp)
{
struct bnxt_en_dev *edev = bp->edev;
Expand Down Expand Up @@ -475,6 +467,135 @@ static const struct bnxt_en_ops bnxt_en_ops_tbl = {
.bnxt_register_fw_async_events = bnxt_register_async_events,
};

void bnxt_aux_dev_free(struct bnxt *bp)
{
kfree(bp->aux_dev);
bp->aux_dev = NULL;
}

static struct bnxt_aux_dev *bnxt_aux_dev_alloc(struct bnxt *bp)
{
struct bnxt_aux_dev *bnxt_adev;

bnxt_adev = kzalloc(sizeof(*bnxt_adev), GFP_KERNEL);
if (!bnxt_adev)
return ERR_PTR(-ENOMEM);

return bnxt_adev;
}

void bnxt_rdma_aux_device_uninit(struct bnxt_aux_dev *bnxt_adev)
{
struct auxiliary_device *adev;

if (IS_ERR_OR_NULL(bnxt_adev))
return;

adev = &bnxt_adev->aux_dev;
auxiliary_device_delete(adev);
auxiliary_device_uninit(adev);
if (bnxt_adev->id >= 0)
ida_free(&bnxt_aux_dev_ids, bnxt_adev->id);
}

void bnxt_rdma_aux_device_init(struct bnxt *bp)
{
struct net_device *dev = bp->dev;
int rc;

if (bp->flags & BNXT_FLAG_ROCE_CAP) {
bp->aux_dev = bnxt_aux_dev_alloc(bp);
if (IS_ERR_OR_NULL(bp->aux_dev)) {
netdev_warn(dev, "Failed to init auxiliary device for ROCE\n");
goto skip_aux_init;
}

bp->aux_dev->id = ida_alloc(&bnxt_aux_dev_ids, GFP_KERNEL);
if (bp->aux_dev->id < 0) {
netdev_warn(dev, "ida alloc failed for ROCE auxiliary device\n");
bnxt_aux_dev_free(bp);
goto skip_aux_init;
}

/* If aux bus init fails, continue with netdev init. */
rc = bnxt_rdma_aux_device_add(bp);
if (rc) {
netdev_warn(dev, "Failed to add auxiliary device for ROCE\n");
ida_free(&bnxt_aux_dev_ids, bp->aux_dev->id);
bnxt_aux_dev_free(bp);
}
}
skip_aux_init:
return;
}

void bnxt_aux_dev_release(struct device *dev)
{
struct bnxt_aux_dev *bnxt_adev =
container_of(dev, struct bnxt_aux_dev, aux_dev.dev);
struct bnxt *bp = netdev_priv(bnxt_adev->edev->net);

bnxt_adev->edev->en_ops = NULL;
kfree(bnxt_adev->edev);
bnxt_adev->edev = NULL;
bp->edev = NULL;
}

static inline void bnxt_set_edev_info(struct bnxt_en_dev *edev, struct bnxt *bp)
{
edev->en_ops = &bnxt_en_ops_tbl;
edev->net = bp->dev;
edev->pdev = bp->pdev;
edev->l2_db_size = bp->db_size;
edev->l2_db_size_nc = bp->db_size;

if (bp->flags & BNXT_FLAG_ROCEV1_CAP)
edev->flags |= BNXT_EN_FLAG_ROCEV1_CAP;
if (bp->flags & BNXT_FLAG_ROCEV2_CAP)
edev->flags |= BNXT_EN_FLAG_ROCEV2_CAP;
}

int bnxt_rdma_aux_device_add(struct bnxt *bp)
{
struct bnxt_aux_dev *bnxt_adev = bp->aux_dev;
struct bnxt_en_dev *edev = bnxt_adev->edev;
struct auxiliary_device *aux_dev;
int ret;

aux_dev = &bnxt_adev->aux_dev;
aux_dev->id = bnxt_adev->id;
aux_dev->name = "rdma";
aux_dev->dev.parent = &bp->pdev->dev;
aux_dev->dev.release = bnxt_aux_dev_release;

if (!edev) {
edev = kzalloc(sizeof(*edev), GFP_KERNEL);
if (!edev)
return -ENOMEM;
}

bnxt_set_edev_info(edev, bp);
bnxt_adev->edev = edev;
bp->edev = edev;

ret = auxiliary_device_init(aux_dev);
if (ret) {
kfree(edev);
kfree(bnxt_adev);
return ret;
}

ret = auxiliary_device_add(aux_dev);
if (ret) {
kfree(edev);
kfree(bnxt_adev);
auxiliary_device_uninit(aux_dev);
return ret;
}

return 0;
}

struct bnxt_en_dev *bnxt_ulp_probe(struct net_device *dev)
{
struct bnxt *bp = netdev_priv(dev);
Expand Down
7 changes: 6 additions & 1 deletion drivers/net/ethernet/broadcom/bnxt/bnxt_ulp.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@
#define BNXT_MIN_ROCE_STAT_CTXS 1

struct hwrm_async_event_cmpl;
struct bnxt_aux_dev;
struct bnxt;

struct bnxt_msix_entry {
Expand Down Expand Up @@ -102,10 +103,14 @@ int bnxt_get_ulp_stat_ctxs(struct bnxt *bp);
void bnxt_ulp_stop(struct bnxt *bp);
void bnxt_ulp_start(struct bnxt *bp, int err);
void bnxt_ulp_sriov_cfg(struct bnxt *bp, int num_vfs);
void bnxt_ulp_shutdown(struct bnxt *bp);
void bnxt_ulp_irq_stop(struct bnxt *bp);
void bnxt_ulp_irq_restart(struct bnxt *bp, int err);
void bnxt_ulp_async_events(struct bnxt *bp, struct hwrm_async_event_cmpl *cmpl);
void bnxt_aux_dev_release(struct device *dev);
int bnxt_rdma_aux_device_add(struct bnxt *bp);
void bnxt_rdma_aux_device_uninit(struct bnxt_aux_dev *bnxt_adev);
void bnxt_rdma_aux_device_init(struct bnxt *bp);
void bnxt_aux_dev_free(struct bnxt *bp);
struct bnxt_en_dev *bnxt_ulp_probe(struct net_device *dev);

#endif

0 comments on commit aadefb5

Please sign in to comment.