Skip to content

Commit 0fbc81b

Browse files
Hariprasad Shenaidavem330
Hariprasad Shenai
authored and
davem330
committed
chcr/cxgb4i/cxgbit/RDMA/cxgb4: Allocate resources dynamically for all cxgb4 ULD's
Allocate resources dynamically to cxgb4's Upper layer driver's(ULD) like cxgbit, iw_cxgb4 and cxgb4i. Allocate resources when they register with cxgb4 driver and free them while unregistering. All the queues and the interrupts for them will be allocated during ULD probe only and freed during remove. Signed-off-by: Hariprasad Shenai <hariprasad@chelsio.com> Signed-off-by: David S. Miller <davem@davemloft.net>
1 parent e8bc8f9 commit 0fbc81b

File tree

10 files changed

+385
-694
lines changed

10 files changed

+385
-694
lines changed

‎drivers/crypto/chelsio/chcr_core.c‎

Lines changed: 4 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -39,12 +39,10 @@ static chcr_handler_func work_handlers[NUM_CPL_CMDS] = {
3939
[CPL_FW6_PLD] = cpl_fw6_pld_handler,
4040
};
4141

42-
static struct cxgb4_pci_uld_info chcr_uld_info = {
42+
static struct cxgb4_uld_info chcr_uld_info = {
4343
.name = DRV_MODULE_NAME,
44-
.nrxq = 4,
44+
.nrxq = MAX_ULD_QSETS,
4545
.rxq_size = 1024,
46-
.nciq = 0,
47-
.ciq_size = 0,
4846
.add = chcr_uld_add,
4947
.state_change = chcr_uld_state_change,
5048
.rx_handler = chcr_uld_rx_handler,
@@ -205,7 +203,7 @@ static int chcr_uld_state_change(void *handle, enum cxgb4_state state)
205203

206204
static int __init chcr_crypto_init(void)
207205
{
208-
if (cxgb4_register_pci_uld(CXGB4_PCI_ULD1, &chcr_uld_info)) {
206+
if (cxgb4_register_uld(CXGB4_ULD_CRYPTO, &chcr_uld_info)) {
209207
pr_err("ULD register fail: No chcr crypto support in cxgb4");
210208
return -1;
211209
}
@@ -228,7 +226,7 @@ static void __exit chcr_crypto_exit(void)
228226
kfree(u_ctx);
229227
}
230228
mutex_unlock(&dev_mutex);
231-
cxgb4_unregister_pci_uld(CXGB4_PCI_ULD1);
229+
cxgb4_unregister_uld(CXGB4_ULD_CRYPTO);
232230
}
233231

234232
module_init(chcr_crypto_init);

‎drivers/infiniband/hw/cxgb4/device.c‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1475,6 +1475,10 @@ static int c4iw_uld_control(void *handle, enum cxgb4_control control, ...)
14751475

14761476
static struct cxgb4_uld_info c4iw_uld_info = {
14771477
.name = DRV_NAME,
1478+
.nrxq = MAX_ULD_QSETS,
1479+
.rxq_size = 511,
1480+
.ciq = true,
1481+
.lro = false,
14781482
.add = c4iw_uld_add,
14791483
.rx_handler = c4iw_uld_rx_handler,
14801484
.state_change = c4iw_uld_state_change,

‎drivers/net/ethernet/chelsio/cxgb4/cxgb4.h‎

Lines changed: 20 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -437,11 +437,6 @@ enum {
437437
MAX_ETH_QSETS = 32, /* # of Ethernet Tx/Rx queue sets */
438438
MAX_OFLD_QSETS = 16, /* # of offload Tx, iscsi Rx queue sets */
439439
MAX_CTRL_QUEUES = NCHAN, /* # of control Tx queues */
440-
MAX_RDMA_QUEUES = NCHAN, /* # of streaming RDMA Rx queues */
441-
MAX_RDMA_CIQS = 32, /* # of RDMA concentrator IQs */
442-
443-
/* # of streaming iSCSIT Rx queues */
444-
MAX_ISCSIT_QUEUES = MAX_OFLD_QSETS,
445440
};
446441

447442
enum {
@@ -458,8 +453,7 @@ enum {
458453
enum {
459454
INGQ_EXTRAS = 2, /* firmware event queue and */
460455
/* forwarded interrupts */
461-
MAX_INGQ = MAX_ETH_QSETS + MAX_OFLD_QSETS + MAX_RDMA_QUEUES +
462-
MAX_RDMA_CIQS + MAX_ISCSIT_QUEUES + INGQ_EXTRAS,
456+
MAX_INGQ = MAX_ETH_QSETS + INGQ_EXTRAS,
463457
};
464458

465459
struct adapter;
@@ -704,10 +698,6 @@ struct sge {
704698
struct sge_ctrl_txq ctrlq[MAX_CTRL_QUEUES];
705699

706700
struct sge_eth_rxq ethrxq[MAX_ETH_QSETS];
707-
struct sge_ofld_rxq iscsirxq[MAX_OFLD_QSETS];
708-
struct sge_ofld_rxq iscsitrxq[MAX_ISCSIT_QUEUES];
709-
struct sge_ofld_rxq rdmarxq[MAX_RDMA_QUEUES];
710-
struct sge_ofld_rxq rdmaciq[MAX_RDMA_CIQS];
711701
struct sge_rspq fw_evtq ____cacheline_aligned_in_smp;
712702
struct sge_uld_rxq_info **uld_rxq_info;
713703

@@ -717,15 +707,8 @@ struct sge {
717707
u16 max_ethqsets; /* # of available Ethernet queue sets */
718708
u16 ethqsets; /* # of active Ethernet queue sets */
719709
u16 ethtxq_rover; /* Tx queue to clean up next */
720-
u16 iscsiqsets; /* # of active iSCSI queue sets */
721-
u16 niscsitq; /* # of available iSCST Rx queues */
722-
u16 rdmaqs; /* # of available RDMA Rx queues */
723-
u16 rdmaciqs; /* # of available RDMA concentrator IQs */
710+
u16 ofldqsets; /* # of active ofld queue sets */
724711
u16 nqs_per_uld; /* # of Rx queues per ULD */
725-
u16 iscsi_rxq[MAX_OFLD_QSETS];
726-
u16 iscsit_rxq[MAX_ISCSIT_QUEUES];
727-
u16 rdma_rxq[MAX_RDMA_QUEUES];
728-
u16 rdma_ciq[MAX_RDMA_CIQS];
729712
u16 timer_val[SGE_NTIMERS];
730713
u8 counter_val[SGE_NCOUNTERS];
731714
u32 fl_pg_order; /* large page allocation size */
@@ -749,10 +732,7 @@ struct sge {
749732
};
750733

751734
#define for_each_ethrxq(sge, i) for (i = 0; i < (sge)->ethqsets; i++)
752-
#define for_each_iscsirxq(sge, i) for (i = 0; i < (sge)->iscsiqsets; i++)
753-
#define for_each_iscsitrxq(sge, i) for (i = 0; i < (sge)->niscsitq; i++)
754-
#define for_each_rdmarxq(sge, i) for (i = 0; i < (sge)->rdmaqs; i++)
755-
#define for_each_rdmaciq(sge, i) for (i = 0; i < (sge)->rdmaciqs; i++)
735+
#define for_each_ofldtxq(sge, i) for (i = 0; i < (sge)->ofldqsets; i++)
756736

757737
struct l2t_data;
758738

@@ -786,6 +766,7 @@ struct uld_msix_bmap {
786766
struct uld_msix_info {
787767
unsigned short vec;
788768
char desc[IFNAMSIZ + 10];
769+
unsigned int idx;
789770
};
790771

791772
struct vf_info {
@@ -818,7 +799,7 @@ struct adapter {
818799
} msix_info[MAX_INGQ + 1];
819800
struct uld_msix_info *msix_info_ulds; /* msix info for uld's */
820801
struct uld_msix_bmap msix_bmap_ulds; /* msix bitmap for all uld */
821-
unsigned int msi_idx;
802+
int msi_idx;
822803

823804
struct doorbell_stats db_stats;
824805
struct sge sge;
@@ -836,9 +817,10 @@ struct adapter {
836817
unsigned int clipt_start;
837818
unsigned int clipt_end;
838819
struct clip_tbl *clipt;
839-
struct cxgb4_pci_uld_info *uld;
820+
struct cxgb4_uld_info *uld;
840821
void *uld_handle[CXGB4_ULD_MAX];
841822
unsigned int num_uld;
823+
unsigned int num_ofld_uld;
842824
struct list_head list_node;
843825
struct list_head rcu_node;
844826
struct list_head mac_hlist; /* list of MAC addresses in MPS Hash */
@@ -858,6 +840,8 @@ struct adapter {
858840
#define T4_OS_LOG_MBOX_CMDS 256
859841
struct mbox_cmd_log *mbox_log;
860842

843+
struct mutex uld_mutex;
844+
861845
struct dentry *debugfs_root;
862846
bool use_bd; /* Use SGE Back Door intfc for reading SGE Contexts */
863847
bool trace_rss; /* 1 implies that different RSS flit per filter is
@@ -1051,6 +1035,11 @@ static inline int is_pci_uld(const struct adapter *adap)
10511035
return adap->params.crypto;
10521036
}
10531037

1038+
static inline int is_uld(const struct adapter *adap)
1039+
{
1040+
return (adap->params.offload || adap->params.crypto);
1041+
}
1042+
10541043
static inline u32 t4_read_reg(struct adapter *adap, u32 reg_addr)
10551044
{
10561045
return readl(adap->regs + reg_addr);
@@ -1277,6 +1266,8 @@ int t4_sge_alloc_eth_txq(struct adapter *adap, struct sge_eth_txq *txq,
12771266
int t4_sge_alloc_ctrl_txq(struct adapter *adap, struct sge_ctrl_txq *txq,
12781267
struct net_device *dev, unsigned int iqid,
12791268
unsigned int cmplqid);
1269+
int t4_sge_mod_ctrl_txq(struct adapter *adap, unsigned int eqid,
1270+
unsigned int cmplqid);
12801271
int t4_sge_alloc_ofld_txq(struct adapter *adap, struct sge_ofld_txq *txq,
12811272
struct net_device *dev, unsigned int iqid);
12821273
irqreturn_t t4_sge_intr_msix(int irq, void *cookie);
@@ -1635,7 +1626,9 @@ void t4_idma_monitor(struct adapter *adapter,
16351626
int hz, int ticks);
16361627
int t4_set_vf_mac_acl(struct adapter *adapter, unsigned int vf,
16371628
unsigned int naddr, u8 *addr);
1638-
void uld_mem_free(struct adapter *adap);
1639-
int uld_mem_alloc(struct adapter *adap);
1629+
void t4_uld_mem_free(struct adapter *adap);
1630+
int t4_uld_mem_alloc(struct adapter *adap);
1631+
void t4_uld_clean_up(struct adapter *adap);
1632+
void t4_register_netevent_notifier(void);
16401633
void free_rspq_fl(struct adapter *adap, struct sge_rspq *rq, struct sge_fl *fl);
16411634
#endif /* __CXGB4_H__ */

‎drivers/net/ethernet/chelsio/cxgb4/cxgb4_debugfs.c‎

Lines changed: 8 additions & 119 deletions
Original file line numberDiff line numberDiff line change
@@ -2432,17 +2432,11 @@ static int sge_qinfo_show(struct seq_file *seq, void *v)
24322432
{
24332433
struct adapter *adap = seq->private;
24342434
int eth_entries = DIV_ROUND_UP(adap->sge.ethqsets, 4);
2435-
int iscsi_entries = DIV_ROUND_UP(adap->sge.iscsiqsets, 4);
2436-
int iscsit_entries = DIV_ROUND_UP(adap->sge.niscsitq, 4);
2437-
int rdma_entries = DIV_ROUND_UP(adap->sge.rdmaqs, 4);
2438-
int ciq_entries = DIV_ROUND_UP(adap->sge.rdmaciqs, 4);
2435+
int ofld_entries = DIV_ROUND_UP(adap->sge.ofldqsets, 4);
24392436
int ctrl_entries = DIV_ROUND_UP(MAX_CTRL_QUEUES, 4);
24402437
int i, r = (uintptr_t)v - 1;
2441-
int iscsi_idx = r - eth_entries;
2442-
int iscsit_idx = iscsi_idx - iscsi_entries;
2443-
int rdma_idx = iscsit_idx - iscsit_entries;
2444-
int ciq_idx = rdma_idx - rdma_entries;
2445-
int ctrl_idx = ciq_idx - ciq_entries;
2438+
int ofld_idx = r - eth_entries;
2439+
int ctrl_idx = ofld_idx - ofld_entries;
24462440
int fq_idx = ctrl_idx - ctrl_entries;
24472441

24482442
if (r)
@@ -2518,119 +2512,17 @@ do { \
25182512
RL("FLLow:", fl.low);
25192513
RL("FLStarving:", fl.starving);
25202514

2521-
} else if (iscsi_idx < iscsi_entries) {
2522-
const struct sge_ofld_rxq *rx =
2523-
&adap->sge.iscsirxq[iscsi_idx * 4];
2515+
} else if (ofld_idx < ofld_entries) {
25242516
const struct sge_ofld_txq *tx =
2525-
&adap->sge.ofldtxq[iscsi_idx * 4];
2526-
int n = min(4, adap->sge.iscsiqsets - 4 * iscsi_idx);
2517+
&adap->sge.ofldtxq[ofld_idx * 4];
2518+
int n = min(4, adap->sge.ofldqsets - 4 * ofld_idx);
25272519

2528-
S("QType:", "iSCSI");
2520+
S("QType:", "OFLD-Txq");
25292521
T("TxQ ID:", q.cntxt_id);
25302522
T("TxQ size:", q.size);
25312523
T("TxQ inuse:", q.in_use);
25322524
T("TxQ CIDX:", q.cidx);
25332525
T("TxQ PIDX:", q.pidx);
2534-
R("RspQ ID:", rspq.abs_id);
2535-
R("RspQ size:", rspq.size);
2536-
R("RspQE size:", rspq.iqe_len);
2537-
R("RspQ CIDX:", rspq.cidx);
2538-
R("RspQ Gen:", rspq.gen);
2539-
S3("u", "Intr delay:", qtimer_val(adap, &rx[i].rspq));
2540-
S3("u", "Intr pktcnt:",
2541-
adap->sge.counter_val[rx[i].rspq.pktcnt_idx]);
2542-
R("FL ID:", fl.cntxt_id);
2543-
R("FL size:", fl.size - 8);
2544-
R("FL pend:", fl.pend_cred);
2545-
R("FL avail:", fl.avail);
2546-
R("FL PIDX:", fl.pidx);
2547-
R("FL CIDX:", fl.cidx);
2548-
RL("RxPackets:", stats.pkts);
2549-
RL("RxImmPkts:", stats.imm);
2550-
RL("RxNoMem:", stats.nomem);
2551-
RL("FLAllocErr:", fl.alloc_failed);
2552-
RL("FLLrgAlcErr:", fl.large_alloc_failed);
2553-
RL("FLMapErr:", fl.mapping_err);
2554-
RL("FLLow:", fl.low);
2555-
RL("FLStarving:", fl.starving);
2556-
2557-
} else if (iscsit_idx < iscsit_entries) {
2558-
const struct sge_ofld_rxq *rx =
2559-
&adap->sge.iscsitrxq[iscsit_idx * 4];
2560-
int n = min(4, adap->sge.niscsitq - 4 * iscsit_idx);
2561-
2562-
S("QType:", "iSCSIT");
2563-
R("RspQ ID:", rspq.abs_id);
2564-
R("RspQ size:", rspq.size);
2565-
R("RspQE size:", rspq.iqe_len);
2566-
R("RspQ CIDX:", rspq.cidx);
2567-
R("RspQ Gen:", rspq.gen);
2568-
S3("u", "Intr delay:", qtimer_val(adap, &rx[i].rspq));
2569-
S3("u", "Intr pktcnt:",
2570-
adap->sge.counter_val[rx[i].rspq.pktcnt_idx]);
2571-
R("FL ID:", fl.cntxt_id);
2572-
R("FL size:", fl.size - 8);
2573-
R("FL pend:", fl.pend_cred);
2574-
R("FL avail:", fl.avail);
2575-
R("FL PIDX:", fl.pidx);
2576-
R("FL CIDX:", fl.cidx);
2577-
RL("RxPackets:", stats.pkts);
2578-
RL("RxImmPkts:", stats.imm);
2579-
RL("RxNoMem:", stats.nomem);
2580-
RL("FLAllocErr:", fl.alloc_failed);
2581-
RL("FLLrgAlcErr:", fl.large_alloc_failed);
2582-
RL("FLMapErr:", fl.mapping_err);
2583-
RL("FLLow:", fl.low);
2584-
RL("FLStarving:", fl.starving);
2585-
2586-
} else if (rdma_idx < rdma_entries) {
2587-
const struct sge_ofld_rxq *rx =
2588-
&adap->sge.rdmarxq[rdma_idx * 4];
2589-
int n = min(4, adap->sge.rdmaqs - 4 * rdma_idx);
2590-
2591-
S("QType:", "RDMA-CPL");
2592-
S("Interface:",
2593-
rx[i].rspq.netdev ? rx[i].rspq.netdev->name : "N/A");
2594-
R("RspQ ID:", rspq.abs_id);
2595-
R("RspQ size:", rspq.size);
2596-
R("RspQE size:", rspq.iqe_len);
2597-
R("RspQ CIDX:", rspq.cidx);
2598-
R("RspQ Gen:", rspq.gen);
2599-
S3("u", "Intr delay:", qtimer_val(adap, &rx[i].rspq));
2600-
S3("u", "Intr pktcnt:",
2601-
adap->sge.counter_val[rx[i].rspq.pktcnt_idx]);
2602-
R("FL ID:", fl.cntxt_id);
2603-
R("FL size:", fl.size - 8);
2604-
R("FL pend:", fl.pend_cred);
2605-
R("FL avail:", fl.avail);
2606-
R("FL PIDX:", fl.pidx);
2607-
R("FL CIDX:", fl.cidx);
2608-
RL("RxPackets:", stats.pkts);
2609-
RL("RxImmPkts:", stats.imm);
2610-
RL("RxNoMem:", stats.nomem);
2611-
RL("FLAllocErr:", fl.alloc_failed);
2612-
RL("FLLrgAlcErr:", fl.large_alloc_failed);
2613-
RL("FLMapErr:", fl.mapping_err);
2614-
RL("FLLow:", fl.low);
2615-
RL("FLStarving:", fl.starving);
2616-
2617-
} else if (ciq_idx < ciq_entries) {
2618-
const struct sge_ofld_rxq *rx = &adap->sge.rdmaciq[ciq_idx * 4];
2619-
int n = min(4, adap->sge.rdmaciqs - 4 * ciq_idx);
2620-
2621-
S("QType:", "RDMA-CIQ");
2622-
S("Interface:",
2623-
rx[i].rspq.netdev ? rx[i].rspq.netdev->name : "N/A");
2624-
R("RspQ ID:", rspq.abs_id);
2625-
R("RspQ size:", rspq.size);
2626-
R("RspQE size:", rspq.iqe_len);
2627-
R("RspQ CIDX:", rspq.cidx);
2628-
R("RspQ Gen:", rspq.gen);
2629-
S3("u", "Intr delay:", qtimer_val(adap, &rx[i].rspq));
2630-
S3("u", "Intr pktcnt:",
2631-
adap->sge.counter_val[rx[i].rspq.pktcnt_idx]);
2632-
RL("RxAN:", stats.an);
2633-
RL("RxNoMem:", stats.nomem);
26342526

26352527
} else if (ctrl_idx < ctrl_entries) {
26362528
const struct sge_ctrl_txq *tx = &adap->sge.ctrlq[ctrl_idx * 4];
@@ -2672,10 +2564,7 @@ do { \
26722564
static int sge_queue_entries(const struct adapter *adap)
26732565
{
26742566
return DIV_ROUND_UP(adap->sge.ethqsets, 4) +
2675-
DIV_ROUND_UP(adap->sge.iscsiqsets, 4) +
2676-
DIV_ROUND_UP(adap->sge.niscsitq, 4) +
2677-
DIV_ROUND_UP(adap->sge.rdmaqs, 4) +
2678-
DIV_ROUND_UP(adap->sge.rdmaciqs, 4) +
2567+
DIV_ROUND_UP(adap->sge.ofldqsets, 4) +
26792568
DIV_ROUND_UP(MAX_CTRL_QUEUES, 4) + 1;
26802569
}
26812570

0 commit comments

Comments
 (0)