Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

mlxcx + Robert's IPD 39 changes to support mlxcx #444

Open
wants to merge 14 commits into
base: master
Choose a base branch
from
Open

mlxcx + Robert's IPD 39 changes to support mlxcx #444

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
14 commits
Select commit Hold shift + click to select a range
fb17419
14677 mlxcx NULL deref panic due to race in mlxcx_cmd_taskq
arekinath May 16, 2022
448f240
xxxxx mlxcx needs DDI_DEVICE_ATTR_V1
danmcd Jan 29, 2024
9bb6578
inline ddi_{get,put} wrappers
danmcd Sep 28, 2023
518ed3f
xxxxx mlxcx should lower DMA threshold
gwr Feb 9, 2024
e5984eb
15445 mlxcx MAC_PROP_MEDIA support
rmustacc Feb 24, 2023
3f2b2d4
mlxcx: batch up work under bufshard locks in ISR, don't take and rele…
arekinath Oct 31, 2023
206c020
mlxcx: bufbmtx very hot thanks to sq_ring_dbell
arekinath Nov 7, 2023
f8aaea4
mlxcx: multi-TIS and bf alternation
arekinath Nov 8, 2023
b2f543d
mlxcx: prepare sqes outside wq lock
arekinath Nov 8, 2023
ca01060
mlxcx: lso support
arekinath Nov 15, 2023
cc586c2
mlxcx should wait to free inlined header mblks until tx is done
arekinath Dec 4, 2023
6a16c08
mlxcx: bump max pointers for wq mem, allow deeper rings
arekinath Dec 12, 2023
37df6b6
mlxcx: add tx latency timers with DEBUG
arekinath Dec 4, 2023
3d151f2
Typo in LSO ASSERT() statement
danmcd Feb 9, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
142 changes: 66 additions & 76 deletions usr/src/uts/common/io/mlxcx/mlxcx.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -650,61 +650,6 @@ mlxcx_panic(mlxcx_t *mlxp, const char *fmt, ...)
va_end(ap);
}

uint16_t
mlxcx_get16(mlxcx_t *mlxp, uintptr_t off)
{
uintptr_t addr = off + (uintptr_t)mlxp->mlx_regs_base;
return (ddi_get16(mlxp->mlx_regs_handle, (void *)addr));
}

uint32_t
mlxcx_get32(mlxcx_t *mlxp, uintptr_t off)
{
uintptr_t addr = off + (uintptr_t)mlxp->mlx_regs_base;
return (ddi_get32(mlxp->mlx_regs_handle, (void *)addr));
}

uint64_t
mlxcx_get64(mlxcx_t *mlxp, uintptr_t off)
{
uintptr_t addr = off + (uintptr_t)mlxp->mlx_regs_base;
return (ddi_get64(mlxp->mlx_regs_handle, (void *)addr));
}

void
mlxcx_put32(mlxcx_t *mlxp, uintptr_t off, uint32_t val)
{
uintptr_t addr = off + (uintptr_t)mlxp->mlx_regs_base;
ddi_put32(mlxp->mlx_regs_handle, (void *)addr, val);
}

void
mlxcx_put64(mlxcx_t *mlxp, uintptr_t off, uint64_t val)
{
uintptr_t addr = off + (uintptr_t)mlxp->mlx_regs_base;
ddi_put64(mlxp->mlx_regs_handle, (void *)addr, val);
}

void
mlxcx_uar_put32(mlxcx_t *mlxp, mlxcx_uar_t *mlu, uintptr_t off, uint32_t val)
{
/*
* The UAR is always inside the first BAR, which we mapped as
* mlx_regs
*/
uintptr_t addr = off + (uintptr_t)mlu->mlu_base +
(uintptr_t)mlxp->mlx_regs_base;
ddi_put32(mlxp->mlx_regs_handle, (void *)addr, val);
}

void
mlxcx_uar_put64(mlxcx_t *mlxp, mlxcx_uar_t *mlu, uintptr_t off, uint64_t val)
{
uintptr_t addr = off + (uintptr_t)mlu->mlu_base +
(uintptr_t)mlxp->mlx_regs_base;
ddi_put64(mlxp->mlx_regs_handle, (void *)addr, val);
}

static void
mlxcx_fm_fini(mlxcx_t *mlxp)
{
Expand Down Expand Up @@ -816,6 +761,7 @@ mlxcx_teardown_bufs(mlxcx_t *mlxp)
list_destroy(&mlxp->mlx_buf_shards);

kmem_cache_destroy(mlxp->mlx_bufs_cache);
kmem_cache_destroy(mlxp->mlx_mbrm_cache);
}

static void
Expand Down Expand Up @@ -1259,7 +1205,7 @@ mlxcx_regs_map(mlxcx_t *mlxp)
* device.
*/
bzero(&da, sizeof (ddi_device_acc_attr_t));
da.devacc_attr_version = DDI_DEVICE_ATTR_V0;
da.devacc_attr_version = DDI_DEVICE_ATTR_V1;
da.devacc_attr_endian_flags = DDI_STRUCTURE_BE_ACC;
da.devacc_attr_dataorder = DDI_STRICTORDER_ACC;
if (DDI_FM_ACC_ERR_CAP(mlxp->mlx_fm_caps)) {
Expand Down Expand Up @@ -1434,6 +1380,26 @@ mlxcx_bufs_cache_destr(void *arg, void *cookie)
list_destroy(&b->mlb_tx_chain);
}

static int
mlxcx_mbrm_cache_constr(void *arg, void *cookie, int kmflags)
{
mlxcx_t *mlxp = cookie;
mlxcx_buf_return_mblk_t *mbrm = arg;
(void)mlxp;
bzero(mbrm, sizeof (mlxcx_buf_return_mblk_t));
return (0);
}

static void
mlxcx_mbrm_cache_destr(void *arg, void *cookie)
{
mlxcx_t *mlxp = cookie;
mlxcx_buf_return_mblk_t *mbrm = arg;
(void)mlxp;
VERIFY3P(mbrm->mbrm_mp, ==, NULL);
VERIFY(!list_link_active(&mbrm->mbrm_entry));
}

mlxcx_buf_shard_t *
mlxcx_mlbs_create(mlxcx_t *mlxp)
{
Expand Down Expand Up @@ -1467,6 +1433,12 @@ mlxcx_setup_bufs(mlxcx_t *mlxp)
sizeof (mlxcx_buffer_t), sizeof (uint64_t),
mlxcx_bufs_cache_constr, mlxcx_bufs_cache_destr,
NULL, mlxp, NULL, 0);
(void) snprintf(namebuf, KSTAT_STRLEN, "mlxcx%d_mbrm_cache",
ddi_get_instance(mlxp->mlx_dip));
mlxp->mlx_mbrm_cache = kmem_cache_create(namebuf,
sizeof (mlxcx_buf_return_mblk_t), sizeof (uint64_t),
mlxcx_mbrm_cache_constr, mlxcx_mbrm_cache_destr,
NULL, mlxp, NULL, 0);

list_create(&mlxp->mlx_buf_shards, sizeof (mlxcx_buf_shard_t),
offsetof(mlxcx_buf_shard_t, mlbs_entry));
Expand Down Expand Up @@ -1518,11 +1490,12 @@ mlxcx_eq_check(void *arg)
{
mlxcx_t *mlxp = (mlxcx_t *)arg;
mlxcx_event_queue_t *eq;
mlxcx_eventq_ctx_t ctx;
mlxcx_eventq_ctx_t *ctx;
const char *str;

uint_t i;

ctx = kmem_zalloc(sizeof (*ctx), KM_SLEEP);

for (i = 0; i < mlxp->mlx_intr_count; ++i) {
eq = &mlxp->mlx_eqs[i];

Expand All @@ -1536,26 +1509,26 @@ mlxcx_eq_check(void *arg)
*/
ASSERT0(eq->mleq_state & MLXCX_EQ_DESTROYED);

if (!mlxcx_cmd_query_eq(mlxp, eq, &ctx))
if (!mlxcx_cmd_query_eq(mlxp, eq, ctx))
continue;

str = "???";
switch (ctx.mleqc_status) {
switch (ctx->mleqc_status) {
case MLXCX_EQ_STATUS_OK:
break;
case MLXCX_EQ_STATUS_WRITE_FAILURE:
str = "WRITE_FAILURE";
break;
}

if (ctx.mleqc_status != MLXCX_EQ_STATUS_OK) {
if (ctx->mleqc_status != MLXCX_EQ_STATUS_OK) {
mlxcx_fm_qstate_ereport(mlxp, "event",
eq->mleq_num, str, ctx.mleqc_status);
eq->mleq_num, str, ctx->mleqc_status);
mlxcx_warn(mlxp, "EQ %u is in bad status: %x (%s)",
eq->mleq_intr_index, ctx.mleqc_status, str);
eq->mleq_intr_index, ctx->mleqc_status, str);
}

if (ctx.mleqc_state != MLXCX_EQ_ST_ARMED &&
if (ctx->mleqc_state != MLXCX_EQ_ST_ARMED &&
(eq->mleq_state & MLXCX_EQ_ARMED)) {
if (eq->mleq_cc == eq->mleq_check_disarm_cc &&
++eq->mleq_check_disarm_cnt >= 3) {
Expand All @@ -1569,17 +1542,21 @@ mlxcx_eq_check(void *arg)
eq->mleq_check_disarm_cnt = 0;
}
}

kmem_free(ctx, sizeof (*ctx));
}

static void
mlxcx_cq_check(void *arg)
{
mlxcx_t *mlxp = (mlxcx_t *)arg;
mlxcx_completion_queue_t *cq;
mlxcx_completionq_ctx_t ctx;
mlxcx_completionq_ctx_t *ctx;
const char *str, *type;
uint_t v;

ctx = kmem_zalloc(sizeof (*ctx), KM_SLEEP);

for (cq = list_head(&mlxp->mlx_cqs); cq != NULL;
cq = list_next(&mlxp->mlx_cqs, cq)) {

Expand All @@ -1597,7 +1574,7 @@ mlxcx_cq_check(void *arg)
if (cq->mlcq_fm_repd_qstate)
continue;

if (!mlxcx_cmd_query_cq(mlxp, cq, &ctx))
if (!mlxcx_cmd_query_cq(mlxp, cq, ctx))
continue;

if (cq->mlcq_wq != NULL) {
Expand All @@ -1613,7 +1590,7 @@ mlxcx_cq_check(void *arg)
}

str = "???";
v = get_bits32(ctx.mlcqc_flags, MLXCX_CQ_CTX_STATUS);
v = get_bits32(ctx->mlcqc_flags, MLXCX_CQ_CTX_STATUS);
switch (v) {
case MLXCX_CQC_STATUS_OK:
break;
Expand All @@ -1636,7 +1613,7 @@ mlxcx_cq_check(void *arg)
cq->mlcq_fm_repd_qstate = B_TRUE;
}

v = get_bits32(ctx.mlcqc_flags, MLXCX_CQ_CTX_STATE);
v = get_bits32(ctx->mlcqc_flags, MLXCX_CQ_CTX_STATE);
if (v != MLXCX_CQC_STATE_ARMED &&
(cq->mlcq_state & MLXCX_CQ_ARMED) &&
!(cq->mlcq_state & MLXCX_CQ_POLLING)) {
Expand All @@ -1652,19 +1629,25 @@ mlxcx_cq_check(void *arg)
cq->mlcq_check_disarm_cc = 0;
}
}

kmem_free(ctx, sizeof (*ctx));
}

void
mlxcx_check_sq(mlxcx_t *mlxp, mlxcx_work_queue_t *sq)
{
mlxcx_sq_ctx_t ctx;
mlxcx_sq_ctx_t *ctx;
mlxcx_sq_state_t state;

if (!mlxcx_cmd_query_sq(mlxp, sq, &ctx))
ctx = kmem_zalloc(sizeof (mlxcx_sq_ctx_t), KM_SLEEP);

if (!mlxcx_cmd_query_sq(mlxp, sq, ctx)) {
kmem_free(ctx, sizeof (*ctx));
return;
}

ASSERT3U(from_be24(ctx.mlsqc_cqn), ==, sq->mlwq_cq->mlcq_num);
state = get_bits32(ctx.mlsqc_flags, MLXCX_SQ_STATE);
ASSERT3U(from_be24(ctx->mlsqc_cqn), ==, sq->mlwq_cq->mlcq_num);
state = get_bits32(ctx->mlsqc_flags, MLXCX_SQ_STATE);
switch (state) {
case MLXCX_SQ_STATE_RST:
if (sq->mlwq_state & MLXCX_WQ_STARTED) {
Expand All @@ -1691,20 +1674,25 @@ mlxcx_check_sq(mlxcx_t *mlxp, mlxcx_work_queue_t *sq)
sq->mlwq_fm_repd_qstate = B_TRUE;
break;
}

kmem_free(ctx, sizeof (mlxcx_sq_ctx_t));
}

void
mlxcx_check_rq(mlxcx_t *mlxp, mlxcx_work_queue_t *rq)
{
mlxcx_rq_ctx_t ctx;
mlxcx_rq_ctx_t *ctx;
mlxcx_rq_state_t state;

ctx = kmem_zalloc(sizeof (*ctx), KM_SLEEP);

if (!mlxcx_cmd_query_rq(mlxp, rq, &ctx))
if (!mlxcx_cmd_query_rq(mlxp, rq, ctx)) {
kmem_free(ctx, sizeof (*ctx));
return;
}

ASSERT3U(from_be24(ctx.mlrqc_cqn), ==, rq->mlwq_cq->mlcq_num);
state = get_bits32(ctx.mlrqc_flags, MLXCX_RQ_STATE);
ASSERT3U(from_be24(ctx->mlrqc_cqn), ==, rq->mlwq_cq->mlcq_num);
state = get_bits32(ctx->mlrqc_flags, MLXCX_RQ_STATE);
switch (state) {
case MLXCX_RQ_STATE_RST:
if (rq->mlwq_state & MLXCX_WQ_STARTED) {
Expand All @@ -1731,6 +1719,8 @@ mlxcx_check_rq(mlxcx_t *mlxp, mlxcx_work_queue_t *rq)
rq->mlwq_fm_repd_qstate = B_TRUE;
break;
}

kmem_free(ctx, sizeof (*ctx));
}

static void
Expand Down