diff --git a/drivers/rknpu/rknpu_drv.c b/drivers/rknpu/rknpu_drv.c index 4090344b4..1f66e4118 100644 --- a/drivers/rknpu/rknpu_drv.c +++ b/drivers/rknpu/rknpu_drv.c @@ -37,6 +37,7 @@ #include #endif +#include #include "rknpu_ioctl.h" #include "rknpu_reset.h" #include "rknpu_fence.h" @@ -417,6 +418,27 @@ static int rknpu_action(struct rknpu_device *rknpu_dev, break; case RKNPU_SET_VOLT: break; + case RKNPU_ACT_RESET_CORE: { + /* #patch: per-core counterpart of RKNPU_ACT_RESET. Cheaper than the device-wide path — no + * msleep(100), only two reset lines — but it still needs the same quiesce for the IOMMU + * re-attach, which is unsafe asynchronously (it raced rknpu_iommu_dma_map_sg and panicked). */ + int _core = (int)args->value; + if (_core < 0 || _core >= rknpu_dev->config->num_irqs) + return -EINVAL; + ret = rknpu_reset_core_and_remap(rknpu_dev, _core); + break; + } + case RKNPU_ACT_SET_DOMAIN: { /* #patchB11: make a domain live before an import's PRIME ioctl */ + int _id = (int)args->value; + + ret = rknpu_iommu_domain_get_and_switch(rknpu_dev, _id); + if (!ret) + rknpu_iommu_domain_put(rknpu_dev); /* switch only; hold no reference */ + break; + } + case RKNPU_ACT_REINIT: /* #patch61 */ + ret = rknpu_iommu_reinit(rknpu_dev); + break; case RKNPU_ACT_RESET: ret = rknpu_soft_reset(rknpu_dev); break; @@ -778,17 +800,520 @@ static enum hrtimer_restart hrtimer_handler(struct hrtimer *timer) return HRTIMER_RESTART; } +/* ================= PROGRESS WATCHDOG (#patch) ========================================== + * WHY. rknpu_job_timeout_clean is not a watchdog: it runs only when ANOTHER submit arrives on + * the same core, so a stalled job with no follow-up submit is never noticed (which is why + * userspace has to fire "reap dummy" submits). It also measures TOTAL JOB AGE, meaningless for + * a PC-chained job whose task list is extensible at runtime. "No task completed in N samples" + * is well defined for a chain of any length. + * + * CONFIRMED FINDING: on a doorbell miss the PC completed-task counter reads 0 with the job many + * ms old -- the job is accepted and NEVER DISPATCHES. (Control: at an 8 ms threshold a healthy + * ~30 ms job does NOT report, so the counter genuinely advances and the offset is right.) + * + * POWER SAFETY -- the hard part, and v1/v2 got it wrong. + * v1 read NPU MMIO straight from the hrtimer. A register read on a powered-down block is an + * external abort => instant HARD hang (no ping, power cycle required). It hung the board on + * its first NPU workload. + * v2 guarded with pm_runtime_get_if_in_use(rknpu_dev->dev). STILL WRONG on RK3588: this driver + * keeps its OWN refcount (power_refcount/power_lock) and rknpu_power_off() puts the three + * PER-CORE genpds (genpd_dev_npu0/1/2) directly. A reference on the parent does not hold those + * up, so core i's domain can still drop between the check and the readl on base[i]. + * v3 (this): sample in PROCESS context and hold power_lock across the read. rknpu_power_off() + * takes the same mutex, so the genpds cannot go down while we read. We deliberately do NOT call + * rknpu_power_get(): that would POWER THE NPU ON just to read a counter and pin it up forever. + * mutex_trylock means the watchdog can never delay a submit; a missed sample is simply skipped + * rather than counted as a no-advance. + * + * The hrtimer only decides cadence and queues the work; all MMIO happens in the worker. */ +/* DEFAULT OFF. This watchdog has destabilised the board three times: v1 hard-hung it (unguarded MMIO, + * fixed), and even the power_lock-guarded v3 COLD-RESET it twice mid-run (instant SoC reset, no panic, + * no console, ramoops cleared). Discriminator, same workload: wd_period_us=1000 -> reset; + * wd_period_us=0 -> 4000 reps clean, same boot_id. It is a DIAGNOSTIC: set wd_period_us=1000 only for + * a deliberate investigation, accepting that the board may reset. */ +/* #patch40: the progress watchdog is ON BY DEFAULT. + * + * It was opt-in because two earlier versions killed the board (v1 read NPU MMIO straight from the + * hrtimer; v2's pm_runtime_get_if_in_use on the parent missed the per-core genpds). v3 samples in + * process context holding power_lock and has since run clean. The other reason -- "it caused board + * resets" -- was RETRACTED; that was the vendor IRQ bug fixed as change 6. + * + * Opt-in made it useless in practice: the one time it was needed, it was enabled by hand and + * recorded wd_ticks=0 across 8 stalls, because the arm site was NONBLOCK-only (fixed in #patch39). + * A stalled BLOCKING submit otherwise sits in rknpu_job_wait() for its whole timeout -- 60.6 s + * measured -- with nothing reporting it. Detection at ~120 ms is a ~500x cut in time-to-visibility. + * + * Cost, measured A/B on this board (2000-rep int4 probe, ndom=1, interleaved x3, warmup discarded): + * OFF 133/133/133 commits/s ON 131/133/129 commits/s => ~1.5% + * Paid on every submit since #patch39. Judged worth it: the stalls it exposes cost ~23% of 27B + * prefill and were previously invisible. + * + * Detection latency is wd_samples * wd_period_us ~= 120 ms. Do NOT drop the period to 1000 without + * also dropping wd_samples: 1 kHz sampling costs ~100x throughput (2.8 vs 283-490 commits/s). */ +/* #patchB12: DEFAULT OFF — this is a DEVELOPER tool, not a production mechanism. + * + * The progress watchdog was written to chase "job committed, PC task counter 0x0, no interrupt", + * which we believed was a hardware stall. It was not: it was our own multi-domain page-table bugs + * (see the wiki, Kernel-Modifications changes 20-22 and Vendor-Kernel-Behaviour). Mainline + * accel/rocket carries no stall detection at all, and does not need any. + * + * With those fixed the watchdog should never fire, so leaving it armed only costs a 100 Hz timer and + * the risk of misreading a slow-but-healthy job: the detection threshold is wd_samples * wd_period_us + * ~= 120 ms, and a cold weight-DMA can legitimately exceed that and still complete bit-exact + * (measured 2026-08-27 — several detections per suite run with every output bit-exact). + * + * It stays useful for DEVELOPERS: a wedge-prone shape can still hang the NPU, and a hang costs a + * reboot — or, for wide 27B prefill, a physical SPI reflash. So arm it while bringing up new shapes: + * echo 10000 > /sys/module/rknpu/parameters/wd_period_us # detect + log + * echo 2000 > /sys/module/rknpu/parameters/wd_abort_ms # AND fail the submit (opt-in on top) + * The cheap counters (cnt_commit / cnt_done / cnt_irq) are always on and cost nothing — they are what + * actually localised the bug this was built for. */ +static unsigned int rknpu_wd_period_us; /* 0 = off; see above */ +module_param_named(wd_period_us, rknpu_wd_period_us, uint, 0644); +MODULE_PARM_DESC(wd_period_us, "progress watchdog sample period in us (0 = off, the default; set 10000 to arm developer stall detection)"); + +/* diagnostics: is the watchdog actually sampling, and why not when it isn't */ +static unsigned long rknpu_wd_ticks, rknpu_wd_seen_job, rknpu_wd_pm_skip, rknpu_wd_stalls; +module_param_named(wd_ticks, rknpu_wd_ticks, ulong, 0444); +module_param_named(wd_seen_job, rknpu_wd_seen_job, ulong, 0444); +module_param_named(wd_pm_skip, rknpu_wd_pm_skip, ulong, 0444); +module_param_named(wd_stalls, rknpu_wd_stalls, ulong, 0444); + +/* START-CONFIRM-AND-RETRY (wd_kick=1). + * A stalled job is committed with byte-identical registers, a valid mapping, a fresh descriptor and + * verified-correct regcmd bytes in DRAM — it simply never begins, and the completed-task counter stays 0. + * The registers are still correct at that point, so rather than the current recovery (userspace waits out + * a ~614 ms window, then issues a DEVICE-WIDE ACT_RESET that destroys warm state on all three cores, then + * resubmits — ~3.6 s per miss), just RE-PULSE the start trigger for that one core. + * + * Cheap and targeted: two register writes, no reset, no warm-state loss, no resubmit. If the PC merely + * missed its start, this restarts it; if it does not, we have lost nothing and learned that the fault is + * not a missed start trigger. Fires ONCE per stall episode, on the same condition as the report, and only + * with the counter still reading 0 (i.e. nothing has executed, so re-pulsing cannot disturb work in + * flight). Opt-in while under evaluation. */ +static unsigned long rknpu_cnt_kick; +module_param_named(cnt_kick, rknpu_cnt_kick, ulong, 0444); + +static unsigned int rknpu_wd_samples = 12; +/* #patch41: FAST-ABORT. When a BLOCKING submit stalls, its caller sits in rknpu_job_wait() for + * args->timeout x 3 -- 60.8 s measured on a real 1.5B-Q8 run -- before getting -ETIMEDOUT and + * self-healing. Waking it as soon as the stall is established turns that into ~wd_abort_ms. + * + * DELIBERATELY NOT the logging threshold. The PC counter advances per COMPLETED TASK, so a single + * long-running task is indistinguishable from a stall by this signal alone; aborting at the ~120 ms + * log threshold would kill slow-but-healthy jobs. This is a separate, much larger bound: still + * ~30x faster than 60 s while leaving any plausible single task room to finish. + * + * 0 disables. OPT-IN on purpose -- the previous automatic stall intervention (wd_kick) was deleted + * for having no measurable benefit, so this one earns its default only once measured. */ +static unsigned int rknpu_wd_abort_ms; +module_param_named(wd_abort_ms, rknpu_wd_abort_ms, uint, 0644); +MODULE_PARM_DESC(wd_abort_ms, "fail a stalled BLOCKING submit after this many ms (0 = off)"); +/* #patch47: opt-in timing profiler. prof_mask != 0 enables it (bit0 = per-stall breakdown, + * bit1 = also log healthy commits). Everything it reports is derived from timestamps the driver + * already takes, so the cost when disabled is one branch and when enabled is one printk. */ +/* #patch49: CAUSAL TEST for the change-16 correlation. + * + * Every stalled job was committed 0.24-0.9 ms after the previous commit on the same core, against a + * healthy mean three orders of magnitude larger. That is correlation. This enforces a MINIMUM spacing + * between commits on a core so the hypothesis can be tested directly: if stalls vanish as this rises, + * back-to-back commits cause them; if they persist, the short gap is a symptom of something else. + * + * 0 = off (default). Clamped to 5000 us because this MUST busy-wait: rknpu_job_next() can be reached + * from the completion IRQ, where sleeping is illegal. A diagnostic knob, not a shipping fix -- a real + * fix would reshape submission rather than burn CPU in udelay(). */ +static unsigned int rknpu_min_commit_gap_us; +module_param_named(min_commit_gap_us, rknpu_min_commit_gap_us, uint, 0644); +MODULE_PARM_DESC(min_commit_gap_us, "enforce >= this gap between commits on a core (0=off, max 5000)"); +unsigned long rknpu_gap_enforced_n, rknpu_gap_enforced_us; +module_param_named(gap_enforced_n, rknpu_gap_enforced_n, ulong, 0444); +module_param_named(gap_enforced_us_sum, rknpu_gap_enforced_us, ulong, 0444); +unsigned int rknpu_min_commit_gap_get(void) { return rknpu_min_commit_gap_us; } + +/* #patch55: SUB-BLOCK register dump — is the NPU stuck mid-read, or did it never issue one? + * + * At stall time the PC completed-task counter reads 0x0 and all four MMU banks read IDLE + * (0x19 = paging|idle|replay-empty). Those two facts pull in opposite directions: MMU-idle says no + * translation is outstanding, i.e. nothing was ever requested; but the job WAS triggered, so + * something should have been. The missing instrument is the NPU's own sub-blocks — we only ever + * dumped the PC block. + * + * CNA 0x1000 does the convolution feeder think it is running? + * DPU 0x4000 output stage state + * CDMA 0x5000 IS A DMA READ OUTSTANDING? <- the discriminator + * + * CDMA busy while the MMU is idle == a transaction stuck UPSTREAM of translation (a hung read). + * CDMA idle too == the trigger never took and nothing was issued. + * + * Deliberately dumped BLIND rather than by name: mainline drivers/accel/rocket names these, but it + * is a much newer kernel than this tree, and we do not need names to diff a stalled snapshot against + * a healthy one. Names can follow once we know which words move. + * + * Safety: each core maps 0x10000 (rk3588s.dtsi), so 0x1000/0x4000/0x5000 are in range; and this runs + * only from the watchdog worker, which is process context holding power_lock with power_refcount > 0 + * (patch06: an MMIO read of a powered-down block kills the machine instantly). Capped so a stall + * storm cannot flood the log. */ +/* #patch58: does the PC completed-task counter EVER read non-zero while a job is in flight? + * + * The cnt>0 healthy control never fired across a whole run despite tens of thousands of samples. + * If cnt is always 0 for in-flight jobs then `cnt == wd_last` is trivially true and the watchdog is + * not a progress detector at all — it is a "job present for > wd_samples*period" duration timer, and + * the "counter 0x0" in every stall report is NOT evidence that the hardware never started, because + * healthy jobs would report it too. That would invalidate the framing this whole investigation has + * carried. Record the maximum ever observed and settle it. */ +static unsigned int rknpu_wd_max_cnt; +module_param_named(wd_max_cnt, rknpu_wd_max_cnt, uint, 0444); +static unsigned long rknpu_wd_cnt_nonzero; +module_param_named(wd_cnt_nonzero, rknpu_wd_cnt_nonzero, ulong, 0444); + +/* #patch67: TIME SERIES of CNA/DPU S_STATUS across in-flight jobs (prof_mask bit 3). + * + * The one-shot healthy dump showed CNA/DPU S_STATUS = 0x00000000 healthy vs 0x00000005 stalled, which + * suggests the PC DID start the downstream units and they are parked. But `cnt > 0` only proves the + * job executed a task — the single sample may have landed BETWEEN tasks, so "healthy = 0" could be a + * quiet moment rather than the running state, which would invalidate the comparison. + * + * Sample every watchdog tick for every in-flight job and log wd_flat alongside, so healthy + * (flat small, still progressing) and stalling (flat large) samples sit in one stream and can be + * separated offline. If 0x5 ever appears transiently while healthy, it means RUNNING and the units are + * merely stuck mid-op; if it never does, 0x5 is specific to the failure. */ +static unsigned long rknpu_prof_stat_n; +module_param_named(prof_stat_n, rknpu_prof_stat_n, ulong, 0444); +static unsigned long rknpu_prof_stat_cap = 400; +module_param_named(prof_stat_cap, rknpu_prof_stat_cap, ulong, 0644); + +static unsigned int rknpu_prof_blk_cap = 4; +module_param_named(prof_blk_cap, rknpu_prof_blk_cap, uint, 0644); +static unsigned int rknpu_prof_blk_done; +static unsigned int rknpu_prof_blk_healthy_done; + +static void rknpu_blk_dump(void __iomem *base, int core, const char *tag) +{ + /* #patch65: include the PC block (0x0000) — the one we never compared. + * + * The CPU->NPU protocol is a doorbell plus a DMA-fetched program: write PC_DATA_ADDR (0x10) + * with the regcmd's IOVA, PC_DATA_AMOUNT (0x14), the interrupt regs, then PC_TASK_CONTROL + * (0x30) whose 0x6 field is the trigger; the NPU then DMA-reads the regcmd and executes it. + * At a stall the trigger IS consumed (0x7002 written, 0x1002 read back) but the MMU stays + * idle, so the fetch never happens — the doorbell is acknowledged and the program is never + * collected. + * + * A stall dump showed PC_OP_EN (0x08) reading 0x00000000 with INT_STATUS and INT_RAW_STATUS + * both 0. The commit sequence never writes PC_OP_EN, and one watchdog kick mode was + * "re-pulse PC_OP_EN", so it is plausibly the run-enable — but we have never dumped this + * block for a HEALTHY job, so there is nothing to compare against. Add it. */ + static const u32 blk[] = { 0x0000, 0x1000, 0x4000, 0x5000 }; + static const char *nm[] = { "PC", "CNA", "DPU", "CDMA" }; + int b, i; + + /* #patch57: 16 words per block, not 8. The first 8 were byte-identical between healthy and + * stalled, which may simply mean they are static config. Widen so a word that actually moves + * has a chance to appear. */ + for (b = 0; b < 4; b++) { + u32 v[16]; + int h; + + for (i = 0; i < 16; i++) + v[i] = readl(base + blk[b] + i * 4); + for (h = 0; h < 2; h++) + LOG_ERROR("RKNPU: BLK[%s] core%d %-4s +0x%04x: %08x %08x %08x %08x %08x %08x %08x %08x\n", + tag, core, nm[b], blk[b] + h * 32, + v[h*8+0], v[h*8+1], v[h*8+2], v[h*8+3], + v[h*8+4], v[h*8+5], v[h*8+6], v[h*8+7]); + } +} + +extern unsigned long rknpu_dbg_destroy_bailed; /* #patch69 */ +module_param_named(dbg_destroy_bailed, rknpu_dbg_destroy_bailed, ulong, 0444); + +static unsigned int rknpu_prof_mask; +module_param_named(prof_mask, rknpu_prof_mask, uint, 0644); +MODULE_PARM_DESC(prof_mask, "timing profiler: 0=off, 1=stall breakdown, 3=+healthy commits"); +/* #patch50: shape-axis profiling. bit1 = log every stall with its shape (already on), + * bit2 = ALSO log every commit with its shape, so the HEALTHY distribution can be built. + * ~1100 lines per run — trivial for the persistent disk log, and it keeps all the binning + * offline in awk instead of hard-coding a histogram in the kernel. */ +unsigned int rknpu_prof_mask_get(void) { return rknpu_prof_mask; } +/* #patch48: non-static — rknpu_job_done() (other TU) accumulates the healthy side */ +unsigned long rknpu_prof_ok_n, rknpu_prof_ok_gap, rknpu_prof_ok_queue; +module_param_named(prof_ok_n, rknpu_prof_ok_n, ulong, 0444); +module_param_named(prof_ok_gap_us_sum, rknpu_prof_ok_gap, ulong, 0444); +module_param_named(prof_ok_queue_us_sum, rknpu_prof_ok_queue, ulong, 0444); +static unsigned long rknpu_prof_bad_n, rknpu_prof_bad_gap, rknpu_prof_bad_queue; +module_param_named(prof_bad_n, rknpu_prof_bad_n, ulong, 0444); +module_param_named(prof_bad_gap_us_sum, rknpu_prof_bad_gap, ulong, 0444); +module_param_named(prof_bad_queue_us_sum, rknpu_prof_bad_queue, ulong, 0444); + +static unsigned long rknpu_wd_queued_aborts; +module_param_named(wd_queued_aborts, rknpu_wd_queued_aborts, ulong, 0444); +static unsigned long rknpu_wd_aborts; +module_param_named(wd_aborts, rknpu_wd_aborts, ulong, 0444); /* #patch40: 12 x 10 ms ~= 120 ms detection */ +module_param_named(wd_samples, rknpu_wd_samples, uint, 0644); +MODULE_PARM_DESC(wd_samples, "consecutive no-advance samples before reporting a stall"); + +/* Process context: safe to take power_lock, so safe to touch MMIO. */ +static void rknpu_wd_work_fn(struct work_struct *work) +{ + struct rknpu_device *rknpu_dev = + container_of(work, struct rknpu_device, wd_work); + struct rknpu_subcore_data *subcore_data = NULL; + struct rknpu_job *job = NULL; + unsigned long flags; + int i; + + /* never block a submit: power_lock is held by rknpu_power_get/put on the hot path */ + if (!mutex_trylock(&rknpu_dev->power_lock)) { + rknpu_wd_pm_skip++; + return; + } + /* powered off => nothing is running and MMIO is unsafe; skip without penalising wd_flat */ + if (atomic_read(&rknpu_dev->power_refcount) <= 0) { + rknpu_wd_pm_skip++; + mutex_unlock(&rknpu_dev->power_lock); + return; + } + + for (i = 0; i < rknpu_dev->config->num_irqs; i++) { + void __iomem *rknpu_core_base = rknpu_dev->base[i]; + u32 cnt; + + subcore_data = &rknpu_dev->subcore_datas[i]; + + spin_lock_irqsave(&rknpu_dev->irq_lock, flags); + job = subcore_data->job; + spin_unlock_irqrestore(&rknpu_dev->irq_lock, flags); + + if (!job) { + subcore_data->wd_flat = 0; + subcore_data->wd_job = NULL; /* #patch44 */ + continue; + } + rknpu_wd_seen_job++; + + cnt = readl(rknpu_core_base + + rknpu_dev->config->pc_task_status_offset) & + rknpu_dev->config->pc_task_number_mask; + + /* #patch44: a stall run must be attributed to a JOB, not just to a counter value. + * + * wd_flat only reset when the PC counter CHANGED. Consecutive jobs on one core each + * start their counter at 0, so across a job boundary the counter looks unchanged and + * wd_flat keeps climbing. Both the stall report and the fast-abort fire on EQUALITY + * (== wd_samples / == an), so only the FIRST stall in a run of them was ever seen and + * every later one was silently absorbed. + * + * Measured before this fix: 10 timed-out waits, 5 detected+aborted at 2.0 s, 5 missed + * entirely at ~61 s -- exactly 50% coverage, with the misses in a clean second mode. + * Reset the run whenever the job identity changes. */ + if (subcore_data->wd_job != job) { + subcore_data->wd_job = job; + subcore_data->wd_last = cnt; + subcore_data->wd_flat = 0; + } + + if ((rknpu_prof_mask & 8) && + rknpu_prof_stat_n < rknpu_prof_stat_cap) { + rknpu_prof_stat_n++; + /* #patch68: INT_RAW_STATUS (PC block, 0x2c) in the same series. + * + * It read 0x00000008 healthy vs 0xc0000000 stalled — but from a SINGLE sample + * of each, the same one-shot method that made CNA/DPU 0x5 look like a failure + * signature when it is really the RUNNING state. Sample it across both + * populations before believing it. */ + LOG_ERROR("RKNPU: SSTAT core%d cnt=%u flat=%u CNA=%08x DPU=%08x RAW=%08x\n", + i, cnt, subcore_data->wd_flat, + readl(rknpu_core_base + 0x1000), + readl(rknpu_core_base + 0x4000), + readl(rknpu_core_base + 0x2c)); + } + + if (cnt > rknpu_wd_max_cnt) + rknpu_wd_max_cnt = cnt; /* #patch58 */ + if (cnt) + rknpu_wd_cnt_nonzero++; + + /* #patch57: a VALID healthy control. + * + * The previous baseline fired at wd_flat == 0, which only means RECENTLY COMMITTED — + * plausibly the identical register state to a job that is about to be reported + * stalled. Comparing that against a stall compares a state with itself, and indeed + * every word matched. Gate on cnt > 0 instead: the PC completed-task counter has + * advanced, so this job has provably executed at least one task and cannot be in the + * failure state (whose defining symptom is counter stuck at 0x0). Also far more + * catchable than requiring the counter to CHANGE between two 10 ms samples. */ + if ((rknpu_prof_mask & 4) && cnt > 0 && + rknpu_prof_blk_healthy_done < rknpu_prof_blk_cap) { + rknpu_prof_blk_healthy_done++; + rknpu_blk_dump(rknpu_core_base, i, "OK"); + } + + if (cnt == subcore_data->wd_last) { + /* fire ONCE per stall episode (equality, not >=) */ + if (++subcore_data->wd_flat == rknpu_wd_samples) { + rknpu_wd_stalls++; + if ((rknpu_prof_mask & 4) && + rknpu_prof_blk_done < rknpu_prof_blk_cap) { + rknpu_prof_blk_done++; + rknpu_blk_dump(rknpu_core_base, i, "STALL"); /* #patch55 */ + } + /* #patch47: attribute this stall's timeline */ + rknpu_prof_bad_n++; + rknpu_prof_bad_gap += (unsigned long)job->prof_gap_us; + rknpu_prof_bad_queue += (unsigned long)job->prof_queue_us; + if (rknpu_prof_mask & 1) + LOG_ERROR("RKNPU: PROF stalled core %d: tn=%u cm=%#x dom=%d sinceswitch=%u tsince=%lldus gap_since_prev_commit=%lldus queue(submit->commit)=%lldus | healthy mean gap=%luus queue=%luus over n=%lu\n", + i, job->args->task_number, + job->args->core_mask, + job->iommu_domain_id, + job->prof_since_switch, + job->prof_tsince_us, + job->prof_gap_us, job->prof_queue_us, + rknpu_prof_ok_n ? rknpu_prof_ok_gap / rknpu_prof_ok_n : 0UL, + rknpu_prof_ok_n ? rknpu_prof_ok_queue / rknpu_prof_ok_n : 0UL, + rknpu_prof_ok_n); + /* #patch: report COMPLETED vs SUBMITTED so the line is self-interpreting. + * The kernel has both: the PC counter (completed) and the submit's task + * counts. "0/21" says the job never started; "17/21" would say it stalled + * partway. core_mask is echoed so this pairs with the userspace + * ORK_KMSG_SUBMIT stamp (`ork: sub#N dom=D cm=0xM tasks=T`). */ + LOG_ERROR("RKNPU: core %d NO TASK PROGRESS for %u samples (%u us), tasks=%u/%u done (counter %#x), cm=0x%x dom=%u, job age %lld us\n", + i, subcore_data->wd_flat, + subcore_data->wd_flat * rknpu_wd_period_us, + cnt, + (job->args->subcore_task[i].task_number ? + job->args->subcore_task[i].task_number : + job->args->task_number), + cnt, + job->args->core_mask, + job->args->iommu_domain_id, + ktime_us_delta(ktime_get(), job->timestamp)); + if (job->dbg_valid) + LOG_ERROR("RKNPU: commit snapshot: seq=%u WROTE data_addr=%#x amount=%#x int_mask=%#x task_ctrl=%#x dma_base=%#x tasks=%u | READBACK data_addr=%#x amount=%#x int_mask=%#x task_ctrl=%#x dma_base=%#x status(pre)=%#x status(post)=%#x\n", + job->dbg_seq, job->dbg_wr[0], job->dbg_wr[1], job->dbg_wr[2], + job->dbg_wr[3], job->dbg_wr[4], job->dbg_wr[5], + job->dbg_rd[0], job->dbg_rd[1], job->dbg_rd[2], + job->dbg_rd[3], job->dbg_rd[4], job->dbg_rd[5], + job->dbg_rd[6]); + LOG_ERROR("RKNPU: PCBLK %08x %08x %08x %08x %08x %08x %08x %08x %08x %08x %08x %08x %08x %08x %08x %08x | en=%08x\n", + job->dbg_blk[0], job->dbg_blk[1], job->dbg_blk[2], job->dbg_blk[3], + job->dbg_blk[4], job->dbg_blk[5], job->dbg_blk[6], job->dbg_blk[7], + job->dbg_blk[8], job->dbg_blk[9], job->dbg_blk[10], job->dbg_blk[11], + job->dbg_blk[12], job->dbg_blk[13], job->dbg_blk[14], job->dbg_blk[15], + job->dbg_blk[16]); + } + + /* #patch41: fast-abort a stalled BLOCKING submit. ASYNC jobs have no + * waiter, so there is nothing to wake. Equality fires it once. */ + if (rknpu_wd_abort_ms && !(job->flags & RKNPU_JOB_ASYNC)) { + unsigned int an = (rknpu_wd_abort_ms * 1000u) / + (rknpu_wd_period_us ? + rknpu_wd_period_us : 1u); + + if (an && subcore_data->wd_flat == an) { + struct rknpu_job *entry = NULL; + unsigned int qn = 0; + + spin_lock_irqsave(&rknpu_dev->irq_lock, flags); + job->flags |= RKNPU_JOB_STALLED; + /* #patch43: also abort everything QUEUED behind this head. + * + * This is the detection-coverage gap: the sampler only ever + * inspects subcore_data->job, the COMMITTED job, so a submit + * waiting in todo_list is invisible and its waiter burns the + * whole timeout for a job that never even started. Measured: + * 14 userspace submit failures produced only 2 watchdog + * episodes, and fast-abort therefore only helped 2 of 14. + * + * Safe to declare these stalled: rknpu_job_next() promotes + * from todo_list ONLY when subcore_data->job is NULL, so while + * this head is stuck nothing behind it can run. A woken job + * that never committed has last_task == NULL, and + * rknpu_job_wait() already handles exactly that -- it unlinks + * itself from todo_list and returns. No new path needed. */ + list_for_each_entry(entry, + &subcore_data->todo_list, + head[i]) { + if (entry->flags & RKNPU_JOB_ASYNC) + continue; + entry->flags |= RKNPU_JOB_STALLED; + qn++; + } + spin_unlock_irqrestore(&rknpu_dev->irq_lock, flags); + /* one wake serves the whole queue: every waiter sleeps on this + * same job_done_wq and re-tests its OWN job->flags */ + wake_up(&subcore_data->job_done_wq); + rknpu_wd_aborts++; + rknpu_wd_queued_aborts += qn; + LOG_ERROR("RKNPU: core %d FAST-ABORT after %u ms, +%u queued behind it (would have waited %u ms x3)\n", + i, rknpu_wd_abort_ms, qn, + job->args->timeout); + } + } + } else { + subcore_data->wd_last = cnt; + subcore_data->wd_flat = 0; + } + } + + mutex_unlock(&rknpu_dev->power_lock); +} + +/* Cadence only — NO MMIO here. Self-disarms once no core has an outstanding job. */ +static enum hrtimer_restart rknpu_wd_handler(struct hrtimer *timer) +{ + struct rknpu_device *rknpu_dev = + container_of(timer, struct rknpu_device, wd_timer); + unsigned long flags; + int i, active = 0; + + rknpu_wd_ticks++; + + spin_lock_irqsave(&rknpu_dev->irq_lock, flags); + for (i = 0; i < rknpu_dev->config->num_irqs; i++) + if (rknpu_dev->subcore_datas[i].job) + active++; + spin_unlock_irqrestore(&rknpu_dev->irq_lock, flags); + + if (!active) { + rknpu_dev->wd_on = 0; + return HRTIMER_NORESTART; + } + /* a no-op if the previous sample is still queued, which self-throttles */ + queue_work(system_wq, &rknpu_dev->wd_work); + + hrtimer_forward_now(timer, rknpu_dev->wd_kt); + return HRTIMER_RESTART; +} + +/* Armed from the submit path; self-disarms in the handler once no core has a job. Re-arming an + * already-running hrtimer is harmless, so the wd_on race is benign. */ +void rknpu_wd_arm(struct rknpu_device *rknpu_dev) +{ + if (!rknpu_wd_period_us || rknpu_dev->wd_on) + return; + rknpu_dev->wd_on = 1; + rknpu_dev->wd_kt = ktime_set(0, (u64)rknpu_wd_period_us * 1000); + hrtimer_start(&rknpu_dev->wd_timer, rknpu_dev->wd_kt, HRTIMER_MODE_REL); +} + static void rknpu_init_timer(struct rknpu_device *rknpu_dev) { rknpu_dev->kt = ktime_set(0, RKNPU_LOAD_INTERVAL); hrtimer_init(&rknpu_dev->timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL); rknpu_dev->timer.function = hrtimer_handler; hrtimer_start(&rknpu_dev->timer, rknpu_dev->kt, HRTIMER_MODE_REL); + /* #patch: watchdog timer is initialised but NOT started — armed on submit */ + hrtimer_init(&rknpu_dev->wd_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL); + rknpu_dev->wd_timer.function = rknpu_wd_handler; + rknpu_dev->wd_on = 0; + INIT_WORK(&rknpu_dev->wd_work, rknpu_wd_work_fn); } static void rknpu_cancel_timer(struct rknpu_device *rknpu_dev) { hrtimer_cancel(&rknpu_dev->timer); + hrtimer_cancel(&rknpu_dev->wd_timer); /* #patch */ + cancel_work_sync(&rknpu_dev->wd_work); /* #patch */ } static bool rknpu_is_iommu_enable(struct device *dev)