dsp + golden net: prep for the encoder relocation (modular-backends part 3a/3b)

analysis/bandwaveform.go

@@ -2,7 +2,10 @@
 
 package analysis
 
-import "github.com/vynulldev/vynull/core"
+import (
+	"github.com/vynulldev/vynull/core"
+	"github.com/vynulldev/vynull/dsp"
+)
 
 // Neutral waveform resolutions. Detail matches the PWV5 detail rate; overview
 // is a fixed coarse width (like the PWV4 overview) whose points-per-second
@@ -20,7 +23,7 @@ func BandWaveformDetail(samples []float32, sampleRate int) core.BandWaveform {
 	if numPoints < 1 {
 		numPoints = 1
 	}
-	bass, mid, treble, _ := splitBandsAndPeaks(samples, sampleRate, numPoints)
+	bass, mid, treble, _ := dsp.SplitBandsAndPeaks(samples, sampleRate, numPoints, BandBassMidHz, BandMidTrebleHz)
 	return bandsToWaveform(detailEntriesPerSec, bass, mid, treble)
 }
 
@@ -31,7 +34,7 @@ func BandWaveformOverview(samples []float32, sampleRate int) core.BandWaveform {
 	if durationSec <= 0 {
 		return core.BandWaveform{}
 	}
-	bass, mid, treble, _ := splitBandsAndPeaks(samples, sampleRate, overviewPoints)
+	bass, mid, treble, _ := dsp.SplitBandsAndPeaks(samples, sampleRate, overviewPoints, BandBassMidHz, BandMidTrebleHz)
 	return bandsToWaveform(float64(overviewPoints)/durationSec, bass, mid, treble)
 }
 

analysis/blob_golden_test.go

@@ -0,0 +1,53 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+package analysis
+
+import "testing"
+
+// goldenBeats is a deterministic 32-beat grid at 120 BPM (500 ms apart), used
+// to lock the beat-grid / phrase blob encoders independently of the detector.
+func goldenBeats() []float64 {
+	b := make([]float64, 32)
+	for i := range b {
+		b[i] = float64(i) * 500
+	}
+	return b
+}
+
+func TestPQT2GoldenHash(t *testing.T) {
+	checkGolden(t, "PQT2 (beat grid, 0x2c04)",
+		GeneratePQT2(120, goldenBeats(), 0),
+		"056e1cb0129614b1746adc9ae3696ee49c778f6a24510078aea746b3bb55ce3b")
+}
+
+func TestBeatGridGoldenHash(t *testing.T) {
+	checkGolden(t, "beat grid (0x2204)",
+		GenerateBeatGrid(120, 60000, 0),
+		"6ab4b71bf3643d6d1be9f3ff6659e85fddf950365e1dcec650980743ca86e620")
+}
+
+func TestBeatGridFromBeatsGoldenHash(t *testing.T) {
+	checkGolden(t, "beat grid from beats",
+		GenerateBeatGridFromBeats(&BeatResult{BPM: 120, Beats: goldenBeats(), Downbeat: 0}),
+		"8a9ba25e847b08ae25b12dde6d22c50053761cb5d757cd5a2c3b6892dcbde41e")
+}
+
+func TestPSSIGoldenHash(t *testing.T) {
+	phrases := []Phrase{
+		{StartBeat: 1, EndBeat: 16, Kind: 1, StartMs: 0, EndMs: 8000},
+		{StartBeat: 17, EndBeat: 32, Kind: 5, StartMs: 8000, EndMs: 16000},
+	}
+	checkGolden(t, "PSSI (song structure)",
+		GeneratePSSI(phrases, 120),
+		"b919eb9fd2a5cb5e32e7cebdf3f53d48794effacaeec2964a3526a80011d31a9")
+}
+
+func TestPVB2GoldenHash(t *testing.T) {
+	checkGolden(t, "PVB2", GeneratePVB2(),
+		"28a54fd044a18b6041d33ac63e8dcc2722ce7d9df7a5c24f70b28f975654aac8")
+}
+
+func TestPVBRGoldenHash(t *testing.T) {
+	checkGolden(t, "PVBR", GeneratePVBR(1000000),
+		"229cfc7bde86ad0a4c49541930d5a411533342397107c7ca844616d41daac7f1")
+}

analysis/waveform.go

@@ -5,6 +5,8 @@ package analysis
 import (
 	"encoding/binary"
 	"math"
+
+	"github.com/vynulldev/vynull/dsp"
 )
 
 const (
@@ -163,25 +165,25 @@ func generateColorPreviewV2(samples []float32, sampleRate int) []byte {
 	// at 2 kHz; the target shows essentially zero in that transition zone,
 	// implying a near-brick-wall response.
 	bp8Low := func(s []float32, cutoff float64) []float32 {
-		c := butterworthLow(cutoff, sr)
-		return applyBiquad(applyBiquad(applyBiquad(applyBiquad(s, c), c), c), c)
+		c := dsp.ButterworthLow(cutoff, sr)
+		return dsp.ApplyBiquad(dsp.ApplyBiquad(dsp.ApplyBiquad(dsp.ApplyBiquad(s, c), c), c), c)
 	}
 	bp8High := func(s []float32, cutoff float64) []float32 {
-		c := butterworthHigh(cutoff, sr)
-		return applyBiquad(applyBiquad(applyBiquad(applyBiquad(s, c), c), c), c)
+		c := dsp.ButterworthHigh(cutoff, sr)
+		return dsp.ApplyBiquad(dsp.ApplyBiquad(dsp.ApplyBiquad(dsp.ApplyBiquad(s, c), c), c), c)
 	}
 	// Mid uses an 8th-order HP@200 for sharp bass rejection followed by a
 	// gentle 2nd-order LP@800 — the mid has a humped response
 	// peaking around 400-600 Hz then rolling off at ~3-6 dB/octave above.
 	// A flat band-pass 200-2000 Hz over-emits the 1-2 kHz region by ~2×.
-	midLP := butterworthLow(800, sr)
+	midLP := dsp.ButterworthLow(800, sr)
 	// d2 is a 2nd-order LP@400 with gentle 12 dB/octave rolloff. The target is
 	// 16→8→2 from 100→500→1000 Hz which an 8th-order would overshoot at the
 	// top end (still 8 at 1 kHz when the target is at 2).
-	lowLP := butterworthLow(400, sr)
+	lowLP := dsp.ButterworthLow(400, sr)
 	bassSamples := bp8Low(samples, 200)
-	lowSamples := applyBiquad(samples, lowLP)
-	midSamples := applyBiquad(bp8High(samples, 200), midLP)
+	lowSamples := dsp.ApplyBiquad(samples, lowLP)
+	midSamples := dsp.ApplyBiquad(bp8High(samples, 200), midLP)
 	trebleSamples := bp8High(samples, PreviewTrebleHz)
 
 	const entrySize = 6
@@ -269,123 +271,6 @@ func generateColorPreviewV2(samples []float32, sampleRate int) []byte {
 	return buf
 }
 
-// biquadCoeffs holds coefficients for a 2nd-order IIR (biquad) filter.
-type biquadCoeffs struct {
-	b0, b1, b2, a1, a2 float64
-}
-
-// butterworthLow computes 2nd-order Butterworth low-pass filter coefficients.
-// Butterworth Q = 1/√2 ≈ 0.707 (maximally flat passband, no resonant peak).
-// alpha = sin(w0) / (2*Q) = sin(w0) / √2.
-func butterworthLow(cutoff, sampleRate float64) biquadCoeffs {
-	w0 := 2.0 * math.Pi * cutoff / sampleRate
-	alpha := math.Sin(w0) / math.Sqrt2
-	cosW0 := math.Cos(w0)
-	a0 := 1.0 + alpha
-	return biquadCoeffs{
-		b0: (1.0 - cosW0) / 2.0 / a0,
-		b1: (1.0 - cosW0) / a0,
-		b2: (1.0 - cosW0) / 2.0 / a0,
-		a1: -2.0 * cosW0 / a0,
-		a2: (1.0 - alpha) / a0,
-	}
-}
-
-// butterworthHigh computes 2nd-order Butterworth high-pass filter coefficients.
-func butterworthHigh(cutoff, sampleRate float64) biquadCoeffs {
-	w0 := 2.0 * math.Pi * cutoff / sampleRate
-	alpha := math.Sin(w0) / math.Sqrt2
-	cosW0 := math.Cos(w0)
-	a0 := 1.0 + alpha
-	return biquadCoeffs{
-		b0: (1.0 + cosW0) / 2.0 / a0,
-		b1: -(1.0 + cosW0) / a0,
-		b2: (1.0 + cosW0) / 2.0 / a0,
-		a1: -2.0 * cosW0 / a0,
-		a2: (1.0 - alpha) / a0,
-	}
-}
-
-// applyBiquad applies a biquad IIR filter to the samples (in-place would mutate, so returns new slice).
-func applyBiquad(samples []float32, c biquadCoeffs) []float32 {
-	out := make([]float32, len(samples))
-	var x1, x2, y1, y2 float64
-	for i, s := range samples {
-		x0 := float64(s)
-		y0 := c.b0*x0 + c.b1*x1 + c.b2*x2 - c.a1*y1 - c.a2*y2
-		out[i] = float32(y0)
-		x2 = x1
-		x1 = x0
-		y2 = y1
-		y1 = y0
-	}
-	return out
-}
-
-// splitBandsAndPeaks runs IIR band-splitting on samples and returns
-// per-segment peak amplitudes for bass, mid, treble, and the unfiltered overall.
-// Used by PWV5 (per-entry relative color) and the 3-band JSON generator.
-//
-// Cutoffs ~200/2000 Hz, matching the crossover points
-// (confirmed via ramp/tone tests). Mid uses a proper band-pass (HP@200 + LP@2000)
-// rather than complementary subtraction, which had constructive-interference
-// artefacts near the bass cutoff that rendered mid-range tones as bass-tinted
-// on the CDJ (yellow/orange instead of green).
-func splitBandsAndPeaks(samples []float32, sampleRate, numPoints int) (allBass, allMid, allTreble, allTotal []float64) {
-	segLen := len(samples) / numPoints
-	if segLen < 1 {
-		segLen = 1
-	}
-
-	sr := float64(sampleRate)
-	bassFiltered := applyBiquad(samples, butterworthLow(BandBassMidHz, sr))
-	trebleFiltered := applyBiquad(samples, butterworthHigh(BandMidTrebleHz, sr))
-	midFiltered := applyBiquad(
-		applyBiquad(samples, butterworthHigh(BandBassMidHz, sr)),
-		butterworthLow(BandMidTrebleHz, sr),
-	)
-
-	allBass = make([]float64, numPoints)
-	allMid = make([]float64, numPoints)
-	allTreble = make([]float64, numPoints)
-	allTotal = make([]float64, numPoints)
-
-	for i := 0; i < numPoints; i++ {
-		start := i * segLen
-		end := start + segLen
-		if end > len(samples) {
-			end = len(samples)
-		}
-		if start >= len(samples) {
-			break
-		}
-		var bp, mp, tp, overall float64
-		for j := start; j < end; j++ {
-			bv := math.Abs(float64(bassFiltered[j]))
-			mv := math.Abs(float64(midFiltered[j]))
-			tv := math.Abs(float64(trebleFiltered[j]))
-			sv := math.Abs(float64(samples[j]))
-			if bv > bp {
-				bp = bv
-			}
-			if mv > mp {
-				mp = mv
-			}
-			if tv > tp {
-				tp = tv
-			}
-			if sv > overall {
-				overall = sv
-			}
-		}
-		allBass[i] = bp
-		allMid[i] = mp
-		allTreble[i] = tp
-		allTotal[i] = overall
-	}
-	return
-}
-
 // detailEntriesPerSec is the entry rate for PWV3/PWV5 scrolling waveforms.
 // Exports use ~150 entries/sec (e.g. a 484s track has 72,741 PWV5
 // entries → 150.3/s; a 165s track has 24,796 entries → 150.3/s).
@@ -403,7 +288,7 @@ func GenerateDetail(samples []float32, sampleRate int) []byte {
 		numPoints = 1
 	}
 
-	allBass, allMid, allTreble, allTotal := splitBandsAndPeaks(samples, sampleRate, numPoints)
+	allBass, allMid, allTreble, allTotal := dsp.SplitBandsAndPeaks(samples, sampleRate, numPoints, BandBassMidHz, BandMidTrebleHz)
 
 	// Find global maxes — overall (classic mode) and per-band (3-band mode).
 	totalMax, bassMax, midMax, trebleMax := 0.0, 0.0, 0.0, 0.0
@@ -751,9 +636,9 @@ func splitBands3RMS(samples []float32, sampleRate, numPoints int) (bass, mid, tr
 		numPoints = 1
 	}
 	sr := float64(sampleRate)
-	bf := applyBiquad(samples, butterworthLow(BandBassMidHz, sr))
-	tf := applyBiquad(samples, butterworthHigh(BandMidTrebleHz, sr))
-	mf := applyBiquad(applyBiquad(samples, butterworthHigh(BandBassMidHz, sr)), butterworthLow(BandMidTrebleHz, sr))
+	bf := dsp.ApplyBiquad(samples, dsp.ButterworthLow(BandBassMidHz, sr))
+	tf := dsp.ApplyBiquad(samples, dsp.ButterworthHigh(BandMidTrebleHz, sr))
+	mf := dsp.ApplyBiquad(dsp.ApplyBiquad(samples, dsp.ButterworthHigh(BandBassMidHz, sr)), dsp.ButterworthLow(BandMidTrebleHz, sr))
 	segLen := len(samples) / numPoints
 	if segLen < 1 {
 		segLen = 1

docs/design/part3-encoder-relocation.md

@@ -0,0 +1,124 @@
+# Part 3 plan — relocate the Pioneer encoders (modular-backends step 2, final half)
+
+## Goal / end state
+
+- **`analysis/`** = pure DSP only. `AnalyzeTrack` produces a neutral
+  `core.Analysis` (tempo, beats, key, band waveforms, phrases, cues) — no
+  Pioneer bytes.
+- **`link/prolink/anlz/`** = every Pioneer wire encoder (PWV4/5/6/7, ANLZ
+  sections, PQT2, PVB, PSSI, beat-grid blob), consuming `core.Analysis` (+ raw
+  samples where a format needs its own band pass).
+- **Cache** stores neutral `core.Analysis`; the prolink backend encodes to
+  Pioneer bytes (cached per-backend or on demand).
+- **Serving path** (`dbserver`/`api`/`proto`) gets encoded bytes from the
+  prolink backend, not from `analysis.Result`.
+
+## What's actually coupled today (facts)
+
+- **Encoders** live in `analysis/waveform.go` (PWV4/5/6/7 + mono/tiny previews),
+  `analysis/anlz.go` (ANLZ section writers, PVB2), `analysis/pqt2.go`,
+  `analysis/beatgrid.go`, `analysis/pvbr.go`, and `analysis/phrase.go`
+  (`GeneratePSSI` — sits next to the pure-DSP `DetectPhrases`).
+- **Shared low-level DSP** used by both encoders and the neutral extraction:
+  `fft` (`fft.go`), `butterworthLow/High`, `applyBiquad`, `biquadCoeffs`,
+  `splitBandsAndPeaks`, `computeHeights` (all in `waveform.go`).
+- **`AnalyzeTrack` calls the encoders 11×** → `analysis` currently *owns*
+  encoding. This is the dependency to break.
+- **`Result` is gob-cached** (`%x.gob`) with the encoded blobs embedded —
+  encoding happens at analyze-time, and the bytes are what's cached.
+- **~14 serving-path read sites** across `dbserver`/`api`/`proto` read
+  `Result.{BeatGrid×14, BeatGridPQT2×7, WaveDetail×5, WaveColorPreview×5,
+  WavePreview×4, WaveDetailMono×4, WaveDetail3Band×4, SongStructure×4,
+  WavePreview3Band×2}`.
+- **Not encoders:** `render.go` (`RenderPreviewPNG`) is the *web-UI PNG* renderer,
+  neutral-ish — leave it in `analysis` (or later move to `api`); it's out of
+  scope for the prolink relocation.
+
+## Key challenges (why this is the risky half)
+
+1. **Dependency inversion.** If encoders move to `prolink` while `AnalyzeTrack`
+   still calls them, `analysis → prolink` — backwards. So encoding must move out
+   of `AnalyzeTrack` and become a prolink-side step.
+2. **Shared low-level DSP.** `fft`/`biquad`/`splitBandsAndPeaks` can't live in a
+   package that `prolink` shouldn't depend on wholesale — extract them so both
+   pure DSP and the encoders can use them.
+3. **Cache flip.** Moving encoding to serve-time means the cache stores neutral
+   `core.Analysis` instead of Pioneer blobs (cacheVersion bump; existing caches
+   re-analyze once — harmless).
+4. **Band-representation mismatch.** Each encoder computes its *own* bands
+   (PWV4 uses 8th-order filters at 1200 pts; PWV5 uses `splitBandsAndPeaks` at
+   150/s; PWV6/7 use different scales). There is no single neutral waveform all
+   four consume, so encoders keep taking `samples` for their own band pass — the
+   neutral `core.Analysis.Detail/Overview` is a *convenience* for new backends,
+   not a forced input for the Pioneer ones.
+5. **Byte-exactness.** The golden tests (PWV4/5/6/7) guard the four colour
+   waveforms; PVB/PQT2/ANLZ/PSSI have **no golden test yet** — add those before
+   moving them.
+
+## Staged sub-PRs
+
+Each is shippable and golden-guarded; risk rises at 3c.
+
+### 3a — extract shared low-level DSP into a `dsp` package  *(low risk)*
+Move `fft`, `butterworth*`, `applyBiquad`, `biquadCoeffs`, `splitBandsAndPeaks`,
+`computeHeights` into `analysis/dsp` (or top-level `dsp`). Update `analysis`
+(incl. `bandwaveform.go`) and the encoders to import it. Pure move — golden
+tests + full suite still pass. Verifiable by build + golden hashes.
+
+### 3b — add golden tests for the remaining blobs  *(test-only, low risk)*
+Lock `GeneratePQT2`, `GenerateBeatGrid[FromBeats]`, `GeneratePSSI`, the ANLZ
+section writers, and PVB against the deterministic signal (extend
+`waveform_golden_test.go`'s pattern). This completes the safety net before the
+move.
+
+### 3c — move the Pioneer encoders into `link/prolink/anlz`  *(structural, byte-safe)*
+Move the encoder functions (keeping `samples`/inputs → bytes signatures so the
+bytes are identical), importing `dsp` + `core`. Move the golden tests with them.
+`GeneratePSSI` splits from `DetectPhrases` (DSP stays, encoder moves). Build +
+golden hashes verify byte-for-byte.
+
+### 3d — flip the flow: neutral cache + serve-time encoding  *(the risky one — HW validate)*
+- `AnalyzeTrack` stops calling encoders; caches neutral `core.Analysis`
+  (+ artwork). `cacheVersion` bump.
+- The **prolink backend** gains an encode layer: `core.Analysis` (+ samples as
+  needed) → the Pioneer blobs, cached per-track on the prolink side.
+- Repoint the ~14 serving-path reads: `dbserver`/`api`/`proto` request bytes
+  from the prolink backend instead of `analysis.Result`.
+- Delete the encoded fields from `Result` (or retire `Result` in favour of
+  `core.Analysis` + a prolink `Encoded` struct).
+- **Golden tests still guard the bytes, but the wiring change needs a real-CDJ
+  smoke test:** load a track on a deck and confirm waveform + beat grid +
+  cues + phrases render, on at least one NXS2 and (if available) a CDJ-3000
+  (for the 3-band PWV6/7 path).
+
+### 3e — (optional) make encoders consume `core.Analysis`  *(cleanup)*
+Where an encoder's bands match the neutral `Detail`/`Overview` (PWV5 especially),
+have it take `core.Analysis` instead of recomputing from samples — deduping the
+double band pass. Golden-guarded.
+
+## Recommended sequencing
+
+- **PR A = 3a + 3b + 3c** — all byte-safe structural work (extract DSP, finish
+  golden net, move encoders). No serving-path or cache change; golden hashes
+  prove zero byte drift. Mergeable with confidence.
+- **PR B = 3d** — the flow flip + cache change, on its own, **with the hardware
+  smoke test as the merge gate.** This is where a deck must be in the loop.
+- **PR C = 3e** — optional dedupe cleanup, later.
+
+## Open decisions
+
+- **Cache granularity:** prolink caches encoded blobs per-track (fast serve,
+  more disk) vs encodes on demand from cached `core.Analysis` (less disk, CPU on
+  first serve). Suggest per-track encoded cache to match today's serve latency.
+- **`Result`'s fate:** keep a slimmed neutral `Result` (metadata + `core.Analysis`)
+  or replace it outright with `core.Analysis`. Prefer replacing, with a prolink
+  `EncodedTrack` for the blobs.
+- **`render.go`:** leave in `analysis` for now; a later pass can move web-PNG
+  rendering to `api` since it consumes `core` band data, not Pioneer bytes.
+
+## Verification checklist (every PR)
+
+- `go build ./...`, `go vet`, `gofmt -l` clean.
+- `go test ./analysis/... ./link/...` incl. **all golden hashes unchanged**.
+- PR B only: **real-CDJ smoke test** (waveform / beat grid / cues / phrases on a
+  deck) before merge.