feat(rdp): chroma-key alpha compare (key_en, R-10)

CHANGELOG.md

@@ -8,6 +8,19 @@ All notable changes to RustyN64 are documented here. The format is based on
 
 Work toward `v0.8.0 "Breadth"` — the accuracy battery (Phase 7).
 
+### Added — RDP chroma-key alpha compare (gap-analysis Stage D, ledger R-10)
+
+- **`key_en` (Set Other Modes bit 40) chroma-keying.** `Set Key GB`/`R` now also
+  store the per-channel `key_width`, and when `key_en` is set `Rdp::combine` takes
+  the Angrylion `combiner_1cycle` key path: the RGB output is the sub-A chromabypass
+  colour and the pixel alpha is `chroma_key_min` over the pre-`>>8` 17-bit combined
+  colour and the key widths. The new behaviour is gated on `key_en`, so the common
+  combiner path is byte-identical (all 31 prior RDP conformance vectors unchanged).
+  Validated byte-for-byte against Angrylion by `tex_tri_chromakey_alpha_16` (golden
+  `0x4321`, non-vacuous — clearing `key_en` outputs the combined colour) plus
+  mutation-checked unit tests. n64-systemtest impact: none. Still deferred under
+  R-10: noise (un-oracled), the derivative `lod_frac`, and the YUV `K0`–`K3` convert.
+
 ### Added — RDP chroma-key combiner inputs (gap-analysis Stage D, ledger R-10)
 
 - **`Set Key GB` (0x2A) / `Set Key R` (0x2B) now decode** the per-channel chroma-key
@@ -20,8 +33,8 @@ Work toward `v0.8.0 "Breadth"` — the accuracy battery (Phase 7).
   (`combine_cycle_routes_chroma_key`), and the `tex_tri_chromakey_16` conformance
   vector validates the path byte-for-byte end-to-end against Angrylion (RGBA5551
   `0x3b1f`, black if unwired). n64-systemtest impact: none (no RDP-combiner
-  coverage). Still deferred under R-10: noise, the derivative `lod_frac`, the
-  chroma-key alpha compare, and the YUV convert `K0`–`K3`.
+  coverage). Still deferred under R-10: noise, the derivative `lod_frac`, and the
+  YUV convert `K0`–`K3` (the chroma-key alpha compare is wired above).
 
 ### Changed — the frontend now presents the accurate VI scan-out (ledger R-5)
 

crates/rustyn64-rdp/src/lib.rs

@@ -597,6 +597,35 @@ fn clamp_9bit(color: i32) -> u8 {
     special_expand(color).clamp(0, 0xFF) as u8
 }
 
+/// The combiner equation's **pre-`>>8` 17-bit** result — what the chroma-key alpha
+/// compare consumes (Angrylion `color_combiner_equation`). Same terms as
+/// [`combine_channel`] but returning `((A − B) * C + (D << 8) + 0x80) & 0x1ffff`
+/// rather than the `>> 8`'d colour.
+const fn combine_channel_17bit(a: i32, b: i32, c: i32, d: i32) -> i32 {
+    (((special_expand(a) - special_expand(b)) * sext9(c)) + (special_expand(d) << 8) + 0x80)
+        & 0x1_FFFF
+}
+
+/// The chroma-key alpha (Angrylion `chroma_key_min`): per channel, fold the sign of
+/// the 17-bit combined value into a distance, offset by the programmed half-width,
+/// take the minimum across R/G/B, and clamp to `[0, 0xff]`. `col17` is the pre-`>>8`
+/// combined colour ([`combine_channel_17bit`]); `width` is the 12-bit `Set Key` width
+/// per channel.
+#[allow(clippy::cast_possible_truncation, clippy::cast_sign_loss)]
+fn chroma_key_min(col17: [i32; 3], width: [u16; 3]) -> u8 {
+    let mut keyalpha = i32::MAX;
+    for ch in 0..3 {
+        // Sign-extend the 17-bit value (mask first so an unmasked caller is safe).
+        let mut k = ((col17[ch] & 0x1_FFFF) << 15) >> 15; // SIGN(col, 17)
+        if k > 0 {
+            k = if (k & 0xf) == 8 { -k + 0x10 } else { -k };
+        }
+        k += i32::from(width[ch]) << 4;
+        keyalpha = keyalpha.min(k);
+    }
+    keyalpha.clamp(0, 0xff) as u8
+}
+
 /// The combiner's "1" input — `0x100` in the internal `.8` representation, not
 /// `0xFF` (N64brew *…/Commands* §0x3C).
 const COMBINER_ONE: i16 = 0x100;
@@ -869,6 +898,9 @@ pub struct OtherModes {
     pub z_mode: u8,
     /// Alpha-compare enable (gates the pixel write; R-11).
     pub alpha_compare_en: bool,
+    /// Chroma-key enable (bit 40): the combiner outputs the sub-A colour and derives
+    /// the pixel alpha from the key window (`chroma_key_min`). R-10.
+    pub key_en: bool,
     /// Perspective-correct texturing (bit 51): divide the interpolated `S`/`T` by `W`.
     pub persp_tex_en: bool,
     /// Anti-aliasing enable (bit 3): sub-pixel edge coverage governs which edge
@@ -1384,13 +1416,15 @@ pub struct Rdp {
     /// `Set Convert` `K5` (see `k4`).
     pub k5: i16,
     /// Chroma-key **centre** per channel `[r, g, b]` (`Set Key R` 0x2B / `Set Key GB`
-    /// 0x2A) — the combiner RGB sub-B input (select 6). `0..=255`. The key **width**
-    /// (used only by the deferred chroma-key alpha compare, not the combiner mux) is
-    /// not stored — it has no consumer yet, so it lands with that path. R-10.
+    /// 0x2A) — the combiner RGB sub-B input (select 6). `0..=255`. R-10.
     pub key_center: [u8; 3],
     /// Chroma-key **scale** per channel `[r, g, b]` (`Set Key R`/`GB`) — the combiner
     /// RGB mul input (select 6). `0..=255`. R-10.
     pub key_scale: [u8; 3],
+    /// Chroma-key **width** per channel `[r, g, b]` (`Set Key R`/`GB`, 12-bit) — the
+    /// half-width of the key window, consumed by the `key_en` chroma-key alpha compare
+    /// (`chroma_key_min`), not the combiner mux. R-10.
+    pub key_width: [u16; 3],
     /// The environment colour, RGBA8888 (`Set Env Color`, 0x3B).
     pub env_color: u32,
     /// The blend colour, RGBA8888 (`Set Blend Color`, 0x39).
@@ -1608,18 +1642,20 @@ impl Rdp {
                 self.k5 = (lo & 0x1FF) as i16;
             }
             OP_SET_KEY_GB => {
-                // word 1 (lo): centre_g[31:24], scale_g[23:16], centre_b[15:8],
-                // scale_b[7:0] (Angrylion `rdp_set_key_gb`). The widths — word 0 (hi):
-                // width_g[23:12], width_b[11:0] — drive only the deferred chroma-key
-                // alpha compare, not the combiner mux, so they are not stored.
+                // word 0 (hi): width_g[23:12], width_b[11:0]; word 1 (lo):
+                // centre_g[31:24], scale_g[23:16], centre_b[15:8], scale_b[7:0]
+                // (Angrylion `rdp_set_key_gb`). Widths feed the `key_en` alpha compare.
+                self.key_width[1] = ((hi >> 12) & 0xFFF) as u16;
+                self.key_width[2] = (hi & 0xFFF) as u16;
                 self.key_center[1] = (lo >> 24) as u8;
                 self.key_scale[1] = (lo >> 16) as u8;
                 self.key_center[2] = (lo >> 8) as u8;
                 self.key_scale[2] = lo as u8;
             }
             OP_SET_KEY_R => {
-                // word 1 (lo): width_r[31:16] (deferred), centre_r[15:8], scale_r[7:0]
-                // (Angrylion `rdp_set_key_r`).
+                // word 1 (lo): width_r[27:16] (12-bit), centre_r[15:8], scale_r[7:0]
+                // (Angrylion `rdp_set_key_r`, `(args[1] >> 16) & 0xfff`).
+                self.key_width[0] = ((lo >> 16) & 0xFFF) as u16;
                 self.key_center[0] = (lo >> 8) as u8;
                 self.key_scale[0] = lo as u8;
             }
@@ -2687,7 +2723,26 @@ impl Rdp {
             // before cycle 1 (`combiner_2cycle_cycle1`, R-13).
             core::mem::swap(&mut inp.texel0, &mut inp.texel1);
         }
-        Self::combine_cycle(self.combine.cyc1, &inp)
+        let cyc = self.combine.cyc1;
+        if self.other_modes.key_en {
+            // Chroma-key alpha compare (Angrylion `combiner_1cycle` key_en path, R-10):
+            // the RGB output is the sub-A "chromabypass" colour (clamped), and the pixel
+            // alpha is derived from the key window over the pre-`>>8` 17-bit combined
+            // colour. Gated on `key_en` so the common path (below) is byte-identical.
+            let mut col17 = [0i32; 3];
+            let mut rgb = [0u8; 3];
+            for (ch, (c17, out)) in col17.iter_mut().zip(rgb.iter_mut()).enumerate() {
+                let a = i32::from(rgb_input_a(cyc.rgb_a, &inp, ch));
+                let b = i32::from(rgb_input_b(cyc.rgb_b, &inp, ch));
+                let c = i32::from(rgb_input_c(cyc.rgb_c, &inp, ch));
+                let d = i32::from(rgb_input_d(cyc.rgb_d, &inp, ch));
+                *c17 = combine_channel_17bit(a, b, c, d);
+                *out = clamp_9bit(a); // chromabypass = sub-A input, clamped
+            }
+            let keyalpha = chroma_key_min(col17, self.key_width);
+            return [rgb[0], rgb[1], rgb[2], keyalpha];
+        }
+        Self::combine_cycle(cyc, &inp)
     }
 
     /// Decode `Set Other Modes` (0x2F) into [`OtherModes`]. The blend selects and
@@ -2723,10 +2778,11 @@ impl Rdp {
             z_update_en: (lo >> 5) & 1 != 0,
             z_mode: ((lo >> 10) & 0x3) as u8,
             alpha_compare_en: lo & 1 != 0,
-            persp_tex_en: (hi >> 19) & 1 != 0, // command bit 51
-            aa_enable: (lo >> 3) & 1 != 0,     // command bit 3
+            key_en: (hi >> 8) & 1 != 0,               // command bit 40
+            persp_tex_en: (hi >> 19) & 1 != 0,        // command bit 51
+            aa_enable: (lo >> 3) & 1 != 0,            // command bit 3
             rgb_dither_mode: ((hi >> 6) & 0x3) as u8, // command bits 39:38
-            sample_type: (hi >> 13) & 1 != 0,  // command bit 45
+            sample_type: (hi >> 13) & 1 != 0,         // command bit 45
         };
     }
 
@@ -3750,19 +3806,23 @@ mod tests {
         assert_eq!(rdp.color_image, 0x0010_0000);
     }
 
-    /// **`Set Key GB`/`Set Key R` decode the chroma-key centre/scale per channel
-    /// (R-10).** Pins the bit-layout ported from Angrylion `rdp_set_key_gb`/`_r`:
-    /// GB word-1 is `centre_g[31:24] scale_g[23:16] centre_b[15:8] scale_b[7:0]`, and
-    /// R word-1 is `width_r[31:16] centre_r[15:8] scale_r[7:0]` (width unused). Distinct
-    /// per-channel values so a field-swap in the decode is caught.
+    /// **`Set Key GB`/`Set Key R` decode the chroma-key centre/scale/width per channel
+    /// (R-10).** Pins the bit-layout ported from Angrylion `rdp_set_key_gb`/`_r`: GB
+    /// word-0 is `width_g[23:12] width_b[11:0]`, word-1 `centre_g[31:24] scale_g[23:16]
+    /// centre_b[15:8] scale_b[7:0]`; R word-1 is `width_r[27:16] centre_r[15:8]
+    /// scale_r[7:0]`. Distinct per-channel values (incl. distinct 12-bit widths) so a
+    /// field-swap or wrong extraction in the decode is caught.
     #[test]
-    fn set_key_decodes_centre_and_scale_per_channel() {
+    fn set_key_decodes_centre_scale_and_width_per_channel() {
         let (rdp, _) = run_commands(&[
-            (0x2A00_0000, 0x4080_60C0), // Set Key GB: cg=0x40 sg=0x80 cb=0x60 sb=0xC0
-            (0x2B00_0000, 0x0000_2040), // Set Key R:  wr=0 cr=0x20 sr=0x40
+            // Set Key GB: wg=0x111 wb=0x222; cg=0x40 sg=0x80 cb=0x60 sb=0xC0.
+            (0x2A11_1222, 0x4080_60C0),
+            // Set Key R: wr=0x333; cr=0x20 sr=0x40.
+            (0x2B00_0000, 0x0333_2040),
         ]);
         assert_eq!(rdp.key_center, [0x20, 0x40, 0x60], "centre [r, g, b]");
         assert_eq!(rdp.key_scale, [0x40, 0x80, 0xC0], "scale [r, g, b]");
+        assert_eq!(rdp.key_width, [0x333, 0x111, 0x222], "width [r, g, b]");
     }
 
     /// **`Set Fill Color` and `Set Scissor` store their values.**
@@ -5155,6 +5215,30 @@ mod tests {
         assert_eq!(Rdp::combine_cycle(cfg, &inp), [56, 96, 120, 255]);
     }
 
+    /// **`chroma_key_min` folds each channel, offsets by the width, and takes the
+    /// minimum (R-10).** Per channel `k = SIGN(col17, 17)`; if `k > 0`, `k = -k`
+    /// (or `-k + 0x10` when the low nibble is 8); then `k = (width << 4) + k`; the
+    /// result is `min(kr, kg, kb)` clamped to `[0, 0xff]`. Hand-computed cases:
+    #[test]
+    fn chroma_key_min_folds_and_takes_the_minimum() {
+        // Positive col17, low nibble != 8: k = (width<<4) - col17.
+        // r: 128 - 0x10 = 112; g: 128 - 0x20 = 96; b: 128 - 0x30 = 80 -> min 80.
+        assert_eq!(chroma_key_min([0x10, 0x20, 0x30], [8, 8, 8]), 80);
+        // Low-nibble == 8 special fold: k = -col17 + 0x10. col17 0x18 -> -0x18+0x10 = -8;
+        // r: 128 - 8 = 120; the wider g/b (256) leave r the minimum.
+        assert_eq!(chroma_key_min([0x18, 0x300, 0x300], [8, 0x40, 0x40]), 120);
+        // A large col17 drives k negative -> the minimum clamps to 0.
+        assert_eq!(chroma_key_min([0x400, 0x10, 0x10], [8, 8, 8]), 0);
+        // Bit 16 set: SIGN(col17, 17) is NEGATIVE (col17 - 0x20000), so the `k > 0`
+        // fold is skipped. 0x1FFF0 -> -16; r: (8<<4) - 16 = 112, the minimum vs the
+        // wider g/b. This exercises the signed-fold branch (a broken sign-extend that
+        // read 0x1FFF0 as positive would fold to a large negative and clamp to 0).
+        assert_eq!(
+            chroma_key_min([0x1_FFF0, 0x300, 0x300], [8, 0x40, 0x40]),
+            112
+        );
+    }
+
     /// **Two-cycle mode chains cycle 0 into cycle 1's `Combined` input.** Cycle 0
     /// passes texel0 through; cycle 1 selects `Combined` for A (with C=One-ish),
     /// so the final output reflects cycle 0's result — not cycle 1 reading a stale

crates/rustyn64-test-harness/tests/rdp_conformance.rs

@@ -459,6 +459,26 @@ fn tex_tri_chromakey_16_matches_angrylion() {
     );
 }
 
+/// **The chroma-key alpha compare (`key_en`, Set Other Modes bit 40) — R-10.** With
+/// `key_en`, the combiner outputs the sub-A "chromabypass" colour (Shade) and derives
+/// the pixel alpha from `chroma_key_min` over the pre-`>>8` 17-bit combined colour + the
+/// `Set Key` widths. The **key alpha is made observable** by enabling alpha-compare
+/// (bit 0) with a `Set Blend Color` threshold of `0x80`: the Shade triangle is written
+/// only where the key alpha `>= 0x80`. The combine (`rgb_a=rgb_b=Shade` → 17-bit `0x80`,
+/// `width_r=0x10`) yields key alpha `0x80` exactly, so the triangle IS drawn — and a
+/// broken `chroma_key_min` shifting it below `0x80` makes the triangle vanish (a −1
+/// mutation is verified to fail), while clearing `key_en` outputs the combined colour
+/// (black) instead of Shade. Pins the `key_en`/`key_width` decode, the 17-bit combined
+/// value, and `chroma_key_min` end-to-end against Angrylion (the `chroma_key_min` fold
+/// itself is additionally unit-tested with hand-computed values).
+#[test]
+fn tex_tri_chromakey_alpha_16_matches_angrylion() {
+    assert_matches(
+        "tex_tri_chromakey_alpha_16",
+        include_bytes!("vectors/tex_tri_chromakey_alpha_16.rvec"),
+    );
+}
+
 /// A **COPY-mode Texture Rectangle** (16-bit) — the first texture path validated
 /// against Angrylion. Copy mode blits texels straight from TMEM to the colour image
 /// (no combiner, no 1-cycle texel pipeline), so it sidesteps the gaps `tex_tri_16`

crates/rustyn64-test-harness/vectors-gen/driver.c

@@ -953,6 +953,31 @@ static const uint32_t V29_TEX_TRI_CHROMAKEY_16[] = {
     SHADE_BLOCK_FLAT(0x40, 0x40, 0x40, 0xFF), // flat shade (ignored by the combine)
 };
 
+// V30: the chroma-key ALPHA compare (key_en, Set Other Modes bit 40) — ledger R-10.
+// The key alpha (`chroma_key_min`) must be OBSERVABLE, so alpha-compare is enabled
+// (bit 0) with a Set Blend Color threshold of 0x80: the pixel is written only if the
+// key alpha >= 0x80. With rgb_a=rgb_b=Shade the 17-bit combined colour is 0x80 (A−B=0
+// → 0x80 constant), so per channel `SIGN(0x80)=128` folds to `-128`, then `+width<<4`:
+// width_r=0x10 → 256−128 = 128 = 0x80 (the min across r/g/b), which meets the 0x80
+// threshold, so the Shade chromabypass triangle IS drawn. A broken `chroma_key_min`
+// (wrong sign/fold/width/min) shifts the key alpha below 0x80 → the triangle vanishes;
+// clearing key_en outputs the *combined* colour (black, A−B=0) instead of Shade. Both
+// make this non-vacuous — the golden's drawn Shade pixels observe the key alpha.
+static const uint32_t V30_TEX_TRI_CHROMAKEY_ALPHA_16[] = {
+    0x2F0001F0u, 0x00000001u, // Set Other Modes: 1-cycle, KEY_EN (bit 40), ALPHA_COMPARE (bit 0)
+    0x39000000u, 0x00000080u, // Set Blend Color: alpha threshold = 0x80
+    0x2A014018u, 0x00000000u, // Set Key GB: wg=0x14 wb=0x18 (centre/scale unused here)
+    0x2B000000u, 0x00100000u, // Set Key R:  wr=0x10 (centre/scale unused)
+    0x3C000080u, 0x041C01C6u, // Set Combine: rgb_a=Shade rgb_b=Shade rgb_c=Combined rgb_d=Zero; a=One
+    0x3F100007u, 0x00001000u, // Set Color Image: 16-bit, width 8, addr 0x1000
+    0x2D000000u, 0x00020020u, // Set Scissor: (0,0)-(8,8)
+    0x0C800020u, 0x00200000u, // op=0x0C (shade), lft=1, yl=32, ym=32, yh=0
+    0x00000000u, 0x00000000u, // XL, DxLDy
+    0x00020000u, 0x00000000u, // XH = 2.0
+    0x00020000u, 0x00010000u, // XM = 2.0, DxMDy = 1.0
+    SHADE_BLOCK_FLAT(0x40, 0x60, 0x80, 0xFF), // flat shade -> chromabypass + col17 source
+};
+
 // ---- Seeded fuzz generator (SplitMix64) ----
 //
 // A reproducible pseudo-random corpus: the seed and this generator's source fully
@@ -1622,6 +1647,11 @@ int main(int argc, char **argv) {
                   0, 0, NULL};
     if (emit_vector(&v29, out_dir)) return 1;
 
+    Vector v30 = {"tex_tri_chromakey_alpha_16", 0x2000, 0x1000, 8, 8, 2,
+                  sizeof(V30_TEX_TRI_CHROMAKEY_ALPHA_16) / 4, V30_TEX_TRI_CHROMAKEY_ALPHA_16,
+                  0, 0, NULL};
+    if (emit_vector(&v30, out_dir)) return 1;
+
     if (emit_vi_vectors(out_dir)) return 1;
 
     return 0;