From 1d91e31c391fa4ed4a5e6ba25df2e7d10f4b1f08 Mon Sep 17 00:00:00 2001 From: Davide Angelocola Date: Tue, 7 Jul 2026 08:37:23 +0200 Subject: [PATCH] test: cover new decode/validity branches; fix S3034 dict bit-set masking (Sonar gate) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fix the two java:S3034 findings (byte promoted to int before OR without masking) that gate-block new_reliability_rating; both are behavior-preserving (result is immediately re-cast to byte): - reader/decode/DictEncodingDecoder.java setBit (~L207) - reader/layout/DictLayoutDecoder.java validity-mask gather (~L200) Add coverage for the previously untested decode/validity branches these paths introduced: - B1: RleEncodingDecoder F32/F64 decode (run boundary, empty->constant array) and the readDoubles/readFloats broadcast helpers (new RleEncodingDecoderTest; helpers made package-private, unreachable via decode() as constant children fully materialize). - B6: LazyRleFloatArray.materialize constant-run + empty-chunk fast path (mirrors the double sibling). - B2: DictEncodingDecoder validity broadcast slow path (codesCap < rowCount, the setBit fix site) plus U16/U32 readCode widths with combined codes/pool nulls. - B3: DictLayoutDecoder byte/short dict pools (DictByteArray/DictShortArray) and the gatherRowValidity Short/Int/Long codes-type arms. - B4: ScanIterator merged-grid out-of-range guard (mismatched column totals) and the sliceArray StructArray branch via a hand-assembled nested-struct file (also drives collectFlats' struct arm). - B5: ZonedStatsSchema boolean/size stat arms, fixed-width (I64/I32) field skip, the >63-bit varint overflow, and inner-spec overflow-tag / unknown-wire-type bail-outs. Not reached (reported, not forced): the readCode/gatherRowValidity `default` throws are pre-empted upstream (the expand switch and DictArrays.validateCodes reject non-U8/U16/U32 codes before those arms run) — defensive dead code, left as-is rather than backed by an unkillable test. Co-Authored-By: Claude Opus 4.8 --- .../reader/decode/DictEncodingDecoder.java | 2 +- .../reader/decode/RleEncodingDecoder.java | 7 +- .../reader/layout/DictLayoutDecoder.java | 2 +- .../reader/ScanIteratorChunkGridTest.java | 17 ++ .../reader/VortexReaderDecodeChunkTest.java | 135 +++++++++++++ .../vortex/reader/array/LazyRleArrayTest.java | 22 +++ .../decode/DictEncodingDecoderTest.java | 47 +++++ .../reader/decode/RleEncodingDecoderTest.java | 187 ++++++++++++++++++ .../reader/layout/DictLayoutDecoderTest.java | 126 ++++++++++++ .../reader/layout/ZonedStatsSchemaTest.java | 114 +++++++++++ 10 files changed, 655 insertions(+), 4 deletions(-) create mode 100644 reader/src/test/java/io/github/dfa1/vortex/reader/decode/RleEncodingDecoderTest.java diff --git a/reader/src/main/java/io/github/dfa1/vortex/reader/decode/DictEncodingDecoder.java b/reader/src/main/java/io/github/dfa1/vortex/reader/decode/DictEncodingDecoder.java index 63aacf9e..a26ad3df 100644 --- a/reader/src/main/java/io/github/dfa1/vortex/reader/decode/DictEncodingDecoder.java +++ b/reader/src/main/java/io/github/dfa1/vortex/reader/decode/DictEncodingDecoder.java @@ -204,7 +204,7 @@ private static long readCode(MemorySegment codes, long i, PType codePType) { private static void setBit(MemorySegment bits, long i) { long byteIdx = i >>> 3; byte cur = bits.get(ValueLayout.JAVA_BYTE, byteIdx); - bits.set(ValueLayout.JAVA_BYTE, byteIdx, (byte) (cur | (1 << (i & 7)))); + bits.set(ValueLayout.JAVA_BYTE, byteIdx, (byte) ((cur & 0xff) | (1 << (i & 7)))); } private static Array decodeUtf8DictLegacy(DecodeContext ctx, MemorySegment meta) { diff --git a/reader/src/main/java/io/github/dfa1/vortex/reader/decode/RleEncodingDecoder.java b/reader/src/main/java/io/github/dfa1/vortex/reader/decode/RleEncodingDecoder.java index 82ec9aaf..c0fa0871 100644 --- a/reader/src/main/java/io/github/dfa1/vortex/reader/decode/RleEncodingDecoder.java +++ b/reader/src/main/java/io/github/dfa1/vortex/reader/decode/RleEncodingDecoder.java @@ -167,7 +167,10 @@ private static short[] readShorts(MemorySegment buf, int count, PType ptype) { return out; } - private static double[] readDoubles(MemorySegment buf, int count) { + // Package-private (not private) so the broadcast branch (cap < count, a + // ConstantEncoding values pool) is testable directly: through decode() a + // constant child always materializes to `count` full elements, so cap == count. + static double[] readDoubles(MemorySegment buf, int count) { double[] out = new double[count]; long cap = SegmentBroadcast.capacity(buf, 8); for (int i = 0; i < count; i++) { @@ -176,7 +179,7 @@ private static double[] readDoubles(MemorySegment buf, int count) { return out; } - private static float[] readFloats(MemorySegment buf, int count) { + static float[] readFloats(MemorySegment buf, int count) { float[] out = new float[count]; long cap = SegmentBroadcast.capacity(buf, 4); for (int i = 0; i < count; i++) { diff --git a/reader/src/main/java/io/github/dfa1/vortex/reader/layout/DictLayoutDecoder.java b/reader/src/main/java/io/github/dfa1/vortex/reader/layout/DictLayoutDecoder.java index c9b13390..189c96f4 100644 --- a/reader/src/main/java/io/github/dfa1/vortex/reader/layout/DictLayoutDecoder.java +++ b/reader/src/main/java/io/github/dfa1/vortex/reader/layout/DictLayoutDecoder.java @@ -197,7 +197,7 @@ private static BoolArray gatherRowValidity(Array codes, BoolArray codesValidity, if (valid) { long byteIdx = i >>> 3; byte cur = bits.get(ValueLayout.JAVA_BYTE, byteIdx); - bits.set(ValueLayout.JAVA_BYTE, byteIdx, (byte) (cur | (1 << (i & 7)))); + bits.set(ValueLayout.JAVA_BYTE, byteIdx, (byte) ((cur & 0xff) | (1 << (i & 7)))); } } return new MaterializedBoolArray(DType.BOOL, n, bits.asReadOnly()); diff --git a/reader/src/test/java/io/github/dfa1/vortex/reader/ScanIteratorChunkGridTest.java b/reader/src/test/java/io/github/dfa1/vortex/reader/ScanIteratorChunkGridTest.java index 91a7d35d..5cd40c95 100644 --- a/reader/src/test/java/io/github/dfa1/vortex/reader/ScanIteratorChunkGridTest.java +++ b/reader/src/test/java/io/github/dfa1/vortex/reader/ScanIteratorChunkGridTest.java @@ -1,5 +1,6 @@ package io.github.dfa1.vortex.reader; +import io.github.dfa1.vortex.core.error.VortexException; import io.github.dfa1.vortex.core.model.ColumnName; import io.github.dfa1.vortex.core.model.LayoutId; import io.github.dfa1.vortex.reader.ScanIterator.ChunkSpec; @@ -12,6 +13,7 @@ import java.util.Map; import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; /// Unit tests for [ScanIterator#buildChunks(Map)] — the scan planner that merges each column's /// chunk grid into one split grid (the Rust reference's `StructReader::register_splits` union of @@ -120,6 +122,21 @@ void disjointBoundariesSpliceAtTheMergedGrid() { assertThat(coveringFlat(result.get(2), B)).isSameAs(coveringFlat(result.get(3), B)); } + @Test + void columnShorterThanMergedGridThrows() { + // Given two columns whose totals disagree — A spans 8 rows [4, 4] but B only 6 [3, 3]. A + // well-formed file keeps every column the same length; a malformed/untrusted layout may not. + // The merged grid {0, 3, 4, 6, 8} then produces a window [6, 8) that lies beyond B's last + // chunk, so the covering-chunk advance loop runs B's cursor off the end. The guard must + // surface that as a VortexException rather than an ArrayIndexOutOfBoundsException. + var columnFlats = flats(A, new long[]{4, 4}, B, new long[]{3, 3}); + + // When / Then + assertThatThrownBy(() -> ScanIterator.buildChunks(columnFlats)) + .isInstanceOf(VortexException.class) + .hasMessageContaining("no chunk covering"); + } + @Test void emptyProjectionYieldsNoChunks() { // Given no columns diff --git a/reader/src/test/java/io/github/dfa1/vortex/reader/VortexReaderDecodeChunkTest.java b/reader/src/test/java/io/github/dfa1/vortex/reader/VortexReaderDecodeChunkTest.java index 334c9009..837376b0 100644 --- a/reader/src/test/java/io/github/dfa1/vortex/reader/VortexReaderDecodeChunkTest.java +++ b/reader/src/test/java/io/github/dfa1/vortex/reader/VortexReaderDecodeChunkTest.java @@ -310,6 +310,46 @@ void scan_disjointMixedGrid_streamsExactValuesAcrossMergedWindows(@TempDir Path assertThat(streamedB).isEqualTo(MIXED_B_EXPECTED); } + // A NESTED struct column "s" (a vortex.struct under the root struct) with two I64 fields x, y, + // stored as ONE full-range subtree beside the per-chunk column "a" [2, 3, 2]. collectFlats treats + // the whole struct subtree as a single full-range chunk source, so the scan decodes it once as a + // StructArray and slices it per window — the ScanIterator.sliceArray StructArray branch that no + // primitive-only file reaches. + private static final long[] NESTED_X = {1000, 1001, 1002, 1003, 1004, 1005, 1006}; + private static final long[] NESTED_Y = {2000, 2001, 2002, 2003, 2004, 2005, 2006}; + private static final List NESTED_A_EXPECTED = + List.of(10L, 11L, 12L, 13L, 14L, 15L, 16L); + + @Test + void scan_nestedStructColumn_slicesEachFieldPerWindow(@TempDir Path tmp) throws Exception { + // Given a struct whose second column "s" is itself a struct {x, y} spanning all 7 rows, beside + // a per-chunk column "a" chunked [2, 3, 2]. Neither the Java nor JNI writer emits a nested + // struct beside a per-chunk column in one file, so it is hand-assembled here. + Path file = writeNestedStructFile(tmp); + var streamedA = new ArrayList(); + var streamedX = new ArrayList(); + var streamedY = new ArrayList(); + + // When streaming the whole scan — "s" is decoded once over the full range then sliced into + // each merged-grid window {0,2,5,7} via ScanIterator.sliceArray's StructArray arm, which + // recursively slices each field into the window. + try (var reader = VortexReader.open(file, registry()); + var iter = reader.scan(ScanOptions.all())) { + iter.forEachRemaining(chunk -> { + streamedA.addAll(values(chunk.column("a"))); + io.github.dfa1.vortex.reader.array.StructArray s = chunk.column("s"); + streamedX.addAll(fieldValues(s, 0)); + streamedY.addAll(fieldValues(s, 1)); + }); + } + + // Then every field value survives the per-window struct slice, including the middle window + // [2,5) sliced at a nonzero offset within the single full-range struct. + assertThat(streamedA).isEqualTo(NESTED_A_EXPECTED); + assertThat(streamedX).containsExactly(1000L, 1001L, 1002L, 1003L, 1004L, 1005L, 1006L); + assertThat(streamedY).containsExactly(2000L, 2001L, 2002L, 2003L, 2004L, 2005L, 2006L); + } + // ── Helpers ──────────────────────────────────────────────────────────────── private static ReadRegistry registry() { @@ -330,6 +370,101 @@ private static List>> streamAll(VortexReader reader) { return out; } + /// Reads field `fieldIdx` of a (possibly sliced) [StructArray] as a Long list. + private static List fieldValues(io.github.dfa1.vortex.reader.array.StructArray s, int fieldIdx) { + return values(s.field(fieldIdx)); + } + + /// Assembles a struct file: column "a" chunked over [CHUNK_ROWS] (one flat per chunk) and a nested + /// struct column "s" {x, y} stored as one full-range struct subtree (flat x, flat y). Segment + /// order: a-chunk0..a-chunkN, then x, then y. Layout: flat=0, chunked=1, struct=2; array spec + /// vortex.primitive=0. + private static Path writeNestedStructFile(Path dir) throws Exception { + int numChunks = CHUNK_ROWS.length; + List segments = new ArrayList<>(); + for (int k = 0; k < numChunks; k++) { + segments.add(primitiveSegment(COL_A[k], null)); + } + int xSegIdx = segments.size(); + segments.add(primitiveSegment(NESTED_X, null)); + int ySegIdx = segments.size(); + segments.add(primitiveSegment(NESTED_Y, null)); + + long[] segOffsets = new long[segments.size()]; + long[] segLengths = new long[segments.size()]; + long off = 0; + for (int i = 0; i < segments.size(); i++) { + segOffsets[i] = off; + segLengths[i] = segments.get(i).length; + off += segments.get(i).length; + } + + ByteBuffer footerBuf = MalformedFiles.buildFooter( + new String[]{"vortex.primitive", "vortex.bool"}, + new String[]{"vortex.flat", "vortex.chunked", "vortex.struct"}, + segOffsets, segLengths); + ByteBuffer dtypeBuf = buildNestedStructDtype(); + ByteBuffer layoutBuf = buildNestedStructLayout(CHUNK_ROWS, xSegIdx, ySegIdx); + + return writeVtxFile(dir, "nested.vtx", segments, footerBuf, dtypeBuf, layoutBuf); + } + + /// Root struct DType {a: I64 non-null, s: struct {x: I64, y: I64} non-null}. + private static ByteBuffer buildNestedStructDtype() { + var fbb = new FbsBuilder(512); + int primA = FbsPrimitive.createFbsPrimitive(fbb, FbsPType.I64, false); + int dtA = FbsDType.createFbsDType(fbb, FbsType.FbsPrimitive, primA); + + int primX = FbsPrimitive.createFbsPrimitive(fbb, FbsPType.I64, false); + int dtX = FbsDType.createFbsDType(fbb, FbsType.FbsPrimitive, primX); + int primY = FbsPrimitive.createFbsPrimitive(fbb, FbsPType.I64, false); + int dtY = FbsDType.createFbsDType(fbb, FbsType.FbsPrimitive, primY); + int innerNames = FbsStruct_.createNamesVector(fbb, + new int[]{fbb.createString("x"), fbb.createString("y")}); + int innerDtypes = FbsStruct_.createDtypesVector(fbb, new int[]{dtX, dtY}); + int innerStruct = FbsStruct_.createFbsStruct_(fbb, innerNames, innerDtypes, false); + int dtS = FbsDType.createFbsDType(fbb, FbsType.FbsStruct_, innerStruct); + + int names = FbsStruct_.createNamesVector(fbb, + new int[]{fbb.createString("a"), fbb.createString("s")}); + int dtypes = FbsStruct_.createDtypesVector(fbb, new int[]{dtA, dtS}); + int rootStruct = FbsStruct_.createFbsStruct_(fbb, names, dtypes, false); + int dt = FbsDType.createFbsDType(fbb, FbsType.FbsStruct_, rootStruct); + FbsDType.finishFbsDTypeBuffer(fbb, dt); + return MalformedFiles.slice(fbb); + } + + /// Layout: root struct(2) → [chunked "a" (flat per chunk), struct(2) "s" → [flat x, flat y]]. + /// Both "s" fields are single full-range flats, so the whole subtree is one full-range chunk + /// source alongside "a"'s per-chunk grid. + private static ByteBuffer buildNestedStructLayout(long[] chunkRows, int xSegIdx, int ySegIdx) { + var fbb = new FbsBuilder(1024); + int numChunks = chunkRows.length; + long total = 0; + for (long r : chunkRows) { + total += r; + } + int[] flatOffs = new int[numChunks]; + for (int k = 0; k < numChunks; k++) { + int segV = FbsLayout.createSegmentsVector(fbb, new long[]{k}); + flatOffs[k] = FbsLayout.createFbsLayout(fbb, 0, chunkRows[k], 0, 0, segV); + } + int chunkedChildV = FbsLayout.createChildrenVector(fbb, flatOffs); + int chunkedA = FbsLayout.createFbsLayout(fbb, 1, total, 0, chunkedChildV, 0); + + int xSegV = FbsLayout.createSegmentsVector(fbb, new long[]{xSegIdx}); + int flatX = FbsLayout.createFbsLayout(fbb, 0, total, 0, 0, xSegV); + int ySegV = FbsLayout.createSegmentsVector(fbb, new long[]{ySegIdx}); + int flatY = FbsLayout.createFbsLayout(fbb, 0, total, 0, 0, ySegV); + int sChildV = FbsLayout.createChildrenVector(fbb, new int[]{flatX, flatY}); + int structS = FbsLayout.createFbsLayout(fbb, 2, total, 0, sChildV, 0); + + int structChildV = FbsLayout.createChildrenVector(fbb, new int[]{chunkedA, structS}); + int structOff = FbsLayout.createFbsLayout(fbb, 2, total, 0, structChildV, 0); + FbsLayout.finishFbsLayoutBuffer(fbb, structOff); + return MalformedFiles.slice(fbb); + } + private static List values(Array arr) { List out = new ArrayList<>(); if (arr instanceof MaskedArray masked) { diff --git a/reader/src/test/java/io/github/dfa1/vortex/reader/array/LazyRleArrayTest.java b/reader/src/test/java/io/github/dfa1/vortex/reader/array/LazyRleArrayTest.java index 483d294a..b870dd81 100644 --- a/reader/src/test/java/io/github/dfa1/vortex/reader/array/LazyRleArrayTest.java +++ b/reader/src/test/java/io/github/dfa1/vortex/reader/array/LazyRleArrayTest.java @@ -521,6 +521,28 @@ void indexedClampAndEmptyChunk() { assertThat(empty.getFloat(0)).isZero(); } + @Test + void materializeConstantRunAndEmptyChunk() { + // Given a constant-run chunk (1 distinct value) and, separately, an empty chunk + // (0 distinct values): materialize's processChunk fast path must emit the lone + // value — or 0.0f for the empty pool — for every row without touching indices. + var constant = new LazyRleFloatArray(F32, 3, new float[]{6.25f}, new int[1024], + new long[]{0L}, 0L, 1L, 1, 0); + var empty = new LazyRleFloatArray(F32, 2, new float[0], new int[1024], + new long[]{0L}, 0L, 0L, 1, 0); + + // When / Then + try (var arena = Arena.ofConfined()) { + var constantResult = constant.materialize(arena); + assertThat(constantResult.getAtIndex(VortexFormat.LE_FLOAT, 0)).isEqualTo(6.25f); + assertThat(constantResult.getAtIndex(VortexFormat.LE_FLOAT, 2)).isEqualTo(6.25f); + + var emptyResult = empty.materialize(arena); + assertThat(emptyResult.getAtIndex(VortexFormat.LE_FLOAT, 0)).isEqualTo(0.0f); + assertThat(emptyResult.getAtIndex(VortexFormat.LE_FLOAT, 1)).isEqualTo(0.0f); + } + } + @Test void materializeDecodesEveryRow() { // Given a mixed chunk; materialize must emit each logical row once, honoring diff --git a/reader/src/test/java/io/github/dfa1/vortex/reader/decode/DictEncodingDecoderTest.java b/reader/src/test/java/io/github/dfa1/vortex/reader/decode/DictEncodingDecoderTest.java index 85096013..685fd3b3 100644 --- a/reader/src/test/java/io/github/dfa1/vortex/reader/decode/DictEncodingDecoderTest.java +++ b/reader/src/test/java/io/github/dfa1/vortex/reader/decode/DictEncodingDecoderTest.java @@ -508,6 +508,53 @@ void codeOutOfRangeWithPoolValidity_throwsVortexException() { .hasMessageContaining("out of range for pool validity"); } + @Test + void codesBroadcast_slowPathSetsValidityBits() { + // Given — a single-element codes buffer (codesCap == 1 < rowCount == 4, the + // ConstantEncoding fan-out shape) forces rowValidity's broadcast slow path + // (the `i % codesCap` loop and the setBit S3034 fix site). The lone code points at + // valid pool slot 0, so every row is valid and setBit runs on each iteration. + ArrayNode codesNode = primitiveNode(0); + ArrayNode valuesNode = maskedPrimitiveNode(1, 2); + MemorySegment[] segs = { + u8Codes(0), // 1 code, broadcast across 4 rows + TestSegments.leInts(10, 20), + boolBitmap(true, false) // pool slot 0 valid, slot 1 invalid + }; + + // When + Array result = decodeDict(DType.I32, PType.U8, 2, 4, segs, codesNode, valuesNode); + + // Then — the broadcast code resolves to valid slot 0 for every row + MaskedArray masked = assertMasked(result); + assertValidity(masked, true, true, true, true); + assertIntValues(masked, 10, 10, 10, 10); + } + + @ParameterizedTest(name = "codes={0}") + @org.junit.jupiter.params.provider.EnumSource(value = PType.class, names = {"U16", "U32"}) + void poolNull_readsWiderCodeWidths(PType codePType) { + // Given — codes at the wider widths (U16/U32) with a pool-null slot. rowValidity must + // read each code at the correct stride via readCode's U16/U32 arms (only exercised when + // pool validity is present), then mask rows whose code points at the dead slot. Here + // codesCap == rowCount, so the fast validity loop runs. + ArrayNode codesNode = primitiveNode(0); + ArrayNode valuesNode = maskedPrimitiveNode(1, 2); + MemorySegment[] segs = { + codeSegment(codePType, new long[]{0, 1, 0, 1}), + TestSegments.leInts(10, 20), + boolBitmap(true, false) // slot 1 invalid + }; + + // When + Array result = decodeDict(DType.I32, codePType, 2, 4, segs, codesNode, valuesNode); + + // Then — rows referencing the dead slot 1 are null; their expanded value is still present + MaskedArray masked = assertMasked(result); + assertValidity(masked, true, false, true, false); + assertIntValues(masked, 10, 20, 10, 20); + } + private Array decodeDict(DType dtype, PType codePType, int valuesLen, long rowCount, MemorySegment[] segs, ArrayNode codesNode, ArrayNode valuesNode) { MemorySegment meta = MemorySegment.ofArray( diff --git a/reader/src/test/java/io/github/dfa1/vortex/reader/decode/RleEncodingDecoderTest.java b/reader/src/test/java/io/github/dfa1/vortex/reader/decode/RleEncodingDecoderTest.java new file mode 100644 index 00000000..2aec9f85 --- /dev/null +++ b/reader/src/test/java/io/github/dfa1/vortex/reader/decode/RleEncodingDecoderTest.java @@ -0,0 +1,187 @@ +package io.github.dfa1.vortex.reader.decode; + +import io.github.dfa1.vortex.core.model.DType; +import io.github.dfa1.vortex.core.model.EncodingId; +import io.github.dfa1.vortex.core.proto.ProtoPType; +import io.github.dfa1.vortex.core.proto.ProtoRLEMetadata; +import io.github.dfa1.vortex.encoding.TestSegments; +import io.github.dfa1.vortex.reader.ReadRegistry; +import io.github.dfa1.vortex.reader.array.Array; +import io.github.dfa1.vortex.reader.array.LazyConstantDoubleArray; +import io.github.dfa1.vortex.reader.array.LazyConstantFloatArray; +import io.github.dfa1.vortex.reader.array.LazyRleDoubleArray; +import io.github.dfa1.vortex.reader.array.LazyRleFloatArray; +import org.junit.jupiter.api.Nested; +import org.junit.jupiter.api.Test; + +import java.lang.foreign.Arena; +import java.lang.foreign.MemorySegment; + +import static org.assertj.core.api.Assertions.assertThat; + +/// Decoder-level tests for `fastlanes.rle`. The array records themselves are covered by +/// [io.github.dfa1.vortex.reader.array.LazyRleArrayTest]; here we drive +/// [RleEncodingDecoder#decode] with hand-assembled [ArrayNode]/[DecodeContext] fixtures so the +/// F32/F64 switch arms, the empty-array constant fallback, and the broadcast value helpers are +/// exercised at the seam that [io.github.dfa1.vortex.reader.array.LazyRleArrayTest] cannot reach. +/// +/// Wire shape mirrors what the Python Vortex writer emits for F64/F32 weather columns with long +/// constant runs (issue #209): one FastLanes chunk (indices_len = 1024), a small distinct-values +/// pool, and a single per-chunk offset of 0. +class RleEncodingDecoderTest { + + private static final int FL_CHUNK_SIZE = 1024; + + private static final RleEncodingDecoder SUT = new RleEncodingDecoder(); + private static final ReadRegistry REGISTRY = TestRegistry.ofDecoders( + SUT, new PrimitiveEncodingDecoder()); + + @Test + void encodingId_isFastlanesRle() { + // Given / When / Then + assertThat(SUT.encodingId()).isEqualTo(EncodingId.FASTLANES_RLE); + } + + @Nested + class DoubleDecode { + + @Test + void decodesAcrossRunBoundary() { + // Given — one chunk, pool [1.5, 2.5, 3.5], indices 0,0,1,2: rows 0-1 share value 1.5 + // then two distinct runs. Fractional values catch any accidental long-narrowing read. + MemorySegment values = TestSegments.leDoubles(1.5, 2.5, 3.5); + MemorySegment indices = u8Indices(0, 0, 1, 2); + MemorySegment offsets = TestSegments.leLongs(0); + + // When + Array result = decodeRle(DType.F64, 3, 4, indices, offsets, values, 0); + + // Then — the run boundary (row 1 -> row 2) resolves to the correct pool entries + assertThat(result).isInstanceOf(LazyRleDoubleArray.class); + LazyRleDoubleArray rle = (LazyRleDoubleArray) result; + assertThat(rle.getDouble(0)).isEqualTo(1.5); + assertThat(rle.getDouble(1)).isEqualTo(1.5); + assertThat(rle.getDouble(2)).isEqualTo(2.5); + assertThat(rle.getDouble(3)).isEqualTo(3.5); + } + + @Test + void emptyIndices_yieldsConstantZeroArray() { + // Given — indices_len = 0 short-circuits to emptyArray (L127): the all-null/no-run + // column the writer emits for a fully-empty F64 chunk. + MemorySegment values = TestSegments.leDoubles(9.5); + MemorySegment indices = u8Indices(); + MemorySegment offsets = TestSegments.leLongs(0); + + // When + Array result = decodeRle(DType.F64, 1, 0, indices, offsets, values, 0); + + // Then — a length-0 constant array, not an RLE array + assertThat(result).isInstanceOf(LazyConstantDoubleArray.class); + assertThat(result.length()).isZero(); + } + } + + @Nested + class FloatDecode { + + @Test + void decodesAcrossRunBoundary() { + // Given — one chunk, pool [1.25f, 2.75f], indices 0,0,1: the 4-byte value read must not + // widen to a double; fractional values chosen so a wrong width surfaces as a bad assertion. + MemorySegment values = TestSegments.leFloats(1.25f, 2.75f); + MemorySegment indices = u8Indices(0, 0, 1); + MemorySegment offsets = TestSegments.leLongs(0); + + // When + Array result = decodeRle(DType.F32, 2, 3, indices, offsets, values, 0); + + // Then + assertThat(result).isInstanceOf(LazyRleFloatArray.class); + LazyRleFloatArray rle = (LazyRleFloatArray) result; + assertThat(rle.getFloat(0)).isEqualTo(1.25f); + assertThat(rle.getFloat(1)).isEqualTo(1.25f); + assertThat(rle.getFloat(2)).isEqualTo(2.75f); + } + + @Test + void emptyIndices_yieldsConstantZeroArray() { + // Given — indices_len = 0 short-circuits to emptyArray (L128) for an empty F32 chunk + MemorySegment values = TestSegments.leFloats(9.5f); + MemorySegment indices = u8Indices(); + MemorySegment offsets = TestSegments.leLongs(0); + + // When + Array result = decodeRle(DType.F32, 1, 0, indices, offsets, values, 0); + + // Then + assertThat(result).isInstanceOf(LazyConstantFloatArray.class); + assertThat(result.length()).isZero(); + } + } + + /// The broadcast branch of `readDoubles`/`readFloats` (cap < count) is unreachable through + /// `decode()` — a ConstantEncoding values child always materializes to `count` full elements, + /// so cap == count. These tests exercise the helper directly on a single-element pool, the + /// exact zip-bomb-defense shape [SegmentBroadcast] guards against. + @Nested + class ValueBroadcast { + + @Test + void readDoubles_broadcastsLoneElementAcrossCount() { + // Given — one physical double but a requested count of 4 (cap = 1 < count) + MemorySegment buf = TestSegments.leDoubles(7.25); + + // When + double[] result = RleEncodingDecoder.readDoubles(buf, 4); + + // Then — the lone value repeats for every logical index + assertThat(result).containsExactly(7.25, 7.25, 7.25, 7.25); + } + + @Test + void readFloats_broadcastsLoneElementAcrossCount() { + // Given — one physical float but a requested count of 3 (cap = 1 < count) + MemorySegment buf = TestSegments.leFloats(-3.5f); + + // When + float[] result = RleEncodingDecoder.readFloats(buf, 3); + + // Then + assertThat(result).containsExactly(-3.5f, -3.5f, -3.5f); + } + } + + // ── decode harness ───────────────────────────────────────────────────────── + + /// Builds a single-chunk RLE node: child[0] = values, child[1] = indices (U8), + /// child[2] = values-idx-offsets (U64), and drives [RleEncodingDecoder#decode]. + private static Array decodeRle(DType dtype, int valuesLen, long rowCount, + MemorySegment indices, MemorySegment offsets, MemorySegment values, int offset) { + long indicesLen = indices.byteSize(); // U8 indices: 1 byte per element + long offsetsLen = offsets.byteSize() / 8; // U64 offsets + byte[] metaBytes = new ProtoRLEMetadata( + valuesLen, indicesLen, ProtoPType.U8, offsetsLen, ProtoPType.U64, offset).encode(); + MemorySegment meta = MemorySegment.ofArray(metaBytes); + MemorySegment[] segs = {values, indices, offsets}; + ArrayNode node = new ArrayNode(EncodingId.FASTLANES_RLE, meta, + new ArrayNode[]{primitiveNode(0), primitiveNode(1), primitiveNode(2)}, new int[]{}); + DecodeContext ctx = new DecodeContext(node, dtype, rowCount, segs, REGISTRY, Arena.ofAuto()); + return SUT.decode(ctx); + } + + private static ArrayNode primitiveNode(int bufferIndex) { + return new ArrayNode(EncodingId.VORTEX_PRIMITIVE, null, new ArrayNode[0], new int[]{bufferIndex}); + } + + /// A full 1024-byte (one FastLanes chunk) U8 index buffer with the leading entries set; + /// an empty varargs list yields the length-0 buffer that drives the empty-array path. + private static MemorySegment u8Indices(int... leading) { + int len = leading.length == 0 ? 0 : FL_CHUNK_SIZE; + byte[] a = new byte[len]; + for (int i = 0; i < leading.length; i++) { + a[i] = (byte) leading[i]; + } + return MemorySegment.ofArray(a); + } +} diff --git a/reader/src/test/java/io/github/dfa1/vortex/reader/layout/DictLayoutDecoderTest.java b/reader/src/test/java/io/github/dfa1/vortex/reader/layout/DictLayoutDecoderTest.java index 2cd117f4..8aa042d6 100644 --- a/reader/src/test/java/io/github/dfa1/vortex/reader/layout/DictLayoutDecoderTest.java +++ b/reader/src/test/java/io/github/dfa1/vortex/reader/layout/DictLayoutDecoderTest.java @@ -8,11 +8,17 @@ import io.github.dfa1.vortex.reader.SegmentSpec; import io.github.dfa1.vortex.reader.array.Array; import io.github.dfa1.vortex.reader.array.BoolArray; +import io.github.dfa1.vortex.reader.array.ByteArray; +import io.github.dfa1.vortex.reader.array.DictByteArray; +import io.github.dfa1.vortex.reader.array.DictShortArray; import io.github.dfa1.vortex.reader.array.IntArray; import io.github.dfa1.vortex.reader.array.MaskedArray; import io.github.dfa1.vortex.reader.array.MaterializedBoolArray; import io.github.dfa1.vortex.reader.array.MaterializedByteArray; import io.github.dfa1.vortex.reader.array.MaterializedIntArray; +import io.github.dfa1.vortex.reader.array.MaterializedLongArray; +import io.github.dfa1.vortex.reader.array.MaterializedShortArray; +import io.github.dfa1.vortex.reader.array.ShortArray; import org.junit.jupiter.api.Test; import java.lang.foreign.Arena; @@ -105,6 +111,89 @@ void noValidityEitherSide_returnsPlainDictArray() { assertThat(result).isInstanceOf(IntArray.class).isNotInstanceOf(MaskedArray.class); } + @Test + void bytePool_buildsDictByteArray() { + // Given — an I8/U8 dictionary pool: buildLazyDictPrimitive must dispatch to DictByteArray, + // the narrow-value arm untested until now (real files use byte pools for tiny enum columns). + DType u8 = new DType.Primitive(PType.U8, true); + Array values = bytePool(11, 22, 33); + Array codes = byteCodes(2, 0, 1); + + // When + Array result = decode(3, u8, values, codes); + + // Then — a lazy DictByteArray resolving each row through its code + assertThat(result).isInstanceOf(DictByteArray.class); + ByteArray bytes = (ByteArray) result; + assertThat(bytes.getByte(0)).isEqualTo((byte) 33); + assertThat(bytes.getByte(1)).isEqualTo((byte) 11); + assertThat(bytes.getByte(2)).isEqualTo((byte) 22); + } + + @Test + void shortPool_buildsDictShortArray() { + // Given — an I16/U16 dictionary pool: dispatches to DictShortArray (the 2-byte value arm) + DType i16 = new DType.Primitive(PType.I16, true); + Array values = shortPool(-100, 200, 300); + Array codes = byteCodes(0, 2, 1); + + // When + Array result = decode(3, i16, values, codes); + + // Then + assertThat(result).isInstanceOf(DictShortArray.class); + ShortArray shorts = (ShortArray) result; + assertThat(shorts.getShort(0)).isEqualTo((short) -100); + assertThat(shorts.getShort(1)).isEqualTo((short) 300); + assertThat(shorts.getShort(2)).isEqualTo((short) 200); + } + + @Test + void poolNull_gathersShortCodes() { + // Given — pool-null validity with U16-width codes (a ShortArray): gatherRowValidity's + // codes-type switch must take the ShortArray arm to read each code before masking. + Array values = new MaskedArray(intPool(10, 20, 30), boolArray(true, false, true)); + Array codes = shortCodes(0, 1, 2, 1); + + // When + Array result = decode(4, I32, values, codes); + + // Then — rows referencing the dead slot 1 are null + MaskedArray masked = assertMasked(result); + assertValidity(masked, true, false, true, false); + assertIntValues(masked, 10, 20, 30, 20); + } + + @Test + void poolNull_gathersIntCodes() { + // Given — pool-null validity with U32-width codes (an IntArray): the IntArray gather arm + Array values = new MaskedArray(intPool(10, 20, 30), boolArray(true, false, true)); + Array codes = intCodes(0, 1, 2, 1); + + // When + Array result = decode(4, I32, values, codes); + + // Then + MaskedArray masked = assertMasked(result); + assertValidity(masked, true, false, true, false); + assertIntValues(masked, 10, 20, 30, 20); + } + + @Test + void poolNull_gathersLongCodes() { + // Given — pool-null validity with U64-width codes (a LongArray): the LongArray gather arm + Array values = new MaskedArray(intPool(10, 20, 30), boolArray(true, false, true)); + Array codes = longCodes(0, 1, 2, 1); + + // When + Array result = decode(4, I32, values, codes); + + // Then + MaskedArray masked = assertMasked(result); + assertValidity(masked, true, false, true, false); + assertIntValues(masked, 10, 20, 30, 20); + } + // ── harness ───────────────────────────────────────────────────────────────── private static Array decode(long n, DType dtype, Array values, Array codes) { @@ -130,6 +219,43 @@ private static Array byteCodes(int... codes) { MemorySegment.ofArray(bytes)); } + private static Array shortCodes(int... codes) { + short[] shorts = new short[codes.length]; + for (int i = 0; i < codes.length; i++) { + shorts[i] = (short) codes[i]; + } + return new MaterializedShortArray(new DType.Primitive(PType.U16, false), codes.length, + TestSegments.leShorts(shorts)); + } + + private static Array intCodes(int... codes) { + return new MaterializedIntArray(new DType.Primitive(PType.U32, false), codes.length, + TestSegments.leInts(codes)); + } + + private static Array longCodes(long... codes) { + return new MaterializedLongArray(new DType.Primitive(PType.U64, false), codes.length, + TestSegments.leLongs(codes)); + } + + private static ByteArray bytePool(int... values) { + byte[] bytes = new byte[values.length]; + for (int i = 0; i < values.length; i++) { + bytes[i] = (byte) values[i]; + } + return new MaterializedByteArray(new DType.Primitive(PType.U8, true), values.length, + MemorySegment.ofArray(bytes)); + } + + private static ShortArray shortPool(int... values) { + short[] shorts = new short[values.length]; + for (int i = 0; i < values.length; i++) { + shorts[i] = (short) values[i]; + } + return new MaterializedShortArray(new DType.Primitive(PType.I16, true), values.length, + TestSegments.leShorts(shorts)); + } + private static BoolArray boolArray(boolean... valid) { byte[] bytes = new byte[(valid.length + 7) / 8]; for (int i = 0; i < valid.length; i++) { diff --git a/reader/src/test/java/io/github/dfa1/vortex/reader/layout/ZonedStatsSchemaTest.java b/reader/src/test/java/io/github/dfa1/vortex/reader/layout/ZonedStatsSchemaTest.java index 71780efa..e52403ac 100644 --- a/reader/src/test/java/io/github/dfa1/vortex/reader/layout/ZonedStatsSchemaTest.java +++ b/reader/src/test/java/io/github/dfa1/vortex/reader/layout/ZonedStatsSchemaTest.java @@ -347,6 +347,58 @@ void returnsEmptyStructWhenNoAggregatesPresent() { assertThat(result).isNotNull(); assertThat(result.fieldNames()).isEmpty(); } + + @Test + void buildsBooleanAndSizeStatColumns() { + // Given — the boolean/size aggregates Rust can emit (is_constant/is_sorted/ + // is_strict_sorted → nullable Bool; uncompressed_size_in_bytes → nullable U64). These + // exercise the statForAggregate arms that the max/min/sum/count tests never reach. + MemorySegment meta = aggregateMeta(2048, + "vortex.is_constant", "vortex.is_sorted", "vortex.is_strict_sorted", + "vortex.uncompressed_size_in_bytes"); + + // When + DType.Struct result = ZonedStatsSchema.aggregateStatsTableDtype(DType.I64, meta); + + // Then + assertThat(result.fieldNames().stream().map(ColumnName::value).toList()) + .containsExactly("is_constant", "is_sorted", "is_strict_sorted", + "uncompressed_size_in_bytes"); + assertThat(result.fieldTypes()).containsExactly( + DType.BOOL.withNullable(true), + DType.BOOL.withNullable(true), + DType.BOOL.withNullable(true), + DType.U64.withNullable(true)); + } + + @Test + void skipsFixedWidthFieldsBeforeSpec() { + // Given — an envelope carrying unknown 64-bit and 32-bit fixed-width fields (wire types + // I64 and I32) ahead of a real aggregate spec. A forward-compatible reader must skip both + // fixed-width fields (the ProtoCursor WIRE_I64/WIRE_I32 skip arms) and still read the spec. + java.io.ByteArrayOutputStream out = new java.io.ByteArrayOutputStream(); + out.write(1); // envelope version + writeTag(out, 3, 1); // field 3, wire I64 → advance(8) + out.writeBytes(new byte[8]); + writeTag(out, 4, 5); // field 4, wire I32 → advance(4) + out.writeBytes(new byte[4]); + byte[] idBytes = "vortex.max".getBytes(java.nio.charset.StandardCharsets.UTF_8); + java.io.ByteArrayOutputStream spec = new java.io.ByteArrayOutputStream(); + writeTag(spec, 1, 2); // id, wire LEN + writeVarint(spec, idBytes.length); + spec.writeBytes(idBytes); + writeTag(out, 2, 2); // aggregate_specs, wire LEN + writeVarint(out, spec.size()); + out.writeBytes(spec.toByteArray()); + + // When + DType.Struct result = ZonedStatsSchema.aggregateStatsTableDtype( + DType.I64, MemorySegment.ofArray(out.toByteArray())); + + // Then — the fixed-width fields are skipped and only the max spec survives + assertThat(result.fieldNames().stream().map(ColumnName::value).toList()) + .containsExactly("max"); + } } @Nested @@ -423,6 +475,68 @@ void bailsOnUnknownWireType() { assertThat(result).isNull(); } + @Test + void bailsOnOverlongVarintOverflow() { + // Given — a tag varint of ten continuation bytes: it never terminates and pushes the + // shift past 63 bits. readVarint must return -1 (the > 63-bit overflow arm) rather than + // silently wrapping, so the caller bails. + java.io.ByteArrayOutputStream out = new java.io.ByteArrayOutputStream(); + out.write(1); // envelope version + for (int i = 0; i < 10; i++) { + out.write(0x80); // continuation bit set, no terminating byte + } + + // When + DType.Struct result = ZonedStatsSchema.aggregateStatsTableDtype( + DType.I64, MemorySegment.ofArray(out.toByteArray())); + + // Then + assertThat(result).isNull(); + } + + @Test + void bailsOnOverflowTagInsideSpec() { + // Given — a framed AggregateSpecProto whose first inner tag is a ten-byte overflow + // varint. Inside readAggregateSpecId the tag reads as -1, so it must bail (the inner + // `tag < 0` arm) rather than treat the negative value as a field number. + byte[] specBytes = new byte[10]; + java.util.Arrays.fill(specBytes, (byte) 0x80); + java.io.ByteArrayOutputStream out = new java.io.ByteArrayOutputStream(); + out.write(1); // envelope version + writeTag(out, 2, 2); // aggregate_specs, wire LEN + writeVarint(out, specBytes.length); + out.writeBytes(specBytes); + + // When + DType.Struct result = ZonedStatsSchema.aggregateStatsTableDtype( + DType.I64, MemorySegment.ofArray(out.toByteArray())); + + // Then + assertThat(result).isNull(); + } + + @Test + void bailsOnUnknownWireTypeInsideSpec() { + // Given — an AggregateSpecProto whose inner field uses wire type 3 (group start), which + // the cursor cannot skip. readAggregateSpecId must bail on the failed skipField, not + // loop forever or misread. + java.io.ByteArrayOutputStream spec = new java.io.ByteArrayOutputStream(); + writeTag(spec, 2, 3); // inner field 2, wire type 3 (unsupported) + byte[] specBytes = spec.toByteArray(); + java.io.ByteArrayOutputStream out = new java.io.ByteArrayOutputStream(); + out.write(1); // envelope version + writeTag(out, 2, 2); // aggregate_specs, wire LEN + writeVarint(out, specBytes.length); + out.writeBytes(specBytes); + + // When + DType.Struct result = ZonedStatsSchema.aggregateStatsTableDtype( + DType.I64, MemorySegment.ofArray(out.toByteArray())); + + // Then + assertThat(result).isNull(); + } + @Test void bailsOnMessageLengthPastEnd() { // Given — an aggregate_specs field whose declared length runs past the buffer end.