From 9f1f8a2f92a4b2a13d4006408340590416912889 Mon Sep 17 00:00:00 2001 From: Davide Angelocola Date: Sat, 25 Jul 2026 13:05:20 +0200 Subject: [PATCH] feat(writer): add ALP-RD to the cascade competition (#304) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit AlpRdEncodingEncoder was registered on WriteRegistry but never added to the top-level cascade codec list, so it could only appear as a child encoding, never be selected for an F64/F32 column. High-precision floats that plain ALP cannot fit without an exception on nearly every row (nyc-311 Latitude) therefore fell back to raw vortex.primitive (or dict), while the Rust reference uses vortex.alprd. Add it as a top-level candidate next to ALP. It competes on measured size, so it only wins where it is actually smaller (Latitude flips to alprd; Longitude keeps plain alp). No wire/reader change — alprd is already decodable (the reader reads Rust files that use it). nyc-311 re-encode: 1678.86 MB -> 1644.25 MB (0.95x -> 0.93x vortex-jni). Round-trip + ALP-RD-selection integration test added; 217 Java-writes-Rust-reads interop tests pass; full verify green. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../AlpRdCascadeSelectionIntegrationTest.java | 79 +++++++++++++++++++ .../dfa1/vortex/writer/VortexWriter.java | 6 ++ 2 files changed, 85 insertions(+) create mode 100644 integration/src/test/java/io/github/dfa1/vortex/integration/AlpRdCascadeSelectionIntegrationTest.java diff --git a/integration/src/test/java/io/github/dfa1/vortex/integration/AlpRdCascadeSelectionIntegrationTest.java b/integration/src/test/java/io/github/dfa1/vortex/integration/AlpRdCascadeSelectionIntegrationTest.java new file mode 100644 index 00000000..1982ad97 --- /dev/null +++ b/integration/src/test/java/io/github/dfa1/vortex/integration/AlpRdCascadeSelectionIntegrationTest.java @@ -0,0 +1,79 @@ +package io.github.dfa1.vortex.integration; + +import io.github.dfa1.vortex.core.model.ColumnName; +import io.github.dfa1.vortex.core.model.DType; +import io.github.dfa1.vortex.core.model.PType; +import io.github.dfa1.vortex.inspect.InspectorTree; +import io.github.dfa1.vortex.reader.ReadRegistry; +import io.github.dfa1.vortex.reader.ScanOptions; +import io.github.dfa1.vortex.reader.VortexReader; +import io.github.dfa1.vortex.reader.array.DoubleArray; +import io.github.dfa1.vortex.writer.VortexWriter; +import io.github.dfa1.vortex.writer.WriteOptions; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import java.io.IOException; +import java.nio.channels.FileChannel; +import java.nio.file.Path; +import java.nio.file.StandardOpenOption; +import java.util.List; +import java.util.Map; +import java.util.Random; + +import static org.assertj.core.api.Assertions.assertThat; + +/// High-precision F64 columns that plain ALP cannot fit (too many exceptions) must be able to pick +/// ALP-RD in the cascade competition, matching the Rust reference — before #304 ALP-RD was +/// registered but not a top-level cascade candidate, so such columns fell back to raw primitive. +class AlpRdCascadeSelectionIntegrationTest { + + private static final ColumnName COL = ColumnName.of("lat"); + private static final DType.Struct SCHEMA = new DType.Struct( + List.of(COL), List.of(new DType.Primitive(PType.F64, false)), false); + + @Test + void highPrecisionF64_selectsAlpRd_andRoundTrips(@TempDir Path tmp) throws IOException { + // Given — NYC-latitude-like doubles: a tight value range (shared high/exponent bits) with + // full random mantissas (so plain ALP produces exceptions on nearly every row, but ALP-RD's + // left-parts dictionary captures the shared bits). All distinct, so dict is skipped too. + Path file = tmp.resolve("lat.vortex"); + var rng = new Random(42L); + int rows = 60_000; + double[] data = new double[rows]; + for (int i = 0; i < rows; i++) { + data[i] = 40.5 + rng.nextDouble() * 0.4; + } + + try (var ch = FileChannel.open(file, StandardOpenOption.CREATE, StandardOpenOption.WRITE); + var sut = VortexWriter.create(ch, SCHEMA, WriteOptions.cascading(3))) { + // When + sut.writeChunk(Map.of(COL, data)); + } + + // Then — the cascade picks ALP-RD. + try (var vf = VortexReader.open(file, ReadRegistry.loadAll())) { + assertThat(InspectorTree.build(vf).usedEncodings()) + .as("high-precision F64 selects ALP-RD via the cascade") + .contains("vortex.alprd"); + } + + // And every value round-trips exactly. + try (var vf = VortexReader.open(file, ReadRegistry.loadAll())) { + double[] got = new double[rows]; + int[] idx = {0}; + try (var iter = vf.scan(ScanOptions.columns("lat"))) { + while (iter.hasNext()) { + try (var chunk = iter.next()) { + DoubleArray col = chunk.column("lat"); + long n = col.length(); + for (long i = 0; i < n; i++) { + got[idx[0]++] = col.getDouble(i); + } + } + } + } + assertThat(got).containsExactly(data); + } + } +} diff --git a/writer/src/main/java/io/github/dfa1/vortex/writer/VortexWriter.java b/writer/src/main/java/io/github/dfa1/vortex/writer/VortexWriter.java index 97012d62..c6d5038c 100644 --- a/writer/src/main/java/io/github/dfa1/vortex/writer/VortexWriter.java +++ b/writer/src/main/java/io/github/dfa1/vortex/writer/VortexWriter.java @@ -20,6 +20,7 @@ import io.github.dfa1.vortex.writer.encode.StructEncodingEncoder; import io.github.dfa1.vortex.writer.encode.EncodingEncoder; import io.github.dfa1.vortex.writer.encode.AlpEncodingEncoder; +import io.github.dfa1.vortex.writer.encode.AlpRdEncodingEncoder; import io.github.dfa1.vortex.writer.encode.BitpackedEncodingEncoder; import io.github.dfa1.vortex.writer.encode.BoolEncodingEncoder; import io.github.dfa1.vortex.writer.encode.CascadingCompressor; @@ -222,6 +223,11 @@ private static List buildCascadeCodecs(WriteOptions options) { codecs.add(new ListEncodingEncoder()); codecs.add(new ConstantEncodingEncoder()); codecs.add(new AlpEncodingEncoder()); + // ALP-RD competes for high-precision F64/F32 that plain ALP can't fit without too many + // exceptions (e.g. nyc-311 Latitude): without it in the competition such columns fall back + // to raw vortex.primitive or dict, matching neither the Rust reference (which uses alprd) nor + // its size (#304). Registered on WriteRegistry already; this adds it as a top-level candidate. + codecs.add(new AlpRdEncodingEncoder()); codecs.add(new FrameOfReferenceEncodingEncoder()); codecs.add(new RunEndEncodingEncoder()); codecs.add(new RleEncodingEncoder());