diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0a58330..2afa63d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -59,3 +59,79 @@ jobs: exit 1 fi echo "conformance/ matches upstream" + + # The port exists to be wire compatible; everything else is detail. This job is + # the only place that claim is MEASURED against the reference itself: the exact + # release candidate in this repository exchanges bytes with the exact C++ + # library, in both directions, on every push and pull request. Nothing here is a + # local one-off, and no expectation is regenerated from this port's own codec. + # + # What is exchanged: interop/serialize/interop/Interop.java and + # interop/interop.cpp serialize the SAME boundary message -- every operation + # STANDARD.md defines, at the values where implementations disagree. Both halves + # write it and the files must be byte identical; each half then decodes the + # other's file, checks every value and re-encodes it byte for byte. Then the + # hostile half: every proper prefix of the other's stream is a truncated stream + # and both must refuse it, and both run the shared conformance corpus, where the + # refusal vectors are the point. + # + # THE FAMILY PIN -- ONE POLICY, ONE VERSION. The C++ checkout is pinned to a + # RELEASED tag of mas-bandwidth/serialize, never main, and to the SAME tag in + # every port's wire-compat gate, so "compatible with C++" means the same thing + # in all of them. Never main, because an upstream merge mid-day must not change + # what a commit's CI proves. Find the newest tag with: + # gh release list --repo mas-bandwidth/serialize --limit 1 + # Bump family-wide, deliberately, in its own commit per repo. SERIALIZE_TAG + # below is the ONE place this repository names a version of the reference. + # (The spec-sync job above deliberately stays on upstream main: drift DETECTION + # is that job's whole purpose.) + interop: + name: interop with the C++ reference + runs-on: ubuntu-latest + env: + SERIALIZE_TAG: v1.16.0 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-java@v4 + with: + distribution: temurin + java-version: 21 + + - name: Check out the C++ reference at the pinned release + run: git clone --quiet --depth 1 --branch "$SERIALIZE_TAG" https://github.com/mas-bandwidth/serialize.git /tmp/cpp-serialize + + # Asserts stay ON (no -DNDEBUG): they are the C++ half of "API misuse + # panics", and the degenerate ranges the message carries must pass with the + # library's own checks enabled rather than around them. No -ffp-contract=off + # either: the compressed float arithmetic is pinned in the library's source, + # and building with default flags proves that rather than masking it. + - name: Build the C++ halves against the pinned library + run: | + c++ -O2 -std=c++17 -Wall -I /tmp/cpp-serialize -o /tmp/cpp_interop interop/interop.cpp + c++ -O2 -std=c++17 -Wall -I /tmp/cpp-serialize -o /tmp/cpp_conformance /tmp/cpp-serialize/conformance.cpp + + - name: Both halves write byte identical wire data + run: | + /tmp/cpp_interop write /tmp/cpp.bin + make interop JDK_HOME="$JAVA_HOME" MODE=write FILE=/tmp/java.bin + cmp /tmp/cpp.bin /tmp/java.bin + + - name: Each half decodes the other's bytes and re-encodes them exactly + run: | + make interop JDK_HOME="$JAVA_HOME" MODE=read FILE=/tmp/cpp.bin + /tmp/cpp_interop read /tmp/java.bin + + - name: Both halves refuse every truncation of the other's stream + run: | + make interop JDK_HOME="$JAVA_HOME" MODE=refuse FILE=/tmp/cpp.bin + /tmp/cpp_interop refuse /tmp/java.bin + + # The corpus is the family's one conformance instrument, and its refusal + # vectors are what this step is for: the pinned reference reader and this + # port's reader are held to the same accepts and the same refusals, from the + # same vendored files. The spec-sync job above proves those files match + # upstream. + - name: Both halves run the shared conformance corpus + run: | + /tmp/cpp_conformance conformance/*.txt + make conformance JDK_HOME="$JAVA_HOME" diff --git a/Makefile b/Makefile index 9cdad97..0d794bd 100644 --- a/Makefile +++ b/Makefile @@ -6,12 +6,14 @@ JAVAC := $(JDK_HOME)/bin/javac JAVA := $(JDK_HOME)/bin/java SRC := $(wildcard src/serialize/*.java) -TEST_SRC := $(wildcard test/serialize/tests/*.java) +TEST_SRC := $(wildcard test/serialize/tests/*.java) +INTEROP_SRC := $(wildcard interop/serialize/interop/*.java) -CLASSES := build/classes -TEST_CLASSES := build/test-classes +CLASSES := build/classes +TEST_CLASSES := build/test-classes +INTEROP_CLASSES := build/interop-classes -.PHONY: all test test-release clean +.PHONY: all test test-release conformance interop clean all: test test-release @@ -37,5 +39,23 @@ test: $(TEST_CLASSES)/.stamp test-release: $(TEST_CLASSES)/.stamp $(JAVA) -da -cp $(CLASSES):$(TEST_CLASSES) serialize.tests.AllTests --release +# the shared conformance corpus alone, so the interop job can hold this reader +# and the pinned C++ reader to the same vendored files in one place. make test +# runs it too, as one suite among many. +conformance: $(TEST_CLASSES)/.stamp + $(JAVA) -ea -cp $(CLASSES):$(TEST_CLASSES) serialize.tests.ConformanceTests + +# the interop harness: compiled against the library classes only, like the tests. +# One exchange with the C++ reference per invocation: +# make interop MODE=write FILE=/tmp/java.bin +$(INTEROP_CLASSES)/.stamp: $(INTEROP_SRC) $(CLASSES)/.stamp + $(JAVAC) --release 17 -Xlint:all -Werror -cp $(CLASSES) -d $(INTEROP_CLASSES) $(INTEROP_SRC) + @touch $@ + +# asserts on: the write-side contracts are asserts here as everywhere, and the +# degenerate ranges the message carries must pass with them enabled +interop: $(INTEROP_CLASSES)/.stamp + $(JAVA) -ea -cp $(CLASSES):$(INTEROP_CLASSES) serialize.interop.Interop $(MODE) $(FILE) + clean: rm -rf build diff --git a/README.md b/README.md index 8864acd..f5d77aa 100644 --- a/README.md +++ b/README.md @@ -17,7 +17,7 @@ specification in [mas-bandwidth/serialize](https://github.com/mas-bandwidth/serialize), which CI checks for drift — is the authority on every byte. -Version 1.1.0 (`SerializeUtil.VERSION`). +Version 1.1.1 (`SerializeUtil.VERSION`). ## The surface @@ -129,6 +129,17 @@ plus a sabotage sweep proving every consumed bit of the golden stream is load bearing, refusal and terminality proofs for hostile input, and the measure bound. +[`interop/`](interop) takes it further: the CI `interop` job builds the +C++ reference at a pinned release and runs it head to head with this +port. Both halves write the same boundary message — every operation the +standard defines, at its boundary values — and the files must be byte +identical; each then decodes the other's bytes and re-encodes them +exactly; both must refuse every truncation of the other's stream; and +both run the corpus. The release candidate in this repository exchanges +bytes with the reference on every push, so wire compatibility is +measured rather than asserted. `make interop MODE=write FILE=out.bin` +runs one exchange by hand. + Benchmarking for the serialize family lives in [mas-bandwidth/schema](https://github.com/mas-bandwidth/schema)'s data-driven bench, which measures the generated codecs across every language on one corpus. ## License diff --git a/interop/interop.cpp b/interop/interop.cpp new file mode 100644 index 0000000..5fbe0a1 --- /dev/null +++ b/interop/interop.cpp @@ -0,0 +1,732 @@ +/* + The C++ half of the cross language interop harness. + + Built in CI against the real C++ serialize library (github.com/mas-bandwidth/serialize) + at the release the workflow pins, and run head to head with this port's half. Neither + half is a local one-off: the exact release candidate in this repository exchanges bytes + with the exact reference, on every push and pull request. + + interop write write the boundary message and hand the bytes over + interop read decode the other half's bytes, check every value, re-encode + and require the result to be byte identical to the input + interop refuse every proper prefix of the other half's bytes is a truncated + stream, and every one of them must be REFUSED + + THE MESSAGE is the boundary set: every operation STANDARD.md defines, at the values where + implementations disagree. Zero bit ranges on all three ranged widths and on fixed point; + the domain edges of int, int64, int128 and int_relative; the maximum widths of bits, + uint128 and the four group fixed point path; both sides of the alignment rule, including + the align inside a zero length bytes; empty and full strings; and the wide string cases + the surrogate rule governs, up to the largest code unit. Each section is preceded by an + align, so a divergence localizes to a section instead of shifting every byte after it. + + WHAT IT DELIBERATELY DOES NOT CARRY: a NaN payload. STANDARD.md's bit transparency claim + covers it, but a NaN's payload bits do not survive every language's float type on the way + to the wire, so a difference here would say nothing about the wire format. Each port pins + its own NaN patterns in its own suite, where the claim can be tested honestly. + + Build with asserts ON (no -DNDEBUG): they are the C++ half of "API misuse panics", and the + degenerate ranges must pass with the library's own checks enabled rather than around them. + + The library tag is pinned in ONE place, .github/workflows/ci.yml, and this file names no + version. Any change to the sequence below must be mirrored in this port's half, and never + changes the wire format. +*/ + +#include "serialize.h" + +#include +#include +#include +#include + +// --------------------------------------------------------------------------------------- +// section 1: raw bit groups, every width boundary + +struct BitsVector { int bits; uint64_t value; }; + +static const BitsVector bits_vectors[] = +{ + { 1, 1 }, // the minimum width, at its maximum value + { 1, 0 }, // and at its minimum + { 7, 0x7F }, // a sub-byte width, all ones + { 31, 0x7FFFFFFF }, // one below the single group maximum + { 32, 0xFFFFFFFFULL }, // the widest single group, all ones + { 32, 0 }, // and all zeros + { 33, 0x1FFFFFFFFULL }, // the first width past the 32 bit split + { 64, 0xFFFFFFFFFFFFFFFFULL }, // the maximum width, all ones + { 64, 0 }, // and all zeros +}; +static const int bits_count = (int) ( sizeof( bits_vectors ) / sizeof( bits_vectors[0] ) ); + +// --------------------------------------------------------------------------------------- +// section 3: the fixed width unsigned helpers at their domain edges + +static const uint32_t uint8_values[] = { 0x00u, 0xFFu }; +static const uint32_t uint16_values[] = { 0x0000u, 0xFFFFu }; +static const uint32_t uint32_values[] = { 0x00000000u, 0xFFFFFFFFu }; +static const uint64_t uint64_values[] = { 0ULL, 0xFFFFFFFFFFFFFFFFULL }; +static const int uint8_count = (int) ( sizeof( uint8_values ) / sizeof( uint8_values[0] ) ); +static const int uint16_count = (int) ( sizeof( uint16_values ) / sizeof( uint16_values[0] ) ); +static const int uint32_count = (int) ( sizeof( uint32_values ) / sizeof( uint32_values[0] ) ); +static const int uint64_count = (int) ( sizeof( uint64_values ) / sizeof( uint64_values[0] ) ); + +static const int uint128_count = 3; + +// --------------------------------------------------------------------------------------- +// sections 5 to 7: the ranged integers + +struct IntVector { int32_t min; int32_t max; int32_t value; }; + +static const IntVector int_vectors[] = +{ + { 42, 42, 42 }, // degenerate: zero bits, mid sequence + { -100, +100, -100 }, // the bottom of the range + { -100, +100, +100 }, // the top of the range + { INT32_MIN, INT32_MAX, INT32_MIN }, // the full domain, 32 bits on the wire + { INT32_MIN, INT32_MAX, INT32_MAX }, + { -100, +100, -37 }, // a live field after the degenerate one +}; +static const int int_count = (int) ( sizeof( int_vectors ) / sizeof( int_vectors[0] ) ); + +struct Int64Vector { int64_t min; int64_t max; int64_t value; }; + +static const Int64Vector int64_vectors[] = +{ + { 10000000000LL, 10000000000LL, 10000000000LL }, // degenerate, with bounds past 2^32 + { -5000000000LL, +5000000000LL, -5000000000LL }, // a range wider than 32 bits, bottom + { -5000000000LL, +5000000000LL, +5000000000LL }, // and top + { INT64_MIN, INT64_MAX, INT64_MIN }, // the full domain, 64 bits on the wire + { INT64_MIN, INT64_MAX, INT64_MAX }, +}; +static const int int64_count = (int) ( sizeof( int64_vectors ) / sizeof( int64_vectors[0] ) ); + +static const int int128_count = 4; + +// the 128 bit constants are built rather than tabulated: serialize::int128_t is native +// __int128 where the compiler has it and the emulated pair where it does not, and only one +// of those is a literal type. +static serialize::int128_t int128_min_value() +{ + return serialize::int128_t( serialize::uint128_t( 1 ) << 127 ); +} + +static serialize::int128_t int128_max_value() +{ + return ~int128_min_value(); +} + +// 2^100 + 7: a degenerate bound that no 64 bit path can carry +static serialize::int128_t int128_degenerate_value() +{ + return ( serialize::int128_t( 1 ) << 100 ) + serialize::int128_t( 7 ); +} + +// --------------------------------------------------------------------------------------- +// section 9: int_relative, every tier at both ends and the domain edges + +struct RelativeVector { int32_t previous; int32_t current; }; + +static const RelativeVector relative_vectors[] = +{ + { 0, 1 }, // one-bit + { 0, 2 }, { 0, 6 }, // bounded-3, both ends + { 0, 7 }, { 0, 23 }, // bounded-5 + { 0, 24 }, { 0, 280 }, // bounded-9 + { 0, 281 }, { 0, 4377 }, // bounded-13 + { 0, 4378 }, { 0, 69914 }, // bounded-17 + { 0, 69915 }, // absolute, at its smallest difference + { 2147483646, 2147483647 }, // one-bit, at the top of the domain + { 0, 2147483647 }, // absolute, at the top of the domain +}; +static const int relative_count = (int) ( sizeof( relative_vectors ) / sizeof( relative_vectors[0] ) ); + +// --------------------------------------------------------------------------------------- +// section 10: float and double, given as bit patterns so no decimal literal is parsed twice + +static const uint32_t float_bits[] = +{ + 0x00000000u, // +0 + 0x80000000u, // -0 + 0x7F800000u, // +infinity + 0xFF800000u, // -infinity + 0x7F7FFFFFu, // the largest finite float32 + 0x00800000u, // the smallest normal + 0x00000001u, // the smallest subnormal + 0x3F800000u, // 1.0f + 0xBF800000u, // -1.0f +}; +static const int float_count = (int) ( sizeof( float_bits ) / sizeof( float_bits[0] ) ); + +static const uint64_t double_bits[] = +{ + 0x0000000000000000ULL, // +0 + 0x8000000000000000ULL, // -0 + 0x7FF0000000000000ULL, // +infinity + 0xFFF0000000000000ULL, // -infinity + 0x7FEFFFFFFFFFFFFFULL, // the largest finite float64 + 0x0010000000000000ULL, // the smallest normal + 0x0000000000000001ULL, // the smallest subnormal + 0x3FF0000000000000ULL, // 1.0 + 0xBFF0000000000000ULL, // -1.0 +}; +static const int double_count = (int) ( sizeof( double_bits ) / sizeof( double_bits[0] ) ); + +// --------------------------------------------------------------------------------------- +// section 11: compressed_float + +struct CompressedFloatVector { float value; float min; float max; float res; }; + +static const CompressedFloatVector compressed_float_vectors[] = +{ + { 0.0f, 0.0f, 10.0f, 0.01f }, // the bottom of the range: integer 0 + { 10.0f, 0.0f, 10.0f, 0.01f }, // the top: the maximum integer + { 0.005f, 0.0f, 10.0f, 0.01f }, // between quanta: 1 under float32, 0 widened + { 0.025f, 0.0f, 10.0f, 0.01f }, // between quanta: 3 vs 2 + { 0.105f, 0.0f, 10.0f, 0.01f }, // between quanta: 11 vs 10 + { 9.995f, 0.0f, 10.0f, 0.01f }, // between quanta: 1000 vs 999 + { -100.0f, -100.0f, 100.0f, 0.01f }, // the bottom of a range with a non-zero min + { -42.573f, -100.0f, 100.0f, 0.01f }, // off quantum over a non-zero min + { 8388609.0f, 0.0f, 8388609.0f, 1.0f }, // clamp witness A: an unclamped writer emits + // a code its own reader rejects (schema#109) + { 16777215.0f, 0.0f, 16777215.0f, 1.0f }, // clamp witness B: one bit wider than the field + { 0.0f, 0.0f, 1.0f, 1.0f }, // a one bit field, both codes + { 1.0f, 0.0f, 1.0f, 1.0f }, +}; +static const int compressed_float_count = (int) ( sizeof( compressed_float_vectors ) / sizeof( compressed_float_vectors[0] ) ); + +// --------------------------------------------------------------------------------------- +// section 12: bytes. The block lengths, and the byte each block is filled with. + +struct BytesVector { int length; uint8_t fill; }; + +static const BytesVector bytes_vectors[] = +{ + { 0, 0x00 }, // zero length: the align happens anyway + { 8, 0x00 }, + { 8, 0xFF }, + { 1, 0x5A }, +}; +static const int bytes_count = (int) ( sizeof( bytes_vectors ) / sizeof( bytes_vectors[0] ) ); +static const int bytes_max = 8; + +// --------------------------------------------------------------------------------------- +// section 13: string. buffer_size 16 throughout, so the length field is four bits. + +static const int string_buffer_size = 16; +static const int string_count = 3; + +// --------------------------------------------------------------------------------------- +// section 14: wstring. buffer_size 8 throughout: at most seven UTF-16 code units. + +static const int wstring_buffer_size = 8; +static const int wstring_count = 6; + +// the wide strings as explicit code points, so no source file encoding can reach the wire. +// On a four byte wchar_t the astral entry is one wchar_t and two code units, and the library +// splits it into its surrogate pair at the boundary; on a two byte wchar_t it is already the +// pair. Both produce the same bytes, which is the rule this entry exists to hold. +static void init_wide_strings( wchar_t strings[wstring_count][wstring_buffer_size] ) +{ + memset( (void*) strings, 0, sizeof( wchar_t ) * wstring_count * wstring_buffer_size ); + + // 0: empty + // 1: cyrillic "мир", basic plane + strings[1][0] = (wchar_t) 0x043C; + strings[1][1] = (wchar_t) 0x0438; + strings[1][2] = (wchar_t) 0x0440; + // 2: the first code unit above the surrogate block + strings[2][0] = (wchar_t) 0xE000; + // 3: the largest code unit there is + strings[3][0] = (wchar_t) 0xFFFF; + // 4: an astral code point between two basic plane ones: four code units, one pair + strings[4][0] = (wchar_t) 0x0041; + strings[4][1] = (wchar_t) 0x1F600; + strings[4][2] = (wchar_t) 0x0042; + // 5: seven code units, the most buffer_size 8 carries + for ( int i = 0; i < 7; i++ ) + { + strings[5][i] = (wchar_t) ( 0x0061 + i ); + } +} + +// --------------------------------------------------------------------------------------- +// the message + +struct InteropData +{ + uint64_t bits_values[bits_count]; + bool bool_values[2]; + uint32_t uint8_values_data[uint8_count]; + uint32_t uint16_values_data[uint16_count]; + uint32_t uint32_values_data[uint32_count]; + uint64_t uint64_values_data[uint64_count]; + serialize::uint128_t uint128_values_data[uint128_count]; + int32_t int_values[int_count]; + int64_t int64_values[int64_count]; + serialize::int128_t int128_values[int128_count]; + int16_t fixed_q8_8_min; + int16_t fixed_q8_8_max; + int32_t fixed_q16_16_degenerate; + int64_t fixed_q48_16_min; + int64_t fixed_q48_16_max; + serialize::int128_t fixed_q112_16_max; + serialize::int128_t fixed_q64_64_degenerate; + serialize::int128_t fixed_q64_64_max; + int32_t relative_values[relative_count]; + float float_values[float_count]; + double double_values[double_count]; + float compressed_float_values[compressed_float_count]; + uint32_t filler; + uint8_t bytes_values[bytes_count][bytes_max]; + char strings[string_count][string_buffer_size]; + wchar_t wide_strings[wstring_count][wstring_buffer_size]; +}; + +static float float_from_bits( uint32_t bits ) +{ + float value; + memcpy( &value, &bits, sizeof( value ) ); + return value; +} + +static uint32_t bits_from_float( float value ) +{ + uint32_t bits; + memcpy( &bits, &value, sizeof( bits ) ); + return bits; +} + +static double double_from_bits( uint64_t bits ) +{ + double value; + memcpy( &value, &bits, sizeof( value ) ); + return value; +} + +static uint64_t bits_from_double( double value ) +{ + uint64_t bits; + memcpy( &bits, &value, sizeof( bits ) ); + return bits; +} + +static void InteropInit( InteropData & data ) +{ + memset( (void*) &data, 0, sizeof( data ) ); + + for ( int i = 0; i < bits_count; i++ ) + { + data.bits_values[i] = bits_vectors[i].value; + } + + data.bool_values[0] = true; + data.bool_values[1] = false; + + for ( int i = 0; i < uint8_count; i++ ) { data.uint8_values_data[i] = uint8_values[i]; } + for ( int i = 0; i < uint16_count; i++ ) { data.uint16_values_data[i] = uint16_values[i]; } + for ( int i = 0; i < uint32_count; i++ ) { data.uint32_values_data[i] = uint32_values[i]; } + for ( int i = 0; i < uint64_count; i++ ) { data.uint64_values_data[i] = uint64_values[i]; } + + data.uint128_values_data[0] = serialize::uint128_t( 0u ); + data.uint128_values_data[1] = ~serialize::uint128_t( 0u ); + data.uint128_values_data[2] = ( serialize::uint128_t( 0x0123456789ABCDEFULL ) << 64 ) + | serialize::uint128_t( 0x0FEDCBA987654321ULL ); + + for ( int i = 0; i < int_count; i++ ) { data.int_values[i] = int_vectors[i].value; } + for ( int i = 0; i < int64_count; i++ ) { data.int64_values[i] = int64_vectors[i].value; } + + data.int128_values[0] = int128_degenerate_value(); + data.int128_values[1] = serialize::int128_t( 5000000000LL ); + data.int128_values[2] = int128_min_value(); + data.int128_values[3] = int128_max_value(); + + data.fixed_q8_8_min = (int16_t) ( -100 * 256 ); // the bottom of the range + data.fixed_q8_8_max = (int16_t) ( 100 * 256 ); // the top + data.fixed_q16_16_degenerate = (int32_t) ( 7 * 65536 ); // min == max: zero bits + data.fixed_q48_16_min = -( (int64_t) 100000 * 65536 ); // 34 bits on the wire + data.fixed_q48_16_max = (int64_t) 100000 * 65536; + data.fixed_q112_16_max = serialize::int128_t( 144115188075855872LL ) << 16; // 75 bits, three groups + data.fixed_q64_64_degenerate = serialize::int128_t( 5 ) << 64; // zero bits at 128 bit storage + data.fixed_q64_64_max = serialize::int128_t( INT64_MAX ) << 64; // 128 bits, four groups + + for ( int i = 0; i < relative_count; i++ ) + { + data.relative_values[i] = relative_vectors[i].current; + } + + for ( int i = 0; i < float_count; i++ ) { data.float_values[i] = float_from_bits( float_bits[i] ); } + for ( int i = 0; i < double_count; i++ ) { data.double_values[i] = double_from_bits( double_bits[i] ); } + + for ( int i = 0; i < compressed_float_count; i++ ) + { + data.compressed_float_values[i] = compressed_float_vectors[i].value; + } + + data.filler = 5; + for ( int i = 0; i < bytes_count; i++ ) + { + memset( data.bytes_values[i], bytes_vectors[i].fill, (size_t) bytes_vectors[i].length ); + } + + serialize_copy_string( data.strings[0], "", string_buffer_size ); + serialize_copy_string( data.strings[1], "0123456789abcde", string_buffer_size ); // fifteen bytes: full + serialize_copy_string( data.strings[2], "\xD0\xBC\xD0\xB8\xD1\x80", string_buffer_size ); // "мир", six UTF-8 bytes + + init_wide_strings( data.wide_strings ); +} + +template bool InteropSerialize( Stream & stream, InteropData & data ) +{ + // ----- raw bit groups + for ( int i = 0; i < bits_count; i++ ) + { + serialize_bits( stream, data.bits_values[i], bits_vectors[i].bits ); + } + + // ----- bool, both codes + for ( int i = 0; i < 2; i++ ) + { + serialize_bool( stream, data.bool_values[i] ); + } + + // both sides of the alignment rule: the stream is unaligned here, so the first align + // pads, and the second must write nothing at all + serialize_align( stream ); + serialize_align( stream ); + + // ----- the fixed width unsigned helpers + for ( int i = 0; i < uint8_count; i++ ) { serialize_uint8( stream, data.uint8_values_data[i] ); } + for ( int i = 0; i < uint16_count; i++ ) { serialize_uint16( stream, data.uint16_values_data[i] ); } + for ( int i = 0; i < uint32_count; i++ ) { serialize_uint32( stream, data.uint32_values_data[i] ); } + for ( int i = 0; i < uint64_count; i++ ) { serialize_uint64( stream, data.uint64_values_data[i] ); } + for ( int i = 0; i < uint128_count; i++ ) { serialize_uint128( stream, data.uint128_values_data[i] ); } + + // ----- ranged integers + for ( int i = 0; i < int_count; i++ ) + { + serialize_int( stream, data.int_values[i], int_vectors[i].min, int_vectors[i].max ); + } + for ( int i = 0; i < int64_count; i++ ) + { + serialize_int64( stream, data.int64_values[i], int64_vectors[i].min, int64_vectors[i].max ); + } + { + const serialize::int128_t degenerate = int128_degenerate_value(); + const serialize::int128_t minimum = int128_min_value(); + const serialize::int128_t maximum = int128_max_value(); + serialize_int128( stream, data.int128_values[0], degenerate, degenerate ); + // bounds inside the 64 bit domain: the bytes are identical to serialize_int64 here + serialize_int128( stream, data.int128_values[1], serialize::int128_t( -5000000000LL ), serialize::int128_t( 5000000000LL ) ); + serialize_int128( stream, data.int128_values[2], minimum, maximum ); + serialize_int128( stream, data.int128_values[3], minimum, maximum ); + } + + // ----- fixed point, at the ends of its ranges and degenerate on two storage widths + serialize_align( stream ); + serialize_fixed( stream, data.fixed_q8_8_min, 8, 8, -100, +100 ); + serialize_fixed( stream, data.fixed_q8_8_max, 8, 8, -100, +100 ); + serialize_fixed( stream, data.fixed_q16_16_degenerate, 16, 16, 7, 7 ); + serialize_fixed( stream, data.fixed_q48_16_min, 48, 16, -100000, +100000 ); + serialize_fixed( stream, data.fixed_q48_16_max, 48, 16, -100000, +100000 ); + serialize_fixed( stream, data.fixed_q112_16_max, 112, 16, -144115188075855872LL, +144115188075855872LL ); + serialize_fixed( stream, data.fixed_q64_64_degenerate, 64, 64, 5, 5 ); + serialize_fixed( stream, data.fixed_q64_64_max, 64, 64, INT64_MIN, INT64_MAX ); + + // ----- int_relative + for ( int i = 0; i < relative_count; i++ ) + { + serialize_int_relative( stream, relative_vectors[i].previous, data.relative_values[i] ); + } + + // ----- float and double + for ( int i = 0; i < float_count; i++ ) { serialize_float( stream, data.float_values[i] ); } + for ( int i = 0; i < double_count; i++ ) { serialize_double( stream, data.double_values[i] ); } + + // ----- compressed_float + for ( int i = 0; i < compressed_float_count; i++ ) + { + serialize_compressed_float( stream, data.compressed_float_values[i], + compressed_float_vectors[i].min, + compressed_float_vectors[i].max, + compressed_float_vectors[i].res ); + } + + // ----- bytes. The three bit filler leaves the stream unaligned, so the align that + // begins the first block -- a ZERO LENGTH one -- is load bearing. + serialize_bits( stream, data.filler, 3 ); + for ( int i = 0; i < bytes_count; i++ ) + { + serialize_bytes( stream, data.bytes_values[i], bytes_vectors[i].length ); + } + + // ----- string + for ( int i = 0; i < string_count; i++ ) + { + serialize_string( stream, data.strings[i], string_buffer_size ); + } + + // ----- wstring + for ( int i = 0; i < wstring_count; i++ ) + { + serialize_wstring( stream, data.wide_strings[i], wstring_buffer_size ); + } + + return true; +} + +// What a conforming reader recovers. Everything is exact except the compressed floats, which +// are lossy by construction: the reader returns the nearest quantum, so they are compared +// within one resolution step. Floats compare by BIT PATTERN -- a value comparison cannot see +// -0.0, which is the whole point of half of this section. +static bool InteropCheck( const InteropData & data ) +{ + InteropData expected; + InteropInit( expected ); + + for ( int i = 0; i < bits_count; i++ ) + { + if ( data.bits_values[i] != expected.bits_values[i] ) { printf( "mismatch: bits[%d]\n", i ); return false; } + } + for ( int i = 0; i < 2; i++ ) + { + if ( data.bool_values[i] != expected.bool_values[i] ) { printf( "mismatch: bool[%d]\n", i ); return false; } + } + for ( int i = 0; i < uint8_count; i++ ) + { + if ( data.uint8_values_data[i] != expected.uint8_values_data[i] ) { printf( "mismatch: uint8[%d]\n", i ); return false; } + } + for ( int i = 0; i < uint16_count; i++ ) + { + if ( data.uint16_values_data[i] != expected.uint16_values_data[i] ) { printf( "mismatch: uint16[%d]\n", i ); return false; } + } + for ( int i = 0; i < uint32_count; i++ ) + { + if ( data.uint32_values_data[i] != expected.uint32_values_data[i] ) { printf( "mismatch: uint32[%d]\n", i ); return false; } + } + for ( int i = 0; i < uint64_count; i++ ) + { + if ( data.uint64_values_data[i] != expected.uint64_values_data[i] ) { printf( "mismatch: uint64[%d]\n", i ); return false; } + } + for ( int i = 0; i < uint128_count; i++ ) + { + if ( !( data.uint128_values_data[i] == expected.uint128_values_data[i] ) ) { printf( "mismatch: uint128[%d]\n", i ); return false; } + } + for ( int i = 0; i < int_count; i++ ) + { + if ( data.int_values[i] != expected.int_values[i] ) { printf( "mismatch: int[%d]\n", i ); return false; } + } + for ( int i = 0; i < int64_count; i++ ) + { + if ( data.int64_values[i] != expected.int64_values[i] ) { printf( "mismatch: int64[%d]\n", i ); return false; } + } + for ( int i = 0; i < int128_count; i++ ) + { + if ( !( data.int128_values[i] == expected.int128_values[i] ) ) { printf( "mismatch: int128[%d]\n", i ); return false; } + } + if ( data.fixed_q8_8_min != expected.fixed_q8_8_min ) { printf( "mismatch: fixed q8.8 min\n" ); return false; } + if ( data.fixed_q8_8_max != expected.fixed_q8_8_max ) { printf( "mismatch: fixed q8.8 max\n" ); return false; } + if ( data.fixed_q16_16_degenerate != expected.fixed_q16_16_degenerate ) { printf( "mismatch: fixed q16.16 degenerate\n" ); return false; } + if ( data.fixed_q48_16_min != expected.fixed_q48_16_min ) { printf( "mismatch: fixed q48.16 min\n" ); return false; } + if ( data.fixed_q48_16_max != expected.fixed_q48_16_max ) { printf( "mismatch: fixed q48.16 max\n" ); return false; } + if ( !( data.fixed_q112_16_max == expected.fixed_q112_16_max ) ) { printf( "mismatch: fixed q112.16 max\n" ); return false; } + if ( !( data.fixed_q64_64_degenerate == expected.fixed_q64_64_degenerate ) ) { printf( "mismatch: fixed q64.64 degenerate\n" ); return false; } + if ( !( data.fixed_q64_64_max == expected.fixed_q64_64_max ) ) { printf( "mismatch: fixed q64.64 max\n" ); return false; } + for ( int i = 0; i < relative_count; i++ ) + { + if ( data.relative_values[i] != expected.relative_values[i] ) { printf( "mismatch: int_relative[%d]\n", i ); return false; } + } + for ( int i = 0; i < float_count; i++ ) + { + if ( bits_from_float( data.float_values[i] ) != float_bits[i] ) { printf( "mismatch: float[%d]\n", i ); return false; } + } + for ( int i = 0; i < double_count; i++ ) + { + if ( bits_from_double( data.double_values[i] ) != double_bits[i] ) { printf( "mismatch: double[%d]\n", i ); return false; } + } + for ( int i = 0; i < compressed_float_count; i++ ) + { + const float difference = data.compressed_float_values[i] - compressed_float_vectors[i].value; + const float magnitude = difference < 0.0f ? -difference : difference; + if ( magnitude > compressed_float_vectors[i].res ) { printf( "mismatch: compressed_float[%d]\n", i ); return false; } + } + if ( data.filler != expected.filler ) { printf( "mismatch: filler\n" ); return false; } + for ( int i = 0; i < bytes_count; i++ ) + { + if ( memcmp( data.bytes_values[i], expected.bytes_values[i], (size_t) bytes_vectors[i].length ) != 0 ) { printf( "mismatch: bytes[%d]\n", i ); return false; } + } + for ( int i = 0; i < string_count; i++ ) + { + if ( strcmp( data.strings[i], expected.strings[i] ) != 0 ) { printf( "mismatch: string[%d]\n", i ); return false; } + } + for ( int i = 0; i < wstring_count; i++ ) + { + if ( wcscmp( data.wide_strings[i], expected.wide_strings[i] ) != 0 ) { printf( "mismatch: wstring[%d]\n", i ); return false; } + } + return true; +} + +// --------------------------------------------------------------------------------------- + +static const int buffer_size = 1024; + +// + 8: read buffer allocations extend eight bytes past the data, per the read allocation +// contract, and the writer shares the array so the two halves cannot disagree about size +static uint8_t write_buffer[buffer_size + 8]; +static uint8_t read_buffer[buffer_size + 8]; +static uint8_t reencode_buffer[buffer_size + 8]; + +static bool encode( uint8_t * buffer, InteropData & data, int64_t & bytes ) +{ + memset( buffer, 0, (size_t) buffer_size + 8 ); + serialize::WriteStream stream( buffer, buffer_size ); + if ( !InteropSerialize( stream, data ) ) + { + return false; + } + stream.Flush(); + bytes = stream.GetBytesProcessed(); + return true; +} + +static int Write( const char * path ) +{ + InteropData data; + InteropInit( data ); + + int64_t bytes = 0; + if ( !encode( write_buffer, data, bytes ) ) + { + fprintf( stderr, "interop cpp write: serialize failed\n" ); + return 1; + } + + FILE * file = fopen( path, "wb" ); + if ( !file ) + { + fprintf( stderr, "interop cpp write: could not open %s\n", path ); + return 1; + } + if ( fwrite( write_buffer, 1, (size_t) bytes, file ) != (size_t) bytes ) + { + fprintf( stderr, "interop cpp write: short write to %s\n", path ); + fclose( file ); + return 1; + } + fclose( file ); + + printf( "interop cpp: wrote %d bytes to %s\n", (int) bytes, path ); + return 0; +} + +static int64_t load( const char * path, uint8_t * buffer ) +{ + FILE * file = fopen( path, "rb" ); + if ( !file ) + { + fprintf( stderr, "interop cpp: could not open %s\n", path ); + return -1; + } + memset( buffer, 0, (size_t) buffer_size + 8 ); + const size_t bytes = fread( buffer, 1, (size_t) buffer_size, file ); + fclose( file ); + if ( bytes == 0 || bytes >= (size_t) buffer_size ) + { + fprintf( stderr, "interop cpp: unexpected size %d for %s\n", (int) bytes, path ); + return -1; + } + return (int64_t) bytes; +} + +static bool decode( uint8_t * buffer, int64_t bytes, InteropData & data ) +{ + memset( (void*) &data, 0, sizeof( data ) ); + serialize::ReadStream stream( buffer, bytes ); + return InteropSerialize( stream, data ); +} + +static int Read( const char * path ) +{ + const int64_t bytes = load( path, read_buffer ); + if ( bytes < 0 ) + { + return 1; + } + + InteropData data; + if ( !decode( read_buffer, bytes, data ) ) + { + fprintf( stderr, "interop cpp read: could not decode %s\n", path ); + return 1; + } + if ( !InteropCheck( data ) ) + { + fprintf( stderr, "interop cpp read: %s decoded to unexpected values\n", path ); + return 1; + } + + // re-encode what was decoded: the bytes must be identical to the input + int64_t reencoded = 0; + if ( !encode( reencode_buffer, data, reencoded ) ) + { + fprintf( stderr, "interop cpp read: re-encode failed\n" ); + return 1; + } + if ( reencoded != bytes || memcmp( reencode_buffer, read_buffer, (size_t) bytes ) != 0 ) + { + fprintf( stderr, "interop cpp read: re-encoded bytes differ from %s\n", path ); + return 1; + } + + printf( "interop cpp: decoded and re-encoded %d bytes from %s, byte identical\n", (int) bytes, path ); + return 0; +} + +// The hostile half: every proper prefix of a valid stream is a truncated stream, and a +// conforming reader refuses every one of them. Nothing here may crash, hang or accept. +static int Refuse( const char * path ) +{ + const int64_t bytes = load( path, read_buffer ); + if ( bytes < 0 ) + { + return 1; + } + + for ( int64_t length = 0; length < bytes; length++ ) + { + memset( reencode_buffer, 0, (size_t) buffer_size + 8 ); + memcpy( reencode_buffer, read_buffer, (size_t) length ); + InteropData data; + if ( decode( reencode_buffer, length, data ) ) + { + fprintf( stderr, "interop cpp refuse: the %d byte prefix of %s was ACCEPTED\n", (int) length, path ); + return 1; + } + } + + printf( "interop cpp: refused all %d truncated prefixes of %s\n", (int) bytes, path ); + return 0; +} + +int main( int argc, char ** argv ) +{ + if ( argc != 3 ) + { + fprintf( stderr, "usage: interop write|read|refuse \n" ); + return 2; + } + if ( strcmp( argv[1], "write" ) == 0 ) + { + return Write( argv[2] ); + } + if ( strcmp( argv[1], "read" ) == 0 ) + { + return Read( argv[2] ); + } + if ( strcmp( argv[1], "refuse" ) == 0 ) + { + return Refuse( argv[2] ); + } + fprintf( stderr, "usage: interop write|read|refuse \n" ); + return 2; +} diff --git a/interop/serialize/interop/Interop.java b/interop/serialize/interop/Interop.java new file mode 100644 index 0000000..771fe53 --- /dev/null +++ b/interop/serialize/interop/Interop.java @@ -0,0 +1,640 @@ +package serialize.interop; + +import serialize.BitStream; +import serialize.BoolRef; +import serialize.DoubleRef; +import serialize.FloatRef; +import serialize.Int128Value; +import serialize.IntRef; +import serialize.LongRef; +import serialize.ReadStream; +import serialize.Ref; +import serialize.UInt128Value; +import serialize.WriteStream; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +/** + * The Java half of the cross language interop harness. + * + * Its twin is {@code interop/interop.cpp}, built in CI against the real C++ + * serialize library at the release {@code .github/workflows/ci.yml} pins. The two + * halves run head to head on every push and pull request: each writes the boundary + * message, the two files must be byte identical, and each must decode the other's + * file to the exact values and re-encode it to the exact bytes. + * + * 
+ *   make interop MODE=write  FILE=/tmp/java.bin
+ *   make interop MODE=read   FILE=/tmp/cpp.bin
+ *   make interop MODE=refuse FILE=/tmp/cpp.bin
+ *
+ * + * THE MESSAGE is the boundary set: every operation STANDARD.md defines, at the + * values where implementations disagree. Zero bit ranges on all three ranged + * widths and on fixed point; the domain edges of int, int64, int128 and + * int_relative; the maximum widths of bits, uint128 and the four group fixed point + * path; both sides of the alignment rule, including the align inside a zero length + * bytes; empty and full strings; and the wide string cases the surrogate rule + * governs, up to the largest code unit. + * + * WHAT IT DELIBERATELY DOES NOT CARRY: a NaN payload. STANDARD.md's bit + * transparency claim covers it, but a NaN's payload bits do not survive every + * language's float type on the way to the wire, so a difference here would say + * nothing about the wire format. This port pins its own NaN patterns in its own + * suite, where the claim can be tested honestly. + * + * Any change to the sequence below must be mirrored in {@code interop/interop.cpp}, + * and never changes the wire format. + */ +public final class Interop +{ + private Interop() {} + + // ----------------------------------------------------------------------- + // the sequence, mirrored field for field from interop/interop.cpp + + // raw bit groups, every width boundary + private static final int[] BITS_WIDTHS = { 1, 1, 7, 31, 32, 32, 33, 64, 64 }; + private static final long[] BITS_VALUES = + { + 1, // the minimum width, at its maximum value + 0, // and at its minimum + 0x7F, // a sub-byte width, all ones + 0x7FFFFFFF, // one below the single group maximum + 0xFFFFFFFFL, // the widest single group, all ones + 0, // and all zeros + 0x1FFFFFFFFL, // the first width past the 32 bit split + 0xFFFFFFFFFFFFFFFFL, // the maximum width, all ones + 0, // and all zeros + }; + + private static final int[] UINT8_VALUES = { 0x00, 0xFF }; + private static final int[] UINT16_VALUES = { 0x0000, 0xFFFF }; + private static final int[] UINT32_VALUES = { 0x00000000, 0xFFFFFFFF }; + private static final long[] UINT64_VALUES = { 0L, 0xFFFFFFFFFFFFFFFFL }; + + private static UInt128Value[] uint128Values() + { + return new UInt128Value[] + { + UInt128Value.ZERO, + UInt128Value.ZERO.not(), // all ones + new UInt128Value( 0x0123456789ABCDEFL, 0x0FEDCBA987654321L ), + }; + } + + // ranged 32 bit integers: min, max, value + private static final int[] INT_MIN = + { 42, -100, -100, Integer.MIN_VALUE, Integer.MIN_VALUE, -100 }; + private static final int[] INT_MAX = + { 42, +100, +100, Integer.MAX_VALUE, Integer.MAX_VALUE, +100 }; + private static final int[] INT_VALUE = + { + 42, // degenerate: zero bits, mid sequence + -100, // the bottom of the range + +100, // the top of the range + Integer.MIN_VALUE, // the full domain, 32 bits on the wire + Integer.MAX_VALUE, + -37, // a live field after the degenerate one + }; + + private static final long[] INT64_MIN = + { 10000000000L, -5000000000L, -5000000000L, Long.MIN_VALUE, Long.MIN_VALUE }; + private static final long[] INT64_MAX = + { 10000000000L, +5000000000L, +5000000000L, Long.MAX_VALUE, Long.MAX_VALUE }; + private static final long[] INT64_VALUE = + { + 10000000000L, // degenerate, with bounds past 2^32 + -5000000000L, // a range wider than 32 bits, bottom + +5000000000L, // and top + Long.MIN_VALUE, // the full domain, 64 bits on the wire + Long.MAX_VALUE, + }; + + // 2^100 + 7: a degenerate bound no 64 bit path can carry + private static Int128Value int128Degenerate() + { + return Int128Value.fromLong( 1 ).shiftLeft( 100 ).add( Int128Value.fromLong( 7 ) ); + } + + private static Int128Value[] int128Bounds( boolean minimum ) + { + Int128Value degenerate = int128Degenerate(); + return new Int128Value[] + { + degenerate, + // bounds inside the 64 bit domain: the bytes are identical to serializeInt64 here + Int128Value.fromLong( minimum ? -5000000000L : +5000000000L ), + minimum ? Int128Value.MIN_VALUE : Int128Value.MAX_VALUE, + minimum ? Int128Value.MIN_VALUE : Int128Value.MAX_VALUE, + }; + } + + private static Int128Value[] int128Values() + { + return new Int128Value[] + { + int128Degenerate(), + Int128Value.fromLong( 5000000000L ), + Int128Value.MIN_VALUE, + Int128Value.MAX_VALUE, + }; + } + + // int_relative: every tier at both ends, and the domain edges + private static final int[] RELATIVE_PREVIOUS = + { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2147483646, 0 }; + private static final int[] RELATIVE_CURRENT = + { + 1, // one-bit + 2, 6, // bounded-3, both ends + 7, 23, // bounded-5 + 24, 280, // bounded-9 + 281, 4377, // bounded-13 + 4378, 69914, // bounded-17 + 69915, // absolute, at its smallest difference + 2147483647, // one-bit, at the top of the domain + 2147483647, // absolute, at the top of the domain + }; + + // floats are given as bit patterns, so no decimal literal is parsed twice + private static final int[] FLOAT_BITS = + { + 0x00000000, // +0 + 0x80000000, // -0 + 0x7F800000, // +infinity + 0xFF800000, // -infinity + 0x7F7FFFFF, // the largest finite float32 + 0x00800000, // the smallest normal + 0x00000001, // the smallest subnormal + 0x3F800000, // 1.0f + 0xBF800000, // -1.0f + }; + + private static final long[] DOUBLE_BITS = + { + 0x0000000000000000L, // +0 + 0x8000000000000000L, // -0 + 0x7FF0000000000000L, // +infinity + 0xFFF0000000000000L, // -infinity + 0x7FEFFFFFFFFFFFFFL, // the largest finite float64 + 0x0010000000000000L, // the smallest normal + 0x0000000000000001L, // the smallest subnormal + 0x3FF0000000000000L, // 1.0 + 0xBFF0000000000000L, // -1.0 + }; + + private static final float[] CF_VALUE = + { + 0.0f, // the bottom of the range: integer 0 + 10.0f, // the top: the maximum integer + 0.005f, // between quanta: 1 under float32, 0 widened + 0.025f, // between quanta: 3 vs 2 + 0.105f, // between quanta: 11 vs 10 + 9.995f, // between quanta: 1000 vs 999 + -100.0f, // the bottom of a range with a non-zero min + -42.573f, // off quantum over a non-zero min + 8388609.0f, // clamp witness A (schema#109) + 16777215.0f, // clamp witness B + 0.0f, // a one bit field, both codes + 1.0f, + }; + private static final float[] CF_MIN = + { 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, -100.0f, -100.0f, 0.0f, 0.0f, 0.0f, 0.0f }; + private static final float[] CF_MAX = + { 10.0f, 10.0f, 10.0f, 10.0f, 10.0f, 10.0f, 100.0f, 100.0f, 8388609.0f, 16777215.0f, 1.0f, 1.0f }; + private static final float[] CF_RES = + { 0.01f, 0.01f, 0.01f, 0.01f, 0.01f, 0.01f, 0.01f, 0.01f, 1.0f, 1.0f, 1.0f, 1.0f }; + + // a zero length block first: the align happens anyway + private static final int[] BYTES_LENGTH = { 0, 8, 8, 1 }; + private static final int[] BYTES_FILL = { 0x00, 0x00, 0xFF, 0x5A }; + + private static final int STRING_BUFFER_SIZE = 16; + private static final String[] STRINGS = + { + "", // empty + "0123456789abcde", // fifteen bytes: the most buffer size 16 carries + "\u043C\u0438\u0440", // six UTF-8 bytes, as explicit code points so no + // source file encoding can reach the wire + }; + + private static final int WSTRING_BUFFER_SIZE = 8; + private static final String[] WIDE_STRINGS = + { + "", // empty + "\u043C\u0438\u0440", // basic plane + "\uE000", // the first code unit above the surrogate block + "\uFFFF", // the largest code unit there is + "A\uD83D\uDE00B", // U+1F600 as its surrogate pair: four code units + "abcdefg", // seven code units, the most buffer size 8 carries + }; + + // ----------------------------------------------------------------------- + // the message + + /** + * Holders for every field. The raw bit groups keep both a narrow and a wide + * holder per entry, because the port splits raw bits at 32 where the wire does + * not; only the one the width selects is ever used. + */ + private static final class Data + { + final IntRef[] bitsNarrow = new IntRef[BITS_WIDTHS.length]; + final LongRef[] bitsWide = new LongRef[BITS_WIDTHS.length]; + final BoolRef[] bools = { new BoolRef(), new BoolRef() }; + final IntRef[] uint8 = new IntRef[UINT8_VALUES.length]; + final IntRef[] uint16 = new IntRef[UINT16_VALUES.length]; + final IntRef[] uint32 = new IntRef[UINT32_VALUES.length]; + final LongRef[] uint64 = new LongRef[UINT64_VALUES.length]; + final List> uint128 = new ArrayList<>(); + final IntRef[] ints = new IntRef[INT_VALUE.length]; + final LongRef[] int64s = new LongRef[INT64_VALUE.length]; + final List> int128s = new ArrayList<>(); + final LongRef fixedQ8_8Min = new LongRef(); + final LongRef fixedQ8_8Max = new LongRef(); + final LongRef fixedQ16_16Degenerate = new LongRef(); + final LongRef fixedQ48_16Min = new LongRef(); + final LongRef fixedQ48_16Max = new LongRef(); + final Ref fixedQ112_16Max = new Ref<>( Int128Value.ZERO ); + final Ref fixedQ64_64Degenerate = new Ref<>( Int128Value.ZERO ); + final Ref fixedQ64_64Max = new Ref<>( Int128Value.ZERO ); + final IntRef[] relative = new IntRef[RELATIVE_CURRENT.length]; + final FloatRef[] floats = new FloatRef[FLOAT_BITS.length]; + final DoubleRef[] doubles = new DoubleRef[DOUBLE_BITS.length]; + final FloatRef[] compressedFloats = new FloatRef[CF_VALUE.length]; + final IntRef filler = new IntRef(); + final byte[][] bytes = new byte[BYTES_LENGTH.length][]; + final List> strings = new ArrayList<>(); + final List> wideStrings = new ArrayList<>(); + + Data() + { + for ( int i = 0; i < BITS_WIDTHS.length; i++ ) + { + bitsNarrow[i] = new IntRef(); + bitsWide[i] = new LongRef(); + } + for ( int i = 0; i < UINT8_VALUES.length; i++ ) { uint8[i] = new IntRef(); } + for ( int i = 0; i < UINT16_VALUES.length; i++ ) { uint16[i] = new IntRef(); } + for ( int i = 0; i < UINT32_VALUES.length; i++ ) { uint32[i] = new IntRef(); } + for ( int i = 0; i < UINT64_VALUES.length; i++ ) { uint64[i] = new LongRef(); } + for ( int i = 0; i < 3; i++ ) { uint128.add( new Ref<>( UInt128Value.ZERO ) ); } + for ( int i = 0; i < INT_VALUE.length; i++ ) { ints[i] = new IntRef(); } + for ( int i = 0; i < INT64_VALUE.length; i++ ) { int64s[i] = new LongRef(); } + for ( int i = 0; i < 4; i++ ) { int128s.add( new Ref<>( Int128Value.ZERO ) ); } + for ( int i = 0; i < RELATIVE_CURRENT.length; i++ ) { relative[i] = new IntRef(); } + for ( int i = 0; i < FLOAT_BITS.length; i++ ) { floats[i] = new FloatRef(); } + for ( int i = 0; i < DOUBLE_BITS.length; i++ ) { doubles[i] = new DoubleRef(); } + for ( int i = 0; i < CF_VALUE.length; i++ ) { compressedFloats[i] = new FloatRef(); } + for ( int i = 0; i < BYTES_LENGTH.length; i++ ) { bytes[i] = new byte[BYTES_LENGTH[i]]; } + for ( int i = 0; i < STRINGS.length; i++ ) { strings.add( new Ref<>( "" ) ); } + for ( int i = 0; i < WIDE_STRINGS.length; i++ ) { wideStrings.add( new Ref<>( "" ) ); } + } + } + + /** The write side: every holder carries the value the sequence pins. */ + private static Data boundaryData() + { + Data data = new Data(); + for ( int i = 0; i < BITS_WIDTHS.length; i++ ) + { + data.bitsNarrow[i].value = (int) BITS_VALUES[i]; + data.bitsWide[i].value = BITS_VALUES[i]; + } + data.bools[0].value = true; + data.bools[1].value = false; + for ( int i = 0; i < UINT8_VALUES.length; i++ ) { data.uint8[i].value = UINT8_VALUES[i]; } + for ( int i = 0; i < UINT16_VALUES.length; i++ ) { data.uint16[i].value = UINT16_VALUES[i]; } + for ( int i = 0; i < UINT32_VALUES.length; i++ ) { data.uint32[i].value = UINT32_VALUES[i]; } + for ( int i = 0; i < UINT64_VALUES.length; i++ ) { data.uint64[i].value = UINT64_VALUES[i]; } + UInt128Value[] unsigned = uint128Values(); + for ( int i = 0; i < unsigned.length; i++ ) { data.uint128.get( i ).value = unsigned[i]; } + for ( int i = 0; i < INT_VALUE.length; i++ ) { data.ints[i].value = INT_VALUE[i]; } + for ( int i = 0; i < INT64_VALUE.length; i++ ) { data.int64s[i].value = INT64_VALUE[i]; } + Int128Value[] signed = int128Values(); + for ( int i = 0; i < signed.length; i++ ) { data.int128s.get( i ).value = signed[i]; } + data.fixedQ8_8Min.value = -100L * 256; // the bottom of the range + data.fixedQ8_8Max.value = 100L * 256; // the top + data.fixedQ16_16Degenerate.value = 7L * 65536; // min == max: zero bits + data.fixedQ48_16Min.value = -100000L * 65536; // 34 bits on the wire + data.fixedQ48_16Max.value = 100000L * 65536; + data.fixedQ112_16Max.value = Int128Value.fromLong( 144115188075855872L ).shiftLeft( 16 ); + data.fixedQ64_64Degenerate.value = Int128Value.fromLong( 5 ).shiftLeft( 64 ); + data.fixedQ64_64Max.value = Int128Value.fromLong( Long.MAX_VALUE ).shiftLeft( 64 ); + for ( int i = 0; i < RELATIVE_CURRENT.length; i++ ) { data.relative[i].value = RELATIVE_CURRENT[i]; } + for ( int i = 0; i < FLOAT_BITS.length; i++ ) { data.floats[i].value = Float.intBitsToFloat( FLOAT_BITS[i] ); } + for ( int i = 0; i < DOUBLE_BITS.length; i++ ) { data.doubles[i].value = Double.longBitsToDouble( DOUBLE_BITS[i] ); } + for ( int i = 0; i < CF_VALUE.length; i++ ) { data.compressedFloats[i].value = CF_VALUE[i]; } + data.filler.value = 5; + for ( int i = 0; i < BYTES_LENGTH.length; i++ ) { Arrays.fill( data.bytes[i], (byte) BYTES_FILL[i] ); } + for ( int i = 0; i < STRINGS.length; i++ ) { data.strings.get( i ).value = STRINGS[i]; } + for ( int i = 0; i < WIDE_STRINGS.length; i++ ) { data.wideStrings.get( i ).value = WIDE_STRINGS[i]; } + return data; + } + + /** The message, operation for operation. The chain stops at the first refusal. */ + private static boolean interopSerialize( BitStream stream, Data d ) + { + // ----- raw bit groups + for ( int i = 0; i < BITS_WIDTHS.length; i++ ) + { + int width = BITS_WIDTHS[i]; + boolean ok = width <= 32 ? stream.serializeBits( d.bitsNarrow[i], width ) + : stream.serializeBits64( d.bitsWide[i], width ); + if ( !ok ) { return false; } + } + + // ----- bool, both codes + for ( BoolRef ref : d.bools ) + { + if ( !stream.serializeBool( ref ) ) { return false; } + } + + // both sides of the alignment rule: the stream is unaligned here, so the + // first align pads and the second must write nothing at all + if ( !stream.serializeAlign() ) { return false; } + if ( !stream.serializeAlign() ) { return false; } + + // ----- the fixed width unsigned helpers, at their domain edges + for ( IntRef ref : d.uint8 ) { if ( !stream.serializeUint8( ref ) ) { return false; } } + for ( IntRef ref : d.uint16 ) { if ( !stream.serializeUint16( ref ) ) { return false; } } + for ( IntRef ref : d.uint32 ) { if ( !stream.serializeUint32( ref ) ) { return false; } } + for ( LongRef ref : d.uint64 ) { if ( !stream.serializeUint64( ref ) ) { return false; } } + for ( Ref ref : d.uint128 ) { if ( !stream.serializeUint128( ref ) ) { return false; } } + + // ----- ranged integers + for ( int i = 0; i < INT_VALUE.length; i++ ) + { + if ( !stream.serializeInt( d.ints[i], INT_MIN[i], INT_MAX[i] ) ) { return false; } + } + for ( int i = 0; i < INT64_VALUE.length; i++ ) + { + if ( !stream.serializeInt64( d.int64s[i], INT64_MIN[i], INT64_MAX[i] ) ) { return false; } + } + Int128Value[] lower = int128Bounds( true ); + Int128Value[] upper = int128Bounds( false ); + for ( int i = 0; i < lower.length; i++ ) + { + if ( !stream.serializeInt128( d.int128s.get( i ), lower[i], upper[i] ) ) { return false; } + } + + // ----- fixed point, at the ends of its ranges and degenerate on two storage widths + if ( !stream.serializeAlign() ) { return false; } + if ( !stream.serializeFixed( d.fixedQ8_8Min, 8, 8, -100, +100 ) ) { return false; } + if ( !stream.serializeFixed( d.fixedQ8_8Max, 8, 8, -100, +100 ) ) { return false; } + if ( !stream.serializeFixed( d.fixedQ16_16Degenerate, 16, 16, 7, 7 ) ) { return false; } + if ( !stream.serializeFixed( d.fixedQ48_16Min, 48, 16, -100000, +100000 ) ) { return false; } + if ( !stream.serializeFixed( d.fixedQ48_16Max, 48, 16, -100000, +100000 ) ) { return false; } + if ( !stream.serializeFixed128( d.fixedQ112_16Max, 112, 16, -144115188075855872L, +144115188075855872L ) ) { return false; } + if ( !stream.serializeFixed128( d.fixedQ64_64Degenerate, 64, 64, 5, 5 ) ) { return false; } + if ( !stream.serializeFixed128( d.fixedQ64_64Max, 64, 64, Long.MIN_VALUE, Long.MAX_VALUE ) ) { return false; } + + // ----- int_relative + for ( int i = 0; i < RELATIVE_CURRENT.length; i++ ) + { + if ( !stream.serializeIntRelative( RELATIVE_PREVIOUS[i], d.relative[i] ) ) { return false; } + } + + // ----- float and double, bit transparent at the domain edges + for ( FloatRef ref : d.floats ) { if ( !stream.serializeFloat( ref ) ) { return false; } } + for ( DoubleRef ref : d.doubles ) { if ( !stream.serializeDouble( ref ) ) { return false; } } + + // ----- compressed_float + for ( int i = 0; i < CF_VALUE.length; i++ ) + { + if ( !stream.serializeCompressedFloat( d.compressedFloats[i], CF_MIN[i], CF_MAX[i], CF_RES[i] ) ) { return false; } + } + + // ----- bytes. The three bit filler leaves the stream unaligned, so the + // align that begins the first block -- a ZERO LENGTH one -- is load bearing. + if ( !stream.serializeBits( d.filler, 3 ) ) { return false; } + for ( int i = 0; i < BYTES_LENGTH.length; i++ ) + { + if ( !stream.serializeBytes( d.bytes[i], BYTES_LENGTH[i] ) ) { return false; } + } + + // ----- string: empty, full, and multi-byte UTF-8 + for ( Ref ref : d.strings ) + { + if ( !stream.serializeString( ref, STRING_BUFFER_SIZE ) ) { return false; } + } + + // ----- wstring: empty, basic plane, the code unit boundaries, a pair, full + for ( Ref ref : d.wideStrings ) + { + if ( !stream.serializeWideString( ref, WSTRING_BUFFER_SIZE ) ) { return false; } + } + + return true; + } + + /** + * What a conforming reader recovers. Everything is exact except the compressed + * floats, which are lossy by construction: the reader returns the nearest + * quantum, so they are compared within one resolution step. Floats compare by + * RAW BIT PATTERN -- a value comparison cannot see -0.0. + */ + private static List interopCheck( Data d ) + { + List problems = new ArrayList<>(); + Data expected = boundaryData(); + + for ( int i = 0; i < BITS_WIDTHS.length; i++ ) + { + long actual = BITS_WIDTHS[i] <= 32 ? ( d.bitsNarrow[i].value & 0xFFFFFFFFL ) : d.bitsWide[i].value; + long want = BITS_WIDTHS[i] <= 32 ? ( (int) BITS_VALUES[i] & 0xFFFFFFFFL ) : BITS_VALUES[i]; + if ( actual != want ) { problems.add( "bits[" + i + "]: got " + actual + ", expected " + want ); } + } + if ( !d.bools[0].value ) { problems.add( "bool[0]" ); } + if ( d.bools[1].value ) { problems.add( "bool[1]" ); } + for ( int i = 0; i < UINT8_VALUES.length; i++ ) + { + if ( d.uint8[i].value != UINT8_VALUES[i] ) { problems.add( "uint8[" + i + "]" ); } + } + for ( int i = 0; i < UINT16_VALUES.length; i++ ) + { + if ( d.uint16[i].value != UINT16_VALUES[i] ) { problems.add( "uint16[" + i + "]" ); } + } + for ( int i = 0; i < UINT32_VALUES.length; i++ ) + { + if ( d.uint32[i].value != UINT32_VALUES[i] ) { problems.add( "uint32[" + i + "]" ); } + } + for ( int i = 0; i < UINT64_VALUES.length; i++ ) + { + if ( d.uint64[i].value != UINT64_VALUES[i] ) { problems.add( "uint64[" + i + "]" ); } + } + for ( int i = 0; i < d.uint128.size(); i++ ) + { + if ( !d.uint128.get( i ).value.equals( expected.uint128.get( i ).value ) ) { problems.add( "uint128[" + i + "]" ); } + } + for ( int i = 0; i < INT_VALUE.length; i++ ) + { + if ( d.ints[i].value != INT_VALUE[i] ) { problems.add( "int[" + i + "]" ); } + } + for ( int i = 0; i < INT64_VALUE.length; i++ ) + { + if ( d.int64s[i].value != INT64_VALUE[i] ) { problems.add( "int64[" + i + "]" ); } + } + for ( int i = 0; i < d.int128s.size(); i++ ) + { + if ( !d.int128s.get( i ).value.equals( expected.int128s.get( i ).value ) ) { problems.add( "int128[" + i + "]" ); } + } + if ( d.fixedQ8_8Min.value != expected.fixedQ8_8Min.value ) { problems.add( "fixed q8.8 min" ); } + if ( d.fixedQ8_8Max.value != expected.fixedQ8_8Max.value ) { problems.add( "fixed q8.8 max" ); } + if ( d.fixedQ16_16Degenerate.value != expected.fixedQ16_16Degenerate.value ) { problems.add( "fixed q16.16 degenerate" ); } + if ( d.fixedQ48_16Min.value != expected.fixedQ48_16Min.value ) { problems.add( "fixed q48.16 min" ); } + if ( d.fixedQ48_16Max.value != expected.fixedQ48_16Max.value ) { problems.add( "fixed q48.16 max" ); } + if ( !d.fixedQ112_16Max.value.equals( expected.fixedQ112_16Max.value ) ) { problems.add( "fixed q112.16 max" ); } + if ( !d.fixedQ64_64Degenerate.value.equals( expected.fixedQ64_64Degenerate.value ) ) { problems.add( "fixed q64.64 degenerate" ); } + if ( !d.fixedQ64_64Max.value.equals( expected.fixedQ64_64Max.value ) ) { problems.add( "fixed q64.64 max" ); } + if ( d.filler.value != expected.filler.value ) { problems.add( "filler" ); } + for ( int i = 0; i < RELATIVE_CURRENT.length; i++ ) + { + if ( d.relative[i].value != RELATIVE_CURRENT[i] ) { problems.add( "int_relative[" + i + "]" ); } + } + for ( int i = 0; i < FLOAT_BITS.length; i++ ) + { + if ( Float.floatToRawIntBits( d.floats[i].value ) != FLOAT_BITS[i] ) { problems.add( "float[" + i + "]" ); } + } + for ( int i = 0; i < DOUBLE_BITS.length; i++ ) + { + if ( Double.doubleToRawLongBits( d.doubles[i].value ) != DOUBLE_BITS[i] ) { problems.add( "double[" + i + "]" ); } + } + for ( int i = 0; i < CF_VALUE.length; i++ ) + { + if ( !( Math.abs( d.compressedFloats[i].value - CF_VALUE[i] ) <= CF_RES[i] ) ) + { + problems.add( "compressed_float[" + i + "]: got " + d.compressedFloats[i].value ); + } + } + for ( int i = 0; i < BYTES_LENGTH.length; i++ ) + { + if ( !Arrays.equals( d.bytes[i], expected.bytes[i] ) ) { problems.add( "bytes[" + i + "]" ); } + } + for ( int i = 0; i < STRINGS.length; i++ ) + { + if ( !STRINGS[i].equals( d.strings.get( i ).value ) ) { problems.add( "string[" + i + "]: got " + d.strings.get( i ).value ); } + } + for ( int i = 0; i < WIDE_STRINGS.length; i++ ) + { + if ( !WIDE_STRINGS[i].equals( d.wideStrings.get( i ).value ) ) { problems.add( "wstring[" + i + "]" ); } + } + return problems; + } + + // ----------------------------------------------------------------------- + // the three modes + + private static final int BUFFER_BYTES = 1024; + + /** + * The read allocation contract: this port's reader requires the array to extend + * eight bytes past the data (BitReader). The slack is zero filled, so nothing in + * it can reach a decoded value -- STANDARD.md ruled that bytes past the stream + * end are never interpreted, and a truncated prefix must be refused for the bits + * it does not have, not accepted from what follows it. + */ + private static byte[] withSlack( byte[] input, int length ) + { + byte[] buffer = new byte[length + 8]; + System.arraycopy( input, 0, buffer, 0, length ); + return buffer; + } + + private static byte[] encode( Data data ) + { + WriteStream stream = new WriteStream( new byte[BUFFER_BYTES], BUFFER_BYTES ); + if ( !interopSerialize( stream, data ) ) + { + throw new IllegalStateException( "interop java: serialize failed" ); + } + stream.flush(); + return Arrays.copyOf( stream.getData(), (int) stream.getBytesProcessed() ); + } + + private static void write( String path ) throws IOException + { + byte[] bytes = encode( boundaryData() ); + Files.write( Path.of( path ), bytes ); + System.out.println( "interop java: wrote " + bytes.length + " bytes to " + path ); + } + + private static void read( String path ) throws IOException + { + byte[] input = Files.readAllBytes( Path.of( path ) ); + Data data = new Data(); + if ( !interopSerialize( new ReadStream( withSlack( input, input.length ), input.length ), data ) ) + { + throw new IllegalStateException( "interop java: could not decode " + path ); + } + List problems = interopCheck( data ); + if ( !problems.isEmpty() ) + { + throw new IllegalStateException( "interop java: " + path + " decoded to unexpected values: " + problems ); + } + // re-encode what was decoded: the bytes must be identical to the input + if ( !Arrays.equals( encode( data ), input ) ) + { + throw new IllegalStateException( "interop java: re-encoded bytes differ from " + path ); + } + System.out.println( "interop java: decoded and re-encoded " + input.length + " bytes from " + path + ", byte identical" ); + } + + /** + * The hostile half: every proper prefix of a valid stream is a truncated + * stream, and a conforming reader refuses every one of them without throwing. + */ + private static void refuse( String path ) throws IOException + { + byte[] input = Files.readAllBytes( Path.of( path ) ); + for ( int length = 0; length < input.length; length++ ) + { + byte[] truncated = withSlack( input, length ); + boolean accepted; + try + { + accepted = interopSerialize( new ReadStream( truncated, length ), new Data() ); + } + catch ( RuntimeException error ) + { + throw new IllegalStateException( "interop java refuse: the " + length + " byte prefix of " + path + " THREW", error ); + } + if ( accepted ) + { + throw new IllegalStateException( "interop java refuse: the " + length + " byte prefix of " + path + " was ACCEPTED" ); + } + } + System.out.println( "interop java: refused all " + input.length + " truncated prefixes of " + path ); + } + + public static void main( String[] args ) throws IOException + { + if ( args.length != 2 ) + { + System.err.println( "usage: Interop write|read|refuse " ); + System.exit( 2 ); + } + switch ( args[0] ) + { + case "write": + write( args[1] ); + break; + case "read": + read( args[1] ); + break; + case "refuse": + refuse( args[1] ); + break; + default: + System.err.println( "usage: Interop write|read|refuse " ); + System.exit( 2 ); + break; + } + } +} diff --git a/src/serialize/SerializeUtil.java b/src/serialize/SerializeUtil.java index 51ebcb7..a2e2f85 100644 --- a/src/serialize/SerializeUtil.java +++ b/src/serialize/SerializeUtil.java @@ -11,7 +11,7 @@ public final class SerializeUtil private SerializeUtil() {} /** The library version, matching the release tag. */ - public static final String VERSION = "1.1.0"; + public static final String VERSION = "1.1.1"; /** * The number of bits required to serialize an integer in [min,max]. diff --git a/test/serialize/tests/ConformanceTests.java b/test/serialize/tests/ConformanceTests.java index 7410780..0178be3 100644 --- a/test/serialize/tests/ConformanceTests.java +++ b/test/serialize/tests/ConformanceTests.java @@ -38,7 +38,7 @@ * below fails the suite rather than being skipped: a vector that does not * run is a vector that proves nothing. */ -final class ConformanceTests +public final class ConformanceTests { private ConformanceTests() {} @@ -53,6 +53,17 @@ private ConformanceTests() {} private static final BigInteger MASK_128 = BigInteger.ONE.shiftLeft( 128 ).subtract( BigInteger.ONE ); + /** + * Runs this suite on its own ({@code java serialize.tests.ConformanceTests}), so + * the interop job can hold this reader and the pinned C++ reader to the same + * corpus in one place. AllTests calls {@link #run} directly. + */ + public static void main( String[] args ) + { + run(); + System.exit( Harness.finish() ); + } + static void run() { List files = corpusFiles();