From 0156650a6e4872582c1d60da49394c70fa630825 Mon Sep 17 00:00:00 2001 From: Curt Hagenlocher Date: Fri, 24 Apr 2026 19:16:30 -0700 Subject: [PATCH] Add Parquet variant shredding support --- .../Apache.Arrow.Operations.csproj | 1 + .../Shredding/ShredOptions.cs | 50 ++ .../Shredding/ShredResult.cs | 109 ++++ .../Shredding/ShredSchema.cs | 293 ++++++++++ .../Shredding/ShredSchemaInferer.cs | 201 +++++++ .../Shredding/ShredType.cs | 90 +++ .../Shredding/ShreddedArray.cs | 151 ++++++ .../Shredding/ShreddedObject.cs | 172 ++++++ .../Shredding/ShreddedVariant.cs | 358 ++++++++++++ .../Shredding/ShreddedVariantArrayBuilder.cs | 513 ++++++++++++++++++ .../Shredding/ShreddingHelpers.cs | 48 ++ .../VariantArrayShreddingExtensions.cs | 93 ++++ .../Shredding/VariantShredder.cs | 312 +++++++++++ .../Shredding/VariantUnshredder.cs | 178 ++++++ .../Variant/VariantMetadataBuilder.cs | 32 ++ .../Variant/VariantValueWriter.cs | 86 +++ src/Apache.Arrow/Arrays/VariantArray.cs | 154 +++++- .../Shredding/ShredRoundTripTests.cs | 404 ++++++++++++++ .../Shredding/ShredSchemaInfererTests.cs | 259 +++++++++ .../Shredding/ShredSchemaTests.cs | 159 ++++++ .../ShreddedVariantArrayBuilderTests.cs | 424 +++++++++++++++ .../ShreddedVariantConformanceTests.cs | 236 ++++++++ .../ShreddedVariantErrorCaseTests.cs | 178 ++++++ .../Shredding/ShreddedVariantReaderTests.cs | 443 +++++++++++++++ .../Shredding/VariantShredderTests.cs | 496 +++++++++++++++++ .../Shredding/VariantUnshredderTests.cs | 265 +++++++++ .../VariantValueWriterCopyValueTests.cs | 413 ++++++++++++++ test/shredded_variant_ipc/case-001.arrow | Bin 0 -> 3138 bytes test/shredded_variant_ipc/case-002.arrow | Bin 0 -> 3098 bytes test/shredded_variant_ipc/case-004.arrow | Bin 0 -> 2298 bytes test/shredded_variant_ipc/case-005.arrow | Bin 0 -> 2298 bytes test/shredded_variant_ipc/case-006.arrow | Bin 0 -> 2298 bytes test/shredded_variant_ipc/case-007.arrow | Bin 0 -> 2298 bytes test/shredded_variant_ipc/case-008.arrow | Bin 0 -> 2298 bytes test/shredded_variant_ipc/case-009.arrow | Bin 0 -> 2298 bytes test/shredded_variant_ipc/case-010.arrow | Bin 0 -> 2298 bytes test/shredded_variant_ipc/case-011.arrow | Bin 0 -> 2298 bytes test/shredded_variant_ipc/case-012.arrow | Bin 0 -> 2298 bytes test/shredded_variant_ipc/case-013.arrow | Bin 0 -> 2298 bytes test/shredded_variant_ipc/case-014.arrow | Bin 0 -> 2314 bytes test/shredded_variant_ipc/case-015.arrow | Bin 0 -> 2314 bytes test/shredded_variant_ipc/case-016.arrow | Bin 0 -> 2314 bytes test/shredded_variant_ipc/case-017.arrow | Bin 0 -> 2314 bytes test/shredded_variant_ipc/case-018.arrow | Bin 0 -> 2362 bytes test/shredded_variant_ipc/case-019.arrow | Bin 0 -> 2362 bytes test/shredded_variant_ipc/case-020.arrow | Bin 0 -> 2426 bytes test/shredded_variant_ipc/case-021.arrow | Bin 0 -> 2426 bytes test/shredded_variant_ipc/case-022.arrow | Bin 0 -> 2410 bytes test/shredded_variant_ipc/case-023.arrow | Bin 0 -> 2410 bytes test/shredded_variant_ipc/case-024.arrow | Bin 0 -> 2306 bytes test/shredded_variant_ipc/case-025.arrow | Bin 0 -> 2306 bytes test/shredded_variant_ipc/case-026.arrow | Bin 0 -> 2306 bytes test/shredded_variant_ipc/case-027.arrow | Bin 0 -> 2306 bytes test/shredded_variant_ipc/case-028.arrow | Bin 0 -> 2434 bytes test/shredded_variant_ipc/case-029.arrow | Bin 0 -> 2434 bytes test/shredded_variant_ipc/case-030.arrow | Bin 0 -> 2306 bytes test/shredded_variant_ipc/case-031.arrow | Bin 0 -> 2322 bytes test/shredded_variant_ipc/case-032.arrow | Bin 0 -> 2394 bytes test/shredded_variant_ipc/case-033.arrow | Bin 0 -> 2330 bytes test/shredded_variant_ipc/case-034.arrow | Bin 0 -> 2330 bytes test/shredded_variant_ipc/case-035.arrow | Bin 0 -> 2314 bytes test/shredded_variant_ipc/case-036.arrow | Bin 0 -> 2314 bytes test/shredded_variant_ipc/case-037.arrow | Bin 0 -> 2626 bytes test/shredded_variant_ipc/case-038.arrow | Bin 0 -> 3386 bytes test/shredded_variant_ipc/case-039.arrow | Bin 0 -> 3986 bytes test/shredded_variant_ipc/case-040.arrow | Bin 0 -> 3114 bytes test/shredded_variant_ipc/case-041.arrow | Bin 0 -> 2866 bytes test/shredded_variant_ipc/case-042.arrow | Bin 0 -> 2298 bytes .../case-043-INVALID.arrow | Bin 0 -> 3986 bytes test/shredded_variant_ipc/case-044.arrow | Bin 0 -> 5754 bytes test/shredded_variant_ipc/case-045.arrow | Bin 0 -> 3274 bytes test/shredded_variant_ipc/case-046.arrow | Bin 0 -> 3970 bytes test/shredded_variant_ipc/case-047.arrow | Bin 0 -> 1842 bytes test/shredded_variant_ipc/case-048.arrow | Bin 0 -> 1842 bytes test/shredded_variant_ipc/case-049.arrow | Bin 0 -> 1842 bytes test/shredded_variant_ipc/case-050.arrow | Bin 0 -> 1842 bytes test/shredded_variant_ipc/case-051.arrow | Bin 0 -> 1842 bytes test/shredded_variant_ipc/case-052.arrow | Bin 0 -> 1842 bytes test/shredded_variant_ipc/case-053.arrow | Bin 0 -> 1842 bytes test/shredded_variant_ipc/case-054.arrow | Bin 0 -> 1842 bytes test/shredded_variant_ipc/case-055.arrow | Bin 0 -> 1842 bytes test/shredded_variant_ipc/case-056.arrow | Bin 0 -> 1850 bytes test/shredded_variant_ipc/case-057.arrow | Bin 0 -> 1850 bytes test/shredded_variant_ipc/case-058.arrow | Bin 0 -> 1842 bytes test/shredded_variant_ipc/case-059.arrow | Bin 0 -> 1842 bytes test/shredded_variant_ipc/case-060.arrow | Bin 0 -> 1850 bytes test/shredded_variant_ipc/case-061.arrow | Bin 0 -> 1850 bytes test/shredded_variant_ipc/case-062.arrow | Bin 0 -> 1842 bytes test/shredded_variant_ipc/case-063.arrow | Bin 0 -> 1842 bytes test/shredded_variant_ipc/case-064.arrow | Bin 0 -> 1850 bytes test/shredded_variant_ipc/case-065.arrow | Bin 0 -> 1850 bytes test/shredded_variant_ipc/case-066.arrow | Bin 0 -> 1850 bytes test/shredded_variant_ipc/case-067.arrow | Bin 0 -> 1850 bytes test/shredded_variant_ipc/case-068.arrow | Bin 0 -> 1842 bytes test/shredded_variant_ipc/case-069.arrow | Bin 0 -> 1842 bytes test/shredded_variant_ipc/case-070.arrow | Bin 0 -> 1850 bytes test/shredded_variant_ipc/case-071.arrow | Bin 0 -> 1850 bytes test/shredded_variant_ipc/case-072.arrow | Bin 0 -> 1858 bytes test/shredded_variant_ipc/case-073.arrow | Bin 0 -> 1858 bytes test/shredded_variant_ipc/case-074.arrow | Bin 0 -> 1850 bytes test/shredded_variant_ipc/case-075.arrow | Bin 0 -> 1842 bytes test/shredded_variant_ipc/case-076.arrow | Bin 0 -> 1850 bytes test/shredded_variant_ipc/case-077.arrow | Bin 0 -> 1850 bytes test/shredded_variant_ipc/case-078.arrow | Bin 0 -> 1850 bytes test/shredded_variant_ipc/case-079.arrow | Bin 0 -> 1850 bytes test/shredded_variant_ipc/case-080.arrow | Bin 0 -> 1850 bytes test/shredded_variant_ipc/case-081.arrow | Bin 0 -> 1858 bytes test/shredded_variant_ipc/case-082.arrow | Bin 0 -> 1858 bytes test/shredded_variant_ipc/case-083.arrow | Bin 0 -> 5994 bytes .../case-084-INVALID.arrow | Bin 0 -> 5794 bytes test/shredded_variant_ipc/case-085.arrow | Bin 0 -> 3114 bytes test/shredded_variant_ipc/case-086.arrow | Bin 0 -> 3154 bytes test/shredded_variant_ipc/case-087.arrow | Bin 0 -> 3986 bytes test/shredded_variant_ipc/case-088.arrow | Bin 0 -> 2858 bytes test/shredded_variant_ipc/case-089.arrow | Bin 0 -> 2322 bytes test/shredded_variant_ipc/case-090.arrow | Bin 0 -> 2322 bytes test/shredded_variant_ipc/case-091.arrow | Bin 0 -> 2322 bytes test/shredded_variant_ipc/case-092.arrow | Bin 0 -> 2322 bytes test/shredded_variant_ipc/case-093.arrow | Bin 0 -> 2322 bytes test/shredded_variant_ipc/case-094.arrow | Bin 0 -> 2322 bytes test/shredded_variant_ipc/case-095.arrow | Bin 0 -> 2322 bytes test/shredded_variant_ipc/case-096.arrow | Bin 0 -> 2322 bytes test/shredded_variant_ipc/case-097.arrow | Bin 0 -> 2322 bytes test/shredded_variant_ipc/case-098.arrow | Bin 0 -> 2330 bytes test/shredded_variant_ipc/case-099.arrow | Bin 0 -> 2330 bytes test/shredded_variant_ipc/case-100.arrow | Bin 0 -> 2322 bytes test/shredded_variant_ipc/case-101.arrow | Bin 0 -> 2322 bytes test/shredded_variant_ipc/case-102.arrow | Bin 0 -> 2330 bytes test/shredded_variant_ipc/case-103.arrow | Bin 0 -> 2330 bytes test/shredded_variant_ipc/case-104.arrow | Bin 0 -> 2322 bytes test/shredded_variant_ipc/case-105.arrow | Bin 0 -> 2322 bytes test/shredded_variant_ipc/case-106.arrow | Bin 0 -> 2330 bytes test/shredded_variant_ipc/case-107.arrow | Bin 0 -> 2330 bytes test/shredded_variant_ipc/case-108.arrow | Bin 0 -> 2330 bytes test/shredded_variant_ipc/case-109.arrow | Bin 0 -> 2330 bytes test/shredded_variant_ipc/case-110.arrow | Bin 0 -> 2322 bytes test/shredded_variant_ipc/case-111.arrow | Bin 0 -> 2322 bytes test/shredded_variant_ipc/case-112.arrow | Bin 0 -> 2330 bytes test/shredded_variant_ipc/case-113.arrow | Bin 0 -> 2330 bytes test/shredded_variant_ipc/case-114.arrow | Bin 0 -> 2338 bytes test/shredded_variant_ipc/case-115.arrow | Bin 0 -> 2338 bytes test/shredded_variant_ipc/case-116.arrow | Bin 0 -> 2330 bytes test/shredded_variant_ipc/case-117.arrow | Bin 0 -> 2322 bytes test/shredded_variant_ipc/case-118.arrow | Bin 0 -> 2330 bytes test/shredded_variant_ipc/case-119.arrow | Bin 0 -> 2330 bytes test/shredded_variant_ipc/case-120.arrow | Bin 0 -> 2330 bytes test/shredded_variant_ipc/case-121.arrow | Bin 0 -> 2330 bytes test/shredded_variant_ipc/case-122.arrow | Bin 0 -> 2330 bytes test/shredded_variant_ipc/case-123.arrow | Bin 0 -> 2338 bytes test/shredded_variant_ipc/case-124.arrow | Bin 0 -> 2338 bytes .../case-125-INVALID.arrow | Bin 0 -> 3986 bytes test/shredded_variant_ipc/case-126.arrow | Bin 0 -> 4930 bytes test/shredded_variant_ipc/case-127.arrow | Bin 0 -> 2322 bytes test/shredded_variant_ipc/case-128.arrow | Bin 0 -> 3978 bytes test/shredded_variant_ipc/case-129.arrow | Bin 0 -> 2306 bytes test/shredded_variant_ipc/case-130.arrow | Bin 0 -> 3978 bytes test/shredded_variant_ipc/case-131.arrow | Bin 0 -> 1930 bytes test/shredded_variant_ipc/case-132.arrow | Bin 0 -> 3418 bytes test/shredded_variant_ipc/case-133.arrow | Bin 0 -> 3978 bytes test/shredded_variant_ipc/case-134.arrow | Bin 0 -> 3986 bytes test/shredded_variant_ipc/case-135.arrow | Bin 0 -> 3106 bytes test/shredded_variant_ipc/case-136.arrow | Bin 0 -> 4010 bytes test/shredded_variant_ipc/case-137.arrow | Bin 0 -> 2402 bytes test/shredded_variant_ipc/case-138.arrow | Bin 0 -> 3610 bytes test/shredded_variant_ipc/regen.py | 79 +++ 165 files changed, 6189 insertions(+), 8 deletions(-) create mode 100644 src/Apache.Arrow.Operations/Shredding/ShredOptions.cs create mode 100644 src/Apache.Arrow.Operations/Shredding/ShredResult.cs create mode 100644 src/Apache.Arrow.Operations/Shredding/ShredSchema.cs create mode 100644 src/Apache.Arrow.Operations/Shredding/ShredSchemaInferer.cs create mode 100644 src/Apache.Arrow.Operations/Shredding/ShredType.cs create mode 100644 src/Apache.Arrow.Operations/Shredding/ShreddedArray.cs create mode 100644 src/Apache.Arrow.Operations/Shredding/ShreddedObject.cs create mode 100644 src/Apache.Arrow.Operations/Shredding/ShreddedVariant.cs create mode 100644 src/Apache.Arrow.Operations/Shredding/ShreddedVariantArrayBuilder.cs create mode 100644 src/Apache.Arrow.Operations/Shredding/ShreddingHelpers.cs create mode 100644 src/Apache.Arrow.Operations/Shredding/VariantArrayShreddingExtensions.cs create mode 100644 src/Apache.Arrow.Operations/Shredding/VariantShredder.cs create mode 100644 src/Apache.Arrow.Operations/Shredding/VariantUnshredder.cs create mode 100644 test/Apache.Arrow.Operations.Tests/Shredding/ShredRoundTripTests.cs create mode 100644 test/Apache.Arrow.Operations.Tests/Shredding/ShredSchemaInfererTests.cs create mode 100644 test/Apache.Arrow.Operations.Tests/Shredding/ShredSchemaTests.cs create mode 100644 test/Apache.Arrow.Operations.Tests/Shredding/ShreddedVariantArrayBuilderTests.cs create mode 100644 test/Apache.Arrow.Operations.Tests/Shredding/ShreddedVariantConformanceTests.cs create mode 100644 test/Apache.Arrow.Operations.Tests/Shredding/ShreddedVariantErrorCaseTests.cs create mode 100644 test/Apache.Arrow.Operations.Tests/Shredding/ShreddedVariantReaderTests.cs create mode 100644 test/Apache.Arrow.Operations.Tests/Shredding/VariantShredderTests.cs create mode 100644 test/Apache.Arrow.Operations.Tests/Shredding/VariantUnshredderTests.cs create mode 100644 test/Apache.Arrow.Scalars.Tests/VariantValueWriterCopyValueTests.cs create mode 100644 test/shredded_variant_ipc/case-001.arrow create mode 100644 test/shredded_variant_ipc/case-002.arrow create mode 100644 test/shredded_variant_ipc/case-004.arrow create mode 100644 test/shredded_variant_ipc/case-005.arrow create mode 100644 test/shredded_variant_ipc/case-006.arrow create mode 100644 test/shredded_variant_ipc/case-007.arrow create mode 100644 test/shredded_variant_ipc/case-008.arrow create mode 100644 test/shredded_variant_ipc/case-009.arrow create mode 100644 test/shredded_variant_ipc/case-010.arrow create mode 100644 test/shredded_variant_ipc/case-011.arrow create mode 100644 test/shredded_variant_ipc/case-012.arrow create mode 100644 test/shredded_variant_ipc/case-013.arrow create mode 100644 test/shredded_variant_ipc/case-014.arrow create mode 100644 test/shredded_variant_ipc/case-015.arrow create mode 100644 test/shredded_variant_ipc/case-016.arrow create mode 100644 test/shredded_variant_ipc/case-017.arrow create mode 100644 test/shredded_variant_ipc/case-018.arrow create mode 100644 test/shredded_variant_ipc/case-019.arrow create mode 100644 test/shredded_variant_ipc/case-020.arrow create mode 100644 test/shredded_variant_ipc/case-021.arrow create mode 100644 test/shredded_variant_ipc/case-022.arrow create mode 100644 test/shredded_variant_ipc/case-023.arrow create mode 100644 test/shredded_variant_ipc/case-024.arrow create mode 100644 test/shredded_variant_ipc/case-025.arrow create mode 100644 test/shredded_variant_ipc/case-026.arrow create mode 100644 test/shredded_variant_ipc/case-027.arrow create mode 100644 test/shredded_variant_ipc/case-028.arrow create mode 100644 test/shredded_variant_ipc/case-029.arrow create mode 100644 test/shredded_variant_ipc/case-030.arrow create mode 100644 test/shredded_variant_ipc/case-031.arrow create mode 100644 test/shredded_variant_ipc/case-032.arrow create mode 100644 test/shredded_variant_ipc/case-033.arrow create mode 100644 test/shredded_variant_ipc/case-034.arrow create mode 100644 test/shredded_variant_ipc/case-035.arrow create mode 100644 test/shredded_variant_ipc/case-036.arrow create mode 100644 test/shredded_variant_ipc/case-037.arrow create mode 100644 test/shredded_variant_ipc/case-038.arrow create mode 100644 test/shredded_variant_ipc/case-039.arrow create mode 100644 test/shredded_variant_ipc/case-040.arrow create mode 100644 test/shredded_variant_ipc/case-041.arrow create mode 100644 test/shredded_variant_ipc/case-042.arrow create mode 100644 test/shredded_variant_ipc/case-043-INVALID.arrow create mode 100644 test/shredded_variant_ipc/case-044.arrow create mode 100644 test/shredded_variant_ipc/case-045.arrow create mode 100644 test/shredded_variant_ipc/case-046.arrow create mode 100644 test/shredded_variant_ipc/case-047.arrow create mode 100644 test/shredded_variant_ipc/case-048.arrow create mode 100644 test/shredded_variant_ipc/case-049.arrow create mode 100644 test/shredded_variant_ipc/case-050.arrow create mode 100644 test/shredded_variant_ipc/case-051.arrow create mode 100644 test/shredded_variant_ipc/case-052.arrow create mode 100644 test/shredded_variant_ipc/case-053.arrow create mode 100644 test/shredded_variant_ipc/case-054.arrow create mode 100644 test/shredded_variant_ipc/case-055.arrow create mode 100644 test/shredded_variant_ipc/case-056.arrow create mode 100644 test/shredded_variant_ipc/case-057.arrow create mode 100644 test/shredded_variant_ipc/case-058.arrow create mode 100644 test/shredded_variant_ipc/case-059.arrow create mode 100644 test/shredded_variant_ipc/case-060.arrow create mode 100644 test/shredded_variant_ipc/case-061.arrow create mode 100644 test/shredded_variant_ipc/case-062.arrow create mode 100644 test/shredded_variant_ipc/case-063.arrow create mode 100644 test/shredded_variant_ipc/case-064.arrow create mode 100644 test/shredded_variant_ipc/case-065.arrow create mode 100644 test/shredded_variant_ipc/case-066.arrow create mode 100644 test/shredded_variant_ipc/case-067.arrow create mode 100644 test/shredded_variant_ipc/case-068.arrow create mode 100644 test/shredded_variant_ipc/case-069.arrow create mode 100644 test/shredded_variant_ipc/case-070.arrow create mode 100644 test/shredded_variant_ipc/case-071.arrow create mode 100644 test/shredded_variant_ipc/case-072.arrow create mode 100644 test/shredded_variant_ipc/case-073.arrow create mode 100644 test/shredded_variant_ipc/case-074.arrow create mode 100644 test/shredded_variant_ipc/case-075.arrow create mode 100644 test/shredded_variant_ipc/case-076.arrow create mode 100644 test/shredded_variant_ipc/case-077.arrow create mode 100644 test/shredded_variant_ipc/case-078.arrow create mode 100644 test/shredded_variant_ipc/case-079.arrow create mode 100644 test/shredded_variant_ipc/case-080.arrow create mode 100644 test/shredded_variant_ipc/case-081.arrow create mode 100644 test/shredded_variant_ipc/case-082.arrow create mode 100644 test/shredded_variant_ipc/case-083.arrow create mode 100644 test/shredded_variant_ipc/case-084-INVALID.arrow create mode 100644 test/shredded_variant_ipc/case-085.arrow create mode 100644 test/shredded_variant_ipc/case-086.arrow create mode 100644 test/shredded_variant_ipc/case-087.arrow create mode 100644 test/shredded_variant_ipc/case-088.arrow create mode 100644 test/shredded_variant_ipc/case-089.arrow create mode 100644 test/shredded_variant_ipc/case-090.arrow create mode 100644 test/shredded_variant_ipc/case-091.arrow create mode 100644 test/shredded_variant_ipc/case-092.arrow create mode 100644 test/shredded_variant_ipc/case-093.arrow create mode 100644 test/shredded_variant_ipc/case-094.arrow create mode 100644 test/shredded_variant_ipc/case-095.arrow create mode 100644 test/shredded_variant_ipc/case-096.arrow create mode 100644 test/shredded_variant_ipc/case-097.arrow create mode 100644 test/shredded_variant_ipc/case-098.arrow create mode 100644 test/shredded_variant_ipc/case-099.arrow create mode 100644 test/shredded_variant_ipc/case-100.arrow create mode 100644 test/shredded_variant_ipc/case-101.arrow create mode 100644 test/shredded_variant_ipc/case-102.arrow create mode 100644 test/shredded_variant_ipc/case-103.arrow create mode 100644 test/shredded_variant_ipc/case-104.arrow create mode 100644 test/shredded_variant_ipc/case-105.arrow create mode 100644 test/shredded_variant_ipc/case-106.arrow create mode 100644 test/shredded_variant_ipc/case-107.arrow create mode 100644 test/shredded_variant_ipc/case-108.arrow create mode 100644 test/shredded_variant_ipc/case-109.arrow create mode 100644 test/shredded_variant_ipc/case-110.arrow create mode 100644 test/shredded_variant_ipc/case-111.arrow create mode 100644 test/shredded_variant_ipc/case-112.arrow create mode 100644 test/shredded_variant_ipc/case-113.arrow create mode 100644 test/shredded_variant_ipc/case-114.arrow create mode 100644 test/shredded_variant_ipc/case-115.arrow create mode 100644 test/shredded_variant_ipc/case-116.arrow create mode 100644 test/shredded_variant_ipc/case-117.arrow create mode 100644 test/shredded_variant_ipc/case-118.arrow create mode 100644 test/shredded_variant_ipc/case-119.arrow create mode 100644 test/shredded_variant_ipc/case-120.arrow create mode 100644 test/shredded_variant_ipc/case-121.arrow create mode 100644 test/shredded_variant_ipc/case-122.arrow create mode 100644 test/shredded_variant_ipc/case-123.arrow create mode 100644 test/shredded_variant_ipc/case-124.arrow create mode 100644 test/shredded_variant_ipc/case-125-INVALID.arrow create mode 100644 test/shredded_variant_ipc/case-126.arrow create mode 100644 test/shredded_variant_ipc/case-127.arrow create mode 100644 test/shredded_variant_ipc/case-128.arrow create mode 100644 test/shredded_variant_ipc/case-129.arrow create mode 100644 test/shredded_variant_ipc/case-130.arrow create mode 100644 test/shredded_variant_ipc/case-131.arrow create mode 100644 test/shredded_variant_ipc/case-132.arrow create mode 100644 test/shredded_variant_ipc/case-133.arrow create mode 100644 test/shredded_variant_ipc/case-134.arrow create mode 100644 test/shredded_variant_ipc/case-135.arrow create mode 100644 test/shredded_variant_ipc/case-136.arrow create mode 100644 test/shredded_variant_ipc/case-137.arrow create mode 100644 test/shredded_variant_ipc/case-138.arrow create mode 100644 test/shredded_variant_ipc/regen.py diff --git a/src/Apache.Arrow.Operations/Apache.Arrow.Operations.csproj b/src/Apache.Arrow.Operations/Apache.Arrow.Operations.csproj index c4f0c453..524eb12d 100644 --- a/src/Apache.Arrow.Operations/Apache.Arrow.Operations.csproj +++ b/src/Apache.Arrow.Operations/Apache.Arrow.Operations.csproj @@ -18,6 +18,7 @@ + diff --git a/src/Apache.Arrow.Operations/Shredding/ShredOptions.cs b/src/Apache.Arrow.Operations/Shredding/ShredOptions.cs new file mode 100644 index 00000000..38450061 --- /dev/null +++ b/src/Apache.Arrow.Operations/Shredding/ShredOptions.cs @@ -0,0 +1,50 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +namespace Apache.Arrow.Operations.Shredding +{ + /// + /// Options controlling how infers a shredding schema. + /// + public sealed class ShredOptions + { + /// + /// Maximum nesting depth for shredded objects and arrays. + /// 0 means only top-level fields are shredded. + /// Default is 3. + /// + public int MaxDepth { get; set; } = 3; + + /// + /// Minimum fraction of values (0.0–1.0) in which a field must appear + /// to be considered for shredding. Fields appearing less frequently + /// than this threshold are left in the binary residual. + /// Default is 0.5 (50%). + /// + public double MinFieldFrequency { get; set; } = 0.5; + + /// + /// Minimum fraction of non-null values (0.0–1.0) for a field that must + /// share the same type for the field to be shredded as a typed column. + /// If the type consistency is below this threshold, the field gets a + /// schema (binary-only). + /// Default is 0.8 (80%). + /// + public double MinTypeConsistency { get; set; } = 0.8; + + /// Default options. + public static readonly ShredOptions Default = new ShredOptions(); + } +} diff --git a/src/Apache.Arrow.Operations/Shredding/ShredResult.cs b/src/Apache.Arrow.Operations/Shredding/ShredResult.cs new file mode 100644 index 00000000..dec9b6bd --- /dev/null +++ b/src/Apache.Arrow.Operations/Shredding/ShredResult.cs @@ -0,0 +1,109 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System.Collections.Generic; + +namespace Apache.Arrow.Operations.Shredding +{ + /// + /// The result of shredding a single variant value: a (value, typed_value) pair. + /// + /// Follows the Parquet variant shredding spec encoding matrix: + /// + /// Both null → missing (only valid for object sub-fields) + /// value non-null, typed_value null → unshredded (value in binary) + /// value null, typed_value non-null → fully shredded into typed column + /// Both non-null → partially shredded object (typed_value has shredded fields, value has residual) + /// + /// + /// + public sealed class ShredResult + { + /// + /// The residual variant value bytes. These reference the column-level metadata + /// returned by ; + /// they are NOT self-contained. Non-null when the value (or part of it) could + /// not be shredded into the typed column. For partially shredded objects this + /// contains only the unshredded fields. + /// + public byte[] Value { get; } + + /// + /// The typed value extracted according to the schema. The runtime type depends + /// on the : + /// + /// Primitives: the corresponding CLR type (bool, int, long, double, string, etc.) + /// Object: + /// Array: + /// + /// Null when the value does not match the schema type (falls back to binary). + /// + public object TypedValue { get; } + + /// + /// True when both and are null, + /// indicating the field is missing (only valid for object sub-fields). + /// + public bool IsMissing => Value == null && TypedValue == null; + + /// Creates a shred result. + public ShredResult(byte[] value, object typedValue) + { + Value = value; + TypedValue = typedValue; + } + + /// A missing result (both null). + public static readonly ShredResult Missing = new ShredResult(null, null); + } + + /// + /// The typed_value result for a shredded object. Contains one + /// per field defined in the object's . + /// + public sealed class ShredObjectResult + { + /// + /// Shredded fields, keyed by field name matching the . + /// Each entry is the shredded (value, typed_value) pair for that field. + /// + public IReadOnlyDictionary Fields { get; } + + /// Creates a shredded object result. + public ShredObjectResult(IReadOnlyDictionary fields) + { + Fields = fields; + } + } + + /// + /// The typed_value result for a shredded array. Contains one + /// per element in the source array. + /// + public sealed class ShredArrayResult + { + /// + /// Shredded elements. Each entry is the shredded (value, typed_value) pair for that element. + /// Array elements are never missing — null elements are encoded as variant null in the value column. + /// + public IReadOnlyList Elements { get; } + + /// Creates a shredded array result. + public ShredArrayResult(IReadOnlyList elements) + { + Elements = elements; + } + } +} diff --git a/src/Apache.Arrow.Operations/Shredding/ShredSchema.cs b/src/Apache.Arrow.Operations/Shredding/ShredSchema.cs new file mode 100644 index 00000000..e1ea5e3c --- /dev/null +++ b/src/Apache.Arrow.Operations/Shredding/ShredSchema.cs @@ -0,0 +1,293 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System; +using System.Collections.Generic; +using Apache.Arrow; +using Apache.Arrow.Scalars.Variant; +using Apache.Arrow.Types; + +namespace Apache.Arrow.Operations.Shredding +{ + /// + /// Describes the shredding schema for a variant column — which fields + /// to extract into typed Parquet columns and at what types. + /// + public sealed class ShredSchema + { + /// + /// The type of the typed_value column. For primitives, this is the + /// expected scalar type. For objects, use + /// and populate . For arrays, use + /// and populate . + /// means no typed_value — everything goes to binary value. + /// + public ShredType TypedValueType { get; } + + /// + /// For : the shredding schemas for each named sub-field. + /// Null for non-object types. + /// + public IReadOnlyDictionary ObjectFields { get; } + + /// + /// For : the shredding schema applied to each element. + /// Null for non-array types. + /// + public ShredSchema ArrayElement { get; } + + private ShredSchema(ShredType typedValueType, IReadOnlyDictionary objectFields, ShredSchema arrayElement) + { + TypedValueType = typedValueType; + ObjectFields = objectFields; + ArrayElement = arrayElement; + } + + /// Creates a schema that does no shredding (all values go to binary). + public static ShredSchema Unshredded() => new ShredSchema(ShredType.None, null, null); + + /// + /// Creates a schema that shreds values into a typed primitive column. + /// Values not matching this type fall back to the binary value column. + /// + public static ShredSchema Primitive(ShredType type) + { + if (type == ShredType.None || type == ShredType.Object || type == ShredType.Array) + { + throw new ArgumentException($"Use the appropriate factory method for {type}.", nameof(type)); + } + return new ShredSchema(type, null, null); + } + + /// + /// Creates a schema that shreds object values by extracting named fields + /// into typed sub-columns. + /// + public static ShredSchema ForObject(IDictionary fields) + { + if (fields == null) throw new ArgumentNullException(nameof(fields)); + Dictionary copy = new Dictionary(fields); + return new ShredSchema(ShredType.Object, copy, null); + } + + /// + /// Creates a schema that shreds array values by applying the element + /// schema to each element. + /// + public static ShredSchema ForArray(ShredSchema elementSchema) + { + if (elementSchema == null) throw new ArgumentNullException(nameof(elementSchema)); + return new ShredSchema(ShredType.Array, null, elementSchema); + } + + /// + /// Maps a to the corresponding . + /// + public static ShredType ShredTypeFromPrimitive(VariantPrimitiveType primitiveType) + { + switch (primitiveType) + { + case VariantPrimitiveType.BooleanTrue: + case VariantPrimitiveType.BooleanFalse: + return ShredType.Boolean; + case VariantPrimitiveType.Int8: return ShredType.Int8; + case VariantPrimitiveType.Int16: return ShredType.Int16; + case VariantPrimitiveType.Int32: return ShredType.Int32; + case VariantPrimitiveType.Int64: return ShredType.Int64; + case VariantPrimitiveType.Float: return ShredType.Float; + case VariantPrimitiveType.Double: return ShredType.Double; + case VariantPrimitiveType.Decimal4: return ShredType.Decimal4; + case VariantPrimitiveType.Decimal8: return ShredType.Decimal8; + case VariantPrimitiveType.Decimal16: return ShredType.Decimal16; + case VariantPrimitiveType.Date: return ShredType.Date; + case VariantPrimitiveType.Timestamp: return ShredType.Timestamp; + case VariantPrimitiveType.TimestampNtz: return ShredType.TimestampNtz; + case VariantPrimitiveType.TimeNtz: return ShredType.TimeNtz; + case VariantPrimitiveType.TimestampTzNanos: return ShredType.TimestampTzNanos; + case VariantPrimitiveType.TimestampNtzNanos: return ShredType.TimestampNtzNanos; + case VariantPrimitiveType.String: return ShredType.String; + case VariantPrimitiveType.Binary: return ShredType.Binary; + case VariantPrimitiveType.Uuid: return ShredType.Uuid; + default: return ShredType.None; + } + } + + /// + /// Derives a from the Arrow type of a typed_value column. + /// + /// + /// The typed_value Arrow type, or null for a fully unshredded column. + /// + /// A describing the shredding. + /// + /// Thrown when is not a valid shredded type + /// per the Parquet variant shredding spec (for example, an unsigned integer or + /// a fixed-size binary that isn't UUID). + /// + public static ShredSchema FromArrowType(IArrowType typedValueType) + { + if (typedValueType == null) return Unshredded(); + return MapArrowType(typedValueType); + } + + private static ShredSchema MapArrowType(IArrowType type) + { + switch (type) + { + case BooleanType _: return Primitive(ShredType.Boolean); + case Int8Type _: return Primitive(ShredType.Int8); + case Int16Type _: return Primitive(ShredType.Int16); + case Int32Type _: return Primitive(ShredType.Int32); + case Int64Type _: return Primitive(ShredType.Int64); + case FloatType _: return Primitive(ShredType.Float); + case DoubleType _: return Primitive(ShredType.Double); + case StringType _: return Primitive(ShredType.String); + case BinaryType _: return Primitive(ShredType.Binary); + case LargeBinaryType _: return Primitive(ShredType.Binary); + case LargeStringType _: return Primitive(ShredType.String); + case Date32Type _: return Primitive(ShredType.Date); + + case Time64Type t when t.Unit == TimeUnit.Microsecond: + return Primitive(ShredType.TimeNtz); + + case TimestampType ts when ts.Unit == TimeUnit.Microsecond && ts.IsTimeZoneAware: + return Primitive(ShredType.Timestamp); + case TimestampType ts when ts.Unit == TimeUnit.Microsecond && !ts.IsTimeZoneAware: + return Primitive(ShredType.TimestampNtz); + case TimestampType ts when ts.Unit == TimeUnit.Nanosecond && ts.IsTimeZoneAware: + return Primitive(ShredType.TimestampTzNanos); + case TimestampType ts when ts.Unit == TimeUnit.Nanosecond && !ts.IsTimeZoneAware: + return Primitive(ShredType.TimestampNtzNanos); + + // The Parquet variant spec allows any Arrow decimal representation + // whose precision fits in one of the variant's decimal widths + // (≤9 digits → 4-byte unscaled, ≤18 → 8-byte, ≤38 → 16-byte). + // Decimal128Type extends FixedSizeBinaryType with byte_width=16, so + // we MUST match the decimal cases before the UUID fallback below, + // and dispatch by precision inside the cases rather than via `when` + // guards that can fall through into the FSB(16) branch. + case Decimal32Type d32: return MapDecimalByPrecision(d32.Precision, type); + case Decimal64Type d64: return MapDecimalByPrecision(d64.Precision, type); + case Decimal128Type d128: return MapDecimalByPrecision(d128.Precision, type); + + case ExtensionType ext when ext.Name == "arrow.uuid": + return Primitive(ShredType.Uuid); + + // When the Arrow IPC reader has no UUID extension registered, the + // column comes through as its storage type (16-byte fixed binary). + // Per the Parquet variant shredding spec, fixed_size_binary(16) is + // the only valid fixed-size binary type and represents UUID. + case FixedSizeBinaryType fsb when fsb.ByteWidth == 16: + return Primitive(ShredType.Uuid); + + case ListType list: + return MapArrayType(list); + + case StructType structType: + return MapObjectType(structType); + + default: + throw new ArgumentException( + $"Unsupported shredded value type: {type}", + nameof(type)); + } + } + + private static ShredSchema MapDecimalByPrecision(int precision, IArrowType type) + { + if (precision <= 9) return Primitive(ShredType.Decimal4); + if (precision <= 18) return Primitive(ShredType.Decimal8); + if (precision <= 38) return Primitive(ShredType.Decimal16); + throw new ArgumentException( + $"Unsupported decimal precision {precision} (max 38): {type}", + nameof(type)); + } + + private static ShredSchema MapArrayType(ListType list) + { + if (!(list.ValueDataType is StructType elementStruct) || !IsElementGroupStruct(elementStruct)) + { + throw new ArgumentException( + "Shredded array element must be a struct with 'value' and/or 'typed_value' fields.", + nameof(list)); + } + return ForArray(ParseElementGroup(elementStruct)); + } + + private static ShredSchema MapObjectType(StructType structType) + { + Dictionary fields = new Dictionary(structType.Fields.Count); + foreach (Field field in structType.Fields) + { + if (!(field.DataType is StructType elementGroup) || !IsElementGroupStruct(elementGroup)) + { + throw new ArgumentException( + $"Shredded object field '{field.Name}' must be a struct with 'value' and/or 'typed_value' fields.", + nameof(structType)); + } + fields[field.Name] = ParseElementGroup(elementGroup); + } + return ForObject(fields); + } + + /// + /// Tests whether a struct type is a valid shredded "element group": + /// a struct with at least one of value (binary) or typed_value, + /// and no other fields. + /// + private static bool IsElementGroupStruct(StructType st) + { + int valueIdx = st.GetFieldIndex("value"); + int typedIdx = st.GetFieldIndex("typed_value"); + + if (valueIdx < 0 && typedIdx < 0) + { + return false; + } + + if (valueIdx >= 0) + { + IArrowType valueFieldType = st.Fields[valueIdx].DataType; + if (!(valueFieldType is BinaryType || + valueFieldType is LargeBinaryType || + valueFieldType is BinaryViewType)) + { + return false; + } + } + + // Reject structs with unexpected extra fields. + foreach (Field f in st.Fields) + { + if (f.Name != "value" && f.Name != "typed_value") + { + return false; + } + } + + return true; + } + + private static ShredSchema ParseElementGroup(StructType elementStruct) + { + int typedIdx = elementStruct.GetFieldIndex("typed_value"); + if (typedIdx < 0) + { + return Unshredded(); + } + return MapArrowType(elementStruct.Fields[typedIdx].DataType); + } + } +} diff --git a/src/Apache.Arrow.Operations/Shredding/ShredSchemaInferer.cs b/src/Apache.Arrow.Operations/Shredding/ShredSchemaInferer.cs new file mode 100644 index 00000000..4537c3cb --- /dev/null +++ b/src/Apache.Arrow.Operations/Shredding/ShredSchemaInferer.cs @@ -0,0 +1,201 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System; +using System.Collections.Generic; +using Apache.Arrow.Scalars.Variant; + +namespace Apache.Arrow.Operations.Shredding +{ + /// + /// Analyzes a batch of s and infers an optimal + /// for shredding them. + /// + public sealed class ShredSchemaInferer + { + /// + /// Infers a shredding schema by analyzing the given values. + /// + /// The variant values to analyze. + /// Options controlling depth, frequency, and type consistency thresholds. + /// An inferred . + public ShredSchema Infer(IEnumerable values, ShredOptions options = null) + { + if (values == null) throw new ArgumentNullException(nameof(values)); + if (options == null) options = ShredOptions.Default; + + TypeStats stats = new TypeStats(); + int totalCount = 0; + + foreach (VariantValue value in values) + { + CollectStats(value, stats, 0, options.MaxDepth); + totalCount++; + } + + if (totalCount == 0) + { + return ShredSchema.Unshredded(); + } + + return BuildSchema(stats, totalCount, options, 0); + } + + private void CollectStats(VariantValue value, TypeStats stats, int depth, int maxDepth) + { + ShredType type = VariantShredder.GetShredType(value); + stats.TypeCounts.TryGetValue(type, out int count); + stats.TypeCounts[type] = count + 1; + + if (type == ShredType.Object && depth <= maxDepth && value.IsObject) + { + if (stats.ObjectFieldStats == null) + { + stats.ObjectFieldStats = new Dictionary(); + } + + foreach (KeyValuePair field in value.AsObject()) + { + if (!stats.ObjectFieldStats.TryGetValue(field.Key, out TypeStats fieldStats)) + { + fieldStats = new TypeStats(); + stats.ObjectFieldStats[field.Key] = fieldStats; + } + CollectStats(field.Value, fieldStats, depth + 1, maxDepth); + } + } + else if (type == ShredType.Array && depth <= maxDepth && value.IsArray) + { + if (stats.ArrayElementStats == null) + { + stats.ArrayElementStats = new TypeStats(); + } + + foreach (VariantValue element in value.AsArray()) + { + CollectStats(element, stats.ArrayElementStats, depth + 1, maxDepth); + } + } + } + + private ShredSchema BuildSchema(TypeStats stats, int totalCount, ShredOptions options, int depth) + { + // Find the dominant type. + ShredType dominantType = ShredType.None; + int dominantCount = 0; + int nonNullCount = 0; + + foreach (KeyValuePair entry in stats.TypeCounts) + { + if (entry.Key != ShredType.None) + { + nonNullCount += entry.Value; + if (entry.Value > dominantCount) + { + dominantCount = entry.Value; + dominantType = entry.Key; + } + } + } + + if (nonNullCount == 0) + { + return ShredSchema.Unshredded(); + } + + // Check type consistency. + double consistency = (double)dominantCount / nonNullCount; + if (consistency < options.MinTypeConsistency) + { + return ShredSchema.Unshredded(); + } + + if (dominantType == ShredType.Object && stats.ObjectFieldStats != null) + { + return BuildObjectSchema(stats, totalCount, dominantCount, options, depth); + } + + if (dominantType == ShredType.Array && stats.ArrayElementStats != null) + { + return BuildArraySchema(stats, dominantCount, options, depth); + } + + // Object/Array without collected sub-stats (e.g., maxDepth reached) — can't shred further. + if (dominantType == ShredType.Object || dominantType == ShredType.Array) + { + return ShredSchema.Unshredded(); + } + + // Primitive type — shred as that type. + return ShredSchema.Primitive(dominantType); + } + + private ShredSchema BuildObjectSchema(TypeStats stats, int totalCount, int objectCount, ShredOptions options, int depth) + { + Dictionary fields = new Dictionary(); + + foreach (KeyValuePair fieldEntry in stats.ObjectFieldStats) + { + // Check field frequency: how often does this field appear relative to the number of objects? + int fieldAppearances = 0; + foreach (KeyValuePair tc in fieldEntry.Value.TypeCounts) + { + fieldAppearances += tc.Value; + } + + double frequency = (double)fieldAppearances / objectCount; + if (frequency < options.MinFieldFrequency) + { + continue; + } + + ShredSchema fieldSchema = BuildSchema(fieldEntry.Value, fieldAppearances, options, depth + 1); + fields[fieldEntry.Key] = fieldSchema; + } + + if (fields.Count == 0) + { + return ShredSchema.Unshredded(); + } + + return ShredSchema.ForObject(fields); + } + + private ShredSchema BuildArraySchema(TypeStats stats, int arrayCount, ShredOptions options, int depth) + { + // Count total elements across all arrays. + int totalElements = 0; + foreach (KeyValuePair entry in stats.ArrayElementStats.TypeCounts) + { + totalElements += entry.Value; + } + + if (totalElements == 0) + { + return ShredSchema.Unshredded(); + } + + ShredSchema elementSchema = BuildSchema(stats.ArrayElementStats, totalElements, options, depth + 1); + return ShredSchema.ForArray(elementSchema); + } + + private sealed class TypeStats + { + public Dictionary TypeCounts { get; } = new Dictionary(); + public Dictionary ObjectFieldStats { get; set; } + public TypeStats ArrayElementStats { get; set; } + } + } +} diff --git a/src/Apache.Arrow.Operations/Shredding/ShredType.cs b/src/Apache.Arrow.Operations/Shredding/ShredType.cs new file mode 100644 index 00000000..19f1a98c --- /dev/null +++ b/src/Apache.Arrow.Operations/Shredding/ShredType.cs @@ -0,0 +1,90 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +namespace Apache.Arrow.Operations.Shredding +{ + /// + /// Describes the type that a shredded typed_value column expects. + /// Maps variant primitive types to the logical Parquet types used for shredding. + /// + public enum ShredType : byte + { + /// No typed_value column — all values go to the binary value column. + None = 0, + + /// Boolean (Parquet BOOLEAN). + Boolean, + + /// 8-bit signed integer (Parquet INT32 with INT_8 annotation). + Int8, + + /// 16-bit signed integer (Parquet INT32 with INT_16 annotation). + Int16, + + /// 32-bit signed integer (Parquet INT32). + Int32, + + /// 64-bit signed integer (Parquet INT64). + Int64, + + /// 32-bit float (Parquet FLOAT). + Float, + + /// 64-bit double (Parquet DOUBLE). + Double, + + /// Decimal with 4-byte unscaled value. + Decimal4, + + /// Decimal with 8-byte unscaled value. + Decimal8, + + /// Decimal with 16-byte unscaled value. + Decimal16, + + /// Date as days since epoch (Parquet DATE). + Date, + + /// Timestamp with UTC microseconds (Parquet TIMESTAMP with isAdjustedToUTC=true, MICROS). + Timestamp, + + /// Timestamp without timezone, microseconds (Parquet TIMESTAMP with isAdjustedToUTC=false, MICROS). + TimestampNtz, + + /// Time without timezone, microseconds (Parquet TIME with MICROS). + TimeNtz, + + /// Timestamp with UTC nanoseconds. + TimestampTzNanos, + + /// Timestamp without timezone, nanoseconds. + TimestampNtzNanos, + + /// UTF-8 string (Parquet BINARY with STRING logical type). + String, + + /// Binary data (Parquet BINARY). + Binary, + + /// UUID (Parquet FIXED_LEN_BYTE_ARRAY(16) with UUID logical type). + Uuid, + + /// Shredded as an object group with named sub-fields. + Object, + + /// Shredded as an array (Parquet LIST). + Array, + } +} diff --git a/src/Apache.Arrow.Operations/Shredding/ShreddedArray.cs b/src/Apache.Arrow.Operations/Shredding/ShreddedArray.cs new file mode 100644 index 00000000..9d1afed9 --- /dev/null +++ b/src/Apache.Arrow.Operations/Shredding/ShreddedArray.cs @@ -0,0 +1,151 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System; +using System.Collections.Generic; +using Apache.Arrow; +using Apache.Arrow.Scalars.Variant; + +namespace Apache.Arrow.Operations.Shredding +{ + /// + /// Reader for a single row of a shredded-array slot. The underlying storage + /// is a list of element groups (each a {value, typed_value} struct). + /// + public ref struct ShreddedArray + { + private readonly ShredSchema _schema; + private readonly ReadOnlySpan _metadata; + // The typed_value list (elements are {value, typed_value} structs). May be null. + private readonly ListArray _list; + // The residual binary column at the array level (for unshredded arrays). May be null. + private readonly IArrowArray _residual; + private readonly int _row; + + internal ShreddedArray( + ShredSchema schema, + ReadOnlySpan metadata, + ListArray list, + IArrowArray residual, + int row) + { + _schema = schema; + _metadata = metadata; + _list = list; + _residual = residual; + _row = row; + } + + /// + /// True when the typed list is populated at this row (the array is stored + /// element-by-element in the shredded column). + /// + public bool IsTypedList => _list != null && !_list.IsNull(_row); + + /// + /// The number of shredded elements at this row. Only valid when + /// is true. + /// + /// If the array is stored as a residual. + public int ElementCount + { + get + { + if (!IsTypedList) + { + throw new InvalidOperationException( + "Array at this row is stored as a residual (not a typed list). " + + "Use TryGetResidualReader and iterate via VariantArrayReader."); + } + return _list.ValueOffsets[_row + 1] - _list.ValueOffsets[_row]; + } + } + + /// + /// Gets a reader for the element at position + /// . Only valid when is true. + /// + public ShreddedVariant GetElement(int index) + { + if (!IsTypedList) + { + throw new InvalidOperationException( + "Array at this row is stored as a residual (not a typed list)."); + } + int start = _list.ValueOffsets[_row]; + int end = _list.ValueOffsets[_row + 1]; + if ((uint)index >= (uint)(end - start)) + { + throw new ArgumentOutOfRangeException(nameof(index)); + } + StructArray elementGroup = (StructArray)_list.Values; + return ShreddingHelpers.BuildSlot(_schema.ArrayElement, _metadata, elementGroup, start + index); + } + + /// + /// If the array is stored as a residual at this row (not shredded), returns + /// a over the residual bytes. Callers can then + /// inspect the array via VariantArrayReader. + /// + public bool TryGetResidualReader(out VariantReader reader) + { + if (_residual == null || _residual.IsNull(_row)) + { + reader = default; + return false; + } + ReadOnlySpan bytes = ((BinaryArray)_residual).GetBytes(_row, out _); + reader = new VariantReader(_metadata, bytes); + return true; + } + + /// + /// Materializes the array into a . If the typed + /// list is null at this row, falls back to the residual binary (the array + /// was stored unshredded for this row). + /// + public VariantValue ToVariantValue() + { + if (_list != null && !_list.IsNull(_row)) + { + int start = _list.ValueOffsets[_row]; + int end = _list.ValueOffsets[_row + 1]; + int count = end - start; + + StructArray elementGroup = (StructArray)_list.Values; + List elements = new List(count); + for (int i = start; i < end; i++) + { + // For array elements, a both-null slot encodes a variant null + // (arrays cannot contain "missing"). ShreddedVariant.ToVariantValue + // already returns VariantValue.Null for a missing slot. + ShreddedVariant slot = ShreddingHelpers.BuildSlot(_schema.ArrayElement, _metadata, elementGroup, i); + elements.Add(slot.ToVariantValue()); + } + return VariantValue.FromArray(elements); + } + + // No typed list at this row — decode from the residual. + if (_residual == null || _residual.IsNull(_row)) + { + throw new InvalidOperationException( + "Shredded array slot has neither typed list nor residual bytes."); + } + BinaryArray residualBinary = (BinaryArray)_residual; + ReadOnlySpan bytes = residualBinary.GetBytes(_row, out _); + return new VariantReader(_metadata, bytes).ToVariantValue(); + } + } +} diff --git a/src/Apache.Arrow.Operations/Shredding/ShreddedObject.cs b/src/Apache.Arrow.Operations/Shredding/ShreddedObject.cs new file mode 100644 index 00000000..07e23811 --- /dev/null +++ b/src/Apache.Arrow.Operations/Shredding/ShreddedObject.cs @@ -0,0 +1,172 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System; +using System.Collections.Generic; +using Apache.Arrow; +using Apache.Arrow.Scalars.Variant; +using Apache.Arrow.Types; + +namespace Apache.Arrow.Operations.Shredding +{ + /// + /// Reader for a single row of a shredded-object slot. Provides field-wise + /// access to both typed sub-columns and residual unshredded fields. + /// + public ref struct ShreddedObject + { + private readonly ShredSchema _schema; + private readonly ReadOnlySpan _metadata; + // The typed_value struct (one field per shredded field, each itself a {value, typed_value} struct). + // May be null if this row's typed_value column is null (i.e., the whole slot is in residual). + private readonly StructArray _fields; + // The residual value at this level (a binary column holding unshredded fields). May be null. + private readonly IArrowArray _residual; + private readonly int _index; + + internal ShreddedObject( + ShredSchema schema, + ReadOnlySpan metadata, + StructArray typedValueStruct, + IArrowArray residualValue, + int index) + { + _schema = schema; + _metadata = metadata; + _fields = typedValueStruct; + _residual = residualValue; + _index = index; + } + + /// The names of the shredded fields, in schema order. + public IEnumerable FieldNames => _schema.ObjectFields.Keys; + + /// + /// Gets the shredded reader for a named field. The field must exist in the schema. + /// + /// If is not a shredded field. + public ShreddedVariant GetField(string name) + { + if (!TryGetField(name, out ShreddedVariant field)) + { + throw new KeyNotFoundException($"Field '{name}' is not in the shredded object schema."); + } + return field; + } + + /// + /// Tries to get a reader for a shredded sub-field by name. Returns false if + /// isn't a shredded field (it may still exist in the + /// residual — use to inspect). + /// + public bool TryGetField(string name, out ShreddedVariant field) + { + if (!_schema.ObjectFields.TryGetValue(name, out ShredSchema fieldSchema)) + { + field = default; + return false; + } + if (_fields == null || _fields.IsNull(_index)) + { + // typed_value is null at this row — the field is effectively missing + // from the typed column. Return a slot with no typed/residual set. + field = new ShreddedVariant(fieldSchema, _metadata, null, null, _index); + return true; + } + StructType fieldsStructType = (StructType)_fields.Data.DataType; + int fieldIdx = fieldsStructType.GetFieldIndex(name); + StructArray elementGroup = (StructArray)_fields.Fields[fieldIdx]; + field = ShreddingHelpers.BuildSlot(fieldSchema, _metadata, elementGroup, _index); + return true; + } + + /// + /// If the object's residual binary is populated at this row, returns a + /// over it. The residual holds whatever fields + /// were not shredded (or, for a non-object row, the whole value). + /// + public bool TryGetResidualReader(out VariantReader reader) + { + if (_residual == null || _residual.IsNull(_index)) + { + reader = default; + return false; + } + ReadOnlySpan bytes = ((BinaryArray)_residual).GetBytes(_index, out _); + reader = new VariantReader(_metadata, bytes); + return true; + } + + /// + /// Materializes the whole shredded object into a , + /// merging typed-column fields with residual unshredded fields. When the + /// typed_value column is null at this row, the residual is returned + /// as-is (it may be any variant type, not just an object). + /// + public VariantValue ToVariantValue() + { + bool typedPopulated = _fields != null && !_fields.IsNull(_index); + bool residualPopulated = _residual != null && !_residual.IsNull(_index); + + if (!typedPopulated && !residualPopulated) + { + return VariantValue.Null; + } + + // No shredded fields at this row — whatever is in the residual IS the value. + if (!typedPopulated) + { + BinaryArray binary = (BinaryArray)_residual; + ReadOnlySpan bytes = binary.GetBytes(_index, out _); + return new VariantReader(_metadata, bytes).ToVariantValue(); + } + + Dictionary fields = new Dictionary(); + + // Shredded fields (from typed_value). + StructType fieldsStructType = (StructType)_fields.Data.DataType; + foreach (KeyValuePair entry in _schema.ObjectFields) + { + int fieldIdx = fieldsStructType.GetFieldIndex(entry.Key); + StructArray elementGroup = (StructArray)_fields.Fields[fieldIdx]; + ShreddedVariant slot = ShreddingHelpers.BuildSlot(entry.Value, _metadata, elementGroup, _index); + if (!slot.IsMissing) + { + fields[entry.Key] = slot.ToVariantValue(); + } + } + + // Partially shredded object — merge residual unshredded fields. + if (residualPopulated) + { + BinaryArray residualBinary = (BinaryArray)_residual; + ReadOnlySpan residualBytes = residualBinary.GetBytes(_index, out _); + VariantReader residualReader = new VariantReader(_metadata, residualBytes); + if (!residualReader.IsObject) + { + throw new InvalidOperationException( + "Residual value for a partially shredded object must itself be a variant object."); + } + VariantValue residual = residualReader.ToVariantValue(); + foreach (KeyValuePair kv in residual.AsObject()) + { + fields[kv.Key] = kv.Value; + } + } + + return VariantValue.FromObject(fields); + } + } +} diff --git a/src/Apache.Arrow.Operations/Shredding/ShreddedVariant.cs b/src/Apache.Arrow.Operations/Shredding/ShreddedVariant.cs new file mode 100644 index 00000000..8f3ae919 --- /dev/null +++ b/src/Apache.Arrow.Operations/Shredding/ShreddedVariant.cs @@ -0,0 +1,358 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System; +using System.Data.SqlTypes; +using Apache.Arrow.Arrays; +using Apache.Arrow.Scalars.Variant; + +namespace Apache.Arrow.Operations.Shredding +{ + /// + /// Zero-copy reader for a single row of a (possibly shredded) variant column. + /// Composes with the for this position to expose the + /// typed columns and residual bytes side-by-side, or to materialize the logical + /// value on demand. + /// + /// A does not own any Arrow buffers; it is only + /// valid while the underlying Arrow arrays are alive. + /// + /// + public ref struct ShreddedVariant + { + private readonly ShredSchema _schema; + private readonly ReadOnlySpan _metadata; + // _valueArray is the residual binary column at this level (may be null). + private readonly IArrowArray _valueArray; + // _typedValueArray is the typed column at this level (may be null if no shredding here). + private readonly IArrowArray _typedValueArray; + private readonly int _index; + + internal ShreddedVariant( + ShredSchema schema, + ReadOnlySpan metadata, + IArrowArray valueArray, + IArrowArray typedValueArray, + int index) + { + _schema = schema ?? throw new ArgumentNullException(nameof(schema)); + _metadata = metadata; + _valueArray = valueArray; + _typedValueArray = typedValueArray; + _index = index; + } + + /// The schema describing how this slot is shredded. + public ShredSchema Schema => _schema; + + /// The column-level variant metadata. + public ReadOnlySpan Metadata => _metadata; + + /// True when the residual value column has a value at this index. + public bool HasResidual => _valueArray != null && !_valueArray.IsNull(_index); + + /// True when the typed_value column has a value at this index. + public bool HasTypedValue => _typedValueArray != null && !_typedValueArray.IsNull(_index); + + /// + /// True when neither the residual nor the typed column is populated at this index + /// — valid only for sub-fields of shredded objects. + /// + public bool IsMissing => !HasResidual && !HasTypedValue; + + /// + /// Materializes this slot into a logical , merging + /// typed-column values with residual bytes per the shredding spec. + /// + /// If the slot is missing. + public VariantValue ToVariantValue() + { + // Both-null at this slot means the logical value is variant null. (The + // "missing" encoding — omitting the field entirely from the output — is + // a choice made by the container: see ShreddedObject, which uses + // IsMissing to decide whether to drop a sub-field.) + if (IsMissing) + { + return VariantValue.Null; + } + + switch (_schema.TypedValueType) + { + case ShredType.None: + return ReadResidual(); + + case ShredType.Object: + return GetObject().ToVariantValue(); + + case ShredType.Array: + return GetArray().ToVariantValue(); + + default: + // Primitive shredding. Per the Parquet variant shredding spec, a + // primitive slot may have at most one of value / typed_value set. + // If both are populated at the same row, the shredded data is + // invalid and implementations should reject it. + if (HasTypedValue) + { + if (HasResidual) + { + throw new InvalidOperationException( + "Invalid shredded variant: primitive slot has both 'value' and 'typed_value' populated."); + } + return ReadTypedPrimitive(); + } + return ReadResidual(); + } + } + + /// + /// If the residual column has a value at this index, returns a + /// over its bytes. + /// + public bool TryGetResidualReader(out VariantReader reader) + { + if (HasResidual) + { + BinaryArray binary = (BinaryArray)_valueArray; + ReadOnlySpan bytes = binary.GetBytes(_index, out _); + reader = new VariantReader(_metadata, bytes); + return true; + } + reader = default; + return false; + } + + /// + /// Reader for a shredded object at this slot. Valid only when the schema's + /// is . + /// + public ShreddedObject GetObject() + { + if (_schema.TypedValueType != ShredType.Object) + { + throw new InvalidOperationException( + $"Slot is not shredded as an object (schema type {_schema.TypedValueType})."); + } + return new ShreddedObject(_schema, _metadata, _typedValueArray as StructArray, _valueArray, _index); + } + + /// + /// Reader for a shredded array at this slot. Valid only when the schema's + /// is . + /// + public ShreddedArray GetArray() + { + if (_schema.TypedValueType != ShredType.Array) + { + throw new InvalidOperationException( + $"Slot is not shredded as an array (schema type {_schema.TypedValueType})."); + } + return new ShreddedArray(_schema, _metadata, _typedValueArray as ListArray, _valueArray, _index); + } + + private VariantValue ReadResidual() + { + if (!HasResidual) + { + throw new InvalidOperationException("No residual value to read."); + } + BinaryArray binary = (BinaryArray)_valueArray; + ReadOnlySpan bytes = binary.GetBytes(_index, out _); + return new VariantReader(_metadata, bytes).ToVariantValue(); + } + + // --------------------------------------------------------------- + // Typed-column accessors — zero-copy access to the shredded value + // without materializing a VariantValue. + // + // Each getter requires: + // (a) the slot's schema to match the requested type, and + // (b) the typed_value column to be populated at this index. + // Otherwise it throws. Callers that want automatic residual fallback + // should use ToVariantValue instead. + // --------------------------------------------------------------- + + /// Reads the shredded boolean value at this slot. + public bool GetBoolean() => ((BooleanArray)RequireTyped(ShredType.Boolean)).GetValue(_index).Value; + + /// Reads the shredded 8-bit signed integer at this slot. + public sbyte GetInt8() => ((Int8Array)RequireTyped(ShredType.Int8)).GetValue(_index).Value; + + /// Reads the shredded 16-bit signed integer at this slot. + public short GetInt16() => ((Int16Array)RequireTyped(ShredType.Int16)).GetValue(_index).Value; + + /// Reads the shredded 32-bit signed integer at this slot. + public int GetInt32() => ((Int32Array)RequireTyped(ShredType.Int32)).GetValue(_index).Value; + + /// Reads the shredded 64-bit signed integer at this slot. + public long GetInt64() => ((Int64Array)RequireTyped(ShredType.Int64)).GetValue(_index).Value; + + /// Reads the shredded 32-bit float at this slot. + public float GetFloat() => ((FloatArray)RequireTyped(ShredType.Float)).GetValue(_index).Value; + + /// Reads the shredded 64-bit double at this slot. + public double GetDouble() => ((DoubleArray)RequireTyped(ShredType.Double)).GetValue(_index).Value; + + /// + /// Reads the shredded decimal value at this slot. Works for Decimal4, Decimal8, + /// and Decimal16 shred types, regardless of whether the Arrow column is backed + /// by Decimal32Array, Decimal64Array, or Decimal128Array. + /// + public decimal GetDecimal() + { + RequireDecimalSchema(); + if (!HasTypedValue) ThrowNoTyped(); + IArrowArray arr = UnwrapExtension(_typedValueArray); + switch (arr) + { + case Decimal32Array d32: return d32.GetValue(_index).Value; + case Decimal64Array d64: return d64.GetValue(_index).Value; + case Decimal128Array d128: return d128.GetValue(_index).Value; + default: + throw new InvalidOperationException( + $"Shredded decimal column is backed by {arr.GetType().Name}, which is not a supported decimal array type."); + } + } + + /// + /// Reads the shredded decimal value at this slot. Works for Decimal4, Decimal8, + /// and Decimal16 shred types, regardless of whether the Arrow column is backed + /// by Decimal32Array, Decimal64Array, or Decimal128Array. + /// + public SqlDecimal GetSqlDecimal() + { + RequireDecimalSchema(); + if (!HasTypedValue) ThrowNoTyped(); + IArrowArray arr = UnwrapExtension(_typedValueArray); + switch (arr) + { + case Decimal32Array d32: return d32.GetValue(_index).Value; + case Decimal64Array d64: return d64.GetValue(_index).Value; + case Decimal128Array d128: return d128.GetSqlDecimal(_index).Value; + default: + throw new InvalidOperationException( + $"Shredded decimal column is backed by {arr.GetType().Name}, which is not a supported decimal array type."); + } + } + + /// Reads the shredded date (days since epoch) at this slot. + public int GetDateDays() => ((Date32Array)RequireTyped(ShredType.Date)).GetValue(_index).Value; + + /// Reads the shredded timestamp (microseconds since epoch, UTC) at this slot. + public long GetTimestampMicros() => ((TimestampArray)RequireTyped(ShredType.Timestamp)).GetValue(_index).Value; + + /// Reads the shredded timestamp-without-tz (microseconds since epoch) at this slot. + public long GetTimestampNtzMicros() => ((TimestampArray)RequireTyped(ShredType.TimestampNtz)).GetValue(_index).Value; + + /// Reads the shredded time-without-tz (microseconds since midnight) at this slot. + public long GetTimeNtzMicros() => ((Time64Array)RequireTyped(ShredType.TimeNtz)).GetValue(_index).Value; + + /// Reads the shredded timestamp-with-tz (nanoseconds since epoch) at this slot. + public long GetTimestampTzNanos() => ((TimestampArray)RequireTyped(ShredType.TimestampTzNanos)).GetValue(_index).Value; + + /// Reads the shredded timestamp-without-tz (nanoseconds since epoch) at this slot. + public long GetTimestampNtzNanos() => ((TimestampArray)RequireTyped(ShredType.TimestampNtzNanos)).GetValue(_index).Value; + + /// Reads the shredded string value at this slot. + public string GetString() => ((StringArray)RequireTyped(ShredType.String)).GetString(_index); + + /// Reads the shredded binary value at this slot as a byte span. + public ReadOnlySpan GetBinaryBytes() => ((BinaryArray)RequireTyped(ShredType.Binary)).GetBytes(_index); + + /// Reads the shredded UUID at this slot. + public Guid GetUuid() + { + FixedSizeBinaryArray arr = (FixedSizeBinaryArray)RequireTyped(ShredType.Uuid); + ReadOnlySpan raw = arr.GetBytes(_index); +#if NET8_0_OR_GREATER + return new Guid(raw, bigEndian: true); +#else + byte[] bytes = new byte[16]; + bytes[0] = raw[3]; bytes[1] = raw[2]; bytes[2] = raw[1]; bytes[3] = raw[0]; + bytes[4] = raw[5]; bytes[5] = raw[4]; + bytes[6] = raw[7]; bytes[7] = raw[6]; + raw.Slice(8, 8).CopyTo(bytes.AsSpan(8)); + return new Guid(bytes); +#endif + } + + /// Reads the shredded UUID at this slot as raw big-endian (RFC 4122) bytes. + public ReadOnlySpan GetUuidBytes() + => ((FixedSizeBinaryArray)RequireTyped(ShredType.Uuid)).GetBytes(_index); + + // --------------------------------------------------------------- + // Primitive dispatch for internal materialization. Delegates to the + // typed getters so the two paths stay in sync. + // --------------------------------------------------------------- + + private VariantValue ReadTypedPrimitive() + { + switch (_schema.TypedValueType) + { + case ShredType.Boolean: return VariantValue.FromBoolean(GetBoolean()); + case ShredType.Int8: return VariantValue.FromInt8(GetInt8()); + case ShredType.Int16: return VariantValue.FromInt16(GetInt16()); + case ShredType.Int32: return VariantValue.FromInt32(GetInt32()); + case ShredType.Int64: return VariantValue.FromInt64(GetInt64()); + case ShredType.Float: return VariantValue.FromFloat(GetFloat()); + case ShredType.Double: return VariantValue.FromDouble(GetDouble()); + case ShredType.Decimal4: return VariantValue.FromDecimal4(GetDecimal()); + case ShredType.Decimal8: return VariantValue.FromDecimal8(GetDecimal()); + case ShredType.Decimal16: return VariantValue.FromSqlDecimal(GetSqlDecimal()); + case ShredType.Date: return VariantValue.FromDate(GetDateDays()); + case ShredType.Timestamp: return VariantValue.FromTimestamp(GetTimestampMicros()); + case ShredType.TimestampNtz: return VariantValue.FromTimestampNtz(GetTimestampNtzMicros()); + case ShredType.TimeNtz: return VariantValue.FromTimeNtz(GetTimeNtzMicros()); + case ShredType.TimestampTzNanos: return VariantValue.FromTimestampTzNanos(GetTimestampTzNanos()); + case ShredType.TimestampNtzNanos: return VariantValue.FromTimestampNtzNanos(GetTimestampNtzNanos()); + case ShredType.String: return VariantValue.FromString(GetString()); + case ShredType.Binary: return VariantValue.FromBinary(GetBinaryBytes().ToArray()); + case ShredType.Uuid: return VariantValue.FromUuid(GetUuid()); + default: + throw new InvalidOperationException( + $"Unexpected primitive shred type {_schema.TypedValueType}."); + } + } + + private IArrowArray RequireTyped(ShredType expected) + { + if (_schema.TypedValueType != expected) + { + throw new InvalidOperationException( + $"Slot schema is {_schema.TypedValueType}, not {expected}."); + } + if (!HasTypedValue) ThrowNoTyped(); + return UnwrapExtension(_typedValueArray); + } + + private void RequireDecimalSchema() + { + if (_schema.TypedValueType != ShredType.Decimal4 && + _schema.TypedValueType != ShredType.Decimal8 && + _schema.TypedValueType != ShredType.Decimal16) + { + throw new InvalidOperationException( + $"Slot schema is {_schema.TypedValueType}, not a decimal type."); + } + } + + private void ThrowNoTyped() => + throw new InvalidOperationException( + "No typed_value at this index (check HasTypedValue first, or use ToVariantValue for residual fallback)."); + + private static IArrowArray UnwrapExtension(IArrowArray arr) => + arr is ExtensionArray ext ? ext.Storage : arr; + } +} diff --git a/src/Apache.Arrow.Operations/Shredding/ShreddedVariantArrayBuilder.cs b/src/Apache.Arrow.Operations/Shredding/ShreddedVariantArrayBuilder.cs new file mode 100644 index 00000000..b30344b8 --- /dev/null +++ b/src/Apache.Arrow.Operations/Shredding/ShreddedVariantArrayBuilder.cs @@ -0,0 +1,513 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System; +using System.Collections.Generic; +using Apache.Arrow; +using Apache.Arrow.Arrays; +using Apache.Arrow.Memory; +using Apache.Arrow.Scalars.Variant; +using Apache.Arrow.Types; + +namespace Apache.Arrow.Operations.Shredding +{ + /// + /// Assembles a shredded from pre-shredded rows. + /// Produces an Arrow struct with shared metadata, residual value, + /// and the typed_value tree whose Arrow shape matches the . + /// + public static class ShreddedVariantArrayBuilder + { + /// + /// Builds a shredded from the output of + /// . + /// + /// The shredding schema applied to each row. + /// The column-level variant metadata (shared across rows). + /// Per-row shred results whose residual bytes reference . + /// Arrow memory allocator, or default if null. + public static VariantArray Build( + ShredSchema schema, + byte[] metadata, + IReadOnlyList rows, + MemoryAllocator allocator = null) + { + if (schema == null) throw new ArgumentNullException(nameof(schema)); + if (metadata == null) throw new ArgumentNullException(nameof(metadata)); + if (rows == null) throw new ArgumentNullException(nameof(rows)); + + int rowCount = rows.Count; + + // metadata column: emit the shared bytes once per row. (A dictionary-encoded + // or run-end-encoded representation would compress this; VariantArray's reader + // already handles those, but for simplicity we emit the plain binary form.) + BinaryArray.Builder metadataBuilder = new BinaryArray.Builder(); + for (int i = 0; i < rowCount; i++) + { + metadataBuilder.Append((ReadOnlySpan)metadata); + } + BinaryArray metadataArr = metadataBuilder.Build(allocator); + + // value column: residual bytes (or null). + BinaryArray valueArr = BuildBinaryColumn(rows, allocator); + + // typed_value column (if the schema has one). + List fields = new List + { + new Field("metadata", BinaryType.Default, false), + new Field("value", BinaryType.Default, true), + }; + List children = new List { metadataArr, valueArr }; + + if (schema.TypedValueType != ShredType.None) + { + List typedValues = new List(rowCount); + for (int i = 0; i < rowCount; i++) typedValues.Add(rows[i].TypedValue); + IArrowArray typedValueArr = BuildTypedValueArray(schema, typedValues, allocator); + fields.Add(new Field("typed_value", typedValueArr.Data.DataType, true)); + children.Add(typedValueArr); + } + + StructType structType = new StructType(fields); + StructArray structArr = new StructArray( + structType, rowCount, children, ArrowBuffer.Empty, nullCount: 0); + // The public VariantArray(IArrowArray) constructor infers the VariantType + // from the struct's shape (including detecting the shredded layout). + return new VariantArray(structArr); + } + + // --------------------------------------------------------------- + // Recursive builders + // --------------------------------------------------------------- + + private static BinaryArray BuildBinaryColumn(IReadOnlyList rows, MemoryAllocator allocator) + { + BinaryArray.Builder b = new BinaryArray.Builder(); + foreach (ShredResult r in rows) + { + if (r.Value == null) b.AppendNull(); + else b.Append((ReadOnlySpan)r.Value); + } + return b.Build(allocator); + } + + private static IArrowArray BuildTypedValueArray( + ShredSchema schema, + IList typedValues, + MemoryAllocator allocator) + { + switch (schema.TypedValueType) + { + case ShredType.Object: return BuildObjectTyped(schema, typedValues, allocator); + case ShredType.Array: return BuildArrayTyped(schema, typedValues, allocator); + default: return BuildPrimitiveTyped(schema.TypedValueType, typedValues, allocator); + } + } + + private static StructArray BuildObjectTyped( + ShredSchema schema, + IList typedValues, + MemoryAllocator allocator) + { + int rowCount = typedValues.Count; + List fieldDefs = new List(schema.ObjectFields.Count); + List fieldArrays = new List(schema.ObjectFields.Count); + + foreach (KeyValuePair entry in schema.ObjectFields) + { + List fieldShreds = new List(rowCount); + foreach (object tv in typedValues) + { + if (tv is ShredObjectResult obj && + obj.Fields.TryGetValue(entry.Key, out ShredResult r)) + { + fieldShreds.Add(r); + } + else + { + fieldShreds.Add(ShredResult.Missing); + } + } + StructArray elementGroup = BuildElementGroupArray(entry.Value, fieldShreds, allocator); + fieldArrays.Add(elementGroup); + fieldDefs.Add(new Field(entry.Key, elementGroup.Data.DataType, true)); + } + + ArrowBuffer nullBitmap = BuildNullBitmap(typedValues, v => v != null, rowCount, + allocator, out int nullCount); + StructType structType = new StructType(fieldDefs); + return new StructArray(structType, rowCount, fieldArrays, nullBitmap, nullCount); + } + + private static ListArray BuildArrayTyped( + ShredSchema schema, + IList typedValues, + MemoryAllocator allocator) + { + int rowCount = typedValues.Count; + List flatElements = new List(); + ArrowBuffer.Builder offsets = new ArrowBuffer.Builder(); + offsets.Append(0); + ArrowBuffer.BitmapBuilder validity = new ArrowBuffer.BitmapBuilder(); + int nullCount = 0; + + foreach (object tv in typedValues) + { + if (tv is ShredArrayResult arr) + { + foreach (ShredResult e in arr.Elements) flatElements.Add(e); + validity.Append(true); + } + else + { + validity.Append(false); + nullCount++; + } + offsets.Append(flatElements.Count); + } + + StructArray elementGroup = BuildElementGroupArray(schema.ArrayElement, flatElements, allocator); + Field elementField = new Field("element", elementGroup.Data.DataType, true); + ListType listType = new ListType(elementField); + ArrowBuffer nullBitmap = nullCount > 0 ? validity.Build(allocator) : ArrowBuffer.Empty; + return new ListArray(listType, rowCount, offsets.Build(allocator), elementGroup, nullBitmap, nullCount); + } + + /// + /// Builds a {value?, typed_value?} element group. Always emits both + /// sub-fields (for simplicity) — readers tolerate the absent-field case via + /// null entries. + /// + private static StructArray BuildElementGroupArray( + ShredSchema schema, + IReadOnlyList results, + MemoryAllocator allocator) + { + int rowCount = results.Count; + List fieldDefs = new List(2); + List children = new List(2); + + // value column. + BinaryArray valueArr = BuildBinaryColumn(results, allocator); + fieldDefs.Add(new Field("value", BinaryType.Default, true)); + children.Add(valueArr); + + // typed_value column (only when schema has one). + if (schema.TypedValueType != ShredType.None) + { + List typedValues = new List(rowCount); + foreach (ShredResult r in results) typedValues.Add(r.TypedValue); + IArrowArray typedArr = BuildTypedValueArray(schema, typedValues, allocator); + fieldDefs.Add(new Field("typed_value", typedArr.Data.DataType, true)); + children.Add(typedArr); + } + + // Outer slot validity: non-null iff the slot isn't "missing" (both columns null). + ArrowBuffer.BitmapBuilder validity = new ArrowBuffer.BitmapBuilder(); + int nullCount = 0; + foreach (ShredResult r in results) + { + if (r.IsMissing) { validity.Append(false); nullCount++; } + else validity.Append(true); + } + ArrowBuffer nullBitmap = nullCount > 0 ? validity.Build(allocator) : ArrowBuffer.Empty; + + StructType structType = new StructType(fieldDefs); + return new StructArray(structType, rowCount, children, nullBitmap, nullCount); + } + + // --------------------------------------------------------------- + // Primitive typed-value builders + // --------------------------------------------------------------- + + private static IArrowArray BuildPrimitiveTyped( + ShredType shredType, + IList typedValues, + MemoryAllocator allocator) + { + switch (shredType) + { + case ShredType.Boolean: + { + BooleanArray.Builder b = new BooleanArray.Builder(); + foreach (object v in typedValues) + { + if (v == null) b.AppendNull(); else b.Append((bool)v); + } + return b.Build(allocator); + } + case ShredType.Int8: + { + Int8Array.Builder b = new Int8Array.Builder(); + foreach (object v in typedValues) + { + if (v == null) b.AppendNull(); else b.Append((sbyte)v); + } + return b.Build(allocator); + } + case ShredType.Int16: + { + Int16Array.Builder b = new Int16Array.Builder(); + foreach (object v in typedValues) + { + if (v == null) b.AppendNull(); else b.Append((short)v); + } + return b.Build(allocator); + } + case ShredType.Int32: + { + Int32Array.Builder b = new Int32Array.Builder(); + foreach (object v in typedValues) + { + if (v == null) b.AppendNull(); else b.Append((int)v); + } + return b.Build(allocator); + } + case ShredType.Int64: + { + Int64Array.Builder b = new Int64Array.Builder(); + foreach (object v in typedValues) + { + if (v == null) b.AppendNull(); else b.Append((long)v); + } + return b.Build(allocator); + } + case ShredType.Float: + { + FloatArray.Builder b = new FloatArray.Builder(); + foreach (object v in typedValues) + { + if (v == null) b.AppendNull(); else b.Append((float)v); + } + return b.Build(allocator); + } + case ShredType.Double: + { + DoubleArray.Builder b = new DoubleArray.Builder(); + foreach (object v in typedValues) + { + if (v == null) b.AppendNull(); else b.Append((double)v); + } + return b.Build(allocator); + } + case ShredType.Decimal4: + case ShredType.Decimal8: + case ShredType.Decimal16: + return BuildDecimalArray(shredType, typedValues, allocator); + case ShredType.Date: + return BuildDate32(typedValues, allocator); + case ShredType.Timestamp: + return BuildTimestamp(typedValues, TimeUnit.Microsecond, "UTC", allocator); + case ShredType.TimestampNtz: + return BuildTimestamp(typedValues, TimeUnit.Microsecond, null, allocator); + case ShredType.TimestampTzNanos: + return BuildTimestamp(typedValues, TimeUnit.Nanosecond, "UTC", allocator); + case ShredType.TimestampNtzNanos: + return BuildTimestamp(typedValues, TimeUnit.Nanosecond, null, allocator); + case ShredType.TimeNtz: + return BuildTime64(typedValues, allocator); + case ShredType.String: + { + StringArray.Builder b = new StringArray.Builder(); + foreach (object v in typedValues) + { + if (v == null) b.AppendNull(); else b.Append((string)v); + } + return b.Build(allocator); + } + case ShredType.Binary: + { + BinaryArray.Builder b = new BinaryArray.Builder(); + foreach (object v in typedValues) + { + if (v == null) b.AppendNull(); else b.Append((ReadOnlySpan)(byte[])v); + } + return b.Build(allocator); + } + case ShredType.Uuid: + return BuildUuidArray(typedValues, allocator); + default: + throw new NotSupportedException($"Cannot build typed column for ShredType.{shredType}."); + } + } + + private static TimestampArray BuildTimestamp( + IList typedValues, TimeUnit unit, string timezone, MemoryAllocator allocator) + { + TimestampType type = new TimestampType(unit, timezone); + (ArrowBuffer values, ArrowBuffer nullBitmap, int nullCount) = + BuildLongBuffers(typedValues, allocator); + return new TimestampArray(type, values, nullBitmap, typedValues.Count, nullCount, offset: 0); + } + + private static Date32Array BuildDate32(IList typedValues, MemoryAllocator allocator) + { + (ArrowBuffer values, ArrowBuffer nullBitmap, int nullCount) = + BuildIntBuffers(typedValues, allocator); + return new Date32Array(values, nullBitmap, typedValues.Count, nullCount, offset: 0); + } + + private static Time64Array BuildTime64(IList typedValues, MemoryAllocator allocator) + { + Time64Type type = new Time64Type(TimeUnit.Microsecond); + (ArrowBuffer values, ArrowBuffer nullBitmap, int nullCount) = + BuildLongBuffers(typedValues, allocator); + return new Time64Array(type, values, nullBitmap, typedValues.Count, nullCount, offset: 0); + } + + private static (ArrowBuffer values, ArrowBuffer nullBitmap, int nullCount) BuildLongBuffers( + IList typedValues, MemoryAllocator allocator) + { + ArrowBuffer.Builder values = new ArrowBuffer.Builder(typedValues.Count); + ArrowBuffer.BitmapBuilder bitmap = new ArrowBuffer.BitmapBuilder(typedValues.Count); + int nullCount = 0; + foreach (object v in typedValues) + { + if (v == null) + { + values.Append(0L); + bitmap.Append(false); + nullCount++; + } + else + { + values.Append((long)v); + bitmap.Append(true); + } + } + ArrowBuffer nullBitmap = nullCount > 0 ? bitmap.Build(allocator) : ArrowBuffer.Empty; + return (values.Build(allocator), nullBitmap, nullCount); + } + + private static (ArrowBuffer values, ArrowBuffer nullBitmap, int nullCount) BuildIntBuffers( + IList typedValues, MemoryAllocator allocator) + { + ArrowBuffer.Builder values = new ArrowBuffer.Builder(typedValues.Count); + ArrowBuffer.BitmapBuilder bitmap = new ArrowBuffer.BitmapBuilder(typedValues.Count); + int nullCount = 0; + foreach (object v in typedValues) + { + if (v == null) + { + values.Append(0); + bitmap.Append(false); + nullCount++; + } + else + { + values.Append((int)v); + bitmap.Append(true); + } + } + ArrowBuffer nullBitmap = nullCount > 0 ? bitmap.Build(allocator) : ArrowBuffer.Empty; + return (values.Build(allocator), nullBitmap, nullCount); + } + + private static Decimal128Array BuildDecimalArray( + ShredType shredType, IList typedValues, MemoryAllocator allocator) + { + int precision = shredType == ShredType.Decimal4 ? 9 + : shredType == ShredType.Decimal8 ? 18 + : 38; + // Scale: pick the max scale seen across all rows. Arrow's builder will rescale + // individual values to match; if rows have heterogeneous scales the larger one + // accommodates all values exactly (assuming precision is not exceeded). + int scale = 0; + foreach (object v in typedValues) + { + if (v is decimal d) + { + int s = (decimal.GetBits(d)[3] >> 16) & 0x7F; + if (s > scale) scale = s; + } + } + + Decimal128Array.Builder b = new Decimal128Array.Builder(new Decimal128Type(precision, scale)); + foreach (object v in typedValues) + { + if (v == null) b.AppendNull(); else b.Append((decimal)v); + } + return b.Build(allocator); + } + + /// + /// UUID is encoded as FixedSizeBinary(16) in big-endian (RFC 4122) byte order. + /// FixedSizeBinaryArray has no concrete public Builder, so we construct the + /// value buffer manually (16 bytes per row). + /// + private static FixedSizeBinaryArray BuildUuidArray( + IList typedValues, MemoryAllocator allocator) + { + FixedSizeBinaryType type = new FixedSizeBinaryType(16); + int rowCount = typedValues.Count; + ArrowBuffer.Builder values = new ArrowBuffer.Builder(rowCount * 16); + ArrowBuffer.BitmapBuilder bitmap = new ArrowBuffer.BitmapBuilder(rowCount); + int nullCount = 0; + byte[] scratch = new byte[16]; + + foreach (object v in typedValues) + { + if (v == null) + { + // Emit 16 zero bytes as a placeholder; the null bitmap marks it invalid. + for (int i = 0; i < 16; i++) values.Append((byte)0); + bitmap.Append(false); + nullCount++; + continue; + } + + Guid g = (Guid)v; +#if NET8_0_OR_GREATER + g.TryWriteBytes(scratch.AsSpan(), bigEndian: true, out _); +#else + byte[] native = g.ToByteArray(); + // Convert .NET mixed-endian to big-endian. + scratch[0] = native[3]; scratch[1] = native[2]; scratch[2] = native[1]; scratch[3] = native[0]; + scratch[4] = native[5]; scratch[5] = native[4]; + scratch[6] = native[7]; scratch[7] = native[6]; + Buffer.BlockCopy(native, 8, scratch, 8, 8); +#endif + for (int i = 0; i < 16; i++) values.Append(scratch[i]); + bitmap.Append(true); + } + + ArrowBuffer nullBitmap = nullCount > 0 ? bitmap.Build(allocator) : ArrowBuffer.Empty; + ArrayData data = new ArrayData( + type, rowCount, nullCount, 0, + new[] { nullBitmap, values.Build(allocator) }); + return new FixedSizeBinaryArray(data); + } + + // --------------------------------------------------------------- + // Utility + // --------------------------------------------------------------- + + private static ArrowBuffer BuildNullBitmap( + IList items, Func isValid, int rowCount, + MemoryAllocator allocator, out int nullCount) + { + ArrowBuffer.BitmapBuilder bitmap = new ArrowBuffer.BitmapBuilder(rowCount); + int nulls = 0; + for (int i = 0; i < rowCount; i++) + { + bool valid = isValid(items[i]); + bitmap.Append(valid); + if (!valid) nulls++; + } + nullCount = nulls; + return nulls > 0 ? bitmap.Build(allocator) : ArrowBuffer.Empty; + } + } +} diff --git a/src/Apache.Arrow.Operations/Shredding/ShreddingHelpers.cs b/src/Apache.Arrow.Operations/Shredding/ShreddingHelpers.cs new file mode 100644 index 00000000..f7be85e8 --- /dev/null +++ b/src/Apache.Arrow.Operations/Shredding/ShreddingHelpers.cs @@ -0,0 +1,48 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System; +using Apache.Arrow; +using Apache.Arrow.Types; + +namespace Apache.Arrow.Operations.Shredding +{ + /// + /// Internal helpers shared by the shredded-variant reader trio. + /// + internal static class ShreddingHelpers + { + /// + /// Builds a slot for the given index of an element-group + /// struct (one with value and/or typed_value sub-fields). Either sub-field + /// may be absent from the struct. + /// + public static ShreddedVariant BuildSlot( + ShredSchema slotSchema, + ReadOnlySpan metadata, + StructArray elementGroup, + int index) + { + StructType elementGroupType = (StructType)elementGroup.Data.DataType; + int valueIdx = elementGroupType.GetFieldIndex("value"); + int typedIdx = elementGroupType.GetFieldIndex("typed_value"); + + IArrowArray valueArr = valueIdx >= 0 ? elementGroup.Fields[valueIdx] : null; + IArrowArray typedArr = typedIdx >= 0 ? elementGroup.Fields[typedIdx] : null; + + return new ShreddedVariant(slotSchema, metadata, valueArr, typedArr, index); + } + } +} diff --git a/src/Apache.Arrow.Operations/Shredding/VariantArrayShreddingExtensions.cs b/src/Apache.Arrow.Operations/Shredding/VariantArrayShreddingExtensions.cs new file mode 100644 index 00000000..4e1fbfe1 --- /dev/null +++ b/src/Apache.Arrow.Operations/Shredding/VariantArrayShreddingExtensions.cs @@ -0,0 +1,93 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System; +using Apache.Arrow; +using Apache.Arrow.Scalars.Variant; + +namespace Apache.Arrow.Operations.Shredding +{ + /// + /// Shredding-aware extensions on . Provides both + /// transparent materialization () and a + /// reader-style API () that exposes typed + /// columns and residual bytes side-by-side. + /// + public static class VariantArrayShreddingExtensions + { + /// + /// Gets the for a variant array, derived from + /// its Arrow storage type. Returns + /// for unshredded columns. + /// + public static ShredSchema GetShredSchema(this VariantArray array) + { + if (array == null) throw new ArgumentNullException(nameof(array)); + return ShredSchema.FromArrowType(array.VariantType.TypedValueField?.DataType); + } + + /// + /// Gets a reader for the element at the given index. + /// Exposes typed-column access and residual bytes without materializing the + /// full logical variant. Works for both shredded and unshredded columns. + /// + /// If the element is null. + public static ShreddedVariant GetShreddedVariant(this VariantArray array, int index) + { + if (array == null) throw new ArgumentNullException(nameof(array)); + if (index < 0 || index >= array.Length) + throw new ArgumentOutOfRangeException(nameof(index)); + if (array.IsNull(index)) + throw new InvalidOperationException("Cannot create a ShreddedVariant for a null element."); + + ShredSchema schema = GetShredSchema(array); + ReadOnlySpan metadata = array.GetMetadataBytes(index); + IArrowArray valueArr = array.VariantType.HasValueColumn + ? GetValueArray(array) + : null; + IArrowArray typedValueArr = array.TypedValueArray; + + return new ShreddedVariant(schema, metadata, valueArr, typedValueArr, index); + } + + /// + /// Materializes the element at into a logical + /// , transparently merging shredded columns and + /// residual bytes. Works for both shredded and unshredded columns. + /// + public static VariantValue GetLogicalVariantValue(this VariantArray array, int index) + { + if (array == null) throw new ArgumentNullException(nameof(array)); + if (index < 0 || index >= array.Length) + throw new ArgumentOutOfRangeException(nameof(index)); + if (array.IsNull(index)) + return VariantValue.Null; + + return GetShreddedVariant(array, index).ToVariantValue(); + } + + /// + /// Returns the underlying value sub-array of the VariantArray's struct storage. + /// This mirrors what uses internally. + /// + private static IArrowArray GetValueArray(VariantArray array) + { + StructArray storage = array.StorageArray; + var structType = (Apache.Arrow.Types.StructType)storage.Data.DataType; + int idx = structType.GetFieldIndex("value"); + return storage.Fields[idx]; + } + } +} diff --git a/src/Apache.Arrow.Operations/Shredding/VariantShredder.cs b/src/Apache.Arrow.Operations/Shredding/VariantShredder.cs new file mode 100644 index 00000000..9c4f7fab --- /dev/null +++ b/src/Apache.Arrow.Operations/Shredding/VariantShredder.cs @@ -0,0 +1,312 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System; +using System.Collections.Generic; +using Apache.Arrow.Scalars.Variant; + +namespace Apache.Arrow.Operations.Shredding +{ + /// + /// Decomposes s into shredded (value, typed_value) pairs + /// according to a . + /// + /// Per the Parquet variant shredding spec, the variant metadata dictionary is shared + /// across an entire column. The + /// batch entrypoint builds that shared metadata and emits per-row value bytes that + /// reference it — ready to drop into a Parquet value column. + /// + /// + public static class VariantShredder + { + /// + /// Shreds a column of variant values into a shared metadata dictionary and + /// per-row s. The residual + /// bytes for each row reference the returned metadata. + /// + public static (byte[] Metadata, IReadOnlyList Rows) Shred( + IEnumerable values, + ShredSchema schema) + { + if (values == null) throw new ArgumentNullException(nameof(values)); + if (schema == null) throw new ArgumentNullException(nameof(schema)); + + // Materialize so we can make two passes (metadata collection, then shredding). + List rows = values as List ?? new List(values); + + // Pass 1: collect every field name into the shared metadata dictionary. + // A superset is fine per-spec; the Parquet value column just needs to resolve + // any field ID it references. + VariantMetadataBuilder metadata = new VariantMetadataBuilder(); + foreach (VariantValue row in rows) + { + CollectFieldNames(row, metadata); + } + byte[] metadataBytes = metadata.Build(out int[] idRemap); + + // Pass 2: shred each row against the finalized metadata. + ShredResult[] results = new ShredResult[rows.Count]; + for (int i = 0; i < rows.Count; i++) + { + results[i] = Shred(rows[i], schema, metadata, idRemap); + } + + return (metadataBytes, results); + } + + /// + /// Shreds a single variant value against a caller-managed metadata dictionary. + /// Use this when combining shredded columns with external metadata, or when + /// streaming rows one at a time. The caller is responsible for ensuring + /// already contains every field name the residual + /// may reference. + /// + public static ShredResult Shred( + VariantValue value, + ShredSchema schema, + VariantMetadataBuilder metadata, + int[] idRemap) + { + if (schema == null) throw new ArgumentNullException(nameof(schema)); + if (metadata == null) throw new ArgumentNullException(nameof(metadata)); + if (idRemap == null) throw new ArgumentNullException(nameof(idRemap)); + + switch (schema.TypedValueType) + { + case ShredType.None: + return ShredAsUntyped(value, metadata, idRemap); + case ShredType.Object: + return ShredAsObject(value, schema, metadata, idRemap); + case ShredType.Array: + return ShredAsArray(value, schema, metadata, idRemap); + default: + return ShredAsPrimitive(value, schema, metadata, idRemap); + } + } + + private static ShredResult ShredAsUntyped(VariantValue value, VariantMetadataBuilder metadata, int[] idRemap) + { + return new ShredResult(EncodeValue(value, metadata, idRemap), null); + } + + private static ShredResult ShredAsPrimitive(VariantValue value, ShredSchema schema, VariantMetadataBuilder metadata, int[] idRemap) + { + ShredType actualType = GetShredType(value); + if (actualType == schema.TypedValueType) + { + return new ShredResult(null, ExtractTypedValue(value, schema.TypedValueType)); + } + return new ShredResult(EncodeValue(value, metadata, idRemap), null); + } + + private static ShredResult ShredAsObject(VariantValue value, ShredSchema schema, VariantMetadataBuilder metadata, int[] idRemap) + { + if (!value.IsObject) + { + return new ShredResult(EncodeValue(value, metadata, idRemap), null); + } + + IReadOnlyDictionary fields = value.AsObject(); + Dictionary shreddedFields = new Dictionary(schema.ObjectFields.Count); + List> residualFields = null; + + foreach (KeyValuePair schemaField in schema.ObjectFields) + { + if (fields.TryGetValue(schemaField.Key, out VariantValue fieldValue)) + { + shreddedFields[schemaField.Key] = Shred(fieldValue, schemaField.Value, metadata, idRemap); + } + else + { + shreddedFields[schemaField.Key] = ShredResult.Missing; + } + } + + foreach (KeyValuePair field in fields) + { + if (!schema.ObjectFields.ContainsKey(field.Key)) + { + if (residualFields == null) + { + residualFields = new List>(); + } + residualFields.Add(field); + } + } + + ShredObjectResult typedValue = new ShredObjectResult(shreddedFields); + if (residualFields != null) + { + return new ShredResult(EncodeResidualObject(residualFields, metadata, idRemap), typedValue); + } + return new ShredResult(null, typedValue); + } + + private static ShredResult ShredAsArray(VariantValue value, ShredSchema schema, VariantMetadataBuilder metadata, int[] idRemap) + { + if (!value.IsArray) + { + return new ShredResult(EncodeValue(value, metadata, idRemap), null); + } + + IReadOnlyList elements = value.AsArray(); + List shreddedElements = new List(elements.Count); + for (int i = 0; i < elements.Count; i++) + { + shreddedElements.Add(Shred(elements[i], schema.ArrayElement, metadata, idRemap)); + } + return new ShredResult(null, new ShredArrayResult(shreddedElements)); + } + + // --------------------------------------------------------------- + // Encoding helpers — write value bytes referencing the shared metadata + // --------------------------------------------------------------- + + private static byte[] EncodeValue(VariantValue value, VariantMetadataBuilder metadata, int[] idRemap) + { + VariantValueWriter writer = new VariantValueWriter(metadata, idRemap); + WriteVariantValue(writer, value); + return writer.ToArray(); + } + + private static byte[] EncodeResidualObject(List> fields, VariantMetadataBuilder metadata, int[] idRemap) + { + VariantValueWriter writer = new VariantValueWriter(metadata, idRemap); + writer.BeginObject(); + foreach (KeyValuePair field in fields) + { + writer.WriteFieldName(field.Key); + WriteVariantValue(writer, field.Value); + } + writer.EndObject(); + return writer.ToArray(); + } + + private static void WriteVariantValue(VariantValueWriter writer, VariantValue variant) + { + if (variant.IsNull) { writer.WriteNull(); return; } + if (variant.IsBoolean) { writer.WriteBoolean(variant.AsBoolean()); return; } + if (variant.IsObject) + { + writer.BeginObject(); + foreach (KeyValuePair field in variant.AsObject()) + { + writer.WriteFieldName(field.Key); + WriteVariantValue(writer, field.Value); + } + writer.EndObject(); + return; + } + if (variant.IsArray) + { + writer.BeginArray(); + foreach (VariantValue element in variant.AsArray()) + { + WriteVariantValue(writer, element); + } + writer.EndArray(); + return; + } + + switch (variant.PrimitiveType) + { + case VariantPrimitiveType.Int8: writer.WriteInt8(variant.AsInt8()); break; + case VariantPrimitiveType.Int16: writer.WriteInt16(variant.AsInt16()); break; + case VariantPrimitiveType.Int32: writer.WriteInt32(variant.AsInt32()); break; + case VariantPrimitiveType.Int64: writer.WriteInt64(variant.AsInt64()); break; + case VariantPrimitiveType.Float: writer.WriteFloat(variant.AsFloat()); break; + case VariantPrimitiveType.Double: writer.WriteDouble(variant.AsDouble()); break; + case VariantPrimitiveType.Decimal4: writer.WriteDecimal4(variant.AsDecimal()); break; + case VariantPrimitiveType.Decimal8: writer.WriteDecimal8(variant.AsDecimal()); break; + case VariantPrimitiveType.Decimal16: writer.WriteDecimal16(variant.AsSqlDecimal()); break; + case VariantPrimitiveType.Date: writer.WriteDateDays(variant.AsDateDays()); break; + case VariantPrimitiveType.Timestamp: writer.WriteTimestampMicros(variant.AsTimestampMicros()); break; + case VariantPrimitiveType.TimestampNtz: writer.WriteTimestampNtzMicros(variant.AsTimestampNtzMicros()); break; + case VariantPrimitiveType.TimeNtz: writer.WriteTimeNtzMicros(variant.AsTimeNtzMicros()); break; + case VariantPrimitiveType.TimestampTzNanos: writer.WriteTimestampTzNanos(variant.AsTimestampTzNanos()); break; + case VariantPrimitiveType.TimestampNtzNanos: writer.WriteTimestampNtzNanos(variant.AsTimestampNtzNanos()); break; + case VariantPrimitiveType.String: writer.WriteString(variant.AsString()); break; + case VariantPrimitiveType.Binary: writer.WriteBinary(variant.AsBinary()); break; + case VariantPrimitiveType.Uuid: writer.WriteUuid(variant.AsUuid()); break; + default: throw new NotSupportedException($"Unsupported primitive type: {variant.PrimitiveType}"); + } + } + + private static void CollectFieldNames(VariantValue variant, VariantMetadataBuilder builder) + { + if (variant.IsObject) + { + foreach (KeyValuePair field in variant.AsObject()) + { + builder.Add(field.Key); + CollectFieldNames(field.Value, builder); + } + } + else if (variant.IsArray) + { + foreach (VariantValue element in variant.AsArray()) + { + CollectFieldNames(element, builder); + } + } + } + + // --------------------------------------------------------------- + // Type extraction + // --------------------------------------------------------------- + + /// + /// Extracts the native CLR value from a for + /// storage in a typed Parquet column. + /// + internal static object ExtractTypedValue(VariantValue value, ShredType shredType) + { + switch (shredType) + { + case ShredType.Boolean: return value.AsBoolean(); + case ShredType.Int8: return value.AsInt8(); + case ShredType.Int16: return value.AsInt16(); + case ShredType.Int32: return value.AsInt32(); + case ShredType.Int64: return value.AsInt64(); + case ShredType.Float: return value.AsFloat(); + case ShredType.Double: return value.AsDouble(); + case ShredType.Decimal4: + case ShredType.Decimal8: + case ShredType.Decimal16: return value.AsDecimal(); + case ShredType.Date: return value.AsDateDays(); + case ShredType.Timestamp: return value.AsTimestampMicros(); + case ShredType.TimestampNtz: return value.AsTimestampNtzMicros(); + case ShredType.TimeNtz: return value.AsTimeNtzMicros(); + case ShredType.TimestampTzNanos: return value.AsTimestampTzNanos(); + case ShredType.TimestampNtzNanos: return value.AsTimestampNtzNanos(); + case ShredType.String: return value.AsString(); + case ShredType.Binary: return value.AsBinary(); + case ShredType.Uuid: return value.AsUuid(); + default: + throw new InvalidOperationException($"Cannot extract typed value for ShredType.{shredType}."); + } + } + + /// + /// Determines the of a . + /// + internal static ShredType GetShredType(VariantValue value) + { + if (value.IsObject) return ShredType.Object; + if (value.IsArray) return ShredType.Array; + return ShredSchema.ShredTypeFromPrimitive(value.PrimitiveType); + } + } +} diff --git a/src/Apache.Arrow.Operations/Shredding/VariantUnshredder.cs b/src/Apache.Arrow.Operations/Shredding/VariantUnshredder.cs new file mode 100644 index 00000000..346e57d8 --- /dev/null +++ b/src/Apache.Arrow.Operations/Shredding/VariantUnshredder.cs @@ -0,0 +1,178 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System; +using System.Collections.Generic; +using Apache.Arrow.Scalars.Variant; + +namespace Apache.Arrow.Operations.Shredding +{ + /// + /// Reconstructs s from shredded (value, typed_value) pairs + /// according to a . The residual + /// bytes are interpreted against the column-level variant metadata supplied to + /// . + /// + public static class VariantUnshredder + { + /// + /// Reconstructs a variant value from a shredded result. + /// + /// The shredded (value, typed_value) pair. + /// The shredding schema that was used to produce the result. + /// The column-level variant metadata bytes. + /// + /// The reconstructed , or null if the field is missing + /// (both value and typed_value are null). + /// + public static VariantValue? Reconstruct(ShredResult shredded, ShredSchema schema, ReadOnlySpan metadata) + { + if (shredded == null) throw new ArgumentNullException(nameof(shredded)); + if (schema == null) throw new ArgumentNullException(nameof(schema)); + + if (shredded.IsMissing) + { + return null; + } + + switch (schema.TypedValueType) + { + case ShredType.Object: + return ReconstructObject(shredded, schema, metadata); + case ShredType.Array: + return ReconstructArray(shredded, schema, metadata); + case ShredType.None: + return DecodeValue(metadata, shredded.Value); + default: + return ReconstructPrimitive(shredded, schema, metadata); + } + } + + private static VariantValue ReconstructPrimitive(ShredResult shredded, ShredSchema schema, ReadOnlySpan metadata) + { + if (shredded.TypedValue != null) + { + return CreateVariantFromTyped(shredded.TypedValue, schema.TypedValueType); + } + return DecodeValue(metadata, shredded.Value); + } + + private static VariantValue ReconstructObject(ShredResult shredded, ShredSchema schema, ReadOnlySpan metadata) + { + if (shredded.TypedValue == null) + { + // Source value wasn't an object — the whole thing is in the residual. + return DecodeValue(metadata, shredded.Value); + } + + ShredObjectResult objectResult = (ShredObjectResult)shredded.TypedValue; + Dictionary fields = new Dictionary(); + + foreach (KeyValuePair fieldEntry in objectResult.Fields) + { + if (!schema.ObjectFields.TryGetValue(fieldEntry.Key, out ShredSchema fieldSchema)) + { + throw new InvalidOperationException( + $"Shredded object contains field '{fieldEntry.Key}' not in schema."); + } + + VariantValue? fieldValue = Reconstruct(fieldEntry.Value, fieldSchema, metadata); + if (fieldValue.HasValue) + { + fields[fieldEntry.Key] = fieldValue.Value; + } + // If null (missing), the field is omitted from the result. + } + + if (shredded.Value != null) + { + VariantValue residual = DecodeValue(metadata, shredded.Value); + if (!residual.IsObject) + { + throw new InvalidOperationException( + "Residual value for a partially shredded object must be an object."); + } + foreach (KeyValuePair residualField in residual.AsObject()) + { + fields[residualField.Key] = residualField.Value; + } + } + + return VariantValue.FromObject(fields); + } + + private static VariantValue ReconstructArray(ShredResult shredded, ShredSchema schema, ReadOnlySpan metadata) + { + if (shredded.TypedValue == null) + { + return DecodeValue(metadata, shredded.Value); + } + + ShredArrayResult arrayResult = (ShredArrayResult)shredded.TypedValue; + List elements = new List(arrayResult.Elements.Count); + + for (int i = 0; i < arrayResult.Elements.Count; i++) + { + VariantValue? elementValue = Reconstruct(arrayResult.Elements[i], schema.ArrayElement, metadata); + if (!elementValue.HasValue) + { + throw new InvalidOperationException( + $"Array element at index {i} is missing, but array elements cannot be missing."); + } + elements.Add(elementValue.Value); + } + + return VariantValue.FromArray(elements); + } + + private static VariantValue DecodeValue(ReadOnlySpan metadata, byte[] value) + { + if (value == null) throw new ArgumentNullException(nameof(value)); + VariantReader reader = new VariantReader(metadata, value); + return reader.ToVariantValue(); + } + + /// + /// Creates a from a typed CLR value and its . + /// + internal static VariantValue CreateVariantFromTyped(object typedValue, ShredType shredType) + { + switch (shredType) + { + case ShredType.Boolean: return VariantValue.FromBoolean((bool)typedValue); + case ShredType.Int8: return VariantValue.FromInt8((sbyte)typedValue); + case ShredType.Int16: return VariantValue.FromInt16((short)typedValue); + case ShredType.Int32: return VariantValue.FromInt32((int)typedValue); + case ShredType.Int64: return VariantValue.FromInt64((long)typedValue); + case ShredType.Float: return VariantValue.FromFloat((float)typedValue); + case ShredType.Double: return VariantValue.FromDouble((double)typedValue); + case ShredType.Decimal4: return VariantValue.FromDecimal4((decimal)typedValue); + case ShredType.Decimal8: return VariantValue.FromDecimal8((decimal)typedValue); + case ShredType.Decimal16: return VariantValue.FromDecimal16((decimal)typedValue); + case ShredType.Date: return VariantValue.FromDate((int)typedValue); + case ShredType.Timestamp: return VariantValue.FromTimestamp((long)typedValue); + case ShredType.TimestampNtz: return VariantValue.FromTimestampNtz((long)typedValue); + case ShredType.TimeNtz: return VariantValue.FromTimeNtz((long)typedValue); + case ShredType.TimestampTzNanos: return VariantValue.FromTimestampTzNanos((long)typedValue); + case ShredType.TimestampNtzNanos: return VariantValue.FromTimestampNtzNanos((long)typedValue); + case ShredType.String: return VariantValue.FromString((string)typedValue); + case ShredType.Binary: return VariantValue.FromBinary((byte[])typedValue); + case ShredType.Uuid: return VariantValue.FromUuid((Guid)typedValue); + default: + throw new InvalidOperationException($"Cannot create VariantValue for ShredType.{shredType}."); + } + } + } +} diff --git a/src/Apache.Arrow.Scalars/Variant/VariantMetadataBuilder.cs b/src/Apache.Arrow.Scalars/Variant/VariantMetadataBuilder.cs index 9e06feaf..5c9fa77d 100644 --- a/src/Apache.Arrow.Scalars/Variant/VariantMetadataBuilder.cs +++ b/src/Apache.Arrow.Scalars/Variant/VariantMetadataBuilder.cs @@ -57,6 +57,38 @@ public int Add(string name) /// public int Count => _names.Count; + /// + /// Recursively walks and adds every field name it + /// references to this builder. Use during the metadata-collection phase of + /// a two-pass encode — build the metadata first, finalize it via + /// , then pass the resulting remap to a + /// and call . + /// + public void CollectFieldNames(VariantReader source) + { + switch (source.BasicType) + { + case VariantBasicType.Object: + VariantObjectReader obj = new VariantObjectReader(source.Metadata, source.Value); + for (int i = 0; i < obj.FieldCount; i++) + { + Add(obj.GetFieldName(i)); + CollectFieldNames(obj.GetFieldValue(i)); + } + return; + + case VariantBasicType.Array: + VariantArrayReader arr = new VariantArrayReader(source.Metadata, source.Value); + for (int i = 0; i < arr.ElementCount; i++) + { + CollectFieldNames(arr.GetElement(i)); + } + return; + + // Primitive values and short strings have no field-name references. + } + } + /// /// Builds the binary metadata with the dictionary sorted by UTF-8 byte order. /// diff --git a/src/Apache.Arrow.Scalars/Variant/VariantValueWriter.cs b/src/Apache.Arrow.Scalars/Variant/VariantValueWriter.cs index dc54aa50..0fada099 100644 --- a/src/Apache.Arrow.Scalars/Variant/VariantValueWriter.cs +++ b/src/Apache.Arrow.Scalars/Variant/VariantValueWriter.cs @@ -424,6 +424,92 @@ public void WriteTimestampNtzNanos(long nanos) WriteInt64LE(ms, nanos); } + // --------------------------------------------------------------- + // Transcode from a VariantReader + // --------------------------------------------------------------- + + /// + /// Copies the variant value pointed to by into this + /// writer. Useful when copying between metadata dictionaries: field IDs in the + /// source are re-looked-up against this writer's + /// on the fly, via . + /// + /// + /// All field names referenced anywhere in must already + /// exist in the metadata builder used to construct this writer. Use + /// during + /// the metadata-collection phase of a two-pass encode to accumulate them. + /// + public void CopyValue(VariantReader source) + { + switch (source.BasicType) + { + case VariantBasicType.Primitive: + CopyPrimitive(source); + return; + + case VariantBasicType.ShortString: + WriteString(source.GetString()); + return; + + case VariantBasicType.Object: + VariantObjectReader obj = new VariantObjectReader(source.Metadata, source.Value); + BeginObject(); + for (int i = 0; i < obj.FieldCount; i++) + { + WriteFieldName(obj.GetFieldName(i)); + CopyValue(obj.GetFieldValue(i)); + } + EndObject(); + return; + + case VariantBasicType.Array: + VariantArrayReader arr = new VariantArrayReader(source.Metadata, source.Value); + BeginArray(); + for (int i = 0; i < arr.ElementCount; i++) + { + CopyValue(arr.GetElement(i)); + } + EndArray(); + return; + + default: + throw new NotSupportedException($"Unsupported basic type: {source.BasicType}"); + } + } + + private void CopyPrimitive(VariantReader source) + { + VariantPrimitiveType? pt = source.PrimitiveType; + switch (pt) + { + case VariantPrimitiveType.NullType: WriteNull(); return; + case VariantPrimitiveType.BooleanTrue: WriteBoolean(true); return; + case VariantPrimitiveType.BooleanFalse: WriteBoolean(false); return; + case VariantPrimitiveType.Int8: WriteInt8(source.GetInt8()); return; + case VariantPrimitiveType.Int16: WriteInt16(source.GetInt16()); return; + case VariantPrimitiveType.Int32: WriteInt32(source.GetInt32()); return; + case VariantPrimitiveType.Int64: WriteInt64(source.GetInt64()); return; + case VariantPrimitiveType.Float: WriteFloat(source.GetFloat()); return; + case VariantPrimitiveType.Double: WriteDouble(source.GetDouble()); return; + case VariantPrimitiveType.Decimal4: WriteDecimal4(source.GetDecimal4()); return; + case VariantPrimitiveType.Decimal8: WriteDecimal8(source.GetDecimal8()); return; + // Decimal16 may exceed System.Decimal's range, so route through SqlDecimal. + case VariantPrimitiveType.Decimal16: WriteDecimal16(source.GetSqlDecimal()); return; + case VariantPrimitiveType.Date: WriteDateDays(source.GetDateDays()); return; + case VariantPrimitiveType.Timestamp: WriteTimestampMicros(source.GetTimestampMicros()); return; + case VariantPrimitiveType.TimestampNtz: WriteTimestampNtzMicros(source.GetTimestampNtzMicros()); return; + case VariantPrimitiveType.TimeNtz: WriteTimeNtzMicros(source.GetTimeNtzMicros()); return; + case VariantPrimitiveType.TimestampTzNanos: WriteTimestampTzNanos(source.GetTimestampTzNanos()); return; + case VariantPrimitiveType.TimestampNtzNanos: WriteTimestampNtzNanos(source.GetTimestampNtzNanos()); return; + case VariantPrimitiveType.String: WriteString(source.GetString()); return; + case VariantPrimitiveType.Binary: WriteBinary(source.GetBinary().ToArray()); return; + case VariantPrimitiveType.Uuid: WriteUuid(source.GetUuid()); return; + default: + throw new NotSupportedException($"Unsupported primitive type: {pt}"); + } + } + // --------------------------------------------------------------- // Internal bookkeeping // --------------------------------------------------------------- diff --git a/src/Apache.Arrow/Arrays/VariantArray.cs b/src/Apache.Arrow/Arrays/VariantArray.cs index c31a382a..3bc9fd53 100644 --- a/src/Apache.Arrow/Arrays/VariantArray.cs +++ b/src/Apache.Arrow/Arrays/VariantArray.cs @@ -37,9 +37,15 @@ private VariantExtensionDefinition() { } public override bool TryCreateType(IArrowType storageType, string metadata, out ExtensionType type) { + // Accept the Parquet variant storage layouts: + // struct (unshredded) + // struct (shredded with residual) + // struct (fully shredded, no value column) + // The metadata field is required. At least one of value/typed_value must be present. if (storageType is StructType structType && FindBinaryFieldIndex(structType, "metadata") >= 0 && - FindBinaryFieldIndex(structType, "value") >= 0) + (FindBinaryFieldIndex(structType, "value") >= 0 || + structType.GetFieldIndex("typed_value") >= 0)) { type = new VariantType(structType); return true; @@ -67,8 +73,15 @@ internal static int FindBinaryFieldIndex(StructType structType, string name) } /// - /// Extension type representing Parquet Variant values, stored as - /// struct<metadata: binary, value: binary>. + /// Extension type representing Parquet Variant values. The underlying storage is + /// a struct with a required metadata binary field and at least one of: + /// + /// value: binary — the variant value bytes (possibly residual when shredded). + /// typed_value: T — a typed column populated by variant shredding, where + /// T is an Arrow primitive, struct, or list per the Parquet variant shredding spec. + /// + /// Use to check for shredded layouts. Decoding shredded + /// values requires Apache.Arrow.Operations.Shredding. /// public class VariantType : ExtensionType { @@ -79,14 +92,46 @@ public class VariantType : ExtensionType public override string Name => ExtensionName; public override string ExtensionMetadata => ""; + /// + /// True if the storage layout has a value binary field (unshredded or + /// partially shredded). False for fully shredded layouts that omit the column. + /// + public bool HasValueColumn { get; } + + /// + /// True if the storage layout has a typed_value field (shredded). + /// + public bool HasTypedValueColumn { get; } + + /// + /// True if the storage layout includes any shredding (has a typed_value + /// column, or lacks a value column indicating full shredding). + /// + public bool IsShredded => HasTypedValueColumn || !HasValueColumn; + + /// + /// The typed_value field when is true; otherwise null. + /// + public Field TypedValueField { get; } + public VariantType() : base(new StructType(new[] { new Field("metadata", BinaryType.Default, false), new Field("value", BinaryType.Default, false), })) - { } + { + HasValueColumn = true; + HasTypedValueColumn = false; + TypedValueField = null; + } - internal VariantType(StructType storageType) : base(storageType) { } + internal VariantType(StructType storageType) : base(storageType) + { + HasValueColumn = VariantExtensionDefinition.FindBinaryFieldIndex(storageType, "value") >= 0; + int typedIdx = storageType.GetFieldIndex("typed_value"); + HasTypedValueColumn = typedIdx >= 0; + TypedValueField = typedIdx >= 0 ? storageType.Fields[typedIdx] : null; + } public override ExtensionArray CreateArray(IArrowArray storageArray) { @@ -107,15 +152,53 @@ public class VariantArray : ExtensionArray, IReadOnlyList public StructArray StorageArray => (StructArray)Storage; + /// + /// The variant type metadata describing which columns are present. + /// + public VariantType VariantType => (VariantType)ExtensionType; + + /// + /// True when the underlying column includes shredded data (has a + /// typed_value column, or lacks a value column). Reads + /// on shredded columns require Apache.Arrow.Operations.Shredding. + /// + public bool IsShredded => VariantType.IsShredded; + + /// + /// The typed_value child array when the column is shredded; otherwise null. + /// + public IArrowArray TypedValueArray { get; } + public VariantArray(VariantType variantType, IArrowArray storage) : base(variantType, storage) { var structType = (StructType)variantType.StorageType; _metadataArray = DecodeBinaryArray(StorageArray.Fields[structType.GetFieldIndex("metadata")], out _metadataIndexes); - _valueArray = DecodeBinaryArray(StorageArray.Fields[structType.GetFieldIndex("value")], out _valueIndexes); + + if (variantType.HasValueColumn) + { + _valueArray = DecodeBinaryArray(StorageArray.Fields[structType.GetFieldIndex("value")], out _valueIndexes); + } + + if (variantType.HasTypedValueColumn) + { + TypedValueArray = StorageArray.Fields[structType.GetFieldIndex("typed_value")]; + } } - public VariantArray(IArrowArray storage) : this(VariantType.Default, storage) { } + public VariantArray(IArrowArray storage) : this(InferVariantType(storage), storage) { } + + private static VariantType InferVariantType(IArrowArray storage) + { + if (storage == null) throw new ArgumentNullException(nameof(storage)); + if (VariantExtensionDefinition.Instance.TryCreateType(storage.Data.DataType, null, out ExtensionType ext)) + { + return (VariantType)ext; + } + throw new ArgumentException( + "Storage array does not match a variant layout (expected struct).", + nameof(storage)); + } /// /// Gets the metadata bytes for the element at the given index. @@ -129,18 +212,52 @@ public ReadOnlySpan GetMetadataBytes(int index) /// /// Gets the value bytes for the element at the given index. /// + /// If the column has no value field. public ReadOnlySpan GetValueBytes(int index) { + if (_valueArray == null) + { + throw new InvalidOperationException( + "This VariantArray has no 'value' column (fully shredded layout). " + + "Use the shredding-aware readers in Apache.Arrow.Operations.Shredding."); + } int physicalIndex = _valueIndexes.GetPhysicalIndex(index); return _valueArray.GetBytes(physicalIndex, out bool isNull); } + /// + /// Returns true and sets when the element at + /// has value bytes, false otherwise. Shredded + /// elements whose residual is null will return false. + /// + public bool TryGetValueBytes(int index, out ReadOnlySpan value) + { + if (_valueArray == null) + { + value = default; + return false; + } + int physicalIndex = _valueIndexes.GetPhysicalIndex(index); + ReadOnlySpan bytes = _valueArray.GetBytes(physicalIndex, out bool isNull); + if (isNull) + { + value = default; + return false; + } + value = bytes; + return true; + } + /// /// Gets a zero-copy for the element at the given index. /// The reader is only valid while the underlying array buffers are alive. /// /// If is out of range. - /// If the element at is null. + /// + /// If the element at is null, or the column is shredded + /// (a over residual bytes alone does not represent the + /// logical variant — use Apache.Arrow.Operations.Shredding instead). + /// public VariantReader GetVariantReader(int index) { if (index < 0 || index >= Length) @@ -149,12 +266,20 @@ public VariantReader GetVariantReader(int index) if (IsNull(index)) throw new InvalidOperationException("Cannot create a VariantReader for a null element."); + if (IsShredded) + throw new InvalidOperationException( + "Cannot create a VariantReader for a shredded column. Use the shredding-aware " + + "readers in Apache.Arrow.Operations.Shredding."); + return new VariantReader(GetMetadataBytes(index), GetValueBytes(index)); } /// /// Gets a materialized for the element at the given index. + /// Shredded columns require Apache.Arrow.Operations.Shredding; call the + /// GetShreddedVariant / GetLogicalVariantValue extension methods instead. /// + /// If the column is shredded. public VariantValue GetVariantValue(int index) { if (index < 0 || index >= Length) @@ -163,6 +288,12 @@ public VariantValue GetVariantValue(int index) if (IsNull(index)) return VariantValue.Null; + if (IsShredded) + throw new InvalidOperationException( + "GetVariantValue is not supported on shredded VariantArrays. " + + "Reference Apache.Arrow.Operations.Shredding and use GetLogicalVariantValue " + + "(transparent materialization) or GetShreddedVariant (reader-style access)."); + var metadata = GetMetadataBytes(index); var value = GetValueBytes(index); var reader = new VariantReader(metadata, value); @@ -174,6 +305,13 @@ public VariantValue GetVariantValue(int index) public IEnumerator GetEnumerator() { + if (IsShredded) + { + throw new InvalidOperationException( + "Enumeration is not supported on shredded VariantArrays. " + + "Reference Apache.Arrow.Operations.Shredding and iterate via GetLogicalVariantValue."); + } + IEnumerator metadataIdx = _metadataIndexes.EnumeratePhysicalIndices().GetEnumerator(); IEnumerator valueIdx = _valueIndexes.EnumeratePhysicalIndices().GetEnumerator(); for (int i = 0; metadataIdx.MoveNext() && valueIdx.MoveNext(); i++) diff --git a/test/Apache.Arrow.Operations.Tests/Shredding/ShredRoundTripTests.cs b/test/Apache.Arrow.Operations.Tests/Shredding/ShredRoundTripTests.cs new file mode 100644 index 00000000..bfcd4114 --- /dev/null +++ b/test/Apache.Arrow.Operations.Tests/Shredding/ShredRoundTripTests.cs @@ -0,0 +1,404 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System; +using System.Collections.Generic; +using Apache.Arrow.Operations.VariantJson; +using Apache.Arrow.Operations.Shredding; +using Apache.Arrow.Scalars.Variant; +using Xunit; + +namespace Apache.Arrow.Operations.Tests.Shredding +{ + /// + /// Round-trip tests: VariantValue → Shred → Reconstruct → VariantValue, verifying equality. + /// + public class ShredRoundTripTests + { + private static VariantValue RoundTrip(VariantValue original, ShredSchema schema) + { + (byte[] metadata, IReadOnlyList rows) = VariantShredder.Shred(new[] { original }, schema); + VariantValue? result = VariantUnshredder.Reconstruct(rows[0], schema, metadata); + Assert.True(result.HasValue, "Round-trip should not produce a missing value."); + return result.Value; + } + + // --------------------------------------------------------------- + // Primitives through typed columns + // --------------------------------------------------------------- + + [Fact] + public void RoundTrip_Boolean_True() + { + ShredSchema schema = ShredSchema.Primitive(ShredType.Boolean); + Assert.Equal(VariantValue.True, RoundTrip(VariantValue.True, schema)); + } + + [Fact] + public void RoundTrip_Boolean_False() + { + ShredSchema schema = ShredSchema.Primitive(ShredType.Boolean); + Assert.Equal(VariantValue.False, RoundTrip(VariantValue.False, schema)); + } + + [Fact] + public void RoundTrip_Int8() + { + ShredSchema schema = ShredSchema.Primitive(ShredType.Int8); + VariantValue v = VariantValue.FromInt8(-42); + Assert.Equal(v, RoundTrip(v, schema)); + } + + [Fact] + public void RoundTrip_Int16() + { + ShredSchema schema = ShredSchema.Primitive(ShredType.Int16); + VariantValue v = VariantValue.FromInt16(short.MaxValue); + Assert.Equal(v, RoundTrip(v, schema)); + } + + [Fact] + public void RoundTrip_Int32() + { + ShredSchema schema = ShredSchema.Primitive(ShredType.Int32); + VariantValue v = VariantValue.FromInt32(42); + Assert.Equal(v, RoundTrip(v, schema)); + } + + [Fact] + public void RoundTrip_Int64() + { + ShredSchema schema = ShredSchema.Primitive(ShredType.Int64); + VariantValue v = VariantValue.FromInt64(long.MaxValue); + Assert.Equal(v, RoundTrip(v, schema)); + } + + [Fact] + public void RoundTrip_Float() + { + ShredSchema schema = ShredSchema.Primitive(ShredType.Float); + VariantValue v = VariantValue.FromFloat(3.14f); + Assert.Equal(v, RoundTrip(v, schema)); + } + + [Fact] + public void RoundTrip_Double() + { + ShredSchema schema = ShredSchema.Primitive(ShredType.Double); + VariantValue v = VariantValue.FromDouble(Math.PI); + Assert.Equal(v, RoundTrip(v, schema)); + } + + [Fact] + public void RoundTrip_String() + { + ShredSchema schema = ShredSchema.Primitive(ShredType.String); + VariantValue v = VariantValue.FromString("hello world"); + Assert.Equal(v, RoundTrip(v, schema)); + } + + [Fact] + public void RoundTrip_Binary() + { + ShredSchema schema = ShredSchema.Primitive(ShredType.Binary); + VariantValue v = VariantValue.FromBinary(new byte[] { 0, 1, 2, 255 }); + Assert.Equal(v, RoundTrip(v, schema)); + } + + [Fact] + public void RoundTrip_Uuid() + { + ShredSchema schema = ShredSchema.Primitive(ShredType.Uuid); + VariantValue v = VariantValue.FromUuid(Guid.NewGuid()); + Assert.Equal(v, RoundTrip(v, schema)); + } + + [Fact] + public void RoundTrip_Date() + { + ShredSchema schema = ShredSchema.Primitive(ShredType.Date); + VariantValue v = VariantValue.FromDate(19000); + Assert.Equal(v, RoundTrip(v, schema)); + } + + [Fact] + public void RoundTrip_Timestamp() + { + ShredSchema schema = ShredSchema.Primitive(ShredType.Timestamp); + VariantValue v = VariantValue.FromTimestamp(1640995200000000L); + Assert.Equal(v, RoundTrip(v, schema)); + } + + [Fact] + public void RoundTrip_Decimal4() + { + ShredSchema schema = ShredSchema.Primitive(ShredType.Decimal4); + VariantValue v = VariantValue.FromDecimal4(99.99m); + Assert.Equal(v, RoundTrip(v, schema)); + } + + // --------------------------------------------------------------- + // Primitives through binary fallback (type mismatch) + // --------------------------------------------------------------- + + [Fact] + public void RoundTrip_TypeMismatch_FallsBackToBinary() + { + // Schema expects Int32, but value is a string — goes through binary. + ShredSchema schema = ShredSchema.Primitive(ShredType.Int32); + VariantValue v = VariantValue.FromString("hello"); + Assert.Equal(v, RoundTrip(v, schema)); + } + + [Fact] + public void RoundTrip_Null_FallsBackToBinary() + { + ShredSchema schema = ShredSchema.Primitive(ShredType.Int32); + Assert.Equal(VariantValue.Null, RoundTrip(VariantValue.Null, schema)); + } + + // --------------------------------------------------------------- + // Objects + // --------------------------------------------------------------- + + [Fact] + public void RoundTrip_FullyShreddedObject() + { + VariantValue original = VariantValue.FromObject(new Dictionary + { + { "name", VariantValue.FromString("Alice") }, + { "age", VariantValue.FromInt32(30) }, + }); + + ShredSchema schema = ShredSchema.ForObject(new Dictionary + { + { "name", ShredSchema.Primitive(ShredType.String) }, + { "age", ShredSchema.Primitive(ShredType.Int32) }, + }); + + Assert.Equal(original, RoundTrip(original, schema)); + } + + [Fact] + public void RoundTrip_PartiallyShreddedObject() + { + VariantValue original = VariantValue.FromObject(new Dictionary + { + { "name", VariantValue.FromString("Alice") }, + { "age", VariantValue.FromInt32(30) }, + { "active", VariantValue.True }, + }); + + // Only shred "name" — "age" and "active" go to residual binary. + ShredSchema schema = ShredSchema.ForObject(new Dictionary + { + { "name", ShredSchema.Primitive(ShredType.String) }, + }); + + Assert.Equal(original, RoundTrip(original, schema)); + } + + [Fact] + public void RoundTrip_ObjectWithMissingField() + { + // Only "name" present, schema expects "name" and "age". + VariantValue original = VariantValue.FromObject(new Dictionary + { + { "name", VariantValue.FromString("Alice") }, + }); + + ShredSchema schema = ShredSchema.ForObject(new Dictionary + { + { "name", ShredSchema.Primitive(ShredType.String) }, + { "age", ShredSchema.Primitive(ShredType.Int32) }, + }); + + Assert.Equal(original, RoundTrip(original, schema)); + } + + [Fact] + public void RoundTrip_EmptyObject() + { + VariantValue original = VariantValue.FromObject(new Dictionary()); + ShredSchema schema = ShredSchema.ForObject(new Dictionary + { + { "x", ShredSchema.Primitive(ShredType.Int32) }, + }); + + Assert.Equal(original, RoundTrip(original, schema)); + } + + // --------------------------------------------------------------- + // Arrays + // --------------------------------------------------------------- + + [Fact] + public void RoundTrip_Array_Homogeneous() + { + VariantValue original = VariantValue.FromArray( + VariantValue.FromInt32(1), + VariantValue.FromInt32(2), + VariantValue.FromInt32(3)); + + ShredSchema schema = ShredSchema.ForArray(ShredSchema.Primitive(ShredType.Int32)); + Assert.Equal(original, RoundTrip(original, schema)); + } + + [Fact] + public void RoundTrip_Array_Mixed() + { + VariantValue original = VariantValue.FromArray( + VariantValue.FromInt32(1), + VariantValue.FromString("two"), + VariantValue.Null, + VariantValue.True); + + ShredSchema schema = ShredSchema.ForArray(ShredSchema.Primitive(ShredType.Int32)); + Assert.Equal(original, RoundTrip(original, schema)); + } + + [Fact] + public void RoundTrip_EmptyArray() + { + VariantValue original = VariantValue.FromArray(new List()); + ShredSchema schema = ShredSchema.ForArray(ShredSchema.Primitive(ShredType.Int32)); + Assert.Equal(original, RoundTrip(original, schema)); + } + + // --------------------------------------------------------------- + // Nested structures + // --------------------------------------------------------------- + + [Fact] + public void RoundTrip_NestedObjectsAndArrays() + { + VariantValue original = VariantValue.FromObject(new Dictionary + { + { "users", VariantValue.FromArray( + VariantValue.FromObject(new Dictionary + { + { "name", VariantValue.FromString("Alice") }, + { "scores", VariantValue.FromArray( + VariantValue.FromInt32(95), + VariantValue.FromInt32(87)) + }, + }), + VariantValue.FromObject(new Dictionary + { + { "name", VariantValue.FromString("Bob") }, + { "scores", VariantValue.FromArray( + VariantValue.FromInt32(88)) + }, + })) + }, + { "count", VariantValue.FromInt32(2) }, + }); + + ShredSchema schema = ShredSchema.ForObject(new Dictionary + { + { "users", ShredSchema.ForArray( + ShredSchema.ForObject(new Dictionary + { + { "name", ShredSchema.Primitive(ShredType.String) }, + { "scores", ShredSchema.ForArray(ShredSchema.Primitive(ShredType.Int32)) }, + })) + }, + { "count", ShredSchema.Primitive(ShredType.Int32) }, + }); + + Assert.Equal(original, RoundTrip(original, schema)); + } + + // --------------------------------------------------------------- + // Unshredded fallback + // --------------------------------------------------------------- + + [Fact] + public void RoundTrip_Unshredded() + { + VariantValue original = VariantValue.FromObject(new Dictionary + { + { "name", VariantValue.FromString("Alice") }, + { "age", VariantValue.FromInt32(30) }, + }); + + ShredSchema schema = ShredSchema.Unshredded(); + Assert.Equal(original, RoundTrip(original, schema)); + } + + // --------------------------------------------------------------- + // Cross-codec: JSON → shred → unshred → JSON + // --------------------------------------------------------------- + + [Fact] + public void RoundTrip_JsonThroughShredding() + { + string originalJson = "{\"name\":\"Alice\",\"age\":30,\"tags\":[\"a\",\"b\"]}"; + (byte[] metaIn, byte[] valueIn) = VariantJsonReader.Parse(originalJson); + VariantValue parsed = new VariantReader(metaIn, valueIn).ToVariantValue(); + + ShredSchema schema = ShredSchema.ForObject(new Dictionary + { + { "name", ShredSchema.Primitive(ShredType.String) }, + { "age", ShredSchema.Primitive(ShredType.Int8) }, + }); + + VariantValue reconstructed = RoundTrip(parsed, schema); + string resultJson = VariantJsonWriter.ToJson(reconstructed); + + // Field order in JSON output is not guaranteed to match; compare via parsed equality. + (byte[] metaOut, byte[] valueOut) = VariantJsonReader.Parse(resultJson); + VariantValue reparsed = new VariantReader(metaOut, valueOut).ToVariantValue(); + Assert.Equal(parsed, reparsed); + } + + // --------------------------------------------------------------- + // Column-level shared metadata: rows share a single dictionary + // --------------------------------------------------------------- + + [Fact] + public void RoundTrip_MultipleRows_SharedMetadata() + { + List values = new List + { + VariantValue.FromObject(new Dictionary + { + { "name", VariantValue.FromString("Alice") }, + { "extra1", VariantValue.FromInt32(1) }, + }), + VariantValue.FromObject(new Dictionary + { + { "name", VariantValue.FromString("Bob") }, + { "extra2", VariantValue.FromString("hi") }, + }), + }; + + ShredSchema schema = ShredSchema.ForObject(new Dictionary + { + { "name", ShredSchema.Primitive(ShredType.String) }, + }); + + (byte[] metadata, IReadOnlyList rows) = VariantShredder.Shred(values, schema); + + Assert.Equal(values.Count, rows.Count); + for (int i = 0; i < values.Count; i++) + { + VariantValue? reconstructed = VariantUnshredder.Reconstruct(rows[i], schema, metadata); + Assert.True(reconstructed.HasValue); + Assert.Equal(values[i], reconstructed.Value); + } + } + } +} diff --git a/test/Apache.Arrow.Operations.Tests/Shredding/ShredSchemaInfererTests.cs b/test/Apache.Arrow.Operations.Tests/Shredding/ShredSchemaInfererTests.cs new file mode 100644 index 00000000..9aaee5d9 --- /dev/null +++ b/test/Apache.Arrow.Operations.Tests/Shredding/ShredSchemaInfererTests.cs @@ -0,0 +1,259 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System.Collections.Generic; +using Apache.Arrow.Operations.Shredding; +using Apache.Arrow.Scalars.Variant; +using Xunit; + +namespace Apache.Arrow.Operations.Tests.Shredding +{ + public class ShredSchemaInfererTests + { + private readonly ShredSchemaInferer _inferer = new ShredSchemaInferer(); + + [Fact] + public void Infer_EmptyValues_ReturnsUnshredded() + { + ShredSchema schema = _inferer.Infer(new List()); + Assert.Equal(ShredType.None, schema.TypedValueType); + } + + [Fact] + public void Infer_AllSameType_ReturnsPrimitive() + { + List values = new List + { + VariantValue.FromInt32(1), + VariantValue.FromInt32(2), + VariantValue.FromInt32(3), + }; + + ShredSchema schema = _inferer.Infer(values); + Assert.Equal(ShredType.Int32, schema.TypedValueType); + } + + [Fact] + public void Infer_MixedTypes_BelowConsistency_ReturnsUnshredded() + { + // 2 strings vs 2 ints = 50% consistency, below default 80% threshold. + List values = new List + { + VariantValue.FromInt32(1), + VariantValue.FromString("two"), + VariantValue.FromInt32(3), + VariantValue.FromString("four"), + }; + + ShredSchema schema = _inferer.Infer(values); + Assert.Equal(ShredType.None, schema.TypedValueType); + } + + [Fact] + public void Infer_MixedTypes_AboveConsistency_ReturnsDominant() + { + // 9 ints vs 1 string = 90% consistency, above default 80%. + List values = new List(); + for (int i = 0; i < 9; i++) + { + values.Add(VariantValue.FromInt32(i)); + } + values.Add(VariantValue.FromString("outlier")); + + ShredSchema schema = _inferer.Infer(values); + Assert.Equal(ShredType.Int32, schema.TypedValueType); + } + + [Fact] + public void Infer_Objects_InfersFieldSchemas() + { + List values = new List + { + VariantValue.FromObject(new Dictionary + { + { "name", VariantValue.FromString("Alice") }, + { "age", VariantValue.FromInt32(30) }, + }), + VariantValue.FromObject(new Dictionary + { + { "name", VariantValue.FromString("Bob") }, + { "age", VariantValue.FromInt32(25) }, + }), + }; + + ShredSchema schema = _inferer.Infer(values); + Assert.Equal(ShredType.Object, schema.TypedValueType); + Assert.NotNull(schema.ObjectFields); + Assert.True(schema.ObjectFields.ContainsKey("name")); + Assert.True(schema.ObjectFields.ContainsKey("age")); + Assert.Equal(ShredType.String, schema.ObjectFields["name"].TypedValueType); + Assert.Equal(ShredType.Int32, schema.ObjectFields["age"].TypedValueType); + } + + [Fact] + public void Infer_Objects_RareField_Excluded() + { + // "rare" appears in only 1 of 4 objects = 25%, below 50% threshold. + List values = new List(); + for (int i = 0; i < 4; i++) + { + Dictionary fields = new Dictionary + { + { "name", VariantValue.FromString($"user{i}") }, + }; + if (i == 0) + { + fields["rare"] = VariantValue.FromInt32(99); + } + values.Add(VariantValue.FromObject(fields)); + } + + ShredSchema schema = _inferer.Infer(values); + Assert.Equal(ShredType.Object, schema.TypedValueType); + Assert.True(schema.ObjectFields.ContainsKey("name")); + Assert.False(schema.ObjectFields.ContainsKey("rare")); + } + + [Fact] + public void Infer_Arrays_InfersElementSchema() + { + List values = new List + { + VariantValue.FromArray( + VariantValue.FromInt32(1), + VariantValue.FromInt32(2)), + VariantValue.FromArray( + VariantValue.FromInt32(3)), + }; + + ShredSchema schema = _inferer.Infer(values); + Assert.Equal(ShredType.Array, schema.TypedValueType); + Assert.NotNull(schema.ArrayElement); + Assert.Equal(ShredType.Int32, schema.ArrayElement.TypedValueType); + } + + [Fact] + public void Infer_NestedObjects() + { + List values = new List + { + VariantValue.FromObject(new Dictionary + { + { "address", VariantValue.FromObject(new Dictionary + { + { "city", VariantValue.FromString("NYC") }, + }) + }, + }), + VariantValue.FromObject(new Dictionary + { + { "address", VariantValue.FromObject(new Dictionary + { + { "city", VariantValue.FromString("LA") }, + }) + }, + }), + }; + + ShredSchema schema = _inferer.Infer(values); + Assert.Equal(ShredType.Object, schema.TypedValueType); + Assert.True(schema.ObjectFields.ContainsKey("address")); + + ShredSchema addressSchema = schema.ObjectFields["address"]; + Assert.Equal(ShredType.Object, addressSchema.TypedValueType); + Assert.True(addressSchema.ObjectFields.ContainsKey("city")); + Assert.Equal(ShredType.String, addressSchema.ObjectFields["city"].TypedValueType); + } + + [Fact] + public void Infer_RespectsMaxDepth() + { + List values = new List + { + VariantValue.FromObject(new Dictionary + { + { "deep", VariantValue.FromObject(new Dictionary + { + { "value", VariantValue.FromInt32(1) }, + }) + }, + }), + VariantValue.FromObject(new Dictionary + { + { "deep", VariantValue.FromObject(new Dictionary + { + { "value", VariantValue.FromInt32(2) }, + }) + }, + }), + }; + + // MaxDepth=0 means only top level — nested objects not explored. + ShredOptions options = new ShredOptions { MaxDepth = 0 }; + ShredSchema schema = _inferer.Infer(values, options); + + // Top level is object, but fields shouldn't be further explored. + Assert.Equal(ShredType.Object, schema.TypedValueType); + // "deep" field is an object, but since we can't recurse (maxDepth=0), + // it falls back to unshredded. + Assert.True(schema.ObjectFields.ContainsKey("deep")); + Assert.Equal(ShredType.None, schema.ObjectFields["deep"].TypedValueType); + } + + [Fact] + public void Infer_CustomOptions() + { + // 3 strings + 1 int = 75% string. Default (80%) would reject, but custom 70% accepts. + List values = new List + { + VariantValue.FromString("a"), + VariantValue.FromString("b"), + VariantValue.FromString("c"), + VariantValue.FromInt32(1), + }; + + ShredOptions options = new ShredOptions { MinTypeConsistency = 0.7 }; + ShredSchema schema = _inferer.Infer(values, options); + Assert.Equal(ShredType.String, schema.TypedValueType); + } + + [Fact] + public void Infer_AllStrings() + { + List values = new List + { + VariantValue.FromString("hello"), + VariantValue.FromString("world"), + }; + + ShredSchema schema = _inferer.Infer(values); + Assert.Equal(ShredType.String, schema.TypedValueType); + } + + [Fact] + public void Infer_AllBooleans() + { + List values = new List + { + VariantValue.True, + VariantValue.False, + VariantValue.True, + }; + + ShredSchema schema = _inferer.Infer(values); + Assert.Equal(ShredType.Boolean, schema.TypedValueType); + } + } +} diff --git a/test/Apache.Arrow.Operations.Tests/Shredding/ShredSchemaTests.cs b/test/Apache.Arrow.Operations.Tests/Shredding/ShredSchemaTests.cs new file mode 100644 index 00000000..f5f0eb5e --- /dev/null +++ b/test/Apache.Arrow.Operations.Tests/Shredding/ShredSchemaTests.cs @@ -0,0 +1,159 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System; +using System.Collections.Generic; +using Apache.Arrow; +using Apache.Arrow.Operations.Shredding; +using Apache.Arrow.Scalars.Variant; +using Apache.Arrow.Types; +using Xunit; + +namespace Apache.Arrow.Operations.Tests.Shredding +{ + public class ShredSchemaTests + { + [Fact] + public void Unshredded_HasNoneType() + { + ShredSchema schema = ShredSchema.Unshredded(); + Assert.Equal(ShredType.None, schema.TypedValueType); + Assert.Null(schema.ObjectFields); + Assert.Null(schema.ArrayElement); + } + + [Fact] + public void Primitive_HasCorrectType() + { + ShredSchema schema = ShredSchema.Primitive(ShredType.Int32); + Assert.Equal(ShredType.Int32, schema.TypedValueType); + Assert.Null(schema.ObjectFields); + Assert.Null(schema.ArrayElement); + } + + [Theory] + [InlineData(ShredType.None)] + [InlineData(ShredType.Object)] + [InlineData(ShredType.Array)] + public void Primitive_RejectsNonPrimitiveTypes(ShredType type) + { + Assert.Throws(() => ShredSchema.Primitive(type)); + } + + [Fact] + public void ForObject_HasObjectType() + { + ShredSchema schema = ShredSchema.ForObject(new Dictionary + { + { "x", ShredSchema.Primitive(ShredType.Int32) }, + }); + + Assert.Equal(ShredType.Object, schema.TypedValueType); + Assert.NotNull(schema.ObjectFields); + Assert.Single(schema.ObjectFields); + Assert.Null(schema.ArrayElement); + } + + [Fact] + public void ForObject_NullThrows() + { + Assert.Throws(() => ShredSchema.ForObject(null)); + } + + [Fact] + public void ForArray_HasArrayType() + { + ShredSchema schema = ShredSchema.ForArray(ShredSchema.Primitive(ShredType.String)); + + Assert.Equal(ShredType.Array, schema.TypedValueType); + Assert.Null(schema.ObjectFields); + Assert.NotNull(schema.ArrayElement); + Assert.Equal(ShredType.String, schema.ArrayElement.TypedValueType); + } + + [Fact] + public void ForArray_NullThrows() + { + Assert.Throws(() => ShredSchema.ForArray(null)); + } + + [Fact] + public void ShredTypeFromPrimitive_MapsAllTypes() + { + Assert.Equal(ShredType.Boolean, ShredSchema.ShredTypeFromPrimitive(VariantPrimitiveType.BooleanTrue)); + Assert.Equal(ShredType.Boolean, ShredSchema.ShredTypeFromPrimitive(VariantPrimitiveType.BooleanFalse)); + Assert.Equal(ShredType.Int8, ShredSchema.ShredTypeFromPrimitive(VariantPrimitiveType.Int8)); + Assert.Equal(ShredType.Int16, ShredSchema.ShredTypeFromPrimitive(VariantPrimitiveType.Int16)); + Assert.Equal(ShredType.Int32, ShredSchema.ShredTypeFromPrimitive(VariantPrimitiveType.Int32)); + Assert.Equal(ShredType.Int64, ShredSchema.ShredTypeFromPrimitive(VariantPrimitiveType.Int64)); + Assert.Equal(ShredType.Float, ShredSchema.ShredTypeFromPrimitive(VariantPrimitiveType.Float)); + Assert.Equal(ShredType.Double, ShredSchema.ShredTypeFromPrimitive(VariantPrimitiveType.Double)); + Assert.Equal(ShredType.Decimal4, ShredSchema.ShredTypeFromPrimitive(VariantPrimitiveType.Decimal4)); + Assert.Equal(ShredType.Decimal8, ShredSchema.ShredTypeFromPrimitive(VariantPrimitiveType.Decimal8)); + Assert.Equal(ShredType.Decimal16, ShredSchema.ShredTypeFromPrimitive(VariantPrimitiveType.Decimal16)); + Assert.Equal(ShredType.Date, ShredSchema.ShredTypeFromPrimitive(VariantPrimitiveType.Date)); + Assert.Equal(ShredType.Timestamp, ShredSchema.ShredTypeFromPrimitive(VariantPrimitiveType.Timestamp)); + Assert.Equal(ShredType.TimestampNtz, ShredSchema.ShredTypeFromPrimitive(VariantPrimitiveType.TimestampNtz)); + Assert.Equal(ShredType.TimeNtz, ShredSchema.ShredTypeFromPrimitive(VariantPrimitiveType.TimeNtz)); + Assert.Equal(ShredType.TimestampTzNanos, ShredSchema.ShredTypeFromPrimitive(VariantPrimitiveType.TimestampTzNanos)); + Assert.Equal(ShredType.TimestampNtzNanos, ShredSchema.ShredTypeFromPrimitive(VariantPrimitiveType.TimestampNtzNanos)); + Assert.Equal(ShredType.String, ShredSchema.ShredTypeFromPrimitive(VariantPrimitiveType.String)); + Assert.Equal(ShredType.Binary, ShredSchema.ShredTypeFromPrimitive(VariantPrimitiveType.Binary)); + Assert.Equal(ShredType.Uuid, ShredSchema.ShredTypeFromPrimitive(VariantPrimitiveType.Uuid)); + Assert.Equal(ShredType.None, ShredSchema.ShredTypeFromPrimitive(VariantPrimitiveType.NullType)); + } + + // --------------------------------------------------------------- + // FromArrowType — decimal width inference across Decimal32/64/128. + // --------------------------------------------------------------- + + [Theory] + // Decimal32: any precision ≤ 9 maps to Decimal4. + [InlineData(typeof(Decimal32Type), 4, 2, ShredType.Decimal4)] + [InlineData(typeof(Decimal32Type), 9, 4, ShredType.Decimal4)] + // Decimal64: ≤9 still fits Decimal4; 10–18 maps to Decimal8. + [InlineData(typeof(Decimal64Type), 9, 2, ShredType.Decimal4)] + [InlineData(typeof(Decimal64Type), 10, 2, ShredType.Decimal8)] + [InlineData(typeof(Decimal64Type), 18, 9, ShredType.Decimal8)] + // Decimal128: width chosen by precision bucket. + [InlineData(typeof(Decimal128Type), 9, 4, ShredType.Decimal4)] + [InlineData(typeof(Decimal128Type), 18, 9, ShredType.Decimal8)] + [InlineData(typeof(Decimal128Type), 38, 9, ShredType.Decimal16)] + public void FromArrowType_DecimalTypes_MapToCorrectShredWidth( + Type arrowTypeKind, int precision, int scale, ShredType expected) + { + IArrowType arrowType = (IArrowType)Activator.CreateInstance(arrowTypeKind, precision, scale); + ShredSchema schema = ShredSchema.FromArrowType(arrowType); + Assert.Equal(expected, schema.TypedValueType); + } + + [Fact] + public void FromArrowType_Decimal128_PrecisionGreaterThan38_Throws() + { + // Precision 39 exceeds the spec max (38). + Assert.Throws( + () => ShredSchema.FromArrowType(new Decimal128Type(39, 0))); + } + + [Fact] + public void FromArrowType_Decimal256_Unsupported() + { + // Decimal256 exists in Arrow but the variant spec only defines 4/8/16-byte + // decimal widths, so 32-byte unscaled storage isn't a valid shred target. + Assert.Throws( + () => ShredSchema.FromArrowType(new Decimal256Type(10, 2))); + } + } +} diff --git a/test/Apache.Arrow.Operations.Tests/Shredding/ShreddedVariantArrayBuilderTests.cs b/test/Apache.Arrow.Operations.Tests/Shredding/ShreddedVariantArrayBuilderTests.cs new file mode 100644 index 00000000..049b1948 --- /dev/null +++ b/test/Apache.Arrow.Operations.Tests/Shredding/ShreddedVariantArrayBuilderTests.cs @@ -0,0 +1,424 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System; +using System.Collections.Generic; +using Apache.Arrow; +using Apache.Arrow.Operations.Shredding; +using Apache.Arrow.Scalars.Variant; +using Xunit; + +namespace Apache.Arrow.Operations.Tests.Shredding +{ + /// + /// Round-trip tests for the producer path: take s, + /// shred them, assemble into a shredded , and then + /// read each row back via GetLogicalVariantValue. The reader is the + /// trusted oracle (validated against the Iceberg corpus), so equality here + /// confirms the builder produces a correct Arrow structure. + /// + public class ShreddedVariantArrayBuilderTests + { + private static VariantArray ShredAndBuild(IReadOnlyList values, ShredSchema schema) + { + (byte[] metadata, IReadOnlyList rows) = VariantShredder.Shred(values, schema); + return ShreddedVariantArrayBuilder.Build(schema, metadata, rows); + } + + private static void AssertRoundTrip(IReadOnlyList values, ShredSchema schema) + { + VariantArray array = ShredAndBuild(values, schema); + Assert.Equal(values.Count, array.Length); + for (int i = 0; i < values.Count; i++) + { + VariantValue actual = array.GetLogicalVariantValue(i); + Assert.Equal(values[i], actual); + } + } + + // --------------------------------------------------------------- + // Unshredded (schema = None) + // --------------------------------------------------------------- + + [Fact] + public void Unshredded_Column_HasNoTypedValue() + { + var values = new List + { + VariantValue.FromInt32(42), + VariantValue.FromString("hello"), + }; + VariantArray array = ShredAndBuild(values, ShredSchema.Unshredded()); + + Assert.False(array.IsShredded); + AssertRoundTrip(values, ShredSchema.Unshredded()); + } + + // --------------------------------------------------------------- + // Primitive shredding + // --------------------------------------------------------------- + + [Fact] + public void Primitive_Int32() + { + var values = new List + { + VariantValue.FromInt32(1), + VariantValue.FromInt32(2), + VariantValue.FromInt32(-42), + }; + AssertRoundTrip(values, ShredSchema.Primitive(ShredType.Int32)); + } + + [Fact] + public void Primitive_Boolean() + { + var values = new List { VariantValue.True, VariantValue.False, VariantValue.True }; + AssertRoundTrip(values, ShredSchema.Primitive(ShredType.Boolean)); + } + + [Fact] + public void Primitive_String() + { + var values = new List + { + VariantValue.FromString("alpha"), + VariantValue.FromString("beta"), + VariantValue.FromString(""), + }; + AssertRoundTrip(values, ShredSchema.Primitive(ShredType.String)); + } + + [Fact] + public void Primitive_Int64() + { + var values = new List + { + VariantValue.FromInt64(long.MaxValue), + VariantValue.FromInt64(long.MinValue), + }; + AssertRoundTrip(values, ShredSchema.Primitive(ShredType.Int64)); + } + + [Fact] + public void Primitive_Double() + { + var values = new List + { + VariantValue.FromDouble(Math.PI), + VariantValue.FromDouble(-0.0), + VariantValue.FromDouble(double.MaxValue), + }; + AssertRoundTrip(values, ShredSchema.Primitive(ShredType.Double)); + } + + [Fact] + public void Primitive_Decimal4() + { + var values = new List + { + VariantValue.FromDecimal4(123.45m), + VariantValue.FromDecimal4(-99.99m), + }; + AssertRoundTrip(values, ShredSchema.Primitive(ShredType.Decimal4)); + } + + [Fact] + public void Primitive_Date() + { + var values = new List + { + VariantValue.FromDate(19000), + VariantValue.FromDate(0), + }; + AssertRoundTrip(values, ShredSchema.Primitive(ShredType.Date)); + } + + [Fact] + public void Primitive_Timestamp() + { + var values = new List + { + VariantValue.FromTimestamp(1640995200000000L), + VariantValue.FromTimestamp(0L), + }; + AssertRoundTrip(values, ShredSchema.Primitive(ShredType.Timestamp)); + } + + [Fact] + public void Primitive_Uuid() + { + var values = new List + { + VariantValue.FromUuid(Guid.NewGuid()), + VariantValue.FromUuid(Guid.Empty), + }; + AssertRoundTrip(values, ShredSchema.Primitive(ShredType.Uuid)); + } + + [Fact] + public void Primitive_Binary() + { + var values = new List + { + VariantValue.FromBinary(new byte[] { 1, 2, 3 }), + VariantValue.FromBinary(new byte[] { 0xff, 0x00 }), + }; + AssertRoundTrip(values, ShredSchema.Primitive(ShredType.Binary)); + } + + // --------------------------------------------------------------- + // Primitive type mismatch — falls back to residual + // --------------------------------------------------------------- + + [Fact] + public void Primitive_TypeMismatch_FallsBackToBinary() + { + // Schema expects Int32, values include a string — the string goes to residual. + var values = new List + { + VariantValue.FromInt32(42), + VariantValue.FromString("not an int"), + VariantValue.FromInt32(99), + }; + AssertRoundTrip(values, ShredSchema.Primitive(ShredType.Int32)); + } + + // --------------------------------------------------------------- + // Object shredding + // --------------------------------------------------------------- + + [Fact] + public void Object_FullyShredded() + { + var values = new List + { + VariantValue.FromObject(new Dictionary + { + { "name", VariantValue.FromString("Alice") }, + { "age", VariantValue.FromInt32(30) }, + }), + VariantValue.FromObject(new Dictionary + { + { "name", VariantValue.FromString("Bob") }, + { "age", VariantValue.FromInt32(25) }, + }), + }; + + ShredSchema schema = ShredSchema.ForObject(new Dictionary + { + { "name", ShredSchema.Primitive(ShredType.String) }, + { "age", ShredSchema.Primitive(ShredType.Int32) }, + }); + + VariantArray array = ShredAndBuild(values, schema); + Assert.True(array.IsShredded); + AssertRoundTrip(values, schema); + } + + [Fact] + public void Object_PartiallyShredded_MergesResidualFields() + { + var values = new List + { + VariantValue.FromObject(new Dictionary + { + { "name", VariantValue.FromString("Alice") }, + { "age", VariantValue.FromInt32(30) }, + { "extra", VariantValue.True }, + }), + }; + ShredSchema schema = ShredSchema.ForObject(new Dictionary + { + { "name", ShredSchema.Primitive(ShredType.String) }, + }); + + AssertRoundTrip(values, schema); + } + + [Fact] + public void Object_MissingField_NotInReconstruction() + { + var values = new List + { + VariantValue.FromObject(new Dictionary + { + { "name", VariantValue.FromString("only-name") }, + }), + }; + ShredSchema schema = ShredSchema.ForObject(new Dictionary + { + { "name", ShredSchema.Primitive(ShredType.String) }, + { "age", ShredSchema.Primitive(ShredType.Int32) }, + }); + + AssertRoundTrip(values, schema); + } + + // --------------------------------------------------------------- + // Array shredding + // --------------------------------------------------------------- + + [Fact] + public void Array_Homogeneous() + { + var values = new List + { + VariantValue.FromArray( + VariantValue.FromInt32(1), + VariantValue.FromInt32(2), + VariantValue.FromInt32(3)), + VariantValue.FromArray(VariantValue.FromInt32(4)), + VariantValue.FromArray(new List()), + }; + ShredSchema schema = ShredSchema.ForArray(ShredSchema.Primitive(ShredType.Int32)); + AssertRoundTrip(values, schema); + } + + [Fact] + public void Array_MixedElements_FallbackToBinary() + { + var values = new List + { + VariantValue.FromArray( + VariantValue.FromInt32(1), + VariantValue.FromString("two"), + VariantValue.FromInt32(3)), + }; + ShredSchema schema = ShredSchema.ForArray(ShredSchema.Primitive(ShredType.Int32)); + AssertRoundTrip(values, schema); + } + + // --------------------------------------------------------------- + // Nested structures + // --------------------------------------------------------------- + + [Fact] + public void Nested_ObjectsAndArrays() + { + var values = new List + { + VariantValue.FromObject(new Dictionary + { + { "users", VariantValue.FromArray( + VariantValue.FromObject(new Dictionary + { + { "name", VariantValue.FromString("Alice") }, + { "score", VariantValue.FromInt32(95) }, + }), + VariantValue.FromObject(new Dictionary + { + { "name", VariantValue.FromString("Bob") }, + { "score", VariantValue.FromInt32(88) }, + })) + }, + { "count", VariantValue.FromInt32(2) }, + }), + }; + + ShredSchema schema = ShredSchema.ForObject(new Dictionary + { + { "users", ShredSchema.ForArray( + ShredSchema.ForObject(new Dictionary + { + { "name", ShredSchema.Primitive(ShredType.String) }, + { "score", ShredSchema.Primitive(ShredType.Int32) }, + })) + }, + { "count", ShredSchema.Primitive(ShredType.Int32) }, + }); + + AssertRoundTrip(values, schema); + } + + // --------------------------------------------------------------- + // Shape of the built Arrow array + // --------------------------------------------------------------- + + [Fact] + public void Build_ProducesExpectedArrowShape_PrimitiveInt32() + { + var values = new List { VariantValue.FromInt32(42) }; + VariantArray array = ShredAndBuild(values, ShredSchema.Primitive(ShredType.Int32)); + Assert.True(array.IsShredded); + Assert.NotNull(array.TypedValueArray); + Assert.IsType(array.TypedValueArray); + } + + [Fact] + public void Build_ProducesExpectedArrowShape_Object() + { + var values = new List + { + VariantValue.FromObject(new Dictionary + { + { "x", VariantValue.FromInt32(1) }, + }), + }; + ShredSchema schema = ShredSchema.ForObject(new Dictionary + { + { "x", ShredSchema.Primitive(ShredType.Int32) }, + }); + VariantArray array = ShredAndBuild(values, schema); + Assert.True(array.IsShredded); + Assert.IsType(array.TypedValueArray); + } + + [Fact] + public void Build_ProducesExpectedArrowShape_Array() + { + var values = new List + { + VariantValue.FromArray(VariantValue.FromInt32(1), VariantValue.FromInt32(2)), + }; + ShredSchema schema = ShredSchema.ForArray(ShredSchema.Primitive(ShredType.Int32)); + VariantArray array = ShredAndBuild(values, schema); + Assert.True(array.IsShredded); + Assert.IsType(array.TypedValueArray); + } + + // --------------------------------------------------------------- + // Reader-side composition: built array is usable by the shredded reader. + // --------------------------------------------------------------- + + [Fact] + public void BuiltArray_SupportsShreddedReaderAccess() + { + var values = new List + { + VariantValue.FromObject(new Dictionary + { + { "name", VariantValue.FromString("alice") }, + { "age", VariantValue.FromInt32(42) }, + }), + }; + ShredSchema schema = ShredSchema.ForObject(new Dictionary + { + { "name", ShredSchema.Primitive(ShredType.String) }, + { "age", ShredSchema.Primitive(ShredType.Int32) }, + }); + VariantArray array = ShredAndBuild(values, schema); + + ShreddedVariant slot = array.GetShreddedVariant(0); + ShreddedObject obj = slot.GetObject(); + + Assert.True(obj.TryGetField("name", out ShreddedVariant nameField)); + Assert.Equal("alice", nameField.GetString()); + + Assert.True(obj.TryGetField("age", out ShreddedVariant ageField)); + Assert.Equal(42, ageField.GetInt32()); + } + } +} diff --git a/test/Apache.Arrow.Operations.Tests/Shredding/ShreddedVariantConformanceTests.cs b/test/Apache.Arrow.Operations.Tests/Shredding/ShreddedVariantConformanceTests.cs new file mode 100644 index 00000000..b592ffe7 --- /dev/null +++ b/test/Apache.Arrow.Operations.Tests/Shredding/ShreddedVariantConformanceTests.cs @@ -0,0 +1,236 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Text.Json; +using Apache.Arrow; +using Apache.Arrow.Ipc; +using Apache.Arrow.Operations.Shredding; +using Apache.Arrow.Scalars.Variant; +using Apache.Arrow.Types; +using Xunit; + +namespace Apache.Arrow.Operations.Tests.Shredding +{ + /// + /// Conformance tests against the Iceberg-derived shredded variant test corpus + /// from apache/parquet-testing. For each case, the sibling .arrow IPC + /// file (produced by test/shredded_variant_ipc/regen.py) is loaded, + /// the var column is projected as a , and + /// each row's materialization is compared against the expected + /// *.variant.bin payload. + /// + public class ShreddedVariantConformanceTests + { + private static readonly string IpcDir = FindIpcDir(); + private static readonly string ShreddedVariantDir = FindShreddedVariantDir(); + + private static string FindIpcDir() + { + string dir = AppContext.BaseDirectory; + for (int i = 0; i < 10; i++) + { + string candidate = Path.Combine(dir, "test", "shredded_variant_ipc"); + if (Directory.Exists(candidate) && Directory.GetFiles(candidate, "*.arrow").Length > 0) + return candidate; + string parent = Path.GetDirectoryName(dir); + if (parent == null || parent == dir) break; + dir = parent; + } + return null; + } + + private static string FindShreddedVariantDir() + { + string dir = AppContext.BaseDirectory; + for (int i = 0; i < 10; i++) + { + string candidate = Path.Combine(dir, "test", "parquet-testing", "shredded_variant"); + if (File.Exists(Path.Combine(candidate, "cases.json"))) + return candidate; + string parent = Path.GetDirectoryName(dir); + if (parent == null || parent == dir) break; + dir = parent; + } + return null; + } + + public static IEnumerable SingleRecordCases() + { + string shreddedDir = FindShreddedVariantDir(); + if (shreddedDir == null) + { + yield return new object[] { 0, null, null, null }; + yield break; + } + + string casesPath = Path.Combine(shreddedDir, "cases.json"); + using JsonDocument doc = JsonDocument.Parse(File.ReadAllText(casesPath)); + foreach (JsonElement c in doc.RootElement.EnumerateArray()) + { + if (!c.TryGetProperty("variant_file", out JsonElement vf)) continue; + if (!c.TryGetProperty("parquet_file", out JsonElement pf)) continue; + // Skip spec-INVALID cases here — the cases.json notes explicitly say + // "implementations can choose to error, or read the shredded value". + // They're covered by separate deliberate tests. + if (pf.GetString().Contains("INVALID")) continue; + int caseNumber = c.GetProperty("case_number").GetInt32(); + string testName = c.TryGetProperty("test", out JsonElement t) ? t.GetString() : ""; + yield return new object[] { caseNumber, testName, pf.GetString(), vf.GetString() }; + } + } + + public static IEnumerable MultiRecordCases() + { + string shreddedDir = FindShreddedVariantDir(); + if (shreddedDir == null) + { + yield return new object[] { 0, null, null }; + yield break; + } + + string casesPath = Path.Combine(shreddedDir, "cases.json"); + using JsonDocument doc = JsonDocument.Parse(File.ReadAllText(casesPath)); + foreach (JsonElement c in doc.RootElement.EnumerateArray()) + { + if (!c.TryGetProperty("variant_files", out _)) continue; + if (!c.TryGetProperty("parquet_file", out JsonElement pf)) continue; + int caseNumber = c.GetProperty("case_number").GetInt32(); + string testName = c.TryGetProperty("test", out JsonElement t) ? t.GetString() : ""; + yield return new object[] { caseNumber, testName, pf.GetString() }; + } + } + + [SkippableTheory] + [MemberData(nameof(SingleRecordCases))] + public void SingleRecord(int caseNumber, string testName, string parquetFile, string variantFile) + { + Skip.If(ShreddedVariantDir == null, "parquet-testing submodule not checked out"); + Skip.If(IpcDir == null, "regen.py has not been run (test/shredded_variant_ipc/*.arrow missing)"); + + string stem = Path.GetFileNameWithoutExtension(parquetFile); + string ipcPath = Path.Combine(IpcDir, stem + ".arrow"); + string variantBinPath = Path.Combine(ShreddedVariantDir, variantFile); + Skip.IfNot(File.Exists(ipcPath), $"Missing {ipcPath} (case {caseNumber}: {testName})"); + + VariantArray variantArray = LoadVariantArray(ipcPath); + Assert.True(variantArray.Length >= 1, $"Expected at least 1 row (case {caseNumber}: {testName})"); + + VariantValue actual = variantArray.GetLogicalVariantValue(0); + VariantValue expected = LoadExpectedVariant(variantBinPath); + + Assert.Equal(expected, actual); + } + + [SkippableTheory] + [MemberData(nameof(MultiRecordCases))] + public void MultiRecord(int caseNumber, string testName, string parquetFile) + { + Skip.If(ShreddedVariantDir == null, "parquet-testing submodule not checked out"); + Skip.If(IpcDir == null, "regen.py has not been run"); + + string stem = Path.GetFileNameWithoutExtension(parquetFile); + string ipcPath = Path.Combine(IpcDir, stem + ".arrow"); + Skip.IfNot(File.Exists(ipcPath), $"Missing {ipcPath} (case {caseNumber}: {testName})"); + + VariantArray variantArray = LoadVariantArray(ipcPath); + + // Load the list of expected files from cases.json. + string casesPath = Path.Combine(ShreddedVariantDir, "cases.json"); + using JsonDocument doc = JsonDocument.Parse(File.ReadAllText(casesPath)); + JsonElement caseElement = doc.RootElement.EnumerateArray() + .First(c => c.GetProperty("case_number").GetInt32() == caseNumber); + JsonElement variantFiles = caseElement.GetProperty("variant_files"); + + Assert.Equal(variantFiles.GetArrayLength(), variantArray.Length); + + for (int i = 0; i < variantArray.Length; i++) + { + JsonElement vf = variantFiles[i]; + if (vf.ValueKind == JsonValueKind.Null) + { + Assert.True(variantArray.IsNull(i), $"Case {caseNumber} ({testName}) row {i} expected struct-level null"); + continue; + } + + Assert.False(variantArray.IsNull(i), $"Case {caseNumber} ({testName}) row {i} unexpectedly null"); + string binPath = Path.Combine(ShreddedVariantDir, vf.GetString()); + VariantValue expected = LoadExpectedVariant(binPath); + VariantValue actual = variantArray.GetLogicalVariantValue(i); + Assert.Equal(expected, actual); + } + } + + // --------------------------------------------------------------- + // IPC loading helpers + // --------------------------------------------------------------- + + private static VariantArray LoadVariantArray(string ipcPath) + { + using Stream stream = File.OpenRead(ipcPath); + using ArrowFileReader reader = new ArrowFileReader(stream); + RecordBatch batch = reader.ReadNextRecordBatch(); + if (batch == null) + { + throw new InvalidOperationException($"No record batches in {ipcPath}"); + } + + int varIdx = batch.Schema.GetFieldIndex("var"); + Assert.True(varIdx >= 0, "IPC schema missing 'var' column"); + + IArrowArray varArray = batch.Column(varIdx); + return new VariantArray(varArray); + } + + // --------------------------------------------------------------- + // Expected-variant loading: decode the .variant.bin format + // = concatenated metadata bytes | value bytes + // --------------------------------------------------------------- + + private static VariantValue LoadExpectedVariant(string variantBinPath) + { + byte[] bytes = File.ReadAllBytes(variantBinPath); + int metadataLength = ComputeMetadataLength(bytes); + ReadOnlySpan metadata = new ReadOnlySpan(bytes, 0, metadataLength); + ReadOnlySpan value = new ReadOnlySpan(bytes, metadataLength, bytes.Length - metadataLength); + VariantReader reader = new VariantReader(metadata, value); + return reader.ToVariantValue(); + } + + private static int ComputeMetadataLength(byte[] bytes) + { + byte header = bytes[0]; + int offsetSize = ((header >> 6) & 0x3) + 1; + int dictSize = ReadLittleEndianInt(bytes, 1, offsetSize); + int offsetsStart = 1 + offsetSize; + int stringsStart = offsetsStart + (dictSize + 1) * offsetSize; + int lastOffset = ReadLittleEndianInt(bytes, offsetsStart + dictSize * offsetSize, offsetSize); + return stringsStart + lastOffset; + } + + private static int ReadLittleEndianInt(byte[] buf, int pos, int size) + { + int result = 0; + for (int i = 0; i < size; i++) + { + result |= buf[pos + i] << (8 * i); + } + return result; + } + } +} diff --git a/test/Apache.Arrow.Operations.Tests/Shredding/ShreddedVariantErrorCaseTests.cs b/test/Apache.Arrow.Operations.Tests/Shredding/ShreddedVariantErrorCaseTests.cs new file mode 100644 index 00000000..fbf3e3d8 --- /dev/null +++ b/test/Apache.Arrow.Operations.Tests/Shredding/ShreddedVariantErrorCaseTests.cs @@ -0,0 +1,178 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System; +using System.IO; +using Apache.Arrow; +using Apache.Arrow.Ipc; +using Apache.Arrow.Operations.Shredding; +using Apache.Arrow.Scalars.Variant; +using Xunit; + +namespace Apache.Arrow.Operations.Tests.Shredding +{ + /// + /// Regression tests against the spec-invalid cases in the Iceberg corpus: + /// (1) cases with error_message in cases.json — malformed schemas + /// (unsupported Arrow types) or malformed data (value/typed_value conflicts). + /// (2) cases with case-NNN-INVALID.parquet — spec-invalid but whose + /// published "implementations may error or read" leaves behavior to the reader. + /// + public class ShreddedVariantErrorCaseTests + { + private static readonly string IpcDir = FindIpcDir(); + + private static string FindIpcDir() + { + string dir = AppContext.BaseDirectory; + for (int i = 0; i < 10; i++) + { + string candidate = Path.Combine(dir, "test", "shredded_variant_ipc"); + if (Directory.Exists(candidate) && Directory.GetFiles(candidate, "*.arrow").Length > 0) + return candidate; + string parent = Path.GetDirectoryName(dir); + if (parent == null || parent == dir) break; + dir = parent; + } + return null; + } + + private static VariantArray LoadCase(string stem) + { + Skip.If(IpcDir == null, "regen.py has not been run"); + string path = Path.Combine(IpcDir, stem + ".arrow"); + Skip.IfNot(File.Exists(path), $"Missing {path}"); + + using Stream stream = File.OpenRead(path); + using ArrowFileReader reader = new ArrowFileReader(stream); + RecordBatch batch = reader.ReadNextRecordBatch(); + return new VariantArray(batch.Column(batch.Schema.GetFieldIndex("var"))); + } + + // =============================================================== + // Schema-level errors: unsupported Arrow types in typed_value. + // These should fail as soon as the shredding schema is derived. + // =============================================================== + + [SkippableFact] + public void Case127_UnsignedInteger_RejectedAtSchemaDerivation() + { + // typed_value: uint32 — not a supported shredded type per spec. + VariantArray array = LoadCase("case-127"); + ArgumentException ex = Assert.Throws(() => array.GetShredSchema()); + Assert.Contains("Unsupported shredded value type", ex.Message); + } + + [SkippableFact] + public void Case137_FixedLengthByteArray_NotUuid_RejectedAtSchemaDerivation() + { + // typed_value: fixed_size_binary[4] — only fsb(16) is valid (UUID). + VariantArray array = LoadCase("case-137"); + ArgumentException ex = Assert.Throws(() => array.GetShredSchema()); + Assert.Contains("Unsupported shredded value type", ex.Message); + } + + [SkippableFact] + public void Case127_GetLogicalVariantValue_AlsoThrows() + { + // Any reader-facing entrypoint should surface the schema error. + VariantArray array = LoadCase("case-127"); + Assert.Throws(() => array.GetLogicalVariantValue(0)); + } + + // =============================================================== + // Data-level errors: both value and typed_value populated where + // the spec forbids it (primitive and array-element slots). + // =============================================================== + + [SkippableFact] + public void Case42_PrimitiveSlot_ValueAndTypedValueConflict_Throws() + { + // Top-level row: value has residual bytes AND typed_value is an int32. + VariantArray array = LoadCase("case-042"); + InvalidOperationException ex = Assert.Throws( + () => array.GetLogicalVariantValue(0)); + Assert.Contains("both", ex.Message); + Assert.Contains("value", ex.Message); + Assert.Contains("typed_value", ex.Message); + } + + [SkippableFact] + public void Case40_ArrayElement_ValueAndTypedValueConflict_Throws() + { + // Array element 0 has both value and typed_value set. + VariantArray array = LoadCase("case-040"); + InvalidOperationException ex = Assert.Throws( + () => array.GetLogicalVariantValue(0)); + Assert.Contains("both", ex.Message); + Assert.Contains("value", ex.Message); + Assert.Contains("typed_value", ex.Message); + } + + [SkippableFact] + public void Case87_NonObjectResidualWithShreddedFields_Throws() + { + // Top-level typed_value is a shredded-object struct, but the residual + // 'value' column holds a non-object variant (int32 = 34). Spec invalid. + VariantArray array = LoadCase("case-087"); + InvalidOperationException ex = Assert.Throws( + () => array.GetLogicalVariantValue(0)); + Assert.Contains("object", ex.Message); + } + + [SkippableFact] + public void Case128_NonObjectResidualWithEmptyShreddedFields_Throws() + { + // typed_value has all-null fields, value is a variant null (not an object). + VariantArray array = LoadCase("case-128"); + Assert.Throws(() => array.GetLogicalVariantValue(0)); + } + + // =============================================================== + // "INVALID" parquet files: spec-noncompliant but whose cases.json + // notes say "implementations can choose to error, or read". We + // document current behavior: we read (and the merged value may + // differ from the Iceberg-published expected value). + // =============================================================== + + [SkippableFact] + public void Case043_INVALID_FieldConflict_ReadsWithoutThrowing() + { + // case-043-INVALID: a shredded field has typed_value=null but the + // residual object re-declares it. We merge both, producing a result + // that differs from Iceberg's published "typed wins" expectation. + VariantArray array = LoadCase("case-043-INVALID"); + VariantValue v = array.GetLogicalVariantValue(0); + Assert.True(v.IsObject, + "Invalid-043 row is expected to materialize as an object under our permissive reader."); + } + + [SkippableFact] + public void Case125_INVALID_FieldConflict_ReadsWithoutThrowing() + { + VariantArray array = LoadCase("case-125-INVALID"); + VariantValue v = array.GetLogicalVariantValue(0); + Assert.True(v.IsObject); + } + + [SkippableFact] + public void Case084_INVALID_OptionalFieldStructs_ReadsWithoutThrowing() + { + VariantArray array = LoadCase("case-084-INVALID"); + VariantValue v = array.GetLogicalVariantValue(0); + Assert.True(v.IsObject); + } + } +} diff --git a/test/Apache.Arrow.Operations.Tests/Shredding/ShreddedVariantReaderTests.cs b/test/Apache.Arrow.Operations.Tests/Shredding/ShreddedVariantReaderTests.cs new file mode 100644 index 00000000..fb997770 --- /dev/null +++ b/test/Apache.Arrow.Operations.Tests/Shredding/ShreddedVariantReaderTests.cs @@ -0,0 +1,443 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System; +using System.Collections.Generic; +using System.IO; +using Apache.Arrow; +using Apache.Arrow.Ipc; +using Apache.Arrow.Operations.Shredding; +using Apache.Arrow.Scalars.Variant; +using Apache.Arrow.Types; +using Xunit; + +namespace Apache.Arrow.Operations.Tests.Shredding +{ + /// + /// Reader-style API tests: exercise , + /// , and typed accessors + /// without going through full variant materialization. These mirror what a + /// query engine would do for push-down reads against typed Parquet columns. + /// + public class ShreddedVariantReaderTests + { + private static readonly string IpcDir = FindIpcDir(); + + private static string FindIpcDir() + { + string dir = AppContext.BaseDirectory; + for (int i = 0; i < 10; i++) + { + string candidate = Path.Combine(dir, "test", "shredded_variant_ipc"); + if (Directory.Exists(candidate) && Directory.GetFiles(candidate, "*.arrow").Length > 0) + return candidate; + string parent = Path.GetDirectoryName(dir); + if (parent == null || parent == dir) break; + dir = parent; + } + return null; + } + + private static VariantArray LoadCase(string caseStem) + { + Skip.If(IpcDir == null, "regen.py has not been run"); + string path = Path.Combine(IpcDir, caseStem + ".arrow"); + Skip.IfNot(File.Exists(path), $"Missing {path}"); + + using Stream stream = File.OpenRead(path); + using ArrowFileReader reader = new ArrowFileReader(stream); + RecordBatch batch = reader.ReadNextRecordBatch(); + return new VariantArray(batch.Column(batch.Schema.GetFieldIndex("var"))); + } + + // --------------------------------------------------------------- + // Schema + state introspection + // --------------------------------------------------------------- + + [SkippableFact] + public void GetShredSchema_ReflectsPrimitiveTypedValue() + { + VariantArray array = LoadCase("case-010"); // typed_value: int32 + ShredSchema schema = array.GetShredSchema(); + Assert.Equal(ShredType.Int32, schema.TypedValueType); + } + + [SkippableFact] + public void GetShreddedVariant_HasTypedValue_WhenColumnPopulated() + { + VariantArray array = LoadCase("case-010"); // Int32 = 12345 + ShreddedVariant slot = array.GetShreddedVariant(0); + Assert.True(slot.HasTypedValue); + Assert.False(slot.HasResidual); + Assert.False(slot.IsMissing); + } + + [SkippableFact] + public void GetShreddedVariant_HasResidual_WhenUnshredded() + { + VariantArray array = LoadCase("case-048"); // testUnshreddedVariants (bool true) + ShreddedVariant slot = array.GetShreddedVariant(0); + Assert.False(slot.HasTypedValue); + Assert.True(slot.HasResidual); + } + + // --------------------------------------------------------------- + // Typed primitive accessors + // --------------------------------------------------------------- + + [SkippableFact] + public void GetInt32_ReadsShreddedValue() + { + VariantArray array = LoadCase("case-010"); // Int32 = 12345 + ShreddedVariant slot = array.GetShreddedVariant(0); + Assert.Equal(12345, slot.GetInt32()); + } + + [SkippableFact] + public void GetInt8_ReadsShreddedValue() + { + VariantArray array = LoadCase("case-006"); // Int8 = 34 + ShreddedVariant slot = array.GetShreddedVariant(0); + Assert.Equal((sbyte)34, slot.GetInt8()); + } + + [SkippableFact] + public void GetInt64_ReadsShreddedValue() + { + VariantArray array = LoadCase("case-012"); // Int64 = 9876543210 + ShreddedVariant slot = array.GetShreddedVariant(0); + Assert.Equal(9876543210L, slot.GetInt64()); + } + + [SkippableFact] + public void GetBoolean_ReadsShreddedValue() + { + VariantArray array = LoadCase("case-004"); // Bool = true + ShreddedVariant slot = array.GetShreddedVariant(0); + Assert.True(slot.GetBoolean()); + } + + [SkippableFact] + public void GetString_ReadsShreddedValue() + { + VariantArray array = LoadCase("case-031"); // String + ShreddedVariant slot = array.GetShreddedVariant(0); + Assert.Equal("iceberg", slot.GetString()); + } + + [SkippableFact] + public void GetDouble_ReadsShreddedValue() + { + VariantArray array = LoadCase("case-016"); // Double = 14.3 + ShreddedVariant slot = array.GetShreddedVariant(0); + Assert.Equal(14.3, slot.GetDouble()); + } + + [SkippableFact] + public void GetDecimal_ReadsShreddedDecimal4() + { + VariantArray array = LoadCase("case-024"); // Decimal4 = 123456.789 (scale 3?) + ShreddedVariant slot = array.GetShreddedVariant(0); + decimal d = slot.GetDecimal(); + Assert.NotEqual(0m, d); + } + + // --------------------------------------------------------------- + // Type-mismatch errors + // --------------------------------------------------------------- + + [SkippableFact] + public void GetInt32_OnStringSchema_Throws() + { + VariantArray array = LoadCase("case-031"); // String + Assert.Throws(() => array.GetShreddedVariant(0).GetInt32()); + } + + [SkippableFact] + public void GetString_OnInt32Schema_Throws() + { + VariantArray array = LoadCase("case-010"); // Int32 + Assert.Throws(() => array.GetShreddedVariant(0).GetString()); + } + + [SkippableFact] + public void GetInt32_WithResidualOnly_Throws() + { + // Case 48 is unshredded — typed column absent on this row. + VariantArray array = LoadCase("case-048"); + Assert.False(array.GetShreddedVariant(0).HasTypedValue); + Assert.Throws(() => array.GetShreddedVariant(0).GetInt32()); + } + + // --------------------------------------------------------------- + // Residual reader access + // --------------------------------------------------------------- + + [SkippableFact] + public void TryGetResidualReader_ReturnsUnderlyingBytes() + { + VariantArray array = LoadCase("case-048"); // unshredded bool=true + ShreddedVariant slot = array.GetShreddedVariant(0); + Assert.True(slot.TryGetResidualReader(out VariantReader reader)); + Assert.True(reader.IsBoolean); + Assert.True(reader.GetBoolean()); + } + + [SkippableFact] + public void TryGetResidualReader_FalseWhenNoResidual() + { + VariantArray array = LoadCase("case-010"); // fully shredded Int32 + ShreddedVariant slot = array.GetShreddedVariant(0); + Assert.False(slot.TryGetResidualReader(out VariantReader _)); + } + + // --------------------------------------------------------------- + // Object traversal + // --------------------------------------------------------------- + + [SkippableFact] + public void GetObject_Nested_FullyTypedLeaves() + { + // case-044: outer object with field 'c' whose typed_value is itself a + // shredded object {a: int32=34, b: string="iceberg"}. + VariantArray array = LoadCase("case-044"); + ShreddedVariant slot = array.GetShreddedVariant(0); + Assert.Equal(ShredType.Object, slot.Schema.TypedValueType); + + ShreddedObject outerObj = slot.GetObject(); + Assert.True(outerObj.TryGetField("c", out ShreddedVariant cField)); + Assert.True(cField.HasTypedValue); + Assert.Equal(ShredType.Object, cField.Schema.TypedValueType); + + ShreddedObject innerObj = cField.GetObject(); + Assert.True(innerObj.TryGetField("a", out ShreddedVariant aField)); + Assert.True(aField.HasTypedValue); + Assert.Equal(34, aField.GetInt32()); + + Assert.True(innerObj.TryGetField("b", out ShreddedVariant bField)); + Assert.True(bField.HasTypedValue); + Assert.Equal("iceberg", bField.GetString()); + } + + [SkippableFact] + public void GetObject_MixedTypedAndResidualFields() + { + // case-138: top-level value column is missing (fully shredded at the top), + // but individual field typed_values can still be null at this row — + // their values live in the field-level residual ('value' sub-column). + // Field 'a' has a shredded typed of int32 but the actual value is int16, + // so 'a' falls to residual; field 'b' is typed string "iceberg". + VariantArray array = LoadCase("case-138"); + ShreddedObject obj = array.GetShreddedVariant(0).GetObject(); + + Assert.True(obj.TryGetField("a", out ShreddedVariant aField)); + Assert.False(aField.HasTypedValue); // schema says int32, value is int16 + Assert.True(aField.HasResidual); // residual holds the int16 bytes + Assert.True(aField.TryGetResidualReader(out VariantReader aReader)); + Assert.Equal((short)1234, aReader.GetInt16()); + + Assert.True(obj.TryGetField("b", out ShreddedVariant bField)); + Assert.True(bField.HasTypedValue); + Assert.Equal("iceberg", bField.GetString()); + } + + [SkippableFact] + public void TryGetField_ReturnsFalseForUnknownField() + { + VariantArray array = LoadCase("case-138"); + ShreddedObject obj = array.GetShreddedVariant(0).GetObject(); + Assert.False(obj.TryGetField("nonexistent", out _)); + } + + [SkippableFact] + public void Object_PartialShred_ExposesResidual() + { + // case-134: partially shredded — typed fields a, b plus residual field d (date). + VariantArray array = LoadCase("case-134"); + ShreddedVariant slot = array.GetShreddedVariant(0); + ShreddedObject obj = slot.GetObject(); + + Assert.True(obj.TryGetField("b", out ShreddedVariant bField)); + Assert.Equal("iceberg", bField.GetString()); + + Assert.True(obj.TryGetResidualReader(out VariantReader residualReader)); + Assert.True(residualReader.IsObject); + // Residual is a variant object holding the unshredded field(s). + VariantValue residualValue = residualReader.ToVariantValue(); + Assert.True(residualValue.IsObject); + Assert.Contains("d", residualValue.AsObject().Keys); + } + + [SkippableFact] + public void Object_MissingField_IsDetectable() + { + // case-132: typed struct but typed_value for `a` is null at row 0. + // Expected variant has only `b`; field `a` is missing. + VariantArray array = LoadCase("case-132"); + ShreddedObject obj = array.GetShreddedVariant(0).GetObject(); + + Assert.True(obj.TryGetField("a", out ShreddedVariant aField)); + Assert.True(aField.IsMissing); + + Assert.True(obj.TryGetField("b", out ShreddedVariant bField)); + Assert.False(bField.IsMissing); + Assert.Equal("iceberg", bField.GetString()); + } + + // --------------------------------------------------------------- + // Array traversal + // --------------------------------------------------------------- + + [SkippableFact] + public void GetArray_Shredded_IteratesElements() + { + // case-001: shredded array of strings [comedy, drama] + VariantArray array = LoadCase("case-001"); + ShreddedVariant slot = array.GetShreddedVariant(0); + + Assert.Equal(ShredType.Array, slot.Schema.TypedValueType); + ShreddedArray arr = slot.GetArray(); + Assert.True(arr.IsTypedList); + Assert.Equal(2, arr.ElementCount); + + Assert.Equal("comedy", arr.GetElement(0).GetString()); + Assert.Equal("drama", arr.GetElement(1).GetString()); + } + + [SkippableFact] + public void GetArray_Empty() + { + // case-002: empty array + VariantArray array = LoadCase("case-002"); + ShreddedArray arr = array.GetShreddedVariant(0).GetArray(); + Assert.True(arr.IsTypedList); + Assert.Equal(0, arr.ElementCount); + } + + [SkippableFact] + public void GetArray_ElementAccessMatches_Materialization() + { + VariantArray array = LoadCase("case-001"); + ShreddedArray arr = array.GetShreddedVariant(0).GetArray(); + // Cross-check that per-element typed access agrees with whole-array materialization. + VariantValue materialized = array.GetLogicalVariantValue(0); + Assert.True(materialized.IsArray); + var elements = materialized.AsArray(); + Assert.Equal(elements.Count, arr.ElementCount); + for (int i = 0; i < arr.ElementCount; i++) + { + Assert.Equal(elements[i].AsString(), arr.GetElement(i).GetString()); + } + } + + // --------------------------------------------------------------- + // Decimal32 / Decimal64 typed_value: construct the Arrow struct + // directly (the Iceberg corpus only exercises Decimal128Type). + // --------------------------------------------------------------- + + [Fact] + public void GetDecimal_BackedByDecimal32Array() + { + // struct + decimal expected = 123.45m; + VariantArray array = BuildShreddedColumn( + new Decimal32Type(5, 2), + new Decimal32Array.Builder(new Decimal32Type(5, 2)).Append(expected).Build()); + + Assert.Equal(ShredType.Decimal4, array.GetShredSchema().TypedValueType); + Assert.Equal(expected, array.GetShreddedVariant(0).GetDecimal()); + } + + [Fact] + public void GetDecimal_BackedByDecimal64Array() + { + // struct + decimal expected = 987654321.123456789m; + VariantArray array = BuildShreddedColumn( + new Decimal64Type(18, 9), + new Decimal64Array.Builder(new Decimal64Type(18, 9)).Append(expected).Build()); + + Assert.Equal(ShredType.Decimal8, array.GetShredSchema().TypedValueType); + Assert.Equal(expected, array.GetShreddedVariant(0).GetDecimal()); + } + + [Fact] + public void GetDecimal_BackedByDecimal32Array_MaterializesCorrectly() + { + // End-to-end: via GetLogicalVariantValue. + decimal expected = 42.50m; + VariantArray array = BuildShreddedColumn( + new Decimal32Type(4, 2), + new Decimal32Array.Builder(new Decimal32Type(4, 2)).Append(expected).Build()); + + VariantValue v = array.GetLogicalVariantValue(0); + Assert.Equal(expected, v.AsDecimal()); + } + + [Fact] + public void GetSqlDecimal_BackedByDecimal128Array_ExceedingSystemDecimalRange() + { + // Decimal16 value larger than System.Decimal (max ~7.9228e28). Precision 38, + // scale 0, value 10^38 - 1 fits in SqlDecimal/Decimal128 but overflows decimal. + System.Data.SqlTypes.SqlDecimal expected = + System.Data.SqlTypes.SqlDecimal.Parse("99999999999999999999999999999999999999"); + Decimal128Type type = new Decimal128Type(38, 0); + VariantArray array = BuildShreddedColumn( + type, + new Decimal128Array.Builder(type).Append(expected).Build()); + + Assert.Equal(ShredType.Decimal16, array.GetShredSchema().TypedValueType); + + // Typed accessor: SqlDecimal path preserves full precision. + Assert.Equal(expected, array.GetShreddedVariant(0).GetSqlDecimal()); + + // GetDecimal() overflows System.Decimal for this value. ShreddedVariant + // is a ref struct, so it cannot be captured by the Throws lambda — + // the call must be made on a fresh slot inside the delegate. + Assert.Throws(() => array.GetShreddedVariant(0).GetDecimal()); + + // Materialization must not throw: ReadTypedPrimitive dispatches the + // Decimal16 case through GetSqlDecimal / FromSqlDecimal, so the value + // is retained with SqlDecimal storage inside the VariantValue. + VariantValue v = array.GetLogicalVariantValue(0); + Assert.Equal(expected, v.AsSqlDecimal()); + Assert.Throws(() => v.AsDecimal()); + } + + /// + /// Helper: builds a one-row shredded VariantArray whose typed_value column + /// is (of type ), + /// with empty metadata and null value. + /// + private static VariantArray BuildShreddedColumn(IArrowType typedType, IArrowArray typedArray) + { + byte[] emptyMetadata = new VariantMetadataBuilder().Build(); + BinaryArray metadataArr = new BinaryArray.Builder().Append(emptyMetadata).Build(); + BinaryArray valueArr = new BinaryArray.Builder().AppendNull().Build(); + + StructType storageType = new StructType(new List + { + new Field("metadata", BinaryType.Default, false), + new Field("value", BinaryType.Default, true), + new Field("typed_value", typedType, true), + }); + StructArray storage = new StructArray( + storageType, length: 1, + new IArrowArray[] { metadataArr, valueArr, typedArray }, + ArrowBuffer.Empty, nullCount: 0); + return new VariantArray(storage); + } + } +} diff --git a/test/Apache.Arrow.Operations.Tests/Shredding/VariantShredderTests.cs b/test/Apache.Arrow.Operations.Tests/Shredding/VariantShredderTests.cs new file mode 100644 index 00000000..ae33d1a6 --- /dev/null +++ b/test/Apache.Arrow.Operations.Tests/Shredding/VariantShredderTests.cs @@ -0,0 +1,496 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System; +using System.Collections.Generic; +using Apache.Arrow.Operations.Shredding; +using Apache.Arrow.Scalars.Variant; +using Xunit; + +namespace Apache.Arrow.Operations.Tests.Shredding +{ + public class VariantShredderTests + { + private static ShredResult ShredOne(VariantValue value, ShredSchema schema) + { + (_, IReadOnlyList rows) = VariantShredder.Shred(new[] { value }, schema); + return rows[0]; + } + + // --------------------------------------------------------------- + // Column-level metadata is shared, not per-row framed + // --------------------------------------------------------------- + + [Fact] + public void Shred_EmptyColumn_ProducesEmptyMetadata() + { + (byte[] metadata, IReadOnlyList rows) = VariantShredder.Shred( + System.Array.Empty(), + ShredSchema.Primitive(ShredType.Int32)); + + Assert.NotNull(metadata); + Assert.Empty(rows); + } + + [Fact] + public void Shred_Column_ReturnsSharedMetadata() + { + // Two rows with overlapping field names should produce a single + // metadata dictionary containing all unique names. + List values = new List + { + VariantValue.FromObject(new Dictionary + { + { "name", VariantValue.FromString("Alice") }, + { "age", VariantValue.FromInt32(30) }, + }), + VariantValue.FromObject(new Dictionary + { + { "name", VariantValue.FromString("Bob") }, + { "city", VariantValue.FromString("NYC") }, + }), + }; + + ShredSchema schema = ShredSchema.ForObject(new Dictionary + { + { "name", ShredSchema.Primitive(ShredType.String) }, + }); + + (byte[] metadata, IReadOnlyList rows) = VariantShredder.Shred(values, schema); + + // Metadata dictionary should contain at least "age" and "city" (the residual fields). + VariantMetadata meta = new VariantMetadata(metadata); + HashSet names = new HashSet(); + for (int i = 0; i < meta.DictionarySize; i++) + { + names.Add(meta.GetString(i)); + } + Assert.Contains("age", names); + Assert.Contains("city", names); + + Assert.Equal(2, rows.Count); + } + + // --------------------------------------------------------------- + // Unshredded (ShredType.None) + // --------------------------------------------------------------- + + [Fact] + public void Shred_Unshredded_EncodesAsBinary() + { + VariantValue value = VariantValue.FromInt32(42); + ShredResult result = ShredOne(value, ShredSchema.Unshredded()); + + Assert.NotNull(result.Value); + Assert.Null(result.TypedValue); + Assert.False(result.IsMissing); + } + + // --------------------------------------------------------------- + // Primitive shredding + // --------------------------------------------------------------- + + [Fact] + public void Shred_Boolean_MatchingType() + { + ShredResult result = ShredOne(VariantValue.True, ShredSchema.Primitive(ShredType.Boolean)); + + Assert.Null(result.Value); + Assert.NotNull(result.TypedValue); + Assert.Equal(true, result.TypedValue); + } + + [Fact] + public void Shred_Boolean_False() + { + ShredResult result = ShredOne(VariantValue.False, ShredSchema.Primitive(ShredType.Boolean)); + + Assert.Null(result.Value); + Assert.Equal(false, result.TypedValue); + } + + [Fact] + public void Shred_Int32_MatchingType() + { + ShredResult result = ShredOne(VariantValue.FromInt32(42), ShredSchema.Primitive(ShredType.Int32)); + + Assert.Null(result.Value); + Assert.Equal(42, result.TypedValue); + } + + [Fact] + public void Shred_Int64_MatchingType() + { + ShredResult result = ShredOne(VariantValue.FromInt64(long.MaxValue), ShredSchema.Primitive(ShredType.Int64)); + + Assert.Null(result.Value); + Assert.Equal(long.MaxValue, result.TypedValue); + } + + [Fact] + public void Shred_Float_MatchingType() + { + ShredResult result = ShredOne(VariantValue.FromFloat(3.14f), ShredSchema.Primitive(ShredType.Float)); + + Assert.Null(result.Value); + Assert.Equal(3.14f, result.TypedValue); + } + + [Fact] + public void Shred_Double_MatchingType() + { + ShredResult result = ShredOne(VariantValue.FromDouble(Math.PI), ShredSchema.Primitive(ShredType.Double)); + + Assert.Null(result.Value); + Assert.Equal(Math.PI, result.TypedValue); + } + + [Fact] + public void Shred_String_MatchingType() + { + ShredResult result = ShredOne(VariantValue.FromString("hello"), ShredSchema.Primitive(ShredType.String)); + + Assert.Null(result.Value); + Assert.Equal("hello", result.TypedValue); + } + + [Fact] + public void Shred_Binary_MatchingType() + { + byte[] data = new byte[] { 1, 2, 3 }; + ShredResult result = ShredOne(VariantValue.FromBinary(data), ShredSchema.Primitive(ShredType.Binary)); + + Assert.Null(result.Value); + Assert.Equal(data, result.TypedValue); + } + + [Fact] + public void Shred_Uuid_MatchingType() + { + Guid guid = Guid.NewGuid(); + ShredResult result = ShredOne(VariantValue.FromUuid(guid), ShredSchema.Primitive(ShredType.Uuid)); + + Assert.Null(result.Value); + Assert.Equal(guid, result.TypedValue); + } + + [Fact] + public void Shred_Date_MatchingType() + { + ShredResult result = ShredOne(VariantValue.FromDate(19000), ShredSchema.Primitive(ShredType.Date)); + + Assert.Null(result.Value); + Assert.Equal(19000, result.TypedValue); + } + + [Fact] + public void Shred_Decimal4_MatchingType() + { + ShredResult result = ShredOne(VariantValue.FromDecimal4(99.99m), ShredSchema.Primitive(ShredType.Decimal4)); + + Assert.Null(result.Value); + Assert.Equal(99.99m, result.TypedValue); + } + + [Fact] + public void Shred_TypeMismatch_FallsBackToBinary() + { + // Schema expects Int32, but value is a string. + ShredResult result = ShredOne(VariantValue.FromString("hello"), ShredSchema.Primitive(ShredType.Int32)); + + Assert.NotNull(result.Value); + Assert.Null(result.TypedValue); + } + + [Fact] + public void Shred_NullVariant_FallsBackToBinary() + { + // Variant null doesn't match any primitive shred type. + ShredResult result = ShredOne(VariantValue.Null, ShredSchema.Primitive(ShredType.Int32)); + + Assert.NotNull(result.Value); + Assert.Null(result.TypedValue); + } + + // --------------------------------------------------------------- + // Object shredding + // --------------------------------------------------------------- + + [Fact] + public void Shred_Object_FullyShredded() + { + VariantValue value = VariantValue.FromObject(new Dictionary + { + { "name", VariantValue.FromString("Alice") }, + { "age", VariantValue.FromInt32(30) }, + }); + + ShredSchema schema = ShredSchema.ForObject(new Dictionary + { + { "name", ShredSchema.Primitive(ShredType.String) }, + { "age", ShredSchema.Primitive(ShredType.Int32) }, + }); + + ShredResult result = ShredOne(value, schema); + + // Fully shredded: value is null. + Assert.Null(result.Value); + Assert.NotNull(result.TypedValue); + + ShredObjectResult objectResult = (ShredObjectResult)result.TypedValue; + Assert.Equal(2, objectResult.Fields.Count); + + Assert.Null(objectResult.Fields["name"].Value); + Assert.Equal("Alice", objectResult.Fields["name"].TypedValue); + + Assert.Null(objectResult.Fields["age"].Value); + Assert.Equal(30, objectResult.Fields["age"].TypedValue); + } + + [Fact] + public void Shred_Object_PartiallyShredded() + { + VariantValue value = VariantValue.FromObject(new Dictionary + { + { "name", VariantValue.FromString("Alice") }, + { "age", VariantValue.FromInt32(30) }, + { "extra", VariantValue.True }, + }); + + // Schema only shreds "name" — "age" and "extra" go to residual. + ShredSchema schema = ShredSchema.ForObject(new Dictionary + { + { "name", ShredSchema.Primitive(ShredType.String) }, + }); + + ShredResult result = ShredOne(value, schema); + + // Partially shredded: both value and typed_value are non-null. + Assert.NotNull(result.Value); + Assert.NotNull(result.TypedValue); + + ShredObjectResult objectResult = (ShredObjectResult)result.TypedValue; + Assert.Equal("Alice", objectResult.Fields["name"].TypedValue); + } + + [Fact] + public void Shred_Object_MissingField() + { + VariantValue value = VariantValue.FromObject(new Dictionary + { + { "name", VariantValue.FromString("Alice") }, + }); + + // Schema expects "name" and "age", but "age" is missing. + ShredSchema schema = ShredSchema.ForObject(new Dictionary + { + { "name", ShredSchema.Primitive(ShredType.String) }, + { "age", ShredSchema.Primitive(ShredType.Int32) }, + }); + + ShredResult result = ShredOne(value, schema); + + ShredObjectResult objectResult = (ShredObjectResult)result.TypedValue; + Assert.True(objectResult.Fields["age"].IsMissing); + } + + [Fact] + public void Shred_NotObject_WithObjectSchema_FallsBackToBinary() + { + VariantValue value = VariantValue.FromInt32(42); + ShredSchema schema = ShredSchema.ForObject(new Dictionary + { + { "x", ShredSchema.Primitive(ShredType.Int32) }, + }); + + ShredResult result = ShredOne(value, schema); + + Assert.NotNull(result.Value); + Assert.Null(result.TypedValue); + } + + [Fact] + public void Shred_Object_FieldTypeMismatch() + { + // Field "age" is a string but schema expects Int32. + VariantValue value = VariantValue.FromObject(new Dictionary + { + { "age", VariantValue.FromString("thirty") }, + }); + + ShredSchema schema = ShredSchema.ForObject(new Dictionary + { + { "age", ShredSchema.Primitive(ShredType.Int32) }, + }); + + ShredResult result = ShredOne(value, schema); + ShredObjectResult objectResult = (ShredObjectResult)result.TypedValue; + + // Field falls back to binary within the shredded object. + Assert.NotNull(objectResult.Fields["age"].Value); + Assert.Null(objectResult.Fields["age"].TypedValue); + } + + // --------------------------------------------------------------- + // Array shredding + // --------------------------------------------------------------- + + [Fact] + public void Shred_Array_AllMatchingType() + { + VariantValue value = VariantValue.FromArray( + VariantValue.FromInt32(1), + VariantValue.FromInt32(2), + VariantValue.FromInt32(3)); + + ShredSchema schema = ShredSchema.ForArray(ShredSchema.Primitive(ShredType.Int32)); + + ShredResult result = ShredOne(value, schema); + + Assert.Null(result.Value); + Assert.NotNull(result.TypedValue); + + ShredArrayResult arrayResult = (ShredArrayResult)result.TypedValue; + Assert.Equal(3, arrayResult.Elements.Count); + Assert.Equal(1, arrayResult.Elements[0].TypedValue); + Assert.Equal(2, arrayResult.Elements[1].TypedValue); + Assert.Equal(3, arrayResult.Elements[2].TypedValue); + } + + [Fact] + public void Shred_Array_MixedTypes() + { + VariantValue value = VariantValue.FromArray( + VariantValue.FromInt32(1), + VariantValue.FromString("two"), + VariantValue.FromInt32(3)); + + ShredSchema schema = ShredSchema.ForArray(ShredSchema.Primitive(ShredType.Int32)); + + ShredResult result = ShredOne(value, schema); + ShredArrayResult arrayResult = (ShredArrayResult)result.TypedValue; + + // Element 0: matches, goes to typed. + Assert.Null(arrayResult.Elements[0].Value); + Assert.Equal(1, arrayResult.Elements[0].TypedValue); + + // Element 1: doesn't match, goes to binary. + Assert.NotNull(arrayResult.Elements[1].Value); + Assert.Null(arrayResult.Elements[1].TypedValue); + + // Element 2: matches. + Assert.Null(arrayResult.Elements[2].Value); + Assert.Equal(3, arrayResult.Elements[2].TypedValue); + } + + [Fact] + public void Shred_Array_NullElement_FallsToBinary() + { + // Variant null in an array goes to binary (it doesn't match Int32). + VariantValue value = VariantValue.FromArray( + VariantValue.FromInt32(1), + VariantValue.Null, + VariantValue.FromInt32(3)); + + ShredSchema schema = ShredSchema.ForArray(ShredSchema.Primitive(ShredType.Int32)); + + ShredResult result = ShredOne(value, schema); + ShredArrayResult arrayResult = (ShredArrayResult)result.TypedValue; + + // Null element falls back to binary (not missing — arrays can't have missing). + Assert.NotNull(arrayResult.Elements[1].Value); + Assert.Null(arrayResult.Elements[1].TypedValue); + Assert.False(arrayResult.Elements[1].IsMissing); + } + + [Fact] + public void Shred_NotArray_WithArraySchema_FallsBackToBinary() + { + VariantValue value = VariantValue.FromInt32(42); + ShredSchema schema = ShredSchema.ForArray(ShredSchema.Primitive(ShredType.Int32)); + + ShredResult result = ShredOne(value, schema); + + Assert.NotNull(result.Value); + Assert.Null(result.TypedValue); + } + + // --------------------------------------------------------------- + // Nested shredding + // --------------------------------------------------------------- + + [Fact] + public void Shred_NestedObject() + { + VariantValue value = VariantValue.FromObject(new Dictionary + { + { "user", VariantValue.FromObject(new Dictionary + { + { "name", VariantValue.FromString("Alice") }, + { "age", VariantValue.FromInt32(30) }, + }) + }, + }); + + ShredSchema schema = ShredSchema.ForObject(new Dictionary + { + { "user", ShredSchema.ForObject(new Dictionary + { + { "name", ShredSchema.Primitive(ShredType.String) }, + { "age", ShredSchema.Primitive(ShredType.Int32) }, + }) + }, + }); + + ShredResult result = ShredOne(value, schema); + + Assert.Null(result.Value); + ShredObjectResult outer = (ShredObjectResult)result.TypedValue; + + ShredResult userResult = outer.Fields["user"]; + Assert.Null(userResult.Value); + ShredObjectResult userObj = (ShredObjectResult)userResult.TypedValue; + + Assert.Equal("Alice", userObj.Fields["name"].TypedValue); + Assert.Equal(30, userObj.Fields["age"].TypedValue); + } + + [Fact] + public void Shred_ObjectWithArrayField() + { + VariantValue value = VariantValue.FromObject(new Dictionary + { + { "scores", VariantValue.FromArray( + VariantValue.FromInt32(95), + VariantValue.FromInt32(87)) + }, + }); + + ShredSchema schema = ShredSchema.ForObject(new Dictionary + { + { "scores", ShredSchema.ForArray(ShredSchema.Primitive(ShredType.Int32)) }, + }); + + ShredResult result = ShredOne(value, schema); + + ShredObjectResult obj = (ShredObjectResult)result.TypedValue; + ShredResult scoresResult = obj.Fields["scores"]; + ShredArrayResult arr = (ShredArrayResult)scoresResult.TypedValue; + Assert.Equal(2, arr.Elements.Count); + Assert.Equal(95, arr.Elements[0].TypedValue); + Assert.Equal(87, arr.Elements[1].TypedValue); + } + } +} diff --git a/test/Apache.Arrow.Operations.Tests/Shredding/VariantUnshredderTests.cs b/test/Apache.Arrow.Operations.Tests/Shredding/VariantUnshredderTests.cs new file mode 100644 index 00000000..b1714c01 --- /dev/null +++ b/test/Apache.Arrow.Operations.Tests/Shredding/VariantUnshredderTests.cs @@ -0,0 +1,265 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System; +using System.Collections.Generic; +using Apache.Arrow.Operations.Shredding; +using Apache.Arrow.Scalars.Variant; +using Xunit; + +namespace Apache.Arrow.Operations.Tests.Shredding +{ + public class VariantUnshredderTests + { + // For typed_value-only results the metadata is unused, but the unshredder + // still needs a valid empty metadata span. Build one once. + private static readonly byte[] EmptyMetadata = new VariantMetadataBuilder().Build(); + + // --------------------------------------------------------------- + // Missing values + // --------------------------------------------------------------- + + [Fact] + public void Reconstruct_Missing_ReturnsNull() + { + ShredSchema schema = ShredSchema.Primitive(ShredType.Int32); + VariantValue? result = VariantUnshredder.Reconstruct(ShredResult.Missing, schema, EmptyMetadata); + + Assert.Null(result); + } + + // --------------------------------------------------------------- + // Primitive reconstruction + // --------------------------------------------------------------- + + [Fact] + public void Reconstruct_Boolean() + { + ShredSchema schema = ShredSchema.Primitive(ShredType.Boolean); + ShredResult shredded = new ShredResult(null, true); + + VariantValue? result = VariantUnshredder.Reconstruct(shredded, schema, EmptyMetadata); + + Assert.True(result.HasValue); + Assert.Equal(VariantValue.True, result.Value); + } + + [Fact] + public void Reconstruct_Int32() + { + ShredSchema schema = ShredSchema.Primitive(ShredType.Int32); + ShredResult shredded = new ShredResult(null, 42); + + VariantValue? result = VariantUnshredder.Reconstruct(shredded, schema, EmptyMetadata); + + Assert.True(result.HasValue); + Assert.Equal(VariantValue.FromInt32(42), result.Value); + } + + [Fact] + public void Reconstruct_String() + { + ShredSchema schema = ShredSchema.Primitive(ShredType.String); + ShredResult shredded = new ShredResult(null, "hello"); + + VariantValue? result = VariantUnshredder.Reconstruct(shredded, schema, EmptyMetadata); + + Assert.True(result.HasValue); + Assert.Equal(VariantValue.FromString("hello"), result.Value); + } + + [Fact] + public void Reconstruct_Double() + { + ShredSchema schema = ShredSchema.Primitive(ShredType.Double); + ShredResult shredded = new ShredResult(null, Math.PI); + + VariantValue? result = VariantUnshredder.Reconstruct(shredded, schema, EmptyMetadata); + + Assert.True(result.HasValue); + Assert.Equal(VariantValue.FromDouble(Math.PI), result.Value); + } + + [Fact] + public void Reconstruct_Uuid() + { + Guid guid = Guid.NewGuid(); + ShredSchema schema = ShredSchema.Primitive(ShredType.Uuid); + ShredResult shredded = new ShredResult(null, guid); + + VariantValue? result = VariantUnshredder.Reconstruct(shredded, schema, EmptyMetadata); + + Assert.True(result.HasValue); + Assert.Equal(VariantValue.FromUuid(guid), result.Value); + } + + // --------------------------------------------------------------- + // Object reconstruction + // --------------------------------------------------------------- + + [Fact] + public void Reconstruct_FullyShreddedObject() + { + ShredSchema schema = ShredSchema.ForObject(new Dictionary + { + { "name", ShredSchema.Primitive(ShredType.String) }, + { "age", ShredSchema.Primitive(ShredType.Int32) }, + }); + + ShredObjectResult objectResult = new ShredObjectResult(new Dictionary + { + { "name", new ShredResult(null, "Alice") }, + { "age", new ShredResult(null, 30) }, + }); + + ShredResult shredded = new ShredResult(null, objectResult); + VariantValue? result = VariantUnshredder.Reconstruct(shredded, schema, EmptyMetadata); + + Assert.True(result.HasValue); + Assert.True(result.Value.IsObject); + IReadOnlyDictionary fields = result.Value.AsObject(); + Assert.Equal(2, fields.Count); + Assert.Equal(VariantValue.FromString("Alice"), fields["name"]); + Assert.Equal(VariantValue.FromInt32(30), fields["age"]); + } + + [Fact] + public void Reconstruct_ObjectWithMissingField() + { + ShredSchema schema = ShredSchema.ForObject(new Dictionary + { + { "name", ShredSchema.Primitive(ShredType.String) }, + { "age", ShredSchema.Primitive(ShredType.Int32) }, + }); + + ShredObjectResult objectResult = new ShredObjectResult(new Dictionary + { + { "name", new ShredResult(null, "Alice") }, + { "age", ShredResult.Missing }, + }); + + ShredResult shredded = new ShredResult(null, objectResult); + VariantValue? result = VariantUnshredder.Reconstruct(shredded, schema, EmptyMetadata); + + Assert.True(result.HasValue); + IReadOnlyDictionary fields = result.Value.AsObject(); + Assert.Single(fields); + Assert.Equal(VariantValue.FromString("Alice"), fields["name"]); + } + + [Fact] + public void Reconstruct_NonObjectWithObjectSchema_DecodesFromBinary() + { + // Value was not an object, so it went to binary with typed_value=null. + VariantValue original = VariantValue.FromInt32(42); + + ShredSchema schema = ShredSchema.ForObject(new Dictionary + { + { "x", ShredSchema.Primitive(ShredType.Int32) }, + }); + + (byte[] metadata, IReadOnlyList rows) = VariantShredder.Shred(new[] { original }, schema); + VariantValue? result = VariantUnshredder.Reconstruct(rows[0], schema, metadata); + + Assert.True(result.HasValue); + Assert.Equal(VariantValue.FromInt32(42), result.Value); + } + + // --------------------------------------------------------------- + // Array reconstruction + // --------------------------------------------------------------- + + [Fact] + public void Reconstruct_Array() + { + ShredSchema schema = ShredSchema.ForArray(ShredSchema.Primitive(ShredType.Int32)); + + ShredArrayResult arrayResult = new ShredArrayResult(new List + { + new ShredResult(null, 1), + new ShredResult(null, 2), + new ShredResult(null, 3), + }); + + ShredResult shredded = new ShredResult(null, arrayResult); + VariantValue? result = VariantUnshredder.Reconstruct(shredded, schema, EmptyMetadata); + + Assert.True(result.HasValue); + Assert.True(result.Value.IsArray); + IReadOnlyList elements = result.Value.AsArray(); + Assert.Equal(3, elements.Count); + Assert.Equal(VariantValue.FromInt32(1), elements[0]); + Assert.Equal(VariantValue.FromInt32(2), elements[1]); + Assert.Equal(VariantValue.FromInt32(3), elements[2]); + } + + [Fact] + public void Reconstruct_ArrayWithMixedElements() + { + // Element 1 didn't match the shred type, so it's in binary. + VariantValue original = VariantValue.FromArray( + VariantValue.FromInt32(1), + VariantValue.FromString("two"), + VariantValue.FromInt32(3)); + + ShredSchema schema = ShredSchema.ForArray(ShredSchema.Primitive(ShredType.Int32)); + + (byte[] metadata, IReadOnlyList rows) = VariantShredder.Shred(new[] { original }, schema); + VariantValue? result = VariantUnshredder.Reconstruct(rows[0], schema, metadata); + + Assert.True(result.HasValue); + IReadOnlyList elements = result.Value.AsArray(); + Assert.Equal(3, elements.Count); + Assert.Equal(VariantValue.FromInt32(1), elements[0]); + Assert.Equal(VariantValue.FromString("two"), elements[1]); + Assert.Equal(VariantValue.FromInt32(3), elements[2]); + } + + // --------------------------------------------------------------- + // Nested reconstruction + // --------------------------------------------------------------- + + [Fact] + public void Reconstruct_NestedObject() + { + ShredSchema schema = ShredSchema.ForObject(new Dictionary + { + { "user", ShredSchema.ForObject(new Dictionary + { + { "name", ShredSchema.Primitive(ShredType.String) }, + }) + }, + }); + + ShredObjectResult innerObj = new ShredObjectResult(new Dictionary + { + { "name", new ShredResult(null, "Alice") }, + }); + + ShredObjectResult outerObj = new ShredObjectResult(new Dictionary + { + { "user", new ShredResult(null, innerObj) }, + }); + + ShredResult shredded = new ShredResult(null, outerObj); + VariantValue? result = VariantUnshredder.Reconstruct(shredded, schema, EmptyMetadata); + + Assert.True(result.HasValue); + IReadOnlyDictionary outer = result.Value.AsObject(); + IReadOnlyDictionary inner = outer["user"].AsObject(); + Assert.Equal(VariantValue.FromString("Alice"), inner["name"]); + } + } +} diff --git a/test/Apache.Arrow.Scalars.Tests/VariantValueWriterCopyValueTests.cs b/test/Apache.Arrow.Scalars.Tests/VariantValueWriterCopyValueTests.cs new file mode 100644 index 00000000..399bace4 --- /dev/null +++ b/test/Apache.Arrow.Scalars.Tests/VariantValueWriterCopyValueTests.cs @@ -0,0 +1,413 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System; +using System.Collections.Generic; +using System.Data.SqlTypes; +using Apache.Arrow.Scalars.Variant; +using Xunit; + +namespace Apache.Arrow.Scalars.Tests +{ + /// + /// Tests for and + /// — the transcoder + /// that walks a and re-emits it into a writer + /// with a (potentially different) metadata dictionary. + /// + public class VariantValueWriterCopyValueTests + { + private static readonly VariantBuilder _encoder = new VariantBuilder(); + + /// + /// Encodes a value, then transcodes it through CopyValue into a fresh writer + /// whose metadata is collected via CollectFieldNames. Returns the transcoded + /// VariantValue for equality comparison. + /// + private static VariantValue Transcode(VariantValue original) + { + (byte[] srcMetadata, byte[] srcValue) = _encoder.Encode(original); + VariantReader srcReader = new VariantReader(srcMetadata, srcValue); + + VariantMetadataBuilder dstMetadata = new VariantMetadataBuilder(); + dstMetadata.CollectFieldNames(srcReader); + byte[] dstMetadataBytes = dstMetadata.Build(out int[] idRemap); + + VariantValueWriter writer = new VariantValueWriter(dstMetadata, idRemap); + writer.CopyValue(srcReader); + byte[] dstValue = writer.ToArray(); + + return new VariantReader(dstMetadataBytes, dstValue).ToVariantValue(); + } + + // --------------------------------------------------------------- + // Primitives + // --------------------------------------------------------------- + + [Fact] + public void CopyValue_Null() => + Assert.Equal(VariantValue.Null, Transcode(VariantValue.Null)); + + [Fact] + public void CopyValue_BooleanTrue() => + Assert.Equal(VariantValue.True, Transcode(VariantValue.True)); + + [Fact] + public void CopyValue_BooleanFalse() => + Assert.Equal(VariantValue.False, Transcode(VariantValue.False)); + + [Theory] + [InlineData(sbyte.MinValue)] + [InlineData((sbyte)-1)] + [InlineData((sbyte)0)] + [InlineData(sbyte.MaxValue)] + public void CopyValue_Int8(sbyte v) => + Assert.Equal(VariantValue.FromInt8(v), Transcode(VariantValue.FromInt8(v))); + + [Fact] + public void CopyValue_Int16() => + Assert.Equal(VariantValue.FromInt16(short.MaxValue), Transcode(VariantValue.FromInt16(short.MaxValue))); + + [Fact] + public void CopyValue_Int32() => + Assert.Equal(VariantValue.FromInt32(int.MinValue), Transcode(VariantValue.FromInt32(int.MinValue))); + + [Fact] + public void CopyValue_Int64() => + Assert.Equal(VariantValue.FromInt64(long.MaxValue), Transcode(VariantValue.FromInt64(long.MaxValue))); + + [Fact] + public void CopyValue_Float() => + Assert.Equal(VariantValue.FromFloat(3.14f), Transcode(VariantValue.FromFloat(3.14f))); + + [Fact] + public void CopyValue_Double() => + Assert.Equal(VariantValue.FromDouble(Math.PI), Transcode(VariantValue.FromDouble(Math.PI))); + + [Fact] + public void CopyValue_Decimal4() => + Assert.Equal(VariantValue.FromDecimal4(123.45m), Transcode(VariantValue.FromDecimal4(123.45m))); + + [Fact] + public void CopyValue_Decimal8() + { + // Must fit in 64-bit unscaled (precision ≤ 18). 17 significant digits. + VariantValue v = VariantValue.FromDecimal8(987654321.12345678m); + Assert.Equal(v, Transcode(v)); + } + + [Fact] + public void CopyValue_Decimal16() => + Assert.Equal(VariantValue.FromDecimal16(99999999.99m), Transcode(VariantValue.FromDecimal16(99999999.99m))); + + [Fact] + public void CopyValue_Decimal16_SqlDecimalRange() + { + // Exceeds System.Decimal range — should route through SqlDecimal internally. + SqlDecimal large = SqlDecimal.Parse("12345678901234567890123456789012345678"); + VariantValue original = VariantValue.FromSqlDecimal(large); + VariantValue roundTripped = Transcode(original); + Assert.Equal(original, roundTripped); + } + + [Fact] + public void CopyValue_Date() => + Assert.Equal(VariantValue.FromDate(19000), Transcode(VariantValue.FromDate(19000))); + + [Fact] + public void CopyValue_Timestamp() => + Assert.Equal(VariantValue.FromTimestamp(1640995200000000L), Transcode(VariantValue.FromTimestamp(1640995200000000L))); + + [Fact] + public void CopyValue_TimestampNtz() => + Assert.Equal(VariantValue.FromTimestampNtz(1640995200000000L), Transcode(VariantValue.FromTimestampNtz(1640995200000000L))); + + [Fact] + public void CopyValue_TimeNtz() => + Assert.Equal(VariantValue.FromTimeNtz(123456789L), Transcode(VariantValue.FromTimeNtz(123456789L))); + + [Fact] + public void CopyValue_TimestampTzNanos() => + Assert.Equal(VariantValue.FromTimestampTzNanos(1700000000_123456789L), Transcode(VariantValue.FromTimestampTzNanos(1700000000_123456789L))); + + [Fact] + public void CopyValue_TimestampNtzNanos() => + Assert.Equal(VariantValue.FromTimestampNtzNanos(1700000000_123456789L), Transcode(VariantValue.FromTimestampNtzNanos(1700000000_123456789L))); + + [Fact] + public void CopyValue_ShortString() => + // 5-byte string triggers the short-string encoding (≤ 63 bytes). + Assert.Equal(VariantValue.FromString("hello"), Transcode(VariantValue.FromString("hello"))); + + [Fact] + public void CopyValue_LongString_PrimitiveEncoding() + { + // 64+ bytes forces the long-string primitive path. + string longString = new string('x', 100); + VariantValue v = VariantValue.FromString(longString); + Assert.Equal(v, Transcode(v)); + } + + [Fact] + public void CopyValue_EmptyString() => + Assert.Equal(VariantValue.FromString(""), Transcode(VariantValue.FromString(""))); + + [Fact] + public void CopyValue_Binary() => + Assert.Equal( + VariantValue.FromBinary(new byte[] { 0, 1, 2, 255 }), + Transcode(VariantValue.FromBinary(new byte[] { 0, 1, 2, 255 }))); + + [Fact] + public void CopyValue_Uuid() + { + Guid g = Guid.NewGuid(); + Assert.Equal(VariantValue.FromUuid(g), Transcode(VariantValue.FromUuid(g))); + } + + // --------------------------------------------------------------- + // Containers + // --------------------------------------------------------------- + + [Fact] + public void CopyValue_EmptyObject() => + Assert.Equal( + VariantValue.FromObject(new Dictionary()), + Transcode(VariantValue.FromObject(new Dictionary()))); + + [Fact] + public void CopyValue_EmptyArray() => + Assert.Equal( + VariantValue.FromArray(new List()), + Transcode(VariantValue.FromArray(new List()))); + + [Fact] + public void CopyValue_Object_FlatFields() + { + VariantValue v = VariantValue.FromObject(new Dictionary + { + { "name", VariantValue.FromString("alice") }, + { "age", VariantValue.FromInt32(30) }, + { "active", VariantValue.True }, + }); + Assert.Equal(v, Transcode(v)); + } + + [Fact] + public void CopyValue_ArrayOfPrimitives() + { + VariantValue v = VariantValue.FromArray( + VariantValue.FromInt32(1), + VariantValue.FromInt32(2), + VariantValue.FromInt32(3)); + Assert.Equal(v, Transcode(v)); + } + + [Fact] + public void CopyValue_NestedObjectInArray() + { + VariantValue v = VariantValue.FromArray( + VariantValue.FromObject(new Dictionary + { + { "name", VariantValue.FromString("Alice") }, + }), + VariantValue.FromObject(new Dictionary + { + { "name", VariantValue.FromString("Bob") }, + })); + Assert.Equal(v, Transcode(v)); + } + + [Fact] + public void CopyValue_DeeplyNested() + { + VariantValue v = VariantValue.FromObject(new Dictionary + { + { "users", VariantValue.FromArray( + VariantValue.FromObject(new Dictionary + { + { "id", VariantValue.FromInt32(1) }, + { "tags", VariantValue.FromArray( + VariantValue.FromString("admin"), + VariantValue.FromString("beta")) + }, + }), + VariantValue.FromObject(new Dictionary + { + { "id", VariantValue.FromInt32(2) }, + { "tags", VariantValue.FromArray(VariantValue.FromString("user")) }, + })) + }, + { "count", VariantValue.FromInt32(2) }, + }); + Assert.Equal(v, Transcode(v)); + } + + [Fact] + public void CopyValue_MixedArray() + { + VariantValue v = VariantValue.FromArray( + VariantValue.FromInt32(1), + VariantValue.FromString("two"), + VariantValue.Null, + VariantValue.True, + VariantValue.FromArray(VariantValue.FromInt32(4), VariantValue.FromInt32(5))); + Assert.Equal(v, Transcode(v)); + } + + // --------------------------------------------------------------- + // Field-ID remap: transcoding between distinct metadata dictionaries. + // --------------------------------------------------------------- + + [Fact] + public void CopyValue_RemapsFieldIds_WhenTargetMetadataIsSuperset() + { + // Source has fields {"age", "name"} — sorted metadata assigns IDs based + // on byte order (age < name lexicographically). + VariantValue source = VariantValue.FromObject(new Dictionary + { + { "name", VariantValue.FromString("Alice") }, + { "age", VariantValue.FromInt32(30) }, + }); + + (byte[] srcMetadata, byte[] srcValue) = _encoder.Encode(source); + VariantReader srcReader = new VariantReader(srcMetadata, srcValue); + + // Build target metadata that contains extra names the source doesn't use. + // This forces the field IDs to differ between source and target. + VariantMetadataBuilder dstMetadata = new VariantMetadataBuilder(); + dstMetadata.Add("zzz-decoy-1"); + dstMetadata.Add("aaa-decoy-2"); // sorts before "age" + dstMetadata.CollectFieldNames(srcReader); + dstMetadata.Add("mmm-decoy-3"); + + byte[] dstMetadataBytes = dstMetadata.Build(out int[] idRemap); + VariantValueWriter writer = new VariantValueWriter(dstMetadata, idRemap); + writer.CopyValue(srcReader); + byte[] dstValue = writer.ToArray(); + + // Reading back through the target metadata should yield an equivalent + // VariantValue even though the field IDs are numerically different. + VariantValue reconstructed = new VariantReader(dstMetadataBytes, dstValue).ToVariantValue(); + Assert.Equal(source, reconstructed); + } + + [Fact] + public void CopyValue_ThrowsIfFieldNameNotInTargetMetadata() + { + VariantValue source = VariantValue.FromObject(new Dictionary + { + { "unknown-field", VariantValue.FromInt32(42) }, + }); + + (byte[] srcMetadata, byte[] srcValue) = _encoder.Encode(source); + + // Deliberately skip CollectFieldNames — "unknown-field" is not in dst metadata. + VariantMetadataBuilder dstMetadata = new VariantMetadataBuilder(); + byte[] _ = dstMetadata.Build(out int[] idRemap); + VariantValueWriter writer = new VariantValueWriter(dstMetadata, idRemap); + + // Ref structs (VariantReader) can't be captured by lambdas; reconstruct inside. + Assert.Throws(() => + writer.CopyValue(new VariantReader(srcMetadata, srcValue))); + } + + // --------------------------------------------------------------- + // CollectFieldNames — idempotence + coverage. + // --------------------------------------------------------------- + + [Fact] + public void CollectFieldNames_AccumulatesNamesFromNestedSources() + { + VariantValue v = VariantValue.FromObject(new Dictionary + { + { "outer1", VariantValue.FromObject(new Dictionary + { + { "inner1", VariantValue.FromInt32(1) }, + { "inner2", VariantValue.FromInt32(2) }, + }) + }, + { "outer2", VariantValue.FromArray( + VariantValue.FromObject(new Dictionary + { + { "inner3", VariantValue.FromInt32(3) }, + })) + }, + }); + + (byte[] metadata, byte[] value) = _encoder.Encode(v); + VariantReader reader = new VariantReader(metadata, value); + + VariantMetadataBuilder dst = new VariantMetadataBuilder(); + dst.CollectFieldNames(reader); + + // All five distinct field names should be present. + Assert.Equal(5, dst.Count); + Assert.Equal(0, dst.GetId("outer1")); // insertion order (pre-sort) + // Exact insertion IDs matter less than "every name is addable"; verify via re-add: + int before = dst.Count; + dst.Add("outer1"); // duplicate — no change + Assert.Equal(before, dst.Count); + } + + [Fact] + public void CollectFieldNames_PrimitiveReader_IsNoOp() + { + (byte[] metadata, byte[] value) = _encoder.Encode(VariantValue.FromInt32(42)); + VariantReader reader = new VariantReader(metadata, value); + + VariantMetadataBuilder dst = new VariantMetadataBuilder(); + dst.CollectFieldNames(reader); + Assert.Equal(0, dst.Count); + } + + // --------------------------------------------------------------- + // Merging values from multiple sources into one target. + // --------------------------------------------------------------- + + [Fact] + public void CopyValue_MergesTwoObjectsIntoOneTargetDictionary() + { + VariantValue a = VariantValue.FromObject(new Dictionary + { + { "alpha", VariantValue.FromInt32(1) }, + }); + VariantValue b = VariantValue.FromObject(new Dictionary + { + { "beta", VariantValue.FromInt32(2) }, + }); + + (byte[] aMeta, byte[] aVal) = _encoder.Encode(a); + (byte[] bMeta, byte[] bVal) = _encoder.Encode(b); + VariantReader aReader = new VariantReader(aMeta, aVal); + VariantReader bReader = new VariantReader(bMeta, bVal); + + // Single target metadata that covers both sources. + VariantMetadataBuilder dst = new VariantMetadataBuilder(); + dst.CollectFieldNames(aReader); + dst.CollectFieldNames(bReader); + byte[] dstMeta = dst.Build(out int[] remap); + + // Transcode each into its own value stream (still referencing `dst`). + VariantValueWriter writerA = new VariantValueWriter(dst, remap); + writerA.CopyValue(aReader); + VariantValueWriter writerB = new VariantValueWriter(dst, remap); + writerB.CopyValue(bReader); + + Assert.Equal(a, new VariantReader(dstMeta, writerA.ToArray()).ToVariantValue()); + Assert.Equal(b, new VariantReader(dstMeta, writerB.ToArray()).ToVariantValue()); + } + } +} diff --git a/test/shredded_variant_ipc/case-001.arrow b/test/shredded_variant_ipc/case-001.arrow new file mode 100644 index 0000000000000000000000000000000000000000..65c39caf231fc08b18f9cfba2c74c9f7760f7f07 GIT binary patch literal 3138 zcmeHKL2DCH5PoUXG}NtBE#kpLczEzof_f^!gM|tn3Tja-w57gwUu|GFshdqDl>Qfg zfIq@xe?tF+(4(Ljzwf;_>o!e5@Zuo@v$Hev&CJ_*vu~5N&CQqZ9xF8jomZ7wQltfS zQ`OX*s^bn+sTTGKYBkt*W_Yg@gj#iWrM~_c4wu6{edZ+VLa1BdPgy&+Su^d%HfeTD zYGJzs{%6JK`38Ba^TDyzEuGneZWikmO(68#>?aoLVcRBgPq(&D#nUD#M72A)K3geV zCnhU=+q%QADvJ$Vx4?Dqeef)J z191^}J?Pk21=Y>!`(MK$XPmQ1+Y>PLS8JQE-#&lSa@)Vxj$u;EcqcZ%Kt`PD23X>_ zt{dRe4ytuvpE=&PZj%(ohi6@Qb{tVoO$5x4De!a#C>8+9V=!GsM5d%Z5Vz>AZwB$bvK-7k*lL- ztFDJf-i_-5bYGii-c!VKjjS!?XHK-MvfP)lC#b{E);FCGNkUm|;yHTU1)H#4U(&j0`b literal 0 HcmV?d00001 diff --git a/test/shredded_variant_ipc/case-002.arrow b/test/shredded_variant_ipc/case-002.arrow new file mode 100644 index 0000000000000000000000000000000000000000..381be6a6f148eeb6163264dd8130c4a44f7c89ff GIT binary patch literal 3098 zcmeHKPiqrF6n|;cHq@_=d$`To`v*CLo?#qW!J{V^CUOjVq)k+=)$V5Ibh!{6 z*|d;tnr=Ty41{Am1n^b+9hceIX11_oc8@dHyX+_FGq_+`J>|IVWfQx7+fOpnWC*4B zh>v%U>?rLbPMfCo7{-yj4qm+2AI^|VT*u-4N9Aa=i+>By^s=rQ+)p| z5Os6AJ~zxZYeA1SJbAw;hW;qS6*aa_)Ttq2>R?prcpwdf?@**qj!5q2H(KQV$aK}` zu;rzE%!3cUdFMTaE@Px`9iiuxyF8ckQkLW~6bLH>kTWhH=Dbeh zc(wmXf%!a+*Z5b4@Z)&Jw}$X!>09Gh@pXB{>8$#HJD2sD+cJM0TkId=Sf2=8G>f;2 zz_CZ!?aa#qzY4a`H6c9g7%l(Q$Gq2@d^acGP5c0vd^d3{Cg07>@^<*&zMB`g$5#D+ Fg5OVPEOGz< literal 0 HcmV?d00001 diff --git a/test/shredded_variant_ipc/case-004.arrow b/test/shredded_variant_ipc/case-004.arrow new file mode 100644 index 0000000000000000000000000000000000000000..4626f07a03576e60821958c71e69a670aed9b836 GIT binary patch literal 2298 zcmeHJy>1gh5FRHNb20`6h!6#ZwH1YB0ToyjSOSR-us}o#QP|`*!qV9e=d)21rQ-p3 z018STfd`-ZqJuy zI-9R;snIk|6&5RYaK@+O4^6HUo$GldwR@PG$)EeAQzqvkhn>9Lc3UN8Ur*Cqb(umb zKiVfdGkv!Ac$^wNQm6d4kIzs6%PZgmoxPcwyhGeOlfBnwsBL}j1KVp@7rBf%Ms?2# zYq>5P(2!~uS{OIMG59(d?I2&k@4yed43Gq&Y~6qSDhl=s>rCBkFy$w`{?q3VpLH#i zJA(u&`GEfV4u)l&HjGa{(s^(R3Hb^;=2gzp^<^#k+jkAV7C+39{kb#o4B>>%SFCT# z+F=0ECKQ))1XNs34x>ecfRU`Qw_k$k>*HE{c}ikHOk<3^Bf*9-xrS?2i4T2ZWK17W zt%ZRED2h!ub6-+ZuKJtVw0p^FWb1m(F5|z2UygJ_N0dOUB0QyH1gh5FRHNb20`6h!6#ZwH1YB0ToyjSOSR-us}o#QP|`*!qV9e=d)21rQ-p3 z018STfd`-ZqJuy zI-9R;snIk|6&5RYaK@+O4^6HUo$GldwR@PG$)EeAQzqvkhn>9Lc3UN8Ur*Cqb(umb zKiVfdGkv!Ac$^wNQm6d4kIzs6%PZgmoxPcwyhGeOlfBnwsBL}j1KVp@7rBf%Ms?2# zYq>5P(2!~uS{OIMG59(d?I2&k@4yed43Gq&Y~6qSDhl=s>rCBkFy$w`{?q3VpLH#i zJA(u&`GEfV4u)l&HjGa{(s^(R3Hb^;=2gzp^<^#k+jkAV7C+39{kb#o4B>>%SFCT# z+F=0ECKQ))1XNs34x>ecfRU`Qw_k$k>*HE{c}ikHOk<3^Bf*9-xrS?2i4T2ZWK17W zt%ZRED2h!ub6-+ZuKJtVw0p^FWb1mTTy11CykkOd_T94P3bJ z0bKYHK13hF$8e#J-*-V3%ejri+KCXzY3!x#`WldJ4 zjuRk?OMVk%?D=0Jo<(ZF$CfYh{YO!3_%mgCmW{E=4dDB1kejSC94992jP%fexB0Uu zw!EK5*iTh{a%5CjW#(X-YnP&#&Pv&i z8cowwLAW4-Q@-ebXmXwCT$i5I{z+~of5u6tOwM@@k-Xk+TP5aDPt#m=nL;H$>L+_M zeW^9S<$~?**(EI0@fPq%XU9{McUbezma)fUl#@9S&lhDL7Q z=5brzQ_$b^bl4Wr)x6o~DoxverGsABL)w~4N-p<7K*>^Y~0 p@8CUqa=t#(Dd4SlJns{^4Qsc2PnYlM@;&`O@99_E$DaMq^BYgCZ5IFl literal 0 HcmV?d00001 diff --git a/test/shredded_variant_ipc/case-007.arrow b/test/shredded_variant_ipc/case-007.arrow new file mode 100644 index 0000000000000000000000000000000000000000..8f38ba04283b32f8161e94fa40875e0cdb462f1b GIT binary patch literal 2298 zcmeHJ&1w@-6h4zqYlu;*7IEPsTwHV!f(n9U7pUM)4T^TTy11CyjqCXrJ51}+x#&N=szUcdk3ST zb({cET=JVBW6%E_@hnmUKD2z1?>~xS!=EYBvuun_t^wa=gWP1D;W#mAXQYP)yv?6Q zvE}_dz<#3g<0GTGDl-S;EKzMmMbNq4PYuY!fk~5z>h7Eb$$(nkYB0*xX}c87bXLlC z)M%Qf3c>{uobq}9LzC-7=eqQy_K$Nj`7=&BWpd7Qh~)Kl+bS`KdYa~{%M>d4Q9s$6 z=?ks-E$3`^PcLAhjyHfuI(s`cd51OcOb*_dp|-twjJn=}T;w|TIy!qv*vEC*0EZA= zX<^+4)`7Qy7zH^4z6Wl38U+%#vOWLyt0*`t>@j)wfux`I`p;fGdfv5V+#4id$w$oA z84#K^>d-&!2$zACDoAYgBw`-SubQo!%_#oc$0~Gf4Y9}e7i^{cQfDjHwq^CO02n7U zm~@QlxZGT=<{m<=q{jJv1#~&m%H08eEo;Hr*0J)Q1ReUM3|DPoooG`>#GD1OKkw-m+{d2%&+{7|5^d7} literal 0 HcmV?d00001 diff --git a/test/shredded_variant_ipc/case-008.arrow b/test/shredded_variant_ipc/case-008.arrow new file mode 100644 index 0000000000000000000000000000000000000000..10a237305144d4402c142c8143c3db037b236743 GIT binary patch literal 2298 zcmeHJ&uSAv82>ii)+Me|wTK4~VersH2r3AYTcCnBH7FKpsnhNh1Dm97Hjz^L1|B^4 z03Lh@AEGCpz{l{QuD{>x%x1L>c=D7ld^7v~oo{~M%#dEc|K#O8kplE#MPyw_O}Q#- zvMLe2KtWjPV~C;Ce-8L9(f~iSdXeuxiekfk)23(H7?WHBzsm->$vVSvV$#k?4-Isi zKZ|0^<#~YlMCHdvMs-zY4#ruc+KP^lbG@G$h=&7{CKJ`&Iq{YOy!ec_S7!DrIltYuO3b02rn%~}gj#*{ zPxfZ|!f5X0TzE^S_L8uV>#_k2 zDY(+YxDAfLx4}3Hat3}6-f}Vk5~Q*{|MsgWcvjeB>h6OnKkfCOy?FGzYhm0QBv8pm zoUJo3JR9_3e#Vh5gKH>AX#FIhAM2}EOC1m6-#J#HwHRU!?a#+bwbL?IY;4Q=VE}Qz zzQL43pi|aXVl~g;v62SQ_bV`CU0#bbcS|gYZG@5c#QQKOZMbR+abiq_iWvh6S{MLF zv5i2kNs8sByP2_FM^-}H+qpc!UH>C2s z&DU*NZHXB_H=XRKMKa$OPwaDjAJqS=?EU=gbJk_nUYETO=dTVJ_JcC_8>1_0=vEAg tJ?H7+J9x)_mAO7SKwi7V^FGnPY2%jf>GC~YzNi1^J^h0F*t7q6egi*vZQK9= literal 0 HcmV?d00001 diff --git a/test/shredded_variant_ipc/case-009.arrow b/test/shredded_variant_ipc/case-009.arrow new file mode 100644 index 0000000000000000000000000000000000000000..19922bbf8f5654330a2306074593a410e0adeea9 GIT binary patch literal 2298 zcmeHJ&59F25U$D2t|3NQbrBC9LgQf%A*dioZh;lNsX=jJUDn>6UJOi}v`sOhfis_N>ms+;us{Up@D28!ZR--H-@{pW~tksA1+)r)-lQ4||~&NV&D#u(%p_+2*0P1YHX6O(pEdT5~A z{8l)mAiwoa_D6Ks+3nG?}RG&PiAq(8_xaM!7m|mrFC9 zm1{d{G)+^5#RWS!4y$>DfR)tP-><;*^>HoEJT0*xwsm}YPl63&at&8)Ax`wEBV+o2q82`Y zqu7Qc_arstroWj@yN|5IwzhZOGQJJSZg1XQ){r@J??A+S5rw+=97?lAa?l4H*f%8C zZ}WXyR_ie1bkoaWSS0gp3B)>A&q4ja%E2y-A!b!(?o~P1aO`|{66?Vk&)Y;*)=;e` uB-WhW!*}qGow-<_JRq-q;CY>B->`nm_jLK5F5lDt^PYaeeeBu)Jih^Zrfx3) literal 0 HcmV?d00001 diff --git a/test/shredded_variant_ipc/case-010.arrow b/test/shredded_variant_ipc/case-010.arrow new file mode 100644 index 0000000000000000000000000000000000000000..17f694287a5369e2871658974c311bd2324320b5 GIT binary patch literal 2298 zcmeHJPiqrF6n~p;;}Tb?TEv5gFnH)81Qi6$El|Om8Wan))M|^rEa?vhPhZ^=DL@}rMAn4VlB?2` zRf(_y1!1L+A%;%>CE!}50e)!pBHw=$#kyN_PtUUnCb>Z#1^PqIXH6b&KgdM`B)4~8a9rmDAn?01H=a$du6u1-4D)?8=R z-nJUg(o|t{L59-!eEgBgb)s`!MN)f5xtac%C!MjlGP2L)>~=aTF$a2<=BmdMYW2}R z*`4bPy?K^%zB?xusDR@Q@S)D$%}m}U=H2Q3TQkzOH&1};E$oY2#~h)vmxO&>lXYlF z!Id_~EpPMDU3*+uEfl5B% zZk>W**`N*M(~oo+Tth)ZYbQY)de&F3w!E7`_;-v|*jfxRhxX@vr8;TpE7q5>eSV(T zHyBIkn{Kbf>MVlCN*cW1ufgae)c%<-jK@s zc9yqowIya;ZaCRbi)3F9?bzq~IjH+r+1vS`@U u#Gdo^@H=?NuF6=S9UyO9U>AnQpqBMpeovR*)8+T{|9nrs;yw25f1ckS$8C`S literal 0 HcmV?d00001 diff --git a/test/shredded_variant_ipc/case-011.arrow b/test/shredded_variant_ipc/case-011.arrow new file mode 100644 index 0000000000000000000000000000000000000000..4736beaa10455ac16d39324e0983ffaf1bd2cb59 GIT binary patch literal 2298 zcmeHJPiqrF6n~p;;}WA(E#kpL7(Db4f(nA<7O3D&4T^a;t>z$R%on@B1B1|B^4 z0X+C2{03eMeg!{<2X+1ZW@k35ZNQVKyzpk`y?OuU_hy##`u!&_?}_A~_bVc6LTboW zsmrQFSb>7D)Hfl9PX8(3TBHVkX!RoBf8_bPTXRp((lI8v27a3kGLv?Ok#wKWdDsBYTKK~!1WgPMXqCx(Ai7EKCa0+ zG^F543*#0z0^bJXD##i59eC5p07wwZ_WawgJm+0ukEy#4ru?+mfA-?h^R9()cMwA* zA8@zMz_6^*hVkh~x(u$MAfdIBpanhat5#dw%^>_c#wu(rhL}V9^S)A^wD1+{%h*0Y z&+8kECG>T-S70@d;IWb#@AoS(eO+9OGfzt_h;4+C@5I|MANTOo7UD#o2o=)@6tpk^ zj(iJ_%t>m>P4_aJHjk`?wzqeSZHx`bMQ*-b_K-C)w1LU;}Jns|j8`f|6JzaiJm*3O>^F95H_t>-ld42;lCvdy~ literal 0 HcmV?d00001 diff --git a/test/shredded_variant_ipc/case-012.arrow b/test/shredded_variant_ipc/case-012.arrow new file mode 100644 index 0000000000000000000000000000000000000000..3c507cbf1a2e74979eb9d330f432451f7534a559 GIT binary patch literal 2298 zcmeHJJ#G_06n^V$63ZAALKIO@SR;xQmIV|Ma)qTJQ6K>ph)5v{o9qNEt?jU08%0rW zKtaI)C^!HmhsYtQIROQ*eBb=67ZOWURDRZ*nfE(yzc({-r`vt>{I*CAdcPpDBBYvJ zktJD_DpsH%EcA7Vq0@f~xE85^?;5?x_aAw_=GL@DlXQqlu7cmDy-cUA{xH@_YY_D{ zbQ?eOe9Pt8#C)Q%<0GxwD%A(WG*(STN61;UpJ<4OJ)OiO)!sSrmL9!aRBw=}(`I3s zL}_8$QG;=kC>Uq#V4ctB@9Qjzqbw>isr}+3)21cAF~JhtW96RGTG~ z>Z5Rj;d=>;m0Yryg-O5coi)*|Mu(ZOrok4$eK0N3lVi>zXG{`RdCs%K9#eM*O!;xA`{ddEr)>k{ zUN449KHzMff#F%95A!pQbO~HSK|leufdsLODu?O6(jG7_hC-jaMfnw#F#27 zW(+83U;rHXHUhaPsV?j8X4cITRn zgDQS+9bH*Mx9X7CbDkc)gE!5}=KAOWdFc{Q`$YemiJQNt^Y?WAp8lWr^b77|$NcB{ E4b87`HUIzs literal 0 HcmV?d00001 diff --git a/test/shredded_variant_ipc/case-013.arrow b/test/shredded_variant_ipc/case-013.arrow new file mode 100644 index 0000000000000000000000000000000000000000..dfccfe466d5de4badac4e6cc6e6184355fc7bca4 GIT binary patch literal 2298 zcmeHJJ#G_06n^V$63ZAALKIO@SR;xQmIV|Ma)qTJ(FH6JkwO$E*$G%$+hM&nilW?r zf`S83a040+kwegO0}5dIzS)`eLSl)E%FlW;^M2>;_hv@!bi3Qn?}}ug_X{E`LaNC% zS&~JmVg(AqLSKg%I{l}BYmo~0zSWC-|B+>DZcUq>BtuMc9sD-wr6y_hhmna}1Kl^! zZT!r#Eth8#^QlTtj*V)o#2gNjNHrB5A*cEvHV}_`CXPm`y?g2{J$kvQ-XK+H&B8R% zNnzVngK-=y7-#H|J73J-H>r+vs*6nO;3PGpKlh2pEH2ONcXEDvO%<6VJ&se=W(lSG z=pXG*^rg}9aPUf92;MupL?v}AxT^9Cn zMb@Ap1y>swcfeKfEijIPoP*zix10=s1gUJ#zy8WHo)z|(x_e;Ck2~Eb&mKN)TNw9y z5mfR4XX_jc&kB8*pK+us;1UWFT0aRMK+pQh)#hh22>;Hp2w#gK=Ft9ptQ23G$BK<@ zSU(IPey?vZ@ol7R0uSk@v*=Feh!eYBOvkVm2yJig z#qX`7D@*8B9TI!a)5CZ0wq519J~}{Ny2R5y(Z6Qn=I`nJJ)OU&|K~mZg8SI9|9O4` D^ZIl% literal 0 HcmV?d00001 diff --git a/test/shredded_variant_ipc/case-014.arrow b/test/shredded_variant_ipc/case-014.arrow new file mode 100644 index 0000000000000000000000000000000000000000..0d12fea6d54fa4e9335af7d5d27aa05e7b9be188 GIT binary patch literal 2314 zcmeHJPiqrF6n|;9afwl?7V+RA4n24XL3>exmq5JKfRM+2cc4o8E20VGngEuqp@4WfFnPt1Z-s2Z{MKaL)s>qs<>T+3X zvMM1~ph{5a8xRAh|4`vtWCeWR>P3G1%(8X2rcKY1F($bJewz$ZleCB9$i(fD9vbL2 zer4H~%X1I&u}Y5)jq0ex9E_7lwGlYM-$cAIrf$Ty#RF3wFqz&*mSRR7X11MJBa>l$y!k`@~ZgmuL1nIltYOicDWm<5YE6La9Fb zM|(4UZgf8$>r|bI-aR=-CD(xEp-$dRP1+{r?a9GwGt{;>kAUk9*hQ{l4$;{Q!alCa zIy9upr6$I0a0tE$##NAS;8XCHlL3$*mF@ZW-&w}H!X8t%38ws{+k5)_;j@l~ac>Yo zC8xMsU%>FJ(q{wAIMPLM2?ak}8rUA+$;}w+kn{cdhQESYC-?V6>=%*8)zwkBEt0+7 zW5N4F%J1EL-=@{(xN&*g$$nZS`+DfdUYE~7-M`A-&(Dk{m1WjmmaT&u-hIB+<&yp2 z%=0$T72etgB=(&*=mAEa#rOQ91LT!UJns|zNp%~${I)K?t;=ug|NFLn!n^F+f1p3R C@oI?x literal 0 HcmV?d00001 diff --git a/test/shredded_variant_ipc/case-015.arrow b/test/shredded_variant_ipc/case-015.arrow new file mode 100644 index 0000000000000000000000000000000000000000..f96f50fdf7126bd4463f41dd12348336336c48fe GIT binary patch literal 2314 zcmeHJy>1gh5FW=DW0?a5h!6#ZS*4(`ET92dG?qe#1Xv&a zS(Q3gph{5aLx_RXzprpDvI4$m^&&rhX4$%1)23(17?WHAzextEN!r74Wa9Qn4-Ir1 zzp`x8<++RbNTr7dMs-wT_Qy%2T8fU4Q@s}(i2Z?yqlxNlA9>4wUM^}dO4V_zFwJyQ z*tXSZ8pjI81v})>`&j*U{N?!alCa zIy9upg(k)=a2Yg*~S34w&-eZtuyn2TwZ|#@#^# zm7L&ieF4L>N}mld<4EVhB@`glY=t!%k(o7?D`q_O0cY0f^Q>ZRakRR?>hoABPMgPy zjct}9Cd2_tx89ZLH!aM!~%vhJ#V$Bm158_+L$hYEsSSxLKZVRzuOdS<7 z22^QcAn|=}A(;D;Lb>W*XlVD6YG8YOD>q|oK+gB)8~zGro!s9Ov0p?US64^jwn+AR zj|J}!DZh8~eVbOBDRrCob6! z&OC33uJG1|kl1(Lp!*nk7T@y^4v?2F@w`v;CpB#B^4q%nwl2S||L@!S5%02V|AGDh Dr;lq1 literal 0 HcmV?d00001 diff --git a/test/shredded_variant_ipc/case-016.arrow b/test/shredded_variant_ipc/case-016.arrow new file mode 100644 index 0000000000000000000000000000000000000000..aaa269af7df4cf538be85559dc1b8ef131dc61d8 GIT binary patch literal 2314 zcmeHJ!EO^V5S=t>ScQNJREPtImDK}>C<0DZPDBEU0}>E{s8p)rc3lx^la$SFP*wFG zdIY|ppTUs>2R?w0-~)i=J+Hl4)JEdODI?9=GoBfb-;Ay84+c+P-4n?{AL=4<brX zY09bun1O1IQs0KyaQd?v=OQcMO{*9A@iWiY-JENBmP|0nHSqgnn3|+JnnWhCZ~NG<3~A-OhT~M7bjqcf zPRg}yHO}H#VR69@h4K0LBa`Y#r@D-!_Ks3B{d=D{V{%1gzmv1u>8QvY=qyfEk1160 zqkXhH(-(S2lZ;0`=e~1tfr=R403Yh)U1riQaqmv|-H;OTt>N z$vQNo+LaLBEpPz74Msc2H}EO=p_2iUAe629_uqNWeqo)dyAP)PtUq}E^3jW)g>rWo zK_#c?uPU)MP$z$s~OXuP05jY`Z%kYTP&@vVfBTtl&39x z#rlR7U!K_)8dLZ-UEBh%c?6G_tgz2tgX!zyTC90u;z4`^e0g8I4RhrhuG~Vb=o27g z`haR7KD?h>aO9q(w%l|#v~Bm1dc)TDZn2DS3v#hH@9;m0Dq92h`Sg+0dBA#5 z#q+jN6?SYJ5^K)x;d{7YXGN?t2gn;2cwQ&kx2)gtU0uGbym!lY_5Z!ApK&ky_8;gE DU;k~R literal 0 HcmV?d00001 diff --git a/test/shredded_variant_ipc/case-017.arrow b/test/shredded_variant_ipc/case-017.arrow new file mode 100644 index 0000000000000000000000000000000000000000..9e4e3dd9a5985d388ad89e05f2c3ee25e12996e8 GIT binary patch literal 2314 zcmeHJ!EO^V5S=t>ScQNJREPtImDK}>C<0DZZjnIZfCNM!DwV3ZT~|cfBxSP+s;d4& zkH8o7GkWB}fe+v#_yAye&uecMwJUMrl#yob8PANzZ^qX42ZLv?ABbe2k9CnXA+_YX zG-Xu+%s{n9p>IQMIQ?0TbCDJBrqzr5_?czvZq7A5PbL`T2KYlVOij`qO(GL_$9iO- z3x8$V6BlO_pjl zC&k*f8qeZbVX8QvY=vkbq9#bgg zNBd}Zt}peDCNm!Sg8REh;iEh2clWQBeH225WU*J8~R6A$7W;LH2sZI~<9aOEnoqECQ~ z=>w{T`0##i!I68C+H%|7(6-%2>J3}p`}s1yEy(%ayu)9?%#nM0BJNe>F?BfvD+ zdmPw1q`c=A`-WDVW5(${C;OpF_VH=QI+xEu-M`A-&X4pZm1X8$maT#NeEvl0oU@SC{W9@7?lU{eSQ3IqqfO{sa91 DNu6#Z literal 0 HcmV?d00001 diff --git a/test/shredded_variant_ipc/case-018.arrow b/test/shredded_variant_ipc/case-018.arrow new file mode 100644 index 0000000000000000000000000000000000000000..3b5d531e5634ade3448de7012421cf2dabd96650 GIT binary patch literal 2362 zcmeHJy>1gh5FRIvIT?cjM2LdITBV?{ETD?q8HEn z@CYd>C{m{2QGnz7_IA&Q5=(S68EI~JW@lz+znNWKx7U01{)tEi`noK#Dx|vHmYS?c zfElP#E%XhDRi{6za4xb0-mrR+i=SDx=H^_}(`1A}?ts4}QEHO*U=*6TJ=6mOUGOW* z_FbH}7*AAsd}LHdC8j@0Le)|4>Peic4pS)QNBeMZ zs;}}c^5mIXDjt1|`a0gq(Lp{&onDH(dzDJ=0p}x~e4d!J&3d%Q{ZD3~ZQWh~^9NWL zxr;GC)o%!EyeezZkSe#D_-=y(@Ix@#M9#r8@TQXio*U!K`F0H*M*xwtuA^9UXyY!kdAGlVnIrf1MBEpV z$JFIem@SgM-s8aDA>}p0|OjuwxsLSaWs{-^q15%VV86K=xhWd7Wrqw|>{}?e%+mj&7KNH^FR~ R>-YBmytfwszia=Q{s1+-e9`~_ literal 0 HcmV?d00001 diff --git a/test/shredded_variant_ipc/case-019.arrow b/test/shredded_variant_ipc/case-019.arrow new file mode 100644 index 0000000000000000000000000000000000000000..ebe4ef0fcac0039ef6a2606a204f432c6f4a5195 GIT binary patch literal 2362 zcmeHJy>1gh5FRIvIT?cjM2LdITBV?{ETD?qSqdEzV1kGgqU7S+z)5F2oXGvdF5C>T+9Z zvLXRyph~sSHy~D>{;a~e$P#$N=tVAm=J}eNb1j^vBMfo}{3(qyowf&~NGI)KIMC1q zzw&(F#d(eKL}kZET6I*a`=c~cEk#4fS$L3Wh=;LGqOt1iocNWPR?aIPX6m$6EKS3- zSldy$C)1gy-zY>ayGKx$=U6;RHP5XNs_4!Qz+#} z`)F?(Ugurp$uqT7GI}5P!(_{%gM73)y%Kr%I+ff5&PQSTaiX&}>(L(fKj=Yd>h>I% zKft=kU5o*$enVK}Rat|ERJql}cN-jlAA->)at@w>H=PXd1ffhneE*&2>?78nx~E{u zFT1^0Z=P*+48(hJ1eMItcVEG<3}~|srXT4hxP%a-n#r)nMG$i=XH0)KD@W$(T)Q|=E+{~abWL|Y|mZp+ca8>8K;k&?1y=>&xdxbbNL+9{j2Ql{77F?S!V8K z*%-Lb=O650TMv#rZv$0f$2K6b=IkE6lj~+?W1Trb_Fdr1I?=vv{1)%+;=MgbH_X7B TV7AQSz5PG$?FGQ^nt!H09)f?^ literal 0 HcmV?d00001 diff --git a/test/shredded_variant_ipc/case-020.arrow b/test/shredded_variant_ipc/case-020.arrow new file mode 100644 index 0000000000000000000000000000000000000000..f7107d677e525a5d9a488df749ff5a57a89da9d8 GIT binary patch literal 2426 zcmeHJy>1gh5FRI&V3`92h!6#ZbwY^3ScD2$G!{@pEJs8NQL^!E!bx{Nb3PkI5gm`g zQ&6Yi0VrHjQ1Ad0loSZ_eS5q1p~Mm;MMj$4o!R-H+0(T{%?}PL#e^TLDWC{4l@I_94O{eSj%sVyCQfzV!_%@4joi&p*)N!+~5)IyW z$PcWYee5Seemu}YE68*&&BCA&P!M#k4q^@RFw${23|c!U?k1v?)r$Ig@UAht8mnw} zw-fY7aU9@sfrmo)to=miDpa|eSqcu0b3OdqPds99g=Oc-%I!9SP#>yMoCht2P>PT8 z;oexCr#*`^PWwl&9j8Z8PsQD$Jb9GpVXl(F{UqvTDWW;!X7@Z6dvOCeP}!T2&YSE{ zbJ%;W6J`3ijUL>hkX*-J#S~l;_Iyp&!68(x)Uj>@tH9epjE#H+egHnTG>R!OW#(h@ zdphM@aURHf2qgWY-PzoF)@_-R??oY4@*Z<|3WQ{hGIxN~BU}cSDhB^P;;WiCIE9pB zEL+_tCf;)N#avfB?u^;lT*2AY-|#%cXRgf6WnSot(XAUfEC9;5;UMiHn!;{TySW8d zJ6Yn}cm<@cwQK6m>{%PuntAb!IT_mI9iHjJx>2W!h^Ygp)UgowuGv%iCHQjFzHHz4 zl4Z~I`$2JywFbKI&$s^>$T;cW0sF8BTwQG(GqFXqyGot)dLODNNfy-XVcyL|fi^mpuF;?c{ mdj8E8WZxQ|_lfd^n$f-ZCSQD$FTTnD|2O$3-gDdhbNvDOh>ryT literal 0 HcmV?d00001 diff --git a/test/shredded_variant_ipc/case-021.arrow b/test/shredded_variant_ipc/case-021.arrow new file mode 100644 index 0000000000000000000000000000000000000000..6d2fe77b3e6b2b245d60635359247cf7ef1e6264 GIT binary patch literal 2426 zcmeHJy>1gh5FRHESmrVA?2e$%HQ=u90%8suT9qhRE>cTU_*NGYoo4l@6Jv$&e5 zw7A>x2jeL6ak;=lE_~L0tTPp;Ocj>=gX2t({`M1%8C-7Jd9rf5O+V0wY8+*Ln<13q zqkOP8QRitFaYo7D2)3i-DD11Km!~HSV?D}LJiHr+{WL)|XWZ?`&Id7D7eFFM_qTTgp!lk&YV082ig4^M%RtW)MTka~nmz*5HG-$Q&g69>DH za*SoG_t?Z+jy~_}N|iffb~e{=Hu*Qas^Rmke4BfDt}8~jVdSs?DC3%gbQRI$cJtiL zEx6pt3TNXrkh<2c$va=q+%VUC7iY}L&?fINrVH~%of;yh4xrM&Lf~Apru0kjE8kCun1gUZ5)NzBHHm?E?7T8-h*?$4a3dz#^W7J zyKNEe>{E`lF294kf2EzA+o?+^(~P}L8v(m~`J6m>_6dUkAdBzyP!;yC2a0uP2R*{d nx44>rw+Y#|hUaynJfUuMm*3>&H+lI@{{O$pKQYf8^Uw7MG3<~~ literal 0 HcmV?d00001 diff --git a/test/shredded_variant_ipc/case-022.arrow b/test/shredded_variant_ipc/case-022.arrow new file mode 100644 index 0000000000000000000000000000000000000000..72c41eefe46724b3ae228809928a230948e3880d GIT binary patch literal 2410 zcmeHJy>1gh5FRIvv5Y|hB1A!9oe-jMETBRbjV0)iAXX5OLKHT>bvWtH7tZUTDB>OR zEIa`P1qB5KMG77w1tr3K-`?)|P-2Oa!XwS@&j0Mp?CE;_{_|H4M5cf*%Oa~nXvkHm z$%@o*0#vF6zX`JH_}?o$i!1>jS-!~UU(@NDJ5#0(vlN?L1HR89qq9zuhC1$yRHDKA z34P=I46z>v=IB5N-5}G0Gz)`vKt<3-?Zq18Fw${24!YaNUJ_BuSw$lgd}tS z?O-&C;{d`r5pw18{u6CfsEjH+1$#$EkN=JnPncZp*++7EJMAFULp6y_&}9mx{HP!9 z9;%D1i#+3Wv=85Lx*rWxyqT9LPH|p0QKH92B?s$CG{{oqbS~J=MJ$mUsQrP;-c7XW zut%No;H^%S?d3D{;TGf~*Rj_z373R@UzIg*2$d@>tXsf3@HP;mBj120z$cDIK?Sbt zjQsdLopQD~6XZPvl77+aZ@hfE*|lZgjY6>GgtG{QW`jC+fV3lA29~M@s%A5+QHboB zV>x5mbAY)sPw!_1XIpp6t6F|;E5&VdTd}sSk}dD-hXO~~)|}tGUNaBAUb4hD@ft{5 z=hxPnH)cIp-#S*lEw95|DZ}%eTPxbskuhxml@=BP-}4rX8JEzMo9;z7Z7f-?+8*D} z#aJ7l^Z0!8UxCb%@jbBi^T5~D1gh5FRIvv5Y|hB1A!9ohT?A3nB_+(O7~G1Xw{t3Q^ejHsPc@pE<9MqKJ3M zJMacP0R;sG1qB5K4?#hRFyFVgdp?v{qNK=3v%B*@J2QK_UcdkN`E8LI;M20mst_7- zQEIXxb({c|YQb-UtUCVJ3eO@-z`K?&^6}Sfw&u>1scDvClS{xiS!8t9NzzcqouNuJ zct4?UoS%K{M}avU>7W~AdXQ#e&<>~w+Nix)gWQjF9FBwT_K}xF)N)qQ&;)PWg=ng> zLbe?YCvhA=xFABVeA0iSjS7`fg{NTe(CG2sapDP+%RT!@PH(3jgnD01ViRZt^2905>n2L{*r;T*o&P_TVB=jb6Y8Ho7;-DZIx_!XFn7;!nWr8=JlF;@b!`&D|vNBeCN?fp}a zy)NH_ynm&=p5JLpDAUZnOj`xFe0ZPUdGrp$4>HS nrE@&*6ZHuVYkT%BKKmB8Fb!wlVx#yf{VH(7+iD_0v3dlU7&(HH7FKhsngym25y>qa}z10Z{Wg( z58%Rw@FD6u_!utK>-Wvi&DA#G%2f`WnS0KhzxmG0aJ#+Ula~)fa?pomku@RJ<+`lO ziq!B0szimp0WtFW&lP@)EP=PIUgZ0aJYV=1C?j&P5KHyxnd~#pX~?l1z12La9F5 z$9q$KsW!FTuU5*2W~1wPj4+lk5A#JoK|cw>gz_U18gy^D2`o0x0pY*pCDHCcy- zRJqo~xC5?%?}2d?nCVL@%dEXDTLVAb{@SmWW%h+K>uaDp xtLRz-5_`{6#D3hjUoPe+56o+yVAe0%*R9{;U0uAZi+A<^ysKYuAG`M7=QqoTZqfh% literal 0 HcmV?d00001 diff --git a/test/shredded_variant_ipc/case-025.arrow b/test/shredded_variant_ipc/case-025.arrow new file mode 100644 index 0000000000000000000000000000000000000000..e6aea2c44814fcdd26848449efa7b152af011566 GIT binary patch literal 2306 zcmeHJ&1zFY6h3Kh+Zdx%E#ksO7+iD_0v3dlU7&(HH7FKpsngym25yq}<|a}~-@t_n zAHYQ)!iVTXxc3FRP_N%NKQ~v~fGbxyaAxlLoipE=8E&WBeg5XLNDlh6D6%S~n%tIU zS&}NgK$WP_*C9q;e_r9Y$O3rN>P0Sp&|5D$AMiASova~do?diki{AX8_}!Zguo zVcSuIagrz)XRP3yujcQYOvgIYMJBa>l9|z;>m*|q=Q4+tyx(q9#pY0tlT5W)La9Fb z$9offZ8Z0CC3g4h8WnK72R_p2@z`W7V%{1Zyf=Mqd-DvqK7d{1F6JsayCUr4s;ogn zs@!T|+y+;{55YJJasmDb-tsa45~Q*{|Mn}-c~;nC>NdcXUv#=JUq5}-wlMDXVyNUZ z&ejDOo-6cWe#Vh*f=ejKCj4$dM?32)SL@DZCHgzZ5_~O&HZCf~O0iQHD>k-a{V;&I z-_T&n5zuk9Ijp7`0#>rX^ZgdgSfAJ8%-s?TVq3+?dlGz@lQvwnnK&_~ii#Nnsx&YF zj(i(|T$5Cn`~GIu?K-j;+1~R0IjTW+YxC~1hpdrn2O{RP$nD+jt?c7#DpWxZbHIV; zhUCuMblry4PV=sNIZU(Uke7b!b9o=s{i_`O!pvAwS!V5J**f^)`^Nbtdr>BuSYI98 ySw`3Dkl1^kBKG5v{c<_KcwpZ71*ZL?f6d0t-_`lMI)7LH&%62+_pxLDeSQM~rF=U8 literal 0 HcmV?d00001 diff --git a/test/shredded_variant_ipc/case-026.arrow b/test/shredded_variant_ipc/case-026.arrow new file mode 100644 index 0000000000000000000000000000000000000000..47732a05eed103a15ec2f06fa00314e44871676f GIT binary patch literal 2306 zcmeHJ&1zFY6h3Kh+Zdx%E#ksO7+iD_0=kfrU7&*CLJf+ATI#fSs)3uN-rPh==^MCk z;m(ZIr9N&hMQ2otYt>ZujA{+afvW-Mq-MkZN*S z7G*)ISb-{0p|3-Xy#7;#Ymqtdn$?SZ`;q4>eodR6q(e+{1^g!MWhQO)hp|ao1Kl^! zt^UmOb)V-h=0lYo>>Jfqso5Q-v1%$hLeBI~Vj%AIOcIY&d-E_@di3&9y+NjqnuTei z)55l?2IC}AFwWS)IiJqoH<^xgri)B!=O8nqKle$-EY4*PJ9)pYri#s;9w(V5F>j4_UzxtPy}1uuZ@?~c6>}AxT@vAQPwzcxTNt-{ zF;wy%XX_XY&n5aWKjTOj!6g)A4SuZY0+{uct956y68)WH0lpSP%#r zSU(IPes5?ndPz5XykwO$UxeZu4+u?jRilY1h z1qB~KK}$!$N8}^;3<}_QZ*TW}kXWLk@>sLGGdnXo_hxqGPPe=D{GLb#dcP>LBBZ)p zl_jZ44J%M3DD)x3!0A6#xE5IeZ&ek%TlVpfVu7Tesz0@SF{xC9eYoPlE zy0xEKw&~*B$9$sF<0GTmDlrGcBvMU9L&&M#j}64bo{6K8YVVx*ogS^6S8tH2(`KBjQ%_)9y7T-vOme$?KV|p4)r)rRhuc4 z@}qsUH_;b*$HT!Zb*#?-BYDL?LXpFDf;v~6MB z>qSt>2lUn%7?#VlVSM_LE`v)b$Oi0~(S``V9Ei|$#cuGnnv(g$pZWP1(?1ruEm*OODu?O4I}S~w_!}~;i}EVi9R)C zOdn9CfdO!2+i>KWq)=|Un;F`3WHGR{<^9X44mm%Ycb7F}jy&5FF`q@gE-r_{Y?kcx z9tZXfDet%Gxecq$G2?R0$$pw8`?$1Yoy*@r-M`A-&d>BEm1X8$maTyswyK}+e2E^R z>!-vm<`<$mOQ>21iM3}Ju^xBrDv$Zm0rS!wFs&Eu>(+1nuFl`p`Mdgm-qo+Tj~)Bp F=QqCSdF=oI literal 0 HcmV?d00001 diff --git a/test/shredded_variant_ipc/case-028.arrow b/test/shredded_variant_ipc/case-028.arrow new file mode 100644 index 0000000000000000000000000000000000000000..40668aced8379edb41f3dbf9988998b5ef43da75 GIT binary patch literal 2434 zcmeHJ&1zFY6h3LwHpVDXtGI9xW+5&@z@J47ZUPGK)KIZdOP%IU^}@~H%}u0~f;$mh z`XDY{_y}G22zdY>z=asUZ|=-Zv<=>5omJP1t`3{_+2$V)zw_8<#InbjhQ4OXrmml?m zy|F&aI?FSSR#j2&b=cO?R$8CHw8M_ZeN-Q%3Ws5@s~UIfRW*bT3U>eaRNUROSm?nO zU|+|Bkx6RoS8dpSV>;UQ^)Y&K9deP&*bA774dD!wWfL4i?m`9YHn0G^0mK-|3GgHE zp`%e;fh#*NlV8&*=Zy0~-W?$6r_I*0=a04;wgr1(0G52hT%G`-xj`NJryb!uaIS(< zhu%%_&H=zbUIJCrQm5IeE^PjEvx$~c@B@e^ zKaStP9{dQsD|q!F#@}ytX0zG`JbK6rZ)V^BH}Aa}(rULip5GCf0KP1WtO}tlmt|R& zq=YX(p_uV2Ad8NFTHv?H0W!jEK{z8qs(jXe*Caa7iOxK!y~D%||BMrjm|W`FM{;^QH5HhBJ&F?5WD2?bs2}W( z^?BA=o@un2iu$j@j*d3d`UIvEb~VnU1}IfH4EsIRTyNCW5IQK>z2h@+cg|y>2iJfD z9UqKLQfI&F!_I5d)wZvX(UTjHi(JKC!c?pXXJA#0n6CgBKs6+p>BU}XLDku%; z-2(3dkoo1aP3O8${M*M8bZz~y7wym6N_NuJR;+E+>R|!!ynZ-H7g3!wx3pGM5588i zz`OARNL%OE)|sbeEm+$UR^BnML!Xr4n$D~fZA!?PHh@AE3jrmWY{8gu2^G2SZg$1S zlEtF!FYj*(WzcDS-hK9wc{08S)_oTE-ncw6wOO?1`&w{z2~;Per>-zBU4Uu7s81+c+xgo(f1BrT^Z)-g|G<52+5cX@0mdAq^8f$< literal 0 HcmV?d00001 diff --git a/test/shredded_variant_ipc/case-030.arrow b/test/shredded_variant_ipc/case-030.arrow new file mode 100644 index 0000000000000000000000000000000000000000..38691ab3038ee1bcee618c05163ed60d6661ac42 GIT binary patch literal 2306 zcmeHJy>1gh5FRI&U^y2QAVL%r){26{vH}gr6_!E`SRjys6>V}G#nSnQ^Vukh+)?ld z6qGyw4?)2LQ1Sr01RUSDKlUNZ5)^bEYj$sD=WoB+UAf)qJb8IfqyW9&6xkC}O|Hte zY)KVgph`5?#}Fg0|5V|($OibK)noWs6#M?0YkHauG08RX+pL$Htl1wXCT$LM-#~W& z`$He+0p?4UUyO`usmz=XvqUu%4I$_HBsCDvdL~WAs&#Z3R(iDZUcEuCW{vsMRA=+G zBQ=<$slwul49<8te&6Ie(Yc;SQYRO=8UL9lov=6;Ib`y7-G)lcnVzJ%YO#b;eY8)G zr}|&KyR)T$-I?3~mLr|LnV7su%$wuWbJN$hH;;hpEv$=N$6Q5cuL%3NC;QNlD!X-z zufSFCZ7`04d;z}$A9@)82}0SP|M*oDJS*%mb@#!PpSC;CUOax@vM?U^5~$=O&ek_D z4-#z{pMInra0vzZ3Om+SuF{>!W#29MTI?`K_UHD*J%k-PU$J@X)(!)RHleYUBcS4H zau_Wl1dL>Z=lV66zCNzSm%Ahe#I%Z$_aoRaCf9JqR^mgSDk`QAs8YuO7>aW^GAAjP zoBl?|HjivZwx_&zj%tuyZr<7NVAjaofmq)v3jOdk%*|HGK_777c_F!Tw#ZwzT89}w z?|L~*tK<;>z#E48D6;S6eNp$fa-v4=eQ^IR{>S_J8TYVl|NHy~^e%4W literal 0 HcmV?d00001 diff --git a/test/shredded_variant_ipc/case-031.arrow b/test/shredded_variant_ipc/case-031.arrow new file mode 100644 index 0000000000000000000000000000000000000000..b1ac2901933fb8b93d8030d7202caceb4a11af85 GIT binary patch literal 2322 zcmeHJy>1gh5FRJz#4-j2h!6#ZwH1YB0aavCUH8C>;;L z15i-%2s{7<4?xKS@DPFH`}W5^NKT@m^H{TcGdnXo`_1gi-Cpm>%ex{u=>59LmXPXl zMK+`=HGF|8!9w4F7&!f>3cp3xz&l1Sa(0&I+wPld;XEBr;%zY8bZ#(exe~B#5#$lsxeGtx)Om&z- zAwSwj`}6QZ?`f9CqeFGheed)F6*0aJo`mU}na0xN<^8nc1q>)_1Ttjs? zg|*z0ZD>f9%T0{0z%}qKFxo-BfZu^1IvF4dLYcb%_?74E7uK1&dtk~>yS-;G9zE|E zDEH$CD*1r^`UZw&l{Sn|Khh;|0SWmEJLXl)()MK~_}jM%UxOd!!2Il)cm{t$>np~$ zY3wk7Xyc1ZIRGj)CyUWCg2zbK*xRqc^mTC!zC0x{Af`2pyd&O*F}a3oR*DaOYRH&A zph^=12~gxmaOS?GhTL#Bvtjm<^}y7X_s>!tvfZC|_dA$*a(_?Ev5b68TpkOvGTG}r z7VH_4?X~5;O{2BA@$?KqVWHEY^YChn{Ejz3|ZiHE|m(>jxu4b>q8wYgcbAduH|4+FQZ@ Q8vo<1{fztAHUEWv17@dmjsO4v literal 0 HcmV?d00001 diff --git a/test/shredded_variant_ipc/case-032.arrow b/test/shredded_variant_ipc/case-032.arrow new file mode 100644 index 0000000000000000000000000000000000000000..bf63b6a6327639f617a16707f84b62a20b74af63 GIT binary patch literal 2394 zcmeHJJ#Q015FIB@Fvg$&5u%{5RumuwD^MYe0!vU6tRNzpC~SP2H8h@OIy zAC!WEA_WBnKY^M$!o0V)dp?v{qNMOhv%B*>J2QK_PPhBw^+S;f;ABx`MF@4dCd;xU zHJkvIYQb-StUCVJ3eO@7z{i#^a(XtIthzI0dYmTM20s6!0hQ!l&LmTDCI}}U}vl^ zvM%zBlEFTFN6CKJ*U@HPo-B;bgE;J`2@*LMVf!MM+(f+(b^2~(vKIT&8ukxNtZn}` zQ2X1Ei`>9oL-#KUd%Ys7;1DWTn^?DiHQ*f}#znpZKLejR8buVivUBm{_hiC(;vA6o z2uOOP(|!5s`DWXedM6CPl24ewFFEkC!F;>NkHSledFmiP9|o!`7>QxCpovcR|S21r}y*Vdc2 zW=&XQ-iL3>>o8Z!@GNK6i#9c6OdCL@iG{#-yai*%B{bxgd&vzOOBSoP&+EAuYaMhR zpKtvX$UGU}1M5Evd|h20h1x9I^L;HiLxg1gh5FRIIIGFvZu|!AXk>+-1cK-I8-PLuv-A7OFiWH!C%OWd6s>v0( zAXN!5166{lK7trH{rd{%B1_==MlbT?XHl%WIoIMzHpC!T!LPGkuCrEunCP@Mi2EA4 z`mdtcaB=QoJW=`akydS$>4RaGsHUPJ1_G+y27BO#)4qobhYPY9-*vntjYF<_1%v z3yhw7<=*_xt*;p0hOxs3L>u35$^ojeaV>VU2p&6G;(31trmu@@aOduc4YB6FcyGK7 zV{#2wZ!T{12~jb9K$QkQyr=9ZbCM#t>27Ld^2l;vdVR+(<6DDlbMr2L1+zxx_QW2{ zBaf-8VQMx{_Ii&4dq=Y8Z1gh5FRHNu*`u11eAipTBV?{ETBNhokgetD-n@G6gIgHSUTI`d^U=rbUXqj z1qCe<55NOZP*70PBJm6eIKFRx>_dqqI+~0$w>z`*x8LlpuHEU}e{@r%0KJ(PSrk%D zF31^KkSb=NN;J{O5F@XDTj5+}4t&SzMZW(miX}hinjU8Z3~~|tBJ1WRYxV|-Nt=D$ zGte#nDvDJf=QhSel^^UI)l!++9b}1WC>lb}^-gLa?sZL?3{`9MFsyWG<-NN7Tpcwg zOJki));3jtl%@)cGcq{i1o@W4dA+rb&*RLtLW@m zVILP|2^v!6d>!8na20$FjH4i5!LPxqUIsvdP`2lve-{PM3VTf5EimN=?assXduuHV z<90WJN?zh@eE`Femq%mvIX zrdAhOJ@?AJg`GQJvA%U{hYyH0q2ZJxRO8}0?4}U{b~4BF{uoSOAJ^i}-4h#P&3*CS z1RKWW8m`_<+~`w9#qiF=UvY*UJish=msj(7R=k6(Kd` zvecy}0al<&HP<&GR-OKRg=>)|@I9*+`SCMNSKXR>dKQl{$rbS1c#xR5GaQE|>WuWz zK)3!YO*dVfyO@tva&%}^SH)(39EYl{Xb3scdy#>7FfdU#QQhrhzcZkf^BRm2b<)na zW;)LIw$*4FMGBiGCuGKF;}1=uL!Ib6lG-~;%;fKLqA8QhBKwn^-A-GD=0H!QM0J@$ zAwSxOyEA>RcQhWoQfHiZPR>z@TmzPeI({=XNr#wsCi}0=P}|x(1g>k?7rBZ#KxHoo z>$oDT(2y#ZS{S#$0r(~utsvjPr{GN|10X>tTl4R~)0Dl!8dJ9cru?|qfAZ|X)2@Ya zcMw7)r|7LOU|2S2vks;o=_0s*0;Ialu+B|n!5oVj)1M8=nR)s+HLNX;R#&z9%vbW; zX1-#5TLoX<*%uj0_|{$A46k_vkC!a5$6tWy>*89hd1K;1d;^SpE8d2=au3g~6f61! z$e2E$N(%#t?{f>zJeSmz8}5ZR?OC#1wKcw-ZDVXe&d%o>{t9NEJl_+sFC&ku%Of`{ zlfB+!!Tup-eK$Y1WwjY@T;6iBpUPw(pLVQu@gCIutL*LkOkYw_X6{AV8o1$luN!^( zSX*N~sN#K_s0zEb35j)Q2i?cWyZD-aaDcpYf#-FiJ*i=R7vI*!w{`Jt{eR!q&v=(T H`w#R7Xew%* literal 0 HcmV?d00001 diff --git a/test/shredded_variant_ipc/case-036.arrow b/test/shredded_variant_ipc/case-036.arrow new file mode 100644 index 0000000000000000000000000000000000000000..3ad810534790618cac19acca69d1bb6a88f75a2e GIT binary patch literal 2314 zcmeHJv2GJV5FICuIT?cj1Vlk$tx`}}7NkJPqOlY@B)|d@DMVqD+kmCB9nNQ?D9S&i zpx^^2X!r+83JMBJTK<3v0mplLyXQm6NmNuGX?AyJXJ%*L%$}~>>)n5HQzQkwSrl0o zQeDnVO_n6U3RJ1)`Ub?R)4#27EwTWwcaV^%oG4UY20Y<(RZ^K-C)KU*{M$PJw$8t;|L@!S L0q?SF|AGDhfJ|(< literal 0 HcmV?d00001 diff --git a/test/shredded_variant_ipc/case-037.arrow b/test/shredded_variant_ipc/case-037.arrow new file mode 100644 index 0000000000000000000000000000000000000000..44b39988921e8516dc4939a4cd151a3e9d8e4714 GIT binary patch literal 2626 zcmeHIy>1gh5FRHNV~l|S5lZ30x{AWGNUF%9zy$>j304S};wWrqM<`!s}{sDnsDJPdUd^!)1XncE3zm3oDp#6K^OwgxI5 z?d|%VeiZrGoRPsAPsVTSLthwa>+7UKOe($BTisR$ zJBQ;(%mBc?U4m@`oC6oZi{K`(DT1k|jAjW$ga6j=;gEAfys29UQ{Jl8U%q;_T{SJ; z4+E&=61F1uWfjr0azi*z%d!FuDRY;z^BK(Ct61@wl>Qj7?es z8+@^P=4a2#^SRSoUopNVV}}QbHsf_OyNzr6cOAj?PZn5zGxbg5ntt(Y#D@LO;mL29 z@oB@D+`~066F2(gP%(W#nGzl(-Wv@#GcTzi57V2YU~uG+j!<7+x+bP6__!Vb5%PiT&r5&+$@toiP%FfQM)BIWc zzx&Sa$L6Qt!_Vj6zP`A)`Q>|Yhq!Q$^%Z~)r?&u!_!Dt{KRlDe{%P9SH)(=vyl79# Y8{hfeKfn9ucmMqEZ{WT#|FLU-05UAS*8l(j literal 0 HcmV?d00001 diff --git a/test/shredded_variant_ipc/case-038.arrow b/test/shredded_variant_ipc/case-038.arrow new file mode 100644 index 0000000000000000000000000000000000000000..77ce40712098133a14c2545afd66fb671775de7c GIT binary patch literal 3386 zcmeHK%}!H66h4I(D;5Jvj0;Vg8x|~@MqQDH1tgf@%1ShnAR*JE6fi zF?<9c!AI}`+?iOvZ|wn%pg7mk`=(GxRcnp-aQR%R6j9Y3tj^K zeedT5@I{cEpV^=h#I}7Jhe0i%BIv}lA`9}kW23MaG!8CQ(xH~O>YOCOu$GAiCeCCB z!AU=g0thEWNR`L^cWq)qlbFm?&^k|S?`oW=&*W0i8p-P&)`HL;n|_o84W^LGkNV-! zz+7iNG0&?;{E~DFT34424R>9#YELAb@1ImC$fO$X!yBc7}(`Xt2 z>CVB{pV5fthkZ}pGXUx4_TH;E&tEs(*B^C4u;dqXYdCX^!#vgiwBuQd0OZ{P7uUU+)04^?u!6 zvl`qt*1n9Hcbn=k9%Z;5C)SZRWn@enj6xL?2ex8VMll(eqaqLdjaqTBWWMO?(oDse z7eS}-dFO5-F!N-59cf=Gbrt9LCNJ@MWh|u?>s~B*jjG2Qciq#f|HRXZrlr;2#y|eH z)RD*2(wfh%w^N$rZGgt};;Cu1Q+$)X>M6cSpK|W?N=XWH^Tm>BwL`vx(+qy&<4)VQ zZ2X@6MH|+yf-YM?2Ub9_k9i_CF|WH-x)=LAa2U(HU@)2~%FT<;cJ{W<-u7rXd)xWe Y&))XS-wLz0T?$q7#2N0}NLktW-M+bu!}i64N2 z2ag{87#=)u@B?t*;79Nim}vd|X6G$S|ENJ@V$w<8&b;@2@6XJe z85OxIY-4goMkOx=jLh5?a*^(9Rk5eZY}V^Zr1(4YdMvb1J%@t zvR3tGs$DlsbSqA?9Rv=P10uxWUjGe0ay>V46HiWUKk{4W;{bo_4kD+*6jJ#S@2%~+{j3M(dETfu-?Kyqx*(+2uzwzZHVcP4(z61qhp9!VB?~t& zUqb#-zO5*%H(&HkdG)Z*MEZzO%5ojFfJMUY%WU%*Mb zBs^DFWfBs$%yJU5o?Z-p4+(zl#4ko>Qg- zyVKa_r+It=hWnJ>HO9bPja_=K>HKq;73US(O0wgztr*)9^g4I$yIwpPLOyDBam}20 z(3+7f@7?Dh+FHM6Pr2*d8`f9>U*6w}VLa;a6JTum)eCJ3$e1>0nG!x&hRz-ehB4WS za>M>&6ipl%$(g;}i`Dp!fsbSJH{}RHnImIsyQzadeR^HeYMWCcQeJyqhjEK2CCGT7 zyy9o9o_oi3JMNM^=Pa+C@;Z-st5AQ73beiQimzI^&TqL3ue{>dtzP|ab>Wp){7x5s zuM1C>XI=5Cz9_GD+IRa?`JnzHc<${)UNYH{T;8p1czjo|aStaQGyH2G);*oEu8OGs zQJjh*IG!z@Qr5$?8O3Wlhc^}5YhJEs=huJC*bU!P!}rwhz6yu$so{Gn{l@5irw!jz S=9>NQ-cwclPAr)J;y(cLx%ET< literal 0 HcmV?d00001 diff --git a/test/shredded_variant_ipc/case-040.arrow b/test/shredded_variant_ipc/case-040.arrow new file mode 100644 index 0000000000000000000000000000000000000000..04a1ce4b0964eb878a9c708eda2c6da6b0ed5695 GIT binary patch literal 3114 zcmeHK&1zFY6h3LwG}K$E8pMT*Ft~70g1RcfMGF;N6s$#2XiJ^;PBn0EQg3c5q4ZsR z03X6-AE7THbSWst@0*|7G)+Kp<01oRX3m`N{LDG$-kYp#ZM}H&SY&{4Ru)+jjs>|X z6`7MNb{M5{aUOv!`}3U{?nSW3iai(k`eQJt`8{R&IO|}MTcA%_D>qpq?ZhT&v~_B5 zb^-j)lC$#->xs(yM@BVOW)3=8tdob?tt;r>A4i#vGyk zcR)uv``9yigT2z|9(*vVwmo%*ez*s@$Zf1ubmY9S|CXeN6OPhi9rIgI6?7jo3)+BQ z1X}kz3M*i3ufF>=7_i6Ln>>30BEPw|_44(zS4~^~y;h7Pso|N}009}Xr|Y1g$GC2Q zhAJrQz^+=nEK*(v5zpsi9Yh=RAYUlSf&Jp_&oKtZPdvw{?a#oSJ8YR)+7H+{>}`Ru zo&m%B3}=gYKE$IwhV{y@Mte?|ak@-9U9q}#$b(PjJ3JWJ;=JNzu2z>8Lak(m_um0g z*W0zZVYXQddaPpR`z2uLk21WXCf12MRYXi3j8YvF(lGcAMf&83$$*PU5E2BEb%cFM}ZG($OHS9gLUNye(RQVIpg-B=fg6I z5B47gpkF8P0l(sL>^G*ky)Wm()Phf3NMChstZW)1N&$9cOw+D9T?45^(;KPrx=@vgPSiR|Ybow2| c50dG36i>zUJDO|09R9cO=mp+mYxY0FZwKlxWB>pF literal 0 HcmV?d00001 diff --git a/test/shredded_variant_ipc/case-041.arrow b/test/shredded_variant_ipc/case-041.arrow new file mode 100644 index 0000000000000000000000000000000000000000..0dba3f13c2d95b6addcb0628abf73aa0da822db0 GIT binary patch literal 2866 zcmeHJPiqrF6n|+qX{cMRTEv5gF!bP|1oTuwF9ii}YN=SLp-j6|8`w?iW)lr1caMI6 zehtB+2M>M#KaLoGznPu5CL6(fUU>87{hc>&-p*wI;Nbb|M%gmt+oHK|G+ zGe9Mn%0sY0F5h0^Tm(dRtz2aL=VY>#&#BYHvcl{H2>Wjp z7w0qRmC8n!MzvIG&U$I2b`%Z4XZkcYV9z@yj{2%~eC0JATIF7yZl=aNQ`JzXQ{A!Z z4&qorxnKonJRd(XnT~X(r;*g@C^P-yI*b^d40SwUiZ1@#dOA(@2x=4|!3GxQVPcaQ z?4xY2!Un;N`2Qhb?=AC!Gi)*+XAvKv;(C%LdTdz$4 zV`{O8_i|qg_8@^hQ3TI>`$kcTx#xYJ4|6^1N!WE{Ixgb9{j=PXz2$-Z=;M3Y|MKkb zB6xn5dYfW=)~3jNA8$KLYY_PS&w7c8Mp3G9R}k12q^x_0?x>-2LU3;Xf$Txf8v5S* wkRR;VJb@$V{)TnyyL_{jZ&u-DY>&|-%QvgCe6vdaGnBnqpK#CY+yBbH0nxzpt^fc4 literal 0 HcmV?d00001 diff --git a/test/shredded_variant_ipc/case-042.arrow b/test/shredded_variant_ipc/case-042.arrow new file mode 100644 index 0000000000000000000000000000000000000000..bae03e12ac9c777280a6747ee183a35fcc814f1b GIT binary patch literal 2298 zcmeHJ&uSAv82>iirb|qvY7q|}!r-BY5L6H(w?GANYEUfHQm5T11~y6EY$B!f4SMk4 z19jxq^=;MmWhL9R^UFxzb z5muleEcH!@q0@f}xE85_A6vc1_aAw_>DILASvtlfH^3j#L1xm=GWh4bb7LzC%PXS&R!4o)&N`Lj+k<>rdazLN9XYpd8C>1mRwF1JwGAN}L~ znZ7cbXSo!+cXoveINkyu>-7E9WF2DOnH;_|Lv4HW6u91jUF0U_2%TLM_Hjctp&%K?{2Bue#gfZU*7sIac9oF~l6&pO2ODq(!XQSmyTid0yXO zETOMETY=R)gU3p0yx(uYjCFY}&O9x#Ahr=kz7y}mdbHuGEyRg25q8WNP|(5vIPx6? zvL>l1x82KZ+B&il+TPwT%orPxi`sm<>>>Ba+MbB{BC6t2^>KSDRbKY9$AR~TRNS{> zU6s9m%b9$=t+}gRdD+{kWS^_&pzdGF+jY;9nZ*3G82l!70i z2M-=S`Y}9s=)n)rLl1rgKY^g}_j~(hyUiah2n8VzX5X8c-@KXk-puaX#pUIvFYk$T z(dKd@lfpJGHzX%n$zz0;%Ea;l*o>7w7~@zZ4VpJ{k&PeS?zA0K=k5hN=wu4?HfV%i zP;TzjJ-^&?n;vBEfj`9r#Gar(bi#vO?px49G4osvT!_|UES9@wph=hxd#W%W=s z4Wg`7qZK+wrC7D+2C;6{X?1+xfpUn42;4t^(+l0Y8@lmKPVFG{+86!!9R?T8tez}x zt>o0bEw|%`PK6;P;v>GkzUL0I9vbIGqyBi$106aAA^nDfA!j_sS;<*CKgYJS*YI2$q`2c!kc_gfWHMMj0v)g6uvW_Ub4ZA z@$-t=z4b;NB02um?Vf<>hjGk;XvaGK2%_vdD3NIL1^Ca#MvM((Qkw;g>5G1ApzpxV z;(Y>1mU6{n*f;MCaMcK#A(p;5FIyI5auWRlFb@r`-~STy>fgoK2kuieXZmmczeQ_G z6=8Q4+59n%ufXt}lBY%=SgWy1t~D8d2CL$_B3p@1JhByITZCTM&U4on4~CHEtS-u# z^BFW}B+d8kD-dn%yyi@K>O33fSRPNlzZFA&)Zr6gYzE~EZSshiHfX6L9@vKNJ`#pL z*$Q&oelZHhkBnu^S?)(_JjcODzWGf#hEv8!-`eizU_hVTm!#Up6pNJCUiV?bB8o9G z8Yr*$Ijc9{J&flquia&Noo~S!>N=Nt_^Rb~eJhsNPIw`q94EzsiU8#Pe*YvXV-VWwLH6NAp3dV=#rBRbSG0@!KW^+s_o>l+>aw@O(S2%kpGw{_ XdhcnY`;>Xj{&)AOD!vno=3o3b^FsBw literal 0 HcmV?d00001 diff --git a/test/shredded_variant_ipc/case-044.arrow b/test/shredded_variant_ipc/case-044.arrow new file mode 100644 index 0000000000000000000000000000000000000000..7abb0bbee9f1b0d208b0ac6157e37ee873ee42dc GIT binary patch literal 5754 zcmeHLJx>%t7@mVWIPox`#86mVHWn0wAh95X0uoFpOhlrQ1PSilE_liD;ocrXkc7g5 z!h*uWg2Ix*f`Xqw;ZNWvSlZd3=kv_Ydn~uRfQe!>H{{)!cjkFN=IgzA8K0hh^z5Ee zHI(c%rMeVLhq|HKR7PddLP@3T{W-8{(|?j?Td5Y%u_W*cI~8kW zc2h=5L>Z~lvTtqm*N4_@uRb7oq9`gyt&i4!CwtTR^6pO~-36Bya?xvkKq7vM~AP!ioI=3?ks1e%778)a_mH6fQ(zWK&!U*s_Oa!jAa9r;!AhQ)-lg%RjQXDbrW?K z5wl$}K6a@d^suBl^Z3qyvYD^8lQgr z^ud!6y?XN{2VH9Yd#!c=q8<8i7eqP6<9!hOu7g4WLwy2%U&|;hL!U(EA=

yEJeN zT-Oh+zm@%>AI*qfh1qij+BgUgCIh)N<>&`(uw+2`-k}}_=Dx;7eVKw2;!6=&2e`dJ zXJoE>7>}Fl2jJf)=pTcO=r5ps1l&oWKf``m7c?Vt{R&(kelM|7=gzMem>!Lb&TkTW z#lDU91aNNxoh=vHKMdxS%0uoTkU7_nkH8#X0{0mhp55?CrY|zj@S248F^%2j{sywD zehUIwrG3j|T+RURg8ZnWiT*Y-7Tlv#4RViE3*QvaL6kLm^*QHF;Cbb}%i_zoiC}1t zV|d6iXU0b$D&U!^W?Am+(@{T67vj-qd2Z&+vjDG`v)o z%kqMg`XYJ3N&PS2dDgo#D%H}O&e(;b!#v_w%XxQ^dtEBJ1=m|lQoGIgz`-t14pD-? zemQVDHXsU(qQ4I$F{l^070WIk0IY*5^zG2H&D^S)Tm9w?F`P(*o`TKXDo8i4E+uK^ lR`MT^xh)31W^UEYts3*wW^Q#hNB-Y(t3kYj#`XW4e**beo16du literal 0 HcmV?d00001 diff --git a/test/shredded_variant_ipc/case-045.arrow b/test/shredded_variant_ipc/case-045.arrow new file mode 100644 index 0000000000000000000000000000000000000000..a558a30ca9bedd8bfa0a2542bec700a044574170 GIT binary patch literal 3274 zcmeHKPis>_5Z|Opn$}cWjeiab;o-qU3FxT=4;CtTC|HZ4(3ZN%+qMthOX|x@C6t0k zKSqxp{183%BlJrMISPvL_j|jOrfC9#7Y`XaJ3BkSnVp^e^U~GL%~$W9DAfm@=agDh zqLMlIij z)uxUN#(Kd2%GfyHA&a_Pu3~KSMMgKeRZC&!{lsanaSaHmObC>=}wq< zHKs6>AMyU~ksjwg$WzC$?jksf7h&Op@zDyI&_o97A6KLCQ~Xeb|+{1zE(2oK-x3i>izfQf8ro_C2@= zegK{UufwkdUUND&RzXeA>WAO`KF>JMCdZzFsc)=qzJB}SO)cI2-N46?D&tP9gMlpZ zOjp3-$GmQUhjvh{0sB0~s|DICVB&c{*1(K03w9f&5@5gC_<82Q{D~)x#y$n^!eVp8 zGQME*=%*VDc@7NgGh8j=d5K5o81~B{8sk}C=4mT!eO2mPfnDZgz2#z{i}8Z9*}b}q zAbX{mzhzAkQB8&;cp!5NEa`MwB-^U;PUYT`c8r-+Q{1Ikp;pbY&VaO9k%lDg|& zv{E{cn#rX#>{&C~dFXa-zIo5!%N%9BgxNF4-gT1oh!n&mUGjSDfE6cSI>|MYHmn$| zI{NE6QGlmbirRZ+`S>cBKpI1qv>yT-W8JIM|Eqx zv}iB4-@YUYcKULdr-1j#LmpA zhe5;CO}vkxIj|6O5FVGO;6^3q<~zV|r_{I6ij3d&V;yPTNP<@LpcTihI6d$5=XopP zv=nd~O3-+xSgjSbkJ6s4%Y`Ju#6UeOTsxooO~3Qg?>v4mO~3QF71Qs0qWL)dZ{PVI K-m%s6f0#dv+(lmi literal 0 HcmV?d00001 diff --git a/test/shredded_variant_ipc/case-046.arrow b/test/shredded_variant_ipc/case-046.arrow new file mode 100644 index 0000000000000000000000000000000000000000..41d10fa3b627bb933c442709c1ae35923c68a45e GIT binary patch literal 3970 zcmeHK%}x_h6h192HdqWOQ5G(m2@4jaQR9L%E+El_ot0=LK>|}|kS5cD(_i4VZS zg-e${h6@)gd;k_Kd<37sMC6REZiBliuX?-gC}(e(uk?bGx{_{P5{*kq+AI zsK|t{jmZ@mmAn)%Ld)d3@*>!rmERrVSR@OYH*%4Uubs}69aHD-g*)hE67({xM}Amt z?09}qZn_O0vNyn=zyYzx=ntG|f7f>^PUvs$gq}0!5C}eUYk?1TtL_J0%c-m$sHRSo zwW>EG=Wwp8+H=FMZq;eFgTR4uK!h0F@4w+kuIEN>*OOD*kNnp8I6<4q#hx{i#jVXb zp1+#5l}j21Gm7@jDP@mqCexCLe+SU~I(LFekNHz?iWZw+8wG+zjp$ zNb-~`7Q;Tj)4)|@Y>rsQreC%^$ec;^3&7knxc>M{&})1b;{dq#vCr(U;(w0klqtdP zG`9I^9-n~WJ|%aJF)&wSmz--d{~Tt;dBwKU-SOB~jBN>eojdnkFCGjbAGNyp%$#}9 zGb35vyU#$hwSLW>a@V;xJYxm?^8Qu~<57oCfU!wCFSIEjW7?o)O8CJtbY9`WFeY13 zuG=q0(ZrFFoY~9mSdHH?@NsN@Q;ranIWo4k8#+kolk1XH+nl;0<+ay!7`KR$?KSSK zOP1I9RV}Zb@|y38RjB_B1=@ZPulVgAJXzMd`d9TudBth} zx9!!h_G$jPx089vWJhv&x3=l=U6Iy$eWYP}WjyDUO@N9x?W3rIA~@C(tCIUWV@B~B z&f^Wm_L`R|+WGY!Gj_xG)A0Rt(O2N`{WN?(CEpai@3P_h$y~Gl-TSGE@5G|{@BIUS C_4H~0 literal 0 HcmV?d00001 diff --git a/test/shredded_variant_ipc/case-047.arrow b/test/shredded_variant_ipc/case-047.arrow new file mode 100644 index 0000000000000000000000000000000000000000..4280e661fd0fe2c761d7d6049f8b35ec6e16d54d GIT binary patch literal 1842 zcmeHIyG|QH6upLp7;8a zlRkw%A_YGXIp@q{wFoHaD7e<0nfrds-EpVe{c^A^G6npa5m^>OO_pR{=A?=fpb`}P z5Ma?dpo~OE(45RkpotLE4a#rbO9*s8((MV;5>_yL3>8ano z&Q+pvRd|R_Z*x6-?7w%zq+E0-KOT>d$Ef8K@JeOZH#(2mhj@5)q5I18>n~)iIY!^H zEv{qP16F}+K=$+=xC@*GB1u8Jjm-vZcu%w^ZwE;FYp45d|MOnklyZ_LV95l!e{76^ zI*da*!W?j1B8CVr&~m4-0^K`Ua5>FfZ2c3i7Aad#K9_6;KlA z;s{8ooike}Gi4o&SjEcu@;dZM8AjCDJXa%as)$D$K&62NB~P0$WK2RRt8Su06GvtP z)90oYW37R zlRkw%A_YGXIp@q{wFoHaD7e<0nfrds-EpVe{c^A^G6npa5m^>OO_pR{=A?=fpb`}P z5Ma?dpo~OE(45RkpotLE4a#rbO9*s8((MV;5>_yL3>8ano z&Q+pvRd|R_Z*x6-?7w%zq+E0-KOT>d$Ef8K@JeOZH#(2mhj@5)q5I18>n~)iIY!^H zEv{qP16F}+K=$+=xC@*GB1u8Jjm-vZcu%w^ZwE;FYp45d|MOnklyZ_LV95l!e{76^ zI*da*!W?j1B8CVr&~m4-0^K`Ua5>FfZ2c3i7Aad#K9_6;KlA z;s{8ooike}Gi4o&SjEcu@;dZM8AjCDJXa%as)$D$K&62NB~P0$WK2RRt8Su06GvtP z)90oYW37R zlRkw%A_YGXIp@q{wFoHaD7e<0nfrds-EpVe{c^A^G6npa5m^>OO_pR{=A?=fpb`}P z5Ma?dpo~OE(45RkpotLE4a#rbO9*s8((MV;5>_yL3>8ano z&Q+pvRd|R_Z*x6-?7w%zq+E0-KOT>d$Ef8K@JeOZH#(2mhj@5)q5I18>n~)iIY!^H zEv{qP16F}+K=$+=xC@*GB1u8Jjm-vZcu%w^ZwE;FYp45d|MOnklyZ_LV95l!e{76^ zI*da*!W?j1B8CVr&~m4-0^K`Ua5>FfZ2c3i7Aad#K9_6;KlA z;s{8ooike}Gi4o&SjEcu@;dZM8AjCDJXa%as)$D$K&62NB~P0$WK2RRt8Su06GvtP z)90oYW37RV{SVSoZLIXR9m26Dbiz9^G@j3a3 zl+dT}M^Nws$ec5em7_pGN5Qr3%-r{5?v6X%?)LXjB2&Qi8<7UE|0yAk@zQ~{Z>9p?8l<84Ez$Wj2zw)dwdD0)Grk9-Rz5(wJ`V;Kby#ZdT z;^NGxw#v=XAWzkXq9W)*ANCB$L+Uf3m`?A}%r5t1_STcd`Ul${y z4&%^{Fb^DQRDeCQKl-WL9Nr3Vdd1(nSlfox!vbKQp+Tf0%!~88iu@@J9_q461(d|O z_yMHU&6%r{nX(Q>tYYPS1s(dN3?pi79;=ZyRm7tWpwhsClBZ1=GA1FG_kN;d8%O3M z+vlbeW37R9u{raB_JA~2JN;fvfnP9{}%rc4d99ya+1yvWjAXK}Ze=uUj9x*EI(=#Q~e_YQa! z<(FqVYDSqp?qx}|9#Ii=u8ukyOv*)f^25>SaD-Yu0nb!+KG1p0KE(aw6Wvv&UwZJo1YUj+>$xK-XBbKppzPt{7Qic&VHm}u4n=<0j22iSDLCKRk3>lLU%Bq{_(8Q6M z!1TFp#aJt#ZEVi`HIO+nwuc3OcxCrJbvYCoZ_%Fb`@lOU@b0I<(_X*kgq#l#zDJ(s zKArc!(#&Prp0UDvda-@w&b&i(cMe?-LGkW51r4lg=45@0U7{P#VV!CPRE*vGd*Yum V^EJM|Czy}_?LB$G9JkDWM#!N2(!SULIv0Z^P``d$>F2$rmODW#n{%39u@%e^bH~%U|y`>MdVMZ_fV5XDxf6J z#U7ASJ7=~|X39Dkv5b}T<#p(jGK{FPd9Fs5qAb=1gh5FV3DFxG)gVxbfi)+vRH=mb=jJ4>Vnte_MSp-pZPC)sg0UmQh|J06FJ zpbUKqKLka36gl5FyL&PsDd;FN*6i;5|LoklKN!5(dnvL2Ts#nI3ZWrSWKCA3juW63 zRs0xa2M0@gH0ZG5@58l1s+3jg5pYjYWnM3!| z#fYfGIJ6_I0;dKQV2|`iKP}DSq41`g{@z7x+p32Jz&t~PNJp3#=XV|XQyM(9WSt5q ziF5H0NU57MS0^)N9gJAV%J~X9^hp^;RBdjnkv4V2qYa?e#)6U;9T+kuA(m%;qGOFC zE0OlOo9S=5(Y|tL-Z8qnhOWn;cz4XCi*-v+&c~dC40z!kb{y%~hT7e~ YC*dj6ukrpp!F2p@@5vSBxUc_}H_!nZQUCw| literal 0 HcmV?d00001 diff --git a/test/shredded_variant_ipc/case-054.arrow b/test/shredded_variant_ipc/case-054.arrow new file mode 100644 index 0000000000000000000000000000000000000000..e39c1a61de6c0092d5b956c758210e7f5377a7f4 GIT binary patch literal 1842 zcmeHIyG|QH6upLp7;8ZUER-UJwW4r=R)orNX9;S8MMRMxG_ZqM$;M>8I6}xBpTkE` zLQmn3NWl*vbIv?gMnI8{f@|HGxpUu-9k<%;?Y(azW5Df{$f6J`vLLfEBW0WbrJ%SE zK?d%=QQ}zyYSJ+GA~%oYan+s4Q-iFBO+EwvW@)aoxZ6u~Cq7kOjk^cPkFZnr4tN>m z7iT(ZMwve9Wl6LaQ4n;l4m%p;ajH8>KWhHE^qiDZPAfglqv2Y?8mO$`y=mDBy|mla zxk^;73JcNUMXvkL?RU-@l=JTR`@`Y>5Vr{R59V$EZ8D z`L9@Zfo0$qAZvOB+yqVo5u~8pMrRc|>=X5Qw*e&mqt)K|xwYFgsT`yUu4Dw+d+Q^h z4E<1#FasPYWPm*|Kia997(NL*z3BQbMz?0Rh*MtQFAKH)sAD$QbF{!<;|7viq947z&BEXn*hPz#bFW`$_P$m#;Y{d*{K|$kW^> z?+K>ky}c(7nB$iDuY3ZdCK;Ll literal 0 HcmV?d00001 diff --git a/test/shredded_variant_ipc/case-055.arrow b/test/shredded_variant_ipc/case-055.arrow new file mode 100644 index 0000000000000000000000000000000000000000..5459b9ac81e397897caf5f3d8893b7747975151c GIT binary patch literal 1842 zcmeHIyG|QH6upK89BV;5ER-UJwW4r=RzPLBvy{|;MU)~z(ZCL3B^$$fF+#{)K8KG; z2dUCf_#-Iz0py%BkChQnq@&6%+bryGfiSEQFs;hDL0QnJi%Kiaf zMfv5aj+#-X4|`b>tw$6DovVY626>d~PSTH>->*C;rIgc3Px5HEUa$r#D|oM3wnESC zc6F{2m8-%+ba0vL{!{y%a|Y$SJO2J~xIaWLE5K8got^7EW*y@G;j!*2Q?GmQm~)J} zW1IhkWe->eegv|nzkr{C(?A3%D7VpBgAV&deco*WN$<4UyI;TTHBBl%(*##Cg6y63 z5m1JHs7IIq4iqxL9+)5PR80(X!cH%`zKhYV896Kf#_0=0I>5YGyUU25TyLQ&%M?IL zoQoeoa_yYiJQ*qTpvN*+&X<>=P4Y0JM(3p*sZ)kO>HtbLEJ%4=ha!CvLRoba9U4C} z9hf@TEgNeEwDrxIzXmc!`u6a_A70pfO83GDqOc-qU?9Fx8C;A`Y* z?vruq>i!J4AJ7QS}fMd&engU|lmOYh%Pg25dNkbxXRnV)WkM Z6aSQ%ukrmo!F0T}_v8U{+%o@_{{TCs9K`?t literal 0 HcmV?d00001 diff --git a/test/shredded_variant_ipc/case-056.arrow b/test/shredded_variant_ipc/case-056.arrow new file mode 100644 index 0000000000000000000000000000000000000000..4b76e6498ce942bc4ac2094ccd04887e4dc6193f GIT binary patch literal 1850 zcmeHIy-piJ5FWz;#ySuo7NVfAP81aA1XPwgOQb`BSVRj5WJE|35poZnj$6d+Q=2z~3>EIU!VJRwiXa$~XZ^ zUe5PHdX`@=@hk!}sT;n?^}o@mYR{CZe$vGzpMXD;IMqqm=|;L8o~n)p?*aNl?9{yh zUIpppnGPC3qEEU>6s!hR1f8nmwg&k<*6pYlH1@BYB&L?NiciyEu$qhdD#>N9dd`%d z`|apdMJiRfhv4`!)xD?w+ZRmAMz`|A!QgO!T0R5MRC0cy(~x}#dnZ41N11-zL&lmz z^c~yG5|&+H8Mp{!Pk#Y7f#X0VDQLH`S%wYoiT31e07>sOTVKC@*=?9oj^YR`8AA6q zi{Vj+acD=F0QMCs!0wqJ{Zvg3Q^K2`v-d8>wr2FO0GOw15NQwdlKGuS{**coRhg#( zO5$7`04dGpEUS~5vJOTpW957~9r~mUBWi43s*yHj#G?(MRKtRjM=LO7OoA^9cA|X~ zN5(zV=e10XwE{Ye&6)oVWR8sOVA>sC*j-O;4!Oo_wBx%z@Qw+*`yx2n>DR2Vn2YOC z(XN;``EQ|_OHrS(7Vj4RY;h^t=< c#_r=?aZj50DnH&8Ge7U^U3tJ9H_d literal 0 HcmV?d00001 diff --git a/test/shredded_variant_ipc/case-057.arrow b/test/shredded_variant_ipc/case-057.arrow new file mode 100644 index 0000000000000000000000000000000000000000..196dbe69de6d269f6adee98b3a10bdd987258979 GIT binary patch literal 1850 zcmeHIyG|QH6upLpIM#xAScrncT2Z7xD?(+tvjjC@5v2fx26j+ZvN2gNMhLm%bJFnx zl+aW77mz6U0py%BJF{#QDCj7-)}5LAe$3r*v(?)C`CViLxE&Lj5kf_#Wl|=jj1!>b z<$NEcXZiIK&mu6By5Wo5JdZ|Id!|hFlP)%y1)e8ys*|wOjdVLaQ5_B56ZD7Jse1>! z4AP5J9W;VOA9a%`SP7^II#q{l4e~hF?Wh+tb}yYIrk1sePtstpl8gE($z^YP&XoT5 z+tI0tRH||h!Qn-!d;j`xpD`&L-OBd|gZ%+&nFpS#)4P?a5mMlHO{zwtsBwG)yT6aRinOp?jaj z@TkK$v?ELa`wA6c_soxeswRgo!keD4_b$e^X7sQCn5SzHX%F*~`JF@llsXSpnWF+q z;#}+jDb40AtCN|s4n{0v<$O6E`lJjaYHVJskv3(-qYa=`!-A4W-(bj?1YZ{HMEfR= zjC-cftC<*U1#}jhGyez392wieS9f@2cRjT^-wJ0|e%i{NOdU$equF0MyK zyJFttzlUZnMSaFvTzJ^L#<$90C9uETnZJ)NPon!iDBdBbp^kOgoHAcS4m4oZI)3Rp exfNsg`L4Jp&3u)g?+Pa4qrEFnnB%7T&-@E{Kp#i| literal 0 HcmV?d00001 diff --git a/test/shredded_variant_ipc/case-058.arrow b/test/shredded_variant_ipc/case-058.arrow new file mode 100644 index 0000000000000000000000000000000000000000..4dda132df22e0c2cd1644c1e059f286f496e3a7f GIT binary patch literal 1842 zcmeHIyG|QH6uk}$IM#x|SWyZJXGP%xtr9BBoh7JAED!~tXkZ7il8woFF|v?5dVVAy z5v2-!3V%cjejqaE%wy#!P^3%2weHN^x$no0+nvtt!3U8k;KwVG6(Lk)Sr%ko$~XZ^ zL2(~~4BUOQ#Ip$0q-pL&zWts~Ywk>*8fJZLvI@M;(p+b8ub=2{e4%<8cMp)CV5jUo z@FvPfmpW=inLh1jNwgVJ5Ol6ix*FtJs=LV`Y8~EqPD&}Km0sl0c(Y&)RaWqxwQPm{ zw%gOWN>r{23(?6a*MmRpcdrsHN6FH1LuGUQc!NAvjH9UiTb?T0+RmN?tI$c*=w0pKBoz;WCGcr z)<-}Y`k@|S9ynCU0DEA5v{N%NEDAfl;`%N|w{GOH02rq)5a|H(V(nHDKe^sQO{x?? zN}P)$Ah~wVY@UpidC+4SE9cA0&?b2pQKR!zj?^i`A9VnwIu@inZ9tJe38B1k6CD~q zG8dRSH!K@#1+?|eng0r8jP&i{wLd(u`Mo$_At?5aQ_#e^VNTY@go6xt?+i99>DG$T bdwEa%Q)a%#m-htI@xQ$%518Y&`LBEg_{|## literal 0 HcmV?d00001 diff --git a/test/shredded_variant_ipc/case-059.arrow b/test/shredded_variant_ipc/case-059.arrow new file mode 100644 index 0000000000000000000000000000000000000000..f22b0317da25d53cdc9d3c5246df3ba04f0a4d95 GIT binary patch literal 1842 zcmeHIyG|QH6ul0MW2{X?h!s&#m=%Q!v`SK0?kt5m!2(eLipK09RXI zNpKfzAf zU%=}qzq-&-E6ViAAWNd{h=QPV)$M7Jr>X8G!>Dz5?Kvr>oK||CN8{~+HBwo@TWHw| z{cE?cbCsxE6&9lIRj!AB+wWa6DCgbrkH_QVF>-kcyinPvOP$B8Lp(e=(|u*?bqgMI zj!}1P&o;2^0V}|DAZz*s_zt)PM391V8=Woauus(I-CH2(5ADwG{?1;@r1CLMa3vGS z{%Fqw>2+P2cLI&6a^P`=biQ%cR)2puUVsz_94hw*B`T~&-FfZ2b8saC{Td2tz z1&|Ww;s{8toim#!BV``+Si#Eq@-nnZ9!AvY%*&BF75JkLpj5|#l&1|S(kCI5S8k$1 z<42YPQ|E?dW37U=zB%*XfQ*s8J*@b{oZZ*d#ZX8rqW!(E1A9zh?`Of&UcT;_?41W+ zBTsXmjr)IT#xkohyVZp literal 0 HcmV?d00001 diff --git a/test/shredded_variant_ipc/case-060.arrow b/test/shredded_variant_ipc/case-060.arrow new file mode 100644 index 0000000000000000000000000000000000000000..65a8ddcc5d4d74c04a49113e2f964ac8809f6153 GIT binary patch literal 1850 zcmeHIyG|QH6upK89BV;5tSAMAwW6S~RzPLB!V=U;EFua(XkZ7il8s@#7$M}2pCI3m zkDw%d3V%cjejswrnVnfS3KVn{TzxYcfd-dz_N10E(s=7ms^IhmFzDdPkv zc{$$)=~;fG#Ip#@q+$3X_kYIYsy$Pt21yT_EC8>PIMqqm?M1p1o~f<|?-BYV?9}}U zyb03la~(8;L?8E(D5wWi1f8m*js|%W>rT`UntL}+5>v}s#b;?StmmSEN^;qYo-?KY z{dRS#B9*G#LvVDR>i)m}JC{t#Mz`{V;qYLHT9$z4D!I7SX~;f={o_;JRiz&j@J?u+1Pr(d(eVlJ*n zMZ03&rkdsiMY$1U@p`4@?K9aI1S literal 0 HcmV?d00001 diff --git a/test/shredded_variant_ipc/case-061.arrow b/test/shredded_variant_ipc/case-061.arrow new file mode 100644 index 0000000000000000000000000000000000000000..f3fc282cdee2c6c1addbeb283910f9924dcd0063 GIT binary patch literal 1850 zcmeHIJx&`z6n=(K?oI@msy#SGERV! zm-Bs)p5-@6Jd40g8ip@&{VZJ?aA8&lHP8$zkUC@(=?^*#}QaEhVES! z!=n!4(2g(z94b_R-7`P>shS)<2yc49-n$svn$g1oV4ki)q&>__=64bKQ|dfaWswRf ziF5H2NNF}_S)I(3bueNXE9cAU&?jXWQDgI5jkGBv9&G@n8Wxm1`3ytGB>1vyC)zi0 zWZE-*UeCl>E1eVHoevLDM{pHU5eRO#i-SM`Q;0z96zDgo>=nvMfm%CqOAE z?n97)yKk0w7J-^H&ArI|@7b*C&g7|K*2gAqfZwt-*IC@_C%PM7s-DK(1LUXJDfWgbl$1#76Xg7>UtEA+SB zp3YUGa#dJ}PDi;OJhtDxVNlMy;~!5Z#}nl87I>wy>l>ZNtV29FyU;yl>h%je<{YE$ z*j6^N901F}cR<$k7Ptpo1R_X5xsA>ibl4~A^KKVN`g6Op|K-C$%cOFWCb*I*WPe&8 z0cGfidW0q5P$2{Cf%(x+)x_{h*y%ObcQLv(BZmdRIDLUg2bdRYcOCJQ>n&7eodQUS zbMX~OuAMWRCnIGZ^jOBq`SLQfNghVj=scApb;|HZ9YCpu1u4(!P^3>nC>w5~L*qvl z15@X^Wn-;?w!S&@cRz7Fg$fxVvxPkZ^AW3qQ1e2qNK zeLn7gr5Ve-K7BRbf20{xSMH2EM0J-@^$-+$$0=xH-7+U@W6D7WY&(N>OS-jU^j_W* Y|CE`p@#Q_ibo_7c$phxNZT>6&0Agkt*#H0l literal 0 HcmV?d00001 diff --git a/test/shredded_variant_ipc/case-063.arrow b/test/shredded_variant_ipc/case-063.arrow new file mode 100644 index 0000000000000000000000000000000000000000..f8a048bf91c9e424b9f52f75bdbe4eccf78f45b7 GIT binary patch literal 1842 zcmeHIyJ{Oj6uq+7jw05^c7#chLM$#+h{bUw)CrP0iEv1f2s8Fh@PcGHtCfr~>hd)q zACWTYQ|Kc?@DCW}oOvuVmI`+%bj6*SJNNzAQM=Q5^Wmk)4DfSJWJd@U*_KV&kTOny zQc&E7AOm;bEb%M?HEEiAkso)nS=F7%Q{!xaO?H7-S(@uC?hg{(i!W4Pq{NAqD-F+vLtFm6a<~C)1C(TIn}*n7_~m$cuq#-C8kv atM|k|W#(&Ky(gHC|Lr}w#T>WIf8{@lh#cPl literal 0 HcmV?d00001 diff --git a/test/shredded_variant_ipc/case-064.arrow b/test/shredded_variant_ipc/case-064.arrow new file mode 100644 index 0000000000000000000000000000000000000000..e6f00e9ccf72d7a28d4684da7990a69d08557ab8 GIT binary patch literal 1850 zcmeHIy-piJ5FW!}9P1K+V?`+_tP_e-pp&7p+*u+u!6HgQP_%Fx8`}TG(7sW2prC_Yt-TD97xpk-8J$k<{vH*Nr6WJ0%Rh~&%Hl%_R zpycIzAEambtrE{7Fq4+yi`@TQEb8`5nHnbpY_bi!PU2K2VSf^PUFzDb~Gc7_?7soFt}}wTds(V7ix!#wy8WD?MjQ zfBWt0R7EOPxrg9rj~MJ>;PS1S23(rL&(gu{!^y01*Xen7^W zL-ZZn(;X})zzXmskUhNx9s<{aNK(*lW3vkz-V^P~I{=b?+vy&^dvnq@rF@Jduw(|^ z?^z6wI*da*!Uk}xPyu$&{OG4{a@Z8!^p?GMF}4k(hXueqU4uw_n3v3N4f#{*Jk+H| z1(d|O_yD9do3pG=X39Dkv4WNJ<#gziGK{FPSym%$Du_oLK&gQRB`;pXkTD6qys#7P zn>e!WnLal&G1e;REH-EU4#*rC+rbleShBmG+8lC?RkY)~KJbnSy!#?J+UYl}u$YVM zQPHlLH~IfUGnb-1V>R!;&S%H3%JfBGf4MV%A6+h^`#vb%A*Z2*b=RCSUo#FgplKcd g?mM|vWB2&3xF^kgm5=WVCgXp5SLT@Gj``301>!{=8~^|S literal 0 HcmV?d00001 diff --git a/test/shredded_variant_ipc/case-065.arrow b/test/shredded_variant_ipc/case-065.arrow new file mode 100644 index 0000000000000000000000000000000000000000..e00dcfee595c471f55e24c315c7012bc77b51120 GIT binary patch literal 1850 zcmeHIyG|QH6uk}$IMycOVMQq@tQAEHv~nT}%bg`sMi2``0Vo>SiL#Q7$$BxeEO-0~ zZ7RNiGV~Pwh!p%l;+!)(vn&FNbSb#jotgW7%-wOT-QL`NFER!Ed?B(Rgo?bDIhmC* zPJoh^^L>z>T2+Ap+CV+ z-D}_|NQWmnXaLpRI7Elp%st!9EaKcQLj#qlX2+JY9oGdzhEZ?-KH-)Oo1N z5*1Jq=VBj7X*OqBoy?SVFk%@i=gaBPCuJB>WAj*zv?(JVZ2+Yj7L+`F2Sdgr`0~b1 zv~S|bjA!~>&%{_OptIPV`5!>$$k+~Ey2B&8>#5Bl*LaF{eAfrwF@bkq1V=mlniUpv zaXl*974s(lUufo1)Mu>v)n#w})5RYSD}nvx&is9Jc@Ew8LGcbb4GpZT=9KxGaG(Ko i>-etkWHf97A!KptlR literal 0 HcmV?d00001 diff --git a/test/shredded_variant_ipc/case-066.arrow b/test/shredded_variant_ipc/case-066.arrow new file mode 100644 index 0000000000000000000000000000000000000000..148062486f563127b059e41805ff9c93736676fc GIT binary patch literal 1850 zcmeHIy-piJ5FUra80!)Nu_6iz>x3c-bRtm-%blfACs;-l1VszCC@0yNoDU;}-0?V6 zJOL&26n=;leu2pOzP;V!P)w071!K+b&i~KOty}H(m%R@nQ^4ITktHEi<&BhOUMe^N zN?y+QL3)^?!okrG-BYGtzaV4H zA^MK(^%|BAumW5KvZq(T&%ik#k`%Pt*sQ~b_e6X0J_1R9ZMAoHw>wQ!%J(<|OUBTB z$YOZZVI0~K=7B?n3b1?TM?W=_!-DXpm+ZZZv8@|DECA-|8bsQ|ykve?kUyo)Lrqqw zfRZ>D-++{6bC%V~Oj!pbRGM`5###lP#pcXk1DPXZJ6Lpw8N2JL%^}x#j&^+42i`G(cV7fYJN>#97ISeu zD%utECjVb(=2FyWtWUQ$lkv{`GJO%)U+&D`N0-a!z7L9b$Z2R`T{ox9*O&th*s_iv g`%Z4v*uA_f?nyIW<;%N*$@t&il?mp!W&Sh&0Pc+)AOHXW literal 0 HcmV?d00001 diff --git a/test/shredded_variant_ipc/case-067.arrow b/test/shredded_variant_ipc/case-067.arrow new file mode 100644 index 0000000000000000000000000000000000000000..ca9d91af8604259ec78142ab8b3cfe417067df76 GIT binary patch literal 1850 zcmeHIy-piJ5FWz;j&&dcR+NImI#E!d6Ceu9oh4F6Bo>H*Ahd9cILXFvJ{%$BjyIuA z#S>75K7}751uqcH_wDUoE(#QM6pS^yJO4jBw{Estn>(u_Bf!rwky#;BWJV@sLdrM+ zN?y+QL3);7FYzn_GpQTC$nBresA|uYseaPMChvjYlQ`8$*y%>P9iFI;2JasFL+sSO z16~H{#igSG~F6zg`>3mUtZP7+hgTE!=6Fj&q-eU;?0XFX?1 z|NHIeR7EOPxrgB3BGtWr{kPAVl#OoXdxOE=0JY2mPgQbuuG5fx2z!Uex}!|LenZBZ zL-ZZn^a7SGU>W!k$evySH-O_nBq?aOu~~!-?}_&0tpQ1YX|}#@f7)u8QugBrEEz)g zR~Eyg4&%^{FahijBU;6VF55t*C5g!<|Xqxhx{pZ9;z}& z1(d|O_y(jjo3pG=X39Dkv5b}T<#gziGK{FPd8$U*lo5|MfKm+$N**o2kTD6qe6SPk zn>aG=nLe*%VyqR=S!~Yy4jUqYz`HMkqn&=u3X8e8 z9u@71d6WMVnzgYenG#t$@mMXDO*kETR;EqJbU6N;Zb|;s_yk`J8-2 z%23dy@JFQJ2O{U3d8`%zqD{fI?#$fxWA2W-z25GZZIKz^*PO_*5bE+;s- zNBPycjyh4MkB3X|elm(W``2EQQp;JTXL&T)EJS0K6|yHiTcwA7 z2Rc`Y%2nYZI=agB=)V8{C6jW|o&0bzIh>%DRp7bGE-rN*vk&p;_*4&+>DM2~SaXcN zV|%rZWe->bz5}wS-+}Le^FSmiXt%N1fDP}7_T=pVNq_A2K7IbM*D4epilJ8N8 zbuVx7%XK_QGncYHV>N@fxBH8>ue4|0A-Y>d*F#XeJ5E6x>xMa5A5#u8V9PmdS<s-sSn>BC``L>mzmLFekAuR$KAx}S`q&ZjFcNvY+m(vv)zY!srg$_m+&o~_c~ zeg`^NiON;sAv(Cs_2{Ah{yCF!(VcvMGTEP?mY2X&m7SgIJZ2x_(c!TkDATW>kg?_% zeaE)+2Fo6>27C=*%xn;#z>!59H&ipNqIWo3~7yj_b?tAKTC^Vj;J>U0%cTC{jm%-Cszv+b4l9KOH ziS;0F^2>Gni)Jomea31AciT6=Y+q^5yhC)iimr#Ccz2wFHr91>vOcC9WWc6#*s`Qs d>&EW+J@HSO`5K?!6HLed_MY5hj=Sc+@(*OK@IuR<%ouyDw02U}kg3!V(;v^fB^VtX?cf19! zKtq$7GU+K?@&GBQk&yF!d%G7SprA{^ShKtH|Fd)JMzgv0^|Qzb@M}V3RtP0|C(|+| zMVtTyFX#IpJV7mT+cRaVpLDRv9Pl`aQ=NqEPNZAmp=xXJZlOQK zPTk+Y(;z)L(m_2)^j;^4g2jM}pi{Nm(jfO^-HN(F{oARN#MH7@@nISa7IRTwCAsWb z&zaI=zipkWNTn+G5bU0$y8F<7tH-2lbSvK;47LZT@h-DL41TFyC(=*@~;3N=93fgUKs<7cb(Vo0zAnEl+b7N~|vu;Y+i6gLN z2;J)}hDRO7p&elg*jK0kyJvp%Q#Lug5#IEyy>~IT6{Cj*z&u@pNPC!<%tMtpR?e5xp-;*%qQ>T_8fjBRJlX&X6)Y%u^a+NHN$}-^ zooL_0kx9?=c_|ZPErHHrbLKCA%#pDj%(%l7yX&dVA=h}0c6`?d-Z6o9KMsy|`V}iQ z-=#gf9*x_L=S}`gXy!7m&sf!&%e&^y>iJJAV1MbC`TOYdG`jDD;vI4tYFMl0l=&KR lpaDzP{$Jn8Eg8Glcf~zv=Bs>tS1=j>+q-g$Ic}K$%)cwh9o_%{ literal 0 HcmV?d00001 diff --git a/test/shredded_variant_ipc/case-071.arrow b/test/shredded_variant_ipc/case-071.arrow new file mode 100644 index 0000000000000000000000000000000000000000..bd9e558aff23911c74c169bd8e52a984da1d2046 GIT binary patch literal 1850 zcmeHIyG|QH6upKuG1ev`#EMcx!dkgVfmV>pa%YJYsUQ|nih$6-PL!2wOxBAdgxv8t z(D4D3p{MXiQ1AnZbIv>#BcMo^f@|HGx$no^9k)B3^)IU;6Tq(-kp&@C<%P^iSt>XI zNz*?Gx`vE3 z$LKq@=WnrW0xQ5bK=$+mxCWdBB1u8JjZFhKyeHa|_Z~=kquu%R`NL+*l(Lg1V96M| zS2jjK9mb&@p$r@Xn?|vFQ?e*(UXueB# z_C1=mo6ej3f1#Pnv_4}s=GV(-OXtbQKlEn%%bod$=<*!8AA;f?avGXg8|GwvjXBVO j73cW2@8njE-P60`pEUDTKD{fLjQ{OjxxpN_&41=!D1RK| literal 0 HcmV?d00001 diff --git a/test/shredded_variant_ipc/case-072.arrow b/test/shredded_variant_ipc/case-072.arrow new file mode 100644 index 0000000000000000000000000000000000000000..d005bc3d923d774eea4b27fe612d63f2ba9f5b5b GIT binary patch literal 1858 zcmeHIy-piJ5FW!pjCDi~7(_u~ohT^KDIf~u&JxrFizo%4XyF!dl8xbfI6}x>B*eR< zNJ&WveF{HB3SJ;G-?z7W7y(5(3dWk_ z2fPT<^AjD^fHy)gq?#U-BPAse<5Sd zA^MJOavsYzun3$3vZp_R>%dVUk`%Pt*et+?_e6X0R)D0x)Eir0H@0i0l<#o_mh_?f zCyU`xhjC~}7z1_{D!}fUAN^EJ4j+U!UAFfw#;fsx<}9m|nX(Q>EMnz+IUV|>3?pi6UaFBcMZ}{Gpisqvk_U@0WK4oDvv#6= z6Guip(`U|Y1|`s0Y|i{IAai7F2NUk_!tQ!%bI3JbqaEM%fp<*c-4BDKoqp8{ZBGs5 zdNgb|oHzOZLo=6Qea2dxm^oejeSKNJqPIt2pGmO}AKf0uE%=~#mz)arcfp)8pM4HC k;Ij?z*ne_M#_s)naZj81I^W+HOvYP#U+yu-b@Si(0DV{-$^ZZW literal 0 HcmV?d00001 diff --git a/test/shredded_variant_ipc/case-073.arrow b/test/shredded_variant_ipc/case-073.arrow new file mode 100644 index 0000000000000000000000000000000000000000..5aace7639a837f544a75d6f363e7d12b66478a34 GIT binary patch literal 1858 zcmeHIyG|QH6uk}$7;6&&u~3Q>){24xtwJi~&JrjkiA6*a5E|G)S;@w*UJMAiqvlKa z2uh-~Df|yX3O^8;bLO!a1&VYOTzxY2B`Z7hil0oSiYriDmPQpO2T z3JQJ*GI0Fm63-$qlV!sfx%xXCR^6F0)yq2AWCnPgrMb@Hb|=xTcwe3WV5ja6 z;8~QP9_XkZWqP-hCDCF;MbNq0X=#vqsct3RsQ&HDOHyh%t8_n)`iq6Ar?Nu!q-U%2 zuiv)LRibiLc!+jRbKQODzjefi4($sO2s2KxKzVI*-|hxV!sZx0UJF4P>l2 zM&Gf$evf4xSO&fWvZv?372p^UNebF+Z02FZd!jvgAAzJlH=19*uCCWjDcflRmJFc# z%f<+(!#K1fj01ZL6<`m{kAA8qhe_d0PrG{;V_P$NSOCn^H;8nAd9i+Hkw2y0Lse#} zfRZ>Dn?OqKoY^{=DeGXwGFHx)*P&0!Frvohu^MSpMm*X8N;NDfdAI;W#w3I?=O#Kd zabzqoedgR+sDQSyIrEo5=E&F{ru^ZN-S^bxP-r|wd%o`j@0h^59|cc){hAZHo*K&c zXw+^rZ}R_#W-g=pjI}WJ`{L)?r;|TDc9uY|w$I#IM~H4u;1)tqyh~06`#Wz=*5`nO l4fx;!-1nc{im`inU;NW%zRs8T1(WgI-j`dUUDl#LJG9hK003|Qy z`yf5bua|fhftl0|U*!7#XjHXl%2Yq;Vv{-Gza&m|5_YY%Mb{*84z>IIG63nz)GWv$}lG#ISpqP|LU*_)m-rI&s? zI#rQMRqi1;I8Sx&x&QVVld{pRe19<5AE1^6;E763&vY8H4`J`{NOzR!*F9vcIYi&F zO)p{D0hWP_K=$+!xB(moB1u8Jjm*Z7F1HWWxuCsVJPV^uisi6k%9{O|a)cpax ziSp}99krrN_s3ZhH6kj4&Q))qL4Hj2Aelt1cQ;;=Qp;JTqdb~53ei+$g>0>7tMs?u zq0UvJa#eVUde^z0{ONyi#iU$xCqJLf&S$9Q5O}Gw&sRE+*@t-2ztBTv`t=hs)*Pen z*d9H@(gl`*Pl4>|E${@m4MdWHb{m@`*zlfcPu@!)=~wN}>o=#}mMP^!nt&y9=-$~F z0d*LMc7z?^RG|Xwf%(x-)#UI%c+>mt-o@C~j2;#M^YjfO9bjIp-vi`NsrOKo11g{- z&c%BmrFPD2oy?SVFk%@i=gaHRCuJB>W3#G8+LRHGHh@wM3rb$pVaS+-P@cGn4ow`{ z4osixR*ba*+Q#P0e+4o}#`f^gA6D$Xr!I#=V;$}Jz7M=(0`GnqJni*sPUvZpzDG`E zw&lFZ_!pYFEbBAY^R%yfI{U=F(jW5<(cN8iJp{$OV_#Sud8QsiUu z5tO7);g6s|=_7N_JT@Bz3OWj|b!XP|)H&Y^uLa51t%*u>ZaRO9= zf**nm9KTuNSp;U%G<=ckKa)w_ohehptdC8WfM2sT*IC@_C%PM-s-6b#4*Fy4)cpy( zit@`d9krrNpY*dN+Ki|OI#UzJA{}5}tlwqiPpS7%mt`uT zB+kVrAfW$?7uZ#ZE&7vH0@ zT{&;^pF%U2vOZ(&UjN?sZXN@Av;F1H{6lnk7Tpg)@eVl+O{{C?WPOb}(10!H_;26I dtr@$Qcf~(x=Bs>pS1=jR?OnOU9JkGX<~=Xc8|DB2 literal 0 HcmV?d00001 diff --git a/test/shredded_variant_ipc/case-077.arrow b/test/shredded_variant_ipc/case-077.arrow new file mode 100644 index 0000000000000000000000000000000000000000..33fd3189212f74a5b4b1767c3fb4ad8523e573f1 GIT binary patch literal 1850 zcmeHIKTq306n_B+(#WkSq*aFwk;TvfS-=!!iv$uIG@|OzB6Q%ZYRMtgF{u#B#>YWo zWMzb%As>Q)FMz`DcfNC70YkSAyy)HC`~T;jFP(08^XrPp1aLJaGAD$(ypb8HN)0DK z#VhzeNYCp53? z?6;>g6{$=W9)i8&Ob`C{pBynM8{NuxMx&h(YFPjts`S^9&O-Jf9PIzpJ!SfJ3mIz; z(RXaGKVsPi)_@;??CB|R4LA)%l7e;{n?>00o@h_rDv?^wTgo%nEOM&fdEi+osXO0$`r5L8Lv*OYV0b`BUmVG-RF% zD2a3N9Y|?DXL+5>lyxv-4J+r%>Ch)-7*S*MRE@N$As%f2l_nOHJoyAe#w7Ui-cGb{ z;>fgT`n;TrvDQK7u{rZ+K<3EU4qmy#6T9oF&7shEj&^+42i`G(cV7laJN>2=mUD4E zD%+LwCjVb(=2F&Ytc`b_c5-oEb@{Nr^vnExba@8d_d)RvISnnWi{_O38grll%hvIq fzLQ%wb}#RWd(zBT`SPw{GXA%BqJ3;PC#Y3vqYqHNi3oifY8D%;v^fx`EZ1gJ02mA zk-S67q^Ix;q)R~^DRRDVZ}&I^6m%4fHM=|iKRdT>cRKGsuZv6oKc_@$La54|lx0RL zH~~st&i6rjmftM#ECMrW8otPnf0Iewo+(qqq>oMJfnSn1)k)avN4gsxtDXk$8v0}G z)cpm#2-5Qt9khZ(AM}$bSPiHMI#v5!4e~J7-DnWBwlAC{rk1sekJDhZnu~@i$z>0E z&Xn%@?deoSDpk3MVE;VTgWLYQXH3dQxANW5Xm^BKUI9;3a(brIkbMXT2S>W6Ouzm> z#+pO)9oy^@mMvffxCmrVFM*rDX&{mmwA-wJ0|e%i{NOd->|}B zF0MyKyJFttKZa&5MSaG4_qyG=`uh9(UpfP@zucL>k1m(deIFF>kkin_x@=CFuQ3N2 kux1_K^qt(Qv3q`3+>>U$%I9|llkwEvm21p#+x%z#2i%Ds82|tP literal 0 HcmV?d00001 diff --git a/test/shredded_variant_ipc/case-079.arrow b/test/shredded_variant_ipc/case-079.arrow new file mode 100644 index 0000000000000000000000000000000000000000..f2cf41b66a7821dcab9f3a848c02bfcdc17aa8ad GIT binary patch literal 1850 zcmeHIKTq306n_B+(#WkSq*aFwk;TvfS-=!!iv$}IG@|OzB6Q%ZYRMtgF{u#B#>YWo zWMzb%As>Q)FMz`DcfNC70YkSAyy)HC`~T;jFP(08^XrPp1aLJaGAD$(ypb8HN)0DK z#VhzeNYCp53? z?6;>g6{$=W9)i8&Ob`C{pBynM8{NuxMx&h(YFPjts`S^9&O-Jf9PIzpJ!SfJ3mIz; z(RXaGKVsPi)_@;??CB|R4LA)%l7e;{n?>00o@h_rDv?^wTgo%nEOM&fdEi+osXO0$`r5L8Lv*OYV0b`BUmVG-RF% zD2a3N9Y|?DXL+5>lyxv-4J+r%>Ch)-7*S*MRE@N$As%f2l_nOHJoyAe#w7Ui-cGb{ z;>fgT`n;TrvDQK7u{rZ+K<3EU4qmy#6T9oF&7shEj&^+42i`G(cV7laJN>2=mUD4E zD%+LwCjVb(=2F&Ytc`b_c5-oEb@{Nr^vnExba@8d_d)RvISnnWi{_O38grll%hvIq fzLQ%wb}#RWd(zBT`SPw{GXA%BqJ3;PC#Y3vqZW`Ni3oifY8D%;v^fx`EZ1gJ02mA zk-S67q^Ix;q)R~^DRRDVZ}&I^6m%4fHM=|iKRdT>cRKGsuZv6oKc_@$La54|lx0RL zH~~st&i6rjmftM#ECMrW8otPnf0Iewo+(qqq>oMJfnSn1)k)avN4gsxtDXk$8v0}G z)cpm#2-5Qt9khZ(AM}$bSPiHMI#v5!4e~J7-DnWBwlAC{rk1sekJDhZnu~@i$z>0E z&Xn%@?deoSDpk3MVE;VTgWLYQXH3dQxANW5Xm^BKUI9;3a(brIkbMXT2S>W6Ouzm> z#+pO)9oy^@mMvffxCmrVFM*rDX&{mmwA-wJ0|e%i{NOd->|}B zF0MyKyJFttKZa&5MSaG4_qyG=`uh9(UpfP@zucL>k1m(deIFF>kkin_x@=CFuQ3N2 kux1_K^qt(Qv3q`3+>>U$%I9|llkwEvm21p#+x%z#2k6lr9RL6T literal 0 HcmV?d00001 diff --git a/test/shredded_variant_ipc/case-081.arrow b/test/shredded_variant_ipc/case-081.arrow new file mode 100644 index 0000000000000000000000000000000000000000..2f94b8ada5e7b1e1cc78a2d59b96f35ba7e0e1c5 GIT binary patch literal 1858 zcmeHIzfTiU9RF4hEX`p68{@#xTo@S01)Z91O>}S(Xb=Y=7=v3`BHOPZlH=|Ba+v+<>Of72_AEm)yITv+R zlFRP&oGIP(+tR6uRH||h!Co)bo$LOar%cL5xAL9AU}u0@9s`e6a&oHEkbMX{`-i%v zOuv3Z#+pO)9ox(TmJMJL_!P*VegM7zP5_alpxwr15jMOh+LQMjNP4Z_Sby_!qh?Ck zjU%vR2;HAq439dDLp#DGu&Yo3cF+9ir)+X~D7@)ed+%awD@G3sfO)zGk@hezncsQj zPpR`zmU$|mB+kV)kkV|tMtpR?e5xp-;*%qQ>U78fjBRJlX&X6)Y%uv;;%O zB>3{gPPA|0$b@J5%(=~=1Uie&ng0T0j*RVK+8u7$T~BQexyD_zz*8Z%1ir!zm`tkMk$LY_%7B=Y=z&>+l9X`4}glC`3sK&y}nxwAxSh((klLDASdBrDk%){7&A+~srl zh?JqH@JFQJ2O{U3c`QUgk&c3E-I=-X$J`xvy4~H6TOxD7uZqaB5E}Aa7NjP1oB%;s z@*|L;<8KE%i@;2_4PWHvpZUD$&XlQ1KEfs|!0&lh=sX#YQawnH)KG(W4gDE*>RtlR z+u;D$?p1ij}(tDlm`w#E-+oqH+SqhfSpnGLw zgw$ai+7W8Ni9!X~L-V7brpe)%@TQmDy^FDJ89gij=II+mI>fwKzpKcfQtzQDt5iTq zoQuyuO6{E4I+-c!V8l9B&X?DrPs%W&#^$~nX;Vi$+5mzU7L+{SfFWZNB6;a1Ix=yj z8k#3m|i3Y!8e6aL?|0>T)PG9-=+p_hHFdv`R~R@U++8aB^=)()Y-T z%yv6(GX95VE-N(&D%G&2di|91L14^E1>+bj%39XfJ)M7LpS>(Xw-;~=5j1(1%#8hA rHz%9JjDrn$;}W{*Ke-KK_w>H_r_FqwPwxvRQECneA62SstDOkzg?IxUb{=waz zlPFknL6L%jf`Y+xyAB0061uV8zIpF`@6DT? znRh#0U0eI?tB;HspjF>8rf%3~%_TEqDyE7aTB$tL*C3X?{-n&lF;k#rt2gHRKL&$2 z-={6y?QVma%bv$z#*#?U zo1v|F(X=u11etfhtEiY44C`av%t6Cey4b*T9aIIq4}yz%iGq0uy6I&U9#cY}_438Z zV89y8K6TeXly9%Deg5SqU#!@-cefouWgh)G7#x8(4)gc`L_gN!LlAXugHi><`~v); z^)c3mImzc!^f?yCR$bbjRWv%${#xhDKy?%lVb*$2>R&TLom#9r!XZ zA6wjbF1L|{a%lk@0e9eHj?MJ|_Pn|N4E#?8@o&IK@tf$s0Pe)auTZb|g6`N{UjcU& zzn6GZXU=aPm^q6ZpWiLS)%YR$H-Y=k#hiNG_|M&(N)7lOIiEB0cm~YzKfwJ44EJt& zCo>n#GyP7|>sZF>a($hz8J+^?Yiw=}*tG|E7WAWvF7z|r=FZV5g3ghd;y189bLP`%|x1yk}l!&0#xq%)WPakGw1DiA5FICW@Z}toKy*q$VJ#^rtcXJtQL01&i6YX42y{psg0*)6OB*|E)*(@Z zM2SR0nc|8R6qG3_Qc_S*Qc_S{c?JDZ9B1C!|Hs~V*CvEhD*tG9{>;4jGk^d8ASJNgpDyw!isvoA6NnveA&&;Dt*r){4$cQb5) z$rR{uSdaX0ZmsG0!Cb>#^PxKce%Frk9DK)#w%2`U!3q6ZGxVH_Lm=eHtp+~C`*lC? zHk^gqJ8G#DWxeW+$azxfm^R(8W4rA%T0!8z*e8Mkr^jFOBiC~yw=NBi_5a>1KKN&GWERI^S-hnn>TF2u%-=ozIWW#^K7O z-o%|JDOGpzs;L*+cOHbE&^$Jxu-bRppd#oT2rlwBisdWN6)U6YNDh70%D4Zv+pNFrQ+EkOd1Yz!`pv617UFk# zx9&kDTVLAkeUQefKu15)KAbC{KhbB7KZ6oQLtex0ll7&YxzUexd=bQ$jL$%S0XOqs z9A}Jr#iE-Uw*=f3gUu7mu{kbV0TkC2@XNpi1~<_ge;Ia-Uqk;MaJvROO8qQI$LyV# zU=_F$W594W?~UKUT;2fl+2B&SEMgSRrHcM1;AZ(dDY*jmgWkmqa4QBoF#aMiTLzbk ze=QN;MLz)UrNNGTEB?oLPLeCb?}PCfn8!O{xObB~nK3Ze*e^NP((_v!Fky=t4hcZRQ+xN6>R5BmAED14LgtZoF4{61Y!!?uvNwyF@XJM;ktE zVxO!tM4uuurVm=Kj0avrdlv_lG1*G;3o_)FPzir%qxpDWAFv;{amXe%zm=A8h?pZ| zYdf!<44!;1NwdwVqf%M9zK02mC|j;^FIZXe7p<)9xU!BnZwKODU|Woz-FaWLvd(YC z%F0yMd~aEUj(<;qmiLnt|Ip%2S-x($GL<#nmNlsVQw3VCvf`gvz2aY3S?#+$@?H=A zm6esLtp5AfpfW{%Gbd&0jw&loIepoVDl1O8@$UMoUU{#*S61Br#W_Cr{!~G7Bcu6( zTdjG#Ph>Uj(P<~klC9$&s@MFgA3n$i97D^xLbpj!3Dt$aR3%7pO+n@2d71@j-I~N* zjBSq}4QL03+2?rdH@t@p?_uA2s~q0L4&TxHUMYw7uw(qs!WrJfP@!4FNyB>>f3Al2 bFua6Z8IR#TEc<^MS@*CD`2Jan|2O>=gASh1 literal 0 HcmV?d00001 diff --git a/test/shredded_variant_ipc/case-085.arrow b/test/shredded_variant_ipc/case-085.arrow new file mode 100644 index 0000000000000000000000000000000000000000..4a7fb36bbe916ad5a2e3fd9612c583c7d0ddea16 GIT binary patch literal 3114 zcmeHK!Dafg2_gUD z2lx>l^9lJ0AxA-ReXo0JoNRVg@Zw<`YPzfHRaH+_bTT(4@LSIXH}7T;h2@{ zGA&b5!wREPE#wi{YA)ZJ;93NeEP1)e=WqS~LcXTX9;FAE@Gc zL3RQB_e${d1@no?jt`w_n$+zbq_J5v1cJ}(ZsNf1w_Fl;O>^@^H7%lYt5zp7r;CN^ z$fkvE({y@CVxSxmA%L&?Z@bLKHnW8%vwNJm?&Ua1kDCjgHBye-UNo`Wx4k4YO>UvI zKjPz^BRkBx*r!cXdkp7DUK=l7$Okjz64!Qk|6w^A9^&6NXu4V2>bx~q-f#Q#bck$- z2=%`OI<)Egp355Sl}2~(ooid)Q)lRhJJ5^V#9Tv1&ItQ&UKSwXsLa*zd=09B?t&&k zYp{zzt2vLt3RvH(Z+`Uq>~Z!cWsgAQ*H<>4zkKqd>C3;l&z3L0JWM+2du7`Z|bso{v=!eawLJLP-Yfr{I5zd*J?w=NR<;5xDb!trAQBik-sR z7Z~#yFw9RmTg3Aa55_3lRpE{Pf-UzMG97Hi+t#7in9R2x3@jm^&UK+yzdUHIWP4*XM@WW-S=8h9~bA#V{UqxS~eZi8eLtm^K)dIvz+v|0@g`lOvMb`HdF&I5JuF zIb08FJZHg&*u3)|!#TWT-*T|792&RoxsWq1@8`Tu zqj>fIK!N$Jy!v0t<%(Y(z$>r#rvv!)0X$jyQoO4FmRFq4f3jEHsG0W6t?JcZdH$VF zaqK6qS+59PHI287z_Dl9{mkP%zY4L=wIO);F)rt6*4vH0qvP)=evpj6qc|Vq?`URu QPyBD+(F@#TEB-&iPgvJ1+5i9m literal 0 HcmV?d00001 diff --git a/test/shredded_variant_ipc/case-086.arrow b/test/shredded_variant_ipc/case-086.arrow new file mode 100644 index 0000000000000000000000000000000000000000..89515025a7bb08f51eb8222e93b9db7cda1106b7 GIT binary patch literal 3154 zcmeHKPiqrF6rZ$d8tPX3Q^bRZFnI7#f_f^!gM|tn3f7`nXiJ@Tr#7&g)Xko!e6@Zup4-oAPN-@bV>n=G%dzk2sXsUhI3s?;q-m{T`Z zP0gq}R)9*ikVjyvLB2J`wNfBz$;p-a`eQg;2y5!hQPu@fw}GFsc5bs~+Kp|}?3mO- zb^-gJ6(8pt=!wn;hgP?AX7{>TtQR#6!RKZ-v0(SxHi>(>wRs|%Hm!nJyOZnF#X@yt zvO>41JN+clP+pV48()o|+T6q@H$^19JIHPCa-O8m;(TPu6zsMab!_)dKgo5AC6wx; zeY|sI#(5WYnk+K|1V{5yym%oW?T}4uYVrQ#dNe-8zZIx^dDiZH&{y7X`}A~-Y>Wu| ze;0UYvX6b6H#sZK-rj*tjhm@6%)@=?mAV63$3)I5&fhJy012To-@tPNSO-1;P6Jor z7XeoSkBwDe-K@UWsf z082ZlR$%+w*{gZ#8$jCge5?Q&V-^@TO69Qq;^WV-2G&n|!f5Qzusip*RoXJX*v??> zHW>5_HtbKiTeRmP9?enqt0EfXd0*D)JMDdy^KC#cIoWSH7+6AH3p&47e*29i<2eW3=jNUF48E+9xdr^p33r*6`%>0~nhKHdLZuI!TLSl$AbA^(^F8D8QQ(DK z$4mUjVi>6K5`QVkrH+?@7ya4@zcFHe5O~h30CS7K$o`fWJE7t){TbzXwg{pZKcS*4 z^Uk64ps&OcG40DfJSze4pbL7W-fSp?11(8u<8>cpOa6s%4`h6?d+jgz0<-6-!zGKZ;1cDFjy61x3Xt{h zxYZfUaX0La7g$xMkjRgCXKlw$v+kScX`|kJyNUL7K}fG*dLI8a3yT%$S%KAq#3Iz= zh3lIyA(xbIJMbH=SE(tl9;8epMTD&^H$mIB|GMJ_6`p}gdwtVw+Ul&-u+Jw6k{jrY zIB8de=W0~OAz{mml<{2#6+yQ_IBIg75qS@Kpm}Uq0jti=$*(Zv+2uK+>@J9WZDHxj zvxiTsYW3C{4n%VJI}DFNjKe&pK(ymI{s5xv8Yr>Ri8jf3Ps6I)XuCzPi8cu$n z`1FVuVV}l*Pt_=>gxz1jsVITt+2Sc>y{FVDTGKhaso36XpQqPJ@pr^Il!cb9Xqb&{e2#j(5?H zb^f7miYCvz**iEgnYOiYj&r?_d67HdbyU`Zu#T2x1sYOqse%42S{>~H+ALZFe!sw< zdl?5LP}!RM^k+0;t+K|bdxA!Jb7Sk(>lfQCdyISS1S(laZJ(mSGG=Wy(1IWHx`9># zNRDBfT6d2TrpjrbR2~do_A~4q_#d6QzII&$zW+PiOn{FEYB4oEd@1z*M%#*nVx)lZ$ z-0k1(uRQJZ8modFc))`7NOE<2v2Md^FXBG*a$u9>5dTpy^n8*`{7LJ!ceUk*wjRqk zQ2)PjunRoI81V zcO=iCCzhe+PW^#?^9yfc?lLV}-|1U5eXB}O@9q#qGJUIR)3>VPe?rw;^%?ifhW(HH E7luyt*Z=?k literal 0 HcmV?d00001 diff --git a/test/shredded_variant_ipc/case-089.arrow b/test/shredded_variant_ipc/case-089.arrow new file mode 100644 index 0000000000000000000000000000000000000000..0bd0354416c05562780bab1e9f364c80069d2a6b GIT binary patch literal 2322 zcmeHJy>1gh5FRJzU>Sn~M2LdI+KR%mfGV;mumlnvV1YmiQMAcz$Vum)oXc+=`dPEU(s!>_rgr`ZsbTm!$$lH6qN{xCLad!YLU zy4J6vc;e%1Vm?;+(a5Nd%FNy{i&aa}5OS_}Qv-27F=;$jovq`rlhDe0C4*d@v}RjV zoz3>P)L@dP3Y!Z~aK`iT`zF`1&h;#k+C9q6`0sPl36pb?!%5z5yQN~YuP14)I!vLI zAMN9vslL#AoM*}4t(tS+KDj_ejBkKPIy;=0yiMHOUxfyvkX+zN|#&eXH=b_+gIh&)tc42sd=TVtt#| z4g-ibp}3SIpyG0J7@b837|9BI`wf`BKCZ==w=}~lHMcIy!M^FYLSA>y$#=aR?9cA!H!!jG!9QsKRSte(W<1`dESJ~eyfCv)ToZ=| zsE+U2*FJ+u7Q0t68MYy6LI?H9bquKh3c E2Y1VHu>b%7 literal 0 HcmV?d00001 diff --git a/test/shredded_variant_ipc/case-090.arrow b/test/shredded_variant_ipc/case-090.arrow new file mode 100644 index 0000000000000000000000000000000000000000..be943a6721dd7582242e4387b53da5c9e2d5aa24 GIT binary patch literal 2322 zcmeHJy>1gh5FRJzU>Sn~M2LdI+KR%mfGV;mumlnvV1YmiQMAcz$Vum)oXc+=`dPEU(s!>_rgr`ZsbTm!$$lH6qN{xCLad!YLU zy4J6vc;e%1Vm?;+(a5Nd%FNy{i&aa}5OS_}Qv-27F=;$jovq`rlhDe0C4*d@v}RjV zoz3>P)L@dP3Y!Z~aK`iT`zF`1&h;#k+C9q6`0sPl36pb?!%5z5yQN~YuP14)I!vLI zAMN9vslL#AoM*}4t(tS+KDj_ejBkKPIy;=0yiMHOUxfyvkX+zN|#&eXH=b_+gIh&)tc42sd=TVtt#| z4g-ibp}3SIpyG0J7@b837|9BI`wf`BKCZ==w=}~lHMcIy!M^FYLSA>y$#=aR?9cA!H!!jG!9QsKRSte(W<1`dEOQ<3;eMG|C(6X3 zfoiRxk_|`>>qs6z&pU=*_~d~a`2*(fSGT^)Z|(A1%br<&YXJg?|26){xAqI(W7qx{ F`U8hkakT&d literal 0 HcmV?d00001 diff --git a/test/shredded_variant_ipc/case-091.arrow b/test/shredded_variant_ipc/case-091.arrow new file mode 100644 index 0000000000000000000000000000000000000000..54db61693c3de1f85c8d68f6c43ade95621bd142 GIT binary patch literal 2322 zcmeHJy>1gh5FRJzU>Sn~M2LdI+KR%mfGV;mumlnvV1YmiQMAcz$Vum)oXc+=`dPEU(s!>_rgr`ZsbTm!$$lH6qN{xCLad!YLU zy4J6vc;e%1Vm?;+(a5Nd%FNy{i&aa}5OS_}Qv-27F=;$jovq`rlhDe0C4*d@v}RjV zoz3>P)L@dP3Y!Z~aK`iT`zF`1&h;#k+C9q6`0sPl36pb?!%5z5yQN~YuP14)I!vLI zAMN9vslL#AoM*}4t(tS+KDj_ejBkKPIy;=0yiMHOUxfyvkX+zN|#&eXH=b_+gIh&)tc42sd=TVtt#| z4g-ibp}3SIpyG0J7@b837|9BI`wf`BKCZ==w=}~lHMcIy!M^FYLSA>y$#=aR?9cA!H!!jG!9QsKRSte(W<1`dEOQ<5;eMG|C(6X3 zfoiS6vH{6q9mxaedB@NTpFB__f580x>ehGptzCX=*)z*;EkFSAzsCRg)_%cz?ArfA Fe*l)-ak&5h literal 0 HcmV?d00001 diff --git a/test/shredded_variant_ipc/case-092.arrow b/test/shredded_variant_ipc/case-092.arrow new file mode 100644 index 0000000000000000000000000000000000000000..b7216cd3aba7b0fbf49995eca052da6c9cc1d5ef GIT binary patch literal 2322 zcmeHJJ#G_06n;+DiDecPAVL%r)>IUh1yqqmfu$hP0Tu|P5Jj8pgsimwVZAnrqI4X9 z15i+M1P(yK0Vp{DhX^d+H$R&N#|;vlpY>+;{mz^3&5YdZ_aDEwD^h^ouZnC4sVP@v zP3qFX7pNAL`VeB^^q*?{7Fhvr8NJBaSy61dZ`z_+Ho_!V!Edu9*I8#UignrJ4e>D1X*^NgofB_K=;fl4VXjWwrD+yr zrENzIr)jEST(E+5p3gtfc@#%^RAy59$GM*TxlTG|aW=DG$@%TJRjd!AX_~7pOQ_UG z|9Ec}T^K#dvt;;M&4urtUZ5hz*TLf`do$H}hq!kp2S<7knf^Qgwm0b{*DyEG-F0Cv zH)InUQtfgJ<4bS@d<%@@AYZ`mz}rp+NP<+R?>~MO1I!1?+HhG(5V%+EN|C2$1^`3gVQRjtyV%UbYvY#p%%Kg@yo**$R&eup+zOl-^e zVF1y`HGR}#f$;f zS{O)xqBue@*CmB=!`;l#tR<^~=_~J_r6y#%KJV^#Fze*{o>=1|@;z~Nlx~Y;ulHE+ z%#iF^v)@(O`?s8lZwKkNoyt1zINAH3@6Q=9vG>(|(ekgdkMlEgaW7SwwT6n8d19Za zB@Q9FwT4cHkSx}L+=rff^k?Ct18VFRnBQO1#4g|3X95FrW*YbpxM0;!b*jb7yRv?w;*H*L`@8)1?w;5S*4>#Q>v#X9W_qk)F5 z{j(?@xjdVgk5zs&)~c&Ay*J8Y)mC(boJYHTG@hvL*0HxF^m0+jFjpt-(lm>* z(zd0B(==5uE?B`j&*vZLJc^?{Dl@6wqg+q^Tqm8fIGfq8#_k2sdlM_@g=wcz6r*0kk8<^;0I0yNP<+R?>~MO1|$NBmShG(5V%+EN|MQ{ZP`2s)IRjtyV%UbYvY#p%%Kg@yo**$R&eup+zOl-^e zVF1y`HGR}#f$;f zS{O)xqBuY>*CmB=-QCR4tR<^~=_~J_r6y#%KJV@~Fze*{o>=1|@;z~Nlx~Y;ulHE+ z%#iF^v)@(O`?s8lZwKj?oyt0IJK6i6?avu7vG>(|(ekgdkMlEgaW7SwwT9nmnJ4y% zTH+9*TWjcK2+3j{$UW$}M}HPRIH1OEf%*M4P3-cmUB0zEGt0LYAb@xW|HoVV3HPyQ H{tNvE3^j6{ literal 0 HcmV?d00001 diff --git a/test/shredded_variant_ipc/case-094.arrow b/test/shredded_variant_ipc/case-094.arrow new file mode 100644 index 0000000000000000000000000000000000000000..e9ed083a6a43fe309a42b7dc7496118a2ff12618 GIT binary patch literal 2322 zcmeHJy>1gh5FRJz#4-jMh!6#ZwH1YB0aavCU@1s+fCU06MA0UH8C>;;L z15i-%2s{7IBt;WJl5>)%+ActelvS=r`vt};;u*mdcP`C7gANO z$eNU;f)%I~P4zX1k=K7JaV@d}-n4p=)6=5Z@N4erNjAVFSHW+yBsW>BH;7Hz>g%3? zuKBAd9{D(%n2%L{G&HKMGP6I(V%1bMgq-WW)IdB)Od5|=d;2)-B((BgNk3O7&FR)e zXVblH)gPy+!sdb#obi18p2>Brb3Kiu_KtEh`um)8%;a3;aFVy%X{y*9=y95>HdC0% zkM{BIL|^DV%Cn^ZTFtrdoLrzH#@E3^oxK^GyhYqwqy0nE)3!bjfbC5h$u-OsRCiri z%erhpLn>WvV0;O#fNz1(4)O*34!q@MfFuZI>;Ch1QLtZFXX@^ODL?6SpFV&1tZkv( zO=76z1N!Sb7?x$)Fh2cAm%uYf$XD1guh}eJUzVctzGe7Y{4huM=kCNigc~|vvAzv! zhXF*JP+ZCpP;ogqjLsqijAVtq{R&K9AJ^i`TM`3eTEWOS5^NZgdw6CG@u5!z8Pf+; zYG5D%isBH?JeO3H8~$b1>{+rJ*}C%mIjTZ-=kx9U05eaXABZ_FqEHi`$JA_*9P|MT z_6*7Onp@Ax!M@?Qg8gkThdl3iIoO}w&u?I2?Sp;L{;M4P!pwNg?Yvz777z+2x|wz2 zia69ztu<7#2FYO^$$jW~$IuHO{l$m=fcg7Xt?%+%yZqL&XO`bufB@oujsNkj{fzh6 IvHyks00k*>n*aa+ literal 0 HcmV?d00001 diff --git a/test/shredded_variant_ipc/case-095.arrow b/test/shredded_variant_ipc/case-095.arrow new file mode 100644 index 0000000000000000000000000000000000000000..62bd1f9eac113ff912ce35e4f8bc4383d091b834 GIT binary patch literal 2322 zcmeHJy>1gh5FRJzU>So9M2LdI+KR%mfGV;mumlnvV1YmiQMAcz$Vum)oXrDH^~Y(du(=?EGoFv%Gr5j+uBVaI&S7pwXY-_E7Uv>|Ox|v*sbaII$7!zGEMcZT z+Q-`yeWCX#&yxOIHRrx{e1VD>-vAGFb}%-1i@3K&yZff6ZGRpD+gmh}>sTx3?j>O_ z>#`0FsdS}*=WB2Ud>f2$kgwqP;748tNP)Wt) zcz|dVnoBtXDy}An(P@N$k*sjG-+<}s<63-qN@74vD|qsb1RKWW9>GY7*x&JTsPnFugZ=6G{00{GKG+BCzskWctc=In&dc@IuaN4@%05v+ z9BSy+8ai2nH~un+}>H5G+r0aapAUmr*%s>@~B zkgC-12C4*wz5y|C`cD!gSj9%pIEYF+noqORd9bu6x;J0a<>9jK#MLOvW!-0mb z{WH%Wxj0)`PgHh1)~c&geK1NR)mAiwoQ3;|hIkn3B$}x1&WYcNY301)VWv*o#nvoL zi@hB+oF<9F=8_E7crpG!XJHg(VG&8~A7^^ z#Sv8U0q5%*7?xGqFh2cA7r`YYjfys}PNu-qSd;lN7 zg-aj72M~1U#-$J7LsZA_yFb&>G#kNPPB?St{N8iFbMBB{zyI*rZIK-Gc2#6UNOid+ zYf_aO-awV0&^I6kPXD38caatFrqPR>p5}Sez0(%X(h(N941S%)nNB-{QKXa3FdS&; z+CTIBfy=Xr^;l&`W39R>)%&9~Qf);?$XU3TXov@~PNIqGZXbI~OfMG|4>NVrE=;p9 zEo|FrI871-8q*EI>fy**+0~S(DdgXu)R(vxq`KZ?yd`a zxgkwxNR^8%JfDMW;2U6!gM0$N0dF}OAPG{LzW?}@=bRVznYue*%8z>e$4~D+=^7|^ z;|MBwhxz&jhG&&N%+EN|1#k%o`2s)IRj$&`WhMAKwu)GTAJ)M9?3s85e?l88Cbnh# z@Bq=rHB(h#f$+} zT6mBEMSh52?n`ROHFq-`W-nO{Oka8bEY%^~{dsr4gIOo{_rw|(k?)DCqi|a!d%eek zGefeoX5Y)Q_is59`$xKIm!i&FPWJw1=W_-uj4hvwmVcFfoL`xXXDQ3vule~ahj@b)&c|&|7-k@Z|z6i J$Da8w^c%x(bzuMi literal 0 HcmV?d00001 diff --git a/test/shredded_variant_ipc/case-098.arrow b/test/shredded_variant_ipc/case-098.arrow new file mode 100644 index 0000000000000000000000000000000000000000..af2f04140bd04bde0e2eebc0543801a46af0c3a4 GIT binary patch literal 2330 zcmeHJyKWOf6unN?VKD{;h@un})>IUh1yqqmVM`!UAORK#q!2}$?1Zeeeq_BiilTIU z03Sd>NlD2EQ1Ahid;sDb0?Rq`FgA%BBsyQ~&g^~8opWbKZg)D5p5GQJKyQ~tR)tiN zE3zVG3GoFg1rvQ0V&L>2O8gdC0&g0<$mwZOthsO6qH)&8Bv-+&vn1D9quY;l+UP}H z4PE_bQ9N*YHZdQo{Ai$6Q)POupT(-K=m*4G=>4?SI%zh>3w_R7U-j7CUu9_@i zsy_P1JLBlw=wY5Ey;tf?`1Z*;Dq?&cJczQxk`=p@%Lhv@DF zVJ}x@4H{DEat-4Pa0tEu#&M9(;5XoVP6kMVRHpCWe-#DK3;Rsn9Wdp`?aq^D_n$Tm zlsicbmAu3G`U-|;nLfO+E_8M zHRFc?L?7Q=$^lTZHCc@QWbhct63_NaFk@X_gD-bU42Wrnk$1%VFeh!eW^?gjOo)mZ z11i-pkN`z-fMBjms>-^%nN_ovEC;5qEo0?Sfo#|3-Tek;om}4&Yn(^EC$5f(+dSFp zJr+DOBzxBE_q6Q&YtD4h&9~g_>*RYv^8PI&%?|9n_xF~6m3^F_nU8y!mRak@kM)la z`QXhV_KPOsQAO8Qu!AZji+Lb-p}z_CXX3pBY~U7Z!8S`36T5hG7jG`l%;L?pw}V&k Of4sS$a39;|KhbY;;B)@~ literal 0 HcmV?d00001 diff --git a/test/shredded_variant_ipc/case-099.arrow b/test/shredded_variant_ipc/case-099.arrow new file mode 100644 index 0000000000000000000000000000000000000000..34be6798cc7a8d8ef3367c1d5e7e2d964b38a2b7 GIT binary patch literal 2330 zcmeHJy>1gh5FRJz#2AADL{SO~Yby%N0;E`;J3&Uc+2QTE-vzX-F?#*j?x|`xdA>&<4mXZPA}3)y&HBk zbhV#({@AUvh51Znr+uv&D%A(QG*UH1N61;YpJ<4Ou}-3aYV4eOOH3~p6?ZdrUK^W6 zVLG<$sO~UH6pV9ru+Ep)@8~Ry!YmxGr1noUJ(%7n88SJ$vfs)1?bcMJ55r-SsRmP+ z$dCTf-YA?IJ;>6y`&wNJ-#wq9BF4AC{V+Wq>a0%O>w|+M-3d*7n!t9GPI41-3DsQ{ z)^bhOp&=EnRWZH$_^x^u9BV7egAR%Ai$Gj%9w0&6!{*Emo*5HRZFh6@Hp245c#)^rp z8b1sm`uO5f4uFcy$zt?p1&@&|vA17=8SB?#?tTL^PwwxDInEZJzA) z9t-vi$@ZH4o|L_R)tRoi`L3INo_tS8-oI(2>4CNP{@(JhvXApK*W+0xW#+oM@nicV z-YhioOk%xgARZM|Z3QQ&K(d$z@&NifVBZt(9bkR8Q4_XlDx28Fo4a^(*)xkb*WM2P Q=lCCQ?kC*GmibTg8}#OP`~Uy| literal 0 HcmV?d00001 diff --git a/test/shredded_variant_ipc/case-100.arrow b/test/shredded_variant_ipc/case-100.arrow new file mode 100644 index 0000000000000000000000000000000000000000..074083f992b26e14351bf5c2400961dbba7a8353 GIT binary patch literal 2322 zcmeHJy>1gh5FRJzU>Sn~M2LdIY(-&NKxJ7JSOSR-us|S%D7xe}7+dj2O7H8 zuRMR~@^r8ss_bB_Rad2Yca%n|rRWGb3wII?aWB?MG*R8nLvM-c<)Y$YrjA;LX%?o1 zZBq@WNupp}u!D7;&p*&v7=>9_WKuf^nV$T;Pcmh3HnZQ!`E9jSr1!#UlBq6BDAh;* zXnPi(89m9;c=$@qg>N05p(4gtz~eA|J=Iy8xVI;}`+5+X{@eq$*XSgdvDVPtbHZM( z%LX*0%7rGLFTgeMbuh+3zJlL^?>iYF2~wH9|NNcjoEP?)x;tRXk9++m&mKJO8Ys8p z2rBu2`T7oqXO%w8&p6U~a0vp^X(2+cbW7 zfav3!OE~~4wkC_wNd}LRtZ=qpf*I@b8hm+5Vn9r5c=C>TALgVD*K8p^jH#hw#(*kK zJV<~d-$yX_B{k%#yO|BMm#hY+ue^Vj>X7aJyt_ZZtdskDVvUQ)_r%pvxGj>s-ebX; zA=z28?`7HhH=T+7Bi*!1QRgiud;gR3IRh5Pmd{1Yzsf$&ugt}>lx6PMdH3Z;kH%BP zK2b#+8tB#1gh5FRJzU>Sn~M2LdIY(-&NKxJ7JSOSR-us|S%D7xe}?oV{qChqO=_O2d;ra$+9?KL{dWvn%H_nfeo ztFi_SsdAx-=L>KRd>xE&kgwo3;QLMnNP<+R?>~R%Ip>9artS`y^5b6r$+HJfy9Ub5 zID$&vW4^wF;aQ~*^D~Zg9$Z2~zQK=mm8-OKSqVTYJk>?O;A=_~J_r8;E0Kkx1jFze+0o>=2N@;z~N6mIimulHDR zW=M9{?0Z@E{!M3M|429OQq+0N$=?6ue9nM{vE_5o@~^Uw^DA@lEM=Mdb>4ot@tMX` z#6D3)92)4>3Od<zBa(yOu8OP+sVY}w zP0CWi7pN3W^)-lr(|;=QTVw^iW%MFvXL-KizG(|5=>U^l1;0+?OsCD>Aks;*ANDkK zjh}h`(B;{}e4?`Bp;j%G>it0)sfMB>d7H2d2m7L#BLq+-^94DD-v4lc> z^pAEY;oRs^md5>8>RkBF=^Pa?z78IS>CsqcP2%1h?H}r1X!_Fzwm0b{*DzPm-3!8A zuFD2Aq|)U&#uwlUcoU4{AYZ_5!1tXDkOZkr-+%ndbDkIWnYz1R%8xtUC(j-{Z5b$c z;|MDGfb;bY49_xsn4fW^OW*<$@)drpt5~Hym!;tE*fL@bewYLEvwPwm{0?obnAp1U z!vLa>Z!YBksMwk;Mt?GRjAVsp`z4sMF0a9tyCeq0w1Sa$#QQKOZMbF&@nK8_6*C4@ zs$(Dliu@44T$fan8}4S-%v!PU7vUNJD7EHeNU`$5&53DI;L)mWUu#F z@XV0xS+n0o+56X>>7tu&yV=*t_k`sA+eVrl*n98qE&nR}I6pHV_fnKu>*o7+q(&Rx z^N9VTo_N&IwKeRZ2FYR`$UW$9f&H2I=l~nKh1#&qQq{yR-`wS!%QLflbM5Wme~$n0 N=6=R~?3n*VzX4w`b_f6f literal 0 HcmV?d00001 diff --git a/test/shredded_variant_ipc/case-103.arrow b/test/shredded_variant_ipc/case-103.arrow new file mode 100644 index 0000000000000000000000000000000000000000..5be970b2203437562b2114ce1a0712b470b60268 GIT binary patch literal 2330 zcmeHJyKWOf6unN?!I%XFh!8~zYbpxM0;hzG;ibSs#;J2EWRZTxX4LKh|lZ7j-ps z^`Av?-{slDe5~@LfmThG>HU5dtGc2i*BH;8*+%|Wxl3X|Oe+|9N4yVn(uQlc5Ff@=P%&db zr5XkjpePOz%ymhjTyr-wG;7IfVEWoJRt{Cjc75L6?_k!+^*yo1MdW+p>X^7KlD*zz z!81d$XU%?3%ih1{Oy}Kv)6Kq4z9%H_-!jtdz}|a*Z~0f*$N8E0xR+^}wQjz9OR9gy z_dH_1s3#sFy0(TLgpe%ef!u-q2H2m84-T+_Tc`!wELBbH^37eoxjZw=H`m?{{^$4~ NZ|*1D$F})T^czr4c8CA~ literal 0 HcmV?d00001 diff --git a/test/shredded_variant_ipc/case-104.arrow b/test/shredded_variant_ipc/case-104.arrow new file mode 100644 index 0000000000000000000000000000000000000000..a9fe52335766ffa63834c221906540673bf4b782 GIT binary patch literal 2322 zcmeHJOKuZE5G^O;Ffs-Oh!6`FR*f&;MR030GPyw{(}fa47kJC{|{Q{S$BRXuXA-+%n#mPi45zbdjWq=sCS zHK|L8H&87o^-YL@(|@Y*U1SBkY4jo|Cq=R0-f4?w*$9hV0>90YTxXrZDAs9b7!5RZ z?Vm;Q(B;|0dZhBhu~uD`>Ag`FtG1#e>cPqWcqU#*j}TPT*exryXSl!F` zk{Bxafcg3chG(5V%+EN|d2j^@`3gVQRjty_Wi9wSwvJeXAJ)M9?3s85e?l88Cbnh# z@Bq=rHH~h!6#ZH5J9m0;;mj z02Gvn6L0_u4nWBPs5n7j`M&wtEI4kE==`j=v+sA_eDBT5y?+1Uvs)qs=TVr*OF}Ofl?-!r9G0e8 zl$EwEHJqlYf^oqP)_FevK<7~$@#(@!IU5M`j4O9f6_Hj z?j$i(@(%O$6%5ZheVCtdr1Rhk67mIptgBk3oy%JAcWfQ820yHU`Pnn^4E}^RR!nT$ z_~8Mfk8dvJ0I1lSEJmjpJVvs@*?s|LtjlZgT<W$)j1CiahX!!Bi=H=XSLPtWHJSQuM97cKuP`#8Tc7td0axnKD42Zwl$ z*e5E9LkrznLnm91EY^YCfu3jdv+&*lHFg`!pRZ|Rm*3jux0W-r{MG^l5dUlZk8kZK K+{d2zFZ3HIQFVI& literal 0 HcmV?d00001 diff --git a/test/shredded_variant_ipc/case-106.arrow b/test/shredded_variant_ipc/case-106.arrow new file mode 100644 index 0000000000000000000000000000000000000000..1b901f509bfda4f00734d3970f40ba4dee6cbd05 GIT binary patch literal 2330 zcmeHJyKWOf6unN?!5D)AM2LdIngUT+R!9|D6t)Bs9mE2G6ryO8osgB*57uj=C`!i% z@BtK*)Krud6np>$1s_1k2LzUL=3#7NH%N58)}7h=oIB^vjNEFsA3V7&l7n6^i>wN% zESF?Oic-QCs1S_x6^Mb;zbo)tWC^@!^dhIHdA{bpX$wbb50hL5zf9vyr}a)R(n-A= zb~JRgpLu@Y<=Mo1tg@rNRt=Tv{azZWnxZ4*EZj>p#DiET(Lgn}kG&m)-KXEXbioZn7OMfxBdCYfrm zgo*m-AMK99nbCtRjl0j)ned&H87g9Y72FTg7ek%ZiF!CXRj z&k1|EDr?Y?3Ky#wpMp!^8(CizUyRwBuHiY{{2^;^SrRn)NOz%KWw$P9^ZS^ zFi`Hs5mfR9=j#g?o<;gFKjTOjz!ONwXZW$M$tvx+ECheY77=Uk!yK5O-4plVcW7h9 z#8!s literal 0 HcmV?d00001 diff --git a/test/shredded_variant_ipc/case-107.arrow b/test/shredded_variant_ipc/case-107.arrow new file mode 100644 index 0000000000000000000000000000000000000000..587fafa069187a04e124d11dec8ccb2dc14458a4 GIT binary patch literal 2330 zcmeHJy>1gh5FRJzV2nWlB1A!9ZAD>O0i}p63R?n+1_2fbq!2}y+=iTV{^5K!ilTHp z01rTcL_9wxaAex1gdPV1dsq?39# z>}cp}zw-RPTW1sVk;)GHS~XOv_j+lhYKo4Kvv4=j5cgx9L<7~>I`Wp7UM?!`X6m>$ zHjTn`Y}-=ZVUj2q=j>pe&#vFmSr~;`I9^HZ9%gzly-zY^a&~3Elk?lIsYvgK!z5D; zrZABo{iB^xI5T>XrE&MAIupKqJVQl{uY&tw`f8}NI&rTL_6~F>H1%l$+v{|aE0{~D z?m1yCS7i+vQsH71<8yEcd;^Sjkgwo3;JZ!+NP?wI-G83uIs1imrtUVF@}pM!@zZ-x z8V1UpID$&vp})R^;aQ{)*Jm8*0(b%m`367cHJPRD%R=yXY!R^rKg@yo*)#DB{)9GG zOl;NoVF1y`7ngDXRBTQbqdzNnjAV(u{Q}Hbx30mLrz8f%w1km&#QShf+HlS0;=`B{ zGG+{@P{lw36!`&yxi6_A*WAsln7w20rO{+^iQJn}Vhd5qoW z$zJcVV9$_jui5WO+51AaiQ-R$$^dqVR5O(RVYtiAX5mVcFfoS(TK&oU`9*Yz)- zdK(WuoT8Cu66-|+@u;9`D>y*~lEplbJJ8<*`<{630PDMrny^h%*~Bj1+{K&Ao>{!P T_IB_;$NzY9KjS{O%zvWaiw<`i literal 0 HcmV?d00001 diff --git a/test/shredded_variant_ipc/case-108.arrow b/test/shredded_variant_ipc/case-108.arrow new file mode 100644 index 0000000000000000000000000000000000000000..98a8c574af0b8da0e8c5563035a7b4a2c7446480 GIT binary patch literal 2330 zcmeHJyKWOf6unN?!5D)AM2LdIngUT+7DN=0MPW-I(LpQ_NFjNlisbLBR)5Q1Ahid_Z71XCB5Tc7sIcYu%Z>&$)B%%*d^F`@xf2B01>wvdF5C z%5p(gq$nkPfeOJ`Ux64n{ksCcMV7!DMlW)5lILsgo3?P2_Atps@XIvLbXxE9BAwK` zVMjw(`Mx-s`22swp}`&cfY9L)?#b5)D*i>&RPTdby~$o2ld4 z*fa{$v29Crhe@JfoU?*;KApd#voH#?aGXi)9%gzly-qS@aW=DG$@y*9RHXOAVUno^ zOPHvS{?X1ToEbgH(zyFvoeJMRo}nVfm%;roeKFKoow(Nrdk4A`n*KC_?NvI-CCnvs z_l&TYtFi_Ssc^oE@hP|jz6Qo|kWb)O;JZ!+NP<+R@85srInN9GOx-$|^21hp^YOh$ z4Fly)96=>-aK65P;aQ{)^D~Zg4m^Q`e1;$Enyk{E%R=yXY!R^rKg@yo**$R&eup+z zOl;NoVF1y`HZz9-f=k9<#D9b>n7 zve$bocxFiUtl95L+518zV?y4lys_k`sAn?{-**n98qE&nR}I6pHV_cAH7*4rQ7 z|M1gh5FRJzV2nWlB1A!9ZAD>OfKo&jg)M9wxaAex1gdPV1dsq?39# z>}cp}zw-QnTW16Fk;)GHS~XOvcYA51YKo4Kvv4QT5cgu8L<7~>Jo1*9UM?!`X6m>$ zHjTn`Y}-`bVUj2q=j>pe&#vFmSr~;`I9^HZ9AXrE&L_IupKiJVQl{uY&twdN9;kow(NryZgEmn))<>?R7fI70e}6 z_nfentFi_Ssc^B1@dda9z5zx%$XDobmY0X%_(e1jkJn#|JnWg+-Gwuo4RALhXP?3s85e?l88 zCbnw)Fo5Xei%U5GDmEvJ(VrDOMzX}-ehFr*Ti4*rQxXGWTEfUX;(fR#ZMbH0@nK8} z88Ze{sA3=iihLiz+?Q05Ywl)N%wDn_n7THNl|vb_-Jf^&2bg(se^1PD9{HNMJjQPG zWUu#FuxCiN*X;MC?ES0Gbl%Ok-0bt@dqVR5O(RVYtiAX5mVcFfoS(TK&oU`9*Yz)- zdg~8AoT8Cu66-|+@u;9`D>y*~lEplbyU^bR`<{630PDMrny^h%*~Bj1+{K&Ao>{!P T_IB_;$NzY9KjS{O%zvWanznZz literal 0 HcmV?d00001 diff --git a/test/shredded_variant_ipc/case-110.arrow b/test/shredded_variant_ipc/case-110.arrow new file mode 100644 index 0000000000000000000000000000000000000000..d99d985417d0b3408164e0933afb40b0283ea168 GIT binary patch literal 2322 zcmeHJy>1gh5FRJz#4-j22#A8h+KR%mfGV;mumlnvV1Ym~QMAcz$VulP&S#@2O2-56 z02Guw0uMkzLr2L2@DPFH`}W6iaNHo#d91m;`Q6!XW>0Q+I*(u65h+0LS4B32RFzAz zCS|GM4OEJz`WnQ@>pzwFF0umNvU-t|lcL!4@3iSjHozj6!Edu9H(9ech)vq;>z;wG z@vA5v`8-=#k5qm*G^(XCvp2|M)lhVVoa^1xK-^DE8jn<~dlW1Qy?j*C&((2bYMSV5 zYU`^0I87Cd3wChM^Z9!w*Rjs^G?Usr%+2WRKIxdnxy)fF@3-AhvDw$-G*>N_FjF7> zYLGd6jXxHm_82d1ZOf9?a@8+4K@SS#r6y0DiU zvIz~Tbg_=-OK=5z6O3_?FW`6J2VMq9f>gHeKYtem=Y@Ty?k6K z0MREjmvRJDTulz6(+mM4S>bHI0yEab6J@`hW#z zhU8|=z0b@;qKKKXizseyltjx!=%*w2_9$ov^U8nUd zu}{KaPXr28qtkdOQ1m=gs%ttla5#A3eV#Qh?qri>wH# zCYPlmWvSu~REj40I>gB9KbH6|vIJhYdXbZpqFD9swCQoSk43J4UuQ{fvQ}?DHfgJ` zdj`7ZucCP9^Q>b%Qu*P)sJ6sTpc$j zrm@Z@whh%ErKy5(&JNCbHh<6LI@YAzAl;hV?jsEF}3@IYq=Ba^p?duzC}XL{Q9=N_=VK_|J2wTkXu z5cYCKR-qx4E;aCc0j`2?f-w&A8Tnj+ZW%@8b<4705Q%J}c__40(D&1U`qQ7Iyh_(1(jqJ~ziDw8Wbg^P% z8`cjG5Pd>(DMvuX)#NZb%@8n>CC>IsFk^jQi!V<}42WqJPu`K>!<@9?n$5+BF;!H| z7*MH!2MJIVdkE&fq`F-9H?wZ{lI6(umG{q44YJ#xclR5Zb#nhetZ^QNp7=T@Zu8`z z4_I(!NN(2L`?MVV8{Sm$>n*>AI&XV9_@AE78?dnV!9QsKRSt1sWj>x|T4t?{=*QZJ zpR}GO_K6zeP)D~a=wuy|!#a|?(DRI87T$ZH27ZIt^VMwZ;#<4;)^cVR-&%kG;(v|* O@vZ%Y``EGnh5i7y3UrtN literal 0 HcmV?d00001 diff --git a/test/shredded_variant_ipc/case-112.arrow b/test/shredded_variant_ipc/case-112.arrow new file mode 100644 index 0000000000000000000000000000000000000000..199596ff5b7be4dd56ad3be32b77ead3b55905ee GIT binary patch literal 2330 zcmeHJyKWOf6unN?!5D)AM2LdI8c|SK7Nm+S3R?mRi2|@dAcZK}WG7^$^@H`=D2meY z0rXH%($XM4fPxR8q@d&*P+&P{9%fDA28k}$y0deibLZTdkvrY)y~j61a?qxc^KY3Ewz4Mn#M-f`?)He5|u3ac_>ccXcl`{b>W+%XE?p_*T%} zQ^H;@%L+84(%Cx3C*TVB3K+*hK7n6?Z#fwt2~wH9fB%)|JTL4sb*o^?_dDGOk8VF~ z87Md72r7Al^YsM`&oX_OpK+u!-~tlz8Gfv*SfxFerQq+_GGYyW_y*=@_ryK;9okqi zv328z0Yo3)T*?7Zu{BwY{$%hN$pX*zQ!ry)UV|@pNeqZ-1taf>_hC-jaLwl8!=2N@;z~NOx@UK**Iq9f!i+)XsZ{a7c_KsC0Gyd|cWi;BCMI&L!H%$dwsBXpgW=IPZQW)r;}X4Ttaux z346IJYtWDi7poXwfJ@*TU>pbe3VsW|?__`^NM-u|^EA(SUf5^q?t&>lZnd90d+@Yj zpxlWgsN_A)*LN^Hi}YcB#*r?7Cyy;OBG%xCIWRxFC+@-T(8h|1 ztr|ZJAo}>`QVxKMt;u5aCxgdGmUy;bf*I@b8hp7+Vn9qw7}j#*|Po zV?c!}1`?pi4-m|CNfo)~Zf3=-CCh>7YtvXclp)*od3S$+Str-`#2V+3?}@8p>^4vK zdXELq49T7~`#mXp|Ee>Y_XXc~ob2o5dqVR5O(RVY?7jE*mVcFfoS&JGdzqA3>#g&f z#ZNasM~_bFF-zuB~7P6-XBIK<+_*8|=@-2M1W+E!2c_e|8xA0H}?zfW6S&}`VF5Cb(a7D literal 0 HcmV?d00001 diff --git a/test/shredded_variant_ipc/case-114.arrow b/test/shredded_variant_ipc/case-114.arrow new file mode 100644 index 0000000000000000000000000000000000000000..39ee297f2f3da17fb4229e5217825c774e5ea8e9 GIT binary patch literal 2338 zcmeHJyKWOf6unN?!5D)K1Vlk$O_jp31WFNE6tM)7JHP^g6ryO8osgB*kF3{5QIrk| z(eVKkl$4bD017^Uk`Le`0?Rq`Fb2mB5}mJgXZAkl&bc!qw>q7N&+dripm)n6RUwt- zimXUcO85d5f{DHYF>v~i1%8Vxfj5j^dv-#$4u$x*MAQYy#U`bdqbBOX%(e zVK1w)3Js}nxrXsMxCFip#&M9(;Md^$P6kMVRHpCWf95&Q3;RsnT`=WGtO+E_8M zHRFc?L?7Q=$^lTZHCc>)XYd%w63_MvFk@X_gD-bU42Wq7Bkzd!VNTj`&F136m=Y>x z45(1UKmru`0fMZCf+OGGx0x@9sA+>*V^LSmQkMJ#lqR+~&z% z@3G*SA=$HLzo%vIUvnmV_U!vXm9KNt*}Z?;$_GjaR18(3JoZWBP#4g_6#oNm>vv_*} T0*L=T{>R&Uiu>3y|BZeDwuW_< literal 0 HcmV?d00001 diff --git a/test/shredded_variant_ipc/case-115.arrow b/test/shredded_variant_ipc/case-115.arrow new file mode 100644 index 0000000000000000000000000000000000000000..1a62a33fbde7837d7fc4b9bd64de486c827cd574 GIT binary patch literal 2338 zcmeHJyKWOf6unN?!5D)AM2LdInkt253987Vh$Rs20uczL5Jj8pgsilFWW6>-Q95eG z2T)M*348zr5-lYkfJDI$1eSB=VHO-WNOZo|o!R@GJLk@f+}_;WczjbN2fbbtSr$@N zF3FOVrGhU|DVXSM5Cf-wSK_zG0(i~nMNUuie8qj!7LLEWO2BqJ7QGy9dC-*!VqdOsW`nQF0w zLVfg)cE;hE(Zei_d(YHV`1Z*eDq?&UJP6ankpbe0DcL+<79v&NM-u|^=F>*ys*#I-2zj--`;%i=`QVxKMt;u5aJA=na7I?Owf*I@b8hp7+Vn9qQ7-f##B%- zV?d=k1`?pi4-m|CNj16VZf4D_C5wUSYsZ=~Rw3K^wjBzjJhw-A>ay0?VR*3b!yeIU1?Uj_TK@zw!1a0^cFw`yYNZ}0r=<(Zkk Vy#N8k{~rJ2?R}5?*f#%-egSuHe**vj literal 0 HcmV?d00001 diff --git a/test/shredded_variant_ipc/case-116.arrow b/test/shredded_variant_ipc/case-116.arrow new file mode 100644 index 0000000000000000000000000000000000000000..51e705a7421e70742174b157d4840222405900fd GIT binary patch literal 2330 zcmeHJJ#G_06n;+D!8jWfAVL%r)>IUh1yqrh!j?dy11u0oA&NHH30Z0Vll9stiqdfa z4nRT45jX$^2cYBt93rrM-~1Sx*bNe$pY>+;{mz^3&5Ydcbe_DtFH(TsuZnC6sVdiG zP0CWi7pN3W^dZE+=|7eDEwTdMHhPhZi=xDcWH%e0l%zpi?f;iO3rVuu3~)@jnZ5-S;AC( z^pE$)(cI`^o+Z6E>QeaL`5YB7z6l;g*~v)f4dUJy9v}akOZkr-+%ln3Z57CnYssH%1_&!XD=Q-ZyG4~ zlNc)bfb;bY49_xsn4fW^b?_7t@)drpYr0B%E=$4Rv1P;>{4fXRXZOTC_#N6xjQ(Ws7|9CH_G>U>U0#DPcS#J0X$2$ii1%Sm+HlPl;=`B8hLWx!Kpr_k`sATSl54*n98qE&nR}I6pHV_cATB)*S*=T@N=f z%_H`Udg2kHYirm+2+3j|$PV;(!TwBqbbt-qLM_;4scK@EZ|?HV<(XN&x%PJOKga)g Mb3fxgw#|Q{-})1BR{#J2 literal 0 HcmV?d00001 diff --git a/test/shredded_variant_ipc/case-117.arrow b/test/shredded_variant_ipc/case-117.arrow new file mode 100644 index 0000000000000000000000000000000000000000..91c2bd7607260c6c9dc483bb187f7994acf27d40 GIT binary patch literal 2322 zcmeHJ&1w@-6h3JuZHQ5-7IC2v?#e|7>Z$}61{B<>K~bosOft9az~rZsNlGc*`2aqE z3zt5E58%QFaOng15Y_Sf?oT>4%|>vS6VBW@zxUkloI9l5={$e)KqLo!SQXh2QdO?W znv|u2H&7{<>1z-Jr~h2yyT}T7%jiYU&hmWIz0(#>(;*hQ4n9ibOsCEMFw#kL5cV~6 zjbC~G%;nj_dZM!9kyb60>fSJoR724bau#+I4RJ5lNiE)v0L8eX{Gt)Fo zXSQuMm?VjUalsDOc|Lz%XJHg(;VhHt9%p*|_ddyl#o5e$C+D}*P?6pXCrPGSETK>z z{iEG!cxm)FOXI=5nhW1Ky+lQfZ-Pf*dN|Qplejm>y#w73O@AH(+uL-K8(1so?z*s- z8?p%vsdTlD=UZ?Ed~R%Ip>9artTq_@{4xo?Ny#=_~J_r7C2*Kkx1jFze+0o>=1|@;z~N%-j~qUhlEs z%#iG?+4rLC{p-%;<$F%{b>6>VKR=%{U}5jQzt{e&?Bo2(d^}50X03PQp6=>&pY|fL zPn3y64c%HpCu@)_)`2{No@ey4@W}x+avRK_uWDkK-`eH3mNT>b)&c|&|7-k@Z|xV{ J$F}(|^alusbgcjY literal 0 HcmV?d00001 diff --git a/test/shredded_variant_ipc/case-118.arrow b/test/shredded_variant_ipc/case-118.arrow new file mode 100644 index 0000000000000000000000000000000000000000..c5da1cd5ac4a6a3318a75916cf69a2de41f1565a GIT binary patch literal 2330 zcmeHJyKWOf6unN?!5D)=h!6#ZH5Evf1*sy7!j^(W2UsAGLKJPX6SC6!!Fuf|iqcU~ z@&Oc-d;%o}1s_1k2T)T7mUHG|Y#cX8biUS|+54P3=gy4WYPTOgyDO4|-Y$!*3aKnt zWJQWn!WXCzjP(_Wfzy8|@LOaFylM0zXJ>i7?!IXYM`;g}Tm>Jcai-IHrx)p@-VHk% zy4ue?f8g?LVm?vXabK&3O7(s(jZ{t15povpB^u&EtdnS<8rvt{64T2?#obJu*2bn$ zn2v4Rsyj>)1>>9*tn+OCj?Tg;%))UdwRfE9!Sp)Gkj2@|ekJF(Q&W*X2!~0g8Z2R= zKKe(yqww75L6*kdmue<_=ky#EF}?=whv}=K&g#UyKG;9hozV2B32blBN!Boz(A^8d zUarbIG^E1iD#qvF68I(<$3Z@WUxT-t43Gq=Oy9r%%5$C<_L;g3Fy%+B_T#7bpEL}V zyKw}Syu>`adnK{=E+{~ zvEZ2@*|TQ9CuQ$nb*77MzT;+JC*Kp2_iq|$dSLIpzqkCW?Bo2*eB8^V%v!fT{kZ+c zJc{&~CH9M2;!#1@R6WfQx2a~E$e&&=Y@wYP)+ QIsV6+`w{oCW&RWW2GE;yng9R* literal 0 HcmV?d00001 diff --git a/test/shredded_variant_ipc/case-119.arrow b/test/shredded_variant_ipc/case-119.arrow new file mode 100644 index 0000000000000000000000000000000000000000..5d6f695d629360602cbf01c7124d4b07325a66e0 GIT binary patch literal 2330 zcmeHJyKWOf6unN?!5D)A1Vlk$O+{f@kSek$Yzd@vfCU06MA0TYAuFvPtk*_Sl#UPJ z11KozDB%Mr_y9^ifR6|)=gh;blej^m^R@2G-sjvocV^^HxBKY%J&_#rZdqhiNL9Ha zD^ivUzCfj5s;@x|oc?2p-y%!k4Wk!1Imz=i_f1zrXF6^629Zvh{jjH@ zYy8UdO_yf_^O4F9hg!8%s`mzIq#BBjkh5?%(Gd4zokSzm+B))lc?{U7qgW*}G5A!pQbO~HQLcYO|brq|$=du)>jx8hB;D+%|Wxl3X|Oe+|9N4yVn(uQj`7azt{P%&db zr8))@pvVsp%ymgMx#@0Z&8#KMf$3}8SUFT7+x2;Oe}Gvh*Z0I4=aKJ;t7GamPxg9` z1E`;J3&Uc*E#LPEYfE&3)4rj?x|`xeR`l#+gp*onEAqdN=H7 z=xRUneADIGzPe90&Ofeht3oWPl_{W%~a8SDy2{u+P-3gDF33wI4mX|F~hG z+=(Nok}2^gM5`Y=D^NEg5pNXQrXv98G~?YS%jf5#RPYw*Jyn4jGf_uzMEW5vW) zjUNUOeSC8%2SCNvWHI`a!DA##JloH~jCFYpzT71-Af_dZyd&O+IcdW+n~M))N~oAI zph6V`2~gw*2&VoqSJ7-oI(2>4ClX{@(JhvXApK^KmbeGHczu)@q-; z{r2$(9cB>wMK$rLpld7GK?RbM`6np?BAHYWhmUHH@*h${EkQtdbcdHDx|7h zk`*aS1z(_2Q0QwA1E>F3;>9*tn=yoJ)MP7n1w|qwR4c^(d;_On8n%5ekJF()liY%3&%;OS}b9z zKKe)7lkm*wQI^L2SL#&w*3lU%Vtf@m4Aa+Roi&MjbF{m!d!gx18`$2UlU%`EL3hsy zd$}rW(2zlj~vE8v@890&Odeha?mWPl_{W%~Z}cb@aSu+P-3gDF4mbe}xC|FmVG z+>RrtVQgYINOZo|o!R@GJLk@f+-kQUJiR57gWfEQtO}_t z7i2|>Qo4VCKMUK**Iq9f!i+(|UVy;vvFKs7cGy(OlXi;BCMI;xFL zqc9!YHdS|+BnrkkD_G~#`8zrbqc97{nbgifrU%pOBtsTwGy9dC-&RdUdM_L%nQE|v ziTdatZI8m4(St0FyD!wK@U5d6Dq?&Y+z->2L!H%$dwsCGuREdXPZQW)qmx|1Ttaux z2z$9IYtWDi=c^c>flJ`)U>pbe41Nv1<79v&NM-u|{a2pzys*#It%E5)Y_%Ufx%;?b zpxlllsN@~a*D)BLMfxy5<4EVg6G+Gx__40ZD($%}1b@dC5o_?n9GIWo6ZhbEXk*31 zR*fG95Pf`eDF;Bs)?_jIlfh#oOFY}p!Hjiz4ZhqZF(9TTjJzY>hdF7(HJghMV@jx) zF`z;f0|`*%`v~T`q>5a1H?v~alI6hkwP~yz%8>2)yt`k)tdr||VvY02_r%pPcAF=A zy~l!QhGfs0{hpM)f7O}Jy7{J?eVu$yNZ!9`r0IdZ_x|4UudPj93HPyO{uBKMXfJlr literal 0 HcmV?d00001 diff --git a/test/shredded_variant_ipc/case-123.arrow b/test/shredded_variant_ipc/case-123.arrow new file mode 100644 index 0000000000000000000000000000000000000000..79fffdf759429484d25f05980872a69f20841955 GIT binary patch literal 2338 zcmeHJJ8u&~5FRJzV2nWlB1DnG+KR%m1XW~F#1crRA;ID$g($k@F65;1gY(%aic%m^ zQqV&|h4=^X11M;bDCuaZDN*3~zI~Vj+YJ(3#+u!m*_qkdZ)R6+w%cp>E{kNK=kp?q zLMqD{S&*WX@C7P(V|@jpXZ3Fi{1%x5uV}r<(NUHy*>A4-qojvP&VnB&Q5q)oPA>@K zde`rS(A9oqS%~uV}->TI~e21_?&4Cmu366WQ%#?KW#F2zUKq zoT>&>$mK`-U~A-0^&X^2)P1NX+&2%WsEF}-aNkcJ4a2lf-0Op#y|Cl!`m6%mi!_pR zm`kYc31KZ4WeFNm;dB+_eQ*i93`RT1d+<~6H7f%oK`34KuRpVl{lYp^cLhv&z1hBX z=lbo2hH@(kppuv9uTNlD7HPxy^dp@D=a7&Ouw!2NEKOe)yuW>m@YVQX_Vmx3iDz&p zG`^yJtJ)3&h&Hacls%wgaxxhGj^HqoIrjDgFnw)YjW17042Wq7Bkzc_VN9;!n$5(A zJ|$#KA5fu+fdnYBJveh;QbjJHk%Ber{$wo+U3c z*Y$5}FM=oEZ+v`v^V!MQUzYFEa0;<*R1=p9s<(j3Rv;PdJ-G_~C9vy_R~ERw-Ei`} gW$kGhtxyf2c0-jzi*gjAF3 zvL+R&;tS;Yg}x5axB8DBzeP&mZKD@CKhLr)`%POgPsW&J6Z|HLQk}Gh<50(~Q83id zHGgDT*XG&Ae5%ruiB@fu=)pJ%Ra4OsavJnw4e=n-aX3}&-BV|Y=w+j#QL4_Gg=ron zg>6@jW^t@wT(Ls#yqtfi(;y7epva{9C#jzPxlTM|ad~F9lJ(nbs!$&Uvp7|4matGC z{loovaAEW`O`_4ES_q+Xk|34o`}dz&#`D5HQ}+N&`Dv&3?8W2fZ3E?g z6hbBMalXES;aQ;%^D~Zg4ZMJae1RY9TC6fZm!AK3Yz46fKg_=Q<$K~D+z#_tF|iHf zhXF(%*IddzP|0h`G5VdsVI(DCZ(!ER^&PRsRpfeN>nPk-$xiRE zSho?4A~OGQvh%-Z?Jkb*2g%jBW278ses-`d-?YA%v`>`=DPg1xAzn7W5@hA`USc1cU=Gg literal 0 HcmV?d00001 diff --git a/test/shredded_variant_ipc/case-125-INVALID.arrow b/test/shredded_variant_ipc/case-125-INVALID.arrow new file mode 100644 index 0000000000000000000000000000000000000000..3640a6980a3639149bad05b873fd3fba309d02c4 GIT binary patch literal 3986 zcmeHKPiqrF6rW9#G}KgDE!u;Ju=LPF2q+#(@X&%nFJdf;g<9H8vTXyKv~D)Fr4;-C zJ$Ufw(U0N5Ll1s{9(wR2_z47Ef4|vz+ikMZf>01L@OI|C_j`Y4=FRTx(#p!SSND|) z(dP3?O)0hsbyMY4P8Bdhv$C3a-}wZgyD=CQ|IjayXe$3=&j!h zT))!ZZMt5i$&f$*vy;ZufE=ugByf&l1Vt z*2{L&-Ew+fU{{$!B0u7r8~aY0_0T*o8x7`rk?2qtgbW&{=kaf|u(=}xE3kf?ScH1K za6|JY(PyQGI-ey--9lf$ zNgG!@S5s;R5;kkHgy$-#0J;alQB!A@QXfE%4Ud9VVD;HK{~3lnyF5pfJphrfEv-C% z`Q$}auii$h36VPa6^5rE#$g_FAlmU9e*{r>1C%H<^#%CP+D2&`<|H6a}B(lv>G5tv6BHyD2zdWr8~>;d;F@|pQt{BO~m ztP17=z4Wbky zqk-^(pEr8_-NSgn@X}qC7k`V!Q0BSZ$JY!m^IJ8%bi#}MrcsFh9RZqt7%%uegBwrS z@vMt~5nmTCais5$FLLSo_2jv?(>Y~j#e9ws zt5(CUyZ*MWdtz8uMO1$tr=kdsXN#wl^)aVM(V8ycPQ~_)FITjKs58$AZ8y44jqX!d iy%mn`Q=|J-@{ZAePaEB*^sDv1yHC~domkTU;=cjaOZbHV literal 0 HcmV?d00001 diff --git a/test/shredded_variant_ipc/case-126.arrow b/test/shredded_variant_ipc/case-126.arrow new file mode 100644 index 0000000000000000000000000000000000000000..e5071f15a83526694f0d6227e2d76448d5e4c666 GIT binary patch literal 4930 zcmeHLJ#Q387@j-tnG2keAy#4`BAr$gE-Wh`3S?2(5|L=47|SS;z#?b3&S&=hoB8+iGv8Mc+Gg05J(QWxHiMhm{pKD{3g zP^mMZU*k?<;+5VYGToKF?it9ofPd)i3{fA1$?%>DSHsw}2XPcGhXjI8bgOH?-tCxf zv>UG8I*^_YQC_RlPr}3Hv}<3-Y2U4|zt`=C=q&Nzz?1fSCee{jbZRMV4HL6_)K7Ph z!8yy~$>X+{!^qs#d)*{lWeA!0h>z~<>muzmP94X32;-Q%9tNJu^D|_+re`qzqB$yd z@jnHa59H{mCMR?}lci%JWK!{Cj3Q*pOf1p%VP=(D>B^SIOUNgRcQ=VU{a*@0-aafE zNP(XX?(dh_MZIyPKOhUXTiK$fP|$o*TnBn zpgQP02-{iRhTeJ5HP2&{sS4UW8#dmLMm%rWrtA`k{Kop`jho+Xt=b8_(}^Hb!9SzX zRS=L3o-a+xK-6Pgr$Cv!Q6*LznspoaXbO$w)DbatP%2IQuwW`ib!-s&Wof9d{EgeNzSK-$*L{%=Y&i+u z`RASd8gj^;w>gtg}7g5U7GRAs#2e-@yx zg%|rfUM}^igSLwUhTEoNn}JILy!3q*cuZU3=3OVgj)B{*S^V<8;+t63 zxjpEaXc)!1r*%8&4Ej5RI3C3IoS+i(uYuEI7CX2B&W$HfS0P^l(KloHjh7nAJOe@* zpC69r+n&19H~jPsf857V<}FqDw4J`;`3X4jjvjw{9-TmP_-6H=pyBil&-<=v&mnhi pEuuYr!;ilUcz0a~@!aRRu0hi`JSM?^e}3*a{2#by*X@62{{nVrz?A?1 literal 0 HcmV?d00001 diff --git a/test/shredded_variant_ipc/case-127.arrow b/test/shredded_variant_ipc/case-127.arrow new file mode 100644 index 0000000000000000000000000000000000000000..2a7bdfd3f77c4deb6a117579c48fed80f2bb076b GIT binary patch literal 2322 zcmeHJPiqrF6n{y#b&0D~E#kq$GW6gf1Qi4cUIO(ds6nw%OPzM77}z9rvx$__@6e+k zz=I#dqX!Ru06&IDb^ZNjXEu>`!IP&vcr)|8pDp1>nPq$hr_(a#hx3 zRYIHqjiBO3AOpw$*x*^D34CbzB0qi>#fCdmrf1n0n_L6F&jz{4I>T{d(#}W^4S1Ws ziek(8d4T;y<;O=xbya2###y4;ii)6fy`LJ8hXa!)6V=^0@sa_xoYi2ItJ8KRn(3^P z?WoZ-O%;SS5lZFr{)Z;liOzN9N$nr!X7YEObjsvP&pwjV+ik1F9O`MBt1eSm$dCHT z-b`O;&AXhl-95d4g*x5<9_j4e)Z`u3yfZm?YlhnP<_YS03v!X`*h6&olCY2KvH=dE zaV5sO4Ge*|ffxn(20R09IT{5LxUxO}{<|nRE9^0O_kpCJ_4?0WK7P@)W!xJiV96O~ z>kAN?P3mj{X-BvWT&M!5HRKRr6^Vh&aWP}sb0E1hPw!_HXIn?h3oO61mFl*otytT5 z!IpRSO~%niFa4JFntSl|k|t;THITN>udOw2%zCiCAy&Q>uS1`d;knh;iZ&rKrVXGG zVkNgqF2keq)#4*yT6&|9xXW<6ZXbztA7X7iW$D literal 0 HcmV?d00001 diff --git a/test/shredded_variant_ipc/case-128.arrow b/test/shredded_variant_ipc/case-128.arrow new file mode 100644 index 0000000000000000000000000000000000000000..9d4264434a005de1b07f4fac703463e5fdb69824 GIT binary patch literal 3978 zcmeHK&1w@-6h29l8fq%77VW}C7`o`91Qi!0xM)G4J7ZBS)Y48mQw>bgIGI#SDfj?g zxNzyx$8h1I3m>41E_?)^K+y5~?#!7^Cy9!pAmqZCd(S!F`ME#m=4N?y_5S0VA|cxR zxX6^SO~@q~m!gy~Ldz8rc^PaWmfs%ZSR@ZxG;)!xFJU+nkEyfw{9SZ14SL}>1J|#% zb{*HNw(XV+*=yiW;DFd;^aob3-*K&)<-6OvzGE#|1cDFjhUbFaX}X@%wQB1Js%a7x zTQ%E(b-0kI_G~}Vty}G$=UGq=i4cMN{kPn}cI?1TJXwwX!0n!okjjsEXLHXUWIZ&`^G5yoo+Uce1tI;0gY)>aSy9YxfoR7%ehZ@PA}Dpx;d;K@|pix{7=!GauwK} zMm9gp<0CNKr}VBd2Igw)(sNDcU%;$5ugF%C9gl3q*jAv|xpUw3;=vH|@mLp~nKKW1 zW+cyh_X&u$v0t;N+;#2^&sYgx-rtI0JnHZhU~C4R7uuANF>TOt6@0J^;Y%DC#$+qY z)%X{qY~sjR!R%!xQsX-TK8nrXlp_RXj*PACnhpl^>2*n~xf{X^r zD}FxKYhE`p_+h-_7h}BoTguogulSW%uk)*C@X9NGE!M05tqfjy#Xrm7I~hD#*1Fpg`(ZqcKEJfOJgF*?|*sTX*~rh;$z z`tP80rH>57B z(!dTBgq1#m7&`rzfP0Y|__5WCeE(4tn{H2=o@OH~aufVMOLCKS2BX-douM8W=(c_q z#kR}y2~dYvRJhh9U=GWrSs+d1C#4m=eo+I4o-73{U&;CHwN-46^d!wymo3coNB?+# zs;`XZyIhLhJHJ8&9B+Y-b#^i_d54&H#)t3BK-<|o1+I5s7umqtz+~5ib6l5AXh^~J z7M?rc2KX)*M?o&YAHdsA20(&TcIMxH6$Q@CJALeHq=^A(r1qrR61TE;<-+Z^_*$l$JbF9MGVu&@gKOZaAo0hR+W0~96=ll8r zV+(!V*-ETt89Y`}AllR2?upVu=Y722C`c`2oh({ rQ^a}Pv%50qX9vt%S70_T`ZsOd^1HhHt}ef;|L42<75A}c|NHy~$^~sg literal 0 HcmV?d00001 diff --git a/test/shredded_variant_ipc/case-130.arrow b/test/shredded_variant_ipc/case-130.arrow new file mode 100644 index 0000000000000000000000000000000000000000..f904f49dbd08243ddad9abd12a60a4de8c322ad3 GIT binary patch literal 3978 zcmeHK%}x_h6h19&ZLk!$3%I}YHERqH-7`e#i_fBWpj;V9^!d-MS1$rIUB0ntE zcRfESHQc%n**oA*;DFd;^aoC~-}Ifb6Z%`bq36sy1cHy;YT$$2uK9u2a>{E5s;LoW zt!j*?v>$#EJ_2g9dBfoV%PS9p@v1g5BaqIJr z=Wo01Aacq~A(0>P-o~Cg$a-j==Z*UFJxg?`3qtx02j}r;vv9a0eJikjm{^2rcj1QS zOUOUUw-tr8#>;^zuOAMW$N&+JvfKbQ-SAc0k4mh8Qfq6+ue+vJDmdpe1j%*uc~sgJ zVO>qiG$d@Pi6VYiL3z+^5K2u>QX=m`4=s-aD_~9Soc`!^Si7tv%I<;4SC&?uJbU!C zY*ufh=0PM!KRcac5aTe9SrF}5#~(nHT?Hi$ntTTSld%zF!<^J+5o5+;+$!iBaI?5i zAjwj$SPc98&Hz`5u^D0+n||4{Aaf?sF9P$x;QHe)L$C2&j0504#y-;*#s3=5DOH5s zX>9Y`JiY+KeM;^cV_>evE;-j^{u#`Q^NMYyyW_E~7~3NBI(P27UOX5=p0m36%$#}9 zGb3ra@V;xJY#wM^8Qu~<57oCfUy~LUTBj?#YDReJEsD}L4LHQ&u1yz+|Q>A^R9@MKx*idXez zdBtgek-g%E^}Y7Pa4ILBdpnhtRC+9vb*oz*-xYtm@x0ULB&dK>pF=eiz_F%So!s|X zGm6)80dFa`x4cx*&aeBpu^YXkM(?Q0z6M9{sL?wr`L^hNpN-y8=9>NQ-cc2NCzi~A F@Lvdz^qc?y literal 0 HcmV?d00001 diff --git a/test/shredded_variant_ipc/case-131.arrow b/test/shredded_variant_ipc/case-131.arrow new file mode 100644 index 0000000000000000000000000000000000000000..6883e4bc03afab65bc7f85ac21729310a44e0045 GIT binary patch literal 1930 zcmeHIO-~a+7=9OO-PF~T(hv?FHWLpXnkIT8J=ue2w83D4gbeHuH`%thoz@W2J3qpM zKZ8HyKj3QW^UTiF74_iJgcsh~dB4AAro+)_|L}##3Lu*zZ6UPei8SSb#Ml9%#+sji zY;!E9cz$;}h zFN_+f(wt4pO!X8ML0f%V7?9_=DYBUw99;!TPAzYhPpn$@)}nlOi+Np#Pu+S4QI%HlJ^ow`pt0k z_TB4)fh+SQ&%lBX|0wUmcT*k;r`z_kjy7n=*wi6B0@f;)S5r%jA_ zv;jmZ7L>Hwg&|`S64~)LA#rhJv*BitIw_YH=qfhv$`>GWWb6Rme;|MP*85Ty+_edG z_=Y}k{sf+jIs`iSr!4fOu2;{K{C}YtJLoI#l>OtD`6cLL6WvWfaYoEw59>3xt8Cl8 pfYdumA^Qm}XLtYJgr}}{`|jTxcW=XM;qSwL_uf3m`5(I9;2)XqC2s%# literal 0 HcmV?d00001 diff --git a/test/shredded_variant_ipc/case-132.arrow b/test/shredded_variant_ipc/case-132.arrow new file mode 100644 index 0000000000000000000000000000000000000000..4d062d96576cbca732423e57b573fd149e65aebc GIT binary patch literal 3418 zcmeHK%}x_h6h19esaOmsF)Ua#6BaC*23?WH1tgld6A4BVBslF{V3KL6(auf zsNiM#3PjK9zZdx~f19{>4L?z>^Ca|ZK7o+apcyNOCy|Q7p5HjUax)Q8)+*|x{&g*zx(JeN?$Gb_ zcy^I2?x^O6>LlpLso!7X|ii6hsQyFtHV077s`0@+;bRLYaE@q>q?t4GNuivP{jk~7_i?MlT?u>_N1-o zI5OwywcIt+crHLTv3VBPV9Oktmm{97dF0mG=8(biWT$s^ShN;ZJG!_VR(AMJE4$&Y zvh%lP4PCz5&XFDSWEcOH#mx`d-N(E0@6>md9nOu@>>bXH)9{>UZI??@D9(AMpxFv} zAKr!djZsTARdTBPi8frX3cBhZx~&3M3{RM@xe0u-@ literal 0 HcmV?d00001 diff --git a/test/shredded_variant_ipc/case-133.arrow b/test/shredded_variant_ipc/case-133.arrow new file mode 100644 index 0000000000000000000000000000000000000000..d4cb7b36fc0234c2e07ebbace20aba35dda6427b GIT binary patch literal 3978 zcmeHKPfrs;6n`ykZLkrTO>rA z%ZW@1+qhhjoMa`B5n3wKlNZ2dto-g6$0BLaf{}}Cehb5CJEqRvZSSCyDbTBSE%4i= z#*XLLOHH@oL-rQlr82>rej?6rKS?6m!@own!9I|PCc+-ltiyIu3^UdJh~?W?9n zl(nig1Lt7Ar`mPfJ>8nq?AGfJltUs!;6eWlKX5%aaC@Gd>R#Y?&c>;CnOx*qBU#+~ zyyN-XZnqvdWu_3zk9com*G;k>n&(-g!F*2>9qNLRLBr%c{%jTwcVu7%)(>KfQ0*<; z(0mE`NBMSwcCGn5G3E7xgoz}GP|9)*)NcxB!@r3@Cd{>%wrZrJD%hBAj&R-VudE3fd6Q0#Mm$=wOPcNu^6`s`U>1E z?h{C|lq(j)0lzcARU&MLSjMJbwk*ihB>F{Q?it)*{AK7hzKd}kxDS!f^snN7iRP3l z!tNxpIX92bz;K`9yT%xptFepEHJ*P4v*NrWTj}k1WGlwD2))jo`>q!chLGp1E~=SR z4_Y&l=Dqt2L|g0E>?wDhd&3&b0%MiXs!7wIU zL9W{`M#039v5eWvR;0%7IQS?wzbS_Z${ZP6+YKEg^zn6xt8Gp_k@DK>I!stZ(e@g5 z&hm=CWqIv>msfuaeeo9i_+`uM{3@2$PI=9D)hg8grUGrRyy9P2x#C-Wc(Od}idXeT zdBth}x9!#Mu%7X_w^Lb3rN=T^x4Px=U6Iu2L)Rfmf1Zr>PW>dPfU3{IQvn>$6i+Aj zdDe`gwOqhkitPY}f~(K~ANj*7o6`rl`xca*ti|GRfo1>cD! H^B?>txs>#j literal 0 HcmV?d00001 diff --git a/test/shredded_variant_ipc/case-134.arrow b/test/shredded_variant_ipc/case-134.arrow new file mode 100644 index 0000000000000000000000000000000000000000..c18e1b3d1461e02ef2778f34469d9f8192a1e202 GIT binary patch literal 3986 zcmeHKPiqrF6rcQSLrtaCqCI#BOAkGSfa0M94=pJ4BF3UvsHNRxw{2jP*3G82l!70i z2M-=S`Y}9s=)n)rLl1rgKY^g@?>9SdyG=G)5DG#D-p;)De(%rByxE;yT3LDa>b_DT z+WeSOQ;Kau-Be?$po$owW%9AS1U7Hv4|5zVl?5$mxl)@y!f?insk8U}U36+1^ww_$ zu3v5MI<8mk*licG_rRaQ0kNm(kF4Nu&$Vin?>2XR$EsKaf)DJ5=Yrj8xt`OtYHLTL zX%S_tTAjc;uEeT++mCf?R;TBA7L-FGMBqXHZ8xwTJFsImyO-m5Jth}HRbi=l!>H>P|E5yXwUZF^xUAzGf?d|cigtE&q^J2K1-0gg}#WB zHm-QCrqm21Y?;Y2o~xiD=pG11O`TBkY32FL zCogJx^)^}#MC#;M7@mR{hk49_XvcH>5k%PyP@>S(7vMi@8>MZSlh`a`%vg-u0DT8; z4(}643X}^L!vVjuz||vco><1FU$z2B*ChHyU><4QVEkq1CBBWZ2i&K~XZCOLzeRJ( zlwo%s+59n&ufTAhlDozjn5(u+&NZ2T9<$=SB3l*jcx0=zZ5eu*JNI2)JQzYgW^_@_ zTzHV0QCYrsUxR3C{OUdBu5)i#V?{jq{uT`5QHM`}wn?iO+7yv7ZO}4hJg^Mm0SbmO z*-Gk;`C^oG9F@!Ky*!B2cus(iV)L7Ff}qTiv8COWK}w%om!#U{6pMtHUe;mKAj&Z^ z8VE1=d860gJ&YF&FWpsn@waFUWuD7@eBJOezg5FaC%o8i8in}Z5uoXZ@q*toxbcJ? z&${>*@pbVMNBaKwBA32jPo8@_T~L{9E?=-4O^5diHtu0IlT(Gt{8KG3kA7cCUD}vy z)oQv8*WcE4PYmm-gz6u|sVITt+2Sc>eaz`mw5AKVQ?b3{%N6Y)>dbRO+l}s1qx;lV hZ-t}#)aX8yykqp=(?<6x{c8R1?o)MqCzkZT_;2E?_=f-h literal 0 HcmV?d00001 diff --git a/test/shredded_variant_ipc/case-135.arrow b/test/shredded_variant_ipc/case-135.arrow new file mode 100644 index 0000000000000000000000000000000000000000..369f88eb1af3e4c7eec562709ffdef46659aafcf GIT binary patch literal 3106 zcmeHK&1zFY6h3LwG{jq}TEvBmFt~6Lg1RcfMGFeXqb4_{ zB6Cv33Zqoc_am@nzrQuZwFo9zw)-Mqe+-99eodJ^N)Ir}Ezqa5lbN*HJ%~-x?CGw- z-UaYKOU}+W%qJ>4J~XPOQnPoE#;TzR1fS{M#DLxJm?ZA2*5*l&bcpg=onEF+8@cF6 zr@3rX^#)0zAe`XAfv?){noP$!)43(Jdz_j6Wk1P)!8yyplgDj0RBZP3AjwpVAr#^x zKHfRfS(Cle?C*UrU2S{n4E=Bqa*^AZtLVsiVgD`45_ULB3w1o-f~uhVpjprc z^diuj=TTSzYkT$Gui=nA&fetS6A<|qtL>MspS^0?^6zwFY{?SN#0Ch+h&^2g1wF=f z15~J>tO2`X@v=a99Yj3O#~O$>=0Uztk^%e0*`H$!jGuUpQQMz^J9pSJv9uqsb6DE~ zV?G0h`6*_Lcpl0Tg-|P*;rnla zsO#<8+%Vg$1wB^ra6K2e%X2X=MTw6o-wJ$~Lmt?-9IR_`1U}&FUKrwfIKn^j ze84YzK1`GN5dTU5`gsx`@arRZvh*F|3*x`!NA-fe>*k^!b6ez#>+9^WkBDKNB6L#) zZx?}Mud?fz&j)tpe4K~G@X%vi&QHziO~0Sh?yQej3&y z9hU1&UkBxeSJ%*;0DlSx#J)#==tkX^b}MeEtIg1N7hM7&N8VnbA@0|7;J4k%_Mup6 zMA=!jM&uqXrltcgOl{k4qZ0%!i~}Mh;Pd|LI`Vuk@={OkUN_S1U*iNFCYN}YNEWxV z==yr!>jaToVG5c2i1&97yneIOJiRdVx^RxgtK-8{eeVkCK-V?Czuy}55An|ueb(_a2-dAsc>tTi6F=k{gi=x9h?9~jQ>6>!T7pLBFo=D9DotB-ZvGxf27vpom9 zQkT&eQC*XYwKSt*WSK$cja4p`Rt_|2(2HZZenamF##jSDCpE?2;SzT`n68p)Xila%O&d zkTavkxDOA&oNN7>J>stOOfX^*E$?i>Fdl8V_6MF9&M6{e&H?2dG$brV{1S(NaY-e0 z)&5eHOe{5?H|u?zn9)u_PU7>o?OdRYtB5J_T4T*qjW zb>ncs%F+#z#s883Z7;I;U$J_wKWRPb(sqz6_^lL~3`7?H53ODBtseX!S@6#-p0y*y zvG^C=u(IH!m-)OevLteEFBsAH%A%KE?AdtS>#Qso>3ipjUV7SphancvBy;C*Ty561 z-}OTe_dJR9!F~o@LX{LyK_y7Zy64pm=y|rJ2AAjzbZi43Ct{|ZGIK|7uF;z-`*{8H gN8#wrHF|T6-dwzw9dPzt@jt)0*75tWYW}1D0MQcv761SM literal 0 HcmV?d00001 diff --git a/test/shredded_variant_ipc/case-137.arrow b/test/shredded_variant_ipc/case-137.arrow new file mode 100644 index 0000000000000000000000000000000000000000..7e6834203f2dc91176ebe4c083bc95bed9ad1bf1 GIT binary patch literal 2402 zcmeHJy>1gh5FRIvV;KVpM2LdIT2W9~79a{_(O7~G39vv!3L$KAYjM)q4(GEWM0tli z5+#L;JOB^CLqwSG+n=+eKlH{hK^KkD-grKg!JLVw_~mrVI0yBBZU zw%mJh1eTm)27dyfd6PQ#fwUuB11?kxR0TPdu{s^*xR^2RIk?=JXYjLzv#qz~l`Y@d zN`6~sE7rELV9Ps)VZhNwcYa;H<{m=5WQA|x1CX}fudOw2%zCiCRjhndL5Dsm!?P@` z6>X}>m^Oe?0}FxgcpJuyOQ_2o|B~xAmaLX-kNI95)IhuVeCtyn^JM%0tbGxLy81kF zwIVw3LoGNz1UGl{xDCs7b>s1Yr^6QLkmo}$2>Fm?uY=r+|CtVUMLYd5_aG1ci+1*f zGTyt6?r?hRpxA#-(le~QQ|SGfFU%(&VBRn46Kd9W`7K_4i}|LO-{OjWJK?wMe}9XA K1gh5FRJCV;KiI!9o-i)+q%=WC2y=%8@{#Kq5pzASG6E_FWQ7wiC|hKoq&- z5hy5lfII*Xzynb63@Ioms6pcRzP+2{@Yx{^R32$|c4p`AW_Q->_07#E&+mx@fOk2O zvJi@LUGkEX0%m|rHkOybW+VB<9Ooh+vS#EW-@XSyC7N^1KJkt~WC{4n>-er$>m506 zx7N3NE@agoLGUK>_`%G6l)QYS+ zecu|b#!Dx*7q3nA94e*#_FUg~Y~PMOSgljv9VGoTsNggM3pZ43X2s@*)~TB^H%)nQ}d zKhKCf0G$WoKnYmW8v+~l6nlZP`#|y=>zhxXJ>06Bs_b?gh~xuwznU1CMfzO<(vJS$ z11Y-(Ozf9@f&PWD5o5#L)TV(sV=-#!JURAbF>a#dda-HGhkSA@GEF*W3LE#fubM#L*oo;4>t z3W?TMHX>fhG+y)JnIY3u|@GW(73ZFn~QX>QFRx xvv-C*aS}Iv{9`+NXQUsAvv&r&m%TrGXP9@+?45zLz&rV>?~ISQm)FgI!e1-0uYUjl literal 0 HcmV?d00001 diff --git a/test/shredded_variant_ipc/regen.py b/test/shredded_variant_ipc/regen.py new file mode 100644 index 00000000..be008f55 --- /dev/null +++ b/test/shredded_variant_ipc/regen.py @@ -0,0 +1,79 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Converts shredded-variant Parquet test cases from +test/parquet-testing/shredded_variant/*.parquet into Arrow IPC (.arrow) files +under this directory, so that .NET tests can load them without a Parquet +reader. The Parquet test corpus comes from apache/parquet-testing. + +Requires: pyarrow (tested with 23.0). + +Run from the repo root: + + python test/shredded_variant_ipc/regen.py + +Existing .arrow files are overwritten in place. +""" + +import json +import os +import sys + +import pyarrow as pa +import pyarrow.ipc as ipc +import pyarrow.parquet as pq + + +def main() -> int: + repo_root = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..")) + src = os.path.join(repo_root, "test", "parquet-testing", "shredded_variant") + dst = os.path.join(repo_root, "test", "shredded_variant_ipc") + cases_json = os.path.join(src, "cases.json") + + if not os.path.exists(cases_json): + print(f"cases.json not found at {cases_json}", file=sys.stderr) + return 1 + + with open(cases_json) as f: + cases = json.load(f) + + os.makedirs(dst, exist_ok=True) + + written = 0 + for case in cases: + parquet_files = [] + if "parquet_file" in case: + parquet_files.append(case["parquet_file"]) + + for pf in parquet_files: + src_path = os.path.join(src, pf) + if not os.path.exists(src_path): + continue + + table = pq.read_table(src_path) + dst_path = os.path.join(dst, os.path.splitext(pf)[0] + ".arrow") + + with ipc.new_file(dst_path, table.schema) as writer: + writer.write_table(table) + + written += 1 + + print(f"Wrote {written} .arrow files to {dst}") + return 0 + + +if __name__ == "__main__": + sys.exit(main())