feat: RunEndEncodedIterator

.github/patches/enable-binaryview-integration-tests.patch

@@ -0,0 +1,29 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+diff --git a/dev/archery/archery/integration/datagen.py b/dev/archery/archery/integration/datagen.py
+index 83913dc379..7ace28e1be 100644
+--- a/dev/archery/archery/integration/datagen.py
++++ b/dev/archery/archery/integration/datagen.py
+@@ -2003,7 +2003,6 @@ def get_generated_json_files(tempdir=None):
+         .skip_tester('Rust'),
+
+         generate_binary_view_case()
+-        .skip_tester('JS')
+         # TODO(https://github.com/apache/arrow-nanoarrow/issues/618)
+         .skip_tester('nanoarrow')
+         .skip_tester('Rust'),

.github/workflows/test.yaml

@@ -193,6 +193,9 @@ jobs:
         uses: actions/setup-python@e797f83bcb11b83ae66e0230d6156d7c80228e7c # v6.0.0
         with:
           python-version: 3
+      - name: Patch Archery to enable BinaryView tests
+        run: |
+          patch -p1 < js/.github/patches/enable-binaryview-integration-tests.patch
       - name: Setup Archery
         run: pip install -e dev/archery[docker]
       - name: Execute Docker Build

scripts/update_flatbuffers.sh

@@ -0,0 +1,77 @@
+#!/usr/bin/env bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# Regenerate the FlatBuffers helper files used by arrow-js. Requires a sibling
+# checkout of apache/arrow (../arrow) if not provided in env and a working flatc on PATH.
+
+set -euo pipefail
+
+PROJECT_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
+FORMAT_DIR="${PROJECT_ROOT}/../arrow/format"
+
+if [[ ! -d "${FORMAT_DIR}" ]]; then
+  echo "error: expected FlatBuffers schemas in ../arrow/format" >&2
+  exit 1
+fi
+
+if ! command -v flatc >/dev/null 2>&1; then
+  echo "error: flatc not found on PATH" >&2
+  exit 1
+fi
+
+TMPDIR="$(mktemp -d "${PROJECT_ROOT}/.flatc.XXXXXX")"
+cleanup() {
+  rm -rf "${TMPDIR}"
+}
+trap cleanup EXIT
+
+schemas=(File Schema Message Tensor SparseTensor)
+
+for schema in "${schemas[@]}"; do
+  cp "${FORMAT_DIR}/${schema}.fbs" "${TMPDIR}/${schema}.fbs"
+  sed -i '' \
+    -e 's/namespace org.apache.arrow.flatbuf;//g' \
+    -e 's/org\.apache\.arrow\.flatbuf\.//g' \
+    "${TMPDIR}/${schema}.fbs"
+done
+
+flatc --ts --ts-flat-files --ts-omit-entrypoint \
+  -o "${TMPDIR}" \
+  "${TMPDIR}"/{File,Schema,Message,Tensor,SparseTensor}.fbs
+
+rm -f "${TMPDIR}"/{File,Schema,Message,Tensor,SparseTensor}.fbs
+
+generated_files=(
+  binary-view.ts
+  list-view.ts
+  large-list-view.ts
+  message.ts
+  record-batch.ts
+  schema.ts
+  type.ts
+  utf8-view.ts
+)
+
+for file in "${generated_files[@]}"; do
+  if [[ ! -f "${TMPDIR}/${file}" ]]; then
+    echo "error: expected generated file ${file} not found" >&2
+    exit 1
+  fi
+  install -m 0644 "${TMPDIR}/${file}" "${PROJECT_ROOT}/src/fb/${file}"
+done

src/Arrow.dom.ts

@@ -49,15 +49,16 @@ export {
     Bool,
     Int, Int8, Int16, Int32, Int64, Uint8, Uint16, Uint32, Uint64,
     Float, Float16, Float32, Float64,
-    Utf8, LargeUtf8,
-    Binary, LargeBinary,
+    Utf8, LargeUtf8, Utf8View,
+    Binary, LargeBinary, BinaryView,
     FixedSizeBinary,
     Date_, DateDay, DateMillisecond,
     Timestamp, TimestampSecond, TimestampMillisecond, TimestampMicrosecond, TimestampNanosecond,
     Time, TimeSecond, TimeMillisecond, TimeMicrosecond, TimeNanosecond,
     Decimal,
-    List,
+    List, LargeList, ListView, LargeListView,
     Struct, StructRow,
+    RunEndEncoded,
     Union, DenseUnion, SparseUnion,
     Dictionary,
     Interval, IntervalDayTime, IntervalYearMonth, IntervalMonthDayNano,
@@ -81,7 +82,7 @@ export {
 } from './Arrow.js';
 
 export {
-    BinaryBuilder, LargeBinaryBuilder,
+    BinaryBuilder, BinaryViewBuilder, LargeBinaryBuilder,
     BoolBuilder,
     DateBuilder, DateDayBuilder, DateMillisecondBuilder,
     DecimalBuilder,
@@ -92,12 +93,12 @@ export {
     IntervalBuilder, IntervalDayTimeBuilder, IntervalYearMonthBuilder, IntervalMonthDayNanoBuilder,
     DurationBuilder, DurationSecondBuilder, DurationMillisecondBuilder, DurationMicrosecondBuilder, DurationNanosecondBuilder,
     IntBuilder, Int8Builder, Int16Builder, Int32Builder, Int64Builder, Uint8Builder, Uint16Builder, Uint32Builder, Uint64Builder,
-    ListBuilder,
+    ListBuilder, ListViewBuilder, LargeListViewBuilder,
     MapBuilder,
     NullBuilder,
     StructBuilder,
     TimestampBuilder, TimestampSecondBuilder, TimestampMillisecondBuilder, TimestampMicrosecondBuilder, TimestampNanosecondBuilder,
     TimeBuilder, TimeSecondBuilder, TimeMillisecondBuilder, TimeMicrosecondBuilder, TimeNanosecondBuilder,
     UnionBuilder, DenseUnionBuilder, SparseUnionBuilder,
-    Utf8Builder, LargeUtf8Builder
+    Utf8Builder, Utf8ViewBuilder, LargeUtf8Builder
 } from './Arrow.js';

src/Arrow.ts

@@ -37,21 +37,22 @@ export {
     Bool,
     Int, Int8, Int16, Int32, Int64, Uint8, Uint16, Uint32, Uint64,
     Float, Float16, Float32, Float64,
-    Utf8, LargeUtf8,
-    Binary, LargeBinary,
+    Utf8, LargeUtf8, Utf8View,
+    Binary, LargeBinary, BinaryView,
     FixedSizeBinary,
     Date_, DateDay, DateMillisecond,
     Timestamp, TimestampSecond, TimestampMillisecond, TimestampMicrosecond, TimestampNanosecond,
     Time, TimeSecond, TimeMillisecond, TimeMicrosecond, TimeNanosecond,
     Decimal,
-    List,
+    List, LargeList, ListView, LargeListView,
     Struct,
     Union, DenseUnion, SparseUnion,
     Dictionary,
     Interval, IntervalDayTime, IntervalYearMonth, IntervalMonthDayNano,
     Duration, DurationSecond, DurationMillisecond, DurationMicrosecond, DurationNanosecond,
     FixedSizeList,
-    Map_
+    Map_,
+    RunEndEncoded
 } from './type.js';
 
 export { Table, makeTable, tableFromArrays } from './table.js';
@@ -79,10 +80,14 @@ export { TimestampBuilder, TimestampSecondBuilder, TimestampMillisecondBuilder,
 export { IntervalBuilder, IntervalDayTimeBuilder, IntervalYearMonthBuilder, IntervalMonthDayNanoBuilder } from './builder/interval.js';
 export { DurationBuilder, DurationSecondBuilder, DurationMillisecondBuilder, DurationMicrosecondBuilder, DurationNanosecondBuilder } from './builder/duration.js';
 export { Utf8Builder } from './builder/utf8.js';
+export { Utf8ViewBuilder } from './builder/utf8view.js';
 export { LargeUtf8Builder } from './builder/largeutf8.js';
 export { BinaryBuilder } from './builder/binary.js';
+export { BinaryViewBuilder } from './builder/binaryview.js';
 export { LargeBinaryBuilder } from './builder/largebinary.js';
 export { ListBuilder } from './builder/list.js';
+export { LargeListBuilder } from './builder/largelist.js';
+export { ListViewBuilder, LargeListViewBuilder } from './builder/listview.js';
 export { FixedSizeListBuilder } from './builder/fixedsizelist.js';
 export { MapBuilder } from './builder/map.js';
 export { StructBuilder } from './builder/struct.js';

src/builder.ts

@@ -22,7 +22,7 @@ import {
     DataType, strideForType,
     Float, Int, Decimal, FixedSizeBinary,
     Date_, Time, Timestamp, Interval, Duration,
-    Utf8, LargeUtf8, Binary, LargeBinary, List, Map_,
+    Utf8, LargeUtf8, Binary, LargeBinary, List, LargeList, Map_,
 } from './type.js';
 import { createIsValidFunction } from './builder/valid.js';
 import { BufferBuilder, BitmapBufferBuilder, DataBufferBuilder, OffsetsBufferBuilder } from './builder/buffer.js';
@@ -357,7 +357,7 @@ export abstract class FixedWidthBuilder<T extends Int | Float | FixedSizeBinary
 }
 
 /** @ignore */
-export abstract class VariableWidthBuilder<T extends Binary | LargeBinary | Utf8 | LargeUtf8 | List | Map_, TNull = any> extends Builder<T, TNull> {
+export abstract class VariableWidthBuilder<T extends Binary | LargeBinary | Utf8 | LargeUtf8 | List | LargeList | Map_, TNull = any> extends Builder<T, TNull> {
     protected _pendingLength = 0;
     protected _offsets: OffsetsBufferBuilder<T>;
     protected _pending: Map<number, any> | undefined;

src/builder/binaryview.ts

@@ -0,0 +1,169 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import { BinaryView } from '../type.js';
+import { Builder, BuilderOptions } from '../builder.js';
+import { BufferBuilder } from './buffer.js';
+import { toUint8Array } from '../util/buffer.js';
+import { makeData } from '../data.js';
+
+/** @ignore */
+export class BinaryViewBuilder<TNull = any> extends Builder<BinaryView, TNull> {
+    protected _views: BufferBuilder<Uint8Array>;
+    protected _variadicBuffers: Uint8Array[] = [];
+    protected _currentBuffer: BufferBuilder<Uint8Array> | null = null;
+    protected _currentBufferIndex = 0;
+    protected _currentBufferOffset = 0;
+    protected readonly _bufferSize = 32 * 1024 * 1024; // 32MB per buffer as per spec recommendation
+
+    constructor(opts: BuilderOptions<BinaryView, TNull>) {
+        super(opts);
+        this._views = new BufferBuilder(Uint8Array);
+    }
+
+    public get byteLength(): number {
+        let size = 0;
+        this._views && (size += this._views.byteLength);
+        this._nulls && (size += this._nulls.byteLength);
+        for (const buffer of this._variadicBuffers) {
+            size += buffer.byteLength;
+        }
+        this._currentBuffer && (size += this._currentBuffer.byteLength);
+        return size;
+    }
+
+    public setValue(index: number, value: Uint8Array) {
+        const data = toUint8Array(value);
+        const length = data.length;
+
+        // Ensure views buffer has space up to this index (similar to FixedWidthBuilder)
+        const bytesNeeded = (index + 1) * BinaryView.ELEMENT_WIDTH;
+        const currentBytes = this._views.length;
+        if (bytesNeeded > currentBytes) {
+            this._views.reserve(bytesNeeded - currentBytes);
+        }
+
+        const viewBuffer = this._views.buffer;
+        const viewOffset = index * BinaryView.ELEMENT_WIDTH;
+        const view = new DataView(viewBuffer.buffer, viewBuffer.byteOffset + viewOffset, BinaryView.ELEMENT_WIDTH);
+
+        // Write length (4 bytes, little-endian)
+        view.setInt32(BinaryView.LENGTH_OFFSET, length, true);
+
+        if (length <= BinaryView.INLINE_CAPACITY) {
+            // Inline: store data directly in view struct (up to 12 bytes)
+            viewBuffer.set(data, viewOffset + BinaryView.INLINE_OFFSET);
+            // Zero out remaining bytes
+            for (let i = length; i < BinaryView.INLINE_CAPACITY; i++) {
+                viewBuffer[viewOffset + BinaryView.INLINE_OFFSET + i] = 0;
+            }
+        } else {
+            // Out-of-line: store in variadic buffer
+            // Write prefix (first 4 bytes of data)
+            const prefix = new DataView(data.buffer, data.byteOffset, Math.min(4, length));
+            view.setUint32(BinaryView.INLINE_OFFSET, prefix.getUint32(0, true), true);
+
+            // Allocate space in variadic buffer
+            if (!this._currentBuffer || this._currentBufferOffset + length > this._bufferSize) {
+                // Start a new buffer
+                if (this._currentBuffer) {
+                    this._variadicBuffers.push(this._currentBuffer.buffer.slice(0, this._currentBufferOffset));
+                }
+                this._currentBuffer = new BufferBuilder(Uint8Array);
+                this._currentBufferIndex = this._variadicBuffers.length;
+                this._currentBufferOffset = 0;
+            }
+
+            // Write data to current buffer
+            const bufferData = this._currentBuffer.reserve(length).buffer;
+            bufferData.set(data, this._currentBufferOffset);
+
+            // Write buffer index and offset to view struct
+            view.setInt32(BinaryView.BUFFER_INDEX_OFFSET, this._currentBufferIndex, true);
+            view.setInt32(BinaryView.BUFFER_OFFSET_OFFSET, this._currentBufferOffset, true);
+
+            this._currentBufferOffset += length;
+        }
+
+        return this;
+    }
+
+    public setValid(index: number, isValid: boolean) {
+        if (!super.setValid(index, isValid)) {
+            // For null values, write a zero-length view
+            // Ensure space is allocated
+            const bytesNeeded = (index + 1) * BinaryView.ELEMENT_WIDTH;
+            const currentBytes = this._views.length;
+            if (bytesNeeded > currentBytes) {
+                this._views.reserve(bytesNeeded - currentBytes);
+            }
+
+            const viewBuffer = this._views.buffer;
+            const viewOffset = index * BinaryView.ELEMENT_WIDTH;
+            // Zero out the entire view struct
+            for (let i = 0; i < BinaryView.ELEMENT_WIDTH; i++) {
+                viewBuffer[viewOffset + i] = 0;
+            }
+            return false;
+        }
+        return true;
+    }
+
+    public clear() {
+        this._variadicBuffers = [];
+        this._currentBuffer = null;
+        this._currentBufferIndex = 0;
+        this._currentBufferOffset = 0;
+        this._views.clear();
+        return super.clear();
+    }
+
+    public flush() {
+        const { type, length, nullCount, _views, _nulls } = this;
+
+        // Finalize current buffer if it exists
+        if (this._currentBuffer && this._currentBufferOffset > 0) {
+            this._variadicBuffers.push(this._currentBuffer.buffer.slice(0, this._currentBufferOffset));
+            this._currentBuffer = null;
+            this._currentBufferOffset = 0;
+        }
+
+        const views = _views.flush(length * BinaryView.ELEMENT_WIDTH);
+        const nullBitmap = nullCount > 0 ? _nulls.flush(length) : undefined;
+        const variadicBuffers = this._variadicBuffers.slice();
+
+        // Reset variadic buffers for next batch
+        this._variadicBuffers = [];
+        this._currentBufferIndex = 0;
+
+        this.clear();
+
+        return makeData({
+            type,
+            length,
+            nullCount,
+            nullBitmap,
+            views,
+            variadicBuffers
+        });
+    }
+
+    public finish() {
+        this.finished = true;
+        return this;
+    }
+}