diff --git a/js/src/data.ts b/js/src/data.ts index e05b5aefff978..81d19a3cf63cd 100644 --- a/js/src/data.ts +++ b/js/src/data.ts @@ -60,42 +60,38 @@ export type kUnknownNullCount = -1; export const kUnknownNullCount = -1; export class BaseData implements VectorLike { - protected _type: T; - protected _length: number; - protected _offset: number; + public type: T; + public length: number; + public offset: number; // @ts-ignore - protected _childData: Data[]; + public childData: Data[]; protected _nullCount: number | kUnknownNullCount; protected /* [VectorType.OFFSET]:*/ 0?: Int32Array; protected /* [VectorType.DATA]:*/ 1?: T['TArray']; protected /*[VectorType.VALIDITY]:*/ 2?: Uint8Array; protected /* [VectorType.TYPE]:*/ 3?: Int8Array; constructor(type: T, length: number, offset?: number, nullCount?: number) { - this._type = type; - this._length = Math.floor(Math.max(length || 0, 0)); - this._offset = Math.floor(Math.max(offset || 0, 0)); + this.type = type; + this.length = Math.floor(Math.max(length || 0, 0)); + this.offset = Math.floor(Math.max(offset || 0, 0)); this._nullCount = Math.floor(Math.max(nullCount || 0, -1)); } - public get type() { return this._type; } - public get length() { return this._length; } - public get offset() { return this._offset; } - public get typeId() { return this._type.TType; } - public get childData() { return this._childData; } + public get typeId() { return this.type.TType; } public get nullBitmap() { return this[VectorType.VALIDITY]; } public get nullCount() { let nullCount = this._nullCount; let nullBitmap: Uint8Array | undefined; if (nullCount === -1 && (nullBitmap = this[VectorType.VALIDITY])) { - this._nullCount = nullCount = this._length - popcnt_bit_range(nullBitmap, this._offset, this._offset + this._length); + this._nullCount = nullCount = this.length - popcnt_bit_range(nullBitmap, this.offset, this.offset + this.length); } return nullCount; } - public clone(type: R, length = this._length, offset = this._offset, nullCount = this._nullCount) { + public clone(type: R, length = this.length, offset = this.offset, nullCount = this._nullCount) { return new BaseData(type, length, offset, nullCount); } public slice(offset: number, length: number) { return length <= 0 ? this : this.sliceInternal(this.clone( - this._type, length, this._offset + offset, +(this._nullCount === 0) - 1 + this.type, length, this.offset + offset, +(this._nullCount === 0) - 1 ) as any, offset, length); } protected sliceInternal(clone: this, offset: number, length: number) { @@ -125,8 +121,8 @@ export class FlatData extends BaseData { this[VectorType.DATA] = toTypedArray(this.ArrayType, data); this[VectorType.VALIDITY] = toTypedArray(Uint8Array, nullBitmap); } - public get ArrayType(): T['ArrayType'] { return this._type.ArrayType; } - public clone(type: R, length = this._length, offset = this._offset, nullCount = this._nullCount) { + public get ArrayType(): T['ArrayType'] { return this.type.ArrayType; } + public clone(type: R, length = this.length, offset = this.offset, nullCount = this._nullCount) { return new (this.constructor as any)(type, length, this[VectorType.VALIDITY], this[VectorType.DATA], offset, nullCount) as FlatData; } } @@ -145,7 +141,7 @@ export class FlatListData extends FlatData { super(type, length, nullBitmap, data, offset, nullCount); this[VectorType.OFFSET] = toTypedArray(Int32Array, valueOffsets); } - public clone(type: R, length = this._length, offset = this._offset, nullCount = this._nullCount) { + public clone(type: R, length = this.length, offset = this.offset, nullCount = this._nullCount) { return new FlatListData(type, length, this[VectorType.VALIDITY], this[VectorType.OFFSET], this[VectorType.DATA], offset, nullCount); } } @@ -159,19 +155,19 @@ export class DictionaryData extends BaseData> super(type, indicies.length, (indicies as any)._nullCount); this._indicies = indicies; this._dictionary = dictionary; + this.length = this._indicies.length; } - public get length() { return this._indicies.length; } public get nullCount() { return this._indicies.nullCount; } - public clone>(type: R, length = this._length, offset = this._offset) { + public clone>(type: R, length = this.length, offset = this.offset) { const data = this._dictionary.data.clone(type.dictionary as any); return new DictionaryData( - this._type as any, + this.type as any, this._dictionary.clone(data) as any, - this._indicies.slice(offset - this._offset, length) + this._indicies.slice(offset - this.offset, length) ) as any; } protected sliceInternal(clone: this, _offset: number, _length: number) { - clone._length = clone._indicies.length; + clone.length = clone._indicies.length; clone._nullCount = (clone._indicies as any)._nullCount; return clone; } @@ -181,15 +177,15 @@ export class NestedData extends BaseData { public /*[VectorType.VALIDITY]:*/ 2: Uint8Array; constructor(type: T, length: number, nullBitmap: Uint8Array | null | undefined, childData: Data[], offset?: number, nullCount?: number) { super(type, length, offset, nullCount); - this._childData = childData; + this.childData = childData; this[VectorType.VALIDITY] = toTypedArray(Uint8Array, nullBitmap); } - public clone(type: R, length = this._length, offset = this._offset, nullCount = this._nullCount) { - return new NestedData(type, length, this[VectorType.VALIDITY], this._childData, offset, nullCount); + public clone(type: R, length = this.length, offset = this.offset, nullCount = this._nullCount) { + return new NestedData(type, length, this[VectorType.VALIDITY], this.childData, offset, nullCount); } protected sliceInternal(clone: this, offset: number, length: number) { if (!this[VectorType.OFFSET]) { - clone._childData = this._childData.map((child) => child.slice(offset, length)); + clone.childData = this.childData.map((child) => child.slice(offset, length)); } return super.sliceInternal(clone, offset, length); } @@ -212,7 +208,7 @@ export class ListData extends SingleNestedData { super(type, length, nullBitmap, valueChildData, offset, nullCount); this[VectorType.OFFSET] = toTypedArray(Int32Array, valueOffsets); } - public clone(type: R, length = this._length, offset = this._offset, nullCount = this._nullCount) { + public clone(type: R, length = this.length, offset = this.offset, nullCount = this._nullCount) { return new ListData(type, length, this[VectorType.VALIDITY], this[VectorType.OFFSET], this._valuesData as any, offset, nullCount); } } @@ -224,8 +220,8 @@ export class UnionData extends Neste super(type, length, nullBitmap, childData, offset, nullCount); this[VectorType.TYPE] = toTypedArray(Int8Array, typeIds); } - public clone(type: R, length = this._length, offset = this._offset, nullCount = this._nullCount) { - return new UnionData(type, length, this[VectorType.VALIDITY], this[VectorType.TYPE], this._childData, offset, nullCount); + public clone(type: R, length = this.length, offset = this.offset, nullCount = this._nullCount) { + return new UnionData(type, length, this[VectorType.VALIDITY], this[VectorType.TYPE], this.childData, offset, nullCount); } } @@ -233,13 +229,13 @@ export class SparseUnionData extends UnionData { constructor(type: SparseUnion, length: number, nullBitmap: Uint8Array | null | undefined, typeIds: Iterable, childData: Data[], offset?: number, nullCount?: number) { super(type, length, nullBitmap, typeIds, childData, offset, nullCount); } - public clone(type: R, length = this._length, offset = this._offset, nullCount = this._nullCount) { + public clone(type: R, length = this.length, offset = this.offset, nullCount = this._nullCount) { return new SparseUnionData( type, length, this[VectorType.VALIDITY], this[VectorType.TYPE], - this._childData, + this.childData, offset, nullCount ) as any as UnionData; } @@ -252,50 +248,52 @@ export class DenseUnionData extends UnionData { super(type, length, nullBitmap, typeIds, childData, offset, nullCount); this[VectorType.OFFSET] = toTypedArray(Int32Array, valueOffsets); } - public clone(type: R, length = this._length, offset = this._offset, nullCount = this._nullCount) { + public clone(type: R, length = this.length, offset = this.offset, nullCount = this._nullCount) { return new DenseUnionData( type, length, this[VectorType.VALIDITY], this[VectorType.TYPE], this[VectorType.OFFSET], - this._childData, + this.childData, offset, nullCount ) as any as UnionData; } } export class ChunkedData extends BaseData { - protected _childVectors: Vector[]; - protected _childOffsets: Uint32Array; - public get childVectors() { return this._childVectors; } - public get childOffsets() { return this._childOffsets; } - public get childData() { - return this._childData || ( - this._childData = this._childVectors.map(({ data }) => data)); - } - constructor(type: T, length: number, childVectors: Vector[], offset?: number, nullCount?: number, childOffsets?: Uint32Array) { + // @ts-ignore + protected _chunkData: Data[]; + protected _chunkVectors: Vector[]; + protected _chunkOffsets: Uint32Array; + public get chunkVectors() { return this._chunkVectors; } + public get chunkOffsets() { return this._chunkOffsets; } + public get chunkData() { + return this._chunkData || ( + this._chunkData = this._chunkVectors.map(({ data }) => data)); + } + constructor(type: T, length: number, chunkVectors: Vector[], offset?: number, nullCount?: number, chunkOffsets?: Uint32Array) { super(type, length, offset, nullCount); - this._childVectors = childVectors; - this._childOffsets = childOffsets || ChunkedData.computeOffsets(childVectors); + this._chunkVectors = chunkVectors; + this._chunkOffsets = chunkOffsets || ChunkedData.computeOffsets(chunkVectors); } public get nullCount() { let nullCount = this._nullCount; if (nullCount === -1) { - this._nullCount = nullCount = this._childVectors.reduce((x, c) => x + c.nullCount, 0); + this._nullCount = nullCount = this._chunkVectors.reduce((x, c) => x + c.nullCount, 0); } return nullCount; } - public clone(type: R, length = this._length, offset = this._offset, nullCount = this._nullCount) { + public clone(type: R, length = this.length, offset = this.offset, nullCount = this._nullCount) { return new ChunkedData( type, length, - this._childVectors.map((vec) => vec.clone(vec.data.clone(type))) as any, - offset, nullCount, this._childOffsets + this._chunkVectors.map((vec) => vec.clone(vec.data.clone(type))) as any, + offset, nullCount, this._chunkOffsets ); } protected sliceInternal(clone: this, offset: number, length: number) { - const chunks = this._childVectors; - const offsets = this._childOffsets; + const chunks = this._chunkVectors; + const offsets = this._chunkOffsets; const chunkSlices: Vector[] = []; for (let childIndex = -1, numChildren = chunks.length; ++childIndex < numChildren;) { const child = chunks[childIndex]; @@ -315,8 +313,8 @@ export class ChunkedData extends BaseData { const end = begin + Math.min(childLength - begin, (offset + length) - childOffset); chunkSlices.push(child.slice(begin, end)); } - clone._childVectors = chunkSlices; - clone._childOffsets = ChunkedData.computeOffsets(chunkSlices); + clone._chunkVectors = chunkSlices; + clone._chunkOffsets = ChunkedData.computeOffsets(chunkSlices); return clone; } static computeOffsets(childVectors: Vector[]) { diff --git a/js/src/vector.ts b/js/src/vector.ts index a1b5cb3405017..4190c64c59fe4 100644 --- a/js/src/vector.ts +++ b/js/src/vector.ts @@ -87,7 +87,7 @@ export class Vector implements VectorLike, View, Vi const { view } = this; const vecs = !(view instanceof ChunkedView) ? [this, ...others] - : [...view.childVectors, ...others]; + : [...view.chunkVectors, ...others]; const offsets = ChunkedData.computeOffsets(vecs); const chunksLength = offsets[offsets.length - 1]; const chunkedData = new ChunkedData(this.type, chunksLength, vecs, 0, -1, offsets); @@ -380,7 +380,7 @@ export class DictionaryVector extends Vector[]; + const chunks = view.chunkVectors as DictionaryVector[]; // Assume the last chunk's dictionary data is the most up-to-date, // including data from DictionaryBatches that were marked as deltas this.dictionary = chunks[chunks.length - 1].dictionary; diff --git a/js/src/vector/chunked.ts b/js/src/vector/chunked.ts index 9f5141f2cd430..c0087fdd75833 100644 --- a/js/src/vector/chunked.ts +++ b/js/src/vector/chunked.ts @@ -20,33 +20,35 @@ import { View, Vector, NestedVector } from '../vector'; import { DataType, TypedArray, IterableArrayLike } from '../type'; export class ChunkedView implements View { - public childVectors: Vector[]; - public childOffsets: Uint32Array; - protected _childColumns: Vector[]; + public chunkVectors: Vector[]; + public chunkOffsets: Uint32Array; + protected _children: Vector[]; constructor(data: ChunkedData) { - this.childVectors = data.childVectors; - this.childOffsets = data.childOffsets; + this.chunkVectors = data.chunkVectors; + this.chunkOffsets = data.chunkOffsets; } public clone(data: ChunkedData): this { return new ChunkedView(data) as this; } public *[Symbol.iterator](): IterableIterator { - for (const vector of this.childVectors) { + for (const vector of this.chunkVectors) { yield* vector; } } public getChildAt(index: number) { - return (this._childColumns || (this._childColumns = []))[index] || ( - this._childColumns[index] = Vector.concat( - ...( this.childVectors as NestedVector[]).map((v) => v.getChildAt(index)))); + return index < 0 ? null + : (this._children || (this._children = []))[index] || + (this._children[index] = Vector.concat( + ...( this.chunkVectors as NestedVector[]) + .map((chunk) => chunk.getChildAt(index)))); } public isValid(index: number): boolean { // binary search to find the child vector and value index offset (inlined for speed) - let offsets = this.childOffsets, pos = 0; + let offsets = this.chunkOffsets, pos = 0; let lhs = 0, mid = 0, rhs = offsets.length - 1; while (index < offsets[rhs] && index >= (pos = offsets[lhs])) { if (lhs + 1 === rhs) { - return this.childVectors[lhs].isValid(index - pos); + return this.chunkVectors[lhs].isValid(index - pos); } mid = lhs + ((rhs - lhs) / 2) | 0; index >= offsets[mid] ? (lhs = mid) : (rhs = mid); @@ -55,11 +57,11 @@ export class ChunkedView implements View { } public get(index: number): T['TValue'] | null { // binary search to find the child vector and value index offset (inlined for speed) - let offsets = this.childOffsets, pos = 0; + let offsets = this.chunkOffsets, pos = 0; let lhs = 0, mid = 0, rhs = offsets.length - 1; while (index < offsets[rhs] && index >= (pos = offsets[lhs])) { if (lhs + 1 === rhs) { - return this.childVectors[lhs].get(index - pos); + return this.chunkVectors[lhs].get(index - pos); } mid = lhs + ((rhs - lhs) / 2) | 0; index >= offsets[mid] ? (lhs = mid) : (rhs = mid); @@ -68,18 +70,18 @@ export class ChunkedView implements View { } public set(index: number, value: T['TValue'] | null): void { // binary search to find the child vector and value index offset (inlined for speed) - let offsets = this.childOffsets, pos = 0; + let offsets = this.chunkOffsets, pos = 0; let lhs = 0, mid = 0, rhs = offsets.length - 1; while (index < offsets[rhs] && index >= (pos = offsets[lhs])) { if (lhs + 1 === rhs) { - return this.childVectors[lhs].set(index - pos, value); + return this.chunkVectors[lhs].set(index - pos, value); } mid = lhs + ((rhs - lhs) / 2) | 0; index >= offsets[mid] ? (lhs = mid) : (rhs = mid); } } public toArray(): IterableArrayLike { - const chunks = this.childVectors; + const chunks = this.chunkVectors; const numChunks = chunks.length; if (numChunks === 1) { return chunks[0].toArray();