Skip to content

Commit

Permalink
rename ChunkData's fields so it's more clear they're not semantically…
Browse files Browse the repository at this point in the history 
… similar to other similarly named fields
  • Loading branch information
@trxcllnt
trxcllnt committed Jan 25, 2018
1 parent 7e43b78 commit 18807c6
Show file tree
Hide file tree
Showing 3 changed files with 71 additions and 71 deletions.
104 changes: 51 additions & 53 deletions js/src/data.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -60,42 +60,38 @@ export type kUnknownNullCount = -1;
export const kUnknownNullCount = -1;

export class BaseData<T extends DataType = DataType> implements VectorLike {
protected _type: T;
protected _length: number;
protected _offset: number;
public type: T;
public length: number;
public offset: number;
// @ts-ignore
protected _childData: Data<any>[];
public childData: Data<any>[];
protected _nullCount: number | kUnknownNullCount;
protected /* [VectorType.OFFSET]:*/ 0?: Int32Array;
protected /* [VectorType.DATA]:*/ 1?: T['TArray'];
protected /*[VectorType.VALIDITY]:*/ 2?: Uint8Array;
protected /* [VectorType.TYPE]:*/ 3?: Int8Array;
constructor(type: T, length: number, offset?: number, nullCount?: number) {
this._type = type;
this._length = Math.floor(Math.max(length || 0, 0));
this._offset = Math.floor(Math.max(offset || 0, 0));
this.type = type;
this.length = Math.floor(Math.max(length || 0, 0));
this.offset = Math.floor(Math.max(offset || 0, 0));
this._nullCount = Math.floor(Math.max(nullCount || 0, -1));
}
public get type() { return this._type; }
public get length() { return this._length; }
public get offset() { return this._offset; }
public get typeId() { return this._type.TType; }
public get childData() { return this._childData; }
public get typeId() { return this.type.TType; }
public get nullBitmap() { return this[VectorType.VALIDITY]; }
public get nullCount() {
let nullCount = this._nullCount;
let nullBitmap: Uint8Array | undefined;
if (nullCount === -1 && (nullBitmap = this[VectorType.VALIDITY])) {
this._nullCount = nullCount = this._length - popcnt_bit_range(nullBitmap, this._offset, this._offset + this._length);
this._nullCount = nullCount = this.length - popcnt_bit_range(nullBitmap, this.offset, this.offset + this.length);
}
return nullCount;
}
public clone<R extends T>(type: R, length = this._length, offset = this._offset, nullCount = this._nullCount) {
public clone<R extends T>(type: R, length = this.length, offset = this.offset, nullCount = this._nullCount) {
return new BaseData(type, length, offset, nullCount);
}
public slice(offset: number, length: number) {
return length <= 0 ? this : this.sliceInternal(this.clone(
this._type, length, this._offset + offset, +(this._nullCount === 0) - 1
this.type, length, this.offset + offset, +(this._nullCount === 0) - 1
) as any, offset, length);
}
protected sliceInternal(clone: this, offset: number, length: number) {
Expand Down Expand Up @@ -125,8 +121,8 @@ export class FlatData<T extends FlatType> extends BaseData<T> {
this[VectorType.DATA] = toTypedArray(this.ArrayType, data);
this[VectorType.VALIDITY] = toTypedArray(Uint8Array, nullBitmap);
}
public get ArrayType(): T['ArrayType'] { return this._type.ArrayType; }
public clone<R extends T>(type: R, length = this._length, offset = this._offset, nullCount = this._nullCount) {
public get ArrayType(): T['ArrayType'] { return this.type.ArrayType; }
public clone<R extends T>(type: R, length = this.length, offset = this.offset, nullCount = this._nullCount) {
return new (this.constructor as any)(type, length, this[VectorType.VALIDITY], this[VectorType.DATA], offset, nullCount) as FlatData<R>;
}
}
Expand All @@ -145,7 +141,7 @@ export class FlatListData<T extends FlatListType> extends FlatData<T> {
super(type, length, nullBitmap, data, offset, nullCount);
this[VectorType.OFFSET] = toTypedArray(Int32Array, valueOffsets);
}
public clone<R extends T>(type: R, length = this._length, offset = this._offset, nullCount = this._nullCount) {
public clone<R extends T>(type: R, length = this.length, offset = this.offset, nullCount = this._nullCount) {
return new FlatListData(type, length, this[VectorType.VALIDITY], this[VectorType.OFFSET], this[VectorType.DATA], offset, nullCount);
}
}
Expand All @@ -159,19 +155,19 @@ export class DictionaryData<T extends DataType> extends BaseData<Dictionary<T>>
super(type, indicies.length, (indicies as any)._nullCount);
this._indicies = indicies;
this._dictionary = dictionary;
this.length = this._indicies.length;
}
public get length() { return this._indicies.length; }
public get nullCount() { return this._indicies.nullCount; }
public clone<R extends Dictionary<T>>(type: R, length = this._length, offset = this._offset) {
public clone<R extends Dictionary<T>>(type: R, length = this.length, offset = this.offset) {
const data = this._dictionary.data.clone(type.dictionary as any);
return new DictionaryData<R>(
this._type as any,
this.type as any,
this._dictionary.clone(data) as any,
this._indicies.slice(offset - this._offset, length)
this._indicies.slice(offset - this.offset, length)
) as any;
}
protected sliceInternal(clone: this, _offset: number, _length: number) {
clone._length = clone._indicies.length;
clone.length = clone._indicies.length;
clone._nullCount = (clone._indicies as any)._nullCount;
return clone;
}
Expand All @@ -181,15 +177,15 @@ export class NestedData<T extends NestedType = NestedType> extends BaseData<T> {
public /*[VectorType.VALIDITY]:*/ 2: Uint8Array;
constructor(type: T, length: number, nullBitmap: Uint8Array | null | undefined, childData: Data<any>[], offset?: number, nullCount?: number) {
super(type, length, offset, nullCount);
this._childData = childData;
this.childData = childData;
this[VectorType.VALIDITY] = toTypedArray(Uint8Array, nullBitmap);
}
public clone<R extends T>(type: R, length = this._length, offset = this._offset, nullCount = this._nullCount) {
return new NestedData<R>(type, length, this[VectorType.VALIDITY], this._childData, offset, nullCount);
public clone<R extends T>(type: R, length = this.length, offset = this.offset, nullCount = this._nullCount) {
return new NestedData<R>(type, length, this[VectorType.VALIDITY], this.childData, offset, nullCount);
}
protected sliceInternal(clone: this, offset: number, length: number) {
if (!this[VectorType.OFFSET]) {
clone._childData = this._childData.map((child) => child.slice(offset, length));
clone.childData = this.childData.map((child) => child.slice(offset, length));
}
return super.sliceInternal(clone, offset, length);
}
Expand All @@ -212,7 +208,7 @@ export class ListData<T extends ListType> extends SingleNestedData<T> {
super(type, length, nullBitmap, valueChildData, offset, nullCount);
this[VectorType.OFFSET] = toTypedArray(Int32Array, valueOffsets);
}
public clone<R extends T>(type: R, length = this._length, offset = this._offset, nullCount = this._nullCount) {
public clone<R extends T>(type: R, length = this.length, offset = this.offset, nullCount = this._nullCount) {
return new ListData<R>(type, length, this[VectorType.VALIDITY], this[VectorType.OFFSET], this._valuesData as any, offset, nullCount);
}
}
Expand All @@ -224,22 +220,22 @@ export class UnionData<T extends (DenseUnion | SparseUnion) = any> extends Neste
super(type, length, nullBitmap, childData, offset, nullCount);
this[VectorType.TYPE] = toTypedArray(Int8Array, typeIds);
}
public clone<R extends T>(type: R, length = this._length, offset = this._offset, nullCount = this._nullCount) {
return new UnionData<R>(type, length, this[VectorType.VALIDITY], this[VectorType.TYPE], this._childData, offset, nullCount);
public clone<R extends T>(type: R, length = this.length, offset = this.offset, nullCount = this._nullCount) {
return new UnionData<R>(type, length, this[VectorType.VALIDITY], this[VectorType.TYPE], this.childData, offset, nullCount);
}
}

export class SparseUnionData extends UnionData<SparseUnion> {
constructor(type: SparseUnion, length: number, nullBitmap: Uint8Array | null | undefined, typeIds: Iterable<number>, childData: Data<any>[], offset?: number, nullCount?: number) {
super(type, length, nullBitmap, typeIds, childData, offset, nullCount);
}
public clone<R extends SparseUnion>(type: R, length = this._length, offset = this._offset, nullCount = this._nullCount) {
public clone<R extends SparseUnion>(type: R, length = this.length, offset = this.offset, nullCount = this._nullCount) {
return new SparseUnionData(
type,
length,
this[VectorType.VALIDITY],
this[VectorType.TYPE],
this._childData,
this.childData,
offset, nullCount
) as any as UnionData<R>;
}
Expand All @@ -252,50 +248,52 @@ export class DenseUnionData extends UnionData<DenseUnion> {
super(type, length, nullBitmap, typeIds, childData, offset, nullCount);
this[VectorType.OFFSET] = toTypedArray(Int32Array, valueOffsets);
}
public clone<R extends DenseUnion>(type: R, length = this._length, offset = this._offset, nullCount = this._nullCount) {
public clone<R extends DenseUnion>(type: R, length = this.length, offset = this.offset, nullCount = this._nullCount) {
return new DenseUnionData(
type,
length,
this[VectorType.VALIDITY],
this[VectorType.TYPE],
this[VectorType.OFFSET],
this._childData,
this.childData,
offset, nullCount
) as any as UnionData<R>;
}
}

export class ChunkedData<T extends DataType> extends BaseData<T> {
protected _childVectors: Vector<T>[];
protected _childOffsets: Uint32Array;
public get childVectors() { return this._childVectors; }
public get childOffsets() { return this._childOffsets; }
public get childData() {
return this._childData || (
this._childData = this._childVectors.map(({ data }) => data));
}
constructor(type: T, length: number, childVectors: Vector<T>[], offset?: number, nullCount?: number, childOffsets?: Uint32Array) {
// @ts-ignore
protected _chunkData: Data<T>[];
protected _chunkVectors: Vector<T>[];
protected _chunkOffsets: Uint32Array;
public get chunkVectors() { return this._chunkVectors; }
public get chunkOffsets() { return this._chunkOffsets; }
public get chunkData() {
return this._chunkData || (
this._chunkData = this._chunkVectors.map(({ data }) => data));
}
constructor(type: T, length: number, chunkVectors: Vector<T>[], offset?: number, nullCount?: number, chunkOffsets?: Uint32Array) {
super(type, length, offset, nullCount);
this._childVectors = childVectors;
this._childOffsets = childOffsets || ChunkedData.computeOffsets(childVectors);
this._chunkVectors = chunkVectors;
this._chunkOffsets = chunkOffsets || ChunkedData.computeOffsets(chunkVectors);
}
public get nullCount() {
let nullCount = this._nullCount;
if (nullCount === -1) {
this._nullCount = nullCount = this._childVectors.reduce((x, c) => x + c.nullCount, 0);
this._nullCount = nullCount = this._chunkVectors.reduce((x, c) => x + c.nullCount, 0);
}
return nullCount;
}
public clone<R extends T>(type: R, length = this._length, offset = this._offset, nullCount = this._nullCount) {
public clone<R extends T>(type: R, length = this.length, offset = this.offset, nullCount = this._nullCount) {
return new ChunkedData<R>(
type, length,
this._childVectors.map((vec) => vec.clone(vec.data.clone(type))) as any,
offset, nullCount, this._childOffsets
this._chunkVectors.map((vec) => vec.clone(vec.data.clone(type))) as any,
offset, nullCount, this._chunkOffsets
);
}
protected sliceInternal(clone: this, offset: number, length: number) {
const chunks = this._childVectors;
const offsets = this._childOffsets;
const chunks = this._chunkVectors;
const offsets = this._chunkOffsets;
const chunkSlices: Vector<T>[] = [];
for (let childIndex = -1, numChildren = chunks.length; ++childIndex < numChildren;) {
const child = chunks[childIndex];
Expand All @@ -315,8 +313,8 @@ export class ChunkedData<T extends DataType> extends BaseData<T> {
const end = begin + Math.min(childLength - begin, (offset + length) - childOffset);
chunkSlices.push(child.slice(begin, end));
}
clone._childVectors = chunkSlices;
clone._childOffsets = ChunkedData.computeOffsets(chunkSlices);
clone._chunkVectors = chunkSlices;
clone._chunkOffsets = ChunkedData.computeOffsets(chunkSlices);
return clone;
}
static computeOffsets<T extends DataType>(childVectors: Vector<T>[]) {
Expand Down
4 changes: 2 additions & 2 deletions js/src/vector.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -87,7 +87,7 @@ export class Vector<T extends DataType = any> implements VectorLike, View<T>, Vi
const { view } = this;
const vecs = !(view instanceof ChunkedView)
? [this, ...others]
: [...view.childVectors, ...others];
: [...view.chunkVectors, ...others];
const offsets = ChunkedData.computeOffsets(vecs);
const chunksLength = offsets[offsets.length - 1];
const chunkedData = new ChunkedData(this.type, chunksLength, vecs, 0, -1, offsets);
Expand Down Expand Up @@ -380,7 +380,7 @@ export class DictionaryVector<T extends DataType = DataType> extends Vector<Dict
this.indicies = view.indicies;
this.dictionary = data.dictionary;
} else if (data instanceof ChunkedData && view instanceof ChunkedView) {
const chunks = view.childVectors as DictionaryVector<T>[];
const chunks = view.chunkVectors as DictionaryVector<T>[];
// Assume the last chunk's dictionary data is the most up-to-date,
// including data from DictionaryBatches that were marked as deltas
this.dictionary = chunks[chunks.length - 1].dictionary;
Expand Down
34 changes: 18 additions & 16 deletions js/src/vector/chunked.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,33 +20,35 @@ import { View, Vector, NestedVector } from '../vector';
import { DataType, TypedArray, IterableArrayLike } from '../type';

export class ChunkedView<T extends DataType> implements View<T> {
public childVectors: Vector<T>[];
public childOffsets: Uint32Array;
protected _childColumns: Vector<any>[];
public chunkVectors: Vector<T>[];
public chunkOffsets: Uint32Array;
protected _children: Vector<any>[];
constructor(data: ChunkedData<T>) {
this.childVectors = data.childVectors;
this.childOffsets = data.childOffsets;
this.chunkVectors = data.chunkVectors;
this.chunkOffsets = data.chunkOffsets;
}
public clone(data: ChunkedData<T>): this {
return new ChunkedView(data) as this;
}
public *[Symbol.iterator](): IterableIterator<T['TValue'] | null> {
for (const vector of this.childVectors) {
for (const vector of this.chunkVectors) {
yield* vector;
}
}
public getChildAt<R extends DataType = DataType>(index: number) {
return (this._childColumns || (this._childColumns = []))[index] || (
this._childColumns[index] = Vector.concat<R>(
...(<any> this.childVectors as NestedVector<any>[]).map((v) => v.getChildAt(index))));
return index < 0 ? null
: (this._children || (this._children = []))[index] ||
(this._children[index] = Vector.concat<R>(
...(<any> this.chunkVectors as NestedVector<any>[])
.map((chunk) => chunk.getChildAt<R>(index))));
}
public isValid(index: number): boolean {
// binary search to find the child vector and value index offset (inlined for speed)
let offsets = this.childOffsets, pos = 0;
let offsets = this.chunkOffsets, pos = 0;
let lhs = 0, mid = 0, rhs = offsets.length - 1;
while (index < offsets[rhs] && index >= (pos = offsets[lhs])) {
if (lhs + 1 === rhs) {
return this.childVectors[lhs].isValid(index - pos);
return this.chunkVectors[lhs].isValid(index - pos);
}
mid = lhs + ((rhs - lhs) / 2) | 0;
index >= offsets[mid] ? (lhs = mid) : (rhs = mid);
Expand All @@ -55,11 +57,11 @@ export class ChunkedView<T extends DataType> implements View<T> {
}
public get(index: number): T['TValue'] | null {
// binary search to find the child vector and value index offset (inlined for speed)
let offsets = this.childOffsets, pos = 0;
let offsets = this.chunkOffsets, pos = 0;
let lhs = 0, mid = 0, rhs = offsets.length - 1;
while (index < offsets[rhs] && index >= (pos = offsets[lhs])) {
if (lhs + 1 === rhs) {
return this.childVectors[lhs].get(index - pos);
return this.chunkVectors[lhs].get(index - pos);
}
mid = lhs + ((rhs - lhs) / 2) | 0;
index >= offsets[mid] ? (lhs = mid) : (rhs = mid);
Expand All @@ -68,18 +70,18 @@ export class ChunkedView<T extends DataType> implements View<T> {
}
public set(index: number, value: T['TValue'] | null): void {
// binary search to find the child vector and value index offset (inlined for speed)
let offsets = this.childOffsets, pos = 0;
let offsets = this.chunkOffsets, pos = 0;
let lhs = 0, mid = 0, rhs = offsets.length - 1;
while (index < offsets[rhs] && index >= (pos = offsets[lhs])) {
if (lhs + 1 === rhs) {
return this.childVectors[lhs].set(index - pos, value);
return this.chunkVectors[lhs].set(index - pos, value);
}
mid = lhs + ((rhs - lhs) / 2) | 0;
index >= offsets[mid] ? (lhs = mid) : (rhs = mid);
}
}
public toArray(): IterableArrayLike<T['TValue'] | null> {
const chunks = this.childVectors;
const chunks = this.chunkVectors;
const numChunks = chunks.length;
if (numChunks === 1) {
return chunks[0].toArray();
Expand Down

0 comments on commit 18807c6

Please sign in to comment.