Skip to content

Commit

Permalink
export vector views, allow cloning data as another type
Browse files Browse the repository at this point in the history
  • Loading branch information
@trxcllnt
trxcllnt committed Jan 22, 2018
1 parent 700a47c commit e81082f
Show file tree
Hide file tree
Showing 7 changed files with 160 additions and 34 deletions.
6 changes: 6 additions & 0 deletions js/src/Arrow.externs.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -374,6 +374,12 @@ ChunkedData.computeOffsets = function() {};
var FlatVector = function() {};
/** @type {?} */
FlatVector.prototype.values;
/** @type {?} */
FlatVector.prototype.lows;
/** @type {?} */
FlatVector.prototype.highs;
/** @type {?} */
FlatVector.prototype.asInt32;

var ListVectorBase = function() {};
/** @type {?} */
Expand Down
28 changes: 28 additions & 0 deletions js/src/Arrow.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ import * as data_ from './data';
import * as vector_ from './vector';
import * as util_ from './util/int';
import * as visitor_ from './visitor';
import * as view_ from './vector/view';
import { Vector } from './vector';
import { RecordBatch } from './recordbatch';
import { Schema, Field, Type } from './type';
Expand Down Expand Up @@ -122,6 +123,32 @@ export namespace visitor {
export import VectorVisitor = visitor_.VectorVisitor;
}

export namespace view {
export import ChunkedView = view_.ChunkedView;
export import DictionaryView = view_.DictionaryView;
export import ListView = view_.ListView;
export import FixedSizeListView = view_.FixedSizeListView;
export import BinaryView = view_.BinaryView;
export import Utf8View = view_.Utf8View;
export import UnionView = view_.UnionView;
export import DenseUnionView = view_.DenseUnionView;
export import NestedView = view_.NestedView;
export import StructView = view_.StructView;
export import MapView = view_.MapView;
export import FlatView = view_.FlatView;
export import NullView = view_.NullView;
export import BoolView = view_.BoolView;
export import ValidityView = view_.ValidityView;
export import FixedSizeView = view_.FixedSizeView;
export import Float16View = view_.Float16View;
export import DateDayView = view_.DateDayView;
export import DateMillisecondView = view_.DateMillisecondView;
export import IntervalYearMonthView = view_.IntervalYearMonthView;
export import IntervalYearView = view_.IntervalYearView;
export import IntervalMonthView = view_.IntervalMonthView;
export import PrimitiveView = view_.PrimitiveView;
}

/* These exports are needed for the closure and uglify umd targets */
try {
let Arrow: any = eval('exports');
Expand All @@ -130,6 +157,7 @@ try {
Arrow['data'] = data;
Arrow['type'] = type;
Arrow['util'] = util;
Arrow['view'] = view;
Arrow['vector'] = vector;
Arrow['visitor'] = visitor;

Expand Down
71 changes: 46 additions & 25 deletions js/src/data.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -90,13 +90,13 @@ export class BaseData<T extends DataType = DataType> implements VectorLike {
}
return nullCount;
}
public clone(length = this._length, offset = this._offset, nullCount = this._nullCount) {
return new BaseData<T>(this._type, length, offset, nullCount) as this;
public clone<R extends T>(type: R, length = this._length, offset = this._offset, nullCount = this._nullCount) {
return new BaseData(type, length, offset, nullCount);
}
public slice(offset: number, length: number) {
return length <= 0 ? this : this.sliceInternal(this.clone(
length, this._offset + offset, +(this._nullCount === 0) - 1
), offset, length);
this._type, length, this._offset + offset, +(this._nullCount === 0) - 1
) as any, offset, length);
}
protected sliceInternal(clone: this, offset: number, length: number) {
let arr: any;
Expand Down Expand Up @@ -126,16 +126,13 @@ export class FlatData<T extends FlatType> extends BaseData<T> {
this[VectorType.VALIDITY] = toTypedArray(Uint8Array, nullBitmap);
}
public get ArrayType(): T['ArrayType'] { return this._type.ArrayType; }
public clone(length = this._length, offset = this._offset, nullCount = this._nullCount) {
return new FlatData<T>(this._type, length, this[VectorType.VALIDITY], this[VectorType.DATA], offset, nullCount) as this;
public clone<R extends T>(type: R, length = this._length, offset = this._offset, nullCount = this._nullCount) {
return new (this.constructor as any)(type, length, this[VectorType.VALIDITY], this[VectorType.DATA], offset, nullCount) as FlatData<R>;
}
}

export class BoolData extends FlatData<Bool> {
protected sliceData(data: Uint8Array) { return data; }
public clone(length = this._length, offset = this._offset, nullCount = this._nullCount) {
return new BoolData(this._type, length, this[VectorType.VALIDITY], this[VectorType.DATA], offset, nullCount) as this;
}
}

export class FlatListData<T extends FlatListType> extends FlatData<T> {
Expand All @@ -148,8 +145,8 @@ export class FlatListData<T extends FlatListType> extends FlatData<T> {
super(type, length, nullBitmap, data, offset, nullCount);
this[VectorType.OFFSET] = toTypedArray(Int32Array, valueOffsets);
}
public clone(length = this._length, offset = this._offset, nullCount = this._nullCount) {
return new FlatListData<T>(this._type, length, this[VectorType.VALIDITY], this[VectorType.OFFSET], this[VectorType.DATA], offset, nullCount) as this;
public clone<R extends T>(type: R, length = this._length, offset = this._offset, nullCount = this._nullCount) {
return new FlatListData(type, length, this[VectorType.VALIDITY], this[VectorType.OFFSET], this[VectorType.DATA], offset, nullCount);
}
}

Expand All @@ -165,8 +162,13 @@ export class DictionaryData<T extends DataType> extends BaseData<Dictionary<T>>
}
public get length() { return this._indicies.length; }
public get nullCount() { return this._indicies.nullCount; }
public clone(length = this._length, offset = this._offset) {
return new DictionaryData<T>(this._type, this._dictionary, this._indicies.slice(offset - this._offset, length)) as this;
public clone<R extends Dictionary<T>>(type: R, length = this._length, offset = this._offset) {
const data = this._dictionary.data.clone(type.dictionary as any);
return new DictionaryData<R>(
this._type as any,
this._dictionary.clone(data) as any,
this._indicies.slice(offset - this._offset, length)
) as any;
}
protected sliceInternal(clone: this, _offset: number, _length: number) {
clone._length = clone._indicies.length;
Expand All @@ -182,8 +184,8 @@ export class NestedData<T extends NestedType = NestedType> extends BaseData<T> {
this._childData = childData;
this[VectorType.VALIDITY] = toTypedArray(Uint8Array, nullBitmap);
}
public clone(length = this._length, offset = this._offset, nullCount = this._nullCount) {
return new NestedData<T>(this._type, length, this[VectorType.VALIDITY], this._childData, offset, nullCount) as this;
public clone<R extends T>(type: R, length = this._length, offset = this._offset, nullCount = this._nullCount) {
return new NestedData<R>(type, length, this[VectorType.VALIDITY], this._childData, offset, nullCount);
}
protected sliceInternal(clone: this, offset: number, length: number) {
if (!this[VectorType.OFFSET]) {
Expand All @@ -204,8 +206,8 @@ export class ListData<T extends ListType> extends NestedData<T> {
this._valuesData = valueChildData;
this[VectorType.OFFSET] = toTypedArray(Int32Array, valueOffsets);
}
public clone(length = this._length, offset = this._offset, nullCount = this._nullCount) {
return new ListData<T>(this._type, length, this[VectorType.VALIDITY], this[VectorType.OFFSET], this._valuesData, offset, nullCount) as this;
public clone<R extends T>(type: R, length = this._length, offset = this._offset, nullCount = this._nullCount) {
return new ListData<R>(type, length, this[VectorType.VALIDITY], this[VectorType.OFFSET], this._valuesData as any, offset, nullCount);
}
}

Expand All @@ -216,17 +218,24 @@ export class UnionData<T extends (DenseUnion | SparseUnion) = any> extends Neste
super(type, length, nullBitmap, childData, offset, nullCount);
this[VectorType.TYPE] = toTypedArray(Int8Array, typeIds);
}
public clone(length = this._length, offset = this._offset, nullCount = this._nullCount) {
return new UnionData<T>(this._type, length, this[VectorType.VALIDITY], this[VectorType.TYPE], this._childData, offset, nullCount) as this;
public clone<R extends T>(type: R, length = this._length, offset = this._offset, nullCount = this._nullCount) {
return new UnionData<R>(type, length, this[VectorType.VALIDITY], this[VectorType.TYPE], this._childData, offset, nullCount);
}
}

export class SparseUnionData extends UnionData<SparseUnion> {
constructor(type: SparseUnion, length: number, nullBitmap: Uint8Array | null | undefined, typeIds: Iterable<number>, childData: Data<any>[], offset?: number, nullCount?: number) {
super(type, length, nullBitmap, typeIds, childData, offset, nullCount);
}
public clone(length = this._length, offset = this._offset, nullCount = this._nullCount) {
return new SparseUnionData(this._type, length, this[VectorType.VALIDITY], this[VectorType.TYPE], this._childData, offset, nullCount) as this;
public clone<R extends SparseUnion>(type: R, length = this._length, offset = this._offset, nullCount = this._nullCount) {
return new SparseUnionData(
type,
length,
this[VectorType.VALIDITY],
this[VectorType.TYPE],
this._childData,
offset, nullCount
) as any as UnionData<R>;
}
}

Expand All @@ -237,8 +246,16 @@ export class DenseUnionData extends UnionData<DenseUnion> {
super(type, length, nullBitmap, typeIds, childData, offset, nullCount);
this[VectorType.OFFSET] = toTypedArray(Int32Array, valueOffsets);
}
public clone(length = this._length, offset = this._offset, nullCount = this._nullCount) {
return new DenseUnionData(this._type, length, this[VectorType.VALIDITY], this[VectorType.TYPE], this[VectorType.OFFSET], this._childData, offset, nullCount) as this;
public clone<R extends DenseUnion>(type: R, length = this._length, offset = this._offset, nullCount = this._nullCount) {
return new DenseUnionData(
type,
length,
this[VectorType.VALIDITY],
this[VectorType.TYPE],
this[VectorType.OFFSET],
this._childData,
offset, nullCount
) as any as UnionData<R>;
}
}

Expand All @@ -263,8 +280,12 @@ export class ChunkedData<T extends DataType> extends BaseData<T> {
}
return nullCount;
}
public clone(length = this._length, offset = this._offset, nullCount = this._nullCount) {
return new ChunkedData<T>(this._type, length, this._childVectors, offset, nullCount, this._childOffsets) as this;
public clone<R extends T>(type: R, length = this._length, offset = this._offset, nullCount = this._nullCount) {
return new ChunkedData<R>(
type, length,
this._childVectors.map((vec) => vec.clone(vec.data.clone(type))) as any,
offset, nullCount, this._childOffsets
);
}
protected sliceInternal(clone: this, offset: number, length: number) {
const chunks = this._childVectors;
Expand Down
4 changes: 2 additions & 2 deletions js/src/recordbatch.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -64,8 +64,8 @@ export class RecordBatch extends StructVector {
this.numCols = schema.fields.length;
}
}
public clone(data: Data<Struct>, view: View<Struct> = this.view.clone(data)): this {
return new RecordBatch(this.schema, data, view) as this;
public clone<R extends Struct>(data: Data<R>, view: View<R> = this.view.clone(data)): this {
return new RecordBatch(this.schema, data as any, view) as any;
}
public select(...columnNames: string[]) {
const fields = this.schema.fields;
Expand Down
53 changes: 48 additions & 5 deletions js/src/vector.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@
// specific language governing permissions and limitations
// under the License.

import { Data, ChunkedData, FlatData, BoolData } from './data';
import { Data, ChunkedData, FlatData, BoolData, FlatListData, NestedData } from './data';
import { VisitorNode, TypeVisitor, VectorVisitor } from './visitor';
import { DataType, ListType, FlatType, NestedType, FlatListType } from './type';
import { IterableArrayLike, Precision, DateUnit, IntervalUnit, UnionMode } from './type';
Expand All @@ -35,10 +35,14 @@ export class Vector<T extends DataType = any> implements VectorLike, View<T>, Vi
public static create<T extends DataType>(data: Data<T>): Vector<T> {
return createVector(data);
}
public type: T;
public length: number;
public readonly data: Data<T>;
public readonly view: View<T>;
constructor(data: Data<T>, view: View<T>) {
this.data = data;
this.type = data.type;
this.length = data.length;
let nulls: Uint8Array;
if ((<any> data instanceof ChunkedData) && !(view instanceof ChunkedView)) {
this.view = new ChunkedView(data);
Expand All @@ -49,15 +53,13 @@ export class Vector<T extends DataType = any> implements VectorLike, View<T>, Vi
}
}

public get type() { return this.data.type; }
public get length() { return this.data.length; }
public get nullCount() { return this.data.nullCount; }
public get nullBitmap() { return this.data.nullBitmap; }
public get [Symbol.toStringTag]() {
return `Vector<${this.type[Symbol.toStringTag]}>`;
}
public toJSON(): any { return this.toArray(); }
public clone(data: Data<T>, view: View<T> = this.view.clone(data)): this {
public clone<R extends T>(data: Data<R>, view: View<R> = this.view.clone(data) as any): this {
return new (this.constructor as any)(data, view);
}
public isValid(index: number): boolean {
Expand Down Expand Up @@ -111,6 +113,17 @@ export class Vector<T extends DataType = any> implements VectorLike, View<T>, Vi

export abstract class FlatVector<T extends FlatType> extends Vector<T> {
public get values() { return this.data.values; }
public lows(): IntVector<Int32> { return this.asInt32(0, 2); }
public highs(): IntVector<Int32> { return this.asInt32(1, 2); }
public asInt32(offset: number = 0, stride: number = 2): IntVector<Int32> {
let data = (this.data as FlatData<any>).clone(new Int32());
if (offset > 0) {
data = data.slice(offset, this.length - offset);
}
const int32s = new IntVector(data, new PrimitiveView(data, stride));
int32s.length = this.length / stride | 0;
return int32s;
}
}

export abstract class ListVectorBase<T extends (ListType | FlatListType)> extends Vector<T> {
Expand Down Expand Up @@ -144,7 +157,7 @@ import { ChunkedView } from './vector/chunked';
import { DictionaryView } from './vector/dictionary';
import { ListView, FixedSizeListView, BinaryView, Utf8View } from './vector/list';
import { UnionView, DenseUnionView, NestedView, StructView, MapView } from './vector/nested';
import { FlatView, NullView, BoolView, ValidityView, FixedSizeView, Float16View, DateDayView, DateMillisecondView, IntervalYearMonthView } from './vector/flat';
import { FlatView, NullView, BoolView, ValidityView, FixedSizeView, Float16View, DateDayView, DateMillisecondView, IntervalYearMonthView, PrimitiveView } from './vector/flat';
import { packBools } from './util/bit';

export class NullVector extends Vector<Null> {
Expand Down Expand Up @@ -223,6 +236,12 @@ export class DateVector extends FlatVector<Date_> {
constructor(data: Data<Date_>, view: View<Date_> = DateVector.defaultView(data)) {
super(data, view);
}
public lows(): IntVector<Int32> {
return this.type.unit === DateUnit.DAY ? this.asInt32(0, 1) : this.asInt32(0, 2);
}
public highs(): IntVector<Int32> {
return this.type.unit === DateUnit.DAY ? this.asInt32(0, 1) : this.asInt32(1, 2);
}
}

export class DecimalVector extends FlatVector<Decimal> {
Expand All @@ -238,6 +257,12 @@ export class TimeVector extends FlatVector<Time> {
constructor(data: Data<Time>, view: View<Time> = TimeVector.defaultView(data)) {
super(data, view);
}
public lows(): IntVector<Int32> {
return this.type.bitWidth <= 32 ? this.asInt32(0, 1) : this.asInt32(0, 2);
}
public highs(): IntVector<Int32> {
return this.type.bitWidth <= 32 ? this.asInt32(0, 1) : this.asInt32(1, 2);
}
}

export class TimestampVector extends FlatVector<Timestamp> {
Expand All @@ -253,12 +278,21 @@ export class IntervalVector extends FlatVector<Interval> {
constructor(data: Data<Interval>, view: View<Interval> = IntervalVector.defaultView(data)) {
super(data, view);
}
public lows(): IntVector<Int32> {
return this.type.unit === IntervalUnit.YEAR_MONTH ? this.asInt32(0, 1) : this.asInt32(0, 2);
}
public highs(): IntVector<Int32> {
return this.type.unit === IntervalUnit.YEAR_MONTH ? this.asInt32(0, 1) : this.asInt32(1, 2);
}
}

export class BinaryVector extends ListVectorBase<Binary> {
constructor(data: Data<Binary>, view: View<Binary> = new BinaryView(data)) {
super(data, view);
}
public asUtf8() {
return new Utf8Vector((this.data as FlatListData<any>).clone(new Utf8()));
}
}

export class FixedSizeBinaryVector extends FlatVector<FixedSizeBinary> {
Expand All @@ -271,6 +305,9 @@ export class Utf8Vector extends ListVectorBase<Utf8> {
constructor(data: Data<Utf8>, view: View<Utf8> = new Utf8View(data)) {
super(data, view);
}
public asBinary() {
return new BinaryVector((this.data as FlatListData<any>).clone(new Binary()));
}
}

export class ListVector<T extends DataType = DataType> extends ListVectorBase<List<T>> {
Expand All @@ -289,12 +326,18 @@ export class MapVector extends NestedVector<Map_> {
constructor(data: Data<Map_>, view: View<Map_> = new MapView(data)) {
super(data, view);
}
public asStruct() {
return new StructVector((this.data as NestedData<any>).clone(new Struct(this.type.children)));
}
}

export class StructVector extends NestedVector<Struct> {
constructor(data: Data<Struct>, view: View<Struct> = new StructView(data)) {
super(data, view);
}
public asMap(keysSorted: boolean = false) {
return new MapVector((this.data as NestedData<any>).clone(new Map_(keysSorted, this.type.children)));
}
}

export class UnionVector<T extends (SparseUnion | DenseUnion) = any> extends NestedVector<T> {
Expand Down

0 comments on commit e81082f

Please sign in to comment.