Skip to content

Commit

Permalink
WIP: adding StorageType to in-memory #1
Browse files Browse the repository at this point in the history
  • Loading branch information
radeusgd committed Mar 31, 2023
1 parent 38450cb commit d42a860
Show file tree
Hide file tree
Showing 19 changed files with 222 additions and 60 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -11,8 +11,8 @@ type SQL_Type_Mapping
have at least a few exceptions, and some like SQLite may have very
limited support). If an SQL_Type that matches the Value_Type cannot be
found, a closest approximate match is returned instead. If an exact match
cannot be found, an `Inexact_Type_???` warning is reported according to
the `on_problems` setting.
cannot be found, an `Inexact_Type_Coercion` warning is reported according
to the `on_problems` setting.

If the conversion is exact, it should be reversible, i.e.
`sql_type_to_value_type (value_type_to_sql x) = x`.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,13 @@ import Standard.Table.Data.Type.Value_Type.Value_Type
polyglot java import org.enso.table.data.column.builder.object.Builder
polyglot java import org.enso.table.data.column.storage.Storage as Java_Storage

## PRIVATE
Gets the value type represented by this Java Storage.
value_type_of_storage : Java_Storage -> Value_Type
value_type_of_storage storage = case storage.getStoredType of
"Integer" ->
""

## Represents different types of underlying storage for Columns.
type Storage
## A column storing text data.
Expand Down

This file was deleted.

Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,12 @@ type Bits
Bits.Bits_32 -> 32
Bits.Bits_64 -> 64

to_text : Text
to_text self = case self of
Bits.Bits_16 -> "16 bits"
Bits.Bits_32 -> "32 bits"
Bits.Bits_64 -> "64 bits"

type Bits_Comparator
compare x y = Comparable.from x.to_bits . compare x.to_bits y.to_bits
hash x = Comparable.from x.to_bits . hash x.to_bits
Expand Down Expand Up @@ -126,6 +132,8 @@ type Value_Type
Value_Type.Char _ _ -> True
_ -> False

# TODO move these functions below to a separate file

## PRIVATE
Finds a type that can fit both a current type and a new type.
reconcile_types current new = case current of
Expand Down
14 changes: 14 additions & 0 deletions distribution/lib/Standard/Table/0.0.0-dev/src/Errors.enso
Original file line number Diff line number Diff line change
Expand Up @@ -431,3 +431,17 @@ type Invalid_Aggregate_Column
to_display_text : Text
to_display_text self =
"The name ["+self.name+"] is not a valid column name nor expression."

type Inexact_Type_Coercion
## Indicates that the requested `Value_Type` is not available in the given
backend, so it was replaced by its closest available type.
Warning (requested_type : Value_Type) (actual_type : Value_Type)

type Invalid_Value_For_Type
## Indicates that a column construction/transformation failed because the
provided value is not valid for the requested column type.
Error (value : Any) (value_type : Value_Type)

to_display_text : Text
to_display_text self =
"The value ["+self.value.to_text+"] is not valid for the column type ["+self.value_type.to_text+"]."
Original file line number Diff line number Diff line change
@@ -1,11 +1,12 @@
package org.enso.table.data.column.builder.object;

import org.enso.table.data.column.storage.Storage;
import org.enso.table.data.column.storage.type.StorageType;

/** A builder for creating columns dynamically. */
public abstract class Builder {
public static Builder getForType(int type, int size) {
return switch (type) {
public static Builder getForType(StorageType type, int size) {
Builder builder = switch (type) {
case Storage.Type.OBJECT -> new ObjectBuilder(size);
case Storage.Type.LONG -> NumericBuilder.createLongBuilder(size);
case Storage.Type.DOUBLE -> NumericBuilder.createDoubleBuilder(size);
Expand All @@ -16,6 +17,8 @@ public static Builder getForType(int type, int size) {
case Storage.Type.DATE_TIME -> new DateTimeBuilder(size);
default -> new InferredBuilder(size);
};
assert builder.getType().equals(type);
return builder;
}

/**
Expand Down Expand Up @@ -66,4 +69,7 @@ public static Builder getForType(int type, int size) {
* @return a storage containing all the items appended so far
*/
public abstract Storage<?> seal();

/** @return the current storage type of this builder */
public abstract StorageType getType();
}
Original file line number Diff line number Diff line change
Expand Up @@ -26,9 +26,6 @@ public abstract class TypedBuilder extends Builder {
*/
public abstract TypedBuilder retypeTo(long type);

/** @return the current storage type of this builder */
public abstract int getType();

/** Specifies if the following object will be accepted by this builder's append* methods. */
public abstract boolean accepts(Object o);
}
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@
import org.enso.table.data.column.operation.map.MapOperationProblemBuilder;
import org.enso.table.data.column.operation.map.UnaryMapOperation;
import org.enso.table.data.column.operation.map.bool.BooleanIsInOp;
import org.enso.table.data.column.storage.type.StorageType;
import org.enso.table.data.index.Index;
import org.enso.table.data.mask.OrderMask;
import org.enso.table.data.mask.SliceRange;
Expand Down Expand Up @@ -55,8 +56,8 @@ public int countMissing() {
}

@Override
public int getType() {
return Type.BOOL;
public StorageType getType() {
return StorageType.BOOLEAN;
}

@Override
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -5,8 +5,8 @@
import org.enso.table.data.column.builder.object.Builder;
import org.enso.table.data.column.builder.object.DateBuilder;
import org.enso.table.data.column.operation.map.MapOpStorage;
import org.enso.table.data.column.operation.map.SpecializedIsInOp;
import org.enso.table.data.column.operation.map.datetime.DateTimeIsInOp;
import org.enso.table.data.column.storage.type.StorageType;

public final class DateStorage extends SpecializedStorage<LocalDate> {
/**
Expand Down Expand Up @@ -36,8 +36,8 @@ protected LocalDate[] newUnderlyingArray(int size) {
}

@Override
public int getType() {
return Type.DATE;
public StorageType getType() {
return StorageType.DATE;
}

@Override
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,8 +3,8 @@
import org.enso.table.data.column.builder.object.Builder;
import org.enso.table.data.column.builder.object.DateTimeBuilder;
import org.enso.table.data.column.operation.map.MapOpStorage;
import org.enso.table.data.column.operation.map.SpecializedIsInOp;
import org.enso.table.data.column.operation.map.datetime.DateTimeIsInOp;
import org.enso.table.data.column.storage.type.StorageType;

import java.time.ZonedDateTime;

Expand Down Expand Up @@ -38,8 +38,8 @@ protected ZonedDateTime[] newUnderlyingArray(int size) {
}

@Override
public int getType() {
return Type.DATE_TIME;
public StorageType getType() {
return StorageType.DATE_TIME;
}

@Override
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@
import org.enso.table.data.column.operation.map.numeric.DoubleBooleanOp;
import org.enso.table.data.column.operation.map.numeric.DoubleIsInOp;
import org.enso.table.data.column.operation.map.numeric.DoubleNumericOp;
import org.enso.table.data.column.storage.type.StorageType;
import org.enso.table.data.index.Index;
import org.enso.table.data.mask.OrderMask;
import org.enso.table.data.mask.SliceRange;
Expand Down Expand Up @@ -71,10 +72,12 @@ public Double getItemBoxed(int idx) {
return isMissing.get(idx) ? null : Double.longBitsToDouble(data[idx]);
}

/** @inheritDoc */
/**
* @inheritDoc
*/
@Override
public int getType() {
return Type.DOUBLE;
public StorageType getType() {
return StorageType.FLOAT_64;
}

/** @inheritDoc */
Expand Down
Original file line number Diff line number Diff line change
@@ -1,8 +1,5 @@
package org.enso.table.data.column.storage;

import java.util.BitSet;
import java.util.List;

import org.enso.table.data.column.builder.object.Builder;
import org.enso.table.data.column.builder.object.NumericBuilder;
import org.enso.table.data.column.operation.map.MapOpStorage;
Expand All @@ -11,13 +8,20 @@
import org.enso.table.data.column.operation.map.numeric.LongBooleanOp;
import org.enso.table.data.column.operation.map.numeric.LongIsInOp;
import org.enso.table.data.column.operation.map.numeric.LongNumericOp;
import org.enso.table.data.column.storage.type.StorageType;
import org.enso.table.data.index.Index;
import org.enso.table.data.mask.OrderMask;
import org.enso.table.data.mask.SliceRange;
import org.graalvm.polyglot.Value;

import java.util.BitSet;
import java.util.List;

/** A column storing 64-bit integers. */
public final class LongStorage extends NumericStorage<Long> {
// TODO at some point we will want to add separate storage classes for byte, short and int, for more compact
// storage and more efficient handling of smaller integers; for now we will be handling this just by checking the
// bounds
private final long[] data;
private final BitSet isMissing;
private final int size;
Expand Down Expand Up @@ -75,13 +79,18 @@ public Long getItemBoxed(int idx) {
return isMissing.get(idx) ? null : data[idx];
}

/** @inheritDoc */
/**
* @inheritDoc
*/
@Override
public int getType() {
return Type.LONG;
public StorageType getType() {
// TODO add possibility to set integer bit limit
return StorageType.INTEGER_64;
}

/** @inheritDoc */
/**
* @inheritDoc
*/
@Override
public boolean isNa(long idx) {
return isMissing.get((int) idx);
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,92 @@
package org.enso.table.data.column.storage;

import org.enso.table.data.column.builder.object.Builder;
import org.enso.table.data.column.operation.map.MapOperationProblemBuilder;
import org.enso.table.data.column.storage.type.StorageType;
import org.enso.table.data.mask.OrderMask;
import org.enso.table.data.mask.SliceRange;

import java.util.BitSet;
import java.util.List;

public class MixedStorageFacade extends Storage<Object> {
private final Storage<?> underlyingStorage;
public MixedStorageFacade(Storage<?> storage) {
underlyingStorage = storage;
}

@Override
public int size() {
return underlyingStorage.size();
}

@Override
public int countMissing() {
return underlyingStorage.countMissing();
}

@Override
public StorageType getType() {
return StorageType.ANY_OBJECT;
}

@Override
public boolean isNa(long idx) {
return underlyingStorage.isNa(idx);
}

@Override
public Object getItemBoxed(int idx) {
return underlyingStorage.getItemBoxed(idx);
}

@Override
public boolean isOpVectorized(String name) {
return underlyingStorage.isOpVectorized(name);
}

@Override
protected Storage<?> runVectorizedMap(String name, Object argument, MapOperationProblemBuilder problemBuilder) {
return underlyingStorage.runVectorizedMap(name, argument, problemBuilder);
}

@Override
protected Storage<?> runVectorizedZip(String name, Storage<?> argument, MapOperationProblemBuilder problemBuilder) {
return underlyingStorage.runVectorizedZip(name, argument, problemBuilder);
}

@Override
public Storage<Object> mask(BitSet mask, int cardinality) {
Storage<?> newStorage = underlyingStorage.mask(mask, cardinality);
return new MixedStorageFacade(newStorage);
}

@Override
public Storage<Object> applyMask(OrderMask mask) {
Storage<?> newStorage = underlyingStorage.applyMask(mask);
return new MixedStorageFacade(newStorage);
}

@Override
public Storage<Object> countMask(int[] counts, int total) {
Storage<?> newStorage = underlyingStorage.countMask(counts, total);
return new MixedStorageFacade(newStorage);
}

@Override
public Storage<Object> slice(int offset, int limit) {
Storage<?> newStorage = underlyingStorage.slice(offset, limit);
return new MixedStorageFacade(newStorage);
}

@Override
public Builder createDefaultBuilderOfSameType(int capacity) {
throw new UnsupportedOperationException("TODO");
}

@Override
public Storage<Object> slice(List<SliceRange> ranges) {
Storage<?> newStorage = underlyingStorage.slice(ranges);
return new MixedStorageFacade(newStorage);
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
import org.enso.table.data.column.builder.object.ObjectBuilder;
import org.enso.table.data.column.operation.map.MapOpStorage;
import org.enso.table.data.column.operation.map.UnaryMapOperation;
import org.enso.table.data.column.storage.type.StorageType;

/** A column storing arbitrary objects. */
public final class ObjectStorage extends SpecializedStorage<Object> {
Expand All @@ -28,8 +29,8 @@ protected Object[] newUnderlyingArray(int size) {
}

@Override
public int getType() {
return Type.OBJECT;
public StorageType getType() {
return StorageType.ANY_OBJECT;
}

@Override
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
import java.util.List;
import org.enso.table.data.column.operation.map.MapOpStorage;
import org.enso.table.data.column.operation.map.MapOperationProblemBuilder;
import org.enso.table.data.column.storage.type.StorageType;
import org.enso.table.data.index.Index;
import org.enso.table.data.mask.OrderMask;
import org.enso.table.data.mask.SliceRange;
Expand All @@ -15,7 +16,7 @@ public abstract class SpecializedStorage<T> extends Storage<T> {
protected abstract T[] newUnderlyingArray(int size);

@Override
public abstract int getType();
public abstract StorageType getType();

/**
* @param data the underlying data
Expand Down
Loading

0 comments on commit d42a860

Please sign in to comment.