diff --git a/common/src/main/java/io/druid/math/expr/Evals.java b/common/src/main/java/io/druid/math/expr/Evals.java index 702037309a91..1ff14385738b 100644 --- a/common/src/main/java/io/druid/math/expr/Evals.java +++ b/common/src/main/java/io/druid/math/expr/Evals.java @@ -20,7 +20,6 @@ package io.druid.math.expr; import com.google.common.base.Strings; -import io.druid.common.guava.GuavaUtils; import io.druid.java.util.common.logger.Logger; import java.util.Arrays; @@ -32,27 +31,6 @@ public class Evals { private static final Logger log = new Logger(Evals.class); - public static Number toNumber(Object value) - { - if (value == null) { - return 0L; - } - if (value instanceof Number) { - return (Number) value; - } - String stringValue = String.valueOf(value); - Long longValue = GuavaUtils.tryParseLong(stringValue); - if (longValue == null) { - return Double.valueOf(stringValue); - } - return longValue; - } - - public static boolean isConstant(Expr expr) - { - return expr instanceof ConstantExpr; - } - public static boolean isAllConstants(Expr... exprs) { return isAllConstants(Arrays.asList(exprs)); diff --git a/extensions-contrib/virtual-columns/src/main/java/io/druid/segment/MapVirtualColumn.java b/extensions-contrib/virtual-columns/src/main/java/io/druid/segment/MapVirtualColumn.java index c94cba1c0fe5..33ce97c231ee 100644 --- a/extensions-contrib/virtual-columns/src/main/java/io/druid/segment/MapVirtualColumn.java +++ b/extensions-contrib/virtual-columns/src/main/java/io/druid/segment/MapVirtualColumn.java @@ -22,13 +22,20 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; import com.google.common.collect.Maps; import com.metamx.common.StringUtils; import io.druid.query.dimension.DefaultDimensionSpec; +import io.druid.query.dimension.DimensionSpec; import io.druid.query.filter.DimFilterUtils; +import io.druid.segment.column.ColumnCapabilities; +import io.druid.segment.column.ColumnCapabilitiesImpl; +import io.druid.segment.column.ValueType; import io.druid.segment.data.IndexedInts; +import io.druid.segment.virtual.VirtualColumnCacheHelper; import java.nio.ByteBuffer; +import java.util.List; import java.util.Map; import java.util.Objects; @@ -36,8 +43,6 @@ */ public class MapVirtualColumn implements VirtualColumn { - private static final byte VC_TYPE_ID = 0x00; - private final String outputName; private final String keyDimension; private final String valueDimension; @@ -59,13 +64,14 @@ public MapVirtualColumn( } @Override - public ObjectColumnSelector init(String dimension, ColumnSelectorFactory factory) + public ObjectColumnSelector makeObjectColumnSelector(String dimension, ColumnSelectorFactory factory) { final DimensionSelector keySelector = factory.makeDimensionSelector(DefaultDimensionSpec.of(keyDimension)); final DimensionSelector valueSelector = factory.makeDimensionSelector(DefaultDimensionSpec.of(valueDimension)); - int index = dimension.indexOf('.'); - if (index < 0) { + final String subColumnName = VirtualColumns.splitColumnName(dimension).rhs; + + if (subColumnName == null) { return new ObjectColumnSelector() { @Override @@ -97,7 +103,7 @@ public Map get() IdLookup keyIdLookup = keySelector.idLookup(); if (keyIdLookup != null) { - final int keyId = keyIdLookup.lookupId(dimension.substring(index + 1)); + final int keyId = keyIdLookup.lookupId(subColumnName); if (keyId < 0) { return NullStringObjectColumnSelector.instance(); } @@ -127,7 +133,6 @@ public String get() } }; } else { - final String key = dimension.substring(index + 1); return new ObjectColumnSelector() { @Override @@ -146,7 +151,7 @@ public String get() } final int limit = Math.min(keyIndices.size(), valueIndices.size()); for (int i = 0; i < limit; i++) { - if (Objects.equals(keySelector.lookupName(keyIndices.get(i)), key)) { + if (Objects.equals(keySelector.lookupName(keyIndices.get(i)), subColumnName)) { return valueSelector.lookupName(valueIndices.get(i)); } } @@ -156,6 +161,38 @@ public String get() } } + @Override + public DimensionSelector makeDimensionSelector(DimensionSpec dimensionSpec, ColumnSelectorFactory factory) + { + // Could probably do something useful here if the column name is dot-style. But for now just return nothing. + return null; + } + + @Override + public FloatColumnSelector makeFloatColumnSelector(String columnName, ColumnSelectorFactory factory) + { + return null; + } + + @Override + public LongColumnSelector makeLongColumnSelector(String columnName, ColumnSelectorFactory factory) + { + return null; + } + + @Override + public ColumnCapabilities capabilities(String columnName) + { + final ValueType valueType = columnName.indexOf('.') < 0 ? ValueType.COMPLEX : ValueType.STRING; + return new ColumnCapabilitiesImpl().setType(valueType); + } + + @Override + public List requiredColumns() + { + return ImmutableList.of(keyDimension, valueDimension); + } + @Override public boolean usesDotNotation() { @@ -170,7 +207,7 @@ public byte[] getCacheKey() byte[] output = StringUtils.toUtf8(outputName); return ByteBuffer.allocate(3 + key.length + value.length + output.length) - .put(VC_TYPE_ID) + .put(VirtualColumnCacheHelper.CACHE_TYPE_ID_MAP) .put(key).put(DimFilterUtils.STRING_SEPARATOR) .put(value).put(DimFilterUtils.STRING_SEPARATOR) .put(output) diff --git a/indexing-hadoop/src/main/java/io/druid/indexer/InputRowSerde.java b/indexing-hadoop/src/main/java/io/druid/indexer/InputRowSerde.java index a8b25460b1c4..937c0db262db 100644 --- a/indexing-hadoop/src/main/java/io/druid/indexer/InputRowSerde.java +++ b/indexing-hadoop/src/main/java/io/druid/indexer/InputRowSerde.java @@ -36,6 +36,7 @@ import io.druid.segment.incremental.IncrementalIndex; import io.druid.segment.serde.ComplexMetricSerde; import io.druid.segment.serde.ComplexMetrics; +import io.druid.segment.VirtualColumns; import org.apache.hadoop.io.ArrayWritable; import org.apache.hadoop.io.Text; import org.apache.hadoop.io.WritableUtils; @@ -87,6 +88,7 @@ public InputRow get() Aggregator agg = aggFactory.factorize( IncrementalIndex.makeColumnSelectorFactory( + VirtualColumns.EMPTY, aggFactory, supplier, true diff --git a/processing/src/main/java/io/druid/query/Druids.java b/processing/src/main/java/io/druid/query/Druids.java index dfa7d238fb50..b9f9db4b8a04 100644 --- a/processing/src/main/java/io/druid/query/Druids.java +++ b/processing/src/main/java/io/druid/query/Druids.java @@ -55,6 +55,7 @@ import io.druid.query.timeboundary.TimeBoundaryResultValue; import io.druid.query.timeseries.TimeseriesQuery; import io.druid.segment.VirtualColumn; +import io.druid.segment.VirtualColumns; import org.joda.time.DateTime; import org.joda.time.Interval; @@ -1104,7 +1105,7 @@ public static class SelectQueryBuilder private QueryGranularity granularity; private List dimensions; private List metrics; - private List virtualColumns; + private VirtualColumns virtualColumns; private PagingSpec pagingSpec; public SelectQueryBuilder() @@ -1233,12 +1234,22 @@ public SelectQueryBuilder metrics(List m) return this; } - public SelectQueryBuilder virtualColumns(List vcs) + public SelectQueryBuilder virtualColumns(VirtualColumns vcs) { virtualColumns = vcs; return this; } + public SelectQueryBuilder virtualColumns(List vcs) + { + return virtualColumns(VirtualColumns.create(vcs)); + } + + public SelectQueryBuilder virtualColumns(VirtualColumn... vcs) + { + return virtualColumns(VirtualColumns.create(Arrays.asList(vcs))); + } + public SelectQueryBuilder pagingSpec(PagingSpec p) { pagingSpec = p; diff --git a/processing/src/main/java/io/druid/query/extraction/BucketExtractionFn.java b/processing/src/main/java/io/druid/query/extraction/BucketExtractionFn.java index 0f9089921128..1606883954d0 100644 --- a/processing/src/main/java/io/druid/query/extraction/BucketExtractionFn.java +++ b/processing/src/main/java/io/druid/query/extraction/BucketExtractionFn.java @@ -58,7 +58,7 @@ public double getOffset() public String apply(Object value) { if (value instanceof Number) { - return bucket((Double) value); + return bucket(((Number) value).doubleValue()); } else if (value instanceof String) { return apply(value); } diff --git a/processing/src/main/java/io/druid/query/select/SelectQuery.java b/processing/src/main/java/io/druid/query/select/SelectQuery.java index 4dd27a593083..d33ff67fe7d8 100644 --- a/processing/src/main/java/io/druid/query/select/SelectQuery.java +++ b/processing/src/main/java/io/druid/query/select/SelectQuery.java @@ -31,7 +31,7 @@ import io.druid.query.dimension.DimensionSpec; import io.druid.query.filter.DimFilter; import io.druid.query.spec.QuerySegmentSpec; -import io.druid.segment.VirtualColumn; +import io.druid.segment.VirtualColumns; import java.util.List; import java.util.Map; @@ -46,7 +46,7 @@ public class SelectQuery extends BaseQuery> private final QueryGranularity granularity; private final List dimensions; private final List metrics; - private final List virtualColumns; + private final VirtualColumns virtualColumns; private final PagingSpec pagingSpec; @JsonCreator @@ -58,7 +58,7 @@ public SelectQuery( @JsonProperty("granularity") QueryGranularity granularity, @JsonProperty("dimensions") List dimensions, @JsonProperty("metrics") List metrics, - @JsonProperty("virtualColumns") List virtualColumns, + @JsonProperty("virtualColumns") VirtualColumns virtualColumns, @JsonProperty("pagingSpec") PagingSpec pagingSpec, @JsonProperty("context") Map context ) @@ -67,7 +67,7 @@ public SelectQuery( this.dimFilter = dimFilter; this.granularity = granularity; this.dimensions = dimensions; - this.virtualColumns = virtualColumns; + this.virtualColumns = virtualColumns == null ? VirtualColumns.EMPTY : virtualColumns; this.metrics = metrics; this.pagingSpec = pagingSpec; @@ -134,7 +134,7 @@ public List getMetrics() } @JsonProperty - public List getVirtualColumns() + public VirtualColumns getVirtualColumns() { return virtualColumns; } diff --git a/processing/src/main/java/io/druid/query/select/SelectQueryEngine.java b/processing/src/main/java/io/druid/query/select/SelectQueryEngine.java index 41597aab2bb8..31a160f08036 100644 --- a/processing/src/main/java/io/druid/query/select/SelectQueryEngine.java +++ b/processing/src/main/java/io/druid/query/select/SelectQueryEngine.java @@ -43,7 +43,6 @@ import io.druid.segment.ObjectColumnSelector; import io.druid.segment.Segment; import io.druid.segment.StorageAdapter; -import io.druid.segment.VirtualColumns; import io.druid.segment.column.Column; import io.druid.segment.column.ColumnCapabilities; import io.druid.segment.column.ValueType; @@ -161,7 +160,7 @@ public Sequence> process(final SelectQuery query, fina adapter, query.getQuerySegmentSpec().getIntervals(), filter, - VirtualColumns.valueOf(query.getVirtualColumns()), + query.getVirtualColumns(), query.isDescending(), query.getGranularity(), new Function>() diff --git a/processing/src/main/java/io/druid/query/select/SelectQueryQueryToolChest.java b/processing/src/main/java/io/druid/query/select/SelectQueryQueryToolChest.java index 99ff72251fd4..2bb009835e2d 100644 --- a/processing/src/main/java/io/druid/query/select/SelectQueryQueryToolChest.java +++ b/processing/src/main/java/io/druid/query/select/SelectQueryQueryToolChest.java @@ -49,7 +49,6 @@ import io.druid.query.dimension.DimensionSpec; import io.druid.query.filter.DimFilter; import io.druid.timeline.DataSegmentUtils; -import io.druid.segment.VirtualColumn; import io.druid.timeline.LogicalSegment; import org.joda.time.DateTime; import org.joda.time.Interval; @@ -192,20 +191,7 @@ public byte[] computeCacheKey(SelectQuery query) ++index; } - List virtualColumns = query.getVirtualColumns(); - if (virtualColumns == null) { - virtualColumns = Collections.emptyList(); - } - - final byte[][] virtualColumnsBytes = new byte[virtualColumns.size()][]; - int virtualColumnsBytesSize = 0; - index = 0; - for (VirtualColumn vc : virtualColumns) { - virtualColumnsBytes[index] = vc.getCacheKey(); - virtualColumnsBytesSize += virtualColumnsBytes[index].length; - ++index; - } - + final byte[] virtualColumnsCacheKey = query.getVirtualColumns().getCacheKey(); final ByteBuffer queryCacheKey = ByteBuffer .allocate( 1 @@ -214,7 +200,7 @@ public byte[] computeCacheKey(SelectQuery query) + query.getPagingSpec().getCacheKey().length + dimensionsBytesSize + metricBytesSize - + virtualColumnsBytesSize + + virtualColumnsCacheKey.length ) .put(SELECT_QUERY) .put(granularityBytes) @@ -229,9 +215,7 @@ public byte[] computeCacheKey(SelectQuery query) queryCacheKey.put(metricByte); } - for (byte[] vcByte : virtualColumnsBytes) { - queryCacheKey.put(vcByte); - } + queryCacheKey.put(virtualColumnsCacheKey); return queryCacheKey.array(); } diff --git a/processing/src/main/java/io/druid/segment/ColumnSelectorFactory.java b/processing/src/main/java/io/druid/segment/ColumnSelectorFactory.java index f550fef14e5c..a13afabc253c 100644 --- a/processing/src/main/java/io/druid/segment/ColumnSelectorFactory.java +++ b/processing/src/main/java/io/druid/segment/ColumnSelectorFactory.java @@ -22,14 +22,29 @@ import io.druid.query.dimension.DimensionSpec; import io.druid.segment.column.ColumnCapabilities; +import javax.annotation.Nullable; + /** * Factory class for MetricSelectors */ public interface ColumnSelectorFactory { - public DimensionSelector makeDimensionSelector(DimensionSpec dimensionSpec); - public FloatColumnSelector makeFloatColumnSelector(String columnName); - public LongColumnSelector makeLongColumnSelector(String columnName); - public ObjectColumnSelector makeObjectColumnSelector(String columnName); - public ColumnCapabilities getColumnCapabilities(String columnName); + DimensionSelector makeDimensionSelector(DimensionSpec dimensionSpec); + FloatColumnSelector makeFloatColumnSelector(String columnName); + LongColumnSelector makeLongColumnSelector(String columnName); + + @Nullable + ObjectColumnSelector makeObjectColumnSelector(String columnName); + + /** + * Returns capabilities of a particular column, if known. May be null if the column doesn't exist, or if + * the column does exist but the capabilities are unknown. The latter is possible with dynamically discovered + * columns. + * + * @param column column name + * + * @return capabilities, or null + */ + @Nullable + ColumnCapabilities getColumnCapabilities(String column); } diff --git a/processing/src/main/java/io/druid/segment/NullDimensionSelector.java b/processing/src/main/java/io/druid/segment/NullDimensionSelector.java index cdbd371bf04e..b1c7d03b006e 100644 --- a/processing/src/main/java/io/druid/segment/NullDimensionSelector.java +++ b/processing/src/main/java/io/druid/segment/NullDimensionSelector.java @@ -30,6 +30,18 @@ public class NullDimensionSelector implements DimensionSelector, IdLookup { + private static final NullDimensionSelector INSTANCE = new NullDimensionSelector(); + + private NullDimensionSelector() + { + // Singleton. + } + + public static NullDimensionSelector instance() + { + return INSTANCE; + } + @Override public IndexedInts getRow() { diff --git a/processing/src/main/java/io/druid/segment/QueryableIndexStorageAdapter.java b/processing/src/main/java/io/druid/segment/QueryableIndexStorageAdapter.java index dc3d4804a68a..6e778b7aa399 100644 --- a/processing/src/main/java/io/druid/segment/QueryableIndexStorageAdapter.java +++ b/processing/src/main/java/io/druid/segment/QueryableIndexStorageAdapter.java @@ -41,7 +41,6 @@ import io.druid.segment.column.BitmapIndex; import io.druid.segment.column.Column; import io.druid.segment.column.ColumnCapabilities; -import io.druid.segment.column.ColumnCapabilitiesImpl; import io.druid.segment.column.ComplexColumn; import io.druid.segment.column.DictionaryEncodedColumn; import io.druid.segment.column.GenericColumn; @@ -68,8 +67,6 @@ */ public class QueryableIndexStorageAdapter implements StorageAdapter { - private static final NullDimensionSelector NULL_DIMENSION_SELECTOR = new NullDimensionSelector(); - private final QueryableIndex index; public QueryableIndexStorageAdapter( @@ -430,6 +427,10 @@ public DimensionSelector makeDimensionSelector( DimensionSpec dimensionSpec ) { + if (virtualColumns.exists(dimensionSpec.getDimension())) { + return virtualColumns.makeDimensionSelector(dimensionSpec, this); + } + return dimensionSpec.decorate(makeDimensionSelectorUndecorated(dimensionSpec)); } @@ -442,7 +443,7 @@ private DimensionSelector makeDimensionSelectorUndecorated( final Column columnDesc = index.getColumn(dimension); if (columnDesc == null) { - return NULL_DIMENSION_SELECTOR; + return NullDimensionSelector.instance(); } if (dimension.equals(Column.TIME_COLUMN_NAME)) { @@ -463,7 +464,7 @@ private DimensionSelector makeDimensionSelectorUndecorated( final DictionaryEncodedColumn column = cachedColumn; if (column == null) { - return NULL_DIMENSION_SELECTOR; + return NullDimensionSelector.instance(); } else if (columnDesc.getCapabilities().hasMultipleValues()) { class MultiValueDimensionSelector implements DimensionSelector, IdLookup { @@ -652,6 +653,10 @@ public int lookupId(String name) @Override public FloatColumnSelector makeFloatColumnSelector(String columnName) { + if (virtualColumns.exists(columnName)) { + return virtualColumns.makeFloatColumnSelector(columnName, this); + } + GenericColumn cachedMetricVals = genericColumnCache.get(columnName); if (cachedMetricVals == null) { @@ -665,14 +670,7 @@ public FloatColumnSelector makeFloatColumnSelector(String columnName) } if (cachedMetricVals == null) { - return new FloatColumnSelector() - { - @Override - public float get() - { - return 0.0f; - } - }; + return ZeroFloatColumnSelector.instance(); } final GenericColumn metricVals = cachedMetricVals; @@ -689,6 +687,10 @@ public float get() @Override public LongColumnSelector makeLongColumnSelector(String columnName) { + if (virtualColumns.exists(columnName)) { + return virtualColumns.makeLongColumnSelector(columnName, this); + } + GenericColumn cachedMetricVals = genericColumnCache.get(columnName); if (cachedMetricVals == null) { @@ -702,14 +704,7 @@ public LongColumnSelector makeLongColumnSelector(String columnName) } if (cachedMetricVals == null) { - return new LongColumnSelector() - { - @Override - public long get() - { - return 0L; - } - }; + return ZeroLongColumnSelector.instance(); } final GenericColumn metricVals = cachedMetricVals; @@ -727,6 +722,10 @@ public long get() @Override public ObjectColumnSelector makeObjectColumnSelector(String column) { + if (virtualColumns.exists(column)) { + return virtualColumns.makeObjectColumnSelector(column, this); + } + Object cachedColumnVals = objectColumnCache.get(column); if (cachedColumnVals == null) { @@ -751,10 +750,6 @@ public ObjectColumnSelector makeObjectColumnSelector(String column) } if (cachedColumnVals == null) { - VirtualColumn vc = virtualColumns.getVirtualColumn(column); - if (vc != null) { - return vc.init(column, this); - } return null; } @@ -881,19 +876,14 @@ public Object get() }; } - @Nullable @Override public ColumnCapabilities getColumnCapabilities(String columnName) { - ColumnCapabilities capabilities = getColumnCapabilites(index, columnName); - if (capabilities == null && !virtualColumns.isEmpty()) { - VirtualColumn virtualColumn = virtualColumns.getVirtualColumn(columnName); - if (virtualColumn != null) { - Class clazz = virtualColumn.init(columnName, this).classOfObject(); - capabilities = new ColumnCapabilitiesImpl().setType(ValueType.typeFor(clazz)); - } + if (virtualColumns.exists(columnName)) { + return virtualColumns.getColumnCapabilities(columnName); } - return capabilities; + + return getColumnCapabilites(index, columnName); } } diff --git a/processing/src/main/java/io/druid/segment/VirtualColumn.java b/processing/src/main/java/io/druid/segment/VirtualColumn.java index 855affe8bf4f..beef428fef41 100644 --- a/processing/src/main/java/io/druid/segment/VirtualColumn.java +++ b/processing/src/main/java/io/druid/segment/VirtualColumn.java @@ -19,15 +19,26 @@ package io.druid.segment; +import com.fasterxml.jackson.annotation.JsonSubTypes; import com.fasterxml.jackson.annotation.JsonTypeInfo; +import io.druid.query.dimension.DimensionSpec; +import io.druid.segment.column.ColumnCapabilities; +import io.druid.segment.virtual.ExpressionVirtualColumn; + +import javax.annotation.Nullable; +import java.util.List; -/** - */ -@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "type") /** * Virtual columns are "views" created over a ColumnSelectorFactory. They can potentially draw from multiple * underlying columns, although they always present themselves as if they were a single column. + * + * A virtual column object will be shared amongst threads and must be thread safe. The selectors returned + * from the various makeXXXSelector methods need not be thread safe. */ +@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "type") +@JsonSubTypes(value = { + @JsonSubTypes.Type(name = "expression", value = ExpressionVirtualColumn.class) +}) public interface VirtualColumn { /** @@ -42,10 +53,73 @@ public interface VirtualColumn * virtual column was referenced with, which is useful if this column uses dot notation. * * @param columnName the name this virtual column was referenced with - * @param factory column selector factory - * @return the selector + * @param factory column selector factory + * + * @return the selector, must not be null + */ + ObjectColumnSelector makeObjectColumnSelector(String columnName, ColumnSelectorFactory factory); + + /** + * Build a selector corresponding to this virtual column. Also provides the name that the + * virtual column was referenced with (through {@link DimensionSpec#getDimension()}, which + * is useful if this column uses dot notation. The virtual column is expected to apply any + * necessary decoration from the dimensionSpec. + * + * @param dimensionSpec the dimensionSpec this column was referenced with + * @param factory column selector factory + * + * @return the selector, or null if we can't make a selector + */ + @Nullable + DimensionSelector makeDimensionSelector(DimensionSpec dimensionSpec, ColumnSelectorFactory factory); + + /** + * Build a selector corresponding to this virtual column. Also provides the name that the + * virtual column was referenced with, which is useful if this column uses dot notation. + * + * @param columnName the name this virtual column was referenced with + * @param factory column selector factory + * + * @return the selector, or null if we can't make a selector + */ + @Nullable + FloatColumnSelector makeFloatColumnSelector(String columnName, ColumnSelectorFactory factory); + + /** + * Build a selector corresponding to this virtual column. Also provides the name that the + * virtual column was referenced with, which is useful if this column uses dot notation. + * + * @param columnName the name this virtual column was referenced with + * @param factory column selector factory + * + * @return the selector, or null if we can't make a selector + */ + @Nullable + LongColumnSelector makeLongColumnSelector(String columnName, ColumnSelectorFactory factory); + + /** + * Returns the capabilities of this virtual column, which includes a type that should match + * the type returned by "makeObjectColumnSelector" and should correspond to the best + * performing selector. May vary based on columnName if this column uses dot notation. + * + * @param columnName the name this virtual column was referenced with + * + * @return capabilities, must not be null + */ + ColumnCapabilities capabilities(String columnName); + + /** + * Returns a list of columns that this virtual column will access. This may include the + * names of other virtual columns. May be empty if a virtual column doesn't access any + * underlying columns. + * + * Does not pass columnName because there is an assumption that the list of columns + * needed by a dot-notation supporting virtual column will not vary based on the + * columnName. + * + * @return column names */ - ObjectColumnSelector init(String columnName, ColumnSelectorFactory factory); + List requiredColumns(); /** * Indicates that this virtual column can be referenced with dot notation. For example, diff --git a/processing/src/main/java/io/druid/segment/VirtualColumns.java b/processing/src/main/java/io/druid/segment/VirtualColumns.java index 799a0ca366e7..bf4acccb76b0 100644 --- a/processing/src/main/java/io/druid/segment/VirtualColumns.java +++ b/processing/src/main/java/io/druid/segment/VirtualColumns.java @@ -19,62 +19,256 @@ package io.druid.segment; +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonValue; +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Maps; +import com.google.common.collect.Sets; +import com.google.common.primitives.Ints; +import io.druid.java.util.common.IAE; +import io.druid.java.util.common.Pair; +import io.druid.query.dimension.DimensionSpec; +import io.druid.segment.column.Column; +import io.druid.segment.column.ColumnCapabilities; +import io.druid.segment.virtual.VirtualizedColumnSelectorFactory; +import java.nio.ByteBuffer; import java.util.List; import java.util.Map; +import java.util.Set; /** + * Class allowing lookup and usage of virtual columns. */ public class VirtualColumns { public static final VirtualColumns EMPTY = new VirtualColumns( - ImmutableMap.of(), ImmutableMap.of() + ImmutableList.of(), + ImmutableMap.of(), + ImmutableMap.of() ); - public static VirtualColumns valueOf(List virtualColumns) { + /** + * Split a dot-style columnName into the "main" columnName and the subColumn name after the dot. Useful for + * columns that support dot notation. + * + * @param columnName columnName like "foo" or "foo.bar" + * + * @return pair of main column name (will not be null) and subColumn name (may be null) + */ + public static Pair splitColumnName(String columnName) + { + final int i = columnName.indexOf('.'); + if (i < 0) { + return Pair.of(columnName, null); + } else { + return Pair.of(columnName.substring(0, i), columnName.substring(i + 1)); + } + } + + @JsonCreator + public static VirtualColumns create(List virtualColumns) + { if (virtualColumns == null || virtualColumns.isEmpty()) { return EMPTY; } Map withDotSupport = Maps.newHashMap(); Map withoutDotSupport = Maps.newHashMap(); for (VirtualColumn vc : virtualColumns) { + if (vc.getOutputName().equals(Column.TIME_COLUMN_NAME)) { + throw new IAE("virtualColumn name[%s] not allowed", vc.getOutputName()); + } + + if (withDotSupport.containsKey(vc.getOutputName()) || withoutDotSupport.containsKey(vc.getOutputName())) { + throw new IAE("Duplicate virtualColumn name[%s]", vc.getOutputName()); + } + if (vc.usesDotNotation()) { withDotSupport.put(vc.getOutputName(), vc); } else { withoutDotSupport.put(vc.getOutputName(), vc); } } - return new VirtualColumns(withDotSupport, withoutDotSupport); + return new VirtualColumns(ImmutableList.copyOf(virtualColumns), withDotSupport, withoutDotSupport); } - public VirtualColumns(Map withDotSupport, Map withoutDotSupport) + private VirtualColumns( + List virtualColumns, + Map withDotSupport, + Map withoutDotSupport + ) { + this.virtualColumns = virtualColumns; this.withDotSupport = withDotSupport; this.withoutDotSupport = withoutDotSupport; + + for (VirtualColumn virtualColumn : virtualColumns) { + detectCycles(virtualColumn, null); + } } + // For equals, hashCode, toString, and serialization: + private final List virtualColumns; + + // For getVirtualColumn: private final Map withDotSupport; private final Map withoutDotSupport; - public VirtualColumn getVirtualColumn(String dimension) + public boolean exists(String columnName) + { + return getVirtualColumn(columnName) != null; + } + + public VirtualColumn getVirtualColumn(String columnName) { - VirtualColumn vc = withoutDotSupport.get(dimension); + final VirtualColumn vc = withoutDotSupport.get(columnName); if (vc != null) { return vc; } - for (int index = dimension.indexOf('.'); index >= 0; index = dimension.indexOf('.', index + 1)) { - vc = withDotSupport.get(dimension.substring(0, index)); - if (vc != null) { - return vc; - } + final String baseColumnName = splitColumnName(columnName).lhs; + return withDotSupport.get(baseColumnName); + } + + public ObjectColumnSelector makeObjectColumnSelector(String columnName, ColumnSelectorFactory factory) + { + final VirtualColumn virtualColumn = getVirtualColumn(columnName); + if (virtualColumn == null) { + return null; + } else { + return Preconditions.checkNotNull( + virtualColumn.makeObjectColumnSelector(columnName, factory), + "VirtualColumn[%s] returned a null ObjectColumnSelector for columnName[%s]", + virtualColumn.getOutputName(), + columnName + ); + } + } + + public DimensionSelector makeDimensionSelector(DimensionSpec dimensionSpec, ColumnSelectorFactory factory) + { + final VirtualColumn virtualColumn = getVirtualColumn(dimensionSpec.getDimension()); + if (virtualColumn == null) { + return dimensionSpec.decorate(NullDimensionSelector.instance()); + } else { + final DimensionSelector selector = virtualColumn.makeDimensionSelector(dimensionSpec, factory); + return selector == null ? dimensionSpec.decorate(NullDimensionSelector.instance()) : selector; + } + } + + public FloatColumnSelector makeFloatColumnSelector(String columnName, ColumnSelectorFactory factory) + { + final VirtualColumn virtualColumn = getVirtualColumn(columnName); + if (virtualColumn == null) { + return ZeroFloatColumnSelector.instance(); + } else { + final FloatColumnSelector selector = virtualColumn.makeFloatColumnSelector(columnName, factory); + return selector == null ? ZeroFloatColumnSelector.instance() : selector; + } + } + + public LongColumnSelector makeLongColumnSelector(String columnName, ColumnSelectorFactory factory) + { + final VirtualColumn virtualColumn = getVirtualColumn(columnName); + if (virtualColumn == null) { + return ZeroLongColumnSelector.instance(); + } else { + final LongColumnSelector selector = virtualColumn.makeLongColumnSelector(columnName, factory); + return selector == null ? ZeroLongColumnSelector.instance() : selector; + } + } + + public ColumnCapabilities getColumnCapabilities(String columnName) + { + final VirtualColumn virtualColumn = getVirtualColumn(columnName); + if (virtualColumn != null) { + return Preconditions.checkNotNull( + virtualColumn.capabilities(columnName), + "capabilities for column[%s]", + columnName + ); + } else { + return null; } - return withDotSupport.get(dimension); } public boolean isEmpty() { return withDotSupport.isEmpty() && withoutDotSupport.isEmpty(); } + + @JsonValue + public VirtualColumn[] getVirtualColumns() + { + // VirtualColumn[] instead of List to aid Jackson serialization. + return virtualColumns.toArray(new VirtualColumn[]{}); + } + + public ColumnSelectorFactory wrap(final ColumnSelectorFactory baseFactory) + { + return new VirtualizedColumnSelectorFactory(baseFactory, this); + } + + public byte[] getCacheKey() + { + final byte[][] cacheKeys = new byte[virtualColumns.size()][]; + int len = Ints.BYTES; + for (int i = 0; i < virtualColumns.size(); i++) { + cacheKeys[i] = virtualColumns.get(i).getCacheKey(); + len += Ints.BYTES + cacheKeys[i].length; + } + final ByteBuffer buf = ByteBuffer.allocate(len).putInt(virtualColumns.size()); + for (byte[] cacheKey : cacheKeys) { + buf.putInt(cacheKey.length); + buf.put(cacheKey); + } + return buf.array(); + } + + private void detectCycles(VirtualColumn virtualColumn, Set columnNames) + { + // Copy columnNames to avoid modifying it + final Set nextSet = columnNames == null + ? Sets.newHashSet(virtualColumn.getOutputName()) + : Sets.newHashSet(columnNames); + + for (String columnName : virtualColumn.requiredColumns()) { + if (!nextSet.add(columnName)) { + throw new IAE("Self-referential column[%s]", columnName); + } + + final VirtualColumn dependency = getVirtualColumn(columnName); + if (dependency != null) { + detectCycles(dependency, nextSet); + } + } + } + + @Override + public boolean equals(Object o) + { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + + VirtualColumns that = (VirtualColumns) o; + + return virtualColumns.equals(that.virtualColumns); + } + + @Override + public int hashCode() + { + return virtualColumns.hashCode(); + } + + @Override + public String toString() + { + return virtualColumns.toString(); + } } diff --git a/processing/src/main/java/io/druid/segment/ZeroFloatColumnSelector.java b/processing/src/main/java/io/druid/segment/ZeroFloatColumnSelector.java new file mode 100644 index 000000000000..888214862f81 --- /dev/null +++ b/processing/src/main/java/io/druid/segment/ZeroFloatColumnSelector.java @@ -0,0 +1,41 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.segment; + +public final class ZeroFloatColumnSelector implements FloatColumnSelector +{ + private static final ZeroFloatColumnSelector INSTANCE = new ZeroFloatColumnSelector(); + + private ZeroFloatColumnSelector() + { + // No instantiation. + } + + public static ZeroFloatColumnSelector instance() + { + return INSTANCE; + } + + @Override + public float get() + { + return 0.0f; + } +} diff --git a/processing/src/main/java/io/druid/segment/ZeroLongColumnSelector.java b/processing/src/main/java/io/druid/segment/ZeroLongColumnSelector.java new file mode 100644 index 000000000000..911c0e24b265 --- /dev/null +++ b/processing/src/main/java/io/druid/segment/ZeroLongColumnSelector.java @@ -0,0 +1,41 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.segment; + +public final class ZeroLongColumnSelector implements LongColumnSelector +{ + private static final ZeroLongColumnSelector INSTANCE = new ZeroLongColumnSelector(); + + private ZeroLongColumnSelector() + { + // No instantiation. + } + + public static ZeroLongColumnSelector instance() + { + return INSTANCE; + } + + @Override + public long get() + { + return 0; + } +} diff --git a/processing/src/main/java/io/druid/segment/column/ValueType.java b/processing/src/main/java/io/druid/segment/column/ValueType.java index 1760fea8c725..aa9919a129b9 100644 --- a/processing/src/main/java/io/druid/segment/column/ValueType.java +++ b/processing/src/main/java/io/druid/segment/column/ValueType.java @@ -26,17 +26,5 @@ public enum ValueType FLOAT, LONG, STRING, - COMPLEX; - - public static ValueType typeFor(Class clazz) - { - if (clazz == String.class) { - return STRING; - } else if (clazz == float.class || clazz == Float.TYPE) { - return FLOAT; - } else if (clazz == long.class || clazz == Long.TYPE) { - return LONG; - } - return COMPLEX; - } + COMPLEX } diff --git a/processing/src/main/java/io/druid/segment/incremental/IncrementalIndex.java b/processing/src/main/java/io/druid/segment/incremental/IncrementalIndex.java index 431ebe18df67..24d0ce390a33 100644 --- a/processing/src/main/java/io/druid/segment/incremental/IncrementalIndex.java +++ b/processing/src/main/java/io/druid/segment/incremental/IncrementalIndex.java @@ -53,6 +53,7 @@ import io.druid.segment.LongColumnSelector; import io.druid.segment.Metadata; import io.druid.segment.ObjectColumnSelector; +import io.druid.segment.VirtualColumns; import io.druid.segment.column.Column; import io.druid.segment.column.ColumnCapabilities; import io.druid.segment.column.ColumnCapabilitiesImpl; @@ -100,14 +101,25 @@ public abstract class IncrementalIndex implements Iterable, .put(DimensionSchema.ValueType.STRING, ValueType.STRING) .build(); + /** + * Column selector used at ingestion time for inputs to aggregators. + * + * @param agg the aggregator + * @param in ingestion-time input row supplier + * @param deserializeComplexMetrics whether complex objects should be deserialized by a {@link ComplexMetricExtractor} + * + * @return column selector factory + */ public static ColumnSelectorFactory makeColumnSelectorFactory( + final VirtualColumns virtualColumns, final AggregatorFactory agg, final Supplier in, final boolean deserializeComplexMetrics ) { final RowBasedColumnSelectorFactory baseSelectorFactory = RowBasedColumnSelectorFactory.create(in, null); - return new ColumnSelectorFactory() + + class IncrementalIndexInputRowColumnSelectorFactory implements ColumnSelectorFactory { @Override public LongColumnSelector makeLongColumnSelector(final String columnName) @@ -167,13 +179,16 @@ public ColumnCapabilities getColumnCapabilities(String columnName) { return baseSelectorFactory.getColumnCapabilities(columnName); } - }; + } + + return virtualColumns.wrap(new IncrementalIndexInputRowColumnSelectorFactory()); } private final long minTimestamp; private final QueryGranularity gran; private final boolean rollup; private final List> rowTransformers; + private final VirtualColumns virtualColumns; private final AggregatorFactory[] metrics; private final AggregatorType[] aggs; private final boolean deserializeComplexMetrics; @@ -217,6 +232,7 @@ public IncrementalIndex( this.minTimestamp = incrementalIndexSchema.getMinTimestamp(); this.gran = incrementalIndexSchema.getGran(); this.rollup = incrementalIndexSchema.isRollup(); + this.virtualColumns = incrementalIndexSchema.getVirtualColumns(); this.metrics = incrementalIndexSchema.getMetrics(); this.rowTransformers = new CopyOnWriteArrayList<>(); this.deserializeComplexMetrics = deserializeComplexMetrics; @@ -894,6 +910,15 @@ public int hashCode() } } + protected ColumnSelectorFactory makeColumnSelectorFactory( + final AggregatorFactory agg, + final Supplier in, + final boolean deserializeComplexMetrics + ) + { + return makeColumnSelectorFactory(virtualColumns, agg, in, deserializeComplexMetrics); + } + protected final Comparator dimsComparator() { return new TimeAndDimsComp(dimensionDescsList); diff --git a/processing/src/main/java/io/druid/segment/incremental/IncrementalIndexSchema.java b/processing/src/main/java/io/druid/segment/incremental/IncrementalIndexSchema.java index b3bbfb187896..ca26a95b9d2b 100644 --- a/processing/src/main/java/io/druid/segment/incremental/IncrementalIndexSchema.java +++ b/processing/src/main/java/io/druid/segment/incremental/IncrementalIndexSchema.java @@ -25,6 +25,7 @@ import io.druid.granularity.QueryGranularities; import io.druid.granularity.QueryGranularity; import io.druid.query.aggregation.AggregatorFactory; +import io.druid.segment.VirtualColumns; /** */ @@ -34,6 +35,7 @@ public class IncrementalIndexSchema private final long minTimestamp; private final TimestampSpec timestampSpec; private final QueryGranularity gran; + private final VirtualColumns virtualColumns; private final DimensionsSpec dimensionsSpec; private final AggregatorFactory[] metrics; private final boolean rollup; @@ -42,6 +44,7 @@ public IncrementalIndexSchema( long minTimestamp, TimestampSpec timestampSpec, QueryGranularity gran, + VirtualColumns virtualColumns, DimensionsSpec dimensionsSpec, AggregatorFactory[] metrics, boolean rollup @@ -50,6 +53,7 @@ public IncrementalIndexSchema( this.minTimestamp = minTimestamp; this.timestampSpec = timestampSpec; this.gran = gran; + this.virtualColumns = virtualColumns == null ? VirtualColumns.EMPTY : virtualColumns; this.dimensionsSpec = dimensionsSpec; this.metrics = metrics; this.rollup = rollup; @@ -70,6 +74,11 @@ public QueryGranularity getGran() return gran; } + public VirtualColumns getVirtualColumns() + { + return virtualColumns; + } + public DimensionsSpec getDimensionsSpec() { return dimensionsSpec; @@ -90,6 +99,7 @@ public static class Builder private long minTimestamp; private TimestampSpec timestampSpec; private QueryGranularity gran; + private VirtualColumns virtualColumns; private DimensionsSpec dimensionsSpec; private AggregatorFactory[] metrics; private boolean rollup; @@ -98,6 +108,7 @@ public Builder() { this.minTimestamp = 0L; this.gran = QueryGranularities.NONE; + this.virtualColumns = VirtualColumns.EMPTY; this.dimensionsSpec = new DimensionsSpec(null, null, null); this.metrics = new AggregatorFactory[]{}; this.rollup = true; @@ -133,6 +144,12 @@ public Builder withQueryGranularity(QueryGranularity gran) return this; } + public Builder withVirtualColumns(VirtualColumns virtualColumns) + { + this.virtualColumns = virtualColumns; + return this; + } + public Builder withDimensionsSpec(DimensionsSpec dimensionsSpec) { this.dimensionsSpec = dimensionsSpec == null ? DimensionsSpec.ofEmpty() : dimensionsSpec; @@ -167,7 +184,7 @@ public Builder withRollup(boolean rollup) public IncrementalIndexSchema build() { return new IncrementalIndexSchema( - minTimestamp, timestampSpec, gran, dimensionsSpec, metrics, rollup + minTimestamp, timestampSpec, gran, virtualColumns, dimensionsSpec, metrics, rollup ); } } diff --git a/processing/src/main/java/io/druid/segment/incremental/IncrementalIndexStorageAdapter.java b/processing/src/main/java/io/druid/segment/incremental/IncrementalIndexStorageAdapter.java index 44d67fc686f6..b553774aa62b 100644 --- a/processing/src/main/java/io/druid/segment/incremental/IncrementalIndexStorageAdapter.java +++ b/processing/src/main/java/io/druid/segment/incremental/IncrementalIndexStorageAdapter.java @@ -44,12 +44,11 @@ import io.druid.segment.ObjectColumnSelector; import io.druid.segment.SingleScanTimeDimSelector; import io.druid.segment.StorageAdapter; -import io.druid.segment.VirtualColumn; import io.druid.segment.VirtualColumns; +import io.druid.segment.ZeroFloatColumnSelector; +import io.druid.segment.ZeroLongColumnSelector; import io.druid.segment.column.Column; import io.druid.segment.column.ColumnCapabilities; -import io.druid.segment.column.ColumnCapabilitiesImpl; -import io.druid.segment.column.ValueType; import io.druid.segment.data.Indexed; import io.druid.segment.data.ListIndexed; import io.druid.segment.filter.BooleanValueMatcher; @@ -64,8 +63,6 @@ */ public class IncrementalIndexStorageAdapter implements StorageAdapter { - private static final NullDimensionSelector NULL_DIMENSION_SELECTOR = new NullDimensionSelector(); - private final IncrementalIndex index; public IncrementalIndexStorageAdapter( @@ -340,6 +337,10 @@ public DimensionSelector makeDimensionSelector( DimensionSpec dimensionSpec ) { + if (virtualColumns.exists(dimensionSpec.getDimension())) { + return virtualColumns.makeDimensionSelector(dimensionSpec, this); + } + final String dimension = dimensionSpec.getDimension(); final ExtractionFn extractionFn = dimensionSpec.getExtractionFn(); @@ -354,7 +355,7 @@ public DimensionSelector makeDimensionSelector( final IncrementalIndex.DimensionDesc dimensionDesc = index.getDimension(dimensionSpec.getDimension()); if (dimensionDesc == null) { - return dimensionSpec.decorate(NULL_DIMENSION_SELECTOR); + return dimensionSpec.decorate(NullDimensionSelector.instance()); } final DimensionIndexer indexer = dimensionDesc.getIndexer(); @@ -364,6 +365,10 @@ public DimensionSelector makeDimensionSelector( @Override public FloatColumnSelector makeFloatColumnSelector(String columnName) { + if (virtualColumns.exists(columnName)) { + return virtualColumns.makeFloatColumnSelector(columnName, this); + } + final Integer dimIndex = index.getDimensionIndex(columnName); if (dimIndex != null) { final IncrementalIndex.DimensionDesc dimensionDesc = index.getDimension(columnName); @@ -377,14 +382,7 @@ public FloatColumnSelector makeFloatColumnSelector(String columnName) final Integer metricIndexInt = index.getMetricIndex(columnName); if (metricIndexInt == null) { - return new FloatColumnSelector() - { - @Override - public float get() - { - return 0.0f; - } - }; + return ZeroFloatColumnSelector.instance(); } final int metricIndex = metricIndexInt; @@ -401,6 +399,10 @@ public float get() @Override public LongColumnSelector makeLongColumnSelector(String columnName) { + if (virtualColumns.exists(columnName)) { + return virtualColumns.makeLongColumnSelector(columnName, this); + } + if (columnName.equals(Column.TIME_COLUMN_NAME)) { return new LongColumnSelector() { @@ -425,14 +427,7 @@ public long get() final Integer metricIndexInt = index.getMetricIndex(columnName); if (metricIndexInt == null) { - return new LongColumnSelector() - { - @Override - public long get() - { - return 0L; - } - }; + return ZeroLongColumnSelector.instance(); } final int metricIndex = metricIndexInt; @@ -453,6 +448,10 @@ public long get() @Override public ObjectColumnSelector makeObjectColumnSelector(String column) { + if (virtualColumns.exists(column)) { + return virtualColumns.makeObjectColumnSelector(column, this); + } + if (column.equals(Column.TIME_COLUMN_NAME)) { return new ObjectColumnSelector() { @@ -496,10 +495,6 @@ public Object get() IncrementalIndex.DimensionDesc dimensionDesc = index.getDimension(column); if (dimensionDesc == null) { - VirtualColumn virtualColumn = virtualColumns.getVirtualColumn(column); - if (virtualColumn != null) { - return virtualColumn.init(column, this); - } return null; } else { @@ -539,15 +534,11 @@ public Object get() @Override public ColumnCapabilities getColumnCapabilities(String columnName) { - ColumnCapabilities capabilities = index.getCapabilities(columnName); - if (capabilities == null && !virtualColumns.isEmpty()) { - VirtualColumn virtualColumn = virtualColumns.getVirtualColumn(columnName); - if (virtualColumn != null) { - Class clazz = virtualColumn.init(columnName, this).classOfObject(); - capabilities = new ColumnCapabilitiesImpl().setType(ValueType.typeFor(clazz)); - } + if (virtualColumns.exists(columnName)) { + return virtualColumns.getColumnCapabilities(columnName); } - return capabilities; + + return index.getCapabilities(columnName); } }; } diff --git a/processing/src/main/java/io/druid/segment/incremental/OffheapIncrementalIndex.java b/processing/src/main/java/io/druid/segment/incremental/OffheapIncrementalIndex.java index 2100e8ff0df1..4655d7c27623 100644 --- a/processing/src/main/java/io/druid/segment/incremental/OffheapIncrementalIndex.java +++ b/processing/src/main/java/io/druid/segment/incremental/OffheapIncrementalIndex.java @@ -91,31 +91,6 @@ public OffheapIncrementalIndex( aggBuffers.add(bb); } - public OffheapIncrementalIndex( - long minTimestamp, - QueryGranularity gran, - final AggregatorFactory[] metrics, - boolean deserializeComplexMetrics, - boolean reportParseExceptions, - boolean sortFacts, - int maxRowCount, - StupidPool bufferPool - ) - { - this( - new IncrementalIndexSchema.Builder().withMinTimestamp(minTimestamp) - .withQueryGranularity(gran) - .withMetrics(metrics) - .withRollup(IncrementalIndexSchema.DEFAULT_ROLLUP) - .build(), - deserializeComplexMetrics, - reportParseExceptions, - sortFacts, - maxRowCount, - bufferPool - ); - } - public OffheapIncrementalIndex( long minTimestamp, QueryGranularity gran, diff --git a/processing/src/main/java/io/druid/segment/virtual/BaseSingleValueDimensionSelector.java b/processing/src/main/java/io/druid/segment/virtual/BaseSingleValueDimensionSelector.java new file mode 100644 index 000000000000..5fb64c653eaf --- /dev/null +++ b/processing/src/main/java/io/druid/segment/virtual/BaseSingleValueDimensionSelector.java @@ -0,0 +1,92 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.segment.virtual; + +import com.google.common.base.Predicate; +import io.druid.query.filter.ValueMatcher; +import io.druid.segment.DimensionSelector; +import io.druid.segment.IdLookup; +import io.druid.segment.data.IndexedInts; +import io.druid.segment.data.ZeroIndexedInts; + +import javax.annotation.Nullable; +import java.util.Objects; + +public abstract class BaseSingleValueDimensionSelector implements DimensionSelector +{ + protected abstract String getValue(); + + @Override + public IndexedInts getRow() + { + return ZeroIndexedInts.instance(); + } + + @Override + public int getValueCardinality() + { + return DimensionSelector.CARDINALITY_UNKNOWN; + } + + @Override + public String lookupName(int id) + { + return getValue(); + } + + @Override + public ValueMatcher makeValueMatcher(final String value) + { + return new ValueMatcher() + { + @Override + public boolean matches() + { + return Objects.equals(getValue(), value); + } + }; + } + + @Override + public ValueMatcher makeValueMatcher(final Predicate predicate) + { + return new ValueMatcher() + { + @Override + public boolean matches() + { + return predicate.apply(getValue()); + } + }; + } + + @Override + public boolean nameLookupPossibleInAdvance() + { + return false; + } + + @Nullable + @Override + public IdLookup idLookup() + { + return null; + } +} diff --git a/processing/src/main/java/io/druid/segment/virtual/ExpressionSelectors.java b/processing/src/main/java/io/druid/segment/virtual/ExpressionSelectors.java index 7273d95759ed..f1d02439b4e0 100644 --- a/processing/src/main/java/io/druid/segment/virtual/ExpressionSelectors.java +++ b/processing/src/main/java/io/druid/segment/virtual/ExpressionSelectors.java @@ -20,7 +20,9 @@ package io.druid.segment.virtual; import io.druid.math.expr.Expr; +import io.druid.query.extraction.ExtractionFn; import io.druid.segment.ColumnSelectorFactory; +import io.druid.segment.DimensionSelector; import io.druid.segment.FloatColumnSelector; import io.druid.segment.LongColumnSelector; @@ -46,7 +48,7 @@ public static LongColumnSelector makeLongColumnSelector( ) { final ExpressionObjectSelector baseSelector = ExpressionObjectSelector.from(columnSelectorFactory, expression); - return new LongColumnSelector() + class ExpressionLongColumnSelector implements LongColumnSelector { @Override public long get() @@ -54,7 +56,8 @@ public long get() final Number number = baseSelector.get(); return number != null ? number.longValue() : nullValue; } - }; + } + return new ExpressionLongColumnSelector(); } public static FloatColumnSelector makeFloatColumnSelector( @@ -64,7 +67,7 @@ public static FloatColumnSelector makeFloatColumnSelector( ) { final ExpressionObjectSelector baseSelector = ExpressionObjectSelector.from(columnSelectorFactory, expression); - return new FloatColumnSelector() + class ExpressionFloatColumnSelector implements FloatColumnSelector { @Override public float get() @@ -72,6 +75,39 @@ public float get() final Number number = baseSelector.get(); return number != null ? number.floatValue() : nullValue; } - }; + } + return new ExpressionFloatColumnSelector(); + } + + public static DimensionSelector makeDimensionSelector( + final ColumnSelectorFactory columnSelectorFactory, + final Expr expression, + final ExtractionFn extractionFn + ) + { + final ExpressionObjectSelector baseSelector = ExpressionObjectSelector.from(columnSelectorFactory, expression); + + if (extractionFn == null) { + class DefaultExpressionDimensionSelector extends BaseSingleValueDimensionSelector + { + @Override + protected String getValue() + { + final Number number = baseSelector.get(); + return number == null ? null : String.valueOf(number); + } + } + return new DefaultExpressionDimensionSelector(); + } else { + class ExtractionExpressionDimensionSelector extends BaseSingleValueDimensionSelector + { + @Override + protected String getValue() + { + return extractionFn.apply(baseSelector.get()); + } + } + return new ExtractionExpressionDimensionSelector(); + } } } diff --git a/processing/src/main/java/io/druid/segment/virtual/ExpressionVirtualColumn.java b/processing/src/main/java/io/druid/segment/virtual/ExpressionVirtualColumn.java new file mode 100644 index 000000000000..650063cdb669 --- /dev/null +++ b/processing/src/main/java/io/druid/segment/virtual/ExpressionVirtualColumn.java @@ -0,0 +1,185 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.segment.virtual; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.common.base.Preconditions; +import com.google.common.primitives.Ints; +import io.druid.math.expr.Expr; +import io.druid.math.expr.Parser; +import io.druid.query.dimension.DimensionSpec; +import io.druid.segment.ColumnSelectorFactory; +import io.druid.segment.DimensionSelector; +import io.druid.segment.FloatColumnSelector; +import io.druid.segment.LongColumnSelector; +import io.druid.segment.ObjectColumnSelector; +import io.druid.segment.VirtualColumn; +import io.druid.segment.column.ColumnCapabilities; +import io.druid.segment.column.ColumnCapabilitiesImpl; +import io.druid.segment.column.ValueType; +import org.apache.commons.codec.Charsets; + +import java.nio.ByteBuffer; +import java.util.List; + +public class ExpressionVirtualColumn implements VirtualColumn +{ + private static final ColumnCapabilities CAPABILITIES = new ColumnCapabilitiesImpl().setType(ValueType.FLOAT); + + private final String name; + private final String expression; + private final Expr parsedExpression; + + @JsonCreator + public ExpressionVirtualColumn( + @JsonProperty("name") String name, + @JsonProperty("expression") String expression + ) + { + this.name = Preconditions.checkNotNull(name, "name"); + this.expression = Preconditions.checkNotNull(expression, "expression"); + this.parsedExpression = Parser.parse(expression); + } + + @JsonProperty("name") + @Override + public String getOutputName() + { + return name; + } + + @JsonProperty + public String getExpression() + { + return expression; + } + + @Override + public ObjectColumnSelector makeObjectColumnSelector( + final String columnName, + final ColumnSelectorFactory columnSelectorFactory + ) + { + return ExpressionSelectors.makeObjectColumnSelector(columnSelectorFactory, parsedExpression); + } + + @Override + public DimensionSelector makeDimensionSelector( + final DimensionSpec dimensionSpec, + final ColumnSelectorFactory columnSelectorFactory + ) + { + return dimensionSpec.decorate( + ExpressionSelectors.makeDimensionSelector( + columnSelectorFactory, + parsedExpression, + dimensionSpec.getExtractionFn() + ) + ); + } + + @Override + public FloatColumnSelector makeFloatColumnSelector( + final String columnName, + final ColumnSelectorFactory columnSelectorFactory + ) + { + return ExpressionSelectors.makeFloatColumnSelector(columnSelectorFactory, parsedExpression, 0.0f); + } + + @Override + public LongColumnSelector makeLongColumnSelector( + final String columnName, + final ColumnSelectorFactory columnSelectorFactory + ) + { + return ExpressionSelectors.makeLongColumnSelector(columnSelectorFactory, parsedExpression, 0L); + } + + @Override + public ColumnCapabilities capabilities(String columnName) + { + return CAPABILITIES; + } + + @Override + public List requiredColumns() + { + return Parser.findRequiredBindings(expression); + } + + @Override + public boolean usesDotNotation() + { + return false; + } + + @Override + public byte[] getCacheKey() + { + final byte[] nameBytes = name.getBytes(Charsets.UTF_8); + final byte[] expressionBytes = expression.getBytes(Charsets.UTF_8); + + return ByteBuffer + .allocate(1 + Ints.BYTES * 2 + nameBytes.length + expressionBytes.length) + .put(VirtualColumnCacheHelper.CACHE_TYPE_ID_EXPRESSION) + .putInt(nameBytes.length) + .put(nameBytes) + .putInt(expressionBytes.length) + .put(expressionBytes) + .array(); + } + + @Override + public boolean equals(Object o) + { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + + ExpressionVirtualColumn that = (ExpressionVirtualColumn) o; + + if (!name.equals(that.name)) { + return false; + } + return expression.equals(that.expression); + } + + @Override + public int hashCode() + { + int result = name.hashCode(); + result = 31 * result + expression.hashCode(); + return result; + } + + @Override + public String toString() + { + return "ExpressionVirtualColumn{" + + "name='" + name + '\'' + + ", expression='" + expression + '\'' + + '}'; + } +} diff --git a/processing/src/main/java/io/druid/segment/virtual/VirtualColumnCacheHelper.java b/processing/src/main/java/io/druid/segment/virtual/VirtualColumnCacheHelper.java new file mode 100644 index 000000000000..7c7bba2edd9d --- /dev/null +++ b/processing/src/main/java/io/druid/segment/virtual/VirtualColumnCacheHelper.java @@ -0,0 +1,34 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.segment.virtual; + +public class VirtualColumnCacheHelper +{ + public static final byte CACHE_TYPE_ID_MAP = 0x00; + public static final byte CACHE_TYPE_ID_EXPRESSION = 0x01; + + // Starting byte 0xFF is reserved for site-specific virtual columns. + public static final byte CACHE_TYPE_ID_USER_DEFINED = (byte) 0xFF; + + private VirtualColumnCacheHelper() + { + // No instantiation. + } +} diff --git a/processing/src/main/java/io/druid/segment/virtual/VirtualizedColumnSelectorFactory.java b/processing/src/main/java/io/druid/segment/virtual/VirtualizedColumnSelectorFactory.java new file mode 100644 index 000000000000..fd01c0ef6fd2 --- /dev/null +++ b/processing/src/main/java/io/druid/segment/virtual/VirtualizedColumnSelectorFactory.java @@ -0,0 +1,99 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.segment.virtual; + +import com.google.common.base.Preconditions; +import io.druid.query.dimension.DimensionSpec; +import io.druid.segment.ColumnSelectorFactory; +import io.druid.segment.DimensionSelector; +import io.druid.segment.FloatColumnSelector; +import io.druid.segment.LongColumnSelector; +import io.druid.segment.ObjectColumnSelector; +import io.druid.segment.VirtualColumns; +import io.druid.segment.column.ColumnCapabilities; + +import javax.annotation.Nullable; + +public class VirtualizedColumnSelectorFactory implements ColumnSelectorFactory +{ + private final ColumnSelectorFactory baseFactory; + private final VirtualColumns virtualColumns; + + public VirtualizedColumnSelectorFactory( + ColumnSelectorFactory baseFactory, + VirtualColumns virtualColumns + ) + { + this.baseFactory = Preconditions.checkNotNull(baseFactory, "baseFactory"); + this.virtualColumns = Preconditions.checkNotNull(virtualColumns, "virtualColumns"); + } + + @Override + public DimensionSelector makeDimensionSelector(DimensionSpec dimensionSpec) + { + if (virtualColumns.exists(dimensionSpec.getDimension())) { + return virtualColumns.makeDimensionSelector(dimensionSpec, baseFactory); + } else { + return baseFactory.makeDimensionSelector(dimensionSpec); + } + } + + @Override + public FloatColumnSelector makeFloatColumnSelector(String columnName) + { + if (virtualColumns.exists(columnName)) { + return virtualColumns.makeFloatColumnSelector(columnName, baseFactory); + } else { + return baseFactory.makeFloatColumnSelector(columnName); + } + } + + @Override + public LongColumnSelector makeLongColumnSelector(String columnName) + { + if (virtualColumns.exists(columnName)) { + return virtualColumns.makeLongColumnSelector(columnName, baseFactory); + } else { + return baseFactory.makeLongColumnSelector(columnName); + } + } + + @Nullable + @Override + public ObjectColumnSelector makeObjectColumnSelector(String columnName) + { + if (virtualColumns.exists(columnName)) { + return virtualColumns.makeObjectColumnSelector(columnName, baseFactory); + } else { + return baseFactory.makeObjectColumnSelector(columnName); + } + } + + @Nullable + @Override + public ColumnCapabilities getColumnCapabilities(String columnName) + { + if (virtualColumns.exists(columnName)) { + return virtualColumns.getColumnCapabilities(columnName); + } else { + return baseFactory.getColumnCapabilities(columnName); + } + } +} diff --git a/processing/src/test/java/io/druid/query/select/SelectQuerySpecTest.java b/processing/src/test/java/io/druid/query/select/SelectQuerySpecTest.java index b6f918fa6baa..9c4ee4c58ec6 100644 --- a/processing/src/test/java/io/druid/query/select/SelectQuerySpecTest.java +++ b/processing/src/test/java/io/druid/query/select/SelectQuerySpecTest.java @@ -60,7 +60,7 @@ public void testSerializationLegacyString() throws Exception + "\"granularity\":{\"type\":\"all\"}," + "\"dimensions\":[{\"type\":\"default\",\"dimension\":\"market\",\"outputName\":\"market\"},{\"type\":\"default\",\"dimension\":\"quality\",\"outputName\":\"quality\"}]," + "\"metrics\":[\"index\"]," - + "\"virtualColumns\":null," + + "\"virtualColumns\":[]," + "\"pagingSpec\":{\"pagingIdentifiers\":{},\"threshold\":3,\"fromNext\":false}," + "\"context\":null}"; diff --git a/processing/src/test/java/io/druid/segment/NullDimensionSelectorTest.java b/processing/src/test/java/io/druid/segment/NullDimensionSelectorTest.java index da8b7c837120..9b34cda11efb 100644 --- a/processing/src/test/java/io/druid/segment/NullDimensionSelectorTest.java +++ b/processing/src/test/java/io/druid/segment/NullDimensionSelectorTest.java @@ -27,7 +27,7 @@ public class NullDimensionSelectorTest { - private final NullDimensionSelector selector = new NullDimensionSelector(); + private final NullDimensionSelector selector = NullDimensionSelector.instance(); @Test public void testGetRow() throws Exception { diff --git a/processing/src/test/java/io/druid/segment/TestIndex.java b/processing/src/test/java/io/druid/segment/TestIndex.java index 084785933807..efc7a6405aeb 100644 --- a/processing/src/test/java/io/druid/segment/TestIndex.java +++ b/processing/src/test/java/io/druid/segment/TestIndex.java @@ -41,6 +41,7 @@ import io.druid.segment.incremental.IncrementalIndexSchema; import io.druid.segment.incremental.OnheapIncrementalIndex; import io.druid.segment.serde.ComplexMetrics; +import io.druid.segment.virtual.ExpressionVirtualColumn; import org.joda.time.DateTime; import org.joda.time.Interval; @@ -78,10 +79,15 @@ public class TestIndex public static final String[] METRICS = new String[]{"index", "indexMin", "indexMaxPlusTen"}; private static final Logger log = new Logger(TestIndex.class); private static final Interval DATA_INTERVAL = new Interval("2011-01-12T00:00:00.000Z/2011-05-01T00:00:00.000Z"); + private static final VirtualColumns VIRTUAL_COLUMNS = VirtualColumns.create( + Arrays.asList( + new ExpressionVirtualColumn("expr", "index + 10") + ) + ); public static final AggregatorFactory[] METRIC_AGGS = new AggregatorFactory[]{ new DoubleSumAggregatorFactory(METRICS[0], METRICS[0]), new DoubleMinAggregatorFactory(METRICS[1], METRICS[0]), - new DoubleMaxAggregatorFactory(METRICS[2], null, "index + 10"), + new DoubleMaxAggregatorFactory(METRICS[2], VIRTUAL_COLUMNS.getVirtualColumns()[0].getOutputName()), new HyperUniquesAggregatorFactory("quality_uniques", "quality") }; private static final IndexSpec indexSpec = new IndexSpec(); @@ -224,6 +230,7 @@ public static IncrementalIndex makeRealtimeIndex(final CharSource source, boolea .withMinTimestamp(new DateTime("2011-01-12T00:00:00.000Z").getMillis()) .withTimestampSpec(new TimestampSpec("ds", "auto", null)) .withQueryGranularity(QueryGranularities.NONE) + .withVirtualColumns(VIRTUAL_COLUMNS) .withMetrics(METRIC_AGGS) .withRollup(rollup) .build(); diff --git a/processing/src/test/java/io/druid/segment/incremental/IncrementalIndexMultiValueSpecTest.java b/processing/src/test/java/io/druid/segment/incremental/IncrementalIndexMultiValueSpecTest.java index 48c98f2ce892..00d0e66a0995 100644 --- a/processing/src/test/java/io/druid/segment/incremental/IncrementalIndexMultiValueSpecTest.java +++ b/processing/src/test/java/io/druid/segment/incremental/IncrementalIndexMultiValueSpecTest.java @@ -28,6 +28,7 @@ import io.druid.data.input.impl.TimestampSpec; import io.druid.granularity.QueryGranularities; import io.druid.query.aggregation.AggregatorFactory; +import io.druid.segment.VirtualColumns; import org.junit.Assert; import org.junit.Test; @@ -54,6 +55,7 @@ public void test() throws IndexSizeExceededException 0, new TimestampSpec("ds", "auto", null), QueryGranularities.ALL, + VirtualColumns.EMPTY, dimensionsSpec, new AggregatorFactory[0], false diff --git a/processing/src/test/java/io/druid/segment/incremental/IncrementalIndexStorageAdapterTest.java b/processing/src/test/java/io/druid/segment/incremental/IncrementalIndexStorageAdapterTest.java index eaaa14da3da7..c041e842547c 100644 --- a/processing/src/test/java/io/druid/segment/incremental/IncrementalIndexStorageAdapterTest.java +++ b/processing/src/test/java/io/druid/segment/incremental/IncrementalIndexStorageAdapterTest.java @@ -49,9 +49,9 @@ import io.druid.segment.Cursor; import io.druid.segment.DimensionSelector; import io.druid.segment.StorageAdapter; -import io.druid.segment.VirtualColumns; import io.druid.segment.data.IndexedInts; import io.druid.segment.filter.SelectorFilter; +import io.druid.segment.VirtualColumns; import org.joda.time.DateTime; import org.joda.time.Interval; import org.junit.Assert; @@ -265,7 +265,7 @@ public void testResetSanity() throws IOException Sequence cursorSequence = adapter.makeCursors( new SelectorFilter("sally", "bo"), interval, - null, + VirtualColumns.EMPTY, QueryGranularities.NONE, descending ); diff --git a/processing/src/test/java/io/druid/segment/virtual/ExpressionVirtualColumnTest.java b/processing/src/test/java/io/druid/segment/virtual/ExpressionVirtualColumnTest.java new file mode 100644 index 000000000000..45ac94c8682b --- /dev/null +++ b/processing/src/test/java/io/druid/segment/virtual/ExpressionVirtualColumnTest.java @@ -0,0 +1,175 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.segment.virtual; + +import com.google.common.base.Predicates; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import io.druid.data.input.InputRow; +import io.druid.data.input.MapBasedInputRow; +import io.druid.query.dimension.DefaultDimensionSpec; +import io.druid.query.dimension.ExtractionDimensionSpec; +import io.druid.query.extraction.BucketExtractionFn; +import io.druid.query.filter.ValueMatcher; +import io.druid.query.groupby.epinephelinae.TestColumnSelectorFactory; +import io.druid.segment.DimensionSelector; +import io.druid.segment.FloatColumnSelector; +import io.druid.segment.LongColumnSelector; +import io.druid.segment.ObjectColumnSelector; +import org.junit.Assert; +import org.junit.Test; + +public class ExpressionVirtualColumnTest +{ + private static final InputRow ROW0 = new MapBasedInputRow( + 0, + ImmutableList.of(), + ImmutableMap.of() + ); + + private static final InputRow ROW1 = new MapBasedInputRow( + 0, + ImmutableList.of(), + ImmutableMap.of("x", 4) + ); + + private static final InputRow ROW2 = new MapBasedInputRow( + 0, + ImmutableList.of(), + ImmutableMap.of("x", 2.1, "y", 3L) + ); + + private static final ExpressionVirtualColumn XPLUSY = new ExpressionVirtualColumn("expr", "x + y"); + private static final TestColumnSelectorFactory COLUMN_SELECTOR_FACTORY = new TestColumnSelectorFactory(); + + @Test + public void testObjectSelector() + { + final ObjectColumnSelector selector = XPLUSY.makeObjectColumnSelector("expr", COLUMN_SELECTOR_FACTORY); + + COLUMN_SELECTOR_FACTORY.setRow(ROW0); + Assert.assertEquals(null, selector.get()); + + COLUMN_SELECTOR_FACTORY.setRow(ROW1); + Assert.assertEquals(null, selector.get()); + + COLUMN_SELECTOR_FACTORY.setRow(ROW2); + Assert.assertEquals(5.1d, selector.get()); + } + + @Test + public void testLongSelector() + { + final LongColumnSelector selector = XPLUSY.makeLongColumnSelector("expr", COLUMN_SELECTOR_FACTORY); + + COLUMN_SELECTOR_FACTORY.setRow(ROW0); + Assert.assertEquals(0L, selector.get()); + + COLUMN_SELECTOR_FACTORY.setRow(ROW1); + Assert.assertEquals(0L, selector.get()); + + COLUMN_SELECTOR_FACTORY.setRow(ROW2); + Assert.assertEquals(5L, selector.get()); + } + + @Test + public void testFloatSelector() + { + final FloatColumnSelector selector = XPLUSY.makeFloatColumnSelector("expr", COLUMN_SELECTOR_FACTORY); + + COLUMN_SELECTOR_FACTORY.setRow(ROW0); + Assert.assertEquals(0.0f, selector.get(), 0.0f); + + COLUMN_SELECTOR_FACTORY.setRow(ROW1); + Assert.assertEquals(0.0f, selector.get(), 0.0f); + + COLUMN_SELECTOR_FACTORY.setRow(ROW2); + Assert.assertEquals(5.1f, selector.get(), 0.0f); + } + + @Test + public void testDimensionSelector() + { + final DimensionSelector selector = XPLUSY.makeDimensionSelector( + new DefaultDimensionSpec("expr", "x"), + COLUMN_SELECTOR_FACTORY + ); + + final ValueMatcher nullMatcher = selector.makeValueMatcher((String) null); + final ValueMatcher fiveMatcher = selector.makeValueMatcher("5"); + final ValueMatcher nonNullMatcher = selector.makeValueMatcher(Predicates.notNull()); + + COLUMN_SELECTOR_FACTORY.setRow(ROW0); + Assert.assertEquals(true, nullMatcher.matches()); + Assert.assertEquals(false, fiveMatcher.matches()); + Assert.assertEquals(false, nonNullMatcher.matches()); + Assert.assertEquals(null, selector.lookupName(selector.getRow().get(0))); + + COLUMN_SELECTOR_FACTORY.setRow(ROW1); + Assert.assertEquals(true, nullMatcher.matches()); + Assert.assertEquals(false, fiveMatcher.matches()); + Assert.assertEquals(false, nonNullMatcher.matches()); + Assert.assertEquals(null, selector.lookupName(selector.getRow().get(0))); + + COLUMN_SELECTOR_FACTORY.setRow(ROW2); + Assert.assertEquals(false, nullMatcher.matches()); + Assert.assertEquals(false, fiveMatcher.matches()); + Assert.assertEquals(true, nonNullMatcher.matches()); + Assert.assertEquals("5.1", selector.lookupName(selector.getRow().get(0))); + } + + @Test + public void testDimensionSelectorWithExtraction() + { + final DimensionSelector selector = XPLUSY.makeDimensionSelector( + new ExtractionDimensionSpec("expr", "x", new BucketExtractionFn(1.0, 0.0)), + COLUMN_SELECTOR_FACTORY + ); + + final ValueMatcher nullMatcher = selector.makeValueMatcher((String) null); + final ValueMatcher fiveMatcher = selector.makeValueMatcher("5"); + final ValueMatcher nonNullMatcher = selector.makeValueMatcher(Predicates.notNull()); + + COLUMN_SELECTOR_FACTORY.setRow(ROW0); + Assert.assertEquals(true, nullMatcher.matches()); + Assert.assertEquals(false, fiveMatcher.matches()); + Assert.assertEquals(false, nonNullMatcher.matches()); + Assert.assertEquals(null, selector.lookupName(selector.getRow().get(0))); + + COLUMN_SELECTOR_FACTORY.setRow(ROW1); + Assert.assertEquals(true, nullMatcher.matches()); + Assert.assertEquals(false, fiveMatcher.matches()); + Assert.assertEquals(false, nonNullMatcher.matches()); + Assert.assertEquals(null, selector.lookupName(selector.getRow().get(0))); + + COLUMN_SELECTOR_FACTORY.setRow(ROW2); + Assert.assertEquals(false, nullMatcher.matches()); + Assert.assertEquals(true, fiveMatcher.matches()); + Assert.assertEquals(true, nonNullMatcher.matches()); + Assert.assertEquals("5", selector.lookupName(selector.getRow().get(0))); + } + + @Test + public void testRequiredColumns() + { + final ExpressionVirtualColumn virtualColumn = new ExpressionVirtualColumn("expr", "x + y"); + Assert.assertEquals(ImmutableList.of("x", "y"), virtualColumn.requiredColumns()); + } +} diff --git a/processing/src/test/java/io/druid/segment/virtual/VirtualColumnsTest.java b/processing/src/test/java/io/druid/segment/virtual/VirtualColumnsTest.java new file mode 100644 index 000000000000..f37d34940bfc --- /dev/null +++ b/processing/src/test/java/io/druid/segment/virtual/VirtualColumnsTest.java @@ -0,0 +1,417 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.segment.virtual; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.base.Predicate; +import com.google.common.collect.ImmutableList; +import com.google.common.primitives.Longs; +import io.druid.jackson.DefaultObjectMapper; +import io.druid.query.dimension.DefaultDimensionSpec; +import io.druid.query.dimension.DimensionSpec; +import io.druid.query.dimension.ExtractionDimensionSpec; +import io.druid.query.extraction.BucketExtractionFn; +import io.druid.query.extraction.ExtractionFn; +import io.druid.query.filter.ValueMatcher; +import io.druid.segment.ColumnSelectorFactory; +import io.druid.segment.DimensionSelector; +import io.druid.segment.DimensionSelectorUtils; +import io.druid.segment.FloatColumnSelector; +import io.druid.segment.IdLookup; +import io.druid.segment.LongColumnSelector; +import io.druid.segment.ObjectColumnSelector; +import io.druid.segment.VirtualColumn; +import io.druid.segment.VirtualColumns; +import io.druid.segment.column.ColumnCapabilities; +import io.druid.segment.column.ColumnCapabilitiesImpl; +import io.druid.segment.column.ValueType; +import io.druid.segment.data.IndexedInts; +import it.unimi.dsi.fastutil.ints.IntIterator; +import it.unimi.dsi.fastutil.ints.IntIterators; +import org.junit.Assert; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.ExpectedException; + +import javax.annotation.Nullable; +import java.io.IOException; +import java.util.Arrays; +import java.util.List; + +public class VirtualColumnsTest +{ + @Rule + public ExpectedException expectedException = ExpectedException.none(); + + @Test + public void testMakeSelectors() + { + final VirtualColumns virtualColumns = makeVirtualColumns(); + final ObjectColumnSelector objectSelector = virtualColumns.makeObjectColumnSelector("expr", null); + final DimensionSelector dimensionSelector = virtualColumns.makeDimensionSelector( + new DefaultDimensionSpec("expr", "x"), + null + ); + final DimensionSelector extractionDimensionSelector = virtualColumns.makeDimensionSelector( + new ExtractionDimensionSpec("expr", "x", new BucketExtractionFn(1.0, 0.5)), + null + ); + final FloatColumnSelector floatSelector = virtualColumns.makeFloatColumnSelector("expr", null); + final LongColumnSelector longSelector = virtualColumns.makeLongColumnSelector("expr", null); + + Assert.assertEquals(1L, objectSelector.get()); + Assert.assertEquals("1", dimensionSelector.lookupName(dimensionSelector.getRow().get(0))); + Assert.assertEquals("0.5", extractionDimensionSelector.lookupName(extractionDimensionSelector.getRow().get(0))); + Assert.assertEquals(1.0f, floatSelector.get(), 0.0f); + Assert.assertEquals(1L, longSelector.get()); + } + + @Test + public void testMakeSelectorsWithDotSupport() + { + final VirtualColumns virtualColumns = makeVirtualColumns(); + final ObjectColumnSelector objectSelector = virtualColumns.makeObjectColumnSelector("foo.5", null); + final DimensionSelector dimensionSelector = virtualColumns.makeDimensionSelector( + new DefaultDimensionSpec("foo.5", "x"), + null + ); + final FloatColumnSelector floatSelector = virtualColumns.makeFloatColumnSelector("foo.5", null); + final LongColumnSelector longSelector = virtualColumns.makeLongColumnSelector("foo.5", null); + + Assert.assertEquals(5L, objectSelector.get()); + Assert.assertEquals("5", dimensionSelector.lookupName(dimensionSelector.getRow().get(0))); + Assert.assertEquals(5.0f, floatSelector.get(), 0.0f); + Assert.assertEquals(5L, longSelector.get()); + } + + @Test + public void testMakeSelectorsWithDotSupportBaseNameOnly() + { + final VirtualColumns virtualColumns = makeVirtualColumns(); + final ObjectColumnSelector objectSelector = virtualColumns.makeObjectColumnSelector("foo", null); + final DimensionSelector dimensionSelector = virtualColumns.makeDimensionSelector( + new DefaultDimensionSpec("foo", "x"), + null + ); + final FloatColumnSelector floatSelector = virtualColumns.makeFloatColumnSelector("foo", null); + final LongColumnSelector longSelector = virtualColumns.makeLongColumnSelector("foo", null); + + Assert.assertEquals(-1L, objectSelector.get()); + Assert.assertEquals("-1", dimensionSelector.lookupName(dimensionSelector.getRow().get(0))); + Assert.assertEquals(-1.0f, floatSelector.get(), 0.0f); + Assert.assertEquals(-1L, longSelector.get()); + } + + @Test + public void testTimeNotAllowed() + { + final ExpressionVirtualColumn expr = new ExpressionVirtualColumn("__time", "x + y"); + + expectedException.expect(IllegalArgumentException.class); + expectedException.expectMessage("virtualColumn name[__time] not allowed"); + + VirtualColumns.create(ImmutableList.of(expr)); + } + + @Test + public void testDuplicateNameDetection() + { + final ExpressionVirtualColumn expr = new ExpressionVirtualColumn("expr", "x + y"); + final ExpressionVirtualColumn expr2 = new ExpressionVirtualColumn("expr", "x * 2"); + + expectedException.expect(IllegalArgumentException.class); + expectedException.expectMessage("Duplicate virtualColumn name[expr]"); + + VirtualColumns.create(ImmutableList.of(expr, expr2)); + } + + @Test + public void testCycleDetection() + { + final ExpressionVirtualColumn expr = new ExpressionVirtualColumn("expr", "x + expr2"); + final ExpressionVirtualColumn expr2 = new ExpressionVirtualColumn("expr2", "expr * 2"); + + expectedException.expect(IllegalArgumentException.class); + expectedException.expectMessage("Self-referential column[expr]"); + + VirtualColumns.create(ImmutableList.of(expr, expr2)); + } + + @Test + public void testGetCacheKey() throws Exception + { + final VirtualColumns virtualColumns = VirtualColumns.create( + ImmutableList.of( + new ExpressionVirtualColumn("expr", "x + y") + ) + ); + + final VirtualColumns virtualColumns2 = VirtualColumns.create( + ImmutableList.of( + new ExpressionVirtualColumn("expr", "x + y") + ) + ); + + Assert.assertArrayEquals(virtualColumns.getCacheKey(), virtualColumns2.getCacheKey()); + Assert.assertFalse(Arrays.equals(virtualColumns.getCacheKey(), VirtualColumns.EMPTY.getCacheKey())); + } + + @Test + public void testEqualsAndHashCode() throws Exception + { + final VirtualColumns virtualColumns = VirtualColumns.create( + ImmutableList.of( + new ExpressionVirtualColumn("expr", "x + y") + ) + ); + + final VirtualColumns virtualColumns2 = VirtualColumns.create( + ImmutableList.of( + new ExpressionVirtualColumn("expr", "x + y") + ) + ); + + Assert.assertEquals(virtualColumns, virtualColumns); + Assert.assertEquals(virtualColumns, virtualColumns2); + Assert.assertNotEquals(VirtualColumns.EMPTY, virtualColumns); + Assert.assertNotEquals(VirtualColumns.EMPTY, null); + + Assert.assertEquals(virtualColumns.hashCode(), virtualColumns.hashCode()); + Assert.assertEquals(virtualColumns.hashCode(), virtualColumns2.hashCode()); + Assert.assertNotEquals(VirtualColumns.EMPTY.hashCode(), virtualColumns.hashCode()); + } + + @Test + public void testSerde() throws Exception + { + final ObjectMapper mapper = new DefaultObjectMapper(); + final ImmutableList theColumns = ImmutableList.of( + new ExpressionVirtualColumn("expr", "x + y"), + new ExpressionVirtualColumn("expr2", "x + z") + ); + final VirtualColumns virtualColumns = VirtualColumns.create(theColumns); + + Assert.assertEquals( + virtualColumns, + mapper.readValue( + mapper.writeValueAsString(virtualColumns), + VirtualColumns.class + ) + ); + + Assert.assertEquals( + theColumns, + mapper.readValue( + mapper.writeValueAsString(virtualColumns), + mapper.getTypeFactory().constructParametricType(List.class, VirtualColumn.class) + ) + ); + } + + private VirtualColumns makeVirtualColumns() + { + final ExpressionVirtualColumn expr = new ExpressionVirtualColumn("expr", "1"); + final DottyVirtualColumn dotty = new DottyVirtualColumn("foo"); + return VirtualColumns.create(ImmutableList.of(expr, dotty)); + } + + static class DottyVirtualColumn implements VirtualColumn + { + private final String name; + + public DottyVirtualColumn(String name) + { + this.name = name; + } + + @Override + public String getOutputName() + { + return name; + } + + @Override + public ObjectColumnSelector makeObjectColumnSelector(String columnName, ColumnSelectorFactory factory) + { + final LongColumnSelector selector = makeLongColumnSelector(columnName, factory); + return new ObjectColumnSelector() + { + @Override + public Class classOfObject() + { + return Long.class; + } + + @Override + public Object get() + { + return selector.get(); + } + }; + } + + @Override + public DimensionSelector makeDimensionSelector(DimensionSpec dimensionSpec, ColumnSelectorFactory factory) + { + final LongColumnSelector selector = makeLongColumnSelector(dimensionSpec.getDimension(), factory); + final ExtractionFn extractionFn = dimensionSpec.getExtractionFn(); + final DimensionSelector dimensionSelector = new DimensionSelector() + { + @Override + public IndexedInts getRow() + { + return new IndexedInts() + { + @Override + public int size() + { + return 1; + } + + @Override + public int get(int index) + { + return 0; + } + + @Override + public IntIterator iterator() + { + return IntIterators.singleton(0); + } + + @Override + public void fill(int index, int[] toFill) + { + throw new UnsupportedOperationException("fill not supported"); + } + + @Override + public void close() throws IOException + { + + } + }; + } + + @Override + public int getValueCardinality() + { + return DimensionSelector.CARDINALITY_UNKNOWN; + } + + @Override + public String lookupName(int id) + { + final String stringValue = String.valueOf(selector.get()); + return extractionFn == null ? stringValue : extractionFn.apply(stringValue); + } + + @Override + public ValueMatcher makeValueMatcher(final String value) + { + return DimensionSelectorUtils.makeValueMatcherGeneric(this, value); + } + + @Override + public ValueMatcher makeValueMatcher(final Predicate predicate) + { + return DimensionSelectorUtils.makeValueMatcherGeneric(this, predicate); + } + + @Override + public boolean nameLookupPossibleInAdvance() + { + return false; + } + + @Nullable + @Override + public IdLookup idLookup() + { + return new IdLookup() + { + @Override + public int lookupId(final String name) + { + return 0; + } + }; + } + }; + + return dimensionSpec.decorate(dimensionSelector); + } + + @Override + public FloatColumnSelector makeFloatColumnSelector(String columnName, ColumnSelectorFactory factory) + { + final LongColumnSelector selector = makeLongColumnSelector(columnName, factory); + return new FloatColumnSelector() + { + @Override + public float get() + { + return selector.get(); + } + }; + } + + @Override + public LongColumnSelector makeLongColumnSelector(String columnName, ColumnSelectorFactory factory) + { + final String subColumn = VirtualColumns.splitColumnName(columnName).rhs; + final Long boxed = subColumn == null ? null : Longs.tryParse(subColumn); + final long theLong = boxed == null ? -1 : boxed; + return new LongColumnSelector() + { + @Override + public long get() + { + return theLong; + } + }; + } + + @Override + public ColumnCapabilities capabilities(String columnName) + { + return new ColumnCapabilitiesImpl().setType(ValueType.LONG); + } + + @Override + public List requiredColumns() + { + return ImmutableList.of(); + } + + @Override + public boolean usesDotNotation() + { + return true; + } + + @Override + public byte[] getCacheKey() + { + throw new UnsupportedOperationException(); + } + } +}