Skip to content

Commit

Permalink
Merge 12914b7 into 5611760
Browse files Browse the repository at this point in the history
  • Loading branch information
kyotoYaho committed May 14, 2018
2 parents 5611760 + 12914b7 commit 071ecba
Show file tree
Hide file tree
Showing 68 changed files with 3,965 additions and 525 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -4996,7 +4996,7 @@ private void translateAgg(SqlCall call, SqlNode filter, SqlCall outerCall) {
// special case for COUNT(*): delete the *
if (operand instanceof SqlIdentifier) {
SqlIdentifier id = (SqlIdentifier) operand;
if (id.isStar() || isSimpleCount(call)) { /* OVERRIDE POINT */
if (id.isStar()) {
assert call.operandCount() == 1;
assert args.isEmpty();
break;
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,45 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package org.apache.kylin.common.util;

import java.math.BigDecimal;
import java.math.BigInteger;

public class DecimalUtil {

// Copy from org.apache.calcite.runtime.SqlFunctions.toBigDecimal
public static BigDecimal toBigDecimal(String s) {
return new BigDecimal(s.trim());
}

public static BigDecimal toBigDecimal(Number number) {
// There are some values of "long" that cannot be represented as "double".
// Not so "int". If it isn't a long, go straight to double.
return number instanceof BigDecimal ? (BigDecimal) number
: number instanceof BigInteger ? new BigDecimal((BigInteger) number)
: number instanceof Long ? new BigDecimal(number.longValue())
: new BigDecimal(number.doubleValue());
}

public static BigDecimal toBigDecimal(Object o) {
if (o == null)
return null;
return o instanceof Number ? toBigDecimal((Number) o) : toBigDecimal(o.toString());
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@
import org.apache.kylin.measure.MeasureType;
import org.apache.kylin.measure.basic.BasicMeasureType;
import org.apache.kylin.metadata.filter.UDF.MassInTupleFilter;
import org.apache.kylin.metadata.model.DynamicFunctionDesc;
import org.apache.kylin.metadata.model.FunctionDesc;
import org.apache.kylin.metadata.model.IStorageAware;
import org.apache.kylin.metadata.model.MeasureDesc;
Expand Down Expand Up @@ -155,10 +156,12 @@ public MeasureDesc getInvolvedMeasure() {
private static Collection<TblColRef> getDimensionColumns(SQLDigest sqlDigest) {
Collection<TblColRef> groupByColumns = sqlDigest.groupbyColumns;
Collection<TblColRef> filterColumns = sqlDigest.filterColumns;
Collection<TblColRef> rtDimColumns = sqlDigest.rtDimensionColumns;

Collection<TblColRef> dimensionColumns = new HashSet<TblColRef>();
dimensionColumns.addAll(groupByColumns);
dimensionColumns.addAll(filterColumns);
dimensionColumns.addAll(rtDimColumns);
return dimensionColumns;
}

Expand All @@ -171,8 +174,39 @@ private static Set<TblColRef> unmatchedDimensions(Collection<TblColRef> dimensio

private static Set<FunctionDesc> unmatchedAggregations(Collection<FunctionDesc> aggregations, CubeInstance cube) {
HashSet<FunctionDesc> result = Sets.newHashSet(aggregations);

CubeDesc cubeDesc = cube.getDescriptor();
result.removeAll(cubeDesc.listAllFunctions());
List<FunctionDesc> definedFuncs = cubeDesc.listAllFunctions();

// check normal aggregations
result.removeAll(definedFuncs);

// check dynamic aggregations
Iterator<FunctionDesc> funcIterator = result.iterator();
while (funcIterator.hasNext()) {
FunctionDesc entry = funcIterator.next();
if (entry instanceof DynamicFunctionDesc) {
DynamicFunctionDesc dynFunc = (DynamicFunctionDesc) entry;
// Filter columns cannot be derived
Collection<TblColRef> definedCols = dynFunc.ifFriendlyForDerivedFilter()
? cubeDesc.listDimensionColumnsIncludingDerived()
: cubeDesc.listDimensionColumnsExcludingDerived(true);
Set<TblColRef> filterCols = Sets.newHashSet(dynFunc.getFilterColumnSet());
filterCols.removeAll(definedCols);
if (!filterCols.isEmpty()) {
continue;
}

// All inner funcs should be defined
Set<FunctionDesc> innerFuncSet = Sets.newHashSet(dynFunc.getRuntimeFuncs());
innerFuncSet.removeAll(definedFuncs);
if (!innerFuncSet.isEmpty()) {
continue;
}

funcIterator.remove();
}
}
return result;
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,7 @@
import org.apache.kylin.gridtable.IGTComparator;
import org.apache.kylin.measure.MeasureAggregator;
import org.apache.kylin.metadata.datatype.DataTypeSerializer;
import org.apache.kylin.metadata.datatype.DynamicDimSerializer;

/**
* defines how column values will be encoded to/ decoded from GTRecord
Expand Down Expand Up @@ -68,7 +69,7 @@ public TrimmedCubeCodeSystem trimForCoprocessor() {
@Override
public void init(GTInfo info) {
this.info = info;

ImmutableBitSet dDims = info.getDynamicDims();
this.serializers = new DataTypeSerializer[info.getColumnCount()];
for (int i = 0; i < serializers.length; i++) {
DimensionEncoding dimEnc = i < dimEncs.length ? dimEncs[i] : null;
Expand All @@ -77,8 +78,14 @@ public void init(GTInfo info) {
// for dimensions
serializers[i] = dimEnc.asDataTypeSerializer();
} else {
// for measures
serializers[i] = DataTypeSerializer.create(info.getColumnType(i));
DataTypeSerializer dSerializer = DataTypeSerializer.create(info.getColumnType(i));
if (dDims != null && dDims.get(i)) {
// for dynamic dimensions
dSerializer = new DynamicDimSerializer(dSerializer);
} else {
// for measures
}
serializers[i] = dSerializer;
}
}
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -28,10 +28,25 @@ public static GTInfo newGTInfo(Cuboid cuboid, IDimensionEncodingMap dimEncMap) {

GTInfo.Builder builder = GTInfo.builder();
builder.setTableName("Cuboid " + cuboid.getId());
builder.setCodeSystem(new CubeCodeSystem(mapping.getDimensionEncodings(dimEncMap), mapping.getDependentMetricsMap()));
builder.setCodeSystem(
new CubeCodeSystem(mapping.getDimensionEncodings(dimEncMap), mapping.getDependentMetricsMap()));
builder.setColumns(mapping.getDataTypes());
builder.setPrimaryKey(mapping.getPrimaryKey());
builder.enableColumnBlock(mapping.getColumnBlocks());
return builder.build();
}

public static GTInfo newGTInfo(Cuboid cuboid, IDimensionEncodingMap dimEncMap, CuboidToGridTableMapping mapping) {
GTInfo.Builder builder = GTInfo.builder();
builder.setTableName("Cuboid " + cuboid.getId());
builder.setCodeSystem(
new CubeCodeSystem(mapping.getDimensionEncodings(dimEncMap), mapping.getDependentMetricsMap()));
builder.setColumns(mapping.getDataTypes());
builder.setPrimaryKey(mapping.getPrimaryKey());
builder.enableColumnBlock(mapping.getColumnBlocks());
if (mapping instanceof CuboidToGridTableMappingExt) {
builder.enableDynamicDims(((CuboidToGridTableMappingExt) mapping).getDynamicDims());
}
return builder.build();
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,7 @@
import org.apache.kylin.metadata.model.MeasureDesc;
import org.apache.kylin.metadata.model.TblColRef;

import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;

Expand Down Expand Up @@ -201,17 +202,17 @@ public Map<Integer, Integer> getDependentMetricsMap() {
public ImmutableBitSet makeGridTableColumns(Set<TblColRef> dimensions) {
BitSet result = new BitSet();
for (TblColRef dim : dimensions) {
int idx = this.getIndexOf(dim);
int idx = getIndexOf(dim);
if (idx >= 0)
result.set(idx);
}
return new ImmutableBitSet(result);
}

public ImmutableBitSet makeGridTableColumns(Collection<FunctionDesc> metrics) {
public ImmutableBitSet makeGridTableColumns(Collection<? extends FunctionDesc> metrics) {
BitSet result = new BitSet();
for (FunctionDesc metric : metrics) {
int idx = this.getIndexOf(metric);
int idx = getIndexOf(metric);
if (idx < 0)
throw new IllegalStateException(metric + " not found in " + this);
result.set(idx);
Expand All @@ -227,8 +228,8 @@ public String[] makeAggrFuncs(Collection<FunctionDesc> metrics) {
Collections.sort(metricList, new Comparator<FunctionDesc>() {
@Override
public int compare(FunctionDesc o1, FunctionDesc o2) {
int a = CuboidToGridTableMapping.this.getIndexOf(o1);
int b = CuboidToGridTableMapping.this.getIndexOf(o2);
int a = getIndexOf(o1);
int b = getIndexOf(o2);
return a - b;
}
});
Expand All @@ -241,4 +242,7 @@ public int compare(FunctionDesc o1, FunctionDesc o2) {
return result;
}

public Map<TblColRef, Integer> getDim2gt() {
return ImmutableMap.copyOf(dim2gt);
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,134 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package org.apache.kylin.cube.gridtable;

import java.util.BitSet;
import java.util.Collection;
import java.util.List;
import java.util.Map;

import org.apache.commons.lang.ArrayUtils;
import org.apache.kylin.common.util.ImmutableBitSet;
import org.apache.kylin.cube.cuboid.Cuboid;
import org.apache.kylin.metadata.datatype.DataType;
import org.apache.kylin.metadata.model.DynamicFunctionDesc;
import org.apache.kylin.metadata.model.FunctionDesc;
import org.apache.kylin.metadata.model.TblColRef;

import com.google.common.collect.Lists;
import com.google.common.collect.Maps;

public class CuboidToGridTableMappingExt extends CuboidToGridTableMapping {
private final List<TblColRef> dynDims;
private final List<DynamicFunctionDesc> dynFuncs;

private ImmutableBitSet dynamicDims;

private List<DataType> dynGtDataTypes;
private List<ImmutableBitSet> dynGtColBlocks;

private Map<TblColRef, Integer> dynDim2gt;

private Map<FunctionDesc, Integer> dynMetrics2gt;

public CuboidToGridTableMappingExt(Cuboid cuboid, List<TblColRef> dynDims, List<DynamicFunctionDesc> dynFuncs) {
super(cuboid);
this.dynDims = dynDims;
this.dynFuncs = dynFuncs;
init();
}

private void init() {
dynGtDataTypes = Lists.newArrayList();
dynGtColBlocks = Lists.newArrayList();
dynDim2gt = Maps.newHashMap();
dynMetrics2gt = Maps.newHashMap();

int gtColIdx = super.getColumnCount();

BitSet rtColBlock = new BitSet();
// dynamic dimensions
for (TblColRef rtDim : dynDims) {
dynDim2gt.put(rtDim, gtColIdx);
dynGtDataTypes.add(rtDim.getType());
rtColBlock.set(gtColIdx);
gtColIdx++;
}
dynamicDims = new ImmutableBitSet(rtColBlock);

// dynamic metrics
for (DynamicFunctionDesc rtFunc : dynFuncs) {
dynMetrics2gt.put(rtFunc, gtColIdx);
dynGtDataTypes.add(rtFunc.getReturnDataType());
rtColBlock.set(gtColIdx);
gtColIdx++;
}

dynGtColBlocks.add(new ImmutableBitSet(rtColBlock));
}

public ImmutableBitSet getDynamicDims() {
return dynamicDims;
}

@Override
public int getColumnCount() {
return super.getColumnCount() + dynDims.size() + dynFuncs.size();
}

@Override
public DataType[] getDataTypes() {
return (DataType[]) ArrayUtils.addAll(super.getDataTypes(),
dynGtDataTypes.toArray(new DataType[dynGtDataTypes.size()]));
}

@Override
public ImmutableBitSet[] getColumnBlocks() {
return (ImmutableBitSet[]) ArrayUtils.addAll(super.getColumnBlocks(),
dynGtColBlocks.toArray(new ImmutableBitSet[dynGtColBlocks.size()]));
}

@Override
public int getIndexOf(TblColRef dimension) {
Integer i = super.getIndexOf(dimension);
if (i < 0) {
i = dynDim2gt.get(dimension);
}
return i == null ? -1 : i;
}

@Override
public int getIndexOf(FunctionDesc metric) {
Integer r = super.getIndexOf(metric);
if (r < 0) {
r = dynMetrics2gt.get(metric);
}
return r == null ? -1 : r;
}

@Override
public int[] getMetricsIndexes(Collection<FunctionDesc> metrics) {
int[] result = new int[metrics.size()];
int i = 0;
for (FunctionDesc metric : metrics) {
result[i++] = getIndexOf(metric);
}
return result;
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@
import org.apache.kylin.common.util.Pair;
import org.apache.kylin.gridtable.GTInfo;
import org.apache.kylin.gridtable.GTScanRequest;
import org.apache.kylin.metadata.expression.TupleExpression;
import org.apache.kylin.metadata.filter.CompareTupleFilter;
import org.apache.kylin.metadata.filter.ConstantTupleFilter;
import org.apache.kylin.metadata.filter.LogicalTupleFilter;
Expand Down Expand Up @@ -61,6 +62,10 @@ public abstract class ScanRangePlannerBase {
protected RecordComparator rangeEndComparator;
protected RecordComparator rangeStartEndComparator;

protected ImmutableBitSet gtDynColumns;
protected ImmutableBitSet gtRtAggrMetrics;
protected List<TupleExpression> tupleExpressionList;

public abstract GTScanRequest planScanRequest();

protected TupleFilter flattenToOrAndFilter(TupleFilter filter) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -354,7 +354,7 @@ private boolean[] createCompareMask() {
for (int i = 0; i < dimensions.trueBitCount(); i++) {
int c = dimensions.trueBitAt(i);
int l = info.codeSystem.maxCodeLength(c);
boolean m = groupBy.get(c) ? true : false;
boolean m = groupBy.get(c);
for (int j = 0; j < l; j++) {
mask[p++] = m;
}
Expand Down

0 comments on commit 071ecba

Please sign in to comment.