From c114a8f3d363d6d216254a4d7b9bc9a7d8d4c431 Mon Sep 17 00:00:00 2001 From: tangjj Date: Fri, 24 Aug 2018 18:21:45 +0800 Subject: [PATCH 1/4] auc draft --- core-metadata/pom.xml | 16 +++++ .../kylin/measure/MeasureTypeFactory.java | 2 + .../apache/kylin/measure/auc/AucAggFunc.java | 10 +++ .../kylin/measure/auc/AucAggregator.java | 44 ++++++++++++ .../apache/kylin/measure/auc/AucCounter.java | 54 ++++++++++++++ .../kylin/measure/auc/AucMesureType.java | 70 +++++++++++++++++++ .../kylin/measure/auc/AucSerializer.java | 32 +++++++++ pom.xml | 6 ++ 8 files changed, 234 insertions(+) create mode 100644 core-metadata/src/main/java/org/apache/kylin/measure/auc/AucAggFunc.java create mode 100644 core-metadata/src/main/java/org/apache/kylin/measure/auc/AucAggregator.java create mode 100644 core-metadata/src/main/java/org/apache/kylin/measure/auc/AucCounter.java create mode 100644 core-metadata/src/main/java/org/apache/kylin/measure/auc/AucMesureType.java create mode 100644 core-metadata/src/main/java/org/apache/kylin/measure/auc/AucSerializer.java diff --git a/core-metadata/pom.xml b/core-metadata/pom.xml index acc2371d773..d99b6bc12b2 100644 --- a/core-metadata/pom.xml +++ b/core-metadata/pom.xml @@ -24,6 +24,18 @@ jar Apache Kylin - Core Metadata Apache Kylin - Core Metadata + + + + org.apache.maven.plugins + maven-compiler-plugin + + 1.8 + 1.8 + + + + org.apache.kylin @@ -87,6 +99,10 @@ com.tdunning t-digest + + com.github.haifengl + smile-core + diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/MeasureTypeFactory.java b/core-metadata/src/main/java/org/apache/kylin/measure/MeasureTypeFactory.java index 9699d2ea55d..d18a8ac3eee 100644 --- a/core-metadata/src/main/java/org/apache/kylin/measure/MeasureTypeFactory.java +++ b/core-metadata/src/main/java/org/apache/kylin/measure/MeasureTypeFactory.java @@ -23,6 +23,7 @@ import org.apache.kylin.common.KylinConfig; import org.apache.kylin.common.KylinConfigCannotInitException; +import org.apache.kylin.measure.auc.AucMesureType; import org.apache.kylin.measure.basic.BasicMeasureType; import org.apache.kylin.measure.bitmap.BitmapMeasureType; import org.apache.kylin.measure.dim.DimCountDistinctMeasureType; @@ -112,6 +113,7 @@ public static synchronized void init() { factoryInsts.add(new ExtendedColumnMeasureType.Factory()); factoryInsts.add(new PercentileMeasureType.Factory()); factoryInsts.add(new DimCountDistinctMeasureType.Factory()); + factoryInsts.add(new AucMesureType.Factory()); logger.info("Checking custom measure types from kylin config"); diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/auc/AucAggFunc.java b/core-metadata/src/main/java/org/apache/kylin/measure/auc/AucAggFunc.java new file mode 100644 index 00000000000..c35fd3a0046 --- /dev/null +++ b/core-metadata/src/main/java/org/apache/kylin/measure/auc/AucAggFunc.java @@ -0,0 +1,10 @@ +package org.apache.kylin.measure.auc; + +import org.apache.kylin.measure.ParamAsMeasureCount; + +public class AucAggFunc implements ParamAsMeasureCount { + @Override + public int getParamAsMeasureCount() { + return 2; + } +} diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/auc/AucAggregator.java b/core-metadata/src/main/java/org/apache/kylin/measure/auc/AucAggregator.java new file mode 100644 index 00000000000..83b82cdfdc3 --- /dev/null +++ b/core-metadata/src/main/java/org/apache/kylin/measure/auc/AucAggregator.java @@ -0,0 +1,44 @@ +package org.apache.kylin.measure.auc; + +import org.apache.kylin.measure.MeasureAggregator; + +public class AucAggregator extends MeasureAggregator { + AucCounter auc = null; + + public AucAggregator() { + } + + @Override + public void reset() { + auc = null; + } + + @Override + public void aggregate(AucCounter value) { + if (auc == null) + auc = new AucCounter(value); + else + auc.merge(value); + } + + @Override + public AucCounter aggregate(AucCounter value1, AucCounter value2) { + if (value1 == null) { + return value2; + } else if (value2 == null) { + return value1; + } + value1.merge(value2); + return value1; + } + + @Override + public AucCounter getState() { + return auc; + } + + @Override + public int getMemBytesEstimate() { + return 0; + } +} diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/auc/AucCounter.java b/core-metadata/src/main/java/org/apache/kylin/measure/auc/AucCounter.java new file mode 100644 index 00000000000..48e35296b7c --- /dev/null +++ b/core-metadata/src/main/java/org/apache/kylin/measure/auc/AucCounter.java @@ -0,0 +1,54 @@ +package org.apache.kylin.measure.auc; + + +import com.google.common.collect.Lists; +import org.apache.commons.collections.CollectionUtils; +import smile.validation.AUC; + +import java.io.Serializable; +import java.util.List; + +public class AucCounter implements Serializable { + private List truth = Lists.newLinkedList(); + private List pred = Lists.newLinkedList(); + + public AucCounter() { + } + + public AucCounter(AucCounter another) { + merge(another); + } + + public void merge(AucCounter value) { + + if (value == null) { + return; + } + if (CollectionUtils.isEmpty(value.getTruth())) { + return; + } + if (CollectionUtils.isEmpty(value.getPred())) { + return; + } + + + this.getTruth().addAll(value.getTruth()); + this.getPred().addAll(value.getPred()); + } + + public List getTruth() { + return truth; + } + + public List getPred() { + return pred; + } + + public double auc() { + + int[] t = truth.stream().mapToInt(Integer::valueOf).toArray(); + double[] p = pred.stream().mapToDouble(Double::valueOf).toArray(); + + return AUC.measure(t, p); + } +} diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/auc/AucMesureType.java b/core-metadata/src/main/java/org/apache/kylin/measure/auc/AucMesureType.java new file mode 100644 index 00000000000..e561136982f --- /dev/null +++ b/core-metadata/src/main/java/org/apache/kylin/measure/auc/AucMesureType.java @@ -0,0 +1,70 @@ +package org.apache.kylin.measure.auc; + + +import org.apache.kylin.measure.MeasureAggregator; +import org.apache.kylin.measure.MeasureIngester; +import org.apache.kylin.measure.MeasureType; +import org.apache.kylin.measure.MeasureTypeFactory; + +import com.google.common.collect.ImmutableMap; +import org.apache.kylin.metadata.datatype.DataType; +import org.apache.kylin.metadata.datatype.DataTypeSerializer; + +import java.util.Map; + +public class AucMesureType extends MeasureType { + + private final DataType dataType; + public static final String FUNC_AUC = "AUC"; + public static final String DATATYPE_AUC = "auc"; + + public AucMesureType(String funcName, DataType dataType) { + this.dataType = dataType; + } + + public static class Factory extends MeasureTypeFactory { + + @Override + public MeasureType createMeasureType(String funcName, DataType dataType) { + return new AucMesureType(funcName, dataType); + } + + @Override + public String getAggrFunctionName() { + return FUNC_AUC; + } + + @Override + public String getAggrDataTypeName() { + return DATATYPE_AUC; + } + + @Override + public Class> getAggrDataTypeSerializer() { + return AucSerializer.class; + } + } + + @Override + public MeasureIngester newIngester() { + return null; + } + + @Override + public MeasureAggregator newAggregator() { + return new AucAggregator(); + } + + static final Map> UDAF_MAP = ImmutableMap.>of( + AucMesureType.FUNC_AUC, AucAggFunc.class); + + @Override + public Map> getRewriteCalciteAggrFunctions() { + return UDAF_MAP; + } + + @Override + public boolean needRewrite() { + return true; + } +} diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/auc/AucSerializer.java b/core-metadata/src/main/java/org/apache/kylin/measure/auc/AucSerializer.java new file mode 100644 index 00000000000..5f4a3af723e --- /dev/null +++ b/core-metadata/src/main/java/org/apache/kylin/measure/auc/AucSerializer.java @@ -0,0 +1,32 @@ +package org.apache.kylin.measure.auc; + +import org.apache.kylin.metadata.datatype.DataTypeSerializer; + +import java.nio.ByteBuffer; + +public class AucSerializer extends DataTypeSerializer { + @Override + public int peekLength(ByteBuffer in) { + return 0; + } + + @Override + public int maxLength() { + return 0; + } + + @Override + public int getStorageBytesEstimate() { + return 0; + } + + @Override + public void serialize(AucCounter value, ByteBuffer out) { + + } + + @Override + public AucCounter deserialize(ByteBuffer in) { + return null; + } +} diff --git a/pom.xml b/pom.xml index d9b9efe20b3..e6608ad5b2f 100644 --- a/pom.xml +++ b/pom.xml @@ -125,6 +125,7 @@ 4.2.3.RELEASE 1.0.2.RELEASE 2.6.6 + 1.5.0 1.8.9 @@ -805,6 +806,11 @@ opensaml ${opensaml.version} + + com.github.haifengl + smile-core + ${smile.version} + From 5e3847dcec0f361c540704c58b8f315e26d1dfb7 Mon Sep 17 00:00:00 2001 From: tangjj Date: Mon, 27 Aug 2018 16:31:33 +0800 Subject: [PATCH 2/4] add junit test and auc serializer --- .../apache/kylin/measure/auc/AucAggFunc.java | 23 ++++++ .../kylin/measure/auc/AucAggregator.java | 2 +- .../apache/kylin/measure/auc/AucCounter.java | 36 +++++++-- .../kylin/measure/auc/AucSerializer.java | 55 +++++++++++--- .../java/org/apache/kylin/util/KryoUtils.java | 7 +- .../kylin/measure/auc/AucAggregatorTest.java | 48 ++++++++++++ .../kylin/measure/auc/AucSerializerTest.java | 75 +++++++++++++++++++ 7 files changed, 227 insertions(+), 19 deletions(-) create mode 100644 core-metadata/src/test/java/org/apache/kylin/measure/auc/AucAggregatorTest.java create mode 100644 core-metadata/src/test/java/org/apache/kylin/measure/auc/AucSerializerTest.java diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/auc/AucAggFunc.java b/core-metadata/src/main/java/org/apache/kylin/measure/auc/AucAggFunc.java index c35fd3a0046..d7c488fade8 100644 --- a/core-metadata/src/main/java/org/apache/kylin/measure/auc/AucAggFunc.java +++ b/core-metadata/src/main/java/org/apache/kylin/measure/auc/AucAggFunc.java @@ -3,6 +3,29 @@ import org.apache.kylin.measure.ParamAsMeasureCount; public class AucAggFunc implements ParamAsMeasureCount { + + public static AucCounter init() { + return null; + } + + public static AucCounter add(AucCounter counter, Object t, Object p) { + if (counter == null) { + counter = new AucCounter(); + } + counter.addTruth(t); + counter.addPred(p); + return counter; + } + + public static AucCounter merge(AucCounter counter0, AucCounter counter1) { + counter0.merge(counter1); + return counter0; + } + + public static double result(AucCounter counter) { + return counter == null ? -1D : counter.auc(); + } + @Override public int getParamAsMeasureCount() { return 2; diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/auc/AucAggregator.java b/core-metadata/src/main/java/org/apache/kylin/measure/auc/AucAggregator.java index 83b82cdfdc3..d6d25990942 100644 --- a/core-metadata/src/main/java/org/apache/kylin/measure/auc/AucAggregator.java +++ b/core-metadata/src/main/java/org/apache/kylin/measure/auc/AucAggregator.java @@ -39,6 +39,6 @@ public AucCounter getState() { @Override public int getMemBytesEstimate() { - return 0; + return auc.getTruth().size() * 4 + auc.getPred().size() * 4; } } diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/auc/AucCounter.java b/core-metadata/src/main/java/org/apache/kylin/measure/auc/AucCounter.java index 48e35296b7c..67b6dedcd77 100644 --- a/core-metadata/src/main/java/org/apache/kylin/measure/auc/AucCounter.java +++ b/core-metadata/src/main/java/org/apache/kylin/measure/auc/AucCounter.java @@ -19,19 +19,22 @@ public AucCounter(AucCounter another) { merge(another); } + public AucCounter(List truth, List pred) { + this.truth = truth == null ? Lists.newLinkedList() : truth; + this.pred = pred == null ? Lists.newLinkedList() : pred; + } + + public void merge(AucCounter value) { if (value == null) { return; } - if (CollectionUtils.isEmpty(value.getTruth())) { - return; - } - if (CollectionUtils.isEmpty(value.getPred())) { + + if (CollectionUtils.isEmpty(value.getTruth()) || CollectionUtils.isEmpty(value.getPred())) { return; } - this.getTruth().addAll(value.getTruth()); this.getPred().addAll(value.getPred()); } @@ -45,10 +48,31 @@ public List getPred() { } public double auc() { + if (CollectionUtils.isEmpty(truth) || CollectionUtils.isEmpty(pred)) { + return -1; + } int[] t = truth.stream().mapToInt(Integer::valueOf).toArray(); double[] p = pred.stream().mapToDouble(Double::valueOf).toArray(); + double result = AUC.measure(t, p); + if (Double.isNaN(result)) { + return -1; + } + return result; + } - return AUC.measure(t, p); + public void addTruth(Object t) { + + if (t == null) { + throw new RuntimeException("Truth of dimension is null "); + } + truth.add((Integer) t); + } + + public void addPred(Object p) { + if (p == null) { + throw new RuntimeException("Pred of dimension is null "); + } + pred.add((Double) p); } } diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/auc/AucSerializer.java b/core-metadata/src/main/java/org/apache/kylin/measure/auc/AucSerializer.java index 5f4a3af723e..02623ec222d 100644 --- a/core-metadata/src/main/java/org/apache/kylin/measure/auc/AucSerializer.java +++ b/core-metadata/src/main/java/org/apache/kylin/measure/auc/AucSerializer.java @@ -1,32 +1,65 @@ package org.apache.kylin.measure.auc; +import org.apache.kylin.metadata.datatype.DataType; import org.apache.kylin.metadata.datatype.DataTypeSerializer; +import org.apache.kylin.util.KryoUtils; import java.nio.ByteBuffer; +import java.util.LinkedList; +import java.util.List; -public class AucSerializer extends DataTypeSerializer { - @Override - public int peekLength(ByteBuffer in) { - return 0; +public class AucSerializer extends DataTypeSerializer { + + + // called by reflection + public AucSerializer(DataType type) { } + @Override - public int maxLength() { - return 0; + public void serialize(AucCounter value, ByteBuffer out) { + byte[] tserialize = KryoUtils.serialize(value.getTruth()); + byte[] pserialize = KryoUtils.serialize(value.getPred()); + out.putInt(4 + 4 + tserialize.length); + out.putInt(tserialize.length); + out.put(tserialize); + out.putInt(4 + 4 + pserialize.length); + out.putInt(pserialize.length); + out.put(pserialize); + } @Override - public int getStorageBytesEstimate() { - return 0; + public AucCounter deserialize(ByteBuffer in) { + int totalTruthLength = in.getInt(); + int tarrayLength = in.getInt(); + byte[] tdata = new byte[tarrayLength]; + in.get(tdata); + List truth = KryoUtils.deserialize(tdata, LinkedList.class); + + int totalpredLength = in.getInt(); + int parrayLength = in.getInt(); + byte[] pdata = new byte[parrayLength]; + in.get(pdata); + List pred = KryoUtils.deserialize(pdata, LinkedList.class); + return new AucCounter(truth, pred); } @Override - public void serialize(AucCounter value, ByteBuffer out) { + public int peekLength(ByteBuffer in) { + int mark = in.position(); + int ret = in.getInt(); + in.position(mark); + return ret; + } + @Override + public int maxLength() { + return 8 * 1024 * 1024; } @Override - public AucCounter deserialize(ByteBuffer in) { - return null; + public int getStorageBytesEstimate() { + return 1024; } } diff --git a/core-metadata/src/main/java/org/apache/kylin/util/KryoUtils.java b/core-metadata/src/main/java/org/apache/kylin/util/KryoUtils.java index 4059e9475cd..76ac538ca9c 100644 --- a/core-metadata/src/main/java/org/apache/kylin/util/KryoUtils.java +++ b/core-metadata/src/main/java/org/apache/kylin/util/KryoUtils.java @@ -19,9 +19,11 @@ package org.apache.kylin.util; import java.util.BitSet; +import java.util.LinkedList; -import org.objenesis.strategy.StdInstantiatorStrategy; +import org.objenesis.strategy.StdInstantiatorStrategy; +import com.esotericsoftware.kryo.serializers.CollectionSerializer; import com.esotericsoftware.kryo.Kryo; import com.esotericsoftware.kryo.io.Input; import com.esotericsoftware.kryo.io.Output; @@ -53,6 +55,9 @@ public static Kryo getKryo() { Kryo kryo = new Kryo(); kryo.setInstantiatorStrategy(new Kryo.DefaultInstantiatorStrategy(new StdInstantiatorStrategy())); kryo.register(BitSet.class, new BitSetSerializer()); + CollectionSerializer listSerializer = new CollectionSerializer(); + listSerializer.setElementsCanBeNull(false); + kryo.register(LinkedList.class, listSerializer); _Kryo.set(kryo); } diff --git a/core-metadata/src/test/java/org/apache/kylin/measure/auc/AucAggregatorTest.java b/core-metadata/src/test/java/org/apache/kylin/measure/auc/AucAggregatorTest.java new file mode 100644 index 00000000000..036c67f7aa1 --- /dev/null +++ b/core-metadata/src/test/java/org/apache/kylin/measure/auc/AucAggregatorTest.java @@ -0,0 +1,48 @@ +package org.apache.kylin.measure.auc; + +import com.google.common.collect.Lists; +import org.apache.commons.math3.random.RandomDataGenerator; +import org.junit.Test; +import smile.validation.AUC; + +import java.util.List; + +import static org.junit.Assert.assertEquals; + +public class AucAggregatorTest { + @Test + public void testAggregate() { + int datasize = 10000; + AucAggregator aggregator = new AucAggregator(); + RandomDataGenerator randomData = new RandomDataGenerator(); + List truths = Lists.newArrayListWithCapacity(datasize); + List preds = Lists.newArrayListWithCapacity(datasize); + for (int i = 0; i < datasize; i++) { + int t = randomData.nextInt(0, 1); + double p = Math.random(); + truths.add(t); + preds.add(p); + + AucCounter c = new AucCounter(); + c.addTruth(t); + c.addPred(p); + aggregator.aggregate(c); + } + + double actualResult = aggregator.getState().auc(); + double expectResult = auc(truths, preds); + assertEquals(expectResult, actualResult, 0.001); + } + + + public double auc(List truth, List pred) { + + int[] t = truth.stream().mapToInt(Integer::valueOf).toArray(); + double[] p = pred.stream().mapToDouble(Double::valueOf).toArray(); + double result = AUC.measure(t, p); + if (Double.isNaN(result)) { + return -1; + } + return result; + } +} diff --git a/core-metadata/src/test/java/org/apache/kylin/measure/auc/AucSerializerTest.java b/core-metadata/src/test/java/org/apache/kylin/measure/auc/AucSerializerTest.java new file mode 100644 index 00000000000..0c21fa896da --- /dev/null +++ b/core-metadata/src/test/java/org/apache/kylin/measure/auc/AucSerializerTest.java @@ -0,0 +1,75 @@ +package org.apache.kylin.measure.auc; + +import org.apache.kylin.common.util.LocalFileMetadataTestCase; +import org.apache.kylin.metadata.datatype.DataType; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; + +import java.nio.ByteBuffer; +import java.util.Arrays; +import java.util.LinkedList; +import java.util.List; + +import static org.junit.Assert.assertEquals; + +public class AucSerializerTest extends LocalFileMetadataTestCase { + + @BeforeClass + public static void setUp() throws Exception { + staticCreateTestMetadata(); + } + + @AfterClass + public static void after() throws Exception { + cleanAfterClass(); + } + + + @Test + public void testBasic() { + AucSerializer serializer = new AucSerializer(DataType.getType("auc")); + + List truth = new LinkedList<>(Arrays.asList(0, 0, 1, 1)); + List pred = new LinkedList<>(Arrays.asList(0.1, 0.4, 0.35, 0.8)); + + AucCounter counter = new AucCounter(truth, pred); + double markResult = counter.auc(); + assertEquals(markResult, 0.75, 0.01); + + ByteBuffer buffer = ByteBuffer.allocateDirect(serializer.getStorageBytesEstimate()); + serializer.serialize(counter, buffer); + + buffer.flip(); + counter = serializer.deserialize(buffer); + List truth1 = new LinkedList<>(Arrays.asList(1, 0, 1, 1)); + List pred1 = new LinkedList<>(Arrays.asList(0.9, 0.4, 0.65, 0.8)); + AucCounter counter1 = new AucCounter(truth1, pred1); + counter1.merge(counter); + + assertEquals(0.86, counter1.auc(), 0.01); + } + + @Test + public void testNull() { + + List truth = null; + List pred = null; + + AucCounter counter = new AucCounter(truth, pred); + double markResult = counter.auc(); + assertEquals(markResult, -1.0, 0.01); + + } + + @Test + public void testNan() { + + List truth = new LinkedList<>(Arrays.asList(1, 1, 1, 1)); + List pred = new LinkedList<>(Arrays.asList(0.1, 0.4, 0.35, 0.8)); + + AucCounter counter = new AucCounter(truth, pred); + double markResult = counter.auc(); + assertEquals(markResult, -1.0, 0.01); + } +} From 887897cc8ffa2505f20809f78418646fd4b7f598 Mon Sep 17 00:00:00 2001 From: tangjj Date: Mon, 27 Aug 2018 16:48:10 +0800 Subject: [PATCH 3/4] add newIngester --- .../main/java/org/apache/kylin/measure/auc/AucMesureType.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/auc/AucMesureType.java b/core-metadata/src/main/java/org/apache/kylin/measure/auc/AucMesureType.java index e561136982f..de1826fd2d4 100644 --- a/core-metadata/src/main/java/org/apache/kylin/measure/auc/AucMesureType.java +++ b/core-metadata/src/main/java/org/apache/kylin/measure/auc/AucMesureType.java @@ -47,7 +47,7 @@ public Class> getAggrDataTypeSerializer @Override public MeasureIngester newIngester() { - return null; + throw new UnsupportedOperationException("No ingester for this measure type."); } @Override From b7dd0d2b5393ecdee80f8a0b4084633e38730d01 Mon Sep 17 00:00:00 2001 From: tangjj Date: Mon, 27 Aug 2018 17:01:38 +0800 Subject: [PATCH 4/4] add apache license --- .../apache/kylin/measure/auc/AucAggFunc.java | 18 ++++++++++++++++++ .../kylin/measure/auc/AucAggregator.java | 18 ++++++++++++++++++ .../apache/kylin/measure/auc/AucCounter.java | 18 ++++++++++++++++++ .../kylin/measure/auc/AucMesureType.java | 18 ++++++++++++++++++ .../kylin/measure/auc/AucSerializer.java | 18 ++++++++++++++++++ storage-hbase/pom.xml | 1 + 6 files changed, 91 insertions(+) diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/auc/AucAggFunc.java b/core-metadata/src/main/java/org/apache/kylin/measure/auc/AucAggFunc.java index d7c488fade8..018f371362a 100644 --- a/core-metadata/src/main/java/org/apache/kylin/measure/auc/AucAggFunc.java +++ b/core-metadata/src/main/java/org/apache/kylin/measure/auc/AucAggFunc.java @@ -1,3 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + package org.apache.kylin.measure.auc; import org.apache.kylin.measure.ParamAsMeasureCount; diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/auc/AucAggregator.java b/core-metadata/src/main/java/org/apache/kylin/measure/auc/AucAggregator.java index d6d25990942..3b8edaa1d4d 100644 --- a/core-metadata/src/main/java/org/apache/kylin/measure/auc/AucAggregator.java +++ b/core-metadata/src/main/java/org/apache/kylin/measure/auc/AucAggregator.java @@ -1,3 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + package org.apache.kylin.measure.auc; import org.apache.kylin.measure.MeasureAggregator; diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/auc/AucCounter.java b/core-metadata/src/main/java/org/apache/kylin/measure/auc/AucCounter.java index 67b6dedcd77..fd94bcaa5bb 100644 --- a/core-metadata/src/main/java/org/apache/kylin/measure/auc/AucCounter.java +++ b/core-metadata/src/main/java/org/apache/kylin/measure/auc/AucCounter.java @@ -1,3 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + package org.apache.kylin.measure.auc; diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/auc/AucMesureType.java b/core-metadata/src/main/java/org/apache/kylin/measure/auc/AucMesureType.java index de1826fd2d4..0ec5ea21836 100644 --- a/core-metadata/src/main/java/org/apache/kylin/measure/auc/AucMesureType.java +++ b/core-metadata/src/main/java/org/apache/kylin/measure/auc/AucMesureType.java @@ -1,3 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + package org.apache.kylin.measure.auc; diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/auc/AucSerializer.java b/core-metadata/src/main/java/org/apache/kylin/measure/auc/AucSerializer.java index 02623ec222d..b13f14bbd8a 100644 --- a/core-metadata/src/main/java/org/apache/kylin/measure/auc/AucSerializer.java +++ b/core-metadata/src/main/java/org/apache/kylin/measure/auc/AucSerializer.java @@ -1,3 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + package org.apache.kylin.measure.auc; import org.apache.kylin.metadata.datatype.DataType; diff --git a/storage-hbase/pom.xml b/storage-hbase/pom.xml index c1b4cea76c6..2ce142ece83 100644 --- a/storage-hbase/pom.xml +++ b/storage-hbase/pom.xml @@ -141,6 +141,7 @@ org.apache.kylin:kylin-core-cube org.roaringbitmap:RoaringBitmap com.tdunning:t-digest + com.github.haifengl:smile-core