From 11e393d5251e98ea342d7bbdb84b484efc6cee2b Mon Sep 17 00:00:00 2001 From: Matthias Wahl Date: Wed, 28 Jan 2015 18:19:04 +0100 Subject: [PATCH] added support for the stddev, variance and geometric_mean aggregation functions --- CHANGES.txt | 3 + .../main/java/io/crate/types/BooleanType.java | 2 +- .../main/java/io/crate/types/ByteType.java | 2 +- .../main/java/io/crate/types/DoubleType.java | 2 +- ...FixedWithType.java => FixedWidthType.java} | 4 +- .../main/java/io/crate/types/FloatType.java | 2 +- .../java/io/crate/types/GeoPointType.java | 2 +- .../main/java/io/crate/types/IntegerType.java | 2 +- .../main/java/io/crate/types/LongType.java | 3 +- .../main/java/io/crate/types/ShortType.java | 2 +- docs/sql/aggregation.txt | 98 +++++++- docs/sql/system.txt | 2 - sql/build.gradle | 2 +- .../crate/breaker/SizeEstimatorFactory.java | 4 +- .../impl/AggregationImplModule.java | 4 + .../aggregation/impl/AverageAggregation.java | 89 +++---- .../impl/GeometricMeanAggregation.java | 218 ++++++++++++++++++ .../aggregation/impl/MaximumAggregation.java | 6 +- .../aggregation/impl/MinimumAggregation.java | 6 +- .../impl/StandardDeviationAggregation.java | 187 +++++++++++++++ .../aggregation/impl/VarianceAggregation.java | 189 +++++++++++++++ .../statistics/moment/StandardDeviation.java | 31 +++ .../statistics/moment/Variance.java | 74 ++++++ .../GroupByAggregateTest.java | 60 ++++- .../impl/AverageAggregationTest.java | 3 + .../impl/GeometricMeanAggregationtest.java | 104 +++++++++ .../impl/StdDevAggregationTest.java | 111 +++++++++ .../impl/VarianceAggregationTest.java | 104 +++++++++ 28 files changed, 1245 insertions(+), 71 deletions(-) rename core/src/main/java/io/crate/types/{FixedWithType.java => FixedWidthType.java} (95%) create mode 100644 sql/src/main/java/io/crate/operation/aggregation/impl/GeometricMeanAggregation.java create mode 100644 sql/src/main/java/io/crate/operation/aggregation/impl/StandardDeviationAggregation.java create mode 100644 sql/src/main/java/io/crate/operation/aggregation/impl/VarianceAggregation.java create mode 100644 sql/src/main/java/io/crate/operation/aggregation/statistics/moment/StandardDeviation.java create mode 100644 sql/src/main/java/io/crate/operation/aggregation/statistics/moment/Variance.java create mode 100644 sql/src/test/java/io/crate/operation/aggregation/impl/GeometricMeanAggregationtest.java create mode 100644 sql/src/test/java/io/crate/operation/aggregation/impl/StdDevAggregationTest.java create mode 100644 sql/src/test/java/io/crate/operation/aggregation/impl/VarianceAggregationTest.java diff --git a/CHANGES.txt b/CHANGES.txt index c2a625787e5c..a2def8ff49d6 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -5,6 +5,9 @@ Changes for Crate Unreleased ========== + - Added support for the ``std_dev``, ``variance`` and ``geometric_mean`` + aggregation functions + - Added support for the CURRENT_TIMESTAMP expression - Throw an error while using unknown column references in ``SELECT`` diff --git a/core/src/main/java/io/crate/types/BooleanType.java b/core/src/main/java/io/crate/types/BooleanType.java index c7eab7781b28..81ba330f62af 100644 --- a/core/src/main/java/io/crate/types/BooleanType.java +++ b/core/src/main/java/io/crate/types/BooleanType.java @@ -31,7 +31,7 @@ import java.util.Locale; import java.util.Map; -public class BooleanType extends DataType implements DataTypeFactory, Streamer, FixedWithType { +public class BooleanType extends DataType implements DataTypeFactory, Streamer, FixedWidthType { public static final int ID = 3; public static final BooleanType INSTANCE = new BooleanType(); diff --git a/core/src/main/java/io/crate/types/ByteType.java b/core/src/main/java/io/crate/types/ByteType.java index 6c7b0925c679..14165f9a31a8 100644 --- a/core/src/main/java/io/crate/types/ByteType.java +++ b/core/src/main/java/io/crate/types/ByteType.java @@ -28,7 +28,7 @@ import java.io.IOException; -public class ByteType extends DataType implements DataTypeFactory, Streamer, FixedWithType { +public class ByteType extends DataType implements DataTypeFactory, Streamer, FixedWidthType { public final static ByteType INSTANCE = new ByteType(); public final static int ID = 2; diff --git a/core/src/main/java/io/crate/types/DoubleType.java b/core/src/main/java/io/crate/types/DoubleType.java index f07a51acfa41..c9dde8cdf2e7 100644 --- a/core/src/main/java/io/crate/types/DoubleType.java +++ b/core/src/main/java/io/crate/types/DoubleType.java @@ -28,7 +28,7 @@ import java.io.IOException; -public class DoubleType extends DataType implements FixedWithType, Streamer, DataTypeFactory { +public class DoubleType extends DataType implements FixedWidthType, Streamer, DataTypeFactory { public static final DoubleType INSTANCE = new DoubleType(); public static final int ID = 6; diff --git a/core/src/main/java/io/crate/types/FixedWithType.java b/core/src/main/java/io/crate/types/FixedWidthType.java similarity index 95% rename from core/src/main/java/io/crate/types/FixedWithType.java rename to core/src/main/java/io/crate/types/FixedWidthType.java index 4516ef8bfe57..614ae3a7aad1 100644 --- a/core/src/main/java/io/crate/types/FixedWithType.java +++ b/core/src/main/java/io/crate/types/FixedWidthType.java @@ -24,11 +24,11 @@ /** * A type that has a fixed size for every value */ -public interface FixedWithType { +public interface FixedWidthType { /** * The fixed amount of memory a value object instance of type t requires. - * (t is the type described by our DataType interface or something that implements FixedWithType) + * (t is the type described by our DataType interface or something that implements FixedWidthType) * * * Implementations here may not be 100% accurate because sizes may vary between JVM implementations diff --git a/core/src/main/java/io/crate/types/FloatType.java b/core/src/main/java/io/crate/types/FloatType.java index 12a30e97c9ce..64321ea41f1f 100644 --- a/core/src/main/java/io/crate/types/FloatType.java +++ b/core/src/main/java/io/crate/types/FloatType.java @@ -28,7 +28,7 @@ import java.io.IOException; -public class FloatType extends DataType implements Streamer, DataTypeFactory, FixedWithType { +public class FloatType extends DataType implements Streamer, DataTypeFactory, FixedWidthType { public static final FloatType INSTANCE = new FloatType(); public static final int ID = 7; diff --git a/core/src/main/java/io/crate/types/GeoPointType.java b/core/src/main/java/io/crate/types/GeoPointType.java index f6e520240b69..6c3485047c82 100644 --- a/core/src/main/java/io/crate/types/GeoPointType.java +++ b/core/src/main/java/io/crate/types/GeoPointType.java @@ -35,7 +35,7 @@ import java.util.Arrays; import java.util.List; -public class GeoPointType extends DataType implements Streamer, DataTypeFactory, FixedWithType { +public class GeoPointType extends DataType implements Streamer, DataTypeFactory, FixedWidthType { public static final int ID = 13; public static final GeoPointType INSTANCE = new GeoPointType(); diff --git a/core/src/main/java/io/crate/types/IntegerType.java b/core/src/main/java/io/crate/types/IntegerType.java index b3e7f01cdb4e..a2ba7175bdfb 100644 --- a/core/src/main/java/io/crate/types/IntegerType.java +++ b/core/src/main/java/io/crate/types/IntegerType.java @@ -28,7 +28,7 @@ import java.io.IOException; -public class IntegerType extends DataType implements Streamer, DataTypeFactory, FixedWithType { +public class IntegerType extends DataType implements Streamer, DataTypeFactory, FixedWidthType { public static final IntegerType INSTANCE = new IntegerType(); public static final int ID = 9; diff --git a/core/src/main/java/io/crate/types/LongType.java b/core/src/main/java/io/crate/types/LongType.java index 7b05145ede8b..bcebdfc07887 100644 --- a/core/src/main/java/io/crate/types/LongType.java +++ b/core/src/main/java/io/crate/types/LongType.java @@ -28,7 +28,8 @@ import java.io.IOException; -public class LongType extends DataType implements FixedWithType, Streamer, DataTypeFactory { +public class + LongType extends DataType implements FixedWidthType, Streamer, DataTypeFactory { public static final LongType INSTANCE = new LongType(); public static final int ID = 10; diff --git a/core/src/main/java/io/crate/types/ShortType.java b/core/src/main/java/io/crate/types/ShortType.java index 5c91e9762c85..6ed57fb5037b 100644 --- a/core/src/main/java/io/crate/types/ShortType.java +++ b/core/src/main/java/io/crate/types/ShortType.java @@ -28,7 +28,7 @@ import java.io.IOException; -public class ShortType extends DataType implements DataTypeFactory, Streamer, FixedWithType { +public class ShortType extends DataType implements DataTypeFactory, Streamer, FixedWidthType { public static final ShortType INSTANCE = new ShortType(); public static final int ID = 8; diff --git a/docs/sql/aggregation.txt b/docs/sql/aggregation.txt index 06b2be410e6f..b0fbdf550a86 100644 --- a/docs/sql/aggregation.txt +++ b/docs/sql/aggregation.txt @@ -221,10 +221,10 @@ as a double value. Its single argument is the column name of a numeric column or cr> select sum(name), kind from locations group by kind order by sum(name) desc; SQLActionException[unknown function: sum(string)] -avg -=== +avg / mean +========== -The ``avg`` aggregation function returns the arithmetic mean, the *average*, +The ``avg`` or ``mean`` aggregation function returns the arithmetic mean, the *average*, of all values in a column that are not ``NULL`` as a double value. It accepts all numeric columns and timestamp columns as single argument. Using ``avg`` on other column types is not allowed. @@ -242,6 +242,94 @@ Example:: SELECT 3 rows in set (... sec) +geometric_mean +============== + +The ``geometric_mean`` aggregation function computes the geometric mean, +a mean for positive numbers. For details see: `Geometric Mean`_. + +``geometric mean`` is defined on all numeric types and on timestamp. It always +returns double values. If a value is negative, all values were null or we got no +value at all ``NULL`` is returned. If any of the aggregated values is ``0`` the result will be ``0.0`` +as well. + +.. note:: + + Due to java double precision arithmetic it is possible that any two executions + of the aggregation function on the same data produce slightly differing results. + +Example:: + + cr> select geometric_mean(position), kind from locations + ... group by kind order by kind; + +--------------------------+-------------+ + | geometric_mean(position) | kind | + +--------------------------+-------------+ + | 2.6321480259 | Galaxy | + | 2.6051710847 | Planet | + | 2.2133638394 | Star System | + +--------------------------+-------------+ + SELECT 3 rows in set (... sec) + + +variance +======== + +The ``variance`` aggregation function computes the `Variance`_ of the set of non-null +values in a column. It is a measure about how far a set of numbers is spread. +A variance of ``0.0`` indicates that all values are the same. + +``variance`` is defined on all numeric types and on timestamp. It returns a +double value. If all values were null or we got no value at all ``NULL`` is +returned. + +Example:: + + cr> select variance(position), kind from locations + ... group by kind order by kind desc; + +--------------------+-------------+ + | variance(position) | kind | + +--------------------+-------------+ + | 1.25 | Star System | + | 2.0 | Planet | + | 3.6875 | Galaxy | + +--------------------+-------------+ + SELECT 3 rows in set (... sec) + +.. note:: + + Due to java double precision arithmetic it is possible that any two executions + of the aggregation function on the same data produce slightly differing results. + +stddev +====== + +The ``stddev`` aggregation function computes the `Standard Deviation`_ of the set +of non-null values in a column. It is a measure of the variation of data values. +A low standard deviation indicates that the values tend to be near the mean. + +``stddev`` is defined on all numeric types and on timestamp. It always +returns double values. If all values were null or we got no +value at all ``NULL`` is returned. + +Example:: + + cr> select stddev(position), kind from locations + ... group by kind order by kind; + +------------------+-------------+ + | stddev(position) | kind | + +------------------+-------------+ + | 1.92028643697 | Galaxy | + | 1.41421356237 | Planet | + | 1.11803398875 | Star System | + +------------------+-------------+ + SELECT 3 rows in set (... sec) + +.. note:: + + Due to java double precision arithmetic it is possible that any two executions + of the aggregation function on the same data produce slightly differing results. + arbitrary ========= @@ -284,3 +372,7 @@ user. This works as rows with same ``user_id`` have the same strings. The advantage is that the ``arbitrary`` function does very little to no computation as for example ``max`` aggregation function would do. + +.. _Geometric Mean: https://en.wikipedia.org/wiki/Mean#Geometric_mean_.28GM.29 +.. _Variance: https://en.wikipedia.org/wiki/Variance +.. _Standard Deviation: https://en.wikipedia.org/wiki/Standard_deviation \ No newline at end of file diff --git a/docs/sql/system.txt b/docs/sql/system.txt index 7dc9307de393..515c370a02d1 100644 --- a/docs/sql/system.txt +++ b/docs/sql/system.txt @@ -653,8 +653,6 @@ Note, that querying a time setting will always return a ``string`` value:: +---------------------------------------------------...-------------------+ SELECT 1 row in set (... sec) -:: - The default configuration in ``crate.yml`` looks like: .. code-block:: yaml diff --git a/sql/build.gradle b/sql/build.gradle index 0d4fd29ee37a..46293b6c105a 100644 --- a/sql/build.gradle +++ b/sql/build.gradle @@ -15,7 +15,7 @@ dependencies { compile project(':blob') compile project(':sql-parser') compile 'com.amazonaws:aws-java-sdk:1.8.7' - + compile 'org.apache.commons:commons-math3:3.4.1' testCompile project(':testing') testCompile 'org.skyscreamer:jsonassert:1.2.0' testCompile 'com.h2database:h2:1.3.173' diff --git a/sql/src/main/java/io/crate/breaker/SizeEstimatorFactory.java b/sql/src/main/java/io/crate/breaker/SizeEstimatorFactory.java index 4baf90940db3..a8dfd30c0007 100644 --- a/sql/src/main/java/io/crate/breaker/SizeEstimatorFactory.java +++ b/sql/src/main/java/io/crate/breaker/SizeEstimatorFactory.java @@ -32,8 +32,8 @@ public static SizeEstimator create(DataType type) { case IpType.ID: return (SizeEstimator)new BytesRefSizeEstimator(); default: - if (type instanceof FixedWithType) { - return (SizeEstimator) new ConstSizeEstimator(((FixedWithType) type).fixedSize()); + if (type instanceof FixedWidthType) { + return (SizeEstimator) new ConstSizeEstimator(((FixedWidthType) type).fixedSize()); } throw new UnsupportedOperationException(String.format("Cannot get SizeEstimator for type %s", type)); } diff --git a/sql/src/main/java/io/crate/operation/aggregation/impl/AggregationImplModule.java b/sql/src/main/java/io/crate/operation/aggregation/impl/AggregationImplModule.java index 7db2810453d6..08b397a80b7b 100644 --- a/sql/src/main/java/io/crate/operation/aggregation/impl/AggregationImplModule.java +++ b/sql/src/main/java/io/crate/operation/aggregation/impl/AggregationImplModule.java @@ -53,5 +53,9 @@ protected void configure() { SumAggregation.register(this); CountAggregation.register(this); CollectSetAggregation.register(this); + + VarianceAggregation.register(this); + GeometricMeanAggregation.register(this); + StandardDeviationAggregation.register(this); } } diff --git a/sql/src/main/java/io/crate/operation/aggregation/impl/AverageAggregation.java b/sql/src/main/java/io/crate/operation/aggregation/impl/AverageAggregation.java index f726923a58a1..3849325114bf 100644 --- a/sql/src/main/java/io/crate/operation/aggregation/impl/AverageAggregation.java +++ b/sql/src/main/java/io/crate/operation/aggregation/impl/AverageAggregation.java @@ -31,7 +31,7 @@ import io.crate.types.DataType; import io.crate.types.DataTypeFactory; import io.crate.types.DataTypes; -import io.crate.types.FixedWithType; +import io.crate.types.FixedWidthType; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; @@ -39,26 +39,60 @@ public class AverageAggregation extends AggregationFunction { - public static final String NAME = "avg"; + public static final String[] NAMES = new String[] {"avg", "mean"}; + public static final String NAME = NAMES[0]; private final FunctionInfo info; + /** + * register as "avg" and "mean" + */ public static void register(AggregationImplModule mod) { - for (DataType t :DataTypes.NUMERIC_PRIMITIVE_TYPES) { + for (String name :NAMES) { + for (DataType t : DataTypes.NUMERIC_PRIMITIVE_TYPES) { + mod.register(new AverageAggregation(new FunctionInfo( + new FunctionIdent(name, ImmutableList.of(t)), DataTypes.DOUBLE, + FunctionInfo.Type.AGGREGATE))); + } mod.register(new AverageAggregation(new FunctionInfo( - new FunctionIdent(NAME, ImmutableList.of(t)), DataTypes.DOUBLE, - FunctionInfo.Type.AGGREGATE))); + new FunctionIdent(name, ImmutableList.of(DataTypes.TIMESTAMP)), DataTypes.DOUBLE, + FunctionInfo.Type.AGGREGATE))); } - mod.register(new AverageAggregation(new FunctionInfo( - new FunctionIdent(NAME, ImmutableList.of(DataTypes.TIMESTAMP)), DataTypes.DOUBLE, - FunctionInfo.Type.AGGREGATE))); } - AverageAggregation(FunctionInfo info) { - this.info = info; + public static class AverageState implements Comparable { + + private double sum = 0; + private long count = 0; + + public Double value() { + if (count > 0) { + return sum / count; + } else { + return null; + } + } + + @Override + public int compareTo(AverageState o) { + if (o == null) { + return 1; + } else { + int compare = Double.compare(sum, o.sum); + if (compare == 0) { + return Long.compare(count, o.count); + } + return compare; + } + } + + @Override + public String toString() { + return "sum: " + sum + " count: " + count; + } } public static class AverageStateType extends DataType - implements FixedWithType, Streamer, DataTypeFactory { + implements FixedWidthType, Streamer, DataTypeFactory { public static final int ID = 1024; private static final AverageStateType INSTANCE = new AverageStateType(); @@ -119,39 +153,10 @@ public int fixedSize() { } } - public static class AverageState implements Comparable { - - private double sum = 0; - private long count = 0; - - public Double value() { - if (count > 0) { - return sum / count; - } else { - return null; - } - } - - @Override - public int compareTo(AverageState o) { - if (o == null) { - return 1; - } else { - int compare = Double.compare(sum, o.sum); - if (compare == 0) { - return Long.compare(count, o.count); - } - return compare; - } - } - - @Override - public String toString() { - return "sum: " + sum + " count: " + count; - } + AverageAggregation(FunctionInfo info) { + this.info = info; } - @Override public AverageState iterate(RamAccountingContext ramAccountingContext, AverageState state, Input... args) { if (state != null) { diff --git a/sql/src/main/java/io/crate/operation/aggregation/impl/GeometricMeanAggregation.java b/sql/src/main/java/io/crate/operation/aggregation/impl/GeometricMeanAggregation.java new file mode 100644 index 000000000000..3f7d70e69358 --- /dev/null +++ b/sql/src/main/java/io/crate/operation/aggregation/impl/GeometricMeanAggregation.java @@ -0,0 +1,218 @@ +/* + * Licensed to CRATE Technology GmbH ("Crate") under one or more contributor + * license agreements. See the NOTICE file distributed with this work for + * additional information regarding copyright ownership. Crate licenses + * this file to you under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. You may + * obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * However, if you have executed another commercial license agreement + * with Crate these terms will supersede the license and you may use the + * software solely pursuant to the terms of the relevant commercial agreement. + */ + +package io.crate.operation.aggregation.impl; + +import com.google.common.collect.ComparisonChain; +import com.google.common.collect.ImmutableList; +import io.crate.Streamer; +import io.crate.breaker.RamAccountingContext; +import io.crate.metadata.FunctionIdent; +import io.crate.metadata.FunctionInfo; +import io.crate.operation.Input; +import io.crate.operation.aggregation.AggregationFunction; +import io.crate.types.DataType; +import io.crate.types.DataTypeFactory; +import io.crate.types.DataTypes; +import io.crate.types.FixedWidthType; +import org.apache.commons.math3.util.FastMath; +import org.elasticsearch.common.breaker.CircuitBreakingException; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.io.stream.Streamable; + +import javax.annotation.Nullable; +import java.io.IOException; + +public class GeometricMeanAggregation extends AggregationFunction { + + public static final String NAME = "geometric_mean"; + + public static void register(AggregationImplModule mod) { + for (DataType t : DataTypes.NUMERIC_PRIMITIVE_TYPES) { + mod.register(new GeometricMeanAggregation(new FunctionInfo( + new FunctionIdent(NAME, ImmutableList.of(t)), DataTypes.DOUBLE, + FunctionInfo.Type.AGGREGATE))); + } + mod.register(new GeometricMeanAggregation(new FunctionInfo( + new FunctionIdent(NAME, ImmutableList.of(DataTypes.TIMESTAMP)), DataTypes.DOUBLE, + FunctionInfo.Type.AGGREGATE))); + } + + public static class GeometricMeanState implements Comparable, Streamable { + /**Number of values that have been added */ + private long n; + + /** + * The currently running value + */ + private double value; + + public GeometricMeanState() { + value = 0d; + n = 0; + } + + private void addValue(double val) { + this.value += FastMath.log(val); + n++; + } + + private Double value() { + if (n > 0) { + return FastMath.exp(value / n); + } else { + return null; + } + } + + private void merge(GeometricMeanState other) { + this.value += other.value; + this.n += other.n; + } + + @Override + public int compareTo(GeometricMeanState o) { + return ComparisonChain.start() + .compare(value, o.value) + .compare(n, o.n) + .result(); + } + + @Override + public void readFrom(StreamInput in) throws IOException { + n = in.readVLong(); + value = in.readDouble(); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeVLong(n); + out.writeDouble(value); + } + } + + public static class GeometricMeanStateType extends DataType + implements Streamer, FixedWidthType, DataTypeFactory { + + public static final GeometricMeanStateType INSTANCE = new GeometricMeanStateType(); + public static final int ID = 4096; + + @Override + public int id() { + return ID; + } + + @Override + public String getName() { + return "geometric_mean_state"; + } + + @Override + public Streamer streamer() { + return this; + } + + @Override + public GeometricMeanState value(Object value) throws IllegalArgumentException, ClassCastException { + return (GeometricMeanState)value; + } + + @Override + public int compareValueTo(GeometricMeanState val1, GeometricMeanState val2) { + return val1.compareTo(val2); + } + + @Override + public DataType create() { + return INSTANCE; + } + + @Override + public int fixedSize() { + return 40; + } + + @Override + public GeometricMeanState readValueFrom(StreamInput in) throws IOException { + GeometricMeanState state = new GeometricMeanState(); + state.readFrom(in); + return state; + } + + @Override + public void writeValueTo(StreamOutput out, Object v) throws IOException { + GeometricMeanState state = (GeometricMeanState)v; + state.writeTo(out); + } + } + + private final FunctionInfo info; + + public GeometricMeanAggregation(FunctionInfo info) { + this.info = info; + } + + @Nullable + @Override + public GeometricMeanState newState(RamAccountingContext ramAccountingContext) { + ramAccountingContext.addBytes(GeometricMeanStateType.INSTANCE.fixedSize()); + return new GeometricMeanState(); + } + + @Override + public GeometricMeanState iterate(RamAccountingContext ramAccountingContext, GeometricMeanState state, Input... args) throws CircuitBreakingException { + if (state != null) { + Number value = (Number) args[0].value(); + if (value != null) { + state.addValue(value.doubleValue()); + } + } + return state; + } + + @Override + public GeometricMeanState reduce(RamAccountingContext ramAccountingContext, GeometricMeanState state1, GeometricMeanState state2) { + if (state1 == null) { + return state2; + } + if (state2 == null) { + return state1; + } + state1.merge(state2); + return state1; + } + + @Override + public Double terminatePartial(RamAccountingContext ramAccountingContext, GeometricMeanState state) { + return state.value(); + } + + @Override + public DataType partialType() { + return GeometricMeanStateType.INSTANCE; + } + + @Override + public FunctionInfo info() { + return info; + } +} diff --git a/sql/src/main/java/io/crate/operation/aggregation/impl/MaximumAggregation.java b/sql/src/main/java/io/crate/operation/aggregation/impl/MaximumAggregation.java index 2fb97884e31c..11d57a487b57 100644 --- a/sql/src/main/java/io/crate/operation/aggregation/impl/MaximumAggregation.java +++ b/sql/src/main/java/io/crate/operation/aggregation/impl/MaximumAggregation.java @@ -31,7 +31,7 @@ import io.crate.operation.aggregation.AggregationFunction; import io.crate.types.DataType; import io.crate.types.DataTypes; -import io.crate.types.FixedWithType; +import io.crate.types.FixedWidthType; import org.elasticsearch.common.breaker.CircuitBreakingException; public abstract class MaximumAggregation extends AggregationFunction { @@ -45,7 +45,7 @@ public static void register(AggregationImplModule mod) { FunctionInfo functionInfo = new FunctionInfo( new FunctionIdent(NAME, ImmutableList.of(dataType)), dataType, FunctionInfo.Type.AGGREGATE); - if (dataType instanceof FixedWithType) { + if (dataType instanceof FixedWidthType) { mod.register(new FixedMaximumAggregation(functionInfo)); } else { mod.register(new VariableMaximumAggregation(functionInfo)); @@ -59,7 +59,7 @@ private static class FixedMaximumAggregation extends MaximumAggregation { public FixedMaximumAggregation(FunctionInfo info) { super(info); - size = ((FixedWithType) partialType()).fixedSize(); + size = ((FixedWidthType) partialType()).fixedSize(); } @Override diff --git a/sql/src/main/java/io/crate/operation/aggregation/impl/MinimumAggregation.java b/sql/src/main/java/io/crate/operation/aggregation/impl/MinimumAggregation.java index cb335263d872..7781c4b713d9 100644 --- a/sql/src/main/java/io/crate/operation/aggregation/impl/MinimumAggregation.java +++ b/sql/src/main/java/io/crate/operation/aggregation/impl/MinimumAggregation.java @@ -32,7 +32,7 @@ import io.crate.operation.aggregation.AggregationFunction; import io.crate.types.DataType; import io.crate.types.DataTypes; -import io.crate.types.FixedWithType; +import io.crate.types.FixedWidthType; public abstract class MinimumAggregation extends AggregationFunction { @@ -45,7 +45,7 @@ public static void register(AggregationImplModule mod) { FunctionInfo functionInfo = new FunctionInfo(new FunctionIdent(NAME, ImmutableList.of(dataType)), dataType, FunctionInfo.Type.AGGREGATE); - if (dataType instanceof FixedWithType) { + if (dataType instanceof FixedWidthType) { mod.register(new FixedMinimumAggregation(functionInfo)); } else { mod.register(new VariableMinimumAggregation(functionInfo)); @@ -92,7 +92,7 @@ private static class FixedMinimumAggregation extends MinimumAggregation { FixedMinimumAggregation(FunctionInfo info) { super(info); - size = ((FixedWithType) partialType()).fixedSize(); + size = ((FixedWidthType) partialType()).fixedSize(); } @Override diff --git a/sql/src/main/java/io/crate/operation/aggregation/impl/StandardDeviationAggregation.java b/sql/src/main/java/io/crate/operation/aggregation/impl/StandardDeviationAggregation.java new file mode 100644 index 000000000000..e60ae907ebde --- /dev/null +++ b/sql/src/main/java/io/crate/operation/aggregation/impl/StandardDeviationAggregation.java @@ -0,0 +1,187 @@ +/* + * Licensed to CRATE Technology GmbH ("Crate") under one or more contributor + * license agreements. See the NOTICE file distributed with this work for + * additional information regarding copyright ownership. Crate licenses + * this file to you under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. You may + * obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * However, if you have executed another commercial license agreement + * with Crate these terms will supersede the license and you may use the + * software solely pursuant to the terms of the relevant commercial agreement. + */ + +package io.crate.operation.aggregation.impl; + +import com.google.common.collect.ImmutableList; +import io.crate.Streamer; +import io.crate.breaker.RamAccountingContext; +import io.crate.metadata.FunctionIdent; +import io.crate.metadata.FunctionInfo; +import io.crate.operation.Input; +import io.crate.operation.aggregation.AggregationFunction; +import io.crate.operation.aggregation.statistics.moment.StandardDeviation; +import io.crate.types.DataType; +import io.crate.types.DataTypeFactory; +import io.crate.types.DataTypes; +import io.crate.types.FixedWidthType; +import org.elasticsearch.common.breaker.CircuitBreakingException; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; + +import javax.annotation.Nullable; +import java.io.IOException; + +public class StandardDeviationAggregation extends AggregationFunction { + + public static final String NAME = "stddev"; + + + public static void register(AggregationImplModule mod) { + for (DataType t : DataTypes.NUMERIC_PRIMITIVE_TYPES) { + mod.register(new StandardDeviationAggregation(new FunctionInfo( + new FunctionIdent(NAME, ImmutableList.of(t)), DataTypes.DOUBLE, + FunctionInfo.Type.AGGREGATE))); + } + mod.register(new StandardDeviationAggregation(new FunctionInfo( + new FunctionIdent(NAME, ImmutableList.of(DataTypes.TIMESTAMP)), DataTypes.DOUBLE, + FunctionInfo.Type.AGGREGATE))); + } + + public static class StdDevState implements Comparable { + + private final StandardDeviation stdDev; + + public StdDevState() { + this.stdDev = new StandardDeviation(); + } + + private void addValue(double val) { + this.stdDev.increment(val); + } + + private Double value() { + double result = stdDev.result(); + return (Double.isNaN(result) ? null : result); + } + + @Override + public int compareTo(StdDevState o) { + return Double.compare(stdDev.result(), o.stdDev.result()); + } + } + + public static class StdDevStateType extends DataType + implements Streamer, FixedWidthType, DataTypeFactory { + + public static final StdDevStateType INSTANCE = new StdDevStateType(); + public static final int ID = 8192; + + @Override + public int id() { + return ID; + } + + @Override + public String getName() { + return "stddev_state"; + } + + @Override + public Streamer streamer() { + return this; + } + + @Override + public StdDevState value(Object value) throws IllegalArgumentException, ClassCastException { + return (StdDevState)value; + } + + @Override + public int compareValueTo(StdDevState val1, StdDevState val2) { + return val1.compareTo(val2); + } + + @Override + public DataType create() { + return INSTANCE; + } + + @Override + public int fixedSize() { + return 56; + } + + @Override + public StdDevState readValueFrom(StreamInput in) throws IOException { + StdDevState state = new StdDevState(); + state.stdDev.readFrom(in); + return state; + } + + @Override + public void writeValueTo(StreamOutput out, Object v) throws IOException { + StdDevState state = (StdDevState)v; + state.stdDev.writeTo(out); + } + } + + private final FunctionInfo info; + + public StandardDeviationAggregation(FunctionInfo functionInfo) { + this.info = functionInfo; + } + + @Nullable + @Override + public StdDevState newState(RamAccountingContext ramAccountingContext) { + ramAccountingContext.addBytes(StdDevStateType.INSTANCE.fixedSize()); + return new StdDevState(); + } + + @Override + public StdDevState iterate(RamAccountingContext ramAccountingContext, StdDevState state, Input... args) throws CircuitBreakingException { + if (state != null) { + Number value = (Number) args[0].value(); + if (value != null) { + state.addValue(value.doubleValue()); + } + } + return state; + } + + @Override + public StdDevState reduce(RamAccountingContext ramAccountingContext, StdDevState state1, StdDevState state2) { + if (state1 == null) { + return state2; + } + if (state2 == null) { + return state1; + } + state1.stdDev.merge(state2.stdDev); + return state1; + } + + @Override + public Double terminatePartial(RamAccountingContext ramAccountingContext, StdDevState state) { + return state.value(); + } + + @Override + public DataType partialType() { + return StdDevStateType.INSTANCE; + } + + @Override + public FunctionInfo info() { + return info; + } +} diff --git a/sql/src/main/java/io/crate/operation/aggregation/impl/VarianceAggregation.java b/sql/src/main/java/io/crate/operation/aggregation/impl/VarianceAggregation.java new file mode 100644 index 000000000000..8bb2ea94b643 --- /dev/null +++ b/sql/src/main/java/io/crate/operation/aggregation/impl/VarianceAggregation.java @@ -0,0 +1,189 @@ +/* + * Licensed to CRATE Technology GmbH ("Crate") under one or more contributor + * license agreements. See the NOTICE file distributed with this work for + * additional information regarding copyright ownership. Crate licenses + * this file to you under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. You may + * obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * However, if you have executed another commercial license agreement + * with Crate these terms will supersede the license and you may use the + * software solely pursuant to the terms of the relevant commercial agreement. + */ + +package io.crate.operation.aggregation.impl; + +import com.google.common.collect.ImmutableList; +import io.crate.Streamer; +import io.crate.breaker.RamAccountingContext; +import io.crate.metadata.FunctionIdent; +import io.crate.metadata.FunctionInfo; +import io.crate.operation.Input; +import io.crate.operation.aggregation.AggregationFunction; +import io.crate.operation.aggregation.statistics.moment.Variance; +import io.crate.types.DataType; +import io.crate.types.DataTypeFactory; +import io.crate.types.DataTypes; +import io.crate.types.FixedWidthType; +import org.elasticsearch.common.breaker.CircuitBreakingException; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; + +import javax.annotation.Nullable; +import java.io.IOException; + + +public class VarianceAggregation extends AggregationFunction { + + public static final String NAME = "variance"; + + public static void register(AggregationImplModule mod) { + for (DataType t : DataTypes.NUMERIC_PRIMITIVE_TYPES) { + mod.register(new VarianceAggregation(new FunctionInfo( + new FunctionIdent(NAME, ImmutableList.of(t)), DataTypes.DOUBLE, + FunctionInfo.Type.AGGREGATE))); + } + mod.register(new VarianceAggregation(new FunctionInfo( + new FunctionIdent(NAME, ImmutableList.of(DataTypes.TIMESTAMP)), DataTypes.DOUBLE, + FunctionInfo.Type.AGGREGATE))); + } + + public static class VarianceState implements Comparable { + + private final Variance variance; + + public VarianceState() { + this.variance = new Variance(); + } + + private void addValue(double val) { + variance.increment(val); + } + + private Double value() { + double result = variance.result(); + return (Double.isNaN(result) ? null : result); + } + + @Override + public int compareTo(VarianceState o) { + return Double.compare(variance.result(), o.variance.result()); + } + } + + public static class VarianceStateType extends DataType + implements Streamer, FixedWidthType, DataTypeFactory { + + public static final VarianceStateType INSTANCE = new VarianceStateType(); + public static final int ID = 2048; + + + @Override + public int id() { + return ID; + } + + @Override + public String getName() { + return "variance_state"; + } + + @Override + public Streamer streamer() { + return this; + } + + @Override + public VarianceState value(Object value) throws IllegalArgumentException, ClassCastException { + return (VarianceState)value; + } + + @Override + public int compareValueTo(VarianceState val1, VarianceState val2) { + return val1.compareTo(val2); + } + + @Override + public DataType create() { + return INSTANCE; + } + + @Override + public VarianceState readValueFrom(StreamInput in) throws IOException { + VarianceState state = new VarianceState(); + state.variance.readFrom(in); + return state; + } + + @Override + public void writeValueTo(StreamOutput out, Object v) throws IOException { + VarianceState state = (VarianceState)v; + state.variance.writeTo(out); + } + + @Override + public int fixedSize() { + return 56; + } + } + + private final FunctionInfo info; + + public VarianceAggregation(FunctionInfo info) { + this.info = info; + } + + + @Nullable + @Override + public VarianceAggregation.VarianceState newState(RamAccountingContext ramAccountingContext) { + ramAccountingContext.addBytes(VarianceStateType.INSTANCE.fixedSize()); + return new VarianceState(); + } + + @Override + public VarianceAggregation.VarianceState iterate(RamAccountingContext ramAccountingContext, VarianceAggregation.VarianceState state, Input... args) throws CircuitBreakingException { + if (state != null) { + Number value = (Number) args[0].value(); + if (value != null) { + state.addValue(value.doubleValue()); + } + } + return state; + } + + @Override + public VarianceAggregation.VarianceState reduce(RamAccountingContext ramAccountingContext, VarianceAggregation.VarianceState state1, VarianceAggregation.VarianceState state2) { + if (state1 == null) { + return state2; + } + if (state2 == null) { + return state1; + } + state1.variance.merge(state2.variance); + return state1; + } + + @Override + public Double terminatePartial(RamAccountingContext ramAccountingContext, VarianceAggregation.VarianceState state) { + return state.value(); + } + + @Override + public DataType partialType() { + return VarianceStateType.INSTANCE; + } + + @Override + public FunctionInfo info() { + return info; + } +} diff --git a/sql/src/main/java/io/crate/operation/aggregation/statistics/moment/StandardDeviation.java b/sql/src/main/java/io/crate/operation/aggregation/statistics/moment/StandardDeviation.java new file mode 100644 index 000000000000..70b914222119 --- /dev/null +++ b/sql/src/main/java/io/crate/operation/aggregation/statistics/moment/StandardDeviation.java @@ -0,0 +1,31 @@ +/* + * Licensed to CRATE Technology GmbH ("Crate") under one or more contributor + * license agreements. See the NOTICE file distributed with this work for + * additional information regarding copyright ownership. Crate licenses + * this file to you under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. You may + * obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * However, if you have executed another commercial license agreement + * with Crate these terms will supersede the license and you may use the + * software solely pursuant to the terms of the relevant commercial agreement. + */ + +package io.crate.operation.aggregation.statistics.moment; + +import org.apache.commons.math3.util.FastMath; + +public class StandardDeviation extends Variance { + @Override + public double result() { + return FastMath.sqrt(super.result()); + } +} diff --git a/sql/src/main/java/io/crate/operation/aggregation/statistics/moment/Variance.java b/sql/src/main/java/io/crate/operation/aggregation/statistics/moment/Variance.java new file mode 100644 index 000000000000..e06c70d9d09e --- /dev/null +++ b/sql/src/main/java/io/crate/operation/aggregation/statistics/moment/Variance.java @@ -0,0 +1,74 @@ +/* + * Licensed to CRATE Technology GmbH ("Crate") under one or more contributor + * license agreements. See the NOTICE file distributed with this work for + * additional information regarding copyright ownership. Crate licenses + * this file to you under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. You may + * obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * However, if you have executed another commercial license agreement + * with Crate these terms will supersede the license and you may use the + * software solely pursuant to the terms of the relevant commercial agreement. + */ + +package io.crate.operation.aggregation.statistics.moment; + +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.io.stream.Streamable; + +import java.io.IOException; + +public class Variance implements Streamable { + + double sumOfSqrs; + double sum; + long count; + + public Variance() { + sumOfSqrs = 0.0; + sum = 0.0; + count = 0; + } + + public void increment(double value) { + sumOfSqrs += (value * value); + sum += value; + count++; + } + + public synchronized double result() { + if (count == 0) { + return Double.NaN; + } + return (sumOfSqrs - ((sum * sum) / count)) / count; + } + + public void merge(Variance other) { + sumOfSqrs += other.sumOfSqrs; + sum += other.sum; + count += other.count; + } + + @Override + public void readFrom(StreamInput in) throws IOException { + sumOfSqrs = in.readDouble(); + sum = in.readDouble(); + count = in.readVLong(); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeDouble(sumOfSqrs); + out.writeDouble(sum); + out.writeVLong(count); + } +} diff --git a/sql/src/test/java/io/crate/integrationtests/GroupByAggregateTest.java b/sql/src/test/java/io/crate/integrationtests/GroupByAggregateTest.java index d1a22a495e5b..587e7689311b 100644 --- a/sql/src/test/java/io/crate/integrationtests/GroupByAggregateTest.java +++ b/sql/src/test/java/io/crate/integrationtests/GroupByAggregateTest.java @@ -35,6 +35,7 @@ import java.util.HashMap; import static org.hamcrest.CoreMatchers.is; +import static org.hamcrest.Matchers.closeTo; import static org.hamcrest.Matchers.isIn; @CrateIntegrationTest.ClusterScope(scope = CrateIntegrationTest.Scope.GLOBAL) @@ -388,20 +389,26 @@ public void testGroupByAggSumShort() throws Exception { @Test public void testGroupByAvgDouble() throws Exception { - execute("select avg(income), department from employees group by department order by department asc"); + execute("select avg(income), mean(income), department from employees group by department order by department asc"); assertEquals(4, response.rowCount()); - assertEquals("HR", response.rows()[0][1]); + assertEquals(500000000.245d, response.rows()[0][0]); + assertEquals(500000000.245d, response.rows()[0][1]); + assertEquals("HR", response.rows()[0][2]); - assertEquals("engineering", response.rows()[1][1]); assertEquals(5000.0d, response.rows()[1][0]); + assertEquals(5000.0d, response.rows()[1][1]); + assertEquals("engineering", response.rows()[1][2]); + - assertEquals("internship", response.rows()[2][1]); assertEquals(null, response.rows()[2][0]); + assertEquals(null, response.rows()[2][1]); + assertEquals("internship", response.rows()[2][2]); - assertEquals("management", response.rows()[3][1]); assertEquals(Double.MAX_VALUE, response.rows()[3][0]); + assertEquals(Double.MAX_VALUE, response.rows()[3][1]); + assertEquals("management", response.rows()[3][2]); } @Test @@ -947,4 +954,47 @@ public void testNonDistributedGroupByWithHavingAndLimit() throws Exception { fail(e.getMessage()); } } + + @Test + public void groupByAggregateStdDevByte() throws Exception { + this.setup.groupBySetup("byte"); + + execute("select stddev(age), gender from characters group by gender order by gender"); + assertEquals(2L, response.rowCount()); + assertEquals(5.5d, response.rows()[0][0]); + assertEquals(39.0d, response.rows()[1][0]); + } + + @Test + public void groupByAggregateVarianceByte() throws Exception { + this.setup.groupBySetup("byte"); + + execute("select variance(age), gender from characters group by gender order by gender"); + assertEquals(2L, response.rowCount()); + assertEquals(30.25d, response.rows()[0][0]); + assertEquals(1521.0d, response.rows()[1][0]); + } + + @Test + public void groupByAggregateStdDevDouble() throws Exception { + this.setup.groupBySetup("double"); + + execute("select stddev(age), gender from characters group by gender order by gender"); + assertEquals(2L, response.rowCount()); + assertEquals(5.5d, response.rows()[0][0]); + assertEquals(39.0d, response.rows()[1][0]); + } + + @Test + public void groupByStatsAggregatesGlobal() throws Exception { + this.setup.groupBySetup("short"); + execute("select min(age), mean(age), geometric_mean(age), max(age), variance(age), stddev(age) from characters"); + assertThat((Short) response.rows()[0][0], is((short) 32)); + assertThat((Double)response.rows()[0][1], is(55.25d)); + + assertThat((Double)response.rows()[0][2], closeTo(47.84415001097868d, 0.0000001)); + assertThat((Short)response.rows()[0][3], is((short)112)); + assertThat((Double)response.rows()[0][4], is(1090.6875d)); + assertThat((Double)response.rows()[0][5], is(33.025558284456d)); + } } diff --git a/sql/src/test/java/io/crate/operation/aggregation/impl/AverageAggregationTest.java b/sql/src/test/java/io/crate/operation/aggregation/impl/AverageAggregationTest.java index 90318eb53ec0..5951e28859cd 100644 --- a/sql/src/test/java/io/crate/operation/aggregation/impl/AverageAggregationTest.java +++ b/sql/src/test/java/io/crate/operation/aggregation/impl/AverageAggregationTest.java @@ -41,6 +41,9 @@ public void testReturnType() throws Exception { FunctionIdent fi = new FunctionIdent("avg", ImmutableList.of(DataTypes.INTEGER)); // Return type is fixed to Double assertEquals(DataTypes.DOUBLE, functions.get(fi).info().returnType()); + + FunctionIdent meanFi = new FunctionIdent("mean", ImmutableList.of(DataTypes.INTEGER)); + assertEquals(DataTypes.DOUBLE, functions.get(meanFi).info().returnType()); } @Test diff --git a/sql/src/test/java/io/crate/operation/aggregation/impl/GeometricMeanAggregationtest.java b/sql/src/test/java/io/crate/operation/aggregation/impl/GeometricMeanAggregationtest.java new file mode 100644 index 000000000000..45087e325d05 --- /dev/null +++ b/sql/src/test/java/io/crate/operation/aggregation/impl/GeometricMeanAggregationtest.java @@ -0,0 +1,104 @@ +/* + * Licensed to CRATE Technology GmbH ("Crate") under one or more contributor + * license agreements. See the NOTICE file distributed with this work for + * additional information regarding copyright ownership. Crate licenses + * this file to you under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. You may + * obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * However, if you have executed another commercial license agreement + * with Crate these terms will supersede the license and you may use the + * software solely pursuant to the terms of the relevant commercial agreement. + */ + +package io.crate.operation.aggregation.impl; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Iterables; +import io.crate.metadata.FunctionIdent; +import io.crate.operation.aggregation.AggregationTest; +import io.crate.types.DataType; +import io.crate.types.DataTypes; +import org.junit.Test; + +import java.util.Arrays; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; + +public class GeometricMeanAggregationtest extends AggregationTest { + + private Object[][] executeAggregation(DataType dataType, Object[][] data) throws Exception { + return executeAggregation("geometric_mean", dataType, data); + } + + @Test + public void testReturnType() throws Exception { + for (DataType type : Iterables.concat(DataTypes.NUMERIC_PRIMITIVE_TYPES, Arrays.asList(DataTypes.TIMESTAMP))) { + FunctionIdent fi = new FunctionIdent("geometric_mean", ImmutableList.of(type)); + // Return type is fixed to Double + assertEquals(DataTypes.DOUBLE, functions.get(fi).info().returnType()); + } + } + + @Test + public void withNullArg() throws Exception { + Object[][] result = executeAggregation(DataTypes.DOUBLE, new Object[][]{{null}, {null}}); + assertNull(result[0][0]); + } + + @Test + public void testDouble() throws Exception { + Object[][] result = executeAggregation(DataTypes.DOUBLE, new Object[][]{{1.0d}, {1000.0d}, {1.0d}, {null}}); + + assertEquals(9.999999999999998d, result[0][0]); + } + + @Test + public void testFloat() throws Exception { + Object[][] result = executeAggregation(DataTypes.FLOAT, new Object[][]{{0.7f}, {0.3f}, {0.7f}}); + + assertEquals(0.5277632097890468d, result[0][0]); + } + + @Test + public void testInteger() throws Exception { + Object[][] result = executeAggregation(DataTypes.INTEGER, new Object[][]{{7}, {3}}); + + assertEquals(4.58257569495584d, result[0][0]); + } + + @Test + public void testLong() throws Exception { + Object[][] result = executeAggregation(DataTypes.LONG, new Object[][]{{1L}, {3L}, {2L}}); + + assertEquals(1.8171205928321397d, result[0][0]); + } + + @Test + public void testShort() throws Exception { + Object[][] result = executeAggregation(DataTypes.SHORT, new Object[][]{{(short) 0}, {(short) 3}, {(short) 1000}}); + + assertEquals(0d, result[0][0]); + } + + @Test + public void testByte() throws Exception { + Object[][] result = executeAggregation(DataTypes.SHORT, new Object[][]{{(short) 1}, {(short) 1}}); + + assertEquals(1.0d, result[0][0]); + } + + @Test(expected = NullPointerException.class) + public void testUnsupportedType() throws Exception { + Object[][] result = executeAggregation(DataTypes.BOOLEAN, new Object[][]{{true}, {false}}); + } +} diff --git a/sql/src/test/java/io/crate/operation/aggregation/impl/StdDevAggregationTest.java b/sql/src/test/java/io/crate/operation/aggregation/impl/StdDevAggregationTest.java new file mode 100644 index 000000000000..54387b14a0c5 --- /dev/null +++ b/sql/src/test/java/io/crate/operation/aggregation/impl/StdDevAggregationTest.java @@ -0,0 +1,111 @@ +/* + * Licensed to CRATE Technology GmbH ("Crate") under one or more contributor + * license agreements. See the NOTICE file distributed with this work for + * additional information regarding copyright ownership. Crate licenses + * this file to you under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. You may + * obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * However, if you have executed another commercial license agreement + * with Crate these terms will supersede the license and you may use the + * software solely pursuant to the terms of the relevant commercial agreement. + */ + +package io.crate.operation.aggregation.impl; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Iterables; +import io.crate.metadata.FunctionIdent; +import io.crate.operation.aggregation.AggregationTest; +import io.crate.types.DataType; +import io.crate.types.DataTypes; +import org.junit.Test; + +import java.util.Arrays; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; + +public class StdDevAggregationTest extends AggregationTest { + + private Object[][] executeAggregation(DataType dataType, Object[][] data) throws Exception { + return executeAggregation("stddev", dataType, data); + } + + @Test + public void testReturnType() throws Exception { + for (DataType type : Iterables.concat(DataTypes.NUMERIC_PRIMITIVE_TYPES, Arrays.asList(DataTypes.TIMESTAMP))) { + FunctionIdent fi = new FunctionIdent("stddev", ImmutableList.of(type)); + // Return type is fixed to Double + assertEquals(DataTypes.DOUBLE, functions.get(fi).info().returnType()); + } + } + + @Test + public void withNullArg() throws Exception { + Object[][] result = executeAggregation(DataTypes.DOUBLE, new Object[][]{{null}, {null}}); + assertNull(result[0][0]); + } + + @Test + public void withSomeNullArgs() throws Exception { + Object[][] result = executeAggregation(DataTypes.DOUBLE, new Object[][]{{10.7d}, {42.9D}, {0.3d}, {null}}); + assertEquals(18.13455878212156, result[0][0]); + } + + @Test + public void testDouble() throws Exception { + Object[][] result = executeAggregation(DataTypes.DOUBLE, new Object[][]{{10.7d}, {42.9D}, {0.3d}}); + + assertEquals(18.13455878212156, result[0][0]); + } + + @Test + public void testFloat() throws Exception { + Object[][] result = executeAggregation(DataTypes.FLOAT, new Object[][]{{1.5f}, {1.25f}, {1.75f}}); + + assertEquals(0.2041241452319315, result[0][0]); + } + + @Test + public void testInteger() throws Exception { + Object[][] result = executeAggregation(DataTypes.INTEGER, new Object[][]{{7}, {3}}); + + assertEquals(2d, result[0][0]); + } + + @Test + public void testLong() throws Exception { + Object[][] result = executeAggregation(DataTypes.LONG, new Object[][]{{7L}, {3L}}); + + assertEquals(2d, result[0][0]); + } + + @Test + public void testShort() throws Exception { + Object[][] result = executeAggregation(DataTypes.SHORT, new Object[][]{{(short) 7}, {(short) 3}}); + + assertEquals(2d, result[0][0]); + } + + @Test + public void testByte() throws Exception { + Object[][] result = executeAggregation(DataTypes.SHORT, new Object[][]{{(short) 1}, {(short) 1}}); + + assertEquals(0d, result[0][0]); + } + + @Test(expected = NullPointerException.class) + public void testUnsupportedType() throws Exception { + Object[][] result = executeAggregation(DataTypes.STRING, new Object[][]{{"Youri"}, {"Ruben"}}); + } + +} diff --git a/sql/src/test/java/io/crate/operation/aggregation/impl/VarianceAggregationTest.java b/sql/src/test/java/io/crate/operation/aggregation/impl/VarianceAggregationTest.java new file mode 100644 index 000000000000..734b894fdbf5 --- /dev/null +++ b/sql/src/test/java/io/crate/operation/aggregation/impl/VarianceAggregationTest.java @@ -0,0 +1,104 @@ +/* + * Licensed to CRATE Technology GmbH ("Crate") under one or more contributor + * license agreements. See the NOTICE file distributed with this work for + * additional information regarding copyright ownership. Crate licenses + * this file to you under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. You may + * obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * However, if you have executed another commercial license agreement + * with Crate these terms will supersede the license and you may use the + * software solely pursuant to the terms of the relevant commercial agreement. + */ + +package io.crate.operation.aggregation.impl; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Iterables; +import io.crate.metadata.FunctionIdent; +import io.crate.operation.aggregation.AggregationTest; +import io.crate.types.DataType; +import io.crate.types.DataTypes; +import org.junit.Test; + +import java.util.Arrays; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; + +public class VarianceAggregationTest extends AggregationTest { + + private Object[][] executeAggregation(DataType dataType, Object[][] data) throws Exception { + return executeAggregation("variance", dataType, data); + } + + @Test + public void testReturnType() throws Exception { + for (DataType type : Iterables.concat(DataTypes.NUMERIC_PRIMITIVE_TYPES, Arrays.asList(DataTypes.TIMESTAMP))) { + FunctionIdent fi = new FunctionIdent("variance", ImmutableList.of(type)); + // Return type is fixed to Double + assertEquals(DataTypes.DOUBLE, functions.get(fi).info().returnType()); + } + } + + @Test + public void withNullArg() throws Exception { + Object[][] result = executeAggregation(DataTypes.DOUBLE, new Object[][]{{null}, {null}}); + assertNull(result[0][0]); + } + + @Test + public void testDouble() throws Exception { + Object[][] result = executeAggregation(DataTypes.DOUBLE, new Object[][]{{1.0d}, {1.0d}, {1.0d}, {null}}); + + assertEquals(0.0d, result[0][0]); + } + + @Test + public void testFloat() throws Exception { + Object[][] result = executeAggregation(DataTypes.FLOAT, new Object[][]{{0.7f}, {0.3f}, {0.7f}}); + + assertEquals(0.035555551317003165d, result[0][0]); + } + + @Test + public void testInteger() throws Exception { + Object[][] result = executeAggregation(DataTypes.INTEGER, new Object[][]{{7}, {3}}); + + assertEquals(4d, result[0][0]); + } + + @Test + public void testLong() throws Exception { + Object[][] result = executeAggregation(DataTypes.LONG, new Object[][]{{7L}, {3L}}); + + assertEquals(4d, result[0][0]); + } + + @Test + public void testShort() throws Exception { + Object[][] result = executeAggregation(DataTypes.SHORT, new Object[][]{{(short) 7}, {(short) 3}}); + + assertEquals(4d, result[0][0]); + } + + @Test + public void testByte() throws Exception { + Object[][] result = executeAggregation(DataTypes.SHORT, new Object[][]{{(short) 1}, {(short) 1}}); + + assertEquals(0d, result[0][0]); + } + + @Test(expected = NullPointerException.class) + public void testUnsupportedType() throws Exception { + Object[][] result = executeAggregation(DataTypes.STRING, new Object[][]{{"Youri"}, {"Ruben"}}); + } +}