From 46c5dd47deac696b19536ee5bb78667006bad8a5 Mon Sep 17 00:00:00 2001 From: sirpkt Date: Mon, 22 Dec 2014 14:26:04 +0900 Subject: [PATCH 1/6] TAJO-1265: min(), max() does not handle null properly --- .../tajo/engine/function/builtin/Max.java | 69 +++++++++++ .../engine/function/builtin/MaxDouble.java | 43 +------ .../engine/function/builtin/MaxFloat.java | 43 ++----- .../tajo/engine/function/builtin/MaxInt.java | 44 ++----- .../tajo/engine/function/builtin/MaxLong.java | 32 +---- .../engine/function/builtin/MaxString.java | 52 ++------ .../tajo/engine/function/builtin/Min.java | 69 +++++++++++ .../engine/function/builtin/MinDouble.java | 41 +------ .../engine/function/builtin/MinFloat.java | 42 +------ .../tajo/engine/function/builtin/MinInt.java | 43 ++----- .../tajo/engine/function/builtin/MinLong.java | 32 +---- .../engine/function/builtin/MinString.java | 46 +------- .../engine/function/TestBuiltinFunctions.java | 111 ++++++++++++++++++ 13 files changed, 305 insertions(+), 362 deletions(-) create mode 100644 tajo-core/src/main/java/org/apache/tajo/engine/function/builtin/Max.java create mode 100644 tajo-core/src/main/java/org/apache/tajo/engine/function/builtin/Min.java diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/function/builtin/Max.java b/tajo-core/src/main/java/org/apache/tajo/engine/function/builtin/Max.java new file mode 100644 index 0000000000..141df6cc90 --- /dev/null +++ b/tajo-core/src/main/java/org/apache/tajo/engine/function/builtin/Max.java @@ -0,0 +1,69 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tajo.engine.function.builtin; + +import org.apache.tajo.catalog.Column; +import org.apache.tajo.datum.Datum; +import org.apache.tajo.datum.NullDatum; +import org.apache.tajo.plan.function.AggFunction; +import org.apache.tajo.plan.function.FunctionContext; +import org.apache.tajo.storage.Tuple; + +public abstract class Max extends AggFunction { + public Max(Column[] definedArgs) { + super(definedArgs); + } + + @Override + public FunctionContext newContext() { + return new MaxContext(); + } + + @Override + public void eval(FunctionContext ctx, Tuple params) { + MaxContext maxCtx = (MaxContext) ctx; + Datum datum = params.get(0); + if (datum.isNotNull()) { + if (maxCtx.max == null || maxCtx.max.compareTo(datum) < 0) { + maxCtx.max = datum; + } + } + } + + @Override + public Datum getPartialResult(FunctionContext ctx) { + return ((MaxContext) ctx).max; + } + + @Override + public Datum terminate(FunctionContext ctx) { + Datum max = ((MaxContext)ctx).max; + + if (max == null) { + return NullDatum.get(); + } + else { + return max; + } + } + + private class MaxContext implements FunctionContext { + Datum max = null; + } +} diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/function/builtin/MaxDouble.java b/tajo-core/src/main/java/org/apache/tajo/engine/function/builtin/MaxDouble.java index a6840c4e20..c5f5f41b06 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/function/builtin/MaxDouble.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/function/builtin/MaxDouble.java @@ -22,57 +22,26 @@ import org.apache.tajo.catalog.Column; import org.apache.tajo.common.TajoDataTypes.DataType; import org.apache.tajo.common.TajoDataTypes.Type; -import org.apache.tajo.datum.Datum; -import org.apache.tajo.datum.DatumFactory; -import org.apache.tajo.datum.Float8Datum; -import org.apache.tajo.plan.function.AggFunction; -import org.apache.tajo.plan.function.FunctionContext; import org.apache.tajo.engine.function.annotation.Description; import org.apache.tajo.engine.function.annotation.ParamTypes; -import org.apache.tajo.storage.Tuple; @Description( - functionName = "max", - description = "the maximum value of expr", - example = "> SELECT max(expr);", - returnType = Type.FLOAT8, - paramTypes = {@ParamTypes(paramTypes = {Type.FLOAT8})} + functionName = "max", + description = "the maximum value of expr", + example = "> SELECT max(expr);", + returnType = Type.FLOAT8, + paramTypes = {@ParamTypes(paramTypes = {Type.FLOAT8})} ) -public class MaxDouble extends AggFunction { - +public class MaxDouble extends Max { public MaxDouble() { super(new Column[] { new Column("expr", Type.FLOAT8) }); } - @Override - public FunctionContext newContext() { - return new MaxContext(); - } - - @Override - public void eval(FunctionContext ctx, Tuple params) { - MaxContext maxCtx = (MaxContext) ctx; - maxCtx.max = Math.max(maxCtx.max, params.get(0).asFloat8()); - } - - @Override - public Datum getPartialResult(FunctionContext ctx) { - return DatumFactory.createFloat8(((MaxContext) ctx).max); - } - @Override public DataType getPartialResultType() { return CatalogUtil.newSimpleDataType(Type.FLOAT8); } - @Override - public Float8Datum terminate(FunctionContext ctx) { - return DatumFactory.createFloat8(((MaxContext) ctx).max); - } - - private class MaxContext implements FunctionContext { - double max; - } } diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/function/builtin/MaxFloat.java b/tajo-core/src/main/java/org/apache/tajo/engine/function/builtin/MaxFloat.java index 6b725aff6e..85fa855678 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/function/builtin/MaxFloat.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/function/builtin/MaxFloat.java @@ -22,55 +22,26 @@ import org.apache.tajo.catalog.Column; import org.apache.tajo.common.TajoDataTypes.DataType; import org.apache.tajo.common.TajoDataTypes.Type; -import org.apache.tajo.datum.Datum; -import org.apache.tajo.datum.DatumFactory; -import org.apache.tajo.plan.function.AggFunction; -import org.apache.tajo.plan.function.FunctionContext; import org.apache.tajo.engine.function.annotation.Description; import org.apache.tajo.engine.function.annotation.ParamTypes; -import org.apache.tajo.storage.Tuple; @Description( - functionName = "max", - description = "the maximum value of expr", - example = "> SELECT max(expr);", - returnType = Type.FLOAT4, - paramTypes = {@ParamTypes(paramTypes = {Type.FLOAT4})} + functionName = "max", + description = "the maximum value of expr", + example = "> SELECT max(expr);", + returnType = Type.FLOAT4, + paramTypes = {@ParamTypes(paramTypes = {Type.FLOAT4})} ) -public class MaxFloat extends AggFunction { +public class MaxFloat extends Max { public MaxFloat() { super(new Column[] { - new Column("expr", Type.FLOAT8) + new Column("expr", Type.FLOAT4) }); } - @Override - public FunctionContext newContext() { - return new MaxContext(); - } - - @Override - public void eval(FunctionContext ctx, Tuple params) { - MaxContext maxCtx = (MaxContext) ctx; - maxCtx.max = Math.max(maxCtx.max, params.get(0).asFloat4()); - } - - @Override - public Datum getPartialResult(FunctionContext ctx) { - return DatumFactory.createFloat4(((MaxContext) ctx).max); - } - @Override public DataType getPartialResultType() { return CatalogUtil.newSimpleDataType(Type.FLOAT4); } - @Override - public Datum terminate(FunctionContext ctx) { - return DatumFactory.createFloat4(((MaxContext) ctx).max); - } - - private class MaxContext implements FunctionContext { - float max; - } } diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/function/builtin/MaxInt.java b/tajo-core/src/main/java/org/apache/tajo/engine/function/builtin/MaxInt.java index 2d9cf481ec..d879f876d4 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/function/builtin/MaxInt.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/function/builtin/MaxInt.java @@ -22,56 +22,26 @@ import org.apache.tajo.catalog.Column; import org.apache.tajo.common.TajoDataTypes.DataType; import org.apache.tajo.common.TajoDataTypes.Type; -import org.apache.tajo.datum.Datum; -import org.apache.tajo.datum.DatumFactory; -import org.apache.tajo.plan.function.AggFunction; -import org.apache.tajo.plan.function.FunctionContext; import org.apache.tajo.engine.function.annotation.Description; import org.apache.tajo.engine.function.annotation.ParamTypes; -import org.apache.tajo.storage.Tuple; @Description( - functionName = "max", - description = "the maximum value of expr", - example = "> SELECT max(expr);", - returnType = Type.INT4, - paramTypes = {@ParamTypes(paramTypes = {Type.INT4})} + functionName = "max", + description = "the maximum value of expr", + example = "> SELECT max(expr);", + returnType = Type.INT4, + paramTypes = {@ParamTypes(paramTypes = {Type.INT4})} ) -public class MaxInt extends AggFunction { - +public class MaxInt extends Max { public MaxInt() { super(new Column[] { - new Column("expr", Type.INT8) + new Column("expr", Type.INT4) }); } - @Override - public FunctionContext newContext() { - return new MaxContext(); - } - - @Override - public void eval(FunctionContext ctx, Tuple params) { - MaxContext maxCtx = (MaxContext) ctx; - maxCtx.max = Math.max(maxCtx.max, params.get(0).asInt4()); - } - - @Override - public Datum getPartialResult(FunctionContext ctx) { - return DatumFactory.createInt4(((MaxContext) ctx).max); - } - @Override public DataType getPartialResultType() { return CatalogUtil.newSimpleDataType(Type.INT4); } - @Override - public Datum terminate(FunctionContext ctx) { - return DatumFactory.createInt4(((MaxContext) ctx).max); - } - - private class MaxContext implements FunctionContext { - int max; - } } diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/function/builtin/MaxLong.java b/tajo-core/src/main/java/org/apache/tajo/engine/function/builtin/MaxLong.java index b1cc30b4d8..904eca5dc2 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/function/builtin/MaxLong.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/function/builtin/MaxLong.java @@ -22,14 +22,8 @@ import org.apache.tajo.catalog.Column; import org.apache.tajo.common.TajoDataTypes.DataType; import org.apache.tajo.common.TajoDataTypes.Type; -import org.apache.tajo.datum.Datum; -import org.apache.tajo.datum.DatumFactory; -import org.apache.tajo.datum.Int8Datum; -import org.apache.tajo.plan.function.AggFunction; -import org.apache.tajo.plan.function.FunctionContext; import org.apache.tajo.engine.function.annotation.Description; import org.apache.tajo.engine.function.annotation.ParamTypes; -import org.apache.tajo.storage.Tuple; @Description( functionName = "max", @@ -38,40 +32,16 @@ returnType = Type.INT8, paramTypes = {@ParamTypes(paramTypes = {Type.INT8})} ) -public class MaxLong extends AggFunction { +public class MaxLong extends Max { public MaxLong() { super(new Column[] { new Column("expr", Type.INT8) }); } - @Override - public FunctionContext newContext() { - return new MaxContext(); - } - - @Override - public void eval(FunctionContext ctx, Tuple params) { - MaxContext maxCtx = (MaxContext) ctx; - maxCtx.max = Math.max(maxCtx.max, params.get(0).asInt8()); - } - - @Override - public Datum getPartialResult(FunctionContext ctx) { - return DatumFactory.createInt8(((MaxContext) ctx).max); - } - @Override public DataType getPartialResultType() { return CatalogUtil.newSimpleDataType(Type.INT8); } - @Override - public Int8Datum terminate(FunctionContext ctx) { - return DatumFactory.createInt8(((MaxContext) ctx).max); - } - - private class MaxContext implements FunctionContext { - long max; - } } diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/function/builtin/MaxString.java b/tajo-core/src/main/java/org/apache/tajo/engine/function/builtin/MaxString.java index d3c67ce5ff..239c8b6579 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/function/builtin/MaxString.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/function/builtin/MaxString.java @@ -20,62 +20,28 @@ import org.apache.tajo.catalog.CatalogUtil; import org.apache.tajo.catalog.Column; -import org.apache.tajo.common.TajoDataTypes; -import org.apache.tajo.datum.Datum; -import org.apache.tajo.datum.DatumFactory; -import org.apache.tajo.datum.TextDatum; -import org.apache.tajo.plan.function.AggFunction; -import org.apache.tajo.plan.function.FunctionContext; +import org.apache.tajo.common.TajoDataTypes.DataType; +import org.apache.tajo.common.TajoDataTypes.Type; import org.apache.tajo.engine.function.annotation.Description; import org.apache.tajo.engine.function.annotation.ParamTypes; -import org.apache.tajo.storage.Tuple; @Description( functionName = "max", description = "the maximum value of expr", example = "> SELECT max(expr);", - returnType = TajoDataTypes.Type.TEXT, - paramTypes = {@ParamTypes(paramTypes = {TajoDataTypes.Type.TEXT})} + returnType = Type.TEXT, + paramTypes = {@ParamTypes(paramTypes = {Type.TEXT})} ) -public class MaxString extends AggFunction { - +public class MaxString extends Max { public MaxString() { super(new Column[] { - new Column("expr", TajoDataTypes.Type.TEXT) + new Column("expr", Type.TEXT) }); } @Override - public FunctionContext newContext() { - return new MaxContext(); - } - - @Override - public void eval(FunctionContext ctx, Tuple params) { - MaxContext maxCtx = (MaxContext) ctx; - if (maxCtx.max == null) { - maxCtx.max = params.get(0).asChars(); - } else if (params.get(0).asChars().compareTo(maxCtx.max) > 0) { - maxCtx.max = params.get(0).asChars(); - } - } - - @Override - public Datum getPartialResult(FunctionContext ctx) { - return DatumFactory.createText(((MaxContext) ctx).max); + public DataType getPartialResultType() { + return CatalogUtil.newSimpleDataType(Type.TEXT); } - @Override - public TajoDataTypes.DataType getPartialResultType() { - return CatalogUtil.newSimpleDataType(TajoDataTypes.Type.TEXT); - } - - @Override - public TextDatum terminate(FunctionContext ctx) { - return DatumFactory.createText(((MaxContext) ctx).max); - } - - private class MaxContext implements FunctionContext { - String max; - } -} \ No newline at end of file +} diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/function/builtin/Min.java b/tajo-core/src/main/java/org/apache/tajo/engine/function/builtin/Min.java new file mode 100644 index 0000000000..ac051e51c2 --- /dev/null +++ b/tajo-core/src/main/java/org/apache/tajo/engine/function/builtin/Min.java @@ -0,0 +1,69 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tajo.engine.function.builtin; + +import org.apache.tajo.catalog.Column; +import org.apache.tajo.datum.Datum; +import org.apache.tajo.datum.NullDatum; +import org.apache.tajo.plan.function.AggFunction; +import org.apache.tajo.plan.function.FunctionContext; +import org.apache.tajo.storage.Tuple; + +public abstract class Min extends AggFunction { + public Min(Column[] definedArgs) { + super(definedArgs); + } + + @Override + public FunctionContext newContext() { + return new MinContext(); + } + + @Override + public void eval(FunctionContext ctx, Tuple params) { + MinContext minCtx = (MinContext) ctx; + Datum datum = params.get(0); + if (datum.isNotNull()) { + if (minCtx.min == null || minCtx.min.compareTo(datum) > 0) { + minCtx.min = datum; + } + } + } + + @Override + public Datum getPartialResult(FunctionContext ctx) { + return ((MinContext) ctx).min; + } + + @Override + public Datum terminate(FunctionContext ctx) { + Datum min = ((MinContext)ctx).min; + + if (min == null) { + return NullDatum.get(); + } + else { + return min; + } + } + + private class MinContext implements FunctionContext { + Datum min = null; + } +} diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/function/builtin/MinDouble.java b/tajo-core/src/main/java/org/apache/tajo/engine/function/builtin/MinDouble.java index 38eba48159..8999f0a6af 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/function/builtin/MinDouble.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/function/builtin/MinDouble.java @@ -22,22 +22,17 @@ import org.apache.tajo.catalog.Column; import org.apache.tajo.common.TajoDataTypes.DataType; import org.apache.tajo.common.TajoDataTypes.Type; -import org.apache.tajo.datum.Datum; -import org.apache.tajo.datum.DatumFactory; -import org.apache.tajo.plan.function.AggFunction; -import org.apache.tajo.plan.function.FunctionContext; import org.apache.tajo.engine.function.annotation.Description; import org.apache.tajo.engine.function.annotation.ParamTypes; -import org.apache.tajo.storage.Tuple; @Description( - functionName = "min", - description = "the minimum value of expr", - example = "> SELECT min(expr);", - returnType = Type.FLOAT8, - paramTypes = {@ParamTypes(paramTypes = {Type.FLOAT8})} + functionName = "min", + description = "the minimum value of expr", + example = "> SELECT min(expr);", + returnType = Type.FLOAT8, + paramTypes = {@ParamTypes(paramTypes = {Type.FLOAT8})} ) -public class MinDouble extends AggFunction { +public class MinDouble extends Min { public MinDouble() { super(new Column[] { @@ -45,33 +40,9 @@ public MinDouble() { }); } - @Override - public FunctionContext newContext() { - return new MinContext(); - } - - @Override - public void eval(FunctionContext ctx, Tuple params) { - MinContext minCtx = (MinContext) ctx; - minCtx.min = Math.min(minCtx.min, params.get(0).asFloat8()); - } - - @Override - public Datum getPartialResult(FunctionContext ctx) { - return DatumFactory.createFloat8(((MinContext) ctx).min); - } - @Override public DataType getPartialResultType() { return CatalogUtil.newSimpleDataType(Type.FLOAT8); } - @Override - public Datum terminate(FunctionContext ctx) { - return DatumFactory.createFloat8(((MinContext) ctx).min); - } - - private class MinContext implements FunctionContext { - double min = Double.MAX_VALUE; - } } diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/function/builtin/MinFloat.java b/tajo-core/src/main/java/org/apache/tajo/engine/function/builtin/MinFloat.java index 17e63e244c..e064f28b4f 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/function/builtin/MinFloat.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/function/builtin/MinFloat.java @@ -22,23 +22,17 @@ import org.apache.tajo.catalog.Column; import org.apache.tajo.common.TajoDataTypes.DataType; import org.apache.tajo.common.TajoDataTypes.Type; -import org.apache.tajo.datum.Datum; -import org.apache.tajo.datum.DatumFactory; -import org.apache.tajo.datum.Float4Datum; -import org.apache.tajo.plan.function.AggFunction; -import org.apache.tajo.plan.function.FunctionContext; import org.apache.tajo.engine.function.annotation.Description; import org.apache.tajo.engine.function.annotation.ParamTypes; -import org.apache.tajo.storage.Tuple; @Description( - functionName = "min", - description = "the minimum value of expr", - example = "> SELECT min(expr);", - returnType = Type.FLOAT4, - paramTypes = {@ParamTypes(paramTypes = {Type.FLOAT4})} + functionName = "min", + description = "the minimum value of expr", + example = "> SELECT min(expr);", + returnType = Type.FLOAT4, + paramTypes = {@ParamTypes(paramTypes = {Type.FLOAT4})} ) -public class MinFloat extends AggFunction { +public class MinFloat extends Min { public MinFloat() { super(new Column[] { @@ -46,33 +40,9 @@ public MinFloat() { }); } - @Override - public FunctionContext newContext() { - return new MinContext(); - } - - @Override - public void eval(FunctionContext ctx, Tuple params) { - MinContext minCtx = (MinContext) ctx; - minCtx.min = Math.min(minCtx.min, params.get(0).asFloat4()); - } - - @Override - public Datum getPartialResult(FunctionContext ctx) { - return DatumFactory.createFloat4(((MinContext) ctx).min); - } - @Override public DataType getPartialResultType() { return CatalogUtil.newSimpleDataType(Type.FLOAT4); } - @Override - public Float4Datum terminate(FunctionContext ctx) { - return DatumFactory.createFloat4(((MinContext) ctx).min); - } - - private class MinContext implements FunctionContext { - float min = Float.MAX_VALUE; - } } diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/function/builtin/MinInt.java b/tajo-core/src/main/java/org/apache/tajo/engine/function/builtin/MinInt.java index d2a5a2d74d..4b6630997e 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/function/builtin/MinInt.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/function/builtin/MinInt.java @@ -22,56 +22,27 @@ import org.apache.tajo.catalog.Column; import org.apache.tajo.common.TajoDataTypes.DataType; import org.apache.tajo.common.TajoDataTypes.Type; -import org.apache.tajo.datum.Datum; -import org.apache.tajo.datum.DatumFactory; -import org.apache.tajo.plan.function.AggFunction; -import org.apache.tajo.plan.function.FunctionContext; import org.apache.tajo.engine.function.annotation.Description; import org.apache.tajo.engine.function.annotation.ParamTypes; -import org.apache.tajo.storage.Tuple; @Description( - functionName = "min", - description = "the minimum value of expr", - example = "> SELECT min(expr);", - returnType = Type.INT4, - paramTypes = {@ParamTypes(paramTypes = {Type.INT4})} + functionName = "min", + description = "the minimum value of expr", + example = "> SELECT min(expr);", + returnType = Type.INT4, + paramTypes = {@ParamTypes(paramTypes = {Type.INT4})} ) -public class MinInt extends AggFunction { +public class MinInt extends Min { public MinInt() { super(new Column[] { - new Column("expr", Type.INT8) + new Column("expr", Type.INT4) }); } - @Override - public FunctionContext newContext() { - return new MinContext(); - } - - @Override - public void eval(FunctionContext ctx, Tuple params) { - MinContext minCtx = (MinContext) ctx; - minCtx.min = Math.min(minCtx.min, params.get(0).asInt4()); - } - - @Override - public Datum getPartialResult(FunctionContext ctx) { - return DatumFactory.createInt4(((MinContext) ctx).min); - } - @Override public DataType getPartialResultType() { return CatalogUtil.newSimpleDataType(Type.INT4); } - @Override - public Datum terminate(FunctionContext ctx) { - return DatumFactory.createInt4(((MinContext) ctx).min); - } - - private class MinContext implements FunctionContext { - int min = Integer.MAX_VALUE; - } } diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/function/builtin/MinLong.java b/tajo-core/src/main/java/org/apache/tajo/engine/function/builtin/MinLong.java index e346ac7b15..5861037371 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/function/builtin/MinLong.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/function/builtin/MinLong.java @@ -22,14 +22,8 @@ import org.apache.tajo.catalog.Column; import org.apache.tajo.common.TajoDataTypes.DataType; import org.apache.tajo.common.TajoDataTypes.Type; -import org.apache.tajo.datum.Datum; -import org.apache.tajo.datum.DatumFactory; -import org.apache.tajo.datum.Int8Datum; -import org.apache.tajo.plan.function.AggFunction; -import org.apache.tajo.plan.function.FunctionContext; import org.apache.tajo.engine.function.annotation.Description; import org.apache.tajo.engine.function.annotation.ParamTypes; -import org.apache.tajo.storage.Tuple; @Description( functionName = "min", @@ -38,7 +32,7 @@ returnType = Type.INT8, paramTypes = {@ParamTypes(paramTypes = {Type.INT8})} ) -public class MinLong extends AggFunction { +public class MinLong extends Min { public MinLong() { super(new Column[] { @@ -46,33 +40,9 @@ public MinLong() { }); } - @Override - public FunctionContext newContext() { - return new MinContext(); - } - - @Override - public void eval(FunctionContext ctx, Tuple params) { - MinContext minCtx = (MinContext)ctx; - minCtx.min = Math.min(minCtx.min, params.get(0).asInt8()); - } - - @Override - public Datum getPartialResult(FunctionContext ctx) { - return DatumFactory.createInt8(((MinContext) ctx).min); - } - @Override public DataType getPartialResultType() { return CatalogUtil.newSimpleDataType(Type.INT8); } - @Override - public Int8Datum terminate(FunctionContext ctx) { - return DatumFactory.createInt8(((MinContext) ctx).min); - } - - private class MinContext implements FunctionContext { - long min = Long.MAX_VALUE; - } } diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/function/builtin/MinString.java b/tajo-core/src/main/java/org/apache/tajo/engine/function/builtin/MinString.java index 7e87a49044..3b8e6d5ffe 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/function/builtin/MinString.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/function/builtin/MinString.java @@ -22,23 +22,17 @@ import org.apache.tajo.catalog.Column; import org.apache.tajo.common.TajoDataTypes.DataType; import org.apache.tajo.common.TajoDataTypes.Type; -import org.apache.tajo.datum.Datum; -import org.apache.tajo.datum.DatumFactory; -import org.apache.tajo.datum.TextDatum; -import org.apache.tajo.plan.function.AggFunction; -import org.apache.tajo.plan.function.FunctionContext; import org.apache.tajo.engine.function.annotation.Description; import org.apache.tajo.engine.function.annotation.ParamTypes; -import org.apache.tajo.storage.Tuple; @Description( - functionName = "min", - description = "the minimum value of expr", - example = "> SELECT min(expr);", - returnType = Type.TEXT, - paramTypes = {@ParamTypes(paramTypes = {Type.TEXT})} + functionName = "min", + description = "the minimum value of expr", + example = "> SELECT min(expr);", + returnType = Type.TEXT, + paramTypes = {@ParamTypes(paramTypes = {Type.TEXT})} ) -public class MinString extends AggFunction { +public class MinString extends Min { public MinString() { super(new Column[] { @@ -46,37 +40,9 @@ public MinString() { }); } - @Override - public FunctionContext newContext() { - return new MinContext(); - } - - @Override - public void eval(FunctionContext ctx, Tuple params) { - MinContext minCtx = (MinContext) ctx; - if (minCtx.min == null) { - minCtx.min = params.get(0).asChars(); - } else if (params.get(0).asChars().compareTo(minCtx.min) < 0) { - minCtx.min = params.get(0).asChars(); - } - } - - @Override - public Datum getPartialResult(FunctionContext ctx) { - return DatumFactory.createText(((MinContext) ctx).min); - } - @Override public DataType getPartialResultType() { return CatalogUtil.newSimpleDataType(Type.TEXT); } - @Override - public TextDatum terminate(FunctionContext ctx) { - return DatumFactory.createText(((MinContext) ctx).min); - } - - private class MinContext implements FunctionContext { - String min; - } } diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/function/TestBuiltinFunctions.java b/tajo-core/src/test/java/org/apache/tajo/engine/function/TestBuiltinFunctions.java index 4fefe07a1f..a408400bfa 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/function/TestBuiltinFunctions.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/function/TestBuiltinFunctions.java @@ -21,11 +21,18 @@ import org.apache.tajo.IntegrationTest; import org.apache.tajo.QueryTestCaseBase; import org.apache.tajo.TajoConstants; +import org.apache.tajo.TajoTestingCluster; +import org.apache.tajo.catalog.Schema; +import org.apache.tajo.common.TajoDataTypes; +import org.apache.tajo.storage.StorageConstants; +import org.apache.tajo.util.KeyValueSet; import org.junit.Test; import org.junit.experimental.categories.Category; import java.sql.ResultSet; +import static org.junit.Assert.assertEquals; + @Category(IntegrationTest.class) public class TestBuiltinFunctions extends QueryTestCaseBase { @@ -40,6 +47,32 @@ public void testMaxLong() throws Exception { cleanupQuery(res); } + @Test + public void testMaxLongWithNull() throws Exception { + KeyValueSet tableOptions = new KeyValueSet(); + tableOptions.set(StorageConstants.TEXT_DELIMITER, StorageConstants.DEFAULT_FIELD_DELIMITER); + tableOptions.set(StorageConstants.TEXT_NULL, "\\\\N"); + + Schema schema = new Schema(); + schema.addColumn("id", TajoDataTypes.Type.INT4); + schema.addColumn("value", TajoDataTypes.Type.INT8); + String[] data = new String[]{ "1|-111", "2|\\N", "3|-333" }; + TajoTestingCluster.createTable("table11", schema, tableOptions, data, 1); + + try { + ResultSet res = executeString("select max(value) as max_value from table11"); + String ascExpected = "max_value\n" + + "-------------------------------\n" + + "-111\n"; + + assertEquals(ascExpected, resultSetToString(res)); + res.close(); + } finally { + executeString("DROP TABLE table11 PURGE"); + } + + } + @Test public void testMinLong() throws Exception { ResultSet res = executeQuery(); @@ -47,6 +80,32 @@ public void testMinLong() throws Exception { cleanupQuery(res); } + @Test + public void testMinLongWithNull() throws Exception { + KeyValueSet tableOptions = new KeyValueSet(); + tableOptions.set(StorageConstants.TEXT_DELIMITER, StorageConstants.DEFAULT_FIELD_DELIMITER); + tableOptions.set(StorageConstants.TEXT_NULL, "\\\\N"); + + Schema schema = new Schema(); + schema.addColumn("id", TajoDataTypes.Type.INT4); + schema.addColumn("value", TajoDataTypes.Type.INT8); + String[] data = new String[]{ "1|111", "2|\\N", "3|333" }; + TajoTestingCluster.createTable("table11", schema, tableOptions, data, 1); + + try { + ResultSet res = executeString("select min(value) as min_value from table11"); + String ascExpected = "min_value\n" + + "-------------------------------\n" + + "111\n"; + + assertEquals(ascExpected, resultSetToString(res)); + res.close(); + } finally { + executeString("DROP TABLE table11 PURGE"); + } + + } + @Test public void testMaxString() throws Exception { ResultSet res = executeQuery(); @@ -54,6 +113,32 @@ public void testMaxString() throws Exception { cleanupQuery(res); } + @Test + public void testMaxStringWithNull() throws Exception { + KeyValueSet tableOptions = new KeyValueSet(); + tableOptions.set(StorageConstants.TEXT_DELIMITER, StorageConstants.DEFAULT_FIELD_DELIMITER); + tableOptions.set(StorageConstants.TEXT_NULL, "\\\\N"); + + Schema schema = new Schema(); + schema.addColumn("id", TajoDataTypes.Type.INT4); + schema.addColumn("name", TajoDataTypes.Type.TEXT); + String[] data = new String[]{ "1|\\N", "2|\\N", "3|\\N" }; + TajoTestingCluster.createTable("table11", schema, tableOptions, data, 1); + + try { + ResultSet res = executeString("select max(name) as max_name from table11"); + String ascExpected = "max_name\n" + + "-------------------------------\n" + + "null\n"; + + assertEquals(ascExpected, resultSetToString(res)); + res.close(); + } finally { + executeString("DROP TABLE table11 PURGE"); + } + + } + @Test public void testMinString() throws Exception { ResultSet res = executeQuery(); @@ -61,6 +146,32 @@ public void testMinString() throws Exception { cleanupQuery(res); } + @Test + public void testMinStringWithNull() throws Exception { + KeyValueSet tableOptions = new KeyValueSet(); + tableOptions.set(StorageConstants.TEXT_DELIMITER, StorageConstants.DEFAULT_FIELD_DELIMITER); + tableOptions.set(StorageConstants.TEXT_NULL, "\\\\N"); + + Schema schema = new Schema(); + schema.addColumn("id", TajoDataTypes.Type.INT4); + schema.addColumn("name", TajoDataTypes.Type.TEXT); + String[] data = new String[]{ "1|def", "2|\\N", "3|abc" }; + TajoTestingCluster.createTable("table11", schema, tableOptions, data, 1); + + try { + ResultSet res = executeString("select min(name) as min_name from table11"); + String ascExpected = "min_name\n" + + "-------------------------------\n" + + "abc\n"; + + assertEquals(ascExpected, resultSetToString(res)); + res.close(); + } finally { + executeString("DROP TABLE table11 PURGE"); + } + + } + @Test public void testCount() throws Exception { ResultSet res = executeQuery(); From 52f6f2a8eed78f5154ebce58eb85dc254125fcf4 Mon Sep 17 00:00:00 2001 From: sirpkt Date: Tue, 23 Dec 2014 09:51:47 +0900 Subject: [PATCH 2/6] TAJO-1265: min(), max() does not handle null properly --- .../results/TestGroupByQuery/testGroupByWithNullData2.result | 2 +- .../results/TestGroupByQuery/testGroupByWithNullData3.result | 2 +- .../results/TestGroupByQuery/testGroupByWithNullData4.result | 2 +- .../results/TestGroupByQuery/testGroupByWithNullData6.result | 2 +- .../results/TestGroupByQuery/testGroupByWithNullData7.result | 2 +- .../results/TestGroupByQuery/testGroupByWithNullData8.result | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/tajo-core/src/test/resources/results/TestGroupByQuery/testGroupByWithNullData2.result b/tajo-core/src/test/resources/results/TestGroupByQuery/testGroupByWithNullData2.result index f4f9a5b7c7..8746e36e87 100644 --- a/tajo-core/src/test/resources/results/TestGroupByQuery/testGroupByWithNullData2.result +++ b/tajo-core/src/test/resources/results/TestGroupByQuery/testGroupByWithNullData2.result @@ -1,3 +1,3 @@ unique_key,max_key ------------------------------- -0,0 \ No newline at end of file +0,null \ No newline at end of file diff --git a/tajo-core/src/test/resources/results/TestGroupByQuery/testGroupByWithNullData3.result b/tajo-core/src/test/resources/results/TestGroupByQuery/testGroupByWithNullData3.result index fef3d0cbd9..c2cc22e273 100644 --- a/tajo-core/src/test/resources/results/TestGroupByQuery/testGroupByWithNullData3.result +++ b/tajo-core/src/test/resources/results/TestGroupByQuery/testGroupByWithNullData3.result @@ -1,3 +1,3 @@ maximum,unique_key ------------------------------- -0,0 \ No newline at end of file +null,0 \ No newline at end of file diff --git a/tajo-core/src/test/resources/results/TestGroupByQuery/testGroupByWithNullData4.result b/tajo-core/src/test/resources/results/TestGroupByQuery/testGroupByWithNullData4.result index fef3d0cbd9..c2cc22e273 100644 --- a/tajo-core/src/test/resources/results/TestGroupByQuery/testGroupByWithNullData4.result +++ b/tajo-core/src/test/resources/results/TestGroupByQuery/testGroupByWithNullData4.result @@ -1,3 +1,3 @@ maximum,unique_key ------------------------------- -0,0 \ No newline at end of file +null,0 \ No newline at end of file diff --git a/tajo-core/src/test/resources/results/TestGroupByQuery/testGroupByWithNullData6.result b/tajo-core/src/test/resources/results/TestGroupByQuery/testGroupByWithNullData6.result index 58aaa20ad0..4014f6d913 100644 --- a/tajo-core/src/test/resources/results/TestGroupByQuery/testGroupByWithNullData6.result +++ b/tajo-core/src/test/resources/results/TestGroupByQuery/testGroupByWithNullData6.result @@ -1,3 +1,3 @@ unique_key,maximum ------------------------------- -0,0.0 \ No newline at end of file +0,null \ No newline at end of file diff --git a/tajo-core/src/test/resources/results/TestGroupByQuery/testGroupByWithNullData7.result b/tajo-core/src/test/resources/results/TestGroupByQuery/testGroupByWithNullData7.result index e19a62303e..c2cc22e273 100644 --- a/tajo-core/src/test/resources/results/TestGroupByQuery/testGroupByWithNullData7.result +++ b/tajo-core/src/test/resources/results/TestGroupByQuery/testGroupByWithNullData7.result @@ -1,3 +1,3 @@ maximum,unique_key ------------------------------- -0.0,0 \ No newline at end of file +null,0 \ No newline at end of file diff --git a/tajo-core/src/test/resources/results/TestGroupByQuery/testGroupByWithNullData8.result b/tajo-core/src/test/resources/results/TestGroupByQuery/testGroupByWithNullData8.result index e19a62303e..c2cc22e273 100644 --- a/tajo-core/src/test/resources/results/TestGroupByQuery/testGroupByWithNullData8.result +++ b/tajo-core/src/test/resources/results/TestGroupByQuery/testGroupByWithNullData8.result @@ -1,3 +1,3 @@ maximum,unique_key ------------------------------- -0.0,0 \ No newline at end of file +null,0 \ No newline at end of file From 96946489eaadb5418b2bcdb60b2f7c8f5b52588e Mon Sep 17 00:00:00 2001 From: sirpkt Date: Tue, 23 Dec 2014 11:32:40 +0900 Subject: [PATCH 3/6] TAJO-1265: fix bug in partial result return --- .../org/apache/tajo/engine/function/builtin/Max.java | 9 ++++++++- .../org/apache/tajo/engine/function/builtin/Min.java | 9 ++++++++- .../org/apache/tajo/engine/query/TestGroupByQuery.java | 4 ++-- .../testLeftOuterJoinWithEmptyTable2.result | 10 +++++----- .../testLeftOuterJoinWithEmptyTable4.result | 2 +- .../testLeftOuterJoinWithEmptyTable5.result | 4 ++-- 6 files changed, 26 insertions(+), 12 deletions(-) diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/function/builtin/Max.java b/tajo-core/src/main/java/org/apache/tajo/engine/function/builtin/Max.java index 141df6cc90..f8519941aa 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/function/builtin/Max.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/function/builtin/Max.java @@ -48,7 +48,14 @@ public void eval(FunctionContext ctx, Tuple params) { @Override public Datum getPartialResult(FunctionContext ctx) { - return ((MaxContext) ctx).max; + Datum max = ((MaxContext)ctx).max; + + if (max == null) { + return NullDatum.get(); + } + else { + return max; + } } @Override diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/function/builtin/Min.java b/tajo-core/src/main/java/org/apache/tajo/engine/function/builtin/Min.java index ac051e51c2..2695b6adcc 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/function/builtin/Min.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/function/builtin/Min.java @@ -48,7 +48,14 @@ public void eval(FunctionContext ctx, Tuple params) { @Override public Datum getPartialResult(FunctionContext ctx) { - return ((MinContext) ctx).min; + Datum min = ((MinContext)ctx).min; + + if (min == null) { + return NullDatum.get(); + } + else { + return min; + } } @Override diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestGroupByQuery.java b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestGroupByQuery.java index edbe029071..584df9325b 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestGroupByQuery.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestGroupByQuery.java @@ -412,8 +412,8 @@ public final void testDistinctAggregationCasebyCase() throws Exception { String expected = "id,?count_4,?avg_5,?min_6,?max_7,?sum_8,?cast_9,?cast_10,?cast_11,?cast_12\n" + "-------------------------------\n" + - "1,2,4.0,0,5,12,4,0,5,12\n" + - "2,3,2.0,0,3,6,7,0,8,21\n"; + "1,2,4.0,3,5,12,4,3,5,12\n" + + "2,3,2.0,1,3,6,7,6,8,21\n"; assertEquals(expected, resultSetToString(res)); diff --git a/tajo-core/src/test/resources/results/TestJoinQuery/testLeftOuterJoinWithEmptyTable2.result b/tajo-core/src/test/resources/results/TestJoinQuery/testLeftOuterJoinWithEmptyTable2.result index 3bf0ed2478..3e8a623f8e 100644 --- a/tajo-core/src/test/resources/results/TestJoinQuery/testLeftOuterJoinWithEmptyTable2.result +++ b/tajo-core/src/test/resources/results/TestJoinQuery/testLeftOuterJoinWithEmptyTable2.result @@ -1,7 +1,7 @@ c_custkey,?sum,?max_1,?max_2 ------------------------------- -1,0,, -2,0,, -3,0,, -4,0,, -5,0,, \ No newline at end of file +1,0,null,null +2,0,null,null +3,0,null,null +4,0,null,null +5,0,null,null \ No newline at end of file diff --git a/tajo-core/src/test/resources/results/TestJoinQuery/testLeftOuterJoinWithEmptyTable4.result b/tajo-core/src/test/resources/results/TestJoinQuery/testLeftOuterJoinWithEmptyTable4.result index b9ac208f04..0acebad0bc 100644 --- a/tajo-core/src/test/resources/results/TestJoinQuery/testLeftOuterJoinWithEmptyTable4.result +++ b/tajo-core/src/test/resources/results/TestJoinQuery/testLeftOuterJoinWithEmptyTable4.result @@ -1,4 +1,4 @@ ?max,?sum_1,?max_2,?max_3 ------------------------------- 5,6,O,1996-12-01 -5,0,, \ No newline at end of file +5,0,null,null \ No newline at end of file diff --git a/tajo-core/src/test/resources/results/TestJoinQuery/testLeftOuterJoinWithEmptyTable5.result b/tajo-core/src/test/resources/results/TestJoinQuery/testLeftOuterJoinWithEmptyTable5.result index f1d80e4e85..18462e9d23 100644 --- a/tajo-core/src/test/resources/results/TestJoinQuery/testLeftOuterJoinWithEmptyTable5.result +++ b/tajo-core/src/test/resources/results/TestJoinQuery/testLeftOuterJoinWithEmptyTable5.result @@ -1,4 +1,4 @@ l_linenumber,?sum,?max_1,?max_2,?avg_3,?sum_4 ------------------------------- -1,0,,,33.333333333333336,100.0 -2,0,,,42.5,85.0 \ No newline at end of file +1,0,null,null,33.333333333333336,100.0 +2,0,null,null,42.5,85.0 \ No newline at end of file From 0be8ce89847b663edfcb644e27a2012c61d28345 Mon Sep 17 00:00:00 2001 From: sirpkt Date: Wed, 24 Dec 2014 12:50:06 +0900 Subject: [PATCH 4/6] TAJO-1265: min(), max() does not handle null properly --- .../DistinctGroupbySortAggregationExec.java | 41 ++++++++++++------- .../testLeftOuterJoinWithEmptyTable2.result | 10 ++--- .../testLeftOuterJoinWithEmptyTable4.result | 2 +- 3 files changed, 32 insertions(+), 21 deletions(-) diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/DistinctGroupbySortAggregationExec.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/DistinctGroupbySortAggregationExec.java index 06b241c734..e8d35e427e 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/DistinctGroupbySortAggregationExec.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/DistinctGroupbySortAggregationExec.java @@ -22,6 +22,7 @@ import org.apache.tajo.common.TajoDataTypes; import org.apache.tajo.datum.DatumFactory; import org.apache.tajo.datum.NullDatum; +import org.apache.tajo.plan.expr.AggregationFunctionCallEval; import org.apache.tajo.plan.logical.DistinctGroupbyNode; import org.apache.tajo.plan.logical.GroupbyNode; import org.apache.tajo.storage.Tuple; @@ -127,23 +128,33 @@ public Tuple next() throws IOException { } private Tuple getEmptyTuple() { - Tuple tuple = new VTuple(outColumnNum); + Tuple tuple = new VTuple(outSchema.size()); NullDatum nullDatum = DatumFactory.createNullDatum(); - for (int i = 0; i < outColumnNum; i++) { - TajoDataTypes.Type type = outSchema.getColumn(i).getDataType().getType(); - if (type == TajoDataTypes.Type.INT8) { - tuple.put(i, DatumFactory.createInt8(nullDatum.asInt8())); - } else if (type == TajoDataTypes.Type.INT4) { - tuple.put(i, DatumFactory.createInt4(nullDatum.asInt4())); - } else if (type == TajoDataTypes.Type.INT2) { - tuple.put(i, DatumFactory.createInt2(nullDatum.asInt2())); - } else if (type == TajoDataTypes.Type.FLOAT4) { - tuple.put(i, DatumFactory.createFloat4(nullDatum.asFloat4())); - } else if (type == TajoDataTypes.Type.FLOAT8) { - tuple.put(i, DatumFactory.createFloat8(nullDatum.asFloat8())); - } else { - tuple.put(i, DatumFactory.createNullDatum()); + int tupleIndex = 0; + for (SortAggregateExec aggExec: aggregateExecs) { + for (int i = 0; i < aggExec.aggFunctionsNum; i++, tupleIndex++) { + String funcName = aggExec.aggFunctions[i].getName(); + if ("min".equals(funcName) || "max".equals(funcName)) { + tuple.put(resultColumnIdIndexes[tupleIndex], DatumFactory.createNullDatum()); + } + else + { + TajoDataTypes.Type type = outSchema.getColumn(resultColumnIdIndexes[tupleIndex]).getDataType().getType(); + if (type == TajoDataTypes.Type.INT8) { + tuple.put(resultColumnIdIndexes[tupleIndex], DatumFactory.createInt8(nullDatum.asInt8())); + } else if (type == TajoDataTypes.Type.INT4) { + tuple.put(resultColumnIdIndexes[tupleIndex], DatumFactory.createInt4(nullDatum.asInt4())); + } else if (type == TajoDataTypes.Type.INT2) { + tuple.put(resultColumnIdIndexes[tupleIndex], DatumFactory.createInt2(nullDatum.asInt2())); + } else if (type == TajoDataTypes.Type.FLOAT4) { + tuple.put(resultColumnIdIndexes[tupleIndex], DatumFactory.createFloat4(nullDatum.asFloat4())); + } else if (type == TajoDataTypes.Type.FLOAT8) { + tuple.put(resultColumnIdIndexes[tupleIndex], DatumFactory.createFloat8(nullDatum.asFloat8())); + } else { + tuple.put(resultColumnIdIndexes[tupleIndex], DatumFactory.createNullDatum()); + } + } } } diff --git a/tajo-core/src/test/resources/results/TestJoinBroadcast/testLeftOuterJoinWithEmptyTable2.result b/tajo-core/src/test/resources/results/TestJoinBroadcast/testLeftOuterJoinWithEmptyTable2.result index 3bf0ed2478..3e8a623f8e 100644 --- a/tajo-core/src/test/resources/results/TestJoinBroadcast/testLeftOuterJoinWithEmptyTable2.result +++ b/tajo-core/src/test/resources/results/TestJoinBroadcast/testLeftOuterJoinWithEmptyTable2.result @@ -1,7 +1,7 @@ c_custkey,?sum,?max_1,?max_2 ------------------------------- -1,0,, -2,0,, -3,0,, -4,0,, -5,0,, \ No newline at end of file +1,0,null,null +2,0,null,null +3,0,null,null +4,0,null,null +5,0,null,null \ No newline at end of file diff --git a/tajo-core/src/test/resources/results/TestJoinBroadcast/testLeftOuterJoinWithEmptyTable4.result b/tajo-core/src/test/resources/results/TestJoinBroadcast/testLeftOuterJoinWithEmptyTable4.result index b9ac208f04..0acebad0bc 100644 --- a/tajo-core/src/test/resources/results/TestJoinBroadcast/testLeftOuterJoinWithEmptyTable4.result +++ b/tajo-core/src/test/resources/results/TestJoinBroadcast/testLeftOuterJoinWithEmptyTable4.result @@ -1,4 +1,4 @@ ?max,?sum_1,?max_2,?max_3 ------------------------------- 5,6,O,1996-12-01 -5,0,, \ No newline at end of file +5,0,null,null \ No newline at end of file From d6be22bb92a50ab84cd3475cdbea88127f15d520 Mon Sep 17 00:00:00 2001 From: sirpkt Date: Mon, 5 Jan 2015 14:51:23 +0900 Subject: [PATCH 5/6] sum() and avg() are fixed to support null correctly. Test cases for sum() and avg() are modified also. --- .../engine/function/builtin/AvgDouble.java | 26 +++- .../engine/function/builtin/AvgFloat.java | 12 +- .../tajo/engine/function/builtin/AvgInt.java | 14 +- .../tajo/engine/function/builtin/AvgLong.java | 28 +++- .../engine/function/builtin/SumDouble.java | 33 ++++- .../engine/function/builtin/SumFloat.java | 33 +---- .../tajo/engine/function/builtin/SumInt.java | 32 +---- .../tajo/engine/function/builtin/SumLong.java | 34 ++++- .../DistinctGroupbySortAggregationExec.java | 2 +- .../engine/function/TestBuiltinFunctions.java | 123 ++++++++++++++++++ .../testAvgLongOverflow.sql | 1 + .../testAvgLongOverflow.result | 3 + .../testLeftOuterJoinWithEmptyTable2.result | 10 +- .../testLeftOuterJoinWithEmptyTable4.result | 2 +- .../testLeftOuterJoinWithEmptyTable2.result | 10 +- .../testLeftOuterJoinWithEmptyTable4.result | 2 +- .../testLeftOuterJoinWithEmptyTable5.result | 4 +- 17 files changed, 267 insertions(+), 102 deletions(-) create mode 100644 tajo-core/src/test/resources/queries/TestBuiltinFunctions/testAvgLongOverflow.sql create mode 100644 tajo-core/src/test/resources/results/TestBuiltinFunctions/testAvgLongOverflow.result diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/function/builtin/AvgDouble.java b/tajo-core/src/main/java/org/apache/tajo/engine/function/builtin/AvgDouble.java index f337c368c0..a74dcad417 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/function/builtin/AvgDouble.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/function/builtin/AvgDouble.java @@ -18,6 +18,7 @@ package org.apache.tajo.engine.function.builtin; +import org.apache.commons.lang.ObjectUtils; import org.apache.tajo.catalog.CatalogUtil; import org.apache.tajo.catalog.Column; import org.apache.tajo.common.TajoDataTypes.DataType; @@ -41,13 +42,17 @@ returnType = Type.FLOAT8, paramTypes = {@ParamTypes(paramTypes = {Type.FLOAT8})} ) -public class AvgDouble extends AggFunction { +public class AvgDouble extends AggFunction { public AvgDouble() { super(new Column[] { new Column("expr", Type.FLOAT8) }); } + public AvgDouble(Column[] definedArgs) { + super(definedArgs); + } + public AvgContext newContext() { return new AvgContext(); } @@ -55,8 +60,11 @@ public AvgContext newContext() { @Override public void eval(FunctionContext ctx, Tuple params) { AvgContext avgCtx = (AvgContext) ctx; - avgCtx.sum += params.get(0).asFloat8(); - avgCtx.count++; + Datum datum = params.get(0); + if (datum.isNotNull()) { + avgCtx.sum += datum.asFloat8(); + avgCtx.count++; + } } @Override @@ -75,6 +83,9 @@ public void merge(FunctionContext ctx, Tuple part) { @Override public Datum getPartialResult(FunctionContext ctx) { AvgContext avgCtx = (AvgContext) ctx; + if (avgCtx.count == 0) { + return NullDatum.get(); + } AvgDoubleProto.Builder builder = AvgDoubleProto.newBuilder(); builder.setSum(avgCtx.sum); builder.setCount(avgCtx.count); @@ -89,11 +100,14 @@ public DataType getPartialResultType() { @Override public Datum terminate(FunctionContext ctx) { AvgContext avgCtx = (AvgContext) ctx; + if (avgCtx.count == 0) { + return NullDatum.get(); + } return DatumFactory.createFloat8(avgCtx.sum / avgCtx.count); } protected class AvgContext implements FunctionContext { - double sum; - long count; + double sum = 0.0; + long count = 0; } -} +} \ No newline at end of file diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/function/builtin/AvgFloat.java b/tajo-core/src/main/java/org/apache/tajo/engine/function/builtin/AvgFloat.java index 2370421fe2..8162319459 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/function/builtin/AvgFloat.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/function/builtin/AvgFloat.java @@ -18,7 +18,9 @@ package org.apache.tajo.engine.function.builtin; +import org.apache.tajo.catalog.Column; import org.apache.tajo.common.TajoDataTypes; +import org.apache.tajo.datum.Datum; import org.apache.tajo.plan.function.FunctionContext; import org.apache.tajo.engine.function.annotation.Description; import org.apache.tajo.engine.function.annotation.ParamTypes; @@ -34,13 +36,19 @@ public class AvgFloat extends AvgDouble { public AvgFloat() { + super(new Column[] { + new Column("expr", TajoDataTypes.Type.FLOAT4) + }); } @Override public void eval(FunctionContext ctx, Tuple params) { AvgContext avgCtx = (AvgContext) ctx; - avgCtx.sum += params.get(0).asFloat4(); - avgCtx.count++; + Datum datum = params.get(0); + if (datum.isNotNull()) { + avgCtx.sum += datum.asFloat4(); + avgCtx.count++; + } } } diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/function/builtin/AvgInt.java b/tajo-core/src/main/java/org/apache/tajo/engine/function/builtin/AvgInt.java index 07cf37382e..1950fb1208 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/function/builtin/AvgInt.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/function/builtin/AvgInt.java @@ -18,7 +18,9 @@ package org.apache.tajo.engine.function.builtin; +import org.apache.tajo.catalog.Column; import org.apache.tajo.common.TajoDataTypes; +import org.apache.tajo.datum.Datum; import org.apache.tajo.plan.function.FunctionContext; import org.apache.tajo.engine.function.annotation.Description; import org.apache.tajo.engine.function.annotation.ParamTypes; @@ -34,13 +36,19 @@ public class AvgInt extends AvgLong { public AvgInt() { - super(); + super(new Column[] { + new Column("expr", TajoDataTypes.Type.INT4) + }); + } @Override public void eval(FunctionContext ctx, Tuple params) { AvgContext avgCtx = (AvgContext) ctx; - avgCtx.sum += params.get(0).asInt4(); - avgCtx.count++; + Datum datum = params.get(0); + if (datum.isNotNull()) { + avgCtx.sum += datum.asInt4(); + avgCtx.count++; + } } } diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/function/builtin/AvgLong.java b/tajo-core/src/main/java/org/apache/tajo/engine/function/builtin/AvgLong.java index f48de6c4a6..85950117cf 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/function/builtin/AvgLong.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/function/builtin/AvgLong.java @@ -18,6 +18,7 @@ package org.apache.tajo.engine.function.builtin; +import org.apache.commons.lang.ObjectUtils; import org.apache.tajo.catalog.CatalogUtil; import org.apache.tajo.catalog.Column; import org.apache.tajo.common.TajoDataTypes.DataType; @@ -38,14 +39,18 @@ returnType = Type.FLOAT8, paramTypes = {@ParamTypes(paramTypes = {Type.INT8})} ) -public class AvgLong extends AggFunction { +public class AvgLong extends AggFunction { public AvgLong() { super(new Column[] { - new Column("expr", Type.FLOAT8) + new Column("expr", Type.INT8) }); } + public AvgLong(Column[] definedArgs) { + super(definedArgs); + } + public AvgContext newContext() { return new AvgContext(); } @@ -53,8 +58,11 @@ public AvgContext newContext() { @Override public void eval(FunctionContext ctx, Tuple params) { AvgContext avgCtx = (AvgContext) ctx; - avgCtx.sum += params.get(0).asInt8(); - avgCtx.count++; + Datum datum = params.get(0); + if (datum.isNotNull()) { + avgCtx.sum += datum.asInt8(); + avgCtx.count++; + } } @Override @@ -73,6 +81,9 @@ public void merge(FunctionContext ctx, Tuple part) { @Override public Datum getPartialResult(FunctionContext ctx) { AvgContext avgCtx = (AvgContext) ctx; + if (avgCtx.count == 0) { + return NullDatum.get(); + } AvgLongProto.Builder builder = AvgLongProto.newBuilder(); builder.setSum(avgCtx.sum); builder.setCount(avgCtx.count); @@ -85,13 +96,16 @@ public DataType getPartialResultType() { } @Override - public Float8Datum terminate(FunctionContext ctx) { + public Datum terminate(FunctionContext ctx) { AvgContext avgCtx = (AvgContext) ctx; + if (avgCtx.count == 0) { + return NullDatum.get(); + } return DatumFactory.createFloat8((double) avgCtx.sum / avgCtx.count); } protected class AvgContext implements FunctionContext { - long sum; - long count; + long sum = 0; + long count = 0; } } diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/function/builtin/SumDouble.java b/tajo-core/src/main/java/org/apache/tajo/engine/function/builtin/SumDouble.java index a2da84d668..5bb5979d43 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/function/builtin/SumDouble.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/function/builtin/SumDouble.java @@ -25,6 +25,7 @@ import org.apache.tajo.datum.Datum; import org.apache.tajo.datum.DatumFactory; import org.apache.tajo.datum.Float8Datum; +import org.apache.tajo.datum.NullDatum; import org.apache.tajo.plan.function.AggFunction; import org.apache.tajo.plan.function.FunctionContext; import org.apache.tajo.engine.function.annotation.Description; @@ -51,6 +52,10 @@ public SumDouble() { }); } + public SumDouble(Column[] definedArgs) { + super(definedArgs); + } + @Override public FunctionContext newContext() { return new SumContext(); @@ -58,12 +63,22 @@ public FunctionContext newContext() { @Override public void eval(FunctionContext ctx, Tuple params) { - ((SumContext)ctx).sum += params.get(0).asFloat8(); + Datum datum = params.get(0); + if (datum.isNotNull()) { + SumContext sumCtx = (SumContext)ctx; + sumCtx.hasNonNull = true; + sumCtx.sum += datum.asFloat8(); + } } @Override public Datum getPartialResult(FunctionContext ctx) { - return DatumFactory.createFloat8(((SumContext) ctx).sum); + SumContext sumCtx = (SumContext)ctx; + if (sumCtx.hasNonNull) { + return DatumFactory.createFloat8(sumCtx.sum); + } else { + return NullDatum.get(); + } } @Override @@ -72,11 +87,17 @@ public DataType getPartialResultType() { } @Override - public Float8Datum terminate(FunctionContext ctx) { - return DatumFactory.createFloat8(((SumContext) ctx).sum); + public Datum terminate(FunctionContext ctx) { + SumContext sumCtx = (SumContext)ctx; + if (sumCtx.hasNonNull) { + return DatumFactory.createFloat8(sumCtx.sum); + } else { + return NullDatum.get(); + } } - private class SumContext implements FunctionContext { - double sum; + protected class SumContext implements FunctionContext { + boolean hasNonNull = false; + double sum = 0.0; } } diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/function/builtin/SumFloat.java b/tajo-core/src/main/java/org/apache/tajo/engine/function/builtin/SumFloat.java index 73257e2e2f..f63aa561be 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/function/builtin/SumFloat.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/function/builtin/SumFloat.java @@ -42,39 +42,20 @@ returnType = Type.FLOAT8, paramTypes = {@ParamTypes(paramTypes = {Type.FLOAT4})} ) -public class SumFloat extends AggFunction { +public class SumFloat extends SumDouble { public SumFloat() { super(new Column[] { new Column("expr", Type.FLOAT4) }); } - @Override - public FunctionContext newContext() { - return new SumContext(); - } - @Override public void eval(FunctionContext ctx, Tuple params) { - ((SumContext)ctx).sum += params.get(0).asFloat4(); - } - - @Override - public Datum getPartialResult(FunctionContext ctx) { - return DatumFactory.createFloat8(((SumContext) ctx).sum); - } - - @Override - public DataType getPartialResultType() { - return CatalogUtil.newSimpleDataType(Type.FLOAT8); - } - - @Override - public Datum terminate(FunctionContext ctx) { - return DatumFactory.createFloat8(((SumContext) ctx).sum); - } - - private class SumContext implements FunctionContext { - private double sum; + Datum datum = params.get(0); + if (datum.isNotNull()) { + SumContext sumCtx = (SumContext)ctx; + sumCtx.hasNonNull = true; + sumCtx.sum += datum.asFloat4(); + } } } diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/function/builtin/SumInt.java b/tajo-core/src/main/java/org/apache/tajo/engine/function/builtin/SumInt.java index da8bf16aab..5f68e21ad8 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/function/builtin/SumInt.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/function/builtin/SumInt.java @@ -42,41 +42,11 @@ returnType = Type.INT8, paramTypes = {@ParamTypes(paramTypes = {Type.INT4})} ) -public class SumInt extends AggFunction { +public class SumInt extends SumLong { public SumInt() { super(new Column[] { new Column("expr", Type.INT4) }); } - - @Override - public SumIntContext newContext() { - return new SumIntContext(); - } - - @Override - public void eval(FunctionContext ctx, Tuple params) { - SumIntContext sumCtx = (SumIntContext) ctx; - sumCtx.sum += params.get(0).asInt8(); - } - - @Override - public Datum getPartialResult(FunctionContext ctx) { - return DatumFactory.createInt8(((SumIntContext) ctx).sum); - } - - @Override - public DataType getPartialResultType() { - return CatalogUtil.newSimpleDataType(Type.INT8); - } - - @Override - public Datum terminate(FunctionContext ctx) { - return DatumFactory.createInt8(((SumIntContext) ctx).sum); - } - - private class SumIntContext implements FunctionContext { - long sum; - } } diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/function/builtin/SumLong.java b/tajo-core/src/main/java/org/apache/tajo/engine/function/builtin/SumLong.java index 7a6e707239..0ef4b0af27 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/function/builtin/SumLong.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/function/builtin/SumLong.java @@ -18,6 +18,7 @@ package org.apache.tajo.engine.function.builtin; +import org.apache.commons.math.stat.descriptive.summary.Sum; import org.apache.tajo.catalog.CatalogUtil; import org.apache.tajo.catalog.Column; import org.apache.tajo.common.TajoDataTypes.DataType; @@ -25,6 +26,7 @@ import org.apache.tajo.datum.Datum; import org.apache.tajo.datum.DatumFactory; import org.apache.tajo.datum.Int8Datum; +import org.apache.tajo.datum.NullDatum; import org.apache.tajo.plan.function.AggFunction; import org.apache.tajo.plan.function.FunctionContext; import org.apache.tajo.engine.function.annotation.Description; @@ -51,6 +53,10 @@ public SumLong() { }); } + public SumLong(Column[] definedArgs) { + super(definedArgs); + } + @Override public FunctionContext newContext() { return new SumContext(); @@ -58,12 +64,22 @@ public FunctionContext newContext() { @Override public void eval(FunctionContext ctx, Tuple params) { - ((SumContext)ctx).sum += params.get(0).asInt8(); + Datum datum = params.get(0); + if (datum.isNotNull()) { + SumContext sumCtx = (SumContext) ctx; + sumCtx.hasNonNull = true; + sumCtx.sum += datum.asInt8(); + } } @Override public Datum getPartialResult(FunctionContext ctx) { - return DatumFactory.createInt8(((SumContext) ctx).sum); + SumContext sumCtx = (SumContext) ctx; + if (sumCtx.hasNonNull) { + return DatumFactory.createInt8(sumCtx.sum); + } else { + return NullDatum.get(); + } } @Override @@ -72,11 +88,17 @@ public DataType getPartialResultType() { } @Override - public Int8Datum terminate(FunctionContext ctx) { - return DatumFactory.createInt8(((SumContext) ctx).sum); + public Datum terminate(FunctionContext ctx) { + SumContext sumCtx = (SumContext) ctx; + if (sumCtx.hasNonNull) { + return DatumFactory.createInt8(sumCtx.sum); + } else { + return NullDatum.get(); + } } - private class SumContext implements FunctionContext { - long sum; + protected class SumContext implements FunctionContext { + boolean hasNonNull; + long sum = 0; } } diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/DistinctGroupbySortAggregationExec.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/DistinctGroupbySortAggregationExec.java index 5b404ce0e2..3a84f989d6 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/DistinctGroupbySortAggregationExec.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/DistinctGroupbySortAggregationExec.java @@ -135,7 +135,7 @@ private Tuple getEmptyTuple() { for (SortAggregateExec aggExec: aggregateExecs) { for (int i = 0; i < aggExec.aggFunctionsNum; i++, tupleIndex++) { String funcName = aggExec.aggFunctions[i].getName(); - if ("min".equals(funcName) || "max".equals(funcName)) { + if ("min".equals(funcName) || "max".equals(funcName) || "avg".equals(funcName) || "sum".equals(funcName)) { tuple.put(resultColumnIdIndexes[tupleIndex], DatumFactory.createNullDatum()); } else diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/function/TestBuiltinFunctions.java b/tajo-core/src/test/java/org/apache/tajo/engine/function/TestBuiltinFunctions.java index a408400bfa..4578ae506c 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/function/TestBuiltinFunctions.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/function/TestBuiltinFunctions.java @@ -200,6 +200,129 @@ public void testAvgInt() throws Exception { cleanupQuery(res); } + @Test + public void testAvgLongOverflow() throws Exception { + ResultSet res = executeQuery(); + assertResultSet(res); + cleanupQuery(res); + } + + @Test + public void testAvgWithNull() throws Exception { + KeyValueSet tableOptions = new KeyValueSet(); + tableOptions.set(StorageConstants.TEXT_DELIMITER, StorageConstants.DEFAULT_FIELD_DELIMITER); + tableOptions.set(StorageConstants.TEXT_NULL, "\\\\N"); + + Schema schema = new Schema(); + schema.addColumn("id", TajoDataTypes.Type.INT4); + schema.addColumn("value_int", TajoDataTypes.Type.INT4); + schema.addColumn("value_long", TajoDataTypes.Type.INT8); + schema.addColumn("value_float", TajoDataTypes.Type.FLOAT4); + schema.addColumn("value_double", TajoDataTypes.Type.FLOAT8); + String[] data = new String[]{ "1|\\N|-111|1.2|-50.5", "2|1|\\N|\\N|52.5", "3|2|-333|2.8|\\N" }; + TajoTestingCluster.createTable("table11", schema, tableOptions, data, 1); + + try { + ResultSet res = executeString("select avg(value_int) as avg_int, avg(value_long) as avg_long, avg(value_float) as avg_float, avg(value_double) as avg_double from table11"); + String ascExpected = "avg_int,avg_long,avg_float,avg_double\n" + + "-------------------------------\n" + + "1.5,-222.0,2.0,1.0\n"; + + assertEquals(ascExpected, resultSetToString(res)); + res.close(); + } finally { + executeString("DROP TABLE table11 PURGE"); + } + + } + + @Test + public void testAvgWithAllNulls() throws Exception { + KeyValueSet tableOptions = new KeyValueSet(); + tableOptions.set(StorageConstants.TEXT_DELIMITER, StorageConstants.DEFAULT_FIELD_DELIMITER); + tableOptions.set(StorageConstants.TEXT_NULL, "\\\\N"); + + Schema schema = new Schema(); + schema.addColumn("id", TajoDataTypes.Type.INT4); + schema.addColumn("value_int", TajoDataTypes.Type.INT4); + schema.addColumn("value_long", TajoDataTypes.Type.INT8); + schema.addColumn("value_float", TajoDataTypes.Type.FLOAT4); + schema.addColumn("value_double", TajoDataTypes.Type.FLOAT8); + String[] data = new String[]{ "1|\\N|\\N|\\N|\\N", "2|\\N|\\N|\\N|\\N", "3|\\N|\\N|\\N|\\N" }; + TajoTestingCluster.createTable("table11", schema, tableOptions, data, 1); + + try { + ResultSet res = executeString("select avg(value_int) as avg_int, avg(value_long) as avg_long, avg(value_float) as avg_float, avg(value_double) as avg_double from table11"); + String ascExpected = "avg_int,avg_long,avg_float,avg_double\n" + + "-------------------------------\n" + + "null,null,null,null\n"; + + assertEquals(ascExpected, resultSetToString(res)); + res.close(); + } finally { + executeString("DROP TABLE table11 PURGE"); + } + + } + + @Test + public void testSumWithNull() throws Exception { + KeyValueSet tableOptions = new KeyValueSet(); + tableOptions.set(StorageConstants.TEXT_DELIMITER, StorageConstants.DEFAULT_FIELD_DELIMITER); + tableOptions.set(StorageConstants.TEXT_NULL, "\\\\N"); + + Schema schema = new Schema(); + schema.addColumn("id", TajoDataTypes.Type.INT4); + schema.addColumn("value_int", TajoDataTypes.Type.INT4); + schema.addColumn("value_long", TajoDataTypes.Type.INT8); + schema.addColumn("value_float", TajoDataTypes.Type.FLOAT4); + schema.addColumn("value_double", TajoDataTypes.Type.FLOAT8); + String[] data = new String[]{ "1|\\N|-111|1.2|-50.5", "2|1|\\N|\\N|52.5", "3|2|-333|2.8|\\N" }; + TajoTestingCluster.createTable("table11", schema, tableOptions, data, 1); + + try { + ResultSet res = executeString("select sum(value_int) as sum_int, sum(value_long) as sum_long, sum(value_float) as sum_float, sum(value_double) as sum_double from table11"); + String ascExpected = "sum_int,sum_long,sum_float,sum_double\n" + + "-------------------------------\n" + + "3,-444,4.0,2.0\n"; + + assertEquals(ascExpected, resultSetToString(res)); + res.close(); + } finally { + executeString("DROP TABLE table11 PURGE"); + } + + } + + @Test + public void testSumWithAllNulls() throws Exception { + KeyValueSet tableOptions = new KeyValueSet(); + tableOptions.set(StorageConstants.TEXT_DELIMITER, StorageConstants.DEFAULT_FIELD_DELIMITER); + tableOptions.set(StorageConstants.TEXT_NULL, "\\\\N"); + + Schema schema = new Schema(); + schema.addColumn("id", TajoDataTypes.Type.INT4); + schema.addColumn("value_int", TajoDataTypes.Type.INT4); + schema.addColumn("value_long", TajoDataTypes.Type.INT8); + schema.addColumn("value_float", TajoDataTypes.Type.FLOAT4); + schema.addColumn("value_double", TajoDataTypes.Type.FLOAT8); + String[] data = new String[]{ "1|\\N|\\N|\\N|\\N", "2|\\N|\\N|\\N|\\N", "3|\\N|\\N|\\N|\\N" }; + TajoTestingCluster.createTable("table11", schema, tableOptions, data, 1); + + try { + ResultSet res = executeString("select sum(value_int) as sum_int, sum(value_long) as sum_long, sum(value_float) as sum_float, sum(value_double) as sum_double from table11"); + String ascExpected = "sum_int,sum_long,sum_float,sum_double\n" + + "-------------------------------\n" + + "null,null,null,null\n"; + + assertEquals(ascExpected, resultSetToString(res)); + res.close(); + } finally { + executeString("DROP TABLE table11 PURGE"); + } + + } + // @Test // public void testRandom() throws Exception { // ResultSet res = executeQuery(); diff --git a/tajo-core/src/test/resources/queries/TestBuiltinFunctions/testAvgLongOverflow.sql b/tajo-core/src/test/resources/queries/TestBuiltinFunctions/testAvgLongOverflow.sql new file mode 100644 index 0000000000..51a2ac14a5 --- /dev/null +++ b/tajo-core/src/test/resources/queries/TestBuiltinFunctions/testAvgLongOverflow.sql @@ -0,0 +1 @@ +select avg(cast(l_quantity * 25264513 as INT4)) as avg from lineitem where l_quantity > 0; \ No newline at end of file diff --git a/tajo-core/src/test/resources/results/TestBuiltinFunctions/testAvgLongOverflow.result b/tajo-core/src/test/resources/results/TestBuiltinFunctions/testAvgLongOverflow.result new file mode 100644 index 0000000000..5e1c937b83 --- /dev/null +++ b/tajo-core/src/test/resources/results/TestBuiltinFunctions/testAvgLongOverflow.result @@ -0,0 +1,3 @@ +avg +------------------------------- +9.34786981E8 \ No newline at end of file diff --git a/tajo-core/src/test/resources/results/TestJoinBroadcast/testLeftOuterJoinWithEmptyTable2.result b/tajo-core/src/test/resources/results/TestJoinBroadcast/testLeftOuterJoinWithEmptyTable2.result index 3e8a623f8e..0830847314 100644 --- a/tajo-core/src/test/resources/results/TestJoinBroadcast/testLeftOuterJoinWithEmptyTable2.result +++ b/tajo-core/src/test/resources/results/TestJoinBroadcast/testLeftOuterJoinWithEmptyTable2.result @@ -1,7 +1,7 @@ c_custkey,?sum,?max_1,?max_2 ------------------------------- -1,0,null,null -2,0,null,null -3,0,null,null -4,0,null,null -5,0,null,null \ No newline at end of file +1,null,null,null +2,null,null,null +3,null,null,null +4,null,null,null +5,null,null,null \ No newline at end of file diff --git a/tajo-core/src/test/resources/results/TestJoinBroadcast/testLeftOuterJoinWithEmptyTable4.result b/tajo-core/src/test/resources/results/TestJoinBroadcast/testLeftOuterJoinWithEmptyTable4.result index 0acebad0bc..d4d0b53a72 100644 --- a/tajo-core/src/test/resources/results/TestJoinBroadcast/testLeftOuterJoinWithEmptyTable4.result +++ b/tajo-core/src/test/resources/results/TestJoinBroadcast/testLeftOuterJoinWithEmptyTable4.result @@ -1,4 +1,4 @@ ?max,?sum_1,?max_2,?max_3 ------------------------------- 5,6,O,1996-12-01 -5,0,null,null \ No newline at end of file +5,null,null,null \ No newline at end of file diff --git a/tajo-core/src/test/resources/results/TestJoinQuery/testLeftOuterJoinWithEmptyTable2.result b/tajo-core/src/test/resources/results/TestJoinQuery/testLeftOuterJoinWithEmptyTable2.result index 3e8a623f8e..0830847314 100644 --- a/tajo-core/src/test/resources/results/TestJoinQuery/testLeftOuterJoinWithEmptyTable2.result +++ b/tajo-core/src/test/resources/results/TestJoinQuery/testLeftOuterJoinWithEmptyTable2.result @@ -1,7 +1,7 @@ c_custkey,?sum,?max_1,?max_2 ------------------------------- -1,0,null,null -2,0,null,null -3,0,null,null -4,0,null,null -5,0,null,null \ No newline at end of file +1,null,null,null +2,null,null,null +3,null,null,null +4,null,null,null +5,null,null,null \ No newline at end of file diff --git a/tajo-core/src/test/resources/results/TestJoinQuery/testLeftOuterJoinWithEmptyTable4.result b/tajo-core/src/test/resources/results/TestJoinQuery/testLeftOuterJoinWithEmptyTable4.result index 0acebad0bc..d4d0b53a72 100644 --- a/tajo-core/src/test/resources/results/TestJoinQuery/testLeftOuterJoinWithEmptyTable4.result +++ b/tajo-core/src/test/resources/results/TestJoinQuery/testLeftOuterJoinWithEmptyTable4.result @@ -1,4 +1,4 @@ ?max,?sum_1,?max_2,?max_3 ------------------------------- 5,6,O,1996-12-01 -5,0,null,null \ No newline at end of file +5,null,null,null \ No newline at end of file diff --git a/tajo-core/src/test/resources/results/TestJoinQuery/testLeftOuterJoinWithEmptyTable5.result b/tajo-core/src/test/resources/results/TestJoinQuery/testLeftOuterJoinWithEmptyTable5.result index 18462e9d23..61c58b9960 100644 --- a/tajo-core/src/test/resources/results/TestJoinQuery/testLeftOuterJoinWithEmptyTable5.result +++ b/tajo-core/src/test/resources/results/TestJoinQuery/testLeftOuterJoinWithEmptyTable5.result @@ -1,4 +1,4 @@ l_linenumber,?sum,?max_1,?max_2,?avg_3,?sum_4 ------------------------------- -1,0,null,null,33.333333333333336,100.0 -2,0,null,null,42.5,85.0 \ No newline at end of file +1,null,null,null,33.333333333333336,100.0 +2,null,null,null,42.5,85.0 \ No newline at end of file From 41e6ddbdd887870d33bb3c626a53decc46aa747a Mon Sep 17 00:00:00 2001 From: sirpkt Date: Thu, 8 Jan 2015 09:07:59 +0900 Subject: [PATCH 6/6] remove unused imports --- .../java/org/apache/tajo/engine/function/builtin/AvgDouble.java | 1 - .../java/org/apache/tajo/engine/function/builtin/AvgLong.java | 1 - .../java/org/apache/tajo/engine/function/builtin/SumDouble.java | 1 - .../java/org/apache/tajo/engine/function/builtin/SumLong.java | 2 -- 4 files changed, 5 deletions(-) diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/function/builtin/AvgDouble.java b/tajo-core/src/main/java/org/apache/tajo/engine/function/builtin/AvgDouble.java index a74dcad417..a69becacd5 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/function/builtin/AvgDouble.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/function/builtin/AvgDouble.java @@ -18,7 +18,6 @@ package org.apache.tajo.engine.function.builtin; -import org.apache.commons.lang.ObjectUtils; import org.apache.tajo.catalog.CatalogUtil; import org.apache.tajo.catalog.Column; import org.apache.tajo.common.TajoDataTypes.DataType; diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/function/builtin/AvgLong.java b/tajo-core/src/main/java/org/apache/tajo/engine/function/builtin/AvgLong.java index 85950117cf..417ecb79bf 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/function/builtin/AvgLong.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/function/builtin/AvgLong.java @@ -18,7 +18,6 @@ package org.apache.tajo.engine.function.builtin; -import org.apache.commons.lang.ObjectUtils; import org.apache.tajo.catalog.CatalogUtil; import org.apache.tajo.catalog.Column; import org.apache.tajo.common.TajoDataTypes.DataType; diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/function/builtin/SumDouble.java b/tajo-core/src/main/java/org/apache/tajo/engine/function/builtin/SumDouble.java index 5bb5979d43..2f422724eb 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/function/builtin/SumDouble.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/function/builtin/SumDouble.java @@ -24,7 +24,6 @@ import org.apache.tajo.common.TajoDataTypes.Type; import org.apache.tajo.datum.Datum; import org.apache.tajo.datum.DatumFactory; -import org.apache.tajo.datum.Float8Datum; import org.apache.tajo.datum.NullDatum; import org.apache.tajo.plan.function.AggFunction; import org.apache.tajo.plan.function.FunctionContext; diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/function/builtin/SumLong.java b/tajo-core/src/main/java/org/apache/tajo/engine/function/builtin/SumLong.java index 0ef4b0af27..55484d7c37 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/function/builtin/SumLong.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/function/builtin/SumLong.java @@ -18,14 +18,12 @@ package org.apache.tajo.engine.function.builtin; -import org.apache.commons.math.stat.descriptive.summary.Sum; import org.apache.tajo.catalog.CatalogUtil; import org.apache.tajo.catalog.Column; import org.apache.tajo.common.TajoDataTypes.DataType; import org.apache.tajo.common.TajoDataTypes.Type; import org.apache.tajo.datum.Datum; import org.apache.tajo.datum.DatumFactory; -import org.apache.tajo.datum.Int8Datum; import org.apache.tajo.datum.NullDatum; import org.apache.tajo.plan.function.AggFunction; import org.apache.tajo.plan.function.FunctionContext;