From 5afd9fd3c1bf307749d7ea0cf0dab279297062f2 Mon Sep 17 00:00:00 2001 From: Ali Alsuliman Date: Sun, 17 Jun 2018 00:23:35 -0700 Subject: [PATCH] [NO ISSUE][FUN] Implement array_append function - user model changes: no - storage format changes: no - interface changes: no details: This is part of implementing array functions. The array_append takes an input list and values to be appended and return a new list with all the values. array_append(list, val1,...) Change-Id: I0a65549957060beee4579b903e6bd323745fb875 Reviewed-on: https://asterix-gerrit.ics.uci.edu/2710 Tested-by: Jenkins Contrib: Jenkins Integration-Tests: Jenkins Reviewed-by: Dmitry Lychagin --- .../apache/asterix/runtime/ExceptionIT.java | 4 +- .../asterix/runtime/NullMissingTest.java | 4 +- .../array_append/array_append.1.ddl.sqlpp | 56 ++++++ .../array_append/array_append.2.update.sqlpp | 27 +++ .../array_append/array_append.3.query.sqlpp | 31 +++ .../array_append/array_append.4.ddl.sqlpp | 20 ++ .../array_fun/array_append/array_append.3.adm | 1 + .../resources/runtimets/testsuite_sqlpp.xml | 7 + .../asterix/common/exceptions/ErrorCode.java | 1 + .../main/resources/asx_errormsg/en.properties | 1 + .../om/functions/BuiltinFunctions.java | 23 ++- .../base/AbstractResultTypeComputer.java | 3 +- .../impl/ArrayAppendTypeComputer.java | 54 ++++++ .../functions/ArrayAppendDescriptor.java | 176 ++++++++++++++++++ .../functions/CastTypeEvaluator.java | 16 +- .../records/RecordReplaceDescriptor.java | 2 +- .../runtime/functions/FunctionCollection.java | 6 +- .../functions/FunctionTypeInferers.java | 32 ++-- 18 files changed, 430 insertions(+), 34 deletions(-) create mode 100755 asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array_fun/array_append/array_append.1.ddl.sqlpp create mode 100755 asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array_fun/array_append/array_append.2.update.sqlpp create mode 100755 asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array_fun/array_append/array_append.3.query.sqlpp create mode 100755 asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array_fun/array_append/array_append.4.ddl.sqlpp create mode 100644 asterixdb/asterix-app/src/test/resources/runtimets/results/array_fun/array_append/array_append.3.adm create mode 100755 asterixdb/asterix-om/src/main/java/org/apache/asterix/om/typecomputer/impl/ArrayAppendTypeComputer.java create mode 100755 asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/ArrayAppendDescriptor.java diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/runtime/ExceptionIT.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/runtime/ExceptionIT.java index ccbf4e31cba..e764c32159f 100644 --- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/runtime/ExceptionIT.java +++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/runtime/ExceptionIT.java @@ -50,7 +50,9 @@ public void test() throws Exception { String className = func.getClass().getName(); // We test all generated functions except // record and cast functions, which requires type settings. - if (className.contains("Gen") && !className.contains("record") && !className.contains("Cast")) { + String[] splits = className.split("\\."); + if (className.contains("Gen") && !className.contains("record") && !className.contains("Cast") + && !splits[splits.length - 1].startsWith("Array")) { testFunction(func); ++testedFunctions; } diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/runtime/NullMissingTest.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/runtime/NullMissingTest.java index d96fec9cf8c..89494c473ff 100644 --- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/runtime/NullMissingTest.java +++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/runtime/NullMissingTest.java @@ -50,7 +50,9 @@ public void test() throws Exception { String className = func.getClass().getName(); // We test all generated functions except // record and cast functions, which requires type settings (we test them in runtime tests). - if (className.contains("Gen") && !className.contains("record") && !className.contains("Cast")) { + String[] splits = className.split("\\."); + if (className.contains("Gen") && !className.contains("record") && !className.contains("Cast") + && !splits[splits.length - 1].startsWith("Array")) { testFunction(func); ++testedFunctions; } diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array_fun/array_append/array_append.1.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array_fun/array_append/array_append.1.ddl.sqlpp new file mode 100755 index 00000000000..1c55a9a5d02 --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array_fun/array_append/array_append.1.ddl.sqlpp @@ -0,0 +1,56 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +drop dataverse TinySocial if exists; +create dataverse TinySocial; + +use TinySocial; + + +create type TinySocial.TwitterUserType as +{ + `screen-name` : string, + lang : string, + friends_count : bigint, + statuses_count : bigint, + name : string, + followers_count : bigint +}; + +create type TinySocial.TweetMessageType as + closed { + tweetid : string, + user : TwitterUserType, + `sender-location` : point?, + `send-time` : datetime, + `referred-topics` : {{string}}, + `message-text` : string +}; + +create type t1 AS { + +}; + +create type t2 AS { +id: int, +compType: t1 +}; + +create dataset TweetMessages(TweetMessageType) primary key tweetid hints (`CARDINALITY`=`100`); +create dataset d1(t2) primary key id; \ No newline at end of file diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array_fun/array_append/array_append.2.update.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array_fun/array_append/array_append.2.update.sqlpp new file mode 100755 index 00000000000..27672103d5a --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array_fun/array_append/array_append.2.update.sqlpp @@ -0,0 +1,27 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +use TinySocial; + +load dataset TweetMessages using localfs ((`path`=`asterix_nc1://data/tinysocial/twm.adm`),(`format`=`adm`)); + +insert into d1([ +{"id":1, "compType":{"sth":33}}, +{"id":2, "compType":{"sth":44}} +]); \ No newline at end of file diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array_fun/array_append/array_append.3.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array_fun/array_append/array_append.3.query.sqlpp new file mode 100755 index 00000000000..8488e22b500 --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array_fun/array_append/array_append.3.query.sqlpp @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +use TinySocial; + +{ + "t1": (select array_append(t.`referred-topics`, "sth", 5) from TweetMessages t order by t.tweetid), + "t2": (select array_append([3, "John"], (select value v.compType from d1 v))), + "t3": (select array_append([3], 7, null, missing)), + "t4": (select array_append("non_array", 5)), + "t5": (select array_append("non_array", 5, missing)), + "t6": (select array_append([], 5, 10, 12.0, "sth")), + "t7": (select array_append(missing, 3, 9)), + "t8": (select array_append(null, 3, 9)) +}; \ No newline at end of file diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array_fun/array_append/array_append.4.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array_fun/array_append/array_append.4.ddl.sqlpp new file mode 100755 index 00000000000..3f8c8ec49ab --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array_fun/array_append/array_append.4.ddl.sqlpp @@ -0,0 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +drop dataverse TinySocial; \ No newline at end of file diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/array_fun/array_append/array_append.3.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/array_fun/array_append/array_append.3.adm new file mode 100644 index 00000000000..c435e4e654b --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/array_fun/array_append/array_append.3.adm @@ -0,0 +1 @@ +{ "t1": [ { "$1": {{ "t-mobile", "customization", "sth", 5 }} }, { "$1": {{ "verizon", "voice-clarity", "sth", 5 }} }, { "$1": {{ "iphone", "platform", "sth", 5 }} }, { "$1": {{ "samsung", "voice-command", "sth", 5 }} }, { "$1": {{ "verizon", "shortcut-menu", "sth", 5 }} }, { "$1": {{ "motorola", "speed", "sth", 5 }} }, { "$1": {{ "sprint", "voice-command", "sth", 5 }} }, { "$1": {{ "motorola", "speed", "sth", 5 }} }, { "$1": {{ "iphone", "voice-clarity", "sth", 5 }} }, { "$1": {{ "samsung", "platform", "sth", 5 }} }, { "$1": {{ "t-mobile", "shortcut-menu", "sth", 5 }} }, { "$1": {{ "verizon", "voicemail-service", "sth", 5 }} } ], "t2": [ { "$2": [ 3, "John", [ { "sth": 33 }, { "sth": 44 } ] ] } ], "t3": [ { } ], "t4": [ { "$4": null } ], "t5": [ { } ], "t6": [ { "$6": [ 5, 10, 12.0, "sth" ] } ], "t7": [ { } ], "t8": [ { "$8": null } ] } diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml index 21c14d12d48..a6613cd460b 100644 --- a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml +++ b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml @@ -982,6 +982,13 @@ + + + + array_append + + + diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/exceptions/ErrorCode.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/exceptions/ErrorCode.java index ae59ddcf05a..447f8516143 100644 --- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/exceptions/ErrorCode.java +++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/exceptions/ErrorCode.java @@ -167,6 +167,7 @@ public class ErrorCode { public static final int INDEX_EXISTS = 1084; public static final int TYPE_EXISTS = 1085; public static final int PARAMETER_NO_VALUE = 1086; + public static final int COMPILATION_INVALID_NUM_OF_ARGS = 1087; // Feed errors public static final int DATAFLOW_ILLEGAL_STATE = 3001; diff --git a/asterixdb/asterix-common/src/main/resources/asx_errormsg/en.properties b/asterixdb/asterix-common/src/main/resources/asx_errormsg/en.properties index 01e217eb432..720ff57fc53 100644 --- a/asterixdb/asterix-common/src/main/resources/asx_errormsg/en.properties +++ b/asterixdb/asterix-common/src/main/resources/asx_errormsg/en.properties @@ -154,6 +154,7 @@ 1084 = An index with this name %1$s already exists 1085 = A datatype with this name %1$s already exists 1086 = No value for parameter: %1$s +1087 = Invalid number of arguments: at least %1$d arguments are required for function %2$s # Feed Errors 3001 = Illegal state. diff --git a/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/functions/BuiltinFunctions.java b/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/functions/BuiltinFunctions.java index 3263ae120e1..f17bc47beb5 100644 --- a/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/functions/BuiltinFunctions.java +++ b/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/functions/BuiltinFunctions.java @@ -53,6 +53,7 @@ import org.apache.asterix.om.typecomputer.impl.AUUIDTypeComputer; import org.apache.asterix.om.typecomputer.impl.AYearMonthDurationTypeComputer; import org.apache.asterix.om.typecomputer.impl.AnyTypeComputer; +import org.apache.asterix.om.typecomputer.impl.ArrayAppendTypeComputer; import org.apache.asterix.om.typecomputer.impl.BooleanFunctionTypeComputer; import org.apache.asterix.om.typecomputer.impl.BooleanOnlyTypeComputer; import org.apache.asterix.om.typecomputer.impl.BooleanOrMissingTypeComputer; @@ -135,15 +136,12 @@ public enum SpatialFilterKind { } private static final FunctionInfoRepository registeredFunctions = new FunctionInfoRepository(); - private static final Map registeredFunctionsDomain = new HashMap<>(); // it is supposed to be an identity mapping private static final Map builtinPublicFunctionsSet = new HashMap<>(); private static final Map builtinPrivateFunctionsSet = new HashMap<>(); - private static final Map funTypeComputer = new HashMap<>(); - private static final Set builtinAggregateFunctions = new HashSet<>(); private static final Map datasourceFunctions = new HashMap<>(); private static final Set similarityFunctions = new HashSet<>(); @@ -164,14 +162,12 @@ public enum SpatialFilterKind { new FunctionIdentifier(FunctionConstants.ASTERIX_NS, "get-handle", 2); public static final FunctionIdentifier GET_DATA = new FunctionIdentifier(FunctionConstants.ASTERIX_NS, "get-data", 2); - public static final FunctionIdentifier GET_ITEM = new FunctionIdentifier(FunctionConstants.ASTERIX_NS, "get-item", 2); public static final FunctionIdentifier ANY_COLLECTION_MEMBER = new FunctionIdentifier(FunctionConstants.ASTERIX_NS, "any-collection-member", 1); public static final FunctionIdentifier LISTIFY = new FunctionIdentifier(FunctionConstants.ASTERIX_NS, "listify", 1); public static final FunctionIdentifier LEN = new FunctionIdentifier(FunctionConstants.ASTERIX_NS, "len", 1); - public static final FunctionIdentifier CONCAT_NON_NULL = new FunctionIdentifier(FunctionConstants.ASTERIX_NS, "concat-non-null", FunctionIdentifier.VARARGS); public static final FunctionIdentifier EMPTY_STREAM = @@ -186,6 +182,10 @@ public enum SpatialFilterKind { public static final FunctionIdentifier DEEP_EQUAL = new FunctionIdentifier(FunctionConstants.ASTERIX_NS, "deep-equal", 2); + // array functions + public static final FunctionIdentifier ARRAY_APPEND = + new FunctionIdentifier(FunctionConstants.ASTERIX_NS, "array-append", FunctionIdentifier.VARARGS); + // objects public static final FunctionIdentifier RECORD_MERGE = new FunctionIdentifier(FunctionConstants.ASTERIX_NS, "object-merge", 2); @@ -300,7 +300,8 @@ public enum SpatialFilterKind { new FunctionIdentifier(FunctionConstants.ASTERIX_NS, "find-binary", 2); public static final FunctionIdentifier FIND_BINARY_FROM = new FunctionIdentifier(FunctionConstants.ASTERIX_NS, "find-binary", 3); - // String funcitons + + // String functions public static final FunctionIdentifier STRING_EQUAL = new FunctionIdentifier(FunctionConstants.ASTERIX_NS, "string-equal", 2); public static final FunctionIdentifier STRING_MATCHES = @@ -392,6 +393,7 @@ public enum SpatialFilterKind { public static final FunctionIdentifier MAKE_FIELD_NAME_HANDLE = new FunctionIdentifier(FunctionConstants.ASTERIX_NS, "make-field-name-handle", 1); + // aggregate functions public static final FunctionIdentifier AVG = new FunctionIdentifier(FunctionConstants.ASTERIX_NS, "agg-avg", 1); public static final FunctionIdentifier COUNT = new FunctionIdentifier(FunctionConstants.ASTERIX_NS, "agg-count", 1); public static final FunctionIdentifier SUM = new FunctionIdentifier(FunctionConstants.ASTERIX_NS, "agg-sum", 1); @@ -444,7 +446,6 @@ public enum SpatialFilterKind { new FunctionIdentifier(FunctionConstants.ASTERIX_NS, "intermediate-avg-serial", 1); // distinct aggregate functions - public static final FunctionIdentifier COUNT_DISTINCT = new FunctionIdentifier(FunctionConstants.ASTERIX_NS, "agg-count-distinct", 1); public static final FunctionIdentifier SCALAR_COUNT_DISTINCT = @@ -543,6 +544,7 @@ public enum SpatialFilterKind { public static final FunctionIdentifier SCALAR_SQL_MIN_DISTINCT = new FunctionIdentifier(FunctionConstants.ASTERIX_NS, "sql-min-distinct", 1); + // unnesting functions public static final FunctionIdentifier SCAN_COLLECTION = new FunctionIdentifier(FunctionConstants.ASTERIX_NS, "scan-collection", 1); public static final FunctionIdentifier SUBSET_COLLECTION = @@ -550,7 +552,7 @@ public enum SpatialFilterKind { public static final FunctionIdentifier RANGE = new FunctionIdentifier(FunctionConstants.ASTERIX_NS, "range", 2); - // fuzzy functions: + // fuzzy functions public static final FunctionIdentifier FUZZY_EQ = new FunctionIdentifier(FunctionConstants.ASTERIX_NS, "fuzzy-eq", 2); @@ -1093,7 +1095,7 @@ public static FunctionInfo lookupFunction(FunctionIdentifier fid) { addPrivateFunction(OR, BooleanFunctionTypeComputer.INSTANCE, true); addPrivateFunction(NUMERIC_ADD, NumericAddSubMulDivTypeComputer.INSTANCE, true); - // Deep equality + // deep equality addFunction(DEEP_EQUAL, BooleanFunctionTypeComputer.INSTANCE, true); // and then, Asterix builtin functions @@ -1451,6 +1453,9 @@ public static FunctionInfo lookupFunction(FunctionIdentifier fid) { addPrivateFunction(UNORDERED_LIST_CONSTRUCTOR, UnorderedListConstructorTypeComputer.INSTANCE, true); addFunction(WORD_TOKENS, OrderedListOfAStringTypeComputer.INSTANCE, true); + // array functions + addFunction(ARRAY_APPEND, ArrayAppendTypeComputer.INSTANCE, true); + // objects addFunction(RECORD_MERGE, RecordMergeTypeComputer.INSTANCE, true); addFunction(RECORD_CONCAT, OpenARecordTypeComputer.INSTANCE, true); diff --git a/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/typecomputer/base/AbstractResultTypeComputer.java b/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/typecomputer/base/AbstractResultTypeComputer.java index ae7c996ee42..60a49853641 100644 --- a/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/typecomputer/base/AbstractResultTypeComputer.java +++ b/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/typecomputer/base/AbstractResultTypeComputer.java @@ -68,7 +68,8 @@ protected void checkArgType(String funcName, int argIndex, IAType type, SourceLo * @param expr * the expression under consideration. * @param strippedInputTypes, - * the stripped input types. + * the stripped input types. When the function propagates null & missing, they can be any type except + * null, missing, and union. * @return the result type without considering optional types. * @throws AlgebricksException */ diff --git a/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/typecomputer/impl/ArrayAppendTypeComputer.java b/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/typecomputer/impl/ArrayAppendTypeComputer.java new file mode 100755 index 00000000000..f2fed42114d --- /dev/null +++ b/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/typecomputer/impl/ArrayAppendTypeComputer.java @@ -0,0 +1,54 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.asterix.om.typecomputer.impl; + +import org.apache.asterix.common.exceptions.CompilationException; +import org.apache.asterix.common.exceptions.ErrorCode; +import org.apache.asterix.om.pointables.base.DefaultOpenFieldType; +import org.apache.asterix.om.typecomputer.base.AbstractResultTypeComputer; +import org.apache.asterix.om.types.ATypeTag; +import org.apache.asterix.om.types.BuiltinType; +import org.apache.asterix.om.types.IAType; +import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException; +import org.apache.hyracks.algebricks.core.algebra.base.ILogicalExpression; +import org.apache.hyracks.algebricks.core.algebra.expressions.AbstractFunctionCallExpression; + +public class ArrayAppendTypeComputer extends AbstractResultTypeComputer { + + public static final ArrayAppendTypeComputer INSTANCE = new ArrayAppendTypeComputer(); + + @Override + protected IAType getResultType(ILogicalExpression expr, IAType... strippedInputTypes) throws AlgebricksException { + if (strippedInputTypes.length < 2) { + String functionName = ((AbstractFunctionCallExpression) expr).getFunctionIdentifier().getName(); + throw new CompilationException(ErrorCode.COMPILATION_INVALID_NUM_OF_ARGS, expr.getSourceLocation(), 2, + functionName); + } + // type tag at [0] should be array or multiset. + ATypeTag typeTag = strippedInputTypes[0].getTypeTag(); + if (typeTag == ATypeTag.ARRAY) { + return DefaultOpenFieldType.NESTED_OPEN_AORDERED_LIST_TYPE; + } else if (typeTag == ATypeTag.MULTISET) { + return DefaultOpenFieldType.NESTED_OPEN_AUNORDERED_LIST_TYPE; + } else { + return BuiltinType.ANY; + } + } +} diff --git a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/ArrayAppendDescriptor.java b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/ArrayAppendDescriptor.java new file mode 100755 index 00000000000..a44397c1e91 --- /dev/null +++ b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/ArrayAppendDescriptor.java @@ -0,0 +1,176 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.asterix.runtime.evaluators.functions; + +import java.io.IOException; +import java.util.Arrays; + +import org.apache.asterix.builders.IAsterixListBuilder; +import org.apache.asterix.builders.OrderedListBuilder; +import org.apache.asterix.builders.UnorderedListBuilder; +import org.apache.asterix.om.functions.BuiltinFunctions; +import org.apache.asterix.om.functions.IFunctionDescriptor; +import org.apache.asterix.om.functions.IFunctionDescriptorFactory; +import org.apache.asterix.om.functions.IFunctionTypeInferer; +import org.apache.asterix.om.pointables.base.DefaultOpenFieldType; +import org.apache.asterix.om.types.AOrderedListType; +import org.apache.asterix.om.types.ATypeTag; +import org.apache.asterix.om.types.AbstractCollectionType; +import org.apache.asterix.om.types.IAType; +import org.apache.asterix.runtime.evaluators.base.AbstractScalarFunctionDynamicDescriptor; +import org.apache.asterix.runtime.evaluators.common.ListAccessor; +import org.apache.asterix.runtime.functions.FunctionTypeInferers; +import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException; +import org.apache.hyracks.algebricks.core.algebra.functions.FunctionIdentifier; +import org.apache.hyracks.algebricks.runtime.base.IScalarEvaluator; +import org.apache.hyracks.algebricks.runtime.base.IScalarEvaluatorFactory; +import org.apache.hyracks.api.context.IHyracksTaskContext; +import org.apache.hyracks.api.exceptions.HyracksDataException; +import org.apache.hyracks.data.std.api.IPointable; +import org.apache.hyracks.data.std.primitive.VoidPointable; +import org.apache.hyracks.data.std.util.ArrayBackedValueStorage; +import org.apache.hyracks.dataflow.common.data.accessors.IFrameTupleReference; + +public class ArrayAppendDescriptor extends AbstractScalarFunctionDynamicDescriptor { + private static final long serialVersionUID = 1L; + private IAType[] argTypes; + + public static final IFunctionDescriptorFactory FACTORY = new IFunctionDescriptorFactory() { + @Override + public IFunctionDescriptor createFunctionDescriptor() { + return new ArrayAppendDescriptor(); + } + + @Override + public IFunctionTypeInferer createFunctionTypeInferer() { + return FunctionTypeInferers.SET_ARGUMENTS_TYPE; + } + }; + + @Override + public FunctionIdentifier getIdentifier() { + return BuiltinFunctions.ARRAY_APPEND; + } + + @Override + public IScalarEvaluatorFactory createEvaluatorFactory(final IScalarEvaluatorFactory[] args) + throws AlgebricksException { + return new IScalarEvaluatorFactory() { + private static final long serialVersionUID = 1L; + + @Override + public IScalarEvaluator createScalarEvaluator(final IHyracksTaskContext ctx) throws HyracksDataException { + return new ArrayAppendFunction(args, ctx); + } + }; + } + + @Override + public void setImmutableStates(Object... states) { + argTypes = Arrays.copyOf(states, states.length, IAType[].class); + } + + public class ArrayAppendFunction implements IScalarEvaluator { + private final ArrayBackedValueStorage storage; + private final IPointable listArg; + private final IPointable[] appendedValues; + private final IScalarEvaluator listArgEval; + private final IScalarEvaluator[] appendedValuesEval; + + public ArrayAppendFunction(IScalarEvaluatorFactory[] args, IHyracksTaskContext ctx) + throws HyracksDataException { + storage = new ArrayBackedValueStorage(); + listArg = new VoidPointable(); + listArgEval = args[0].createScalarEvaluator(ctx); + appendedValues = new IPointable[args.length - 1]; + appendedValuesEval = new IScalarEvaluator[args.length - 1]; + for (int i = 1; i < args.length; i++) { + appendedValues[i - 1] = new VoidPointable(); + appendedValuesEval[i - 1] = args[i].createScalarEvaluator(ctx); + } + } + + @Override + public void evaluate(IFrameTupleReference tuple, IPointable result) throws HyracksDataException { + // get the list argument, 1st argument, make sure it's a list + listArgEval.evaluate(tuple, listArg); + byte listArgType = listArg.getByteArray()[listArg.getStartOffset()]; + + CastTypeEvaluator caster = null; + AbstractCollectionType listType = null; + IAsterixListBuilder listBuilder = null; + // create the new list to be returned. The item type is always "ANY" + // cast the input list and make it open + if (listArgType == ATypeTag.SERIALIZED_ORDEREDLIST_TYPE_TAG) { + listBuilder = new OrderedListBuilder(); + listType = DefaultOpenFieldType.NESTED_OPEN_AORDERED_LIST_TYPE; + caster = new CastTypeEvaluator(listType, argTypes[0], listArgEval); + caster.evaluate(tuple, listArg); + } else if (listArgType == ATypeTag.SERIALIZED_UNORDEREDLIST_TYPE_TAG) { + listBuilder = new UnorderedListBuilder(); + listType = DefaultOpenFieldType.NESTED_OPEN_AUNORDERED_LIST_TYPE; + caster = new CastTypeEvaluator(listType, argTypes[0], listArgEval); + caster.evaluate(tuple, listArg); + } + // else, don't return null right away. evaluate rest of args as some may be missing, return missing instead + IAType defaultOpenType; + for (int i = 0; i < appendedValuesEval.length; i++) { + // cast to open if necessary + defaultOpenType = DefaultOpenFieldType.getDefaultOpenFieldType(argTypes[i + 1].getTypeTag()); + if (defaultOpenType != null && caster != null) { + caster.reset(defaultOpenType, argTypes[i + 1], appendedValuesEval[i]); + caster.evaluate(tuple, appendedValues[i]); + } else { + // either no casting is needed (e.g. int and the like) or evaluate normally for the below case: + // when caster == null, it means the first arg was not a list and a null would be returned but + // evaluate values to be appended normally in case missing exists and return missing instead of null + appendedValuesEval[i].evaluate(tuple, appendedValues[i]); + } + } + + if (listArgType != ATypeTag.SERIALIZED_ORDEREDLIST_TYPE_TAG + && listArgType != ATypeTag.SERIALIZED_UNORDEREDLIST_TYPE_TAG) { + PointableHelper.setNull(result); + return; + } + + // arguments are good: no nulls/missings and 1st arg is a list + listBuilder.reset(listType); + ListAccessor listAccessor = new ListAccessor(); + listAccessor.reset(listArg.getByteArray(), listArg.getStartOffset()); + try { + // get the list items one by one and append to the new list + for (int i = 0; i < listAccessor.size(); i++) { + storage.reset(); + listAccessor.writeItem(i, storage.getDataOutput()); + listBuilder.addItem(storage); + } + // append the values arguments + for (IPointable appendedValue : appendedValues) { + listBuilder.addItem(appendedValue); + } + storage.reset(); + listBuilder.write(storage.getDataOutput(), true); + result.set(storage); + } catch (IOException e) { + throw HyracksDataException.create(e); + } + } + } +} diff --git a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/CastTypeEvaluator.java b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/CastTypeEvaluator.java index 22efe9aafeb..7e3195104d1 100644 --- a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/CastTypeEvaluator.java +++ b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/CastTypeEvaluator.java @@ -35,12 +35,12 @@ public class CastTypeEvaluator implements IScalarEvaluator { - private final IScalarEvaluator argEvaluator; + private IScalarEvaluator argEvaluator; private final IPointable argPointable = new VoidPointable(); private final PointableAllocator allocator = new PointableAllocator(); - private final IVisitablePointable inputPointable; - private final IVisitablePointable resultPointable; + private IVisitablePointable inputPointable; + private IVisitablePointable resultPointable; private final ACastVisitor castVisitor; private final Triple arg; @@ -53,6 +53,14 @@ public CastTypeEvaluator(IAType reqType, IAType inputType, IScalarEvaluator argE this.castVisitor = createCastVisitor(); } + public void reset(IAType reqType, IAType inputType, IScalarEvaluator argEvaluator) { + this.argEvaluator = argEvaluator; + this.inputPointable = allocatePointable(inputType, reqType); + this.resultPointable = allocatePointable(reqType, inputType); + this.arg.first = resultPointable; + this.arg.second = reqType; + } + protected ACastVisitor createCastVisitor() { return new ACastVisitor(); } @@ -70,7 +78,7 @@ protected void cast(IPointable result) throws HyracksDataException { } // Allocates the result pointable. - private final IVisitablePointable allocatePointable(IAType typeForPointable, IAType typeForOtherSide) { + private IVisitablePointable allocatePointable(IAType typeForPointable, IAType typeForOtherSide) { if (!typeForPointable.equals(BuiltinType.ANY)) { return allocator.allocateFieldValue(typeForPointable); } diff --git a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/records/RecordReplaceDescriptor.java b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/records/RecordReplaceDescriptor.java index 665f3f6b263..03d0088cd02 100644 --- a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/records/RecordReplaceDescriptor.java +++ b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/records/RecordReplaceDescriptor.java @@ -41,7 +41,7 @@ public IFunctionDescriptor createFunctionDescriptor() { @Override public IFunctionTypeInferer createFunctionTypeInferer() { - return new FunctionTypeInferers.ArgsTypeInferer(); + return FunctionTypeInferers.SET_ARGUMENTS_TYPE; } }; diff --git a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/functions/FunctionCollection.java b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/functions/FunctionCollection.java index 58d32e4afd5..7663f1958ad 100644 --- a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/functions/FunctionCollection.java +++ b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/functions/FunctionCollection.java @@ -143,6 +143,7 @@ import org.apache.asterix.runtime.evaluators.constructors.UnorderedListConstructorDescriptor; import org.apache.asterix.runtime.evaluators.functions.AndDescriptor; import org.apache.asterix.runtime.evaluators.functions.AnyCollectionMemberDescriptor; +import org.apache.asterix.runtime.evaluators.functions.ArrayAppendDescriptor; import org.apache.asterix.runtime.evaluators.functions.CastTypeDescriptor; import org.apache.asterix.runtime.evaluators.functions.CastTypeLaxDescriptor; import org.apache.asterix.runtime.evaluators.functions.CheckUnknownDescriptor; @@ -366,7 +367,10 @@ public void addGenerated(IFunctionDescriptorFactory descriptorFactory) { public static FunctionCollection createDefaultFunctionCollection() { FunctionCollection fc = new FunctionCollection(); - // unnesting function + // array functions + fc.addGenerated(ArrayAppendDescriptor.FACTORY); + + // unnesting functions fc.add(TidRunningAggregateDescriptor.FACTORY); fc.add(ScanCollectionDescriptor.FACTORY); fc.add(RangeDescriptor.FACTORY); diff --git a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/functions/FunctionTypeInferers.java b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/functions/FunctionTypeInferers.java index 45749a71ae5..b9c58c72ce7 100644 --- a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/functions/FunctionTypeInferers.java +++ b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/functions/FunctionTypeInferers.java @@ -79,6 +79,22 @@ public void infer(ILogicalExpression expr, IFunctionDescriptor fd, IVariableType } }; + /** Sets the types of the function arguments */ + public static final IFunctionTypeInferer SET_ARGUMENTS_TYPE = new IFunctionTypeInferer() { + @Override + public void infer(ILogicalExpression expr, IFunctionDescriptor fd, IVariableTypeEnvironment context, + CompilerProperties compilerProps) throws AlgebricksException { + AbstractFunctionCallExpression fce = (AbstractFunctionCallExpression) expr; + IAType[] argsTypes = new IAType[fce.getArguments().size()]; + int i = 0; + for (Mutable arg : fce.getArguments()) { + argsTypes[i] = TypeComputeUtils.getActualType((IAType) context.getType(arg.getValue())); + i++; + } + fd.setImmutableStates((Object[]) argsTypes); + } + }; + public static final class CastTypeInferer implements IFunctionTypeInferer { @Override public void infer(ILogicalExpression expr, IFunctionDescriptor fd, IVariableTypeEnvironment context, @@ -295,20 +311,4 @@ public void infer(ILogicalExpression expr, IFunctionDescriptor fd, IVariableType fd.setImmutableStates((Object[]) argRecordTypes); } } - - public static final class ArgsTypeInferer implements IFunctionTypeInferer { - @Override - public void infer(ILogicalExpression expr, IFunctionDescriptor fd, IVariableTypeEnvironment context, - CompilerProperties compilerProps) throws AlgebricksException { - final AbstractFunctionCallExpression f = (AbstractFunctionCallExpression) expr; - final List> args = f.getArguments(); - final IAType[] types = new IAType[f.getArguments().size()]; - for (int i = 0; i < types.length; i++) { - final IAType argType = (IAType) context.getType(args.get(i).getValue()); - final IAType actualType = TypeComputeUtils.getActualType(argType); - types[i] = actualType; - } - fd.setImmutableStates((Object[]) types); - } - } }