Skip to content

Commit

Permalink
[NO ISSUE][FUN] Implement array_append function
Browse files Browse the repository at this point in the history
- user model changes: no
- storage format changes: no
- interface changes: no

details:
This is part of implementing array functions.
The array_append takes an input list and values to be
appended and return a new list with all the values.
array_append(list, val1,...)

Change-Id: I0a65549957060beee4579b903e6bd323745fb875
Reviewed-on: https://asterix-gerrit.ics.uci.edu/2710
Tested-by: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Contrib: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Integration-Tests: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Reviewed-by: Dmitry Lychagin <dmitry.lychagin@couchbase.com>
  • Loading branch information
AliSolaiman authored and mblow committed Jun 17, 2018
1 parent baa4b59 commit 5afd9fd
Show file tree
Hide file tree
Showing 18 changed files with 430 additions and 34 deletions.
Expand Up @@ -50,7 +50,9 @@ public void test() throws Exception {
String className = func.getClass().getName();
// We test all generated functions except
// record and cast functions, which requires type settings.
if (className.contains("Gen") && !className.contains("record") && !className.contains("Cast")) {
String[] splits = className.split("\\.");
if (className.contains("Gen") && !className.contains("record") && !className.contains("Cast")
&& !splits[splits.length - 1].startsWith("Array")) {
testFunction(func);
++testedFunctions;
}
Expand Down
Expand Up @@ -50,7 +50,9 @@ public void test() throws Exception {
String className = func.getClass().getName();
// We test all generated functions except
// record and cast functions, which requires type settings (we test them in runtime tests).
if (className.contains("Gen") && !className.contains("record") && !className.contains("Cast")) {
String[] splits = className.split("\\.");
if (className.contains("Gen") && !className.contains("record") && !className.contains("Cast")
&& !splits[splits.length - 1].startsWith("Array")) {
testFunction(func);
++testedFunctions;
}
Expand Down
@@ -0,0 +1,56 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/

drop dataverse TinySocial if exists;
create dataverse TinySocial;

use TinySocial;


create type TinySocial.TwitterUserType as
{
`screen-name` : string,
lang : string,
friends_count : bigint,
statuses_count : bigint,
name : string,
followers_count : bigint
};

create type TinySocial.TweetMessageType as
closed {
tweetid : string,
user : TwitterUserType,
`sender-location` : point?,
`send-time` : datetime,
`referred-topics` : {{string}},
`message-text` : string
};

create type t1 AS {

};

create type t2 AS {
id: int,
compType: t1
};

create dataset TweetMessages(TweetMessageType) primary key tweetid hints (`CARDINALITY`=`100`);
create dataset d1(t2) primary key id;
@@ -0,0 +1,27 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/

use TinySocial;

load dataset TweetMessages using localfs ((`path`=`asterix_nc1://data/tinysocial/twm.adm`),(`format`=`adm`));

insert into d1([
{"id":1, "compType":{"sth":33}},
{"id":2, "compType":{"sth":44}}
]);
@@ -0,0 +1,31 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/

use TinySocial;

{
"t1": (select array_append(t.`referred-topics`, "sth", 5) from TweetMessages t order by t.tweetid),
"t2": (select array_append([3, "John"], (select value v.compType from d1 v))),
"t3": (select array_append([3], 7, null, missing)),
"t4": (select array_append("non_array", 5)),
"t5": (select array_append("non_array", 5, missing)),
"t6": (select array_append([], 5, 10, 12.0, "sth")),
"t7": (select array_append(missing, 3, 9)),
"t8": (select array_append(null, 3, 9))
};
@@ -0,0 +1,20 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/

drop dataverse TinySocial;
@@ -0,0 +1 @@
{ "t1": [ { "$1": {{ "t-mobile", "customization", "sth", 5 }} }, { "$1": {{ "verizon", "voice-clarity", "sth", 5 }} }, { "$1": {{ "iphone", "platform", "sth", 5 }} }, { "$1": {{ "samsung", "voice-command", "sth", 5 }} }, { "$1": {{ "verizon", "shortcut-menu", "sth", 5 }} }, { "$1": {{ "motorola", "speed", "sth", 5 }} }, { "$1": {{ "sprint", "voice-command", "sth", 5 }} }, { "$1": {{ "motorola", "speed", "sth", 5 }} }, { "$1": {{ "iphone", "voice-clarity", "sth", 5 }} }, { "$1": {{ "samsung", "platform", "sth", 5 }} }, { "$1": {{ "t-mobile", "shortcut-menu", "sth", 5 }} }, { "$1": {{ "verizon", "voicemail-service", "sth", 5 }} } ], "t2": [ { "$2": [ 3, "John", [ { "sth": 33 }, { "sth": 44 } ] ] } ], "t3": [ { } ], "t4": [ { "$4": null } ], "t5": [ { } ], "t6": [ { "$6": [ 5, 10, 12.0, "sth" ] } ], "t7": [ { } ], "t8": [ { "$8": null } ] }
Expand Up @@ -982,6 +982,13 @@
</compilation-unit>
</test-case>
</test-group>
<test-group name="array_fun">
<test-case FilePath="array_fun">
<compilation-unit name="array_append">
<output-dir compare="Text">array_append</output-dir>
</compilation-unit>
</test-case>
</test-group>
<test-group name="boolean">
<test-case FilePath="boolean">
<compilation-unit name="and_01">
Expand Down
Expand Up @@ -167,6 +167,7 @@ public class ErrorCode {
public static final int INDEX_EXISTS = 1084;
public static final int TYPE_EXISTS = 1085;
public static final int PARAMETER_NO_VALUE = 1086;
public static final int COMPILATION_INVALID_NUM_OF_ARGS = 1087;

// Feed errors
public static final int DATAFLOW_ILLEGAL_STATE = 3001;
Expand Down
Expand Up @@ -154,6 +154,7 @@
1084 = An index with this name %1$s already exists
1085 = A datatype with this name %1$s already exists
1086 = No value for parameter: %1$s
1087 = Invalid number of arguments: at least %1$d arguments are required for function %2$s

# Feed Errors
3001 = Illegal state.
Expand Down
Expand Up @@ -53,6 +53,7 @@
import org.apache.asterix.om.typecomputer.impl.AUUIDTypeComputer;
import org.apache.asterix.om.typecomputer.impl.AYearMonthDurationTypeComputer;
import org.apache.asterix.om.typecomputer.impl.AnyTypeComputer;
import org.apache.asterix.om.typecomputer.impl.ArrayAppendTypeComputer;
import org.apache.asterix.om.typecomputer.impl.BooleanFunctionTypeComputer;
import org.apache.asterix.om.typecomputer.impl.BooleanOnlyTypeComputer;
import org.apache.asterix.om.typecomputer.impl.BooleanOrMissingTypeComputer;
Expand Down Expand Up @@ -135,15 +136,12 @@ public enum SpatialFilterKind {
}

private static final FunctionInfoRepository registeredFunctions = new FunctionInfoRepository();

private static final Map<IFunctionInfo, ATypeHierarchy.Domain> registeredFunctionsDomain = new HashMap<>();

// it is supposed to be an identity mapping
private static final Map<IFunctionInfo, IFunctionInfo> builtinPublicFunctionsSet = new HashMap<>();
private static final Map<IFunctionInfo, IFunctionInfo> builtinPrivateFunctionsSet = new HashMap<>();

private static final Map<IFunctionInfo, IResultTypeComputer> funTypeComputer = new HashMap<>();

private static final Set<IFunctionInfo> builtinAggregateFunctions = new HashSet<>();
private static final Map<IFunctionInfo, IFunctionToDataSourceRewriter> datasourceFunctions = new HashMap<>();
private static final Set<IFunctionInfo> similarityFunctions = new HashSet<>();
Expand All @@ -164,14 +162,12 @@ public enum SpatialFilterKind {
new FunctionIdentifier(FunctionConstants.ASTERIX_NS, "get-handle", 2);
public static final FunctionIdentifier GET_DATA =
new FunctionIdentifier(FunctionConstants.ASTERIX_NS, "get-data", 2);

public static final FunctionIdentifier GET_ITEM =
new FunctionIdentifier(FunctionConstants.ASTERIX_NS, "get-item", 2);
public static final FunctionIdentifier ANY_COLLECTION_MEMBER =
new FunctionIdentifier(FunctionConstants.ASTERIX_NS, "any-collection-member", 1);
public static final FunctionIdentifier LISTIFY = new FunctionIdentifier(FunctionConstants.ASTERIX_NS, "listify", 1);
public static final FunctionIdentifier LEN = new FunctionIdentifier(FunctionConstants.ASTERIX_NS, "len", 1);

public static final FunctionIdentifier CONCAT_NON_NULL =
new FunctionIdentifier(FunctionConstants.ASTERIX_NS, "concat-non-null", FunctionIdentifier.VARARGS);
public static final FunctionIdentifier EMPTY_STREAM =
Expand All @@ -186,6 +182,10 @@ public enum SpatialFilterKind {
public static final FunctionIdentifier DEEP_EQUAL =
new FunctionIdentifier(FunctionConstants.ASTERIX_NS, "deep-equal", 2);

// array functions
public static final FunctionIdentifier ARRAY_APPEND =
new FunctionIdentifier(FunctionConstants.ASTERIX_NS, "array-append", FunctionIdentifier.VARARGS);

// objects
public static final FunctionIdentifier RECORD_MERGE =
new FunctionIdentifier(FunctionConstants.ASTERIX_NS, "object-merge", 2);
Expand Down Expand Up @@ -300,7 +300,8 @@ public enum SpatialFilterKind {
new FunctionIdentifier(FunctionConstants.ASTERIX_NS, "find-binary", 2);
public static final FunctionIdentifier FIND_BINARY_FROM =
new FunctionIdentifier(FunctionConstants.ASTERIX_NS, "find-binary", 3);
// String funcitons

// String functions
public static final FunctionIdentifier STRING_EQUAL =
new FunctionIdentifier(FunctionConstants.ASTERIX_NS, "string-equal", 2);
public static final FunctionIdentifier STRING_MATCHES =
Expand Down Expand Up @@ -392,6 +393,7 @@ public enum SpatialFilterKind {
public static final FunctionIdentifier MAKE_FIELD_NAME_HANDLE =
new FunctionIdentifier(FunctionConstants.ASTERIX_NS, "make-field-name-handle", 1);

// aggregate functions
public static final FunctionIdentifier AVG = new FunctionIdentifier(FunctionConstants.ASTERIX_NS, "agg-avg", 1);
public static final FunctionIdentifier COUNT = new FunctionIdentifier(FunctionConstants.ASTERIX_NS, "agg-count", 1);
public static final FunctionIdentifier SUM = new FunctionIdentifier(FunctionConstants.ASTERIX_NS, "agg-sum", 1);
Expand Down Expand Up @@ -444,7 +446,6 @@ public enum SpatialFilterKind {
new FunctionIdentifier(FunctionConstants.ASTERIX_NS, "intermediate-avg-serial", 1);

// distinct aggregate functions

public static final FunctionIdentifier COUNT_DISTINCT =
new FunctionIdentifier(FunctionConstants.ASTERIX_NS, "agg-count-distinct", 1);
public static final FunctionIdentifier SCALAR_COUNT_DISTINCT =
Expand Down Expand Up @@ -543,14 +544,15 @@ public enum SpatialFilterKind {
public static final FunctionIdentifier SCALAR_SQL_MIN_DISTINCT =
new FunctionIdentifier(FunctionConstants.ASTERIX_NS, "sql-min-distinct", 1);

// unnesting functions
public static final FunctionIdentifier SCAN_COLLECTION =
new FunctionIdentifier(FunctionConstants.ASTERIX_NS, "scan-collection", 1);
public static final FunctionIdentifier SUBSET_COLLECTION =
new FunctionIdentifier(FunctionConstants.ASTERIX_NS, "subset-collection", 3);

public static final FunctionIdentifier RANGE = new FunctionIdentifier(FunctionConstants.ASTERIX_NS, "range", 2);

// fuzzy functions:
// fuzzy functions
public static final FunctionIdentifier FUZZY_EQ =
new FunctionIdentifier(FunctionConstants.ASTERIX_NS, "fuzzy-eq", 2);

Expand Down Expand Up @@ -1093,7 +1095,7 @@ public static FunctionInfo lookupFunction(FunctionIdentifier fid) {
addPrivateFunction(OR, BooleanFunctionTypeComputer.INSTANCE, true);
addPrivateFunction(NUMERIC_ADD, NumericAddSubMulDivTypeComputer.INSTANCE, true);

// Deep equality
// deep equality
addFunction(DEEP_EQUAL, BooleanFunctionTypeComputer.INSTANCE, true);

// and then, Asterix builtin functions
Expand Down Expand Up @@ -1451,6 +1453,9 @@ public static FunctionInfo lookupFunction(FunctionIdentifier fid) {
addPrivateFunction(UNORDERED_LIST_CONSTRUCTOR, UnorderedListConstructorTypeComputer.INSTANCE, true);
addFunction(WORD_TOKENS, OrderedListOfAStringTypeComputer.INSTANCE, true);

// array functions
addFunction(ARRAY_APPEND, ArrayAppendTypeComputer.INSTANCE, true);

// objects
addFunction(RECORD_MERGE, RecordMergeTypeComputer.INSTANCE, true);
addFunction(RECORD_CONCAT, OpenARecordTypeComputer.INSTANCE, true);
Expand Down
Expand Up @@ -68,7 +68,8 @@ protected void checkArgType(String funcName, int argIndex, IAType type, SourceLo
* @param expr
* the expression under consideration.
* @param strippedInputTypes,
* the stripped input types.
* the stripped input types. When the function propagates null & missing, they can be any type except
* null, missing, and union.
* @return the result type without considering optional types.
* @throws AlgebricksException
*/
Expand Down
@@ -0,0 +1,54 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/

package org.apache.asterix.om.typecomputer.impl;

import org.apache.asterix.common.exceptions.CompilationException;
import org.apache.asterix.common.exceptions.ErrorCode;
import org.apache.asterix.om.pointables.base.DefaultOpenFieldType;
import org.apache.asterix.om.typecomputer.base.AbstractResultTypeComputer;
import org.apache.asterix.om.types.ATypeTag;
import org.apache.asterix.om.types.BuiltinType;
import org.apache.asterix.om.types.IAType;
import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
import org.apache.hyracks.algebricks.core.algebra.base.ILogicalExpression;
import org.apache.hyracks.algebricks.core.algebra.expressions.AbstractFunctionCallExpression;

public class ArrayAppendTypeComputer extends AbstractResultTypeComputer {

public static final ArrayAppendTypeComputer INSTANCE = new ArrayAppendTypeComputer();

@Override
protected IAType getResultType(ILogicalExpression expr, IAType... strippedInputTypes) throws AlgebricksException {
if (strippedInputTypes.length < 2) {
String functionName = ((AbstractFunctionCallExpression) expr).getFunctionIdentifier().getName();
throw new CompilationException(ErrorCode.COMPILATION_INVALID_NUM_OF_ARGS, expr.getSourceLocation(), 2,
functionName);
}
// type tag at [0] should be array or multiset.
ATypeTag typeTag = strippedInputTypes[0].getTypeTag();
if (typeTag == ATypeTag.ARRAY) {
return DefaultOpenFieldType.NESTED_OPEN_AORDERED_LIST_TYPE;
} else if (typeTag == ATypeTag.MULTISET) {
return DefaultOpenFieldType.NESTED_OPEN_AUNORDERED_LIST_TYPE;
} else {
return BuiltinType.ANY;
}
}
}

0 comments on commit 5afd9fd

Please sign in to comment.