Skip to content
This repository has been archived by the owner on Sep 20, 2022. It is now read-only.

[HIVEMALL-145] Merge Brickhouse functions #135

Closed
wants to merge 56 commits into from
Closed
Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
56 commits
Select commit Hold shift + click to select a range
c19fd7c
Added array UDFs (array_append, element_at, array_union, first_elemen…
myui Mar 20, 2018
8f0aeec
Added array_flatten UDF
myui Mar 20, 2018
5d386e0
Added try_cast UDF
myui Mar 20, 2018
391472c
Added to_json/from_json UDFs
myui Mar 27, 2018
56ced64
Added MovingAverage class
myui Apr 3, 2018
c23cc60
Moved MovingAverage and OnlineVariance to hivemall.utils.stats
myui Apr 4, 2018
b127d0d
Added moving_avg UDTF
myui Apr 4, 2018
a713d9b
Added conditional_emit UDTF
myui Apr 5, 2018
d1b1f09
Added array_slice UDF
myui Apr 5, 2018
da9f0fa
Aded vector_add and vector_dot UDFs
myui Apr 6, 2018
0c79958
Fixed possible serialization errors
myui Apr 10, 2018
67615f7
Added Jerome to the Committer list
myui Apr 10, 2018
53c279d
Fixed a bug in vector_dot UDF
myui Apr 10, 2018
5bebe90
Fixed an error message of tree_predict
myui Apr 12, 2018
727775a
Fix array_append UDF behavior
myui Apr 12, 2018
49bf31a
Added map_key_values UDF
myui Apr 18, 2018
98c44da
Fixed UDFType of moving_avg UDF
myui Apr 20, 2018
1466de3
Added sessionize UDF
myui Apr 20, 2018
7b40218
Refactored generate_series UDTF to be more flexible
myui Apr 20, 2018
6bc0948
Added a new line in EoF
myui Apr 24, 2018
65fdffb
Added merge_maps UDAF
myui Apr 24, 2018
443f49c
Applied formatter
myui Apr 26, 2018
9ce97d1
Fixed warning for duplicate entry
myui Apr 27, 2018
3efefc4
Applied spotless-maven-plugin formatter
myui Apr 27, 2018
acea0f7
Fixed SSL related test error
myui May 14, 2018
b47468d
Moved package of moving_avg
myui May 14, 2018
052e45d
Updated UDF description of to_json UDF
myui May 14, 2018
8436dbe
Included timeseries doc generation
myui May 14, 2018
2e293a1
Updated function usage doc
myui May 16, 2018
d11735b
Add a script to generate function desc
myui May 17, 2018
6a507d4
Fixed markdown generation scheme for UDF descriptions
myui May 21, 2018
533140b
Fixed UDF descriptions
myui May 21, 2018
00a7e49
Added map_include_keys and map_exclude_keys UDFs
myui May 24, 2018
b7fd6a9
Updated DDLs
myui May 24, 2018
2982865
Added array_to_str UDF
myui May 25, 2018
91c1013
Added map_index UDF
myui May 25, 2018
85cc27f
Updated DDLs for map_key_values
myui May 25, 2018
bf70541
Updated DDLs
myui May 29, 2018
aae2450
Added assert and raise_error UDFs
myui May 31, 2018
09b246d
Fixed unit tests for raise_error UDF
myui May 31, 2018
099673f
Added moving_avg UDTF
myui May 31, 2018
7661efc
Fixed a typo in assert UDF
myui May 31, 2018
7726c8b
Added vector_add, vector_dot UDFs
myui May 31, 2018
1263b9a
Renamed BloomUDAF to BloomFilterUDAF
myui Jun 1, 2018
9ebf51a
Added DDLs for bloom filter
myui Jun 1, 2018
60be082
Fixed NPE bug in bloom UDAF
myui Jun 1, 2018
68e9511
Added bloom_contains_any UDF
myui Jun 1, 2018
2d95334
Supported contains all in bloom_contains
myui Jun 1, 2018
78c2dbd
Updated DDLs for bloom filter UDFs
myui Jun 1, 2018
512f930
Fixed bugs in bloom filter UDFs
myui Jun 5, 2018
e21606c
Updated UDF documents
myui Jun 5, 2018
906009a
Fixed DDLs for array_to_str UDF
myui Jun 6, 2018
e6bd7a5
formatted TD DDLs
myui Jun 6, 2018
bf81830
Formatted DDLs
myui Jun 6, 2018
5d73101
Updated DDL usage in tutorial
myui Jun 6, 2018
849940c
Updated user guide for bloom filter
myui Jun 6, 2018
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
6 changes: 4 additions & 2 deletions core/src/main/java/hivemall/tools/sanity/AssertUDF.java
Expand Up @@ -25,8 +25,10 @@

@Description(name = "assert",
value = "_FUNC_(boolean condition) or _FUNC_(boolean condition, string errMsg)"
+ "- Throws HiveException if condition is not met")
@UDFType(deterministic = true, stateful = false)
+ "- Throws HiveException if condition is not met",
extended = "SELECT count(1) FROM stock_price WHERE assert(price > 0.0);\n"
+ "SELECT count(1) FROM stock_price WHRE assert(price > 0.0, 'price MUST be more than 0.0')")
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

typo s/WHRE/WHERE/

@UDFType(deterministic = false, stateful = false)
public final class AssertUDF extends UDF {

public boolean evaluate(boolean condition) throws HiveException {
Expand Down
43 changes: 35 additions & 8 deletions core/src/main/java/hivemall/tools/sanity/RaiseErrorUDF.java
Expand Up @@ -18,21 +18,48 @@
*/
package hivemall.tools.sanity;

import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.hive.ql.exec.Description;
import org.apache.hadoop.hive.ql.exec.UDF;
import org.apache.hadoop.hive.ql.exec.UDFArgumentException;
import org.apache.hadoop.hive.ql.exec.UDFArgumentLengthException;
import org.apache.hadoop.hive.ql.metadata.HiveException;
import org.apache.hadoop.hive.ql.udf.UDFType;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDF;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory;

@Description(name = "raise_error", value = "_FUNC_() or _FUNC_(string msg) - Throws an error")
@UDFType(deterministic = true, stateful = false)
public final class RaiseErrorUDF extends UDF {
@Description(name = "raise_error", value = "_FUNC_() or _FUNC_(string msg) - Throws an error",
extended = "SELECT product_id, price, raise_error('Found an invalid record') FROM xxx WHERE price < 0.0")
@UDFType(deterministic = false, stateful = false)
public class RaiseErrorUDF extends GenericUDF {

public boolean evaluate() throws HiveException {
throw new HiveException();
@Override
public ObjectInspector initialize(ObjectInspector[] argOIs) throws UDFArgumentException {
if (argOIs.length != 0 && argOIs.length != 1) {
throw new UDFArgumentLengthException(
"Expected one or two arguments for raise_error UDF: " + argOIs.length);
}

return PrimitiveObjectInspectorFactory.writableBooleanObjectInspector;
}

@Override
public Object evaluate(DeferredObject[] arguments) throws HiveException {
if (arguments.length == 1) {
Object arg0 = arguments[0].get();
if (arg0 == null) {
throw new HiveException();
}
String msg = arg0.toString();
throw new HiveException(msg);
} else {
throw new HiveException();
}
}

public boolean evaluate(String errorMessage) throws HiveException {
throw new HiveException(errorMessage);
@Override
public String getDisplayString(String[] children) {
return "raise_error(" + StringUtils.join(children, ',') + ')';
}

}
6 changes: 6 additions & 0 deletions resources/ddl/define-all-as-permanent.hive
Expand Up @@ -812,3 +812,9 @@ CREATE FUNCTION to_json as 'hivemall.tools.json.ToJsonUDF' USING JAR '${hivemall

DROP FUNCTION IF EXISTS from_json;
CREATE FUNCTION from_json as 'hivemall.tools.json.FromJsonUDF' USING JAR '${hivemall_jar}';

DROP FUNCTION IF EXISTS assert;
CREATE FUNCTION assert as 'hivemall.tools.sanity.AssertUDF' USING JAR '${hivemall_jar}';

DROP FUNCTION IF EXISTS raise_error;
CREATE FUNCTION raise_error as 'hivemall.tools.sanity.RaiseErrorUDF' USING JAR '${hivemall_jar}';
6 changes: 6 additions & 0 deletions resources/ddl/define-all.hive
Expand Up @@ -818,3 +818,9 @@ create temporary function to_json as 'hivemall.tools.json.ToJsonUDF';

drop temporary function if exists from_json;
create temporary function from_json as 'hivemall.tools.json.FromJsonUDF';

drop temporary function if exists assert;
create temporary function assert as 'hivemall.tools.sanity.AssertUDF';

drop temporary function if exists raise_error;
create temporary function raise_error as 'hivemall.tools.sanity.RaiseErrorUDF';
6 changes: 6 additions & 0 deletions resources/ddl/define-all.spark
Expand Up @@ -771,3 +771,9 @@ sqlContext.sql("CREATE TEMPORARY FUNCTION to_json AS 'hivemall.tools.json.ToJson

sqlContext.sql("DROP TEMPORARY FUNCTION IF EXISTS from_json")
sqlContext.sql("CREATE TEMPORARY FUNCTION from_json AS 'hivemall.tools.json.FromJsonUDF'")

sqlContext.sql("DROP TEMPORARY FUNCTION IF EXISTS assert")
sqlContext.sql("CREATE TEMPORARY FUNCTION assert AS 'hivemall.tools.sanity.AssertUDF'")

sqlContext.sql("DROP TEMPORARY FUNCTION IF EXISTS raise_error")
sqlContext.sql("CREATE TEMPORARY FUNCTION raise_error AS 'hivemall.tools.sanity.RaiseErrorUDF'")
4 changes: 4 additions & 0 deletions resources/ddl/define-udfs.td.hql
Expand Up @@ -230,3 +230,7 @@ create temporary function sessionize as 'hivemall.tools.datetime.SessionizeUDF';
create temporary function to_json as 'hivemall.tools.json.ToJsonUDF';

create temporary function from_json as 'hivemall.tools.json.FromJsonUDF';

create temporary function assert as 'hivemall.tools.sanity.AssertUDF';

create temporary function raise_error as 'hivemall.tools.sanity.RaiseErrorUDF';