[HIVEMALL-145] Merge Brickhouse functions

NOTICE

@@ -10,4 +10,4 @@ the following organizations and individuals:
  - Copyright 2013-2015 National Institute of Advanced Industrial Science and Technology (AIST)
  - Copyright 2015-2016 Makoto Yui
  - Copyright 2015-2016 Treasure Data, Inc.
- - Copyright 2012 Klout, Inc.
+ - Copyright 2012 Klout, Inc.

bin/update_func_md.sh

@@ -0,0 +1,47 @@
+#!/bin/sh
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+if [ "$HIVEMALL_HOME" = "" ]; then
+    if [ -e ../bin/${0##*/} ]; then
+	HIVEMALL_HOME=".."
+    elif [ -e ./bin/${0##*/} ]; then
+	HIVEMALL_HOME="."
+    else
+	echo "env HIVEMALL_HOME not defined"
+	exit 1
+    fi
+fi
+
+cd $HIVEMALL_HOME
+HIVEMALL_HOME=`pwd`
+
+# Deploy to local Maven repos
+
+export MAVEN_OPTS=-XX:MaxPermSize=256m
+mvn clean install -DskipTests=true -Dmaven.test.skip=true -pl '.,core,nlp,xgboost,tools/hivemall-docs'
+
+# Generate docs
+
+mvn org.apache.hivemall:hivemall-docs:generate-funcs-list -pl '.,core,nlp,xgboost,tools/hivemall-docs' -X
+
+# Run HTTP server on localhost:040
+
+cd ${HIVEMALL_HOME}/docs/gitbook
+gitbook install && gitbook serve

core/src/main/java/hivemall/HivemallVersionUDF.java

@@ -24,7 +24,7 @@
 import org.apache.hadoop.io.Text;
 
 @Description(name = "hivemall_version", value = "_FUNC_() - Returns the version of Hivemall",
-        extended = "Usage: SELECT hivemall_version();")
+        extended = "SELECT hivemall_version();")
 @UDFType(deterministic = true, stateful = false)
 public final class HivemallVersionUDF extends UDF {
 

core/src/main/java/hivemall/smile/classification/DecisionTree.java

@@ -19,6 +19,7 @@
 
 import static hivemall.smile.utils.SmileExtUtils.resolveFeatureName;
 import static hivemall.smile.utils.SmileExtUtils.resolveName;
+
 import hivemall.annotations.VisibleForTesting;
 import hivemall.math.matrix.Matrix;
 import hivemall.math.matrix.ints.ColumnMajorIntMatrix;
@@ -36,6 +37,8 @@
 import hivemall.utils.lang.StringUtils;
 import hivemall.utils.lang.mutable.MutableInt;
 import hivemall.utils.sampling.IntReservoirSampler;
+import smile.classification.Classifier;
+import smile.math.Math;
 
 import java.io.Externalizable;
 import java.io.IOException;
@@ -53,9 +56,6 @@
 import org.roaringbitmap.IntConsumer;
 import org.roaringbitmap.RoaringBitmap;
 
-import smile.classification.Classifier;
-import smile.math.Math;
-
 /**
  * Decision tree for classification. A decision tree can be learned by splitting the training set
  * into subsets based on an attribute value test. This process is repeated on each derived subset in

core/src/main/java/hivemall/smile/regression/RegressionTree.java

@@ -18,6 +18,7 @@
 package hivemall.smile.regression;
 
 import static hivemall.smile.utils.SmileExtUtils.resolveFeatureName;
+
 import hivemall.annotations.VisibleForTesting;
 import hivemall.math.matrix.Matrix;
 import hivemall.math.matrix.ints.ColumnMajorIntMatrix;
@@ -36,6 +37,10 @@
 import hivemall.utils.lang.StringUtils;
 import hivemall.utils.lang.mutable.MutableInt;
 import hivemall.utils.math.MathUtils;
+import smile.math.Math;
+import smile.regression.GradientTreeBoost;
+import smile.regression.RandomForest;
+import smile.regression.Regression;
 
 import java.io.Externalizable;
 import java.io.IOException;
@@ -51,11 +56,6 @@
 
 import org.apache.hadoop.hive.ql.metadata.HiveException;
 
-import smile.math.Math;
-import smile.regression.GradientTreeBoost;
-import smile.regression.RandomForest;
-import smile.regression.Regression;
-
 /**
  * Decision tree for regression. A decision tree can be learned by splitting the training set into
  * subsets based on an attribute value test. This process is repeated on each derived subset in a

core/src/main/java/hivemall/smile/tools/TreePredictUDF.java

@@ -158,7 +158,7 @@ public Object evaluate(@Nonnull DeferredObject[] arguments) throws HiveException
 
         Object arg2 = arguments[2].get();
         if (arg2 == null) {
-            throw new HiveException("array<double> features was null");
+            throw new HiveException("features was null");
         }
         this.featuresProbe = parseFeatures(arg2, featuresProbe);
 

core/src/main/java/hivemall/tools/GenerateSeriesUDTF.java

@@ -21,77 +21,187 @@
 import hivemall.utils.hadoop.HiveUtils;
 
 import java.util.ArrayList;
+import java.util.List;
+
+import javax.annotation.Nonnull;
+import javax.annotation.Nullable;
 
 import org.apache.hadoop.hive.ql.exec.Description;
 import org.apache.hadoop.hive.ql.exec.UDFArgumentException;
 import org.apache.hadoop.hive.ql.metadata.HiveException;
 import org.apache.hadoop.hive.ql.udf.generic.GenericUDTF;
 import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
 import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory;
+import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector;
 import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector;
 import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils;
+import org.apache.hadoop.io.IntWritable;
+import org.apache.hadoop.io.LongWritable;
+import org.apache.hadoop.io.Writable;
 
+// @formatter:off
 @Description(name = "generate_series",
         value = "_FUNC_(const int|bigint start, const int|bigint end) - "
-                + "Generate a series of values, from start to end. A similar function to PostgreSQL's `generate_serics`. http://www.postgresql.org/docs/current/static/functions-srf.html",
-        extended = "select generate_series(1,9);\n\n" + "1\n" + "2\n" + "3\n" + "4\n" + "5\n"
-                + "6\n" + "7\n" + "8\n" + "9")
+                + "Generate a series of values, from start to end. " + 
+                "A similar function to PostgreSQL's [generate_serics](http://www.postgresql.org/docs/current/static/functions-srf.html)",
+        extended = "SELECT generate_series(2,4);\n" + 
+                "\n" + 
+                " 2\n" + 
+                " 3\n" + 
+                " 4\n" + 
+                "\n" + 
+                "SELECT generate_series(5,1,-2);\n" + 
+                "\n" + 
+                " 5\n" + 
+                " 3\n" + 
+                " 1\n" + 
+                "\n" + 
+                "SELECT generate_series(4,3);\n" + 
+                "\n" + 
+                " (no return)\n" + 
+                "\n" + 
+                "SELECT date_add(current_date(),value),value from (SELECT generate_series(1,3)) t;\n" + 
+                "\n" + 
+                " 2018-04-21      1\n" + 
+                " 2018-04-22      2\n" + 
+                " 2018-04-23      3\n" + 
+                "\n" + 
+                "WITH input as (\n" + 
+                " SELECT 1 as c1, 10 as c2, 3 as step\n" + 
+                " UNION ALL\n" + 
+                " SELECT 10, 2, -3\n" + 
+                ")\n" + 
+                "SELECT generate_series(c1, c2, step) as series\n" + 
+                "FROM input;\n" + 
+                "\n" + 
+                " 1\n" + 
+                " 4\n" + 
+                " 7\n" + 
+                " 10\n" + 
+                " 10\n" + 
+                " 7\n" + 
+                " 4")
+// @formatter:on
 public final class GenerateSeriesUDTF extends GenericUDTF {
 
-    private long start, end;
-    private boolean useBigInt;
+    private PrimitiveObjectInspector startOI, endOI;
+    @Nullable
+    private PrimitiveObjectInspector stepOI;
+
+    @Nonnull
+    private final Writable[] row = new Writable[1];
+    private boolean returnLong;
 
     @Override
     public StructObjectInspector initialize(ObjectInspector[] argOIs) throws UDFArgumentException {
-        if (argOIs.length != 2) {
-            throw new UDFArgumentException("Expected number of arguments is 2: " + argOIs.length);
+        if (argOIs.length != 2 && argOIs.length != 3) {
+            throw new UDFArgumentException(
+                "Expected number of arguments is 2 or 3: " + argOIs.length);
+        }
+        if (!HiveUtils.isIntegerOI(argOIs[0])) {
+            throw new UDFArgumentException(
+                "Expected Integer type for the first argument: " + argOIs[0].getTypeName());
+        }
+        if (!HiveUtils.isIntegerOI(argOIs[1])) {
+            throw new UDFArgumentException(
+                "Expected Integer type for the second argument: " + argOIs[1].getTypeName());
         }
+        this.startOI = HiveUtils.asIntegerOI(argOIs[0]);
+        this.endOI = HiveUtils.asIntegerOI(argOIs[1]);
 
-        ArrayList<String> fieldNames = new ArrayList<String>(1);
+        if (argOIs.length == 3) {
+            if (!HiveUtils.isIntegerOI(argOIs[2])) {
+                throw new UDFArgumentException(
+                    "Expected Integer type for the third argument: " + argOIs[2].getTypeName());
+            }
+            this.stepOI = HiveUtils.asIntegerOI(argOIs[2]);
+        }
+
+        this.returnLong = HiveUtils.isBigIntOI(startOI) || HiveUtils.isBigIntOI(endOI);
+
+        List<String> fieldNames = new ArrayList<>(1);
         fieldNames.add("value");
-        ArrayList<ObjectInspector> fieldOIs = new ArrayList<ObjectInspector>(1);
+        List<ObjectInspector> fieldOIs = new ArrayList<>(1);
+        if (returnLong) {
+            fieldOIs.add(PrimitiveObjectInspectorFactory.writableLongObjectInspector);
+        } else {
+            fieldOIs.add(PrimitiveObjectInspectorFactory.writableIntObjectInspector);
+        }
+        return ObjectInspectorFactory.getStandardStructObjectInspector(fieldNames, fieldOIs);
+    }
 
-        this.useBigInt = HiveUtils.isBigIntOI(argOIs[1]);
-        if (useBigInt) {
-            fieldOIs.add(PrimitiveObjectInspectorFactory.javaLongObjectInspector);
+    @Override
+    public void process(Object[] args) throws HiveException {
+        if (returnLong) {
+            generateLongSeries(args);
         } else {
-            fieldOIs.add(PrimitiveObjectInspectorFactory.javaIntObjectInspector);
+            generateIntSeries(args);
         }
+    }
 
-        this.start = HiveUtils.getAsConstLong(argOIs[0]);
-        this.end = HiveUtils.getAsConstLong(argOIs[1]);
-        if (start > end) {
-            throw new UDFArgumentException(
-                "start '" + start + "' must be less than or equals to end '" + end + "'");
+    private void generateLongSeries(@Nonnull final Object[] args) throws HiveException {
+        final long start, end;
+        long step = 1L;
+        switch (args.length) {
+            case 3:
+                step = PrimitiveObjectInspectorUtils.getLong(args[2], stepOI);
+                if (step == 0) {
+                    throw new UDFArgumentException("Step MUST NOT be zero");
+                }
+                // fall through
+            case 2:
+                start = PrimitiveObjectInspectorUtils.getLong(args[0], startOI);
+                end = PrimitiveObjectInspectorUtils.getLong(args[1], endOI);
+                break;
+            default:
+                throw new UDFArgumentException("Expected number of arguments: " + args.length);
         }
 
-        return ObjectInspectorFactory.getStandardStructObjectInspector(fieldNames, fieldOIs);
+        final LongWritable row0 = new LongWritable();
+        row[0] = row0;
+        if (step > 0) {
+            for (long i = start; i <= end; i += step) {
+                row0.set(i);
+                forward(row);
+            }
+        } else {
+            for (long i = start; i >= end; i += step) {
+                row0.set(i);
+                forward(row);
+            }
+        }
     }
 
-    @Override
-    public void process(Object[] argOIs) throws HiveException {
-        final Object[] forwardObjs = new Object[1];
-        if (useBigInt) {
-            if (start == end) {
-                forwardObjs[0] = start;
-                forward(forwardObjs);
-            } else {
-                for (long i = start; i <= end; i++) {
-                    forwardObjs[0] = i;
-                    forward(forwardObjs);
+    private void generateIntSeries(@Nonnull final Object[] args) throws HiveException {
+        final int start, end;
+        int step = 1;
+        switch (args.length) {
+            case 3:
+                step = PrimitiveObjectInspectorUtils.getInt(args[2], stepOI);
+                if (step == 0) {
+                    throw new UDFArgumentException("Step MUST NOT be zero");
                 }
+                // fall through
+            case 2:
+                start = PrimitiveObjectInspectorUtils.getInt(args[0], startOI);
+                end = PrimitiveObjectInspectorUtils.getInt(args[1], endOI);
+                break;
+            default:
+                throw new UDFArgumentException("Expected number of arguments: " + args.length);
+        }
+
+        final IntWritable row0 = new IntWritable();
+        row[0] = row0;
+        if (step > 0) {
+            for (int i = start; i <= end; i += step) {
+                row0.set(i);
+                forward(row);
             }
         } else {
-            int starti = (int) start;
-            int endi = (int) end;
-            if (starti == endi) {
-                forwardObjs[0] = starti;
-                forward(forwardObjs);
-            } else {
-                for (int i = starti; i <= endi; i++) {
-                    forwardObjs[0] = i;
-                    forward(forwardObjs);
-                }
+            for (int i = start; i >= end; i += step) {
+                row0.set(i);
+                forward(row);
             }
         }
     }

core/src/main/java/hivemall/tools/TryCastUDF.java

@@ -35,8 +35,8 @@
 @Description(name = "try_cast",
         value = "_FUNC_(ANY src, const string typeName)"
                 + " - Explicitly cast a value as a type. Returns null if cast fails.",
-        extended = "Usage: select try_cast(array(1.0,2.0,3.0), 'array<string>')\n"
-                + "     select try_cast(map('A',10,'B',20,'C',30), 'map<string,double>')")
+        extended = "SELECT try_cast(array(1.0,2.0,3.0), 'array<string>')\n"
+                + "SELECT try_cast(map('A',10,'B',20,'C',30), 'map<string,double>')")
 @UDFType(deterministic = true, stateful = false)
 public final class TryCastUDF extends GenericUDF {