Added quantified_features and introduces new package

`hivemall.ftvec.trans`.

scripts/ddl/define-all-as-permanent.hive

@@ -252,18 +252,9 @@ CREATE FUNCTION extract_weight as 'hivemall.ftvec.ExtractWeightUDF' USING JAR '$
 DROP FUNCTION IF EXISTS add_feature_index;
 CREATE FUNCTION add_feature_index as 'hivemall.ftvec.AddFeatureIndexUDF' USING JAR '${hivemall_jar}';
 
-DROP FUNCTION IF EXISTS vectorize_features;
-CREATE FUNCTION vectorize_features as 'hivemall.ftvec.VectorizeFeaturesUDF' USING JAR '${hivemall_jar}';
-
 DROP FUNCTION IF EXISTS feature;
 CREATE FUNCTION feature as 'hivemall.ftvec.FeatureUDF' USING JAR '${hivemall_jar}';
 
-DROP FUNCTION IF EXISTS categorical_features;
-CREATE FUNCTION categorical_features as 'hivemall.ftvec.CategoricalFeaturesUDF' USING JAR '${hivemall_jar}';
-
-DROP FUNCTION IF EXISTS indexed_features;
-CREATE FUNCTION indexed_features as 'hivemall.ftvec.IndexedFeatures' USING JAR '${hivemall_jar}';
-
 ----------------------------------
 -- feature converting functions --
 ----------------------------------
@@ -288,6 +279,22 @@ CREATE FUNCTION to_sparse as 'hivemall.ftvec.conv.ToSparseFeaturesUDF' USING JAR
 DROP FUNCTION IF EXISTS quantify;
 CREATE FUNCTION quantify as 'hivemall.ftvec.conv.QuantifyColumnsUDTF' USING JAR '${hivemall_jar}';
 
+--------------------------
+-- feature transformers --
+--------------------------
+
+DROP FUNCTION IF EXISTS vectorize_features;
+CREATE FUNCTION vectorize_features as 'hivemall.ftvec.trans.VectorizeFeaturesUDF' USING JAR '${hivemall_jar}';
+
+DROP FUNCTION IF EXISTS categorical_features;
+CREATE FUNCTION categorical_features as 'hivemall.ftvec.trans.CategoricalFeaturesUDF' USING JAR '${hivemall_jar}';
+
+DROP FUNCTION IF EXISTS indexed_features;
+CREATE FUNCTION indexed_features as 'hivemall.ftvec.trans.IndexedFeatures' USING JAR '${hivemall_jar}';
+
+DROP FUNCTION IF EXISTS quantified_features;
+CREATE FUNCTION quantified_features as 'hivemall.ftvec.trans.QuantifiedFeaturesUDTF' USING JAR '${hivemall_jar}';
+
 --------------------------
 -- ftvec/text functions --
 --------------------------

scripts/ddl/define-all.hive

@@ -248,18 +248,9 @@ create temporary function extract_weight as 'hivemall.ftvec.ExtractWeightUDF';
 drop temporary function add_feature_index;
 create temporary function add_feature_index as 'hivemall.ftvec.AddFeatureIndexUDF';
 
-drop temporary function vectorize_features;
-create temporary function vectorize_features as 'hivemall.ftvec.VectorizeFeaturesUDF';
-
 drop temporary function feature;
 create temporary function feature as 'hivemall.ftvec.FeatureUDF';
 
-drop temporary function categorical_features;
-create temporary function categorical_features as 'hivemall.ftvec.CategoricalFeaturesUDF';
-
-drop temporary function indexed_features;
-create temporary function indexed_features as 'hivemall.ftvec.IndexedFeatures';
-
 ----------------------------------
 -- feature converting functions --
 ----------------------------------
@@ -284,6 +275,22 @@ create temporary function to_sparse as 'hivemall.ftvec.conv.ToSparseFeaturesUDF'
 drop temporary function quantify;
 create temporary function quantify as 'hivemall.ftvec.conv.QuantifyColumnsUDTF';
 
+--------------------------
+-- feature transformers --
+--------------------------
+
+drop temporary function vectorize_features;
+create temporary function vectorize_features as 'hivemall.ftvec.trans.VectorizeFeaturesUDF';
+
+drop temporary function categorical_features;
+create temporary function categorical_features as 'hivemall.ftvec.trans.CategoricalFeaturesUDF';
+
+drop temporary function indexed_features;
+create temporary function indexed_features as 'hivemall.ftvec.trans.IndexedFeatures';
+
+drop temporary function quantified_features;
+create temporary function quantified_features as 'hivemall.ftvec.trans.QuantifiedFeaturesUDTF';
+
 --------------------------
 -- ftvec/text functions --
 --------------------------

src/main/java/hivemall/ftvec/CategoricalFeaturesUDF.java → src/main/java/hivemall/ftvec/trans/CategoricalFeaturesUDF.java

@@ -16,7 +16,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package hivemall.ftvec;
+package hivemall.ftvec.trans;
 
 import hivemall.utils.hadoop.HiveUtils;
 import hivemall.utils.lang.StringUtils;

src/main/java/hivemall/ftvec/IndexedFeatures.java → src/main/java/hivemall/ftvec/trans/IndexedFeatures.java

@@ -15,7 +15,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package hivemall.ftvec;
+package hivemall.ftvec.trans;
 
 import hivemall.utils.lang.StringUtils;
 

src/main/java/hivemall/ftvec/trans/QuantifiedFeaturesUDTF.java

@@ -0,0 +1,111 @@
+/*
+ * Hivemall: Hive scalable Machine Learning Library
+ *
+ * Copyright (C) 2015 Makoto YUI
+ * Copyright (C) 2013-2015 National Institute of Advanced Industrial Science and Technology (AIST)
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *         http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package hivemall.ftvec.trans;
+
+import hivemall.utils.hadoop.HiveUtils;
+import hivemall.utils.lang.Identifier;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+
+import org.apache.hadoop.hive.ql.exec.Description;
+import org.apache.hadoop.hive.ql.exec.UDFArgumentException;
+import org.apache.hadoop.hive.ql.metadata.HiveException;
+import org.apache.hadoop.hive.ql.udf.generic.GenericUDTF;
+import org.apache.hadoop.hive.serde2.io.DoubleWritable;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory;
+import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils;
+
+@Description(name = "quantified_features", value = "_FUNC_(col1, col2, ...) - Returns an identified features in a dence array<double>")
+public final class QuantifiedFeaturesUDTF extends GenericUDTF {
+
+    private PrimitiveObjectInspector[] doubleOIs;
+    private Identifier<String>[] identifiers;
+    private DoubleWritable[] columnValues;
+
+    // org.apache.hive.com.esotericsoftware.kryo.KryoException: java.lang.NullPointerException
+    private transient Object[] fowardObjs;
+
+    @SuppressWarnings("unchecked")
+    @Override
+    public StructObjectInspector initialize(ObjectInspector[] argOIs) throws UDFArgumentException {
+        int size = argOIs.length;
+        this.doubleOIs = new PrimitiveObjectInspector[size];
+        this.columnValues = new DoubleWritable[size];
+        this.identifiers = new Identifier[size];
+        this.fowardObjs = null;
+
+        for(int i = 0; i < size; i++) {
+            columnValues[i] = new DoubleWritable(Double.NaN);
+            if(HiveUtils.isNumberOI(argOIs[i])) {
+                doubleOIs[i] = HiveUtils.asDoubleCompatibleOI(argOIs[i]);
+            } else {
+                identifiers[i] = new Identifier<String>();
+            }
+        }
+
+
+        ArrayList<String> fieldNames = new ArrayList<String>();
+        ArrayList<ObjectInspector> fieldOIs = new ArrayList<ObjectInspector>();
+        fieldNames.add("features");
+        fieldOIs.add(ObjectInspectorFactory.getStandardListObjectInspector(PrimitiveObjectInspectorFactory.writableDoubleObjectInspector));
+
+        return ObjectInspectorFactory.getStandardStructObjectInspector(fieldNames, fieldOIs);
+    }
+
+    @Override
+    public void process(Object[] args) throws HiveException {
+        if(fowardObjs == null) {
+            this.fowardObjs = new Object[] { Arrays.asList(columnValues) };
+        }
+
+        final DoubleWritable[] values = this.columnValues;
+        for(int i = 0; i < args.length; i++) {
+            Object arg = args[i];
+
+            Identifier<String> identifier = identifiers[i];
+            if(identifier == null) {
+                double v = PrimitiveObjectInspectorUtils.getDouble(arg, doubleOIs[i]);
+                values[i].set(v);
+            } else {
+                if(arg == null) {
+                    throw new HiveException("Found Null in the input: " + Arrays.toString(args));
+                } else {
+                    String k = arg.toString();
+                    int id = identifier.valueOf(k);
+                    values[i].set(id);
+                }
+            }
+
+        }
+        forward(fowardObjs);
+    }
+
+    @Override
+    public void close() throws HiveException {
+        this.doubleOIs = null;
+        this.identifiers = null;
+        this.columnValues = null;
+    }
+
+}

src/main/java/hivemall/ftvec/VectorizeFeaturesUDF.java → src/main/java/hivemall/ftvec/trans/VectorizeFeaturesUDF.java

@@ -16,7 +16,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package hivemall.ftvec;
+package hivemall.ftvec.trans;
 
 import hivemall.utils.hadoop.HiveUtils;
 import hivemall.utils.lang.StringUtils;

src/main/java/hivemall/utils/hadoop/HiveUtils.java

@@ -184,6 +184,8 @@ public static boolean isNumberOI(@Nonnull final ObjectInspector argOI)
             case LONG:
             case FLOAT:
             case DOUBLE:
+            case BYTE:
+            case TIMESTAMP:
                 return true;
             default:
                 return false;

src/test/java/hivemall/ftvec/VectorizeFeaturesUDFTest.java → src/test/java/hivemall/ftvec/trans/VectorizeFeaturesUDFTest.java

@@ -16,8 +16,9 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package hivemall.ftvec;
+package hivemall.ftvec.trans;
 
+import hivemall.ftvec.trans.VectorizeFeaturesUDF;
 import hivemall.utils.hadoop.WritableUtils;
 
 import java.io.IOException;