Merge pull request #64 from myui/feature/distcache

Feature/distcache
myui · Jul 1, 2014 · 4647931 · 4647931
2 parents 1008ff8 + d9951b6
commit 4647931
Show file tree

Hide file tree

Showing 36 changed files with 260 additions and 34 deletions.
diff --git a/scripts/ddl/define-all.hive b/scripts/ddl/define-all.hive
@@ -294,6 +294,9 @@ create temporary function taskid as 'hivemall.tools.mapred.TaskIdUDF';
 drop temporary function rowid;
 create temporary function rowid as 'hivemall.tools.mapred.RowIdUDF';
 
+drop temporary function distcache_gets;
+create temporary function distcache_gets as 'hivemall.tools.mapred.DistributedCacheLookupUDF';
+
 ----------------------
 -- string functions --
 ----------------------

diff --git a/scripts/ddl/define-tools-udf.hive b/scripts/ddl/define-tools-udf.hive
@@ -67,6 +67,9 @@ create temporary function taskid as 'hivemall.tools.mapred.TaskIdUDF';
 drop temporary function rowid;
 create temporary function rowid as 'hivemall.tools.mapred.RowIdUDF';
 
+drop temporary function distcache_gets;
+create temporary function distcache_gets as 'hivemall.tools.mapred.DistributedCacheLookupUDF';
+
 ----------------------
 -- string functions --
 ----------------------

diff --git a/src/main/hivemall/classifier/BinaryOnlineClassifierUDTF.java b/src/main/hivemall/classifier/BinaryOnlineClassifierUDTF.java
@@ -31,6 +31,7 @@
 import hivemall.common.WeightValue;
 import hivemall.utils.collections.OpenHashMap;
 import hivemall.utils.collections.OpenHashMap.IMapIterator;
+import hivemall.utils.hadoop.HiveUtils;
 
 import java.util.ArrayList;
 import java.util.List;
@@ -47,7 +48,6 @@
 import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector;
 import org.apache.hadoop.hive.serde2.objectinspector.primitive.IntObjectInspector;
 import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory;
-import org.apache.hadoop.hive.serde2.objectinspector.primitive.WritableConstantStringObjectInspector;
 import org.apache.hadoop.io.FloatWritable;
 import org.apache.hadoop.io.Text;
 
@@ -122,7 +122,7 @@ protected CommandLine processOptions(ObjectInspector[] argOIs) throws UDFArgumen
 
         CommandLine cl = null;
         if(argOIs.length >= 3) {
-            String rawArgs = ((WritableConstantStringObjectInspector) argOIs[2]).getWritableConstantValue().toString();
+            String rawArgs = HiveUtils.getConstString(argOIs[2]);
             cl = parseOptions(rawArgs);
 
             if(cl.hasOption("fh")) {

diff --git a/src/main/hivemall/common/RandomizedAmplifier.java b/src/main/hivemall/common/RandomizedAmplifier.java
@@ -20,7 +20,7 @@
  */
 package hivemall.common;
 
-import hivemall.utils.ArrayUtils;
+import hivemall.utils.lang.ArrayUtils;
 
 import java.util.Random;
 

diff --git a/src/main/hivemall/ensemble/MaxValueLabelUDAF.java b/src/main/hivemall/ensemble/MaxValueLabelUDAF.java
@@ -20,7 +20,7 @@
  */
 package hivemall.ensemble;
 
-import hivemall.utils.WritableUtils;
+import hivemall.utils.hadoop.WritableUtils;
 
 import org.apache.hadoop.hive.ql.exec.UDAF;
 import org.apache.hadoop.hive.ql.exec.UDAFEvaluator;

diff --git a/src/main/hivemall/ensemble/bagging/VotedAvgUDAF.java b/src/main/hivemall/ensemble/bagging/VotedAvgUDAF.java
@@ -20,7 +20,7 @@
  */
 package hivemall.ensemble.bagging;
 
-import static hivemall.utils.WritableUtils.val;
+import static hivemall.utils.hadoop.WritableUtils.val;
 
 import org.apache.hadoop.hive.ql.exec.UDAF;
 import org.apache.hadoop.hive.ql.exec.UDAFEvaluator;

diff --git a/src/main/hivemall/ensemble/bagging/WeightVotedAvgUDAF.java b/src/main/hivemall/ensemble/bagging/WeightVotedAvgUDAF.java
@@ -20,7 +20,7 @@
  */
 package hivemall.ensemble.bagging;
 
-import static hivemall.utils.WritableUtils.val;
+import static hivemall.utils.hadoop.WritableUtils.val;
 
 import org.apache.hadoop.hive.ql.exec.UDAF;
 import org.apache.hadoop.hive.ql.exec.UDAFEvaluator;

diff --git a/src/main/hivemall/ftvec/AddBiasUDF.java b/src/main/hivemall/ftvec/AddBiasUDF.java
@@ -21,7 +21,7 @@
 package hivemall.ftvec;
 
 import hivemall.HivemallConstants;
-import hivemall.utils.WritableUtils;
+import hivemall.utils.hadoop.WritableUtils;
 
 import java.util.List;
 

diff --git a/src/main/hivemall/ftvec/hashing/ArrayHashValuesUDF.java b/src/main/hivemall/ftvec/hashing/ArrayHashValuesUDF.java
@@ -20,7 +20,7 @@
  */
 package hivemall.ftvec.hashing;
 
-import static hivemall.utils.WritableUtils.val;
+import static hivemall.utils.hadoop.WritableUtils.val;
 import hivemall.utils.hashing.MurmurHash3;
 
 import java.util.Arrays;

diff --git a/src/main/hivemall/ftvec/hashing/ArrayPrefixedHashValuesUDF.java b/src/main/hivemall/ftvec/hashing/ArrayPrefixedHashValuesUDF.java
@@ -20,7 +20,7 @@
  */
 package hivemall.ftvec.hashing;
 
-import static hivemall.utils.WritableUtils.val;
+import static hivemall.utils.hadoop.WritableUtils.val;
 import hivemall.utils.hashing.MurmurHash3;
 
 import java.util.Arrays;

diff --git a/src/main/hivemall/ftvec/hashing/MurmurHash3UDF.java b/src/main/hivemall/ftvec/hashing/MurmurHash3UDF.java
@@ -20,7 +20,7 @@
  */
 package hivemall.ftvec.hashing;
 
-import static hivemall.utils.WritableUtils.val;
+import static hivemall.utils.hadoop.WritableUtils.val;
 import hivemall.utils.hashing.MurmurHash3;
 
 import java.util.List;

diff --git a/src/main/hivemall/ftvec/hashing/Sha1UDF.java b/src/main/hivemall/ftvec/hashing/Sha1UDF.java
@@ -20,7 +20,7 @@
  */
 package hivemall.ftvec.hashing;
 
-import static hivemall.utils.WritableUtils.val;
+import static hivemall.utils.hadoop.WritableUtils.val;
 
 import java.security.MessageDigest;
 import java.security.NoSuchAlgorithmException;

diff --git a/src/main/hivemall/ftvec/scaling/RescaleFeatureValueUDF.java b/src/main/hivemall/ftvec/scaling/RescaleFeatureValueUDF.java
@@ -20,7 +20,7 @@
  */
 package hivemall.ftvec.scaling;
 
-import static hivemall.utils.WritableUtils.val;
+import static hivemall.utils.hadoop.WritableUtils.val;
 
 import org.apache.hadoop.hive.ql.exec.UDF;
 import org.apache.hadoop.io.Text;

diff --git a/src/main/hivemall/ftvec/scaling/RescaleUDF.java b/src/main/hivemall/ftvec/scaling/RescaleUDF.java
@@ -20,7 +20,7 @@
  */
 package hivemall.ftvec.scaling;
 
-import static hivemall.utils.WritableUtils.val;
+import static hivemall.utils.hadoop.WritableUtils.val;
 
 import org.apache.hadoop.hive.ql.exec.UDF;
 import org.apache.hadoop.io.FloatWritable;

diff --git a/src/main/hivemall/ftvec/scaling/ZScoreUDF.java b/src/main/hivemall/ftvec/scaling/ZScoreUDF.java
@@ -20,7 +20,7 @@
  */
 package hivemall.ftvec.scaling;
 
-import static hivemall.utils.WritableUtils.val;
+import static hivemall.utils.hadoop.WritableUtils.val;
 
 import org.apache.hadoop.hive.ql.exec.UDF;
 import org.apache.hadoop.io.FloatWritable;

diff --git a/src/main/hivemall/knn/distance/CosineSimilarityUDF.java b/src/main/hivemall/knn/distance/CosineSimilarityUDF.java
@@ -20,7 +20,7 @@
  */
 package hivemall.knn.distance;
 
-import static hivemall.utils.WritableUtils.val;
+import static hivemall.utils.hadoop.WritableUtils.val;
 import hivemall.common.FeatureValue;
 
 import java.util.Collections;

diff --git a/src/main/hivemall/knn/distance/HammingDistanceUDF.java b/src/main/hivemall/knn/distance/HammingDistanceUDF.java
@@ -20,7 +20,7 @@
  */
 package hivemall.knn.distance;
 
-import static hivemall.utils.WritableUtils.val;
+import static hivemall.utils.hadoop.WritableUtils.val;
 
 import java.math.BigInteger;
 import java.util.List;

diff --git a/src/main/hivemall/knn/distance/JaccardIndexUDF.java b/src/main/hivemall/knn/distance/JaccardIndexUDF.java
@@ -20,7 +20,7 @@
  */
 package hivemall.knn.distance;
 
-import static hivemall.utils.WritableUtils.val;
+import static hivemall.utils.hadoop.WritableUtils.val;
 
 import java.math.BigInteger;
 

diff --git a/src/main/hivemall/knn/distance/PopcountUDF.java b/src/main/hivemall/knn/distance/PopcountUDF.java
@@ -20,7 +20,7 @@
  */
 package hivemall.knn.distance;
 
-import static hivemall.utils.WritableUtils.val;
+import static hivemall.utils.hadoop.WritableUtils.val;
 
 import java.math.BigInteger;
 import java.util.List;

diff --git a/src/main/hivemall/knn/lsh/MinHashesUDF.java b/src/main/hivemall/knn/lsh/MinHashesUDF.java
@@ -19,7 +19,7 @@
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
  */package hivemall.knn.lsh;
 
-import static hivemall.utils.WritableUtils.val;
+import static hivemall.utils.hadoop.WritableUtils.val;
 import hivemall.common.FeatureValue;
 import hivemall.utils.hashing.MurmurHash3;
 

diff --git a/src/main/hivemall/knn/lsh/bBitMinHashUDF.java b/src/main/hivemall/knn/lsh/bBitMinHashUDF.java
@@ -19,7 +19,7 @@
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
  */package hivemall.knn.lsh;
 
-import static hivemall.utils.WritableUtils.val;
+import static hivemall.utils.hadoop.WritableUtils.val;
 import hivemall.common.FeatureValue;
 import hivemall.utils.hashing.MurmurHash3;
 

diff --git a/src/main/hivemall/regression/OnlineRegressionUDTF.java b/src/main/hivemall/regression/OnlineRegressionUDTF.java
@@ -31,6 +31,7 @@
 import hivemall.common.WeightValue;
 import hivemall.utils.collections.OpenHashMap;
 import hivemall.utils.collections.OpenHashMap.IMapIterator;
+import hivemall.utils.hadoop.HiveUtils;
 
 import java.util.ArrayList;
 import java.util.Collection;
@@ -48,7 +49,6 @@
 import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector;
 import org.apache.hadoop.hive.serde2.objectinspector.primitive.FloatObjectInspector;
 import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory;
-import org.apache.hadoop.hive.serde2.objectinspector.primitive.WritableConstantStringObjectInspector;
 import org.apache.hadoop.io.FloatWritable;
 import org.apache.hadoop.io.Text;
 
@@ -133,7 +133,7 @@ protected CommandLine processOptions(ObjectInspector[] argOIs) throws UDFArgumen
 
         CommandLine cl = null;
         if(argOIs.length >= 3) {
-            String rawArgs = ((WritableConstantStringObjectInspector) argOIs[2]).getWritableConstantValue().toString();
+            String rawArgs = HiveUtils.getConstString(argOIs[2]);
             cl = parseOptions(rawArgs);
 
             if(cl.hasOption("fh")) {

diff --git a/src/main/hivemall/tools/map/MapGetSumUDF.java b/src/main/hivemall/tools/map/MapGetSumUDF.java
@@ -20,7 +20,7 @@
  */
 package hivemall.tools.map;
 
-import static hivemall.utils.WritableUtils.val;
+import static hivemall.utils.hadoop.WritableUtils.val;
 
 import java.util.List;
 import java.util.Map;