3804789 replay starts

.github/PULL_REQUEST_TEMPLATE

@@ -23,4 +23,3 @@
 (Please remove this section if not needed; check `x` for YES, blank for NO)
 
 - [x] Did you apply source code formatter, i.e., `mvn formatter:format`, for your commit?
-- [ ] Did you run system tests on Hive (or Spark)?

.travis.yml

@@ -1,41 +1,45 @@
-sudo: false
-
+after_success:
+- mvn clean cobertura:cobertura coveralls:report
+before_cache:
+- find $TRAVIS_BUILD_DIR -name .starts | while read line; do du -sh $line;         DIR=$(dirname
+  $HOME/CACHE$line); mkdir -p $DIR; cp -r $line $DIR; done
+- find $TRAVIS_BUILD_DIR -name jdeps-cache | while read line; do du -sh $line;         DIR=$(dirname
+  $HOME/CACHE$line); mkdir -p $DIR; cp -r $line $DIR; done
+- echo CACHESIZE $(du -sh $HOME/CACHE)
+before_install:
+- find $HOME/CACHE -name jdeps-cache | while read line; do du -sh $line;         DIR=$(dirname
+  ${line#$HOME/CACHE}); cp -r $line $DIR; done
+- find $HOME/CACHE -name .starts | while read line; do du -sh $line;         DIR=$(dirname
+  ${line#$HOME/CACHE}); cp -r $line $DIR; done
+- mvn validate -Pxgboost
+before_script:
+- find $TRAVIS_BUILD_DIR -name .starts | while read line; do rm -rf $line; done
+- find $TRAVIS_BUILD_DIR -name jdeps-cache | while read line; do rm -rf $line; done
+- find $HOME/CACHE -name jdeps-cache | while read line; do du -sh $line;         DIR=$(dirname
+  ${line#$HOME/CACHE}); cp -r $line $DIR; done
+- find $HOME/CACHE -name .starts | while read line; do du -sh $line;         DIR=$(dirname
+  ${line#$HOME/CACHE}); cp -r $line $DIR; done
+branches:
+  only:
+  - master
+  - develop
+cache:
+  directories:
+  - $HOME/CACHE
 env:
   global:
-   # The next declaration is the encrypted COVERITY_SCAN_TOKEN, created
-   #   via the "travis encrypt" command using the project repo's public key
-   - secure: "gYAzVukFw3ssmQzfovoKTNJnzQbUhF13tksHHhe5dqQWglxdix39wY405nMmzRm/21ZpIq3OFIlI7fnHdZSGkVPS/nPPxbzUkjYUh+zkujYwsqgjZecq9vGaoarnyV9CcI+2D6qUVnArHQ02FKxirtXduGvElZ/eRIa/hBfpgn4="
-
-#addons:
-#  coverity_scan:
-#    project:
-#      name: "myui/hivemall"
-#      description: "Build submitted via Travis CI"
-#    notification_email: false
-#    build_command_prepend: "mvn clean"
-#    build_command:   "mvn -DskipTests=true compile"
-#    branch_pattern: master
-
-language: java
+  - secure: gYAzVukFw3ssmQzfovoKTNJnzQbUhF13tksHHhe5dqQWglxdix39wY405nMmzRm/21ZpIq3OFIlI7fnHdZSGkVPS/nPPxbzUkjYUh+zkujYwsqgjZecq9vGaoarnyV9CcI+2D6qUVnArHQ02FKxirtXduGvElZ/eRIa/hBfpgn4=
 jdk:
-  - openjdk7
-# - oraclejdk7
-  - oraclejdk8
-# - oraclejdk9
-
-branches:
-  only:
-    - master
-    - develop
-
-before_install:
-  - mvn validate -Pxgboost
-
-notifications:
-  email: false
-
+- oraclejdk8
+language: java
 script:
-  - ./bin/run_travis_tests.sh
-
-after_success:
-  - mvn clean cobertura:cobertura coveralls:report
+- touch NOW
+- echo GIBSTUDYSTART $(date)
+- mvn -q scalastyle:check test -Pspark-2.1
+- mvn -q scalastyle:check clean -Pspark-2.0 -pl spark/spark-2.0 -am test -Dtest=none
+- echo GIBSTUDYEND $(date)
+- cd $TRAVIS_BUILD_DIR
+- if [ "$TRAVIS_OS_NAME" == "osx" ];        then echo TESTNUM $(find . -name TEST-*.xml
+  -newer NOW | wc -l);        else echo TESTNUM $(find -name TEST-*.xml -newer NOW
+  | wc -l);        fi
+sudo: false

NOTICE

@@ -33,17 +33,6 @@ o hivemall/core/src/main/java/hivemall/utils/collections/OpenHashMap.java
     https://github.com/slipperyseal/atomicobjects/
     Licensed under the Apache License, Version 2.0
 
-o hivemall/core/src/main/java/hivemall/utils/math/FastMath.java
-
-     Copyright 2012-2015 Jeff Hain
-
-     https://github.com/jeffhain/jafama/
-     Licensed under the Apache License, Version 2.0
-
-     Copyright (C) 1993 by Sun Microsystems, Inc.
-
-     Permission to use, copy, modify, and distribute this software is freely granted, provided that this notice is preserved.
-
 ------------------------------------------------------------------------------------------------------
 Copyright notifications which have been relocated from ASF projects
 
@@ -61,7 +50,7 @@ o hivemall/core/src/main/java/hivemall/utils/buffer/DynamicByteArray.java
     https://orc.apache.org/
     Licensed under the Apache License, Version 2.0
 
-o hivemall/spark/spark-2.0/extra-src/hive/src/main/scala/org/apache/spark/sql/hive/HiveShim.scala
+  hivemall/spark/spark-2.0/extra-src/hive/src/main/scala/org/apache/spark/sql/hive/HiveShim.scala
   hivemall/spark/spark-2.0/src/test/scala/org/apache/spark/sql/QueryTest.scala
   hivemall/spark/spark-2.0/src/test/scala/org/apache/spark/sql/catalyst/plans/PlanTest.scala
   hivemall/spark/spark-2.0/src/test/scala/org/apache/spark/sql/hive/test/TestHiveSingleton.scala
@@ -77,5 +66,3 @@ o hivemall/spark/spark-2.0/extra-src/hive/src/main/scala/org/apache/spark/sql/hi
 
     http://spark.apache.org/
     Licensed under the Apache License, Version 2.0
-
-

README.md

@@ -46,12 +46,6 @@ Contributing
 
 If you are planning to contribute to this repository, we first request you to create an issue at [our JIRA page](https://issues.apache.org/jira/projects/HIVEMALL) even if the topic is not related to source code itself (e.g., documentation, new idea and proposal).
 
-All Hivemall functions are defined under [resources/ddl](resources/ddl). In order to update the definition files, the following script helps inserting function name and class path of your new UDF:
-
-```
-$ ./bin/update_ddls.sh
-```
-
 Note that, before creating a pull request including Java code, please make sure your code follows our coding conventions by applying formatter:
 
 ```

core/pom.xml

@@ -139,21 +139,9 @@
 		<dependency>
 			<groupId>org.roaringbitmap</groupId>
 			<artifactId>RoaringBitmap</artifactId>
-			<version>[0.6,0.7)</version>
+			<version>[0.6,)</version>
 			<scope>compile</scope>
 		</dependency>
-		<dependency>
-			<groupId>it.unimi.dsi</groupId>
-			<artifactId>fastutil</artifactId>
-			<version>[7.2.1,7.3)</version>
-			<scope>compile</scope>
-		</dependency>
-		<dependency>
-		  <groupId>com.clearspring.analytics</groupId>
-		  <artifactId>stream</artifactId>
-		  <version>2.9.5</version>
-		  <scope>compile</scope>
-		</dependency>
 
 		<!-- test scope -->
 		<dependency>
@@ -202,7 +190,7 @@
 			<plugin>
 				<groupId>org.apache.maven.plugins</groupId>
 				<artifactId>maven-shade-plugin</artifactId>
-				<version>3.1.0</version>
+				<version>2.3</version>
 				<executions>
 					<execution>
 						<id>jar-with-dependencies</id>
@@ -224,8 +212,6 @@
 									<include>org.tukaani:xz</include>
 									<include>org.apache.commons:commons-math3</include>
 									<include>org.roaringbitmap:RoaringBitmap</include>
-									<include>it.unimi.dsi:fastutil</include>
-									<include>com.clearspring.analytics:stream</include>
 								</includes>
 							</artifactSet>
 							<transformers>

core/src/main/java/hivemall/HivemallConstants.java

@@ -18,7 +18,6 @@
  */
 package hivemall;
 
-
 public final class HivemallConstants {
 
     public static final String VERSION = "0.4.2-rc.2";
@@ -36,7 +35,6 @@ public final class HivemallConstants {
     public static final String BIGINT_TYPE_NAME = "bigint";
     public static final String FLOAT_TYPE_NAME = "float";
     public static final String DOUBLE_TYPE_NAME = "double";
-    public static final String DECIMAL_TYPE_NAME = "decimal";
     public static final String STRING_TYPE_NAME = "string";
     public static final String DATE_TYPE_NAME = "date";
     public static final String DATETIME_TYPE_NAME = "datetime";

core/src/main/java/hivemall/LearnerBaseUDTF.java

@@ -55,8 +55,6 @@
 
 public abstract class LearnerBaseUDTF extends UDTFWithOptions {
     private static final Log logger = LogFactory.getLog(LearnerBaseUDTF.class);
-    private static final int DEFAULT_SPARSE_DIMS = 16384;
-    private static final int DEFAULT_DENSE_DIMS = 16777216;
 
     protected final boolean enableNewModel;
     protected boolean dense_model;
@@ -122,7 +120,7 @@ protected CommandLine processOptions(@Nonnull ObjectInspector[] argOIs)
 
             denseModel = cl.hasOption("dense");
             if (denseModel) {
-                modelDims = Primitives.parseInt(cl.getOptionValue("dims"), DEFAULT_DENSE_DIMS);
+                modelDims = Primitives.parseInt(cl.getOptionValue("dims"), 16777216);
             }
             disableHalfFloat = cl.hasOption("disable_halffloat");
 
@@ -170,7 +168,7 @@ private final PredictionModel createOldModel(@Nullable String label) {
         PredictionModel model;
         final boolean useCovar = useCovariance();
         if (dense_model) {
-            if (disable_halffloat == false && model_dims > DEFAULT_DENSE_DIMS) {
+            if (disable_halffloat == false && model_dims > 16777216) {
                 logger.info("Build a space efficient dense model with " + model_dims
                         + " initial dimensions" + (useCovar ? " w/ covariances" : ""));
                 model = new SpaceEfficientDenseModel(model_dims, useCovar);
@@ -201,7 +199,7 @@ private final PredictionModel createNewModel(@Nullable String label) {
         PredictionModel model;
         final boolean useCovar = useCovariance();
         if (dense_model) {
-            if (disable_halffloat == false && model_dims > DEFAULT_DENSE_DIMS) {
+            if (disable_halffloat == false && model_dims > 16777216) {
                 logger.info("Build a space efficient dense model with " + model_dims
                         + " initial dimensions" + (useCovar ? " w/ covariances" : ""));
                 model = new NewSpaceEfficientDenseModel(model_dims, useCovar);
@@ -231,11 +229,9 @@ private final PredictionModel createNewModel(@Nullable String label) {
     protected final Optimizer createOptimizer(@CheckForNull Map<String, String> options) {
         Preconditions.checkNotNull(options);
         if (dense_model) {
-            return DenseOptimizerFactory.create(model_dims < 0 ? DEFAULT_DENSE_DIMS : model_dims,
-                options);
+            return DenseOptimizerFactory.create(model_dims, options);
         } else {
-            return SparseOptimizerFactory.create(model_dims < 0 ? DEFAULT_SPARSE_DIMS : model_dims,
-                options);
+            return SparseOptimizerFactory.create(model_dims, options);
         }
     }