diff --git a/jvm-packages/checkstyle-suppressions.xml b/jvm-packages/checkstyle-suppressions.xml
index 366f6a01e4e0..14a9efd130fc 100644
--- a/jvm-packages/checkstyle-suppressions.xml
+++ b/jvm-packages/checkstyle-suppressions.xml
@@ -16,8 +16,8 @@
   -->
 
 <!DOCTYPE suppressions PUBLIC
-"-//Puppy Crawl//DTD Suppressions 1.1//EN"
-"http://www.puppycrawl.com/dtds/suppressions_1_1.dtd">
+    "-//Checkstyle//DTD SuppressionFilter Configuration 1.1//EN"
+    "https://checkstyle.org/dtds/suppressions_1_1.dtd">
 
 <!--
 
diff --git a/jvm-packages/checkstyle.xml b/jvm-packages/checkstyle.xml
index a926eb720c1c..123dbe011820 100644
--- a/jvm-packages/checkstyle.xml
+++ b/jvm-packages/checkstyle.xml
@@ -16,8 +16,8 @@
   -->
 
 <!DOCTYPE module PUBLIC
-          "-//Puppy Crawl//DTD Check Configuration 1.3//EN"
-          "http://www.puppycrawl.com/dtds/configuration_1_3.dtd">
+    "-//Checkstyle//DTD Checkstyle Configuration 1.3//EN"
+    "https://checkstyle.org/dtds/configuration_1_3.dtd">
 
 <!--
 
diff --git a/jvm-packages/create_jni.py b/jvm-packages/create_jni.py
index 4d627cb7c748..21d88b80fb36 100755
--- a/jvm-packages/create_jni.py
+++ b/jvm-packages/create_jni.py
@@ -110,16 +110,13 @@ def normpath(path):
        "xgboost4j/src/main/resources")
 
     print("copying train/test files")
-    maybe_makedirs("xgboost4j-spark/src/test/resources")
+    maybe_makedirs("xgboost4j/src/test/resources")
     with cd("../demo/regression"):
         run("{} mapfeat.py".format(sys.executable))
         run("{} mknfold.py machine.txt 1".format(sys.executable))
 
-    for file in glob.glob("../demo/regression/machine.txt.t*"):
-        cp(file, "xgboost4j-spark/src/test/resources")
-    for file in glob.glob("../demo/data/agaricus.*"):
-        cp(file, "xgboost4j-spark/src/test/resources")
-
     maybe_makedirs("xgboost4j/src/test/resources")
+    for file in glob.glob("../demo/regression/machine.txt.t*"):
+        cp(file, "xgboost4j/src/test/resources")
     for file in glob.glob("../demo/data/agaricus.*"):
         cp(file, "xgboost4j/src/test/resources")
diff --git a/jvm-packages/xgboost4j-spark/pom.xml b/jvm-packages/xgboost4j-spark/pom.xml
index cf4f00445bc0..98bdefe58167 100644
--- a/jvm-packages/xgboost4j-spark/pom.xml
+++ b/jvm-packages/xgboost4j-spark/pom.xml
@@ -26,6 +26,13 @@
             <artifactId>xgboost4j_${scala.binary.version}</artifactId>
             <version>1.0.0-SNAPSHOT</version>
         </dependency>
+        <dependency>
+          <groupId>ml.dmlc</groupId>
+          <artifactId>xgboost4j_${scala.binary.version}</artifactId>
+          <version>1.0.0-SNAPSHOT</version>
+          <type>test-jar</type>
+          <scope>test</scope>
+        </dependency>
         <dependency>
             <groupId>org.apache.spark</groupId>
             <artifactId>spark-core_${scala.binary.version}</artifactId>
diff --git a/jvm-packages/xgboost4j-spark/src/test/scala/ml/dmlc/xgboost4j/scala/spark/CheckpointManagerSuite.scala b/jvm-packages/xgboost4j-spark/src/test/scala/ml/dmlc/xgboost4j/scala/spark/CheckpointManagerSuite.scala
index ddeb48241c86..a67334ead956 100755
--- a/jvm-packages/xgboost4j-spark/src/test/scala/ml/dmlc/xgboost4j/scala/spark/CheckpointManagerSuite.scala
+++ b/jvm-packages/xgboost4j-spark/src/test/scala/ml/dmlc/xgboost4j/scala/spark/CheckpointManagerSuite.scala
@@ -18,7 +18,7 @@ package ml.dmlc.xgboost4j.scala.spark
 
 import java.io.File
 
-import ml.dmlc.xgboost4j.scala.{Booster, DMatrix, XGBoost => SXGBoost}
+import ml.dmlc.xgboost4j.scala.{Classification, Booster, DMatrix, XGBoost => SXGBoost}
 import org.scalatest.FunSuite
 import org.apache.hadoop.fs.{FileSystem, Path}
 
diff --git a/jvm-packages/xgboost4j-spark/src/test/scala/ml/dmlc/xgboost4j/scala/spark/DeterministicPartitioningSuite.scala b/jvm-packages/xgboost4j-spark/src/test/scala/ml/dmlc/xgboost4j/scala/spark/DeterministicPartitioningSuite.scala
old mode 100644
new mode 100755
index 986b0843b5f3..739ea185736e
--- a/jvm-packages/xgboost4j-spark/src/test/scala/ml/dmlc/xgboost4j/scala/spark/DeterministicPartitioningSuite.scala
+++ b/jvm-packages/xgboost4j-spark/src/test/scala/ml/dmlc/xgboost4j/scala/spark/DeterministicPartitioningSuite.scala
@@ -16,8 +16,8 @@
 
 package ml.dmlc.xgboost4j.scala.spark
 
+import ml.dmlc.xgboost4j.scala.Classification
 import org.scalatest.FunSuite
-
 import org.apache.spark.sql.functions._
 
 class DeterministicPartitioningSuite extends FunSuite with TmpFolderPerSuite with PerTest {
diff --git a/jvm-packages/xgboost4j-spark/src/test/scala/ml/dmlc/xgboost4j/scala/spark/ParameterSuite.scala b/jvm-packages/xgboost4j-spark/src/test/scala/ml/dmlc/xgboost4j/scala/spark/ParameterSuite.scala
index e0450f90dfd8..9e87847c1b19 100644
--- a/jvm-packages/xgboost4j-spark/src/test/scala/ml/dmlc/xgboost4j/scala/spark/ParameterSuite.scala
+++ b/jvm-packages/xgboost4j-spark/src/test/scala/ml/dmlc/xgboost4j/scala/spark/ParameterSuite.scala
@@ -17,8 +17,8 @@
 package ml.dmlc.xgboost4j.scala.spark
 
 import ml.dmlc.xgboost4j.java.XGBoostError
+import ml.dmlc.xgboost4j.scala.MultiClassification
 import org.scalatest.{BeforeAndAfterAll, FunSuite}
-
 import org.apache.spark.ml.param.ParamMap
 
 class ParameterSuite extends FunSuite with PerTest with BeforeAndAfterAll {
diff --git a/jvm-packages/xgboost4j-spark/src/test/scala/ml/dmlc/xgboost4j/scala/spark/PersistenceSuite.scala b/jvm-packages/xgboost4j-spark/src/test/scala/ml/dmlc/xgboost4j/scala/spark/PersistenceSuite.scala
index a3841772c796..c9d5704a5b92 100755
--- a/jvm-packages/xgboost4j-spark/src/test/scala/ml/dmlc/xgboost4j/scala/spark/PersistenceSuite.scala
+++ b/jvm-packages/xgboost4j-spark/src/test/scala/ml/dmlc/xgboost4j/scala/spark/PersistenceSuite.scala
@@ -20,10 +20,9 @@ import java.io.File
 import java.util.Arrays
 
 import scala.io.Source
+import ml.dmlc.xgboost4j.scala.{Classification, DMatrix, Regression}
 
-import ml.dmlc.xgboost4j.scala.DMatrix
 import scala.util.Random
-
 import org.apache.spark.ml.feature._
 import org.apache.spark.ml.{Pipeline, PipelineModel}
 import org.scalatest.FunSuite
diff --git a/jvm-packages/xgboost4j-spark/src/test/scala/ml/dmlc/xgboost4j/scala/spark/RabitRobustnessSuite.scala b/jvm-packages/xgboost4j-spark/src/test/scala/ml/dmlc/xgboost4j/scala/spark/RabitRobustnessSuite.scala
index 9655104c35d7..7498a4b172b3 100644
--- a/jvm-packages/xgboost4j-spark/src/test/scala/ml/dmlc/xgboost4j/scala/spark/RabitRobustnessSuite.scala
+++ b/jvm-packages/xgboost4j-spark/src/test/scala/ml/dmlc/xgboost4j/scala/spark/RabitRobustnessSuite.scala
@@ -19,12 +19,10 @@ package ml.dmlc.xgboost4j.scala.spark
 import java.util.concurrent.LinkedBlockingDeque
 
 import scala.util.Random
-
 import ml.dmlc.xgboost4j.java.{IRabitTracker, Rabit, RabitTracker => PyRabitTracker}
 import ml.dmlc.xgboost4j.scala.rabit.{RabitTracker => ScalaRabitTracker}
 import ml.dmlc.xgboost4j.java.IRabitTracker.TrackerStatus
-import ml.dmlc.xgboost4j.scala.DMatrix
-
+import ml.dmlc.xgboost4j.scala.{Classification, DMatrix}
 import org.apache.spark.{SparkConf, SparkContext}
 import org.scalatest.FunSuite
 
diff --git a/jvm-packages/xgboost4j-spark/src/test/scala/ml/dmlc/xgboost4j/scala/spark/XGBoostClassifierSuite.scala b/jvm-packages/xgboost4j-spark/src/test/scala/ml/dmlc/xgboost4j/scala/spark/XGBoostClassifierSuite.scala
index f2ccdd44ee73..92f050214d0a 100644
--- a/jvm-packages/xgboost4j-spark/src/test/scala/ml/dmlc/xgboost4j/scala/spark/XGBoostClassifierSuite.scala
+++ b/jvm-packages/xgboost4j-spark/src/test/scala/ml/dmlc/xgboost4j/scala/spark/XGBoostClassifierSuite.scala
@@ -16,13 +16,11 @@
 
 package ml.dmlc.xgboost4j.scala.spark
 
-import ml.dmlc.xgboost4j.scala.{DMatrix, XGBoost => ScalaXGBoost}
-
+import ml.dmlc.xgboost4j.scala.{Classification, DMatrix, MultiClassification, Synthetic, XGBoost => ScalaXGBoost}
 import org.apache.spark.ml.linalg._
 import org.apache.spark.ml.param.ParamMap
 import org.apache.spark.sql._
 import org.scalatest.FunSuite
-
 import org.apache.spark.Partitioner
 
 class XGBoostClassifierSuite extends FunSuite with PerTest {
diff --git a/jvm-packages/xgboost4j-spark/src/test/scala/ml/dmlc/xgboost4j/scala/spark/XGBoostConfigureSuite.scala b/jvm-packages/xgboost4j-spark/src/test/scala/ml/dmlc/xgboost4j/scala/spark/XGBoostConfigureSuite.scala
index fe16bcda56bf..16dfa4ed945d 100644
--- a/jvm-packages/xgboost4j-spark/src/test/scala/ml/dmlc/xgboost4j/scala/spark/XGBoostConfigureSuite.scala
+++ b/jvm-packages/xgboost4j-spark/src/test/scala/ml/dmlc/xgboost4j/scala/spark/XGBoostConfigureSuite.scala
@@ -16,7 +16,7 @@
 
 package ml.dmlc.xgboost4j.scala.spark
 
-import ml.dmlc.xgboost4j.scala.{Booster, DMatrix}
+import ml.dmlc.xgboost4j.scala.{Booster, Classification, DMatrix}
 import org.apache.spark.sql._
 import org.scalatest.FunSuite
 
diff --git a/jvm-packages/xgboost4j-spark/src/test/scala/ml/dmlc/xgboost4j/scala/spark/XGBoostRegressorSuite.scala b/jvm-packages/xgboost4j-spark/src/test/scala/ml/dmlc/xgboost4j/scala/spark/XGBoostRegressorSuite.scala
index ff88ff328659..90f10dc5ef08 100644
--- a/jvm-packages/xgboost4j-spark/src/test/scala/ml/dmlc/xgboost4j/scala/spark/XGBoostRegressorSuite.scala
+++ b/jvm-packages/xgboost4j-spark/src/test/scala/ml/dmlc/xgboost4j/scala/spark/XGBoostRegressorSuite.scala
@@ -16,7 +16,7 @@
 
 package ml.dmlc.xgboost4j.scala.spark
 
-import ml.dmlc.xgboost4j.scala.{DMatrix, XGBoost => ScalaXGBoost}
+import ml.dmlc.xgboost4j.scala.{DMatrix, Ranking, Regression, XGBoost => ScalaXGBoost}
 import org.apache.spark.ml.linalg.Vector
 import org.apache.spark.sql.functions._
 import org.apache.spark.sql.{DataFrame, Row}
diff --git a/jvm-packages/xgboost4j/src/test/java/ml/dmlc/xgboost4j/java/BoosterImplTest.java b/jvm-packages/xgboost4j/src/test/java/ml/dmlc/xgboost4j/java/BoosterImplTest.java
index 3b565ab8fe94..8bd674723bb0 100755
--- a/jvm-packages/xgboost4j/src/test/java/ml/dmlc/xgboost4j/java/BoosterImplTest.java
+++ b/jvm-packages/xgboost4j/src/test/java/ml/dmlc/xgboost4j/java/BoosterImplTest.java
@@ -24,6 +24,7 @@
 import java.util.Map;
 
 import junit.framework.TestCase;
+import org.junit.Before;
 import org.junit.Test;
 
 /**
@@ -84,12 +85,18 @@ private Booster trainBooster(DMatrix trainMat, DMatrix testMat) throws XGBoostEr
     return XGBoost.train(trainMat, paramMap, round, watches, null, null);
   }
 
+  private DMatrix trainMat;
+  private DMatrix testMat;
+
+  @Before
+  public void setUp() throws XGBoostError {
+    trainMat = new DMatrix(Classification.trainFile.toString());
+    testMat = new DMatrix(Classification.testFile.toString());
+  }
+
   @Test
   public void testBoosterBasic() throws XGBoostError, IOException {
 
-    DMatrix trainMat = new DMatrix("../../demo/data/agaricus.txt.train");
-    DMatrix testMat = new DMatrix("../../demo/data/agaricus.txt.test");
-
     Booster booster = trainBooster(trainMat, testMat);
 
     //predict raw output
@@ -103,8 +110,6 @@ public void testBoosterBasic() throws XGBoostError, IOException {
 
   @Test
   public void saveLoadModelWithPath() throws XGBoostError, IOException {
-    DMatrix trainMat = new DMatrix("../../demo/data/agaricus.txt.train");
-    DMatrix testMat = new DMatrix("../../demo/data/agaricus.txt.test");
     IEvaluation eval = new EvalError();
 
     Booster booster = trainBooster(trainMat, testMat);
@@ -121,8 +126,6 @@ public void saveLoadModelWithPath() throws XGBoostError, IOException {
 
   @Test
   public void saveLoadModelWithStream() throws XGBoostError, IOException {
-    DMatrix trainMat = new DMatrix("../../demo/data/agaricus.txt.train");
-    DMatrix testMat = new DMatrix("../../demo/data/agaricus.txt.test");
 
     Booster booster = trainBooster(trainMat, testMat);
 
@@ -310,8 +313,6 @@ public void testAscendMetrics() {
 
   @Test
   public void testBoosterEarlyStop() throws XGBoostError, IOException {
-    DMatrix trainMat = new DMatrix("../../demo/data/agaricus.txt.train");
-    DMatrix testMat = new DMatrix("../../demo/data/agaricus.txt.test");
     Map<String, Object> paramMap = new HashMap<String, Object>() {
       {
         put("max_depth", 3);
@@ -363,8 +364,6 @@ private void testWithQuantileHisto(DMatrix trainingSet, Map<String, DMatrix> wat
 
   @Test
   public void testQuantileHistoDepthWise() throws XGBoostError {
-    DMatrix trainMat = new DMatrix("../../demo/data/agaricus.txt.train");
-    DMatrix testMat = new DMatrix("../../demo/data/agaricus.txt.test");
     Map<String, Object> paramMap = new HashMap<String, Object>() {
       {
         put("max_depth", 3);
@@ -383,8 +382,6 @@ public void testQuantileHistoDepthWise() throws XGBoostError {
 
   @Test
   public void testQuantileHistoLossGuide() throws XGBoostError {
-    DMatrix trainMat = new DMatrix("../../demo/data/agaricus.txt.train");
-    DMatrix testMat = new DMatrix("../../demo/data/agaricus.txt.test");
     Map<String, Object> paramMap = new HashMap<String, Object>() {
       {
         put("max_depth", 0);
@@ -404,8 +401,6 @@ public void testQuantileHistoLossGuide() throws XGBoostError {
 
   @Test
   public void testQuantileHistoLossGuideMaxBin() throws XGBoostError {
-    DMatrix trainMat = new DMatrix("../../demo/data/agaricus.txt.train");
-    DMatrix testMat = new DMatrix("../../demo/data/agaricus.txt.test");
     Map<String, Object> paramMap = new HashMap<String, Object>() {
       {
         put("max_depth", 0);
@@ -425,8 +420,6 @@ public void testQuantileHistoLossGuideMaxBin() throws XGBoostError {
 
   @Test
   public void testDumpModelJson() throws XGBoostError {
-    DMatrix trainMat = new DMatrix("../../demo/data/agaricus.txt.train");
-    DMatrix testMat = new DMatrix("../../demo/data/agaricus.txt.test");
 
     Booster booster = trainBooster(trainMat, testMat);
     String[] dump = booster.getModelDump("", false, "json");
@@ -441,9 +434,6 @@ public void testDumpModelJson() throws XGBoostError {
 
   @Test
   public void testGetFeatureScore() throws XGBoostError {
-    DMatrix trainMat = new DMatrix("../../demo/data/agaricus.txt.train");
-    DMatrix testMat = new DMatrix("../../demo/data/agaricus.txt.test");
-
     Booster booster = trainBooster(trainMat, testMat);
     String[] featureNames = new String[126];
     for(int i = 0; i < 126; i++) featureNames[i] = "test_feature_name_" + i;
@@ -453,9 +443,6 @@ public void testGetFeatureScore() throws XGBoostError {
 
   @Test
   public void testGetFeatureImportanceGain() throws XGBoostError {
-    DMatrix trainMat = new DMatrix("../../demo/data/agaricus.txt.train");
-    DMatrix testMat = new DMatrix("../../demo/data/agaricus.txt.test");
-
     Booster booster = trainBooster(trainMat, testMat);
     String[] featureNames = new String[126];
     for(int i = 0; i < 126; i++) featureNames[i] = "test_feature_name_" + i;
@@ -465,9 +452,6 @@ public void testGetFeatureImportanceGain() throws XGBoostError {
 
   @Test
   public void testGetFeatureImportanceTotalGain() throws XGBoostError {
-    DMatrix trainMat = new DMatrix("../../demo/data/agaricus.txt.train");
-    DMatrix testMat = new DMatrix("../../demo/data/agaricus.txt.test");
-
     Booster booster = trainBooster(trainMat, testMat);
     String[] featureNames = new String[126];
     for(int i = 0; i < 126; i++) featureNames[i] = "test_feature_name_" + i;
@@ -477,9 +461,6 @@ public void testGetFeatureImportanceTotalGain() throws XGBoostError {
 
   @Test
   public void testGetFeatureImportanceCover() throws XGBoostError {
-    DMatrix trainMat = new DMatrix("../../demo/data/agaricus.txt.train");
-    DMatrix testMat = new DMatrix("../../demo/data/agaricus.txt.test");
-
     Booster booster = trainBooster(trainMat, testMat);
     String[] featureNames = new String[126];
     for(int i = 0; i < 126; i++) featureNames[i] = "test_feature_name_" + i;
@@ -489,9 +470,6 @@ public void testGetFeatureImportanceCover() throws XGBoostError {
 
   @Test
   public void testGetFeatureImportanceTotalCover() throws XGBoostError {
-    DMatrix trainMat = new DMatrix("../../demo/data/agaricus.txt.train");
-    DMatrix testMat = new DMatrix("../../demo/data/agaricus.txt.test");
-
     Booster booster = trainBooster(trainMat, testMat);
     String[] featureNames = new String[126];
     for(int i = 0; i < 126; i++) featureNames[i] = "test_feature_name_" + i;
@@ -501,7 +479,6 @@ public void testGetFeatureImportanceTotalCover() throws XGBoostError {
 
   @Test
   public void testQuantileHistoDepthwiseMaxDepth() throws XGBoostError {
-    DMatrix trainMat = new DMatrix("../../demo/data/agaricus.txt.train");
     Map<String, Object> paramMap = new HashMap<String, Object>() {
       {
         put("max_depth", 3);
@@ -519,8 +496,6 @@ public void testQuantileHistoDepthwiseMaxDepth() throws XGBoostError {
 
   @Test
   public void testQuantileHistoDepthwiseMaxDepthMaxBin() throws XGBoostError {
-    DMatrix trainMat = new DMatrix("../../demo/data/agaricus.txt.train");
-    DMatrix testMat = new DMatrix("../../demo/data/agaricus.txt.test");
     Map<String, Object> paramMap = new HashMap<String, Object>() {
       {
         put("max_depth", 3);
@@ -544,9 +519,6 @@ public void testQuantileHistoDepthwiseMaxDepthMaxBin() throws XGBoostError {
    */
   @Test
   public void testCV() throws XGBoostError {
-    //load train mat
-    DMatrix trainMat = new DMatrix("../../demo/data/agaricus.txt.train");
-
     //set params
     Map<String, Object> param = new HashMap<String, Object>() {
       {
@@ -573,8 +545,6 @@ public void testCV() throws XGBoostError {
    */
   @Test
   public void testTrainFromExistingModel() throws XGBoostError, IOException {
-    DMatrix trainMat = new DMatrix("../../demo/data/agaricus.txt.train");
-    DMatrix testMat = new DMatrix("../../demo/data/agaricus.txt.test");
     IEvaluation eval = new EvalError();
 
     Map<String, Object> paramMap = new HashMap<String, Object>() {
@@ -624,9 +594,6 @@ public void testTrainFromExistingModel() throws XGBoostError, IOException {
    */
   @Test
   public void testSetAndGetAttrs() throws XGBoostError {
-    DMatrix trainMat = new DMatrix("../../demo/data/agaricus.txt.train");
-    DMatrix testMat = new DMatrix("../../demo/data/agaricus.txt.test");
-
     Booster booster = trainBooster(trainMat, testMat);
     booster.setAttr("testKey1", "testValue1");
     TestCase.assertEquals(booster.getAttr("testKey1"), "testValue1");
diff --git a/jvm-packages/xgboost4j/src/test/java/ml/dmlc/xgboost4j/java/Classification.java b/jvm-packages/xgboost4j/src/test/java/ml/dmlc/xgboost4j/java/Classification.java
new file mode 100644
index 000000000000..8ca3ecf231a0
--- /dev/null
+++ b/jvm-packages/xgboost4j/src/test/java/ml/dmlc/xgboost4j/java/Classification.java
@@ -0,0 +1,10 @@
+package ml.dmlc.xgboost4j.java;
+
+
+import java.io.File;
+
+public class Classification {
+    public static File trainFile = TrainTestData.getResourceFile("/agaricus.txt.train");
+    public static File testFile = TrainTestData.getResourceFile("/agaricus.txt.test");
+}
+
diff --git a/jvm-packages/xgboost4j/src/test/java/ml/dmlc/xgboost4j/java/DMatrixTest.java b/jvm-packages/xgboost4j/src/test/java/ml/dmlc/xgboost4j/java/DMatrixTest.java
old mode 100644
new mode 100755
index 75f52d877eac..c47c1aaba783
--- a/jvm-packages/xgboost4j/src/test/java/ml/dmlc/xgboost4j/java/DMatrixTest.java
+++ b/jvm-packages/xgboost4j/src/test/java/ml/dmlc/xgboost4j/java/DMatrixTest.java
@@ -60,8 +60,7 @@ public void testCreateFromDataIterator() throws XGBoostError {
   @Test
   public void testCreateFromFile() throws XGBoostError {
     //create DMatrix from file
-    String filePath = writeResourceIntoTempFile("/agaricus.txt.test");
-    DMatrix dmat = new DMatrix(filePath);
+    DMatrix dmat = new DMatrix(Classification.testFile.toString());
     //get label
     float[] labels = dmat.getLabel();
     //check length
diff --git a/jvm-packages/xgboost4j/src/test/java/ml/dmlc/xgboost4j/java/TrainTestData.java b/jvm-packages/xgboost4j/src/test/java/ml/dmlc/xgboost4j/java/TrainTestData.java
new file mode 100644
index 000000000000..c37346984c0b
--- /dev/null
+++ b/jvm-packages/xgboost4j/src/test/java/ml/dmlc/xgboost4j/java/TrainTestData.java
@@ -0,0 +1,27 @@
+package ml.dmlc.xgboost4j.java;
+
+import java.io.File;
+import java.io.InputStream;
+import java.nio.file.Files;
+import java.nio.file.StandardCopyOption;
+
+public class TrainTestData {
+    static File getResourceFile(String resource) {
+        InputStream is = TrainTestData.class.getResourceAsStream(resource);
+        if (is == null) {
+            throw new RuntimeException("Failed to resolve resource " + resource);
+        }
+        try {
+            try {
+                File file = File.createTempFile(resource.substring(1), "");
+                Files.copy(is, file.toPath(), StandardCopyOption.REPLACE_EXISTING);
+                file.deleteOnExit();
+                return file;
+            } finally {
+                is.close();
+            }
+        } catch (Exception e) {
+            throw new RuntimeException("Failed to load the resource " + resource, e);
+        }
+    }
+}
diff --git a/jvm-packages/xgboost4j/src/test/scala/ml/dmlc/xgboost4j/scala/DMatrixSuite.scala b/jvm-packages/xgboost4j/src/test/scala/ml/dmlc/xgboost4j/scala/DMatrixSuite.scala
index 87ff8e006a98..c3d6cc2f2abb 100644
--- a/jvm-packages/xgboost4j/src/test/scala/ml/dmlc/xgboost4j/scala/DMatrixSuite.scala
+++ b/jvm-packages/xgboost4j/src/test/scala/ml/dmlc/xgboost4j/scala/DMatrixSuite.scala
@@ -25,7 +25,7 @@ import ml.dmlc.xgboost4j.java.{DMatrix => JDMatrix}
 
 class DMatrixSuite extends FunSuite {
   test("create DMatrix from File") {
-    val dmat = new DMatrix("../../demo/data/agaricus.txt.test")
+    val dmat = new DMatrix(Classification.testFile.toString)
     // get label
     val labels: Array[Float] = dmat.getLabel
     // check length
diff --git a/jvm-packages/xgboost4j/src/test/scala/ml/dmlc/xgboost4j/scala/ScalaBoosterImplSuite.scala b/jvm-packages/xgboost4j/src/test/scala/ml/dmlc/xgboost4j/scala/ScalaBoosterImplSuite.scala
index adea1b1ec93a..785b4a3302d1 100644
--- a/jvm-packages/xgboost4j/src/test/scala/ml/dmlc/xgboost4j/scala/ScalaBoosterImplSuite.scala
+++ b/jvm-packages/xgboost4j/src/test/scala/ml/dmlc/xgboost4j/scala/ScalaBoosterImplSuite.scala
@@ -26,6 +26,9 @@ import ml.dmlc.xgboost4j.java.XGBoostError
 
 class ScalaBoosterImplSuite extends FunSuite {
 
+  private val trainMat = new DMatrix(Classification.trainFile.toString)
+  private val testMat = new DMatrix(Classification.testFile.toString)
+
   private class EvalError extends EvalTrait {
 
     val logger = LogFactory.getLog(classOf[EvalError])
@@ -95,9 +98,6 @@ class ScalaBoosterImplSuite extends FunSuite {
   }
 
   test("basic operation of booster") {
-    val trainMat = new DMatrix("../../demo/data/agaricus.txt.train")
-    val testMat = new DMatrix("../../demo/data/agaricus.txt.test")
-
     val booster = trainBooster(trainMat, testMat)
     val predicts = booster.predict(testMat, true)
     val eval = new EvalError
@@ -105,9 +105,6 @@ class ScalaBoosterImplSuite extends FunSuite {
   }
 
   test("save/load model with path") {
-
-    val trainMat = new DMatrix("../../demo/data/agaricus.txt.train")
-    val testMat = new DMatrix("../../demo/data/agaricus.txt.test")
     val eval = new EvalError
     val booster = trainBooster(trainMat, testMat)
     // save and load
@@ -122,8 +119,6 @@ class ScalaBoosterImplSuite extends FunSuite {
   }
 
   test("save/load model with stream") {
-    val trainMat = new DMatrix("../../demo/data/agaricus.txt.train")
-    val testMat = new DMatrix("../../demo/data/agaricus.txt.test")
     val eval = new EvalError
     val booster = trainBooster(trainMat, testMat)
     // save and load
@@ -138,7 +133,6 @@ class ScalaBoosterImplSuite extends FunSuite {
   }
 
   test("cross validation") {
-    val trainMat = new DMatrix("../../demo/data/agaricus.txt.train")
     val params = List("eta" -> "1.0", "max_depth" -> "3", "silent" -> "1", "nthread" -> "6",
       "objective" -> "binary:logistic", "gamma" -> "1.0", "eval_metric" -> "error").toMap
     val round = 2
@@ -147,8 +141,6 @@ class ScalaBoosterImplSuite extends FunSuite {
   }
 
   test("test with quantile histo depthwise") {
-    val trainMat = new DMatrix("../../demo/data/agaricus.txt.train")
-    val testMat = new DMatrix("../../demo/data/agaricus.txt.test")
     val paramMap = List("max_depth" -> "3", "silent" -> "0",
       "objective" -> "binary:logistic", "tree_method" -> "hist",
       "grow_policy" -> "depthwise", "eval_metric" -> "auc").toMap
@@ -157,8 +149,6 @@ class ScalaBoosterImplSuite extends FunSuite {
   }
 
   test("test with quantile histo lossguide") {
-    val trainMat = new DMatrix("../../demo/data/agaricus.txt.train")
-    val testMat = new DMatrix("../../demo/data/agaricus.txt.test")
     val paramMap = List("max_depth" -> "0", "silent" -> "0",
       "objective" -> "binary:logistic", "tree_method" -> "hist",
       "grow_policy" -> "lossguide", "max_leaves" -> "8", "eval_metric" -> "auc").toMap
@@ -167,8 +157,6 @@ class ScalaBoosterImplSuite extends FunSuite {
   }
 
   test("test with quantile histo lossguide with max bin") {
-    val trainMat = new DMatrix("../../demo/data/agaricus.txt.train")
-    val testMat = new DMatrix("../../demo/data/agaricus.txt.test")
     val paramMap = List("max_depth" -> "0", "silent" -> "0",
       "objective" -> "binary:logistic", "tree_method" -> "hist",
       "grow_policy" -> "lossguide", "max_leaves" -> "8", "max_bin" -> "16",
@@ -178,8 +166,6 @@ class ScalaBoosterImplSuite extends FunSuite {
   }
 
   test("test with quantile histo depthwidth with max depth") {
-    val trainMat = new DMatrix("../../demo/data/agaricus.txt.train")
-    val testMat = new DMatrix("../../demo/data/agaricus.txt.test")
     val paramMap = List("max_depth" -> "0", "silent" -> "0",
       "objective" -> "binary:logistic", "tree_method" -> "hist",
       "grow_policy" -> "depthwise", "max_leaves" -> "8", "max_depth" -> "2",
@@ -189,8 +175,6 @@ class ScalaBoosterImplSuite extends FunSuite {
   }
 
   test("test with quantile histo depthwidth with max depth and max bin") {
-    val trainMat = new DMatrix("../../demo/data/agaricus.txt.train")
-    val testMat = new DMatrix("../../demo/data/agaricus.txt.test")
     val paramMap = List("max_depth" -> "0", "silent" -> "0",
       "objective" -> "binary:logistic", "tree_method" -> "hist",
       "grow_policy" -> "depthwise", "max_depth" -> "2", "max_bin" -> "2",
@@ -200,7 +184,6 @@ class ScalaBoosterImplSuite extends FunSuite {
   }
 
   test("test training from existing model in scala") {
-    val trainMat = new DMatrix("../../demo/data/agaricus.txt.train")
     val paramMap = List("max_depth" -> "0", "silent" -> "0",
       "objective" -> "binary:logistic", "tree_method" -> "hist",
       "grow_policy" -> "depthwise", "max_depth" -> "2", "max_bin" -> "2",
diff --git a/jvm-packages/xgboost4j-spark/src/test/scala/ml/dmlc/xgboost4j/scala/spark/TrainTestData.scala b/jvm-packages/xgboost4j/src/test/scala/ml/dmlc/xgboost4j/scala/TrainTestData.scala
similarity index 94%
rename from jvm-packages/xgboost4j-spark/src/test/scala/ml/dmlc/xgboost4j/scala/spark/TrainTestData.scala
rename to jvm-packages/xgboost4j/src/test/scala/ml/dmlc/xgboost4j/scala/TrainTestData.scala
index 1a64e2d03529..43a9aa8f664a 100644
--- a/jvm-packages/xgboost4j-spark/src/test/scala/ml/dmlc/xgboost4j/scala/spark/TrainTestData.scala
+++ b/jvm-packages/xgboost4j/src/test/scala/ml/dmlc/xgboost4j/scala/TrainTestData.scala
@@ -14,12 +14,14 @@
  limitations under the License.
  */
 
-package ml.dmlc.xgboost4j.scala.spark
+package ml.dmlc.xgboost4j.scala
 
 import scala.collection.mutable
 import scala.io.Source
 import ml.dmlc.xgboost4j.{LabeledPoint => XGBLabeledPoint}
 
+import ml.dmlc.xgboost4j.java.{Classification => JClassification}
+
 trait TrainTestData {
   protected def getResourceLines(resource: String): Iterator[String] = {
     require(resource.startsWith("/"), "resource must start with /")
@@ -62,6 +64,9 @@ trait TrainTestData {
 }
 
 object Classification extends TrainTestData {
+  private[xgboost4j] val trainFile = JClassification.trainFile
+  private[xgboost4j] val testFile = JClassification.testFile
+
   val train: Seq[XGBLabeledPoint] = getLabeledPoints("/agaricus.txt.train", zeroBased = false)
   val test: Seq[XGBLabeledPoint] = getLabeledPoints("/agaricus.txt.test", zeroBased = false)
 }
diff --git a/tests/travis/run_test.sh b/tests/travis/run_test.sh
index 3d9602fd134d..b4a7004a4db1 100755
--- a/tests/travis/run_test.sh
+++ b/tests/travis/run_test.sh
@@ -27,8 +27,8 @@ fi
 if [ ${TASK} == "java_test" ]; then
     set -e
     cd jvm-packages
-    mvn -q clean install -DskipTests -Dmaven.test.skip
-    mvn -q test
+    mvn clean install -DskipTests -Dmaven.test.skip
+    mvn test
 fi
 
 if [ ${TASK} == "cmake_test" ]; then