From db1bb0f3c912fb49a4f3e620a0d757783607819e Mon Sep 17 00:00:00 2001 From: Oleksandr Pryimak Date: Thu, 18 Jul 2019 15:01:11 -0700 Subject: [PATCH 1/3] Removed hardcoded paths from 'xgboost4j' To achieve this I had to move the train/test data to the main package and replace all hardcoded occurences of "../../demo" in the tests. Original author is Sergei Lebedev https://github.com/criteo-forks/xgboost/commit/d10b607cc274169a1bbb37ab8a8a15d5c4b7a819 --- jvm-packages/create_jni.py | 9 ++-- jvm-packages/xgboost4j-spark/pom.xml | 7 +++ .../scala/spark/CheckpointManagerSuite.scala | 2 +- .../DeterministicPartitioningSuite.scala | 2 +- .../scala/spark/ParameterSuite.scala | 2 +- .../scala/spark/PersistenceSuite.scala | 3 +- .../scala/spark/RabitRobustnessSuite.scala | 4 +- .../scala/spark/XGBoostClassifierSuite.scala | 4 +- .../scala/spark/XGBoostConfigureSuite.scala | 2 +- .../scala/spark/XGBoostRegressorSuite.scala | 2 +- .../dmlc/xgboost4j/java/BoosterImplTest.java | 53 ++++--------------- .../dmlc/xgboost4j/java/Classification.java | 10 ++++ .../ml/dmlc/xgboost4j/java/DMatrixTest.java | 3 +- .../ml/dmlc/xgboost4j/java/TrainTestData.java | 27 ++++++++++ .../dmlc/xgboost4j/scala/DMatrixSuite.scala | 2 +- .../scala/ScalaBoosterImplSuite.scala | 23 ++------ .../dmlc/xgboost4j/scala}/TrainTestData.scala | 7 ++- 17 files changed, 76 insertions(+), 86 deletions(-) mode change 100644 => 100755 jvm-packages/xgboost4j-spark/src/test/scala/ml/dmlc/xgboost4j/scala/spark/DeterministicPartitioningSuite.scala create mode 100644 jvm-packages/xgboost4j/src/test/java/ml/dmlc/xgboost4j/java/Classification.java mode change 100644 => 100755 jvm-packages/xgboost4j/src/test/java/ml/dmlc/xgboost4j/java/DMatrixTest.java create mode 100644 jvm-packages/xgboost4j/src/test/java/ml/dmlc/xgboost4j/java/TrainTestData.java rename jvm-packages/{xgboost4j-spark/src/test/scala/ml/dmlc/xgboost4j/scala/spark => xgboost4j/src/test/scala/ml/dmlc/xgboost4j/scala}/TrainTestData.scala (94%) diff --git a/jvm-packages/create_jni.py b/jvm-packages/create_jni.py index 4d627cb7c748..21d88b80fb36 100755 --- a/jvm-packages/create_jni.py +++ b/jvm-packages/create_jni.py @@ -110,16 +110,13 @@ def normpath(path): "xgboost4j/src/main/resources") print("copying train/test files") - maybe_makedirs("xgboost4j-spark/src/test/resources") + maybe_makedirs("xgboost4j/src/test/resources") with cd("../demo/regression"): run("{} mapfeat.py".format(sys.executable)) run("{} mknfold.py machine.txt 1".format(sys.executable)) - for file in glob.glob("../demo/regression/machine.txt.t*"): - cp(file, "xgboost4j-spark/src/test/resources") - for file in glob.glob("../demo/data/agaricus.*"): - cp(file, "xgboost4j-spark/src/test/resources") - maybe_makedirs("xgboost4j/src/test/resources") + for file in glob.glob("../demo/regression/machine.txt.t*"): + cp(file, "xgboost4j/src/test/resources") for file in glob.glob("../demo/data/agaricus.*"): cp(file, "xgboost4j/src/test/resources") diff --git a/jvm-packages/xgboost4j-spark/pom.xml b/jvm-packages/xgboost4j-spark/pom.xml index cf4f00445bc0..98bdefe58167 100644 --- a/jvm-packages/xgboost4j-spark/pom.xml +++ b/jvm-packages/xgboost4j-spark/pom.xml @@ -26,6 +26,13 @@ xgboost4j_${scala.binary.version} 1.0.0-SNAPSHOT + + ml.dmlc + xgboost4j_${scala.binary.version} + 1.0.0-SNAPSHOT + test-jar + test + org.apache.spark spark-core_${scala.binary.version} diff --git a/jvm-packages/xgboost4j-spark/src/test/scala/ml/dmlc/xgboost4j/scala/spark/CheckpointManagerSuite.scala b/jvm-packages/xgboost4j-spark/src/test/scala/ml/dmlc/xgboost4j/scala/spark/CheckpointManagerSuite.scala index ddeb48241c86..a67334ead956 100755 --- a/jvm-packages/xgboost4j-spark/src/test/scala/ml/dmlc/xgboost4j/scala/spark/CheckpointManagerSuite.scala +++ b/jvm-packages/xgboost4j-spark/src/test/scala/ml/dmlc/xgboost4j/scala/spark/CheckpointManagerSuite.scala @@ -18,7 +18,7 @@ package ml.dmlc.xgboost4j.scala.spark import java.io.File -import ml.dmlc.xgboost4j.scala.{Booster, DMatrix, XGBoost => SXGBoost} +import ml.dmlc.xgboost4j.scala.{Classification, Booster, DMatrix, XGBoost => SXGBoost} import org.scalatest.FunSuite import org.apache.hadoop.fs.{FileSystem, Path} diff --git a/jvm-packages/xgboost4j-spark/src/test/scala/ml/dmlc/xgboost4j/scala/spark/DeterministicPartitioningSuite.scala b/jvm-packages/xgboost4j-spark/src/test/scala/ml/dmlc/xgboost4j/scala/spark/DeterministicPartitioningSuite.scala old mode 100644 new mode 100755 index 986b0843b5f3..739ea185736e --- a/jvm-packages/xgboost4j-spark/src/test/scala/ml/dmlc/xgboost4j/scala/spark/DeterministicPartitioningSuite.scala +++ b/jvm-packages/xgboost4j-spark/src/test/scala/ml/dmlc/xgboost4j/scala/spark/DeterministicPartitioningSuite.scala @@ -16,8 +16,8 @@ package ml.dmlc.xgboost4j.scala.spark +import ml.dmlc.xgboost4j.scala.Classification import org.scalatest.FunSuite - import org.apache.spark.sql.functions._ class DeterministicPartitioningSuite extends FunSuite with TmpFolderPerSuite with PerTest { diff --git a/jvm-packages/xgboost4j-spark/src/test/scala/ml/dmlc/xgboost4j/scala/spark/ParameterSuite.scala b/jvm-packages/xgboost4j-spark/src/test/scala/ml/dmlc/xgboost4j/scala/spark/ParameterSuite.scala index e0450f90dfd8..9e87847c1b19 100644 --- a/jvm-packages/xgboost4j-spark/src/test/scala/ml/dmlc/xgboost4j/scala/spark/ParameterSuite.scala +++ b/jvm-packages/xgboost4j-spark/src/test/scala/ml/dmlc/xgboost4j/scala/spark/ParameterSuite.scala @@ -17,8 +17,8 @@ package ml.dmlc.xgboost4j.scala.spark import ml.dmlc.xgboost4j.java.XGBoostError +import ml.dmlc.xgboost4j.scala.MultiClassification import org.scalatest.{BeforeAndAfterAll, FunSuite} - import org.apache.spark.ml.param.ParamMap class ParameterSuite extends FunSuite with PerTest with BeforeAndAfterAll { diff --git a/jvm-packages/xgboost4j-spark/src/test/scala/ml/dmlc/xgboost4j/scala/spark/PersistenceSuite.scala b/jvm-packages/xgboost4j-spark/src/test/scala/ml/dmlc/xgboost4j/scala/spark/PersistenceSuite.scala index a3841772c796..c9d5704a5b92 100755 --- a/jvm-packages/xgboost4j-spark/src/test/scala/ml/dmlc/xgboost4j/scala/spark/PersistenceSuite.scala +++ b/jvm-packages/xgboost4j-spark/src/test/scala/ml/dmlc/xgboost4j/scala/spark/PersistenceSuite.scala @@ -20,10 +20,9 @@ import java.io.File import java.util.Arrays import scala.io.Source +import ml.dmlc.xgboost4j.scala.{Classification, DMatrix, Regression} -import ml.dmlc.xgboost4j.scala.DMatrix import scala.util.Random - import org.apache.spark.ml.feature._ import org.apache.spark.ml.{Pipeline, PipelineModel} import org.scalatest.FunSuite diff --git a/jvm-packages/xgboost4j-spark/src/test/scala/ml/dmlc/xgboost4j/scala/spark/RabitRobustnessSuite.scala b/jvm-packages/xgboost4j-spark/src/test/scala/ml/dmlc/xgboost4j/scala/spark/RabitRobustnessSuite.scala index 9655104c35d7..7498a4b172b3 100644 --- a/jvm-packages/xgboost4j-spark/src/test/scala/ml/dmlc/xgboost4j/scala/spark/RabitRobustnessSuite.scala +++ b/jvm-packages/xgboost4j-spark/src/test/scala/ml/dmlc/xgboost4j/scala/spark/RabitRobustnessSuite.scala @@ -19,12 +19,10 @@ package ml.dmlc.xgboost4j.scala.spark import java.util.concurrent.LinkedBlockingDeque import scala.util.Random - import ml.dmlc.xgboost4j.java.{IRabitTracker, Rabit, RabitTracker => PyRabitTracker} import ml.dmlc.xgboost4j.scala.rabit.{RabitTracker => ScalaRabitTracker} import ml.dmlc.xgboost4j.java.IRabitTracker.TrackerStatus -import ml.dmlc.xgboost4j.scala.DMatrix - +import ml.dmlc.xgboost4j.scala.{Classification, DMatrix} import org.apache.spark.{SparkConf, SparkContext} import org.scalatest.FunSuite diff --git a/jvm-packages/xgboost4j-spark/src/test/scala/ml/dmlc/xgboost4j/scala/spark/XGBoostClassifierSuite.scala b/jvm-packages/xgboost4j-spark/src/test/scala/ml/dmlc/xgboost4j/scala/spark/XGBoostClassifierSuite.scala index f2ccdd44ee73..92f050214d0a 100644 --- a/jvm-packages/xgboost4j-spark/src/test/scala/ml/dmlc/xgboost4j/scala/spark/XGBoostClassifierSuite.scala +++ b/jvm-packages/xgboost4j-spark/src/test/scala/ml/dmlc/xgboost4j/scala/spark/XGBoostClassifierSuite.scala @@ -16,13 +16,11 @@ package ml.dmlc.xgboost4j.scala.spark -import ml.dmlc.xgboost4j.scala.{DMatrix, XGBoost => ScalaXGBoost} - +import ml.dmlc.xgboost4j.scala.{Classification, DMatrix, MultiClassification, Synthetic, XGBoost => ScalaXGBoost} import org.apache.spark.ml.linalg._ import org.apache.spark.ml.param.ParamMap import org.apache.spark.sql._ import org.scalatest.FunSuite - import org.apache.spark.Partitioner class XGBoostClassifierSuite extends FunSuite with PerTest { diff --git a/jvm-packages/xgboost4j-spark/src/test/scala/ml/dmlc/xgboost4j/scala/spark/XGBoostConfigureSuite.scala b/jvm-packages/xgboost4j-spark/src/test/scala/ml/dmlc/xgboost4j/scala/spark/XGBoostConfigureSuite.scala index fe16bcda56bf..16dfa4ed945d 100644 --- a/jvm-packages/xgboost4j-spark/src/test/scala/ml/dmlc/xgboost4j/scala/spark/XGBoostConfigureSuite.scala +++ b/jvm-packages/xgboost4j-spark/src/test/scala/ml/dmlc/xgboost4j/scala/spark/XGBoostConfigureSuite.scala @@ -16,7 +16,7 @@ package ml.dmlc.xgboost4j.scala.spark -import ml.dmlc.xgboost4j.scala.{Booster, DMatrix} +import ml.dmlc.xgboost4j.scala.{Booster, Classification, DMatrix} import org.apache.spark.sql._ import org.scalatest.FunSuite diff --git a/jvm-packages/xgboost4j-spark/src/test/scala/ml/dmlc/xgboost4j/scala/spark/XGBoostRegressorSuite.scala b/jvm-packages/xgboost4j-spark/src/test/scala/ml/dmlc/xgboost4j/scala/spark/XGBoostRegressorSuite.scala index ff88ff328659..90f10dc5ef08 100644 --- a/jvm-packages/xgboost4j-spark/src/test/scala/ml/dmlc/xgboost4j/scala/spark/XGBoostRegressorSuite.scala +++ b/jvm-packages/xgboost4j-spark/src/test/scala/ml/dmlc/xgboost4j/scala/spark/XGBoostRegressorSuite.scala @@ -16,7 +16,7 @@ package ml.dmlc.xgboost4j.scala.spark -import ml.dmlc.xgboost4j.scala.{DMatrix, XGBoost => ScalaXGBoost} +import ml.dmlc.xgboost4j.scala.{DMatrix, Ranking, Regression, XGBoost => ScalaXGBoost} import org.apache.spark.ml.linalg.Vector import org.apache.spark.sql.functions._ import org.apache.spark.sql.{DataFrame, Row} diff --git a/jvm-packages/xgboost4j/src/test/java/ml/dmlc/xgboost4j/java/BoosterImplTest.java b/jvm-packages/xgboost4j/src/test/java/ml/dmlc/xgboost4j/java/BoosterImplTest.java index 3b565ab8fe94..8bd674723bb0 100755 --- a/jvm-packages/xgboost4j/src/test/java/ml/dmlc/xgboost4j/java/BoosterImplTest.java +++ b/jvm-packages/xgboost4j/src/test/java/ml/dmlc/xgboost4j/java/BoosterImplTest.java @@ -24,6 +24,7 @@ import java.util.Map; import junit.framework.TestCase; +import org.junit.Before; import org.junit.Test; /** @@ -84,12 +85,18 @@ private Booster trainBooster(DMatrix trainMat, DMatrix testMat) throws XGBoostEr return XGBoost.train(trainMat, paramMap, round, watches, null, null); } + private DMatrix trainMat; + private DMatrix testMat; + + @Before + public void setUp() throws XGBoostError { + trainMat = new DMatrix(Classification.trainFile.toString()); + testMat = new DMatrix(Classification.testFile.toString()); + } + @Test public void testBoosterBasic() throws XGBoostError, IOException { - DMatrix trainMat = new DMatrix("../../demo/data/agaricus.txt.train"); - DMatrix testMat = new DMatrix("../../demo/data/agaricus.txt.test"); - Booster booster = trainBooster(trainMat, testMat); //predict raw output @@ -103,8 +110,6 @@ public void testBoosterBasic() throws XGBoostError, IOException { @Test public void saveLoadModelWithPath() throws XGBoostError, IOException { - DMatrix trainMat = new DMatrix("../../demo/data/agaricus.txt.train"); - DMatrix testMat = new DMatrix("../../demo/data/agaricus.txt.test"); IEvaluation eval = new EvalError(); Booster booster = trainBooster(trainMat, testMat); @@ -121,8 +126,6 @@ public void saveLoadModelWithPath() throws XGBoostError, IOException { @Test public void saveLoadModelWithStream() throws XGBoostError, IOException { - DMatrix trainMat = new DMatrix("../../demo/data/agaricus.txt.train"); - DMatrix testMat = new DMatrix("../../demo/data/agaricus.txt.test"); Booster booster = trainBooster(trainMat, testMat); @@ -310,8 +313,6 @@ public void testAscendMetrics() { @Test public void testBoosterEarlyStop() throws XGBoostError, IOException { - DMatrix trainMat = new DMatrix("../../demo/data/agaricus.txt.train"); - DMatrix testMat = new DMatrix("../../demo/data/agaricus.txt.test"); Map paramMap = new HashMap() { { put("max_depth", 3); @@ -363,8 +364,6 @@ private void testWithQuantileHisto(DMatrix trainingSet, Map wat @Test public void testQuantileHistoDepthWise() throws XGBoostError { - DMatrix trainMat = new DMatrix("../../demo/data/agaricus.txt.train"); - DMatrix testMat = new DMatrix("../../demo/data/agaricus.txt.test"); Map paramMap = new HashMap() { { put("max_depth", 3); @@ -383,8 +382,6 @@ public void testQuantileHistoDepthWise() throws XGBoostError { @Test public void testQuantileHistoLossGuide() throws XGBoostError { - DMatrix trainMat = new DMatrix("../../demo/data/agaricus.txt.train"); - DMatrix testMat = new DMatrix("../../demo/data/agaricus.txt.test"); Map paramMap = new HashMap() { { put("max_depth", 0); @@ -404,8 +401,6 @@ public void testQuantileHistoLossGuide() throws XGBoostError { @Test public void testQuantileHistoLossGuideMaxBin() throws XGBoostError { - DMatrix trainMat = new DMatrix("../../demo/data/agaricus.txt.train"); - DMatrix testMat = new DMatrix("../../demo/data/agaricus.txt.test"); Map paramMap = new HashMap() { { put("max_depth", 0); @@ -425,8 +420,6 @@ public void testQuantileHistoLossGuideMaxBin() throws XGBoostError { @Test public void testDumpModelJson() throws XGBoostError { - DMatrix trainMat = new DMatrix("../../demo/data/agaricus.txt.train"); - DMatrix testMat = new DMatrix("../../demo/data/agaricus.txt.test"); Booster booster = trainBooster(trainMat, testMat); String[] dump = booster.getModelDump("", false, "json"); @@ -441,9 +434,6 @@ public void testDumpModelJson() throws XGBoostError { @Test public void testGetFeatureScore() throws XGBoostError { - DMatrix trainMat = new DMatrix("../../demo/data/agaricus.txt.train"); - DMatrix testMat = new DMatrix("../../demo/data/agaricus.txt.test"); - Booster booster = trainBooster(trainMat, testMat); String[] featureNames = new String[126]; for(int i = 0; i < 126; i++) featureNames[i] = "test_feature_name_" + i; @@ -453,9 +443,6 @@ public void testGetFeatureScore() throws XGBoostError { @Test public void testGetFeatureImportanceGain() throws XGBoostError { - DMatrix trainMat = new DMatrix("../../demo/data/agaricus.txt.train"); - DMatrix testMat = new DMatrix("../../demo/data/agaricus.txt.test"); - Booster booster = trainBooster(trainMat, testMat); String[] featureNames = new String[126]; for(int i = 0; i < 126; i++) featureNames[i] = "test_feature_name_" + i; @@ -465,9 +452,6 @@ public void testGetFeatureImportanceGain() throws XGBoostError { @Test public void testGetFeatureImportanceTotalGain() throws XGBoostError { - DMatrix trainMat = new DMatrix("../../demo/data/agaricus.txt.train"); - DMatrix testMat = new DMatrix("../../demo/data/agaricus.txt.test"); - Booster booster = trainBooster(trainMat, testMat); String[] featureNames = new String[126]; for(int i = 0; i < 126; i++) featureNames[i] = "test_feature_name_" + i; @@ -477,9 +461,6 @@ public void testGetFeatureImportanceTotalGain() throws XGBoostError { @Test public void testGetFeatureImportanceCover() throws XGBoostError { - DMatrix trainMat = new DMatrix("../../demo/data/agaricus.txt.train"); - DMatrix testMat = new DMatrix("../../demo/data/agaricus.txt.test"); - Booster booster = trainBooster(trainMat, testMat); String[] featureNames = new String[126]; for(int i = 0; i < 126; i++) featureNames[i] = "test_feature_name_" + i; @@ -489,9 +470,6 @@ public void testGetFeatureImportanceCover() throws XGBoostError { @Test public void testGetFeatureImportanceTotalCover() throws XGBoostError { - DMatrix trainMat = new DMatrix("../../demo/data/agaricus.txt.train"); - DMatrix testMat = new DMatrix("../../demo/data/agaricus.txt.test"); - Booster booster = trainBooster(trainMat, testMat); String[] featureNames = new String[126]; for(int i = 0; i < 126; i++) featureNames[i] = "test_feature_name_" + i; @@ -501,7 +479,6 @@ public void testGetFeatureImportanceTotalCover() throws XGBoostError { @Test public void testQuantileHistoDepthwiseMaxDepth() throws XGBoostError { - DMatrix trainMat = new DMatrix("../../demo/data/agaricus.txt.train"); Map paramMap = new HashMap() { { put("max_depth", 3); @@ -519,8 +496,6 @@ public void testQuantileHistoDepthwiseMaxDepth() throws XGBoostError { @Test public void testQuantileHistoDepthwiseMaxDepthMaxBin() throws XGBoostError { - DMatrix trainMat = new DMatrix("../../demo/data/agaricus.txt.train"); - DMatrix testMat = new DMatrix("../../demo/data/agaricus.txt.test"); Map paramMap = new HashMap() { { put("max_depth", 3); @@ -544,9 +519,6 @@ public void testQuantileHistoDepthwiseMaxDepthMaxBin() throws XGBoostError { */ @Test public void testCV() throws XGBoostError { - //load train mat - DMatrix trainMat = new DMatrix("../../demo/data/agaricus.txt.train"); - //set params Map param = new HashMap() { { @@ -573,8 +545,6 @@ public void testCV() throws XGBoostError { */ @Test public void testTrainFromExistingModel() throws XGBoostError, IOException { - DMatrix trainMat = new DMatrix("../../demo/data/agaricus.txt.train"); - DMatrix testMat = new DMatrix("../../demo/data/agaricus.txt.test"); IEvaluation eval = new EvalError(); Map paramMap = new HashMap() { @@ -624,9 +594,6 @@ public void testTrainFromExistingModel() throws XGBoostError, IOException { */ @Test public void testSetAndGetAttrs() throws XGBoostError { - DMatrix trainMat = new DMatrix("../../demo/data/agaricus.txt.train"); - DMatrix testMat = new DMatrix("../../demo/data/agaricus.txt.test"); - Booster booster = trainBooster(trainMat, testMat); booster.setAttr("testKey1", "testValue1"); TestCase.assertEquals(booster.getAttr("testKey1"), "testValue1"); diff --git a/jvm-packages/xgboost4j/src/test/java/ml/dmlc/xgboost4j/java/Classification.java b/jvm-packages/xgboost4j/src/test/java/ml/dmlc/xgboost4j/java/Classification.java new file mode 100644 index 000000000000..8ca3ecf231a0 --- /dev/null +++ b/jvm-packages/xgboost4j/src/test/java/ml/dmlc/xgboost4j/java/Classification.java @@ -0,0 +1,10 @@ +package ml.dmlc.xgboost4j.java; + + +import java.io.File; + +public class Classification { + public static File trainFile = TrainTestData.getResourceFile("/agaricus.txt.train"); + public static File testFile = TrainTestData.getResourceFile("/agaricus.txt.test"); +} + diff --git a/jvm-packages/xgboost4j/src/test/java/ml/dmlc/xgboost4j/java/DMatrixTest.java b/jvm-packages/xgboost4j/src/test/java/ml/dmlc/xgboost4j/java/DMatrixTest.java old mode 100644 new mode 100755 index 75f52d877eac..c47c1aaba783 --- a/jvm-packages/xgboost4j/src/test/java/ml/dmlc/xgboost4j/java/DMatrixTest.java +++ b/jvm-packages/xgboost4j/src/test/java/ml/dmlc/xgboost4j/java/DMatrixTest.java @@ -60,8 +60,7 @@ public void testCreateFromDataIterator() throws XGBoostError { @Test public void testCreateFromFile() throws XGBoostError { //create DMatrix from file - String filePath = writeResourceIntoTempFile("/agaricus.txt.test"); - DMatrix dmat = new DMatrix(filePath); + DMatrix dmat = new DMatrix(Classification.testFile.toString()); //get label float[] labels = dmat.getLabel(); //check length diff --git a/jvm-packages/xgboost4j/src/test/java/ml/dmlc/xgboost4j/java/TrainTestData.java b/jvm-packages/xgboost4j/src/test/java/ml/dmlc/xgboost4j/java/TrainTestData.java new file mode 100644 index 000000000000..c37346984c0b --- /dev/null +++ b/jvm-packages/xgboost4j/src/test/java/ml/dmlc/xgboost4j/java/TrainTestData.java @@ -0,0 +1,27 @@ +package ml.dmlc.xgboost4j.java; + +import java.io.File; +import java.io.InputStream; +import java.nio.file.Files; +import java.nio.file.StandardCopyOption; + +public class TrainTestData { + static File getResourceFile(String resource) { + InputStream is = TrainTestData.class.getResourceAsStream(resource); + if (is == null) { + throw new RuntimeException("Failed to resolve resource " + resource); + } + try { + try { + File file = File.createTempFile(resource.substring(1), ""); + Files.copy(is, file.toPath(), StandardCopyOption.REPLACE_EXISTING); + file.deleteOnExit(); + return file; + } finally { + is.close(); + } + } catch (Exception e) { + throw new RuntimeException("Failed to load the resource " + resource, e); + } + } +} diff --git a/jvm-packages/xgboost4j/src/test/scala/ml/dmlc/xgboost4j/scala/DMatrixSuite.scala b/jvm-packages/xgboost4j/src/test/scala/ml/dmlc/xgboost4j/scala/DMatrixSuite.scala index 87ff8e006a98..c3d6cc2f2abb 100644 --- a/jvm-packages/xgboost4j/src/test/scala/ml/dmlc/xgboost4j/scala/DMatrixSuite.scala +++ b/jvm-packages/xgboost4j/src/test/scala/ml/dmlc/xgboost4j/scala/DMatrixSuite.scala @@ -25,7 +25,7 @@ import ml.dmlc.xgboost4j.java.{DMatrix => JDMatrix} class DMatrixSuite extends FunSuite { test("create DMatrix from File") { - val dmat = new DMatrix("../../demo/data/agaricus.txt.test") + val dmat = new DMatrix(Classification.testFile.toString) // get label val labels: Array[Float] = dmat.getLabel // check length diff --git a/jvm-packages/xgboost4j/src/test/scala/ml/dmlc/xgboost4j/scala/ScalaBoosterImplSuite.scala b/jvm-packages/xgboost4j/src/test/scala/ml/dmlc/xgboost4j/scala/ScalaBoosterImplSuite.scala index adea1b1ec93a..785b4a3302d1 100644 --- a/jvm-packages/xgboost4j/src/test/scala/ml/dmlc/xgboost4j/scala/ScalaBoosterImplSuite.scala +++ b/jvm-packages/xgboost4j/src/test/scala/ml/dmlc/xgboost4j/scala/ScalaBoosterImplSuite.scala @@ -26,6 +26,9 @@ import ml.dmlc.xgboost4j.java.XGBoostError class ScalaBoosterImplSuite extends FunSuite { + private val trainMat = new DMatrix(Classification.trainFile.toString) + private val testMat = new DMatrix(Classification.testFile.toString) + private class EvalError extends EvalTrait { val logger = LogFactory.getLog(classOf[EvalError]) @@ -95,9 +98,6 @@ class ScalaBoosterImplSuite extends FunSuite { } test("basic operation of booster") { - val trainMat = new DMatrix("../../demo/data/agaricus.txt.train") - val testMat = new DMatrix("../../demo/data/agaricus.txt.test") - val booster = trainBooster(trainMat, testMat) val predicts = booster.predict(testMat, true) val eval = new EvalError @@ -105,9 +105,6 @@ class ScalaBoosterImplSuite extends FunSuite { } test("save/load model with path") { - - val trainMat = new DMatrix("../../demo/data/agaricus.txt.train") - val testMat = new DMatrix("../../demo/data/agaricus.txt.test") val eval = new EvalError val booster = trainBooster(trainMat, testMat) // save and load @@ -122,8 +119,6 @@ class ScalaBoosterImplSuite extends FunSuite { } test("save/load model with stream") { - val trainMat = new DMatrix("../../demo/data/agaricus.txt.train") - val testMat = new DMatrix("../../demo/data/agaricus.txt.test") val eval = new EvalError val booster = trainBooster(trainMat, testMat) // save and load @@ -138,7 +133,6 @@ class ScalaBoosterImplSuite extends FunSuite { } test("cross validation") { - val trainMat = new DMatrix("../../demo/data/agaricus.txt.train") val params = List("eta" -> "1.0", "max_depth" -> "3", "silent" -> "1", "nthread" -> "6", "objective" -> "binary:logistic", "gamma" -> "1.0", "eval_metric" -> "error").toMap val round = 2 @@ -147,8 +141,6 @@ class ScalaBoosterImplSuite extends FunSuite { } test("test with quantile histo depthwise") { - val trainMat = new DMatrix("../../demo/data/agaricus.txt.train") - val testMat = new DMatrix("../../demo/data/agaricus.txt.test") val paramMap = List("max_depth" -> "3", "silent" -> "0", "objective" -> "binary:logistic", "tree_method" -> "hist", "grow_policy" -> "depthwise", "eval_metric" -> "auc").toMap @@ -157,8 +149,6 @@ class ScalaBoosterImplSuite extends FunSuite { } test("test with quantile histo lossguide") { - val trainMat = new DMatrix("../../demo/data/agaricus.txt.train") - val testMat = new DMatrix("../../demo/data/agaricus.txt.test") val paramMap = List("max_depth" -> "0", "silent" -> "0", "objective" -> "binary:logistic", "tree_method" -> "hist", "grow_policy" -> "lossguide", "max_leaves" -> "8", "eval_metric" -> "auc").toMap @@ -167,8 +157,6 @@ class ScalaBoosterImplSuite extends FunSuite { } test("test with quantile histo lossguide with max bin") { - val trainMat = new DMatrix("../../demo/data/agaricus.txt.train") - val testMat = new DMatrix("../../demo/data/agaricus.txt.test") val paramMap = List("max_depth" -> "0", "silent" -> "0", "objective" -> "binary:logistic", "tree_method" -> "hist", "grow_policy" -> "lossguide", "max_leaves" -> "8", "max_bin" -> "16", @@ -178,8 +166,6 @@ class ScalaBoosterImplSuite extends FunSuite { } test("test with quantile histo depthwidth with max depth") { - val trainMat = new DMatrix("../../demo/data/agaricus.txt.train") - val testMat = new DMatrix("../../demo/data/agaricus.txt.test") val paramMap = List("max_depth" -> "0", "silent" -> "0", "objective" -> "binary:logistic", "tree_method" -> "hist", "grow_policy" -> "depthwise", "max_leaves" -> "8", "max_depth" -> "2", @@ -189,8 +175,6 @@ class ScalaBoosterImplSuite extends FunSuite { } test("test with quantile histo depthwidth with max depth and max bin") { - val trainMat = new DMatrix("../../demo/data/agaricus.txt.train") - val testMat = new DMatrix("../../demo/data/agaricus.txt.test") val paramMap = List("max_depth" -> "0", "silent" -> "0", "objective" -> "binary:logistic", "tree_method" -> "hist", "grow_policy" -> "depthwise", "max_depth" -> "2", "max_bin" -> "2", @@ -200,7 +184,6 @@ class ScalaBoosterImplSuite extends FunSuite { } test("test training from existing model in scala") { - val trainMat = new DMatrix("../../demo/data/agaricus.txt.train") val paramMap = List("max_depth" -> "0", "silent" -> "0", "objective" -> "binary:logistic", "tree_method" -> "hist", "grow_policy" -> "depthwise", "max_depth" -> "2", "max_bin" -> "2", diff --git a/jvm-packages/xgboost4j-spark/src/test/scala/ml/dmlc/xgboost4j/scala/spark/TrainTestData.scala b/jvm-packages/xgboost4j/src/test/scala/ml/dmlc/xgboost4j/scala/TrainTestData.scala similarity index 94% rename from jvm-packages/xgboost4j-spark/src/test/scala/ml/dmlc/xgboost4j/scala/spark/TrainTestData.scala rename to jvm-packages/xgboost4j/src/test/scala/ml/dmlc/xgboost4j/scala/TrainTestData.scala index 1a64e2d03529..43a9aa8f664a 100644 --- a/jvm-packages/xgboost4j-spark/src/test/scala/ml/dmlc/xgboost4j/scala/spark/TrainTestData.scala +++ b/jvm-packages/xgboost4j/src/test/scala/ml/dmlc/xgboost4j/scala/TrainTestData.scala @@ -14,12 +14,14 @@ limitations under the License. */ -package ml.dmlc.xgboost4j.scala.spark +package ml.dmlc.xgboost4j.scala import scala.collection.mutable import scala.io.Source import ml.dmlc.xgboost4j.{LabeledPoint => XGBLabeledPoint} +import ml.dmlc.xgboost4j.java.{Classification => JClassification} + trait TrainTestData { protected def getResourceLines(resource: String): Iterator[String] = { require(resource.startsWith("/"), "resource must start with /") @@ -62,6 +64,9 @@ trait TrainTestData { } object Classification extends TrainTestData { + private[xgboost4j] val trainFile = JClassification.trainFile + private[xgboost4j] val testFile = JClassification.testFile + val train: Seq[XGBLabeledPoint] = getLabeledPoints("/agaricus.txt.train", zeroBased = false) val test: Seq[XGBLabeledPoint] = getLabeledPoints("/agaricus.txt.test", zeroBased = false) } From 73c520f76386e36b7cc843e8d2a719ad81c393b0 Mon Sep 17 00:00:00 2001 From: Oleksandr Pryimak Date: Fri, 19 Jul 2019 16:03:59 -0700 Subject: [PATCH 2/3] Update checktyle xml to point to proper DTD --- jvm-packages/checkstyle-suppressions.xml | 4 ++-- jvm-packages/checkstyle.xml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/jvm-packages/checkstyle-suppressions.xml b/jvm-packages/checkstyle-suppressions.xml index 366f6a01e4e0..14a9efd130fc 100644 --- a/jvm-packages/checkstyle-suppressions.xml +++ b/jvm-packages/checkstyle-suppressions.xml @@ -16,8 +16,8 @@ --> + "-//Checkstyle//DTD SuppressionFilter Configuration 1.1//EN" + "https://checkstyle.org/dtds/suppressions_1_1.dtd"> + "-//Checkstyle//DTD Checkstyle Configuration 1.3//EN" + "https://checkstyle.org/dtds/configuration_1_3.dtd">