Skip to content

Commit

Permalink
Revert "fix spark tests on machines with many cores (dmlc#4634)"
Browse files Browse the repository at this point in the history
This reverts commit e3906de.
  • Loading branch information
sperlingxx committed Jul 17, 2019
1 parent 06ce287 commit 7703e02
Show file tree
Hide file tree
Showing 2 changed files with 6 additions and 8 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -19,24 +19,23 @@ package ml.dmlc.xgboost4j.scala.spark
import java.io.File

import ml.dmlc.xgboost4j.{LabeledPoint => XGBLabeledPoint}

import org.apache.spark.{SparkConf, SparkContext, TaskFailedListener}
import org.apache.spark.sql._
import org.scalatest.{BeforeAndAfterEach, FunSuite}

import scala.math.min
import scala.util.Random

trait PerTest extends BeforeAndAfterEach { self: FunSuite =>

protected val numWorkers: Int = min(Runtime.getRuntime.availableProcessors(), 4)
protected val numWorkers: Int = Runtime.getRuntime.availableProcessors()

@transient private var currentSession: SparkSession = _

def ss: SparkSession = getOrCreateSession
implicit def sc: SparkContext = ss.sparkContext

protected def sparkSessionBuilder: SparkSession.Builder = SparkSession.builder()
.master(s"local[${numWorkers}]")
.master("local[*]")
.appName("XGBoostSuite")
.config("spark.ui.enabled", false)
.config("spark.driver.memory", "512m")
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -18,17 +18,16 @@ package org.apache.spark

import org.scalatest.FunSuite
import _root_.ml.dmlc.xgboost4j.scala.spark.PerTest

import org.apache.spark.rdd.RDD
import org.apache.spark.sql.SparkSession

import scala.math.min

class SparkParallelismTrackerSuite extends FunSuite with PerTest {

val numParallelism: Int = min(Runtime.getRuntime.availableProcessors(), 4)
val numParallelism: Int = Runtime.getRuntime.availableProcessors()

override protected def sparkSessionBuilder: SparkSession.Builder = SparkSession.builder()
.master(s"local[${numParallelism}]")
.master("local[*]")
.appName("XGBoostSuite")
.config("spark.ui.enabled", true)
.config("spark.driver.memory", "512m")
Expand Down

0 comments on commit 7703e02

Please sign in to comment.