Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

perf: improve lightgbm training performance 4x-10x by setting num_threads to be cores-1 by default for single dataset mode #1282

Merged
merged 2 commits into from
Dec 3, 2021
Merged

perf: improve lightgbm training performance 4x-10x by setting num_threads to be cores-1 by default for single dataset mode #1282

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
6fb7fc4
perf: improve lightgbm training performance 4x-10x by setting num_thr…
imatiach-msft Dec 2, 2021
a52e0f8
Merge branch 'master' into ilmat/def-num-threads
imatiach-msft Dec 3, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 6 additions & 2 deletions lightgbm/src/main/scala/com/microsoft/azure/synapse/ml/lightgbm/LightGBMBase.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -247,8 +247,12 @@ trait LightGBMBase[TrainedModel <: Model[TrainedModel]] extends Estimator[Traine
*
* @return ExecutionParams object containing parameters related to LightGBM execution.
*/
protected def getExecutionParams: ExecutionParams = {
ExecutionParams(getChunkSize, getMatrixType, getNumThreads, getUseSingleDatasetMode)
protected def getExecutionParams(numTasksPerExec: Int): ExecutionParams = {
val execNumThreads =
if (getUseSingleDatasetMode) get(numThreads).getOrElse(numTasksPerExec - 1)
else getNumThreads

ExecutionParams(getChunkSize, getMatrixType, execNumThreads, getUseSingleDatasetMode)
}

protected def getColumnParams: ColumnParams = {
Expand Down
2 changes: 1 addition & 1 deletion lightgbm/src/main/scala/com/microsoft/azure/synapse/ml/lightgbm/LightGBMClassifier.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,7 @@ class LightGBMClassifier(override val uid: String)
getIsUnbalance, getVerbosity, categoricalIndexes, actualNumClasses, getBoostFromAverage,
getBoostingType, get(lambdaL1), get(lambdaL2), get(isProvideTrainingMetric),
get(metric), get(minGainToSplit), get(maxDeltaStep), getMaxBinByFeature, get(minDataInLeaf), getSlotNames,
getDelegate, getDartParams, getExecutionParams, getObjectiveParams)
getDelegate, getDartParams, getExecutionParams(numTasksPerExec), getObjectiveParams)
}

def getModel(trainParams: TrainParams, lightGBMBooster: LightGBMBooster): LightGBMClassificationModel = {
Expand Down
2 changes: 1 addition & 1 deletion lightgbm/src/main/scala/com/microsoft/azure/synapse/ml/lightgbm/LightGBMRanker.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -60,7 +60,7 @@ class LightGBMRanker(override val uid: String)
getVerbosity, categoricalIndexes, getBoostingType, get(lambdaL1), get(lambdaL2), getMaxPosition, getLabelGain,
get(isProvideTrainingMetric), get(metric), getEvalAt, get(minGainToSplit), get(maxDeltaStep),
getMaxBinByFeature, get(minDataInLeaf), getSlotNames, getDelegate, getDartParams,
getExecutionParams, getObjectiveParams)
getExecutionParams(numTasksPerExec), getObjectiveParams)
}

def getModel(trainParams: TrainParams, lightGBMBooster: LightGBMBooster): LightGBMRankerModel = {
Expand Down
2 changes: 1 addition & 1 deletion lightgbm/src/main/scala/com/microsoft/azure/synapse/ml/lightgbm/LightGBMRegressor.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -70,7 +70,7 @@ class LightGBMRegressor(override val uid: String)
getBoostFromAverage, getBoostingType, get(lambdaL1), get(lambdaL2), get(isProvideTrainingMetric),
get(metric), get(minGainToSplit), get(maxDeltaStep),
getMaxBinByFeature, get(minDataInLeaf), getSlotNames, getDelegate,
getDartParams, getExecutionParams, getObjectiveParams)
getDartParams, getExecutionParams(numTasksPerExec), getObjectiveParams)
}

def getModel(trainParams: TrainParams, lightGBMBooster: LightGBMBooster): LightGBMRegressionModel = {
Expand Down