Named loss functions similar to scikit-learn instead of R gbm package.

src/gb_dt.jl

@@ -18,7 +18,7 @@ type GBDT <: GradientBoost
   num_iterations::Int
   tree_options::Dict
 
-  function GBDT(loss_function=GaussianLoss(),
+  function GBDT(loss_function=LeastSquares(),
     sampling_rate=0.6, learning_rate=0.1, 
     num_iterations=100, tree_options=Dict())
 
@@ -53,8 +53,8 @@ function GB.build_base_func(
   function val_func(node)
     inst_ind = inst_node_index.n2i[node]
 
-    # If loss function is Gaussian, we don't need need to change values.
-    if typeof(gb.loss_function) <: GaussianLoss
+    # If loss function is LeastSquares, we don't need need to change values.
+    if typeof(gb.loss_function) <: LeastSquares
       val = node.majority
     else
       val = fit_best_constant(gb.loss_function,
@@ -131,13 +131,13 @@ function update_regions!{T}(n2v::Dict{Leaf, T}, leaf::Leaf, val_func::Function)
 end
 
 # Loss function fits
-function fit_best_constant(lf::LaplaceLoss,
+function fit_best_constant(lf::LeastAbsoluteDeviation,
   labels, psuedo, psuedo_pred, prev_func_pred)
 
   values = labels .- prev_func_pred
   median(values)
 end
-function fit_best_constant(lf::BernoulliLoss,
+function fit_best_constant(lf::BinomialDeviance,
   labels, psuedo, psuedo_pred, prev_func_pred)
 
   num = sum(psuedo)

src/gb_learner.jl

@@ -18,7 +18,7 @@ type GBL <: GradientBoost
   num_iterations::Int
   learner
 
-  function GBL(learner, loss_function=GaussianLoss(),
+  function GBL(learner, loss_function=LeastSquares(),
     sampling_rate=0.8, learning_rate=0.1, 
     num_iterations=100)
 
@@ -70,14 +70,14 @@ function learner_predict(lf::LossFunction, learner, model, instances)
 end
 
 # Loss function fits
-function fit_best_constant(lf::GaussianLoss,
+function fit_best_constant(lf::LeastSquares,
   labels, psuedo, psuedo_pred, prev_func_pred)
 
   # No refitting required
   1.0
 end
 
-function fit_best_constant(lf::LaplaceLoss,
+function fit_best_constant(lf::LeastAbsoluteDeviation,
   labels, psuedo, psuedo_pred, prev_func_pred)
 
   weights = abs(psuedo_pred)
@@ -91,10 +91,10 @@ function fit_best_constant(lf::LaplaceLoss,
 
   weighted_median(weights, values)
 end
-function fit_best_constant(lf::BernoulliLoss,
+function fit_best_constant(lf::BinomialDeviance,
   labels, psuedo, psuedo_pred, prev_func_pred)
 
-  # TODO(svs14): Add fit_best_constant (BernoulliLoss) for base learner.
+  # TODO(svs14): Add fit_best_constant (BinomialDeviance) for base learner.
   error("$(typeof(lf)) is not implemented for GBLearner.")
 end
 

src/loss.jl

@@ -7,9 +7,9 @@ export LossFunction,
        loss,
        negative_gradient,
        minimizing_scalar,
-       GaussianLoss,
-       LaplaceLoss,
-       BernoulliLoss
+       LeastSquares,
+       LeastAbsoluteDeviation,
+       BinomialDeviance
 
 # Loss function.
 abstract LossFunction
@@ -37,50 +37,50 @@ negative_gradient(lf::LossFunction, y, y_pred) = err_must_be_overriden()
 # @return Scalar value.
 minimizing_scalar(lf::LossFunction, y) = err_must_be_overriden()
 
-# Gaussian (Least Squares)
-type GaussianLoss <: LossFunction; end
+# LeastSquares
+type LeastSquares <: LossFunction; end
 
-function loss(lf::GaussianLoss, y, y_pred)
+function loss(lf::LeastSquares, y, y_pred)
   mean((y .- y_pred) .^ 2.0)
 end
 
-function negative_gradient(lf::GaussianLoss, y, y_pred)
+function negative_gradient(lf::LeastSquares, y, y_pred)
   y .- y_pred
 end
 
-function minimizing_scalar(lf::GaussianLoss, y)
+function minimizing_scalar(lf::LeastSquares, y)
   mean(y)
 end
 
 
-# Laplace (Least Absolute Deviation)
-type LaplaceLoss <: LossFunction; end
+# LeastAbsoluteDeviation
+type LeastAbsoluteDeviation <: LossFunction; end
 
-function loss(lf::LaplaceLoss, y, y_pred)
+function loss(lf::LeastAbsoluteDeviation, y, y_pred)
   mean(abs(y .- y_pred))
 end
 
-function negative_gradient(lf::LaplaceLoss, y, y_pred)
+function negative_gradient(lf::LeastAbsoluteDeviation, y, y_pred)
   sign(y .- y_pred)
 end
 
-function minimizing_scalar(lf::LaplaceLoss, y)
+function minimizing_scalar(lf::LeastAbsoluteDeviation, y)
   median(y)
 end
 
 
-# Bernoulli Loss (Two Classes {0,1})
-type BernoulliLoss <: LossFunction; end
+# Binomial Deviance (Two Classes {0,1})
+type BinomialDeviance <: LossFunction; end
 
-function loss(lf::BernoulliLoss, y, y_pred)
+function loss(lf::BinomialDeviance, y, y_pred)
   -2.0 .* mean(y .* y_pred .- log(1.0 .+ exp(y_pred)))
 end
 
-function negative_gradient(lf::BernoulliLoss, y, y_pred)
+function negative_gradient(lf::BinomialDeviance, y, y_pred)
   y .- 1.0 ./ (1.0 .+ exp(-y_pred))
 end
 
-function minimizing_scalar(lf::BernoulliLoss, y)
+function minimizing_scalar(lf::BinomialDeviance, y)
   y_sum = sum(y)
   y_length = length(y)
   log(y_sum / (y_length - y_sum))

src/ml.jl

@@ -10,9 +10,9 @@ export GBProblem,
        fit!,
        predict!,
        LossFunction,
-       GaussianLoss,
-       LaplaceLoss,
-       BernoulliLoss,
+       LeastSquares,
+       LeastAbsoluteDeviation,
+       BinomialDeviance,
        GBDT,
        GBL,
        learner_fit,
@@ -65,11 +65,11 @@ end
 function postprocess_pred(
   output::Symbol, lf::LossFunction, predictions::Vector{Float64})
 
-  if output == :class && typeof(lf) <: BernoulliLoss
+  if output == :class && typeof(lf) <: BinomialDeviance
     return round(logistic(predictions))
-  elseif output == :class_prob && typeof(lf) <: BernoulliLoss
+  elseif output == :class_prob && typeof(lf) <: BinomialDeviance
     return logistic(predictions)
-  elseif output == :regression && !(typeof(lf) <: BernoulliLoss)
+  elseif output == :regression && !(typeof(lf) <: BinomialDeviance)
     return predictions
   else
     error("Cannot handle $(output) and $(typeof(lf)) together.")

test/test_gb.jl

@@ -58,13 +58,13 @@ facts("Gradient Boost") do
 
   context("stochastic_gradient_boost works") do
     # Sanity check
-    sgb = StubGradientBoost(GaussianLoss(), 1.0, 0.5, 1)
+    sgb = StubGradientBoost(LeastSquares(), 1.0, 0.5, 1)
     model = stochastic_gradient_boost(sgb, sgb_instances, sgb_labels)
     @fact 1 => 1
   end
 
   context("fit returns model") do
-    sgb = StubGradientBoost(GaussianLoss(), 1.0, 0.5, 1)
+    sgb = StubGradientBoost(LeastSquares(), 1.0, 0.5, 1)
     model = stochastic_gradient_boost(sgb, sgb_instances, sgb_labels)
     @fact typeof(model) <: GBModel => true
   end
@@ -74,7 +74,7 @@ facts("Gradient Boost") do
       3.0
       3.5
     ]
-    sgb = StubGradientBoost(GaussianLoss(), 1.0, 0.5, 1)
+    sgb = StubGradientBoost(LeastSquares(), 1.0, 0.5, 1)
     model = stochastic_gradient_boost(sgb, sgb_instances, sgb_labels)
     predictions = predict(model, sgb_instances)
     @fact predictions => expected
@@ -84,14 +84,14 @@ facts("Gradient Boost") do
     instances = [1:5 6:10]
     labels = [1:5]
 
-    sgb = StubGradientBoost(GaussianLoss(), 1, 1, 1)
+    sgb = StubGradientBoost(LeastSquares(), 1, 1, 1)
     indices = create_sample_indices(sgb, instances, labels)
     @fact length(indices) => 5
     @fact length(unique(indices)) => 5
     @fact minimum(indices) >= 1 => true
     @fact maximum(indices) <= 5 => true
 
-    sgb = StubGradientBoost(GaussianLoss(), 0.5, 1, 1)
+    sgb = StubGradientBoost(LeastSquares(), 0.5, 1, 1)
     indices = create_sample_indices(sgb, instances, labels)
     @fact length(indices) => 3
     @fact length(unique(indices)) => 3

test/test_gb_dt.jl

@@ -26,7 +26,7 @@ end
 facts("GB Decision Tree") do
   context("build_base_func works") do
     gb = GBDT(
-      GaussianLoss(),
+      LeastSquares(),
       0.5,
       0.01,
       100
@@ -85,8 +85,8 @@ facts("GB Decision Tree") do
     end
   end
 
-  context("LaplaceLoss fit_best_constant works") do
-    lf = LaplaceLoss()
+  context("LeastAbsoluteDeviation fit_best_constant works") do
+    lf = LeastAbsoluteDeviation()
     dummy_vec = [0.0,0.0,0.0,0.0]
     labels = [0.0,1.0,2.0,3.0]
     prev_func_pred = [3.0,2.0,1.0,0.0]
@@ -97,8 +97,8 @@ facts("GB Decision Tree") do
     )
     @fact actual => expected
   end
-  context("BernoulliLoss fit_best_constant works") do
-    lf = BernoulliLoss()
+  context("BinomialDeviance fit_best_constant works") do
+    lf = BinomialDeviance()
     dummy_vec = [0.0,0.0,0.0,0.0]
 
     labels = [0.0,0.0,1.0,1.0]

test/test_gb_learner.jl

@@ -30,13 +30,13 @@ facts("GB Learner") do
     dummy_model = emp_vec
 
     @fact_throws learner_fit(
-      GaussianLoss(),
+      LeastSquares(),
       dgb,
       emp_mat,
       emp_vec
     )
     @fact_throws learner_predict(
-      GaussianLoss(),
+      LeastSquares(),
       dgb,
       dummy_model,
       emp_mat
@@ -76,8 +76,8 @@ facts("GB Learner") do
     @fact predictions => roughly(expected)
   end
 
-  context("GaussianLoss fit_best_constant works") do
-    lf = GaussianLoss()
+  context("LeastSquares fit_best_constant works") do
+    lf = LeastSquares()
     dummy_vec = [0.0,0.0,0.0,0.0]
     expected = 1.0
 
@@ -86,8 +86,8 @@ facts("GB Learner") do
     )
     @fact actual => expected
   end
-  context("LaplaceLoss fit_best_constant works") do
-    lf = LaplaceLoss()
+  context("LeastAbsoluteDeviation fit_best_constant works") do
+    lf = LeastAbsoluteDeviation()
     dummy_vec = [0.0,0.0,0.0,0.0]
     labels = [0.0,1.0,2.0,3.0]
     psuedo_pred = [3.0,2.0,1.0,0.0]
@@ -99,8 +99,8 @@ facts("GB Learner") do
     )
     @fact actual => roughly(expected)
   end
-  context("BernoulliLoss fit_best_constant throws error") do
-    lf = BernoulliLoss()
+  context("BinomialDeviance fit_best_constant throws error") do
+    lf = BinomialDeviance()
     dummy_vec = [0.0,0.0,0.0,0.0]
 
     @fact_throws GBLearner.fit_best_constant(

test/test_loss.jl

@@ -61,47 +61,47 @@ facts("Loss functions") do
     @fact_throws minimizing_scalar(dlf, emp_vec)
   end
 
-  context("GaussianLoss loss works") do
-    lf = GaussianLoss()
+  context("LeastSquares loss works") do
+    lf = LeastSquares()
     expected = { 0.0, 0.5, 1.0, 2.0, 4.0 }
     test_loss(lf, y_examples, y_pred_examples, expected)
   end
-  context("GaussianLoss negative_gradient works") do
-    lf = GaussianLoss()
+  context("LeastSquares negative_gradient works") do
+    lf = LeastSquares()
     expected = { [0,0], [0,-1], [-1,-1], [-2,0], [-2,-2] }
     test_negative_gradient(lf, y_examples, y_pred_examples, expected)
   end
-  context("GaussianLoss minimizing_scalar works") do
-    lf = GaussianLoss()
+  context("LeastSquares minimizing_scalar works") do
+    lf = LeastSquares()
     expected = { 1.0, 0.5, 0.0, 0.0, -1.0 }
     test_minimizing_scalar(lf, y_examples, expected)
   end
 
-  context("LaplaceLoss loss works") do
-    lf = LaplaceLoss()
+  context("LeastAbsoluteDeviation loss works") do
+    lf = LeastAbsoluteDeviation()
     expected = { 0.0, 0.5, 1.0, 1.0, 2.0 }
     test_loss(lf, y_examples, y_pred_examples, expected)
   end
-  context("LaplaceLoss negative_gradient works") do
-    lf = LaplaceLoss()
+  context("LeastAbsoluteDeviation negative_gradient works") do
+    lf = LeastAbsoluteDeviation()
     expected = { [0,0], [0,-1], [-1,-1], [-1,0], [-1,-1] }
     test_negative_gradient(lf, y_examples, y_pred_examples, expected)
   end
-  context("LaplaceLoss minimizing_scalar works") do
-    lf = LaplaceLoss()
+  context("LeastAbsoluteDeviation minimizing_scalar works") do
+    lf = LeastAbsoluteDeviation()
     expected = { 1.0, 0.5, 0.0, 0.0, -1.0 }
     test_minimizing_scalar(lf, y_examples, expected)
   end
 
-  context("BernoulliLoss loss works") do
-    lf = BernoulliLoss()
+  context("BinomialDeviance loss works") do
+    lf = BinomialDeviance()
     expected = { 
       0.626523, 1.626523, 1.626523, 2.626523, 2.626523,
     }
     test_loss(lf, bern_y_examples, y_pred_examples, expected)
   end
-  context("BernoulliLoss negative_gradient works") do
-    lf = BernoulliLoss()
+  context("BinomialDeviance negative_gradient works") do
+    lf = BinomialDeviance()
     expected = { 
       [0.268941, 0.268941], 
       [0.268941, -0.731059], 
@@ -111,8 +111,8 @@ facts("Loss functions") do
     }
     test_negative_gradient(lf, bern_y_examples, y_pred_examples, expected)
   end
-  context("BernoulliLoss minimizing_scalar works") do
-    lf = BernoulliLoss()
+  context("BinomialDeviance minimizing_scalar works") do
+    lf = BinomialDeviance()
     expected = { Inf, 0.0, 0.0, -Inf, -Inf}
     test_minimizing_scalar(lf, bern_y_examples, expected)
   end