From 7a0b52807b870685abb6a6b75bc7fb09670b6be1 Mon Sep 17 00:00:00 2001 From: schalkdaniel Date: Fri, 15 Nov 2019 15:04:00 +0100 Subject: [PATCH 1/3] move optimizing steps to the optimizer class --- src/compboost.cpp | 31 +++++++------------------------ src/optimizer.cpp | 19 +++++++++++++++++++ src/optimizer.h | 7 +++++-- 3 files changed, 31 insertions(+), 26 deletions(-) diff --git a/src/compboost.cpp b/src/compboost.cpp index adf92dbd..4f279094 100644 --- a/src/compboost.cpp +++ b/src/compboost.cpp @@ -57,47 +57,30 @@ void Compboost::train (const unsigned int& trace, std::shared_ptrsetActualIteration(actual_iteration); sh_ptr_response->updatePseudoResiduals(sh_ptr_loss); - // Cast integer k to string for baselearner identifier: - std::string temp_string = std::to_string(k); - std::shared_ptr sh_ptr_blearner_selected = sh_ptr_optimizer->findBestBaselearner(temp_string, - sh_ptr_response, used_baselearner_list.getMap()); - - // Prediction is needed more often, use a temp vector to avoid multiple computations: - blearner_pred_temp = sh_ptr_blearner_selected->predict(); - - sh_ptr_optimizer->calculateStepSize(sh_ptr_loss, sh_ptr_response, blearner_pred_temp); - - // Insert new base-learner to vector of selected base-learner. The parameter are estimated here, hence - // the contribution to the old parameter is the estimated parameter times the learning rate times - // the step size. Therefore we have to pass the step size which changes in each iteration: - blearner_track.insertBaselearner(sh_ptr_blearner_selected, sh_ptr_optimizer->getStepSize(actual_iteration)); - sh_ptr_response->updatePrediction(learning_rate, sh_ptr_optimizer->getStepSize(actual_iteration), blearner_pred_temp); + sh_ptr_optimizer->optimize(actual_iteration, learning_rate, sh_ptr_loss, sh_ptr_response, blearner_track, + used_baselearner_list); - // Log the current step: - // The last term has to be the prediction or anything like that. This is - // important to track the risk (inbag or oob)!!!! - logger_list->logCurrent(actual_iteration, sh_ptr_response, sh_ptr_blearner_selected, learning_rate, sh_ptr_optimizer->getStepSize(actual_iteration)); + logger_list->logCurrent(actual_iteration, sh_ptr_response, blearner_track.getBaselearnerVector().back(), learning_rate, sh_ptr_optimizer->getStepSize(actual_iteration)); // Calculate and log risk: risk.push_back(sh_ptr_response->calculateEmpiricalRisk(sh_ptr_loss)); // Get status of the algorithm (is the stopping criteria reached?). The negation here // seems a bit weird, but it makes the while loop easier to read: - stop_the_algorithm = ! logger_list->getStopperStatus(stop_if_all_stopper_fulfilled); + is_stopc_reached = ! logger_list->getStopperStatus(stop_if_all_stopper_fulfilled); if (helper::checkTracePrinter(actual_iteration, trace)) logger_list->printLoggerStatus(risk.back()); k += 1; diff --git a/src/optimizer.cpp b/src/optimizer.cpp index 6d501c94..bbf46383 100644 --- a/src/optimizer.cpp +++ b/src/optimizer.cpp @@ -148,6 +148,25 @@ std::shared_ptr OptimizerCoordinateDescent::findBestBasel } } +void OptimizerCoordinateDescent::optimize (const unsigned int& actual_iteration, const double& learning_rate, const std::shared_ptr sh_ptr_loss, const std::shared_ptr sh_ptr_response, + blearnertrack::BaselearnerTrack& blearner_track, const blearnerlist::BaselearnerFactoryList& blearner_list) +{ + std::string temp_string = std::to_string(actual_iteration); + std::shared_ptr sh_ptr_blearner_selected = findBestBaselearner(temp_string, + sh_ptr_response, blearner_list.getMap()); + + // Prediction is needed more often, use a temp vector to avoid multiple computations: + arma::mat blearner_pred_temp = sh_ptr_blearner_selected->predict(); + + calculateStepSize(sh_ptr_loss, sh_ptr_response, blearner_pred_temp); + + // Insert new base-learner to vector of selected base-learner. The parameter are estimated here, hence + // the contribution to the old parameter is the estimated parameter times the learning rate times + // the step size. Therefore we have to pass the step size which changes in each iteration: + blearner_track.insertBaselearner(sh_ptr_blearner_selected, getStepSize(actual_iteration)); + sh_ptr_response->updatePrediction(learning_rate, getStepSize(actual_iteration), blearner_pred_temp); +} + void OptimizerCoordinateDescent::calculateStepSize (std::shared_ptr sh_ptr_loss, std::shared_ptr sh_ptr_response, const arma::vec& baselearner_prediction) { diff --git a/src/optimizer.h b/src/optimizer.h index 4c250ed1..487012e5 100644 --- a/src/optimizer.h +++ b/src/optimizer.h @@ -30,6 +30,7 @@ #include "baselearner.h" #include "baselearner_factory_list.h" +#include "baselearner_track.h" #include "loss.h" #include "line_search.h" #include "helper.h" @@ -50,6 +51,8 @@ class Optimizer virtual std::shared_ptr findBestBaselearner (const std::string&, std::shared_ptr, const blearner_factory_map&) const = 0; + virtual void optimize (const unsigned int&, const double&, const std::shared_ptr, const std::shared_ptr, + blearnertrack::BaselearnerTrack&, const blearnerlist::BaselearnerFactoryList&) = 0; // loss, target, model_prediction, base_learner_prediction (prediction of newly selected base-learner) virtual void calculateStepSize (std::shared_ptr, std::shared_ptr, const arma::vec&) = 0; @@ -82,14 +85,14 @@ class OptimizerCoordinateDescent : public Optimizer std::shared_ptr findBestBaselearner (const std::string&, std::shared_ptr, const blearner_factory_map&) const; + void optimize (const unsigned int&, const double&, const std::shared_ptr, const std::shared_ptr, + blearnertrack::BaselearnerTrack&, const blearnerlist::BaselearnerFactoryList&); void calculateStepSize (std::shared_ptr, std::shared_ptr, const arma::vec&); std::vector getStepSize () const; double getStepSize (const unsigned int&) const; }; -// Coordinate Descent with line search: -// ------------------------------------------- class OptimizerCoordinateDescentLineSearch : public OptimizerCoordinateDescent { public: From 261e61435d3716078e2e454574ab46a51889b4f5 Mon Sep 17 00:00:00 2001 From: schalkdaniel Date: Fri, 15 Nov 2019 15:59:07 +0100 Subject: [PATCH 2/3] updates of optimizer and response structure --- src/optimizer.cpp | 8 +++++++- src/optimizer.h | 2 ++ src/response.cpp | 7 ++++++- src/response.h | 3 ++- 4 files changed, 17 insertions(+), 3 deletions(-) diff --git a/src/optimizer.cpp b/src/optimizer.cpp index bbf46383..14d7707b 100644 --- a/src/optimizer.cpp +++ b/src/optimizer.cpp @@ -148,6 +148,12 @@ std::shared_ptr OptimizerCoordinateDescent::findBestBasel } } +arma::mat OptimizerCoordinateDescent::calculateUpdate (const double& learning_rate, const double& step_size, + const arma::mat& blearner_pred) const +{ + return learning_rate * step_size * blearner_pred; +} + void OptimizerCoordinateDescent::optimize (const unsigned int& actual_iteration, const double& learning_rate, const std::shared_ptr sh_ptr_loss, const std::shared_ptr sh_ptr_response, blearnertrack::BaselearnerTrack& blearner_track, const blearnerlist::BaselearnerFactoryList& blearner_list) { @@ -164,7 +170,7 @@ void OptimizerCoordinateDescent::optimize (const unsigned int& actual_iteration, // the contribution to the old parameter is the estimated parameter times the learning rate times // the step size. Therefore we have to pass the step size which changes in each iteration: blearner_track.insertBaselearner(sh_ptr_blearner_selected, getStepSize(actual_iteration)); - sh_ptr_response->updatePrediction(learning_rate, getStepSize(actual_iteration), blearner_pred_temp); + sh_ptr_response->updatePrediction(calculateUpdate(learning_rate, getStepSize(actual_iteration), blearner_pred_temp)); } void OptimizerCoordinateDescent::calculateStepSize (std::shared_ptr sh_ptr_loss, std::shared_ptr sh_ptr_response, diff --git a/src/optimizer.h b/src/optimizer.h index 487012e5..6df04028 100644 --- a/src/optimizer.h +++ b/src/optimizer.h @@ -51,6 +51,7 @@ class Optimizer virtual std::shared_ptr findBestBaselearner (const std::string&, std::shared_ptr, const blearner_factory_map&) const = 0; + virtual arma::mat calculateUpdate (const double&, const double&, const arma::mat&) const = 0; virtual void optimize (const unsigned int&, const double&, const std::shared_ptr, const std::shared_ptr, blearnertrack::BaselearnerTrack&, const blearnerlist::BaselearnerFactoryList&) = 0; @@ -85,6 +86,7 @@ class OptimizerCoordinateDescent : public Optimizer std::shared_ptr findBestBaselearner (const std::string&, std::shared_ptr, const blearner_factory_map&) const; + arma::mat calculateUpdate (const double&, const double&, const arma::mat&) const; void optimize (const unsigned int&, const double&, const std::shared_ptr, const std::shared_ptr, blearnertrack::BaselearnerTrack&, const blearnerlist::BaselearnerFactoryList&); diff --git a/src/response.cpp b/src/response.cpp index 3ee18629..cd84de63 100644 --- a/src/response.cpp +++ b/src/response.cpp @@ -65,6 +65,11 @@ void Response::updatePseudoResiduals (std::shared_ptr sh_ptr_loss) } } +void Response::updatePrediction (const arma::mat& update) +{ + prediction_scores += update; +} + void Response::updatePrediction (const double& learning_rate, const double& step_size, const arma::mat& update) { prediction_scores += learning_rate * step_size * update; @@ -370,4 +375,4 @@ void ResponseFDA::filter (const arma::uvec& idx) prediction_scores = prediction_scores.elem(idx); } -} // namespace response \ No newline at end of file +} // namespace response diff --git a/src/response.h b/src/response.h index 4eda46df..04f53990 100644 --- a/src/response.h +++ b/src/response.h @@ -68,6 +68,7 @@ class Response void checkLossCompatibility (std::shared_ptr) const; void updatePseudoResiduals (std::shared_ptr); + void updatePrediction (const arma::mat&); void updatePrediction (const double&, const double&, const arma::mat&); void constantInitialization (std::shared_ptr); @@ -148,4 +149,4 @@ class ResponseFDA : public Response } // namespace response -#endif // RESPONSE_H_ \ No newline at end of file +#endif // RESPONSE_H_ From 4861b4249d072c7285b79931f016c6bc745c3765 Mon Sep 17 00:00:00 2001 From: schalkdaniel Date: Mon, 18 Nov 2019 12:44:32 +0100 Subject: [PATCH 3/3] set logger to be able to handle different optimizer --- src/compboost.cpp | 3 ++- src/logger.cpp | 14 +++++++++----- src/logger.h | 16 +++++++++++----- src/loggerlist.cpp | 11 ++++++----- src/loggerlist.h | 4 +++- 5 files changed, 31 insertions(+), 17 deletions(-) diff --git a/src/compboost.cpp b/src/compboost.cpp index 4f279094..b9bde52d 100644 --- a/src/compboost.cpp +++ b/src/compboost.cpp @@ -73,7 +73,8 @@ void Compboost::train (const unsigned int& trace, std::shared_ptroptimize(actual_iteration, learning_rate, sh_ptr_loss, sh_ptr_response, blearner_track, used_baselearner_list); - logger_list->logCurrent(actual_iteration, sh_ptr_response, blearner_track.getBaselearnerVector().back(), learning_rate, sh_ptr_optimizer->getStepSize(actual_iteration)); + logger_list->logCurrent(actual_iteration, sh_ptr_response, blearner_track.getBaselearnerVector().back(), + learning_rate, sh_ptr_optimizer->getStepSize(actual_iteration), sh_ptr_optimizer); // Calculate and log risk: risk.push_back(sh_ptr_response->calculateEmpiricalRisk(sh_ptr_loss)); diff --git a/src/logger.cpp b/src/logger.cpp index 3a7b7961..b070af2b 100644 --- a/src/logger.cpp +++ b/src/logger.cpp @@ -79,7 +79,8 @@ LoggerIteration::LoggerIteration (const std::string& logger_id0, const bool& is_ */ void LoggerIteration::logStep (const unsigned int& current_iteration, std::shared_ptr sh_ptr_response, - std::shared_ptr sh_ptr_blearner, const double& learning_rate, const double& step_size) + std::shared_ptr sh_ptr_blearner, const double& learning_rate, const double& step_size, + std::shared_ptr sh_ptr_optimizer) { iterations.push_back(current_iteration); } @@ -226,7 +227,8 @@ LoggerInbagRisk::LoggerInbagRisk (const std::string& logger_id0, const bool& is_ */ void LoggerInbagRisk::logStep (const unsigned int& current_iteration, std::shared_ptr sh_ptr_response, - std::shared_ptr sh_ptr_blearner, const double& learning_rate, const double& step_size) + std::shared_ptr sh_ptr_blearner, const double& learning_rate, const double& step_size, + std::shared_ptr sh_ptr_optimizer) { // Calculate empirical risk. Calculateion of the temporary vector ensures // // that stuff like auc logging is possible: @@ -394,7 +396,8 @@ LoggerOobRisk::LoggerOobRisk (const std::string& logger_id0, const bool& is_a_st */ void LoggerOobRisk::logStep (const unsigned int& current_iteration, std::shared_ptr sh_ptr_response, - std::shared_ptr sh_ptr_blearner, const double& learning_rate, const double& step_size) + std::shared_ptr sh_ptr_blearner, const double& learning_rate, const double& step_size, + std::shared_ptr sh_ptr_optimizer) { if (current_iteration == 1) { sh_ptr_oob_response->constantInitialization(sh_ptr_response->getInitialization()); @@ -413,7 +416,7 @@ void LoggerOobRisk::logStep (const unsigned int& current_iteration, std::shared_ // Predict this data using the selected baselearner: arma::mat temp_oob_prediction = sh_ptr_blearner->predict(oob_blearner_data); - sh_ptr_oob_response->updatePrediction(learning_rate, step_size, temp_oob_prediction); + sh_ptr_oob_response->updatePrediction(sh_ptr_optimizer->calculateUpdate(learning_rate, step_size, temp_oob_prediction)); } @@ -591,7 +594,8 @@ LoggerTime::LoggerTime (const std::string& logger_id0, const bool& is_a_stopper0 */ void LoggerTime::logStep (const unsigned int& current_iteration, std::shared_ptr sh_ptr_response, - std::shared_ptr sh_ptr_blearner, const double& learning_rate, const double& step_size) + std::shared_ptr sh_ptr_blearner, const double& learning_rate, const double& step_size, + std::shared_ptr sh_ptr_optimizer) { if (current_time.size() == 0) { init_time = std::chrono::steady_clock::now(); diff --git a/src/logger.h b/src/logger.h index cda3643e..396aa512 100644 --- a/src/logger.h +++ b/src/logger.h @@ -50,6 +50,7 @@ #include "loss.h" #include "baselearner.h" #include "response.h" +#include "optimizer.h" namespace logger { @@ -82,7 +83,8 @@ class Logger /// Log current step of compboost iteration dependent on the child class virtual void logStep (const unsigned int&, std::shared_ptr, - std::shared_ptr, const double&, const double&) = 0; + std::shared_ptr, const double&, const double&, + std::shared_ptr) = 0; /// Class dependent check if the stopping criteria is fulfilled virtual bool reachedStopCriteria () = 0; @@ -145,7 +147,8 @@ class LoggerIteration : public Logger /// Log current step of compboost iteration of class `LoggerIteration` void logStep (const unsigned int&, std::shared_ptr, - std::shared_ptr, const double&, const double&); + std::shared_ptr, const double&, const double&, + std::shared_ptr); /// Stop criteria is fulfilled if the current iteration exceed `max_iteration` bool reachedStopCriteria (); @@ -201,7 +204,8 @@ class LoggerInbagRisk : public Logger /// Log current step of compboost iteration for class `LoggerInbagRisk` void logStep (const unsigned int&, std::shared_ptr, - std::shared_ptr, const double&, const double&); + std::shared_ptr, const double&, const double&, + std::shared_ptr); /// Stop criteria is fulfilled if the relative improvement falls below `eps_for_break` bool reachedStopCriteria (); @@ -270,7 +274,8 @@ class LoggerOobRisk : public Logger /// Log current step of compboost iteration for class `LoggerOobRisk` void logStep (const unsigned int&, std::shared_ptr, - std::shared_ptr, const double&, const double&); + std::shared_ptr, const double&, const double&, + std::shared_ptr); /// Stop criteria is fulfilled if the relative improvement falls below `eps_for_break` bool reachedStopCriteria (); @@ -330,7 +335,8 @@ class LoggerTime : public Logger /// Log current step of compboost iteration for class `LoggerTime` void logStep (const unsigned int&, std::shared_ptr, - std::shared_ptr, const double&, const double&); + std::shared_ptr, const double&, const double&, + std::shared_ptr); /// Stop criteria is fulfilled if the passed time exceeds `max_time` bool reachedStopCriteria (); diff --git a/src/loggerlist.cpp b/src/loggerlist.cpp index 356da25e..4ee063bf 100644 --- a/src/loggerlist.cpp +++ b/src/loggerlist.cpp @@ -96,7 +96,8 @@ std::pair, arma::mat> LoggerList::getLoggerData () cons } void LoggerList::logCurrent (const unsigned int& current_iteration, std::shared_ptr sh_ptr_response, - std::shared_ptr sh_ptr_blearner, const double& learning_rate, const double& step_size) + std::shared_ptr sh_ptr_blearner, const double& learning_rate, const double& step_size, + std::shared_ptr sh_ptr_optimizer) { // Think about how to implement this the best way. I think the computations // e.g. for the risk should be done within the logger object. If so, the @@ -111,7 +112,7 @@ void LoggerList::logCurrent (const unsigned int& current_iteration, std::shared_ // data specified by initializing the logger list. for (logger_map::iterator it = log_list.begin(); it != log_list.end(); ++it) { it->second->logStep(current_iteration, sh_ptr_response, sh_ptr_blearner, - learning_rate, step_size); + learning_rate, step_size, sh_ptr_optimizer); } } // Print logger: @@ -133,19 +134,19 @@ void LoggerList::printLoggerStatus (const double& current_risk) const void LoggerList::prepareForRetraining (const unsigned int& new_max_iters) { - bool iters_in_list = false; + bool has_iteration_logger = false; for (auto& it : log_list) { it.second->is_a_stopper = false; if (it.second->getLoggerType() == "iteration") { std::static_pointer_cast(it.second)->updateMaxIterations(new_max_iters); it.second->is_a_stopper = true; - iters_in_list = true; + has_iteration_logger = true; } if (it.second->getLoggerType() == "time") { std::static_pointer_cast(it.second)->reInitializeTime(); } } - if (! iters_in_list) { + if (! has_iteration_logger) { std::shared_ptr new_logger = std::make_shared("iters_re", true, new_max_iters); log_list.insert(std::pair>("_iteration", new_logger)); } diff --git a/src/loggerlist.h b/src/loggerlist.h index 7deed0aa..08794afe 100644 --- a/src/loggerlist.h +++ b/src/loggerlist.h @@ -27,6 +27,7 @@ #include "logger.h" #include "response.h" +#include "optimizer.h" typedef std::map> logger_map; @@ -70,7 +71,8 @@ class LoggerList // Log the current step (structure ). // This is given to the instantiated logger: void logCurrent (const unsigned int&, std::shared_ptr, - std::shared_ptr, const double&, const double&); + std::shared_ptr, const double&, const double&, + std::shared_ptr); // Print the logger status: void printLoggerStatus (const double&) const;