Fixed auto-tuning process to correctly bubble up parameters (#744)

horovod · Jan 10, 2019 · e5a4de0 · e5a4de0
1 parent 65bb358
commit e5a4de0
Show file tree

Hide file tree

Showing 2 changed files with 10 additions and 4 deletions.
diff --git a/horovod/common/parameter_manager.cc b/horovod/common/parameter_manager.cc
@@ -182,8 +182,12 @@ void ParameterManager::Tune(double score) {
     // Only do the tuning on the coordinator to ensure consistency.
     if (rank_ == root_rank_) {
       bool finished_tuning = true;
+      double best_score = score;
       for (auto* param : parameter_chain_) {
-        bool finished = param->Tune(score);
+        double new_best_score;
+        bool finished = param->Tune(best_score, &new_best_score);
+        best_score = new_best_score;
+
         if (!finished) {
           finished_tuning = false;
           break;
@@ -294,8 +298,10 @@ ParameterManager::TunableParameter<T>::TunableParameter(T initial_value) :
     tunable_(true) {}
 
 template <class T>
-bool ParameterManager::TunableParameter<T>::Tune(double score) {
+bool ParameterManager::TunableParameter<T>::Tune(double score, double* best_score) {
   UpdateBestValue(score);
+  *best_score = best_score_;
+
   if (!tunable_) {
     return true;
   }

diff --git a/horovod/common/parameter_manager.h b/horovod/common/parameter_manager.h
@@ -105,7 +105,7 @@ class ParameterManager {
   // Interface used to represent a parameter (or group of parameters) being tuned.
   class ITunableParameter {
   public:
-    virtual bool Tune(double score) = 0;
+    virtual bool Tune(double score, double* best_score) = 0;
     virtual void UpdateBestValue(double score) = 0;
     virtual double BestScore() const = 0;
     virtual bool IsTunable() const = 0;
@@ -116,7 +116,7 @@ class ParameterManager {
   class TunableParameter : public ITunableParameter {
   public:
     TunableParameter(T initial_value);
-    bool Tune(double score) override;
+    bool Tune(double score, double* best_score) override;
     void UpdateBestValue(double score) override;
 
     void SetValue(T value, bool fixed);