From 91e0db2cbdcf6d24d99b51cb849aa94ea2b561db Mon Sep 17 00:00:00 2001 From: Chenzhe Diao Date: Tue, 6 Jun 2017 13:04:20 -0600 Subject: [PATCH 01/16] Add the FrankWolfe optimization, the OMP is working now. --- src/mlpack/core/optimizers/CMakeLists.txt | 1 + src/mlpack/core/optimizers/fw/CMakeLists.txt | 15 ++ .../core/optimizers/fw/constr_lpball.hpp | 106 +++++++++++ src/mlpack/core/optimizers/fw/frank_wolfe.hpp | 175 ++++++++++++++++++ .../core/optimizers/fw/frank_wolfe_impl.hpp | 100 ++++++++++ .../core/optimizers/fw/test_func_sq.hpp | 56 ++++++ .../core/optimizers/fw/update_classic.hpp | 89 +++++++++ src/mlpack/core/optimizers/fw/update_span.hpp | 145 +++++++++++++++ src/mlpack/tests/CMakeLists.txt | 1 + src/mlpack/tests/frankwolfe_test.cpp | 52 ++++++ 10 files changed, 740 insertions(+) create mode 100644 src/mlpack/core/optimizers/fw/CMakeLists.txt create mode 100644 src/mlpack/core/optimizers/fw/constr_lpball.hpp create mode 100644 src/mlpack/core/optimizers/fw/frank_wolfe.hpp create mode 100644 src/mlpack/core/optimizers/fw/frank_wolfe_impl.hpp create mode 100644 src/mlpack/core/optimizers/fw/test_func_sq.hpp create mode 100644 src/mlpack/core/optimizers/fw/update_classic.hpp create mode 100644 src/mlpack/core/optimizers/fw/update_span.hpp create mode 100644 src/mlpack/tests/frankwolfe_test.cpp diff --git a/src/mlpack/core/optimizers/CMakeLists.txt b/src/mlpack/core/optimizers/CMakeLists.txt index 072fd27b32e..8f17d87a165 100644 --- a/src/mlpack/core/optimizers/CMakeLists.txt +++ b/src/mlpack/core/optimizers/CMakeLists.txt @@ -3,6 +3,7 @@ set(DIRS ada_grad adam aug_lagrangian + fw gradient_descent lbfgs minibatch_sgd diff --git a/src/mlpack/core/optimizers/fw/CMakeLists.txt b/src/mlpack/core/optimizers/fw/CMakeLists.txt new file mode 100644 index 00000000000..b979a2aacf2 --- /dev/null +++ b/src/mlpack/core/optimizers/fw/CMakeLists.txt @@ -0,0 +1,15 @@ +set(SOURCES + frank_wolfe.hpp + frank_wolfe_impl.hpp + constr_lpball.hpp + update_classic.hpp + update_span.hpp + test_func_sq.hpp +) + +set(DIR_SRCS) +foreach(file ${SOURCES}) + set(DIR_SRCS ${DIR_SRCS} ${CMAKE_CURRENT_SOURCE_DIR}/${file}) +endforeach() + +set(MLPACK_SRCS ${MLPACK_SRCS} ${DIR_SRCS} PARENT_SCOPE) diff --git a/src/mlpack/core/optimizers/fw/constr_lpball.hpp b/src/mlpack/core/optimizers/fw/constr_lpball.hpp new file mode 100644 index 00000000000..4028531681c --- /dev/null +++ b/src/mlpack/core/optimizers/fw/constr_lpball.hpp @@ -0,0 +1,106 @@ +/** + * @file constr_lpball.hpp + * @author Chenzhe Diao + * + * Lp ball constrained for FrankWolfe algorithm. Used as LinearConstrSolverType. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ +#ifndef MLPACK_CORE_OPTIMIZERS_FW_CONSTR_LPBALL_HPP +#define MLPACK_CORE_OPTIMIZERS_FW_CONSTR_LPBALL_HPP + +#include + +namespace mlpack { +namespace optimization { + +/** + * LinearConstrSolver for FrankWolfe algorithm. Constraint domain given in the + * form of lp ball. That is, given \f$ v \f$, solve + * \f[ + * s:=arg\min_{s\in D} + * \f] + * when \f$ D \f$ is an lp ball. + * + * For \f$ p=1 \f$: take (one) \f$ k = arg\max_j |v_j|\f$, then the solution is: + * \f[ + * s_k = -sign(v_k), \qquad s_j = 0, j\neq k. + * \f] + * + * For \f$ 11.0) + { + // lp ball with 1 +#include "update_classic.hpp" +#include "update_span.hpp" +#include "constr_lpball.hpp" +#include "test_func_sq.hpp" + +namespace mlpack { +namespace optimization { + +/** + * Frank-Wolfe is a technique to minimize a continuously differentiable convex + * function \f$ f \f$ over a compact convex subset \f$ D \f$ of a vector space. + * It is also known as conditional gradient method. + * + * To find minimum of a function using Frank-Wolfe in each iteration \f$ k \f$: + * 1. One optimize the linearized constrained problem, using LinearConstrSolver: + * \f[ + * s_k:= arg\min_{s\in D} + * \f] + * + * 2. Update \f$ x \f$ using UpdateRule: + * \f[ + * x_{k+1} := (1-\gamma) x_k + \gamma s_k + * \f] + * for some \f$ \gamma \in (0, 1) \f$, or use Fully-Corrective Variant: + * \f[ + * x_{k+1}:= arg\min_{x\in conv(s_0, \cdots, s_k)} f(x) + * \f] + * + * + * The algorithm continues until \f$ k \f$ reaches the maximum number of iterations, + * or when the duality gap is bounded by a certain tolerance \f$ \epsilon \f$. + * That is, + * + * \f[ + * g(x):= \max_{s\in D} \quad \leq \epsilon, + * \f] + * + * we also know that \f$ g(x) \geq f(x) - f(x^*) \f$, where \f$ x^* \f$ is the optimal + * solution. + * + * The parameter \f$ \epsilon \f$ is specified by the tolerance parameter to the + * constructor. + * + * For FrankWolfe to work, FunctionType, LinearConstrSolverType and UpdateRuleType + * template parameters are required. + * These classes must implement the following functions: + * + * FunctionType: + * + * double Evaluate(const arma::mat& coordinates); + * void Gradient(const arma::mat& coordinates, + * arma::mat& gradient); + * + * LinearConstrSolverType: + * + * void Optimize(const arma::mat& gradient, + * arma::mat& s); + * + * UpdateRuleType: + * + * void Update(const arma::mat& old_coords, + * const arma::mat& s, + * arma::mat& new_coords, + * const size_t num_iter); + * + * @tparam FunctionType Objective function type to be + * minimized. + * @tparam LinearConstrSolverType Solver for the linear constrained problem. + * @tparam UpdateRuleType Rule to update the solution in each iteration. + * + */ +template< + typename FunctionType, + typename LinearConstrSolverType, + typename UpdateRuleType +> +class FrankWolfe +{ + public: + /** + * Construct the Frank-Wolfe optimizer with the given function and + * parameters. Notice that the constraint domain \f$ D \f$ is input + * at the initialization of linear_constr_solver, the function to be + * optimized is stored in update_rule. + * + * @param function Function to be optimized + * @param linear_constr_solver Solver for linear constrained problem. + * @param update_rule Rule for updating solution in each iteration. + * @param maxIterations Maximum number of iterations allowed (0 means no + * limit). + * @param tolerance Maximum absolute tolerance to terminate algorithm. + */ + FrankWolfe( FunctionType& function, + const LinearConstrSolverType linear_constr_solver, + const UpdateRuleType update_rule, + const size_t maxIterations = 100000, + const double tolerance = 1e-5); + + /** + * Optimize the given function using FrankWolfe. The given starting + * point will be modified to store the finishing point of the algorithm, and + * the final objective value is returned. + * + * @param iterate Starting point (will be modified). + * @return Objective value of the final point. + */ + double Optimize(arma::mat& iterate); + + //! Get the instantiated function to be optimized. + const FunctionType& Function() const { return function; } // Chenzhe: why return const? + //! Modify the instantiated function. + FunctionType& Function() { return function; } + + //! Get the linear constrained solver. + LinearConstrSolverType LinearConstrSolver() const { return linear_constr_solver; } + //! Modify the linear constrained solver. + LinearConstrSolverType& LinearConstrSolver() { return linear_constr_solver; } + + //! Get the update rule. + UpdateRuleType UpdateRule() const { return update_rule; } + //! Modify the update rule. + UpdateRuleType& UpdateRule() { return update_rule; } + + //! Get the maximum number of iterations (0 indicates no limit). + size_t MaxIterations() const { return maxIterations; } + //! Modify the maximum number of iterations (0 indicates no limit). + size_t& MaxIterations() { return maxIterations; } + + //! Get the tolerance for termination. + double Tolerance() const { return tolerance; } + //! Modify the tolerance for termination. + double& Tolerance() { return tolerance; } + + private: + //! The instantiated function. + FunctionType& function; + + //! The solver for constrained linear problem in first step. + LinearConstrSolverType linear_constr_solver; + + //! The rule to update, used in the second step. + UpdateRuleType update_rule; + + //! The maximum number of allowed iterations. + size_t maxIterations; + + //! The tolerance for termination. + double tolerance; +}; + +using OMP = FrankWolfe>; + +} // namespace optimization +} // namespace mlpack + +// Include implementation. +#include "frank_wolfe_impl.hpp" + +#endif diff --git a/src/mlpack/core/optimizers/fw/frank_wolfe_impl.hpp b/src/mlpack/core/optimizers/fw/frank_wolfe_impl.hpp new file mode 100644 index 00000000000..b477c0b9a0d --- /dev/null +++ b/src/mlpack/core/optimizers/fw/frank_wolfe_impl.hpp @@ -0,0 +1,100 @@ +/** + * @file frank_wolfe_impl.hpp + * @author Chenzhe Diao + * + * Frank-Wolfe Algorithm. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ +#ifndef MLPACK_CORE_OPTIMIZERS_FW_FRANK_WOLFE_IMPL_HPP +#define MLPACK_CORE_OPTIMIZERS_FW_FRANK_WOLFE_IMPL_HPP + +// In case it hasn't been included yet. +#include "frank_wolfe.hpp" + +namespace mlpack { +namespace optimization { + +template< + typename FunctionType, + typename LinearConstrSolverType, + typename UpdateRuleType +> +FrankWolfe::FrankWolfe( + FunctionType& function, + const LinearConstrSolverType linear_constr_solver, + const UpdateRuleType update_rule, + const size_t maxIterations, + const double tolerance) : + function(function), + linear_constr_solver(linear_constr_solver), + update_rule(update_rule), + maxIterations(maxIterations), + tolerance(tolerance) +{ /* Nothing to do*/ } + + +//! Optimize the function (minimize). +template< + typename FunctionType, + typename LinearConstrSolverType, + typename UpdateRuleType +> +double FrankWolfe::Optimize( + arma::mat& iterate) +{ + // To keep track of the function value + double CurrentObjective = function.Evaluate(iterate); + double PreviousObjective = DBL_MAX; + + arma::mat gradient(iterate.n_rows, iterate.n_cols); + arma::mat s(iterate.n_rows, iterate.n_cols); + arma::mat iterate_new(iterate.n_rows, iterate.n_cols); + double gap = 0; + + for(size_t i=1; i != maxIterations; ++i) + { + //Output current objective function + Log::Info << "Iteration " << i << ", objective " + << CurrentObjective << "." << std::endl; + + // Reset counter variables. + PreviousObjective = CurrentObjective; + + // Calculate the gradient + function.Gradient(iterate, gradient); + + // Solve linear constrained problem, solution saved in s. + linear_constr_solver.Optimize(gradient, s); + + // Check duality gap for return condition + gap = dot(iterate-s, gradient); + if (gap < tolerance) + { + Log::Info << "FrankWolfe: minimized within tolerance " + << tolerance << "; " << "terminating optimization." << std::endl; + return CurrentObjective; + } + + + // Update solution, save in iterate_new + update_rule.Update(iterate, s, iterate_new, i); + + + iterate = iterate_new; + CurrentObjective = function.Evaluate(iterate); + } + Log::Info << "Frank Wolfe: maximum iterations (" << maxIterations + << ") reached; " << "terminating optimization." << std::endl; + return CurrentObjective; + +} + + +} // namespace optimization +} // namespace mlpack + +#endif diff --git a/src/mlpack/core/optimizers/fw/test_func_sq.hpp b/src/mlpack/core/optimizers/fw/test_func_sq.hpp new file mode 100644 index 00000000000..a269a368713 --- /dev/null +++ b/src/mlpack/core/optimizers/fw/test_func_sq.hpp @@ -0,0 +1,56 @@ +/** + * @file test_func_sq.hpp + * @author Chenzhe Diao + * + * Update method for FrankWolfe algorithm, recalculate the optimal in the span + * of previous solution space. Used as UpdateRuleType. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ +#ifndef MLPACK_CORE_OPTIMIZERS_FW_TEST_FUNC_SQ_HPP +#define MLPACK_CORE_OPTIMIZERS_FW_TEST_FUNC_SQ_HPP + +#include + +namespace mlpack { +namespace optimization { + +class TestFuncSq +{ + public: + TestFuncSq(const arma::mat A, const arma::mat b ) : A(A), b(b) + {/* Nothing to do. */} + + double Evaluate(const arma::mat& coords) + { + arma::vec r = A*coords - b; + arma::mat y = (r.t() * r) * 0.5; + return y(0,0); + } + + void Gradient(const arma::mat& coords, arma::mat& gradient) + { + arma::vec r = A*coords - b; + gradient = A.t() * r; + } + + + arma::mat MatrixA() const {return A;} + arma::mat& MatrixA() {return A;} + + arma::vec Vectorb() const { return b; } + arma::vec& Vectorb() { return b; } + + private: + arma::mat A; // matrix + arma::vec b; // vector + +}; + +} // namespace optimization +} // namespace mlpack + +#endif diff --git a/src/mlpack/core/optimizers/fw/update_classic.hpp b/src/mlpack/core/optimizers/fw/update_classic.hpp new file mode 100644 index 00000000000..539ae762551 --- /dev/null +++ b/src/mlpack/core/optimizers/fw/update_classic.hpp @@ -0,0 +1,89 @@ +/** + * @file update_classic.hpp + * @author Chenzhe Diao + * + * Classic update method for FrankWolfe algorithm. Used as UpdateRuleType. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ +#ifndef MLPACK_CORE_OPTIMIZERS_FW_UPDATE_CLASSIC_HPP +#define MLPACK_CORE_OPTIMIZERS_FW_UPDATE_CLASSIC_HPP + +#include + +namespace mlpack { +namespace optimization { + +/** + * Use classic rule in the update step for FrankWolfe algorithm. That is, + * take \f$ \gamma = \frac{2}{k+2} \f$, where \f$ k \f$ is the iteration + * number. The update rule would be: + * \f[ + * x_{k+1} = (1-\gamma) x_k + \gamma s + * \f] + * + * For UpdateClassic to work, FunctionType template parameters are required. + * This class must implement the following functions: + * + * FunctionType: + * + * double Evaluate(const arma::mat& coordinates); + * void Gradient(const arma::mat& coordinates, + * arma::mat& gradient); + * + * + * Although the function is not used in classic update rule, we still put it + * here since we want the same template code interface. + * + * @tparam FunctionType Objective function type to be minimized in FrankWolfe algorithm. + */ +template +class UpdateClassic +{ + public: + /** + * Construct the classic update rule. The function to be optimized is + * input here, although not used. + * + * @param function Function to be optimized in FrankWolfe algorithm. + */ + UpdateClassic(FunctionType& function): function(function) + { /* Do nothing. */ } + + /** + * Classic update rule for FrankWolfe. + * + * \f$ x_{k+1} = (1-\gamma)x_k + \gamma s \f$, where \f$ \gamma = 2/(k+2) \f$ + * + * @param old_coords previous solution coords. + * @param s current linear_constr_solution result. + * @param new_coords new output solution coords. + * @param num_iter current iteration number + */ + void Update(const arma::mat& old_coords, + const arma::mat& s, + arma::mat& new_coords, + const size_t num_iter) + { + double gamma = 2.0/(num_iter + 2.0); + new_coords = (1.0-gamma)*old_coords + gamma*s; + } + + //! Get the instantiated function to be optimized. + const FunctionType& Function() const { return function; } + //! Modify the instantiated function. + FunctionType& Function() { return function; } + + private: + //! The instantiated function. + FunctionType& function; + +}; + +} // namespace optimization +} // namespace mlpack + +#endif diff --git a/src/mlpack/core/optimizers/fw/update_span.hpp b/src/mlpack/core/optimizers/fw/update_span.hpp new file mode 100644 index 00000000000..9470ffb01a2 --- /dev/null +++ b/src/mlpack/core/optimizers/fw/update_span.hpp @@ -0,0 +1,145 @@ +/** + * @file update_span.hpp + * @author Chenzhe Diao + * + * Update method for FrankWolfe algorithm, recalculate the optimal in the span + * of previous solution space. Used as UpdateRuleType. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ +#ifndef MLPACK_CORE_OPTIMIZERS_FW_UPDATE_SPAN_HPP +#define MLPACK_CORE_OPTIMIZERS_FW_UPDATE_SPAN_HPP + +#include + +namespace mlpack { +namespace optimization { + +/** + * Recalculate the optimal solution in the span of all previous solution space, + * used as update step for FrankWolfe algorithm. + * + * + * For UpdateSpan to work, FunctionType template parameters are required. + * This class must implement the following functions: + * + * FunctionType: + * + * double Evaluate(const arma::mat& coordinates); + * void Gradient(const arma::mat& coordinates, + * arma::mat& gradient); + * + * + * @tparam FunctionType Objective function type to be minimized in FrankWolfe algorithm. + */ +template +class UpdateSpan +{ + public: + /** + * Construct the span update rule. The function to be optimized is + * input here. + * + * @param function Function to be optimized in FrankWolfe algorithm. + */ + UpdateSpan(FunctionType& function): function(function) + { /* Do nothing. */ } + // Chenzhe: should we use reference to function? what's the difference? + + /** + * Update rule for FrankWolfe, reoptimize in the span of original solution space. + * + * + * @param old_coords previous solution coords. + * @param s current linear_constr_solution result. + * @param new_coords new output solution coords. + * @param num_iter current iteration number + */ + void Update(const arma::mat& old_coords, + const arma::mat& s, + arma::mat& new_coords, + const size_t num_iter) + { + //Need to add atom here + arma::uvec ind = find(s, 1); + arma::uword d = ind(0); + AddAtom(d); + + arma::vec b = function.Vectorb(); + arma::mat x = solve(atoms_current, b); + + new_coords = RecoverVector(x); + } + + + + //! Get the instantiated function to be optimized. + FunctionType Function() const { return function; } + //! Modify the instantiated function. + FunctionType& Function() { return function; } + + //! Get the current atom indices. + arma::uvec CurrentIndices() const { return current_indices; } + //! Modify the current atom indices. + arma::uvec& CurrentIndices() { return current_indices; } + + //! Get the current atoms. + arma::mat CurrentAtoms() const { return atoms_current; } + //! Modify the current atoms. + arma::mat& CurrentAtoms() { return atoms_current; } + + //! Add atom into the solution space, modify the current_indices and atoms_current. + void AddAtom(const arma::uword k) + { + if (isEmpty){ + CurrentIndices() = k; + CurrentAtoms() = (function.MatrixA()).col(k); + isEmpty = false; + } + else{ + arma::uvec vk(1); + vk = k; + current_indices.insert_rows(0, vk); + + arma::mat atom = (function.MatrixA()).col(k); + atoms_current.insert_cols(0, atom); + + } + } + + arma::vec RecoverVector(const arma::vec& x ) + { + int n = (function.MatrixA()).n_cols; + arma::vec y = arma::zeros(n); + + arma::uword len = current_indices.size(); + for (size_t ii = 0; ii < len; ++ii) + { + y(current_indices(ii)) = x(ii); + } + + return y; + } + + private: + //! The instantiated function. + FunctionType& function; + + //! Current indices. + arma::uvec current_indices; + + //! Current atoms. + arma::mat atoms_current; + + //! Flag current indices is empty + bool isEmpty=true; + +}; + +} // namespace optimization +} // namespace mlpack + +#endif diff --git a/src/mlpack/tests/CMakeLists.txt b/src/mlpack/tests/CMakeLists.txt index 694c23d877f..2fd98a68c30 100644 --- a/src/mlpack/tests/CMakeLists.txt +++ b/src/mlpack/tests/CMakeLists.txt @@ -27,6 +27,7 @@ add_executable(mlpack_test emst_test.cpp fastmks_test.cpp feedforward_network_test.cpp + frankwolfe_test.cpp gmm_test.cpp gradient_descent_test.cpp hmm_test.cpp diff --git a/src/mlpack/tests/frankwolfe_test.cpp b/src/mlpack/tests/frankwolfe_test.cpp new file mode 100644 index 00000000000..e24cf05ff59 --- /dev/null +++ b/src/mlpack/tests/frankwolfe_test.cpp @@ -0,0 +1,52 @@ +/** + * @file frankwolfe_test.cpp + * @author Chenzhe Diao + * + * Test file for Frank-Wolfe optimizer. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ +#include +#include +#include +#include +#include + +#include +#include "test_tools.hpp" + +using namespace std; +using namespace arma; +using namespace mlpack; +using namespace mlpack::optimization; +//using namespace mlpack::optimization::test; + +BOOST_AUTO_TEST_SUITE(FrankWolfeTest); + +BOOST_AUTO_TEST_CASE(OMPTest) +{ + mat A("1, 0, 0, 0.2, -0.15; 0, 1, 0, -0.03, -0.3; 0, 0, 1, 0.1, 0.1"); + vec b("1; 1; 0"); + + TestFuncSq f(A, b); + ConstrLpBallSolver linear_constr_solver(1); + UpdateSpan update_rule(f); + + OMP s(f, linear_constr_solver, update_rule); + + arma::vec coordinates("0; 0; 0; 0; 0"); + double result = s.Optimize(coordinates); + + BOOST_REQUIRE_SMALL(result, 1e-10); + BOOST_REQUIRE_SMALL(coordinates[0]-1, 1e-10); + BOOST_REQUIRE_SMALL(coordinates[1]-1, 1e-10); + BOOST_REQUIRE_SMALL(coordinates[2], 1e-10); + BOOST_REQUIRE_SMALL(coordinates[3], 1e-10); + BOOST_REQUIRE_SMALL(coordinates[4], 1e-10); +} + + +BOOST_AUTO_TEST_SUITE_END(); From 6cb9f3abd05e1a42eb37de81d0886a0cd145894b Mon Sep 17 00:00:00 2001 From: Chenzhe Diao Date: Mon, 26 Jun 2017 21:30:30 -0600 Subject: [PATCH 02/16] Classic FrankWolfe and OMP working. Both algorithms are tested with simple functions. --- src/mlpack/core/optimizers/fw/CMakeLists.txt | 3 +- src/mlpack/core/optimizers/fw/frank_wolfe.hpp | 9 +-- .../core/optimizers/fw/frank_wolfe_impl.hpp | 2 +- .../core/optimizers/fw/test_func_sq.hpp | 56 ------------------- src/mlpack/core/optimizers/fw/update_span.hpp | 1 - src/mlpack/tests/frankwolfe_test.cpp | 56 +++++++++++++------ 6 files changed, 48 insertions(+), 79 deletions(-) delete mode 100644 src/mlpack/core/optimizers/fw/test_func_sq.hpp diff --git a/src/mlpack/core/optimizers/fw/CMakeLists.txt b/src/mlpack/core/optimizers/fw/CMakeLists.txt index b979a2aacf2..eed067963b1 100644 --- a/src/mlpack/core/optimizers/fw/CMakeLists.txt +++ b/src/mlpack/core/optimizers/fw/CMakeLists.txt @@ -4,7 +4,8 @@ set(SOURCES constr_lpball.hpp update_classic.hpp update_span.hpp - test_func_sq.hpp + func_sq.hpp + test_func_fw.hpp ) set(DIR_SRCS) diff --git a/src/mlpack/core/optimizers/fw/frank_wolfe.hpp b/src/mlpack/core/optimizers/fw/frank_wolfe.hpp index 777358aad75..02ac400483c 100644 --- a/src/mlpack/core/optimizers/fw/frank_wolfe.hpp +++ b/src/mlpack/core/optimizers/fw/frank_wolfe.hpp @@ -17,7 +17,7 @@ #include "update_classic.hpp" #include "update_span.hpp" #include "constr_lpball.hpp" -#include "test_func_sq.hpp" +#include "func_sq.hpp" namespace mlpack { namespace optimization { @@ -110,7 +110,7 @@ class FrankWolfe const LinearConstrSolverType linear_constr_solver, const UpdateRuleType update_rule, const size_t maxIterations = 100000, - const double tolerance = 1e-5); + const double tolerance = 1e-10); /** * Optimize the given function using FrankWolfe. The given starting @@ -123,7 +123,7 @@ class FrankWolfe double Optimize(arma::mat& iterate); //! Get the instantiated function to be optimized. - const FunctionType& Function() const { return function; } // Chenzhe: why return const? + const FunctionType& Function() const { return function; } //! Modify the instantiated function. FunctionType& Function() { return function; } @@ -164,7 +164,8 @@ class FrankWolfe double tolerance; }; -using OMP = FrankWolfe>; +//! Orthogonal Matching Pursuit +using OMP = FrankWolfe>; } // namespace optimization } // namespace mlpack diff --git a/src/mlpack/core/optimizers/fw/frank_wolfe_impl.hpp b/src/mlpack/core/optimizers/fw/frank_wolfe_impl.hpp index b477c0b9a0d..848ecbe5639 100644 --- a/src/mlpack/core/optimizers/fw/frank_wolfe_impl.hpp +++ b/src/mlpack/core/optimizers/fw/frank_wolfe_impl.hpp @@ -71,7 +71,7 @@ double FrankWolfe::Optimiz linear_constr_solver.Optimize(gradient, s); // Check duality gap for return condition - gap = dot(iterate-s, gradient); + gap = std::fabs(dot(iterate-s, gradient)); if (gap < tolerance) { Log::Info << "FrankWolfe: minimized within tolerance " diff --git a/src/mlpack/core/optimizers/fw/test_func_sq.hpp b/src/mlpack/core/optimizers/fw/test_func_sq.hpp deleted file mode 100644 index a269a368713..00000000000 --- a/src/mlpack/core/optimizers/fw/test_func_sq.hpp +++ /dev/null @@ -1,56 +0,0 @@ -/** - * @file test_func_sq.hpp - * @author Chenzhe Diao - * - * Update method for FrankWolfe algorithm, recalculate the optimal in the span - * of previous solution space. Used as UpdateRuleType. - * - * mlpack is free software; you may redistribute it and/or modify it under the - * terms of the 3-clause BSD license. You should have received a copy of the - * 3-clause BSD license along with mlpack. If not, see - * http://www.opensource.org/licenses/BSD-3-Clause for more information. - */ -#ifndef MLPACK_CORE_OPTIMIZERS_FW_TEST_FUNC_SQ_HPP -#define MLPACK_CORE_OPTIMIZERS_FW_TEST_FUNC_SQ_HPP - -#include - -namespace mlpack { -namespace optimization { - -class TestFuncSq -{ - public: - TestFuncSq(const arma::mat A, const arma::mat b ) : A(A), b(b) - {/* Nothing to do. */} - - double Evaluate(const arma::mat& coords) - { - arma::vec r = A*coords - b; - arma::mat y = (r.t() * r) * 0.5; - return y(0,0); - } - - void Gradient(const arma::mat& coords, arma::mat& gradient) - { - arma::vec r = A*coords - b; - gradient = A.t() * r; - } - - - arma::mat MatrixA() const {return A;} - arma::mat& MatrixA() {return A;} - - arma::vec Vectorb() const { return b; } - arma::vec& Vectorb() { return b; } - - private: - arma::mat A; // matrix - arma::vec b; // vector - -}; - -} // namespace optimization -} // namespace mlpack - -#endif diff --git a/src/mlpack/core/optimizers/fw/update_span.hpp b/src/mlpack/core/optimizers/fw/update_span.hpp index 9470ffb01a2..6e00810904f 100644 --- a/src/mlpack/core/optimizers/fw/update_span.hpp +++ b/src/mlpack/core/optimizers/fw/update_span.hpp @@ -47,7 +47,6 @@ class UpdateSpan */ UpdateSpan(FunctionType& function): function(function) { /* Do nothing. */ } - // Chenzhe: should we use reference to function? what's the difference? /** * Update rule for FrankWolfe, reoptimize in the span of original solution space. diff --git a/src/mlpack/tests/frankwolfe_test.cpp b/src/mlpack/tests/frankwolfe_test.cpp index e24cf05ff59..c6ccf7a9f80 100644 --- a/src/mlpack/tests/frankwolfe_test.cpp +++ b/src/mlpack/tests/frankwolfe_test.cpp @@ -13,7 +13,9 @@ #include #include #include -#include +#include +#include +#include #include #include "test_tools.hpp" @@ -22,30 +24,52 @@ using namespace std; using namespace arma; using namespace mlpack; using namespace mlpack::optimization; -//using namespace mlpack::optimization::test; BOOST_AUTO_TEST_SUITE(FrankWolfeTest); BOOST_AUTO_TEST_CASE(OMPTest) { - mat A("1, 0, 0, 0.2, -0.15; 0, 1, 0, -0.03, -0.3; 0, 0, 1, 0.1, 0.1"); - vec b("1; 1; 0"); + int k = 5; + mat B1 = eye(3, 3); + mat B2 = 0.1*randn(3, k); + mat A = join_horiz(B1, B2); + vec b("1; 1; 0"); - TestFuncSq f(A, b); - ConstrLpBallSolver linear_constr_solver(1); - UpdateSpan update_rule(f); + FuncSq f(A, b); + ConstrLpBallSolver linear_constr_solver(1); + UpdateSpan update_rule(f); - OMP s(f, linear_constr_solver, update_rule); + OMP s(f, linear_constr_solver, update_rule); - arma::vec coordinates("0; 0; 0; 0; 0"); - double result = s.Optimize(coordinates); + vec coordinates = zeros(k+3); + double result = s.Optimize(coordinates); - BOOST_REQUIRE_SMALL(result, 1e-10); - BOOST_REQUIRE_SMALL(coordinates[0]-1, 1e-10); - BOOST_REQUIRE_SMALL(coordinates[1]-1, 1e-10); - BOOST_REQUIRE_SMALL(coordinates[2], 1e-10); - BOOST_REQUIRE_SMALL(coordinates[3], 1e-10); - BOOST_REQUIRE_SMALL(coordinates[4], 1e-10); + BOOST_REQUIRE_SMALL(result, 1e-10); + BOOST_REQUIRE_SMALL(coordinates[0]-1, 1e-10); + BOOST_REQUIRE_SMALL(coordinates[1]-1, 1e-10); + BOOST_REQUIRE_SMALL(coordinates[2], 1e-10); + for (int ii = 0; ii update_rule(f); + + FrankWolfe> s(f, linear_constr_solver, update_rule); + + vec coordinates = zeros(3); + double result = s.Optimize(coordinates); + + BOOST_REQUIRE_SMALL(result, 1e-4); + BOOST_REQUIRE_SMALL(coordinates[0]-0.1, 1e-4); + BOOST_REQUIRE_SMALL(coordinates[1]-0.2, 1e-4); + BOOST_REQUIRE_SMALL(coordinates[2]-0.3, 1e-4); } From b95969401f94200aceaafff140ed0f530aa8ee6d Mon Sep 17 00:00:00 2001 From: Chenzhe Diao Date: Mon, 26 Jun 2017 21:33:32 -0600 Subject: [PATCH 03/16] Forgot to add 2 files. --- src/mlpack/core/optimizers/fw/func_sq.hpp | 55 +++++++++++++++++++ .../core/optimizers/fw/test_func_fw.hpp | 48 ++++++++++++++++ 2 files changed, 103 insertions(+) create mode 100644 src/mlpack/core/optimizers/fw/func_sq.hpp create mode 100644 src/mlpack/core/optimizers/fw/test_func_fw.hpp diff --git a/src/mlpack/core/optimizers/fw/func_sq.hpp b/src/mlpack/core/optimizers/fw/func_sq.hpp new file mode 100644 index 00000000000..729c919a8f4 --- /dev/null +++ b/src/mlpack/core/optimizers/fw/func_sq.hpp @@ -0,0 +1,55 @@ +/** + * @file func_sq.hpp + * @author Chenzhe Diao + * + * Square loss function: \f$ x-> 0.5 * || Ax - b ||_2^2 \f$. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ +#ifndef MLPACK_CORE_OPTIMIZERS_FW_FUNC_SQ_HPP +#define MLPACK_CORE_OPTIMIZERS_FW_FUNC_SQ_HPP + +#include + +namespace mlpack { +namespace optimization { + +class FuncSq +{ + public: + FuncSq(const arma::mat A, const arma::mat b ) : A(A), b(b) + {/* Nothing to do. */} + + double Evaluate(const arma::mat& coords) + { + arma::vec r = A*coords - b; + arma::mat y = (r.t() * r) * 0.5; + return y(0,0); + } + + void Gradient(const arma::mat& coords, arma::mat& gradient) + { + arma::vec r = A*coords - b; + gradient = A.t() * r; + } + + + arma::mat MatrixA() const {return A;} + arma::mat& MatrixA() {return A;} + + arma::vec Vectorb() const { return b; } + arma::vec& Vectorb() { return b; } + + private: + arma::mat A; // matrix + arma::vec b; // vector + +}; + +} // namespace optimization +} // namespace mlpack + +#endif diff --git a/src/mlpack/core/optimizers/fw/test_func_fw.hpp b/src/mlpack/core/optimizers/fw/test_func_fw.hpp new file mode 100644 index 00000000000..b6fcd0302e2 --- /dev/null +++ b/src/mlpack/core/optimizers/fw/test_func_fw.hpp @@ -0,0 +1,48 @@ +/** + * @file test_func_sq.hpp + * @author Chenzhe Diao + * + * Simple test function for classic Frank Wolfe Algorithm: + * + * \f$ f(x) = (x1 - 0.1)^4 + (x2 - 0.2)^4 + (x3 - 0.3)^4 \f$ + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ +#ifndef MLPACK_CORE_OPTIMIZERS_FW_TEST_FUNC_FW_HPP +#define MLPACK_CORE_OPTIMIZERS_FW_TEST_FUNC_FW_HPP + +#include + +namespace mlpack { +namespace optimization { + +class TestFuncFW +{ + public: + TestFuncFW() + {/* Nothing to do. */} + + double Evaluate(const arma::mat& coords) + { + double f = std::pow(coords[0]-0.1, 2); + f += std::pow(coords[1]-0.2, 2); + f += std::pow(coords[2]-0.3, 2); + return f; + } + + void Gradient(const arma::mat& coords, arma::mat& gradient) + { + gradient[0] = coords[0]-0.1; + gradient[1] = coords[1]-0.2; + gradient[2] = coords[2]-0.3; + } + +}; + +} // namespace optimization +} // namespace mlpack + +#endif From 20aee0c1658269e0eab4102fd9beca46c238b41b Mon Sep 17 00:00:00 2001 From: Chenzhe Diao Date: Tue, 27 Jun 2017 01:47:14 -0600 Subject: [PATCH 04/16] Fixed some style problems. --- .../core/optimizers/fw/constr_lpball.hpp | 40 +++++----- src/mlpack/core/optimizers/fw/frank_wolfe.hpp | 15 ++-- .../core/optimizers/fw/frank_wolfe_impl.hpp | 74 +++++++++---------- src/mlpack/core/optimizers/fw/func_sq.hpp | 5 +- .../core/optimizers/fw/test_func_fw.hpp | 17 ++--- .../core/optimizers/fw/update_classic.hpp | 9 +-- src/mlpack/core/optimizers/fw/update_span.hpp | 46 ++++++------ src/mlpack/tests/frankwolfe_test.cpp | 9 ++- 8 files changed, 105 insertions(+), 110 deletions(-) diff --git a/src/mlpack/core/optimizers/fw/constr_lpball.hpp b/src/mlpack/core/optimizers/fw/constr_lpball.hpp index 4028531681c..023d66edf37 100644 --- a/src/mlpack/core/optimizers/fw/constr_lpball.hpp +++ b/src/mlpack/core/optimizers/fw/constr_lpball.hpp @@ -62,37 +62,35 @@ class ConstrLpBallSolver * @param s Output optimal solution in the constrained domain (lp ball). */ void Optimize(const arma::mat& v, - arma::mat& s) + arma::mat& s) { - - if (p==-1.0) + if (p == -1.0) { - // l-inf ball - s = -sign(v); - return; + // l-inf ball + s = -sign(v); + return; } - else if(p>1.0) + else if (p > 1.0) { - // lp ball with 1 class FrankWolfe @@ -106,8 +106,8 @@ class FrankWolfe * limit). * @param tolerance Maximum absolute tolerance to terminate algorithm. */ - FrankWolfe( FunctionType& function, - const LinearConstrSolverType linear_constr_solver, + FrankWolfe(FunctionType& function, + const LinearConstrSolverType linear_constr_solver, const UpdateRuleType update_rule, const size_t maxIterations = 100000, const double tolerance = 1e-10); @@ -123,12 +123,13 @@ class FrankWolfe double Optimize(arma::mat& iterate); //! Get the instantiated function to be optimized. - const FunctionType& Function() const { return function; } + const FunctionType& Function() const { return function; } //! Modify the instantiated function. FunctionType& Function() { return function; } //! Get the linear constrained solver. - LinearConstrSolverType LinearConstrSolver() const { return linear_constr_solver; } + LinearConstrSolverType LinearConstrSolver() + const { return linear_constr_solver; } //! Modify the linear constrained solver. LinearConstrSolverType& LinearConstrSolver() { return linear_constr_solver; } @@ -150,7 +151,7 @@ class FrankWolfe private: //! The instantiated function. FunctionType& function; - + //! The solver for constrained linear problem in first step. LinearConstrSolverType linear_constr_solver; diff --git a/src/mlpack/core/optimizers/fw/frank_wolfe_impl.hpp b/src/mlpack/core/optimizers/fw/frank_wolfe_impl.hpp index 848ecbe5639..dceaaf811b1 100644 --- a/src/mlpack/core/optimizers/fw/frank_wolfe_impl.hpp +++ b/src/mlpack/core/optimizers/fw/frank_wolfe_impl.hpp @@ -19,8 +19,8 @@ namespace mlpack { namespace optimization { template< - typename FunctionType, - typename LinearConstrSolverType, + typename FunctionType, + typename LinearConstrSolverType, typename UpdateRuleType > FrankWolfe::FrankWolfe( @@ -37,14 +37,14 @@ FrankWolfe::FrankWolfe( { /* Nothing to do*/ } -//! Optimize the function (minimize). +//! Optimize the function (minimize). template< - typename FunctionType, - typename LinearConstrSolverType, + typename FunctionType, + typename LinearConstrSolverType, typename UpdateRuleType > -double FrankWolfe::Optimize( - arma::mat& iterate) +double FrankWolfe +::Optimize(arma::mat& iterate) { // To keep track of the function value double CurrentObjective = function.Evaluate(iterate); @@ -55,42 +55,40 @@ double FrankWolfe::Optimiz arma::mat iterate_new(iterate.n_rows, iterate.n_cols); double gap = 0; - for(size_t i=1; i != maxIterations; ++i) + for (size_t i=1; i != maxIterations; ++i) { - //Output current objective function - Log::Info << "Iteration " << i << ", objective " - << CurrentObjective << "." << std::endl; - - // Reset counter variables. - PreviousObjective = CurrentObjective; - - // Calculate the gradient - function.Gradient(iterate, gradient); - - // Solve linear constrained problem, solution saved in s. - linear_constr_solver.Optimize(gradient, s); - - // Check duality gap for return condition - gap = std::fabs(dot(iterate-s, gradient)); - if (gap < tolerance) - { - Log::Info << "FrankWolfe: minimized within tolerance " - << tolerance << "; " << "terminating optimization." << std::endl; - return CurrentObjective; - } - - - // Update solution, save in iterate_new - update_rule.Update(iterate, s, iterate_new, i); - - - iterate = iterate_new; - CurrentObjective = function.Evaluate(iterate); + // Output current objective function + Log::Info << "Iteration " << i << ", objective " + << CurrentObjective << "." << std::endl; + + // Reset counter variables. + PreviousObjective = CurrentObjective; + + // Calculate the gradient + function.Gradient(iterate, gradient); + + // Solve linear constrained problem, solution saved in s. + linear_constr_solver.Optimize(gradient, s); + + // Check duality gap for return condition + gap = std::fabs(dot(iterate-s, gradient)); + if (gap < tolerance) + { + Log::Info << "FrankWolfe: minimized within tolerance " + << tolerance << "; " << "terminating optimization." << std::endl; + return CurrentObjective; + } + + + // Update solution, save in iterate_new + update_rule.Update(iterate, s, iterate_new, i); + + iterate = iterate_new; + CurrentObjective = function.Evaluate(iterate); } Log::Info << "Frank Wolfe: maximum iterations (" << maxIterations << ") reached; " << "terminating optimization." << std::endl; return CurrentObjective; - } diff --git a/src/mlpack/core/optimizers/fw/func_sq.hpp b/src/mlpack/core/optimizers/fw/func_sq.hpp index 729c919a8f4..62758c2fb57 100644 --- a/src/mlpack/core/optimizers/fw/func_sq.hpp +++ b/src/mlpack/core/optimizers/fw/func_sq.hpp @@ -20,14 +20,14 @@ namespace optimization { class FuncSq { public: - FuncSq(const arma::mat A, const arma::mat b ) : A(A), b(b) + FuncSq(const arma::mat A, const arma::mat b) : A(A), b(b) {/* Nothing to do. */} double Evaluate(const arma::mat& coords) { arma::vec r = A*coords - b; arma::mat y = (r.t() * r) * 0.5; - return y(0,0); + return y(0, 0); } void Gradient(const arma::mat& coords, arma::mat& gradient) @@ -46,7 +46,6 @@ class FuncSq private: arma::mat A; // matrix arma::vec b; // vector - }; } // namespace optimization diff --git a/src/mlpack/core/optimizers/fw/test_func_fw.hpp b/src/mlpack/core/optimizers/fw/test_func_fw.hpp index b6fcd0302e2..3425f4e516a 100644 --- a/src/mlpack/core/optimizers/fw/test_func_fw.hpp +++ b/src/mlpack/core/optimizers/fw/test_func_fw.hpp @@ -4,7 +4,7 @@ * * Simple test function for classic Frank Wolfe Algorithm: * - * \f$ f(x) = (x1 - 0.1)^4 + (x2 - 0.2)^4 + (x3 - 0.3)^4 \f$ + * \f$ f(x) = (x1 - 0.1)^2 + (x2 - 0.2)^2 + (x3 - 0.3)^2 \f$ * * mlpack is free software; you may redistribute it and/or modify it under the * terms of the 3-clause BSD license. You should have received a copy of the @@ -27,19 +27,18 @@ class TestFuncFW double Evaluate(const arma::mat& coords) { - double f = std::pow(coords[0]-0.1, 2); - f += std::pow(coords[1]-0.2, 2); - f += std::pow(coords[2]-0.3, 2); - return f; + double f = std::pow(coords[0]-0.1, 2); + f += std::pow(coords[1]-0.2, 2); + f += std::pow(coords[2]-0.3, 2); + return f; } void Gradient(const arma::mat& coords, arma::mat& gradient) { - gradient[0] = coords[0]-0.1; - gradient[1] = coords[1]-0.2; - gradient[2] = coords[2]-0.3; + gradient[0] = coords[0]-0.1; + gradient[1] = coords[1]-0.2; + gradient[2] = coords[2]-0.3; } - }; } // namespace optimization diff --git a/src/mlpack/core/optimizers/fw/update_classic.hpp b/src/mlpack/core/optimizers/fw/update_classic.hpp index 539ae762551..43d0b34aef0 100644 --- a/src/mlpack/core/optimizers/fw/update_classic.hpp +++ b/src/mlpack/core/optimizers/fw/update_classic.hpp @@ -64,9 +64,9 @@ class UpdateClassic * @param num_iter current iteration number */ void Update(const arma::mat& old_coords, - const arma::mat& s, - arma::mat& new_coords, - const size_t num_iter) + const arma::mat& s, + arma::mat& new_coords, + const size_t num_iter) { double gamma = 2.0/(num_iter + 2.0); new_coords = (1.0-gamma)*old_coords + gamma*s; @@ -79,8 +79,7 @@ class UpdateClassic private: //! The instantiated function. - FunctionType& function; - + FunctionType& function; }; } // namespace optimization diff --git a/src/mlpack/core/optimizers/fw/update_span.hpp b/src/mlpack/core/optimizers/fw/update_span.hpp index 6e00810904f..4a69a17de31 100644 --- a/src/mlpack/core/optimizers/fw/update_span.hpp +++ b/src/mlpack/core/optimizers/fw/update_span.hpp @@ -45,7 +45,7 @@ class UpdateSpan * * @param function Function to be optimized in FrankWolfe algorithm. */ - UpdateSpan(FunctionType& function): function(function) + UpdateSpan(FunctionType& function): function(function) { /* Do nothing. */ } /** @@ -58,11 +58,11 @@ class UpdateSpan * @param num_iter current iteration number */ void Update(const arma::mat& old_coords, - const arma::mat& s, - arma::mat& new_coords, - const size_t num_iter) + const arma::mat& s, + arma::mat& new_coords, + const size_t num_iter) { - //Need to add atom here + // add atom here arma::uvec ind = find(s, 1); arma::uword d = ind(0); AddAtom(d); @@ -90,22 +90,23 @@ class UpdateSpan //! Modify the current atoms. arma::mat& CurrentAtoms() { return atoms_current; } - //! Add atom into the solution space, modify the current_indices and atoms_current. - void AddAtom(const arma::uword k) + //! Add atom into the solution space. + void AddAtom(const arma::uword k) { - if (isEmpty){ - CurrentIndices() = k; - CurrentAtoms() = (function.MatrixA()).col(k); - isEmpty = false; + if (isEmpty) + { + CurrentIndices() = k; + CurrentAtoms() = (function.MatrixA()).col(k); + isEmpty = false; } - else{ - arma::uvec vk(1); - vk = k; - current_indices.insert_rows(0, vk); - - arma::mat atom = (function.MatrixA()).col(k); - atoms_current.insert_cols(0, atom); - + else + { + arma::uvec vk(1); + vk = k; + current_indices.insert_rows(0, vk); + + arma::mat atom = (function.MatrixA()).col(k); + atoms_current.insert_cols(0, atom); } } @@ -117,7 +118,7 @@ class UpdateSpan arma::uword len = current_indices.size(); for (size_t ii = 0; ii < len; ++ii) { - y(current_indices(ii)) = x(ii); + y(current_indices(ii)) = x(ii); } return y; @@ -128,14 +129,13 @@ class UpdateSpan FunctionType& function; //! Current indices. - arma::uvec current_indices; + arma::uvec current_indices; //! Current atoms. arma::mat atoms_current; //! Flag current indices is empty - bool isEmpty=true; - + bool isEmpty = true; }; } // namespace optimization diff --git a/src/mlpack/tests/frankwolfe_test.cpp b/src/mlpack/tests/frankwolfe_test.cpp index c6ccf7a9f80..fc307b6c018 100644 --- a/src/mlpack/tests/frankwolfe_test.cpp +++ b/src/mlpack/tests/frankwolfe_test.cpp @@ -50,18 +50,19 @@ BOOST_AUTO_TEST_CASE(OMPTest) BOOST_REQUIRE_SMALL(coordinates[2], 1e-10); for (int ii = 0; ii update_rule(f); - FrankWolfe> s(f, linear_constr_solver, update_rule); + FrankWolfe> + s(f, linear_constr_solver, update_rule); vec coordinates = zeros(3); double result = s.Optimize(coordinates); From b2ba0bc7e3d0e34b6ccc80b26082f4873a23f6d4 Mon Sep 17 00:00:00 2001 From: Chenzhe Diao Date: Tue, 27 Jun 2017 01:54:23 -0600 Subject: [PATCH 05/16] Fixed style problem again. --- src/mlpack/core/optimizers/fw/frank_wolfe.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/mlpack/core/optimizers/fw/frank_wolfe.hpp b/src/mlpack/core/optimizers/fw/frank_wolfe.hpp index 068061ead6b..3d58cde6941 100644 --- a/src/mlpack/core/optimizers/fw/frank_wolfe.hpp +++ b/src/mlpack/core/optimizers/fw/frank_wolfe.hpp @@ -128,7 +128,7 @@ class FrankWolfe FunctionType& Function() { return function; } //! Get the linear constrained solver. - LinearConstrSolverType LinearConstrSolver() + LinearConstrSolverType LinearConstrSolver() const { return linear_constr_solver; } //! Modify the linear constrained solver. LinearConstrSolverType& LinearConstrSolver() { return linear_constr_solver; } From 3e1ebac9e7ce0f55c9ad8b0d6b948d9e9fc3c253 Mon Sep 17 00:00:00 2001 From: Chenzhe Diao Date: Mon, 3 Jul 2017 17:35:08 -0600 Subject: [PATCH 06/16] Modified the optimization API, and some other issues. --- .../core/optimizers/fw/constr_lpball.hpp | 15 +++--- src/mlpack/core/optimizers/fw/frank_wolfe.hpp | 19 ++----- .../core/optimizers/fw/frank_wolfe_impl.hpp | 15 +++--- .../core/optimizers/fw/update_classic.hpp | 35 ++----------- src/mlpack/core/optimizers/fw/update_span.hpp | 52 +++++++++---------- src/mlpack/tests/frankwolfe_test.cpp | 14 ++--- 6 files changed, 57 insertions(+), 93 deletions(-) diff --git a/src/mlpack/core/optimizers/fw/constr_lpball.hpp b/src/mlpack/core/optimizers/fw/constr_lpball.hpp index 023d66edf37..96a51010d20 100644 --- a/src/mlpack/core/optimizers/fw/constr_lpball.hpp +++ b/src/mlpack/core/optimizers/fw/constr_lpball.hpp @@ -64,7 +64,7 @@ class ConstrLpBallSolver void Optimize(const arma::mat& v, arma::mat& s) { - if (p == -1.0) + if (p == std::numeric_limits::infinity()) { // l-inf ball s = -sign(v); @@ -79,11 +79,10 @@ class ConstrLpBallSolver else if (p == 1.0) { // l1 ball, used in OMP - arma::mat tmp = abs(v); + arma::mat tmp = arma::abs(v); arma::uword k = tmp.index_max(); // linear index of matrix - tmp = 0 * tmp; - tmp(k) = v(k); - s = -sign(tmp); + s.zeros(v.n_rows, v.n_cols); + s(k) = -sign_double(v(k)); return; } else @@ -94,8 +93,12 @@ class ConstrLpBallSolver } private: - // lp norm, take 1::infinity() for inf norm. double p; + + //! Signum function for double. + double sign_double(const double x) const {return (x > 0) - (x < 0);} }; } // namespace optimization diff --git a/src/mlpack/core/optimizers/fw/frank_wolfe.hpp b/src/mlpack/core/optimizers/fw/frank_wolfe.hpp index 3d58cde6941..0185d07cb25 100644 --- a/src/mlpack/core/optimizers/fw/frank_wolfe.hpp +++ b/src/mlpack/core/optimizers/fw/frank_wolfe.hpp @@ -86,7 +86,6 @@ namespace optimization { * */ template< - typename FunctionType, typename LinearConstrSolverType, typename UpdateRuleType > @@ -99,15 +98,13 @@ class FrankWolfe * at the initialization of linear_constr_solver, the function to be * optimized is stored in update_rule. * - * @param function Function to be optimized * @param linear_constr_solver Solver for linear constrained problem. * @param update_rule Rule for updating solution in each iteration. * @param maxIterations Maximum number of iterations allowed (0 means no * limit). * @param tolerance Maximum absolute tolerance to terminate algorithm. */ - FrankWolfe(FunctionType& function, - const LinearConstrSolverType linear_constr_solver, + FrankWolfe(const LinearConstrSolverType linear_constr_solver, const UpdateRuleType update_rule, const size_t maxIterations = 100000, const double tolerance = 1e-10); @@ -117,15 +114,12 @@ class FrankWolfe * point will be modified to store the finishing point of the algorithm, and * the final objective value is returned. * + * @tparam function Function to be optimized. * @param iterate Starting point (will be modified). * @return Objective value of the final point. */ - double Optimize(arma::mat& iterate); - - //! Get the instantiated function to be optimized. - const FunctionType& Function() const { return function; } - //! Modify the instantiated function. - FunctionType& Function() { return function; } + template + double Optimize(FunctionType& function,arma::mat& iterate); //! Get the linear constrained solver. LinearConstrSolverType LinearConstrSolver() @@ -149,9 +143,6 @@ class FrankWolfe double& Tolerance() { return tolerance; } private: - //! The instantiated function. - FunctionType& function; - //! The solver for constrained linear problem in first step. LinearConstrSolverType linear_constr_solver; @@ -166,7 +157,7 @@ class FrankWolfe }; //! Orthogonal Matching Pursuit -using OMP = FrankWolfe>; +using OMP = FrankWolfe; } // namespace optimization } // namespace mlpack diff --git a/src/mlpack/core/optimizers/fw/frank_wolfe_impl.hpp b/src/mlpack/core/optimizers/fw/frank_wolfe_impl.hpp index dceaaf811b1..dee933548d9 100644 --- a/src/mlpack/core/optimizers/fw/frank_wolfe_impl.hpp +++ b/src/mlpack/core/optimizers/fw/frank_wolfe_impl.hpp @@ -19,17 +19,14 @@ namespace mlpack { namespace optimization { template< - typename FunctionType, typename LinearConstrSolverType, typename UpdateRuleType > -FrankWolfe::FrankWolfe( - FunctionType& function, +FrankWolfe::FrankWolfe( const LinearConstrSolverType linear_constr_solver, const UpdateRuleType update_rule, const size_t maxIterations, const double tolerance) : - function(function), linear_constr_solver(linear_constr_solver), update_rule(update_rule), maxIterations(maxIterations), @@ -39,12 +36,12 @@ FrankWolfe::FrankWolfe( //! Optimize the function (minimize). template< - typename FunctionType, typename LinearConstrSolverType, typename UpdateRuleType > -double FrankWolfe -::Optimize(arma::mat& iterate) +template +double FrankWolfe +::Optimize(FunctionType& function, arma::mat& iterate) { // To keep track of the function value double CurrentObjective = function.Evaluate(iterate); @@ -81,9 +78,9 @@ ::Optimize(arma::mat& iterate) // Update solution, save in iterate_new - update_rule.Update(iterate, s, iterate_new, i); + update_rule.Update(function, iterate, s, iterate_new, i); - iterate = iterate_new; + iterate = std::move(iterate_new); CurrentObjective = function.Evaluate(iterate); } Log::Info << "Frank Wolfe: maximum iterations (" << maxIterations diff --git a/src/mlpack/core/optimizers/fw/update_classic.hpp b/src/mlpack/core/optimizers/fw/update_classic.hpp index 43d0b34aef0..055a0585724 100644 --- a/src/mlpack/core/optimizers/fw/update_classic.hpp +++ b/src/mlpack/core/optimizers/fw/update_classic.hpp @@ -25,32 +25,14 @@ namespace optimization { * x_{k+1} = (1-\gamma) x_k + \gamma s * \f] * - * For UpdateClassic to work, FunctionType template parameters are required. - * This class must implement the following functions: - * - * FunctionType: - * - * double Evaluate(const arma::mat& coordinates); - * void Gradient(const arma::mat& coordinates, - * arma::mat& gradient); - * - * - * Although the function is not used in classic update rule, we still put it - * here since we want the same template code interface. - * - * @tparam FunctionType Objective function type to be minimized in FrankWolfe algorithm. */ -template class UpdateClassic { public: /** - * Construct the classic update rule. The function to be optimized is - * input here, although not used. - * - * @param function Function to be optimized in FrankWolfe algorithm. + * Construct the classic update rule for FrankWolfe algorithm. */ - UpdateClassic(FunctionType& function): function(function) + UpdateClassic() { /* Do nothing. */ } /** @@ -63,7 +45,9 @@ class UpdateClassic * @param new_coords new output solution coords. * @param num_iter current iteration number */ - void Update(const arma::mat& old_coords, + template + void Update(FunctionType& function, + const arma::mat& old_coords, const arma::mat& s, arma::mat& new_coords, const size_t num_iter) @@ -71,15 +55,6 @@ class UpdateClassic double gamma = 2.0/(num_iter + 2.0); new_coords = (1.0-gamma)*old_coords + gamma*s; } - - //! Get the instantiated function to be optimized. - const FunctionType& Function() const { return function; } - //! Modify the instantiated function. - FunctionType& Function() { return function; } - - private: - //! The instantiated function. - FunctionType& function; }; } // namespace optimization diff --git a/src/mlpack/core/optimizers/fw/update_span.hpp b/src/mlpack/core/optimizers/fw/update_span.hpp index 4a69a17de31..49bdd165b7a 100644 --- a/src/mlpack/core/optimizers/fw/update_span.hpp +++ b/src/mlpack/core/optimizers/fw/update_span.hpp @@ -31,11 +31,12 @@ namespace optimization { * double Evaluate(const arma::mat& coordinates); * void Gradient(const arma::mat& coordinates, * arma::mat& gradient); + * arma::mat MatrixA() + * arma::vec Vectorb() * * * @tparam FunctionType Objective function type to be minimized in FrankWolfe algorithm. */ -template class UpdateSpan { public: @@ -45,19 +46,22 @@ class UpdateSpan * * @param function Function to be optimized in FrankWolfe algorithm. */ - UpdateSpan(FunctionType& function): function(function) + UpdateSpan() { /* Do nothing. */ } /** * Update rule for FrankWolfe, reoptimize in the span of original solution space. * * + * @tparam function function to be optimized. * @param old_coords previous solution coords. * @param s current linear_constr_solution result. * @param new_coords new output solution coords. * @param num_iter current iteration number */ - void Update(const arma::mat& old_coords, + template + void Update(FunctionType& function, + const arma::mat& old_coords, const arma::mat& s, arma::mat& new_coords, const size_t num_iter) @@ -65,21 +69,14 @@ class UpdateSpan // add atom here arma::uvec ind = find(s, 1); arma::uword d = ind(0); - AddAtom(d); + AddAtom(function, d); arma::vec b = function.Vectorb(); arma::mat x = solve(atoms_current, b); - new_coords = RecoverVector(x); + new_coords = RecoverVector(function, x); } - - - //! Get the instantiated function to be optimized. - FunctionType Function() const { return function; } - //! Modify the instantiated function. - FunctionType& Function() { return function; } - //! Get the current atom indices. arma::uvec CurrentIndices() const { return current_indices; } //! Modify the current atom indices. @@ -90,8 +87,21 @@ class UpdateSpan //! Modify the current atoms. arma::mat& CurrentAtoms() { return atoms_current; } + + + private: + //! Current indices. + arma::uvec current_indices; + + //! Current atoms. + arma::mat atoms_current; + + //! Flag current indices is empty + bool isEmpty = true; + //! Add atom into the solution space. - void AddAtom(const arma::uword k) + template + void AddAtom(FunctionType& function, const arma::uword k) { if (isEmpty) { @@ -110,7 +120,8 @@ class UpdateSpan } } - arma::vec RecoverVector(const arma::vec& x ) + template + arma::vec RecoverVector(FunctionType& function, const arma::vec& x) { int n = (function.MatrixA()).n_cols; arma::vec y = arma::zeros(n); @@ -123,19 +134,6 @@ class UpdateSpan return y; } - - private: - //! The instantiated function. - FunctionType& function; - - //! Current indices. - arma::uvec current_indices; - - //! Current atoms. - arma::mat atoms_current; - - //! Flag current indices is empty - bool isEmpty = true; }; } // namespace optimization diff --git a/src/mlpack/tests/frankwolfe_test.cpp b/src/mlpack/tests/frankwolfe_test.cpp index fc307b6c018..669a83a4c54 100644 --- a/src/mlpack/tests/frankwolfe_test.cpp +++ b/src/mlpack/tests/frankwolfe_test.cpp @@ -37,12 +37,12 @@ BOOST_AUTO_TEST_CASE(OMPTest) FuncSq f(A, b); ConstrLpBallSolver linear_constr_solver(1); - UpdateSpan update_rule(f); + UpdateSpan update_rule; - OMP s(f, linear_constr_solver, update_rule); + OMP s(linear_constr_solver, update_rule); vec coordinates = zeros(k+3); - double result = s.Optimize(coordinates); + double result = s.Optimize(f, coordinates); BOOST_REQUIRE_SMALL(result, 1e-10); BOOST_REQUIRE_SMALL(coordinates[0]-1, 1e-10); @@ -59,13 +59,13 @@ BOOST_AUTO_TEST_CASE(ClassicFW) TestFuncFW f; double p = 1; // constraint set is unit lp ball ConstrLpBallSolver linear_constr_solver(p); - UpdateClassic update_rule(f); + UpdateClassic update_rule; - FrankWolfe> - s(f, linear_constr_solver, update_rule); + FrankWolfe + s(linear_constr_solver, update_rule); vec coordinates = zeros(3); - double result = s.Optimize(coordinates); + double result = s.Optimize(f, coordinates); BOOST_REQUIRE_SMALL(result, 1e-4); BOOST_REQUIRE_SMALL(coordinates[0]-0.1, 1e-4); From bc056f1bfc1ca09e6730c2170bf940fc04c895d8 Mon Sep 17 00:00:00 2001 From: Chenzhe Diao Date: Mon, 3 Jul 2017 18:05:16 -0600 Subject: [PATCH 07/16] Fixed style problem. --- src/mlpack/core/optimizers/fw/frank_wolfe.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/mlpack/core/optimizers/fw/frank_wolfe.hpp b/src/mlpack/core/optimizers/fw/frank_wolfe.hpp index 0185d07cb25..01c65151466 100644 --- a/src/mlpack/core/optimizers/fw/frank_wolfe.hpp +++ b/src/mlpack/core/optimizers/fw/frank_wolfe.hpp @@ -119,7 +119,7 @@ class FrankWolfe * @return Objective value of the final point. */ template - double Optimize(FunctionType& function,arma::mat& iterate); + double Optimize(FunctionType& function, arma::mat& iterate); //! Get the linear constrained solver. LinearConstrSolverType LinearConstrSolver() From 41fb64143a439030a19732212bdc69da6f5f7e7c Mon Sep 17 00:00:00 2001 From: Chenzhe Diao Date: Sat, 8 Jul 2017 18:14:35 -0600 Subject: [PATCH 08/16] Fixed some code indentation and alignment problem. --- .../core/optimizers/fw/constr_lpball.hpp | 48 +++++----- src/mlpack/core/optimizers/fw/frank_wolfe.hpp | 2 +- .../core/optimizers/fw/frank_wolfe_impl.hpp | 58 ++++++------ src/mlpack/core/optimizers/fw/func_sq.hpp | 10 +-- .../core/optimizers/fw/test_func_fw.hpp | 30 +++---- .../core/optimizers/fw/update_classic.hpp | 32 +++---- src/mlpack/core/optimizers/fw/update_span.hpp | 88 +++++++++---------- src/mlpack/tests/frankwolfe_test.cpp | 60 ++++++------- 8 files changed, 160 insertions(+), 168 deletions(-) diff --git a/src/mlpack/core/optimizers/fw/constr_lpball.hpp b/src/mlpack/core/optimizers/fw/constr_lpball.hpp index 96a51010d20..cc4288621d2 100644 --- a/src/mlpack/core/optimizers/fw/constr_lpball.hpp +++ b/src/mlpack/core/optimizers/fw/constr_lpball.hpp @@ -55,46 +55,44 @@ class ConstrLpBallSolver ConstrLpBallSolver(const double p) : p(p) { /* Do nothing. */ } - /** - * Optimizer of Linear Constrained Problem for FrankWolfe. - * - * @param v Input local gradient. - * @param s Output optimal solution in the constrained domain (lp ball). - */ + /** + * Optimizer of Linear Constrained Problem for FrankWolfe. + * + * @param v Input local gradient. + * @param s Output optimal solution in the constrained domain (lp ball). + */ void Optimize(const arma::mat& v, - arma::mat& s) + arma::mat& s) { - if (p == std::numeric_limits::infinity()) - { + if (p == std::numeric_limits::infinity()) + { // l-inf ball s = -sign(v); - return; - } - else if (p > 1.0) - { + } + else if (p > 1.0) + { // lp ball with 1::infinity() for inf norm. + //! lp norm, 1<=p<=inf; + //! use std::numeric_limits::infinity() for inf norm. double p; //! Signum function for double. diff --git a/src/mlpack/core/optimizers/fw/frank_wolfe.hpp b/src/mlpack/core/optimizers/fw/frank_wolfe.hpp index 01c65151466..8f9d5e61baf 100644 --- a/src/mlpack/core/optimizers/fw/frank_wolfe.hpp +++ b/src/mlpack/core/optimizers/fw/frank_wolfe.hpp @@ -156,7 +156,7 @@ class FrankWolfe double tolerance; }; -//! Orthogonal Matching Pursuit +//! Orthogonal Matching Pursuit. using OMP = FrankWolfe; } // namespace optimization diff --git a/src/mlpack/core/optimizers/fw/frank_wolfe_impl.hpp b/src/mlpack/core/optimizers/fw/frank_wolfe_impl.hpp index dee933548d9..10755843466 100644 --- a/src/mlpack/core/optimizers/fw/frank_wolfe_impl.hpp +++ b/src/mlpack/core/optimizers/fw/frank_wolfe_impl.hpp @@ -20,13 +20,12 @@ namespace optimization { template< typename LinearConstrSolverType, - typename UpdateRuleType -> -FrankWolfe::FrankWolfe( - const LinearConstrSolverType linear_constr_solver, - const UpdateRuleType update_rule, - const size_t maxIterations, - const double tolerance) : + typename UpdateRuleType> +FrankWolfe:: +FrankWolfe(const LinearConstrSolverType linear_constr_solver, + const UpdateRuleType update_rule, + const size_t maxIterations, + const double tolerance) : linear_constr_solver(linear_constr_solver), update_rule(update_rule), maxIterations(maxIterations), @@ -37,52 +36,51 @@ FrankWolfe::FrankWolfe( //! Optimize the function (minimize). template< typename LinearConstrSolverType, - typename UpdateRuleType -> + typename UpdateRuleType> template -double FrankWolfe -::Optimize(FunctionType& function, arma::mat& iterate) +double FrankWolfe:: +Optimize(FunctionType& function, arma::mat& iterate) { - // To keep track of the function value - double CurrentObjective = function.Evaluate(iterate); - double PreviousObjective = DBL_MAX; - - arma::mat gradient(iterate.n_rows, iterate.n_cols); - arma::mat s(iterate.n_rows, iterate.n_cols); - arma::mat iterate_new(iterate.n_rows, iterate.n_cols); - double gap = 0; - - for (size_t i=1; i != maxIterations; ++i) - { - // Output current objective function + // To keep track of the function value + double CurrentObjective = function.Evaluate(iterate); + double PreviousObjective = DBL_MAX; + + arma::mat gradient(iterate.n_rows, iterate.n_cols); + arma::mat s(iterate.n_rows, iterate.n_cols); + arma::mat iterate_new(iterate.n_rows, iterate.n_cols); + double gap = 0; + + for (size_t i=1; i != maxIterations; ++i) + { + // Output current objective function. Log::Info << "Iteration " << i << ", objective " << CurrentObjective << "." << std::endl; // Reset counter variables. PreviousObjective = CurrentObjective; - // Calculate the gradient + // Calculate the gradient. function.Gradient(iterate, gradient); // Solve linear constrained problem, solution saved in s. linear_constr_solver.Optimize(gradient, s); - // Check duality gap for return condition + // Check duality gap for return condition. gap = std::fabs(dot(iterate-s, gradient)); if (gap < tolerance) { - Log::Info << "FrankWolfe: minimized within tolerance " - << tolerance << "; " << "terminating optimization." << std::endl; - return CurrentObjective; + Log::Info << "FrankWolfe: minimized within tolerance " + << tolerance << "; " << "terminating optimization." << std::endl; + return CurrentObjective; } - // Update solution, save in iterate_new + // Update solution, save in iterate_new. update_rule.Update(function, iterate, s, iterate_new, i); iterate = std::move(iterate_new); CurrentObjective = function.Evaluate(iterate); - } + } Log::Info << "Frank Wolfe: maximum iterations (" << maxIterations << ") reached; " << "terminating optimization." << std::endl; return CurrentObjective; diff --git a/src/mlpack/core/optimizers/fw/func_sq.hpp b/src/mlpack/core/optimizers/fw/func_sq.hpp index 62758c2fb57..81f90eb0f41 100644 --- a/src/mlpack/core/optimizers/fw/func_sq.hpp +++ b/src/mlpack/core/optimizers/fw/func_sq.hpp @@ -25,15 +25,15 @@ class FuncSq double Evaluate(const arma::mat& coords) { - arma::vec r = A*coords - b; - arma::mat y = (r.t() * r) * 0.5; - return y(0, 0); + arma::vec r = A*coords - b; + arma::mat y = (r.t() * r) * 0.5; + return y(0, 0); } void Gradient(const arma::mat& coords, arma::mat& gradient) { - arma::vec r = A*coords - b; - gradient = A.t() * r; + arma::vec r = A*coords - b; + gradient = A.t() * r; } diff --git a/src/mlpack/core/optimizers/fw/test_func_fw.hpp b/src/mlpack/core/optimizers/fw/test_func_fw.hpp index 3425f4e516a..6617b50deae 100644 --- a/src/mlpack/core/optimizers/fw/test_func_fw.hpp +++ b/src/mlpack/core/optimizers/fw/test_func_fw.hpp @@ -22,23 +22,23 @@ namespace optimization { class TestFuncFW { public: - TestFuncFW() - {/* Nothing to do. */} + TestFuncFW() + {/* Nothing to do. */} - double Evaluate(const arma::mat& coords) - { - double f = std::pow(coords[0]-0.1, 2); - f += std::pow(coords[1]-0.2, 2); - f += std::pow(coords[2]-0.3, 2); - return f; - } + double Evaluate(const arma::mat& coords) + { + double f = std::pow(coords[0]-0.1, 2); + f += std::pow(coords[1]-0.2, 2); + f += std::pow(coords[2]-0.3, 2); + return f; + } - void Gradient(const arma::mat& coords, arma::mat& gradient) - { - gradient[0] = coords[0]-0.1; - gradient[1] = coords[1]-0.2; - gradient[2] = coords[2]-0.3; - } + void Gradient(const arma::mat& coords, arma::mat& gradient) + { + gradient[0] = coords[0]-0.1; + gradient[1] = coords[1]-0.2; + gradient[2] = coords[2]-0.3; + } }; } // namespace optimization diff --git a/src/mlpack/core/optimizers/fw/update_classic.hpp b/src/mlpack/core/optimizers/fw/update_classic.hpp index 055a0585724..19e1ce94ee6 100644 --- a/src/mlpack/core/optimizers/fw/update_classic.hpp +++ b/src/mlpack/core/optimizers/fw/update_classic.hpp @@ -35,25 +35,25 @@ class UpdateClassic UpdateClassic() { /* Do nothing. */ } - /** - * Classic update rule for FrankWolfe. - * - * \f$ x_{k+1} = (1-\gamma)x_k + \gamma s \f$, where \f$ \gamma = 2/(k+2) \f$ - * - * @param old_coords previous solution coords. - * @param s current linear_constr_solution result. - * @param new_coords new output solution coords. - * @param num_iter current iteration number - */ + /** + * Classic update rule for FrankWolfe. + * + * \f$ x_{k+1} = (1-\gamma)x_k + \gamma s \f$, where \f$ \gamma = 2/(k+2) \f$ + * + * @param old_coords previous solution coords. + * @param s current linear_constr_solution result. + * @param new_coords new output solution coords. + * @param num_iter current iteration number + */ template void Update(FunctionType& function, - const arma::mat& old_coords, - const arma::mat& s, - arma::mat& new_coords, - const size_t num_iter) + const arma::mat& old_coords, + const arma::mat& s, + arma::mat& new_coords, + const size_t num_iter) { - double gamma = 2.0/(num_iter + 2.0); - new_coords = (1.0-gamma)*old_coords + gamma*s; + double gamma = 2.0 / (num_iter + 2.0); + new_coords = (1.0 - gamma) * old_coords + gamma * s; } }; diff --git a/src/mlpack/core/optimizers/fw/update_span.hpp b/src/mlpack/core/optimizers/fw/update_span.hpp index 49bdd165b7a..5e7e28d75ad 100644 --- a/src/mlpack/core/optimizers/fw/update_span.hpp +++ b/src/mlpack/core/optimizers/fw/update_span.hpp @@ -49,32 +49,32 @@ class UpdateSpan UpdateSpan() { /* Do nothing. */ } - /** - * Update rule for FrankWolfe, reoptimize in the span of original solution space. - * - * - * @tparam function function to be optimized. - * @param old_coords previous solution coords. - * @param s current linear_constr_solution result. - * @param new_coords new output solution coords. - * @param num_iter current iteration number - */ + /** + * Update rule for FrankWolfe, reoptimize in the span of original solution space. + * + * + * @tparam function function to be optimized. + * @param old_coords previous solution coords. + * @param s current linear_constr_solution result. + * @param new_coords new output solution coords. + * @param num_iter current iteration number + */ template void Update(FunctionType& function, - const arma::mat& old_coords, - const arma::mat& s, - arma::mat& new_coords, - const size_t num_iter) + const arma::mat& old_coords, + const arma::mat& s, + arma::mat& new_coords, + const size_t num_iter) { - // add atom here - arma::uvec ind = find(s, 1); - arma::uword d = ind(0); - AddAtom(function, d); + // Add atom here. + arma::uvec ind = find(s, 1); + arma::uword d = ind(0); + AddAtom(function, d); - arma::vec b = function.Vectorb(); - arma::mat x = solve(atoms_current, b); + arma::vec b = function.Vectorb(); + arma::mat x = solve(atoms_current, b); - new_coords = RecoverVector(function, x); + new_coords = RecoverVector(function, x); } //! Get the current atom indices. @@ -87,8 +87,6 @@ class UpdateSpan //! Modify the current atoms. arma::mat& CurrentAtoms() { return atoms_current; } - - private: //! Current indices. arma::uvec current_indices; @@ -96,43 +94,41 @@ class UpdateSpan //! Current atoms. arma::mat atoms_current; - //! Flag current indices is empty + //! Flag current indices is empty. bool isEmpty = true; //! Add atom into the solution space. template void AddAtom(FunctionType& function, const arma::uword k) { - if (isEmpty) - { - CurrentIndices() = k; - CurrentAtoms() = (function.MatrixA()).col(k); - isEmpty = false; - } - else - { - arma::uvec vk(1); - vk = k; - current_indices.insert_rows(0, vk); - - arma::mat atom = (function.MatrixA()).col(k); - atoms_current.insert_cols(0, atom); - } + if (isEmpty) + { + CurrentIndices() = k; + CurrentAtoms() = (function.MatrixA()).col(k); + isEmpty = false; + } + else + { + arma::uvec vk(1); + vk = k; + current_indices.insert_rows(0, vk); + + arma::mat atom = (function.MatrixA()).col(k); + atoms_current.insert_cols(0, atom); + } } template arma::vec RecoverVector(FunctionType& function, const arma::vec& x) { - int n = (function.MatrixA()).n_cols; - arma::vec y = arma::zeros(n); + int n = (function.MatrixA()).n_cols; + arma::vec y = arma::zeros(n); - arma::uword len = current_indices.size(); - for (size_t ii = 0; ii < len; ++ii) - { + arma::uword len = current_indices.size(); + for (size_t ii = 0; ii < len; ++ii) y(current_indices(ii)) = x(ii); - } - return y; + return y; } }; diff --git a/src/mlpack/tests/frankwolfe_test.cpp b/src/mlpack/tests/frankwolfe_test.cpp index 669a83a4c54..9451c5ad0e3 100644 --- a/src/mlpack/tests/frankwolfe_test.cpp +++ b/src/mlpack/tests/frankwolfe_test.cpp @@ -29,48 +29,48 @@ BOOST_AUTO_TEST_SUITE(FrankWolfeTest); BOOST_AUTO_TEST_CASE(OMPTest) { - int k = 5; - mat B1 = eye(3, 3); - mat B2 = 0.1*randn(3, k); - mat A = join_horiz(B1, B2); - vec b("1; 1; 0"); + int k = 5; + mat B1 = eye(3, 3); + mat B2 = 0.1 * randn(3, k); + mat A = join_horiz(B1, B2); + vec b("1; 1; 0"); - FuncSq f(A, b); - ConstrLpBallSolver linear_constr_solver(1); - UpdateSpan update_rule; + FuncSq f(A, b); + ConstrLpBallSolver linear_constr_solver(1); + UpdateSpan update_rule; - OMP s(linear_constr_solver, update_rule); + OMP s(linear_constr_solver, update_rule); - vec coordinates = zeros(k+3); - double result = s.Optimize(f, coordinates); + vec coordinates = zeros(k + 3); + double result = s.Optimize(f, coordinates); - BOOST_REQUIRE_SMALL(result, 1e-10); - BOOST_REQUIRE_SMALL(coordinates[0]-1, 1e-10); - BOOST_REQUIRE_SMALL(coordinates[1]-1, 1e-10); - BOOST_REQUIRE_SMALL(coordinates[2], 1e-10); - for (int ii = 0; ii - s(linear_constr_solver, update_rule); + FrankWolfe + s(linear_constr_solver, update_rule); - vec coordinates = zeros(3); - double result = s.Optimize(f, coordinates); + vec coordinates = zeros(3); + double result = s.Optimize(f, coordinates); - BOOST_REQUIRE_SMALL(result, 1e-4); - BOOST_REQUIRE_SMALL(coordinates[0]-0.1, 1e-4); - BOOST_REQUIRE_SMALL(coordinates[1]-0.2, 1e-4); - BOOST_REQUIRE_SMALL(coordinates[2]-0.3, 1e-4); + BOOST_REQUIRE_SMALL(result, 1e-4); + BOOST_REQUIRE_SMALL(coordinates[0]-0.1, 1e-4); + BOOST_REQUIRE_SMALL(coordinates[1]-0.2, 1e-4); + BOOST_REQUIRE_SMALL(coordinates[2]-0.3, 1e-4); } From 4d43b4e34f81ad0378bee7d889d5395caaa208ec Mon Sep 17 00:00:00 2001 From: Chenzhe Diao Date: Sat, 8 Jul 2017 18:47:48 -0600 Subject: [PATCH 09/16] Added signum function to lin_alg.hpp. --- src/mlpack/core/math/lin_alg.hpp | 13 +++++++++++++ src/mlpack/core/optimizers/fw/constr_lpball.hpp | 13 +++++-------- src/mlpack/core/optimizers/fw/frank_wolfe.hpp | 9 ++++----- 3 files changed, 22 insertions(+), 13 deletions(-) diff --git a/src/mlpack/core/math/lin_alg.hpp b/src/mlpack/core/math/lin_alg.hpp index 6552dc85753..eba80dcad39 100644 --- a/src/mlpack/core/math/lin_alg.hpp +++ b/src/mlpack/core/math/lin_alg.hpp @@ -124,6 +124,19 @@ inline size_t SvecIndex(size_t i, size_t j, size_t n); */ void SymKronId(const arma::mat& A, arma::mat& op); +/** + * Signum function. + * Return 1 if x>0; return 0 if x=0; return -1 if x<0. + * Return type are the same as input type. + * + * @param x + */ +template +T Sign(const T x) +{ + return (T(0) < x) - (x < T(0)); +} + } // namespace math } // namespace mlpack diff --git a/src/mlpack/core/optimizers/fw/constr_lpball.hpp b/src/mlpack/core/optimizers/fw/constr_lpball.hpp index cc4288621d2..01f90563e68 100644 --- a/src/mlpack/core/optimizers/fw/constr_lpball.hpp +++ b/src/mlpack/core/optimizers/fw/constr_lpball.hpp @@ -66,21 +66,21 @@ class ConstrLpBallSolver { if (p == std::numeric_limits::infinity()) { - // l-inf ball + // l-inf ball. s = -sign(v); } else if (p > 1.0) { - // lp ball with 1::infinity() for inf norm. double p; - - //! Signum function for double. - double sign_double(const double x) const {return (x > 0) - (x < 0);} }; } // namespace optimization diff --git a/src/mlpack/core/optimizers/fw/frank_wolfe.hpp b/src/mlpack/core/optimizers/fw/frank_wolfe.hpp index 8f9d5e61baf..e728da45cdf 100644 --- a/src/mlpack/core/optimizers/fw/frank_wolfe.hpp +++ b/src/mlpack/core/optimizers/fw/frank_wolfe.hpp @@ -87,8 +87,7 @@ namespace optimization { */ template< typename LinearConstrSolverType, - typename UpdateRuleType -> + typename UpdateRuleType> class FrankWolfe { public: @@ -105,9 +104,9 @@ class FrankWolfe * @param tolerance Maximum absolute tolerance to terminate algorithm. */ FrankWolfe(const LinearConstrSolverType linear_constr_solver, - const UpdateRuleType update_rule, - const size_t maxIterations = 100000, - const double tolerance = 1e-10); + const UpdateRuleType update_rule, + const size_t maxIterations = 100000, + const double tolerance = 1e-10); /** * Optimize the given function using FrankWolfe. The given starting From 49b23c091f8c5f33c109aa1db4782e2608b68d09 Mon Sep 17 00:00:00 2001 From: Chenzhe Diao Date: Sat, 8 Jul 2017 19:36:28 -0600 Subject: [PATCH 10/16] Fixed camel casing for variables. --- src/mlpack/core/optimizers/fw/frank_wolfe.hpp | 20 ++++----- .../core/optimizers/fw/frank_wolfe_impl.hpp | 18 ++++---- .../core/optimizers/fw/update_classic.hpp | 13 +++--- src/mlpack/core/optimizers/fw/update_span.hpp | 45 ++++++++++--------- 4 files changed, 49 insertions(+), 47 deletions(-) diff --git a/src/mlpack/core/optimizers/fw/frank_wolfe.hpp b/src/mlpack/core/optimizers/fw/frank_wolfe.hpp index e728da45cdf..459a6ed87e3 100644 --- a/src/mlpack/core/optimizers/fw/frank_wolfe.hpp +++ b/src/mlpack/core/optimizers/fw/frank_wolfe.hpp @@ -97,14 +97,14 @@ class FrankWolfe * at the initialization of linear_constr_solver, the function to be * optimized is stored in update_rule. * - * @param linear_constr_solver Solver for linear constrained problem. - * @param update_rule Rule for updating solution in each iteration. + * @param linearConstrSolver Solver for linear constrained problem. + * @param updateRule Rule for updating solution in each iteration. * @param maxIterations Maximum number of iterations allowed (0 means no * limit). * @param tolerance Maximum absolute tolerance to terminate algorithm. */ - FrankWolfe(const LinearConstrSolverType linear_constr_solver, - const UpdateRuleType update_rule, + FrankWolfe(const LinearConstrSolverType linearConstrSolver, + const UpdateRuleType updateRule, const size_t maxIterations = 100000, const double tolerance = 1e-10); @@ -122,14 +122,14 @@ class FrankWolfe //! Get the linear constrained solver. LinearConstrSolverType LinearConstrSolver() - const { return linear_constr_solver; } + const { return linearConstrSolver; } //! Modify the linear constrained solver. - LinearConstrSolverType& LinearConstrSolver() { return linear_constr_solver; } + LinearConstrSolverType& LinearConstrSolver() { return linearConstrSolver; } //! Get the update rule. - UpdateRuleType UpdateRule() const { return update_rule; } + UpdateRuleType UpdateRule() const { return updateRule; } //! Modify the update rule. - UpdateRuleType& UpdateRule() { return update_rule; } + UpdateRuleType& UpdateRule() { return updateRule; } //! Get the maximum number of iterations (0 indicates no limit). size_t MaxIterations() const { return maxIterations; } @@ -143,10 +143,10 @@ class FrankWolfe private: //! The solver for constrained linear problem in first step. - LinearConstrSolverType linear_constr_solver; + LinearConstrSolverType linearConstrSolver; //! The rule to update, used in the second step. - UpdateRuleType update_rule; + UpdateRuleType updateRule; //! The maximum number of allowed iterations. size_t maxIterations; diff --git a/src/mlpack/core/optimizers/fw/frank_wolfe_impl.hpp b/src/mlpack/core/optimizers/fw/frank_wolfe_impl.hpp index 10755843466..e0b3d3d0030 100644 --- a/src/mlpack/core/optimizers/fw/frank_wolfe_impl.hpp +++ b/src/mlpack/core/optimizers/fw/frank_wolfe_impl.hpp @@ -22,12 +22,12 @@ template< typename LinearConstrSolverType, typename UpdateRuleType> FrankWolfe:: -FrankWolfe(const LinearConstrSolverType linear_constr_solver, - const UpdateRuleType update_rule, +FrankWolfe(const LinearConstrSolverType linearConstrSolver, + const UpdateRuleType updateRule, const size_t maxIterations, const double tolerance) : - linear_constr_solver(linear_constr_solver), - update_rule(update_rule), + linearConstrSolver(linearConstrSolver), + updateRule(updateRule), maxIterations(maxIterations), tolerance(tolerance) { /* Nothing to do*/ } @@ -47,7 +47,7 @@ Optimize(FunctionType& function, arma::mat& iterate) arma::mat gradient(iterate.n_rows, iterate.n_cols); arma::mat s(iterate.n_rows, iterate.n_cols); - arma::mat iterate_new(iterate.n_rows, iterate.n_cols); + arma::mat iterateNew(iterate.n_rows, iterate.n_cols); double gap = 0; for (size_t i=1; i != maxIterations; ++i) @@ -63,7 +63,7 @@ Optimize(FunctionType& function, arma::mat& iterate) function.Gradient(iterate, gradient); // Solve linear constrained problem, solution saved in s. - linear_constr_solver.Optimize(gradient, s); + linearConstrSolver.Optimize(gradient, s); // Check duality gap for return condition. gap = std::fabs(dot(iterate-s, gradient)); @@ -75,10 +75,10 @@ Optimize(FunctionType& function, arma::mat& iterate) } - // Update solution, save in iterate_new. - update_rule.Update(function, iterate, s, iterate_new, i); + // Update solution, save in iterateNew. + updateRule.Update(function, iterate, s, iterateNew, i); - iterate = std::move(iterate_new); + iterate = std::move(iterateNew); CurrentObjective = function.Evaluate(iterate); } Log::Info << "Frank Wolfe: maximum iterations (" << maxIterations diff --git a/src/mlpack/core/optimizers/fw/update_classic.hpp b/src/mlpack/core/optimizers/fw/update_classic.hpp index 19e1ce94ee6..e5ebae22415 100644 --- a/src/mlpack/core/optimizers/fw/update_classic.hpp +++ b/src/mlpack/core/optimizers/fw/update_classic.hpp @@ -40,20 +40,21 @@ class UpdateClassic * * \f$ x_{k+1} = (1-\gamma)x_k + \gamma s \f$, where \f$ \gamma = 2/(k+2) \f$ * - * @param old_coords previous solution coords. + * @param function function to be optimized, not used in this update rule. + * @param oldCoords previous solution coords. * @param s current linear_constr_solution result. * @param new_coords new output solution coords. * @param num_iter current iteration number */ template void Update(FunctionType& function, - const arma::mat& old_coords, + const arma::mat& oldCoords, const arma::mat& s, - arma::mat& new_coords, - const size_t num_iter) + arma::mat& newCoords, + const size_t numIter) { - double gamma = 2.0 / (num_iter + 2.0); - new_coords = (1.0 - gamma) * old_coords + gamma * s; + double gamma = 2.0 / (numIter + 2.0); + newCoords = (1.0 - gamma) * oldCoords + gamma * s; } }; diff --git a/src/mlpack/core/optimizers/fw/update_span.hpp b/src/mlpack/core/optimizers/fw/update_span.hpp index 5e7e28d75ad..615b5925e66 100644 --- a/src/mlpack/core/optimizers/fw/update_span.hpp +++ b/src/mlpack/core/optimizers/fw/update_span.hpp @@ -35,7 +35,8 @@ namespace optimization { * arma::vec Vectorb() * * - * @tparam FunctionType Objective function type to be minimized in FrankWolfe algorithm. + * @tparam FunctionType Objective function type to be minimized in + * FrankWolfe algorithm. */ class UpdateSpan { @@ -50,21 +51,21 @@ class UpdateSpan { /* Do nothing. */ } /** - * Update rule for FrankWolfe, reoptimize in the span of original solution space. - * + * Update rule for FrankWolfe, reoptimize in the span of original + * solution space. * * @tparam function function to be optimized. - * @param old_coords previous solution coords. - * @param s current linear_constr_solution result. - * @param new_coords new output solution coords. - * @param num_iter current iteration number + * @param oldCoords previous solution coords, not used in this update rule. + * @param s current linearConstrSolution result. + * @param newCoords new output solution coords. + * @param numIter current iteration number. */ template void Update(FunctionType& function, - const arma::mat& old_coords, + const arma::mat& oldCoords, const arma::mat& s, - arma::mat& new_coords, - const size_t num_iter) + arma::mat& newCoords, + const size_t numIter) { // Add atom here. arma::uvec ind = find(s, 1); @@ -72,27 +73,27 @@ class UpdateSpan AddAtom(function, d); arma::vec b = function.Vectorb(); - arma::mat x = solve(atoms_current, b); + arma::mat x = solve(currentAtoms, b); - new_coords = RecoverVector(function, x); + newCoords = RecoverVector(function, x); } //! Get the current atom indices. - arma::uvec CurrentIndices() const { return current_indices; } + arma::uvec CurrentIndices() const { return currentIndices; } //! Modify the current atom indices. - arma::uvec& CurrentIndices() { return current_indices; } + arma::uvec& CurrentIndices() { return currentIndices; } //! Get the current atoms. - arma::mat CurrentAtoms() const { return atoms_current; } + arma::mat CurrentAtoms() const { return currentAtoms; } //! Modify the current atoms. - arma::mat& CurrentAtoms() { return atoms_current; } + arma::mat& CurrentAtoms() { return currentAtoms; } private: //! Current indices. - arma::uvec current_indices; + arma::uvec currentIndices; //! Current atoms. - arma::mat atoms_current; + arma::mat currentAtoms; //! Flag current indices is empty. bool isEmpty = true; @@ -111,10 +112,10 @@ class UpdateSpan { arma::uvec vk(1); vk = k; - current_indices.insert_rows(0, vk); + currentIndices.insert_rows(0, vk); arma::mat atom = (function.MatrixA()).col(k); - atoms_current.insert_cols(0, atom); + currentAtoms.insert_cols(0, atom); } } @@ -124,9 +125,9 @@ class UpdateSpan int n = (function.MatrixA()).n_cols; arma::vec y = arma::zeros(n); - arma::uword len = current_indices.size(); + arma::uword len = currentIndices.size(); for (size_t ii = 0; ii < len; ++ii) - y(current_indices(ii)) = x(ii); + y(currentIndices(ii)) = x(ii); return y; } From d395596fe7b4c58e5f741b70fd22ca5da943a8c2 Mon Sep 17 00:00:00 2001 From: Chenzhe Diao Date: Sat, 8 Jul 2017 20:50:38 -0600 Subject: [PATCH 11/16] Added more comments. Removed the function template in UpdateSpan class since it only supports FuncSq class. --- .../core/optimizers/fw/constr_lpball.hpp | 2 +- src/mlpack/core/optimizers/fw/frank_wolfe.hpp | 27 +++++----- .../core/optimizers/fw/frank_wolfe_impl.hpp | 4 +- src/mlpack/core/optimizers/fw/func_sq.hpp | 37 ++++++++++++-- .../core/optimizers/fw/test_func_fw.hpp | 16 ++++++ .../core/optimizers/fw/update_classic.hpp | 4 +- src/mlpack/core/optimizers/fw/update_span.hpp | 51 ++++++++----------- 7 files changed, 91 insertions(+), 50 deletions(-) diff --git a/src/mlpack/core/optimizers/fw/constr_lpball.hpp b/src/mlpack/core/optimizers/fw/constr_lpball.hpp index 01f90563e68..7c55d59c78d 100644 --- a/src/mlpack/core/optimizers/fw/constr_lpball.hpp +++ b/src/mlpack/core/optimizers/fw/constr_lpball.hpp @@ -76,7 +76,7 @@ class ConstrLpBallSolver } else if (p == 1.0) { - // l1 ball, used in OMP. + // l1 ball, also used in OMP. arma::mat tmp = arma::abs(v); arma::uword k = tmp.index_max(); // linear index of matrix. s.zeros(v.n_rows, v.n_cols); diff --git a/src/mlpack/core/optimizers/fw/frank_wolfe.hpp b/src/mlpack/core/optimizers/fw/frank_wolfe.hpp index 459a6ed87e3..658ede8b246 100644 --- a/src/mlpack/core/optimizers/fw/frank_wolfe.hpp +++ b/src/mlpack/core/optimizers/fw/frank_wolfe.hpp @@ -51,22 +51,16 @@ namespace optimization { * g(x):= \max_{s\in D} \quad \leq \epsilon, * \f] * - * we also know that \f$ g(x) \geq f(x) - f(x^*) \f$, where \f$ x^* \f$ is the optimal + * we also know that \f$ g(x) \geq f(x) - f(x^*) \f$, where \f$ x^* \f$ is the optimal * solution. * * The parameter \f$ \epsilon \f$ is specified by the tolerance parameter to the * constructor. * - * For FrankWolfe to work, FunctionType, LinearConstrSolverType and UpdateRuleType + * For FrankWolfe to work, LinearConstrSolverType and UpdateRuleType * template parameters are required. * These classes must implement the following functions: * - * FunctionType: - * - * double Evaluate(const arma::mat& coordinates); - * void Gradient(const arma::mat& coordinates, - * arma::mat& gradient); - * * LinearConstrSolverType: * * void Optimize(const arma::mat& gradient, @@ -79,8 +73,6 @@ namespace optimization { * arma::mat& new_coords, * const size_t num_iter); * - * @tparam FunctionType Objective function type to be - * minimized. * @tparam LinearConstrSolverType Solver for the linear constrained problem. * @tparam UpdateRuleType Rule to update the solution in each iteration. * @@ -110,12 +102,19 @@ class FrankWolfe /** * Optimize the given function using FrankWolfe. The given starting - * point will be modified to store the finishing point of the algorithm, and + * point will be modified to store the finishing point of the algorithm, * the final objective value is returned. * - * @tparam function Function to be optimized. - * @param iterate Starting point (will be modified). - * @return Objective value of the final point. + * FunctionType template class must provide the following functions: + * + * double Evaluate(const arma::mat& coordinates); + * void Gradient(const arma::mat& coordinates, + * arma::mat& gradient); + * + * @param function Function to be optimized. + * @param iterate Input with starting point, and will be modified to save + * the output optimial solution coordinates. + * @return Objective value at the final solution. */ template double Optimize(FunctionType& function, arma::mat& iterate); diff --git a/src/mlpack/core/optimizers/fw/frank_wolfe_impl.hpp b/src/mlpack/core/optimizers/fw/frank_wolfe_impl.hpp index e0b3d3d0030..4a5847705e1 100644 --- a/src/mlpack/core/optimizers/fw/frank_wolfe_impl.hpp +++ b/src/mlpack/core/optimizers/fw/frank_wolfe_impl.hpp @@ -18,6 +18,7 @@ namespace mlpack { namespace optimization { +//! Constructor of the FrankWolfe class. template< typename LinearConstrSolverType, typename UpdateRuleType> @@ -41,7 +42,7 @@ template double FrankWolfe:: Optimize(FunctionType& function, arma::mat& iterate) { - // To keep track of the function value + // To keep track of the function value. double CurrentObjective = function.Evaluate(iterate); double PreviousObjective = DBL_MAX; @@ -81,6 +82,7 @@ Optimize(FunctionType& function, arma::mat& iterate) iterate = std::move(iterateNew); CurrentObjective = function.Evaluate(iterate); } + Log::Info << "Frank Wolfe: maximum iterations (" << maxIterations << ") reached; " << "terminating optimization." << std::endl; return CurrentObjective; diff --git a/src/mlpack/core/optimizers/fw/func_sq.hpp b/src/mlpack/core/optimizers/fw/func_sq.hpp index 81f90eb0f41..f926c7492dc 100644 --- a/src/mlpack/core/optimizers/fw/func_sq.hpp +++ b/src/mlpack/core/optimizers/fw/func_sq.hpp @@ -17,12 +17,30 @@ namespace mlpack { namespace optimization { +/** + * Square loss function \f$ f(x) = 0.5 * ||Ax - b||_2^2 \f$. + * + * Contains matrix \f$ A \f$ and vector \f$ b \f$. + */ class FuncSq { public: + /** + * Construct the square loss function. + * + * @param A matrix A. + * @param b vector b. + */ FuncSq(const arma::mat A, const arma::mat b) : A(A), b(b) {/* Nothing to do. */} + /** + * Evaluation of the function. + * \f$ f(x) = 0.5 * ||Ax - b||_2^2 \f$ + * + * @param coords vector x. + * @return \f$ f(x) \f$. + */ double Evaluate(const arma::mat& coords) { arma::vec r = A*coords - b; @@ -30,22 +48,35 @@ class FuncSq return y(0, 0); } + /** + * Gradient of square loss function. + * \f$ \nabla f(x) = A^T(Ax - b) \f$ + * + * @param coords input vector x. + * @param gradient output gradient vector. + */ void Gradient(const arma::mat& coords, arma::mat& gradient) { arma::vec r = A*coords - b; gradient = A.t() * r; } - + //! Get the matrix A. arma::mat MatrixA() const {return A;} + //! Modify the matrix A. arma::mat& MatrixA() {return A;} + //! Get the vector b. arma::vec Vectorb() const { return b; } + //! Modify the vector b. arma::vec& Vectorb() { return b; } private: - arma::mat A; // matrix - arma::vec b; // vector + //! Matrix A in square loss function. + arma::mat A; + + //! Vector b in square loss function. + arma::vec b; }; } // namespace optimization diff --git a/src/mlpack/core/optimizers/fw/test_func_fw.hpp b/src/mlpack/core/optimizers/fw/test_func_fw.hpp index 6617b50deae..c5a3c045548 100644 --- a/src/mlpack/core/optimizers/fw/test_func_fw.hpp +++ b/src/mlpack/core/optimizers/fw/test_func_fw.hpp @@ -19,12 +19,22 @@ namespace mlpack { namespace optimization { +/** + * Simple test function for classic Frank Wolfe Algorithm: + * + * \f$ f(x) = (x1 - 0.1)^2 + (x2 - 0.2)^2 + (x3 - 0.3)^2 \f$. + */ class TestFuncFW { public: TestFuncFW() {/* Nothing to do. */} + /** + * Evaluation of the function. + * + * @param coords input vector x. + */ double Evaluate(const arma::mat& coords) { double f = std::pow(coords[0]-0.1, 2); @@ -33,6 +43,12 @@ class TestFuncFW return f; } + /** + * Gradient of the function. + * + * @param coords input vector x. + * @param gradient output gradient vector. + */ void Gradient(const arma::mat& coords, arma::mat& gradient) { gradient[0] = coords[0]-0.1; diff --git a/src/mlpack/core/optimizers/fw/update_classic.hpp b/src/mlpack/core/optimizers/fw/update_classic.hpp index e5ebae22415..55166bc507b 100644 --- a/src/mlpack/core/optimizers/fw/update_classic.hpp +++ b/src/mlpack/core/optimizers/fw/update_classic.hpp @@ -43,8 +43,8 @@ class UpdateClassic * @param function function to be optimized, not used in this update rule. * @param oldCoords previous solution coords. * @param s current linear_constr_solution result. - * @param new_coords new output solution coords. - * @param num_iter current iteration number + * @param newCoords output new solution coords. + * @param numIter current iteration number */ template void Update(FunctionType& function, diff --git a/src/mlpack/core/optimizers/fw/update_span.hpp b/src/mlpack/core/optimizers/fw/update_span.hpp index 615b5925e66..9a1889e5a48 100644 --- a/src/mlpack/core/optimizers/fw/update_span.hpp +++ b/src/mlpack/core/optimizers/fw/update_span.hpp @@ -14,29 +14,17 @@ #define MLPACK_CORE_OPTIMIZERS_FW_UPDATE_SPAN_HPP #include +#include "func_sq.hpp" namespace mlpack { namespace optimization { /** - * Recalculate the optimal solution in the span of all previous solution space, + * Recalculate the optimal solution in the span of all previous solution space, * used as update step for FrankWolfe algorithm. * + * Currently only works for function in FuncSq class. * - * For UpdateSpan to work, FunctionType template parameters are required. - * This class must implement the following functions: - * - * FunctionType: - * - * double Evaluate(const arma::mat& coordinates); - * void Gradient(const arma::mat& coordinates, - * arma::mat& gradient); - * arma::mat MatrixA() - * arma::vec Vectorb() - * - * - * @tparam FunctionType Objective function type to be minimized in - * FrankWolfe algorithm. */ class UpdateSpan { @@ -51,31 +39,33 @@ class UpdateSpan { /* Do nothing. */ } /** - * Update rule for FrankWolfe, reoptimize in the span of original + * Update rule for FrankWolfe, reoptimize in the span of current * solution space. * - * @tparam function function to be optimized. + * @param function function to be optimized. * @param oldCoords previous solution coords, not used in this update rule. * @param s current linearConstrSolution result. - * @param newCoords new output solution coords. + * @param newCoords output new solution coords. * @param numIter current iteration number. */ - template - void Update(FunctionType& function, + void Update(FuncSq& function, const arma::mat& oldCoords, const arma::mat& s, arma::mat& newCoords, const size_t numIter) { - // Add atom here. + // Add new atom into soluton space. arma::uvec ind = find(s, 1); arma::uword d = ind(0); AddAtom(function, d); + // Reoptimize the solution in the current space. arma::vec b = function.Vectorb(); arma::mat x = solve(currentAtoms, b); - newCoords = RecoverVector(function, x); + // x has coords of only the current atoms, recover the solution + // to the original size. + newCoords = RecoverVector((function.MatrixA()).n_cols, x); } //! Get the current atom indices. @@ -89,18 +79,17 @@ class UpdateSpan arma::mat& CurrentAtoms() { return currentAtoms; } private: - //! Current indices. + //! Indices of the atoms in the current solution space. arma::uvec currentIndices; - //! Current atoms. + //! Current atoms in the solution space, ordered as currentIndices. arma::mat currentAtoms; //! Flag current indices is empty. bool isEmpty = true; //! Add atom into the solution space. - template - void AddAtom(FunctionType& function, const arma::uword k) + void AddAtom(FuncSq& function, const arma::uword k) { if (isEmpty) { @@ -119,10 +108,14 @@ class UpdateSpan } } - template - arma::vec RecoverVector(FunctionType& function, const arma::vec& x) + /** + * Recover the solution coordinate from the coefficients of current atoms. + * + * @param n dimension of original solution space. + * @param x coefficients of current atoms. + */ + arma::vec RecoverVector(const size_t n, const arma::vec& x) { - int n = (function.MatrixA()).n_cols; arma::vec y = arma::zeros(n); arma::uword len = currentIndices.size(); From 19f14d8fc5be7eae371ce5771b06fcc8bfe9dd1b Mon Sep 17 00:00:00 2001 From: Chenzhe Diao Date: Wed, 12 Jul 2017 14:37:09 -0600 Subject: [PATCH 12/16] Add comment for Sign() function. --- src/mlpack/core/math/lin_alg.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/mlpack/core/math/lin_alg.hpp b/src/mlpack/core/math/lin_alg.hpp index eba80dcad39..07b9bf720de 100644 --- a/src/mlpack/core/math/lin_alg.hpp +++ b/src/mlpack/core/math/lin_alg.hpp @@ -129,7 +129,7 @@ void SymKronId(const arma::mat& A, arma::mat& op); * Return 1 if x>0; return 0 if x=0; return -1 if x<0. * Return type are the same as input type. * - * @param x + * @param x Number of any type. */ template T Sign(const T x) From 7a4b8c4b762bf6196b8445d78d5004b4b8af91ed Mon Sep 17 00:00:00 2001 From: Chenzhe Diao Date: Sat, 15 Jul 2017 20:51:15 -0600 Subject: [PATCH 13/16] Add document for OMP, and fix some stype problems. --- src/mlpack/core/optimizers/fw/frank_wolfe.hpp | 23 ++++++++++++------- .../core/optimizers/fw/frank_wolfe_impl.hpp | 14 +++++------ src/mlpack/core/optimizers/fw/func_sq.hpp | 4 ++-- src/mlpack/core/optimizers/fw/update_span.hpp | 4 ++-- 4 files changed, 26 insertions(+), 19 deletions(-) diff --git a/src/mlpack/core/optimizers/fw/frank_wolfe.hpp b/src/mlpack/core/optimizers/fw/frank_wolfe.hpp index 658ede8b246..14ff32c5200 100644 --- a/src/mlpack/core/optimizers/fw/frank_wolfe.hpp +++ b/src/mlpack/core/optimizers/fw/frank_wolfe.hpp @@ -43,21 +43,22 @@ namespace optimization { * \f] * * - * The algorithm continues until \f$ k \f$ reaches the maximum number of iterations, - * or when the duality gap is bounded by a certain tolerance \f$ \epsilon \f$. + * The algorithm continues until \f$ k \f$ reaches the maximum number of + * iterations, or when the duality gap is bounded by a certain tolerance + * \f$ \epsilon \f$. * That is, * * \f[ * g(x):= \max_{s\in D} \quad \leq \epsilon, * \f] * - * we also know that \f$ g(x) \geq f(x) - f(x^*) \f$, where \f$ x^* \f$ is the optimal - * solution. + * we also know that \f$ g(x) \geq f(x) - f(x^*) \f$, where \f$ x^* \f$ is the + * optimal solution. * * The parameter \f$ \epsilon \f$ is specified by the tolerance parameter to the * constructor. * - * For FrankWolfe to work, LinearConstrSolverType and UpdateRuleType + * For FrankWolfe to work, LinearConstrSolverType and UpdateRuleType * template parameters are required. * These classes must implement the following functions: * @@ -120,13 +121,13 @@ class FrankWolfe double Optimize(FunctionType& function, arma::mat& iterate); //! Get the linear constrained solver. - LinearConstrSolverType LinearConstrSolver() + const LinearConstrSolverType& LinearConstrSolver() const { return linearConstrSolver; } //! Modify the linear constrained solver. LinearConstrSolverType& LinearConstrSolver() { return linearConstrSolver; } //! Get the update rule. - UpdateRuleType UpdateRule() const { return updateRule; } + const UpdateRuleType& UpdateRule() const { return updateRule; } //! Modify the update rule. UpdateRuleType& UpdateRule() { return updateRule; } @@ -154,7 +155,13 @@ class FrankWolfe double tolerance; }; -//! Orthogonal Matching Pursuit. +/** + * Orthogonal Matching Pursuit. It is a sparse approximation algorithm which + * involves finding the "best matching" projections of multidimensional data + * onto the span of an over-complete dictionary. To use it, the dictionary is + * input as the columns of MatrixA() in FuncSq class, and the vector to be + * approximated is input as the Vectorb() in FuncSq class. + */ using OMP = FrankWolfe; } // namespace optimization diff --git a/src/mlpack/core/optimizers/fw/frank_wolfe_impl.hpp b/src/mlpack/core/optimizers/fw/frank_wolfe_impl.hpp index 4a5847705e1..c9650483c34 100644 --- a/src/mlpack/core/optimizers/fw/frank_wolfe_impl.hpp +++ b/src/mlpack/core/optimizers/fw/frank_wolfe_impl.hpp @@ -43,8 +43,8 @@ double FrankWolfe:: Optimize(FunctionType& function, arma::mat& iterate) { // To keep track of the function value. - double CurrentObjective = function.Evaluate(iterate); - double PreviousObjective = DBL_MAX; + double currentObjective = function.Evaluate(iterate); + double previousObjective = DBL_MAX; arma::mat gradient(iterate.n_rows, iterate.n_cols); arma::mat s(iterate.n_rows, iterate.n_cols); @@ -55,10 +55,10 @@ Optimize(FunctionType& function, arma::mat& iterate) { // Output current objective function. Log::Info << "Iteration " << i << ", objective " - << CurrentObjective << "." << std::endl; + << currentObjective << "." << std::endl; // Reset counter variables. - PreviousObjective = CurrentObjective; + previousObjective = currentObjective; // Calculate the gradient. function.Gradient(iterate, gradient); @@ -72,7 +72,7 @@ Optimize(FunctionType& function, arma::mat& iterate) { Log::Info << "FrankWolfe: minimized within tolerance " << tolerance << "; " << "terminating optimization." << std::endl; - return CurrentObjective; + return currentObjective; } @@ -80,12 +80,12 @@ Optimize(FunctionType& function, arma::mat& iterate) updateRule.Update(function, iterate, s, iterateNew, i); iterate = std::move(iterateNew); - CurrentObjective = function.Evaluate(iterate); + currentObjective = function.Evaluate(iterate); } Log::Info << "Frank Wolfe: maximum iterations (" << maxIterations << ") reached; " << "terminating optimization." << std::endl; - return CurrentObjective; + return currentObjective; } diff --git a/src/mlpack/core/optimizers/fw/func_sq.hpp b/src/mlpack/core/optimizers/fw/func_sq.hpp index f926c7492dc..baa21cf75b7 100644 --- a/src/mlpack/core/optimizers/fw/func_sq.hpp +++ b/src/mlpack/core/optimizers/fw/func_sq.hpp @@ -31,7 +31,7 @@ class FuncSq * @param A matrix A. * @param b vector b. */ - FuncSq(const arma::mat A, const arma::mat b) : A(A), b(b) + FuncSq(const arma::mat& A, const arma::vec& b) : A(A), b(b) {/* Nothing to do. */} /** @@ -43,7 +43,7 @@ class FuncSq */ double Evaluate(const arma::mat& coords) { - arma::vec r = A*coords - b; + arma::vec r = A * coords - b; arma::mat y = (r.t() * r) * 0.5; return y(0, 0); } diff --git a/src/mlpack/core/optimizers/fw/update_span.hpp b/src/mlpack/core/optimizers/fw/update_span.hpp index 9a1889e5a48..c0b4687a7fd 100644 --- a/src/mlpack/core/optimizers/fw/update_span.hpp +++ b/src/mlpack/core/optimizers/fw/update_span.hpp @@ -69,12 +69,12 @@ class UpdateSpan } //! Get the current atom indices. - arma::uvec CurrentIndices() const { return currentIndices; } + const arma::uvec& CurrentIndices() const { return currentIndices; } //! Modify the current atom indices. arma::uvec& CurrentIndices() { return currentIndices; } //! Get the current atoms. - arma::mat CurrentAtoms() const { return currentAtoms; } + const arma::mat& CurrentAtoms() const { return currentAtoms; } //! Modify the current atoms. arma::mat& CurrentAtoms() { return currentAtoms; } From 918cfb332dc51836e9214ebdd647fdd4349a8f1f Mon Sep 17 00:00:00 2001 From: Chenzhe Diao Date: Sat, 15 Jul 2017 20:58:41 -0600 Subject: [PATCH 14/16] Fix style issue. --- src/mlpack/core/optimizers/fw/func_sq.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/mlpack/core/optimizers/fw/func_sq.hpp b/src/mlpack/core/optimizers/fw/func_sq.hpp index baa21cf75b7..6065d383a22 100644 --- a/src/mlpack/core/optimizers/fw/func_sq.hpp +++ b/src/mlpack/core/optimizers/fw/func_sq.hpp @@ -57,7 +57,7 @@ class FuncSq */ void Gradient(const arma::mat& coords, arma::mat& gradient) { - arma::vec r = A*coords - b; + arma::vec r = A * coords - b; gradient = A.t() * r; } From 4baa16e3a9dec5e6313145be4ffcb969c2499f8e Mon Sep 17 00:00:00 2001 From: Chenzhe Diao Date: Sun, 16 Jul 2017 15:02:09 -0600 Subject: [PATCH 15/16] .index_max() function is changed to .max(k) function. --- src/mlpack/core/optimizers/fw/constr_lpball.hpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/mlpack/core/optimizers/fw/constr_lpball.hpp b/src/mlpack/core/optimizers/fw/constr_lpball.hpp index 7c55d59c78d..f3cbf8ee1e8 100644 --- a/src/mlpack/core/optimizers/fw/constr_lpball.hpp +++ b/src/mlpack/core/optimizers/fw/constr_lpball.hpp @@ -78,7 +78,8 @@ class ConstrLpBallSolver { // l1 ball, also used in OMP. arma::mat tmp = arma::abs(v); - arma::uword k = tmp.index_max(); // linear index of matrix. + arma::uword k; + tmp.max(k); // k is the linear index of the largest element. s.zeros(v.n_rows, v.n_cols); s(k) = - mlpack::math::Sign(v(k)); } From 61a5864ba45fb675893d602d9d0f6809f7713853 Mon Sep 17 00:00:00 2001 From: Chenzhe Diao Date: Mon, 17 Jul 2017 16:31:40 -0600 Subject: [PATCH 16/16] RecoverVector() changed to output as reference. --- .../core/optimizers/fw/frank_wolfe_impl.hpp | 8 ++---- .../core/optimizers/fw/test_func_fw.hpp | 12 ++++----- src/mlpack/core/optimizers/fw/update_span.hpp | 25 +++++++---------- src/mlpack/tests/frankwolfe_test.cpp | 27 ++++++++++++------- 4 files changed, 34 insertions(+), 38 deletions(-) diff --git a/src/mlpack/core/optimizers/fw/frank_wolfe_impl.hpp b/src/mlpack/core/optimizers/fw/frank_wolfe_impl.hpp index c9650483c34..abc84dffe23 100644 --- a/src/mlpack/core/optimizers/fw/frank_wolfe_impl.hpp +++ b/src/mlpack/core/optimizers/fw/frank_wolfe_impl.hpp @@ -44,22 +44,18 @@ Optimize(FunctionType& function, arma::mat& iterate) { // To keep track of the function value. double currentObjective = function.Evaluate(iterate); - double previousObjective = DBL_MAX; arma::mat gradient(iterate.n_rows, iterate.n_cols); arma::mat s(iterate.n_rows, iterate.n_cols); arma::mat iterateNew(iterate.n_rows, iterate.n_cols); double gap = 0; - for (size_t i=1; i != maxIterations; ++i) + for (size_t i = 1; i != maxIterations; ++i) { // Output current objective function. Log::Info << "Iteration " << i << ", objective " << currentObjective << "." << std::endl; - // Reset counter variables. - previousObjective = currentObjective; - // Calculate the gradient. function.Gradient(iterate, gradient); @@ -67,7 +63,7 @@ Optimize(FunctionType& function, arma::mat& iterate) linearConstrSolver.Optimize(gradient, s); // Check duality gap for return condition. - gap = std::fabs(dot(iterate-s, gradient)); + gap = std::fabs(dot(iterate - s, gradient)); if (gap < tolerance) { Log::Info << "FrankWolfe: minimized within tolerance " diff --git a/src/mlpack/core/optimizers/fw/test_func_fw.hpp b/src/mlpack/core/optimizers/fw/test_func_fw.hpp index c5a3c045548..305cad3a34a 100644 --- a/src/mlpack/core/optimizers/fw/test_func_fw.hpp +++ b/src/mlpack/core/optimizers/fw/test_func_fw.hpp @@ -37,9 +37,9 @@ class TestFuncFW */ double Evaluate(const arma::mat& coords) { - double f = std::pow(coords[0]-0.1, 2); - f += std::pow(coords[1]-0.2, 2); - f += std::pow(coords[2]-0.3, 2); + double f = std::pow(coords[0] - 0.1, 2); + f += std::pow(coords[1] - 0.2, 2); + f += std::pow(coords[2] - 0.3, 2); return f; } @@ -51,9 +51,9 @@ class TestFuncFW */ void Gradient(const arma::mat& coords, arma::mat& gradient) { - gradient[0] = coords[0]-0.1; - gradient[1] = coords[1]-0.2; - gradient[2] = coords[2]-0.3; + gradient[0] = coords[0] - 0.1; + gradient[1] = coords[1] - 0.2; + gradient[2] = coords[2] - 0.3; } }; diff --git a/src/mlpack/core/optimizers/fw/update_span.hpp b/src/mlpack/core/optimizers/fw/update_span.hpp index c0b4687a7fd..a01c7bb93de 100644 --- a/src/mlpack/core/optimizers/fw/update_span.hpp +++ b/src/mlpack/core/optimizers/fw/update_span.hpp @@ -56,16 +56,15 @@ class UpdateSpan { // Add new atom into soluton space. arma::uvec ind = find(s, 1); - arma::uword d = ind(0); - AddAtom(function, d); + AddAtom(function, ind(0)); // Reoptimize the solution in the current space. arma::vec b = function.Vectorb(); - arma::mat x = solve(currentAtoms, b); + arma::vec x = solve(currentAtoms, b); // x has coords of only the current atoms, recover the solution // to the original size. - newCoords = RecoverVector((function.MatrixA()).n_cols, x); + RecoverVector(x, function.MatrixA().n_cols, newCoords); } //! Get the current atom indices. @@ -85,17 +84,13 @@ class UpdateSpan //! Current atoms in the solution space, ordered as currentIndices. arma::mat currentAtoms; - //! Flag current indices is empty. - bool isEmpty = true; - //! Add atom into the solution space. void AddAtom(FuncSq& function, const arma::uword k) { - if (isEmpty) + if (currentIndices.is_empty()) { CurrentIndices() = k; CurrentAtoms() = (function.MatrixA()).col(k); - isEmpty = false; } else { @@ -111,18 +106,16 @@ class UpdateSpan /** * Recover the solution coordinate from the coefficients of current atoms. * - * @param n dimension of original solution space. - * @param x coefficients of current atoms. + * @param x input coefficients of current atoms. + * @param n dimension of the recovered vector. + * @param y output recovered vector. */ - arma::vec RecoverVector(const size_t n, const arma::vec& x) + void RecoverVector(const arma::vec& x, const size_t n, arma::mat& y) { - arma::vec y = arma::zeros(n); - + y.zeros(n, 1); arma::uword len = currentIndices.size(); for (size_t ii = 0; ii < len; ++ii) y(currentIndices(ii)) = x(ii); - - return y; } }; diff --git a/src/mlpack/tests/frankwolfe_test.cpp b/src/mlpack/tests/frankwolfe_test.cpp index 9451c5ad0e3..7dbad719f14 100644 --- a/src/mlpack/tests/frankwolfe_test.cpp +++ b/src/mlpack/tests/frankwolfe_test.cpp @@ -2,7 +2,7 @@ * @file frankwolfe_test.cpp * @author Chenzhe Diao * - * Test file for Frank-Wolfe optimizer. + * Test file for Frank-Wolfe type optimizer. * * mlpack is free software; you may redistribute it and/or modify it under the * terms of the 3-clause BSD license. You should have received a copy of the @@ -27,13 +27,16 @@ using namespace mlpack::optimization; BOOST_AUTO_TEST_SUITE(FrankWolfeTest); +/** + * Simple test of Orthogonal Matching Pursuit algorithm. + */ BOOST_AUTO_TEST_CASE(OMPTest) { int k = 5; mat B1 = eye(3, 3); mat B2 = 0.1 * randn(3, k); - mat A = join_horiz(B1, B2); - vec b("1; 1; 0"); + mat A = join_horiz(B1, B2); // The dictionary is input as columns of A. + vec b("1; 1; 0"); // Vector to be sparsely approximated. FuncSq f(A, b); ConstrLpBallSolver linear_constr_solver(1); @@ -45,19 +48,23 @@ BOOST_AUTO_TEST_CASE(OMPTest) double result = s.Optimize(f, coordinates); BOOST_REQUIRE_SMALL(result, 1e-10); - BOOST_REQUIRE_SMALL(coordinates[0]-1, 1e-10); - BOOST_REQUIRE_SMALL(coordinates[1]-1, 1e-10); + BOOST_REQUIRE_SMALL(coordinates[0] - 1, 1e-10); + BOOST_REQUIRE_SMALL(coordinates[1] - 1, 1e-10); BOOST_REQUIRE_SMALL(coordinates[2], 1e-10); for (int ii = 0; ii < k; ++ii) { - BOOST_REQUIRE_SMALL(coordinates[ii+3], 1e-10); + BOOST_REQUIRE_SMALL(coordinates[ii + 3], 1e-10); } } +/** + * A very simple test of classic Frank-Wolfe algorithm. + * The constrained domain used is unit lp ball. + */ BOOST_AUTO_TEST_CASE(ClassicFW) { TestFuncFW f; - double p = 1; // constraint set is unit lp ball + double p = 1; // Constraint set is unit lp ball. ConstrLpBallSolver linear_constr_solver(p); UpdateClassic update_rule; @@ -68,9 +75,9 @@ BOOST_AUTO_TEST_CASE(ClassicFW) double result = s.Optimize(f, coordinates); BOOST_REQUIRE_SMALL(result, 1e-4); - BOOST_REQUIRE_SMALL(coordinates[0]-0.1, 1e-4); - BOOST_REQUIRE_SMALL(coordinates[1]-0.2, 1e-4); - BOOST_REQUIRE_SMALL(coordinates[2]-0.3, 1e-4); + BOOST_REQUIRE_SMALL(coordinates[0] - 0.1, 1e-4); + BOOST_REQUIRE_SMALL(coordinates[1] - 0.2, 1e-4); + BOOST_REQUIRE_SMALL(coordinates[2] - 0.3, 1e-4); }