Merge pull request #946 from accord-net/feature/GH-820

GH-820: Augmented Lagrangian to support linear constraints

Sources/Accord.Math/Accord.Math.csproj

@@ -217,6 +217,7 @@
     <Compile Include="Matrix\Vector.MinMax.cs" />
     <Compile Include="Matrix\Matrix.MinMax.cs" />
     <Compile Include="Optimization\Base\IFunctionOptimizationMethod.cs" />
+    <Compile Include="Optimization\Constrained\Constraints\ConstraintExtensions.cs" />
     <Compile Include="Optimization\Losses\BinaryCrossEntropyLoss.cs" />
     <Compile Include="Optimization\Losses\CategoryCrossEntropyLoss.cs" />
     <Compile Include="Optimization\Losses\HammingLoss.cs" />

Sources/Accord.Math/Optimization/Constrained/AugmentedLagrangian.cs

@@ -56,7 +56,7 @@ public enum AugmentedLagrangianStatus
 
         /// <summary>
         ///   The optimization could not make progress towards finding a feasible
-        ///   solution. Try increasing the <see cref="NonlinearConstraint.Tolerance"/>
+        ///   solution. Try increasing the <see cref="IConstraint.Tolerance"/>
         ///   of the constraints.
         /// </summary>
         /// 
@@ -115,9 +115,9 @@ public class AugmentedLagrangian : BaseGradientOptimizationMethod, IGradientOpti
 
         IGradientOptimizationMethod dualSolver;
 
-        NonlinearConstraint[] lesserThanConstraints;
-        NonlinearConstraint[] greaterThanConstraints;
-        NonlinearConstraint[] equalityConstraints;
+        IConstraint[] lesserThanConstraints;
+        IConstraint[] greaterThanConstraints;
+        IConstraint[] equalityConstraints;
 
 
         private double rho;
@@ -206,9 +206,9 @@ public int MaxEvaluations
         /// 
         /// <param name="numberOfVariables">The number of free parameters in the optimization problem.</param>
         /// <param name="constraints">
-        ///   The <see cref="NonlinearConstraint"/>s to which the solution must be subjected.</param>
+        ///   The <see cref="IConstraint"/>s to which the solution must be subjected.</param>
         /// 
-        public AugmentedLagrangian(int numberOfVariables, IEnumerable<NonlinearConstraint> constraints)
+        public AugmentedLagrangian(int numberOfVariables, IEnumerable<IConstraint> constraints)
             : base(numberOfVariables)
         {
             init(null, constraints, null);
@@ -220,9 +220,9 @@ public AugmentedLagrangian(int numberOfVariables, IEnumerable<NonlinearConstrain
         /// 
         /// <param name="function">The objective function to be optimized.</param>
         /// <param name="constraints">
-        ///   The <see cref="NonlinearConstraint"/>s to which the solution must be subjected.</param>
+        ///   The <see cref="IConstraint"/>s to which the solution must be subjected.</param>
         /// 
-        public AugmentedLagrangian(NonlinearObjectiveFunction function, IEnumerable<NonlinearConstraint> constraints)
+        public AugmentedLagrangian(NonlinearObjectiveFunction function, IEnumerable<IConstraint> constraints)
             : base(function.NumberOfVariables)
         {
             init(function, constraints, null);
@@ -237,10 +237,10 @@ public AugmentedLagrangian(NonlinearObjectiveFunction function, IEnumerable<Nonl
         ///   problem.</param>
         /// <param name="function">The objective function to be optimized.</param>
         /// <param name="constraints">
-        ///   The <see cref="NonlinearConstraint"/>s to which the solution must be subjected.</param>
+        ///   The <see cref="IConstraint"/>s to which the solution must be subjected.</param>
         /// 
         public AugmentedLagrangian(IGradientOptimizationMethod innerSolver,
-            NonlinearObjectiveFunction function, IEnumerable<NonlinearConstraint> constraints)
+            NonlinearObjectiveFunction function, IEnumerable<IConstraint> constraints)
             : base(innerSolver.NumberOfVariables)
         {
             if (innerSolver.NumberOfVariables != function.NumberOfVariables)
@@ -258,17 +258,17 @@ public AugmentedLagrangian(NonlinearObjectiveFunction function, IEnumerable<Nonl
         ///   optimization method</see> used internally to solve the dual of this optimization 
         ///   problem.</param>
         /// <param name="constraints">
-        ///   The <see cref="NonlinearConstraint"/>s to which the solution must be subjected.</param>
+        ///   The <see cref="IConstraint"/>s to which the solution must be subjected.</param>
         /// 
-        public AugmentedLagrangian(IGradientOptimizationMethod innerSolver, IEnumerable<NonlinearConstraint> constraints)
+        public AugmentedLagrangian(IGradientOptimizationMethod innerSolver, IEnumerable<IConstraint> constraints)
             : base(innerSolver.NumberOfVariables)
         {
             init(null, constraints, innerSolver);
         }
 
 
         private void init(NonlinearObjectiveFunction function,
-            IEnumerable<NonlinearConstraint> constraints, IGradientOptimizationMethod innerSolver)
+            IEnumerable<IConstraint> constraints, IGradientOptimizationMethod innerSolver)
         {
             if (function != null)
             {
@@ -294,9 +294,9 @@ public AugmentedLagrangian(IGradientOptimizationMethod innerSolver, IEnumerable<
                 };
             }
 
-            var equality = new List<NonlinearConstraint>();
-            var lesserThan = new List<NonlinearConstraint>();
-            var greaterThan = new List<NonlinearConstraint>();
+            var equality = new List<IConstraint>();
+            var lesserThan = new List<IConstraint>();
+            var greaterThan = new List<IConstraint>();
 
             foreach (var c in constraints)
             {

Sources/Accord.Math/Optimization/Constrained/Constraints/ConstraintExtensions.cs

@@ -0,0 +1,77 @@
+// Accord Math Library
+// The Accord.NET Framework
+// http://accord-framework.net
+//
+// Copyright © César Souza, 2009-2017
+// cesarsouza at gmail.com
+//
+//    This library is free software; you can redistribute it and/or
+//    modify it under the terms of the GNU Lesser General Public
+//    License as published by the Free Software Foundation; either
+//    version 2.1 of the License, or (at your option) any later version.
+//
+//    This library is distributed in the hope that it will be useful,
+//    but WITHOUT ANY WARRANTY; without even the implied warranty of
+//    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+//    Lesser General Public License for more details.
+//
+//    You should have received a copy of the GNU Lesser General Public
+//    License along with this library; if not, write to the Free Software
+//    Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+//
+
+namespace Accord.Math.Optimization
+{
+    using System;
+
+    /// <summary>
+    /// Extension methods on the <see cref="IConstraint"/> interface.
+    /// </summary>
+    public static class ConstraintExtensions
+    {
+
+        /// <summary>
+        /// Gets how much the constraint is being violated.
+        /// </summary>
+        /// <param name="constraint">The constraint.</param>
+        /// <param name="input">The function point.</param>
+        /// <returns>
+        /// How much the constraint is being violated at the given point. Positive
+        /// value means the constraint is not being violated with the returned slack,
+        /// while a negative value means the constraint is being violated by the returned
+        /// amount.
+        /// </returns>
+        public static double GetViolation(this IConstraint constraint, double[] input)
+        {
+            double fx = constraint.Function(input);
+
+            switch (constraint.ShouldBe)
+            {
+                case ConstraintType.EqualTo:
+                    return Math.Abs(fx - constraint.Value);
+
+                case ConstraintType.GreaterThanOrEqualTo:
+                    return fx - constraint.Value;
+
+                case ConstraintType.LesserThanOrEqualTo:
+                    return constraint.Value - fx;
+            }
+
+            throw new NotSupportedException();
+        }
+
+        /// <summary>
+        /// Gets whether this constraint is being violated
+        /// (within the current tolerance threshold).
+        /// </summary>
+        /// <param name="constraint">The constraint.</param>
+        /// <param name="input">The function point.</param>
+        /// <returns>
+        /// True if the constraint is being violated, false otherwise.
+        /// </returns>
+        public static bool IsViolated(this IConstraint constraint, double[] input)
+        {
+            return constraint.GetViolation(input) + constraint.Tolerance < 0;
+        }
+    }
+}

Sources/Accord.Math/Optimization/Constrained/Constraints/IConstraint.cs

@@ -22,12 +22,10 @@
 
 namespace Accord.Math.Optimization
 {
-    using System;
-    using System.Collections.Generic;
-    using System.Linq;
-    using System.Text;
-
-    interface IConstraint
+    /// <summary>
+    /// Defines an interface for an optimization constraint.
+    /// </summary>
+    public interface IConstraint
     {
         /// <summary>
         ///   Gets the type of the constraint.
@@ -40,41 +38,36 @@ interface IConstraint
         ///   side of the constraint equation.
         /// </summary>
         /// 
-        double Value { get; }
+        double Value { get; }
+
+        /// <summary>
+        ///   Gets the violation tolerance for the constraint.
+        /// </summary>
+        /// 
+        double Tolerance { get; }
 
         /// <summary>
         ///   Gets the number of variables in the constraint.
         /// </summary>
         /// 
-        int NumberOfVariables { get; }
-
-        /// <summary>
-        ///   Gets the left hand side of 
-        ///   the constraint equation.
-        /// </summary>
-        /// 
-        Func<double[], double> Function { get; }
+        int NumberOfVariables { get; }
+
+        /// <summary>
+        /// Calculates the left hand side of the constraint
+        /// equation given a vector x.
+        /// </summary>
+        /// <param name="x">The vector.</param>
+        /// <returns>
+        /// The left hand side of the constraint equation as evaluated at x.
+        /// </returns>
+        double Function(double[] x);
 
-        /// <summary>
-        ///   Gets the gradient of the left hand
-        ///   side of the constraint equation.
-        /// </summary>
-        /// 
-        Func<double[], double[]> Gradient { get; }
-
-        /// <summary>
-        ///   Gets how much the constraint is being violated.
-        /// </summary>
-        /// 
-        /// <param name="input">The function point.</param>
-        /// 
-        /// <returns>
-        ///   How much the constraint is being violated at the given point. Positive
-        ///   value means the constraint is not being violated with the returned slack, 
-        ///   while a negative value means the constraint is being violated by the returned
-        ///   amount.
-        /// </returns>
-        /// 
-        double GetViolation(double[] input);
-    }
+        /// <summary>
+        /// Calculates the gradient of the constraint
+        /// equation given a vector x
+        /// </summary>
+        /// <param name="x">The vector.</param>
+        /// <returns>The gradient of the constraint as evaluated at x.</returns>
+        double[] Gradient(double[] x);
+    }
 }