Restructuring and cleanup to prepare for open-sourcing.

SwiftCMAES/AppDelegate.swift → App/AppDelegate.swift

@@ -11,24 +11,23 @@ import Cocoa
 @NSApplicationMain
 class AppDelegate: NSObject, NSApplicationDelegate {
 
-
-
 	func applicationDidFinishLaunching(_ aNotification: Notification) {
-		// Insert code here to initialize your application
-
-
 
 		// Solution variables.
-//		let startSolution = Vector(repeating: 0.5, count: 10)
 		let startRangeBound: Double = 5.0
-		let startSolution = [Double.random(in: -startRangeBound...startRangeBound), Double.random(in: -startRangeBound...startRangeBound), Double.random(in: -startRangeBound...startRangeBound)]
+		let startSolution = [
+			Double.random(in: -startRangeBound...startRangeBound),
+			Double.random(in: -startRangeBound...startRangeBound),
+			Double.random(in: -startRangeBound...startRangeBound)
+		]
 		var fitness = AckleyObjectiveEvaluator()
 
 		// Hyperparameters.
-		let populationSize = Int(4+floor(3*log(Double(startSolution.count))))
+		let populationSize = CMAES.populationSize(forDimensions: startSolution.count)
 		let varSpan = startRangeBound * 2 // (max - min) of all inputs.
-		let stepSigma = 0.3 * varSpan // from Kaitlin.
+		let stepSigma = 0.3 * varSpan // from Kaitlin Maile.
 
+		// Perform CMA-ES.
 		let cmaes = CMAES(startSolution: startSolution, populationSize: populationSize, stepSigma: stepSigma)
 		var solution: Vector?
 		var solutionFitness: Double?
@@ -46,20 +45,14 @@ class AppDelegate: NSObject, NSApplicationDelegate {
 			print("\(i):   \(cmaes.bestSolution!.1)")
 		}
 
-
+		// Print solution.
 		if let solution = solution, let fitness = solutionFitness {
 			print("Found solution with fitness \(fitness): \(solution)")
 		} else {
 			print("Failed to perfect solution.")
 			print("Best: \(bestSolution!.1): \(bestSolution!.0)")
 		}
-
-	}
-
-	func applicationWillTerminate(_ aNotification: Notification) {
-		// Insert code here to tear down your application
 	}
 
-
 }
 

SwiftCMAES/Base.lproj/Main.storyboard → App/Base.lproj/Main.storyboard

@@ -708,6 +708,20 @@
                     <view key="view" id="m2S-Jp-Qdl">
                         <rect key="frame" x="0.0" y="0.0" width="480" height="480"/>
                         <autoresizingMask key="autoresizingMask"/>
+                        <subviews>
+                            <textField horizontalHuggingPriority="251" verticalHuggingPriority="750" translatesAutoresizingMaskIntoConstraints="NO" id="Wy3-ee-SQg">
+                                <rect key="frame" x="113" y="232" width="254" height="17"/>
+                                <textFieldCell key="cell" lineBreakMode="clipping" title="Nothing to see here. Look at the console." id="Wx5-1p-28M">
+                                    <font key="font" usesAppearanceFont="YES"/>
+                                    <color key="textColor" name="labelColor" catalog="System" colorSpace="catalog"/>
+                                    <color key="backgroundColor" name="textBackgroundColor" catalog="System" colorSpace="catalog"/>
+                                </textFieldCell>
+                            </textField>
+                        </subviews>
+                        <constraints>
+                            <constraint firstItem="Wy3-ee-SQg" firstAttribute="centerY" secondItem="m2S-Jp-Qdl" secondAttribute="centerY" id="7Xt-PS-cly"/>
+                            <constraint firstItem="Wy3-ee-SQg" firstAttribute="centerX" secondItem="m2S-Jp-Qdl" secondAttribute="centerX" id="n8T-v0-2dd"/>
+                        </constraints>
                     </view>
                 </viewController>
                 <customObject id="rPt-NT-nkU" userLabel="First Responder" customClass="NSResponder" sceneMemberID="firstResponder"/>

SwiftCMAES/SwiftCMAES.entitlements → App/SwiftCMAES.entitlements

@@ -2,9 +2,9 @@
 <!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
 <plist version="1.0">
 <dict>
-    <key>com.apple.security.app-sandbox</key>
-    <true/>
-    <key>com.apple.security.files.user-selected.read-only</key>
-    <true/>
+	<key>com.apple.security.app-sandbox</key>
+	<true/>
+	<key>com.apple.security.files.user-selected.read-only</key>
+	<true/>
 </dict>
 </plist>

App/ViewController.swift

@@ -0,0 +1,19 @@
+//
+//  ViewController.swift
+//  SwiftCMAES
+//
+//  Created by Santiago Gonzalez on 4/13/19.
+//  Copyright © 2019 Santiago Gonzalez. All rights reserved.
+//
+
+import Cocoa
+
+class ViewController: NSViewController {
+
+	override func viewDidLoad() {
+		super.viewDidLoad()
+
+		// Do any additional setup after loading the view.
+	}
+
+}

LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2019 Santiago Gonzalez
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

README.md

@@ -0,0 +1,95 @@
+# SwiftCMA
+## *by Santiago Gonzalez*
+### ***A pure-Swift implementation of Covariance Matrix Adaptation Evolutionary Strategy (CMA-ES).***
+
+**SwiftCMA** is a *de novo* implementation of [Covariance Matrix Adaptation Evolutionary Strategy](https://en.wikipedia.org/wiki/CMA-ES) (CMA-ES). CMA-ES is a wonderful population-based optimization technique that can optimize non-convex, non-smooth, non-differentiable functions. While CMA-ES is conceptually simple, it's rather complex mathematically. **SwiftCMA** is written in pure Swift, and makes proper use of functional programming and Swift's type system. This project is provided under the MIT License (see the `LICENSE` file for more info).
+
+## Functionality
+
+### CMA-ES
+
+The specific implementation of CMA-ES is inspired from the MATLAB reference implementation on [Wikipedia](https://en.wikipedia.org/wiki/CMA-ES). The implementation supports arbitrary dimension solution spaces. 
+
+The primary `CMAES` object has two slightly different implementations of the main `epoch()` method.
+* One takes a closure that takes an array of candidate solution vectors and returns an array of corresponding objective function values. This allows your code to potentially calculate objective function values concurrently.
+* Alternatively, for simplicity, you can use the flavor of `epoch()` that takes an objective evaluator. Objective functions can be represented by types that conform to the `ObjectiveEvaluator` protocol. In this case, objective function values are calculated sequentially on the same thread.
+
+### Linear Algebra API
+
+Swift isn't traditionally thought of as a good language for linear algebra code, though I feel that's mainly due to the lack of linear algebra APIs. **SwiftCMA** provides a clean API for vectors and matrices, based on top of Swift arrays, that should feel familiar if you've used Eigen / MATLAB / Octave, or similar systems. This API has not been optimized to be as fast as it could be since objective-function evaluation is the biggest bottleneck by far for what I created this library for (metalearning). Pull requests are welcome!
+
+Functionality:
+* Vectors and vector operations
+* Matrices and matrix operations
+* Vector-matrix operations
+* Eigendecomposition of matrices to get eigenvalues and an eigenbasis
+
+### Unit Tests
+
+Testing is great, so we have some unit tests as part of the Xcode project! More tests would be great, right now the tests just cover the linear algebra APIs.
+
+### Built-in Objective Functions
+
+**SwiftCMA** has some built-in objective functions. These are useful for testing / benchmarking how well the system is able to optimize some relatively well-understood functions.
+
+* N-dimensional sphere: `SphereObjectiveEvaluator`
+* Rastrigin function: `RastriginObjectiveEvaluator`
+* Ackley function: `AckleyObjectiveEvaluator`
+
+### Test App
+
+**SwiftCMA** comes with an Xcode project that builds a test app bundle. All code specific to this is in the `App/` directory. A quick note: the project builds an app bundle, rather than a basic executable, since it needs to link to `Accelerate.framework`.
+
+## Usage
+
+Everything you need to use **SwiftCMA** is in the `Sources/` directory.
+
+    let startSolution: Vector = ...
+		var fitness = MyObjectiveEvaluator()
+		let populationSize = CMAES.populationSize(forDimensions: startSolution.count)
+		let stepSigma: Double = ...
+		
+		let cmaes = CMAES(startSolution: startSolution, populationSize: populationSize, stepSigma: stepSigma)
+		var bestSolution: (Vector, Double)?
+		for i in 0..<1000 {
+			cmaes.epoch(evaluator: &fitness) { newSolution, newFitness in
+				print("Found solution with fitness \(fitness): \(solution)")
+			}
+
+			if bestSolution == nil || bestSolution!.1 > cmaes.bestSolution!.1 {
+				bestSolution = cmaes.bestSolution
+			}
+			print("\(i):   \(cmaes.bestSolution!.1)")
+		}
+		
+		print("Best: \(bestSolution!.1): \(bestSolution!.0)")
+
+### Defining an Objective Function
+
+CMA-ES aims to find the global minimum, so your objective function must be formulated so that smaller values are better.
+
+	struct SphereObjectiveEvaluator: ObjectiveEvaluator {
+		typealias Genome = Vector
+
+		func objective(genome: Vector, solutionCallback: (Vector, Double) -> ()) -> Double {
+			let value = genome.squaredMagnitude // Distance from origin is the error.
+			if diff < 0.01 { // We have found a solution when the difference is below a threshold.
+				solutionCallback(genome, diff)
+			}
+			return genome.squared.sum
+		}
+	}
+
+### Dependencies
+
+The only external dependency is `LAPACK` (for eigendecomposition). On macOS, this is fulfilled by the built-in `Accelerate` framework. On Linux, you should use the [CLapacke-Linux](https://github.com/indisoluble/CLapacke-Linux) Swift wrapper around LAPACK, which is very easy to install using `APT`.
+
+
+## Future Work
+
+* Integrate with Swift Package Manager.
+* Separate linear algebra API into its own library.
+* Faster.
+* Support fun variants of CMA-ES.
+* More tests (unit, integration, performance).
+* More engaging test app that visualizes the CMA-ES process.

SwiftCMAES/CMAES.swift → Sources/CMAES.swift

@@ -8,8 +8,14 @@
 
 import Foundation
 
+/// Embodies Covariance Matrix Adaptation Evolutionary Strategy (CMA-ES)
 class CMAES {
 
+	/// Returns a good population size for the specified number of dimensions.
+	static func populationSize(forDimensions n: Int) -> Int {
+		return Int(4+floor(3*log(Double(n))))
+	}
+
 	/// Dimensionality.
 	let n: Int
 	/// The number of solutions at each generation.
@@ -53,6 +59,7 @@ class CMAES {
 	/// Gap to postpone eigendecomposition to achieve O(N**2) per eval.
 	let lazyGapEvals: Double
 
+	/// Initializes a new CMA-ES run.
 	init(startSolution: Vector, populationSize: Int, stepSigma: Double) {
 		n = startSolution.count
 		xmean = startSolution
@@ -90,8 +97,19 @@ class CMAES {
 		lazyGapEvals = 0.5 * Double(n) * Double(populationSize) * (1.0 / (c1 + cmu)) / (Double(n) * Double(n)) // 0.5 is chosen such that eig takes 2 times the time of tell in >= 20-D		
 	}
 
+	typealias SolutionCallback = (Vector, Double) -> ()
+
+	/// A convenient wrapper for `epoch` that takes an objective evaluator.
+	func epoch<E: ObjectiveEvaluator>(evaluator: inout E, solutionCallback: @escaping SolutionCallback) where E.Genome == Vector {
+		epoch(valuesForCandidates: { candidateSolutions, innerSolutionCallback in
+			return candidateSolutions.map { solution in
+				return evaluator.objective(genome: solution, solutionCallback: innerSolutionCallback)
+			}
+		}, solutionCallback: solutionCallback)
+	}
+
 	/// Performs an evolutionary epoch.
-	func epoch<E: ObjectiveEvaluator>(evaluator: inout E, solutionCallback: (Vector, Double) -> ()) where E.Genome == Vector {
+	func epoch(valuesForCandidates: ([Vector], @escaping SolutionCallback) -> ([Double]), solutionCallback: @escaping SolutionCallback) {
 		// Generate offspring.
 		C.updateEigensystem(currentEval: countEval, lazyGapEvals: lazyGapEvals)
 		var candidateSolutions = [Vector]()
@@ -102,11 +120,9 @@ class CMAES {
 		}
 
 		// Evaluate fitnesses.
-		var fitnesses = candidateSolutions.map { solution in
-			return evaluator.objective(genome: solution, solutionCallback: solutionCallback)
-		}
+		var fitnesses = valuesForCandidates(candidateSolutions, solutionCallback)
 
-		// Bookkeeping
+		// Bookkeeping.
 		countEval += Double(fitnesses.count)
 
 		// Sort by fitness.
@@ -122,17 +138,11 @@ class CMAES {
 		let y = xmean - xold
 		let z = C.invsqrt.dot(vec: y)
 		let csn = sqrt(cs * (2.0 - cs) * mueff) / stepSigma
-//		ps = ps.enumerated().map { (1.0 - cs) * $1 + csn * z[$0] } // Update evolution path.
-		for i in 0..<n {
-			ps[i] = (1.0 - cs) * ps[i] + csn * z[i]
-		}
+		ps = ps.indexedMap { (1.0 - cs) * $1 + csn * z[$0] } // Update evolution path.
 		let ccn = sqrt(cc * (2.0 - cc) * mueff) / stepSigma
 		let hsig: Bool = (ps.squared.sum / Double(n) / (1.0 - pow(1.0 - cs, 2.0 * countEval / Double(populationSize)))) < 2.0 + 4.0 / (Double(n) + 1.0) // Turn off rank-one accumulation when sigma increases quickly.
 		let hsigValue = hsig ? 1.0 : 0.0
-//		pc = pc.enumerated().map { (1.0 - cc) * $1 + ccn * hsigValue * y[$0] } // Update evolution path.
-		for i in 0..<n {
-			pc[i] = (1.0 - cc) * pc[i] + ccn * hsigValue * y[i]
-		}
+		pc = pc.indexedMap { (1.0 - cc) * $1 + ccn * hsigValue * y[$0] } // Update evolution path.
 
 		// Adapt covariance matrix C.
 		let c1a: Double = c1 * (1.0 - (1.0 - hsigValue * hsigValue) * cc * (2.0 - cc))

SwiftCMAES/DecomposingPositiveDefiniteMatrix.swift → Sources/DecomposingPositiveDefiniteMatrix.swift

@@ -10,6 +10,7 @@ import Foundation
 
 /// A symmetric, positive-definite matrix that maintains its own eigendecomposition.
 class DecomposingPositiveDefiniteMatrix {
+
 	/// The backing matrix.
 	var matrix: Matrix
 	/// The matrix's dimension.
@@ -20,12 +21,13 @@ class DecomposingPositiveDefiniteMatrix {
 	/// The matrix's eigenvalues.
 	var eigenvalues: Vector
 
-	/// The "squooshinnes" of the distribution. More spherical distributions
+	/// The "squooshinnes" of the distribution; the ratio of the largest eigenvalue to the smallest eigenvalue. More spherical distributions
 	/// have a smaller condition number.
 	var conditionNumber: Double
 	var invsqrt: Matrix
 	var updatedEval: Double
 
+	/// Creates a new matrix of the specified dimension.
 	init(dim: Int) {
 		n = dim
 		matrix = Matrix.identity(dim: n)
@@ -42,7 +44,7 @@ class DecomposingPositiveDefiniteMatrix {
 
 		enforceSymmetry()
 
-		(eigenvalues, eigenbasis) = eigenDecompose(matrix) // O(N**3)
+		(eigenvalues, eigenbasis) = eigenDecompose(matrix)
 		guard eigenvalues.min()! > 0 else {
 			fatalError("Found negative eigenvalue!")
 		}
@@ -52,7 +54,7 @@ class DecomposingPositiveDefiniteMatrix {
 		// O(n^3) and takes about 25% of the time of eig
 		for i in 0..<n {
 			for j in 0..<(i+1) {
-				let sum = eigenvalues.enumerated().map { k, eigenvalue in
+				let sum = eigenvalues.indexedMap { k, eigenvalue in
 					return eigenbasis[i][k] * eigenbasis[j][k] / sqrt(eigenvalue)
 				}.sum
 				invsqrt[i][j] = sum