-
Notifications
You must be signed in to change notification settings - Fork 214
/
Optim.jl
213 lines (164 loc) · 6.43 KB
/
Optim.jl
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
"""
# Optim.jl
Welcome to Optim.jl!
Optim.jl is a package used to solve continuous optimization problems. It is
written in Julia for Julians to help take advantage of arbitrary number types,
fast computation, and excellent automatic differentiation tools.
## REPL help
`?` followed by an algorithm name (`?BFGS`), constructors (`?Optim.Options`)
prints help to the terminal.
## Documentation
Besides the help provided at the REPL, it is possible to find help and general
documentation online at http://julianlsolvers.github.io/Optim.jl/stable/ .
"""
module Optim
using NLSolversBase # for shared infrastructure in JuliaNLSolvers
using PositiveFactorizations # for globalization strategy in Newton
import PositiveFactorizations: cholesky!, cholesky
using LineSearches # for globalization strategy in Quasi-Newton algs
using DiffEqDiffTools # for finite difference derivatives
# using ReverseDiff # for reverse mode AD (not really suitable for scalar output)
using ForwardDiff # for forward mode AD
import NaNMath # for functions that ignore NaNs (no poisoning)
import Parameters: @with_kw, # for types where constructors are simply defined
@unpack # by their default values, and simple unpacking
# of fields
using Printf # For printing, maybe look into other options
using FillArrays # For handling scalar bounds in Fminbox
#using Compat # for compatibility across multiple julia versions
# for extensions of functions defined in Base.
import Base: length, push!, show, getindex, setindex!, maximum, minimum
# objective and constraints types and functions relevant to them.
import NLSolversBase: NonDifferentiable, OnceDifferentiable, TwiceDifferentiable,
nconstraints, nconstraints_x, NotInplaceObjective, InplaceObjective
# var for NelderMead
import StatsBase: var
import LinearAlgebra: Diagonal, diag, Hermitian, Symmetric,
rmul!, mul!, ldiv!,
dot, norm, normalize!,
eigen, BLAS,
cholesky, Cholesky, # factorizations
I,
svd
import SparseArrays: AbstractSparseMatrix
# exported functions and types
export optimize, maximize, # main function
# Re-export objective types from NLSolversBase
NonDifferentiable,
OnceDifferentiable,
TwiceDifferentiable,
# Re-export constraint types from NLSolversBase
TwiceDifferentiableConstraints,
# I don't think these should be here [pkofod]
OptimizationState,
OptimizationTrace,
# Optimization algorithms
## Zeroth order methods (heuristics)
NelderMead,
ParticleSwarm,
SimulatedAnnealing,
## First order
### Quasi-Newton
GradientDescent,
BFGS,
LBFGS,
### Conjugate gradient
ConjugateGradient,
### Acceleration methods
AcceleratedGradientDescent,
MomentumGradientDescent,
### Nonlinear GMRES
NGMRES,
OACCEL,
## Second order
### (Quasi-)Newton
Newton,
### Trust region
NewtonTrustRegion,
# Constrained
## Box constraints, x_i in [lb_i, ub_i]
### Specifically Univariate, R -> R
GoldenSection,
Brent,
### Multivariate, R^N -> R
Fminbox,
SAMIN,
## Manifold constraints
Manifold,
Flat,
Sphere,
Stiefel,
## Non-linear constraints
IPNewton
include("types.jl") # types used throughout
include("Manifolds.jl") # code to handle manifold constraints
include("multivariate/precon.jl") # preconditioning functionality
# utilities
include("utilities/generic.jl") # generic utilities
include("utilities/maxdiff.jl") # find largest difference
include("utilities/update.jl") # trace code
# Unconstrained optimization
## Grid Search
include("multivariate/solvers/zeroth_order/grid_search.jl")
## Zeroth order (Heuristic) Optimization Methods
include("multivariate/solvers/zeroth_order/nelder_mead.jl")
include("multivariate/solvers/zeroth_order/simulated_annealing.jl")
include("multivariate/solvers/zeroth_order/particle_swarm.jl")
## Quasi-Newton
include("multivariate/solvers/first_order/gradient_descent.jl")
include("multivariate/solvers/first_order/bfgs.jl")
include("multivariate/solvers/first_order/l_bfgs.jl")
## Acceleration methods
include("multivariate/solvers/first_order/accelerated_gradient_descent.jl")
include("multivariate/solvers/first_order/momentum_gradient_descent.jl")
## Conjugate gradient
include("multivariate/solvers/first_order/cg.jl")
## Newton
### Line search
include("multivariate/solvers/second_order/newton.jl")
include("multivariate/solvers/second_order/krylov_trust_region.jl")
### Trust region
include("multivariate/solvers/second_order/newton_trust_region.jl")
## Nonlinear GMRES
include("multivariate/solvers/first_order/ngmres.jl")
# Constrained optimization
## Box constraints
include("multivariate/solvers/constrained/fminbox.jl")
include("multivariate/solvers/constrained/samin.jl")
# Univariate methods
include("univariate/solvers/golden_section.jl")
include("univariate/solvers/brent.jl")
include("univariate/types.jl")
include("univariate/printing.jl")
# Line search generic code
include("utilities/perform_linesearch.jl")
# Backward compatibility
include("deprecate.jl")
# convenient user facing optimize methods
include("univariate/optimize/interface.jl")
include("multivariate/optimize/interface.jl")
# actual optimize methods
include("univariate/optimize/optimize.jl")
include("multivariate/optimize/optimize.jl")
# Convergence
include("utilities/assess_convergence.jl")
include("multivariate/solvers/zeroth_order/zeroth_utils.jl")
# Traces
include("utilities/trace.jl")
# API
include("api.jl")
## Interior point includes
include("multivariate/solvers/constrained/ipnewton/types.jl")
# Tracing
include("multivariate/solvers/constrained/ipnewton/utilities/update.jl")
# Constrained optimization
include("multivariate/solvers/constrained/ipnewton/iplinesearch.jl")
include("multivariate/solvers/constrained/ipnewton/interior.jl")
include("multivariate/solvers/constrained/ipnewton/ipnewton.jl")
# Convergence
include("multivariate/solvers/constrained/ipnewton/utilities/assess_convergence.jl")
# Traces
include("multivariate/solvers/constrained/ipnewton/utilities/trace.jl")
# Maximization convenience wrapper
include("maximize.jl")
end