Skip to content

Commit

Permalink
Default parameters of optimization algorithms can be changed
Browse files Browse the repository at this point in the history
  • Loading branch information
@andreArtelt
andreArtelt committed Jan 13, 2021
1 parent c909f30 commit b160ea6
Show file tree
Hide file tree
Showing 16 changed files with 247 additions and 118 deletions.
2 changes: 1 addition & 1 deletion ceml/VERSION
View file
Original file line number Diff line number Diff line change
@@ -1 +1 @@
0.5.2
0.6
129 changes: 99 additions & 30 deletions ceml/optim/cvx.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,37 @@
import cvxpy as cp


def cvx_desc_to_solver(solver_desc):
if solver_desc == "SCS":
return cp.SCS
elif solver_desc == "MOSEK":
return cp.MOSEK
elif solver_desc == "CVXOPT":
return cp.CVXOPT
elif solver_desc == "OSQP":
return cp.OSQP
elif solver_desc == "ECOS":
return cp.ECOS
elif solver_desc == "CPLEX":
return cp.CPLEX
elif solver_desc == "CBC":
return cp.CBC
elif solver_desc == "NAG":
return cp.NAG
elif solver_desc == "GLPK":
return cp.GLPK
elif solver_desc == "GLPK_MI":
return cp.GLPK_MI
elif solver_desc == "GUROBI":
return cp.GUROBI
elif solver_desc == "SCIP":
return cp.SCIP
elif solver_desc == "XPRESS":
return cp.XPRESS
else:
raise ValueError(f"Solver '{solver_desc}' is not supported or unkown.")


class MathematicalProgram():
"""Base class for a mathematical program.
"""
Expand Down Expand Up @@ -58,6 +89,7 @@ class ConvexQuadraticProgram(ABC, SupportAffinePreprocessing):
"""
def __init__(self, **kwds):
self.epsilon = 1e-2
self.solver = cp.SCS

super().__init__(**kwds)

Expand All @@ -80,9 +112,9 @@ def _build_constraints(self, var_x, y):
raise NotImplementedError()

def _solve(self, prob):
prob.solve(solver=cp.SCS, verbose=False)
prob.solve(solver=self.solver, verbose=False)

def build_solve_opt(self, x_orig, y, features_whitelist=None, mad=None):
def build_solve_opt(self, x_orig, y, features_whitelist=None, mad=None, optimizer_args=None):
"""Builds and solves the convex quadratic optimization problem.
Parameters
Expand All @@ -102,6 +134,10 @@ def build_solve_opt(self, x_orig, y, features_whitelist=None, mad=None):
If `mad` is None, the Euclidean distance is used.
The default is None.
optimizer_args : `dict`, optional
Dictionary for overriding the default hyperparameters of the optimization algorithm.
The default is None.
Returns
Expand All @@ -111,6 +147,12 @@ def build_solve_opt(self, x_orig, y, features_whitelist=None, mad=None):
If no solution exists, `None` is returned.
"""
if optimizer_args is not None:
if "epsilon" in optimizer_args:
self.epsilon = optimizer_args["epsilon"]
if "solver" in optimizer_args:
self.solver = cvx_desc_to_solver(optimizer_args["solver"])

dim = x_orig.shape[0]

# Variables
Expand Down Expand Up @@ -175,6 +217,7 @@ class SDP(ABC):
"""
def __init__(self, **kwds):
self.epsilon = 1e-2
self.solver = cp.SCS

super().__init__(**kwds)

Expand All @@ -199,9 +242,9 @@ def _build_constraints(self, var_X, var_x, y):
raise NotImplementedError()

def _solve(self, prob):
prob.solve(solver=cp.SCS, verbose=False)
prob.solve(solver=self.solver, verbose=False)

def build_solve_opt(self, x_orig, y, features_whitelist=None):
def build_solve_opt(self, x_orig, y, features_whitelist=None, optimizer_args=None):
"""Builds and solves the SDP.
Parameters
Expand All @@ -215,6 +258,10 @@ def build_solve_opt(self, x_orig, y, features_whitelist=None):
If `features_whitelist` is None, all features can be used.
The default is None.
optimizer_args : `dict`, optional
Dictionary for overriding the default hyperparameters of the optimization algorithm.
The default is None.
Returns
Expand All @@ -224,6 +271,12 @@ def build_solve_opt(self, x_orig, y, features_whitelist=None):
If no solution exists, `None` is returned.
"""
if optimizer_args is not None:
if "epsilon" in optimizer_args:
self.epsilon = optimizer_args["epsilon"]
if "solver" in optimizer_args:
self.solver = cvx_desc_to_solver(optimizer_args["solver"])

dim = x_orig.shape[0]

# Variables
Expand Down Expand Up @@ -278,11 +331,10 @@ class DCQP(SupportAffinePreprocessing):
"""
def __init__(self, **kwds):
self.pccp = None
self.epsilon = 1e-2

super().__init__(**kwds)

def build_program(self, model, x_orig, y_target, Q0, Q1, q, c, A0_i, A1_i, b_i, r_i, features_whitelist=None, mad=None):
def build_program(self, model, x_orig, y_target, Q0, Q1, q, c, A0_i, A1_i, b_i, r_i, features_whitelist=None, mad=None, optimizer_args=None):
"""Builds the DCQP.
Parameters
Expand Down Expand Up @@ -320,40 +372,32 @@ def build_program(self, model, x_orig, y_target, Q0, Q1, q, c, A0_i, A1_i, b_i,
If `mad` is None, the Euclidean distance is used.
The default is None.
optimizer_args : `dict`, optional
Dictionary for overriding the default hyperparameters of the optimization algorithm.
The default is None.
"""
self.x_orig = x_orig
self.y_target = y_target
self.pccp = PenaltyConvexConcaveProcedure(model, Q0, Q1, q, c, A0_i, A1_i, b_i, r_i, features_whitelist, mad)
self.pccp = PenaltyConvexConcaveProcedure(model, Q0, Q1, q, c, A0_i, A1_i, b_i, r_i, features_whitelist, mad, optimizer_args)

def solve(self, x0, tao=1.2, tao_max=100, mu=1.5):
def solve(self, x0):
"""Approximately solves the DCQP by using the penalty convex-concave procedure.
Parameters
----------
x0 : `numpy.ndarray`
The initial data point for the penalty convex-concave procedure - this could be anything, however a "good" initial solution might lead to a better result.
tao : `float`, optional
Hyperparameter - see paper for details.
The default is 1.2
tao_max : `float`, optional
Hyperparameter - see paper for details.
The default is 100
mu : `float`, optional
Hyperparameter - see paper for details.
The default is 1.5
"""
self.pccp.set_affine_preprocessing(**self.get_affine_preprocessing())
return self.pccp.compute_counterfactual(self.x_orig, self.y_target, x0, tao=1.2, tao_max=100, mu=1.5)
return self.pccp.compute_counterfactual(self.x_orig, self.y_target, x0)


class PenaltyConvexConcaveProcedure(SupportAffinePreprocessing):
"""Implementation of the penalty convex-concave procedure for approximately solving a DCQP.
"""
def __init__(self, model, Q0, Q1, q, c, A0_i, A1_i, b_i, r_i, features_whitelist=None, mad=None, **kwds):
def __init__(self, model, Q0, Q1, q, c, A0_i, A1_i, b_i, r_i, features_whitelist=None, mad=None, optimizer_args=None, **kwds):
self.model = model
self.mad = mad
self.features_whitelist = features_whitelist
Expand All @@ -369,14 +413,31 @@ def __init__(self, model, Q0, Q1, q, c, A0_i, A1_i, b_i, r_i, features_whitelist
self.dim = None

self.epsilon = 1e-2
self.tao = 1.2
self.tao_max = 100
self.mu = 1.5

self.solver = cp.SCS

if optimizer_args is not None:
if "epsilon" in optimizer_args:
self.epsilon = optimizer_args["epsilon"]
if "tao" in optimizer_args:
self.tao = optimizer_args["tao"]
if "tao_max" in optimizer_args:
self.tao_max = optimizer_args["tao_max"]
if "mu" in optimizer_args:
self.mu = optimizer_args["mu"]
if "solver" in optimizer_args:
self.solver = cvx_desc_to_solver(optimizer_args["solver"])

if not(len(self.A0s) == len(self.A1s) and len(self.A0s) == len(self.bs) and len(self.rs) == len(self.bs)):
raise ValueError("Inconsistent number of constraint parameters")

super().__init__(**kwds)

def _solve(self, prob):
prob.solve(solver=cp.SCS, verbose=False)
prob.solve(solver=self.solver, verbose=False)

def solve_aux(self, xcf, tao, x_orig):
try:
Expand Down Expand Up @@ -442,7 +503,7 @@ def solve_aux(self, xcf, tao, x_orig):

return x_orig

def compute_counterfactual(self, x_orig, y_target, x0, tao, tao_max, mu):
def compute_counterfactual(self, x_orig, y_target, x0):
####################################
# Penalty convex-concave procedure #
####################################
Expand All @@ -451,18 +512,18 @@ def compute_counterfactual(self, x_orig, y_target, x0, tao, tao_max, mu):
xcf = x0

# Hyperparameters
cur_tao = tao
cur_tao = self.tao

# Solve a bunch of CCPs
while cur_tao < tao_max:
while cur_tao < self.tao_max:
xcf_ = self.solve_aux(xcf, cur_tao, x_orig)
xcf = xcf_

if y_target == self.model.predict([xcf_])[0]:
break

# Increase penalty parameter
cur_tao *= mu
cur_tao *= self.mu

return xcf

Expand All @@ -472,22 +533,29 @@ def compute_counterfactual(self, x_orig, y_target, x0, tao, tao_max, mu):
#################################################

class HighDensityEllipsoids:
def __init__(self, X, X_densities, cluster_probs, means, covariances, density_threshold=None, **kwds):
def __init__(self, X, X_densities, cluster_probs, means, covariances, density_threshold=None, optimizer_args=None, **kwds):
self.X = X
self.X_densities = X_densities
self.density_threshold = density_threshold if density_threshold is not None else float("-inf")
self.cluster_probs = cluster_probs
self.means = means
self.covariances = covariances

self.epsilon = 1e-5
self.solver = cp.SCS
if optimizer_args is not None:
if "epsilon" in optimizer_args:
self.epsilon = optimizer_args["epsilon"]
if "solver" in optimizer_args:
self.solver = cvx_desc_to_solver(optimizer_args["solver"])

super().__init__(**kwds)

def compute_ellipsoids(self):
return self.build_solve_opt()

def _solve(self, prob):
prob.solve(solver=cp.SCS, verbose=False)
prob.solve(solver=self.solver, verbose=False)

def build_solve_opt(self):
n_ellipsoids = self.cluster_probs.shape[1]
Expand Down Expand Up @@ -536,6 +604,7 @@ def __init__(self, w, b, n_dims, **kwds):

self.min_density = None
self.epsilon = 1e-2
self.solver = cp.SCS
self.gmm_cluster_index = 0 # For internal use only!

super().__init__(**kwds)
Expand Down Expand Up @@ -593,7 +662,7 @@ def compute_plausible_counterfactual(self, x, y, regularizer="l1"):
return xcf

def _solve_plausibility_opt(self, prob):
prob.solve(solver=cp.SCS, verbose=False)
prob.solve(solver=self.solver, verbose=False)

def build_solve_plausibility_opt(self, x_orig, y, mad=None):
dim = x_orig.shape[0]
Expand Down
23 changes: 11 additions & 12 deletions ceml/optim/optimizer.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -76,7 +76,7 @@ def is_optimizer_grad_based(optim):
raise TypeError(f"optim has to be either a string or an instance of 'ceml.optim.optimizer.Optimizer' but not of {type(optim)}")


def prepare_optim(optim, f, x0, f_grad=None, tol=None, max_iter=None):
def prepare_optim(optim, f, x0, f_grad=None, optimizer_args=None):
"""
Creates and initializes an optimization algorithm (instance of :class:`ceml.optim.optimizer.Optimizer`) specified by a description of the algorithm.
Expand All @@ -101,19 +101,10 @@ def prepare_optim(optim, f, x0, f_grad=None, tol=None, max_iter=None):
If `f_grad` is None, no gradient is used. Note that some optimization algorithms require a gradient!
The default is None.
tol : `float`, optional
Tolerance for termination.
`tol=None` is equivalent to `tol=0`.
optimizer_args : `dict`, optional
Dictionary for overriding the default hyperparameters of the optimization algorithm.
The default is None.
max_iter : `int`, optional
Maximum number of iterations.
If `max_iter` is None, the default value of the particular optimization algorithm is used.
Default is None.
Returns
-------
`callable`
Expand All @@ -129,6 +120,14 @@ def prepare_optim(optim, f, x0, f_grad=None, tol=None, max_iter=None):
if is_optimizer_grad_based(optim) and f_grad is None:
raise ValueError("You have to specify the gradient of the cost function if you want to use a gradient-based optimization algorithm.")

tol = None
max_iter = None
if optimizer_args is not None:
if "tol" in optimizer_args:
tol = optimizer_args["tol"]
if "max_iter" in optimizer_args:
max_iter = optimizer_args["max_iter"]

if isinstance(optim, str):
if optim == "nelder-mead":
optim = NelderMead()
Expand Down

0 comments on commit b160ea6

Please sign in to comment.