Skip to content

Commit

Permalink
Merge branch 'development' of github.com:automl/HPOlib into development
Browse files Browse the repository at this point in the history
  • Loading branch information
@KEggensperger
KEggensperger committed Feb 23, 2015
2 parents 237cd3d + 13c54dd commit eb53dd3
Show file tree
Hide file tree
Showing 2 changed files with 110 additions and 29 deletions.
110 changes: 97 additions & 13 deletions HPOlib/Plotting/plotParam.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,8 @@
import os
import sys

from matplotlib.pyplot import tight_layout, figure, subplots_adjust, subplot, savefig, show
from matplotlib.pyplot import tight_layout, figure, subplots_adjust, subplot,\
savefig, show, xticks
from matplotlib.gridspec import GridSpec

import numpy as np
Expand Down Expand Up @@ -65,8 +66,43 @@ def translate_para(key, value):
return new_name, value


def plot_params(value_list, result_list, name, min_=0.0, max_=0.0, save="",
title="", jitter=0):
def plot_params(value_list, result_list, name, min_=0.0, max_=0.0, save=None,
title="", jitter=0, xticks_=None, rotation=0):
"""
Parameters
----------
value_list : list
List of the parameter values. Will be plotted on the x axis.
result_list : list
List of the accompanying results. Will be plotted on the y axis.
name : str
Name of the hyperparameter.
min : float, optional
Minimum value on the y axis. If min_ == max_, the minumum and maximum
of result_list are used.
max : float, optional
Maximum value on the y axis.
save : str or None
Save the plot to disc. Use save as filename.
title : str, optional
Optional suptitle.
jitter : float, optional
Jitter the data to make it easier to display.
xticks_ : list, optional
List of hyperparameter values in sorted order. This should only be
used for categorical hyperparameters.
rotation : int, optional
Rotate the xticks by this many degrees.
"""
color = 'k'
marker = 'o'
size = 1
Expand Down Expand Up @@ -100,8 +136,8 @@ def plot_params(value_list, result_list, name, min_=0.0, max_=0.0, save="",

# Maybe jitter data
# But before save best values
best_value = value_list[np.argmin(result_list)]
best_result = min(result_list)
best_value = value_list[np.nanargmin(result_list)]
best_result = np.nanmin(result_list)

for idx in range(len(result_list)):
# noinspection PyArgumentList
Expand All @@ -126,6 +162,9 @@ def plot_params(value_list, result_list, name, min_=0.0, max_=0.0, save="",
ax.set_xlim([min_x, max_x])
ax.set_ylim([min_y, max_y])

if xticks_:
xticks(range(1, 1 + len(xticks_)), xticks_, rotation=rotation)

# Save Plot
tight_layout()
subplots_adjust(top=0.85)
Expand All @@ -137,8 +176,44 @@ def plot_params(value_list, result_list, name, min_=0.0, max_=0.0, save="",
show()


def main(pkl_list, name_list, param, min_=0.0, max_=0.0,
save="", title="", jitter=0):
def main(pkl_list, name_list, param=None, min_=0.0, max_=0.0,
save=None, title="", jitter=0, rotation=0):
"""
Parameters
----------
pkl_list : list of lists
The first list contains a list for each optimization algorithm. Each
of these lists lists HPOlib experiment pickles. plotParam will read the
configuration/result pairs from these pickles. Currently, the first
list is only allowed to have one entry.
name_list : list
Names of the optimizers in pkl_list. Currently, only one optimizer is
supported.
param : str or None
If this is a string, plot a single hyperparameter. Otherwise,
plot all hyperparameters.
min : float
See documentation of :meth:`HPOlib.Plotting.plotParam.plot_params`
max : float
See documentation of :meth:`HPOlib.Plotting.plotParam.plot_params`
save : str or None
If param is None, save should either be pdf or png. Otherwise,
see documentation of :meth:`HPOlib.Plotting.plotParam.plot_params`
title : str
See documentation of :meth:`HPOlib.Plotting.plotParam.plot_params`
jitter : float
See documentation of :meth:`HPOlib.Plotting.plotParam.plot_params`
rotation : int, optional
See documentation of :meth:`HPOlib.Plotting.plotParam.plot_params`
"""
if len(name_list) > 1:
raise NotImplementedError("No support for more than one experiment")

Expand All @@ -157,14 +232,17 @@ def main(pkl_list, name_list, param, min_=0.0, max_=0.0,
else:
params = [param]

for param in params:
for param in sorted(params):
if len(params) > 1:
try:
os.mkdir("params")
except:
pass

save = "params/%s.png" % param
if save is None or save.lower() not in ["png", "pdf"]:
save = ".png"

save = "params/%s.%s" % (param, save)

mod_param = param

Expand All @@ -181,7 +259,7 @@ def main(pkl_list, name_list, param, min_=0.0, max_=0.0,
if mod_param in t["params"]:
k, value = translate_para(param, t["params"][mod_param])
k = re.sub('^-', '', k)
print k, value
# print k, value
value = value.strip()
try:
value = float(value)
Expand All @@ -204,14 +282,19 @@ def main(pkl_list, name_list, param, min_=0.0, max_=0.0,
print "Found %s values for %s" % (str(len(value_list)), param)

plot_params(value_list=value_list, result_list=result_list, name=param,
min_=min_, max_=max_, save=save, title=title, jitter=jitter)
min_=min_, max_=max_, save=save, title=title,
jitter=jitter,
xticks_=[key for key, value in
sorted(string_to_value_map.items(), key=lambda
t: t[1])],
rotation=rotation)

if __name__ == "__main__":
prog = "python plot_param.py WhatIsThis <pathTo.pkl>* "
description = "Plot one param wrt to minfunction value"

parser = ArgumentParser(description=description, prog=prog)
parser.add_argument("-s", "--save", dest="save", default="",
parser.add_argument("-s", "--save", dest="save", default=None,
help="Where to save plot instead of showing it?")
parser.add_argument("--max", dest="max", type=float,
default=0, help="Maximum value of the y axis.")
Expand All @@ -223,11 +306,12 @@ def main(pkl_list, name_list, param, min_=0.0, max_=0.0,
help="Jitter data?")
parser.add_argument("-t", "--title", dest="title", default="",
help="Choose a supertitle for the plot")
parser.add_argument("-r", "--rotation", default=0, type=int)

args, unknown = parser.parse_known_args()

sys.stdout.write("\nFound " + str(len(unknown)) + " argument[s]...\n")
pkl_list_main, name_list_main = plot_util.get_pkl_and_name_list(unknown)
main(pkl_list=pkl_list_main, name_list=name_list_main, param=args.param,
min_=args.min, max_=args.max, save=args.save, title=args.title,
jitter=args.jitter)
jitter=args.jitter, rotation=args.rotation)
29 changes: 13 additions & 16 deletions HPOlib/wrapping.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -158,25 +158,13 @@ def main():
else:
experiment_dir = os.getcwd()

# Call get_configuration here to get the log level!
config = wrapping_util.get_configuration(experiment_dir,
None,
unknown_arguments)

formatter = logging.Formatter('[%(levelname)s] [%(asctime)s:%(name)s] %('
'message)s', datefmt='%H:%M:%S')
handler = logging.StreamHandler(sys.stdout)
handler.setFormatter(formatter)
hpolib_logger.addHandler(handler)
hpolib_logger.setLevel(1)

loglevel = config.getint("HPOLIB", "HPOlib_loglevel")
hpolib_logger.setLevel(loglevel)
if args.silent:
hpolib_logger.setLevel(60)
if args.verbose:
hpolib_logger.setLevel(10)

# DO NOT LOG UNTIL HERE UNLESS SOMETHING DRAMATIC HAS HAPPENED!!!
# First of all print the infodevel
if IS_DEVELOPMENT:
logger.critical(INFODEVEL)
Expand Down Expand Up @@ -212,6 +200,14 @@ def main():
config = wrapping_util.get_configuration(experiment_dir,
optimizer_version, unknown_arguments)

# DO NOT LOG UNTIL HERE UNLESS SOMETHING DRAMATIC HAS HAPPENED!!!
loglevel = config.getint("HPOLIB", "HPOlib_loglevel")
hpolib_logger.setLevel(loglevel)
if args.silent:
hpolib_logger.setLevel(60)
if args.verbose:
hpolib_logger.setLevel(10)

# Saving the config file is down further at the bottom, as soon as we get
# hold of the new optimizer directory
# wrapping_dir = os.path.dirname(os.path.realpath(__file__))
Expand Down Expand Up @@ -425,6 +421,7 @@ def main():
sent_SIGTERM_time = np.inf
sent_SIGKILL = False
sent_SIGKILL_time = np.inf
children_to_kill = list()

def enqueue_output(out, queue):
for line in iter(out.readline, b''):
Expand Down Expand Up @@ -484,21 +481,21 @@ def enqueue_output(out, queue):

if exit_.get_exit() == True and not sent_SIGINT:
logger.critical("Shutdown procedure: Sending SIGINT")
wrapping_util.kill_children(signal.SIGINT)
wrapping_util.kill_processes(signal.SIGINT)
sent_SIGINT_time = time.time()
sent_SIGINT = True

if exit_.get_exit() == True and not sent_SIGTERM and time.time() \
> sent_SIGINT_time + 5:
logger.critical("Shutdown procedure: Sending SIGTERM")
wrapping_util.kill_children(signal.SIGTERM)
wrapping_util.kill_processes(signal.SIGTERM)
sent_SIGTERM_time = time.time()
sent_SIGTERM = True

if exit_.get_exit() == True and not sent_SIGKILL and time.time() \
> sent_SIGTERM_time + 5:
logger.critical("Shutdown procedure: Sending SIGKILL")
wrapping_util.kill_children(signal.SIGKILL)
wrapping_util.kill_processes(signal.SIGKILL)
sent_SIGKILL_time = time.time()
sent_SIGKILL = True

Expand Down

0 comments on commit eb53dd3

Please sign in to comment.