Skip to content
Permalink
Branch: master
Find file Copy path
Find file Copy path
Fetching contributors…
Cannot retrieve contributors at this time
734 lines (674 sloc) 26.7 KB
"""
Provides the learning rate scheduling logic.
The base class is :class:`LearningRateControl`.
"""
from __future__ import print_function
import os
import typing
from Util import better_repr, simple_obj_repr, ObjAsDict, unicode
from Log import log
import numpy
class LearningRateControl(object):
"""
Base class for learning rate control / scheduling.
"""
need_error_info = True
class EpochData:
"""
Encapsulates all relevant information for one epoch,
needed to perform learning rate scheduling,
such as the individual scores (cv or train; cross-entropy or frame-error or whatever).
"""
# Need to keep the non-PEP8 name for compatibility, because we store the repr of the object.
# noinspection PyPep8Naming
def __init__(self, learningRate, error=None):
"""
:type learningRate: float
:type error: dict[str,float] | None
"""
self.learning_rate = learningRate
if isinstance(error, float): # Old format.
error = {"old_format_score": error}
if error is None:
error = {}
self.error = error
def __repr__(self):
# This is being used for serialization, and we want some forward/backward compatibility,
# so we should try to keep this consistent.
return "EpochData(learningRate=%s, error=%s)" % (
better_repr(self.learning_rate), better_repr(self.error))
@classmethod
def load_initial_kwargs_from_config(cls, config):
"""
:type config: Config.Config
:rtype: dict[str]
"""
return {
"default_learning_rate": config.float('learning_rate', 1.0),
"min_learning_rate": config.float('min_learning_rate', 0.0),
"default_learning_rates": config.typed_value('learning_rates') or config.float_list('learning_rates'),
"error_measure_key": (
config.typed_value('learning_rate_control_error_measure')
or config.value('learning_rate_control_error_measure', None)),
"relative_error_also_relative_to_learning_rate": (
config.bool('learning_rate_control_relative_error_relative_lr', False)),
"min_num_epochs_per_new_learning_rate": config.int("learning_rate_control_min_num_epochs_per_new_lr", 0),
"relative_error_div_by_old": config.bool('newbob_relative_error_div_by_old', False),
"filename": config.value('learning_rate_file', None),
}
@classmethod
def load_initial_from_config(cls, config):
"""
:type config: Config.Config
:rtype: LearningRateControl
"""
kwargs = cls.load_initial_kwargs_from_config(config)
return cls(**kwargs)
def __init__(self, default_learning_rate, min_learning_rate=0.0, default_learning_rates=None,
error_measure_key=None,
relative_error_also_relative_to_learning_rate=False,
min_num_epochs_per_new_learning_rate=0,
relative_error_div_by_old=False,
filename=None):
"""
:param float default_learning_rate: default learning rate. usually for epoch 1
:param list[float] | dict[int,float] default_learning_rates: learning rates
:param str|list[str]|None error_measure_key: for get_epoch_error_value() the key for EpochData.error which is a dict
:param int min_num_epochs_per_new_learning_rate: if the lr was recently updated, use it for at least N epochs
:param bool relative_error_div_by_old: if True, compute relative error as (new - old) / old.
:param str filename: load from and save to file
"""
self.epoch_data = {} # type: typing.Dict[int,LearningRateControl.EpochData]
self.filename = filename
if filename:
if os.path.exists(filename):
print("Learning-rate-control: loading file %s" % filename, file=log.v4)
# Load now, such that default_learning_rates is correctly handled.
self.load()
else:
print("Learning-rate-control: file %s does not exist yet" % filename, file=log.v4)
else:
print("Learning-rate-control: no file specified, not saving history (no proper restart possible)", file=log.v4)
self.default_learning_rate = default_learning_rate
self.min_learning_rate = min_learning_rate
if default_learning_rates:
if isinstance(default_learning_rates, list):
default_learning_rates = {i + 1: v for (i, v) in enumerate(default_learning_rates)}
if isinstance(default_learning_rates, (str, unicode)):
default_learning_rates = eval(default_learning_rates)
assert isinstance(default_learning_rates, dict)
for epoch, v in default_learning_rates.items():
self.set_default_learning_rate_for_epoch(epoch, v)
self.default_learning_rates = default_learning_rates
self.error_measure_key = error_measure_key
self.relative_error_also_relative_to_learning_rate = relative_error_also_relative_to_learning_rate
self.min_num_epochs_per_new_learning_rate = min_num_epochs_per_new_learning_rate
self.relative_error_div_by_old = relative_error_div_by_old
__repr__ = simple_obj_repr
def __str__(self):
epochs = sorted(self.epoch_data.keys())
if len(epochs) > 6:
epoch_str = ", ".join(
["%i: %s" % (epoch, self.epoch_data[epoch]) for epoch in epochs[:3]] +
["..."] +
["%i: %s" % (epoch, self.epoch_data[epoch]) for epoch in epochs[-3:]])
else:
epoch_str = ", ".join(["%i: %s" % (epoch, self.epoch_data[epoch]) for epoch in epochs])
return "%r, epoch data: %s, error key: %s" % (self, epoch_str, self.get_error_key(epoch=1))
def calc_learning_rate_for_epoch(self, epoch):
"""
:type epoch: int
:returns learning rate
:rtype: float
"""
raise NotImplementedError
def calc_new_learning_rate_for_epoch(self, epoch):
"""
:param int epoch:
:return: new learning rate for this epoch
:rtype: float
"""
if self.min_num_epochs_per_new_learning_rate > 1:
last_lrs = [self.epoch_data[e].learning_rate
for e in self._last_epochs_for_epoch(epoch, num_epochs=self.min_num_epochs_per_new_learning_rate)]
if len(set(last_lrs)) >= 2 or 0 < len(last_lrs) < self.min_num_epochs_per_new_learning_rate:
return last_lrs[-1]
learning_rate = self.calc_learning_rate_for_epoch(epoch)
if learning_rate < self.min_learning_rate:
return self.min_learning_rate
return learning_rate
def _last_epochs_for_epoch(self, epoch, num_epochs):
"""
:param int epoch:
:param int num_epochs:
:return: last N epochs where we have some epoch data
:rtype: list[int]
"""
last_epochs = sorted([e for e in self.epoch_data.keys() if e < epoch])
if not last_epochs:
return []
last_epochs = last_epochs[-num_epochs:]
return last_epochs
def get_learning_rate_for_epoch(self, epoch):
"""
:type epoch: int
:rtype: float
"""
assert epoch >= 1
if epoch in self.epoch_data:
return self.epoch_data[epoch].learning_rate
learning_rate = self.calc_new_learning_rate_for_epoch(epoch)
self.set_default_learning_rate_for_epoch(epoch, learning_rate)
return learning_rate
def set_default_learning_rate_for_epoch(self, epoch, learning_rate):
"""
:type epoch: int
:type learning_rate: float
"""
if epoch in self.epoch_data:
if not self.epoch_data[epoch].learning_rate:
self.epoch_data[epoch].learning_rate = learning_rate
else:
self.epoch_data[epoch] = self.EpochData(learning_rate)
def get_last_epoch(self, epoch):
"""
:param int epoch:
:return: last epoch before ``epoch`` where we have some epoch data
:rtype: int
"""
epochs = sorted([e for e in self.epoch_data.keys() if e < epoch])
if not epochs:
return None
return epochs[-1]
def get_most_recent_learning_rate(self, epoch, exclude_current=True):
"""
:param int epoch:
:param bool exclude_current:
:return: most learning rate before or including ``epoch``
:rtype: float
"""
for e, data in reversed(sorted(self.epoch_data.items())):
assert isinstance(data, LearningRateControl.EpochData)
if e > epoch:
continue
if exclude_current and e == epoch:
continue
if data.learning_rate is None:
continue
return data.learning_rate
return self.default_learning_rate
def calc_relative_error(self, old_epoch, new_epoch):
"""
:param int old_epoch:
:param int new_epoch:
:return: relative error between old epoch and new epoch
:rtype: float
"""
old_key, old_error = self.get_epoch_error_key_value(old_epoch)
new_key, new_error = self.get_epoch_error_key_value(new_epoch)
if old_error is None or new_error is None:
return None
if old_key != new_key:
return None
if self.relative_error_div_by_old:
relative_error = (new_error - old_error) / abs(old_error)
else:
relative_error = (new_error - old_error) / abs(new_error)
if self.relative_error_also_relative_to_learning_rate:
learning_rate = self.get_most_recent_learning_rate(new_epoch, exclude_current=False)
if learning_rate > 0:
# If the learning rate is lower than the initial learning rate,
# the relative error is also expected to be lower, so correct for that here.
relative_error /= learning_rate / self.default_learning_rate
return relative_error
def set_epoch_error(self, epoch, error):
"""
:type epoch: int
:type error: dict[str,float|dict[str,float]]
"""
if epoch not in self.epoch_data:
print("Learning rate not set for epoch %i. Assuming default." % epoch, file=log.v4)
self.get_learning_rate_for_epoch(epoch) # This will set it.
assert isinstance(error, dict)
error = error.copy()
for k, v in list(error.items()):
if isinstance(v, dict): # like error = {"dev_score": {"cost:output1": .., "cost:output2": ...}, ...}
del error[k]
if len(v) == 1:
error[k] = list(v.values())[0]
continue
for k1, v1 in v.items():
if ":" in k1:
k1 = k1[k1.index(":") + 1:]
error[k + "_" + k1] = v1
for v in error.values():
assert isinstance(v, float)
self.epoch_data[epoch].error.update(error)
if epoch == 1:
print("Learning-rate-control: error key %r from %r" % (self.get_error_key(epoch), error), file=log.v4)
def get_error_key(self, epoch):
"""
:param int epoch:
:return: key which we should look in scores/errors, for this epoch
:rtype: str
"""
if epoch not in self.epoch_data:
if isinstance(self.error_measure_key, list):
return self.error_measure_key[0]
assert isinstance(self.error_measure_key, (str, type(None)))
return self.error_measure_key
epoch_data = self.epoch_data[epoch]
if not epoch_data.error:
return None
if len(epoch_data.error) == 1 and "old_format_score" in epoch_data.error:
return "old_format_score"
keys = []
if isinstance(self.error_measure_key, list):
for key in self.error_measure_key:
keys += [key, key + "_output"] # for multiple outputs, try default output
elif isinstance(self.error_measure_key, str):
keys += [self.error_measure_key, self.error_measure_key + "_output"]
else:
assert self.error_measure_key is None
keys += ["dev_score", "dev_score_output"]
for key in keys:
if key in epoch_data.error:
return key
for key in sorted(epoch_data.error.keys()):
if key == "dev_score_output/output" or key.startswith("dev_score_output/output_"):
return key
for key in sorted(epoch_data.error.keys()):
if key.startswith("dev_score_output/"):
return key
for key in sorted(epoch_data.error.keys()):
if key.startswith("dev_"):
return key
for key in ["train_score", "train_score_output"]:
if key in epoch_data.error:
return key
return min(epoch_data.error.keys())
def get_epoch_error_dict(self, epoch):
"""
:param int epoch:
:rtype: dict[str,float]
"""
if epoch not in self.epoch_data:
return {}
return self.epoch_data[epoch].error
def get_epoch_error_value(self, epoch):
"""
:param int epoch:
:return: error/score for the specific epoch, given the error-key, see :func:`get_error_key`
:rtype: float
"""
error = self.get_epoch_error_dict(epoch)
if not error:
return None
key = self.get_error_key(epoch)
assert key
assert key in error, (
"%r not in %r. fix %r in config. set it to %r or so." % (
key, error, 'learning_rate_control_error_measure', 'dev_error'))
return error[key]
def get_epoch_error_key_value(self, epoch):
"""
:param int epoch:
:return: key, error
:rtype: (str, float)
"""
error = self.get_epoch_error_dict(epoch)
if not error:
return None, None
key = self.get_error_key(epoch)
assert key
assert key in error, (
"%r not in %r. fix %r in config. set it to %r or so." %
(key, error, 'learning_rate_control_error_measure', 'dev_error'))
return key, error[key]
def get_last_best_epoch(self, last_epoch, first_epoch=1, filter_score=float("inf"), only_last_n=-1,
min_score_dist=0.0):
"""
:param int first_epoch: will check all epochs >= first_epoch
:param int last_epoch: inclusive. will check all epochs <= last_epoch
:param float filter_score: all epochs which values over this score are not considered
:param int only_last_n: if set (>=1), from the resulting list, we consider only the last only_last_n
:param float min_score_dist: filter out epochs where the diff to the most recent is not big enough
:return: the last best epoch. to get the details then, you might want to use getEpochErrorDict.
:rtype: int|None
"""
if first_epoch > last_epoch:
return None
values = [(self.get_epoch_error_key_value(ep), ep) for ep in range(first_epoch, last_epoch + 1)]
# Note that the order of the checks here is a bit arbitrary but I had some thoughts on it.
# Changing the order will also slightly change the behavior, so be sure it make sense.
values = [((key, v), ep) for ((key, v), ep) in values if v is not None]
if not values:
return None
last_key, latest_score = values[-1][0]
values = [(v, ep) for ((key, v), ep) in values if key == last_key] # only same key
values = [(v, ep) for (v, ep) in values if v <= filter_score]
if not values:
return None
if only_last_n >= 1:
values = values[-only_last_n:]
values = [(v, ep) for (v, ep) in values if v + min_score_dist < latest_score]
if not values:
return None
return min(values)[1]
def save(self):
"""
Save the current epoch data to file (self.filename).
"""
if not self.filename:
return
# First write to a temp-file, to be sure that the write happens without errors.
# Otherwise, it could happen that we delete the old existing file, then
# some error happens (e.g. disk quota), and we loose the newbob data.
# Loosing that data is very bad because it basically means that we have to redo all the training.
tmp_filename = self.filename + ".new_tmp"
f = open(tmp_filename, "w")
f.write(better_repr(self.epoch_data))
f.write("\n")
f.close()
os.rename(tmp_filename, self.filename)
def load(self):
"""
Loads the saved epoch data from file (self.filename).
"""
s = open(self.filename).read()
self.epoch_data = eval(s, {"nan": float("nan"), "inf": float("inf")}, ObjAsDict(self))
class ConstantLearningRate(LearningRateControl):
"""
Just a constant learning rate.
"""
need_error_info = False
def calc_learning_rate_for_epoch(self, epoch):
"""
Dummy constant learning rate. Returns initial learning rate.
:type epoch: int
:returns learning rate
:rtype: float
"""
while True:
last_epoch = self.get_last_epoch(epoch)
if last_epoch is None:
return self.default_learning_rate
learning_rate = self.epoch_data[last_epoch].learning_rate
if learning_rate is None:
epoch = last_epoch
continue
return learning_rate
class NewbobRelative(LearningRateControl):
"""
If relative diff between old and new error is over some threshold, decay learning rate.
"""
@classmethod
def load_initial_kwargs_from_config(cls, config):
"""
:type config: Config.Config
:rtype: dict[str]
"""
kwargs = super(NewbobRelative, cls).load_initial_kwargs_from_config(config)
kwargs.update({
"relative_error_threshold": config.float('newbob_relative_error_threshold', -0.01),
"learning_rate_decay_factor": config.float('newbob_learning_rate_decay', 0.5)})
return kwargs
def __init__(self, relative_error_threshold, learning_rate_decay_factor, **kwargs):
"""
:type relative_error_threshold: float
:type learning_rate_decay_factor: float
:type filename: str
"""
super(NewbobRelative, self).__init__(**kwargs)
self.relative_error_threshold = relative_error_threshold
self.learning_rate_decay_factor = learning_rate_decay_factor
def calc_learning_rate_for_epoch(self, epoch):
"""
Newbob+ on train data.
:type epoch: int
:returns learning rate
:rtype: float
"""
last_epoch = self.get_last_epoch(epoch)
if last_epoch is None:
return self.default_learning_rate
learning_rate = self.epoch_data[last_epoch].learning_rate
if learning_rate is None:
return self.default_learning_rate
last2_epoch = self.get_last_epoch(last_epoch)
if last2_epoch is None:
return learning_rate
relative_error = self.calc_relative_error(last2_epoch, last_epoch)
if relative_error is None:
return learning_rate
if relative_error > self.relative_error_threshold:
learning_rate *= self.learning_rate_decay_factor
return learning_rate
class NewbobAbs(LearningRateControl):
"""
If absolute diff between old and new error is over some threshold, decay learning rate.
"""
@classmethod
def load_initial_kwargs_from_config(cls, config):
"""
:type config: Config.Config
:rtype: dict[str]
"""
kwargs = super(NewbobAbs, cls).load_initial_kwargs_from_config(config)
kwargs.update({
"error_threshold": config.float('newbob_error_threshold', -0.01),
"learning_rate_decay_factor": config.float('newbob_learning_rate_decay', 0.5)})
return kwargs
def __init__(self, error_threshold, learning_rate_decay_factor, **kwargs):
"""
:type error_threshold: float
:type learning_rate_decay_factor: float
"""
super(NewbobAbs, self).__init__(**kwargs)
self.error_threshold = error_threshold
self.learning_rate_decay_factor = learning_rate_decay_factor
def calc_learning_rate_for_epoch(self, epoch):
"""
Newbob+ on train data.
:type epoch: int
:returns learning rate
:rtype: float
"""
last_epoch = self.get_last_epoch(epoch)
if last_epoch is None:
return self.default_learning_rate
learning_rate = self.epoch_data[last_epoch].learning_rate
if learning_rate is None:
return self.default_learning_rate
last2_epoch = self.get_last_epoch(last_epoch)
if last2_epoch is None:
return learning_rate
old_key, old_error = self.get_epoch_error_key_value(last2_epoch)
new_key, new_error = self.get_epoch_error_key_value(last_epoch)
if old_error is None or new_error is None:
return learning_rate
if old_key != new_key:
return learning_rate
error_diff = new_error - old_error
if error_diff > self.error_threshold:
learning_rate *= self.learning_rate_decay_factor
return learning_rate
class NewbobMultiEpoch(LearningRateControl):
"""
Like :class:`NewbobRelative`, but looks at the average relative error over multiple epochs.
This is useful together with ``partition_epoch`` from :class:`Dataset`.
"""
@classmethod
def load_initial_kwargs_from_config(cls, config):
"""
:type config: Config.Config
:rtype: dict[str]
"""
kwargs = super(NewbobMultiEpoch, cls).load_initial_kwargs_from_config(config)
kwargs.update({
"num_epochs": config.int("newbob_multi_num_epochs", 5),
"update_interval": config.int("newbob_multi_update_interval", config.int("newbob_multi_num_epochs", 5)),
"relative_error_threshold": config.float('newbob_relative_error_threshold', -0.01),
"learning_rate_decay_factor": config.float('newbob_learning_rate_decay', 0.5),
"learning_rate_growth_factor": config.float('newbob_learning_rate_growth', 1.0),
})
return kwargs
def __init__(self, num_epochs, update_interval,
relative_error_threshold, learning_rate_decay_factor, learning_rate_growth_factor=1.0,
**kwargs):
"""
:param int num_epochs:
:param int update_interval:
:param float relative_error_threshold:
:param float learning_rate_decay_factor:
:param int filename:
"""
super(NewbobMultiEpoch, self).__init__(**kwargs)
self.num_epochs = num_epochs
assert self.num_epochs >= 1
self.update_interval = update_interval
assert self.update_interval >= 1
self.relative_error_threshold = relative_error_threshold
self.learning_rate_decay_factor = learning_rate_decay_factor
self.learning_rate_growth_factor = learning_rate_growth_factor
def _calc_mean_relative_error(self, epochs):
"""
:param list[int] epochs:
:return: mean of relative errors
:rtype: float|None
"""
assert len(epochs) >= 2
errors = [self.calc_relative_error(epochs[i], epochs[i + 1]) for i in range(len(epochs) - 1)]
if any([e is None for e in errors]):
return None
return numpy.mean(errors)
def _calc_recent_mean_relative_error(self, epoch):
"""
:param int epoch:
:return: recent mean of relative errors
:rtype: float|None
"""
# Take one more than numEpochs because we are looking at the diffs.
last_epochs = self._last_epochs_for_epoch(epoch, num_epochs=self.num_epochs + 1)
if not last_epochs:
return None
# We could also use the self.numEpochs limit here. But maybe this is better.
if len(last_epochs) <= 1:
return None
return self._calc_mean_relative_error(last_epochs)
def calc_learning_rate_for_epoch(self, epoch):
"""
Newbob+ on train data.
:type epoch: int
:returns learning rate
:rtype: float
"""
learning_rate = self.get_most_recent_learning_rate(epoch)
# We start counting epochs at 1.
if self.update_interval > 1 and epoch % self.update_interval != 1:
return learning_rate
mean_relative_error = self._calc_recent_mean_relative_error(epoch)
if mean_relative_error is None:
return learning_rate
if mean_relative_error > self.relative_error_threshold:
learning_rate *= self.learning_rate_decay_factor
else:
learning_rate *= self.learning_rate_growth_factor
return learning_rate
def learning_rate_control_type(type_name):
"""
:param str type_name:
:rtype: type[LearningRateControl]|LearningRateControl
"""
if type_name == "constant":
return ConstantLearningRate
elif type_name in ("newbob", "newbob_rel", "newbob_relative"): # Old setups expect the relative version.
return NewbobRelative
elif type_name == "newbob_abs":
return NewbobAbs
elif type_name == "newbob_multi_epoch":
return NewbobMultiEpoch
else:
assert False, "unknown learning-rate-control type %s" % type_name
def load_learning_rate_control_from_config(config):
"""
:type config: Config.Config
:rtype: LearningRateControl
"""
control_type = config.value("learning_rate_control", "constant")
cls = learning_rate_control_type(control_type)
return cls.load_initial_from_config(config)
def demo():
"""
Demo run. Given some learning rate file (with scores / existing lrs), will calculate how lrs would have been set,
given some config.
"""
import better_exchook
better_exchook.install()
import rnn
import sys
if len(sys.argv) <= 1:
print("usage: python %s [config] [other options] [++check_learning_rates 1]" % __file__)
print(
("example usage: "
"python %s ++learning_rate_control newbob ++learning_rate_file newbob.data ++learning_rate 0.001") % __file__)
rnn.init_config(command_line_options=sys.argv[1:])
# noinspection PyProtectedMember
rnn.config._hack_value_reading_debug()
rnn.config.update({"log": []})
rnn.init_log()
rnn.init_backend_engine()
check_lr = rnn.config.bool("check_learning_rates", False)
from Pretrain import pretrain_from_config
pretrain = pretrain_from_config(rnn.config)
first_non_pretrain_epoch = 1
pretrain_learning_rate = None
if pretrain:
first_non_pretrain_epoch = pretrain.get_train_num_epochs() + 1
log.initialize(verbosity=[5])
control = load_learning_rate_control_from_config(rnn.config)
print("LearningRateControl: %r" % control)
if not control.epoch_data:
print("No epoch data so far.")
return
first_epoch = min(control.epoch_data.keys())
if first_epoch != 1:
print("Strange, first epoch from epoch data is %i." % first_epoch)
print("Error key: %s from %r" % (control.get_error_key(epoch=first_epoch), control.epoch_data[first_epoch].error))
if pretrain:
pretrain_learning_rate = rnn.config.float('pretrain_learning_rate', control.default_learning_rate)
max_epoch = max(control.epoch_data.keys())
for epoch in range(1, max_epoch + 2): # all epochs [1..max_epoch+1]
old_learning_rate = None
if epoch in control.epoch_data:
old_learning_rate = control.epoch_data[epoch].learning_rate
if epoch < first_non_pretrain_epoch:
learning_rate = pretrain_learning_rate
s = "Pretrain epoch %i, fixed learning rate: %s (was: %s)" % (epoch, learning_rate, old_learning_rate)
elif 1 < first_non_pretrain_epoch == epoch:
learning_rate = control.default_learning_rate
s = "First epoch after pretrain, epoch %i, fixed learning rate: %s (was %s)" % (
epoch, learning_rate, old_learning_rate)
else:
learning_rate = control.calc_new_learning_rate_for_epoch(epoch)
s = "Calculated learning rate for epoch %i: %s (was: %s)" % (epoch, learning_rate, old_learning_rate)
if learning_rate < control.min_learning_rate:
learning_rate = control.min_learning_rate
s += ", clipped to %s" % learning_rate
s += ", previous relative error: %s" % control.calc_relative_error(epoch - 2, epoch - 1)
if hasattr(control, "_calc_recent_mean_relative_error"):
# noinspection PyProtectedMember
s += ", previous mean relative error: %s" % control._calc_recent_mean_relative_error(epoch)
print(s)
if check_lr and old_learning_rate is not None:
if old_learning_rate != learning_rate:
print("Learning rate is different in epoch %i!" % epoch)
sys.exit(1)
# Overwrite new learning rate so that the calculation for further learning rates stays consistent.
if epoch in control.epoch_data:
control.epoch_data[epoch].learning_rate = learning_rate
else:
control.epoch_data[epoch] = control.EpochData(learningRate=learning_rate)
print("Finished, last stored epoch was %i." % max_epoch)
if __name__ == "__main__":
demo()
You can’t perform that action at this time.