From faca8aa01203df4b1fdc50a00c8eb4788d8432a2 Mon Sep 17 00:00:00 2001 From: Maxim Kochurov Date: Tue, 13 Jun 2017 12:23:39 +0300 Subject: [PATCH 01/24] beginning of refactoring, simple tests pass for MeanField --- pymc3/model.py | 17 +- pymc3/tests/test_variational_inference.py | 10 +- pymc3/variational/approximations.py | 32 +- pymc3/variational/operators.py | 12 +- pymc3/variational/opvi.py | 661 +++++++++++++--------- 5 files changed, 440 insertions(+), 292 deletions(-) diff --git a/pymc3/model.py b/pymc3/model.py index 052e3cc1b3..da415bf25c 100644 --- a/pymc3/model.py +++ b/pymc3/model.py @@ -676,7 +676,7 @@ def profile(self, outs, n=1000, point=None, profile=True, *args, **kwargs): return f.profile - def flatten(self, vars=None): + def flatten(self, vars=None, order=None, inputvar=None): """Flattens model's input and returns: FlatView with * input vector variable @@ -687,6 +687,10 @@ def flatten(self, vars=None): ---------- vars : list of variables or None if None, then all model.free_RVs are used for flattening input + order : ArrayOrdering + Optional, use predefined ordering + inputvar : tt.vector + Optional, use predefined inputvar Returns ------- @@ -694,9 +698,14 @@ def flatten(self, vars=None): """ if vars is None: vars = self.free_RVs - order = ArrayOrdering(vars) - inputvar = tt.vector('flat_view', dtype=theano.config.floatX) - inputvar.tag.test_value = flatten_list(vars).tag.test_value + if order is None: + order = ArrayOrdering(vars) + if inputvar is None: + inputvar = tt.vector('flat_view', dtype=theano.config.floatX) + if vars: + inputvar.tag.test_value = flatten_list(vars).tag.test_value + else: + inputvar.tag.test_value = np.asarray([], inputvar.dtype) replacements = {self.named_vars[name]: inputvar[slc].reshape(shape).astype(dtype) for name, slc, shape, dtype in order.vmap} view = {vm.var: vm for vm in order.vmap} diff --git a/pymc3/tests/test_variational_inference.py b/pymc3/tests/test_variational_inference.py index af3ba2fe09..4915c4980c 100644 --- a/pymc3/tests/test_variational_inference.py +++ b/pymc3/tests/test_variational_inference.py @@ -84,8 +84,8 @@ def test_vars_view(self): _, model, _ = models.multidimensional_model() with model: app = self.inference().approx - posterior = app.random(10) - x_sampled = app.view(posterior, 'x').eval() + posterior = app.random_global(10) + x_sampled = app.view_global(posterior, 'x').eval() assert x_sampled.shape == (10,) + model['x'].dshape def test_vars_view_dynamic_size(self): @@ -94,10 +94,10 @@ def test_vars_view_dynamic_size(self): app = self.inference().approx i = tt.iscalar('i') i.tag.test_value = 1 - posterior = app.random(i) - x_sampled = app.view(posterior, 'x').eval({i: 10}) + posterior = app.random_global(i) + x_sampled = app.view_global(posterior, 'x').eval({i: 10}) assert x_sampled.shape == (10,) + model['x'].dshape - x_sampled = app.view(posterior, 'x').eval({i: 1}) + x_sampled = app.view_global(posterior, 'x').eval({i: 1}) assert x_sampled.shape == (1,) + model['x'].dshape def test_vars_view_dynamic_size_numpy(self): diff --git a/pymc3/variational/approximations.py b/pymc3/variational/approximations.py index cd3ee0672d..7f29b98d08 100644 --- a/pymc3/variational/approximations.py +++ b/pymc3/variational/approximations.py @@ -80,22 +80,26 @@ def create_shared_params(self, **kwargs): 'rho': theano.shared( pm.floatX(np.zeros((self.global_size,))), 'rho')} - def log_q_W_global(self, z): + @property + @memoize + def symbolic_random_global_matrix(self): + initial = self._symbolic_initial_global_matrix + sd = rho2sd(self.rho) + mu = self.mean + return sd * initial + mu + + @property + @memoize + def _symbolic_log_q_W_global(self): """ log_q_W samples over q for global vars """ mu = self.scale_grad(self.mean) rho = self.scale_grad(self.rho) - z = z[self.global_slc] + z = self.symbolic_random_global_matrix logq = tt.sum(log_normal(z, mu, rho=rho)) return logq - def random_global(self, size=None, no_rand=False): - initial = self.initial(size, no_rand, l=self.global_size) - sd = rho2sd(self.rho) - mu = self.mean - return sd * initial + mu - class FullRank(Approximation): """Full Rank approximation to the posterior where Multivariate Gaussian family @@ -203,9 +207,9 @@ def log_q_W_global(self, z): z = z[self.global_slc] return log_normal_mv(z, mu, chol=L, gpu_compat=self.gpu_compat) - def random_global(self, size=None, no_rand=False): + def random_global(self, size=None, deterministic=False): # (samples, dim) or (dim, ) - initial = self.initial(size, no_rand, l=self.global_size).T + initial = self.initial(size, deterministic, l=self.global_size).T # (dim, dim) L = self.L # (dim, ) @@ -322,14 +326,14 @@ def randidx(self, size=None): high=pm.floatX(self.histogram.shape[0]) - pm.floatX(1e-16)) .astype('int32')) - def random_global(self, size=None, no_rand=False): - theano_condition_is_here = isinstance(no_rand, tt.Variable) + def random_global(self, size=None, deterministic=False): + theano_condition_is_here = isinstance(deterministic, tt.Variable) if theano_condition_is_here: - return tt.switch(no_rand, + return tt.switch(deterministic, self.mean, self.histogram[self.randidx(size)]) else: - if no_rand: + if deterministic: return self.mean else: return self.histogram[self.randidx(size)] diff --git a/pymc3/variational/operators.py b/pymc3/variational/operators.py index 26933a7982..795a6fcb27 100644 --- a/pymc3/variational/operators.py +++ b/pymc3/variational/operators.py @@ -19,9 +19,8 @@ class KL(Operator): KL[q(v)||p(v)] = \int q(v)\log\\frac{q(v)}{p(v)}dv """ - def apply(self, f): - z = self.input - return self.logq_norm(z) - self.logp_norm(z) + def apply(self, f, nmc): + return self.logq_norm(nmc) - self.logp_norm(nmc) # SVGD Implementation @@ -48,12 +47,12 @@ def get_input(self, n_mc): _warn_not_used('n_mc', self.op) return self.approx.histogram elif n_mc is not None and n_mc > 1: - return self.approx.random(n_mc) + return self.approx.random_total(n_mc) else: raise ValueError('Variational type approximation requires ' 'sample size (`n_mc` : int > 1 should be passed)') - def __call__(self, z, **kwargs): + def __call__(self, nmc, **kwargs): op = self.op # type: KSD grad = op.apply(self.tf) if 'more_obj_params' in kwargs: @@ -99,14 +98,13 @@ class KSD(Operator): def __init__(self, approx, temperature=1): Operator.__init__(self, approx) self.temperature = temperature - self.input_matrix = tt.matrix('KSD input matrix') def apply(self, f): # f: kernel function for KSD f(histogram) -> (k(x,.), \nabla_x k(x,.)) stein = Stein( approx=self.approx, kernel=f, - input_matrix=self.input_matrix, + input_matrix=self.approx.symbolic_random_total_matrix, temperature=self.temperature) return pm.floatX(-1) * stein.grad diff --git a/pymc3/variational/opvi.py b/pymc3/variational/opvi.py index e43aaf329f..2bf57554e0 100644 --- a/pymc3/variational/opvi.py +++ b/pymc3/variational/opvi.py @@ -39,7 +39,10 @@ import pymc3 as pm from .updates import adagrad_window from ..distributions.dist_math import rho2sd, log_normal -from ..model import modelcontext, ArrayOrdering, DictToArrayBijection +from ..model import modelcontext +from ..blocking import ( + ArrayOrdering, DictToArrayBijection, +) from ..util import get_default_varnames from ..theanof import tt_rng, memoize, change_flags, identity @@ -82,21 +85,6 @@ def __init__(self, op, tf): test_params = property(lambda self: self.tf.params) approx = property(lambda self: self.op.approx) - def random(self, size=None): - """ - Posterior distribution from initial latent space - - Parameters - ---------- - size : `int` - number of samples from distribution - - Returns - ------- - posterior space (theano) - """ - return self.op.approx.random(size) - def updates(self, obj_n_mc=None, tf_n_mc=None, obj_optimizer=adagrad_window, test_optimizer=adagrad_window, more_obj_params=None, more_tf_params=None, more_updates=None, more_replacements=None, total_grad_norm_constraint=None): @@ -163,8 +151,7 @@ def add_test_updates(self, updates, tf_n_mc=None, test_optimizer=adagrad_window, more_tf_params = [] if more_replacements is None: more_replacements = dict() - tf_z = self.get_input(tf_n_mc) - tf_target = self(tf_z, more_tf_params=more_tf_params) + tf_target = self(tf_n_mc, more_tf_params=more_tf_params) tf_target = theano.clone(tf_target, more_replacements, strict=False) grads = pm.updates.get_or_compute_grads(tf_target, self.obj_params + more_tf_params) if total_grad_norm_constraint is not None: @@ -182,8 +169,7 @@ def add_obj_updates(self, updates, obj_n_mc=None, obj_optimizer=adagrad_window, more_obj_params = [] if more_replacements is None: more_replacements = dict() - obj_z = self.get_input(obj_n_mc) - obj_target = self(obj_z, more_obj_params=more_obj_params) + obj_target = self(obj_n_mc, more_obj_params=more_obj_params) obj_target = theano.clone(obj_target, more_replacements, strict=False) grads = pm.updates.get_or_compute_grads(obj_target, self.obj_params + more_obj_params) if total_grad_norm_constraint is not None: @@ -196,9 +182,6 @@ def add_obj_updates(self, updates, obj_n_mc=None, obj_optimizer=adagrad_window, if self.op.RETURNS_LOSS: updates.loss = obj_target - def get_input(self, n_mc): - return self.random(n_mc) - @memoize @change_flags(compute_test_value='off') def step_function(self, obj_n_mc=None, tf_n_mc=None, @@ -288,7 +271,7 @@ def score_function(self, sc_n_mc=None, more_replacements=None, fn_kwargs=None): if more_replacements is None: more_replacements = {} loss = theano.clone( - self(self.random(sc_n_mc)), + self(sc_n_mc), more_replacements, strict=False) return theano.function([], loss, **fn_kwargs) @@ -299,21 +282,13 @@ def __getstate__(self): def __setstate__(self, state): self.__init__(*state) - def __call__(self, z, **kwargs): + def __call__(self, nmc, **kwargs): if 'more_tf_params' in kwargs: - m = -1 - else: - m = 1 - if z.ndim > 1: - a = theano.scan( - lambda z_: theano.clone( - self.op.apply(self.tf), - {self.op.input: z_}, strict=False), - sequences=z, n_steps=z.shape[0])[0].mean() + m = -1. else: - a = theano.clone( - self.op.apply(self.tf), - {self.op.input: z}, strict=False) + m = 1. + a = self.op.apply(self.tf) + a = self.approx.set_size_deterministic(a, nmc, 0) return m * self.op.T(a) @@ -364,6 +339,9 @@ def apply(self, f): # pragma: no cover function that takes `z = self.input` and returns same dimensional output + nmc : n + monte carlo samples to use + Returns ------- `TensorVariable` @@ -543,8 +521,10 @@ class Approximation(object): - Kingma, D. P., & Welling, M. (2014). Auto-Encoding Variational Bayes. stat, 1050, 1. """ - initial_dist_name = 'normal' - initial_dist_map = 0. + initial_dist_local_name = 'normal' + initial_dist_global_name = 'normal' + initial_dist_local_map = 0. + initial_dist_global_map = 0. def __init__(self, local_rv=None, model=None, cost_part_grad_scale=1, @@ -571,14 +551,48 @@ def get_transformed(v): self.known = known self.local_vars = self.get_local_vars(**kwargs) self.global_vars = self.get_global_vars(**kwargs) - self.order = ArrayOrdering(self.local_vars + self.global_vars) - self.gbij = DictToArrayBijection(ArrayOrdering(self.global_vars), {}) - self.lbij = DictToArrayBijection(ArrayOrdering(self.local_vars), {}) - self.flat_view = model.flatten( - vars=self.local_vars + self.global_vars + self._g_order = ArrayOrdering(self.global_vars) + self._l_order = ArrayOrdering(self.local_vars) + self.gbij = DictToArrayBijection(self._g_order, {}) + self._symbolic_initial_local_matrix = tt.matrix(self.__class__.__name__ + '_symbolic_initial_local_matrix') + self._symbolic_initial_local_matrix.tag.test_value = np.random.rand(1, self.local_size).astype( + self._symbolic_initial_local_matrix.dtype + ) + self._symbolic_initial_global_matrix = tt.matrix(self.__class__.__name__ + '_symbolic_initial_global_matrix') + self._symbolic_initial_global_matrix.tag.test_value = np.random.rand(1, self.global_size).astype( + self._symbolic_initial_global_matrix.dtype + ) + + self.global_flat_view = model.flatten( + vars=self.global_vars, + order=self._g_order, + inputvar=self._symbolic_initial_global_matrix + ) + self.local_flat_view = model.flatten( + vars=self.local_vars, + order=self._l_order, + inputvar=self._symbolic_initial_local_matrix ) self._setup(**kwargs) self.shared_params = self.create_shared_params(**kwargs) + self._n_samples = self._symbolic_initial_global_matrix.shape[0] + + _global_view = property(lambda self: self.global_flat_view.view) + _local_view = property(lambda self: self.local_flat_view.view) + local_input = property(lambda self: self.local_flat_view.input) + global_input = property(lambda self: self.global_flat_view.input) + + local_names = property(lambda self: tuple(v.name for v in self.local_vars)) + global_names = property(lambda self: tuple(v.name for v in self.global_vars)) + + @staticmethod + def _choose_alternative(part, loc, glob): + if part == 'local': + return loc + elif part == 'global': + return glob + else: + raise ValueError("part is restricted to be in {'local', 'global'}, got %r" % part) def seed(self, random_seed=None): """ @@ -592,16 +606,6 @@ def seed(self, random_seed=None): self._seed = random_seed self._rng.seed(random_seed) - @property - def normalizing_constant(self): - t = self.to_flat_input( - tt.max([v.scaling for v in self.model.basic_RVs])) - t = theano.clone(t, {self.input: tt.zeros(self.total_size)}) - # if not scale_cost_to_minibatch: t=1 - t = tt.switch(self.scale_cost_to_minibatch, t, - tt.constant(1, dtype=t.dtype)) - return pm.floatX(t) - def _setup(self, **kwargs): pass @@ -611,9 +615,6 @@ def get_global_vars(self, **kwargs): def get_local_vars(self, **kwargs): return [v for v in self.model.free_RVs if v in self.known] - _view = property(lambda self: self.flat_view.view) - input = property(lambda self: self.flat_view.input) - def check_model(self, model, **kwargs): """Checks that model is valid for variational inference """ @@ -633,16 +634,6 @@ def create_shared_params(self, **kwargs): """ pass - def _local_mu_rho(self): - mu = [] - rho = [] - for var in self.local_vars: - mu.append(self.known[var][0].ravel()) - rho.append(self.known[var][1].ravel()) - mu = tt.concatenate(mu) - rho = tt.concatenate(rho) - return mu, rho - def construct_replacements(self, include=None, exclude=None, more_replacements=None): """Construct replacements with given conditions @@ -664,18 +655,28 @@ def construct_replacements(self, include=None, exclude=None, if include is not None and exclude is not None: raise ValueError( 'Only one parameter is supported {include|exclude}, got two') + _replacements = dict() + _replacements.update(self.global_flat_view.replacements) + _replacements.update(self.local_flat_view.replacements) if include is not None: # pragma: no cover replacements = {k: v for k, v - in self.flat_view.replacements.items() if k in include} + in _replacements.items() if k in include} elif exclude is not None: # pragma: no cover replacements = {k: v for k, v - in self.flat_view.replacements.items() if k not in exclude} + in _replacements.items() if k not in exclude} else: - replacements = self.flat_view.replacements.copy() + replacements = _replacements if more_replacements is not None: # pragma: no cover replacements.update(more_replacements) return replacements + def to_flat_input(self, node): + """ + Replaces vars with flattened view stored in self.input + """ + replacements = self.construct_replacements() + return theano.clone(node, replacements, strict=False) + def apply_replacements(self, node, deterministic=False, include=None, exclude=None, more_replacements=None): @@ -694,6 +695,8 @@ def apply_replacements(self, node, deterministic=False, latent variables to be excluded for replacements more_replacements : `dict` add custom replacements to graph, e.g. change input source + new : bool + reinit random generator for replacements? Returns ------- @@ -703,8 +706,12 @@ def apply_replacements(self, node, deterministic=False, include, exclude, more_replacements ) node = theano.clone(node, replacements, strict=False) - posterior = self.random(no_rand=deterministic) - return theano.clone(node, {self.input: posterior}, strict=False) + posterior_glob = self.random_global(deterministic=deterministic) + posterior_loc = self.random_local(deterministic=deterministic) + return theano.clone(node, { + self.global_input: posterior_glob, + self.local_input: posterior_loc + }, strict=False) def sample_node(self, node, size=100, more_replacements=None): @@ -724,11 +731,32 @@ def sample_node(self, node, size=100, """ if more_replacements is not None: # pragma: no cover node = theano.clone(node, more_replacements, strict=False) - posterior = self.random(size) + if size is None: + size = 1 + posterior_loc = self.random_local(size) + posterior_glob = self.random_global(size) node = self.to_flat_input(node) - def sample(z): return theano.clone(node, {self.input: z}, strict=False) - nodes, _ = theano.scan(sample, posterior, n_steps=size) + def sample(zl, zg): + return theano.clone(node, { + self.local_input: zl, + self.global_input: zg + }, strict=False) + nodes, _ = theano.scan(sample, [posterior_loc, posterior_glob], n_steps=size) + return nodes + + def _sample_over_posterior(self, node): + node = self.to_flat_input(node) + posterior_loc = self.symbolic_random_local_matrix + posterior_glob = self.symbolic_random_global_matrix + def sample(zl, zg): + return theano.clone(node, { + self.local_input: zl, + self.global_input: zg + }, strict=False) + nodes, _ = theano.scan( + sample, [posterior_loc, posterior_glob], + n_steps=posterior_glob.shape[0]) return nodes def scale_grad(self, inp): @@ -742,156 +770,113 @@ def scale_grad(self, inp): """ return theano.gradient.grad_scale(inp, self.cost_part_grad_scale) - def to_flat_input(self, node): - """ - Replaces vars with flattened view stored in self.input - """ - return theano.clone(node, self.flat_view.replacements, strict=False) - @property def params(self): return cast_to_list(self.shared_params) - def initial(self, size, no_rand=False, l=None): - """Initial distribution for constructing posterior - - Parameters - ---------- - size : `int` - number of samples - no_rand : `bool` - return zeros if True - l : `int` - length of sample, defaults to latent space dim - - Returns - ------- - `tt.TensorVariable` - sampled latent space - """ - - theano_condition_is_here = isinstance(no_rand, tt.Variable) - if l is None: # pragma: no cover - l = self.total_size + def _random_part(self, part, size=None, deterministic=False): + r_part = self._choose_alternative( + part, + self.symbolic_random_local_matrix, + self.symbolic_random_global_matrix + ) + if not isinstance(deterministic, tt.Variable): + deterministic = np.int8(deterministic) if size is None: - shape = (l, ) + i_size = np.int32(1) + else: + i_size = size + r_part = self.set_size_deterministic(r_part, i_size, deterministic) + if size is None: + r_part = r_part[0] + return r_part + + def _initial_part_matrix(self, part, size, deterministic): + length = self._choose_alternative( + part, + self.local_size, + self.global_size + ) + dist_name = self._choose_alternative( + part, + self.initial_dist_local_name, + self.initial_dist_global_name, + ) + dist_map = self._choose_alternative( + part, + self.initial_dist_local_map, + self.initial_dist_global_map, + ) + length = tt.as_tensor(length) + size = tt.as_tensor(size) + shape = (size, length) + dtype = self._symbolic_initial_global_matrix.dtype + if not isinstance(deterministic, tt.Variable): + if deterministic: + return tt.ones(shape, dtype) * dist_map + else: + return getattr(self._rng, dist_name)(shape) else: - shape = (size, l) - shape = tt.stack(*shape) - if theano_condition_is_here: - no_rand = tt.as_tensor(no_rand) - sample = getattr(self._rng, self.initial_dist_name)(shape) - space = tt.switch( - no_rand, - tt.ones_like(sample) * self.initial_dist_map, + deterministic = tt.as_tensor(deterministic) + sample = getattr(self._rng, dist_name)(shape) + initial = tt.switch( + deterministic, + tt.ones(shape, dtype) * dist_map, sample ) - else: - if no_rand: - return tt.ones(shape) * self.initial_dist_map - else: - return getattr(self._rng, self.initial_dist_name)(shape) - return space + return initial - def random_local(self, size=None, no_rand=False): + def random_local(self, size=None, deterministic=False): """Implements posterior distribution from initial latent space Parameters ---------- size : `scalar` number of samples from distribution - no_rand : `bool` + deterministic : `bool` whether use deterministic distribution Returns ------- local posterior space """ + return self._random_part('local', size=size, deterministic=deterministic) - mu, rho = self._local_mu_rho() - e = self.initial(size, no_rand, self.local_size) - return e * rho2sd(rho) + mu - - def random_global(self, size=None, no_rand=False): # pragma: no cover + def random_global(self, size=None, deterministic=False): # pragma: no cover """Implements posterior distribution from initial latent space Parameters ---------- size : `scalar` number of samples from distribution - no_rand : `bool` + deterministic : `bool` whether use deterministic distribution Returns ------- global posterior space """ - raise NotImplementedError - - def random(self, size=None, no_rand=False): - """Implements posterior distribution from initial latent space - - Parameters - ---------- - size : `scalar` - number of samples from distribution - no_rand : `bool` - whether use deterministic distribution - - Returns - ------- - posterior space (theano) - """ - if size is None: - ax = 0 - else: - ax = 1 - if self.local_vars and self.global_vars: - return tt.concatenate([ - self.random_local(size, no_rand), - self.random_global(size, no_rand) - ], axis=ax) - elif self.local_vars: # pragma: no cover - return self.random_local(size, no_rand) - elif self.global_vars: - return self.random_global(size, no_rand) - else: # pragma: no cover - raise ValueError('No FreeVARs in model') + return self._random_part('global', size=size, deterministic=deterministic) @property @memoize - @change_flags(compute_test_value='off') - def random_fn(self): - """Implements posterior distribution from initial latent space - - Parameters - ---------- - size : `int` - number of samples from distribution - no_rand : `bool` - whether use deterministic distribution - - Returns - ------- - posterior space (numpy) - """ - In = theano.In - size = tt.iscalar('size') - no_rand = tt.bscalar('no_rand') - posterior = self.random(size, no_rand) - fn = theano.function([In(size, 'size', 1, allow_downcast=True), - In(no_rand, 'no_rand', 0, allow_downcast=True)], - posterior) - - def inner(size=None, no_rand=False): - if size is None: - return fn(1, int(no_rand))[0] - else: - return fn(size, int(no_rand)) - + def sample_dict_fn(self): + s = tt.iscalar() + l_posterior = self.random_local(s) + g_posterior = self.random_global(s) + sampled = ([self.view_global(g_posterior, name) + for name in self.global_names] + + [self.view_local(l_posterior, name) + for name in self.local_names] + ) + sample_fn = theano.function([theano.In(s, 'draws', 1)], sampled) + + def inner(draws=1): + _samples = sample_fn(draws) + return dict([(n_, s_) for n_, s_ in zip(self.global_names+self.local_names, _samples)]) return inner - def sample(self, draws=1, include_transformed=False): + def sample(self, draws=500, include_transformed=False): """Draw samples from variational posterior. Parameters @@ -908,14 +893,11 @@ def sample(self, draws=1, include_transformed=False): """ vars_sampled = get_default_varnames(self.model.unobserved_RVs, include_transformed=include_transformed) - posterior = self.random_fn(draws) - names = [var.name for var in self.local_vars + self.global_vars] - samples = {name: self.view(posterior, name) - for name in names} + samples = self.sample_dict_fn(draws) # type: dict def points(): for i in range(draws): - yield {name: samples[name][i] for name in names} + yield {name: samples[name][i] for name in samples.keys()} trace = pm.sampling.NDArray(model=self.model, vars=vars_sampled) try: @@ -926,54 +908,223 @@ def points(): trace.close() return pm.sampling.MultiTrace([trace]) - def log_q_W_local(self, z): + @property + @memoize + def __local_mu_rho(self): + if not self.local_vars: + mu, rho = ( + tt.constant(pm.floatX(np.asarray([]))), + tt.constant(pm.floatX(np.asarray([]))) + ) + else: + mu = [] + rho = [] + for var in self.local_vars: + mu.append(self.known[var][0].ravel()) + rho.append(self.known[var][1].ravel()) + mu = tt.concatenate(mu) + rho = tt.concatenate(rho) + mu.name = self.__class__.__name__ + '_local_mu' + rho.name = self.__class__.__name__ + '_local_rho' + return mu, rho + + @property + @memoize + def normalizing_constant(self): + """ + Constant to divide when we want to scale down loss from minibatches + """ + t = self.to_flat_input( + tt.max([v.scaling for v in self.model.basic_RVs])) + t = theano.clone(t, { + self.global_input: self.symbolic_random_global_matrix[0], + self.local_input: self.symbolic_random_local_matrix[0] + }) + t = self.set_size_deterministic(t, 1, 1) # remove random, we do not it here at all + # if not scale_cost_to_minibatch: t=1 + t = tt.switch(self.scale_cost_to_minibatch, t, + tt.constant(1, dtype=t.dtype)) + return pm.floatX(t) + + def set_size_deterministic(self, smth, s, d): + """ + Replaces self._n_samples and self._deterministic_flag + with non symbolic input. Used whenever user specifies + `sample size` and `deterministic` option + """ + initial_local = self._initial_part_matrix('local', s, d) + initial_global = self._initial_part_matrix('global', s, d) + return theano.clone(smth, { + self._symbolic_initial_local_matrix: initial_local, + self._symbolic_initial_global_matrix: initial_global, + }, strict=False) + + @property + @memoize + def symbolic_random_global_matrix(self): + raise NotImplementedError + + @property + @memoize + def symbolic_random_local_matrix(self): + mu, rho = self.__local_mu_rho + e = self._symbolic_initial_local_matrix + return e * rho2sd(rho) + mu + + @property + @memoize + def symbolic_random_total_matrix(self): + if self.local_vars and self.global_vars: + return tt.stack([ + self.symbolic_random_local_matrix, + self.symbolic_random_global_matrix, + ], axis=-1) + elif self.local_vars: + return self.symbolic_random_local_matrix + elif self.global_vars: + return self.symbolic_random_global_matrix + else: + raise TypeError('No free vars in the Model') + + @property + @memoize + def _symbolic_log_q_W_local(self): """log_q_W samples over q for local vars Gradient wrt mu, rho in density parametrization can be scaled to lower variance of ELBO """ - if not self.local_vars: - return tt.constant(0) - mu, rho = self._local_mu_rho() + mu, rho = self.__local_mu_rho mu = self.scale_grad(mu) rho = self.scale_grad(rho) - logp = log_normal(z[self.local_slc], mu, rho=rho) + z = self.symbolic_random_local_matrix + logp = log_normal(z, mu, rho=rho) scaling = [] for var in self.local_vars: - scaling.append(tt.repeat(var.scaling, var.dsize)) + scaling.append(tt.repeat(var.scaling, var.size)) scaling = tt.concatenate(scaling) + # we need only dimensions here + # from incoming unobserved + # to get rid of input_view + # I replace it with the first row + # of total_random matrix + # that always exists + scaling = theano.clone(scaling, { + self.local_input: self.symbolic_random_local_matrix[0] + }) logp *= scaling - return self.to_flat_input(tt.sum(logp)) + logp = logp.sum() + return logp - def log_q_W_global(self, z): # pragma: no cover - """log_q_W samples over q for global vars - """ + @property + @memoize + def _symbolic_log_q_W_global(self): raise NotImplementedError - def logq(self, z): + @property + @memoize + def _symbolic_log_q_W(self): + q_w_local = self._symbolic_log_q_W_local + q_w_global = self._symbolic_log_q_W_global + return q_w_global + q_w_local + + def logq(self, nmc=None): """Total logq for approximation """ - return self.log_q_W_global(z) + self.log_q_W_local(z) - - def logq_norm(self, z): - return self.logq(z) / self.normalizing_constant - - def logp(self, z): - factors = ([tt.sum(var.logpt)for var in self.model.basic_RVs] + - [tt.sum(var) for var in self.model.potentials]) - p = self.to_flat_input(tt.add(*factors)) - p = theano.clone(p, {self.input: z}) - return p - - def logp_norm(self, z): - t = self.normalizing_constant - factors = ([tt.sum(var.logpt) / t for var in self.model.basic_RVs] + - [tt.sum(var) / t for var in self.model.potentials]) - logpt = tt.add(*factors) - p = self.to_flat_input(logpt) - p = theano.clone(p, {self.input: z}) - return p - - def view(self, space, name, reshape=True): + if nmc is None: + nmc = 1 + log_q = self._symbolic_log_q_W / pm.floatX(self._n_samples) + return self.set_size_deterministic(log_q, nmc, 0) + + def logq_norm(self, nmc=None): + return self.logq(nmc) / self.normalizing_constant + + @property + @memoize + def sized_symbolic_logp_local(self): + local_post = self.symbolic_random_local_matrix + free_logp_local = tt.sum([ + var.distribution.logp( + self.view_local(local_post, var.name, reshape=True) + ) for var in self.model.free_RVs if var.name in self.local_names + ]) + return free_logp_local + + @property + @memoize + def sized_symbolic_logp_global(self): + global_post = self.symbolic_random_global_matrix + free_logp_global = tt.sum([ + var.distribution.logp( + self.view_local(global_post, var.name, reshape=True) + ) for var in self.model.free_RVs if var.name in self.global_names + ]) + return free_logp_global + + @property + @memoize + def sized_symbolic_logp_observed(self): + observed_logp = tt.sum([ + var.logpt + for var in self.model.observed_RVs + ]) + observed_logp = self._sample_over_posterior( + observed_logp + ) + return observed_logp + + @property + @memoize + def sized_symbolic_logp_potentials(self): + potentials = [tt.sum(var) for var in self.model.potentials] + potentials = self.sample_node(potentials, size=self._n_samples).sum() + return potentials + + @property + @memoize + def sized_symbolic_logp(self): + return (self.sized_symbolic_logp_local + + self.sized_symbolic_logp_global + + self.sized_symbolic_logp_observed + + self.sized_symbolic_logp_potentials) + + @property + @memoize + def single_symbolic_logp(self): + return self.apply_replacements(self.model.logpt) + + def logp(self, nmc=None): + if nmc is None: + _logp = self.single_symbolic_logp + nmc = 1 + else: + _logp = self.sized_symbolic_logp + _logp = _logp / pm.floatX(self._n_samples) + return self.set_size_deterministic(_logp, nmc, 0) + + def logp_norm(self, nmc): + return self.logp(nmc) / self.normalizing_constant + + def _view_part(self, part, space, name, reshape=True): + theano_is_here = isinstance(space, tt.Variable) + if not theano_is_here: + raise TypeError('View on numpy arrays is not supported') + if part == 'global': + _, slc, _shape, dtype = self._global_view[name] + elif part == 'local': + _, slc, _shape, dtype = self._local_view[name] + else: + raise ValueError("%r part is not supported, you can use only {'local', 'global'}") + if space.ndim > 2: + raise ValueError('Space should have <= 2 dimensions, got %r' % space.ndim) + view = space[..., slc] + if reshape: + shape = np.asarray((-1,) + _shape, int) + view = view.reshape(shape, ndim=len(_shape) + 1) + if space.ndim == 1: + view = view[0] + return view.astype(dtype) + + def view_global(self, space, name, reshape=True): """Construct view on a variable from flattened `space` Parameters @@ -990,51 +1141,37 @@ def view(self, space, name, reshape=True): (reshaped) slice of matrix variable view """ - theano_is_here = isinstance(space, tt.TensorVariable) - slc = self._view[name].slc - _, _, _shape, dtype = self._view[name] - if space.ndim == 2: - view = space[:, slc] - elif space.ndim < 2: - view = space[slc] - else: # pragma: no cover - raise ValueError( - 'Space should have no more than 2 dims, got %d' % - space.ndim) - if reshape: - if len(_shape) > 0: - if theano_is_here: - shape = tt.concatenate([space.shape[:-1], - tt.as_tensor(_shape)]) - else: - shape = np.concatenate([space.shape[:-1], - _shape]).astype(int) + return self._view_part('global', space, name, reshape) - else: - shape = space.shape[:-1] - if theano_is_here: - view = view.reshape(shape, ndim=space.ndim + len(_shape) - 1) - else: - view = view.reshape(shape) - return view.astype(dtype) + def view_local(self, space, name, reshape=True): + """Construct view on a variable from flattened `space` + + Parameters + ---------- + space : matrix or vector + space to take view of variable from + name : `str` + name of variable + reshape : `bool` + whether to reshape variable from vectorized view + + Returns + ------- + (reshaped) slice of matrix + variable view + """ + return self._view_part('local', space, name, reshape) @property + @memoize def total_size(self): - return self.order.dimensions + return self.local_size + self.global_size @property + @memoize def local_size(self): - size = sum([0] + [v.dsize for v in self.local_vars]) - return size + return self._l_order.dimensions @property def global_size(self): - return self.total_size - self.local_size - - @property - def local_slc(self): - return slice(0, self.local_size) - - @property - def global_slc(self): - return slice(self.local_size, self.total_size) + return self._g_order.dimensions From 12e6173b3b8114b618fd6836175bd7e01b62998d Mon Sep 17 00:00:00 2001 From: Maxim Kochurov Date: Fri, 16 Jun 2017 01:54:37 +0300 Subject: [PATCH 02/24] more tests pass --- pymc3/tests/conftest.py | 10 +- pymc3/tests/test_variational_inference.py | 17 +- pymc3/variational/approximations.py | 24 ++- pymc3/variational/operators.py | 6 +- pymc3/variational/opvi.py | 223 ++++++++++++---------- 5 files changed, 148 insertions(+), 132 deletions(-) diff --git a/pymc3/tests/conftest.py b/pymc3/tests/conftest.py index 6081bca1e9..cac0f3d9b8 100644 --- a/pymc3/tests/conftest.py +++ b/pymc3/tests/conftest.py @@ -34,7 +34,15 @@ def theano_config(): yield -@pytest.fixture(scope='function') +@pytest.fixture(scope='session', autouse=True) +def exception_verbosity(): + config = theano.configparser.change_flags( + exception_verbosity='high') + with config: + yield + + +@pytest.fixture(scope='session') def strict_float32(): config = theano.configparser.change_flags( warn_float64='raise', diff --git a/pymc3/tests/test_variational_inference.py b/pymc3/tests/test_variational_inference.py index 4915c4980c..7ca2d149d9 100644 --- a/pymc3/tests/test_variational_inference.py +++ b/pymc3/tests/test_variational_inference.py @@ -30,13 +30,13 @@ def test_elbo(): # Create variational gradient tensor mean_field = MeanField(model=model) - elbo = -KL(mean_field)()(mean_field.random()) + elbo = -KL(mean_field)()(10000) mean_field.shared_params['mu'].set_value(post_mu) mean_field.shared_params['rho'].set_value(np.log(np.exp(post_sd) - 1)) f = theano.function([], elbo) - elbo_mc = sum(f() for _ in range(10000)) / 10000 + elbo_mc = f() # Exact value elbo_true = (-0.5 * ( @@ -100,19 +100,6 @@ def test_vars_view_dynamic_size(self): x_sampled = app.view_global(posterior, 'x').eval({i: 1}) assert x_sampled.shape == (1,) + model['x'].dshape - def test_vars_view_dynamic_size_numpy(self): - _, model, _ = models.multidimensional_model() - with model: - app = self.inference().approx - i = tt.iscalar('i') - i.tag.test_value = 1 - x_sampled = app.view(app.random_fn(10), 'x') - assert x_sampled.shape == (10,) + model['x'].dshape - x_sampled = app.view(app.random_fn(1), 'x') - assert x_sampled.shape == (1,) + model['x'].dshape - x_sampled = app.view(app.random_fn(), 'x') - assert x_sampled.shape == () + model['x'].dshape - def test_sample(self): n_samples = 100 xs = np.random.binomial(n=1, p=0.2, size=n_samples) diff --git a/pymc3/variational/approximations.py b/pymc3/variational/approximations.py index 7f29b98d08..71cdab2898 100644 --- a/pymc3/variational/approximations.py +++ b/pymc3/variational/approximations.py @@ -5,7 +5,7 @@ import pymc3 as pm from pymc3.distributions.dist_math import rho2sd, log_normal, log_normal_mv from pymc3.variational.opvi import Approximation -from pymc3.theanof import memoize +from pymc3.theanof import memoize, change_flags __all__ = [ @@ -82,6 +82,7 @@ def create_shared_params(self, **kwargs): @property @memoize + @change_flags(compute_test_value='off') def symbolic_random_global_matrix(self): initial = self._symbolic_initial_global_matrix sd = rho2sd(self.rho) @@ -90,15 +91,16 @@ def symbolic_random_global_matrix(self): @property @memoize - def _symbolic_log_q_W_global(self): + @change_flags(compute_test_value='off') + def symbolic_log_q_W_global(self): """ log_q_W samples over q for global vars """ mu = self.scale_grad(self.mean) rho = self.scale_grad(self.rho) z = self.symbolic_random_global_matrix - logq = tt.sum(log_normal(z, mu, rho=rho)) - return logq + logq = log_normal(z, mu, rho=rho) + return logq.sum(1) class FullRank(Approximation): @@ -199,17 +201,23 @@ def create_shared_params(self, **kwargs): 'L_tril': theano.shared(L_tril, 'L_tril') } - def log_q_W_global(self, z): + @property + @memoize + @change_flags(compute_test_value='off') + def log_q_W_global(self): """log_q_W samples over q for global vars """ mu = self.scale_grad(self.mean) L = self.scale_grad(self.L) - z = z[self.global_slc] + z = self.symbolic_random_global_matrix return log_normal_mv(z, mu, chol=L, gpu_compat=self.gpu_compat) - def random_global(self, size=None, deterministic=False): + @property + @memoize + @change_flags(compute_test_value='off') + def symbolic_random_global_matrix(self): # (samples, dim) or (dim, ) - initial = self.initial(size, deterministic, l=self.global_size).T + initial = self._symbolic_initial_global_matrix.T # (dim, dim) L = self.L # (dim, ) diff --git a/pymc3/variational/operators.py b/pymc3/variational/operators.py index 795a6fcb27..56f9f70dbb 100644 --- a/pymc3/variational/operators.py +++ b/pymc3/variational/operators.py @@ -19,8 +19,8 @@ class KL(Operator): KL[q(v)||p(v)] = \int q(v)\log\\frac{q(v)}{p(v)}dv """ - def apply(self, f, nmc): - return self.logq_norm(nmc) - self.logp_norm(nmc) + def apply(self, f): + return self.logq_norm - self.logp_norm # SVGD Implementation @@ -60,7 +60,7 @@ def __call__(self, nmc, **kwargs): else: params = self.test_params + kwargs['more_tf_params'] grad *= pm.floatX(-1) - grad = theano.clone(grad, {op.input_matrix: z}) + z = op.approx.symbolic_random_total_matrix grad = tt.grad(None, params, known_grads={z: grad}) return grad diff --git a/pymc3/variational/opvi.py b/pymc3/variational/opvi.py index 2bf57554e0..6ac07d719d 100644 --- a/pymc3/variational/opvi.py +++ b/pymc3/variational/opvi.py @@ -35,7 +35,7 @@ import numpy as np import theano import theano.tensor as tt - +from theano.ifelse import ifelse import pymc3 as pm from .updates import adagrad_window from ..distributions.dist_math import rho2sd, log_normal @@ -288,7 +288,7 @@ def __call__(self, nmc, **kwargs): else: m = 1. a = self.op.apply(self.tf) - a = self.approx.set_size_deterministic(a, nmc, 0) + a = self.approx.set_size_and_deterministic(a, nmc, 0) return m * self.op.T(a) @@ -555,27 +555,27 @@ def get_transformed(v): self._l_order = ArrayOrdering(self.local_vars) self.gbij = DictToArrayBijection(self._g_order, {}) self._symbolic_initial_local_matrix = tt.matrix(self.__class__.__name__ + '_symbolic_initial_local_matrix') - self._symbolic_initial_local_matrix.tag.test_value = np.random.rand(1, self.local_size).astype( + self._symbolic_initial_local_matrix.tag.test_value = np.random.rand(2, self.local_size).astype( self._symbolic_initial_local_matrix.dtype ) self._symbolic_initial_global_matrix = tt.matrix(self.__class__.__name__ + '_symbolic_initial_global_matrix') - self._symbolic_initial_global_matrix.tag.test_value = np.random.rand(1, self.global_size).astype( + self._symbolic_initial_global_matrix.tag.test_value = np.random.rand(2, self.global_size).astype( self._symbolic_initial_global_matrix.dtype ) self.global_flat_view = model.flatten( vars=self.global_vars, order=self._g_order, - inputvar=self._symbolic_initial_global_matrix + # inputvar=self._symbolic_initial_global_matrix ) self.local_flat_view = model.flatten( vars=self.local_vars, order=self._l_order, - inputvar=self._symbolic_initial_local_matrix + # inputvar=self._symbolic_initial_local_matrix ) self._setup(**kwargs) self.shared_params = self.create_shared_params(**kwargs) - self._n_samples = self._symbolic_initial_global_matrix.shape[0] + self.symbolic_n_samples = self._symbolic_initial_global_matrix.shape[0] _global_view = property(lambda self: self.global_flat_view.view) _local_view = property(lambda self: self.local_flat_view.view) @@ -733,30 +733,23 @@ def sample_node(self, node, size=100, node = theano.clone(node, more_replacements, strict=False) if size is None: size = 1 - posterior_loc = self.random_local(size) - posterior_glob = self.random_global(size) - node = self.to_flat_input(node) - - def sample(zl, zg): - return theano.clone(node, { - self.local_input: zl, - self.global_input: zg - }, strict=False) - nodes, _ = theano.scan(sample, [posterior_loc, posterior_glob], n_steps=size) + nodes = self.sample_over_posterior(node) + nodes = self.set_size_and_deterministic(nodes, size, 0) return nodes - def _sample_over_posterior(self, node): + def sample_over_posterior(self, node): node = self.to_flat_input(node) posterior_loc = self.symbolic_random_local_matrix posterior_glob = self.symbolic_random_global_matrix + def sample(zl, zg): return theano.clone(node, { self.local_input: zl, self.global_input: zg }, strict=False) + nodes, _ = theano.scan( - sample, [posterior_loc, posterior_glob], - n_steps=posterior_glob.shape[0]) + sample, [posterior_loc, posterior_glob]) return nodes def scale_grad(self, inp): @@ -786,38 +779,32 @@ def _random_part(self, part, size=None, deterministic=False): i_size = np.int32(1) else: i_size = size - r_part = self.set_size_deterministic(r_part, i_size, deterministic) + r_part = self.set_size_and_deterministic(r_part, i_size, deterministic) if size is None: r_part = r_part[0] return r_part def _initial_part_matrix(self, part, size, deterministic): - length = self._choose_alternative( - part, - self.local_size, - self.global_size - ) - dist_name = self._choose_alternative( + length, dist_name, dist_map = self._choose_alternative( part, - self.initial_dist_local_name, - self.initial_dist_global_name, - ) - dist_map = self._choose_alternative( - part, - self.initial_dist_local_map, - self.initial_dist_global_map, + (self.local_size, self.initial_dist_local_name, self.initial_dist_local_map), + (self.global_size, self.initial_dist_global_name, self.initial_dist_global_map) ) + dtype = self._symbolic_initial_global_matrix.dtype + if size is None: + size = 1 + if length == 0: # in this case theano fails to compute sample of correct size + return tt.ones((size, 0), dtype) length = tt.as_tensor(length) size = tt.as_tensor(size) - shape = (size, length) - dtype = self._symbolic_initial_global_matrix.dtype + shape = tt.stack((size, length)) + # apply optimizations if possible if not isinstance(deterministic, tt.Variable): if deterministic: return tt.ones(shape, dtype) * dist_map else: return getattr(self._rng, dist_name)(shape) else: - deterministic = tt.as_tensor(deterministic) sample = getattr(self._rng, dist_name)(shape) initial = tt.switch( deterministic, @@ -858,8 +845,22 @@ def random_global(self, size=None, deterministic=False): # pragma: no cover """ return self._random_part('global', size=size, deterministic=deterministic) + def set_size_and_deterministic(self, node, s, d): + """ + Replaces self.symbolic_n_samples and self._deterministic_flag + with non symbolic input. Used whenever user specifies + `sample size` and `deterministic` option + """ + initial_local = self._initial_part_matrix('local', s, d) + initial_global = self._initial_part_matrix('global', s, d) + return theano.clone(node, { + self._symbolic_initial_local_matrix: initial_local, + self._symbolic_initial_global_matrix: initial_global, + }) + @property @memoize + @change_flags(compute_test_value='off') def sample_dict_fn(self): s = tt.iscalar() l_posterior = self.random_local(s) @@ -910,6 +911,7 @@ def points(): @property @memoize + @change_flags(compute_test_value='raise') def __local_mu_rho(self): if not self.local_vars: mu, rho = ( @@ -930,6 +932,7 @@ def __local_mu_rho(self): @property @memoize + @change_flags(compute_test_value='raise') def normalizing_constant(self): """ Constant to divide when we want to scale down loss from minibatches @@ -940,25 +943,12 @@ def normalizing_constant(self): self.global_input: self.symbolic_random_global_matrix[0], self.local_input: self.symbolic_random_local_matrix[0] }) - t = self.set_size_deterministic(t, 1, 1) # remove random, we do not it here at all + t = self.set_size_and_deterministic(t, 1, 1) # remove random, we do not it here at all # if not scale_cost_to_minibatch: t=1 t = tt.switch(self.scale_cost_to_minibatch, t, tt.constant(1, dtype=t.dtype)) return pm.floatX(t) - def set_size_deterministic(self, smth, s, d): - """ - Replaces self._n_samples and self._deterministic_flag - with non symbolic input. Used whenever user specifies - `sample size` and `deterministic` option - """ - initial_local = self._initial_part_matrix('local', s, d) - initial_global = self._initial_part_matrix('global', s, d) - return theano.clone(smth, { - self._symbolic_initial_local_matrix: initial_local, - self._symbolic_initial_global_matrix: initial_global, - }, strict=False) - @property @memoize def symbolic_random_global_matrix(self): @@ -966,6 +956,7 @@ def symbolic_random_global_matrix(self): @property @memoize + @change_flags(compute_test_value='raise') def symbolic_random_local_matrix(self): mu, rho = self.__local_mu_rho e = self._symbolic_initial_local_matrix @@ -973,6 +964,7 @@ def symbolic_random_local_matrix(self): @property @memoize + @change_flags(compute_test_value='raise') def symbolic_random_total_matrix(self): if self.local_vars and self.global_vars: return tt.stack([ @@ -988,99 +980,106 @@ def symbolic_random_total_matrix(self): @property @memoize - def _symbolic_log_q_W_local(self): - """log_q_W samples over q for local vars - Gradient wrt mu, rho in density parametrization - can be scaled to lower variance of ELBO - """ + @change_flags(compute_test_value='raise') + def symbolic_log_q_W_local(self): mu, rho = self.__local_mu_rho mu = self.scale_grad(mu) rho = self.scale_grad(rho) z = self.symbolic_random_local_matrix logp = log_normal(z, mu, rho=rho) - scaling = [] - for var in self.local_vars: - scaling.append(tt.repeat(var.scaling, var.size)) - scaling = tt.concatenate(scaling) + if self.local_size == 0: + scaling = tt.constant(1, mu.dtype) + else: + scaling = [] + for var in self.local_vars: + scaling.append(tt.repeat(var.scaling, var.dsize)) + scaling = tt.concatenate(scaling) # we need only dimensions here # from incoming unobserved # to get rid of input_view # I replace it with the first row # of total_random matrix # that always exists + scaling = self.to_flat_input(scaling) scaling = theano.clone(scaling, { - self.local_input: self.symbolic_random_local_matrix[0] + self.local_input: self.symbolic_random_local_matrix[0], + self.global_input: self.symbolic_random_global_matrix[0] }) logp *= scaling - logp = logp.sum() - return logp + logp = logp.sum(1) + return logp # shape (s,) @property @memoize - def _symbolic_log_q_W_global(self): - raise NotImplementedError + def symbolic_log_q_W_global(self): + raise NotImplementedError # shape (s,) @property @memoize - def _symbolic_log_q_W(self): - q_w_local = self._symbolic_log_q_W_local - q_w_global = self._symbolic_log_q_W_global - return q_w_global + q_w_local + @change_flags(compute_test_value='raise') + def symbolic_log_q_W(self): + q_w_local = self.symbolic_log_q_W_local + q_w_global = self.symbolic_log_q_W_global + return q_w_global + q_w_local # shape (s,) - def logq(self, nmc=None): + @property + @memoize + @change_flags(compute_test_value='raise') + def logq(self): """Total logq for approximation """ - if nmc is None: - nmc = 1 - log_q = self._symbolic_log_q_W / pm.floatX(self._n_samples) - return self.set_size_deterministic(log_q, nmc, 0) + return self.symbolic_log_q_W.mean(0) - def logq_norm(self, nmc=None): - return self.logq(nmc) / self.normalizing_constant + @property + @memoize + @change_flags(compute_test_value='raise') + def logq_norm(self): + return self.logq / self.normalizing_constant @property @memoize + @change_flags(compute_test_value='raise') def sized_symbolic_logp_local(self): - local_post = self.symbolic_random_local_matrix free_logp_local = tt.sum([ - var.distribution.logp( - self.view_local(local_post, var.name, reshape=True) - ) for var in self.model.free_RVs if var.name in self.local_names + var.logpt + for var in self.model.free_RVs if var.name in self.local_names ]) - return free_logp_local + free_logp_local = self.sample_over_posterior(free_logp_local) + return free_logp_local # shape (s,) @property @memoize + @change_flags(compute_test_value='raise') def sized_symbolic_logp_global(self): - global_post = self.symbolic_random_global_matrix free_logp_global = tt.sum([ - var.distribution.logp( - self.view_local(global_post, var.name, reshape=True) - ) for var in self.model.free_RVs if var.name in self.global_names + var.logpt + for var in self.model.free_RVs if var.name in self.global_names ]) - return free_logp_global + free_logp_global = self.sample_over_posterior(free_logp_global) + return free_logp_global # shape (s,) @property @memoize + @change_flags(compute_test_value='raise') def sized_symbolic_logp_observed(self): observed_logp = tt.sum([ var.logpt for var in self.model.observed_RVs ]) - observed_logp = self._sample_over_posterior( - observed_logp - ) - return observed_logp + observed_logp = self.sample_over_posterior(observed_logp) + return observed_logp # shape (s,) @property @memoize + @change_flags(compute_test_value='raise') def sized_symbolic_logp_potentials(self): - potentials = [tt.sum(var) for var in self.model.potentials] - potentials = self.sample_node(potentials, size=self._n_samples).sum() + potentials = tt.sum(self.model.potentials) + potentials = self.sample_over_posterior(potentials) return potentials @property @memoize + @change_flags(compute_test_value='raise') def sized_symbolic_logp(self): return (self.sized_symbolic_logp_local + self.sized_symbolic_logp_global + @@ -1089,20 +1088,36 @@ def sized_symbolic_logp(self): @property @memoize + @change_flags(compute_test_value='raise') def single_symbolic_logp(self): - return self.apply_replacements(self.model.logpt) + logp = self.to_flat_input(self.model.logpt) + loc = self.symbolic_random_local_matrix[0] + glob = self.symbolic_random_global_matrix[0] + iloc = self.local_input + iglob = self.global_input + return theano.clone( + logp, { + iloc: loc, + iglob: glob + } + ) - def logp(self, nmc=None): - if nmc is None: - _logp = self.single_symbolic_logp - nmc = 1 - else: - _logp = self.sized_symbolic_logp - _logp = _logp / pm.floatX(self._n_samples) - return self.set_size_deterministic(_logp, nmc, 0) + @property + @memoize + @change_flags(compute_test_value='raise') + def logp(self): + return ifelse( + # computed first, so lazy evaluation will work + tt.eq(self._symbolic_initial_global_matrix.shape[0], 1), + self.single_symbolic_logp, + self.sized_symbolic_logp.mean(0) + ) - def logp_norm(self, nmc): - return self.logp(nmc) / self.normalizing_constant + @property + @memoize + @change_flags(compute_test_value='raise') + def logp_norm(self): + return self.logp / self.normalizing_constant def _view_part(self, part, space, name, reshape=True): theano_is_here = isinstance(space, tt.Variable) @@ -1163,12 +1178,10 @@ def view_local(self, space, name, reshape=True): return self._view_part('local', space, name, reshape) @property - @memoize def total_size(self): return self.local_size + self.global_size @property - @memoize def local_size(self): return self._l_order.dimensions From 1be7aeb6cf47a2838a2d1c3afa9e3f94f3138a3b Mon Sep 17 00:00:00 2001 From: Maxim Kochurov Date: Sun, 18 Jun 2017 20:50:45 +0300 Subject: [PATCH 03/24] mv normal refactor --- pymc3/distributions/dist_math.py | 8 ++- pymc3/variational/approximations.py | 34 +++++-------- pymc3/variational/opvi.py | 77 ++++++++++------------------- 3 files changed, 44 insertions(+), 75 deletions(-) diff --git a/pymc3/distributions/dist_math.py b/pymc3/distributions/dist_math.py index 42a0603856..11d8bd4bd0 100644 --- a/pymc3/distributions/dist_math.py +++ b/pymc3/distributions/dist_math.py @@ -217,8 +217,12 @@ def logdet(m): log_det = -logdet(S) delta = x - mean k = f(S.shape[0]) - result = k * tt.log(2. * np.pi) - log_det - result += delta.dot(T).dot(delta) + result = delta.dot(T) + if delta.ndim > 1: + result = tt.batched_dot(result, delta) + else: + result = result.dot(delta.T) + result += k * tt.log(2. * np.pi) - log_det return -.5 * result diff --git a/pymc3/variational/approximations.py b/pymc3/variational/approximations.py index 71cdab2898..8c4cbe4c3f 100644 --- a/pymc3/variational/approximations.py +++ b/pymc3/variational/approximations.py @@ -4,7 +4,7 @@ import pymc3 as pm from pymc3.distributions.dist_math import rho2sd, log_normal, log_normal_mv -from pymc3.variational.opvi import Approximation +from pymc3.variational.opvi import Approximation, node_property from pymc3.theanof import memoize, change_flags @@ -50,19 +50,19 @@ class MeanField(Approximation): Sticking the Landing: A Simple Reduced-Variance Gradient for ADVI approximateinference.org/accepted/RoederEtAl2016.pdf """ - @property + @node_property def mean(self): return self.shared_params['mu'] - @property + @node_property def rho(self): return self.shared_params['rho'] - @property + @node_property def cov(self): return tt.diag(rho2sd(self.rho)**2) - @property + @node_property def std(self): return rho2sd(self.rho) @@ -80,18 +80,14 @@ def create_shared_params(self, **kwargs): 'rho': theano.shared( pm.floatX(np.zeros((self.global_size,))), 'rho')} - @property - @memoize - @change_flags(compute_test_value='off') + @node_property def symbolic_random_global_matrix(self): initial = self._symbolic_initial_global_matrix sd = rho2sd(self.rho) mu = self.mean return sd * initial + mu - @property - @memoize - @change_flags(compute_test_value='off') + @node_property def symbolic_log_q_W_global(self): """ log_q_W samples over q for global vars @@ -153,15 +149,15 @@ def __init__(self, local_rv=None, model=None, cost_part_grad_scale=1, ) self.gpu_compat = gpu_compat - @property + @node_property def L(self): return self.shared_params['L_tril'][self.tril_index_matrix] - @property + @node_property def mean(self): return self.shared_params['mu'] - @property + @node_property def cov(self): L = self.L return L.dot(L.T) @@ -201,10 +197,8 @@ def create_shared_params(self, **kwargs): 'L_tril': theano.shared(L_tril, 'L_tril') } - @property - @memoize - @change_flags(compute_test_value='off') - def log_q_W_global(self): + @node_property + def symbolic_log_q_W_global(self): """log_q_W samples over q for global vars """ mu = self.scale_grad(self.mean) @@ -212,9 +206,7 @@ def log_q_W_global(self): z = self.symbolic_random_global_matrix return log_normal_mv(z, mu, chol=L, gpu_compat=self.gpu_compat) - @property - @memoize - @change_flags(compute_test_value='off') + @node_property def symbolic_random_global_matrix(self): # (samples, dim) or (dim, ) initial = self._symbolic_initial_global_matrix.T diff --git a/pymc3/variational/opvi.py b/pymc3/variational/opvi.py index 6ac07d719d..f456f6227b 100644 --- a/pymc3/variational/opvi.py +++ b/pymc3/variational/opvi.py @@ -55,6 +55,13 @@ ] +def node_property(f): + """ + A shortcut for wrapping method to accessible tensor + """ + return property(memoize(change_flags(compute_test_value='raise')(f))) + + class ObjectiveUpdates(theano.OrderedUpdates): """ OrderedUpdates extension for storing loss @@ -909,9 +916,7 @@ def points(): trace.close() return pm.sampling.MultiTrace([trace]) - @property - @memoize - @change_flags(compute_test_value='raise') + @node_property def __local_mu_rho(self): if not self.local_vars: mu, rho = ( @@ -930,9 +935,7 @@ def __local_mu_rho(self): rho.name = self.__class__.__name__ + '_local_rho' return mu, rho - @property - @memoize - @change_flags(compute_test_value='raise') + @node_property def normalizing_constant(self): """ Constant to divide when we want to scale down loss from minibatches @@ -949,22 +952,17 @@ def normalizing_constant(self): tt.constant(1, dtype=t.dtype)) return pm.floatX(t) - @property - @memoize + @node_property def symbolic_random_global_matrix(self): raise NotImplementedError - @property - @memoize - @change_flags(compute_test_value='raise') + @node_property def symbolic_random_local_matrix(self): mu, rho = self.__local_mu_rho e = self._symbolic_initial_local_matrix return e * rho2sd(rho) + mu - @property - @memoize - @change_flags(compute_test_value='raise') + @node_property def symbolic_random_total_matrix(self): if self.local_vars and self.global_vars: return tt.stack([ @@ -978,9 +976,7 @@ def symbolic_random_total_matrix(self): else: raise TypeError('No free vars in the Model') - @property - @memoize - @change_flags(compute_test_value='raise') + @node_property def symbolic_log_q_W_local(self): mu, rho = self.__local_mu_rho mu = self.scale_grad(mu) @@ -1009,36 +1005,27 @@ def symbolic_log_q_W_local(self): logp = logp.sum(1) return logp # shape (s,) - @property - @memoize + @node_property def symbolic_log_q_W_global(self): raise NotImplementedError # shape (s,) - @property - @memoize - @change_flags(compute_test_value='raise') + @node_property def symbolic_log_q_W(self): q_w_local = self.symbolic_log_q_W_local q_w_global = self.symbolic_log_q_W_global return q_w_global + q_w_local # shape (s,) - @property - @memoize - @change_flags(compute_test_value='raise') + @node_property def logq(self): """Total logq for approximation """ return self.symbolic_log_q_W.mean(0) - @property - @memoize - @change_flags(compute_test_value='raise') + @node_property def logq_norm(self): return self.logq / self.normalizing_constant - @property - @memoize - @change_flags(compute_test_value='raise') + @node_property def sized_symbolic_logp_local(self): free_logp_local = tt.sum([ var.logpt @@ -1047,9 +1034,7 @@ def sized_symbolic_logp_local(self): free_logp_local = self.sample_over_posterior(free_logp_local) return free_logp_local # shape (s,) - @property - @memoize - @change_flags(compute_test_value='raise') + @node_property def sized_symbolic_logp_global(self): free_logp_global = tt.sum([ var.logpt @@ -1058,9 +1043,7 @@ def sized_symbolic_logp_global(self): free_logp_global = self.sample_over_posterior(free_logp_global) return free_logp_global # shape (s,) - @property - @memoize - @change_flags(compute_test_value='raise') + @node_property def sized_symbolic_logp_observed(self): observed_logp = tt.sum([ var.logpt @@ -1069,26 +1052,20 @@ def sized_symbolic_logp_observed(self): observed_logp = self.sample_over_posterior(observed_logp) return observed_logp # shape (s,) - @property - @memoize - @change_flags(compute_test_value='raise') + @node_property def sized_symbolic_logp_potentials(self): potentials = tt.sum(self.model.potentials) potentials = self.sample_over_posterior(potentials) return potentials - @property - @memoize - @change_flags(compute_test_value='raise') + @node_property def sized_symbolic_logp(self): return (self.sized_symbolic_logp_local + self.sized_symbolic_logp_global + self.sized_symbolic_logp_observed + self.sized_symbolic_logp_potentials) - @property - @memoize - @change_flags(compute_test_value='raise') + @node_property def single_symbolic_logp(self): logp = self.to_flat_input(self.model.logpt) loc = self.symbolic_random_local_matrix[0] @@ -1102,9 +1079,7 @@ def single_symbolic_logp(self): } ) - @property - @memoize - @change_flags(compute_test_value='raise') + @node_property def logp(self): return ifelse( # computed first, so lazy evaluation will work @@ -1113,9 +1088,7 @@ def logp(self): self.sized_symbolic_logp.mean(0) ) - @property - @memoize - @change_flags(compute_test_value='raise') + @node_property def logp_norm(self): return self.logp / self.normalizing_constant From ae3ff31df6938c10734a5367cb7eb24443025490 Mon Sep 17 00:00:00 2001 From: Maxim Kochurov Date: Sun, 18 Jun 2017 21:18:23 +0300 Subject: [PATCH 04/24] mv normal refactor --- pymc3/tests/test_vi.py | 79 +++++++++++++++++++++++++++++ pymc3/variational/approximations.py | 2 +- 2 files changed, 80 insertions(+), 1 deletion(-) create mode 100644 pymc3/tests/test_vi.py diff --git a/pymc3/tests/test_vi.py b/pymc3/tests/test_vi.py new file mode 100644 index 0000000000..9bb36f25b8 --- /dev/null +++ b/pymc3/tests/test_vi.py @@ -0,0 +1,79 @@ +import pytest +import pickle +import functools +import numpy as np +from theano import theano, tensor as tt +import pymc3 as pm +from pymc3 import Model, Normal +from pymc3.variational import ( + ADVI, FullRankADVI, SVGD, + Empirical, ASVGD, + MeanField, fit +) +from pymc3.variational.operators import KL + +from pymc3.tests import models +from pymc3.tests.helpers import SeededTest + +def test_elbo(): + mu0 = 1.5 + sigma = 1.0 + y_obs = np.array([1.6, 1.4]) + + post_mu = np.array([1.88], dtype=theano.config.floatX) + post_sd = np.array([1], dtype=theano.config.floatX) + # Create a model for test + with Model() as model: + mu = Normal('mu', mu=mu0, sd=sigma) + Normal('y', mu=mu, sd=1, observed=y_obs) + + # Create variational gradient tensor + mean_field = MeanField(model=model) + elbo = -KL(mean_field)()(10000) + + mean_field.shared_params['mu'].set_value(post_mu) + mean_field.shared_params['rho'].set_value(np.log(np.exp(post_sd) - 1)) + + f = theano.function([], elbo) + elbo_mc = f() + + # Exact value + elbo_true = (-0.5 * ( + 3 + 3 * post_mu ** 2 - 2 * (y_obs[0] + y_obs[1] + mu0) * post_mu + + y_obs[0] ** 2 + y_obs[1] ** 2 + mu0 ** 2 + 3 * np.log(2 * np.pi)) + + 0.5 * (np.log(2 * np.pi) + 1)) + np.testing.assert_allclose(elbo_mc, elbo_true, rtol=0, atol=1e-1) + + +@pytest.fixture(scope='module', params=['mini', 'full']) +def simple_model_data(request): + n = 1000 + sd0 = 2. + mu0 = 4. + sd = 3. + mu = -5. + + data = sd * np.random.randn(n) + mu + if request.param == 'mini': + data = pm.Minibatch(data) + d = n / sd ** 2 + 1 / sd0 ** 2 + mu_post = (n * np.mean(data) / sd ** 2 + mu0 / sd0 ** 2) / d + return dict( + data=data, + mu_post=mu_post, + mu0=mu0, + sd0=sd0, + sd=sd + ) + + +@pytest.fixture(scope='module') +def simple_model(simple_model_data): + with Model(): + mu_ = Normal( + 'mu', mu=simple_model_data['mu0'], + sd=simple_model_data['sd0'], testval=0) + Normal('x', mu=mu_, sd=simple_model_data['sd'], + observed=simple_model_data['data']) + + diff --git a/pymc3/variational/approximations.py b/pymc3/variational/approximations.py index 8c4cbe4c3f..ca052a500e 100644 --- a/pymc3/variational/approximations.py +++ b/pymc3/variational/approximations.py @@ -204,7 +204,7 @@ def symbolic_log_q_W_global(self): mu = self.scale_grad(self.mean) L = self.scale_grad(self.L) z = self.symbolic_random_global_matrix - return log_normal_mv(z, mu, chol=L, gpu_compat=self.gpu_compat) + return pm.MvNormal.dist(mu=mu, chol=L).logp(z) @node_property def symbolic_random_global_matrix(self): From 241b303af7fe5c6eef5d800e273e527680c31fd6 Mon Sep 17 00:00:00 2001 From: Maxim Kochurov Date: Mon, 19 Jun 2017 01:21:20 +0300 Subject: [PATCH 05/24] refactor tests for VI --- pymc3/tests/test_vi.py | 123 ++++++++++++++++++++++++++++++++++++++--- 1 file changed, 114 insertions(+), 9 deletions(-) diff --git a/pymc3/tests/test_vi.py b/pymc3/tests/test_vi.py index 9bb36f25b8..870acc16a8 100644 --- a/pymc3/tests/test_vi.py +++ b/pymc3/tests/test_vi.py @@ -12,8 +12,12 @@ ) from pymc3.variational.operators import KL -from pymc3.tests import models -from pymc3.tests.helpers import SeededTest + +@pytest.fixture('function', autouse=True) +def set_seed(): + np.random.seed(42) + pm.set_tt_rng(42) + def test_elbo(): mu0 = 1.5 @@ -45,8 +49,30 @@ def test_elbo(): np.testing.assert_allclose(elbo_mc, elbo_true, rtol=0, atol=1e-1) -@pytest.fixture(scope='module', params=['mini', 'full']) -def simple_model_data(request): +@pytest.fixture( + 'module', + params=[ + dict(mini=True, scale=False), + dict(mini=False, scale=True), + ], + ids=['mini-noscale', 'full-scale'] +) +def minibatch_and_scaling(request): + return request.param + + +@pytest.fixture('module') +def using_minibatch(minibatch_and_scaling): + return minibatch_and_scaling['mini'] + + +@pytest.fixture('module') +def scale_cost_to_minibatch(minibatch_and_scaling): + return minibatch_and_scaling['scale'] + + +@pytest.fixture('module') +def simple_model_data(using_minibatch): n = 1000 sd0 = 2. mu0 = 4. @@ -54,26 +80,105 @@ def simple_model_data(request): mu = -5. data = sd * np.random.randn(n) + mu - if request.param == 'mini': - data = pm.Minibatch(data) d = n / sd ** 2 + 1 / sd0 ** 2 mu_post = (n * np.mean(data) / sd ** 2 + mu0 / sd0 ** 2) / d + if using_minibatch: + data = pm.Minibatch(data) return dict( + n=n, data=data, mu_post=mu_post, + d=d, mu0=mu0, sd0=sd0, - sd=sd + sd=sd, ) @pytest.fixture(scope='module') def simple_model(simple_model_data): - with Model(): + with Model() as model: mu_ = Normal( 'mu', mu=simple_model_data['mu0'], sd=simple_model_data['sd0'], testval=0) Normal('x', mu=mu_, sd=simple_model_data['sd'], - observed=simple_model_data['data']) + observed=simple_model_data['data'], + total_size=simple_model_data['n']) + return model + + +@pytest.fixture( + scope='function', + params=[ + dict(cls=ADVI, init=dict()), + dict(cls=FullRankADVI, init=dict()), + dict(cls=SVGD, init=dict(n_particles=500)), + dict(cls=ASVGD, init=dict(temperature=1.5)), + ], + ids=lambda d: d['cls'].__name__ +) +def inference(request, simple_model, scale_cost_to_minibatch): + cls = request.param['cls'] + init = request.param['init'] + with simple_model: + return cls(scale_cost_to_minibatch=scale_cost_to_minibatch, **init) + + +@pytest.fixture('function') +def fit_kwargs(inference, using_minibatch): + cb = [pm.callbacks.CheckParametersConvergence( + every=500, + diff='relative', tolerance=0.001), + pm.callbacks.CheckParametersConvergence( + every=500, + diff='absolute', tolerance=0.0001)] + _select = { + (ADVI, 'full'): dict( + obj_optimizer=pm.adagrad_window(learning_rate=0.02, n_win=50), + n=5000, callbacks=cb + ), + (ADVI, 'mini'): dict( + obj_optimizer=pm.adagrad_window(learning_rate=0.01, n_win=50), + n=12000, callbacks=cb + ), + (FullRankADVI, 'full'): dict( + obj_optimizer=pm.adagrad_window(learning_rate=0.01, n_win=50), + n=6000, callbacks=cb + ), + (FullRankADVI, 'mini'): dict( + obj_optimizer=pm.rmsprop(learning_rate=0.001), + n=12000, callbacks=cb + ), + (SVGD, 'full'): dict( + obj_optimizer=pm.sgd(learning_rate=0.01), + n=1000, callbacks=cb + ), + (SVGD, 'mini'): dict( + obj_optimizer=pm.adagrad_window(learning_rate=0.01, n_win=5), + n=3000, callbacks=cb + ), + (ASVGD, 'full'): dict( + obj_optimizer=pm.adagrad_window(learning_rate=0.01, n_win=50), + n=1000, callbacks=cb + ), + (ASVGD, 'mini'): dict( + obj_optimizer=pm.adagrad_window(learning_rate=0.01, n_win=50), + n=3000, callbacks=cb + ) + } + if using_minibatch: + key = 'mini' + else: + key = 'full' + return _select[(type(inference), key)] +def test_fit(inference, + fit_kwargs, + simple_model_data + ): + trace = inference.fit(**fit_kwargs).sample(10000) + mu_post = simple_model_data['mu_post'] + d = simple_model_data['d'] + np.testing.assert_allclose(np.mean(trace['mu']), mu_post, rtol=0.05) + np.testing.assert_allclose(np.std(trace['mu']), np.sqrt(1. / d), rtol=0.1) From a7467da64a3f59c09489372f15f28569e3201efc Mon Sep 17 00:00:00 2001 From: Maxim Kochurov Date: Mon, 19 Jun 2017 01:40:22 +0300 Subject: [PATCH 06/24] more tests --- pymc3/tests/test_vi.py | 112 ++++++++++++++++++++++++++++++++++++++++- 1 file changed, 111 insertions(+), 1 deletion(-) diff --git a/pymc3/tests/test_vi.py b/pymc3/tests/test_vi.py index 870acc16a8..dd9923528c 100644 --- a/pymc3/tests/test_vi.py +++ b/pymc3/tests/test_vi.py @@ -11,6 +11,7 @@ MeanField, fit ) from pymc3.variational.operators import KL +from pymc3.tests import models @pytest.fixture('function', autouse=True) @@ -173,7 +174,7 @@ def fit_kwargs(inference, using_minibatch): return _select[(type(inference), key)] -def test_fit(inference, +def test_fit_oo(inference, fit_kwargs, simple_model_data ): @@ -182,3 +183,112 @@ def test_fit(inference, d = simple_model_data['d'] np.testing.assert_allclose(np.mean(trace['mu']), mu_post, rtol=0.05) np.testing.assert_allclose(np.std(trace['mu']), np.sqrt(1. / d), rtol=0.1) + + +@pytest.fixture('module') +def another_simple_model(): + _model = models.simple_model()[1] + with _model: + pm.Potential('pot', tt.ones((10, 10))) + return _model + + +@pytest.fixture(params=[ + dict(name='advi', kw=dict(start={})), + dict(name='fullrank_advi', kw=dict(start={})), + dict(name='svgd', kw=dict(start={}))], + ids=lambda d: d['name'] +) +def fit_method_with_object(request, another_simple_model): + _select = dict( + advi=ADVI, + fullrank_advi=FullRankADVI, + svgd=SVGD + ) + with another_simple_model: + return _select[request.param['name']]( + **request.param['kw']) + + +@pytest.mark.parametrize( + ['method', 'kwargs', 'error'], + [ + ('undefined', dict(), KeyError), + (1, dict(), TypeError), + ('advi', dict(total_grad_norm_constraint=10), None), + ('advi->fullrank_advi', dict(frac=.1), None), + ('advi->fullrank_advi', dict(frac=1), ValueError), + ('fullrank_advi', dict(), None), + ('svgd', dict(total_grad_norm_constraint=10), None), + ('svgd', dict(start={}), None), + ('asvgd', dict(start={}, total_grad_norm_constraint=10), None), + ], +) +def test_fit_fn_text(method, kwargs, error, another_simple_model): + with another_simple_model: + if error is not None: + with pytest.raises(error): + fit(10, method=method, **kwargs) + else: + fit(10, method=method, **kwargs) + + +def test_fit_fn_oo(fit_method_with_object, another_simple_model): + with another_simple_model: + fit(10, method=fit_method_with_object) + + +def test_error_on_local_rv_for_svgd(another_simple_model): + with another_simple_model: + with pytest.raises(ValueError) as e: + fit(10, method='svgd', local_rv={ + another_simple_model.free_RVs[0]: (0, 1)}) + assert e.match('does not support AEVB') + + +@pytest.mark.parametrize( + 'diff', + [ + 'relative', + 'absolute' + ] +) +@pytest.mark.parametrize( + 'ord', + [1, 2, np.inf] +) +def test_callbacks_convergence(diff, ord): + cb = pm.variational.callbacks.CheckParametersConvergence(every=1, diff=diff, ord=ord) + + class _approx: + params = (theano.shared(np.asarray([1, 2, 3])), ) + + approx = _approx() + + with pytest.raises(StopIteration): + cb(approx, None, 1) + cb(approx, None, 10) + + +def test_tracker_callback(): + import time + tracker = pm.callbacks.Tracker( + ints=lambda *t: t[-1], + ints2=lambda ap, h, j: j, + time=time.time, + ) + for i in range(10): + tracker(None, None, i) + assert 'time' in tracker.hist + assert 'ints' in tracker.hist + assert 'ints2' in tracker.hist + assert (len(tracker['ints']) + == len(tracker['ints2']) + == len(tracker['time']) + == 10) + assert tracker['ints'] == tracker['ints2'] == list(range(10)) + tracker = pm.callbacks.Tracker( + bad=lambda t: t # bad signature + ) + with pytest.raises(TypeError): + tracker(None, None, 1) \ No newline at end of file From 02996f2f5acbf6275b68d7921aa59c5c1a872407 Mon Sep 17 00:00:00 2001 From: Maxim Kochurov Date: Fri, 23 Jun 2017 00:44:36 +0300 Subject: [PATCH 07/24] refactor rest code --- pymc3/variational/approximations.py | 54 ++++++++++++------------- pymc3/variational/inference.py | 61 ++++++++++++++++++----------- pymc3/variational/operators.py | 38 ++++++++++++------ pymc3/variational/opvi.py | 14 +++---- pymc3/variational/stein.py | 46 +++++++++------------- 5 files changed, 114 insertions(+), 99 deletions(-) diff --git a/pymc3/variational/approximations.py b/pymc3/variational/approximations.py index ca052a500e..a2d95e4ce7 100644 --- a/pymc3/variational/approximations.py +++ b/pymc3/variational/approximations.py @@ -3,9 +3,8 @@ from theano import tensor as tt import pymc3 as pm -from pymc3.distributions.dist_math import rho2sd, log_normal, log_normal_mv +from pymc3.distributions.dist_math import rho2sd, log_normal from pymc3.variational.opvi import Approximation, node_property -from pymc3.theanof import memoize, change_flags __all__ = [ @@ -310,7 +309,7 @@ def create_shared_params(self, **kwargs): def randidx(self, size=None): if size is None: - size = () + size = (1,) elif isinstance(size, tt.TensorVariable): if size.ndim < 1: size = size[None] @@ -326,17 +325,31 @@ def randidx(self, size=None): high=pm.floatX(self.histogram.shape[0]) - pm.floatX(1e-16)) .astype('int32')) - def random_global(self, size=None, deterministic=False): - theano_condition_is_here = isinstance(deterministic, tt.Variable) - if theano_condition_is_here: - return tt.switch(deterministic, - self.mean, - self.histogram[self.randidx(size)]) - else: - if deterministic: - return self.mean + def _initial_part_matrix(self, part, size, deterministic): + if part == 'local': + return super(Empirical, self)._initial_part_matrix( + part, size, deterministic + ) + elif part == 'global': + theano_condition_is_here = isinstance(deterministic, tt.Variable) + if theano_condition_is_here: + return tt.switch( + deterministic, + tt.repeat( + self.mean.reshape((1, -1)), + size if size is not None else 1), + self.histogram[self.randidx(size)]) else: - return self.histogram[self.randidx(size)] + if deterministic: + return tt.repeat( + self.mean.reshape((1, -1)), + size if size is not None else 1) + else: + return self.histogram[self.randidx(size)] + + @property + def symbolic_random_global_matrix(self): + return self._symbolic_initial_global_matrix @property def histogram(self): @@ -344,21 +357,6 @@ def histogram(self): """ return self.shared_params - @property - @memoize - def histogram_logp(self): - """Symbolic logp for every point in trace - """ - node = self.to_flat_input(self.model.logpt) - - def mapping(z): - return theano.clone(node, {self.input: z}) - x = self.histogram - _histogram_logp, _ = theano.scan( - mapping, x, n_steps=x.shape[0] - ) - return _histogram_logp - @property def mean(self): return self.histogram.mean(0) diff --git a/pymc3/variational/inference.py b/pymc3/variational/inference.py index c8ffaeb7d3..664bd1208e 100644 --- a/pymc3/variational/inference.py +++ b/pymc3/variational/inference.py @@ -46,6 +46,9 @@ class Inference(object): kwargs : kwargs additional kwargs for :class:`Approximation` """ + OP = None + APPROX = None + TF = None def __init__(self, op, approx, tf, local_rv=None, model=None, op_kwargs=None, **kwargs): if op_kwargs is None: @@ -174,7 +177,7 @@ def _infmean(input_array): 'Average Loss = {:,.5g}'.format(avg_loss)) for callback in callbacks: callback(self.approx, scores[:i + 1], i) - except (KeyboardInterrupt, StopIteration) as e: + except (KeyboardInterrupt, StopIteration) as e: # pragma: no cover # do not print log on the same line progress.close() scores = scores[:i] @@ -352,13 +355,16 @@ class ADVI(Inference): - Kingma, D. P., & Welling, M. (2014). Auto-Encoding Variational Bayes. stat, 1050, 1. """ + OP = KL + APPROX = MeanField + TF = None def __init__(self, local_rv=None, model=None, cost_part_grad_scale=1, scale_cost_to_minibatch=False, random_seed=None, start=None): super(ADVI, self).__init__( - KL, MeanField, None, + self.OP, self.APPROX, self.TF, local_rv=local_rv, model=model, cost_part_grad_scale=cost_part_grad_scale, scale_cost_to_minibatch=scale_cost_to_minibatch, @@ -381,9 +387,7 @@ def from_mean_field(cls, mean_field): if not isinstance(mean_field, MeanField): raise TypeError('Expected MeanField, got %r' % mean_field) inference = object.__new__(cls) - objective = KL(mean_field)(None) - inference.hist = np.asarray(()) - inference.objective = objective + Inference.__init__(inference, KL, mean_field, None) return inference @@ -426,13 +430,16 @@ class FullRankADVI(Inference): - Kingma, D. P., & Welling, M. (2014). Auto-Encoding Variational Bayes. stat, 1050, 1. """ + OP = KL + APPROX = FullRank + TF = None def __init__(self, local_rv=None, model=None, cost_part_grad_scale=1, scale_cost_to_minibatch=False, gpu_compat=False, random_seed=None, start=None): super(FullRankADVI, self).__init__( - KL, FullRank, None, + self.OP, self.APPROX, self.TF, local_rv=local_rv, model=model, cost_part_grad_scale=cost_part_grad_scale, scale_cost_to_minibatch=scale_cost_to_minibatch, @@ -453,11 +460,9 @@ def from_full_rank(cls, full_rank): :class:`FullRankADVI` """ if not isinstance(full_rank, FullRank): - raise TypeError('Expected MeanField, got %r' % full_rank) + raise TypeError('Expected FullRank, got %r' % full_rank) inference = object.__new__(cls) - objective = KL(full_rank)(None) - inference.hist = np.asarray(()) - inference.objective = objective + Inference.__init__(inference, KL, full_rank, None) return inference @classmethod @@ -481,9 +486,7 @@ def from_mean_field(cls, mean_field, gpu_compat=False): """ full_rank = FullRank.from_mean_field(mean_field, gpu_compat) inference = object.__new__(cls) - objective = KL(full_rank)(None) - inference.objective = objective - inference.hist = np.asarray(()) + Inference.__init__(inference, KL, full_rank, None) return inference @classmethod @@ -563,20 +566,31 @@ class SVGD(Inference): Stein Variational Policy Gradient arXiv:1704.02399 """ + OP = KSD + APPROX = Empirical + TF = test_functions.Kernel def __init__(self, n_particles=100, jitter=.01, model=None, kernel=test_functions.rbf, - temperature=1, scale_cost_to_minibatch=False, start=None, histogram=None, + temperature=1, scale_cost_to_minibatch=False, start=None, random_seed=None, local_rv=None): - if histogram is None: - histogram = Empirical.from_noise( - n_particles, jitter=jitter, - scale_cost_to_minibatch=scale_cost_to_minibatch, - start=start, model=model, local_rv=local_rv, random_seed=random_seed) + empirical = Empirical.from_noise( + n_particles, jitter=jitter, + scale_cost_to_minibatch=scale_cost_to_minibatch, + start=start, model=model, local_rv=local_rv, random_seed=random_seed) super(SVGD, self).__init__( - KSD, histogram, + KSD, empirical, kernel, op_kwargs=dict(temperature=temperature), model=model, random_seed=random_seed) + @classmethod + def from_empirical(cls, empirical, kernel=test_functions.rbf, + temperature=1): + instance = object.__new__(cls) + Inference.__init__( + instance, KSD, empirical, + kernel, op_kwargs=dict(temperature=temperature)) + return instance + class ASVGD(Inference): R""" @@ -626,6 +640,9 @@ class ASVGD(Inference): Stein Variational Policy Gradient arXiv:1704.02399 """ + OP = AKSD + APPROX = None + TF = test_functions.Kernel def __init__(self, approx=FullRank, local_rv=None, kernel=test_functions.rbf, temperature=1, model=None, **kwargs): @@ -640,7 +657,7 @@ def __init__(self, approx=FullRank, local_rv=None, ) def fit(self, n=10000, score=None, callbacks=None, progressbar=True, - obj_n_mc=30, **kwargs): + obj_n_mc=300, **kwargs): """ Performs Amortized Stein Variational Gradient Descent @@ -667,7 +684,7 @@ def fit(self, n=10000, score=None, callbacks=None, progressbar=True, n=n, score=score, callbacks=callbacks, progressbar=progressbar, obj_n_mc=obj_n_mc, **kwargs) - def run_profiling(self, n=1000, score=None, obj_n_mc=30, **kwargs): + def run_profiling(self, n=1000, score=None, obj_n_mc=300, **kwargs): return super(ASVGD, self).run_profiling( n=n, score=score, obj_n_mc=obj_n_mc, **kwargs) diff --git a/pymc3/variational/operators.py b/pymc3/variational/operators.py index 56f9f70dbb..f491e0026f 100644 --- a/pymc3/variational/operators.py +++ b/pymc3/variational/operators.py @@ -1,6 +1,7 @@ import warnings -from theano import theano, tensor as tt -from pymc3.variational.opvi import Operator, ObjectiveFunction, _warn_not_used +import collections +from theano import tensor as tt +from pymc3.variational.opvi import Operator, ObjectiveFunction from pymc3.variational.stein import Stein import pymc3 as pm @@ -41,27 +42,31 @@ def __init__(self, op, tf): raise TypeError('Op should be KSD') ObjectiveFunction.__init__(self, op, tf) - def get_input(self, n_mc): + def get_input(self): if hasattr(self.approx, 'histogram'): - if n_mc is not None: - _warn_not_used('n_mc', self.op) - return self.approx.histogram - elif n_mc is not None and n_mc > 1: - return self.approx.random_total(n_mc) + return self.approx.symbolic_random_local_matrix, self.approx.histogram else: - raise ValueError('Variational type approximation requires ' - 'sample size (`n_mc` : int > 1 should be passed)') + return self.approx.symbolic_random_local_matrix, self.approx.symbolic_random_global_matrix def __call__(self, nmc, **kwargs): op = self.op # type: KSD grad = op.apply(self.tf) + loc_size = self.approx.local_size + local_grad = grad[..., :loc_size] + global_grad = grad[..., loc_size:] if 'more_obj_params' in kwargs: params = self.obj_params + kwargs['more_obj_params'] else: params = self.test_params + kwargs['more_tf_params'] grad *= pm.floatX(-1) - z = op.approx.symbolic_random_total_matrix - grad = tt.grad(None, params, known_grads={z: grad}) + zl, zg = self.get_input() + zl, zg, grad, local_grad, global_grad = self.approx.set_size_and_deterministic( + (zl, zg, grad, local_grad, global_grad), + nmc, 0) + grad = tt.grad(None, params, known_grads=collections.OrderedDict([ + (zl, local_grad), + (zg, global_grad) + ]), disconnected_inputs='ignore') return grad @@ -99,12 +104,19 @@ def __init__(self, approx, temperature=1): Operator.__init__(self, approx) self.temperature = temperature + def get_input(self): + if hasattr(self.approx, 'histogram'): + return self.approx.histogram + else: + return self.approx.symbolic_random_total_matrix + def apply(self, f): # f: kernel function for KSD f(histogram) -> (k(x,.), \nabla_x k(x,.)) + input_matrix = self.get_input() stein = Stein( approx=self.approx, kernel=f, - input_matrix=self.approx.symbolic_random_total_matrix, + input_matrix=input_matrix, temperature=self.temperature) return pm.floatX(-1) * stein.grad diff --git a/pymc3/variational/opvi.py b/pymc3/variational/opvi.py index f456f6227b..3c13e7f9e1 100644 --- a/pymc3/variational/opvi.py +++ b/pymc3/variational/opvi.py @@ -35,7 +35,6 @@ import numpy as np import theano import theano.tensor as tt -from theano.ifelse import ifelse import pymc3 as pm from .updates import adagrad_window from ..distributions.dist_math import rho2sd, log_normal @@ -860,6 +859,10 @@ def set_size_and_deterministic(self, node, s, d): """ initial_local = self._initial_part_matrix('local', s, d) initial_global = self._initial_part_matrix('global', s, d) + if isinstance(s, int) and (s == 1) or s is None: + node = theano.clone(node, { + self.logp: self.single_symbolic_logp + }) return theano.clone(node, { self._symbolic_initial_local_matrix: initial_local, self._symbolic_initial_global_matrix: initial_global, @@ -965,7 +968,7 @@ def symbolic_random_local_matrix(self): @node_property def symbolic_random_total_matrix(self): if self.local_vars and self.global_vars: - return tt.stack([ + return tt.concatenate([ self.symbolic_random_local_matrix, self.symbolic_random_global_matrix, ], axis=-1) @@ -1081,12 +1084,7 @@ def single_symbolic_logp(self): @node_property def logp(self): - return ifelse( - # computed first, so lazy evaluation will work - tt.eq(self._symbolic_initial_global_matrix.shape[0], 1), - self.single_symbolic_logp, - self.sized_symbolic_logp.mean(0) - ) + return self.sized_symbolic_logp.mean(0) @node_property def logp_norm(self): diff --git a/pymc3/variational/stein.py b/pymc3/variational/stein.py index 4ebcd53227..9ab257d2c9 100644 --- a/pymc3/variational/stein.py +++ b/pymc3/variational/stein.py @@ -1,4 +1,5 @@ from theano import theano, tensor as tt +from pymc3.variational.opvi import node_property from pymc3.variational.test_functions import rbf from pymc3.theanof import memoize, floatX @@ -49,37 +50,26 @@ def dxkxy(self): return self._kernel()[1] @property - @memoize - def logp(self): - return theano.scan( - fn=lambda zg: self.approx.logp_norm(zg), - sequences=[self.input_matrix] - )[0] + def logp_norm(self): + return self.approx.sized_symbolic_logp / self.approx.normalizing_constant - @property - @memoize + @node_property def dlogp(self): - return theano.scan( - fn=lambda zg: theano.grad(self.approx.logp_norm(zg), zg), - sequences=[self.input_matrix] - )[0] + loc_random = self.input_matrix[..., :self.approx.local_size] + glob_random = self.input_matrix[..., self.approx.local_size:] + loc_grad, glob_grad = tt.grad( + self.logp_norm.sum(), + [self.approx.symbolic_random_local_matrix, + self.approx.symbolic_random_global_matrix], + disconnected_inputs='ignore' + ) + loc_grad, glob_grad = theano.clone( + [loc_grad, glob_grad], + {self.approx.symbolic_random_local_matrix: loc_random, + self.approx.symbolic_random_global_matrix: glob_random} + ) + return tt.concatenate([loc_grad, glob_grad], axis=-1) @memoize def _kernel(self): return self._kernel_f(self.input_matrix) - - def get_approx_input(self, size=100): - """ - - Parameters - ---------- - size : if approx is not Empirical, takes `n=size` random samples - - Returns - ------- - matrix - """ - if hasattr(self.approx, 'histogram'): - return self.approx.histogram - else: - return self.approx.random(size) From 9e53c050f368d0a243c9ea34dfcbdaa7d002191c Mon Sep 17 00:00:00 2001 From: Maxim Kochurov Date: Fri, 23 Jun 2017 00:44:51 +0300 Subject: [PATCH 08/24] refactor tests --- pymc3/tests/test_variational_inference.py | 620 +++++++++++----------- pymc3/tests/test_vi.py | 294 ---------- 2 files changed, 316 insertions(+), 598 deletions(-) delete mode 100644 pymc3/tests/test_vi.py diff --git a/pymc3/tests/test_variational_inference.py b/pymc3/tests/test_variational_inference.py index 7ca2d149d9..83d0e53010 100644 --- a/pymc3/tests/test_variational_inference.py +++ b/pymc3/tests/test_variational_inference.py @@ -1,6 +1,5 @@ import pytest import pickle -import functools import numpy as np from theano import theano, tensor as tt import pymc3 as pm @@ -8,12 +7,17 @@ from pymc3.variational import ( ADVI, FullRankADVI, SVGD, Empirical, ASVGD, - MeanField, fit + MeanField, FullRank, + fit ) from pymc3.variational.operators import KL - from pymc3.tests import models -from pymc3.tests.helpers import SeededTest + + +@pytest.fixture('function', autouse=True) +def set_seed(): + np.random.seed(42) + pm.set_tt_rng(42) def test_elbo(): @@ -46,300 +50,177 @@ def test_elbo(): np.testing.assert_allclose(elbo_mc, elbo_true, rtol=0, atol=1e-1) -def _test_aevb(self): - # add to inference that supports aevb - with pm.Model() as model: - x = pm.Normal('x') - pm.Normal('y', x) - x = model.x - y = model.y - mu = theano.shared(x.init_value) * 2 - rho = theano.shared(np.zeros_like(x.init_value)) - with model: - inference = self.inference(local_rv={x: (mu, rho)}) - approx = inference.fit(3, obj_n_mc=2, obj_optimizer=self.optimizer) - approx.sample(10) - approx.apply_replacements( - y, - more_replacements={x: np.asarray([1, 1], dtype=x.dtype)} - ).eval() +@pytest.fixture( + 'module', + params=[ + dict(mini=True, scale=False), + dict(mini=False, scale=True), + ], + ids=['mini-noscale', 'full-scale'] +) +def minibatch_and_scaling(request): + return request.param + + +@pytest.fixture('module') +def using_minibatch(minibatch_and_scaling): + return minibatch_and_scaling['mini'] + + +@pytest.fixture('module') +def scale_cost_to_minibatch(minibatch_and_scaling): + return minibatch_and_scaling['scale'] + + +@pytest.fixture('module') +def simple_model_data(using_minibatch): + n = 1000 + sd0 = 2. + mu0 = 4. + sd = 3. + mu = -5. + + data = sd * np.random.randn(n) + mu + d = n / sd ** 2 + 1 / sd0 ** 2 + mu_post = (n * np.mean(data) / sd ** 2 + mu0 / sd0 ** 2) / d + if using_minibatch: + data = pm.Minibatch(data) + return dict( + n=n, + data=data, + mu_post=mu_post, + d=d, + mu0=mu0, + sd0=sd0, + sd=sd, + ) -class TestApproximates: - @pytest.mark.usefixtures('strict_float32') - class Base(SeededTest): - inference = None - NITER = 12000 - optimizer = pm.adagrad_window(learning_rate=0.01, n_win=50) - conv_cb = property(lambda self: [ - pm.callbacks.CheckParametersConvergence( - every=500, - diff='relative', tolerance=0.001), - pm.callbacks.CheckParametersConvergence( - every=500, - diff='absolute', tolerance=0.0001) - ]) - - def test_vars_view(self): - _, model, _ = models.multidimensional_model() - with model: - app = self.inference().approx - posterior = app.random_global(10) - x_sampled = app.view_global(posterior, 'x').eval() - assert x_sampled.shape == (10,) + model['x'].dshape - - def test_vars_view_dynamic_size(self): - _, model, _ = models.multidimensional_model() - with model: - app = self.inference().approx - i = tt.iscalar('i') - i.tag.test_value = 1 - posterior = app.random_global(i) - x_sampled = app.view_global(posterior, 'x').eval({i: 10}) - assert x_sampled.shape == (10,) + model['x'].dshape - x_sampled = app.view_global(posterior, 'x').eval({i: 1}) - assert x_sampled.shape == (1,) + model['x'].dshape - - def test_sample(self): - n_samples = 100 - xs = np.random.binomial(n=1, p=0.2, size=n_samples) - with pm.Model(): - p = pm.Beta('p', alpha=1, beta=1) - pm.Binomial('xs', n=1, p=p, observed=xs) - app = self.inference().approx - trace = app.sample(draws=1, include_transformed=False) - assert trace.varnames == ['p'] - assert len(trace) == 1 - trace = app.sample(draws=10, include_transformed=True) - assert sorted(trace.varnames) == ['p', 'p_logodds__'] - assert len(trace) == 10 - - def test_sample_node(self): - n_samples = 100 - xs = np.random.binomial(n=1, p=0.2, size=n_samples) - with pm.Model(): - p = pm.Beta('p', alpha=1, beta=1) - pm.Binomial('xs', n=1, p=p, observed=xs) - app = self.inference().approx - app.sample_node(p).eval() # should be evaluated without errors - - def test_optimizer_with_full_data(self): - n = 1000 - sd0 = 2. - mu0 = 4. - sd = 3. - mu = -5. - - data = sd * np.random.randn(n) + mu - - d = n / sd ** 2 + 1 / sd0 ** 2 - mu_post = (n * np.mean(data) / sd ** 2 + mu0 / sd0 ** 2) / d - - with Model(): - mu_ = Normal('mu', mu=mu0, sd=sd0, testval=0) - Normal('x', mu=mu_, sd=sd, observed=data) - inf = self.inference(start={}) - approx = inf.fit(self.NITER, - obj_optimizer=self.optimizer, - callbacks=self.conv_cb,) - trace = approx.sample(10000) - np.testing.assert_allclose(np.mean(trace['mu']), mu_post, rtol=0.05) - np.testing.assert_allclose(np.std(trace['mu']), np.sqrt(1. / d), rtol=0.1) - - def test_optimizer_minibatch_with_generator(self): - n = 1000 - sd0 = 2. - mu0 = 4. - sd = 3. - mu = -5. - - data = sd * np.random.randn(n) + mu - - d = n / sd**2 + 1 / sd0**2 - mu_post = (n * np.mean(data) / sd**2 + mu0 / sd0**2) / d - - def create_minibatch(data): - while True: - data = np.roll(data, 100, axis=0) - yield data[:100] - - minibatches = create_minibatch(data) - with Model(): - mu_ = Normal('mu', mu=mu0, sd=sd0, testval=0) - Normal('x', mu=mu_, sd=sd, observed=minibatches, total_size=n) - inf = self.inference() - approx = inf.fit(self.NITER * 3, obj_optimizer=self.optimizer, - callbacks=self.conv_cb) - trace = approx.sample(10000) - np.testing.assert_allclose(np.mean(trace['mu']), mu_post, rtol=0.05) - np.testing.assert_allclose(np.std(trace['mu']), np.sqrt(1. / d), rtol=0.1) - - def test_optimizer_minibatch_with_callback(self): - n = 1000 - sd0 = 2. - mu0 = 4. - sd = 3. - mu = -5. - - data = sd * np.random.randn(n) + mu - - d = n / sd ** 2 + 1 / sd0 ** 2 - mu_post = (n * np.mean(data) / sd ** 2 + mu0 / sd0 ** 2) / d - - def create_minibatch(data): - while True: - data = np.roll(data, 100, axis=0) - yield pm.floatX(data[:100]) - - minibatches = create_minibatch(data) - with Model(): - data_t = theano.shared(next(minibatches)) - - def cb(*_): - data_t.set_value(next(minibatches)) - mu_ = Normal('mu', mu=mu0, sd=sd0, testval=0) - Normal('x', mu=mu_, sd=sd, observed=data_t, total_size=n) - inf = self.inference(scale_cost_to_minibatch=True) - approx = inf.fit( - self.NITER * 3, callbacks=[cb] + self.conv_cb, obj_optimizer=self.optimizer) - trace = approx.sample(10000) - np.testing.assert_allclose(np.mean(trace['mu']), mu_post, rtol=0.05) - np.testing.assert_allclose(np.std(trace['mu']), np.sqrt(1. / d), rtol=0.1) - - def test_n_obj_mc(self): - n_samples = 100 - xs = np.random.binomial(n=1, p=0.2, size=n_samples) - with pm.Model(): - p = pm.Beta('p', alpha=1, beta=1) - pm.Binomial('xs', n=1, p=p, observed=xs) - inf = self.inference(scale_cost_to_minibatch=True) - # should just work - inf.fit(10, obj_n_mc=10, obj_optimizer=self.optimizer) - - def test_pickling(self): - with models.multidimensional_model()[1]: - inference = self.inference() - - inference = pickle.loads(pickle.dumps(inference)) - inference.fit(20) - - def test_profile(self): - with models.multidimensional_model()[1]: - self.inference().run_profiling(10) - - def test_multiple_replacements(self): - _, model, _ = models.exponential_beta(n=2) - x = model.x - y = model.y - xy = x*y - xpy = x+y - with model: - mf = self.inference().approx - xy_, xpy_ = mf.apply_replacements([xy, xpy]) - xy_s, xpy_s = mf.sample_node([xy, xpy]) - xy_.eval() - xpy_.eval() - xy_s.eval() - xpy_s.eval() - - -class TestMeanField(TestApproximates.Base): - inference = ADVI - test_aevb = _test_aevb - - def test_length_of_hist(self): - with models.multidimensional_model()[1]: - inf = self.inference() - assert len(inf.hist) == 0 - inf.fit(10) - assert len(inf.hist) == 10 - assert not np.isnan(inf.hist).any() - inf.fit(self.NITER, obj_optimizer=self.optimizer) - assert len(inf.hist) == self.NITER + 10 - assert not np.isnan(inf.hist).any() - - -class TestFullRank(TestApproximates.Base): - inference = FullRankADVI - test_aevb = _test_aevb - - def test_from_mean_field(self): - with models.multidimensional_model()[1]: - advi = ADVI() - full_rank = FullRankADVI.from_mean_field(advi.approx) - full_rank.fit(20) - - def test_from_advi(self): - with models.multidimensional_model()[1]: - advi = ADVI() - full_rank = FullRankADVI.from_advi(advi) - full_rank.fit(20) - - -class TestSVGD(TestApproximates.Base): - inference = functools.partial(SVGD, n_particles=100) - - -class TestASVGD(TestApproximates.Base): - NITER = 5000 - inference = functools.partial(ASVGD, temperature=1.5) - test_aevb = _test_aevb - - -class TestEmpirical(SeededTest): - def test_sampling(self): - with models.multidimensional_model()[1]: - full_rank = FullRankADVI() - approx = full_rank.fit(20) - trace0 = approx.sample(10000) - approx = Empirical(trace0) - trace1 = approx.sample(100000) - np.testing.assert_allclose(trace0['x'].mean(0), trace1['x'].mean(0), atol=0.01) - np.testing.assert_allclose(trace0['x'].var(0), trace1['x'].var(0), atol=0.01) - - def test_aevb_empirical(self): - _, model, _ = models.exponential_beta(n=2) - x = model.x - mu = theano.shared(x.init_value) - rho = theano.shared(np.zeros_like(x.init_value)) - with model: - inference = ADVI(local_rv={x: (mu, rho)}) - approx = inference.approx - trace0 = approx.sample(10000) - approx = Empirical(trace0, local_rv={x: (mu, rho)}) - trace1 = approx.sample(10000) - approx.random(no_rand=True) - approx.random_fn(no_rand=True) - np.testing.assert_allclose(trace0['y'].mean(0), trace1['y'].mean(0), atol=0.02) - np.testing.assert_allclose(trace0['y'].var(0), trace1['y'].var(0), atol=0.02) - np.testing.assert_allclose(trace0['x'].mean(0), trace1['x'].mean(0), atol=0.02) - np.testing.assert_allclose(trace0['x'].var(0), trace1['x'].var(0), atol=0.02) - - def test_random_with_transformed(self): - p = .2 - trials = (np.random.uniform(size=10) < p).astype('int8') - with pm.Model(): - p = pm.Uniform('p') - pm.Bernoulli('trials', p, observed=trials) - trace = pm.sample(1000, step=pm.Metropolis()) - approx = Empirical(trace) - approx.randidx(None).eval() - approx.randidx(1).eval() - approx.random_fn(no_rand=True) - approx.random_fn(no_rand=False) - approx.histogram_logp.eval() - - def test_init_from_noize(self): - with models.multidimensional_model()[1]: - approx = Empirical.from_noise(100) - assert approx.histogram.eval().shape == (100, 6) - -_model = models.simple_model()[1] -with _model: - pm.Potential('pot', tt.ones((10, 10))) - _advi = ADVI() - _fullrank_advi = FullRankADVI() - _svgd = SVGD() +@pytest.fixture(scope='module') +def simple_model(simple_model_data): + with Model() as model: + mu_ = Normal( + 'mu', mu=simple_model_data['mu0'], + sd=simple_model_data['sd0'], testval=0) + Normal('x', mu=mu_, sd=simple_model_data['sd'], + observed=simple_model_data['data'], + total_size=simple_model_data['n']) + return model + + +@pytest.fixture('module', params=[ + dict(cls=ADVI, init=dict()), + dict(cls=FullRankADVI, init=dict()), + dict(cls=SVGD, init=dict(n_particles=500)), + dict(cls=ASVGD, init=dict(temperature=1.)), + ], ids=lambda d: d['cls'].__name__) +def inference_spec(request): + cls = request.param['cls'] + init = request.param['init'] + + def init_(**kw): + k = init.copy() + k.update(kw) + return cls(**k) + init_.cls = cls + return init_ + + +@pytest.fixture('function') +def inference(inference_spec, simple_model, scale_cost_to_minibatch): + + with simple_model: + return inference_spec(scale_cost_to_minibatch=scale_cost_to_minibatch) + + +@pytest.fixture('function') +def fit_kwargs(inference, using_minibatch): + cb = [pm.callbacks.CheckParametersConvergence( + every=500, + diff='relative', tolerance=0.001), + pm.callbacks.CheckParametersConvergence( + every=500, + diff='absolute', tolerance=0.0001)] + _select = { + (ADVI, 'full'): dict( + obj_optimizer=pm.adagrad_window(learning_rate=0.02, n_win=50), + n=5000, callbacks=cb + ), + (ADVI, 'mini'): dict( + obj_optimizer=pm.adagrad_window(learning_rate=0.01, n_win=50), + n=12000, callbacks=cb + ), + (FullRankADVI, 'full'): dict( + obj_optimizer=pm.adagrad_window(learning_rate=0.01, n_win=50), + n=6000, callbacks=cb + ), + (FullRankADVI, 'mini'): dict( + obj_optimizer=pm.rmsprop(learning_rate=0.007), + n=12000, callbacks=cb + ), + (SVGD, 'full'): dict( + obj_optimizer=pm.sgd(learning_rate=0.01), + n=1000, callbacks=cb + ), + (SVGD, 'mini'): dict( + obj_optimizer=pm.adagrad_window(learning_rate=0.01, n_win=5), + n=3000, callbacks=cb + ), + (ASVGD, 'full'): dict( + obj_optimizer=pm.adagrad_window(learning_rate=0.02, n_win=50), + n=2000, obj_n_mc=300, callbacks=cb + ), + (ASVGD, 'mini'): dict( + obj_optimizer=pm.adagrad_window(learning_rate=0.02, n_win=50), + n=1700, obj_n_mc=300, callbacks=cb + ) + } + if using_minibatch: + key = 'mini' + else: + key = 'full' + return _select[(type(inference), key)] + + +def test_fit_oo(inference, + fit_kwargs, + simple_model_data): + trace = inference.fit(**fit_kwargs).sample(10000) + mu_post = simple_model_data['mu_post'] + d = simple_model_data['d'] + np.testing.assert_allclose(np.mean(trace['mu']), mu_post, rtol=0.05) + np.testing.assert_allclose(np.std(trace['mu']), np.sqrt(1. / d), rtol=0.1) + + +def test_profile(inference, fit_kwargs): + fit_kwargs['n'] = 100 + fit_kwargs.pop('callbacks') + inference.run_profiling(**fit_kwargs).summary() + + +@pytest.fixture('module') +def another_simple_model(): + _model = models.simple_model()[1] + with _model: + pm.Potential('pot', tt.ones((10, 10))) + return _model + + +@pytest.fixture(params=[ + dict(name='advi', kw=dict(start={})), + dict(name='fullrank_advi', kw=dict(start={})), + dict(name='svgd', kw=dict(start={}))], + ids=lambda d: d['name'] +) +def fit_method_with_object(request, another_simple_model): + _select = dict( + advi=ADVI, + fullrank_advi=FullRankADVI, + svgd=SVGD + ) + with another_simple_model: + return _select[request.param['name']]( + **request.param['kw']) @pytest.mark.parametrize( @@ -347,9 +228,6 @@ def test_init_from_noize(self): [ ('undefined', dict(), KeyError), (1, dict(), TypeError), - (_advi, dict(start={}), None), - (_fullrank_advi, dict(), None), - (_svgd, dict(), None), ('advi', dict(total_grad_norm_constraint=10), None), ('advi->fullrank_advi', dict(frac=.1), None), ('advi->fullrank_advi', dict(frac=1), ValueError), @@ -357,11 +235,10 @@ def test_init_from_noize(self): ('svgd', dict(total_grad_norm_constraint=10), None), ('svgd', dict(start={}), None), ('asvgd', dict(start={}, total_grad_norm_constraint=10), None), - ('svgd', dict(local_rv={_model.free_RVs[0]: (0, 1)}), ValueError) - ] + ], ) -def test_fit(method, kwargs, error): - with _model: +def test_fit_fn_text(method, kwargs, error, another_simple_model): + with another_simple_model: if error is not None: with pytest.raises(error): fit(10, method=method, **kwargs) @@ -369,6 +246,141 @@ def test_fit(method, kwargs, error): fit(10, method=method, **kwargs) +def test_fit_fn_oo(fit_method_with_object, another_simple_model): + with another_simple_model: + fit(10, method=fit_method_with_object) + + +def test_error_on_local_rv_for_svgd(another_simple_model): + with another_simple_model: + with pytest.raises(ValueError) as e: + fit(10, method='svgd', local_rv={ + another_simple_model.free_RVs[0]: (0, 1)}) + assert e.match('does not support AEVB') + + +@pytest.fixture('module') +def aevb_model(): + with pm.Model() as model: + x = pm.Normal('x') + pm.Normal('y', x) + x = model.x + y = model.y + mu = theano.shared(x.init_value) * 2 + rho = theano.shared(np.zeros_like(x.init_value)) + return { + 'model': model, + 'y': y, + 'x': x, + 'replace': (mu, rho) + } + + +def test_aevb(inference_spec, aevb_model): + # add to inference that supports aevb + cls = inference_spec.cls + if not cls.OP.SUPPORT_AEVB: + raise pytest.skip('%s does not support aevb' % cls) + x = aevb_model['x'] + y = aevb_model['y'] + model = aevb_model['model'] + replace = aevb_model['replace'] + with model: + inference = inference_spec(local_rv={x: replace}) + approx = inference.fit(3, obj_n_mc=2) + approx.sample(10) + approx.apply_replacements( + y, + more_replacements={x: np.asarray([1, 1], dtype=x.dtype)} + ).eval() + + +@pytest.fixture('module') +def binomial_model(): + n_samples = 100 + xs = np.random.binomial(n=1, p=0.2, size=n_samples) + with pm.Model() as model: + p = pm.Beta('p', alpha=1, beta=1) + pm.Binomial('xs', n=1, p=p, observed=xs) + return model + + +@pytest.fixture('module') +def binomial_model_inference(binomial_model, inference_spec): + with binomial_model: + return inference_spec() + + +def test_n_mc(binomial_model_inference): + binomial_model_inference.fit(10, obj_n_mc=2) + + +def test_pickling(binomial_model_inference): + inference = pickle.loads(pickle.dumps(binomial_model_inference)) + inference.fit(20) + + +def test_multiple_replacements(inference_spec): + _, model, _ = models.exponential_beta(n=2) + x = model.x + y = model.y + xy = x*y + xpy = x+y + with model: + ap = inference_spec().approx + xy_, xpy_ = ap.apply_replacements([xy, xpy]) + xy_s, xpy_s = ap.sample_node([xy, xpy]) + xy_.eval() + xpy_.eval() + xy_s.eval() + xpy_s.eval() + + +def test_from_mean_field(another_simple_model): + with another_simple_model: + advi = ADVI() + full_rank = FullRankADVI.from_mean_field(advi.approx) + full_rank.fit(20) + + +def test_from_advi(another_simple_model): + with another_simple_model: + advi = ADVI() + full_rank = FullRankADVI.from_advi(advi) + full_rank.fit(20) + + +def test_from_full_rank(another_simple_model): + with another_simple_model: + fr = FullRank() + full_rank = FullRankADVI.from_full_rank(fr) + full_rank.fit(20) + + +def test_from_empirical(another_simple_model): + with another_simple_model: + emp = Empirical.from_noise(1000) + svgd = SVGD.from_empirical(emp) + svgd.fit(20) + + +def test_aevb_empirical(): + _, model, _ = models.exponential_beta(n=2) + x = model.x + mu = theano.shared(x.init_value) + rho = theano.shared(np.zeros_like(x.init_value)) + with model: + inference = ADVI(local_rv={x: (mu, rho)}) + approx = inference.approx + trace0 = approx.sample(10000) + approx = Empirical(trace0, local_rv={x: (mu, rho)}) + trace1 = approx.sample(10000) + np.testing.assert_allclose(trace0['y'].mean(0), trace1['y'].mean(0), atol=0.02) + np.testing.assert_allclose(trace0['y'].var(0), trace1['y'].var(0), atol=0.02) + np.testing.assert_allclose(trace0['x'].mean(0), trace1['x'].mean(0), atol=0.02) + np.testing.assert_allclose(trace0['x'].var(0), trace1['x'].var(0), atol=0.02) + + @pytest.mark.parametrize( 'diff', [ @@ -414,4 +426,4 @@ def test_tracker_callback(): bad=lambda t: t # bad signature ) with pytest.raises(TypeError): - tracker(None, None, 1) + tracker(None, None, 1) \ No newline at end of file diff --git a/pymc3/tests/test_vi.py b/pymc3/tests/test_vi.py deleted file mode 100644 index dd9923528c..0000000000 --- a/pymc3/tests/test_vi.py +++ /dev/null @@ -1,294 +0,0 @@ -import pytest -import pickle -import functools -import numpy as np -from theano import theano, tensor as tt -import pymc3 as pm -from pymc3 import Model, Normal -from pymc3.variational import ( - ADVI, FullRankADVI, SVGD, - Empirical, ASVGD, - MeanField, fit -) -from pymc3.variational.operators import KL -from pymc3.tests import models - - -@pytest.fixture('function', autouse=True) -def set_seed(): - np.random.seed(42) - pm.set_tt_rng(42) - - -def test_elbo(): - mu0 = 1.5 - sigma = 1.0 - y_obs = np.array([1.6, 1.4]) - - post_mu = np.array([1.88], dtype=theano.config.floatX) - post_sd = np.array([1], dtype=theano.config.floatX) - # Create a model for test - with Model() as model: - mu = Normal('mu', mu=mu0, sd=sigma) - Normal('y', mu=mu, sd=1, observed=y_obs) - - # Create variational gradient tensor - mean_field = MeanField(model=model) - elbo = -KL(mean_field)()(10000) - - mean_field.shared_params['mu'].set_value(post_mu) - mean_field.shared_params['rho'].set_value(np.log(np.exp(post_sd) - 1)) - - f = theano.function([], elbo) - elbo_mc = f() - - # Exact value - elbo_true = (-0.5 * ( - 3 + 3 * post_mu ** 2 - 2 * (y_obs[0] + y_obs[1] + mu0) * post_mu + - y_obs[0] ** 2 + y_obs[1] ** 2 + mu0 ** 2 + 3 * np.log(2 * np.pi)) + - 0.5 * (np.log(2 * np.pi) + 1)) - np.testing.assert_allclose(elbo_mc, elbo_true, rtol=0, atol=1e-1) - - -@pytest.fixture( - 'module', - params=[ - dict(mini=True, scale=False), - dict(mini=False, scale=True), - ], - ids=['mini-noscale', 'full-scale'] -) -def minibatch_and_scaling(request): - return request.param - - -@pytest.fixture('module') -def using_minibatch(minibatch_and_scaling): - return minibatch_and_scaling['mini'] - - -@pytest.fixture('module') -def scale_cost_to_minibatch(minibatch_and_scaling): - return minibatch_and_scaling['scale'] - - -@pytest.fixture('module') -def simple_model_data(using_minibatch): - n = 1000 - sd0 = 2. - mu0 = 4. - sd = 3. - mu = -5. - - data = sd * np.random.randn(n) + mu - d = n / sd ** 2 + 1 / sd0 ** 2 - mu_post = (n * np.mean(data) / sd ** 2 + mu0 / sd0 ** 2) / d - if using_minibatch: - data = pm.Minibatch(data) - return dict( - n=n, - data=data, - mu_post=mu_post, - d=d, - mu0=mu0, - sd0=sd0, - sd=sd, - ) - - -@pytest.fixture(scope='module') -def simple_model(simple_model_data): - with Model() as model: - mu_ = Normal( - 'mu', mu=simple_model_data['mu0'], - sd=simple_model_data['sd0'], testval=0) - Normal('x', mu=mu_, sd=simple_model_data['sd'], - observed=simple_model_data['data'], - total_size=simple_model_data['n']) - return model - - -@pytest.fixture( - scope='function', - params=[ - dict(cls=ADVI, init=dict()), - dict(cls=FullRankADVI, init=dict()), - dict(cls=SVGD, init=dict(n_particles=500)), - dict(cls=ASVGD, init=dict(temperature=1.5)), - ], - ids=lambda d: d['cls'].__name__ -) -def inference(request, simple_model, scale_cost_to_minibatch): - cls = request.param['cls'] - init = request.param['init'] - with simple_model: - return cls(scale_cost_to_minibatch=scale_cost_to_minibatch, **init) - - -@pytest.fixture('function') -def fit_kwargs(inference, using_minibatch): - cb = [pm.callbacks.CheckParametersConvergence( - every=500, - diff='relative', tolerance=0.001), - pm.callbacks.CheckParametersConvergence( - every=500, - diff='absolute', tolerance=0.0001)] - _select = { - (ADVI, 'full'): dict( - obj_optimizer=pm.adagrad_window(learning_rate=0.02, n_win=50), - n=5000, callbacks=cb - ), - (ADVI, 'mini'): dict( - obj_optimizer=pm.adagrad_window(learning_rate=0.01, n_win=50), - n=12000, callbacks=cb - ), - (FullRankADVI, 'full'): dict( - obj_optimizer=pm.adagrad_window(learning_rate=0.01, n_win=50), - n=6000, callbacks=cb - ), - (FullRankADVI, 'mini'): dict( - obj_optimizer=pm.rmsprop(learning_rate=0.001), - n=12000, callbacks=cb - ), - (SVGD, 'full'): dict( - obj_optimizer=pm.sgd(learning_rate=0.01), - n=1000, callbacks=cb - ), - (SVGD, 'mini'): dict( - obj_optimizer=pm.adagrad_window(learning_rate=0.01, n_win=5), - n=3000, callbacks=cb - ), - (ASVGD, 'full'): dict( - obj_optimizer=pm.adagrad_window(learning_rate=0.01, n_win=50), - n=1000, callbacks=cb - ), - (ASVGD, 'mini'): dict( - obj_optimizer=pm.adagrad_window(learning_rate=0.01, n_win=50), - n=3000, callbacks=cb - ) - } - if using_minibatch: - key = 'mini' - else: - key = 'full' - return _select[(type(inference), key)] - - -def test_fit_oo(inference, - fit_kwargs, - simple_model_data - ): - trace = inference.fit(**fit_kwargs).sample(10000) - mu_post = simple_model_data['mu_post'] - d = simple_model_data['d'] - np.testing.assert_allclose(np.mean(trace['mu']), mu_post, rtol=0.05) - np.testing.assert_allclose(np.std(trace['mu']), np.sqrt(1. / d), rtol=0.1) - - -@pytest.fixture('module') -def another_simple_model(): - _model = models.simple_model()[1] - with _model: - pm.Potential('pot', tt.ones((10, 10))) - return _model - - -@pytest.fixture(params=[ - dict(name='advi', kw=dict(start={})), - dict(name='fullrank_advi', kw=dict(start={})), - dict(name='svgd', kw=dict(start={}))], - ids=lambda d: d['name'] -) -def fit_method_with_object(request, another_simple_model): - _select = dict( - advi=ADVI, - fullrank_advi=FullRankADVI, - svgd=SVGD - ) - with another_simple_model: - return _select[request.param['name']]( - **request.param['kw']) - - -@pytest.mark.parametrize( - ['method', 'kwargs', 'error'], - [ - ('undefined', dict(), KeyError), - (1, dict(), TypeError), - ('advi', dict(total_grad_norm_constraint=10), None), - ('advi->fullrank_advi', dict(frac=.1), None), - ('advi->fullrank_advi', dict(frac=1), ValueError), - ('fullrank_advi', dict(), None), - ('svgd', dict(total_grad_norm_constraint=10), None), - ('svgd', dict(start={}), None), - ('asvgd', dict(start={}, total_grad_norm_constraint=10), None), - ], -) -def test_fit_fn_text(method, kwargs, error, another_simple_model): - with another_simple_model: - if error is not None: - with pytest.raises(error): - fit(10, method=method, **kwargs) - else: - fit(10, method=method, **kwargs) - - -def test_fit_fn_oo(fit_method_with_object, another_simple_model): - with another_simple_model: - fit(10, method=fit_method_with_object) - - -def test_error_on_local_rv_for_svgd(another_simple_model): - with another_simple_model: - with pytest.raises(ValueError) as e: - fit(10, method='svgd', local_rv={ - another_simple_model.free_RVs[0]: (0, 1)}) - assert e.match('does not support AEVB') - - -@pytest.mark.parametrize( - 'diff', - [ - 'relative', - 'absolute' - ] -) -@pytest.mark.parametrize( - 'ord', - [1, 2, np.inf] -) -def test_callbacks_convergence(diff, ord): - cb = pm.variational.callbacks.CheckParametersConvergence(every=1, diff=diff, ord=ord) - - class _approx: - params = (theano.shared(np.asarray([1, 2, 3])), ) - - approx = _approx() - - with pytest.raises(StopIteration): - cb(approx, None, 1) - cb(approx, None, 10) - - -def test_tracker_callback(): - import time - tracker = pm.callbacks.Tracker( - ints=lambda *t: t[-1], - ints2=lambda ap, h, j: j, - time=time.time, - ) - for i in range(10): - tracker(None, None, i) - assert 'time' in tracker.hist - assert 'ints' in tracker.hist - assert 'ints2' in tracker.hist - assert (len(tracker['ints']) - == len(tracker['ints2']) - == len(tracker['time']) - == 10) - assert tracker['ints'] == tracker['ints2'] == list(range(10)) - tracker = pm.callbacks.Tracker( - bad=lambda t: t # bad signature - ) - with pytest.raises(TypeError): - tracker(None, None, 1) \ No newline at end of file From 03b6d2111f7fcb036d991bad68c54b8c7cf1be38 Mon Sep 17 00:00:00 2001 From: Maxim Kochurov Date: Sat, 24 Jun 2017 01:06:31 +0300 Subject: [PATCH 09/24] refactor tests --- pymc3/tests/conftest.py | 43 ++++++----------------- pymc3/tests/test_minibatches.py | 29 +++++++++++++++ pymc3/tests/test_variational_inference.py | 10 +++--- 3 files changed, 45 insertions(+), 37 deletions(-) diff --git a/pymc3/tests/conftest.py b/pymc3/tests/conftest.py index cac0f3d9b8..ff6199870c 100644 --- a/pymc3/tests/conftest.py +++ b/pymc3/tests/conftest.py @@ -1,40 +1,17 @@ -import theano import numpy as np +import theano +import pymc3 as pm import pytest -class DataSampler(object): - """ - Not for users - """ - def __init__(self, data, batchsize=50, random_seed=42, dtype='floatX'): - self.dtype = theano.config.floatX if dtype == 'floatX' else dtype - self.rng = np.random.RandomState(random_seed) - self.data = data - self.n = batchsize - - def __iter__(self): - return self - - def __next__(self): - idx = (self.rng - .uniform(size=self.n, - low=0.0, - high=self.data.shape[0] - 1e-16) - .astype('int64')) - return np.asarray(self.data[idx], self.dtype) - - next = __next__ - - -@pytest.fixture(scope="session", autouse=True) +@pytest.fixture(scope="function", autouse=True) def theano_config(): config = theano.configparser.change_flags(compute_test_value='raise') with config: yield -@pytest.fixture(scope='session', autouse=True) +@pytest.fixture(scope='function', autouse=True) def exception_verbosity(): config = theano.configparser.change_flags( exception_verbosity='high') @@ -42,7 +19,7 @@ def exception_verbosity(): yield -@pytest.fixture(scope='session') +@pytest.fixture(scope='function', autouse=False) def strict_float32(): config = theano.configparser.change_flags( warn_float64='raise', @@ -51,8 +28,8 @@ def strict_float32(): yield -@pytest.fixture('session', params=[ - np.random.uniform(size=(1000, 10)) -]) -def datagen(request): - return DataSampler(request.param) +@pytest.fixture('function', autouse=False) +def seeded_test(): + # TODO: use this instead of SeededTest + np.random.seed(42) + pm.set_tt_rng(42) diff --git a/pymc3/tests/test_minibatches.py b/pymc3/tests/test_minibatches.py index 6bb57bd446..96d916cc4c 100644 --- a/pymc3/tests/test_minibatches.py +++ b/pymc3/tests/test_minibatches.py @@ -13,6 +13,35 @@ from pymc3.theanof import GeneratorOp +class _DataSampler(object): + """ + Not for users + """ + def __init__(self, data, batchsize=50, random_seed=42, dtype='floatX'): + self.dtype = theano.config.floatX if dtype == 'floatX' else dtype + self.rng = np.random.RandomState(random_seed) + self.data = data + self.n = batchsize + + def __iter__(self): + return self + + def __next__(self): + idx = (self.rng + .uniform(size=self.n, + low=0.0, + high=self.data.shape[0] - 1e-16) + .astype('int64')) + return np.asarray(self.data[idx], self.dtype) + + next = __next__ + + +@pytest.fixture('module') +def datagen(): + return _DataSampler(np.random.uniform(size=(1000, 10))) + + def integers(): i = 0 while True: diff --git a/pymc3/tests/test_variational_inference.py b/pymc3/tests/test_variational_inference.py index 83d0e53010..f3a4e78941 100644 --- a/pymc3/tests/test_variational_inference.py +++ b/pymc3/tests/test_variational_inference.py @@ -14,10 +14,10 @@ from pymc3.tests import models -@pytest.fixture('function', autouse=True) -def set_seed(): - np.random.seed(42) - pm.set_tt_rng(42) +pytestmark = pytest.mark.usefixtures( + 'strict_float32', + 'seeded_test' +) def test_elbo(): @@ -182,6 +182,7 @@ def fit_kwargs(inference, using_minibatch): return _select[(type(inference), key)] +@pytest.mark.run('first') def test_fit_oo(inference, fit_kwargs, simple_model_data): @@ -193,6 +194,7 @@ def test_fit_oo(inference, def test_profile(inference, fit_kwargs): + fit_kwargs = fit_kwargs.copy() # they are module level fit_kwargs['n'] = 100 fit_kwargs.pop('callbacks') inference.run_profiling(**fit_kwargs).summary() From 3ef423ece37c44a5eaeba3f64d4f63ddf65cac30 Mon Sep 17 00:00:00 2001 From: Maxim Kochurov Date: Sat, 24 Jun 2017 01:12:05 +0300 Subject: [PATCH 10/24] use yield fixtures --- pymc3/tests/conftest.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pymc3/tests/conftest.py b/pymc3/tests/conftest.py index ff6199870c..42a0b33780 100644 --- a/pymc3/tests/conftest.py +++ b/pymc3/tests/conftest.py @@ -4,14 +4,14 @@ import pytest -@pytest.fixture(scope="function", autouse=True) +@pytest.yield_fixture(scope="function", autouse=True) def theano_config(): config = theano.configparser.change_flags(compute_test_value='raise') with config: yield -@pytest.fixture(scope='function', autouse=True) +@pytest.yield_fixture(scope='function', autouse=True) def exception_verbosity(): config = theano.configparser.change_flags( exception_verbosity='high') @@ -19,7 +19,7 @@ def exception_verbosity(): yield -@pytest.fixture(scope='function', autouse=False) +@pytest.yield_fixture(scope='function', autouse=False) def strict_float32(): config = theano.configparser.change_flags( warn_float64='raise', From 289a5597580d391287ba29ceef6d6c84b200107b Mon Sep 17 00:00:00 2001 From: Maxim Kochurov Date: Sat, 24 Jun 2017 15:15:41 +0300 Subject: [PATCH 11/24] refactor tests, make them faster --- pymc3/tests/test_variational_inference.py | 32 ++++++++++------------- 1 file changed, 14 insertions(+), 18 deletions(-) diff --git a/pymc3/tests/test_variational_inference.py b/pymc3/tests/test_variational_inference.py index f3a4e78941..67459720fa 100644 --- a/pymc3/tests/test_variational_inference.py +++ b/pymc3/tests/test_variational_inference.py @@ -128,7 +128,6 @@ def init_(**kw): @pytest.fixture('function') def inference(inference_spec, simple_model, scale_cost_to_minibatch): - with simple_model: return inference_spec(scale_cost_to_minibatch=scale_cost_to_minibatch) @@ -137,10 +136,10 @@ def inference(inference_spec, simple_model, scale_cost_to_minibatch): def fit_kwargs(inference, using_minibatch): cb = [pm.callbacks.CheckParametersConvergence( every=500, - diff='relative', tolerance=0.001), + diff='relative', tolerance=0.01), pm.callbacks.CheckParametersConvergence( every=500, - diff='absolute', tolerance=0.0001)] + diff='absolute', tolerance=0.01)] _select = { (ADVI, 'full'): dict( obj_optimizer=pm.adagrad_window(learning_rate=0.02, n_win=50), @@ -151,28 +150,28 @@ def fit_kwargs(inference, using_minibatch): n=12000, callbacks=cb ), (FullRankADVI, 'full'): dict( - obj_optimizer=pm.adagrad_window(learning_rate=0.01, n_win=50), + obj_optimizer=pm.adagrad_window(learning_rate=0.007, n_win=50), n=6000, callbacks=cb ), (FullRankADVI, 'mini'): dict( - obj_optimizer=pm.rmsprop(learning_rate=0.007), + obj_optimizer=pm.adagrad_window(learning_rate=0.007, n_win=50), n=12000, callbacks=cb ), (SVGD, 'full'): dict( - obj_optimizer=pm.sgd(learning_rate=0.01), - n=1000, callbacks=cb + obj_optimizer=pm.adagrad_window(learning_rate=0.07, n_win=7), + n=200, callbacks=cb ), (SVGD, 'mini'): dict( - obj_optimizer=pm.adagrad_window(learning_rate=0.01, n_win=5), - n=3000, callbacks=cb + obj_optimizer=pm.adagrad_window(learning_rate=0.07, n_win=7), + n=200, callbacks=cb ), (ASVGD, 'full'): dict( - obj_optimizer=pm.adagrad_window(learning_rate=0.02, n_win=50), - n=2000, obj_n_mc=300, callbacks=cb + obj_optimizer=pm.adagrad_window(learning_rate=0.07, n_win=10), + n=500, obj_n_mc=300, callbacks=cb ), (ASVGD, 'mini'): dict( - obj_optimizer=pm.adagrad_window(learning_rate=0.02, n_win=50), - n=1700, obj_n_mc=300, callbacks=cb + obj_optimizer=pm.adagrad_window(learning_rate=0.07, n_win=10), + n=500, obj_n_mc=300, callbacks=cb ) } if using_minibatch: @@ -193,11 +192,8 @@ def test_fit_oo(inference, np.testing.assert_allclose(np.std(trace['mu']), np.sqrt(1. / d), rtol=0.1) -def test_profile(inference, fit_kwargs): - fit_kwargs = fit_kwargs.copy() # they are module level - fit_kwargs['n'] = 100 - fit_kwargs.pop('callbacks') - inference.run_profiling(**fit_kwargs).summary() +def test_profile(inference): + inference.run_profiling(n=100).summary() @pytest.fixture('module') From 027ef006448058f83aab6733f167f06471c00444 Mon Sep 17 00:00:00 2001 From: Maxim Kochurov Date: Sat, 24 Jun 2017 18:33:44 +0300 Subject: [PATCH 12/24] fix tests (cherry picked from commit d12fddc) --- pymc3/tests/test_variational_inference.py | 22 ++++++++-------------- 1 file changed, 8 insertions(+), 14 deletions(-) diff --git a/pymc3/tests/test_variational_inference.py b/pymc3/tests/test_variational_inference.py index 67459720fa..f76c566933 100644 --- a/pymc3/tests/test_variational_inference.py +++ b/pymc3/tests/test_variational_inference.py @@ -134,44 +134,38 @@ def inference(inference_spec, simple_model, scale_cost_to_minibatch): @pytest.fixture('function') def fit_kwargs(inference, using_minibatch): - cb = [pm.callbacks.CheckParametersConvergence( - every=500, - diff='relative', tolerance=0.01), - pm.callbacks.CheckParametersConvergence( - every=500, - diff='absolute', tolerance=0.01)] _select = { (ADVI, 'full'): dict( obj_optimizer=pm.adagrad_window(learning_rate=0.02, n_win=50), - n=5000, callbacks=cb + n=5000 ), (ADVI, 'mini'): dict( obj_optimizer=pm.adagrad_window(learning_rate=0.01, n_win=50), - n=12000, callbacks=cb + n=12000 ), (FullRankADVI, 'full'): dict( obj_optimizer=pm.adagrad_window(learning_rate=0.007, n_win=50), - n=6000, callbacks=cb + n=6000 ), (FullRankADVI, 'mini'): dict( obj_optimizer=pm.adagrad_window(learning_rate=0.007, n_win=50), - n=12000, callbacks=cb + n=12000 ), (SVGD, 'full'): dict( obj_optimizer=pm.adagrad_window(learning_rate=0.07, n_win=7), - n=200, callbacks=cb + n=200 ), (SVGD, 'mini'): dict( obj_optimizer=pm.adagrad_window(learning_rate=0.07, n_win=7), - n=200, callbacks=cb + n=200 ), (ASVGD, 'full'): dict( obj_optimizer=pm.adagrad_window(learning_rate=0.07, n_win=10), - n=500, obj_n_mc=300, callbacks=cb + n=500, obj_n_mc=300 ), (ASVGD, 'mini'): dict( obj_optimizer=pm.adagrad_window(learning_rate=0.07, n_win=10), - n=500, obj_n_mc=300, callbacks=cb + n=500, obj_n_mc=300 ) } if using_minibatch: From 33fdca8d1dcc47e0926990605eec19aacbfee4ee Mon Sep 17 00:00:00 2001 From: Maxim Kochurov Date: Sat, 24 Jun 2017 18:44:26 +0300 Subject: [PATCH 13/24] fix tests duration (cherry picked from commit d181cea) --- .travis.yml | 6 ++++-- pymc3/tests/conftest.py | 10 ++++++---- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/.travis.yml b/.travis.yml index 1bccb95685..33c2e75719 100644 --- a/.travis.yml +++ b/.travis.yml @@ -19,13 +19,15 @@ install: env: - PYTHON_VERSION=2.7 FLOATX='float32' TESTCMD="--durations=10 --ignore=pymc3/tests/test_examples.py --cov-append --ignore=pymc3/tests/test_distributions_random.py --ignore=pymc3/tests/test_variational_inference.py --ignore=pymc3/tests/test_shared.py --ignore=pymc3/tests/test_smc.py --ignore=pymc3/tests/test_updates.py --ignore=pymc3/tests/test_posteriors.py --ignore=pymc3/tests/test_sampling.py" - PYTHON_VERSION=2.7 FLOATX='float32' RUN_PYLINT="true" TESTCMD="--durations=10 --cov-append pymc3/tests/test_distributions_random.py pymc3/tests/test_shared.py pymc3/tests/test_smc.py pymc3/tests/test_sampling.py" - - PYTHON_VERSION=2.7 FLOATX='float32' TESTCMD="--durations=10 --cov-append pymc3/tests/test_examples.py pymc3/tests/test_variational_inference.py pymc3/tests/test_updates.py pymc3/tests/test_posteriors.py" + - PYTHON_VERSION=2.7 FLOATX='float32' TESTCMD="--durations=10 --cov-append pymc3/tests/test_examples.py pymc3/tests/test_posteriors.py" + - PYTHON_VERSION=2.7 FLOATX='float32' TESTCMD="--durations=10 --cov-append pymc3/tests/test_variational_inference.py pymc3/tests/test_updates.py" - PYTHON_VERSION=2.7 FLOATX='float64' TESTCMD="--durations=10 --ignore=pymc3/tests/test_examples.py --cov-append --ignore=pymc3/tests/test_distributions_random.py --ignore=pymc3/tests/test_variational_inference.py --ignore=pymc3/tests/test_shared.py --ignore=pymc3/tests/test_smc.py --ignore=pymc3/tests/test_updates.py --ignore=pymc3/tests/test_posteriors.py --ignore=pymc3/tests/test_sampling.py" - PYTHON_VERSION=2.7 FLOATX='float64' RUN_PYLINT="true" TESTCMD="--durations=10 --cov-append pymc3/tests/test_distributions_random.py pymc3/tests/test_shared.py pymc3/tests/test_smc.py pymc3/tests/test_sampling.py" - PYTHON_VERSION=2.7 FLOATX='float64' TESTCMD="--durations=10 --cov-append pymc3/tests/test_examples.py pymc3/tests/test_variational_inference.py pymc3/tests/test_updates.py pymc3/tests/test_posteriors.py" - PYTHON_VERSION=3.6 FLOATX='float64' TESTCMD="--durations=10 --cov-append --ignore=pymc3/tests/test_examples.py --ignore=pymc3/tests/test_distributions_random.py --ignore=pymc3/tests/test_variational_inference.py --ignore=pymc3/tests/test_shared.py --ignore=pymc3/tests/test_smc.py --ignore=pymc3/tests/test_updates.py --ignore=pymc3/tests/test_posteriors.py --ignore=pymc3/tests/test_sampling.py" - PYTHON_VERSION=3.6 FLOATX='float64' TESTCMD="--durations=10 --cov-append pymc3/tests/test_distributions_random.py pymc3/tests/test_shared.py pymc3/tests/test_smc.py pymc3/tests/test_sampling.py" - - PYTHON_VERSION=3.6 FLOATX='float64' TESTCMD="--durations=10 --cov-append pymc3/tests/test_examples.py pymc3/tests/test_variational_inference.py pymc3/tests/test_updates.py pymc3/tests/test_posteriors.py" + - PYTHON_VERSION=3.6 FLOATX='float64' TESTCMD="--durations=10 --cov-append pymc3/tests/test_examples.py pymc3/tests/test_posteriors.py" + - PYTHON_VERSION=3.6 FLOATX='float64' TESTCMD="--durations=10 --cov-append pymc3/tests/test_variational_inference.py pymc3/tests/test_updates.py" script: - . ./scripts/test.sh $TESTCMD diff --git a/pymc3/tests/conftest.py b/pymc3/tests/conftest.py index 42a0b33780..8c2bf0c444 100644 --- a/pymc3/tests/conftest.py +++ b/pymc3/tests/conftest.py @@ -21,10 +21,12 @@ def exception_verbosity(): @pytest.yield_fixture(scope='function', autouse=False) def strict_float32(): - config = theano.configparser.change_flags( - warn_float64='raise', - floatX='float32') - with config: + if theano.config.floatX == 'float32': + config = theano.configparser.change_flags( + warn_float64='raise') + with config: + yield + else: yield From a1800667dffa5b24722519695fa92f31568bc443 Mon Sep 17 00:00:00 2001 From: Maxim Kochurov Date: Sat, 24 Jun 2017 19:55:25 +0300 Subject: [PATCH 14/24] fix deterministic for Empirical, more tests --- pymc3/tests/test_variational_inference.py | 63 ++++++++++++++++++++++- pymc3/variational/approximations.py | 10 ++-- 2 files changed, 67 insertions(+), 6 deletions(-) diff --git a/pymc3/tests/test_variational_inference.py b/pymc3/tests/test_variational_inference.py index f76c566933..2e97550a49 100644 --- a/pymc3/tests/test_variational_inference.py +++ b/pymc3/tests/test_variational_inference.py @@ -1,5 +1,6 @@ import pytest import pickle +import operator import numpy as np from theano import theano, tensor as tt import pymc3 as pm @@ -111,7 +112,7 @@ def simple_model(simple_model_data): @pytest.fixture('module', params=[ dict(cls=ADVI, init=dict()), dict(cls=FullRankADVI, init=dict()), - dict(cls=SVGD, init=dict(n_particles=500)), + dict(cls=SVGD, init=dict(n_particles=500, jitter=1)), dict(cls=ASVGD, init=dict(temperature=1.)), ], ids=lambda d: d['cls'].__name__) def inference_spec(request): @@ -303,10 +304,70 @@ def binomial_model_inference(binomial_model, inference_spec): return inference_spec() +@pytest.mark.run(after='test_replacements') def test_n_mc(binomial_model_inference): binomial_model_inference.fit(10, obj_n_mc=2) +@pytest.mark.run(after='test_sample_replacements') +def test_replacements(binomial_model_inference): + d = tt.bscalar() + d.tag.test_value = 1 + approx = binomial_model_inference.approx + p = approx.model.p + p_t = p ** 3 + p_s = approx.apply_replacements(p_t) + sampled = [p_s.eval() for _ in range(100)] + assert any(map( + operator.ne, + sampled[1:], sampled[:-1]) + ) # stochastic + + p_d = approx.apply_replacements(p_t, deterministic=True) + sampled = [p_d.eval() for _ in range(100)] + assert all(map( + operator.eq, + sampled[1:], sampled[:-1]) + ) # deterministic + + p_r = approx.apply_replacements(p_t, deterministic=d) + sampled = [p_r.eval({d: 1}) for _ in range(100)] + assert all(map( + operator.eq, + sampled[1:], sampled[:-1]) + ) # deterministic + sampled = [p_r.eval({d: 0}) for _ in range(100)] + assert any(map( + operator.ne, + sampled[1:], sampled[:-1]) + ) # stochastic + + +def test_sample_replacements(binomial_model_inference): + i = tt.iscalar() + i.tag.test_value = 1 + approx = binomial_model_inference.approx + p = approx.model.p + p_t = p ** 3 + p_s = approx.sample_node(p_t, size=100) + sampled = p_s.eval() + assert any(map( + operator.ne, + sampled[1:], sampled[:-1]) + ) # stochastic + assert sampled.shape[0] == 100 + + p_d = approx.sample_node(p_t, size=i) + sampled = p_d.eval({i: 100}) + assert any(map( + operator.ne, + sampled[1:], sampled[:-1]) + ) # deterministic + assert sampled.shape[0] == 100 + sampled = p_d.eval({i: 101}) + assert sampled.shape[0] == 101 + + def test_pickling(binomial_model_inference): inference = pickle.loads(pickle.dumps(binomial_model_inference)) inference.fit(20) diff --git a/pymc3/variational/approximations.py b/pymc3/variational/approximations.py index a2d95e4ce7..3453aaf5b1 100644 --- a/pymc3/variational/approximations.py +++ b/pymc3/variational/approximations.py @@ -336,14 +336,14 @@ def _initial_part_matrix(self, part, size, deterministic): return tt.switch( deterministic, tt.repeat( - self.mean.reshape((1, -1)), - size if size is not None else 1), + self.mean, + size if size is not None else 1, -1), self.histogram[self.randidx(size)]) else: if deterministic: return tt.repeat( - self.mean.reshape((1, -1)), - size if size is not None else 1) + self.mean, + size if size is not None else 1, -1) else: return self.histogram[self.randidx(size)] @@ -359,7 +359,7 @@ def histogram(self): @property def mean(self): - return self.histogram.mean(0) + return self.histogram.mean(0, keepdims=True) @property def cov(self): From 7d4562a55419ee36b3fd98178660fe6a2873bb00 Mon Sep 17 00:00:00 2001 From: Maxim Kochurov Date: Sat, 24 Jun 2017 20:01:34 +0300 Subject: [PATCH 15/24] change mean shape for Empirical (cherry picked from commit c74337e) --- pymc3/variational/approximations.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pymc3/variational/approximations.py b/pymc3/variational/approximations.py index 3453aaf5b1..44125ae70c 100644 --- a/pymc3/variational/approximations.py +++ b/pymc3/variational/approximations.py @@ -336,13 +336,13 @@ def _initial_part_matrix(self, part, size, deterministic): return tt.switch( deterministic, tt.repeat( - self.mean, + self.mean.dimshuffle('x', 0), size if size is not None else 1, -1), self.histogram[self.randidx(size)]) else: if deterministic: return tt.repeat( - self.mean, + self.mean.dimshuffle('x', 0), size if size is not None else 1, -1) else: return self.histogram[self.randidx(size)] @@ -359,12 +359,12 @@ def histogram(self): @property def mean(self): - return self.histogram.mean(0, keepdims=True) + return self.histogram.mean(0) @property def cov(self): x = (self.histogram - self.mean) - return x.T.dot(x) / self.histogram.shape[0] + return x.T.dot(x) / pm.floatX(self.histogram.shape[0]) @classmethod def from_noise(cls, size, jitter=.01, local_rv=None, From 64ee80a33e4c213f5cc709db61678dca58ce5762 Mon Sep 17 00:00:00 2001 From: Maxim Kochurov Date: Sat, 24 Jun 2017 20:13:33 +0300 Subject: [PATCH 16/24] support multitrace in Empirical (cherry picked from commit 48c38b2) --- pymc3/tests/test_variational_inference.py | 11 +++++++++++ pymc3/variational/approximations.py | 9 ++++++--- 2 files changed, 17 insertions(+), 3 deletions(-) diff --git a/pymc3/tests/test_variational_inference.py b/pymc3/tests/test_variational_inference.py index 2e97550a49..11a7fa5cc8 100644 --- a/pymc3/tests/test_variational_inference.py +++ b/pymc3/tests/test_variational_inference.py @@ -373,6 +373,17 @@ def test_pickling(binomial_model_inference): inference.fit(20) +def test_empirical_from_trace(another_simple_model): + with another_simple_model: + step = pm.Metropolis() + trace = pm.sample(100, step=step) + emp = Empirical(trace) + assert emp.histogram.shape[0].eval() == 100 + trace = pm.sample(100, step=step, njobs=4) + emp = Empirical(trace) + assert emp.histogram.shape[0].eval() == 400 + + def test_multiple_replacements(inference_spec): _, model, _ = models.exponential_beta(n=2) x = model.x diff --git a/pymc3/variational/approximations.py b/pymc3/variational/approximations.py index 44125ae70c..f4a5b60777 100644 --- a/pymc3/variational/approximations.py +++ b/pymc3/variational/approximations.py @@ -302,9 +302,12 @@ def create_shared_params(self, **kwargs): if trace is None: histogram = np.atleast_2d(self.gbij.map(self.model.test_point)) else: - histogram = np.empty((len(trace), self.global_size)) - for i in range(len(trace)): - histogram[i] = self.gbij.map(trace[i]) + histogram = np.empty((len(trace) * len(trace.chains), self.global_size)) + i = 0 + for t in trace.chains: + for j in range(len(trace)): + histogram[i] = self.gbij.map(trace.point(j, t)) + i += 1 return theano.shared(pm.floatX(histogram), 'histogram') def randidx(self, size=None): From 57376268f7f6fc7458105431b8c17a74083c8633 Mon Sep 17 00:00:00 2001 From: Maxim Kochurov Date: Sat, 24 Jun 2017 20:22:36 +0300 Subject: [PATCH 17/24] fix convergence tests --- pymc3/tests/test_variational_inference.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pymc3/tests/test_variational_inference.py b/pymc3/tests/test_variational_inference.py index 11a7fa5cc8..5f501c04b7 100644 --- a/pymc3/tests/test_variational_inference.py +++ b/pymc3/tests/test_variational_inference.py @@ -154,11 +154,11 @@ def fit_kwargs(inference, using_minibatch): ), (SVGD, 'full'): dict( obj_optimizer=pm.adagrad_window(learning_rate=0.07, n_win=7), - n=200 + n=300 ), (SVGD, 'mini'): dict( obj_optimizer=pm.adagrad_window(learning_rate=0.07, n_win=7), - n=200 + n=300 ), (ASVGD, 'full'): dict( obj_optimizer=pm.adagrad_window(learning_rate=0.07, n_win=10), From 020c428bf6895ae06b8df41d073df72d2f5d3bed Mon Sep 17 00:00:00 2001 From: Maxim Kochurov Date: Sat, 24 Jun 2017 23:15:11 +0300 Subject: [PATCH 18/24] do not duplicate tests (cherry picked from commit 0f97ba3) --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 33c2e75719..c0d10c0997 100644 --- a/.travis.yml +++ b/.travis.yml @@ -23,7 +23,7 @@ env: - PYTHON_VERSION=2.7 FLOATX='float32' TESTCMD="--durations=10 --cov-append pymc3/tests/test_variational_inference.py pymc3/tests/test_updates.py" - PYTHON_VERSION=2.7 FLOATX='float64' TESTCMD="--durations=10 --ignore=pymc3/tests/test_examples.py --cov-append --ignore=pymc3/tests/test_distributions_random.py --ignore=pymc3/tests/test_variational_inference.py --ignore=pymc3/tests/test_shared.py --ignore=pymc3/tests/test_smc.py --ignore=pymc3/tests/test_updates.py --ignore=pymc3/tests/test_posteriors.py --ignore=pymc3/tests/test_sampling.py" - PYTHON_VERSION=2.7 FLOATX='float64' RUN_PYLINT="true" TESTCMD="--durations=10 --cov-append pymc3/tests/test_distributions_random.py pymc3/tests/test_shared.py pymc3/tests/test_smc.py pymc3/tests/test_sampling.py" - - PYTHON_VERSION=2.7 FLOATX='float64' TESTCMD="--durations=10 --cov-append pymc3/tests/test_examples.py pymc3/tests/test_variational_inference.py pymc3/tests/test_updates.py pymc3/tests/test_posteriors.py" + - PYTHON_VERSION=2.7 FLOATX='float64' TESTCMD="--durations=10 --cov-append pymc3/tests/test_examples.py pymc3/tests/test_posteriors.py" - PYTHON_VERSION=3.6 FLOATX='float64' TESTCMD="--durations=10 --cov-append --ignore=pymc3/tests/test_examples.py --ignore=pymc3/tests/test_distributions_random.py --ignore=pymc3/tests/test_variational_inference.py --ignore=pymc3/tests/test_shared.py --ignore=pymc3/tests/test_smc.py --ignore=pymc3/tests/test_updates.py --ignore=pymc3/tests/test_posteriors.py --ignore=pymc3/tests/test_sampling.py" - PYTHON_VERSION=3.6 FLOATX='float64' TESTCMD="--durations=10 --cov-append pymc3/tests/test_distributions_random.py pymc3/tests/test_shared.py pymc3/tests/test_smc.py pymc3/tests/test_sampling.py" - PYTHON_VERSION=3.6 FLOATX='float64' TESTCMD="--durations=10 --cov-append pymc3/tests/test_examples.py pymc3/tests/test_posteriors.py" From 009b095ed851e7a8d2cca611914ac5c6871ceab7 Mon Sep 17 00:00:00 2001 From: Maxim Kochurov Date: Mon, 26 Jun 2017 21:36:24 +0300 Subject: [PATCH 19/24] some small changes in docs+code --- pymc3/variational/approximations.py | 86 ++++++++++++++++------------- pymc3/variational/operators.py | 2 +- pymc3/variational/opvi.py | 82 +++++++++++---------------- 3 files changed, 81 insertions(+), 89 deletions(-) diff --git a/pymc3/variational/approximations.py b/pymc3/variational/approximations.py index f4a5b60777..b2038ed18c 100644 --- a/pymc3/variational/approximations.py +++ b/pymc3/variational/approximations.py @@ -49,6 +49,18 @@ class MeanField(Approximation): Sticking the Landing: A Simple Reduced-Variance Gradient for ADVI approximateinference.org/accepted/RoederEtAl2016.pdf """ + + def __init__(self, local_rv=None, model=None, cost_part_grad_scale=1, + scale_cost_to_minibatch=False, + random_seed=None, start=None, **kwargs): + super(MeanField, self).__init__( + local_rv=local_rv, model=model, + cost_part_grad_scale=cost_part_grad_scale, + scale_cost_to_minibatch=scale_cost_to_minibatch, + random_seed=random_seed, **kwargs + ) + self.shared_params = self.create_shared_params(start=start) + @node_property def mean(self): return self.shared_params['mu'] @@ -65,8 +77,7 @@ def cov(self): def std(self): return rho2sd(self.rho) - def create_shared_params(self, **kwargs): - start = kwargs.get('start') + def create_shared_params(self, start=None): if start is None: start = self.model.test_point else: @@ -81,7 +92,7 @@ def create_shared_params(self, **kwargs): @node_property def symbolic_random_global_matrix(self): - initial = self._symbolic_initial_global_matrix + initial = self.symbolic_initial_global_matrix sd = rho2sd(self.rho) mu = self.mean return sd * initial + mu @@ -139,7 +150,7 @@ class FullRank(Approximation): def __init__(self, local_rv=None, model=None, cost_part_grad_scale=1, scale_cost_to_minibatch=False, - gpu_compat=False, random_seed=None, **kwargs): + gpu_compat=False, random_seed=None, start=None, **kwargs): super(FullRank, self).__init__( local_rv=local_rv, model=model, cost_part_grad_scale=cost_part_grad_scale, @@ -147,6 +158,24 @@ def __init__(self, local_rv=None, model=None, cost_part_grad_scale=1, random_seed=random_seed, **kwargs ) self.gpu_compat = gpu_compat + self.shared_params = self.create_shared_params(start=start) + + def create_shared_params(self, start=None): + if start is None: + start = self.model.test_point + else: + start_ = self.model.test_point.copy() + pm.sampling._update_start_vals(start_, start, self.model) + start = start_ + start = pm.floatX(self.gbij.map(start)) + n = self.global_size + L_tril = ( + np.eye(n) + [np.tril_indices(n)] + .astype(theano.config.floatX) + ) + return {'mu': theano.shared(start, 'mu'), + 'L_tril': theano.shared(L_tril, 'L_tril')} @node_property def L(self): @@ -177,25 +206,6 @@ def tril_index_matrix(self): ] = np.arange(num_tril_entries) return tril_index_matrix - def create_shared_params(self, **kwargs): - start = kwargs.get('start') - if start is None: - start = self.model.test_point - else: - start_ = self.model.test_point.copy() - pm.sampling._update_start_vals(start_, start, self.model) - start = start_ - start = pm.floatX(self.gbij.map(start)) - n = self.global_size - L_tril = ( - np.eye(n) - [np.tril_indices(n)] - .astype(theano.config.floatX) - ) - return {'mu': theano.shared(start, 'mu'), - 'L_tril': theano.shared(L_tril, 'L_tril') - } - @node_property def symbolic_log_q_W_global(self): """log_q_W samples over q for global vars @@ -208,7 +218,7 @@ def symbolic_log_q_W_global(self): @node_property def symbolic_random_global_matrix(self): # (samples, dim) or (dim, ) - initial = self._symbolic_initial_global_matrix.T + initial = self.symbolic_initial_global_matrix.T # (dim, dim) L = self.L # (dim, ) @@ -289,16 +299,9 @@ def __init__(self, trace, local_rv=None, super(Empirical, self).__init__( local_rv=local_rv, scale_cost_to_minibatch=scale_cost_to_minibatch, model=model, trace=trace, random_seed=random_seed, **kwargs) + self.shared_params = self.create_shared_params(trace=trace) - def check_model(self, model, **kwargs): - trace = kwargs.get('trace') - if (trace is not None - and not all([var.name in trace.varnames - for var in model.free_RVs])): - raise ValueError('trace has not all FreeRV') - - def create_shared_params(self, **kwargs): - trace = kwargs.get('trace') + def create_shared_params(self, trace=None): if trace is None: histogram = np.atleast_2d(self.gbij.map(self.model.test_point)) else: @@ -308,7 +311,14 @@ def create_shared_params(self, **kwargs): for j in range(len(trace)): histogram[i] = self.gbij.map(trace.point(j, t)) i += 1 - return theano.shared(pm.floatX(histogram), 'histogram') + return dict(histogram=theano.shared(pm.floatX(histogram), 'histogram')) + + def check_model(self, model, **kwargs): + trace = kwargs.get('trace') + if (trace is not None + and not all([var.name in trace.varnames + for var in model.free_RVs])): + raise ValueError('trace has not all FreeRV') def randidx(self, size=None): if size is None: @@ -352,19 +362,19 @@ def _initial_part_matrix(self, part, size, deterministic): @property def symbolic_random_global_matrix(self): - return self._symbolic_initial_global_matrix + return self.symbolic_initial_global_matrix @property def histogram(self): """Shortcut to flattened Trace """ - return self.shared_params + return self.shared_params['histogram'] - @property + @node_property def mean(self): return self.histogram.mean(0) - @property + @node_property def cov(self): x = (self.histogram - self.mean) return x.T.dot(x) / pm.floatX(self.histogram.shape[0]) diff --git a/pymc3/variational/operators.py b/pymc3/variational/operators.py index f491e0026f..7ef4ef0bd9 100644 --- a/pymc3/variational/operators.py +++ b/pymc3/variational/operators.py @@ -105,7 +105,7 @@ def __init__(self, approx, temperature=1): self.temperature = temperature def get_input(self): - if hasattr(self.approx, 'histogram'): + if isinstance(self.approx, pm.Empirical): return self.approx.histogram else: return self.approx.symbolic_random_total_matrix diff --git a/pymc3/variational/opvi.py b/pymc3/variational/opvi.py index 3c13e7f9e1..4511012375 100644 --- a/pymc3/variational/opvi.py +++ b/pymc3/variational/opvi.py @@ -397,14 +397,8 @@ def cast_to_list(params): ------- list """ - if isinstance(params, list): - return params - elif isinstance(params, tuple): - return list(params) - elif isinstance(params, dict): + if isinstance(params, dict): return list(params.values()) - elif isinstance(params, theano.compile.SharedVariable): - return [params] elif params is None: return [] else: @@ -485,23 +479,21 @@ class Approximation(object): custom implementation of the following methods: - :code:`.create_shared_params(**kwargs)` - Returns {dict|list|theano.shared} + Returns dict - - :code:`.random_global(size=None, no_rand=False)` - Generate samples from posterior. If `no_rand==False`: - sample from MAP of initial distribution. + - :code:`symbolic_random_global_matrix` node property + It takes internally `symbolic_initial_global_matrix` + and performs appropriate transforms. To memoize result + one should use :code:`@node_property` wrapper instead :code:`@property`. Returns TensorVariable - - :code:`.log_q_W_global(z)` - It is needed only if used with operator - that requires :math:`logq` of an approximation - Returns Scalar + - :code:`.symbolic_log_q_W_global` node property + Should use vectorized form if possible and return vector of size `(s,)` + It is needed only if used with operator that requires :math:`logq` + of an approximation. Returns vector You can also override the following methods: - - :code:`._setup(**kwargs)` - Do some specific stuff having `kwargs` before calling :func:`Approximation.create_shared_params` - - :code:`.check_model(model, **kwargs)` Do some specific check for model having `kwargs` @@ -511,12 +503,14 @@ class Approximation(object): There are some defaults class attributes for approximation classes that can be optionally overridden. - - :code:`initial_dist_name` + - :code:`initial_dist_local_name = 'normal'` + :code:`initial_dist_global_name = 'normal'` string that represents name of the initial distribution. In most cases if will be `uniform` or `normal` - - :code:`initial_dist_map` - float where initial distribution has maximum density + - :code:`initial_dist_local_map = 0. + :code:`initial_dist_global_map = 0.` + point where initial distribution has maximum density References ---------- @@ -531,6 +525,7 @@ class Approximation(object): initial_dist_global_name = 'normal' initial_dist_local_map = 0. initial_dist_global_map = 0. + shared_params = None def __init__(self, local_rv=None, model=None, cost_part_grad_scale=1, @@ -540,7 +535,10 @@ def __init__(self, local_rv=None, model=None, self.scale_cost_to_minibatch = theano.shared(np.int8(0)) if scale_cost_to_minibatch: self.scale_cost_to_minibatch.set_value(1) - self.cost_part_grad_scale = pm.floatX(cost_part_grad_scale) + if not isinstance(cost_part_grad_scale, theano.Variable): + self.cost_part_grad_scale = theano.shared(pm.floatX(cost_part_grad_scale)) + else: + self.cost_part_grad_scale = pm.floatX(cost_part_grad_scale) self._seed = random_seed self._rng = tt_rng(random_seed) self.model = model @@ -560,28 +558,25 @@ def get_transformed(v): self._g_order = ArrayOrdering(self.global_vars) self._l_order = ArrayOrdering(self.local_vars) self.gbij = DictToArrayBijection(self._g_order, {}) - self._symbolic_initial_local_matrix = tt.matrix(self.__class__.__name__ + '_symbolic_initial_local_matrix') - self._symbolic_initial_local_matrix.tag.test_value = np.random.rand(2, self.local_size).astype( - self._symbolic_initial_local_matrix.dtype + self.lbij = DictToArrayBijection(self._l_order, {}) + self.symbolic_initial_local_matrix = tt.matrix(self.__class__.__name__ + '_symbolic_initial_local_matrix') + self.symbolic_initial_local_matrix.tag.test_value = np.random.rand(2, self.local_size).astype( + self.symbolic_initial_local_matrix.dtype ) - self._symbolic_initial_global_matrix = tt.matrix(self.__class__.__name__ + '_symbolic_initial_global_matrix') - self._symbolic_initial_global_matrix.tag.test_value = np.random.rand(2, self.global_size).astype( - self._symbolic_initial_global_matrix.dtype + self.symbolic_initial_global_matrix = tt.matrix(self.__class__.__name__ + '_symbolic_initial_global_matrix') + self.symbolic_initial_global_matrix.tag.test_value = np.random.rand(2, self.global_size).astype( + self.symbolic_initial_global_matrix.dtype ) self.global_flat_view = model.flatten( vars=self.global_vars, order=self._g_order, - # inputvar=self._symbolic_initial_global_matrix ) self.local_flat_view = model.flatten( vars=self.local_vars, order=self._l_order, - # inputvar=self._symbolic_initial_local_matrix ) - self._setup(**kwargs) - self.shared_params = self.create_shared_params(**kwargs) - self.symbolic_n_samples = self._symbolic_initial_global_matrix.shape[0] + self.symbolic_n_samples = self.symbolic_initial_global_matrix.shape[0] _global_view = property(lambda self: self.global_flat_view.view) _local_view = property(lambda self: self.local_flat_view.view) @@ -612,9 +607,6 @@ def seed(self, random_seed=None): self._seed = random_seed self._rng.seed(random_seed) - def _setup(self, **kwargs): - pass - def get_global_vars(self, **kwargs): return [v for v in self.model.free_RVs if v not in self.known] @@ -632,14 +624,6 @@ def check_model(self, model, **kwargs): ): # pragma: no cover raise ValueError('Model should not include discrete RVs') - def create_shared_params(self, **kwargs): - """ - Returns - ------- - {dict|list|theano.shared} - """ - pass - def construct_replacements(self, include=None, exclude=None, more_replacements=None): """Construct replacements with given conditions @@ -701,8 +685,6 @@ def apply_replacements(self, node, deterministic=False, latent variables to be excluded for replacements more_replacements : `dict` add custom replacements to graph, e.g. change input source - new : bool - reinit random generator for replacements? Returns ------- @@ -796,7 +778,7 @@ def _initial_part_matrix(self, part, size, deterministic): (self.local_size, self.initial_dist_local_name, self.initial_dist_local_map), (self.global_size, self.initial_dist_global_name, self.initial_dist_global_map) ) - dtype = self._symbolic_initial_global_matrix.dtype + dtype = self.symbolic_initial_global_matrix.dtype if size is None: size = 1 if length == 0: # in this case theano fails to compute sample of correct size @@ -864,8 +846,8 @@ def set_size_and_deterministic(self, node, s, d): self.logp: self.single_symbolic_logp }) return theano.clone(node, { - self._symbolic_initial_local_matrix: initial_local, - self._symbolic_initial_global_matrix: initial_global, + self.symbolic_initial_local_matrix: initial_local, + self.symbolic_initial_global_matrix: initial_global, }) @property @@ -962,7 +944,7 @@ def symbolic_random_global_matrix(self): @node_property def symbolic_random_local_matrix(self): mu, rho = self.__local_mu_rho - e = self._symbolic_initial_local_matrix + e = self.symbolic_initial_local_matrix return e * rho2sd(rho) + mu @node_property From 8708772b324afdd104d67cb27f5e0bb9d2a61b1c Mon Sep 17 00:00:00 2001 From: ferrine Date: Mon, 26 Jun 2017 21:56:28 +0300 Subject: [PATCH 20/24] update LDA notebook --- docs/source/notebooks/lda-advi-aevb.ipynb | 108 ++++++++++++---------- 1 file changed, 58 insertions(+), 50 deletions(-) diff --git a/docs/source/notebooks/lda-advi-aevb.ipynb b/docs/source/notebooks/lda-advi-aevb.ipynb index 03f61d837d..cd04d868b2 100644 --- a/docs/source/notebooks/lda-advi-aevb.ipynb +++ b/docs/source/notebooks/lda-advi-aevb.ipynb @@ -69,9 +69,9 @@ "output_type": "stream", "text": [ "Loading dataset...\n", - "done in 1.742s.\n", + "done in 1.576s.\n", "Extracting tf features for LDA...\n", - "done in 2.463s.\n" + "done in 2.303s.\n" ] } ], @@ -111,9 +111,9 @@ "outputs": [ { "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAeEAAAFJCAYAAACsBZWNAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJztvXucXHV9//86M7P33SS7ZAkiBEgAURCBqG1UDAkKKFCj\nUQKxwVbqAy38UFotkC9S1HwNqbWtUEGwlbaIX02DVS5SQRMM12iQRBPuIVly2SSTvc/sbWbO5/fH\n7Mycy+dzbnNm5uzk9fxjd845n8v7fG7vz/tzO5oQQoAQQgghVSdWawEIIYSQIxUqYUIIIaRGUAkT\nQgghNYJKmBBCCKkRVMKEEEJIjaASJoQQQmpEotoRJpMjoYbX2dmKgYHRUMM8EmE6lg/TsHyYhuXD\nNAyHMNOxu7tD+WzaW8KJRLzWItQFTMfyYRqWD9OwfJiG4VCtdJz2SpgQQgiZrlAJE0IIITWCSpgQ\nQgipEVTChBBCSI2gEiaEEEJqBJUwIYQQUiOohAkhhJAaQSVMCCGE1AhPSrivrw+LFi3Czp07Tfc3\nbNiAZcuWYfny5Vi3bl1FBCSEEELqFddjKzOZDG655RY0Nzfb7q9Zswbr169HS0sLrrjiCixevBjd\n3d0VE5YQQgipJ1wt4bVr1+Lyyy/H0Ucfbbq/c+dOzJ07FzNnzkRjYyMWLFiALVu2VExQQkj9oAuB\nZ3ccwMjoZK1FIaSmOFrCP/3pT9HV1YVzzz0X99xzj+lZKpVCR0fpUOq2tjakUinXCDs7W0M/k9Pp\ncGziHaZj+TANvfHE83vw/YdexGkndOJb133Q9IxpWD5Mw3CoRjo6KuEHHngAmqbh2WefxUsvvYQb\nbrgBd911F7q7u9He3o50Ol10m06nTUpZRdhf9+ju7gj9y0xHIkzH8mEaemfnngEAwMs9A6Y0YxqW\nD9MwHMJMRydl7qiE77///uLvlStX4tZbby3O+c6fPx89PT0YHBxEa2srtmzZgquuuioUgQkhhJAj\nAd/fE37ooYcwOjqK5cuX48Ybb8RVV10FIQSWLVuGOXPmVEJGQgghpC7xrITvu+8+AHkLuMCSJUuw\nZMmS8KUihBBCjgB4WAchhBBSI6iECSGEkBpBJUwIIYTUCCphQgghpEZQCRNCCCE1gkqYEEIIqRFU\nwoQQQkiNoBImhBBCagSVMCGEEFIjqIQJIYSQGkElTAghhNQIKmFCCCGkRlAJE0IIITWCSpgQQgip\nEVTChBBCSI2gEiaEEEJqBJUwIYQQUiOohAkhhJAaQSVMCCGE1AgqYUIIIaRGUAkTQgghNYJKmBBC\nCKkRVMKEEEJIjaASJoQQQmoElTAhhBBSI6iECSGEkBqRcHOQy+Vw8803Y9euXYjH41izZg3mzp1b\nfH7vvfdi/fr16OrqAgB87Wtfw7x58yonMSGEEFInuCrhjRs3AgB+/OMfY/PmzVizZg3uuuuu4vMd\nO3Zg7dq1OOOMMyonJSGEEFKHuCrhD33oQzjvvPMAAPv378fs2bNNz3fs2IF77rkHyWQS5513Hq6+\n+uqKCEoIIYTUG65KGAASiQRuuOEGPP7447j99ttNzy6++GKsWLEC7e3tuPbaa7Fx40YsXrxYGVZn\nZysSiXh5Ulvo7u4INbwjFaZj+TANvdHW1lT8bU0zpmH5MA3DoRrpqAkhhFfHyWQSl112GR555BG0\ntrZCCIFUKoWOjryg999/PwYHB3HNNdc4hDFSvtQGurs7Qg/zSITpWD5MQ+88+PQu/OzJXQCAH9y4\npHifaVg+TMNwCDMdnZS56+ron/3sZ7j77rsBAC0tLdA0DfF43pJNpVK45JJLkE6nIYTA5s2bOTdM\nCCGEeMR1OPqCCy7ATTfdhE9/+tPIZrNYtWoVHnvsMYyOjmL58uW4/vrrceWVV6KxsRELFy7EokWL\nqiE3IYQQMu1xVcKtra34zne+o3y+dOlSLF26NFShCCGEkCMBHtZBCCGE1AgqYUIIIaRGUAkTQggh\nNYJKmBBCCKkRVMKEEEJIjaASJoQQQmoElTAhhBBSI6iECSGEkBpBJUwIIYTUCCphQgghpEZQCRNC\nqo/nb7cRUt9QCRNCCCE1gkqYEFJ9tFoLQEg0oBImhBBCagSVMCGEEFIjqIQJIYSQGkElTAghhNQI\nKmFCCCGkRlAJE0IIITWCSpgQQgipEVTChBBCSI2gEiaEEEJqBJUwIYQQUiOohAkh1YcfcCAEAJUw\nIYQQUjOohAkh1YcfcCAEgAclnMvlcNNNN+Hyyy/Hpz/9abz55pum5xs2bMCyZcuwfPlyrFu3rmKC\nEkIIIfWGqxLeuHEjAODHP/4xrrvuOqxZs6b4LJPJYM2aNfjBD36A++67Dz/5yU+QTCYrJy0hhBBS\nR7gq4Q996EP4xje+AQDYv38/Zs+eXXy2c+dOzJ07FzNnzkRjYyMWLFiALVu2VE5aQohvRrb8DhN7\n3oQQAk/v34z+8QEMPfUkMuwwT3uGB8fw0rZeCMGVbn5445UkkgdGai0GACDhyVEigRtuuAGPP/44\nbr/99uL9VCqFjo6O4nVbWxtSqZRjWJ2drUgk4gHFldPd3eHuiLjCdCyfqKVhbmICr37vuwCAjrtX\n40cvP4CThxtx8cN7EWtqwsJ1P6qJXG1tTcXf1jSLWhpGmX//5ycxOZHDifNnY+5JXcX7TEM1Qhe4\n63+eAADc8u1LHd1WIx09KWEAWLt2Lb785S/jsssuwyOPPILW1la0t7cjnU4X3aTTaZNSljEwMBpc\nWgnd3R1IJqPRo5nOMB3LJ4ppqI+PFX+/eeggACA3nJdRn5iombzp9ETxt1GGKKZhlJmcyAEADuwf\nQkt7AwCmoRu6Xho1cEqnMNPRSZm7Dkf/7Gc/w9133w0AaGlpgaZpiMfzluz8+fPR09ODwcFBTE5O\nYsuWLTj77LNDEZoQQgipd1wt4QsuuAA33XQTPv3pTyObzWLVqlV47LHHMDo6iuXLl+PGG2/EVVdd\nBSEEli1bhjlz5lRDbkIIIWTa46qEW1tb8Z3vfEf5fMmSJViyZEmoQhFCCCGVIVqL2HhYByGEEFIj\nqIQJIYQcMURtNxeVMCGk+kSsISSkVlAJE0IIITWCSpgQUn34AQdCAFAJE0IIOZKI2FQIlTAhhBBS\nI6iECSGEHDGIiJnCVMKEEEJIjaASJqSOidqeSEJqTsTqBJUwIaT6RKwhnP4wQacrVMKE1DM0hQkx\nEbUaQSVMCKk+3CdMCAAqYULqnKj1+wmpMRGrElTChBBCSI2gEiaknolYr79IVOUiRwDRKnxUwoQQ\nQkiNoBImpJ6J6upoLswiNSJqVYJKmBBCCKkRVMKEEEJIjaASJqSeidrYGyE1JmpVgkqYEEIIqRFU\nwoTUMVH7bBshtSdadYJKmBBSfaLVDhJSM6iECalnqOwIiTRUwoSQ6sN9woQAABJODzOZDFatWoV9\n+/ZhcnISX/jCF3D++ecXn997771Yv349urq6AABf+9rXMG/evMpKTAjxTtSWghJSY6JWJRyV8IMP\nPohZs2bhW9/6FgYGBvDxj3/cpIR37NiBtWvX4owzzqi4oIQQQki94aiEL7roIlx44YXF63g8bnq+\nY8cO3HPPPUgmkzjvvPNw9dVXV0ZKQkhAItbtJ4SYcFTCbW1tAIBUKoXrrrsOX/rSl0zPL774YqxY\nsQLt7e249tprsXHjRixevNgxws7OViQScUc3funu7gg1vCMVpmP5RC0NJxNZvDH1e8aMFtvzWsnb\n1taklCFqaTgdmDGjxZRuTEM1o+nJ4m+3dKpGOjoqYQDo7e3FNddcgxUrVuDSSy8t3hdC4DOf+Qw6\nOvJCLlq0CC+++KKrEh4YGC1TZDPd3R1IJkdCDfNIhOlYPlFMw+xgqvh7eHjM9rxW8qbTE1IZopiG\n04Hh4fFiujENnRkbLSlhp3QKMx2dlLnj6ujDhw/js5/9LL7yla/gk5/8pOlZKpXCJZdcgnQ6DSEE\nNm/ezLlhQgghxAeOlvD3vvc9DA8P484778Sdd94JAPjUpz6FsbExLF++HNdffz2uvPJKNDY2YuHC\nhVi0aFFVhCaEeIVzwoREGUclfPPNN+Pmm29WPl+6dCmWLl0aulCEEELIkQAP6yCkjonankhCak3U\n6gSVMCGEEFIjqIQJqWei1u0vEFGxCKk2VMKEEEJIjaASJqSuiajJyQ84kFoRsdEhKmFCCCGkRlAJ\nE1LPRKvTT0jNiVqVoBImhBBCagSVMCF1TdT6/YTUmIhVCSphQgghpEZQCRNSz0RsJWiRiIpF6p+o\nFT0qYUIIIaRGUAkTUs9ErdtfgPuESa2I2OgQlTAhhBBSI6iECaljRGRNYUIIQCVMCKkF7BsQAoBK\nmJD6hsqOEBMRmxKmEiaE1AAuzAqZiGkW4hkqYULqmah1+wkhJqiECSGEkBpBJUxIXUNLmBAjImKj\nQ1TChBBCSI2gEiaknolWp58QYoFKmBBSfdg5IAQAlTAhdQ61HSFRhkqYEFJ9uE+YEABAwulhJpPB\nqlWrsG/fPkxOTuILX/gCzj///OLzDRs24Lvf/S4SiQSWLVuGyy67rOICE0J8ELGVoITUmqhVCUcl\n/OCDD2LWrFn41re+hYGBAXz84x8vKuFMJoM1a9Zg/fr1aGlpwRVXXIHFixeju7u7KoITQggh0x3H\n4eiLLroIX/ziF4vX8Xi8+Hvnzp2YO3cuZs6cicbGRixYsABbtmypnKQkMhw+OIIXt+6very6EPjf\nzW8iOThmeza48deY2Le37Dhe6n8VW5Pbyw7HL2PZMTzWsxGjmdFQw/XS6/9Dcgde7HtF+iwz3ofh\nQ8+h/3AK6//pv/Dipk1lyZPRs3i85wlM6GnffvelerFp77NlxV+vhGHdDYwP4tmffg+jb+4uP7AQ\n0Scm0P/oI8gOD4cUYrRMYUdLuK2tDQCQSqVw3XXX4Utf+lLxWSqVQkdHh8ltKpVyjbCzsxWJRNzV\nnR+6uzvcHRFXvKbjXbc9AQA4693HY2ZnawUlMvP0tv1Yt/F1PL5lD/7r1ouK90fffBOv3n8fAOD9\nP3+grDiu2fBvAIB1y+8K5D9oWbzndw/iV288hYFsP679078IFIaM0fE29BR+Z3Tb8+7uDty94T8B\nyN/594//XwiRw/NbDiM5ORc7Ht+NRcsuDizPw6/8Cj/b+Qt0xucAOLsog1UmGdds+DsAwML5Z+LY\nGccElqEeGcYAurvnFa+DlMM7/t9tuPAXL2PvL54rux6FSc/9D+PwA+uRfeM1nH7rV8sOL66VbE+3\ndKqGbnFUwgDQ29uLa665BitWrMCll15avN/e3o50utSbTafTJqWsYmAg3J5+d3cHksmRUMM8EgmS\njgcPDmMym6uQRHb2HRgCAAyMTJhkHdvfV/wdVlkIEk45ZXHvwMH8/8EDoZbnif5Sx3hwyD6CYIxL\nFq8Q+fwdG83/n4y3lCXf3r5DAIDhXL80Xi9p2JscQMNEW2AZ6pH+4ZFiugUth32DB4u/o9SmDu/p\nBQCk9uwNRa7B/pIOcgovTN3ipMwdh6MPHz6Mz372s/jKV76CT37yk6Zn8+fPR09PDwYHBzE5OYkt\nW7bg7LPPDkVgMk2I1qgOIYRMOxwt4e9973sYHh7GnXfeiTvvvBMA8KlPfQpjY2NYvnw5brzxRlx1\n1VUQQmDZsmWYM2dOVYQm0YA6eBoQUiYxrwmpDI5K+Oabb8bNN9+sfL5kyRIsWbIkdKHINIEtMyGR\ngFVx+sLDOgipa8Juntnck+lN1PYJUwmTwAg2yEcQPOIqyrAuTl+ohElwWO+jT8jdfo15TqY90SrE\nVMKEEB9EqwEjU0RtjJV4hkqYkHqGbTMhZiJWJ6iESWDY+T5yYF5HHWbQdIVKmJRBdSu+MrY60BCV\nW1hjD5dLrOqP6V8D1IRdN6KWVlTCJDB1oPuIZ7Spv8z0SMJsmbZQCZNpg9KC06a/badVyD4Vsp4S\nG2wyjQi9bkSs/FMJE0LINCdieoX4gEqYkCOMsuwKzkFEEh7W4Z2opRWVMAmMdKiT1CWFnJ7+A/91\nCqvitIVKmJB6RtpRYotdb0TNuos0EUsqKmFCiA8i1oKRPMyWaQuVMAkMR6OnAZI8CnT+s+BANCGV\ngEqYEEKmOewPT1+ohElguDBrOhBOHnFhFqkXotZsUQkTQnwQsRaMAODCrOkMlTAh9UzUuv2kMjCb\nfRCtxKISJsGpclnmBxwChGsIthBHoIVZJNLUQRVQEvoHHCKWVlTCJDARK8ukokzNBjPTIwozZrpC\nJUyCU+UuJT/gEIRw84hfUSLVpnJ1IxpQCZPAsDkmJBpwYdb0hUqYBIdzwqFRsUbUkDblzAlP/xSu\nc+o4g+q9g0ElTAJTz/uE6/ndAiFsPwghIUAlTAJTbT1VzTnhave+633ei1SWeu4ahV03otbB9qSE\nt23bhpUrV9ru33vvvbj44ouxcuVKrFy5Em+88UboApIoE63CTMgRC6vitCXh5uD73/8+HnzwQbS0\ntNie7dixA2vXrsUZZ5xREeFItIlYhzJUhBD1cUajcU546new19LK8EsqTx1XxjrH1RKeO3cu7rjj\nDumzHTt24J577sEVV1yBu+++O3ThSLSpZyVMyHSCdXH64moJX3jhhdi7d6/02cUXX4wVK1agvb0d\n1157LTZu3IjFixc7htfZ2YpEIh5MWgXd3R2hhnek4jcdZ85sqWrat3c0F38b4x3ua8Ueyf1ymN3d\ngUTMfzkNGn9DYz6uhkQ81DQdPlxKm9bWxvwPQ4M9e3Z78bcs3jcLP6bm3YXCnVda9+ZlMFrU1vDc\nwp/V2YruLtZ5I83NCVO6BckjpzypJQPNDQCAeCwWilyTY9nib7fwqpEOrkpYhRACn/nMZ9DRkRdy\n0aJFePHFF12V8MDAaNAopXR3dyCZHAk1zCORIOk4ODha1bRPjYwXfxvjHRscld4vh2RyGImYv+pR\nTlnMTOby/7O5UNN0zFDfRkcnbc8PJYeLv53iNS5mKUe+0bG8DEbDzRielzQcGEgjmWOdNzI+nimm\nW+ByaMiUKLWpE+MZAEBO10ORy6iDnMILU7c4KfPAq6NTqRQuueQSpNNpCCGwefNmzg0fadTxEFj9\nvFr9vEmBet83GgSmyPTFtyX80EMPYXR0FMuXL8f111+PK6+8Eo2NjVi4cCEWLVpUCRlJRKn2Un8e\n1hEFuDQrioRReqKas9OnbgTDkxI+7rjjsG7dOgDApZdeWry/dOlSLF26tDKSkchT11WjDhQ7YO4o\nFU/Mgn3FtHtA0Wmi6yRrwiWMNDlC0nVa7hMmREpUPuBQATGqXU2ny2EdIhJyRqsRJZVlutSNoFAJ\nk8BEp0NZCUEi83LhI/nGsGci0B7Wcc6UQfmpEoGsPSKhEiaBqbYSVkUXteGlIER93iva0pE6qAJK\nol43yoVKmJRB/VaOunkz6Zyw4XGVxQmH6Sl1ZQkhTY6UZI3Ye1IJk8BE5gMOFaDa1nXN5r18v2ft\nBy3r2eqrJbXPWTmcEyYk6lSkVa6Tlt5kCUse+w2uLGHCod6HJwMRSpIcGekatbekEiaB4T7h8Ii+\nYqlva2S6o0e+/AQn+nWjPKiESWDqQPcpqctXK3xFyfRydfmmJADakVIUItZwUQmT4ERln3BFqL85\n4XKGo0vuam8R18NqeOIdzgkToiAybWEFBInMu5WL8UUkL+VXoUUjWaIhRaRgkngmaklFJUwCE7XC\nHC7193ZlvVGEkiNCokQGpsn0hUqYBCciC7PqYXiymotPzIN7fk/Mqu+hwSOZqM4Jh143IvaeVMIk\nMHWg+5TU46u5zQk7d2aipHzrMXfKpJ4rY51DJUwCU+2tA+oPOFRgTrhOFmYJ2ZywYnW00ztHqYmn\nvqkMUepmGeHCLEJU1HNjWIfvVi+vVO/7RoNQD1MyRypUwiQwUfmAQz2YRhVTLKa0ka2O9hdMND5l\nSKzU84FZYdeNqHVYqIRJGUSrMIdJPVpb8jcyDEdPmzlhYiOE4socrg1UwiQw0fmAQziC1LKHXJV5\nL1GIy3ZrWlGPHaRyqec04ZwwISqiUu9DksPYkNVjoyZ7J//vHIEGsf6ypmzCSJKoblGqd6iESWCi\n8gGHqM3xBKE6c8LFyEIJhkSIOs6g8OeEQw2ubKiESWAiVpZDJWoVNRRcNgo7v3LeAhaRMITrMXPK\nJYw0YbrWAiphEpjqf/heRSXmhOtjn7BsH7B5Ttj5bGk7tdfCVBV26nl1NOeECVERlUpbATmORGvL\n6Y2jlRrRkiYKhNEhrm9VJycKU1lUwiQw3CccHpWbE5b9DBLXkdhEkygQft2IVntBJUwCU2/WYr29\njx3n1dGOjVOEkiYK1kvUYJJMX6iESXDqrOJ7/5jB9MH4HoWfxq0oQmopS8Ip/qdFHEXC6EAeKVuU\nola1PSnhbdu2YeXKlbb7GzZswLJly7B8+XKsW7cudOFItInMwqyw5Kj3wzqkTL8Ts+p/xCIAdZwk\nlawbUVDICTcH3//+9/Hggw+ipaXFdD+TyWDNmjVYv349WlpacMUVV2Dx4sXo7u6umLCEyJn+X1Gq\nHH4O46iXdz7yqJ/yeuThqoTnzp2LO+64A3/3d39nur9z507MnTsXM2fOBAAsWLAAW7ZswUc+8pHK\nSOpCNpvD757cjdlz2nHKO+YAAF7sewVDE8NYeOx7IITAT//3FXS3NODYk2dj14FhfPjdxxf9b33t\nMMYmslh4xjF5v7v7kRwcw6Kz3goAePS5Hrz9xE6ceMwMW9w7+l7B8MQwOptnoW+sH+9/658AAA7s\nHcKhAyNo2/8k9s9uwLnv+yQAoHd0Aq8MprHoLZ3QNA1CCPz+2TdxwvyjMHtOOwCg/xcPY/il7Tgw\nA3j7B/8MW1IzsLt3GB+K78XsE96KtnecjrGJLB7d/CZOPkVHX/YAFh//gXxa5HQ8/MxuvO+MY3B0\nZ2tRTiEEfrjtYSR7MzhxeD5STXFcev4p6OxoQs/PH8QD+4Zxcnca+8UCvP9dx+OYrlZYOTw0ZgjP\n/EwXAg8/sxsL3nY03jq7DQAw9tprGO/Zjcl3/AkO7B3Cu957PGQIIfCb3gF0tzSid3QCi9/ShXis\n1APe238Yv973ayD2FkAvFdv0aBr/++Qf0d51Fk7s34bfbtqFc943F4lEHLteO4zh5KsYHWvF/skU\n3nLcTLzvne/Ett/uwcHeYVzwsdMBALuG3sSuod34wFsXFsP9Zc9GXH7qx4v588izPXjXybNx/NFT\n+fO/vwA0DV0Xeivvjzz7Eg69sgeDI03oPPEoXJB9DV3vex/++OooTjl9TrER7R+ewJ5DqWI8bkwe\n6MXIbzej65I/gxaTD2xtO6UFHaNxJIdacM7LWWQ0YFfnmehOvwlt3wi6DpyA/mN60DO8F2/rOjkf\n7ugBDP5uI7ZtPwovHxVHhyGvn922AxNdgzinMYaxdBteefB3SLzzXAw1NmDeCbPwtrmdyGXHMHTg\nGbzw6BYMtL8D737PQpx48mxMpPcBAHLIALEcjtbj2P/mII6dO8sk8xv79+HZLS/jpJNOQ1NjIxa8\nrdS5373jDWx/cztOWHwCRl7pROvcmZjd2IgD/aMYHc8gHT+EV3uBPz/vDJw4M1+GdV3H80/34JTT\n52BWVyuEEPj5k0/idfE6jhfvw7tmz0JTYxwnnWo2IlJ/2Ib+Rx7C7KWfQMtpb8fwwafwzAstSA8m\nsGjxfOx5ox+HDqXQOzSGVEsP3r7ndYyceQnet+AkzJmqP0MHBvDL//dbLHzPUZh85wn44e9+i9zQ\nIDq7WjErdjRmam/BR/5kLkaHx7F58x5s7U/jzz5wEuYefBmxtjbs6WrBz198An+98GMYH9yB/3wh\nh4+995049Zi3FOUcTQ57Ki8AsDeZwguvJnHJ+06Epml4+o+9aG9pgAYgEwe2nN6G7vFBdDbn82TH\nUw+j/1ASg3POw4dPasRv/vtJtJ31Hnxg0XwAwEQmh0ee7cF5Zx2LrhnNxXiSB0fw5K9fwcTbenFC\nw4noyMzCz154FW+fpeGM+UfhtLPfhW2/24M5x85A91s68NDTu7HwjGMwpzOfP8MHn0TLzNOQy+k4\n0H4SGuItmCd5nx0v7MesrhZsfeh/0N7Zho4LLkLbpl/jhPcsQPPcE0xuc3oOm3ufB9AAAPjFQ5ux\nILcfxyxdikxGx8Pr/oCGhjg++ql3ek7PcnFVwhdeeCH27t1ru59KpdDR0VG8bmtrQyqVco2ws7MV\niUTcp5jOdHd3YPfrh7F18x4AwPsW5RuS7274dwDAn71rCV7a1Y9D2w7gEID/fK4HAsDF587HzPYm\nAMDtt23Iu118CgDgH6euP/nh09DTO4z/fmInAOChb3/MFv+dU/EUWHrWhwAAd932BJozKby/55eY\nA0CcfymObp+NVb94DQDwruOPwqld7dizqx+/3bQLv920C7d8+1KMHzqEV3+6HgDQBWD/5ldw/8n5\n6YBzX/8R9gF4/88fwL8/uB0PP7Mbse2H0XTaFlx0+rmY0dSOXz7Xgwef3o1nth/AvbdcWJTrYCqJ\n5/qfRFOuAy27ugAA2+YdhaXveQv+++nfY2PubGzc2QFgH574wyH8ePVHbe/69/f+DsdM/W5tbUR3\nd6kMbHnpIH725C48+NQu/Pwf8+n09F/9XwDAr0/OF/p3LzwRsyTK/bX+FB7b11e8fmtXOz44d3bx\n+u9++c9It/Wi4bgUMm++vRjvD+/dgKHUcUh2nYimbBovP9ODrqPasPC8+bjrtiemfE8CAA5t68PH\nlnTgmQ35vJx5ZQsaGxO4ZsO/5tP2lAXF+J7a9xwuOu1cnDp7HrbvPIyfbnoDP930Bh769seQGR7B\nq+vz0y/zl34UifaSwjSmh5Gdv+tDw2gjUvFJ/HHHIeh9u/D2F3Zja+vZ2P77/Wj8UL5O9A2P4+9/\n8FtpOZPx3P/3BeTGxjD7tPmY/f732Z7n2hvwxHvyMs3sOB5Ht/8JWnp34I2jzsEbR50DrHsNx+J0\nDM7eh9u33oN1y+8CADz/2Ncxft8b2HvyX6B9BBDITYWoYeujSez900cxb2a+o5V9swE7xvdjHAL/\n/bTAQ9+x/1UQAAAgAElEQVT+GHZvfwSvbdmE4zcfxPF4CY/2dODvvrEQk+k9Rdkajt6NEw6cgp//\naCtu+falpjT813/agfhkI37+5nNIpjpN6bH32XYA7Tj0+KtoPHg8xJb92CJ027v/c+5l/OjqRQCA\nP2zZgy1P9+DFbb348tcuxK7XD+Px7MMAgBd/ewJeRS9mQzPJAQCv3v7P+Ti//Q8489612NO7Ebu3\nfRAAsP4/ni+6OwiBc0f+F8cfzOC53hhue2Uhfvi1fAftJ3duQv9EE375RBK7GuMYjP0G6AT6BIAc\nMPbbi9DUlMCbG98AAByCjn9atw03vv5fAIDvLjsBomkM/77pPhyfnYWXXpqP13a9gP9ZfWox/pP+\nsNtU9lTlEAA+O9W2LTj9LXjXKd3490fy1+84Fnjhba347RltSL50H9ZesCqfT/+xHnMA3HvyCTjl\nv36EN076c+DZPVi67F3QNA3rfvUqHn5mN156cwD/cv15xXgK9U/f24pBvQ9AH3ZBw+zDMWx8fQBn\n/UkDnvl1vi6+//J34cGnd2PTH3px360XYbjvNezpfQJDvU8g1T8DO47J5+OfWd5L6AKbfvnq1NU8\nIAWMbX4Blzz6IN589EG8/+cPmNw/setZ/O7A73Ei8obSiQ99HyPI4a1nvQPbDrXh4L58Z2bPzgEc\nc8xMx3QMC1clrKK9vR3pdLp4nU6nTUpZxcDAaNAopXR3dyCZHEF/f0mWZHLE5CaZHMGBQ/ae4qHk\nCCbHJm1urde9Br/W5zKMbmIiV/x98PAgtLGmUvx9KXTmBA4dGjH5nTw4aApPthgmmRzB/il/YrJl\n6n2GMNEkcGAq/sND4yZZkun8e8RypU7Q0PA4Dh8cxFiiCSiJivRYRvquvYdTOGZKnnRqwuSmkE66\nUKfToUMjyORytvsHB9Nmd4NpJFtKaTUmpsJrmCi+f/4dJ9AUy79/JpZ3fziZUsZvvH84OYKGxlIV\nOHB4yOT2QN8AOsUIDljyJztUyp/koSEkxvJmYqEsykiMZQEAuVzeWh2NN2M0PQy0AuNjGWQnzUrE\nSzkDgNxYfmRi4EAfhMTP4LC5vo22daBJT9hWg2hTR2EZ4zWvm87PzBVOzEoYymQmnk/35rwLJJMj\nSA33ITdpzue+w+ZyHWvIFH8X4i2kYXyyEQDQMLVaSJYeicmmKdltjwAAekYv+kseyhsIo6lJJJMj\nSB40h5e1yCGjv29Q+SwLYEYq/75tuTEMTcUDAKPj+bzNxRogNPkIR2+yZMBYxzNEUz6PR4XA6GS+\nvGbHmk2yxoWwpaEbBw+NIDmr2XRvrCUf+4GRw9IwYrls8XcyOQJN09A7VT96k2m5H11udB0ytKuF\nOjY4km9TRgdLaT0xkQWaSnEa0XV756shU2rTre57+/tMBTuBvP/Bg33oO1x6cLhQbjzWQzeclHng\n1dHz589HT08PBgcHMTk5iS1btuDss88OGlxkCbL4KPQFSyGtS6jv3Xb+CSx/7Q7W8k8UVp5Enumf\nRrV4gygUrSAymBd61X7BoW9L+KGHHsLo6CiWL1+OG2+8EVdddRWEEFi2bBnmzJlTCRk94Ufx5fvr\nHsMNIkvRbgiHsLaF1PvHsf2i61b5vb2P6b0lw6BBiOLCGnm5q32jFTp+GoQgRC9rpfjN2Xw98Ocr\nWOkJd/961LZieVLCxx13XHEL0qWXluZMlixZgiVLllRGMp9UTB9ELMPKIXSlOd3TxqKEPStCr5tr\nI4BcPP9C1/v5vdOfEPIn4mVZRqHOCqFD06bnsRfTU2oZFSpAegSGoyNrCU/HWmsgoCFsUsJh5XWl\nlFx58kVr2K5yTO9yHBbBLOFq4PDpljqwhOtGCUdJIYSv7CJKZAXzSOBGRHHkVBQp43vCQqus4o2M\nWo+MIOVR9emhKBR9Yfvhz3uFy7gX6kcJO00bWB76SfYoLMwKq6AEseqdiLr+ccM6J+y586SHr4Qr\n1Yn0Hq63MhbqsZVR+DgxgGhok/Ipvyj6CyDYoqggqCMqtLWaD/OWlnClcJy7V8yMeciMIAWtVnkc\n5ESkfCEO2Iuc5lo4qPieP3oQAcpT7lFRksQVrfyOnH/lFJ2y7+fdo1aq60YJOymEcpRFMCUc0Tlh\nycsIMf0t2qBY0yOQJWybWI4WXt9J1gBH+81CJGqmUUCqPhodgWQrWsJ+hImA3EbqRgn7oaDSvCjn\nIAo1fAvR41ChS7yydxF5jwFkqgdL2KKEPb+OYWFWSDW6cguzZHHJJfByL8w5tKhZJJWj8vVEQAtm\nMFTZQAmW51580RKuOU6FSa/6cLR8uNLBVneM1POi3eJyfdXGFMVwdPBxWedrT56CRlY+rvuEVVEa\n08unJVzsAFqUWe3nhCV+HRSu8ZFqpMY95go3hwE6VW7u1FWllAqhvJWyDjutFPY5pwsfbRIA+554\nYXJZTglWmAdTP51GOad+lDuaISpXB92oIyXs/6EnS3gaLcxyi1a6UDZvCvuWyUt8Uce+MMsbogIL\nsyqF17Ko1WKRVFSSrtKvHshiDNLu+I/HGI2bHrPKVG7RD6OdLIURdDi69nZx3ShhJ6w9nKI1InMb\n0Aq1xugP54Lg1xLWFEpbPhwtAs9r2sKraHmufGWxNQrqDxgrfnuIoxB0lZR3+Auz8veM4qsUhluO\nVTxHQ4/Aaci3lAo2JyHK4aSc/ea1BqcdEzKhrUpYmFx6eU2jG3MZcnDpEHBJB5c5HK3V7kCaulHC\njguzfPix3grSVhqHv82Fzltg9o6A18LhFr78fQM31IHmoIJFVQkCb1EyHtYR0sKsyjUAXuWTzP+G\nK4inOGuDnzetjsxVs4SNcbpZwppVCZcXn3f/HtLc1xalUnhRaI7qSAk7PZOf7yu1hBW9PX/CeBTM\ni38fuO0Dlj4XCG4Jlzn8VXNsHS6P50CbXjxKL2SnLOk8zglPe3y8S7VyO0jyVnphVviLyD0aJU7P\nggxHK9cIcU64PHxYwqXV0e7BhLlFybulZVYG3lek+hdWL2efcCRWjgfH+hm0YGdHh/M+FVuYJZFP\nOsgsmROWd1LVYXgQxlWO2uBnPjGA1EHmhH2XKy1gffQTg6W+BCj7quFoZ5wsLA9urDJEpwkCUEdK\n2Ftvyf2+TQkHksU4HG0YuvRa6mwyhLRFSWXphbU6erphkd/riWLCpxKWffO0WpSXRWF/RSmEykWU\nBLOES79dc9bqIEjnokz/VqLUqQ9K/ShhU7tozRiVEpbdK3842uzHoIS9Dr9YLWGv8QZ4rpexRSkK\nK8fLqcjWUXjvnSR/SriWzYSA1yH2WnySLhq2sJ8jD6vV5vudExYIWB9NW4HcZCrfEg7i35sREmxO\n2FQGa1RR60YJmxbLWC0cP6ujJcH6LWtlD0fb8Hqur5slLJ8TrmZvMkodV+uiKtV+crtHw09P29zc\ng6zYYR1l+XWYEw4UXj1QnTevVvr6G44un7AP6yjIH2Qhmy2sskMIRt0oYSdL2N9wtN2v703wBvea\nqXPgMZyg+1fdhqMLW5gMhTo/JRzUEg7kLTLYF+F5tRoN7rwMR3uylqs3JyxDvm1DcldzcO8ujHuc\nNSFsS7j8vAw0JxxoONqHBWmzhP3HFyxutbsgC7NMAx8RKIR1o4QDIR2O9navslgi9HpYh5slLNui\nhGk+HF0GVp3rfbrA5Mmf+8jidWFWiHPCJFwCDUeX8D0EXrWFWfVN3Shh4WBxWi0RTXE/79d8rQew\nhI3hGgu254U/lbKEC89FOJZwFPYJlzXc6lJO1B51+W8FulcLuwJ4t+5lN8Mdjo7qnLCvLUpe5s5D\neC3fw6tasJ2GfjqUVks4CMGG2b0c1RnMEjaFzS1KIWJLSz+DfcLxMqAAPu9blbBXS9jbc81yL6h1\nGiWrNgiuX1EKy8qt4ZywdJ7bp8It37Ui3rKLT1hpFsVyXB2ZfNVhS3JXbzjafU7Yctc5tIhld90o\nYbMlbHkG+cHrsnvW3qQuuecqi3nljvy+kxw25eBVCTv3VEtWkdESLmM4OoifCClut7UDyoVaprOj\nPVjCHgyIKM4Jy/enB1d8quNjywgwHHwd1lGtE7P8E2x6yHuc1nnqag1HO9WNUhiG0ccqtmdhUEdK\n2HRleSZs9/KuPJo6vhdmlX47FTplqJUeHrb2EYKWvkBaOLygysWqP+3DxsLw137fczyRs7Qkza3f\n0U8PWiJgjZOHJakT4alDH1J5clp+fgdRJIEWZvlwG8YK5PA/ZSimXBiVcKBIakbdKGGF8Tn1SJiM\nF6cTs6zzgkLA90yI2SJVW8LKTw9aZfC5MMv9U4ZWSzjYXE+w4Vs32dSxhY092eX7IB0tZg/DJNKy\nV6VzH8sZ8pNZffIpDcW7uIzw+GmS5W/h4t/x1VWjUk6e1KVUQFd/yCBA0VW9mfqzkZr/XRwwv69Z\neUk6Pbazo82dVL+vaRq9lErnHrDMEnYdjpYGVLtRurpRwo7D0Qprz+vCLL+ly1SwjUrY1sjL4wz6\nJSev+4St1nlgwztIz1tlCVdRhgLWk6ysw8/F9LTGYfDnZeGTFzeV2yfsdczP39CzJ3ltUZtv+Hpj\n2WuU8flFU7nR1G2HTYgqfPJRQzCr07exIJy+j2zHPhztM0KY89z+PW85xg6r3WCRxOEmWBnGQiWo\nHyXseKVY4Syt2DLF7L+H6ee+m8UQ2rGVEqWS36IUcNVjJIZ9glg0ipAUFm+5H0ry4r1ic8LSu44f\njvPt10/cLsaWElmHuazNUp4r4zQiwLSZv3rif+TLiskAcFosIUyNlPR2/lpih7u8k6a4qlXO148S\ndrCErU2cdTja6Ne2MMu/IWyKzemwDl0Sv0mwYnjhro42D0d7CloeXgiKrkBIXwT0hX0rmOraIX88\nrLoyOqn+a3rsfMoo97AOl85lbVF13py8CPUWJdNQre3F/Qg2FVyQuuXTvTVOF/cxxXB0UJw/P+vP\nlNGkrZsc2cdKakndKGFzR8iq7OSWcHEO1THc8k7MksVnvKOK04THMlMcblbMNyo/WBGSBvRk8QW2\nQOTv5Oddrdg7XPLxFLfpAjdM7gtKrEoKSW5Blr/qWTO9ksdhRd/NvsGvNL1c/Ds8Vi7kdHgV59EK\nzdBlK6+RdxqOdkpr38aCEObyYQrA+zso58Id3ALm4WjHpVcOW3mlC/Zc65aQ/LLsH65ifzHh5kDX\nddx666145ZVX0NjYiNWrV+OEE04oPl+9ejV+//vfo62tDQBw5513oqOjo3ISK3Cby5GtWJaPZFh7\ne0F6mPJesd3SUsUZ1BJ2FrR4bKXpo9YC/qvvlF8XuVVSeL/rSQg/tx0dWbd4WReeSP0FUPaRRGow\nqxdreVpXJikfshEpT+LJ0jCsdDU18o6msDcVGMLy3GpsUVKtlTFiXDehKRYuBsU6EmUJ3HBhzCAH\nZypHHh7XcuukqxL+1a9+hcnJSfzkJz/B1q1bcdttt+Guu+4qPt+xYwf+7d/+DV1dXRUV1B2DspP0\nlqR1uDjn5+BXcs8PpsokCVtKwAg9n5hlESL4YR3O1178GMQIF0+yuHV+hPQ+TOXFPSIv71bdhVmy\nOeFw9wTn47beCJ7LMq9aGZtm1F86U7mZeli1oUyfCjXA2dHC0u2UNg8GxWt987IXZjmuRZErYXV9\nM1jVLnKp1pfLTaTK4zoc/fzzz+Pcc88FAJx11lnYvn178Zmu6+jp6cEtt9yCyy+/HOvXr6+cpC6Y\n65Q9Ob18OxgSd3kF7i9DzEM8pd+21bdCIUdAS9i94Nifl/MpQ2+9Um9uwu6Ieskz26cMbflTUMLW\nwH1awh5M4Vof1uHXn6d5NbeREh+iSdMntNXRigf2jFeLHPLm1OpZwgblJXNjtIRDtvAdLWHFCKK9\nKgrJEzdLWLZPXt42VwNXSziVSqG9vb14HY/Hkc1mkUgkMDo6ij//8z/HX/7lXyKXy+HKK6/EGWec\ngdNOO00ZXmdnKxKJeDjST9Hd3YG2tqbidVdXOzpmNpviHBjI2fzNnNWC7u4OTGZKzzq72tB9VFvx\netasVkwaMq272zzULiv4nZ2t6O4suCs9nzmz2eS/Y0b++uDeYVP4iZkt2GeMQ/HOzU0NpnszZrbY\n0sIYX/voVJoY3qepqQEzZ5TcW+OwU/Lb2Bg3v09Hv83vqxbfM6dktNKRyZiu29qaFPGbw4/F7P3I\n1tZGzD6q3Xbf6A8A2tvN+dHa1mhyO2Mqf9o7hkz+4zOai/kzc0YzOg1hyGSe8FChGxvNdcLp3Y0U\n0rdDkV4trQ3ApOWmtBEyx9sj4HhilpOS6O7uwEBPwlZwZ8xottg38nole4+jZPnpIV0LYbW3m+tE\nb8cQcNDu/qijOtDUnG8WRS6H1wzPOjqafHUcCnGr1i0YaW5uQKEGaIAnLWBNp87OtuI9L+Wno6MZ\nnZ2lti5fLvLxxrR8GJOTE3ij+Nws06xZreju7kBLa77eaDHNc7kFzHnS1l5qr7u7OxDLNKNv6joe\nL9Xx2Ue1o7GppLYmx7L5uA3hGueErfK09zWby+5U3nS0N6FxqFQH21qbpP4rgasSbm9vRzqdLl7r\nuo5EIu+tpaUFV155JVpaWgAAf/qnf4qXX37ZUQkPDIyWK7OJ7u4OJJMjSKXGi/cO96UwNlFqefr6\nUxgq6bliJgwMjCKZHDEp4b6+FOKG3l//QBpDQxPF62RyxBS/bEilvz+F9uyIKS4AGBwcRTJR8j88\nPIZkwwiGhsZM4acG0jAiawyTyRGMT5gV1+DgKJKxEaTTcnmHh8dsMo2NZzA0aO+gWP0apCmGMD6R\nNbkZGi7lgdzvlIySZwXZCqTTE8owjOFb9/oCwOjoJJLJlKO/QpzG65GRcZPboaG8rEbZkskRpAZL\nZXhocBTZqTAKZdFKX79cFiOTk+Y8cHp3GSMjY4hL/IyOWjWwnIJCdIu30Lw5DZ8nkyOYnMzaFNbQ\nUBoqLZZ0SUN5frort0JYqZS5TpjLmzA9KyrhbNYU1vDwmMOolP29CnF7sVDHx8112YslbE2nvv40\nWhOaMg2tDA2Noa+/UfpMF1P5OFFKJ6sl3N+fRkNzHGNTZUzowjVe43sNG9q9dMrcdqQM+ZPLiaKm\nSiZHTEq4oE/MRwWXflrlSaXNdbwoy/AYJsdLHZJCG+q3HqpwUuauw9HnnHMONm3aBADYunUrTj31\n1OKz3bt3Y8WKFcjlcshkMvj973+P008/PQSR/WMdjhaW4QzFdKjNr2zk0f8+YdV8k9chj4BDiB4X\nZllWogRflBDEW7WGowOMjXtdOOd3TtgLZc8Je16R7/2gDadpkEDSlpFUFR2OtsRUcuND4JCniv1/\nRcn/iVmA9R0lC/FMw9XhDkd7nRM2lkPV6mjzFiW34WjJzyoOP1txtYQ//OEP4+mnn8bll18OIQS+\n+c1v4t5778XcuXNx/vnn49JLL8Vll12GhoYGfOxjH8Mpp5xSDbntWBZXeZl/Kq1+NftVBKuI1u7A\ntE/Ywa2ykbftXw1pdbRqXjzovKElPk8fr1cqi2CUU3ds+4Stqz+LC7OsHn3OCas6HqaDAspsBRRy\neP88Y/gLjqQLJA33/HQ85K/h5+AQdcXWVCdmyeaElXnpGLkvtCCe4L8a68L6PrL2oTRCE8YHHEz+\nHffPq8JWtesO+5gsyI+t1G2GW7VwVcKxWAxf//rXTffmz59f/P25z30On/vc58KXzCfmumNXDrLz\newuezG2q3TpyashkmaUq2EENX6/FwfuJWZZTYkJaHe1F0GoV7SCrlu0L5xRK2NSV9qKEVW7CU3x+\nOjeeOnVCKA7ryN/zdsBAgAKiFEfWi/YTgDU8T5FKAvE+HB15rMaKpCPmmE4BXtm8T9h4/Ks3/ypL\n2HRoZ875AB3zqPqURAIWJeJNnjCom8M6bMakhxQtnVilDqdcS9hkoSsbeWukFotM0hjK41U/Mz63\nW+cBS1yQnrFKNle/QUwQlxBdvqJUOszF8p66Q4HxGremhTuE6aOHJ1vkarNKhXPSaiYrXvEiMkvY\nWTRpTGqnLgnoOZuEvO7IGgNlf6qUCtbh0CBWlerACacOlOdRj2JYTlkwVfZ1gyVc0Q84qBteL/0A\noxvrfmZVnOYHeqB8CgNXS3i6YD220nwtt2blw9HCk9/ic6klXLrnbTja4t/LcLT0fXTVI3P8RktY\nFOLzrxFcRtE9+Sne9+jOEc38XjLMvW/nToSXLUqeTuZSJIwxX8ufE5Y3PF73CcuNVqcjpzyIZGtX\ng5xRrsgDQGq5qUNxyGvFSKY9b4WDEpSHF4RA25Pgv85Yh6Olh7MY9wnb6oe/+AD1FiX77KHHzrqw\nh+t0YpaAkHZCg340JwzqxhK2d7EtvSxJHZEpQWvv0M1QlLYNCivc+3bw8oqAn3ALlnC1Cl2gOTWn\n8JTxqBSfWoEGyh/fc8LejgrwjbLj5dWx7HQstc8wvsTopeNRmjqqTPrYHjpYwsLJEg4Z/4ugNP9l\nyGKsyLybDYLy2wmVUSLr8MhQD0cbDB+nzp6ywbBcB/ymTRDqRglbF1eZTsFSFB7ZcLDMinaq/7Le\nvWdLWMjv2yq/rLGSbM1x/Z7w1H2TTACgC+/ft3WwFLwNh3mrXMFsJi+WsIMStqV7Xgrbe/kcjlZ3\nCMKcEy7TErZ7VLib2idstEJVZUfSYAZtxKWDCX4sYde8Ltw3ObIGopY/9MM6/Ifnt5/iZaGcyRIO\n+wMODpYwhOKZog01W8Lq1sO67sMgjM0YqxZ1o4TtnVm1YrX6MeexXel6PDHWSRipa2WoAUtAabhZ\nFWzRjjH4CRaX1K+HsLxbwhWqBqrKDcmWCXsne+pSnafSKJVuqnEEoqzcS46ttN0TjuauJ8mVVosP\nHL0ETz9dulLTEp10pMRLnCGUXV8jG8GitTmXFRXbcHR5bYd5i5LRUPKGl5Y45vhlMyFfz2Arq9Uz\nhetGCVt7S077hAtZoFqYZbzWhXNhc9+i5GB5KSxX21YZ41ynwo0xPJVFqkuUdH6ITQ9mlblYF3Iv\nqm6LvIfrSxzDRIMqnozDikzblivVyIK1gLigdBKmDlbOOwft0Tk3jN4WZjmXD0/GY6GNlHai3RJQ\n3eFSnR3tNBwNXVeWyzC7jBrklrBwicf3wixhVYT29NQtx1Z6NCOUmFdHq9tGpXnjoWPnOCcsVHPC\n1o9TKIMInbpRwnbL1/BECOnJSlAqQXNFdC7cksris6K6lT+pP0mj66YES0MxZqUudKcFJ2oCGMKB\nw/bEVGfFmv9G9Kz3hVkFKWxJbZnqcEVudKEqC7OkCeGh2gvAaTg6GNbU8j4nLK2Dbklv7WwasK+E\nL7gz3vNRwkMejpaRjyHcbW0m48TTK4QzigbAdC61Pam9DQWUhqMNho/bcLRScKdOQeWoGyVsrTw2\nS9jBj9mwMbvV8wEokc0x+D4xy1YArQ2EB6vDEIzS8pLNCQu3/rUal6lsT36UYQWRx0NExsbX1jDb\n8seDsB5e2tMhJhUafjeVRMchHX8Ls2Kme2a/6q13zhaKI9LHPhSSU/nSSnVAmBsDsx+hQyisb0fp\nfWatyhLOy+AQTQBL2NxOSizhnHk4WnjYgeCEue0xxKMYJZy6cIhTWJ1AcxqOVvbOhd9qHRp1qYRh\nLVxCSA0FYXhuDstcAPxWMN8nZnkyfS2PHIajVQEUFZBlixIslcs7DhVH6UXRuNicWe/I5dMkV06L\n6XK6vHLL4ix1ahzyJ2Bt1USw0QcVQtHwyBtmyZyw3aPisA734AoxWi2rshZmyW6WsTDL+cjEoi/P\n4ZtqXygrxyUxOLRFwT5laC6+srpkTHl7x8B87eW1VVuUnFFbqLJ3drLo81uUjIsKS09Mo1FUwgEw\nDhEKa2aZ5wiLM4eSHLTtnRPOZyu77RN2Grr0sg8178+09k/uxyk8ebAlP24dDVvs8vDKs4RDLPUO\nQem50uED1v27OqzzQkIanmn+yLclbGz1qrEwy4iTrDJN6rA6WlImi14L5dAWvbrRViVjOVuULH1p\ny0PzRalD7iCUgyUc9ho72bxmXmk6jIT5toQtFV9I1KxtdbTEEg5YdY31wnnKx99+dSdLWCmqtZPm\nuYNQPnWjhM11x26hyRVQ/r9uqa3Wyus4BCQdjnaX0emBTX7jEFCxB+G0RUkeTan3b200dds9T7jJ\nrWhIPAUdoA4I05ywPICcQ+XyunBOMdvgEG7pd0F5WUceKvUBB/mWDG9xeVXZyvbTWfPB3KA7J6S0\nUfRjclq86+ZMkTp06xw7BO/rqRXlcSBOJp7mXxcKYV+hXErSqbJvSHfN2lkv9lG9x6y2hB2sbKOB\nqmhzTN0HlxOzVKNntaJ+lLBtuMlcmYzXNkvYZrga/ToXMamiEbq0cKj3JjqblOaiKl9Sb/SnXoGc\nx1iXdQFflrA5OqvSlcvjeg+OVdAHUxXKwbeec8oPhRK2BmKaV3Yf1pRmFcw9/Ep9wMHzdiqrQhOA\nk7KWD106iuJrjthKoM6EwyiUaie62RJWjIzI/PntmbkgnRMWmosl7C8Oe5q47BOGubPuNvImR97x\nUs31mn+pr4x3NcezOpSF0zYCWi3q5thKOCSggHxIuXBHt7i1nsXgdzjak5C2K6cnsn1tdt/FLTWq\nUIsvZulaCj3gnLAlfC9uVHUgQFhWd8LwQ9kRMZyFa3Vin/tVNDI+21uv87Ll4NbxMl3J9glLPKo6\nD3n3TvLbO7cFGU23NPMzGU5TR36qnj2vzQEJmTtJp7Jqw9GSe6oFpgUf5Z6YpUHSF7OOGEimTf3E\nqt6i5Cyn0l3R2CkRczjqxzoEX2ozhEtElaOOLGHLtaknpRqOLjQW6sKQV8JO8cqUe0npa6aw5Zaj\nLQzb2dHSiH3JBMitieLq7wBlznX42eFUL3tYdpf+cd+ilDVWfOvZHNbRlGKnxuE9PVjCsnMhBDSE\nuTDLc+/GJIXRndftSBJrydKBK76vY+9FOQos9SLXwW6WsFP4kkyxRmTLW2/fvy1/s596r6tzW+Qv\nVv76X64AABaZSURBVPv6F3uaGhf82RYTuoy8yVAtVPUylWWIUnJtSH+XOinNH9tIqmMQoVI3StiY\nCVbrVbW4SqKDIYSA7chLn6VbtTpXte/Qftd6x24Jy99HYblZntsaJxFsdbRLG6vw5O1BOR1RVacL\ngPmoPMXws82PUwMeUE4BmFqDSs0JC8s4DyzxqoMTjgvHzA2HYnm0LUybcOpnlqAD7RM2xW1tZCXp\nYg1SIq+vvo4E+XkF5nCVc8KOT4PVGVs7IhsSKT5SGRH+47XGbc9f47V6UljIirSDPDp06acM86M0\n6k5BJakbJWxVpMJasWTWiEwnWaxCy+iFDenXmQxjeeY5YfkBAbJhL9OldGGWIl75o6n7hWpsnpcJ\nvHXEavjaKohucxZ8ONq9ZReaZsgwuXvrKT3mhkA+B+g0UuGlskrLiGYe26vcnLDkystwtM2vQ9SW\na9nJbFNPHGxhFy0se1zGXiD7yFnhvkPeOi1gdPiUoTwm5/vKfcJOIftUHLqtbdNsbZK10yA7n125\nGl6CeWGW4beDEjbliS1E+x3nfcKq+8I6L1k16kgJm60Tc2XSpfsCZY2stWF22nOaj0pSLExhGMKy\nbf6XyA64DEeX5oTtQzMKpVEIVjYcLabC8miJqYaTpkSSBC6X0X7fcu1JGnnaOOVZNmdcVOXc+1Vv\nIdPlvxXIO2qWBs01FGfUc8IeQ/a8MGtqi5LJvWJhlqJ8Fn1pxrSXi1VQC9729arjs+et4sMEph6j\nvVPmRc85KmHZI80ervLYSodOrF/jzf4+mmRO2LIwS7N3nPzEa5oTFg75r2wnFG2oMQ6f7XUhILMh\npw4jbOpGCRsRkJR1SaKWer9qv8K1dKsaWJlLuXJ1HO6EqrFWH6qn7uzJLCGVtO6o+65W3BW8xzro\nGLapP66sa+qj8mz7hM0hyr15kdPgRpP8mhLMQ0BOcfgRxMNeU5uVNHV7yquxLVZZJ7L7JjGth8ZI\nybsp97AOW4dA5Uw4ORJQpZ1XJaCK2eTfwafziXl+tbDFj8y7cU7Y4qRUVAK2Hw7rMyxiKi5KGNPc\naU7YUQk7tq6Vo26UsG04WpgTNGfMJIM74/9COFal7DRJrx6Ozt83LcyyWbiFUiy3wIwylH5rxbDs\n7nSp/5Ks9sKpT71goLOjXeSGdDha1Qgp0saC8+fU3PcJ65YPODhaS7CXj3wgxgJShiUc5mEdHiwH\n54bFur1LOM4Jm+awlQuzXETUzCVbGo+8ihSlcMSQv/YsNMdd6hCr81YI9QccjPLbrFiZ5rI4sJZj\n/wuz/J+YZV2Y5W4JmzshQbYoeV6YpeiWqEb/oBrNsCCE5cQsw33nDljlqCMlbM4E2zCjoiE0/s//\ntg9Hu8QslUXmTdhOZFKEYBubUszYWT2qwrNIal3E4Lz1ITil13A0myxufcZhudbgnGc565ywQ+9X\nNawKk5+AWPK0/BxQdW6kkbvfcRFHtrjF7sZZC2sOjauNgAWklIUOnTeFLncboQqGN+Wq2qLkGHIA\n8dz8eAkycN3VneqRSiO619GYoyWsemBuT70fqVk+daOETQkoaVylZ0dLetk2S9jBqirEZRelpIS9\nLMxymw8xFcFC4y3sX3cqfXrPRVZh6c2KcFZH2/fZ6maZnWRzCduLDKW0UeeZ+bAOt954Xn7bexkr\nqKdPGRqtpELYIW9R8vIpQ1GwZNRblMzD0Q5blIz9KouzwvvKOpfCcl36qUhHp9XRHhC2H87xCWvl\nNz9U5pnxzWyriBXuSh7MHX9NEkZJBA9TCR7RJfXE2g6YtijB/MnTIMPRyt1gDh02pyyRNRSOh3VY\nla1pjU0InesA1I0SdrImheW5ZnFjG462KnBTsOoKZo5b0kVTNdguFoN0TlgxrKV+pugwFHodQVaa\n2uSWi2O+5a14O3R7XH0KOCh7XV0uVJ+3c5bB/X3koyLW6/Kqvcq3zQJ0HcH1JofmcCUdYpLfcKdY\npAP6VVQJbydcSeqlyqnXPTIeO6EqN+EqB2unSKL6rR1ImRb1IZTX4WizlGoKz8zJ77yfW97Bsbbz\nDkGETP0oYSdFaunlFNAlFdQ6lGy3lizxSjJUN1jCZkNEMRzt0ruTfcpQNuRdkFOl6wuWsnWLUt4S\nlvtxwqXvYLjhfzhaqQwc1jOJqaUjThXaZFHpbsPR8vQUljDckK6gV1mkQVF9T9hqh2mqeL1awlOu\njelutZ6mwrAOR9vSQb390+YkWKNYyl+34ehidTX3ymyhOcVlCM6TO7M85mvl6miH0P2OFtjaNtg7\nacK2MMtuCQcdpXA+Mct4wz4PbfUndy2JE5avJRV+WtbYcJ9wEMxdOtsQkNdEtfU2hbWhd6jMEpy2\nK6gPh/Agq6kCCcNf8y9LwLJgYBsmdMDpUAlbKJJhSfVwtJsEpYFcSRTWwNTD0ZaK71zxJD0pW6Qe\nlLAH66fsOl+GhQUYFkCZbqrDcy4H8rit+9HN1ouzpPKzo50x1WXHTp6qDNjzXaUEjSuH7Z0PF2TD\n0V7nbcrANGBXjNkancmKMLmxiuKlS6nc4ujYo3dQ1hJrx/HELNUooTmIECqkd+pGCQtLRjnt/S11\nfoTpv8ytbp0rcKzM8jCK920b30vuLQGYL02HdRTmMOzNkqslXOw2mnuzwqFxccJtn3DpubHiOlQC\nh7CKKEY+Cw9jKHS65N5zxn3CBktJJpt6n7C/HrPF6Mrf0zTpnkvr6m/PqCxhy5dqXC3hQhpYGlyr\nO5PoNks4H5I9qy2tvqctSnkCfVrO0IF2XclvFFHlxmE42viFI8eapFIClttO25Tk8WsBLGHzOfkQ\ncLaE7R+IBuDviEdTiTcUWdtok6k9d7eETWK7fcBB2nE3r3ynJRwES6fKZgsrLCbpPVuBMIftLoox\nPocGW2Z9qOQqPjIuPrLH6+Rf1WHI3y5/YZZSWdl1jUw4p0tv8mgAoDunn633be5wyWRwnC7wooRl\n6Q7Ayarwi7cRhrxidYqr2IEUhrJmxLD4zRiqNQxA2PfLWi1h42EdCnlsKeT/zA4pqrUdwuzI6stj\nPjmmcOB7U6kaLFqlc1OOSByZy7psODqo1ehoCZtd+grXaAnLlLY0BUXxT4AYy8NVCeu6jltuuQXL\nly/HypUr0dPTY3q+bt06fOITn8Bll12GjRs3VkxQN6y9GLslLPNT6MkZG2LrtXWO2ByQdA+oKGlI\nU5OnsIRtlpVtgZBsYZYuaUiEVEar7JrVApFY1Z5wsy6Kw9Hqnmzxvu3am7Y296C1qS1KDvHkzHlt\nnhNTHNbh9J4+LeFS2M7K0DfKdLXIrrKEhdy9CqeFWfkyJasX1hseJoUtj311FUWptFiDN5V408ld\n6rzN103V6mhDcLanLuVfs6+gV21Rckomv7pQuq7EtubCsk/YtLyjUD+8x2kM3truWiKW+lKOtsmz\n056HNkvYMLJo7a9WCddPGf7qV7/C5OQkfvKTn2Dr1q247bbbcNdddwEAkskk7rvvPjzwwAOYmJjA\nihUr8P73vx+NjY0VF9yOvGdb+C3fSiTzas1la0PtHG8xPsljqxxeO5Lmx4UFNJKYi0pPFY7iScAC\nZ1ecwQO2v0sQj9pU8sgSJ4/qkAZArWxtbnxbwiq8DBF4xFPnxkkS45x7If0cFmaZhqOtcaoaZ3X8\n6mTMB17KN3+Dd+qOqYe64NUPnOe3veaB17jUPvz5sbch9o6hfRrPvsgyaMlVfEPDdsM+Ly0NTf5b\nooSlHRyVcq8CmnCJbc2aNTjzzDNx8cUXAwDOPfdcPPnkkwCAX//61/jNb36Dr3/96wCAa665Bldf\nfTXOPPNMZXjJ5EhYsuMnP/5vjL8+DkDDRKwLuVgrAKAp14e4GIOY2jCmCQ06GjCeOAYA0JAbREJP\nldpsGDJh6p7x2tTL8jlq25DV0ZTJf8M23dwAoaFYDLzaQqL4xxy/UMilvK/lG9hMbCay2kwAQHN2\nP2KQnwTk911V8Xr2b7OvDBXRuPlPj8GwXRpjDccBAJozKcQwCGhATmvCRLzbFF5rdi90LYHx+FQ5\n0AfRoKfc5bK8V2NWR+NUnk40xJFJuCuG0URexrg+isZcfzGscmkfywAAMokYJhriEheiNF8pYsjv\n9uzARGKGyVWDPoCEGIEm8u8S0wU0vQUjzbNtIbZk95nyxlj3WjJ7i+/VmMmhMZvPt1RLQ/6mVrJB\ns1oHMrFOAEBz7gBiIlsMs5ReaTTmBgyj4BrGEm+1y5TZK3n3UhpPxDqRi7UByJcDwFCmhAYIzVN+\nZLVWTMa7AAAxMQ5da87Hn92P9vEJaALIxjWMNSSK4Y3Hu6FrTdBEBs25g/aNrHo+zQvlOKGn0KgP\nFh+Xyr4G6CUhNa2UTi2Tw0Bs2NNIgc3INLQtxnpWKAtBmYzNQjbWbroX08ehx/JpZk6/XmjI2cIA\ngFiuEammowEATbkk4mLC5kagBWOJowAAM8f3oymTd1MsdwYysbZiuTsqvRdxPYNMIoZ0Yzsm4/kw\nEujDqm//RWj6qru7Q/nM1RJOpVJoby8lZDweRzabRSKRQCqVQkdHKfC2tjakUs6NWmdnKxIJWWPh\nn5HeIWQSJ9vuT0wlpIpMfBYy8VmhyOBKAkBzdaIKwnji2FqL4B9F8RlvaAfQLn+IUoNVIBObhUzM\nfzkYLSNPc7FWjE0prDAYVddtX2Rincig09FN28QA0k2dUiVYoKBIgKl08kihY2QlF2vD2JTydMIY\nrxvWchCUggIBgLHEsRhTFz0AgNAaMCaL21Kes7F2m/JSuS3G3zgDwAz5wwhRUMCANf3eovZkKEfW\njrWMoeZjPdfPvjZ5WdByregfGEJ390xvAZWBazVpb29HOp0uXuu6jkQiIX2WTqdNSlnGwMBoUFlt\n/NX1f4U//vEFHE4OAQBiifwQR2GYQ4MGTdOKhzAkElr+2EId0OKAbuh4FRarFqZtrdfxOJCTd9Ty\nbqFNDXXETIc+aDFAxBOI5QBdL/X0Y5rlY+9xQBjlScSBWAwa4hCZcWgaEItpyAktP39RkCuhQc8K\naJo53nhCQy5rN28TsQbEYwJZZBHXNExm8m5aOpqRGc9BFwIaclK/RdkTGqAJ6Bn7s3hcQ84w76pp\n+cQT2ZztHW3hTqWJNW0KNCUakEXWJltDYwNEQxOyY2nENFHMVy2W/6NpOoAEhJ6F0PP3tRhQyA4N\ngKbFoYtcMS+BGHSDsNb3Eg35OqBlSnna3NKI8bFJ+bvFgJbWJuRyOWQyOehaHCKTRcxQDmOxBKBl\n82XOx2iY3pBAzCCHlXiiEdByENkcGpqbMTE2jkRrC3LjYxACSCQSiOkJZHLjpWg1QMQaoMV1tMcb\nkZ4YQ1vniUgPDCE/NRiDDg1ADrGmRiAzOTV1YxxFiiPe2AiRG4PITd3X4miKNSCrZyFi2ZJ7YU/D\nxkQi7w6lOh3T4tCRQ0tzCybGx6HFDfk9VWeFrqEpoWEsY7Y4Y5Y6H4triIsm6LGJ4upr6yJzLQag\noRGYnCzKH4/lgBiQy+TDBIB4PAboGvSGBiAzYS+j7U3Ijk5A6EBLogGxBDCZmfq6WwxF9/FE/kX0\nrJiqPPnT/pobWzA+OZZPv0QME5lS5Wtp7cDE+EixrXIqh8UyYSjPsVgp32KNzYhls8ga26pYfEqW\nLPQcEGtuhJicNKWVNW2L8TQCsUwjssgAmkBraxPGxjMQer4N0+JTaS4kYRgajFhrA8R4RroJQMtv\niUC8uRl6NgORzUFvSOTrpqQexWIJIK5D5HRoGpCNlepPvCGfDieddAa6OmdGwxI+55xzsHHjRnz0\nox/F1q1bceqppxafnXnmmfiXf/kXTExMYHJyEjt37jQ9rwZLlnww1CHuI5Xu7g6mY5kwDcuHaVg+\nTMPphasS/vCHP4ynn34al19+OYQQ+OY3v4l7770Xc+fOxfnnn4+VK1dixYoVEELg+uuvR1NTUzXk\nJoQQQqY9rguzwibsHhp7feHAdCwfpmH5MA3Lh2kYDmGmo9NwdP0c1kEIIYRMM6iECSGEkBpBJUwI\nIYTUCCphQgghpEZQCRNCCCE1gkqYEEIIqRFUwoQQQkiNoBImhBBCagSVMCGEEFIjqn5iFiGEEELy\n0BImhBBCagSVMCGEEFIjqIQJIYSQGkElTAghhNQIKmFCCCGkRlAJE0IIITUiUWsBgqLrOm699Va8\n8soraGxsxOrVq3HCCSfUWqzIkslksGrVKuzbtw+Tk5P4whe+gJNPPhk33ngjNE3DKaecgr//+79H\nLBbDv/7rv+KJJ55AIpHAqlWrcOaZZ9Za/EjR19eHT3ziE/jBD36ARCLBNPTJ3XffjQ0bNiCTyeCK\nK67Ae9/7XqahDzKZDG688Ubs27cPsVgM3/jGN1gOfbJt2zb84z/+I+677z709PR4TjuV27IQ05Rf\n/vKX4oYbbhBCCPHCCy+Iz3/+8zWWKNqsX79erF69WgghRH9/v1i0aJG4+uqrxXPPPSeEEOKrX/2q\neOyxx8T27dvFypUrha7rYt++feITn/hELcWOHJOTk+Kv//qvxQUXXCBef/11pqFPnnvuOXH11VeL\nXC4nUqmUuP3225mGPnn88cfFddddJ4QQ4qmnnhLXXnst09AH99xzj7jkkkvEpz71KSGE8JV2Mrfl\nMm2Ho59//nmce+65AICzzjoL27dvr7FE0eaiiy7CF7/4xeJ1PB7Hjh078N73vhcA8MEPfhDPPPMM\nnn/+eXzgAx+Apmk49thjkcvl0N/fXyuxI8fatWtx+eWX4+ijjwYApqFPnnrqKZx66qm45ppr8PnP\nfx7nnXce09AnJ510EnK5HHRdRyqVQiKRYBr6YO7cubjjjjuK137STua2XKatEk6lUmhvby9ex+Nx\nZLPZGkoUbdra2tDe3o5UKoXrrrsOX/rSlyCEgKZpxecjIyO2dC3cJ8BPf/pTdHV1FTt/AJiGPhkY\nGMD27dvxne98B1/72tfw5S9/mWnok9bWVuzbtw8f+chH8NWvfhUrV65kGvrgwgsvRCJRmon1k3Yy\nt+UybeeE29vbkU6ni9e6rpsSltjp7e3FNddcgxUrVuDSSy/Ft771reKzdDqNGTNm2NI1nU6jo6Oj\nFuJGjgceeACapuHZZ5/FSy+9hBtuuMFkWTAN3Zk1axbmzZuHxsZGzJs3D01NTThw4EDxOdPQnf/4\nj//ABz7wAfzt3/4tent78ZnPfAaZTKb4nGnoD+OcrlvaydyWHX/ZIdSIc845B5s2bQIAbN26Faee\nemqNJYo2hw8fxmc/+1l85StfwSc/+UkAwDve8Q5s3rwZALBp0ya8+93vxjnnnIOnnnoKuq5j//79\n0HUdXV1dtRQ9Mtx///344Q9/iPvuuw9vf/vbsXbtWnzwgx9kGvpgwYIFePLJJyGEwMGDBzE2NoaF\nCxcyDX0wY8aMojKdOXMmstks63IZ+Ek7mdtymbYfcCisjn711VchhMA3v/lNzJ8/v9ZiRZbVq1fj\n0Ucfxbx584r3/s//+T9YvXo1MpkM5s2bh9WrVyMej+OOO+7Apk2boOs6brrpplAKWr2xcuVK3Hrr\nrYjFYvjqV7/KNPTBP/zDP2Dz5s0QQuD666/HcccdxzT0QTqdxqpVq5BMJpHJZHDllVfijDPOYBr6\nYO/evfibv/kbrFu3Drt27fKcdiq35TBtlTAhhBAy3Zm2w9GEEELIdIdKmBBCCKkRVMKEEEJIjaAS\nJoQQQmoElTAhhBBSI6iECSGEkBpBJUwIIYTUCCphQgghpEb8/6FianNu1dIKAAAAAElFTkSuQmCC\n", + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAeEAAAFJCAYAAACsBZWNAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJztvXucHFWZ//+p7p77TJIZMgQRAiSAKIhA1N2oGAgKKLDk\na5BA3OCu+HqhCz+EXRXIIguar5B1XVdYQXBX2UX8agBXucgqmmC4RoIkmnAPyZDLJOnMvXtu3V3n\n90dPd9flnLp1dXdN5/P+I5muOnXOU+f2nOc5l9KEEAKEEEIIqTqxWgtACCGEHKxQCRNCCCE1gkqY\nEEIIqRFUwoQQQkiNoBImhBBCagSVMCGEEFIjEtVOMJkcCTW+zs5WDAyMhhrnwQjzsXyYh+XDPCwf\n5mE4hJmP3d0dynvT3hJOJOK1FqEuYD6WD/OwfJiH5cM8DIdq5eO0V8KEEELIdIVKmBBCCKkRVMKE\nEEJIjaASJoQQQmoElTAhhBBSI6iECSGEkBpBJUwIIYTUCCphQgghpEZ4UsJ9fX1YtGgRtm3bZrq+\ndu1aLF26FMuWLcOaNWsqIiAhhBBSr7geW5nJZHDTTTehubnZdv3WW2/Fgw8+iJaWFlx66aVYvHgx\nZs+eXTFhCSGEkHrC1RJevXo1LrnkEhx66KGm69u2bcPcuXMxc+ZMNDY2YsGCBXjhhRcqJighpH7Q\nhcBzW/diZHSy1qIQUlMcLeGf//zn6Orqwumnn4577rnHdC+VSqGjo3QodVtbG1KplGuCnZ2toZ/J\n6XQ4NvEO87F8mIfeePLFnfjBIy/jhKM68a2rP2q6xzwsH+ZhOFQjHx2V8EMPPQRN0/Dcc8/hlVde\nwXXXXYe77roL3d3daG9vRzqdLoZNp9Mmpawi7K97dHd3hP5lpoMR5mP5MA+9s23nAADg1Z4BU54x\nD8uHeRgOYeajkzJ3VML3339/8e8VK1bg5ptvRnd3NwBg/vz56OnpweDgIFpbW7Fx40ZcfvnloQhM\nCCGEHAz4/p7wI488gtHRUSxbtgzXX389Lr/8cgghsHTpUsyZM6cSMhJCCCF1iWclfN999wHIW8AF\nFi9ejMWLF4cvFSGEEHIQwMM6CCGEkBpBJUwIIYTUCCphQgghpEZQCRNCCCE1gkqYEEIIqRFUwoQQ\nQkiNoBImhBBCagSVMCGEEFIjqIQJIYSQGkElTAghhNQIKmFCCCGkRlAJE0IIITWCSpgQQgipEVTC\nhBBCSI2gEiaEEEJqBJUwIYQQUiOohAkhhJAaQSVMCCGE1AgqYUIIIaRGUAkTQgghNYJKmBBCCKkR\nVMKEEEJIjaASJoQQQmoElTAhhBBSI6iECSGEkBpBJUwIIYTUiIRbgFwuhxtvvBHbt2+Hpmm45ZZb\ncPzxxxfv33vvvXjggQfQ1dUFALjlllswb968yklMCCGE1AmuSnjdunUAgJ/+9KfYsGEDvvOd7+Cu\nu+4q3t+yZQtWr16Nk046qXJSEkIIIXWIqxL+2Mc+hjPOOAMAsGfPHsyYMcN0f+vWrbjnnnuQTCZx\nxhln4IorrqiIoIQQQki94aqEASCRSOC6667DE088gdtvv91077zzzsPy5cvR3t6Oq666CuvWrcOZ\nZ56pjKuzsxWJRLw8qS10d3eEGt/BCvOxfJiH3mhrayr+bc0z5mH5MA/DoRr5qAkhhNfAyWQSF198\nMR577DG0trZCCIFUKoWOjryg999/PwYHB3HllVc6xDFSvtQGurs7Qo/zYIT5WD7MQ+88/Mx2/OKp\n7QCAH16/uHideVg+zMNwCDMfnZS56+roX/ziF7j77rsBAC0tLdA0DbFY/rFUKoXzzz8f6XQaQghs\n2LCBc8OEEEKIR1zd0WeffTZuuOEGfOYzn0E2m8XKlSvxxBNPYHR0FMuWLcO1116Lyy67DI2NjVi4\ncCEWLVpUDbkJIYSQaY+rEm5tbcV3v/td5f0lS5ZgyZIloQpFCCGEHAzwsA5CCCGkRlAJE0IIITWC\nSpgQQgipEVTChBBCSI2gEiaEEEJqBJUwIYQQUiOohAkhhJAaQSVMCCGE1AgqYUIIIaRGUAkTQggh\nNYJKmBBSfTx/u42Q+oZKmBBCCKkRVMKEkOqj1VoAQqIBlTAhhBBSI6iECSGEkBpBJUwIIYTUCCph\nQgghpEZQCRNCCCE1gkqYEEIIqRFUwoQQQkiNoBImhBBCagSVMCGEEFIjqIQJIYSQGkElTAipPvyA\nAyEAqIQJIYSQmkElTAipPvyAAyEAPCjhXC6HG264AZdccgkuvfRSvP7666b7a9euxdKlS7Fs2TKs\nWbOmYoISQggh9YarEl63bh0A4Kc//SmuueYafOc73yney2QyuPXWW/HDH/4Q9913H372s5/hwIED\nlZOWEEIIqSNclfDHPvYxfOMb3wAA7NmzBzNmzCje27ZtG+bOnYuZM2eisbERCxYswAsvvFA5aQkh\nvhnZ+AImdr4NIQSe2bMB/eMDGHr6KWSSyVqLRspkeHAMr2zuhRBc6eaHt15LIrl3pNZiAAASngIl\nErjuuuvwxBNP4Pbbby9eT6VS6OjoKP5ua2tDKpVyjKuzsxWJRDyguHK6uzvcAxFXmI/lE7U8zE1M\n4PXvfw8A0HH3Kvzk1Ydw7HAjznt0F2JNTVi45ic1kautran4tzXPopaHUeY/v/MUJidyOHr+bMw9\npqt4nXmoRugCd/3PkwCAm759gWPYauSjJyUMAKtXr8aXv/xlXHzxxXjsscfQ2tqK9vZ2pNPpYph0\nOm1SyjIGBkaDSyuhu7sDyWQ0RjTTGeZj+UQxD/XxseLfb+/fBwDIDedl1CcmaiZvOj1R/NsoQxTz\nMMpMTuQAAHv3DKGlvQEA89ANXS95DZzyKcx8dFLmru7oX/ziF7j77rsBAC0tLdA0DbFY/rH58+ej\np6cHg4ODmJycxMaNG3HqqaeGIjQhhBBS77hawmeffTZuuOEGfOYzn0E2m8XKlSvxxBNPYHR0FMuW\nLcP111+Pyy+/HEIILF26FHPmzKmG3IQQQsi0x1UJt7a24rvf/a7y/uLFi7F48eJQhSKEEEIqQ7QW\nsfGwDkIIIaRGUAkTQgg5aIjabi4qYUJI9YlYR0hIraASJoQQQmoElTAhpPrwAw6EAKASJoQQcjAR\nsakQKmFCCCGkRlAJE0IIOWgQETOFqYQJIYSQGkElTEgdE7U9kYTUnIi1CSphQkj1iVhHOP1hhk5X\nqIQJqWdoChNiImotgkqYEFJ9uE+YEABUwoTUOVEb9xNSYyLWJKiECSGEkBpBJUxIPROxUX+RqMpF\nDgKiVfmohAkhhJAaQSVMSD0T1dXRXJhFakTUmgSVMCGEEFIjqIQJIYSQGkElTEg9EzXfGyE1JmpN\ngkqYEEIIqRFUwoTUMVH7bBshtSdabYJKmBBSfaLVDxJSM6iECalnqOwIiTRUwoSQ6sN9woQAABJO\nNzOZDFauXIndu3djcnISX/ziF3HWWWcV799777144IEH0NXVBQC45ZZbMG/evMpKTAjxTtSWghJS\nY6LWJByV8MMPP4xZs2bhW9/6FgYHB7FkyRKTEt6yZQtWr16Nk046qeKCEkIIIfWGoxI+99xzcc45\n5wAAhBCIx+Om+1u3bsU999yDZDKJM844A1dccUXlJCWEBCBiw35CiAlHJdzW1gYASKVSuPrqq3HN\nNdeY7p933nlYvnw52tvbcdVVV2HdunU488wzHRPs7GxFIhF3DOOX7u6OUOM7WGE+lk/U8nAykcVb\nU3/PmNFiu18redvampQyRC0PpwMzZrSY8o15qGY0PVn82y2fqpGPjkoYAHp7e3HllVdi+fLluOCC\nC4rXhRD47Gc/i46OvJCLFi3Cyy+/7KqEBwZGyxTZTHd3B5LJkVDjPBhhPpZPFPMwO5gq/j08PGa7\nXyt50+kJqQxRzMPpwPDweDHfmIfOjI2WlLBTPoWZj07K3HF19IEDB/C5z30OX/nKV3DRRReZ7qVS\nKZx//vlIp9MQQmDDhg2cGyaEEEJ84GgJf//738fw8DDuvPNO3HnnnQCAT3/60xgbG8OyZctw7bXX\n4rLLLkNjYyMWLlyIRYsWVUVoQohXOCdMSJRxVMI33ngjbrzxRuX9JUuWYMmSJaELRQghhBwM8LAO\nQuqYqO2JJKTWRK1NUAkTQgghNYJKmJB6JmrD/gIRFYuQakMlTAghhNQIKmFC6pqImpz8gAOpFRHz\nDlEJE0IIITWCSpiQeiZag35Cak7UmgSVMCGEEFIjqIQJqWuiNu4npMZErElQCRNCCCE1gkqYkHom\nYitBi0RULFL/RK3qUQkTQgghNYJKmJB6JmrD/gLcJ0xqRcS8Q1TChBBCSI2gEiakjhGRNYUJIQCV\nMCGkFnBsQAgAKmFC6hsqO0JMRGxKmEqYEFIDuDArZCKmWYhnqIQJqWeiNuwnhJigEiaEEEJqBJUw\nIXUNLWFCjIiIeYeohAkhhJAaQSVMSD0TrUE/IcQClTAhpPpwcEAIACphQuocajtCogyVMCGk+nCf\nMCEAgITTzUwmg5UrV2L37t2YnJzEF7/4RZx11lnF+2vXrsX3vvc9JBIJLF26FBdffHHFBSaE+CBi\nK0EJqTVRaxKOSvjhhx/GrFmz8K1vfQuDg4NYsmRJUQlnMhnceuutePDBB9HS0oJLL70UixcvxuzZ\ns6siOCGEEDLdcXRHn3vuufjSl74EIL+3Kh6PF+9t27YNc+fOxcyZM9HY2IgFCxbghRdeqKy0JBIc\n2DeClzftqXq6uhD43w1vIzk4Zrs3uO53mNi9q+w0Xul/HZuSW8qOxy9j2TH8pmcdRjOjocbrZdT/\np+RWvNz3mvReZrwPw/ufR/+BFB781//Gy+vXlyVPRs/iiZ4nMaGnfT+7O9WL9bueKyv9eiUM625g\nfBDP/fz7GH17R/mRhYg+MYH+xx9Ddng4pBijZQo7WsJtbW0AgFQqhauvvhrXXHNN8V4qlUJHR4cp\nbCqVck2ws7MViUTcNZwfurs73AMRV7zm4123PQkAOOX9R2JmZ2sFJTLzzOY9WLPuTTyxcSf+++Zz\ni9dH334br99/HwDgw798qKw0rlz7HwCANcvuCvR80Lp4zwsP47dvPY2BbD+u+su/CRSHjNHxNvQU\n/s7otvvd3R24e+1/AZC/8x+f+L8QIocXNx5AcnIutj6xA4uWnhdYnkdf+y1+se1X6IzPAXBqUQar\nTDKuXPtVAMDC+Sfj8BmHBZahHhnGALq75xV/B6mHd/y/23DOr17Frl89X3Y7CpOe+x/FgYceRPat\nN3DizV8rO764VrI93fKpGrrFUQkDQG9vL6688kosX74cF1xwQfF6e3s70unSaDadTpuUsoqBgXBH\n+t3dHUgmR0KN82AkSD7u2zeMyWyuQhLZ2b13CAAwMDJhknVsT1/x77DqQpB4yqmLuwb25f8f3Btq\nfZ7oLw2MB4fsHgRjWrJ0hciX79ho/v/JeEtZ8u3q2w8AGM71S9P1koe9yQE0TLQFlqEe6R8eKeZb\n0HrYN7iv+HeU+tThnb0AgNTOXaHINdhf0kFO8YWpW5yUuaM7+sCBA/jc5z6Hr3zlK7joootM9+bP\nn4+enh4MDg5icnISGzduxKmnnhqKwGSaEC2vDiGETDscLeHvf//7GB4exp133ok777wTAPDpT38a\nY2NjWLZsGa6//npcfvnlEEJg6dKlmDNnTlWEJtGAOngaEFIhsawJqQyOSvjGG2/EjTfeqLy/ePFi\nLF68OHShyDSBPTMhkYBNcfrCwzoIqWvC7p7Z3ZPpTdT2CVMJk8AIdsgHETziKsqwLU5fqIRJcNju\no0/Iw36NZU6mPdGqxFTChBAfRKsDI1NEzcdKPEMlTEg9w76ZEDMRaxNUwiQwHHwfPLCsow4LaLpC\nJUzKoLoNX5laHWiIyi2sscfLJVb1x/RvAWrCbhtRyysqYRKYOtB9xDPa1L8s9EjCYpm2UAmTaYPS\ngtOmv22nVcg+FbKREjtsMo0IvW1ErP5TCRNCyDQnYnqF+IBKmJCDjLLsCs5BRBIe1uGdqOUVlTAJ\njNTVSeqSQklPf8d/ncKmOG2hEiaknpEOlNhj1xtRs+4iTcSyikqYEOKDiPVgJA+LZdpCJUwCQ2/0\nNEBSRoHOfxZ0RBNSCaiECSFkmsPx8PSFSpgEhguzpgPhlBEXZpF6IWrdFpUwIcQHEevBCAAuzJrO\nUAkTUs9EbdhPKgOL2QfRyiwqYRKcKtdlfsAhQLyGaAtpBFqYRSJNHTQBJaF/wCFieUUlTAITsbpM\nKsrUbDALPaKwYKYrVMIkOFUeUvIDDkEIt4z4FSVSbSrXNqIBlTAJDLtjQqIBF2ZNX6iESXA4Jxwa\nFetEDXlTzpzw9M/hOqeOC6jeBxhUwiQw9bxPuJ7fLRDC9gchJASohElgqq2nqjknXO3Rd73Pe5HK\nUs9Do7DbRtQG2J6U8ObNm7FixQrb9XvvvRfnnXceVqxYgRUrVuCtt94KXUASZaJVmQk5aGFTnLYk\n3AL84Ac/wMMPP4yWlhbbvS1btmD16tU46aSTKiIciTYRG1CGihCiPs5oNM4JT/0d7LW0Mp4llaeO\nG2Od42oJz507F3fccYf03tatW3HPPffg0ksvxd133x26cCTa1LMSJmQ6wbY4fXG1hM855xzs2rVL\neu+8887D8uXL0d7ejquuugrr1q3DmWee6RhfZ2crEol4MGkVdHd3hBrfwYrffJw5s6Wqed/e0Vz8\n25jucF8rdkqul8Ps7g4kYv7radD0GxrzaTUk4qHm6fCBUt60tjbm/zB02LNntxf/lqX7duGPqXl3\noQjnldZdeRmMFrU1Prf4Z3W2oruLbd5Ic3PClG9BysipTGrJQHMDACAei4Ui1+RYtvi3W3zVyAdX\nJaxCCIHPfvaz6OjIC7lo0SK8/PLLrkp4YGA0aJJSurs7kEyOhBrnwUiQfBwcHK1q3qdGxot/G9Md\nGxyVXi+HZHIYiZi/5lFOXcxM5vL/Z3Oh5umYob2Njk7a7u9PDhf/dkrXuJilHPlGx/IyGA03Y3xe\n8nBgII1kjm3eyPh4pphvgeuhoVCi1KdOjGcAADldD0Uuow5yii9M3eKkzAOvjk6lUjj//PORTqch\nhMCGDRs4N3ywUccusPp5tfp5kwL1vm80CMyR6YtvS/iRRx7B6Ogoli1bhmuvvRaXXXYZGhsbsXDh\nQixatKgSMpKIUu2l/jysIwpwaVYUCaP2RLVkp0/bCIYnJXzEEUdgzZo1AIALLrigeH3JkiVYsmRJ\nZSQjkaeum0YdKHbAPFAqnpgF+4pp94ii00XXSdGESxh5cpDk67TcJ0yIlKh8wKECYlS7mU6XwzpE\nJOSMVidKKst0aRtBoRImgYnOgLISgkTm5cJH8o1hz0SgP6zjkimD8nMlAkV7UEIlTAJTbSWsSi5q\n7qUgRH3eK9rSkTpoAkqi3jbKhUqYlEH9No66eTPpnLDhdpXFCYfpKXVlCSFPDpZsjdh7UgmTwETm\nAw4VoNrWdc3mvXy/Z+2dlvVs9dWS2pesHM4JExJ1KtIr10lPb7KEJbf9RleWMOFQ7+7JQISSJQdH\nvkbtLamESWC4Tzg8oq9Y6tsame7oka8/wYl+2ygPKmESmDrQfUrq8tUKX1EyvVxdvikJgHawVIWI\ndVxUwiQ4UdknXBHqb064HHd0KVztLeJ6WA1PvMM5YUIURKYvrIAgkXm3cjG+iOSl/Cq0aGRLNKSI\nFMwSz0Qtq6iESWCiVpnDpf7erqw3ilB2REiUyMA8mb5QCZPgRGRhVj24J6u5+MTs3PN7YlZ9uwYP\nZqI6Jxx624jYe1IJk8DUge5TUo+v5jYn7DyYiZLyrcfSKZN6box1DpUwCUy1tw6oP+BQgTnhOlmY\nJWRzworV0U7vHKUunvqmMkRpmGWEC7MIUVHPnWEdvlu9vFK97xsNQj1MyRysUAmTwETlAw71YBpV\nTLGY8ka2OtpfNNH4lCGxUs8HZoXdNqI2YKESJmUQrcocJvVobcnfyOCOnjZzwsRGCNWVJVwbqIRJ\nYKLzAYdwBKnlCLkq816ikJbt0rSiHgdI5VLPecI5YUJURKXdhySHsSOrx05N9k7+3zkCHWL9FU3Z\nhJElUd2iVO9QCZPAROUDDlGb4wlCdeaEi4mFEg2JEHVcQOHPCYcaXdlQCZPARKwuh0rUGmoouGwU\ndn7lvAUsImEI12PhlEsYecJ8rQVUwiQw1f/wvYpKzAnXxz5h2T5g85yw89nSdmqvhakq7NTz6mjO\nCROiIiqNtgJyHIzWltMbRys3oiVNFAhjQFzfqk5OFKayqIRJYLhPODwqNycs+zNIWgdjF02iQPht\nI1r9BZUwCUy9WYv19j52nFdHO3ZOEcqaKFgvUYNZMn2hEibBqbOG7/1jBtMH43sU/jRuRRFSS1kS\nT/F/WsRRJIwB5MGyRSlqTduTEt68eTNWrFhhu7527VosXboUy5Ytw5o1a0IXjkSbyCzMCkuOej+s\nQ8r0OzGr/j0WAajjLKlk24iCQk64BfjBD36Ahx9+GC0tLabrmUwGt956Kx588EG0tLTg0ksvxeLF\nizF79uyKCUuInOn/FaXK4ecwjnp554OP+qmvBx+uSnju3Lm444478NWvftV0fdu2bZg7dy5mzpwJ\nAFiwYAFeeOEFfOITn6iMpC5kszm88NQOzJ7TjuPeMwcA8HLfaxiaGMbCwz8AIQR+/r+vobulAYcf\nOxvb9w7j4+8/svj8pjcOYGwii4UnHZZ/dkc/koNjWHTKOwEAjz/fg3cf3YmjD5thS3tr32sYnhhG\nZ/Ms9I3148Pv/AsAwN5dQ9i/dwRte57CntkNOP1DFwEAekcn8NpgGove0QlN0yCEwB+fextHzT8E\ns+e0AwD6f/Uohl/Zgr0zgHd/9K+wMTUDO3qH8bH4Lsw+6p1oe8+JGJvI4vENb+PY43T0ZffizCM/\nks+LnI5Hn92BD510GA7tbC3KKYTAjzc/imRvBkcPz0eqKY4LzjoOnR1N6Pnlw3ho9zCO7U5jj1iA\nD7/vSBzW1QorB4bGDPGZ7+lC4NFnd2DBuw7FO2e3AQDG3ngD4z07MPmev8DeXUN43wePhAwhBH7f\nO4Dulkb0jk7gzHd0IR4rjYB39R/A73b/Doi9A9BL1TY9msb/PvVntHedgqP7N+MP67fjtA/NRSIR\nx/Y3DmA4+TpGx1qxZzKFdxwxEx9673ux+Q87sa93GGdfeCIAYPvQ29g+tAMfeefCYry/7lmHS47/\nP8Xyeey5Hrzv2Nk48tCp8vnfXwGahq5zvNX3x557Bftf24nBkSZ0Hn0Izs6+ga4PfQh/fn0Ux504\np9iJ9g9PYOf+VDEdNyb39mLkDxvQdf5fQYvJHVubj2tBx2gcyaEWnPZqFhkN2N55MrrTb0PbPYKu\nvUeh/7Ae9Azvwru6js3HO7oXgy+sw+Yth+DVQ+LoMJT1c5u3YqJrEKc1xjCWbsNrD7+AxHtPx1Bj\nA+YdNQvvmtuJXHYMQ3ufxUuPb8RA+3vw/g8sxNHHzsZEejcAIIcMEMvhUD2OPW8P4vC5s0wyv7Vn\nN57b+CqOOeYENDU2YsG7uov3dmx9C1ve3oKjzjwKI691onXuTMxubMTe/lGMjmeQju/H673AX59x\nEo6ema/Duq7jxWd6cNyJczCrqxVCCPzyqafwpngTR4oP4X2zZ6GpMY5jju82yZH602b0P/YIZi/5\nFFpOeDeG9z2NZ19qQXowgUVnzsfOt/qxf38KvUNjSLX04N0738TIyefjQwuOwZyp9jO0dwC//n9/\nwMIPHILJ9x6FH7/wB+SGBtHZ1YpZsUMxU3sHPvEXczE6PI4NG3ZiU38af/WRYzB336uItbVhZ1cL\nfvnyk/i7hRdifHAr/uulHC784Htx/GHvKMo5mhz2VF8AYFcyhZdeT+L8Dx0NTdPwzJ970d7SAA1A\nJg5sPLEN3eOD6GzOl8nWpx9F//4kBuecgY8f04jfP/AU2k75AD6yaD4AYCKTw2PP9eCMUw5H14zm\nYjrJfSN46nevYeJdvTiq4Wh0ZGbhFy+9jnfP0nDS/ENwwqnvw+YXdmLO4TPQ/Y4OPPLMDiw86TDM\n6cyXz/C+p9Ay8wTkcjr2th+DhngL5kneZ+tLezCrqwWbHvkftHe2oePsc9G2/nc46gML0Dz3KFPY\nnJ7Dht4XATQAAH71yAYsyO3BYUuWIJPR8eiaP6GhIY5Pfvq9nvOzXFyV8DnnnINdu3bZrqdSKXR0\ndBR/t7W1IZVKuSbY2dmKRCLuU0xnurs7sOPNA9i0YScA4EOL8h3J99b+JwDgr963GK9s78f+zXux\nH8B/Pd8DAeC80+djZnsTAOD229bmw555HADgX6Z+X/TxE9DTO4wHntwGAHjk2xfa0r9zKp0CS075\nGADgrtueRHMmhQ/3/BpzAIizLsCh7bOx8ldvAADed+QhOL6rHTu39+MP67fjD+u346ZvX4Dx/fvx\n+s8fBAB0Adiz4TXcf2x+OuD0N3+C3QA+/MuH8J8Pb8Gjz+5AbMsBNJ2wEeeeeDpmNLXj18/34OFn\nduDZLXvxo5vOKcq1L5XE8/1PoSnXgZbtXQCAzfMOwZIPvAMPPPNHrMudinXbOgDsxpN/2o+frvqk\n7V3/6Ucv4LCpv1tbG9HdXaoDG1/Zh188tR0PP70dv/yXfD498/n/CwD43bH5Sv/+hUdjlkS5v9Gf\nwm929xV/v7OrHR+dW/KqfPXX30G6rRcNR6SQefvdxXR//KO1GEodgWTX0WjKpvHqsz3oOqQNC8+Y\nj7tue3Lq6UkAwP7NfbhwcQeeXZsvy5mXtaCxMYEr1/57Pm+PW1BM7+ndz+PcE07H8bPnYcu2A/j5\n+rfw8/Vv4ZFvX4jM8AhefzA//TJ/ySeRaC8pTGN+GNn2Qh8aRhuRik/iz1v3Q+/bjne/tAObWk/F\nlj/uQePH8m2ib3gc//TDP0jrmYzn/78vIjc2htknzMfsD3/Idj/X3oAnP5CXaWbHkTi0/S/Q0rsV\nbx1yGt465DRgzRs4HCdicPZu3L7pHqxZdhcA4MXffB3j972FXcf+DdpHAIHcVIwaNj2exK6/fBzz\nZuYHWtkQVWlZAAAgAElEQVS3G7B1fA/GIfDAMwKPfPtC7NjyGN7YuB5HbtiHI/EKHu/pwFe/sRCT\n6Z1F2RoO3YGj9h6HX/5kE2769gWmPPz3f92K+GQjfvn280imOk35seu5dgDt2P/E62jcdyTExj3Y\nKHTbu38n9yp+csUiAMCfNu7Exmd68PLmXnz5lnOw/c0DeCL7KADg5T8chdfRi9nQTHIAwOu3fyef\n5rf/GSf/aDV29q7Djs0fBQA8eO+LxXD7IHD6yP/iyH0ZPN8bw22vLcSPb8kP0H5253r0TzTh108m\nsb0xjsHY74FOoE8AyAFjfzgXTU0JvL3uLQDAfuj41zWbcf2b/w0A+N7SoyCaxvCf6+/DkdlZeOWV\n+Xhj+0v4n1XHF9M/5k87THVPVQ8B4HNTfduCE9+B9x3Xjf98LP/7PYcDL72rFX84qQ3JV+7D6rNX\n5svp3gcxB8CPjj0Kx/33T/DWMX8NPLcTS5a+D5qmYc1vX8ejz+7AK28P4N+uPaOYTqH96btaMaj3\nAejDdmiYfSCGdW8O4JS/aMCzv8u3xQ9f8j48/MwOrP9TL+67+VwM972Bnb1PYqj3SaT6Z2DrYfly\n/CvLewldYP2vX5/6NQ9IAWMbXsL5jz+Mtx9/GB/+5UOm8E9ufw4v7P0jjkbeUDr6kR9gBDm885T3\nYPP+NuzbnR/M7Nw2gMMOm+mYj2HhqoRVtLe3I51OF3+n02mTUlYxMDAaNEkp3d0dSCZH0N9fkiWZ\nHDGFSSZHsHe/faS4PzmCybFJW1jr717Ds9b7MoxhYiJX/HvfgUFoY02l9PtS6MwJ7N8/Ynp2ct+g\nKT7ZYphkcgR7pp4Tky1T7zOEiSaBvVPpHxgaN8mSTOffI5YrDYKGhsdxYN8gxhJNQElUpMcy0nft\nPZDCYVPypFMTpjCFfNKFOp/27x9BJpezXd83mDaHG0wj2VLKqzExFV/DRPH98+84gaZY/v0zsXz4\nA8mUMn3j9QPJETQ0lprA3gNDprB7+wbQKUaw11I+2aFS+ST3DyExljcTC3VRRmIsCwDI5fLW6mi8\nGaPpYaAVGB/LIDtpViJe6hkA5MbynomBvX0QkmcGh83tbbStA016wrYaRJs6CsuYrnnddH5mrnBi\nVsJQJzPxfL4350MgmRxBargPuUlzOfcdMNfrWEOm+Hch3UIexicbAQANU6uFZPmRmGyakt12CwCg\nZ/Tic8n9eQNhNDWJZHIEyX3m+LIWOWT09w0q72UBzEjl37ctN4ahqXQAYHQ8X7a5WAOEJvdw9CZL\nBozVnyGa8mU8KgRGJ/P1NTvWbJI1LoQtD93Yt38EyVnNpmtjLfnU944ckMYRy2WLfyeTI9A0Db1T\n7aM3mZY/o8uNrv2GfrXQxgZH8n3K6GAprycmskBTKU0jum4ffDVkSn26NXxvf5+pYieQf35wXx/6\nDpRuHCjUG4/t0A0nZR54dfT8+fPR09ODwcFBTE5OYuPGjTj11FODRhdZgiw+Cn3BUkjrEup7t51/\nAstfu4O1/BOFlSeRZ/rnUS3eIApVK4gM5oVetV9w6NsSfuSRRzA6Ooply5bh+uuvx+WXXw4hBJYu\nXYo5c+ZUQkZP+FF8+fG6x3iDyFK0G8IhrG0h9f5xbL/oulV+b+9jem+JGzQIUVxYI693te+0QsdP\nhxCE6BWtFL8lm28H/p4KVnvC3b8eta1YnpTwEUccUdyCdMEFpTmTxYsXY/HixZWRzCcV0wcRK7By\nCF1pTve8sShhz4rQ6+baCCAXz7/Q9X5+7/QnhPKJeF2WUWizQujQtOl57MX0lFpGhSqQHgF3dGQt\n4enYag0ENIRNSjissq6UkitPvmi57SrH9K7HYRHMEq4GDp9uqQNLuG6UcJQUQvjKLqJEVjCPBO5E\nFEdORZEyvicstMoq3sio9cgIUh5Vnx6KQtUXtj/8PV7hOu6F+lHCTtMGlpt+sj0KC7PCqihBrHon\noq5/3LDOCXsePOnhK+FKDSK9x+utjoV6bGUUPk4MIBrapHzKr4r+Igi2KCoI6oQKfa3mw7ylJVwp\nHOfuFTNjHgojSEWrVRkHOREpX4kDjiKnuRYOKr7njx5EgPKUe1SUJHFFK38g5185Rafu+3n3qNXq\nulHCTgqhHGURTAlHdE5Y8jJCTH+LNijW/AhkCdsmlqOF13eSdcDRfrMQiZppFJCqe6MjkG1FS9iP\nMBGQ20jdKGE/FFSaF+UcRKGGbyF6dBW6pCt7F5F/MIBM9WAJW5Sw59cxLMwKqUVXbmGWLC25BF6u\nhTmHFjWLpHJUvp0IaMEMhiobKMHK3MtTtIRrjlNl0qvujpa7Kx1sdcdEPS/aLS7XV21MUbijg/tl\nnX97eihoYuXjuk9YlaQxv3xawsUBoEWZ1X5OWPKsg8I13lJ5atxTrnB3GGBQ5RZO3VRKuRDKWynb\nsNNKYZ9zuvDRJwGw74kXppDl1GCFeTD1p5OXc+qPcr0ZonJt0I06UsL+b3qyhKfRwiy3ZKULZfOm\nsG+ZvKQXdewLs7whKrAwq1J4rYtaLRZJRSXrKv3qgSzGIP2O/3SMybjpMatM5Vb9MPrJUhxB3dG1\nt4vrRgk7YR3hFK0RWdiAVqg1RX84VwS/lrCmUNpyd7QIPK9pi6+i9bnyjcXWKag/YKz420Mahair\npLzDX5iVv2YUX6Uw3Eqs4iUaegJOLt9SLtiChCiHk3L2W9YanHZMyIS2KmFhCunlNY1hzHXIIaRD\nxCUdXKY7WqvdgTR1o4QdF2b5eMZ6KUhfaXR/myudt8jsAwGvlcMtfvn7Bu6oA81BBUuqEgTeomQ8\nrCOkhVmV6wC8yieZ/w1XEE9p1gY/b1odmatmCRvTdLOENasSLi897897yHNfW5RK8UWhO6ojJex0\nT36+r9QSVoz2/AnjUTAvz/vAbR+w9L5AcEu4TPdXzbENuDyeA2168Si9kJ2ypPM4Jzzt8fEu1Srt\nINlb6YVZ4S8i92iUON0L4o5WrhHinHB5+LCES6uj3aMJc4uSd0vLrAy8r0j1L6xezj7hSKwcD471\nM2jBzo4O530qtjBLIp/UySyZE5YPUtVxeBDGVY7a4Gc+MYDUQeaEfdcrLWB79JOCpb0EqPsqd7Qz\nThaWhzBWGaLTBQGoIyXsbbTkft2mhAPJYnRHG1yXXmudTYaQtiipLL2wVkdPNyzyez1RTPhUwrJv\nnlaL8ooo7K8ohdC4iJJglnDpb9eStQYIMrgo83krURrUB6V+lLCpX7QWjEoJy66V7442P2NQwl7d\nL1ZL2Gu6Ae7rZWxRisLK8XIastUL732Q5E8J17KbEPDqYq/FJ+miYQv7OfKwWn2+3zlhgYDt0bQV\nyE2m8i3hIM97M0KCzQmb6mCNGmrdKGHTYhmrheNndbQkWr91rWx3tA2v5/q6WcLyOeFqjiajNHC1\nLqpS7Se3P2j409M2N/coK3ZYR1nPOswJB4qvHqjOm1crf/25o8sn7MM6CvIHWchmi6vsGIJRN0rY\nyRL25462P+t7E7whvGYaHHiMJ+j+VTd3dGELk6FS56eEg1rCgR6LDPZFeF6tRkM4L+5oT9Zy9eaE\nZci3bUiuag7h3YVxT7MmhG0Jl1+WgeaEA7mjfViQNkvYf3rB0laHC7Iwy+T4iEAlrBslHAipO9rb\ntcpiSdDrYR1ulrBsixKmuTu6DKw61/t0gekhf+Eji9eFWSHOCZNwCeSOLuHbBV61hVn1Td0oYeFg\ncVotEU1xPf+s+bcewBI2xmus2J4X/lTKEi7cF+FYwlHYJ1yWu9Wlnqgf1OV/K9C9WtgVwLt1L7sY\nrjs6qnPCvrYoeZk7D+G1fLtXtWA7Df0MKK2WcBCCudm9HNUZzBI2xc0tSiFiy0s/zj7h+DOgAD6v\nW5WwV0vY233Nci2odRolqzYIrl9RCsvKreGcsHSe26fCLT+0It2yq09YeRbFelwdmXy1YUt2V88d\n7T4nbLnqHFvEirtulLDZErbcg/zgddk162hSl1xzlcW8ckd+3UkOm3LwqoSdR6olq8hoCZfhjg7y\nTIQUt9vaAeVCLdPZ0R4sYQ8GRBTnhOX704MrPtXxsWVEGA6+Duuo1olZ/gk2PeQ9Tes8dbXc0U5t\noxSHwftYxf4sDOpICZt+We4J27V8KI+mju+FWaW/nSqdMtZKu4etY4SgtS+QFg4vqnKx6k+721gY\n/rVf95xO5CwtSXfr1/vpQUsEbHHyuCRtIjx16EMqT0HLL+8giiTQwiwfYcNYgRz+pwzFVAijEg6U\nSM2oGyWsMD6nbgmT8eJ0YpZ1XlAI+J4JMVukaktY+elBqww+F2a5f8rQagkHm+sJ5r51k02dWtjY\ns12+D9LRYvbgJpHWvSqd+1iOy09m9cmnNBTv4uLh8dMly9/C5XnHV1d5pZweUtdSAV39IYMAVVf1\nZurPRmr+d3HA/L5m5SUZ9NjOjjYPUv2+psl7KZXOPWKZJezqjpZGVDsvXd0oYUd3tMLa87owy2/t\nMlVsoxK2dfLyNIN+ycnrPmGrdR7Y8A4y8lZZwlWUoYD1JCur+7mYn9Y0DM95WfjkJUzl9gl79fn5\ncz17kteWtPmCrzeWvUYZn1801RtN3XfYhKjCJx81BLM6fRsLwun7yHbs7mifCcJc5vbvecsxDljt\nBoskDTfByjAWKkH9KGHHX4oVztKGLVPM/keYfq67WQyhHVspUSr5LUoBVz1Gwu0TxKJRxKSweMv9\nUJKXxys2Jyy96vjhON/P+knbxdhSIhswl7VZynNjnEYEmDbz1078e76smAwAp8USwtRJSS/nf0vs\ncJd30hS/alXy9aOEHSxhaxdndUcbn7UtzPJvCJtSczqsQ5ekbxKsGF+4q6PN7mhPUcvjC0HRFQjp\ni4C+sG8FU/12KB8Pq66MQar/mh4HnzLKPazDZXBZW1SDN6dHhHqLkslVa3txP4JNRRekbfkMb03T\nJXxM4Y4OivPnZ/2ZMpq0d5Mj+1hJLakbJWweCFmVndwSLs6hOsZb3olZsvSMV1RpmvBYZ4ruZsV8\no/KDFSFpQE8WX2ALRP5Oft7Vin3AJfenuE0XuGEKX1BiVVJIcguy/FXPmumVPLoVfXf7hmel+eXy\nvMNt5UJOh1dx9lZohiFbeZ28kzvaKa99GwtCmOuHKQLv76CcC3cIC5jd0Y5Lrxy28koX7Lm2LSH5\ny7J/uIrjxYRbAF3XcfPNN+O1115DY2MjVq1ahaOOOqp4f9WqVfjjH/+ItrY2AMCdd96Jjo6Oykms\nwG0uR7ZiWe7JsI72goww5aNiu6WlSjOoJewsaPHYStNHrQX8N9+pZ13kVknh/aonIfxcdgxk3eJl\nXXgifS6Aso8kUoNZvVjL07oySf2QeaQ8iSfLw7Dy1dTJO5rC3lRgCMtzq7FFSbVWxohx3YSmWLgY\nFKsnyhK54YexgByCqQJ5uF3LrZOuSvi3v/0tJicn8bOf/QybNm3Cbbfdhrvuuqt4f+vWrfiP//gP\ndHV1VVRQdwzKTjJakrbh4pyfw7OSa34wNSZJ3FICJuj5xCyLEMEP63D+7eUZgxjh4kkWt8GPkF6H\nqb64J+Tl3aq7MEs2JxzunuB82tYLwUtZ9qhWxqYZ9ZfOVGGmblbNlelToQY4O1pYhp3S7sGgeK1v\nXvbCLMe1KHIlrG5vBqvaRS7V+nK5iVR5XN3RL774Ik4//XQAwCmnnIItW7YU7+m6jp6eHtx00024\n5JJL8OCDD1ZOUhfMbcqenV6+HQxJuLwC91cgZhdP6W/b6luhkCOgJexecez3y/mUobdRqbcwYQ9E\nvZSZ7VOGtvIpKGFr5D4tYQ+mcK0P6/D7nKd5NTdPiQ/RpPkT2upoxQ17watFDnlzavUsYYPykoUx\nWsIhW/iOlrDCg2hvikJyx80Slu2Tl/fN1cDVEk6lUmhvby/+jsfjyGazSCQSGB0dxV//9V/jb//2\nb5HL5XDZZZfhpJNOwgknnKCMr7OzFYlEPBzpp+ju7kBbW1Pxd1dXOzpmNpvSHBjI2Z6bOasF3d0d\nmMyU7nV2taH7kLbi71mzWjFpKLTubrOrXVbxOztb0d1ZCFe6P3Nms+n5jhn53/t2DZviT8xswW5j\nGop3bm5qMF2bMbPFlhfG9NpHp/LE8D5NTQ2YOaMU3pqGndKzjY1x8/t09Nuefd3y9MwpGa10ZDKm\n321tTYr0zfHHYvZxZGtrI2Yf0m67bnwOANrbzeXR2tZoCjtjqnzaO4ZMz8dnNBfLZ+aMZnQa4pDJ\nPOGhQTc2mtuE07sbKeRvhyK/WlobgEnLRWknZE63R8DxxCwnJdHd3YGBnoSt4s6Y0Wyxb+TtSvYe\nh8jK00O+FuJqbze3id6OIWCfPfwhh3SgqTnfLYpcDm8Y7nV0NPkaOBTSVq1bMNLc3IBCC9AAT1rA\nmk+dnW3Fa17qT0dHMzo7S31dvl7k041p+TgmJyfwVvG+WaZZs1rR3d2BltZ8u9Fimud6C5jLpK29\n1F93d3cglmlG39TveLzUxmcf0o7GppLamhzL5tM2xGucE7bK097XbK67U2XT0d6ExqFSG2xrbZI+\nXwlclXB7ezvS6XTxt67rSCTyj7W0tOCyyy5DS0sLAOAv//Iv8eqrrzoq4YGB0XJlNtHd3YFkcgSp\n1Hjx2oG+FMYmSj1PX38KQyU9VyyEgYFRJJMjJiXc15dC3DD66x9IY2hoovg7mRwxpS9zqfT3p9Ce\nHTGlBQCDg6NIJkrPDw+PIdkwgqGhMVP8qYE0jMg6w2RyBOMTZsU1ODiKZGwE6bRc3uHhMZtMY+MZ\nDA3aByjWZw3SFGMYn8iawgwNl8pA/uyUjJJ7BdkKpNMTyjiM8Vv3+gLA6OgkksmU43OFNI2/R0bG\nTWGHhvKyGmVLJkeQGizV4aHBUWSn4ijURSt9/XJZjExOmsvA6d1ljIyMIS55ZnTUqoHlFBSiW7qF\n7s3JfZ5MjmByMmtTWENDaai0WNIlD+Xl6a7cCnGlUuY2Ya5vwnSvqISzWVNcw8NjDl4p+3sV0vZi\noY6Pm9uyF0vYmk99/Wm0JjRlHloZGhpDX3+j9J4upspxopRPVku4vz+NhuY4xqbqmNCFa7rG9xo2\n9HvplLnvSBnKJ5cTRU2VTI6YlHBBn5iPCi79aZUnlTa38aIsw2OYHC8NSAp9qN92qMJJmbu6o087\n7TSsX78eALBp0yYcf/zxxXs7duzApZdeilwuh0wmgz/+8Y848cQTQxDZP1Z3tLC4MxTTobZnZZ5H\n//uEVfNNXl0eAV2IHhdmWVaiBF+UEOSxarmjA/jGvS6c8zsn7IWy54Q9r8j3ftCG0zRIIGnLyKqK\nuqMtKZXC+BA45Kli/19R8n9iFmB9R8lCPJO7Olx3tNc5YWM9VK2ONm9RcnNHS/6sovvZiqsl/PGP\nfxzPPPMMLrnkEggh8M1vfhM/+tGPMHfuXJx11lm48MILcfHFF6OhoQEXXnghjjvuuGrIbceyuMrL\n/FNp9av5WUW0imTtAUz7hB3CKjt52/7VkFZHq+bFg84bWtLz9PF6pbIIRjltx7ZP2Lr6s7gwy/qg\nzzlh1cDDdFBAmb2AQg7vn2cMf8GRdIGk4ZqfgYf8NfwcHKJu2JrqxCzZnLCyLB0T94UW5CH4b8a6\nsL6PrH8oeWjC+ICD6XnH/fOquFX9usM+JgvyYyt1m+FWLVyVcCwWw9e//nXTtfnz5xf//vznP4/P\nf/7z4UvmE3PbsSsH2fm9hYfMfardOnLqyGSFparYQQ1fr9XB+4lZllNiQlod7UXQalXtIKuW7Qvn\nFErYNJT2ooRVYcJTfH4GN54GdUIoDuvIX/N2wECACqIURzaK9hOBNT5PiUoi8e6OjjxWY0UyEHPM\npwCvbN4nbDz+1dvzKkvYdGhnzvkAHbNXfUoiAYsS8SZPGNTNYR02Y9JDjpZOrFLHU64lbLLQlZ28\nNVGLRSbpDOXpqu8Z79ut84A1LsjIWCWb67NBTBCXGF2+olQ6zMXynrpDhfGatqaF68L0McKTLXK1\nWaXCOWs1kxWveBGZJewsmjQldVCXDPRcTELedmSdgXI8VcoFqzs0iFWlOnDCaQDl2etRjMupCKbq\nvm6whCv6AQd1x+tlHGAMY93PrErTfEMPVE5h4GoJTxesx1aaf8utWbk7Wnh6tnhfagmXrnlzR1ue\n9+KOlr6PrrplTt9oCYtCev41gosX3dMzxesewzmimd9Lhnn07TyI8LJFydPJXIqMMZZr+XPC8o7H\n6z5hudHqdOSUB5Fs/WqQM8oVZQBILTd1LA5lrfBk2stWOChBeXxBCLQ9Cf7bjNUdLT2cxbhP2NY+\n/KUHqLco2WcPPQ7WhT1epxOzBIR0EBr0ozlhUDeWsH2IbRllSdqITAlaR4duhqK0b1BY4d63g5dX\nBfzEW7CEq1XpAs2pOcWnTEel+NQKNFD5+J4T9nZUgG+UAy+vgWWnY6mfDONLjF4GHqWpo8rkj+2m\ngyUsnCzhkPG/CErzX4csxorscbNBUH4/oTJKZAMeGWp3tMHwcRrsKTsMy++A37QJQt0oYeviKtMp\nWIrKI3MHy6xop/YvG917toSF/Lqt8cs6K8nWHNfvCU9dN8kEALrw/n1bB0vBmzvMW+MKZjN5sYQd\nlLAt3/NS2N7LpztaPSAIc064TEvY/qAi3NQ+YaMVqqo7kg4zaCcudSb4sYRdy7pw3RTIGola/tAP\n6/Afn99xipeFciZLOOwPODhYwhCKe4o+1GwJq3sP67oPgzA2Y6xa1I0Stg9m1YrV+oy5jO1K1+OJ\nsU7CSEMrYw1YA0ruZlW0RTvG8EywtKTPeojLuyVcoWagatyQbJmwD7KnfqrLVJqkMkw1jkCU1XvJ\nsZW2a8LR3PUkudJq8YHjI8HzT5eu1LQkJ/WUeEkzhLrry7MRLFlbcFlVsbmjy+s7zFuUjIaSN7z0\nxDHHL5sJ+XoGW12tnilcN0rYOlpy2idcKALVwizjb104Vzb3LUoOlpfCcrVtlTHOdSrCGONTWaS6\nREnnXWx6MKvMxbqQP6IatshHuL7EMUw0qNLJOKzItG25UnkWrBXEBWWQMHWwct456IjOuWP0tjDL\nuX54Mh4LfaR0EO2WgeoBl+rsaCd3NHRdWS/DHDJqkFvCwiUd3wuzhFUR2vNTtxxb6dGMUGJeHa3u\nG5XmjYeBneOcsFDNCVs/TqGMInTqRgnbLV/DHSGkJytBqQTNDdG5cksai8+G6lb/pM9JOl03JVhy\nxZiVutCdFpyoCWAIB47bE1ODFWv5G9Gz3hdmFaSwZbVlqsMVudGFqizMkmaEh2YvACd3dDCsueV9\nTljaBt2y3jrYNGBfCV8IZ7zmo4aH7I6WkU8h3G1tJuPE0yuE40UDYDqX2p7V3lwBJXe0wfBxc0cr\nBXcaFFSOulHC1sZjs4QdnjEbNuawej4CJbI5Bt8nZtkqoLWD8GB1GKJRWl6yOWHhNr5W4zKV7ekZ\nZVxB5PGQkLHztXXMtvLxIKyHl/Z0iEmF3O+mmujo0vG3MCtmumZ+Vr31ztlCcUR624dCcqpfWqkN\nCHNnYH5G6BAK69tRep9Fq7KE8zI4JBPAEjb3kxJLOGd2RwsPOxCcMPc9hnQUXsKpHw5pCmsQaE7u\naOXoXPht1qFRl0oY1solhNRQEIb75rjMFcBvA/N9YpYn09dyy8EdrYqgqIAsW5RgaVzecWg4ykcU\nnYstmPWKXD5N8stpMV1OlzduWZqlQY1D+QRsrZoI5n1QIRQdj7xjlswJ2x9UHNbhHl0hRatlVdbC\nLNnFMhZmOR+ZWHzKc/ym1hfKynFJCg59UbBPGZqrr6wtGXPePjAw//by2qotSs6oLVTZOztZ9Pkt\nSsZFhaU7Jm8UlXAAjC5CYS0s8xxhceZQUoK2vXPC+Wxlt33CTq5LL/tQ88+Z1v7Jn3GKTx5t6Rm3\ngYYtdXl85VnCIdZ6h6j0XOnwAev+XR3WeSEhjc80f+TbEjb2etVYmGXESVaZJnVYHS2pk8VHC/XQ\nlry601ZlYzlblCxjactN84/SgNxBKAdLOOw1drJ5zbzSdPCE+baELQ1fSNSsbXW0xBIO2HSN7cJ5\nysfffnUnS1gpqnWQ5nmAUD51o4TNbcduockVUP5/3dJarY3X0QUkdUe7y+h0wya/0QVUHEE4bVGS\nJ1Ma/Vs7Td12zRNucis6Ek9RB2gDwjQnLI8g59C4vC6cU8w2OMRb+rugvKyeh0p9wEG+JcNbWl5V\ntrL/dNZ8MHfozhkp7RT9mJyWx3VzoUgDug2OHaL3ddeK8jgQJxNP868LhbCvUC5l6VTdN+S7Zh2s\nF8eo3lNWW8IOVrbRQFX0Oabhg8uJWSrvWa2oHyVsczeZG5Pxt80Sthmuxmedq5hU0QhdWjnUexOd\nTUpzVZUvqTc+p16BnMfYlnUBX5awOTmr0pXL43oNjk3QB1MNyuFpPedUHgolbI3ENK/s7taUFhXM\nI/xKfcDB83Yqq0ITgJOylrsuHUXxNUdsJdBgwsELpdqJbraEFZ4R2XN+R2YuSOeEheZiCftLw54n\nLvuEYR6su3ne5MgHXqq5XvNf6l/Gq5rjWR3KymnzgFaLujm2Eg4ZKCB3KReu6Jaw1rMY/LqjPQlp\n++V0R7avzf50cUuNKtbii1mGlkIPOCdsid9LGFUbCBCXNZww/KEciBjOwrUGsc/9KjoZn/2t13nZ\ncnAbeJl+yfYJSx5UDR7y4Z3ktw9uCzKaLmnmezKcpo78ND17WZsjErJwkkFl1dzRkmuqBaaFJ8o9\nMUuDZCxm9RhIpk39pKreouQspzJc0dgpEXM46sfqgi/1GcIlocpRR5aw5bdpJKVyRxc6C3VlyCth\np3Rlyr2k9DVT3HLL0RaH7exoacK+ZALk1kRx9XeAOufqfnY41cselz2kf9y3KGWNDd96NofVm1Ic\n1Di8pwdLWHYuhICGMBdmeR7dmKQwhvO6HUliLVkGcMX3dRy9KL3A0kfkOtjNEnaKX1Io1oRsZevt\n+4M84h8AABanSURBVLflb/ZT73V17ov8pWpf/2LPU+OCP9tiQhfPmwzVQlUvU1mGJCW/Dfnv0ial\n5WPzpDpGESp1o4SNhWC1XlWLqyQ6GEII2I689Fm7VatzVfsO7VetV+yWsPx9FJab5b6tcxLBVke7\n9LGKh7zdKGcgqhp0ATAfladwP9uecerAA8opAFNvUKk5YWHx88CSrjo64bhwzNxxKJZH2+K0Cae+\nZ4k60D5hU9rWTlaSL9YoJfL6GutIkJ9XYI5XOSfseDdYm7H1IzKXSPGWyojwn641bXv5Gn+rJ4WF\nrEo7yKNDl37KMO+lUQ8KKkndKGGrIhXWhiWzRmQ6yWIVWrwXNqRfZzL48sxzwvIDAmRuL9NP6cIs\nRbryW1PXC83YPC8TeOuI1fC1NRDdFiy4O9q9ZxeaZigweXjrKT3mjkA+B+jkqfDSWKV1RDP79io3\nJyz55cUdbXvWIWnLb9nJbFN3HGxhFy0su13GXiC756xw3aFsnRYwOnzKUJ6S83XlPmGnmH0qDt3W\nt2m2Psk6aJCdz65cDS/BvDDL8LeDEjaViS1G+xXnfcKq68I6L1k16kgJm60Tc2PSpfsCZZ2stWN2\n2nOaT0pSLUxxGOKybf6XyA64uKNLc8J214xCaRSilbmjxVRcHi0xlTtpSiRJ5HIZ7dctvz1JI88b\npzLL5oyLqpxHv+otZLr8bwXygZqlQ3ONxRn1nLDHmD0vzJraomQKr1iYpaifxac0Y97LxSqoBW/7\netXp2ctW8WEC04jRPijzoucclbDslmaPV3lspcMg1q/xZn8fTTInbFmYpdkHTn7SNc0JC4fyV/YT\nij7UmIbP/roQkdmQU8cRNnWjhI0ISOq6JFNLo1/1s8K1dqs6WFlIuXJ1dHdC1VmrD9VTD/ZklpBK\nWnfUY1cr7greYxt0jNs0Hle2NfVRebZ9wuYY5Y95kdMQRpP8NSWYh4ic0vAjiIe9pjYraery1KPG\nvlhlncium8S0HhojJR+m3MM6bAMCVTDhFEhAlXdelYAqZdPzDk86n5jnVwtbnpE9bpwTtgQpVZWA\n/YfD+gyLmIofJYx57jQn7KiEHXvXylE3StjmjhbmDM0ZC8kQzvh/IR6rUnaapFe7o/PXTQuzbBZu\noRbLLTCjDKW/tWJc9nC69PmSrPbKqU+9YKCzo13khtQdreqEFHljwflzau77hHXLBxwcrSXY60c+\nEmMFKcMSDvOwDg+Wg3PHYt3eJRznhE1z2MqFWS4iauaaLU1H3kSKUjhiKF97EZrTLg2I1WUrhPoD\nDkb5bVasTHNZAljrsf+FWf5PzLIuzHK3hM2DkCBblDwvzFIMS1TeP6i8GRaEsJyYZbjuPACrHHWk\nhM2FYHMzKjpC4//5v+3uaJeUpbLIHhO2E5kUMdh8U4oZO+uDqvgskloXMThvfQhO6TUczSZLWJ9p\nWH5rcC6znHVO2GH0q3KrwvRMQCxlWn4JqAY30sTdr7iII1vcYg/jrIU1h87VRsAKUipCh8GbQpe7\neaiC4U25qrYoOcYcQDy3Z7xEGbjt6k7tSKUR3dtozNESVt0w96fej9Qsn7pRwqYMlHSu0rOjJaNs\nmyXsYFUV0rKLUlLCXhZmuc2HmKpgofMW9q87lT695yKrsIxmRTiro+37bHWzzE6yucTtRYZS3qjL\nzHxYh9toPC+/7b2MDdTTpwyNVlIh7pC3KHn5lKEoWDLqLUpmd7TDFiXjuMoSrPC+ssGlsPwu/anI\nR6fV0R4Qtj+c0xPWxm++qSwz45vZVhErwpUeMA/8NUkcJRE8TCV4RJe0E2s/YNqiBPMnT4O4o5W7\nwRwGbE5FIusoHA/rsCpb0xqbEAbXAagbJexkTQrLfc0SxuaOtipwU7TqBmZOWzJEU3XYLhaDdE5Y\n4dZS31MMGAqjjiArTW1yy8UxX/JWvR2GPa5PCjgoe11dL1Sft3OWwf195F4R6+/ymr3qaZsF6OrB\n9SaH5vBL6mKSX3CnWKUDPqtoEt5OuJK0S1VQr3tkPA5CVWHCVQ7WQZFE9VsHkDIt6kMor+5os5Rq\nCvfM2e+8n1s+wLH28w5RhEz9KGEnRWoZ5RTQJQ3U6kq2W0uWdCUFqhssYbMhonBHu4zuZJ8ylLm8\nC3KqdH3BUrZuUcpbwvJnnHAZOxgu+HdHK5WBw3omMbV0xKlBmywq3c0dLc9PYYnDDekKepVFGhTV\n94StdpimSterJTwV2pjvVutpKg6rO9qWD+rtn7YgwTrFUvm6uaOLzdU8KrPF5pSWITpP4czymH8r\nV0c7xO7XW2Dr22AfpAnbwiy7JRzUS+F8Ypbxgn0e2vqcPLQkTVi+llT407LGhvuEg2Ae0tlcQF4z\n1TbaFNaO3qExS3DarqA+HMKDrKYGJAz/mv+yRCyLBjY3oQNOh0rYYpG4JdXuaDcJSo5cSRLWyNTu\naEvDd254kpGULVEPStiD9VN2my/DwgIMC6BMF9XxOdcDedrW/ehm68VZUvnZ0c6Y2rLjIE9VB+zl\nrlKCxpXD9sGHCzJ3tNd5mzIwOeyKKVuTM1kRpjBWUbwMKZVbHB1H9A7KWmLtOJ6YpfISmqMIoUF6\np26UsLAUlNPe39LgR5j+l4XVrXMFjo1ZHkfxum3jeym8JQLzT9NhHYU5DHu35GoJF4eN5tGscOhc\nnHDbJ1y6b2y4Do3AIa4iCs9n4WYMhUGX/PGccZ+wwVKSyabeJ+xvxGwxuvLXNE2659K6+tszKkvY\n8qUaV0u4kAeWDtcaziS6zRLOx2Qvakuv72mLUp5An5YzDKBdV/IbRVSFcXBHG79w5NiSVErActlp\nm5I8fS2AJWw+Jx8Czpaw/QPRAPwd8Wiq8YYqa/M2mfpzd0vYJLbbBxykA3fzyndawkGwDKpstrDC\nYpJes1UIc9zuohjTc+iwZdaHSq7iLePiI3u6Ts+rBgz5y+UvzFIqK7uukQnn9NObPBoA6M75Zxt9\nmwdcMhkcpwu8KGFZvgNwsir84s3DkFesTmkVB5DCUNeMGBa/GWO1xgEI+35ZqyVsPKxDIY8th/yf\n2SFFtbZDmANZn/JYTo45HPjaVK4GS1YZ3FQikkDmui5zRwe1Gh0tYXNIX/EaLWGZ0pbmoCj+EyDF\n8nBVwrqu46abbsKyZcuwYsUK9PT0mO6vWbMGn/rUp3DxxRdj3bp1FRPUDesoxm4Jy54pjOSMHbH1\nt3WO2ByRdA+oKGlIU5ensIRtlpVtgZBsYZYu6UiEVEar7JrVApFY1Z5wsy6K7mj1SLZ43fbbm7Y2\nj6C1qS1KDunkzGVtnhNTHNbh9J4+LeFS3M7K0DfKfLXIrrKEhTy8CqeFWfk6JWsX1gseJoUtt30N\nFUWptlijN9V408ld6rLNt03V6mhDdLa7LvVfs6+gV21Rcsomv7pQuq7EtubCsk/YtLyj0D68p2mM\n3trvWhKWPqX0tsmL016GNkvY4Fm0jlerhOunDH/7299icnISP/vZz7Bp0ybcdtttuOuuuwAAyWQS\n9913Hx566CFMTExg+fLl+PCHP4zGxsaKC25HPrIt/C3fSiR71FrK1o7aOd1iepLbVjm8DiTNtwsL\naCQpF5WeKh7FnYAVzq44g0dsf5cgD2pT2SPLnDyqQxoAtbK1hfFtCavw4iLwiKfBjZMkxjn3Qv45\nLMwyuaOtaao6Z3X66mzMR14qN3/OO/XA1ENb8PoMnOe3vZaB17TUT/h7xt6H2AeG9mk8+yLLoDVX\n8Q0N2wX7vLQ0NvnfEiUsHeColHsV0IRLarfeeitOPvlknHfeeQCA008/HU899RQA4He/+x1+//vf\n4+tf/zoA4Morr8QVV1yBk08+WRlfMjkSluz42U8fwPib4wA0TMS6kIu1AgCacn2IizGIqQ1jmtCg\nowHjicMAAA25QST0VKnPhqEQpq4Zf5tGWT69tg1ZHU2Z/Dds080NEBqK1cCrLSSK/5jTFwq5lNe1\nfAebic1EVpsJAGjO7kEM8pOA/L6rKl3Pz9vsK0NDNG7+02MwbJfGWMMRAIDmTAoxDAIakNOaMBHv\nNsXXmt0FXUtgPD5VD/RBNOgpd7ks79WY1dE4VaYTDXFkEu6KYTSRlzGuj6Ix11+Mq1zaxzIAgEwi\nhomGuCSEKM1Xihjyuz07MJGYYQrVoA8gIUagify7xHQBTW/BSPNsW4wt2d2msjG2vZbMruJ7NWZy\naMzmyy3V0pC/qJVs0KzWgUysEwDQnNuLmMgW4yzlVxqNuQGDF1zDWOKddpkyuyTvXsrjiVgncrE2\nAPl6ABjqlNAAoXkqj6zWisl4FwAgJsaha8359LN70D4+AU0A2biGsYZEMb7xeDd0rQmayKA5t8++\nkVXP53mhHif0FBr1weLtUt3XAL0kpKaV8qllchiIDXvyFNiMTEPfYmxnhboQlMnYLGRj7aZrMX0c\neiyfZ+b864WGnC0OAIjlGpFqOhQA0JRLIi4mbGEEWjCWOAQAMHN8D5oy+TDFemcgE2sr1rtD0rsQ\n1zPIJGJIN7ZjMp6PI4E+rPz234Smr7q7O5T3XC3hVCqF9vZSRsbjcWSzWSQSCaRSKXR0lCJva2tD\nKuXcqXV2tiKRkHUW/hnpHUImcazt+sRURqrIxGchE58VigyuJAA0VyepIIwnDq+1CP5RVJ/xhnYA\n7fKbKHVYBTKxWcjE/NeD0TLKNBdrxdiUwgqDUXXb9kUm1okMOh3DtE0MIN3UKVWCBQqKBJjKJ48U\nBkZWcrE2jE0pTyeM6bphrQdBKSgQABhLHI4xddUDAAitAWOytC31ORtrtykvVdhi+o0zAMyQ34wQ\nBQUMWPPvHeqHDPXIOrCWMdR8uOf22dcmrwtarhX9A0Po7p7pLaIycG0m7e3tSKfTxd+6riORSEjv\npdNpk1KWMTAwGlRWG5+/9vP4859fwoHkEAAglsi7OApuDg0aNE0rHsKQSGj5Ywt1QIsDumHgVVis\nWpi2tf6Ox4GcfKCWDwttytURMx36oMUAEU8glgN0vTTSj2mWj73HAWGUJxEHYjFoiENkxqFpQCym\nISe0/PxFQa6EBj0roGnmdOMJDbms3bxNxBoQjwlkkUVc0zCZyYdp6WhGZjwHXQhoyEmfLcqe0ABN\nQM/Y78XjGnKGeVdNy2eeyOZs72iLdypPrHlToCnRgCyyNtkaGhsgGpqQHUsjpoliuWqx/D+apgNI\nQOhZCD1/XYsBheLQAGhaHLrIFcsSiEE3CGt9L9GQbwNaplSmzS2NGB+blL9bDGhpbUIul0Mmk4Ou\nxSEyWcQM9TAWSwBaNl/nfHjD9IYEYgY5rMQTjYCWg8jm0NDcjImxcSRaW5AbH4MQQCKRQExPIJMb\nLyWrASLWAC2uoz3eiPTEGNo6j0Z6YAj5qcEYdGgAcog1NQKZyampG6MXKY54YyNEbgwiN3Vdi6Mp\n1oCsnoWIZUvhhT0PGxOJfDiU2nRMi0NHDi3NLZgYH4cWN5T3VJsVuoamhIaxjNnijFnafCyuIS6a\noMcmiquvrYvMtRiAhkZgcrIofzyWA2JALpOPEwDi8Riga9AbGoDMhL2OtjchOzoBoQMtiQbEEsBk\nZurrbjEUw8cT+RfRs2Kq8eRP+2tubMH45Fg+/xIxTGRKja+ltQMT4yPFvsqpHhbrhKE+x2Klcos1\nNiOWzSJr7Kti8SlZstBzQKy5EWJy0pRX1rwtptMIxDKNyCIDaAKtrU0YG89A6Pk+TItP5bmQxGHo\nMGKtDRDjGekmAC2/JQLx5mbo2QxENge9IZFvm5J2FIslgLgOkdOhaUA2Vmo/8YZ8PhxzzEno6pwZ\nDUv4tNNOw7p16/DJT34SmzZtwvHHH1+8d/LJJ+Pf/u3fMDExgcnJSWzbts10vxosXvzRUF3cByvd\n3R3MxzJhHpYP87B8mIfTC1cl/PGPfxzPPPMMLrnkEggh8M1vfhM/+tGPMHfuXJx11llYsWIFli9f\nDiEErr32WjQ1NVVDbkIIIWTa47owK2zCHqFx1BcOzMfyYR6WD/OwfJiH4RBmPjq5o+vnsA5CCCFk\nmkElTAghhNQIKmFCCCGkRlAJE0IIITWCSpgQQgipEVTChBBCSI2gEiaEEEJqBJUwIYQQUiOohAkh\nhJAaUfUTswghhBCSh5YwIYQQUiOohAkhhJAaQSVMCCGE1AgqYUIIIaRGUAkTQgghNYJKmBBCCKkR\niVoLEBRd13HzzTfjtddeQ2NjI1atWoWjjjqq1mJFlkwmg5UrV2L37t2YnJzEF7/4RRx77LG4/vrr\noWkajjvuOPzTP/0TYrEY/v3f/x1PPvkkEokEVq5ciZNPPrnW4keKvr4+fOpTn8IPf/hDJBIJ5qFP\n7r77bqxduxaZTAaXXnopPvjBDzIPfZDJZHD99ddj9+7diMVi+MY3vsF66JPNmzfjX/7lX3Dfffeh\np6fHc96pwpaFmKb8+te/Ftddd50QQoiXXnpJfOELX6ixRNHmwQcfFKtWrRJCCDEwMCAWLVokrrji\nCvH8888LIYT42te+Jn7zm9+ILVu2iBUrVghd18Xu3bvFpz71qVqKHTkmJyfF3/3d34mzzz5bvPnm\nm8xDnzz//PPiiiuuELlcTqRSKXH77bczD33yxBNPiKuvvloIIcTTTz8trrrqKuahD+655x5x/vnn\ni09/+tNCCOEr72Rhy2XauqNffPFFnH766QCAU045BVu2bKmxRNHm3HPPxZe+9CUAgBAC8XgcW7du\nxQc/+EEAwEc/+lE8++yzePHFF/GRj3wEmqbh8MMPRy6XQ39/fy1FjxSrV6/GJZdcgkMPPRQAmIc+\nefrpp3H88cfjyiuvxBe+8AWcccYZzEOfHHPMMcjlctB1HalUColEgnnog7lz5+KOO+4o/vaTd7Kw\n5TJtlXAqlUJ7e3vxdzweRzabraFE0aatrQ3t7e1IpVK4+uqrcc0110AIAU3TivdHRkZs+Vq4ToCf\n//zn6OrqKg7+ADAPfTIwMIAtW7bgu9/9Lm655RZ8+ctfZh76pLW1Fbt378YnPvEJfO1rX8OKFSuY\nhz4455xzkEiUZmL95J0sbLlM2znh9vZ2pNPp4m9d100ZS+z09vbiyiuvxPLly3HBBRfgW9/6VvFe\nOp3GjBkzbPmaTqfR0dFRC3Ejx0MPPQRN0/Dcc8/hlVdewXXXXWeyLJiH7syaNQvz5s1DY2Mj5s2b\nh6amJuzdu7d4n3nozr333ouPfOQj+Id/+Af09vbis5/9LDKZTPE+89Afxjldt7yThS07/bJjqBGn\nnXYa1q9fDwDYtGkTjj/++BpLFG0OHDiAz33uc/jKV76Ciy66CADwnve8Bxs2bAAArF+/Hu9///tx\n2mmn4emnn4au69izZw90XUdXV1ctRY8M999/P3784x/jvvvuw7vf/W6sXr0aH/3oR5mHPliwYAGe\neuopCCGwb98+jI2NYeHChcxDH8yYMaOoTGfOnIlsNsu2XAZ+8k4Wtlym7QccCqujX3/9dQgh8M1v\nfhPz58+vtViRZdWqVXj88ccxb9684rV//Md/xKpVq5DJZDBv3jysWrUK8Xgcd9xxB9avXw9d13HD\nDTeEUtHqjRUrVuDmm29GLBbD1772NeahD/75n/8ZGzZsgBAC1157LY444gjmoQ/S6TRWrlyJZDKJ\nTCaDyy67DCeddBLz0Ae7du3C3//932PNmjXYvn2757xThS2HaauECSGEkOnOtHVHE0IIIdMdKmFC\nCCGkRlAJE0IIITWCSpgQQgipEVTChBBCSI2gEiaEEEJqBJUwIYQQUiOohAkhhJAa8f8DNyFzT00i\nJ48AAAAASUVORK5CYII=\n", "text/plain": [ - "" + "" ] }, "metadata": {}, @@ -176,7 +176,9 @@ { "cell_type": "code", "execution_count": 5, - "metadata": {}, + "metadata": { + "collapsed": true + }, "outputs": [], "source": [ "def logp_lda_doc(beta, theta):\n", @@ -229,7 +231,9 @@ { "cell_type": "code", "execution_count": 6, - "metadata": {}, + "metadata": { + "collapsed": true + }, "outputs": [], "source": [ "n_topics = 10\n", @@ -262,7 +266,9 @@ { "cell_type": "code", "execution_count": 7, - "metadata": {}, + "metadata": { + "collapsed": true + }, "outputs": [], "source": [ "class LDAEncoder:\n", @@ -313,7 +319,9 @@ { "cell_type": "code", "execution_count": 8, - "metadata": {}, + "metadata": { + "collapsed": true + }, "outputs": [], "source": [ "encoder = LDAEncoder(n_words=n_words, n_hidden=100, n_topics=n_topics, p_corruption=0.0)\n", @@ -332,7 +340,9 @@ { "cell_type": "code", "execution_count": 9, - "metadata": {}, + "metadata": { + "collapsed": true + }, "outputs": [], "source": [ "encoder_params = encoder.get_params()" @@ -342,8 +352,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "## AEVB with ADVI\n", - "Here we will use class based interface for variational methods as we will decide if we need some more training or not." + "## AEVB with ADVI" ] }, { @@ -355,21 +364,19 @@ "name": "stderr", "output_type": "stream", "text": [ - "Average Loss = 2.9809e+06: 100%|██████████| 6000/6000 [01:11<00:00, 84.38it/s]\n", - "Finished [100%]: Average Loss = 2.9795e+06\n" + "Average Loss = 3.0174e+06: 100%|██████████| 10000/10000 [02:49<00:00, 58.89it/s]\n", + "Finished [100%]: Average Loss = 3.0206e+06\n" ] } ], "source": [ "with model:\n", - " approx1 = pm.fit(6000, method='advi',\n", + " approx1 = pm.fit(10000, method='advi',\n", " local_rv=local_RVs,\n", " more_obj_params=encoder_params,\n", " # https://arxiv.org/pdf/1705.08292.pdf\n", " # sgd(with/without momentum) seems to be good choice for high dimensional problems\n", - " obj_optimizer=pm.sgd,\n", - " # but your gradients will explode here\n", - " total_grad_norm_constraint=1000.)" + " obj_optimizer=pm.adam(learning_rate=0.05))" ] }, { @@ -379,9 +386,9 @@ "outputs": [ { "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAfoAAAFJCAYAAABzS++SAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3XlgFOXdB/DvHrl3cy+BEBISbggIIQTQgChq1KrVVuQq\nVtAqSrFooRzl0GJFxGJVvI8eoqUira/t+2q1CKZcoaKAxFsxCAmYkJBjQ67dff9IstnNXrO7Mzsz\nu9/PP5DdZ2eefXZmfs/zzDPPo7HZbDYQERFRWNLKnQEiIiKSDgM9ERFRGGOgJyIiCmMM9ERERGGM\ngZ6IiCiMMdATERGFMb3cGZBCdXWjqNtLSYlHXV2zqNtUM5aHM5ZHD5aFM5aHM5aHMzHLw2QyenyP\nLXoB9Hqd3FlQFJaHM5ZHD5aFM5aHM5aHs1CVBwM9ERFRGGOgJyIiCmMM9ERERGGMgZ6IiCiMMdAT\nERGFMQZ6IiKiMMZAT0REFMYY6ImIiMIYAz0REVEYY6AnIiIKYwz0ArV3WLHvWBXOt3bInRUiIiLB\nGOgF+tfBE3jhn5/iz//6XO6sEBERCeZz9TqLxYLVq1fj+PHj0Ol02LBhAxobG7Fw4UIMHDgQADB7\n9mxcffXV2LJlC3bv3g29Xo9Vq1ZhzJgxqKiowIoVK6DRaDBkyBCsW7cOWq026LShVlljBgAcr2oI\n+b6JiIgC5TPQ79q1CwCwbds2lJWVYcOGDbj00ksxf/58LFiwwJ6uvLwcBw8exPbt21FVVYXFixdj\nx44d2LBhA5YsWYKJEydi7dq12LlzJzIzM4NOGyqnasw4+Hl1yPZHREQkJp+B/rLLLsO0adMAAJWV\nlUhPT8exY8dw/Phx7Ny5Ezk5OVi1ahUOHTqE4uJiaDQaZGZmwmKxoLa2FuXl5SgqKgIATJ06FXv3\n7kVubm7QaVNTU6UrFQdrXigDAPRLiw/J/oiIiMTkM9ADgF6vx/Lly/Huu+/i8ccfx5kzZzBjxgzk\n5+fj6aefxpNPPgmj0Yjk5GT7ZxISEtDY2AibzQaNRuP0WlNTU9BpvQX6lJR40df5rTrb3FkWOi1M\nJqOo21YjloEzlkcPloUzloczloezUJSHoEAPABs3bsTSpUtx0003Ydu2bcjIyAAAXH755Vi/fj2m\nT58Os9lsT282m2E0GqHVap1eS0xMhMFgCDqtN3V1zUK/lt8sFiuqqxsl274amEzGiC8DRyyPHiwL\nZywPZywPZ2KWh7cKg89R92+88QaeffZZAEBcXBw0Gg1+/vOf4+jRowCA/fv3Y9SoUSgoKMCePXtg\ntVpRWVkJq9WK1NRUjBw5EmVlnd3fpaWlKCwsFCUtERER+eazRX/FFVdg5cqVmDt3Ljo6OrBq1Sr0\n69cP69evR1RUFNLT07F+/XoYDAYUFhZi5syZsFqtWLt2LQBg+fLlWLNmDTZv3oy8vDyUlJRAp9MF\nnVZO355uQGZaAqKjxL09QEREJDaNzWazyZ0JsYnZNbTgoffcvj52cDruvjH0j/kpAbvfnLE8erAs\nnLE8nLE8nCmm6z7S5fZzX3iHv6rBs2+Wo7XdEuIcERERCcdA78OCq0d4fK/skzPYvuurEOaGiIjI\nPwz0QXrvw1NyZ4GIiMgjBnof9HrfRbRt55fY+3EVfv38AZxv7UB7hxUtbVz8hoiI5Cf4OfpIlZES\njxnTh2D7zi89pnnnv9/Z///vQyex+6NTiI3W4bc/m+SUzmq1Yc/HVRg7OB2JCdGS5ZmIiKgbW/QC\n3Hz1SGy4Y5LvhAD+XvoN6hpbUXW2GXWNrag5dx5v/OcbtHdY8d/Pvscf3/oMm/96GADQ1m7BseNn\nYbWG3YMPRESkEGzRC5SWGGv/f0ZKHM7Unff5mV8+uRf90uJRdbYZx6saUX68FgBw4vsmAMDWd77A\nno+rcPOVwzBtbH9pMk5ERBGNLXqBtF1z8APA+tsmCv5c9xz5H39zFlaHKQvOt3bg2PGzAIA/v/05\nzjW1ipLPMJwWgYiIgsBAL5BDnIdOq8GkkRlBbe/1979GY3O7/e/Nfz0S1PYAYH/5ady6cRcqTnNC\nCiIi6sRAL5DGIdJrNBoE227e9eEpWBzuzZ+sbgpyi8Cr734BAPjP0cqgt0VEROGBgd4PMRLPbb/z\n0EmX15pbOh/Xs9lsaG7pQL25DQ3NbXjlnS9cuvu7KyPsvCdSnkOff4/jVQ1yZ4MiEAfj+eHJe6dK\nOkL+lXe/QMXpRkwalYEhWUm445H3vab/+PhZPHTHZPvf3Z0OQm/TNza34bvvmzByIFcDJJLak38/\nBgB4acWlMueEIg0DvR+0Gg20Oo3vhEHY83EV9nxcJSjt93Xncba+Bd/XNeO9j06hw2LtfMNLpG9t\nsyA6SguNRoMH/vwBqs+14P4FRRjQxyBG9omISGEY6AMkbbgXbtnT+1xe2324EglxUfjxxYOcXm8w\nt2HJE3tQNKIPFv4wH9XnWgAAdY2tDPRERGGK9+jD1P/ur8Cf3v4Mn5+ow9tlJ9DeYcG3XaPxD376\nfa/UNvzi8f/gz29/Jtr+681t+PTbWtG2BwBWm41TCxMR+YmBXgbxMaHpSHn/cCU2vvoRXtv1Fe54\n5H38fnvPI3ynHEb5t7Vb0djcjt2He0br7z92Gt+e9j5w6HxrB863Ogdem82GUzVmrHmhDJu2HcaZ\numaRvg2w6dWPcNfmUi4NHCasNhs+P1GH9g7+noFobG7Dzx8tRekRPmVD3jHQB0jtI9vXvHjQ/v8d\n739t/7+5pR1/fOtTPP/PT/CbP36ADosVZ+tbcN8fDuKrU/VO21j0aCkWPVpq/9tms+EPb32GNS+U\noel85xwB9U1tbvf/6be12HNU2FiEbp9/dw4A0OQw/4A/Dn3+PV7pegSR5FdWfgYbX/0If3zrc7mz\nokqHv6xBc2sH/viWeD1xFJ54j14GSqskOE7nu/j3/3F67/ZNu+3/f/DlQ15HDB/89HuX4O1ppr5N\n2zrn+y8e08/f7MLmpQSrzpphSo6DXqe17/+Tijrk9k20j3q+9sKB9kWFmls68P25Zgzsm+h3Pig4\n3Y+aHfmqJuht2Ww2PP+PTzAiJwVTLsgMentqoLTrCCkXW/Tklwf+/AF2f3gSb+45bn+t6qwZX5+q\nx7Nvlrukd3wascNixRffnfP6iOKZumZsfedztLR14PMTdVj/pw9Qb+7VK+Dh4198dw6/fr4ML/zz\nE/trn1bU4XfbDuP3r/fctujubQCA3778AX7zxw/wvYi3GOzZtNl6noRwo6WtAy/88xOcOON9JsNj\nx8+irlGcKZKFaO+w4JV3v8CpGrPgz/z7g+/wx7c+RWVNk995FSNgmVs6cOCTM/gDW7ekcFarDf86\neAI153yvlyIWtugDJMWo+5y+RsVPX/tNZQN+98ohp9d+/XyZx/Sb/vIR5l81HHWNrfjoqxpUnG5E\ndJTn+uXGVz7EuaY2lH1yBuaWzvv/fy/9BiMHptjTtLRZ8FlFHYZlJ0Oj0eB8awfqGlvxTWVnC/Hg\np9/jhinNKD1aiaT4zpb7Vyd7bjusfqHM3jPRvRbBJxV1OPL1WVw2PstpFsTefvvnDxAXo8e9M8d6\nTNPt99uP4uNvzuK5ZdPsPQyOdn10CvuOncbhL2uw5Z6pbrdRU38em/96BNF6LZ5ZOs3nPsVQeqQK\nOw+dxMFPz+Cxu6cI+syr//7S/lnA97PiVqvNqcJF/vO1rkWHxer2uCN5ffD59/jre1/h3Q++w5/W\nXRmSfTLQK8i1Fw7Erg9PovzbOrmzIqreray29p5WrsVqxc8e3g0AyMkw4lzXPf3uIA8ApUcqnQYc\nPb7jKGrqW3DPTRdgdF4a1r10EDX1LRienWxPs/K5A37l8c9vd94nzkpPwLCcFLx1oAKj89KQnWF0\nSvd1pfsBiu0dVnxaUYuRA1PtF9ePv+lctKjqbLPbxxe7y6G5tQNN59thiIuyv3emrhmtbRb7NMlt\nHZ57Bhw1NLfhTG0zcjKMaG23wNhV0Tn2zVn8c38FfnHjGMT5GAxqbukMwI5rMVisVhz6vBr5uWmI\nj3X+vKeAY7PZUNfYilSHlR+7Pfzqh/jiZL2bTwXGsW5mtdqg1SrlAVj36s1teOfgCVw1KQeGuCi0\nd1hw7HgtRuelCQ7O3uJ896O008dnYe7lQ0XKtTCNzW04UH4G08ZlIkof2Gyi7/73O4zISUFWGD72\n291DWdsQul46VvdEkJYYI8p2NAB+OWucKNtSi+4gDwAVPrqwu9XUdz7//+hrR7Dgoffsf3924pzg\n/T77ZjmOfn3W5fV6c+f0wjve/wb3/eG/Tu85BrTulQgPf1mDc42t2L77K/x++1H8rfQbl8C37qWe\ngY9v7jmOA+WnXfb7P123Qj7+5iw+P1GHlc8ecNm/EGteKMOGrR9i4e/exy8e32N/ffNrR/DFd+dw\n4JMzPrfheGul+3uWHq7EM/9T7nRbpJu7SlVruwVvlZ3A0qf2Yd8x10GX/gT5DosVbx2oQG1Di8c0\njmF9y98+FrTdNS+W4ReP/8d3Qgn86a3P8FbZCdz92H9w4kwjXnvvazyx42P834EKwdvw1p7/pmv8\ng7tptaX2/D8+wV92fom3D37nM63VanO5lXfiTCP+svNLrHU4b6TWYbHig8++R2ubsCdAtr7zOTb9\n5aOA9rXvmOv5LzW26APkeGiakuNw1q/amYdTVNmNkLBS9skZlLkJes/9wzmQ3bn5fWy4fRKSDTFO\nF+EjX9UgMT4aj+84Cuw4an/97bITOHGmEZ/06pX5vq4ZfVLi8UZXQJ80qq/TRaX7McVHXxO+imGH\nxYr3PjyFohF9kGyIQYfF6tQKB4CT3zc5t4q6Ave5plYkJUTbb1O0d1gRpe+s97+591t78ts27kLx\nmH72QZaHv6rBzkMnMX18lsN3c73XeOfveqZvPvR5NS7M7xl06Xox9dwjoNFosOfjKmzf/TX2lZ/G\n+lvdLxHteLvlsIfBfVabzb7c9PnWDpyqdh6D8LfSr/HBZ9V44LaJ0Go1aG2z4C87v8CF+f0wdEAy\nms6345dP7sWN0wbh8sIBbvchRHuHBbWNPZWWv773FcxdtzG+rfLj1p2XJr2UU3V7UtfYCmN8lH2B\nrrP1vu9BL31qL2wAHv15MYDO8TOBVHCD9e8PTuK1XV/hwvy+uO2akT7Tv/fhqYD3JcftWbboRcAl\n4MNXa5sF927ZiwUPvYcd739jf72xuR1b33H/qF7vIA8Aa1866NTSrzjdiLcPnrD/3dJmsbeeHbk7\nttraLTj8VQ3e+/AUtu38Er/98wew2WxOT0g47rehuWcwo9UGfH/uPO7dshe3btyFA+Wncd8fDuKO\nR3ajpv48LFbXWwS9n6R45d0v8NWpelitNkEXLZsN9kWZgM7Kk6PzrRa8+u4XTvM2VJxuxK0bd+Hg\np2fslZdT1WZ8efIcDn3+PSpON+JPb3/msZXfe/GYfceqcNvGXfi6srMnoc3NXAz/3FeB07XN9rED\nb5VVoPRIFR565UMAwCff1qK9w4q/dI1HcPw9681t6LBY8fvtR7D34yoc/boGHRYrvjlVj/29WnB3\nPPI+vvu+Zx6L2oYWnOj6+/BXNTjX1IoNWw/h68p6p3EMjsdHzbnzeNnh+Ks43Yjfbz9iT2+RKNDv\nO1aF/93/rdNr/zlSiTf3Hscvn9yL3/65Z/xO6ZEqn/NonGtqsz+C29puwd2PydPD8tqurwCI19p+\nu+wEjn4d/NMkYmGLPkBsfEc2f59dbmu34taNu+x/3/9H51bLh19U467NrosY9b4N8H8HKvD67q+d\nXjvb0IptO7/yuO8lDl34TefbcdxhnIFjD8bru792M2uiew++fMh3oi42mw1/evszlB6pwobbJ7lN\n8+9DJ/HvQyftg/je+W9nt+9fdn6J4tE9vQEbtn7o9Ln3D1fi2aUXo/f4yfV/+gAvLr/E3tJ/4Z+f\nAgD+sfdbXFmUjdiYnnvH7/z3OxgdxkfY0FnB630PtcWhJ+JXT+9Dg7kNl47PQrReizf3fouEWD3M\nLR1ubwnl5zkvHOX4s57p1SPyv/sq8OXJenvQXHfLBCTE6fGrp/fjxxfn4aqJOfjVM/udPtN9PK14\nZj80GuCqSTn299a+WIb7FhRBq9Ggpa0DNhvs4zQ+q6jDw3/5CCvmFmDogGT40l2OV03MQWu7BXEx\neqcxOBVnGpFi7LmVufGVD7G5q7Xuy9l6z7dmAOBPb3+GvqnxKCnKdvv+t6cbsOdoFWZNH+IyzqH6\n3Hm8c/A73DA1z2WMib+DQh3PyWVP7cOtPxgBY3wU3j9ciaEDkjGwr9Feceg9KNWfJ1nExEAvAi+D\ntBVrcP8klwlwRg5MweRRffHi/34qU64im+MgxW6/dQioZ+qaXYJ8t3c/8H0/FOgZC+CO0CDvryMO\nge+BP3/gM/3v/noY5cc7p0+ub2rD/+73ft/6jkfex1Q3z853V6yWzLjA/trRr8+6BOJtO790+rut\n3YJ7ntjj9Nq/Dp7AX9/rqUx1jwt5u6ynV8ZxAGlvNT6CmCNrr1sZn5+osw/s3PH+N9jzsedWZ3PX\nLSDH4+RktRktrR2IjdHjrs2dE1zdN38CskwGbHuv87v/z57jWDZ7HDb95SMM7p+EKydm41S1GYOz\nkjrzZLM5NW7WvXQQp2rM+KWPp0/ONbXhvQ9P4tKCnls9r+/+Goa4KJQU9dz+ePeD7zDKzSqatQ0t\n9sGc73fN3Nkd6D/6shr7jp3GHdeNgl6nxW/+2HlsDR2QjKIRGU7beervx1BxpvOJnxmXDAbQM8D0\nmf9xfiy4sbkNH3xejSlj+kGv0zrd8tn818P2AcMAcLahBc//8xP7I6X/PnTSqZzu2vw+Wtos+PmP\nRqNgqAlrXvD8hJKUGOgDlNXHAHTd4xWr614Twn6CH07JRU6GES/88xP7hW/yqL4Ylcsla5Vq5bP+\nPUmgRN6CIdAZiLuDvD+8TQPrOPWzEMt7tZYBOAX5QPyt9Bvfibrs6nX/t+JMk9OTOGdq/Z/zYd+x\n0/ZHIAG43Af/tKIOHRYrPq2ow6cVdfjHvm9dtnHNhQPt/+9umf7ur4dd0vWeR2HrO1/gZLUZGgA3\nXDrEPtbFsSX9l39/idx+rpNWLX2qc9Euxytj92ODT+zoHHR5+6bduH9Bkf39XR+ewvhhJui0Wphb\n2lF1thn15s48dT9R8lZZBbbvcl9pfuZ/yu3l0dZuwY73v8GF+X1x1aQcHHNzbPYec+IYDrp7gYQO\nEJWKxubrYUwVqq4Wd7CDyWR02WaHxWq/JzpsQLJ9elYhYqN1Tt2A3e6+cQzGDk7HgofeCyq/Qvxy\n1liMGpiKx18/ah+89OLyS9BhseKOR1y7kImIlEKjCayBVTymH6YXZLncOhPC03U7GP/43Q9Fi1cm\nk9HjexyMFyC9Tmu/p+Vv171Sa1YajQZReh2eutf95C1EREoQaPN0z9GqgII8ANGDfCgx0IvA34NO\n6bf0Y6N5R4eIKFww0EeqrspJGN65ISIiBwz0IhBr1L3SW/pERKQ+DPQi8LdRzDZ0YIpG9JE7C0RE\nqsNALwdPM+CySe9Vn5R4ubNARKQ6DPQiEDtAXzEh8Hm0hbJ11TbU1LvAehARkf8Y6EUg9ni2xITo\noD7vbpYwIiKKTAz0ihJcm/Wu6/MxIicFPyzOFSk/yhIbHdja1mLqkxLn8b2RA1NCmBMiImEY6EWg\nlHvr44amY9nscfblRoNx3/wJIuRImMmjMnwnQudywHIb3L9z7m93v7nQnh3HqUSJiKTGQC8C/0fd\nu/9Ad/AI9tl2MSoe2Rmep1MU28+uHSUonUYDt/NhK4XQ3y0zLR5XTXS/AhcRuRqdlyZ3FlSNgT5S\nqWkUXheNRiP7BD8K6bwhCtp0hxXllO7yCerJqxIx0ItAKV33oVz9Ti6yT+TXXcRu8iE0bxqNhjUG\nIn/Ifd6rHAO9CMRbpjb025E9cPpJ/ha959K9YHC6sG0wyJMSBHgc/sZhSVhSB5+B3mKxYOXKlZg1\naxbmzp2LEydOoKKiArNnz8acOXOwbt06WK1WAMCWLVtw4403YtasWTh69CgASJZW1aSKVREQQEYP\nku9e3Z3X53t9/4oi4fMf9FHAwEKShmpmcAzwOpTVxyBuPkhyPgP9rl27AADbtm3D3XffjQ0bNmDD\nhg1YsmQJXn31VdhsNuzcuRPl5eU4ePAgtm/fjs2bN+P+++8HAMnSKkkwy9ROHOkw4jyEgdpm/1dd\nTfrrp8j36OCE4X28/kZagQfCgD4GFI/pJ1KuSGkW/tB7hVAxVNIwSDYIn1dkWNfS4eTM53qkl112\nGaZNmwYAqKysRHp6Onbv3o2ios7um6lTp2Lv3r3Izc1FcXExNBoNMjMzYbFYUFtbi/LycknSpqam\nSlQk/gtmmdp+aVJM66qSMzgAOq28d5u83KIXrF9aQsCfTTZE41xTWxB7p3Ch02pgsQZ+JKrhKvG7\nRRchKSEan1TUCkr/y1ljcfum3dJmSoUELTyu1+uxfPlyvPvuu3j88cexa9euzgFFABISEtDY2Iim\npiYkJ/fUprpft9lskqT1FuhTUuKh14s7uYrJ5Pq4WVRU5z6i/J3IRaNBfKwezS0d6J/R87hYUlI8\nTCYjEhJiAspjH5MRGo0G5vPtLu/pdVp0WKz2vxMT42AyGREd1XMIuPuOUhuVlyZ4v915lovJZERc\nnOfWhdC8BfMdxgwxofSjUwF/nqQXqmO081oZeKCPiw9sBs5QnoOmdANSEmNx6lyLoPTp6fJdHwIV\nivIUFOgBYOPGjVi6dCluuukmtLa22l83m81ITEyEwWCA2Wx2et1oNELr0AITM603dXXNQr+WICaT\nEdXVjS6vt7dbAAAdXf8KZrPh/vkT8eXJc0h36JZqqD+P6upGNDW1evmwZ9XVjdBoNGhu6XB5b9rY\nTPz70En73/Vd+2pt63D6fKgN6ue+bN1paDgvSx67VVc3oqXFc2taaN6C+Q4tLa6VOFKW0B2jwd12\na24OrGdI6u/n2Gt19mwTOlrbUX9O2DW9pka+60OgxCpPbxUGn/2gb7zxBp599lkAQFxcHDQaDfLz\n81FWVgYAKC0tRWFhIQoKCrBnzx5YrVZUVlbCarUiNTUVI0eOlCStkgSyTG1aUiwmjeorSX78GTOQ\nlhgb1L6CnfZV40dm1faEADnT65T/kM/QrCS5sxDx/LkmkDA+W/RXXHEFVq5ciblz56KjowOrVq3C\noEGDsGbNGmzevBl5eXkoKSmBTqdDYWEhZs6cCavVirVr1wIAli9fLklaNfMYsII8vr2dIJ52ec3k\nHJQeqQx4n4tuGI1Fj5YG/Hn14UUonPVLT8AXJ+vlzoZAwXXd80iOHD4DfXx8PB577DGX17du3ery\n2uLFi7F48WKn13JzcyVJqyRiV0DzMkMxzWvnBaJ7nEGgIq3yHWnfN9Ko6+dlFxcJo/y+NBUQu0t5\nUKYE3YcSXROCnY1PXRdW+fM7fphKntEmEgNr1qJgoFcQOQ5pnkbqkuFlmVyKNMHe6xMnF6R8DPQi\nEL3SqaYTMMi86kVYUjekJGhhLLphtOjbJAonQjsk2QHgnsqusuHCwzK1Im3d7VrpEvXdB5vn+FjB\nT3gqghTXkfHDTBJslYioEwO9yvz0ymGibIePqvkny9Q5v3f3POZXT8qRMztOLhufhUmOUymTLJbN\nGovbrhkhdzaIXDDQq8yg/sp6zjdSusqmjcsEAAzLTsHTv7wYl08QvoCNmNxV0JKNMbj9ulGS73v5\nnHGS70PNRgxMxYX5XMOAlIeBXkFSkzonr3GMnY//Ygp+v7jY/reQuOpuJLynBnzwDXvfOUqQsHt+\nxrRBkm3bk5ggH0lUq4xUKdZloMCFf7ecpte/FBgG+iDMnj4EfZLjMPPSIX59zlO3eUaK64XUEBeF\nxITA5qRWCm+z7wV7AqcGObNfwBR074MXwdDpkxyniNs2N4iwimOwj8ZKJVJ6CUOJgT4IOX2NeGjh\nZOT0NeJ3iy4KzU5FOgu6w1SwW4uUkzJCvib5opF3qeRIJXjUPc9UtxjoRZJiDGzFOUm4O9Y9nCnK\naZf6ZoyPcnlNrKcJ+psCXzo2lNx+X17bQkoRlVtFZILUgoFegcLtHO4jwiQvCbF6DJFwwZFBIZl2\nOHDrbpmAJ++ZKmsewuywVL1g7x5JdZ25vFCegaqhdMPUPLmz4BcGepUJ9NyUaB0dQReLm68cHuRe\ngItG95N3Vate+/bnGpvopifCX4kJ0YiL0bu9uLO7MrTCpbylGmai04k4LbZCi7pgSLrcWfALA30Y\n8ufcCPZcF3LRM8R5DnSRsCTlBYPT8eDtk7ymWX9rUYhyQ+FAg/Dr+eumptuJasFALwNFDNhWQh4U\nRdqrZpSPtdj7d03I0216QRY2Lpxs/ztcL+qqpJDfQhHXEaVRyG+jNAz0iiTB0SrVVSFCTqxgvmZA\nQVoDmJJdxzbotBFS4CQ5qSqPQd8OdPMaKzXBYaCnoMgRdh51mEBILlJfeGKj3U/KM6CPwe3rRErB\nmKw8DPQRZHReml/pB/X3PRJdo9HgsvFZgWYpIEkSTCAkdde4v48B9u7qt88Qxj582fEXkBpLWGwM\n9CK6amK2KNvxdi0Xcp33FFKWzBjjkMZ34Fn1k/FY89NCn+nmXD7UZ5q7rs/3mcabgqHqXeHNZgOM\ncZ2Vk+wM8VvkjP2RJ1J+c8V+TZX9AOpaI1ThCoaZ8FbZCZ/ppFoy1vs+/W8NajQa6H0MIhNqVG5q\nwJ/9xY1jMHRAsij58MRnV3yQ53VMtA6P/2IK4mJ6uuR/csVQdHRYJdkfSUej0WD+VcPxh7c+kzsr\nYcr1ZFRZXFUctugldv8CPjYVrGiVLyLTfZEyxEVBp+055S4tyMIVRcH3AsXFsL4eat0LUJEUAo/q\nrA+4xyuExGQZPCVih4FN4lFnwlbjCwX/vqenwXKScdOkueemC/BpRR0uzO8rWx5IPr5OzcT4KDQ0\nt4cmMyJyd5hx1H1w2KIXkZJnzHI5UQSeOOF8gl130UCHv/z77fxpRQdUhgI+0y81HjddMli02yvk\nhzA4L1gWbbzjAAAgAElEQVRvixy8QohoYF+jsIRBXCQ46lqY264Z4TPNuCHqHeAnFa45rx6RcClQ\n6vVOmbnyjIFeRFqBk5kooTEgNA+eBg5mmQx47O7QPM/ua+7s3lPs3nLVcFyY38/PvXgvEW85GD0o\nHT++uHORi9mXDXH9rMCrghrm+CYi9eE9ehlIfQ2P0rurvwVWvejd7Tx7+hBMnzQQWosloO25EFAY\ng/p7X7Wud0Vg6gWZgnYttGLmy+UTs5Gf3flUwJQx/fCXf38Z0HZuvGQQtu/6WpQ8kcRkrogptaVL\nysQWvcoIOb21Wg2evvdivLD8EiQb3E8uE+hlIjZGF9Lu3UH9E6F1c1HLyei8TZKREnhesiRYg16q\ncRputyrBrhg+BJK5Wy7yfidhBc4KkHts0UtoekFgM8Z5G7zlvrXuKqZrVPg1Fw7E1ne+wMSRziOz\nBXfdy3hBe3RxMQxx7g/RZbPH4WR1E3L7JQZ80XO8KPj6nkIvIO5udYTzgEaSj6/jSq7DjqFWeRjo\nJTT3CvczxgVyAt43fwKOVzUgNdG/53cvLcjClDH9EKUP7HEwOSb36eZtqtv4WL19Eh3GUYo0Sj7m\nlZy3SMWu+xC4YsIAv9LrdRr0S4vHZYU9PQLZGUZcPLZ/QPt3F+S7WwO+lk+VujUaikcS3XX9i0nj\n9H/XfYmyezaTyIGQ+S18HTJKfhxY6dR2h4AtegXSaDT47c8mhWRf8bHqPwR8nXPZGQZ8e7rRaxpF\ntEK8ZMLY68kCOajs2kZEXdiil5nia4aKiIDehSKLci8PG6pBRoo/HhVCzltaQOfxIHcepOL+GOSB\nGQz1N+dULstkwLgh6SgakSF3VtyyBtl3P3mUMr+Xv3L7+V6yVwrTxvXH+dYOt+9J0fXKgYPqEHlh\njwdmMBjoZabVarD4x2N8JxSdsBPH5f62n+ebKTnO5bV7b7oAm1874t+GvPB10UuUYP36ULm5ZJjc\nWSAilWPXPXmVlyl+SzY/L03U7fmqe/z0yuGi7s8bdn1TyERAI5fnkzgY6FVo9c2FKCnybyR/oLRa\nDdIdl+T0ceIleZigR04pxhhpd+BwNQqXJXWJKHww0Itsjpu5zsWWl5mIS8YF9qid1IpH+zfHvBiB\nRemxife9SRLBHvhKP3FINAz0IvNn+dJgpCbGIikhGtMCDPiBBp+4aA7rkArrA+SPYAdjMs5HDl61\nQ0CKFp1ep8Wji4thbmnH7o9Oib8DDwqGKm9pVzGK1xgfhQvz+2LkwBR89GUNDn1eLcJWOwW0eh2j\nPvkQ7ON1Sj3E3FVg2CsWHAZ6kbkbZR4uLszvK9qKb0qjgQa3XTMSAJAQGxVUoM/tl4jjVQ1+f87f\npwN4Pz2CRchvHyFfU3LsuhdZdoa8E6sold7HVLtq1vti9MuZF+Demy4IbmCiEq9wSsyTXJTQwgxy\nURv+nJGDLfoQGD/MhHc/+E6SbaulSytKr8WqeeORYpB4BLwCxMdGIT8vjRdSko5NGXWNSKW25XDD\nt5mlIEMHJOPpey+WOxsBE6syMbh/EtKS/Ft9Tw5yncMqu3aQwqn2cFJtxpWLgT5EuteHVzMh558x\nXnnP0ftLLb0kFMHCOBi6+2qsBAfHa9d9e3s7Vq1ahVOnTqGtrQ133nkn+vbti4ULF2LgwIEAgNmz\nZ+Pqq6/Gli1bsHv3buj1eqxatQpjxoxBRUUFVqxYAY1GgyFDhmDdunXQarVBpyXlumRcf2zb+aXg\n9OFwAov1HfIyk8TZEElO7rqgKOscqOjcY+U7OF4D/Ztvvonk5GRs2rQJdXV1uOGGG7Bo0SLMnz8f\nCxYssKcrLy/HwYMHsX37dlRVVWHx4sXYsWMHNmzYgCVLlmDixIlYu3Ytdu7ciczMzKDTknJF6YV1\nEmWkxuNMbTPSEpXXlS925UPoRapvary4O1aYcKjUKUW4rlwH9K5E8aARg9dAf+WVV6KkpMT+t06n\nw7Fjx3D8+HHs3LkTOTk5WLVqFQ4dOoTi4mJoNBpkZmbCYrGgtrYW5eXlKCoqAgBMnToVe/fuRW5u\nbtBpU1NTJSySyODfZUL8i8rKnxTgq5P1GJadEvS2ArkUjMgJfr/kH7bKSBAeJ6LzGugTEhIAAE1N\nTbj77ruxZMkStLW1YcaMGcjPz8fTTz+NJ598EkajEcnJyU6fa2xshM1ms49O7H6tqakp6LS+An1K\nSjz0enHviZtMRkHpWhyWFPX2GaHb8yXG3BbQNhMTY+3pb756BOJj9B4/r+16NC42LsqexlPa3q+n\npya4TWsCMCjH++I2Qr9P72f7HT+X4JBnRw8uKnZ65C8+PtqeLqmm2e+8uHtfq+3cflxctN+/d0JC\njM/PpKUZkC7yvA16Lz0y6WnBPTqqhhZ9bJz3MSY6nRYmkxHJZ8+7vCfWOS2EISEGGo33ylP38edJ\nvI/v6omv7xnodrvpHI7B9HQDEuKikFRtFiVvYklLTRBtW6HIs8/H66qqqrBo0SLMmTMH1157LRoa\nGpCY2Lmi2eWXX47169dj+vTpMJt7fgiz2Qyj0eh0oJnNZiQmJsJgMASd1pe6OtcLdTBMJiOqqxsF\npW1ts9j/7+0zQrfnS9P59oC22WxutaefNqaf189bLVYAQEtLO6qrG72Wh+PrlxVmoWhoesDfVejn\nLFbnq133555bNs3jdupqnS8ccVFae7r6eteLuLe8aKBx+77V2l1ubX6Xgdnh9/Hk7Nkm2Nrdr1Uf\nqI4Oq8f3as42BbVtNbToW863eX3fYrGiuroR9fWu1xixzmkhmsytPsuz+/jzpNnHd/XE1/cMdLvd\nLJaefNfUNKE5Vu/2nAwkb2KprRVW8RBCrDx7qzB4rfLV1NRgwYIFWLZsGW688UYAwK233oqjR48C\nAPbv349Ro0ahoKAAe/bsgdVqRWVlJaxWK1JTUzFy5EiUlZUBAEpLS1FYWChKWuqREKvHpFEZmH+V\nsKVYfz1vPC4Z1x8XDEoXvI95JcOg12lwxYRsv/I257KhIVnNzRAb5fZ1vU4reKKeqRc4LsYjLCJl\nmTpr9Rlp4X1vvZsKGuQho4I6i2rxOBOf1xb9M888g4aGBjz11FN46qmnAAArVqzAgw8+iKioKKSn\np2P9+vUwGAwoLCzEzJkzYbVasXbtWgDA8uXLsWbNGmzevBl5eXkoKSmBTqcLOq3a9RMxMGg0Gtx+\n7SjB6Qf1T8Kg/v6N7h4zKB3PLbtEcPqls8YiKYQT41w/JQ9P/v3joLah89HN6c7yuQU4caYJQ7NT\nQtqak5LaJgKJVGKMuhdl5L47wdaCHI7B7v+yYhUcr4F+9erVWL16tcvr27Ztc3lt8eLFWLx4sdNr\nubm52Lp1q+hp1Wr8UBMOfVENbZhfTEcODG2viyFO7Akehf0+CbFRYTeoz6aG/nUphfm5SZGJE+aE\nEq8hJCK2viUQ6RUdMfCwVBwGeiKVEjpnAYlP7vpAOD9Hrwoqq8xwURsZ8BRVNkU0lL0cJA/ePglV\nZ80wxLkfhNhblsmAk9XBjZgnCpUwXQlbVmwSiM3LQRrOx+/vFxfjwdsnyZ0NUYjVWptXMgwAMH38\nAHE22KVvajzGDTEJTv+bW4sEp5XydoAiKlBhQuhAuhSj+laLHJKV7PIaD53gMNCTKBITooOawnXd\nLROQnRHchCxKM26ICS+tuBQD+gTwvXhlIxFsXDjZ43tKrXi5a9GzFzQ4DPSkCDl9jbhvvvCWZ6Cu\nvXAgNt15odc0Sr0Aqp3c97XFpJZjRC35JGkx0FNEyepjQFqS8hbSIXVRQ6XlgsHp0j0rH+HUVqoM\n9DKI+GeVSbG8XcD4OJ96jBmUhp9cMUzubASGx5noGOhDiQewbEbndS6g0z1trTf8lUjthmQlqfbx\nS55/4uPjdRQRFv94NM41tSI9yfdqb5Hc3xLJ3x0AK+MKw59DHAz0IcRjVj56ndZtkL/z+nzowvjB\n3VnTh8idhTAV8VUiz6QoGhZ3UBjoReYtZPBYVZ4Jw/u4vBZOYb+/gFsV5CCCxs+wtRw51HkTh4hc\n3FwS/OArXvvVQYwgLVmdJhIOIpXVkhjoQ0hdhwapzbRx/fHicuHLCYeayq6NRGGDgV4GEdQ7GLC7\nfzxG7iyoEh+BCw25z2Ex9q/YQ0Xj+F+lZlJdGOhDSLEnlgJlpPoeHU8UqXgtIX8w0BNRSMjdChaT\nagKtl3yG0+9B3jHQy4Dnl2cZXQvjJCZEy5yTCKWWAEZBU01lhYLGx+tIUe6fPwENzW1IiBW21rok\nZLwARkdp0dZuhV7HOjhJjZE+UGorOV5NRMZacnCio3SCZq/zpGiE63PxarJibgEKh5lwybj+8mRA\nRd1NhW7mQAiawBOY3d7S4SVUfGzRU1i5/bpRcmchKAP7JuKuG0bLnQ1VyEyLlzsLsomYYBgxX1Ra\nbNFTWNGyS4VIXhL0dtjU1NWkQAz0IhPUpcd+P1KqSK8n8dykMMRAH0KczIRCTkFxK5wOf7mLVe79\nh4PJo/rKnYWQYaCXAU9SZeNsXKRGw7OT5c6CKEJ1/qUmxgT8WbVdIRjoQ0htBwcRKVPva8mWJVOw\ndPY4WfLiQsQLHa+Z4mCgJ+olI6Xz8b5B/RNlzokIxLzo8qqrWPGxUW4HovInI4CBnshFenIcHlo4\nGb+aXQAAyEyPnDXdpQwMYo9zk3PMS3gEUN5EjBR8jp7IjT7Jna3655ZNg1YbHpd1Eg9DJKkJA70c\neJVQjUibilZNh6Yk1S+13J9QSz5JESLrKhYCXi+UPDcp1NQUuYnUQmXXcgZ6IiK1CeeJfRyCKDsu\nxMFALwNO5yiuuBjegRILr6sC8RT2jGWjOLxChhAvotLIzjDi5pJhGBYmE4aQjNTSUg7jpq7bb6aS\nn0WpGOgpLEyTa1lXpVNQPAjj2BRyCbG8dJNw7LonIkEUF6cVl6HQEaPSpJbOCwoeAz0ROYjg6El+\nGzUwxev7y+eMw8MLJwexB2Uej73n4x+clSRTToRhoJcBa9JEwVNmCAgNoQu/SD174NghJjc77fnv\noP5JSO+afCqcKX1BIQZ6sfFBelISBVUqWcFVFlWNmVB8XpWdQQZ6InLAaKwGYjyiq6qKl5ryqkAM\n9CGkqho0qRYPsyBEyEma288odxbCyogc72MV5MZAT0QOIiPQBUvuSa+E3qP35IJB6SLlRAIqOAR7\n1weVHui9PozZ3t6OVatW4dSpU2hra8Odd96JwYMHY8WKFdBoNBgyZAjWrVsHrVaLLVu2YPfu3dDr\n9Vi1ahXGjBmDiooKSdIqmpeDdFRuKvYdO42iERmhyw9RuIqQ1rc/phdkYeeHJzEsWxmBJ5CfyLES\nw59YHF4D/Ztvvonk5GRs2rQJdXV1uOGGGzB8+HAsWbIEEydOxNq1a7Fz505kZmbi4MGD2L59O6qq\nqrB48WLs2LEDGzZskCStksVE6XDVpGzk9k10eW/SyAwM7GtERmq8DDmjiMQLZUSZc/kQXHPRQCQl\nRIdkfwzE6uA10F955ZUoKSmx/63T6VBeXo6ioiIAwNSpU7F3717k5uaiuLgYGo0GmZmZsFgsqK2t\nlSxtamqqVOUhihnTBrt9XaPRoF9aQohzQ5FGqeOWVBEUVDVCzZVGowlZkAfUPV2tKo5HkXgN9AkJ\nnUGpqakJd999N5YsWYKNGzfan81MSEhAY2MjmpqakJyc7PS5xsZG2Gw2SdL6CvQpKfHQ63X+lINP\nJhMHrzhieThTankkJcb5lTe93vOwnfT04L6j2NfVBAkCWmyc923qdFqYTEYknWlyeS+Ux0BiYmxQ\n+45PiEF7hyWgfTvuy2CMdXk/Lr6nDE3pRuh0/g0Fi4uPsv8/Pd2IKL0WxtOu5e0rb77Ex8f4lS9H\nqamGgPfbWyiOG58TJldVVWHRokWYM2cOrr32WmzatMn+ntlsRmJiIgwGA8xms9PrRqMRWq1WkrS+\n1NU1+0zjD5PJiOrqRlG3qWYsD2dKLo/6+vN+5c1isXp8r6ZG2MXWE7Ebes3mNpG3CLSc975Ni8WK\n6upG1Nefd3kvlMdAQ2NLUPtuNrei3ctv7Y3jvhp75QMAzjf3lGF1TSN0Wv8CfbPD52tqGqHXadHQ\n4FrevvLmez+tfuXLUW2t87kQzG8v1nHjrcLg9ReoqanBggULsGzZMtx4440AgJEjR6KsrAwAUFpa\nisLCQhQUFGDPnj2wWq2orKyE1WpFamqqZGmJyLNIWfAkkrpeXdiAi/L7yp0Ln4J9OkBKKr9L4xev\nV4RnnnkGDQ0NeOqpp/DUU08BAH7961/jgQcewObNm5GXl4eSkhLodDoUFhZi5syZsFqtWLt2LQBg\n+fLlWLNmjehpicizX80pwLqXDsqdjfCmgPhVPKYf9h47LXc2RKfkyoFaeQ30q1evxurVq11e37p1\nq8trixcvxuLFi51ey83NlSQtEXk2oI/BdyIPpGzlRFILKlK4DckqidOR1CPECXOISJUkqTgIvfrL\nXWmJsGVqIygmS4KBnojs1NTKUVNeSTh3v2uwdZIn75mKohF9gtyKejHQE4UzBkPyICQVpSD3IVYe\n42L0ki/Zq2QM9EThTEXdsxRCkRvzRKG2SgMDPREJEuy1TWXXxrAnyj16/qiqwEBPRNRNYPRjR0kn\nNU+BG0ndGgz0RGQXOZc+EoOvmC7W8aSmJwSUiIGeiFRJbfdJlSY2Wtz1QJQuJipyw13kfnMiUrVI\nDvNifPfLCrNE2Iq0xJwlr3h0Zq9XIqebgIGeKAKkda12ZoiL8pFSOmJ3vyr5Mp2ToczVDB3FRusV\nuS6CVB01UV5WZgx3kfvNiSLIuvkTsGJuAfqlxcudFWUTKcosmz1OlO0oXST3qqgJAz1RBDDERWHo\ngGS5s6EKN04bFPQ24mP1yA5izYFwoexxFErOm7gY6IlIlaS6TF89KQfXT8kNejsDMiI00EdO/FQN\nBnoiIikoeRBBF2W3uMFKg0gY6IkoJJQeU/zB57qlw/XoxcdAT0Q9eI1VjGwJu/67f2ZbsDWWCD1e\n1FZpZaAnIuomZlM9yGBw94/HSLbtcPDwwslyZ0E1GOiJSJ0Y7JQpyLqSTeAG0pPjgtpPJB0+DPRE\nYSi3X+eELaYUPy+Garr3LGteBexcTWWpeCzMYChvWiQiCtryOQU429CCPr1aPRcMTseXJ+tlypWK\nREhcUfqoeylzFyE/MQC26InCUnSUDv3SElxev3JiNu6bP8HzByW8sqpipLrgwCdfgNRpNcjPTZNt\n/46kKAWOuhcfW/REEUSr0SBbBfOwhwWJ4tVT905FlD6yVp6TQjA/j9qqImzRE0WgJ5ZMUd3FyoWs\nX0C+7gkGefIXAz1RBEqIjUJcjH8degq/nas8arhVQRGBgZ6IQoIVhQgh4u+s9MGCasFAT0R2vgZC\nJcbLt549KQ8DsTow0BNFqEB6ln8swhKuEUPJMVCCAB2tFymcKLncuvlZfoP7J0mUEWEY6IlIlfx5\nDGv+VcOFJRT4DKCUjwqOH2oCABgjtfckDMc2FA4zYXCWfMGej9cRRSh3YbL39KPZfQw48X1TaDLk\nJ6FTpQJwO6eAUt11Qz4sVhv0OpW2w1QSqEN610HmWxwM9ETkmRq6USUgSqwKcCMajQZ6nQoLXuFZ\nTkuKlTsLslFplZGIpBDxs5JxcJkiiXGrxBDnfCsk2G3OuWxIcBsIIQZ6IgoJWafAlSN+s84QGJHL\nbfKojKC3MdTN/fXEhOigtxsqDPREJJiSWvxKygspU1piDH527SjRt6u2I4/36IlIEA00fg2AC2cs\nBe9+ffN41DW0yp0N6sIWPRH1CKCpcstVw7F01ljfmw6iGTR+mCnwDwPyROYIrg0MykxC4fA+cmcD\n3g7oUA7HkLsHgIGeiIIy9YJMjByYKuk+LhrdT9LtC3XZ+CwAgE0Va+6GGIvEI7mLhoGeKELdfl3n\nvcsri7JlzolvcreIAGDdLRNwWeEA4R8IINPrbpng/4eCIEmrVgk/lp3cIVYZGOiJItSYQWl4acWl\nmDgy+FHJjuL9XBUvJEQIPrHR0i8P29+knol9pKKoekKYYKAnIlE9vmSK6NsMWbssQrrk5Qymuf2M\nXt9XxS+gstoIAz0RSa73ZCVK5+sevKD6gAoiliT1Gh/b/PHFkbcwktz1AgZ6IvIo2GfVDXFRmHXp\nYKz+aaFIOQqQ0IDGmfECF4ZFp9OGx5dioCciJ88unYb8XHFG0Ws1wBVF2eiTHCfK9lQlPGKEV+H8\nFTNSwueYZaAnIidReq37lkyQV/VguonDOaA4CnWHAjswPLvrhtFyZ0E0ggL9kSNHMG/ePABAeXk5\npkyZgnnz5mHevHn4v//7PwDAli1bcOONN2LWrFk4evQoAKCiogKzZ8/GnDlzsG7dOlitVlHSEpE8\nVDvtrMjZDmaGwEGZiZ63q4L7+lJTyhEm9gqCcn4vn8/BPP/883jzzTcRF9fZjfHJJ59g/vz5WLBg\ngT1NeXk5Dh48iO3bt6OqqgqLFy/Gjh07sGHDBixZsgQTJ07E2rVrsXPnTmRmZgadloik4etiJCTA\nKeVCLTsPRZWRGo+vKxtCmxcSldqOcZ8t+uzsbDzxxBP2v48dO4bdu3dj7ty5WLVqFZqamnDo0CEU\nFxdDo9EgMzMTFosFtbW1KC8vR1FREQBg6tSp2LdvnyhpiUh9pOgmzs4wiL9RIdjyDsr8q4eHaE/S\nhWS/el9krhn4bNGXlJTg5MmT9r/HjBmDGTNmID8/H08//TSefPJJGI1GJCcn29MkJCSgsbERNpsN\nmq6zu/u1pqamoNOmpnofKJSSEg+9XtzJLUwm789+RhqWhzM1l0dDq8X+f32UDiaTEdFdk97oo3ra\nAunpBiQa610+3/u7O/6t1WrtfwdzrUtKch0YZTTGQK8Tdp4nJ8ULShcXGwWTyYiEhBiX91JTE2Ay\nGbr2fc7l/d7lEOvhkcLYWM+PGppMRuh13ttf3fs57bBojL/HX0JCDEwmI7R+jipf9pPxTvsyGmPt\n/+++fsfFR3vNT3qaARdPyMF104bghl/9w+X9+Pie5V+7t5NY2eiSzt0+er+m1Wk85sXdb+woNbVn\n8qKoaOdQmZZmQGLtea95cWQwxHiMSaG4dvg9hdXll1+OxMRE+//Xr1+P6dOnw2w229OYzWYYjUZo\ntVqn1xITE2EwGIJO60tdXbO/X8srk8mI6mrXAy1SsTycqb08HM+XjnYLqqsb0dba0fW31f5eTU0T\nGhpbXD7f+7s7/m21Wu1/B9MIrq8/7/LauLw0vHugQtDnz50Tdk0439KO6upGmM2uK6/V1poR1fUt\nhJRDS0u72314er17G74Cffd+6h2+k7/Hn9nciurqRtiswn+VyaMyMCIryWlfjQ7l0D33wPnmNo/5\niYvRI90QherqRnRYrG7TNDe32f/fvZ2GBtff390+er9mtVg95sXdb+yotrYn9gxIT8D5lnZ803XL\n5ezZJjQ09Hx3X+Xf1NiK9g6L2/fEunZ4qzD4Per+1ltvtQ+K279/P0aNGoWCggLs2bMHVqsVlZWV\nsFqtSE1NxciRI1FWVgYAKC0tRWFhoShpiUga+XlpcmfBLXfd/vGxoZ1qN8GxhR5ErWXYgGSkJXpv\nTaqSgDK544bR0Pq6h6PAG+A6nQarbxZ3LgixB/t53Ze/H7jvvvuwfv16REVFIT09HevXr4fBYEBh\nYSFmzpwJq9WKtWvXAgCWL1+ONWvWYPPmzcjLy0NJSQl0Ol3QaYlIGj8sHih3FtwKejS6CNdUsWb3\ni4nWYdNdF2HBQ+/5TDtuSDo++rLG/vctV4lzb1vSx+qC3baA3/rmkmFB7kR+memhW9dAUKDPysrC\na6+9BgAYNWoUtm3b5pJm8eLFWLx4sdNrubm52Lp1q+hpiUgauq5baBxr5l3vpw/6pgobA+Cvn/9o\nNFrbLbhrcykAINUoTU/A6Lw0pCXGYPfhSkm2L7Zp4/rLun+NvzUlN8mj9KGbxoYT5hBFuARvXeAK\n7EZVogd+NlGS7Wo0GsRGS3eLoru6kpYUi5uvDKC3wN3x4aWW6BgffXbhS8zvYC2ym0tC9eRBAF33\nRBRe0pPjsOiG0T4fVVN1zJe4iyJUQUvRPS1+FoG/I/7F5mvhIrE5fltDXBQG9Ando6Fs0RMRxg8z\nweRjPvpgY1moZ30bnp3sO5EHUgZuuVuSjvzLiXz5lvrQkW0+hhBhoCcigeS70Ic6Nk4vzPL6Pqeq\nFUdcjJtOZRkOs3W3TAjp/kJ9PDPQE5EgQi9OnlqssjZk/dx3gpdJbYQa3D8p6G0oS0/tpvu3lHL8\nQCh1joXwb5I1f9Y7kLsPh4GeiEgCUy7IFH2bcgeMbvfPL8JVE7MxfqhJ7qyIZt1851Z9siHG/khl\ndJS4M62GGgM9Eclq6IDA76V7E6p74T+90v0z3XKPKveLn/cisvoYMOOSwbIPqAtG7+MjIyUeBV0V\nl/SkWMTF6LF01lhcNLovLhvv/VaOEHLe7WGgJyJBlBi3vF08nUZVu0mYmBDt+qLQ/Tpsb/ywPh7T\nvfHwtS6viVWxyeh6dn9UbhCzhSrxR1WA7m78/iYDbv3BSPdjCfzQ+/AL9RiP8LjBQkSSU+1a9B4s\nmzUWpuQ4LPzd+/bXxP6GOod56+ddMRTTxvUXrach2RCDx+4uFmU8ATkLt7GWbNETkaykqj44tcLc\n7USjcbr3mmKMwbUXDRS0bX8GYjnuT+zbCcb4aJ/d58vnjMP628SY0MdH3mWIjo8uLg79TrukOqzc\n569Qd6SwRU9EHmk8/qF8V03KcZon3pMovRbtHVasvWUCkoLozleqYdkpod1hCI8TqX4vX19Bo+m8\nBXPX9fkYkuX76QqNgG1KiYGeiARRWZxHnMDHpR5eOBln6s77FTSSHNYyD/UqeqKS4max100q5yjK\nMnVOkpMfxBiHwuGex2c4knuSJBUfoUQUSu4uVtdPyQ16u56eXzbERaHpvOe124PV/W2SDDFIMvi3\nWB+2XYIAABAWSURBVMzovFTcfOUwjMlLE3V0vapG6vcWUNb9r2hsuGNSIDty0TctHg8vnIxkiRYK\nUhLeoyeigLy4/BJcd5HwQH/xWPfPlef0Nbp9PS5GxGeXRW64ajQaTBvbH6mJgd+ndSfkj6upsGLR\nx8dUzdl+zCGfnhwHvcOASanmv5e7mBnoiUiwCcP7YEhWEu6+cYxLC3/RDaMxx8uKXLOmD8GKuQX2\nv/PzUnHf/AkCuzX9v1IqbeS0GNd6pX0nKbh7usOfADz7siGi5CKcMNATkYvrLsqFTqvBjEsGO70e\nG63Hyp+Mx9jB6S6fGT/MhNlXuJ88Bujslk5N7OkmvfemscjOcN+aF114XbdVZ/ywzoloBgkYuCam\n64qDv7Ukht6HX6gPR96jJyIXeZmJeP5Xl8idDUnpZJ7VrX96Ak7VmGXNQ29zLx+KV979wu17wXQ/\n3/nDfJxrakVO30RUVzf6TH9JQX98W+U7nS9Txog/DTEQQKCWue+eLXoiijhXTcxGnxTv93rFktbV\ni9F7VP+anxZi5MCeR99yBPRuBBMuLino7zPNdC9TvQZz+1qr1bgdz5CZluA2/bwrhmHNTwsD36EA\noQy9Gsg7Xz4DPRGJbsygNADAjy8e5DNtMAOgAv3ojEsGh+yRp5U/GY95JcMwdojz7Y7oKB1MPgaW\niWmel9sqct37X/Sj0ZgxbRD6pcUL/kwofjfRd6EBflriufylxq57IhLdkhkXwGq1BT2KfEAfA777\nvgmm5CBHtwcQyeJi9Djf2hHcfgGkJsbiknG+W9NykHvoQrIhBldNysHFYzPx6r+/xL5jpzE5v6/M\nuZJGenIcpo3rj90fnQr5vtmiJyJJCA3y3lpoK+YWYN0tE9DPQxdvt3FDXAcHBmvd/AmYe/lQ0bdL\nruJjo3DbNSPx4vJL0D/d+28djDmijMj3Lref0e+17aXGQE9EihUXo/f4nL1fAmi69kmO83rPmnoR\nof9f6m75ywoHSLp9ALjnprEuAz3l7jlhoCciijDeAmr3+ArXz7hPf82FA3H7tSPFyFZIDOqfCKDz\ntkHg/Avdck+By0BPRBSh0pM6xz4Y43ueCFgy4wK/tvGjqXmYNKqv/M3WLgP6eO8BWjG3AI//Yoq6\n1yjwEwM9EanWT68ahj7JcfjR1Dy5s6JKd16fjx9MzsHVk7KdXr/2woHyZEgEvgK4TquFIS7K7XsS\nzYArex0ocqo0RCS7GAHPEg/KTMTXlQ2CtjcoMwkPLZzs9NpvFhS53i52eCGY1crCTWpirNtHIG+Y\nmod/7Ps29BlSCLkDs9jYoieikDHGR+PuH4/xugLZ8JzOSWT6pAh/ttpRVh8DBnhZ2OTemWMD2q4U\nnCodIYguCV2tXaWNChfDs0svljsLnnX/tlJ1GfjAFj0RhVTviWN6+8HkHMRE6TD1giCnL3W8piq0\niTZ+WB9kpiegssYsaNR6sGFixU/G4/2PTkk2Nawc5l4+FGfrWxClV27lxWWhnhAPzmOgJyJFiY3W\n4xoV3yP2lykptjPQh0D/9ATMCbO5AeR4BNLfOM1laomIiJQkiMAsd1B3h4GeiEglQhlDBod4SVmS\nDgM9EckqkOlrvQ22cyvIm9t6XeRdKgdnMtCLRZTZHYPAe/REJKvsDCOeWzYNt2/aLSj9/KuHh3Qw\n2ZYlUyIy0CvJqBA9EhnMSoruXDKuP34wOcd1id4Qj75noCci2UkeSIPo846PdT+5CoVOQoh/A5dR\n8gHSajSuQV4GrKYSEVHQfn7DaKQnxeLS8cpcktcfNh/3erRuRtxlpMQBAKKFPObHx+uIiDzzNLve\n2lsKodey7SIVX6EpPy8ND995YUjy4su6WyagsbnN788JXXzGXbL1t01ES5sFUXrlHYMM9ETk1ZP3\nTHVZdlMO982fgH3HTmP8MJPb9wf2TQxxjkipAh38JvQevbsKgV6nhSFOeUEeYKAnIh/iYpRxmcjO\nMCI7Q97RyxQZfN2jV+Kz8t4o4wwmIpLQwL5G5PZLxMVjlTf1qzyzn1MwtAro4fIHAz0RhT29Tos1\nPy2UOxsUJgIN83JV6pR5Q4GIKEL4EzTY+peW0PIVOmjP4+eD+rT/GOiJKCwxKFLAPETitMQYQR8f\n0bXUcn9Tglg5CoqgrvsjR47gkUcewcsvv4yKigqsWLECGo0GQ4YMwbp166DVarFlyxbs3r0ber0e\nq1atwpgxYyRLS0ThJy0xFmcbWuTOhqKFsiV4aUF/7C8/jXqz/4+pSeEnVwzFye+bZM3Dhjsmo73D\n6jPd/KtHYPKovrhgsP/TO0vBZ4v++eefx+rVq9Ha2goA2LBhA5YsWYJXX30VNpsNO3fuRHl5OQ4e\nPIjt27dj8+bNuP/++yVNS0ThZ+PCyXhu2TS5s0Fd0pPj8OjiYqQpYGY3ALi0IAs3Xzlc1jzodVpB\nT6HExegxbqhJMYP2fAb67OxsPPHEE/a/y8vLUVRUBACYOnUq9u3bh0OHDqG4uBgajQaZmZmwWCyo\nra2VLC0RhR+tVsM55UkRlBGexePzrCopKYFe31ODsdls9oEICQkJaGxsRFNTEwyGntWkul+XKi0R\nEZHa9E/vvGc/JMRLAPv9eJ3WYYpJs9mMxMREGAwGmM1mp9eNRqNkaX1JSYmHXsh8w34wmThRhyOW\nhzOWRw+llEVTe8+9VDnz5Gvf0V1dwfoorc+0SUnxIf8uOl1nAywmNkqUfSvl+HAnOrr7t9BJks+b\nrhiOAZlJKBjWx75YUijKw+9AP3LkSJSVlWHixIkoLS3FpEmTkJ2djU2bNuHWW2/F6dOnYbVakZqa\nKllaX+rqmgMqDE9MJiOqq9mT0I3l4Yzl0UNJZeF4HZArT0LKo621AwDQ0W71mba+vjnk38Vi6Xx+\nobWlPeh9K+n4cCe9a1R9v5Q4yfI5LDMR5sYWmBtbRC0PbxUGvwP98uXLsWbNGmzevBl5eXkoKSmB\nTqdDYWEhZs6cCavVirVr10qaloiIQiVyHlS8vjgX/VLjUTi8j9xZEZXGJnQWfxURuyam9FpoqLE8\nnLE8eiipLL77vgnrXjoIAHhpxaWy5EFIeTy2/QiOfH0WORlGrJs/wW2aBQ+9BwC496YLkJ+XJno+\nvVn21F6cbWjFRaP74tYfjAxqW0o6PpQgVC16DnElIiIKYwz0REQy6n7WWsurMUmEi9oQEclo1vQh\naGmzYO7lQ+XOCoUpBnoiIhmZkuOwbPY4r2kSE6LRYG5DkkHYXOtEjhjoiSgshdM44/vnT0DFmUYM\n6GPwnVhkxvhonG1oFTT1KykTfzkiIoVLMsRgjEyt+Tuvz8dbByrww+JcWfZPwWOgJyIij0zJcbIv\nJkPB4ThPIiKiMMZAT0REFMYY6ImIiMIYAz0REVEYY6AnorBkSo4DAEwalSFzTojkxVH3RBSW4mL0\neG7ZNOh1bM9QZOMZQERhi0GeiIGeiIgorDHQExERhTEGeiIiojDGQE9ERBTGGOiJiIjCGAM9ERFR\nGGOgJyIiCmMM9ERERGGMgZ6IiCiMMdATERGFMQZ6IiKiMKax2Ww2uTNBRERE0mCLnoiIKIwx0BMR\nEYUxBnoiIqIwxkBPREQUxhjoiYiIwhgDPRERURjTy50BJbNarbjvvvvw+eefIzo6Gg888ABycnLk\nzpakjhw5gkceeQQvv/wyKioqsGLFCmg0GgwZMgTr1q2DVqvFli1bsHv3buj1eqxatQpjxozxmFat\n2tvbsWrVKpw6dQptbW248847MXjw4IgtD4vFgtWrV+P48ePQ6XTYsGEDbDZbxJYHAJw9exY/+tGP\n8NJLL0Gv10d0WVx//fUwGo0AgKysLMycORO//e1vodPpUFxcjJ///Ocer6eHDx92Sat2zz77LN57\n7z20t7dj9uzZKCoqkvf4sJFH//rXv2zLly+32Ww220cffWRbuHChzDmS1nPPPWe75pprbDNmzLDZ\nbDbbHXfcYTtw4IDNZrPZ1qxZY3vnnXdsx44ds82bN89mtVptp06dsv3oRz/ymFbNXn/9ddsDDzxg\ns9lsttraWtvFF18c0eXx7rvv2lasWGGz2Wy2AwcO2BYuXBjR5dHW1ma76667bFdccYXtq6++iuiy\naGlpsf3whz90eu26666zVVRU2KxWq+22226zHTt2zOP11F1aNTtw4IDtjjvusFksFltTU5Pt8ccf\nl/34UHc1UmKHDh3ClClTAABjx47FsWPHZM6RtLKzs/HEE0/Y/y4vL0dRUREAYOrUqdi3bx8OHTqE\n4uJiaDQaZGZmwmKxoLa21m1aNbvyyivxi1/8wv63TqeL6PK47LLLsH79egBAZWUl0tPTI7o8Nm7c\niFmzZqFPnz4AIvtc+eyzz3D+/HksWLAAN998M/773/+ira0N2dnZ0Gg0KC4uxv79+91eT5uamtym\nVbM9e/Zg6NChWLRoERYuXIhp06bJfnww0HvR1NQEg8Fg/1un06Gjo0PGHEmrpKQEen3P3RybzQaN\nRgMASEhIQGNjo0uZdL/uLq2aJSQkwGAwoKmpCXfffTeWLFkS0eUBAHq9HsuXL8f69etRUlISseXx\nt7/9DampqfagBUT2uRIbG4tbb70VL774Iu6//36sXLkScXFx9vc9lYdOp/NYRmpWV1eHY8eO4bHH\nHsP999+PpUuXyn588B69FwaDAWaz2f631Wp1CoThzvG+kNlsRmJiokuZmM1mGI1Gt2nVrqqqCosW\nLcKcOXNw7bXXYtOmTfb3IrE8gM6W7NKlS3HTTTehtbXV/noklceOHTug0Wiwf/9+fPrpp1i+fDlq\na2vt70dSWQBAbm4ucnJyoNFokJubC6PRiHPnztnf7/6OLS0tLtdTd2Wk9vJITk5GXl4eoqOjkZeX\nh5iYGJw+fdr+vhzHB1v0XhQUFKC0tBQAcPjwYQwdOlTmHIXWyJEjUVZWBgAoLS1FYWEhCgoKsGfP\nHlitVlRWVsJqtSI1NdVtWjWrqanBggULsGzZMtx4440AIrs83njjDTz77LMAgLi4OGg0GuTn50dk\nebzyyivYunUrXn75ZYwYMQIbN27E1KlTI7IsAOD111/HQw89BAA4c+YMzp8/j/j4eJw4cQI2mw17\n9uyxl0fv66nBYEBUVJRLWjUbP348/vOf/8Bms9nLY/LkybIeH1zUxovuUaJffPEFbDYbHnzwQQwa\nNEjubEnq5MmTuPfee/Haa6/h+PHjWLNmDdrb25GXl4cHHngAOp0OTzzxBEpLS2G1WrFy5UoUFhZ6\nTKtWDzzwAN566y3k5eXZX/v1r3+NBx54ICLLo7m5GStXrkRNTQ06Ojrws5/9DIMGDYrY46PbvHnz\ncN9990Gr1UZsWbS1tWHlypWorKyERqPB0qVLodVq8eCDD8JisaC4uBj33HOPx+vp4cOHXdKq3cMP\nP4yysjLYbDbcc889yMrKkvX4YKAnIiIKY+y6JyIiCmMM9ERERGGMgZ6IiCiMMdATERGFMQZ6IiKi\nMMZAT0REFMYY6ImIiMIYAz0REVEY+386QTo678eRyQAAAABJRU5ErkJggg==\n", + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAfoAAAFJCAYAAABzS++SAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3XlgFOX9P/D3HknIfZBwBAgkkHAaMYaAyilC0GpLVURA\nWkXrhUE8wSighYqIX7wQUVp7SH9FKdXSVquWQwggKCpIRAQ5BBIwkECSBXLt/v4Iu9l7Z3bn3H2/\n/oHszs48Ozszn+d+DDabzQYiIiIKS0a1E0BERETyYaAnIiIKYwz0REREYYyBnoiIKIwx0BMREYUx\nBnoiIqIwZlY7AXKoqqqTdH+pqXGoqTkn6T4jDc+hNHgeQ8dzGDqew9BJfQ4zMhJ9vscSvQBms0nt\nJOgez6E0eB5Dx3MYOp7D0Cl5DhnoiYiIwhgDPRERURhjoCciIgpjDPRERERhjIGeiIgojDHQExER\nhTEGeiIiojDGQE9ERBTGGOiJiIjCGAM9ERFRGGOgF+DoyTrsOXha7WQQERGJxkAvwP3Pr8eSd3fB\narWpnRQiIiJRGOhFsIGBnoiIhKk91wibTf24wUBPREQksR+On8XMV8rwt//tVzspDPRiaCBjRkRE\nOrD3SA0A4H87j6mcEgZ6IiIiyRkMaqegDQM9ERGRxM7UNaqdBAcGeiIiIomt+1L9Kns7BnoiIqIw\nxkAvwq4DnDSHiIj0hYFehD9+sFftJBAREYnCQE9ERBTGGOhF4DB6IiLSGwZ6IiKiMMZALwrL9ERE\npC8M9CJwClwiItIbBnoiIqIwxkAvAgv0REQkhMmoncnuGejFYKQnIiIBGOiJiIjCmMnEQK9LNhbp\niYhIgPye6WonwYGBnkiHfjh+Fjv2nlQ7GUTkw6U926udBAez2gnQFRboSSN+9/ZOAEBR344qp4SI\nvPnX1sNqJ8GBJXoRGOeJiNTT3GLFzn1VaGxqUTspAVWePqd2EhwY6ImISBc+2vEjXnvvG6xaf0Dt\npOgKA70InBmPiEg9x6ssAIBvD1ernBJ9YaAnIiJdMJtbQ1ZTs1XllOhLwM54LS0teOqpp3Do0CEY\nDAY888wzaG5uxj333IMePXoAACZNmoTrrrsOS5cuxcaNG2E2m1FaWor8/HwcOXIEs2fPhsFgQG5u\nLubNmwej0RjytupgkZ5IqC+/r8KGL4/hwQmXwmximYJCZ5+DxsbqVVECBvoNGzYAAFatWoXt27fj\nxRdfxNVXX4077rgD06ZNc2xXXl6OHTt2YPXq1aisrERJSQnWrFmDhQsXYubMmRg8eDDmzp2LdevW\nITMzM+Rt1cBri0i4pf/4BgDw7eEa5GtoqBHpWWuk57NYnICB/pprrsHIkSMBABUVFUhKSsKePXtw\n6NAhrFu3Dt27d0dpaSl27tyJoUOHwmAwIDMzEy0tLaiurkZ5eTmKiooAAMOHD8eWLVuQnZ0d8rZp\naWnynRUiDdNbaUZDM4GSzhnsJXp1k6E7gsbRm81mzJo1C5988gleeeUVnDx5EhMmTMCAAQPw+uuv\n47XXXkNiYiJSUlIcn4mPj0ddXR1sNhsMF38d+2v19fUhb+sv0KemxsFsNok7EwK0WG3IyEiUfL+R\ngudOGunpbedRD+c0OSVOc+nUWnr0SI1zGBsbDQAwGgyO4zc0teCxVzbhF8N7YvSgLMXTJISvc6XU\nORQ8Yc6iRYvw6KOP4pZbbsGqVavQsWPrRB1jxozB/PnzMXr0aFgsFsf2FosFiYmJMBqNLq8lJSUh\nISEh5G39qamRb/xiVVWdbPsOZxkZiTx3EsjISETVqbbzWFF5FlFmbbd/19ae19Rvr8a1eKyqHv/v\nk+8x7bq+SE+JVfTYclDrfr5woQkA0GK1Oo6/+4fTOFRRi5dWfYX8HqmKp0kIb+dK6nPoL9MQ8Anx\n/vvv44033gAAxMbGwmAw4IEHHsDu3bsBANu2bUP//v1RUFCAsrIyWK1WVFRUwGq1Ii0tDf369cP2\n7dsBAJs2bUJhYaEk2xIRUF13Qe0kBMSae2DFv77Fdz+ewbsbf1A7Kbpmv5ZcW69YkR9IwBL92LFj\n8cQTT2DKlClobm5GaWkpOnfujPnz5yMqKgrp6emYP38+EhISUFhYiIkTJ8JqtWLu3LkAgFmzZmHO\nnDlYsmQJcnJyUFxcDJPJFPK2RKSPTknVtQ1qJ0F1LdbWH0pv/Ss0x0uukac0sICBPi4uDi+//LLH\n66tWrfJ4raSkBCUlJS6vZWdnY+XKlZJvS0T6CBxvfbAXQ/M7q50MVVWcam1+vNDQrHJK9M3o6HXf\ndt1r/w4ALjQ2o120ekvLaLtxj4g8OT3ZdBDnyUn54Rq1k6BvjnH06iZDLLUn+GGgJ9IxPZToSR/q\nzzehuUXbM8452uidX9TBLaD2bcpAH8C5i708xWhobMHaLYdw1tIoQ4paWa02fP7dTzjPqkBN2v3D\nKbyrwMIbaj9ASDvqzzfhZHVwI46aW6yY8fJmzPn9dolTJTFHG71z1b32bwK1M+QM9AE8uUL8hf/v\nbYfx/uZDWPGvcukTdNGnuyrw+vt78If/7JXtGBS8l1bvxn93/IjqWnl7xVsZ6XXl8rwM2fb90Ktl\neOLNz4K6JuxVyydrzkudLEnVXiw8nW/Q/jK1ztS+SxnoA3AvlX9/9EzAz9TUtfYy/knGm+ZYVb3g\n9FDwfjh+1tGRKhhWq9q3OGlBUlwUACAxPlq2Y9h79jcH0R5s1Mn0hTv2/uT5og5uMbXz4wz0Ij33\n1y/VTgKAtgvHoI/7U7d+9/ZOPBVCdaYc97dzVaXaDxASSYEfrCWIzKVO4rxuseqegnPxwjEw0mua\n3Lf398dYo6MLCt6nkZb508PXVfs3YaDXKXumnWE+sgXb+YqU5bW3uEyC6ZymdiBSQ0NTC36oOBvy\nfs43NAcssbNET0FR+8IJxo69J/Hx50dl2//HO37Epl0Vsu0/KE6/01sf7MU/Nh2UdPeff+elzZK0\nR8F11COtX0iwp3TZe3vwu7/sxN4jwc9tUFPXgOkvbsKKf33rdzvnJJ46cx5n6pWdLZKBXqfsHf3k\nHMInteX/LMeqdftl2/+q9Qfwpw+/k23/wXC+wct2V+LfWw+HvE/n4F53znX45+mzF7Dy432oPy9+\nWCjJR8maNx2WASQjJiP1zcHTAIBjP9UHfbwjJ1sXpfns25OC0/X48m14eOmWoI8ZDAb6AOy9ZbXm\nYGWt2kkI6NvD1Xj6jzt0lRmRnMQP3QPHzuDNtb5LDyv+/S3Wf3kc722WtuaApKFEEI68IZdt33fn\nviq1Du1/M7bRa9uogq5qJ8GrlhbPK+dQZS3OXdDOBDovrPoaP56sx4Yvjwna/lhVPWYv34aDFdrP\nxIRi2nPrseSdrwVtu/3bk3hyxWeO3/VMnf8qP3umyiKyRF9raVT+IRlBlOw0q3ZQUcrhE63PCefv\nay9hK8VXf4jjbkNy1f5JGOgDuHJAp5D3cb6hGfcv+RT/2XYY3x89gxkvb8aPIV6Q7rn2ytMWzP/z\nF1i4cmdI+3Vms9nwyedHBY8jb26x4ovvfkJDo+tkFka3h9zr7+/x+vl/fHoQP505j798pK3qd7vP\nyk/grx9/L6p60NeWew5VC/r8G2vLUXn6HHb9cErwMYOx8K9f4rX3vsG+HzkXu5yUeOBHShv9/qOe\nHemE3Jr155uw5lOJlgv2cbzX/vGN62bsjKdtYn6emroG1J3zrKY+VFmLC40tWPPpQSx77xvUn2/C\ni+/uki6RAE6dbZ2BzZ6TPHehCQ1Noc0edbCyFn9bt1/wOPL/fXEMy97fg798tM/1DbfCjLcOZDab\nzTECKdR7ovxwNQ4cF9abVsxD8c1/fYt1Xx7Dl98LL/n6u8F//+9vsfuH06L2E+iBEUwv/IMVtY7P\nnZZhJr9Le7YP+rMVpyz43dtfoPJ08JMWaYF9Ei0lIn0wQSVcsgZCmi3eXX8A/9l2RLY0tFitOOF2\nH6pdy8JAH0BqQozP97buqcTbH7cFtUde24IHXynz2M45ztVe7Dwlpt3aZrOhqdk1aA/p19HnMQDg\ngZc2Y/qSTY6/rVYbWqziZswSO4++fba+/W5juwNVW9ZaGnHnog34an9rqTXU3O//rfoaz74duGbj\n/c0HcdfzG7Dn0GlR7Zo1AarPhdq65wReWr0LKz/eF3A0wu//3TrVsdBUBvo6zouXvLNeWAdJsW2/\nZlPr794lI0HU55z99ZPv8cPxWvxZY50shfC2QEyT02tWm/h7UohQ91hdewFWq80x3azU1m45hI92\n/Bj6ji4+VpzPs7eMu/vzpOybSpe/Q8lEersj3Gs0vaVBaQz0AUSZPU/Rf7YdxrTn1uP3/96LDV8e\nl2RhmU8+P+rRI73ilAXvbTqIF1fvwj0vfIpGpxJ6o/s0l15iqfODufTNz3DvC5+KSpPBaafnG5pR\ntrvSI8Phsv3FzU+dvYBpz633lzQX7je9UvfE2i2HAQBL3tmF/0jQG94bId9l/ZfHhY9GEHhu/OWt\nvj1cjbsXb8Tm3a1DEZ039ZXeTz4/irsWbcDJGvE1Bh98FnzpyWxqvf88rncNOVl9zqNm5tTZ87h7\n8UaPKmLnQDT3Dztw3/9tgtRsIVbdn6w+h9fe+wYzXy3D6bPS1/C8v/kQ3pFiwaeLX9P52nDPjL60\neheeeOMzv7vZ+HXwQ3KdD3fUT+99tWtMGOiDsOZT1x7NX+2vwj/LDnlsd+rshdacnI+n7o8n6xyB\n82/r9nuU6p7+4w78a+th7DnY2p5b69Qs4F59bAgQTn86c1781JhOu1z58T689cFePLliu8/Sp6+S\ne6B+SJ3ax7n87d6RRQlfBOiIFmpP5mBy9KG0l/s73ObdrSUaR/WlgI5if7uYERHTbOHspyAyCEBb\nrUCzl86nStqx9yQ++/aE1/eeePMzvLR6Fy40tmX47WtQuFcROzdbVZyyyLIsbKjXqhVw1K4dkmh0\nz/mGZmz48phLZ2GrzYZV6/bjB4HNbO6qzrQOMY5v1zYyyr2T8u4fTuOnM3Iu1NN2vHlv7XB7xetm\nqmCgFyDFT/U90Fqt6i3QA20PVW+e/uPn+O2fvvD5vtCHm02mKkDnx799nOipsxd8lj59zZftLQPw\noVMpL9D5VUKgQOxeHffD8bOCHoI2t3/FWPT/vvLcnwTVHfZ92H8V518n0Gp7Yg7vfP1u3eM9SAon\n/MANTS2SD+lc/s9yv8MaAdfvq+bU1KH2xfvcaeGYaomaqd5ZfwBvf/w9Zr7a1rS5/+gZfPz5UfzO\nTzOb5UIT/ll2yOu8EP/b2TqaJya6LYw1htgvCQD2Hq7GdwIn0fF2P3h7Te0hjwz0ArxZeo2o7Z0f\nan/68DscOeG7h7176dVenSqmI917mw862nClJKRK12V7Hw+3dTs9h9et3ihRr9eLvAUoUb3jA2z6\nxlrXJYd/9/ZOzP/zFy61LL52fKL6XEiTcrjsLsTtTp0537YCmMEAm82GfU4rIL632XuG1e7vG3/A\nJ18cRfnhatRaGjHtufV46wPXa8/bQ23tlsMumTuh7NeUmMfk469vxUOvlin+cDUYWq+5ac+txz/8\n9Or2dl2eb2jGsve+EVyC9ndtO9csCN6P0+7szyAAgpuU7n1hI6Y9t94l/f/ZdhjrvzyGnft+csxY\n6VyD0eyUI7HZvPcJeHf9Afyz7BD++sn3fr5D2/99zTj33+3C+wQsXvU1nv+bZyZbKG8rij79x8+9\ndtRWCgO9ALExZvx6XO+gP//uBuHtUX/84Ds0t1jxsY/OKs0tVrzpts79v7cekWcmNJGlEl9bB+q8\n5m2oYYvVip/OnA/40Np14BSmPbcei73cmMt8DOPz5nxjM348WedzLLlzG6xzm+BML50vndnQ2j/i\n6T9+HjANdecaHeuC+9xfiLHr8eXbHP83wH+Hy/9sO+y1+eBv/9uP/1v1taPtvcyp1upQZS3uWrTB\n61TEwWTuHNfUxe+98evj2B1gqKFjtkCJ4nz54bahkP6GxVaebmurP13r+5qf8fJmj9c2fnUcX+yr\nwvw/f4HpL27y22xzobEZdy7agNU+niu/+4v/jqj2YLt64wHcuWgDzjc0u2SK3K8xIRkm+z0x/89f\n4EJj69zvaz49iJUff4/X3vN+HzoPu71z0QbMfLXMo0+DfTTRdj8zzzlnesoPez9vYp7BYng7M0vd\nhtbZ/fED9TqUMtAL1Ktrimz7du64BgB3L97os2RVfqgan5X7n27R+eG9rfyE39z/v7e2diz86vsq\nrPhXOf72v/2OB4HYysdgqyvd+zwAwCefH8Ps5dtw/5JNfjMx9pz+yRrPdjjnoL1j70mcOtu6jdVq\n8whw1bUNePqPn+O1975xdJjydd7+7hawzvqZt1rM8L0HXynD7De2+Xz/fRGz3R2sOBtwqJ3B0Bqc\nvDlb34A1nx702nxg5+272Wuz3g3Q2erA8bM4XuW/lqPytAU7L/YJsNd8/eW/+/DS6t1eq+bPXWh2\nqTnZK9GcAP9wuj79ZdiefXsnXv77bo/X3a8Py4Vmlxq7uxZtcCkJn29o9nvefzzZ+h0/FFFKtXv7\no324e/FGnLU04sPPWj9/vMrid5jnDi9Bdv/RGp/NPPcv2eQz2DkzeWnrc+/TYLPZvP7fdRvXv9/+\naJ/Pmoid+6RdG0JMreFPZ87L0idDCAZ6gbqkx6udBLz5r29RLmCilZVOQ/5W/Mt1rPYX3/2EPYfa\n/rYvsvLqP77BtvKT+OSLo46S2gUvw0S8+Xr/Kcx4eTM2fHVc0PZ2/jrJHD/V9sCe8fJmfL0/uAlj\nGhpb8MkXR7H8n+V4/PVtsFptuOv5DZj+ou/ezvbZroTeww85zVt94PhZvLy6bY4EX303fKmpa/BZ\nqm8dJeCZqDm/345zF5qw16nkWV3bgCfe/Axvri1HQ1MLDp+o9Xj4VZ4+hyU+5nMQkj/x1n7rmAvB\nT3G6ucWKZ9/eiTl/2AGrzeaz+eOZP7kG1T992NZE8NCrZdjsVmsw760dmHuxQxTQWvpubGoJWCL9\n9nA1nlzxmdfA1fpZ30O2AjbdwHtA3uI0xMtqs3ntDOptZbW3/rMXz/31y4DH9MV+jzo3Jza3WP1+\nj9O1F/BTzTn8/t/f4siJOjS3WPHwS5vw6LKtOGtpxAq3GkagrTOfP0YfnXqcJ/1y/um8/Yonq895\n3Kcbvjrus8Owr9oFoQLVuPnrK1VT14C7F290/L3DR+dOOZgVO1IY6NU1GQeOhb6sYbAOHDsr6Pjb\n3Er8zqUMe3X2pNG5SEn03gnuYEUt/rv9R7/VXYcqa/Hx50dR2DtD0M3zxV7PUsHs5dvw1uyrvW5/\n9KRrae+LfT9hYG66y2vnLgRurrhvieuQQiGdwr49XINLctpjW7nwG3Hac+sxbnCWR1ugkAeeu3te\n2IiBvdK9vuctZh0/ZcEDL3lWBwOtnShP1pzDoUrvVc7equ4vNDbDZGp7CPs6D9564NurY31lFHbu\nq3IJ2Mv/WY4vvvsJMyfk45Kc9i61Qo1Nrg/NTbtcO7b+8cPvkNuttaatfVI7j8l+Vm/4Aas3tNa+\n/PyqHli75TBeLBmKJkM9Xn3nawzp3xFXDuiMV/6+G43NVnz8+VHcOjr34rFbsG7nMazdctijv8xH\nO37EO+sPoOSmS3CsKvAIEW9BR0hH29/9ZSdeLBmK5Phox2vuY8D3HzuDXC+1jS1WK0xG3+W4E05j\nx5//21f42RXdfW675tODjlq3beUnsOzhEY73/vzhd/j6QHCZcF+97fdffMbV1DW4NPvdtWiDx7bP\n/b8v8atiYc2q3sa32zW3WGE2GT1qD785eBrrdh7DqbMXMKawK/783324ZVQvjBuchd0/nMbyf7pm\nct5Z5/uZ6X6vzf/Ddp/PP6kZbGqP5JdBVZW08x1nZCSiqqoOL63eJXgmMz3L6pCAE9XnFBm7HBNl\nEtTxsF20Ca88OMwxrrpsd6VHJzApPfmrywO2dYazqcW98bb7DId+2APSO+v346MdRxEdZfQI1IHc\nNCIH/XqkoVuHBNhsNtwjYt6H/j1SfbbPOuveMdFlPvS3Zl+NuxdvdFSpTv/lAFzeu4NHc5ov9gyE\nWJ3bx/lsNnFnMhqw4vFRADyb+QA4goXze/NuH4TYdmZkJLdzyTz5+l4dUmIFD0N7/ZERuO//xM3J\nIdbYQd1CXtL6rdlXi/odUxJj8Jf/OtWGPj4Sv3l+o8e29t/joaVlOFsfWgc7KQN9Rkaiz/dYohch\nySlnHc5+lKiHuBBCRxdcaGzB3Ys34qlfFaLuXCP+K8XMWn4sC7GKT+/EBHmgtSr9rdlXO2ocxAZ5\nwLXkKJaQIA94LnrS3GJ1aTd97b09oh6+p4KcUEZokAeAFqsNDy8tw7VDvJe6K09b0Lm9a9Oivdkj\nI6UdfjWuD3K7JCM6yuTzGM0ihucq0c4capAHxM354C2z5i3IA62/h81mU31aWzFYohfAXqI/WFGL\nBX/xPe6dKJK98ehI3PPCRrWTIUpaUgyq3XrIiykJaoWQNP96XG/8+b/iMnDeJMRGyTPKJwIpVaJn\nZzwRcjKT8IdZo9ROBpEmvV8WXGlcTe5BHpBuLQMlCcmYSBHkATDI6xADvUhqznhFpGX24Vp698hr\nWwJvRKQjDPRERERhjIGeiIgojDHQExERhTEGeiIiojDGQE9ERBTGGOh1gP38iYgoWAz0OhAT3Taj\nVZcM9RfXISIi/WCg1xmW7omISAwGeiIiojDGQE9ERBTGAq5e19LSgqeeegqHDh2CwWDAM888g5iY\nGMyePRsGgwG5ubmYN28ejEYjli5dio0bN8JsNqO0tBT5+fk4cuSILNtGEpuP/xPpzZ0/64s//Ee+\n5YWJyFPAQL9hwwYAwKpVq7B9+3a8+OKLsNlsmDlzJgYPHoy5c+di3bp1yMzMxI4dO7B69WpUVlai\npKQEa9aswcKFC2XZNlKxjZ70LDEuSu0kEEWcgIH+mmuuwciRIwEAFRUVSEpKwtatW1FUVAQAGD58\nOLZs2YLs7GwMHToUBoMBmZmZaGlpQXV1NcrLy2XZNi0tTaZToj3OwX1Qnw44VnVItbQQEZG+BAz0\nAGA2mzFr1ix88skneOWVV7BlyxbHKm7x8fGoq6tDfX09UlJSHJ+xv26z2WTZ1l+gT02Ng9ls8vl+\nMPyt9Ss35xXzfjEqF+9tZqAnfUpOjlM7CUSaoVRcERToAWDRokV49NFHccstt6ChoW29ZovFgqSk\nJCQkJMBisbi8npiYCKPRKMu2/tTUnBP6tQTJyEhEVVWd4++pxb3x9kfSrO0sTFvL/OnTFj/bEWnb\nmTPS3ptEeuYcV0LlL9MQsNf9+++/jzfeeAMAEBsbC4PBgAEDBmD79u0AgE2bNqGwsBAFBQUoKyuD\n1WpFRUUFrFYr0tLS0K9fP1m2VdOoy7pgVEEXVdNAkWPi1b3UTgIR6VjAEv3YsWPxxBNPYMqUKWhu\nbkZpaSl69uyJOXPmYMmSJcjJyUFxcTFMJhMKCwsxceJEWK1WzJ07FwAwa9YsWbZVW8eUWMHbDunf\nEZ+Vnwz6WDZ2tY9oxUVZeGf9AbWTQUQ6ZbDZwi+MSFkdAnhW3QPAxzt+xCqBD98r+nfCtvITQR+/\nXbQJFxpbAABLHrgKDy/dEvS+SH/emn01pj23Xu1kSOLBm/Px8t93q50MIk14a/bVku0rpKp7Up+B\nY+qIiChIDPRERERhjIFeEaG1joRf4woRESmFgV4BjNNErdgMRaQ8Bnqd4XOSKHLNuCmy1vnQi9uv\n7aN2EvxioCcixbAZKjQJXCtAk7ReAGOgJ6KIkhDLYEmRhYFeB9iuSSSdIQM6q50EIkUx0BNRRGHG\nmSINA70CQn2uuLRr8ilFREQiMNATaUwiO1zJysDMMklM631MGeh1gM+lyNK/h7qrM4a7MFzeg1Sm\n9Uc0Az0REVEITCZth3oGel0wePkfERFpQUxUwBXfVcVAT6Qx4VyxHM7fjUirGOh1gY9HIiK5dEyN\nVTsJsmKgJ9IYNs/IS9e97pnnl8VDEweG9HmtX1IM9ArgvUmiaPyhQUT6wkCvNwwC4Y85Q/KF9788\nwnzIJQN9kC7NTVfwaLy7iYgoOAz0QeqYGodHbg2tXYco0jDLSqQ8BvoQGBXrgRHe1UpEJBAfBRQE\nBnoFsBRDRBS+tN7Ez0CvCOlCPTMN4U/jz4yQhPN3I9IqBvoQMOgS6Y/WxzwTSY2BnogiitarWYmk\nxkCviFCfLCyCEBH4KKCgMNArgAUIEoPrpZNPvDQoCAz0SpDw5tT1PN1ERKQ4BvoQMOYSEZHWMdCH\nILtzEjJS2gXekBkCIqKwpfVCHwN9CKKjTFh075UYOTDT73YavwaIiCiMMdBLYPKYPLWTQKQP7EwW\nEhtPIAWBgV4CZhNPI0mHHS6JFBbm9xwjlExuHZ3r9Fd4X0REpAwDnyUUBAZ6mfB2JCKKDFqf+oKB\nXhEavwpIU8J6whzmgEmLwvmeA2D292ZTUxNKS0tx/PhxNDY24r777kPnzp1xzz33oEePHgCASZMm\n4brrrsPSpUuxceNGmM1mlJaWIj8/H0eOHMHs2bNhMBiQm5uLefPmwWg0hrytFvXtnoq9R2rUTgYR\nEZELv4F+7dq1SElJweLFi3HmzBmMHz8e06dPxx133IFp06Y5tisvL8eOHTuwevVqVFZWoqSkBGvW\nrMHChQsxc+ZMDB48GHPnzsW6deuQmZkZ8rZaZDT6LqpImVcM8z4jRLLT8z3EXvcUDL+Bfty4cSgu\nLgbQWp1oMpmwZ88eHDp0COvWrUP37t1RWlqKnTt3YujQoTAYDMjMzERLSwuqq6tRXl6OoqIiAMDw\n4cOxZcsWZGdnh7xtWlqazKdFPJNboM/rlqJSSojInzCvpSXy4DfQx8fHAwDq6+sxY8YMzJw5E42N\njZgwYQIGDBiA119/Ha+99hoSExORkpLi8rm6ujrYbDbHUCH7a/X19SFvGyjQp6bGwWw2iTwV/mVk\nJPp9v2TiZfjNs/9z/F14SdskOu1iokI6tnMeIr19Qkj7Iu2LcbteAl17epKcHKt2EnQtJSVO7SSE\npbS00J4Nnl7UAAAgAElEQVSryckCZkj1Qql722+gB4DKykpMnz4dkydPxg033IDa2lokJSUBAMaM\nGYP58+dj9OjRsFgsjs9YLBYkJibCaDS6vJaUlISEhISQtw2kpuZcwG3EyMhIRFVVnd9t3LMVzttf\naGgK6fhWpxLIqdP1Ie2LtK/B7XoJdO3pydkz59VOgq6dOSPts41aVVeH9lw9e/ZCUJ+T8t72l2nw\n2+v+1KlTmDZtGh577DHcfPPNAIA777wTu3fvBgBs27YN/fv3R0FBAcrKymC1WlFRUQGr1Yq0tDT0\n69cP27dvBwBs2rQJhYWFkmxLRBSJOI6eguG3RL98+XLU1tZi2bJlWLZsGQBg9uzZePbZZxEVFYX0\n9HTMnz8fCQkJKCwsxMSJE2G1WjF37lwAwKxZszBnzhwsWbIEOTk5KC4uhslkCnlbvQn11jT4+D8R\nRRZ2xqNg+A30Tz31FJ566imP11etWuXxWklJCUpKSlxey87OxsqVKyXfliicsbMYkb5ofSQHJ8zR\ngYK8DLWTQEREOsVArwPJCdFqJ4GIiHSKgV4RGq/XISKisMVArzvMNBARaYnW+9Uw0CtC41cBERGF\nLQZ6CaUmxnh9nWGexNB6D95QcHgYaVI433RgoJdUQmxoU90SERFJjYGeiBSjhZndwrzwRuSBgV5C\nSnTI4EMq/Gm9Y4/e8fyShzC/KBjoJcQgTHoxe0qB2kkgIoUw0CuA8Z+0pmsGlzvWozAveJJMGOiJ\nIhBrn4giBwO9hHw/O/lUJeFYaCMiKTHQawSrUokoENbEaJPWfxcGekUELqOZTRq/UogkwAlziJTH\nQE9EpBPsjEfBYKAnikAMGETS0fr9xEAvIV+/dWyMWdF0kM5p/alBRLrCQK8AKQO91jt9EBGRtjDQ\nSyiUGMwyHBERyYGBXgEshRMRkVoY6HWAGYUIwx+ciCTEQC+lEJ7PfLSTkpiXIDklx0ernQRywkAv\npRAa2v191LkTthbW8yZ96pgWp3YSiLQpzHO+DPSKCO+LiPQhu1Oi2kkgIhUw0EvJZzxnn3oSQaZx\n9LwKiXwI87krGOgllN+zvdpJICKdG9Kvo9pJoDDDQC+hgrwMtZNARDqVmR6PhfcMwZ3X91U7KSSS\n1pv4Gej1RuMXlLMh/VkyIRLKZrOhY2ocTEY+lklanIRdQgmxUQCA6KjWG/WZaUWIb2fGhq+Oh7Rf\nrecWfQrvZi8iIl1goJdQenIsHrl1IDLbxwMAunVIUDlFpEdy5o8en3QZKqvPwajb3COR9mi9Lx8D\nvcT690hTOwlEPvXpnoo+3VNxobFZ7aQQkULYGKQA98JTlNnLadd4jpBIErzOiRTHQK+ChyZcGvRn\nWeFKRJrHB5WmMNATaQ1LvREpLTFG7SRQmGKgV4HXflDMARNFtNgYdpkieTDQawVLcSQz5iWJIhMD\nvS7wEU3SknoVxOuv7C7p/ohIOn7ripqamlBaWorjx4+jsbER9913H3r16oXZs2fDYDAgNzcX8+bN\ng9FoxNKlS7Fx40aYzWaUlpYiPz8fR44ckWXbSMbhz6RFUSaWGYi0ym+gX7t2LVJSUrB48WKcOXMG\n48ePR58+fTBz5kwMHjwYc+fOxbp165CZmYkdO3Zg9erVqKysRElJCdasWYOFCxfKsm3kYb1+IN07\nJeLIiTq1kxG5mAMl0iy/gX7cuHEoLi4G0DoPs8lkQnl5OYqKigAAw4cPx5YtW5CdnY2hQ4fCYDAg\nMzMTLS0tqK6ulm3btDROSqMHzJ5oi6y/h9CpwZgfiAy8+TXFb6CPj2+dyrW+vh4zZszAzJkzsWjR\nIhgu5t7j4+NRV1eH+vp6pKSkuHyurq4ONptNlm0DBfrU1DiYzSYx5yGgjIzEoD8bGxvt8ndySpzH\nNuYo31WfcXFtw27S04NPh9JiFOxF7HUSIp2KdjtvoVx7zmJizI59XWiQdma8+HhhQ8OSkmIlPW44\niWkXFfC3TvHy7NAio1FfObr27UObrjw5ObjrWqp7O5CAT+LKykpMnz4dkydPxg033IDFixc73rNY\nLEhKSkJCQgIsFovL64mJiTA6rcIk5baB1NScC7iNGBkZiaiqCr5a+Ny5Rpe/z3hJX3OT1c/nGxz/\nP3VKP9XTDRIHE3+amn2fP71pdDtvoVx7zhoamh37amhskWSfdha3a9yX2rPnJT1uOHH+fXw5c0ba\nZ5tcrFZ9FelPn64P6fO1tcFd11Ld24D/TIPfYtCpU6cwbdo0PPbYY7j55psBAP369cP27dsBAJs2\nbUJhYSEKCgpQVlYGq9WKiooKWK1WpKWlybat3ghtvrwsN13I3kJKC2mfvh6RpCSb1ldPsYuwx5TW\nfxa/Jfrly5ejtrYWy5Ytw7JlywAATz75JBYsWIAlS5YgJycHxcXFMJlMKCwsxMSJE2G1WjF37lwA\nwKxZszBnzhzJtw1XXFGMAB09zIlIF/wG+qeeegpPPfWUx+srV670eK2kpAQlJSUur2VnZ8uybTiy\n+S3H6TMDwIBFJIKA+8XAwgAFIXx6MGmY+xhjn/cq7+HgMU9BROQVA70C4tpFebw29/ZCzw0ZrEgh\n/muQiMJLWlJkLxjEQK+SHp2SXP4WOiUpa+7Cn1wtHhkpHNqmaQJubjaHBSfSFwxioNcIlrBIbmYT\nc4mkDM1daRH+eGWgJ13I7qyfiYLItwh/3pJKIv26Y6AnXbjr+n5qJyGsSL16HRFpFwM9kYomjOyp\n2LEY2okiEwO9AkLtBMUHtAA8SepiJzFFcBw9BYOBXgGX5Ohv2l4KPwzF+qeXXvf6SGXkYKBXgHsu\nXOy9ypuGNI8lTdIwua9OrV/+DPREFDKNP+d0gedQv7Re0cJAr5Bn7x6idhKIVMdgRqQ8BnqFdEqL\nc/zfazWPwByh1quIiIhIWxjoycFs4uWgBd46XJXcdIkKKZGexms4iRQzYXSuYsfik50cljxwlaT7\nU7TdKowiiLevcklOe8XTQUTy+dV1yk0CxkCvFQb4bMBUqrY+IdZzlT0iIrHYwqgtDPRaYUNYlUpJ\n27iIElHkYKDXGc5RHl4YbolIbgz0REREYYyBnnRB6xNSSEnK79qvR6rX11kzRHLinPzawkBPpCK5\nH4ejC7rKfAQi7ROSd1Yya9InK0XBozHQa8YVAzqpnQRSQ4QVfCKpZobIlwduVHZeDAZ6DbhpRA7G\nDurm832XWrAICwxhT8HAx0uHlKKXVfacKZniuHbKDmVmoFfQpGty0T4pBj06J7m8npIQwzYtEa4d\nnKV2EnRJf49e0hLnabzl8POresi6/0CS4qNVPb6czGonIJKMKeyGMYWeJff4MJ2oRsrAErBWI4zy\nSRzjTpGoa0aCbPsW8njo0SkRu384LVsa1MQSvQbk9/Q/vakOa8FIJe2TYtROAgWJlXrqCufTz0Cv\nAUYvd/iQfh1VSAkw9JLOqhyXpMHlkInIHQO9RpnN3n8auXOdmenxMh8hOKzVEMr1ClHqtAktjbLU\nSqQ8BnqNKuzdQe0kEAk2qA+vV9Iuo9E1hxkXE1nd0xjoNahLRjxiovjTiBJOJX4Jv4sSBeg/zBqF\nDqnS9sh+ZOJASfenZ7OnFKidBN2Ldqshzeum7IQ1amM0UdGDN+d7v4ndHvS6re5kfXtQgj1rXTPU\naXYRMzRU6CXRPzstyNSEH19B6Z6f91c0HUlxbaOD7v2lshO+yE63D1lhGOhVdGmvdJ83MWMkiXXf\n+AEIr6oN8qdvd+/rGMil2Gn+iqSEMBvdEeYPXAZ6LfKTueTEOiQFjtWXT6xS7b8KPwpiokzKHpAk\nw0BPiunRKVHtJGiO13Ab5qULIgD47bQiTC3urXYyIgIDvUaFY8H9wZvz1U6C5ik1DSiXqZUPz6ww\naUkxyOua7Pg7HJ95WsFAT7rgXMbt1kG+qTKV5v5s652Vykp1nZPz91OqsifUmMsmRm1hoCfdye6U\nFHgjHQvlYc5af5KElzgt5tJScvW628bmYenMYV7fU6rDYveO2i58MNBrFB/Yvkl9akYXdJV4j9rE\nMhYJpsLzJ9hnXl7XFK/LvnZMjXVMlCP310lPiZX5CKERFOh37dqFqVOnAgC+/fZbDBs2DFOnTsXU\nqVPxwQcfAACWLl2Km2++Gbfeeit2794NADhy5AgmTZqEyZMnY968ebBarZJsS+TNE7cFN7HIYJXW\nFVAa846hiY1hr3NNkigHG+z9oYdZ9gKmcMWKFVi7di1iY1tzLOXl5bjjjjswbdo0xzbl5eXYsWMH\nVq9ejcrKSpSUlGDNmjVYuHAhZs6cicGDB2Pu3LlYt24dMjMzQ96WdErmdrvcrpE125VWxUSZ0NDU\n4uNd/WY3LsvNwNY9J9ROhjIM8Jy4S5aDKCPSa7MCluizsrLw6quvOv7es2cPNm7ciClTpqC0tBT1\n9fXYuXMnhg4dCoPBgMzMTLS0tKC6uhrl5eUoKioCAAwfPhxbt26VZFvSB7ke6ZF+02rd3NsL1U5C\n2GIfN/mE86kNWKIvLi7GsWPHHH/n5+djwoQJGDBgAF5//XW89tprSExMREpKW2kqPj4edXV1sNls\njt6X9tfq6+tD3jYtzf/0mKmpcTCbpa1my8iQfwx49MUqILPJiJSUtrnD4+PbZqGSMx0ZGYkuxwpV\njFuVVvv2wU/R2j6t7bNpaZ77iYoyBn1unM+10tzPd0pKLKLcJibJyEhEizVwtiktLR4d0ly/S1Jy\nW9thfHyM4xxdaGgONske3M97fp9OPrdNShLWlqnE/SZWOy/twN4EG4zbxUR5/d7OryUnt/2+6e3l\n6wBmMBg8OtSlJAu/T4zGwK3C7dMTAKdr3flaFSM1Nd7reTOZjIi+uH+z2z3l/mxKS0twPH/FMhgN\noq9X+/ZKXeeiv9mYMWOQlJTk+P/8+fMxevRoWCwWxzYWiwWJiYkuP7bFYkFSUhISEhJC3jaQmppz\nYr+WXxkZiaiqqpN0n940Xnz4NrdYceZM23ewWBoc/3dOx5UDOklalVhVVedyrFA1uAWT06ctPrYM\n7HR122erqz3309RsDfo3cj7XSnM/32fOnEdjo+t5E/q9qqstiDEAt4zqhXc3HAAARDnVq1gsDY59\nNTT6qloXzz19/tJbW3s+qH1qwYULTYK2C7bH+YWGJq/f2/k152v11On6oI4jiJevcOZs27EDZWbs\nfaz8OX2qDjV1bdd/7Vlh14a7mmoL4s2eCWppsaLxYhNSs1tTkvuzqbq63vH8FctmtYm+Xquq6iSP\nK/4yDaJ73d95552OTnHbtm1D//79UVBQgLKyMlitVlRUVMBqtSItLQ39+vXD9u3bAQCbNm1CYWGh\nJNtGAm+PCpPbUou6q2oS8AC8/soeQe47uI9pTQcJeu+Oc5qTPKuj1krG0l61at0DwXb8FGr05eqO\nBOEUycLddX0/tZMQkOgS/dNPP4358+cjKioK6enpmD9/PhISElBYWIiJEyfCarVi7ty5AIBZs2Zh\nzpw5WLJkCXJyclBcXAyTyRTytpFo2cPDPQK9WJflpuOr/ackSpE8fPZgjZDxhu2T22l2spFxg7Pw\n3+0/qp0MV146jSmhY5q8zT1TxuRh3c5jgTdUiVS3o9ks4whvhe6jgbnpihwnFIICfdeuXfHuu+8C\nAPr3749Vq1Z5bFNSUoKSkhKX17Kzs7Fy5UrJt41E7aJDH8Kh+DAQ94dBCDee0OfK7CkF2P7tSWz4\n6njQxwKAe3/RH8v/WR7SPoKl5GQjYnRJl3YZ3EF9OuDz736SdJ9CdGqvXp8MKcVI3A9JDR1T4zB+\nWDb6ZKWi1tKodnLCFifM0ZCxRd0AADcOz1E5JZ6UmoM9VHndUiRZKEPqoEaeUlRa6nT8iJ5Bfc5f\nNjW+XVsmWs58mvOuY6LlC/Ri10J4ZlpREM+I1mP8/Kpsn8t1S+mK/r47iQphAPDcPUOkSYzCGOg1\nJLdrCv4waxQuy81weT2Yqtz+2dL2ZRg/LAf/N/0qSfcpijYLubplDLEZSK9MRgOigiwJ+7sEO4cw\nosQumPt8VEGXkI8bDPekduuQgPHD1C2gPDOtCHdc18fn+4P6dAhp/5kZ8UiIFTbyQmsY6DXGfrMn\nxYV2QRllaJ9KTVSnBAaEV5zv7FR17K2N0rlE+NKMobKkIcpsxAwZVxPs5bQqmSv9/pJyt6gIuWPd\nt0lPbidHUnSpW4cEDMvPlPUYer16Geg1qktG2xhZqUJ2Rqq252NWjYqF2/yc9n7fT4qLdvy/ncRV\ntQN7ydeJKDk+OvBGOiZHRlpLw2jce913yRBXY2E2eQ8toXYopuAw0OtBiPfGuKIs3D9+AAryMgJv\nLKEOEmYstNpBLVRiqtBfnuG6QldHZtwU5Rzb5ajC1VIIFNtG767kJvlqi6SQmhRc7aReH0MM9DoQ\nzC3n/FCKjzWjsE8HQfuRcvzsz6/KxqTRuW1pkmzP4c3XLxDlVs0fae3sznMspCQoX2Mg/9l2PcJt\nY/MCrqwo10iaQBUWUT5K7HbdOiRgwqjgOj1KZdzFzs3e3DS8p246GEuBgV4PgqwmtLepe1vC0e6O\na313XglVTLQJYwb5vtkiVX5P/9X1Uhcb2ie1tuPGC5zCVRIylHw6pbXVYGh1roFQuH+lqwu6YsrY\nPADA2EHd8LMrunuc1qsu6axM4uD6kxb27YjEIPoR+bu0pfpJf//4KCx7eDhGDPTdUTGunVn1zoNK\nYqAPUzYb8Pjky/CzK7pjWL7vh4GvtjStERv7hl8aWqccOWvobhrRE5kXh+8pMcTssUkD8bMruod8\nTqRW7KfE5U2y07oAk6/JQ9eMBNw2NvShlHpw6+hc3ORlWKDZZFRkaJo7k8mIO67tK+ozvbom4/HJ\nl/l8X6r8rdFocMw7MuTiEtRXKpgh8uXWq3updmx9POUjXLAZ3Y6pcbhpRE+vwXz2lAJMGNkTKSJ7\n0j82yfeNqhQhz4Pbr+3jUdWtFWaTEQvuGow/zBoVdBofvuVSwdt2uHgdaO18pCW1w2UCZxW747o+\n6Ncj1fF3l4x4/PbOInTLkG9hFzF+qcG5L6QWaoG79LbLXTIlSnTMu+qSznhpxlCMuizwMMS0JP8j\nGK51mlo6GGrWQmnrziev5Lg+8rql4Noh3UV/rm/3VMEPZyk8e/cQLLw7uEkqhHaYUvL2c+4Q6evG\nF5KRGeClt36yhCsPas2w/EyX86XGI9PfdK19u6f6fM8uUOdJqb/TvNsHYUaQneKkKF0Py89E905+\nFloRGegvDdTk5YPzyBV/bhrRE+OHZXt9b9bky3DlAP+1AvNuHyQ6bUphoCfRlKzu75QWh45pcS6d\nBPXcOmuvSvRH6Pl1n6Qlrp28Uxxnd04K6fPtg+zpLFRMtAmXBBiuGAp/QUsIf31lAAi6sMVc+907\nJQY9D7sUhYuE2CivwW/q2Dxc0b+T6Bqme8cPwAM3XhJ6wnyIa2fGz6/yHuhNAu7JUK8POTHQ61ha\nUjufE2bIWUsUdECRMU3+SgdD/fRR0KJ4gTURt8vYkdKbzPR4vFgS3AQ+U8bkYdG9V8o6PGnZQ8Px\nkIgmDbHcr7Dci5MC9cx0zQD5+oqB7kkhQ9p0OrrLxaiCrvjNDeJXfIuJMiEn00tmU8Gcv16H+TLQ\na1ivLq0Pko6p3hfhMJkMeP6+K11e6yFhrnKwj9KnFufiv+t6cR2DNO3iw6RzgMVX5JqO099EOsFO\nhNM+uZ0swwFdanoUbgO1H829ZGoKNh16rqpSk8Sx9+k7tFsFHywGeg2bcXM+7h8/AJf3Vm6iG+eH\nvK82xUSBbV5SCpSRlmKucdcDSru7YCgRuLwtjNKzS2hV9M4SYqNw58/6Bt2+KpfXHx4h+T5nTylA\nUd8OGNQ3uDnV5fq1RwzMVGXeATkoUaDO6ui7sBTM4UcVdMHvfjNY1UcKA72GJcRGtU50o2BJRfKA\nGcBLJUPx/L1XKHpMd3oZYiiHVyScS3/yNbmO0QD2zOm1Q7Jw1SWdHdewZJdyiPvxt/JbVofgevLn\ndUvBvb8YEPT1JNdt/utxfbDkAXnWTCBXA3ul4waniZ2A1lpWpZ+r7hReoJykMGJgJj79ugK9Lw5V\neXnGUJQfqoblQjN+qDiLwyfqkJHsWRq3t/2aTf6fKFf074ht5SeRk5mESdfkornZKv2XuChJ5TnR\nbxnVS9AQw0t7tseuH04rkCLgF8NycLCyFr8e5zlG/MWHRqDhvHTrdge7kps31xS2jYsf0r8Temel\n6rIkeft1ffDbP32hwpGVrbuPb2eG5UKz1/fSkmJQdeaC42+zMTIzw/b73jGdd4BiuX2hqD2HqnGo\nshZA6NMJS4GBXoemFvfG9Vf0QPuLHfES46Ix5OJay1f074juHRO9To6SkhCDxyddho5p/tt+77iu\nL4qLstCtQwLyfRQzXp4xFC1WGx5euiXEb+PK+XD3jx/g+L/UVXYlN16CKLMRA3La46xFusAphS7p\n8Xjhfu9LAvfqmoKqqjqFUxQcNVc7FKNX12QcOHZW9uOo3Y+ra0YCjlXVO/6+ckBnfPLFUa/b3ji8\nJ95YWw6gdeTL3T/vh5q6BpdtsjoqP4eB0p3hZtycjwuNLYgVOdVwWmIMDlW6vqZmuI/MbJrOGQ0G\nR5B3F9cuCsVFWT4vzD7dU10ewLExniU6s8mIrI6JfpsMEuOikZIQg2sHZ8k2bKrQaf1oKefgB4DL\n8jK8jkWPZPZOZWo1ZdjnGBC7UlqobpK6c6nAS/XJqZe7/C2k6r77xeA6rsj75C3+zt2gPq59fTLT\nfWf4Y2PMGNK/tTPu03cM8lr1HGiCmXBgMBhEB3ktYqCPcN07JmLSNbmYf2dRUJ+fMKoXnrit9YH1\ni6HZgnu/X5LTXtPD3iJtwRgAeOK21s5kI/3MEQ4EHg0QLPczLqT3sxTVomrNWNazSzKGOk3NKiQV\nce2i8IdZo3CL03Sq9qa4/j1ScduYPJ+fvXZId0z/Zds49GH5mS61Zu7uvqE/Vjw+EtFR0i6PLMZ9\nftKnBvc83M+uED/pmBoY6COcwWDAmMJu6BLCVKJpSe3w+1mj8Iuh3ieb8OaWUT0x7ToRQ+IUrvaU\nK5h5M8vP/N+BjAtxWk5nPTol4d5fDEBMtMnvyl5zfl3osiqhXDLT45GR0g7jRVxXcurfIw0Agl71\nLGB+QmCGwz1j8qvi3rgkpz2mFvuf999sMrqM4DEaDS61Zt6YVG6bHxQgfaGyt70nxAbXl0QvUx8z\n0JMkjDKXipRu3jQYDLhphDI3ce+swNOn+iJloHfmb0a1dtHmoDNCYppYzSYjFt17JX6uQqB3nlff\nLj0lFiseHynbqmfB3kEdUuPw0C2XokNqXFCdK6+53P9SuFoidRP9b6cVYfF9VwqeBCzaac6EWZMv\nk/25JxX9Nz6Qrtw3fgB27vsJndPVHW5CbW6/to/H6Aeh84OHq4cnDsS5C82Y8fJml9dlLeFKEDOy\nOyfiZ1d09zvpkbvJY/Lwv53H3F7VwEQSXkjdVyc6yoT2ycIzR7ExZpTceAmMRkNIGXSlMdCTpAK1\nmQ7q08FvdZzPT4d4fxv87CLuYmeb3t1SsO/oGcH7fOPRkbjnhY2hJUwDvI3QULqjldzz4F+Wm46v\n9p8SvL3RYHCZefD6Kz3bYn1dT2kivotzgVCKsmFrTZTncrZSENKX4cUHrkL9+SZZjq8Vl+UpN4GZ\nVFh1T5Lyl+MO5UEWak5+wW8Gu3R8chZlNmL5IyP8rpXt63NiSTlFcTiJjZFnOl+7zBBrkAr8PNzd\nA+A1hd0wUeDa472z2pZt1VY1sGda8rolY2CvdJT4WVgmOSEm6P4+iXGBrwEp531QihZ+VpboSVJq\njxX2pXP7eIy+vCvKvqn0+r4UPYsL+3RAcnw01nlUg7YyoLWdl9r07Z6Kr/afwqW92uPE6XNqJ0cS\nUWYjiouy8M76AwG3HdK/E37/770AgJ+5zaimNSaj0TEhjBzyuqXg9mv7ILdrMqLNJq+Z++T4aNw2\nNg9n6xvxr62HZUtLuGGgJ11wzkDY59rv3D4OlX6Cw33jB+Av/93nMSWlP0/fMQjxgZYT9SEjuR0m\njOrlCPR53VLwvVNTQDsZxuNqoLAQkqsv74qczGR075SA5e+XC/6cHr93opdFiJxL8cEuGBQuDAaD\n12Ykd1cXdMXXIpphiFX3JDElSvRx7cz4v+lX4ek7/I/979UlGb+9swgZIkrRWR0TvU5G5FxNf3nv\nDLz4gOfMde61Anf+TP4V9dSqQLF33gt1MhGjwYCczCTVh3EpYXShfnq3U3gJ/7uLFBXsFJVJF9vn\n4gSWplMTY4JqIxfD3h4YE2XCrU7jxrukxyM5wbPD1dhB3Txe0zN/PbezOibi/vEDgp5oKRJF66p9\nWaNtcDqhxvTA/rDqniQV7OPhidsux5Y9JzCkf0cMyEmDxa3nrn1qTyFVe77TJi51IwZm4ujJOowZ\n1C1gL/QpY/IElG59H3/pzOFoaGoRlT659c9Ow9cHfFeRBppsRayBuenY+X0VhvTrKOl+KQxpvO3m\n2iHd8d7mQ2onw4GBnhTjb3hOx7Q43HhxlqmUhBikuJWY49tFYcXjI0Oq4hVb2RATZcKd1/cL+nhi\nxLUzC560Q8/8NaNcdUln9M5KQfsQh/Z1bh+H+Ngox0I1yx8Z4bfjlinAao6RSdvnxD5LYbTZiEyF\n10YQwmwyYkB2GvYcqkbXEGYdlSw9aieAwoucq0uF2o6r1REBkaR7gOGF6V6WV/bqYqbRWy3NgrsG\nAwDuXLQBgO8RFbOnFOCr/VXI6Zwk7JhhoqhvB3RMVW6KZzlEmY14a/bVaifDr5Kb8lF15nzIQzul\nwEBPktJyLFV6iUuSn7efVOgiNXndUpDXrXUce9eMeByrsgBorckR4pare+Hl1btlm6BGLvf+QlsL\nxYSrKLNRE0EeYGc8kpqGY6kcSYuOar2F2kUHDg7BDtsj36TKvD0zra1TobclWb3pmZmMVx4chpxM\n4e7n0L8AAAqOSURBVDUCSiwGRN71zkpBQmwUJl+jzm+QcnF58A4qzKXBEj1JSsulZjnSNudXhSj7\nphKDA3QgG5rfGdfKtABNJJPqJ1Vqqdoxg7rhb+v2K3Ks0GjnPk5NjEFNXUPI+4mNMeOVB4cJ2vbp\nOwbBbJK2HHx57wzcNjYPl+UqP4UuAz1JSjuPB09y5EG6ZCRg4tWeJYTUxBiXcyFqSV4SzN9PeuvV\nvZAqomNfbIwZvboke33PHAHj/AFg+i8vwYavjqFv9zS1k+Lw/H1XoKVF2SdLVkfpp6o2Ggy4ukCd\nuRQY6ElS3oLpb27oh8Mn6xEvQ6/y3/1msODSmFK1DcsfGQGTyYDTtaGXQsLRL4fnhHwt2H9xf7/p\n2CJxNShLZw7zuJZmTynANwdPO4Z3hkJo27+aLu+d4bJmvRaYjEZIXLiOOAz0JLsr+nfCz0cmoqqq\nTvJ9C21PVZIU8+aHMzFTEgciZfWqtwyjc4e9YD18y6U4VmXR3CQqFDkY6ElSWm6jtyqcNG2PRA6s\nWwftB6bcrskYV5SluVKos64dEjAgp73ayaAIJig7vGvXLkydOhUAcOTIEUyaNAmTJ0/GvHnzYLVa\nAQBLly7FzTffjFtvvRW7d++WdVvSLu2GeYTcSD8gu7XdUou1CHLI65aCKWPy1E6GXwaDAbdc3Qs9\nfbStE5GAEv2KFSuwdu1axMa2DglYuHAhZs6cicGDB2Pu3LlYt24dMjMzsWPHDqxevRqVlZUoKSnB\nmjVrZNuWtEvDBXpYQ/z8feMHYN/RM8jvGTmls0AT3OiFfRGeTmn6nihGLr+dVoRma6h3CGlVwECf\nlZWFV199FY8//jgAoLy8HEVFrWNOhw8fji1btiA7OxtDhw6FwWBAZmYmWlpaUF1dLdu2aWna6RFK\nbrQc6UNMWmyM2e9CLyS/Xw7Lxtbyk0hL8lxUyJ+RA7vgQkMzrrqks0wp803qYVpy6KqDZhoKXsBA\nX1xcjGPHjjn+ttlsjk4r8fHxqKurQ319PVJS2jqs2F+Xa9tAgT41NQ5miVeKysgIj5KN3Ab27YRV\n6w/gyvzOHudM7XOYdPq84/9KpKXFaUiWlMdz3ld0fYPX16U67mlL2+JCav9+ADBtfD6mjQ/+s3ZK\nfJdXHhmJI5W1yM7y/rzS2v0hlnN6haRdb99PCUqdE9Gd8YxODy+LxYKkpCQkJCTAYrG4vJ6YmCjb\ntoHU1JwT+7X8ysiQp8d4OOqQGI35dw1Gx9RYl3OmhXOYHNua+Rt1WRdF0lJ9pi1jIdXx3M9j7blG\nv8cI9bg1Z9ruJbV/P6kodS0mRBnRPyvF57Hsr08dm4eTNed1dX7dz6GQtOvp+ylB6uvQX6ZBdJ1S\nv379sH37dgDApk2bUFhYiIKCApSVlcFqtaKiogJWqxVpaWmybUva1iU9XpPVlUlx0XjzsZGYWtxb\n7aQQOYwq6IpbOTUuyUh0iX7WrFmYM2cOlixZgpycHBQXF8NkMqGwsBATJ06E1WrF3LlzZd2WKFha\nzICEQu9D+IhIfoICfdeuXfHuu+8CALKzs7Fy5UqPbUpKSlBSUuLymlzbElGrhNgoDL+0M/pkpaqd\nFCKvbhubh4bGFrWTEdE4YQ6RjhkMBtx+LefRJ+1Sa353ahNe9ZhERETkgiV6ojBUetvlqHPqkU9E\nkYuBnigM9erKKWG16pfDc5CRInz5XKJQMdATkU89OiWiT1YKhuYrP6NcuJJy9T61zb+zKOxGsoQj\nBnoi8slsMuLxyQVqJ4M0qksGp87VA2bFiIiIwhgDPRERURhjoCciIgpjDPREMkm+uAb65XkZKqeE\niCIZO+MRySQ6yoQ3HxsJk5Ez0hORehjoiWTEoUdEpDY+hYiIiMIYAz0REVEYY6AnIiIKYwz0RERE\nYYyBnoiIKIwx0BMREYUxBnoiIqIwxkBPREQUxhjoiYiIwhgDPRERURhjoCciIgpjBpvNZlM7EURE\nRCQPluiJiIjCGAM9ERFRGGOgJyIiCmMM9ERERGGMgZ6IiCiMMdATERGFMbPaCdAyq9WKp59+Gvv2\n7UN0dDQWLFiA7t27q50sTWlqakJpaSmOHz+OxsZG3HfffejVqxdmz54Ng8GA3NxczJs3D0ajEUuX\nLsXGjRthNptRWlqK/Px8HDlyxOu2kej06dO48cYb8dZbb8FsNvMcivTGG29g/fr1aGpqwqRJk1BU\nVMRzKEJTUxNmz56N48ePw2g0Yv78+bwORdq1axdeeOEFvP322z7Ph5hz523boNjIp48++sg2a9Ys\nm81ms3311Ve2e++9V+UUac/f//5324IFC2w2m81WU1NjGzFihO2ee+6xffbZZzabzWabM2eO7eOP\nP7bt2bPHNnXqVJvVarUdP37cduONN9psNpvXbSNRY2Oj7f7777eNHTvWduDAAZ5DkT777DPbPffc\nY2tpabHV19fbXnnlFZ5DkT755BPbjBkzbDabzVZWVmZ74IEHeA5FePPNN23XX3+9bcKECTabzfv5\nEHPufG0bjMjKbom0c+dODBs2DAAwcOBA7NmzR+UUac+4cePw4IMPAgBsNhtMJhPKy8tRVFQEABg+\nfDi2bt2KnTt3YujQoTAYDMjMzERLSwuqq6u9bhuJFi1ahFtvvRUdOnQAAJ5DkcrKypCXl4fp06fj\n3nvvxciRI3kORcrOzkZLSwusVivq6+thNpt5DkXIysrCq6++6vg71HPna9tgMND7UV9fj4SEBMff\nJpMJzc3NKqZIe+Lj45GQkID6+nrMmDEDM2fOhM1mg8FgcLxfV1fncS7tr3vbNtL84x//QFpamiNT\nCYDnUKSamhrs2bMHL7/8Mp555hk8+uijPIcixcXF4fjx47j22msxZ84cTJ06ledQhOLiYpjNba3h\noZ47X9sGg230fiQkJMBisTj+tlqtLj8ktaqsrMT06dMxefJk3HDDDVi8eLHjPYvFgqSkJI9zabFY\nkJiY6NKGZ9820qxZswYGgwHbtm3D3r17MWvWLJecO89hYCkpKcjJyUF0dDRycnIQExODEydOON7n\nOQzsT3/6E4YOHYpHHnkElZWV+PWvf42mpibH+zyH4ng7H2LOna9tg0pLUJ+KEAUFBdi0aRMA4Ouv\nv0ZeXp7KKdKeU6dOYdq0aXjsscdw8803AwD69euH7du3AwA2bdqEwsJCFBQUoKysDFarFRUVFbBa\nrUhLS/O6baT561//ipUrV+Ltt99G3759sWjRIgwfPpznUITLL78cmzdvhs1mw8mTJ3H+/HlcccUV\nPIciJCUlOQJJcnIympubeS+HINRz52vbYHBRGz/sve6///572Gw2PPvss+jZs6faydKUBQsW4MMP\nP0ROTo7jtSeffBILFixAU1MTcnJysGDBAphMJrz66qvYtGkTrFYrnnjiCRQWFuLQoUOYM2eOx7aR\naurUqXj66adhNBq9nheeQ9+ef/55bN++HTabDQ899BC6du3KcyiCxWJBaWkpqqqq0NTUhF/96lcY\nMGAAz6EIx44dw8MPP4x3333X5/kQc+68bRsMBnoiIqIwxqp7IiKiMMZAT0REFMYY6ImIiMIYAz0R\nEVEYY6AnIiIKYwz0REREYYyBnoiIKIwx0BMREYWx/w/jc70XuQcZ0QAAAABJRU5ErkJggg==\n", "text/plain": [ - "" + "" ] }, "metadata": {}, @@ -410,7 +417,9 @@ { "cell_type": "code", "execution_count": 12, - "metadata": {}, + "metadata": { + "collapsed": true + }, "outputs": [], "source": [ "theano.config.compute_test_value = 'raise'\n", @@ -440,16 +449,16 @@ "name": "stdout", "output_type": "stream", "text": [ - "Topic #0: don people just think know like god time good does\n", - "Topic #1: use edu file like windows data thanks drive does available\n", - "Topic #2: armenian people space armenians jews new turkish 000 university government\n", - "Topic #3: key chip encryption government clipper keys public law use bit\n", - "Topic #4: 00 10 15 20 25 11 17 16 55 12\n", - "Topic #5: like new ground edu don does 10 good air just\n", - "Topic #6: new like edu just good does 10 know interested don\n", - "Topic #7: new ground like edu just space 10 good does know\n", - "Topic #8: new just good like car ground edu does don com\n", - "Topic #9: ax max 75u cx a86 34u 145 b8f 2di g9v\n" + "Topic #0: file edu use program like drive windows does thanks know\n", + "Topic #1: 00 10 25 db 11 15 55 20 40 12\n", + "Topic #2: space like just new car time good don use know\n", + "Topic #3: ax max g9v 34u b8f ah pl cx 75u 145\n", + "Topic #4: team season game hockey year play league games just players\n", + "Topic #5: people don think like just know right make way does\n", + "Topic #6: key encryption chip government clipper use keys security people don\n", + "Topic #7: president mr think people going stephanopoulos know gun said ms\n", + "Topic #8: people armenian said armenians turkish war new government turkey time\n", + "Topic #9: god jesus believe people does bible think don just say\n" ] } ], @@ -483,8 +492,8 @@ "name": "stdout", "output_type": "stream", "text": [ - "CPU times: user 14.6 s, sys: 67.7 ms, total: 14.7 s\n", - "Wall time: 14.8 s\n", + "CPU times: user 14.7 s, sys: 87.2 ms, total: 14.8 s\n", + "Wall time: 15.1 s\n", "Topic #0: people gun armenian war armenians turkish states said state 000\n", "Topic #1: government people law mr president use don think right public\n", "Topic #2: space science nasa program data research center output earth launch\n", @@ -521,7 +530,9 @@ { "cell_type": "code", "execution_count": 15, - "metadata": {}, + "metadata": { + "collapsed": true + }, "outputs": [], "source": [ "def calc_pp(ws, thetas, beta, wix):\n", @@ -599,7 +610,9 @@ { "cell_type": "code", "execution_count": 16, - "metadata": {}, + "metadata": { + "collapsed": true + }, "outputs": [], "source": [ "def transform_pymc3(docs):\n", @@ -610,25 +623,20 @@ " return samples['theta'].mean(axis=0)" ] }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "The mean of the log predictive probability is about -6.00. " - ] - }, { "cell_type": "code", "execution_count": 17, - "metadata": {}, + "metadata": { + "scrolled": true + }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "CPU times: user 51.4 s, sys: 13.1 s, total: 1min 4s\n", - "Wall time: 36.6 s\n", - "Predictive log prob (pm3) = -6.0658545547850276\n" + "CPU times: user 26.1 s, sys: 8.07 s, total: 34.2 s\n", + "Wall time: 32.8 s\n", + "Predictive log prob (pm3) = -6.158563560819359\n" ] } ], @@ -641,7 +649,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "We compare the result with the scikit-learn LDA implemented The log predictive probability is comparable (-6.04) with AEVB-ADVI, and it shows good set of words in the estimated topics." + "We compare the result with the scikit-learn LDA implemented The log predictive probability is comparable (-6.15) with AEVB-ADVI, and it shows good set of words in the estimated topics." ] }, { @@ -653,9 +661,9 @@ "name": "stdout", "output_type": "stream", "text": [ - "CPU times: user 43.1 s, sys: 2.75 s, total: 45.8 s\n", - "Wall time: 39.4 s\n", - "Predictive log prob (sklearn) = -6.014771065227896\n" + "CPU times: user 33 s, sys: 94 ms, total: 33.1 s\n", + "Wall time: 33.3 s\n", + "Predictive log prob (sklearn) = -6.0147710652278965\n" ] } ], @@ -710,7 +718,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.6.1" + "version": "3.6.0" }, "latex_envs": { "bibliofile": "biblio.bib", From 19aeec98a00c9a6bb53091f97a31bc19cee540ae Mon Sep 17 00:00:00 2001 From: Maxim Kochurov Date: Mon, 26 Jun 2017 22:06:33 +0300 Subject: [PATCH 21/24] scale_cost to minibatch refactor --- pymc3/variational/opvi.py | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/pymc3/variational/opvi.py b/pymc3/variational/opvi.py index 4511012375..93c8e19cee 100644 --- a/pymc3/variational/opvi.py +++ b/pymc3/variational/opvi.py @@ -532,9 +532,8 @@ def __init__(self, local_rv=None, model=None, scale_cost_to_minibatch=False, random_seed=None, **kwargs): model = modelcontext(model) - self.scale_cost_to_minibatch = theano.shared(np.int8(0)) - if scale_cost_to_minibatch: - self.scale_cost_to_minibatch.set_value(1) + self._scale_cost_to_minibatch = theano.shared(np.int8(0)) + self.scale_cost_to_minibatch = scale_cost_to_minibatch if not isinstance(cost_part_grad_scale, theano.Variable): self.cost_part_grad_scale = theano.shared(pm.floatX(cost_part_grad_scale)) else: @@ -586,6 +585,14 @@ def get_transformed(v): local_names = property(lambda self: tuple(v.name for v in self.local_vars)) global_names = property(lambda self: tuple(v.name for v in self.global_vars)) + @property + def scale_cost_to_minibatch(self): + return bool(self._scale_cost_to_minibatch.get_value()) + + @scale_cost_to_minibatch.setter + def scale_cost_to_minibatch(self, value): + self._scale_cost_to_minibatch.set_value(int(bool(value))) + @staticmethod def _choose_alternative(part, loc, glob): if part == 'local': @@ -841,6 +848,8 @@ def set_size_and_deterministic(self, node, s, d): """ initial_local = self._initial_part_matrix('local', s, d) initial_global = self._initial_part_matrix('global', s, d) + + # optimizations if isinstance(s, int) and (s == 1) or s is None: node = theano.clone(node, { self.logp: self.single_symbolic_logp @@ -933,7 +942,7 @@ def normalizing_constant(self): }) t = self.set_size_and_deterministic(t, 1, 1) # remove random, we do not it here at all # if not scale_cost_to_minibatch: t=1 - t = tt.switch(self.scale_cost_to_minibatch, t, + t = tt.switch(self._scale_cost_to_minibatch, t, tt.constant(1, dtype=t.dtype)) return pm.floatX(t) From 80e7e470c7013631cac2120243ab42675e3ca3a2 Mon Sep 17 00:00:00 2001 From: Maxim Kochurov Date: Tue, 27 Jun 2017 12:56:43 +0300 Subject: [PATCH 22/24] get feedback and refactor code --- pymc3/distributions/dist_math.py | 63 -------------------------------- pymc3/tests/conftest.py | 6 +-- pymc3/variational/opvi.py | 8 ++-- 3 files changed, 7 insertions(+), 70 deletions(-) diff --git a/pymc3/distributions/dist_math.py b/pymc3/distributions/dist_math.py index 11d8bd4bd0..86b0ea9eca 100644 --- a/pymc3/distributions/dist_math.py +++ b/pymc3/distributions/dist_math.py @@ -11,7 +11,6 @@ import theano from .special import gammaln -from ..math import logdet as _logdet from pymc3.theanof import floatX f = floatX @@ -164,68 +163,6 @@ def log_normal(x, mean, **kwargs): return f(c) - tt.log(tt.abs_(std)) - (x - mean) ** 2 / (2. * std ** 2) -def log_normal_mv(x, mean, gpu_compat=False, **kwargs): - """ - Calculate logarithm of normal distribution at point `x` - with given `mean` and `sigma` matrix - - Parameters - ---------- - x : Tensor - point of evaluation - mean : Tensor - mean of normal distribution - kwargs : one of parameters `{cov, tau, chol}` - - Other Parameters - ---------------- - gpu_compat : False, because LogDet is not GPU compatible yet. - If this is set as true, the GPU compatible (but numerically unstable) log(det) is used. - - Notes - ----- - There are three variants for density parametrization. - They are: - 1) covariance matrix - `cov` - 2) precision matrix - `tau`, - 3) cholesky decomposition matrix - `chol` - ---- - """ - if gpu_compat: - def logdet(m): - return tt.log(tt.abs_(tt.nlinalg.det(m))) - else: - logdet = _logdet - - T = kwargs.get('tau') - S = kwargs.get('cov') - L = kwargs.get('chol') - check = sum(map(lambda a: a is not None, [T, S, L])) - if check > 1: - raise ValueError('more than one required kwarg is passed') - if check == 0: - raise ValueError('none of required kwarg is passed') - # avoid unnecessary computations - if L is not None: - S = L.dot(L.T) - T = tt.nlinalg.matrix_inverse(S) - log_det = -logdet(S) - elif T is not None: - log_det = logdet(T) - else: - T = tt.nlinalg.matrix_inverse(S) - log_det = -logdet(S) - delta = x - mean - k = f(S.shape[0]) - result = delta.dot(T) - if delta.ndim > 1: - result = tt.batched_dot(result, delta) - else: - result = result.dot(delta.T) - result += k * tt.log(2. * np.pi) - log_det - return -.5 * result - - def MvNormalLogp(): """Compute the log pdf of a multivariate normal distribution. diff --git a/pymc3/tests/conftest.py b/pymc3/tests/conftest.py index 8c2bf0c444..481fcba133 100644 --- a/pymc3/tests/conftest.py +++ b/pymc3/tests/conftest.py @@ -4,14 +4,14 @@ import pytest -@pytest.yield_fixture(scope="function", autouse=True) +@pytest.fixture(scope="function", autouse=True) def theano_config(): config = theano.configparser.change_flags(compute_test_value='raise') with config: yield -@pytest.yield_fixture(scope='function', autouse=True) +@pytest.fixture(scope='function', autouse=True) def exception_verbosity(): config = theano.configparser.change_flags( exception_verbosity='high') @@ -19,7 +19,7 @@ def exception_verbosity(): yield -@pytest.yield_fixture(scope='function', autouse=False) +@pytest.fixture(scope='function', autouse=False) def strict_float32(): if theano.config.floatX == 'float32': config = theano.configparser.change_flags( diff --git a/pymc3/variational/opvi.py b/pymc3/variational/opvi.py index 93c8e19cee..89904d7618 100644 --- a/pymc3/variational/opvi.py +++ b/pymc3/variational/opvi.py @@ -391,19 +391,19 @@ def cast_to_list(params): Parameters ---------- - params : {list|tuple|dict|theano.shared|None} + params : {dict|None} Returns ------- list """ if isinstance(params, dict): - return list(params.values()) + return list(t[1] for t in sorted(params.items(), key=lambda t: t[0])) elif params is None: return [] else: raise TypeError( - 'Unknown type %s for %r, need list, dict or shared variable') + 'Unknown type %s for %r, need dict or None') class TestFunction(object): @@ -871,7 +871,7 @@ def sample_dict_fn(self): [self.view_local(l_posterior, name) for name in self.local_names] ) - sample_fn = theano.function([theano.In(s, 'draws', 1)], sampled) + sample_fn = theano.function([s], sampled) def inner(draws=1): _samples = sample_fn(draws) From a4a04a4bbe07247aa2ac5afaa32d717791087de4 Mon Sep 17 00:00:00 2001 From: Maxim Kochurov Date: Wed, 28 Jun 2017 00:41:51 +0300 Subject: [PATCH 23/24] stein refactor and typo fix --- pymc3/variational/stein.py | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/pymc3/variational/stein.py b/pymc3/variational/stein.py index 9ab257d2c9..116f057a83 100644 --- a/pymc3/variational/stein.py +++ b/pymc3/variational/stein.py @@ -1,7 +1,7 @@ from theano import theano, tensor as tt from pymc3.variational.opvi import node_property from pymc3.variational.test_functions import rbf -from pymc3.theanof import memoize, floatX +from pymc3.theanof import memoize, floatX, change_flags __all__ = [ 'Stein' @@ -15,11 +15,10 @@ def __init__(self, approx, kernel=rbf, input_matrix=None, temperature=1): self._kernel_f = kernel if input_matrix is None: input_matrix = tt.matrix('stein_input_matrix') - input_matrix.tag.test_value = approx.random(10).tag.test_value + input_matrix.tag.test_value = approx.symbolic_random_total_matrix.tag.test_value self.input_matrix = input_matrix - @property - @memoize + @node_property def grad(self): n = floatX(self.input_matrix.shape[0]) temperature = self.temperature @@ -27,15 +26,13 @@ def grad(self): self.repulsive_part_grad) return svgd_grad / n - @property - @memoize + @node_property def density_part_grad(self): Kxy = self.Kxy dlogpdx = self.dlogp return tt.dot(Kxy, dlogpdx) - @property - @memoize + @node_property def repulsive_part_grad(self): t = self.approx.normalizing_constant dxkxy = self.dxkxy @@ -49,7 +46,7 @@ def Kxy(self): def dxkxy(self): return self._kernel()[1] - @property + @node_property def logp_norm(self): return self.approx.sized_symbolic_logp / self.approx.normalizing_constant @@ -71,5 +68,6 @@ def dlogp(self): return tt.concatenate([loc_grad, glob_grad], axis=-1) @memoize + @change_flags(compute_test_value='raise') def _kernel(self): return self._kernel_f(self.input_matrix) From 5b983a7e1688ce2f657afb6539636104acb8a95c Mon Sep 17 00:00:00 2001 From: Maxim Kochurov Date: Wed, 28 Jun 2017 03:21:02 +0300 Subject: [PATCH 24/24] test value for stein (cherry picked from commit cc4291e) --- pymc3/variational/stein.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pymc3/variational/stein.py b/pymc3/variational/stein.py index 116f057a83..242622f0af 100644 --- a/pymc3/variational/stein.py +++ b/pymc3/variational/stein.py @@ -65,6 +65,8 @@ def dlogp(self): {self.approx.symbolic_random_local_matrix: loc_random, self.approx.symbolic_random_global_matrix: glob_random} ) + loc_grad.tag.test_value = loc_random.tag.test_value + glob_grad.tag.test_value = glob_random.tag.test_value return tt.concatenate([loc_grad, glob_grad], axis=-1) @memoize