From 8fe9e1ca15a3d38933208b6227443113aebc59dd Mon Sep 17 00:00:00 2001 From: PrimozGodec Date: Tue, 2 Aug 2016 12:10:48 +0200 Subject: [PATCH 001/128] Implemented step logistic regression. --- .../widgets/utils/logistic_regression.py | 73 +++++++++++++++++++ 1 file changed, 73 insertions(+) create mode 100644 orangecontrib/educational/widgets/utils/logistic_regression.py diff --git a/orangecontrib/educational/widgets/utils/logistic_regression.py b/orangecontrib/educational/widgets/utils/logistic_regression.py new file mode 100644 index 00000000..75a49e27 --- /dev/null +++ b/orangecontrib/educational/widgets/utils/logistic_regression.py @@ -0,0 +1,73 @@ +import numpy as np + +from Orange.classification import Learner, Model + + +class LogisticRegression: + + x = None + y = None + theta = None + domain = None + + def __init__(self, alpha, theta=None, data=None): + self.alpha = alpha + self.set_data(data) + self.set_theta(theta) + + def set_data(self, data): + if data is not None: + self.x = data.X + self.y = data.Y + self.domain = data.domain + + def set_theta(self, theta): + self.theta = theta + + @property + def model(self): + return LogisticRegressionModel(self.theta, self.domain) + + def step(self): + grad = self.dj(self.theta) + self.theta -= self.alpha * grad + + def j(self, theta): + """ + Cost function for logistic regression + """ + yh = self.g(self.x.dot(theta)) + return -sum(np.log(self.y * yh + (1 - self.y) * (1 - yh))) + + def dj(self, theta): + """ + Gradient of the cost function with L2 regularization + """ + return (self.g(self.x.dot(theta)) - self.y).dot(self.x) + + @staticmethod + def g(z): + """ + sigmoid function + + Parameters + ---------- + z : array_like + values to evaluate with function + """ + + # limit values in z to avoid log with 0 produced by values almost 0 + z_mod = np.minimum(z, 100 * np.ones(len(z))) + z_mod = np.maximum(z_mod, -100 * np.ones(len(z))) + + return 1.0 / (1 + np.exp(- z_mod)) + + +class LogisticRegressionModel(Model): + + def __init__(self, theta, domain): + super().__init__(domain) + self.theta = theta + + def predict_storage(self, data): + return LogisticRegression.g(data.X.dot(self.theta)) From ecc4aa4e9b7be3024e95720cb449ca4bb375994a Mon Sep 17 00:00:00 2001 From: PrimozGodec Date: Tue, 2 Aug 2016 15:19:35 +0200 Subject: [PATCH 002/128] Partly implemented gradient desecent --- .../educational/widgets/owgradientdescent.py | 383 ++++++++++++++++++ .../widgets/utils/logistic_regression.py | 22 +- 2 files changed, 401 insertions(+), 4 deletions(-) create mode 100644 orangecontrib/educational/widgets/owgradientdescent.py diff --git a/orangecontrib/educational/widgets/owgradientdescent.py b/orangecontrib/educational/widgets/owgradientdescent.py new file mode 100644 index 00000000..f00da059 --- /dev/null +++ b/orangecontrib/educational/widgets/owgradientdescent.py @@ -0,0 +1,383 @@ +from math import isnan +from os import path + +import numpy as np +from Orange.widgets.utils import itemmodels +from PyQt4.QtCore import pyqtSlot, Qt +from PyQt4.QtGui import QSizePolicy, QPixmap, QColor, QIcon + +from Orange.classification import Model +from Orange.data import Table, ContinuousVariable, Domain, DiscreteVariable +from Orange.widgets import gui +from Orange.widgets import highcharts +from Orange.widgets import settings +from Orange.widgets.widget import OWWidget, Msg +from scipy.ndimage import gaussian_filter + +from orangecontrib.educational.widgets.utils.logistic_regression \ + import LogisticRegression + + +class Scatterplot(highcharts.Highchart): + """ + Scatterplot extends Highchart and just defines some sane defaults: + * enables scroll-wheel zooming, + * set callback functions for click (in empty chart), drag and drop + * enables moving of centroids points + * include drag_drop_js script by highchart + """ + + js_click_function = """/**/(function(event) { + window.pybridge.chart_clicked(event.xAxis[0].value, event.yAxis[0].value); + }) + """ + + # to make unit tesest + count_replots = 0 + + def __init__(self, click_callback, **kwargs): + + # read javascript for drag and drop + with open(path.join(path.dirname(__file__), 'resources', 'highcharts-contour.js'), 'r') as f: + contours_js = f.read() + + super().__init__(enable_zoom=True, + bridge=self, + enable_select='', + chart_events_click=self.js_click_function, + plotOptions_series_states_hover_enabled=False, + plotOptions_series_cursor="move", + javascript=contours_js, + **kwargs) + + self.click_callback = click_callback + + def chart(self, *args, **kwargs): + self.count_replots += 1 + super(Scatterplot, self).chart(*args, **kwargs) + + @pyqtSlot(float, float) + def chart_clicked(self, x, y): + self.click_callback(x, y) + + +class OWGradientDescent(OWWidget): + + name = "Gradient Descent" + description = "Widget demonstrates shows the procedure of gradient descent." + icon = "icons/InteractiveKMeans.svg" + want_main_area = True + + inputs = [("Data", Table, "set_data")] + outputs = [("Model", Model), + ("Coefficients", Table)] + + # selected attributes in chart + attr_x = settings.Setting('') + attr_y = settings.Setting('') + target_class = settings.Setting('') + + # models + x_var_model = None + y_var_model = None + + # function used in gradient descent + default_learner = LogisticRegression + learner = None + cost_grid = None + grid_size = 20 + + # data + data = None + selected_data = None + + class Warning(OWWidget.Warning): + to_few_features = Msg("Too few Continuous feature. Min 2 required") + no_class = Msg("No class provided or only one class variable") + + def __init__(self): + super().__init__() + + # var models + self.x_var_model = itemmodels.VariableListModel() + self.y_var_model = itemmodels.VariableListModel() + + # options box + policy = QSizePolicy(QSizePolicy.MinimumExpanding, QSizePolicy.Fixed) + + self.options_box = gui.widgetBox(self.controlArea) + opts = dict( + widget=self.options_box, master=self, orientation=Qt.Horizontal, + callback=self.restart, sendSelectedValue=True + ) + self.cbx = gui.comboBox(value='attr_x', label='X:', **opts) + self.cbx.setSizePolicy(policy) + self.cby = gui.comboBox(value='attr_y', label='Y:', **opts) + self.cby.setSizePolicy(policy) + self.target_class_combobox = gui.comboBox( + value='target_class', label='Target class: ', **opts) + self.target_class_combobox.setSizePolicy(policy) + + self.cbx.setModel(self.x_var_model) + self.cby.setModel(self.y_var_model) + + # graph in mainArea + self.scatter = Scatterplot(click_callback=self.graph_clicked, + xAxis_gridLineWidth=0, + yAxis_gridLineWidth=0, + title_text='', + tooltip_shared=False, + debug=True) + + gui.rubber(self.controlArea) + + # TODO: set false when end of development + # Just render an empty chart so it shows a nice 'No data to display' + self.scatter.chart() + self.mainArea.layout().addWidget(self.scatter) + + # set random learner + + def set_data(self, data): + """ + Function receives data from input and init part of widget if data + satisfy. Otherwise set empty plot and notice + user about that + + Parameters + ---------- + data : Table + Input data + """ + + def reset_combos(): + self.x_var_model[:] = [] + self.y_var_model[:] = [] + self.target_class_combobox.clear() + + def init_combos(): + """ + function initialize the combos with attributes + """ + reset_combos() + + c_vars = [var for var in data.domain.variables if var.is_continuous] + + self.x_var_model[:] = c_vars + self.y_var_model[:] = c_vars + + for i, var in enumerate(data.domain.class_var.values): + pix_map = QPixmap(60, 60) + color = tuple(data.domain.class_var.colors[i].tolist()) + pix_map.fill(QColor(*color)) + self.target_class_combobox.addItem(QIcon(pix_map), var) + + self.Warning.clear() + + # clear variables + self.xv = None + self.yv = None + self.cost_grid = None + + if data is None or len(data) == 0: + self.data = None + reset_combos() + self.set_empty_plot() + elif sum(True for var in data.domain.attributes + if isinstance(var, ContinuousVariable)) < 2: + self.data = None + reset_combos() + self.Warning.to_few_features() + self.set_empty_plot() + elif (data.domain.class_var is None or + len(data.domain.class_var.values) < 2): + self.data = None + reset_combos() + self.Warning.no_class() + self.set_empty_plot() + else: + self.data = data + init_combos() + self.attr_x = self.cbx.itemText(0) + self.attr_y = self.cbx.itemText(1) + self.target_class = self.target_class_combobox.itemText(0) + self.restart() + + def restart(self): + self.selected_data = self.select_data() + self.learner = self.default_learner(data=self.selected_data) + self.replot() + + def replot(self): + """ + This function performs complete replot of the graph + """ + if self.data is None: + return + + attr_x = self.data.domain[self.attr_x] + attr_y = self.data.domain[self.attr_y] + + optimal_theta = self.learner.optimized() + min_x = optimal_theta[0] - 3 + max_x = optimal_theta[0] + 3 + min_y = optimal_theta[1] - 3 + max_y = optimal_theta[1] + 3 + + options = dict(series=[]) + + # gradient and contour + options['series'] += self.plot_gradient_and_contour( + min_x, max_x, min_y, max_y) + + data = options['series'][0]['data'] + data = [d[2] for d in data] + min_value = np.min(data) + max_value = np.max(data) + + # highcharts parameters + kwargs = dict( + xAxis_title_text=attr_x.name, + yAxis_title_text=attr_y.name, + xAxis_min=min_x, + xAxis_max=max_x, + yAxis_min=min_y, + yAxis_max=max_y, + colorAxis=dict( + stops=[ + [min_value, "#ffffff"], + [max_value, "#ff0000"]], + tickInterval=1, max=max_value, min=min_value), + plotOptions_contour_colsize=(max_y - min_y) / 1000, + plotOptions_contour_rowsize=(max_x - min_x) / 1000, + tooltip_headerFormat="", + tooltip_pointFormat="%s: {point.x:.2f}
" + "%s: {point.y:.2f}" % + (self.attr_x, self.attr_y)) + + self.scatter.chart(options, **kwargs) + # hack to destroy the legend for coloraxis + + def plot_gradient_and_contour(self, x_from, x_to, y_from, y_to): + """ + Function constructs series for gradient and contour + Parameters + ---------- + x_from : float + Min grid x value + x_to : float + Max grid x value + y_from : float + Min grid y value + y_to : float + Max grid y value + Returns + ------- + list + List containing series with background gradient and contour + """ + + # grid for gradient + x = np.linspace(x_from, x_to, self.grid_size) + y = np.linspace(y_from, y_to, self.grid_size) + self.xv, self.yv = np.meshgrid(x, y) + thetas = np.column_stack((self.xv.flatten(), self.yv.flatten())) + + cost_values = np.vstack([self.learner.j(theta) for theta in thetas]) + + # results + self.cost_grid = cost_values.reshape(self.xv.shape) + + blurred = self.blur_grid(self.cost_grid) + + return self.plot_gradient(self.xv, self.yv, blurred) + + def plot_gradient(self, x, y, grid): + """ + Function constructs background gradient + """ + return [dict(data=[[x[j, k], y[j, k], grid[j, k]] for j in range(len(x)) + for k in range(y.shape[1])], + grid_width=self.grid_size, + type="contour")] + + def select_data(self): + """ + Function takes two selected columns from data table and merge them + in new Orange.data.Table + Returns + ------- + Table + Table with selected columns + """ + attr_x = self.data.domain[self.attr_x] + attr_y = self.data.domain[self.attr_y] + cols = [] + for attr in (attr_x, attr_y): + subset = self.data[:, attr] + cols.append(subset.X) + x = np.column_stack(cols) + domain = Domain( + [attr_x, attr_y], + [DiscreteVariable(name=self.data.domain.class_var.name, + values=[self.target_class, 'Others'])], + [self.data.domain.class_var]) + y = [(0 if d.get_class().value == self.target_class else 1) + for d in self.data] + + return Table(domain, x, y, self.data.Y[:, None]) + + # def plot_contour(self): + # """ + # Function constructs contour lines + # """ + # self.scatter.remove_contours() + # if self.contours_enabled: + # contour = Contour( + # self.xv, self.yv, self.blur_grid(self.probabilities_grid)) + # contour_lines = contour.contours( + # np.hstack( + # (np.arange(0.5, 0, - self.contour_step)[::-1], + # np.arange(0.5 + self.contour_step, 1, self.contour_step)))) + # # we want to have contour for 0.5 + # + # series = [] + # count = 0 + # for key, value in contour_lines.items(): + # for line in value: + # if len(line) > self.degree: + # # if less than degree interpolation fails + # tck, u = splprep( + # [list(x) for x in zip(*reversed(line))], + # s=0.001, k=self.degree, + # per=(len(line) + # if np.allclose(line[0], line[-1]) + # else 0)) + # new_int = np.arange(0, 1.01, 0.01) + # interpol_line = np.array(splev(new_int, tck)).T.tolist() + # else: + # interpol_line = line + # + # series.append(dict(data=self.labeled(interpol_line, count), + # color=self.contour_color, + # type="spline", + # lineWidth=0.5, + # showInLegend=False, + # marker=dict(enabled=False), + # name="%g" % round(key, 2), + # enableMouseTracking=False + # )) + # count += 1 + # self.scatter.add_series(series) + # self.scatter.redraw_series() + + @staticmethod + def blur_grid(grid): + filtered = gaussian_filter(grid, sigma=1) + filtered[(grid > 0.45) & (grid < 0.55)] = grid[(grid > 0.45) & + (grid < 0.55)] + return filtered + + def graph_clicked(self, x, y): + pass + diff --git a/orangecontrib/educational/widgets/utils/logistic_regression.py b/orangecontrib/educational/widgets/utils/logistic_regression.py index 75a49e27..ad24474f 100644 --- a/orangecontrib/educational/widgets/utils/logistic_regression.py +++ b/orangecontrib/educational/widgets/utils/logistic_regression.py @@ -1,6 +1,7 @@ import numpy as np -from Orange.classification import Learner, Model +from Orange.classification import Model +from scipy.optimize import fmin_l_bfgs_b class LogisticRegression: @@ -10,8 +11,8 @@ class LogisticRegression: theta = None domain = None - def __init__(self, alpha, theta=None, data=None): - self.alpha = alpha + def __init__(self, alpha=0.1, theta=None, data=None): + self.set_alpha(alpha) self.set_data(data) self.set_theta(theta) @@ -24,6 +25,9 @@ def set_data(self, data): def set_theta(self, theta): self.theta = theta + def set_alpha(self, alpha): + self.alpha = alpha + @property def model(self): return LogisticRegressionModel(self.theta, self.domain) @@ -36,8 +40,9 @@ def j(self, theta): """ Cost function for logistic regression """ + # TODO: modify for more thetas yh = self.g(self.x.dot(theta)) - return -sum(np.log(self.y * yh + (1 - self.y) * (1 - yh))) + return -sum(np.log(self.y * yh + (1 - self.y) * (1 - yh))) / len(yh) def dj(self, theta): """ @@ -45,6 +50,15 @@ def dj(self, theta): """ return (self.g(self.x.dot(theta)) - self.y).dot(self.x) + def optimized(self): + """ + Function performs model training + """ + res = fmin_l_bfgs_b(self.j, + np.zeros(self.x.shape[1]), + self.dj) + return res[0] + @staticmethod def g(z): """ From a73ac6793420de5ed9fa5ce3aab3b1b29d73b327 Mon Sep 17 00:00:00 2001 From: PrimozGodec Date: Tue, 2 Aug 2016 15:20:50 +0200 Subject: [PATCH 003/128] Added highcahrts-contour --- .../widgets/resources/highcharts-contour.js | 538 ++++++++++++++++++ 1 file changed, 538 insertions(+) create mode 100644 orangecontrib/educational/widgets/resources/highcharts-contour.js diff --git a/orangecontrib/educational/widgets/resources/highcharts-contour.js b/orangecontrib/educational/widgets/resources/highcharts-contour.js new file mode 100644 index 00000000..e1fdd64e --- /dev/null +++ b/orangecontrib/educational/widgets/resources/highcharts-contour.js @@ -0,0 +1,538 @@ +/** +* Highcharts plugin for contour curves +* +* Author: Paulo Costa +*/ + +(function (Highcharts) { + +"use strict"; + + +var defaultOptions = Highcharts.getOptions(), + each = Highcharts.each, + extendClass = Highcharts.extendClass, + merge = Highcharts.merge, + seriesTypes = Highcharts.seriesTypes, + wrap = Highcharts.wrap, + perspective = Highcharts.perspective, + eps = 0.0001, + SVG_NS = "http://www.w3.org/2000/svg", + XLINK_NS = "http://www.w3.org/1999/xlink", + gradient_id = 0; + +/** +* Extend the default options with map options +*/ + + +defaultOptions.plotOptions.contour = merge(defaultOptions.plotOptions.heatmap, { + marker: defaultOptions.plotOptions.scatter.marker, + turboThreshold:0 +}); + +/** +* Normalize a value into 0-1 range +*/ +Highcharts.ColorAxis.prototype.toRelativePosition = function(value) { + if (this.isLog) { + value = this.val2lin(value); + } + return (value - this.min) / ((this.max - this.min) || 1); +}; + +wrap(Highcharts.ColorAxis.prototype, 'setAxisSize', function (proceed) { + if (this.legendSymbol) { + proceed.call(this); + } else { + this.len = 100; + this.pos = 0; + } +}); + +Highcharts.Axis.prototype.drawCrosshair = function() {}; + +// The Heatmap series type +seriesTypes.contour = extendClass(seriesTypes.heatmap, { + type: 'contour', + hasPointSpecificOptions: true, + getSymbol: seriesTypes.scatter.prototype.getSymbol, + drawPoints: Highcharts.Series.prototype.drawPoints, + + init: function (chart) { + this.is3d = chart.is3d && chart.is3d(); + seriesTypes.scatter.prototype.init.apply(this, arguments); + }, + + bindAxes: function () { + if (this.is3d) { + this.axisTypes = ['xAxis', 'yAxis', 'zAxis', 'colorAxis']; + this.parallelArrays = ['x', 'y', 'z', 'value']; + } else { + this.axisTypes = ['xAxis', 'yAxis', 'colorAxis']; + this.parallelArrays = ['x', 'y', 'value']; + } + seriesTypes.scatter.prototype.bindAxes.apply(this, arguments); + }, + + //FIXME: Once https://github.com/highcharts/highcharts/pull/5497 has landed, this whole method can go away + translate: function () { + if (!this.is3d) { + seriesTypes.scatter.prototype.translate.apply(this, arguments); + return; + } + this.chart.options.chart.options3d.enabled = false; + seriesTypes.scatter.prototype.translate.apply(this, arguments); + this.chart.options.chart.options3d.enabled = true; + + var series = this, + chart = series.chart, + zAxis = series.zAxis; + + Highcharts.each(series.data, function (point) { + var p3d = { + x: point.plotX, + y: point.plotY, + z: zAxis.translate(zAxis.isLog && zAxis.val2lin ? zAxis.val2lin(point.z) : point.z) + }; + point.plotXold = p3d.x; + point.plotYold = p3d.y; + point.plotZold = p3d.z; + + p3d = perspective([p3d], chart, true)[0]; + point.plotX = p3d.x; + point.plotY = p3d.y; + point.plotZ = p3d.z; + }); + }, + + drawTriangle: function (triangle_data, points, edgeCount, show_edges, contours) { + var fill; + var chart = this.chart; + var renderer = this.chart.renderer; + var a = points[triangle_data.a]; + var b = points[triangle_data.b]; + var c = points[triangle_data.c]; + var abc = [a,b,c]; + + //Normalized values of the vertexes + var values = [ + this.colorAxis.toRelativePosition(a.value), + this.colorAxis.toRelativePosition(b.value), + this.colorAxis.toRelativePosition(c.value) + ]; + + //All vertexes have the same value/color + if (Math.abs(values[0] - values[1]) < eps && Math.abs(values[0] - values[2]) < eps) { + fill = this.colorAxis.toColor((a.value + b.value + c.value) / 3); + //Use a linear gradient to interpolate values/colors + } else { + //Find function where "Value = A*X + B*Y + C" at the 3 vertexes + var m = new Matrix([ + [a.plotX, a.plotY, 1, values[0]], + [b.plotX, b.plotY, 1, values[1]], + [c.plotX, c.plotY, 1, values[2]]]); + m.toReducedRowEchelonForm(); + var A = m.mtx[0][3]; + var B = m.mtx[1][3]; + var C = m.mtx[2][3]; + + //For convenience, we place our gradient control points at (k*A, k*B) + //We can find the value of K as: + // Value = A*X + B*Y + C = + // Value = A*(A*k) + B*(B*k) + C + // Value = A²*k + B²*k + C + // Value = k*(A² + B²) + C + // k = (Value - C) / (A² + B²) + var k0 = (0-C) / (A*A + B*B); + var k1 = (1-C) / (A*A + B*B); + var x1 = k0*A; + var y1 = k0*B; + var x2 = k1*A; + var y2 = k1*B; + + // Assign a linear gradient that interpolates all 3 vertexes + if (renderer.isSVG) { + //SVGRenderer implementation of gradient is slow and leaks memory -- Lets do it ourselves + var gradient = triangle_data.gradient; + if (!gradient) { + var gradient = triangle_data.gradient = document.createElementNS(SVG_NS, "linearGradient"); + gradient.setAttribute("id", "contour-gradient-id-" + (gradient_id++)); + renderer.defs.element.appendChild(gradient); + } + gradient.setAttributeNS(XLINK_NS, "xlink:href", this.base_gradient_id); + gradient.setAttribute("x1", x1); + gradient.setAttribute("y1", y1); + gradient.setAttribute("x2", x2); + gradient.setAttribute("y2", y2); + fill = 'url(' + renderer.url + '#' + gradient.getAttribute('id') + ')'; + } else { + fill = { + linearGradient: { + x1: x1, + y1: y1, + x2: x2, + y2: y2, + spreadMethod: 'pad', + gradientUnits:'userSpaceOnUse' + }, + stops: this.colorAxis.stops + }; + } + } + + + var path = [ + 'M', + a.plotX + ',' + a.plotY, + 'L', + b.plotX + ',' + b.plotY, + 'L', + c.plotX + ',' + c.plotY, + 'Z' + ]; + + if (triangle_data.shape) { + triangle_data.shape.attr({ + d: path, + fill: fill, + }); + } else { + triangle_data.shape = renderer.path(path) + .attr({ + 'shape-rendering': 'crispEdges', + fill: fill + }) + } + triangle_data.shape.add(this.surface_group); + + + + // Draw edges around the triangle and/or on contour curves + + var edge_path = []; + if (show_edges) { + var processEdge = function(a,b) { + if (!edgeCount[b + '-' + a]) { + if (edgeCount[a + '-' + b]-- == 1) { + edge_path.push( + 'M', + points[a].plotX + ',' + points[a].plotY, + 'L', + points[b].plotX + ',' + points[b].plotY); + } + } + } + processEdge(triangle_data.a,triangle_data.b); + processEdge(triangle_data.b,triangle_data.c); + processEdge(triangle_data.c,triangle_data.a); + } + + for (var contour_index=0; contour_index= Math.min(v1, v2) && tickValue < Math.max(v1, v2)) { + var q = (tickValue-v1)/(v2-v1); + contourVertexes.push([ + q*(x2-x1) + x1, + q*(y2-y1) + y1 + ]); + } + } + if (contourVertexes.length == 2) { + edge_path.push( + 'M', + contourVertexes[0][0] + ',' + contourVertexes[0][1], + 'L', + contourVertexes[1][0] + ',' + contourVertexes[1][1]); + } + } + } + + if (edge_path.length) { + if (triangle_data.edge) { + triangle_data.edge.attr({ + d: edge_path, + }); + } else { + triangle_data.edge = renderer.path(edge_path) + .attr({ + 'stroke-linecap': 'round', + 'stroke': 'black', + 'stroke-width': 1, + }) + } + triangle_data.edge.add(this.surface_group); + } else if (triangle_data.edge) { + triangle_data.edge.destroy(); + delete triangle_data.edge; + } + }, + drawGraph: function () { + var series = this, + i,j, + points = series.points, + options = this.options, + renderer = series.chart.renderer; + + if (!series.surface_group) { + series.surface_group = renderer.g().add(series.group); + series.triangles = []; + } + + //When creating a SVG, we create a "base" gradient with the right colors, + //And extend it on every triangle to define the orientation. + if (series.chart.renderer.isSVG && !this.base_gradient_id) { + var fake_rect = series.chart.renderer.rect(0,0,1,1).attr({ + fill: { + linearGradient: { + x1: 0, + y1: 0, + x2: 1, + y2: 0, + spreadMethod: 'pad', + gradientUnits:'userSpaceOnUse' + }, + stops: this.colorAxis.stops + } + }); + this.base_gradient_id = /(#.*)[)]/.exec(fake_rect.attr('fill'))[1]; + } + + var group = series.surface_group; + var triangle_count = 0; + + var egde_count = {}; + var validatePoint = function(p) { + return p && (typeof p.x === "number") && (typeof p.y === "number") && (typeof p.z === "number" || !series.is3d) && (typeof p.value === "number"); + }; + var appendEdge = function(a,b) { + egde_count[a+'-'+b] = (egde_count[a+'-'+b] || 0) + 1; + }; + var appendTriangle = function(a,b,c) { + if (validatePoint(points[a]) && validatePoint(points[b]) && validatePoint(points[c])) { + var triangle_data = series.triangles[triangle_count]; + if (!triangle_data) { + triangle_data = series.triangles[triangle_count] = {}; + } + triangle_count++; + + //Make sure the shape is counter-clockwise + if (shapeArea([points[a], points[b], points[c]], 'plotX', 'plotY') > 0) { + var tmp = a; + a = b; + b = tmp; + } + triangle_data.a = a; + triangle_data.b = b; + triangle_data.c = c; + + appendEdge(a,b); + appendEdge(b,c); + appendEdge(c,a); + + triangle_data.z_order = [(points[a].plotZ + points[b].plotZ + points[c].plotZ)/3]; + } + }; + + + var triangles = []; + if (options.triangles) { + for (i=0; i bestRowVal) { + bestRow = row; + bestRowVal = Math.abs(this.mtx[row][col]); + } + } + + //All zeros in this column :( + if (bestRow == null) { + for (var row=0; row Date: Wed, 3 Aug 2016 11:57:43 +0200 Subject: [PATCH 004/128] Basic implementation of the gradient descent. --- .../educational/widgets/owgradientdescent.py | 208 +++++++++----- .../educational/widgets/utils/contour.py | 265 ++++++++++++++++++ .../widgets/utils/logistic_regression.py | 7 +- 3 files changed, 407 insertions(+), 73 deletions(-) create mode 100644 orangecontrib/educational/widgets/utils/contour.py diff --git a/orangecontrib/educational/widgets/owgradientdescent.py b/orangecontrib/educational/widgets/owgradientdescent.py index f00da059..7fd139c4 100644 --- a/orangecontrib/educational/widgets/owgradientdescent.py +++ b/orangecontrib/educational/widgets/owgradientdescent.py @@ -5,6 +5,8 @@ from Orange.widgets.utils import itemmodels from PyQt4.QtCore import pyqtSlot, Qt from PyQt4.QtGui import QSizePolicy, QPixmap, QColor, QIcon +from scipy.interpolate import splev, splprep +from scipy.ndimage import gaussian_filter from Orange.classification import Model from Orange.data import Table, ContinuousVariable, Domain, DiscreteVariable @@ -12,10 +14,11 @@ from Orange.widgets import highcharts from Orange.widgets import settings from Orange.widgets.widget import OWWidget, Msg -from scipy.ndimage import gaussian_filter +from Orange.preprocess.preprocess import Normalize from orangecontrib.educational.widgets.utils.logistic_regression \ import LogisticRegression +from orangecontrib.educational.widgets.utils.contour import Contour class Scatterplot(highcharts.Highchart): @@ -28,9 +31,9 @@ class Scatterplot(highcharts.Highchart): """ js_click_function = """/**/(function(event) { - window.pybridge.chart_clicked(event.xAxis[0].value, event.yAxis[0].value); - }) - """ + window.pybridge.chart_clicked(event.xAxis[0].value, event.yAxis[0].value); + }) + """ # to make unit tesest count_replots = 0 @@ -46,7 +49,6 @@ def __init__(self, click_callback, **kwargs): enable_select='', chart_events_click=self.js_click_function, plotOptions_series_states_hover_enabled=False, - plotOptions_series_cursor="move", javascript=contours_js, **kwargs) @@ -60,6 +62,24 @@ def chart(self, *args, **kwargs): def chart_clicked(self, x, y): self.click_callback(x, y) + def remove_series(self, id): + self.evalJS(""" + series = chart.get('{id}'); + if (series != null) + series.remove(true); + """.format(id=id)) + + def add_series(self, series): + for i, s in enumerate(series): + self.exposeObject('series%d' % i, series[i]) + self.evalJS("chart.addSeries(series%d, true);" % i) + + def add_point_to_series(self, id, x, y): + self.evalJS(""" + series = chart.get('{id}'); + series.addPoint([{x}, {y}]); + """.format(id=id, x=x, y=y)) + class OWGradientDescent(OWWidget): @@ -76,6 +96,7 @@ class OWGradientDescent(OWWidget): attr_x = settings.Setting('') attr_y = settings.Setting('') target_class = settings.Setting('') + alpha = settings.Setting(0.1) # models x_var_model = None @@ -85,7 +106,8 @@ class OWGradientDescent(OWWidget): default_learner = LogisticRegression learner = None cost_grid = None - grid_size = 20 + grid_size = 15 + contour_color = "#aaaaaa" # data data = None @@ -121,8 +143,26 @@ def __init__(self): self.cbx.setModel(self.x_var_model) self.cby.setModel(self.y_var_model) + self.properties_box = gui.widgetBox(self.controlArea) + self.alpha_spin = gui.spin(widget=self.properties_box, + master=self, + callback=self.change_alpha, + value="alpha", + label="Alpha: ", + minv=0.01, + maxv=1, + step=0.01, + spinType=float) + + self.comand_box = gui.widgetBox(self.controlArea) + + self.step_buttton = gui.button(widget=self.comand_box, + master=self, + callback=self.step, + label="Step") + # graph in mainArea - self.scatter = Scatterplot(click_callback=self.graph_clicked, + self.scatter = Scatterplot(click_callback=self.set_theta, xAxis_gridLineWidth=0, yAxis_gridLineWidth=0, title_text='', @@ -203,11 +243,31 @@ def init_combos(): self.target_class = self.target_class_combobox.itemText(0) self.restart() + def set_empty_plot(self): + self.scatter.clear() + def restart(self): self.selected_data = self.select_data() - self.learner = self.default_learner(data=self.selected_data) + self.learner = self.default_learner(data=Normalize(self.selected_data)) self.replot() + def change_alpha(self): + if self.learner is not None: + self.learner.set_alpha(self.alpha) + + def step(self): + if self.data is None: + return + if self.learner.theta is None: + self.set_theta(np.random.uniform(self.min_x, self.max_x), + np.random.uniform(self.min_y, self.max_y)) + self.learner.step() + theta = self.learner.theta + self.plot_point(theta[0], theta[1]) + + def plot_point(self, x, y): + self.scatter.add_point_to_series("path", x, y) + def replot(self): """ This function performs complete replot of the graph @@ -219,44 +279,48 @@ def replot(self): attr_y = self.data.domain[self.attr_y] optimal_theta = self.learner.optimized() - min_x = optimal_theta[0] - 3 - max_x = optimal_theta[0] + 3 - min_y = optimal_theta[1] - 3 - max_y = optimal_theta[1] + 3 + self.min_x = optimal_theta[0] - 5 + self.max_x = optimal_theta[0] + 5 + self.min_y = optimal_theta[1] - 5 + self.max_y = optimal_theta[1] + 5 options = dict(series=[]) # gradient and contour options['series'] += self.plot_gradient_and_contour( - min_x, max_x, min_y, max_y) + self.min_x, self.max_x, self.min_y, self.max_y) - data = options['series'][0]['data'] - data = [d[2] for d in data] - min_value = np.min(data) - max_value = np.max(data) + + min_value = np.min(self.cost_grid) + max_value = np.max(self.cost_grid) # highcharts parameters kwargs = dict( xAxis_title_text=attr_x.name, yAxis_title_text=attr_y.name, - xAxis_min=min_x, - xAxis_max=max_x, - yAxis_min=min_y, - yAxis_max=max_y, + xAxis_min=self.min_x, + xAxis_max=self.max_x, + yAxis_min=self.min_y, + yAxis_max=self.max_y, + xAxis_startOnTick=False, + xAxis_endOnTick=False, + yAxis_startOnTick=False, + yAxis_endOnTick=False, colorAxis=dict( stops=[ [min_value, "#ffffff"], [max_value, "#ff0000"]], tickInterval=1, max=max_value, min=min_value), - plotOptions_contour_colsize=(max_y - min_y) / 1000, - plotOptions_contour_rowsize=(max_x - min_x) / 1000, + plotOptions_contour_colsize=(self.max_y - self.min_y) / 10000, + plotOptions_contour_rowsize=(self.max_x - self.min_x) / 10000, + tooltip_enabled=False, tooltip_headerFormat="", tooltip_pointFormat="%s: {point.x:.2f}
" "%s: {point.y:.2f}" % (self.attr_x, self.attr_y)) self.scatter.chart(options, **kwargs) - # hack to destroy the legend for coloraxis + def plot_gradient_and_contour(self, x_from, x_to, y_from, y_to): """ @@ -290,7 +354,8 @@ def plot_gradient_and_contour(self, x_from, x_to, y_from, y_to): blurred = self.blur_grid(self.cost_grid) - return self.plot_gradient(self.xv, self.yv, blurred) + # return self.plot_gradient(self.xv, self.yv, blurred) + \ + return self.plot_contour() def plot_gradient(self, x, y, grid): """ @@ -327,57 +392,56 @@ def select_data(self): return Table(domain, x, y, self.data.Y[:, None]) - # def plot_contour(self): - # """ - # Function constructs contour lines - # """ - # self.scatter.remove_contours() - # if self.contours_enabled: - # contour = Contour( - # self.xv, self.yv, self.blur_grid(self.probabilities_grid)) - # contour_lines = contour.contours( - # np.hstack( - # (np.arange(0.5, 0, - self.contour_step)[::-1], - # np.arange(0.5 + self.contour_step, 1, self.contour_step)))) - # # we want to have contour for 0.5 - # - # series = [] - # count = 0 - # for key, value in contour_lines.items(): - # for line in value: - # if len(line) > self.degree: - # # if less than degree interpolation fails - # tck, u = splprep( - # [list(x) for x in zip(*reversed(line))], - # s=0.001, k=self.degree, - # per=(len(line) - # if np.allclose(line[0], line[-1]) - # else 0)) - # new_int = np.arange(0, 1.01, 0.01) - # interpol_line = np.array(splev(new_int, tck)).T.tolist() - # else: - # interpol_line = line - # - # series.append(dict(data=self.labeled(interpol_line, count), - # color=self.contour_color, - # type="spline", - # lineWidth=0.5, - # showInLegend=False, - # marker=dict(enabled=False), - # name="%g" % round(key, 2), - # enableMouseTracking=False - # )) - # count += 1 - # self.scatter.add_series(series) - # self.scatter.redraw_series() + def plot_contour(self): + """ + Function constructs contour lines + """ + + contour = Contour( + self.xv, self.yv, self.blur_grid(self.cost_grid)) + contour_lines = contour.contours( + np.linspace(np.min(self.cost_grid), np.max(self.cost_grid), 10)) + + series = [] + count = 0 + for key, value in contour_lines.items(): + for line in value: + # if len(line) > 3: + # # if less than degree interpolation fails + # tck, u = splprep( + # [list(x) for x in zip(*reversed(line))], + # s=0.001, k=3, + # per=(len(line) + # if np.allclose(line[0], line[-1]) + # else 0)) + # new_int = np.arange(0, 1.01, 0.01) + # interpol_line = np.array(splev(new_int, tck)).T.tolist() + # else: + interpol_line = line + + series.append(dict(data=interpol_line, + color=self.contour_color, + type="spline", + lineWidth=0.5, + showInLegend=False, + marker=dict(enabled=False), + name="%g" % round(key, 2), + enableMouseTracking=False + )) + count += 1 + return series @staticmethod def blur_grid(grid): filtered = gaussian_filter(grid, sigma=1) - filtered[(grid > 0.45) & (grid < 0.55)] = grid[(grid > 0.45) & - (grid < 0.55)] return filtered - def graph_clicked(self, x, y): - pass + def set_theta(self, x, y): + if self.learner is not None: + self.learner.set_theta([x, y]) + self.scatter.remove_series("path") + self.scatter.add_series([ + dict(id="path", data=[[x, y]], showInLegend=False, + type="scatter", lineWidth=1, + marker=dict(enabled=True, radius=2))],) diff --git a/orangecontrib/educational/widgets/utils/contour.py b/orangecontrib/educational/widgets/utils/contour.py new file mode 100644 index 00000000..009dd6de --- /dev/null +++ b/orangecontrib/educational/widgets/utils/contour.py @@ -0,0 +1,265 @@ +import numpy as np + + +class Contour: + + # look corners table from + # https://en.wikipedia.org/wiki/Marching_squares#Isoline + # corners table is coded as move in clockwise direction + moves = { + 1: {"to": [1, 0], "from": [0, -1]}, # D + 2: {"to": [0, 1], "from": [1, 0]}, # R + 3: {"to": [0, 1], "from": [0, -1]}, # R + 4: {"to": [-1, 0], "from": [0, 1]}, # U + 6: {"to": [-1, 0], "from": [1, 0]}, # U + 7: {"to": [-1, 0], "from": [0, -1]}, # U + 8: {"to": [0, -1], "from": [-1, 0]}, # L + 9: {"to": [1, 0], "from": [-1, 0]}, # D + 11: {"to": [0, 1], "from": [-1, 0]}, # R + 12: {"to": [0, -1], "from": [0, 1]}, # L + 13: {"to": [1, 0], "from": [0, 1]}, # D + 14: {"to": [0, -1], "from": [1, 0]} # L + } + + moves_up = [4, 6, 7] + moves_down = [1, 9, 13] + moves_left = [8, 12, 14] + moves_right = [2, 3, 11] + + from_up = [8, 9, 11] + from_down = [2, 6, 14] + from_left = [1, 3, 7] + from_right = [4, 12, 13] + + def __init__(self, x, y, z): + self.x = np.array(x) + self.y = np.array(y) + self.z = np.array(z) + self.visited_points = None + + def contours(self, thresholds): + contours = {} + for t in thresholds: + points = self.find_contours(t) + if len(points) > 0: + contours[t] = points + return contours + + def find_contours(self, threshold): + contours = [] + bitmap = (self.z > threshold).astype(int) + self.visited_points = np.zeros(self.z.shape) + # check if contour start on edge (they have to touches the edge) + for i in range(bitmap.shape[0] - 1): + # left + sq_idx = self.corner_idx(bitmap[i:i+2, 0:2]) + upper = (False if sq_idx != 5 else True) + if sq_idx in [1, 3, 5, 7] and not self.visited(i, 0, upper): + contour = self.find_contour_path(bitmap, i, 0, threshold) + contours.append(contour) + # right + sq_idx = self.corner_idx( + bitmap[i:i+2, bitmap.shape[1]-2:bitmap.shape[1]]) + if sq_idx in [4, 5, 12, 13] and \ + not self.visited(i, bitmap.shape[1]-2, False): + contour = self.find_contour_path( + bitmap, i, bitmap.shape[1]-2, threshold) + contours.append(contour) + + for j in range(bitmap.shape[1] - 1): + # top + sq_idx = self.corner_idx(bitmap[0:2, j:j+2]) + upper = (False if sq_idx != 10 else True) + if sq_idx in [8, 9, 10, 11] and not self.visited(0, j, upper): + contours.append(self.find_contour_path(bitmap, 0, j, threshold)) + # bottom + sq_idx = self.corner_idx( + bitmap[bitmap.shape[0]-2:bitmap.shape[0], j:j+2]) + if sq_idx in [2, 6, 10, 14] and \ + not self.visited(bitmap.shape[0]-2, j, False): + contour = self.find_contour_path( + bitmap, bitmap.shape[0]-2, j, threshold) + contours.append(contour) + + nonzero_lines = np.nonzero( + bitmap.shape[1] - np.sum(bitmap[1:-1, :], axis=1))[0] + 1 + # 1:-1 to avoid double check edge + + for i in nonzero_lines: + for j in range(1, bitmap.shape[1] - 2): + sq_idx = self.corner_idx(bitmap[i:i+2, j:j+2]) + if sq_idx not in [0, 15] and not self.visited(i, j, False): + path = self.find_contour_path(bitmap, i, j, threshold) + contours.append(path) + return contours + + def find_contour_path(self, bitmap, start_i, start_j, threshold): + i, j = start_i, start_j + path = [self.to_real_coordinate( + self.start_point( + bitmap[i:i+2, j:j+2], np.array([i, j]), threshold))] + + previous_position = None + step = 0 + while 0 <= i < bitmap.shape[0] - 1 \ + and 0 <= j < bitmap.shape[1] - 1: + square = bitmap[i:i+2, j:j+2] + upper = (True if (self.corner_idx(square) in [5, 10] and + (previous_position is None or + previous_position[0] < i or + previous_position[1] < j)) else False) + + if self.visited(i, j, upper): + # i == start_i and j == start_j and step > 0 and + break # cycle + + new_p = self.new_point( + square, previous_position, np.array([i, j]), threshold) + path.append(self.to_real_coordinate(new_p)) + + self.mark_visited(i, j, upper) + previous_position_tmp = [i, j] + + i, j = self.new_position( + square, previous_position, np.array([i, j])) + previous_position = previous_position_tmp + step += 1 + return path + + def to_real_coordinate(self, point): + """ + Parameters + ---------- + point : list + List that contains point (x, y) in grid coordinate system + + Returns + ------- + list + """ + x_idx = int(point[1]) + y_idx = int(point[0]) + return [self.x[y_idx, x_idx] + + ((point[1] % 1) * (self.x[y_idx, x_idx + 1] - + self.x[y_idx, x_idx]) + if x_idx + 1 < self.x.shape[1] else 0), + self.y[y_idx, x_idx] + + ((point[0] % 1) * (self.y[y_idx + 1, x_idx] - + self.y[y_idx, x_idx]) + if y_idx + 1 < self.x.shape[0] else 0)] + + def new_point(self, sq, previous, position, threshold): + con_idx = self.corner_idx(sq) + if con_idx == 5: + goes_top = ((previous is None and + position[1] != self.z.shape[1] - 2) or + (previous is not None and + (previous[1] + 1 == position[1]))) + heat_from = self.z[position[0] + + (0 if goes_top else 1), position[1]] + heat_to = self.z[position[0] + + (0 if goes_top else 1), position[1] + 1] + pos = position + np.array( + [(0 if goes_top else 1), + self.triangulate(threshold, heat_from, heat_to)]) + elif con_idx == 10: + goes_right = ((previous is None and + position[0] != self.z.shape[0] - 2) or + (previous is not None and + (previous[0] + 1 == position[0]))) + heat_from = self.z[position[0], + position[1] + (1 if goes_right else 0)] + heat_to = self.z[position[0] + 1, + position[1] + (1 if goes_right else 0)] + pos = position + np.array( + [self.triangulate(threshold, heat_from, heat_to), + (1 if goes_right else 0)]) + else: + move_dimension = 0 if self.moves[con_idx]['to'][0] == 0 else 1 + pos = (position + np.array( + self.moves[con_idx]['to']).clip(min=0)).astype(float) + heat_from = self.z[ + (position[0] + 1 if con_idx in self.moves_down + else position[0]), + (position[1] + 1 if con_idx in self.moves_right + else position[1])] + heat_to = self.z[ + (position[0] if con_idx in self.moves_up else position[0] + 1), + (position[1] if con_idx in self.moves_left + else position[1] + 1)] + pos[move_dimension] += self.triangulate( + threshold, heat_from, heat_to) + + return pos.tolist() + + @staticmethod + def triangulate(threshold, heat_from, heat_to): + return ((threshold - heat_from) / (heat_to - heat_from)) \ + if heat_from < heat_to else \ + (1 - (threshold - heat_to) / (heat_from - heat_to)) + + def start_point(self, sq, position, threshold): + con_idx = self.corner_idx(sq) + if con_idx == 5: + from_left = position[1] != self.z.shape[1] - 2 + heat_from = self.z[position[0], + position[1] + (0 if from_left else 1)] + heat_to = self.z[position[0] + 1, + position[1] + (0 if from_left else 1)] + pos = position + np.array([self.triangulate( + threshold, heat_from, heat_to), + 0 if from_left else 1]) # left edge 0 every time, same right + elif con_idx == 10: + from_top = position[0] != self.z.shape[0] - 2 + heat_from = self.z[position[0] + + (0 if from_top else 1), position[1]] + heat_to = self.z[position[0] + + (0 if from_top else 1), position[1] + 1] + pos = position + np.array( + [0 if from_top else 1, + self.triangulate(threshold, heat_from, heat_to)]) + # on top edge 1 every time, same bottom + else: + move_dimension = 0 if self.moves[con_idx]['from'][0] == 0 else 1 + pos = (position + np.array( + self.moves[con_idx]['from']).clip(min=0)).astype(float) + heat_from = self.z[ + (position[0] + 1 if con_idx in self.from_down else position[0]), + (position[1] + 1 if con_idx in self.from_right + else position[1])] + heat_to = self.z[ + (position[0] if con_idx in self.from_up else position[0] + 1), + (position[1] if con_idx in self.from_left else position[1] + 1)] + pos[move_dimension] += self.triangulate( + threshold, heat_from, heat_to) + + return pos.tolist() + + @classmethod + def new_position(cls, sq, previous, position): + con_idx = cls.corner_idx(sq) + if con_idx == 5: + pos = (position + + np.array([(-1 if (previous is None or + previous[1] + 1 == position[1]) + else 1), 0])) + elif con_idx == 10: + pos = (position + + np.array([0, (1 if (previous is None or + previous[0] + 1 == position[0]) + else -1)])) + else: + pos = position + cls.moves[con_idx]['to'] + return pos.tolist() + + @staticmethod + def corner_idx(sq): + return np.sum(np.array([[8, 4], [1, 2]]) * sq) + + def visited(self, i, j, upper=True): + visited = self.visited_points[i, j] + return (visited in [1, 3] and upper) or (visited >= 2 and not upper) + + def mark_visited(self, i, j, upper=True): + if not self.visited(i, j, upper): + self.visited_points[i, j] += (1 if upper else 2) diff --git a/orangecontrib/educational/widgets/utils/logistic_regression.py b/orangecontrib/educational/widgets/utils/logistic_regression.py index ad24474f..ae6669a5 100644 --- a/orangecontrib/educational/widgets/utils/logistic_regression.py +++ b/orangecontrib/educational/widgets/utils/logistic_regression.py @@ -23,7 +23,12 @@ def set_data(self, data): self.domain = data.domain def set_theta(self, theta): - self.theta = theta + if isinstance(theta, (np.ndarray, np.generic)): + self.theta = theta + elif isinstance(theta, list): + self.theta = np.array(theta) + else: + self.theta = None def set_alpha(self, alpha): self.alpha = alpha From 6cf86802815dd318bf571a58912dfd9b0f1ab4ea Mon Sep 17 00:00:00 2001 From: PrimozGodec Date: Wed, 3 Aug 2016 13:02:57 +0200 Subject: [PATCH 005/128] Implemented step back. --- .../educational/widgets/owgradientdescent.py | 18 ++++++++++++++- .../widgets/utils/logistic_regression.py | 23 ++++++++++++++++++- 2 files changed, 39 insertions(+), 2 deletions(-) diff --git a/orangecontrib/educational/widgets/owgradientdescent.py b/orangecontrib/educational/widgets/owgradientdescent.py index 7fd139c4..737c3bca 100644 --- a/orangecontrib/educational/widgets/owgradientdescent.py +++ b/orangecontrib/educational/widgets/owgradientdescent.py @@ -69,6 +69,13 @@ def remove_series(self, id): series.remove(true); """.format(id=id)) + def remove_last_point(self, id): + self.evalJS(""" + series = chart.get('{id}'); + if (series != null) + series.removePoint(series.data.length - 1, true); + """.format(id=id)) + def add_series(self, series): for i, s in enumerate(series): self.exposeObject('series%d' % i, series[i]) @@ -156,10 +163,14 @@ def __init__(self): self.comand_box = gui.widgetBox(self.controlArea) - self.step_buttton = gui.button(widget=self.comand_box, + self.step_button = gui.button(widget=self.comand_box, master=self, callback=self.step, label="Step") + self.step_back_button = gui.button(widget=self.comand_box, + master=self, + callback=self.step_back, + label="Step") # graph in mainArea self.scatter = Scatterplot(click_callback=self.set_theta, @@ -265,6 +276,11 @@ def step(self): theta = self.learner.theta self.plot_point(theta[0], theta[1]) + def step_back(self): + if self.learner.step_no > 0: + self.learner.step_back() + self.scatter.remove_last_point("path") + def plot_point(self, x, y): self.scatter.add_point_to_series("path", x, y) diff --git a/orangecontrib/educational/widgets/utils/logistic_regression.py b/orangecontrib/educational/widgets/utils/logistic_regression.py index ae6669a5..34434b56 100644 --- a/orangecontrib/educational/widgets/utils/logistic_regression.py +++ b/orangecontrib/educational/widgets/utils/logistic_regression.py @@ -10,8 +10,10 @@ class LogisticRegression: y = None theta = None domain = None + step_no = 0 def __init__(self, alpha=0.1, theta=None, data=None): + self.history = [] self.set_alpha(alpha) self.set_data(data) self.set_theta(theta) @@ -29,6 +31,8 @@ def set_theta(self, theta): self.theta = np.array(theta) else: self.theta = None + self.history = self.set_list(self.history, 0, np.copy(self.theta)) + self.step_no = 0 def set_alpha(self, alpha): self.alpha = alpha @@ -38,8 +42,15 @@ def model(self): return LogisticRegressionModel(self.theta, self.domain) def step(self): + self.step_no += 1 grad = self.dj(self.theta) self.theta -= self.alpha * grad + self.history = self.set_list(self.history, self.step_no, np.copy(self.theta)) + + def step_back(self): + if self.step_no > 0: + self.step_no -= 1 + self.theta = np.copy(self.history[self.step_no]) def j(self, theta): """ @@ -47,7 +58,8 @@ def j(self, theta): """ # TODO: modify for more thetas yh = self.g(self.x.dot(theta)) - return -sum(np.log(self.y * yh + (1 - self.y) * (1 - yh))) / len(yh) + # return -sum(np.log(self.y * yh + (1 - self.y) * (1 - yh))) / len(yh) + return -sum(self.y * np.log(yh) + (1 - self.y) * np.log(1 - yh)) / len(yh) def dj(self, theta): """ @@ -81,6 +93,15 @@ def g(z): return 1.0 / (1 + np.exp(- z_mod)) + @staticmethod + def set_list(l, i, v): + try: + l[i] = v + except IndexError: + for _ in range(i-len(l)): + l.append(None) + l.append(v) + return l class LogisticRegressionModel(Model): From 42588134164a2ab4e35a55313cf8d1126e1cdc39 Mon Sep 17 00:00:00 2001 From: PrimozGodec Date: Wed, 3 Aug 2016 13:48:12 +0200 Subject: [PATCH 006/128] Auto step. --- .../educational/widgets/owgradientdescent.py | 116 +++++++++++++++--- .../widgets/utils/logistic_regression.py | 6 + 2 files changed, 106 insertions(+), 16 deletions(-) diff --git a/orangecontrib/educational/widgets/owgradientdescent.py b/orangecontrib/educational/widgets/owgradientdescent.py index 737c3bca..e58df245 100644 --- a/orangecontrib/educational/widgets/owgradientdescent.py +++ b/orangecontrib/educational/widgets/owgradientdescent.py @@ -1,9 +1,9 @@ -from math import isnan from os import path +import time import numpy as np from Orange.widgets.utils import itemmodels -from PyQt4.QtCore import pyqtSlot, Qt +from PyQt4.QtCore import pyqtSlot, Qt, QThread, SIGNAL from PyQt4.QtGui import QSizePolicy, QPixmap, QColor, QIcon from scipy.interpolate import splev, splprep from scipy.ndimage import gaussian_filter @@ -88,6 +88,33 @@ def add_point_to_series(self, id, x, y): """.format(id=id, x=x, y=y)) +class Autoplay(QThread): + """ + Class used for separated thread when using "Autoplay" for k-means + Parameters + ---------- + owkmeans : OWKmeans + Instance of OWKmeans class + """ + + def __init__(self, ow_gradient_descent): + QThread.__init__(self) + self.ow_gradient_descent = ow_gradient_descent + + def __del__(self): + self.wait() + + def run(self): + """ + Stepping through the algorithm until converge or user interrupts + """ + while (not self.ow_gradient_descent.learner.converged and + self.ow_gradient_descent.auto_play_enabled): + self.emit(SIGNAL('step()')) + time.sleep(2 - self.ow_gradient_descent.auto_play_speed) + self.emit(SIGNAL('stop_auto_play()')) + + class OWGradientDescent(OWWidget): name = "Gradient Descent" @@ -104,6 +131,7 @@ class OWGradientDescent(OWWidget): attr_y = settings.Setting('') target_class = settings.Setting('') alpha = settings.Setting(0.1) + auto_play_speed = settings.Setting(1) # models x_var_model = None @@ -120,6 +148,10 @@ class OWGradientDescent(OWWidget): data = None selected_data = None + # autoplay + auto_play_enabled = False + autoplay_button_text = ["Run", "Stop"] + class Warning(OWWidget.Warning): to_few_features = Msg("Too few Continuous feature. Min 2 required") no_class = Msg("No class provided or only one class variable") @@ -134,7 +166,7 @@ def __init__(self): # options box policy = QSizePolicy(QSizePolicy.MinimumExpanding, QSizePolicy.Fixed) - self.options_box = gui.widgetBox(self.controlArea) + self.options_box = gui.widgetBox(self.controlArea, "Data") opts = dict( widget=self.options_box, master=self, orientation=Qt.Horizontal, callback=self.restart, sendSelectedValue=True @@ -150,7 +182,7 @@ def __init__(self): self.cbx.setModel(self.x_var_model) self.cby.setModel(self.y_var_model) - self.properties_box = gui.widgetBox(self.controlArea) + self.properties_box = gui.widgetBox(self.controlArea, "Properties") self.alpha_spin = gui.spin(widget=self.properties_box, master=self, callback=self.change_alpha, @@ -160,17 +192,35 @@ def __init__(self): maxv=1, step=0.01, spinType=float) + self.restart_button = gui.button(widget=self.properties_box, + master=self, + callback=self.restart, + label="Restart") - self.comand_box = gui.widgetBox(self.controlArea) + self.step_box = gui.widgetBox(self.controlArea, "Manually step through") - self.step_button = gui.button(widget=self.comand_box, + self.step_button = gui.button(widget=self.step_box, master=self, callback=self.step, label="Step") - self.step_back_button = gui.button(widget=self.comand_box, + self.step_back_button = gui.button(widget=self.step_box, master=self, callback=self.step_back, - label="Step") + label="Step back") + + self.run_box = gui.widgetBox(self.controlArea, "Run") + self.auto_play_button = gui.button( + self.run_box, self, self.autoplay_button_text[0], + callback=self.auto_play) + self.auto_play_speed_spinner = gui.hSlider(self.run_box, + self, + 'auto_play_speed', + minValue=0, + maxValue=1.91, + step=0.1, + intOnly=False, + createLabel=False, + label='Speed:') # graph in mainArea self.scatter = Scatterplot(click_callback=self.set_theta, @@ -259,7 +309,8 @@ def set_empty_plot(self): def restart(self): self.selected_data = self.select_data() - self.learner = self.default_learner(data=Normalize(self.selected_data)) + self.learner = self.default_learner(data=Normalize(self.selected_data), + alpha=self.alpha) self.replot() def change_alpha(self): @@ -312,8 +363,8 @@ def replot(self): # highcharts parameters kwargs = dict( - xAxis_title_text=attr_x.name, - yAxis_title_text=attr_y.name, + xAxis_title_text="theta 0", + yAxis_title_text="theta 1", xAxis_min=self.min_x, xAxis_max=self.max_x, yAxis_min=self.min_y, @@ -322,11 +373,11 @@ def replot(self): xAxis_endOnTick=False, yAxis_startOnTick=False, yAxis_endOnTick=False, - colorAxis=dict( - stops=[ - [min_value, "#ffffff"], - [max_value, "#ff0000"]], - tickInterval=1, max=max_value, min=min_value), + # colorAxis=dict( + # stops=[ + # [min_value, "#ffffff"], + # [max_value, "#ff0000"]], + # tickInterval=1, max=max_value, min=min_value), plotOptions_contour_colsize=(self.max_y - self.min_y) / 10000, plotOptions_contour_rowsize=(self.max_x - self.min_x) / 10000, tooltip_enabled=False, @@ -461,3 +512,36 @@ def set_theta(self, x, y): type="scatter", lineWidth=1, marker=dict(enabled=True, radius=2))],) + def auto_play(self): + """ + Function called when autoplay button pressed + """ + self.auto_play_enabled = not self.auto_play_enabled + self.auto_play_button.setText( + self.autoplay_button_text[self.auto_play_enabled]) + if self.auto_play_enabled: + self.disable_controls(self.auto_play_enabled) + self.autoPlayThread = Autoplay(self) + self.connect(self.autoPlayThread, SIGNAL("step()"), self.step) + self.connect( + self.autoPlayThread, SIGNAL("stop_auto_play()"), + self.stop_auto_play) + self.autoPlayThread.start() + else: + self.stop_auto_play() + + def stop_auto_play(self): + """ + Called when stop autoplay button pressed or in the end of autoplay + """ + self.auto_play_enabled = False + self.disable_controls(self.auto_play_enabled) + self.auto_play_button.setText( + self.autoplay_button_text[self.auto_play_enabled]) + + def disable_controls(self, disabled): + self.step_box.setDisabled(disabled) + self.options_box.setDisabled(disabled) + self.properties_box.setDisabled(disabled) + + diff --git a/orangecontrib/educational/widgets/utils/logistic_regression.py b/orangecontrib/educational/widgets/utils/logistic_regression.py index 34434b56..0662b3a8 100644 --- a/orangecontrib/educational/widgets/utils/logistic_regression.py +++ b/orangecontrib/educational/widgets/utils/logistic_regression.py @@ -41,6 +41,12 @@ def set_alpha(self, alpha): def model(self): return LogisticRegressionModel(self.theta, self.domain) + @property + def converged(self): + if self.step_no == 0: + return False + return np.sum(np.abs(self.theta - self.history[self.step_no - 1])) < 1e-2 + def step(self): self.step_no += 1 grad = self.dj(self.theta) From 6cb21b792130a5267f2fb619a7b69569fa846f2d Mon Sep 17 00:00:00 2001 From: PrimozGodec Date: Wed, 3 Aug 2016 15:49:02 +0200 Subject: [PATCH 007/128] Stochastic option in gradient descent --- .../educational/widgets/owgradientdescent.py | 15 +++++- .../widgets/utils/logistic_regression.py | 50 +++++++++++++++---- 2 files changed, 53 insertions(+), 12 deletions(-) diff --git a/orangecontrib/educational/widgets/owgradientdescent.py b/orangecontrib/educational/widgets/owgradientdescent.py index e58df245..226676b9 100644 --- a/orangecontrib/educational/widgets/owgradientdescent.py +++ b/orangecontrib/educational/widgets/owgradientdescent.py @@ -132,6 +132,7 @@ class OWGradientDescent(OWWidget): target_class = settings.Setting('') alpha = settings.Setting(0.1) auto_play_speed = settings.Setting(1) + stochastic = settings.Setting(False) # models x_var_model = None @@ -187,16 +188,22 @@ def __init__(self): master=self, callback=self.change_alpha, value="alpha", - label="Alpha: ", + label="Learning rate: ", minv=0.01, maxv=1, step=0.01, spinType=float) + self.stochastic_checkbox = gui.checkBox(widget=self.properties_box, + master=self, + callback=self.change_stochastic, + value="stochastic", + label="Stochastic: ") self.restart_button = gui.button(widget=self.properties_box, master=self, callback=self.restart, label="Restart") + self.step_box = gui.widgetBox(self.controlArea, "Manually step through") self.step_button = gui.button(widget=self.step_box, @@ -310,13 +317,17 @@ def set_empty_plot(self): def restart(self): self.selected_data = self.select_data() self.learner = self.default_learner(data=Normalize(self.selected_data), - alpha=self.alpha) + alpha=self.alpha, stochastic=self.stochastic) self.replot() def change_alpha(self): if self.learner is not None: self.learner.set_alpha(self.alpha) + def change_stochastic(self): + if self.learner is not None: + self.learner.stochastic = self.stochastic + def step(self): if self.data is None: return diff --git a/orangecontrib/educational/widgets/utils/logistic_regression.py b/orangecontrib/educational/widgets/utils/logistic_regression.py index 0662b3a8..7fdce26f 100644 --- a/orangecontrib/educational/widgets/utils/logistic_regression.py +++ b/orangecontrib/educational/widgets/utils/logistic_regression.py @@ -11,12 +11,15 @@ class LogisticRegression: theta = None domain = None step_no = 0 + stochastic_i = 0 + stochastic_num_steps = 30 # number of steps in one step - def __init__(self, alpha=0.1, theta=None, data=None): + def __init__(self, alpha=0.1, theta=None, data=None, stochastic=False): self.history = [] self.set_alpha(alpha) self.set_data(data) self.set_theta(theta) + self.stochastic = stochastic def set_data(self, data): if data is not None: @@ -31,7 +34,7 @@ def set_theta(self, theta): self.theta = np.array(theta) else: self.theta = None - self.history = self.set_list(self.history, 0, np.copy(self.theta)) + self.history = self.set_list(self.history, 0, (np.copy(self.theta), 0)) self.step_no = 0 def set_alpha(self, alpha): @@ -45,18 +48,39 @@ def model(self): def converged(self): if self.step_no == 0: return False - return np.sum(np.abs(self.theta - self.history[self.step_no - 1])) < 1e-2 + return np.sum(np.abs(self.theta - self.history[self.step_no - 1][0])) < (1e-2 if not self.stochastic else 1e-5) def step(self): self.step_no += 1 - grad = self.dj(self.theta) + grad = self.dj(self.theta, self.stochastic) self.theta -= self.alpha * grad - self.history = self.set_list(self.history, self.step_no, np.copy(self.theta)) + + self.stochastic_i += self.stochastic_num_steps + + seed = None # seed that will be stored to revert the shuffle + if self.stochastic_i >= len(self.x): + self.stochastic_i = 0 + seed = np.random.randint(100) # random seed + np.random.seed(seed) # set seed of permutation used to shuffle + indices = np.random.permutation(len(self.x)) + self.x = self.x[indices] # permutation + self.y = self.y[indices] + + self.history = self.set_list(self.history, self.step_no, (np.copy(self.theta), self.stochastic_i, seed)) def step_back(self): if self.step_no > 0: self.step_no -= 1 - self.theta = np.copy(self.history[self.step_no]) + self.theta = np.copy(self.history[self.step_no][0]) + self.stochastic_i = self.history[self.step_no][1] + seed = self.history[self.step_no + 1][2] + if seed is not None: # it means data had been permuted on this pos + np.random.seed(seed) # use same seed to revert + indices = np.random.permutation(len(self.x)) + indices_reverse = np.argsort(indices) + # indices of sorted indices gives us reversing shuffle list + self.x = self.x[indices_reverse] + self.y = self.y[indices_reverse] def j(self, theta): """ @@ -67,11 +91,17 @@ def j(self, theta): # return -sum(np.log(self.y * yh + (1 - self.y) * (1 - yh))) / len(yh) return -sum(self.y * np.log(yh) + (1 - self.y) * np.log(1 - yh)) / len(yh) - def dj(self, theta): + def dj(self, theta, stochastic=False): """ Gradient of the cost function with L2 regularization """ - return (self.g(self.x.dot(theta)) - self.y).dot(self.x) + if stochastic: + ns = self.stochastic_num_steps + x = self.x[self.stochastic_i : self.stochastic_i + ns] + y = self.y[self.stochastic_i : self.stochastic_i + ns] + return x.T.dot(self.g(x.dot(theta)) - y) + else: + return (self.g(self.x.dot(theta)) - self.y).dot(self.x) def optimized(self): """ @@ -94,8 +124,8 @@ def g(z): """ # limit values in z to avoid log with 0 produced by values almost 0 - z_mod = np.minimum(z, 100 * np.ones(len(z))) - z_mod = np.maximum(z_mod, -100 * np.ones(len(z))) + z_mod = np.minimum(z, 100) + z_mod = np.maximum(z_mod, -100) return 1.0 / (1 + np.exp(- z_mod)) From a3892519a236a358f763c6ba33ec950266b2da0a Mon Sep 17 00:00:00 2001 From: PrimozGodec Date: Wed, 3 Aug 2016 16:22:29 +0200 Subject: [PATCH 008/128] Cost function (j) modified to work with matrix of thetas --- .../educational/widgets/owgradientdescent.py | 3 ++- .../widgets/utils/logistic_regression.py | 16 ++++++++++------ 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/orangecontrib/educational/widgets/owgradientdescent.py b/orangecontrib/educational/widgets/owgradientdescent.py index 226676b9..4a282389 100644 --- a/orangecontrib/educational/widgets/owgradientdescent.py +++ b/orangecontrib/educational/widgets/owgradientdescent.py @@ -425,7 +425,8 @@ def plot_gradient_and_contour(self, x_from, x_to, y_from, y_to): self.xv, self.yv = np.meshgrid(x, y) thetas = np.column_stack((self.xv.flatten(), self.yv.flatten())) - cost_values = np.vstack([self.learner.j(theta) for theta in thetas]) + # cost_values = np.vstack([self.learner.j(theta) for theta in thetas]) + cost_values = self.learner.j(thetas) # results self.cost_grid = cost_values.reshape(self.xv.shape) diff --git a/orangecontrib/educational/widgets/utils/logistic_regression.py b/orangecontrib/educational/widgets/utils/logistic_regression.py index 7fdce26f..415ba208 100644 --- a/orangecontrib/educational/widgets/utils/logistic_regression.py +++ b/orangecontrib/educational/widgets/utils/logistic_regression.py @@ -9,6 +9,7 @@ class LogisticRegression: x = None y = None theta = None + alpha = None domain = None step_no = 0 stochastic_i = 0 @@ -48,7 +49,8 @@ def model(self): def converged(self): if self.step_no == 0: return False - return np.sum(np.abs(self.theta - self.history[self.step_no - 1][0])) < (1e-2 if not self.stochastic else 1e-5) + return (np.sum(np.abs(self.theta - self.history[self.step_no - 1][0])) < + (1e-2 if not self.stochastic else 1e-5)) def step(self): self.step_no += 1 @@ -66,7 +68,9 @@ def step(self): self.x = self.x[indices] # permutation self.y = self.y[indices] - self.history = self.set_list(self.history, self.step_no, (np.copy(self.theta), self.stochastic_i, seed)) + self.history = self.set_list( + self.history, self.step_no, + (np.copy(self.theta), self.stochastic_i, seed)) def step_back(self): if self.step_no > 0: @@ -86,10 +90,10 @@ def j(self, theta): """ Cost function for logistic regression """ - # TODO: modify for more thetas - yh = self.g(self.x.dot(theta)) - # return -sum(np.log(self.y * yh + (1 - self.y) * (1 - yh))) / len(yh) - return -sum(self.y * np.log(yh) + (1 - self.y) * np.log(1 - yh)) / len(yh) + yh = self.g(self.x.dot(theta.T)).T + y = self.y + return -np.sum( + (self.y * np.log(yh) + (1 - y) * np.log(1 - yh)).T, axis=0) / len(y) def dj(self, theta, stochastic=False): """ From b42b0329412505b9dbc81a602a9a89e9d47f4a67 Mon Sep 17 00:00:00 2001 From: PrimozGodec Date: Wed, 3 Aug 2016 16:50:37 +0200 Subject: [PATCH 009/128] Code refactor for logistic_regression --- .../widgets/utils/logistic_regression.py | 77 +++++++++++++++++-- 1 file changed, 70 insertions(+), 7 deletions(-) diff --git a/orangecontrib/educational/widgets/utils/logistic_regression.py b/orangecontrib/educational/widgets/utils/logistic_regression.py index 415ba208..aad7609c 100644 --- a/orangecontrib/educational/widgets/utils/logistic_regression.py +++ b/orangecontrib/educational/widgets/utils/logistic_regression.py @@ -1,10 +1,23 @@ import numpy as np +from scipy.optimize import fmin_l_bfgs_b from Orange.classification import Model -from scipy.optimize import fmin_l_bfgs_b class LogisticRegression: + """ + Logistic regression algorithm with custom cost and gradient function, + which allow to perform algorithm step by step + + Parameters + ---------- + alpha : float + Learning rate + theta : array_like(float, ndim=1) + Logistic function parameters + data : Orange.data.Table + Data + """ x = None y = None @@ -23,12 +36,18 @@ def __init__(self, alpha=0.1, theta=None, data=None, stochastic=False): self.stochastic = stochastic def set_data(self, data): + """ + Function set the data + """ if data is not None: self.x = data.X self.y = data.Y self.domain = data.domain def set_theta(self, theta): + """ + Function sets theta. Can be called from constructor or outside. + """ if isinstance(theta, (np.ndarray, np.generic)): self.theta = theta elif isinstance(theta, list): @@ -39,35 +58,54 @@ def set_theta(self, theta): self.step_no = 0 def set_alpha(self, alpha): + """ + Function sets alpha and can be called from constructor or from outside. + """ self.alpha = alpha @property def model(self): + """ + Function returns model based on current parameters. + """ return LogisticRegressionModel(self.theta, self.domain) @property def converged(self): + """ + Function returns True if gradient descent already converged. + """ if self.step_no == 0: return False return (np.sum(np.abs(self.theta - self.history[self.step_no - 1][0])) < (1e-2 if not self.stochastic else 1e-5)) def step(self): + """ + Function performs one step of the gradient descent + """ self.step_no += 1 + + # calculates gradient and modify theta grad = self.dj(self.theta, self.stochastic) self.theta -= self.alpha * grad + # increase index used by stochastic gradient descent self.stochastic_i += self.stochastic_num_steps seed = None # seed that will be stored to revert the shuffle + # if we came around all data set index to zero and permute data if self.stochastic_i >= len(self.x): self.stochastic_i = 0 + + # shuffle data seed = np.random.randint(100) # random seed np.random.seed(seed) # set seed of permutation used to shuffle indices = np.random.permutation(len(self.x)) self.x = self.x[indices] # permutation self.y = self.y[indices] + # save history for step back self.history = self.set_list( self.history, self.step_no, (np.copy(self.theta), self.stochastic_i, seed)) @@ -75,8 +113,14 @@ def step(self): def step_back(self): if self.step_no > 0: self.step_no -= 1 + + # modify theta self.theta = np.copy(self.history[self.step_no][0]) + + # modify index for stochastic gradient descent self.stochastic_i = self.history[self.step_no][1] + + # if necessary restore data shuffle seed = self.history[self.step_no + 1][2] if seed is not None: # it means data had been permuted on this pos np.random.seed(seed) # use same seed to revert @@ -97,19 +141,19 @@ def j(self, theta): def dj(self, theta, stochastic=False): """ - Gradient of the cost function with L2 regularization + Gradient of the cost function for logistic regression """ if stochastic: ns = self.stochastic_num_steps - x = self.x[self.stochastic_i : self.stochastic_i + ns] - y = self.y[self.stochastic_i : self.stochastic_i + ns] + x = self.x[self.stochastic_i: self.stochastic_i + ns] + y = self.y[self.stochastic_i: self.stochastic_i + ns] return x.T.dot(self.g(x.dot(theta)) - y) else: return (self.g(self.x.dot(theta)) - self.y).dot(self.x) def optimized(self): """ - Function performs model training + Function performs whole model training. Not step by step. """ res = fmin_l_bfgs_b(self.j, np.zeros(self.x.shape[1]), @@ -119,11 +163,11 @@ def optimized(self): @staticmethod def g(z): """ - sigmoid function + Sigmoid function Parameters ---------- - z : array_like + z : array_like(float) values to evaluate with function """ @@ -135,6 +179,24 @@ def g(z): @staticmethod def set_list(l, i, v): + """ + Function sets i-th value in list to v. If i does not exist in l + it is initialized else value is modified + + Parameters + ---------- + l : list + List + i : int + Index of position in list + v : any + Value to insert in list + + Returns + ------- + list + List with inserted value v on position i + """ try: l[i] = v except IndexError: @@ -143,6 +205,7 @@ def set_list(l, i, v): l.append(v) return l + class LogisticRegressionModel(Model): def __init__(self, theta, domain): From ea60cfdb60310e6b0bfd5b3ae5a88a0e09e0034c Mon Sep 17 00:00:00 2001 From: PrimozGodec Date: Wed, 3 Aug 2016 17:28:34 +0200 Subject: [PATCH 010/128] Code refactor in OWGradientDescent. --- .../educational/widgets/owgradientdescent.py | 252 ++++++++++-------- 1 file changed, 139 insertions(+), 113 deletions(-) diff --git a/orangecontrib/educational/widgets/owgradientdescent.py b/orangecontrib/educational/widgets/owgradientdescent.py index 4a282389..c0a23c7e 100644 --- a/orangecontrib/educational/widgets/owgradientdescent.py +++ b/orangecontrib/educational/widgets/owgradientdescent.py @@ -2,12 +2,11 @@ import time import numpy as np -from Orange.widgets.utils import itemmodels +from scipy.ndimage import gaussian_filter from PyQt4.QtCore import pyqtSlot, Qt, QThread, SIGNAL from PyQt4.QtGui import QSizePolicy, QPixmap, QColor, QIcon -from scipy.interpolate import splev, splprep -from scipy.ndimage import gaussian_filter +from Orange.widgets.utils import itemmodels from Orange.classification import Model from Orange.data import Table, ContinuousVariable, Domain, DiscreteVariable from Orange.widgets import gui @@ -27,11 +26,11 @@ class Scatterplot(highcharts.Highchart): * enables scroll-wheel zooming, * set callback functions for click (in empty chart), drag and drop * enables moving of centroids points - * include drag_drop_js script by highchart + * include drag_drop_js script by highcharts """ - js_click_function = """/**/(function(event) { - window.pybridge.chart_clicked(event.xAxis[0].value, event.yAxis[0].value); + js_click_function = """/**/(function(e) { + window.pybridge.chart_clicked(e.xAxis[0].value, e.yAxis[0].value); }) """ @@ -41,7 +40,9 @@ class Scatterplot(highcharts.Highchart): def __init__(self, click_callback, **kwargs): # read javascript for drag and drop - with open(path.join(path.dirname(__file__), 'resources', 'highcharts-contour.js'), 'r') as f: + with open( + path.join(path.dirname(__file__), 'resources', + 'highcharts-contour.js'), 'r') as f: contours_js = f.read() super().__init__(enable_zoom=True, @@ -60,41 +61,57 @@ def chart(self, *args, **kwargs): @pyqtSlot(float, float) def chart_clicked(self, x, y): + """ + Function is called from javascript when click event happens + """ self.click_callback(x, y) - def remove_series(self, id): + def remove_series(self, idx): + """ + Function remove series with id idx + """ self.evalJS(""" series = chart.get('{id}'); if (series != null) series.remove(true); - """.format(id=id)) + """.format(id=idx)) - def remove_last_point(self, id): + def remove_last_point(self, idx): + """ + Function remove last point from series with id idx + """ self.evalJS(""" series = chart.get('{id}'); if (series != null) series.removePoint(series.data.length - 1, true); - """.format(id=id)) + """.format(id=idx)) def add_series(self, series): + """ + Function add series to the chart + """ for i, s in enumerate(series): self.exposeObject('series%d' % i, series[i]) self.evalJS("chart.addSeries(series%d, true);" % i) - def add_point_to_series(self, id, x, y): + def add_point_to_series(self, idx, x, y): + """ + Function add point to the series with id idx + """ self.evalJS(""" series = chart.get('{id}'); series.addPoint([{x}, {y}]); - """.format(id=id, x=x, y=y)) + """.format(id=idx, x=x, y=y)) class Autoplay(QThread): """ - Class used for separated thread when using "Autoplay" for k-means + Class used for separated thread when using "Autoplay" for gradient descent + Parameters ---------- - owkmeans : OWKmeans - Instance of OWKmeans class + ow_gradient_descent : OWGradientDescent + Instance of OWGradientDescent class """ def __init__(self, ow_gradient_descent): @@ -109,16 +126,19 @@ def run(self): Stepping through the algorithm until converge or user interrupts """ while (not self.ow_gradient_descent.learner.converged and - self.ow_gradient_descent.auto_play_enabled): + self.ow_gradient_descent.auto_play_enabled): self.emit(SIGNAL('step()')) time.sleep(2 - self.ow_gradient_descent.auto_play_speed) self.emit(SIGNAL('stop_auto_play()')) class OWGradientDescent(OWWidget): + """ + Gradient descent widget algorithm + """ name = "Gradient Descent" - description = "Widget demonstrates shows the procedure of gradient descent." + description = "Widget shows the procedure of gradient descent." icon = "icons/InteractiveKMeans.svg" want_main_area = True @@ -144,6 +164,10 @@ class OWGradientDescent(OWWidget): cost_grid = None grid_size = 15 contour_color = "#aaaaaa" + min_x = None + max_x = None + min_y = None + max_y = None # data data = None @@ -151,9 +175,13 @@ class OWGradientDescent(OWWidget): # autoplay auto_play_enabled = False - autoplay_button_text = ["Run", "Stop"] + auto_play_button_text = ["Run", "Stop"] + auto_play_thread = None class Warning(OWWidget.Warning): + """ + Class used fro widget warnings. + """ to_few_features = Msg("Too few Continuous feature. Min 2 required") no_class = Msg("No class provided or only one class variable") @@ -183,69 +211,52 @@ def __init__(self): self.cbx.setModel(self.x_var_model) self.cby.setModel(self.y_var_model) + # properties box self.properties_box = gui.widgetBox(self.controlArea, "Properties") - self.alpha_spin = gui.spin(widget=self.properties_box, - master=self, - callback=self.change_alpha, - value="alpha", - label="Learning rate: ", - minv=0.01, - maxv=1, - step=0.01, - spinType=float) - self.stochastic_checkbox = gui.checkBox(widget=self.properties_box, - master=self, - callback=self.change_stochastic, - value="stochastic", - label="Stochastic: ") - self.restart_button = gui.button(widget=self.properties_box, - master=self, - callback=self.restart, - label="Restart") - - + self.alpha_spin = gui.spin( + widget=self.properties_box, master=self, callback=self.change_alpha, + value="alpha", label="Learning rate: ", + minv=0.01, maxv=1, step=0.01, spinType=float) + self.stochastic_checkbox = gui.checkBox( + widget=self.properties_box, master=self, + callback=self.change_stochastic, value="stochastic", + label="Stochastic: ") + self.restart_button = gui.button( + widget=self.properties_box, master=self, + callback=self.restart, label="Restart") + + # step box self.step_box = gui.widgetBox(self.controlArea, "Manually step through") + self.step_button = gui.button( + widget=self.step_box, master=self, callback=self.step, label="Step") + self.step_back_button = gui.button( + widget=self.step_box, master=self, callback=self.step_back, + label="Step back") - self.step_button = gui.button(widget=self.step_box, - master=self, - callback=self.step, - label="Step") - self.step_back_button = gui.button(widget=self.step_box, - master=self, - callback=self.step_back, - label="Step back") - + # run box self.run_box = gui.widgetBox(self.controlArea, "Run") self.auto_play_button = gui.button( - self.run_box, self, self.autoplay_button_text[0], - callback=self.auto_play) - self.auto_play_speed_spinner = gui.hSlider(self.run_box, - self, - 'auto_play_speed', - minValue=0, - maxValue=1.91, - step=0.1, - intOnly=False, - createLabel=False, - label='Speed:') + widget=self.run_box, master=self, + label=self.auto_play_button_text[0], callback=self.auto_play) + self.auto_play_speed_spinner = gui.hSlider( + widget=self.run_box, master=self, value='auto_play_speed', + minValue=0, maxValue=1.91, step=0.1, + intOnly=False, createLabel=False, label='Speed:') # graph in mainArea - self.scatter = Scatterplot(click_callback=self.set_theta, + self.scatter = Scatterplot(click_callback=self.change_theta, xAxis_gridLineWidth=0, yAxis_gridLineWidth=0, title_text='', tooltip_shared=False, debug=True) - + # TODO: set false when end of development gui.rubber(self.controlArea) - # TODO: set false when end of development # Just render an empty chart so it shows a nice 'No data to display' self.scatter.chart() self.mainArea.layout().addWidget(self.scatter) - # set random learner - def set_data(self, data): """ Function receives data from input and init part of widget if data @@ -283,22 +294,21 @@ def init_combos(): self.Warning.clear() # clear variables - self.xv = None - self.yv = None self.cost_grid = None + dd = data.domain + if data is None or len(data) == 0: self.data = None reset_combos() self.set_empty_plot() - elif sum(True for var in data.domain.attributes + elif sum(True for var in dd.attributes if isinstance(var, ContinuousVariable)) < 2: self.data = None reset_combos() self.Warning.to_few_features() self.set_empty_plot() - elif (data.domain.class_var is None or - len(data.domain.class_var.values) < 2): + elif dd.class_var is None or len(dd.class_var.values) < 2: self.data = None reset_combos() self.Warning.no_class() @@ -312,38 +322,72 @@ def init_combos(): self.restart() def set_empty_plot(self): + """ + Function render empty plot + """ self.scatter.clear() def restart(self): + """ + Function restarts the algorithm + """ self.selected_data = self.select_data() - self.learner = self.default_learner(data=Normalize(self.selected_data), - alpha=self.alpha, stochastic=self.stochastic) + self.learner = self.default_learner( + data=Normalize(self.selected_data), + alpha=self.alpha, stochastic=self.stochastic) self.replot() def change_alpha(self): + """ + Function changes alpha parameter of the alogrithm + """ if self.learner is not None: self.learner.set_alpha(self.alpha) def change_stochastic(self): + """ + Function changes switches between stochastic or usual algorithm + """ if self.learner is not None: self.learner.stochastic = self.stochastic + def change_theta(self, x, y): + """ + Function set new theta + """ + if self.learner is not None: + self.learner.set_theta([x, y]) + self.scatter.remove_series("path") + self.scatter.add_series([ + dict(id="path", data=[[x, y]], showInLegend=False, + type="scatter", lineWidth=1, + marker=dict(enabled=True, radius=2))],) + def step(self): + """ + Function performs one step of the algorithm + """ if self.data is None: return if self.learner.theta is None: - self.set_theta(np.random.uniform(self.min_x, self.max_x), - np.random.uniform(self.min_y, self.max_y)) + self.change_theta(np.random.uniform(self.min_x, self.max_x), + np.random.uniform(self.min_y, self.max_y)) self.learner.step() theta = self.learner.theta self.plot_point(theta[0], theta[1]) def step_back(self): + """ + Function performs step back + """ if self.learner.step_no > 0: self.learner.step_back() self.scatter.remove_last_point("path") def plot_point(self, x, y): + """ + Function add point to the path + """ self.scatter.add_point_to_series("path", x, y) def replot(self): @@ -353,9 +397,6 @@ def replot(self): if self.data is None: return - attr_x = self.data.domain[self.attr_x] - attr_y = self.data.domain[self.attr_y] - optimal_theta = self.learner.optimized() self.min_x = optimal_theta[0] - 5 self.max_x = optimal_theta[0] + 5 @@ -368,7 +409,6 @@ def replot(self): options['series'] += self.plot_gradient_and_contour( self.min_x, self.max_x, self.min_y, self.max_y) - min_value = np.min(self.cost_grid) max_value = np.max(self.cost_grid) @@ -399,10 +439,10 @@ def replot(self): self.scatter.chart(options, **kwargs) - def plot_gradient_and_contour(self, x_from, x_to, y_from, y_to): """ Function constructs series for gradient and contour + Parameters ---------- x_from : float @@ -413,6 +453,7 @@ def plot_gradient_and_contour(self, x_from, x_to, y_from, y_to): Min grid y value y_to : float Max grid y value + Returns ------- list @@ -422,19 +463,19 @@ def plot_gradient_and_contour(self, x_from, x_to, y_from, y_to): # grid for gradient x = np.linspace(x_from, x_to, self.grid_size) y = np.linspace(y_from, y_to, self.grid_size) - self.xv, self.yv = np.meshgrid(x, y) - thetas = np.column_stack((self.xv.flatten(), self.yv.flatten())) + xv, yv = np.meshgrid(x, y) + thetas = np.column_stack((xv.flatten(), yv.flatten())) # cost_values = np.vstack([self.learner.j(theta) for theta in thetas]) cost_values = self.learner.j(thetas) # results - self.cost_grid = cost_values.reshape(self.xv.shape) + self.cost_grid = cost_values.reshape(xv.shape) blurred = self.blur_grid(self.cost_grid) # return self.plot_gradient(self.xv, self.yv, blurred) + \ - return self.plot_contour() + return self.plot_contour(xv, yv, blurred) def plot_gradient(self, x, y, grid): """ @@ -449,6 +490,7 @@ def select_data(self): """ Function takes two selected columns from data table and merge them in new Orange.data.Table + Returns ------- Table @@ -471,31 +513,20 @@ def select_data(self): return Table(domain, x, y, self.data.Y[:, None]) - def plot_contour(self): + def plot_contour(self, xv, yv, cost_grid): """ Function constructs contour lines """ contour = Contour( - self.xv, self.yv, self.blur_grid(self.cost_grid)) + xv, yv, cost_grid) contour_lines = contour.contours( - np.linspace(np.min(self.cost_grid), np.max(self.cost_grid), 10)) + np.linspace(np.min(cost_grid), np.max(cost_grid), 10)) series = [] count = 0 for key, value in contour_lines.items(): for line in value: - # if len(line) > 3: - # # if less than degree interpolation fails - # tck, u = splprep( - # [list(x) for x in zip(*reversed(line))], - # s=0.001, k=3, - # per=(len(line) - # if np.allclose(line[0], line[-1]) - # else 0)) - # new_int = np.arange(0, 1.01, 0.01) - # interpol_line = np.array(splev(new_int, tck)).T.tolist() - # else: interpol_line = line series.append(dict(data=interpol_line, @@ -512,33 +543,27 @@ def plot_contour(self): @staticmethod def blur_grid(grid): + """ + Function blur the grid, to make crossings smoother + """ filtered = gaussian_filter(grid, sigma=1) return filtered - def set_theta(self, x, y): - if self.learner is not None: - self.learner.set_theta([x, y]) - self.scatter.remove_series("path") - self.scatter.add_series([ - dict(id="path", data=[[x, y]], showInLegend=False, - type="scatter", lineWidth=1, - marker=dict(enabled=True, radius=2))],) - def auto_play(self): """ Function called when autoplay button pressed """ self.auto_play_enabled = not self.auto_play_enabled self.auto_play_button.setText( - self.autoplay_button_text[self.auto_play_enabled]) + self.auto_play_button_text[self.auto_play_enabled]) if self.auto_play_enabled: self.disable_controls(self.auto_play_enabled) - self.autoPlayThread = Autoplay(self) - self.connect(self.autoPlayThread, SIGNAL("step()"), self.step) + self.auto_play_thread = Autoplay(self) + self.connect(self.auto_play_thread, SIGNAL("step()"), self.step) self.connect( - self.autoPlayThread, SIGNAL("stop_auto_play()"), + self.auto_play_thread, SIGNAL("stop_auto_play()"), self.stop_auto_play) - self.autoPlayThread.start() + self.auto_play_thread.start() else: self.stop_auto_play() @@ -549,11 +574,12 @@ def stop_auto_play(self): self.auto_play_enabled = False self.disable_controls(self.auto_play_enabled) self.auto_play_button.setText( - self.autoplay_button_text[self.auto_play_enabled]) + self.auto_play_button_text[self.auto_play_enabled]) def disable_controls(self, disabled): + """ + Function disable or enable all controls except those from run part + """ self.step_box.setDisabled(disabled) self.options_box.setDisabled(disabled) self.properties_box.setDisabled(disabled) - - From f24f60d5c54bb15356494d447fedbf49941ab8b6 Mon Sep 17 00:00:00 2001 From: PrimozGodec Date: Thu, 4 Aug 2016 08:44:39 +0200 Subject: [PATCH 011/128] Icon for gradient descent. --- .../widgets/icons/GradientDescent.svg | 152 ++++++++++++++++++ .../educational/widgets/owgradientdescent.py | 2 +- 2 files changed, 153 insertions(+), 1 deletion(-) create mode 100644 orangecontrib/educational/widgets/icons/GradientDescent.svg diff --git a/orangecontrib/educational/widgets/icons/GradientDescent.svg b/orangecontrib/educational/widgets/icons/GradientDescent.svg new file mode 100644 index 00000000..10a2ff9a --- /dev/null +++ b/orangecontrib/educational/widgets/icons/GradientDescent.svg @@ -0,0 +1,152 @@ + + + + + + + + + + image/svg+xml + + + + + + + + + + + + + + + + + + + + + + + diff --git a/orangecontrib/educational/widgets/owgradientdescent.py b/orangecontrib/educational/widgets/owgradientdescent.py index c0a23c7e..4276c278 100644 --- a/orangecontrib/educational/widgets/owgradientdescent.py +++ b/orangecontrib/educational/widgets/owgradientdescent.py @@ -139,7 +139,7 @@ class OWGradientDescent(OWWidget): name = "Gradient Descent" description = "Widget shows the procedure of gradient descent." - icon = "icons/InteractiveKMeans.svg" + icon = "icons/GradientDescent.svg" want_main_area = True inputs = [("Data", Table, "set_data")] From ae985d2b759aa9343dd9da18e8fae0732f1a39f2 Mon Sep 17 00:00:00 2001 From: PrimozGodec Date: Thu, 4 Aug 2016 08:49:42 +0200 Subject: [PATCH 012/128] Icon updated --- .../widgets/icons/GradientDescent.svg | 96 ++++++++----------- 1 file changed, 39 insertions(+), 57 deletions(-) diff --git a/orangecontrib/educational/widgets/icons/GradientDescent.svg b/orangecontrib/educational/widgets/icons/GradientDescent.svg index 10a2ff9a..83948785 100644 --- a/orangecontrib/educational/widgets/icons/GradientDescent.svg +++ b/orangecontrib/educational/widgets/icons/GradientDescent.svg @@ -55,98 +55,80 @@ id="layer1" transform="translate(0,-1004.3622)"> - + style="fill:#000000;fill-opacity:1;stroke:#000000;stroke-width:1.23305607;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1" + id="path4719-1" + cx="41.132149" + cy="1010.5355" + r="0.55258733" /> + style="fill:#000000;fill-opacity:1;stroke:#000000;stroke-width:1.23305607;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1" + id="path4719-1-6" + cx="35.544743" + cy="1019.5954" + r="0.55258733" /> + style="fill:#000000;fill-opacity:1;stroke:#000000;stroke-width:1.23305607;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1" + id="path4719-1-0" + cx="31.504133" + cy="1022.8152" + r="0.55258733" /> + style="fill:#000000;fill-opacity:1;stroke:#000000;stroke-width:1.23305607;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1" + id="path4719-1-62" + cx="27.905464" + cy="1023.7622" + r="0.55258733" /> - - + r="0.55258733" /> + style="fill:#000000;fill-opacity:1;stroke:#000000;stroke-width:1.23305607;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1" + id="path4719-1-8" + cx="21.339472" + cy="1023.6992" + r="0.55258733" /> + style="fill:#000000;fill-opacity:1;stroke:#000000;stroke-width:1.23305607;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1" + id="path4719-1-7" + cx="18.372149" + cy="1023.0677" + r="0.55258733" /> From 5bf3b3ac6f8bcc8e2b64ebad71b902f37097b1ba Mon Sep 17 00:00:00 2001 From: PrimozGodec Date: Thu, 4 Aug 2016 09:35:20 +0200 Subject: [PATCH 013/128] Implemented widget output --- .../educational/widgets/owgradientdescent.py | 50 ++++++++++++++++--- 1 file changed, 44 insertions(+), 6 deletions(-) diff --git a/orangecontrib/educational/widgets/owgradientdescent.py b/orangecontrib/educational/widgets/owgradientdescent.py index 4276c278..dec24d3c 100644 --- a/orangecontrib/educational/widgets/owgradientdescent.py +++ b/orangecontrib/educational/widgets/owgradientdescent.py @@ -8,7 +8,8 @@ from Orange.widgets.utils import itemmodels from Orange.classification import Model -from Orange.data import Table, ContinuousVariable, Domain, DiscreteVariable +from Orange.data import Table, ContinuousVariable, Domain, DiscreteVariable, \ + StringVariable from Orange.widgets import gui from Orange.widgets import highcharts from Orange.widgets import settings @@ -143,8 +144,9 @@ class OWGradientDescent(OWWidget): want_main_area = True inputs = [("Data", Table, "set_data")] - outputs = [("Model", Model), - ("Coefficients", Table)] + outputs = [("Classifier", Model), + ("Coefficients", Table), + ("Data", Table)] # selected attributes in chart attr_x = settings.Setting('') @@ -295,20 +297,22 @@ def init_combos(): # clear variables self.cost_grid = None + self.learner = None - dd = data.domain + d = data + self.send_output() if data is None or len(data) == 0: self.data = None reset_combos() self.set_empty_plot() - elif sum(True for var in dd.attributes + elif sum(True for var in d.domain.attributes if isinstance(var, ContinuousVariable)) < 2: self.data = None reset_combos() self.Warning.to_few_features() self.set_empty_plot() - elif dd.class_var is None or len(dd.class_var.values) < 2: + elif d.domain.class_var is None or len(d.domain.class_var.values) < 2: self.data = None reset_combos() self.Warning.no_class() @@ -336,6 +340,7 @@ def restart(self): data=Normalize(self.selected_data), alpha=self.alpha, stochastic=self.stochastic) self.replot() + self.send_output() def change_alpha(self): """ @@ -362,6 +367,7 @@ def change_theta(self, x, y): dict(id="path", data=[[x, y]], showInLegend=False, type="scatter", lineWidth=1, marker=dict(enabled=True, radius=2))],) + self.send_output() def step(self): """ @@ -375,6 +381,7 @@ def step(self): self.learner.step() theta = self.learner.theta self.plot_point(theta[0], theta[1]) + self.send_output() def step_back(self): """ @@ -383,6 +390,7 @@ def step_back(self): if self.learner.step_no > 0: self.learner.step_back() self.scatter.remove_last_point("path") + self.send_output() def plot_point(self, x, y): """ @@ -583,3 +591,33 @@ def disable_controls(self, disabled): self.step_box.setDisabled(disabled) self.options_box.setDisabled(disabled) self.properties_box.setDisabled(disabled) + + def send_output(self): + self.send_model() + self.send_coefficients() + self.send_data() + + def send_model(self): + if self.learner is not None and self.learner.theta is not None: + self.send("Classifier", self.learner.model) + else: + self.send("Classifier", None) + + def send_coefficients(self): + if self.learner is not None and self.learner.theta is not None: + domain = Domain( + [ContinuousVariable("coef", number_of_decimals=7)], + metas=[StringVariable("name")]) + names = ["theta 0", "theta 1"] + + coefficients_table = Table( + domain, list(zip(list(self.learner.theta), names))) + self.send("Coefficients", coefficients_table) + else: + self.send("Coefficients", None) + + def send_data(self): + if self.selected_data is not None: + self.send("Data", self.selected_data) + else: + self.send("Data", None) \ No newline at end of file From f3914dd50f1e586ab97ecf18cd51d0a34a6eb412 Mon Sep 17 00:00:00 2001 From: PrimozGodec Date: Thu, 4 Aug 2016 09:36:53 +0200 Subject: [PATCH 014/128] [FIX] Predictor to return correct predictions. --- .../educational/widgets/utils/logistic_regression.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/orangecontrib/educational/widgets/utils/logistic_regression.py b/orangecontrib/educational/widgets/utils/logistic_regression.py index aad7609c..4291dbbc 100644 --- a/orangecontrib/educational/widgets/utils/logistic_regression.py +++ b/orangecontrib/educational/widgets/utils/logistic_regression.py @@ -211,6 +211,12 @@ class LogisticRegressionModel(Model): def __init__(self, theta, domain): super().__init__(domain) self.theta = theta + self.name = "Logistic Regression" + print("a") def predict_storage(self, data): - return LogisticRegression.g(data.X.dot(self.theta)) + probabilities = LogisticRegression.g(data.X.dot(self.theta)) + values = np.around(probabilities) + probabilities0 = 1 - probabilities + probabilities = np.column_stack((probabilities0, probabilities)) + return values, probabilities From 3c52867ff3d89841b46c5a01e46d0d0dacfe1166 Mon Sep 17 00:00:00 2001 From: PrimozGodec Date: Thu, 4 Aug 2016 10:03:34 +0200 Subject: [PATCH 015/128] Data normalization moved to select_data --- orangecontrib/educational/widgets/owgradientdescent.py | 4 ++-- .../educational/widgets/utils/logistic_regression.py | 1 - 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/orangecontrib/educational/widgets/owgradientdescent.py b/orangecontrib/educational/widgets/owgradientdescent.py index dec24d3c..81d0dcb7 100644 --- a/orangecontrib/educational/widgets/owgradientdescent.py +++ b/orangecontrib/educational/widgets/owgradientdescent.py @@ -337,7 +337,7 @@ def restart(self): """ self.selected_data = self.select_data() self.learner = self.default_learner( - data=Normalize(self.selected_data), + data=self.selected_data, alpha=self.alpha, stochastic=self.stochastic) self.replot() self.send_output() @@ -519,7 +519,7 @@ def select_data(self): y = [(0 if d.get_class().value == self.target_class else 1) for d in self.data] - return Table(domain, x, y, self.data.Y[:, None]) + return Normalize(Table(domain, x, y, self.data.Y[:, None])) def plot_contour(self, xv, yv, cost_grid): """ diff --git a/orangecontrib/educational/widgets/utils/logistic_regression.py b/orangecontrib/educational/widgets/utils/logistic_regression.py index 4291dbbc..e890ff80 100644 --- a/orangecontrib/educational/widgets/utils/logistic_regression.py +++ b/orangecontrib/educational/widgets/utils/logistic_regression.py @@ -212,7 +212,6 @@ def __init__(self, theta, domain): super().__init__(domain) self.theta = theta self.name = "Logistic Regression" - print("a") def predict_storage(self, data): probabilities = LogisticRegression.g(data.X.dot(self.theta)) From 092e8663bba383835b48c5c900390bc7554bc123 Mon Sep 17 00:00:00 2001 From: PrimozGodec Date: Thu, 4 Aug 2016 12:37:54 +0200 Subject: [PATCH 016/128] Introduced very small regularization rate to avoid high numbers when data are good separated between classes, Alpha field max set to 10 --- .../educational/widgets/owgradientdescent.py | 16 +++------------- .../widgets/utils/logistic_regression.py | 13 ++++++++----- 2 files changed, 11 insertions(+), 18 deletions(-) diff --git a/orangecontrib/educational/widgets/owgradientdescent.py b/orangecontrib/educational/widgets/owgradientdescent.py index 81d0dcb7..bdfe4a0d 100644 --- a/orangecontrib/educational/widgets/owgradientdescent.py +++ b/orangecontrib/educational/widgets/owgradientdescent.py @@ -218,7 +218,7 @@ def __init__(self): self.alpha_spin = gui.spin( widget=self.properties_box, master=self, callback=self.change_alpha, value="alpha", label="Learning rate: ", - minv=0.01, maxv=1, step=0.01, spinType=float) + minv=0.01, maxv=10, step=0.01, spinType=float) self.stochastic_checkbox = gui.checkBox( widget=self.properties_box, master=self, callback=self.change_stochastic, value="stochastic", @@ -480,10 +480,8 @@ def plot_gradient_and_contour(self, x_from, x_to, y_from, y_to): # results self.cost_grid = cost_values.reshape(xv.shape) - blurred = self.blur_grid(self.cost_grid) - # return self.plot_gradient(self.xv, self.yv, blurred) + \ - return self.plot_contour(xv, yv, blurred) + return self.plot_contour(xv, yv, self.cost_grid) def plot_gradient(self, x, y, grid): """ @@ -529,7 +527,7 @@ def plot_contour(self, xv, yv, cost_grid): contour = Contour( xv, yv, cost_grid) contour_lines = contour.contours( - np.linspace(np.min(cost_grid), np.max(cost_grid), 10)) + np.linspace(np.min(cost_grid), np.max(cost_grid), 20)) series = [] count = 0 @@ -549,14 +547,6 @@ def plot_contour(self, xv, yv, cost_grid): count += 1 return series - @staticmethod - def blur_grid(grid): - """ - Function blur the grid, to make crossings smoother - """ - filtered = gaussian_filter(grid, sigma=1) - return filtered - def auto_play(self): """ Function called when autoplay button pressed diff --git a/orangecontrib/educational/widgets/utils/logistic_regression.py b/orangecontrib/educational/widgets/utils/logistic_regression.py index e890ff80..fbb9804a 100644 --- a/orangecontrib/educational/widgets/utils/logistic_regression.py +++ b/orangecontrib/educational/widgets/utils/logistic_regression.py @@ -27,6 +27,8 @@ class LogisticRegression: step_no = 0 stochastic_i = 0 stochastic_num_steps = 30 # number of steps in one step + regularization_rate = 0.001 + # very small regularization rate to avoid big parameters def __init__(self, alpha=0.1, theta=None, data=None, stochastic=False): self.history = [] @@ -136,8 +138,8 @@ def j(self, theta): """ yh = self.g(self.x.dot(theta.T)).T y = self.y - return -np.sum( - (self.y * np.log(yh) + (1 - y) * np.log(1 - yh)).T, axis=0) / len(y) + return (-np.sum((y * np.log(yh) + (1 - y) * np.log(1 - yh)).T, axis=0) + + self.regularization_rate * np.sum(np.square(theta.T), axis=0)) def dj(self, theta, stochastic=False): """ @@ -149,7 +151,8 @@ def dj(self, theta, stochastic=False): y = self.y[self.stochastic_i: self.stochastic_i + ns] return x.T.dot(self.g(x.dot(theta)) - y) else: - return (self.g(self.x.dot(theta)) - self.y).dot(self.x) + return ((self.g(self.x.dot(theta)) - self.y).dot(self.x) + + self.regularization_rate * theta) def optimized(self): """ @@ -172,8 +175,8 @@ def g(z): """ # limit values in z to avoid log with 0 produced by values almost 0 - z_mod = np.minimum(z, 100) - z_mod = np.maximum(z_mod, -100) + z_mod = np.minimum(z, 20) + z_mod = np.maximum(z_mod, -20) return 1.0 / (1 + np.exp(- z_mod)) From d0a11368490a5139195fa359ed25f7d9b4de6094 Mon Sep 17 00:00:00 2001 From: PrimozGodec Date: Thu, 4 Aug 2016 14:25:37 +0200 Subject: [PATCH 017/128] Solved bug with zoom --- orangecontrib/educational/widgets/owgradientdescent.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/orangecontrib/educational/widgets/owgradientdescent.py b/orangecontrib/educational/widgets/owgradientdescent.py index bdfe4a0d..aefd1b0e 100644 --- a/orangecontrib/educational/widgets/owgradientdescent.py +++ b/orangecontrib/educational/widgets/owgradientdescent.py @@ -51,6 +51,7 @@ def __init__(self, click_callback, **kwargs): enable_select='', chart_events_click=self.js_click_function, plotOptions_series_states_hover_enabled=False, + chart_panning=False, javascript=contours_js, **kwargs) @@ -437,8 +438,8 @@ def replot(self): # [min_value, "#ffffff"], # [max_value, "#ff0000"]], # tickInterval=1, max=max_value, min=min_value), - plotOptions_contour_colsize=(self.max_y - self.min_y) / 10000, - plotOptions_contour_rowsize=(self.max_x - self.min_x) / 10000, + # plotOptions_contour_colsize=(self.max_y - self.min_y) / 10000, + # plotOptions_contour_rowsize=(self.max_x - self.min_x) / 10000, tooltip_enabled=False, tooltip_headerFormat="", tooltip_pointFormat="%s: {point.x:.2f}
" @@ -480,7 +481,7 @@ def plot_gradient_and_contour(self, x_from, x_to, y_from, y_to): # results self.cost_grid = cost_values.reshape(xv.shape) - # return self.plot_gradient(self.xv, self.yv, blurred) + \ + # return self.plot_gradient(xv, yv, self.cost_grid) + \ return self.plot_contour(xv, yv, self.cost_grid) def plot_gradient(self, x, y, grid): From b9707c215364a0e742163f7bcb04a71bb7e5f1c0 Mon Sep 17 00:00:00 2001 From: PrimozGodec Date: Thu, 4 Aug 2016 14:30:52 +0200 Subject: [PATCH 018/128] Code refactor. --- .../educational/widgets/owgradientdescent.py | 20 +++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/orangecontrib/educational/widgets/owgradientdescent.py b/orangecontrib/educational/widgets/owgradientdescent.py index aefd1b0e..82983af0 100644 --- a/orangecontrib/educational/widgets/owgradientdescent.py +++ b/orangecontrib/educational/widgets/owgradientdescent.py @@ -407,10 +407,10 @@ def replot(self): return optimal_theta = self.learner.optimized() - self.min_x = optimal_theta[0] - 5 - self.max_x = optimal_theta[0] + 5 - self.min_y = optimal_theta[1] - 5 - self.max_y = optimal_theta[1] + 5 + self.min_x = optimal_theta[0] - 10 + self.max_x = optimal_theta[0] + 10 + self.min_y = optimal_theta[1] - 10 + self.max_y = optimal_theta[1] + 10 options = dict(series=[]) @@ -584,17 +584,26 @@ def disable_controls(self, disabled): self.properties_box.setDisabled(disabled) def send_output(self): + """ + Function sends output + """ self.send_model() self.send_coefficients() self.send_data() def send_model(self): + """ + Function sends model on output. + """ if self.learner is not None and self.learner.theta is not None: self.send("Classifier", self.learner.model) else: self.send("Classifier", None) def send_coefficients(self): + """ + Function sends logistic regression coefficients on output. + """ if self.learner is not None and self.learner.theta is not None: domain = Domain( [ContinuousVariable("coef", number_of_decimals=7)], @@ -608,6 +617,9 @@ def send_coefficients(self): self.send("Coefficients", None) def send_data(self): + """ + Function sends data on output. + """ if self.selected_data is not None: self.send("Data", self.selected_data) else: From 76172bcefadabb0d5ab401f92f3aec7c6bce06c5 Mon Sep 17 00:00:00 2001 From: PrimozGodec Date: Thu, 4 Aug 2016 14:30:52 +0200 Subject: [PATCH 019/128] Code refactor. --- .../educational/widgets/owgradientdescent.py | 31 ++++++++++--------- 1 file changed, 16 insertions(+), 15 deletions(-) diff --git a/orangecontrib/educational/widgets/owgradientdescent.py b/orangecontrib/educational/widgets/owgradientdescent.py index aefd1b0e..42fbe07d 100644 --- a/orangecontrib/educational/widgets/owgradientdescent.py +++ b/orangecontrib/educational/widgets/owgradientdescent.py @@ -407,10 +407,10 @@ def replot(self): return optimal_theta = self.learner.optimized() - self.min_x = optimal_theta[0] - 5 - self.max_x = optimal_theta[0] + 5 - self.min_y = optimal_theta[1] - 5 - self.max_y = optimal_theta[1] + 5 + self.min_x = optimal_theta[0] - 10 + self.max_x = optimal_theta[0] + 10 + self.min_y = optimal_theta[1] - 10 + self.max_y = optimal_theta[1] + 10 options = dict(series=[]) @@ -418,9 +418,6 @@ def replot(self): options['series'] += self.plot_gradient_and_contour( self.min_x, self.max_x, self.min_y, self.max_y) - min_value = np.min(self.cost_grid) - max_value = np.max(self.cost_grid) - # highcharts parameters kwargs = dict( xAxis_title_text="theta 0", @@ -433,13 +430,6 @@ def replot(self): xAxis_endOnTick=False, yAxis_startOnTick=False, yAxis_endOnTick=False, - # colorAxis=dict( - # stops=[ - # [min_value, "#ffffff"], - # [max_value, "#ff0000"]], - # tickInterval=1, max=max_value, min=min_value), - # plotOptions_contour_colsize=(self.max_y - self.min_y) / 10000, - # plotOptions_contour_rowsize=(self.max_x - self.min_x) / 10000, tooltip_enabled=False, tooltip_headerFormat="", tooltip_pointFormat="%s: {point.x:.2f}
" @@ -481,7 +471,6 @@ def plot_gradient_and_contour(self, x_from, x_to, y_from, y_to): # results self.cost_grid = cost_values.reshape(xv.shape) - # return self.plot_gradient(xv, yv, self.cost_grid) + \ return self.plot_contour(xv, yv, self.cost_grid) def plot_gradient(self, x, y, grid): @@ -584,17 +573,26 @@ def disable_controls(self, disabled): self.properties_box.setDisabled(disabled) def send_output(self): + """ + Function sends output + """ self.send_model() self.send_coefficients() self.send_data() def send_model(self): + """ + Function sends model on output. + """ if self.learner is not None and self.learner.theta is not None: self.send("Classifier", self.learner.model) else: self.send("Classifier", None) def send_coefficients(self): + """ + Function sends logistic regression coefficients on output. + """ if self.learner is not None and self.learner.theta is not None: domain = Domain( [ContinuousVariable("coef", number_of_decimals=7)], @@ -608,6 +606,9 @@ def send_coefficients(self): self.send("Coefficients", None) def send_data(self): + """ + Function sends data on output. + """ if self.selected_data is not None: self.send("Data", self.selected_data) else: From 684e0cc3fc210156655d318a2f5f0bd62c99874e Mon Sep 17 00:00:00 2001 From: PrimozGodec Date: Thu, 4 Aug 2016 15:55:26 +0200 Subject: [PATCH 020/128] Part one of unit test for logistic regression. --- .../widgets/utils/logistic_regression.py | 23 +- .../utils/tests/test_logistic_regression.py | 265 ++++++++++++++++++ 2 files changed, 273 insertions(+), 15 deletions(-) create mode 100644 orangecontrib/educational/widgets/utils/tests/test_logistic_regression.py diff --git a/orangecontrib/educational/widgets/utils/logistic_regression.py b/orangecontrib/educational/widgets/utils/logistic_regression.py index fbb9804a..13e835cf 100644 --- a/orangecontrib/educational/widgets/utils/logistic_regression.py +++ b/orangecontrib/educational/widgets/utils/logistic_regression.py @@ -45,6 +45,10 @@ def set_data(self, data): self.x = data.X self.y = data.Y self.domain = data.domain + else: + self.x = None + self.y = None + self.domain = None def set_theta(self, theta): """ @@ -70,7 +74,10 @@ def model(self): """ Function returns model based on current parameters. """ - return LogisticRegressionModel(self.theta, self.domain) + if self.theta is None or self.domain is None: + return None + else: + return LogisticRegressionModel(self.theta, self.domain) @property def converged(self): @@ -185,20 +192,6 @@ def set_list(l, i, v): """ Function sets i-th value in list to v. If i does not exist in l it is initialized else value is modified - - Parameters - ---------- - l : list - List - i : int - Index of position in list - v : any - Value to insert in list - - Returns - ------- - list - List with inserted value v on position i """ try: l[i] = v diff --git a/orangecontrib/educational/widgets/utils/tests/test_logistic_regression.py b/orangecontrib/educational/widgets/utils/tests/test_logistic_regression.py new file mode 100644 index 00000000..eacad98f --- /dev/null +++ b/orangecontrib/educational/widgets/utils/tests/test_logistic_regression.py @@ -0,0 +1,265 @@ +import unittest +from Orange.data import Table, Domain +from orangecontrib.educational.widgets.utils.logistic_regression import \ + LogisticRegression +from numpy.testing import * +import numpy as np + +class TestKmeans(unittest.TestCase): + + def setUp(self): + self.iris = Table('iris') + # new_domain = Domain(self.data.domain.attributes[:2]) + # self.data = Table(new_domain, self.data) + self.logistic_regression = LogisticRegression() + + def test_set_data(self): + """ + Test set data + """ + + # check if None on beginning + self.assertIsNone(self.logistic_regression.x, None) + self.assertIsNone(self.logistic_regression.y, None) + self.assertIsNone(self.logistic_regression.domain, None) + + # check if correct data are provided + self.logistic_regression.set_data(self.iris) + + assert_array_equal(self.logistic_regression.x, self.iris.X) + assert_array_equal(self.logistic_regression.y, self.iris.Y) + self.assertEqual(self.logistic_regression.domain, self.iris.domain) + + # check data remove + self.logistic_regression.set_data(None) + + self.assertIsNone(self.logistic_regression.x, None) + self.assertIsNone(self.logistic_regression.y, None) + self.assertIsNone(self.logistic_regression.domain, None) + + def test_set_theta(self): + """ + Check set theta + """ + + lr = self.logistic_regression + + # theta must be none on beginning + self.assertIsNone(lr.theta, None) + + # check if theta set correctly + # theta from np array + lr.set_theta(np.array([1, 2])) + assert_array_equal(lr.theta, np.array([1, 2])) + # history of 0 have to be equal theta + assert_array_equal(lr.history[0][0], np.array([1, 2])) + # step no have to reset to 0 + self.assertEqual(lr.step_no, 0) + + # theta from list + lr.set_theta([2, 3]) + assert_array_equal(lr.theta, np.array([2, 3])) + assert_array_equal(lr.history[0][0], np.array([2, 3])) + self.assertEqual(lr.step_no, 0) + + # theta None + lr.set_theta(None) + self.assertIsNone(lr.theta) + + # theta anything else + lr.set_theta("abc") + self.assertIsNone(lr.theta) + + def test_set_alpha(self): + """ + Check if alpha set correctly + """ + lr = self.logistic_regression + + # check alpha 0.1 in the beginning + self.assertEqual(lr.alpha, 0.1) + + # check if alpha set correctly + lr.set_alpha(0.2) + self.assertEqual(lr.alpha, 0.2) + + # check if alpha removed correctly + lr.set_alpha(None) + self.assertIsNone(lr.alpha) + + def test_model(self): + """ + Test if model is correct + """ + lr = self.logistic_regression + + # test if model None when no data + lr.set_theta([1, 2]) + self.assertIsNone(lr.model) + + # test if model None when no theta + lr.set_theta(None) + lr.set_data(self.iris) + self.assertIsNone(lr.model) + + # test if model None when no theta and no Data + lr.set_data(None) + self.assertIsNone(lr.model) + + # test when model is not none + lr.set_data(self.iris) + lr.set_theta([1, 1, 1, 1]) + model = lr.model + + # test parameters are ok + self.assertIsNotNone(model) + assert_array_equal(model.theta, np.array([1, 1, 1, 1])) + self.assertEqual(model.name, "Logistic Regression") + + # test class returns correct predictions + values, probabilities = model(self.iris, ret=2) + self.assertEqual(len(values), len(self.iris)) + self.assertEqual(len(probabilities), len(self.iris)) + # values have to be 0 if prob <0.5 else 1 + assert_array_equal(values, np.around(probabilities)[:,1]) + + def test_converged(self): + """ + Test convergence flag or the algorithm + """ + lr = self.logistic_regression + lr.set_data(self.iris) + lr.set_theta([1., 1., 1., 1.]) + lr.set_alpha(1) + # we found out for example in test convergence is faster with this alpha + + # it can not converge in the first step + self.assertFalse(lr.converged) + + # it converge when distance between current theta and this is < 1e-2 + converge = False + while not converge: + lr.step() + converge = np.sum( + np.abs(lr.theta - lr.history[lr.step_no - 1][0])) < 1e-2 + self.assertEqual(lr.converged, converge) + + def test_step(self): + """ + Test step method + """ + lr = self.logistic_regression + + lr.set_theta([1., 1., 1., 1.]) + lr.set_data(self.iris) + + # check beginning + self.assertEqual(lr.step_no, 0) + + # perform step + lr.step() + + # check if parameters are fine + self.assertEqual(len(lr.theta), 4) + assert_array_equal(lr.history[1][0], lr.theta) + + # perform step + lr.step() + + # check if parameters are fine + self.assertEqual(len(lr.theta), 4) + assert_array_equal(lr.history[2][0], lr.theta) + + # check for stochastic + lr.stochastic = True + + # perform step + lr.step() + self.assertEqual(len(lr.theta), 4) + assert_array_equal(lr.history[3][0], lr.theta) + + # check if stochastic_i indices are ok + self.assertEqual(lr.history[3][1], lr.stochastic_i) + + # reset algorithm + lr.set_data(self.iris) + + # wait for shuffle and check if fine + shuffle = False + while not shuffle: + lr.step() + shuffle = lr.history[lr.step_no][2] is not None + if shuffle: + self.assertEqual(len(lr.x), len(self.iris)) + self.assertEqual(len(lr.y), len(self.iris)) + + def test_step_back(self): + """ + Test step back function + """ + lr = self.logistic_regression + theta = [1., 1., 1., 1.] + + lr.set_data(self.iris) + lr.set_theta(theta) + + # check no step back when no step done before + lr.step_back() + assert_array_equal(lr.theta, theta) + self.assertEqual(lr.step_no, 0) + + # perform step and step back + lr.step() + lr.step_back() + assert_array_equal(lr.theta, theta) + self.assertEqual(lr.step_no, 0) + + lr.step() + theta1 = np.copy(lr.theta) + lr.step() + lr.step_back() + + assert_array_equal(lr.theta, theta1) + self.assertEqual(lr.step_no, 1) + + lr.step_back() + + assert_array_equal(lr.theta, theta) + self.assertEqual(lr.step_no, 0) + + # test for stochastic + lr.stochastic = True + + lr.step() + lr.step_back() + self.assertEqual(lr.stochastic_i, 0) + self.assertEqual(lr.step_no, 0) + + lr.step() + theta1 = np.copy(lr.theta) + lr.step() + lr.step_back() + + self.assertEqual(lr.stochastic_i, lr.stochastic_num_steps) + self.assertEqual(lr.step_no, 1) + + lr.step_back() + + self.assertEqual(lr.stochastic_i, 0) + self.assertEqual(lr.step_no, 0) + + # wait for shuffle and check if fine + shuffle = False + before = np.copy(lr.x) + while not shuffle: + lr.step() + shuffle = lr.history[lr.step_no][2] is not None + + lr.step_back() + assert_array_equal(lr.x, before) + + + + + + From a8ca9d6f76bd3a9cedd4db0d6c42f4d9e1b13449 Mon Sep 17 00:00:00 2001 From: PrimozGodec Date: Fri, 5 Aug 2016 10:18:03 +0200 Subject: [PATCH 021/128] Complete unittest for logistic regression --- .../utils/tests/test_logistic_regression.py | 96 ++++++++++++++++++- 1 file changed, 93 insertions(+), 3 deletions(-) diff --git a/orangecontrib/educational/widgets/utils/tests/test_logistic_regression.py b/orangecontrib/educational/widgets/utils/tests/test_logistic_regression.py index eacad98f..99c15fbd 100644 --- a/orangecontrib/educational/widgets/utils/tests/test_logistic_regression.py +++ b/orangecontrib/educational/widgets/utils/tests/test_logistic_regression.py @@ -1,10 +1,11 @@ import unittest -from Orange.data import Table, Domain +from Orange.data import Table from orangecontrib.educational.widgets.utils.logistic_regression import \ LogisticRegression from numpy.testing import * import numpy as np + class TestKmeans(unittest.TestCase): def setUp(self): @@ -121,7 +122,7 @@ def test_model(self): self.assertEqual(len(values), len(self.iris)) self.assertEqual(len(probabilities), len(self.iris)) # values have to be 0 if prob <0.5 else 1 - assert_array_equal(values, np.around(probabilities)[:,1]) + assert_array_equal(values, np.around(probabilities)[:, 1]) def test_converged(self): """ @@ -236,7 +237,6 @@ def test_step_back(self): self.assertEqual(lr.step_no, 0) lr.step() - theta1 = np.copy(lr.theta) lr.step() lr.step_back() @@ -258,8 +258,98 @@ def test_step_back(self): lr.step_back() assert_array_equal(lr.x, before) + def test_j(self): + """ + Test cost function j + """ + lr = self.logistic_regression + + lr.set_data(self.iris) + + # test with one theta and with list of thetas + self.assertEqual(type(lr.j(np.array([1., 1., 1., 1.]))), np.float64) + self.assertEqual( + len(lr.j(np.array([[1., 1., 1., 1.], [2, 2, 2, 2]]))), 2) + + def test_dj(self): + """ + Test gradient function + """ + lr = self.logistic_regression + lr.set_data(self.iris) + # check length with stochastic and usual + self.assertEqual(len(lr.dj(np.array([1, 1, 1, 1]))), 4) + lr.stochastic = True + self.assertEqual(len(lr.dj(np.array([1, 1, 1, 1]))), 4) + def test_optimized(self): + """ + Test if optimized works well + """ + lr = self.logistic_regression + lr.set_data(self.iris) + op_theta = lr.optimized() + self.assertEqual(len(op_theta), 4) + + # check if really minimal, function is monotonic so everywhere around + # j should be higher + self.assertLessEqual( + lr.j(op_theta), lr.j(op_theta + np.array([1, 0, 0, 0]))) + self.assertLessEqual( + lr.j(op_theta), lr.j(op_theta + np.array([0, 1, 0, 0]))) + self.assertLessEqual( + lr.j(op_theta), lr.j(op_theta + np.array([0, 0, 1, 0]))) + self.assertLessEqual( + lr.j(op_theta), lr.j(op_theta + np.array([0, 0, 0, 1]))) + + def test_g(self): + """ + Test sigmoid function + """ + lr = self.logistic_regression + # test length + self.assertEqual(type(lr.g(1)), np.float64) + self.assertEqual(len(lr.g(np.array([1, 1]))), 2) + self.assertEqual(len(lr.g(np.array([1, 1, 1]))), 3) + self.assertEqual(len(lr.g(np.array([1, 1, 1, 1]))), 4) + + # test correctness, function between 0 and 1 + self.assertGreaterEqual(lr.g(-10000), 0) + self.assertGreaterEqual(lr.g(-1000), 0) + self.assertGreaterEqual(lr.g(-10), 0) + self.assertGreaterEqual(lr.g(-1), 0) + self.assertGreaterEqual(lr.g(0), 0) + self.assertGreaterEqual(lr.g(1), 0) + self.assertGreaterEqual(lr.g(10), 0) + self.assertGreaterEqual(lr.g(1000), 0) + self.assertGreaterEqual(lr.g(10000), 0) + + self.assertLessEqual(lr.g(-10000), 1) + self.assertLessEqual(lr.g(-1000), 1) + self.assertLessEqual(lr.g(-10), 1) + self.assertLessEqual(lr.g(-1), 1) + self.assertLessEqual(lr.g(0), 1) + self.assertLessEqual(lr.g(1), 1) + self.assertLessEqual(lr.g(10), 1) + self.assertLessEqual(lr.g(1000), 1) + self.assertLessEqual(lr.g(10000), 1) + + def test_set_list(self): + """ + Test set list + """ + lr = self.logistic_regression + # test adding Nones if list too short + self.assertEqual(lr.set_list([], 2, 1), [None, None, 1]) + # test adding Nones if list too short + self.assertEqual(lr.set_list([2], 2, 1), [2, None, 1]) + # adding to end + self.assertEqual(lr.set_list([2, 1], 2, 1), [2, 1, 1]) + # changing the element in the last place + self.assertEqual(lr.set_list([2, 1], 1, 3), [2, 3]) + # changing the element in the middle place + self.assertEqual(lr.set_list([2, 1, 3], 1, 3), [2, 3, 3]) From 008098904f467bd0d1fabc349d22f2b00f931929 Mon Sep 17 00:00:00 2001 From: PrimozGodec Date: Mon, 8 Aug 2016 10:15:50 +0200 Subject: [PATCH 022/128] Added field for step size for stochastic GDS --- .../educational/widgets/owgradientdescent.py | 12 +++++++++++- .../educational/widgets/utils/logistic_regression.py | 10 ++++++---- .../widgets/utils/tests/test_logistic_regression.py | 2 +- 3 files changed, 18 insertions(+), 6 deletions(-) diff --git a/orangecontrib/educational/widgets/owgradientdescent.py b/orangecontrib/educational/widgets/owgradientdescent.py index 42fbe07d..0e4d7a63 100644 --- a/orangecontrib/educational/widgets/owgradientdescent.py +++ b/orangecontrib/educational/widgets/owgradientdescent.py @@ -154,6 +154,7 @@ class OWGradientDescent(OWWidget): attr_y = settings.Setting('') target_class = settings.Setting('') alpha = settings.Setting(0.1) + step_size = settings.Setting(30) # step size for stochastic gds auto_play_speed = settings.Setting(1) stochastic = settings.Setting(False) @@ -224,6 +225,10 @@ def __init__(self): widget=self.properties_box, master=self, callback=self.change_stochastic, value="stochastic", label="Stochastic: ") + self.step_size_spin = gui.spin( + widget=self.properties_box, master=self, callback=self.change_step, + value="step_size", label="Step size: ", + minv=1, maxv=100, step=1) self.restart_button = gui.button( widget=self.properties_box, master=self, callback=self.restart, label="Restart") @@ -339,7 +344,8 @@ def restart(self): self.selected_data = self.select_data() self.learner = self.default_learner( data=self.selected_data, - alpha=self.alpha, stochastic=self.stochastic) + alpha=self.alpha, stochastic=self.stochastic, + step_size=self.step_size) self.replot() self.send_output() @@ -357,6 +363,10 @@ def change_stochastic(self): if self.learner is not None: self.learner.stochastic = self.stochastic + def change_step(self): + if self.learner is not None: + self.learner.stochastic_step_size = self.step_size + def change_theta(self, x, y): """ Function set new theta diff --git a/orangecontrib/educational/widgets/utils/logistic_regression.py b/orangecontrib/educational/widgets/utils/logistic_regression.py index 13e835cf..87a275e9 100644 --- a/orangecontrib/educational/widgets/utils/logistic_regression.py +++ b/orangecontrib/educational/widgets/utils/logistic_regression.py @@ -26,16 +26,18 @@ class LogisticRegression: domain = None step_no = 0 stochastic_i = 0 - stochastic_num_steps = 30 # number of steps in one step + stochastic_step_size = 30 # number of steps in one step regularization_rate = 0.001 # very small regularization rate to avoid big parameters - def __init__(self, alpha=0.1, theta=None, data=None, stochastic=False): + def __init__(self, alpha=0.1, theta=None, data=None, stochastic=False, + step_size=30): self.history = [] self.set_alpha(alpha) self.set_data(data) self.set_theta(theta) self.stochastic = stochastic + self.stochastic_step_size = step_size def set_data(self, data): """ @@ -100,7 +102,7 @@ def step(self): self.theta -= self.alpha * grad # increase index used by stochastic gradient descent - self.stochastic_i += self.stochastic_num_steps + self.stochastic_i += self.stochastic_step_size seed = None # seed that will be stored to revert the shuffle # if we came around all data set index to zero and permute data @@ -153,7 +155,7 @@ def dj(self, theta, stochastic=False): Gradient of the cost function for logistic regression """ if stochastic: - ns = self.stochastic_num_steps + ns = self.stochastic_step_size x = self.x[self.stochastic_i: self.stochastic_i + ns] y = self.y[self.stochastic_i: self.stochastic_i + ns] return x.T.dot(self.g(x.dot(theta)) - y) diff --git a/orangecontrib/educational/widgets/utils/tests/test_logistic_regression.py b/orangecontrib/educational/widgets/utils/tests/test_logistic_regression.py index 99c15fbd..54e49dbc 100644 --- a/orangecontrib/educational/widgets/utils/tests/test_logistic_regression.py +++ b/orangecontrib/educational/widgets/utils/tests/test_logistic_regression.py @@ -240,7 +240,7 @@ def test_step_back(self): lr.step() lr.step_back() - self.assertEqual(lr.stochastic_i, lr.stochastic_num_steps) + self.assertEqual(lr.stochastic_i, lr.stochastic_step_size) self.assertEqual(lr.step_no, 1) lr.step_back() From ae2597a37a55d7a568d07fd22d07158743f76c61 Mon Sep 17 00:00:00 2001 From: PrimozGodec Date: Mon, 8 Aug 2016 12:53:27 +0200 Subject: [PATCH 023/128] Unit test for OWGradientDescent and some bug fix and small upgrades of code --- .../educational/widgets/owgradientdescent.py | 92 +-- .../widgets/tests/test_owgradientdescent.py | 525 ++++++++++++++++++ 2 files changed, 575 insertions(+), 42 deletions(-) create mode 100644 orangecontrib/educational/widgets/tests/test_owgradientdescent.py diff --git a/orangecontrib/educational/widgets/owgradientdescent.py b/orangecontrib/educational/widgets/owgradientdescent.py index 0e4d7a63..de0275c1 100644 --- a/orangecontrib/educational/widgets/owgradientdescent.py +++ b/orangecontrib/educational/widgets/owgradientdescent.py @@ -304,6 +304,7 @@ def init_combos(): # clear variables self.cost_grid = None self.learner = None + self.selected_data = None d = data self.send_output() @@ -314,6 +315,7 @@ def init_combos(): self.set_empty_plot() elif sum(True for var in d.domain.attributes if isinstance(var, ContinuousVariable)) < 2: + # not enough (2) continuous variable self.data = None reset_combos() self.Warning.to_few_features() @@ -398,6 +400,8 @@ def step_back(self): """ Function performs step back """ + if self.data is None: + return if self.learner.step_no > 0: self.learner.step_back() self.scatter.remove_last_point("path") @@ -492,33 +496,6 @@ def plot_gradient(self, x, y, grid): grid_width=self.grid_size, type="contour")] - def select_data(self): - """ - Function takes two selected columns from data table and merge them - in new Orange.data.Table - - Returns - ------- - Table - Table with selected columns - """ - attr_x = self.data.domain[self.attr_x] - attr_y = self.data.domain[self.attr_y] - cols = [] - for attr in (attr_x, attr_y): - subset = self.data[:, attr] - cols.append(subset.X) - x = np.column_stack(cols) - domain = Domain( - [attr_x, attr_y], - [DiscreteVariable(name=self.data.domain.class_var.name, - values=[self.target_class, 'Others'])], - [self.data.domain.class_var]) - y = [(0 if d.get_class().value == self.target_class else 1) - for d in self.data] - - return Normalize(Table(domain, x, y, self.data.Y[:, None])) - def plot_contour(self, xv, yv, cost_grid): """ Function constructs contour lines @@ -547,23 +524,54 @@ def plot_contour(self, xv, yv, cost_grid): count += 1 return series + def select_data(self): + """ + Function takes two selected columns from data table and merge them + in new Orange.data.Table + + Returns + ------- + Table + Table with selected columns + """ + if self.data is None: + return + + attr_x = self.data.domain[self.attr_x] + attr_y = self.data.domain[self.attr_y] + cols = [] + for attr in (attr_x, attr_y): + subset = self.data[:, attr] + cols.append(subset.X) + x = np.column_stack(cols) + domain = Domain( + [attr_x, attr_y], + [DiscreteVariable(name=self.data.domain.class_var.name, + values=[self.target_class, 'Others'])], + [self.data.domain.class_var]) + y = [(0 if d.get_class().value == self.target_class else 1) + for d in self.data] + + return Normalize(Table(domain, x, y, self.data.Y[:, None])) + def auto_play(self): """ Function called when autoplay button pressed """ - self.auto_play_enabled = not self.auto_play_enabled - self.auto_play_button.setText( - self.auto_play_button_text[self.auto_play_enabled]) - if self.auto_play_enabled: - self.disable_controls(self.auto_play_enabled) - self.auto_play_thread = Autoplay(self) - self.connect(self.auto_play_thread, SIGNAL("step()"), self.step) - self.connect( - self.auto_play_thread, SIGNAL("stop_auto_play()"), - self.stop_auto_play) - self.auto_play_thread.start() - else: - self.stop_auto_play() + if self.data is not None: + self.auto_play_enabled = not self.auto_play_enabled + self.auto_play_button.setText( + self.auto_play_button_text[self.auto_play_enabled]) + if self.auto_play_enabled: + self.disable_controls(self.auto_play_enabled) + self.auto_play_thread = Autoplay(self) + self.connect(self.auto_play_thread, SIGNAL("step()"), self.step) + self.connect( + self.auto_play_thread, SIGNAL("stop_auto_play()"), + self.stop_auto_play) + self.auto_play_thread.start() + else: + self.stop_auto_play() def stop_auto_play(self): """ @@ -605,8 +613,8 @@ def send_coefficients(self): """ if self.learner is not None and self.learner.theta is not None: domain = Domain( - [ContinuousVariable("coef", number_of_decimals=7)], - metas=[StringVariable("name")]) + [ContinuousVariable("Coefficients", number_of_decimals=7)], + metas=[StringVariable("Name")]) names = ["theta 0", "theta 1"] coefficients_table = Table( diff --git a/orangecontrib/educational/widgets/tests/test_owgradientdescent.py b/orangecontrib/educational/widgets/tests/test_owgradientdescent.py new file mode 100644 index 00000000..23863c31 --- /dev/null +++ b/orangecontrib/educational/widgets/tests/test_owgradientdescent.py @@ -0,0 +1,525 @@ +from numpy.testing import * +import numpy as np + +from Orange.data import Table, Domain, ContinuousVariable, DiscreteVariable +from Orange.widgets.tests.base import WidgetTest + +from orangecontrib.educational.widgets.owgradientdescent import \ + OWGradientDescent + + +class TestOWGradientDescent(WidgetTest): + + def setUp(self): + self.widget = self.create_widget(OWGradientDescent) + self.iris = Table('iris') + + def test_set_data(self): + """ + Test set data + """ + w = self.widget + + # test on init + self.assertIsNone(w.data) + self.assertEqual(w.cbx.count(), 0) + self.assertEqual(w.cby.count(), 0) + self.assertEqual(w.target_class_combobox.count(), 0) + self.assertIsNone(w.learner) + self.assertIsNone(w.cost_grid) + + # call with none data + self.send_signal("Data", None) + self.assertIsNone(w.data) + self.assertEqual(w.cbx.count(), 0) + self.assertEqual(w.cby.count(), 0) + self.assertEqual(w.target_class_combobox.count(), 0) + self.assertIsNone(w.learner) + self.assertIsNone(w.cost_grid) + + # call with no class variable + table_no_class = Table( + Domain([ContinuousVariable("x"), ContinuousVariable("y")]), + [[1, 2], [2, 3]]) + self.send_signal("Data", table_no_class) + self.assertIsNone(w.data) + self.assertEqual(w.cbx.count(), 0) + self.assertEqual(w.cby.count(), 0) + self.assertEqual(w.target_class_combobox.count(), 0) + self.assertIsNone(w.learner) + self.assertIsNone(w.cost_grid) + self.assertTrue(w.Warning.no_class.is_shown()) + + # with only one class value + table_one_class = Table( + Domain([ContinuousVariable("x"), ContinuousVariable("y")], + DiscreteVariable("a", values=["k"])), + [[1, 2], [2, 3]], [0, 0]) + self.send_signal("Data", table_one_class) + self.assertIsNone(w.data) + self.assertEqual(w.cbx.count(), 0) + self.assertEqual(w.cby.count(), 0) + self.assertEqual(w.target_class_combobox.count(), 0) + self.assertIsNone(w.learner) + self.assertIsNone(w.cost_grid) + self.assertTrue(w.Warning.no_class.is_shown()) + + # not enough continuous variables + table_no_enough_cont = Table( + Domain( + [ContinuousVariable("x"), + DiscreteVariable("y", values=["a", "b"])], + ContinuousVariable("a")), + [[1, 0], [2, 1]], [0, 0]) + self.send_signal("Data", table_no_enough_cont) + self.assertIsNone(w.data) + self.assertEqual(w.cbx.count(), 0) + self.assertEqual(w.cby.count(), 0) + self.assertEqual(w.target_class_combobox.count(), 0) + self.assertIsNone(w.learner) + self.assertIsNone(w.cost_grid) + self.assertTrue(w.Warning.to_few_features.is_shown()) + + # init with ok data + num_continuous_attributes = sum( + True for var in self.iris.domain.attributes + if isinstance(var, ContinuousVariable)) + + self.send_signal("Data", self.iris) + self.assertEqual(w.cbx.count(), num_continuous_attributes) + self.assertEqual(w.cby.count(), num_continuous_attributes) + self.assertEqual( + w.target_class_combobox.count(), + len(self.iris.domain.class_var.values)) + self.assertEqual(w.cbx.currentText(), self.iris.domain[0].name) + self.assertEqual(w.cby.currentText(), self.iris.domain[1].name) + self.assertEqual( + w.target_class_combobox.currentText(), + self.iris.domain.class_var.values[0]) + + self.assertEqual(w.attr_x, self.iris.domain[0].name) + self.assertEqual(w.attr_y, self.iris.domain[1].name) + self.assertEqual(w.target_class, self.iris.domain.class_var.values[0]) + + # change showed attributes + w.attr_x = self.iris.domain[1].name + w.attr_y = self.iris.domain[2].name + w.target_class = self.iris.domain.class_var.values[1] + + self.assertEqual(w.cbx.currentText(), self.iris.domain[1].name) + self.assertEqual(w.cby.currentText(), self.iris.domain[2].name) + self.assertEqual( + w.target_class_combobox.currentText(), + self.iris.domain.class_var.values[1]) + + self.assertEqual(w.attr_x, self.iris.domain[1].name) + self.assertEqual(w.attr_y, self.iris.domain[2].name) + self.assertEqual(w.target_class, self.iris.domain.class_var.values[1]) + + # remove data + self.send_signal("Data", None) + self.assertIsNone(w.data) + self.assertEqual(w.cbx.count(), 0) + self.assertEqual(w.cby.count(), 0) + self.assertEqual(w.target_class_combobox.count(), 0) + self.assertIsNone(w.learner) + self.assertIsNone(w.cost_grid) + + def test_restart(self): + """ + Test if restart works fine + """ + w = self.widget + + # check if init is as expected + self.assertIsNone(w.selected_data) + self.assertIsNone(w.learner) + + # with data + self.send_signal("Data", self.iris) + self.assertEqual(len(w.selected_data), len(self.iris)) + assert_array_equal(w.learner.x, w.selected_data.X) + assert_array_equal(w.learner.y, w.selected_data.Y) + assert_array_equal(w.learner.domain, w.selected_data.domain) + self.assertEqual(w.learner.alpha, w.alpha) + self.assertEqual(w.learner.stochastic, False) + self.assertEqual(w.learner.stochastic_step_size, w.step_size) + + # again no data + self.send_signal("Data", None) + self.assertIsNone(w.selected_data) + self.assertIsNone(w.learner) + + def test_change_alpha(self): + """ + Function check if alpha is changing correctly + """ + w = self.widget + + # to define learner + self.send_signal("Data", self.iris) + + # check init alpha + self.assertEqual(w.learner.alpha, 0.1) + + # change alpha + w.alpha_spin.setValue(1) + self.assertEqual(w.learner.alpha, 1) + w.alpha_spin.setValue(0.3) + self.assertEqual(w.learner.alpha, 0.3) + + # just check if nothing happens when no learner + self.send_signal("Data", None) + self.assertIsNone(w.learner) + w.alpha_spin.setValue(5) + + def test_change_stochastic(self): + """ + Test changing stochastic + """ + w = self.widget + + # define learner + self.send_signal("Data", self.iris) + + # check init + self.assertFalse(w.learner.stochastic) + + # change stochastic + w.stochastic_checkbox.click() + self.assertTrue(w.learner.stochastic) + w.stochastic_checkbox.click() + self.assertFalse(w.learner.stochastic) + + # just check if nothing happens when no learner + self.send_signal("Data", None) + self.assertIsNone(w.learner) + w.stochastic_checkbox.click() + + def change_step(self): + """ + Function check if change step works correctly + """ + w = self.widget + + # to define learner + self.send_signal("Data", self.iris) + + # check init alpha + self.assertEqual(w.learner.stochastic_step_size, 30) + + # change alpha + w.step_size_spin.setValue(50) + self.assertEqual(w.learner.stochastic_step_size, 50) + w.step_size_spin.setValue(40) + self.assertEqual(w.learner.stochastic_step_size, 40) + + # just check if nothing happens when no learner + self.send_signal("Data", None) + self.assertIsNone(w.learner) + w.step_size_spin.setValue(40) + + def test_change_theta(self): + """ + Test setting theta + """ + w = self.widget + + # to define learner + self.send_signal("Data", self.iris) + + # check init alpha + self.assertIsNone(w.learner.theta) + + # change alpha + w.change_theta(1, 1) + assert_array_equal(w.learner.theta, [1, 1]) + w.change_theta(1, 2) + assert_array_equal(w.learner.theta, [1, 2]) + + # just check if nothing happens when no learner + self.send_signal("Data", None) + self.assertIsNone(w.learner) + w.change_theta(1, 1) + + def test_step(self): + """ + Test step + """ + w = self.widget + + # test function not crashes when no data and learner + w.step() + + self.send_signal("Data", self.iris) + + # test theta set when none + self.assertIsNone(w.learner.theta) + w.step() + self.assertIsNotNone(w.learner.theta) + + # check theta is changing when step + old_theta = np.copy(w.learner.theta) + w.step() + self.assertNotEqual(sum(old_theta - w.learner.theta), 0) + + def test_step_back(self): + """ + Test stepping back + """ + w = self.widget + + # test function not crashes when no data and learner + w.step_back() + + self.send_signal("Data", self.iris) + + # test step back not performed when step_no == 0 + old_theta = np.copy(w.learner.theta) + w.step_back() + assert_array_equal(w.learner.theta, old_theta) + + # test same theta when step performed + w.change_theta(1.0, 1.0) + theta = np.copy(w.learner.theta) + w.step() + w.step_back() + assert_array_equal(theta, w.learner.theta) + + w.change_theta(1.0, 1.0) + theta1 = np.copy(w.learner.theta) + w.step() + theta2 = np.copy(w.learner.theta) + w.step() + theta3 = np.copy(w.learner.theta) + w.step() + w.step_back() + assert_array_equal(theta3, w.learner.theta) + w.step_back() + assert_array_equal(theta2, w.learner.theta) + w.step_back() + assert_array_equal(theta1, w.learner.theta) + w.step_back() + assert_array_equal(theta1, w.learner.theta) + + # test for stochastic + w.stochastic_checkbox.click() + + w.change_theta(1.0, 1.0) + theta = np.copy(w.learner.theta) + w.step() + w.step_back() + assert_array_equal(theta, w.learner.theta) + + w.change_theta(1.0, 1.0) + theta1 = np.copy(w.learner.theta) + w.step() + theta2 = np.copy(w.learner.theta) + w.step() + theta3 = np.copy(w.learner.theta) + w.step() + w.step_back() + assert_array_equal(theta3, w.learner.theta) + w.step_back() + assert_array_equal(theta2, w.learner.theta) + w.step_back() + assert_array_equal(theta1, w.learner.theta) + w.step_back() + assert_array_equal(theta1, w.learner.theta) + + # test mix stochastic and normal + # now it is stochastic + + w.change_theta(1.0, 1.0) + theta1 = np.copy(w.learner.theta) + w.step() + theta2 = np.copy(w.learner.theta) + w.step() + w.stochastic_checkbox.click() + theta3 = np.copy(w.learner.theta) + w.step() + w.step_back() + assert_array_equal(theta3, w.learner.theta) + w.step_back() + assert_array_equal(theta2, w.learner.theta) + w.step_back() + w.stochastic_checkbox.click() + assert_array_equal(theta1, w.learner.theta) + w.step_back() + assert_array_equal(theta1, w.learner.theta) + + def test_replot(self): + """ + Test replot function and all functions connected with it + """ + w = self.widget + # nothing happens when no data + w.replot() + + self.assertIsNone(w.cost_grid) + self.assertEqual(w.scatter.count_replots, 1) + + self.send_signal("Data", self.iris) + self.assertTupleEqual(w.cost_grid.shape, (w.grid_size, w.grid_size)) + self.assertEqual(w.scatter.count_replots, 2) + + # when step no new re-plots + w.step() + self.assertEqual(w.scatter.count_replots, 2) + + # triggered new re-plot + self.send_signal("Data", self.iris) + self.assertTupleEqual(w.cost_grid.shape, (w.grid_size, w.grid_size)) + self.assertEqual(w.scatter.count_replots, 3) + + def test_select_data(self): + """ + Test select data function + """ + w = self.widget + + # test for none data + self.send_signal("Data", None) + + self.assertIsNone(w.select_data()) # result is none + + # test on iris + self.send_signal("Data", self.iris) + self.assertEqual(len(w.select_data()), len(self.iris)) + self.assertEqual(len(w.select_data().domain.attributes), 2) + self.assertEqual(len(w.select_data().domain.class_var.values), 2) + self.assertEqual(w.select_data().domain.class_var.values[1], 'Others') + self.assertEqual(w.select_data().domain.attributes[0].name, w.attr_x) + self.assertEqual(w.select_data().domain.attributes[1].name, w.attr_y) + self.assertEqual( + w.select_data().domain.class_var.values[0], w.target_class) + + def test_autoplay(self): + """ + Test autoplay functionalities + """ + w = self.widget + + # test if not chrashes when data is none + w.auto_play() + + # set data + self.send_signal("Data", self.iris) + + # check init + self.assertFalse(w.auto_play_enabled) + self.assertEqual(w.auto_play_button.text(), w.auto_play_button_text[0]) + self.assertTrue((w.step_box.isEnabled())) + self.assertTrue((w.options_box.isEnabled())) + self.assertTrue((w.properties_box.isEnabled())) + + # auto play on + w.auto_play() + self.assertTrue(w.auto_play_enabled) + self.assertEqual(w.auto_play_button.text(), w.auto_play_button_text[1]) + self.assertFalse((w.step_box.isEnabled())) + self.assertFalse((w.options_box.isEnabled())) + self.assertFalse((w.properties_box.isEnabled())) + + # stop auto play + w.auto_play() + self.assertFalse(w.auto_play_enabled) + self.assertEqual(w.auto_play_button.text(), w.auto_play_button_text[0]) + self.assertTrue((w.step_box.isEnabled())) + self.assertTrue((w.options_box.isEnabled())) + self.assertTrue((w.properties_box.isEnabled())) + + def test_disable_controls(self): + """ + Test disabling controls + """ + w = self.widget + + # check init + self.assertTrue((w.step_box.isEnabled())) + self.assertTrue((w.options_box.isEnabled())) + self.assertTrue((w.properties_box.isEnabled())) + + # disable + w.disable_controls(True) + self.assertFalse((w.step_box.isEnabled())) + self.assertFalse((w.options_box.isEnabled())) + self.assertFalse((w.properties_box.isEnabled())) + + w.disable_controls(True) + self.assertFalse((w.step_box.isEnabled())) + self.assertFalse((w.options_box.isEnabled())) + self.assertFalse((w.properties_box.isEnabled())) + + # enable + w.disable_controls(False) + self.assertTrue((w.step_box.isEnabled())) + self.assertTrue((w.options_box.isEnabled())) + self.assertTrue((w.properties_box.isEnabled())) + + w.disable_controls(False) + self.assertTrue((w.step_box.isEnabled())) + self.assertTrue((w.options_box.isEnabled())) + self.assertTrue((w.properties_box.isEnabled())) + + def test_send_model(self): + """ + Test sending model + """ + w = self.widget + + # when no learner + self.assertIsNone(self.get_output("Classifier")) + + # when learner but no theta + self.send_signal("Data", self.iris) + self.assertIsNone(self.get_output("Classifier")) + + # when everything fine + w.change_theta(1., 1.) + assert_array_equal(self.get_output("Classifier").theta, [1., 1.]) + + # when data deleted + self.send_signal("Data", None) + self.assertIsNone(self.get_output("Classifier")) + + def test_send_coefficients(self): + w = self.widget + + # when no learner + self.assertIsNone(self.get_output("Coefficients")) + + # when learner but no theta + self.send_signal("Data", self.iris) + self.assertIsNone(self.get_output("Coefficients")) + + # when everything fine + w.change_theta(1., 1.) + coef_out = self.get_output("Coefficients") + self.assertEqual(len(coef_out), 2) + self.assertEqual(len(coef_out.domain.attributes), 1) + self.assertEqual(coef_out.domain.attributes[0].name, "Coefficients") + self.assertEqual(len(coef_out.domain.metas), 1) + self.assertEqual(coef_out.domain.metas[0].name, "Name") + + # when data deleted + self.send_signal("Data", None) + self.assertIsNone(self.get_output("Coefficients")) + + def test_send_data(self): + """ + Test sending selected data to output + """ + w = self.widget + + # when no data + self.assertIsNone(self.get_output("Data")) + + # when everything fine + self.send_signal("Data", self.iris) + w.change_theta(1., 1.) + assert_array_equal(self.get_output("Data"), w.selected_data) + + # when data deleted + self.send_signal("Data", None) + self.assertIsNone(self.get_output("Data")) From 911e9a07b4e2c82cddd743a6df8a60425eb3c875 Mon Sep 17 00:00:00 2001 From: PrimozGodec Date: Mon, 8 Aug 2016 13:17:44 +0200 Subject: [PATCH 024/128] Added unit test for contour --- .../widgets/utils/tests/test_contours.py | 452 ++++++++++++++++++ 1 file changed, 452 insertions(+) create mode 100644 orangecontrib/educational/widgets/utils/tests/test_contours.py diff --git a/orangecontrib/educational/widgets/utils/tests/test_contours.py b/orangecontrib/educational/widgets/utils/tests/test_contours.py new file mode 100644 index 00000000..bd57074f --- /dev/null +++ b/orangecontrib/educational/widgets/utils/tests/test_contours.py @@ -0,0 +1,452 @@ +import unittest + +import numpy as np +from numpy.testing import assert_array_equal + +from orangecontrib.educational.widgets.utils.contour import Contour + + +class TestContours(unittest.TestCase): + + def setUp(self): + x = np.linspace(0, 10, 11) + y = np.linspace(0, 10, 11) + self.xv, self.yv = np.meshgrid(x, y) + self.z_vertical_asc = self.yv + self.z_vertical_desc = np.max(self.yv) - self.yv + self.z_horizontal_asc = self.xv + self.z_horizontal_desc = np.max(self.xv) - self.xv + + # lt = left top, rt = right top, lb = left bottom, lt = left top + self.z_rt_lb_desc = self.xv + (np.max(self.yv) - self.yv) + self.z_rt_lb_asc = (np.max(self.xv) - self.xv) + self.yv + self.z_lt_rb_asc = self.xv + self.yv + self.z_lt_rb_desc = (np.max(self.xv) - self.xv) + \ + (np.max(self.yv) - self.yv) + + # test for testing cycles and 5s and 10s + self.cycle1 = np.array([[0, 0, 0, 0, 0], + [0, 1, 0, 0, 0], + [0, 0, 1, 0, 0], + [0, 1, 0, 0, 0], + [0, 0, 0, 0, 0]]) + self.cycle2 = np.array([[0, 0, 0, 0, 0], + [0, 0, 1, 0, 0], + [0, 1, 0, 0, 0], + [0, 0, 1, 0, 0], + [0, 0, 0, 0, 0]]) + x = np.linspace(0, 4, 5) + y = np.linspace(0, 4, 5) + self.xv_cycle, self.yv_cycle = np.meshgrid(x, y) + + def test_contours(self): + """ + Test if right amount of values + """ + c = Contour(self.xv, self.yv, self.z_vertical_asc) + c_lines = c.contours([1, 2, 3]) + + # all line exists in particular data + self.assertIn(1, c_lines.keys()) + self.assertIn(2, c_lines.keys()) + self.assertIn(3, c_lines.keys()) + + # in particular data none line are in more peaces + self.assertEqual(len(c_lines[1]), 1) + self.assertEqual(len(c_lines[2]), 1) + self.assertEqual(len(c_lines[3]), 1) + + c = Contour(self.xv, self.yv, self.z_vertical_desc) + c_lines = c.contours([1, 2, 3]) + + # all line exists in particular data + self.assertIn(1, c_lines.keys()) + self.assertIn(2, c_lines.keys()) + self.assertIn(3, c_lines.keys()) + + # in particular data none line are in more peaces + self.assertEqual(len(c_lines[1]), 1) + self.assertEqual(len(c_lines[2]), 1) + self.assertEqual(len(c_lines[3]), 1) + + c = Contour(self.xv, self.yv, self.z_horizontal_asc) + c_lines = c.contours([1, 2, 3]) + + # all line exists in particular data + self.assertIn(1, c_lines.keys()) + self.assertIn(2, c_lines.keys()) + self.assertIn(3, c_lines.keys()) + + # in particular data none line are in more peaces + self.assertEqual(len(c_lines[1]), 1) + self.assertEqual(len(c_lines[2]), 1) + self.assertEqual(len(c_lines[3]), 1) + + c = Contour(self.xv, self.yv, self.z_horizontal_desc) + c_lines = c.contours([1, 2, 3]) + + # all line exists in particular data + self.assertIn(1, c_lines.keys()) + self.assertIn(2, c_lines.keys()) + self.assertIn(3, c_lines.keys()) + + # in particular data none line are in more peaces + self.assertEqual(len(c_lines[1]), 1) + self.assertEqual(len(c_lines[2]), 1) + self.assertEqual(len(c_lines[3]), 1) + + c = Contour(self.xv, self.yv, self.z_lt_rb_asc) + c_lines = c.contours([1, 2, 3]) + + # all line exists in particular data + self.assertIn(1, c_lines.keys()) + self.assertIn(2, c_lines.keys()) + self.assertIn(3, c_lines.keys()) + + # in particular data none line are in more peaces + self.assertEqual(len(c_lines[1]), 1) + self.assertEqual(len(c_lines[2]), 1) + self.assertEqual(len(c_lines[3]), 1) + + c = Contour(self.xv, self.yv, self.z_lt_rb_desc) + c_lines = c.contours([1, 2, 3]) + + # all line exists in particular data + self.assertIn(1, c_lines.keys()) + self.assertIn(2, c_lines.keys()) + self.assertIn(3, c_lines.keys()) + + # in particular data none line are in more peaces + self.assertEqual(len(c_lines[1]), 1) + self.assertEqual(len(c_lines[2]), 1) + self.assertEqual(len(c_lines[3]), 1) + + c = Contour(self.xv, self.yv, self.z_rt_lb_asc) + c_lines = c.contours([1, 2, 3]) + + # all line exists in particular data + self.assertIn(1, c_lines.keys()) + self.assertIn(2, c_lines.keys()) + self.assertIn(3, c_lines.keys()) + + # in particular data none line are in more peaces + self.assertEqual(len(c_lines[1]), 1) + self.assertEqual(len(c_lines[2]), 1) + self.assertEqual(len(c_lines[3]), 1) + + c = Contour(self.xv, self.yv, self.z_rt_lb_desc) + c_lines = c.contours([1, 2, 3]) + + # all line exists in particular data + self.assertIn(1, c_lines.keys()) + self.assertIn(2, c_lines.keys()) + self.assertIn(3, c_lines.keys()) + + # in particular data none line are in more peaces + self.assertEqual(len(c_lines[1]), 1) + self.assertEqual(len(c_lines[2]), 1) + self.assertEqual(len(c_lines[3]), 1) + + # test in cycle set + c = Contour(self.xv_cycle, self.yv_cycle, self.cycle1) + c_lines = c.contours([0.5]) + + self.assertIn(0.5, c_lines.keys()) + self.assertEqual(len(c_lines[0.5]), 1) + + # test start with square 5, before only 10 was checked + c = Contour(self.xv_cycle, self.yv_cycle, self.cycle2) + c_lines = c.contours([0.5]) + + self.assertIn(0.5, c_lines.keys()) + self.assertEqual(len(c_lines[0.5]), 1) + + # test no contours, then no key in dict + c = Contour(self.xv_cycle, self.yv_cycle, self.cycle2) + c_lines = c.contours([1.5]) + + self.assertNotIn(1.5, c_lines.keys()) + + def test_find_contours(self): + """ + Test if right contours found for threshold + """ + # check all horizontal edges + c = Contour(self.xv, self.yv, self.z_horizontal_asc) + + points = c.find_contours(1) + self.assertEqual(len(points), 1) # only one line in particular example + for i in range(11): + self.assertIn([1, i], points[0]) + + points = c.find_contours(5) + self.assertEqual(len(points), 1) # only one line in particular example + for i in range(11): + self.assertIn([5, i], points[0]) + + c = Contour(self.xv, self.yv, self.z_horizontal_desc) + + points = c.find_contours(1) + self.assertEqual(len(points), 1) # only one line in particular example + for i in range(11): + self.assertIn([9, i], points[0]) + + points = c.find_contours(5) + self.assertEqual(len(points), 1) # only one line in particular example + for i in range(11): + self.assertIn([5, i], points[0]) + + # check all vertical edges + c = Contour(self.xv, self.yv, self.z_vertical_asc) + + points = c.find_contours(1) + self.assertEqual(len(points), 1) # only one line in particular example + for i in range(11): + self.assertIn([i, 1], points[0]) + + points = c.find_contours(5) + self.assertEqual(len(points), 1) # only one line in particular example + for i in range(11): + self.assertIn([i, 5], points[0]) + + c = Contour(self.xv, self.yv, self.z_vertical_desc) + + points = c.find_contours(1) + self.assertEqual(len(points), 1) # only one line in particular example + for i in range(11): + self.assertIn([i, 9], points[0]) + + points = c.find_contours(5) + self.assertEqual(len(points), 1) # only one line in particular example + for i in range(11): + self.assertIn([i, 5], points[0]) + + # check all top-left bottom-right edges + c = Contour(self.xv, self.yv, self.z_lt_rb_asc) + + points = c.find_contours(1) + self.assertEqual(len(points), 1) # only one line in particular example + self.assertIn([0, 1], points[0]) + self.assertIn([1, 0], points[0]) + + points = c.find_contours(10) + self.assertEqual(len(points), 1) # only one line in particular example + for i in range(11): + self.assertIn([i, 10-i], points[0]) + + c = Contour(self.xv, self.yv, self.z_lt_rb_desc) + + points = c.find_contours(1) + self.assertEqual(len(points), 1) # only one line in particular example + self.assertIn([10, 9], points[0]) + self.assertIn([9, 10], points[0]) + + points = c.find_contours(10) + self.assertEqual(len(points), 1) # only one line in particular example + for i in range(11): + self.assertIn([i, 10-i], points[0]) + + # check all top-right bottom-left edges + c = Contour(self.xv, self.yv, self.z_rt_lb_asc) + + points = c.find_contours(1) + self.assertEqual(len(points), 1) # only one line in particular example + self.assertIn([9, 0], points[0]) + self.assertIn([10, 1], points[0]) + + points = c.find_contours(10) + self.assertEqual(len(points), 1) # only one line in particular example + for i in range(11): + self.assertIn([10-i, 10-i], points[0]) + + c = Contour(self.xv, self.yv, self.z_rt_lb_desc) + + points = c.find_contours(1) + self.assertEqual(len(points), 1) # only one line in particular example + self.assertIn([0, 9], points[0]) + self.assertIn([1, 10], points[0]) + + points = c.find_contours(10) + self.assertEqual(len(points), 1) # only one line in particular example + for i in range(11): + self.assertIn([10-i, 10-i], points[0]) + + c = Contour(self.xv_cycle, self.yv_cycle, self.cycle1) + + points = c.find_contours(0.5) + self.assertEqual(len(points[0]), 13) + self.assertIn([1, 0.5], points[0]) + self.assertIn([1.5, 1], points[0]) + self.assertIn([2, 1.5], points[0]) + self.assertIn([2.5, 2], points[0]) + self.assertIn([2, 2.5], points[0]) + self.assertIn([1.5, 3], points[0]) + self.assertIn([1, 3.5], points[0]) + self.assertIn([0.5, 3], points[0]) + self.assertIn([1, 2.5], points[0]) + self.assertIn([1.5, 2], points[0]) + self.assertIn([1, 1.5], points[0]) + self.assertIn([0.5, 1], points[0]) + + c = Contour(self.xv_cycle, self.yv_cycle, self.cycle2) + + points = c.find_contours(0.5) + self.assertEqual(len(points[0]), 13) + self.assertIn([2, 0.5], points[0]) + self.assertIn([2.5, 1], points[0]) + self.assertIn([2, 1.5], points[0]) + self.assertIn([1.5, 2], points[0]) + self.assertIn([2, 2.5], points[0]) + self.assertIn([2.5, 3], points[0]) + self.assertIn([2, 3.5], points[0]) + self.assertIn([1.5, 3], points[0]) + self.assertIn([1, 2.5], points[0]) + self.assertIn([0.5, 2], points[0]) + self.assertIn([1, 1.5], points[0]) + self.assertIn([1.5, 1], points[0]) + + def test_to_real_coordinate(self): + c = Contour(self.xv, self.yv, self.z_horizontal_asc) + + # integers same because of grid with integers + self.assertEqual(c.to_real_coordinate([1, 1]), [1, 1]) + + # coordinate have to have x on first place (before row first) + self.assertEqual(c.to_real_coordinate([1, 2]), [2, 1]) + + # middle values + self.assertEqual(c.to_real_coordinate([1, 1.5]), [1.5, 1]) + self.assertEqual(c.to_real_coordinate([1.5, 1.5]), [1.5, 1.5]) + self.assertEqual(c.to_real_coordinate([1.5, 1]), [1, 1.5]) + self.assertEqual(c.to_real_coordinate([5, 5.5]), [5.5, 5]) + self.assertEqual(c.to_real_coordinate([5.5, 5.5]), [5.5, 5.5]) + self.assertEqual(c.to_real_coordinate([5.5, 5]), [5, 5.5]) + + # meshgrid no integers + xv, yv = np.meshgrid(np.linspace(0, 5, 11), np.linspace(0, 5, 11)) + c = Contour(xv, yv, self.z_horizontal_asc) + + self.assertEqual(c.to_real_coordinate([1, 1]), [0.5, 0.5]) + self.assertEqual(c.to_real_coordinate([1, 1.5]), [0.75, 0.5]) + self.assertEqual(c.to_real_coordinate([1.5, 1.5]), [0.75, 0.75]) + self.assertEqual(c.to_real_coordinate([1.5, 1]), [0.5, 0.75]) + self.assertEqual(c.to_real_coordinate([5, 5.5]), [2.75, 2.5]) + self.assertEqual(c.to_real_coordinate([5.5, 5.5]), [2.75, 2.75]) + self.assertEqual(c.to_real_coordinate([5.5, 5]), [2.5, 2.75]) + + def test_triangulate(self): + self.assertEqual(Contour.triangulate(0, 0, 1), 0) + self.assertEqual(Contour.triangulate(1, 0, 1), 1) + self.assertEqual(Contour.triangulate(0.5, 0, 1), 0.5) + self.assertEqual(Contour.triangulate(0.3, 0, 1), 0.3) + + self.assertEqual(Contour.triangulate(0, 1, 0), 1) + self.assertEqual(Contour.triangulate(1, 1, 0), 0) + self.assertEqual(Contour.triangulate(0.5, 1, 0), 0.5) + self.assertEqual(Contour.triangulate(0.3, 1, 0), 0.7) + + def test_new_position(self): + # when sq not equal 5 or 10 previous position does not matter + assert_array_equal(Contour.new_position( + np.array([[0, 0], [1, 0]]), None, np.array([1, 1])), [2, 1]) + assert_array_equal(Contour.new_position( + np.array([[0, 0], [0, 1]]), None, np.array([1, 1])), [1, 2]) + assert_array_equal(Contour.new_position( + np.array([[0, 0], [1, 1]]), None, np.array([1, 1])), [1, 2]) + assert_array_equal(Contour.new_position( + np.array([[0, 1], [0, 0]]), None, np.array([1, 1])), [0, 1]) + assert_array_equal(Contour.new_position( + np.array([[0, 1], [0, 1]]), None, np.array([1, 1])), [0, 1]) + assert_array_equal(Contour.new_position( + np.array([[0, 1], [1, 1]]), None, np.array([1, 1])), [0, 1]) + assert_array_equal(Contour.new_position( + np.array([[1, 0], [0, 0]]), None, np.array([1, 1])), [1, 0]) + assert_array_equal(Contour.new_position( + np.array([[1, 0], [1, 0]]), None, np.array([1, 1])), [2, 1]) + assert_array_equal(Contour.new_position( + np.array([[1, 0], [1, 1]]), None, np.array([1, 1])), [1, 2]) + assert_array_equal(Contour.new_position( + np.array([[1, 1], [0, 0]]), None, np.array([1, 1])), [1, 0]) + assert_array_equal(Contour.new_position( + np.array([[1, 1], [1, 0]]), None, np.array([1, 1])), [2, 1]) + assert_array_equal(Contour.new_position( + np.array([[1, 1], [0, 1]]), None, np.array([1, 1])), [1, 0]) + + # sq = 5 + # start on edge + assert_array_equal(Contour.new_position( + np.array([[0, 1], [1, 0]]), None, np.array([1, 1])), [0, 1]) + # previous from left + assert_array_equal(Contour.new_position( + np.array([[0, 1], [1, 0]]), np.array([1, 0]), + np.array([1, 1])), [0, 1]) + # previous from right + assert_array_equal(Contour.new_position( + np.array([[0, 1], [1, 0]]), np.array([1, 2]), + np.array([1, 1])), [2, 1]) + + # sq = 10 + # start on edge + assert_array_equal(Contour.new_position( + np.array([[1, 0], [0, 1]]), None, np.array([1, 1])), [1, 2]) + # previous from top + assert_array_equal(Contour.new_position( + np.array([[1, 0], [0, 1]]), np.array([0, 1]), + np.array([1, 1])), [1, 2]) + # previous from bottom + assert_array_equal(Contour.new_position( + np.array([[1, 0], [0, 1]]), np.array([2, 1]), + np.array([1, 1])), [1, 0]) + + def test_corner_idx(self): + self.assertEqual(Contour.corner_idx([[0, 0], [0, 0]]), 0) + self.assertEqual(Contour.corner_idx([[0, 0], [1, 0]]), 1) + self.assertEqual(Contour.corner_idx([[0, 0], [0, 1]]), 2) + self.assertEqual(Contour.corner_idx([[0, 0], [1, 1]]), 3) + self.assertEqual(Contour.corner_idx([[0, 1], [0, 0]]), 4) + self.assertEqual(Contour.corner_idx([[0, 1], [1, 0]]), 5) + self.assertEqual(Contour.corner_idx([[0, 1], [0, 1]]), 6) + self.assertEqual(Contour.corner_idx([[0, 1], [1, 1]]), 7) + self.assertEqual(Contour.corner_idx([[1, 0], [0, 0]]), 8) + self.assertEqual(Contour.corner_idx([[1, 0], [1, 0]]), 9) + self.assertEqual(Contour.corner_idx([[1, 0], [0, 1]]), 10) + self.assertEqual(Contour.corner_idx([[1, 0], [1, 1]]), 11) + self.assertEqual(Contour.corner_idx([[1, 1], [0, 0]]), 12) + self.assertEqual(Contour.corner_idx([[1, 1], [1, 0]]), 13) + self.assertEqual(Contour.corner_idx([[1, 1], [0, 1]]), 14) + self.assertEqual(Contour.corner_idx([[1, 1], [1, 1]]), 15) + + def test_visited(self): + c = Contour(self.xv, self.yv, self.z_rt_lb_desc) + c.visited_points = np.zeros(self.xv.shape) + + self.assertFalse(c.visited(0, 0, True)) + self.assertFalse(c.visited(0, 0, False)) + + # check if upper + c.mark_visited(0, 0, True) + self.assertTrue(c.visited(0, 0, True)) + self.assertFalse(c.visited(0, 0, False)) + + # check if lower + c.mark_visited(1, 1, False) + self.assertFalse(c.visited(1, 1, True)) + self.assertTrue(c.visited(1, 1, False)) + + # check if ok when mark again + c.mark_visited(1, 1, False) + self.assertFalse(c.visited(1, 1, True)) + self.assertTrue(c.visited(1, 1, False)) + + c.mark_visited(0, 0, True) + self.assertTrue(c.visited(0, 0, True)) + self.assertFalse(c.visited(0, 0, False)) + + # check if booth lower fist, and upper first + c.mark_visited(1, 1, True) + self.assertTrue(c.visited(1, 1, True)) + self.assertTrue(c.visited(1, 1, False)) + + c.mark_visited(0, 0, False) + self.assertTrue(c.visited(0, 0, True)) + self.assertTrue(c.visited(0, 0, False)) From 9717692bf903f97a7983b213d785b62992a21a7f Mon Sep 17 00:00:00 2001 From: PrimozGodec Date: Mon, 8 Aug 2016 13:23:50 +0200 Subject: [PATCH 025/128] Updated unit test to reach higher coverage --- orangecontrib/educational/widgets/owgradientdescent.py | 9 --------- .../educational/widgets/tests/test_owgradientdescent.py | 4 ++-- 2 files changed, 2 insertions(+), 11 deletions(-) diff --git a/orangecontrib/educational/widgets/owgradientdescent.py b/orangecontrib/educational/widgets/owgradientdescent.py index de0275c1..6710a68f 100644 --- a/orangecontrib/educational/widgets/owgradientdescent.py +++ b/orangecontrib/educational/widgets/owgradientdescent.py @@ -487,15 +487,6 @@ def plot_gradient_and_contour(self, x_from, x_to, y_from, y_to): return self.plot_contour(xv, yv, self.cost_grid) - def plot_gradient(self, x, y, grid): - """ - Function constructs background gradient - """ - return [dict(data=[[x[j, k], y[j, k], grid[j, k]] for j in range(len(x)) - for k in range(y.shape[1])], - grid_width=self.grid_size, - type="contour")] - def plot_contour(self, xv, yv, cost_grid): """ Function constructs contour lines diff --git a/orangecontrib/educational/widgets/tests/test_owgradientdescent.py b/orangecontrib/educational/widgets/tests/test_owgradientdescent.py index 23863c31..cff31e9d 100644 --- a/orangecontrib/educational/widgets/tests/test_owgradientdescent.py +++ b/orangecontrib/educational/widgets/tests/test_owgradientdescent.py @@ -196,7 +196,7 @@ def test_change_stochastic(self): self.assertIsNone(w.learner) w.stochastic_checkbox.click() - def change_step(self): + def test_change_step(self): """ Function check if change step works correctly """ @@ -234,7 +234,7 @@ def test_change_theta(self): # change alpha w.change_theta(1, 1) assert_array_equal(w.learner.theta, [1, 1]) - w.change_theta(1, 2) + w.scatter.chart_clicked(1, 2) assert_array_equal(w.learner.theta, [1, 2]) # just check if nothing happens when no learner From 0becef7c94c6d11df495fc35dfd2a9b0933f9da3 Mon Sep 17 00:00:00 2001 From: PrimozGodec Date: Mon, 8 Aug 2016 13:33:37 +0200 Subject: [PATCH 026/128] Updated unit test to reach higher coverage --- .../educational/widgets/tests/test_owgradientdescent.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/orangecontrib/educational/widgets/tests/test_owgradientdescent.py b/orangecontrib/educational/widgets/tests/test_owgradientdescent.py index cff31e9d..bf393400 100644 --- a/orangecontrib/educational/widgets/tests/test_owgradientdescent.py +++ b/orangecontrib/educational/widgets/tests/test_owgradientdescent.py @@ -217,7 +217,7 @@ def test_change_step(self): # just check if nothing happens when no learner self.send_signal("Data", None) self.assertIsNone(w.learner) - w.step_size_spin.setValue(40) + w.step_size_spin.setValue(30) def test_change_theta(self): """ From 815d9b3608a1b69ff8790deeab13bb912d0bc2c0 Mon Sep 17 00:00:00 2001 From: PrimozGodec Date: Tue, 9 Aug 2016 10:27:50 +0200 Subject: [PATCH 027/128] Added gradient for function values. --- .../educational/widgets/owgradientdescent.py | 35 ++++++++++++++++--- 1 file changed, 30 insertions(+), 5 deletions(-) diff --git a/orangecontrib/educational/widgets/owgradientdescent.py b/orangecontrib/educational/widgets/owgradientdescent.py index 6710a68f..db72f889 100644 --- a/orangecontrib/educational/widgets/owgradientdescent.py +++ b/orangecontrib/educational/widgets/owgradientdescent.py @@ -257,13 +257,16 @@ def __init__(self): yAxis_gridLineWidth=0, title_text='', tooltip_shared=False, - debug=True) + debug=True, + legend_symbolWidth=0, + legend_symbolHeight=0) # TODO: set false when end of development gui.rubber(self.controlArea) # Just render an empty chart so it shows a nice 'No data to display' self.scatter.chart() self.mainArea.layout().addWidget(self.scatter) + # to remove the legend def set_data(self, data): """ @@ -378,8 +381,10 @@ def change_theta(self, x, y): self.scatter.remove_series("path") self.scatter.add_series([ dict(id="path", data=[[x, y]], showInLegend=False, - type="scatter", lineWidth=1, - marker=dict(enabled=True, radius=2))],) + type="scatter", lineWidth=1, enableMouseTracking=False, + color="#ff0000", + marker=dict( + enabled=True, radius=2))],) self.send_output() def step(self): @@ -432,6 +437,9 @@ def replot(self): options['series'] += self.plot_gradient_and_contour( self.min_x, self.max_x, self.min_y, self.max_y) + min_value = np.min(self.cost_grid) + max_value = np.max(self.cost_grid) + # highcharts parameters kwargs = dict( xAxis_title_text="theta 0", @@ -444,13 +452,20 @@ def replot(self): xAxis_endOnTick=False, yAxis_startOnTick=False, yAxis_endOnTick=False, - tooltip_enabled=False, + # tooltip_enabled=False, + colorAxis=dict( + minColor="#ffffff", maxColor="#00BFFF", + endOnTick=False, startOnTick=False), + plotOptions_contour_colsize=(self.max_y - self.min_y) / 1000, + plotOptions_contour_rowsize=(self.max_x - self.min_x) / 1000, tooltip_headerFormat="", tooltip_pointFormat="%s: {point.x:.2f}
" "%s: {point.y:.2f}" % (self.attr_x, self.attr_y)) self.scatter.chart(options, **kwargs) + # to remove the colorAxis legend + self.scatter.evalJS("chart.colorAxis[0].axisParent.destroy();") def plot_gradient_and_contour(self, x_from, x_to, y_from, y_to): """ @@ -485,7 +500,17 @@ def plot_gradient_and_contour(self, x_from, x_to, y_from, y_to): # results self.cost_grid = cost_values.reshape(xv.shape) - return self.plot_contour(xv, yv, self.cost_grid) + return self.plot_gradient(xv, yv, self.cost_grid) + \ + self.plot_contour(xv, yv, self.cost_grid) + + def plot_gradient(self, x, y, grid): + """ + Function constructs background gradient + """ + return [dict(data=[[x[j, k], y[j, k], grid[j, k]] for j in range(len(x)) + for k in range(y.shape[1])], + grid_width=self.grid_size, + type="contour")] def plot_contour(self, xv, yv, cost_grid): """ From 01a6acf2c350a4d2d53d85f1a0245b4139a1966b Mon Sep 17 00:00:00 2001 From: PrimozGodec Date: Thu, 11 Aug 2016 10:38:27 +0200 Subject: [PATCH 028/128] Modified control areas. Additional separators and removed spaced between label and spin. --- orangecontrib/educational/widgets/owgradientdescent.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/orangecontrib/educational/widgets/owgradientdescent.py b/orangecontrib/educational/widgets/owgradientdescent.py index db72f889..5c173564 100644 --- a/orangecontrib/educational/widgets/owgradientdescent.py +++ b/orangecontrib/educational/widgets/owgradientdescent.py @@ -2,7 +2,6 @@ import time import numpy as np -from scipy.ndimage import gaussian_filter from PyQt4.QtCore import pyqtSlot, Qt, QThread, SIGNAL from PyQt4.QtGui import QSizePolicy, QPixmap, QColor, QIcon @@ -215,6 +214,8 @@ def __init__(self): self.cbx.setModel(self.x_var_model) self.cby.setModel(self.y_var_model) + gui.separator(self.controlArea, 20, 20) + # properties box self.properties_box = gui.widgetBox(self.controlArea, "Properties") self.alpha_spin = gui.spin( @@ -233,6 +234,11 @@ def __init__(self): widget=self.properties_box, master=self, callback=self.restart, label="Restart") + self.alpha_spin.setSizePolicy(policy) + self.step_size_spin.setSizePolicy(policy) + + gui.separator(self.controlArea, 20, 20) + # step box self.step_box = gui.widgetBox(self.controlArea, "Manually step through") self.step_button = gui.button( @@ -241,6 +247,8 @@ def __init__(self): widget=self.step_box, master=self, callback=self.step_back, label="Step back") + gui.separator(self.controlArea, 20, 20) + # run box self.run_box = gui.widgetBox(self.controlArea, "Run") self.auto_play_button = gui.button( From f2aefd5ba3d8e6083c2ad6c38de0c5b579b39a02 Mon Sep 17 00:00:00 2001 From: PrimozGodec Date: Thu, 11 Aug 2016 11:01:48 +0200 Subject: [PATCH 029/128] Added special mark for the last point --- orangecontrib/educational/widgets/owgradientdescent.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/orangecontrib/educational/widgets/owgradientdescent.py b/orangecontrib/educational/widgets/owgradientdescent.py index 5c173564..1eaaf0b6 100644 --- a/orangecontrib/educational/widgets/owgradientdescent.py +++ b/orangecontrib/educational/widgets/owgradientdescent.py @@ -387,12 +387,16 @@ def change_theta(self, x, y): if self.learner is not None: self.learner.set_theta([x, y]) self.scatter.remove_series("path") + self.scatter.remove_series("last_point") self.scatter.add_series([ + dict(id="last_point", data=[[x, y]], showInLegend=False, + type="scatter", enableMouseTracking=False, + color="#ffcc00", marker=dict(radius=4)), dict(id="path", data=[[x, y]], showInLegend=False, type="scatter", lineWidth=1, enableMouseTracking=False, color="#ff0000", marker=dict( - enabled=True, radius=2))],) + enabled=True, radius=2))]) self.send_output() def step(self): @@ -425,6 +429,8 @@ def plot_point(self, x, y): Function add point to the path """ self.scatter.add_point_to_series("path", x, y) + self.scatter.remove_last_point("last_point") + self.scatter.add_point_to_series("last_point", x, y) def replot(self): """ From b6da19f0b5b5a588b3185cc1f656ac7efbc9b74f Mon Sep 17 00:00:00 2001 From: PrimozGodec Date: Thu, 11 Aug 2016 11:09:39 +0200 Subject: [PATCH 030/128] SIGNAL changed with pyqtSignal --- .../educational/widgets/owgradientdescent.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/orangecontrib/educational/widgets/owgradientdescent.py b/orangecontrib/educational/widgets/owgradientdescent.py index 1eaaf0b6..c65df7cb 100644 --- a/orangecontrib/educational/widgets/owgradientdescent.py +++ b/orangecontrib/educational/widgets/owgradientdescent.py @@ -2,7 +2,7 @@ import time import numpy as np -from PyQt4.QtCore import pyqtSlot, Qt, QThread, SIGNAL +from PyQt4.QtCore import pyqtSlot, Qt, QThread, pyqtSignal from PyQt4.QtGui import QSizePolicy, QPixmap, QColor, QIcon from Orange.widgets.utils import itemmodels @@ -128,9 +128,9 @@ def run(self): """ while (not self.ow_gradient_descent.learner.converged and self.ow_gradient_descent.auto_play_enabled): - self.emit(SIGNAL('step()')) + self.ow_gradient_descent.step_trigger.emit() time.sleep(2 - self.ow_gradient_descent.auto_play_speed) - self.emit(SIGNAL('stop_auto_play()')) + self.ow_gradient_descent.stop_auto_play_trigger.emit() class OWGradientDescent(OWWidget): @@ -181,6 +181,10 @@ class OWGradientDescent(OWWidget): auto_play_button_text = ["Run", "Stop"] auto_play_thread = None + # signals + step_trigger = pyqtSignal() + stop_auto_play_trigger = pyqtSignal() + class Warning(OWWidget.Warning): """ Class used fro widget warnings. @@ -595,10 +599,8 @@ def auto_play(self): if self.auto_play_enabled: self.disable_controls(self.auto_play_enabled) self.auto_play_thread = Autoplay(self) - self.connect(self.auto_play_thread, SIGNAL("step()"), self.step) - self.connect( - self.auto_play_thread, SIGNAL("stop_auto_play()"), - self.stop_auto_play) + self.step_trigger.connect(self.step) + self.stop_auto_play_trigger.connect(self.stop_auto_play) self.auto_play_thread.start() else: self.stop_auto_play() From ce086b55a01d08acc7637f14a55a614445b8cff3 Mon Sep 17 00:00:00 2001 From: PrimozGodec Date: Thu, 11 Aug 2016 11:32:44 +0200 Subject: [PATCH 031/128] Modified icon for gradient descent. --- .../widgets/icons/GradientDescent.svg | 87 ++++++++----------- 1 file changed, 37 insertions(+), 50 deletions(-) diff --git a/orangecontrib/educational/widgets/icons/GradientDescent.svg b/orangecontrib/educational/widgets/icons/GradientDescent.svg index 83948785..cc773e74 100644 --- a/orangecontrib/educational/widgets/icons/GradientDescent.svg +++ b/orangecontrib/educational/widgets/icons/GradientDescent.svg @@ -26,8 +26,8 @@ inkscape:pageopacity="0.0" inkscape:pageshadow="2" inkscape:zoom="15.839192" - inkscape:cx="29.162429" - inkscape:cy="26.524336" + inkscape:cx="18.492693" + inkscape:cy="31.575099" inkscape:document-units="px" inkscape:current-layer="layer1" showgrid="false" @@ -45,7 +45,7 @@ image/svg+xml - + @@ -55,80 +55,67 @@ id="layer1" transform="translate(0,-1004.3622)"> - + cx="24" + cy="1028.3622" + rx="13.710499" + ry="12.214576" /> - + cx="24" + cy="1028.3622" + rx="7.7435975" + ry="6.3237586" /> From 6a9f577fdd8bfc5c79d3ea0147f00cfd2892ca6f Mon Sep 17 00:00:00 2001 From: PrimozGodec Date: Thu, 11 Aug 2016 11:50:36 +0200 Subject: [PATCH 032/128] Code clean --- .../educational/widgets/owgradientdescent.py | 17 +++++------------ 1 file changed, 5 insertions(+), 12 deletions(-) diff --git a/orangecontrib/educational/widgets/owgradientdescent.py b/orangecontrib/educational/widgets/owgradientdescent.py index c65df7cb..90772162 100644 --- a/orangecontrib/educational/widgets/owgradientdescent.py +++ b/orangecontrib/educational/widgets/owgradientdescent.py @@ -278,7 +278,6 @@ def __init__(self): # Just render an empty chart so it shows a nice 'No data to display' self.scatter.chart() self.mainArea.layout().addWidget(self.scatter) - # to remove the legend def set_data(self, data): """ @@ -368,7 +367,7 @@ def restart(self): def change_alpha(self): """ - Function changes alpha parameter of the alogrithm + Function changes alpha parameter of the algorithm """ if self.learner is not None: self.learner.set_alpha(self.alpha) @@ -455,9 +454,6 @@ def replot(self): options['series'] += self.plot_gradient_and_contour( self.min_x, self.max_x, self.min_y, self.max_y) - min_value = np.min(self.cost_grid) - max_value = np.max(self.cost_grid) - # highcharts parameters kwargs = dict( xAxis_title_text="theta 0", @@ -470,7 +466,6 @@ def replot(self): xAxis_endOnTick=False, yAxis_startOnTick=False, yAxis_endOnTick=False, - # tooltip_enabled=False, colorAxis=dict( minColor="#ffffff", maxColor="#00BFFF", endOnTick=False, startOnTick=False), @@ -518,8 +513,8 @@ def plot_gradient_and_contour(self, x_from, x_to, y_from, y_to): # results self.cost_grid = cost_values.reshape(xv.shape) - return self.plot_gradient(xv, yv, self.cost_grid) + \ - self.plot_contour(xv, yv, self.cost_grid) + return (self.plot_gradient(xv, yv, self.cost_grid) + + self.plot_contour(xv, yv, self.cost_grid)) def plot_gradient(self, x, y, grid): """ @@ -534,9 +529,7 @@ def plot_contour(self, xv, yv, cost_grid): """ Function constructs contour lines """ - - contour = Contour( - xv, yv, cost_grid) + contour = Contour(xv, yv, cost_grid) contour_lines = contour.contours( np.linspace(np.min(cost_grid), np.max(cost_grid), 20)) @@ -662,4 +655,4 @@ def send_data(self): if self.selected_data is not None: self.send("Data", self.selected_data) else: - self.send("Data", None) \ No newline at end of file + self.send("Data", None) From 51972c3a2f0fc661da7b0a3cb16fb7ce15a6b521 Mon Sep 17 00:00:00 2001 From: PrimozGodec Date: Thu, 11 Aug 2016 14:23:14 +0200 Subject: [PATCH 033/128] Max number of steps limited to 500 --- orangecontrib/educational/widgets/owgradientdescent.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/orangecontrib/educational/widgets/owgradientdescent.py b/orangecontrib/educational/widgets/owgradientdescent.py index 90772162..ca532088 100644 --- a/orangecontrib/educational/widgets/owgradientdescent.py +++ b/orangecontrib/educational/widgets/owgradientdescent.py @@ -127,7 +127,8 @@ def run(self): Stepping through the algorithm until converge or user interrupts """ while (not self.ow_gradient_descent.learner.converged and - self.ow_gradient_descent.auto_play_enabled): + self.ow_gradient_descent.auto_play_enabled and + self.ow_gradient_descent.learner.step_no <= 500): self.ow_gradient_descent.step_trigger.emit() time.sleep(2 - self.ow_gradient_descent.auto_play_speed) self.ow_gradient_descent.stop_auto_play_trigger.emit() @@ -408,6 +409,8 @@ def step(self): """ if self.data is None: return + if self.learner.step_no > 500: # limit step no to avoid freezes + return if self.learner.theta is None: self.change_theta(np.random.uniform(self.min_x, self.max_x), np.random.uniform(self.min_y, self.max_y)) From 75478504b165e90faa10b80edda578024793bce6 Mon Sep 17 00:00:00 2001 From: PrimozGodec Date: Thu, 11 Aug 2016 14:34:20 +0200 Subject: [PATCH 034/128] No others when only two classes. --- orangecontrib/educational/widgets/owgradientdescent.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/orangecontrib/educational/widgets/owgradientdescent.py b/orangecontrib/educational/widgets/owgradientdescent.py index ca532088..5a71c05d 100644 --- a/orangecontrib/educational/widgets/owgradientdescent.py +++ b/orangecontrib/educational/widgets/owgradientdescent.py @@ -574,9 +574,12 @@ def select_data(self): subset = self.data[:, attr] cols.append(subset.X) x = np.column_stack(cols) + if len(self.data.domain.class_var.values) == 2: + return self.data + domain = Domain( [attr_x, attr_y], - [DiscreteVariable(name=self.data.domain.class_var.name, + [DiscreteVariable(name=self.data.domain.class_var.name + "-bin", values=[self.target_class, 'Others'])], [self.data.domain.class_var]) y = [(0 if d.get_class().value == self.target_class else 1) From c6400e922df7510166bd371595b027d0e66aee32 Mon Sep 17 00:00:00 2001 From: PrimozGodec Date: Thu, 11 Aug 2016 14:44:56 +0200 Subject: [PATCH 035/128] Keep theta on restart and set theta on setup. --- orangecontrib/educational/widgets/owgradientdescent.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/orangecontrib/educational/widgets/owgradientdescent.py b/orangecontrib/educational/widgets/owgradientdescent.py index 5a71c05d..9b0375c6 100644 --- a/orangecontrib/educational/widgets/owgradientdescent.py +++ b/orangecontrib/educational/widgets/owgradientdescent.py @@ -359,11 +359,18 @@ def restart(self): Function restarts the algorithm """ self.selected_data = self.select_data() + theta = self.learner.history[0][0] if self.learner is not None else None self.learner = self.default_learner( data=self.selected_data, alpha=self.alpha, stochastic=self.stochastic, + theta=theta, step_size=self.step_size) self.replot() + if theta is None: # no previous theta exist + self.change_theta(np.random.uniform(self.min_x, self.max_x), + np.random.uniform(self.min_y, self.max_y)) + else: # theta already exist + self.change_theta(theta[0], theta[1]) self.send_output() def change_alpha(self): From f7da2643df778fa986388cd06890d49e8f306026 Mon Sep 17 00:00:00 2001 From: PrimozGodec Date: Thu, 11 Aug 2016 14:57:49 +0200 Subject: [PATCH 036/128] Test for owgradientdescent fixed. --- .../widgets/tests/test_owgradientdescent.py | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/orangecontrib/educational/widgets/tests/test_owgradientdescent.py b/orangecontrib/educational/widgets/tests/test_owgradientdescent.py index bf393400..dc9b35bd 100644 --- a/orangecontrib/educational/widgets/tests/test_owgradientdescent.py +++ b/orangecontrib/educational/widgets/tests/test_owgradientdescent.py @@ -228,10 +228,10 @@ def test_change_theta(self): # to define learner self.send_signal("Data", self.iris) - # check init alpha - self.assertIsNone(w.learner.theta) + # check init theta + self.assertIsNotNone(w.learner.theta) - # change alpha + # change theta w.change_theta(1, 1) assert_array_equal(w.learner.theta, [1, 1]) w.scatter.chart_clicked(1, 2) @@ -253,8 +253,7 @@ def test_step(self): self.send_signal("Data", self.iris) - # test theta set when none - self.assertIsNone(w.learner.theta) + # test theta set after step if not set yet w.step() self.assertIsNotNone(w.learner.theta) @@ -471,9 +470,9 @@ def test_send_model(self): # when no learner self.assertIsNone(self.get_output("Classifier")) - # when learner but no theta + # when learner theta set automatically self.send_signal("Data", self.iris) - self.assertIsNone(self.get_output("Classifier")) + self.assertIsNotNone(self.get_output("Classifier")) # when everything fine w.change_theta(1., 1.) @@ -491,7 +490,7 @@ def test_send_coefficients(self): # when learner but no theta self.send_signal("Data", self.iris) - self.assertIsNone(self.get_output("Coefficients")) + self.assertIsNotNone(self.get_output("Coefficients")) # when everything fine w.change_theta(1., 1.) From 58ca46da19afab22bcc2ef9d94f7fd230464658e Mon Sep 17 00:00:00 2001 From: PrimozGodec Date: Thu, 11 Aug 2016 16:12:31 +0200 Subject: [PATCH 037/128] Updated description of the widget. --- orangecontrib/educational/widgets/owgradientdescent.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/orangecontrib/educational/widgets/owgradientdescent.py b/orangecontrib/educational/widgets/owgradientdescent.py index 9b0375c6..8aaa7afd 100644 --- a/orangecontrib/educational/widgets/owgradientdescent.py +++ b/orangecontrib/educational/widgets/owgradientdescent.py @@ -140,7 +140,8 @@ class OWGradientDescent(OWWidget): """ name = "Gradient Descent" - description = "Widget shows the procedure of gradient descent." + description = "Widget shows the procedure of gradient descent " \ + "on logistic regression." icon = "icons/GradientDescent.svg" want_main_area = True From e88834f70d109ab6e3448dbda9da669dcd7f68b9 Mon Sep 17 00:00:00 2001 From: PrimozGodec Date: Thu, 11 Aug 2016 17:16:10 +0200 Subject: [PATCH 038/128] Documentation for gradient descent. --- doc/index.rst | 1 + doc/widgets/gradientdescent.rst | 92 ++++++++++++++++++ doc/widgets/images/gradient-descent-flow.png | Bin 0 -> 43736 bytes doc/widgets/images/gradient-descent.png | Bin 0 -> 153862 bytes doc/widgets/images/gradient-descent1.png | Bin 0 -> 170624 bytes doc/widgets/images/gradient-descent2.png | Bin 0 -> 174226 bytes doc/widgets/images/gradient-descent3.png | Bin 0 -> 173190 bytes doc/widgets/images/gradient-descent4.png | Bin 0 -> 222606 bytes .../educational/widgets/owgradientdescent.py | 5 + 9 files changed, 98 insertions(+) create mode 100644 doc/widgets/gradientdescent.rst create mode 100644 doc/widgets/images/gradient-descent-flow.png create mode 100644 doc/widgets/images/gradient-descent.png create mode 100644 doc/widgets/images/gradient-descent1.png create mode 100644 doc/widgets/images/gradient-descent2.png create mode 100644 doc/widgets/images/gradient-descent3.png create mode 100644 doc/widgets/images/gradient-descent4.png diff --git a/doc/index.rst b/doc/index.rst index cefae851..1df4dfc2 100644 --- a/doc/index.rst +++ b/doc/index.rst @@ -13,6 +13,7 @@ Widgets widgets/kmeans widgets/univariatepolynomialregression + widgets/gradientdescent Indices and tables ================== diff --git a/doc/widgets/gradientdescent.rst b/doc/widgets/gradientdescent.rst new file mode 100644 index 00000000..6e469a4e --- /dev/null +++ b/doc/widgets/gradientdescent.rst @@ -0,0 +1,92 @@ +Gradient Descent +================ + +.. figure:: icons/gradient_descent.png + +Educational widget that show gradient descent algorithm on logistic regression. + +Signals +------- + +**Inputs**: + +- **Data** + +Input data set. + +**Outputs**: + +- **Data** + +Data with columns selected in widget. + +- **Classifier** + +Model produced on the current step of the algorithm. + +- **Coefficients** + +Logistic regression coefficient on the current step of the algorithm. + +Description +----------- + +This widget shows steps of `gradient descent `__ for logistic regression +step by step. Gradient descent is demonstrated on two attributes that are selected by user. + +.. figure:: images/gradient-descent.png + +1. Select two attributes (**x** and **y**) on which logistic regression algorithm is preformed. + Select **target class**. It is class that is classified against all other classes. + +2. **Learning rate** is step size in gradient descent + + With **stochastic** checkbox you can select whether gradient descent is + `stochastic `__ or not. + If stochastic is checked you can set **step size** that is amount of steps of stochastic gradient descent + performed in one step. + + **Restart**: start algorithm from beginning + +3. **Step**: perform one step of the algorithm + + **Step back**: make a step back in algorithm + +4. **Run**: perform several steps until algorithm converge automatically + + **Speed**: set speed of automatic stepping + +5. **Save Image** saves the image to the computer in a .svg or .png + format. + + **Report** includes widget parameters and visualization in the report. + +Example +------- + +In Orange we connected *File* widget with *Iris* data set to *Gradient Descent* widget. We connected outputs of +the widget to *Predictions* widget to see how data are classified and *Data Table* widget where we inspect coefficients +of logistic regression. + +.. figure:: images/gradient-descent-flow.png + +We opened *Gradient Descent* widget and set *X* to *sepal width* and *Y* to *sepal length*. Target class is set to +*Iris-virginica*. We set *learning rate* to 0.02. With click in graph we set beginning coefficients (red dot). + +.. figure:: images/gradient-descent1.png + +We performs step of the algorithm with pressing **Step** button. When we get bored with clicking we can finish steping +with press on **Run** button. + +.. figure:: images/gradient-descent2.png + +If we want to go back in the algorithm we can do it with pressing **Step back** button. This will also change model. +Current model uses positions of last coefficients (red-yellow dot). + +.. figure:: images/gradient-descent3.png + +In the end we want to see predictions for input data so we can open *Predictions* widget. Predictions are listed in +left column. We can compare this predictions to real classes. + +.. figure:: images/gradient-descent4.png + diff --git a/doc/widgets/images/gradient-descent-flow.png b/doc/widgets/images/gradient-descent-flow.png new file mode 100644 index 0000000000000000000000000000000000000000..59ec28bd069e23466aff1da707fe99072cd4877f GIT binary patch literal 43736 zcmcF~1y@yF+cnZicZa0(p&JCGOX-F~cXvv7XprtM>28n`knZk~PNnnPct7t?_#BKO za&YXm*VS{*E9{e^6zW@|w@^?}s4~(Ym7$=Z(V(E-@FKzkKjCoj#sU6`j)fc&1qgl2}N{>Y5?)V9dLVfu`44D*~y8Z)J zk2se0J~wm11eO@$J5Ck(Fc|L9{^p^$S(>sNx7OOGwsCTIewjuaxkla88^(dm%9AyT zIY0MZ9*#IL0em0@BJh<@adt$C0V|3DyV>Ry(H+Kd@O6;VBGM5^I_g)-47v78r8FRpM$P z!6PBb#EI(>=lVjodU<`u`HEO<%<8X_PxsHJ4xh9Qw7W}DX`o2dc@qcY*#o-O;a4Xn zIY||eVd1Mvw=TRK?^N$`;IdcWsw0Cmhe3uLdll$8x!E1UOm?i`G|043zgdG3S31la z$_1i#_PAu)O9CaU=9v= z!d26vkcwfx&;@I2`IxIr3RkwM=MDEe$bz7Lr)i!5(jrM2&6qp+v zmuSc%Ng-&_O6HZl)NfXf#{our@bDqsmaYG1LxjZsIX7Y5XNBreOl9mXO@{>%YOS6L zR8GC#DoYPv2C}Lc;h1DaEiqL0K&BAr-iZlN5*~ta*)sf>O5+1p%nD*&TXh?CWc3Kb zcq6e`Q)(^fUPI?AycIH5C|ZeJLuEYLc=qV^u~Tdgskea95Tg8Il3)D(>b>XuDJ69g zbXOM}5)7YxIB;^Y>Eyld;BDc$^qHzR)`XByvR!()#lLhAK<&;%C6+LE9o-KgL+Joko>O4r`GCgmVHtZRN&F0v9#;8u+( zNW_>TPfKfB0N=_mV%BrrODTi-8fL*kVwP7)pB!_kWk}rj-z}6UeXy8tQGGGAu;4%u zoQ&-i>zUgOz_hEIJ;~L>)j@y{+sp;$1a^O;zR#zcX!GhmL0hqek;e+B-mcb46=ZY1 z>HH@2@n48;e%ZUa(`?D*y>tfqafj7R?zaY19^7(?kX6YVbf$Bow8w5zLB5{o86J0o zkM0ptH{bPO<0|Ta1f{!W@~GvdWply}4eP(+WqcX3SrS28d>GLG7c6#~D!S40#n2eI zn37;WH^a-|-0gXouixxqlF{^fiES(ih=|D8~2&)wq)Z%p(#U`eGl;8P5o<|=}1qWOROxo~CQFATr<=@Aik^uUwD?|LR zfK9I^Pp&qj!wK+kAb8w%Rn3kC4~ZM5^>A@Ioi7u<`!V4;%SV0NSGo6GE_&z{F_UN4 zTl1Vyhv~}`At;M08vkV^Gz((cGBJdi`kx-kTgD;s=Qt?~gTF7PNKX;n5Nww7KuMF> z#vPG3pCEeFVU-SaPh_mZ+2IDr5CjU!5yHc~Pfvbn%K|qOO}t+J9;M{iv|1A#rU^eT17>rlp0~*f``XB)|b`8S*vYfx7FP zsTUJ1i9T!6219xPO^a#KZ~21!;^beTJz;e8|J@nvD(L)y>oG06Y7P$-85L?4RY86{ z8zVL6hC|{o04WT$Ep-@^>MC^#7Eqtm67%vXx3;!YgnjvukdStEca?Q@D5^V zac^0zcXOFun^n-!;5pF~%7x34q?Ey;mFPB6KEI!_h0vw}wTvFMemoq(~thGHRAzP!K#fB%3igsoC2bVrze7OKj6- zTQwR^M4|^)#z7&3XBSt|K@jp~iQ6_tT9=ys9I*kEeus&P&qDmQ-f-y5ESviSYkM{4 zZh{w|A0#XQC+2LdpPpD!G?pV4s(iN1GG5c9+L(O#6 znCH1Tm@fgf`08*L2M;eUDM?aM z5iK(_laq&MTKTi6BzUvhHoE>PS0@&#*R1d8BF~bjl?_xJ6nv;*X<-Qi)NY`NUXRw# z{AXZTl+KW`v#gDzc{=BDdlG4F)>(reqh*wn&MlT(LLnJdx=a4>TU)fr4X3n$U`;Wn5f1;o#taV*?)ebACOhZ*zf`+Q!}=p3Hri9cW7C zQ6mjY3OP+;I!gHupNQ@MJMaFC)wA1`!lP1ZPm6n|v{7U~bEoyrD5^dsvf@@xewpX) z7~S2vjFOYpW0nG1T9L~7l&pd{M?=(TZX&}Q9Eo@IERcj-O=YYtzyb@u{T@z*HTKNb_sfVdP2j)^UBI1eq6g5 zL105-wFxDc@T-{eA%`T;<2S~;SyYatcg;v;z1(TarmxthWfxK0CPAw0}oYDTWG%by-wKtP^+ zn4cF}7-d_K1;d(0a-LiBr#z-38}SX$Dqt*sU725)po)}7>n#Z+)O z)lk3Y1{fj64+gU?EN$FazsLJ8kxhPTE2%pA7%fM|xAO6La^87%S>t@myJ19#6`O-S zS~_f~7_W}WOISn1iQc)VpNF^J-Bm?hMX4|89?Xnf^BL2mi2Ah>xqQ5BwsY>9pp*?6 z<=?@C5l7F6J))g<(;;&ExEy8oWoSqiI1?y+wCzg2buoDNDvy|79RDyn95KJJ1G}s* zu&>=7ZS&tP9alyAg}*$%H;`}3l0)u=MiA|lTOEjT8rXW7vTNG3YI68F8uO>^YY9{P zfnbvE4TzlV0aQ#&zd)PW{UYSu-+UT3aFkzO6kVIv3g$xoya@)vne833XO-%McTk;# zm|-*wBAI>=Gg@n<@vkWsxR6!C>wTw|{nJxPRCstxd>u^`t+$Qb7>A`_KG;g>x=`cQ zd1j-I<8%XU4cMP%QF`dIdgsSLD6de&%=Va>S+DWRxd|)2&;>W0x%NJ*Un;TPBQxus zaBtzFxx-(2^7Es&YBmFDWOEf?==Ou6jKVB%!p|Bcj_Af$rDWKUSA2@GksRL1&+bJq#r) zk0I$pgU}a~ag*zr)#cLRlics^;iGB|w}E=l>JeYt%U%A^nH|@Kg+Ss$VDUP+H6!^h z%U3}tk%ncxhKXI*_UMQaQ4(zF?k&P#3eFU!B-_B=u4^~1Wl*3VW^bF0m)_G1Pt&H$ zg0zzCLgCZ0-hWOsAYY@B>aWUzl$h- zVpA^Q-$r`4e+m}2q)F5vnh1&~?;H5Y=4nKWJ+1sH;i=fQo4)|$K(PXwb}O1x&@~j- z+Fd!~!v)fFk(21NJ23{T7Rx-ZbZY?9aJNJye2{{%wZIb@I`9Q##0A^7hnM?>6Om51 z*Le-!i;mI2BNF{k_42f#{jnI~uAa+d&c@^W-Nu{B`wK^sh7F%npE9f%FnCMJ&P?=a zTCnk5@~|Br_K`r7!*$HqUge8~(}nTPpX~%a5%h9<%k5rVM$fycw_g+@~IGutK`u$Dc4 zxQ&IN<6WWBzfJsIQO0qN_HT%-ZW;trr_4Lvh?6`15`yDpR@QTgVe;wUm>L(H_dbO)DmN3g|2<1?C7Kqk?-HlgTzU7N zTa4v;xgwqAbP+1?6)4RXz0iV)RzflfCfm|*VCEh8mkCxc8_);;pc^sCbmc%6X^$HC z;h80}=E30KdCx%BX@@!BSDzNjf&aI6Q06^#?Dw|s_g*yDTFx?QouTG0had8085s?Vwk$EPlmVqFeFX{ItyQTNT>YatrFub=b}X~4_axF}e_LIl*Q5?k zhZOpu&{Est2}LXEMhjeu_}a)m1ahT2+Y81GpZY;&>+Cjn{!X>*bkC<=^-A;4YfQVl zS>@bLYIKW`dnK(5m^qro1sbh3n8B}tzOPJmwZ=KMdco8Y^4q8v?I}>&i}hE(i$Vxo zg6vM%_I?4H#>uk_<&Y|LXgQl6O zC4FLH;jdbGF(-EL=H~5_-;%cW{?P&Wd-R-oD?fi|co?BG{?|9s`sC_bukUs<{Gm!p zQJuf^aNWF4GVq^yyki5bPbmGaJP`0juI&{E>n9BUyyaQHgnH4vMcG?-VUerU5#3}q zuelzSf^2H|lj0Y$ekDk{Y*ZqlQ_Q~3DAT9ffeL5C*4 zk>JnW$$O3J=lLSQve``K1a$B&l8S&71WH@7z%)ypZV+2VHU zOVxMi*Ki8+mPh+}H5EZcb8btmtrz!_S*+bwY}mWi+x5#x&I^tUwkZm<^5OEHwn9-A zC%swZDdCHOMb__YBNq!WPUkGt)^)4=x6H}5v zhGr(sv{*V%f-UQlw=scbd+l8%Ft7;~o2P3On7cov(#!b#xjLLMM_ozgwIM^O8w~EX zDJotMBHCTBb08qXgXm}bO-j^ra{n1M&;A;YCO(L(CUxB9nkG8kLA@;tZzqRqOD|Eg`cZSp1 z)&IT3jW~b0RIC&c4!Fi;J>MWkt86<`joYr1agTobGO@b1wH!sGI?waQkUKRcNg%dl z5nBxT!-+!aVR2<);!;0eBWL5r)g45Lj1*JR$%jeif{ps;$IX?fhIeyG33aU&96-Bx z3)Bi@0{FDbZ51_LJJ!rfza`4vZzhDPAKrg!Y$a=EwsqBk)+QplJ&j;}zg`~(S<>cT zS~_oDC4dBs+R^cICnZFpI29|JUdKvdB`0t)q3r9dlF`fW!-}cvYOqc!A}F@uMl42B zGD>5W)3YG^?zSTl@}Bh0cP*rUjCgfT-f3Tbl)_B+Q?KyrY{B8q7&Wbnl_$Z5$`q$N zjmOaLLdgOw1QHaiQNd)AG$c}8={5#H8*Rb3P>KiIg^{#HyD{C@hHk`H8 z`M=((*AW&k$}3eN8<^Zy<3VCu-y^P(5s_@EDDz$%*SWV&?MjORvx`x|1Cx+1?+9&q zcNc|RKxKIDQSkfPb|te2lK;@W$iy|N*?IE6cV;|;Az42i;U)v>Kze=C_k=gA*EDo1 zbzgW`{R1qM@oRS*?8Aglk{`EVzs49?Z|ut{QF!twEHR0y=TK7*&xP*N9%}`CC;i zT|3HN5?)*G5i^zV4Pav}wf&d9gkavdEhD!Ffo0OXzSsVD#>Fdq{dd?ywn-{o_M+wL z39bHdNhh=`gc%&u~+Mh?RNus-g-J*OpioyY%_X8XbgUhbePOH z+7ZD2Zn*Mks|kMI;2fq$avaG_QnFFq;q@CMJrmb+X2fl{n&IRO!Apsw4y!Nlpe>*U zTG3EnRz+$qE_dj3hSEuwOh^*JAD6xFG5n5~{HgEU4Ke3^&O)QAWiYi;!4FTP+ z!11^t!qXT`wpp7TzQ?*5ZZ5JL*kT^AlcRL>p zzu1cRc9|Ly>_ywKavqmobx$se2hWC&S|K%Z{_1WahSnJwlP)YPjsdW2UttFLf7O_P zv@dXG=?nXwI*UPyR5OYpEd$NlmQLarwB34g$KkuPi!%Hq$4wYIOpyaYG9RWe4gI@s zJlxcvqh$Zh$UA!D``+cFL-Sb(77$r)s&$x0(&ChWnm|ULR7?!7L!`5yt0t!hxTJ|_ z^R%d<8BsjD@=ZDyv)PWhin>k+FpIU#_oXSAj%1}F#6{9;9_1{f7`dQ!l;JC>-l825 zHU8+@*Cv6lbax{}M4hW7%fp3h8x5Z|>ZfMxewQ3{M!Z{nHp}uo6Z0R`mtVlu(Tj@T zA-&YW>O3rueU&XTKz^ntd(4{sTZxVa42|NBsif;9VB6^tAP^n8^qY%Db#{Dd^tTOD znb^FmiI$nw80_1>wKuoYNHOyRyQSLPYBLjO7f=E3qEky`fToUuPuMMcoCXwEo5nBv z;zo+G&#&dNlS|#-mqQGnhDTt!d2#OmU`+?`S>MmWn}+5MsK%&h;`T8!df=nOcZx}l zOD7Aqcq6pVhEsGhJ=<5C7k8m=(U;GDSpS>QBozvGXUkfSuXv+J-MDAM?&aA-v!SLW@I$8i4xXVi7TQ1buGxV<^XbSlv z43rbhU1K^#{Znde4jmn7@KWaJY_z7*79colVrpGGbW<)@0OB#S;Vkhk5wNQN&xw$g z=*RWF=&K~xf6Ad}OWsSb-g}{&u5-nKIAVUWjkL`yP4;OVgQ+-8zCzpoxVB(6HiruDga|v9-|hHh+avX>XWS0n?d40FVpygo4g|NJ1sh+$h6e{!+B^B-^vkH8HC+EZqF?rGx2%nI1ATS=yy-!S+7BO@&}B6Tn3I7j z_uWq13BlPhYB*l{vna4V&na8hRGnKN_v5*1AF*vzw!47o51mQm(T@CsGSGcx zS7;O8dtN690x<~G_f~Mukyp-<*wwb0TDv}Oa_N+<@~6ye@Ed5iH3c@qbbbWAfnz7^ zgL#LvQOydo^4+e4DVSk1^zttU7xxRQpGg<8sAJ<(8x3$J&=xw4B7wRsTs+-khBUmY z!sm7Rky7fOkzUGae|DZ@yQfd^YShOro1AZ#@)wY!e`xD`)UYSV;APAbF@8H4>%A+# zUR-KZ`W=7)-p25N;9Q!gyDwK{0F36OY1sO#^f*$rU6|aUY8e*etx+5`TsVa(Arg{S z92@;xym-QhQllF$N7w5OWs`PpMoEXH{+fE;4-S?#O(6~n19oi2?)=p$qbUk+!}DMx zC}5nqQ0f^L%khjTdk;?BoTR;<%_6uDdmDv_BTPl6;5iCk?7pMz#X?P-@1J`qgj`Imkx4ku5C+T8{ zJuNRMU4cMc{W~WLj112=HIeWl@GR#sBtT6al&%20gPF>Pb}S5|K0HJR4ILE$>)%KY zii~7_mnxl~Zr6~^%-FQiVKRp}&Vea#g(0ca`2B#Nc(_zkM;Br+{mnGhvIU!dWVqji z=p~U+>MaVuWsUqArxl_HuQWAv%F2WiUp#lg${7*_z%(<-y_Yqlw~MfooWC+;4#q>l z@p~*KA3Z{yQAEjbmQ&tnsC^}b7i|DSSfyI+9R?0G_}v7J;|4tB%|FBi1td7*`;3K- z?)3Bc|Fi%iPq>@K`s4XGwmW0R#tz43jDEgy#0C|M((c*FUjQ~~7gzBe`v@PJfKZ0g zm5QoU;vp97b{O30HvRG~|}wZzXFd*oTmf zpyVoseZn9>K?-t34hj$r3;<^%I9U8g6+pq*Zzn%K#TMU=Jc?K9TrxRtje10Me)(Rl z_bh(k`Ia3wT{$%stopC%%0-*$jFL|lkGoYfDMMfi< zU(X1#$6*h+>a3fKqsR7h^8^-vH3F6qv@{81U%5~;FUY$1p1YF6@KJor%{a2m0t^-U zsy-{@Jidy)4*Hi~JL9#>Kp}SRyvCYTHDbB_LnM^nDDM2c1r*6L2lNP6Z>$ZL7dQVr z{8^27y&g0>D=z=Gd)nt>`OTF!MUtBdLjri3#0DQ7)j%aw`7KT^b7&5_JwxuOv+RYr zXcl!lrreSSQbhR`P7aZfcBF%cKBr-Xwlg`<0^w`syjkep*_#eZe2fGsZyE900GE(m*)lLYei)TrQf>{^Wy8)!Wsjv& zOLrWbn-N$|_3FtE7?b!^H{CQPN7z1ANY(hV;*ibkBwd|SJw#D;3 ztXv8D@1eH;=$7|_q$a7E$~PYgD`q8gk&#pWJo;Nx)SR(3WaYB@A&DA5IN6<@#zZ-| z>K!&PZFK@ENWQqJWtTYUG$dyb0|=GCC0tR{WdTs87+CHxeo z(l~$LTM?j3m~ag$pP?QT+_c!mSH#Q}T_}Dn=$la6xjP5h&TB#j(~Cdi_Q-wWnudj- zZi&=CpJvD|{1I{Nt6J}N88a4i&F%I+Q#Ufn>K-k2IbYct-zOM;qaS9ZEKAC|`o!t} zd;ng%HDb2wjEdB9<~#d@AYsom@EFm?TXW4Scx6g#=Xx_G?&^#Ea<|X%L(3piNlhj+ zYB#|cZTr@eUyEAI0D(Q1sAlDP9miT3G zcoez$rp3F(Ltn6xOYYXinU$a0iM2Um{l$f7)A)9H?}j*`^z*Ikb02;K{3bUtvY(p( z>!TUb+3Ev-QTF;EZisn2VOh;1kGaF5!SKat&w$m@@ovZXkFDt{jp^+}x)JbRvpD+Z zj4>J&8@Ct!+GmVwl+WA1!QP0T=?|~t`->f_b()-xqwN8s-B!b?ixIjA(OGG6vxpab zOqS#O`4=Z)4Z*)}jCT(wbiGXonf7l{x5~X2Lpr zt`rzUNL)GLYPe^Hl692ez@{853d=@$Ux(>=>Pn>I9IVY+BL7=vuFPlKcDQ=-_Py3U>vpH(bZ?JHyAO@ zxX?q5o^k|N(-<5GkRVi2+p>gV&!Ai-xRVnvp0EY;m9|35$iuH;7 zHz-XZ%MCjB6bu==VU-lUQ-F8u$tT5O^3>0i}NHnRB$qBhr|H#^UV) z&B~-JDeH#|C?xC+Klkcu5naF>~T9 zn=^cl^2eGztE}~o_Z+Ow57d`ACXYH#xUerqB!=5@eK z{|q|q<%I()eL8LbJ0@^*r3gqeY_g*duXEBz-pj&Tj@fHn>WUQWdkRa*$~C{BatvgF zzLwJysy_b)&+TQ7Jemw6BDjbs0Mv67k8pSYjPqS*$mzzEO7!+^yrMDq6fdjoNJ8Bb zY_ZD4X)~E}(VNfP-xS!XK|aQ+WVc(zdINg7jOSI=9?EPI>C+$?3UkKl=d7>mC<`%V zoUhL1te$ZVp6>qaYzFd{C38EPtDp?5_Q$dX2T~k2U9Ls7I}K}*`1c@XxQsHY-kTW3 z{K%5aOFrL7M=i*5<)CX48r~Y`r!eByc(NkvkU(9gS4?mWzM9U%KaU^w`!*zG z6N7c?cy66Q?9^m>NDPrJ%8Z(>a}r^8SS=(S60M$dpa|n$k+q9|Idhd z`99&HabV)PH5!FqyLaJJs5k*YzFqm@y-4h6ydzme1d`7r^oTuvLsESrr<+mLM;^Wh zE-R$!>`YLv*-Q5yLr3y}vA6Dnga2~D>Lc*Ly-Vhb?QVDTWJ0;CJIGq3--WGIWprxm z{JrA4iLE1_p8}O#KbyS4%^N*>_itISjOqy#zq7@U=Kc33J6l_}3S)R#=7I$wKaf^I z?@P+TuAeO}N{&{_FFzm$Db#}y5ojmX-T_B4k!}hun7&(0XDR_1pD`HqJyiX@M@R=L zq>I~m343-_IlfF0C_WEblB6b!dU=sFW;W!?Dzek2-+4^d`4md!-_!#mAHan%@!lh1 zMXZ~Yc)lrCjG%cpJaVs*WiF!yOSkHTO;@A^!~v1^((O*!Y{KkJ?g2j%+h-CHf4q1) zv*-sRyugUWrtChon8zRu179wOyBlToreKv%ys^JlZ(FafW14>-`o}jLQW`uSLgU?# zjfJUDN!eA-FW;W;mfb1+ks%>l?jfE(-MK?}f3eI6yqDaQ~Y_MqZUyFT_ShAgd1 z4%+AA7~L`-*$(=!-Z4I`r}#F{KlU^vu1B0eicDYE;0*=wg$Yrct1j;!x&L+5gNK_| z6ZWv@gA)-+ibtad^lwO!in?T^b*qt)!BjUSr(PEsZ2r*z8oge={t-yuP-y9ieABcM ziHtYAj3s+(z1jX5kw)Jy=gK##ZUno7Ea3!QyoOH6k9RC~^?hJ0k=P=wTro@|$3?#U z>r?y;ckGZB<9fqHIQ|YEts710EPgtry}q@O|H293Bji*t7uTY(mK&Pnj0$cMF>yOc z$8knYE~b7Xd?=zn5^w6M`)9dXk6&PhGs-ocQPV&>py=RN?a!HvFq~cU2s}u~$3}h! z$X76QPZ7(;h-a1Kz`djoo$H@l*B6Sjc1~fRnn>ekACcnsjGhOvoqsm_@uahQwdSA0 zor|O>hVb-%N2iyjHM~MIUxc?3jY}m(5(zxFIl`)_-uRm;ghqffroG4JLxD~v{ zhOToiKqT~vZS1beP6&>a6MB-;4pXZE-(@id{5tue>T-7SYYzOCdm zC(zLGZO+)d6M6S&Xo;f+JyCQG8ZH zS1NL}+1QK^Cebw#OoOe^@Z1%a5F0n>PhUsad=f*3Pp>Np@#;vNH-6o_Gb^%}f#DT* zNb1e0&d+2${z}ww=d-d*636A~_Qzkp3#z=ciEh;G&(?lM+_XdX3PjF(hRjrZ40-&{O12PwDepWT8~&06}V#2zkM*cRqLkJ_wXaF?6{$^OTLZ5g9VSXt*H8) z6Va&n4fTxVn#=Nc;0|lmx}f@ED{dCknVnVZ7(H{vu)n0$*tP<&)cjHS_gDGn%7LNZ zCeBnm7PJ!Yy*A6!&3N-uUZT9fk%r*aI5}vLuhF@ra!vSkm=%{JhQb6Z?vB1ESbydeh*(p7 zQs$?>hLN@df;ld&@w-$GR=(&$_bc?}xZ-2qAN3i(^pI=A*m`64*1x>ypc)GdCwP>J zY$9(H*gagRZk1!WZ%Je{@3;vw{dJ({FG4DI-aYm>R%Qwz%j)TjSlJH-=WIJ)Z8z;Y z?f;JOSTZKNlFky6dF|$}Nd_ZlJHk6x_Dt@#>SQ=zAQ1}wFQiQ!bG?zq&KCY#v_w<} zyVt#q>&$F;rHAGUGK3PQYWt7?na?S)*`xItVZG+YoP|z@r08**jE3q(#`2?8rH6ZM zTkPD4=yv%IPG~C5v^kNELzdunO==}?(=6;qnUOT{d7^CNL-1X);|3xR|{33&NE0^{A z8vlXs&x$Pbm4)RXE&;^jUSqRszmtgu#o691%n9y%Dv7?9Ldh!kTwJu zcScgV35!>`x2%~4>ry!&F+5Y;v_-`rcZ6k(Ycl7Lf?+Z;R4-%F>`o4vk;kF$C^xC? zeEwKL4}hbdqZJ>)idGsr2bBjOXz-9X&c5dliyTLRzDng@a!=-q79gQ$5N1DDMGzv> zMm8ISbKqIqlV&ndYrgTPa}}b^>lu}13B$BDds?k0Y{_sQ;LzQ%GwGc&_t;PxkGoun zl3yO^w4x+J4g?85D)?N6L^NObS;a*QCVu63^zoUu_o!j`T4ygD&|gfx7okWh&JpXk zBZd}Lx}E8LX8Lr2?y&sRRPcPueFm8AF4Lqu@9dSF9z4H(jHzA8Y3Yb}Pxc+$Td(Xr zxxxaVzd@<-y_6P?fs-$5ON)JNFT^X7E+)Km#k52gL*xLM&SpAhn&D}}Mo*Pp3 zG+f>de#xCWBYTFno>B4t3L@DDhFky015PHYcX_4wQ*c;(5}p&1tH zWEi(lNIy1^Zs7>f7$o~Hl^($ZjCki8(;Dof^e}V|TP^%?GLq~pTSa&H&SxCf9vAcT zwH8j;XHUC$KZH)-_`UQx%-Mx@wOuypu3uVbJ@;PxjQW#MRJA?|NYVk`8p^OiGew9d z2AET~xi10T?VpqW(FApSB)i^`cW|CNN+TcedKuB>ljO(RPq6sOpE!~`W&%h=GF2)& zV~w|PHnDc)n; z;7&pv(>>z>!;$AHjZm%X;<2dr?ypdB9OsY<;q00D>h4?aB=x+5fZQ0k_}cd)7L3Jq$b8SK?Ls4nl+OrJwa9U(N{IuV8w* zQezUjV12wFGO6@AH^{h5{@Z*!WV!M*lCMhQ#O=P~`RBVBK;Z%{)`jh)*-F-jC{Z(6 zO6Ot)TFMbVyX>MLg;7=Wc$2fs2p<6O`Y&=Z;EZf`%YZaG@s+)ggh3|!%WmnrE6y<9 z2?2jycOpQFWk_7-!glRN&3GB(61+S`*L2yn_q^Ykw9f^^k%`3N2ge6+6twJto*Zwb zqhm;$lqQc3_W(q2h7)Dv-okjS-lvvBdBB!9&>;sK(1!&(QegWoNIY(oEPGs5d=rLr+22by#JP(#1Tt)7ylDhF|Zo<-yq7%=3wgsDf7P)rWff9A|A_-DQ`-{ty&p>%BFT4 z=5Tj7wkRV*!V#`y0Y*k!AtFklo*NfUY#Z~18UQnfJEHH6v+%#=_j)O1e!kXy3KIEOOaM$`=~C$(G(hH)7E|W?fx2ZNb5w2qC6ccuU%>gs!?%1{SJ6>IVdK~! z?pJn-=;n^@@wR5d0ubXoN8FPuYe*kdJqSX^-cvBM`K>g*Lqwo@#b_3&TpGxExHEPq z8`y7EMTmzP7B$ts9->DY{Q?+`>lq))ftvH0_3hsPpADGyjUik5D||GFT_=VOixgPx z`-;g_E8THLDd5hQuj>W?3i@wYew2N3uhopZut00ND$C-CL)= zb6HUWn9o;K092i8$)8?kt)-((Mx>pJf!zoE&w|KpX{0Zut&-|??~AHLsz<;8MgG?e z2LqcPqVe{Bc!YkTB@vBU1Mo~ye-P;BM@f*iEE(b8c3Y{5wG{q(IeED;B9ISV?oHTL z;`&_$0?&OZU-cH=U)QTXE35V1k0$2A0ea`26h6KXq>{mH?s}}Q-)?nQ+hz>#k|rq% zBO~Lv6asSP`t*ox|5fPs*@wjir4ZD@L|}7*Vxs30FJF3jLV+VHO6mGI080Y=uGlnZ z!A5ALIBf(!yeI)Ha63I{1dwjSyXu-zm!YYN7um=o(en>`PTubss`3g-x$Eb|ZB(fN zSnXSE2B>m9v69%Pvh;V`*3>6n8M*Mi2}`(-62ZVd0$K#XgBx>+B`HArfF=N#V_NNU zj`$&L1mx7z)JP^;l1Vf;T9UE2fR-$-;Ot~5EOWJ^oVBc=wvkLIr4>+%r^aS0cun2T zfuI2TwgxDTtG+Byc~rfr0=Bg&F+c}`fk_e^q^_sJSh}vHemIuii=d6?q9--`_`Njy zISEgYcs^f4jk2l9==4At5K{$?`FpQAWYzddS4&RmJksDZj#J$f5S(CLjdGHS#p&TDVjh*{ z1r%6XhqTz_QGkHLF?}q`1G-uhfP3fNe=Q|n;h_wwzZzOYz&Qu}`q?pA%&yHMnqnT4 z+LS0TCWK5$-5XyxKmDw}N{3}Zfk*Imr@JhwC$YyS&-x*bvWuI{;Lgp&JnT7?* zXaVz;MLZ4;CEx&)2S6* zPMq_Lf-ztN%=%;8LrDn`AImc$7HR!ctm=~s zOK7lY;m9#*N?T+WTkC4TCe%Lv|h;awyCWQArP*> zwhpz7iFtvOv?vR(Xw_`5gb_>%c&04=*hFTASw8$jOy|6d;0+R~MEs;zFx5Le1{;*hWQx zp~{`0R$J0e~Af+>#0fjCBpDr8iLy!{I-Ocj&5!@8xrE;w^4^a6V} zjW&efvuEPP2~jpwbTpV;QT4!Fa2bCH7&6b3LmZF0JAR7*g|ZL}nQ z1q=8TkbQ8}RJU&spvA3>1nY`-F!4!Sq#6N`XCJ_6NpN1dN1rP%3WqNOcl);KcU-_3 zuO}LJY$=8BC@?VOi@G-q+2MM`zR!x1*|>fSd?RL5-PS`ewPc85YfG?nm=my0{uV7@ znO_JQ56k^e3GYdxZI8H$wZIwc3nailO=_HeW%~*c?0;ULk_4cKSHXT?5s*iE1_$Mh zl_lSzrQ61w1kD#iv?kaQ-GmrpAUr0&hi}U}*8pbRw{dRGTp%J2Yl3 zAN8B!9VS|v?E>d7nTs0v@yuVXqqrY(G})>6-b5@4R<7#+#3m~939tqMoRo;a4Ivc8 zJuWfsXK=lVQU3L@)(=4Y0Fbzpm;|t!UjQkdLzj+8_gK&5^+tKo~Z;x(8?vYFfW10KJQwwO=5ahRCQb z>FdtM#FS#^0LSq~^S*Sga@;R@1xf~=@p%kjF%Qs2DJwV7aI%ZT!iz4vyqbGL*Nz8J zyXI!^;VY7=yI2sw0}`|R^2loU7g9tRvuQ&MP->S6*_tg_`GflYgzKEkb#+No48-q- zds+eg+ZUX+D0o>gaJ$a3sw1 zH*u0GHIcfC!mSvZI6+koZ{w#_+-@@gJwi-bcH@}cQd?`#ywI(x>%!k-b>u-L$AX^& zacKmd^OEmfZdx=uTLS+qwSKBGl@%ian6&|4K3ibs%SJ@&d1EVXiQ8HU^9|sDM+2m$ zU4Y!QS!fRP^_$sW!<(NUEC`4waj+HqM96s>YJl@kCO{E>=vWr-SneHQwv`Vj)y z9T0qmMMP6@GsG=?m)-(&5*Fa?>-|`u@_p#c{-UzfEzfWhshy>A^NxspU1M(8JZoD@b2G1gX-PgCFN!j} z84WFnl_o49)PdS`;6)W>R7qTk++dEUx~Ktl6?9wPkoiX#AHRqe(~=vLsMT^m=nA9z zsG9Xf>I_o~JUH@!8?Ld?hHJbRDr8kMcsgLPz>q5)$(a|?c>&3J31%McfsB}ePg_4* zRz{5((3KsyBUsUom|=Tr9mo%6QPA391;eDgdQVw-5sQm}KOm_da9Qm-Qn~P{4t7{Y zjLsq8f1LW6bV`OJQM5EW-l^+)2`etGs7hUIszl8_LipYZ<&Py%#gpBAPy;ktU6@A* zOhg_9J*{$nKCb|KhOw85XI**VP?rATDp^)}$@WmBs*!vVm>D%UmLNR3y!6tyI`~|U zRq8X2i3}wbA7@#5*}N;MhBY!+!WuzJ`u)&`AZmCIak9R= zggv>O5Q9)R$U#h!Xuzz1Ds_}X=9V*RHfpro(ypkon1?g*3Q8+ZItJ0)OqsGSE*>=e z_5u@ASJ;>+TXpu_Qcq~LG~3*oxS5*+x7g@Af~EKWX#t3YnH%zgi;B^f&q;lmD>SE- zV{zfi`bK*4_b#rM6lR(lA>pJMs_t*U$3&Gv0ft==Y~Q?8c`;+jV`?*_3o)Aebg=xmd<6ZolMfr z?Rs%_L^%6S*nqx*qO&=nKS_Q5IVBXtu8e>|j$5f#HZ#E+&D-b~z{zWEyPVZ zVhmDK!%b(Jt}$wt@gUm zXtGH}{ym&Pz0aRi6XTN9Wwo{Md^M1Y3#mES*`YKnGS z<5@mYb$9uuvEJ2xNVLSH4*X(%PE8mxd?D3=i+prMVea%uX_|I8Kfl)MR6jM4I=eTH zW1RJt+A)`p>)2YqnnC}R{HpR<2mOQE>LfOr0<-NgQ(4aU@;En@Vrzd@kM}|F)ZcKh zXUlR@Advoxue2nb_cVHB_fVg~S&Ffxc}ArbeQh;kpNWz6IJC6cn{ecqbK=st&-P>LGi__WUUK#6)g(G@V>VRMaSA^w+x=ICe|jZFK9} zT;Irw`Jkp|BGI4Gfh&?*)>^2OD7OdI(OQTa!ST{Nn=geyj?tBXK%P5ow->Bchlq@} zdpMFyjo`Y6Z*j~SzrG+H^jA?}W;y+G!ceTeV^R)_4o#Yx?fn<@rrlSIjDPLR$V!V; ztlF#p$&2^JNwr$(CCz;r`?POwm zVkZ-8Vq;=^V%z3K-+A7(zPs-Kd%8|{)vmpF^))roTfhV%6$On9{!vuP(b`NtF>c)1 zsHvXXQHmu;Z%b2AS=!#b`IrIhO%Xi~+FMqMurRA0cs#s7QI6e$xoa!yYo{9Mf3H3k z(Ss`KVzf*yaFCXFl$vXnsOl(M*9`iEyN5%4J4P7Y@hdkh9!r1sFE6Kt#G%?H#2Z)p zpCS3jDyp{a1H+|4+4&j6M2eFZcZ`?_a;B+Q+ZmYYsoB|rwG17hB0^D2IA*Qt8-US+ z)6X85;Ov6avtpV83~VxbHab4dCums=*qi&3)y$uj&BgnPR8D72S{1dn1w}3K^W!=R zGDzJ@(tc_Xw6Ls$QsLLE3Toow`*lJ8?IyxYsubI9O|7Ccm>~@OvyO8yn5#X}f?3qTLu5hU$M4#Gh?)+2n2$%EMK&xLdd(=^)OV_Qq6gJW#;BEQd8N#`tPGIS(Tx2;wYdYS^4;qJsVo?=2{3Gh`F(3 zF)|3Sw4 z{%<8FQ)zlOIZAOmG>F|E>mF2bIA?2Rm2}B}Hv~yK5yxSaEgL)H<8GdyL<_1*Ffpae z4~Ni4(D$z+WXtwzxwnUZj59~wag+j0QB1$)f@ECT9FT;a%a zLk{WntCty*7JLd8(wlIWw(OaDtDKv2rxax3;*m)3q;vv5Z+1kcDhmn==cs|B;XGMT zPFDlK>DK0j`7P1T3Ut*n+?3VWzcvFVji%1{51lL8I=iF<%A!r{gt%m#CoWhh5DFec z&Fkx#Uxh=wWyDq-WUyfD;fRM5m#@Em{hHNL5j7Vl->%Ug`G(Iuw=ow)0`vZKu>a!A zKPV#Qg|mHdX{6~2UtC0%BoPE0B&x70r!DZ0>4s>7 zH;MGxGF$bF z?neUg7j>4zEK%nqYRBogxxc02>B|=hG<^5>zj=J>ADMtJ7EJ)MlB$EydW@W49$?+P z_C^E6$VHm--?`FJSC>YDq|WugH3WyijRR@YsUEG2rhoyqctd?<*2Nm%@UX=FtDq++ zuvJt*UOLpXCo4Q9^bnk+OjgV?HL3>q&<+G@H`nZCZ%`$rYv7)Hc)86q1tT1&BNjuo zTCf57ZHkmuR$+AV_c^>LPf7nFpPh>S?(VI=&Gh;N2K)c0qL zV4>p%q~vv>*Wsf7+nSX>%a_k(qNB)YxuP>}Sll{3=)V|b$;7{>==J~E9x0PH00mV+ z`3(tW%8@Mqbx@nU1Z%Ka`c|~Wl`I}|yM&L6fhXqR4^BTpF9Q!NMIEijY6s(JX@A&^ zn2m#vt_bW{)Q*1(9!o$ zFfud2(Z*^VN$PrnYB-w_5@Jc_m($x#rEmR3Np0z?v{Y42UtNOxZX%tg6%_w$pUVGk zFO4URQ-tY@0!I@~Qo&4RjElEQWC_V*=|-y7u*}Ec4k|7R4I_{4_QzY}drIyV`!C`! zg+Ro`!^i9_#65%{mA(baqRGm~T3W4vfPEn8GlD;Nz|>q4(nRmUQDQimX=5a-8tJCV zGq@qo!Ub2dG0ooGH!$VLOa@r&-Vd3AaqZM^!W4P^rq~E84yRb@} zAVf!J;zE~93A}A!h~cra?2~U}K{4Y4vXyXAAuK zv%yW1mM;EgQd`M3WXi(SSr!+hhcD_DG=OA1*Iz~Ot(?xw5ZT;^;_t(30Y=^b=fmg>$@t`55>^ZN?$UowR~v94n~-?WKitlms9UDt**s zIcc*_UAkF%CZLiYYQ;Y>{3udW`o&%fTk>=oGl^q7oHcadk_`@`3vvB&(`QQ0@B)~b zj1?s*i^<4kk>N?#mXV1qub_d-YwpX>kB7girY;^a1=V+QLNl%S%~h?nX~NG?O`&)Q z)tHr@0fIRPOv>YF?5{ASsuqN(abdR$1HjXCERl-~Rt5bp$;ke<%SFm7$Cuv4xa_xs zipn8g{;^meD4UQEB#fR*M?^H3rtdTL01oolMbO$22~m-{Wm9}ssB7WRV}f7iiNCJg zRZPAY5VsRjMe`|CS5{CCPK^Ex^`hv`gNH6-XWfB?!C{E zv*K^Y|2^~tbe7ZXiX$H*R$A@4LW!%$qzqwSM&aqnuaB5S9-KY00aOT3plzwH8nJ0K zp5^anfCQd8E(wen*^45?>)9i6u)A{5cp#9f`RIlZTEBc3=cMnMs;5wHArLdkKh1D1gkMjFGx`M+c;xHX>|D)a@ImGd8K6 zUdAe278$nDB6)$AXrJ1cgx#73xh#8NHd6EZ#@cXu1+4_cOJO<%P?FfWsPHvC&fL)@#F}RkgPY!uk_E5~g6;mrMWIhyHhYSGbF1G8HHeF41u4 z3eFz*2kNhGTRZg@)wPZBvuL@NMeep_QK}a&S4lgHPY|x`;#K>+a}9hrlI?pKiih?gbHE0b{t##JK#*;n!mjD+J z{DIFwCUWFjP-rt^UmmKa{6?Tqgetet9GzwsxHW>~e}?R5LobeuG40bh?dE$l4= zss3QF|Bw?9x7h-b_l-(dq6r#8$?AXZoUhGTV&PKN)Qh!Qei zu4Rhb;uu@+5@*AdwvBHf3S^LR(B~u^Jn3X6E1K`jk+nAqbl8snH8}(%q!$cy%Ic(L zOyI}|9415%?!MIC`DNIq!-z?<_s>~}Yp3e<+)TbrR=fwGmqSBGl~a(w5uL_Z8$p6& z%--7Vffg>d*pba)S;-Hd>aDNzheHm0CuKxk_Oiz)(Dkj&Y0-SMy6WQOE zL@^{qc7_CDH@9VmINMbDf^nt zWJd(SCs&$7(pnv{w{~r3?dg__jn&mEzbB*x3H@=3k%OaId3#aJ-xVIKYHA;cn3>sx zOe`u6hFpm6uKZ0=D_ZzC9$$zIxWCBjA{HVRte~m>Fb=;>`0KlnmKPLi)}g5Qet9ua zl{bM48UPl@bflKeEqSn7)-;kAPASp<&*mA*$@N4#wc@=@;x8IHi(^1|uV~3G zJbte#U$R|d>&YF&;(Fo65vd9{Fx)@ZP3T#dMwFDWIJ8-^)_s)J3;na4{K{UfSQJvb`oJrf)>w-tTf>+ zHn*VNOe9Q8Z&ZEvu}Hf>+ID-9IDrhZQlVIrhd^+c88H@AR!@*nBHB%B$Q}}H>3k<# zQ_59*X|l$N3pe;A!*}&;lL%=|Pf7|=MgHe{_x$*0dpROV_O`rIPSqA47~%b?x!evR zX>Kn_-`EfZY-U=nEPyiUiVgSAe8ENT-?e@l7p$y$FK#GI?lW zb}%66t6~-{+FK`X5Yj>(qm_wMI;1jUa^KFSqJBR`YH#Y)CAuj02wPH}p?H5_zkOu$ zdEo?6TUxqbxNEMxL?4c$;1xP@3h>ja)&Tg_6?GLwNqfTRy%%j1$)L~^nR&GRs2I5G z)VVulAoX=AYJJJF&plh5N&DnT|gr{ZoX0nPM*HJyv9;flCzVp^7e@>ISXZ?@7Ig?l~x6^ zB2~FPCFLcYHL2jalgvxFl`CLp9)X)TO%QxqnY#)duK!aTZKJvM3bgN5Vil% z^2V|S-f_&@BgB6dXNHKY)OKdx%oX3t$F|3xnDjg?Zi2uSzlx zs~3l66%ahM@;v=fUg2nVd&kM7>I|_ZPu{sx&gXN>?AT zv!r{`8@sDE`1rg4B&EW_NV3ZmOBAnR?U&cL!sSz+SB{lSHsLxw>A1#>REp>vR)4Zx zQqhRZ#x=vrbNsPwbtz|W%7%+bR@j?IV<$N@@RU6yzSEyJw8ziOd3!_GF{t2bc=%m7 zfeYk0>hBo$Hq-lK5!2=(Z$3gkN02RwSu-)3WxLJS3jh!!FyYvp-LBVYu$$|AUrU_4 z=rtT{i~RVy<#Nw@br^$Fuz)`W-^*mPc`-gYNiwyxh77FTzE~qlNm4p0*qRQjtSq8| zv#5$|8JANK{`j%oq8J&ymq|13pJn3NU%2cRR!mzY^LC@+mCu*`Bg3i-!AC-6O@jx%Wujku{cJBQ+iT2bb?_XG?sv51GGWsNMy$=}BaI_^w<9uv9-cvi9-w)^ zE?9*DB1I-Sv{hv=G2_Qq^YWa5uJn2(rc5^Xo~mIZ{w~NS@am37iTeLwUFnh@>ocM^Zd)o%mB!MozTnFiY2f7=Z0hD95agjq(FPDl5N#*AAI0 zxaLNxxLV|H{_QSJI&`RB>mS9cVpdv4u;CWdocLVZnaf!A8B@kY6kAsldQH@pIis4l zXJAa!`iU~LAwAtseJqXTzdZcP%)ApE%2a4oM*D{RSsiIKKB>^(G?)TFR|rL(2VJ$S zz~0Pz1I7K4gxUg&Ex$EKzvN6vWasm7=MAT!)7ZGjwew9)O;k!*j7*GbBhZgO&o}<4!+uNa*nR)w$m304olvZ&s+PBtn zWo^B!>W)dM!;;3J+|AB6_59jE{2&9NblVBra>#ldz!b^_is}#4YLJkqbJk4H!~v+s zQ;D$lFS*=0W%pj8xzS%7XzcD-`zFszAl4E~==g{zr5dvqN1yZ)aif zCE*+V1B%LowX%3*+|x+UE9|%l(@qfd%wWbFv4Pi_PY)`eayXn3^YJ=0)?F{7B*pd@ zQGz0Jsp}iIfX6V-#uL4BH1V=7SiWAWH-V0sch0Sg?F;)P97 zBX6rOQ5h&yciu}m4A3+cRL;(Bh`H+Wdk>WWw_u_Y?DXba*GM7+h(f&Lqk@XoSe};2 zvnmKip;`}Qc${JM^IstF-J69&;MQAoj}+9Cj-=pVM=HH# zad05v*jE+_7d9n+94>4;kJJ5PKf3)Rhz^SdYPVSP#k_4FwA(mivLRFn!?gZ8EI8RE z80?+L_vj~i9<|7RT|~vgu=5Q z^noaigKs;^H%B16m?$_W-oseB8A#)q=I19Qo;oWibQZDdFYG1bPJUb!$op5GV;snj zY3hJ)DHK31{*1P~<#2Y8y`5&8LKBjVo}d@V$*}>*KStsnL;z#}WjS#Nj(&1d z)*`*O=63z}rV}ZLbvG_YMxGd%35&emqkOkX{{(1i=-Gxn*dqqK$@@BeiJYTN0MPYa zkZXP0UlA?jJjT!S1$;Vh6+e(jLzB`42c+u^Wl!Q67Ol%tIFH_+jUg zvaDu#Dk1*8y6lC{P3Eyijm{cXbMc=y-oM-h%CDX_NnJ48NF!}C!0udo<}3YN6n*Oa zGI=e~eeI1Booz_d#I|7H4PS`*D-jYtELx~d@DH}7p|0&?m>E{Uo~^~O$HMn6Rp6m7 zH5Pe!uJZ<9+OBBKK3W92;-sdtIWODBmkZ{;R@GIe{#^*@tn-7wAj6u7SwNw_;r3V1 z!9{`{eohPA3T^d__m|BG%y4IC-^gW7V|s2(hlzvdcq+pY zx-*^(1UK^mcRc%wHu5|`FJdhL$F-#j`|br1n}yV!gL=5;IH^~EIH#)aaLDf7Ab$%; z7Q;DyAf@rRwNX%Urt!a4Xr}WybEC-FSnwVeIl1mXi0QRE0vhRS+znAAP(d+cC}UNx zyF+jb!R)mi*#bYTt~%Bdn6Cn|s$-#F_FD739hGQefKlJ=b3%9S#U=O2uQ&5|S9AFB zU7up0-hyV$)e0Ncm@GVTJ)l2m!S|Xb-&@OM*iaLqvZ_&0FavX0kjWG;eD-QZUiwL}lVayu3c`GVp=rLJmBjm!iH5 zb^3K&1x&})Sg2}4q>Ox{5UV&WJKieYy?}gIl)ow{tpCPwJZcJKdrLjpecDU0$(gVF zEpo()$?}rh@&3-Y3Z}~rbmmhK{cAe+{qqs6vPfyCD`oy>J!DEI&HVHJ%|&O84~L6I zyOGQ4!PZRMSXUG^``&|Wf~|;#&CiGpXeyt{gEAHN!xJ*T`j4w??o3A8J#Say&{fg} zMuo8`Z!gweWhy`8khkB0E?SJj^ZZ!TbJ#LOrkJV6?x6NtttA-5{bRU$2mNhvhx8S9RGxYw76t9=B@~! zYcF*Cff=5KOm*B-xz8L+4g}~|=F=hTt*qhFi zB1!5qrcN|tEGhE87S;p6oH>3A2#cT5IA}WC(kxF}R6_Jlm-YfKrx{5y&37I30bs+9 zwGc|y>fpRdNpHL|-#kqX+^T1<^e7r{sJGruuHW-6ytB4${v%pfjJK(9d7KgOzGnC9 zvSWvI$KqhG88*aK_HItV#V@*2O;+q82OvNIWv-tpR|d@idj=KAIJj!4$R`! zY^O=TY{QA41wH}ZSL!Xmt{pNM&u8B3-{9t&pD)=}=eNkVh?;#l zFL>o4upL#jd-Wqq=xtB>eSfvTd;X0i4{WRZlgWiSAlti4x`A5rH{gi@tT9+B7(flj z02;)2t7T4(c&ae-)c+1p*$|k>zaJ38NQ9wtv)FYc%4;q4Cqd9J)qFWLzLaOD%r{5E zQ1eggA@8OtWZ!zd1#yMIGPrJE%$Zql4n5l_l#1;4!H&AHh8i99L`}90vwyb&|B*HL7)HId0RSx?Vm^` zk7xDq?s?DdB6q%VgS4m$zxwvpaE_;Zhv%ylyV+gXhS#Png*PF1drYcNK+l^}i@_t+ zGs3pKXv#iF)zonKg?F!)DKk#qx z4B~^PJ}JVtox##S_j=-n-DBP9D&gKf_soc=gm9qO4jY0o{{G64=;J>%yYh3zSP%|K zE9;HlANEw*EVcsC>ZXKNEz7(~BmuHKL3=F`2A#d`*51{~C&kOfdE5RPLeT@_!TUc_ zM}ZffT=@K>_fT zs*6bD+4D>e7n+JF=jxTF)Y3GrwCYU>KUs(rDWstpOXfpUPzdtfiUC;YKBe{r@4L@G zTQEP?>Qg;Gn}bPPyJyNt$W4@S>vYv(A>Fm8p`HCE*bot+*S@XQ#|p)=YR(+r_pZ)) zUutt=q-&~Z=>Vp21!8tE6&|Q7m$%@WpVx+8xVQbs-tY4i1g1U^?6p3TDCVFst&2wT zFr?>m>o2C`tC5EOQul{-w=X*cTlrs5!iQWY>{2Xy7wsGV+Ek!L3aU$G41SONEgW7+ zd>xCo*Tu|6l^lo@&8Cm!ba&Ihp+mmRRKKFYz zakQngeio?iC7qDpr+&b6ZP-b@p3}X(9)|!(voj;{WMBw!<9QM|w96Udv-9RpC$}e` zB_^Fw-I!JtdI|Q=J^uEOcQ-$4>aX6qyK(HQmA_Zt)~apWT&ERG5DBY&&6s)>yHURK zg$@&fk@~6h2L?RzBCY3&ySj@}&(U3PeGVv(koZ3%S|bxlrZag{?IFX=4zZLq^e2{k zKgTj+-2jw8`f!Tp%=fKQNf_p@#Pr+Do%FcgRxVzw`P;cbD_@B8m%nhP6!`GKM=1cG zQ8^~oSQCD1$@h-@OD|Por*CWlWeHd@0{eJNR=i#jshu%#H|4|5&5Mj66}d5e;Y@Je zzVMQ^s+=Fq`nH7bJ|?)q@}1)NLsJ{qn-gB>J=Bv}82tR(b$yePl)cW>#FtL&XLaD+ z)!U3y;WJ#CnznCOWms;$z?kpfl(yrVZ{AXJqZX3 z3%!w5W$}-BKbxn8>5uhG#(3Rc8&~@ryTUQBHTN>^48@IY+}!ngzwM@+zFUwQV(MDW z_{5a;YVhpeBEK9=t>YTpKz;Kwh{Y7KH`z@0M(Dlg6!N+Jl9~G8CO)S7Wq8lCjEG=- zHV}w7HycI+2Vi!-9O?mafTja#=V5PP^{@Jp<^)UUXsAAeS8(Q!yuPo^05IbvKY&rf zMsw(iJKgOG_2T~haOm51Tl(g8Is(SWVJI87j|24TrfTYkGl;rCvKy2PXXl~3weqy? zOH4X?h&lzei z^Qrgt&NRKj6vy%0Wk~+$jv^4sB>l9|nS^G6gOLIQbIZ`>t5;(<42&~vf7)Y}c!f^k zJJ(@?o7qngLoXonY%%@rC!6CNwHlg5I&!PlspO`Z%rY=|HIO}n74OE@XVkt}^{&9#qfM5YlO-JFb$H!*_edl<>dVz|? zUV%y~BoDBJ^t|~);=9}>9rgsacC6fceSVDa(ry5w=U*ny4{;mLVmM@UHMZDnf!;ip z`fR8R)UEfkGtE}{UBkE0Tl3}MX&=B7iaLuHrjhP<97QP7Z)C~T|%r)7v2IW}xI zLZD_+zGFzTb1W&Xk1lyO9Qw-15^K($i#wGNMgC60s!uJ#b7M{XSI)MIuujBru?0^D z@9QASmWzj(2eVV#+^BOjk@xw&H9^>7i__6KK7SM!TYvm@-+nQxD+-C7k5Lh-h&orQ zNpB)S@e;$|i9S7W?;_n~2+s{KzDt4+%oI!y+r2llflurgAmV~yIopt!9-ubW!}}LeTho;N6~ZzkEMJ16(={*J zFjTvvq~J-C&kqQKH+BCJ-QAsp1$IkD3=t&GxhQJK>uZ@x5i!Z&imXx!CMGzTht2ZmWXmio(BHD2 zlxk*rnnVt0&#Z}wcLo`S^h-e?$sQ+|cEv$i#Nx{WlZjOgEpzi*e-Z*~Er6!WGy-Y3 z;CkR&5l%mB0)vDeIe7(Rwa=Xpm;MWLUfYH9mDdA*p6(4?U5rnYi#FZ}{hFGj@Ssg& zquC`DhPt_o92)E#s&uD3bU10)eI4Pq0qE!3!I6nkrizkUvV=oo;femk+!N2|&D$|XTE{6-lI)JI4hef<9+Oym09+MQr0llk<1t4IY*2Z zw^_BO*ky;#T+fzgQvG_9!p#QXg%&!``K>u6UtMy{^{<`ovMmzfN3dxEqdpYw5NiW^ z?FWA*M8cY%b#IBjj^26hHH3EF9|lC9&rp>LWI~`pUQs`of?=tplHabGSJW2d{6Cqy zKF+u=5<_r2hLtOYzd}>Z?cElS?-$;6+ny)BU%Z`l+6*kV&lG>MQAkKTeO8|$I4fNr zVy50b0)yl_>I-GZF`wYP#!E);5V)?okocMxWJYVe2n_m@(R`@H-Bddq@rE2R;=*P_ z!!=Fr88C=&C&%i{l47MBL-e3uyEQ=SzPrb(7L`TF_8?6V5DZ(+zdrfEs1wz5c_qoy z@)AskMMdC0{O-Wl8MbWjnxW0CKQZ-;al1i}>~xq7_1(f?kTT>;I{WP&l9;WwCm-Ix zH%hwJFIB@7VSly=v7mq$kD-q2*4l9`vwuNa5T z8VUX==Kx_y;o$uZnAi*pyiGvHM;(Z8J=);UKqU(P2&dQP4g_5ebQHUZv)l{wbP`QQ z?x}xvQhgLN=(uIWE5R8y^y5xPe&{!B&+L>pJZSD@J~X5XOfJHMVn-3r z@R$*_3%mfGelHKZ!9{kG|M}Ed>Fnn^h$d1|DfdiY(e2U=fAt0bz^ZpvCp|r_U%i?f zVo z;waX>vESJ|1R_;ADXaY@#lgVTP6EaYBvn)}Ak{C^kip`;%~dc^B|&8|k|f4h>llGC z)HP=*fWVJQuaQzwvrxhr`mrKpF|Yvepy+XoH4NKrsX#sl*(>S?7_pTebm>dXu$F4> zw);%GaDzZ*R|_f^1`3S7vpmX3?M^2R3^dWfiY=y%tE|~NiMn)&42UPGNU>UT@E#P$ zL*!LPs_t59;9O(at2ogZI1%aJKzzE1x+zGbP4XE~0lHn!AnZ>8H51#ra!+${^qAez z`nKk*bqfoJJ;;6pGVhxH{45U>;Oor^HsLzr>*svrBQy<1GI4V07u6QCf5d;eXg_|< zRBiv5?q#J?K$hd@U%sSpnuO_Z#zl$?=qJTQiY@|rc&uzhRbO_OeWmwm^%r>yf(4Xd zSFWP-y>p$)P5EY9-4ey8qD_<#CJso)Z4!VdNv0u<_U-X%gpbzlM|xpq@HF_hdId6vjFh>+jVF(GFT-pWp}FzTS{q8gAkLp%s`nJ=&Z3YX7)HoYer zCciXl%f`LaNut5ZzGcgpS>62UnkpU+&+%(HxRj_{V0{Y(M!{dM-(>&u_WX=jVh57H zijF$Gh?O#ksx&e+2hC02w@Sd;klQ}_Q68RR_!SxtqmkF!jXgClHAYa6`xt)jeFjY%rjmK!0WP***0^UBgT_W>7 zkzxZP2NSN&z$NeQ* z1UspSm>WeSRUB_(tB)E@ZMn%K7EYJeaH>0RXkqK9Mt-kJLLeg1 zA2CcP`iBFORfod=d!#QGu90 z7!bz?0HS6I>A&n{JXBhvW#1x{E6xISt{KY*zRVPY3@hekB&7NVv zW%gPEyn_t%+5c-}dH)mcqSYx86EbC0%-0)>^@)u84~aW`SZ#|gX+3d`JWX4^>Y{g- zMXa8T@2dqM9XIS{BJlu5=Uwm={dK^q_4L{=r1MjK`3|tl(NvD#JYcXbs3v%%1MX1| zaXmAhfi*vt zK<6nzkknuZF76YcpX7OHDx-%?N}AcyaaI*MykH>9v~Exkd>i=MqJp}$@gZ^}nhTH5 zhepGiG>(kxwN!lklE|TvDP}YE=j`emcpZ-m-yV=y0X`LJu7CzG5gT31iUNQKc-Ow7 zz!3wu!&Pd9{~YH^jbYo%b_cD0@mbxokWX^t2J*XZOpwU0GbVnb%;Z~@IPelPsNe+3 zv2{9^EUv%{Li;pVRlKeCk>B4Ms)#&j^ORRcPg#_e6eiQS<{gt<8N072u5UJll!L2W z+xcc~t!%dc+U)W8#T#)HE@#KpIg(BzONK6G*4E+*?`h0N>?_riTY||n$HKvw37e@C zOpO@#DJCI<0??jeD&kON(PWgVfB%c19W%q~jOdWk-+5-OO;ph$;ylV-`&5R!FR=B9 zsoly#p@Q@5q7XpmUbTGM)ORCC!kguXQh{cHPj938MN3 zZzbAle>ty*SvpBM6KO0s=$T~QZ9W*GHFeI6By}lz z`INrbtSUnr|L*g~)|3UepxerQNhxtq>p6ILCm4x=YJe8=Y2 z2YK|Q_X{G|A;zVqWia@M_w8IZPIA@{Ux+Yy7Me~}q)LS7&E`5e1ric#0l?gsK!E($ z9r3-*4iN;ZJ6>K%7&iCn#K5|t`Zt>`J|yKKOmJW5Bq@TdEMKMne{1tnQ#X%bcmDa; z;y5Y-wF@OtxV%i#5Xa z)6XtgLhhur24Dpy?CW#hr_UKS*Is3T9EahnL6{o$#japds?5TZtrQ+|SyQoOzvpBs{N1<$u0mB7fa$gLHMr3bxXT`jU9gx_rb=6vRA8Ljyuzl|W%NX(r`3aI*}37+shZ6r7{PBbf5H zF}cq1Sb{{r&w&ytTkR+O)-~SRTkCNjbJJw;;G^GarkY$(v~+0Hyz0X%ha{)uR6{&9 zJ2_v8*Yn}e>#jKU^7fAfsH%XVILm`{d&|b=`c7k^h<6UR`R@G+WY`Ip!;DqtpB8Dt z&ir+bTyvRUl;5_U`4f76N0uNKno+)CD*^I=70)BT*Z{yi zFC?UV!6!G?fJuj>vmm3tL2kU#7MI@*?5s4JK|T~I`wwJ#iA%fQP;0B6?9-&T zUJ~Aq5{052bll!@F}%xxo7b54CO5=MB2^Td*04Xj(5FWiCvNc%UX#v6o4r3=;fl_E z(IEb;szEf{ZmxnpN5k1XbA(q`j>J70*Y%64tJ^iib~~2=gS5VQr<>-HZs@3QvQAFd zI%tmm?}j#Kc-}i-OyX#iWy&X{IBMLx1$0VPA;H~+D7qpv$-1lzz?iF>77bu21iXdb zih{tL;@jpjc;T;xEcI+x7Wb?qiX_0k)w$OTOfSR-KJXFlha*yPlBT6^lHBR}DL=K^?$MJq?hq7SMx9NIF78W=-)m z9n3`CKaTlEZpnzA=m;>>hb$U*Eo;DX7lL>4q0IPt{?^k42I`YxQ*FDmg zl|8FKxV5(24~8_0>rH=_g4kuIYxcwRlAQT{A6iQI2CVfP5^rs#ZHGiicB-f| z9O$>CoeT|S^}oE`1#L^}%dU%8C=*N*;^`%S+P5fCB&kxRE&)u`4SCTU$SbD=={q5i_fGbgZHai6-UqqL3RvjW)2~fa-&0Ql!;u&ktnq1FSSbz#TiX3t3+;O;kINIf9bkC2g5m=QW!xJ4$2kYA@<(?w z7%$DFX$Khv4TuT~Fh&kc6Psy^CL`rSIoes9ug8-`kJACIunG!JPVyk2!F|;7l>uWf zyF=?%|4EjB&E~g{D;MHa4Fh0wdEfg^6_3*~a|%yu20{I;UiDK_h|pLSW;Uqi2K>s) zO^}>9Q{nRb`L2gE8G;j^_mrwXA*E8Q8+i4HA}X5_!FvQF7?6@4u0B68%~tDs0lKz9 zy>#o>{9=VseaPP{FTxO{SU#9rEZ9fXNZ3Wx6H4F@cav$zWQ$B`qa%|e$)aV7LTM^x ziXx*UX@0Djw>V`H|LgQxSy&#e_nU-_< z7jy8<*JZBvPXbLqO>M38$<563Uyf~=W72M+tTV#m$3b|3yxGTj|B;x8B4Z#X(+{gH zBXb91JSMjTX<$O)-A4|A21B}nazVgySKnpbxHMP0`drgm*>RmvSzCLRnKaz;>lC$j zhWD_x*5J$|F8;c;=ed=JtJ+`)QI|_ul~R2RLR2efY1rp;J(u}#bc|iV{e&i@c;D|H zTG(V^&B4Qd(<>?)DMtO-|EF-px^%yq)v_pcN84+&<9>W=Z^){D;PATe>~v?(jSC*r zw%~jNDi-1~bi~--g#{7YxWeXiH6yP| zV;>hNdXpqMD$OP@ExkY+=1$VRX{ag;qTMJTP|#bjjd03hEOZ1q3LF>-iNU--?#I$& zOKOyv8F`>xlwsIp1P$#1*iaF?3*4itpQ3iX{FW3Z_122m>pn~0a~1UVbb%Yj1Q1GA zIsy=FHQjFx{*V*!`^S@0&p6%b5pz3ij;6U+vBpYL;Wh@HUj>x3Zr^)+?gV|iE9Y3e)@_#|8>AfMTyViR35ZB_k5Z;PZ@-I}zfNiliT}R?wKN1Oy z?>@GZnvZU4SWV)YNE-%S8i}xqQ9$K+(bL1j?_7uI!K(g&!aFk4Y}=cvm(oHV6+(*^(F%?d=qHUD5zie;Pm zv4%!7m*G3S8Ycer8#s|)v0~}9?)&(RW&3eFl7^3kg%$?^zbg|8A_(vLQ#x^UFsbAE zR+=Xd+t2;EEdq4yq0$!;M+fit^C~7UF;XoHZf!fc*sLrE+wiMF49_JZ>SLO`%=0~H zRo&O=dX1-?sr&Iq6|QCpXMOkymk%#^^rmI@F^}Axw`pVq)EF(bx9I`7{p8aORCCqu z)He0R-kgD5CIQ2fU?YqKcjie<3D<$~vfK6`C&aj>Gd&?2zsoRJbY#z>!15jsY)%qPNcuh zZf^A}^0z2!s`}qDvj8I8^G#>3>Xd>BWgfn)BV$2NsNqxU2jbL91H+u2)B9 zAkp>|H&XNbg5&*}qVrjD8nzuSZPPGYMOB4~50fHSwz%!oDLlBH&ibV$kAZgWz)&}R zoF-$#uc9!xut+AicQ{qF5oM37hW#{`WB;yayX`Jh>2JCFCl#*Pg7$+AR@HSSC57=% zrBXWh=TQR`5?#q>lZ&_CwtPw;=$ePvTzOSm;5KnTG-xI@UGK?Zks zcO4uC3GN4XC%6O%g9Z&SxI=&tAh^2+cL?zAob&Cyb$`H}nyQ)FA9nBE{XDDJv%0z$ zpLh65J<(jP3%LMKWh*RQw*A3>6rys(m@@jp*rX`^c;SV-*+VdOdJ_P2g09rPy@OC6 z^N_ZtBeesX<~msHdX|J`W)S^waBp`gEGnOjZ9Lp@UIM5K)t9f}-N2Y!8QK&P{wIF*qXE^z8XU`topd5cM?$xaW3? zOT@=V{-w2jBVcyLX~(v_a6?i)&iy+lA2MRam_hHYK8Os${kG5bD!`GeKm<@M*r z(k<4epd%!~mIN4wAm_Q6`WbryR^V*Vo;^EzGcEfMgJU_nWxMel+#g1YR0MlFXc^}k zptKbN3u(TQY#HL4@8YahlZ@;5svE_jN(9-n9`naM%8t|+cD$>jE@v@9VWvxp!~uK>KDz&JKz})r6U-rt(s?3fD(%h*)6(* z{R0pdUG^NEiVGU8r1}IvtYrAa;WSR#LCO(iVNysEw}r-K`k396&=>2t#<0^6wR0%4|D2ci&|l*v~h(<5euMCiUo* z*<|D@xNc3&uZ_nU@FEUZ{MmaG5j8LP>;aZhaY*#!IeE>8k(B6#+KQ=Nwuz0Ij2;t_ z+&oZB$NtM*be_7YVCt)C*BH-pMz>$I6Q$!ixfTP4Seacl$ekhkjUB=jR)V^L#x}28 zo1M-ra&4)zs+SgFt}g9X_`4{2U0HUfSq)?3FN2@V@{BZDshEUiHV3lVG3qVA@rRs3 zo3DW&EVHxap-1f*9^}GkysEUE%yPuc2gahPJvLV5d>o=Ou4yuImF)}k)Ym8!&;aTg5Z`)$tz%dXM0m1~*yjPS(C)2S$qJbn&b{bW>bTl7Kh zRFHkTj7(ZTACh=sB6NLJuGJ@NZh8{i@2BL&%8bn2_#N;0_yoGsimfS0JZ}>TW|S`e z2-nM93mdnb>z#pUPjW&cOnEaB2CE}xVqrsOY#9#%}g=E=&aY z+U0nYT=J|>bA6hL!x}moEF<{ARE8S5X+!aGhjVRJ*%FM(u4{90(gTclPkAM!6=hbt zu&c_$1H(2n%i_-fknsgv`#Ia#kCAjfhaRRUd4L5oiagO*Cff!_@|fVSgPe)iU7s6I z%F*wZZ;mZy2MBvur6lM^UdH*}wyvkUx-kvTc;Jn2I4f3HOF<-9LVAs*hLX6(jUCSf)rIItIRb~>I39j8F zBUCRG0fsgWvVC>%09P!h=3QmSh1yI>!GXR@^yjNB;g0KZTESUgltmBLM=71mnnmY( zagn>WS(ozKs~bxB+E!9gG{?N?Nrs8OmHD0dxMKB? z$P*1&gplDhM#VM#E$x;U{!8{xX3i#$DY4f05hxB0CS3UXJ~wXezZRytTC~WG<*n8c zHJQ>C2V->^v^RZnx-~Mwx(2CG)x**l#N$dX2~=xiqQYwy*WaMayr;nsk_6RDf-PLV zQNm?HR%hnx#9xL3#5a!-W?SQJ#7jEs;d0<7@+$s{lb@qM2*Swc^#{K~K!1LwKCKZJ zZb=G(J4*1UK%Cvk5PI479PGQVT1^(>6;@olHXQPl&cu9hi;xJYiR7Ex^zq3edg?mV zGTxK((1*`sQ*K4ov;5wRG#6k^tKeuD;8Y^9)5r*CMnfrn&oi^0{D~IBGO@HAl4_-Y zStBCGMn{~pf#p#^EO}1z4kQAPXQRmJmkRBSE7dNq&>*%vdsFiB2KfwwoJFax@O}GH z3=lXMy{geyM&y+z2d2&zu@dbieEw!sb3ii=`iGXbpuuO_HzWA|ArjrSjW86@B$d-5 zd;t&3AI~^_gttjhrxHo;n4if$z-RlDj?e*>~DID_^|$fAeho-G$x%Z0#0u zPT{1OM{hMpM#i=8cq$dba0~6-#QX8H*cNIB1h8K1G+p{sMGu!^xjiW{iGftBFcB{L z)34A9K`rFGMNq^-ZKY4u^$%iafVzR?QrX<-W z@mJ3L^$vev84U;*4Y6>K{|A_(kX#^2X15h%OwbcFw^~<4TA`>5xHXE-9QGokAmDS;bq?x1hw7BG?@r# ztrziq#BDYES@c280x-b4UPtT0j5A);86_kuOgAFGgG2{ErNhuj+M2-fEh7Ly#0d$ zK~~4Rv2|lgN}7y}H?9gbWH?nQ-VIDd9P5Z0E46uWqE;8P1z59okr7cG5tS_yL7y|Y zD#lXVDz||C`BOXko0x?GOcYLoy&n|k#;7bxUSp5LN%9dqowBLt)BWaXHD5Sq#oy&C z?-xsZ-|`%gE=1{S59udvf(1KHn%C~G^BRawOxS?2L1HT2v#~8KDvnBDstE~7 zQd2e}Ui%UD!fNLi8NW+3FpZiqNx4FVa-V#&5AAoC`&f`M6vqe_5do?vhW-z)gpKrW z+Z~LBbDQNA5}wERb?tSCPSh!%!hkwH4;05~$j@Qs$`IdZNpZ5Tp}mCwPysS0DtNDV zyDyR>!V4n%3%@KhjeD|6L`)vGuWh^z6&1BpIA61g4NAV@60`!%p*Ru=GV@5+9OMze8!*u(+m|9KZ=b{{T7 z{5IbLK*a7Oj=X?l;_!k}m13Ln>f$n1DTqG{Mj8GN{c$05aDDttqx&Lrp3h;C^)(Hd zLhDqfYk*&fR?Ziq-ygj{k-r-5dsAHe6%sD-HPcMcC=3%#$h+QDNh`Q^$hI&Eb>jQB zi&GWSn6CxQEHQR)yrQb)L{RGNpX+6m`YGt=HJ-LapEV*^a>H+jzlin=tfsH%Hfz1N ziyp$I!Op)ePXEjtHsY48iG&02ATRb#%0MH;Y@$mOVm*jpz1qA)cU#cqp zPyh(f56krO5I6#j9m7MFIE@_^nnY{OW>`hlDEcl)eiL~$AR8vei8E-}@Tp+J zM;+o@##7%hX^M@+hhncNy^pLJng&9vQ;4|kZD#^FSu7r;<&5EgbTLY%=wfE#qjz=$ z<)Q*Z_RG)AIJ@yi0>*nP1pktTiAX5}S6b+3|F5U`P4*AzWsfUNprI6CeK$DX2ty_NF3^F6?$8kkICxh|lHH)WWH)^ya{5 zJUTs1xZxD$CXYR9$$4?t}yp-^5LpK78_0M{L>*6!jdTzeVUHnp{SOchQCkXqOl=G1hnPI`W}h$++OY^vA;a#k*_Ha zE!j0)ep~V$eQ7Q@?tfAQZ0gsUYkVIF0o7>l#z1MdxO^Va9I7f__KH|_*Wj@|u9+*W z=;WR@$4s;L{%CO_0rke@gUB?a0cQPc9<=X}25ZsBJ=5*)=k2n@##}>g%YezzSYbmQ z=+o^?9jp+2zonNOioE+^6bnU7d*72_R#I?k_`z~8GtSeT`%={?mxd(Dlw?Qh;d?}G z*kaY{jyowHFr3;*00*>OJ`e?K(&OsFU4L%XRN#eQ-}X01oH%uzRkT}&(o zsX>Nlyj|2BN$yGvx!_rP(YyX!G(;LoJ6Gc&1;TXh{w{WW`OJ)PIj&!^xps8d^yVc(DMH)Xm@n{^r^+!_nHD7LDJ} zbE?3=PLUI>MTc_EN4|P0$~Mg@#5ApDC1N&T$3pet{52)TNjj2R***x|JuZOKJ?^Mw zi#q_NqPhiN5+ej&z>e{_=IIxC-0hk~v!Rpif#F6{+d~CYv-nl{;5)(;v z{|GYHvA8k9hRrw7eo8QrV%;8y{K=Pc+8oO{DPq-?t2T$t4Sk=!y0xar?MXQwru+_^U-o1!$EqIuS=F_OxKNovV$FJ zS+u9_+mwE!gwbvP*bC|A3@d2c{(S_+sjC|L;A(WiXxCKfdp>Pm;2JS-eW}Q+2dyg_ zMN8Xy6E8tMLycP-ypQlUrb9V$zz=Wd||dLt%rIc@&C(^Vii|9J%nHE2G@*3q0$GHY)LF z)@4qN#wLTQL2eVu)i_p9QndJ&3fPzmF7#4 z3cldlY~}Zzi>;5|%tByOrKmfW6}gSClkO z+eiOYCo;S3vwiNRG=7O%@mfS3<<{yyYcB`8uaQF;X(+OdWtLg|>ojwC2&Ua|5Xq@tm~*7;!7D5gkV22veQm(#v?2$*@`(Gky+J}z{C+He{H zdi?%=)7`H)~=(=>Q z@lIb=9)k%8f0Io5!li@L=RGN@sf6nk5e5UyWXjpikMnkD6vI7(-o5laUh!@d^1;U0=hd$yU|U|z=(M4fAl3YWHBcz!%ZbESvjZMvPv1@`!reZI)C-9DYo!7e02eN` zxn2frqijT|?dv-8+=7u4@+UEMBXKC+?4s^%G1U(j%}S z_z$kKr2*ViA$CDh41}Nd4j*P#*V6t8fcu%Rb~oix+EvD=*#8ARnON%!d;y-Xj1(6n z#L8noT;Mn8u=RdENx!q2M(SaEo(w^r)}-p#-)K=0=W;?OIj75pW#9}(CPJ^}(NF#| zWu{D#;cN-Ncg`{?j^{0EVI1;(b&(mcCQTgq-b$eUvVrR3)M6mh4FD-2R;rsC?8P+< z)N6MWlMgSze&^`WxT~qz;_-#^dlkFUtK39auGPckpE;zP*4crvdb(Cjn(17IssSkB zoG~0&^e&j|#1Y6O+F@^$k&*Kq#xWZ=&(}^By!>Dh0<(ICvpBRQwULq_&`1v##(ARz ze|Aw7to+NZM%ZB(LKNVLCWg&S__h5>THn6od6N+kUC>1Z3Ac|IooLMUwxEj%-lrQB z*GAvZDF?}C22pN{(`3pLEFLpZ0zbqS+I+?am7yx$p3i+x4%aI9HDmeibI~0*|E(W= zN&VX=tJ#qflS1qaD78Km%ArVTYo@*rjZ#a)(YwDFvnZ zQ;GY>ytHY9X21`UHz` z+GW4k!f`%w`sKH%%Y-6NW_(Bpm?Gngn zWhx3T^!V8G<-xpD1uh8%h>+(J{AC;)1zCjOiYpbvjspdM)_Pxtm4sj7!$H1 z5Q8P;>==I3ley_xA@eiQ`g1M+5J@B!lh(bB1y$dUUfvBy~b<&6D>(PLVp~GNzBlJ8T4~{Jp@lo(s zHbk&!4v=+zPMJ9Z0)pMa2CkGGd4#t$7K?_L5o>3xyhesT$zS zo4uG%mG-~#tG6X#nY8;Jm@wch_LQA3BZoW&4$wHm9vUF4z2DY8edRrcg{<_ydS+-TpY8jlOEa;^I5wCCQHkZkk z@-5$E)`h9j5Th0L`)V>-nj*1$dm$2IUL1u2--wrpjc@WxO?Y+_uv$;#j2s{cIq?&z z&?jMk+gc~_H-oC)1{)K~PfYyUV-&DKM`zpS%*gq)8g8&oi~`#?Z)S+!tb+{|wyxk2 zY1l@*nZ$(Dvu^G)b9yA_Y%Z(lWR9D@HBA2p#+(vQlA_r|^b!_ii!UX9YLgM*!uZnC z%e-3^%`#M-RY#ZIf=|bK=nw@1t(ORkby$@+99ZU#54CTBS4C9_dD2i! zaut6&vDougVsD?uGKnOX;xK2b?r&NbKr{2~R+T<|@^&ZDhTHrT9qlsyL-2y{Y3q>H z>9LpXH2EOK@3Dw-GoiC3tux?3UsB+2njuGTbmP4Vdp(cr)HAb?-PRyYiZ;vb+#uab zqSI>gRfxJkXhO8W7zy9N=GFB3*Ez90A0@ z^V^alT~VBw3`{e~+oB&8VmLRf^G+~FqKG?6c+Z*C7DCo2QX^4Sks}HunL_)6(c~p7 z-6jxp48f8%zMB!Zr}m3~h+l;zl`<*^JQ?KjNe@_q@4{}*8rdF*W=4}LS|j*whU3-$ z24yCGUr3XonuFXCxwcKQkXk{*4cU=aK5Hb3{7Y6MKDnVUlA(%9)J z@Y3XwGa)zI%43V^`7Fw0y{W^4(6@Q;Fa6f!3L;$-dKdTpG@T$e8A0ad2mC2w<$Nw8 zKC4iQ7_sEy7fDjoF3@O{($Y0qsUw!>6s(-b_re#zuKGDc4142igwBWfxV|)B8{i?u z`|-e72*@{Ln|$bIuX?abL49Hf-JFBL@e)Ir#Z^6p5B(fQ_Oe5^e+#uN+9k3Ic+p56 zk>?KOjT$}H1!gF+#I65qp(&H9So~iKKxJs$6EC(ViV#L_lyUEV_loeTFbt`$;XMwa zG}&K5xt>+xS8njbTQ?^xCw#nOwx(A#9{evGv@GPFm+26{t__qLQ)a!fsREamr_Cv> zO8zFei3nz6#&Y?0`iZ8y*>p;Jnkh;5CDzI9BMtnt$5L%@YzC6w{sQ?+{=_4f>P2*q zsX=d=OFBE3B;ZW{;rK9}K+_VfE(K@(h){40P0!z{96OYr_J97_&-i;A%OZ8jfHQl# zS$wShD9BubfB0nrCfmZ)FWTU{^0CC&fY|*&k=kz;{Z2&(y}$uZ!8!LtR>EgZ;0qsa zhMJXY?+uxN%uGg^-Gt}mK$XkRm7(8B!|^7%X#YJ9fhgh4x{0;b7=msCam5ME6HG;& z;91M6bJln0Txsk?(>L>>_o3|(_D~vdqs$rj^f33fYqW^S=JZcW=)c_DOzQJR{oW-y zdx+~PW#)*ds_U}>d*CgOgD=4Z?-S1%sgLRa*>M3I$Bl}~;L(#(vZn{6G(ipeOj!S% zNnO|VkmDfhW3woNl{kcm`NewtttdUa$vt(l$Q5oZT>3^td}kz3%5m=5;bNI?^@f24 zN_ad`o%FQu+{-G5$)P5MmFU+?TAO7KXY7>QslZCkH{|pGO#d%M(hah?+&?j^e^Egl zzHt4%uN5otY2+dEzE9>muaKyp2Acn$lpl|4pku7Sn9K+tv!olA%y3`XrD?_E=O>T` zr8m~Tt3c%QlvO8^JL1LlGI;fm6ioP@`r`yI09$dzQdAn%RCirQaI9w%jle&~oJCD3 zx0KcWx?=UU|3*{Ka9^&}E~Kl@k0+j*d~?Zv68Km$I3Anycyog7KP!|m0eLz^8A#(by%zeu6jX16ad#pt zd&#>Js&(bG1=EqHw*I%20h%inm#TyFy|SS?&$_|ecMQH{Z}BMDS7JN!C+g;Acl<+- zyEtnG$AjPTqncBk?{(LjxAjlG{nq@J0REbxTsZ9DPRTCd*}osM!}lBhdh$J_S^VD; zqXto}l)JiT1MCV>tK<#L*z04=e4H{~GDJLtV&UfBWZG3eq@8#4v$H$ik|!JFSskB< z1PSHJ7(H80{bp^W5NqHmB$j+x?nuro66t}q%w=>rg?``MX?9O}?Sm-Mui7A&Nc=Z5 z#o%qFvINX(apjJKOHVw>H>*LdyA@!)Ndw0kS8UsXr8|T(maEb%5|Iv3F~NBCbe~mv z8H8FSINvvU(@KBtv)0Cf4Gbo{u0HtLZd8@M^t6e}<;rZ=lhfuFTE3@q!k)pAhPo9@PVnNij6ZvTVB7M)t z{|`h;70rjXDZ0TEdg`#3ugZa6siaCYpxX;*qE_yG+&toc zCp;kwedt)~ZQdx&@-g>ZlDUM?;La^Wtf#8~i;BJEs?wSxZxH}|ha336k4(>iU41>E5a(Tb| zopauM=TASgpWVBAuUfmRS5-xPQj^DeL;eN<0Rc-To2f|Y|K(2~Q|+{Mz;!PVN)?F6Y)0s(;* zK~YBPt5?=xwr3Wjymx2r{_&uVYZIlrXUiXigt!mrucW2rOppZ>1*8`K;v@zShO_?_ zFfnKROB;mFgBZyk91?6HTV6iw>3ipKBcQ6bV%RCke0aFM=;r$=R-$ar`@nzAr>yt} z)TyyDYc{)YiO(o)?dIt|1(}TiZ^@y3m;U|fE#aGx|1+=qqSTVCueo#d;=AJ*@NSDmufa42Dwk3Wz<28cOm z_ZVy_WNDUZ*VsbeNdNX&c3>&_8tUW~^oWFnc?eOr&T-{nLPBqOt_X0F8+QYRPu zZ>I|i1xev$RX)FG>hJ4PCUt@k5@IDBk=Vphcrstvz;?Vs5Ps89s-fUB3of_*AVut4 z)8|c<*~*LL?1fz5#F*e&~peiF-c3!tyn8;b5!MO*4NXzYnd6FgkU zv^tTRsk!ni&k2Of<}(tY*J?Qk=uf-*QKPog;pm#GusI47D5CCU%%2^3R{q#9KFv7@ z?60=tsCt-?bdm)Jba?ipdWlpdx{nx4FmrIl*Elc1MfcQj@Q54^$@9ii)cWN@6$|PT zaUAdwi+gCd`O$9uif1rg*vz&6OKS{N2nZvUOuN%cOG%NB95xNB>dwS8pC_pG_G{@v z9XA*eJ*N2!<-_%MW+-IDk9OTFBuN~#*#D`HFwHmVo_#UqOYFJgKAr`&O_7;X<*Rf) zHZ~ZJ9AE3*S0kZSjG^4#lMfi@3cSLd;ua0kT7u?=^0ohP|oM|-b+CU8-hpRPMRo#1$$ z*ZE#gPHxausGbZ5-4!I)u>I%j`xz3Fg#Qt-Atmr9i-y4rjY&yPzfuu`f?mQ$^hEbp z@{^9*e8W6d-{r^Ad!d|y;|EvgU~BTKdBc$gi@`4joFwTguH$tL>==!Z(S*K8QW~1F z%!A6-0At>9rc?t1UTxk2RW9_pFu}m%3WEjRCHhmhign#_$X~~R2U=vRsF1A!jgrlg z3I`#bMXF6NM#Xn{;DlzO;^AMC+udW4#pO_1ax#{51yiIBR-)bA_^{i34(c&S(GkN~ zCa2p6i^4H4b}13!zIv%?td<}VqbeGV@0P7;X{2Y{Atlx znDz$V+Grpm_Lx@ZO@Gw$q@RrFoi4~T#hm!44E;=A)ywoKg|$li8GJQ{Uu zSz1|VXR4b>2vE?-$+i|Hv*Rfs1_xVU#2uQf{k8XblcckEX#B1A@GNGHLIUs88ilL$ zoJhB0=NVPNHTYwcWwsl;KDek-N{tkSz?lZtwmI8&I>KLmc=j8qgyFfTSmInm%kYA8 z^}Y@2=Iu_~cRGgJe55Z4pv|G(*`>|6blRh++}d|dC77CyicTIoU9zZKX}jd-+E!r* zy54x^e8I9W+xiJqT1&!=dA3yE|o^PSXY`k&fruKAQbf3R?z>3_%X~zBF zVJ~^}eCWIEvu4U?y!Y7C-gb^*3`<4`3-}9>^b^cclO$ud>QRnOFmDWq^L*SU1r=wi zralp!c#V<;YZ-M(wmmpY+~y-SNsC8CTiMRRFkTDCPn=b?$9cBhn}M#IrHaypZgHQ_ zLQZ%sC@|&NQw+TLtJ=>g1>9O`MY}53{E`FP9}!G>2kwO_TUW`f?{EL-B`=g60HByO ze$_;x(1>iMZDL=(B96tDwbs_4_%`$>3eH1~tSc2Pa0>1AZ>Ys{4*3@iaR{x`!9>s{#5bX?h>#M=o@ zYKau)bs+dBiqOfOzOFffg<>Z5vR<0P9R2oD-gG=2ImkK8@3Oozyt7rJ2flV;o@e1T z)+*WEkm*vlvX~?#FaPK(7z=iv>$s=6Ter;qrpRu#abhrB=;$YTw1;zA!P2t;Ajupv z>Z3=BBbx#dXoKHzwq37;o1bif!+JDa0Qm$mssmXHzsT@gW&UbGjHzBBtrXY5V;s-N zBPB<%NKki$p6OcxPJ!6LiRA%TsCIKV(JWtdV(Pr$t(;cv2y9{_PW>OX!xDA60k}H% z=LdH)%J=85({y44q!*D1Y)I^k1RKq@#|^dq<1vXjzbIisD*b=gB(2wdS=iey;7&Bl z7VNPTv>~D&OE!X%grrx}g4+7~<&5>zKn~0)K9G_wRjmP>lu)c-?6V^`KolWD-C3rHn zdD^sxaVLBl?ddu?D)RIR!5F;Qc9B?R;SQ4%2o4yLI-;yx+D4#%c3$a*t?_;C?exoclQ%9G zPXd*C5|>)f7_vpBbrFAJq%binJ>spuUv+SRQI7&w%6heoL)9c z4q64erX}s<&7;;EzK5BQgPi>S12y5#yFL)b$jBSIa0?ab5b)`?x2;#u33ar~3dr(d zJ)MDF-=d~ZkSWfoUuK%w>M|IY5uY_Ut=`aOkBL7rGPtL&$xCeZJ&DQ-%Dlq7`sroD z^@E~sE!9?aL5W}A4HyTkw`ajZkDkv3B(5!NqD;J7rKV2bQ}QlcYoBa!q*j^k_m!!& zsZEH^PoTyvjEReC-2O03Tb!!-%86#lzBIyvKzvq8;kct&IC>O`Zu4G*Np5|P;Bbi_ zyKL#q{r%$s(n_( zs!v?4428feO%20 z`q`J8#!I8+)d;l|&5ypGF_>*KsY)p$JFNK;C(;%VBw!nB4)V-$C4PSFqG+nX{NVE4{1y}yp`w?CUWK1koMTl?Kk%caOpAJ%Qc=nh6>IV9g4WFkrXMjCjqBz)HfjgxzH`lFa z%*{WkqyyMTJ3{(8K{VqNKbAefQk$T^c!zn~DkdV*;pZy`vcO@LsHWGuz;eCqow+au zw2#${yR=NqCy?Y?9zQYc?hUB!d|P^BPOAPVo73YpUPdY%to=GF>Zz&dHR|;-$7cf1 zC#Tm{wFEa* zC(AF@(y6q-tUXdTW|ZplNO9eGXDtifg+0P4&Hz@YM9@>yZr?_^t{nOYIto< zNKJ`iExNTkVb*F29XtN>wkN&uzvUv~o0?s5gdVa89)9tLWSfc$I2`hmR5g z_mv_yBKJLlWYcp<+l}xmP}`HaAN-10*Jd4;jQVLr^zQgcrU%0^1YuNUtOssCCNw&z zd-6cPrfT8LcmnM?xuXNk%ZfaolIs=$NDMDn1H49O{341Sp}EiSw-Qycy$e0=2k?99 z2!jVPhlk$2Tl)McMmL{(1oqI_zJVOCZ_hrreQg(uMq@3d1Ygjz#KQOp^kOrcAe95Tbs9LoIOcj>K!&|9jV zfetD$w;QU5{qH@ukDOE8{jg;BgSvR3C`S3vMEk?N0!cKjZM4|sJ7YiCb9X&l^lnRA zvP+UGkg2N4i+4B(+bw;2_mA9Pysb%L>Q=o`o&PXZr%8boZ@lxe`OEY>iv>cSJ$5PH zK}olM9UF_milDD9wOq)wr0=u+oM+@0Hu|A%b0e0E2~5hwTQ1E%LsW`}&>P)Edg2-6 zC){?Jw0SLnEF~70Y?)o}#8$P?FbyY;rWN+7dcmQC^g8&2j988x%-K3iy-Iz7SV!Dyck^u!Q#EieeH6CJ27t<_5f*QF6T641I?8OwR{|nzZmUmI`fei_ z?Q2U&L+N(WZmW?CI7s%qNT>PYzd2Z6zRoUqUle7$NlP9b$6=WDP7=G5B1>(YkZkc! zNAMmq1x4RTBf9Fk>-YJiK72%F5d~(AwDfKXyr!&)or{Dn;i~_ifWu2Q@{{_=jnPc> zoi`m5CPa`08>}9Rr=zvc`(kc9H;9DJf@9S#l%+-~_Xx*QouU-k1YE~N5t7QWz_sfK>srjBYN4k zfBQ+}zcK$!$mZ7%Q@#1#|C;t&M}hAuOZUHWbL_t^ z5MrP(0f_$hKF&kGd;dQZgp<-V|F?7>YZ!jr|L*=R1bwgO@#-Z>!ibBReTe_x9lgel z4lY}QSpnvhxSj34{UBKM-c$P*3V};AS?Q|ct^_CWZvU(SLUu$?IR2cyk3g!y8U2{< zz7*f|5BIFx&gb@r^|$2#7oF#47?8}!e}j;R!fypQx+GVE4Klh z$H6tIonx?0xIDIMK^$7!V`kGXR~0lFcD?ylvEjtd5jJa#qyrXpczO>L!a3dI@=ISk zCT9j9#&~FnAGjFZ+z$(nR_!?c3)0WoJ16Nx*g1FCJD=fx#}M`S zxH@5#f0_D$b@JS{-v{CYy8PQ=N-8cSm!kGy)gI;;pq=nDG@>u;b_FImIv$37w)oQah7r(!c<@8|+-`RnH;MU5ZnT`^@`qmi&<#j@{u_ z7wgJrae=I`iB8HLL(7ceNMTiD!BTj7q0RRD{(sT*o1hP{dO&CFGWCfm1|?W_SKzE?OX zbHrOE@%ctT1o8CuR>3#tPH-Q3Y~!A@*&(c77=v>)h6R{{twC($_ayyax#to@UJ-+1 zrb%c3o?tzEwG+ZM)+&z6t!h=R=wrGvQK6Y;6kV1gE#uL8c%o?w3%0o(3RqOzlq?a0 zYeb7C+LQKHNm>u3^T2Z%L&-(*x9CkwaDtRWjW-UlGFFB(EiAI z2YVT0b&=vp4W)SYW{IT{?_PaRyX?lUJG$4-iHx0MaEHCh=fm5$D)>PtUB(f^#0pER z?a;7|`-vmK<(WDZ`_L543E4*%WW(Q;!u9P%yJCF*;=G{nKBYraoOh)E5?2z_p$C7d zKICxd@U0bN>}lPE7LT-}aK}0X(BDj3n<_#>0=M=gElrWJRyxL|SDwpfp?x zYd4y*kXLx<15EEr%pe!BZQsm(nN}-;2>p-xzEH)cAC$+URIK%-XVXXdm+cOjZZVX9 z(^c`YuDF}LG$unK0&C0{~Ns(=`SX?6J9De-Y@`a@k&q8g_|L91%Tnpww@!+H&wl zmu1)yCyoLPUHeKtkSY!Ea*sfO`bUdd8~knW|UnBo{7S{GpZL+V5E{ z-V`LI4X*EXvN>*u{}^1$owKVGYl!=(I`^f*IILa>XwM}O)J|nr&0hhiEcLd%+#2rH ze5}Q|begGi@$i37Y zR1Og=Y(MrOFmn^(MtskM?neN22H@*tsJ)v}9-5cp|GuJ7&CeiEHYXn2TyIYGq+$bei27Q3FkQw5m_%N7l$w(}y6 z4lgdx3LJH(Q*q{Nt72(>3-WfgXX_8x${D`WarA5UPlg`GQ&C=Bom)FCCbg7o8dvn6 z2c*ixPG(Y?YQxDB)Eq52_-#2>9HHDT^}5_GB!&te*ACrtmp{*D4~pAj&pmpErUX3@ ztOWX;;5?xsTJ?m+RqYwEY*fXrmk8D(`G#!VRFz9Ln&&fD=|!EQ5;1~e;egCEHLi&3 zfV{i$szr$*NNIbOO_^C%q=An}3?2noL1yB-xF}_#`@A9k&@f_#3JacIHXL-&FYw0U zckbT3+HmLWH{E^D4j0u_Gb~U@H88H3#uTCeL4?R{0~b8htQR%bhTCGee z&dk1Q5idtG?p8t>wdN-LJHe16579nlGss;-AeFff^f8~PhRIvm|B@PRnpU)~%;>nsM=2^SHVUBTh2#n@GisTJ@@ zlNS|;n={nd=e(-WQZ8vX!LZoM5&)%28FR(n*t1^ia}}irh?w_Wf@7S2xniLg2M(oC zj2@;mCf`5@igfq{zQrZ{jv*?zxVe}cGNfKp68$DfR^)$(Thuce(c!+(@>x*cCg+WB z#S!ykNd1{i=$VW>uEJD@2mPa6O~YUY`K-q3+z8xO*!F&uHZ@K_jy?!4_k%h(l0`aY zIBd_3VX+2H2HGwU%{AuV=i|4pg3Qr6=LcAAuDKj*Ar(!66^EIa7KnA2yLs^LfJ&f{ zY+qx-d`)Ywyc`nKFVilK+}6TU%zW)l(OT693Jij+Aw{MPzZl2HVE7pVtaF5#x@f+D zRf(Jgt~c*8GSI3Dn=J_>iba%0-Vy%PEhM9B~9{jjWzAUXps@Z4Rek%>9=N-8_ zdwggC@+xjvzc}|+j)3rQ$(w>lkPiezXQMF~CW<~uSCb#b7Mw*C+t28=8eEX9rEA^o z1&fZ?zaInpuJz1I2K0BTlr!D8eDS`V%iJ&*RL&E5HP~`AHjfgP$$1c5*Vp9->d(H; zA%~V~{t!+46=;v3yi0ka7hNI>Q|Yz+Cg~A0(aW8XgZnX3kp3doMR4e|ZCUF|K_|fS zECkh=g}FLJj%TLlHes4Iyv3G38CEqf-*6MXsSnZ|WGZKu^Fw^`Q5>D5iG<`uEFT1B zREHY!b|N4KBeen|idhk#9OkS0hmqML-`AM{TvMY`+sx8OuUL_H zMZWNU5RA*f_6dOOjkNEhSfhNIFS#xpFql*jN*XiCey0KxN;^6m-Kn`P?k{m;A7%c( zxJs)eqo=!SgxNIWFOx56?QxNVNdg3lV`|@0>~umvba)zmb`ip#0lWpHvKv&C&hBc> zb zZ;a8AO+w>AAJe+8eO7OOr)Fe%e*%C~n_dC|Y4U1Gt67j^Pf(c%{oRmhX;)Z@C&+D? zypKs6)KQ~nX?)E3ulPrKrD>K%uu2_&X4dZabeTar*MDwd?U+Tjvhgda zVrEM1_2m8S+r;bTb096`h@r#E`Q~K(k0@m3KfM4?fAyFtpIFbX?yCTff?HYQHId+t zcBHN-#oMej`t3_^nziL)C(_X`Om5~j#fwL`(-PR$c0g11q${IMt>&AiUU(T6MtPOg7E#jh!U>KHE7$5Qtu{={|AADJxF z*Z*17%8_yUD$4%N^X+HKbmXE9##K5@9)!Yi&zJ$0g)k$=?e2;yvS zp*Fx<_tNJ0tYRk(kNsYgz;|{NWep#w_kT;E7(T+N{#TTAC0yaodH&kS?A zR+1POR!!%l+SF|>PxId?{MLGHxB-r1wu(JA8JO;)5fv05Y=>SYL$pJ2@JD18CmV9g zkN=E(&d-^8)TI9BlpTA>eB=bbkY;q(^*9$P26kv-t;9CV~s)?y_)3tmq;=?wMPs+nz=7dE$Z#0>;c^LzCiJXxDXhj>@Bt(<+sN<7!iqBHu(4*L7~lc?vh1WaF3qtYk+_3-f%d zAdB*->MlL=%GSC_T`C3Ifg&6a-yVNO9^Rmz>qz@EJTC+qe3*U6j4mP8KAlYboHdDe z`IWMu*Td)i55pHCll<5}O4NI7Fe^*0`#hqu1ek)kJs z7&ur|6YRMO^4sY>36CZ%4)H>o0g2u%nx6e5vkc}twhtHA;hB==#8baTb7^jZ2iqiNA%0&U## zCoi;7dLShdJe5^==f-HIe#l%X+FV@Ltm$jFgVLszP}lEK%T}N8SzG_ed8VZ5w1a^P zFxne%U0=bx`9|xCYepxdbx8-Ks&G7NWwz1#{Aq<8UTjo3^(31A(M=|XccCVzYb*f{ z%p{B-1^FhJipb+6xOl-`Y9jIJrLx{g@qT_5vN1NE8oW=C@wx-TsaydkjUlTtb z^d*6bmZB{Pt+tW%-eG@(qQ_xf#qFP|4t_3s2JHW_Dod`a48-!~J#yu*)8DWIOiU>% zuuSkM$}np>%W+|$PHmtxk}SUPly%eOa(bXKzFq5K47yblvQu7sIunRix&mXB#KF)M%C*S%WWWyUo7v5fEghdboKu zVcV~-vmN2m+2${MNZP)~O})QlU{KN+Z&IO^%M6^0J?bJRkG)n~?a>>h$)I5Spluc` z8_$60R?fWfW@Z^Sy8SMyCo`tT+JXx&5C=};m^r0&ED3Gs^9Zz00!1(stVp4dI7-{< zDW8pkQXN_oOdq!7@uryZ4lV5mi_A>;pQqZ~{WDG-1&St$3JIFJdv*uk#t~HQed6+{ zsgh(6=T@iBMp{-9s)Q2v>RYS@zH7wV9~+!go-X zHSm*8euve*-7{Yw$I)Q(w=N%E9kT8T%shD0_vW(} zz?ei*+LL)i+)%$dM1te9_APg{kgEl1{;>!3-ttV6Nrk4dzQ~Bp7JOl)CdHb+Yf&Z| zDv+8EnB8?SJI00BiSdHy)sb2qGTEtCe^l&I?yeaa?70cZmLnj>(@Ae^F+OBd8n?aX zn{<0{TjJ;a=ydo#MkOeTrB~*;Ww3^OFbVj%M*`_tjyof{50AP(JeuLf+jtKK`zTzM zTPw9`i3-{-Y4zzG7?f%HRy?To&DtgMc&gjJ%ONN)@#@P56gX@HGz^Ujs&F8CCHg9R zC$hXF$&Nh+b?H%y6+i*qKkg0P(xL|(YsI5;jh`eVL-e$HM_@HIxPWDnq%_rHUN5m6 zR?wH<^auqqe+e_$@dSTLQdvAHM}FCO!<5@B;H_@o{L8)Bz@m2Xj_sPT7n8=)?+|zG zcHJ;b!CTj3Ou6}*2sW{m7BJ=yzdkeWsz}Q_cbT#6uFMcJ%%FlTM*MgXs}HGvulJK{ zA!}-LD#B`E&#;$HTU)hDU)3wQ!&52ppV&IGGKfS)QG994E3a!<*xWj0DiC~gAp`c= zoHp2Iy)mi{u;)S?H21eM!T21Xzi#~5HNE|`^i?DnXMMbXVXYlo!&7agk>4eGPII*q zeV0s9hE}+1^0=XZDktM{zO?moD}pou7)>rL0>xJtrCU7lD0!g&?or{@$Nrn-^#>ydN->MpDL_QZ$?woo(`Q7iM?oY zqn%vOBg3%~U-(peYHcO!4PSM)XSs4N_FixX3OrsTbUtm8H1Aas`JINet9aIWTVDfVCVz~(wz?~H*k_Od$8kx*Suj8-H4O^N| zeE-fWeHhL;DwS?X;B?8nQkQOsziliYs2K0(>4#~= zHB$WA!iat~e()H`;Pgk$+?I{}VRdw)7A8jO+@KpIf9J`UXZtJmI?d90Fq z+7==JUVUWkcYyQnOrTAb6S8$rIA6L?dyFQt-z`gyd5?0KX0LZRKB5L1oa?P#=k-~N zN)E>xul0JOjBRo_dSg?uxYHR$#)k!VAHlWHYVF4I9vNr91nfCAC=X(Q6*nT|f~tw6 z31eAg-J_l`yzi8$g!o^TZ$7}T&LLp+xa{v~A#<|%2yS%V0TwJjVrV;Y%I%cUwCgHV zuvN9MZM=mnU?EivP9Y zC?yk!p8)QxPey&P5;{}=X3h30d584~=R+fhR`d?%^U7o=kuFrY8OjZ99)eI!Sqx4% zsPqgzF(r1lCmSRI)gzJv4w#XejzD*224pZc1AnwNj7KH4YQIVO zVuoD4?^@-dFq&5^h`|NX6tcj{wFi>0_k{XaHQbk6cND4eZD{mq&A*thQ0&VNv9^-T z1pen>;X6X)l05}Sggp#1Pma=->MwUz3rga=*TBB!zGYWD$u)0%==d38yB5ruxSLZC zjejIb!}Q)>;O0V_<|GKe?mB5$X*dLv(tn;s*m?RU{%%;FKR5O1pelbWS$k8jnUF1q z_H1_=GvF8?_jCmLW}p)^mtiuGgmZrRy6SJG5MOPS#9gPDpS`eSTB9x?^Y(YKmu-10 zZ{rnDb0{$MH>v{+=4;#!trAMS_AV{LYnJ?42rqJ?xhiPCRF3oEeSDKSDY-mgmD3S^ z(*Z@i@5NPra1MF``j5DAZ0Y6Um12D6(lru0Tc)u?k8WK$SfkO` zGjATXv=VkSLxuZA3hf-1viT@qd^t{SM~ zWGxIAr{cd+mG#rjnYymLecv8$PPwb6Jhhi8GWTw?(ui|lRKio0l8%%Hsxxk*$;ImM zRok0_Nq~;&Xe;cDsh`YIzxy_!Q1iyJfs&(9IpQUU$>kFi#q#$y?@B>6w>I|euvByV z+%@LIl|A|V?b&j-YKEM^gsjV8=w>Qz$bwII-2-y*68gBCm0{W4%#ikcXU6BDWy>z_ z$)7;FfyZGkL!(oaYIq7Pr&y24xT0f!mfKMBkN8gdPIk<*AN1W$R%8K2xL%Ja8OuYU z68!#a%ir^_i$`@0$|Uz*ReIg0B(`(!JO6<(pJjEV{N`GdJ`8*gcWOPAgt1?KLJ&=4@;j)~ z@!LYgudChun~;pEd*-4iY}`4cnA3J=zHHr*s1Lf_d@HP1Jk$K)={swCmQcFN=9$;| z&5&9}&1r0_Ayn7@CnEXExIdEi5&T`Q4?K`wTq-38{%-YhnG+y_w>I=x%Vr9 zcVciJ)zx>K3nwujI&j_UH~kb4i?%@+2NnTtjU{ScdJ zFZ{h*@u$L~ZJ0scT3!9rVjNI@t}K`Nr)v2n$x$v+6B*w4BRi5@(&&#{pyu{CWF=il zS^dzIePR6XW`%}~bDg`U9n!=({ahxgRhvI60oCx%ONtI8YjTM@_BBaHpC*5}NJ67g ztb-49{9;$C!MBxXTjBQPdrt8VCoHpdd3jF&E90{4q>;Ho@T|-1MTKpufF4h1N;qx8 z=iuz;eQmVU+xS&lEqUYNy7Gp@=%sgl`c4i^j>GQtX{|&%A8AKFl3a`pQcoiaYi0@* z*S~I?X`iKlr9BG;F$Z9ma5~At2T6<~!i*>%@Vd3e>rdhu9-K1FiyJk40 zGa&uMv-QRDi;J7WUrLeBex0`jC;fHaRH9?g%9}XiM)k3er?9@G%k|T4k#38P;tXka zZ?3YpE5dAVpG69Dq@P`=?QEhD;u6GtuTLxV9X1-Oys-`h?LQ2!HEMZ@shhIz^K79A zkvLCgN_NRQ=B(5=$e7z?BB-h3)Uxvq9RVTLcn$t`y$rM#OwI4x@^>!GStF_}hYhvF zzG>(!Pd7K5OAMfx)%lelrK4r`-<->UMK)h5nA|g^qo!7iVzoc$90k?g9ZMIDEBw`( zRh3dga5e{f|H<`fc&hBR&ZZk8U}ZA(_)b=n?dJMnAGvX=S2CMH-tg8ODW~rmq@1{x z8hd9$?=o!!%h|gKkao2~6<|DO;%u4!Z-T01<0L9}i&8YEnuA^ao5hVwL62q>YgMtB zcHJW%f&ToQOl8LPgGw1OPCirqrzTCA={Sey?ZPhUvVmzgnhKDhi#6T6n}6{Z4vRcm zRxoGrXvtT*?^^2ZYJJ=(wI9^@i~FRBH|v;w5TD{&?eUmzm|e{}LZPw#;WW>1F5!z% z65Pj0)#c=cZv9GM>JI4o4XBOzI{(y|_?JL5|m4^1V9ox~HhsLdZ3 zkL?DoO2`gqndlJ-$0=Sv-rGIED{gpj~e>*=Q3s^^@}b_-UAS9|`F zo376uB&{(~an%U&B*n(pFG2ktV|Td9lFr=s_KIU|v-sxNj1QxGx@*dvdWjT)cl1za7Xw?mMgRJ#9JcoZHLbGoI7#cc8yU zkBjCs+}JtVFyE1bMIn$>?#3Q}XDkenNPe~vb7v*tf_BRyaYmic(NR7KG}UyxliK8_Nqh20h*T@nX)9Tj$fzhIf~W+SDX>i9o7Q`gjx5h=N>9g8OM(Q0XB*I1-&LX8 z2n0?8RiBP*q3X+7cc9H>de>whY~>v7BGDKPU{`kf48$H-ylFE19w&Pjr32N*a=fmEo`%im=r<&0lMh=YO$~OJ_ze&E7ESkXAF|y*j zKq>rKiIt|xTFONvNhrpeSNO5rYESXlu}lmN%hY@*XBYWBhhV^MW|q*6<1M(;Ff!m& zts~$9cBitg?Du4;{9Zn1WAarO#`6=6w-##f%^EoHaoyN!jn?n3on+$o_f>BJ?dKKd zpd;#9F!eUv_>s=|c1+H}=U}hN@~EX~fsq!-O#L7zaa#phJe`03%hGr&?7ZfAJEScU zVKI5;BP^ZZEvIfOz~>-?^a#M~nGyiVOO&XLA&+_-JS z-~NC<&Fhl5yn<(Z!*5Wa}aH}UA-^v3;m{ya~x?Hm7h_ORne>c-_u z8$Y&=k+n*9{?!xLzd7x~beU?!ipA3ieixZ*n~u*rjyjzI{$nLhqm27Gi;1$$UaaC| zo~CD;1Jh#1y4`eERvLLV8E9f`0^jI>0rMS&Of@?1lzezhK{BXbzx5mJxXDzrD*UH5z zKQ^Q&7vS5;QGYgT{(J1h_&(yN7^c*Vl(G}3el&p}ywTr9^J{x5iQjf++b=c0 z&wdIKNg<1aw$EnU*G2Z5lWb77b-go!-c#3|9gcE+s8lucNh)AnyYb9z{f&d)aHQ@Q zvYLLkJA`J_{6=hKLNn4P5#gv@63z~K{n}nMQT535>4{BpyHS2@BUu@%e%(e4u6s8& z$G+vRzS0-w;b6ts`n@lN`!i3+pi<;k@)!D1y5qaa#|2nmT%)l+io`356l3sr)G8Rx z4dph!-TaQHs_4PENMeQ5t4!^D@kc<1*Yk9gaRb`(F{B9?n;I$MCtkIZ(YnkXf5&t` zp5H7{FamXQud}%0k?u<@cXQ}$0Imy<3r7Y}fhsR*Z#H5t253;dM>8x();||a14j2} z{q_@cv-q~v^Gwh2&C>(1xuxH7`zGMtJtzNE_=YXob&)UH%v=3ET;htX?9+}hlTmn< zP^(Z|_K!-3YhNGzoKcRpHA2zwaWz-ZMaw>@zebOd-gH9m`{7NncfBh^I!dINW>j6T zU|e*asT)bF=p4Q~itV-bXKgHBs|V!A$$U(=u`xt-=?_-hLPwLPF_&3#0CEg5%{sde zv^)xLd1RV^c=A?n%?=FT91bv|Wt0A$!|Hx4rgI0E-+ZnyJeO05VBq=5CYD4{-Bp=L zZ*tH#dtOq#_;LseBBuISfZr!GMbV4xW^%(`meHiQYO`p^KNDdC@?Ywx5K^y;T@24m zl>mHN_KULXANd%s8j&@1CB0J`Cu~M1{?VglBoxQsNP><7T7NYTVU$ zAkz2&F{e%>m@}VpiT_Q0=emF3V`Pb9(2rnVVyW~_%mq;O(eeh#(ye{U8I73#+kQ4; zxFYp~_!@kS%gbxrnYk8Qj2?@Ml_w>H8lVRN!KsK%!3UN3xiwh>F?+~gOcs< z0^(P@Kz3jli(Gyhh(BMxafC8n)aGq-wl<>l&Pk*6FXedA9Kws2v@GZyZ}-j&w*wBa zk1+8@GX*0%BTxO?nv$!cm*@@Z9M8t7`oWWqpAw;~yX@x9ka?Fa5h{uJNu3q+^3sVwB_K_tX z^7feXx3mjd8i=5W?U!DJ{rhBC8G-g}N@u5LY>X6rhTSxuy(8}8WZ2R9Tx@Pc`&hbB zc2SWUZswUyQOryAn(DcE=tTx<+yOCEFxUy~G13G3YF(;8A?$6b*V}fc?U9R=Jtg6| zcQjaZ$X~Xf`Uo>p;OCm##uqSA>FE#F&1nxFzp+%2REPTrm)xj=1)kx}9cEsYENkZUoq&&>@7fjWQfo=@SG^X$B!uR>P$Pa>5gFeJ~_AUh=9vVHqmDegkh#m zfAo-@uZdhSk`eX1joo5&!-MjCM^WH$T$btjCWfhPyT^E=92_{#<=C0c)%jff@L(x; zI@x(R5;52pI2g9(j16RJeCZTaZ!|UEv8x8Qz6AhTe+sa5Wa~>f^WIz=4Q8%gc(UJE zExJNrclS~e^+$uV?uFUI+znHkEP@u)ub_T%x~+m&tSl}Qdq zYH@Wx!OQMCah{T@9vUKon2FZF0qyy20=omocT09LhIA zGTO@ zWBqbf+sRZ`9D!%CEcT`#!SY%=(8Z%-kg~8NyTn;bjnc&TX0PC-Zd8&HcL^XUCtn=> zCHFCa+RF)O3zeRQl6RLJ7NORhKK1h369sifc6+v`bJoNzUxVfFmBrCesPX~;1%(_acCsS9>4q+b`q8*^u_Ar866gN3B_ z&X$TUX$-O>Q~zW|6t8f{4;+v;r+AuT8kjh&DVQ)-cyAqaj4Yi2Vd)kQ)LpD{Tch-% zqVmMWp?kIamXV!Kh`WTy&C=N@iVxS_%FI4Fmw1OO==JISue-M0W}YjY9WQkftuJ(1 z`XhRV&O#*Mst5lW%CoE9U${9MP2&L2CqaM9aN0W%_|3RqAGs7 zXkT~TTf%GKwM&JyBM%Wf(#x6R5%`kI6a?x2+z(9Jc=%ZHx&2{XS2p?LL*ZQQ2LP6N zr+P2?;~XI${q5DlLH`M}@-IO_5--JA68)zInuMS zHKO=|G)1IB%04Nsmnuc#85rJ@J;RkvAZHzqp)fa{OuB)t?X{c9?{6BvCe{BpToU6X zqGxaGFq6g+Bq0U%S`28(p6d9Zc26hNx7|cy`hyaJY55r7F9r$tzIO(QEGY@3b?(s8bbI2(KvFoDmm(1ohQ;~CD)|=0S)!V0|Bm~4 zDG)L!*B-nDkzuh8XRx-;5 z2;L0OZ9?8($=rwuc!QM1)9t8;LTx2vsq@pmTH#Hil1MPv#hA{j9A+YJ{g53@t=^t0 zYGret_sn!KZBMMi=&JhOVAbH|@PsxCvPJp5Q0L10&D|u#XXdTfnU5NS zVO}1CX4gR#zFEze ztzi|Z`Zd+&>d1siB+}O+#E`Z(3k?ZXe9|9j8{=5h&hZLj^s$EH)Fn`q45QTFxS;o5 zYrlzTYP?*kVk4KiZ@P5OmslQ_;ut%?|1kg$NY zG2?3YPL~qAl@P?AfTYh(^0>+Xcz?wdjQHkD?D&El8uvBiZXG0Ql4c6tU_!;| zq1RtVaj%Zxy85KLTxp~L-BVb#5A~;^)%%-jBs=URZ#W)Zpdq}@mxKg zY+LgWj&wVs^UZZgR^=v1oX@rFT-EnN^3A_8ZS|$;OQHUfevByqHQ#?9(>do^XdllMRM8vWsyUrbBS;jn)pIY2I7~1a62)pel2NWV$)|!ZN4FH43tw@)o}ePEJ9lzGhQ9GDLowWi3Me{#0qe{7b~Ou@k3~Xt@R=IkIpW1 zh&C1CM*2cZ6Wj7NYTx^XYTpFhk0-ei3%0@66<@POen`U{#AeJOOZ8n^5HSt%a zB_V#UUAq!@Z-aXHCxG4SEzrc}#Lc158wr28Dc>uSx*RV9;KhCu%)QZZ?&o;m+2R~? zE+Pp4b~^=izda4f<)4?8-TVdmQS9yQ{eJC(FDu(dg?U;R;_pEf`jE1zTb@~2K${AP zKgzo_Fa>nSef+e^4zLzdehsQ}_qR(+-S29w=bv}`-GWaK9i9SF0I}MD^(d}%Yc(aZ zczbJF*~IV+or^G1d*sC?8|Y%t9(CaGAh&9kNF>k9j^5ro^4@QDYhA7BV>la$O_v^N zD4M{#1TLOIui|x@Ha%oe{R{*s_+I%B!Qsni9XW63o%OUaR*ryJ^}@`?O#|~NJ`F@t z`?W$_S0;nrGzD~mbunhN(lS9}bA*H5W%Ax5Le=|(3j!%bkK|+8IR(0vl}cKW@{o+8 zOjSYEIC-6QVSc^`valB5CyQ2l082GL9|`@a)|kh$PvzZ|-h_~ggwE|79g;gcoiq(M zT$iRQmN;oLTnQQcejLiotVIkZe%aDN`dZ$hN_l~I3>yvqC}(67^6#Jt={m2O`qx*N z1i({8d?FLm?Tmm5@w#5YXw&hMTkyW7U;qoj6?HSQUi|^LJLewn$RGB09$@_+i;A(` zX4CBrNAu60X(MuACyWyenT7&dOLdleeF@p3THuohgvZUlCi!0PY5mUfWM~&tO5lP} zJ%K0tle0Ob)Ctz`eZ>S8*4Kx4^7oz^zOYBf#o!jSSKDSv=@~{sni!%Q}5z)g*qm+UDWwY5wMY}P~&k-xa9L#Lww?Vj!Wxks?!=c`42&i zVd)RvvJ~60$_xn$m-nKDl^(#vc%L+*+Ler+&!9;v%^CWi;Xs;9jbKftT}kJ6|UO4*YkLo=c6qz_H()q^kE_Bl#?Ko7o;ideFJg&0B z%7zUaoB8Fk3{VTs%~`R`ba|`fx?}ZwrT1SZM>_w<6-BT6@6OP#5;wcp2O zrr5fn>c^{Jo5d=NuCIUoi*XJF!O9ay0g4TjZl?r>)_e0xyHc9DzmGZbjxK@Y2&_RC zdN^wif)@teB1+VQ{cl6l*w_Q)r-w+ZGXrz_7da-J>uL^LA+de;R%bZb$DET$ z1cDj`(4%jf_k@_Ypb?lpeSE$kwEPR$oYz()F&E1ijYtn7hP2AIUolVGn z?}nKekz0s%yND$EKjw{p-CgqDM!_*b5s9FxirD_v6$=R-RJZY^QAw<=(TeY$$43$b zTY$FhWT}sehb-T+D1h;_>8V=E%_0=!U?6jn=-=J=(GU%_lUx|n+5+W&(zoYClTxZg zUmUSt%VZWMO7Qg>RT=E;;GAtN3&KV97Jv$J><&gD`hIp-HUn#KU5oobwC!8IF6r`B zsktmOW8E2j>b=a|7BrD6@zB;`!H+`|=TKYVmM59^z+Hrmm0l5^(A`4y)m=MKM9N1> zu<-{;-hx{+Z3TKB1LGNme1gu4MwG`tG(G!Ags;&*^lzvJ)F5aS6|p$S9)iA6}b+mBzFZ5cZYb~BW? z%#uN9oXE4}7&Fa|i4OfPx1;j!wE)<~=EQh8AfM6fp{6|rOBrYiFLm;Y4Gm(r^ux`( zOneB|xy2f}#=36(Sebl}I8pT#*3_**2+vdO$fuR_j zF&EaDFsjL~=zpt7!!NaEus@6Evma79cfT5$3Ag{_+?g`({NXA z{I;Jh#h>meTZk7IeZG4Win#6%GT3+K8Fp+|^%SH3Wpbrn1n)E+5=kWd@+L8$-u2q} zoCBwU9U)HFA;7)iqjCT9B;xYBAkTtofX8qy+x8IzH9A@{PY$lY$R1I<&Xq?Bn1`Q_-K7Sl63Pxey0?M)1I>p{M2{OkKKf?GjtX$dRg;`7`9vxW@ zlHu#I|6btaO{>~f8bU_nYZ+=}v5(V8e&N=HXELOj6ZkiHfk`zLo{3UV%+M`AnHQUx zg%1$ZN&l|;n3(D?oL_EtYzsxXP!8${9;wm~elY<(={_1`*gOlCyXVe~-pa+iXy)vXOl~m3 zwaMD^1WK0+MFvuo`6iYrOT>ewcQu{AG zVeja$^CwZDagkT^XO;L}xYYXH6<)#3&^!a7ZT$6OL9fA{F>O4iaZY0-?7rj(KU+@g z$}+J6Rlaufm#c|Ot0RT1=P_^V1M#lfP;avHxkPxnrH6qtGykxRwH>gY*O3u_b}^Kz zZAyAA|F!GK$?Xx4rfckXp^X8GJftN3l{Xe1fPM20(YWxV$&DN=6(gW5uv$7u!xt2G zVy~RdR6m1Y**xrO&_8P5kpGkEe3{vA4PGe2q#R+#8&}?Du+s|0%qAHy@yl@yVZ5wt zG}}Wc$yTfCYT1LBDqY^8QI78pi(EnlUW!~^;oN+x%TMoo5|z;lGRYlLU!{I7ZZ&ZubCYh}1~wf%QO2 zkHxjF?1LmrqGm`I39l-`Z-c-*%>TB{#~5-OdyEf?IM0e*%h#1$P%Bvdx1ir{NNnw= z+Fk&k24jJ;Rt^I>tr#YEaPesurHjb^Qaw#^tODPcZGfqWHdG*R19u9xW7S+*2mfX4&Cq98-gcpuRJ~i7SZX51>iz`i5slbO|7-f%WYYu) zP1q3V*890_ee~c z9(S_a+tetKr=PKws_0TQ`gu&kR$4YO5ep3Z-+1LUk(W=dPV}b0{C>4M4#@IJUmLKW z&Fh$b%P;ffvwELz#eLk-&w=!g#Zm-^&Hyfn$5+_8UatH-srbd0W96ljf?{X7uM(E< z|5Artk?X#@t*gD+nAb2s$In7k*HR5z2*aC6v8*0#{w+dne+H=+b9Mc4?SrVe{xj1h z8X*)oyZBWggtqsS@yfYJgzG}IFSSE{UzDw^sa=@=&BbM_yixYs%fwSlEt*<77 zBfuo}-7ebkj+jZY$OFn9c1<&2)%j}M+jlkJ0ctJ7ZO8WqZ1T)GrUL8zDYclHkN^G#oISY<(o28AVa`9o zuWy$HkZ$LfC&pA2F+8!!Xbz#|F$k`4>7Htzd(mkXDq8K}cg^!2=cr7PE~2n{CwvD) zsS|SYh9}BEK~;7V*n1-QLauo$f*hL8&Q=$GCVK3y+!Trb-Fv>PNY_fT+DvkuJM=cx63#}!cheUccUlYucH$~p7U*qZ#5Q7=L?RwC1&wnm9qtfH zRlL7k(l;6@05-!hoc{*P5{UJci5g9IUSp18~oVYo4si z5-6G5xa*n~fCKm@jVtz($_3K;&f$XzW<2y7y{}Ute!>MwKG#88cvywIm*km!`2|^e z9ejkzKg**qzhsCRkKJQn3(~&!aB8)mM>o>-vH-xaX6TzSw)Rj!4?*mFddP$X1%_&i z=GJXU%;{z?Y4!fj4K67&54;bs>tAG2B44 zQa3PldguSlJe3AC(V+>9Z~8IFtogSah`1sr74pPzw!fNXH1Sz?%m^Q?%1~;o)Yv163)fHVEZX>N%w3Z8?+MJC>ky&Z zT58sds-Ym3vgAi)&6KPffl7d5Ox=R2m>VChOZ~O_ig;fsa??BaHyF;4b#N!1e-nA# z($()$IpYG6EBAF?%l0X;D)AWaSdagSAp}x}LQAjrlcZPMB{HsgC%mh27(&=w@>a_O z%Ct4f^oeRNyC!2;ZADV2DX_w-vkdVmpQ=TFW7Wt|tX-BE1BhqA zzN1O{%)MR~kzk~SY;ENzH)N1q;>dITIFfu4J%kB?kArXdR`w~PYxQIeYXi`ZRdp2? z+m%Pc6gonq4YwMNU{#KKeFY2cRH*`8w@}zRWYO#9^l7S`NCf9 zc+^CO>mse5QnoG@*SezHA&|}NG#~_j;pkIks~!`z#-w_>uT;s^gapIw)Ob`06|!c$ zwP(?dwx(&MDVwd$ICx@{Y)7ru+J|x;AMp$ANj*i{2k!0d4L(o3>N9!KkmY_(dt&9q zo;5FAiL$xy@aMw$m{TmAIrTS`Tc@jIkpV8inP6m^9?JtFopyQ6>z`$oZRGGx!8O4` z1=n6*Fy;8UJPAu_tArPk@927}j6UB30|J7p7rxVQi_G9;J&403+hQVIXt~Cx{n2t38t$hG&pmv!a zTbC!g!o7>zos|U1*^|WvguqN@!;sb=62ZVj)YavRO-U=$^vM*03S^}uP~uTMHU8tE3Q z+E+lTw|_8(ZGNk(B1{Wk^vu#j&;+3#3io$2#hTzci8giusgu7v>R`O!!p_6c1V^#3al&*!4ZU*sWH5ROi&%f#RRA=Hre|gz29UD+-pgQxHJ$Keim*s92J*yroIym z@-Jm$YFQX(naQL5;!MO9|tW3_?%2ZhQgv#WncI? z^puOm{?j4CD3@T%LsH;_c=FB>XB8@mh}xT*gemNFvvZ7*O-Snu#gUB{3=yt@;_DTd z{Cq~QB8HF@w*+amlFKd*3i=!@Lukec&GwxFrifO^RZ_0ct)@++ix3-Zow!PD!2wD-jOxiTK7>%WpyX{PX; zvEbA--|q>sc3sJ|cE$&yqAR{AHNiVjQ#fnF}1%jh1=GZmzhP^${`pL=Sp@sf}8?9?%*=lCS&z zV?iCQ+${0wunHmVN`K@wL z%OgaiHd7LLORsJPIX;G)0vk3DMw!WnL93F6B#*`t>feZ`ATJOkv(e@7PK%bgy#(Sm zjNS)MkavryPgF9UIwJNXUezt^CE!1QTWoxbm$DhvzrcT4Pr9Hl(Gm7s3)FG>w;*e7 zJ#;8a7OkF01nw#z|2+ZFyKKL2WlhSbwIZIYcQ2_mgVW9O3_ab5;0dcQ_pP6T0^dL> z_RPH=Pgc2@onyoyIY_#7E;8{8nqKpGC#et<@W(2THW&Pawm7r}&i`NyhXNk@fhK=|4FYckk+V#_w8&DsN+)7vx{9wfVO(p z$eC!%vikukP6k2umE{wctrh*mWuJP?=7cLClEsnScJQALZgpbv*jc#=h3?iBARa4&Eq8Ec%M?+fnVFL@;|~-&vh+Cwv&A+Ma~cl+A^)f82W1SgEj<0{7KDftvhpPn=L!#W$(V>tMCOQ?jNSbw>syN z`_jMfK(+y%O`|`0`7$nCGzyt+Vo?3!FQY$@y6-4V6)v6#Z?|5tfO=l*b zi1+q|{4$2x1CoJlj#e-HNM3b+QPzr*aC1`n#^Ry=sP220+mP8qVmtSOKB6fUg zWpW?zUN~(3Z@T^%1I_NcRp6#D+6_x!vhZ+?<1MsyDYBb{fB`}|Z14S(&e-1UFs{P= z!%xn(tX~E7PUSv zbnlTYzIYug5f&;?rwhDM))O^fH$+T42Uv~yKDq!8digF6G#-%`57ZcjpX@mv4UC!Z zskdIq|1qt0i%n3_#9P7*<}~@IH|&CZh*LPmRn}-ba0$#osU)VSj`fW>yH`uaP}J@52>FmQWTo_BH6F4T|T^?0aZnwn@&ZY*Inwv{T8aR$~%u zz_9Zy-Nq5AYK$vF`$CHhQ(i(jb9?;cO&voVmk`BFUXKUI0!#jlUeY&3c-XTxIquc_ zx^w5&f(W5#FSNZ&vx-U^i$ zd_VmV9v+FDhK6?%N2M`45SbIyG`JqO`-+gbh&DvB1+5-+0zDa#$^NZ+y%icP;QR)Id zUPRh$!l@bP6f0Gdy72yFOQ$460f9zS1=&L(i!NBZ&B8O36cZHKFRL0|KC$c;47@7j zVT2#w+7Ws%1zs2y8|!zLTnGXDBSjYi|3tH^NXL=)>3a4@sS45VC*maPY(FP=Bfh0D z)b^$k8OtD&;lJN9_{MY}wZ#qx5rpZ)O({0qq)ZWg0lXrf7v#DvbHD!~%znY-2) zn=A=-Wb68>$<)0@jwXuSykmPQdQ2ixE92$Lrg`pJ+Phc&_tZW##-f_5-JoPes9W$1N6zoQR|w?F)g zTV}giF<@3H+)NxmuR_&6@kH!MGd_*R>a)90Z3N~cIZOg}j}ld{h@lAs4%UEAP`6?% zhfG5IC}w0p3p+_FYFF~0^;eT?@3ZEB(5o5ftX9y$KayvMgwNHp0gO4tAe=LmYs~~k zU$YJWQs}9Jw2$ppuTJ;23fcvZ(|JagbF(IvuMl*tPCHN}Z6E&rheGdCD8#(K5=mZh znm}~Sm))43I+ZdQvrnRp#ygV1o?<)|(b&*T8wu-2lDQ$mv z7Ecj{kjy@Yc^9Q`d`Zy2^rMdshgT}1={^mTzx+~Bm?PKnPX^zF-4-#1myv-22X2!7 z)Cm*$HTL14gYB~HfPJl3U%5*~3^C(MR8#KQNEjwy!TFaZ3+XRWlh>MB-C_m3(b$Dl zxd=z|FXr}ND?JL~O$dzp%Z9G`lSG*}ME_!Lqkb|CokggKX>2S@#3x!4xG27GUgNDAOEc_r&}V;>tZK18+ol@=KI}U=7#rGX z+ThI^bmNEJI7Bc1R%4a*vw;f!lByBIa*ENI_X`e~@v@Mq9b}egjE^@Gp;57^7hs2x zGMU#-z3*AO$c4{P6kgXI*(SI>j`jL*>!4ep9IoBI)Xf0rI_J7ZS2gC=#VoXJ_M;7G z_UtIA2W2)!Tbyz%Ox=1H+PxnTaC533Ap?)%3?c|fY8DEB$Ci2R!+qL@6P#FLsmh0i z;{o?AT}KJMA$@s^Z+-Q6-FfLAT{(^V$BA}sn)MW5I6@F`LOd8e$TXo3Ch*^D{s_9Q zB}6kmn}6{)(p_jsq-1ihFGK2=%H z(vfA_`-Ta0-=}rB?E{_{fG?~emtfLvcYY+c-Xa$N`0l6nYL}lH6rY$K z%qB{w+k)@w;ZYS|RR`QRGOwy%7 z8^i`Smkfm!BaCE^6}N11_4yJbX_eVF&SXZ``JD5{p{6*b?ZMn-`9_;1ea^UQ3-7h= z9kFJ~c`2v*M1sLpV1dSwafYmy2FCt+%Vn!eONUph2qM1cI%k3pECH#hah*E*jS6`# zQhWxgolx7Sq*F@C2QuiLI>BZEO+AM%wSEOu3ECYePP87gYR)+d6rVd;c!S`7DT9a5 zo);1}PPhqrc;BklZtPuGM#l7Um}+B*0o)MfbZuo%Ated!zC;_eA*LyyGTjRisrXlFC0U$2L=0B3dGi>R8FjFavj) z)^f1ZnTq)|+|vb7!}`xBdwn48z^daAlW4zM&q*1|LgQM0hl$m^hjx}N0wjL7_j zB1T+x#aXI^)WZq%>&%DOuOO|in61_VRJ?2|uJsstk<1Ua?vKfClWRQ3Z6e1k$<7CC z!7JQ=;Y;_|uK`bWKb_@@Jh9R2ssE>-fd1|C&A+`&jSLkw4kh=AYOTS}3 zMv~=X7M!eb%vh5DMQS=r!I??|Hr;#3s0zzyM2z!W;q=T!tD-e{xSQYtMLd ztrc)jw)qss?R1q}>ly!?vN4&pQkjPVVFd+%6d3YjP`bI?c!_vkBNWNUbixWGiD}0L z?z=7-EN`uc@k$VXqI5JOA+7)@yy_>y%`mHI7m!pVLAhc3~}P^@@$-XWVl z9-m=_-eMf~D$I0Ot+?YI^({sC@Bu}!Ca@e9xU&p(Zd+dwODJ6Zv&e*9sHRrb^BTW2 zNM<)g&0KJWxSS~s!_EB50_49{$aY=fifrOib7Si-pZqP3I3A(Yuhr50Pcef~<08Ot z$yTMPW5SIJLdOok9Mx1C_W9)~Iy{6<#bF%xY%N5wHI$pJaY&gnKj`~as#WRgu%zhT zE4326ClFw$*t?18pYb#mPx6##tlS72zFFZsIY{qM_LU%>;VAM48>WxpXmLzQ;1X$a zOj$<4^h<0r*fr_joav?9(|=t+2(!I!#wTh#vB%=HgQP_l>|uDH>*Rlhd%W#>d~~wD zMsxIi$#!LcLg|hWdduW)U1WhmrAuHVX>E}6oA@}@TK&CVVyFvgap)qj31=2^Km@&A|*8tsCp3CIEpy}@@sPZx}TJbl~1$PEwiSf7kggrv!tv=^{_ z?RycLOmp0vD0hEW=6!To%JwpSq4qpa`sn-M_sJ+aBO64p@0;)9rxR%YIi#Fg?fdej zln*6;oPY2+8MQ%;zc&7$$1}A&&Mat_ z{I)4z_pWUQF4Q0XA)1ZnyRQ6ms!qsO?W4m*t0@WOw$m2lmj6qL@2z)6F;STbSs>1I zSEtW&_G5=-Qi2jo*aVho`c~98vQPdAM+Lp-Pmh-N)YVx^l5hDVtIPs1mIhRau|R*e z9F14UR$+mtK{?CZjNDOV9J?Q!+^b8yEgZ9lMD?~9=ykX@12j~2BtMyLTv-WR$dS?N zIsL1az#DQ4n(-B}pjJ$NGF|#Yc=zWyGPS_tf-qfK>Ze16T7c&c!wQEzJ7}hls6r__ zw??aKp*YbL`Cx#jmIGuHn#oS;?^^ zktNXHjYfc>nT`qpd7m;2@lh8B!yb-c{LjY?a^IG1J?GeoL2)EGmLIkOp)cT079{4P z@0OZVB-s%5?(UERIi${K$`aXah6tW-?SMWx)PT%xe-Mk)&I-%ERKOi9zpW!~nLY^I zYOeiGXcIY!i-xC#-Ab-r=dBY=R1$C{>nRIQ6+xG*J+05hy|tVqY)~@ z)#lNDOI7*T@P604^rpO-X|?*S^za?a%RFc%k~ij~skK=-dt4IM&UKn1A6x-S?akrb z(>04)Tf_snzS*n}WKsN5I*cIt(7i8veDnHsP?@p%iIx>PJEwk9Beec{@y9T)2px({ zy%8xc{Jc)dS-Z_C^>bR=bkd(_u|=(Y;PbE%kFYDG36+t|$r<3sg!2&Tn0-i5lbvuL zCK|Fe9W`ju$;C;iR9H{OkNZV4JMZqxXE^B)W>I5a&^r2rHmzJ!oJKWNPI55f6X>)= zW~`EacZc{SY~6!8yA8({jtEaivlxDJe$*}^o4}&!i7-zECx15ev_7$^!#byWB83H6 z8;vEZT^V52d@Al~6O{(fiearjNgJo!fO&u7@*>pi;nTpuD18aMIKhQIv4WD6rlv$) z0q!OhaCwgD+Aql;LFmGrh#PUguo4DIVk1pU^_X{cT<;>D+t6Hc>sOzlR zb7|~lnu_(odGmR({X5fe^i?DK_T0#U-+%C93pct*ekT8nmw*?RC-8LKwTt7+lzfwA z>(#N)KJ*f$>JqosVoFX@D`d-vvx%Ovfi{m?^EYa5=G z3@}f3gf#k^j1Tw<$Oo!t0RH}i_cu!D2kTkQPWO(gC$=}+<`y<@JD0AifmB?>%NtIp z^k3*PjSkRe)@dtnGy&KK1(p|tt;#+lvIC1gCuaSlk5o$$YUV zXRsUYY1z7~+c6x5mKNlktZH9;#^hT|hYWC&JI z_m)}&8`6|D1)Ln8>Vir%3ySecX>NR;%ZN6fh|KQn2DMx0Q)nSOVA-K+TXkIo6;?B5 z^-n#9PTwl#Jz{>JY357%SvDygDJ&_V>(C|CK@8t8q&?5?k}Gp>zPG#bKjIPk-_?)5 z^#U;i&anC}DV%p%kH?mD-jATeI?fR~$s`p!esdEApggXBxci?*0YS5J{;@Ni0#_^` zopVn*skX{y$wa0w2=kfhn6Wr^F(H<3U-K&s4o$|VY5|^fQ-T;@>a0SHTP$oaNjZr& z5-pI>{&d9g@sjMHe!P+Ehe%#RH}%~`$dYCWF_vDCAH`7iL*Z`vZONZ-T=@j{2X zZYjrD@XVDr&gx&+ASng={H3dlrwiBh1IO;_a*qJUR7PsvH;Zb$G@0aNXwQP^ONSHeUoSCrC<#p1V(80Y?pYreLwS%3v8#wi>4mMn3i(kUK%3@gdeO{L8e?Pg z2y6VrdS7V9USQ z5}>7wL$M$^ctg3x4E2USL-i1dTXv_va#DJOxL$H8kJwW|>2>J8sbb7&s-r!j%>tog z4dsC0Rui{A4h6B&DxtV_jUl=94EW|bSB^(SB>I<~QVC%6?)M0ti}FyQbGUGb(>)<1 z1o`QCgf~y~++F?htNm+U-6^cl;CsM3ZNU2riP6D;2kT58W@e_8zytpU@m=0AMoS+^ z@IB5kM=t@+!Mjq{dLMZAr^cIWaRI;#dLIzc(Ud2fs!ZJTs~5uW$P zM_ScCI68IEUeAaOv};7&5;#0`tvw!aae1XU5GSfjCu_?mZhg9u z>=Z+%860QCUp-bLm$>d{I2#tsoo&ZHsWN4(`3DE6{`P2l2Koh$ z8oh{1ox#JL`NHBatv2ST8p}?48memNOC=qB?KLC==1#{l_B>KA$s+jy8w!Ed{FKN~VG-ni^(ZD&KeVWevN-%T<-i zw=)?f&M9^1Qi+6>QH7N(2HV1YU9}iK!w1pD_Z!S3JX1DzcjI(hM=#u^&6_GV1M=~oLZJ7@k~DN)S4fA` zqf4u;@I;8s|FcERk+G#g#-G~%hzH5CY0Pq`0pZC?JvVPsm~U4j$s0p& z!w~AX2V)@=d;et!)?^UI5Dst-H8^edv^(&`mX4uwzQFv)#Y*1|2X;@x@*+@_%a)BCPJ&&4DUF^CM9~xVudyhZp((?(l;8 z!&cPC+;Q7!$A4w+j3^h+5QmQdjO1OG0x=khZ#i}p<-Jo8fB0>_ZZbN)&wcdP^LA~$ zM>g@{z4^WO!M;E2k+4L3D=!<(^nZwnj*BCJw#v&bLwFYsoO7+t&W%_AB?e!bO-&*N&?p&P$d>zp4d zh=wo)EU%1_QjH!Fi<7D|o6ZR>-X_}as7zkyE~nXEe``JP`lK@QN}k%huiYMfpsL+J z^c6oWntWCA)(bAe#QDkH5M#Ta#Xt(VkqlvGL)ecmeSWqO^yrzS7gxD*)4Y|Ls{}1Fm~odHuomUYGR1{2IWZWoh{5Yu)zBe7gK0*$E0uKe>Q_H#TO2hJPZ)Z8z;1sJIR6j7Iu3tUw(0DypN& zQw;V@Mu%8Ia$eMyR*t?9xrQ4vCj_t9z(v{V=oHx!K*%!7xtkN)`(Z->^k{l_B6b4{ zbRldX<8%gZ;ZgLUG(*p=H@HrV&c6r}C`>p27%0*glot|u5fm3QnodRBBsA_9t0koj zyzQeoDC$l%!g2lXpLQ1An@g%_{LDk4I8jv@Bs>t0$IWL9j4N*RVcL6ZP7QbUt~)ol zj5JSp6DN4;LJ4RSBu}i2RVWbZDHL#-eoniti_}-rSo1^9&CPXyHL7knd;tSH)d^O0#&On@T4}=>T@hJ>7j^@?Ebl7#J%JAh^nrOG_CDH-Jtmj zPajxBgMBc5^wm`s1P=Cj6KxGPbAD&McH(aKrY3~U$yQ{qrtwZvw#iO6pbHF%=C$>lwHm^;Sp>(K912wlO% zjNTp(It}pi85P`4*g4)H!NEni(Nn*F!d&R8v3e!mB zB{%|>ztBR>4`T=$x}SwD>__G4wfHI03`*Uv7^@^-<|M!gj=^qkmWOZ>lWmuy#Y6N7 z_yblf^Q+i_JdVLS(onk^rbr)G6z)fV#g5{*TI%CkF@X@yPJ-ke^?E6p_7krLb1z3y z=mh;UfsSQqw<<-MZ1<9|xl0-XUvxv$Zr&^QVvr{-#<%uIYqtt4w~#&qFv zBQ#41XG+f!$yOZN5gYUI(XS!|NucbLS3Qd5cv~SN_M^~VTWy#9#;BtdZjm3(9Q83_ zUEPd_v+b73P~&}GS@QaaCXoHA8GurNxqK}9TLia&eE1l-&lR_Fwux2$jeE(K8|L0( zaKNUgu#NEhx|S@jBWMK%QgvN~c-OtuhU%>x@3maP0PWQ;$#n&Eq@84?B~NtsDgazZ z=+gO3V76Rxz&uiS3$3tC)X*Ii{(Eq3J3a&*2DK8+q4#4ox$Y zO4;WN%z9>|MGTLf$SBH$8;9N%7feZRU zuK0_FWH^}~_w>VdLN~d*a`?p-fOhH$z7INU${e?f3$ILZ<(x;fV70$QTaOQHZS^8rzal>T#>6ds zCY)&~qI}#ZYGl3aHTxCKLFxmG5iJJ*OLsMLWrV!@x`hO?rZ}R-S?re#G4bj9d8QlC z#N?GqDUfK8FHJ}?+IHd1kA=tj%#@}-yVYOZ+F(_JY0bqDLIHt%3N2P=L9SDQ$7zFA zESl3CiOpxUnDcD*Sb+I`imS&HpD#~5?-}RkOh&~3fl;7{I>Y>%*i3vJQV4pj)f-E_ z*!bpEH4aL0yy;>VyCY=#bkT-jb6VbHwPxrug$zi34$pQg;CkST9$_*m;WE* z81?^S9MgnzfjgOiGGuqym=ENVY|Q`6skM<%Z#Yp?_G9Tg*16@G3uf%oo1V=Obqy=H zN3`qQth^VI%R5&9xD$A=)RURXC{Ud83SFS4Th1*WQpz+HM=o1wmql9l>953B{9)^J zVtwnc_+M;udZMsGVe;PQ8QC+%6pHP+z|RW@ z3k3Y?HfTS2P;)9CHK6m1-<*K!Xt7MfHufVZn%`nL!evt2;@nEF1=Tp4`Vunxkbcul zS-e3@JisPCz;?YgX#Hwd#_)Rp%Z~krC^qHE1ysMTgex}+aKvp{-`4tGOt5ZS;#gGq zyE29deYwmSIJfXL7RV#X+Mwd=`9U~`;|R4A#)SJbXR!00bPNW29sYs#ZJYd^qvtAv zXSK1k-Gx?2U!}3!&(C)yoV;8Z9m&3oTkkT#TQYOd@MZ$DF8p#2hDjz}FDhqG561cK z%MuKkVtiR~zC77=rOA0o&RFt|;efzfoZ$Uie;2B85k}3r10HogU3M(9 z*b@{Ma{Y(XOQ19Z!C6 z5#@Ppbc5|xN*SwZDnarpZt)asMXd8@k=-33Y)&w8^GoC<7@{*9>!@M?Z!!2KrYq~( zAVjrT72vbpS>#S#llU@*eA9UC36r`f`GROm_qOt!Uz_u7N5Ke*A|L4sbS!p*IP{!& z8bYQHW29qt64@3KL>E-rgT{Um>n7GP{>Fo^uBiuBBD7kySTl(=E|-1N@ZUrEL7ne; zqW4iaRc11X^XfB{`l&sfh(i(k-HSa3LJ9=?GhYFV6Q|*N67C#P?c#5pWb5^{fPY-{ zN-9&fd@nqydnIcSy5b8R5&!F(IjK;U4mN``K&1GZZD97bp}#nw&6GLfFS-!8KZns% zg;{7kd^=zW-Z8<=>G}C@>@8hp$4}#vaXnzHNMDAj-90W>*o!>kdYqf0UyYDpUH}vD zyF200XhPaO-XX=(_wuDv!}cg#1z@rQ3H9@)`^7lMP;2w7 zkwlJA+*J8D!4OHd-TCiyEg>R)h@YHfw*vGzLXUMLCHvZ+zIet%t3(#^08NQcaBtKj zNOyGRE23|ewff(761peGUhOj1hLF}CJCjij>Z^V~QiC5i1W+*1yULPyCdz#)zVHy0 zHvciH$9q#p;}#%VliO%nZ9db`AXiw8WZOKd=p}qT1MC9LY`qnrR?KV{a3k(Kd-%!= z_2$8$GaFDGV>Ssj&$jlRdu!#{@ksAX*_!XHe)Ig#0hZbEQ}NCwfzIhUT=k=8E$@V` z-_U?qr!*Y_s!Z{rIR}cb^;c-+jS}K+r2JkOp{7e)vRJ=5D70ov#-6NE`fsCyxI43oB9tL?YOK&42^WiSl)1S3<4F}moGHm7%I`4vgx|qzj8Vxn z0tsP-f585N1-a^n^$))9zg+;}^@@el#Z6z4f}4uFY{E*0=y)v>Oz^3vLyNB zEu*@BpJN%oL`*NQnuQz8t2HBwQsmtyt&SM>@b*K9>S;yy%m^UwP{6BDPxOQjpQ<*Jet2kW!h+Jn>0sixK#4K1O#r{Y=s`r0bz zsA^m5$^qEnLLK3PD3mBFGgL3|n27$ZlE#zAV;2r7s^{PM7nD9CT*>bLh#X0R7tfu! zx)q>&pgLxt_DQmZ_goL8KV=`@oqkc2k3~`rDCidk6zsDCqC%4<_y#t zjR8NWzlAW)X%2qcpoOim_@@Uuj&VxFnvEEeNHf;Lu`tozx)_m@XH|h_)wNX0CW9^S z6nMMU{jl#PcP0{%|Ax`Mmu@`s`zmtvH2`us?R@CW!xU#^ba`aF%T2E*IJJ(_S*FG$ z)q)Q6-3)Pv56mj(gVC+K_iYr6qs8ywcAHs<#M9wgPow>*(AXx>a`PJN8TaI?{E!k;er&mZgf04$W+sqR-^V6;y(MHsh!~( zp|3M^;U3sDP?rT<5Bx29UgH+t>ZC=nC1LCfI9-5)P(TSn#={M-w80@W4QHmkg9NKc z&T`)GJ68ytmSHl{o;+r-P{URCv*6881bJcDLw&4HOMoxpO;_X$Vy!Pi{v5#u<+IP! zFdjLjtUm(~Ik|{JrjHr7yaRUPbXPq6^REHM> zb*_}{6Xk*6cGo2Mbz?{4Gl}ulyY9V`k49v5fC38M|W>+<;=q^K1Y!&r3}Q&(&)< zv-u=Yhe$N?-wiK@1+`xlg4%|Ia(>fYylkM2J1lpPtX)~|mg9m7_eaA)UN4M!!MGze zEr;RJp}Jl>>7j{$vC`zo@^7+b#K;YF?@nMRiq@IFt(Jp@ZL5GH`vb4mu?wg4gpOp| zC6_d@cZ&IOLRmqrrN+1e@iJIrql&!N_0x;`G{mm&O#8rjbVp);{pWfUGMTxJGd&sJ zfYR<_PT{w^&Q2AM)L%&|DMooaI|s@zDL;r}c4LRW4St`zK7^kf<0;fGXVN_phM!E| z*xQJWmjUyACzuim{VFOd%Z_z0HpOAq@HqJKcrIzqv$2DDGWLdkHmSFR*= zdxHJxweHBnK*sFH&FtB6u~+MtMog=gAymJC?|)Ymce3d|d>*@CPdC~(X3P;bg$*Qu z$&K69Jv5A6REAsAaY`J|L*672HuiK%t7@ z;mtm-W!yr=yd#Z!J!Z}&S7g%@XHg7Vjw~XDNu*f$ZI+LoMZSq)$osp_a`6Db^M&RC zYJ%4sCcmH#v}iqw!Ym*EkvZO8{GS6i>|K{4y)w0I4&io;%=>63nW+-Cm+@>mAS5>j zvM656xtD8_3zt;kqm4dp8eTu8p6cr0=U(d(c2 ze+o5F-1=zczIFap!jSzXYW`?MAcmRC?{C9`N=h33y<&<+4^ky?q(yb)UN&S3)XI3J zUvta|uKe-|HJl&t%Hd?xH*s`?1P>pEj1AhO%Gw_7izWB(_R!pXwHCSN6(o6Y{Seoe ziSI?Qww=Ad<@`hgT4P^S?F|L`FQ%ezkl&p|{~gAqe>i%a+I~oEz9lccoq*YBp(wJ&E?vIw$=0s_fmzI-2*LhE>cp z9xCnY-7u3Usp842$XwN_p>ZesRiFh3GkTq^^MiujX9{m9n_lq9xg?@i_zcaEzF?KI zuSs`01`1L@wgah(Q@M9ywQvy1<-_)WNHqFyl%yPp}BsSqOb} z2du=tWzg@_mDThhi9451B|0zR5Q3q!88gVJ|i573=`qKj}l#3v5j3|0fP-vvDyP#qXk?}(DjVTg|UVnNy+ zKrTs6EKdY+Kp|_ks++|NEAJ>PMLlw(l~FhwHD9Y+_;N3GNtmm)Vb`PCIEI;D6|+jN zYzkOBKW7?}o^voLT;eC=_~O9L^YgK&<%MBbXuSBa$;R-&@ToH*2qNI?pY*Hf7N|&G za}FOBkGQ(U9J1)mu~2bb(0?dvuYGgT~Xs2Ce|$=#3k$iX(mE#t|VQ_5AfWB|a5 z{VR)gl`=-&`tLE02%fci#LgHAiWNzU6?I@4-Zie;hemyST&~l%AK}cI6B)&>{>eWN zXHFwOpk(*$u-uVxjyWDSnOCu`t=jx@=+Hb1U6YM->kumX_B(=J9-HlL<5zqH?{gJF zbH_?E2F*n9CW=83=lr5Q4;u&kx)_Dnw$9nLQt@A+pof`)j8&ac&%`kj8wI1_d3OIV(mFWbnL1-j5uR})LMgM9 z0SU}71HOX4Co7J%i9E;_wD3vO%`g^zfXok;IMvFzH5Yx6-Y_(jpA9+{I=$iyJo;PM zegTd7c` zPc90-H0|pSnMxy5QS0`CKa>7Q1Uv~%i3;imlvR&4V6u`r+!=Xt79TIGaPj1gtbAZ^ zAD%yLX;pbv>4ZA_76f5>;JN*K8bx?JWUb5*i)>#62CyJce%mS<8&{jGUF9M#r}x8Ue`b(rxd04~T=tP0%%I3b2lVCe3TXY{@z zRB2=qaBE>W<;24zqlbe6S67ua&RD}s?qA(GPk^$(Yl}3Yi4;^qBBzT!8y>%S0*fTC8Aav^-hi)vR#f`_`k!tDW)q0McX>bMd0pL;KL$uK%q(RFGVQ#*it9c7Mut1Qmc`LYGw?3w zEwJMH+OK6(Xa(nhgeP-pa}bV7sN%4BuznJL6mouzxAI)357!u$^B|tGMk1g^(6m;D zhbcCX3jHcp)~~qPK|wvg%N=S>9pE-5A5%AItXQe|&$S%lifXs|Gs6>{{|WduV}i#B zWqqo=&mK<=n}-E|r_xi9G{t>jHsfyDnD}f)UqIXDLLi^r6t}9})Xs<& zP-1b+u`W{~GF&WvC@Mjj8X3+Jwho|*P);A7SswM@%nxfEKPN5e69m}u7 zCONz$(9K2IrpeeJvzGyrrhpp{a^cWTcs72gIcTjQa~k`3>a2vI*45MRr{Zy>t-mU9 zJlMWquOPGB=Z;cA?~J>&4UAs>U7rX<{;nH%;uX_~~*ch)$yI%o$x+^;;nzYIUaydaa=i>7ZdQ zkL#{;g%q1`Oh(5OmoAv;RwQ$U;k?A1Yp%SMjs5UF^Vd7g_JJ7$c`-bPp5H4k4Os&* znMCa1TUuY(Ukq3T? zZG%3fRhROmW93KBmhnu8)~m8~bSc?ANlNo5R&Jj-H!2K3!*E%5mZ;HNYNswcSUpYF zVi`iG8%Fw%ZHrC&KDJyuzFGdwie;orpG2`S;Nj|)#w%pXN4ir9k6J@baRxt9c*j2# zir3}kMU3wBJB&Ub52k=M-e8v?loD@}-2N;D#YqZDM#d78)>R`G1>`~1#~H>-PvPhB0L#BNtKdx1tc3UWXvOfamK<^&KmqnAR&ZQ zb%mSCU3&~uSv_u*oL&mv-_xFuzd_oy)t{_pn$NqjU|Z6-JD$^(gul^P#4x$8gQtfF z)$Ha_$-(992TPP^T@AEvLt7q|fy7@7Iq! zNI6%|<%{Q-)CnNBN$X>z7du|~Z%2huGjBwd!Hp z>*4hH=)nJB0jS~ul^5Z~Z~4}JAq7FL;enBIj4vIYAp8rT3qPN`Dx=|w#q#7*wTNOj zAywK8Yp1NMK`8#KnU$tWkN+KOYM8!Ch%LCr4Gg@_)$5zMYe2(*U3+7nAY8)_IX}7f zTxbwaX`oc>;|B5spp~>nJ5D$Glo(*2yQ0SlaTicFy5ub_W!;l>Q6uy|5l4MX`#Ys8 z>vOZZH2wj0(hDnFD>L4aU(pW1=Wr|9QWr2aB(PTR*8<;={vfd^=gh3Wx+Qb}gn|)| zlgGCJfU)PMtk0oYx?(x^SpXF(6;H6NIfTYR^a3VyNB%8P$`&B&a3Asf%bS=3RBCkS z7D5LYOKB}CGCQ>jz`{7*VZ>R5K;hIv-zD`|SPA?r1K|Sd%vIVRfy9hi&G_E~-|!By zdGpVfgl(Sk74fef;o4Q6G{AgeF#d`w%iFWWGfr7U=zsJEu;b8?WI>3QEWD8_82qfdpRrUcH`p;Hv zBZ{3&ZchpJzN5DwB8Hy7?pz^Xyx~CpjUf4i#t9t@dfz;Vme&?Mb&(%90^Y{hZu)i} zHoD&z+g4da>mI%uzrS}!Yb(9X@w{hYzTUq)uBLmsjY(M>O$9xVYr9jusJ=eDzkYNQ z1&uyFU5LmI4U#T(;k`Do z;d3{8__X)Xn|C~l=%)?&=>=Jdd@3LQDrZX9`xn`J15+fD%)L)4LNMrQqx&W-XtqZf zHpDdFdrS4Ej3BX)!sXd!r_(d%q5D{P$c$tArK@k!-smPTS0rG)ez8F;?auQJYdKOh zYzz&HSg{71(V~4Ts9Rv*9&VKp7}(Qa0Ua1eE}TeL1^qo2UKWVf?s~s06d9=&;-LAN zD{9N%#-*@c@HpIqtkmKD_}L?*5<4j8mHXxGU4p7sdOea1n@W>9I0pTs>omMEXjt-7r&Um7x>-4PrKD-dou^`oteay2nr` z#~huLwhUNmMoG!21q41=5gXOfdil%>jYnLV6oWZgIlf9tpm@|o88K6Rt-0-eW&DcK zqay|6{!vExBcayT*k9B^0lV=R1ruXqB^486%xqgw&GR=6^2YV;bc^X)$(H2@v6?`@ zN19^WK?i~pVL6x+X`tH9kWF*kOa1Nldt?+0bR443wR;3jgs2t5YLl3A6U3_WDL=5$ zGbhrEuR5uJd@MO%4an+q(2kZPrF_3qJdyc z)hl~vzHm{l*<%r}#LK=XvTctBa(-b=9(9ob;~B$BVm-{o_b^A!E=9sL_Y z!2`j4KDhM>^YvNv9kW(75Toun*_x}8Zf#0+vADnTI53yu>eTBxr(nVRG{_rmmQ-7= z+7Sn&Eq;DA+jm3k?{U}B4H-2aQeq!`j`$OWIfnXGS<)WHW4RiBOSYvZwLXY#!QKCe z%M0ZldNt_txr1|Awx}~OuCqX||JWduyfeKKZT>riX5koGZd_7R_rZVOME5ec-v}dm zpJaVr_Ti(zmiID#-w|x8g2Fhju_v*dw^(7<3|M7ZGvRlGAhQpb(7l}*>g6inQ8fsr z?@?`5z~cde3^G>^xp_J+TRyltp-!OkwV6Wyd)euUh<-?CKLjkEJ7BL}06N0l=LnNSS)wON9!CP|hJmjVC! zWEf7b=o{iMBe|3k#^6+3#8UKKTwU?&Cq>6bHGvtq48@DX^)q(10H<_;C-q8J1PM0y z2eSs$f3Z&_z5<#=Kr`5$6Mu_>5{BO*qR?XM{3bm?;p$1TU+2zr+hXC2%eqVpwhoWf z1JFuHIt%EI%Zgr`5oy-bT2I3mmzbpv7GTy@<^C*z5n;!Zk1g?Ss%)GdFz!&R)i+3^ zJ37kj^pCTmY7hpYuS|7GqOWOtCmP$xR3RGsw^X=+6E@-Z`wEkSj!>)v+z`Bu3# zrktUM{f5g8;>wy=bD)GK^mrNhs11hOc{i^+?C8OUM)@oX@^u#ZsGixIBGrE`M;OkB z`uY0LE$=6wwx{z%|6tXv>Fa7VOZ5R*y7Ka9{RE?saOn2W*&EvKD_P^|eePJvCX8G2 zj6Yh~pM)k((aL|uE4oxOmmjXJu@v4G?EcBpMZ)6){MuZ3p>ci%(-RX-GK%5AQBDwQ z?B!11O8hafVn!GF0Rsoc#zL+0Hg+&rS?)chDFKM;_c-E$k86YgKF>25<>+rV7ty44EN4OoZW&`sb2 z=aN+P?CUujmQ)>6=|%4O3ck}qnl`ycJ~8F8Gin;Ip1UYv!c6Rp(9Vi~hkt^@o0Y1T z>rbCXtT;9!E5BWo|4JCG*85}6JVlcns!g9^YFtzip)7@&#D-jEH+ZnmW3}Kk`^J~%6}IJ3 z%&|aJLJe;o3QPSXfj+HxSPIy^-w=!$;0E_Kduu_^zRh*CqZ-rAa;bCU8I!fE-x*Jy zz?ho5>NVL#Vh$T`5cn57OjGFjX8BXt!y95jSJNMr0a>{TapQzvhM3aXPaqKc8>$L$ z`F93GALOfI_Q*_EZ-I7iRqJ$H9#qVTAh12Ets6YxbrCv)T-fellH(XZq2@o{zVBuR z9ZU;dEwk;}_Z|5N@$5>fJ^Re?b|>|{9(vu3kiXFPp3$-$ma+-W$`y9hSyL+5TispT z<-l%qH%t3v&BALIhn9&aAM!imj$ zUQa$ihZKb3j3!AYkA&1m8wWU>OHP)_vj~gX5be!>_+_rX@(ci`D0W5K(-8M!)P2|0 zf00O1EgvsWJy4gA=~yTCp@jw`^gXl)lwrSb@YKtm6A6%YrM%0<)76cRk><7WUnLBW zX$x=mV+B|!HFA`hRaP5lS-8UnAA5A_kp@yU;JP6&b`()W=zq}=l9#V_aai*JVDlXM zHI=<)_)BibNtgM5O695f+5v*Z@wGSAi?$r<;IfSwPHWwa4@15vL%wzvcV;0HW^K)< z8mzGzM^!~~p4OJ~w$&PIti1^^28-e9co8|GY=Oi@Ghu7z%7GWh+tcr45Z-P`yecOt zi_3B1JMl%}ZVAnthnN<4wi_2tJc=H&pYwE7JTknB9n#B(&5t>t8C)1yN17S_#L!^nt>385xT7yfb)(eg2-7PRcyiP9sZ!X{epVJJ&LsS>cU@d!SC z<05})5$o`R>pn&F2NMT{(!0?L^K*&|&CLSe4g?=lW`klx-!CJd?%rSN zU)OlM(+i$QU_DGV$qYJ0UqtU?*5Aq|azvu9?GMp@*7)=XCp2 z;ZGBdr_;QU4kLe(tLqcfC7GD~5H1A2hN_x@(4?5DOUS1{hgEfBYYm+yfs%I>UcUNS zL`CFtOFRL6js$zXce=*~L&s{N`4m8?BZ-Sz(jxEh5UA|0oO!o39W<6Il}FDz6zDIvE!yKsby$ALxmv8w zPfu9vQG?PYJ0{aJ6P;MNTwhS54LOOK3PC88Nt)!nI+zff#KWg{@Dw{QwR6S(71zm8 z8{rw>&t<@ctKaPC(}fS>tSB)Y>d7Fb_iiCgIAKo0ptZ>C+pT+tO#@d-&&k2k=?!8Ry2FJgrCImw6|&+PuO#17*+m^>U;>C_T3rW3b$;u$}^q20BmjEwKaGdApqEV0M6 zX7g<|{~*?6JM>4nT| z=%sun>k`}IZ!ZH*gg>3m&$Wd2rYGR+I6=3j5V-3$MjN_C_hh@77*j-v5>bl?X{DCl zaL(KKaQ-zZG{nhMNj>HxMQ|kF|0Pb&qBeQhz;jO{&uD4Mk|4;DT5F5zu8X>c{$vq) zJ3=HKd+oVQ&H&~lh6d1ZD8$(5+3V?=ot*>Q>FJBVG`8BFoX0X-I)o&{wPrwXY;?Qv z!43Scl-S#@)mh6IoR&%ZA&4(Vlq5Vkn$Z{-bAZ!TdgJO&h*kPdNsBu`e!*htdidx&uWzHyI9XXG zz)Z`lGgRF_w~x{_j)md#0HC=6g;@>bF zLxC65!P#fe)7^2D5r2vOKhJoqm<6(24lo+&Eil~NLX><6{35SU z3G3pyR03zpzUAmi;cIhh_QXBEM>7u2pw&oAI&)ciZ8gK=;HTHUsA4CMg5sakNwX#6 z|D4pvmM=Wj5W9=mA)LD{@w2!N^dSW~7;LTQw}Rg}*QWwJ7QqacjUavE#txRzBPvX2 z9Y3%Q8`=YpL@O?m$86~e%$S(`D1QH>2*bO+c-7>Z(~(*G?D)ue;h*Jt^0f|<>`e8N z-?wWh`fd#)7s9ts5b;5PL==?c%P$Hki0OWLBR_5n+)JGQwNG-TK5;G=jJvn#1;Kr< zIDUBCrQaS5dZ|ZJh^t`p!<+TlbES&;Go~HLTi5yY5pv=cloixJdy?6k5DOiX-=FkR zQ>HhY{OwED&U;VdCB*n0!3Xzm6WgI|L3jU@Vqf^N@slCsHsC$-xcd!6@?4af`yvzs zDPpV7XBy^lJq0}tXy;S?c5d)~OMR^iGN8i1A~`x&?903e>_-05VtjI*eh-{DJCWF7 z<914(%F_Mr!yRgg<5JuB>Ex8#>1`1Cap{774%YbH)#&mWc$F^cM!L#+zIn(M^sN25 z4fymeirgpH`5jVcu7@W#Xl$5XcScB8E)m5zh9^;()&GHphBCy}&^7a8ZX8IFtyW~X zWq=D8_xM_qtM1y%?u}?NV6i0BRk83C0ftEoU~7_5jgd55VveLAV`O2r2mL2i)zN>D zyqFNgs!egVdxwLgxomn1=_%~*KN$rDm)Jua`T`S#j%!zkO4y#K`0KZ}KSWO$ospb1 z13Rj9P^FU+^Q!I~Ic8K^?IDTogl;!&-z7dtjLuR|x+!;ot2F!!>CH{y4e48MgUtlQ zEgZ5>4wRtE?khT=J7e>Zcu(`(He4uc%dycAvZc-JSe~jgYeCy8vJRv=s-)17m68rv_)wfOlPuKlIm3l2;$I8gp<~o61S~lAX4*V!Q-m&m9r7#4)^+L;dza-|Fta06@s`JFXG}_r~C;Js=)hl zs^9iF_|Vr!w^+r1TK8TH=QTaw;A54Az1+V z18?MVg;Lk91}8A9*Ea?3)m00;uZMgQDX$7#eO?htPX< z6069|b*2KRuQR~-z}8nzNgJZI9#T$0{bf#5>AO#7r_`&9v1U4hHkKRr4YnK+tiMQ$ zt=Eftws7hr`HOBkRc2D1a>i$R=R8r@ln5`&#;q4TkFl$ujsX`0W4=(f&wx#zjRkGA zJbRgz?|E9Iy;%@4skrI;NtV=&$^!U~@ymg{G5w-P2WHca(icB2GTwueRTRfb8D5Jl|P2=&e616)j}zR@&=sXj=LI z44TK1kWHac$>p1A3jnoULNNW4yoF>K(~kuE_Lr4Zl(a_0esu?Gd+-MP>d?R*^VGSE zifnyN%G>H5R`D*hrSNmO`jVj!(^#ox;(x`-gmXNn(dS5>>)mS*14%Dm%k6}TSx<60 z?NNCh&}@WH#Vrn~ed~{AL+A}Bj&jNUduuD}LvMmg0d}NXA%I>nS;qT~i!GcwKwC{0ERvx3^Kkk+VhT7v<- zU8Tz6xZ&Tr+sc$yXUb9ydF^gL`6VSD$R1wp%&0BOoCAFQv#0;9o$)r%wP!uWfZwB- z1v9>)Q8FAg9xQ%o?AkbIVN2_G>Ikh-!v>>Cqe%TK_3R?c^<()2b=*^M<~s#C+|d#n zVy?XctT=P#D;Bv7qB?o+1IYkK>M5454P>tk>qAPEsZ@P!>P>5De=&Ds@BSt}tT>*n zRB*Qdo~IXz5S#S=lD$7)i5{=Jg$iMv6dXotL-^_f2gG7u?nTyVJV?L8Iz*k(AHNYi zFn~XG8G9If8E47sIlFG)pR&Q9FvQE{a}9NV%W&RphL)a?ig4hkv0T3T0j&y)mMEw|ey?m*8L}z|*4B(Mx zoBc#aR=zWstwWDxPivkq4&si`L0dyVg;qS#YNc^!m{_OeCDs<*W&2t?M=F4tu}P_4 zhNx~&fN8iA&Eezta9cbJ7kYs#u9;*?E6D(;k-rJZYX zgeOQ;&}?LbG3U7Ig{)wjvuvBwC%IIfG2u);A84bOEMb=Uy8Thl5@+CAz#ym_lUqZ7 zeD3W1Ce^swa>TH)tRec#zwOT9Ora~1m|?1@>!HdHcev&wYL2+K z4-BYq-`7dXmUNcYanq|*1472BxsZvRGZmIll7=aGQ*=hk34UIPuHYsKH@d~gZ_6uE z=-=%lSfJ^SXw^1-^@KIlvOk#W1T-L~=16&EuH_0Czw{mBjo~*+5!9vT3Lqc4k+Z$= zPH3(}>1~Qh+(bhcgleb$Gy^1P57hD*SB?FguwHL{#q%1iO&kUKYM`7{7!|8#IFO&( zlF`z5R>`f}SSr=T(!c5Wm2uu*WyGAau1DY06h7ybc|9de40Bo9s+9 zM+S%G`*Xl)L&3;U;1nS^hp<4)8qUbSHO17aBX?b*aGp6^pTx;Gg$A6-j+wC3ni=B6 znr`Ck6Y|TH-by~>O7_=sTF~*kH(hvxxMxhKzs$RG8ADomQPz015Gz_Icg=jFb9X+_35^ItiG27a%_Kh-)% z^v);E{8)iQbG`ccz)XKh8G%I_M6a;bEwk0P^qNj6z zTM4c2jm{_R{d)GKMOfQ%Ov$l735RQ}#6Bgb+T2Lg5OH6NIeXN4Yfkmt-53{GbIw?| zty1s(xLDk{)zB>XvXN9EE4Q_QIJt$@J-d}?4wRyPJFy`Df%V_+GAZh~6n8^O1l6P8vrs|g#`KiPpxk%deDCW? zZ3z^Q4a$IPmil|HA@|H!3lZSXo(Rn2#okp1b7FA0YZ`KK87UMl66f z$wU@$lSntZW$Cc&QO1bgklp1YfKLDU+4tZqz}|a3nxUtp+7$M@WD30q7OAo`nKO2d z*kk{3C>#pfK!XW%)|e~%naLo5y}R5KbE9p5!=UA`L{b5hWi&NCTRP_o-KECOI!#o1 z@YD$E+MT7D`sk(VgU%Vx-_g`K`PwA;)J+}Q0tG60?4o=Tq{Jzzs@{lxdqFWfYCzAb zJv}MSYD;lg!X5?ikKVd{(crYjE3>EaLD`O5L~9nGk__>5Wyh0ytZs^>#5~`C%^SKg z6aE-2j>QAup0^n8w?zmwM`(k$ArG)N2yB!ID%qHct0;$fFuVk=Hnx+-k4qb+(7ev` z1rAzk!VD|Ly#5263T<9nYkY{f9DTWz^*aNoIi+`++<&5x{>}$@^(LtIJpGn}XVMHW z(cqsFy~Xboms^*92SY*o0m+MCo;Itd{iT6ht5qV)5Q+C9Gq{?jD^#|M?~mXlLgNR( z2$?;Kx0W{^MNW3dH=`S{A#hdI(;W6oULl`fh2CEW0$zMTRrmE%!w>T@^_46)b7q}2 zmdO~2{Hd)fgwe!bL%uAICXMQ&ymzp%trBjD2Y(DD-UiWgeKJa&r~`?jhYbYI(ZZuY zCa|-6;xbj{Fv#)v6X#p&B6q3y^V(@$)IVP-fX85X*2@8K{A~9^FO4pn=nQU4&S6f} ztsl|mJQtz9X$5D|y={n|pTsx0^088Ehv{_Q>n}m_6(JNMHo0QV31^smFYN3eE1A0% zck5NkhdFoP(=(Y5Skf@n$!XAul?g3c93oz~dAc4Rywv{V6KXJu!W8iKDrfx(R?F;M z(rSbcuoA;(wD`H!IUU~+8>~Tnj4;H=*;9cL`t(&%lKxB#o6I46apTsJtzO6n))X$t z2-ecW?^8>Xsn@xMx|kNL#RFHPC51gADydIrg7Y4dl;3L}}TWYT7CntQp{`igJ|vtbQ6!@EIk zKdc|A+)qjt8!^Xadc&)4Lnhi>*vRPmHNz;EOrkC?z0>Os8xQVVNxQv^|Ie`j+~6An zS!Sc}K%1Ytq7t~FMGxN0zxhlZOQ+jvrrj#oZ>KQ zkVBf!`V3O|G~Q&RSrdj*H`IiLTvVp~zleGZw_OtIDYp=cboKio=wo8w->P!|)mxftj zo{zC<5Tj~f?2q3?8c$xf@k2>mKFOGORh_k&uBs@jC(fXveeFhG)yI0yd3Q02GU`tI zS5t4r`F??~>>$WDB;d7)?YW+PE%!N2v^NiFd>#t z@Poa>;M4cX7@$^>J~?-dGmPDWc&%-?{ro%c3&-H>J^N*FGwrj13yWI1Vn!W$ww19j zkV1jIp3irL`;6ye+bn4Q4{u`lXm;D7&gyU(`OGYO(Mt5Y$CcF42h%P!hk}~1JhIM6 z+SAhqpq{P0TMl-8?={_(a^l%>bGY418;L_^Gb^Y?+}$VSZidr$(z9QJB@!=Eh3+6n zG+d0V?#DuBZHBtuxyZ2KyhDT-L{_dHFeP#Cl<%I5FLh~dPa1Moc({@cQ5Q5j{bK2n z<=c7yF&M9^fsObt4n7W1@!2lgH2)a=p>$Hm9gvfcxcwltVN3tn3Aq`qPvWWw3)g;| zvg*+IUo0S=<$=%_7d>`R8PxnFDEOmlCf(a97MV^Z6qV5sSgkc{T1#Utrg) zn%O*}WMW)zEIiN)R+ ztejYxD_11T8%ejSDpFH0$VfL0*@%Y6ul@^Z3ZIqn4sRz&!PYBoU;qWVgnd?+3TQ!Y z=b6Ci@8B{h0-#*Z3vdp9DJzIOCmD43vy ziUgWM`y-4BZl$a3kD-Siu-12KW5w!x?>|zAIASp!^^HZd2kAD26+URhWjdo~I1odh zru=ASs5$%_I7!ij)c@)$)5)_f(@jjv{XI|`2yd{Dtqj3o=jyA&YZ>cnHyEh@)03w3 zYdnzGDsI$26Fw6f3}1_kMS}^R3rWbDNMbaYz0CuD?fuXziD<;!Zrh6#X85{Fw-!A6 z!$d(=L)$`IMnlU&MnWeUPfe$Mu0CI@hVF9IEzb(UmD_yntY@NEL8!hH{b!#2EF@}@ z5y58O1pS&eGc%#$Ui_;dF+93uzDO}Vx7#mssZ#3YPrd1= zB6f8U_}0(=b}4Br&%*r*3U#6mI7>(V+MCJ2C!sJ6Js{%mdE_ z^6gUDfp%FqQikxQ$blECppB?vJMrg=?BO53T0`)#r#v>v67%_y)Q=aHxvc!J{RDnq z-O&3CUt*I9sOAb|CSeEbJXo#8!HHXr-9ULA+mUd5Olmk2hM->WRWnPkZ?+kAybfco zO|M@oPO#XI^1%A5Jt>V@O~Al=x$Aok-wyg?{VR#NkA&a9>er#<^1{hp>bbd=le{=a za}&I9DZX}q;7Xz;>g=hUhF___m}o%m15+!R*kmLFd%V=OI|j?p=29oLkhG_>M7S+3UA)ZA80z4s&D!l4pUztwPv?Ji5=J zoA5lze482GeShI_!kVg;tGwI;7y8>9qAb89xeA>BSx2_1Z;@p|$G;4Ux>DNQR_ znM(qYB8lN=qME4g_kuTojS|`8Q=U7DQ@Eh^DTB743(ZO_k!9=h@qEMM{f)RzwVO&s z@RKC98e7(fW{?bg+)h0HYOp?7jDRT~h#SBWdRW0e+Hkoi?Xb04FY+eI0HQ01;|#kZ zMYFNOH8U?HHW@txzcNcl)KR(rc%hp+s{hF%c=3628 zGruyzxkhxmYDd@IQk_K?mfQasn>hK}y7BA?^{1JfaFD!m+v*Hjd37BjWfk&}v0blb z2BUrgtZUMn{7qwU8S7J8d%BB7)pq74M$JkA(SgkO8(!>SX5qij%wLljL@a;SZ2?hx zR)!hj^aWJO>>xW8F&y)se&6tUq2D_L3T;S^ z$of}@KBU1^(5xxDKa1u|a$zhk@ICTzS5nNlzj0KnS6Ljj=C(+rn@AtGDz}e5q#=lX z3xipI^L;hw+MwP_PvX!=a$%0=aF_Z_sEMfK#1tKzA4)}ZVNsyVqB&qWXw6;d;W2M) zB+F;f!kqooMHK{}Kr<>+5y;m4Iu>K81!gbnA5w7r02cYoBRaH#AZ~&xHzkuDTHD$I zfp_~T=aZPZuCgB99^b3bA(P)_e$WQjQG9p~VcaYPZ}2-ti~R*-3(G@EBRlt< z5!vi?HT)HQQ{aOnxjN5d!*`k{aOe+$SM85oA*wBqm@7 zJ8&gx=^ugN?wl`06O-Mc&TRgrn~e>m_MKj>P3QO(^?>zM;jQ~1Ek5s&>T?%v-q)C* zNv9_YiB#4F)UD+iPTTpC?~+nURrFR?<>mS18j{6sp|YPOF)kLco0ZdNPF^oDEA6-F z0Q>hCaN}^Jr7=Tk+!ij}Yj*6>?(`0M5uAn6qU?3X%X6}jnnn?Vbb^$*e@~3Y8!5w^ zRB|GEeE}|c3@#s`V>Wj65lh6g2%6AG^m940*Ayhkx;mT-kqbj&iU{X6TJay{&VKpvBWl7Wk&K$V{`7?D zlYb|wv>R-Z{d=JTC}G!Ky!~^@08y`3;M!t1@|7oSapOchJoJ+;M1Onak<7hVvGqCd zxbD();MnY4d=^gkk(jdUn_*jlk#DVajKdCHfnm|%q$>h+EJt7ihX7O}I;<7sp-p!rZ2bqWkhi2#v3>K(Ir>nWdUlm-sYHzHD>(LF zap6)%d+<+?ra;KOdkP&sau`jaZ4@2xJ3I3dYCxEjycajOkzVa?f*k2hqk`aNM(U#C zyIQVR|D?xf(AWA6Y^^fmu~!&Fcw}&NZ+YJ))XT3ljI|U@hYY7Q>k^CLL~9mUzpHrH zM6TO5GJJDI>S^&04vGRSD(R__7U0@Qqt!oOdO&$Oz$t3W2(?%te_D(Vcm8nsFj@gqH9ok7h0X zW!fLSeJSl8qNy^8w2}!#=RwrrC01~S!m#vNp|4_9BSDKR{f&KQ zrn-yQ81C<6sPRsz@z&NjYVyn9y(*%Vb8np2&8%_Iq*Kr=vlAepA&V{GY5th9a@WZE z{m67==r#|vNyqO9fl(NjoLjz*W5QO9>B1Pg6293Ef9(}1qBysCA?s~47rW;HT3z;7 zGyms+d6MB>GbDU+<3hm$Olx-x5&>o@`I|IvPim0s9U2iH+zCHEbX$DzyY?6E!Buvo zv5|2;f1m-{!a*W9^DZD3-UrYsxUbkOL^D|9@mdcqUUv;}FCzVLC4V0K1aB)3p#$mt zzoFq>Iq-Y7{ks+g&RIqV`9D-9g?tv@1}+9E4pQyY(k40p zOSASr?csjlOwt(ufm$R5nu9ZdJxge@8X8<4WE56Ofv48^jBtU2D z3>&ct773%Ohx{4Xr{Zrg@@VCk7mjiTizmIypkD1VL`y&zLST+PP2eiKuOES-$7aB0M;+6F z0w=vh3>ihzPuKA=vkg1Wdi8@SLO|wiu7gBo1xX#PKXv?hx|!mPG3R#c#fYtPD}sKl z9gr_vYiwD?SF~hZRVg{$2vG7<6^2LCx)VUgGEH5FWZu)lbp(y{L&RAc;0x&53E_5~ zqsjngah&y#Q1@?22gf?Bb&MLWO}4nz>#PrAmlNc1w%_o0(6b2!iwYTZNd62OePmHJ zu0)MzYb;lRGwxht;j{an;tO%sD4#b{ga~}(CXo}o19qEOJ)P?U15QZ7%-@SA3Wu0- z2NGQJ{g5k8kbK-b1RzW1VO#j1pQl0Lb;6f6)#cVGc0vYW9;=1FI$cHQ#>Z7GiL0v? zUvD_I0&@R#Kzma9`X9VU7M~Ovf4=`nc!5Hr5nd9f6^~S($j4QAym$|k?PCT{>o7b) zeClV$Ufk=Y=Vs%A78vJoP=ZmAB)rkWZF|0lAuU)0w~x9x?_4jd>!*^&^*g{<) zub@jh<#6eflJ{AScZUf40*vH?SG&1f6J1|3O?ixW_NG5hEs<`+Oa67Na77_{xh-C? zf%(Og-y-{kPhXAbA>-pX>WWyt{I$)P;+&fo=1NVPo?b zyg%vLs0P7QwW|H};T_;5+6XJ$@&J?7S8WP}jdU0%mBIOm@oxl&(#$I^TyfT(hY764 zg7Uo&k*y2@bvwQj9LYtWtVLJ_*`0K$_=+I2BI~(ktt@@Uj8(-wKB!6RwLo{t$a~}F zn-ueNxZdfOezNOr-$*~KY5x3!`IY6a`A_f_mD2N^y}CqXR>Rd zqqGM$pBGhB#%4>F+P25VvKF)ttTMG6GQ6)w-E`})_p}8>vDYcs9>&o90B_Fr(4%zt zq<&P*D%V?MO53sPE4auWrK8o8IH9y59~fJajJ@9EChz=Iuw`1{%o>gA?r22qe?{A& zUMOmXzODP9n zHgjKE(R~V@WG&km8-P=j*b5H;UY<|JM_)TJZPADf?j3vL44z;}t;P)P<>PW~j)9(Q z;<37-8Px5N;h|@tef_tLR6v*=fPg5&xn z)K<8|G&1}ABhylmw!8AGkseEkfmlL^M6ie-IO&%Zu8+|68?GDpF!P_5Q!adDee_kJ zw#DWFtA&v{pP&UMky9OSgL_7MHs`CL#ZhuYM2q|J93{ME18bi{L-L7=sJU>4_gJB+ z4he$ZgIxaX!fVUCSCa*}*g}24U1C_me6TW}*I^;=8Ba&5-Qv*u`!J!>4Mw=mq8#b2 zj~N96YX-g3Em}K#O*dJh#S0sIW2TMop>!+5B07zS^ftTL!!@xZisq;I+*HS8ww#HM z{ktLBn|Z@@ISh%v1N>kK7~>aeKErl>qt}E&10@& zT$UZL?YQtk)wD<29a=CoQWtJ}cY$`c4ojlyae93d9=xvq>PQ*%t&mN~U3bW<>WNI= zE_R3yQ9V6sP-QRoal-Ctmpln9nLPtVi&~m{+Y(n<9{Pxed401)Ved9kC&skinD+NQ z`~OSLI_o&d;~4(Fy=2`&m&>FcCJaE-U2pf#6Yo+iTx{VzPAt07^(5_alg!lErSy^7Ba-8Z;N%RpoR5YnPPH4f zTf7vESqIJFHgQ{z2>3%hU+7oY0?YQ5TJQ$dZwj2REG}Xi8qA6@=2TVpUop*`Z4c96 zT-dI$zBF2VA+5E}zJne#Df??{R}F8%yCd2gc{|7YjA-|5c%17+%#%D?uxpSLfBl?(6Fb^;Hfu^^C2n>q4|6lag$!XpomNNyoD$&dqmnY7waDVC?-}5iN&fN1Fb18Q-xSkboixU$!vrFumr$ulsQ2m%g z<8v#ELCr74i0Yk&K!?RxOcbC%gp|c)Nu6`?_MesdNy4^zM zF)Y^I@zsOdshG~ro0V1M`8ye^0tbKz3(OfQS zN?~Yim{7!v?u(?G4i7wP(kXcX^zZJHko1bY20_r#AoQ3I8u8ng_~SA7f30& zqsKH@5cKB`;vP-&H&!BE7c(AdL3H+w2iFFbahcLTp9I%=Lk6JXYKkF~*GoP^@39_; zDoD~BH)ctXtyFXGC{O5@?Dd>mS(8m9Fhdgx6tZ&ko?(DBs9><697RT*cFkL&0-hhI zCcPeqibt~5&w}7bkC2JJJ}ySCu0i~27YuQmb5RT=DTZ+mR6TN9lE8u!6GXt?DfL{r zvau(L1ozPrRxZi)>}qj2lL3&Kz?l@D_>1AO_~OwGGsfW|#!l7m4W6M-l*qonw^!qo zy(naA6@%nP0AT?3vtL29fJ{UxS0=H&lN2dIH&dQv)ftAeeE+la-TIBsK_#24kBehs z?w&sy%<)RdJDCc&_6=9bLS}Fd6B3Ko+lCIY>I!xPF?(?w4JMUzl#@xp} z&>FamvwflD&xeuTF`&8AsOy}Lw)&|^gzM9X;O*(z<&$^~R(mhf3}9&rUf032NYL@v zGbzfZ!XKKgF>#$~IMD$nT4im{VEx(^&}dwdX!NVVxatQP`Cd;^i4anZgEH8w$}Zf! ztLG$_oln%klJAyEyv|_l8BLVInYUo^<`3((AZ$&w!=KHEH|DMKrgC2usv$xz%D3*x zrX#wcMM0l%rX*U`nlCkO0xz4ALUB^zEdBS7hMj3O`$w-K@l)Iok)0DwpUxMk_rE3? z!ro86F>P0Nw$4INO0}1=@Toq(UfQ1K=2^7X;&meG;r)yC9{ToO+RAq5mrT9tVwFTv zX`Pe##WMvgKO`!hVfl*NvK*dl?M9FbXL+>qsX*ZYC^G0rb^Pmq0)0;m{|g<$<{u2c zPi-)=ug)0dLpQL!s2G*EWQrNP6LAG1=p;VlPpY4a__jjDy{cIbeLL>T?c#Es&u*N-63Xv&1#Cg% z5lE0l%R&DDAjG2eTSY}t{M>VGO*83#J~NE5k04E81yz@cK>I%~Ks5yJ480A##5*KR zm72GxA7wPje_Nx-$nv5Lq2>#b+EKL5@Q#fT8Z{}}JbW0%$%&zwtn}YTdkH(@Q59Wr zqNtwbo!B)^6m3M$(4=67VQ<`3TyL1-eRo zMZ=ZlYtX6vGP^r7?!Z>kwQ_OeS~EpAKlbaP9c96uQuV=|+uVv|56{(T_6tol`<6ZT zJfbA`Z`M(xpsO@!dt50hCe6`#}uygM2{1{4fUJQz7Y3cWwFJxRo(YVnR38sI4x+xOiw z)28G1Na-7q`I@13>^|CHS~_MVMV2KSNiX>1x`iv)Usxr_3Y*Y1q4hodI46K2LmhB= zNozH}9^dnTR1ceI0(D5*a1ZaNeU}#vD2X?v0PdJW-;4XZrAfMHZSV8@x+k>A>EYVzPmWGpq zorb!8y|xhA=TpO5XwtHAt8tAN1PC2|kAlwSVnfr% z(RlAM{viQDK5<5nzVu+KbZ&msvEKKVF!}>BZd`9vNjRpAd7Q0_5Cu&4waf0bN=(v2 z*6wK;VaYwYu@7W|;U1ZMZy8{#?G8F3&>@7Qv45n)xlzocI%Z%EH~z7poQr>}h*ZLT zN$`hr=j`V5h+#c9_)XLVB#vh9tn2g3()2xr1}D~5`niFr%WVes+(=P!@$_EslJ6{D z5X>!_TP+Q0tKG>Cfy*G~ODAQ`p@}dOcV#1#Ue3MMG>?pG;2lVRZZpZzuTalwUBDlB zRXzl*V!y(%SgQHrB{UA*MCK|P+j0~wC?AZHs7#4Z=%f+{8=76O+M5mXFtCYL#2+Op zJ%Rvzxq}HS2|1?SB@rEOm~#>gyEFPs(cN(_n`VN>1{NbhDRLYGZaoNds+nv$wS4rQ zV&b;0nKS$RObzbpY<5<9CY}g^wD*zppL!B)d||B_OQjWiKvNeo_e;v2({rk$Q@%{0 zvK1RvtCtS(AKTkJ95~|bnpS*}v6~k(>md2${WC`Y5bMsB|4jG(zW}I+#gSjm(GG`I zKfU2op1LPN5TE|Jnpr|bnnxH(6r4-Q(x9GZ*gUA}O(K8`K#{l494j7>PIavJd9pF` zx37`~lxc~xz$Si)I{J7uK#SQ{C5cER$Vg5CWn!Fd5Gs++NZgoitKSN^Se^=~hjcUu z#2zlfgI}FFgAdw*&I(d)`S!`01TG3an`p>9AVk%`E<)w8CQR*U@3{j?u`E?R`%BaV zb8yckS;^zl>gc|WNqnST*n$I!9*_17uBWWNZIu?ZG8Uz5QuZ6}`D=pc&sQLpD zASHTxZ0D-|4Yz#C0n$H~zpBG8b^$!zG;5&W#Gxw|%DI9dE4EX!mNc?LasI*4qyZ(I zA(fqTx29s9y8Cl(w-J8|i>(Z9^vmQqA*%Ly7~)if3;^M1c7hY775xCmv^oa0>}*C% zq><`XQ?Dzs^0RL`vly!r^C(n?3)H&Ibw9J6u6*r3)ji=kGb*y)d;?5ZqS1Kb2f~RG z(MfyaxUHgd<>07T!H!v<8MUUhW?iG^c-$x&dT#PFYqamhrey=kC&F^uGoK<8$9>Cc z6ykQp!hGhUV#tB^H-!~&`iJR)NxLS!&n?=)0e&dnpz7C|nGnEgPyc`kZmteno$nrf z0XIYBYr1Qv*UrR)1Iw`P#d@RKjjddEhUSM^6S19L*_KnU7FW}>r7q1qXAmd7I>qp|_;3LUpRe*CwN{S59lwem?b zW0VlUg$1&5dmbT3$k2oQGm6rz$wf?-8X|^Jk#>*_KD}o_oU%=G^n4b*p5eFUn0l8T zYxi;wNh~Tdj!L7IX}Krg z|ImJy7Td~)(HlV|bgRX{V{2>kcW&q(FS`+)NOf4F!c*h$J4vL=bssLW=k{3{JwY*q z-CYUIq@u0+aXDZ)HpLf6eqmGK4sVpd=S}j-N855Ywa( zBj1d7tgL(i`lT$sV5h@-A+*PkuKLUonyLZ$h~(s1pA;l_KqJEK^aH z%7u%E1_}JU18q75but;Wiip3mWn~~lPc2*c+9;MUzu{Mf-6C{2nDra4%hAn4P~PGzti;CeM_~zqSCpTVQw@+jpU#PZx=tjLQ3!qmKUCMwwVDQ+wJPM`5|j-pPRpxo5zy}-^Yf$Q&cI4*R5_!( zzu2T}_=I}-W%p$MiGRS`P1*l^3lHVjf|y$4vw$848V66Kc$IWvL+{Xbk*corUkF&F zdLem0G&eb0`WzK!tAW>bhWglE_nkr8*vWZ`0f46S^)KKsE5P>7d3Z+6u�q`wm50 zCapEx^`ag&?`|NZt3$C&#laQC4%jBKNFXJkSpMMmLOe$*3m!_ggq|8R5Jho$DS&)x zQ8^j|z8%`hzC4P}Fj@X|FtUDlqC^xzn^?UW5Q8OJP;Re)Tv)2S!9^6 z$)r)GQ#xl`T3)VX3K^GB!j+bGIu(=fy#p~SRLxDjj3?4kM^CMGCeCgIffsFnnT6!9U$v%f8Es88!uF!aOp18iD5Mq8>le=(Hka%=n&GjL zeW849pqOP(;^uUfMJkHnNPv`yV=68nkH#n_mdWZvS&59Q@GSUh>m1|?l z+~NhAR$DjPTqW%Ad9ItlVolgaC%|`+y1u@(doiSytn-Q|@w>ItTb4y0G00#45sTY^ z3I~7Z0*NooYVfaJ2Z(6ILGXbf4mYY_J}w3q8_liO`ZH3<{Y{%ubbc<=&}wISG|RA0 zjW(*xPYnW8y`LI~Zni0}qFHE8|IR%%w=3GWCo*olLBkCDgCa#8oW~KAdymFwd>T&=L}MRI5R@j;T_v{@>I4Y1G5r!B@u zQqhAaRy(A1gZ2n6L1h1cxd$)+0sKP>e#@&sO0e`Wa`tyVAw96-GV%o>Xu~PX(Pw25 z7*w2{71k;(S8^66*+K@p2D*IpOYx39&H^eM`WCxktNH}ke`(B}n&_;mYiO5o@Cw>W zy&0bZ4_N;C#8Im^Qg;s$!@gseSHD)LeXw>g;x~zs& zAs-{3pX2LK%QX0-0_*2#En9N8O0*mM?B@p4DoE#ZeyAf#_*%IYNzdI}Hpf)5v^V&G z)C{nmKfXC4kbl54cQJe}g)xq-qJ>Zg?{0Nt`kTZ zftPnsU*5i{K;%!Sx1YLDUf8bdsYm;oy8+#dy~{+x}36I34@)p*~ne zOjhSHk;TeL?#Z@`T~!U2&xZ4Ov*4n#vyETOOjY}enznZ1Wrm$YCCxCGXlKsG2xCD~ zQkLmziRc@HcBqXzJFNoV#@FO;Jhmjvd?WA2CJ*4evno*ic`t+5HgT_$B&lQ zSD&a_HBLQU9Gy;WYJ9(9RZ;oir-^X;yWe4}&xAk*UZ6Y05RwSe5|+un_h&;9 z6UiB06J9dr$IJ3NVC2X;!Fcb=>5%TV6+x-jZjc?WdPI!q7)O|BY|_oq+)V-7au1)+ z>f}7QQPv;1$dFdpr4N^7v@ai353EtrbQO3NW-aIoe*ThJ(9YWFnUgn2?EBa%NP1m@ zEZ6nhYP+U6PUxO2EY{6x(VfK|xm5MYHZ^T!sK&S=!N|a}q&Aic1G{K?BwI9O-1?Me zQ8mZ?h@=pW6ANff4gua{7?a zzF=5#tvfrzI-WPSoleVjc?1^-Bo=C=xnZFnAZ3;CpJOYeMDNx_ zMwef?x2DO{n%bXQbDtHALhUFGS_c>Jn&U2YM|0KCJ{MaD(OY@EU_Q-Fb%u%RiPY}hq#@wm~kK^W-&Y z1@nxp_R|YO+PM?GwUu%k?0RSBG4LQenU^6mjn0GHK<3Up;`d6Dz3QJSd%1sqDB)etvLI zx5EcZCgH72RL>d4`OKL(U&`SbMDH=z_TZz(0ckc5U#JA=Rl7mdbS3xPBEJSgK znUA{wU60zdWP^X2+iCodQFpqbLtt`}K+3c~`e_5Y%h#25pH%rAqE9P7;q=;|{DCKX z{wk4X_lQ692XWRRnRbHj3|hKQMm|wRHm1@+mI|xfX1%Eju>j)bO0wgL~fdW2{D_<`dJ2CJVtpb!9H2vz!kH0e!Y|qI~hq zi}5ugQoMAgDei3~LiL!*BhYKa$`SSV-lbcn*V25%5_F_$9Ig_dJfJ=`$qhl*fLqlm z%iwV}KzReILY@eJ^4>)w4h~sQn`8)Gx|#C>L>=l)N;fC}pe-c~njfMr*)8@x#71Xf zSZ(SpgE_*dTSK?6Z(oC4Bf5%xowM@(FfsR6?+5;Y-=(b(LYUzl{vP|lh1C5UN&=Qf zqwTDsYi~K{Ih4QGCSj;?t(rZ1z{q7=k;&x0Kl76BqJsX0w)CrO+hTb{X8ZNwvBVxu z{wXv3r~xO)W_b1AgKC#y>h9$vb|*4=*eiW(W(7S;p>v- zH|zS3QeQMbfmjXPJR-8dmbN>1?~T8u8UI&z_?oltt@%$s-WNeS6aH4q^54&{jY7y> zI+N;jU>x&b9FcSDz!{aXS->h!Xi&NW8A7?D+4bw@g%X^N*>W@l32n^X&y3lgvXtft z6-F;b83WvVD!|!A_%RK-8Ch~_z>E#w;ZkAoLmytyIq`nIz>jJbWWVxq%_fhKtNW5` zhhR48Tm7Yx$ z@CuJaf%f#}kgxOogTHX6?%*hhF@6iQWv@xqe9aV3g%SGai{u7J0lp9SyP_2%XliPy z)=*@=v|!Q~Mv(j$obU2lu^T$9JhhHmLfAk6 zZEZc@Z}EynBO_6tLL0d40hdkJ2Zz>M!x#)xK~Y=lFN(L1p3)c6W}H@LB|+^4p%)7{ zCu?nS`HZ178eCiWS)fM1dfE4-yxxOqg&fC=l|F0|?3K2XxL0I==k7_OjU;RJ1))86 z7t1KyGI=TjE6)p}pl8c`xr0R?Q{c%F|1GsKu=PJI^)tTh2$h89yTp6CpT#)`F`QA; z!9`lkTNNn_gP9yC!l0&N>E@sst=#l9*cIPk5$Ge=)b6X;Pg$T}Fh5)SMc>0dJV|eC zY2Z32_;=r~V276MT*SKuB1Go`L+P?P44k5CCo3w8shsk1_*{tF>p-vjrSgy?Nn>dW zMyi}?w{$*7U8s*#Iciu7NgIM4jdOE~bSobwN2Mx=i9EPobKkFAGx6Q)^RbGcE=?C; zKoV&=xd3AW^ZW=qW(n6RmP|W%`HTr9#MA=9W)qUg@rtmJ${h`qrEKRnxOP4X z=%8$2ufvKF(@JuGS?Fmkr#FeR=`>5Vjatph2Qjz7Hg=Hj<8#>OQ}yimjJI{c4xp!w zl-7Zy64u9=ML}a`oS7<|LT-B@4lJ||BfIP4i&ap)z3Ae3dLy`f)$7~$nP%oyH6#}(*eLbykT;}XO6Ibm$1V5$EXXiJC4w&az)Jj3&bfBhLP1J34=(`S``; zHslO;_hFk8)u0vJl=81dS}T9 zv~{_gRB(bJA7cfsc|3k)-B!@fgi(lZ#KTqC>$|#g6CqV_Q|$U1*UXOi9Uq%XUG1$0jm-GD%PFz;|*IzTN zg|GCb-UpyGc|4OasOErC77AX~x%;BsvwC-0U?j2GjIwTD3&m%VcF42z*)4$@Gf0n? z+x_2ZYe)5MV#mjNP~2Zdjx8Qm?*uyhdP6`<`K1<&rQfCA1SV+KsHtmi8R^?xUUj~& z!_1c&M6P*GKrujTt-N(YGWZYO)wBOYur{h0N2`J3?!k+!wCG%<<5(x+RXxq7+ur~| zeMLGjsnzdNzG$cesB40YnNr{RdIy3_3#6EDan+XABo+p0vW>b@DVBuB`jISL&q~i( zg_~qH@AS5s0Jx`LxesdUL9y^f(YcqAWQEr>3y8yT_%<<()7L*>{DGb@BnJj$AG|ia$MWkBo5Y> zNc4?~iGAys4O}WhfPg7b#1GAcYejqmNOR#-ZK4d(xS>^Z;(PErXXfu1!2{}6$)1NB z(ZLKjcI12hC|!+Pa24*k#j5{}_NK*WgSE8jQcUdZA1u(?ssK1-Xi5e=z7iGz101Rx zQ9vkFIaAm1hgOCh*ctb*xy5CbNUs0pMq1O#;lv^watxicN$=k$LM+y^8&=aU+tZ24 z_G#=c>0U$OUplIS*==CS{kX9v1emu7PEqWP8nFC!whdJ(=xN`v>^sL)&9uA^Ts>HH4;d*8>0nSwxlS_ug#zUl6d^s zQ`UoIQk?oDU}X(UK{?ZT9xaXe*4@*~Nt7YIx3lx6t%3`% z!zTc7qFwhINCpTR1PbvgKPWQ9HMHUNA)%v*oVGT6LO3|03#5!du6V{oFz9c7++W&Y z=l|NB1bSoNq{+x7E9kC!r?!kVOy2WhO>oirE+cDqjw|; z)JA_>$mrcawtnSjUUKU$eg2zVv?Sj@<{CR2Y#5>9Ky_;OHuD!Orf%OL6^qfa5|@G) zU1I{Q*Gmk}V^SM&HVjDWAtM^L|7e%w1hE|S? z+*<0^<>$0RYeVfAedfa(6g$+Q?E%KY1Puc1qNXhCgE18?ZE8Hq z4+0P0c)KKKMF)xoRG)J~>1Nsw&%^YziyNB%*a{3D*Ab11joY4+P71D1VbNnUIdRwnUx{nF+$qN|sSU60#X@9Y+q z#N}BmH1N8^LG`lFd0tgI(HOyzxYz|LZ&}iJ%Gs8w`TZ7gy+^9& zggn#NgBh2%^5dC2Cv&BuNopL-`-VBK{l{U$;R2QG(>fWZu(|4;FagzU5ywS`!jMp1 zA<^fSApT=(bMPUDhF!8Wflo%T5qC#4E;R zdeqnXKJAa)&LR6nxW|(~M%63n4>TadyZnFwKHn{|b!EfOprJ{P*w)S)NW#{k|v8oBq#+IpM~kMy0<0hj4qISbi4JM1KfM-i>?A@S9ZbiK~*zI|eYafmG< zeC4b7Ek?l|QrtQyMM)dedT&ACC5FzIMB>{t{fV8sFf$OE1BFhV5K7XBKp#`wjS`jV|0N-If&1`3G9F^-KS$=N`wC_ zmxvBTh(41+c3rfzaLueecLIEoM^SAq_Wp9O-PtBSO$C>SYz&5(Z2buD(5TUy^2JJ8J z8w1bozmJ-IaTh3H2DOMvQ*h*md$?BX>sDgv#>Jbz-yN5&U2YDlovfNakh*Ab{eLul zWl-DQ6E0e$6e$kHDXxX$TAbqUE`{J;EVw(x9g4dKch^!RIK?%%1-*Iy_uiSz6IKJ&3^#C0&ww3crUNxn=>(9ikNX7wQC8`BEdugcc0(wE6 ze@3fjr+Lo^o|#X|GT);!&F;GpyFKUh)--{&;g!f_UlD0~I^vvkq?;^R%VGM6!JYrE zEONHjiJYETPN6TiS@Ztb*2|bK!;4CRr7lrofw{=|Oc0%%U zD3_LYp*>ww@`%`oNTDRfDF@J9pN&aKj^~B>rV|Db0^f6em-o+;0FLS#wSn2mtMLV? zqp`%p5$K|M2cP_TxfI&JW<(ztAXZ)W-+-R0PhDkX!p zM$ij*Yx7wd@poc1K2y1K=E&0F<56m*stglhSqwQSM46d`AX=XxG2}#7*In>^q#Iy| ziA%J|0P+>AwGV@=N|L*y?@0o?4R?E76rm~6^&6)uGn%z_FUEJ)>Iq?atm+z4B z{QNhC&`egN?fR?ai+=6hl>+k)0C4bOT#4aQJl2i_=(k}3=`VN`j{~2uw?b=u1UH}d zB&vYhr4I#IYi38l7oNPgV~X9O>RjB;Q`YnC4yA?Gy8?@C`8r>)Fc~^fIs8=3}|4Orplw`D$^g z@ytw+hfl2g?1d(4Dc7l&p8>L*U7i-W02wQ!#Ig%z@_81 zq9}AK8#QQ6B{BDx;5+490|vE?5`w6L#~YwkJ1akdX3F8Tz;Z^o%!pgCaDjqSP0Uc_P`ovpX#DG4llRf!>^lI`)AW|HN z($KUN*28AiYXYDzIdB#Xz%RZ1#Iw}Dq1!@}JVKNq6b7r^t-*PRjdmz#yTSgoe5#)5 z_P%kR{$P;jO35qE+paGWLDMv~KgF-Ko?dFP!!K`#QpC>)3jRkE{I`MnYZWuj?$y)L zrA-B#(_P#6lh3ax>9s=tq+29!p!~;hP|<((wJ*V#_titJBp3xL|E_6NPXww;=C?m- zgW)Y4I`2`urhHaO_C8G@d-d%Ra!pacucX+MF}hC^x^4hY^1dbmdzVAx)L?ojlds}G z`0fMt$|uNpn6d46M;fshVkSL8l^YGziX)gJ7y_NNQF$8B0(3T{(4O35_MJaki^C8Rs+W4Fh80>9zG!r-u(qr!nfuHtkxQ&Itlh3*rVXn zM^O)CsD^PB!Qpv&=;Y=VX;p4tLy3tg=r>w!uItihDE?R1l=W@pQ&w67N&Av$Gjc6S z$>bAB$nK{bnooR2&r=*4dM0%+l0Jz}hZ1(h)V{T|?|I2CWmR+k>WEhqM?WR}U09aS z@FWpqxW8x2srY2IRrp z>F{k=9oR8?Ao6M?a-Zh;Q1Eu6S2KutMbMXADkCv#7Nt}3*i zTW_yz0bwgPC$xu>EW`PpVEvdh`13@#qd?#9{8sTuMYud33_V(27)$>%=qrd_gkzfE}UjZ<8-Q22_1K(BE#EDNnJY(F#t zV48p`B+9O~-fhmQkEWC2b!7Qri|49D!pk_NQ;xqXZLSGH2?Nd=W8j7(7o8;}v9eSZ zKlr81z@?^lh@=7S9mum(>oCO$+d2SVMRT{GuJ1CV=1*lX*N5j6EdtrxZi|gHaV0J| z&4sR54WT7;X3YviH2`asG*0&`lbcr84xLaputmoB?T1+FR_RnbS+Q?G$fqGq=p z@T*=-ibBo6RESdGLYvQMtU#ZW#W~BWiEP@cGpnI5tX}t}a#1m+`?UYj7Stfhbt3k6 zH0`4L?ZOjUE;@a^>H=j zZ$96a>5<(b;hTodsQxVSvQdfmdbR%CYyA4rKTp03eoXtB@BA@Nw}AqP;`jh@Hyaoj zoENHO63aA~Ky35P6%zgnqxu?2XWZ7g#}30Zl1^k}nPN?vxTHDaI2%HF_hCa^>-PZf zhPxM@Mn|TP{S+y6PD)|pUCDdqO}P+w8&{C>yX!+lxh#pzQyYcz)GDbTUr>|E0F6;x z*UN^+R3ZYVdeMxnB%@`u&22t0Q@OrNRtbZsovh;E!Nm`8Dbk2?SHvLCdK*$mQ1Loo zfBc~A4+QS)$2<7ec*M1KNC$3welwFF^!p6p zbCUlh;3Ue}vGgi|^5wD~|Bw5y@G^@spTZ?QS^p{Qcu>^c-(I1nAz!qndK$`|_zYmpS0qHmoOkD7gFa@5Zc9;``3W5nxCn z)(b)2L)NMVvpXKu7$lEQSNJ(Js=J~U7pay3EmL`o3-nJG~9<1P9lL*#7Q3(5xh zE1Ca}Ik2CQz_X0EH!t17L-w1L)LlFmzOgwYXqXKlF><2wCeI(G_F{VPa#ya@!aouw z@)WewqFlm+k=H(W!%D(2eMAIPc7ZvcZgoi9fztD1oFdh)w)oheFQAHjtEi~@wUw3^ zDEe6nI>SUUMMv{X;~#~eVqW~eUQSVXrX55m!qPHR8ov3JEYY+p{$X|lPJ8ek*M#N8 zcNLNk$IVOyKNJWpkQ0^Q7G$krK;7-%tjzlPdu=~?BmD8-b3$DRPFnengL&@%_@VV> z9IV{BU^qj=4FFhYOHU+f6YS$41>VSp{f=Hw#Imrdk4`Uw3sW*z*t)=h)6ePoAci4R z7^0jjZh6kv@_hUyMGYr!M%mUbNOqNw73JNpcI6AFyLO;(FcZ>+JI;f|l(2glS=U&Akn z7EBY>H>L};CXl4#Gw#fpODh`OpZL`dYSJIKduv8NxZFE(dX!wZ7T@XBWmK+j%wOeX zS;S3YV?cz#W;a50cg7*ppW5#{vcyFyvq3c?;QGNO25I_a5xdroyLvm`*6H|67Du&5 zCzQFFtb8NaTj34yZ@hjT)gXNXSI=Ev4@4=|h@t;5)(hm7GkkkDe-(2Vk~#Z#mEdOw zvv>Sqh}A>oJ+Ui$-Z@jv`0|3Q5rw}6uSy-8`7R;XPx~AvA~x5mi#h!*1?&3s|1)~o zSXzeW{DxLl-Ocz5ntaB{O>q}uCZ%xT=i?tPwXD=u)pVl-90G3i62((-p5YB+7Wh^Q zu=BgSC3kQ3&iI=knb~`%Oo=i6R`u&(f_zA2YyFCz*&6G#4MnUTzEr9{B>*Y3?cUYt{|&E=a3Jli96v`q^y?9TUmubX>}*Mq?L=2>%} zg9;D@6fuEl`qOx#Xn5Da1?7nc)BnZNc@-5E3kU?MG%Gz??d3H{WL?O9zA=#H^h8## zlaI4D04}Sl$T;q5Hzaa}%4bafOKp#b5&93^tk>;TOi@!Z!SdBX*r?xD)odU-lf81Q zGGBYdVgAPf9(7c1L>1F>yXAsNbDc3o3WPfEH#S^)D~*6SiyLSK z^Xaqkd0bypE_P$+w#f8C5Gk_u1vFamPiIbByEZ>*7_cN#rY@~A6l+yC)VKFv^?wSY zOq#Zk#f%rq{!EIE-5%+@7CyB(8`z(#{s#XoRsBtT-oaxxzK~%sOpR7%B+<7!5d69n zXpbczl>R|PvT?)W)Xb>A09IFzbIub+c5|=VcD;ha(Fb%IUtH;L0@2l5@%_( z8XF)l#LI);?&r02qDu>4ZsYm@K7RwkA|HGBvfC8?Al$}vSGTd{vR?^7!^W68@bQc{ zj8R*zy(Jd_Dtn)1gKt;R3tsD{SpONGtMryEKE0u|?^1>JxGgT^pJ5sOfphBChg}?) z;0FohVJ*9M$6wY8gYD)|olPA1ul9!p?i~yyU}$b6mGtoNCG=$<4danNf>?)_Q)Y{Q zwFn8uBl!f$k&-o2K#LHDs)d;E9cU(EyJ�?i|h4^CWmGBABr9tTW^PDwY++xDfwh zUZ`69md5ADK0=QtYg@G|SVI`fD4q1P52$y%CBK?y^ut8t;K@!|UzUt06*zG~6#kCt z(5VEsF5c(J(sKU4HVzjtc`VXNJN_GuOl zPoE^S8KPhu31~WKgVCy12CI>&@V(#khTh1)haKYRs~Bn;(4n1@-M%S(6{w-%h) z7+$>vIzGesSsjmOo^+S=ef7cgEH>&+>d@oDe|8zUarGew9BC8OIHh%Ph z#vBh)?8)vaZB}X&YRU-KEKQ&^Pra77l+#-ki)bTn*4{LFI;LI% zf%Mu}hLQW`BEj~zL+nc{BMH*KJ5mQMmqjf?KqEHD?~Z1gfi9QG*8ZOkOi`j^SRkY= z^k;80o@ogtM@6$E|BoVzfSFiFk6g>ds@3Q~4U*Mr!v|QWkeE-(ZVw&hL$iFyu|47! zfqwiZG$7pn=f9~y4t~cQ+HM%3iEnI~HeA-{TH*F+3sDSRq)^EfvPP%(u(B-P6J(fs z{3LB+$H&YJ7RM9EBa?7?I`MW$xfuJt{|CTj{TU?v!%G3Ym0LNkP%N$`Kn+wOw3VPDL(Mhq3}O+7o$l0mZ*PWOxXJn@0#Rv^mEkM4YjRuvCN z3MX$fo-$$`QRssPM9dZ`uzm{JBfM1bg_TkVfJ^!nTN0KQk%LU_s(q&|$Ouui8) zna?%XvQx(C#TPN>Gkt;e2ak&oFlo+cY-m$DwmWbbH5@U0mya+nFNI)_GU7R5N!Wh{ zV5a3VZ^ycHfV1jYADlhD?`UuB-L1Eo^T61M_Bc6S9@&1`Dajj=LqTt46ZFy8(jv6J zR%)VnxdoKKn$*VajSl6IEfNrxEmXk7D&B6t3N%(`s~~}sy#hBNL~RtqIZCPtrB|b8 z6x9wQravuY5$@fa(guYie& z`(GVKrJa>b4BB#u`^G+1T~kwjh+3Vi z(L??VWJ1wLn+mx`0t@m(KSC>-9?b78D3Jh_G$NeG7 za(hzyAP!%XwKo=yT-tS09o?uaX)M;R%vngANSsH>)qNkju4I~3{K91)+=rbJDc3)x z_&aX&i;=dUm&t1KTRwnLDpA8KM#Bn2N^^OA#R5i);YrbbR8Ant=>@$ZN29dlT$$W) z{$k-hZ-w3b|^r>DZ- z38VQdZB#`3LUcMrWOS`=v2Jvi{SVx{=sk&?Rsfqyjv>vQN{^&?Vz#`Add9jXYHVBA z&@lyt7Av=|1G^`kc=b1$|1M7mIW$@J1MF3VP@IDRxV}#$y=e1r_SYZANts&fb22&{sX#Z5aQ zX~-T#XsbY@w04#d8S|0B;UMy>c1$ah1`V;eBghM$!ER62Sa)kOG&&42MRDr?;EnC$ zM1mB7$y9q3TM)bZ5~5yu`4TWy)EfCKCn=}^dyf7tj;|7I3**=uh2G$LM0=O!>%+NP z2y)tqX9@ygVwB(@VVD@@18g0d^NQDVoe2Vd2#Is&idmi;+`-tM1$km1rL4DalFIzU z_oYDLRog~~j<)|TN;#gCI9~ApTajn^@dgkRA05etdw1kb=?rxoohd8Iku4#w{l^4o zeKAD9_$SKeY>rS%K@*l#bcUUg+g{s`tGPXl?ymS$!ZpDhMUU>at1(y81O@-W6@Xik z{H7Mk?Ll1$3Uw83#6Nx#Hmru* z=Uac*&{D8mvYr}0_VBndgPR|Wux8|P>_SGpMlM;k=Y=8!Sgj=jQuh9rTVdI0rx7rw zMTPX^(8Vc&QUmndz32M~axwT$x3}9>i#R4*RH9 z2aL)&=XLJEnV80HflN>qziN|wv?(e_xc{#Ofc3d2IGV?Ir68@m zk6hnouw2p(9lsSwnsJ-}I~cXiiv98}QL;^YFj`$rAt5&H|HrSZ9G|B5*7JeVTJ-Ap z*j2}gedh#a=??E8ACC9s1NATkYLk^6bMeRMrBYf?!{m zx~s3Ml!l##q08@yr{}S~OouPhNCuPrPYnR?GVT3T6TByo!@u=-WW!=x$Iuus%XRr7ZAd5VujQ>dF{9z(pxL{~d+Ajqe>NSn`{1R$^h z|E)TiPmr@fHU!uDo~Lo*py59WlIpMY>|81fb7B`C(clm_(Ca^6x|h+Q0;vn}g`J0c z%f|ooX}e=wJ~K-MExP{`i4&6$TmG<^U*P{Q(Qe!CI^-0IP^WtW`bV8VPJ=GTnv1R{ zf9p*+ADq9%b%fF%oPsxgAHR0^JrSJyo%_hqZg;}ocOGv6Vg9+8GO1z}6%QnuFo6!# z?$}>~p+n5l0thxnw5}vxFBx1Hl4mKtJ{jhMOpQMx{9S6Y>9CCBk@;!;-k91lvuIT( zQxuH;Ge|D^VAlspg}(c4I}TIm@dcjb);oOa>EzO4^wd#B)a8z*mMuPbuxrtosGiK3 zC%KjEAm0oomdI{9<=yy=tFBJ^3*qOh ztTzfXnQd|@+5{qN+X|&GPPFI<`Sn+fuP4A)E~AUZxBha;e4k#isHJ~18`qexC*f~l zDYoe_uo_?VLZWsnlyT>r=T~*l0eBP2_57GOt%?zI8RYnCv~lCu9po&UuciAqhjnq? zbH(|(y+P0*I@OluPr7;kQoCF3b-tNGB>cta@rKy{iXNzTV12tX*Ef#QOWi^GYdb@! zLxM7eNl;^0sc9wWTYVW5WkhvcM_be!f zoL|eboRTH*-2y=s$qTu?wmFQ(n}>g9yhlNpuUH7TmKP`=E5JNXK0Nf@7ACRnF_Lj6 zcHx>SYDZeOv6Jb-Oll=A%_4ye!iKC_Ddi$ig6gLsl`}Z#p&6ONal7}c$4zM~m=?2;juAWzS|NnHnbd$3HIbAwLIF(x}hLT)}jy-Z5#!?`d;i z(w-k0!wpAN)nhv-ybv89Z>f!;kDx2R4F1=V4*yWzXQyne*ELZ~`KP!9?T7WZ2gK*A zV8V;qjVo^@2mdJJ2PnnXhd|>kZ6lx5-m`;;g}28ZM&J^Jrnp+mseq%O9DGis0F25_#MCFDUzwn@@*x-rav5gudD4UPj$w2IyC z)jQw(D0aB+TM`k8*}9%JpR*i);t-4G??NZpJ7e#k_ek)2(*&lpeNRrjroIoO{V{flKx5+b2EOg3$IxPNe)BPth>| zuw?oKc**Lk>z3T}pa}q~l(W3@_do$X1EsY!>c~N3a!S6ism+^R`%=FJ$!EFBPd0^z zrs-ebJHAb^*Xbf65VQF$R6@@j{Y>?r3G#d=diImyCNDBQUyzJ*a9C>+<)M#H4+Wmj z_*}I2! z_MPc-{sT;(WTo71XOG}7R8_AXQ_8qikkF9&F70dqkS?c6;y*s92qf<<>4d-WMVU>i&F*e_(fff?Fo+%6z+k zlqL|orr&qnTSGG*1b?pbDs1BxWj{ta?)ezyQnQ^oog|;{1ozxT!%Q{rjZ+fGj9-{* z@Y$}8&!6PuWJolg=}{|JCq;o-&o{8g2!zN5E#Qkh%^Dv_h(gRC4QmfF%Jx4p4%NK6 zoXtSnDExS5;(KDWU!K^_m^8NcWq^_KLLf1@+5Rdy!=Mna%gTqi^QU)eswr!P{_*#| zyM$gdN1m6#><+n=#5>VFPbkn~V$lfkzcy$;U$9tfW|1nGXeQ9*nBUHbr*0ysPF7D9=^2R@l_@`IFoK=o3Y|z? z6$vL0Uj-Wv2VNW=-xOXdk&xDLnqOhMKwZ-hsIYW)q)|Vhun9(9HFVE?5-M#keOz^*S9tKA4W%5;4vseCHU5)D#mTMramu0Ji-tIe;+hs zxVHSDb;GziTFHdh1eT)3=~9VRy8vRHNO774qUdA6xp8QOk2kbW0I>J@(|Y7h^-phSxq(zsg?}~|`f?RdV4nD> z6>oX@iODf3sh>2X0qGy=Pufa#&iPYF*#$$Fl}wCi((HVLvy6U5$HgzITf3??-Z0n> zQ(HlOpfdoh%d=N(io5uq3Ej(|{2$D0N08N8u>!V-C3IWd2$V1_3wnY*yKfy~4LP^# zpfzKWzZuzKz7%>d8NGh(ZfDMsUua_32PY1>hu>#A$9}>OO-=B*Og3fAZClr6*zZ&_me zv3yFMNa>FBj_6`0f<0P*>hRR&vw>0R#4K5OHHfTyo$UDo zI&^hx3n6;ey=y+;S1_=thug;A{={E>A3VHg246+kIF9<%;vsPl^HpqHo6kU3h!gVT z$TxhBsmOsp$$SLD3znqPQKM z_Bp;iGm+lkV%OpCUjTg`js(O#>M$8~JYdVfIcpPrwgm6&J= zsZ@;~$1^GujOTbwvv?ni?c7QqcZBmhE|4uu4^xprjjFAP)NgVTEN(Ge`1>Gv4=1<= zF+%gUeosAO*1|XO%p?2tv5C|6OgeE({B$UHklr2j&t*fR3U&wA?PkNGLA9g+;n!Ky zj)6o!PQfrc;~s;6WpFPOd?uQyz7wE%=!h)H4yG(M%Vs0w+MkwC#cLN7Tx$1c_2>w0 zA1;}N_;I&q>s>Ye)s0~j#VP88Xy_|TuO>+JFQS8VP3-*bC2%!AD54yI-$^rEd6k#h z_mVe9WTdWDw>w((W{{D-E;{jmjq9@;e#DyI|MQe}>vTc^3C#{&y%(LaCeYYA^vg*B z2~~G7&4xt;9VfPO?%~rCV?)C3n9jrXI^)M=zWA8qIdNgy+d47Oopo*SJAcKjvv)|R;Zv-=U^ygDM~_m72;f5xtV zTZObtJvwqm)6=I0Ikpf!(%^#EnLK-Kjk?yB37Vz(6r7daTbuQco2aF+Ln4uj`U|j8 z?#@gtyikp8h!<`@6y2MrK*qmv4xLO(*PA^7box@Q8%X!Nt;lqx@iB9@Mxj05*;!yU%O142<)&nJKDP;;h?^o+#S6`R zeCKPqmpPQ5`dmfO@65@+>ruf53 zWxb6kw9U-JgdU(x=v+6!v~9y5LR^|V+(}5Wy`g_rdf!B2b2oYeYBAOK{y0%d_HeM+ z4|v;EX6L>tXv60VyXQpxm6>TEz?boJT#oy7N6x>#A#MgXi+A%v%Sewa|A?v0c3)+v z@%dKyt=IDuLw<;DMH({~q&;aReaYb{^2+_j4}h=fX3LI_vsN7kSa3Oh-Ps=UJzmQY zx<4UpYd&JmKHZ$kbM%Sq-G1B3Zeue^2X(vBY1h0k^!V-qbvUlg4eD+<{U3KIE;}lx z&8+U4Rc4yU4gFH2%%@Llyl=sE$QD&qI^k7^`}N8e^0y-7(pKtg zz$bJQ*LHPu16C+wRFZ~Q32y%wLP?YkDFm@j=m;vav z-r3rG2^~3qq!;fsYzt7g&Kte5C0w9*{$jlL#!UV)F!P}7#a+D}mBDgn;ZaytbcE3j zy+yr$*+M?+g38e<3$rST;o*AZh&V0 z&eUD)m*V)k^1U42H(Gd904xSNYXg#(^SV5jIFVd3udTh(%jdUbBGX5}Qx~nmAj`w?VAdyX~K?@CZI$=HcIH-FUVA*xJcO z57zHdwP|%DzOzaXor;4kCQ_?~uEJ#!e+Q4)@sOq&JnNE>lA3R$kBT7m&NZgJRuI>Z zXrikTk(tGeQw>KR=LN7(E!mCDRBoM~fBTV1s4tQjoJChE3&d*q8x?ZU^sScR&&M?8 zeu?*g`F}4vsl+bYSawIYR<}m)UwMw9!)I2qIjJRraw+hfq&*We z)>(1{=(zBS`FJDzl!bbm7#}#hM>k%wg)8$8`>7)VU_F?+m0zbcOzfw$Tu-oCHFUIt zyGJxcywrx{d2Jv$OF*d1v_bpx)#is&jdot~sj>mhpqE z=L9kj*faS?ez*Rv4Sf)pWazNp5!h$Z`E}zfFg^CiS4q}c>2vL}<;Ok@UbamWO_VWo zqoQo*O952H8LByFl8Bpm8>z{MLRCYE{gR!BEh!1(rMLNxhE~4Y|tz&?i@5zX2C-& z4zM+_Uzqipa~)M#ldKARcUV3bm3}ByM7_mP{dCvh3C5HcGCC&7s#}Jnvb5kC8Y#zU zI#lNheAzhThRNP%e*DDmgUu$pR1&1JfPKQX!A!DM=Zgrrz=BpmheozD@whdJVWOq_isf_d_5S z;2_MpYLkwjisufQL8n~$#6Rb(*gR83nQ*iA=YkS5oTf#mkErOm=1rhNX+hI0O9Q$e%2KHJ>t)|)#g8ot5MAM9UpMvGrVI5YVy(K!3QX*i#&C!mU1)KSzgRTh48m-^kyaaXwC zRIe8j1~-~0J4xO+M~rLcmBh42bAdaYe)F4h6^m3MlsL9t=bu%7_1EZC7H3`@$3N8z z$>Bz(A}Q;7U!8*}|J7Cx80!-@dRBwAIbzuI0R09K z&#@ynRbKNJ#(PjudQMQ?@+oXV>P;a_>aCCOB=N7WU2vib|6NwBy6Yt3!zv-zKNZ8i z&<0~ihOkc81z23%oLScid6=>(O2ojcrmEQ8MP7RkmMv{c{k9QziNnjbby;{cA}8`0yq4%fG4;MWS!9z3u&O-WAcX*C0`X+7viMJ|>Baum zisWtFNgCGQwL6Yg#Uj^jZGg4*p=v3y*6|oDpM3H}gR6KXX#i z-xO0-#aRV8Y(aR9W<2EERW_FfQ>REZ(+H8^!w1B5e-4!;dZrPz1KcqZls@ z=b!chAC?XL`<&;c0m*-q$lK1Igf_qW=f~iK*C(Z1r*Jr0{-7o`$|ifoim*w48}%sl zxa8CQ$AlhHFUVE8lkTg8hVe~RZu|=)v`}jMg0kj#xYb1u$*vGl+F(gCfyUJCZY^Cf zFP>wnnQY*fiy^LfQG}wW(Ok|Zcd=cyK7gl3w~!}nAir*^Nxzbw$F%bud|9y)vs6A< zI|{5Vz=`p5;<0%Nf~h!=;99LHH*pYgoekS-?NfT!;Aq%`E5r^MXA*tA^c%UtZlJZH z*e?sH8lBT0MJOOsc72;LVr#uTU~vXr#^=4I`xs|VfRw8FThn!ahihH_sA)U6X5nx? zvW2e;MU?f>o~)PL`}=_tGewKOT4KtF>f<+OKFt@j-yehf#jt`v)UbW1t)nS?xvi(^ zp!XJo8)1+_1toB8RF9HdtDn)J4Tgpu8k>eeP{iFT_S~R)8AXJSS%FXK; zOpCA%nIA&0v;+^5NVa@#y2!xgMTwALGcA%I#hZrFGeJ0qx!s5uDd3S$a{_})BJ`WqlyumY#H`;ICr(I}c2KE-Q zIfXfNPpxb>s)(rtOC{T-sI>qo6_09kWx~l%4sy`gMN{cqIm7RR?kStuq z??p}H8;;$+*VjnXQG9d#x4~zp{Q#Yruj1+D=?1t5?bp0VVaC{v$aoxklxla~-=W7( z(wkE48{0<*J{i~L?+X1|WH&{;+dW-3Z`o&Y7)pLZGYs6$Tq+cabdNDGzWmE8m+Pc+ z)26o{!*w+pQdIeL{8`O^RJ@c&kqzzOAA0W+fw(Q%mgGSaRZaH0Epjvrb60%Wbn;?z zQnYZV6t4viy(k4#4pr<#OJda`bE5!E3-Q5{AgjQv9fxXN_w>)*J6>+IX*PY>?LT1j2xv0+f!T1>FlnXKj@8Txo@&ZEF++g z`bhpTbf@>M5fEEi9<>K_n#NHgU2Kb2<_nIvUySz~2+`)8)~FseE~ zCg^o{x7v14Ol$y@k1g;^k>U^f)O_rycA5Ea@PSj`iD*o-N=`_6L|H5q$L{f9g04TL zcLAq1BnZ{!u0v7xoK`jv=Tz?Q2iGGFRp_^+qA=ovUzOHh>^ub3XoG%S)!8;4OznSF z<7NV{RLcL$JX|Enuj|BYB-%&9L1{}W5FnW8(2!k_P0<#!J7WqU;E zvtPu7Lu(AONs%6p17SaF4@&mS=Z=V1Cd?+np|6pK^VJXjRAYuk#XRzYUtb z6ltX)f#SalcO?uKPz{{xP=?fVn>IB5+joIRNXok7k4ZUMr`zK+5)o)2^+u&y=qNZj zH4g%c0fh{xKij>swfIgksnuwU+>fE+pvR+@a%@>>FWqL3E`r>O>9?IEmnpM6BE5IrijTOr5mjl^+V)Xjr$b+YoXEg%MiQux;T5%MXsgI(sDCl=usQF zp1Ik0B?eEDw)A+V?4LjG4f&lUc-gS%O?N?^Yq1i{0c`95(l%h)x@Sc2brIL;b^tFHBsM)t`)5z@<>H9jConwARZ_oOLO>{}rH zFNW1ImG+H%QLqfM57$0*`<>UD*PcLo_js=YC#EXf8XJ>wlr1MYnFZJbQvgDi5 zbaPG^wa1qvkKGuiVYg?x-WWXC)1!)i=fWkKm8(N@=?ybW-)0b5H?EdDI>5u4tA1PU z?m`NNkNPe&KhHZ@HIM)ch*~03 zveUcAWz7#jaKVH?AV|L9w5OdFF52+cKorxQYs+Nk;ZEhg@|EfZol+7a$c42!N4lYB z$*P$r9}#-YZmuXFlkqhyU91Unq7v(KxLOQK-E@ep#fRSk2K zqF|mX>l-yKUp!UvpM_xkV`f!n4d8m$mUlZNJaGIM>-y5X;ysTCJo4{AmI;PJd$$Kb z(6#MgnY~3oevjpLHUX!=s}U-40@>KTvEmT1>s$&oDvvu5T7s&BUud%3ob*g@Di9sv zChKN@>VX2QHOa(l%H|BjVCTG)9*E~GB+{KIL@uo1NC?+}*aa{Z!%O?cy$L+IHF#vE zHiYjZG?~py5l|2ae;13k`Z1mmXxLQ@TllAnQcQd#b@3&u?Ujna@<*1Bn3rd(j%u4r zll(AT!XB!^G1_t7IR>ch?k_L(n@mX8gbiiY2fa7FGo8H4XsSt>pU6DO-P;WB$!m-+ z=LvVkNbJt3TK)3vx&ir@a{AwoIqaMLyg<^e&a9ff-OM0;Loxrpoy);aW$r;fT!%_~ z*cqaq*wo474^MO~!*E$XO=HO~C5|mQx3(_pL~9kV%SH8F?OAGyhecsz%cX*_QO;!L z&p&QS{(|E7)LMjBnoIkc`lM5qgVhUJGCXTW!e(Rhj|wsW6#_@Gge^wKjIZt9b{-uS zQLd^O!$UvZ1a9Y(nXf})4fKmRG$+R!6l7!sOY?4>HTQDZ2xUI~G;l%wl+Ud)%RK4-lLC-HZ?U-o+=u%Wf0*Hi2p)*s)9VMEOudY@!x1(n#tsWlH<@a8uT>+}gj&nosGo)Ipi<4q z@lPGCLP|T>}gcE>S6Hn+X7j$1v@t zM<}`r!Emka_xf|;SXo2K$RM7%OHRc}RntnI;S=Ve6J{wG)3CDoAJc#tp4?ty^<7(X zT0TbFG%I?#G37d>x!;Ym(wd6?D`x@Xc^&_*<l*m>q3f;nOS(M)-$rXG{1O+c;hnUc z4*9`Ow~R&BKCic2HD2}Mfe`_vmwws)N5$DZj`KLGVPNOCd*HO_Jths!wmpAlBYRJJ zvNawR1I?+qyeFG< z_VpLCxLY(i*QXWdr?!KY6g$Ja5B|?>L%;B1)(~7U^gx^NP8j!`2C8}7(pc64IN5gH zED3H#w%J6Qi4j6yLb{r${lrt&3cAS$iLXEADGV2X^sDx<(ewTlhI*^=E1QTxKY(Pn z40hopvs3d@gjQ$xTD`ZRaj4!9nv9xBgMnsd3xlxXHJXa7nqQkkg2eC6H>5a-@+(SG z<$> z8X8Lp?MC|tn;m&mT%zMc+Ux~QZ^F5+5I^6Z)oE5^!8rM>{$8J+{SHwe{yAFSE|AT3 zSorT$T@Qj_PLVq=84N#T&sRA62N4w>qyasLm8|v*ntnzIs*Ed|fg+rJvmHzP|#V^HNPam4L#w z|3le31y|Zd?V?G?PRB{d?%1}Sj-7P0Vs~uYwr$(CZQJ%<-`@MLI;ZN~oT_!VuHK1( z2V=T+fMEcR@4XEgUBLC)&*D3#sX^dUbvDMnH@?8>FSMp8aNJxUYrUBnt`)62E=0=m z93NBa9wvC=@{zIwQPuyfj%jVbu~%Or*sXYAu289aEVQgMr4Pw>jgIr{j8El8D={w& z5|ZIIxiH=X(1{q_D>ha(XtWp?Cns6j_sejpmr{AylkOY!=;YluVJQih8R^6zG}6>z zIFCBS`2H&wwX))b(TjN+LEPJ(h?E9BjoGyYnfq96-EeUB+jk18Y8Q5b%6re_pT5CV`;uB}x0l{-ySYc0z zLkFd!6sl-D%HE&gsA_Oo9V0s0pO67}Mp)>-qVf!WfPE&2gle#Eq@6deSuOTlQd8%5 zS10Gitprv#*vtQRW*aYCj5xd38Hpk>&YmG9GjEc%AU6?Q0O&5HX%>gt;b%v;!eOGq zLasJjP{c~41RY_cx$=}PChPWQu`;Fv5$?O1(b^oqgud@nbJs3lXg$4;%b>b3NZmf8AWSj_N+a@w&;szHO{O z!bdBg0?2iL(RsZ^GMk=y_$F?5`CapLMf#i}DH>Wspv6!~kc#2?ZwBz@=i@$j98=({ zjmDJ?X3rgAz!GvShH5mP0_J>WgO!_!(#K^z zUWjr)Gh#)om8((~*?Ey&c*a}#@AF&0f@y6V(0d`8Q3=^D*-9Ybx;T?`YB~c#{?(eP z#mwbTALWY^^i3RLPoygN8Y*Tv273mKcX?88zt%+KO@C)z3jP$Y1?%gN5K;Wtz!|Od z`B4YW3&$F&h&AYC8wKEEG%sS1If;V=+mYco5_bQb4aZMC=_#sIHSw9;6@d+A3017m z6%9#Kh^7o132W+{ia#_+1CbR@kj+gW7g8WN$o zvP`E<0vjyslS||IwC9uS({$I>%RPkN56*hYqX6tk7;E=9y((B{ z=UGd0>C6!@ch~#s@)Hz*y=Es|yA2z2Sg`B;ZSm{02Au7qp$2X{i>%@kEMS@Y zW&K?K_F0rbKejt$W{Z#5u$TnT=BCfw)Vy>6@9OX56%i#>@d){!IgV)9!)u<`;bc3! zUBen3gDV*Ey~tLEQ3v<$44uHm2)GgIlT~wZ=3*dukJnL&{aXJ=2y|d>MZd|-dmvE) ztM^JmYl>6s=8}n$&d~duycSuQRKrTo?|D+bX zbUBrt`VOdWwV0WH0B^((wi(>M^ zWoj9n)YfoR)Xan2{x?Am7?Ca!q3URe%#j;Uk0+=k{G#`^FkRlhOimEMT5hLxri!b$ zv*XPzdw$Thd>C;NxCAA#^GEKa_n)eDXOq>d*&aF@dp+-Dw|pmcqu41kT(^wBYy}Q0 zn1Lbx*7_2)j=|sWrN}Zkqy|L}GkfbvcHNE}xPK-mOt4=#?&>q^_DCtdY6%qwwG>TY zC6H_tCY)=^T>g4+_F6}Uw0r^JjzXw;L|`K#YfkNin_TT~b#D#Js|wo9L$w;bz8ef$ zhHW5bgN-%3uhJ1_=Nx{K{)eQRX!@oO@PYuUbV%BEK1Dxee#NSTgq{1#jP;*L3tI|Lc~yd zCjU=+s}^y6Le%b2G+BC)E$=Q7@;xRP%2tJa2v==9^Tg%4$uVI%bkb&TTk>_Fkx>vh zWzhmSHaE?6bUnL5j`jG-YD?hr!WF7GMFu6{RF`7?7ssvz!p3JOFE|*9^73N&ew17M z2^8mI^uIa>nCRjXVH;la>0)YAG4TO`N7R%mA-11!x^SrUap=q3n87{6c4ma^Y2s}A zO0tZj5}D+~4FoIl&L&VQi>yGX2+-iE5vm{=6S}dG@LcY(#6Pp0eA7}oRk68iwAB%K zvC*1&{QdNA?aq%<@r>Gdcdu|nQD4y%6hcNE+Q+aZIyNme|2pvHzO2=+f23+JT$^81KRxY^?qksRiYZGqC1BPwO&ohp`!&@ z(ts<%^OCPhUOJ|Tp!b`FwwseDL)z!N8Zq;cvk6-*q5ksRmPTjqs+3oA|MhFRLtI0Y zjP((rRz8o`Tu;?NSr@Buncj>SBVid#X|Q4v(7$_I<1)$wuCOUzo<}(n7eHg7<&y6M z%(b#7slTWt^wH8uTJH