From d8b75aeb5ece02afe35d3c811347c70e18a419b6 Mon Sep 17 00:00:00 2001 From: Trevor Grant Date: Mon, 28 Mar 2016 17:58:28 -0500 Subject: [PATCH 1/8] [FLINK][WIP][neural-nets] Initial Commit- working MLP --- .../flink/ml/neuralnetwork/ActivationFunction.scala | 8 ++++++++ .../org/apache/flink/ml/neuralnetwork/LossFunction.scala | 8 ++++++++ .../flink/ml/neuralnetwork/MultiLayerPerceptron.scala | 8 ++++++++ 3 files changed, 24 insertions(+) create mode 100644 flink-libraries/flink-ml/src/main/scala/org/apache/flink/ml/neuralnetwork/ActivationFunction.scala create mode 100644 flink-libraries/flink-ml/src/main/scala/org/apache/flink/ml/neuralnetwork/LossFunction.scala create mode 100644 flink-libraries/flink-ml/src/main/scala/org/apache/flink/ml/neuralnetwork/MultiLayerPerceptron.scala diff --git a/flink-libraries/flink-ml/src/main/scala/org/apache/flink/ml/neuralnetwork/ActivationFunction.scala b/flink-libraries/flink-ml/src/main/scala/org/apache/flink/ml/neuralnetwork/ActivationFunction.scala new file mode 100644 index 0000000000000..76fb4851c0b2c --- /dev/null +++ b/flink-libraries/flink-ml/src/main/scala/org/apache/flink/ml/neuralnetwork/ActivationFunction.scala @@ -0,0 +1,8 @@ +package org.apache.flink.ml.neuralnetwork + +/** + * Created by trevor on 3/5/16. + */ +class ActivationFunction { + +} diff --git a/flink-libraries/flink-ml/src/main/scala/org/apache/flink/ml/neuralnetwork/LossFunction.scala b/flink-libraries/flink-ml/src/main/scala/org/apache/flink/ml/neuralnetwork/LossFunction.scala new file mode 100644 index 0000000000000..756aedec1bedb --- /dev/null +++ b/flink-libraries/flink-ml/src/main/scala/org/apache/flink/ml/neuralnetwork/LossFunction.scala @@ -0,0 +1,8 @@ +package org.apache.flink.ml.neuralnetwork + +/** + * Created by trevor on 3/28/16. + */ +class LossFunction { + +} diff --git a/flink-libraries/flink-ml/src/main/scala/org/apache/flink/ml/neuralnetwork/MultiLayerPerceptron.scala b/flink-libraries/flink-ml/src/main/scala/org/apache/flink/ml/neuralnetwork/MultiLayerPerceptron.scala new file mode 100644 index 0000000000000..cb3c840947913 --- /dev/null +++ b/flink-libraries/flink-ml/src/main/scala/org/apache/flink/ml/neuralnetwork/MultiLayerPerceptron.scala @@ -0,0 +1,8 @@ +package org.apache.flink.ml.neuralnetwork + +/** + * Created by trevor on 3/5/16. + */ +class MultiLayerPerceptron { + +} From 8ce81d55cfffb70c8971fae47ea315990a7ea97f Mon Sep 17 00:00:00 2001 From: Theodore Vasiloudis Date: Mon, 4 Apr 2016 11:36:26 +0200 Subject: [PATCH 2/8] [FLINK-2157] [ml] Create evaluation framework for ML library --- .../flink/ml/classification/Classifier.scala | 30 ++++ .../apache/flink/ml/classification/SVM.scala | 2 +- .../apache/flink/ml/evaluation/Score.scala | 145 ++++++++++++++++++ .../apache/flink/ml/evaluation/Scorer.scala | 39 +++++ .../scala/org/apache/flink/ml/package.scala | 36 +++++ .../apache/flink/ml/pipeline/Predictor.scala | 44 +++++- .../apache/flink/ml/recommendation/ALS.scala | 51 ++++-- .../regression/MultipleLinearRegression.scala | 2 +- .../flink/ml/regression/Regressor.scala | 30 ++++ .../org/apache/flink/ml/MLUtilsSuite.scala | 10 ++ .../flink/ml/classification/SVMITSuite.scala | 45 +++--- .../flink/ml/evaluation/ScoreITSuite.scala | 115 ++++++++++++++ .../flink/ml/evaluation/ScorerITSuite.scala | 78 ++++++++++ .../flink/ml/recommendation/ALSITSuite.scala | 34 ++-- 14 files changed, 601 insertions(+), 60 deletions(-) create mode 100644 flink-libraries/flink-ml/src/main/scala/org/apache/flink/ml/classification/Classifier.scala create mode 100644 flink-libraries/flink-ml/src/main/scala/org/apache/flink/ml/evaluation/Score.scala create mode 100644 flink-libraries/flink-ml/src/main/scala/org/apache/flink/ml/evaluation/Scorer.scala create mode 100644 flink-libraries/flink-ml/src/main/scala/org/apache/flink/ml/regression/Regressor.scala create mode 100644 flink-libraries/flink-ml/src/test/scala/org/apache/flink/ml/evaluation/ScoreITSuite.scala create mode 100644 flink-libraries/flink-ml/src/test/scala/org/apache/flink/ml/evaluation/ScorerITSuite.scala diff --git a/flink-libraries/flink-ml/src/main/scala/org/apache/flink/ml/classification/Classifier.scala b/flink-libraries/flink-ml/src/main/scala/org/apache/flink/ml/classification/Classifier.scala new file mode 100644 index 0000000000000..c0ca6eb7bd2f3 --- /dev/null +++ b/flink-libraries/flink-ml/src/main/scala/org/apache/flink/ml/classification/Classifier.scala @@ -0,0 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.ml.classification + +import org.apache.flink.ml.pipeline.Predictor + +/** Trait that classification algorithms should implement + * + * @tparam Self Type of the implementing class + */ +trait Classifier[Self] extends Predictor[Self]{ + that: Self => + +} diff --git a/flink-libraries/flink-ml/src/main/scala/org/apache/flink/ml/classification/SVM.scala b/flink-libraries/flink-ml/src/main/scala/org/apache/flink/ml/classification/SVM.scala index c9544f904333b..40ee2a2fd6695 100644 --- a/flink-libraries/flink-ml/src/main/scala/org/apache/flink/ml/classification/SVM.scala +++ b/flink-libraries/flink-ml/src/main/scala/org/apache/flink/ml/classification/SVM.scala @@ -130,7 +130,7 @@ import breeze.linalg.{DenseVector => BreezeDenseVector, Vector => BreezeVector} * distance to the hyperplane for each example. Setting it to false will return the binary * class label (+1.0, -1.0) (Default value: '''false''') */ -class SVM extends Predictor[SVM] { +class SVM extends Classifier[SVM] { import SVM._ diff --git a/flink-libraries/flink-ml/src/main/scala/org/apache/flink/ml/evaluation/Score.scala b/flink-libraries/flink-ml/src/main/scala/org/apache/flink/ml/evaluation/Score.scala new file mode 100644 index 0000000000000..442f808cd1553 --- /dev/null +++ b/flink-libraries/flink-ml/src/main/scala/org/apache/flink/ml/evaluation/Score.scala @@ -0,0 +1,145 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.ml.evaluation + +import org.apache.flink.api.common.typeinfo.TypeInformation +import org.apache.flink.api.scala._ +import org.apache.flink.ml._ + +import scala.reflect.ClassTag + +/** + * Evaluation score + * + * Can be used to calculate a performance score for an algorithm, when provided with a DataSet + * of (truth, prediction) tuples + * + * @tparam PredictionType output type + */ +trait Score[PredictionType] { + def evaluate(trueAndPredicted: DataSet[(PredictionType, PredictionType)]): DataSet[Double] +} + +/** Traits to allow us to determine at runtime if a Score is a loss (lower is better) or a + * performance score (higher is better) + */ +trait Loss + +trait PerformanceScore + +/** + * Metrics expressible as a mean of a function taking output pairs as input + * + * @param scoringFct function to apply to all elements + * @tparam PredictionType output type + */ +abstract class MeanScore[PredictionType: TypeInformation: ClassTag]( + scoringFct: (PredictionType, PredictionType) => Double) + (implicit yyt: TypeInformation[(PredictionType, PredictionType)]) + extends Score[PredictionType] with Serializable { + + def evaluate(trueAndPredicted: DataSet[(PredictionType, PredictionType)]): DataSet[Double] = { + trueAndPredicted.map(yy => scoringFct(yy._1, yy._2)).mean() + } +} + +/** Scores aimed at evaluating the performance of regression algorithms + * + */ +object RegressionScores { + /** + * Mean Squared loss function + * + * Calculates (y1 - y2)^2^ and returns the mean. + * + * @return a Loss object + */ + def squaredLoss = new MeanScore[Double]((y1,y2) => (y1 - y2) * (y1 - y2)) with Loss + + /** + * Mean Zero One Loss Function also usable for score information + * + * Assigns 1 if sign of outputs differ and 0 if the signs are equal, and returns the mean + * + * @return a Loss object + */ + def zeroOneSignumLoss = new MeanScore[Double]({ (y1, y2) => + val sy1 = y1.signum + val sy2 = y2.signum + if (sy1 == sy2) 0 else 1 + }) with Loss + + /** Calculates the coefficient of determination, $R^2^$ + * + * $R^2^$ indicates how well the data fit the a calculated model + * Reference: [[http://en.wikipedia.org/wiki/Coefficient_of_determination]] + */ + def r2Score = new Score[Double] with PerformanceScore { + override def evaluate(trueAndPredicted: DataSet[(Double, Double)]): DataSet[Double] = { + val onlyTrue = trueAndPredicted.map(truthPrediction => truthPrediction._1) + val meanTruth = onlyTrue.mean() + + val ssRes = trueAndPredicted + .map(tp => (tp._1 - tp._2) * (tp._1 - tp._2)).reduce(_ + _) + val ssTot = onlyTrue + .mapWithBcVariable(meanTruth) { + case (truth: Double, meanTruth: Double) => (truth - meanTruth) * (truth - meanTruth) + }.reduce(_ + _) + + val r2 = ssRes + .mapWithBcVariable(ssTot) { + case (ssRes: Double, ssTot: Double) => + // We avoid dividing by 0 and just assign 0.0 + if (ssTot == 0.0) { + 0.0 + } + else { + 1 - (ssRes / ssTot) + } + } + r2 + } + } +} + +/** Scores aimed at evaluating the performance of classification algorithms + * + */ +object ClassificationScores { + /** Calculates the fraction of correct predictions + * + */ + def accuracyScore = { + new MeanScore[Double]((y1, y2) => if (y1.approximatelyEquals(y2)) 1 else 0) + with PerformanceScore + } + + /** + * Mean Zero One Loss Function + * + * Assigns 1 if outputs differ and 0 if they are equal, and returns the mean. + * + * @return a Loss object + */ + def zeroOneLoss = { + new MeanScore[Double]((y1, y2) => if (y1.approximatelyEquals(y2)) 0 else 1) with Loss + } +} + + diff --git a/flink-libraries/flink-ml/src/main/scala/org/apache/flink/ml/evaluation/Scorer.scala b/flink-libraries/flink-ml/src/main/scala/org/apache/flink/ml/evaluation/Scorer.scala new file mode 100644 index 0000000000000..cf2baa227ed8b --- /dev/null +++ b/flink-libraries/flink-ml/src/main/scala/org/apache/flink/ml/evaluation/Scorer.scala @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.flink.ml.evaluation + +import org.apache.flink.api.scala._ +import org.apache.flink.ml.common.{ParameterMap, WithParameters} +import org.apache.flink.ml.pipeline.{EvaluateDataSetOperation, Predictor} + +//TODO: Need to generalize type of Score (and evaluateOperation) +class Scorer(val score: Score[Double]) extends WithParameters { + + def evaluate[Testing, PredictorInstance <: Predictor[PredictorInstance]]( + testing: DataSet[Testing], + predictorInstance: PredictorInstance, + evaluateParameters: ParameterMap = ParameterMap.Empty) + (implicit evaluateOperation: EvaluateDataSetOperation[PredictorInstance, Testing, Double]): + DataSet[Double] = { + + val resultingParameters = predictorInstance.parameters ++ evaluateParameters + val predictions = predictorInstance.evaluate[Testing, Double](testing, resultingParameters) + score.evaluate(predictions) + } + +} diff --git a/flink-libraries/flink-ml/src/main/scala/org/apache/flink/ml/package.scala b/flink-libraries/flink-ml/src/main/scala/org/apache/flink/ml/package.scala index 554e155201045..8edc4f178c9d5 100644 --- a/flink-libraries/flink-ml/src/main/scala/org/apache/flink/ml/package.scala +++ b/flink-libraries/flink-ml/src/main/scala/org/apache/flink/ml/package.scala @@ -18,6 +18,7 @@ package org.apache.flink +import org.apache.flink.api.scala._ import org.apache.flink.api.common.functions.{RichFilterFunction, RichMapFunction} import org.apache.flink.api.common.typeinfo.TypeInformation import org.apache.flink.api.java.operators.DataSink @@ -29,6 +30,20 @@ import scala.reflect.ClassTag package object ml { + /** Pimp my [[Double]] to allow for approximate equals comparison + * + * @param double + */ + implicit class RichDouble(double: Double) { + def approximatelyEquals(other: Double, precision: Double = 1e-9): Boolean = { + if (scala.math.abs(double - other) < precision) { + true + } else { + false + } + } + } + /** Pimp my [[ExecutionEnvironment]] to directly support `readLibSVM` * * @param executionEnvironment @@ -49,6 +64,27 @@ package object ml { } } + /** Pimp my [[DataSet]] for [[Numeric]] to allow for the calculation of the mean value. + * + * @param dataSet + * @tparam T + */ + implicit class RichNumericDataSet[T : Numeric](dataSet: DataSet[T]) { + /** Calculates the mean value of a DataSet[T <: Numeric[T] ] + * + * @return A DataSet[Double] with the mean value as its only element + */ + def mean()(implicit num: Numeric[T], ttit: TypeInformation[(T, Int)]): DataSet[Double] = + dataSet.map(x => (x, 1)) + .reduce((xc, yc) => (num.plus(xc._1, yc._1), xc._2 + yc._2)) + .map(xc => num.toDouble(xc._1) / xc._2) + } + + /** Pimp my [[DataSet]] to minimize boilerplate for broadcast variables. + * + * @param dataSet + * @tparam T + */ implicit class RichDataSet[T](dataSet: DataSet[T]) { def mapWithBcVariable[B, O: TypeInformation: ClassTag]( broadcastVariable: DataSet[B])( diff --git a/flink-libraries/flink-ml/src/main/scala/org/apache/flink/ml/pipeline/Predictor.scala b/flink-libraries/flink-ml/src/main/scala/org/apache/flink/ml/pipeline/Predictor.scala index 9d11cff9e933c..8f86c4edb4878 100644 --- a/flink-libraries/flink-ml/src/main/scala/org/apache/flink/ml/pipeline/Predictor.scala +++ b/flink-libraries/flink-ml/src/main/scala/org/apache/flink/ml/pipeline/Predictor.scala @@ -19,10 +19,10 @@ package org.apache.flink.ml.pipeline import org.apache.flink.api.common.typeinfo.TypeInformation - import org.apache.flink.api.scala._ import org.apache.flink.ml._ -import org.apache.flink.ml.common.{FlinkMLTools, ParameterMap, WithParameters} +import org.apache.flink.ml.common.{FlinkMLTools, LabeledVector, ParameterMap, WithParameters} +import org.apache.flink.ml.math.{Vector => FlinkVector} /** Predictor trait for Flink's pipeline operators. * @@ -59,7 +59,7 @@ trait Predictor[Self] extends Estimator[Self] with WithParameters { predictor.predictDataSet(this, predictParameters, testing) } - /** Evaluates the testing data by computing the prediction value and returning a pair of true + /** Computes a prediction value for each example in the testing data and returns a pair of true * label value and prediction value. It is important that the implementation chooses a Testing * type from which it can extract the true label value. * @@ -72,8 +72,8 @@ trait Predictor[Self] extends Estimator[Self] with WithParameters { */ def evaluate[Testing, PredictionValue]( testing: DataSet[Testing], - evaluateParameters: ParameterMap = ParameterMap.Empty)(implicit - evaluator: EvaluateDataSetOperation[Self, Testing, PredictionValue]) + evaluateParameters: ParameterMap = ParameterMap.Empty) + (implicit evaluator: EvaluateDataSetOperation[Self, Testing, PredictionValue]) : DataSet[(PredictionValue, PredictionValue)] = { FlinkMLTools.registerFlinkMLTypes(testing.getExecutionEnvironment) evaluator.evaluateDataSet(this, evaluateParameters, testing) @@ -172,6 +172,37 @@ object Predictor { } } } + + /** [[EvaluateDataSetOperation]] which takes a [[PredictOperation]] to calculate a tuple + * of true label value and predicted label value, when provided with a DataSet of + * [[LabeledVector]]. + * + * @param predictOperation An implicit PredictOperation that takes a Flink Vector and returns + * a Double + * @tparam Instance The [[Predictor]] instance that calls the function + * @tparam Model The model that the calling [[Predictor]] uses for predictions + * @return An EvaluateDataSetOperation for LabeledVector + */ + implicit def LabeledVectorEvaluateDataSetOperation[Instance <: Predictor[Instance],Model] + (implicit predictOperation: PredictOperation[Instance, Model, FlinkVector, Double]) + : EvaluateDataSetOperation[Instance, LabeledVector, Double] = { + new EvaluateDataSetOperation[Instance, LabeledVector, Double] { + override def evaluateDataSet( + instance: Instance, + evaluateParameters: ParameterMap, + testing: DataSet[LabeledVector]) + : DataSet[(Double, Double)] = { + val resultingParameters = instance.parameters ++ evaluateParameters + val model = predictOperation.getModel(instance, resultingParameters) + + testing.mapWithBcVariable(model){ + (element, model) => { + (element.label, predictOperation.predict(element.vector, model)) + } + } + } + } + } } /** Type class for the predict operation of [[Predictor]]. This predict operation works on DataSets. @@ -233,8 +264,7 @@ trait PredictOperation[Instance, Model, Testing, Prediction] extends Serializabl * @param model The model representation of the prediciton algorithm * @return A label for the provided example of type [[Prediction]] */ - def predict(value: Testing, model: Model): - Prediction + def predict(value: Testing, model: Model): Prediction } /** Type class for the evaluate operation of [[Predictor]]. This evaluate operation works on diff --git a/flink-libraries/flink-ml/src/main/scala/org/apache/flink/ml/recommendation/ALS.scala b/flink-libraries/flink-ml/src/main/scala/org/apache/flink/ml/recommendation/ALS.scala index d8af42f34f132..947d26aa4ac9f 100644 --- a/flink-libraries/flink-ml/src/main/scala/org/apache/flink/ml/recommendation/ALS.scala +++ b/flink-libraries/flink-ml/src/main/scala/org/apache/flink/ml/recommendation/ALS.scala @@ -25,10 +25,13 @@ import org.apache.flink.api.scala._ import org.apache.flink.api.common.operators.Order import org.apache.flink.core.memory.{DataOutputView, DataInputView} import org.apache.flink.ml.common._ -import org.apache.flink.ml.pipeline.{FitOperation, PredictDataSetOperation, Predictor} +import org.apache.flink.ml.evaluation.RegressionScores +import org.apache.flink.ml.math.{DenseVector, BLAS} +import org.apache.flink.ml.pipeline._ import org.apache.flink.types.Value import org.apache.flink.util.Collector -import org.apache.flink.api.common.functions.{Partitioner => FlinkPartitioner, GroupReduceFunction, CoGroupFunction} +import org.apache.flink.api.common.functions.{Partitioner => FlinkPartitioner, + GroupReduceFunction, CoGroupFunction} import com.github.fommil.netlib.BLAS.{ getInstance => blas } import com.github.fommil.netlib.LAPACK.{ getInstance => lapack } @@ -147,7 +150,7 @@ class ALS extends Predictor[ALS] { } /** Sets the number of iterations of the ALS algorithm - * + * * @param iterations * @return */ @@ -157,7 +160,7 @@ class ALS extends Predictor[ALS] { } /** Sets the number of blocks into which the user and item matrix shall be partitioned - * + * * @param blocks * @return */ @@ -167,7 +170,7 @@ class ALS extends Predictor[ALS] { } /** Sets the random seed for the initial item matrix initialization - * + * * @param seed * @return */ @@ -178,7 +181,7 @@ class ALS extends Predictor[ALS] { /** Sets the temporary path into which intermediate results are written in order to increase * performance. - * + * * @param temporaryPath * @return */ @@ -407,12 +410,7 @@ object ALS { val uFactorsVector = uFactors.factors val iFactorsVector = iFactors.factors - val prediction = blas.ddot( - uFactorsVector.length, - uFactorsVector, - 1, - iFactorsVector, - 1) + val prediction = BLAS.dot(DenseVector(uFactorsVector), DenseVector(iFactorsVector)) (uID, iID, prediction) } @@ -425,6 +423,35 @@ object ALS { } } + implicit val evaluateRatings = new EvaluateDataSetOperation[ALS, (Int, Int, Double), Double] { + override def evaluateDataSet( + instance: ALS, + evaluateParameters: ParameterMap, + testing: DataSet[(Int, Int, Double)]) + : DataSet[(Double, Double)] = { + instance.factorsOption match { + case Some((userFactors, itemFactors)) => { + testing.join(userFactors, JoinHint.REPARTITION_HASH_SECOND).where(0).equalTo(0) + .join(itemFactors, JoinHint.REPARTITION_HASH_SECOND).where("_1._2").equalTo(0).map { + tuple => { + val (((_, _, truth), uFactors), iFactors) = tuple + + val uFactorsVector = uFactors.factors + val iFactorsVector = iFactors.factors + + val prediction = BLAS.dot(DenseVector(uFactorsVector), DenseVector(iFactorsVector)) + + (truth, prediction) + } + } + } + + case None => throw new RuntimeException("The ALS model has not been fitted to data. " + + "Prior to predicting values, it has to be trained on data.") + } + } + } + /** Calculates the matrix factorization for the given ratings. A rating is defined as * a tuple of user ID, item ID and the corresponding rating. * diff --git a/flink-libraries/flink-ml/src/main/scala/org/apache/flink/ml/regression/MultipleLinearRegression.scala b/flink-libraries/flink-ml/src/main/scala/org/apache/flink/ml/regression/MultipleLinearRegression.scala index ef06033599c09..85ea0353a7c15 100644 --- a/flink-libraries/flink-ml/src/main/scala/org/apache/flink/ml/regression/MultipleLinearRegression.scala +++ b/flink-libraries/flink-ml/src/main/scala/org/apache/flink/ml/regression/MultipleLinearRegression.scala @@ -90,7 +90,7 @@ import org.apache.flink.ml.pipeline.{PredictOperation, FitOperation, Predictor} * [[LearningRateMethod]] for all supported methods. * */ -class MultipleLinearRegression extends Predictor[MultipleLinearRegression] { +class MultipleLinearRegression extends Regressor[MultipleLinearRegression] { import org.apache.flink.ml._ import MultipleLinearRegression._ diff --git a/flink-libraries/flink-ml/src/main/scala/org/apache/flink/ml/regression/Regressor.scala b/flink-libraries/flink-ml/src/main/scala/org/apache/flink/ml/regression/Regressor.scala new file mode 100644 index 0000000000000..308d682aad57a --- /dev/null +++ b/flink-libraries/flink-ml/src/main/scala/org/apache/flink/ml/regression/Regressor.scala @@ -0,0 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.ml.regression + +import org.apache.flink.ml.pipeline.Predictor + +/** Trait that regression algorithms should implement + * + * @tparam Self Type of the implementing class + */ +trait Regressor[Self] extends Predictor[Self]{ + that: Self => + +} diff --git a/flink-libraries/flink-ml/src/test/scala/org/apache/flink/ml/MLUtilsSuite.scala b/flink-libraries/flink-ml/src/test/scala/org/apache/flink/ml/MLUtilsSuite.scala index f02f5ffc1c245..4489b6dc35e34 100644 --- a/flink-libraries/flink-ml/src/test/scala/org/apache/flink/ml/MLUtilsSuite.scala +++ b/flink-libraries/flink-ml/src/test/scala/org/apache/flink/ml/MLUtilsSuite.scala @@ -109,4 +109,14 @@ class MLUtilsSuite extends FlatSpec with Matchers with FlinkTestBase { tempFile.delete() } + + it should "correctly find the mean of a DataSet" in { + val env = ExecutionEnvironment.getExecutionEnvironment + + val ds = env.fromCollection(List(5.0, -5.0, 0.0)) + + val mean = ds.mean().collect().head + + mean should be (0.0 +- 1e-9) + } } diff --git a/flink-libraries/flink-ml/src/test/scala/org/apache/flink/ml/classification/SVMITSuite.scala b/flink-libraries/flink-ml/src/test/scala/org/apache/flink/ml/classification/SVMITSuite.scala index e6eb87330d479..37d954d8f731c 100644 --- a/flink-libraries/flink-ml/src/test/scala/org/apache/flink/ml/classification/SVMITSuite.scala +++ b/flink-libraries/flink-ml/src/test/scala/org/apache/flink/ml/classification/SVMITSuite.scala @@ -19,7 +19,7 @@ package org.apache.flink.ml.classification import org.scalatest.{FlatSpec, Matchers} -import org.apache.flink.ml.math.DenseVector +import org.apache.flink.ml.math.{Vector => FlinkVector, DenseVector} import org.apache.flink.api.scala._ import org.apache.flink.test.util.FlinkTestBase @@ -28,22 +28,30 @@ class SVMITSuite extends FlatSpec with Matchers with FlinkTestBase { behavior of "The SVM using CoCoA implementation" - it should "train a SVM" in { + def fixture = new { val env = ExecutionEnvironment.getExecutionEnvironment val svm = SVM(). - setBlocks(env.getParallelism). - setIterations(100). - setLocalIterations(100). - setRegularization(0.002). - setStepsize(0.1). - setSeed(0) + setBlocks(env.getParallelism). + setIterations(100). + setLocalIterations(100). + setRegularization(0.002). + setStepsize(0.1). + setSeed(0) + val trainingDS = env.fromCollection(Classification.trainingData) + val test = trainingDS.map(x => x.vector) + svm.fit(trainingDS) + } - val weightVector = svm.weightsOption.get.collect().head + it should "train a SVM" in { + + val f = fixture + + val weightVector = f.svm.weightsOption.get.collect().head weightVector.valueIterator.zip(Classification.expectedWeightVector.valueIterator).foreach { case (weight, expectedWeight) => @@ -52,23 +60,12 @@ class SVMITSuite extends FlatSpec with Matchers with FlinkTestBase { } it should "make (mostly) correct predictions" in { - val env = ExecutionEnvironment.getExecutionEnvironment - val svm = SVM(). - setBlocks(env.getParallelism). - setIterations(100). - setLocalIterations(100). - setRegularization(0.002). - setStepsize(0.1). - setSeed(0) + val f = fixture - val trainingDS = env.fromCollection(Classification.trainingData) - - val test = trainingDS.map(x => (x.vector, x.label)) - - svm.fit(trainingDS) + val test = f.trainingDS - val predictionPairs = svm.evaluate(test) + val predictionPairs = f.svm.evaluate(test) val absoluteErrorSum = predictionPairs.collect().map{ case (truth, prediction) => Math.abs(truth - prediction)}.sum @@ -98,7 +95,5 @@ class SVMITSuite extends FlatSpec with Matchers with FlinkTestBase { val rawPrediction = svm.predict(test).map(vectorLabel => vectorLabel._2).collect().head rawPrediction should be (15.0 +- 1e-9) - - } } diff --git a/flink-libraries/flink-ml/src/test/scala/org/apache/flink/ml/evaluation/ScoreITSuite.scala b/flink-libraries/flink-ml/src/test/scala/org/apache/flink/ml/evaluation/ScoreITSuite.scala new file mode 100644 index 0000000000000..3aaa916d89eae --- /dev/null +++ b/flink-libraries/flink-ml/src/test/scala/org/apache/flink/ml/evaluation/ScoreITSuite.scala @@ -0,0 +1,115 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +package org.apache.flink.ml.evaluation + +import org.apache.flink.api.scala._ +import org.apache.flink.test.util.FlinkTestBase +import org.scalatest.{FlatSpec, Matchers} + + +class ScoreITSuite extends FlatSpec with Matchers with FlinkTestBase { + + behavior of "Evaluation Score functions" + + it should "work for squared loss" in { + val env = ExecutionEnvironment.getExecutionEnvironment + + val yy = env.fromCollection(Seq((0.0, 1.0), (0.0, 0.0), (3.0, 5.0))) + + val loss = RegressionScores.squaredLoss + + val result = loss.evaluate(yy).collect() + + result.length shouldBe 1 + result.head shouldBe (1.6666666666 +- 1e-4) + } + + it should "work for zero one loss" in { + val env = ExecutionEnvironment.getExecutionEnvironment + + val yy = env.fromCollection(Seq(1.0 -> 1.0, 2.0 -> 2.0, 3.0 -> 4.0, 4.0 -> 5.0)) + + val loss = ClassificationScores.zeroOneLoss + + val result = loss.evaluate(yy).collect() + + result.length shouldBe 1 + result.head shouldBe (0.5 +- 1e9) + } + + it should "work for zero one loss applied to signs" in { + val env = ExecutionEnvironment.getExecutionEnvironment + + val yy = env.fromCollection(Seq[(Double,Double)]( + -2.3 -> 2.3, -1.0 -> -10.5, 2.0 -> 3.0, 4.0 -> -5.0)) + + val loss = RegressionScores.zeroOneSignumLoss + + val result = loss.evaluate(yy).collect() + + result.length shouldBe 1 + result.head shouldBe (0.5 +- 1e9) + } + + it should "work for accuracy score" in { + val env = ExecutionEnvironment.getExecutionEnvironment + + val yy = env.fromCollection(Seq(0.0 -> 0.0, 1.0 -> 1.0, 2.0 -> 2.0, 3.0 -> 2.0)) + + val accuracyScore = ClassificationScores.accuracyScore + + val result = accuracyScore.evaluate(yy).collect() + + result.length shouldBe 1 + result.head shouldBe (0.75 +- 1e9) + } + + it should "calculate the R2 score correctly" in { + val env = ExecutionEnvironment.getExecutionEnvironment + + // List of 50 (i, i + 1.0) tuples, where i the index + val valueList = Range.Double(0.0, 50.0, 1.0) zip Range.Double(0.0, 50.0, 1.0).map(_ + 1) + + val yy = env.fromCollection(valueList) + + val r2 = RegressionScores.r2Score + + val result = r2.evaluate(yy).collect() + + result.length shouldBe 1 + result.head shouldBe (0.99519807923169268 +- 1e9) + } + + it should "calculate the R2 score correctly for edge cases" in { + val env = ExecutionEnvironment.getExecutionEnvironment + + // List of 50 (0.0, 1.0) tuples + val valueList = Array.ofDim[Double](50) zip Array.ofDim[Double](50).map(_ + 1.0) + + val yy = env.fromCollection(valueList) + + val r2 = RegressionScores.r2Score + + val result = r2.evaluate(yy).collect() + + result.length shouldBe 1 + result.head shouldBe (0.0 +- 1e9) + } +} diff --git a/flink-libraries/flink-ml/src/test/scala/org/apache/flink/ml/evaluation/ScorerITSuite.scala b/flink-libraries/flink-ml/src/test/scala/org/apache/flink/ml/evaluation/ScorerITSuite.scala new file mode 100644 index 0000000000000..0d1b481eaa43a --- /dev/null +++ b/flink-libraries/flink-ml/src/test/scala/org/apache/flink/ml/evaluation/ScorerITSuite.scala @@ -0,0 +1,78 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.flink.ml.evaluation + +import org.apache.flink.api.scala._ +import org.apache.flink.ml.common.ParameterMap +import org.apache.flink.ml.preprocessing.StandardScaler +import org.apache.flink.ml.regression.MultipleLinearRegression +import org.apache.flink.ml.regression.RegressionData._ +import org.apache.flink.test.util.FlinkTestBase +import org.scalatest.{FlatSpec, Matchers} + +class ScorerITSuite extends FlatSpec with Matchers with FlinkTestBase { + + behavior of "the Scorer class" + + def fixture = new { + val env = ExecutionEnvironment.getExecutionEnvironment + + val loss = RegressionScores.squaredLoss + + val scorer = new Scorer(loss) + + val mlr = MultipleLinearRegression() + + val parameters = ParameterMap() + + parameters.add(MultipleLinearRegression.Stepsize, 1.0) + parameters.add(MultipleLinearRegression.Iterations, 10) + parameters.add(MultipleLinearRegression.ConvergenceThreshold, 0.001) + + val inputDS = env.fromCollection(data) + } + + it should "work for squared loss" in { + + val f = fixture + + f.mlr.fit(f.inputDS, f.parameters) + + val evaluationDS = f.inputDS.map(x => (x.vector, x.label)) + + val mse = f.scorer.evaluate(evaluationDS, f.mlr).collect().head + + mse should be < 2.0 + } + + it should "be possible to obtain scores for a chained predictor" in { + val f = fixture + + val scaler = StandardScaler() + + val chainedMLR = scaler.chainPredictor(f.mlr) + + chainedMLR.fit(f.inputDS, f.parameters) + + val evaluationDS = f.inputDS.map(x => (x.vector, x.label)) + + val mse = f.scorer.evaluate(evaluationDS, chainedMLR).collect().head + + mse should be < 2.0 + } +} diff --git a/flink-libraries/flink-ml/src/test/scala/org/apache/flink/ml/recommendation/ALSITSuite.scala b/flink-libraries/flink-ml/src/test/scala/org/apache/flink/ml/recommendation/ALSITSuite.scala index 9c241fdf6e29a..46b324bb9dcae 100644 --- a/flink-libraries/flink-ml/src/test/scala/org/apache/flink/ml/recommendation/ALSITSuite.scala +++ b/flink-libraries/flink-ml/src/test/scala/org/apache/flink/ml/recommendation/ALSITSuite.scala @@ -18,12 +18,14 @@ package org.apache.flink.ml.recommendation -import org.scalatest._ - import scala.language.postfixOps +import org.scalatest._ + +import org.apache.flink.api.scala.ExecutionEnvironment import org.apache.flink.api.scala._ import org.apache.flink.test.util.FlinkTestBase +import Recommendation._ class ALSITSuite extends FlatSpec @@ -34,8 +36,7 @@ class ALSITSuite behavior of "The alternating least squares (ALS) implementation" - it should "properly factorize a matrix" in { - import Recommendation._ + def fixture = new { val env = ExecutionEnvironment.getExecutionEnvironment @@ -49,28 +50,33 @@ class ALSITSuite als.fit(inputDS) - val testData = env.fromCollection(expectedResult.map{ - case (userID, itemID, rating) => (userID, itemID) - }) + val evaluationData = env.fromCollection(expectedResult) + + val testData = evaluationData.map(idsAndRating => (idsAndRating._1 , idsAndRating._2)) + } + + it should "properly factorize a matrix" in { + + val f = fixture - val predictions = als.predict(testData).collect() + val predictions = f.als.predict(f.testData).collect() predictions.length should equal(expectedResult.length) val resultMap = expectedResult map { - case (uID, iID, value) => (uID, iID) -> value + case (uID, iID, rating) => (uID, iID) -> rating } toMap predictions foreach { - case (uID, iID, value) => { - resultMap.isDefinedAt((uID, iID)) should be(true) + case (uID, iID, rating) => { + resultMap.isDefinedAt((uID, iID)) should be (true) - value should be(resultMap((uID, iID)) +- 0.1) + rating should be(resultMap((uID, iID)) +- 0.1) } } - val risk = als.empiricalRisk(inputDS).collect().head + val risk = f.als.empiricalRisk(f.inputDS).collect().head - risk should be(expectedEmpiricalRisk +- 1) + risk should be (expectedEmpiricalRisk +- 1) } } From ec1e65a31d80b33589b73619f2a5dd0a8e09c568 Mon Sep 17 00:00:00 2001 From: Trevor Grant Date: Fri, 15 Apr 2016 17:37:51 -0500 Subject: [PATCH 3/8] Add Splitter Pre-processing --- .../org/apache/flink/ml/preprocessing/Splitter.scala | 8 ++++++++ .../apache/flink/ml/preprocessing/SplitterITSuite.scala | 8 ++++++++ 2 files changed, 16 insertions(+) create mode 100644 flink-libraries/flink-ml/src/main/scala/org/apache/flink/ml/preprocessing/Splitter.scala create mode 100644 flink-libraries/flink-ml/src/test/scala/org/apache/flink/ml/preprocessing/SplitterITSuite.scala diff --git a/flink-libraries/flink-ml/src/main/scala/org/apache/flink/ml/preprocessing/Splitter.scala b/flink-libraries/flink-ml/src/main/scala/org/apache/flink/ml/preprocessing/Splitter.scala new file mode 100644 index 0000000000000..6db256419b3d7 --- /dev/null +++ b/flink-libraries/flink-ml/src/main/scala/org/apache/flink/ml/preprocessing/Splitter.scala @@ -0,0 +1,8 @@ +package org.apache.flink.ml.preprocessing + +/** + * Created by trevor on 4/13/16. + */ +class Splitter { + +} diff --git a/flink-libraries/flink-ml/src/test/scala/org/apache/flink/ml/preprocessing/SplitterITSuite.scala b/flink-libraries/flink-ml/src/test/scala/org/apache/flink/ml/preprocessing/SplitterITSuite.scala new file mode 100644 index 0000000000000..b9c715c39ff15 --- /dev/null +++ b/flink-libraries/flink-ml/src/test/scala/org/apache/flink/ml/preprocessing/SplitterITSuite.scala @@ -0,0 +1,8 @@ +package org.apache.flink.ml.preprocessing + +/** + * Created by trevor on 4/15/16. + */ +class SplitterITSuite { + +} From 3ecdc3818dd11a847136510dabe96f444924d319 Mon Sep 17 00:00:00 2001 From: Trevor Grant Date: Fri, 15 Apr 2016 17:40:38 -0500 Subject: [PATCH 4/8] Add Splitter Pre-processing --- .../flink/ml/preprocessing/Splitter.scala | 215 +++++++++++++++++- .../ml/preprocessing/SplitterITSuite.scala | 73 +++++- 2 files changed, 280 insertions(+), 8 deletions(-) diff --git a/flink-libraries/flink-ml/src/main/scala/org/apache/flink/ml/preprocessing/Splitter.scala b/flink-libraries/flink-ml/src/main/scala/org/apache/flink/ml/preprocessing/Splitter.scala index 6db256419b3d7..5249d8562e637 100644 --- a/flink-libraries/flink-ml/src/main/scala/org/apache/flink/ml/preprocessing/Splitter.scala +++ b/flink-libraries/flink-ml/src/main/scala/org/apache/flink/ml/preprocessing/Splitter.scala @@ -1,8 +1,215 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.apache.flink.ml.preprocessing -/** - * Created by trevor on 4/13/16. - */ -class Splitter { +import org.apache.flink.api.common.typeinfo.{TypeInformation, BasicTypeInfo} +import org.apache.flink.api.java.Utils +import org.apache.flink.api.scala. DataSet +import org.apache.flink.api.scala.utils._ + +import org.apache.flink.ml.common.{FlinkMLTools, ParameterMap, WithParameters} +import _root_.scala.reflect.ClassTag + +object Splitter { + + case class TrainTestDataSet[T: TypeInformation : ClassTag](training: DataSet[T], + testing: DataSet[T]) + + case class TrainTestHoldoutDataSet[T: TypeInformation : ClassTag](training: DataSet[T], + testing: DataSet[T], + holdout: DataSet[T]) + // -------------------------------------------------------------------------------------------- + // randomSplit + // -------------------------------------------------------------------------------------------- + /** + * Split a DataSet by the probability fraction of each element. + * + * @param input DataSet to be split + * @param fraction Probability that each element is chosen, should be [0,1] without + * replacement, and [0, ∞) with replacement. While fraction is larger + * than 1, the elements are expected to be selected multi times into + * sample on average. This fraction refers to the first element in the + * resulting array. + * @param precise Sampling by default is random and can result in slightly lop-sided + * sample sets. When precise is true, equal sample set size are forced, + * however this is somewhat less efficient. + * @param seed Random number generator seed. + * @return An array of two datasets + */ + + def randomSplit[T: TypeInformation : ClassTag]( input: DataSet[T], + fraction: Double, + precise: Boolean = false, + seed: Long = Utils.RNG.nextLong()) + : Array[DataSet[T]] = { + import org.apache.flink.api.scala._ + + val indexedInput: DataSet[(Long, T)] = input.zipWithIndex + + val leftSplit: DataSet[(Long, T)] = precise match { + case false => indexedInput.sample(false, fraction, seed) + case true => { + val count = indexedInput.count() + val numOfSamples = math.round(fraction * count).toInt + indexedInput.sampleWithSize(false, numOfSamples, seed) + } + } + + val rightSplit: DataSet[(Long, T)] = indexedInput.leftOuterJoin[(Long, T)](leftSplit) + .where(0) + .equalTo(0) { + (full: (Long,T) , left: (Long, T)) => (if (left == null) full else null) + } + .filter( o => o != null ) + Array(leftSplit.map(o => o._2), rightSplit.map(o => o._2)) + } + + // -------------------------------------------------------------------------------------------- + // multiRandomSplit + // -------------------------------------------------------------------------------------------- + /** + * Split a DataSet by the probability fraction of each element of a vector. + * + * @param input DataSet to be split + * @param fracArray An array of PROPORTIONS for splitting the DataSet. Unlike the + * randomSplit function, number greater than 1 do not lead to over + * sampling. The number of splits is dictated by the length of this array. + * The number are normalized, eg. Array(1.0, 2.0) would yield + * two data sets with a 33/66% split. + * @param precise Sampling by default is random and can result in slightly lop-sided + * sample sets. When precise is true, equal sample set size are forced, + * however this is somewhat less efficient. + * @param seed Random number generator seed. + * @return An array of DataSets whose length is equal to the length of fracArray + */ + def multiRandomSplit[T: TypeInformation : ClassTag](input: DataSet[T], + fracArray: Array[Double], + precise: Boolean = false, + seed: Long = Utils.RNG.nextLong()) + : Array[DataSet[T]] = { + val splits = fracArray.length + val output = new Array[DataSet[T]](splits) + val aggs = fracArray.scanRight((0.0))( _ + _ ) + val fracs = fracArray.zip(aggs).map( o => o._1 / o._2) + + //// + var tempDS = input + for (k <- 0 to splits-2){ + println( (splits -k)) + var temp = Splitter.randomSplit(tempDS, fracs(k), true) + output(k) = temp(0) + tempDS = temp(1) + } + output(splits-1) = tempDS + output + } + + // -------------------------------------------------------------------------------------------- + // kFoldSplit + // -------------------------------------------------------------------------------------------- + /** + * Split a DataSet into an array of TrainTest DataSets + * + * @param input DataSet to be split + * @param kFolds The number of TrainTest DataSets to be returns. Each 'testing' will be + * 1/k of the dataset, randomly sampled, the training will be the remainder + * of the dataset. The DataSet is split into kFolds first, so that no + * observation will occurin in multiple folds. + * @param precise Sampling by default is random and can result in slightly lop-sided + * sample sets. When precise is true, equal sample set size are forced, + * however this is somewhat less efficient. + * @param seed Random number generator seed. + * @return An array of TrainTestDataSets + */ + def kFoldSplit[T: TypeInformation : ClassTag](input: DataSet[T], + kFolds: Int, + precise: Boolean = false, + seed: Long = Utils.RNG.nextLong()) + : Array[TrainTestDataSet[T]] = { + + val fracs = Array.fill(kFolds)(1.0) + val dataSetArray = multiRandomSplit(input, fracs, precise, seed) + + dataSetArray.zipWithIndex.map( ds => TrainTestDataSet(ds._1, + unionDataSetArray(dataSetArray.filter(_ != ds._1))) ) + + } + + def unionDataSetArray[T: TypeInformation : ClassTag](dsa : Array[DataSet[T]]): DataSet[T] = { + var dsu = dsa(0) + for (k <- 1 to dsa.length-1) { + dsu = dsu.union(dsa(k)) + } + dsu + } + + // -------------------------------------------------------------------------------------------- + // trainTestSplit + // -------------------------------------------------------------------------------------------- + /** + * A wrapper for randomSplit that yields a TrainTestDataSet + * + * @param input DataSet to be split + * @param fraction Probability that each element is chosen, should be [0,1] without + * replacement, and [0, ∞) with replacement. While fraction is larger + * than 1, the elements are expected to be selected multi times into + * sample on average. This fraction refers to the training element in + * TrainTestSplit + * @param precise Sampling by default is random and can result in slightly lop-sided + * sample sets. When precise is true, equal sample set size are forced, + * however this is somewhat less efficient. + * @param seed Random number generator seed. + * @return A TrainTestDataSet + */ + def trainTestSplit[T: TypeInformation : ClassTag]( input: DataSet[T], + fraction: Double = 0.6, + precise: Boolean = false, + seed: Long = Utils.RNG.nextLong()) + : TrainTestDataSet[T] = { + val dataSetArray = randomSplit(input, fraction, precise, seed) + TrainTestDataSet(dataSetArray(0), dataSetArray(1)) + } + // -------------------------------------------------------------------------------------------- + // trainTestHoldoutSplit + // -------------------------------------------------------------------------------------------- + /** + * A wrapper for multiRandomSplit that yields a TrainTestHoldoutDataSet + * + * @param input DataSet to be split + * @param fracArray An array of length 3, where the first element specifies the size of the + * training set, the second element the testing set, and the third + * element is the holdout set. These are proportional and will be + * normalized internally. + * @param precise Sampling by default is random and can result in slightly lop-sided + * sample sets. When precise is true, equal sample set size are forced, + * however this is somewhat less efficient. + * @param seed Random number generator seed. + * @return A TrainTestDataSet + */ + def trainTestHoldoutSplit[T: TypeInformation : ClassTag](input: DataSet[T], + fracArray: Array[Double] = Array(0.6,0.3,0.1), + precise: Boolean = false, + seed: Long = Utils.RNG.nextLong()) + : TrainTestHoldoutDataSet[T] = { + // throw error if fracArray isn't length = 3 + val dataSetArray = multiRandomSplit(input, fracArray, precise, seed) + TrainTestHoldoutDataSet(dataSetArray(0), dataSetArray(1), dataSetArray(2)) + } } diff --git a/flink-libraries/flink-ml/src/test/scala/org/apache/flink/ml/preprocessing/SplitterITSuite.scala b/flink-libraries/flink-ml/src/test/scala/org/apache/flink/ml/preprocessing/SplitterITSuite.scala index b9c715c39ff15..1138505fe2a73 100644 --- a/flink-libraries/flink-ml/src/test/scala/org/apache/flink/ml/preprocessing/SplitterITSuite.scala +++ b/flink-libraries/flink-ml/src/test/scala/org/apache/flink/ml/preprocessing/SplitterITSuite.scala @@ -1,8 +1,73 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.flink.ml.preprocessing -/** - * Created by trevor on 4/15/16. - */ -class SplitterITSuite { +import org.apache.flink.api.scala.ExecutionEnvironment +import org.apache.flink.api.scala._ +import org.apache.flink.test.util.FlinkTestBase +import org.scalatest.{Matchers, FlatSpec} +import org.apache.flink.ml.math.Vector +import org.apache.flink.api.scala.utils._ + + +class SplitterITSuite extends FlatSpec + with Matchers + with FlinkTestBase { + + behavior of "Flink's DataSet Splitter" + + import MinMaxScalerData._ + + it should "result in datasets with no elements in common and all elements used" in { + val env = ExecutionEnvironment.getExecutionEnvironment + + val dataSet = env.fromCollection(data) + + val splitDataSets = Splitter.randomSplit(dataSet.zipWithIndex, 0.5) + + (splitDataSets(0).count() + splitDataSets(1).count()) should equal(dataSet.count()) + + + splitDataSets(0).join(splitDataSets(1)).where(0).equalTo(0).count() should equal(0) + } + + it should "result in datasets of an expected size when precise" in { + val env = ExecutionEnvironment.getExecutionEnvironment + + val dataSet = env.fromCollection(data) + + val splitDataSets = Splitter.randomSplit(dataSet, 0.5) + + val expectedLength = dataSet.count().toDouble * 0.5 + + splitDataSets(0).count().toDouble should equal(expectedLength +- 5.0) + } + + it should "result in expected number of datasets" in { + val env = ExecutionEnvironment.getExecutionEnvironment + + val dataSet = env.fromCollection(data) + + val fracArray = Array(0.5, 0.25, 0.25) + + val splitDataSets = Splitter.multiRandomSplit(dataSet, fracArray) + + splitDataSets.length should equal(fracArray.length) + } } From cac2ee8eaf2267504806335bf3cdef5899b0c9b4 Mon Sep 17 00:00:00 2001 From: Trevor Grant Date: Mon, 28 Mar 2016 17:58:46 -0500 Subject: [PATCH 5/8] [FLINK][WIP][neural-nets] Initial Commit- working MLP [FLINK][WIP][neural-nets] Refactored for working PredictoinFunction [FLINK][WIP][neural-nets] Cleaned up MLP etc [FLINK][WIP][neural-nets] Added Warm Starts [FLINK][WIP][neural-nets] Clean up Multi-layer Perceptron [FLINK-3724][ml] Added placeholder for docs to retrigger CI build --- docs/apis/batch/libs/ml/neural_networks.md | 37 ++++ .../ml/neuralnetwork/ActivationFunction.scala | 73 +++++- .../flink/ml/neuralnetwork/LossFunction.scala | 76 ++++++- .../neuralnetwork/MultiLayerPerceptron.scala | 209 +++++++++++++++++- .../ml/optimization/GradientDescent.scala | 32 ++- .../flink/ml/optimization/LossFunction.scala | 1 + .../ml/optimization/PredictionFunction.scala | 83 ++++++- .../apache/flink/ml/optimization/Solver.scala | 13 ++ .../regression/MultipleLinearRegression.scala | 16 +- .../optimization/GradientDescentITSuite.scala | 37 ++++ 10 files changed, 554 insertions(+), 23 deletions(-) create mode 100644 docs/apis/batch/libs/ml/neural_networks.md diff --git a/docs/apis/batch/libs/ml/neural_networks.md b/docs/apis/batch/libs/ml/neural_networks.md new file mode 100644 index 0000000000000..b82cf620f6a2c --- /dev/null +++ b/docs/apis/batch/libs/ml/neural_networks.md @@ -0,0 +1,37 @@ +--- +mathjax: include +title: Multiple linear regression + +# Sub navigation +sub-nav-group: batch +sub-nav-parent: flinkml +sub-nav-title: Neural Networks +--- + + +* This will be replaced by the TOC +{:toc} + +## Description + + Neural networks use a directed graph consisting of activation functions (nodes) and parameter + weights (edges) to solve complex problems with interdependent variables. + + Placeholder. \ No newline at end of file diff --git a/flink-libraries/flink-ml/src/main/scala/org/apache/flink/ml/neuralnetwork/ActivationFunction.scala b/flink-libraries/flink-ml/src/main/scala/org/apache/flink/ml/neuralnetwork/ActivationFunction.scala index 76fb4851c0b2c..366cc1014d693 100644 --- a/flink-libraries/flink-ml/src/main/scala/org/apache/flink/ml/neuralnetwork/ActivationFunction.scala +++ b/flink-libraries/flink-ml/src/main/scala/org/apache/flink/ml/neuralnetwork/ActivationFunction.scala @@ -1,8 +1,77 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + package org.apache.flink.ml.neuralnetwork /** - * Created by trevor on 3/5/16. + * Docs */ -class ActivationFunction { + + +import breeze.linalg.{DenseVector => BreezeDenseVector} +import breeze.numerics.{sigmoid, tanh} + +abstract class ActivationFunction extends Serializable { + def func(x: BreezeDenseVector[Double]): BreezeDenseVector[Double] + + def derivative(x: BreezeDenseVector[Double]): BreezeDenseVector[Double] +} + + +object sigmoidActivationFn extends ActivationFunction { + import breeze.numerics.sigmoid + override def func(x: BreezeDenseVector[Double]): BreezeDenseVector[Double] ={ + sigmoid(x) + } + + override def derivative(x: BreezeDenseVector[Double]): BreezeDenseVector[Double] = { + sigmoid(x) :* (1.0 - sigmoid(x)) + } } + +object tanhActivationFn extends ActivationFunction { + import breeze.numerics.tanh + override def func(x: BreezeDenseVector[Double]): BreezeDenseVector[Double] = { + tanh(x) + } + + override def derivative(x: BreezeDenseVector[Double]): BreezeDenseVector[Double] ={ + 1.0 - tanh(x) :* tanh(x) + } +} + +/** + * Elliots Fast Squash Function + * by David Elliot + * http://ufnalski.edu.pl/zne/ci_2014/papers/Elliott_TR_93-8.pdf + * + */ + +object elliotsSquashActivationFn extends ActivationFunction { + import breeze.numerics.abs + override def func(x: BreezeDenseVector[Double]): BreezeDenseVector[Double] ={ + x / ( abs(x) + 1.0 ) + } + + override def derivative(x: BreezeDenseVector[Double]): BreezeDenseVector[Double] = { + 1.0 / (( abs(x) + 1.0 ) :* ( abs(x) + 1.0)) + } +} + diff --git a/flink-libraries/flink-ml/src/main/scala/org/apache/flink/ml/neuralnetwork/LossFunction.scala b/flink-libraries/flink-ml/src/main/scala/org/apache/flink/ml/neuralnetwork/LossFunction.scala index 756aedec1bedb..ff2511778d2ad 100644 --- a/flink-libraries/flink-ml/src/main/scala/org/apache/flink/ml/neuralnetwork/LossFunction.scala +++ b/flink-libraries/flink-ml/src/main/scala/org/apache/flink/ml/neuralnetwork/LossFunction.scala @@ -1,8 +1,76 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.apache.flink.ml.neuralnetwork -/** - * Created by trevor on 3/28/16. - */ -class LossFunction { +import breeze.linalg.{DenseVector => BreezeDenseVector, DenseMatrix => BreezeDenseMatrix} + +import org.apache.flink.ml.common.{LabeledVector, WeightVector} +import org.apache.flink.ml.math._ +import org.apache.flink.ml.math.Breeze.{Vector2BreezeConverter, Breeze2VectorConverter} +import org.apache.flink.ml.optimization.{ PartialLossFunction, + MultiLayerPerceptronPrediction, + LossFunction} + +case class GenericMLPLossFunction( + partialLossFunction: PartialLossFunction, + predictionFunction: MultiLayerPerceptronPrediction, + arch: List[Int]) extends LossFunction { + + /** Calculates the gradient as well as the loss given a data point and the weight vector + * + * @param dataPoint + * @param weightVector + * @return + */ + // Make this spit out / take weight vectors, not reg vetors and you're gtg + def makeWeightVector(U: Array[BreezeDenseMatrix[Double]]): WeightVector = { + val fVector = DenseVector( U.map(_.toDenseVector) + .reduceLeft(BreezeDenseVector.vertcat(_,_)).data ) + WeightVector( fVector, 0) + } + + def makeWeightArray(v: WeightVector, arch: List[Int]): Array[BreezeDenseMatrix[Double]] = { + val weightVector = Vector2BreezeConverter(v.weights).asBreeze.toDenseVector + val breakPoints = arch.iterator.sliding(2).toList.map(o => o(0) * o(1)).scanLeft(0)(_ + _) + var U = new Array[BreezeDenseMatrix[Double]](arch.length-1) + // takes weight vector and gives back weight array + for (l <- (0 to arch.length - 2)){ + U(l) = new BreezeDenseMatrix( arch(l + 1), + arch(l), + weightVector.data.slice(breakPoints(l), breakPoints(l + 1))) + } + U + } + + def lossGradient(dataPoint: LabeledVector, weightVector: WeightVector): (Double, WeightVector) = { + val ffr = predictionFunction.feedForward(weightVector, dataPoint.vector) + val L = arch.length - 1 + // BP1 + var delta = new Array[BreezeDenseVector[Double]](L + 1) + val f = predictionFunction.f + val loss = partialLossFunction.derivative(ffr.A(L).data(0), dataPoint.label) + delta(L) = loss * f.derivative(ffr.Z( L )) + + val grad = predictionFunction.gradient(ffr, delta) + + (loss, grad) + } } + diff --git a/flink-libraries/flink-ml/src/main/scala/org/apache/flink/ml/neuralnetwork/MultiLayerPerceptron.scala b/flink-libraries/flink-ml/src/main/scala/org/apache/flink/ml/neuralnetwork/MultiLayerPerceptron.scala index cb3c840947913..bee841bed4c30 100644 --- a/flink-libraries/flink-ml/src/main/scala/org/apache/flink/ml/neuralnetwork/MultiLayerPerceptron.scala +++ b/flink-libraries/flink-ml/src/main/scala/org/apache/flink/ml/neuralnetwork/MultiLayerPerceptron.scala @@ -1,8 +1,209 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.apache.flink.ml.neuralnetwork -/** - * Created by trevor on 3/5/16. - */ -class MultiLayerPerceptron { +import breeze.linalg.{DenseMatrix => BreezeDenseMatrix} + +import org.apache.flink.api.scala._ + +import org.apache.flink.ml.common._ +import org.apache.flink.ml.math.Vector +import org.apache.flink.ml.optimization.{MultiLayerPerceptronPrediction, + IterativeSolver, + SquaredLoss} + +import org.apache.flink.api.scala.DataSet + +import org.apache.flink.ml.pipeline.{PredictOperation, FitOperation, Predictor} + + +/** Multi-layer Perceptron regression. + * + * docs + * + */ + +class MultiLayerPerceptron extends Predictor[MultiLayerPerceptron] { + import MultiLayerPerceptron._ + + var weightsOption: Option[DataSet[WeightVector]] = None + + var predictionFunctionOption: Option[MultiLayerPerceptronPrediction] = None + + var lossFunction: Option[GenericMLPLossFunction] = None + + var networkArchitecture = List(0) + + def setIterations(iterations: Int): MultiLayerPerceptron = { + parameters.add(Iterations, iterations) + this + } + + def setHiddenLayerArchitecture(arch: List[Int]): MultiLayerPerceptron = { + parameters.add(HiddenLayerNetworkArchitecture, arch) + this + } + + def setOptimizer[A <: IterativeSolver](optimizer: IterativeSolver): MultiLayerPerceptron = { + parameters.add(Optimizer, optimizer) + this + } + + def setActivatoinFunction(f: ActivationFunction): MultiLayerPerceptron = { + parameters.add(ActivationFunc, f) + this + } + + def squaredResidualSum(input: DataSet[LabeledVector]): DataSet[Double] = { + import scala.math.pow + val pf = predictionFunctionOption.get + weightsOption match { + case Some(weights) => { + input.mapWithBcVariable(weights){ // convenience function to be revoved soon. + (dataPoint, weights) => pow(dataPoint.label - pf.predict(dataPoint.vector, weights), 2) + }.reduce { + _ + _ + } + } + + case None => { + throw new RuntimeException("The MultipleLinearRegression has not been fitted to the " + + "data. This is necessary to learn the weight vector of the linear function.") + } + } + } + + +} + +object MultiLayerPerceptron { + + val WEIGHTVECTOR_BROADCAST = "weights_broadcast" + + + case object Iterations extends Parameter[Int] { + val defaultValue = Some(10) + } + + case object HiddenLayerNetworkArchitecture extends Parameter[List[Int]] { + val defaultValue = Some(List(5,5)) + } + + case object Optimizer extends Parameter[IterativeSolver] { + val defaultValue = None + } + + case object ActivationFunc extends Parameter[ActivationFunction] { + import org.apache.flink.ml.neuralnetwork.elliotsSquashActivationFn + val defaultValue = Some(elliotsSquashActivationFn) + } + + // ======================================== Factory methods ====================================== + + def apply(): MultiLayerPerceptron = { + new MultiLayerPerceptron() + } + + // ====================================== Operations ============================================= + + /** Trains the linear model to fit the training data. The resulting weight vector is stored in + * the [[MultiLayerPerceptron]] instance. + * + */ + implicit val fitMLP = { + new FitOperation[MultiLayerPerceptron, LabeledVector] { + override def fit( + instance: MultiLayerPerceptron, + fitParameters: ParameterMap, + input: DataSet[LabeledVector]): Unit = { + + val map = instance.parameters ++ fitParameters + + val hiddenLayers = map(HiddenLayerNetworkArchitecture) + val inputVectorSample = input.first(1).collect()(0) + val inputDim = inputVectorSample.vector.size + val outputDim = 1 // change this when you introduce vector targets + val actFunc = map(ActivationFunc) + instance.networkArchitecture = List(inputDim) ::: hiddenLayers ::: List(outputDim) + + instance.predictionFunctionOption = instance.predictionFunctionOption match { + case Some(pfo) => Some(pfo) + case None => Some(new MultiLayerPerceptronPrediction(instance.networkArchitecture, + actFunc)) + } + + val predFunc = instance.predictionFunctionOption.get + + // retrieve parameters of the algorithm + val initialWeightsDS: DataSet[WeightVector] = instance.weightsOption match { + case Some(weights) => weights + case None => { + val initialWeights = predFunc.makeWeightVector( + buildUWeightSet(instance.networkArchitecture )) + input.map(o => initialWeights) + } + + } + + instance.lossFunction = Some(GenericMLPLossFunction(SquaredLoss, + predFunc, + instance.networkArchitecture )) + + + val optimizer = map(Optimizer) + .setLossFunction(instance.lossFunction.get) + + + instance.weightsOption = Some(optimizer.optimize(input, Some(initialWeightsDS))) + } + } + } + + implicit def predictVectors[T <: Vector] = { + new PredictOperation[MultiLayerPerceptron, WeightVector, T, Double]() { + + var predictionFunction: MultiLayerPerceptronPrediction = _ + + override def getModel(self: MultiLayerPerceptron, predictParameters: ParameterMap) + : DataSet[WeightVector] = { + predictionFunction = self.predictionFunctionOption.get + self.weightsOption match { + case Some(weights) => weights + case None => { + throw new RuntimeException("The MultiLayerPerceptron has not been fitted to the " + + "data. This is necessary to learn the weight vector of the neural net function.") + } + } + } + + override def predict(value: T, model: WeightVector): Double = { + predictionFunction.predict( value, model ) + } + } + } + private def buildUWeightSet(arch: List[Int]): Array[BreezeDenseMatrix[Double]] = { + // A 'U' Weight set is for parameters moving up e.g. Xs, lower layers + var uTemplate = scala.collection.mutable.ArrayBuffer.empty[BreezeDenseMatrix[Double]] + for (i <- (1 until arch.length)){ + uTemplate += BreezeDenseMatrix.rand[Double](arch(i), arch(i-1)) + } + uTemplate.toArray + } } diff --git a/flink-libraries/flink-ml/src/main/scala/org/apache/flink/ml/optimization/GradientDescent.scala b/flink-libraries/flink-ml/src/main/scala/org/apache/flink/ml/optimization/GradientDescent.scala index 407c074b7f12c..3d70d4ea2b04e 100644 --- a/flink-libraries/flink-ml/src/main/scala/org/apache/flink/ml/optimization/GradientDescent.scala +++ b/flink-libraries/flink-ml/src/main/scala/org/apache/flink/ml/optimization/GradientDescent.scala @@ -55,6 +55,7 @@ abstract class GradientDescent extends IterativeSolver { * @param initialWeights The initial weights that will be optimized * @return The weights, optimized for the provided data. */ + override def optimize( data: DataSet[LabeledVector], initialWeights: Option[DataSet[WeightVector]]): DataSet[WeightVector] = { @@ -65,9 +66,14 @@ abstract class GradientDescent extends IterativeSolver { val learningRate = parameters(LearningRate) val regularizationConstant = parameters(RegularizationConstant) val learningRateMethod = parameters(LearningRateMethodValue) + val warmStart = parameters(WarmStart) // Initialize weights val initialWeightsDS: DataSet[WeightVector] = createInitialWeightsDS(initialWeights, data) + val startingIter = warmStart match { + case true => CURRENT_ITER + case false => 0 + } // Perform the iterations convergenceThresholdOption match { // No convergence criterion @@ -79,7 +85,8 @@ abstract class GradientDescent extends IterativeSolver { regularizationConstant, learningRate, lossFunction, - learningRateMethod) + learningRateMethod, + startingIter) case Some(convergence) => optimizeWithConvergenceCriterion( data, @@ -89,7 +96,8 @@ abstract class GradientDescent extends IterativeSolver { learningRate, convergence, lossFunction, - learningRateMethod) + learningRateMethod, + startingIter) } } @@ -101,7 +109,8 @@ abstract class GradientDescent extends IterativeSolver { learningRate: Double, convergenceThreshold: Double, lossFunction: LossFunction, - learningRateMethod: LearningRateMethodTrait) + learningRateMethod: LearningRateMethodTrait, + startingIter: Int) : DataSet[WeightVector] = { // We have to calculate for each weight vector the sum of squared residuals, // and then sum them and apply regularization @@ -114,7 +123,7 @@ abstract class GradientDescent extends IterativeSolver { val resultWithLoss = initialWeightsWithLossSum.iterateWithTermination(numberOfIterations) { weightsWithPreviousLossSum => - + CURRENT_ITER = CURRENT_ITER + numberOfIterations // Extract weight vector and loss val previousWeightsDS = weightsWithPreviousLossSum.map{_._1} val previousLossSumDS = weightsWithPreviousLossSum.map{_._2} @@ -125,7 +134,8 @@ abstract class GradientDescent extends IterativeSolver { lossFunction, regularizationConstant, learningRate, - learningRateMethod) + learningRateMethod, + startingIter) val currentLossSumDS = calculateLoss(dataPoints, currentWeightsDS, lossFunction) @@ -155,8 +165,10 @@ abstract class GradientDescent extends IterativeSolver { regularizationConstant: Double, learningRate: Double, lossFunction: LossFunction, - optimizationMethod: LearningRateMethodTrait) + optimizationMethod: LearningRateMethodTrait, + startingIter: Int) : DataSet[WeightVector] = { + CURRENT_ITER = CURRENT_ITER + numberOfIterations initialWeightsDS.iterate(numberOfIterations) { weightVectorDS => { SGDStep(data, @@ -164,7 +176,8 @@ abstract class GradientDescent extends IterativeSolver { lossFunction, regularizationConstant, learningRate, - optimizationMethod) + optimizationMethod, + startingIter) } } } @@ -181,7 +194,8 @@ abstract class GradientDescent extends IterativeSolver { lossFunction: LossFunction, regularizationConstant: Double, learningRate: Double, - learningRateMethod: LearningRateMethodTrait) + learningRateMethod: LearningRateMethodTrait, + startingIter: Int) : DataSet[WeightVector] = { data.mapWithBcVariable(currentWeights){ @@ -214,7 +228,7 @@ abstract class GradientDescent extends IterativeSolver { val gradient = WeightVector(weights, intercept/count) val effectiveLearningRate = learningRateMethod.calculateLearningRate( learningRate, - iteration, + iteration + startingIter, regularizationConstant) val newWeights = takeStep( diff --git a/flink-libraries/flink-ml/src/main/scala/org/apache/flink/ml/optimization/LossFunction.scala b/flink-libraries/flink-ml/src/main/scala/org/apache/flink/ml/optimization/LossFunction.scala index 1ff5d97bbb065..235d30505e495 100644 --- a/flink-libraries/flink-ml/src/main/scala/org/apache/flink/ml/optimization/LossFunction.scala +++ b/flink-libraries/flink-ml/src/main/scala/org/apache/flink/ml/optimization/LossFunction.scala @@ -94,3 +94,4 @@ case class GenericLossFunction( (loss, WeightVector(weightGradient, lossDerivative * interceptGradient)) } } + diff --git a/flink-libraries/flink-ml/src/main/scala/org/apache/flink/ml/optimization/PredictionFunction.scala b/flink-libraries/flink-ml/src/main/scala/org/apache/flink/ml/optimization/PredictionFunction.scala index 38f340aceefcf..50293439f6bf0 100644 --- a/flink-libraries/flink-ml/src/main/scala/org/apache/flink/ml/optimization/PredictionFunction.scala +++ b/flink-libraries/flink-ml/src/main/scala/org/apache/flink/ml/optimization/PredictionFunction.scala @@ -19,7 +19,16 @@ package org.apache.flink.ml.optimization import org.apache.flink.ml.common.WeightVector -import org.apache.flink.ml.math.{Vector => FlinkVector, BLAS} +import org.apache.flink.ml.math.{Vector => FlinkVector, Breeze, DenseVector, BLAS} +import org.apache.flink.ml.neuralnetwork.ActivationFunction + +import breeze.linalg.{ Vector => BreezeVector, +SparseVector => BreezeSparseVector, +DenseVector => BreezeDenseVector, +DenseMatrix => BreezeDenseMatrix} + +import org.apache.flink.ml.math.Breeze.{Vector2BreezeConverter, Breeze2VectorConverter} + /** An abstract class for prediction functions to be used in optimization **/ abstract class PredictionFunction extends Serializable { @@ -37,4 +46,76 @@ object LinearPrediction extends PredictionFunction { override def gradient(features: FlinkVector, weights: WeightVector): WeightVector = { WeightVector(features.copy, 1) } + +} + +case class MultiLayerPerceptronPrediction( + arch: List[Int], + f: ActivationFunction + ) extends Serializable { + // Todo: this is used multiple times- move to one spot + def makeWeightArray(v: WeightVector): Array[BreezeDenseMatrix[Double]] = { + val weightVector = Vector2BreezeConverter(v.weights).asBreeze.toDenseVector + val breakPoints = arch.iterator.sliding(2).toList.map(o => o(0) * o(1)).scanLeft(0)(_ + _) + var U = new Array[BreezeDenseMatrix[Double]](arch.length-1) + // takes weight vector and gives back weight array + for (l <- (0 to arch.length - 2)){ + U(l) = new BreezeDenseMatrix(arch(l + 1), arch(l), + weightVector.data.slice(breakPoints(l), breakPoints(l + 1))) + } + U + } + + def makeWeightVector(U: Array[BreezeDenseMatrix[Double]]): WeightVector = { + val fVector = DenseVector( U.map(_.toDenseVector) + .reduceLeft(BreezeDenseVector.vertcat(_,_)) + .data ) + WeightVector( fVector, 0) + } + + case class FeedForwardResult( + U: Array[BreezeDenseMatrix[Double]], + A: Array[BreezeDenseVector[Double]], + Z: Array[BreezeDenseVector[Double]] + ) extends Serializable + + def feedForward(weightVector: WeightVector, features: FlinkVector): FeedForwardResult = { + val L = arch.length - 1 + val U = makeWeightArray(weightVector) + var A = new Array[BreezeDenseVector[Double]](L + 1); + A(0) = Vector2BreezeConverter(features).asBreeze.toDenseVector + var Z = new Array[BreezeDenseVector[Double]](L + 1); + // Feed Forward + for (l <- (0 until L)){ + Z(l + 1) = U(l) * A(l) + A(l + 1) = f.func( Z(l + 1) ) + } + FeedForwardResult(U, A, Z) + } + + def predict(features: FlinkVector, weights: WeightVector): Double = { + feedForward(weights, features).A.last(0) + } + + def gradient(ffr: FeedForwardResult, + delta: Array[BreezeDenseVector[Double]]): WeightVector = { + + val L = arch.length - 1 + // BP1 + + // BP2 + for (l <- (L -1 until 0 by -1)){ + delta(l) = (ffr.U(l).t * delta(l + 1)) :* f.derivative(ffr.Z(l)) + } + + // BP4 + var grads = new Array[BreezeDenseMatrix[Double]](L) + for (l <- (0 to L-1)){ + grads(l) = BreezeDenseMatrix.tabulate(ffr.U(l).rows, ffr.U(l).cols){ + case (i, j) => ffr.A(l)(j)*delta(l + 1)(i) + } + } + + makeWeightVector(grads) + } } diff --git a/flink-libraries/flink-ml/src/main/scala/org/apache/flink/ml/optimization/Solver.scala b/flink-libraries/flink-ml/src/main/scala/org/apache/flink/ml/optimization/Solver.scala index 3bad03857eeb1..6e514d458d3a3 100644 --- a/flink-libraries/flink-ml/src/main/scala/org/apache/flink/ml/optimization/Solver.scala +++ b/flink-libraries/flink-ml/src/main/scala/org/apache/flink/ml/optimization/Solver.scala @@ -137,12 +137,20 @@ abstract class IterativeSolver() extends Solver { parameters.add(LearningRateMethodValue, learningRateMethod) this } + + def setWarmStart(warmStart: Boolean): this.type = { + parameters.add(WarmStart, warmStart) + this + } + } object IterativeSolver { val MAX_DLOSS: Double = 1e12 + var CURRENT_ITER: Int = 0 + // Define parameters for IterativeSolver case object LearningRate extends Parameter[Double] { val defaultValue = Some(0.1) @@ -159,6 +167,11 @@ object IterativeSolver { case object LearningRateMethodValue extends Parameter[LearningRateMethodTrait] { val defaultValue = Some(LearningRateMethod.Default) } + + case object WarmStart extends Parameter[Boolean] { + val defaultValue = Some(false) + } + } object LearningRateMethod { diff --git a/flink-libraries/flink-ml/src/main/scala/org/apache/flink/ml/regression/MultipleLinearRegression.scala b/flink-libraries/flink-ml/src/main/scala/org/apache/flink/ml/regression/MultipleLinearRegression.scala index ef06033599c09..b0f967a0d7a83 100644 --- a/flink-libraries/flink-ml/src/main/scala/org/apache/flink/ml/regression/MultipleLinearRegression.scala +++ b/flink-libraries/flink-ml/src/main/scala/org/apache/flink/ml/regression/MultipleLinearRegression.scala @@ -119,13 +119,18 @@ class MultipleLinearRegression extends Predictor[MultipleLinearRegression] { this } + def setWarmStart(warmStart: Boolean): MultipleLinearRegression = { + parameters.add(WarmStart, warmStart) + this + } + def squaredResidualSum(input: DataSet[LabeledVector]): DataSet[Double] = { weightsOption match { case Some(weights) => { input.mapWithBcVariable(weights){ (dataPoint, weights) => lossFunction.loss(dataPoint, weights) }.reduce { - _ + _ + (_ + _) } } @@ -162,6 +167,10 @@ object MultipleLinearRegression { val defaultValue = None } + case object WarmStart extends Parameter[Boolean] { + val defaultValue = Some(false) + } + // ======================================== Factory methods ====================================== def apply(): MultipleLinearRegression = { @@ -187,13 +196,14 @@ object MultipleLinearRegression { val stepsize = map(Stepsize) val convergenceThreshold = map.get(ConvergenceThreshold) val learningRateMethod = map.get(LearningRateMethodValue) - + val warmStart = map.get(WarmStart).get val lossFunction = GenericLossFunction(SquaredLoss, LinearPrediction) val optimizer = SimpleGradientDescent() .setIterations(numberOfIterations) .setStepsize(stepsize) .setLossFunction(lossFunction) + .setWarmStart(warmStart) convergenceThreshold match { case Some(threshold) => optimizer.setConvergenceThreshold(threshold) @@ -205,7 +215,7 @@ object MultipleLinearRegression { case None => } - instance.weightsOption = Some(optimizer.optimize(input, None)) + instance.weightsOption = Some(optimizer.optimize(input, instance.weightsOption)) } } diff --git a/flink-libraries/flink-ml/src/test/scala/org/apache/flink/ml/optimization/GradientDescentITSuite.scala b/flink-libraries/flink-ml/src/test/scala/org/apache/flink/ml/optimization/GradientDescentITSuite.scala index aed3bfd062666..3ecea7e247387 100644 --- a/flink-libraries/flink-ml/src/test/scala/org/apache/flink/ml/optimization/GradientDescentITSuite.scala +++ b/flink-libraries/flink-ml/src/test/scala/org/apache/flink/ml/optimization/GradientDescentITSuite.scala @@ -272,5 +272,42 @@ class GradientDescentITSuite extends FlatSpec with Matchers with FlinkTestBase { weight0 should be (expectedWeight0 +- 0.1) } // TODO: Need more corner cases, see sklearn tests for SGD linear model + it should "estimate a linear function without an intercept in partial fits" in { + val env = ExecutionEnvironment.getExecutionEnvironment + + env.setParallelism(2) + + val lossFunction = GenericLossFunction(SquaredLoss, LinearPrediction) + + val sgd = SimpleGradientDescent() + .setStepsize(0.001) + .setIterations(20) + .setLossFunction(lossFunction) + .setWarmStart(true) + val inputDS = env.fromCollection(noInterceptData) + + + var weightsOption: Option[DataSet[WeightVector]] = None + for (i <- 1 to 10) { // will perform 10*20=200 iterations + weightsOption = Some(sgd.optimize(inputDS, weightsOption)) + } + + var weightDS = weightsOption.get + + val weightList: Seq[WeightVector] = weightDS.collect() + + weightList.size should equal(1) + + val weightVector: WeightVector = weightList.head + + val weights = weightVector.weights.asInstanceOf[DenseVector].data + val weight0 = weightVector.intercept + + expectedNoInterceptWeights zip weights foreach { + case (expectedWeight, weight) => + weight should be (expectedWeight +- 0.1) + } + weight0 should be (expectedNoInterceptWeight0 +- 0.1) + } } From f7ddc943e9bedce59d32c928721d88efad986e91 Mon Sep 17 00:00:00 2001 From: Trevor Grant Date: Fri, 15 Apr 2016 18:05:03 -0500 Subject: [PATCH 6/8] [FLINK-3724][ml] Add Neural Nets --- docs/apis/batch/libs/ml/neural_networks.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/apis/batch/libs/ml/neural_networks.md b/docs/apis/batch/libs/ml/neural_networks.md index b82cf620f6a2c..72d49b4216fa9 100644 --- a/docs/apis/batch/libs/ml/neural_networks.md +++ b/docs/apis/batch/libs/ml/neural_networks.md @@ -1,6 +1,6 @@ --- mathjax: include -title: Multiple linear regression +title: Neural Networks # Sub navigation sub-nav-group: batch From 50732b577aee7714bf3a66cd25f46901e9ce71f3 Mon Sep 17 00:00:00 2001 From: Trevor Grant Date: Mon, 18 Apr 2016 13:08:26 -0500 Subject: [PATCH 7/8] [FLINK-3724][ml] Add Neuralnet Docs --- .../batch/fig/linear_regression_graph.png | Bin 0 -> 22303 bytes docs/apis/batch/fig/neural_net_graph.png | Bin 0 -> 76939 bytes docs/apis/batch/libs/ml/index.md | 1 + docs/apis/batch/libs/ml/neural_networks.md | 174 +++++++++++++++++- .../neuralnetwork/MultiLayerPerceptron.scala | 10 - 5 files changed, 172 insertions(+), 13 deletions(-) create mode 100644 docs/apis/batch/fig/linear_regression_graph.png create mode 100644 docs/apis/batch/fig/neural_net_graph.png diff --git a/docs/apis/batch/fig/linear_regression_graph.png b/docs/apis/batch/fig/linear_regression_graph.png new file mode 100644 index 0000000000000000000000000000000000000000..6206405ac019e998daf993927cb0b777ed1adccf GIT binary patch literal 22303 zcmeFZWl&wgwl2DGf;++8-CaU(*Wm8%?!i5{LkJ$+9g+aS-QC^Y;ZF8G=j{8c-l_M_ zkNe|R?NzMBVs_7--D7n37~>mX6QL+Cfe42S2Lgc*r6fg_K_Dtc#*V#Cy$Ul{2VDxFh~y(+JH!{jB?v zOM>Or(z;aLjl{^8D}u(MJ^kgF%Sk3$J;D4fq8Q&#%$Hs*=0l7P2m}Xh3G_%qPL|i$ z-j>nO#NNo1(ZkjO=qd=rFXZ81Xl!ljLTqGeZfPe#de+uON^EH&K&s9m$1LaY+0?>P z(#y$I#Y+Gm#Siql$~Q z0I7zYBJpQ?CsSf}Ms`MK1~CsyH&#+XIAVS$6Ej|AQStw10{BaS)WXHZftQKN-QAth zosH4n$()IWhlhuWnU#r^l>w;1;OuGVV(7tO=S=qA#6Q~*HFY+2vUG5)#LgAIbRN;`+C^{znq{ zA65SM?E1I3{znq{A65SM?E1e;TyXz#?o90fkIo%%-DaHSKLTzaq_c>WDlG8wfi(#S zfrvp;qMuYf7LT*d4OC{H`noi^@-4;P7mFu-+@bJgN8#OwTvrmeU~<%4jo&E5Q;GqJaGUrQ7UY~CS=Ac zWWq$;FrE+!%4>TU`?mq}@2OsztEt~|;vEy?PYOd8MtTy1}O6+`75g|qi50`PE;Pf5-YCGpYGQuEg zAYk7d{@_*>pl9EM?n--k6-8f0>3)chy6u>gg#`>$6v1F?;2tsaEj)7DktP`O;j4V{ zjD?u8a^1OTa-;$@=cA2DQ?=>#Gzx6+oK_FtSed_8+uN>0C8N*%ljp3KX>Be1`MFZ9 z?R@6%HGfv-O=l-BSjay|pqB@1`IRZZTDP~6IR53323kXq1L_qj+VSTlKT@~Hz^J&t z@l&6PQS{mf9o+(zI9*;RAh8l7#tm|L%T&$2fNc^|XmX7mPq1sycFM@32MT-Gdi5mw zcYd^G&p^fB*DX$Va^2{j>JE9m7@VT&ry)gwt%?bE&+%@j*Gt~TN847}Lf)kfS{{So z4eE7$*+}DYqnj?sd(9BC&@xsyq=^x)oA6*=!Q0}piKMA+Nh}g3Mmf~$q1r0)@`C;C zzR4}#yU!jB{V?hG!e-_6DPBqS1udpNjP6PFHp|&D!shUFB-L)s#?Q>P%P|D6@Aib{ zYO$6yKq>H-^q)#UzczC#dXr}ny zt>Kf%jFry44u^=(J6ym6D3kX9mC788V|0+l#G^IpSfds!;X_Tok4YZ=zw|3&GFvEUjg`zn>PXa}uBIE=|8Y=zStaft^9Wh=a+W zV@XFif4GfC#EbX!gE}p_cCccH>c)DB#`^S!HkI-MCG)NQd0Y2sW(@A6krBeTjh?%h zp!#qyuKU@~6#UCS2ACbLk(^aVZH@Gy&WnRnLfpX9gzmWQJ43IkxCsHy1^q3b8-|^A zR8SZDcrytsUJFD%Roqvu+`z+a!{%l<>O$VK76Ny?%~2V~6MZ3b+00C16U4<4L4Srd z%0Q{l*;?!-$`DPnhx!Q&F_Kq6&l^ZLm%#pGCX4lRX9Znh$1Xfi8Y{O)^2~*|R(&*jf_6)q6CU3p{Pyc3{b6wkE zUu8cf=_>SnWHr{E%#`_5!)&fzN7cm76 zQR>CfhS`Nw^XzCv?<1i$3YtKORFj8epaaPXrh$JT7}tM-#!mLE{vG*eNyd@#FQ|HH zeWd1v#AMn=OjEX%N1%9o?iA`MaRFI*gDfSofm~h|SO=m#SandcW)E1VlA(Eh=dk1P z5&MYN-?IEZU^ixkqRN+WS#e@iM&njdqkkDHgx}M)X2+Fs9h6!W=-T^nVoqi}O#>!X zWBys)pWSRA<#~28E)?eUM|k)LND+cx^uRH%R2 zVDOf-fI|L?4jZu;a%u}JVMd>u-jn(qyY%?gfCgH`16IIvS5LsHyiQb;(4;NuQtdT4 zYPIKUv$F8xr3(}2lBLCTbaeV-@VGM0Ry(}Il}lB(cX#1TS<_oJhChA!bO9G4LW633 z1^Pwf^VO+)vSG_TiBw&$j>r8jJ8de69UeSTv5d56%91&=;ybNQ=qQ};q^$|&(GW@y zq$}Z9yhuen6-do;&gM4mLQsGt`CvqNcr!XGs^i|n*v|HLI6H34_GmUAYH4X{&cwvT zMw|N?;SVXL2Yz`8(97Iwr*72~3|CBeGefd;;jwMW=$iQ=!#Jp!K9QL`rP1UK;{i#14e?>4_cv`$7$`TtXks-snC$NQ%kCt1m z4)!Rrv$Gk#&?dtrOH;(f#Ub;0oEvnBMPd2oeJL}w;ann49a4LJef)&sL&4$r^GxnkDmns1Z28^V|*FkI$!W){;g1)q>t%J*Ohk3noeB+~4*Au>9 zqSV|kG7_A3!+tZ)z{yRc_TmsC6Y|4GMMc3&$;w{Sz49?JDJWWnhKc@oP>9Y1y3i_M z1oEiT-hbAA4SzUYBW)x;o8-@moCUiUw;<9S+~=W%9a$v`n{PRa$Gjql)wKxyMBpnN zh9j;^VQs(|+>&L8p^&KTyp3jfJ49{x-3f=V$$rf)Athz^LLoFvoJ^@~hBaNbh|$kO zl9eAeSiM~W0Yp%I%JJyYQu8`RSNXXdtL*ghM&ou?Q$W9MChNvap!oiGo>oOlZ*Omi z1v~DCfpfqyF+@G%xpEkcdpfJIn56#oO6RH?ABK!RrJKF*Q;w~ziM;r5Ex}K)JHMb_ zDikL0JR$NU=+V!WdIqjlLh_q^w)$`Zopd{4^7_kD(+dPeB|is`T5_t3W2l6P$k(^- zFyx>pitMzseF>$ccoPp4SS0$*ucY}UOV?|4;5%p;ZJFOsX`!?x9_a3P5}`R>eg5Hx zu`k4Jo`-%ZyNQg9v?@_9-Tv_5!>gmCBgvPtWcl~e@?x=oZ1`Rc7Jt6nb=s@05si*6 z^M0!y@!0wM+1l4X_2{U`{X|$^>Ez-WpU|p7>&vryy0gbO1Z42vucXic956q+mR$JF z%tV8{)-UE8XgBbhwnmHuwY}Hd{J^%axl;;cMmJEgv9W)s(qfVWQ>ugn72?-p_rzM| z8{2N#BZ+cSLs7OsjRt)yUK7#8a!3{i=|*bQBrAU;OmkA|#wT*Q5+xhx(HK0wlr{e! zw%Dm6S74w(bbUPPo7PqOA4_O0qU+3E+k+U6j%l045K2)iE0EV9{|_r@8p!2F_yWzduY=Xqdiizy#Hi{ zu^%Ik4#WQXl|Rkq#G%|~&k+VnjHn&^-iHs3;MjdpFgaRs6V444leyeF1*%5u~US<+bvOO^La z51<$s4!yH=o`2&7>CeVH2WRBd;t+~XRy7496Q$icT+I+4#JzBKYk$+-P#+ci(0zv; zx>NC3Z%@}Es$rfNDIN9EBW1PmxM@i!tF?mAhELSF zZ&w}nuvc%o1W9dXUR}GZcF(+0Q$8f9soNr|(q{tZMpR;DtXf!D>5$(iEbt>Ojq1`V zd9&^6w91(^=@IG%M7YxEF*19{z1uEw*%5E}*^E7|A_^8ztdu9oySHvgA4mTtZ_Tk% z+Tyojk(|;nJ-^6t@o6IU-ubm!+r5pqhU2Jf_Ke*nsj$Mo7^~tiB(lN14tStkEtH-z z%G0c54!9*0i8w$?hW62gZ^RqjFL&ROW#$&T!pHt}VwsBdOCrFZ^qslyMBHmnBBeyp z!<15|?EVb64k98Vo06en#?_X63Na+DdR0OmBUg3k5dGpZM>EH{tqyYFdcZOPtSmft)#LXZ=0nLaAUz2727 zuA8;-Zf@QebGxLimDNx4fmP>=?(R*WgSnS^-xizL%D_HjdCefspWQHFozjG>bx#G0 zhnVQ7=&(qGQDz?8Ubs1cOLMl^U_Bp32@T85)i=`6m=9_lKIeAYmaGE&qPGLU z?D2YDZYR&6UjE75HH$1QW<=M@{TbgX7=`Iyio^B&>B+XcBMt@VO|X?rEBA9eu#MQ*>D6H zbDz)~ub9%`i>LeXl#ITj(lUV+XZW;&F)+!#&#CyE`XoY{b)Y{S_0( zSZ$H>HZ_j2SaikT`^jB4nVmiHDW#mKQ1nQp%{)ZK#5e#CNUzub2=GtdJ&>O`gao`U z-0rW>54WZr`t;kSD#ex5&X$bUITqK;4Vpwvg5uVUzRFvD#v36Y6m)&O>`D$M+|;mY zMlDdYo3k$`BJh6xrb!R+<9XmU{1ljD*svg6JQdhAWLlSQRuhm(K=0b}Cf+>uDj(z9 z$a|$BIu7|qhW?&0eurLmA+W!1wVwBqfYVu79oS>6T`s{3d<&bxdoiX`>CBKbz<0(Re7>Y3h+qk2|6H zt!C@dY*kz$>k42CzIn9;_2vI`_!HgVKN=>UXnJvn)+0}V&%#35zeR632>GXs?tn}p z_8OPnLNWPk>l>`+BTSe4{lRQ9(UWbSnrUdm&srKQRUg8(wLqyLQo$-V)T3yuiQ|zu z1-;hH`RSQn96v$3b%zow`+2K|PnkK06_i8MIvcLfNAB9O@qRo_pCR zGc-ZHMq1B@T*f*YIS3Ja`Ll;P31@a0_j{=vxH$$Xce_D7?UhsNxdJ6@TFo1VtmoO> zZcEJB@~EdD5dxE6_NMb*-27YT)KKR>&_^sr|QOhQpHm! z`1_y71{oc2x{kLWpw5~Iv@#QIf#^SD?{uXtW(eKxBB=9Ag_bVjhYt~Swjb|cuCOb7 z`0DKl={T;Xro;rK?N!Q11--5gt{=MebT-euuk~BUd1nwG)SD_Dso%G>IoFNZG8ypgobzjxxx>-RY;IV%BwmJ9N9XK-zR|{~ zagoH(N9&Cq1+!-tkHZf3MC4|_N4K-l2iu$4{sYAOdKae_6j+Y#NKCq-(2x*;lEeOv z{rh=8L~={FY||4lSa*No`$GlW;%Cyt63QIY7wcYy>tZ$u9Z~rP#tkD6ez==M zDUUk8j(RqVd||t;g#S<6=Lg*84zL{_)>}^PI=O}_==*+KZQ&gvR>S5h_JQA=cT}avkx?-WYAKLc$k6Byi|a|wxEb2fy*)xKD-FuDLo{^J^K@O=d+D-8 zpSU*bQ-XId$3J+Kxr|V$K#N&V910P3khLqh*RwUbaMcQn4gC3Ip9+Vg)~Qd3wDX4A z`g|m<;M+G4Q5g66>8X*tS5DsQ@2I%GMT!3aghF9@;P;sYy(i#|<0Bv{$A9H@%VzKe zu6vkRFd{Ir*IM3cz?JesMx1)-M@Stg<&)~w9{t6&w@;--oGgitoUr^REf8#tw5t70 z-vmvqOO%fO%lLt8XvnR5iHNh6V;c^PWw)37Y1=?*1nK1V2gp7$`Q&*e%F&W5ui$2W zjGqlHc^pcd=%CT%DOp7-^OkxWT%)-^wyk|0E>7^Lx^2Np+p=k3dzyIt}Sa*w98g!QH*1L$6MtR3PRi zypW=)!1bb-^%Pjnx+J;!UOX)-$U*?(B=lyMIoyJ#OtfqII{Ok_F2B(+tv33`b0cDVU8>;nMak9j`-$?|gP28p4_V>p(%dCCNobp$ z^i{|{IfZ%;4OiaK5B6~-{6RDr7ib;f6)MYoI&=99{xl7>m(s!OkxgX?51wb&KfZco zRm-Y4hkO`mdFa~9cj$m8cVe=vqcNQTQpfw)5&zE*!_v321^tR$`EEeH3XP}fxJN_7~zf1$#;S}3w07#j` zU6E*EB-57x{WmK2_u$~1_k5jf>w*nfqGqQUPJ-b+3qQDv38_~%E_t#3pAd#(!(jBo zm<+@aYf}ar(bMFX`MN8@^7Z`H5_{BBy%rk6taL*zf{X)?Hek)8Mj4#tRM7-67fO zC%4gdM+kBF>ax$YzcSi3>1im7wIZoS8F-ExLV;%7=J~E9ojY{JW*5d#iX=|0dvCb9JbJ3SM+?= zVT0t^tm<)Ay&Ee}h788NBYHP&KT z5!3mwn^`6FeBM1nz~{!#XEj|i(&2T(tDM~u78N1GIcvR>z4PsffRx1=y}kvmMf7`p zRCpobGnK+*WPu&yS0~k5^Txly|kAG5`SdgW6b(D=4y2#*Q3P zqSu05xi60B%hUtlX@W*UY(hb z^K!-0{;dOPCxjf%j{et0<^yO3I(zj5POw+_K5anJuK^+B$l39y<>zb5jn^(zmsxuL z1x`e+1h9Kiqx1T^jgg8+7sM}#pQc(hgZ6py`{YU^g0wQq%PzeJG6N?p3L?dFGc)Pv zA08s@$h|sp@}~}($GY^M{WC|z=`;mf;!zGG6Hb~6lHAW$1x)TvS3VagTPnZ3z93(p zt-T5Fjbg$50B zm2P~R*F*HHEDh31QPeAn%H!oqNch3@Dr;&-Zn_do>zw~{m?`1X4(+x%p1-RCyCuFo z`lO&yR41XJfc)U6TA83`gVhX`aw)qdFwNVb5k8)KAI(-&C9ORTqGRGNIBf0kZTm&> z;AHL7=da8r8Y|O>3iAJYk=OE)ceWpJrsLGESG3a!pn?uSy~Js4Oiq&I%3ohJC0x|O z`8uR9w$&3Vw9Z#Zl!XH&`FKSP2L_6Bu>2yULt6!1Zcv|Hhsoe89Y-LeJHKaSbon_m z%qs_aqoFlE$u-d-81DmUdvA|Vh{+2##*9wxbms+^TIRI3O30tAbU0=Z)t+2RN@}uD zCRJX))#c#&`}gnnuP;wT;nC5{C5qIq=;kB+9+O!LIQWdN6(5O;A+hDm^14<;5kOmB zb8kMAKkKME_0qM+o9Q zH=M|%)rU|)9o{L>Gbu1PceSNl(p}-9gxwauWL=R20nZQ6U*bv_T>dECNr;I#Q%&1P zFDNKjqA|F!u`yviTb@2%WO;r`{-0a`xy+%rV2XOFk@k?-O!A$RBx3miY9Lp|ObHT& z^vt@s!w{S);%4J3&M6BDr2q*F@6^FXBDl#c zJW*xeO_87y$vs{Se?WFeoyq|~->~iN>Gr@Ah`D*^6f*EYgD$**cF(I_)!RB?X?kDp zJKDT2O|_)Wl&r$5D=coixHO}T-+QcADy(qZt9CxgE28iD_S=!7+Kwhd%TP#|$LW>` zQ={3&D}~EUCY(2MA*WIl*jTB7-AF zOMN7yBG^v>zVqYa1X}LtVEOkq1O1O@DHr{XJ;N3w^I%=Th&FZdV8rd9H043D(zf|7 z9+9oJiF$ZUr3}3R%FKC$-emauRP-tS(P}<+5^BD8FatRnB?>GK6n_y}aTORSmGy_X zcMnB3Mw{849m*~lHwIqjx8&sXvbw!kNYC@z?ZRUHD7C_;ikEVkFhN@g&IC$HViW`{ zO8hPfonQ3hI421+iiR1uy+-?cakf{ZcSH^TLqFyOn*H^YHNWY9Gi+G=S^o(-+bpT2 z7FiqKIgX&^{JYp#%H>F=nVUULDr|DW11c<}#+9g;INYdB_Gk5}^uhc~xRWWe%IoLj zjhp34n}?qgbutq)9d0?%*hXjprvNH})a6yJiB8H^RrFtT?1`(BX)?cw5Y}`h ziTf=Z%7?4^gt2o0@eeLe3sk2aQVt6k2;7Z?tgF$xXKd@_uX@ZrTI;4ii~`jF6Kh0d zYC8ZD;|HSiEu{|)*7WtGg9n~_8L%Hg;lsosLx4hb%!3AuQft$0ej#~XttJ-w4%noRm@c~ULAc+z(BH4j)Q@;y;tcC$#x55t82$Xbf(&&G*hwnW_ z!7VmE7{o9H6qV;8bJ8LC{eWiR2dn?>>2VBFLLoH*4_wzE<-}-JM3{>5kN@5C{jS*n zf?0p8+fVL*rm+!UU{=@jb`|f8`*&9)n3j1X;RCPA$u0DxnPq%?lBLDn!7N zUr*i~jrYK93i`EcBn<=oQEeqnZO^32Yw5&L*Dh~@jiQX2ylH&jj8%I9v`%Y41tSeP zCsoM|L?ra$=o&WSGn-KC=14`?rUrX%@JZhA4sQZ1q|$eGAK6fJ7@uT0TewnZe=T8YkAeu#Cn$_W8^fEtr zN2ulbZSsBQg=9du#$2Tsi4i7FqBW|E`T6)GkDZ-xGkW}lCtQx^*Wr^$x%Fii(d#U| z!0bvjn1MkLA3kiEk^*NpWL>XSDohj;xlvv3-vxgf!`s`$!GXnT{2YzpFpYi*#qY@X z@`dJ(8Wo%$V<^q^0vY#fSvmH{=;C17;J6RYZc5%(mQEnl6U-!t+q2%2xIp$?)%Ehu zNe9lFgEu27I&J{b`P?uGs?GglHThTgiM3u!roF9pr~oD+qTnx8Y%(k(0V9HKItL#w zPOpBRK@Y#OSwXj6Vw8ehw~_^+gQbFG5O7!P)8f6?ix z&%y1;uKAZcJoRRPyF|!V#XwiP6(j?`;Tq7ru}>5E7#9w>!v*hT1ZCx6bV^x`x_3F^ z#)vB`wfG=aFs~_&fLZ$>QQr8bAUirXAt^9$@(ZX3&~XM#i3fBN&aC+dg+O6N8CLhz z(5nb)D+uY`T;<@Qzq|;npByL~Qzgn5|6ZDc0vC*HaCnMAtT`+%LWkjeU4H!+g^KT= z6sjDTeH2tL@0}I`#HC#B8KH^!NFr&HFdaQeksVaHkQN{@vGe*vR$4ltQn$_R`8^Q; z5ui#dhs$X*U-=G-o&a{(GG0;@VvYbY1{(hRdZ+CE15UZ0DZ3P~iWu>EstuFv&Vp!eFse^Rpkfxz`1gnaoJ zC5!J60*ypq=A?-V%OtKwo{^M9uKVu5eF}m;xw^iVwFNxU=~x0j@#it=c5v_Mm`mqB z)8Zh5o<)Y1`z~%!rF`0aFjCug*2D?b=G%36+^Prf$=(VVB{Ym@x0p^_8MiYxhRD~K ztlM0F_5s~~Il^U{d#a~oyO3Tu-t2+u@mC}@3wZ`Rt}S9)jj?c+wQARrOJR>J3 zC((t41tTD2F=YOzmg5%cVwhTAZcmnSiG=)~7g5Q?kEu_5IeQ1V(vGkC z|8_K>1c~w541j^K1o3}?_vyhToMypf=swQAVO`X7V2{;lAc)(BkPUwA%U?g*8cn|$ zD%4%-I$h|0CA->3ao2OQe0wxDZe+q?Yx&f6f3-I{e6%{5UNj+~fzE0;nPXZii$_Pb zH1g0fqFwBZG1w`>sQ;BLiXGTxvVioKQif#I8zWX}U7d3$K<$Z_B*#Sx%f|*f4lMMg zp&^+eAjky*ky2psDCz0xX#u57DRvioGP9=F`xBX*_N)9qfA9{Ou-<$l`RwNZu96)m z&xfnTDVm@aB&=K!6K&qcXG^Gxcu=>Gwr~XmPMpv1tLB}GK#h9d(r0lmXULJ+$6LFar)a{_+rBa41+&$GE*r9Y?>J9H!10?d zk^~YAqzKX9VWyu;_AcXH^fPrtJ*ZaPb zajRkb@~T1oQq&O@{E?u?l)&tZ(7aUofX9S< zcIn}I7Jo(`m+oe#ckjY!K!WS#DyBt0I44&y(+eb82t01rEBxPHVf57^oOv&;NU*&-)t|}&Z8cqdn`skb4l9Vn_>W1k_*m-Rd-65J) z@9j4xhN?okW{2$~UgF*VX#6{^RQpm}#{VN>bJW*Sx2L1uDzi;13AfJ37xM8vJSz@e ziUwd7qJ~UWlDYo#qM}wp7;qs@x5o=tWs6S?YCZDurDAG21>Gg1ebr-E4tJ#=ezu!e zJ#?MKa@e!}s=n&PtFVdc$kAuHij+lDQ!bLI3m4{78;i~wvPP|f5N`MsH)bY6LVJ2S z`3r?Ze5;1;+uc3E@-Y3^!(-e1OgFG5gqa#@ESX<>7iR|NZ28@dvO|UJdmF`Re`8j&8!MaKHl%ce&z!p;==O~po#@{1 za}T%vtYUEd>8p(w2d?xB>JpJGJ>lxQ4bq+`AEM3&o=;H`_`c;SCBP?2vR zP%>w=7ZMsFqw<9)DB5YXx;@9TiXa{b06v_~$C+#$qh#z-fJ-!abc0W-{I$cu46pnF z8fi;ROcn|j6_cSnnHRTQCx6AOXD)g16kM?C{|yeO=&_P}BA~jZ1FK%=+tg;iGy*gD zoquQ4zS6e54rMdjHRVPFM(ZWmZ{XZ+JYy+$jK*0rB|2J_2<(6qq!iOw;IRu?_Uo(x zHenU{qdg6{m!C-4tVYP5OfvG!@gC6ZqEu54ajrbKWd~L_KP_xy!-%%!;2Da{uyT ze?m?#@7I9~Zm_(nym+O&vMK7-;&zox-Hbn`sI*d2&2~2hsq!B>?2Pqsc=%|YJmB&9 z0iS1@)T5gv?TwVnFP(2cz9ZJp2%Pp^K>*kGd)fppi?)0>EY&m~2Oivw1thYOD{;j=C4w0@g>nWi%Rv}`HJyMi06QuB^w3#P$?+@==x z7bojB$t`gaDf&dyri|k8UD1$DKZTc^X&!tFoiM3zqco_IC zN6?GjX`h$U9M`8^_rYx-+l*Bcp=`k>FEyp}*1mk4YDN5u<76rc^oK)!NG=RDr+NrP zmqz%WK-~$_?+nk&5=MQ7m}qW{zqg9YQ0S@G_R}o~+D|JUkAF-Uc)uE|Sduc}kO8w3 zMUQd0=TGpJr%_l%?!+ICLj)`;Zwh4Q1^S{1n?|P+IOy8{9_IG=jk?TE$iIjfu(Yw$ z&%Dx|ZzAv&s{OKQY*U4csASFJx9yw-LrdU zXnxy~o{o=uOW5fxltF6N#cgi?tz-WBb&ap?yMyPgM$85WCJp`KTkW2 z{naUuJMuNR6X)s6Yw-Q{vcK0~&KDJO!Z$c?|0v%t^8K)oZOQ)C3+zYv=U#T|Qdi_C zdPF)|*F^|0Cz#e({LzAT^0eN`5@+N6=2Zh0tOqxR!F<5}!NvFs9pV@41A^|7v~-oCc(vL6Pf4xPb%g2)yTJ}{DD3o zhohhzqU(_FUz0u*(h7XHc3xndWCzGbFfExRR6KJmMDwPHqLkQRTr+m0MH&dc@FCmn zoF@+sVNjl_yBn(YNZ+|cp8A!=$CN%<+`e)*gP8k#!i=#Ka&LSreIy^t`-M;YBcRu`ey^$A|ozapx}dMW-auav@}OD3k4MF#zb*x#OxIH!)CKNw<_MS(lLlsIc2lA<9xxqaBEqq?Cf5vCLY2x|5j|QoR zGm~w_r^l#fLQ3+hu!jIDY=8dtI2n9Inf5zOBr3GE~$JekJb-111IY3F$g+F+A9=yld zYb;lh8schjbGXOVF>J?JEjgF`B|1ylJtO1Pvv}#;FugrIXIBv<(ZH^bXWD27N=^-< zp6U_jB{Mhz&zO)pW=9)b{&yd+6-OnH(P%eBU! z^iuUFUgg~$?WGg}-8;=DAY7+Tq1`mX@vptSmdXo1+uqsvvciSn2OfI|j6f*9RWI-NzkE|181B zm)fRAo}SJf?dQa{s)t`Mmxmi}x_?WDColkO({ftJcw+YAF;0z2DE%ZV?bI36LTm;)ix-9tZ4zAdyhO3qOiZY5+MN%ccF){kglC=(; zuTKiNqMI{fvKxy=Auok`6OoLh3VU$8WA;^pcP6aqUVo!;qR7a|xKea}k@PHO;HRFR zoH6uQUin7NN^0n8N99}T_E3DrjQZQ;tz9^vV)JnH8Xcu{Cvf&#P!+Ds%-X#!o3*Lma|1A*hR>)5r7?_MHO)mh-i_Di5{ruIE|p&|ylZpK7@2GJt^25{l-oS{+Hss2^jUZ{ zrP)=2<1SG$gOTA1gBy6VW{ioynJ+wt`4b1kF%P3hZ6XJ*oiybg1BF zw5lzA)?InY*wqVxl|Ck!GHRrda?)yOIHLp(u=K{vnmo{n>F^+M!elv0XMj%0xbRbz z*3-wwN71wPown)k{}x5--!$jK;1!^aT!Qcqp2?zhl0A@f84UNlrl_ z6=jMCFycvp7~q6tDN>OdW6%gyUF{cz<^EWtz!i77m~Q0#^M6jKebv}NrM&(1u);!9__T*_;e{X@%?0!H8) z94RO7R8PJHo1yL;H0T@wf4J0)8Lr&fvPAaAr_#X3#~Yoh8P$h(UT^$~%qNe4y+Ymr zDznzYoDO>nmY;EVWd-54{&Q!tDhLarc$@%f7Q1f7NpsIBI zNr(3xZedZ87uo_Z)=H))0rwrbjcP?i+D4W)>EN={7Iy6b64N{29HB5^d%>R1%wF|t z{U?`JRu=cLjXDWeZ(s7$_Z(4=F_O_AQlDQM95$)g{CN3q^fCrubcB4{-1nXvy)?J4 zp?$P?EqAM$Y~D}FMD6uc)Vj*HHrTf9XFuSa#n`+4{8CHBScud~bA?uG<<;<=HsYP? zE59Q5uzsCgVCZ-_{J0p+A2?zLk=hNNC;14jnP8^K9le4Sr}_!Mzs$ zb!kRINM-egfoqAZej9PjLuDzI8`6gTvaI{q#$x5XU%d7Abv_6#Ur1-h3IUlpu}I!T zCN^hl^W+o_kDSGtsWgZ};H!0~zq??brC-x-?G{HzHufOdEDpmD{n)>+KF_T`{k6mi z=T>xK=hr2#vJd=(P-4|g9VPwy^;_57 zJZfgdlaHgR?2AVE_o;LQ_>(imklpD`H#Vfn&>Ni<_ZlX2v4RM`PMf7B zd;gsN+nc}XeS`NBz@7v8QR_*jP$b4BKDaoSFshkY-2BOf2l`jI)hmi`+~qh3>O3}C6Z05ZK6 z{Ze>%IN$a@sd#?zSoN(muAB%O z@^|yN4`Gg-Lv7(ARaCstsPzcpH~XL8u?-T&96kk&?M+LI0Sw!iKM)$lXXKqjCgV4` zNS4Zw&vtqs5CZP|{{_hR^!_Kev1#%COI5(u%8}S=R;~4NTGUp-8Ve{>%)8(D)+^d} z|8uES;#-wXvUDLSBO_yhMq{t@650ZqRU)cDn=B5eIyU;kF@ASWsg3!h;6wM&j3%z$ zXJ=$}>7F*GlFlVx(jxnAX3Ywz``kh11I6IMW{Jwu)x8>=7W7As4Be?Eljf%3W390t zI&Ne%%gpkF>zeHKYv#6@xV-+8UZ>_Te^D1+6&itK7rXz^ih8MPhNy{530?b&uF+HD z?flzT;b~CU|LWw-!=Y}wIR3MgA~8lLTb2mPjIr-LBC;=`ERBruP@~B*S;i6~8B1h) zGzev~XB#5BiF&d`jA_Oal|6fQ^G@%3UGMe&^ZfJvb^mwnb1&Dq&vow4_bi09Ls3HW z5s&4Q-?Xf4y;|*c#v%a2w`E)hhnv_qD2!9%9Nv=?7CqWX}>XnEN1nXc@#zP2=^?WxPecbt*QLv z+8*y_)$L;XU7LY%8~csZ%vB?f{J&ev$f#9ZKz3$CbrpzbgsyuX`uoh1ZnOx|l*6Tb zwh!{~lEp@?@h)SS zJyM8hn}0!jWOi z_w)$N$v2RLiFdG~6WE)=GNfG42{0lSaDA;7nrA+}Wpff!7OFZdzthvS$)#M@F6$(XYOM1Xa*fITsw1_$D`ZOr z+P2P~sKOZdOS7?woIVx=AugH2-?xO;tyb;be*IQ$cX0^4b;q9cTeV1z1XKA=s9MwF zYS_4X+BDtgt!{_gJ0#kBKGlz%h+blZL7P@v&U2-xrf%eFtk>sbm#$NCb$%4LrB7;b zH9mjENM2Sfb3*RZvAyiVHE<5aDQ?&S?ny1^hRGG=@HuG%Onfp|#2;YTXrv)u==T(j z`)b_xBDmZ*R656nVUAY|+8>9=L+g?CMy?z++`r))TEmKjqzh|-07~R%_=}J^8q4HL z+R?9khbFuMtgczszm9hQdse!hXX4v(sYAK6C8&XAQQWcm$ zz$s>?%2*#)L+Z~7!_8m5cc43=NFU{u64<1z7A8c=L57E}cPU@^-)4aP4u(I5=AA>K z%wB(;n9+XghdTk4B9KemAXDnCY>g^_fI_;9lQ@-C@r?3dOA&t}C)BeiV%H<_`t1vadr1em?^o|&FDOqjhQUh`YzL4W9)!~`))mRW>g zm!>-<&u4W(adttkSqHzx$OWt1(Nv+dc94I8=J?*3d|G~~0Y(C2zNuU`34Tcg@Cn=HZjGn? zq0@6>xv+D67-q0%%7o=WwDJrBPwrRzYV|N^pUf&XE5vf2BQT_ai!jIFzB=dOI5ie) zVKB&}8%HYjH!YO9;>I#F_1;(4MI93s)tkvwMB7Pv)tq&#eKWnv?jEZ*uU3<=5y=fV zwV9P@ESN*IxVdyV5E$RZ6Q*LXfc0V%G5HR<*L`Nd9JnV&unu)gLwqiwol{YPxCDx1N@LdVmcNCEC^tEWz#qn#%B zhxTj5tTAZKlx)%mjpUg~E6qm}CN9vJ%u2Kzcl-sdnYX;6r<0l* zFAcNv&2mACR(@E}6TNbzyZ`1Vl$}=`?=G=$=gNu_ja7o8VsG*g)e&bQHGr1FVEW{& zFM=Qwo`%WO(VPvD;`AYi+8+mOA=92o==6NhDSs;KqjnD8z^$iR3s|sU2xjo|NRS;P zP+Aq?TexCHH@DVn*!m%0EW5g$Mz!oexgE{3ZGFQhC*E}a`f8^}>_buE23*}2_P+%G zV^;fA&aYol!g1{N{u2r(DX>R#AFHe+E1#I<%gSiV6cUJt!k;lKgIjoPAW1xngfV+@ z*2C0xPp-wTsU6%hJCBDj=V1&GtHxc8_PyeihCaunZnCNY291iS-KLZaGNK!>xaG;A z?H(6X@#M{vHY@5v+f4^>MZ`nWa_B*Bp}r}4CStaq`gy> zfx7{`84^FPgAnOus!6nBOogImKV_H-g@FPfXY^HjC;^02H60hr+mh>u2)q@;{iS<0 zN4C_TIW5tMgfno!s4Uw?*61>_zuaW;85HA+;U~XS0;w8IL=@G_%50VOe5jcq2eKGk ztBijfKa_Sx*Z57z1w2-=@Ho+_nz&c-t2|fw-TCZ`K>PwP7}WAsxy@@xO(9q()z7*( z8zs`_CO9;D?VD&fjSqM&X-LQsS6`|4K%2yz14-U;9~!hM$_Bp?o2dDD*#Zod$1_bh zpl@uq>isbq(x*f!R~HdhLC4G&?shbd2d%v0XK7F1g!d%uY~MEj`xs3Bw&2aYLywk= zm1~Qcd1+m$Gry z7x}Mv0A47(z{a~@>9v{Qbr_cwx6Vf@iOpP}+?;zUoS6{1rMPZNUYI zGh@B?GHqjR{wJH#8>8~kZkYr!oe{Bm(Wl&4gb!MW<%XbpE$)_#S0B~ib34Te@M0w{ zNY+^5?DaH8ZcJ|0hESYyy(~&?yirPUTr*F4b)e-3b{aI$nUf+Vk1K7M{XFwJ2r!7p zbj9~&OoaKoDlK@qatt9ZrRF^{bLnrh?CVgt0lbu&Z>nr`_vJNDP;dts7m`AeXQ$yz z@IPtjo$}JLVE$PX062KWJK)v&v-j>{_$!ZlA5*EH^ldffr<`!X`O#&Bb%clTN^HCBi;jh2ksO*OeQ|so6qJPtUi7Chl+O-(#)jWr>({?? zz7Her_x_mnuv<};5}D>Mc$Q;VoR}ck5LJe6BU$lbu6V;E_7?pqgTKrqqe>R`v_A}n ziewG$Nn1gl?~S89vx%9bsX4Q!y%V@u7)(Ie)5+wGt+^|Oskx=KgCNyWYbOkLqWT+e+c&SwsDy=31w8q{1?eFg!KW%A`L9cWzX?)V zxw<;>v9Ne}crbf#FgrS1var2+^@@d+orRs92^_)X;^p9K;>qOT@*KLv|Gb8Txyu`8 zYbRH0M+XY%nkJ@>Zmxn)CGTM(q;WMlj1V*mbB=l}SWormk6PoXRE$=I7%nrm5` zxmx}Ew10mfZEkI81)k*pIz`FZ+8o3HdRjp$HYyelN%e-wfL=-~g(uK$MXKZ?MAbnt&?*Z;r5h5B!BXYK$HodoUgO@L|Sp*D50h5ssRrg%@ZQ!A;c5xlRfx=IR{SH;ayobI!R6EJ!i9AD&CSnEA zXQ~~h{6}4jwRVVh_8$!qYfw#lBk;Q#>2y^{dP=C#xdNY~bJ=|p3W>kH-w|xTQfUm- zKK05HN=i)J%s9iz8~xd!Ku&+@Wh1LEhwC0<6wcinKt zFl0g1mnt;-Gs@o@Qg>|$m_KuVsb6o`>6_9(rs|!}@Qinrk$y$!Da7NIRF%Y4N#nL8 zn5f65t;$0dx7%d`H~Tzk&73_AhoGgX?4=-2M%jRiKHJTl?`dfYzLrwZF_NSx_k$g$ zCfAti2Z!EYLSb?lDypu?a6wF5dT`= z{KleM<+q{u%JDJq-QOJhoI3?R8sIEP_N^5Q`@r!vC{*93sWKASxHYWyqTe~RztoL; zBW!dWWU3|5K^$(E(4*N@1ZTA+M&afj6D41cPWQ+37hX+x@UE!v1!7lJd1c@`JG`fO zADB^I&au1_#>$t-zCS~Js;(J#?IB&G?J1-|IH*wU7O++`7SNFS_TBnm9BJ0Fj|Fkh z^<5rvw3`2RuCYY&kTPRtUlEe(@2LSdVgG^8Dt1(O&>i39H8h5{JdZF?kLA zP{899L<{d%s3>%->0e(Uk2!uXyY%A3tf7AocxKjDTl0*=$&)3;ZO7%IfcQ&WlpYSXocnQ43;MRbfL zhg6FE4Bq}T_H7fIhL+DxZiv@Y+PqK(+`aJ<|YxL6Nuk}c%;<^2pVbZ2swDB!)v zac@T)S&VECc4!z1^1y&CJ8xz^yz$YLYg($1+Ui!cCR3nua|?d!U#dDco>9D)!C#6x z@Semd?ABL-s(W?#JEbHCHVqo_pFNg+nHt(7Ivf!&Jj09V1%-*Xq&N4X!BIBKj`jfu z`;@a>8-L4*&BG_r&gly{*!7h^6I5O34&BQAD{`jZ;BF8dp^QbOZ5TzsqAv;Z` zdUfcmj|Nw5roHUJlvS|LM}k8VqTTiK3(GU|3!=i+(wX5~)TVXIzuc%O806t2KOd5> zYdc1X(q}1^D%4StE`D%!^1B@gqwdn>P-HYsmA&e#KK*q6dn-hj(2xo)IEpCZ=2bwp zm~3}KDX_^{P8 zB;6}pD6+q2Gt{AXm>WMeHm18M<|nH>Q8>k^*#Eb*(@yziz`VSc9R(Cay`t1X^QU_f z8uBn0{gqOYVTWEWMFu6+-R~s>deY4Ni6grTZtNoM-Ba7Eq;mWTRpWxM49r~_h|71V zwDNJTPoq#CX~&DETvGnDx_w0E#tjD#3siwK*wQiP7$A~I@%;LpW}dSVM|ZKc|L<_7 z`fHxGE|ovZJFKNO=HCW!VS&oJg%_zpy<(J1hX4IO@|)!^-wjVaKsZU-VWG_RI+XP)~~BK7vt%HjWTGm9=0?OcXnnH=xM( zC&GB5_SI8l`1f8rdIS|qzo0@xfKN|K!gtI$UGNrpU77D9_5*h%7Tep~+lgg=(JHEH z43&;CqX||?x@7jrjQ4KzMPPh#w+)jz+uciduC36|A!pl03Yjsi3zhtl>kje?YB%*w zI-2l7f!rR4KYej=6KhvjSN%vZid;9yf9|5R3z8t6t9E5jkqw^7HNhCk=#%2~ZR zNj8OriwaAjPOy>K)|>W5{QwU;{S;mDocoRky5}>Zp--<)BkSr;sl~;+qwa1l0^E<5 z+Ywu=D?6#3iB&MJUs20>FM2_ot-H-n3dWTgW1y|R&MO>MB4uJHBke4k}3hD z@UoGwNwMznvC2h)fn7Ew^jrzjFJ5qRa(e$rW}=m#O2}iV2i5S3_FYAp`ijESj5f@) zdXg&6nV_oLUk_-^7aqP8#?lfsVW>4IFyny_)J?9gu6wLc-!r~=@dAlDJAluMGu?Qh z-daPeO8>1yTth=cDI5$tvZwPKy{zB|vKkx{`I)hKuSX7Y+uw?;^-TE9`P!_8!gc5S z&FGV5QT7HAp1{7QbJFqEl=KRE_o+{&!oDb`Ij{xOkV1@Kc`s4x+#1Z?h%1F~n&o@FH_=06qBfeOeA z}ibb+}!s+#6H)jp8I5^vrGTMZIXyGwfB|M37w-iiWAKr zN2s$H_CCA4IxW38TFLHo-WrT1=5Y`(cXSLm$Sx_FvukW^XyEdI=0%l$^ZI2UdBD&c z)~7o3^z@|KjGFWW(UCjNnsp_!dv4#^&nSbglcSyB(ol|+^5EaTAo$R%@+_vT>&Wku zWIoD=#Kg4a<>fy7V`3<=hf(nIdS-WY-2Z5>oz04liCJ=6`-Yg3oJYNTvyi`+p+!V z`zLV)U7}hL_%jiged_+5RQPh&?RSUBH**3K1>fJluK(C!2tD-vY_*bJlf#luuUZ8Z zaoF$Wb|02dvC;%#n#2KCC?+R;Gu1hL8fvskb1Mlels?5Qy**~^i5cnX^%D~lGvQ%j zl8(!5uR^H7-@2HR-Yi8M&$ir4%* z{`XX_hlE$d4Ss}kw(GQrBmR#7+NEHvf7}J4{VZZit2w)`QB0ejFG- zIeh%2*gt>%Nb_Zmu+Y(c-gIuBakN<%(bOb(=mQeJpP#-D9%#toU)-^x5JU;K4bgw{++NeQch!mtHrRl=uF zL#|Fv{i4_*cqAlkM-zo|BlGj~ikw9D-!4|gW__Vqy3rTs!_Uim8)L?z|7EonWSLXd zQoQoYN&_P;t>VcWlD8gM*4^9NS@P>A#XD2n3aKknHQXI8Dn^dqn+2I^_UHqO1hL#% ze0+S!w~swF)#)|HkM-a}%{iWY-9FugLKPGg^tj{U{zj=;zv?fz2}c5u+7&bT{=OoM zOM$_p3nw}U27J12S)SYt*#AwWRg~A<&Nol{Yi}wWAo1u=alk`wG@)9E7-iDv_9PYw zFZRyHfB@Yhf392O? zeGoO7%a7Kl$CLAt>yhzk++H(NkJJ9H;~UxQYzn5f_I7nqT8?~r)%#EJUKa@nc;G!F z5f<*e+}t!Ps;+kYc7JyxoKZ4U0@Bs&@G#THU=l-Y$zlbIgp^eH%-QR&{T#plZtwP5Ib3SJi0U<6&map{gHR+RsM;Xi zP=Ff1oFI*tl9EDvb#>MIBfERe$qok>*H2zSL9SRn?ZkAlNP$+XLi=rV^{9_y?syua zOOc_MsRe~!W~ZD?%#>=oufx_2cm2npcl~sYcJt18jg6WqV02!Q#470Pe|lJI^&l)N zD)JsYis#_qFbKF@3s>5Vw4&-CTUily=!+!{pSNq|Pq&{TSIiPF6|1c6V=_ z8E2E%Opdg>h55U5+m3Kg8FKHRzcT47`^*&Gwq z+0xbBsb`~x9Rylvm%Qe)1FY!}C{9j9QKn+pA!f$L%2&sIgfR1k>EzeLY z+eT?^t+ShArZgVSTZl#53X=(eq zCMP+Py4U8~P>y*Z+)NzP*4EaGM#$Wj0hY9Y-*vr|vom{L=VKO7cvfo7e>hCd%WY+nL%UItV_^C@AUtzl+2bo%IVPYCB+@9@ z4E6Ur;i87BgXwFiuCA`_Xe1mdA?AOzm7>AT%}t`GxA=2*`Yi>xtsuqX)9`g}@hABB z_%jm|d2BxCzaK*jLj11E+@;w3_k3f|Y3}fEMF}12pOL?Ehxz*jT9zBMD2ZUO$B)B5 z1qKENM_tCiqyDf!?*h5u33%x<&w2xh*(xX~sP#fE!`}_O*)!6%%*Gs;ooKjQQ&n zA&=}MVWRBF!KrWG@|;#W{JKzwL3@JtlsZC!sx~f8DX*~5JP5o+y}X*ojU}Fg%mqCs zROp{GFhoO;*mHlTlDc~yW8bzMWuoIPNAGC1ite8AG;1=yX?{|&&Zxc}z0XJFi3Ds6 z*w^*m=u=Rk(Nn;Ld`?N(MK~-er>JOgu2sIawUyKZ9_k*Lb9jlCmL+zr5`CQoi;W$x_mf16N9~zO8y&`t%tjA;A z3sdoEREv)lTEnG@3|vC)0uL_vo?BsC7H>uOxBbdEhgR881;|)RL5VA?s+!>d0C@BC z%GJl`W`A#Q&mLr)!iVeG&igv+$)d!Qq;nYT`}gm0!k4STM!~_s7hv9|@q->iRINt} z7wE#601UPcn6WF7@VndD9=V|-5JcmDOiSBs^uN2V?~5ZRX3?qLY>bH+&P=H24$Qmv zOe^d{G_0Gu%`FOxb(Z6gof_w|smQ;`x@w?*%&=C+T-qfhyx8!AvFM(CVOG_^p#31z zW7+GA*Xgg15H_5gn810-`F-(mD6u_W{Kt=*_yG7ifB^tVca{H>ft{pen4YomYS6oP z@IRn6Pf1D17EQ=9!okJGM9MthFHT{X`|!u*zT*Qe7a9h=E!!Y<5}&NWb7F!DHiM&= zCeywn!+VW|M`QKP*%dYD368a|R5!P_m@L@S20+o7U)|eF*HKqDi~TO<>+6g21c9=! zpup}h?&oZ!o(F*_wyLRV-l!E)jucqtT_ERaLN%bgT&}dferjrPFp5VP?qo<$sfosd zKw^!*2jOnrvMhacUtQ8>VCPl(wm@2uRGZfF7qcIO9;S`0so`;dU2sTANkOCC`^9-H zYQw2t57g7!qb@?Zf!A9xv2A@Aio>MH&4mrNAHrHx^D)!T?834Z+Ra^xZl4pMz2zmz zfO>#jWobdDRgqPv4P>$0cpD1~P3PSSYD-So2q??q;^O&Ia-t|9OlP<{Snif93?9W# zSVw75i2tNc5UOb`@;_FW9Hdc)8+mtLAgLCdYBX+77K5HWuyjrCMIfEqggsG-NEAQyMF$u-RI&^QATFo7XXSRSH8@LG6xRe zG~kAbSXdP8D}R7_S6hzfz4q8~@^mp zPkg6N|LAQKf_H(cGUb!PL{1H&S?0`+6_;k~p$y+~1b8?m(C1o|bUoa!Zf<^zLF)Dr z$eIRtgN25cmLnfLgp`^nU=(;fR9RNaowfx$clA}rd6j4qO=uW4_LJCBnW;&Tx{TQ- z&dJi~3F1=JIUAK|S8Gg9pYeIcqM?tr@%zZmNUyVtFCB1maaq6O=QlD1Q!yzi3F$;r z%xHJ4y$hr(KQI-IsNV*Hl2>8ae(ftF@>TRo$fCO;0u>F73EbsDPb99M;wRpkEe)or zE^eG=L=6WoEF(TNL(b0N?tX7hU$K+7fn$ej+fYqo?*NsD*VOa zdXlUo-~Q*28V%GtKXOO&9)co=NGb$1%q%01ZTeM$f!PTNJ!1 z#>U2W5?UU8`0$|(lT6^Lva+%<${$N9Gk_Jp0Zczxqpp0id0N{MGqW(RKp^KW9ig39 zQCZ5UPPK|nlyqxV(O<|jmMjRxsIjrJ84eEa@m7+uuWy@?jZN9MaTVbd_4eW6NNiLT zD)9)>H-L8HNM6<6T^?^NKc_B-vIYcBAI1*U5zH9o=YRdZ#d7(pnqWCt&@j`C01dWd{EPSJM5Z|x5K-) zKbp`O2&G>Lvf4*s?x;vECzj`nqi<<&js2W}lN%~FTxhymO2W-F8~gG;I{HT_6a+0V zX)t!f3k&HP85y>6#gkWH@ydc9TU%R_P^oa#dqUDOi61Gk31K$?1$6Hhu(;d^>>o+w zD%+^4s&ZpBKEVdqgeEbK5mZemSw3fa3QjPSll$?mI~?mM0=zEICY3s8r>84P?Fkoz zQ5Xpr=mZ7*VyOI2&yB|_l5Jm42a!t}*tPgS%)|Tbz0hWzy-bQgf{DBiMXr-C4(jFS zr1MlgYV&;Dx@|H3GQmuM}5*zISw3Ais&2k@$vCoAt9ki0GeH- z(rIpJ-u&I0-exgq*&H%vt>xqAPuiNy8lTbujE0c*9%GCou9>AJKVYQHb|(sX9@E3N zoOro4H8tB_M-wcqf#$M$BF%WQ(O$|xU!PgIRNchfyfCP>m7kx9$^Onsb#{gO62SAs z0ked>fb?&*aInCBb;IRpekD6jE)A2GT(Z^U{OoO(q`{n)DwIJ6o30t(!KpOfmiutjvljA8dU5cDqqQs+TmKtMnc3QhZC z*WC6}kwp%_hr5ex`gI;hwt2g_Rt-JF{x1Tbsd1tyh4ATh zDE1OU)DOV(1fWbTw5t3n5A55}0kzru$4yx`095o03>XmK;#l{twxMBoslisOA~;w) zHZoFXK8?eY`-}#KoIG11L0UplPEJo)y?hFACz+s}?(goV@##8n;GN8eK*SYvpHSxp z4IVyoeQpqxZM19FPmEeo%CR(eemw{f%+zRczDcTo146CG0dEJ5GN1|R%E6G6{Www|1o&^ zw~s=NDhoY5yRhrd*hsD~O>kHLI+y_v28O^O)LZ7Sfuai)G%+z#DBxfsTMegk?V7Rc z6gcH#kODaV>B|@10KkY1@w^o)hp#rCm&jV*GT%;`Pfe4DV^U~%YZNa*6?*%P)>!Dx z$x%3b3V&>$M;>V!)XQCI1V7wSGBPqo{n*m|(Hg$Kv2n-F&i=Eut}fFTz};m@!t}Nz zP_fhv)zrk{V61FxFBupa>vt4TeTA|e?CpoQzzvI6mPfvSkN$AIU)}jh(C7RSX80Qp zA~ESu&kD`8`e?{ zVhsD^$(Y*t2I0!Q;RlC<;5_#XjR0Tr<=O5eS-wGJEENlj<6Eeo{GTJ;Y|!f73x7K= zvueMLp;oG%3z)~h6%~v~Y4QYD*Vif0w*C8FamY8 zwNHC{dtH7FrKt%7+~3|*FT6?zQ|&cqs%ybA4-}r9gE-pBUDx|Xhul~^TBWR?Q$9rr z@DIkdkwR`qCF%6mKlLr|f+VQEp{EPcaQ|IiYs5S*85R5mASlFAVqwKiLfIP79BxOK zU#;Lou|G65T^vJwy{|Dpfcm*06uYad@%hTi0j7Y$m4a#-;;K^08<5_!rC(fU+_r;R zh4jO09b}5+&0f|}a=bf!`-ZhmpmhqHfS`( zz{G*U@R;ipgic>W%wdvG&9?kFrLO6I;5|CI~HAPuXwG3e>*yGct;{RprN&`u=zFINbi zot;S_;_^b%)62^%dffZ{Y-LV-vZAw}Lb-{YII%g^Z-;|}gX+TK;%Yb;ur~0{WxI(N zs)nMVpmab>bbMB&=^4m=5X*H}$i?yWeZ|k8`io!<_;P*Xi-QU;#0G3F%LJi2x9R`7WhWw6pJ6CD72*|lkAxZexUC6Fe4j1f4cKN9ir|os?c6}{xOCT zl|N%aq~84y@67zimlrYVNZ4y!VA~vPrylqCl8gk?Nl=iY+xVgNCVrsatd?7xX`=46 zCQ@hnLr`x(Jy%knJ>DtcHxzrme?L4qKNoT^H}9>hsaY!k!4M^O9qfikD~J%w;I`*$ z1~|B#jEt-pDqHA<1J5%Rw;{Er1wdy$&|U%Rz(^D0BQHeJWo2cRo}6rgkOBI%2s+6q zJTwXjRxDYqI-Hr{ zrmy_!mo$WY9j&c(oOZ@?Gx*q{0SIa(>$+Z%l9*FFyFo3u^!x8acL zKw1%)dS##@F6EzAMyb1oi|g6|FFCHw`N_tuVul0M9 zIjH-OeX%s@5rO8On63*)D$cm2WlUVpRZM~tg?Sf05`ASBqtUPZ0TXajK(U-DRV(H` z{MmdO53m3x$VSja-8C?022+lh8 z*rJsX&;hhGG}k$OfLrDQkWoQLCk@3Wmxzc+^F8(W&)cMJe2)*rTBW%~l78dJ$r@a& zV4RYszE^58Km_gdi7pW1gv?F3!T8PgO8!RkbH0uS$d?UvxL zrma|!9z8WzOZ}@t&-$n_u!9JbwfeL4flRtnoKd?k9lyGZ%PEzb+T_HC4>)zRqMRkI zTTsGpC&z}GW#`TQ^P2PH2Rx6foSRM#a(vCE;kr#hBss(R90?9=xZNr4AannhGrUI$ zzp#Xt1=1>q&NMtpw-g;G5jXEkS30EE~eiv6^7{nv* z>siM}M_Ghky-M2!T>R&Nhx^e@^Izash}58F8HIvfAK_AAU{%?`G^NxYCy4q=%T=Go zMoZ7P9D7K1@G*6|KT2aA8ajQs$=ZRb@Ad(X+|k(Cw}8H_Pq-cz@+En)EG*S8`1xD+ zqb`SqL+lJ^8i=e*{tEuVa@@gq&qV_pL&Ptw`J;XIuq&zSLsnIA3Ks}d6YNAn6P)6a z3>kQTx7zjzQ#Mu0XtO|O@5^48d;YyNjgZjtXn%i+J^%&LQBgzrd3oOqwP@!th`B$Z zKp9_)-VLh6DJfbwx+S~|sHoPvF9P=NHuA+0x$eWZSlE)=YM(F^bmS~AU?vBAH2wI6 zn)71m8Fk;tDOKM(5wl?%Z#6*9??IV_yM30C;5as~)tCk@_aQgeq_>>Y-3+j2a*h8N&3*z(fzOQPUAiI#7z2Ug_mW?(6rLS!}~xxq2!Yja05DFGqWM!JNSHSddm#RT z|MaQn>Yuf4ek9c2v4o(fwTIRQ6pXtkDvlrs%=HVjT&^n-XdZm6FUmBp3iBIJNu3^z zEu)+|5HJS4O-g`ab9cu;hkQ+s zdI-Sv{mIqowi@W1xqLiG;#t;77zh`x3NqH%X0!Y`?snzCJh0I}G|60`@!TfM{=!4a zvqe;GGc2K^dU8zs`fiw5k=w7=b~7lyCAcme-#8a8DvtJRbjzX>2leBg9Ms9C5I8?} zhao~))o*SqES{e|A;k3?0HEgPX7i6pNonsrl63-%0b2vG8z@r@hDOzO0ASZBD@)7K zdTnc^d9?K66ux?`WSsNr?2XCR(3${M621oW)%{s}aQ@XiXat5IUvhAFgCd}BV89WM z6D&TNtHLuEQT6g&JX97oTaP+!1VBeJ2pT=?hGcKTx3QsU2bvc5Ge^~ev8h*=R}pXQ zpZ69NX{I`Fk8k$pHz_VNl)9(VMWM#5*gM$9gF7T#2`+t*;8l~bVs=Um zuyWe+5Y09U$JA*><0RgRN8dR7{8D2F%}sQ4bb1W%wzs!;fdmc`VgH}}K$bFTlSU>z z{z?9J6LjE^5H>g)@jmRG7_h&-?M#t;iW>5UfR_w&N^ikr?^HM9F7js3J`gxQ;(^bp zFkdRpA%ENo{;DVq>ZPEk>)MRIz6wvoWCB9fgsAA?m4K;Se2f*reL#+hVbwG3la2&OO71;r}E!`BXmx6(RTWIlIug$ z+NZ$>9*-G!TQI{B9#d*Mp&rV8my-1Eg^#H%b~}~Y+fO;N*908p0HCl01Oya~T3iQo z40o$_>S)@lkuuxae!5yg3=1J)3YuRpTquy?0d_KKcJ3Yj{yppsu-7mF81O2cxjqDa zc@_zO+RuT^FkGNU_2ttg7LAn)To1A4ikS+XLVT^uaa zsj2Rb#3#xNm?HJJs6un+OmjpVa0|5*H&a|zz9-i68=4PElKyHMGhf6^q4v&y<6e25l*xBi&WRRIwe*kXt zslz!=cy|rdHHjQO<7zjsgNVnD-EFHcW4g9_C7pt{x?Fej(UEQ(GKD!0C1P9B7He4v zUxqiO;B#jCAa)4|U>}7&pc-(;weveBw@5tS06q_3iED2`fG`|4zQ+{P;mImMaw_O1 zB7g?CZq=ZA^=fTlnGfH5qJA(Pg<8$ofWz?h&n1_0-T9C*7l$#xj1@Ygn+S;P=ITZjF{NFBaukd$5>?Oa~*y+Pp;r6 z$DMNTybqU)*-O#O)fPF|97+c8xd#7oF(LJi?uGwWlUlOyi$TZ%EA096d}puPoXK5Z zVq)SE2)|>NF^0!uE*Yi3ymB^L$2%bb9msf<&~%4H(Q5A{=AZt1_S%Mb57+4mcsoCt zatjRCl;{d$zdPnZ;sjrYbkH&4{gTJoNP&S53Bui(WFogg$Vy^W@Y4w6&&P5(O4l62 z&x)JKG%JSR;aXn(*}R-pHoX-bYxqus*iu(1&Z(f)o(?QszeckI9)Jf54eHzL>+92@ zN|-4e;O`djvd=FF_=HN;%RYNxF@glY1^6DJY^8wP%bMd*G(vsQJv4y99|Igs5A+7Q z5e10y3;>$G6h`gJbOl*_ZZ>a#N2Gq4xn2vy;XavmPTo&Pn9`9B^gG^pGgY>2Wz#N2 zwk}968M$?Gz~jYuQ+KXeZRmI3pYVKfYe^4U{%P_SC(5-SNPuAps-9+0KMrh(SBoXb z9W4lFjW(c?qHa~DLcJs)%!Ewp<`3m*IJ}@Wrvz*k)-N&{pcx*ABKz?*7Y43s5fMh^ zWo@!z=0q6{!=$xtTVyRWQ4EbV5R-w;ujs^6pgf2j4?h*wVp5l{gEujaVSnH!@El-9 zrE8=tH@H=K;m3U*rOwtP5~;U_FBr2>VFo;_gU3d>_zGlJotWDm+IRCD!#_|vE_9Ix zwjUNhH&Y_|pCtI$FY12Lc;yt>x^A97!{@Wn4TC{`Dv-VKaBy&fF~8{oQ%Q_gDNzb7 zCFC3cy*?Qq5CDyVi^3i>X)+257}QYn@h>?O$hB^*fUp}{@uDohHGyRDka*IG~Lt{0xdBYeqIE-OhZ*;M>Ef)$d3c* zFSQ6j<@KFCdAKI-M?=jTg*q55*f zeqboS>(!fLw1DTTy&~prNeRbEn#o0nc{h`1pU==)U zJlcA9DgI7d0KGQ^j3`Qee4SA+zd|F@@foML%AnQsITaOE*oJ$8Qur^x3I!Kd=C8~zqK*K?B`28);C6^l z*R0a_8ed+%*EKX;USHE^VK&;4CTLL5_JW6DBTg^Gm8Ub(XAnK!DGgHj#?@J?_5;(PGNkC=Y5 z+6VxV3;;fb($5{Dml1vjkE0^Q$#mhF0q!{f6x&5$7QKCAZC(0uLH%SPk(T4{*;!dk zObof6j?VGW@NhI_83DX^22e&{1K+^Ka*NwO+nYBVeGtFz7gKS6zHj8;G2qweK0~ov z^Py_rY15pt9u6^iUoq(2g0B?}#|lRs_S_yxWi((|`8T>#+O^Qw+f@9;T`!|1HlGWUU>F6m_1d>k56BcZmw|pQuB!S06cN;xw;SKtq2771 z$x)acI1)iEu;bxrdcQ0~1}!vD@mg4%odFjMA^J+aCNkoQL*LVy*ix(?N#jpoxGGs- zhmhyziI)4p3-UdAiPtoEsHHPAGd}^B{Y&`>5z+9`JPh{b_C|~}NMeFJy@=n0fOh}T z-)sN=O*_d34s7HGRSs2<0&_w*4PJejMHj>4W@`K_ViMBmtO3Q!cIsZZDqq zH8sZ@|6p4p(uc2;Z7q4Sa^22r*aJvdd)ka8w!Fb;X-Mf$fl;g8Z8=bFI+OxFP)0Z7 zYlZ-dB*!|9;l*L*RjkW|zzWBkT!*?f`H9~c0A+B46e1p8HqiP7*oJ-nv2F?5-d+E- zhDc1v0e!MLO$>=50R2?9wf&S9BG@=OB9{Ve@m=$BSnILZ?0$&d;oS!~e%DS1Flz;_ z#~SG==f*XS^b}a1B z)!pk!r9oVbuh~JOkPqvX%P%~LN6W3;0l6m!7#u$Uv{S9yu$PiB!o4du{+B1ehzDBc zkU#hwle-vLR*mW`6ugbB7pAADy&QcRQDI-fgh^&JXgMna`pZ8CRmeL9 z=`bK`;D9mC?T?hFiH0iZ-`H`$wo?O__=3|1X5eadKU?b#Kk~gj?i0pzH8h$Jfdvyf zPXNUR%Ix4>nV-Lb4@Lm+lE2QAFlYgz{eqW-4y&2<;%`4LOoS%Mw6&$MAhK)-uz;G7 zQQ-Bhh3R@V<|obXEj$FHxrW#Iznf15qMud!nXqKd3iX+YKL2U^q^^3l3DnFN+}!S3 zz?0$WtO?nwo7QZLT})!{BG#$^UyBhcs<}!NC{?Y%rfrm^$!qCf0#n zwjMa$z5ti#n=If8+=htKDBuzUsTcfk24xKqCZeWRr=z4)oen_9fLX1yoLo8NQwEzr z79e1Pj)B485BVt}i4asw2{7q)F5!T{%ZYadSsOrx8lHcbOYN``Q5NvM=Zd;vmIpvkxgz`o-q&`MUjYfCKo)@|R-d zufwHUN0h0;q1^bP?HLPj=C?D9g{OzkOidMRfzxlR@8}5_>}_q;GT6-rjixgY7#sDX zNMnN~Fz7sevm2jd1`{=^#t*`b@s@5?;m=tPnxSk{Q1GHUEG%q5tJ=^Y3n;{ki`B6d z$fc#FGUY8Tx=J8Ikhxug>Pd4)hhb}3nT_t-x5uBLGqA`6Qrn^V0d_%f0yhIFnOx8x z89zV?f&9h*94dn$)0L7cjswgoqoPw+i7=`|M)iazs zBmQ)#gTS2Y3OP`YIv?)IV1erNz3?z?V7nf7QBQKJB%NEcdcwFG{V1@;Pn-c>>_$yh{1-!rDzwvoVz%Q|H z;d$~~%l&!$OR9vI3{Q?TaScjUO4TV_bU^Pam!%(7_&!P-6_(~)-v^Vu*$~f;1QbQ9p>ut*ESQhKQrNltp#0 zn1B^o80hzGKpjL26{82k1ArCt-N_G@PQwo0tLb0AEQTSr12mbVeQQpFY;0_lbTUsO zCD<^-_jeBuX)b}Jf98IgTP1w;0A1mlJV<0GOE}>E4mu6Gw`7lFY%^7B9=LFgJ)*lq zxVr&G7XQYto`kL5)u!+1D`@LrWLp@hBajqdfQ zbPMwepAF@l;BL&BY1<`1!KWL$*Qqsz(o%B!b+_Rfk{O>Y4j#;6rsI7pt@1>m9Ttxs zV=fxoF-#xapq_7HDL#TNNtH^h| zEBM+slR(Nj_y<4+Y;AvJUNTSr6O?M;U0)QpF)v#yn=H}4T}X5S6THwT-VqkAk}0a%<-8 zE+F3%Gcz-xtgP(Nm{|~bzADgA&=&)q_XO+jgh$LIyMy4JXNALK>Y){mCDy0r%DrHt zOHNhjzJ1+cW~7iuvd$lPcOoQy<7h%$&fE0SJxDx zbRaeK6znNl&f>IIeIq9qjReTAX8*gErOkoF2OBt(ib~Cif~s0&u*ZLjdLjEaE2}`- z`A~tTUfNDF_5*Ppm`*Yh*dd&FY3I?sS*}dlLR6^I^gMms2E?1}boEhe7&0u)P1i{=4sf z$Ms9E&GW6HyL;m#7SPcKM22xyF6NcV^FYI|3|6DN(P73t{2*|?0Z-OPQeZQ2gF1@H z$Vj0Q4QLM=RDOYB+|~c??=03PQ_Xx`2@f^BEDj2p@JLf7!k!-UH$*dyDtzx5KTb?u6XH^3*D`p{D54g!m#cZLw zrkR((2jvLJG=`NN7diuY7!2|>8g=-3^+H>7&NVh)O)+wM0ioClvL*Q*EO6ZbTv^!; ztkQ4v^fCaP2#>l&`sOfxRv$KyUM=b_p7PSaF*mTN_$r0k3qlK^=l=nKlT!ts$9tr) z%NS_?khCsJ`v4u2;X{&MBDcB?z;fa#E*M^`}zWDWAAh5fB ztLj#OSx^Z$%j2fKk?aX`zzF%B9^=HF8nr5F^$rXk9+P$FPteO`w@-yx4RB8L2yOgi z+V#4xm9gKpi5{yj(Qw@}ONvT3G=zL`Swemb!C=lpwxmbR9bo?~+?``f- z$?gHX9JK0YJOkC>q5Xn@*@^TWbby){Km;hy>N+-&Bm|1TA1J8RT{`|m zJpE|6GKC`l4QjRp&}Wntzqs2g%VnJBsVXr9Wo6GZ;um!#vW7onWt{C?hc2wu5w6JyUIz z&8?Eu3{kJ(PR@P}XQ)Y*pF8AjY$axQoC(w1|E@Fd?CKHD)02uN znU7V}eUTvDFVe~ccE`23AEZN6C1l_cbrf-xR5-eN`G^wG*;oqAq(h-$x*U)=^82@4 z@v;L2z#jNOoQf+#y^!)O&s@G*>zz8@KO37gK<;d?dRKR-pM&n)w~EaBwnp1xNLxZ~ zH|{s&!qd>uSoGofUNE_P3;7mCMijRIgSvb$Dn^|k4e{irMl*m{!FX&x_UvEwFeDqP z=W)R6!>C(pHc)B5*hqR1Z70c|payLJp&}Hn7ad3^0sFfpDH%590%bFkL_G5tQblUZ z8K*w>a>e)X9POVvDh9x}an$d)a}uq`BHrHKt^`=C7AS=5vva>fCS=qAp9@$hVtCT&v^xpfgCPu&;tyZKHMf&PHPMS`XNL`}vb? z7>0dUAhVtzu;3YPVIW2nI0Gm5cd+0{MbnPLd7XsT-W{6vnTP>F2?H2jW^;bSRJ5If&`Rhf-wKTtZ45c0DbyAlV8z9wVK~Dynp4Uf!C?W zK~wFM5%Zt%k@J2B*=YwcZy4lggXP}e8}CXslB;SMZ4vTJ9~XMnuFnUKI0&}YIsRi9Kc1s+!nNXN#< zPZNO?ARRJ*G#En^k^G>Tc9LJz{tnCy@;~+9P#S}FhMg0o25vUCpAjIF?e-enfFJ1^ z_z;AEuZFBqJh(gv{&@ZuJ2NuMc!*9hvGc)G-_uS9;-BxFG|Y`n8#A|hDLmG&@veS8 zUnYNJe6Zq)-=uFK47@2d=(Pg2@)3-!Z@u1s0M=lGCCzi=9K^d;Zaf?s8L9n;Bm)l@ zLNp*zfC`pABa-k5Nn)AjyZ-0+LNF28*--WrWuEH=?{8s&YkdWlB+3>@9)3Xz!y|0Y-GEWDt&*Qb+x1FxlDl2;v6eEce#ya@tg&6vLo-SK}$^$ zHHufQtgI|Ge$%easq|bQ-&GMv-i+)29=*riOd{l(9aO%gTQPMyP_t6Z-1W)lGJW7x zB}p1Xnv0waLtNI0zu1%2#Z_@MuXQurkG`lw+2?O=YJMY2Dv47z$d^pqinM^W5TpL} zBbRCuQBy+keczeGubVrvJ&flc%xxjm?cUq+Jk=S<%7Il3{;@XWlog z&VwnM+Oy`R4%jNA5%J{bfcMI!l%ESbzzBKRTKD!Ud)*p`kAc;gp zBqwVZm6X^g!j|q=*&?6y3z0u{4vvm*%F4?0dBCDR- z>1|NJq%*7V{P;^tPEI}nk8xpo`fVCOHSDez$s$bOhlZXSzGMtm1j}awI;nUt!_2+Id^kS6WOG^C-M5bqKGr| zCtsC!wLSNn;Q+|w^U^eQ(cih@`8R&ZcJ73rsilR91ynEr&>Xv9RmzbhYbgt{gl@n| zOV9mUPFGDWv83|JIo^~P;xXrnH92HS@fr=t0`qz9jd(0a9cYybbXbafZ6~ZP0a*B0fAYtA#Rzproj$G#8o*oq#@=glSNWv65zipjSkBnuSb96zr`Pg*XOR1J%WA@uvI1_8fkiFp!vjrGZ!UIo z?{%0I;McB{5H8p2)N2cx?;a#KjF7<_x8^Ez?hfHAIcD@i2nY+0-Ur_CE2OpcyQxW- zA^CExiPRJ59}Q4JT3|h{MsTZxPvJ3y6*3{#CYS6usP z7momRonO0cKkGspsi~VJp}S<0M5;0L5QzuC%8C3_;mXy zl>rK0v?1L{4kDu0*1umx4}&@}T+W+ltD{0p44; zx^QtTq!)VEWMB1D@m&enckbGDW*%#KCENDMQSwLHh_YIvlYEjGnb|Rf*yhQ|$%m^e zD`yN#G?Vy0KQx;C3>q5~78Vv^Xh~xEVrqV zwQbkli&uY{=1q6#j(&}o;H}z*1|LtbKivMk{03a1_q6^+G>q`VX~@LM*#YD7hA;gi zjOelNN2@TEH2~e8SxLidz)3oNB&_u#)dK`*CT8Xp1$p^x65abeR2^J5ZnSZ7bN2|t z*||RuK~~gg-u+e zBUn=Dte+TK{4UR*cOENtj^DQIdT`YFCt=u3m3O-T`Q7T=gW{DILF}^CPggb~HL~ig zh$XMcedxi(nZ91iHZVL)zOcC1ANE!5Nr|Q=WR4{Qn=b}5788qohm`R{c#DJKI}?Cv zm4@fg-9mrm;sJByd>i+#wVfwkRoXwJp z|Cgu2maAx$dM@W^!q$q#;;hzyCv+e2fJu1}bTdJTVO@QF8)lh#j>Jby=!bHw(p5!! zsGpzsa+>}F>~prj*)acD3p?ug{X35NNAtcE#pJk}c?jWELA2z}pFQ(xYi-SK2IQ9F z(-~Ai4Iazz=H}+n^Jo1O=!yQtB^hqEElY6{X5_dy5YY8y&F$2>wLd5z2sE(G{kR~Y z%_+-lt4q9{y#mLc@xP^|R1lDd6M#tHEzD~I7JTD2#YG*8_FfEdzEwZIs5@u-!5X9A_#Hl4NP_%@H4O#>P}Uo;E-e7v6Oh=*;-8tr z`HWKBvKY+En|ec}q@+%AT3T-DkVT$`d2I3W<;yOcA%^IIaCUZ1X-*u$dn{L`$+49A zFoFtp=ZgjxMq;>m{nn#z)c@Y9?Rl`OkVs`V(@|X5Ic{dZPNv;xHz@I`)qM5t@Y5&t zu9jhOEd6C zfZ*eX>(#4o{x&s9c);?qh5e=6bAN*cC^9e6y}Sm6>NhOo zBRL1zkvNg1o@rBaRz;=XvjPE?=Ull zZjKwEkq|bt&Am#B9Xwc^EBsFJ!RL9zn_om!4HqL@zxI_Gks{y??KB+Y??1Ka6O7vOG7M z_KHWul6sGR#*usc^-w}gRZf0Bk9hiiM@L5+Wca+8ZzSR1;R)`5l~xVpNJ&Vy|C0X6 z=|daUXgEO}=z)SDlC=@n>&kKO!(}gJedsf|c2&^n zpAtHVe*6fnt@MySD!mkK6rIK?x}Sc~QoXN8mH3zU_S48f$MxrJjulrDwoJOYvSMpQ z80%!{C6zNCDJpjH*uR|`CK7z30NqJ9*xA@hrlc;XEpO;qs>c6u&FfrDx~Ir2WqVkw z)Tf;`U6FYE+165nUMKd1T81z50mz?J-UjRz1iWt*90k&?%9|mxA&tdIh~M9smEHdJ zOSua2P9eAvJ%IX#;S}Nx!^ekyr>77u|FFgqZ)|LQ?%|a?a%$bBzn7k=yUTp|8hTz> zhhafDC2{&HU7=xA>raKlOiTu0Zi(J3bGaYF!R80f4X#T0fE8q;DgXh zAYo-?wcCS_?}DHylQnGkkDff4T7hXhzvR;=PBJyFHWMy;ggF7u-_+ztx;rS$c*PV; zt8Y?|ZiTNw{7EvqD+m13u(@qZXA9c3xUk=!dRJylW zQDBbIPxSBa>!YHdw3Qhm918^c%pJ~$;<`GojLN%wc_#UnU5>-5JD*~2@p<3S$coVJ zZ>84HbFh^ZN;#Rlqx(#Dbs?!!ysm!oPJt1xl(G!%XdWv|&piW`oK9(tI@jtu$NeR* z(3Y^cxVRm&3NF}sQ4Wr!wJy=qT4#2r5*!6sU9Xma2UbW0M^^uTFgw_a89zfA^a^Nx z!|~aRdi0(IEGkW~RgU3E5d;j{zH-l5=ni0MSNx+~TaTree`k#QujgQ)$V268l$1>E z>He`c!XAUQE7)w51;J6?_gScq=~At?6zX#bWM1u-0HN2jv9ZB$G^ylmi}N#rDC~iy zw|BkqqetN`pe0>R>G%fW6_4{5E_??z3fp_Vg?M)Ci8BpGeu2965V&Eu_k4-J%_r*38@mylxY)2;k2Ju-Y4@+GZOkh^Q-dZz_t@cJOND?#GyCriY5IgH}&I4HC#EefalS>$Lz2-l6 zE?@{c%d|Fa?ZCY>i(t1(fkes*vvbWx%iq?gdU?Y&A!L((Zg+f{%}|{+asQ4n`NrZ} z%AotjsWf0gC8Rv=hN6gT>+fTQFyALNT80 z<=~8mFd9OX5P$#h;}dwy*+}tl?@*a>@vuJ{s=lz^PsV@`ch<4ssOv@G`81=n zIGpBqUuR&r4-Jv8Tc+2)fW>hM`WXC#XfFZDX+R2?pM7Nn)GPz=V|5l67vs=IUx0&C z4NPSf4^F(NzYz=ruIxQsG&mg!I2wn~m;1AgJ~dDme_zObD>VK6`(6<)E`PW9Y5FDZ zR?6=@<=d2fybX*A{kNaBS-tw4*d?e@@tj~{e5Y(J`a2-ul>P582vC0pE_b=EuFfYv z5zIny_4VGIr!Me)jf{?R$Lx!Vz?}68S}f^$;5Y+|*2TMAT)9BShfccz>s@^_D#7uoMl zt>=>0B`PiopY5`=?VNDheHJE^vW`ppw`B`h1HKqBSN0~ zf9J`p44Zy_qqm3-SoaP*W0zgPC@fs(4r=Tcc%ty%(Ntw;e*+b%Hove?!Ql4WC|7R4 z>?yIa;=DXO`9Hh5Tq485aLFz@gN1}*2w-fSG~**kI%a<)(gwqrt)wKj)|w}HyiL5k zjs<0^98aj5J39-13=RtPLlc#}rujKGVN|FnDV_)YeI&+Dp7;kPzJUOCCW81H8ZU!O zOE<)zl0m}TiWk@vm1CUiL^W^2nS8YZP~XuT_QQ7flcC_Hfwcc?1w*uM}GSwU@9OFCBG(zmeq z;_qG=H?5?H()g!iB|*+IpBT;tr*$DEjg+dwzwpYHmp`}vE|8VaV30p1GOC%*KM3=6 zi&cMN!PDCmuHG#qfGP9q>hw}Xd2fVce}UJJ_>J;0UfW|Mj=z#}aFG*cPgBrJ< zNIEUVFcBZ{6y)9d@AC02wts6mDtw?>NPzq7futlYdUz!_t~ZhTMnx%bKkj8-T1A_| zRSF6vXk99SlE!y;0Pg}tK`;OQBKEmkY;Q2|uW^ElY7lbLbn98_5cPS91*gCf-*2LZ zu;L`clVzWMYTu8|3|HHV8?0=&Z~Elzc6?;4`$&DT<(<$k)u|$^VSQJ7A#sSOVSw7L z)Ld(S7q`t!6$+kMS<}UR20atVYzhw=)69cP=4hGPwQse?n*j&!0Rk`T>Pf@Hfi*JH z(rY(v-Qt^O*-N>?)fZNyR+Lla{;Q|LdvQC$)?7H>Dowp5UE0*~-`3Y*$_7<;ihrEG zOs*qzGN*jk9DCIa=i~Mty_BV{s~6VeHTE`AQtG|~3Chy{AV8ApliL(KjR6-LK!qTM zb4DAHBNzF(Va_t)`WOpsOd&<}2MhPWb2uiS!m9@zHvGOCGR8n&5tA{?!iXvn*LxZ- zDl085?F0eBh6zvDTcq=Ha>AHA>>V6hFMpc|YYzc^V*@deV+$lbuK;oGD^Q^0V=^{TzW_Yr<;;; zts=27-lpo9$m%bl*_(p`;l>QkoOOYe%Dhsg^L!C2qI$+AAMZ_%Mg9wW81M_Pc4YMX za_6-i7lUDTgsve6uYe32pU~Lvzo!YS0NQusJ)l}RVH(_7LCP1nDK2`#+pup^-Xsky zjd~K)U@~>U7&!wf=OSbtB;a(DS=&qVKA6Xs;kP=`2?9n6u~9_l&WJ0Dpm2)0L|Wzl zdjY^hx9lNz+U~&i zC!?USBVPH!>?^;JjbY)jo$~R$nuhhGw|==#Riyk+B1#2ZD()P#*120vUx^TtnO}a= zlcDt3@Z*Po9Ls%j2F*5X#wJjnmQ$AiuHk+1kKf}n*{|0 zw##7k*|Od{-d+4I-#0a`H(T9uRD*A$Q9mHmyLqBYzxJ-m=gsbVUcXs2kzsDrWaNE4 zFYX&7qffqvNN#>q5vDV`Fk_3&2)hK-FkI%d8QB{EC?Et|0)_H*C7Mo|^Po z1I-qy1{|xIi*UqMLVf4oxvsQZY%r=!Na2HXu^GDk-hj?|*EPj>NV|=v&a=wWrKK$D zE`e8;>;VL93(HB<1*YkRMF<}Y`PyqyQoNM(D^DPGDa==m{&{`-v5r+hk2+aISKabm zU&g~|7!$^U=zQBS$PEWp0-SI`$};C_{b@@8jy|B`kB^J*|NmZ5R0F^~v%bnLyZ6HO5#Qc13!|s?zLqkD~tQBVu&u05^Rj=N~TE|uN;K#M>gQ-*rIqD7&uXt0xwo=QEK^*V=o+Iy3 z>ekc>%fOE?dbxf83}DVZ`&zCO6p$Tpl5gmJ0!vAypq9B!_Dtt3r|6!SK9{_NxJu;Q zJDMo5G^f8s=BAcP1m>RC2oujMJ!}gE84$Ez)TYGk^!z1sI6Cc%| z+yar{wNEFb+d2rpdFW5Us9Fa$ulff%IyJ{&l%QLun}*5^Xfo9Tr#QYuaW_=f=n?1N zG)f^CedL?&#|=)WWAKo^2N~7$Xuz6#Irb#geN(4ZIM_BN&cA7-P+!So@KWx%ujwPA zj9WA}@vvwFvE}U@h!?A>seYnO<~bUcj}6Mp{*WzN zc7-mpA*=1UcfROxOxaNqDpsn1V4kcL?9J93Ca29E@cLpkFrGg z1R1U{g6uR-+R2Gu|2*%|C%4#=E^ILOdGq_uF?nu}V2VeQ*IGKZ`gjqx=lvTWcL{y@ z?G&nh-BDV-iW!w;e)Y@yrF--a#pc?#_=1%BgDNJj+2`NnHsT`Ub933J0Sa0)(%?PL zrh${>X80bEAw^?QGmC5_PQU~SUm&E}lhp+1$it5I6C^RQ^LXdqgg68HNmCrSRm8+Q zX-q~<&CIIlgk0B#0a!Z#hIGrmQ~yos`<*!z8_A*go&$xXyFNKbnX-)weGIpAy$SKP zYaXye6)5RT{8zEc-8QR!v(c2r)$h)}S62U!nD8({Z`C8od0l<9@YKXKp!*y>eZI8G zw0z(yAa>LvVzJ8Zvsvj^)#r+Bo@SsSM^?M!agf<>(rQ#r+4n|zM*r|7<#4;xQ;Iiu)VDK=<}Nc@)P##hNf9w9J>*xQAs~h-gNu} z3Y0Yh1>RJ?@DqN4u4)@1ER`qVicU#N`l|(9$hUC+gc{;-vat(^XBA?>??AINM^v(aK$WtQaS|Y`6fk7B(kU3jQ3DjfNOiT~@yUsIB0`g!TU&Bg zz%VjecA)m1=Ll&SGCW0W0sxP4&?Or|Bk~zT$BxmJ2xVpE43HVKAy3zNG}obef1+$- zu8(+~uk-GY%(F1j3o`5C9-)Nx9C=<)5Eok!Tx-3ZKRxqYX`}j0G0v)!^q*Cta8V{h?Du}t(evi3;C=4QHK^7kc#)~Z%7|tnh|S~!=nw_8$VEx{dL0}n zZ^(mw`--(?DX|t7ODbRo`0Z|NJVG7A$pIHGd3&r)G};CgoD)~-dToir!S8%y>hK~PlFG%0{!Hl z`3+Ke+suQKG%x3DKIh@+)ptw#iXLBzhhAB&_|;pD+xEPAOz}dy)h;DkC9R5u((7XR zfX@x8eKo_At~8^r=-FbvS-!Ps(Q}KU?m?tnV)y}8vwHM|U&ZvbwC3jGK8Q?7@X0Y> zCI>_cMVk|#{lBFI@$3yOc{b^DzJO&yLl~9>p~hqo828_$rKQI4Y*Z&iRF997V4bqJ z#&H5bL#^70_4OjX&Wn9|X%>8^KaZa-3KO1g4FC97ndvqk&zB8VUw?Oszbrj}-+oN3 zs9PVp_uex*@68V&)PYHFLW1J-x@$0!){6x|H(RQ#tn|IiqOgQ&V-u0G4V|*q555RM zbYhwst?H=_V-PN)B(ntIZ#)^7R$(7C0!PUU*LHu?exxQewGPAve_2{uviMqhdP6A9 z;mBKMU~2A}p=QsXBq;%jSQ|H-Tm|BCmh=M=7}Up-LN;FV4^($|06Bgf|tGXXDf z&CmzsQ7iu_aAl{}*;sepr4aon>|3o-FQSdB4<+qLipub?tsHNZX(&6c$OX~6-QJiX z?Y?&gQ*Ja{@2Xj6dFRLRut%)A)|u5RmM>D97z}^6)3nINuwT69|vY|H-Wu97NG6aW!t5( zEbzR*HJelcQY419wnbDv{#Rdf8{s7b6YnPU_78B^8fs8tutALrmYq!iY42E6mS_ks z97D^DY17H`)05*~h|Ux;F)^t(RSzIKJLvB5_1SL37WcfKYcA?aYct)tN39`(MN@XT zY)voBaXa(4(C*-6Yj^&Q=z!CnD}=vspg-YN^w#5!Kc7IA|Bsjkd5YR%%Dcy*A-7Y@ z*5)vuay&8O8)YxSD2axCrnlEX<$Y!1b6i|L%d0fDkNo8>olRXU_Dx$Y|7tL($iEi1 zfX5<)V>gBG;`&Df8#8@dF#YUMMADbkD}{!&(!?imNwJNM*Qb4?IP^sVGVZKj3R6>) zI6crQi!W~Qg34qmq;L}No&Id4gVfU1kgs=wdjBE{pYY;ER!ET?>uPEa7?n)E@IQM2 z=t(wM{&fIR&Fo3PAw~l_u;ih)jR(!dFtS32R+Yumry5(;cpn0OnNHAoZ3jO&O)oc+ zpVrZ{X1z(iWtprr+RMp&dbDy-ROBHHfWL1xLf%$aAILUNpT}2->1YG-PZHEThZP&kM0 z*^rY(cBBwk61HK!dA)F?ppiUmP}tpXo^sbX@55kYfz7|qF|G|CZH)?=bJ_8Zvmy&w zh^}r!?k2g^c!F)QYaw{p?M&fZKgO4X!|s?&uH5vD7+JB^r2v&ZH@j0ymXB?rYd$BV z7cl0gF|Pw}kp@Xx;q;^B?rB9{P1{& z!Cfds_-4hKD8UzvT1GOXkZXDcZek*nBLvpYU|=qggXbUuWa}T$`%*6* zEep(95#rv#-u5 zM&FH8oeksFx~M=9p5TEgf46xk__)7S&!lkEdWV&IThLm?qoKapqa~u#n}#fpb*jrD zz2q86ABV)5V%tW7)~gia68e{yl^g}hv{^7FAYXU7Uz3(~?_XCp&tHLy{aR+p(%BhK z>;weLIifH8mL3a+4FU2-;V5svvpVjho2qfm@*KqY&wKAy04N~xqXMMb{@;kY)gX-{ zKVZhlis&ThVvwSULCz*?q$z>TW9+R0`Ichr%ior!cPd@po1e@~CWxqa9=$)n1PqE6 zbl{5P5+(kq0AdMEzo^yq76aZ}QI-%T8QA%rUVp!c1Vf;Y9H4fUAd~d0f6!mpKF~cC z(X2O6&pAj3WWBebtp6RrIV&)I7fk(&jOBU;e;3DC??WH%{_yo()=v`;`|rI`QR|~` zFR)a)7bn@~+Ym+{ETOHPJcqO)MCaGr;Bv<`pqbOPM;bfvnKnhOdw)KqVwONdN6KT0 zXMw`Zw{9?)RlL+Yd!z0wCgiN)%d37hcaI?#a8X#0kk5w>D@KQa&Sw&nOJbF*?5gES zVJ?S>VfJnLHcQVvct7(UCLoE{Vv(|$g^6iD8ARH%n0!Q#K3P_9_5+QG-RG+r;5$)K z9^~nMjL20Z{Era%gd}JUlthkLm;SA5ZjS%m1Df9ZlhScj1L(nH=zzWOIJq<~(f(X(DDt zNm<#5U9PbPq-V&(saua$Z$D+Uhz4OC$eW5ltx_B^Dw45_2&+U!3Vi{A5PGbBB7rbd zNd3WUC@rZ%N=oSBs@T*a4qwE_@xAFg3(+qqB9p!K+U}((Vlc}|H(#IpwHf8^x2g|p zh+W%#9L7z`c*VpUocTRtr0TiBysa*(_4&RwG0#vZu*hY zLd*Dqp%I(>Y6q+Lm+vCTru9UE4?Pjs;uQaALd$3*!aC5ic=H|HjRByw!4q6ckCgingUAjii5>{; zUNiR~ph9XCe(`6`x<@U%oPUuqfgEZd7}z=53~#TYBlkxw96pwy>|TjzX`%x03U}BJ z!k@h`ha|Duz8)3z7nbswtluI?B$j z!#7oA>5NsEi|B?vdfPE?vtRVP&f#=IEOKoA)@1Iin7m9BRn}yVf{V-U2*S&AcKqfy zZsY+=75@lLJ3y}z1I)Y%IL;jyM`3hwf?8yt2gVXsYhvVg0T44YH+K)5HQ31Qf^YL2 zR(UvZ#z&rHh_^gGK4yo;o}BnZDf9&4)f9ewwf!V_c0vL~#Qw1=mm%3X;i$I8fotQXF23Q&m zX-tvlu-HP#)UP;;94j;z=SQ{==L*HElIj!wV80agrHg61evOY5&Q?yKweJ%#=(G4x z!KdC!JQ`6A92~8%uD4vxrxu+KVz?T2T}<}1+nd<}UcQ9TqTpdK+Rh-FpgvqoOqsu; zv4U^?!pwsIcZ)z4%9@POZ|5#Jp~bw)B3i<~V;LtwmGqN*|IMOrJij z0U9T_M3aL9l#X5at@HEemSAl(hiubDps%jIQ-@lZY77RsSm4LXDJU3?0`>)-6438? zi=f$R43am+a4{G+c=9esw3HI9$)CT3Yvco#q~#Q-O<9CuC3OYtgAeej-kX2rsl}&w z>`Ni(Unb4}Y_QZ+F)uea5^Hs{$o7^Qx%#K7{DT7Z8Ru_Ow&IU;m@YQy?zv{**WLhIPS#vlKZu-bliLnMD2O}R}8!oO$2C@7^> z-j|g*u7FX)F_Jucwo(f^u2}HWKOq4P89-l1 zIwJOI51$rF{2PJhewUx$p6a7#{aog%y4x?NhV)NXeI0Yf)f)qjIudiU#!>hawj4Mr zMTpvWDp^MUN%$;ApsmGpZ@T1e97lhD+ZF3p`s&F0XFgR~0$v(R*2dzk!)Hu#;o;XA zLXuuj*nI-O8i^J;__qR?rni>}3E7_w6i*_50z+m(`KM3A zh^N|5JR7Hz9G(dZcl#VCCnr99^5Mb3fohtFSM?GS4C;?I+E(|YYgE>s#H&lK=6hLu zsy?hN|K3h8l|T?jo&i$+iToSMEG>7i;X?bk8Xx#5@oe^_*{s~%t3%RD#f#^V#R?>{ zVetQi0|g5$!@b~e=KhZ!=JmhE^Nwl-jP2vy^G*^bCNpQ0i3E@V;szlu&H7`+&?1u& z(j$lkjgij}Xf8;@QKG$);$rae@C%mVd!$!pw$0{K*=uQH5-F))x+@La95R0#ej0(2 zrsxM}2clCG?vZ-~Wd;>FO`~hO<7?-(>I-?EKeEuKsOYM`)048PyZ#dUC;RVrBKsS2 z87+Hmc#pzg$z&R~hzYMT{UKocQoHM%K3$>xO6j`)@h^tr2M;R~TTDm&r#^~(TsV(; zCbhTJX4st)@wP8IH}|8BkHZN@5zkc@YQ=xUB?1NW9Kj8>(s+1ZqoShr;mVGE-B%7^ zfKs(#ND#y9!aQE8&^~_?TSz!3zXS{pI6AA}3g27N2h7Hk~_b?n6LS7^Y zo?GC}S+5NqT#F@_Y`q=1cwj&X{_CO!DjW9$~p~^K(I!-ZkFO zr>9B9*!epDF5am81h%sKu3W$Its(S`)tz8Kn6&UW{nm|h+&{mzDkst9l^u%ZGg-r= z(;OYu4vaP8p6u=I&)}GT{^+O$(k`%!wClXFtH(!2Bp?q`E0cuo`rD-e$mU^~L;t(! zA$3&w97&@WuV2Wx#>&b0DdtZogc$CO00nTx`yb>-#;Qb7xz*C9+X?{DKJ#ATbCfnvv z*^)Q?!kX-88`J%a{-dFdhS7XJi&tJZ)icT(d|XTSD20T*a2A3w-Pz+`p4LPK^m0-9 zinEixsl&Z55pL}I)v0oK^u|Mmx@L0eaZllA53^WVMt{0)D0xf=6KJ6Q1AL;}{~7n4 zsml>pjar)i?JN&lVVwe=9)K;?0%p51xE3Y?swZkh`1wCNfv6NcJvDU}0ZpItlUi<3;muK!FlMni?AK*f;+Y*?(8$MU4rsNSwMEo%WZCPTXTA1M& z6FAPlA`(@#RxR)!J_eKHDbbp-xw&~2ud$)*A zBvHXr(R{ksd9c#MW0LgIKZcFc_P#v!)qvbG1yI5#)RIYuJ+R4T{OPY7mp+)**YeuE zZa-PpctL7%$o3rOU$0@A<5kv-_l-Y#=q&3`XkX2SOnhTE9x%yXKkXykW#UQQC!sAr z`^EFpz{#wr;oGAv<94pvu&19FMGNGQEl0^QMXd^R?XP>LVHRmXmQ|ed?W#r8aGwFX z`UWu1~}(cyQc8ah_>bYU&6poJXP6gc-Ak2?WmQOb#(>HbjfV3 zaY|fmVcb#Bfx7dv#@UENniaw=e_D0D4G}h6lloeYvcmT-=|itark_U*pA6wF;GY+S z!^)7IMid#UkUkIme<+k?<%9A_4zZH!jca}2B=<-DQ?!;xM@C8^=Q#yuic>66GJW0+ zhp~+NA%xIMPmTh$Qn9OG|7e2S1dB}tQ#8QIN)D!vgS|a4eCdqCk=&n?oy|O|@8Q}&Mc!@XN1aNl7vg!S@}%mDj1TpPIBknpFi5hy_@j$P$}E8WN| zDx|MRepbild|$66AZh;EUMcP+?a~T$qkJ$$l=f-8f>|_0Uih#FmbZKZ7L8ayXHw|{$8DD=r*V)6km)M7ok)6D6kk$J@)HlIy#;}G`YdV)S~mJ1R@>AVEa@AI#Uwf%G64I zTwsM?CWO!eG=gMB0L0C~jkJW7GE&v4^?c57fS1>kYTCuy_kB zQOMTz_9Anh7%9t9q9Pr%%hau}+f!mNH(h9w1=-EEJzFEAzv`OGg|OOgufpBez&?OQ zv5i40$`dJb>}+IIu#m4Es})`Ln_sao5J^cEz%q?xF3M8#clL9)jXk)Ep?#_}fXz!c zT(kZEUVvYOcju)tj7Y__Pj6`MUW+PHr6h0^3G?m6#~5z$>M9;$VKZYe*cr19*J?}a z(i-am{uUMYJn*Vb&B#|f%3D9|iJDy$?e3VZ;Fo43VFF=h0Lq2F!tYzYef#@Ym6_Bp z`;4pF8A-74H+4n>t%ikxD6S?AGLsVp$ZCd$HV8O9Qa?-85f&PX(*-XKT)SWn>?KSN%&U^?e^R82@{D%%FWVevS1EK#XTk zo;;Sh9?OalM^8u1wYtY}WRI~a5Nm2{{~m#dW@lye@GJ7+!syJd1%JqV9~TD{?$C$` ze6dtU8!(`hGC2KtH)>kn>-*yx3_SPNlN& zSzj){PqY$BXS>@bj(wKSI%6nKg-Hs>EU<1y<6mB4bYCb7~27IENv+xC{rxnQn*F9BZaW=}e^8B=Fw7SMmvbRK-p6S4!qgg>C5O z1zRk{OMn7ONkgOB3TZJ7b@k*>|Lj;*h^@SVCkb8f$ZV`8?73&BsiG2ZBIX{2t z6(j2z^p%j=7A=1?A#$s=9`q=RdaaLntR91kak9~?DlPu?_BqL_tg`1KXE3C{eg}+t zedXV~b%l`zXqd`-P`|D4TT;ouV4M-~ak9;+{OXs`9h&{YJnICL5R-!tf%sAzxT3pq zfFq*bX>~9Pt5Sf=oToJAy}0Q!lD&dAGiR%7{8H-=702hVrhl!cJh_(s`6mEaDbKj z2RcOhuREQ^2`Wd%JIE8K#e5`Qld?B0nF<7Sjm)VqJ0D2Vc{yne{0)}KU}jcQ!IjMv zGsAl?L!)p-*QSBsSkuK$MH!n7Ghu&kqGE+wjP7cX{g*G-?_M8Z%L(tm{UMVzz{1aH-5diIr4?CU(%t1X?8fp+Dzn-sFxwu-l#O?M?=82j*AgpyKD}Ck?%iJF)xr z1oyV;Y2}lZSAKTx)B zQelH!yn*JJ7)cm})e$~TL_r!vVQ_{EYAY+a8|q3PyoltB9>(c6cJlvkbJ+EWJ<}q5^TC=M!K2e*Eklg!6fDY)tBsYJ2G7u_y)fLV_VSDeXv37w2{sC;WB+t-2JvlOf5TiMQJHJF2pkc zRQ{ELEt6Xwhkv*(6^%lO3I{E8294cQ-lC72r$8ta=sl9Bmeqd7Ym|u_UaDK<&-k{3 zj(J00>hkH!x^r0p0cBt9loR4tj+X85z0)=i&zKYUk%AEkZa6($V_Aj@XJhpGK441) z@Ti0wcN|VAx&{}gO`&ita-gB}ai3CPe-**0rb`xafTYY}qZQ~q&jmyU2vAqRx=#XF z3Q?x;k4Lob|7gp)z%mw78Y{u+t5}{drYM{dd>hWEW@t@JagsI!iBt2SUsJ#1AwWw$LhU23V^?WHgB^JU?+ui zv7-%K+4%tAa~^AQR6_(mFk%dxM45K$L?z%Xy?XCC2Rtmw*_p3AuuVz-=5&rRuejk| zbe6%BI#J?$5U|y=oqLQu^X@^jd1Zm!_Cpdzr*gkB8e^pt#)O`4iOE@c?;l^l zfIgs<0{gCqwY6LyjCT&%=>u>rg9L*8sIkumo>FhFtuf?-17BSss)3#9?v9x<5i1NO zPldIjrw)bjf|XiFP+P16uE6OlN4}Xut=04)aVh(|(vN6Km02+qA%)(87k@Tf(0;6v zrb3yj-9K<%<9T7O&}Cc}a4Y^r*O|jx8iSG}XzD@}7?gnEOG!!1!X<>@=3mC2otcpY zfWCmEu-55e~3m;qc+jlT+8Gz{v9sQgLwSW+MCiw$5RK=^SgjewW!>hZO$@ zSZ9w;2Q5D>y~`-1TAttJU8(WhYjNzJXNpeld+1(ZpEToYB;$cR3{bO4hYWZ>n%;7J z@Q0BRp*LtDr(v{arjWS2WmA80u$a{k9(u57y}Mz-K_jUV{?WvnCl#0I;dxzN{tKRbrs2Xzt*^ zy$-nt(C^_GG=RV84u_m(1thZ`D=W{VjS0?084HogFLKpEt|l>q|%&BhhJ!_Z`|5K%Q_Mu;!K2 z#F6jj)JGE;5a_NlZBHmKVvm7;svOa7Mz2#__i8I>Ttf1E|3aAww~PEJyt4M?~Y#yp!3c2OIMJg-%YOpA?w! z(3mvbR%oByg^7agU4jts;_u4JRN@;B6kl=Omv|2?T_wn-R_(co!_oESi!T>m(19`0 zm96&i?jw1z@h@tu6mKu3@M!pF|JnLmelG926^9ZryFKgbxcLww)2Kn`Rsw4_I$nC< zzsr#8N0th3bw48oNL2_c`yE_YVPR^jMgn5fZd!IAkuK7QQ5K#gcgcxcG2mwk-{*1L1fX>upG~gL1NY zbjkQGvxg*zeO%B|=jh_ng*5ZHKzFf$BaVKhgE~E(panGqk}FBdY-vQ|{5jxb2B*0bo6J96iL%{C9us6I@^N97(Tm<@DhC^!Hu-fL<^U zY{9)eK8QTBZ^)>zHE*;g%(#|W$T8eA%6GkwD1K@^@UZjQXu#e%(XqD0t5gQ{6rzga zBzXSw;XIGRv$EK|{vH938nvolV_N;A6nSjWH-0+OnG za)dIXgzaxH{#c+pD^`TH>ULgmQ}N&RLB}-i}%DAFmFdIP>(P ztI)-XUePOB%r`y+&KDj1DVu=QcKsAo4kJ5Q)Gg0edvtwK z^rBIUJ=wLEnfyXlab_#~_6NrbZT+gwcrC51QT1h)Pp_|)U&ArBdiJc~3?LjH;LNW} z!t-Z=wXvEr(O)+|7X<=r3}{YCaDeI*ziEV~p-VZ|m!EsteI z%EEG>x+!2$eCU}E1hcL2%=poxAK-GQ0gMo{6YR2{`h&I% z+Ul)P;yu(TfZ{3?Q5^fxCE>+hh`t$=R;}vH>bw|Trac&gAJw}$`1zYgYL;&@hsoxP zlzs9GBB#D1J->SX^p1@-n{~+@*+s=IadWzj4aWVhcFUH%YcorGuvFKt%oBP@1={Mt zcE17LZ1g%BI_dcB=ppy@7~&ehL@CFFvjWKo(fN+Qt9{>rT!WrqAq|KIy=jjJw99@6 zRy99J;UbNma3aL5A}Q?xgp*8wxa~;DJcAkd)WVZ*KjPm)#{nAzi)lnI_ii<%dTwA!~&Aff<}!i6(@+i=(>B_>|IEZHCNKX1Wjo6!Sq z@bp~Ue~T?_?L zSV#&=F0;J+YsAvND(r5lrfSi!HI7!;jYq}c411ad}R&C+*+9(Vsu?9a6TnI-% zB&o#c6bUDnbZktq-O9v7ad7F#(rT7k5;gU-+*wn`zB1x1hf<#c0wKq`zjvdh^CpT%7x#N)pFs1SzN?bGJBuCp88*_La#Rn_wB&xUKMoT z`qvM)zKv=$NtA?YYH20nxOe&YeMUY8RNR6>C2JwiUW4ojHl@EHFH>WEtx|mJlHx<` zwf(kb^pQo?uM!uLG>1p8yo2L=dkXs8Wp?N|_i<0Ko4`4ON;A-T1`F}vM+2;p@9@tA z1IL4u3#gj>1GmPvN^x7zZ>)+UM;mgW5Zz({v48>TSWwMtg=$LfNR-)^unND_-yC2& zicfpGT`4(o@3C2R(?&w2^YxO_$?=)amqOc|?6wZe|E~V-eVaD|zI+~Ruo(%lB@FSC zlXja^PT0HAXjxFGHc9hkzISbJY&5dj-t63k0S;J9 z$*-ea$ZhP|N%X7C8nfM8liLhgt4<{ynq=iPMYB<5VH zGJae#E*{qF9kP_x=4|=XIW+^D|B-P+5nbRb9Qtb0@;pr&djU-F&C`{-VD9 zlRj^h_Xijiy@mOfShH}lb_>NhGhDSa+~ORuqZcDk!f9>(oSHdAh~;>h@Oc`Lp4 zO-b!hOMIa9K1I9a-e;%0-nZKFvU5KLhyt4dF%Bk#kv9*XxC(%k)RN{f?h8Yna|VtFMc5{itr>ztfr3%}i3Ar5THee&iROtEc=$9K=K z-O}QM{&(}U>zMXj7Y>%VhpqGbAB_Aou_qU##K$)AZTu!s5`NL-$Men4d-b4u$>HST zc?Oy&TiXp;TKlMuxpP%uvcCfU*w3ZGYUg?GQ{-;}yKxs!^#ACF->9j^#Rn5HEmIA% z-%_PQ9In{-_*zzwvn$dS(X-GBe}DA-BuVoyS9^VR!!d}Or+#nQeoVqyQtMQ`^8XL42LqP7+zV^h8 zXq%UwjxOZ6m)A0Qf6e@8|B#n=m=G`0*p6f!=|U^UCSTUPA-y72DO$C8e>20FZH7++aeMEm0Cru)NszGA-RodQB_J)s;H=+-8W`8##5=9-LK8~B6!@) z4G(Kd!Vry@zZ-0IRk><1=DNXS)KzRZyEK9^cgl5m9RGN>_3*;@xu?q5;)X`%Kk0qY zx@q;-e$!6-+_5yi5Z&3JH24|~A-6^uwdIuWp8ylX9uS=eaBeSFo&&3rB{~s@Y2V7q zXPz+mlSG$MuRm!PlH-pXrv@-i0hB&BG_jqn?ac>`TTnpULZV=QZEr;WK(|2{uOQ zpn}0M@a4>zL}cJuRZ_y<;TzCvu4W}bOPNP$xLyjL11|yPnncvHa9GZLl4-Gr3ws5) z{d8^@M1RC#e?2tN|5s3!^E}S`eCSb~jfk%Kw@cqSFLeJ}jnwAz&a0AKJv7Qul7G_; zjGlNna`-?H`2-rJz2NN&CFbN@%vmtO=cJ|aBgX*K(RcHuX0E`o)&tc|BH!h6!qivR zdceW+J>r8+fN%PO`edGELRq<{Yf<+>y2*kx`{a{>G$F}m=}phzSsR}ft&Qr5hVh=} z$Jo7VV)bf*%y2zq8@|4NHi&Zi_dh}j2I=1r2< zqFRYHM~?ShmFPP(C0TZ^?E_w=_cLf{qhv$_E)&2Kc@{X6iCbwbiLY=Z4%Qn+KG!%7 z<^J;!FPmgYaz9%7E$W7yENnk=DJ%2>x3=LY1z}eqU%Y9;57vg@M&yVS2s8c@M4bV! zQ-NX?(~j}0C1YsI$+I}~EnaBy|P~Qjy6P@F4 zjABheCXcxB1_G&4d#LAh9`~< zhZ~>v9#Fj}12b8$FJA}5TP!dz@IKrais;qN&Hl!3UiD;x9~_uCk^G?xaHTGo2CkpM zfDimxpuRe%6ct^64hfA7AYF1BwHm%Q>9r+(v+&!j=)pln%Y|g4bKQo8}z7 z;y(;Cx5(0t^tpUEo0>PkJT|Ntz0mhnBEx@hf47A%lz*2s_mD(O^V3o{lcp=FD9?-|Gf8pme4JEqXVUNa17xs-#`arUJnwOb`zJj+_$m>Y<)X&}1-?Oo6i3(-VZAeqw zk$!&{Ob{)>eOU?`>GQE8pFp|diLPW&V{0?BFPsB=+ZS-FLJnjpx^uzu#eH@zLi&VG zR#gTVTrQn&nR~&f{TUGOM>5z?z$Oi1HmvOK+zFT&%5{}e)aBl;I*3dR{p&}2hsk7q zjC*^2%;R0M`JF*Gn}s%f;&RjYPcOqHCwUdTlB4i%nmPQB3ox%e<^-xrLiqHd2~o|3 z-0fn;K#v-NIk^e-4cZKkw6ylHh9AP=?2iQ}UY^&P!QBgQ(f!YxrU@ptSou<{j}+6O z6@ucxH4tqsPbG>`iGKiNUiS-^h~%~(iCA;*jz(4T?4=)D^b7M|_9?S!8tR$LRuxRx zuPhojB2ms)e|Xtx?^6+xw{A#w%NEj^HDx`*JDvUSqn{{ce-GNr8~lnmYFBd?9x&gQ;L47yqfiogJos zhxD{Bcs2|g>T;pr-FLufribB8)*r2Ui!ixbxf@4j2RitkowsgcfI@6hC$}Ba*kZP!^s`q#M5f6 z$H}p-4P)(h71zzHSFfJNOqlzmd|(SnlzQ&54QYu&GY%qtH!uHV%KkT(oRZjhI!P*N z5~X_gvKtda>HYhd)uk)dbIN$}4lJ6ySkykZjYu$G?6Vs$&1aqRIN>zmL`d_AlVCUR zW_$ivGnH2}{Z9Y-xz=Xt$_qn^cU0#tQ<(YPi^PRl&Fch*04J*P5@XPtH+{$f$9xRF7L)cQp$@&hPu!Onb_Pd1U;Z^UbDVQ3t>*LR zdJ-nynt+c`4f`!NAJ^yn9a6#2u`A-$&m6xLD3z%(fY zO#BJJq+~zAgZnq?kDCJ$)nplwQxyCs+yjp8P&I>s=G-)j#dwhTJNk(`7b&Sv}Y?)LRBsBi!JByh({$b;lAfKI~XeYa^~VJRhRq>j5fNq)`H5I5%uX z;0Eu)R{U%@*qx>tUYv-D>1N(PJY)s3+}6zWbm|{yY)4E6&&F;J_7t0jwO@9&j)!ATT>Z0NPb^G9N3%tsKzf-&@?SmW~$()6)1 zzL0Y@;dQ-gdXaZwcdhP_ab{sA3Gq_i8GrJWgH{}WdCS#h-20wnJ~j0R6{mG3_0NT} zZD0uh13ganizRvhCaoa~;9;?q55Uo0Kvhu~*ufL}^6ERGgrsB{^26K(#|B(EGTcaNR&* z&wTIQ>83OKBz2%&Tmk=h_Kf!s}vo2ZM{5pwF;IV;UN&p%*fWJFjXnGBzfjUQ`ruGyt@pTj>L_Keo>i z5=PiLcRSOOTBjV)iS@bqPU5Hgc6GL>EL9kW(!%2FX6@#kA8k0i`+Hv1^--7?7XOm> z*CfviN~*~T4GA$Ma#@`WZX3?nnVTfU{c_^q($C*k0+7PDrLz&cOG4YMR=x76!+00` zl?elyz30g;s-WlKdckgb8+sL^xAi|v(8Uc2MCY1ALIEq=1@!h##>1azrUrA1Kp+&1 zg}OJP2?s^l{sG$p&W-V!vPW-e=hc%vK$PqT@BTMwSU$_@g6q)ICHp!@RY-hLR9B{y zRYmz_l7KQRxXsCtay4Eo{YhkwF;@NY2e9%DWMJZg$rV^ z;?pjpTnTA>7GN$8F6pfP>9wM~HKCUH{seps+T!!VQs6Bd5kSd|CCl!ax6ofGmDPlD6#eebMbs@x^)- z%3X*cVg(!9XU5o%VEpJB7$~`y*soz}x%^>w_nBSKAFR8c&QOB!~6^gPm6b6d{>l zIo|>6`_g4GxK4pSGEYoKNGvX1RQIRL4lv0|u=UBfuC--G#l#-+21j6aux+5D-YPRE z=DK(M6M+xcs2_>%5cK%OV%*Fa%8#;fww{VF`v~EEu_m>Y%q58L6LDl$7N)&V(T~Gy z@Y%EEV3M{>E4%MJaLuWxf{TqsH8yljV)V8K{@l)geYpTsu?nyv9yvKaBqR2$XT6{J zgZ&8>YE#&`t|S3w5C?;y1?*K>y7ZazT_P?McmM`c0mp9<_B{MUn->AxkcIXmbBO-b zKzQc$ZJ07qS!|hqdXeO%1{}SrY900&9swX}UTD_~?EC|pqcULL(;y@W)EAjRr2YjD zjY!Qczo*ikVN z2>=q#^48bEwqG4mSyF%rS-QN$=3Gcb}Q`qVs%caO&ZGFx}3kp3UaAwml)BbQuLn3*lM`%(j+Pvgg9AB%pLGFL0df z$-!^<&H>iV4J+>wV5;u}trKRB9M6wNJ|M}Lc&Mdi0SUI+eHZ+LMzEVzToH zu4G~`68AcngX}M143n?cHBjL5mpR@tjkLm#+DsoSMlT9J;)!_|8%y)zZw5^Kc|caTgF$(7GGWC9wKxE0d!Z2sEEr~6E10RF zQ$y(lAzv*JS&s&l=EvJg|GBap0gpj5cpz!^W}tG(j5xdhgGG4S%TZajsM@*&uR8o+%b z1K61hz;@*Aya0Ya>W4MFAieBY#X}2SP=`uhXan>yw_G)>7`%!ZYMp-#j{fpp>}0v0 zNdNXu9ar_nJA8Lc_2CftI6mgNmYMcnZ1>qy9HFp-uk`L2Toyxc(#CPNxO~e@2fW(y zs>X!QYgD-@yGleaqDf2Dn@hf^^Xu1LZGb&8fPKig2~P2>uebo2=KC@=b$UJ3!p=^g z5A-cZoe)z3z%D;!<1KQWa6p~eIIbKICdYof@)Zp+(4~BaNlU>|`9fMiV%j2nmIsd> ztwlyhSK1gGV@r#QvOitYEV2P_4*zwYD4a-buHFJyq zMtq9HU?mXn>bd8sGTgc=>-U58{Ex6dEca3dIV1YAvi;s({ZqxmbC&hYK`qt$>t%(E zHmeM2_g%Q6&z`{q!{@3-NbFk)19*`U1OW-gKVU_KkE$zRr>>xPT~F&U?#Tzg0-af* zO9=4-WrMvGx==_QJ~B`2{Q1-M5l&Z8!^uHFjP!!^X*Q%ICjwqEB+J;5MnN0i4vSUE zYX_r1w;Rh@J_OO93*4yEzzAn|_w=-YnxCueaty294&cu!J;YJ?@IVCJ3lalz2MDVp7p)(4Xa@l!YKj5vZ z3{cUU152eIFlr6(F~0*Or+;tmXB{He+Y~T*TIr;{y{bNLJ32day_Fe_2heT+m<*fW zwG2M<5P1{$rvmxwHyA!(UDx`XVg{|9Hq{&~^g!yP{S8hUx+lzO+kz(pQ0)iC$g>+W z^6IxugDgmGU%E%?gGA$^+n5&Cg`QkXTyNbDmM5Uf%disJ-)LnA!YB+mc=IgF{hwwc=*ah`byqN zvtj~&2qI-8H3yJ2djQr}NaW6#027@lSlPvbWA3Lo>v_z1bg z@S}Zt1LW<_XNzUwEtbVLhU7>zYG+$`dvu7+6*1E@5o5MYxZEBo80vRf6&Np7^tBVWy?IVpR?N;%?E1E_W^ndAZ@UYsX~iIi zSA(D=ab_ZcALc_NBXWh%BTziCy1+0`0vN0?xD#K8L-Op|Gah8Qw+F*m3ePuG&mg-+ zWwOFXi*|T_9uSxRM!rGJS9EKDNeS&4Z0xtmRWEktJYMT8=B%84EB1W66sBTL!sH~P zBgIJm4qv3{+B7SRq0(ytFMqBrg3?qRf7NvN5LYcduC9r?runUHycGDF48q9O+|C4} zanZ~ceh?l9I32)}mw5Z;&5SYvX4EWEOOKFl>YHl#NEvBVvgo+NgP>Xa45*Yp+Uhkw9DGY4W$UNB26{DEmbRw(5s3A>=PqOm&b}2ltol^N&YiB22#G7M@M}HVjJGXN8_; z6XluLJmlwn`9-P2l^qO-2+tbrn^>EgrM=!=>lsqoJ;iH~@gh)8$MA1gH6FO0X}o7Z zy2VRKIj&p%{yZM@_asazCUh?;AiOA7p3DFDK6drLZSMvCe1A9;s&mfnkCVduT@`@mQ1R9#gT=B*e{LE7}zlEN`lus*7*c`(H(`hFJVdm z4ea!T5v?#0rya;)z8Q58Gz1kEZUlfI)er(4QKb#*Q4`Ss(l!m^-=5j}*jocC`3r}a z%8=Isut6WquxY3-xCj-hZMHD1{NCX-NU~p<))yPzdCpK9%TKD5 z-}&J;)`fTTcy%M(ht^|wQAH;s)kDQ>nQ-fL0__aVw{z~1^NR}$HJ~Ew2TtaC@^Bm% zhxoDGe#Ppcabl^(k?!Nw%Wf~u-t0|xG$fO7SMmwHp#Jp?9)5X^ z)I|0Wc=<%NeyCIU%=m5c0JowE4U9rd|*W8nQ>(6y11J}*D34?cK(j9+E z*mN|;v4=)k`mI*uDPYQRz{X-b8{xkZ@J!p~GedqM735tm6ucK-bRk#asY zuvd0K!(X#iHuxq5fFDA^tS{cDuV}!A8(|q{8W^KBNo;RM0cie0FCK&y6Rj1+a9E9T zl7Jr~5A2i+$ix?I9-~4EF0p9zE;z}@E{2ON`lFUTt>PLCLTJAx?)7i$*kAu^*KyqkYm%;e*Ib!=f_ioq4$iPOl3H@_&w5Ia4Z|OEDK~X z^XptR8lMp{^Z_0RL~`LWAMEdEfVP;SNr)Z#k_rTi!OY7dzfGCOE-=#pGhhqCAt!+% zwVCnx3o34C%{Oyf8b+Z^&+Wwo`x7AzN=$nKq9=-9q8MCmWV1Rg+}wPbQm7a;bC$iZ zru7B0GTDrkOy!A4#{i%9;0}3ycicbE3|IOp??i*sZQS7o^VPfI(9Tjm;Y$f&zWD=` z35viVTR{R8zWbkG-L}~W0j3vTUxOg?<`^YYTIrHj>( z-PjmxuB`(Sr3ylfukWIsWAQ?I)bxzq*;$PtRwIh23g%qjHD_l+H~+8UFVA}VvbG(9 z=s*~@M}FYFz#T@+4c{6Jb!iB2$*!#(^%8p{;JAhjt+y%K_@aFIH@jDXUW#HjaxAqd zg_IyyN&hzl(x395p9UT&HDLN>yl=l`Rd|kUj8z{V_iKt@<@GK&;Ni0JASk=S$VK;? zMnJ;%{rRd_9sGIqMEeJ~e=Ef@T+0vu)P|(w!c!}Y4Bl0cKzM9T`@#Nsg#r3>N*7+} zU)S15UlGYMLR2R1kBW|~Y6#?lt``CfOQyg#Fb5du!Qq#Lli$xb8cu_$g7!|4&++P| zH9q)DkmG3n%BCo${l1q* zbj2T?CZy4zsRiv=xkBCus0r(7f;SYBh9rRwi~o{~wu>OQ=3en&V#DH@7HnCweXFYp z3>(3jY5s}V9^sO9TsI7tI*Z{B2>lsjM5XAiTz+OX$jwax>#_|U-q}jxTe|&;YZf={ zCN4uZT7ieMaX=%ZHd82T*l(7|rZE3e1G?% zsz4wM9bX{at>(~x_Pl850k2Zo@$wqBX@VQdp}GJh{zJUU0rd_gGp)YKo#)p->|7-Y z`_iEKoH}x#Wa*g+``L)*yqMMCUMfi_A8>u_4;QGEnC8wVOM9BV;+KYwo;;gOX0vtXCrCfOUg zv~(*jBgeJ&h;*~kpD)i$lOA7{)wn60hc~C1dv87By@5}P7Ui)@gfUu|m$0z_r+-n_ z8HYbkDIQT;Y_Ui&m~*P@G;@@TM-_}RxLsXDLo!0gE$M%pUaXHjGsCXx+b_9bb;>V1b7-Sg5n&$`s9+w0G2At#CSI74*T`%RsU_p zQBNJWl?>|tn8?m~DMu=1n3{k&N>l-q+op#%W+z*I{D?hIP97;R(a!dDrE5jl+c_jI z^SGMWzQ@}AiZJ#4%a}R6JqIR@aL#55g|A=VNoa^Dw7nETbro`xl!`4i-a6JidW$BO zfpob&J76K0BBBzaX$%NbvWj2%H43&d??Ue~Y9AoTR|n{odmW5E<9v2q=Wg|b>(KfH zYMk+WLCeGpQ1p$c+Hv$m8c6Q{h@Y^<`I}75l$#?WLBvT@b>3S{Yp;u$1C+QUZ_65rK*V zHYSijhs2#%NW4My@oJF5dC|*rCePo`aqPYD$}M*V)5GWg;{rJR3oK;k7sKo3c*4GT zVe;%zPPyZhG;;{Zf*SqMB?1^mHubr-;4|^C0X4L``sbpo5RL?gTx={IT~tb*BGyD* zMnq@Vy^-=*(|VvNTj!Y#HO#d#WglDLc$B~7m(OwT-?2ob-g$f}&k>a*yo9xv;weQz zv!2b|79ltJy(pE0z^JDtTK!G$P1n7Olm-Xk7xI3Muw>r>JMwPe$4gky7Q(wJMNaxO zP|&9MQU$zqqM(8XcaE3UYebO$muJRL8M=S9-Nfo`vQIFx!gzD3I--_!Kpo&~QXA zU(PCyXYs!b^f;q{V*Vv(v^d&;`ZNoCJi@IdAq#+>V7vfp%1eh^Eo2Q;-^1Io$v<&sd!FO$SWn|B7)Ylucty8i-<&b?ZNG&7? z+$=N~=GQMmx3Bz=pSKr*rk-`v%{x+$4yZ?TzJkMvhdHrCWR9?G3vz&Bb1k(g`(<&oIk?qPIKuYbIzFwGauAZ~ zwI9KqBC;FTZ1?pC`Ldq<=fA69g|_t-8E?nuwvgN+7XWs;FuV&`tJVpxQ=_EYR|j-$#1a_`zyw)VZ)Cm&*#TN3(1ueAKIv$Uf3mH$QQ=^LeX=FeLf!}=xs ztsggWeT^(T#c=lUGVwn}e|m4dOcR1`P(quCp4Mv_{k&qA20Gg0@2Qx9Rqw^2idAFZ zIdn45IuvUzga6%&9I+oBjJpq1Dt7E|m`ak1n=~J1SoLap+AssHz0t|~ zg|@!IiyaxF3}COJjs3#n{R_#T2*I~CSE$Yq1wAGTYqgLypoHQtbB)3s-3m!o$K8{Y z^)it5u>;O}m)a^sA)43JpxVJm-^?rn0xgcRU%iq~`uL8mk@8|FqKim+?tV005;}hi zz?$S}l4ynkfM)}76@tB;osJz>`x=IT zqt^y6f&dI?v+k(hd`(-JUf{Fgue_H%yyd)x1#&1^*_v~c|8@*-h(5r}=pZ(tqPn@H z!PIz%YOE@;b2m#5YCm0&=i6`>@4uRN!pX%VRPf3pz;~7=NQ}?ebXT8mMbF#V~k&=luuaiatN=$sj z>v+l>vj52!4|7uoOPW_g$~6I}I{VKHb;GiocNS*)`vnp{r(TNp7Qv9VU}VZp8^<=E z6ACGsVZ9_u{b$NfLOe%Wgq2HC+48EN)Zt*3DUv`I`((3E8RKKT@-H|zh(}hExaNk1 zlQ5VtKLTh|3%+3iV3eX48Gc#;RD8Dp5WBCaIL2IPr}L1A8|`uv&BJ-wdOlii!mfnX zZiO4DlaDE;!N(^M5G&l^?L>TWd^U|cUS(~)5liFYS*rR)F7sIlL74zh&%SG=h(sWg zpwByCZ3jQ$cNUwFdrQRG81-xgopD7+)iq}!AmuHut}-G2wespEPz0m+%re4Ibtp=` zvO4%OSYd!L17o@sHZRuT2J{e#04&Roj^7Y>xbgG<`v2pNxd-x>& zf^C>{=l2#S6Sd5~mxa&IE)|hslNU5?B)O!@Ji>53!H3m@U_P*l^4(?`XLb=*l5)^0Clchvwp)!tYg~ zxE8j-PeItaC#!=EhZxs}u>iiENP)j8u*R)HKAMMNBvCN6aiG9s8v07-6NgW zx@h?xOM-pe0!AL>yn;B%B;=MrXMoo|8D2h^^=QGiLI(!`3`FCWg~#yx^muPG8OUx= z!8{xA@8Gjk3Z-#W>m_&n;)=0A{eO-5+BF=Lhu@poX$G)vVSEP{8Swvptrzv&&Usq< zdqFVb5wk2kWSp}@KO=FahawU&oMC;-4`d=>0+&XX|5SQ^`wRwn#f^Tz=aJz`yQiQ) z6sg(T1K++l8fdz&HyjlMRH#Yk@E}KB2+)7xB0?Z~B91g#(afezL&3j>Chh-rU#R+{ zj;ZHLjrZ@3VIEv}nV_K?tRrgmKEbIRXIkFU1!ui7Ms+tSF_8$pkZ3~>5fMQ27*qNB zQ+;6bX4$j#QN=;A6-~mfILl{)BF*WrQCR%3xo;&VegA zaMm?yDrnWttMSsyCUY|4$kX&g5;7tWmdk3}loD$;fzcge#XZdg^A7ws`^$ZC|LIy9 zI{J?EzG$ZqQ(w$#sqi<=7D*s#QZHv)@dxO4{V zAfX{;%44;_0HqqkOuY{_Zb=|65b@23$Aj896(jKS;)rytQ`#fm0c>GBabNt9Vi_0F z?B6;MMm2C6oZsIPts*Mio$}A*qnclEHe_JHFU$Uk-+E$HlNa3l&_2k_AOt%Q^kBJZ zX|usKHT!}}eHNROpF3a{TG-phbI!=Dr4?P741A(@nWAB{6z>dM+JL_Nwoim_t}Hsa<=#nh3eik_VU% zU4Jk~lE+5G6|I`A5bq0;8=sT?rIb3@|4(i2d-=!2XqiJN{sfWC`Qv&5MQv&NV-pgz z5NUfe^u$Z_s-wt$%;D+o^8&98eY%U-*MASvzHjU;=apa(HY+)Yk11~Q`2DJI1M5u| zi{Z4jz@s`rBHtT<74^p2*ujc3-(9E92x6wEPOw9Gq`fJczOZ4LwYY9Q{$^olR73Gi zeIDbBf8>beo6iw>;iqqoC$!5KNb#)IM(E#WqiD;(<)NpJ6_>x7*RBaXNZj zQGZ%sUix;PTc)cyyKg7^K$=f9Z!cwySD}#*r*VEmlFGE_#8QTD)h+6VXf%DUAyLQCm+> z3+$?3W36wcs2J+)WGwo5)+4mLR9sE2;HZTEo>gApl}KDOy0h;z=!}?N5#@sH4x5Jd zGY;#RjUO?_dfwHLTAtKAioEi9cf7c`WG~aY_w!~Lk2tGRkpys z{;lGg?0sL>3V5%wnxmsHJRkTH)iU#di;F9_b)XZ{=qZG3**zFw9z>697-n>~fj&xl z%=s$$=a6a42<|#=Ss5AY6vN_9j*gvG<>mSLU?BPwet()H<6wIwPvd@py~EeSEEt73 z?dydezgd%i#}Xn6D)jq60WdK#QmsIj{wbBiIsH3u{Gbzw5kmjk)1;-+rLy#i&4R9^ zz0u~wrz^Dj>kdU-^%eD}35JJOd!lq0-%@P9PcwhpSQ#W%lGO}e$WR@CAwX#WccdGTolE9a8}#>lkOjYSD?XL`8KE~N;lDadIXe?t zR;E0&4(@uUd=#`8_-2X|keF;FxweV6si5xJh6Z;RHovgN8L)?@DuM8Ghq}Hc%}i!B z6)Wr%;kw5O9XvgwqOHQuAwv zC7cNl>}4<-U;#duC%ROqrWV0tovj#cU40T^EZ=Wlc2mkezo^RI^1c_L3-j!{qVSKR zbytJ)5)&l|b$g4E8^3G;ceUGt2M^NJLzOa0{{tf013DrY`#NSE{NX-Z2N$)=`Rk)l zw&beUX@_pP>;A;d+zkfj!6k@Rx{DrExwBON>~2r%rz_3I=b!)Mlo!QPCsg?t&4Sy) zE7iTqdPD5~8a>VT=93c$kT|^qM;dXJQPbjx_(L}|ir&4u4KB)5TpAp|DiA;hhG*}a zf0ZMA25}wbOM8Lx%>6o*Qn3#Vsq=m%5^@zhLxQ~6ShUGS)5P<`^{P!9Ys+UY0`G2Z z8_p#@25+BIi4XP;JAK+vRaV7O^inAj#~R`yH&J7z+pOyCWiOBl5ELmr+5QZya%@ATiXz-UJV~ALFTF>NSu39GVPpmc?66ozUNHNp${4x&jO?B zOTfPobbtT*)|FSOQSg_ML{|k2{B&r8GQc^+Pl;vEDE|!NOoH3|an23OpU)oh3;}wg6h9D8H~@i;vw6y=LM#AwB!5tjsgTy!SGRT4 z2L|)_mI}X{@Z8c-(aX3MZ*uB9qRW#`^)rt9s3!OKhgB!0&kf;ZR(@}LB4XleisUxR z$N8d05GQ3=!1tVFNM=83Rz8MXUxOiDYY`DwAP&G8f;JCdBeEy&D~zp)DKCvCNGlhC z6oiOnsE5apQtHcLE0XD73&u?V^*DlL-Lq zfMJ8iF&AzMP*xlw;s9C&0v`wu4f*huf-Bxx7;YdeD?+?AqaVg6W+tWYAE`TKHvcW#xj@G%yCqp-TR!-pgGXFexGPG{CfA z(+tOQi^eXB!q&X_I-&0LZF_3VuDoW04q5lDSUo~|5z$EVI;G2Uv5mvv9dsQGTc^RD zd-gR#RJRv;F13Y)~<-au>oq~HORXK>A`nYdjOz?CCUs)I-#fKZk+o~ zi7pWuaw{8~DkN(D1@H!B>_|c8MUWpQp6QQMVmU}!sVVyONl$;2fD@{V7>s_msB9CH zl8B4NBFovE(CCB+=`}U(70XXsC|goF^7!*?#Kb%Z3yCzF#figjdLPjq{cxugTNv@J z?CBYKCrw!DB=b|%m-U9>mVr4R;9JkVQKcO95bU$8+qds~cS_t=_!=ml%zpaVg0nZW zU)@&?suXxp^;esAzBQk54u!$H-FEyQki$@bUm3!rBj`jaM7}Q~v+P>{3^W&fhCOu` zsK$8oYs)LMv%f$aIyeZHpP*Z;gs`FFCg^8zy4@lA0~gj#P(A9UcVF;Ku90E7rY|%m z=n@9|$zvvuOtxi zj3nOR(S8NEo^rr?(tgKG@(`&l!Im~1(WaWMB=ULwV}ZhyD$cR)@)`kfjEtwDOv0@x z&x>!(PraZIreo{ED0RKPiKTmzz0~8Z{3*yiUTpi_Nr_8rj%{DEWDeqqLt(AFQNUbJ zz(lc?UAgk_!5BuT?SaV?U*%)ph0htp@(-VycPbnhPvMRI49a@kX=m$gS&)@i5OTS7 z>Dge2)4%&CTZS4&vRNh`F9H^9mn{IX;()3F;j7-qXo2$^RXl}+CvS$4f=5tHf!Lqn1@E zuO`!8+h6?c5*{|$kU|BMn%w80r&{u>5rrXM^m zVc4XUD~$^EkN|pR#d^DjNFX+Dc_w~k)LHAIb$Rcr?V6!_awQP6eVo$IygB$oGH=3L zqVHhKt!X_Xh9`|+u~#k0rA9=be}*eP)9lU5+`943(>gODJ>jPSA+EyeDdG_5CA)kBCB$M;&Tsei0>JNMFpl(`V`?$aXB0fTPz zpYkWws{}wCIY>D@S{j1y_bBXlMtDR7C7=vDSsXaTS^pp>Iutbc&$RUQKYK##?*f={ z6T=8Yh_vJY$hbNKkGl^@Md&Wm)>i=MBl1b>Z&iz7@eB$Yv zp7q}!JX6WPrYRbxU%(AkDalme%&Cg`%6bJV75do#Fwqqs@x#Ly#?{}|FbhA)Ey#TT zbk^&Mq^6p#g=}HeuG`OFx2yE+KmW;@P?$)OKK#JHzDr+O^7?gc8m*4bu8xYjvg)pf zg{R7CQB`fh?aBfY$I{fV*4Mq^+qOZC|1e=W5awC(lN^P(VPe>y!?%)pze70$nkvK5}5#oSCpinMZdr0^!)N2 z(0^_Q&b%pekRO6fo>vW|>bB7uXU?*zW!y#wY-b02GXerA(R9d+#TV4_p&agnSI<5y zEH2i9e|v;N-W~%`&jx{l9Qf)3mW0uc(2W~;=d_MG!l1(fiBpV1Mky1_9S|t{BL#|j zz!smIg!gSos#W_ZZn*j#*_j=OVhw6e!==D^ES{-^ysd`znC~bF)aw zto7?Xi_y(rK!h>`6!XvT-yez~9_ThA+zgcKg4J$NHP3%Igcb1-YUp8olm?oqyKc{6 zwNeHvV5E~RDFmKrDDZ9w5NloP88CaZ(4Hr7gzyZz;~Qs?(5xdug&-JG)}Y=3E_91E z8om!-Xq#KOYGg;kgWP%bl_UQWPO;V6c*8qoPxQJl94bm@Z?c5p0Xd<%{df}Jj)~hkXBz` zQI%IPw`=L+)BEqs7Cp^U1-Y+}%31sZoaw2k4wY{?6gbq5CM^q=2KMgmZThq8FNM}e z3_c`spLm37PMhm4Z-2oGJ?qpC6Axm`zl)9@O%ijl)`I|^rk_U=O1C{YtFo*g!zYn+6@eha+qP2w;{yCg8uoHRDj9ew(93|TM_OtBED6W}0mM># zm~bQft_e||)%_)H1Y^HCyP^x=2r?nL*UhsIi=Yvp)8ZV8gEV-m-PX)3Z^Kq4Smowa zKewC6W3nZNtXqRE0{1-B2}o}|QZvIJ)u30-I2^nma!TpT0y`z-`29p^Xr*r~7)K)a z&g`CnPxfV=>E=-LEUe{kGqd!{?b2-z|FGIE-LU07UOMZ+HPkh z2R&@px!>vA>hoiy)NgG7qN9B*_>IT?9PV8s9szKXs6fh6nv@HfrC{DvsaAGz28@>f z)7~N0?WxUX-I;i?PtxA}1TpkNs^n>d|K$j8waeul|UhG1NA|9qY0@Ur|9~ zU{=mCs9DSrsC&+q*gqOmdDW8i4rwYyGc%%xXi!sNfBWTPuJQBYSAWK9o=QQCPY_Hz6~HdrRknsGj|P?C>h(`Y+8H06 zULFY82&UGWw7sXrQB|3&>U#EZv-j)Q$CO~lxC$Z(hV$oLEZMFLKQiPa zW50~Ov>X@T`#C3EdBlqv8_xvElBi1JUV}fgsGEn!3*){#=R@T-+5G#QP%a%Fd^(M;tq!>K>v@jIakg0cJqC+Ld z=6X?Pzxv9S<|gZ;qSrMSZyJKA&W|3Ro>`IiuARVU)?UK<$nCzdaazN_P0*e`=WXfz zuo2kRrH~#Ir~v3@37GxMgJ0{fZea21`BfLAh?Y;))ux|qZ9$KZ-3)6KFevh>$;`ZK zAW<@ZtWtV1-Ji#8E*&RDuU%2~^HE(>$MtLb(WJN&?-u!Exbu%et{06DVL}ld7Ezt1&mbbQ8c?*^2SvXj=)eFUVt5Dvpbn8^>mLP3QbLE5BX(_2<)qY35O_jlH&4f$z_HeuuLClM)7E zyEgF4n^=Vd=C;@qx7Xi>ViP2o2~%L~(+9nm0{tueV;j<(I;B?iV{^U$cMOJKHp4XP2*y&D5{m)e>;SL_KQOX#=Gt8xaeA!ugjq=bA|b z$R_g)D@FWae`Hpdm#}SzNZnkxhy{Ifd|o=lnfIwQHGQ#;!M{2W)s!G7EN8Gxdqx?m z=>k5d&tN#ZZe-JY&3n5VjO|4r0V5OwkJto-gq}IQdD8(&h{NQ;FLT$PET{+mTrz%0 zM85m@sGVAASRi)TONZpQb+-ej-UdXU|)dikL88t_|!n;VOqlGHQT_o%rUnf1HB zO*t<#W8O@;2XOHXD}3R2v8OOgd=%O$*&=D@`--;PnDYnVgePD$J&o9X1Mkn`L#0$` zbm*-9BG;rs*|LcBOd58`s_+JuvQ|wqXP;(GRj-HYoHM195iNr^4d^H)y)kfzgmoYg zq*ro-%te?AD#W*JFdT4Yt%crTp4w**)vnh1Ze{&r5w12E(~|ev zah%CZ?L(3tfQErtXCvX`IZM2{ounvLqo}$t;fM3x@?KhRGbG^vOzXyv3 z!Gf{g{;{7m&?yGFZhdS{%nq(QGmnefa`tMemryDWhEx%UaQLGNMxQ3gIR#G2p3+Gk zf8i`W1Duh(Wi2*!nj6f&BYZ3IZe5+<0Jdr|OoDF*OMip8Rpx9JII;s!hGapHgzQs% zza!Oh=*LIap+~B5R^j@3^6>*YZiW5LH|OztHje-LBMx#3-J%GtAc^c78ykBt)bn}f zCv78HV3)g@_fg?ndtEK7hlsVvJu~ht59?K{J_#k5$VM0AzGVE>Wbtjox?y1_3X{xF6$War{?(af=Kd7$rsNm$g_OWS?L zFuLuMr$kGiJ6Kk4rCsxo5z$!{iRQi4^sx_<=nV`9H7Fjra%%|FH^UEsBrFPmNA@AM z01w&DyW86ADBEqcZqL9ruvJoAOok=T+XkFk?H%IzA8;MldCdxfO6 zR6NZMN(;I2Ehf3=-2U=qMWP}8@oOKtV^tqILwd5E5A-@Obk3hj$yHWlYp?n2*NEU~22?@C=f)PPKq;W+y7$zko)R=ZgKllP(zyI9C#ouo@C*cJ)ry@U{ z{{H=wZ!cO+i`s#8B4=Ksps5*^QZCK_pI%;B8HT~NMM&lrQDpN&5x4ydhU3FkzEzx) zSY!F@e|Pw&@6KY7cm0giwboOxUD4%C%F-%q9cHzsBf^b*zzqYbWAHzCo>$eEye<&^ z&pzkE!J5HmSMIo)dJWx+$w=>C{FsSQmekPK=8PW+ul~6yu1!qb@9{QMq_9-BO^W|c z?RWQgin5{auYQ&29Ihc%=2TL3P6px;0ST7LK%76_d55 zAZ>_jiBa>A5W~2}p!laVD_s9%$+GZW*7(pn24=3;l!s^dRrPr;>|{@YwHOM(nqEeFx(nzO zWx!JTAr}mCZ;9WUF{;vl0{-Vsa^5pWxz6SAfxZc;nx`rR+k>{DHR&f(5|7>|YsOAX zHIDMW5+jIXwQ=%M4_k|lOF#Q*>&xtym(!9<{F^%l(qGqpnk60TSWs~3kbly?NfaoR z(r^9`DC2E>QlJ*wmEXE*I@J~;RPoS6O{AAkX!CHRC$Le5+>|nEO!(CXuTW$Yzh_yc zi^WRDd2CpvZ=R5naE3eK<9n$AB_v|&F zv+h4-6stdvlbH$iM^rY?#f}2oqACV=9V@_VydB)WIwc2UsVO;COg4 z{dku3W{$y7swjE9Wn!UXOfY3j32Vq}HwI+`ExoL4Wg2>b9Q>Ux)=B^P;h1IjKe)k- zP28jPEmV7>OIFf+*@2vyK0xFe8_09OEaN9~K=cABHTIJRz;~lTi1|_C!p+{->R)MiCPJ&WUp#$w5jk@2FsYdBhlv9TU-k0H+WbBf zV@LP||Fk!PqHbR&C?!sBb^^n4HMpUGjW zAO9sKk${<>GLrs6+V${V>jJHOG%FM6(gFycb!TK|RslqRzxCIz4?(TDfc(-Tk339M zZfdHjt&(JolgD-6Cnq7%_=cIVY-5)Ee95?fd{D-dvdR*-dJ$j+J|tbpaKMouk6fnq zNqVn@r{Qky@y7M@_|L1og*uN;f}d~corE5c-1hG-8nG$zF$?g@avk7~OijPR#gxSn z&D&3>D#~4249z3X`w(A={nvFSYtrOg(vs$|n9{)!K%9WbA`qts=7%D}AexkclQIJi z3IRsB5u8;bVGdeU!e^jFQ9jFTEGqEr(_tlQ2E9pPZmyD3e!C#-B(((vs;~0d;($#6 zmaNAoNEex$lG`7T2iMaDvXlX~i*7}_(inwRyOu=sn}SHSIstuFZI>L`%YPCGwS^Gw z%0-5uUx3U_M=XLu&-3=BqhBUO1jDfubt3Nq@$}}6zjK{TWPeFCTFv8mO%*=cF~L6- z6KzXcPr+=W0GvL!_(r}N*g9W(?qU0B8n7xYQ=fWC&3M6t3cjpkY- zL=;zp-B3oM)p+3~gcj9uMPzyH5DlP*k^>7I+&mU{?kiKF;|L&?3An`k#*7;Rq0i`z zFN;ByxDadu41lyNilJ2Pj~Ah5SO<8|5dw-6%v-g=h^`vhhWJ01^h2g6(u`2&{^f2N z`v3(}#NVfN~-Zc=D90ZY*aU)vx6J&ylt1qDJu%YuUWHW*7EZccNNlW+z?F#)>R z5{j;76wmVGxMd(-{Q*Y8LO5?%lXFEMxKT%k44rN|{PhXueQ$9Xtr+COBA#-0pN|k{ z_U3*3mx)_`w_^|M+OjJ;jY!!!lW+jQC2%b;Hjy*3;=r+3S$BIiIluNdhw`a^ZM-?H z88kpiX(QZsYYHIA&T9(BZBM~x1(NpC;0Yr2{L>ch0~v~>VA5D8c)LMdU-n!3)!&hy z3kn`CfO%@%jB$ZDjsT4C&VhF}h}FpqBI7C&7`!rAJW>QAv;-)LUjDcGO|#+LTZN?3 zxUjHW$V2h)a}xO7Rli;8IYu;2|BUEv!*Ke2rR&{mjrv8ViU%M`cQR7{JvI-=C+yV@ zP`(H3uzvC!*0mBomBGN+Ejnx6Yds%3R=nUV)*$?fBlEoaSgip6+G+SB23t}0nJ!hX zXZ&tmv_;|TZ*DC1{KC1OCUB}mctPKe3)bgv7Xk=bJQ4JYE0HO5S@Jpf$yx%W%f`;m z2U8BbJ4x=)=;MO6$`q-{Lmt~fymKUIxpjblcBFE)git7XH3m!g9((}I9kPj*zHo0l ze2?3cm1Ka)lA)fSWxEkaP_qG6Ujc%gK)%L<{~F5g-#J`jny*8N?B?QH7{c3kA>W}%MH1{vy#$f{6nQFwX3E*o~TYgXE`BssFph+-eu!Pu#5-9DE zU_~cJ^;W?CB!Sbsf>cL$@#DYAg!)QLNm1U$i$fkfAQ+J14!f@@UQbD}nMRMihm_^W zDwbYbgu6M8Z1l=dhaz7M^CMgUpG-|n9oeBp`T@APJw2%+f+m8Y@;|JDVzi@YK1z^D zoeJCuT}hsyrm%Il6_76hiC*9DdiNZw0sm4&Y@zFZnMr5M##bghpTObIe-eY!NmKbh zz`)e&;`RBV*`+{|LBY%S1#C2*hCV;#27ApXPj=IoO(QG0@n;2+vRuvbN5M+-1MoAy zc|O^9>ZBT%1XLL(#%p+JV9%WR0_&0Y8i;0y*pH$W=B z$2$JvyMO#U;;#Pwvq!sb;k~X4qT7Y^7hgk0>&w_K|4lTyFgT9eXoTGhmd8F9)hA|q zCL3~mad=m7!jgvFPhsU_)52@9;kXnMuk`gj-x0P|6jetP^m*x4H9dC$%?eChalW&MiF2E5Yk`7^Cdue{d%p%%KPh??|9eXse#hxbqj_65-%$KDn-@fh? zKRl7K`prr0-noS1KXGT+;`Kc2eqssf4Wh2|N{uSC#o&2hFf$Zti2d3-KO5fvUy;zG~-$^AaY z>^x!n-=x!=n&DlPxwn)v(b&pO>o~UC`%OMr%45$qOAYQO-UMf7lHL~ujIfG*B}ut) zrJ(3_fj&%CF6%<3N<9EdPT;bXd5z=xLn(#SFR=gOn@=1V^J8Cg#gNrJ9{k&Nq7@mv zsFXnUUIkZ~{$G1XSmliyB?0?O<`*d$KUDZgpWzmV5ul8q8~}s8SNp%59>K{R1gbWk zm#;sfHF3J^_YCjJpdn)fXfT_Qius3~k#p1! z=OLrAt)bpgygVf-qBUXCez&b+xAW@D-7B5HJXkhb^|ZOZkGZ?fhfaAHo(yN~F%%A) zi~i$GrQJWew|q5B7DBQNtBs1H(&rt!Y^uCU`-@DwvtU&138RowTl#&;7cd_{t`W?n zv|uQCVNO@mh5)rdRA&Gy!O1$-Uouet0>O|N_;EK?)MbDcJF!zb{e$uQRG7p0#b#Bn zfL5@YT0r@QXW0CKw?-1`ngqU>m*}T$=Q(+K@le()S1GPJ{7FyBEgg`btnUAP^jJaT zx1l*O%5;$ewwZbWiLTUxUO8x_Ar5dr$YsBsmY(i5I5;@%N8km>z!}1{Hv+yNKiTMe zs4(6E^U8Uq(&1Ib(Tgh%eqvR@ynKA2@_AKP-4J&I$)o`3LXiO;LY*?gi#i- zgD)TDq}nVr{PC7&+KQ)B(~Q06s!Zs#R`O_NF&Z|Y${?Anz2p}Q&NSIhzR}DUbZZtT zZcpY%L(h{_DbM|lU);UDFWG$Fy57~qrh%c$8VvIMEfu#ibXhWQp2^ zUB8tt&K$01_DT%l@dp#}elIQ*FNhRc{~7?dyF*qoTaZ+bTl3^2HfIe>S234&^_R_9 zC80#XX^z0imt4WTrKg*7G5YS0y7YAePu&9(>ub3qTgUfKn41^F@Vd=Xr>3vJdgNr= zUa7et^5qE{1uox1F9%Vq#0(5EPZPejLC1H3**64F|Gs~*gy#BZ-r%|&KgBI^9X~(M zP4}$T1O%h$=~0q`C0Y!&`RANNr{F@%0~}*Vs7Et@xrP%fkBE^27T*)cx0O;=b=nM> zVRyi(LYRED1ZmC2YdzeuAgVgR+gsvGY3acpgzDZug5?l0GP*Vq;|(*s<4$Eha{dD{ z2gS5wFA=ny4BoJ<-`@0(tUq0$ZF71pUQ|I@ycW0pfL+jGV2jU&oh|#BS%KYR-a>ea zw`ocO%mN>QC8-3rem2O6rht6M1n$DG({x|JfcOeykO)r!18`)k*Z|NM)WKzJwr)W{5%WLYV`nT9SFvo%H7{LVAM3|PW6``6!fZF-aT z>f3cFZu~m6p86Gl!urARrB~AdnuD*ZjfMz3&^NDEGi1qsVJT z3x}0Gj7MC5$p7#V{t^sYNIN1-*t+AcwQVT>yd-Pn`K$?5X~u-8($Mi->fokf#d&hW zu-#^tS6Dn!r+h|HWTN(&xW0HV7D6BOJp?{0y$4Ycw<%qbhpPgA%b>Ia3myYduW zmEn#*FGn*S<_nperE6T6k156E*f05b(X;ohv?^96nCAmPRi?1PCX1TzpDcfm&gOpvX2xROe4V3 zI0sZF@`wY`m5o0qEjQ2mV`@4i5B2ldyqtKzfZkQXB_%<*Ppm3rJ>8m}@q)W*3PD=H zHs{{gz94pUH67j zf#{SOOH*~&4ZERB0#05}W~lLdi}G}+KV3r;_7iZG`Ar7xkrY;k^t_W4DU zf>?PC!S11bXM6qKB7tgM%+GENk_qcFgR!WaDNM}F;O-GZH=#v?U=2X#cA+Pg{e2w; z3eEn7;ZMj$reA7R@uBo6`kyD+(NF(&V>KXiby~vVj#R}+|4zTZ6DZ`&wBV-IJ6W_! zCSK%(>OfkzPp8rli-!k{cTir-1e)J2Xm>E*FOZ;AKk}H#6kPif>qzYIV*vF zPO0Lx^0K@aGtA(GX^f4&$^44LRXXt}^tbT`YU`>?piW}g+zbi`0%yc~0Kt0d4qs~I zn|8;p!|croOpd4C-;Za6DIbV4n);pVz-Yk`;wN&`RtmLyXGKUN^mc20 zJQkL0#FcIf3G@Zqg6*AkXQ0q+fo?_rzxENjC8Se|sAh*-DFn|lHKhyg9i z5F03M;(=Fb>oxxHYR#2D-`?#x4UkvZ#(f@D_zo@s<~+5i!Bi~SiIRZxBo7ZE6V8wK zVd8cI>8d?n`xJz+3)QS7DgSat3am;+p8v$1?eC5Xk0NckRItR=Il@H8z^i9DzGK71 z--k!HTze|;M|2q@tkjoq%lc}%7gO*xXS@ZP%w$5b*XaC}gJPyPLO*=|gMSbuPiz3v zab^`30u5uN;Q8;i@`aNs*nuG@8}2=R*bOY;(t<2{j|wVW=nMGe)^)MjznoBad)e!R zwtH9bC^Iaz;QJdXX+9D!@cywvp72&*ZBK*ygO;{-<|(A*3V^^C19%9{P_u$f0TbAz zZ@cS~U?`%J`_1jaSM@V;QT@rBu8KP&R0m{sIggCjp%DM#iwmMi7MS^s0rve7o?;XV zB!n^utpFKC0&y+gP^+Ar$~`4NU%%YEKfy2g*JkYJhf?J=M!G6&Qgxa&e>;nXUK5%yRWez}D&T8+j)+y6Vs#^QJ*Sk+o1*VCvJAhP%;43vCizO&QgnH4akFx% z9F2yg*Hw}1ii)E3x@kVY$klO{ms!jky_P8Pci+k=kY4_l*bgtUC5_p3)j)s$aedeX zYZ1G|P^w1*m(Z2EBVxEm16b|q@4<^X7|d4?V5tMp`%55`1J3jaavZ^$Typ~m%5?zL zFA=foY%f7&S_`H{ytJ2Q{UYL9P4o#{6alfLSvT_&V}NFhS`p9G*%kDr8`C$OnvdM@J{8 zPFO7aND$hE(bl8${=f3N>$RGuCOkB=%?U$0zxEG(PKtm~&7i0p%Y_$S$_pi#Bq-07**Z4#R&?+ZQdjFE?<4AaD0<~_Faal}q?B=b9-?9JLF z!XDwYL*U$pMpMO#ZAHY`ptCo%+Tj)6x^O)efh++<_;j?}$$kicjZUx=&T{+2yfe^s zZj?5`WWpYyEFnjqEPy-+8G)ih^c`$L17z+=-l)&=+Q1es8R zs*%AyUe!kl?10l!BrFK|SoqwVe%tU?Y2U<7m{#Xv`t+EiZu$NCUvfO3O;L;V;KLo{ zvVuR|JxJ*q+*I6BI+ganiR*{{UhE1Br8h6D`<5t2B#LHv4qbM#vBR209QSf-oAglp z#-AxywB+S0N;dK>cSIz^RW%4dJ*KOMVU;y9iO{hm%pgOs*7>VT2vY~-AJrhMEPGPod_RTzMX(6Ski)+dN)L$0 zbAbWX6i9x4AmoYnDcn?k@)z_!I6M~NnZ`-DNvsybs(TS(D5nDq@Yj zs8{9C5`QB2St2DLITyfHH4EA4ynTFL0_DC6(uYavEWIOvgn&RjLiuZVKT14Dys{_+ z?HM~JCMrKQ%`@UE+;KK+=d5a6)w4Zgc-!|vM0`t;zd6X~W!Ki6(|hyAOR7{cl;9y1 z6V+wzpP^1i9?+jSpB zin{B@d)m;jbDIn{&^rp|GYWPgr<{WJ`-}nXZ(TQ+{)itm;H6~g6PdEcVkp*Wd#iZf zET_1*v&i`S5;cl@l_1pzQT-yWMdrwR@PlxwTv`X2X-pLm zU?`RceMjQlA?nW!q_K)vg4kD&GlYNItnl(a$Rqt%?mZJ1Hl%`Ukc`Pw3^fuxFtnnfDTd%7&U~5c6V;nwSCs-0k`0l6PdLcvUsKKj zm<(n!Qb0e-hYBDOiiLD|%+&_6rJ4$li691LX|juauC4Y9 z_rb0LV|Iihg>>M6bK1BMOU#li6w?g%(}^zFxh~#h;Z4>0etiE*RwbIOX;OV99#Q?b zkTTl5`{Zxx-oD%LWVyJWxsTJIqs(VtdV~5cQGSvn7U~@xmal#`?PK%hMeI8&DdJ6} z$P)NH&kY1ySS^LMu(R&MSfvtVOmS?wg=u4eSpDkiT2BPT^1@y|j~j3^NcBnuA>X9b zyE<2=WUtv2NxxLcBRfY{h4EKviIK{Q9n-Oqo3J-BxU8Que(R^@JqKg zxf5QwMqc-EOH>ezHC(vsX4|{{=+%gOXm(yc)Y3XbKYZsrO++Avex}Uwu~OUbswAY$ z2Q_vn;^hIecqHYfI6uE!$iT)EUfVj%%1S^zwny#pumAbKC4~K%t({k^3gA7&ynL;K zQ==&Sl>J=7YpSsE=TBF2+N9Jd4wTrit6o#>;TNxb`}v?tmr_KVC~4WXPI zA0Ho>^M8a95HeUfYY8ULZf!y{%4)IWAAOH7F<{gHu_y-#Jg>CQA-6FAS^G{K6Am2~ zYuYujC5rUe*$xxI4BB$FTPX>oOG{t7Ex!@Qf~n@*j-yVib4h*OORgmyH<_g{avarD zhoiq!igW1KO|8#Kf;2Cfztfu6pl9=uqKgp=3!ai{l;SX~(+lw%JdIOzK1i-gLRt&3 zOD9GK9*m6n)A`SvLcz35w;H(^p!lyriZLYm2Uf_upk{C7)x7T^(7?H008Q6|u4$q1Grl+8H5=Nk9hHUG56z92eN%ESNPS{+dio2vrVU40lS8 zT?=~aH!sJIYOtgJsrLlm4O8;Ua8$UFq8C;`B4J~%nb(yKBVrYrpsnl7ZL&Q5)ZBWQ zm9dy1tveu-+}k>6noVI@g@Qy6u>LXN#$%)`et{X!COjhYP}09#xhgpD7BQwl3V96F z(ek%YTv(4^;P*|3P{9fqlb(wFjbM*^9AVhzO1rX1xZQ_Wb0(USzk%o6W)03EC?z6p z<3%*ZX6zg3pEweUem=ol)6D+IxQH%kj_jhyi2N|V+N)z z)CE$JB}Bclp$aGXyMD@u>Q$CPT7QjovWy?v{=1w(_eQzR>XxthJ~%I z7l=?9OyE>y7|qc?gab-dwSNFcRPL9MVr0*c42=p3!UsS)3E>_#N#B1mg4pc^@O>XmvySy^u;Fh~FsI)kNBG76KKijVYM+Ulp{+Zge5&mWxik6(3HK`{@Y@sWA? zFkC?iFiC|q?7V0AE$Q?OT?44*S>7Vw7g}vy8xOBv>&BdTWZNAVo|!>Ecn!6UM3y@5 zEh<)(mhuh*WiIC&tb9umy6)MW4_>VZ%wgQ_V5jdm9IN5`KSm3dW&uJ{&VQsk2p}Fp z&S5kZJOGIx{djd|K;jA^_l%kk$-DoF1*VOSb3K4l8P!UBi#nVzig&VRDj%OP7JmOT zQKX#Dw;)2F+w$ZE=11Y1tti}=O}g|hF2)n_@i+YYRMia*-IGpJAF2P94$SRXu@}q% zi9!Ty%)r%s2vZgznAYjSY*^{nrXM^V$V)-U7T;hngHS*LAa*(1k*SF4-sOm}*x!D; zciDR6pZu=X$2*n4ibh@hIu?q?o6UySeN+e63ZF(yE8`N1S zV1`1Oc75%yhy6Us<6yiM0yf_J`ub{uZTv-EmE!gPhx;yKnfbxd96+`;r~m$^L|ByD zXgzV}z>I)(tNMr5rj4k;Q=3G0nb-UansD+UH&zr(gdQS`*9;#UB?i3AQo)`;mytf^ zSS5x`S{zZPcKI@;^gh&?=HYMPkH1FBOfU(UM5c)#Oq{8#taJeDRbB*8fg`UQd5?g_ z$d>cc;{eR3Vvs#~*6Z?I2yE#BK>j0uFwF`JjpsKS*=aKOwXcWLqo@k6%xFn7WYqZ@ z7`EJwKCj6&Un)iOiq9j}rJpNx+T})}5G({yNC2gk1#aCo^e9t81B=Fa%sv&ku+k-y zx^;m@xBq6+?hAFix1WLLprh~(9e&#rf-2v=ddZLXp>)F~APe%^IO@OLdCeWtR;jsX z_=;RcQBmr3hq6Lo&{=r6c7cdU+l))40XOu!2xay6{uKbp0yaI7LKVoLU%tk~Oc%k7 zjspS&z}l*pdgV_5b_i4jB0%p;6n5bR2BtnHYED3I;9HncNGjD?6;}2gM!9y>c6lHB zs;>wtjI{OD<2jric79<+)H@_hbq#W6<0~s&J0U1S2~K%XWNWGvgqSOS+i zNS~q9D>Rmad*<{6nbU#yr?Ui*1`zANVH9ek(Q)~P0CvHXlYV7CxTM1B*7dy;haL&Jsqxuqv^_umdZ^S{Eu zy@T=~VNP^Jly~J2T+QA*mRaFE{f=P8N^OC6Yyw0ib1O4qAJL}fk#R0wnY z4l!9fJgX9{zwmF|O?rklN`~`Zat7Bhz^lRq#}#LeWkmAU?{pLvO6}7FtKmMZisvDN zGQSquIK4_jAez9cqpNF*f9MrJ-vgizL@G7dIy&I$>LKp|_yb)di2EelqvYS0g{tB+UEdS9s^bvhFWW{y0qo;3eAjdPD* zlF<}3iQ}|P6eJ<WsI_{jO>K#?&U~n6M786dsySjy8QL(?BTgfX+R?`%jb~K>o z?B9YnogSod71{^y%?Oo;5*XQ#^t=J>svrctL<>{liDLvoMThM6gm>-;Han>;UxH+$ z6c!AQtJ3$dV6(Pf$f?r>HfSbdAFzdABhcIW1+AFAgf5EB!F(4D*|cCQmPpisd>~; z&ch2&Nu{sVx3sm@UbN#600WO<{8^I}p}dTY`-(shkokD+oSj(*$WN3=alJD=Y(zH< z@>C=x=fg!VSP~b=*83gQ?$rIokBQo5qr3Tr|D7itsc{O?iL^k*dZLJ?rc@d};eS;d zLhgA#<46M3yPM4Xmp9+&kh)c@KQ`3OS&AR$YQH8Cn;)qXac^XgpR!JS$dTldZ>-_9 zZ|oWQcnMz4af;fHE<~Y)anvb}+Sr5Wkt1h{YbJ!;;_1YEv}Hcro3l!{Qawff+1W}? z6xl3e-us$aNEY4M(O~Dx#9FJ;B&A+pS!wtR*wGNvP2ZdbI+d=!Z%mL~zV8y3a97gH zPvnb4(j##HSV?V#bzm%uL2qM?}o+R;7y z;`cU9;0Zw`NV--4J~NW=d3e7-9K)=xHPygZu)|4I`SQH(MB_njKvu-A^OmV8GyYr8 z2e}XRIYW2pEM~I~XFgDN%i6)D6ERSd&~9+*Ov;%cI9&_=ixH`9r5vVrt@CqKgN_pA z_Sx+G3w1P(a2^gbk`kAx0qe)*6)qf73may9%MquGBs6Y4X($>t;Y=Fhkt5V73dVER!<4o3*~xgAuVddJ z+>zr+CqDLrQATA>L5V@;b~~!8KR9QIqK{!nEKud@+ijm-g@Mw&tt>{`Rohiuhwla~ zCr`70s`SL2FDg=d_$}STOIG2SPs&6RGG@5O2VEcb$%vBQlTUvwH2eN<;;ZE|nakko zH(O-G+XSOQspMWU>F9CfIH47DQA^h%Y5G?G)nrj(Vt7=@k2d*7YSFhBw}mFW+H1Nw zGP_4Bqp4~~t`fxJE1aoaoq3^nJaAtzO77$_e(~GFIl3y06O@NXMt+m>IJh@ZXJV|4ov)c;PZvcB(u&w~gW&`mFbv(8QVEg6S3-F0I@= z1`2g9K}(F)H$B~bm%Gd<`)p?FhgpqL%QTPtKTrG8R8K#Hi~jkdU?L)o*=WA9#g+1t zE}SF#^08m<9G^4TFR)3_nPHBp#y80JP!z}JHH;>pP@EhxCO4OF?zHb(Vk?$SxG8#D z|MMsak8nqO&ZEy$7yYBwwWs`m@wSB9sE6}Jt6z5SrA?vVw|D5x*KbI3WE8R}$S+wk zb{Yw31{J&mbsP15+_8tmAD35rC50Cso0}#|n}J-24WiNQ(+{$FVxPV-tQSx-5&_f0)=76bg{?~n+r>RT5k z{yQXp4r^0ud+eSwk8YuzX^*f$l<4X|#=*t-6JGXo_88@Mp7`{qzQKKM%NmFtFWwD(l3>^V3JTzrnLwz7h6o=nsv1iVrAWGfPmHaN+towi9j zeepBG{d1Q}q}hoL$Mrl;t~q7DJ$$PS!Z>pCJ4V-qNxqI%&wq!Db>OOpDdOCzLPMjJ z@H#6?poCTCaj|f5LMeK7BpLY@e@`e6N5&md-uQd*9eWVov$%eq!Nz5^^X|R>O-vN3 zd~B7$R`0p}uCn}sA-?CXpSnZhfln3AqHIMTM?jNS{YyWPm`P#f+0S>!4Y#HBe2sYf zwPyODs5C8wyq&7Y$yW`dErNrq8^L9wiCkmyk)OJkN4E+|-xOx62ir zew&VO`ao2PvF%AgWMh2vTZQl43LYDB)AK6s6t#5&OnRi$##=#;+?q&~k&bdq(|H&x z{2lR_PFKfj<3hUa_3Ne`H`&wtz5Nru74x$8w2uz-8Ca)(A=>THr+)E`W5!$Zg%ImN zO@u+G;VfHVSX;0){S-;Z9Ya?~0RI4r@yHtnfrRC_cPgtE#@<{Jxe4-qgEciU;|8bV#v4}jAi29k# zhHH&b6gKll!omfqrTjm6>X@#LzxzJ_!ttU(@*D>@$4K@nqCE+BWT;(K9Zc0F4E?J~ zZWtus_Krqsj>SsXb)ifo{s)zmKmI{(x+pJ`B9%aTNn$4ZY_OJd18o*DmnqHE%tLDR zk6Ami`KJBTv$2(5Yz(a3ui$)qVf4c1VHV*|p3@p%LY3=fWKRfH$+&XdRZhGl%aCG0 z1LSLpyS~1To3zcPVVK^mYM^hV+G~p~d|0zBZ_-vKfOq+=X{n@hCU02H-1#A$`cv;a z@AFDJ=(mm)94HcoO*YDP_@*aaJh@Bl>g#`DzU(3xO6a)nm$a;0I_uJ=_rrT@s}KL$ z1V*p^aC)o&cKv2V)7xV;ng`8a)Wj;&9ENm~Q-&Ib)i!PJ%C^sVP1KH!JY{&h|6)4u zYnwvSLqs4Frmwd)KGfK+>34zKZNo-Btqr&E~d;vygM%!k> zj~0C9&O;4{-Tl|j@ohcut%`CyuiWQEuChWK{r;qnFOL@|6&n^83WdfY_gGv*{{c&4 z@qpg=(x%EZXJEhEoT2ua1oRk$YnDt}4-7c&lJ6D8)p8d*ss>(I z*=4zO@S5;ClJFJVs~%s$A87u|B8Pvy2D^~{;0BLChW8Es8oN@xf}9wo#kDcBYv=^z zpr3=cpFNlm@qzxoAH#b1pSOYr!0(Vt`2YSavRwXq`@f(4|9tv?Kg;p|{}&&{U7}Nr Xa9FZ6C4kR43MD6{^f*_-$ou~Qzt38U literal 0 HcmV?d00001 diff --git a/docs/apis/batch/libs/ml/index.md b/docs/apis/batch/libs/ml/index.md index c3b6316af87c4..17773d2dfaef9 100644 --- a/docs/apis/batch/libs/ml/index.md +++ b/docs/apis/batch/libs/ml/index.md @@ -48,6 +48,7 @@ FlinkML currently supports the following algorithms: * [SVM using Communication efficient distributed dual coordinate ascent (CoCoA)](svm.html) * [Multiple linear regression](multiple_linear_regression.html) * [Optimization Framework](optimization.html) +* [Neural Networks](neural_networks.html) ### Data Preprocessing diff --git a/docs/apis/batch/libs/ml/neural_networks.md b/docs/apis/batch/libs/ml/neural_networks.md index 72d49b4216fa9..57c013b59a9a2 100644 --- a/docs/apis/batch/libs/ml/neural_networks.md +++ b/docs/apis/batch/libs/ml/neural_networks.md @@ -31,7 +31,175 @@ under the License. ## Description - Neural networks use a directed graph consisting of activation functions (nodes) and parameter + Neural networks use a directed graph consisting of activation functions (nodes) and parameter weights (edges) to solve complex problems with interdependent variables. - - Placeholder. \ No newline at end of file + +One of the simplest types of neural networks (sometimes referred to as Artificial Neural +Networks or *ANN*s) is the multiple-layer perceptron (or *MLP*). The Multi-Layer Perceptron is a feed forward neural network that has multiple layers which are fully connected with +*non-linear* activation function at each node and weights at each layer. + +### Network Architecture + +A neural network consists +- *N* input features + - *H* hidden layers, where each layer has + - *Lk* nodes in the *lth* layer +- *M* output targets +- *L* total layers. If we consider the input features and output targets to be layers as well then the +network has *H*+2 layers in total. + +Consider a simple multiple linear regression (with no intercept): + +$$y = \beta_0 \cdot x_0 + \beta_1 \cdot x_1 + \beta_2 \cdot x_2 + \beta_3 +\cdot x_3 + \beta_4 \cdot x_4$$ + +As a network graph, this linear regression would look as follows: + +
Multiple linear regression as a graph.
+ +A multiple linear regression maps inputs to some output estimate at $$y$$ via a linear function. + +A neural network uses hidden layer(s) of activation functions to capture interactions +between variables and mimics the biological way in which inputs are processed. + +Consider a neural networks based on the same number of inputs but with two hidden layers +of five neurons each. Graphically, this network would appear as such: + +
Multiple linear regression as a graph.
+ +Where *hl,n* is the activation function at the *nth* node of the *lth* layer. (More on activation functions shortly.) + +Flink infers the number of inputs and outputs when fitting, however the user must specify +the architecture of the hidden layers. + +{% highlight scala %} +val mlp = MultiLayerPerceptron() + +val hiddenArch = List( 5, 5) + +mlp.setHiddenLayerArchitecture(arch) +{% endhighlight %} + +### Activation functions + +Activations functions are non linear functions that (traditionally) "squash" the output to some number between zero and one. This is akin to the biological concept of a neuron being 'active' or not. There are three popular options available for activation functions included in Flink. + +Each node/activatoin function takes as input the sum of the output of each activation in the layer below scaled by some weight. Consider the activations $$a$$ of the 0th layer are the observed features. Each edge of the graph represents some weight $$w$$ (just as the + $$\beta$$s represents a weight in the linear regression graph). The input of the *n*th + node in the *l*th hidden layer is given as follows: +
$$\textbf{z}_{n,l} = \sum_{i}^{N_{l-1}} w_i \cdot a_i$$
+ +Where $$N_{l-1}$$ is the number of activations in the layer below. + + + + + + + + + + + + + + + + + + + + + + + + + + +
Activation FunctionDescriptionFormulation
sigmoidActivationFn +

+ The Sigmoidal Function. The most commonly used activation function in academic literature. see wikipedia +

+ 		$$\frac{1}{1 + e^{-\textbf{z}}}$$
tanhActivationFn +

+ The Hyperbolic Tangent Function. Another commonly used activation function. + see Wolfram +

+ 		$$\frac{e^{2\textbf{z}}-1}{e^{2\textbf{z}}+1}$$
elliotsSquashActivationFn +

A squash function proposed by David L. Elliot. This function has the desired + properties of 'squashing' an input vector to (-1,1) and being differentiable, + however it is simpler to compute than the Sigmoid or Hyperbolic Tangent functions. + original paper +

+ 		$$\frac{\textbf{z}}{1+|\textbf{z}|}$$
+ +By default `elliotsSquashActivationFn` is used. To change this use the +`setActivatoinFunction(...)` method. + +{% highlight scala %} +val mlp = MultiLayerPerceptron() + +mlp.setActivatoinFunction(tanhActivationFn) +{% endhighlight %} + +### Explicitly setting the optimizer + +The Flink MultiLayer Perceptron expects the user to build an optimizer externally +and set the optimizer as an argument. This differs because MultiLayer perceptrons can be +very complex and the user may wish to specify some other optimization strategy. + +For more information on Flink-ML optimizers see [Optimization](optimization.html) + +The optimizer is set with the `setOptimizer(...)` method. + +{% highlight scala %} +val mlp = MultiLayerPerceptron() + +val sgd = SimpleGradientDescent() + .setIterations(10) + .setStepsize(0.5) + .setLearningRateMethod(LearningRateMethod.Xu(-0.75)) + .setWarmStart(true) + + +mlp.setOptimizer(sgd) + +{% endhighlight %} + +### Full Example + +This example creates two `MultiLayerPerceptron`s and optimizes them in 10 iteration +bursts to show differences in convergence with different step size strategies +(`default` vs. `Xu`). + +{% highlight scala %} + +// LabeledVector is a feature vector with a label (class or real value) +val trainingData: DataSet[LabeledVector] = ... +val testingData: DataSet[Vector] = ... + +val mlp_default = MultiLayerPerceptron() + .setOptimizer( SimpleGradientDescent() + .setIterations(10) + .setStepsize(0.5) + .setWarmStart(true)) + .setHiddenLayerArchitecture(arch) + +val mlp_xu = MultiLayerPerceptron() + .setOptimizer( SimpleGradientDescent() + .setIterations(10) + .setStepsize(0.5) + .setLearningRateMethod(LearningRateMethod.Xu(-0.75)) + .setWarmStart(true)) + .setHiddenLayerArchitecture(arch) + +println("Iteration\tDefaultSSR\tXuSSR\n") +for (i <- 1 to 40){ + mlp_default.fit(trainingData) + mlp_xu.fit(trainingData) + val resid_default = mlp_default.squaredResidualSum(trainingData).collect()(0) + val resid_xu = mlp_xu.squaredResidualSum(trainingData).collect()(0) + println(s"${(10*i).toString}\t${resid_default.toString}\t${resid_xu.toString}") +} + +{% endhighlight %} diff --git a/flink-libraries/flink-ml/src/main/scala/org/apache/flink/ml/neuralnetwork/MultiLayerPerceptron.scala b/flink-libraries/flink-ml/src/main/scala/org/apache/flink/ml/neuralnetwork/MultiLayerPerceptron.scala index bee841bed4c30..6706c3b4c31a3 100644 --- a/flink-libraries/flink-ml/src/main/scala/org/apache/flink/ml/neuralnetwork/MultiLayerPerceptron.scala +++ b/flink-libraries/flink-ml/src/main/scala/org/apache/flink/ml/neuralnetwork/MultiLayerPerceptron.scala @@ -50,11 +50,6 @@ class MultiLayerPerceptron extends Predictor[MultiLayerPerceptron] { var networkArchitecture = List(0) - def setIterations(iterations: Int): MultiLayerPerceptron = { - parameters.add(Iterations, iterations) - this - } - def setHiddenLayerArchitecture(arch: List[Int]): MultiLayerPerceptron = { parameters.add(HiddenLayerNetworkArchitecture, arch) this @@ -96,11 +91,6 @@ object MultiLayerPerceptron { val WEIGHTVECTOR_BROADCAST = "weights_broadcast" - - case object Iterations extends Parameter[Int] { - val defaultValue = Some(10) - } - case object HiddenLayerNetworkArchitecture extends Parameter[List[Int]] { val defaultValue = Some(List(5,5)) } From 0d470e508089e5410cfee1a8f65c79da5b595d72 Mon Sep 17 00:00:00 2001 From: Trevor Grant Date: Wed, 20 Apr 2016 08:21:06 -0500 Subject: [PATCH 8/8] [FLINK-3742][ml] Added ScalaDocs --- .../ml/neuralnetwork/ActivationFunction.scala | 64 +++++++++++++++---- .../flink/ml/neuralnetwork/LossFunction.scala | 32 +++++++--- .../neuralnetwork/MultiLayerPerceptron.scala | 31 +++++++-- .../ml/optimization/PredictionFunction.scala | 9 +++ 4 files changed, 111 insertions(+), 25 deletions(-) diff --git a/flink-libraries/flink-ml/src/main/scala/org/apache/flink/ml/neuralnetwork/ActivationFunction.scala b/flink-libraries/flink-ml/src/main/scala/org/apache/flink/ml/neuralnetwork/ActivationFunction.scala index 366cc1014d693..2f724d1059783 100644 --- a/flink-libraries/flink-ml/src/main/scala/org/apache/flink/ml/neuralnetwork/ActivationFunction.scala +++ b/flink-libraries/flink-ml/src/main/scala/org/apache/flink/ml/neuralnetwork/ActivationFunction.scala @@ -19,57 +19,99 @@ package org.apache.flink.ml.neuralnetwork -/** - * Docs - */ - - - import breeze.linalg.{DenseVector => BreezeDenseVector} import breeze.numerics.{sigmoid, tanh} +/** + * Represents Activation Functions which can be used with [[MultiLayerPerceptron]] + */ abstract class ActivationFunction extends Serializable { + /** + * Applies a function to a [[BreezeDenseVector]] + * @param x + * @return + */ def func(x: BreezeDenseVector[Double]): BreezeDenseVector[Double] + /** + * Applied the derivative of the function to a [[BreezeDenseVector]] + * @param x + * @return + */ def derivative(x: BreezeDenseVector[Double]): BreezeDenseVector[Double] } - +/** + * A sigmoid function + * https://en.wikipedia.org/wiki/Sigmoid_function + */ object sigmoidActivationFn extends ActivationFunction { import breeze.numerics.sigmoid + + /** + * Applies sigmoid function to a [[BreezeDenseVector]] + * @param x input [[BreezeDenseVector]] + * @return function applied to input [[BreezeDenseVector]] + */ override def func(x: BreezeDenseVector[Double]): BreezeDenseVector[Double] ={ sigmoid(x) } + /** + * Applies a derivative of sigmoid function to a [[BreezeDenseVector]] + * @param x input [[BreezeDenseVector]] + * @return derivative of function applied to input [[BreezeDenseVector]] + */ override def derivative(x: BreezeDenseVector[Double]): BreezeDenseVector[Double] = { sigmoid(x) :* (1.0 - sigmoid(x)) } } +/** + * A hyperbolic tangent function + * http://mathworld.wolfram.com/HyperbolicTangent.html + */ object tanhActivationFn extends ActivationFunction { import breeze.numerics.tanh + /** + * Applies a hyperbolic tangent function to a [[BreezeDenseVector]] + * @param x input [[BreezeDenseVector]] + * @return function applied to input [[BreezeDenseVector]] + */ override def func(x: BreezeDenseVector[Double]): BreezeDenseVector[Double] = { tanh(x) } + /** + * Applies a derivative of hyperbolic tangent function to a [[BreezeDenseVector]] + * @param x input [[BreezeDenseVector]] + * @return derivative of function applied to input [[BreezeDenseVector]] + */ override def derivative(x: BreezeDenseVector[Double]): BreezeDenseVector[Double] ={ 1.0 - tanh(x) :* tanh(x) } } /** - * Elliots Fast Squash Function - * by David Elliot + * fast computing activation function by David Elliot * http://ufnalski.edu.pl/zne/ci_2014/papers/Elliott_TR_93-8.pdf - * */ - object elliotsSquashActivationFn extends ActivationFunction { import breeze.numerics.abs + /** + * Applies a Elliot's function to a [[BreezeDenseVector]] + * @param x input [[BreezeDenseVector]] + * @return function applied to input [[BreezeDenseVector]] + */ override def func(x: BreezeDenseVector[Double]): BreezeDenseVector[Double] ={ x / ( abs(x) + 1.0 ) } + /** + * Applies derivative of Elliot's function to a [[BreezeDenseVector]] + * @param x input [[BreezeDenseVector]] + * @return derivative of function applied to input [[BreezeDenseVector]] + */ override def derivative(x: BreezeDenseVector[Double]): BreezeDenseVector[Double] = { 1.0 / (( abs(x) + 1.0 ) :* ( abs(x) + 1.0)) } diff --git a/flink-libraries/flink-ml/src/main/scala/org/apache/flink/ml/neuralnetwork/LossFunction.scala b/flink-libraries/flink-ml/src/main/scala/org/apache/flink/ml/neuralnetwork/LossFunction.scala index ff2511778d2ad..fe51fc94b5f41 100644 --- a/flink-libraries/flink-ml/src/main/scala/org/apache/flink/ml/neuralnetwork/LossFunction.scala +++ b/flink-libraries/flink-ml/src/main/scala/org/apache/flink/ml/neuralnetwork/LossFunction.scala @@ -27,25 +27,34 @@ import org.apache.flink.ml.optimization.{ PartialLossFunction, MultiLayerPerceptronPrediction, LossFunction} +/** + * A special LossFunction for [[MultiLayerPerceptron]] + * @param partialLossFunction [[PartialLossFunction]] + * @param predictionFunction [[MultiLayerPerceptronPrediction]] + * @param arch [[List[[Int]]]] + */ case class GenericMLPLossFunction( partialLossFunction: PartialLossFunction, predictionFunction: MultiLayerPerceptronPrediction, arch: List[Int]) extends LossFunction { - /** Calculates the gradient as well as the loss given a data point and the weight vector - * - * @param dataPoint - * @param weightVector - * @return - */ - - // Make this spit out / take weight vectors, not reg vetors and you're gtg + /** + * Flattens an array of weights into a [[WeightVector]] + * @param U: [[Array[[BreezeDenseMatrix[[Double]]]]]] + * @return [[WeightVector]] + */ def makeWeightVector(U: Array[BreezeDenseMatrix[Double]]): WeightVector = { val fVector = DenseVector( U.map(_.toDenseVector) .reduceLeft(BreezeDenseVector.vertcat(_,_)).data ) WeightVector( fVector, 0) } + /** + * Converts [[WeightVector]] into a weight array for use by [[GenericMLPLossFunction]] + * @param v [[WeightVector]] + * @param arch network architecture [[List[[Int]]]] + * @return [[Array[[BreezeDenseMatrix[[Double]]]]]] + */ def makeWeightArray(v: WeightVector, arch: List[Int]): Array[BreezeDenseMatrix[Double]] = { val weightVector = Vector2BreezeConverter(v.weights).asBreeze.toDenseVector val breakPoints = arch.iterator.sliding(2).toList.map(o => o(0) * o(1)).scanLeft(0)(_ + _) @@ -59,6 +68,13 @@ case class GenericMLPLossFunction( U } + /** + * Performs feed-forward and back-propegatoin algorithm at specified data point. Return gradient + * for [[org.apache.flink.ml.optimization.IterativeSolver]] + * @param dataPoint [[LabeledVector]] + * @param weightVector [[WeightVector]] + * @return A new tuple object of the loss ([[Double]]) and gradient vector [[WeightVector]] + */ def lossGradient(dataPoint: LabeledVector, weightVector: WeightVector): (Double, WeightVector) = { val ffr = predictionFunction.feedForward(weightVector, dataPoint.vector) val L = arch.length - 1 diff --git a/flink-libraries/flink-ml/src/main/scala/org/apache/flink/ml/neuralnetwork/MultiLayerPerceptron.scala b/flink-libraries/flink-ml/src/main/scala/org/apache/flink/ml/neuralnetwork/MultiLayerPerceptron.scala index 6706c3b4c31a3..9b8389f1414e9 100644 --- a/flink-libraries/flink-ml/src/main/scala/org/apache/flink/ml/neuralnetwork/MultiLayerPerceptron.scala +++ b/flink-libraries/flink-ml/src/main/scala/org/apache/flink/ml/neuralnetwork/MultiLayerPerceptron.scala @@ -33,12 +33,31 @@ import org.apache.flink.api.scala.DataSet import org.apache.flink.ml.pipeline.{PredictOperation, FitOperation, Predictor} -/** Multi-layer Perceptron regression. - * - * docs - * - */ - +/** + * Multi-Layer Perceptron to estimate data point + * + * Neural networks use a directed graph consisting of activation functions (nodes) and parameter + * weights (edges) to solve complex problems with interdependent variables. + * One of the simplest types of neural networks (sometimes referred to as Artificial Neural + * Networks or *ANN*s) is the multiple-layer perceptron (or *MLP*). The Multi-Layer Perceptron is + * a feed forward neural network that has multiple layers which are fully connected with + * *non-linear* activation function at each node and weights at each layer. + * + * + * + * =Parameters= + * + * - [[org.apache.flink.ml.neuralnetwork.MultiLayerPerceptron.HiddenLayerNetworkArchitecture]] + * A list of integer specifying the number of nodes to have in each hidden layer of the network + * + * - [[org.apache.flink.ml.neuralnetwork.MultiLayerPerceptron.Optimizer]] + * An [[IterativeSolver]] to be used in optimizing the multi-layer perceptron + * + * - [[org.apache.flink.ml.neuralnetwork.MultiLayerPerceptron.ActivationFunc]] + * An [[ActivationFunction]] that is continuous and differentiable to be used at each node in the + * network + * + */ class MultiLayerPerceptron extends Predictor[MultiLayerPerceptron] { import MultiLayerPerceptron._ diff --git a/flink-libraries/flink-ml/src/main/scala/org/apache/flink/ml/optimization/PredictionFunction.scala b/flink-libraries/flink-ml/src/main/scala/org/apache/flink/ml/optimization/PredictionFunction.scala index 50293439f6bf0..1cf19e73b19bd 100644 --- a/flink-libraries/flink-ml/src/main/scala/org/apache/flink/ml/optimization/PredictionFunction.scala +++ b/flink-libraries/flink-ml/src/main/scala/org/apache/flink/ml/optimization/PredictionFunction.scala @@ -49,6 +49,15 @@ object LinearPrediction extends PredictionFunction { } +/** + * Prediction Function for MultiLayerPerceptron. Calculates Feed-forward for prediction and + * back propogation + * + * https://en.wikipedia.org/wiki/Multilayer_perceptron + * + * @param arch Architecture of the Neural Network [[List [[Int]]]] + * @param f activationFunction [[ActivationFunction]] + */ case class MultiLayerPerceptronPrediction( arch: List[Int], f: ActivationFunction