From 5582a4b6ee066168fa4e9406e8d399a256b69a66 Mon Sep 17 00:00:00 2001 From: ysqyang Date: Mon, 12 Oct 2020 10:15:28 +0800 Subject: [PATCH 001/337] fixed model load/dump issues --- examples/cim/dqn/dist_learner.py | 4 +- examples/cim/dqn/single_process_launcher.py | 4 +- maro/rl/__init__.py | 2 +- maro/rl/actor/simple_actor.py | 2 +- maro/rl/agent/abs_agent.py | 41 ++++++---- maro/rl/agent/abs_agent_manager.py | 48 ++++++----- maro/rl/algorithms/abs_algorithm.py | 60 ++++++++++++++ maro/rl/algorithms/torch/dqn.py | 78 +++++++++++------- maro/rl/algorithms/torch/pg.py | 90 +++++++++++++++++++++ maro/rl/learner/simple_learner.py | 16 ++-- maro/rl/shaping/experience_shaper.py | 4 +- maro/rl/shaping/k_step_experience_shaper.py | 59 ++++++++------ maro/rl/storage/column_based_store.py | 9 ++- maro/rl/storage/utils.py | 7 +- maro/rl/utils/trajectory_utils.py | 21 +++++ 15 files changed, 332 insertions(+), 113 deletions(-) create mode 100644 maro/rl/algorithms/abs_algorithm.py create mode 100644 maro/rl/algorithms/torch/pg.py create mode 100644 maro/rl/utils/trajectory_utils.py diff --git a/examples/cim/dqn/dist_learner.py b/examples/cim/dqn/dist_learner.py index 08f1d777e..32c501e11 100644 --- a/examples/cim/dqn/dist_learner.py +++ b/examples/cim/dqn/dist_learner.py @@ -31,6 +31,6 @@ learner = SimpleLearner(trainable_agents=agent_manager, actor=ActorProxy(proxy_params=proxy_params), logger=Logger("distributed_cim_learner", auto_timestamp=False)) - learner.train(total_episodes=config.general.total_training_episodes) + learner.train(total_episodes=config.general.total_training_episodes, + model_dump_dir=os.path.join(os.getcwd(), "models")) learner.test() - learner.dump_models(os.path.join(os.getcwd(), "models")) diff --git a/examples/cim/dqn/single_process_launcher.py b/examples/cim/dqn/single_process_launcher.py index a906513ea..98c9fa34e 100644 --- a/examples/cim/dqn/single_process_launcher.py +++ b/examples/cim/dqn/single_process_launcher.py @@ -52,6 +52,6 @@ learner = SimpleLearner(trainable_agents=agent_manager, actor=actor, logger=Logger("single_host_cim_learner", auto_timestamp=False)) - learner.train(total_episodes=config.general.total_training_episodes) + learner.train(total_episodes=config.general.total_training_episodes, + model_dump_dir=os.path.join(os.getcwd(), "models")) learner.test() - learner.dump_models(os.path.join(os.getcwd(), "models")) diff --git a/maro/rl/__init__.py b/maro/rl/__init__.py index bd5c9f221..dd0f397ba 100644 --- a/maro/rl/__init__.py +++ b/maro/rl/__init__.py @@ -7,7 +7,7 @@ from maro.rl.learner.simple_learner import SimpleLearner from maro.rl.agent.abs_agent import AbsAgent from maro.rl.agent.abs_agent_manager import AbsAgentManager, AgentMode -from maro.rl.algorithms.torch.abs_algorithm import AbsAlgorithm +from maro.rl.algorithms.abs_algorithm import AbsAlgorithm from maro.rl.algorithms.torch.dqn import DQN, DQNHyperParams from maro.rl.models.torch.mlp_representation import MLPRepresentation from maro.rl.models.torch.decision_layers import MLPDecisionLayers diff --git a/maro/rl/actor/simple_actor.py b/maro/rl/actor/simple_actor.py index 19ef8f6c5..0c0740eed 100644 --- a/maro/rl/actor/simple_actor.py +++ b/maro/rl/actor/simple_actor.py @@ -41,7 +41,7 @@ def roll_out(self, model_dict: dict = None, epsilon_dict: dict = None, done: boo # load models if model_dict is not None: - self._inference_agents.load_models(model_dict) + self._inference_agents.load_trainable_models(model_dict) metrics, decision_event, is_done = self._env.step(None) while not is_done: diff --git a/maro/rl/agent/abs_agent.py b/maro/rl/agent/abs_agent.py index 0aa9afd76..907b62b87 100644 --- a/maro/rl/agent/abs_agent.py +++ b/maro/rl/agent/abs_agent.py @@ -7,7 +7,7 @@ import torch -from maro.rl.algorithms.torch.abs_algorithm import AbsAlgorithm +from maro.rl.algorithms.abs_algorithm import AbsAlgorithm from maro.rl.storage.abs_store import AbsStore @@ -70,20 +70,33 @@ def store_experiences(self, experiences): """Store new experiences in the experience pool.""" self._experience_pool.put(experiences) - def load_model_dict(self, model_dict: dict): + def load_trainable_models(self, *models, **model_dict): """Load models from memory.""" - self._algorithm.model_dict = model_dict - - def load_model_dict_from_file(self, file_path): - """Load models from a disk file.""" - model_dict = torch.load(file_path) - for model_key, state_dict in model_dict.items(): - self._algorithm.model_dict[model_key].load_state_dict(state_dict) - - def dump_model_dict(self, dir_path: str): - """Dump models to disk.""" - torch.save({model_key: model.state_dict() for model_key, model in self._algorithm.model_dict.items()}, - os.path.join(dir_path, self._name)) + self._algorithm.load_trainable_models(*models, **model_dict) + + def dump_trainable_models(self): + """Return the algorithm's trainable models.""" + return self._algorithm.dump_trainable_models() + + def load_trainable_models_from_file(self, dir_path: str): + """Load trainable models from disk. + + Load trainable models from the specified directory. The model file is always prefixed with the agent's name. + + Args: + dir_path (str): path to the directory where the models are saved. + """ + self._algorithm.load_trainable_models_from_file(os.path.join(dir_path, self._name)) + + def dump_trainable_models_to_file(self, dir_path: str): + """Dump the algorithm's trainable models to disk. + + Dump trainable models to the specified directory. The model file is always prefixed with the agent's name. + + Args: + dir_path (str): path to the directory where the models are saved. + """ + self._algorithm.dump_trainable_models_to_file(os.path.join(dir_path, self._name)) def dump_experience_store(self, dir_path: str): """Dump the experience pool to disk.""" diff --git a/maro/rl/agent/abs_agent_manager.py b/maro/rl/agent/abs_agent_manager.py index bed57db2a..3e6f1ce11 100644 --- a/maro/rl/agent/abs_agent_manager.py +++ b/maro/rl/agent/abs_agent_manager.py @@ -9,6 +9,7 @@ from maro.rl.shaping.action_shaper import ActionShaper from maro.rl.shaping.experience_shaper import ExperienceShaper from maro.rl.explorer.abs_explorer import AbsExplorer +from maro.rl.storage.column_based_store import ColumnBasedStore from maro.utils.exception.rl_toolkit_exception import UnsupportedAgentModeError, MissingShaperError, WrongAgentModeError @@ -68,7 +69,8 @@ def __init__(self, self._explorer = explorer self._agent_id_list = agent_id_list - self._trajectory = [] + self._current_transition = {} + self._trajectory = ColumnBasedStore() self._agent_dict = {} self._assemble(self._agent_dict) @@ -102,11 +104,11 @@ def choose_action(self, decision_event, snapshot_list): agent_id, model_state = self._state_shaper(decision_event, snapshot_list) model_action = self._agent_dict[agent_id].choose_action( model_state, self._explorer.epsilon[agent_id] if self._explorer else None) - self._trajectory.append({"state": model_state, - "action": model_action, - "reward": None, - "agent_id": agent_id, - "event": decision_event}) + self._current_transition = {"state": model_state, + "action": model_action, + "reward": None, + "agent_id": agent_id, + "event": decision_event} return self._action_shaper(model_action, decision_event, snapshot_list) def on_env_feedback(self, metrics): @@ -115,7 +117,8 @@ def on_env_feedback(self, metrics): Args: metrics: business metrics provided by the environment after an action has been executed. """ - self._trajectory[-1]["metrics"] = metrics + self._current_transition["metrics"] = metrics + self._trajectory.put(self._current_transition) def post_process(self, snapshot_list): """This method processes the latest trajectory into experiences. @@ -125,6 +128,7 @@ def post_process(self, snapshot_list): """ experiences = self._experience_shaper(self._trajectory, snapshot_list) self._trajectory.clear() + self._current_transition = {} self._state_shaper.reset() self._action_shaper.reset() self._experience_shaper.reset() @@ -159,31 +163,31 @@ def train(self): for agent in self._agent_dict.values(): agent.train() - def load_models(self, agent_model_dict): + def load_trainable_models(self, agent_model_dict): """Load models from memory for each agent.""" - for agent_id, model_dict in agent_model_dict.items(): - self._agent_dict[agent_id].load_model_dict(model_dict) + for agent_id, models in agent_model_dict.items(): + self._agent_dict[agent_id].load_trainable_models(models) - def load_models_from_files(self, file_path_dict): + def dump_trainable_models(self): + """Get agents' underlying models. + + This is usually used in distributed mode where models need to be broadcast to remote roll-out actors. + """ + return {agent_id: agent.dump_trainable_models() for agent_id, agent in self._agent_dict.items()} + + def load_trainable_models_from_files(self, dir_path): """Load models from disk for each agent.""" - for agent_id, file_path in file_path_dict.items(): - self._agent_dict[agent_id].load_model_dict_from(file_path) + for agent in self._agent_dict.values(): + agent.load_trainable_models_from_file(dir_path) - def dump_models(self, dir_path: str): + def dump_trainable_models_to_files(self, dir_path: str): """Dump agents' models to disk. Each agent will use its own name to create a separate file under ``dir_path`` for dumping. """ os.makedirs(dir_path, exist_ok=True) for agent in self._agent_dict.values(): - agent.dump_model_dict(dir_path) - - def get_models(self): - """Get agents' underlying models. - - This is usually used in distributed mode where models need to be broadcast to remote roll-out actors. - """ - return {agent_id: agent.algorithm.model_dict for agent_id, agent in self._agent_dict.items()} + agent.dump_trainable_models(dir_path) @property def name(self): diff --git a/maro/rl/algorithms/abs_algorithm.py b/maro/rl/algorithms/abs_algorithm.py new file mode 100644 index 000000000..fabe0d354 --- /dev/null +++ b/maro/rl/algorithms/abs_algorithm.py @@ -0,0 +1,60 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +from abc import ABC, abstractmethod + + +class AbsAlgorithm(ABC): + """Abstract RL algorithm class. + + The class provides uniform policy interfaces such as ``choose_action`` and ``train``. We also provide some + predefined RL algorithm based on it, such DQN, A2C, etc. User can inherit from it to customize their own + algorithms. + """ + def __init__(self): + pass + + @abstractmethod + def choose_action(self, state, epsilon: float = None): + """This method uses the underlying model(s) to compute an action from a shaped state. + + Args: + state: A state object shaped by a ``StateShaper`` to conform to the model input format. + epsilon (float, optional): Exploration rate. For greedy value-based algorithms, this being None means + using the model output without exploration. For algorithms with inherently stochastic policies such + as policy gradient, this is usually ignored. Defaults to None. + + Returns: + The action to be taken given ``state``. It is usually necessary to use an ``ActionShaper`` to convert + this to an environment executable action. + """ + return NotImplementedError + + @abstractmethod + def train(self, *args, **kwargs): + """Train models using samples. + + This method is algorithm-specific and needs to be implemented by the user. For example, for the DQN + algorithm, this may look like train(self, state, action, reward, next_state). + """ + return NotImplementedError + + @abstractmethod + def load_trainable_models(self, *models, **model_dict): + """Load trainable models from memory.""" + return NotImplementedError + + @abstractmethod + def dump_trainable_models(self): + """Return the algorithm's trainable models.""" + return NotImplementedError + + @abstractmethod + def load_trainable_models_from_file(self, path): + """Load trainable models from disk.""" + return NotImplementedError + + @abstractmethod + def dump_trainable_models_to_file(self, path: str): + """Dump the algorithm's trainable models to disk.""" + return NotImplementedError diff --git a/maro/rl/algorithms/torch/dqn.py b/maro/rl/algorithms/torch/dqn.py index 22cc0bade..d6ab05d7d 100644 --- a/maro/rl/algorithms/torch/dqn.py +++ b/maro/rl/algorithms/torch/dqn.py @@ -1,13 +1,11 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT license. -from typing import Union - import numpy as np import torch +import torch.nn as nn - -from maro.rl.algorithms.torch.abs_algorithm import AbsAlgorithm +from maro.rl.algorithms.abs_algorithm import AbsAlgorithm from maro.utils import clone @@ -32,44 +30,48 @@ def __init__(self, num_actions: int, reward_decay: float, num_training_rounds_pe class DQN(AbsAlgorithm): """The Deep-Q-Networks algorithm. - The model_dict must contain the key `eval`. Optionally a model corresponding to the key `target` can be - provided. If the key `target` is absent or model_dict[`target`] is None, the target model will be a deep + The model must contain the key `eval`. Optionally a model corresponding to the key `target` can be + provided. If the key `target` is absent or model[`target`] is None, the target model will be a deep copy of the provided eval model. """ - def __init__(self, model_dict: dict, optimizer_opt: Union[dict, tuple], loss_func_dict: dict, - hyper_params: DQNHyperParams): - if model_dict.get("target", None) is None: - model_dict["target"] = clone(model_dict["eval"]) - super().__init__(model_dict, optimizer_opt, loss_func_dict, hyper_params) + def __init__(self, eval_model: nn.Module, optimizer_cls, optimizer_params, loss_func, hyper_params: DQNHyperParams, + target_model: nn.Module = None): + super().__init__() + self._eval_model = eval_model + self._target_model = clone(eval_model) if target_model is None else target_model + self._optimizer = optimizer_cls(self._eval_model.parameters(), **optimizer_params) + self._loss_func = loss_func + self._hyper_params = hyper_params self._train_cnt = 0 self._device = torch.device("cuda" if torch.cuda.is_available() else "cpu") + @property + def eval_model(self): + return self._eval_model + def choose_action(self, state: np.ndarray, epsilon: float = None): if epsilon is None or np.random.rand() > epsilon: state = torch.from_numpy(state).unsqueeze(0) - self._model_dict["eval"].eval() + self._eval_model.eval() with torch.no_grad(): - q_values = self._model_dict["eval"](state) + q_values = self._eval_model(state) best_action_idx = q_values.argmax(dim=1).item() return best_action_idx return np.random.choice(self._hyper_params.num_actions) - def _prepare_batch(self, raw_batch): - return {key: torch.from_numpy(np.asarray(lst)).to(self._device) for key, lst in raw_batch.items()} - - def train(self, state: np.ndarray, action: np.ndarray, reward: np.ndarray, next_state: np.ndarray): - state = torch.from_numpy(state).to(self._device) - action = torch.from_numpy(action).to(self._device) - reward = torch.from_numpy(reward).to(self._device) - next_state = torch.from_numpy(next_state).to(self._device) - if len(action.shape) == 1: - action = action.unsqueeze(1) - current_q_values = self._model_dict["eval"](state).gather(1, action).squeeze(1) - next_q_values = self._model_dict["target"](next_state).max(dim=1)[0] - target_q_values = (reward + self._hyper_params.reward_decay * next_q_values).detach() - loss = self._loss_func_dict["eval"](current_q_values, target_q_values) - self._model_dict["eval"].train() + def train(self, states: np.ndarray, actions: np.ndarray, rewards: np.ndarray, next_states: np.ndarray): + states = torch.from_numpy(states).to(self._device) + actions = torch.from_numpy(actions).to(self._device) + rewards = torch.from_numpy(rewards).to(self._device) # (N,) + next_states = torch.from_numpy(next_states).to(self._device) + if len(actions.shape) == 1: + actions = actions.unsqueeze(1) # (N, 1) + current_q_values = self._eval_model(states).gather(1, actions).squeeze(1) # (N,) + next_q_values = self._target_model(next_states).max(dim=1)[0] # (N,) + target_q_values = (rewards + self._hyper_params.reward_decay * next_q_values).detach() # (N,) + loss = self._loss_func(current_q_values, target_q_values) + self._eval_model.train() self._optimizer.zero_grad() loss.backward() self._optimizer.step() @@ -80,9 +82,23 @@ def train(self, state: np.ndarray, action: np.ndarray, reward: np.ndarray, next_ return np.abs((current_q_values - target_q_values).detach().numpy()) def _update_target_model(self): - for eval_params, target_params in zip( - self._model_dict["eval"].parameters(), self._model_dict["target"].parameters() - ): + for eval_params, target_params in zip(self._eval_model.parameters(), self._target_model.parameters()): target_params.data = ( self._hyper_params.tau * eval_params.data + (1 - self._hyper_params.tau) * target_params.data ) + + def load_trainable_models(self, eval_model): + """Load the eval model from memory.""" + self._eval_model = eval_model + + def dump_trainable_models(self): + """Return the eval model.""" + return self._eval_model + + def load_trainable_models_from_file(self, path): + """Load the eval model from disk.""" + self._eval_model = torch.load(path) + + def dump_trainable_models_to_file(self, path: str): + """Dump the eval model to disk.""" + torch.save(self._eval_model.state_dict(), path) diff --git a/maro/rl/algorithms/torch/pg.py b/maro/rl/algorithms/torch/pg.py new file mode 100644 index 000000000..b927719b3 --- /dev/null +++ b/maro/rl/algorithms/torch/pg.py @@ -0,0 +1,90 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import numpy as np +import torch +import torch.nn as nn +import torch.nn.functional as F + +from maro.rl.algorithms.abs_algorithm import AbsAlgorithm + + +class PolicyGradientHyperParameters: + """PG hyper-parameters. + + Args: + num_actions (int): number of possible actions + reward_decay (float): reward decay as defined in standard RL terminology + """ + __slots__ = ["num_actions", "reward_decay"] + + def __init__(self, num_actions: int, reward_decay: float): + self.num_actions = num_actions + self.reward_decay = reward_decay + + +class PolicyGradient(AbsAlgorithm): + """Vanilla policy gradient algorithm. + """ + + def __init__(self, policy_model: nn.Module, policy_optimizer_cls, policy_optimizer_params, + hyper_params: PolicyGradientHyperParameters, value_model: nn.Module = None, + value_optimizer_cls=None, value_optimizer_params=None, value_loss_func=None): + super().__init__() + self._policy_model = policy_model + self._policy_optimizer = policy_optimizer_cls(self._policy_model.parameters(), **policy_optimizer_params) + self._value_model = value_model + if self._value_model is not None: + assert value_optimizer_cls is not None and value_optimizer_params is not None, \ + "value_optimizer_cls and value_optimizer_params should not be None if value model is not None" + self._value_optimizer = value_optimizer_cls(self._value_model.parameters(), **value_optimizer_params) + else: + self._value_optimizer = None + self._value_loss_func = value_loss_func + self._hyper_params = hyper_params + self._device = torch.device("cuda" if torch.cuda.is_available() else "cpu") + + @property + def model(self): + return {"policy": self._policy_model, "value": self._value_model} + + def choose_action(self, state: np.ndarray, epsilon: float = None): + state = torch.from_numpy(state).unsqueeze(0).to(self._device) # (1, state_dim) + action_dist = F.softmax(self._policy_model(state), dim=1).squeeze() # (num_actions,) + return np.random.choice(self._hyper_params.num_actions, p=action_dist.numpy()) + + def train(self, states, actions, returns): + states = torch.from_numpy(states).to(self._device) # (N, state_dim) + actions = torch.from_numpy(actions).to(self._device) # (N,) + returns = torch.from_numpy(returns).to(self._device) # (N,) + # policy model training + action_dist = F.softmax(self._policy_model(states), dim=1) # (N, num_actions) + action_prob = action_dist.gather(1, actions.unsqueeze(1)) # (N, 1) + log_action_prob = torch.log(action_prob).squeeze() # (N,) + policy_loss = -(log_action_prob * returns).mean() + self._policy_optimizer.zero_grad() + policy_loss.backward() + self._policy_optimizer.step() + # value model training (if a value model is present) + if self._value_model is not None: + value_loss = self._value_loss_func(self._value_model(states), returns) + self._value_optimizer.zero_grad() + value_loss.backward() + self._value_optimizer.step() + + def load_trainable_models(self, policy_model, value_model): + self._policy_model = policy_model + self._value_model = value_model + + def dump_trainable_models(self): + return {"policy": self._policy_model, "value": self._value_model} + + def load_trainable_models_from_file(self, path): + """Load trainable models from disk.""" + model_dict = torch.load(path) + self._policy_model = model_dict["policy"] + self._value_model = model_dict["value"] + + def dump_trainable_models_to_file(self, path: str): + """Dump the algorithm's trainable models to disk.""" + torch.save({"policy": self._policy_model.state_dict(), "value": self._value_model.state_dict()}, path) diff --git a/maro/rl/learner/simple_learner.py b/maro/rl/learner/simple_learner.py index 5599562a7..c99d766b6 100644 --- a/maro/rl/learner/simple_learner.py +++ b/maro/rl/learner/simple_learner.py @@ -21,14 +21,16 @@ def __init__(self, trainable_agents: AbsAgentManager, actor, logger=DummyLogger( self._actor = actor self._logger = logger - def train(self, total_episodes): + def train(self, total_episodes, model_dump_dir: str = None): """Main loop for collecting experiences from the actor and using them to update policies. Args: total_episodes (int): number of episodes to be run. + model_dump_dir (str): If a path is provided, it will be treated as a directory under which all agents' + trainable models will be dumped. """ for current_ep in range(1, total_episodes + 1): - model_dict = None if self._is_shared_agent_instance() else self._trainable_agents.get_models() + model_dict = None if self._is_shared_agent_instance() else self._trainable_agents.dump_trainable_models() epsilon_dict = self._trainable_agents.explorer.epsilon if self._trainable_agents.explorer else None performance, exp_by_agent = self._actor.roll_out(model_dict=model_dict, epsilon_dict=epsilon_dict) if isinstance(performance, dict): @@ -42,17 +44,17 @@ def train(self, total_episodes): self._trainable_agents.train() self._trainable_agents.update_epsilon(performance) + if model_dump_dir is not None: + self._trainable_agents.dump_trainable_models_to_files(model_dump_dir) + def test(self): """Test policy performance.""" - performance, _ = self._actor.roll_out(model_dict=self._trainable_agents.get_models(), return_details=False) + performance, _ = self._actor.roll_out(model_dict=self._trainable_agents.dump_trainable_models(), + return_details=False) for actor_id, perf in performance.items(): self._logger.info(f"test performance from {actor_id}: {perf}") self._actor.roll_out(done=True) - def dump_models(self, dir_path: str): - """Dump agents' models to disk.""" - self._trainable_agents.dump_models(dir_path) - def _is_shared_agent_instance(self): """If true, the set of agents performing inference in actor is the same as self._trainable_agents.""" return isinstance(self._actor, SimpleActor) and id(self._actor.inference_agents) == id(self._trainable_agents) diff --git a/maro/rl/shaping/experience_shaper.py b/maro/rl/shaping/experience_shaper.py index 428669a83..0fa01d4c8 100644 --- a/maro/rl/shaping/experience_shaper.py +++ b/maro/rl/shaping/experience_shaper.py @@ -23,11 +23,11 @@ def __init__(self, reward_func: Union[Callable, None], *args, **kwargs): self._reward_func = reward_func @abstractmethod - def __call__(self, trajectory: Sequence, snapshot_list) -> Iterable: + def __call__(self, trajectory, snapshot_list) -> Iterable: """Converts transitions along a trajectory to experiences. Args: - trajectory(Sequence): A sequence of transitions recorded by the agent manager during roll-out. + trajectory: A sequence of transitions recorded by the agent manager during roll-out. snapshot_list: Snapshot list stored in the environment at the end of an episode. Returns: Experiences that can be used by the algorithm. diff --git a/maro/rl/shaping/k_step_experience_shaper.py b/maro/rl/shaping/k_step_experience_shaper.py index 1ffa65722..c1782ac68 100644 --- a/maro/rl/shaping/k_step_experience_shaper.py +++ b/maro/rl/shaping/k_step_experience_shaper.py @@ -1,11 +1,13 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT license. -from collections import defaultdict, deque from enum import Enum from typing import Callable +import numpy as np + from .experience_shaper import ExperienceShaper +from maro.rl.utils.trajectory_utils import get_k_step_discounted_sums class KStepExperienceKeys(Enum): @@ -34,28 +36,33 @@ def __init__(self, reward_func: Callable, reward_decay: float, steps: int, is_pe self._is_per_agent = is_per_agent def __call__(self, trajectory, snapshot_list): - experiences = defaultdict(lambda: defaultdict(deque)) if self._is_per_agent else defaultdict(deque) - reward_list = deque() - full_return = partial_return = 0 - for i in range(len(trajectory) - 2, -1, -1): - transition = trajectory[i] - next_transition = trajectory[min(len(trajectory) - 1, i + self._steps)] - reward_list.appendleft(self._reward_func(trajectory[i]["metrics"])) - # compute the full return - full_return = full_return * self._reward_decay + reward_list[0] - # compute the partial return - partial_return = partial_return * self._reward_decay + reward_list[0] - if len(reward_list) > self._steps: - partial_return -= reward_list.pop() * self._reward_decay ** (self._steps - 1) - agent_exp = experiences[transition["agent_id"]] if self._is_per_agent else experiences - agent_exp[KStepExperienceKeys.STATE.value].appendleft(transition["state"]) - agent_exp[KStepExperienceKeys.ACTION.value].appendleft(transition["action"]) - agent_exp[KStepExperienceKeys.REWARD.value].appendleft(partial_return) - agent_exp[KStepExperienceKeys.RETURN.value].appendleft(full_return) - agent_exp[KStepExperienceKeys.NEXT_STATE.value].appendleft(next_transition["state"]) - agent_exp[KStepExperienceKeys.NEXT_ACTION.value].appendleft(next_transition["action"]) - agent_exp[KStepExperienceKeys.DISCOUNT.value].appendleft( - self._reward_decay ** (min(self._steps, len(trajectory) - 1 - i)) - ) - - return dict(experiences) + length = len(trajectory) + agent_ids = np.asarray(trajectory.get_by_key["agent_id"])[:-1] + states = np.asarray(trajectory.get_by_key["state"]) + actions = np.asarray(trajectory.get_by_key["action"]) + reward_array = np.fromiter(map(self._reward_func, trajectory.get_by_key("metrics")[:-1]), dtype=np.float32) + reward_sums = get_k_step_discounted_sums(reward_array, self._reward_decay, k=self._steps)[:-1] + returns = get_k_step_discounted_sums(reward_array, self._reward_decay)[:-1] + discounts = np.array([self._reward_decay ** min(self._steps, length-i-1) for i in range(length-1)]) + next_states = np.pad(states[self._steps:], (0, length-self._steps-1), mode="edge") + next_actions = np.pad(actions[self._steps:], (0, length-self._steps-1), mode="edge") + + states, actions = states[:-1], actions[:-1] + + if self._is_per_agent: + return {agent_id: {KStepExperienceKeys.STATE.value: states[agent_ids == agent_id], + KStepExperienceKeys.ACTION.value: actions[agent_ids == agent_id], + KStepExperienceKeys.REWARD.value: reward_sums[agent_ids == agent_id], + KStepExperienceKeys.RETURN.value: returns[agent_ids == agent_id], + KStepExperienceKeys.NEXT_STATE.value: next_states[agent_ids == agent_id], + KStepExperienceKeys.NEXT_ACTION.value: next_actions[agent_ids == agent_id], + KStepExperienceKeys.DISCOUNT.value: discounts[agent_ids == agent_id]} + for agent_id in set(agent_ids)} + else: + return {KStepExperienceKeys.STATE.value: states, + KStepExperienceKeys.ACTION.value: actions, + KStepExperienceKeys.REWARD.value: reward_sums, + KStepExperienceKeys.RETURN.value: returns, + KStepExperienceKeys.NEXT_STATE.value: next_states, + KStepExperienceKeys.NEXT_ACTION.value: next_actions, + KStepExperienceKeys.DISCOUNT.value: discounts} diff --git a/maro/rl/storage/column_based_store.py b/maro/rl/storage/column_based_store.py index 8709a8d8e..935117de5 100644 --- a/maro/rl/storage/column_based_store.py +++ b/maro/rl/storage/column_based_store.py @@ -75,7 +75,7 @@ def put(self, contents: dict, overwrite_indexes: Sequence = None) -> List[int]: """Put new contents in the store. Args: - contents (Sequence): Item object list. + contents (dict): Item object list. overwrite_indexes (Sequence, optional): indexes where the contents are to be overwritten. This is only used when the store has a fixed capacity and putting ``contents`` in the store would exceed this capacity. If this is None and overwriting is necessary, rolling or random overwriting will be done @@ -87,8 +87,11 @@ def put(self, contents: dict, overwrite_indexes: Sequence = None) -> List[int]: raise ValueError(f"expected keys {list(self._store.keys())}, got {list(contents.keys())}") added_size = len(contents[next(iter(contents))]) if self._capacity < 0: - for key, lst in contents.items(): - self._store[key].extend(lst) + for key, val in contents.items(): + if not isinstance(val, list) and not isinstance(val, np.ndarray): + self._store[key].append(val) + else: + self._store[key].extend(val) self._size += added_size return list(range(self._size - added_size, self._size)) else: diff --git a/maro/rl/storage/utils.py b/maro/rl/storage/utils.py index c942d13bb..ce397eb03 100644 --- a/maro/rl/storage/utils.py +++ b/maro/rl/storage/utils.py @@ -3,6 +3,7 @@ from enum import Enum from functools import wraps +from typing import Sequence import numpy as np @@ -12,9 +13,11 @@ def decorator(func): @wraps(func) def wrapper(*args, **kwargs): contents = args[arg_num] + if all(not isinstance(val, list) for val in contents.values()): + return func(*args, **kwargs) length = len(contents[next(iter(contents))]) - if any(len(lst) != length for lst in contents.values()): - raise ValueError("all sequences in contents should have the same length") + if any(not isinstance(val, list) or len(val) != length for val in contents.values()): + raise ValueError("values of contents should consist of lists of the same length") return func(*args, **kwargs) return wrapper return decorator diff --git a/maro/rl/utils/trajectory_utils.py b/maro/rl/utils/trajectory_utils.py new file mode 100644 index 000000000..47c667f24 --- /dev/null +++ b/maro/rl/utils/trajectory_utils.py @@ -0,0 +1,21 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +from functools import reduce + +import numpy as np + + +def get_k_step_discounted_sums(arr: np.ndarray, discount: float, k: int = -1): + if k < 0: + k = len(arr) + return reduce(lambda x, y: x * discount + y, + [np.pad(arr[i:], (0, i)) for i in range(min(k, len(arr))-1, -1, -1)]) + + +rw = np.asarray([3, 2, 4, 1, 5]) +gamma = 0.8 + +print(get_k_step_discounted_sums(rw, gamma, k=4)) + + From f62b7e419bff45ed183c7e2297c467cf13231a31 Mon Sep 17 00:00:00 2001 From: ysqyang Date: Tue, 13 Oct 2020 00:42:00 +0800 Subject: [PATCH 002/337] added lambda return util function --- maro/rl/utils/trajectory_utils.py | 69 +++++++++++++++++++++---------- 1 file changed, 48 insertions(+), 21 deletions(-) diff --git a/maro/rl/utils/trajectory_utils.py b/maro/rl/utils/trajectory_utils.py index 47c667f24..24ba99d2f 100644 --- a/maro/rl/utils/trajectory_utils.py +++ b/maro/rl/utils/trajectory_utils.py @@ -1,21 +1,48 @@ -# Copyright (c) Microsoft Corporation. -# Licensed under the MIT license. - -from functools import reduce - -import numpy as np - - -def get_k_step_discounted_sums(arr: np.ndarray, discount: float, k: int = -1): - if k < 0: - k = len(arr) - return reduce(lambda x, y: x * discount + y, - [np.pad(arr[i:], (0, i)) for i in range(min(k, len(arr))-1, -1, -1)]) - - -rw = np.asarray([3, 2, 4, 1, 5]) -gamma = 0.8 - -print(get_k_step_discounted_sums(rw, gamma, k=4)) - - +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +from functools import reduce + +import numpy as np + + +def get_k_step_discounted_sums(rewards: np.ndrewardsay, discount: float, k: int = -1, values: np.ndrewardsay = None): + assert values is None or len(rewards) == len(values), "rewards and values should have the same length" + if values is not None: + rewards[-1] = values[-1] + if k < 0: + k = len(rewards) - 1 + return reduce(lambda x, y: x*discount + y, + [np.pad(rewards[i:], (0, i)) for i in range(min(k, len(rewards))-1, -1, -1)], + np.pad(values[k:], (0, k)) if values is not None else np.zeros(len(rewards))) + + +def get_lambda_returns(rewards: np.ndrewardsay, discount: float, lda: float, values: np.ndrewardsay = None, + horizon: int = -1): + if horizon < 0: + horizon = len(rewards) - 1 + + horizon = min(horizon, len(rewards) - 1) + pre_truncate = reduce(lambda x, y: x*lda + y, + [get_k_step_discounted_sums(rewards, discount, k=k, values=values) + for k in range(horizon-1, 0, -1)]) + + post_truncate = get_k_step_discounted_sums(rewards, discount, k=horizon, values=values) * lda**(horizon-1) + return (1 - lda) * pre_truncate + post_truncate + + +b = np.asrewardsay([4, 7, 1, 3, 6]) +rw = np.asrewardsay([3, 2, 4, 1, 5]) +ld = 0.6 +gamma = 0.8 +steps = 4 +hrz = 3 + +print(get_lambda_returns(rw, gamma, ld, values=b, horizon=hrz)) + +""" +2-step: [5.24 7.12 8.64 5.8 6. ] +1-step: [8.6 2.8 6.4 5.8 6. ] +3-step: [8.696 8.912 8.64 5.8 6. ] +[7.82816 6.03712 7.744 5.8 6. ] +""" From 933eb08ae6119c61e44611c2b60d3b846103d785 Mon Sep 17 00:00:00 2001 From: ysqyang Date: Tue, 13 Oct 2020 20:23:52 +0800 Subject: [PATCH 003/337] rewrote training logic for PG using util functions and made choose_action abstract --- examples/cim/dqn/components/agent_manager.py | 13 +++++++ maro/rl/agent/abs_agent_manager.py | 38 ++++++++------------ maro/rl/algorithms/torch/dqn.py | 9 +++-- maro/rl/algorithms/torch/pg.py | 37 ++++++++++++------- maro/rl/shaping/k_step_experience_shaper.py | 8 ++--- maro/rl/utils/trajectory_utils.py | 16 ++++----- 6 files changed, 67 insertions(+), 54 deletions(-) diff --git a/examples/cim/dqn/components/agent_manager.py b/examples/cim/dqn/components/agent_manager.py index 59eaf0956..a1dbcc203 100644 --- a/examples/cim/dqn/components/agent_manager.py +++ b/examples/cim/dqn/components/agent_manager.py @@ -31,6 +31,19 @@ def _assemble(self, agent_dict): agent_dict[agent_id] = CIMAgent(name=agent_id, algorithm=algorithm, experience_pool=experience_pool, **config.agents.training_loop_parameters) + def choose_action(self, decision_event, snapshot_list): + self._assert_inference_mode() + agent_id, model_state = self._state_shaper(decision_event, snapshot_list) + model_action = self._agent_dict[agent_id].choose_action( + model_state, self._explorer.epsilon[agent_id] if self._explorer else None) + + self._transition_cache = {"state": model_state, + "action": model_action, + "reward": None, + "agent_id": agent_id, + "event": decision_event} + return self._action_shaper(model_action, decision_event, snapshot_list) + def store_experiences(self, experiences): for agent_id, exp in experiences.items(): exp.update({"loss": [1e8] * len(exp[next(iter(exp))])}) diff --git a/maro/rl/agent/abs_agent_manager.py b/maro/rl/agent/abs_agent_manager.py index 3e6f1ce11..4a5925491 100644 --- a/maro/rl/agent/abs_agent_manager.py +++ b/maro/rl/agent/abs_agent_manager.py @@ -4,6 +4,7 @@ from abc import ABC, abstractmethod from enum import Enum import os +from typing import Callable from maro.rl.shaping.state_shaper import StateShaper from maro.rl.shaping.action_shaper import ActionShaper @@ -49,7 +50,9 @@ def __init__(self, state_shaper: StateShaper = None, action_shaper: ActionShaper = None, experience_shaper: ExperienceShaper = None, - explorer: AbsExplorer = None): + explorer: AbsExplorer = None, + on_event_callback: Callable = None, + on_feedback_call_back: Callable = None): self._name = name if mode not in AgentMode: raise UnsupportedAgentModeError(msg='mode must be "train", "inference" or "train_inference"') @@ -69,9 +72,12 @@ def __init__(self, self._explorer = explorer self._agent_id_list = agent_id_list - self._current_transition = {} + self._transition_cache = {} self._trajectory = ColumnBasedStore() + self._on_event_callback = on_event_callback + self._on_feedback_callback = on_feedback_call_back + self._agent_dict = {} self._assemble(self._agent_dict) @@ -82,16 +88,11 @@ def _assemble(self, agent_dict): """Assembles agents and fill the ``agent_dict`` with them.""" return NotImplemented + @abstractmethod def choose_action(self, decision_event, snapshot_list): - """This is the interface for interacting with the environment. + """Generate an environment executable action given the current decision event and snapshot list. - The method consists of 4 steps: - - 1. The decision event and snapshot list are converted by the state shaper to a model input. - The state shaper also finds the target agent ID. - 2. The target agent takes the model input and uses its underlying models to compute an action. - 3. Key information regarding the transition is recorded in the ``_trajectory`` attribute. - 4. The action computed by the model is converted to an environment executable action by the action shaper. + Key information can be recorded in the ``_transition_cache`` attribute for experience shaping. Args: decision_event: A decision event that prompts an action. @@ -100,16 +101,7 @@ def choose_action(self, decision_event, snapshot_list): Returns: An action object that can be passed directly to an environment's ``step`` method. """ - self._assert_inference_mode() - agent_id, model_state = self._state_shaper(decision_event, snapshot_list) - model_action = self._agent_dict[agent_id].choose_action( - model_state, self._explorer.epsilon[agent_id] if self._explorer else None) - self._current_transition = {"state": model_state, - "action": model_action, - "reward": None, - "agent_id": agent_id, - "event": decision_event} - return self._action_shaper(model_action, decision_event, snapshot_list) + return NotImplementedError def on_env_feedback(self, metrics): """This method records the environment-generated metrics as part of the latest transition in the trajectory. @@ -117,8 +109,8 @@ def on_env_feedback(self, metrics): Args: metrics: business metrics provided by the environment after an action has been executed. """ - self._current_transition["metrics"] = metrics - self._trajectory.put(self._current_transition) + self._transition_cache["metrics"] = metrics + self._trajectory.put(self._transition_cache) def post_process(self, snapshot_list): """This method processes the latest trajectory into experiences. @@ -128,7 +120,7 @@ def post_process(self, snapshot_list): """ experiences = self._experience_shaper(self._trajectory, snapshot_list) self._trajectory.clear() - self._current_transition = {} + self._transition_cache = {} self._state_shaper.reset() self._action_shaper.reset() self._experience_shaper.reset() diff --git a/maro/rl/algorithms/torch/dqn.py b/maro/rl/algorithms/torch/dqn.py index d6ab05d7d..c24c60f3a 100644 --- a/maro/rl/algorithms/torch/dqn.py +++ b/maro/rl/algorithms/torch/dqn.py @@ -55,16 +55,15 @@ def choose_action(self, state: np.ndarray, epsilon: float = None): self._eval_model.eval() with torch.no_grad(): q_values = self._eval_model(state) - best_action_idx = q_values.argmax(dim=1).item() - return best_action_idx + return q_values.argmax(dim=1).item() return np.random.choice(self._hyper_params.num_actions) def train(self, states: np.ndarray, actions: np.ndarray, rewards: np.ndarray, next_states: np.ndarray): - states = torch.from_numpy(states).to(self._device) - actions = torch.from_numpy(actions).to(self._device) + states = torch.from_numpy(states).to(self._device) # (N, state_dim) + actions = torch.from_numpy(actions).to(self._device) # (N,) rewards = torch.from_numpy(rewards).to(self._device) # (N,) - next_states = torch.from_numpy(next_states).to(self._device) + next_states = torch.from_numpy(next_states).to(self._device) # (N, state_dim) if len(actions.shape) == 1: actions = actions.unsqueeze(1) # (N, 1) current_q_values = self._eval_model(states).gather(1, actions).squeeze(1) # (N,) diff --git a/maro/rl/algorithms/torch/pg.py b/maro/rl/algorithms/torch/pg.py index b927719b3..6161085e1 100644 --- a/maro/rl/algorithms/torch/pg.py +++ b/maro/rl/algorithms/torch/pg.py @@ -7,6 +7,7 @@ import torch.nn.functional as F from maro.rl.algorithms.abs_algorithm import AbsAlgorithm +from maro.rl.utils.trajectory_utils import get_k_step_discounted_sums, get_lambda_returns class PolicyGradientHyperParameters: @@ -15,12 +16,18 @@ class PolicyGradientHyperParameters: Args: num_actions (int): number of possible actions reward_decay (float): reward decay as defined in standard RL terminology + k (int): number of time steps used in computing bootstrapped return estimates. + lmda (float): lambda coefficient used in computing lambda returns. If it is not None, ``k`` will be used as + the roll-out horizon in computing truncated lambda returns. Otherwise, ``k`` will be used as usual in + computing multi-step bootstrapped return estimates. Defaults to None. """ - __slots__ = ["num_actions", "reward_decay"] + __slots__ = ["num_actions", "reward_decay", "k", "lmda"] - def __init__(self, num_actions: int, reward_decay: float): + def __init__(self, num_actions: int, reward_decay: float, k: int = 1, lmda: float = None): self.num_actions = num_actions self.reward_decay = reward_decay + self.k = k + self.lmda = lmda class PolicyGradient(AbsAlgorithm): @@ -31,9 +38,10 @@ def __init__(self, policy_model: nn.Module, policy_optimizer_cls, policy_optimiz hyper_params: PolicyGradientHyperParameters, value_model: nn.Module = None, value_optimizer_cls=None, value_optimizer_params=None, value_loss_func=None): super().__init__() - self._policy_model = policy_model + self._device = torch.device("cuda" if torch.cuda.is_available() else "cpu") + self._policy_model = policy_model.to(self._device) self._policy_optimizer = policy_optimizer_cls(self._policy_model.parameters(), **policy_optimizer_params) - self._value_model = value_model + self._value_model = value_model.to(self._device) if self._value_model is not None: assert value_optimizer_cls is not None and value_optimizer_params is not None, \ "value_optimizer_cls and value_optimizer_params should not be None if value model is not None" @@ -42,7 +50,6 @@ def __init__(self, policy_model: nn.Module, policy_optimizer_cls, policy_optimiz self._value_optimizer = None self._value_loss_func = value_loss_func self._hyper_params = hyper_params - self._device = torch.device("cuda" if torch.cuda.is_available() else "cpu") @property def model(self): @@ -53,11 +60,17 @@ def choose_action(self, state: np.ndarray, epsilon: float = None): action_dist = F.softmax(self._policy_model(state), dim=1).squeeze() # (num_actions,) return np.random.choice(self._hyper_params.num_actions, p=action_dist.numpy()) - def train(self, states, actions, returns): - states = torch.from_numpy(states).to(self._device) # (N, state_dim) - actions = torch.from_numpy(actions).to(self._device) # (N,) - returns = torch.from_numpy(returns).to(self._device) # (N,) + def train(self, state_sequence: np.ndarray, action_sequence: np.ndarray, reward_sequence: np.ndarray): + states = torch.from_numpy(state_sequence).to(self._device) # (N, state_dim) + state_values = self._value_model(states) + if self._hyper_params.lmda is None: + returns = get_k_step_discounted_sums(reward_sequence, self._hyper_params.reward_decay, + self._hyper_params.k, values=state_values) + else: + returns = get_lambda_returns(reward_sequence, self._hyper_params.reward_decay, self._hyper_params.lmda, + state_values, self._hyper_params.k) # policy model training + actions = torch.from_numpy(action_sequence).to(self._device) # (N,) action_dist = F.softmax(self._policy_model(states), dim=1) # (N, num_actions) action_prob = action_dist.gather(1, actions.unsqueeze(1)) # (N, 1) log_action_prob = torch.log(action_prob).squeeze() # (N,) @@ -72,9 +85,9 @@ def train(self, states, actions, returns): value_loss.backward() self._value_optimizer.step() - def load_trainable_models(self, policy_model, value_model): - self._policy_model = policy_model - self._value_model = value_model + def load_trainable_models(self, model_dict): + self._policy_model = model_dict["policy"] + self._value_model = model_dict["value"] def dump_trainable_models(self): return {"policy": self._policy_model, "value": self._value_model} diff --git a/maro/rl/shaping/k_step_experience_shaper.py b/maro/rl/shaping/k_step_experience_shaper.py index c1782ac68..88ae1ab1b 100644 --- a/maro/rl/shaping/k_step_experience_shaper.py +++ b/maro/rl/shaping/k_step_experience_shaper.py @@ -14,7 +14,6 @@ class KStepExperienceKeys(Enum): STATE = "state" ACTION = "action" REWARD = "reward" - RETURN = "return" NEXT_STATE = "next_state" NEXT_ACTION = "next_action" DISCOUNT = "discount" @@ -37,12 +36,11 @@ def __init__(self, reward_func: Callable, reward_decay: float, steps: int, is_pe def __call__(self, trajectory, snapshot_list): length = len(trajectory) - agent_ids = np.asarray(trajectory.get_by_key["agent_id"])[:-1] + agent_ids = np.asarray(trajectory.get_by_key["agent_id"]) states = np.asarray(trajectory.get_by_key["state"]) actions = np.asarray(trajectory.get_by_key["action"]) - reward_array = np.fromiter(map(self._reward_func, trajectory.get_by_key("metrics")[:-1]), dtype=np.float32) + reward_array = np.fromiter(map(self._reward_func, trajectory.get_by_key("metrics")), dtype=np.float32) reward_sums = get_k_step_discounted_sums(reward_array, self._reward_decay, k=self._steps)[:-1] - returns = get_k_step_discounted_sums(reward_array, self._reward_decay)[:-1] discounts = np.array([self._reward_decay ** min(self._steps, length-i-1) for i in range(length-1)]) next_states = np.pad(states[self._steps:], (0, length-self._steps-1), mode="edge") next_actions = np.pad(actions[self._steps:], (0, length-self._steps-1), mode="edge") @@ -53,7 +51,6 @@ def __call__(self, trajectory, snapshot_list): return {agent_id: {KStepExperienceKeys.STATE.value: states[agent_ids == agent_id], KStepExperienceKeys.ACTION.value: actions[agent_ids == agent_id], KStepExperienceKeys.REWARD.value: reward_sums[agent_ids == agent_id], - KStepExperienceKeys.RETURN.value: returns[agent_ids == agent_id], KStepExperienceKeys.NEXT_STATE.value: next_states[agent_ids == agent_id], KStepExperienceKeys.NEXT_ACTION.value: next_actions[agent_ids == agent_id], KStepExperienceKeys.DISCOUNT.value: discounts[agent_ids == agent_id]} @@ -62,7 +59,6 @@ def __call__(self, trajectory, snapshot_list): return {KStepExperienceKeys.STATE.value: states, KStepExperienceKeys.ACTION.value: actions, KStepExperienceKeys.REWARD.value: reward_sums, - KStepExperienceKeys.RETURN.value: returns, KStepExperienceKeys.NEXT_STATE.value: next_states, KStepExperienceKeys.NEXT_ACTION.value: next_actions, KStepExperienceKeys.DISCOUNT.value: discounts} diff --git a/maro/rl/utils/trajectory_utils.py b/maro/rl/utils/trajectory_utils.py index 24ba99d2f..84df8f5cf 100644 --- a/maro/rl/utils/trajectory_utils.py +++ b/maro/rl/utils/trajectory_utils.py @@ -6,7 +6,7 @@ import numpy as np -def get_k_step_discounted_sums(rewards: np.ndrewardsay, discount: float, k: int = -1, values: np.ndrewardsay = None): +def get_k_step_discounted_sums(rewards: np.ndarray, discount: float, k: int = -1, values: np.ndarray = None): assert values is None or len(rewards) == len(values), "rewards and values should have the same length" if values is not None: rewards[-1] = values[-1] @@ -17,28 +17,28 @@ def get_k_step_discounted_sums(rewards: np.ndrewardsay, discount: float, k: int np.pad(values[k:], (0, k)) if values is not None else np.zeros(len(rewards))) -def get_lambda_returns(rewards: np.ndrewardsay, discount: float, lda: float, values: np.ndrewardsay = None, +def get_lambda_returns(rewards: np.ndarray, discount: float, lmda: float, values: np.ndarray = None, horizon: int = -1): if horizon < 0: horizon = len(rewards) - 1 horizon = min(horizon, len(rewards) - 1) - pre_truncate = reduce(lambda x, y: x*lda + y, + pre_truncate = reduce(lambda x, y: x*lmda + y, [get_k_step_discounted_sums(rewards, discount, k=k, values=values) for k in range(horizon-1, 0, -1)]) - post_truncate = get_k_step_discounted_sums(rewards, discount, k=horizon, values=values) * lda**(horizon-1) - return (1 - lda) * pre_truncate + post_truncate + post_truncate = get_k_step_discounted_sums(rewards, discount, k=horizon, values=values) * lmda**(horizon-1) + return (1 - lmda) * pre_truncate + post_truncate -b = np.asrewardsay([4, 7, 1, 3, 6]) -rw = np.asrewardsay([3, 2, 4, 1, 5]) +b = np.asarray([4, 7, 1, 3, 6]) +rw = np.asarray([3, 2, 4, 1, 5]) ld = 0.6 gamma = 0.8 steps = 4 hrz = 3 -print(get_lambda_returns(rw, gamma, ld, values=b, horizon=hrz)) +print(get_k_step_discounted_sums(rw, gamma, k=3)) """ 2-step: [5.24 7.12 8.64 5.8 6. ] From b9d39a0e7d6e01e456a57c568cc6c266791195c6 Mon Sep 17 00:00:00 2001 From: ysqyang Date: Wed, 14 Oct 2020 00:21:10 +0800 Subject: [PATCH 004/337] renamed pg to a2c and added docstrings --- maro/rl/algorithms/torch/{pg.py => a2c.py} | 55 ++++++++++---- maro/rl/algorithms/torch/abs_algorithm.py | 81 --------------------- maro/rl/algorithms/torch/dqn.py | 2 +- maro/rl/shaping/k_step_experience_shaper.py | 4 +- maro/rl/utils/trajectory_utils.py | 45 +++++++++--- 5 files changed, 79 insertions(+), 108 deletions(-) rename maro/rl/algorithms/torch/{pg.py => a2c.py} (63%) delete mode 100644 maro/rl/algorithms/torch/abs_algorithm.py diff --git a/maro/rl/algorithms/torch/pg.py b/maro/rl/algorithms/torch/a2c.py similarity index 63% rename from maro/rl/algorithms/torch/pg.py rename to maro/rl/algorithms/torch/a2c.py index 6161085e1..60ab39c36 100644 --- a/maro/rl/algorithms/torch/pg.py +++ b/maro/rl/algorithms/torch/a2c.py @@ -1,42 +1,61 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT license. +from typing import Callable + import numpy as np import torch import torch.nn as nn import torch.nn.functional as F from maro.rl.algorithms.abs_algorithm import AbsAlgorithm -from maro.rl.utils.trajectory_utils import get_k_step_discounted_sums, get_lambda_returns +from maro.rl.utils.trajectory_utils import get_k_step_returns, get_lambda_returns -class PolicyGradientHyperParameters: - """PG hyper-parameters. +class ActorCriticHyperParameters: + """Hyper-parameter set for the Actor-Critic algorithm. Args: num_actions (int): number of possible actions reward_decay (float): reward decay as defined in standard RL terminology - k (int): number of time steps used in computing bootstrapped return estimates. + k (int): number of time steps used in computing returns or return estimates. Defaults to -1, in which case + rewards are accumulated until the end of the trajectory. lmda (float): lambda coefficient used in computing lambda returns. If it is not None, ``k`` will be used as the roll-out horizon in computing truncated lambda returns. Otherwise, ``k`` will be used as usual in - computing multi-step bootstrapped return estimates. Defaults to None. + computing returns or multi-step bootstrapped return estimates. Defaults to None. """ __slots__ = ["num_actions", "reward_decay", "k", "lmda"] - def __init__(self, num_actions: int, reward_decay: float, k: int = 1, lmda: float = None): + def __init__(self, num_actions: int, reward_decay: float, k: int = -1, lmda: float = None): self.num_actions = num_actions self.reward_decay = reward_decay self.k = k self.lmda = lmda -class PolicyGradient(AbsAlgorithm): - """Vanilla policy gradient algorithm. +class ActorCritic(AbsAlgorithm): + """Actor Critic algorithm. + + The Actor-Critic algorithm base on the policy gradient theorem and with REINFORCE and REINFORCE with baseline as + special cases. + + Args: + policy_model (nn.Module): model for generating actions given states. + policy_optimizer_cls: torch optimizer class for the policy model. + policy_optimizer_params: parameters required for the policy optimizer class. + hyper_params: hyper-parameter set for the AC algorithm. + value_model (nn.Module): model for estimating state values. If None, the sequences passed to ``train()`` are + assumed to end with a terminal state with value zero. If this and the lmda hyper-parameter are both None, + the returns computed in ``train`` are actual full returns and the algorithm reduces to REINFORCE. Defaults + to None. + value_optimizer_cls: torch optimizer class for the value model. Defaults to None. + value_optimizer_params: parameters required for the value optimizer class. Defaults to None. + value_loss_func (Callable): loss function for the value model. """ def __init__(self, policy_model: nn.Module, policy_optimizer_cls, policy_optimizer_params, - hyper_params: PolicyGradientHyperParameters, value_model: nn.Module = None, - value_optimizer_cls=None, value_optimizer_params=None, value_loss_func=None): + hyper_params: ActorCriticHyperParameters, value_model: nn.Module = None, + value_optimizer_cls=None, value_optimizer_params=None, value_loss_func: Callable = None): super().__init__() self._device = torch.device("cuda" if torch.cuda.is_available() else "cpu") self._policy_model = policy_model.to(self._device) @@ -62,13 +81,19 @@ def choose_action(self, state: np.ndarray, epsilon: float = None): def train(self, state_sequence: np.ndarray, action_sequence: np.ndarray, reward_sequence: np.ndarray): states = torch.from_numpy(state_sequence).to(self._device) # (N, state_dim) - state_values = self._value_model(states) + + if self._value_model is None: + state_values = state_values_numpy = None + else: + state_values = self._value_model(states) + state_values_numpy = state_values.numpy() + if self._hyper_params.lmda is None: - returns = get_k_step_discounted_sums(reward_sequence, self._hyper_params.reward_decay, - self._hyper_params.k, values=state_values) + returns = get_k_step_returns(reward_sequence, self._hyper_params.reward_decay, + k=self._hyper_params.k, values=state_values_numpy) else: returns = get_lambda_returns(reward_sequence, self._hyper_params.reward_decay, self._hyper_params.lmda, - state_values, self._hyper_params.k) + values=state_values_numpy, truncate_steps=self._hyper_params.k) # policy model training actions = torch.from_numpy(action_sequence).to(self._device) # (N,) action_dist = F.softmax(self._policy_model(states), dim=1) # (N, num_actions) @@ -80,7 +105,7 @@ def train(self, state_sequence: np.ndarray, action_sequence: np.ndarray, reward_ self._policy_optimizer.step() # value model training (if a value model is present) if self._value_model is not None: - value_loss = self._value_loss_func(self._value_model(states), returns) + value_loss = self._value_loss_func(state_values, returns) self._value_optimizer.zero_grad() value_loss.backward() self._value_optimizer.step() diff --git a/maro/rl/algorithms/torch/abs_algorithm.py b/maro/rl/algorithms/torch/abs_algorithm.py deleted file mode 100644 index ec58585c7..000000000 --- a/maro/rl/algorithms/torch/abs_algorithm.py +++ /dev/null @@ -1,81 +0,0 @@ -# Copyright (c) Microsoft Corporation. -# Licensed under the MIT license. - -from abc import ABC, abstractmethod -import itertools -from typing import Union - - -class AbsAlgorithm(ABC): - """Abstract RL algorithm class. - - The class provides uniform policy interfaces such as ``choose_action`` and ``train``. We also provide some - predefined RL algorithm based on it, such DQN, A2C, etc. User can inherit from it to customize their own - algorithms. - - Args: - model_dict (dict): Underlying models for the algorithm (e.g., for A2C, model_dict could be something like - {"actor": ..., "critic": ...}) - optimizer_opt (tuple or dict): Tuple or dict of tuples of (optimizer_class, optimizer_params) associated - with the models in model_dict. If it is a tuple, the optimizer to be instantiated applies to all - trainable parameters from ``model_dict``. If it is a dict, the optimizer will be applied to the related - model with the same key. - loss_func_dict (dict): Loss function types associated with the models. - hyper_params (object): Algorithm-specific hyper-parameter set. - """ - def __init__(self, model_dict: dict, optimizer_opt: Union[dict, tuple], loss_func_dict: dict, hyper_params: object): - self._loss_func_dict = loss_func_dict - self._hyper_params = hyper_params - self._model_dict = model_dict - self._register_optimizers(optimizer_opt) - - def _register_optimizers(self, optimizer_opt): - if isinstance(optimizer_opt, tuple): - # If a single optimizer_opt tuple is provided, a single optimizer will be created to jointly - # optimize all model parameters involved in the algorithm. - optim_cls, optim_params = optimizer_opt - model_params = [model.parameters() for model in self._model_dict.values()] - self._optimizer = optim_cls(itertools.chain(*model_params), **optim_params) - else: - self._optimizer = {} - for model_key, model in self._model_dict.items(): - # No gradient required - if model_key not in optimizer_opt or optimizer_opt[model_key] is None: - self._model_dict[model_key].eval() - self._optimizer[model_key] = None - else: - optim_cls, optim_params = optimizer_opt[model_key] - self._optimizer[model_key] = optim_cls(model.parameters(), **optim_params) - - @property - def model_dict(self): - return self._model_dict - - @model_dict.setter - def model_dict(self, model_dict): - self._model_dict = model_dict - - @abstractmethod - def train(self, *args, **kwargs): - """Train models using samples. - - This method is algorithm-specific and needs to be implemented by the user. For example, for the DQN - algorithm, this may look like train(self, state, action, reward, next_state). - """ - return NotImplementedError - - @abstractmethod - def choose_action(self, state, epsilon: float = None): - """This method uses the underlying model(s) to compute an action from a shaped state. - - Args: - state: A state object shaped by a ``StateShaper`` to conform to the model input format. - epsilon (float, optional): Exploration rate. For greedy value-based algorithms, this being None means - using the model output without exploration. For algorithms with inherently stochastic policies such - as policy gradient, this is usually ignored. Defaults to None. - - Returns: - The action to be taken given ``state``. It is usually necessary to use an ``ActionShaper`` to convert - this to an environment executable action. - """ - return NotImplementedError diff --git a/maro/rl/algorithms/torch/dqn.py b/maro/rl/algorithms/torch/dqn.py index c24c60f3a..355c31669 100644 --- a/maro/rl/algorithms/torch/dqn.py +++ b/maro/rl/algorithms/torch/dqn.py @@ -10,7 +10,7 @@ class DQNHyperParams: - """DQN hyper-parameters. + """Hyper-parameter set for the DQN algorithm. Args: num_actions (int): number of possible actions diff --git a/maro/rl/shaping/k_step_experience_shaper.py b/maro/rl/shaping/k_step_experience_shaper.py index 88ae1ab1b..33e5a7428 100644 --- a/maro/rl/shaping/k_step_experience_shaper.py +++ b/maro/rl/shaping/k_step_experience_shaper.py @@ -7,7 +7,7 @@ import numpy as np from .experience_shaper import ExperienceShaper -from maro.rl.utils.trajectory_utils import get_k_step_discounted_sums +from maro.rl.utils.trajectory_utils import get_k_step_returns class KStepExperienceKeys(Enum): @@ -40,7 +40,7 @@ def __call__(self, trajectory, snapshot_list): states = np.asarray(trajectory.get_by_key["state"]) actions = np.asarray(trajectory.get_by_key["action"]) reward_array = np.fromiter(map(self._reward_func, trajectory.get_by_key("metrics")), dtype=np.float32) - reward_sums = get_k_step_discounted_sums(reward_array, self._reward_decay, k=self._steps)[:-1] + reward_sums = get_k_step_returns(reward_array, self._reward_decay, k=self._steps)[:-1] discounts = np.array([self._reward_decay ** min(self._steps, length-i-1) for i in range(length-1)]) next_states = np.pad(states[self._steps:], (0, length-self._steps-1), mode="edge") next_actions = np.pad(actions[self._steps:], (0, length-self._steps-1), mode="edge") diff --git a/maro/rl/utils/trajectory_utils.py b/maro/rl/utils/trajectory_utils.py index 84df8f5cf..8c14cd44a 100644 --- a/maro/rl/utils/trajectory_utils.py +++ b/maro/rl/utils/trajectory_utils.py @@ -6,7 +6,20 @@ import numpy as np -def get_k_step_discounted_sums(rewards: np.ndarray, discount: float, k: int = -1, values: np.ndarray = None): +def get_k_step_returns(rewards: np.ndarray, discount: float, k: int = -1, values: np.ndarray = None): + """Compute K-step returns given reward and value sequences. + Args: + rewards (np.ndarray): reward sequence from a trajectory. + discount (float): reward discount as in standard RL. + k (int): number of steps in computing returns. If it is -1, returns are computed using the largest possible + number of steps. Defaults to -1. + values (np.ndarray): sequence of values for the traversed states in a trajectory. If it is None, the state + immediately after the final state in the given sequence is assumed to be terminal with value zero, and the + computed returns for k = -1 are actual full returns. Defaults to None. + + Returns: + An ndarray containing the k-step returns for each time step. + """ assert values is None or len(rewards) == len(values), "rewards and values should have the same length" if values is not None: rewards[-1] = values[-1] @@ -18,16 +31,30 @@ def get_k_step_discounted_sums(rewards: np.ndarray, discount: float, k: int = -1 def get_lambda_returns(rewards: np.ndarray, discount: float, lmda: float, values: np.ndarray = None, - horizon: int = -1): - if horizon < 0: - horizon = len(rewards) - 1 + truncate_steps: int = -1): + """Compute lambda returns given reward and value sequences and a truncate_steps. + Args: + rewards (np.ndarray): reward sequence from a trajectory. + discount (float): reward discount as in standard RL. + lmda (float): the lambda coefficient involved in computing lambda returns. + values (np.ndarray): sequence of values for the traversed states in a trajectory. If it is None, the state + immediately after the final state in the given sequence is assumed to be terminal with value zero. + Defaults to None. + truncate_steps (int): number of steps where the lambda return series is truncated. If it is -1, no truncating + is done and the lambda return is carried out to the end of the sequence. Defaults to -1. + + Returns: + An ndarray containing the lambda returns for each time step. + """ + if truncate_steps < 0: + truncate_steps = len(rewards) - 1 - horizon = min(horizon, len(rewards) - 1) + truncate_steps = min(truncate_steps, len(rewards) - 1) pre_truncate = reduce(lambda x, y: x*lmda + y, - [get_k_step_discounted_sums(rewards, discount, k=k, values=values) - for k in range(horizon-1, 0, -1)]) + [get_k_step_returns(rewards, discount, k=k, values=values) + for k in range(truncate_steps-1, 0, -1)]) - post_truncate = get_k_step_discounted_sums(rewards, discount, k=horizon, values=values) * lmda**(horizon-1) + post_truncate = get_k_step_returns(rewards, discount, k=truncate_steps, values=values) * lmda**(truncate_steps-1) return (1 - lmda) * pre_truncate + post_truncate @@ -38,7 +65,7 @@ def get_lambda_returns(rewards: np.ndarray, discount: float, lmda: float, values steps = 4 hrz = 3 -print(get_k_step_discounted_sums(rw, gamma, k=3)) +print(get_lambda_returns(rw, gamma, ld, values=None, truncate_steps=3)) """ 2-step: [5.24 7.12 8.64 5.8 6. ] From 62c13fe467a7379174a8e8c31f8d5f8fb7c5381f Mon Sep 17 00:00:00 2001 From: ysqyang Date: Wed, 14 Oct 2020 09:59:16 +0800 Subject: [PATCH 005/337] refined docstring and added unittest for trajectory utils --- maro/rl/agent/abs_agent_manager.py | 9 ++----- maro/rl/algorithms/torch/{a2c.py => ac.py} | 20 +++++++-------- maro/rl/algorithms/torch/dqn.py | 18 +++++++++---- maro/rl/utils/trajectory_utils.py | 30 ++++++++++++---------- tests/test_trajectory_utils.py | 29 +++++++++++++++++++++ 5 files changed, 71 insertions(+), 35 deletions(-) rename maro/rl/algorithms/torch/{a2c.py => ac.py} (89%) create mode 100644 tests/test_trajectory_utils.py diff --git a/maro/rl/agent/abs_agent_manager.py b/maro/rl/agent/abs_agent_manager.py index 4a5925491..e9ac1bc62 100644 --- a/maro/rl/agent/abs_agent_manager.py +++ b/maro/rl/agent/abs_agent_manager.py @@ -33,7 +33,7 @@ class AbsAgentManager(ABC): Args: name (str): Name of agent manager. mode (AgentMode): An AgentMode enum member that specifies that role of the agent. Some attributes may - be None under certain modes. + be None under certain modes. agent_id_list (list): List of agent identifiers. experience_shaper (ExperienceShaper, optional): It is responsible for processing data in the replay buffer at the end of an episode. @@ -50,9 +50,7 @@ def __init__(self, state_shaper: StateShaper = None, action_shaper: ActionShaper = None, experience_shaper: ExperienceShaper = None, - explorer: AbsExplorer = None, - on_event_callback: Callable = None, - on_feedback_call_back: Callable = None): + explorer: AbsExplorer = None): self._name = name if mode not in AgentMode: raise UnsupportedAgentModeError(msg='mode must be "train", "inference" or "train_inference"') @@ -75,9 +73,6 @@ def __init__(self, self._transition_cache = {} self._trajectory = ColumnBasedStore() - self._on_event_callback = on_event_callback - self._on_feedback_callback = on_feedback_call_back - self._agent_dict = {} self._assemble(self._agent_dict) diff --git a/maro/rl/algorithms/torch/a2c.py b/maro/rl/algorithms/torch/ac.py similarity index 89% rename from maro/rl/algorithms/torch/a2c.py rename to maro/rl/algorithms/torch/ac.py index 60ab39c36..883590ec8 100644 --- a/maro/rl/algorithms/torch/a2c.py +++ b/maro/rl/algorithms/torch/ac.py @@ -20,13 +20,12 @@ class ActorCriticHyperParameters: reward_decay (float): reward decay as defined in standard RL terminology k (int): number of time steps used in computing returns or return estimates. Defaults to -1, in which case rewards are accumulated until the end of the trajectory. - lmda (float): lambda coefficient used in computing lambda returns. If it is not None, ``k`` will be used as - the roll-out horizon in computing truncated lambda returns. Otherwise, ``k`` will be used as usual in - computing returns or multi-step bootstrapped return estimates. Defaults to None. + lmda (float): lambda coefficient used in computing lambda returns. Defaults to 1.0, in which case the usual + k-step return is computed. """ __slots__ = ["num_actions", "reward_decay", "k", "lmda"] - def __init__(self, num_actions: int, reward_decay: float, k: int = -1, lmda: float = None): + def __init__(self, num_actions: int, reward_decay: float, k: int = -1, lmda: float = 1.0): self.num_actions = num_actions self.reward_decay = reward_decay self.k = k @@ -36,18 +35,16 @@ def __init__(self, num_actions: int, reward_decay: float, k: int = -1, lmda: flo class ActorCritic(AbsAlgorithm): """Actor Critic algorithm. - The Actor-Critic algorithm base on the policy gradient theorem and with REINFORCE and REINFORCE with baseline as - special cases. + The Actor-Critic algorithm base on the policy gradient theorem. If no value model is provided and hyper-parameter + ``k`` is -1, the algorithm reduces to REINFORCE. Args: policy_model (nn.Module): model for generating actions given states. policy_optimizer_cls: torch optimizer class for the policy model. policy_optimizer_params: parameters required for the policy optimizer class. hyper_params: hyper-parameter set for the AC algorithm. - value_model (nn.Module): model for estimating state values. If None, the sequences passed to ``train()`` are - assumed to end with a terminal state with value zero. If this and the lmda hyper-parameter are both None, - the returns computed in ``train`` are actual full returns and the algorithm reduces to REINFORCE. Defaults - to None. + value_model (nn.Module): model for estimating state values. If this is None, the sequences passed to ``train()`` + are assumed to end with a terminal state with value zero. Defaults to None. value_optimizer_cls: torch optimizer class for the value model. Defaults to None. value_optimizer_params: parameters required for the value optimizer class. Defaults to None. value_loss_func (Callable): loss function for the value model. @@ -66,6 +63,9 @@ def __init__(self, policy_model: nn.Module, policy_optimizer_cls, policy_optimiz "value_optimizer_cls and value_optimizer_params should not be None if value model is not None" self._value_optimizer = value_optimizer_cls(self._value_model.parameters(), **value_optimizer_params) else: + assert hyper_params.k == -1 and hyper_params.lmda == 1.0, \ + "if not value model is provided, hyper-parameters k and lmda must be set to defaults of -1 and 1.0, " \ + "respectively. " self._value_optimizer = None self._value_loss_func = value_loss_func self._hyper_params = hyper_params diff --git a/maro/rl/algorithms/torch/dqn.py b/maro/rl/algorithms/torch/dqn.py index 355c31669..deb99f1d5 100644 --- a/maro/rl/algorithms/torch/dqn.py +++ b/maro/rl/algorithms/torch/dqn.py @@ -30,20 +30,28 @@ def __init__(self, num_actions: int, reward_decay: float, num_training_rounds_pe class DQN(AbsAlgorithm): """The Deep-Q-Networks algorithm. - The model must contain the key `eval`. Optionally a model corresponding to the key `target` can be - provided. If the key `target` is absent or model[`target`] is None, the target model will be a deep - copy of the provided eval model. + See https://web.stanford.edu/class/psych209/Readings/MnihEtAlHassibis15NatureControlDeepRL.pdf for details. + + Args: + eval_model (nn.Module): trainable Q-value model for computing actions given states. + optimizer_cls: torch optimizer class for the eval model. + optimizer_params: parameters required for the eval optimizer class. + loss_func (Callable): loss function for the value model. + hyper_params: hyper-parameter set for the DQN algorithm. + target_model (nn.Module): Q-value model to train the ``eval_model`` against and to be updated periodically. If + it is None, the target model will be initialized as a deep copy of the eval model. """ def __init__(self, eval_model: nn.Module, optimizer_cls, optimizer_params, loss_func, hyper_params: DQNHyperParams, target_model: nn.Module = None): super().__init__() - self._eval_model = eval_model + self._device = torch.device("cuda" if torch.cuda.is_available() else "cpu") + self._eval_model = eval_model.to(self._device) self._target_model = clone(eval_model) if target_model is None else target_model + self._target_model = self._target_model.to(self._device) self._optimizer = optimizer_cls(self._eval_model.parameters(), **optimizer_params) self._loss_func = loss_func self._hyper_params = hyper_params self._train_cnt = 0 - self._device = torch.device("cuda" if torch.cuda.is_available() else "cpu") @property def eval_model(self): diff --git a/maro/rl/utils/trajectory_utils.py b/maro/rl/utils/trajectory_utils.py index 8c14cd44a..16285b7a4 100644 --- a/maro/rl/utils/trajectory_utils.py +++ b/maro/rl/utils/trajectory_utils.py @@ -49,6 +49,14 @@ def get_lambda_returns(rewards: np.ndarray, discount: float, lmda: float, values if truncate_steps < 0: truncate_steps = len(rewards) - 1 + # If lambda is zero, lambda return reduces to one-step return + if lmda == .0: + return get_k_step_returns(rewards, discount, k=1, values=values) + + # If lambda is one, lambda return reduces to maximum-step return + if lmda == 1.0: + return get_k_step_returns(rewards, discount, k=truncate_steps, values=values) + truncate_steps = min(truncate_steps, len(rewards) - 1) pre_truncate = reduce(lambda x, y: x*lmda + y, [get_k_step_returns(rewards, discount, k=k, values=values) @@ -58,18 +66,14 @@ def get_lambda_returns(rewards: np.ndarray, discount: float, lmda: float, values return (1 - lmda) * pre_truncate + post_truncate -b = np.asarray([4, 7, 1, 3, 6]) rw = np.asarray([3, 2, 4, 1, 5]) +vals = np.asarray([4, 7, 1, 3, 6]) ld = 0.6 -gamma = 0.8 -steps = 4 -hrz = 3 - -print(get_lambda_returns(rw, gamma, ld, values=None, truncate_steps=3)) - -""" -2-step: [5.24 7.12 8.64 5.8 6. ] -1-step: [8.6 2.8 6.4 5.8 6. ] -3-step: [8.696 8.912 8.64 5.8 6. ] -[7.82816 6.03712 7.744 5.8 6. ] -""" +discount = 0.8 +k = 4 +truncate_steps = 3 + +print(get_k_step_returns(rw, discount, k=k, values=vals)) +print(get_lambda_returns(rw, discount, ld, values=vals, truncate_steps=truncate_steps)) + + diff --git a/tests/test_trajectory_utils.py b/tests/test_trajectory_utils.py new file mode 100644 index 000000000..c79bfe1f6 --- /dev/null +++ b/tests/test_trajectory_utils.py @@ -0,0 +1,29 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import unittest + +import numpy as np + +from maro.rl.utils.trajectory_utils import get_k_step_returns, get_lambda_returns + + +class TestUnboundedStore(unittest.TestCase): + def setUp(self) -> None: + self.rewards = np.asarray([3, 2, 4, 1, 5]) + self.values = np.asarray([4, 7, 1, 3, 6]) + self.lmda = 0.6 + self.discount = 0.8 + self.k = 4 + self.truncate_steps = 3 + + def test_k_step_return(self): + returns = get_k_step_returns(self.rewards, self.discount, k=self.k, values=self.values) + expected = np.asarray([10.1296, 8.912, 8.64, 5.8, 6.0]) + self.assertEqual(returns, expected, msg=f"expected {expected}, got {returns}") + + def test_lambda_return(self): + returns = get_lambda_returns(self.rewards, self.discount, self.lmda, values=None, + truncate_steps=self.truncate_steps) + expected = np.asarray([7.82816, 6.03712, 7.744, 5.8, 6.0]) + self.assertEqual(returns, expected, msg=f"expected {expected}, got {returns}") From 0c3b99ffa24652e3794a7561e958ba9453f9a101 Mon Sep 17 00:00:00 2001 From: ysqyang Date: Wed, 14 Oct 2020 10:08:27 +0800 Subject: [PATCH 006/337] fixed ut bugs --- tests/test_trajectory_utils.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/tests/test_trajectory_utils.py b/tests/test_trajectory_utils.py index c79bfe1f6..4b90cf101 100644 --- a/tests/test_trajectory_utils.py +++ b/tests/test_trajectory_utils.py @@ -20,10 +20,14 @@ def setUp(self) -> None: def test_k_step_return(self): returns = get_k_step_returns(self.rewards, self.discount, k=self.k, values=self.values) expected = np.asarray([10.1296, 8.912, 8.64, 5.8, 6.0]) - self.assertEqual(returns, expected, msg=f"expected {expected}, got {returns}") + self.assertTrue(np.array_equal(returns, expected), msg=f"expected {expected}, got {returns}") def test_lambda_return(self): returns = get_lambda_returns(self.rewards, self.discount, self.lmda, values=None, truncate_steps=self.truncate_steps) expected = np.asarray([7.82816, 6.03712, 7.744, 5.8, 6.0]) - self.assertEqual(returns, expected, msg=f"expected {expected}, got {returns}") + self.assertTrue(np.array_equal(returns, expected), msg=f"expected {expected}, got {returns}") + + +if __name__ == "__main__": + unittest.main() From 2a475f781694399d702630b7f964dc4641aa68a1 Mon Sep 17 00:00:00 2001 From: ysqyang Date: Wed, 14 Oct 2020 10:24:17 +0800 Subject: [PATCH 007/337] 1.renamed truncate_steps to k for consistency; 2.modified unittest --- maro/rl/algorithms/torch/ac.py | 2 +- maro/rl/utils/trajectory_utils.py | 25 ++++++++++++------------- tests/test_trajectory_utils.py | 10 ++++------ 3 files changed, 17 insertions(+), 20 deletions(-) diff --git a/maro/rl/algorithms/torch/ac.py b/maro/rl/algorithms/torch/ac.py index 883590ec8..86a2e9f33 100644 --- a/maro/rl/algorithms/torch/ac.py +++ b/maro/rl/algorithms/torch/ac.py @@ -93,7 +93,7 @@ def train(self, state_sequence: np.ndarray, action_sequence: np.ndarray, reward_ k=self._hyper_params.k, values=state_values_numpy) else: returns = get_lambda_returns(reward_sequence, self._hyper_params.reward_decay, self._hyper_params.lmda, - values=state_values_numpy, truncate_steps=self._hyper_params.k) + k=self._hyper_params.k, values=state_values_numpy) # policy model training actions = torch.from_numpy(action_sequence).to(self._device) # (N,) action_dist = F.softmax(self._policy_model(states), dim=1) # (N, num_actions) diff --git a/maro/rl/utils/trajectory_utils.py b/maro/rl/utils/trajectory_utils.py index 16285b7a4..b36503b6e 100644 --- a/maro/rl/utils/trajectory_utils.py +++ b/maro/rl/utils/trajectory_utils.py @@ -30,24 +30,23 @@ def get_k_step_returns(rewards: np.ndarray, discount: float, k: int = -1, values np.pad(values[k:], (0, k)) if values is not None else np.zeros(len(rewards))) -def get_lambda_returns(rewards: np.ndarray, discount: float, lmda: float, values: np.ndarray = None, - truncate_steps: int = -1): - """Compute lambda returns given reward and value sequences and a truncate_steps. +def get_lambda_returns(rewards: np.ndarray, discount: float, lmda: float, k: int = -1, values: np.ndarray = None): + """Compute lambda returns given reward and value sequences and a k. Args: rewards (np.ndarray): reward sequence from a trajectory. discount (float): reward discount as in standard RL. lmda (float): the lambda coefficient involved in computing lambda returns. + k (int): number of steps where the lambda return series is truncated. If it is -1, no truncating is done and + the lambda return is carried out to the end of the sequence. Defaults to -1. values (np.ndarray): sequence of values for the traversed states in a trajectory. If it is None, the state immediately after the final state in the given sequence is assumed to be terminal with value zero. Defaults to None. - truncate_steps (int): number of steps where the lambda return series is truncated. If it is -1, no truncating - is done and the lambda return is carried out to the end of the sequence. Defaults to -1. Returns: An ndarray containing the lambda returns for each time step. """ - if truncate_steps < 0: - truncate_steps = len(rewards) - 1 + if k < 0: + k = len(rewards) - 1 # If lambda is zero, lambda return reduces to one-step return if lmda == .0: @@ -55,14 +54,14 @@ def get_lambda_returns(rewards: np.ndarray, discount: float, lmda: float, values # If lambda is one, lambda return reduces to maximum-step return if lmda == 1.0: - return get_k_step_returns(rewards, discount, k=truncate_steps, values=values) + return get_k_step_returns(rewards, discount, k=k, values=values) - truncate_steps = min(truncate_steps, len(rewards) - 1) + k = min(k, len(rewards) - 1) pre_truncate = reduce(lambda x, y: x*lmda + y, [get_k_step_returns(rewards, discount, k=k, values=values) - for k in range(truncate_steps-1, 0, -1)]) + for k in range(k-1, 0, -1)]) - post_truncate = get_k_step_returns(rewards, discount, k=truncate_steps, values=values) * lmda**(truncate_steps-1) + post_truncate = get_k_step_returns(rewards, discount, k=k, values=values) * lmda**(k-1) return (1 - lmda) * pre_truncate + post_truncate @@ -71,9 +70,9 @@ def get_lambda_returns(rewards: np.ndarray, discount: float, lmda: float, values ld = 0.6 discount = 0.8 k = 4 -truncate_steps = 3 + print(get_k_step_returns(rw, discount, k=k, values=vals)) -print(get_lambda_returns(rw, discount, ld, values=vals, truncate_steps=truncate_steps)) +print(get_lambda_returns(rw, discount, ld, k=k, values=vals)) diff --git a/tests/test_trajectory_utils.py b/tests/test_trajectory_utils.py index 4b90cf101..08402c086 100644 --- a/tests/test_trajectory_utils.py +++ b/tests/test_trajectory_utils.py @@ -15,18 +15,16 @@ def setUp(self) -> None: self.lmda = 0.6 self.discount = 0.8 self.k = 4 - self.truncate_steps = 3 def test_k_step_return(self): returns = get_k_step_returns(self.rewards, self.discount, k=self.k, values=self.values) expected = np.asarray([10.1296, 8.912, 8.64, 5.8, 6.0]) - self.assertTrue(np.array_equal(returns, expected), msg=f"expected {expected}, got {returns}") + np.testing.assert_allclose(returns, expected, rtol=1e-4) def test_lambda_return(self): - returns = get_lambda_returns(self.rewards, self.discount, self.lmda, values=None, - truncate_steps=self.truncate_steps) - expected = np.asarray([7.82816, 6.03712, 7.744, 5.8, 6.0]) - self.assertTrue(np.array_equal(returns, expected), msg=f"expected {expected}, got {returns}") + returns = get_lambda_returns(self.rewards, self.discount, self.lmda, k=self.k, values=self.values) + expected = np.asarray([8.1378176, 6.03712, 7.744, 5.8, 6.0]) + np.testing.assert_allclose(returns, expected, rtol=1e-4) if __name__ == "__main__": From 20e659327d63cc661ee9a1518cd08b4f5e786f77 Mon Sep 17 00:00:00 2001 From: ysqyang Date: Wed, 14 Oct 2020 10:26:23 +0800 Subject: [PATCH 008/337] fixed copy-paste error in UT --- maro/rl/utils/trajectory_utils.py | 12 ------------ tests/test_trajectory_utils.py | 2 +- 2 files changed, 1 insertion(+), 13 deletions(-) diff --git a/maro/rl/utils/trajectory_utils.py b/maro/rl/utils/trajectory_utils.py index b36503b6e..360f68b95 100644 --- a/maro/rl/utils/trajectory_utils.py +++ b/maro/rl/utils/trajectory_utils.py @@ -64,15 +64,3 @@ def get_lambda_returns(rewards: np.ndarray, discount: float, lmda: float, k: int post_truncate = get_k_step_returns(rewards, discount, k=k, values=values) * lmda**(k-1) return (1 - lmda) * pre_truncate + post_truncate - -rw = np.asarray([3, 2, 4, 1, 5]) -vals = np.asarray([4, 7, 1, 3, 6]) -ld = 0.6 -discount = 0.8 -k = 4 - - -print(get_k_step_returns(rw, discount, k=k, values=vals)) -print(get_lambda_returns(rw, discount, ld, k=k, values=vals)) - - diff --git a/tests/test_trajectory_utils.py b/tests/test_trajectory_utils.py index 08402c086..9687d87a9 100644 --- a/tests/test_trajectory_utils.py +++ b/tests/test_trajectory_utils.py @@ -8,7 +8,7 @@ from maro.rl.utils.trajectory_utils import get_k_step_returns, get_lambda_returns -class TestUnboundedStore(unittest.TestCase): +class TestTrajectoryUtils(unittest.TestCase): def setUp(self) -> None: self.rewards = np.asarray([3, 2, 4, 1, 5]) self.values = np.asarray([4, 7, 1, 3, 6]) From 85cd0274aaf9e78919264df73734da47bbc9e9e3 Mon Sep 17 00:00:00 2001 From: ysqyang Date: Wed, 14 Oct 2020 15:22:50 +0800 Subject: [PATCH 009/337] split pg and ac and added ac with shared layers --- maro/rl/algorithms/torch/ac.py | 141 ++++++++++++++++++++++----------- maro/rl/algorithms/torch/pg.py | 78 ++++++++++++++++++ 2 files changed, 172 insertions(+), 47 deletions(-) create mode 100644 maro/rl/algorithms/torch/pg.py diff --git a/maro/rl/algorithms/torch/ac.py b/maro/rl/algorithms/torch/ac.py index 86a2e9f33..37e3b0e7f 100644 --- a/maro/rl/algorithms/torch/ac.py +++ b/maro/rl/algorithms/torch/ac.py @@ -6,10 +6,9 @@ import numpy as np import torch import torch.nn as nn -import torch.nn.functional as F from maro.rl.algorithms.abs_algorithm import AbsAlgorithm -from maro.rl.utils.trajectory_utils import get_k_step_returns, get_lambda_returns +from maro.rl.utils.trajectory_utils import get_lambda_returns class ActorCriticHyperParameters: @@ -33,40 +32,36 @@ def __init__(self, num_actions: int, reward_decay: float, k: int = -1, lmda: flo class ActorCritic(AbsAlgorithm): - """Actor Critic algorithm. + """Actor Critic algorithm with separate policy and value models (no shared layers). - The Actor-Critic algorithm base on the policy gradient theorem. If no value model is provided and hyper-parameter - ``k`` is -1, the algorithm reduces to REINFORCE. + The Actor-Critic algorithm base on the policy gradient theorem. Args: policy_model (nn.Module): model for generating actions given states. + value_model (nn.Module): model for estimating state values. + value_loss_func (Callable): loss function for the value model. policy_optimizer_cls: torch optimizer class for the policy model. policy_optimizer_params: parameters required for the policy optimizer class. + value_optimizer_cls: torch optimizer class for the value model. + value_optimizer_params: parameters required for the value optimizer class. hyper_params: hyper-parameter set for the AC algorithm. - value_model (nn.Module): model for estimating state values. If this is None, the sequences passed to ``train()`` - are assumed to end with a terminal state with value zero. Defaults to None. - value_optimizer_cls: torch optimizer class for the value model. Defaults to None. - value_optimizer_params: parameters required for the value optimizer class. Defaults to None. - value_loss_func (Callable): loss function for the value model. """ - def __init__(self, policy_model: nn.Module, policy_optimizer_cls, policy_optimizer_params, - hyper_params: ActorCriticHyperParameters, value_model: nn.Module = None, - value_optimizer_cls=None, value_optimizer_params=None, value_loss_func: Callable = None): + def __init__(self, + policy_model: nn.Module, + value_model: nn.Module, + value_loss_func: Callable, + policy_optimizer_cls, + policy_optimizer_params, + value_optimizer_cls, + value_optimizer_params, + hyper_params: ActorCriticHyperParameters): super().__init__() self._device = torch.device("cuda" if torch.cuda.is_available() else "cpu") self._policy_model = policy_model.to(self._device) - self._policy_optimizer = policy_optimizer_cls(self._policy_model.parameters(), **policy_optimizer_params) self._value_model = value_model.to(self._device) - if self._value_model is not None: - assert value_optimizer_cls is not None and value_optimizer_params is not None, \ - "value_optimizer_cls and value_optimizer_params should not be None if value model is not None" - self._value_optimizer = value_optimizer_cls(self._value_model.parameters(), **value_optimizer_params) - else: - assert hyper_params.k == -1 and hyper_params.lmda == 1.0, \ - "if not value model is provided, hyper-parameters k and lmda must be set to defaults of -1 and 1.0, " \ - "respectively. " - self._value_optimizer = None + self._policy_optimizer = policy_optimizer_cls(self._policy_model.parameters(), **policy_optimizer_params) + self._value_optimizer = value_optimizer_cls(self._value_model.parameters(), **value_optimizer_params) self._value_loss_func = value_loss_func self._hyper_params = hyper_params @@ -76,39 +71,27 @@ def model(self): def choose_action(self, state: np.ndarray, epsilon: float = None): state = torch.from_numpy(state).unsqueeze(0).to(self._device) # (1, state_dim) - action_dist = F.softmax(self._policy_model(state), dim=1).squeeze() # (num_actions,) + action_dist = self._policy_model(state).squeeze() # (num_actions,) return np.random.choice(self._hyper_params.num_actions, p=action_dist.numpy()) def train(self, state_sequence: np.ndarray, action_sequence: np.ndarray, reward_sequence: np.ndarray): states = torch.from_numpy(state_sequence).to(self._device) # (N, state_dim) - - if self._value_model is None: - state_values = state_values_numpy = None - else: - state_values = self._value_model(states) - state_values_numpy = state_values.numpy() - - if self._hyper_params.lmda is None: - returns = get_k_step_returns(reward_sequence, self._hyper_params.reward_decay, - k=self._hyper_params.k, values=state_values_numpy) - else: - returns = get_lambda_returns(reward_sequence, self._hyper_params.reward_decay, self._hyper_params.lmda, - k=self._hyper_params.k, values=state_values_numpy) + state_values = self._value_model(states) + state_values_numpy = state_values.numpy() + returns = get_lambda_returns(reward_sequence, self._hyper_params.reward_decay, self._hyper_params.lmda, + k=self._hyper_params.k, values=state_values_numpy) # policy model training actions = torch.from_numpy(action_sequence).to(self._device) # (N,) - action_dist = F.softmax(self._policy_model(states), dim=1) # (N, num_actions) - action_prob = action_dist.gather(1, actions.unsqueeze(1)) # (N, 1) - log_action_prob = torch.log(action_prob).squeeze() # (N,) - policy_loss = -(log_action_prob * returns).mean() + action_prob = self._policy_model(states).gather(1, actions.unsqueeze(1)).squeeze() # (N,) + policy_loss = -(torch.log(action_prob) * returns).mean() self._policy_optimizer.zero_grad() policy_loss.backward() self._policy_optimizer.step() - # value model training (if a value model is present) - if self._value_model is not None: - value_loss = self._value_loss_func(state_values, returns) - self._value_optimizer.zero_grad() - value_loss.backward() - self._value_optimizer.step() + # value model training + value_loss = self._value_loss_func(state_values, returns) + self._value_optimizer.zero_grad() + value_loss.backward() + self._value_optimizer.step() def load_trainable_models(self, model_dict): self._policy_model = model_dict["policy"] @@ -126,3 +109,67 @@ def load_trainable_models_from_file(self, path): def dump_trainable_models_to_file(self, path: str): """Dump the algorithm's trainable models to disk.""" torch.save({"policy": self._policy_model.state_dict(), "value": self._value_model.state_dict()}, path) + + +class ActorCriticWithSharedLayers(AbsAlgorithm): + """Actor Critic algorithm where policy and value models have shared layers. + + Args: + policy_value_model (nn.Module): model for generating action distributions and values for given states using + shared bottom layers. The model, when called, must return (value, action distribution). + value_loss_func (Callable): loss function for the value model. + optimizer_cls: torch optimizer class for the policy model. + optimizer_params: parameters required for the policy optimizer class. + hyper_params: hyper-parameter set for the AC algorithm. + """ + + def __init__(self, + policy_value_model: nn.Module, + value_loss_func: Callable, + optimizer_cls, + optimizer_params, + hyper_params: ActorCriticHyperParameters): + super().__init__() + self._device = torch.device("cuda" if torch.cuda.is_available() else "cpu") + self._policy_value_model = policy_value_model.to(self._device) + self._optimizer = optimizer_cls(self._policy_value_model.parameters(), **optimizer_params) + self._value_loss_func = value_loss_func + self._hyper_params = hyper_params + + @property + def model(self): + return self._policy_value_model + + def choose_action(self, state: np.ndarray, epsilon: float = None): + state = torch.from_numpy(state).unsqueeze(0).to(self._device) # (1, state_dim) + action_dist = self._policy_value_model(state)[1].squeeze() # (num_actions,) + return np.random.choice(self._hyper_params.num_actions, p=action_dist.numpy()) + + def train(self, state_sequence: np.ndarray, action_sequence: np.ndarray, reward_sequence: np.ndarray): + states = torch.from_numpy(state_sequence).to(self._device) # (N, state_dim) + state_values, action_dist = self._policy_value_model(states) + state_values_numpy = state_values.numpy() + returns = get_lambda_returns(reward_sequence, self._hyper_params.reward_decay, self._hyper_params.lmda, + k=self._hyper_params.k, values=state_values_numpy) + actions = torch.from_numpy(action_sequence).to(self._device) # (N,) + action_prob = action_dist.gather(1, actions.unsqueeze(1)).squeeze() # (N,) + policy_loss = -(torch.log(action_prob) * returns).mean() + value_loss = self._value_loss_func(state_values, returns) + loss = policy_loss + value_loss + self._optimizer.zero_grad() + loss.backward() + self._optimizer.step() + + def load_trainable_models(self, policy_value_model): + self._policy_value_model = policy_value_model + + def dump_trainable_models(self): + return self._policy_value_model + + def load_trainable_models_from_file(self, path): + """Load trainable models from disk.""" + self._policy_value_model = torch.load(path) + + def dump_trainable_models_to_file(self, path: str): + """Dump the algorithm's trainable models to disk.""" + torch.save(self._policy_value_model.state_dict(), path) diff --git a/maro/rl/algorithms/torch/pg.py b/maro/rl/algorithms/torch/pg.py new file mode 100644 index 000000000..d8422ede8 --- /dev/null +++ b/maro/rl/algorithms/torch/pg.py @@ -0,0 +1,78 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import numpy as np +import torch +import torch.nn as nn +import torch.nn.functional as F + +from maro.rl.algorithms.abs_algorithm import AbsAlgorithm +from maro.rl.utils.trajectory_utils import get_k_step_returns + + +class PolicyGradientHyperParameters: + """Hyper-parameter set for the Actor-Critic algorithm. + + Args: + num_actions (int): number of possible actions + reward_decay (float): reward decay as defined in standard RL terminology + """ + __slots__ = ["num_actions", "reward_decay"] + + def __init__(self, num_actions: int, reward_decay: float): + self.num_actions = num_actions + self.reward_decay = reward_decay + + +class PolicyGradient(AbsAlgorithm): + """Policy gradient algorithm. + + The policy gradient algorithm base on the policy gradient theorem, a.k.a. REINFORCE. + + Args: + policy_model (nn.Module): model for generating actions given states. + optimizer_cls: torch optimizer class for the policy model. + optimizer_params: parameters required for the policy optimizer class. + hyper_params: hyper-parameter set for the AC algorithm. + """ + + def __init__(self, policy_model: nn.Module, optimizer_cls, optimizer_params, + hyper_params: PolicyGradientHyperParameters): + super().__init__() + self._device = torch.device("cuda" if torch.cuda.is_available() else "cpu") + self._policy_model = policy_model.to(self._device) + self._policy_optimizer = optimizer_cls(self._policy_model.parameters(), **optimizer_params) + self._hyper_params = hyper_params + + @property + def model(self): + return self._policy_model + + def choose_action(self, state: np.ndarray, epsilon: float = None): + state = torch.from_numpy(state).unsqueeze(0).to(self._device) # (1, state_dim) + action_dist = F.softmax(self._policy_model(state), dim=1).squeeze() # (num_actions,) + return np.random.choice(self._hyper_params.num_actions, p=action_dist.numpy()) + + def train(self, state_sequence: np.ndarray, action_sequence: np.ndarray, reward_sequence: np.ndarray): + states = torch.from_numpy(state_sequence).to(self._device) # (N, state_dim) + returns = get_k_step_returns(reward_sequence, self._hyper_params.reward_decay) + actions = torch.from_numpy(action_sequence).to(self._device) # (N,) + action_prob = self._policy_model(states).gather(1, actions.unsqueeze(1)).squeeze() # (N, 1) + policy_loss = -(torch.log(action_prob) * returns).mean() + self._policy_optimizer.zero_grad() + policy_loss.backward() + self._policy_optimizer.step() + + def load_trainable_models(self, policy_model): + self._policy_model = policy_model + + def dump_trainable_models(self): + return self._policy_model + + def load_trainable_models_from_file(self, path): + """Load trainable models from disk.""" + self._policy_model = torch.load(path) + + def dump_trainable_models_to_file(self, path: str): + """Dump the algorithm's trainable models to disk.""" + torch.save(self._policy_model.state_dict(), path) From 765c03ebe6304eeb408d98c4725526001b45f672 Mon Sep 17 00:00:00 2001 From: ysqyang Date: Wed, 14 Oct 2020 15:48:10 +0800 Subject: [PATCH 010/337] minor edits --- maro/rl/shaping/experience_shaper.py | 2 +- maro/rl/utils/trajectory_utils.py | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/maro/rl/shaping/experience_shaper.py b/maro/rl/shaping/experience_shaper.py index 0fa01d4c8..f7e6d1247 100644 --- a/maro/rl/shaping/experience_shaper.py +++ b/maro/rl/shaping/experience_shaper.py @@ -2,7 +2,7 @@ # Licensed under the MIT license. from abc import abstractmethod -from typing import Callable, Iterable, Sequence, Union +from typing import Callable, Iterable, Union from .abs_shaper import AbsShaper diff --git a/maro/rl/utils/trajectory_utils.py b/maro/rl/utils/trajectory_utils.py index 360f68b95..934aa0868 100644 --- a/maro/rl/utils/trajectory_utils.py +++ b/maro/rl/utils/trajectory_utils.py @@ -52,7 +52,7 @@ def get_lambda_returns(rewards: np.ndarray, discount: float, lmda: float, k: int if lmda == .0: return get_k_step_returns(rewards, discount, k=1, values=values) - # If lambda is one, lambda return reduces to maximum-step return + # If lambda is one, lambda return reduces to k-step return if lmda == 1.0: return get_k_step_returns(rewards, discount, k=k, values=values) @@ -63,4 +63,3 @@ def get_lambda_returns(rewards: np.ndarray, discount: float, lmda: float, k: int post_truncate = get_k_step_returns(rewards, discount, k=k, values=values) * lmda**(k-1) return (1 - lmda) * pre_truncate + post_truncate - From a416445a9de6dd1be414b85830a72f2838fa9f94 Mon Sep 17 00:00:00 2001 From: ysqyang Date: Wed, 14 Oct 2020 17:02:39 +0800 Subject: [PATCH 011/337] added advantages in ac and renamed lmda to lamb --- maro/rl/algorithms/torch/ac.py | 23 +++++++++++++---------- maro/rl/algorithms/torch/pg.py | 3 +-- maro/rl/utils/trajectory_utils.py | 14 +++++++------- tests/test_trajectory_utils.py | 4 ++-- 4 files changed, 23 insertions(+), 21 deletions(-) diff --git a/maro/rl/algorithms/torch/ac.py b/maro/rl/algorithms/torch/ac.py index 37e3b0e7f..b432f20d8 100644 --- a/maro/rl/algorithms/torch/ac.py +++ b/maro/rl/algorithms/torch/ac.py @@ -19,16 +19,16 @@ class ActorCriticHyperParameters: reward_decay (float): reward decay as defined in standard RL terminology k (int): number of time steps used in computing returns or return estimates. Defaults to -1, in which case rewards are accumulated until the end of the trajectory. - lmda (float): lambda coefficient used in computing lambda returns. Defaults to 1.0, in which case the usual + lamb (float): lambda coefficient used in computing lambda returns. Defaults to 1.0, in which case the usual k-step return is computed. """ - __slots__ = ["num_actions", "reward_decay", "k", "lmda"] + __slots__ = ["num_actions", "reward_decay", "k", "lamb"] - def __init__(self, num_actions: int, reward_decay: float, k: int = -1, lmda: float = 1.0): + def __init__(self, num_actions: int, reward_decay: float, k: int = -1, lamb: float = 1.0): self.num_actions = num_actions self.reward_decay = reward_decay self.k = k - self.lmda = lmda + self.lamb = lamb class ActorCritic(AbsAlgorithm): @@ -78,15 +78,17 @@ def train(self, state_sequence: np.ndarray, action_sequence: np.ndarray, reward_ states = torch.from_numpy(state_sequence).to(self._device) # (N, state_dim) state_values = self._value_model(states) state_values_numpy = state_values.numpy() - returns = get_lambda_returns(reward_sequence, self._hyper_params.reward_decay, self._hyper_params.lmda, + returns = get_lambda_returns(reward_sequence, self._hyper_params.reward_decay, self._hyper_params.lamb, k=self._hyper_params.k, values=state_values_numpy) + advantages = returns - state_values # policy model training actions = torch.from_numpy(action_sequence).to(self._device) # (N,) action_prob = self._policy_model(states).gather(1, actions.unsqueeze(1)).squeeze() # (N,) - policy_loss = -(torch.log(action_prob) * returns).mean() + policy_loss = -(torch.log(action_prob) * advantages).mean() self._policy_optimizer.zero_grad() policy_loss.backward() self._policy_optimizer.step() + # value model training value_loss = self._value_loss_func(state_values, returns) self._value_optimizer.zero_grad() @@ -147,13 +149,14 @@ def choose_action(self, state: np.ndarray, epsilon: float = None): def train(self, state_sequence: np.ndarray, action_sequence: np.ndarray, reward_sequence: np.ndarray): states = torch.from_numpy(state_sequence).to(self._device) # (N, state_dim) - state_values, action_dist = self._policy_value_model(states) + state_values, action_distribution = self._policy_value_model(states) state_values_numpy = state_values.numpy() - returns = get_lambda_returns(reward_sequence, self._hyper_params.reward_decay, self._hyper_params.lmda, + returns = get_lambda_returns(reward_sequence, self._hyper_params.reward_decay, self._hyper_params.lamb, k=self._hyper_params.k, values=state_values_numpy) + advantages = returns - state_values actions = torch.from_numpy(action_sequence).to(self._device) # (N,) - action_prob = action_dist.gather(1, actions.unsqueeze(1)).squeeze() # (N,) - policy_loss = -(torch.log(action_prob) * returns).mean() + action_prob = action_distribution.gather(1, actions.unsqueeze(1)).squeeze() # (N,) + policy_loss = -(torch.log(action_prob) * advantages).mean() value_loss = self._value_loss_func(state_values, returns) loss = policy_loss + value_loss self._optimizer.zero_grad() diff --git a/maro/rl/algorithms/torch/pg.py b/maro/rl/algorithms/torch/pg.py index d8422ede8..8bf4101d0 100644 --- a/maro/rl/algorithms/torch/pg.py +++ b/maro/rl/algorithms/torch/pg.py @@ -4,7 +4,6 @@ import numpy as np import torch import torch.nn as nn -import torch.nn.functional as F from maro.rl.algorithms.abs_algorithm import AbsAlgorithm from maro.rl.utils.trajectory_utils import get_k_step_returns @@ -50,7 +49,7 @@ def model(self): def choose_action(self, state: np.ndarray, epsilon: float = None): state = torch.from_numpy(state).unsqueeze(0).to(self._device) # (1, state_dim) - action_dist = F.softmax(self._policy_model(state), dim=1).squeeze() # (num_actions,) + action_dist = self._policy_model(state).squeeze() # (num_actions,) return np.random.choice(self._hyper_params.num_actions, p=action_dist.numpy()) def train(self, state_sequence: np.ndarray, action_sequence: np.ndarray, reward_sequence: np.ndarray): diff --git a/maro/rl/utils/trajectory_utils.py b/maro/rl/utils/trajectory_utils.py index 934aa0868..16b4937e3 100644 --- a/maro/rl/utils/trajectory_utils.py +++ b/maro/rl/utils/trajectory_utils.py @@ -30,12 +30,12 @@ def get_k_step_returns(rewards: np.ndarray, discount: float, k: int = -1, values np.pad(values[k:], (0, k)) if values is not None else np.zeros(len(rewards))) -def get_lambda_returns(rewards: np.ndarray, discount: float, lmda: float, k: int = -1, values: np.ndarray = None): +def get_lambda_returns(rewards: np.ndarray, discount: float, lam: float, k: int = -1, values: np.ndarray = None): """Compute lambda returns given reward and value sequences and a k. Args: rewards (np.ndarray): reward sequence from a trajectory. discount (float): reward discount as in standard RL. - lmda (float): the lambda coefficient involved in computing lambda returns. + lam (float): the lambda coefficient involved in computing lambda returns. k (int): number of steps where the lambda return series is truncated. If it is -1, no truncating is done and the lambda return is carried out to the end of the sequence. Defaults to -1. values (np.ndarray): sequence of values for the traversed states in a trajectory. If it is None, the state @@ -49,17 +49,17 @@ def get_lambda_returns(rewards: np.ndarray, discount: float, lmda: float, k: int k = len(rewards) - 1 # If lambda is zero, lambda return reduces to one-step return - if lmda == .0: + if lam == .0: return get_k_step_returns(rewards, discount, k=1, values=values) # If lambda is one, lambda return reduces to k-step return - if lmda == 1.0: + if lam == 1.0: return get_k_step_returns(rewards, discount, k=k, values=values) k = min(k, len(rewards) - 1) - pre_truncate = reduce(lambda x, y: x*lmda + y, + pre_truncate = reduce(lambda x, y: x*lam + y, [get_k_step_returns(rewards, discount, k=k, values=values) for k in range(k-1, 0, -1)]) - post_truncate = get_k_step_returns(rewards, discount, k=k, values=values) * lmda**(k-1) - return (1 - lmda) * pre_truncate + post_truncate + post_truncate = get_k_step_returns(rewards, discount, k=k, values=values) * lam**(k-1) + return (1 - lam) * pre_truncate + post_truncate diff --git a/tests/test_trajectory_utils.py b/tests/test_trajectory_utils.py index 9687d87a9..417c72716 100644 --- a/tests/test_trajectory_utils.py +++ b/tests/test_trajectory_utils.py @@ -12,7 +12,7 @@ class TestTrajectoryUtils(unittest.TestCase): def setUp(self) -> None: self.rewards = np.asarray([3, 2, 4, 1, 5]) self.values = np.asarray([4, 7, 1, 3, 6]) - self.lmda = 0.6 + self.lamb = 0.6 self.discount = 0.8 self.k = 4 @@ -22,7 +22,7 @@ def test_k_step_return(self): np.testing.assert_allclose(returns, expected, rtol=1e-4) def test_lambda_return(self): - returns = get_lambda_returns(self.rewards, self.discount, self.lmda, k=self.k, values=self.values) + returns = get_lambda_returns(self.rewards, self.discount, self.lamb, k=self.k, values=self.values) expected = np.asarray([8.1378176, 6.03712, 7.744, 5.8, 6.0]) np.testing.assert_allclose(returns, expected, rtol=1e-4) From 858ceafb93d31776da92932ef4965767792d41b6 Mon Sep 17 00:00:00 2001 From: ysqyang Date: Fri, 16 Oct 2020 10:03:03 +0800 Subject: [PATCH 012/337] added dueling action value model --- .../torch/dueling_action_value_model.py | 27 +++++++++++++++++++ 1 file changed, 27 insertions(+) create mode 100644 maro/rl/models/torch/dueling_action_value_model.py diff --git a/maro/rl/models/torch/dueling_action_value_model.py b/maro/rl/models/torch/dueling_action_value_model.py new file mode 100644 index 000000000..ebe94bbcd --- /dev/null +++ b/maro/rl/models/torch/dueling_action_value_model.py @@ -0,0 +1,27 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import torch.nn as nn + +from maro.rl.models.torch.learning_model import IdentityLayers + + +class DuelingActionValueModel(nn.Module): + def __init__(self, value_head, advantage_head, representation_layers=IdentityLayers(), + advantage_mode: str = 'mean'): + super().__init__() + self._value_head = value_head + self._advantage_head = advantage_head + self._representation_layers = representation_layers + if self._advantage_mode not in {'mean', 'max'}: + raise ValueError('Advantage mode must be "mean" or "max"') + self._advantage_mode = advantage_mode + + def forward(self, inputs): + representations = self._representation_layers(inputs) + state_values = self._value_layers(representations) + advantages = self._advantage_layers(representations) + # use mean or max correction to address the identifiability issue + corrections = advantages.mean(1) if self._advantage_mode == 'mean' else advantages.max(1)[0] + q_values = state_values + advantages - corrections.unsqueeze(1) + return q_values From c842106e0941c3edd3cbebca169d37757e18fde2 Mon Sep 17 00:00:00 2001 From: ysqyang Date: Fri, 16 Oct 2020 10:23:36 +0800 Subject: [PATCH 013/337] renamed params in dueling_action_value_model --- maro/rl/models/torch/dueling_action_value_model.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/maro/rl/models/torch/dueling_action_value_model.py b/maro/rl/models/torch/dueling_action_value_model.py index ebe94bbcd..9386e98ae 100644 --- a/maro/rl/models/torch/dueling_action_value_model.py +++ b/maro/rl/models/torch/dueling_action_value_model.py @@ -7,20 +7,20 @@ class DuelingActionValueModel(nn.Module): - def __init__(self, value_head, advantage_head, representation_layers=IdentityLayers(), + def __init__(self, value_head: nn.Module, advantage_head: nn.Module, shared_layers:nn.Module = IdentityLayers(), advantage_mode: str = 'mean'): super().__init__() self._value_head = value_head self._advantage_head = advantage_head - self._representation_layers = representation_layers + self._shared_layers = shared_layers if self._advantage_mode not in {'mean', 'max'}: raise ValueError('Advantage mode must be "mean" or "max"') self._advantage_mode = advantage_mode def forward(self, inputs): - representations = self._representation_layers(inputs) - state_values = self._value_layers(representations) - advantages = self._advantage_layers(representations) + shared_features = self._shared_layers(inputs) + state_values = self._value_layers(shared_features) + advantages = self._advantage_layers(shared_features) # use mean or max correction to address the identifiability issue corrections = advantages.mean(1) if self._advantage_mode == 'mean' else advantages.max(1)[0] q_values = state_values + advantages - corrections.unsqueeze(1) From 8f6a9a8e64714cdabe936fda48622517c71028cd Mon Sep 17 00:00:00 2001 From: ysqyang Date: Fri, 16 Oct 2020 10:40:31 +0800 Subject: [PATCH 014/337] renamed shared_features to features --- maro/rl/models/torch/dueling_action_value_model.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/maro/rl/models/torch/dueling_action_value_model.py b/maro/rl/models/torch/dueling_action_value_model.py index 9386e98ae..7e986dc23 100644 --- a/maro/rl/models/torch/dueling_action_value_model.py +++ b/maro/rl/models/torch/dueling_action_value_model.py @@ -18,9 +18,9 @@ def __init__(self, value_head: nn.Module, advantage_head: nn.Module, shared_laye self._advantage_mode = advantage_mode def forward(self, inputs): - shared_features = self._shared_layers(inputs) - state_values = self._value_layers(shared_features) - advantages = self._advantage_layers(shared_features) + features = self._shared_layers(inputs) + state_values = self._value_layers(features) + advantages = self._advantage_layers(features) # use mean or max correction to address the identifiability issue corrections = advantages.mean(1) if self._advantage_mode == 'mean' else advantages.max(1)[0] q_values = state_values + advantages - corrections.unsqueeze(1) From 93646077020782ac29e45bfdb8cbe39776c98660 Mon Sep 17 00:00:00 2001 From: ysqyang Date: Fri, 16 Oct 2020 18:36:22 +0800 Subject: [PATCH 015/337] modified ppo for ac and added helper function to compute return estimates and advantages --- maro/rl/algorithms/torch/ac.py | 146 ++++++++++++-------- maro/rl/algorithms/torch/pg.py | 10 +- maro/rl/algorithms/torch/ppo.py | 233 ++++++++++++++++++++++++++++++++ 3 files changed, 331 insertions(+), 58 deletions(-) create mode 100644 maro/rl/algorithms/torch/ppo.py diff --git a/maro/rl/algorithms/torch/ac.py b/maro/rl/algorithms/torch/ac.py index b432f20d8..7c105277b 100644 --- a/maro/rl/algorithms/torch/ac.py +++ b/maro/rl/algorithms/torch/ac.py @@ -17,16 +17,23 @@ class ActorCriticHyperParameters: Args: num_actions (int): number of possible actions reward_decay (float): reward decay as defined in standard RL terminology + policy_train_iters (int): number of gradient descent steps for the policy model per call to ``train``. + value_train_iters (int): number of gradient descent steps for the value model per call to ``train``. k (int): number of time steps used in computing returns or return estimates. Defaults to -1, in which case rewards are accumulated until the end of the trajectory. lamb (float): lambda coefficient used in computing lambda returns. Defaults to 1.0, in which case the usual k-step return is computed. """ - __slots__ = ["num_actions", "reward_decay", "k", "lamb"] + __slots__ = ["num_actions", "reward_decay", "policy_train_iters", "value_train_iters", "k", "lamb"] - def __init__(self, num_actions: int, reward_decay: float, k: int = -1, lamb: float = 1.0): + def __init__( + self, num_actions: int, reward_decay: float, policy_train_iters, value_train_iters: int, + k: int = -1, lamb: float = 1.0 + ): self.num_actions = num_actions self.reward_decay = reward_decay + self.policy_train_iters = policy_train_iters + self.value_train_iters = value_train_iters self.k = k self.lamb = lamb @@ -47,15 +54,10 @@ class ActorCritic(AbsAlgorithm): hyper_params: hyper-parameter set for the AC algorithm. """ - def __init__(self, - policy_model: nn.Module, - value_model: nn.Module, - value_loss_func: Callable, - policy_optimizer_cls, - policy_optimizer_params, - value_optimizer_cls, - value_optimizer_params, - hyper_params: ActorCriticHyperParameters): + def __init__( + self, policy_model: nn.Module, value_model: nn.Module, value_loss_func: Callable, policy_optimizer_cls, + policy_optimizer_params, value_optimizer_cls, value_optimizer_params, hyper_params: ActorCriticHyperParameters + ): super().__init__() self._device = torch.device("cuda" if torch.cuda.is_available() else "cpu") self._policy_model = policy_model.to(self._device) @@ -71,29 +73,37 @@ def model(self): def choose_action(self, state: np.ndarray, epsilon: float = None): state = torch.from_numpy(state).unsqueeze(0).to(self._device) # (1, state_dim) - action_dist = self._policy_model(state).squeeze() # (num_actions,) - return np.random.choice(self._hyper_params.num_actions, p=action_dist.numpy()) + action_dist = self._policy_model(state).squeeze().numpy() # (num_actions,) + return np.random.choice(self._hyper_params.num_actions, p=action_dist) - def train(self, state_sequence: np.ndarray, action_sequence: np.ndarray, reward_sequence: np.ndarray): - states = torch.from_numpy(state_sequence).to(self._device) # (N, state_dim) - state_values = self._value_model(states) + def _get_bootstrapped_returns_and_advantages(self, states: torch.tensor, rewards: np.ndarray): + state_values = self._value_model(states).detach() state_values_numpy = state_values.numpy() - returns = get_lambda_returns(reward_sequence, self._hyper_params.reward_decay, self._hyper_params.lamb, - k=self._hyper_params.k, values=state_values_numpy) - advantages = returns - state_values + return_est = get_lambda_returns( + rewards, self._hyper_params.reward_decay, self._hyper_params.lamb, + k=self._hyper_params.k, values=state_values_numpy + ) + return_est = torch.from_numpy(return_est) + return return_est, return_est - state_values + + def train(self, state_sequence: np.ndarray, action_sequence: np.ndarray, reward_sequence: np.ndarray): + states = torch.from_numpy(state_sequence).to(self._device) + actions = torch.from_numpy(action_sequence).to(self._device) + return_est, advantages = self._get_bootstrapped_returns_and_advantages(states, reward_sequence) # policy model training - actions = torch.from_numpy(action_sequence).to(self._device) # (N,) - action_prob = self._policy_model(states).gather(1, actions.unsqueeze(1)).squeeze() # (N,) - policy_loss = -(torch.log(action_prob) * advantages).mean() - self._policy_optimizer.zero_grad() - policy_loss.backward() - self._policy_optimizer.step() + for _ in range(self._hyper_params.policy_train_iters): + action_prob = self._policy_model(states).gather(1, actions.unsqueeze(1)).squeeze() # (N,) + policy_loss = -(torch.log(action_prob) * advantages).mean() + self._policy_optimizer.zero_grad() + policy_loss.backward() + self._policy_optimizer.step() # value model training - value_loss = self._value_loss_func(state_values, returns) - self._value_optimizer.zero_grad() - value_loss.backward() - self._value_optimizer.step() + for _ in range(self._hyper_params.value_train_iters): + value_loss = self._value_loss_func(self._value_model(states), return_est) + self._value_optimizer.zero_grad() + value_loss.backward() + self._value_optimizer.step() def load_trainable_models(self, model_dict): self._policy_model = model_dict["policy"] @@ -113,7 +123,29 @@ def dump_trainable_models_to_file(self, path: str): torch.save({"policy": self._policy_model.state_dict(), "value": self._value_model.state_dict()}, path) -class ActorCriticWithSharedLayers(AbsAlgorithm): +class ActorCriticHyperParametersWithCombinedModel: + """Hyper-parameter set for the Actor-Critic algorithm with a combined policy/value model. + + Args: + num_actions (int): number of possible actions + reward_decay (float): reward decay as defined in standard RL terminology + train_iters (int): number of gradient descent steps for the policy-value model per call to ``train``. + k (int): number of time steps used in computing returns or return estimates. Defaults to -1, in which case + rewards are accumulated until the end of the trajectory. + lamb (float): lambda coefficient used in computing lambda returns. Defaults to 1.0, in which case the usual + k-step return is computed. + """ + __slots__ = ["num_actions", "reward_decay", "train_iters", "k", "lamb"] + + def __init__(self, num_actions: int, reward_decay: float, train_iters: int, k: int = -1, lamb: float = 1.0): + self.num_actions = num_actions + self.reward_decay = reward_decay + self.train_iters = train_iters + self.k = k + self.lamb = lamb + + +class ActorCriticWithCombinedModel(AbsAlgorithm): """Actor Critic algorithm where policy and value models have shared layers. Args: @@ -125,12 +157,10 @@ class ActorCriticWithSharedLayers(AbsAlgorithm): hyper_params: hyper-parameter set for the AC algorithm. """ - def __init__(self, - policy_value_model: nn.Module, - value_loss_func: Callable, - optimizer_cls, - optimizer_params, - hyper_params: ActorCriticHyperParameters): + def __init__( + self, policy_value_model: nn.Module, value_loss_func: Callable, optimizer_cls, optimizer_params, + hyper_params: ActorCriticHyperParametersWithCombinedModel + ): super().__init__() self._device = torch.device("cuda" if torch.cuda.is_available() else "cpu") self._policy_value_model = policy_value_model.to(self._device) @@ -144,24 +174,34 @@ def model(self): def choose_action(self, state: np.ndarray, epsilon: float = None): state = torch.from_numpy(state).unsqueeze(0).to(self._device) # (1, state_dim) - action_dist = self._policy_value_model(state)[1].squeeze() # (num_actions,) - return np.random.choice(self._hyper_params.num_actions, p=action_dist.numpy()) + action_dist = self._policy_value_model(state)[1].squeeze().numpy() # (num_actions,) + return np.random.choice(self._hyper_params.num_actions, p=action_dist) - def train(self, state_sequence: np.ndarray, action_sequence: np.ndarray, reward_sequence: np.ndarray): - states = torch.from_numpy(state_sequence).to(self._device) # (N, state_dim) - state_values, action_distribution = self._policy_value_model(states) + def _get_bootstrapped_returns_and_advantages(self, states: torch.tensor, reward_sequence: np.ndarray): + state_values = self._policy_value_model(states)[0].detach() state_values_numpy = state_values.numpy() - returns = get_lambda_returns(reward_sequence, self._hyper_params.reward_decay, self._hyper_params.lamb, - k=self._hyper_params.k, values=state_values_numpy) - advantages = returns - state_values - actions = torch.from_numpy(action_sequence).to(self._device) # (N,) - action_prob = action_distribution.gather(1, actions.unsqueeze(1)).squeeze() # (N,) - policy_loss = -(torch.log(action_prob) * advantages).mean() - value_loss = self._value_loss_func(state_values, returns) - loss = policy_loss + value_loss - self._optimizer.zero_grad() - loss.backward() - self._optimizer.step() + return_est = get_lambda_returns( + reward_sequence, self._hyper_params.reward_decay, self._hyper_params.lamb, + k=self._hyper_params.k, values=state_values_numpy + ) + return_est = torch.from_numpy(return_est) + return return_est, return_est - state_values + + def train(self, state_sequence: np.ndarray, action_sequence: np.ndarray, reward_sequence: np.ndarray): + states = torch.from_numpy(state_sequence).to(self._device) + actions = torch.from_numpy(action_sequence).to(self._device) + return_est, advantages = self._get_bootstrapped_returns_and_advantages(states, reward_sequence) + # policy-value model training + for _ in range(self._hyper_params.train_iters): + state_values, action_distribution = self._policy_value_model(states) + advantages = return_est - state_values + action_prob = action_distribution.gather(1, actions.unsqueeze(1)).squeeze() # (N,) + policy_loss = -(torch.log(action_prob) * advantages).mean() + value_loss = self._value_loss_func(state_values, return_est) + loss = policy_loss + value_loss + self._optimizer.zero_grad() + loss.backward() + self._optimizer.step() def load_trainable_models(self, policy_value_model): self._policy_value_model = policy_value_model @@ -170,9 +210,7 @@ def dump_trainable_models(self): return self._policy_value_model def load_trainable_models_from_file(self, path): - """Load trainable models from disk.""" self._policy_value_model = torch.load(path) def dump_trainable_models_to_file(self, path: str): - """Dump the algorithm's trainable models to disk.""" torch.save(self._policy_value_model.state_dict(), path) diff --git a/maro/rl/algorithms/torch/pg.py b/maro/rl/algorithms/torch/pg.py index 8bf4101d0..ce5f880eb 100644 --- a/maro/rl/algorithms/torch/pg.py +++ b/maro/rl/algorithms/torch/pg.py @@ -35,8 +35,10 @@ class PolicyGradient(AbsAlgorithm): hyper_params: hyper-parameter set for the AC algorithm. """ - def __init__(self, policy_model: nn.Module, optimizer_cls, optimizer_params, - hyper_params: PolicyGradientHyperParameters): + def __init__( + self, policy_model: nn.Module, optimizer_cls, optimizer_params, + hyper_params: PolicyGradientHyperParameters + ): super().__init__() self._device = torch.device("cuda" if torch.cuda.is_available() else "cpu") self._policy_model = policy_model.to(self._device) @@ -49,8 +51,8 @@ def model(self): def choose_action(self, state: np.ndarray, epsilon: float = None): state = torch.from_numpy(state).unsqueeze(0).to(self._device) # (1, state_dim) - action_dist = self._policy_model(state).squeeze() # (num_actions,) - return np.random.choice(self._hyper_params.num_actions, p=action_dist.numpy()) + action_dist = self._policy_model(state).squeeze().numpy() # (num_actions,) + return np.random.choice(self._hyper_params.num_actions, p=action_dist) def train(self, state_sequence: np.ndarray, action_sequence: np.ndarray, reward_sequence: np.ndarray): states = torch.from_numpy(state_sequence).to(self._device) # (N, state_dim) diff --git a/maro/rl/algorithms/torch/ppo.py b/maro/rl/algorithms/torch/ppo.py new file mode 100644 index 000000000..854123b3b --- /dev/null +++ b/maro/rl/algorithms/torch/ppo.py @@ -0,0 +1,233 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +from typing import Callable + +import numpy as np +import torch +import torch.nn as nn + +from maro.rl.algorithms.abs_algorithm import AbsAlgorithm +from maro.rl.utils.trajectory_utils import get_lambda_returns + + +class PPOHyperParameters: + """Hyper-parameter set for the Proximal Policy Optimization (PPO) algorithm. + + Args: + num_actions (int): number of possible actions + reward_decay (float): reward decay as defined in standard RL terminology + clip_ratio (float): clip ratio as defined in PPO's objective function. + policy_train_iters (int): number of gradient descent steps for the policy model per call to ``train``. + value_train_iters (int): number of gradient descent steps for the value model per call to ``train``. + k (int): number of time steps used in computing returns or return estimates. Defaults to -1, in which case + rewards are accumulated until the end of the trajectory. + lamb (float): lambda coefficient used in computing lambda returns. Defaults to 1.0, in which case the usual + k-step return is computed. + """ + __slots__ = ["num_actions", "reward_decay", "clip_ratio", "policy_train_iters", "value_train_iters", "k", "lamb"] + + def __init__( + self, num_actions: int, reward_decay: float, clip_ratio: float, policy_train_iters: int, + value_train_iters: int, k: int = -1, lamb: float = 1.0 + ): + self.num_actions = num_actions + self.reward_decay = reward_decay + self.clip_ratio = clip_ratio + self.policy_train_iters = policy_train_iters + self.value_train_iters = value_train_iters + self.k = k + self.lamb = lamb + + +class PPO(AbsAlgorithm): + """Proximal policy optimization (PPO) algorithm. + + See https://arxiv.org/pdf/1707.06347.pdf for details. + + Args: + policy_model (nn.Module): model for generating actions given states. + value_model (nn.Module): model for estimating state values. + value_loss_func (Callable): loss function for the value model. + policy_optimizer_cls: torch optimizer class for the policy model. + policy_optimizer_params: parameters required for the policy optimizer class. + value_optimizer_cls: torch optimizer class for the value model. + value_optimizer_params: parameters required for the value optimizer class. + hyper_params: hyper-parameter set for the AC algorithm. + """ + + def __init__( + self, policy_model: nn.Module, value_model: nn.Module, value_loss_func: Callable, policy_optimizer_cls, + policy_optimizer_params, value_optimizer_cls, value_optimizer_params, hyper_params: PPOHyperParameters + ): + super().__init__() + self._device = torch.device("cuda" if torch.cuda.is_available() else "cpu") + self._policy_model = policy_model.to(self._device) + self._value_model = value_model.to(self._device) + self._policy_optimizer = policy_optimizer_cls(self._policy_model.parameters(), **policy_optimizer_params) + self._value_optimizer = value_optimizer_cls(self._value_model.parameters(), **value_optimizer_params) + self._value_loss_func = value_loss_func + self._hyper_params = hyper_params + + def choose_action(self, state: np.ndarray, epsilon: float = None): + state = torch.from_numpy(state).unsqueeze(0).to(self._device) # (1, state_dim) + action_dist = self._policy_model(state).squeeze().numpy() # (num_actions,) + return np.random.choice(self._hyper_params.num_actions, p=action_dist) + + def _get_bootstrapped_returns_and_advantages(self, states: torch.tensor, rewards: np.ndarray): + state_values = self._value_model(states).detach() + state_values_numpy = state_values.numpy() + return_est = get_lambda_returns( + rewards, self._hyper_params.reward_decay, self._hyper_params.lamb, + k=self._hyper_params.k, values=state_values_numpy + ) + return_est = torch.from_numpy(return_est) + return return_est, return_est - state_values + + def train( + self, state_sequence: np.ndarray, action_sequence: np.ndarray, log_action_prob_sequence: np.ndarray, + reward_sequence: np.ndarray + ): + states = torch.from_numpy(state_sequence).to(self._device) # (N, state_dim) + actions = torch.from_numpy(action_sequence).to(self._device) # (N,) + return_est, advantages = self._get_bootstrapped_returns_and_advantages(states, reward_sequence) + log_action_prob_old = torch.from_numpy(log_action_prob_sequence).to(self._device) + + # policy model training (with the value model fixed) + for _ in range(self._hyper_params.policy_train_iters): + action_prob = self._policy_model(states).gather(1, actions.unsqueeze(1)).squeeze() # (N, 1) + ratio = torch.exp(torch.log(action_prob) - log_action_prob_old) + clipped_ratio = torch.clamp(ratio, 1-self._hyper_params.clip_ratio, 1+self._hyper_params.clip_ratio) + loss = -(torch.min(ratio*advantages, clipped_ratio*advantages)).mean() + self._policy_optimizer.zero_grad() + loss.backward() + self._policy_optimizer.step() + + # value model training + for _ in range(self._hyper_params.value_train_iters): + value_loss = self._value_loss_func(self._value_model(states), return_est) + self._value_optimizer.zero_grad() + value_loss.backward() + self._value_optimizer.step() + + def load_trainable_models(self, model_dict): + self._policy_model = model_dict["policy"] + self._value_model = model_dict["value"] + + def dump_trainable_models(self): + return {"policy": self._policy_model, "value": self._value_model} + + def load_trainable_models_from_file(self, path): + """Load trainable models from disk.""" + model_dict = torch.load(path) + self._policy_model = model_dict["policy"] + self._value_model = model_dict["value"] + + def dump_trainable_models_to_file(self, path: str): + """Dump the algorithm's trainable models to disk.""" + torch.save({"policy": self._policy_model.state_dict(), "value": self._value_model.state_dict()}, path) + + +class PPOHyperParametersWithCombinedModel: + """Hyper-parameter set for the Proximal Policy Optimization (PPO) algorithm. + + Args: + num_actions (int): number of possible actions + reward_decay (float): reward decay as defined in standard RL terminology + clip_ratio (float): clip ratio as defined in PPO's objective function. + train_iters (int): number of gradient descent steps for the policy-value model per call to ``train``. + k (int): number of time steps used in computing returns or return estimates. Defaults to -1, in which case + rewards are accumulated until the end of the trajectory. + lamb (float): lambda coefficient used in computing lambda returns. Defaults to 1.0, in which case the usual + k-step return is computed. + """ + __slots__ = ["num_actions", "reward_decay", "clip_ratio", "train_iters", "k", "lamb"] + + def __init__( + self, num_actions: int, reward_decay: float, clip_ratio: float, train_iters: int, + k: int = -1, lamb: float = 1.0 + ): + self.num_actions = num_actions + self.reward_decay = reward_decay + self.clip_ratio = clip_ratio + self.train_iters = train_iters + self.k = k + self.lamb = lamb + + +class PPOWithCombinedModel(AbsAlgorithm): + """PPO algorithm where policy and value models have shared layers. + + Args: + policy_value_model (nn.Module): model for generating action distributions and values for given states using + shared bottom layers. The model, when called, must return (value, action distribution). + value_loss_func (Callable): loss function for the value model. + optimizer_cls: torch optimizer class for the policy model. + optimizer_params: parameters required for the policy optimizer class. + hyper_params: hyper-parameter set for the AC algorithm. + """ + + def __init__( + self, policy_value_model: nn.Module, value_loss_func: Callable, optimizer_cls, optimizer_params, + hyper_params: PPOHyperParametersWithCombinedModel + ): + super().__init__() + self._device = torch.device("cuda" if torch.cuda.is_available() else "cpu") + self._policy_value_model = policy_value_model.to(self._device) + self._optimizer = optimizer_cls(self._policy_value_model.parameters(), **optimizer_params) + self._value_loss_func = value_loss_func + self._hyper_params = hyper_params + + @property + def model(self): + return self._policy_value_model + + def choose_action(self, state: np.ndarray, epsilon: float = None): + state = torch.from_numpy(state).unsqueeze(0).to(self._device) # (1, state_dim) + action_dist = self._policy_value_model(state)[1].squeeze().numpy() # (num_actions,) + action_index = np.random.choice(self._hyper_params.num_actions, p=action_dist) + return action_index, np.log(action_dist[action_index]) + + def _get_bootstrapped_returns_and_advantages(self, states: torch.tensor, rewards: np.ndarray): + state_values = self._policy_value_model(states)[0].detach() + state_values_numpy = state_values.numpy() + return_est = get_lambda_returns( + rewards, self._hyper_params.reward_decay, self._hyper_params.lamb, + k=self._hyper_params.k, values=state_values_numpy + ) + return_est = torch.from_numpy(return_est) + return return_est, return_est - state_values + + def train( + self, state_sequence: np.ndarray, action_sequence: np.ndarray, log_action_prob_sequence: np.ndarray, + reward_sequence: np.ndarray + ): + states = torch.from_numpy(state_sequence).to(self._device) # (N, state_dim) + actions = torch.from_numpy(action_sequence).to(self._device) # (N,) + return_est, advantages = self._get_bootstrapped_returns_and_advantages(states, reward_sequence) + log_action_prob_old = torch.from_numpy(log_action_prob_sequence).to(self._device) + for _ in range(self._hyper_params.train_iters): + state_values, action_distribution = self._policy_value_model(states) + action_prob = action_distribution.gather(1, actions.unsqueeze(1)).squeeze() # (N,) + ratio = torch.exp(torch.log(action_prob) - log_action_prob_old) + clipped_ratio = torch.clamp(ratio, 1 - self._hyper_params.clip_ratio, 1 + self._hyper_params.clip_ratio) + policy_loss = -(torch.min(ratio * advantages, clipped_ratio * advantages)).mean() + value_loss = self._value_loss_func(state_values, return_est) + loss = policy_loss + value_loss + self._optimizer.zero_grad() + loss.backward() + self._optimizer.step() + + def load_trainable_models(self, policy_value_model): + self._policy_value_model = policy_value_model + + def dump_trainable_models(self): + return self._policy_value_model + + def load_trainable_models_from_file(self, path): + """Load trainable models from disk.""" + self._policy_value_model = torch.load(path) + + def dump_trainable_models_to_file(self, path: str): + """Dump the algorithm's trainable models to disk.""" + torch.save(self._policy_value_model.state_dict(), path) From 9ad0897b545ecf22f2d7781902de921d89244405 Mon Sep 17 00:00:00 2001 From: ysqyang Date: Sun, 18 Oct 2020 00:35:11 +0800 Subject: [PATCH 016/337] cim examples using pg and ac (in progress) --- examples/cim/ac/README.md | 22 ++++++ examples/cim/ac/components/__init__.py | 2 + examples/cim/ac/components/action_shaper.py | 33 +++++++++ examples/cim/ac/components/agent.py | 28 +++++++ examples/cim/ac/components/agent_manager.py | 62 ++++++++++++++++ examples/cim/ac/components/config.py | 18 +++++ .../cim/ac/components/experience_shaper.py | 49 ++++++++++++ examples/cim/ac/components/state_shaper.py | 29 ++++++++ examples/cim/ac/config.yml | 74 +++++++++++++++++++ examples/cim/ac/dist_actor.py | 45 +++++++++++ examples/cim/ac/dist_learner.py | 36 +++++++++ examples/cim/ac/multi_process_launcher.py | 23 ++++++ examples/cim/ac/single_process_launcher.py | 57 ++++++++++++++ examples/cim/pg/README.md | 22 ++++++ examples/cim/pg/components/__init__.py | 2 + examples/cim/pg/components/action_shaper.py | 33 +++++++++ examples/cim/pg/components/agent.py | 18 +++++ examples/cim/pg/components/agent_manager.py | 52 +++++++++++++ examples/cim/pg/components/config.py | 18 +++++ .../cim/pg/components/experience_shaper.py | 48 ++++++++++++ examples/cim/pg/components/state_shaper.py | 29 ++++++++ examples/cim/pg/config.yml | 72 ++++++++++++++++++ examples/cim/pg/dist_actor.py | 45 +++++++++++ examples/cim/pg/dist_learner.py | 36 +++++++++ examples/cim/pg/multi_process_launcher.py | 23 ++++++ examples/cim/pg/single_process_launcher.py | 57 ++++++++++++++ maro/rl/__init__.py | 15 ++++ maro/rl/agent/abs_agent.py | 15 ++-- maro/rl/algorithms/torch/ac.py | 33 +++++---- maro/rl/algorithms/torch/pg.py | 8 +- maro/rl/algorithms/torch/ppo.py | 36 ++++----- maro/rl/models/torch/decision_layers.py | 11 ++- maro/rl/shaping/k_step_experience_shaper.py | 2 +- maro/rl/storage/column_based_store.py | 5 ++ 34 files changed, 1010 insertions(+), 48 deletions(-) create mode 100644 examples/cim/ac/README.md create mode 100644 examples/cim/ac/components/__init__.py create mode 100644 examples/cim/ac/components/action_shaper.py create mode 100644 examples/cim/ac/components/agent.py create mode 100644 examples/cim/ac/components/agent_manager.py create mode 100644 examples/cim/ac/components/config.py create mode 100644 examples/cim/ac/components/experience_shaper.py create mode 100644 examples/cim/ac/components/state_shaper.py create mode 100644 examples/cim/ac/config.yml create mode 100644 examples/cim/ac/dist_actor.py create mode 100644 examples/cim/ac/dist_learner.py create mode 100644 examples/cim/ac/multi_process_launcher.py create mode 100644 examples/cim/ac/single_process_launcher.py create mode 100644 examples/cim/pg/README.md create mode 100644 examples/cim/pg/components/__init__.py create mode 100644 examples/cim/pg/components/action_shaper.py create mode 100644 examples/cim/pg/components/agent.py create mode 100644 examples/cim/pg/components/agent_manager.py create mode 100644 examples/cim/pg/components/config.py create mode 100644 examples/cim/pg/components/experience_shaper.py create mode 100644 examples/cim/pg/components/state_shaper.py create mode 100644 examples/cim/pg/config.yml create mode 100644 examples/cim/pg/dist_actor.py create mode 100644 examples/cim/pg/dist_learner.py create mode 100644 examples/cim/pg/multi_process_launcher.py create mode 100644 examples/cim/pg/single_process_launcher.py diff --git a/examples/cim/ac/README.md b/examples/cim/ac/README.md new file mode 100644 index 000000000..816ed5052 --- /dev/null +++ b/examples/cim/ac/README.md @@ -0,0 +1,22 @@ +# Overview + +The CIM problem is one of the quintessential use cases of MARO. The example can +be run with a set of scenario configurations that can be found under +maro/simulator/scenarios/cim. General experimental parameters (e.g., type of +topology, type of algorithm to use, number of training episodes) can be configured +through config.yml. Each RL formulation has a dedicated folder, e.g., dqn, and +all algorithm-specific parameters can be configured through +the config.py file in that folder. + +## Single-host Single-process Mode + +To run the CIM example using the DQN algorithm under single-host mode, go to +examples/cim/dqn and run single_process_launcher.py. You may play around with +the configuration if you want to try out different settings. + +## Distributed Mode + +The examples/cim/dqn/components folder contains dist_learner.py and dist_actor.py +for distributed training. For debugging purposes, we provide a script that +simulates distributed mode using multi-processing. Simply go to examples/cim/dqn +and run multi_process_launcher.py to start the learner and actor processes. diff --git a/examples/cim/ac/components/__init__.py b/examples/cim/ac/components/__init__.py new file mode 100644 index 000000000..b14b47650 --- /dev/null +++ b/examples/cim/ac/components/__init__.py @@ -0,0 +1,2 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. diff --git a/examples/cim/ac/components/action_shaper.py b/examples/cim/ac/components/action_shaper.py new file mode 100644 index 000000000..687d18d88 --- /dev/null +++ b/examples/cim/ac/components/action_shaper.py @@ -0,0 +1,33 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +from maro.rl import ActionShaper +from maro.simulator.scenarios.cim.common import Action + + +class CIMActionShaper(ActionShaper): + def __init__(self, action_space): + super().__init__() + self._action_space = action_space + self._zero_action_index = action_space.index(0) + + def __call__(self, model_action, decision_event, snapshot_list): + scope = decision_event.action_scope + tick = decision_event.tick + port_idx = decision_event.port_idx + vessel_idx = decision_event.vessel_idx + + port_empty = snapshot_list["ports"][tick: port_idx: ["empty", "full", "on_shipper", "on_consignee"]][0] + vessel_remaining_space = snapshot_list["vessels"][tick: vessel_idx: ["empty", "full", "remaining_space"]][2] + early_discharge = snapshot_list["vessels"][tick:vessel_idx: "early_discharge"][0] + assert 0 <= model_action < len(self._action_space) + + if model_action < self._zero_action_index: + actual_action = max(round(self._action_space[model_action] * port_empty), -vessel_remaining_space) + elif model_action > self._zero_action_index: + plan_action = self._action_space[model_action] * (scope.discharge + early_discharge) - early_discharge + actual_action = round(plan_action) if plan_action > 0 else round(self._action_space[model_action] * scope.discharge) + else: + actual_action = 0 + + return Action(vessel_idx, port_idx, actual_action) diff --git a/examples/cim/ac/components/agent.py b/examples/cim/ac/components/agent.py new file mode 100644 index 000000000..7f6f24255 --- /dev/null +++ b/examples/cim/ac/components/agent.py @@ -0,0 +1,28 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import numpy as np + +from maro.rl import AbsAgent, ColumnBasedStore + + +class CIMAgent(AbsAgent): + def __init__(self, name, algorithm, experience_pool: ColumnBasedStore, min_experiences_to_train, + num_batches, batch_size): + super().__init__(name, algorithm, experience_pool) + self._min_experiences_to_train = min_experiences_to_train + self._num_batches = num_batches + self._batch_size = batch_size + + def train(self): + if len(self._experience_pool) < self._min_experiences_to_train: + return + + for _ in range(self._num_batches): + indexes, sample = self._experience_pool.sample_by_key("loss", self._batch_size) + state = np.asarray(sample["state"]) + action = np.asarray(sample["action"]) + reward = np.asarray(sample["reward"]) + next_state = np.asarray(sample["next_state"]) + loss = self._algorithm.train(state, action, reward, next_state) + self._experience_pool.update(indexes, {"loss": loss}) diff --git a/examples/cim/ac/components/agent_manager.py b/examples/cim/ac/components/agent_manager.py new file mode 100644 index 000000000..8f37650f1 --- /dev/null +++ b/examples/cim/ac/components/agent_manager.py @@ -0,0 +1,62 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +from torch.nn.functional import smooth_l1_loss +from torch.optim import Adam, RMSprop + +from .agent import CIMAgent +from .config import config +from maro.rl import AbsAgentManager, LearningModel, MLPDecisionLayers, ActorCritic, ActorCriticHyperParameters, \ + ColumnBasedStore +from maro.utils import set_seeds + + +class ACAgentManager(AbsAgentManager): + def _assemble(self, agent_dict): + set_seeds(config.agents.seed) + num_actions = config.agents.algorithm.num_actions + for agent_id in self._agent_id_list: + policy_model = LearningModel(decision_layers=MLPDecisionLayers(name=f'{agent_id}.policy', + input_dim=self._state_shaper.dim, + output_dim=num_actions, + **config.agents.algorithm.policy_model, + softmax=True)) + + value_model = LearningModel(decision_layers=MLPDecisionLayers(name=f'{agent_id}.policy', + input_dim=self._state_shaper.dim, + output_dim=num_actions, + **config.agents.algorithm.value_model)) + + algorithm = ActorCritic(policy_model=policy_model, + value_model=value_model, + value_loss_func=smooth_l1_loss, + policy_optimizer_cls=Adam, + policy_optimizer_params=config.agents.algorithm.policy_optimizer, + value_optimizer_cls=RMSprop, + value_optimizer_params=config.agents.algorithm.value_optimizer, + hyper_params=ActorCriticHyperParameters(num_actions=num_actions, + **config.agents.algorithm.hyper_parameters, + ) + ) + + experience_pool = ColumnBasedStore(**config.agents.experience_pool) + agent_dict[agent_id] = CIMAgent(name=agent_id, algorithm=algorithm, experience_pool=experience_pool, + **config.agents.training_loop_parameters) + + def choose_action(self, decision_event, snapshot_list): + self._assert_inference_mode() + agent_id, model_state = self._state_shaper(decision_event, snapshot_list) + model_action = self._agent_dict[agent_id].choose_action( + model_state, self._explorer.epsilon[agent_id] if self._explorer else None) + + self._transition_cache = {"state": model_state, + "action": model_action, + "reward": None, + "agent_id": agent_id, + "event": decision_event} + return self._action_shaper(model_action, decision_event, snapshot_list) + + def store_experiences(self, experiences): + for agent_id, exp in experiences.items(): + exp.update({"loss": [1e8] * len(exp[next(iter(exp))])}) + self._agent_dict[agent_id].store_experiences(exp) diff --git a/examples/cim/ac/components/config.py b/examples/cim/ac/components/config.py new file mode 100644 index 000000000..d36d60ddd --- /dev/null +++ b/examples/cim/ac/components/config.py @@ -0,0 +1,18 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +""" +This file is used to load config and convert it into a dotted dictionary. +""" + +import io +import os +import yaml + +from maro.utils import convert_dottable + + +CONFIG_PATH = os.path.join(os.path.split(os.path.realpath(__file__))[0], "../config.yml") +with io.open(CONFIG_PATH, "r") as in_file: + raw_config = yaml.safe_load(in_file) + config = convert_dottable(raw_config) diff --git a/examples/cim/ac/components/experience_shaper.py b/examples/cim/ac/components/experience_shaper.py new file mode 100644 index 000000000..2941f2159 --- /dev/null +++ b/examples/cim/ac/components/experience_shaper.py @@ -0,0 +1,49 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +from collections import defaultdict + +import numpy as np + +from maro.rl import ExperienceShaper + + +class TruncatedExperienceShaper(ExperienceShaper): + def __init__(self, *, time_window: int, time_decay_factor: float, fulfillment_factor: float, + shortage_factor: float): + super().__init__(reward_func=None) + self._time_window = time_window + self._time_decay_factor = time_decay_factor + self._fulfillment_factor = fulfillment_factor + self._shortage_factor = shortage_factor + + def __call__(self, trajectory, snapshot_list): + experiences_by_agent = {} + for i in range(len(trajectory) - 1): + transition = trajectory[i] + agent_id = transition["agent_id"] + if agent_id not in experiences_by_agent: + experiences_by_agent[agent_id] = defaultdict(list) + experiences = experiences_by_agent[agent_id] + experiences["state"].append(transition["state"]) + experiences["action"].append(transition["action"]) + experiences["reward"].append(self._compute_reward(transition["event"], snapshot_list)) + experiences["next_state"].append(trajectory[i+1]["state"]) + + return experiences_by_agent + + def _compute_reward(self, decision_event, snapshot_list): + start_tick = decision_event.tick + 1 + end_tick = decision_event.tick + self._time_window + ticks = list(range(start_tick, end_tick)) + + # calculate tc reward + future_fulfillment = snapshot_list["ports"][ticks::"fulfillment"] + future_shortage = snapshot_list["ports"][ticks::"shortage"] + decay_list = [self._time_decay_factor ** i for i in range(end_tick - start_tick) + for _ in range(future_fulfillment.shape[0]//(end_tick-start_tick))] + + tot_fulfillment = np.dot(future_fulfillment, decay_list) + tot_shortage = np.dot(future_shortage, decay_list) + + return np.float(self._fulfillment_factor * tot_fulfillment - self._shortage_factor * tot_shortage) diff --git a/examples/cim/ac/components/state_shaper.py b/examples/cim/ac/components/state_shaper.py new file mode 100644 index 000000000..0e2af0ab3 --- /dev/null +++ b/examples/cim/ac/components/state_shaper.py @@ -0,0 +1,29 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import numpy as np + +from maro.rl import StateShaper + + +class CIMStateShaper(StateShaper): + def __init__(self, *, look_back, max_ports_downstream, port_attributes, vessel_attributes): + super().__init__() + self._look_back = look_back + self._max_ports_downstream = max_ports_downstream + self._port_attributes = port_attributes + self._vessel_attributes = vessel_attributes + self._dim = (look_back + 1) * (max_ports_downstream + 1) * len(port_attributes) + len(vessel_attributes) + + def __call__(self, decision_event, snapshot_list): + tick, port_idx, vessel_idx = decision_event.tick, decision_event.port_idx, decision_event.vessel_idx + ticks = [tick - rt for rt in range(self._look_back-1)] + future_port_idx_list = snapshot_list["vessels"][tick: vessel_idx: 'future_stop_list'].astype('int') + port_features = snapshot_list["ports"][ticks: [port_idx] + list(future_port_idx_list): self._port_attributes] + vessel_features = snapshot_list["vessels"][tick: vessel_idx: self._vessel_attributes] + state = np.concatenate((port_features, vessel_features)) + return str(port_idx), state + + @property + def dim(self): + return self._dim diff --git a/examples/cim/ac/config.yml b/examples/cim/ac/config.yml new file mode 100644 index 000000000..c3e205d8d --- /dev/null +++ b/examples/cim/ac/config.yml @@ -0,0 +1,74 @@ +env: + scenario: "cim" + topology: "toy.4p_ssdd_l0.0" + durations: 1120 +general: + total_training_episodes: 500 # max episode +state_shaping: + look_back: 7 + max_ports_downstream: 2 + port_attributes: + - "empty" + - "full" + - "on_shipper" + - "on_consignee" + - "booking" + - "shortage" + - "fulfillment" + vessel_attributes: + - "empty" + - "full" + - "remaining_space" +experience_shaping: + type: "truncated" + k_step: + reward_decay: 0.9 + steps: 5 + truncated: + time_window: 100 + fulfillment_factor: 1.0 + shortage_factor: 1.0 + time_decay_factor: 0.97 +exploration: + epsilon_range: [0.0, 0.4] + split_point: [0.5, 0.8] + with_cache: true +agents: + algorithm: + num_actions: 21 + policy_model: + hidden_dims: + - 256 + - 128 + - 64 + dropout_p: 0.0 + value_model: + hidden_dims: + - 256 + - 128 + - 64 + dropout_p: 0.0 + policy_optimizer: + lr: 0.001 + value_optimizer: + lr: 0.001 + hyper_parameters: + reward_decay: .0 + num_training_rounds_per_target_replacement: 5 + tau: 0.1 + experience_pool: + capacity: -1 + training_loop_parameters: + min_experiences_to_train: 1024 + num_batches: 10 # number of times the algorithm's step() method is called + batch_size: 128 + seed: 1024 # for reproducibility +distributed: + group_name: "dqn_distributed_test" + actor: + peer: {"actor": 1} + learner: + peer: {"actor_worker": 1} + redis: + host_name: "localhost" + port: 6379 diff --git a/examples/cim/ac/dist_actor.py b/examples/cim/ac/dist_actor.py new file mode 100644 index 000000000..350f493f2 --- /dev/null +++ b/examples/cim/ac/dist_actor.py @@ -0,0 +1,45 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import numpy as np + +from maro.simulator import Env +from maro.rl import AgentMode, SimpleActor, ActorWorker, KStepExperienceShaper, TwoPhaseLinearExplorer + +from components.action_shaper import CIMActionShaper +from components.agent_manager import DQNAgentManager +from components.config import config +from components.experience_shaper import TruncatedExperienceShaper +from components.state_shaper import CIMStateShaper + + +if __name__ == "__main__": + env = Env(config.env.scenario, config.env.topology, durations=config.env.durations) + agent_id_list = [str(agent_id) for agent_id in env.agent_idx_list] + state_shaper = CIMStateShaper(**config.state_shaping) + action_shaper = CIMActionShaper(action_space=list(np.linspace(-1.0, 1.0, config.agents.algorithm.num_actions))) + if config.experience_shaping.type == "truncated": + experience_shaper = TruncatedExperienceShaper(**config.experience_shaping.truncated) + else: + experience_shaper = KStepExperienceShaper(reward_func=lambda mt: 1-mt["container_shortage"]/mt["order_requirements"], + **config.experience_shaping.k_step) + + exploration_config = {"epsilon_range_dict": {"_all_": config.exploration.epsilon_range}, + "split_point_dict": {"_all_": config.exploration.split_point}, + "with_cache": config.exploration.with_cache + } + explorer = TwoPhaseLinearExplorer(agent_id_list, config.general.total_training_episodes, **exploration_config) + agent_manager = DQNAgentManager(name="cim_remote_actor", + agent_id_list=agent_id_list, + mode=AgentMode.INFERENCE, + state_shaper=state_shaper, + action_shaper=action_shaper, + experience_shaper=experience_shaper, + explorer=explorer) + proxy_params = {"group_name": config.distributed.group_name, + "expected_peers": config.distributed.actor.peer, + "redis_address": (config.distributed.redis.host_name, config.distributed.redis.port) + } + actor_worker = ActorWorker(local_actor=SimpleActor(env=env, inference_agents=agent_manager), + proxy_params=proxy_params) + actor_worker.launch() diff --git a/examples/cim/ac/dist_learner.py b/examples/cim/ac/dist_learner.py new file mode 100644 index 000000000..32c501e11 --- /dev/null +++ b/examples/cim/ac/dist_learner.py @@ -0,0 +1,36 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import os + +from maro.rl import ActorProxy, SimpleLearner, AgentMode, TwoPhaseLinearExplorer +from maro.simulator import Env +from maro.utils import Logger + +from components.agent_manager import DQNAgentManager +from components.config import config +from components.state_shaper import CIMStateShaper + + +if __name__ == "__main__": + env = Env(config.env.scenario, config.env.topology, durations=config.env.durations) + agent_id_list = [str(agent_id) for agent_id in env.agent_idx_list] + state_shaper = CIMStateShaper(**config.state_shaping) + exploration_config = {"epsilon_range_dict": {"_all_": config.exploration.epsilon_range}, + "split_point_dict": {"_all_": config.exploration.split_point}, + "with_cache": config.exploration.with_cache + } + explorer = TwoPhaseLinearExplorer(agent_id_list, config.general.total_training_episodes, **exploration_config) + agent_manager = DQNAgentManager(name="cim_remote_learner", agent_id_list=agent_id_list, mode=AgentMode.TRAIN, + state_shaper=state_shaper, explorer=explorer) + + proxy_params = {"group_name": config.distributed.group_name, + "expected_peers": config.distributed.learner.peer, + "redis_address": (config.distributed.redis.host_name, config.distributed.redis.port) + } + learner = SimpleLearner(trainable_agents=agent_manager, + actor=ActorProxy(proxy_params=proxy_params), + logger=Logger("distributed_cim_learner", auto_timestamp=False)) + learner.train(total_episodes=config.general.total_training_episodes, + model_dump_dir=os.path.join(os.getcwd(), "models")) + learner.test() diff --git a/examples/cim/ac/multi_process_launcher.py b/examples/cim/ac/multi_process_launcher.py new file mode 100644 index 000000000..9ec989550 --- /dev/null +++ b/examples/cim/ac/multi_process_launcher.py @@ -0,0 +1,23 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +""" +This script is used to debug distributed algorithm in single host multi-process mode. +""" + +import os + +from components.config import config + + +ACTOR_NUM = config.distributed.learner.peer["actor_worker"] # must be same as in config +LEARNER_NUM = config.distributed.actor.peer["actor"] + +learner_path = f"{os.path.split(os.path.realpath(__file__))[0]}/dist_learner.py &" +actor_path = f"{os.path.split(os.path.realpath(__file__))[0]}/dist_actor.py &" + +for l_num in range(LEARNER_NUM): + os.system(f"python " + learner_path) + +for a_num in range(ACTOR_NUM): + os.system(f"python " + actor_path) diff --git a/examples/cim/ac/single_process_launcher.py b/examples/cim/ac/single_process_launcher.py new file mode 100644 index 000000000..98c9fa34e --- /dev/null +++ b/examples/cim/ac/single_process_launcher.py @@ -0,0 +1,57 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + + +import os + +import numpy as np + +from maro.simulator import Env +from maro.rl import SimpleLearner, SimpleActor, AgentMode, KStepExperienceShaper, TwoPhaseLinearExplorer +from maro.utils import Logger + +from components.action_shaper import CIMActionShaper +from components.agent_manager import DQNAgentManager +from components.config import config +from components.experience_shaper import TruncatedExperienceShaper +from components.state_shaper import CIMStateShaper + + +if __name__ == "__main__": + # Step 1: initialize a CIM environment for using a toy dataset. + env = Env(config.env.scenario, config.env.topology, durations=config.env.durations) + agent_id_list = [str(agent_id) for agent_id in env.agent_idx_list] + + # Step 2: create state, action and experience shapers. We also need to create an explorer here due to the + # greedy nature of the DQN algorithm. + state_shaper = CIMStateShaper(**config.state_shaping) + action_shaper = CIMActionShaper(action_space=list(np.linspace(-1.0, 1.0, config.agents.algorithm.num_actions))) + if config.experience_shaping.type == "truncated": + experience_shaper = TruncatedExperienceShaper(**config.experience_shaping.truncated) + else: + experience_shaper = KStepExperienceShaper(reward_func=lambda mt: 1-mt["container_shortage"]/mt["order_requirements"], + **config.experience_shaping.k_step) + + exploration_config = {"epsilon_range_dict": {"_all_": config.exploration.epsilon_range}, + "split_point_dict": {"_all_": config.exploration.split_point}, + "with_cache": config.exploration.with_cache + } + explorer = TwoPhaseLinearExplorer(agent_id_list, config.general.total_training_episodes, **exploration_config) + + # Step 3: create an agent manager. + agent_manager = DQNAgentManager(name="cim_learner", + mode=AgentMode.TRAIN_INFERENCE, + agent_id_list=agent_id_list, + state_shaper=state_shaper, + action_shaper=action_shaper, + experience_shaper=experience_shaper, + explorer=explorer) + + # Step 4: Create an actor and a learner to start the training process. + actor = SimpleActor(env=env, inference_agents=agent_manager) + learner = SimpleLearner(trainable_agents=agent_manager, actor=actor, + logger=Logger("single_host_cim_learner", auto_timestamp=False)) + + learner.train(total_episodes=config.general.total_training_episodes, + model_dump_dir=os.path.join(os.getcwd(), "models")) + learner.test() diff --git a/examples/cim/pg/README.md b/examples/cim/pg/README.md new file mode 100644 index 000000000..816ed5052 --- /dev/null +++ b/examples/cim/pg/README.md @@ -0,0 +1,22 @@ +# Overview + +The CIM problem is one of the quintessential use cases of MARO. The example can +be run with a set of scenario configurations that can be found under +maro/simulator/scenarios/cim. General experimental parameters (e.g., type of +topology, type of algorithm to use, number of training episodes) can be configured +through config.yml. Each RL formulation has a dedicated folder, e.g., dqn, and +all algorithm-specific parameters can be configured through +the config.py file in that folder. + +## Single-host Single-process Mode + +To run the CIM example using the DQN algorithm under single-host mode, go to +examples/cim/dqn and run single_process_launcher.py. You may play around with +the configuration if you want to try out different settings. + +## Distributed Mode + +The examples/cim/dqn/components folder contains dist_learner.py and dist_actor.py +for distributed training. For debugging purposes, we provide a script that +simulates distributed mode using multi-processing. Simply go to examples/cim/dqn +and run multi_process_launcher.py to start the learner and actor processes. diff --git a/examples/cim/pg/components/__init__.py b/examples/cim/pg/components/__init__.py new file mode 100644 index 000000000..b14b47650 --- /dev/null +++ b/examples/cim/pg/components/__init__.py @@ -0,0 +1,2 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. diff --git a/examples/cim/pg/components/action_shaper.py b/examples/cim/pg/components/action_shaper.py new file mode 100644 index 000000000..687d18d88 --- /dev/null +++ b/examples/cim/pg/components/action_shaper.py @@ -0,0 +1,33 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +from maro.rl import ActionShaper +from maro.simulator.scenarios.cim.common import Action + + +class CIMActionShaper(ActionShaper): + def __init__(self, action_space): + super().__init__() + self._action_space = action_space + self._zero_action_index = action_space.index(0) + + def __call__(self, model_action, decision_event, snapshot_list): + scope = decision_event.action_scope + tick = decision_event.tick + port_idx = decision_event.port_idx + vessel_idx = decision_event.vessel_idx + + port_empty = snapshot_list["ports"][tick: port_idx: ["empty", "full", "on_shipper", "on_consignee"]][0] + vessel_remaining_space = snapshot_list["vessels"][tick: vessel_idx: ["empty", "full", "remaining_space"]][2] + early_discharge = snapshot_list["vessels"][tick:vessel_idx: "early_discharge"][0] + assert 0 <= model_action < len(self._action_space) + + if model_action < self._zero_action_index: + actual_action = max(round(self._action_space[model_action] * port_empty), -vessel_remaining_space) + elif model_action > self._zero_action_index: + plan_action = self._action_space[model_action] * (scope.discharge + early_discharge) - early_discharge + actual_action = round(plan_action) if plan_action > 0 else round(self._action_space[model_action] * scope.discharge) + else: + actual_action = 0 + + return Action(vessel_idx, port_idx, actual_action) diff --git a/examples/cim/pg/components/agent.py b/examples/cim/pg/components/agent.py new file mode 100644 index 000000000..6f322fe59 --- /dev/null +++ b/examples/cim/pg/components/agent.py @@ -0,0 +1,18 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import numpy as np + +from maro.rl import AbsAgent, ColumnBasedStore + + +class CIMAgent(AbsAgent): + def train(self, ): + for _ in range(self._num_batches): + indexes, sample = self._experience_pool.sample_by_key("loss", self._batch_size) + state = np.asarray(sample["state"]) + action = np.asarray(sample["action"]) + reward = np.asarray(sample["reward"]) + next_state = np.asarray(sample["next_state"]) + loss = self._algorithm.train(state, action, reward, next_state) + self._experience_pool.update(indexes, {"loss": loss}) diff --git a/examples/cim/pg/components/agent_manager.py b/examples/cim/pg/components/agent_manager.py new file mode 100644 index 000000000..e31897630 --- /dev/null +++ b/examples/cim/pg/components/agent_manager.py @@ -0,0 +1,52 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +from torch.nn.functional import smooth_l1_loss +from torch.optim import Adam, RMSprop + +from .agent import CIMAgent +from .config import config +from maro.rl import AbsAgentManager, LearningModel, MLPDecisionLayers, PolicyGradient, PolicyGradientHyperParameters +from maro.utils import set_seeds + + +class PGAgentManager(AbsAgentManager): + def _assemble(self, agent_dict): + set_seeds(config.agents.seed) + num_actions = config.agents.algorithm.num_actions + for agent_id in self._agent_id_list: + policy_model = LearningModel(decision_layers=MLPDecisionLayers(name=f'{agent_id}.policy', + input_dim=self._state_shaper.dim, + output_dim=num_actions, + **config.agents.algorithm.policy_model, + softmax=True)) + + algorithm = PolicyGradient( + policy_model=policy_model, + optimizer_cls=Adam, + optimizer_params=config.agents.algorithm.policy_optimizer, + hyper_params=PolicyGradientHyperParameters( + num_actions=num_actions, + **config.agents.algorithm.hyper_parameters, + ) + ) + + agent_dict[agent_id] = CIMAgent(name=agent_id, algorithm=algorithm) + + def choose_action(self, decision_event, snapshot_list): + self._assert_inference_mode() + agent_id, model_state = self._state_shaper(decision_event, snapshot_list) + model_action = self._agent_dict[agent_id].choose_action( + model_state, self._explorer.epsilon[agent_id] if self._explorer else None) + + self._transition_cache = {"state": model_state, + "action": model_action, + "reward": None, + "agent_id": agent_id, + "event": decision_event} + return self._action_shaper(model_action, decision_event, snapshot_list) + + def store_experiences(self, experiences): + for agent_id, exp in experiences.items(): + exp.update({"loss": [1e8] * len(exp[next(iter(exp))])}) + self._agent_dict[agent_id].store_experiences(exp) diff --git a/examples/cim/pg/components/config.py b/examples/cim/pg/components/config.py new file mode 100644 index 000000000..d36d60ddd --- /dev/null +++ b/examples/cim/pg/components/config.py @@ -0,0 +1,18 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +""" +This file is used to load config and convert it into a dotted dictionary. +""" + +import io +import os +import yaml + +from maro.utils import convert_dottable + + +CONFIG_PATH = os.path.join(os.path.split(os.path.realpath(__file__))[0], "../config.yml") +with io.open(CONFIG_PATH, "r") as in_file: + raw_config = yaml.safe_load(in_file) + config = convert_dottable(raw_config) diff --git a/examples/cim/pg/components/experience_shaper.py b/examples/cim/pg/components/experience_shaper.py new file mode 100644 index 000000000..2cfd4053d --- /dev/null +++ b/examples/cim/pg/components/experience_shaper.py @@ -0,0 +1,48 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +from collections import defaultdict + +import numpy as np + +from maro.rl import ExperienceShaper + + +class TruncatedExperienceShaper(ExperienceShaper): + def __init__(self, *, time_window: int, time_decay_factor: float, fulfillment_factor: float, + shortage_factor: float): + super().__init__(reward_func=None) + self._time_window = time_window + self._time_decay_factor = time_decay_factor + self._fulfillment_factor = fulfillment_factor + self._shortage_factor = shortage_factor + + def __call__(self, trajectory, snapshot_list): + agent_ids = np.asarray(trajectory.get_by_key["agent_id"]) + states = np.asarray(trajectory.get_by_key["state"]) + actions = np.asarray(trajectory.get_by_key["action"]) + rewards = np.fromiter( + map(self._compute_reward, trajectory.get_by_key("event"), [snapshot_list]*len(trajectory)), + dtype=np.float32 + ) + return {agent_id: {"state": states[agent_ids == agent_id], + "action": actions[agent_ids == agent_id], + "reward": rewards[agent_ids == agent_id], + } + for agent_id in set(agent_ids)} + + def _compute_reward(self, decision_event, snapshot_list): + start_tick = decision_event.tick + 1 + end_tick = decision_event.tick + self._time_window + ticks = list(range(start_tick, end_tick)) + + # calculate tc reward + future_fulfillment = snapshot_list["ports"][ticks::"fulfillment"] + future_shortage = snapshot_list["ports"][ticks::"shortage"] + decay_list = [self._time_decay_factor ** i for i in range(end_tick - start_tick) + for _ in range(future_fulfillment.shape[0]//(end_tick-start_tick))] + + tot_fulfillment = np.dot(future_fulfillment, decay_list) + tot_shortage = np.dot(future_shortage, decay_list) + + return np.float(self._fulfillment_factor * tot_fulfillment - self._shortage_factor * tot_shortage) diff --git a/examples/cim/pg/components/state_shaper.py b/examples/cim/pg/components/state_shaper.py new file mode 100644 index 000000000..0e2af0ab3 --- /dev/null +++ b/examples/cim/pg/components/state_shaper.py @@ -0,0 +1,29 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import numpy as np + +from maro.rl import StateShaper + + +class CIMStateShaper(StateShaper): + def __init__(self, *, look_back, max_ports_downstream, port_attributes, vessel_attributes): + super().__init__() + self._look_back = look_back + self._max_ports_downstream = max_ports_downstream + self._port_attributes = port_attributes + self._vessel_attributes = vessel_attributes + self._dim = (look_back + 1) * (max_ports_downstream + 1) * len(port_attributes) + len(vessel_attributes) + + def __call__(self, decision_event, snapshot_list): + tick, port_idx, vessel_idx = decision_event.tick, decision_event.port_idx, decision_event.vessel_idx + ticks = [tick - rt for rt in range(self._look_back-1)] + future_port_idx_list = snapshot_list["vessels"][tick: vessel_idx: 'future_stop_list'].astype('int') + port_features = snapshot_list["ports"][ticks: [port_idx] + list(future_port_idx_list): self._port_attributes] + vessel_features = snapshot_list["vessels"][tick: vessel_idx: self._vessel_attributes] + state = np.concatenate((port_features, vessel_features)) + return str(port_idx), state + + @property + def dim(self): + return self._dim diff --git a/examples/cim/pg/config.yml b/examples/cim/pg/config.yml new file mode 100644 index 000000000..839983f35 --- /dev/null +++ b/examples/cim/pg/config.yml @@ -0,0 +1,72 @@ +env: + scenario: "cim" + topology: "toy.4p_ssdd_l0.0" + durations: 1120 +general: + total_training_episodes: 500 # max episode +state_shaping: + look_back: 7 + max_ports_downstream: 2 + port_attributes: + - "empty" + - "full" + - "on_shipper" + - "on_consignee" + - "booking" + - "shortage" + - "fulfillment" + vessel_attributes: + - "empty" + - "full" + - "remaining_space" +experience_shaping: + type: "truncated" + k_step: + reward_decay: 0.9 + steps: 5 + truncated: + time_window: 100 + fulfillment_factor: 1.0 + shortage_factor: 1.0 + time_decay_factor: 0.97 +exploration: + epsilon_range: [0.0, 0.4] + split_point: [0.5, 0.8] + with_cache: true +agents: + algorithm: + num_actions: 21 + policy_model: + hidden_dims: + - 256 + - 128 + - 64 + dropout_p: 0.0 + value_model: + hidden_dims: + - 256 + - 128 + - 64 + dropout_p: 0.0 + policy_optimizer: + lr: 0.001 + value_optimizer: + lr: 0.001 + hyper_parameters: + reward_decay: .0 + experience_pool: + capacity: -1 + training_loop_parameters: + min_experiences_to_train: 1024 + num_batches: 10 # number of times the algorithm's step() method is called + batch_size: 128 + seed: 1024 # for reproducibility +distributed: + group_name: "dqn_distributed_test" + actor: + peer: {"actor": 1} + learner: + peer: {"actor_worker": 1} + redis: + host_name: "localhost" + port: 6379 diff --git a/examples/cim/pg/dist_actor.py b/examples/cim/pg/dist_actor.py new file mode 100644 index 000000000..350f493f2 --- /dev/null +++ b/examples/cim/pg/dist_actor.py @@ -0,0 +1,45 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import numpy as np + +from maro.simulator import Env +from maro.rl import AgentMode, SimpleActor, ActorWorker, KStepExperienceShaper, TwoPhaseLinearExplorer + +from components.action_shaper import CIMActionShaper +from components.agent_manager import DQNAgentManager +from components.config import config +from components.experience_shaper import TruncatedExperienceShaper +from components.state_shaper import CIMStateShaper + + +if __name__ == "__main__": + env = Env(config.env.scenario, config.env.topology, durations=config.env.durations) + agent_id_list = [str(agent_id) for agent_id in env.agent_idx_list] + state_shaper = CIMStateShaper(**config.state_shaping) + action_shaper = CIMActionShaper(action_space=list(np.linspace(-1.0, 1.0, config.agents.algorithm.num_actions))) + if config.experience_shaping.type == "truncated": + experience_shaper = TruncatedExperienceShaper(**config.experience_shaping.truncated) + else: + experience_shaper = KStepExperienceShaper(reward_func=lambda mt: 1-mt["container_shortage"]/mt["order_requirements"], + **config.experience_shaping.k_step) + + exploration_config = {"epsilon_range_dict": {"_all_": config.exploration.epsilon_range}, + "split_point_dict": {"_all_": config.exploration.split_point}, + "with_cache": config.exploration.with_cache + } + explorer = TwoPhaseLinearExplorer(agent_id_list, config.general.total_training_episodes, **exploration_config) + agent_manager = DQNAgentManager(name="cim_remote_actor", + agent_id_list=agent_id_list, + mode=AgentMode.INFERENCE, + state_shaper=state_shaper, + action_shaper=action_shaper, + experience_shaper=experience_shaper, + explorer=explorer) + proxy_params = {"group_name": config.distributed.group_name, + "expected_peers": config.distributed.actor.peer, + "redis_address": (config.distributed.redis.host_name, config.distributed.redis.port) + } + actor_worker = ActorWorker(local_actor=SimpleActor(env=env, inference_agents=agent_manager), + proxy_params=proxy_params) + actor_worker.launch() diff --git a/examples/cim/pg/dist_learner.py b/examples/cim/pg/dist_learner.py new file mode 100644 index 000000000..32c501e11 --- /dev/null +++ b/examples/cim/pg/dist_learner.py @@ -0,0 +1,36 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import os + +from maro.rl import ActorProxy, SimpleLearner, AgentMode, TwoPhaseLinearExplorer +from maro.simulator import Env +from maro.utils import Logger + +from components.agent_manager import DQNAgentManager +from components.config import config +from components.state_shaper import CIMStateShaper + + +if __name__ == "__main__": + env = Env(config.env.scenario, config.env.topology, durations=config.env.durations) + agent_id_list = [str(agent_id) for agent_id in env.agent_idx_list] + state_shaper = CIMStateShaper(**config.state_shaping) + exploration_config = {"epsilon_range_dict": {"_all_": config.exploration.epsilon_range}, + "split_point_dict": {"_all_": config.exploration.split_point}, + "with_cache": config.exploration.with_cache + } + explorer = TwoPhaseLinearExplorer(agent_id_list, config.general.total_training_episodes, **exploration_config) + agent_manager = DQNAgentManager(name="cim_remote_learner", agent_id_list=agent_id_list, mode=AgentMode.TRAIN, + state_shaper=state_shaper, explorer=explorer) + + proxy_params = {"group_name": config.distributed.group_name, + "expected_peers": config.distributed.learner.peer, + "redis_address": (config.distributed.redis.host_name, config.distributed.redis.port) + } + learner = SimpleLearner(trainable_agents=agent_manager, + actor=ActorProxy(proxy_params=proxy_params), + logger=Logger("distributed_cim_learner", auto_timestamp=False)) + learner.train(total_episodes=config.general.total_training_episodes, + model_dump_dir=os.path.join(os.getcwd(), "models")) + learner.test() diff --git a/examples/cim/pg/multi_process_launcher.py b/examples/cim/pg/multi_process_launcher.py new file mode 100644 index 000000000..9ec989550 --- /dev/null +++ b/examples/cim/pg/multi_process_launcher.py @@ -0,0 +1,23 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +""" +This script is used to debug distributed algorithm in single host multi-process mode. +""" + +import os + +from components.config import config + + +ACTOR_NUM = config.distributed.learner.peer["actor_worker"] # must be same as in config +LEARNER_NUM = config.distributed.actor.peer["actor"] + +learner_path = f"{os.path.split(os.path.realpath(__file__))[0]}/dist_learner.py &" +actor_path = f"{os.path.split(os.path.realpath(__file__))[0]}/dist_actor.py &" + +for l_num in range(LEARNER_NUM): + os.system(f"python " + learner_path) + +for a_num in range(ACTOR_NUM): + os.system(f"python " + actor_path) diff --git a/examples/cim/pg/single_process_launcher.py b/examples/cim/pg/single_process_launcher.py new file mode 100644 index 000000000..98c9fa34e --- /dev/null +++ b/examples/cim/pg/single_process_launcher.py @@ -0,0 +1,57 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + + +import os + +import numpy as np + +from maro.simulator import Env +from maro.rl import SimpleLearner, SimpleActor, AgentMode, KStepExperienceShaper, TwoPhaseLinearExplorer +from maro.utils import Logger + +from components.action_shaper import CIMActionShaper +from components.agent_manager import DQNAgentManager +from components.config import config +from components.experience_shaper import TruncatedExperienceShaper +from components.state_shaper import CIMStateShaper + + +if __name__ == "__main__": + # Step 1: initialize a CIM environment for using a toy dataset. + env = Env(config.env.scenario, config.env.topology, durations=config.env.durations) + agent_id_list = [str(agent_id) for agent_id in env.agent_idx_list] + + # Step 2: create state, action and experience shapers. We also need to create an explorer here due to the + # greedy nature of the DQN algorithm. + state_shaper = CIMStateShaper(**config.state_shaping) + action_shaper = CIMActionShaper(action_space=list(np.linspace(-1.0, 1.0, config.agents.algorithm.num_actions))) + if config.experience_shaping.type == "truncated": + experience_shaper = TruncatedExperienceShaper(**config.experience_shaping.truncated) + else: + experience_shaper = KStepExperienceShaper(reward_func=lambda mt: 1-mt["container_shortage"]/mt["order_requirements"], + **config.experience_shaping.k_step) + + exploration_config = {"epsilon_range_dict": {"_all_": config.exploration.epsilon_range}, + "split_point_dict": {"_all_": config.exploration.split_point}, + "with_cache": config.exploration.with_cache + } + explorer = TwoPhaseLinearExplorer(agent_id_list, config.general.total_training_episodes, **exploration_config) + + # Step 3: create an agent manager. + agent_manager = DQNAgentManager(name="cim_learner", + mode=AgentMode.TRAIN_INFERENCE, + agent_id_list=agent_id_list, + state_shaper=state_shaper, + action_shaper=action_shaper, + experience_shaper=experience_shaper, + explorer=explorer) + + # Step 4: Create an actor and a learner to start the training process. + actor = SimpleActor(env=env, inference_agents=agent_manager) + learner = SimpleLearner(trainable_agents=agent_manager, actor=actor, + logger=Logger("single_host_cim_learner", auto_timestamp=False)) + + learner.train(total_episodes=config.general.total_training_episodes, + model_dump_dir=os.path.join(os.getcwd(), "models")) + learner.test() diff --git a/maro/rl/__init__.py b/maro/rl/__init__.py index dd0f397ba..085acbc7e 100644 --- a/maro/rl/__init__.py +++ b/maro/rl/__init__.py @@ -8,6 +8,11 @@ from maro.rl.agent.abs_agent import AbsAgent from maro.rl.agent.abs_agent_manager import AbsAgentManager, AgentMode from maro.rl.algorithms.abs_algorithm import AbsAlgorithm +from maro.rl.algorithms.torch.pg import PolicyGradient, PolicyGradientHyperParameters +from maro.rl.algorithms.torch.ac import ActorCritic, ActorCriticWithCombinedModel, ActorCriticHyperParameters, \ + ActorCriticHyperParametersWithCombinedModel +from maro.rl.algorithms.torch.ppo import PPO, PPOWithCombinedModel, PPOHyperParameters, \ + PPOHyperParametersWithCombinedModel from maro.rl.algorithms.torch.dqn import DQN, DQNHyperParams from maro.rl.models.torch.mlp_representation import MLPRepresentation from maro.rl.models.torch.decision_layers import MLPDecisionLayers @@ -34,6 +39,16 @@ "AbsAgentManager", "AgentMode", "AbsAlgorithm", + "PolicyGradient", + "PolicyGradientHyperParameters", + "ActorCritic", + "ActorCriticWithCombinedModel", + "ActorCriticHyperParameters", + "ActorCriticHyperParametersWithCombinedModel", + "PPO", + "PPOWithCombinedModel", + "PPOHyperParameters", + "PPOHyperParametersWithCombinedModel", "DQN", "DQNHyperParams", "MLPRepresentation", diff --git a/maro/rl/agent/abs_agent.py b/maro/rl/agent/abs_agent.py index 907b62b87..1c5875bd2 100644 --- a/maro/rl/agent/abs_agent.py +++ b/maro/rl/agent/abs_agent.py @@ -5,8 +5,6 @@ import os import pickle -import torch - from maro.rl.algorithms.abs_algorithm import AbsAlgorithm from maro.rl.storage.abs_store import AbsStore @@ -25,12 +23,13 @@ class AbsAgent(ABC): algorithm (AbsAlgorithm): A concrete algorithm instance that inherits from AbstractAlgorithm. This is the centerpiece of the Agent class and is responsible for the most important tasks of an agent: choosing actions and optimizing models. - experience_pool (AbsStore): A data store that stores experiences generated by the experience shaper. + experience_pool (AbsStore): A data store that stores experiences generated by the experience shaper. This is + only necessary for some algorithms. Defaults to None. """ def __init__(self, name: str, algorithm: AbsAlgorithm, - experience_pool: AbsStore + experience_pool: AbsStore = None ): self._name = name self._algorithm = algorithm @@ -68,7 +67,8 @@ def train(self): def store_experiences(self, experiences): """Store new experiences in the experience pool.""" - self._experience_pool.put(experiences) + if self._experience_pool is not None: + self._experience_pool.put(experiences) def load_trainable_models(self, *models, **model_dict): """Load models from memory.""" @@ -100,5 +100,6 @@ def dump_trainable_models_to_file(self, dir_path: str): def dump_experience_store(self, dir_path: str): """Dump the experience pool to disk.""" - with open(os.path.join(dir_path, self._name)) as fp: - pickle.dump(self._experience_pool, fp) + if self._experience_pool is not None: + with open(os.path.join(dir_path, self._name)) as fp: + pickle.dump(self._experience_pool, fp) diff --git a/maro/rl/algorithms/torch/ac.py b/maro/rl/algorithms/torch/ac.py index 7c105277b..7d296903e 100644 --- a/maro/rl/algorithms/torch/ac.py +++ b/maro/rl/algorithms/torch/ac.py @@ -76,20 +76,21 @@ def choose_action(self, state: np.ndarray, epsilon: float = None): action_dist = self._policy_model(state).squeeze().numpy() # (num_actions,) return np.random.choice(self._hyper_params.num_actions, p=action_dist) - def _get_bootstrapped_returns_and_advantages(self, states: torch.tensor, rewards: np.ndarray): - state_values = self._value_model(states).detach() + def _get_values_and_bootstrapped_returns(self, state_sequence, reward_sequence): + state_values = self._value_model(state_sequence).detach() state_values_numpy = state_values.numpy() return_est = get_lambda_returns( - rewards, self._hyper_params.reward_decay, self._hyper_params.lamb, + reward_sequence, self._hyper_params.reward_decay, self._hyper_params.lamb, k=self._hyper_params.k, values=state_values_numpy ) return_est = torch.from_numpy(return_est) - return return_est, return_est - state_values + return state_values, return_est - def train(self, state_sequence: np.ndarray, action_sequence: np.ndarray, reward_sequence: np.ndarray): - states = torch.from_numpy(state_sequence).to(self._device) - actions = torch.from_numpy(action_sequence).to(self._device) - return_est, advantages = self._get_bootstrapped_returns_and_advantages(states, reward_sequence) + def train(self, states: np.ndarray, actions: torch.tensor, rewards: np.ndarray): + states = torch.from_numpy(states).to(self._device) + state_values, return_est = self._get_values_and_bootstrapped_returns(states, rewards) + advantages = return_est - state_values + actions = torch.from_numpy(actions).to(self._device) # policy model training for _ in range(self._hyper_params.policy_train_iters): action_prob = self._policy_model(states).gather(1, actions.unsqueeze(1)).squeeze() # (N,) @@ -177,24 +178,24 @@ def choose_action(self, state: np.ndarray, epsilon: float = None): action_dist = self._policy_value_model(state)[1].squeeze().numpy() # (num_actions,) return np.random.choice(self._hyper_params.num_actions, p=action_dist) - def _get_bootstrapped_returns_and_advantages(self, states: torch.tensor, reward_sequence: np.ndarray): - state_values = self._policy_value_model(states)[0].detach() + def _get_values_and_bootstrapped_returns(self, state_sequence, reward_sequence): + state_values = self._policy_value_model(state_sequence)[0].detach() state_values_numpy = state_values.numpy() return_est = get_lambda_returns( reward_sequence, self._hyper_params.reward_decay, self._hyper_params.lamb, k=self._hyper_params.k, values=state_values_numpy ) return_est = torch.from_numpy(return_est) - return return_est, return_est - state_values + return state_values, return_est - def train(self, state_sequence: np.ndarray, action_sequence: np.ndarray, reward_sequence: np.ndarray): - states = torch.from_numpy(state_sequence).to(self._device) - actions = torch.from_numpy(action_sequence).to(self._device) - return_est, advantages = self._get_bootstrapped_returns_and_advantages(states, reward_sequence) + def train(self, states: np.ndarray, actions: np.ndarray, rewards: np.ndarray): + states = torch.from_numpy(states).to(self._device) + state_values, return_est = self._get_values_and_bootstrapped_returns(states, rewards) + advantages = return_est - state_values + actions = torch.from_numpy(actions).to(self._device) # policy-value model training for _ in range(self._hyper_params.train_iters): state_values, action_distribution = self._policy_value_model(states) - advantages = return_est - state_values action_prob = action_distribution.gather(1, actions.unsqueeze(1)).squeeze() # (N,) policy_loss = -(torch.log(action_prob) * advantages).mean() value_loss = self._value_loss_func(state_values, return_est) diff --git a/maro/rl/algorithms/torch/pg.py b/maro/rl/algorithms/torch/pg.py index ce5f880eb..de17e594d 100644 --- a/maro/rl/algorithms/torch/pg.py +++ b/maro/rl/algorithms/torch/pg.py @@ -54,10 +54,10 @@ def choose_action(self, state: np.ndarray, epsilon: float = None): action_dist = self._policy_model(state).squeeze().numpy() # (num_actions,) return np.random.choice(self._hyper_params.num_actions, p=action_dist) - def train(self, state_sequence: np.ndarray, action_sequence: np.ndarray, reward_sequence: np.ndarray): - states = torch.from_numpy(state_sequence).to(self._device) # (N, state_dim) - returns = get_k_step_returns(reward_sequence, self._hyper_params.reward_decay) - actions = torch.from_numpy(action_sequence).to(self._device) # (N,) + def train(self, states: np.ndarray, actions: np.ndarray, returns: np.ndarray): + states = torch.from_numpy(states).to(self._device) # (N, state_dim) + actions = torch.from_numpy(actions).to(self._device) # (N,) + returns = torch.from_numpy(returns).to(self._device) action_prob = self._policy_model(states).gather(1, actions.unsqueeze(1)).squeeze() # (N, 1) policy_loss = -(torch.log(action_prob) * returns).mean() self._policy_optimizer.zero_grad() diff --git a/maro/rl/algorithms/torch/ppo.py b/maro/rl/algorithms/torch/ppo.py index 854123b3b..c056e717e 100644 --- a/maro/rl/algorithms/torch/ppo.py +++ b/maro/rl/algorithms/torch/ppo.py @@ -74,7 +74,7 @@ def choose_action(self, state: np.ndarray, epsilon: float = None): action_dist = self._policy_model(state).squeeze().numpy() # (num_actions,) return np.random.choice(self._hyper_params.num_actions, p=action_dist) - def _get_bootstrapped_returns_and_advantages(self, states: torch.tensor, rewards: np.ndarray): + def _get_values_and_bootstrapped_returns(self, states: torch.tensor, rewards: np.ndarray): state_values = self._value_model(states).detach() state_values_numpy = state_values.numpy() return_est = get_lambda_returns( @@ -82,16 +82,16 @@ def _get_bootstrapped_returns_and_advantages(self, states: torch.tensor, rewards k=self._hyper_params.k, values=state_values_numpy ) return_est = torch.from_numpy(return_est) - return return_est, return_est - state_values + return state_values, return_est def train( - self, state_sequence: np.ndarray, action_sequence: np.ndarray, log_action_prob_sequence: np.ndarray, - reward_sequence: np.ndarray + self, states: np.ndarray, actions: np.ndarray, log_action_prob: np.ndarray, rewards: np.ndarray ): - states = torch.from_numpy(state_sequence).to(self._device) # (N, state_dim) - actions = torch.from_numpy(action_sequence).to(self._device) # (N,) - return_est, advantages = self._get_bootstrapped_returns_and_advantages(states, reward_sequence) - log_action_prob_old = torch.from_numpy(log_action_prob_sequence).to(self._device) + states = torch.from_numpy(states).to(self._device) # (N, state_dim) + state_values, return_est = self._get_values_and_bootstrapped_returns(states, rewards) + advantages = return_est - state_values + actions = torch.from_numpy(actions).to(self._device) # (N,) + log_action_prob_old = torch.from_numpy(log_action_prob).to(self._device) # policy model training (with the value model fixed) for _ in range(self._hyper_params.policy_train_iters): @@ -188,24 +188,24 @@ def choose_action(self, state: np.ndarray, epsilon: float = None): action_index = np.random.choice(self._hyper_params.num_actions, p=action_dist) return action_index, np.log(action_dist[action_index]) - def _get_bootstrapped_returns_and_advantages(self, states: torch.tensor, rewards: np.ndarray): - state_values = self._policy_value_model(states)[0].detach() + def _get_values_and_bootstrapped_returns(self, state_sequence, reward_sequence): + state_values = self._policy_value_model(state_sequence)[0].detach() state_values_numpy = state_values.numpy() return_est = get_lambda_returns( - rewards, self._hyper_params.reward_decay, self._hyper_params.lamb, + reward_sequence, self._hyper_params.reward_decay, self._hyper_params.lamb, k=self._hyper_params.k, values=state_values_numpy ) return_est = torch.from_numpy(return_est) - return return_est, return_est - state_values + return state_values, return_est def train( - self, state_sequence: np.ndarray, action_sequence: np.ndarray, log_action_prob_sequence: np.ndarray, - reward_sequence: np.ndarray + self, states: np.ndarray, actions: np.ndarray, log_action_prob: np.ndarray, rewards: np.ndarray ): - states = torch.from_numpy(state_sequence).to(self._device) # (N, state_dim) - actions = torch.from_numpy(action_sequence).to(self._device) # (N,) - return_est, advantages = self._get_bootstrapped_returns_and_advantages(states, reward_sequence) - log_action_prob_old = torch.from_numpy(log_action_prob_sequence).to(self._device) + states = torch.from_numpy(states).to(self._device) # (N, state_dim) + state_values, return_est = self._get_values_and_bootstrapped_returns(states, rewards) + advantages = return_est - state_values + actions = torch.from_numpy(actions).to(self._device) # (N,) + log_action_prob_old = torch.from_numpy(log_action_prob).to(self._device) for _ in range(self._hyper_params.train_iters): state_values, action_distribution = self._policy_value_model(states) action_prob = action_distribution.gather(1, actions.unsqueeze(1)).squeeze() # (N,) diff --git a/maro/rl/models/torch/decision_layers.py b/maro/rl/models/torch/decision_layers.py index 36c62f736..715f89f6a 100644 --- a/maro/rl/models/torch/decision_layers.py +++ b/maro/rl/models/torch/decision_layers.py @@ -16,8 +16,13 @@ class MLPDecisionLayers(nn.Module): hidden layer number, which requires larger than 1. output_dim (int): Network output dimension. dropout_p (float): Dropout parameter. + softmax (bool): If true, the output of the net will be a softmax transformation of the top layer's output. + Defaults to False. """ - def __init__(self, *, name: str, input_dim: int, output_dim: int, hidden_dims: [int], dropout_p: float): + def __init__( + self, *, name: str, input_dim: int, output_dim: int, hidden_dims: [int], dropout_p: float, + softmax: bool = False + ): super().__init__() self._name = name self._input_dim = input_dim @@ -30,9 +35,11 @@ def __init__(self, *, name: str, input_dim: int, output_dim: int, hidden_dims: [ else: self._head = nn.Linear(hidden_dims[-1], self._output_dim) self._net = nn.Sequential(*self._layers, self._head) + self._softmax = nn.Softmax(dim=1) if softmax else None def forward(self, x): - return self._net(x).double() + out = self._net(x).double() + return self._softmax(out) if self._softmax else out @property def input_dim(self): diff --git a/maro/rl/shaping/k_step_experience_shaper.py b/maro/rl/shaping/k_step_experience_shaper.py index 33e5a7428..87cd505e1 100644 --- a/maro/rl/shaping/k_step_experience_shaper.py +++ b/maro/rl/shaping/k_step_experience_shaper.py @@ -40,7 +40,7 @@ def __call__(self, trajectory, snapshot_list): states = np.asarray(trajectory.get_by_key["state"]) actions = np.asarray(trajectory.get_by_key["action"]) reward_array = np.fromiter(map(self._reward_func, trajectory.get_by_key("metrics")), dtype=np.float32) - reward_sums = get_k_step_returns(reward_array, self._reward_decay, k=self._steps)[:-1] + reward_sums = get_k_step_returns(reward_array, self._reward_decay, k=self._steps) discounts = np.array([self._reward_decay ** min(self._steps, length-i-1) for i in range(length-1)]) next_states = np.pad(states[self._steps:], (0, length-self._steps-1), mode="edge") next_actions = np.pad(actions[self._steps:], (0, length-self._steps-1), mode="edge") diff --git a/maro/rl/storage/column_based_store.py b/maro/rl/storage/column_based_store.py index 935117de5..38570f79e 100644 --- a/maro/rl/storage/column_based_store.py +++ b/maro/rl/storage/column_based_store.py @@ -53,6 +53,11 @@ def __next__(self): def __getitem__(self, index: int): return {k: lst[index] for k, lst in self._store.items()} + def __getstate__(self): + obj_dict = self.__dict__ + obj_dict["_store"] = dict(obj_dict["_store"]) + return obj_dict + @property def capacity(self): """Store capacity. From 99b397ab33df495713faeac93cbcadd1a291b89e Mon Sep 17 00:00:00 2001 From: ysqyang Date: Sun, 18 Oct 2020 14:53:44 +0800 Subject: [PATCH 017/337] finished ac and pg for cim --- examples/cim/ac/components/agent.py | 23 +----- examples/cim/ac/components/agent_manager.py | 74 +++++++++++-------- examples/cim/ac/dist_actor.py | 45 +++++------ examples/cim/ac/dist_learner.py | 37 +++++----- examples/cim/ac/single_process_launcher.py | 43 +++++------ examples/cim/dqn/components/agent_manager.py | 33 ++++++--- examples/cim/dqn/dist_actor.py | 46 +++++++----- examples/cim/dqn/dist_learner.py | 39 ++++++---- examples/cim/dqn/single_process_launcher.py | 44 ++++++----- examples/cim/pg/components/agent.py | 13 +--- examples/cim/pg/components/agent_manager.py | 12 +-- .../cim/pg/components/experience_shaper.py | 6 +- examples/cim/pg/dist_actor.py | 42 +++++------ examples/cim/pg/dist_learner.py | 39 ++++++---- examples/cim/pg/single_process_launcher.py | 45 +++++------ maro/rl/agent/abs_agent.py | 2 +- maro/rl/agent/abs_agent_manager.py | 12 ++- maro/rl/algorithms/torch/ac.py | 2 +- 18 files changed, 282 insertions(+), 275 deletions(-) diff --git a/examples/cim/ac/components/agent.py b/examples/cim/ac/components/agent.py index 7f6f24255..39d0e994d 100644 --- a/examples/cim/ac/components/agent.py +++ b/examples/cim/ac/components/agent.py @@ -3,26 +3,9 @@ import numpy as np -from maro.rl import AbsAgent, ColumnBasedStore +from maro.rl import AbsAgent class CIMAgent(AbsAgent): - def __init__(self, name, algorithm, experience_pool: ColumnBasedStore, min_experiences_to_train, - num_batches, batch_size): - super().__init__(name, algorithm, experience_pool) - self._min_experiences_to_train = min_experiences_to_train - self._num_batches = num_batches - self._batch_size = batch_size - - def train(self): - if len(self._experience_pool) < self._min_experiences_to_train: - return - - for _ in range(self._num_batches): - indexes, sample = self._experience_pool.sample_by_key("loss", self._batch_size) - state = np.asarray(sample["state"]) - action = np.asarray(sample["action"]) - reward = np.asarray(sample["reward"]) - next_state = np.asarray(sample["next_state"]) - loss = self._algorithm.train(state, action, reward, next_state) - self._experience_pool.update(indexes, {"loss": loss}) + def train(self, states: np.ndarray, actions: np.ndarray, rewards: np.ndarray): + self._algorithm.train(states, actions, rewards) diff --git a/examples/cim/ac/components/agent_manager.py b/examples/cim/ac/components/agent_manager.py index 8f37650f1..c4a402338 100644 --- a/examples/cim/ac/components/agent_manager.py +++ b/examples/cim/ac/components/agent_manager.py @@ -6,8 +6,7 @@ from .agent import CIMAgent from .config import config -from maro.rl import AbsAgentManager, LearningModel, MLPDecisionLayers, ActorCritic, ActorCriticHyperParameters, \ - ColumnBasedStore +from maro.rl import AbsAgentManager, LearningModel, MLPDecisionLayers, ActorCritic, ActorCriticHyperParameters from maro.utils import set_seeds @@ -16,38 +15,47 @@ def _assemble(self, agent_dict): set_seeds(config.agents.seed) num_actions = config.agents.algorithm.num_actions for agent_id in self._agent_id_list: - policy_model = LearningModel(decision_layers=MLPDecisionLayers(name=f'{agent_id}.policy', - input_dim=self._state_shaper.dim, - output_dim=num_actions, - **config.agents.algorithm.policy_model, - softmax=True)) - - value_model = LearningModel(decision_layers=MLPDecisionLayers(name=f'{agent_id}.policy', - input_dim=self._state_shaper.dim, - output_dim=num_actions, - **config.agents.algorithm.value_model)) - - algorithm = ActorCritic(policy_model=policy_model, - value_model=value_model, - value_loss_func=smooth_l1_loss, - policy_optimizer_cls=Adam, - policy_optimizer_params=config.agents.algorithm.policy_optimizer, - value_optimizer_cls=RMSprop, - value_optimizer_params=config.agents.algorithm.value_optimizer, - hyper_params=ActorCriticHyperParameters(num_actions=num_actions, - **config.agents.algorithm.hyper_parameters, - ) - ) - - experience_pool = ColumnBasedStore(**config.agents.experience_pool) - agent_dict[agent_id] = CIMAgent(name=agent_id, algorithm=algorithm, experience_pool=experience_pool, - **config.agents.training_loop_parameters) + policy_model = LearningModel( + decision_layers=MLPDecisionLayers( + name=f'{agent_id}.policy', + input_dim=self._state_shaper.dim, + output_dim=num_actions, + **config.agents.algorithm.policy_model, + softmax=True + ) + ) + + value_model = LearningModel( + decision_layers=MLPDecisionLayers( + name=f'{agent_id}.policy', + input_dim=self._state_shaper.dim, + output_dim=num_actions, + **config.agents.algorithm.value_model + ) + ) + + algorithm = ActorCritic( + policy_model=policy_model, + value_model=value_model, + value_loss_func=smooth_l1_loss, + policy_optimizer_cls=Adam, + policy_optimizer_params=config.agents.algorithm.policy_optimizer, + value_optimizer_cls=RMSprop, + value_optimizer_params=config.agents.algorithm.value_optimizer, + hyper_params=ActorCriticHyperParameters( + num_actions=num_actions, + **config.agents.algorithm.hyper_parameters, + ) + ) + + agent_dict[agent_id] = CIMAgent(name=agent_id, algorithm=algorithm) def choose_action(self, decision_event, snapshot_list): self._assert_inference_mode() agent_id, model_state = self._state_shaper(decision_event, snapshot_list) model_action = self._agent_dict[agent_id].choose_action( - model_state, self._explorer.epsilon[agent_id] if self._explorer else None) + model_state, self._explorer.epsilon[agent_id] if self._explorer else None + ) self._transition_cache = {"state": model_state, "action": model_action, @@ -56,7 +64,9 @@ def choose_action(self, decision_event, snapshot_list): "event": decision_event} return self._action_shaper(model_action, decision_event, snapshot_list) + def train(self, experiences_by_agent: dict): + for agent_id, experiences in experiences_by_agent.items(): + self._agent_dict[agent_id].train(experiences) + def store_experiences(self, experiences): - for agent_id, exp in experiences.items(): - exp.update({"loss": [1e8] * len(exp[next(iter(exp))])}) - self._agent_dict[agent_id].store_experiences(exp) + pass diff --git a/examples/cim/ac/dist_actor.py b/examples/cim/ac/dist_actor.py index 350f493f2..831c8df13 100644 --- a/examples/cim/ac/dist_actor.py +++ b/examples/cim/ac/dist_actor.py @@ -4,10 +4,10 @@ import numpy as np from maro.simulator import Env -from maro.rl import AgentMode, SimpleActor, ActorWorker, KStepExperienceShaper, TwoPhaseLinearExplorer +from maro.rl import AgentMode, SimpleActor, ActorWorker from components.action_shaper import CIMActionShaper -from components.agent_manager import DQNAgentManager +from components.agent_manager import ACAgentManager from components.config import config from components.experience_shaper import TruncatedExperienceShaper from components.state_shaper import CIMStateShaper @@ -18,28 +18,23 @@ agent_id_list = [str(agent_id) for agent_id in env.agent_idx_list] state_shaper = CIMStateShaper(**config.state_shaping) action_shaper = CIMActionShaper(action_space=list(np.linspace(-1.0, 1.0, config.agents.algorithm.num_actions))) - if config.experience_shaping.type == "truncated": - experience_shaper = TruncatedExperienceShaper(**config.experience_shaping.truncated) - else: - experience_shaper = KStepExperienceShaper(reward_func=lambda mt: 1-mt["container_shortage"]/mt["order_requirements"], - **config.experience_shaping.k_step) + experience_shaper = TruncatedExperienceShaper(**config.experience_shaping.truncated) - exploration_config = {"epsilon_range_dict": {"_all_": config.exploration.epsilon_range}, - "split_point_dict": {"_all_": config.exploration.split_point}, - "with_cache": config.exploration.with_cache - } - explorer = TwoPhaseLinearExplorer(agent_id_list, config.general.total_training_episodes, **exploration_config) - agent_manager = DQNAgentManager(name="cim_remote_actor", - agent_id_list=agent_id_list, - mode=AgentMode.INFERENCE, - state_shaper=state_shaper, - action_shaper=action_shaper, - experience_shaper=experience_shaper, - explorer=explorer) - proxy_params = {"group_name": config.distributed.group_name, - "expected_peers": config.distributed.actor.peer, - "redis_address": (config.distributed.redis.host_name, config.distributed.redis.port) - } - actor_worker = ActorWorker(local_actor=SimpleActor(env=env, inference_agents=agent_manager), - proxy_params=proxy_params) + agent_manager = ACAgentManager( + name="cim_remote_actor", + agent_id_list=agent_id_list, + mode=AgentMode.INFERENCE, + state_shaper=state_shaper, + action_shaper=action_shaper, + experience_shaper=experience_shaper, + ) + proxy_params = { + "group_name": config.distributed.group_name, + "expected_peers": config.distributed.actor.peer, + "redis_address": (config.distributed.redis.host_name, config.distributed.redis.port) + } + actor_worker = ActorWorker( + local_actor=SimpleActor(env=env, inference_agents=agent_manager), + proxy_params=proxy_params + ) actor_worker.launch() diff --git a/examples/cim/ac/dist_learner.py b/examples/cim/ac/dist_learner.py index 32c501e11..df9c04b77 100644 --- a/examples/cim/ac/dist_learner.py +++ b/examples/cim/ac/dist_learner.py @@ -3,11 +3,11 @@ import os -from maro.rl import ActorProxy, SimpleLearner, AgentMode, TwoPhaseLinearExplorer +from maro.rl import ActorProxy, SimpleLearner, AgentMode from maro.simulator import Env from maro.utils import Logger -from components.agent_manager import DQNAgentManager +from components.agent_manager import ACAgentManager from components.config import config from components.state_shaper import CIMStateShaper @@ -16,21 +16,22 @@ env = Env(config.env.scenario, config.env.topology, durations=config.env.durations) agent_id_list = [str(agent_id) for agent_id in env.agent_idx_list] state_shaper = CIMStateShaper(**config.state_shaping) - exploration_config = {"epsilon_range_dict": {"_all_": config.exploration.epsilon_range}, - "split_point_dict": {"_all_": config.exploration.split_point}, - "with_cache": config.exploration.with_cache - } - explorer = TwoPhaseLinearExplorer(agent_id_list, config.general.total_training_episodes, **exploration_config) - agent_manager = DQNAgentManager(name="cim_remote_learner", agent_id_list=agent_id_list, mode=AgentMode.TRAIN, - state_shaper=state_shaper, explorer=explorer) + agent_manager = ACAgentManager( + name="cim_remote_learner", agent_id_list=agent_id_list, mode=AgentMode.TRAIN, state_shaper=state_shaper + ) - proxy_params = {"group_name": config.distributed.group_name, - "expected_peers": config.distributed.learner.peer, - "redis_address": (config.distributed.redis.host_name, config.distributed.redis.port) - } - learner = SimpleLearner(trainable_agents=agent_manager, - actor=ActorProxy(proxy_params=proxy_params), - logger=Logger("distributed_cim_learner", auto_timestamp=False)) - learner.train(total_episodes=config.general.total_training_episodes, - model_dump_dir=os.path.join(os.getcwd(), "models")) + proxy_params = { + "group_name": config.distributed.group_name, + "expected_peers": config.distributed.learner.peer, + "redis_address": (config.distributed.redis.host_name, config.distributed.redis.port) + } + learner = SimpleLearner( + trainable_agents=agent_manager, + actor=ActorProxy(proxy_params=proxy_params), + logger=Logger("distributed_cim_learner", auto_timestamp=False) + ) + learner.train( + total_episodes=config.general.total_training_episodes, + model_dump_dir=os.path.join(os.getcwd(), "models") + ) learner.test() diff --git a/examples/cim/ac/single_process_launcher.py b/examples/cim/ac/single_process_launcher.py index 98c9fa34e..bebd22bf8 100644 --- a/examples/cim/ac/single_process_launcher.py +++ b/examples/cim/ac/single_process_launcher.py @@ -1,7 +1,6 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT license. - import os import numpy as np @@ -11,7 +10,7 @@ from maro.utils import Logger from components.action_shaper import CIMActionShaper -from components.agent_manager import DQNAgentManager +from components.agent_manager import ACAgentManager from components.config import config from components.experience_shaper import TruncatedExperienceShaper from components.state_shaper import CIMStateShaper @@ -26,32 +25,26 @@ # greedy nature of the DQN algorithm. state_shaper = CIMStateShaper(**config.state_shaping) action_shaper = CIMActionShaper(action_space=list(np.linspace(-1.0, 1.0, config.agents.algorithm.num_actions))) - if config.experience_shaping.type == "truncated": - experience_shaper = TruncatedExperienceShaper(**config.experience_shaping.truncated) - else: - experience_shaper = KStepExperienceShaper(reward_func=lambda mt: 1-mt["container_shortage"]/mt["order_requirements"], - **config.experience_shaping.k_step) - - exploration_config = {"epsilon_range_dict": {"_all_": config.exploration.epsilon_range}, - "split_point_dict": {"_all_": config.exploration.split_point}, - "with_cache": config.exploration.with_cache - } - explorer = TwoPhaseLinearExplorer(agent_id_list, config.general.total_training_episodes, **exploration_config) + experience_shaper = TruncatedExperienceShaper(**config.experience_shaping.truncated) # Step 3: create an agent manager. - agent_manager = DQNAgentManager(name="cim_learner", - mode=AgentMode.TRAIN_INFERENCE, - agent_id_list=agent_id_list, - state_shaper=state_shaper, - action_shaper=action_shaper, - experience_shaper=experience_shaper, - explorer=explorer) + agent_manager = ACAgentManager( + name="cim_learner", + mode=AgentMode.TRAIN_INFERENCE, + agent_id_list=agent_id_list, + state_shaper=state_shaper, + action_shaper=action_shaper, + experience_shaper=experience_shaper, + ) # Step 4: Create an actor and a learner to start the training process. actor = SimpleActor(env=env, inference_agents=agent_manager) - learner = SimpleLearner(trainable_agents=agent_manager, actor=actor, - logger=Logger("single_host_cim_learner", auto_timestamp=False)) - - learner.train(total_episodes=config.general.total_training_episodes, - model_dump_dir=os.path.join(os.getcwd(), "models")) + learner = SimpleLearner( + trainable_agents=agent_manager, actor=actor, + logger=Logger("single_host_cim_learner", auto_timestamp=False) + ) + learner.train( + total_episodes=config.general.total_training_episodes, + model_dump_dir=os.path.join(os.getcwd(), "models") + ) learner.test() diff --git a/examples/cim/dqn/components/agent_manager.py b/examples/cim/dqn/components/agent_manager.py index a1dbcc203..a9bdd8ca0 100644 --- a/examples/cim/dqn/components/agent_manager.py +++ b/examples/cim/dqn/components/agent_manager.py @@ -15,17 +15,23 @@ def _assemble(self, agent_dict): set_seeds(config.agents.seed) num_actions = config.agents.algorithm.num_actions for agent_id in self._agent_id_list: - eval_model = LearningModel(decision_layers=MLPDecisionLayers(name=f'{agent_id}.policy', - input_dim=self._state_shaper.dim, - output_dim=num_actions, - **config.agents.algorithm.model) - ) - - algorithm = DQN(model_dict={"eval": eval_model}, - optimizer_opt=(RMSprop, config.agents.algorithm.optimizer), - loss_func_dict={"eval": smooth_l1_loss}, - hyper_params=DQNHyperParams(**config.agents.algorithm.hyper_parameters, - num_actions=num_actions)) + eval_model = LearningModel( + decision_layers=MLPDecisionLayers( + name=f'{agent_id}.policy', input_dim=self._state_shaper.dim, + output_dim=num_actions, **config.agents.algorithm.model + ) + ) + + algorithm = DQN( + eval_model=eval_model, + optimizer_cls=RMSprop, + optimizer_params=config.agents.algorithm.optimizer, + loss_func=smooth_l1_loss, + hyper_params=DQNHyperParams( + **config.agents.algorithm.hyper_parameters, + num_actions=num_actions + ) + ) experience_pool = ColumnBasedStore(**config.agents.experience_pool) agent_dict[agent_id] = CIMAgent(name=agent_id, algorithm=algorithm, experience_pool=experience_pool, @@ -44,6 +50,11 @@ def choose_action(self, decision_event, snapshot_list): "event": decision_event} return self._action_shaper(model_action, decision_event, snapshot_list) + def train(self): + self._assert_train_mode() + for agent in self._agent_dict.values(): + agent.train() + def store_experiences(self, experiences): for agent_id, exp in experiences.items(): exp.update({"loss": [1e8] * len(exp[next(iter(exp))])}) diff --git a/examples/cim/dqn/dist_actor.py b/examples/cim/dqn/dist_actor.py index 350f493f2..82faf88e7 100644 --- a/examples/cim/dqn/dist_actor.py +++ b/examples/cim/dqn/dist_actor.py @@ -21,25 +21,33 @@ if config.experience_shaping.type == "truncated": experience_shaper = TruncatedExperienceShaper(**config.experience_shaping.truncated) else: - experience_shaper = KStepExperienceShaper(reward_func=lambda mt: 1-mt["container_shortage"]/mt["order_requirements"], - **config.experience_shaping.k_step) + experience_shaper = KStepExperienceShaper( + reward_func=lambda mt: 1-mt["container_shortage"]/mt["order_requirements"], + **config.experience_shaping.k_step + ) - exploration_config = {"epsilon_range_dict": {"_all_": config.exploration.epsilon_range}, - "split_point_dict": {"_all_": config.exploration.split_point}, - "with_cache": config.exploration.with_cache - } + exploration_config = { + "epsilon_range_dict": {"_all_": config.exploration.epsilon_range}, + "split_point_dict": {"_all_": config.exploration.split_point}, + "with_cache": config.exploration.with_cache + } explorer = TwoPhaseLinearExplorer(agent_id_list, config.general.total_training_episodes, **exploration_config) - agent_manager = DQNAgentManager(name="cim_remote_actor", - agent_id_list=agent_id_list, - mode=AgentMode.INFERENCE, - state_shaper=state_shaper, - action_shaper=action_shaper, - experience_shaper=experience_shaper, - explorer=explorer) - proxy_params = {"group_name": config.distributed.group_name, - "expected_peers": config.distributed.actor.peer, - "redis_address": (config.distributed.redis.host_name, config.distributed.redis.port) - } - actor_worker = ActorWorker(local_actor=SimpleActor(env=env, inference_agents=agent_manager), - proxy_params=proxy_params) + agent_manager = DQNAgentManager( + name="cim_remote_actor", + agent_id_list=agent_id_list, + mode=AgentMode.INFERENCE, + state_shaper=state_shaper, + action_shaper=action_shaper, + experience_shaper=experience_shaper, + explorer=explorer + ) + proxy_params = { + "group_name": config.distributed.group_name, + "expected_peers": config.distributed.actor.peer, + "redis_address": (config.distributed.redis.host_name, config.distributed.redis.port) + } + actor_worker = ActorWorker( + local_actor=SimpleActor(env=env, inference_agents=agent_manager), + proxy_params=proxy_params + ) actor_worker.launch() diff --git a/examples/cim/dqn/dist_learner.py b/examples/cim/dqn/dist_learner.py index 32c501e11..8d39a227d 100644 --- a/examples/cim/dqn/dist_learner.py +++ b/examples/cim/dqn/dist_learner.py @@ -16,21 +16,30 @@ env = Env(config.env.scenario, config.env.topology, durations=config.env.durations) agent_id_list = [str(agent_id) for agent_id in env.agent_idx_list] state_shaper = CIMStateShaper(**config.state_shaping) - exploration_config = {"epsilon_range_dict": {"_all_": config.exploration.epsilon_range}, - "split_point_dict": {"_all_": config.exploration.split_point}, - "with_cache": config.exploration.with_cache - } + exploration_config = { + "epsilon_range_dict": {"_all_": config.exploration.epsilon_range}, + "split_point_dict": {"_all_": config.exploration.split_point}, + "with_cache": config.exploration.with_cache + } explorer = TwoPhaseLinearExplorer(agent_id_list, config.general.total_training_episodes, **exploration_config) - agent_manager = DQNAgentManager(name="cim_remote_learner", agent_id_list=agent_id_list, mode=AgentMode.TRAIN, - state_shaper=state_shaper, explorer=explorer) + agent_manager = DQNAgentManager( + name="cim_remote_learner", agent_id_list=agent_id_list, mode=AgentMode.TRAIN, + state_shaper=state_shaper, explorer=explorer + ) - proxy_params = {"group_name": config.distributed.group_name, - "expected_peers": config.distributed.learner.peer, - "redis_address": (config.distributed.redis.host_name, config.distributed.redis.port) - } - learner = SimpleLearner(trainable_agents=agent_manager, - actor=ActorProxy(proxy_params=proxy_params), - logger=Logger("distributed_cim_learner", auto_timestamp=False)) - learner.train(total_episodes=config.general.total_training_episodes, - model_dump_dir=os.path.join(os.getcwd(), "models")) + proxy_params = { + "group_name": config.distributed.group_name, + "expected_peers": config.distributed.learner.peer, + "redis_address": (config.distributed.redis.host_name, config.distributed.redis.port) + } + + learner = SimpleLearner( + trainable_agents=agent_manager, + actor=ActorProxy(proxy_params=proxy_params), + logger=Logger("distributed_cim_learner", auto_timestamp=False) + ) + learner.train( + total_episodes=config.general.total_training_episodes, + model_dump_dir=os.path.join(os.getcwd(), "models") + ) learner.test() diff --git a/examples/cim/dqn/single_process_launcher.py b/examples/cim/dqn/single_process_launcher.py index 98c9fa34e..b0321f8c5 100644 --- a/examples/cim/dqn/single_process_launcher.py +++ b/examples/cim/dqn/single_process_launcher.py @@ -29,29 +29,37 @@ if config.experience_shaping.type == "truncated": experience_shaper = TruncatedExperienceShaper(**config.experience_shaping.truncated) else: - experience_shaper = KStepExperienceShaper(reward_func=lambda mt: 1-mt["container_shortage"]/mt["order_requirements"], - **config.experience_shaping.k_step) + experience_shaper = KStepExperienceShaper( + reward_func=lambda mt: 1-mt["container_shortage"]/mt["order_requirements"], + **config.experience_shaping.k_step + ) - exploration_config = {"epsilon_range_dict": {"_all_": config.exploration.epsilon_range}, - "split_point_dict": {"_all_": config.exploration.split_point}, - "with_cache": config.exploration.with_cache - } + exploration_config = { + "epsilon_range_dict": {"_all_": config.exploration.epsilon_range}, + "split_point_dict": {"_all_": config.exploration.split_point}, + "with_cache": config.exploration.with_cache + } explorer = TwoPhaseLinearExplorer(agent_id_list, config.general.total_training_episodes, **exploration_config) # Step 3: create an agent manager. - agent_manager = DQNAgentManager(name="cim_learner", - mode=AgentMode.TRAIN_INFERENCE, - agent_id_list=agent_id_list, - state_shaper=state_shaper, - action_shaper=action_shaper, - experience_shaper=experience_shaper, - explorer=explorer) + agent_manager = DQNAgentManager( + name="cim_learner", + mode=AgentMode.TRAIN_INFERENCE, + agent_id_list=agent_id_list, + state_shaper=state_shaper, + action_shaper=action_shaper, + experience_shaper=experience_shaper, + explorer=explorer + ) # Step 4: Create an actor and a learner to start the training process. actor = SimpleActor(env=env, inference_agents=agent_manager) - learner = SimpleLearner(trainable_agents=agent_manager, actor=actor, - logger=Logger("single_host_cim_learner", auto_timestamp=False)) - - learner.train(total_episodes=config.general.total_training_episodes, - model_dump_dir=os.path.join(os.getcwd(), "models")) + learner = SimpleLearner( + trainable_agents=agent_manager, actor=actor, + logger=Logger("single_host_cim_learner", auto_timestamp=False) + ) + learner.train( + total_episodes=config.general.total_training_episodes, + model_dump_dir=os.path.join(os.getcwd(), "models") + ) learner.test() diff --git a/examples/cim/pg/components/agent.py b/examples/cim/pg/components/agent.py index 6f322fe59..39d0e994d 100644 --- a/examples/cim/pg/components/agent.py +++ b/examples/cim/pg/components/agent.py @@ -3,16 +3,9 @@ import numpy as np -from maro.rl import AbsAgent, ColumnBasedStore +from maro.rl import AbsAgent class CIMAgent(AbsAgent): - def train(self, ): - for _ in range(self._num_batches): - indexes, sample = self._experience_pool.sample_by_key("loss", self._batch_size) - state = np.asarray(sample["state"]) - action = np.asarray(sample["action"]) - reward = np.asarray(sample["reward"]) - next_state = np.asarray(sample["next_state"]) - loss = self._algorithm.train(state, action, reward, next_state) - self._experience_pool.update(indexes, {"loss": loss}) + def train(self, states: np.ndarray, actions: np.ndarray, rewards: np.ndarray): + self._algorithm.train(states, actions, rewards) diff --git a/examples/cim/pg/components/agent_manager.py b/examples/cim/pg/components/agent_manager.py index e31897630..6c872364c 100644 --- a/examples/cim/pg/components/agent_manager.py +++ b/examples/cim/pg/components/agent_manager.py @@ -1,7 +1,6 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT license. -from torch.nn.functional import smooth_l1_loss from torch.optim import Adam, RMSprop from .agent import CIMAgent @@ -37,7 +36,8 @@ def choose_action(self, decision_event, snapshot_list): self._assert_inference_mode() agent_id, model_state = self._state_shaper(decision_event, snapshot_list) model_action = self._agent_dict[agent_id].choose_action( - model_state, self._explorer.epsilon[agent_id] if self._explorer else None) + model_state, self._explorer.epsilon[agent_id] if self._explorer else None + ) self._transition_cache = {"state": model_state, "action": model_action, @@ -46,7 +46,9 @@ def choose_action(self, decision_event, snapshot_list): "event": decision_event} return self._action_shaper(model_action, decision_event, snapshot_list) + def train(self, experiences_by_agent: dict): + for agent_id, experiences in experiences_by_agent.items(): + self._agent_dict[agent_id].train(experiences) + def store_experiences(self, experiences): - for agent_id, exp in experiences.items(): - exp.update({"loss": [1e8] * len(exp[next(iter(exp))])}) - self._agent_dict[agent_id].store_experiences(exp) + pass diff --git a/examples/cim/pg/components/experience_shaper.py b/examples/cim/pg/components/experience_shaper.py index 2cfd4053d..b7a23d5bd 100644 --- a/examples/cim/pg/components/experience_shaper.py +++ b/examples/cim/pg/components/experience_shaper.py @@ -25,9 +25,9 @@ def __call__(self, trajectory, snapshot_list): map(self._compute_reward, trajectory.get_by_key("event"), [snapshot_list]*len(trajectory)), dtype=np.float32 ) - return {agent_id: {"state": states[agent_ids == agent_id], - "action": actions[agent_ids == agent_id], - "reward": rewards[agent_ids == agent_id], + return {agent_id: {"states": states[agent_ids == agent_id], + "actions": actions[agent_ids == agent_id], + "rewards": rewards[agent_ids == agent_id], } for agent_id in set(agent_ids)} diff --git a/examples/cim/pg/dist_actor.py b/examples/cim/pg/dist_actor.py index 350f493f2..3fd058d85 100644 --- a/examples/cim/pg/dist_actor.py +++ b/examples/cim/pg/dist_actor.py @@ -18,28 +18,22 @@ agent_id_list = [str(agent_id) for agent_id in env.agent_idx_list] state_shaper = CIMStateShaper(**config.state_shaping) action_shaper = CIMActionShaper(action_space=list(np.linspace(-1.0, 1.0, config.agents.algorithm.num_actions))) - if config.experience_shaping.type == "truncated": - experience_shaper = TruncatedExperienceShaper(**config.experience_shaping.truncated) - else: - experience_shaper = KStepExperienceShaper(reward_func=lambda mt: 1-mt["container_shortage"]/mt["order_requirements"], - **config.experience_shaping.k_step) - - exploration_config = {"epsilon_range_dict": {"_all_": config.exploration.epsilon_range}, - "split_point_dict": {"_all_": config.exploration.split_point}, - "with_cache": config.exploration.with_cache - } - explorer = TwoPhaseLinearExplorer(agent_id_list, config.general.total_training_episodes, **exploration_config) - agent_manager = DQNAgentManager(name="cim_remote_actor", - agent_id_list=agent_id_list, - mode=AgentMode.INFERENCE, - state_shaper=state_shaper, - action_shaper=action_shaper, - experience_shaper=experience_shaper, - explorer=explorer) - proxy_params = {"group_name": config.distributed.group_name, - "expected_peers": config.distributed.actor.peer, - "redis_address": (config.distributed.redis.host_name, config.distributed.redis.port) - } - actor_worker = ActorWorker(local_actor=SimpleActor(env=env, inference_agents=agent_manager), - proxy_params=proxy_params) + experience_shaper = TruncatedExperienceShaper(**config.experience_shaping.truncated) + agent_manager = DQNAgentManager( + name="cim_remote_actor", + agent_id_list=agent_id_list, + mode=AgentMode.INFERENCE, + state_shaper=state_shaper, + action_shaper=action_shaper, + experience_shaper=experience_shaper + ) + proxy_params = { + "group_name": config.distributed.group_name, + "expected_peers": config.distributed.actor.peer, + "redis_address": (config.distributed.redis.host_name, config.distributed.redis.port) + } + actor_worker = ActorWorker( + local_actor=SimpleActor(env=env, inference_agents=agent_manager), + proxy_params=proxy_params + ) actor_worker.launch() diff --git a/examples/cim/pg/dist_learner.py b/examples/cim/pg/dist_learner.py index 32c501e11..8d39a227d 100644 --- a/examples/cim/pg/dist_learner.py +++ b/examples/cim/pg/dist_learner.py @@ -16,21 +16,30 @@ env = Env(config.env.scenario, config.env.topology, durations=config.env.durations) agent_id_list = [str(agent_id) for agent_id in env.agent_idx_list] state_shaper = CIMStateShaper(**config.state_shaping) - exploration_config = {"epsilon_range_dict": {"_all_": config.exploration.epsilon_range}, - "split_point_dict": {"_all_": config.exploration.split_point}, - "with_cache": config.exploration.with_cache - } + exploration_config = { + "epsilon_range_dict": {"_all_": config.exploration.epsilon_range}, + "split_point_dict": {"_all_": config.exploration.split_point}, + "with_cache": config.exploration.with_cache + } explorer = TwoPhaseLinearExplorer(agent_id_list, config.general.total_training_episodes, **exploration_config) - agent_manager = DQNAgentManager(name="cim_remote_learner", agent_id_list=agent_id_list, mode=AgentMode.TRAIN, - state_shaper=state_shaper, explorer=explorer) + agent_manager = DQNAgentManager( + name="cim_remote_learner", agent_id_list=agent_id_list, mode=AgentMode.TRAIN, + state_shaper=state_shaper, explorer=explorer + ) - proxy_params = {"group_name": config.distributed.group_name, - "expected_peers": config.distributed.learner.peer, - "redis_address": (config.distributed.redis.host_name, config.distributed.redis.port) - } - learner = SimpleLearner(trainable_agents=agent_manager, - actor=ActorProxy(proxy_params=proxy_params), - logger=Logger("distributed_cim_learner", auto_timestamp=False)) - learner.train(total_episodes=config.general.total_training_episodes, - model_dump_dir=os.path.join(os.getcwd(), "models")) + proxy_params = { + "group_name": config.distributed.group_name, + "expected_peers": config.distributed.learner.peer, + "redis_address": (config.distributed.redis.host_name, config.distributed.redis.port) + } + + learner = SimpleLearner( + trainable_agents=agent_manager, + actor=ActorProxy(proxy_params=proxy_params), + logger=Logger("distributed_cim_learner", auto_timestamp=False) + ) + learner.train( + total_episodes=config.general.total_training_episodes, + model_dump_dir=os.path.join(os.getcwd(), "models") + ) learner.test() diff --git a/examples/cim/pg/single_process_launcher.py b/examples/cim/pg/single_process_launcher.py index 98c9fa34e..ed460889d 100644 --- a/examples/cim/pg/single_process_launcher.py +++ b/examples/cim/pg/single_process_launcher.py @@ -1,17 +1,16 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT license. - import os import numpy as np from maro.simulator import Env -from maro.rl import SimpleLearner, SimpleActor, AgentMode, KStepExperienceShaper, TwoPhaseLinearExplorer +from maro.rl import SimpleLearner, SimpleActor, AgentMode from maro.utils import Logger from components.action_shaper import CIMActionShaper -from components.agent_manager import DQNAgentManager +from components.agent_manager import PGAgentManager from components.config import config from components.experience_shaper import TruncatedExperienceShaper from components.state_shaper import CIMStateShaper @@ -26,32 +25,26 @@ # greedy nature of the DQN algorithm. state_shaper = CIMStateShaper(**config.state_shaping) action_shaper = CIMActionShaper(action_space=list(np.linspace(-1.0, 1.0, config.agents.algorithm.num_actions))) - if config.experience_shaping.type == "truncated": - experience_shaper = TruncatedExperienceShaper(**config.experience_shaping.truncated) - else: - experience_shaper = KStepExperienceShaper(reward_func=lambda mt: 1-mt["container_shortage"]/mt["order_requirements"], - **config.experience_shaping.k_step) - - exploration_config = {"epsilon_range_dict": {"_all_": config.exploration.epsilon_range}, - "split_point_dict": {"_all_": config.exploration.split_point}, - "with_cache": config.exploration.with_cache - } - explorer = TwoPhaseLinearExplorer(agent_id_list, config.general.total_training_episodes, **exploration_config) + experience_shaper = TruncatedExperienceShaper(**config.experience_shaping.truncated) # Step 3: create an agent manager. - agent_manager = DQNAgentManager(name="cim_learner", - mode=AgentMode.TRAIN_INFERENCE, - agent_id_list=agent_id_list, - state_shaper=state_shaper, - action_shaper=action_shaper, - experience_shaper=experience_shaper, - explorer=explorer) + agent_manager = PGAgentManager( + name="cim_learner", + mode=AgentMode.TRAIN_INFERENCE, + agent_id_list=agent_id_list, + state_shaper=state_shaper, + action_shaper=action_shaper, + experience_shaper=experience_shaper, + ) # Step 4: Create an actor and a learner to start the training process. actor = SimpleActor(env=env, inference_agents=agent_manager) - learner = SimpleLearner(trainable_agents=agent_manager, actor=actor, - logger=Logger("single_host_cim_learner", auto_timestamp=False)) - - learner.train(total_episodes=config.general.total_training_episodes, - model_dump_dir=os.path.join(os.getcwd(), "models")) + learner = SimpleLearner( + trainable_agents=agent_manager, actor=actor, + logger=Logger("single_host_cim_learner", auto_timestamp=False) + ) + learner.train( + total_episodes=config.general.total_training_episodes, + model_dump_dir=os.path.join(os.getcwd(), "models") + ) learner.test() diff --git a/maro/rl/agent/abs_agent.py b/maro/rl/agent/abs_agent.py index 1c5875bd2..4be9ad4ba 100644 --- a/maro/rl/agent/abs_agent.py +++ b/maro/rl/agent/abs_agent.py @@ -57,7 +57,7 @@ def choose_action(self, model_state, epsilon: float = .0): return self._algorithm.choose_action(model_state, epsilon) @abstractmethod - def train(self): + def train(self, *args, **kwargs): """Training logic to be implemented by the user. For example, this may include drawing samples from the experience pool and the algorithm training on diff --git a/maro/rl/agent/abs_agent_manager.py b/maro/rl/agent/abs_agent_manager.py index e9ac1bc62..fef91cf9b 100644 --- a/maro/rl/agent/abs_agent_manager.py +++ b/maro/rl/agent/abs_agent_manager.py @@ -4,7 +4,6 @@ from abc import ABC, abstractmethod from enum import Enum import os -from typing import Callable from maro.rl.shaping.state_shaper import StateShaper from maro.rl.shaping.action_shaper import ActionShaper @@ -134,6 +133,11 @@ def store_experiences(self, experiences): """ return NotImplementedError + @abstractmethod + def train(self, *args, **kwargs): + """Train all agents.""" + return NotImplementedError + def update_epsilon(self, performance): """This method updates the exploration rates for each agent. @@ -144,12 +148,6 @@ def update_epsilon(self, performance): if self._explorer: self._explorer.update(performance) - def train(self): - """Train all agents.""" - self._assert_train_mode() - for agent in self._agent_dict.values(): - agent.train() - def load_trainable_models(self, agent_model_dict): """Load models from memory for each agent.""" for agent_id, models in agent_model_dict.items(): diff --git a/maro/rl/algorithms/torch/ac.py b/maro/rl/algorithms/torch/ac.py index 7d296903e..df8889007 100644 --- a/maro/rl/algorithms/torch/ac.py +++ b/maro/rl/algorithms/torch/ac.py @@ -86,7 +86,7 @@ def _get_values_and_bootstrapped_returns(self, state_sequence, reward_sequence): return_est = torch.from_numpy(return_est) return state_values, return_est - def train(self, states: np.ndarray, actions: torch.tensor, rewards: np.ndarray): + def train(self, states: np.ndarray, actions: np.ndarray, rewards: np.ndarray): states = torch.from_numpy(states).to(self._device) state_values, return_est = self._get_values_and_bootstrapped_returns(states, rewards) advantages = return_est - state_values From bfdf7f88aeee53af062e6367ec0723b7d7448dcc Mon Sep 17 00:00:00 2001 From: ysqyang Date: Sun, 18 Oct 2020 15:04:45 +0800 Subject: [PATCH 018/337] fixed bug in choose_action in ac, pg and ppo --- maro/rl/algorithms/torch/ac.py | 8 ++++++-- maro/rl/algorithms/torch/pg.py | 4 +++- maro/rl/algorithms/torch/ppo.py | 8 ++++++-- 3 files changed, 15 insertions(+), 5 deletions(-) diff --git a/maro/rl/algorithms/torch/ac.py b/maro/rl/algorithms/torch/ac.py index df8889007..4bc085bb9 100644 --- a/maro/rl/algorithms/torch/ac.py +++ b/maro/rl/algorithms/torch/ac.py @@ -73,7 +73,9 @@ def model(self): def choose_action(self, state: np.ndarray, epsilon: float = None): state = torch.from_numpy(state).unsqueeze(0).to(self._device) # (1, state_dim) - action_dist = self._policy_model(state).squeeze().numpy() # (num_actions,) + self._policy_model.eval() + with torch.no_grad(): + action_dist = self._policy_model(state).squeeze().numpy() # (num_actions,) return np.random.choice(self._hyper_params.num_actions, p=action_dist) def _get_values_and_bootstrapped_returns(self, state_sequence, reward_sequence): @@ -175,7 +177,9 @@ def model(self): def choose_action(self, state: np.ndarray, epsilon: float = None): state = torch.from_numpy(state).unsqueeze(0).to(self._device) # (1, state_dim) - action_dist = self._policy_value_model(state)[1].squeeze().numpy() # (num_actions,) + self._policy_value_model.eval() + with torch.no_grad(): + action_dist = self._policy_value_model(state)[1].squeeze().numpy() # (num_actions,) return np.random.choice(self._hyper_params.num_actions, p=action_dist) def _get_values_and_bootstrapped_returns(self, state_sequence, reward_sequence): diff --git a/maro/rl/algorithms/torch/pg.py b/maro/rl/algorithms/torch/pg.py index de17e594d..91c121cb1 100644 --- a/maro/rl/algorithms/torch/pg.py +++ b/maro/rl/algorithms/torch/pg.py @@ -51,7 +51,9 @@ def model(self): def choose_action(self, state: np.ndarray, epsilon: float = None): state = torch.from_numpy(state).unsqueeze(0).to(self._device) # (1, state_dim) - action_dist = self._policy_model(state).squeeze().numpy() # (num_actions,) + self._policy_model.eval() + with torch.no_grad(): + action_dist = self._policy_model(state).squeeze().numpy() # (num_actions,) return np.random.choice(self._hyper_params.num_actions, p=action_dist) def train(self, states: np.ndarray, actions: np.ndarray, returns: np.ndarray): diff --git a/maro/rl/algorithms/torch/ppo.py b/maro/rl/algorithms/torch/ppo.py index c056e717e..c8009ef07 100644 --- a/maro/rl/algorithms/torch/ppo.py +++ b/maro/rl/algorithms/torch/ppo.py @@ -71,7 +71,9 @@ def __init__( def choose_action(self, state: np.ndarray, epsilon: float = None): state = torch.from_numpy(state).unsqueeze(0).to(self._device) # (1, state_dim) - action_dist = self._policy_model(state).squeeze().numpy() # (num_actions,) + self._policy_model.eval() + with torch.no_grad(): + action_dist = self._policy_model(state).squeeze().numpy() # (num_actions,) return np.random.choice(self._hyper_params.num_actions, p=action_dist) def _get_values_and_bootstrapped_returns(self, states: torch.tensor, rewards: np.ndarray): @@ -184,7 +186,9 @@ def model(self): def choose_action(self, state: np.ndarray, epsilon: float = None): state = torch.from_numpy(state).unsqueeze(0).to(self._device) # (1, state_dim) - action_dist = self._policy_value_model(state)[1].squeeze().numpy() # (num_actions,) + self._policy_value_model.eval() + with torch.no_grad(): + action_dist = self._policy_value_model(state)[1].squeeze().numpy() # (num_actions,) action_index = np.random.choice(self._hyper_params.num_actions, p=action_dist) return action_index, np.log(action_dist[action_index]) From 8d8e653bbaabca72f1d71100243c87913a112c45 Mon Sep 17 00:00:00 2001 From: ysqyang Date: Sun, 18 Oct 2020 15:09:39 +0800 Subject: [PATCH 019/337] fixed small bugs --- examples/cim/ac/components/agent_manager.py | 2 +- .../cim/ac/components/experience_shaper.py | 25 +++++++++---------- examples/cim/pg/components/agent_manager.py | 2 +- .../cim/pg/components/experience_shaper.py | 6 ++--- 4 files changed, 17 insertions(+), 18 deletions(-) diff --git a/examples/cim/ac/components/agent_manager.py b/examples/cim/ac/components/agent_manager.py index c4a402338..0343c7a71 100644 --- a/examples/cim/ac/components/agent_manager.py +++ b/examples/cim/ac/components/agent_manager.py @@ -66,7 +66,7 @@ def choose_action(self, decision_event, snapshot_list): def train(self, experiences_by_agent: dict): for agent_id, experiences in experiences_by_agent.items(): - self._agent_dict[agent_id].train(experiences) + self._agent_dict[agent_id].train(experiences["states"], experiences["actions"], experiences["rewards"]) def store_experiences(self, experiences): pass diff --git a/examples/cim/ac/components/experience_shaper.py b/examples/cim/ac/components/experience_shaper.py index 2941f2159..a4ac82585 100644 --- a/examples/cim/ac/components/experience_shaper.py +++ b/examples/cim/ac/components/experience_shaper.py @@ -18,19 +18,18 @@ def __init__(self, *, time_window: int, time_decay_factor: float, fulfillment_fa self._shortage_factor = shortage_factor def __call__(self, trajectory, snapshot_list): - experiences_by_agent = {} - for i in range(len(trajectory) - 1): - transition = trajectory[i] - agent_id = transition["agent_id"] - if agent_id not in experiences_by_agent: - experiences_by_agent[agent_id] = defaultdict(list) - experiences = experiences_by_agent[agent_id] - experiences["state"].append(transition["state"]) - experiences["action"].append(transition["action"]) - experiences["reward"].append(self._compute_reward(transition["event"], snapshot_list)) - experiences["next_state"].append(trajectory[i+1]["state"]) - - return experiences_by_agent + agent_ids = np.asarray(trajectory.get_by_key("agent_id")) + states = np.asarray(trajectory.get_by_key("state")) + actions = np.asarray(trajectory.get_by_key("action")) + rewards = np.fromiter( + map(self._compute_reward, trajectory.get_by_key("event"), [snapshot_list] * len(trajectory)), + dtype=np.float32 + ) + return {agent_id: {"states": states[agent_ids == agent_id], + "actions": actions[agent_ids == agent_id], + "rewards": rewards[agent_ids == agent_id], + } + for agent_id in set(agent_ids)} def _compute_reward(self, decision_event, snapshot_list): start_tick = decision_event.tick + 1 diff --git a/examples/cim/pg/components/agent_manager.py b/examples/cim/pg/components/agent_manager.py index 6c872364c..e271a490f 100644 --- a/examples/cim/pg/components/agent_manager.py +++ b/examples/cim/pg/components/agent_manager.py @@ -48,7 +48,7 @@ def choose_action(self, decision_event, snapshot_list): def train(self, experiences_by_agent: dict): for agent_id, experiences in experiences_by_agent.items(): - self._agent_dict[agent_id].train(experiences) + self._agent_dict[agent_id].train(experiences["states"], experiences["actions"], experiences["rewards"]) def store_experiences(self, experiences): pass diff --git a/examples/cim/pg/components/experience_shaper.py b/examples/cim/pg/components/experience_shaper.py index b7a23d5bd..08bccce49 100644 --- a/examples/cim/pg/components/experience_shaper.py +++ b/examples/cim/pg/components/experience_shaper.py @@ -18,9 +18,9 @@ def __init__(self, *, time_window: int, time_decay_factor: float, fulfillment_fa self._shortage_factor = shortage_factor def __call__(self, trajectory, snapshot_list): - agent_ids = np.asarray(trajectory.get_by_key["agent_id"]) - states = np.asarray(trajectory.get_by_key["state"]) - actions = np.asarray(trajectory.get_by_key["action"]) + agent_ids = np.asarray(trajectory.get_by_key("agent_id")) + states = np.asarray(trajectory.get_by_key("state")) + actions = np.asarray(trajectory.get_by_key("action")) rewards = np.fromiter( map(self._compute_reward, trajectory.get_by_key("event"), [snapshot_list]*len(trajectory)), dtype=np.float32 From fa9c4323b40e7b60f7f59a94be0faa3ced6c7738 Mon Sep 17 00:00:00 2001 From: ysqyang Date: Sun, 18 Oct 2020 15:27:48 +0800 Subject: [PATCH 020/337] fixed store put bug --- maro/rl/storage/column_based_store.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/maro/rl/storage/column_based_store.py b/maro/rl/storage/column_based_store.py index 38570f79e..981e77c59 100644 --- a/maro/rl/storage/column_based_store.py +++ b/maro/rl/storage/column_based_store.py @@ -93,7 +93,7 @@ def put(self, contents: dict, overwrite_indexes: Sequence = None) -> List[int]: added_size = len(contents[next(iter(contents))]) if self._capacity < 0: for key, val in contents.items(): - if not isinstance(val, list) and not isinstance(val, np.ndarray): + if not isinstance(val, list): self._store[key].append(val) else: self._store[key].extend(val) From b9f3e2b92fafc35deb448588b9a04a937b4917d7 Mon Sep 17 00:00:00 2001 From: ysqyang Date: Sun, 18 Oct 2020 16:58:21 +0800 Subject: [PATCH 021/337] changed some agent manager interfaces to accomodate PO algoriths --- examples/cim/ac/components/agent_manager.py | 3 --- examples/cim/dqn/components/agent_manager.py | 15 ++++++++----- examples/cim/pg/components/agent_manager.py | 3 --- maro/rl/agent/abs_agent_manager.py | 23 -------------------- maro/rl/learner/abs_learner.py | 12 +++------- maro/rl/learner/simple_learner.py | 20 ++++++++++------- 6 files changed, 25 insertions(+), 51 deletions(-) diff --git a/examples/cim/ac/components/agent_manager.py b/examples/cim/ac/components/agent_manager.py index 0343c7a71..c5d14ee8a 100644 --- a/examples/cim/ac/components/agent_manager.py +++ b/examples/cim/ac/components/agent_manager.py @@ -67,6 +67,3 @@ def choose_action(self, decision_event, snapshot_list): def train(self, experiences_by_agent: dict): for agent_id, experiences in experiences_by_agent.items(): self._agent_dict[agent_id].train(experiences["states"], experiences["actions"], experiences["rewards"]) - - def store_experiences(self, experiences): - pass diff --git a/examples/cim/dqn/components/agent_manager.py b/examples/cim/dqn/components/agent_manager.py index a9bdd8ca0..a5195381b 100644 --- a/examples/cim/dqn/components/agent_manager.py +++ b/examples/cim/dqn/components/agent_manager.py @@ -50,12 +50,17 @@ def choose_action(self, decision_event, snapshot_list): "event": decision_event} return self._action_shaper(model_action, decision_event, snapshot_list) - def train(self): + def train(self, experiences_by_agent, performance=None): self._assert_train_mode() - for agent in self._agent_dict.values(): - agent.train() - def store_experiences(self, experiences): - for agent_id, exp in experiences.items(): + # store experiences for each agent + for agent_id, exp in experiences_by_agent.items(): exp.update({"loss": [1e8] * len(exp[next(iter(exp))])}) self._agent_dict[agent_id].store_experiences(exp) + + for agent in self._agent_dict.values(): + agent.train() + + # update exploration rates + if self._explorer is not None: + self._explorer.update(performance) diff --git a/examples/cim/pg/components/agent_manager.py b/examples/cim/pg/components/agent_manager.py index e271a490f..b70a983c7 100644 --- a/examples/cim/pg/components/agent_manager.py +++ b/examples/cim/pg/components/agent_manager.py @@ -49,6 +49,3 @@ def choose_action(self, decision_event, snapshot_list): def train(self, experiences_by_agent: dict): for agent_id, experiences in experiences_by_agent.items(): self._agent_dict[agent_id].train(experiences["states"], experiences["actions"], experiences["rewards"]) - - def store_experiences(self, experiences): - pass diff --git a/maro/rl/agent/abs_agent_manager.py b/maro/rl/agent/abs_agent_manager.py index fef91cf9b..c4287d445 100644 --- a/maro/rl/agent/abs_agent_manager.py +++ b/maro/rl/agent/abs_agent_manager.py @@ -120,34 +120,11 @@ def post_process(self, snapshot_list): self._experience_shaper.reset() return experiences - @abstractmethod - def store_experiences(self, experiences): - """Abstract method to store experiences generated by the experience shaper in the experience pool(s). - - Depending on the user's implementation of the experience shaper's ``__call__`` method, ``experiences`` may - come in different formats (e.g., a dictionary with agent ID's as keys). The user must implement this - method in accordance with this format. - - Args: - experiences: experiences generated by the experience shaper during post-processing. - """ - return NotImplementedError - @abstractmethod def train(self, *args, **kwargs): """Train all agents.""" return NotImplementedError - def update_epsilon(self, performance): - """This method updates the exploration rates for each agent. - - Args: - performance: Performance from the latest episode. Depending on the implementation of the explorer, - this may or may not be used in generating the exploration rates. - """ - if self._explorer: - self._explorer.update(performance) - def load_trainable_models(self, agent_model_dict): """Load models from memory for each agent.""" for agent_id, models in agent_model_dict.items(): diff --git a/maro/rl/learner/abs_learner.py b/maro/rl/learner/abs_learner.py index 58966289a..a58582514 100644 --- a/maro/rl/learner/abs_learner.py +++ b/maro/rl/learner/abs_learner.py @@ -1,7 +1,7 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT license. -from abc import ABC, abstractmethod +from abc import ABC class AbsLearner(ABC): @@ -9,16 +9,10 @@ class AbsLearner(ABC): def __init__(self): pass - @abstractmethod - def train(self, total_episodes): - """The outermost training loop logic is implemented here. - - Args: - total_episodes (int): number of episodes to be run. - """ + def train(self, *args, **kwargs): + """The outermost training loop logic is implemented here.""" pass - @abstractmethod def test(self): """Test policy performance.""" pass diff --git a/maro/rl/learner/simple_learner.py b/maro/rl/learner/simple_learner.py index c99d766b6..10c2b74a2 100644 --- a/maro/rl/learner/simple_learner.py +++ b/maro/rl/learner/simple_learner.py @@ -1,6 +1,8 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT license. +from typing import Callable + from .abs_learner import AbsLearner from maro.rl.agent.abs_agent_manager import AbsAgentManager from maro.rl.actor.simple_actor import SimpleActor @@ -15,7 +17,8 @@ class SimpleLearner(AbsLearner): actor (Actor or ActorProxy): an Actor or VectorActorProxy instance. logger: used for logging important messages. """ - def __init__(self, trainable_agents: AbsAgentManager, actor, logger=DummyLogger()): + def __init__(self, trainable_agents: AbsAgentManager, actor, logger=DummyLogger() + ): super().__init__() self._trainable_agents = trainable_agents self._actor = actor @@ -35,22 +38,23 @@ def train(self, total_episodes, model_dump_dir: str = None): performance, exp_by_agent = self._actor.roll_out(model_dict=model_dict, epsilon_dict=epsilon_dict) if isinstance(performance, dict): for actor_id, perf in performance.items(): - self._logger.info(f"ep {current_ep} - performance: {perf}," - f"source: {actor_id}, epsilons: {epsilon_dict}") + self._logger.info( + f"ep {current_ep} - performance: {perf}, source: {actor_id}, epsilons: {epsilon_dict}" + ) else: self._logger.info(f"ep {current_ep} - performance: {performance}, epsilons: {epsilon_dict}") - self._trainable_agents.store_experiences(exp_by_agent) - self._trainable_agents.train() - self._trainable_agents.update_epsilon(performance) + self._trainable_agents.train(exp_by_agent) if model_dump_dir is not None: self._trainable_agents.dump_trainable_models_to_files(model_dump_dir) def test(self): """Test policy performance.""" - performance, _ = self._actor.roll_out(model_dict=self._trainable_agents.dump_trainable_models(), - return_details=False) + performance, _ = self._actor.roll_out( + model_dict=self._trainable_agents.dump_trainable_models(), + return_details=False + ) for actor_id, perf in performance.items(): self._logger.info(f"test performance from {actor_id}: {perf}") self._actor.roll_out(done=True) From a9864afc3e8085b1afb508cd93c9cf014a67fac0 Mon Sep 17 00:00:00 2001 From: ysqyang Date: Mon, 19 Oct 2020 11:41:40 +0800 Subject: [PATCH 022/337] fixed minor issues --- examples/cim/pg/components/agent_manager.py | 13 +++++++------ examples/cim/pg/config.yml | 10 +--------- 2 files changed, 8 insertions(+), 15 deletions(-) diff --git a/examples/cim/pg/components/agent_manager.py b/examples/cim/pg/components/agent_manager.py index b70a983c7..6f32b9a43 100644 --- a/examples/cim/pg/components/agent_manager.py +++ b/examples/cim/pg/components/agent_manager.py @@ -14,16 +14,17 @@ def _assemble(self, agent_dict): set_seeds(config.agents.seed) num_actions = config.agents.algorithm.num_actions for agent_id in self._agent_id_list: - policy_model = LearningModel(decision_layers=MLPDecisionLayers(name=f'{agent_id}.policy', - input_dim=self._state_shaper.dim, - output_dim=num_actions, - **config.agents.algorithm.policy_model, - softmax=True)) + policy_model = LearningModel( + decision_layers=MLPDecisionLayers( + name=f'{agent_id}.policy', input_dim=self._state_shaper.dim, output_dim=num_actions, + **config.agents.algorithm.policy_model, softmax=True + ) + ) algorithm = PolicyGradient( policy_model=policy_model, optimizer_cls=Adam, - optimizer_params=config.agents.algorithm.policy_optimizer, + optimizer_params=config.agents.algorithm.optimizer, hyper_params=PolicyGradientHyperParameters( num_actions=num_actions, **config.agents.algorithm.hyper_parameters, diff --git a/examples/cim/pg/config.yml b/examples/cim/pg/config.yml index 839983f35..b3e75b4c3 100644 --- a/examples/cim/pg/config.yml +++ b/examples/cim/pg/config.yml @@ -42,15 +42,7 @@ agents: - 128 - 64 dropout_p: 0.0 - value_model: - hidden_dims: - - 256 - - 128 - - 64 - dropout_p: 0.0 - policy_optimizer: - lr: 0.001 - value_optimizer: + optimizer: lr: 0.001 hyper_parameters: reward_decay: .0 From a49551e374cde5ed192345d86cc52085b3949cab Mon Sep 17 00:00:00 2001 From: ysqyang Date: Mon, 19 Oct 2020 13:52:06 +0800 Subject: [PATCH 023/337] removed unwanted configs for ac and pg examples --- examples/cim/ac/config.yml | 29 ++++++---------------- examples/cim/ac/dist_actor.py | 2 +- examples/cim/ac/single_process_launcher.py | 2 +- examples/cim/pg/config.yml | 25 ++++--------------- examples/cim/pg/dist_actor.py | 2 +- examples/cim/pg/single_process_launcher.py | 2 +- 6 files changed, 16 insertions(+), 46 deletions(-) diff --git a/examples/cim/ac/config.yml b/examples/cim/ac/config.yml index c3e205d8d..51d218a16 100644 --- a/examples/cim/ac/config.yml +++ b/examples/cim/ac/config.yml @@ -20,19 +20,10 @@ state_shaping: - "full" - "remaining_space" experience_shaping: - type: "truncated" - k_step: - reward_decay: 0.9 - steps: 5 - truncated: - time_window: 100 - fulfillment_factor: 1.0 - shortage_factor: 1.0 - time_decay_factor: 0.97 -exploration: - epsilon_range: [0.0, 0.4] - split_point: [0.5, 0.8] - with_cache: true + time_window: 100 + fulfillment_factor: 1.0 + shortage_factor: 1.0 + time_decay_factor: 0.97 agents: algorithm: num_actions: 21 @@ -54,17 +45,11 @@ agents: lr: 0.001 hyper_parameters: reward_decay: .0 - num_training_rounds_per_target_replacement: 5 - tau: 0.1 - experience_pool: - capacity: -1 - training_loop_parameters: - min_experiences_to_train: 1024 - num_batches: 10 # number of times the algorithm's step() method is called - batch_size: 128 + policy_train_iters: 1 + value_train_iters: 10 seed: 1024 # for reproducibility distributed: - group_name: "dqn_distributed_test" + group_name: "ac_distributed_test" actor: peer: {"actor": 1} learner: diff --git a/examples/cim/ac/dist_actor.py b/examples/cim/ac/dist_actor.py index 831c8df13..2a30915cd 100644 --- a/examples/cim/ac/dist_actor.py +++ b/examples/cim/ac/dist_actor.py @@ -18,7 +18,7 @@ agent_id_list = [str(agent_id) for agent_id in env.agent_idx_list] state_shaper = CIMStateShaper(**config.state_shaping) action_shaper = CIMActionShaper(action_space=list(np.linspace(-1.0, 1.0, config.agents.algorithm.num_actions))) - experience_shaper = TruncatedExperienceShaper(**config.experience_shaping.truncated) + experience_shaper = TruncatedExperienceShaper(**config.experience_shaping) agent_manager = ACAgentManager( name="cim_remote_actor", diff --git a/examples/cim/ac/single_process_launcher.py b/examples/cim/ac/single_process_launcher.py index bebd22bf8..fb28c0e05 100644 --- a/examples/cim/ac/single_process_launcher.py +++ b/examples/cim/ac/single_process_launcher.py @@ -25,7 +25,7 @@ # greedy nature of the DQN algorithm. state_shaper = CIMStateShaper(**config.state_shaping) action_shaper = CIMActionShaper(action_space=list(np.linspace(-1.0, 1.0, config.agents.algorithm.num_actions))) - experience_shaper = TruncatedExperienceShaper(**config.experience_shaping.truncated) + experience_shaper = TruncatedExperienceShaper(**config.experience_shaping) # Step 3: create an agent manager. agent_manager = ACAgentManager( diff --git a/examples/cim/pg/config.yml b/examples/cim/pg/config.yml index b3e75b4c3..310f8a3f9 100644 --- a/examples/cim/pg/config.yml +++ b/examples/cim/pg/config.yml @@ -20,19 +20,10 @@ state_shaping: - "full" - "remaining_space" experience_shaping: - type: "truncated" - k_step: - reward_decay: 0.9 - steps: 5 - truncated: - time_window: 100 - fulfillment_factor: 1.0 - shortage_factor: 1.0 - time_decay_factor: 0.97 -exploration: - epsilon_range: [0.0, 0.4] - split_point: [0.5, 0.8] - with_cache: true + time_window: 100 + fulfillment_factor: 1.0 + shortage_factor: 1.0 + time_decay_factor: 0.97 agents: algorithm: num_actions: 21 @@ -46,15 +37,9 @@ agents: lr: 0.001 hyper_parameters: reward_decay: .0 - experience_pool: - capacity: -1 - training_loop_parameters: - min_experiences_to_train: 1024 - num_batches: 10 # number of times the algorithm's step() method is called - batch_size: 128 seed: 1024 # for reproducibility distributed: - group_name: "dqn_distributed_test" + group_name: "pg_distributed_test" actor: peer: {"actor": 1} learner: diff --git a/examples/cim/pg/dist_actor.py b/examples/cim/pg/dist_actor.py index 3fd058d85..325457103 100644 --- a/examples/cim/pg/dist_actor.py +++ b/examples/cim/pg/dist_actor.py @@ -18,7 +18,7 @@ agent_id_list = [str(agent_id) for agent_id in env.agent_idx_list] state_shaper = CIMStateShaper(**config.state_shaping) action_shaper = CIMActionShaper(action_space=list(np.linspace(-1.0, 1.0, config.agents.algorithm.num_actions))) - experience_shaper = TruncatedExperienceShaper(**config.experience_shaping.truncated) + experience_shaper = TruncatedExperienceShaper(**config.experience_shaping) agent_manager = DQNAgentManager( name="cim_remote_actor", agent_id_list=agent_id_list, diff --git a/examples/cim/pg/single_process_launcher.py b/examples/cim/pg/single_process_launcher.py index ed460889d..9e76f903a 100644 --- a/examples/cim/pg/single_process_launcher.py +++ b/examples/cim/pg/single_process_launcher.py @@ -25,7 +25,7 @@ # greedy nature of the DQN algorithm. state_shaper = CIMStateShaper(**config.state_shaping) action_shaper = CIMActionShaper(action_space=list(np.linspace(-1.0, 1.0, config.agents.algorithm.num_actions))) - experience_shaper = TruncatedExperienceShaper(**config.experience_shaping.truncated) + experience_shaper = TruncatedExperienceShaper(**config.experience_shaping) # Step 3: create an agent manager. agent_manager = PGAgentManager( From 79cb2981687b5c5748549d2b85a1bb18c5f7c071 Mon Sep 17 00:00:00 2001 From: ysqyang Date: Mon, 19 Oct 2020 13:58:55 +0800 Subject: [PATCH 024/337] fixed a bug --- examples/cim/ac/components/agent_manager.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/cim/ac/components/agent_manager.py b/examples/cim/ac/components/agent_manager.py index c5d14ee8a..1945440c0 100644 --- a/examples/cim/ac/components/agent_manager.py +++ b/examples/cim/ac/components/agent_manager.py @@ -29,7 +29,7 @@ def _assemble(self, agent_dict): decision_layers=MLPDecisionLayers( name=f'{agent_id}.policy', input_dim=self._state_shaper.dim, - output_dim=num_actions, + output_dim=1, **config.agents.algorithm.value_model ) ) From 7a5ded4b9e5ee38ee1960e4a0226a4243d06e626 Mon Sep 17 00:00:00 2001 From: ysqyang Date: Mon, 19 Oct 2020 14:03:36 +0800 Subject: [PATCH 025/337] fixed a shape bug in ac --- maro/rl/algorithms/torch/ac.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/maro/rl/algorithms/torch/ac.py b/maro/rl/algorithms/torch/ac.py index 4bc085bb9..90ef67ba3 100644 --- a/maro/rl/algorithms/torch/ac.py +++ b/maro/rl/algorithms/torch/ac.py @@ -103,7 +103,7 @@ def train(self, states: np.ndarray, actions: np.ndarray, rewards: np.ndarray): # value model training for _ in range(self._hyper_params.value_train_iters): - value_loss = self._value_loss_func(self._value_model(states), return_est) + value_loss = self._value_loss_func(self._value_model(states).squeeze(), return_est) self._value_optimizer.zero_grad() value_loss.backward() self._value_optimizer.step() @@ -202,7 +202,7 @@ def train(self, states: np.ndarray, actions: np.ndarray, rewards: np.ndarray): state_values, action_distribution = self._policy_value_model(states) action_prob = action_distribution.gather(1, actions.unsqueeze(1)).squeeze() # (N,) policy_loss = -(torch.log(action_prob) * advantages).mean() - value_loss = self._value_loss_func(state_values, return_est) + value_loss = self._value_loss_func(state_values.squeeze(), return_est) loss = policy_loss + value_loss self._optimizer.zero_grad() loss.backward() From c4d8c397c6c058672bf3c3da8eafd677d9181669 Mon Sep 17 00:00:00 2001 From: ysqyang Date: Mon, 19 Oct 2020 14:27:05 +0800 Subject: [PATCH 026/337] fixed a bug in ac and ppo and added an assert in k-step return function --- examples/cim/ac/config.yml | 2 ++ maro/rl/algorithms/torch/ac.py | 4 ++-- maro/rl/algorithms/torch/ppo.py | 4 ++-- maro/rl/utils/trajectory_utils.py | 3 ++- 4 files changed, 8 insertions(+), 5 deletions(-) diff --git a/examples/cim/ac/config.yml b/examples/cim/ac/config.yml index 51d218a16..ea217305f 100644 --- a/examples/cim/ac/config.yml +++ b/examples/cim/ac/config.yml @@ -47,6 +47,8 @@ agents: reward_decay: .0 policy_train_iters: 1 value_train_iters: 10 + k: -1 + lamb: 1.0 seed: 1024 # for reproducibility distributed: group_name: "ac_distributed_test" diff --git a/maro/rl/algorithms/torch/ac.py b/maro/rl/algorithms/torch/ac.py index 90ef67ba3..5486272d8 100644 --- a/maro/rl/algorithms/torch/ac.py +++ b/maro/rl/algorithms/torch/ac.py @@ -79,7 +79,7 @@ def choose_action(self, state: np.ndarray, epsilon: float = None): return np.random.choice(self._hyper_params.num_actions, p=action_dist) def _get_values_and_bootstrapped_returns(self, state_sequence, reward_sequence): - state_values = self._value_model(state_sequence).detach() + state_values = self._value_model(state_sequence).detach().squeeze() state_values_numpy = state_values.numpy() return_est = get_lambda_returns( reward_sequence, self._hyper_params.reward_decay, self._hyper_params.lamb, @@ -183,7 +183,7 @@ def choose_action(self, state: np.ndarray, epsilon: float = None): return np.random.choice(self._hyper_params.num_actions, p=action_dist) def _get_values_and_bootstrapped_returns(self, state_sequence, reward_sequence): - state_values = self._policy_value_model(state_sequence)[0].detach() + state_values = self._policy_value_model(state_sequence)[0].detach().squeeze() state_values_numpy = state_values.numpy() return_est = get_lambda_returns( reward_sequence, self._hyper_params.reward_decay, self._hyper_params.lamb, diff --git a/maro/rl/algorithms/torch/ppo.py b/maro/rl/algorithms/torch/ppo.py index c8009ef07..3717f2c41 100644 --- a/maro/rl/algorithms/torch/ppo.py +++ b/maro/rl/algorithms/torch/ppo.py @@ -77,7 +77,7 @@ def choose_action(self, state: np.ndarray, epsilon: float = None): return np.random.choice(self._hyper_params.num_actions, p=action_dist) def _get_values_and_bootstrapped_returns(self, states: torch.tensor, rewards: np.ndarray): - state_values = self._value_model(states).detach() + state_values = self._value_model(states).detach().squeeze() state_values_numpy = state_values.numpy() return_est = get_lambda_returns( rewards, self._hyper_params.reward_decay, self._hyper_params.lamb, @@ -193,7 +193,7 @@ def choose_action(self, state: np.ndarray, epsilon: float = None): return action_index, np.log(action_dist[action_index]) def _get_values_and_bootstrapped_returns(self, state_sequence, reward_sequence): - state_values = self._policy_value_model(state_sequence)[0].detach() + state_values = self._policy_value_model(state_sequence)[0].detach().squeeze() state_values_numpy = state_values.numpy() return_est = get_lambda_returns( reward_sequence, self._hyper_params.reward_decay, self._hyper_params.lamb, diff --git a/maro/rl/utils/trajectory_utils.py b/maro/rl/utils/trajectory_utils.py index 16b4937e3..a29b57602 100644 --- a/maro/rl/utils/trajectory_utils.py +++ b/maro/rl/utils/trajectory_utils.py @@ -20,8 +20,9 @@ def get_k_step_returns(rewards: np.ndarray, discount: float, k: int = -1, values Returns: An ndarray containing the k-step returns for each time step. """ - assert values is None or len(rewards) == len(values), "rewards and values should have the same length" if values is not None: + assert len(rewards) == len(values), "rewards and values should have the same length" + assert len(values.shape) == 1, "values should be a one-dimensional array" rewards[-1] = values[-1] if k < 0: k = len(rewards) - 1 From 01fb7388b12b595a4d69d252de96730ac693bac1 Mon Sep 17 00:00:00 2001 From: ysqyang Date: Mon, 19 Oct 2020 15:18:22 +0800 Subject: [PATCH 027/337] added mlp_policy_net --- examples/cim/ac/components/agent_manager.py | 12 ++-- examples/cim/pg/components/agent_manager.py | 6 +- maro/rl/__init__.py | 4 +- ...ision_layers.py => mlp_decision_layers.py} | 4 +- maro/rl/models/torch/mlp_policy_net.py | 62 +++++++++++++++++++ 5 files changed, 75 insertions(+), 13 deletions(-) rename maro/rl/models/torch/{decision_layers.py => mlp_decision_layers.py} (94%) create mode 100644 maro/rl/models/torch/mlp_policy_net.py diff --git a/examples/cim/ac/components/agent_manager.py b/examples/cim/ac/components/agent_manager.py index 1945440c0..6e13c2724 100644 --- a/examples/cim/ac/components/agent_manager.py +++ b/examples/cim/ac/components/agent_manager.py @@ -6,7 +6,8 @@ from .agent import CIMAgent from .config import config -from maro.rl import AbsAgentManager, LearningModel, MLPDecisionLayers, ActorCritic, ActorCriticHyperParameters +from maro.rl import AbsAgentManager, LearningModel, MLPPolicyNet, MLPDecisionLayers, ActorCritic, \ + ActorCriticHyperParameters from maro.utils import set_seeds @@ -16,12 +17,9 @@ def _assemble(self, agent_dict): num_actions = config.agents.algorithm.num_actions for agent_id in self._agent_id_list: policy_model = LearningModel( - decision_layers=MLPDecisionLayers( - name=f'{agent_id}.policy', - input_dim=self._state_shaper.dim, - output_dim=num_actions, - **config.agents.algorithm.policy_model, - softmax=True + decision_layers=MLPPolicyNet( + name=f'{agent_id}.policy', input_dim=self._state_shaper.dim, output_dim=num_actions, + **config.agents.algorithm.policy_model ) ) diff --git a/examples/cim/pg/components/agent_manager.py b/examples/cim/pg/components/agent_manager.py index 6f32b9a43..7b950f800 100644 --- a/examples/cim/pg/components/agent_manager.py +++ b/examples/cim/pg/components/agent_manager.py @@ -5,7 +5,7 @@ from .agent import CIMAgent from .config import config -from maro.rl import AbsAgentManager, LearningModel, MLPDecisionLayers, PolicyGradient, PolicyGradientHyperParameters +from maro.rl import AbsAgentManager, LearningModel, MLPPolicyNet, PolicyGradient, PolicyGradientHyperParameters from maro.utils import set_seeds @@ -15,9 +15,9 @@ def _assemble(self, agent_dict): num_actions = config.agents.algorithm.num_actions for agent_id in self._agent_id_list: policy_model = LearningModel( - decision_layers=MLPDecisionLayers( + decision_layers=MLPPolicyNet( name=f'{agent_id}.policy', input_dim=self._state_shaper.dim, output_dim=num_actions, - **config.agents.algorithm.policy_model, softmax=True + **config.agents.algorithm.policy_model ) ) diff --git a/maro/rl/__init__.py b/maro/rl/__init__.py index 085acbc7e..9c2edcb13 100644 --- a/maro/rl/__init__.py +++ b/maro/rl/__init__.py @@ -15,7 +15,8 @@ PPOHyperParametersWithCombinedModel from maro.rl.algorithms.torch.dqn import DQN, DQNHyperParams from maro.rl.models.torch.mlp_representation import MLPRepresentation -from maro.rl.models.torch.decision_layers import MLPDecisionLayers +from maro.rl.models.torch.mlp_policy_net import MLPPolicyNet +from maro.rl.models.torch.mlp_decision_layers import MLPDecisionLayers from maro.rl.models.torch.learning_model import LearningModel from maro.rl.storage.abs_store import AbsStore from maro.rl.storage.column_based_store import ColumnBasedStore @@ -52,6 +53,7 @@ "DQN", "DQNHyperParams", "MLPRepresentation", + "MLPPolicyNet", "MLPDecisionLayers", "LearningModel", "AbsStore", diff --git a/maro/rl/models/torch/decision_layers.py b/maro/rl/models/torch/mlp_decision_layers.py similarity index 94% rename from maro/rl/models/torch/decision_layers.py rename to maro/rl/models/torch/mlp_decision_layers.py index 715f89f6a..2197fadd1 100644 --- a/maro/rl/models/torch/decision_layers.py +++ b/maro/rl/models/torch/mlp_decision_layers.py @@ -5,9 +5,9 @@ class MLPDecisionLayers(nn.Module): - """Deep Q network. + """NN model to compute state or action values. - Choose multi-layer full connection with dropout as the basic network architecture. + Fully connected network with batch normalization, leaky RELU and dropout as layer components. Args: name (str): Network name. diff --git a/maro/rl/models/torch/mlp_policy_net.py b/maro/rl/models/torch/mlp_policy_net.py new file mode 100644 index 000000000..f9b806bc2 --- /dev/null +++ b/maro/rl/models/torch/mlp_policy_net.py @@ -0,0 +1,62 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import torch.nn as nn + + +class MLPPolicyNet(nn.Module): + """NN model to compute action distributions given states. + + Args: + name (str): Network name. + input_dim (int): Network input dimension. + hidden_dims ([int]): Network hidden layer dimension. The length of ``hidden_dims`` means the + hidden layer number, which requires larger than 1. + output_dim (int): Network output dimension. + init_w (float): If not None, [-init_w, init_w] will be the range from which the initial network parameters + will be uniformly drawn. + """ + def __init__( + self, name: str, input_dim: int, hidden_dims: [int], output_dim: int, init_w: float = 1e-3 + ): + super().__init__() + assert len(hidden_dims) > 1 + self._name = name + self._input_dim = input_dim + self._hidden_dims = hidden_dims + self._output_dim = output_dim + self._num_layers = len(self._hidden_dims) + 1 + + layer_sizes = [input_dim] + self._hidden_dims + layers = [] + for i in range(self._num_layers - 1): + layers += [ + nn.Linear(layer_sizes[i], layer_sizes[i + 1]), + nn.Tanh() + ] + self._hidden_layer = nn.Sequential(*layers) + + self._last_layer = nn.Linear(layer_sizes[-1], output_dim) + if init_w is not None: + self._last_layer.weight.data.uniform_(-init_w, init_w) + self._last_layer.bias.data.uniform_(-init_w, init_w) + + # TODO: dim=1 for batch forward; dim=0 if only one + self._soft_max = nn.Softmax() + + def forward(self, x): + x = self._hidden_layer(x) + x = self._last_layer(x) + return self._soft_max(x) + + @property + def name(self): + return self._name + + @property + def input_dim(self): + return self._input_dim + + @property + def output_dim(self): + return self._output_dim From 9148a2499c18ed44164f857d26d8cc4a0dcd6373 Mon Sep 17 00:00:00 2001 From: ysqyang Date: Mon, 19 Oct 2020 15:20:31 +0800 Subject: [PATCH 028/337] modified policy_model config accordingly --- examples/cim/ac/config.yml | 1 - examples/cim/pg/config.yml | 1 - 2 files changed, 2 deletions(-) diff --git a/examples/cim/ac/config.yml b/examples/cim/ac/config.yml index ea217305f..7a06dfdfd 100644 --- a/examples/cim/ac/config.yml +++ b/examples/cim/ac/config.yml @@ -32,7 +32,6 @@ agents: - 256 - 128 - 64 - dropout_p: 0.0 value_model: hidden_dims: - 256 diff --git a/examples/cim/pg/config.yml b/examples/cim/pg/config.yml index 310f8a3f9..0be9043f3 100644 --- a/examples/cim/pg/config.yml +++ b/examples/cim/pg/config.yml @@ -32,7 +32,6 @@ agents: - 256 - 128 - 64 - dropout_p: 0.0 optimizer: lr: 0.001 hyper_parameters: From ded6c8f717bdf0884dd9b53ea23499ed5caaa24f Mon Sep 17 00:00:00 2001 From: ysqyang Date: Mon, 19 Oct 2020 15:23:36 +0800 Subject: [PATCH 029/337] added dim arg in nn.Softmax for policy_net --- maro/rl/models/torch/mlp_policy_net.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/maro/rl/models/torch/mlp_policy_net.py b/maro/rl/models/torch/mlp_policy_net.py index f9b806bc2..8a890aed0 100644 --- a/maro/rl/models/torch/mlp_policy_net.py +++ b/maro/rl/models/torch/mlp_policy_net.py @@ -42,7 +42,7 @@ def __init__( self._last_layer.bias.data.uniform_(-init_w, init_w) # TODO: dim=1 for batch forward; dim=0 if only one - self._soft_max = nn.Softmax() + self._soft_max = nn.Softmax(dim=1) def forward(self, x): x = self._hidden_layer(x) From 03bdc3818d2a41d5ce5991d814bf5d30ef20f145 Mon Sep 17 00:00:00 2001 From: ysqyang Date: Mon, 19 Oct 2020 23:26:22 +0800 Subject: [PATCH 030/337] fixed Lint formatting issues --- maro/rl/actor/simple_actor.py | 5 +-- maro/rl/agent/abs_agent.py | 6 +--- maro/rl/agent/abs_agent_manager.py | 12 +++---- maro/rl/algorithms/torch/dqn.py | 12 ++++--- maro/rl/algorithms/torch/pg.py | 1 - maro/rl/algorithms/torch/ppo.py | 4 +-- maro/rl/learner/simple_learner.py | 2 -- maro/rl/models/torch/mlp_decision_layers.py | 10 +++--- maro/rl/shaping/k_step_experience_shaper.py | 36 ++++++++++++--------- maro/rl/storage/column_based_store.py | 7 ++-- maro/rl/storage/utils.py | 1 - maro/rl/utils/trajectory_utils.py | 17 ++++++---- 12 files changed, 58 insertions(+), 55 deletions(-) diff --git a/maro/rl/actor/simple_actor.py b/maro/rl/actor/simple_actor.py index 0c0740eed..f12e4a404 100644 --- a/maro/rl/actor/simple_actor.py +++ b/maro/rl/actor/simple_actor.py @@ -16,8 +16,9 @@ class SimpleActor(AbsActor): def __init__(self, env: Env, inference_agents: AbsAgentManager): super().__init__(env, inference_agents) - def roll_out(self, model_dict: dict = None, epsilon_dict: dict = None, done: bool = False, - return_details: bool = True): + def roll_out( + self, model_dict: dict = None, epsilon_dict: dict = None, done: bool = False, return_details: bool = True + ): """Perform one episode of roll-out and return performance and experiences. Args: diff --git a/maro/rl/agent/abs_agent.py b/maro/rl/agent/abs_agent.py index 4be9ad4ba..a57efb8d3 100644 --- a/maro/rl/agent/abs_agent.py +++ b/maro/rl/agent/abs_agent.py @@ -26,11 +26,7 @@ class AbsAgent(ABC): experience_pool (AbsStore): A data store that stores experiences generated by the experience shaper. This is only necessary for some algorithms. Defaults to None. """ - def __init__(self, - name: str, - algorithm: AbsAlgorithm, - experience_pool: AbsStore = None - ): + def __init__(self, name: str, algorithm: AbsAlgorithm, experience_pool: AbsStore = None): self._name = name self._algorithm = algorithm self._experience_pool = experience_pool diff --git a/maro/rl/agent/abs_agent_manager.py b/maro/rl/agent/abs_agent_manager.py index c4287d445..5b88797a9 100644 --- a/maro/rl/agent/abs_agent_manager.py +++ b/maro/rl/agent/abs_agent_manager.py @@ -42,14 +42,10 @@ class AbsAgentManager(ABC): executable action. Cannot be None under Inference and TrainInference modes. explorer (AbsExplorer): It is responsible for storing and updating exploration rates. """ - def __init__(self, - name: str, - mode: AgentMode, - agent_id_list: [str], - state_shaper: StateShaper = None, - action_shaper: ActionShaper = None, - experience_shaper: ExperienceShaper = None, - explorer: AbsExplorer = None): + def __init__( + self, name: str, mode: AgentMode, agent_id_list: [str], state_shaper: StateShaper = None, + action_shaper: ActionShaper = None, experience_shaper: ExperienceShaper = None, explorer: AbsExplorer = None + ): self._name = name if mode not in AgentMode: raise UnsupportedAgentModeError(msg='mode must be "train", "inference" or "train_inference"') diff --git a/maro/rl/algorithms/torch/dqn.py b/maro/rl/algorithms/torch/dqn.py index deb99f1d5..bb04f406b 100644 --- a/maro/rl/algorithms/torch/dqn.py +++ b/maro/rl/algorithms/torch/dqn.py @@ -19,8 +19,10 @@ class DQNHyperParams: tau (float): soft update coefficient, e.g., target_model = tau * eval_model + (1-tau) * target_model """ __slots__ = ["num_actions", "reward_decay", "num_training_rounds_per_target_replacement", "tau"] - def __init__(self, num_actions: int, reward_decay: float, num_training_rounds_per_target_replacement: int, - tau: float = 1.0): + + def __init__( + self, num_actions: int, reward_decay: float, num_training_rounds_per_target_replacement: int, tau: float = 1.0 + ): self.num_actions = num_actions self.reward_decay = reward_decay self.num_training_rounds_per_target_replacement = num_training_rounds_per_target_replacement @@ -41,8 +43,10 @@ class DQN(AbsAlgorithm): target_model (nn.Module): Q-value model to train the ``eval_model`` against and to be updated periodically. If it is None, the target model will be initialized as a deep copy of the eval model. """ - def __init__(self, eval_model: nn.Module, optimizer_cls, optimizer_params, loss_func, hyper_params: DQNHyperParams, - target_model: nn.Module = None): + def __init__( + self, eval_model: nn.Module, optimizer_cls, optimizer_params, loss_func, hyper_params: DQNHyperParams, + target_model: nn.Module = None + ): super().__init__() self._device = torch.device("cuda" if torch.cuda.is_available() else "cpu") self._eval_model = eval_model.to(self._device) diff --git a/maro/rl/algorithms/torch/pg.py b/maro/rl/algorithms/torch/pg.py index 91c121cb1..75ba84648 100644 --- a/maro/rl/algorithms/torch/pg.py +++ b/maro/rl/algorithms/torch/pg.py @@ -6,7 +6,6 @@ import torch.nn as nn from maro.rl.algorithms.abs_algorithm import AbsAlgorithm -from maro.rl.utils.trajectory_utils import get_k_step_returns class PolicyGradientHyperParameters: diff --git a/maro/rl/algorithms/torch/ppo.py b/maro/rl/algorithms/torch/ppo.py index 3717f2c41..27acf77ad 100644 --- a/maro/rl/algorithms/torch/ppo.py +++ b/maro/rl/algorithms/torch/ppo.py @@ -99,8 +99,8 @@ def train( for _ in range(self._hyper_params.policy_train_iters): action_prob = self._policy_model(states).gather(1, actions.unsqueeze(1)).squeeze() # (N, 1) ratio = torch.exp(torch.log(action_prob) - log_action_prob_old) - clipped_ratio = torch.clamp(ratio, 1-self._hyper_params.clip_ratio, 1+self._hyper_params.clip_ratio) - loss = -(torch.min(ratio*advantages, clipped_ratio*advantages)).mean() + clipped_ratio = torch.clamp(ratio, 1 - self._hyper_params.clip_ratio, 1 + self._hyper_params.clip_ratio) + loss = -(torch.min(ratio * advantages, clipped_ratio * advantages)).mean() self._policy_optimizer.zero_grad() loss.backward() self._policy_optimizer.step() diff --git a/maro/rl/learner/simple_learner.py b/maro/rl/learner/simple_learner.py index 10c2b74a2..a8f971fd2 100644 --- a/maro/rl/learner/simple_learner.py +++ b/maro/rl/learner/simple_learner.py @@ -1,8 +1,6 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT license. -from typing import Callable - from .abs_learner import AbsLearner from maro.rl.agent.abs_agent_manager import AbsAgentManager from maro.rl.actor.simple_actor import SimpleActor diff --git a/maro/rl/models/torch/mlp_decision_layers.py b/maro/rl/models/torch/mlp_decision_layers.py index 2197fadd1..494030a6f 100644 --- a/maro/rl/models/torch/mlp_decision_layers.py +++ b/maro/rl/models/torch/mlp_decision_layers.py @@ -58,10 +58,12 @@ def _build_basic_layer(self, input_dim, output_dim): BN -> Linear -> LeakyReLU -> Dropout """ - return nn.Sequential(nn.BatchNorm1d(input_dim), - nn.Linear(input_dim, output_dim), - nn.LeakyReLU(), - nn.Dropout(p=self._dropout_p)) + return nn.Sequential( + nn.BatchNorm1d(input_dim), + nn.Linear(input_dim, output_dim), + nn.LeakyReLU(), + nn.Dropout(p=self._dropout_p) + ) def _build_layers(self, layer_dims: []): """Build multi basic layer. diff --git a/maro/rl/shaping/k_step_experience_shaper.py b/maro/rl/shaping/k_step_experience_shaper.py index 87cd505e1..12fb19c33 100644 --- a/maro/rl/shaping/k_step_experience_shaper.py +++ b/maro/rl/shaping/k_step_experience_shaper.py @@ -41,24 +41,28 @@ def __call__(self, trajectory, snapshot_list): actions = np.asarray(trajectory.get_by_key["action"]) reward_array = np.fromiter(map(self._reward_func, trajectory.get_by_key("metrics")), dtype=np.float32) reward_sums = get_k_step_returns(reward_array, self._reward_decay, k=self._steps) - discounts = np.array([self._reward_decay ** min(self._steps, length-i-1) for i in range(length-1)]) - next_states = np.pad(states[self._steps:], (0, length-self._steps-1), mode="edge") - next_actions = np.pad(actions[self._steps:], (0, length-self._steps-1), mode="edge") + discounts = np.array([self._reward_decay ** min(self._steps, length - i - 1) for i in range(length - 1)]) + next_states = np.pad(states[self._steps:], (0, length - self._steps - 1), mode="edge") + next_actions = np.pad(actions[self._steps:], (0, length - self._steps - 1), mode="edge") states, actions = states[:-1], actions[:-1] if self._is_per_agent: - return {agent_id: {KStepExperienceKeys.STATE.value: states[agent_ids == agent_id], - KStepExperienceKeys.ACTION.value: actions[agent_ids == agent_id], - KStepExperienceKeys.REWARD.value: reward_sums[agent_ids == agent_id], - KStepExperienceKeys.NEXT_STATE.value: next_states[agent_ids == agent_id], - KStepExperienceKeys.NEXT_ACTION.value: next_actions[agent_ids == agent_id], - KStepExperienceKeys.DISCOUNT.value: discounts[agent_ids == agent_id]} - for agent_id in set(agent_ids)} + return {agent_id: { + KStepExperienceKeys.STATE.value: states[agent_ids == agent_id], + KStepExperienceKeys.ACTION.value: actions[agent_ids == agent_id], + KStepExperienceKeys.REWARD.value: reward_sums[agent_ids == agent_id], + KStepExperienceKeys.NEXT_STATE.value: next_states[agent_ids == agent_id], + KStepExperienceKeys.NEXT_ACTION.value: next_actions[agent_ids == agent_id], + KStepExperienceKeys.DISCOUNT.value: discounts[agent_ids == agent_id]} + for agent_id in set(agent_ids) + } else: - return {KStepExperienceKeys.STATE.value: states, - KStepExperienceKeys.ACTION.value: actions, - KStepExperienceKeys.REWARD.value: reward_sums, - KStepExperienceKeys.NEXT_STATE.value: next_states, - KStepExperienceKeys.NEXT_ACTION.value: next_actions, - KStepExperienceKeys.DISCOUNT.value: discounts} + return { + KStepExperienceKeys.STATE.value: states, + KStepExperienceKeys.ACTION.value: actions, + KStepExperienceKeys.REWARD.value: reward_sums, + KStepExperienceKeys.NEXT_STATE.value: next_states, + KStepExperienceKeys.NEXT_ACTION.value: next_actions, + KStepExperienceKeys.DISCOUNT.value: discounts + } diff --git a/maro/rl/storage/column_based_store.py b/maro/rl/storage/column_based_store.py index 981e77c59..a1a29026e 100644 --- a/maro/rl/storage/column_based_store.py +++ b/maro/rl/storage/column_based_store.py @@ -100,8 +100,9 @@ def put(self, contents: dict, overwrite_indexes: Sequence = None) -> List[int]: self._size += added_size return list(range(self._size - added_size, self._size)) else: - write_indexes = get_update_indexes(self._size, added_size, self._capacity, self._overwrite_type, - overwrite_indexes=overwrite_indexes) + write_indexes = get_update_indexes( + self._size, added_size, self._capacity, self._overwrite_type, overwrite_indexes=overwrite_indexes + ) self.update(write_indexes, contents) self._size = min(self._capacity, self._size + added_size) return write_indexes @@ -133,7 +134,7 @@ def apply_multi_filters(self, filters: Sequence[Callable]): Args: filters (Sequence[Callable]): Filter list, each item is a lambda function, - e.g., [lambda d: d['a'] == 1 and d['b'] == 1]. + e.g., [lambda d: d['a'] == 1 and d['b'] == 1]. Returns: Filtered indexes and corresponding objects. """ diff --git a/maro/rl/storage/utils.py b/maro/rl/storage/utils.py index ce397eb03..0e4022c82 100644 --- a/maro/rl/storage/utils.py +++ b/maro/rl/storage/utils.py @@ -3,7 +3,6 @@ from enum import Enum from functools import wraps -from typing import Sequence import numpy as np diff --git a/maro/rl/utils/trajectory_utils.py b/maro/rl/utils/trajectory_utils.py index a29b57602..942c2f6ab 100644 --- a/maro/rl/utils/trajectory_utils.py +++ b/maro/rl/utils/trajectory_utils.py @@ -26,9 +26,11 @@ def get_k_step_returns(rewards: np.ndarray, discount: float, k: int = -1, values rewards[-1] = values[-1] if k < 0: k = len(rewards) - 1 - return reduce(lambda x, y: x*discount + y, - [np.pad(rewards[i:], (0, i)) for i in range(min(k, len(rewards))-1, -1, -1)], - np.pad(values[k:], (0, k)) if values is not None else np.zeros(len(rewards))) + return reduce( + lambda x, y: x * discount + y, + [np.pad(rewards[i:], (0, i)) for i in range(min(k, len(rewards))-1, -1, -1)], + np.pad(values[k:], (0, k)) if values is not None else np.zeros(len(rewards)) + ) def get_lambda_returns(rewards: np.ndarray, discount: float, lam: float, k: int = -1, values: np.ndarray = None): @@ -58,9 +60,10 @@ def get_lambda_returns(rewards: np.ndarray, discount: float, lam: float, k: int return get_k_step_returns(rewards, discount, k=k, values=values) k = min(k, len(rewards) - 1) - pre_truncate = reduce(lambda x, y: x*lam + y, - [get_k_step_returns(rewards, discount, k=k, values=values) - for k in range(k-1, 0, -1)]) + pre_truncate = reduce( + lambda x, y: x * lam + y, + [get_k_step_returns(rewards, discount, k=k, values=values) for k in range(k - 1, 0, -1)] + ) - post_truncate = get_k_step_returns(rewards, discount, k=k, values=values) * lam**(k-1) + post_truncate = get_k_step_returns(rewards, discount, k=k, values=values) * lam**(k - 1) return (1 - lam) * pre_truncate + post_truncate From 55af6ba791e67ec59604c3f47f19b97639d2644b Mon Sep 17 00:00:00 2001 From: ysqyang Date: Mon, 19 Oct 2020 23:42:56 +0800 Subject: [PATCH 031/337] fixed Lint formatting issues --- maro/rl/learner/simple_learner.py | 3 +-- maro/rl/utils/trajectory_utils.py | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/maro/rl/learner/simple_learner.py b/maro/rl/learner/simple_learner.py index a8f971fd2..d03531d61 100644 --- a/maro/rl/learner/simple_learner.py +++ b/maro/rl/learner/simple_learner.py @@ -15,8 +15,7 @@ class SimpleLearner(AbsLearner): actor (Actor or ActorProxy): an Actor or VectorActorProxy instance. logger: used for logging important messages. """ - def __init__(self, trainable_agents: AbsAgentManager, actor, logger=DummyLogger() - ): + def __init__(self, trainable_agents: AbsAgentManager, actor, logger=DummyLogger()): super().__init__() self._trainable_agents = trainable_agents self._actor = actor diff --git a/maro/rl/utils/trajectory_utils.py b/maro/rl/utils/trajectory_utils.py index 942c2f6ab..2366b4624 100644 --- a/maro/rl/utils/trajectory_utils.py +++ b/maro/rl/utils/trajectory_utils.py @@ -28,7 +28,7 @@ def get_k_step_returns(rewards: np.ndarray, discount: float, k: int = -1, values k = len(rewards) - 1 return reduce( lambda x, y: x * discount + y, - [np.pad(rewards[i:], (0, i)) for i in range(min(k, len(rewards))-1, -1, -1)], + [np.pad(rewards[i:], (0, i)) for i in range(min(k, len(rewards)) - 1, -1, -1)], np.pad(values[k:], (0, k)) if values is not None else np.zeros(len(rewards)) ) From 9979b409f92df0b18a70b0d446b080ab06343fd8 Mon Sep 17 00:00:00 2001 From: ysqyang Date: Tue, 20 Oct 2020 13:58:01 +0800 Subject: [PATCH 032/337] merged with v0.2 and fixed a small bug --- maro/rl/agent/abs_agent_manager.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/maro/rl/agent/abs_agent_manager.py b/maro/rl/agent/abs_agent_manager.py index 5b88797a9..ca635e756 100644 --- a/maro/rl/agent/abs_agent_manager.py +++ b/maro/rl/agent/abs_agent_manager.py @@ -145,7 +145,7 @@ def dump_trainable_models_to_files(self, dir_path: str): """ os.makedirs(dir_path, exist_ok=True) for agent in self._agent_dict.values(): - agent.dump_trainable_models(dir_path) + agent.dump_trainable_models_to_file(dir_path) @property def name(self): From 49d4ee59ed3a25e2ae5f3a13ecc06baa8089444a Mon Sep 17 00:00:00 2001 From: ysqyang Date: Tue, 20 Oct 2020 23:27:41 +0800 Subject: [PATCH 033/337] fixed some PR comments --- maro/rl/actor/simple_actor.py | 2 +- maro/rl/agent/abs_agent.py | 16 ++--- maro/rl/agent/abs_agent_manager.py | 16 ++--- maro/rl/algorithms/abs_algorithm.py | 8 +-- maro/rl/algorithms/torch/ac.py | 68 +++++++++--------- maro/rl/algorithms/torch/dqn.py | 39 +++++------ maro/rl/algorithms/torch/pg.py | 8 +-- maro/rl/algorithms/torch/ppo.py | 72 +++++++++----------- maro/rl/learner/simple_learner.py | 14 ++-- maro/rl/shaping/k_step_experience_shaper.py | 10 +-- maro/rl/storage/column_based_store.py | 11 ++- maro/rl/storage/utils.py | 8 ++- maro/utils/exception/error_code.py | 3 +- maro/utils/exception/rl_toolkit_exception.py | 7 ++ tests/test_trajectory_utils.py | 4 +- 15 files changed, 146 insertions(+), 140 deletions(-) diff --git a/maro/rl/actor/simple_actor.py b/maro/rl/actor/simple_actor.py index f12e4a404..86b877fe1 100644 --- a/maro/rl/actor/simple_actor.py +++ b/maro/rl/actor/simple_actor.py @@ -42,7 +42,7 @@ def roll_out( # load models if model_dict is not None: - self._inference_agents.load_trainable_models(model_dict) + self._inference_agents.load_models(model_dict) metrics, decision_event, is_done = self._env.step(None) while not is_done: diff --git a/maro/rl/agent/abs_agent.py b/maro/rl/agent/abs_agent.py index a57efb8d3..27f387e01 100644 --- a/maro/rl/agent/abs_agent.py +++ b/maro/rl/agent/abs_agent.py @@ -66,15 +66,15 @@ def store_experiences(self, experiences): if self._experience_pool is not None: self._experience_pool.put(experiences) - def load_trainable_models(self, *models, **model_dict): + def load_models(self, *models, **model_dict): """Load models from memory.""" - self._algorithm.load_trainable_models(*models, **model_dict) + self._algorithm.load_models(*models, **model_dict) - def dump_trainable_models(self): + def dump_models(self): """Return the algorithm's trainable models.""" - return self._algorithm.dump_trainable_models() + return self._algorithm.dump_models() - def load_trainable_models_from_file(self, dir_path: str): + def load_models_from_file(self, dir_path: str): """Load trainable models from disk. Load trainable models from the specified directory. The model file is always prefixed with the agent's name. @@ -82,9 +82,9 @@ def load_trainable_models_from_file(self, dir_path: str): Args: dir_path (str): path to the directory where the models are saved. """ - self._algorithm.load_trainable_models_from_file(os.path.join(dir_path, self._name)) + self._algorithm.load_models_from_file(os.path.join(dir_path, self._name)) - def dump_trainable_models_to_file(self, dir_path: str): + def dump_models_to_file(self, dir_path: str): """Dump the algorithm's trainable models to disk. Dump trainable models to the specified directory. The model file is always prefixed with the agent's name. @@ -92,7 +92,7 @@ def dump_trainable_models_to_file(self, dir_path: str): Args: dir_path (str): path to the directory where the models are saved. """ - self._algorithm.dump_trainable_models_to_file(os.path.join(dir_path, self._name)) + self._algorithm.dump_models_to_file(os.path.join(dir_path, self._name)) def dump_experience_store(self, dir_path: str): """Dump the experience pool to disk.""" diff --git a/maro/rl/agent/abs_agent_manager.py b/maro/rl/agent/abs_agent_manager.py index ca635e756..3f22f309b 100644 --- a/maro/rl/agent/abs_agent_manager.py +++ b/maro/rl/agent/abs_agent_manager.py @@ -121,31 +121,31 @@ def train(self, *args, **kwargs): """Train all agents.""" return NotImplementedError - def load_trainable_models(self, agent_model_dict): + def load_models(self, agent_model_dict): """Load models from memory for each agent.""" for agent_id, models in agent_model_dict.items(): - self._agent_dict[agent_id].load_trainable_models(models) + self._agent_dict[agent_id].load_models(models) - def dump_trainable_models(self): + def dump_models(self): """Get agents' underlying models. This is usually used in distributed mode where models need to be broadcast to remote roll-out actors. """ - return {agent_id: agent.dump_trainable_models() for agent_id, agent in self._agent_dict.items()} + return {agent_id: agent.dump_models() for agent_id, agent in self._agent_dict.items()} - def load_trainable_models_from_files(self, dir_path): + def load_models_from_files(self, dir_path): """Load models from disk for each agent.""" for agent in self._agent_dict.values(): - agent.load_trainable_models_from_file(dir_path) + agent.load_models_from_file(dir_path) - def dump_trainable_models_to_files(self, dir_path: str): + def dump_models_to_files(self, dir_path: str): """Dump agents' models to disk. Each agent will use its own name to create a separate file under ``dir_path`` for dumping. """ os.makedirs(dir_path, exist_ok=True) for agent in self._agent_dict.values(): - agent.dump_trainable_models_to_file(dir_path) + agent.dump_models_to_file(dir_path) @property def name(self): diff --git a/maro/rl/algorithms/abs_algorithm.py b/maro/rl/algorithms/abs_algorithm.py index fabe0d354..2bab4b874 100644 --- a/maro/rl/algorithms/abs_algorithm.py +++ b/maro/rl/algorithms/abs_algorithm.py @@ -40,21 +40,21 @@ def train(self, *args, **kwargs): return NotImplementedError @abstractmethod - def load_trainable_models(self, *models, **model_dict): + def load_models(self, *models, **model_dict): """Load trainable models from memory.""" return NotImplementedError @abstractmethod - def dump_trainable_models(self): + def dump_models(self): """Return the algorithm's trainable models.""" return NotImplementedError @abstractmethod - def load_trainable_models_from_file(self, path): + def load_models_from_file(self, path): """Load trainable models from disk.""" return NotImplementedError @abstractmethod - def dump_trainable_models_to_file(self, path: str): + def dump_models_to_file(self, path: str): """Dump the algorithm's trainable models to disk.""" return NotImplementedError diff --git a/maro/rl/algorithms/torch/ac.py b/maro/rl/algorithms/torch/ac.py index 5486272d8..5324d8ca7 100644 --- a/maro/rl/algorithms/torch/ac.py +++ b/maro/rl/algorithms/torch/ac.py @@ -21,21 +21,21 @@ class ActorCriticHyperParameters: value_train_iters (int): number of gradient descent steps for the value model per call to ``train``. k (int): number of time steps used in computing returns or return estimates. Defaults to -1, in which case rewards are accumulated until the end of the trajectory. - lamb (float): lambda coefficient used in computing lambda returns. Defaults to 1.0, in which case the usual + lam (float): lambda coefficient used in computing lambda returns. Defaults to 1.0, in which case the usual k-step return is computed. """ - __slots__ = ["num_actions", "reward_decay", "policy_train_iters", "value_train_iters", "k", "lamb"] + __slots__ = ["num_actions", "reward_decay", "policy_train_iters", "value_train_iters", "k", "lam"] def __init__( self, num_actions: int, reward_decay: float, policy_train_iters, value_train_iters: int, - k: int = -1, lamb: float = 1.0 + k: int = -1, lam: float = 1.0 ): self.num_actions = num_actions self.reward_decay = reward_decay self.policy_train_iters = policy_train_iters self.value_train_iters = value_train_iters self.k = k - self.lamb = lamb + self.lam = lam class ActorCritic(AbsAlgorithm): @@ -60,29 +60,28 @@ def __init__( ): super().__init__() self._device = torch.device("cuda" if torch.cuda.is_available() else "cpu") - self._policy_model = policy_model.to(self._device) - self._value_model = value_model.to(self._device) - self._policy_optimizer = policy_optimizer_cls(self._policy_model.parameters(), **policy_optimizer_params) - self._value_optimizer = value_optimizer_cls(self._value_model.parameters(), **value_optimizer_params) + self._model_dict = {"policy": policy_model.to(self._device), "value": value_model.to(self._device)} + self._policy_optimizer = policy_optimizer_cls(self._model_dict["policy"].parameters(), **policy_optimizer_params) + self._value_optimizer = value_optimizer_cls(self._model_dict["value"].parameters(), **value_optimizer_params) self._value_loss_func = value_loss_func self._hyper_params = hyper_params @property def model(self): - return {"policy": self._policy_model, "value": self._value_model} + return self._model_dict def choose_action(self, state: np.ndarray, epsilon: float = None): state = torch.from_numpy(state).unsqueeze(0).to(self._device) # (1, state_dim) - self._policy_model.eval() + self._model_dict["policy"].eval() with torch.no_grad(): - action_dist = self._policy_model(state).squeeze().numpy() # (num_actions,) + action_dist = self._model_dict["policy"](state).squeeze().numpy() # (num_actions,) return np.random.choice(self._hyper_params.num_actions, p=action_dist) def _get_values_and_bootstrapped_returns(self, state_sequence, reward_sequence): - state_values = self._value_model(state_sequence).detach().squeeze() + state_values = self._model_dict["value"](state_sequence).detach().squeeze() state_values_numpy = state_values.numpy() return_est = get_lambda_returns( - reward_sequence, self._hyper_params.reward_decay, self._hyper_params.lamb, + reward_sequence, self._hyper_params.reward_decay, self._hyper_params.lam, k=self._hyper_params.k, values=state_values_numpy ) return_est = torch.from_numpy(return_est) @@ -95,7 +94,7 @@ def train(self, states: np.ndarray, actions: np.ndarray, rewards: np.ndarray): actions = torch.from_numpy(actions).to(self._device) # policy model training for _ in range(self._hyper_params.policy_train_iters): - action_prob = self._policy_model(states).gather(1, actions.unsqueeze(1)).squeeze() # (N,) + action_prob = self._model_dict["policy"](states).gather(1, actions.unsqueeze(1)).squeeze() # (N,) policy_loss = -(torch.log(action_prob) * advantages).mean() self._policy_optimizer.zero_grad() policy_loss.backward() @@ -103,27 +102,22 @@ def train(self, states: np.ndarray, actions: np.ndarray, rewards: np.ndarray): # value model training for _ in range(self._hyper_params.value_train_iters): - value_loss = self._value_loss_func(self._value_model(states).squeeze(), return_est) + value_loss = self._value_loss_func(self._model_dict["value"](states).squeeze(), return_est) self._value_optimizer.zero_grad() value_loss.backward() self._value_optimizer.step() - def load_trainable_models(self, model_dict): - self._policy_model = model_dict["policy"] - self._value_model = model_dict["value"] + def load_models(self, model_dict): + self._model_dict = model_dict - def dump_trainable_models(self): - return {"policy": self._policy_model, "value": self._value_model} + def dump_models(self): + return {name: model.state_dict() for name, model in self._model_dict.items()} - def load_trainable_models_from_file(self, path): - """Load trainable models from disk.""" - model_dict = torch.load(path) - self._policy_model = model_dict["policy"] - self._value_model = model_dict["value"] + def load_models_from_file(self, path): + self._model_dict = torch.load(path) - def dump_trainable_models_to_file(self, path: str): - """Dump the algorithm's trainable models to disk.""" - torch.save({"policy": self._policy_model.state_dict(), "value": self._value_model.state_dict()}, path) + def dump_models_to_file(self, path: str): + torch.save({name: model.state_dict() for name, model in self._model_dict.items()}, path) class ActorCriticHyperParametersWithCombinedModel: @@ -135,17 +129,17 @@ class ActorCriticHyperParametersWithCombinedModel: train_iters (int): number of gradient descent steps for the policy-value model per call to ``train``. k (int): number of time steps used in computing returns or return estimates. Defaults to -1, in which case rewards are accumulated until the end of the trajectory. - lamb (float): lambda coefficient used in computing lambda returns. Defaults to 1.0, in which case the usual + lam (float): lambda coefficient used in computing lambda returns. Defaults to 1.0, in which case the usual k-step return is computed. """ - __slots__ = ["num_actions", "reward_decay", "train_iters", "k", "lamb"] + __slots__ = ["num_actions", "reward_decay", "train_iters", "k", "lam"] - def __init__(self, num_actions: int, reward_decay: float, train_iters: int, k: int = -1, lamb: float = 1.0): + def __init__(self, num_actions: int, reward_decay: float, train_iters: int, k: int = -1, lam: float = 1.0): self.num_actions = num_actions self.reward_decay = reward_decay self.train_iters = train_iters self.k = k - self.lamb = lamb + self.lam = lam class ActorCriticWithCombinedModel(AbsAlgorithm): @@ -186,7 +180,7 @@ def _get_values_and_bootstrapped_returns(self, state_sequence, reward_sequence): state_values = self._policy_value_model(state_sequence)[0].detach().squeeze() state_values_numpy = state_values.numpy() return_est = get_lambda_returns( - reward_sequence, self._hyper_params.reward_decay, self._hyper_params.lamb, + reward_sequence, self._hyper_params.reward_decay, self._hyper_params.lam, k=self._hyper_params.k, values=state_values_numpy ) return_est = torch.from_numpy(return_est) @@ -208,14 +202,14 @@ def train(self, states: np.ndarray, actions: np.ndarray, rewards: np.ndarray): loss.backward() self._optimizer.step() - def load_trainable_models(self, policy_value_model): + def load_models(self, policy_value_model): self._policy_value_model = policy_value_model - def dump_trainable_models(self): + def dump_models(self): return self._policy_value_model - def load_trainable_models_from_file(self, path): + def load_models_from_file(self, path): self._policy_value_model = torch.load(path) - def dump_trainable_models_to_file(self, path: str): + def dump_models_to_file(self, path: str): torch.save(self._policy_value_model.state_dict(), path) diff --git a/maro/rl/algorithms/torch/dqn.py b/maro/rl/algorithms/torch/dqn.py index bb04f406b..e4a0ca1bc 100644 --- a/maro/rl/algorithms/torch/dqn.py +++ b/maro/rl/algorithms/torch/dqn.py @@ -49,24 +49,25 @@ def __init__( ): super().__init__() self._device = torch.device("cuda" if torch.cuda.is_available() else "cpu") - self._eval_model = eval_model.to(self._device) - self._target_model = clone(eval_model) if target_model is None else target_model - self._target_model = self._target_model.to(self._device) - self._optimizer = optimizer_cls(self._eval_model.parameters(), **optimizer_params) + self._model_dict = { + "eval": eval_model.to(self._device), + "target": clone(eval_model).to(self._device) if target_model is None else target_model.to(self._device) + } + self._optimizer = optimizer_cls(self._model_dict["eval"].parameters(), **optimizer_params) self._loss_func = loss_func self._hyper_params = hyper_params self._train_cnt = 0 @property def eval_model(self): - return self._eval_model + return self._model_dict["eval"] def choose_action(self, state: np.ndarray, epsilon: float = None): if epsilon is None or np.random.rand() > epsilon: state = torch.from_numpy(state).unsqueeze(0) - self._eval_model.eval() + self._model_dict["eval"].eval() with torch.no_grad(): - q_values = self._eval_model(state) + q_values = self._model_dict["eval"](state) return q_values.argmax(dim=1).item() return np.random.choice(self._hyper_params.num_actions) @@ -78,11 +79,11 @@ def train(self, states: np.ndarray, actions: np.ndarray, rewards: np.ndarray, ne next_states = torch.from_numpy(next_states).to(self._device) # (N, state_dim) if len(actions.shape) == 1: actions = actions.unsqueeze(1) # (N, 1) - current_q_values = self._eval_model(states).gather(1, actions).squeeze(1) # (N,) - next_q_values = self._target_model(next_states).max(dim=1)[0] # (N,) + current_q_values = self._model_dict["eval"](states).gather(1, actions).squeeze(1) # (N,) + next_q_values = self._model_dict["target"](next_states).max(dim=1)[0] # (N,) target_q_values = (rewards + self._hyper_params.reward_decay * next_q_values).detach() # (N,) loss = self._loss_func(current_q_values, target_q_values) - self._eval_model.train() + self._model_dict["eval"].train() self._optimizer.zero_grad() loss.backward() self._optimizer.step() @@ -93,23 +94,23 @@ def train(self, states: np.ndarray, actions: np.ndarray, rewards: np.ndarray, ne return np.abs((current_q_values - target_q_values).detach().numpy()) def _update_target_model(self): - for eval_params, target_params in zip(self._eval_model.parameters(), self._target_model.parameters()): + for eval_params, target_params in zip(self._model_dict["eval"].parameters(), self._model_dict["target"].parameters()): target_params.data = ( self._hyper_params.tau * eval_params.data + (1 - self._hyper_params.tau) * target_params.data ) - def load_trainable_models(self, eval_model): + def load_models(self, eval_model): """Load the eval model from memory.""" - self._eval_model = eval_model + self._model_dict["eval"] = eval_model - def dump_trainable_models(self): + def dump_models(self): """Return the eval model.""" - return self._eval_model + return self._model_dict["eval"].state_dict() - def load_trainable_models_from_file(self, path): + def load_models_from_file(self, path): """Load the eval model from disk.""" - self._eval_model = torch.load(path) + self._model_dict["eval"] = torch.load(path) - def dump_trainable_models_to_file(self, path: str): + def dump_models_to_file(self, path: str): """Dump the eval model to disk.""" - torch.save(self._eval_model.state_dict(), path) + torch.save(self._model_dict["eval"].state_dict(), path) diff --git a/maro/rl/algorithms/torch/pg.py b/maro/rl/algorithms/torch/pg.py index 75ba84648..fa7ee9bd8 100644 --- a/maro/rl/algorithms/torch/pg.py +++ b/maro/rl/algorithms/torch/pg.py @@ -65,16 +65,16 @@ def train(self, states: np.ndarray, actions: np.ndarray, returns: np.ndarray): policy_loss.backward() self._policy_optimizer.step() - def load_trainable_models(self, policy_model): + def load_models(self, policy_model): self._policy_model = policy_model - def dump_trainable_models(self): + def dump_models(self): return self._policy_model - def load_trainable_models_from_file(self, path): + def load_models_from_file(self, path): """Load trainable models from disk.""" self._policy_model = torch.load(path) - def dump_trainable_models_to_file(self, path: str): + def dump_models_to_file(self, path: str): """Dump the algorithm's trainable models to disk.""" torch.save(self._policy_model.state_dict(), path) diff --git a/maro/rl/algorithms/torch/ppo.py b/maro/rl/algorithms/torch/ppo.py index 27acf77ad..fd4fee259 100644 --- a/maro/rl/algorithms/torch/ppo.py +++ b/maro/rl/algorithms/torch/ppo.py @@ -22,14 +22,14 @@ class PPOHyperParameters: value_train_iters (int): number of gradient descent steps for the value model per call to ``train``. k (int): number of time steps used in computing returns or return estimates. Defaults to -1, in which case rewards are accumulated until the end of the trajectory. - lamb (float): lambda coefficient used in computing lambda returns. Defaults to 1.0, in which case the usual + lam (float): lambda coefficient used in computing lambda returns. Defaults to 1.0, in which case the usual k-step return is computed. """ - __slots__ = ["num_actions", "reward_decay", "clip_ratio", "policy_train_iters", "value_train_iters", "k", "lamb"] + __slots__ = ["num_actions", "reward_decay", "clip_ratio", "policy_train_iters", "value_train_iters", "k", "lam"] def __init__( self, num_actions: int, reward_decay: float, clip_ratio: float, policy_train_iters: int, - value_train_iters: int, k: int = -1, lamb: float = 1.0 + value_train_iters: int, k: int = -1, lam: float = 1.0 ): self.num_actions = num_actions self.reward_decay = reward_decay @@ -37,7 +37,7 @@ def __init__( self.policy_train_iters = policy_train_iters self.value_train_iters = value_train_iters self.k = k - self.lamb = lamb + self.lam = lam class PPO(AbsAlgorithm): @@ -62,25 +62,28 @@ def __init__( ): super().__init__() self._device = torch.device("cuda" if torch.cuda.is_available() else "cpu") - self._policy_model = policy_model.to(self._device) - self._value_model = value_model.to(self._device) - self._policy_optimizer = policy_optimizer_cls(self._policy_model.parameters(), **policy_optimizer_params) - self._value_optimizer = value_optimizer_cls(self._value_model.parameters(), **value_optimizer_params) + self._model_dict = {"policy": policy_model.to(self._device), "value": value_model.to(self._device)} + self._policy_optimizer = policy_optimizer_cls( + self._model_dict["policy"].parameters(), **policy_optimizer_params + ) + self._value_optimizer = value_optimizer_cls( + self._model_dict["value"].parameters(), **value_optimizer_params + ) self._value_loss_func = value_loss_func self._hyper_params = hyper_params def choose_action(self, state: np.ndarray, epsilon: float = None): state = torch.from_numpy(state).unsqueeze(0).to(self._device) # (1, state_dim) - self._policy_model.eval() + self._model_dict["policy"].eval() with torch.no_grad(): - action_dist = self._policy_model(state).squeeze().numpy() # (num_actions,) + action_dist = self._model_dict["policy"](state).squeeze().numpy() # (num_actions,) return np.random.choice(self._hyper_params.num_actions, p=action_dist) def _get_values_and_bootstrapped_returns(self, states: torch.tensor, rewards: np.ndarray): - state_values = self._value_model(states).detach().squeeze() + state_values = self._model_dict["value"](states).detach().squeeze() state_values_numpy = state_values.numpy() return_est = get_lambda_returns( - rewards, self._hyper_params.reward_decay, self._hyper_params.lamb, + rewards, self._hyper_params.reward_decay, self._hyper_params.lam, k=self._hyper_params.k, values=state_values_numpy ) return_est = torch.from_numpy(return_est) @@ -97,7 +100,7 @@ def train( # policy model training (with the value model fixed) for _ in range(self._hyper_params.policy_train_iters): - action_prob = self._policy_model(states).gather(1, actions.unsqueeze(1)).squeeze() # (N, 1) + action_prob = self._model_dict["policy"](states).gather(1, actions.unsqueeze(1)).squeeze() # (N, 1) ratio = torch.exp(torch.log(action_prob) - log_action_prob_old) clipped_ratio = torch.clamp(ratio, 1 - self._hyper_params.clip_ratio, 1 + self._hyper_params.clip_ratio) loss = -(torch.min(ratio * advantages, clipped_ratio * advantages)).mean() @@ -107,27 +110,22 @@ def train( # value model training for _ in range(self._hyper_params.value_train_iters): - value_loss = self._value_loss_func(self._value_model(states), return_est) + value_loss = self._value_loss_func(self._model_dict["value"](states), return_est) self._value_optimizer.zero_grad() value_loss.backward() self._value_optimizer.step() - def load_trainable_models(self, model_dict): - self._policy_model = model_dict["policy"] - self._value_model = model_dict["value"] + def load_models(self, model_dict): + self._model_dict = model_dict - def dump_trainable_models(self): - return {"policy": self._policy_model, "value": self._value_model} + def dump_models(self): + return {name: model.state_dict() for name, model in self._model_dict.items()} - def load_trainable_models_from_file(self, path): - """Load trainable models from disk.""" - model_dict = torch.load(path) - self._policy_model = model_dict["policy"] - self._value_model = model_dict["value"] + def load_models_from_file(self, path): + self._model_dict = torch.load(path) - def dump_trainable_models_to_file(self, path: str): - """Dump the algorithm's trainable models to disk.""" - torch.save({"policy": self._policy_model.state_dict(), "value": self._value_model.state_dict()}, path) + def dump_models_to_file(self, path: str): + torch.save({name: model.state_dict() for name, model in self._model_dict.items()}, path) class PPOHyperParametersWithCombinedModel: @@ -140,21 +138,21 @@ class PPOHyperParametersWithCombinedModel: train_iters (int): number of gradient descent steps for the policy-value model per call to ``train``. k (int): number of time steps used in computing returns or return estimates. Defaults to -1, in which case rewards are accumulated until the end of the trajectory. - lamb (float): lambda coefficient used in computing lambda returns. Defaults to 1.0, in which case the usual + lam (float): lambda coefficient used in computing lambda returns. Defaults to 1.0, in which case the usual k-step return is computed. """ - __slots__ = ["num_actions", "reward_decay", "clip_ratio", "train_iters", "k", "lamb"] + __slots__ = ["num_actions", "reward_decay", "clip_ratio", "train_iters", "k", "lam"] def __init__( self, num_actions: int, reward_decay: float, clip_ratio: float, train_iters: int, - k: int = -1, lamb: float = 1.0 + k: int = -1, lam: float = 1.0 ): self.num_actions = num_actions self.reward_decay = reward_decay self.clip_ratio = clip_ratio self.train_iters = train_iters self.k = k - self.lamb = lamb + self.lam = lam class PPOWithCombinedModel(AbsAlgorithm): @@ -196,7 +194,7 @@ def _get_values_and_bootstrapped_returns(self, state_sequence, reward_sequence): state_values = self._policy_value_model(state_sequence)[0].detach().squeeze() state_values_numpy = state_values.numpy() return_est = get_lambda_returns( - reward_sequence, self._hyper_params.reward_decay, self._hyper_params.lamb, + reward_sequence, self._hyper_params.reward_decay, self._hyper_params.lam, k=self._hyper_params.k, values=state_values_numpy ) return_est = torch.from_numpy(return_est) @@ -222,16 +220,14 @@ def train( loss.backward() self._optimizer.step() - def load_trainable_models(self, policy_value_model): + def load_models(self, policy_value_model): self._policy_value_model = policy_value_model - def dump_trainable_models(self): + def dump_models(self): return self._policy_value_model - def load_trainable_models_from_file(self, path): - """Load trainable models from disk.""" + def load_models_from_file(self, path): self._policy_value_model = torch.load(path) - def dump_trainable_models_to_file(self, path: str): - """Dump the algorithm's trainable models to disk.""" + def dump_models_to_file(self, path: str): torch.save(self._policy_value_model.state_dict(), path) diff --git a/maro/rl/learner/simple_learner.py b/maro/rl/learner/simple_learner.py index d03531d61..c9e7e94d7 100644 --- a/maro/rl/learner/simple_learner.py +++ b/maro/rl/learner/simple_learner.py @@ -21,16 +21,14 @@ def __init__(self, trainable_agents: AbsAgentManager, actor, logger=DummyLogger( self._actor = actor self._logger = logger - def train(self, total_episodes, model_dump_dir: str = None): + def train(self, total_episodes: int): """Main loop for collecting experiences from the actor and using them to update policies. Args: total_episodes (int): number of episodes to be run. - model_dump_dir (str): If a path is provided, it will be treated as a directory under which all agents' - trainable models will be dumped. """ for current_ep in range(1, total_episodes + 1): - model_dict = None if self._is_shared_agent_instance() else self._trainable_agents.dump_trainable_models() + model_dict = None if self._is_shared_agent_instance() else self._trainable_agents.dump_models() epsilon_dict = self._trainable_agents.explorer.epsilon if self._trainable_agents.explorer else None performance, exp_by_agent = self._actor.roll_out(model_dict=model_dict, epsilon_dict=epsilon_dict) if isinstance(performance, dict): @@ -43,19 +41,19 @@ def train(self, total_episodes, model_dump_dir: str = None): self._trainable_agents.train(exp_by_agent) - if model_dump_dir is not None: - self._trainable_agents.dump_trainable_models_to_files(model_dump_dir) - def test(self): """Test policy performance.""" performance, _ = self._actor.roll_out( - model_dict=self._trainable_agents.dump_trainable_models(), + model_dict=self._trainable_agents.dump_models(), return_details=False ) for actor_id, perf in performance.items(): self._logger.info(f"test performance from {actor_id}: {perf}") self._actor.roll_out(done=True) + def save_models(self, model_dump_dir: str): + self._trainable_agents.dump_models_to_files(model_dump_dir) + def _is_shared_agent_instance(self): """If true, the set of agents performing inference in actor is the same as self._trainable_agents.""" return isinstance(self._actor, SimpleActor) and id(self._actor.inference_agents) == id(self._trainable_agents) diff --git a/maro/rl/shaping/k_step_experience_shaper.py b/maro/rl/shaping/k_step_experience_shaper.py index 12fb19c33..4afaacdd6 100644 --- a/maro/rl/shaping/k_step_experience_shaper.py +++ b/maro/rl/shaping/k_step_experience_shaper.py @@ -31,7 +31,7 @@ class KStepExperienceShaper(ExperienceShaper): def __init__(self, reward_func: Callable, reward_decay: float, steps: int, is_per_agent: bool = True): super().__init__(reward_func) self._reward_decay = reward_decay - self._steps = steps + self._num_steps = steps self._is_per_agent = is_per_agent def __call__(self, trajectory, snapshot_list): @@ -40,10 +40,10 @@ def __call__(self, trajectory, snapshot_list): states = np.asarray(trajectory.get_by_key["state"]) actions = np.asarray(trajectory.get_by_key["action"]) reward_array = np.fromiter(map(self._reward_func, trajectory.get_by_key("metrics")), dtype=np.float32) - reward_sums = get_k_step_returns(reward_array, self._reward_decay, k=self._steps) - discounts = np.array([self._reward_decay ** min(self._steps, length - i - 1) for i in range(length - 1)]) - next_states = np.pad(states[self._steps:], (0, length - self._steps - 1), mode="edge") - next_actions = np.pad(actions[self._steps:], (0, length - self._steps - 1), mode="edge") + reward_sums = get_k_step_returns(reward_array, self._reward_decay, k=self._num_steps) + discounts = np.array([self._reward_decay ** min(self._num_steps, length - i - 1) for i in range(length - 1)]) + next_states = np.pad(states[self._num_steps:], (0, length - self._num_steps - 1), mode="edge") + next_actions = np.pad(actions[self._num_steps:], (0, length - self._num_steps - 1), mode="edge") states, actions = states[:-1], actions[:-1] diff --git a/maro/rl/storage/column_based_store.py b/maro/rl/storage/column_based_store.py index a1a29026e..a62a120e4 100644 --- a/maro/rl/storage/column_based_store.py +++ b/maro/rl/storage/column_based_store.py @@ -9,6 +9,7 @@ from .abs_store import AbsStore from .utils import check_uniformity, get_update_indexes, normalize, OverwriteType from maro.utils import clone +from maro.utils.exception.rl_toolkit_exception import StoreMisalignmentError class ColumnBasedStore(AbsStore): @@ -54,6 +55,11 @@ def __getitem__(self, index: int): return {k: lst[index] for k, lst in self._store.items()} def __getstate__(self): + """A small modification to make the object picklable. + + Using the default ``__dict__`` would make the object unpicklable due to the lambda function involved + in the ``defaultdict`` definition of the ``_store`` attribute. + """ obj_dict = self.__dict__ obj_dict["_store"] = dict(obj_dict["_store"]) return obj_dict @@ -80,7 +86,8 @@ def put(self, contents: dict, overwrite_indexes: Sequence = None) -> List[int]: """Put new contents in the store. Args: - contents (dict): Item object list. + contents (dict): dictionary of items to add to the store. If the store is not empty, this must have the + same keys as the store itself. Otherwise an ``StoreMisalignmentError`` will be raised. overwrite_indexes (Sequence, optional): indexes where the contents are to be overwritten. This is only used when the store has a fixed capacity and putting ``contents`` in the store would exceed this capacity. If this is None and overwriting is necessary, rolling or random overwriting will be done @@ -89,7 +96,7 @@ def put(self, contents: dict, overwrite_indexes: Sequence = None) -> List[int]: The indexes where the newly added entries reside in the store. """ if len(self._store) > 0 and contents.keys() != self._store.keys(): - raise ValueError(f"expected keys {list(self._store.keys())}, got {list(contents.keys())}") + raise StoreMisalignmentError(f"expected keys {list(self._store.keys())}, got {list(contents.keys())}") added_size = len(contents[next(iter(contents))]) if self._capacity < 0: for key, val in contents.items(): diff --git a/maro/rl/storage/utils.py b/maro/rl/storage/utils.py index 0e4022c82..9c4da220b 100644 --- a/maro/rl/storage/utils.py +++ b/maro/rl/storage/utils.py @@ -6,6 +6,8 @@ import numpy as np +from maro.utils.exception.rl_toolkit_exception import StoreMisalignmentError + def check_uniformity(arg_num): def decorator(func): @@ -14,9 +16,9 @@ def wrapper(*args, **kwargs): contents = args[arg_num] if all(not isinstance(val, list) for val in contents.values()): return func(*args, **kwargs) - length = len(contents[next(iter(contents))]) - if any(not isinstance(val, list) or len(val) != length for val in contents.values()): - raise ValueError("values of contents should consist of lists of the same length") + col_length = len(contents[next(iter(contents))]) + if any(not isinstance(val, list) or len(val) != col_length for val in contents.values()): + raise StoreMisalignmentError("values of contents should consist of lists of the same length") return func(*args, **kwargs) return wrapper return decorator diff --git a/maro/utils/exception/error_code.py b/maro/utils/exception/error_code.py index 34a6e8903..ae7e63ff5 100644 --- a/maro/utils/exception/error_code.py +++ b/maro/utils/exception/error_code.py @@ -37,5 +37,6 @@ # 4000-4999: Error codes for RL toolkit 4001: "Unsupported Agent Mode Error", 4002: "Missing Shaper Error", - 4003: "Wrong Agent Mode Error" + 4003: "Wrong Agent Mode Error", + 4004: "Store Misalignment Error" } diff --git a/maro/utils/exception/rl_toolkit_exception.py b/maro/utils/exception/rl_toolkit_exception.py index a914ecbdf..e39ddec72 100644 --- a/maro/utils/exception/rl_toolkit_exception.py +++ b/maro/utils/exception/rl_toolkit_exception.py @@ -20,3 +20,10 @@ class WrongAgentModeError(MAROException): """Wrong agent mode error.""" def __init__(self, msg: str = None): super().__init__(4003, msg) + + +class StoreMisalignmentError(MAROException): + """Raised when a ``put`` operation on a ``ColumnBasedStore`` would cause the underlying lists to have different + sizes.""" + def __init__(self, msg: str = None): + super().__init__(4004, msg) diff --git a/tests/test_trajectory_utils.py b/tests/test_trajectory_utils.py index 417c72716..feb9bdc85 100644 --- a/tests/test_trajectory_utils.py +++ b/tests/test_trajectory_utils.py @@ -12,7 +12,7 @@ class TestTrajectoryUtils(unittest.TestCase): def setUp(self) -> None: self.rewards = np.asarray([3, 2, 4, 1, 5]) self.values = np.asarray([4, 7, 1, 3, 6]) - self.lamb = 0.6 + self.lam = 0.6 self.discount = 0.8 self.k = 4 @@ -22,7 +22,7 @@ def test_k_step_return(self): np.testing.assert_allclose(returns, expected, rtol=1e-4) def test_lambda_return(self): - returns = get_lambda_returns(self.rewards, self.discount, self.lamb, k=self.k, values=self.values) + returns = get_lambda_returns(self.rewards, self.discount, self.lam, k=self.k, values=self.values) expected = np.asarray([8.1378176, 6.03712, 7.744, 5.8, 6.0]) np.testing.assert_allclose(returns, expected, rtol=1e-4) From 9812bf6654ec50d93cb5348cf62f53a72bce77fc Mon Sep 17 00:00:00 2001 From: ysqyang Date: Wed, 21 Oct 2020 20:59:47 +0800 Subject: [PATCH 034/337] refined RL abstractions --- examples/cim/ac/components/agent_manager.py | 93 ++++++------- examples/cim/ac/components/config.py | 13 +- examples/cim/ac/components/state_shaper.py | 5 - examples/cim/ac/config.yml | 5 +- examples/cim/ac/dist_actor.py | 8 +- examples/cim/ac/dist_learner.py | 16 +-- examples/cim/ac/single_process_launcher.py | 14 +- examples/cim/dqn/components/agent.py | 4 +- examples/cim/dqn/components/agent_manager.py | 74 +++++----- examples/cim/dqn/components/config.py | 13 +- examples/cim/dqn/dist_actor.py | 8 +- examples/cim/dqn/dist_learner.py | 18 ++- examples/cim/dqn/single_process_launcher.py | 15 +-- examples/cim/pg/components/agent_manager.py | 67 +++++----- examples/cim/pg/components/config.py | 13 +- examples/cim/pg/config.yml | 2 + examples/cim/pg/dist_actor.py | 10 +- examples/cim/pg/dist_learner.py | 19 ++- examples/cim/pg/single_process_launcher.py | 14 +- maro/rl/__init__.py | 29 ++-- maro/rl/actor/simple_actor.py | 4 +- maro/rl/agent/abs_agent.py | 21 ++- maro/rl/agent/abs_agent_manager.py | 126 ++++-------------- maro/rl/agent/simple_agent_manager.py | 105 +++++++++++++++ maro/rl/algorithms/{torch => }/ac.py | 78 ++++++----- maro/rl/algorithms/{torch => }/dqn.py | 64 +++++---- maro/rl/algorithms/{torch => }/pg.py | 22 +-- maro/rl/algorithms/{torch => }/ppo.py | 76 ++++++----- maro/rl/algorithms/torch/__init__.py | 2 - maro/rl/learner/simple_learner.py | 4 +- maro/rl/models/decision_layers.py | 88 ++++++++++++ maro/rl/models/{torch => }/learning_model.py | 8 +- ...esentation.py => representation_layers.py} | 5 +- maro/rl/models/torch/__init__.py | 2 - maro/rl/models/torch/mlp_decision_layers.py | 76 ----------- maro/rl/models/torch/mlp_policy_net.py | 62 --------- maro/rl/storage/column_based_store.py | 4 +- maro/utils/exception/error_code.py | 9 +- maro/utils/exception/rl_toolkit_exception.py | 18 ++- 39 files changed, 633 insertions(+), 581 deletions(-) create mode 100644 maro/rl/agent/simple_agent_manager.py rename maro/rl/algorithms/{torch => }/ac.py (76%) rename maro/rl/algorithms/{torch => }/dqn.py (62%) rename maro/rl/algorithms/{torch => }/pg.py (76%) rename maro/rl/algorithms/{torch => }/ppo.py (77%) delete mode 100644 maro/rl/algorithms/torch/__init__.py create mode 100644 maro/rl/models/decision_layers.py rename maro/rl/models/{torch => }/learning_model.py (84%) rename maro/rl/models/{torch/mlp_representation.py => representation_layers.py} (90%) delete mode 100644 maro/rl/models/torch/__init__.py delete mode 100644 maro/rl/models/torch/mlp_decision_layers.py delete mode 100644 maro/rl/models/torch/mlp_policy_net.py diff --git a/examples/cim/ac/components/agent_manager.py b/examples/cim/ac/components/agent_manager.py index 6e13c2724..2819cf4d2 100644 --- a/examples/cim/ac/components/agent_manager.py +++ b/examples/cim/ac/components/agent_manager.py @@ -1,67 +1,68 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT license. -from torch.nn.functional import smooth_l1_loss +import torch.nn as nn from torch.optim import Adam, RMSprop from .agent import CIMAgent -from .config import config -from maro.rl import AbsAgentManager, LearningModel, MLPPolicyNet, MLPDecisionLayers, ActorCritic, \ +from maro.rl import AgentMode, SimpleAgentManager, LearningModel, DecisionLayers, ActorCritic, \ ActorCriticHyperParameters from maro.utils import set_seeds -class ACAgentManager(AbsAgentManager): - def _assemble(self, agent_dict): - set_seeds(config.agents.seed) - num_actions = config.agents.algorithm.num_actions - for agent_id in self._agent_id_list: - policy_model = LearningModel( - decision_layers=MLPPolicyNet( - name=f'{agent_id}.policy', input_dim=self._state_shaper.dim, output_dim=num_actions, - **config.agents.algorithm.policy_model - ) +def create_ac_agents(agent_id_list, mode, config): + if mode in {AgentMode.TRAIN, AgentMode.TRAIN_INFERENCE}: + return {agent_id: ActorCritic( + policy_model=None, + value_model=None, + value_loss_func=None, + policy_optimizer_cls=None, + policy_optimizer_params=None, + value_optimizer_cls=None, + value_optimizer_params=None, + hyper_params=None, + ) + for agent_id in agent_id_list} + + set_seeds(config.seed) + num_actions = config.algorithm.num_actions + agent_dict = {} + + for agent_id in agent_id_list: + policy_model = LearningModel( + decision_layers=DecisionLayers( + name=f'{agent_id}.policy', input_dim=config.algorithm.input_dim, output_dim=num_actions, + activation=nn.Tanh, **config.algorithm.policy_model ) + ) - value_model = LearningModel( - decision_layers=MLPDecisionLayers( - name=f'{agent_id}.policy', - input_dim=self._state_shaper.dim, - output_dim=1, - **config.agents.algorithm.value_model - ) + value_model = LearningModel( + decision_layers=DecisionLayers( + name=f'{agent_id}.value', input_dim=config.algorithm.input_dim, output_dim=1, + activation=nn.LeakyReLU, **config.algorithm.value_model ) + ) - algorithm = ActorCritic( - policy_model=policy_model, - value_model=value_model, - value_loss_func=smooth_l1_loss, - policy_optimizer_cls=Adam, - policy_optimizer_params=config.agents.algorithm.policy_optimizer, - value_optimizer_cls=RMSprop, - value_optimizer_params=config.agents.algorithm.value_optimizer, - hyper_params=ActorCriticHyperParameters( - num_actions=num_actions, - **config.agents.algorithm.hyper_parameters, - ) + algorithm = ActorCritic( + policy_model=policy_model, + value_model=value_model, + value_loss_func=nn.functional.smooth_l1_loss, + policy_optimizer_cls=Adam, + policy_optimizer_params=config.algorithm.policy_optimizer, + value_optimizer_cls=RMSprop, + value_optimizer_params=config.algorithm.value_optimizer, + hyper_params=ActorCriticHyperParameters( + num_actions=num_actions, + **config.algorithm.hyper_parameters, ) + ) - agent_dict[agent_id] = CIMAgent(name=agent_id, algorithm=algorithm) + agent_dict[agent_id] = CIMAgent(name=agent_id, mode=mode, algorithm=algorithm) - def choose_action(self, decision_event, snapshot_list): - self._assert_inference_mode() - agent_id, model_state = self._state_shaper(decision_event, snapshot_list) - model_action = self._agent_dict[agent_id].choose_action( - model_state, self._explorer.epsilon[agent_id] if self._explorer else None - ) + return agent_dict - self._transition_cache = {"state": model_state, - "action": model_action, - "reward": None, - "agent_id": agent_id, - "event": decision_event} - return self._action_shaper(model_action, decision_event, snapshot_list) +class ACAgentManager(SimpleAgentManager): def train(self, experiences_by_agent: dict): for agent_id, experiences in experiences_by_agent.items(): - self._agent_dict[agent_id].train(experiences["states"], experiences["actions"], experiences["rewards"]) + self.agent_dict[agent_id].train(experiences["states"], experiences["actions"], experiences["rewards"]) diff --git a/examples/cim/ac/components/config.py b/examples/cim/ac/components/config.py index d36d60ddd..26a3a8c43 100644 --- a/examples/cim/ac/components/config.py +++ b/examples/cim/ac/components/config.py @@ -14,5 +14,14 @@ CONFIG_PATH = os.path.join(os.path.split(os.path.realpath(__file__))[0], "../config.yml") with io.open(CONFIG_PATH, "r") as in_file: - raw_config = yaml.safe_load(in_file) - config = convert_dottable(raw_config) + config = yaml.safe_load(in_file) + +# obtain model input dimension from state shaping configurations +look_back = config["state_shaping"]["look_back"] +max_ports_downstream = config["state_shaping"]["max_port_downstream"] +num_port_attributes = len(config["state_shaping"]["port_attributes"]) +num_vessel_attributes = len(config["state_shaping"]["vessel_attributes"]) + +input_dim = (look_back + 1) * (max_ports_downstream + 1) * num_port_attributes + num_vessel_attributes +config["agents"]["algorithm"]["input_dim"] = input_dim +config = convert_dottable(config) diff --git a/examples/cim/ac/components/state_shaper.py b/examples/cim/ac/components/state_shaper.py index 0e2af0ab3..58889baca 100644 --- a/examples/cim/ac/components/state_shaper.py +++ b/examples/cim/ac/components/state_shaper.py @@ -13,7 +13,6 @@ def __init__(self, *, look_back, max_ports_downstream, port_attributes, vessel_a self._max_ports_downstream = max_ports_downstream self._port_attributes = port_attributes self._vessel_attributes = vessel_attributes - self._dim = (look_back + 1) * (max_ports_downstream + 1) * len(port_attributes) + len(vessel_attributes) def __call__(self, decision_event, snapshot_list): tick, port_idx, vessel_idx = decision_event.tick, decision_event.port_idx, decision_event.vessel_idx @@ -23,7 +22,3 @@ def __call__(self, decision_event, snapshot_list): vessel_features = snapshot_list["vessels"][tick: vessel_idx: self._vessel_attributes] state = np.concatenate((port_features, vessel_features)) return str(port_idx), state - - @property - def dim(self): - return self._dim diff --git a/examples/cim/ac/config.yml b/examples/cim/ac/config.yml index 7a06dfdfd..da9dd1ee6 100644 --- a/examples/cim/ac/config.yml +++ b/examples/cim/ac/config.yml @@ -32,12 +32,15 @@ agents: - 256 - 128 - 64 + softmax_enabled: true + batch_norm_enabled: false value_model: hidden_dims: - 256 - 128 - 64 - dropout_p: 0.0 + softmax_enabled: false + batch_norm_enabled: true policy_optimizer: lr: 0.001 value_optimizer: diff --git a/examples/cim/ac/dist_actor.py b/examples/cim/ac/dist_actor.py index 2a30915cd..80f852186 100644 --- a/examples/cim/ac/dist_actor.py +++ b/examples/cim/ac/dist_actor.py @@ -4,10 +4,10 @@ import numpy as np from maro.simulator import Env -from maro.rl import AgentMode, SimpleActor, ActorWorker +from maro.rl import AgentMode, AgentManagerMode, SimpleActor, ActorWorker from components.action_shaper import CIMActionShaper -from components.agent_manager import ACAgentManager +from components.agent_manager import create_ac_agent, ACAgentManager from components.config import config from components.experience_shaper import TruncatedExperienceShaper from components.state_shaper import CIMStateShaper @@ -22,8 +22,8 @@ agent_manager = ACAgentManager( name="cim_remote_actor", - agent_id_list=agent_id_list, - mode=AgentMode.INFERENCE, + mode=AgentManagerMode.INFERENCE, + agent_dict=create_ac_agent(agent_id_list, AgentMode.INFERENCE, config.agents), state_shaper=state_shaper, action_shaper=action_shaper, experience_shaper=experience_shaper, diff --git a/examples/cim/ac/dist_learner.py b/examples/cim/ac/dist_learner.py index df9c04b77..d14b6b697 100644 --- a/examples/cim/ac/dist_learner.py +++ b/examples/cim/ac/dist_learner.py @@ -3,21 +3,21 @@ import os -from maro.rl import ActorProxy, SimpleLearner, AgentMode +from maro.rl import ActorProxy, SimpleLearner, AgentMode, AgentManagerMode from maro.simulator import Env from maro.utils import Logger -from components.agent_manager import ACAgentManager +from components.agent_manager import create_ac_agents, ACAgentManager from components.config import config -from components.state_shaper import CIMStateShaper if __name__ == "__main__": env = Env(config.env.scenario, config.env.topology, durations=config.env.durations) agent_id_list = [str(agent_id) for agent_id in env.agent_idx_list] - state_shaper = CIMStateShaper(**config.state_shaping) agent_manager = ACAgentManager( - name="cim_remote_learner", agent_id_list=agent_id_list, mode=AgentMode.TRAIN, state_shaper=state_shaper + name="cim_remote_learner", + mode=AgentManagerMode.TRAIN, + agent_dict=create_ac_agents(agent_id_list, AgentMode.TRAIN) ) proxy_params = { @@ -30,8 +30,6 @@ actor=ActorProxy(proxy_params=proxy_params), logger=Logger("distributed_cim_learner", auto_timestamp=False) ) - learner.train( - total_episodes=config.general.total_training_episodes, - model_dump_dir=os.path.join(os.getcwd(), "models") - ) + learner.train(total_episodes=config.general.total_training_episodes) learner.test() + learner.save_models(os.path.join(os.getcwd(), "models")) diff --git a/examples/cim/ac/single_process_launcher.py b/examples/cim/ac/single_process_launcher.py index fb28c0e05..c812cb8e5 100644 --- a/examples/cim/ac/single_process_launcher.py +++ b/examples/cim/ac/single_process_launcher.py @@ -6,11 +6,11 @@ import numpy as np from maro.simulator import Env -from maro.rl import SimpleLearner, SimpleActor, AgentMode, KStepExperienceShaper, TwoPhaseLinearExplorer +from maro.rl import SimpleLearner, SimpleActor, AgentMode, AgentManagerMode from maro.utils import Logger from components.action_shaper import CIMActionShaper -from components.agent_manager import ACAgentManager +from components.agent_manager import create_ac_agents, ACAgentManager from components.config import config from components.experience_shaper import TruncatedExperienceShaper from components.state_shaper import CIMStateShaper @@ -30,8 +30,8 @@ # Step 3: create an agent manager. agent_manager = ACAgentManager( name="cim_learner", - mode=AgentMode.TRAIN_INFERENCE, - agent_id_list=agent_id_list, + mode=AgentManagerMode.TRAIN_INFERENCE, + agent_dict=create_ac_agents(agent_id_list, AgentMode.TRAIN_INFERENCE, config.agents), state_shaper=state_shaper, action_shaper=action_shaper, experience_shaper=experience_shaper, @@ -43,8 +43,6 @@ trainable_agents=agent_manager, actor=actor, logger=Logger("single_host_cim_learner", auto_timestamp=False) ) - learner.train( - total_episodes=config.general.total_training_episodes, - model_dump_dir=os.path.join(os.getcwd(), "models") - ) + learner.train(total_episodes=config.general.total_training_episodes) learner.test() + learner.save_models(os.path.join(os.getcwd(), "models")) diff --git a/examples/cim/dqn/components/agent.py b/examples/cim/dqn/components/agent.py index 7f6f24255..e5ed02e1c 100644 --- a/examples/cim/dqn/components/agent.py +++ b/examples/cim/dqn/components/agent.py @@ -7,9 +7,9 @@ class CIMAgent(AbsAgent): - def __init__(self, name, algorithm, experience_pool: ColumnBasedStore, min_experiences_to_train, + def __init__(self, name, mode, algorithm, experience_pool: ColumnBasedStore, min_experiences_to_train, num_batches, batch_size): - super().__init__(name, algorithm, experience_pool) + super().__init__(name, mode, algorithm, experience_pool) self._min_experiences_to_train = min_experiences_to_train self._num_batches = num_batches self._batch_size = batch_size diff --git a/examples/cim/dqn/components/agent_manager.py b/examples/cim/dqn/components/agent_manager.py index a5195381b..4c09bb40b 100644 --- a/examples/cim/dqn/components/agent_manager.py +++ b/examples/cim/dqn/components/agent_manager.py @@ -5,60 +5,58 @@ from torch.optim import RMSprop from .agent import CIMAgent -from .config import config -from maro.rl import AbsAgentManager, LearningModel, MLPDecisionLayers, DQN, DQNHyperParams, ColumnBasedStore +from maro.rl import AgentMode, SimpleAgentManager, LearningModel, DecisionLayers, DQN, DQNHyperParams, ColumnBasedStore from maro.utils import set_seeds -class DQNAgentManager(AbsAgentManager): - def _assemble(self, agent_dict): - set_seeds(config.agents.seed) - num_actions = config.agents.algorithm.num_actions - for agent_id in self._agent_id_list: - eval_model = LearningModel( - decision_layers=MLPDecisionLayers( - name=f'{agent_id}.policy', input_dim=self._state_shaper.dim, - output_dim=num_actions, **config.agents.algorithm.model - ) - ) +def create_dqn_agents(agent_id_list, mode, config): + if mode in {AgentMode.TRAIN, AgentMode.TRAIN_INFERENCE}: + return {agent_id: DQN( + eval_model=None, + optimizer_cls=None, + optimizer_params=None, + loss_func=None, + hyper_params=None + ) + for agent_id in agent_id_list} - algorithm = DQN( - eval_model=eval_model, - optimizer_cls=RMSprop, - optimizer_params=config.agents.algorithm.optimizer, - loss_func=smooth_l1_loss, - hyper_params=DQNHyperParams( - **config.agents.algorithm.hyper_parameters, - num_actions=num_actions - ) + set_seeds(config.seed) + num_actions = config.algorithm.num_actions + agent_dict = {} + for agent_id in agent_id_list: + eval_model = LearningModel( + decision_layers=DecisionLayers( + name=f'{agent_id}.policy', input_dim=config.algorithm.input_dim, + output_dim=num_actions, **config.algorithm.model ) + ) - experience_pool = ColumnBasedStore(**config.agents.experience_pool) - agent_dict[agent_id] = CIMAgent(name=agent_id, algorithm=algorithm, experience_pool=experience_pool, - **config.agents.training_loop_parameters) + algorithm = DQN( + eval_model=eval_model, + optimizer_cls=RMSprop, + optimizer_params=config.algorithm.optimizer, + loss_func=smooth_l1_loss, + hyper_params=DQNHyperParams( + **config.algorithm.hyper_parameters, + num_actions=num_actions + ) + ) - def choose_action(self, decision_event, snapshot_list): - self._assert_inference_mode() - agent_id, model_state = self._state_shaper(decision_event, snapshot_list) - model_action = self._agent_dict[agent_id].choose_action( - model_state, self._explorer.epsilon[agent_id] if self._explorer else None) + experience_pool = ColumnBasedStore(**config.experience_pool) + agent_dict[agent_id] = CIMAgent(name=agent_id, mode=mode, algorithm=algorithm, experience_pool=experience_pool, + **config.training_loop_parameters) - self._transition_cache = {"state": model_state, - "action": model_action, - "reward": None, - "agent_id": agent_id, - "event": decision_event} - return self._action_shaper(model_action, decision_event, snapshot_list) +class DQNAgentManager(SimpleAgentManager): def train(self, experiences_by_agent, performance=None): self._assert_train_mode() # store experiences for each agent for agent_id, exp in experiences_by_agent.items(): exp.update({"loss": [1e8] * len(exp[next(iter(exp))])}) - self._agent_dict[agent_id].store_experiences(exp) + self.agent_dict[agent_id].store_experiences(exp) - for agent in self._agent_dict.values(): + for agent in self.agent_dict.values(): agent.train() # update exploration rates diff --git a/examples/cim/dqn/components/config.py b/examples/cim/dqn/components/config.py index d36d60ddd..26a3a8c43 100644 --- a/examples/cim/dqn/components/config.py +++ b/examples/cim/dqn/components/config.py @@ -14,5 +14,14 @@ CONFIG_PATH = os.path.join(os.path.split(os.path.realpath(__file__))[0], "../config.yml") with io.open(CONFIG_PATH, "r") as in_file: - raw_config = yaml.safe_load(in_file) - config = convert_dottable(raw_config) + config = yaml.safe_load(in_file) + +# obtain model input dimension from state shaping configurations +look_back = config["state_shaping"]["look_back"] +max_ports_downstream = config["state_shaping"]["max_port_downstream"] +num_port_attributes = len(config["state_shaping"]["port_attributes"]) +num_vessel_attributes = len(config["state_shaping"]["vessel_attributes"]) + +input_dim = (look_back + 1) * (max_ports_downstream + 1) * num_port_attributes + num_vessel_attributes +config["agents"]["algorithm"]["input_dim"] = input_dim +config = convert_dottable(config) diff --git a/examples/cim/dqn/dist_actor.py b/examples/cim/dqn/dist_actor.py index 82faf88e7..548efa622 100644 --- a/examples/cim/dqn/dist_actor.py +++ b/examples/cim/dqn/dist_actor.py @@ -4,10 +4,10 @@ import numpy as np from maro.simulator import Env -from maro.rl import AgentMode, SimpleActor, ActorWorker, KStepExperienceShaper, TwoPhaseLinearExplorer +from maro.rl import AgentMode, AgentManagerMode, SimpleActor, ActorWorker, KStepExperienceShaper, TwoPhaseLinearExplorer from components.action_shaper import CIMActionShaper -from components.agent_manager import DQNAgentManager +from components.agent_manager import create_dqn_agents, DQNAgentManager from components.config import config from components.experience_shaper import TruncatedExperienceShaper from components.state_shaper import CIMStateShaper @@ -34,8 +34,8 @@ explorer = TwoPhaseLinearExplorer(agent_id_list, config.general.total_training_episodes, **exploration_config) agent_manager = DQNAgentManager( name="cim_remote_actor", - agent_id_list=agent_id_list, - mode=AgentMode.INFERENCE, + mode=AgentManagerMode.INFERENCE, + agent_dict=create_dqn_agents(agent_id_list, AgentMode.INFERENCE, config.agents), state_shaper=state_shaper, action_shaper=action_shaper, experience_shaper=experience_shaper, diff --git a/examples/cim/dqn/dist_learner.py b/examples/cim/dqn/dist_learner.py index 8d39a227d..b5ce56aac 100644 --- a/examples/cim/dqn/dist_learner.py +++ b/examples/cim/dqn/dist_learner.py @@ -3,19 +3,17 @@ import os -from maro.rl import ActorProxy, SimpleLearner, AgentMode, TwoPhaseLinearExplorer +from maro.rl import ActorProxy, SimpleLearner, AgentMode, AgentManagerMode, TwoPhaseLinearExplorer from maro.simulator import Env from maro.utils import Logger -from components.agent_manager import DQNAgentManager +from components.agent_manager import create_dqn_agents, DQNAgentManager from components.config import config -from components.state_shaper import CIMStateShaper if __name__ == "__main__": env = Env(config.env.scenario, config.env.topology, durations=config.env.durations) agent_id_list = [str(agent_id) for agent_id in env.agent_idx_list] - state_shaper = CIMStateShaper(**config.state_shaping) exploration_config = { "epsilon_range_dict": {"_all_": config.exploration.epsilon_range}, "split_point_dict": {"_all_": config.exploration.split_point}, @@ -23,8 +21,10 @@ } explorer = TwoPhaseLinearExplorer(agent_id_list, config.general.total_training_episodes, **exploration_config) agent_manager = DQNAgentManager( - name="cim_remote_learner", agent_id_list=agent_id_list, mode=AgentMode.TRAIN, - state_shaper=state_shaper, explorer=explorer + name="cim_remote_learner", + mode=AgentManagerMode.TRAIN, + agent_dict=create_dqn_agents(agent_id_list, AgentMode.TRAIN, config.agents), + explorer=explorer ) proxy_params = { @@ -38,8 +38,6 @@ actor=ActorProxy(proxy_params=proxy_params), logger=Logger("distributed_cim_learner", auto_timestamp=False) ) - learner.train( - total_episodes=config.general.total_training_episodes, - model_dump_dir=os.path.join(os.getcwd(), "models") - ) + learner.train(total_episodes=config.general.total_training_episodes) learner.test() + learner.save_models(os.path.join(os.getcwd(), "models")) diff --git a/examples/cim/dqn/single_process_launcher.py b/examples/cim/dqn/single_process_launcher.py index b0321f8c5..a56a11e89 100644 --- a/examples/cim/dqn/single_process_launcher.py +++ b/examples/cim/dqn/single_process_launcher.py @@ -7,11 +7,12 @@ import numpy as np from maro.simulator import Env -from maro.rl import SimpleLearner, SimpleActor, AgentMode, KStepExperienceShaper, TwoPhaseLinearExplorer +from maro.rl import SimpleLearner, SimpleActor, AgentMode, AgentManagerMode, KStepExperienceShaper, \ + TwoPhaseLinearExplorer from maro.utils import Logger from components.action_shaper import CIMActionShaper -from components.agent_manager import DQNAgentManager +from components.agent_manager import create_dqn_agents, DQNAgentManager from components.config import config from components.experience_shaper import TruncatedExperienceShaper from components.state_shaper import CIMStateShaper @@ -44,8 +45,8 @@ # Step 3: create an agent manager. agent_manager = DQNAgentManager( name="cim_learner", - mode=AgentMode.TRAIN_INFERENCE, - agent_id_list=agent_id_list, + mode=AgentManagerMode.TRAIN_INFERENCE, + agent_dict=create_dqn_agents(agent_id_list, AgentMode.TRAIN_INFERENCE, config.agents), state_shaper=state_shaper, action_shaper=action_shaper, experience_shaper=experience_shaper, @@ -58,8 +59,6 @@ trainable_agents=agent_manager, actor=actor, logger=Logger("single_host_cim_learner", auto_timestamp=False) ) - learner.train( - total_episodes=config.general.total_training_episodes, - model_dump_dir=os.path.join(os.getcwd(), "models") - ) + learner.train(total_episodes=config.general.total_training_episodes) learner.test() + learner.save_models(os.path.join(os.getcwd(), "models")) diff --git a/examples/cim/pg/components/agent_manager.py b/examples/cim/pg/components/agent_manager.py index 7b950f800..1ee29374b 100644 --- a/examples/cim/pg/components/agent_manager.py +++ b/examples/cim/pg/components/agent_manager.py @@ -1,52 +1,51 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT license. +import torch.nn as nn from torch.optim import Adam, RMSprop from .agent import CIMAgent -from .config import config -from maro.rl import AbsAgentManager, LearningModel, MLPPolicyNet, PolicyGradient, PolicyGradientHyperParameters +from maro.rl import AgentMode, SimpleAgentManager, LearningModel, DecisionLayers, PolicyGradient, \ + PolicyGradientHyperParameters from maro.utils import set_seeds -class PGAgentManager(AbsAgentManager): - def _assemble(self, agent_dict): - set_seeds(config.agents.seed) - num_actions = config.agents.algorithm.num_actions - for agent_id in self._agent_id_list: - policy_model = LearningModel( - decision_layers=MLPPolicyNet( - name=f'{agent_id}.policy', input_dim=self._state_shaper.dim, output_dim=num_actions, - **config.agents.algorithm.policy_model - ) +def create_pg_agents(agent_id_list, mode, config): + if mode in {AgentMode.TRAIN, AgentMode.TRAIN_INFERENCE}: + return {agent_id: PolicyGradient( + policy_model=None, + optimizer_cls=None, + optimizer_params=None, + hyper_params=None, + ) + for agent_id in agent_id_list} + set_seeds(config.seed) + num_actions = config.algorithm.num_actions + agent_dict = {} + for agent_id in agent_id_list: + policy_model = LearningModel( + decision_layers=DecisionLayers( + name=f'{agent_id}.policy', input_dim=config.algorithm.input_dim, output_dim=num_actions, + activation=nn.Tanh, **config.algorithm.policy_model ) + ) - algorithm = PolicyGradient( - policy_model=policy_model, - optimizer_cls=Adam, - optimizer_params=config.agents.algorithm.optimizer, - hyper_params=PolicyGradientHyperParameters( - num_actions=num_actions, - **config.agents.algorithm.hyper_parameters, - ) + algorithm = PolicyGradient( + policy_model=policy_model, + optimizer_cls=Adam, + optimizer_params=config.algorithm.optimizer, + hyper_params=PolicyGradientHyperParameters( + num_actions=num_actions, + **config.algorithm.hyper_parameters, ) + ) - agent_dict[agent_id] = CIMAgent(name=agent_id, algorithm=algorithm) + agent_dict[agent_id] = CIMAgent(name=agent_id, mode=mode, algorithm=algorithm) - def choose_action(self, decision_event, snapshot_list): - self._assert_inference_mode() - agent_id, model_state = self._state_shaper(decision_event, snapshot_list) - model_action = self._agent_dict[agent_id].choose_action( - model_state, self._explorer.epsilon[agent_id] if self._explorer else None - ) + return agent_dict - self._transition_cache = {"state": model_state, - "action": model_action, - "reward": None, - "agent_id": agent_id, - "event": decision_event} - return self._action_shaper(model_action, decision_event, snapshot_list) +class PGAgentManager(SimpleAgentManager): def train(self, experiences_by_agent: dict): for agent_id, experiences in experiences_by_agent.items(): - self._agent_dict[agent_id].train(experiences["states"], experiences["actions"], experiences["rewards"]) + self.agent_dict[agent_id].train(experiences["states"], experiences["actions"], experiences["rewards"]) diff --git a/examples/cim/pg/components/config.py b/examples/cim/pg/components/config.py index d36d60ddd..26a3a8c43 100644 --- a/examples/cim/pg/components/config.py +++ b/examples/cim/pg/components/config.py @@ -14,5 +14,14 @@ CONFIG_PATH = os.path.join(os.path.split(os.path.realpath(__file__))[0], "../config.yml") with io.open(CONFIG_PATH, "r") as in_file: - raw_config = yaml.safe_load(in_file) - config = convert_dottable(raw_config) + config = yaml.safe_load(in_file) + +# obtain model input dimension from state shaping configurations +look_back = config["state_shaping"]["look_back"] +max_ports_downstream = config["state_shaping"]["max_port_downstream"] +num_port_attributes = len(config["state_shaping"]["port_attributes"]) +num_vessel_attributes = len(config["state_shaping"]["vessel_attributes"]) + +input_dim = (look_back + 1) * (max_ports_downstream + 1) * num_port_attributes + num_vessel_attributes +config["agents"]["algorithm"]["input_dim"] = input_dim +config = convert_dottable(config) diff --git a/examples/cim/pg/config.yml b/examples/cim/pg/config.yml index 0be9043f3..2cdb72567 100644 --- a/examples/cim/pg/config.yml +++ b/examples/cim/pg/config.yml @@ -32,6 +32,8 @@ agents: - 256 - 128 - 64 + softmax_enabled: true + batch_norm_enabled: false optimizer: lr: 0.001 hyper_parameters: diff --git a/examples/cim/pg/dist_actor.py b/examples/cim/pg/dist_actor.py index 325457103..fe2d33d03 100644 --- a/examples/cim/pg/dist_actor.py +++ b/examples/cim/pg/dist_actor.py @@ -4,10 +4,10 @@ import numpy as np from maro.simulator import Env -from maro.rl import AgentMode, SimpleActor, ActorWorker, KStepExperienceShaper, TwoPhaseLinearExplorer +from maro.rl import AgentMode, AgentManagerMode, SimpleActor, ActorWorker from components.action_shaper import CIMActionShaper -from components.agent_manager import DQNAgentManager +from components.agent_manager import create_pg_agents, PGAgentManager from components.config import config from components.experience_shaper import TruncatedExperienceShaper from components.state_shaper import CIMStateShaper @@ -19,10 +19,10 @@ state_shaper = CIMStateShaper(**config.state_shaping) action_shaper = CIMActionShaper(action_space=list(np.linspace(-1.0, 1.0, config.agents.algorithm.num_actions))) experience_shaper = TruncatedExperienceShaper(**config.experience_shaping) - agent_manager = DQNAgentManager( + agent_manager = PGAgentManager( name="cim_remote_actor", - agent_id_list=agent_id_list, - mode=AgentMode.INFERENCE, + mode=AgentManagerMode.INFERENCE, + agent_dict=create_pg_agents(agent_id_list, AgentMode.INFERENCE, config.agents), state_shaper=state_shaper, action_shaper=action_shaper, experience_shaper=experience_shaper diff --git a/examples/cim/pg/dist_learner.py b/examples/cim/pg/dist_learner.py index 8d39a227d..5c8fb6914 100644 --- a/examples/cim/pg/dist_learner.py +++ b/examples/cim/pg/dist_learner.py @@ -3,28 +3,27 @@ import os -from maro.rl import ActorProxy, SimpleLearner, AgentMode, TwoPhaseLinearExplorer +from maro.rl import ActorProxy, SimpleLearner, AgentMode, AgentManagerMode, TwoPhaseLinearExplorer from maro.simulator import Env from maro.utils import Logger -from components.agent_manager import DQNAgentManager +from components.agent_manager import create_pg_agents, PGAgentManager from components.config import config -from components.state_shaper import CIMStateShaper if __name__ == "__main__": env = Env(config.env.scenario, config.env.topology, durations=config.env.durations) agent_id_list = [str(agent_id) for agent_id in env.agent_idx_list] - state_shaper = CIMStateShaper(**config.state_shaping) exploration_config = { "epsilon_range_dict": {"_all_": config.exploration.epsilon_range}, "split_point_dict": {"_all_": config.exploration.split_point}, "with_cache": config.exploration.with_cache } explorer = TwoPhaseLinearExplorer(agent_id_list, config.general.total_training_episodes, **exploration_config) - agent_manager = DQNAgentManager( - name="cim_remote_learner", agent_id_list=agent_id_list, mode=AgentMode.TRAIN, - state_shaper=state_shaper, explorer=explorer + agent_manager = PGAgentManager( + name="cim_remote_learner", + mode=AgentManagerMode.TRAIN, + agent_dict=create_pg_agents(agent_id_list, AgentMode.TRAIN, config.agents), ) proxy_params = { @@ -38,8 +37,6 @@ actor=ActorProxy(proxy_params=proxy_params), logger=Logger("distributed_cim_learner", auto_timestamp=False) ) - learner.train( - total_episodes=config.general.total_training_episodes, - model_dump_dir=os.path.join(os.getcwd(), "models") - ) + learner.train(total_episodes=config.general.total_training_episodes) learner.test() + learner.save_models(os.path.join(os.getcwd(), "models")) diff --git a/examples/cim/pg/single_process_launcher.py b/examples/cim/pg/single_process_launcher.py index 9e76f903a..ee8f8f56d 100644 --- a/examples/cim/pg/single_process_launcher.py +++ b/examples/cim/pg/single_process_launcher.py @@ -6,11 +6,11 @@ import numpy as np from maro.simulator import Env -from maro.rl import SimpleLearner, SimpleActor, AgentMode +from maro.rl import SimpleLearner, SimpleActor, AgentMode, AgentManagerMode from maro.utils import Logger from components.action_shaper import CIMActionShaper -from components.agent_manager import PGAgentManager +from components.agent_manager import create_pg_agents, PGAgentManager from components.config import config from components.experience_shaper import TruncatedExperienceShaper from components.state_shaper import CIMStateShaper @@ -30,8 +30,8 @@ # Step 3: create an agent manager. agent_manager = PGAgentManager( name="cim_learner", - mode=AgentMode.TRAIN_INFERENCE, - agent_id_list=agent_id_list, + mode=AgentManagerMode.TRAIN_INFERENCE, + agent_dict=create_pg_agents(agent_id_list, AgentMode.TRAIN_INFERENCE, config.agents), state_shaper=state_shaper, action_shaper=action_shaper, experience_shaper=experience_shaper, @@ -43,8 +43,6 @@ trainable_agents=agent_manager, actor=actor, logger=Logger("single_host_cim_learner", auto_timestamp=False) ) - learner.train( - total_episodes=config.general.total_training_episodes, - model_dump_dir=os.path.join(os.getcwd(), "models") - ) + learner.train(total_episodes=config.general.total_training_episodes) learner.test() + learner.save_models(os.path.join(os.getcwd(), "models")) diff --git a/maro/rl/__init__.py b/maro/rl/__init__.py index 9c2edcb13..38be480e8 100644 --- a/maro/rl/__init__.py +++ b/maro/rl/__init__.py @@ -5,19 +5,19 @@ from maro.rl.actor.simple_actor import SimpleActor from maro.rl.learner.abs_learner import AbsLearner from maro.rl.learner.simple_learner import SimpleLearner -from maro.rl.agent.abs_agent import AbsAgent -from maro.rl.agent.abs_agent_manager import AbsAgentManager, AgentMode +from maro.rl.agent.abs_agent import AbsAgent, AgentMode +from maro.rl.agent.abs_agent_manager import AbsAgentManager, AgentManagerMode +from maro.rl.agent.simple_agent_manager import SimpleAgentManager from maro.rl.algorithms.abs_algorithm import AbsAlgorithm -from maro.rl.algorithms.torch.pg import PolicyGradient, PolicyGradientHyperParameters -from maro.rl.algorithms.torch.ac import ActorCritic, ActorCriticWithCombinedModel, ActorCriticHyperParameters, \ +from maro.rl.algorithms.pg import PolicyGradient, PolicyGradientHyperParameters +from maro.rl.algorithms.ac import ActorCritic, ActorCriticWithCombinedModel, ActorCriticHyperParameters, \ ActorCriticHyperParametersWithCombinedModel -from maro.rl.algorithms.torch.ppo import PPO, PPOWithCombinedModel, PPOHyperParameters, \ +from maro.rl.algorithms.ppo import PPO, PPOWithCombinedModel, PPOHyperParameters, \ PPOHyperParametersWithCombinedModel -from maro.rl.algorithms.torch.dqn import DQN, DQNHyperParams -from maro.rl.models.torch.mlp_representation import MLPRepresentation -from maro.rl.models.torch.mlp_policy_net import MLPPolicyNet -from maro.rl.models.torch.mlp_decision_layers import MLPDecisionLayers -from maro.rl.models.torch.learning_model import LearningModel +from maro.rl.algorithms.dqn import DQN, DQNHyperParams +from maro.rl.models.learning_model import LearningModel +from maro.rl.models.representation_layers import RepresentationLayers +from maro.rl.models.decision_layers import DecisionLayers from maro.rl.storage.abs_store import AbsStore from maro.rl.storage.column_based_store import ColumnBasedStore from maro.rl.storage.utils import OverwriteType @@ -37,8 +37,10 @@ "AbsLearner", "SimpleLearner", "AbsAgent", - "AbsAgentManager", "AgentMode", + "AbsAgentManager", + "AgentManagerMode", + "SimpleAgentManager", "AbsAlgorithm", "PolicyGradient", "PolicyGradientHyperParameters", @@ -52,10 +54,9 @@ "PPOHyperParametersWithCombinedModel", "DQN", "DQNHyperParams", - "MLPRepresentation", - "MLPPolicyNet", - "MLPDecisionLayers", "LearningModel", + "RepresentationLayers", + "DecisionLayers", "AbsStore", "ColumnBasedStore", "OverwriteType", diff --git a/maro/rl/actor/simple_actor.py b/maro/rl/actor/simple_actor.py index 86b877fe1..f196c3c02 100644 --- a/maro/rl/actor/simple_actor.py +++ b/maro/rl/actor/simple_actor.py @@ -2,7 +2,7 @@ # Licensed under the MIT license. from .abs_actor import AbsActor -from maro.rl.agent.abs_agent_manager import AbsAgentManager +from maro.rl.agent.simple_agent_manager import SimpleAgentManager from maro.simulator import Env @@ -13,7 +13,7 @@ class SimpleActor(AbsActor): env (Env): An Env instance. inference_agents (AbsAgentManager): An AgentManager instance that manages all agents. """ - def __init__(self, env: Env, inference_agents: AbsAgentManager): + def __init__(self, env: Env, inference_agents: SimpleAgentManager): super().__init__(env, inference_agents) def roll_out( diff --git a/maro/rl/agent/abs_agent.py b/maro/rl/agent/abs_agent.py index 27f387e01..f6fb7ffc6 100644 --- a/maro/rl/agent/abs_agent.py +++ b/maro/rl/agent/abs_agent.py @@ -2,11 +2,19 @@ # Licensed under the MIT license. from abc import ABC, abstractmethod +from enum import Enum import os import pickle from maro.rl.algorithms.abs_algorithm import AbsAlgorithm from maro.rl.storage.abs_store import AbsStore +from maro.utils.exception.rl_toolkit_exception import WrongAgentModeError + + +class AgentMode(Enum): + TRAIN = "train" + INFERENCE = "inference" + TRAIN_INFERENCE = "train_inference" class AbsAgent(ABC): @@ -20,14 +28,16 @@ class AbsAgent(ABC): Args: name (str): Agent's name. + mode (AgentMode): An ``AgentMode`` enum member that indicates the role of the agent in the current process. algorithm (AbsAlgorithm): A concrete algorithm instance that inherits from AbstractAlgorithm. This is the centerpiece of the Agent class and is responsible for the most important tasks of an agent: choosing actions and optimizing models. experience_pool (AbsStore): A data store that stores experiences generated by the experience shaper. This is only necessary for some algorithms. Defaults to None. """ - def __init__(self, name: str, algorithm: AbsAlgorithm, experience_pool: AbsStore = None): + def __init__(self, name: str, mode: AgentMode, algorithm: AbsAlgorithm, experience_pool: AbsStore = None): self._name = name + self._mode = mode self._algorithm = algorithm self._experience_pool = experience_pool @@ -50,6 +60,7 @@ def choose_action(self, model_state, epsilon: float = .0): Returns: Action given by the underlying policy model. """ + self._assert_inference_mode() return self._algorithm.choose_action(model_state, epsilon) @abstractmethod @@ -99,3 +110,11 @@ def dump_experience_store(self, dir_path: str): if self._experience_pool is not None: with open(os.path.join(dir_path, self._name)) as fp: pickle.dump(self._experience_pool, fp) + + def _assert_train_mode(self): + if self._mode != AgentMode.TRAIN and self._mode != AgentMode.TRAIN_INFERENCE: + raise WrongAgentModeError(msg=f"this method is unavailable under mode {self._mode}") + + def _assert_inference_mode(self): + if self._mode != AgentMode.INFERENCE and self._mode != AgentMode.TRAIN_INFERENCE: + raise WrongAgentModeError(msg=f"this method is unavailable under mode {self._mode}") diff --git a/maro/rl/agent/abs_agent_manager.py b/maro/rl/agent/abs_agent_manager.py index 3f22f309b..ffaa13d3d 100644 --- a/maro/rl/agent/abs_agent_manager.py +++ b/maro/rl/agent/abs_agent_manager.py @@ -3,17 +3,15 @@ from abc import ABC, abstractmethod from enum import Enum -import os from maro.rl.shaping.state_shaper import StateShaper from maro.rl.shaping.action_shaper import ActionShaper from maro.rl.shaping.experience_shaper import ExperienceShaper from maro.rl.explorer.abs_explorer import AbsExplorer -from maro.rl.storage.column_based_store import ColumnBasedStore -from maro.utils.exception.rl_toolkit_exception import UnsupportedAgentModeError, MissingShaperError, WrongAgentModeError +from maro.utils.exception.rl_toolkit_exception import WrongAgentManagerModeError -class AgentMode(Enum): +class AgentManagerMode(Enum): TRAIN = "train" INFERENCE = "inference" TRAIN_INFERENCE = "train_inference" @@ -31,9 +29,9 @@ class AbsAgentManager(ABC): Args: name (str): Name of agent manager. - mode (AgentMode): An AgentMode enum member that specifies that role of the agent. Some attributes may - be None under certain modes. - agent_id_list (list): List of agent identifiers. + mode (AgentManagerMode): An ``AgentManagerNode`` enum member that indicates the role of the agent manager + in the current process. + agent_dict (dict): List of agent identifiers. experience_shaper (ExperienceShaper, optional): It is responsible for processing data in the replay buffer at the end of an episode. state_shaper (StateShaper, optional): It is responsible for converting the environment observation to model @@ -43,129 +41,59 @@ class AbsAgentManager(ABC): explorer (AbsExplorer): It is responsible for storing and updating exploration rates. """ def __init__( - self, name: str, mode: AgentMode, agent_id_list: [str], state_shaper: StateShaper = None, - action_shaper: ActionShaper = None, experience_shaper: ExperienceShaper = None, explorer: AbsExplorer = None + self, name: str, mode: AgentManagerMode, agent_dict: dict, + state_shaper: StateShaper = None, action_shaper: ActionShaper = None, + experience_shaper: ExperienceShaper = None, explorer: AbsExplorer = None ): self._name = name - if mode not in AgentMode: - raise UnsupportedAgentModeError(msg='mode must be "train", "inference" or "train_inference"') self._mode = mode - - if mode in {AgentMode.INFERENCE, AgentMode.TRAIN_INFERENCE}: - if state_shaper is None: - raise MissingShaperError(msg=f"state shaper cannot be None under mode {self._mode}") - if action_shaper is None: - raise MissingShaperError(msg=f"action_shaper cannot be None under mode {self._mode}") - if experience_shaper is None: - raise MissingShaperError(msg=f"experience_shaper cannot be None under mode {self._mode}") - + self.agent_dict = agent_dict self._state_shaper = state_shaper self._action_shaper = action_shaper self._experience_shaper = experience_shaper self._explorer = explorer - self._agent_id_list = agent_id_list - self._transition_cache = {} - self._trajectory = ColumnBasedStore() - - self._agent_dict = {} - self._assemble(self._agent_dict) - def __getitem__(self, agent_id): - return self._agent_dict[agent_id] - - def _assemble(self, agent_dict): - """Assembles agents and fill the ``agent_dict`` with them.""" - return NotImplemented + return self.agent_dict[agent_id] @abstractmethod - def choose_action(self, decision_event, snapshot_list): + def choose_action(self, *args, **kwargs): """Generate an environment executable action given the current decision event and snapshot list. - - Key information can be recorded in the ``_transition_cache`` attribute for experience shaping. - - Args: - decision_event: A decision event that prompts an action. - snapshot_list: An object that holds the detailed history of past env observations. - - Returns: - An action object that can be passed directly to an environment's ``step`` method. """ - return NotImplementedError - - def on_env_feedback(self, metrics): - """This method records the environment-generated metrics as part of the latest transition in the trajectory. + return NotImplemented - Args: - metrics: business metrics provided by the environment after an action has been executed. - """ - self._transition_cache["metrics"] = metrics - self._trajectory.put(self._transition_cache) + @abstractmethod + def on_env_feedback(self, *args, **kwargs): + """Do things after a feedback is received from the environment following an action.""" + return NotImplemented - def post_process(self, snapshot_list): - """This method processes the latest trajectory into experiences. + @abstractmethod + def post_process(self, *args, **kwargs): + """Do things after an episode is finished. - Args: - snapshot_list: the snapshot list from the env at the end of an episode. + These things may involve shaping experiences and resetting stateful objects. """ - experiences = self._experience_shaper(self._trajectory, snapshot_list) - self._trajectory.clear() - self._transition_cache = {} - self._state_shaper.reset() - self._action_shaper.reset() - self._experience_shaper.reset() - return experiences + return NotImplemented @abstractmethod def train(self, *args, **kwargs): - """Train all agents.""" - return NotImplementedError - - def load_models(self, agent_model_dict): - """Load models from memory for each agent.""" - for agent_id, models in agent_model_dict.items(): - self._agent_dict[agent_id].load_models(models) - - def dump_models(self): - """Get agents' underlying models. - - This is usually used in distributed mode where models need to be broadcast to remote roll-out actors. - """ - return {agent_id: agent.dump_models() for agent_id, agent in self._agent_dict.items()} - - def load_models_from_files(self, dir_path): - """Load models from disk for each agent.""" - for agent in self._agent_dict.values(): - agent.load_models_from_file(dir_path) - - def dump_models_to_files(self, dir_path: str): - """Dump agents' models to disk. - - Each agent will use its own name to create a separate file under ``dir_path`` for dumping. - """ - os.makedirs(dir_path, exist_ok=True) - for agent in self._agent_dict.values(): - agent.dump_models_to_file(dir_path) + """Train the agents.""" + return NotImplemented @property def name(self): """Agent manager's name.""" return self._name - @property - def agents(self): - """Agents managed by the agent manager.""" - return self._agent_dict - @property def explorer(self): """Explorer used by the agent manager.""" return self._explorer def _assert_train_mode(self): - if self._mode != AgentMode.TRAIN and self._mode != AgentMode.TRAIN_INFERENCE: - raise WrongAgentModeError(msg=f"this method is unavailable under mode {self._mode}") + if self._mode != AgentManagerMode.TRAIN and self._mode != AgentManagerMode.TRAIN_INFERENCE: + raise WrongAgentManagerModeError(msg=f"this method is unavailable under mode {self._mode}") def _assert_inference_mode(self): - if self._mode != AgentMode.INFERENCE and self._mode != AgentMode.TRAIN_INFERENCE: - raise WrongAgentModeError(msg=f"this method is unavailable under mode {self._mode}") + if self._mode != AgentManagerMode.INFERENCE and self._mode != AgentManagerMode.TRAIN_INFERENCE: + raise WrongAgentManagerModeError(msg=f"this method is unavailable under mode {self._mode}") diff --git a/maro/rl/agent/simple_agent_manager.py b/maro/rl/agent/simple_agent_manager.py new file mode 100644 index 000000000..6f7237aeb --- /dev/null +++ b/maro/rl/agent/simple_agent_manager.py @@ -0,0 +1,105 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +from abc import abstractmethod +import os + +from .abs_agent_manager import AbsAgentManager, AgentManagerMode +from maro.rl.shaping.state_shaper import StateShaper +from maro.rl.shaping.action_shaper import ActionShaper +from maro.rl.shaping.experience_shaper import ExperienceShaper +from maro.rl.explorer.abs_explorer import AbsExplorer +from maro.rl.storage.column_based_store import ColumnBasedStore +from maro.utils.exception.rl_toolkit_exception import MissingShaperError + + +class SimpleAgentManager(AbsAgentManager): + def __init__( + self, name: str, mode: AgentManagerMode, agent_dict: dict, + state_shaper: StateShaper = None, action_shaper: ActionShaper = None, + experience_shaper: ExperienceShaper = None, explorer: AbsExplorer = None, + ): + if mode in {AgentManagerMode.INFERENCE, AgentManagerMode.TRAIN_INFERENCE}: + if state_shaper is None: + raise MissingShaperError(msg=f"state shaper cannot be None under mode {self._mode}") + if action_shaper is None: + raise MissingShaperError(msg=f"action_shaper cannot be None under mode {self._mode}") + if experience_shaper is None: + raise MissingShaperError(msg=f"experience_shaper cannot be None under mode {self._mode}") + + super().__init__( + name, mode, agent_dict, state_shaper=state_shaper, action_shaper=action_shaper, + experience_shaper=experience_shaper, explorer=explorer + ) + + # data structures to temporarily store transitions and trajectory + self._transition_cache = {} + self._trajectory = ColumnBasedStore() + + def choose_action(self, decision_event, snapshot_list): + self._assert_inference_mode() + agent_id, model_state = self._state_shaper(decision_event, snapshot_list) + model_action = self.agent_dict[agent_id].choose_action( + model_state, self._explorer.epsilon[agent_id] if self._explorer else None + ) + + self._transition_cache = {"state": model_state, + "action": model_action, + "reward": None, + "agent_id": agent_id, + "event": decision_event} + return self._action_shaper(model_action, decision_event, snapshot_list) + + def on_env_feedback(self, metrics): + """This method records the environment-generated metrics as part of the latest transition in the trajectory. + + Args: + metrics: business metrics provided by the environment after an action has been executed. + """ + self._transition_cache["metrics"] = metrics + self._trajectory.put(self._transition_cache) + + def post_process(self, snapshot_list): + """This method processes the latest trajectory into experiences. + + Args: + snapshot_list: the snapshot list from the env at the end of an episode. + """ + experiences = self._experience_shaper(self._trajectory, snapshot_list) + self._trajectory.clear() + self._transition_cache = {} + self._state_shaper.reset() + self._action_shaper.reset() + self._experience_shaper.reset() + return experiences + + @abstractmethod + def train(self, *args, **kwargs): + """Train all agents.""" + return NotImplementedError + + def load_models(self, agent_model_dict): + """Load models from memory for each agent.""" + for agent_id, models in agent_model_dict.items(): + self.agent_dict[agent_id].load_models(models) + + def dump_models(self): + """Get agents' underlying models. + + This is usually used in distributed mode where models need to be broadcast to remote roll-out actors. + """ + return {agent_id: agent.dump_models() for agent_id, agent in self.agent_dict.items()} + + def load_models_from_files(self, dir_path): + """Load models from disk for each agent.""" + for agent in self.agent_dict.values(): + agent.load_models_from_file(dir_path) + + def dump_models_to_files(self, dir_path: str): + """Dump agents' models to disk. + + Each agent will use its own name to create a separate file under ``dir_path`` for dumping. + """ + os.makedirs(dir_path, exist_ok=True) + for agent in self.agent_dict.values(): + agent.dump_models_to_file(dir_path) diff --git a/maro/rl/algorithms/torch/ac.py b/maro/rl/algorithms/ac.py similarity index 76% rename from maro/rl/algorithms/torch/ac.py rename to maro/rl/algorithms/ac.py index 5324d8ca7..79d06957b 100644 --- a/maro/rl/algorithms/torch/ac.py +++ b/maro/rl/algorithms/ac.py @@ -47,9 +47,11 @@ class ActorCritic(AbsAlgorithm): policy_model (nn.Module): model for generating actions given states. value_model (nn.Module): model for estimating state values. value_loss_func (Callable): loss function for the value model. - policy_optimizer_cls: torch optimizer class for the policy model. + policy_optimizer_cls: torch optimizer class for the policy model. If this is None, the policy_model is not + trainable. policy_optimizer_params: parameters required for the policy optimizer class. - value_optimizer_cls: torch optimizer class for the value model. + value_optimizer_cls: torch optimizer class for the value model. If this is None, the value model is not + trainable. value_optimizer_params: parameters required for the value optimizer class. hyper_params: hyper-parameter set for the AC algorithm. """ @@ -61,8 +63,14 @@ def __init__( super().__init__() self._device = torch.device("cuda" if torch.cuda.is_available() else "cpu") self._model_dict = {"policy": policy_model.to(self._device), "value": value_model.to(self._device)} - self._policy_optimizer = policy_optimizer_cls(self._model_dict["policy"].parameters(), **policy_optimizer_params) - self._value_optimizer = value_optimizer_cls(self._model_dict["value"].parameters(), **value_optimizer_params) + if policy_optimizer_cls is not None: + self._policy_optimizer = policy_optimizer_cls( + self._model_dict["policy"].parameters(), **policy_optimizer_params + ) + if value_optimizer_cls is not None: + self._value_optimizer = value_optimizer_cls( + self._model_dict["value"].parameters(), **value_optimizer_params + ) self._value_loss_func = value_loss_func self._hyper_params = hyper_params @@ -88,24 +96,29 @@ def _get_values_and_bootstrapped_returns(self, state_sequence, reward_sequence): return state_values, return_est def train(self, states: np.ndarray, actions: np.ndarray, rewards: np.ndarray): + if not hasattr(self, "_policy_optimizer") and not hasattr(self, "_value_optimizer"): + return + states = torch.from_numpy(states).to(self._device) state_values, return_est = self._get_values_and_bootstrapped_returns(states, rewards) advantages = return_est - state_values actions = torch.from_numpy(actions).to(self._device) # policy model training - for _ in range(self._hyper_params.policy_train_iters): - action_prob = self._model_dict["policy"](states).gather(1, actions.unsqueeze(1)).squeeze() # (N,) - policy_loss = -(torch.log(action_prob) * advantages).mean() - self._policy_optimizer.zero_grad() - policy_loss.backward() - self._policy_optimizer.step() + if hasattr(self, "_policy_optimizer"): + for _ in range(self._hyper_params.policy_train_iters): + action_prob = self._model_dict["policy"](states).gather(1, actions.unsqueeze(1)).squeeze() # (N,) + policy_loss = -(torch.log(action_prob) * advantages).mean() + self._policy_optimizer.zero_grad() + policy_loss.backward() + self._policy_optimizer.step() # value model training - for _ in range(self._hyper_params.value_train_iters): - value_loss = self._value_loss_func(self._model_dict["value"](states).squeeze(), return_est) - self._value_optimizer.zero_grad() - value_loss.backward() - self._value_optimizer.step() + if hasattr(self, "_value_optimizer"): + for _ in range(self._hyper_params.value_train_iters): + value_loss = self._value_loss_func(self._model_dict["value"](states).squeeze(), return_est) + self._value_optimizer.zero_grad() + value_loss.backward() + self._value_optimizer.step() def load_models(self, model_dict): self._model_dict = model_dict @@ -149,7 +162,8 @@ class ActorCriticWithCombinedModel(AbsAlgorithm): policy_value_model (nn.Module): model for generating action distributions and values for given states using shared bottom layers. The model, when called, must return (value, action distribution). value_loss_func (Callable): loss function for the value model. - optimizer_cls: torch optimizer class for the policy model. + optimizer_cls: torch optimizer class for the policy model. If this is None, the policy_model is not + trainable. optimizer_params: parameters required for the policy optimizer class. hyper_params: hyper-parameter set for the AC algorithm. """ @@ -161,7 +175,8 @@ def __init__( super().__init__() self._device = torch.device("cuda" if torch.cuda.is_available() else "cpu") self._policy_value_model = policy_value_model.to(self._device) - self._optimizer = optimizer_cls(self._policy_value_model.parameters(), **optimizer_params) + if optimizer_cls is not None: + self._optimizer = optimizer_cls(self._policy_value_model.parameters(), **optimizer_params) self._value_loss_func = value_loss_func self._hyper_params = hyper_params @@ -187,20 +202,21 @@ def _get_values_and_bootstrapped_returns(self, state_sequence, reward_sequence): return state_values, return_est def train(self, states: np.ndarray, actions: np.ndarray, rewards: np.ndarray): - states = torch.from_numpy(states).to(self._device) - state_values, return_est = self._get_values_and_bootstrapped_returns(states, rewards) - advantages = return_est - state_values - actions = torch.from_numpy(actions).to(self._device) - # policy-value model training - for _ in range(self._hyper_params.train_iters): - state_values, action_distribution = self._policy_value_model(states) - action_prob = action_distribution.gather(1, actions.unsqueeze(1)).squeeze() # (N,) - policy_loss = -(torch.log(action_prob) * advantages).mean() - value_loss = self._value_loss_func(state_values.squeeze(), return_est) - loss = policy_loss + value_loss - self._optimizer.zero_grad() - loss.backward() - self._optimizer.step() + if hasattr(self, "_optimizer"): + states = torch.from_numpy(states).to(self._device) + state_values, return_est = self._get_values_and_bootstrapped_returns(states, rewards) + advantages = return_est - state_values + actions = torch.from_numpy(actions).to(self._device) + # policy-value model training + for _ in range(self._hyper_params.train_iters): + state_values, action_distribution = self._policy_value_model(states) + action_prob = action_distribution.gather(1, actions.unsqueeze(1)).squeeze() # (N,) + policy_loss = -(torch.log(action_prob) * advantages).mean() + value_loss = self._value_loss_func(state_values.squeeze(), return_est) + loss = policy_loss + value_loss + self._optimizer.zero_grad() + loss.backward() + self._optimizer.step() def load_models(self, policy_value_model): self._policy_value_model = policy_value_model diff --git a/maro/rl/algorithms/torch/dqn.py b/maro/rl/algorithms/dqn.py similarity index 62% rename from maro/rl/algorithms/torch/dqn.py rename to maro/rl/algorithms/dqn.py index e4a0ca1bc..24161c8e1 100644 --- a/maro/rl/algorithms/torch/dqn.py +++ b/maro/rl/algorithms/dqn.py @@ -36,7 +36,7 @@ class DQN(AbsAlgorithm): Args: eval_model (nn.Module): trainable Q-value model for computing actions given states. - optimizer_cls: torch optimizer class for the eval model. + optimizer_cls: torch optimizer class for the eval model. If this is None, the eval model is not trainable. optimizer_params: parameters required for the eval optimizer class. loss_func (Callable): loss function for the value model. hyper_params: hyper-parameter set for the DQN algorithm. @@ -49,11 +49,13 @@ def __init__( ): super().__init__() self._device = torch.device("cuda" if torch.cuda.is_available() else "cpu") - self._model_dict = { - "eval": eval_model.to(self._device), - "target": clone(eval_model).to(self._device) if target_model is None else target_model.to(self._device) - } - self._optimizer = optimizer_cls(self._model_dict["eval"].parameters(), **optimizer_params) + self._model_dict = {"eval": eval_model.to(self._device)} + if optimizer_cls is not None: + self._optimizer = optimizer_cls(self._model_dict["eval"].parameters(), **optimizer_params) + if target_model is None: + self._model_dict["target"] = clone(eval_model).to(self._device) + else: + self._model_dict["target"] = target_model.to(self._device) self._loss_func = loss_func self._hyper_params = hyper_params self._train_cnt = 0 @@ -73,31 +75,35 @@ def choose_action(self, state: np.ndarray, epsilon: float = None): return np.random.choice(self._hyper_params.num_actions) def train(self, states: np.ndarray, actions: np.ndarray, rewards: np.ndarray, next_states: np.ndarray): - states = torch.from_numpy(states).to(self._device) # (N, state_dim) - actions = torch.from_numpy(actions).to(self._device) # (N,) - rewards = torch.from_numpy(rewards).to(self._device) # (N,) - next_states = torch.from_numpy(next_states).to(self._device) # (N, state_dim) - if len(actions.shape) == 1: - actions = actions.unsqueeze(1) # (N, 1) - current_q_values = self._model_dict["eval"](states).gather(1, actions).squeeze(1) # (N,) - next_q_values = self._model_dict["target"](next_states).max(dim=1)[0] # (N,) - target_q_values = (rewards + self._hyper_params.reward_decay * next_q_values).detach() # (N,) - loss = self._loss_func(current_q_values, target_q_values) - self._model_dict["eval"].train() - self._optimizer.zero_grad() - loss.backward() - self._optimizer.step() - self._train_cnt += 1 - if self._train_cnt % self._hyper_params.num_training_rounds_per_target_replacement == 0: - self._update_target_model() - - return np.abs((current_q_values - target_q_values).detach().numpy()) + if hasattr(self, "_optimizer"): + states = torch.from_numpy(states).to(self._device) # (N, state_dim) + actions = torch.from_numpy(actions).to(self._device) # (N,) + rewards = torch.from_numpy(rewards).to(self._device) # (N,) + next_states = torch.from_numpy(next_states).to(self._device) # (N, state_dim) + if len(actions.shape) == 1: + actions = actions.unsqueeze(1) # (N, 1) + current_q_values = self._model_dict["eval"](states).gather(1, actions).squeeze(1) # (N,) + next_q_values = self._model_dict["target"](next_states).max(dim=1)[0] # (N,) + target_q_values = (rewards + self._hyper_params.reward_decay * next_q_values).detach() # (N,) + loss = self._loss_func(current_q_values, target_q_values) + self._model_dict["eval"].train() + self._optimizer.zero_grad() + loss.backward() + self._optimizer.step() + self._train_cnt += 1 + if self._train_cnt % self._hyper_params.num_training_rounds_per_target_replacement == 0: + self._update_target_model() + + return np.abs((current_q_values - target_q_values).detach().numpy()) def _update_target_model(self): - for eval_params, target_params in zip(self._model_dict["eval"].parameters(), self._model_dict["target"].parameters()): - target_params.data = ( - self._hyper_params.tau * eval_params.data + (1 - self._hyper_params.tau) * target_params.data - ) + if hasattr(self, "_optimizer"): + for eval_params, target_params in zip( + self._model_dict["eval"].parameters(), self._model_dict["target"].parameters() + ): + target_params.data = ( + self._hyper_params.tau * eval_params.data + (1 - self._hyper_params.tau) * target_params.data + ) def load_models(self, eval_model): """Load the eval model from memory.""" diff --git a/maro/rl/algorithms/torch/pg.py b/maro/rl/algorithms/pg.py similarity index 76% rename from maro/rl/algorithms/torch/pg.py rename to maro/rl/algorithms/pg.py index fa7ee9bd8..19855f003 100644 --- a/maro/rl/algorithms/torch/pg.py +++ b/maro/rl/algorithms/pg.py @@ -29,7 +29,7 @@ class PolicyGradient(AbsAlgorithm): Args: policy_model (nn.Module): model for generating actions given states. - optimizer_cls: torch optimizer class for the policy model. + optimizer_cls: torch optimizer class for the policy model. If this is None, the policy model is not trainable. optimizer_params: parameters required for the policy optimizer class. hyper_params: hyper-parameter set for the AC algorithm. """ @@ -41,7 +41,8 @@ def __init__( super().__init__() self._device = torch.device("cuda" if torch.cuda.is_available() else "cpu") self._policy_model = policy_model.to(self._device) - self._policy_optimizer = optimizer_cls(self._policy_model.parameters(), **optimizer_params) + if optimizer_cls is not None: + self._policy_optimizer = optimizer_cls(self._policy_model.parameters(), **optimizer_params) self._hyper_params = hyper_params @property @@ -56,14 +57,15 @@ def choose_action(self, state: np.ndarray, epsilon: float = None): return np.random.choice(self._hyper_params.num_actions, p=action_dist) def train(self, states: np.ndarray, actions: np.ndarray, returns: np.ndarray): - states = torch.from_numpy(states).to(self._device) # (N, state_dim) - actions = torch.from_numpy(actions).to(self._device) # (N,) - returns = torch.from_numpy(returns).to(self._device) - action_prob = self._policy_model(states).gather(1, actions.unsqueeze(1)).squeeze() # (N, 1) - policy_loss = -(torch.log(action_prob) * returns).mean() - self._policy_optimizer.zero_grad() - policy_loss.backward() - self._policy_optimizer.step() + if hasattr(self, "_optimizer"): + states = torch.from_numpy(states).to(self._device) # (N, state_dim) + actions = torch.from_numpy(actions).to(self._device) # (N,) + returns = torch.from_numpy(returns).to(self._device) + action_prob = self._policy_model(states).gather(1, actions.unsqueeze(1)).squeeze() # (N, 1) + policy_loss = -(torch.log(action_prob) * returns).mean() + self._policy_optimizer.zero_grad() + policy_loss.backward() + self._policy_optimizer.step() def load_models(self, policy_model): self._policy_model = policy_model diff --git a/maro/rl/algorithms/torch/ppo.py b/maro/rl/algorithms/ppo.py similarity index 77% rename from maro/rl/algorithms/torch/ppo.py rename to maro/rl/algorithms/ppo.py index fd4fee259..20eb309dc 100644 --- a/maro/rl/algorithms/torch/ppo.py +++ b/maro/rl/algorithms/ppo.py @@ -49,9 +49,11 @@ class PPO(AbsAlgorithm): policy_model (nn.Module): model for generating actions given states. value_model (nn.Module): model for estimating state values. value_loss_func (Callable): loss function for the value model. - policy_optimizer_cls: torch optimizer class for the policy model. + policy_optimizer_cls: torch optimizer class for the policy model. If this is None, the policy model is not + trainable. policy_optimizer_params: parameters required for the policy optimizer class. - value_optimizer_cls: torch optimizer class for the value model. + value_optimizer_cls: torch optimizer class for the value model. If this is None, the value model is not + trainable. value_optimizer_params: parameters required for the value optimizer class. hyper_params: hyper-parameter set for the AC algorithm. """ @@ -92,6 +94,9 @@ def _get_values_and_bootstrapped_returns(self, states: torch.tensor, rewards: np def train( self, states: np.ndarray, actions: np.ndarray, log_action_prob: np.ndarray, rewards: np.ndarray ): + if not hasattr(self, "_policy_optimizer") and not hasattr(self, "_value_optimizer"): + return + states = torch.from_numpy(states).to(self._device) # (N, state_dim) state_values, return_est = self._get_values_and_bootstrapped_returns(states, rewards) advantages = return_est - state_values @@ -99,21 +104,23 @@ def train( log_action_prob_old = torch.from_numpy(log_action_prob).to(self._device) # policy model training (with the value model fixed) - for _ in range(self._hyper_params.policy_train_iters): - action_prob = self._model_dict["policy"](states).gather(1, actions.unsqueeze(1)).squeeze() # (N, 1) - ratio = torch.exp(torch.log(action_prob) - log_action_prob_old) - clipped_ratio = torch.clamp(ratio, 1 - self._hyper_params.clip_ratio, 1 + self._hyper_params.clip_ratio) - loss = -(torch.min(ratio * advantages, clipped_ratio * advantages)).mean() - self._policy_optimizer.zero_grad() - loss.backward() - self._policy_optimizer.step() + if hasattr(self, "_policy_optimizer"): + for _ in range(self._hyper_params.policy_train_iters): + action_prob = self._model_dict["policy"](states).gather(1, actions.unsqueeze(1)).squeeze() # (N, 1) + ratio = torch.exp(torch.log(action_prob) - log_action_prob_old) + clipped_ratio = torch.clamp(ratio, 1 - self._hyper_params.clip_ratio, 1 + self._hyper_params.clip_ratio) + loss = -(torch.min(ratio * advantages, clipped_ratio * advantages)).mean() + self._policy_optimizer.zero_grad() + loss.backward() + self._policy_optimizer.step() # value model training - for _ in range(self._hyper_params.value_train_iters): - value_loss = self._value_loss_func(self._model_dict["value"](states), return_est) - self._value_optimizer.zero_grad() - value_loss.backward() - self._value_optimizer.step() + if hasattr(self, "_value_optimizer"): + for _ in range(self._hyper_params.value_train_iters): + value_loss = self._value_loss_func(self._model_dict["value"](states), return_est) + self._value_optimizer.zero_grad() + value_loss.backward() + self._value_optimizer.step() def load_models(self, model_dict): self._model_dict = model_dict @@ -162,7 +169,8 @@ class PPOWithCombinedModel(AbsAlgorithm): policy_value_model (nn.Module): model for generating action distributions and values for given states using shared bottom layers. The model, when called, must return (value, action distribution). value_loss_func (Callable): loss function for the value model. - optimizer_cls: torch optimizer class for the policy model. + optimizer_cls: torch optimizer class for the policy model. If this is None, the policy-value model is not + trainable. optimizer_params: parameters required for the policy optimizer class. hyper_params: hyper-parameter set for the AC algorithm. """ @@ -174,7 +182,8 @@ def __init__( super().__init__() self._device = torch.device("cuda" if torch.cuda.is_available() else "cpu") self._policy_value_model = policy_value_model.to(self._device) - self._optimizer = optimizer_cls(self._policy_value_model.parameters(), **optimizer_params) + if optimizer_cls is not None: + self._optimizer = optimizer_cls(self._policy_value_model.parameters(), **optimizer_params) self._value_loss_func = value_loss_func self._hyper_params = hyper_params @@ -203,22 +212,23 @@ def _get_values_and_bootstrapped_returns(self, state_sequence, reward_sequence): def train( self, states: np.ndarray, actions: np.ndarray, log_action_prob: np.ndarray, rewards: np.ndarray ): - states = torch.from_numpy(states).to(self._device) # (N, state_dim) - state_values, return_est = self._get_values_and_bootstrapped_returns(states, rewards) - advantages = return_est - state_values - actions = torch.from_numpy(actions).to(self._device) # (N,) - log_action_prob_old = torch.from_numpy(log_action_prob).to(self._device) - for _ in range(self._hyper_params.train_iters): - state_values, action_distribution = self._policy_value_model(states) - action_prob = action_distribution.gather(1, actions.unsqueeze(1)).squeeze() # (N,) - ratio = torch.exp(torch.log(action_prob) - log_action_prob_old) - clipped_ratio = torch.clamp(ratio, 1 - self._hyper_params.clip_ratio, 1 + self._hyper_params.clip_ratio) - policy_loss = -(torch.min(ratio * advantages, clipped_ratio * advantages)).mean() - value_loss = self._value_loss_func(state_values, return_est) - loss = policy_loss + value_loss - self._optimizer.zero_grad() - loss.backward() - self._optimizer.step() + if hasattr(self, "_optimizer"): + states = torch.from_numpy(states).to(self._device) # (N, state_dim) + state_values, return_est = self._get_values_and_bootstrapped_returns(states, rewards) + advantages = return_est - state_values + actions = torch.from_numpy(actions).to(self._device) # (N,) + log_action_prob_old = torch.from_numpy(log_action_prob).to(self._device) + for _ in range(self._hyper_params.train_iters): + state_values, action_distribution = self._policy_value_model(states) + action_prob = action_distribution.gather(1, actions.unsqueeze(1)).squeeze() # (N,) + ratio = torch.exp(torch.log(action_prob) - log_action_prob_old) + clipped_ratio = torch.clamp(ratio, 1 - self._hyper_params.clip_ratio, 1 + self._hyper_params.clip_ratio) + policy_loss = -(torch.min(ratio * advantages, clipped_ratio * advantages)).mean() + value_loss = self._value_loss_func(state_values, return_est) + loss = policy_loss + value_loss + self._optimizer.zero_grad() + loss.backward() + self._optimizer.step() def load_models(self, policy_value_model): self._policy_value_model = policy_value_model diff --git a/maro/rl/algorithms/torch/__init__.py b/maro/rl/algorithms/torch/__init__.py deleted file mode 100644 index 9a0454564..000000000 --- a/maro/rl/algorithms/torch/__init__.py +++ /dev/null @@ -1,2 +0,0 @@ -# Copyright (c) Microsoft Corporation. -# Licensed under the MIT license. diff --git a/maro/rl/learner/simple_learner.py b/maro/rl/learner/simple_learner.py index c9e7e94d7..d7a2af6dc 100644 --- a/maro/rl/learner/simple_learner.py +++ b/maro/rl/learner/simple_learner.py @@ -2,7 +2,7 @@ # Licensed under the MIT license. from .abs_learner import AbsLearner -from maro.rl.agent.abs_agent_manager import AbsAgentManager +from maro.rl.agent.simple_agent_manager import SimpleAgentManager from maro.rl.actor.simple_actor import SimpleActor from maro.utils import DummyLogger @@ -15,7 +15,7 @@ class SimpleLearner(AbsLearner): actor (Actor or ActorProxy): an Actor or VectorActorProxy instance. logger: used for logging important messages. """ - def __init__(self, trainable_agents: AbsAgentManager, actor, logger=DummyLogger()): + def __init__(self, trainable_agents: SimpleAgentManager, actor, logger=DummyLogger()): super().__init__() self._trainable_agents = trainable_agents self._actor = actor diff --git a/maro/rl/models/decision_layers.py b/maro/rl/models/decision_layers.py new file mode 100644 index 000000000..23c9e4bed --- /dev/null +++ b/maro/rl/models/decision_layers.py @@ -0,0 +1,88 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +from collections import OrderedDict + +import torch.nn as nn + + +class DecisionLayers(nn.Module): + """NN model to compute state or action values. + + Fully connected network with optional batch normalization, activation and dropout components. + + Args: + name (str): Network name. + input_dim (int): Network input dimension. + output_dim (int): Network output dimension. + hidden_dims ([int]): Dimensions of hidden layers. Its length is the number of hidden layers. + activation: A ``torch.nn`` activation type. If None, there will be no activation. Defaults to LeakyReLU. + softmax_enabled (bool): If true, the output of the net will be a softmax transformation of the top layer's + output. Defaults to False. + batch_norm_enabled (bool): If true, batch normalization will be performed at each layer. + dropout_p (float): Dropout probability. Defaults to None, in which case there is no drop-out. + """ + def __init__( + self, *, name: str, input_dim: int, output_dim: int, hidden_dims: [int], activation=nn.LeakyReLU, + softmax_enabled: bool = False, batch_norm_enabled: bool = False, dropout_p: float = None + ): + super().__init__() + self._name = name + self._input_dim = input_dim + self._hidden_dims = hidden_dims if hidden_dims is not None else [] + self._output_dim = output_dim + + # build the net + self._layers = self._build_layers([input_dim] + self._hidden_dims) + if len(self._hidden_dims) == 0: + self._top_layer = nn.Linear(self._input_dim, self._output_dim) + else: + self._top_layer = nn.Linear(hidden_dims[-1], self._output_dim) + self._net = nn.Sequential(*self._layers, self._top_layer) + + # network features + self._activation = activation + self._softmax = nn.Softmax(dim=1) if softmax_enabled else None + self._batch_norm_enabled = batch_norm_enabled + self._dropout_p = dropout_p + + def forward(self, x): + out = self._net(x).double() + return self._softmax(out) if self._softmax else out + + @property + def name(self): + return self._name + + @property + def input_dim(self): + return self._input_dim + + @property + def output_dim(self): + return self._output_dim + + def _build_basic_layer(self, input_dim, output_dim): + """Build basic layer. + + BN -> Linear -> LeakyReLU -> Dropout + """ + components = [] + if self._batch_norm_enabled: + components.append(("batch_norm", nn.BatchNorm1d(input_dim))) + components.append(("linear", nn.Linear(input_dim, output_dim))) + if self._activation is not None: + components.append(("activation", self._activation())) + if self._dropout_p: + components.append(("dropout", nn.Dropout(p=self._dropout_p))) + return nn.Sequential(OrderedDict(components)) + + def _build_layers(self, layer_dims: []): + """Build multi basic layer. + + BasicLayer1 -> BasicLayer2 -> ... + """ + layers = [] + for input_dim, output_dim in zip(layer_dims, layer_dims[1:]): + layers.append(self._build_basic_layer(input_dim, output_dim)) + return layers diff --git a/maro/rl/models/torch/learning_model.py b/maro/rl/models/learning_model.py similarity index 84% rename from maro/rl/models/torch/learning_model.py rename to maro/rl/models/learning_model.py index dcad42ac1..7272b6550 100644 --- a/maro/rl/models/torch/learning_model.py +++ b/maro/rl/models/learning_model.py @@ -20,10 +20,10 @@ class LearningModel(nn.Module): and outputs values of interest in RL (e.g., state & action values). clip_value (float): Threshold used to clip gradients. """ - def __init__(self, - representation_layers: nn.Module = IdentityLayers(), - decision_layers: nn.Module = IdentityLayers(), - clip_value: float = None): + def __init__( + self, representation_layers: nn.Module = IdentityLayers(), decision_layers: nn.Module = IdentityLayers(), + clip_value: float = None + ): super().__init__() self._net = nn.Sequential(representation_layers, decision_layers) diff --git a/maro/rl/models/torch/mlp_representation.py b/maro/rl/models/representation_layers.py similarity index 90% rename from maro/rl/models/torch/mlp_representation.py rename to maro/rl/models/representation_layers.py index 64d3af7ea..6c7408590 100644 --- a/maro/rl/models/torch/mlp_representation.py +++ b/maro/rl/models/representation_layers.py @@ -4,7 +4,7 @@ import torch.nn as nn -class MLPRepresentation(nn.Module): +class RepresentationLayers(nn.Module): def __init__(self, name: str, input_dim: int, hidden_dims: [int], output_dim: int, dropout_p: float): """Deep Q network. @@ -13,8 +13,7 @@ def __init__(self, name: str, input_dim: int, hidden_dims: [int], output_dim: in Args: name (str): Network name. input_dim (int): Network input dimension. - hidden_dims ([int]): Network hiddenlayer dimension. The length of ``hidden_dims`` means the - hidden layer number, which requires larger than 1. + hidden_dims ([int]): Dimensions of hidden layers. Its length is the number of hidden layers. output_dim (int): Network output dimension. dropout_p (float): Dropout parameter. """ diff --git a/maro/rl/models/torch/__init__.py b/maro/rl/models/torch/__init__.py deleted file mode 100644 index 9a0454564..000000000 --- a/maro/rl/models/torch/__init__.py +++ /dev/null @@ -1,2 +0,0 @@ -# Copyright (c) Microsoft Corporation. -# Licensed under the MIT license. diff --git a/maro/rl/models/torch/mlp_decision_layers.py b/maro/rl/models/torch/mlp_decision_layers.py deleted file mode 100644 index 494030a6f..000000000 --- a/maro/rl/models/torch/mlp_decision_layers.py +++ /dev/null @@ -1,76 +0,0 @@ -# Copyright (c) Microsoft Corporation. -# Licensed under the MIT license. - -import torch.nn as nn - - -class MLPDecisionLayers(nn.Module): - """NN model to compute state or action values. - - Fully connected network with batch normalization, leaky RELU and dropout as layer components. - - Args: - name (str): Network name. - input_dim (int): Network input dimension. - hidden_dims ([int]): Network hidden layer dimension. The length of ``hidden_dims`` means the - hidden layer number, which requires larger than 1. - output_dim (int): Network output dimension. - dropout_p (float): Dropout parameter. - softmax (bool): If true, the output of the net will be a softmax transformation of the top layer's output. - Defaults to False. - """ - def __init__( - self, *, name: str, input_dim: int, output_dim: int, hidden_dims: [int], dropout_p: float, - softmax: bool = False - ): - super().__init__() - self._name = name - self._input_dim = input_dim - self._hidden_dims = hidden_dims if hidden_dims is not None else [] - self._output_dim = output_dim - self._dropout_p = dropout_p - self._layers = self._build_layers([input_dim] + self._hidden_dims) - if len(self._hidden_dims) == 0: - self._head = nn.Linear(self._input_dim, self._output_dim) - else: - self._head = nn.Linear(hidden_dims[-1], self._output_dim) - self._net = nn.Sequential(*self._layers, self._head) - self._softmax = nn.Softmax(dim=1) if softmax else None - - def forward(self, x): - out = self._net(x).double() - return self._softmax(out) if self._softmax else out - - @property - def input_dim(self): - return self._input_dim - - @property - def name(self): - return self._name - - @property - def output_dim(self): - return self._output_dim - - def _build_basic_layer(self, input_dim, output_dim): - """Build basic layer. - - BN -> Linear -> LeakyReLU -> Dropout - """ - return nn.Sequential( - nn.BatchNorm1d(input_dim), - nn.Linear(input_dim, output_dim), - nn.LeakyReLU(), - nn.Dropout(p=self._dropout_p) - ) - - def _build_layers(self, layer_dims: []): - """Build multi basic layer. - - BasicLayer1 -> BasicLayer2 -> ... - """ - layers = [] - for input_dim, output_dim in zip(layer_dims, layer_dims[1:]): - layers.append(self._build_basic_layer(input_dim, output_dim)) - return layers diff --git a/maro/rl/models/torch/mlp_policy_net.py b/maro/rl/models/torch/mlp_policy_net.py deleted file mode 100644 index 8a890aed0..000000000 --- a/maro/rl/models/torch/mlp_policy_net.py +++ /dev/null @@ -1,62 +0,0 @@ -# Copyright (c) Microsoft Corporation. -# Licensed under the MIT license. - -import torch.nn as nn - - -class MLPPolicyNet(nn.Module): - """NN model to compute action distributions given states. - - Args: - name (str): Network name. - input_dim (int): Network input dimension. - hidden_dims ([int]): Network hidden layer dimension. The length of ``hidden_dims`` means the - hidden layer number, which requires larger than 1. - output_dim (int): Network output dimension. - init_w (float): If not None, [-init_w, init_w] will be the range from which the initial network parameters - will be uniformly drawn. - """ - def __init__( - self, name: str, input_dim: int, hidden_dims: [int], output_dim: int, init_w: float = 1e-3 - ): - super().__init__() - assert len(hidden_dims) > 1 - self._name = name - self._input_dim = input_dim - self._hidden_dims = hidden_dims - self._output_dim = output_dim - self._num_layers = len(self._hidden_dims) + 1 - - layer_sizes = [input_dim] + self._hidden_dims - layers = [] - for i in range(self._num_layers - 1): - layers += [ - nn.Linear(layer_sizes[i], layer_sizes[i + 1]), - nn.Tanh() - ] - self._hidden_layer = nn.Sequential(*layers) - - self._last_layer = nn.Linear(layer_sizes[-1], output_dim) - if init_w is not None: - self._last_layer.weight.data.uniform_(-init_w, init_w) - self._last_layer.bias.data.uniform_(-init_w, init_w) - - # TODO: dim=1 for batch forward; dim=0 if only one - self._soft_max = nn.Softmax(dim=1) - - def forward(self, x): - x = self._hidden_layer(x) - x = self._last_layer(x) - return self._soft_max(x) - - @property - def name(self): - return self._name - - @property - def input_dim(self): - return self._input_dim - - @property - def output_dim(self): - return self._output_dim diff --git a/maro/rl/storage/column_based_store.py b/maro/rl/storage/column_based_store.py index a62a120e4..21f78c3b6 100644 --- a/maro/rl/storage/column_based_store.py +++ b/maro/rl/storage/column_based_store.py @@ -57,8 +57,8 @@ def __getitem__(self, index: int): def __getstate__(self): """A small modification to make the object picklable. - Using the default ``__dict__`` would make the object unpicklable due to the lambda function involved - in the ``defaultdict`` definition of the ``_store`` attribute. + Using the default ``__dict__`` would make the object unpicklable due to the lambda function involved in the + ``defaultdict`` definition of the ``_store`` attribute. """ obj_dict = self.__dict__ obj_dict["_store"] = dict(obj_dict["_store"]) diff --git a/maro/utils/exception/error_code.py b/maro/utils/exception/error_code.py index ae7e63ff5..da155e1ae 100644 --- a/maro/utils/exception/error_code.py +++ b/maro/utils/exception/error_code.py @@ -35,8 +35,9 @@ 3003: "Deployment Error", # 4000-4999: Error codes for RL toolkit - 4001: "Unsupported Agent Mode Error", - 4002: "Missing Shaper Error", - 4003: "Wrong Agent Mode Error", - 4004: "Store Misalignment Error" + 4001: "Unsupported Agent Mode", + 4002: "Missing Shaper", + 4003: "Wrong Agent Manager Mode", + 4004: "Wrong Agent Mode", + 4005: "Store Misalignment Error" } diff --git a/maro/utils/exception/rl_toolkit_exception.py b/maro/utils/exception/rl_toolkit_exception.py index e39ddec72..33f29ab4f 100644 --- a/maro/utils/exception/rl_toolkit_exception.py +++ b/maro/utils/exception/rl_toolkit_exception.py @@ -5,25 +5,31 @@ class UnsupportedAgentModeError(MAROException): - """Unsupported agent mode error.""" + """Unsupported agent mode.""" def __init__(self, msg: str = None): super().__init__(4001, msg) class MissingShaperError(MAROException): - """Missing shaper error.""" + """Missing shaper.""" def __init__(self, msg: str = None): super().__init__(4002, msg) -class WrongAgentModeError(MAROException): - """Wrong agent mode error.""" +class WrongAgentManagerModeError(MAROException): + """Wrong agent manager mode.""" def __init__(self, msg: str = None): super().__init__(4003, msg) +class WrongAgentModeError(MAROException): + """Wrong agent mode.""" + def __init__(self, msg: str = None): + super().__init__(4004, msg) + + class StoreMisalignmentError(MAROException): """Raised when a ``put`` operation on a ``ColumnBasedStore`` would cause the underlying lists to have different - sizes.""" + sizes.""" def __init__(self, msg: str = None): - super().__init__(4004, msg) + super().__init__(4005, msg) From 0773f80eda2183955a917fc244ce12b3122bcc87 Mon Sep 17 00:00:00 2001 From: ysqyang Date: Wed, 21 Oct 2020 21:05:48 +0800 Subject: [PATCH 035/337] refined rl abstractions --- examples/cim/dqn/components/agent.py | 4 +- examples/cim/dqn/components/agent_manager.py | 79 +++++--- examples/cim/dqn/components/config.py | 13 +- examples/cim/dqn/dist_actor.py | 50 ++--- examples/cim/dqn/dist_learner.py | 43 +++-- examples/cim/dqn/single_process_launcher.py | 45 +++-- maro/rl/__init__.py | 38 +++- maro/rl/actor/simple_actor.py | 9 +- maro/rl/agent/abs_agent.py | 77 +++++--- maro/rl/agent/abs_agent_manager.py | 174 ++++-------------- maro/rl/agent/simple_agent_manager.py | 105 +++++++++++ maro/rl/algorithms/abs_algorithm.py | 60 ++++++ maro/rl/algorithms/dqn.py | 122 ++++++++++++ maro/rl/algorithms/torch/__init__.py | 2 - maro/rl/algorithms/torch/abs_algorithm.py | 81 -------- maro/rl/algorithms/torch/dqn.py | 88 --------- maro/rl/learner/abs_learner.py | 12 +- maro/rl/learner/simple_learner.py | 27 +-- maro/rl/models/decision_layers.py | 88 +++++++++ maro/rl/models/{torch => }/learning_model.py | 8 +- ...esentation.py => representation_layers.py} | 5 +- maro/rl/models/torch/__init__.py | 2 - maro/rl/models/torch/decision_layers.py | 67 ------- maro/rl/shaping/experience_shaper.py | 6 +- maro/rl/storage/column_based_store.py | 30 ++- maro/rl/storage/utils.py | 10 +- 26 files changed, 695 insertions(+), 550 deletions(-) create mode 100644 maro/rl/agent/simple_agent_manager.py create mode 100644 maro/rl/algorithms/abs_algorithm.py create mode 100644 maro/rl/algorithms/dqn.py delete mode 100644 maro/rl/algorithms/torch/__init__.py delete mode 100644 maro/rl/algorithms/torch/abs_algorithm.py delete mode 100644 maro/rl/algorithms/torch/dqn.py create mode 100644 maro/rl/models/decision_layers.py rename maro/rl/models/{torch => }/learning_model.py (84%) rename maro/rl/models/{torch/mlp_representation.py => representation_layers.py} (90%) delete mode 100644 maro/rl/models/torch/__init__.py delete mode 100644 maro/rl/models/torch/decision_layers.py diff --git a/examples/cim/dqn/components/agent.py b/examples/cim/dqn/components/agent.py index 7f6f24255..e5ed02e1c 100644 --- a/examples/cim/dqn/components/agent.py +++ b/examples/cim/dqn/components/agent.py @@ -7,9 +7,9 @@ class CIMAgent(AbsAgent): - def __init__(self, name, algorithm, experience_pool: ColumnBasedStore, min_experiences_to_train, + def __init__(self, name, mode, algorithm, experience_pool: ColumnBasedStore, min_experiences_to_train, num_batches, batch_size): - super().__init__(name, algorithm, experience_pool) + super().__init__(name, mode, algorithm, experience_pool) self._min_experiences_to_train = min_experiences_to_train self._num_batches = num_batches self._batch_size = batch_size diff --git a/examples/cim/dqn/components/agent_manager.py b/examples/cim/dqn/components/agent_manager.py index 59eaf0956..4c09bb40b 100644 --- a/examples/cim/dqn/components/agent_manager.py +++ b/examples/cim/dqn/components/agent_manager.py @@ -5,33 +5,60 @@ from torch.optim import RMSprop from .agent import CIMAgent -from .config import config -from maro.rl import AbsAgentManager, LearningModel, MLPDecisionLayers, DQN, DQNHyperParams, ColumnBasedStore +from maro.rl import AgentMode, SimpleAgentManager, LearningModel, DecisionLayers, DQN, DQNHyperParams, ColumnBasedStore from maro.utils import set_seeds -class DQNAgentManager(AbsAgentManager): - def _assemble(self, agent_dict): - set_seeds(config.agents.seed) - num_actions = config.agents.algorithm.num_actions - for agent_id in self._agent_id_list: - eval_model = LearningModel(decision_layers=MLPDecisionLayers(name=f'{agent_id}.policy', - input_dim=self._state_shaper.dim, - output_dim=num_actions, - **config.agents.algorithm.model) - ) - - algorithm = DQN(model_dict={"eval": eval_model}, - optimizer_opt=(RMSprop, config.agents.algorithm.optimizer), - loss_func_dict={"eval": smooth_l1_loss}, - hyper_params=DQNHyperParams(**config.agents.algorithm.hyper_parameters, - num_actions=num_actions)) - - experience_pool = ColumnBasedStore(**config.agents.experience_pool) - agent_dict[agent_id] = CIMAgent(name=agent_id, algorithm=algorithm, experience_pool=experience_pool, - **config.agents.training_loop_parameters) - - def store_experiences(self, experiences): - for agent_id, exp in experiences.items(): +def create_dqn_agents(agent_id_list, mode, config): + if mode in {AgentMode.TRAIN, AgentMode.TRAIN_INFERENCE}: + return {agent_id: DQN( + eval_model=None, + optimizer_cls=None, + optimizer_params=None, + loss_func=None, + hyper_params=None + ) + for agent_id in agent_id_list} + + set_seeds(config.seed) + num_actions = config.algorithm.num_actions + agent_dict = {} + for agent_id in agent_id_list: + eval_model = LearningModel( + decision_layers=DecisionLayers( + name=f'{agent_id}.policy', input_dim=config.algorithm.input_dim, + output_dim=num_actions, **config.algorithm.model + ) + ) + + algorithm = DQN( + eval_model=eval_model, + optimizer_cls=RMSprop, + optimizer_params=config.algorithm.optimizer, + loss_func=smooth_l1_loss, + hyper_params=DQNHyperParams( + **config.algorithm.hyper_parameters, + num_actions=num_actions + ) + ) + + experience_pool = ColumnBasedStore(**config.experience_pool) + agent_dict[agent_id] = CIMAgent(name=agent_id, mode=mode, algorithm=algorithm, experience_pool=experience_pool, + **config.training_loop_parameters) + + +class DQNAgentManager(SimpleAgentManager): + def train(self, experiences_by_agent, performance=None): + self._assert_train_mode() + + # store experiences for each agent + for agent_id, exp in experiences_by_agent.items(): exp.update({"loss": [1e8] * len(exp[next(iter(exp))])}) - self._agent_dict[agent_id].store_experiences(exp) + self.agent_dict[agent_id].store_experiences(exp) + + for agent in self.agent_dict.values(): + agent.train() + + # update exploration rates + if self._explorer is not None: + self._explorer.update(performance) diff --git a/examples/cim/dqn/components/config.py b/examples/cim/dqn/components/config.py index d36d60ddd..26a3a8c43 100644 --- a/examples/cim/dqn/components/config.py +++ b/examples/cim/dqn/components/config.py @@ -14,5 +14,14 @@ CONFIG_PATH = os.path.join(os.path.split(os.path.realpath(__file__))[0], "../config.yml") with io.open(CONFIG_PATH, "r") as in_file: - raw_config = yaml.safe_load(in_file) - config = convert_dottable(raw_config) + config = yaml.safe_load(in_file) + +# obtain model input dimension from state shaping configurations +look_back = config["state_shaping"]["look_back"] +max_ports_downstream = config["state_shaping"]["max_port_downstream"] +num_port_attributes = len(config["state_shaping"]["port_attributes"]) +num_vessel_attributes = len(config["state_shaping"]["vessel_attributes"]) + +input_dim = (look_back + 1) * (max_ports_downstream + 1) * num_port_attributes + num_vessel_attributes +config["agents"]["algorithm"]["input_dim"] = input_dim +config = convert_dottable(config) diff --git a/examples/cim/dqn/dist_actor.py b/examples/cim/dqn/dist_actor.py index 350f493f2..548efa622 100644 --- a/examples/cim/dqn/dist_actor.py +++ b/examples/cim/dqn/dist_actor.py @@ -4,10 +4,10 @@ import numpy as np from maro.simulator import Env -from maro.rl import AgentMode, SimpleActor, ActorWorker, KStepExperienceShaper, TwoPhaseLinearExplorer +from maro.rl import AgentMode, AgentManagerMode, SimpleActor, ActorWorker, KStepExperienceShaper, TwoPhaseLinearExplorer from components.action_shaper import CIMActionShaper -from components.agent_manager import DQNAgentManager +from components.agent_manager import create_dqn_agents, DQNAgentManager from components.config import config from components.experience_shaper import TruncatedExperienceShaper from components.state_shaper import CIMStateShaper @@ -21,25 +21,33 @@ if config.experience_shaping.type == "truncated": experience_shaper = TruncatedExperienceShaper(**config.experience_shaping.truncated) else: - experience_shaper = KStepExperienceShaper(reward_func=lambda mt: 1-mt["container_shortage"]/mt["order_requirements"], - **config.experience_shaping.k_step) + experience_shaper = KStepExperienceShaper( + reward_func=lambda mt: 1-mt["container_shortage"]/mt["order_requirements"], + **config.experience_shaping.k_step + ) - exploration_config = {"epsilon_range_dict": {"_all_": config.exploration.epsilon_range}, - "split_point_dict": {"_all_": config.exploration.split_point}, - "with_cache": config.exploration.with_cache - } + exploration_config = { + "epsilon_range_dict": {"_all_": config.exploration.epsilon_range}, + "split_point_dict": {"_all_": config.exploration.split_point}, + "with_cache": config.exploration.with_cache + } explorer = TwoPhaseLinearExplorer(agent_id_list, config.general.total_training_episodes, **exploration_config) - agent_manager = DQNAgentManager(name="cim_remote_actor", - agent_id_list=agent_id_list, - mode=AgentMode.INFERENCE, - state_shaper=state_shaper, - action_shaper=action_shaper, - experience_shaper=experience_shaper, - explorer=explorer) - proxy_params = {"group_name": config.distributed.group_name, - "expected_peers": config.distributed.actor.peer, - "redis_address": (config.distributed.redis.host_name, config.distributed.redis.port) - } - actor_worker = ActorWorker(local_actor=SimpleActor(env=env, inference_agents=agent_manager), - proxy_params=proxy_params) + agent_manager = DQNAgentManager( + name="cim_remote_actor", + mode=AgentManagerMode.INFERENCE, + agent_dict=create_dqn_agents(agent_id_list, AgentMode.INFERENCE, config.agents), + state_shaper=state_shaper, + action_shaper=action_shaper, + experience_shaper=experience_shaper, + explorer=explorer + ) + proxy_params = { + "group_name": config.distributed.group_name, + "expected_peers": config.distributed.actor.peer, + "redis_address": (config.distributed.redis.host_name, config.distributed.redis.port) + } + actor_worker = ActorWorker( + local_actor=SimpleActor(env=env, inference_agents=agent_manager), + proxy_params=proxy_params + ) actor_worker.launch() diff --git a/examples/cim/dqn/dist_learner.py b/examples/cim/dqn/dist_learner.py index 08f1d777e..b5ce56aac 100644 --- a/examples/cim/dqn/dist_learner.py +++ b/examples/cim/dqn/dist_learner.py @@ -3,34 +3,41 @@ import os -from maro.rl import ActorProxy, SimpleLearner, AgentMode, TwoPhaseLinearExplorer +from maro.rl import ActorProxy, SimpleLearner, AgentMode, AgentManagerMode, TwoPhaseLinearExplorer from maro.simulator import Env from maro.utils import Logger -from components.agent_manager import DQNAgentManager +from components.agent_manager import create_dqn_agents, DQNAgentManager from components.config import config -from components.state_shaper import CIMStateShaper if __name__ == "__main__": env = Env(config.env.scenario, config.env.topology, durations=config.env.durations) agent_id_list = [str(agent_id) for agent_id in env.agent_idx_list] - state_shaper = CIMStateShaper(**config.state_shaping) - exploration_config = {"epsilon_range_dict": {"_all_": config.exploration.epsilon_range}, - "split_point_dict": {"_all_": config.exploration.split_point}, - "with_cache": config.exploration.with_cache - } + exploration_config = { + "epsilon_range_dict": {"_all_": config.exploration.epsilon_range}, + "split_point_dict": {"_all_": config.exploration.split_point}, + "with_cache": config.exploration.with_cache + } explorer = TwoPhaseLinearExplorer(agent_id_list, config.general.total_training_episodes, **exploration_config) - agent_manager = DQNAgentManager(name="cim_remote_learner", agent_id_list=agent_id_list, mode=AgentMode.TRAIN, - state_shaper=state_shaper, explorer=explorer) + agent_manager = DQNAgentManager( + name="cim_remote_learner", + mode=AgentManagerMode.TRAIN, + agent_dict=create_dqn_agents(agent_id_list, AgentMode.TRAIN, config.agents), + explorer=explorer + ) - proxy_params = {"group_name": config.distributed.group_name, - "expected_peers": config.distributed.learner.peer, - "redis_address": (config.distributed.redis.host_name, config.distributed.redis.port) - } - learner = SimpleLearner(trainable_agents=agent_manager, - actor=ActorProxy(proxy_params=proxy_params), - logger=Logger("distributed_cim_learner", auto_timestamp=False)) + proxy_params = { + "group_name": config.distributed.group_name, + "expected_peers": config.distributed.learner.peer, + "redis_address": (config.distributed.redis.host_name, config.distributed.redis.port) + } + + learner = SimpleLearner( + trainable_agents=agent_manager, + actor=ActorProxy(proxy_params=proxy_params), + logger=Logger("distributed_cim_learner", auto_timestamp=False) + ) learner.train(total_episodes=config.general.total_training_episodes) learner.test() - learner.dump_models(os.path.join(os.getcwd(), "models")) + learner.save_models(os.path.join(os.getcwd(), "models")) diff --git a/examples/cim/dqn/single_process_launcher.py b/examples/cim/dqn/single_process_launcher.py index a906513ea..a56a11e89 100644 --- a/examples/cim/dqn/single_process_launcher.py +++ b/examples/cim/dqn/single_process_launcher.py @@ -7,11 +7,12 @@ import numpy as np from maro.simulator import Env -from maro.rl import SimpleLearner, SimpleActor, AgentMode, KStepExperienceShaper, TwoPhaseLinearExplorer +from maro.rl import SimpleLearner, SimpleActor, AgentMode, AgentManagerMode, KStepExperienceShaper, \ + TwoPhaseLinearExplorer from maro.utils import Logger from components.action_shaper import CIMActionShaper -from components.agent_manager import DQNAgentManager +from components.agent_manager import create_dqn_agents, DQNAgentManager from components.config import config from components.experience_shaper import TruncatedExperienceShaper from components.state_shaper import CIMStateShaper @@ -29,29 +30,35 @@ if config.experience_shaping.type == "truncated": experience_shaper = TruncatedExperienceShaper(**config.experience_shaping.truncated) else: - experience_shaper = KStepExperienceShaper(reward_func=lambda mt: 1-mt["container_shortage"]/mt["order_requirements"], - **config.experience_shaping.k_step) + experience_shaper = KStepExperienceShaper( + reward_func=lambda mt: 1-mt["container_shortage"]/mt["order_requirements"], + **config.experience_shaping.k_step + ) - exploration_config = {"epsilon_range_dict": {"_all_": config.exploration.epsilon_range}, - "split_point_dict": {"_all_": config.exploration.split_point}, - "with_cache": config.exploration.with_cache - } + exploration_config = { + "epsilon_range_dict": {"_all_": config.exploration.epsilon_range}, + "split_point_dict": {"_all_": config.exploration.split_point}, + "with_cache": config.exploration.with_cache + } explorer = TwoPhaseLinearExplorer(agent_id_list, config.general.total_training_episodes, **exploration_config) # Step 3: create an agent manager. - agent_manager = DQNAgentManager(name="cim_learner", - mode=AgentMode.TRAIN_INFERENCE, - agent_id_list=agent_id_list, - state_shaper=state_shaper, - action_shaper=action_shaper, - experience_shaper=experience_shaper, - explorer=explorer) + agent_manager = DQNAgentManager( + name="cim_learner", + mode=AgentManagerMode.TRAIN_INFERENCE, + agent_dict=create_dqn_agents(agent_id_list, AgentMode.TRAIN_INFERENCE, config.agents), + state_shaper=state_shaper, + action_shaper=action_shaper, + experience_shaper=experience_shaper, + explorer=explorer + ) # Step 4: Create an actor and a learner to start the training process. actor = SimpleActor(env=env, inference_agents=agent_manager) - learner = SimpleLearner(trainable_agents=agent_manager, actor=actor, - logger=Logger("single_host_cim_learner", auto_timestamp=False)) - + learner = SimpleLearner( + trainable_agents=agent_manager, actor=actor, + logger=Logger("single_host_cim_learner", auto_timestamp=False) + ) learner.train(total_episodes=config.general.total_training_episodes) learner.test() - learner.dump_models(os.path.join(os.getcwd(), "models")) + learner.save_models(os.path.join(os.getcwd(), "models")) diff --git a/maro/rl/__init__.py b/maro/rl/__init__.py index bd5c9f221..38be480e8 100644 --- a/maro/rl/__init__.py +++ b/maro/rl/__init__.py @@ -5,13 +5,19 @@ from maro.rl.actor.simple_actor import SimpleActor from maro.rl.learner.abs_learner import AbsLearner from maro.rl.learner.simple_learner import SimpleLearner -from maro.rl.agent.abs_agent import AbsAgent -from maro.rl.agent.abs_agent_manager import AbsAgentManager, AgentMode -from maro.rl.algorithms.torch.abs_algorithm import AbsAlgorithm -from maro.rl.algorithms.torch.dqn import DQN, DQNHyperParams -from maro.rl.models.torch.mlp_representation import MLPRepresentation -from maro.rl.models.torch.decision_layers import MLPDecisionLayers -from maro.rl.models.torch.learning_model import LearningModel +from maro.rl.agent.abs_agent import AbsAgent, AgentMode +from maro.rl.agent.abs_agent_manager import AbsAgentManager, AgentManagerMode +from maro.rl.agent.simple_agent_manager import SimpleAgentManager +from maro.rl.algorithms.abs_algorithm import AbsAlgorithm +from maro.rl.algorithms.pg import PolicyGradient, PolicyGradientHyperParameters +from maro.rl.algorithms.ac import ActorCritic, ActorCriticWithCombinedModel, ActorCriticHyperParameters, \ + ActorCriticHyperParametersWithCombinedModel +from maro.rl.algorithms.ppo import PPO, PPOWithCombinedModel, PPOHyperParameters, \ + PPOHyperParametersWithCombinedModel +from maro.rl.algorithms.dqn import DQN, DQNHyperParams +from maro.rl.models.learning_model import LearningModel +from maro.rl.models.representation_layers import RepresentationLayers +from maro.rl.models.decision_layers import DecisionLayers from maro.rl.storage.abs_store import AbsStore from maro.rl.storage.column_based_store import ColumnBasedStore from maro.rl.storage.utils import OverwriteType @@ -31,14 +37,26 @@ "AbsLearner", "SimpleLearner", "AbsAgent", - "AbsAgentManager", "AgentMode", + "AbsAgentManager", + "AgentManagerMode", + "SimpleAgentManager", "AbsAlgorithm", + "PolicyGradient", + "PolicyGradientHyperParameters", + "ActorCritic", + "ActorCriticWithCombinedModel", + "ActorCriticHyperParameters", + "ActorCriticHyperParametersWithCombinedModel", + "PPO", + "PPOWithCombinedModel", + "PPOHyperParameters", + "PPOHyperParametersWithCombinedModel", "DQN", "DQNHyperParams", - "MLPRepresentation", - "MLPDecisionLayers", "LearningModel", + "RepresentationLayers", + "DecisionLayers", "AbsStore", "ColumnBasedStore", "OverwriteType", diff --git a/maro/rl/actor/simple_actor.py b/maro/rl/actor/simple_actor.py index 19ef8f6c5..f196c3c02 100644 --- a/maro/rl/actor/simple_actor.py +++ b/maro/rl/actor/simple_actor.py @@ -2,7 +2,7 @@ # Licensed under the MIT license. from .abs_actor import AbsActor -from maro.rl.agent.abs_agent_manager import AbsAgentManager +from maro.rl.agent.simple_agent_manager import SimpleAgentManager from maro.simulator import Env @@ -13,11 +13,12 @@ class SimpleActor(AbsActor): env (Env): An Env instance. inference_agents (AbsAgentManager): An AgentManager instance that manages all agents. """ - def __init__(self, env: Env, inference_agents: AbsAgentManager): + def __init__(self, env: Env, inference_agents: SimpleAgentManager): super().__init__(env, inference_agents) - def roll_out(self, model_dict: dict = None, epsilon_dict: dict = None, done: bool = False, - return_details: bool = True): + def roll_out( + self, model_dict: dict = None, epsilon_dict: dict = None, done: bool = False, return_details: bool = True + ): """Perform one episode of roll-out and return performance and experiences. Args: diff --git a/maro/rl/agent/abs_agent.py b/maro/rl/agent/abs_agent.py index 0aa9afd76..f6fb7ffc6 100644 --- a/maro/rl/agent/abs_agent.py +++ b/maro/rl/agent/abs_agent.py @@ -2,13 +2,19 @@ # Licensed under the MIT license. from abc import ABC, abstractmethod +from enum import Enum import os import pickle -import torch - -from maro.rl.algorithms.torch.abs_algorithm import AbsAlgorithm +from maro.rl.algorithms.abs_algorithm import AbsAlgorithm from maro.rl.storage.abs_store import AbsStore +from maro.utils.exception.rl_toolkit_exception import WrongAgentModeError + + +class AgentMode(Enum): + TRAIN = "train" + INFERENCE = "inference" + TRAIN_INFERENCE = "train_inference" class AbsAgent(ABC): @@ -22,17 +28,16 @@ class AbsAgent(ABC): Args: name (str): Agent's name. + mode (AgentMode): An ``AgentMode`` enum member that indicates the role of the agent in the current process. algorithm (AbsAlgorithm): A concrete algorithm instance that inherits from AbstractAlgorithm. This is the centerpiece of the Agent class and is responsible for the most important tasks of an agent: choosing actions and optimizing models. - experience_pool (AbsStore): A data store that stores experiences generated by the experience shaper. + experience_pool (AbsStore): A data store that stores experiences generated by the experience shaper. This is + only necessary for some algorithms. Defaults to None. """ - def __init__(self, - name: str, - algorithm: AbsAlgorithm, - experience_pool: AbsStore - ): + def __init__(self, name: str, mode: AgentMode, algorithm: AbsAlgorithm, experience_pool: AbsStore = None): self._name = name + self._mode = mode self._algorithm = algorithm self._experience_pool = experience_pool @@ -55,10 +60,11 @@ def choose_action(self, model_state, epsilon: float = .0): Returns: Action given by the underlying policy model. """ + self._assert_inference_mode() return self._algorithm.choose_action(model_state, epsilon) @abstractmethod - def train(self): + def train(self, *args, **kwargs): """Training logic to be implemented by the user. For example, this may include drawing samples from the experience pool and the algorithm training on @@ -68,24 +74,47 @@ def train(self): def store_experiences(self, experiences): """Store new experiences in the experience pool.""" - self._experience_pool.put(experiences) + if self._experience_pool is not None: + self._experience_pool.put(experiences) - def load_model_dict(self, model_dict: dict): + def load_models(self, *models, **model_dict): """Load models from memory.""" - self._algorithm.model_dict = model_dict + self._algorithm.load_models(*models, **model_dict) + + def dump_models(self): + """Return the algorithm's trainable models.""" + return self._algorithm.dump_models() + + def load_models_from_file(self, dir_path: str): + """Load trainable models from disk. + + Load trainable models from the specified directory. The model file is always prefixed with the agent's name. + + Args: + dir_path (str): path to the directory where the models are saved. + """ + self._algorithm.load_models_from_file(os.path.join(dir_path, self._name)) - def load_model_dict_from_file(self, file_path): - """Load models from a disk file.""" - model_dict = torch.load(file_path) - for model_key, state_dict in model_dict.items(): - self._algorithm.model_dict[model_key].load_state_dict(state_dict) + def dump_models_to_file(self, dir_path: str): + """Dump the algorithm's trainable models to disk. - def dump_model_dict(self, dir_path: str): - """Dump models to disk.""" - torch.save({model_key: model.state_dict() for model_key, model in self._algorithm.model_dict.items()}, - os.path.join(dir_path, self._name)) + Dump trainable models to the specified directory. The model file is always prefixed with the agent's name. + + Args: + dir_path (str): path to the directory where the models are saved. + """ + self._algorithm.dump_models_to_file(os.path.join(dir_path, self._name)) def dump_experience_store(self, dir_path: str): """Dump the experience pool to disk.""" - with open(os.path.join(dir_path, self._name)) as fp: - pickle.dump(self._experience_pool, fp) + if self._experience_pool is not None: + with open(os.path.join(dir_path, self._name)) as fp: + pickle.dump(self._experience_pool, fp) + + def _assert_train_mode(self): + if self._mode != AgentMode.TRAIN and self._mode != AgentMode.TRAIN_INFERENCE: + raise WrongAgentModeError(msg=f"this method is unavailable under mode {self._mode}") + + def _assert_inference_mode(self): + if self._mode != AgentMode.INFERENCE and self._mode != AgentMode.TRAIN_INFERENCE: + raise WrongAgentModeError(msg=f"this method is unavailable under mode {self._mode}") diff --git a/maro/rl/agent/abs_agent_manager.py b/maro/rl/agent/abs_agent_manager.py index bed57db2a..ffaa13d3d 100644 --- a/maro/rl/agent/abs_agent_manager.py +++ b/maro/rl/agent/abs_agent_manager.py @@ -3,16 +3,15 @@ from abc import ABC, abstractmethod from enum import Enum -import os from maro.rl.shaping.state_shaper import StateShaper from maro.rl.shaping.action_shaper import ActionShaper from maro.rl.shaping.experience_shaper import ExperienceShaper from maro.rl.explorer.abs_explorer import AbsExplorer -from maro.utils.exception.rl_toolkit_exception import UnsupportedAgentModeError, MissingShaperError, WrongAgentModeError +from maro.utils.exception.rl_toolkit_exception import WrongAgentManagerModeError -class AgentMode(Enum): +class AgentManagerMode(Enum): TRAIN = "train" INFERENCE = "inference" TRAIN_INFERENCE = "train_inference" @@ -30,9 +29,9 @@ class AbsAgentManager(ABC): Args: name (str): Name of agent manager. - mode (AgentMode): An AgentMode enum member that specifies that role of the agent. Some attributes may - be None under certain modes. - agent_id_list (list): List of agent identifiers. + mode (AgentManagerMode): An ``AgentManagerNode`` enum member that indicates the role of the agent manager + in the current process. + agent_dict (dict): List of agent identifiers. experience_shaper (ExperienceShaper, optional): It is responsible for processing data in the replay buffer at the end of an episode. state_shaper (StateShaper, optional): It is responsible for converting the environment observation to model @@ -41,169 +40,60 @@ class AbsAgentManager(ABC): executable action. Cannot be None under Inference and TrainInference modes. explorer (AbsExplorer): It is responsible for storing and updating exploration rates. """ - def __init__(self, - name: str, - mode: AgentMode, - agent_id_list: [str], - state_shaper: StateShaper = None, - action_shaper: ActionShaper = None, - experience_shaper: ExperienceShaper = None, - explorer: AbsExplorer = None): + def __init__( + self, name: str, mode: AgentManagerMode, agent_dict: dict, + state_shaper: StateShaper = None, action_shaper: ActionShaper = None, + experience_shaper: ExperienceShaper = None, explorer: AbsExplorer = None + ): self._name = name - if mode not in AgentMode: - raise UnsupportedAgentModeError(msg='mode must be "train", "inference" or "train_inference"') self._mode = mode - - if mode in {AgentMode.INFERENCE, AgentMode.TRAIN_INFERENCE}: - if state_shaper is None: - raise MissingShaperError(msg=f"state shaper cannot be None under mode {self._mode}") - if action_shaper is None: - raise MissingShaperError(msg=f"action_shaper cannot be None under mode {self._mode}") - if experience_shaper is None: - raise MissingShaperError(msg=f"experience_shaper cannot be None under mode {self._mode}") - + self.agent_dict = agent_dict self._state_shaper = state_shaper self._action_shaper = action_shaper self._experience_shaper = experience_shaper self._explorer = explorer - self._agent_id_list = agent_id_list - self._trajectory = [] - - self._agent_dict = {} - self._assemble(self._agent_dict) - def __getitem__(self, agent_id): - return self._agent_dict[agent_id] - - def _assemble(self, agent_dict): - """Assembles agents and fill the ``agent_dict`` with them.""" - return NotImplemented - - def choose_action(self, decision_event, snapshot_list): - """This is the interface for interacting with the environment. - - The method consists of 4 steps: - - 1. The decision event and snapshot list are converted by the state shaper to a model input. - The state shaper also finds the target agent ID. - 2. The target agent takes the model input and uses its underlying models to compute an action. - 3. Key information regarding the transition is recorded in the ``_trajectory`` attribute. - 4. The action computed by the model is converted to an environment executable action by the action shaper. - - Args: - decision_event: A decision event that prompts an action. - snapshot_list: An object that holds the detailed history of past env observations. - - Returns: - An action object that can be passed directly to an environment's ``step`` method. - """ - self._assert_inference_mode() - agent_id, model_state = self._state_shaper(decision_event, snapshot_list) - model_action = self._agent_dict[agent_id].choose_action( - model_state, self._explorer.epsilon[agent_id] if self._explorer else None) - self._trajectory.append({"state": model_state, - "action": model_action, - "reward": None, - "agent_id": agent_id, - "event": decision_event}) - return self._action_shaper(model_action, decision_event, snapshot_list) - - def on_env_feedback(self, metrics): - """This method records the environment-generated metrics as part of the latest transition in the trajectory. - - Args: - metrics: business metrics provided by the environment after an action has been executed. - """ - self._trajectory[-1]["metrics"] = metrics - - def post_process(self, snapshot_list): - """This method processes the latest trajectory into experiences. - - Args: - snapshot_list: the snapshot list from the env at the end of an episode. - """ - experiences = self._experience_shaper(self._trajectory, snapshot_list) - self._trajectory.clear() - self._state_shaper.reset() - self._action_shaper.reset() - self._experience_shaper.reset() - return experiences + return self.agent_dict[agent_id] @abstractmethod - def store_experiences(self, experiences): - """Abstract method to store experiences generated by the experience shaper in the experience pool(s). - - Depending on the user's implementation of the experience shaper's ``__call__`` method, ``experiences`` may - come in different formats (e.g., a dictionary with agent ID's as keys). The user must implement this - method in accordance with this format. - - Args: - experiences: experiences generated by the experience shaper during post-processing. + def choose_action(self, *args, **kwargs): + """Generate an environment executable action given the current decision event and snapshot list. """ - return NotImplementedError - - def update_epsilon(self, performance): - """This method updates the exploration rates for each agent. - - Args: - performance: Performance from the latest episode. Depending on the implementation of the explorer, - this may or may not be used in generating the exploration rates. - """ - if self._explorer: - self._explorer.update(performance) - - def train(self): - """Train all agents.""" - self._assert_train_mode() - for agent in self._agent_dict.values(): - agent.train() - - def load_models(self, agent_model_dict): - """Load models from memory for each agent.""" - for agent_id, model_dict in agent_model_dict.items(): - self._agent_dict[agent_id].load_model_dict(model_dict) + return NotImplemented - def load_models_from_files(self, file_path_dict): - """Load models from disk for each agent.""" - for agent_id, file_path in file_path_dict.items(): - self._agent_dict[agent_id].load_model_dict_from(file_path) + @abstractmethod + def on_env_feedback(self, *args, **kwargs): + """Do things after a feedback is received from the environment following an action.""" + return NotImplemented - def dump_models(self, dir_path: str): - """Dump agents' models to disk. + @abstractmethod + def post_process(self, *args, **kwargs): + """Do things after an episode is finished. - Each agent will use its own name to create a separate file under ``dir_path`` for dumping. + These things may involve shaping experiences and resetting stateful objects. """ - os.makedirs(dir_path, exist_ok=True) - for agent in self._agent_dict.values(): - agent.dump_model_dict(dir_path) - - def get_models(self): - """Get agents' underlying models. + return NotImplemented - This is usually used in distributed mode where models need to be broadcast to remote roll-out actors. - """ - return {agent_id: agent.algorithm.model_dict for agent_id, agent in self._agent_dict.items()} + @abstractmethod + def train(self, *args, **kwargs): + """Train the agents.""" + return NotImplemented @property def name(self): """Agent manager's name.""" return self._name - @property - def agents(self): - """Agents managed by the agent manager.""" - return self._agent_dict - @property def explorer(self): """Explorer used by the agent manager.""" return self._explorer def _assert_train_mode(self): - if self._mode != AgentMode.TRAIN and self._mode != AgentMode.TRAIN_INFERENCE: - raise WrongAgentModeError(msg=f"this method is unavailable under mode {self._mode}") + if self._mode != AgentManagerMode.TRAIN and self._mode != AgentManagerMode.TRAIN_INFERENCE: + raise WrongAgentManagerModeError(msg=f"this method is unavailable under mode {self._mode}") def _assert_inference_mode(self): - if self._mode != AgentMode.INFERENCE and self._mode != AgentMode.TRAIN_INFERENCE: - raise WrongAgentModeError(msg=f"this method is unavailable under mode {self._mode}") + if self._mode != AgentManagerMode.INFERENCE and self._mode != AgentManagerMode.TRAIN_INFERENCE: + raise WrongAgentManagerModeError(msg=f"this method is unavailable under mode {self._mode}") diff --git a/maro/rl/agent/simple_agent_manager.py b/maro/rl/agent/simple_agent_manager.py new file mode 100644 index 000000000..6f7237aeb --- /dev/null +++ b/maro/rl/agent/simple_agent_manager.py @@ -0,0 +1,105 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +from abc import abstractmethod +import os + +from .abs_agent_manager import AbsAgentManager, AgentManagerMode +from maro.rl.shaping.state_shaper import StateShaper +from maro.rl.shaping.action_shaper import ActionShaper +from maro.rl.shaping.experience_shaper import ExperienceShaper +from maro.rl.explorer.abs_explorer import AbsExplorer +from maro.rl.storage.column_based_store import ColumnBasedStore +from maro.utils.exception.rl_toolkit_exception import MissingShaperError + + +class SimpleAgentManager(AbsAgentManager): + def __init__( + self, name: str, mode: AgentManagerMode, agent_dict: dict, + state_shaper: StateShaper = None, action_shaper: ActionShaper = None, + experience_shaper: ExperienceShaper = None, explorer: AbsExplorer = None, + ): + if mode in {AgentManagerMode.INFERENCE, AgentManagerMode.TRAIN_INFERENCE}: + if state_shaper is None: + raise MissingShaperError(msg=f"state shaper cannot be None under mode {self._mode}") + if action_shaper is None: + raise MissingShaperError(msg=f"action_shaper cannot be None under mode {self._mode}") + if experience_shaper is None: + raise MissingShaperError(msg=f"experience_shaper cannot be None under mode {self._mode}") + + super().__init__( + name, mode, agent_dict, state_shaper=state_shaper, action_shaper=action_shaper, + experience_shaper=experience_shaper, explorer=explorer + ) + + # data structures to temporarily store transitions and trajectory + self._transition_cache = {} + self._trajectory = ColumnBasedStore() + + def choose_action(self, decision_event, snapshot_list): + self._assert_inference_mode() + agent_id, model_state = self._state_shaper(decision_event, snapshot_list) + model_action = self.agent_dict[agent_id].choose_action( + model_state, self._explorer.epsilon[agent_id] if self._explorer else None + ) + + self._transition_cache = {"state": model_state, + "action": model_action, + "reward": None, + "agent_id": agent_id, + "event": decision_event} + return self._action_shaper(model_action, decision_event, snapshot_list) + + def on_env_feedback(self, metrics): + """This method records the environment-generated metrics as part of the latest transition in the trajectory. + + Args: + metrics: business metrics provided by the environment after an action has been executed. + """ + self._transition_cache["metrics"] = metrics + self._trajectory.put(self._transition_cache) + + def post_process(self, snapshot_list): + """This method processes the latest trajectory into experiences. + + Args: + snapshot_list: the snapshot list from the env at the end of an episode. + """ + experiences = self._experience_shaper(self._trajectory, snapshot_list) + self._trajectory.clear() + self._transition_cache = {} + self._state_shaper.reset() + self._action_shaper.reset() + self._experience_shaper.reset() + return experiences + + @abstractmethod + def train(self, *args, **kwargs): + """Train all agents.""" + return NotImplementedError + + def load_models(self, agent_model_dict): + """Load models from memory for each agent.""" + for agent_id, models in agent_model_dict.items(): + self.agent_dict[agent_id].load_models(models) + + def dump_models(self): + """Get agents' underlying models. + + This is usually used in distributed mode where models need to be broadcast to remote roll-out actors. + """ + return {agent_id: agent.dump_models() for agent_id, agent in self.agent_dict.items()} + + def load_models_from_files(self, dir_path): + """Load models from disk for each agent.""" + for agent in self.agent_dict.values(): + agent.load_models_from_file(dir_path) + + def dump_models_to_files(self, dir_path: str): + """Dump agents' models to disk. + + Each agent will use its own name to create a separate file under ``dir_path`` for dumping. + """ + os.makedirs(dir_path, exist_ok=True) + for agent in self.agent_dict.values(): + agent.dump_models_to_file(dir_path) diff --git a/maro/rl/algorithms/abs_algorithm.py b/maro/rl/algorithms/abs_algorithm.py new file mode 100644 index 000000000..2bab4b874 --- /dev/null +++ b/maro/rl/algorithms/abs_algorithm.py @@ -0,0 +1,60 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +from abc import ABC, abstractmethod + + +class AbsAlgorithm(ABC): + """Abstract RL algorithm class. + + The class provides uniform policy interfaces such as ``choose_action`` and ``train``. We also provide some + predefined RL algorithm based on it, such DQN, A2C, etc. User can inherit from it to customize their own + algorithms. + """ + def __init__(self): + pass + + @abstractmethod + def choose_action(self, state, epsilon: float = None): + """This method uses the underlying model(s) to compute an action from a shaped state. + + Args: + state: A state object shaped by a ``StateShaper`` to conform to the model input format. + epsilon (float, optional): Exploration rate. For greedy value-based algorithms, this being None means + using the model output without exploration. For algorithms with inherently stochastic policies such + as policy gradient, this is usually ignored. Defaults to None. + + Returns: + The action to be taken given ``state``. It is usually necessary to use an ``ActionShaper`` to convert + this to an environment executable action. + """ + return NotImplementedError + + @abstractmethod + def train(self, *args, **kwargs): + """Train models using samples. + + This method is algorithm-specific and needs to be implemented by the user. For example, for the DQN + algorithm, this may look like train(self, state, action, reward, next_state). + """ + return NotImplementedError + + @abstractmethod + def load_models(self, *models, **model_dict): + """Load trainable models from memory.""" + return NotImplementedError + + @abstractmethod + def dump_models(self): + """Return the algorithm's trainable models.""" + return NotImplementedError + + @abstractmethod + def load_models_from_file(self, path): + """Load trainable models from disk.""" + return NotImplementedError + + @abstractmethod + def dump_models_to_file(self, path: str): + """Dump the algorithm's trainable models to disk.""" + return NotImplementedError diff --git a/maro/rl/algorithms/dqn.py b/maro/rl/algorithms/dqn.py new file mode 100644 index 000000000..24161c8e1 --- /dev/null +++ b/maro/rl/algorithms/dqn.py @@ -0,0 +1,122 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import numpy as np +import torch +import torch.nn as nn + +from maro.rl.algorithms.abs_algorithm import AbsAlgorithm +from maro.utils import clone + + +class DQNHyperParams: + """Hyper-parameter set for the DQN algorithm. + + Args: + num_actions (int): number of possible actions + reward_decay (float): reward decay as defined in standard RL terminology + num_training_rounds_per_target_replacement (int): number of training frequency of target model replacement + tau (float): soft update coefficient, e.g., target_model = tau * eval_model + (1-tau) * target_model + """ + __slots__ = ["num_actions", "reward_decay", "num_training_rounds_per_target_replacement", "tau"] + + def __init__( + self, num_actions: int, reward_decay: float, num_training_rounds_per_target_replacement: int, tau: float = 1.0 + ): + self.num_actions = num_actions + self.reward_decay = reward_decay + self.num_training_rounds_per_target_replacement = num_training_rounds_per_target_replacement + self.tau = tau + + +class DQN(AbsAlgorithm): + """The Deep-Q-Networks algorithm. + + See https://web.stanford.edu/class/psych209/Readings/MnihEtAlHassibis15NatureControlDeepRL.pdf for details. + + Args: + eval_model (nn.Module): trainable Q-value model for computing actions given states. + optimizer_cls: torch optimizer class for the eval model. If this is None, the eval model is not trainable. + optimizer_params: parameters required for the eval optimizer class. + loss_func (Callable): loss function for the value model. + hyper_params: hyper-parameter set for the DQN algorithm. + target_model (nn.Module): Q-value model to train the ``eval_model`` against and to be updated periodically. If + it is None, the target model will be initialized as a deep copy of the eval model. + """ + def __init__( + self, eval_model: nn.Module, optimizer_cls, optimizer_params, loss_func, hyper_params: DQNHyperParams, + target_model: nn.Module = None + ): + super().__init__() + self._device = torch.device("cuda" if torch.cuda.is_available() else "cpu") + self._model_dict = {"eval": eval_model.to(self._device)} + if optimizer_cls is not None: + self._optimizer = optimizer_cls(self._model_dict["eval"].parameters(), **optimizer_params) + if target_model is None: + self._model_dict["target"] = clone(eval_model).to(self._device) + else: + self._model_dict["target"] = target_model.to(self._device) + self._loss_func = loss_func + self._hyper_params = hyper_params + self._train_cnt = 0 + + @property + def eval_model(self): + return self._model_dict["eval"] + + def choose_action(self, state: np.ndarray, epsilon: float = None): + if epsilon is None or np.random.rand() > epsilon: + state = torch.from_numpy(state).unsqueeze(0) + self._model_dict["eval"].eval() + with torch.no_grad(): + q_values = self._model_dict["eval"](state) + return q_values.argmax(dim=1).item() + + return np.random.choice(self._hyper_params.num_actions) + + def train(self, states: np.ndarray, actions: np.ndarray, rewards: np.ndarray, next_states: np.ndarray): + if hasattr(self, "_optimizer"): + states = torch.from_numpy(states).to(self._device) # (N, state_dim) + actions = torch.from_numpy(actions).to(self._device) # (N,) + rewards = torch.from_numpy(rewards).to(self._device) # (N,) + next_states = torch.from_numpy(next_states).to(self._device) # (N, state_dim) + if len(actions.shape) == 1: + actions = actions.unsqueeze(1) # (N, 1) + current_q_values = self._model_dict["eval"](states).gather(1, actions).squeeze(1) # (N,) + next_q_values = self._model_dict["target"](next_states).max(dim=1)[0] # (N,) + target_q_values = (rewards + self._hyper_params.reward_decay * next_q_values).detach() # (N,) + loss = self._loss_func(current_q_values, target_q_values) + self._model_dict["eval"].train() + self._optimizer.zero_grad() + loss.backward() + self._optimizer.step() + self._train_cnt += 1 + if self._train_cnt % self._hyper_params.num_training_rounds_per_target_replacement == 0: + self._update_target_model() + + return np.abs((current_q_values - target_q_values).detach().numpy()) + + def _update_target_model(self): + if hasattr(self, "_optimizer"): + for eval_params, target_params in zip( + self._model_dict["eval"].parameters(), self._model_dict["target"].parameters() + ): + target_params.data = ( + self._hyper_params.tau * eval_params.data + (1 - self._hyper_params.tau) * target_params.data + ) + + def load_models(self, eval_model): + """Load the eval model from memory.""" + self._model_dict["eval"] = eval_model + + def dump_models(self): + """Return the eval model.""" + return self._model_dict["eval"].state_dict() + + def load_models_from_file(self, path): + """Load the eval model from disk.""" + self._model_dict["eval"] = torch.load(path) + + def dump_models_to_file(self, path: str): + """Dump the eval model to disk.""" + torch.save(self._model_dict["eval"].state_dict(), path) diff --git a/maro/rl/algorithms/torch/__init__.py b/maro/rl/algorithms/torch/__init__.py deleted file mode 100644 index 9a0454564..000000000 --- a/maro/rl/algorithms/torch/__init__.py +++ /dev/null @@ -1,2 +0,0 @@ -# Copyright (c) Microsoft Corporation. -# Licensed under the MIT license. diff --git a/maro/rl/algorithms/torch/abs_algorithm.py b/maro/rl/algorithms/torch/abs_algorithm.py deleted file mode 100644 index ec58585c7..000000000 --- a/maro/rl/algorithms/torch/abs_algorithm.py +++ /dev/null @@ -1,81 +0,0 @@ -# Copyright (c) Microsoft Corporation. -# Licensed under the MIT license. - -from abc import ABC, abstractmethod -import itertools -from typing import Union - - -class AbsAlgorithm(ABC): - """Abstract RL algorithm class. - - The class provides uniform policy interfaces such as ``choose_action`` and ``train``. We also provide some - predefined RL algorithm based on it, such DQN, A2C, etc. User can inherit from it to customize their own - algorithms. - - Args: - model_dict (dict): Underlying models for the algorithm (e.g., for A2C, model_dict could be something like - {"actor": ..., "critic": ...}) - optimizer_opt (tuple or dict): Tuple or dict of tuples of (optimizer_class, optimizer_params) associated - with the models in model_dict. If it is a tuple, the optimizer to be instantiated applies to all - trainable parameters from ``model_dict``. If it is a dict, the optimizer will be applied to the related - model with the same key. - loss_func_dict (dict): Loss function types associated with the models. - hyper_params (object): Algorithm-specific hyper-parameter set. - """ - def __init__(self, model_dict: dict, optimizer_opt: Union[dict, tuple], loss_func_dict: dict, hyper_params: object): - self._loss_func_dict = loss_func_dict - self._hyper_params = hyper_params - self._model_dict = model_dict - self._register_optimizers(optimizer_opt) - - def _register_optimizers(self, optimizer_opt): - if isinstance(optimizer_opt, tuple): - # If a single optimizer_opt tuple is provided, a single optimizer will be created to jointly - # optimize all model parameters involved in the algorithm. - optim_cls, optim_params = optimizer_opt - model_params = [model.parameters() for model in self._model_dict.values()] - self._optimizer = optim_cls(itertools.chain(*model_params), **optim_params) - else: - self._optimizer = {} - for model_key, model in self._model_dict.items(): - # No gradient required - if model_key not in optimizer_opt or optimizer_opt[model_key] is None: - self._model_dict[model_key].eval() - self._optimizer[model_key] = None - else: - optim_cls, optim_params = optimizer_opt[model_key] - self._optimizer[model_key] = optim_cls(model.parameters(), **optim_params) - - @property - def model_dict(self): - return self._model_dict - - @model_dict.setter - def model_dict(self, model_dict): - self._model_dict = model_dict - - @abstractmethod - def train(self, *args, **kwargs): - """Train models using samples. - - This method is algorithm-specific and needs to be implemented by the user. For example, for the DQN - algorithm, this may look like train(self, state, action, reward, next_state). - """ - return NotImplementedError - - @abstractmethod - def choose_action(self, state, epsilon: float = None): - """This method uses the underlying model(s) to compute an action from a shaped state. - - Args: - state: A state object shaped by a ``StateShaper`` to conform to the model input format. - epsilon (float, optional): Exploration rate. For greedy value-based algorithms, this being None means - using the model output without exploration. For algorithms with inherently stochastic policies such - as policy gradient, this is usually ignored. Defaults to None. - - Returns: - The action to be taken given ``state``. It is usually necessary to use an ``ActionShaper`` to convert - this to an environment executable action. - """ - return NotImplementedError diff --git a/maro/rl/algorithms/torch/dqn.py b/maro/rl/algorithms/torch/dqn.py deleted file mode 100644 index 22cc0bade..000000000 --- a/maro/rl/algorithms/torch/dqn.py +++ /dev/null @@ -1,88 +0,0 @@ -# Copyright (c) Microsoft Corporation. -# Licensed under the MIT license. - -from typing import Union - -import numpy as np -import torch - - -from maro.rl.algorithms.torch.abs_algorithm import AbsAlgorithm -from maro.utils import clone - - -class DQNHyperParams: - """DQN hyper-parameters. - - Args: - num_actions (int): number of possible actions - reward_decay (float): reward decay as defined in standard RL terminology - num_training_rounds_per_target_replacement (int): number of training frequency of target model replacement - tau (float): soft update coefficient, e.g., target_model = tau * eval_model + (1-tau) * target_model - """ - __slots__ = ["num_actions", "reward_decay", "num_training_rounds_per_target_replacement", "tau"] - def __init__(self, num_actions: int, reward_decay: float, num_training_rounds_per_target_replacement: int, - tau: float = 1.0): - self.num_actions = num_actions - self.reward_decay = reward_decay - self.num_training_rounds_per_target_replacement = num_training_rounds_per_target_replacement - self.tau = tau - - -class DQN(AbsAlgorithm): - """The Deep-Q-Networks algorithm. - - The model_dict must contain the key `eval`. Optionally a model corresponding to the key `target` can be - provided. If the key `target` is absent or model_dict[`target`] is None, the target model will be a deep - copy of the provided eval model. - """ - def __init__(self, model_dict: dict, optimizer_opt: Union[dict, tuple], loss_func_dict: dict, - hyper_params: DQNHyperParams): - if model_dict.get("target", None) is None: - model_dict["target"] = clone(model_dict["eval"]) - super().__init__(model_dict, optimizer_opt, loss_func_dict, hyper_params) - self._train_cnt = 0 - self._device = torch.device("cuda" if torch.cuda.is_available() else "cpu") - - def choose_action(self, state: np.ndarray, epsilon: float = None): - if epsilon is None or np.random.rand() > epsilon: - state = torch.from_numpy(state).unsqueeze(0) - self._model_dict["eval"].eval() - with torch.no_grad(): - q_values = self._model_dict["eval"](state) - best_action_idx = q_values.argmax(dim=1).item() - return best_action_idx - - return np.random.choice(self._hyper_params.num_actions) - - def _prepare_batch(self, raw_batch): - return {key: torch.from_numpy(np.asarray(lst)).to(self._device) for key, lst in raw_batch.items()} - - def train(self, state: np.ndarray, action: np.ndarray, reward: np.ndarray, next_state: np.ndarray): - state = torch.from_numpy(state).to(self._device) - action = torch.from_numpy(action).to(self._device) - reward = torch.from_numpy(reward).to(self._device) - next_state = torch.from_numpy(next_state).to(self._device) - if len(action.shape) == 1: - action = action.unsqueeze(1) - current_q_values = self._model_dict["eval"](state).gather(1, action).squeeze(1) - next_q_values = self._model_dict["target"](next_state).max(dim=1)[0] - target_q_values = (reward + self._hyper_params.reward_decay * next_q_values).detach() - loss = self._loss_func_dict["eval"](current_q_values, target_q_values) - self._model_dict["eval"].train() - self._optimizer.zero_grad() - loss.backward() - self._optimizer.step() - self._train_cnt += 1 - if self._train_cnt % self._hyper_params.num_training_rounds_per_target_replacement == 0: - self._update_target_model() - - return np.abs((current_q_values - target_q_values).detach().numpy()) - - def _update_target_model(self): - for eval_params, target_params in zip( - self._model_dict["eval"].parameters(), self._model_dict["target"].parameters() - ): - target_params.data = ( - self._hyper_params.tau * eval_params.data + (1 - self._hyper_params.tau) * target_params.data - ) diff --git a/maro/rl/learner/abs_learner.py b/maro/rl/learner/abs_learner.py index 58966289a..a58582514 100644 --- a/maro/rl/learner/abs_learner.py +++ b/maro/rl/learner/abs_learner.py @@ -1,7 +1,7 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT license. -from abc import ABC, abstractmethod +from abc import ABC class AbsLearner(ABC): @@ -9,16 +9,10 @@ class AbsLearner(ABC): def __init__(self): pass - @abstractmethod - def train(self, total_episodes): - """The outermost training loop logic is implemented here. - - Args: - total_episodes (int): number of episodes to be run. - """ + def train(self, *args, **kwargs): + """The outermost training loop logic is implemented here.""" pass - @abstractmethod def test(self): """Test policy performance.""" pass diff --git a/maro/rl/learner/simple_learner.py b/maro/rl/learner/simple_learner.py index 5599562a7..d7a2af6dc 100644 --- a/maro/rl/learner/simple_learner.py +++ b/maro/rl/learner/simple_learner.py @@ -2,7 +2,7 @@ # Licensed under the MIT license. from .abs_learner import AbsLearner -from maro.rl.agent.abs_agent_manager import AbsAgentManager +from maro.rl.agent.simple_agent_manager import SimpleAgentManager from maro.rl.actor.simple_actor import SimpleActor from maro.utils import DummyLogger @@ -15,43 +15,44 @@ class SimpleLearner(AbsLearner): actor (Actor or ActorProxy): an Actor or VectorActorProxy instance. logger: used for logging important messages. """ - def __init__(self, trainable_agents: AbsAgentManager, actor, logger=DummyLogger()): + def __init__(self, trainable_agents: SimpleAgentManager, actor, logger=DummyLogger()): super().__init__() self._trainable_agents = trainable_agents self._actor = actor self._logger = logger - def train(self, total_episodes): + def train(self, total_episodes: int): """Main loop for collecting experiences from the actor and using them to update policies. Args: total_episodes (int): number of episodes to be run. """ for current_ep in range(1, total_episodes + 1): - model_dict = None if self._is_shared_agent_instance() else self._trainable_agents.get_models() + model_dict = None if self._is_shared_agent_instance() else self._trainable_agents.dump_models() epsilon_dict = self._trainable_agents.explorer.epsilon if self._trainable_agents.explorer else None performance, exp_by_agent = self._actor.roll_out(model_dict=model_dict, epsilon_dict=epsilon_dict) if isinstance(performance, dict): for actor_id, perf in performance.items(): - self._logger.info(f"ep {current_ep} - performance: {perf}," - f"source: {actor_id}, epsilons: {epsilon_dict}") + self._logger.info( + f"ep {current_ep} - performance: {perf}, source: {actor_id}, epsilons: {epsilon_dict}" + ) else: self._logger.info(f"ep {current_ep} - performance: {performance}, epsilons: {epsilon_dict}") - self._trainable_agents.store_experiences(exp_by_agent) - self._trainable_agents.train() - self._trainable_agents.update_epsilon(performance) + self._trainable_agents.train(exp_by_agent) def test(self): """Test policy performance.""" - performance, _ = self._actor.roll_out(model_dict=self._trainable_agents.get_models(), return_details=False) + performance, _ = self._actor.roll_out( + model_dict=self._trainable_agents.dump_models(), + return_details=False + ) for actor_id, perf in performance.items(): self._logger.info(f"test performance from {actor_id}: {perf}") self._actor.roll_out(done=True) - def dump_models(self, dir_path: str): - """Dump agents' models to disk.""" - self._trainable_agents.dump_models(dir_path) + def save_models(self, model_dump_dir: str): + self._trainable_agents.dump_models_to_files(model_dump_dir) def _is_shared_agent_instance(self): """If true, the set of agents performing inference in actor is the same as self._trainable_agents.""" diff --git a/maro/rl/models/decision_layers.py b/maro/rl/models/decision_layers.py new file mode 100644 index 000000000..23c9e4bed --- /dev/null +++ b/maro/rl/models/decision_layers.py @@ -0,0 +1,88 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +from collections import OrderedDict + +import torch.nn as nn + + +class DecisionLayers(nn.Module): + """NN model to compute state or action values. + + Fully connected network with optional batch normalization, activation and dropout components. + + Args: + name (str): Network name. + input_dim (int): Network input dimension. + output_dim (int): Network output dimension. + hidden_dims ([int]): Dimensions of hidden layers. Its length is the number of hidden layers. + activation: A ``torch.nn`` activation type. If None, there will be no activation. Defaults to LeakyReLU. + softmax_enabled (bool): If true, the output of the net will be a softmax transformation of the top layer's + output. Defaults to False. + batch_norm_enabled (bool): If true, batch normalization will be performed at each layer. + dropout_p (float): Dropout probability. Defaults to None, in which case there is no drop-out. + """ + def __init__( + self, *, name: str, input_dim: int, output_dim: int, hidden_dims: [int], activation=nn.LeakyReLU, + softmax_enabled: bool = False, batch_norm_enabled: bool = False, dropout_p: float = None + ): + super().__init__() + self._name = name + self._input_dim = input_dim + self._hidden_dims = hidden_dims if hidden_dims is not None else [] + self._output_dim = output_dim + + # build the net + self._layers = self._build_layers([input_dim] + self._hidden_dims) + if len(self._hidden_dims) == 0: + self._top_layer = nn.Linear(self._input_dim, self._output_dim) + else: + self._top_layer = nn.Linear(hidden_dims[-1], self._output_dim) + self._net = nn.Sequential(*self._layers, self._top_layer) + + # network features + self._activation = activation + self._softmax = nn.Softmax(dim=1) if softmax_enabled else None + self._batch_norm_enabled = batch_norm_enabled + self._dropout_p = dropout_p + + def forward(self, x): + out = self._net(x).double() + return self._softmax(out) if self._softmax else out + + @property + def name(self): + return self._name + + @property + def input_dim(self): + return self._input_dim + + @property + def output_dim(self): + return self._output_dim + + def _build_basic_layer(self, input_dim, output_dim): + """Build basic layer. + + BN -> Linear -> LeakyReLU -> Dropout + """ + components = [] + if self._batch_norm_enabled: + components.append(("batch_norm", nn.BatchNorm1d(input_dim))) + components.append(("linear", nn.Linear(input_dim, output_dim))) + if self._activation is not None: + components.append(("activation", self._activation())) + if self._dropout_p: + components.append(("dropout", nn.Dropout(p=self._dropout_p))) + return nn.Sequential(OrderedDict(components)) + + def _build_layers(self, layer_dims: []): + """Build multi basic layer. + + BasicLayer1 -> BasicLayer2 -> ... + """ + layers = [] + for input_dim, output_dim in zip(layer_dims, layer_dims[1:]): + layers.append(self._build_basic_layer(input_dim, output_dim)) + return layers diff --git a/maro/rl/models/torch/learning_model.py b/maro/rl/models/learning_model.py similarity index 84% rename from maro/rl/models/torch/learning_model.py rename to maro/rl/models/learning_model.py index dcad42ac1..7272b6550 100644 --- a/maro/rl/models/torch/learning_model.py +++ b/maro/rl/models/learning_model.py @@ -20,10 +20,10 @@ class LearningModel(nn.Module): and outputs values of interest in RL (e.g., state & action values). clip_value (float): Threshold used to clip gradients. """ - def __init__(self, - representation_layers: nn.Module = IdentityLayers(), - decision_layers: nn.Module = IdentityLayers(), - clip_value: float = None): + def __init__( + self, representation_layers: nn.Module = IdentityLayers(), decision_layers: nn.Module = IdentityLayers(), + clip_value: float = None + ): super().__init__() self._net = nn.Sequential(representation_layers, decision_layers) diff --git a/maro/rl/models/torch/mlp_representation.py b/maro/rl/models/representation_layers.py similarity index 90% rename from maro/rl/models/torch/mlp_representation.py rename to maro/rl/models/representation_layers.py index 64d3af7ea..6c7408590 100644 --- a/maro/rl/models/torch/mlp_representation.py +++ b/maro/rl/models/representation_layers.py @@ -4,7 +4,7 @@ import torch.nn as nn -class MLPRepresentation(nn.Module): +class RepresentationLayers(nn.Module): def __init__(self, name: str, input_dim: int, hidden_dims: [int], output_dim: int, dropout_p: float): """Deep Q network. @@ -13,8 +13,7 @@ def __init__(self, name: str, input_dim: int, hidden_dims: [int], output_dim: in Args: name (str): Network name. input_dim (int): Network input dimension. - hidden_dims ([int]): Network hiddenlayer dimension. The length of ``hidden_dims`` means the - hidden layer number, which requires larger than 1. + hidden_dims ([int]): Dimensions of hidden layers. Its length is the number of hidden layers. output_dim (int): Network output dimension. dropout_p (float): Dropout parameter. """ diff --git a/maro/rl/models/torch/__init__.py b/maro/rl/models/torch/__init__.py deleted file mode 100644 index 9a0454564..000000000 --- a/maro/rl/models/torch/__init__.py +++ /dev/null @@ -1,2 +0,0 @@ -# Copyright (c) Microsoft Corporation. -# Licensed under the MIT license. diff --git a/maro/rl/models/torch/decision_layers.py b/maro/rl/models/torch/decision_layers.py deleted file mode 100644 index 36c62f736..000000000 --- a/maro/rl/models/torch/decision_layers.py +++ /dev/null @@ -1,67 +0,0 @@ -# Copyright (c) Microsoft Corporation. -# Licensed under the MIT license. - -import torch.nn as nn - - -class MLPDecisionLayers(nn.Module): - """Deep Q network. - - Choose multi-layer full connection with dropout as the basic network architecture. - - Args: - name (str): Network name. - input_dim (int): Network input dimension. - hidden_dims ([int]): Network hidden layer dimension. The length of ``hidden_dims`` means the - hidden layer number, which requires larger than 1. - output_dim (int): Network output dimension. - dropout_p (float): Dropout parameter. - """ - def __init__(self, *, name: str, input_dim: int, output_dim: int, hidden_dims: [int], dropout_p: float): - super().__init__() - self._name = name - self._input_dim = input_dim - self._hidden_dims = hidden_dims if hidden_dims is not None else [] - self._output_dim = output_dim - self._dropout_p = dropout_p - self._layers = self._build_layers([input_dim] + self._hidden_dims) - if len(self._hidden_dims) == 0: - self._head = nn.Linear(self._input_dim, self._output_dim) - else: - self._head = nn.Linear(hidden_dims[-1], self._output_dim) - self._net = nn.Sequential(*self._layers, self._head) - - def forward(self, x): - return self._net(x).double() - - @property - def input_dim(self): - return self._input_dim - - @property - def name(self): - return self._name - - @property - def output_dim(self): - return self._output_dim - - def _build_basic_layer(self, input_dim, output_dim): - """Build basic layer. - - BN -> Linear -> LeakyReLU -> Dropout - """ - return nn.Sequential(nn.BatchNorm1d(input_dim), - nn.Linear(input_dim, output_dim), - nn.LeakyReLU(), - nn.Dropout(p=self._dropout_p)) - - def _build_layers(self, layer_dims: []): - """Build multi basic layer. - - BasicLayer1 -> BasicLayer2 -> ... - """ - layers = [] - for input_dim, output_dim in zip(layer_dims, layer_dims[1:]): - layers.append(self._build_basic_layer(input_dim, output_dim)) - return layers diff --git a/maro/rl/shaping/experience_shaper.py b/maro/rl/shaping/experience_shaper.py index 428669a83..f7e6d1247 100644 --- a/maro/rl/shaping/experience_shaper.py +++ b/maro/rl/shaping/experience_shaper.py @@ -2,7 +2,7 @@ # Licensed under the MIT license. from abc import abstractmethod -from typing import Callable, Iterable, Sequence, Union +from typing import Callable, Iterable, Union from .abs_shaper import AbsShaper @@ -23,11 +23,11 @@ def __init__(self, reward_func: Union[Callable, None], *args, **kwargs): self._reward_func = reward_func @abstractmethod - def __call__(self, trajectory: Sequence, snapshot_list) -> Iterable: + def __call__(self, trajectory, snapshot_list) -> Iterable: """Converts transitions along a trajectory to experiences. Args: - trajectory(Sequence): A sequence of transitions recorded by the agent manager during roll-out. + trajectory: A sequence of transitions recorded by the agent manager during roll-out. snapshot_list: Snapshot list stored in the environment at the end of an episode. Returns: Experiences that can be used by the algorithm. diff --git a/maro/rl/storage/column_based_store.py b/maro/rl/storage/column_based_store.py index 8709a8d8e..21f78c3b6 100644 --- a/maro/rl/storage/column_based_store.py +++ b/maro/rl/storage/column_based_store.py @@ -9,6 +9,7 @@ from .abs_store import AbsStore from .utils import check_uniformity, get_update_indexes, normalize, OverwriteType from maro.utils import clone +from maro.utils.exception.rl_toolkit_exception import StoreMisalignmentError class ColumnBasedStore(AbsStore): @@ -53,6 +54,16 @@ def __next__(self): def __getitem__(self, index: int): return {k: lst[index] for k, lst in self._store.items()} + def __getstate__(self): + """A small modification to make the object picklable. + + Using the default ``__dict__`` would make the object unpicklable due to the lambda function involved in the + ``defaultdict`` definition of the ``_store`` attribute. + """ + obj_dict = self.__dict__ + obj_dict["_store"] = dict(obj_dict["_store"]) + return obj_dict + @property def capacity(self): """Store capacity. @@ -75,7 +86,8 @@ def put(self, contents: dict, overwrite_indexes: Sequence = None) -> List[int]: """Put new contents in the store. Args: - contents (Sequence): Item object list. + contents (dict): dictionary of items to add to the store. If the store is not empty, this must have the + same keys as the store itself. Otherwise an ``StoreMisalignmentError`` will be raised. overwrite_indexes (Sequence, optional): indexes where the contents are to be overwritten. This is only used when the store has a fixed capacity and putting ``contents`` in the store would exceed this capacity. If this is None and overwriting is necessary, rolling or random overwriting will be done @@ -84,16 +96,20 @@ def put(self, contents: dict, overwrite_indexes: Sequence = None) -> List[int]: The indexes where the newly added entries reside in the store. """ if len(self._store) > 0 and contents.keys() != self._store.keys(): - raise ValueError(f"expected keys {list(self._store.keys())}, got {list(contents.keys())}") + raise StoreMisalignmentError(f"expected keys {list(self._store.keys())}, got {list(contents.keys())}") added_size = len(contents[next(iter(contents))]) if self._capacity < 0: - for key, lst in contents.items(): - self._store[key].extend(lst) + for key, val in contents.items(): + if not isinstance(val, list): + self._store[key].append(val) + else: + self._store[key].extend(val) self._size += added_size return list(range(self._size - added_size, self._size)) else: - write_indexes = get_update_indexes(self._size, added_size, self._capacity, self._overwrite_type, - overwrite_indexes=overwrite_indexes) + write_indexes = get_update_indexes( + self._size, added_size, self._capacity, self._overwrite_type, overwrite_indexes=overwrite_indexes + ) self.update(write_indexes, contents) self._size = min(self._capacity, self._size + added_size) return write_indexes @@ -125,7 +141,7 @@ def apply_multi_filters(self, filters: Sequence[Callable]): Args: filters (Sequence[Callable]): Filter list, each item is a lambda function, - e.g., [lambda d: d['a'] == 1 and d['b'] == 1]. + e.g., [lambda d: d['a'] == 1 and d['b'] == 1]. Returns: Filtered indexes and corresponding objects. """ diff --git a/maro/rl/storage/utils.py b/maro/rl/storage/utils.py index c942d13bb..9c4da220b 100644 --- a/maro/rl/storage/utils.py +++ b/maro/rl/storage/utils.py @@ -6,15 +6,19 @@ import numpy as np +from maro.utils.exception.rl_toolkit_exception import StoreMisalignmentError + def check_uniformity(arg_num): def decorator(func): @wraps(func) def wrapper(*args, **kwargs): contents = args[arg_num] - length = len(contents[next(iter(contents))]) - if any(len(lst) != length for lst in contents.values()): - raise ValueError("all sequences in contents should have the same length") + if all(not isinstance(val, list) for val in contents.values()): + return func(*args, **kwargs) + col_length = len(contents[next(iter(contents))]) + if any(not isinstance(val, list) or len(val) != col_length for val in contents.values()): + raise StoreMisalignmentError("values of contents should consist of lists of the same length") return func(*args, **kwargs) return wrapper return decorator From cc368606c3be05760eb773dfdd54ccce2b60d4b1 Mon Sep 17 00:00:00 2001 From: ysqyang Date: Wed, 21 Oct 2020 22:39:32 +0800 Subject: [PATCH 036/337] fixed formattin issues --- maro/rl/agent/simple_agent_manager.py | 13 +++++++------ maro/rl/models/representation_layers.py | 8 +++++--- 2 files changed, 12 insertions(+), 9 deletions(-) diff --git a/maro/rl/agent/simple_agent_manager.py b/maro/rl/agent/simple_agent_manager.py index 6f7237aeb..ef1ca5bd8 100644 --- a/maro/rl/agent/simple_agent_manager.py +++ b/maro/rl/agent/simple_agent_manager.py @@ -42,12 +42,13 @@ def choose_action(self, decision_event, snapshot_list): model_action = self.agent_dict[agent_id].choose_action( model_state, self._explorer.epsilon[agent_id] if self._explorer else None ) - - self._transition_cache = {"state": model_state, - "action": model_action, - "reward": None, - "agent_id": agent_id, - "event": decision_event} + self._transition_cache = { + "state": model_state, + "action": model_action, + "reward": None, + "agent_id": agent_id, + "event": decision_event + } return self._action_shaper(model_action, decision_event, snapshot_list) def on_env_feedback(self, metrics): diff --git a/maro/rl/models/representation_layers.py b/maro/rl/models/representation_layers.py index 6c7408590..3e7812ce2 100644 --- a/maro/rl/models/representation_layers.py +++ b/maro/rl/models/representation_layers.py @@ -50,9 +50,11 @@ def _build_basic_layer(self, input_dim, output_dim): BN -> Linear -> LeakyReLU -> Dropout """ - return nn.Sequential(nn.Linear(input_dim, output_dim), - nn.LeakyReLU(), - nn.Dropout(p=self._dropout_p)) + return nn.Sequential( + nn.Linear(input_dim, output_dim), + nn.LeakyReLU(), + nn.Dropout(p=self._dropout_p) + ) def _build_layers(self, layer_dims: []): """Build multi basic layer. From b826b2a4cc1b6ac666cf7fff0517785ab8c2d742 Mon Sep 17 00:00:00 2001 From: ysqyang Date: Wed, 21 Oct 2020 22:42:46 +0800 Subject: [PATCH 037/337] checked out error-code related code from v0.2_pg --- maro/utils/exception/error_code.py | 8 +++++--- maro/utils/exception/rl_toolkit_exception.py | 21 ++++++++++++++++---- 2 files changed, 22 insertions(+), 7 deletions(-) diff --git a/maro/utils/exception/error_code.py b/maro/utils/exception/error_code.py index 34a6e8903..da155e1ae 100644 --- a/maro/utils/exception/error_code.py +++ b/maro/utils/exception/error_code.py @@ -35,7 +35,9 @@ 3003: "Deployment Error", # 4000-4999: Error codes for RL toolkit - 4001: "Unsupported Agent Mode Error", - 4002: "Missing Shaper Error", - 4003: "Wrong Agent Mode Error" + 4001: "Unsupported Agent Mode", + 4002: "Missing Shaper", + 4003: "Wrong Agent Manager Mode", + 4004: "Wrong Agent Mode", + 4005: "Store Misalignment Error" } diff --git a/maro/utils/exception/rl_toolkit_exception.py b/maro/utils/exception/rl_toolkit_exception.py index a914ecbdf..33f29ab4f 100644 --- a/maro/utils/exception/rl_toolkit_exception.py +++ b/maro/utils/exception/rl_toolkit_exception.py @@ -5,18 +5,31 @@ class UnsupportedAgentModeError(MAROException): - """Unsupported agent mode error.""" + """Unsupported agent mode.""" def __init__(self, msg: str = None): super().__init__(4001, msg) class MissingShaperError(MAROException): - """Missing shaper error.""" + """Missing shaper.""" def __init__(self, msg: str = None): super().__init__(4002, msg) -class WrongAgentModeError(MAROException): - """Wrong agent mode error.""" +class WrongAgentManagerModeError(MAROException): + """Wrong agent manager mode.""" def __init__(self, msg: str = None): super().__init__(4003, msg) + + +class WrongAgentModeError(MAROException): + """Wrong agent mode.""" + def __init__(self, msg: str = None): + super().__init__(4004, msg) + + +class StoreMisalignmentError(MAROException): + """Raised when a ``put`` operation on a ``ColumnBasedStore`` would cause the underlying lists to have different + sizes.""" + def __init__(self, msg: str = None): + super().__init__(4005, msg) From b4067697f193c951a0fc4714533c502ee531c60c Mon Sep 17 00:00:00 2001 From: ysqyang Date: Wed, 21 Oct 2020 22:44:23 +0800 Subject: [PATCH 038/337] fixed a bug --- maro/rl/__init__.py | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/maro/rl/__init__.py b/maro/rl/__init__.py index 38be480e8..97b21630b 100644 --- a/maro/rl/__init__.py +++ b/maro/rl/__init__.py @@ -9,11 +9,6 @@ from maro.rl.agent.abs_agent_manager import AbsAgentManager, AgentManagerMode from maro.rl.agent.simple_agent_manager import SimpleAgentManager from maro.rl.algorithms.abs_algorithm import AbsAlgorithm -from maro.rl.algorithms.pg import PolicyGradient, PolicyGradientHyperParameters -from maro.rl.algorithms.ac import ActorCritic, ActorCriticWithCombinedModel, ActorCriticHyperParameters, \ - ActorCriticHyperParametersWithCombinedModel -from maro.rl.algorithms.ppo import PPO, PPOWithCombinedModel, PPOHyperParameters, \ - PPOHyperParametersWithCombinedModel from maro.rl.algorithms.dqn import DQN, DQNHyperParams from maro.rl.models.learning_model import LearningModel from maro.rl.models.representation_layers import RepresentationLayers @@ -42,16 +37,6 @@ "AgentManagerMode", "SimpleAgentManager", "AbsAlgorithm", - "PolicyGradient", - "PolicyGradientHyperParameters", - "ActorCritic", - "ActorCriticWithCombinedModel", - "ActorCriticHyperParameters", - "ActorCriticHyperParametersWithCombinedModel", - "PPO", - "PPOWithCombinedModel", - "PPOHyperParameters", - "PPOHyperParametersWithCombinedModel", "DQN", "DQNHyperParams", "LearningModel", From 3f4ca3d086865ed61922db744717c8dc86ca118e Mon Sep 17 00:00:00 2001 From: ysqyang Date: Wed, 21 Oct 2020 22:45:08 +0800 Subject: [PATCH 039/337] fixed a bug --- examples/cim/dqn/components/config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/cim/dqn/components/config.py b/examples/cim/dqn/components/config.py index 26a3a8c43..aaebb3ff9 100644 --- a/examples/cim/dqn/components/config.py +++ b/examples/cim/dqn/components/config.py @@ -18,7 +18,7 @@ # obtain model input dimension from state shaping configurations look_back = config["state_shaping"]["look_back"] -max_ports_downstream = config["state_shaping"]["max_port_downstream"] +max_ports_downstream = config["state_shaping"]["max_ports_downstream"] num_port_attributes = len(config["state_shaping"]["port_attributes"]) num_vessel_attributes = len(config["state_shaping"]["vessel_attributes"]) From dc1b42e08524ecea27c05786e4907463d55b82c1 Mon Sep 17 00:00:00 2001 From: ysqyang Date: Wed, 21 Oct 2020 22:47:46 +0800 Subject: [PATCH 040/337] fixed a bug --- maro/rl/algorithms/dqn.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/maro/rl/algorithms/dqn.py b/maro/rl/algorithms/dqn.py index 24161c8e1..cfab9c1c1 100644 --- a/maro/rl/algorithms/dqn.py +++ b/maro/rl/algorithms/dqn.py @@ -44,16 +44,16 @@ class DQN(AbsAlgorithm): it is None, the target model will be initialized as a deep copy of the eval model. """ def __init__( - self, eval_model: nn.Module, optimizer_cls, optimizer_params, loss_func, hyper_params: DQNHyperParams, - target_model: nn.Module = None + self, eval_model, optimizer_cls, optimizer_params, loss_func, hyper_params: DQNHyperParams, + target_model=None ): super().__init__() self._device = torch.device("cuda" if torch.cuda.is_available() else "cpu") - self._model_dict = {"eval": eval_model.to(self._device)} + self._model_dict = {"eval": eval_model.to(self._device) if eval_model is not None else eval_model} if optimizer_cls is not None: self._optimizer = optimizer_cls(self._model_dict["eval"].parameters(), **optimizer_params) if target_model is None: - self._model_dict["target"] = clone(eval_model).to(self._device) + self._model_dict["target"] = clone(eval_model).to(self._device) if eval_model is not None else None else: self._model_dict["target"] = target_model.to(self._device) self._loss_func = loss_func From 46889df5781d3f3b14990ba9a3975f35c1928916 Mon Sep 17 00:00:00 2001 From: ysqyang Date: Wed, 21 Oct 2020 22:51:28 +0800 Subject: [PATCH 041/337] fixed a bug --- examples/cim/dqn/components/agent_manager.py | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/examples/cim/dqn/components/agent_manager.py b/examples/cim/dqn/components/agent_manager.py index 4c09bb40b..207f3fb83 100644 --- a/examples/cim/dqn/components/agent_manager.py +++ b/examples/cim/dqn/components/agent_manager.py @@ -10,18 +10,23 @@ def create_dqn_agents(agent_id_list, mode, config): + num_actions = config.algorithm.num_actions if mode in {AgentMode.TRAIN, AgentMode.TRAIN_INFERENCE}: return {agent_id: DQN( - eval_model=None, - optimizer_cls=None, - optimizer_params=None, - loss_func=None, - hyper_params=None - ) + eval_model=None, + optimizer_cls=None, + optimizer_params=None, + loss_func=None, + hyper_params=DQNHyperParams( + num_actions=num_actions, + reward_decay=None, + num_training_rounds_per_target_replacement=None, + tau=None + ) + ) for agent_id in agent_id_list} set_seeds(config.seed) - num_actions = config.algorithm.num_actions agent_dict = {} for agent_id in agent_id_list: eval_model = LearningModel( From ed0ff9c7740e802b86b8f55de87a9f66c29d3d99 Mon Sep 17 00:00:00 2001 From: ysqyang Date: Wed, 21 Oct 2020 22:53:06 +0800 Subject: [PATCH 042/337] fixed a bug --- examples/cim/dqn/components/agent_manager.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/cim/dqn/components/agent_manager.py b/examples/cim/dqn/components/agent_manager.py index 207f3fb83..5af422257 100644 --- a/examples/cim/dqn/components/agent_manager.py +++ b/examples/cim/dqn/components/agent_manager.py @@ -11,7 +11,7 @@ def create_dqn_agents(agent_id_list, mode, config): num_actions = config.algorithm.num_actions - if mode in {AgentMode.TRAIN, AgentMode.TRAIN_INFERENCE}: + if mode == AgentMode.INFERENCE: return {agent_id: DQN( eval_model=None, optimizer_cls=None, From 53dc74e01509cd74c5e0212b485db6d1e5adb8e2 Mon Sep 17 00:00:00 2001 From: ysqyang Date: Wed, 21 Oct 2020 22:57:37 +0800 Subject: [PATCH 043/337] fixed a bug --- examples/cim/dqn/components/agent_manager.py | 8 ++++---- examples/cim/dqn/config.yml | 4 +++- maro/rl/models/decision_layers.py | 2 +- 3 files changed, 8 insertions(+), 6 deletions(-) diff --git a/examples/cim/dqn/components/agent_manager.py b/examples/cim/dqn/components/agent_manager.py index 5af422257..4440eabb0 100644 --- a/examples/cim/dqn/components/agent_manager.py +++ b/examples/cim/dqn/components/agent_manager.py @@ -1,7 +1,7 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT license. -from torch.nn.functional import smooth_l1_loss +import torch.nn as nn from torch.optim import RMSprop from .agent import CIMAgent @@ -31,8 +31,8 @@ def create_dqn_agents(agent_id_list, mode, config): for agent_id in agent_id_list: eval_model = LearningModel( decision_layers=DecisionLayers( - name=f'{agent_id}.policy', input_dim=config.algorithm.input_dim, - output_dim=num_actions, **config.algorithm.model + name=f'{agent_id}.policy', input_dim=config.algorithm.input_dim, output_dim=num_actions, + activation=nn.LeakyReLU, **config.algorithm.model ) ) @@ -40,7 +40,7 @@ def create_dqn_agents(agent_id_list, mode, config): eval_model=eval_model, optimizer_cls=RMSprop, optimizer_params=config.algorithm.optimizer, - loss_func=smooth_l1_loss, + loss_func=nn.functional.smooth_l1_loss, hyper_params=DQNHyperParams( **config.algorithm.hyper_parameters, num_actions=num_actions diff --git a/examples/cim/dqn/config.yml b/examples/cim/dqn/config.yml index 4836c45e9..488e9b1a8 100644 --- a/examples/cim/dqn/config.yml +++ b/examples/cim/dqn/config.yml @@ -41,6 +41,8 @@ agents: - 256 - 128 - 64 + softmax_enabled: false + batch_norm_enabled: false dropout_p: 0.0 optimizer: lr: 0.05 @@ -63,4 +65,4 @@ distributed: peer: {"actor_worker": 1} redis: host_name: "localhost" - port: 6379 \ No newline at end of file + port: 6379 diff --git a/maro/rl/models/decision_layers.py b/maro/rl/models/decision_layers.py index 23c9e4bed..24af2f970 100644 --- a/maro/rl/models/decision_layers.py +++ b/maro/rl/models/decision_layers.py @@ -23,7 +23,7 @@ class DecisionLayers(nn.Module): dropout_p (float): Dropout probability. Defaults to None, in which case there is no drop-out. """ def __init__( - self, *, name: str, input_dim: int, output_dim: int, hidden_dims: [int], activation=nn.LeakyReLU, + self, name: str, input_dim: int, output_dim: int, hidden_dims: [int], activation=nn.LeakyReLU, softmax_enabled: bool = False, batch_norm_enabled: bool = False, dropout_p: float = None ): super().__init__() From 0754a397f889f57359f99a8bbd3ce385ef118463 Mon Sep 17 00:00:00 2001 From: ysqyang Date: Wed, 21 Oct 2020 23:00:10 +0800 Subject: [PATCH 044/337] fixed a bug --- maro/rl/models/decision_layers.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/maro/rl/models/decision_layers.py b/maro/rl/models/decision_layers.py index 24af2f970..c4f4cbdd9 100644 --- a/maro/rl/models/decision_layers.py +++ b/maro/rl/models/decision_layers.py @@ -32,6 +32,12 @@ def __init__( self._hidden_dims = hidden_dims if hidden_dims is not None else [] self._output_dim = output_dim + # network features + self._activation = activation + self._softmax = nn.Softmax(dim=1) if softmax_enabled else None + self._batch_norm_enabled = batch_norm_enabled + self._dropout_p = dropout_p + # build the net self._layers = self._build_layers([input_dim] + self._hidden_dims) if len(self._hidden_dims) == 0: @@ -40,12 +46,6 @@ def __init__( self._top_layer = nn.Linear(hidden_dims[-1], self._output_dim) self._net = nn.Sequential(*self._layers, self._top_layer) - # network features - self._activation = activation - self._softmax = nn.Softmax(dim=1) if softmax_enabled else None - self._batch_norm_enabled = batch_norm_enabled - self._dropout_p = dropout_p - def forward(self, x): out = self._net(x).double() return self._softmax(out) if self._softmax else out From 27477c8e0ed79a77072a8bc6c733b83cb85f00e2 Mon Sep 17 00:00:00 2001 From: ysqyang Date: Wed, 21 Oct 2020 23:02:28 +0800 Subject: [PATCH 045/337] fixed a bug --- examples/cim/dqn/components/agent_manager.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/examples/cim/dqn/components/agent_manager.py b/examples/cim/dqn/components/agent_manager.py index 4440eabb0..f0512a149 100644 --- a/examples/cim/dqn/components/agent_manager.py +++ b/examples/cim/dqn/components/agent_manager.py @@ -51,6 +51,8 @@ def create_dqn_agents(agent_id_list, mode, config): agent_dict[agent_id] = CIMAgent(name=agent_id, mode=mode, algorithm=algorithm, experience_pool=experience_pool, **config.training_loop_parameters) + return agent_dict + class DQNAgentManager(SimpleAgentManager): def train(self, experiences_by_agent, performance=None): From cbdf8f8b0813bc3ce7f7fb3a477593054f2be89b Mon Sep 17 00:00:00 2001 From: ysqyang Date: Wed, 21 Oct 2020 23:15:11 +0800 Subject: [PATCH 046/337] fixed a bug --- maro/rl/storage/column_based_store.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/maro/rl/storage/column_based_store.py b/maro/rl/storage/column_based_store.py index 21f78c3b6..4888f6e5f 100644 --- a/maro/rl/storage/column_based_store.py +++ b/maro/rl/storage/column_based_store.py @@ -97,7 +97,8 @@ def put(self, contents: dict, overwrite_indexes: Sequence = None) -> List[int]: """ if len(self._store) > 0 and contents.keys() != self._store.keys(): raise StoreMisalignmentError(f"expected keys {list(self._store.keys())}, got {list(contents.keys())}") - added_size = len(contents[next(iter(contents))]) + added = contents[next(iter(contents))] + added_size = len(added) if isinstance(added, list) else 1 if self._capacity < 0: for key, val in contents.items(): if not isinstance(val, list): From 5c24d9759b56f8e8ebf655d14b551b1256448ea8 Mon Sep 17 00:00:00 2001 From: ysqyang Date: Wed, 21 Oct 2020 23:24:01 +0800 Subject: [PATCH 047/337] fixed a bug --- maro/rl/algorithms/dqn.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/maro/rl/algorithms/dqn.py b/maro/rl/algorithms/dqn.py index cfab9c1c1..a27d41e43 100644 --- a/maro/rl/algorithms/dqn.py +++ b/maro/rl/algorithms/dqn.py @@ -105,18 +105,18 @@ def _update_target_model(self): self._hyper_params.tau * eval_params.data + (1 - self._hyper_params.tau) * target_params.data ) - def load_models(self, eval_model): + def load_models(self, model_dict): """Load the eval model from memory.""" - self._model_dict["eval"] = eval_model + self._model_dict = model_dict def dump_models(self): """Return the eval model.""" - return self._model_dict["eval"].state_dict() + return {name: model.state_dict() for name, model in self._model_dict} def load_models_from_file(self, path): """Load the eval model from disk.""" - self._model_dict["eval"] = torch.load(path) + self._model_dict = torch.load(path) def dump_models_to_file(self, path: str): """Dump the eval model to disk.""" - torch.save(self._model_dict["eval"].state_dict(), path) + torch.save(self.dump_models(), path) From 9ea8b04da02cdffd779b5e7c252b1fe271c0e708 Mon Sep 17 00:00:00 2001 From: ysqyang Date: Wed, 21 Oct 2020 23:26:42 +0800 Subject: [PATCH 048/337] fixed a bug --- maro/rl/algorithms/dqn.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/maro/rl/algorithms/dqn.py b/maro/rl/algorithms/dqn.py index a27d41e43..0df8e36bb 100644 --- a/maro/rl/algorithms/dqn.py +++ b/maro/rl/algorithms/dqn.py @@ -105,18 +105,18 @@ def _update_target_model(self): self._hyper_params.tau * eval_params.data + (1 - self._hyper_params.tau) * target_params.data ) - def load_models(self, model_dict): + def load_models(self, eval_model): """Load the eval model from memory.""" - self._model_dict = model_dict + self._model_dict["eval"] = eval_model def dump_models(self): """Return the eval model.""" - return {name: model.state_dict() for name, model in self._model_dict} + return self._model_dict["eval"].state_dict() def load_models_from_file(self, path): """Load the eval model from disk.""" - self._model_dict = torch.load(path) + self._model_dict["eval"] = torch.load(path) def dump_models_to_file(self, path: str): """Dump the eval model to disk.""" - torch.save(self.dump_models(), path) + torch.save(self._model_dict["eval"], path) From 413003fd4dc633f902911564036fd5dd81867619 Mon Sep 17 00:00:00 2001 From: ysqyang Date: Wed, 21 Oct 2020 23:40:48 +0800 Subject: [PATCH 049/337] fixed a bug --- examples/cim/dqn/components/agent_manager.py | 22 ++++---------------- maro/rl/algorithms/dqn.py | 6 +++--- 2 files changed, 7 insertions(+), 21 deletions(-) diff --git a/examples/cim/dqn/components/agent_manager.py b/examples/cim/dqn/components/agent_manager.py index f0512a149..6b31ea348 100644 --- a/examples/cim/dqn/components/agent_manager.py +++ b/examples/cim/dqn/components/agent_manager.py @@ -10,22 +10,8 @@ def create_dqn_agents(agent_id_list, mode, config): + is_trainable = mode in {AgentMode.TRAIN, AgentMode.TRAIN_INFERENCE} num_actions = config.algorithm.num_actions - if mode == AgentMode.INFERENCE: - return {agent_id: DQN( - eval_model=None, - optimizer_cls=None, - optimizer_params=None, - loss_func=None, - hyper_params=DQNHyperParams( - num_actions=num_actions, - reward_decay=None, - num_training_rounds_per_target_replacement=None, - tau=None - ) - ) - for agent_id in agent_id_list} - set_seeds(config.seed) agent_dict = {} for agent_id in agent_id_list: @@ -38,9 +24,9 @@ def create_dqn_agents(agent_id_list, mode, config): algorithm = DQN( eval_model=eval_model, - optimizer_cls=RMSprop, - optimizer_params=config.algorithm.optimizer, - loss_func=nn.functional.smooth_l1_loss, + optimizer_cls=RMSprop if is_trainable else None, + optimizer_params=config.algorithm.optimizer if is_trainable else None, + loss_func=nn.functional.smooth_l1_loss if is_trainable else None, hyper_params=DQNHyperParams( **config.algorithm.hyper_parameters, num_actions=num_actions diff --git a/maro/rl/algorithms/dqn.py b/maro/rl/algorithms/dqn.py index 0df8e36bb..e9731abd6 100644 --- a/maro/rl/algorithms/dqn.py +++ b/maro/rl/algorithms/dqn.py @@ -44,12 +44,12 @@ class DQN(AbsAlgorithm): it is None, the target model will be initialized as a deep copy of the eval model. """ def __init__( - self, eval_model, optimizer_cls, optimizer_params, loss_func, hyper_params: DQNHyperParams, + self, eval_model: nn.Module, optimizer_cls, optimizer_params, loss_func, hyper_params: DQNHyperParams, target_model=None ): super().__init__() self._device = torch.device("cuda" if torch.cuda.is_available() else "cpu") - self._model_dict = {"eval": eval_model.to(self._device) if eval_model is not None else eval_model} + self._model_dict = {"eval": eval_model.to(self._device)} if optimizer_cls is not None: self._optimizer = optimizer_cls(self._model_dict["eval"].parameters(), **optimizer_params) if target_model is None: @@ -107,7 +107,7 @@ def _update_target_model(self): def load_models(self, eval_model): """Load the eval model from memory.""" - self._model_dict["eval"] = eval_model + self._model_dict["eval"].load_state_dict(eval_model) def dump_models(self): """Return the eval model.""" From b6e6a7e1bff6eec8712ef0f597a607d6002642f2 Mon Sep 17 00:00:00 2001 From: ysqyang Date: Wed, 21 Oct 2020 23:55:43 +0800 Subject: [PATCH 050/337] renamed save_models to dump_models --- examples/cim/dqn/dist_learner.py | 2 +- examples/cim/dqn/single_process_launcher.py | 2 +- maro/rl/learner/simple_learner.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/examples/cim/dqn/dist_learner.py b/examples/cim/dqn/dist_learner.py index b5ce56aac..6818317b4 100644 --- a/examples/cim/dqn/dist_learner.py +++ b/examples/cim/dqn/dist_learner.py @@ -40,4 +40,4 @@ ) learner.train(total_episodes=config.general.total_training_episodes) learner.test() - learner.save_models(os.path.join(os.getcwd(), "models")) + learner.dump_models(os.path.join(os.getcwd(), "models")) diff --git a/examples/cim/dqn/single_process_launcher.py b/examples/cim/dqn/single_process_launcher.py index a56a11e89..134f9a97c 100644 --- a/examples/cim/dqn/single_process_launcher.py +++ b/examples/cim/dqn/single_process_launcher.py @@ -61,4 +61,4 @@ ) learner.train(total_episodes=config.general.total_training_episodes) learner.test() - learner.save_models(os.path.join(os.getcwd(), "models")) + learner.dump_models(os.path.join(os.getcwd(), "models")) diff --git a/maro/rl/learner/simple_learner.py b/maro/rl/learner/simple_learner.py index d7a2af6dc..a49a1f422 100644 --- a/maro/rl/learner/simple_learner.py +++ b/maro/rl/learner/simple_learner.py @@ -51,7 +51,7 @@ def test(self): self._logger.info(f"test performance from {actor_id}: {perf}") self._actor.roll_out(done=True) - def save_models(self, model_dump_dir: str): + def dump_models(self, model_dump_dir: str): self._trainable_agents.dump_models_to_files(model_dump_dir) def _is_shared_agent_instance(self): From ad2f92b2f5d94520ef245693d96c7bfd6a7f42fd Mon Sep 17 00:00:00 2001 From: ysqyang Date: Thu, 22 Oct 2020 15:03:54 +0800 Subject: [PATCH 051/337] fixed bugs and issues related to ac and pg --- examples/cim/ac/components/agent_manager.py | 26 +++++-------------- examples/cim/dqn/components/agent_manager.py | 27 ++++++++------------ examples/cim/dqn/components/config.py | 2 +- examples/cim/dqn/config.yml | 4 ++- examples/cim/dqn/dist_learner.py | 2 +- examples/cim/dqn/single_process_launcher.py | 2 +- examples/cim/pg/components/agent_manager.py | 15 +++-------- maro/rl/agent/simple_agent_manager.py | 13 +++++----- maro/rl/algorithms/ac.py | 7 ++--- maro/rl/algorithms/dqn.py | 2 +- maro/rl/algorithms/pg.py | 4 +-- maro/rl/algorithms/ppo.py | 7 ++--- maro/rl/learner/simple_learner.py | 2 +- maro/rl/models/decision_layers.py | 14 +++++----- maro/rl/models/representation_layers.py | 8 +++--- maro/rl/storage/column_based_store.py | 3 ++- 16 files changed, 60 insertions(+), 78 deletions(-) diff --git a/examples/cim/ac/components/agent_manager.py b/examples/cim/ac/components/agent_manager.py index 2819cf4d2..165531d6a 100644 --- a/examples/cim/ac/components/agent_manager.py +++ b/examples/cim/ac/components/agent_manager.py @@ -11,21 +11,9 @@ def create_ac_agents(agent_id_list, mode, config): - if mode in {AgentMode.TRAIN, AgentMode.TRAIN_INFERENCE}: - return {agent_id: ActorCritic( - policy_model=None, - value_model=None, - value_loss_func=None, - policy_optimizer_cls=None, - policy_optimizer_params=None, - value_optimizer_cls=None, - value_optimizer_params=None, - hyper_params=None, - ) - for agent_id in agent_id_list} - - set_seeds(config.seed) + is_trainable = mode in {AgentMode.TRAIN, AgentMode.TRAIN_INFERENCE} num_actions = config.algorithm.num_actions + set_seeds(config.seed) agent_dict = {} for agent_id in agent_id_list: @@ -46,11 +34,11 @@ def create_ac_agents(agent_id_list, mode, config): algorithm = ActorCritic( policy_model=policy_model, value_model=value_model, - value_loss_func=nn.functional.smooth_l1_loss, - policy_optimizer_cls=Adam, - policy_optimizer_params=config.algorithm.policy_optimizer, - value_optimizer_cls=RMSprop, - value_optimizer_params=config.algorithm.value_optimizer, + value_loss_func=nn.functional.smooth_l1_loss if is_trainable else None, + policy_optimizer_cls=Adam if is_trainable else None, + policy_optimizer_params=config.algorithm.policy_optimizer if is_trainable else None, + value_optimizer_cls=RMSprop if is_trainable else None, + value_optimizer_params=config.algorithm.value_optimizer if is_trainable else None, hyper_params=ActorCriticHyperParameters( num_actions=num_actions, **config.algorithm.hyper_parameters, diff --git a/examples/cim/dqn/components/agent_manager.py b/examples/cim/dqn/components/agent_manager.py index 4c09bb40b..6b31ea348 100644 --- a/examples/cim/dqn/components/agent_manager.py +++ b/examples/cim/dqn/components/agent_manager.py @@ -1,7 +1,7 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT license. -from torch.nn.functional import smooth_l1_loss +import torch.nn as nn from torch.optim import RMSprop from .agent import CIMAgent @@ -10,32 +10,23 @@ def create_dqn_agents(agent_id_list, mode, config): - if mode in {AgentMode.TRAIN, AgentMode.TRAIN_INFERENCE}: - return {agent_id: DQN( - eval_model=None, - optimizer_cls=None, - optimizer_params=None, - loss_func=None, - hyper_params=None - ) - for agent_id in agent_id_list} - - set_seeds(config.seed) + is_trainable = mode in {AgentMode.TRAIN, AgentMode.TRAIN_INFERENCE} num_actions = config.algorithm.num_actions + set_seeds(config.seed) agent_dict = {} for agent_id in agent_id_list: eval_model = LearningModel( decision_layers=DecisionLayers( - name=f'{agent_id}.policy', input_dim=config.algorithm.input_dim, - output_dim=num_actions, **config.algorithm.model + name=f'{agent_id}.policy', input_dim=config.algorithm.input_dim, output_dim=num_actions, + activation=nn.LeakyReLU, **config.algorithm.model ) ) algorithm = DQN( eval_model=eval_model, - optimizer_cls=RMSprop, - optimizer_params=config.algorithm.optimizer, - loss_func=smooth_l1_loss, + optimizer_cls=RMSprop if is_trainable else None, + optimizer_params=config.algorithm.optimizer if is_trainable else None, + loss_func=nn.functional.smooth_l1_loss if is_trainable else None, hyper_params=DQNHyperParams( **config.algorithm.hyper_parameters, num_actions=num_actions @@ -46,6 +37,8 @@ def create_dqn_agents(agent_id_list, mode, config): agent_dict[agent_id] = CIMAgent(name=agent_id, mode=mode, algorithm=algorithm, experience_pool=experience_pool, **config.training_loop_parameters) + return agent_dict + class DQNAgentManager(SimpleAgentManager): def train(self, experiences_by_agent, performance=None): diff --git a/examples/cim/dqn/components/config.py b/examples/cim/dqn/components/config.py index 26a3a8c43..aaebb3ff9 100644 --- a/examples/cim/dqn/components/config.py +++ b/examples/cim/dqn/components/config.py @@ -18,7 +18,7 @@ # obtain model input dimension from state shaping configurations look_back = config["state_shaping"]["look_back"] -max_ports_downstream = config["state_shaping"]["max_port_downstream"] +max_ports_downstream = config["state_shaping"]["max_ports_downstream"] num_port_attributes = len(config["state_shaping"]["port_attributes"]) num_vessel_attributes = len(config["state_shaping"]["vessel_attributes"]) diff --git a/examples/cim/dqn/config.yml b/examples/cim/dqn/config.yml index 4836c45e9..2f435ae52 100644 --- a/examples/cim/dqn/config.yml +++ b/examples/cim/dqn/config.yml @@ -41,6 +41,8 @@ agents: - 256 - 128 - 64 + softmax_enabled: false + batch_norm_enabled: true dropout_p: 0.0 optimizer: lr: 0.05 @@ -63,4 +65,4 @@ distributed: peer: {"actor_worker": 1} redis: host_name: "localhost" - port: 6379 \ No newline at end of file + port: 6379 diff --git a/examples/cim/dqn/dist_learner.py b/examples/cim/dqn/dist_learner.py index b5ce56aac..6818317b4 100644 --- a/examples/cim/dqn/dist_learner.py +++ b/examples/cim/dqn/dist_learner.py @@ -40,4 +40,4 @@ ) learner.train(total_episodes=config.general.total_training_episodes) learner.test() - learner.save_models(os.path.join(os.getcwd(), "models")) + learner.dump_models(os.path.join(os.getcwd(), "models")) diff --git a/examples/cim/dqn/single_process_launcher.py b/examples/cim/dqn/single_process_launcher.py index a56a11e89..134f9a97c 100644 --- a/examples/cim/dqn/single_process_launcher.py +++ b/examples/cim/dqn/single_process_launcher.py @@ -61,4 +61,4 @@ ) learner.train(total_episodes=config.general.total_training_episodes) learner.test() - learner.save_models(os.path.join(os.getcwd(), "models")) + learner.dump_models(os.path.join(os.getcwd(), "models")) diff --git a/examples/cim/pg/components/agent_manager.py b/examples/cim/pg/components/agent_manager.py index 1ee29374b..e41abd907 100644 --- a/examples/cim/pg/components/agent_manager.py +++ b/examples/cim/pg/components/agent_manager.py @@ -11,16 +11,9 @@ def create_pg_agents(agent_id_list, mode, config): - if mode in {AgentMode.TRAIN, AgentMode.TRAIN_INFERENCE}: - return {agent_id: PolicyGradient( - policy_model=None, - optimizer_cls=None, - optimizer_params=None, - hyper_params=None, - ) - for agent_id in agent_id_list} - set_seeds(config.seed) + is_trainable = mode in {AgentMode.TRAIN, AgentMode.TRAIN_INFERENCE} num_actions = config.algorithm.num_actions + set_seeds(config.seed) agent_dict = {} for agent_id in agent_id_list: policy_model = LearningModel( @@ -32,8 +25,8 @@ def create_pg_agents(agent_id_list, mode, config): algorithm = PolicyGradient( policy_model=policy_model, - optimizer_cls=Adam, - optimizer_params=config.algorithm.optimizer, + optimizer_cls=Adam if is_trainable else None, + optimizer_params=config.algorithm.optimizer if is_trainable else None, hyper_params=PolicyGradientHyperParameters( num_actions=num_actions, **config.algorithm.hyper_parameters, diff --git a/maro/rl/agent/simple_agent_manager.py b/maro/rl/agent/simple_agent_manager.py index 6f7237aeb..ef1ca5bd8 100644 --- a/maro/rl/agent/simple_agent_manager.py +++ b/maro/rl/agent/simple_agent_manager.py @@ -42,12 +42,13 @@ def choose_action(self, decision_event, snapshot_list): model_action = self.agent_dict[agent_id].choose_action( model_state, self._explorer.epsilon[agent_id] if self._explorer else None ) - - self._transition_cache = {"state": model_state, - "action": model_action, - "reward": None, - "agent_id": agent_id, - "event": decision_event} + self._transition_cache = { + "state": model_state, + "action": model_action, + "reward": None, + "agent_id": agent_id, + "event": decision_event + } return self._action_shaper(model_action, decision_event, snapshot_list) def on_env_feedback(self, metrics): diff --git a/maro/rl/algorithms/ac.py b/maro/rl/algorithms/ac.py index 79d06957b..7a8bebc24 100644 --- a/maro/rl/algorithms/ac.py +++ b/maro/rl/algorithms/ac.py @@ -121,7 +121,8 @@ def train(self, states: np.ndarray, actions: np.ndarray, rewards: np.ndarray): self._value_optimizer.step() def load_models(self, model_dict): - self._model_dict = model_dict + for name, model in self._model_dict.items(): + model.load_state_dict(model_dict[name]) def dump_models(self): return {name: model.state_dict() for name, model in self._model_dict.items()} @@ -219,10 +220,10 @@ def train(self, states: np.ndarray, actions: np.ndarray, rewards: np.ndarray): self._optimizer.step() def load_models(self, policy_value_model): - self._policy_value_model = policy_value_model + self._policy_value_model.load_state_dict(policy_value_model) def dump_models(self): - return self._policy_value_model + return self._policy_value_model.state_dict() def load_models_from_file(self, path): self._policy_value_model = torch.load(path) diff --git a/maro/rl/algorithms/dqn.py b/maro/rl/algorithms/dqn.py index 24161c8e1..e21bff123 100644 --- a/maro/rl/algorithms/dqn.py +++ b/maro/rl/algorithms/dqn.py @@ -107,7 +107,7 @@ def _update_target_model(self): def load_models(self, eval_model): """Load the eval model from memory.""" - self._model_dict["eval"] = eval_model + self._model_dict["eval"].load_state_dict(eval_model) def dump_models(self): """Return the eval model.""" diff --git a/maro/rl/algorithms/pg.py b/maro/rl/algorithms/pg.py index 19855f003..6e5c623d5 100644 --- a/maro/rl/algorithms/pg.py +++ b/maro/rl/algorithms/pg.py @@ -68,10 +68,10 @@ def train(self, states: np.ndarray, actions: np.ndarray, returns: np.ndarray): self._policy_optimizer.step() def load_models(self, policy_model): - self._policy_model = policy_model + self._policy_model.load_state_dict(policy_model) def dump_models(self): - return self._policy_model + return self._policy_model.state_dict() def load_models_from_file(self, path): """Load trainable models from disk.""" diff --git a/maro/rl/algorithms/ppo.py b/maro/rl/algorithms/ppo.py index 20eb309dc..2386ffeec 100644 --- a/maro/rl/algorithms/ppo.py +++ b/maro/rl/algorithms/ppo.py @@ -123,7 +123,8 @@ def train( self._value_optimizer.step() def load_models(self, model_dict): - self._model_dict = model_dict + for name, model in self._model_dict.items(): + model.load_state_dict(model_dict[name]) def dump_models(self): return {name: model.state_dict() for name, model in self._model_dict.items()} @@ -231,10 +232,10 @@ def train( self._optimizer.step() def load_models(self, policy_value_model): - self._policy_value_model = policy_value_model + self._policy_value_model.load_state_dict(policy_value_model) def dump_models(self): - return self._policy_value_model + return self._policy_value_model.state_dict() def load_models_from_file(self, path): self._policy_value_model = torch.load(path) diff --git a/maro/rl/learner/simple_learner.py b/maro/rl/learner/simple_learner.py index d7a2af6dc..a49a1f422 100644 --- a/maro/rl/learner/simple_learner.py +++ b/maro/rl/learner/simple_learner.py @@ -51,7 +51,7 @@ def test(self): self._logger.info(f"test performance from {actor_id}: {perf}") self._actor.roll_out(done=True) - def save_models(self, model_dump_dir: str): + def dump_models(self, model_dump_dir: str): self._trainable_agents.dump_models_to_files(model_dump_dir) def _is_shared_agent_instance(self): diff --git a/maro/rl/models/decision_layers.py b/maro/rl/models/decision_layers.py index 23c9e4bed..c4f4cbdd9 100644 --- a/maro/rl/models/decision_layers.py +++ b/maro/rl/models/decision_layers.py @@ -23,7 +23,7 @@ class DecisionLayers(nn.Module): dropout_p (float): Dropout probability. Defaults to None, in which case there is no drop-out. """ def __init__( - self, *, name: str, input_dim: int, output_dim: int, hidden_dims: [int], activation=nn.LeakyReLU, + self, name: str, input_dim: int, output_dim: int, hidden_dims: [int], activation=nn.LeakyReLU, softmax_enabled: bool = False, batch_norm_enabled: bool = False, dropout_p: float = None ): super().__init__() @@ -32,6 +32,12 @@ def __init__( self._hidden_dims = hidden_dims if hidden_dims is not None else [] self._output_dim = output_dim + # network features + self._activation = activation + self._softmax = nn.Softmax(dim=1) if softmax_enabled else None + self._batch_norm_enabled = batch_norm_enabled + self._dropout_p = dropout_p + # build the net self._layers = self._build_layers([input_dim] + self._hidden_dims) if len(self._hidden_dims) == 0: @@ -40,12 +46,6 @@ def __init__( self._top_layer = nn.Linear(hidden_dims[-1], self._output_dim) self._net = nn.Sequential(*self._layers, self._top_layer) - # network features - self._activation = activation - self._softmax = nn.Softmax(dim=1) if softmax_enabled else None - self._batch_norm_enabled = batch_norm_enabled - self._dropout_p = dropout_p - def forward(self, x): out = self._net(x).double() return self._softmax(out) if self._softmax else out diff --git a/maro/rl/models/representation_layers.py b/maro/rl/models/representation_layers.py index 6c7408590..3e7812ce2 100644 --- a/maro/rl/models/representation_layers.py +++ b/maro/rl/models/representation_layers.py @@ -50,9 +50,11 @@ def _build_basic_layer(self, input_dim, output_dim): BN -> Linear -> LeakyReLU -> Dropout """ - return nn.Sequential(nn.Linear(input_dim, output_dim), - nn.LeakyReLU(), - nn.Dropout(p=self._dropout_p)) + return nn.Sequential( + nn.Linear(input_dim, output_dim), + nn.LeakyReLU(), + nn.Dropout(p=self._dropout_p) + ) def _build_layers(self, layer_dims: []): """Build multi basic layer. diff --git a/maro/rl/storage/column_based_store.py b/maro/rl/storage/column_based_store.py index 21f78c3b6..4888f6e5f 100644 --- a/maro/rl/storage/column_based_store.py +++ b/maro/rl/storage/column_based_store.py @@ -97,7 +97,8 @@ def put(self, contents: dict, overwrite_indexes: Sequence = None) -> List[int]: """ if len(self._store) > 0 and contents.keys() != self._store.keys(): raise StoreMisalignmentError(f"expected keys {list(self._store.keys())}, got {list(contents.keys())}") - added_size = len(contents[next(iter(contents))]) + added = contents[next(iter(contents))] + added_size = len(added) if isinstance(added, list) else 1 if self._capacity < 0: for key, val in contents.items(): if not isinstance(val, list): From a7f6c7f42acbdb0b0b38032a78865e022fe09815 Mon Sep 17 00:00:00 2001 From: ysqyang Date: Thu, 22 Oct 2020 15:05:31 +0800 Subject: [PATCH 052/337] 1. set default batch_norm_enabled to True; 2. used state_dict in dqn model saving --- examples/cim/dqn/config.yml | 2 +- maro/rl/algorithms/dqn.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/cim/dqn/config.yml b/examples/cim/dqn/config.yml index 488e9b1a8..2f435ae52 100644 --- a/examples/cim/dqn/config.yml +++ b/examples/cim/dqn/config.yml @@ -42,7 +42,7 @@ agents: - 128 - 64 softmax_enabled: false - batch_norm_enabled: false + batch_norm_enabled: true dropout_p: 0.0 optimizer: lr: 0.05 diff --git a/maro/rl/algorithms/dqn.py b/maro/rl/algorithms/dqn.py index e9731abd6..066fb661c 100644 --- a/maro/rl/algorithms/dqn.py +++ b/maro/rl/algorithms/dqn.py @@ -119,4 +119,4 @@ def load_models_from_file(self, path): def dump_models_to_file(self, path: str): """Dump the eval model to disk.""" - torch.save(self._model_dict["eval"], path) + torch.save(self._model_dict["eval"].state_dict(), path) From 93d7f2573fe43c77f1d73152b2c40d9a506fc85d Mon Sep 17 00:00:00 2001 From: ysqyang Date: Thu, 22 Oct 2020 15:19:10 +0800 Subject: [PATCH 053/337] fixed a typo --- examples/cim/ac/components/config.py | 2 +- examples/cim/pg/components/config.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/cim/ac/components/config.py b/examples/cim/ac/components/config.py index 26a3a8c43..aaebb3ff9 100644 --- a/examples/cim/ac/components/config.py +++ b/examples/cim/ac/components/config.py @@ -18,7 +18,7 @@ # obtain model input dimension from state shaping configurations look_back = config["state_shaping"]["look_back"] -max_ports_downstream = config["state_shaping"]["max_port_downstream"] +max_ports_downstream = config["state_shaping"]["max_ports_downstream"] num_port_attributes = len(config["state_shaping"]["port_attributes"]) num_vessel_attributes = len(config["state_shaping"]["vessel_attributes"]) diff --git a/examples/cim/pg/components/config.py b/examples/cim/pg/components/config.py index 26a3a8c43..aaebb3ff9 100644 --- a/examples/cim/pg/components/config.py +++ b/examples/cim/pg/components/config.py @@ -18,7 +18,7 @@ # obtain model input dimension from state shaping configurations look_back = config["state_shaping"]["look_back"] -max_ports_downstream = config["state_shaping"]["max_port_downstream"] +max_ports_downstream = config["state_shaping"]["max_ports_downstream"] num_port_attributes = len(config["state_shaping"]["port_attributes"]) num_vessel_attributes = len(config["state_shaping"]["vessel_attributes"]) From 0658303f3adc8e19631efdc1507117db05aea444 Mon Sep 17 00:00:00 2001 From: ysqyang Date: Thu, 22 Oct 2020 15:20:25 +0800 Subject: [PATCH 054/337] fixed a typo --- examples/cim/ac/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/cim/ac/config.yml b/examples/cim/ac/config.yml index da9dd1ee6..0236d6182 100644 --- a/examples/cim/ac/config.yml +++ b/examples/cim/ac/config.yml @@ -50,7 +50,7 @@ agents: policy_train_iters: 1 value_train_iters: 10 k: -1 - lamb: 1.0 + lam: 1.0 seed: 1024 # for reproducibility distributed: group_name: "ac_distributed_test" From abc8e7a4742787e11a91937ac291b6d562d52f0e Mon Sep 17 00:00:00 2001 From: ysqyang Date: Thu, 22 Oct 2020 15:26:20 +0800 Subject: [PATCH 055/337] changed default lam and k values for cim ac --- examples/cim/ac/config.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/cim/ac/config.yml b/examples/cim/ac/config.yml index 0236d6182..09f4184f0 100644 --- a/examples/cim/ac/config.yml +++ b/examples/cim/ac/config.yml @@ -49,8 +49,8 @@ agents: reward_decay: .0 policy_train_iters: 1 value_train_iters: 10 - k: -1 - lam: 1.0 + k: 1 + lam: 0.0 seed: 1024 # for reproducibility distributed: group_name: "ac_distributed_test" From 86f421397486590773b6f13cd613f878a16b723d Mon Sep 17 00:00:00 2001 From: ysqyang Date: Thu, 22 Oct 2020 16:07:55 +0800 Subject: [PATCH 056/337] removed an unwanted import --- examples/cim/pg/components/agent_manager.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/cim/pg/components/agent_manager.py b/examples/cim/pg/components/agent_manager.py index e41abd907..596f87198 100644 --- a/examples/cim/pg/components/agent_manager.py +++ b/examples/cim/pg/components/agent_manager.py @@ -2,7 +2,7 @@ # Licensed under the MIT license. import torch.nn as nn -from torch.optim import Adam, RMSprop +from torch.optim import Adam from .agent import CIMAgent from maro.rl import AgentMode, SimpleAgentManager, LearningModel, DecisionLayers, PolicyGradient, \ From 3c7a25b532eb8c5a1eb9fd0d220e5e96ade9fa7c Mon Sep 17 00:00:00 2001 From: ysqyang Date: Thu, 22 Oct 2020 17:02:40 +0800 Subject: [PATCH 057/337] renamed dump_experience_store to dump_experience_pool --- maro/rl/agent/abs_agent.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/maro/rl/agent/abs_agent.py b/maro/rl/agent/abs_agent.py index f6fb7ffc6..d5439127e 100644 --- a/maro/rl/agent/abs_agent.py +++ b/maro/rl/agent/abs_agent.py @@ -105,7 +105,7 @@ def dump_models_to_file(self, dir_path: str): """ self._algorithm.dump_models_to_file(os.path.join(dir_path, self._name)) - def dump_experience_store(self, dir_path: str): + def dump_experience_pool(self, dir_path: str): """Dump the experience pool to disk.""" if self._experience_pool is not None: with open(os.path.join(dir_path, self._name)) as fp: From ecb411a9dce992631c0a20d23b470a68db06db5a Mon Sep 17 00:00:00 2001 From: ysqyang Date: Thu, 22 Oct 2020 17:04:03 +0800 Subject: [PATCH 058/337] renamed dump_experience_store to dump_experience_pool and fixed a bug in the method --- maro/rl/agent/abs_agent.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/maro/rl/agent/abs_agent.py b/maro/rl/agent/abs_agent.py index f6fb7ffc6..1d5b96648 100644 --- a/maro/rl/agent/abs_agent.py +++ b/maro/rl/agent/abs_agent.py @@ -105,10 +105,11 @@ def dump_models_to_file(self, dir_path: str): """ self._algorithm.dump_models_to_file(os.path.join(dir_path, self._name)) - def dump_experience_store(self, dir_path: str): + def dump_experience_pool(self, dir_path: str): """Dump the experience pool to disk.""" if self._experience_pool is not None: - with open(os.path.join(dir_path, self._name)) as fp: + os.makedirs(dir_path, exist_ok=True) + with open(os.path.join(dir_path, self._name), "wb") as fp: pickle.dump(self._experience_pool, fp) def _assert_train_mode(self): From 5b866e8bee7495a46f8fae2178a31c934629c671 Mon Sep 17 00:00:00 2001 From: ysqyang Date: Thu, 22 Oct 2020 17:04:54 +0800 Subject: [PATCH 059/337] fixed a bug in the dump_experience_pool method --- maro/rl/agent/abs_agent.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/maro/rl/agent/abs_agent.py b/maro/rl/agent/abs_agent.py index d5439127e..1d5b96648 100644 --- a/maro/rl/agent/abs_agent.py +++ b/maro/rl/agent/abs_agent.py @@ -108,7 +108,8 @@ def dump_models_to_file(self, dir_path: str): def dump_experience_pool(self, dir_path: str): """Dump the experience pool to disk.""" if self._experience_pool is not None: - with open(os.path.join(dir_path, self._name)) as fp: + os.makedirs(dir_path, exist_ok=True) + with open(os.path.join(dir_path, self._name), "wb") as fp: pickle.dump(self._experience_pool, fp) def _assert_train_mode(self): From 864f5873aea89e470d9df3f3d19c4988a61f45ee Mon Sep 17 00:00:00 2001 From: ysqyang Date: Thu, 22 Oct 2020 17:06:46 +0800 Subject: [PATCH 060/337] renamed save_models to dump_models --- examples/cim/ac/dist_learner.py | 2 +- examples/cim/ac/single_process_launcher.py | 2 +- examples/cim/pg/dist_learner.py | 2 +- examples/cim/pg/single_process_launcher.py | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/examples/cim/ac/dist_learner.py b/examples/cim/ac/dist_learner.py index d14b6b697..2ce5e0a2f 100644 --- a/examples/cim/ac/dist_learner.py +++ b/examples/cim/ac/dist_learner.py @@ -32,4 +32,4 @@ ) learner.train(total_episodes=config.general.total_training_episodes) learner.test() - learner.save_models(os.path.join(os.getcwd(), "models")) + learner.dump_models(os.path.join(os.getcwd(), "models")) diff --git a/examples/cim/ac/single_process_launcher.py b/examples/cim/ac/single_process_launcher.py index c812cb8e5..005c65ac2 100644 --- a/examples/cim/ac/single_process_launcher.py +++ b/examples/cim/ac/single_process_launcher.py @@ -45,4 +45,4 @@ ) learner.train(total_episodes=config.general.total_training_episodes) learner.test() - learner.save_models(os.path.join(os.getcwd(), "models")) + learner.dump_models(os.path.join(os.getcwd(), "models")) diff --git a/examples/cim/pg/dist_learner.py b/examples/cim/pg/dist_learner.py index 5c8fb6914..f4b7f5dcf 100644 --- a/examples/cim/pg/dist_learner.py +++ b/examples/cim/pg/dist_learner.py @@ -39,4 +39,4 @@ ) learner.train(total_episodes=config.general.total_training_episodes) learner.test() - learner.save_models(os.path.join(os.getcwd(), "models")) + learner.dump_models(os.path.join(os.getcwd(), "models")) diff --git a/examples/cim/pg/single_process_launcher.py b/examples/cim/pg/single_process_launcher.py index ee8f8f56d..9259c64e5 100644 --- a/examples/cim/pg/single_process_launcher.py +++ b/examples/cim/pg/single_process_launcher.py @@ -45,4 +45,4 @@ ) learner.train(total_episodes=config.general.total_training_episodes) learner.test() - learner.save_models(os.path.join(os.getcwd(), "models")) + learner.dump_models(os.path.join(os.getcwd(), "models")) From f0ef4b35ddda7b90dfcf50b244e3cc87db536d1e Mon Sep 17 00:00:00 2001 From: ysqyang Date: Thu, 22 Oct 2020 21:00:58 +0800 Subject: [PATCH 061/337] restored original k-step shaper for performance considerations --- maro/rl/shaping/k_step_experience_shaper.py | 63 +++++++++------------ 1 file changed, 28 insertions(+), 35 deletions(-) diff --git a/maro/rl/shaping/k_step_experience_shaper.py b/maro/rl/shaping/k_step_experience_shaper.py index 4afaacdd6..1ffa65722 100644 --- a/maro/rl/shaping/k_step_experience_shaper.py +++ b/maro/rl/shaping/k_step_experience_shaper.py @@ -1,19 +1,18 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT license. +from collections import defaultdict, deque from enum import Enum from typing import Callable -import numpy as np - from .experience_shaper import ExperienceShaper -from maro.rl.utils.trajectory_utils import get_k_step_returns class KStepExperienceKeys(Enum): STATE = "state" ACTION = "action" REWARD = "reward" + RETURN = "return" NEXT_STATE = "next_state" NEXT_ACTION = "next_action" DISCOUNT = "discount" @@ -31,38 +30,32 @@ class KStepExperienceShaper(ExperienceShaper): def __init__(self, reward_func: Callable, reward_decay: float, steps: int, is_per_agent: bool = True): super().__init__(reward_func) self._reward_decay = reward_decay - self._num_steps = steps + self._steps = steps self._is_per_agent = is_per_agent def __call__(self, trajectory, snapshot_list): - length = len(trajectory) - agent_ids = np.asarray(trajectory.get_by_key["agent_id"]) - states = np.asarray(trajectory.get_by_key["state"]) - actions = np.asarray(trajectory.get_by_key["action"]) - reward_array = np.fromiter(map(self._reward_func, trajectory.get_by_key("metrics")), dtype=np.float32) - reward_sums = get_k_step_returns(reward_array, self._reward_decay, k=self._num_steps) - discounts = np.array([self._reward_decay ** min(self._num_steps, length - i - 1) for i in range(length - 1)]) - next_states = np.pad(states[self._num_steps:], (0, length - self._num_steps - 1), mode="edge") - next_actions = np.pad(actions[self._num_steps:], (0, length - self._num_steps - 1), mode="edge") - - states, actions = states[:-1], actions[:-1] - - if self._is_per_agent: - return {agent_id: { - KStepExperienceKeys.STATE.value: states[agent_ids == agent_id], - KStepExperienceKeys.ACTION.value: actions[agent_ids == agent_id], - KStepExperienceKeys.REWARD.value: reward_sums[agent_ids == agent_id], - KStepExperienceKeys.NEXT_STATE.value: next_states[agent_ids == agent_id], - KStepExperienceKeys.NEXT_ACTION.value: next_actions[agent_ids == agent_id], - KStepExperienceKeys.DISCOUNT.value: discounts[agent_ids == agent_id]} - for agent_id in set(agent_ids) - } - else: - return { - KStepExperienceKeys.STATE.value: states, - KStepExperienceKeys.ACTION.value: actions, - KStepExperienceKeys.REWARD.value: reward_sums, - KStepExperienceKeys.NEXT_STATE.value: next_states, - KStepExperienceKeys.NEXT_ACTION.value: next_actions, - KStepExperienceKeys.DISCOUNT.value: discounts - } + experiences = defaultdict(lambda: defaultdict(deque)) if self._is_per_agent else defaultdict(deque) + reward_list = deque() + full_return = partial_return = 0 + for i in range(len(trajectory) - 2, -1, -1): + transition = trajectory[i] + next_transition = trajectory[min(len(trajectory) - 1, i + self._steps)] + reward_list.appendleft(self._reward_func(trajectory[i]["metrics"])) + # compute the full return + full_return = full_return * self._reward_decay + reward_list[0] + # compute the partial return + partial_return = partial_return * self._reward_decay + reward_list[0] + if len(reward_list) > self._steps: + partial_return -= reward_list.pop() * self._reward_decay ** (self._steps - 1) + agent_exp = experiences[transition["agent_id"]] if self._is_per_agent else experiences + agent_exp[KStepExperienceKeys.STATE.value].appendleft(transition["state"]) + agent_exp[KStepExperienceKeys.ACTION.value].appendleft(transition["action"]) + agent_exp[KStepExperienceKeys.REWARD.value].appendleft(partial_return) + agent_exp[KStepExperienceKeys.RETURN.value].appendleft(full_return) + agent_exp[KStepExperienceKeys.NEXT_STATE.value].appendleft(next_transition["state"]) + agent_exp[KStepExperienceKeys.NEXT_ACTION.value].appendleft(next_transition["action"]) + agent_exp[KStepExperienceKeys.DISCOUNT.value].appendleft( + self._reward_decay ** (min(self._steps, len(trajectory) - 1 - i)) + ) + + return dict(experiences) From 479f1ad24485c516f682649e13a6c771c4779223 Mon Sep 17 00:00:00 2001 From: ysqyang Date: Fri, 23 Oct 2020 10:14:12 +0800 Subject: [PATCH 062/337] fixed a naming bug --- maro/rl/algorithms/pg.py | 6 +++--- maro/rl/shaping/k_step_experience_shaper.py | 14 +++++++------- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/maro/rl/algorithms/pg.py b/maro/rl/algorithms/pg.py index 6e5c623d5..7e6c421b5 100644 --- a/maro/rl/algorithms/pg.py +++ b/maro/rl/algorithms/pg.py @@ -42,7 +42,7 @@ def __init__( self._device = torch.device("cuda" if torch.cuda.is_available() else "cpu") self._policy_model = policy_model.to(self._device) if optimizer_cls is not None: - self._policy_optimizer = optimizer_cls(self._policy_model.parameters(), **optimizer_params) + self._optimizer = optimizer_cls(self._policy_model.parameters(), **optimizer_params) self._hyper_params = hyper_params @property @@ -63,9 +63,9 @@ def train(self, states: np.ndarray, actions: np.ndarray, returns: np.ndarray): returns = torch.from_numpy(returns).to(self._device) action_prob = self._policy_model(states).gather(1, actions.unsqueeze(1)).squeeze() # (N, 1) policy_loss = -(torch.log(action_prob) * returns).mean() - self._policy_optimizer.zero_grad() + self._optimizer.zero_grad() policy_loss.backward() - self._policy_optimizer.step() + self._optimizer.step() def load_models(self, policy_model): self._policy_model.load_state_dict(policy_model) diff --git a/maro/rl/shaping/k_step_experience_shaper.py b/maro/rl/shaping/k_step_experience_shaper.py index 1ffa65722..fbf892b5e 100644 --- a/maro/rl/shaping/k_step_experience_shaper.py +++ b/maro/rl/shaping/k_step_experience_shaper.py @@ -24,13 +24,13 @@ class KStepExperienceShaper(ExperienceShaper): Args: reward_func (Callable): a function used to compute immediate rewards from metrics given by the env. reward_decay (float): decay factor used to evaluate multi-step returns. - steps (int): number of time steps used in computing returns + num_steps (int): number of time steps used in computing returns is_per_agent (bool): if True, the generated experiences will be bucketed by agent ID. """ - def __init__(self, reward_func: Callable, reward_decay: float, steps: int, is_per_agent: bool = True): + def __init__(self, reward_func: Callable, reward_decay: float, num_steps: int, is_per_agent: bool = True): super().__init__(reward_func) self._reward_decay = reward_decay - self._steps = steps + self._num_steps = num_steps self._is_per_agent = is_per_agent def __call__(self, trajectory, snapshot_list): @@ -39,14 +39,14 @@ def __call__(self, trajectory, snapshot_list): full_return = partial_return = 0 for i in range(len(trajectory) - 2, -1, -1): transition = trajectory[i] - next_transition = trajectory[min(len(trajectory) - 1, i + self._steps)] + next_transition = trajectory[min(len(trajectory) - 1, i + self._num_steps)] reward_list.appendleft(self._reward_func(trajectory[i]["metrics"])) # compute the full return full_return = full_return * self._reward_decay + reward_list[0] # compute the partial return partial_return = partial_return * self._reward_decay + reward_list[0] - if len(reward_list) > self._steps: - partial_return -= reward_list.pop() * self._reward_decay ** (self._steps - 1) + if len(reward_list) > self._num_steps: + partial_return -= reward_list.pop() * self._reward_decay ** (self._num_steps - 1) agent_exp = experiences[transition["agent_id"]] if self._is_per_agent else experiences agent_exp[KStepExperienceKeys.STATE.value].appendleft(transition["state"]) agent_exp[KStepExperienceKeys.ACTION.value].appendleft(transition["action"]) @@ -55,7 +55,7 @@ def __call__(self, trajectory, snapshot_list): agent_exp[KStepExperienceKeys.NEXT_STATE.value].appendleft(next_transition["state"]) agent_exp[KStepExperienceKeys.NEXT_ACTION.value].appendleft(next_transition["action"]) agent_exp[KStepExperienceKeys.DISCOUNT.value].appendleft( - self._reward_decay ** (min(self._steps, len(trajectory) - 1 - i)) + self._reward_decay ** (min(self._num_steps, len(trajectory) - 1 - i)) ) return dict(experiences) From 70bbb52032147ca2eb30f6cdfa6716087c94af47 Mon Sep 17 00:00:00 2001 From: ysqyang Date: Tue, 27 Oct 2020 11:24:36 +0800 Subject: [PATCH 063/337] fixed some PR comments --- examples/cim/dqn/components/agent_manager.py | 4 +++- examples/cim/dqn/dist_actor.py | 4 ++-- examples/cim/dqn/dist_learner.py | 2 +- examples/cim/dqn/single_process_launcher.py | 6 +++--- maro/rl/agent/abs_agent.py | 4 ++-- maro/rl/agent/abs_agent_manager.py | 12 +++--------- maro/rl/agent/simple_agent_manager.py | 8 +++++--- maro/rl/algorithms/dqn.py | 10 +++++++--- maro/rl/models/decision_layers.py | 2 +- maro/rl/models/learning_model.py | 4 +++- 10 files changed, 30 insertions(+), 26 deletions(-) diff --git a/examples/cim/dqn/components/agent_manager.py b/examples/cim/dqn/components/agent_manager.py index 6b31ea348..4224755f2 100644 --- a/examples/cim/dqn/components/agent_manager.py +++ b/examples/cim/dqn/components/agent_manager.py @@ -17,7 +17,9 @@ def create_dqn_agents(agent_id_list, mode, config): for agent_id in agent_id_list: eval_model = LearningModel( decision_layers=DecisionLayers( - name=f'{agent_id}.policy', input_dim=config.algorithm.input_dim, output_dim=num_actions, + name=f'{agent_id}.policy', + input_dim=config.algorithm.input_dim, + output_dim=num_actions, activation=nn.LeakyReLU, **config.algorithm.model ) ) diff --git a/examples/cim/dqn/dist_actor.py b/examples/cim/dqn/dist_actor.py index 548efa622..5767ed531 100644 --- a/examples/cim/dqn/dist_actor.py +++ b/examples/cim/dqn/dist_actor.py @@ -22,7 +22,7 @@ experience_shaper = TruncatedExperienceShaper(**config.experience_shaping.truncated) else: experience_shaper = KStepExperienceShaper( - reward_func=lambda mt: 1-mt["container_shortage"]/mt["order_requirements"], + reward_func=lambda mt: 1 - mt["container_shortage"]/mt["order_requirements"], **config.experience_shaping.k_step ) @@ -33,7 +33,7 @@ } explorer = TwoPhaseLinearExplorer(agent_id_list, config.general.total_training_episodes, **exploration_config) agent_manager = DQNAgentManager( - name="cim_remote_actor", + name="distributed_cim_actor", mode=AgentManagerMode.INFERENCE, agent_dict=create_dqn_agents(agent_id_list, AgentMode.INFERENCE, config.agents), state_shaper=state_shaper, diff --git a/examples/cim/dqn/dist_learner.py b/examples/cim/dqn/dist_learner.py index 6818317b4..1f70beb79 100644 --- a/examples/cim/dqn/dist_learner.py +++ b/examples/cim/dqn/dist_learner.py @@ -21,7 +21,7 @@ } explorer = TwoPhaseLinearExplorer(agent_id_list, config.general.total_training_episodes, **exploration_config) agent_manager = DQNAgentManager( - name="cim_remote_learner", + name="distributed_cim_learner", mode=AgentManagerMode.TRAIN, agent_dict=create_dqn_agents(agent_id_list, AgentMode.TRAIN, config.agents), explorer=explorer diff --git a/examples/cim/dqn/single_process_launcher.py b/examples/cim/dqn/single_process_launcher.py index 134f9a97c..d2ee1ac88 100644 --- a/examples/cim/dqn/single_process_launcher.py +++ b/examples/cim/dqn/single_process_launcher.py @@ -19,11 +19,11 @@ if __name__ == "__main__": - # Step 1: initialize a CIM environment for using a toy dataset. + # Step 1: Initialize a CIM environment for using a toy dataset. env = Env(config.env.scenario, config.env.topology, durations=config.env.durations) agent_id_list = [str(agent_id) for agent_id in env.agent_idx_list] - # Step 2: create state, action and experience shapers. We also need to create an explorer here due to the + # Step 2: Create state, action and experience shapers. We also need to create an explorer here due to the # greedy nature of the DQN algorithm. state_shaper = CIMStateShaper(**config.state_shaping) action_shaper = CIMActionShaper(action_space=list(np.linspace(-1.0, 1.0, config.agents.algorithm.num_actions))) @@ -42,7 +42,7 @@ } explorer = TwoPhaseLinearExplorer(agent_id_list, config.general.total_training_episodes, **exploration_config) - # Step 3: create an agent manager. + # Step 3: Create an agent manager. agent_manager = DQNAgentManager( name="cim_learner", mode=AgentManagerMode.TRAIN_INFERENCE, diff --git a/maro/rl/agent/abs_agent.py b/maro/rl/agent/abs_agent.py index 1d5b96648..c111b19e5 100644 --- a/maro/rl/agent/abs_agent.py +++ b/maro/rl/agent/abs_agent.py @@ -32,8 +32,8 @@ class AbsAgent(ABC): algorithm (AbsAlgorithm): A concrete algorithm instance that inherits from AbstractAlgorithm. This is the centerpiece of the Agent class and is responsible for the most important tasks of an agent: choosing actions and optimizing models. - experience_pool (AbsStore): A data store that stores experiences generated by the experience shaper. This is - only necessary for some algorithms. Defaults to None. + experience_pool (AbsStore): It is used to store experiences processed by the experience shaper, which will be + used by some value-based algorithms, such as DQN. Defaults to None. """ def __init__(self, name: str, mode: AgentMode, algorithm: AbsAlgorithm, experience_pool: AbsStore = None): self._name = name diff --git a/maro/rl/agent/abs_agent_manager.py b/maro/rl/agent/abs_agent_manager.py index ffaa13d3d..ff489e9c6 100644 --- a/maro/rl/agent/abs_agent_manager.py +++ b/maro/rl/agent/abs_agent_manager.py @@ -38,12 +38,12 @@ class AbsAgentManager(ABC): input. action_shaper (ActionShaper, optional): It is responsible for converting an agent's model output to environment executable action. Cannot be None under Inference and TrainInference modes. - explorer (AbsExplorer): It is responsible for storing and updating exploration rates. """ def __init__( self, name: str, mode: AgentManagerMode, agent_dict: dict, - state_shaper: StateShaper = None, action_shaper: ActionShaper = None, - experience_shaper: ExperienceShaper = None, explorer: AbsExplorer = None + state_shaper: StateShaper = None, + action_shaper: ActionShaper = None, + experience_shaper: ExperienceShaper = None ): self._name = name self._mode = mode @@ -51,7 +51,6 @@ def __init__( self._state_shaper = state_shaper self._action_shaper = action_shaper self._experience_shaper = experience_shaper - self._explorer = explorer def __getitem__(self, agent_id): return self.agent_dict[agent_id] @@ -85,11 +84,6 @@ def name(self): """Agent manager's name.""" return self._name - @property - def explorer(self): - """Explorer used by the agent manager.""" - return self._explorer - def _assert_train_mode(self): if self._mode != AgentManagerMode.TRAIN and self._mode != AgentManagerMode.TRAIN_INFERENCE: raise WrongAgentManagerModeError(msg=f"this method is unavailable under mode {self._mode}") diff --git a/maro/rl/agent/simple_agent_manager.py b/maro/rl/agent/simple_agent_manager.py index ef1ca5bd8..36dc566c8 100644 --- a/maro/rl/agent/simple_agent_manager.py +++ b/maro/rl/agent/simple_agent_manager.py @@ -28,11 +28,13 @@ def __init__( raise MissingShaperError(msg=f"experience_shaper cannot be None under mode {self._mode}") super().__init__( - name, mode, agent_dict, state_shaper=state_shaper, action_shaper=action_shaper, - experience_shaper=experience_shaper, explorer=explorer + name, mode, agent_dict, + state_shaper=state_shaper, + action_shaper=action_shaper, + experience_shaper=experience_shaper ) - # data structures to temporarily store transitions and trajectory + # Data structures to temporarily store transitions and trajectory self._transition_cache = {} self._trajectory = ColumnBasedStore() diff --git a/maro/rl/algorithms/dqn.py b/maro/rl/algorithms/dqn.py index 066fb661c..ca1fe5a14 100644 --- a/maro/rl/algorithms/dqn.py +++ b/maro/rl/algorithms/dqn.py @@ -35,7 +35,7 @@ class DQN(AbsAlgorithm): See https://web.stanford.edu/class/psych209/Readings/MnihEtAlHassibis15NatureControlDeepRL.pdf for details. Args: - eval_model (nn.Module): trainable Q-value model for computing actions given states. + eval_model (nn.Module): Q-value model for given states and actions. optimizer_cls: torch optimizer class for the eval model. If this is None, the eval model is not trainable. optimizer_params: parameters required for the eval optimizer class. loss_func (Callable): loss function for the value model. @@ -53,9 +53,13 @@ def __init__( if optimizer_cls is not None: self._optimizer = optimizer_cls(self._model_dict["eval"].parameters(), **optimizer_params) if target_model is None: - self._model_dict["target"] = clone(eval_model).to(self._device) if eval_model is not None else None + self._model_dict["target"] = clone(eval_model).to(self._device) else: self._model_dict["target"] = target_model.to(self._device) + # No gradient computation required for the target model + for param in self._model_dict["target"].parameters(): + param.requires_grad = False + self._loss_func = loss_func self._hyper_params = hyper_params self._train_cnt = 0 @@ -94,7 +98,7 @@ def train(self, states: np.ndarray, actions: np.ndarray, rewards: np.ndarray, ne if self._train_cnt % self._hyper_params.num_training_rounds_per_target_replacement == 0: self._update_target_model() - return np.abs((current_q_values - target_q_values).detach().numpy()) + return loss.detach().numpy() def _update_target_model(self): if hasattr(self, "_optimizer"): diff --git a/maro/rl/models/decision_layers.py b/maro/rl/models/decision_layers.py index c4f4cbdd9..d1017c299 100644 --- a/maro/rl/models/decision_layers.py +++ b/maro/rl/models/decision_layers.py @@ -65,7 +65,7 @@ def output_dim(self): def _build_basic_layer(self, input_dim, output_dim): """Build basic layer. - BN -> Linear -> LeakyReLU -> Dropout + BN -> Linear -> Activation -> Dropout """ components = [] if self._batch_norm_enabled: diff --git a/maro/rl/models/learning_model.py b/maro/rl/models/learning_model.py index 7272b6550..b186c528e 100644 --- a/maro/rl/models/learning_model.py +++ b/maro/rl/models/learning_model.py @@ -21,7 +21,9 @@ class LearningModel(nn.Module): clip_value (float): Threshold used to clip gradients. """ def __init__( - self, representation_layers: nn.Module = IdentityLayers(), decision_layers: nn.Module = IdentityLayers(), + self, + representation_layers: nn.Module = IdentityLayers(), + decision_layers: nn.Module = IdentityLayers(), clip_value: float = None ): super().__init__() From 5985ad06400c9b38c8f3aa19b25307dff3052164 Mon Sep 17 00:00:00 2001 From: ysqyang Date: Tue, 27 Oct 2020 11:29:01 +0800 Subject: [PATCH 064/337] fixed more PR comments --- maro/rl/agent/abs_agent_manager.py | 2 +- maro/rl/agent/simple_agent_manager.py | 2 +- maro/rl/algorithms/abs_algorithm.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/maro/rl/agent/abs_agent_manager.py b/maro/rl/agent/abs_agent_manager.py index ff489e9c6..594f3e05f 100644 --- a/maro/rl/agent/abs_agent_manager.py +++ b/maro/rl/agent/abs_agent_manager.py @@ -31,7 +31,7 @@ class AbsAgentManager(ABC): name (str): Name of agent manager. mode (AgentManagerMode): An ``AgentManagerNode`` enum member that indicates the role of the agent manager in the current process. - agent_dict (dict): List of agent identifiers. + agent_dict (dict): A dictionary of agents to be wrapper by the agent manager. experience_shaper (ExperienceShaper, optional): It is responsible for processing data in the replay buffer at the end of an episode. state_shaper (StateShaper, optional): It is responsible for converting the environment observation to model diff --git a/maro/rl/agent/simple_agent_manager.py b/maro/rl/agent/simple_agent_manager.py index 36dc566c8..f856bf1fd 100644 --- a/maro/rl/agent/simple_agent_manager.py +++ b/maro/rl/agent/simple_agent_manager.py @@ -86,7 +86,7 @@ def load_models(self, agent_model_dict): for agent_id, models in agent_model_dict.items(): self.agent_dict[agent_id].load_models(models) - def dump_models(self): + def dump_models(self) -> dict: """Get agents' underlying models. This is usually used in distributed mode where models need to be broadcast to remote roll-out actors. diff --git a/maro/rl/algorithms/abs_algorithm.py b/maro/rl/algorithms/abs_algorithm.py index 2bab4b874..f1f8d7b00 100644 --- a/maro/rl/algorithms/abs_algorithm.py +++ b/maro/rl/algorithms/abs_algorithm.py @@ -35,7 +35,7 @@ def train(self, *args, **kwargs): """Train models using samples. This method is algorithm-specific and needs to be implemented by the user. For example, for the DQN - algorithm, this may look like train(self, state, action, reward, next_state). + algorithm, this may look like train(self, state_batch, action_batch, reward_batch, next_state_batch). """ return NotImplementedError From 41b720ed1d30f4e0cfc679f0a0762cb40e94605d Mon Sep 17 00:00:00 2001 From: ysqyang Date: Wed, 28 Oct 2020 09:32:06 +0800 Subject: [PATCH 065/337] 1.fixed some PR comments; 2.added early_stopping_checker; 3.revised explorer class --- docs/source/key_components/rl_toolkit.rst | 5 +- examples/cim/dqn/components/agent.py | 4 +- examples/cim/dqn/components/agent_manager.py | 19 +++-- examples/cim/dqn/dist_actor.py | 23 +++++- examples/cim/dqn/dist_learner.py | 25 ++++++- examples/cim/dqn/single_process_launcher.py | 25 ++++++- maro/rl/__init__.py | 3 +- maro/rl/actor/simple_actor.py | 7 +- maro/rl/agent/abs_agent.py | 21 +----- maro/rl/agent/abs_agent_manager.py | 11 ++- maro/rl/agent/simple_agent_manager.py | 15 ++-- .../abs_early_stopping_checker.py | 13 ++++ .../simple_early_stopping_checker.py | 20 +++++ maro/rl/explorer/abs_explorer.py | 45 ++---------- maro/rl/explorer/simple_explorer.py | 51 +++++-------- maro/rl/learner/simple_learner.py | 73 ++++++++++++++----- maro/utils/exception/error_code.py | 3 +- maro/utils/exception/rl_toolkit_exception.py | 10 +-- 18 files changed, 208 insertions(+), 165 deletions(-) create mode 100644 maro/rl/early_stopping/abs_early_stopping_checker.py create mode 100644 maro/rl/early_stopping/simple_early_stopping_checker.py diff --git a/docs/source/key_components/rl_toolkit.rst b/docs/source/key_components/rl_toolkit.rst index 857023da8..56ae2f61c 100644 --- a/docs/source/key_components/rl_toolkit.rst +++ b/docs/source/key_components/rl_toolkit.rst @@ -70,10 +70,7 @@ Agent Manager The agent manager provides a unified interactive interface with the environment for RL agent(s). From the actor's perspective, it isolates the complex dependencies of the various homogeneous/heterogeneous agents, so that the whole agent manager -will behave just like a single agent. Besides that, the agent manager also plays -the role of an agent assembler. It can assemble different RL agents according to -the actual requirements, such as whether to share the underlying model, whether -to share the experience pool, etc. +will behave just like a single agent. .. code-block:: python diff --git a/examples/cim/dqn/components/agent.py b/examples/cim/dqn/components/agent.py index e5ed02e1c..7f6f24255 100644 --- a/examples/cim/dqn/components/agent.py +++ b/examples/cim/dqn/components/agent.py @@ -7,9 +7,9 @@ class CIMAgent(AbsAgent): - def __init__(self, name, mode, algorithm, experience_pool: ColumnBasedStore, min_experiences_to_train, + def __init__(self, name, algorithm, experience_pool: ColumnBasedStore, min_experiences_to_train, num_batches, batch_size): - super().__init__(name, mode, algorithm, experience_pool) + super().__init__(name, algorithm, experience_pool) self._min_experiences_to_train = min_experiences_to_train self._num_batches = num_batches self._batch_size = batch_size diff --git a/examples/cim/dqn/components/agent_manager.py b/examples/cim/dqn/components/agent_manager.py index 4224755f2..d9fb449f1 100644 --- a/examples/cim/dqn/components/agent_manager.py +++ b/examples/cim/dqn/components/agent_manager.py @@ -5,12 +5,11 @@ from torch.optim import RMSprop from .agent import CIMAgent -from maro.rl import AgentMode, SimpleAgentManager, LearningModel, DecisionLayers, DQN, DQNHyperParams, ColumnBasedStore +from maro.rl import SimpleAgentManager, LearningModel, DecisionLayers, DQN, DQNHyperParams, ColumnBasedStore from maro.utils import set_seeds -def create_dqn_agents(agent_id_list, mode, config): - is_trainable = mode in {AgentMode.TRAIN, AgentMode.TRAIN_INFERENCE} +def create_dqn_agents(agent_id_list, config): num_actions = config.algorithm.num_actions set_seeds(config.seed) agent_dict = {} @@ -26,9 +25,9 @@ def create_dqn_agents(agent_id_list, mode, config): algorithm = DQN( eval_model=eval_model, - optimizer_cls=RMSprop if is_trainable else None, - optimizer_params=config.algorithm.optimizer if is_trainable else None, - loss_func=nn.functional.smooth_l1_loss if is_trainable else None, + optimizer_cls=RMSprop, + optimizer_params=config.algorithm.optimizer, + loss_func=nn.functional.smooth_l1_loss, hyper_params=DQNHyperParams( **config.algorithm.hyper_parameters, num_actions=num_actions @@ -36,8 +35,12 @@ def create_dqn_agents(agent_id_list, mode, config): ) experience_pool = ColumnBasedStore(**config.experience_pool) - agent_dict[agent_id] = CIMAgent(name=agent_id, mode=mode, algorithm=algorithm, experience_pool=experience_pool, - **config.training_loop_parameters) + agent_dict[agent_id] = CIMAgent( + name=agent_id, + algorithm=algorithm, + experience_pool=experience_pool, + **config.training_loop_parameters + ) return agent_dict diff --git a/examples/cim/dqn/dist_actor.py b/examples/cim/dqn/dist_actor.py index 5767ed531..b4929c557 100644 --- a/examples/cim/dqn/dist_actor.py +++ b/examples/cim/dqn/dist_actor.py @@ -4,16 +4,26 @@ import numpy as np from maro.simulator import Env -from maro.rl import AgentMode, AgentManagerMode, SimpleActor, ActorWorker, KStepExperienceShaper, TwoPhaseLinearExplorer +from maro.rl import AgentManagerMode, SimpleActor, ActorWorker, KStepExperienceShaper, TwoPhaseLinearExplorer from components.action_shaper import CIMActionShaper from components.agent_manager import create_dqn_agents, DQNAgentManager -from components.config import config from components.experience_shaper import TruncatedExperienceShaper from components.state_shaper import CIMStateShaper -if __name__ == "__main__": +def launch(config): + def set_input_dim(): + # obtain model input dimension from state shaping configurations + look_back = config["state_shaping"]["look_back"] + max_ports_downstream = config["state_shaping"]["max_ports_downstream"] + num_port_attributes = len(config["state_shaping"]["port_attributes"]) + num_vessel_attributes = len(config["state_shaping"]["vessel_attributes"]) + + input_dim = (look_back + 1) * (max_ports_downstream + 1) * num_port_attributes + num_vessel_attributes + config["agents"]["algorithm"]["input_dim"] = input_dim + + set_input_dim() env = Env(config.env.scenario, config.env.topology, durations=config.env.durations) agent_id_list = [str(agent_id) for agent_id in env.agent_idx_list] state_shaper = CIMStateShaper(**config.state_shaping) @@ -35,7 +45,7 @@ agent_manager = DQNAgentManager( name="distributed_cim_actor", mode=AgentManagerMode.INFERENCE, - agent_dict=create_dqn_agents(agent_id_list, AgentMode.INFERENCE, config.agents), + agent_dict=create_dqn_agents(agent_id_list, config.agents), state_shaper=state_shaper, action_shaper=action_shaper, experience_shaper=experience_shaper, @@ -51,3 +61,8 @@ proxy_params=proxy_params ) actor_worker.launch() + + +if __name__ == "__main__": + from components.config import config + launch(config) diff --git a/examples/cim/dqn/dist_learner.py b/examples/cim/dqn/dist_learner.py index 1f70beb79..7406d0949 100644 --- a/examples/cim/dqn/dist_learner.py +++ b/examples/cim/dqn/dist_learner.py @@ -3,15 +3,26 @@ import os -from maro.rl import ActorProxy, SimpleLearner, AgentMode, AgentManagerMode, TwoPhaseLinearExplorer +from maro.rl import ActorProxy, SimpleLearner, AgentManagerMode, TwoPhaseLinearExplorer from maro.simulator import Env from maro.utils import Logger from components.agent_manager import create_dqn_agents, DQNAgentManager -from components.config import config -if __name__ == "__main__": +def launch(config): + def set_input_dim(): + # obtain model input dimension from state shaping configurations + look_back = config["state_shaping"]["look_back"] + max_ports_downstream = config["state_shaping"]["max_ports_downstream"] + num_port_attributes = len(config["state_shaping"]["port_attributes"]) + num_vessel_attributes = len(config["state_shaping"]["vessel_attributes"]) + + input_dim = (look_back + 1) * (max_ports_downstream + 1) * num_port_attributes + num_vessel_attributes + config["agents"]["algorithm"]["input_dim"] = input_dim + + set_input_dim() + env = Env(config.env.scenario, config.env.topology, durations=config.env.durations) agent_id_list = [str(agent_id) for agent_id in env.agent_idx_list] exploration_config = { @@ -23,7 +34,7 @@ agent_manager = DQNAgentManager( name="distributed_cim_learner", mode=AgentManagerMode.TRAIN, - agent_dict=create_dqn_agents(agent_id_list, AgentMode.TRAIN, config.agents), + agent_dict=create_dqn_agents(agent_id_list, config.agents), explorer=explorer ) @@ -41,3 +52,9 @@ learner.train(total_episodes=config.general.total_training_episodes) learner.test() learner.dump_models(os.path.join(os.getcwd(), "models")) + + +if __name__ == "__main__": + from components.config import config + launch(config) + diff --git a/examples/cim/dqn/single_process_launcher.py b/examples/cim/dqn/single_process_launcher.py index d2ee1ac88..c7ed7647c 100644 --- a/examples/cim/dqn/single_process_launcher.py +++ b/examples/cim/dqn/single_process_launcher.py @@ -7,18 +7,30 @@ import numpy as np from maro.simulator import Env -from maro.rl import SimpleLearner, SimpleActor, AgentMode, AgentManagerMode, KStepExperienceShaper, \ +from maro.rl import SimpleLearner, SimpleActor, AgentManagerMode, KStepExperienceShaper, \ TwoPhaseLinearExplorer from maro.utils import Logger from components.action_shaper import CIMActionShaper from components.agent_manager import create_dqn_agents, DQNAgentManager -from components.config import config from components.experience_shaper import TruncatedExperienceShaper from components.state_shaper import CIMStateShaper -if __name__ == "__main__": +def launch(config): + # First determine the input dimension and add it to the config. + def set_input_dim(): + # obtain model input dimension from state shaping configurations + look_back = config["state_shaping"]["look_back"] + max_ports_downstream = config["state_shaping"]["max_ports_downstream"] + num_port_attributes = len(config["state_shaping"]["port_attributes"]) + num_vessel_attributes = len(config["state_shaping"]["vessel_attributes"]) + + input_dim = (look_back + 1) * (max_ports_downstream + 1) * num_port_attributes + num_vessel_attributes + config["agents"]["algorithm"]["input_dim"] = input_dim + + set_input_dim() + # Step 1: Initialize a CIM environment for using a toy dataset. env = Env(config.env.scenario, config.env.topology, durations=config.env.durations) agent_id_list = [str(agent_id) for agent_id in env.agent_idx_list] @@ -46,7 +58,7 @@ agent_manager = DQNAgentManager( name="cim_learner", mode=AgentManagerMode.TRAIN_INFERENCE, - agent_dict=create_dqn_agents(agent_id_list, AgentMode.TRAIN_INFERENCE, config.agents), + agent_dict=create_dqn_agents(agent_id_list, config.agents), state_shaper=state_shaper, action_shaper=action_shaper, experience_shaper=experience_shaper, @@ -62,3 +74,8 @@ learner.train(total_episodes=config.general.total_training_episodes) learner.test() learner.dump_models(os.path.join(os.getcwd(), "models")) + + +if __name__ == "__main__": + from components.config import config + launch(config) diff --git a/maro/rl/__init__.py b/maro/rl/__init__.py index 97b21630b..c51793c92 100644 --- a/maro/rl/__init__.py +++ b/maro/rl/__init__.py @@ -5,7 +5,7 @@ from maro.rl.actor.simple_actor import SimpleActor from maro.rl.learner.abs_learner import AbsLearner from maro.rl.learner.simple_learner import SimpleLearner -from maro.rl.agent.abs_agent import AbsAgent, AgentMode +from maro.rl.agent.abs_agent import AbsAgent from maro.rl.agent.abs_agent_manager import AbsAgentManager, AgentManagerMode from maro.rl.agent.simple_agent_manager import SimpleAgentManager from maro.rl.algorithms.abs_algorithm import AbsAlgorithm @@ -32,7 +32,6 @@ "AbsLearner", "SimpleLearner", "AbsAgent", - "AgentMode", "AbsAgentManager", "AgentManagerMode", "SimpleAgentManager", diff --git a/maro/rl/actor/simple_actor.py b/maro/rl/actor/simple_actor.py index f196c3c02..db5e0a6f9 100644 --- a/maro/rl/actor/simple_actor.py +++ b/maro/rl/actor/simple_actor.py @@ -36,9 +36,6 @@ def roll_out( return None, None self._env.reset() - # assign epsilons - if epsilon_dict is not None: - self._inference_agents.explorer.epsilon = epsilon_dict # load models if model_dict is not None: @@ -46,7 +43,9 @@ def roll_out( metrics, decision_event, is_done = self._env.step(None) while not is_done: - action = self._inference_agents.choose_action(decision_event, self._env.snapshot_list) + action = self._inference_agents.choose_action( + decision_event, self._env.snapshot_list, epsilon_dict=epsilon_dict + ) metrics, decision_event, is_done = self._env.step(action) self._inference_agents.on_env_feedback(metrics) diff --git a/maro/rl/agent/abs_agent.py b/maro/rl/agent/abs_agent.py index c111b19e5..24398d4f8 100644 --- a/maro/rl/agent/abs_agent.py +++ b/maro/rl/agent/abs_agent.py @@ -2,19 +2,11 @@ # Licensed under the MIT license. from abc import ABC, abstractmethod -from enum import Enum import os import pickle from maro.rl.algorithms.abs_algorithm import AbsAlgorithm from maro.rl.storage.abs_store import AbsStore -from maro.utils.exception.rl_toolkit_exception import WrongAgentModeError - - -class AgentMode(Enum): - TRAIN = "train" - INFERENCE = "inference" - TRAIN_INFERENCE = "train_inference" class AbsAgent(ABC): @@ -28,16 +20,14 @@ class AbsAgent(ABC): Args: name (str): Agent's name. - mode (AgentMode): An ``AgentMode`` enum member that indicates the role of the agent in the current process. algorithm (AbsAlgorithm): A concrete algorithm instance that inherits from AbstractAlgorithm. This is the centerpiece of the Agent class and is responsible for the most important tasks of an agent: choosing actions and optimizing models. experience_pool (AbsStore): It is used to store experiences processed by the experience shaper, which will be used by some value-based algorithms, such as DQN. Defaults to None. """ - def __init__(self, name: str, mode: AgentMode, algorithm: AbsAlgorithm, experience_pool: AbsStore = None): + def __init__(self, name: str, algorithm: AbsAlgorithm, experience_pool: AbsStore = None): self._name = name - self._mode = mode self._algorithm = algorithm self._experience_pool = experience_pool @@ -60,7 +50,6 @@ def choose_action(self, model_state, epsilon: float = .0): Returns: Action given by the underlying policy model. """ - self._assert_inference_mode() return self._algorithm.choose_action(model_state, epsilon) @abstractmethod @@ -111,11 +100,3 @@ def dump_experience_pool(self, dir_path: str): os.makedirs(dir_path, exist_ok=True) with open(os.path.join(dir_path, self._name), "wb") as fp: pickle.dump(self._experience_pool, fp) - - def _assert_train_mode(self): - if self._mode != AgentMode.TRAIN and self._mode != AgentMode.TRAIN_INFERENCE: - raise WrongAgentModeError(msg=f"this method is unavailable under mode {self._mode}") - - def _assert_inference_mode(self): - if self._mode != AgentMode.INFERENCE and self._mode != AgentMode.TRAIN_INFERENCE: - raise WrongAgentModeError(msg=f"this method is unavailable under mode {self._mode}") diff --git a/maro/rl/agent/abs_agent_manager.py b/maro/rl/agent/abs_agent_manager.py index 594f3e05f..8e0eebc8c 100644 --- a/maro/rl/agent/abs_agent_manager.py +++ b/maro/rl/agent/abs_agent_manager.py @@ -7,7 +7,6 @@ from maro.rl.shaping.state_shaper import StateShaper from maro.rl.shaping.action_shaper import ActionShaper from maro.rl.shaping.experience_shaper import ExperienceShaper -from maro.rl.explorer.abs_explorer import AbsExplorer from maro.utils.exception.rl_toolkit_exception import WrongAgentManagerModeError @@ -22,10 +21,7 @@ class AbsAgentManager(ABC): The agent manager provides a unified interactive interface with the environment for RL agent(s). From the actor’s perspective, it isolates the complex dependencies of the various homogeneous/heterogeneous - agents, so that the whole agent manager will behave just like a single agent. Besides that, the agent - manager also plays the role of an agent assembler. It can assemble different RL agents according to the - actual requirements, such as whether to share the underlying model, whether to share the experience - pool, etc. + agents, so that the whole agent manager will behave just like a single agent. Args: name (str): Name of agent manager. @@ -40,7 +36,10 @@ class AbsAgentManager(ABC): executable action. Cannot be None under Inference and TrainInference modes. """ def __init__( - self, name: str, mode: AgentManagerMode, agent_dict: dict, + self, + name: str, + mode: AgentManagerMode, + agent_dict: dict, state_shaper: StateShaper = None, action_shaper: ActionShaper = None, experience_shaper: ExperienceShaper = None diff --git a/maro/rl/agent/simple_agent_manager.py b/maro/rl/agent/simple_agent_manager.py index f856bf1fd..ceec7158c 100644 --- a/maro/rl/agent/simple_agent_manager.py +++ b/maro/rl/agent/simple_agent_manager.py @@ -8,16 +8,19 @@ from maro.rl.shaping.state_shaper import StateShaper from maro.rl.shaping.action_shaper import ActionShaper from maro.rl.shaping.experience_shaper import ExperienceShaper -from maro.rl.explorer.abs_explorer import AbsExplorer from maro.rl.storage.column_based_store import ColumnBasedStore from maro.utils.exception.rl_toolkit_exception import MissingShaperError class SimpleAgentManager(AbsAgentManager): def __init__( - self, name: str, mode: AgentManagerMode, agent_dict: dict, - state_shaper: StateShaper = None, action_shaper: ActionShaper = None, - experience_shaper: ExperienceShaper = None, explorer: AbsExplorer = None, + self, + name: str, + mode: AgentManagerMode, + agent_dict: dict, + state_shaper: StateShaper = None, + action_shaper: ActionShaper = None, + experience_shaper: ExperienceShaper = None ): if mode in {AgentManagerMode.INFERENCE, AgentManagerMode.TRAIN_INFERENCE}: if state_shaper is None: @@ -38,11 +41,11 @@ def __init__( self._transition_cache = {} self._trajectory = ColumnBasedStore() - def choose_action(self, decision_event, snapshot_list): + def choose_action(self, decision_event, snapshot_list, epsilon_dict: dict = None): self._assert_inference_mode() agent_id, model_state = self._state_shaper(decision_event, snapshot_list) model_action = self.agent_dict[agent_id].choose_action( - model_state, self._explorer.epsilon[agent_id] if self._explorer else None + model_state, epsilon_dict[agent_id] if epsilon_dict else None ) self._transition_cache = { "state": model_state, diff --git a/maro/rl/early_stopping/abs_early_stopping_checker.py b/maro/rl/early_stopping/abs_early_stopping_checker.py new file mode 100644 index 000000000..8a30529cf --- /dev/null +++ b/maro/rl/early_stopping/abs_early_stopping_checker.py @@ -0,0 +1,13 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +from abc import ABC, abstractmethod + + +class AbsEarlyStoppingChecker(ABC): + def __init__(self): + pass + + @abstractmethod + def __call__(self, performance_history): + return NotImplemented diff --git a/maro/rl/early_stopping/simple_early_stopping_checker.py b/maro/rl/early_stopping/simple_early_stopping_checker.py new file mode 100644 index 000000000..734eea5b9 --- /dev/null +++ b/maro/rl/early_stopping/simple_early_stopping_checker.py @@ -0,0 +1,20 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +from typing import Callable + +from statistics import mean, stdev + +from .abs_early_stopping_checker import AbsEarlyStoppingChecker + + +class SimpleEarlyStoppingChecker(AbsEarlyStoppingChecker): + def __init__(self, last_k: int, performance_metric_func: Callable, threshold: float): + super().__init__() + self._last_k = last_k + self._performance_metric_func = performance_metric_func + self._threshold = threshold + + def __call__(self, performance_history): + metrics = map(self._performance_metric_func, performance_history[self._last_k:]) + return stdev(metrics) / mean(metrics) < self._threshold diff --git a/maro/rl/explorer/abs_explorer.py b/maro/rl/explorer/abs_explorer.py index 992a80ada..db9e8ed5b 100644 --- a/maro/rl/explorer/abs_explorer.py +++ b/maro/rl/explorer/abs_explorer.py @@ -8,48 +8,13 @@ class AbsExplorer(ABC): """Abstract explorer class. An explorer is responsible for generating exploration rates. - - Args: - agent_id_list (list): List of agent ID's. - total_episodes (int): Total number of episodes in the training phase. - epsilon_range_dict (dict): A dictionary containing tuples of lower and upper bounds for the generated - exploration rate for each agent. If the dictionary contains `_all_` as a key, the corresponding - value will be shared amongst all agents. - with_cache (bool): If True, incoming performances will be cached. """ - def __init__(self, agent_id_list: list, total_episodes: int, epsilon_range_dict: dict, with_cache: bool = True): - self._total_episodes = total_episodes - self._epsilon_range_dict = epsilon_range_dict - self._performance_cache = [] if with_cache else None - if "_all_" in self._epsilon_range_dict: - self._current_epsilon = {agent_id: self._epsilon_range_dict["_all_"][1] for agent_id in agent_id_list} - else: - self._current_epsilon = {agent_id: self._epsilon_range_dict.get(agent_id, (.0, .0))[1] - for agent_id in agent_id_list} + def __init__(self): + pass # TODO: performance: summary -> total perf (current version), details -> per-agent perf @abstractmethod - def update(self, performance=None): - """Update exploration rates for each agent. - - Args: - performance: Performance from the latest episode. + def generate_epsilon(self, current_ep: int, max_ep: int, performance_history=None): + """Generate an exploration rate. """ - return NotImplementedError - - @property - def epsilon_range_dict(self): - """Exploration rate ranges for each agent.""" - return self._epsilon_range_dict - - @property - def epsilon(self): - """Current exploration rates for each agent.""" - return self._current_epsilon - - @epsilon.setter - def epsilon(self, epsilon_dict: dict): - self._current_epsilon = epsilon_dict - - def epsilon_range_by_id(self, agent_id): - return self._epsilon_range_dict[agent_id] + return NotImplemented diff --git a/maro/rl/explorer/simple_explorer.py b/maro/rl/explorer/simple_explorer.py index 0deb1bd9f..575ec5ceb 100644 --- a/maro/rl/explorer/simple_explorer.py +++ b/maro/rl/explorer/simple_explorer.py @@ -6,40 +6,27 @@ class LinearExplorer(AbsExplorer): """A simple linear exploration scheme.""" - def __init__(self, agent_id_list, total_episodes, epsilon_range_dict, with_cache=True): - super().__init__(agent_id_list, total_episodes, epsilon_range_dict, with_cache=with_cache) - self._step_dict = {} - for agent_id in agent_id_list: - min_eps, max_eps = self._epsilon_range_dict.get(agent_id, (.0, .0)) - self._step_dict[agent_id] = (max_eps - min_eps) / total_episodes + def __init__(self, max_eps: float, min_eps: float = .0): + super().__init__() + self._max_eps = max_eps + self._min_eps = min_eps - def update(self, performance=None): - for agent_id in self._current_epsilon: - self._current_epsilon[agent_id] = max(.0, self._current_epsilon[agent_id] - self._step_dict[agent_id]) + def generate_epsilon(self, current_ep, max_ep, performance_history=None): + return self._min_eps + (self._max_eps - self._min_eps) * (1 - current_ep / max_ep) class TwoPhaseLinearExplorer(AbsExplorer): """An exploration scheme that consists of two linear schedules separated by a split point.""" - def __init__(self, agent_id_list, total_episodes, epsilon_range_dict, split_point_dict: dict, with_cache=True): - super().__init__(agent_id_list, total_episodes, epsilon_range_dict, with_cache=with_cache) - self._step_dict_p1, self._step_dict_p2, self._num_episodes_p1_dict = {}, {}, {} - for agent_id in agent_id_list: - if "_all_" in self._epsilon_range_dict: - min_eps, max_eps = self._epsilon_range_dict["_all_"] - else: - min_eps, max_eps = self._epsilon_range_dict.get(agent_id, (.0, .0)) - eps_range = max_eps - min_eps - split_point = split_point_dict["_all_"] if "_all_" in split_point_dict else split_point_dict[agent_id] - num_episodes_p1 = int(total_episodes * split_point[0]) - num_episodes_p2 = total_episodes - num_episodes_p1 - self._step_dict_p1[agent_id] = eps_range * (1 - split_point[1]) / (num_episodes_p1 - 1 + 1e-10) - self._step_dict_p2[agent_id] = eps_range * split_point[1] / (num_episodes_p2 + 1e-10) - self._num_episodes_p1_dict[agent_id] = num_episodes_p1 - - self._counter = 0 - - def update(self, performance=None): - self._counter += 1 - for agent_id, num_episodes_p1 in self._num_episodes_p1_dict.items(): - step_dict = self._step_dict_p1 if self._counter <= num_episodes_p1 else self._step_dict_p2 - self._current_epsilon[agent_id] = max(.0, self._current_epsilon[agent_id] - step_dict[agent_id]) + def __init__(self, progress_split: float, eps_split: float, max_eps: float, min_eps: float = 0): + super().__init__() + self._progress_split = progress_split + self._eps_split = eps_split + self._max_eps = max_eps + self._min_eps = min_eps + + def generate_epsilon(self, current_ep, max_ep, performance_history=None): + progress = current_ep / max_ep + if progress <= self._progress_split: + return self._max_eps - (self._max_eps - self._eps_split) * progress / self._progress_split + else: + return self._min_eps + (self._eps_split - self._min_eps) * (1 - progress) / (1 - self._progress_split) diff --git a/maro/rl/learner/simple_learner.py b/maro/rl/learner/simple_learner.py index a49a1f422..c684e930e 100644 --- a/maro/rl/learner/simple_learner.py +++ b/maro/rl/learner/simple_learner.py @@ -1,9 +1,13 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT license. +from typing import Callable +import warnings + from .abs_learner import AbsLearner -from maro.rl.agent.simple_agent_manager import SimpleAgentManager from maro.rl.actor.simple_actor import SimpleActor +from maro.rl.agent.simple_agent_manager import SimpleAgentManager +from maro.rl.explorer.abs_explorer import AbsExplorer from maro.utils import DummyLogger @@ -15,31 +19,63 @@ class SimpleLearner(AbsLearner): actor (Actor or ActorProxy): an Actor or VectorActorProxy instance. logger: used for logging important messages. """ - def __init__(self, trainable_agents: SimpleAgentManager, actor, logger=DummyLogger()): + def __init__( + self, + trainable_agents: SimpleAgentManager, + actor, + explorer: AbsExplorer = None, + logger=DummyLogger() + ): super().__init__() self._trainable_agents = trainable_agents self._actor = actor + self._explorer = explorer self._logger = logger + self._performance_history = [] + + def _get_epsilons(self, current_ep, max_ep): + if self._explorer is not None: + return { + agent_id: self._explorer.generate_epsilon(current_ep, max_ep, self._performance_history) + for agent_id in self._trainable_agents.agent_dict + } + else: + return None + + def _sample(self, ep, max_ep): + """One episode""" + model_dict = None if self._is_shared_agent_instance() else self._trainable_agents.dump_models() + epsilon_dict = self._get_epsilons(ep, max_ep) + performance, exp_by_agent = self._actor.roll_out(model_dict=model_dict, epsilon_dict=epsilon_dict) + self._logger.info(f"ep {ep} - performance: {performance}, epsilons: {epsilon_dict}") + return performance, exp_by_agent - def train(self, total_episodes: int): + def train(self, max_episode: int, early_stopping_checker: Callable = None): """Main loop for collecting experiences from the actor and using them to update policies. Args: - total_episodes (int): number of episodes to be run. + max_episode (int): number of episodes to be run. If negative, the training loop will run forever unless + an ``early_stopping_checker`` is provided and the early stopping condition is met. + early_stopping_checker (Callable): A Callable object to judge whether the training loop should be ended + based on the latest performances. """ - for current_ep in range(1, total_episodes + 1): - model_dict = None if self._is_shared_agent_instance() else self._trainable_agents.dump_models() - epsilon_dict = self._trainable_agents.explorer.epsilon if self._trainable_agents.explorer else None - performance, exp_by_agent = self._actor.roll_out(model_dict=model_dict, epsilon_dict=epsilon_dict) - if isinstance(performance, dict): - for actor_id, perf in performance.items(): - self._logger.info( - f"ep {current_ep} - performance: {perf}, source: {actor_id}, epsilons: {epsilon_dict}" - ) - else: - self._logger.info(f"ep {current_ep} - performance: {performance}, epsilons: {epsilon_dict}") - - self._trainable_agents.train(exp_by_agent) + if max_episode < 0: + if early_stopping_checker is None: + warnings.warn("No max episode and early stopping checker provided. The training loop will run forever.") + episode = 1 + while True: + performance, exp_by_agent = self._sample(episode, max_episode) + self._performance_history.append(performance) + if early_stopping_checker is not None and early_stopping_checker(self._performance_history): + break + episode += 1 + else: + for episode in range(1, max_episode + 1): + performance, exp_by_agent = self._sample(episode, max_episode) + self._performance_history.append(performance) + if early_stopping_checker is not None and early_stopping_checker(self._performance_history): + break + self._trainable_agents.train(exp_by_agent) def test(self): """Test policy performance.""" @@ -47,8 +83,7 @@ def test(self): model_dict=self._trainable_agents.dump_models(), return_details=False ) - for actor_id, perf in performance.items(): - self._logger.info(f"test performance from {actor_id}: {perf}") + self._logger.info(f"test performance: {performance}") self._actor.roll_out(done=True) def dump_models(self, model_dump_dir: str): diff --git a/maro/utils/exception/error_code.py b/maro/utils/exception/error_code.py index da155e1ae..efb728208 100644 --- a/maro/utils/exception/error_code.py +++ b/maro/utils/exception/error_code.py @@ -38,6 +38,5 @@ 4001: "Unsupported Agent Mode", 4002: "Missing Shaper", 4003: "Wrong Agent Manager Mode", - 4004: "Wrong Agent Mode", - 4005: "Store Misalignment Error" + 4004: "Store Misalignment Error" } diff --git a/maro/utils/exception/rl_toolkit_exception.py b/maro/utils/exception/rl_toolkit_exception.py index 33f29ab4f..93684b8eb 100644 --- a/maro/utils/exception/rl_toolkit_exception.py +++ b/maro/utils/exception/rl_toolkit_exception.py @@ -4,7 +4,7 @@ from maro.utils.exception import MAROException -class UnsupportedAgentModeError(MAROException): +class UnsupportedAgentManagerModeError(MAROException): """Unsupported agent mode.""" def __init__(self, msg: str = None): super().__init__(4001, msg) @@ -22,14 +22,8 @@ def __init__(self, msg: str = None): super().__init__(4003, msg) -class WrongAgentModeError(MAROException): - """Wrong agent mode.""" - def __init__(self, msg: str = None): - super().__init__(4004, msg) - - class StoreMisalignmentError(MAROException): """Raised when a ``put`` operation on a ``ColumnBasedStore`` would cause the underlying lists to have different sizes.""" def __init__(self, msg: str = None): - super().__init__(4005, msg) + super().__init__(4004, msg) From f34343d3e41923c3222c86e4429fd0982b6fbf9d Mon Sep 17 00:00:00 2001 From: ysqyang Date: Wed, 28 Oct 2020 10:37:07 +0800 Subject: [PATCH 066/337] fixed cim example according to rl toolkit changes --- examples/cim/dqn/components/agent_manager.py | 6 +----- examples/cim/dqn/config.yml | 9 +++++---- examples/cim/dqn/dist_actor.py | 9 +-------- examples/cim/dqn/dist_learner.py | 10 ++-------- examples/cim/dqn/single_process_launcher.py | 16 +++++----------- 5 files changed, 14 insertions(+), 36 deletions(-) diff --git a/examples/cim/dqn/components/agent_manager.py b/examples/cim/dqn/components/agent_manager.py index d9fb449f1..b437b651f 100644 --- a/examples/cim/dqn/components/agent_manager.py +++ b/examples/cim/dqn/components/agent_manager.py @@ -51,12 +51,8 @@ def train(self, experiences_by_agent, performance=None): # store experiences for each agent for agent_id, exp in experiences_by_agent.items(): - exp.update({"loss": [1e8] * len(exp[next(iter(exp))])}) + exp.update({"loss": [1e8] * len(list(exp.items())[0][1])}) self.agent_dict[agent_id].store_experiences(exp) for agent in self.agent_dict.values(): agent.train() - - # update exploration rates - if self._explorer is not None: - self._explorer.update(performance) diff --git a/examples/cim/dqn/config.yml b/examples/cim/dqn/config.yml index 2f435ae52..6af3597ea 100644 --- a/examples/cim/dqn/config.yml +++ b/examples/cim/dqn/config.yml @@ -3,7 +3,7 @@ env: topology: "toy.4p_ssdd_l0.0" durations: 1120 general: - total_training_episodes: 500 # max episode + max_episode: 500 # max episode state_shaping: look_back: 7 max_ports_downstream: 2 @@ -30,9 +30,10 @@ experience_shaping: shortage_factor: 1.0 time_decay_factor: 0.97 exploration: - epsilon_range: [0.0, 0.4] - split_point: [0.5, 0.8] - with_cache: true + min_eps: .0 + max_eps: .4 + eps_split: 0.32 + progress_split: 0.5 agents: algorithm: num_actions: 21 diff --git a/examples/cim/dqn/dist_actor.py b/examples/cim/dqn/dist_actor.py index b4929c557..181d55633 100644 --- a/examples/cim/dqn/dist_actor.py +++ b/examples/cim/dqn/dist_actor.py @@ -4,7 +4,7 @@ import numpy as np from maro.simulator import Env -from maro.rl import AgentManagerMode, SimpleActor, ActorWorker, KStepExperienceShaper, TwoPhaseLinearExplorer +from maro.rl import AgentManagerMode, SimpleActor, ActorWorker, KStepExperienceShaper from components.action_shaper import CIMActionShaper from components.agent_manager import create_dqn_agents, DQNAgentManager @@ -36,12 +36,6 @@ def set_input_dim(): **config.experience_shaping.k_step ) - exploration_config = { - "epsilon_range_dict": {"_all_": config.exploration.epsilon_range}, - "split_point_dict": {"_all_": config.exploration.split_point}, - "with_cache": config.exploration.with_cache - } - explorer = TwoPhaseLinearExplorer(agent_id_list, config.general.total_training_episodes, **exploration_config) agent_manager = DQNAgentManager( name="distributed_cim_actor", mode=AgentManagerMode.INFERENCE, @@ -49,7 +43,6 @@ def set_input_dim(): state_shaper=state_shaper, action_shaper=action_shaper, experience_shaper=experience_shaper, - explorer=explorer ) proxy_params = { "group_name": config.distributed.group_name, diff --git a/examples/cim/dqn/dist_learner.py b/examples/cim/dqn/dist_learner.py index 7406d0949..7793915fa 100644 --- a/examples/cim/dqn/dist_learner.py +++ b/examples/cim/dqn/dist_learner.py @@ -25,17 +25,10 @@ def set_input_dim(): env = Env(config.env.scenario, config.env.topology, durations=config.env.durations) agent_id_list = [str(agent_id) for agent_id in env.agent_idx_list] - exploration_config = { - "epsilon_range_dict": {"_all_": config.exploration.epsilon_range}, - "split_point_dict": {"_all_": config.exploration.split_point}, - "with_cache": config.exploration.with_cache - } - explorer = TwoPhaseLinearExplorer(agent_id_list, config.general.total_training_episodes, **exploration_config) agent_manager = DQNAgentManager( name="distributed_cim_learner", mode=AgentManagerMode.TRAIN, agent_dict=create_dqn_agents(agent_id_list, config.agents), - explorer=explorer ) proxy_params = { @@ -47,9 +40,10 @@ def set_input_dim(): learner = SimpleLearner( trainable_agents=agent_manager, actor=ActorProxy(proxy_params=proxy_params), + explorer=TwoPhaseLinearExplorer(**config.exploration), logger=Logger("distributed_cim_learner", auto_timestamp=False) ) - learner.train(total_episodes=config.general.total_training_episodes) + learner.train(config.general.max_episode) learner.test() learner.dump_models(os.path.join(os.getcwd(), "models")) diff --git a/examples/cim/dqn/single_process_launcher.py b/examples/cim/dqn/single_process_launcher.py index c7ed7647c..53c153538 100644 --- a/examples/cim/dqn/single_process_launcher.py +++ b/examples/cim/dqn/single_process_launcher.py @@ -47,13 +47,6 @@ def set_input_dim(): **config.experience_shaping.k_step ) - exploration_config = { - "epsilon_range_dict": {"_all_": config.exploration.epsilon_range}, - "split_point_dict": {"_all_": config.exploration.split_point}, - "with_cache": config.exploration.with_cache - } - explorer = TwoPhaseLinearExplorer(agent_id_list, config.general.total_training_episodes, **exploration_config) - # Step 3: Create an agent manager. agent_manager = DQNAgentManager( name="cim_learner", @@ -61,17 +54,18 @@ def set_input_dim(): agent_dict=create_dqn_agents(agent_id_list, config.agents), state_shaper=state_shaper, action_shaper=action_shaper, - experience_shaper=experience_shaper, - explorer=explorer + experience_shaper=experience_shaper ) # Step 4: Create an actor and a learner to start the training process. actor = SimpleActor(env=env, inference_agents=agent_manager) learner = SimpleLearner( - trainable_agents=agent_manager, actor=actor, + trainable_agents=agent_manager, + actor=actor, + explorer=TwoPhaseLinearExplorer(**config.exploration), logger=Logger("single_host_cim_learner", auto_timestamp=False) ) - learner.train(total_episodes=config.general.total_training_episodes) + learner.train(max_episode=config.general.max_episodes) learner.test() learner.dump_models(os.path.join(os.getcwd(), "models")) From 2d944afc41ac1b78cb75dc14124297c381239ea0 Mon Sep 17 00:00:00 2001 From: ysqyang Date: Wed, 28 Oct 2020 14:40:19 +0800 Subject: [PATCH 067/337] fixed some more PR comments --- docs/source/examples/multi_agent_dqn_cim.rst | 2 +- examples/cim/dqn/components/agent.py | 12 ++++ examples/cim/dqn/components/agent_manager.py | 4 +- examples/cim/dqn/config.yml | 2 +- maro/rl/__init__.py | 6 +- maro/rl/agent/abs_agent_manager.py | 10 ++- maro/rl/algorithms/dqn.py | 10 +-- .../abs_early_stopping_checker.py | 15 ++++- .../simple_early_stopping_checker.py | 9 +++ maro/rl/explorer/abs_explorer.py | 5 +- maro/rl/explorer/simple_explorer.py | 21 +++++- maro/rl/learner/simple_learner.py | 23 ++++--- ...ision_layers.py => fully_connected_net.py} | 2 +- maro/rl/models/representation_layers.py | 67 ------------------- .../rl_formulation.ipynb | 6 +- 15 files changed, 91 insertions(+), 103 deletions(-) rename maro/rl/models/{decision_layers.py => fully_connected_net.py} (98%) delete mode 100644 maro/rl/models/representation_layers.py diff --git a/docs/source/examples/multi_agent_dqn_cim.rst b/docs/source/examples/multi_agent_dqn_cim.rst index 6c52b74f2..d54df810b 100644 --- a/docs/source/examples/multi_agent_dqn_cim.rst +++ b/docs/source/examples/multi_agent_dqn_cim.rst @@ -142,7 +142,7 @@ the DQN algorithm and an experience pool for each agent. set_seeds(config.agents.seed) num_actions = config.agents.algorithm.num_actions for agent_id in self._agent_id_list: - eval_model = LearningModel(decision_layers=MLPDecisionLayers(name=f'{agent_id}.policy', + eval_model = LearningModel(decision_layers=MLPFullyConnectedNet(name=f'{agent_id}.policy', input_dim=self._state_shaper.dim, output_dim=num_actions, **config.agents.algorithm.model) diff --git a/examples/cim/dqn/components/agent.py b/examples/cim/dqn/components/agent.py index 7f6f24255..9cf358166 100644 --- a/examples/cim/dqn/components/agent.py +++ b/examples/cim/dqn/components/agent.py @@ -7,6 +7,13 @@ class CIMAgent(AbsAgent): + """Implementation of AbsAgent for the DQN algorithm. + + Args: + min_experiences_to_train: minimum number of experiences required for training. + num_batches: number of batches to train the DQN model on per call to ``train``. + batch_size: mini-batch size. + """ def __init__(self, name, algorithm, experience_pool: ColumnBasedStore, min_experiences_to_train, num_batches, batch_size): super().__init__(name, algorithm, experience_pool) @@ -15,6 +22,11 @@ def __init__(self, name, algorithm, experience_pool: ColumnBasedStore, min_exper self._batch_size = batch_size def train(self): + """Implementation of the training loop for DQN. + + Experiences are sampled using their TD errors as weights. After training, the new TD errors are updated + in the experience pool. + """ if len(self._experience_pool) < self._min_experiences_to_train: return diff --git a/examples/cim/dqn/components/agent_manager.py b/examples/cim/dqn/components/agent_manager.py index b437b651f..93301cd6e 100644 --- a/examples/cim/dqn/components/agent_manager.py +++ b/examples/cim/dqn/components/agent_manager.py @@ -5,7 +5,7 @@ from torch.optim import RMSprop from .agent import CIMAgent -from maro.rl import SimpleAgentManager, LearningModel, DecisionLayers, DQN, DQNHyperParams, ColumnBasedStore +from maro.rl import SimpleAgentManager, LearningModel, FullyConnectedNet, DQN, DQNHyperParams, ColumnBasedStore from maro.utils import set_seeds @@ -15,7 +15,7 @@ def create_dqn_agents(agent_id_list, config): agent_dict = {} for agent_id in agent_id_list: eval_model = LearningModel( - decision_layers=DecisionLayers( + decision_layers=FullyConnectedNet( name=f'{agent_id}.policy', input_dim=config.algorithm.input_dim, output_dim=num_actions, diff --git a/examples/cim/dqn/config.yml b/examples/cim/dqn/config.yml index 6af3597ea..187a0a5b3 100644 --- a/examples/cim/dqn/config.yml +++ b/examples/cim/dqn/config.yml @@ -49,7 +49,7 @@ agents: lr: 0.05 hyper_parameters: reward_decay: .0 - num_training_rounds_per_target_replacement: 5 + target_replacement_period: 5 tau: 0.1 experience_pool: capacity: -1 diff --git a/maro/rl/__init__.py b/maro/rl/__init__.py index c51793c92..28c26caa8 100644 --- a/maro/rl/__init__.py +++ b/maro/rl/__init__.py @@ -11,8 +11,7 @@ from maro.rl.algorithms.abs_algorithm import AbsAlgorithm from maro.rl.algorithms.dqn import DQN, DQNHyperParams from maro.rl.models.learning_model import LearningModel -from maro.rl.models.representation_layers import RepresentationLayers -from maro.rl.models.decision_layers import DecisionLayers +from maro.rl.models.fully_connected_net import FullyConnectedNet from maro.rl.storage.abs_store import AbsStore from maro.rl.storage.column_based_store import ColumnBasedStore from maro.rl.storage.utils import OverwriteType @@ -39,8 +38,7 @@ "DQN", "DQNHyperParams", "LearningModel", - "RepresentationLayers", - "DecisionLayers", + "FullyConnectedNet", "AbsStore", "ColumnBasedStore", "OverwriteType", diff --git a/maro/rl/agent/abs_agent_manager.py b/maro/rl/agent/abs_agent_manager.py index 8e0eebc8c..2e84a1c69 100644 --- a/maro/rl/agent/abs_agent_manager.py +++ b/maro/rl/agent/abs_agent_manager.py @@ -62,14 +62,18 @@ def choose_action(self, *args, **kwargs): @abstractmethod def on_env_feedback(self, *args, **kwargs): - """Do things after a feedback is received from the environment following an action.""" + """Processing logic after receiving feedback from the environment is implemented here. + + See ``SimpleAgentManager`` for example. + """ return NotImplemented @abstractmethod def post_process(self, *args, **kwargs): - """Do things after an episode is finished. + """Processing logic after an episode is finished. - These things may involve shaping experiences and resetting stateful objects. + These things may involve generating experiences and resetting stateful objects. See ``SimpleAgentManager`` + for example. """ return NotImplemented diff --git a/maro/rl/algorithms/dqn.py b/maro/rl/algorithms/dqn.py index ca1fe5a14..cb9de8d70 100644 --- a/maro/rl/algorithms/dqn.py +++ b/maro/rl/algorithms/dqn.py @@ -15,17 +15,17 @@ class DQNHyperParams: Args: num_actions (int): number of possible actions reward_decay (float): reward decay as defined in standard RL terminology - num_training_rounds_per_target_replacement (int): number of training frequency of target model replacement + target_replacement_period (int): number of training frequency of target model replacement tau (float): soft update coefficient, e.g., target_model = tau * eval_model + (1-tau) * target_model """ - __slots__ = ["num_actions", "reward_decay", "num_training_rounds_per_target_replacement", "tau"] + __slots__ = ["num_actions", "reward_decay", "target_replacement_period", "tau"] def __init__( - self, num_actions: int, reward_decay: float, num_training_rounds_per_target_replacement: int, tau: float = 1.0 + self, num_actions: int, reward_decay: float, target_replacement_period: int, tau: float = 1.0 ): self.num_actions = num_actions self.reward_decay = reward_decay - self.num_training_rounds_per_target_replacement = num_training_rounds_per_target_replacement + self.target_replacement_period = target_replacement_period self.tau = tau @@ -95,7 +95,7 @@ def train(self, states: np.ndarray, actions: np.ndarray, rewards: np.ndarray, ne loss.backward() self._optimizer.step() self._train_cnt += 1 - if self._train_cnt % self._hyper_params.num_training_rounds_per_target_replacement == 0: + if self._train_cnt % self._hyper_params.target_replacement_period == 0: self._update_target_model() return loss.detach().numpy() diff --git a/maro/rl/early_stopping/abs_early_stopping_checker.py b/maro/rl/early_stopping/abs_early_stopping_checker.py index 8a30529cf..733a00aa1 100644 --- a/maro/rl/early_stopping/abs_early_stopping_checker.py +++ b/maro/rl/early_stopping/abs_early_stopping_checker.py @@ -5,9 +5,22 @@ class AbsEarlyStoppingChecker(ABC): + """Class that checks for early stopping conditions. + + Implementations of this abstract class usually involve user-defined early stopping conditions. + """ def __init__(self): pass @abstractmethod - def __call__(self, performance_history): + def __call__(self, performance_history) -> bool: + """Check whether the early stopping condition (defined in the class) is met. + + Args: + performance_history: History of performances (from actors) used to check whether the early stopping + condition is satisfied. + + Returns: + A boolean value indicating whether early stopping should be triggered. + """ return NotImplemented diff --git a/maro/rl/early_stopping/simple_early_stopping_checker.py b/maro/rl/early_stopping/simple_early_stopping_checker.py index 734eea5b9..9cf65fccc 100644 --- a/maro/rl/early_stopping/simple_early_stopping_checker.py +++ b/maro/rl/early_stopping/simple_early_stopping_checker.py @@ -9,6 +9,15 @@ class SimpleEarlyStoppingChecker(AbsEarlyStoppingChecker): + """Simple early stopping checker based on the mean and standard deviation of the last k performance records. + + Args: + last_k (int): Number of the latest performance records to check for early stopping. + performance_metric_func (Callable): A function to obtain the metric from a performance record to be evaluated + against a threshold value. + threshold (float): The threshold value against which the early stopping metric is compared. The early stopping + condition is satisfied if the metric obtained using the ``performance_metric_func`` is below this threshold. + """ def __init__(self, last_k: int, performance_metric_func: Callable, threshold: float): super().__init__() self._last_k = last_k diff --git a/maro/rl/explorer/abs_explorer.py b/maro/rl/explorer/abs_explorer.py index db9e8ed5b..00c4e620a 100644 --- a/maro/rl/explorer/abs_explorer.py +++ b/maro/rl/explorer/abs_explorer.py @@ -5,9 +5,8 @@ class AbsExplorer(ABC): - """Abstract explorer class. + """Abstract explorer class for generating exploration rates. - An explorer is responsible for generating exploration rates. """ def __init__(self): pass @@ -15,6 +14,6 @@ def __init__(self): # TODO: performance: summary -> total perf (current version), details -> per-agent perf @abstractmethod def generate_epsilon(self, current_ep: int, max_ep: int, performance_history=None): - """Generate an exploration rate. + """Generate an exploration rate based on the performance history. """ return NotImplemented diff --git a/maro/rl/explorer/simple_explorer.py b/maro/rl/explorer/simple_explorer.py index 575ec5ceb..1423b2aff 100644 --- a/maro/rl/explorer/simple_explorer.py +++ b/maro/rl/explorer/simple_explorer.py @@ -5,7 +5,12 @@ class LinearExplorer(AbsExplorer): - """A simple linear exploration scheme.""" + """Exploration schedule where the exploration rate decreases with the number of episodes in a linear fashion. + + Args: + max_eps (float): Maximum exploration rate, i.e., the exploration rate for the first episode. + min_eps (float): Minimum exploration rate, i.e., the exploration rate for the last episode. + """ def __init__(self, max_eps: float, min_eps: float = .0): super().__init__() self._max_eps = max_eps @@ -16,9 +21,21 @@ def generate_epsilon(self, current_ep, max_ep, performance_history=None): class TwoPhaseLinearExplorer(AbsExplorer): - """An exploration scheme that consists of two linear schedules separated by a split point.""" + """Exploration schedule that consists of two linear schedules separated by a split point. + + Args: + progress_split (float): The point where the switch from the first linear schedule to the second occurs. + Here "point" means the percentage of training loop completion, i.e., current_episode / max_episode, + which means it must be a floating point number between 0 and 1.0. + eps_split (float): The exploration rate where the switch from the first linear schedule to the second occurs. + In other words, this is the exploration rate where the first linear schedule ends and the second begins. + max_eps (float): Maximum exploration rate, i.e., the exploration rate for the first episode. + min_eps (float): Minimum exploration rate, i.e., the exploration rate for the last episode. + """ def __init__(self, progress_split: float, eps_split: float, max_eps: float, min_eps: float = 0): super().__init__() + if progress_split > 1.0 or progress_split < 0.0: + raise ValueError("progress_split must be between 0 and 1.0") self._progress_split = progress_split self._eps_split = eps_split self._max_eps = max_eps diff --git a/maro/rl/learner/simple_learner.py b/maro/rl/learner/simple_learner.py index c684e930e..9711a58a7 100644 --- a/maro/rl/learner/simple_learner.py +++ b/maro/rl/learner/simple_learner.py @@ -1,30 +1,33 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT license. -from typing import Callable +from typing import Callable, Union import warnings from .abs_learner import AbsLearner from maro.rl.actor.simple_actor import SimpleActor from maro.rl.agent.simple_agent_manager import SimpleAgentManager from maro.rl.explorer.abs_explorer import AbsExplorer -from maro.utils import DummyLogger +from maro.rl.dist_topologies.single_learner_multi_actor_sync_mode import ActorProxy +from maro.utils import Logger, DummyLogger class SimpleLearner(AbsLearner): """A simple implementation of ``AbsLearner``. Args: - trainable_agents (AbsAgentManager): an AgentManager instance that manages all agents. - actor (Actor or ActorProxy): an Actor or VectorActorProxy instance. - logger: used for logging important messages. + trainable_agents (AbsAgentManager): An AgentManager instance that manages all agents. + actor (SimpleActor or ActorProxy): An SimpleActor or ActorProxy instance responsible for performing roll-outs + (environment sampling). + explorer (AbsExplorer): An explorer instance responsible for generating exploration rates. Defaults to None. + logger (Logger): Used to log important messages. """ def __init__( self, trainable_agents: SimpleAgentManager, - actor, + actor: Union[SimpleActor, ActorProxy], explorer: AbsExplorer = None, - logger=DummyLogger() + logger: Logger = DummyLogger() ): super().__init__() self._trainable_agents = trainable_agents @@ -43,7 +46,7 @@ def _get_epsilons(self, current_ep, max_ep): return None def _sample(self, ep, max_ep): - """One episode""" + """Perform one episode of environment sampling through actor roll-out.""" model_dict = None if self._is_shared_agent_instance() else self._trainable_agents.dump_models() epsilon_dict = self._get_epsilons(ep, max_ep) performance, exp_by_agent = self._actor.roll_out(model_dict=model_dict, epsilon_dict=epsilon_dict) @@ -56,8 +59,8 @@ def train(self, max_episode: int, early_stopping_checker: Callable = None): Args: max_episode (int): number of episodes to be run. If negative, the training loop will run forever unless an ``early_stopping_checker`` is provided and the early stopping condition is met. - early_stopping_checker (Callable): A Callable object to judge whether the training loop should be ended - based on the latest performances. + early_stopping_checker (Callable): A Callable object to determine whether the training loop should be + terminated based on the latest performances. """ if max_episode < 0: if early_stopping_checker is None: diff --git a/maro/rl/models/decision_layers.py b/maro/rl/models/fully_connected_net.py similarity index 98% rename from maro/rl/models/decision_layers.py rename to maro/rl/models/fully_connected_net.py index d1017c299..ef94cbac0 100644 --- a/maro/rl/models/decision_layers.py +++ b/maro/rl/models/fully_connected_net.py @@ -6,7 +6,7 @@ import torch.nn as nn -class DecisionLayers(nn.Module): +class FullyConnectedNet(nn.Module): """NN model to compute state or action values. Fully connected network with optional batch normalization, activation and dropout components. diff --git a/maro/rl/models/representation_layers.py b/maro/rl/models/representation_layers.py deleted file mode 100644 index 3e7812ce2..000000000 --- a/maro/rl/models/representation_layers.py +++ /dev/null @@ -1,67 +0,0 @@ -# Copyright (c) Microsoft Corporation. -# Licensed under the MIT license. - -import torch.nn as nn - - -class RepresentationLayers(nn.Module): - def __init__(self, name: str, input_dim: int, hidden_dims: [int], output_dim: int, dropout_p: float): - """Deep Q network. - - Choose multi-layer full connection with dropout as the basic network architecture. - - Args: - name (str): Network name. - input_dim (int): Network input dimension. - hidden_dims ([int]): Dimensions of hidden layers. Its length is the number of hidden layers. - output_dim (int): Network output dimension. - dropout_p (float): Dropout parameter. - """ - super().__init__() - self._name = name - self._dropout_p = dropout_p - self._input_dim = input_dim - self._hidden_dims = hidden_dims if hidden_dims is not None else [] - self._output_dim = output_dim - self._layers = self._build_layers([input_dim] + self._hidden_dims) - if len(self._hidden_dims) == 0: - self._head = nn.Linear(self._input_dim, self._output_dim) - else: - self._head = nn.Linear(hidden_dims[-1], self._output_dim) - self._net = nn.Sequential(*self._layers, self._head) - - def forward(self, x): - return self._net(x).double() - - @property - def input_dim(self): - return self._input_dim - - @property - def name(self): - return self._name - - @property - def output_dim(self): - return self._output_dim - - def _build_basic_layer(self, input_dim, output_dim): - """Build basic layer. - - BN -> Linear -> LeakyReLU -> Dropout - """ - return nn.Sequential( - nn.Linear(input_dim, output_dim), - nn.LeakyReLU(), - nn.Dropout(p=self._dropout_p) - ) - - def _build_layers(self, layer_dims: []): - """Build multi basic layer. - - BasicLayer1 -> BasicLayer2 -> ... - """ - layers = [] - for input_dim, output_dim in zip(layer_dims, layer_dims[1:]): - layers.append(self._build_basic_layer(input_dim, output_dim)) - return layers diff --git a/notebooks/container_inventory_management/rl_formulation.ipynb b/notebooks/container_inventory_management/rl_formulation.ipynb index 6767a6de8..38b273999 100644 --- a/notebooks/container_inventory_management/rl_formulation.ipynb +++ b/notebooks/container_inventory_management/rl_formulation.ipynb @@ -228,7 +228,7 @@ "from torch.nn.functional import smooth_l1_loss\n", "from torch.optim import RMSprop\n", "\n", - "from maro.rl import AbsAgentManager, LearningModel, MLPDecisionLayers, DQN, DQNHyperParams, ColumnBasedStore\n", + "from maro.rl import AbsAgentManager, LearningModel, MLPFullyConnectedNet, DQN, DQNHyperParams, ColumnBasedStore\n", "\n", "\n", "num_actions = 21\n", @@ -237,7 +237,7 @@ "class DQNAgentManager(AbsAgentManager):\n", " def _assemble(self, agent_dict):\n", " for agent_id in self._agent_id_list:\n", - " eval_model = LearningModel(decision_layers=MLPDecisionLayers(name=f'{agent_id}.policy',\n", + " eval_model = LearningModel(decision_layers=MLPFullyConnectedNet(name=f'{agent_id}.policy',\n", " input_dim=self._state_shaper.dim,\n", " output_dim=num_actions,\n", " hidden_dims=[256, 128, 64],\n", @@ -248,7 +248,7 @@ " optimizer_opt=(RMSprop, {\"lr\": 0.05}),\n", " loss_func_dict={\"eval\": smooth_l1_loss},\n", " hyper_params=DQNHyperParams(num_actions=num_actions, reward_decay=.0,\n", - " num_training_rounds_per_target_replacement=5, tau=0.1)\n", + " target_replacement_period=5, tau=0.1)\n", " )\n", "\n", " experience_pool = ColumnBasedStore()\n", From 095f77659da42698feb497826caf8941e0bec106 Mon Sep 17 00:00:00 2001 From: ysqyang Date: Wed, 28 Oct 2020 15:12:21 +0800 Subject: [PATCH 068/337] rewrote multi_process_launcher to eliminate the distributed section in config --- examples/cim/dqn/config.yml | 9 ---- examples/cim/dqn/dist_actor.py | 8 ++-- examples/cim/dqn/dist_learner.py | 6 +-- examples/cim/dqn/multi_process_launcher.py | 23 ++++++---- .../single_learner_multi_actor_sync_mode.py | 46 +++++++++++-------- maro/rl/storage/column_based_store.py | 2 +- 6 files changed, 50 insertions(+), 44 deletions(-) diff --git a/examples/cim/dqn/config.yml b/examples/cim/dqn/config.yml index 187a0a5b3..4a02d1844 100644 --- a/examples/cim/dqn/config.yml +++ b/examples/cim/dqn/config.yml @@ -58,12 +58,3 @@ agents: num_batches: 10 # number of times the algorithm's step() method is called batch_size: 128 seed: 1024 # for reproducibility -distributed: - group_name: "dqn_distributed_test" - actor: - peer: {"actor": 1} - learner: - peer: {"actor_worker": 1} - redis: - host_name: "localhost" - port: 6379 diff --git a/examples/cim/dqn/dist_actor.py b/examples/cim/dqn/dist_actor.py index 181d55633..18da573d9 100644 --- a/examples/cim/dqn/dist_actor.py +++ b/examples/cim/dqn/dist_actor.py @@ -1,6 +1,8 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT license. +import os + import numpy as np from maro.simulator import Env @@ -45,9 +47,9 @@ def set_input_dim(): experience_shaper=experience_shaper, ) proxy_params = { - "group_name": config.distributed.group_name, - "expected_peers": config.distributed.actor.peer, - "redis_address": (config.distributed.redis.host_name, config.distributed.redis.port) + "group_name": os.environ["GROUP"], + "expected_peers": {"learner": 1}, + "redis_address": ("localhost", 6379) } actor_worker = ActorWorker( local_actor=SimpleActor(env=env, inference_agents=agent_manager), diff --git a/examples/cim/dqn/dist_learner.py b/examples/cim/dqn/dist_learner.py index 7793915fa..8671649fe 100644 --- a/examples/cim/dqn/dist_learner.py +++ b/examples/cim/dqn/dist_learner.py @@ -32,9 +32,9 @@ def set_input_dim(): ) proxy_params = { - "group_name": config.distributed.group_name, - "expected_peers": config.distributed.learner.peer, - "redis_address": (config.distributed.redis.host_name, config.distributed.redis.port) + "group_name": os.environ["GROUP"], + "expected_peers": {"actor": int(os.environ["NUM_ACTORS"])}, + "redis_address": ("localhost", 6379) } learner = SimpleLearner( diff --git a/examples/cim/dqn/multi_process_launcher.py b/examples/cim/dqn/multi_process_launcher.py index 9ec989550..4450b4b3b 100644 --- a/examples/cim/dqn/multi_process_launcher.py +++ b/examples/cim/dqn/multi_process_launcher.py @@ -5,19 +5,22 @@ This script is used to debug distributed algorithm in single host multi-process mode. """ +import argparse import os -from components.config import config +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("group_name", help="group name") + parser.add_argument("num_actors", help="number of actors") + args = parser.parse_args() -ACTOR_NUM = config.distributed.learner.peer["actor_worker"] # must be same as in config -LEARNER_NUM = config.distributed.actor.peer["actor"] + learner_path = f"{os.path.split(os.path.realpath(__file__))[0]}/dist_learner.py &" + actor_path = f"{os.path.split(os.path.realpath(__file__))[0]}/dist_actor.py &" -learner_path = f"{os.path.split(os.path.realpath(__file__))[0]}/dist_learner.py &" -actor_path = f"{os.path.split(os.path.realpath(__file__))[0]}/dist_actor.py &" + # Launch the learner process + os.system(f"GROUP={args.group_name} NUM_ACTORS={args.num_actors} python " + learner_path) -for l_num in range(LEARNER_NUM): - os.system(f"python " + learner_path) - -for a_num in range(ACTOR_NUM): - os.system(f"python " + actor_path) + # Launch the actor processes + for _ in range(args.num_actors): + os.system(f"GROUP={args.group_name} python " + actor_path) diff --git a/maro/rl/dist_topologies/single_learner_multi_actor_sync_mode.py b/maro/rl/dist_topologies/single_learner_multi_actor_sync_mode.py index 8576d2456..c8c73f8f6 100644 --- a/maro/rl/dist_topologies/single_learner_multi_actor_sync_mode.py +++ b/maro/rl/dist_topologies/single_learner_multi_actor_sync_mode.py @@ -23,7 +23,7 @@ class ActorProxy(object): proxy_params: Parameters for instantiating a ``Proxy`` instance. """ def __init__(self, proxy_params): - self._proxy = Proxy(component_type="actor", **proxy_params) + self._proxy = Proxy(component_type="learner", **proxy_params) def roll_out(self, model_dict: dict = None, epsilon_dict: dict = None, done: bool = False, return_details: bool = True): @@ -46,19 +46,24 @@ def roll_out(self, model_dict: dict = None, epsilon_dict: dict = None, done: boo Performance and per-agent experiences from the remote actor. """ if done: - self._proxy.ibroadcast(tag=MessageTag.ROLLOUT, - session_type=SessionType.NOTIFICATION, - payload={PayloadKey.DONE: True}) + self._proxy.ibroadcast( + tag=MessageTag.ROLLOUT, + session_type=SessionType.NOTIFICATION, + payload={PayloadKey.DONE: True} + ) return None, None else: performance, exp_by_agent = {}, {} payloads = [(peer, {PayloadKey.MODEL: model_dict, PayloadKey.EPSILON: epsilon_dict, PayloadKey.RETURN_DETAILS: return_details}) - for peer in self._proxy.peers["actor_worker"]] + for peer in self._proxy.peers["actor"]] # TODO: double check when ack enable - replies = self._proxy.scatter(tag=MessageTag.ROLLOUT, session_type=SessionType.TASK, - destination_payload_list=payloads) + replies = self._proxy.scatter( + tag=MessageTag.ROLLOUT, + session_type=SessionType.TASK, + destination_payload_list=payloads + ) for msg in replies: performance[msg.source] = msg.payload[PayloadKey.PERFORMANCE] if msg.payload[PayloadKey.EXPERIENCE] is not None: @@ -80,9 +85,9 @@ class ActorWorker(object): """ def __init__(self, local_actor: AbsActor, proxy_params): self._local_actor = local_actor - self._proxy = Proxy(component_type="actor_worker", **proxy_params) + self._proxy = Proxy(component_type="actor", **proxy_params) self._registry_table = RegisterTable(self._proxy.get_peers) - self._registry_table.register_event_handler("actor:rollout:1", self.on_rollout_request) + self._registry_table.register_event_handler("learner:rollout:1", self.on_rollout_request) def on_rollout_request(self, message): """Perform local roll-out and send the results back to the request sender. @@ -94,15 +99,20 @@ def on_rollout_request(self, message): if data.get(PayloadKey.DONE, False): sys.exit(0) - performance, experiences = self._local_actor.roll_out(model_dict=data[PayloadKey.MODEL], - epsilon_dict=data[PayloadKey.EPSILON], - return_details=data[PayloadKey.RETURN_DETAILS]) - - self._proxy.reply(received_message=message, - tag=MessageTag.UPDATE, - payload={PayloadKey.PERFORMANCE: performance, - PayloadKey.EXPERIENCE: experiences} - ) + performance, experiences = self._local_actor.roll_out( + model_dict=data[PayloadKey.MODEL], + epsilon_dict=data[PayloadKey.EPSILON], + return_details=data[PayloadKey.RETURN_DETAILS] + ) + + self._proxy.reply( + received_message=message, + tag=MessageTag.UPDATE, + payload={ + PayloadKey.PERFORMANCE: performance, + PayloadKey.EXPERIENCE: experiences + } + ) def launch(self): """Entry point method. diff --git a/maro/rl/storage/column_based_store.py b/maro/rl/storage/column_based_store.py index 4888f6e5f..c2a55b0ca 100644 --- a/maro/rl/storage/column_based_store.py +++ b/maro/rl/storage/column_based_store.py @@ -55,7 +55,7 @@ def __getitem__(self, index: int): return {k: lst[index] for k, lst in self._store.items()} def __getstate__(self): - """A small modification to make the object picklable. + """A patch to make the object picklable. Using the default ``__dict__`` would make the object unpicklable due to the lambda function involved in the ``defaultdict`` definition of the ``_store`` attribute. From 15747d9e672561147f2d76c52817360a017ca835 Mon Sep 17 00:00:00 2001 From: ysqyang Date: Wed, 28 Oct 2020 15:20:33 +0800 Subject: [PATCH 069/337] 1. fixed a typo; 2. added logging before early stopping --- examples/cim/dqn/single_process_launcher.py | 2 +- maro/rl/learner/simple_learner.py | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/examples/cim/dqn/single_process_launcher.py b/examples/cim/dqn/single_process_launcher.py index 53c153538..61257d04b 100644 --- a/examples/cim/dqn/single_process_launcher.py +++ b/examples/cim/dqn/single_process_launcher.py @@ -65,7 +65,7 @@ def set_input_dim(): explorer=TwoPhaseLinearExplorer(**config.exploration), logger=Logger("single_host_cim_learner", auto_timestamp=False) ) - learner.train(max_episode=config.general.max_episodes) + learner.train(max_episode=config.general.max_episode) learner.test() learner.dump_models(os.path.join(os.getcwd(), "models")) diff --git a/maro/rl/learner/simple_learner.py b/maro/rl/learner/simple_learner.py index 9711a58a7..0faf4da6e 100644 --- a/maro/rl/learner/simple_learner.py +++ b/maro/rl/learner/simple_learner.py @@ -70,6 +70,7 @@ def train(self, max_episode: int, early_stopping_checker: Callable = None): performance, exp_by_agent = self._sample(episode, max_episode) self._performance_history.append(performance) if early_stopping_checker is not None and early_stopping_checker(self._performance_history): + self._logger.info("Early stopping condition satisfied. Training complete.") break episode += 1 else: @@ -77,6 +78,7 @@ def train(self, max_episode: int, early_stopping_checker: Callable = None): performance, exp_by_agent = self._sample(episode, max_episode) self._performance_history.append(performance) if early_stopping_checker is not None and early_stopping_checker(self._performance_history): + self._logger.info("Early stopping condition satisfied. Training complete.") break self._trainable_agents.train(exp_by_agent) From ae2a54a5917679606c7845dad2eb595dc4e05de8 Mon Sep 17 00:00:00 2001 From: ysqyang Date: Wed, 28 Oct 2020 15:22:55 +0800 Subject: [PATCH 070/337] fixed a bug --- examples/cim/dqn/components/agent_manager.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/cim/dqn/components/agent_manager.py b/examples/cim/dqn/components/agent_manager.py index 93301cd6e..67367a657 100644 --- a/examples/cim/dqn/components/agent_manager.py +++ b/examples/cim/dqn/components/agent_manager.py @@ -51,7 +51,7 @@ def train(self, experiences_by_agent, performance=None): # store experiences for each agent for agent_id, exp in experiences_by_agent.items(): - exp.update({"loss": [1e8] * len(list(exp.items())[0][1])}) + exp.update({"loss": [1e8] * len(list(exp.values())[0])}) self.agent_dict[agent_id].store_experiences(exp) for agent in self.agent_dict.values(): From 2c2dc45bb3295551d84bf6b007757966bc87b9a2 Mon Sep 17 00:00:00 2001 From: ysqyang Date: Wed, 28 Oct 2020 15:30:30 +0800 Subject: [PATCH 071/337] fixed a bug --- maro/rl/algorithms/dqn.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/maro/rl/algorithms/dqn.py b/maro/rl/algorithms/dqn.py index cb9de8d70..9631ee06c 100644 --- a/maro/rl/algorithms/dqn.py +++ b/maro/rl/algorithms/dqn.py @@ -98,7 +98,7 @@ def train(self, states: np.ndarray, actions: np.ndarray, rewards: np.ndarray, ne if self._train_cnt % self._hyper_params.target_replacement_period == 0: self._update_target_model() - return loss.detach().numpy() + return np.abs((current_q_values - target_q_values).detach().numpy()) def _update_target_model(self): if hasattr(self, "_optimizer"): From ea5a846bea8b6c7681558cf2d8c41d3d00701b0b Mon Sep 17 00:00:00 2001 From: ysqyang Date: Wed, 28 Oct 2020 15:39:05 +0800 Subject: [PATCH 072/337] fixed a bug --- examples/cim/dqn/multi_process_launcher.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/cim/dqn/multi_process_launcher.py b/examples/cim/dqn/multi_process_launcher.py index 4450b4b3b..0de79a77c 100644 --- a/examples/cim/dqn/multi_process_launcher.py +++ b/examples/cim/dqn/multi_process_launcher.py @@ -12,7 +12,7 @@ if __name__ == "__main__": parser = argparse.ArgumentParser() parser.add_argument("group_name", help="group name") - parser.add_argument("num_actors", help="number of actors") + parser.add_argument("num_actors", type=int, help="number of actors") args = parser.parse_args() learner_path = f"{os.path.split(os.path.realpath(__file__))[0]}/dist_learner.py &" From 7d1dc099252b1c6ddafe4e4ece97df0fee1e1886 Mon Sep 17 00:00:00 2001 From: ysqyang Date: Wed, 28 Oct 2020 16:16:38 +0800 Subject: [PATCH 073/337] added early stopping feature to CIM exmaple --- examples/cim/dqn/config.yml | 3 ++ examples/cim/dqn/single_process_launcher.py | 9 ++-- maro/rl/__init__.py | 4 ++ .../single_learner_multi_actor_sync_mode.py | 5 ++- .../simple_early_stopping_checker.py | 10 ++--- maro/rl/learner/simple_learner.py | 41 ++++++++++--------- 6 files changed, 42 insertions(+), 30 deletions(-) diff --git a/examples/cim/dqn/config.yml b/examples/cim/dqn/config.yml index 4a02d1844..7e74a7638 100644 --- a/examples/cim/dqn/config.yml +++ b/examples/cim/dqn/config.yml @@ -4,6 +4,9 @@ env: durations: 1120 general: max_episode: 500 # max episode + early_stopping: + last_k: 3 + threshold: 0.3 state_shaping: look_back: 7 max_ports_downstream: 2 diff --git a/examples/cim/dqn/single_process_launcher.py b/examples/cim/dqn/single_process_launcher.py index 61257d04b..23ea34fd5 100644 --- a/examples/cim/dqn/single_process_launcher.py +++ b/examples/cim/dqn/single_process_launcher.py @@ -7,8 +7,8 @@ import numpy as np from maro.simulator import Env -from maro.rl import SimpleLearner, SimpleActor, AgentManagerMode, KStepExperienceShaper, \ - TwoPhaseLinearExplorer +from maro.rl import AgentManagerMode, SimpleEarlyStoppingChecker, KStepExperienceShaper, SimpleLearner, SimpleActor, \ + TwoPhaseLinearExplorer, from maro.utils import Logger from components.action_shaper import CIMActionShaper @@ -58,6 +58,9 @@ def set_input_dim(): ) # Step 4: Create an actor and a learner to start the training process. + early_stopping_checker = SimpleEarlyStoppingChecker( + metric_func=lambda x: x["container_shortage"], **config.general.early_stopping + ) actor = SimpleActor(env=env, inference_agents=agent_manager) learner = SimpleLearner( trainable_agents=agent_manager, @@ -65,7 +68,7 @@ def set_input_dim(): explorer=TwoPhaseLinearExplorer(**config.exploration), logger=Logger("single_host_cim_learner", auto_timestamp=False) ) - learner.train(max_episode=config.general.max_episode) + learner.train(max_episode=config.general.max_episode, early_stopping_checker=early_stopping_checker) learner.test() learner.dump_models(os.path.join(os.getcwd(), "models")) diff --git a/maro/rl/__init__.py b/maro/rl/__init__.py index 28c26caa8..b8e5c4cbe 100644 --- a/maro/rl/__init__.py +++ b/maro/rl/__init__.py @@ -22,6 +22,8 @@ from maro.rl.shaping.k_step_experience_shaper import KStepExperienceShaper from maro.rl.explorer.abs_explorer import AbsExplorer from maro.rl.explorer.simple_explorer import LinearExplorer, TwoPhaseLinearExplorer +from maro.rl.early_stopping.abs_early_stopping_checker import AbsEarlyStoppingChecker +from maro.rl.early_stopping.simple_early_stopping_checker import SimpleEarlyStoppingChecker from maro.rl.dist_topologies.single_learner_multi_actor_sync_mode import ActorProxy, ActorWorker @@ -50,6 +52,8 @@ "AbsExplorer", "LinearExplorer", "TwoPhaseLinearExplorer", + "AbsEarlyStoppingChecker", + "SimpleEarlyStoppingChecker", "ActorProxy", "ActorWorker" ] diff --git a/maro/rl/dist_topologies/single_learner_multi_actor_sync_mode.py b/maro/rl/dist_topologies/single_learner_multi_actor_sync_mode.py index c8c73f8f6..1460cc52c 100644 --- a/maro/rl/dist_topologies/single_learner_multi_actor_sync_mode.py +++ b/maro/rl/dist_topologies/single_learner_multi_actor_sync_mode.py @@ -25,8 +25,9 @@ class ActorProxy(object): def __init__(self, proxy_params): self._proxy = Proxy(component_type="learner", **proxy_params) - def roll_out(self, model_dict: dict = None, epsilon_dict: dict = None, done: bool = False, - return_details: bool = True): + def roll_out( + self, model_dict: dict = None, epsilon_dict: dict = None, done: bool = False, return_details: bool = True + ): """Send roll-out requests to remote actors. This method has exactly the same signature as ``SimpleActor``'s ``roll_out`` method but instead of doing diff --git a/maro/rl/early_stopping/simple_early_stopping_checker.py b/maro/rl/early_stopping/simple_early_stopping_checker.py index 9cf65fccc..1440554e8 100644 --- a/maro/rl/early_stopping/simple_early_stopping_checker.py +++ b/maro/rl/early_stopping/simple_early_stopping_checker.py @@ -13,17 +13,17 @@ class SimpleEarlyStoppingChecker(AbsEarlyStoppingChecker): Args: last_k (int): Number of the latest performance records to check for early stopping. - performance_metric_func (Callable): A function to obtain the metric from a performance record to be evaluated - against a threshold value. + metric_func (Callable): A function to obtain the metric from a performance record to be evaluated against a + threshold value. threshold (float): The threshold value against which the early stopping metric is compared. The early stopping condition is satisfied if the metric obtained using the ``performance_metric_func`` is below this threshold. """ - def __init__(self, last_k: int, performance_metric_func: Callable, threshold: float): + def __init__(self, last_k: int, metric_func: Callable, threshold: float): super().__init__() self._last_k = last_k - self._performance_metric_func = performance_metric_func + self._metric_func = metric_func self._threshold = threshold def __call__(self, performance_history): - metrics = map(self._performance_metric_func, performance_history[self._last_k:]) + metrics = map(self._metric_func, performance_history[self._last_k:]) return stdev(metrics) / mean(metrics) < self._threshold diff --git a/maro/rl/learner/simple_learner.py b/maro/rl/learner/simple_learner.py index 0faf4da6e..ae98f8c53 100644 --- a/maro/rl/learner/simple_learner.py +++ b/maro/rl/learner/simple_learner.py @@ -1,6 +1,7 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT license. +import sys from typing import Callable, Union import warnings @@ -62,25 +63,20 @@ def train(self, max_episode: int, early_stopping_checker: Callable = None): early_stopping_checker (Callable): A Callable object to determine whether the training loop should be terminated based on the latest performances. """ - if max_episode < 0: - if early_stopping_checker is None: - warnings.warn("No max episode and early stopping checker provided. The training loop will run forever.") - episode = 1 - while True: - performance, exp_by_agent = self._sample(episode, max_episode) - self._performance_history.append(performance) - if early_stopping_checker is not None and early_stopping_checker(self._performance_history): - self._logger.info("Early stopping condition satisfied. Training complete.") - break - episode += 1 - else: - for episode in range(1, max_episode + 1): - performance, exp_by_agent = self._sample(episode, max_episode) - self._performance_history.append(performance) - if early_stopping_checker is not None and early_stopping_checker(self._performance_history): - self._logger.info("Early stopping condition satisfied. Training complete.") - break - self._trainable_agents.train(exp_by_agent) + if max_episode < 0 and early_stopping_checker is None: + warnings.warn( + "The training loop will run forever since neither maximum episode nor early stopping checker " + "is provided. " + ) + episode = 1 + while max_episode < 0 or episode <= max_episode: + performance, exp_by_agent = self._sample(episode, max_episode) + self._performance_history.append(performance) + if early_stopping_checker is not None and early_stopping_checker(self._performance_history): + self._logger.info("Early stopping condition satisfied. Training complete.") + break + self._trainable_agents.train(exp_by_agent) + episode += 1 def test(self): """Test policy performance.""" @@ -89,7 +85,12 @@ def test(self): return_details=False ) self._logger.info(f"test performance: {performance}") - self._actor.roll_out(done=True) + + def exit(self): + """Tell the remote actor to exit""" + if isinstance(self._actor, ActorProxy): + self._actor.roll_out(done=True) + sys.exit() def dump_models(self, model_dump_dir: str): self._trainable_agents.dump_models_to_files(model_dump_dir) From c20f64dd0fa493d4353e86e2a40c35bcf995e15e Mon Sep 17 00:00:00 2001 From: ysqyang Date: Wed, 28 Oct 2020 16:17:52 +0800 Subject: [PATCH 074/337] fixed a typo --- examples/cim/dqn/single_process_launcher.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/cim/dqn/single_process_launcher.py b/examples/cim/dqn/single_process_launcher.py index 23ea34fd5..2591f521b 100644 --- a/examples/cim/dqn/single_process_launcher.py +++ b/examples/cim/dqn/single_process_launcher.py @@ -8,7 +8,7 @@ from maro.simulator import Env from maro.rl import AgentManagerMode, SimpleEarlyStoppingChecker, KStepExperienceShaper, SimpleLearner, SimpleActor, \ - TwoPhaseLinearExplorer, + TwoPhaseLinearExplorer from maro.utils import Logger from components.action_shaper import CIMActionShaper From 3267228a818bd035c7829ac83ed55ff6e2f010a4 Mon Sep 17 00:00:00 2001 From: ysqyang Date: Wed, 28 Oct 2020 16:39:41 +0800 Subject: [PATCH 075/337] fixed some issues with early stopping --- examples/cim/dqn/config.yml | 1 + examples/cim/dqn/single_process_launcher.py | 10 ++++++++-- maro/rl/learner/simple_learner.py | 9 ++++++--- 3 files changed, 15 insertions(+), 5 deletions(-) diff --git a/examples/cim/dqn/config.yml b/examples/cim/dqn/config.yml index 7e74a7638..8a713bfe6 100644 --- a/examples/cim/dqn/config.yml +++ b/examples/cim/dqn/config.yml @@ -5,6 +5,7 @@ env: general: max_episode: 500 # max episode early_stopping: + start_ep: 50 last_k: 3 threshold: 0.3 state_shaping: diff --git a/examples/cim/dqn/single_process_launcher.py b/examples/cim/dqn/single_process_launcher.py index 2591f521b..8d25659f9 100644 --- a/examples/cim/dqn/single_process_launcher.py +++ b/examples/cim/dqn/single_process_launcher.py @@ -59,7 +59,9 @@ def set_input_dim(): # Step 4: Create an actor and a learner to start the training process. early_stopping_checker = SimpleEarlyStoppingChecker( - metric_func=lambda x: x["container_shortage"], **config.general.early_stopping + last_k=config.general.early_stopping.last_k, + metric_func=lambda x: x["container_shortage"], + threshold=config.general.early_stopping.threshold ) actor = SimpleActor(env=env, inference_agents=agent_manager) learner = SimpleLearner( @@ -68,7 +70,11 @@ def set_input_dim(): explorer=TwoPhaseLinearExplorer(**config.exploration), logger=Logger("single_host_cim_learner", auto_timestamp=False) ) - learner.train(max_episode=config.general.max_episode, early_stopping_checker=early_stopping_checker) + learner.train( + max_episode=config.general.max_episode, + early_stopping_checker=early_stopping_checker, + early_stopping_check_ep=config.general.early_stopping.start_ep + ) learner.test() learner.dump_models(os.path.join(os.getcwd(), "models")) diff --git a/maro/rl/learner/simple_learner.py b/maro/rl/learner/simple_learner.py index ae98f8c53..66e2b61eb 100644 --- a/maro/rl/learner/simple_learner.py +++ b/maro/rl/learner/simple_learner.py @@ -54,14 +54,15 @@ def _sample(self, ep, max_ep): self._logger.info(f"ep {ep} - performance: {performance}, epsilons: {epsilon_dict}") return performance, exp_by_agent - def train(self, max_episode: int, early_stopping_checker: Callable = None): + def train(self, max_episode: int, early_stopping_checker: Callable = None, early_stopping_check_ep: int = None): """Main loop for collecting experiences from the actor and using them to update policies. Args: max_episode (int): number of episodes to be run. If negative, the training loop will run forever unless an ``early_stopping_checker`` is provided and the early stopping condition is met. early_stopping_checker (Callable): A Callable object to determine whether the training loop should be - terminated based on the latest performances. + terminated based on the latest performances. Defaults to None. + early_stopping_check_ep (int): Episode from which early stopping check is initiated. Defaults to None. """ if max_episode < 0 and early_stopping_checker is None: warnings.warn( @@ -72,7 +73,9 @@ def train(self, max_episode: int, early_stopping_checker: Callable = None): while max_episode < 0 or episode <= max_episode: performance, exp_by_agent = self._sample(episode, max_episode) self._performance_history.append(performance) - if early_stopping_checker is not None and early_stopping_checker(self._performance_history): + if early_stopping_checker is not None and \ + (early_stopping_check_ep is None or episode >= early_stopping_check_ep) and \ + early_stopping_checker(self._performance_history): self._logger.info("Early stopping condition satisfied. Training complete.") break self._trainable_agents.train(exp_by_agent) From c966c3b20c1eb1d42a528d1ad91c0e40802936d7 Mon Sep 17 00:00:00 2001 From: ysqyang Date: Wed, 28 Oct 2020 16:49:34 +0800 Subject: [PATCH 076/337] changed early stopping metric func --- examples/cim/dqn/config.yml | 4 ++-- examples/cim/dqn/single_process_launcher.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/examples/cim/dqn/config.yml b/examples/cim/dqn/config.yml index 8a713bfe6..422a93dfa 100644 --- a/examples/cim/dqn/config.yml +++ b/examples/cim/dqn/config.yml @@ -6,8 +6,8 @@ general: max_episode: 500 # max episode early_stopping: start_ep: 50 - last_k: 3 - threshold: 0.3 + last_k: 5 + threshold: 0.2 state_shaping: look_back: 7 max_ports_downstream: 2 diff --git a/examples/cim/dqn/single_process_launcher.py b/examples/cim/dqn/single_process_launcher.py index 8d25659f9..e57b3d212 100644 --- a/examples/cim/dqn/single_process_launcher.py +++ b/examples/cim/dqn/single_process_launcher.py @@ -60,7 +60,7 @@ def set_input_dim(): # Step 4: Create an actor and a learner to start the training process. early_stopping_checker = SimpleEarlyStoppingChecker( last_k=config.general.early_stopping.last_k, - metric_func=lambda x: x["container_shortage"], + metric_func=lambda x: 1 - x["container_shortage"] / x["order_requirements"], threshold=config.general.early_stopping.threshold ) actor = SimpleActor(env=env, inference_agents=agent_manager) From 653b945d52beee2488b69fcf8111bdf54cc6b97c Mon Sep 17 00:00:00 2001 From: ysqyang Date: Wed, 28 Oct 2020 17:04:21 +0800 Subject: [PATCH 077/337] fixed a bug --- maro/rl/early_stopping/simple_early_stopping_checker.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/maro/rl/early_stopping/simple_early_stopping_checker.py b/maro/rl/early_stopping/simple_early_stopping_checker.py index 1440554e8..928d3b343 100644 --- a/maro/rl/early_stopping/simple_early_stopping_checker.py +++ b/maro/rl/early_stopping/simple_early_stopping_checker.py @@ -25,5 +25,5 @@ def __init__(self, last_k: int, metric_func: Callable, threshold: float): self._threshold = threshold def __call__(self, performance_history): - metrics = map(self._metric_func, performance_history[self._last_k:]) + metrics = map(self._metric_func, performance_history[-self._last_k:]) return stdev(metrics) / mean(metrics) < self._threshold From bb7819c92fb625044abc6a1f3eb6c7e8f3935822 Mon Sep 17 00:00:00 2001 From: ysqyang Date: Wed, 28 Oct 2020 17:13:50 +0800 Subject: [PATCH 078/337] fixed a bug --- maro/rl/early_stopping/simple_early_stopping_checker.py | 2 +- maro/rl/explorer/simple_explorer.py | 4 ++-- maro/rl/learner/simple_learner.py | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/maro/rl/early_stopping/simple_early_stopping_checker.py b/maro/rl/early_stopping/simple_early_stopping_checker.py index 928d3b343..9d7cdef42 100644 --- a/maro/rl/early_stopping/simple_early_stopping_checker.py +++ b/maro/rl/early_stopping/simple_early_stopping_checker.py @@ -25,5 +25,5 @@ def __init__(self, last_k: int, metric_func: Callable, threshold: float): self._threshold = threshold def __call__(self, performance_history): - metrics = map(self._metric_func, performance_history[-self._last_k:]) + metrics = list(map(self._metric_func, performance_history[-self._last_k:])) return stdev(metrics) / mean(metrics) < self._threshold diff --git a/maro/rl/explorer/simple_explorer.py b/maro/rl/explorer/simple_explorer.py index 1423b2aff..804925f73 100644 --- a/maro/rl/explorer/simple_explorer.py +++ b/maro/rl/explorer/simple_explorer.py @@ -17,7 +17,7 @@ def __init__(self, max_eps: float, min_eps: float = .0): self._min_eps = min_eps def generate_epsilon(self, current_ep, max_ep, performance_history=None): - return self._min_eps + (self._max_eps - self._min_eps) * (1 - current_ep / max_ep) + return self._min_eps + (self._max_eps - self._min_eps) * (1 - current_ep / (max_ep - 1)) class TwoPhaseLinearExplorer(AbsExplorer): @@ -42,7 +42,7 @@ def __init__(self, progress_split: float, eps_split: float, max_eps: float, min_ self._min_eps = min_eps def generate_epsilon(self, current_ep, max_ep, performance_history=None): - progress = current_ep / max_ep + progress = current_ep / (max_ep - 1) if progress <= self._progress_split: return self._max_eps - (self._max_eps - self._eps_split) * progress / self._progress_split else: diff --git a/maro/rl/learner/simple_learner.py b/maro/rl/learner/simple_learner.py index 66e2b61eb..b379f2825 100644 --- a/maro/rl/learner/simple_learner.py +++ b/maro/rl/learner/simple_learner.py @@ -69,8 +69,8 @@ def train(self, max_episode: int, early_stopping_checker: Callable = None, early "The training loop will run forever since neither maximum episode nor early stopping checker " "is provided. " ) - episode = 1 - while max_episode < 0 or episode <= max_episode: + episode = 0 + while max_episode < 0 or episode < max_episode: performance, exp_by_agent = self._sample(episode, max_episode) self._performance_history.append(performance) if early_stopping_checker is not None and \ From 57dcb33d83b53a5cca7d7ca5bcee9be154703105 Mon Sep 17 00:00:00 2001 From: ysqyang Date: Wed, 28 Oct 2020 20:42:53 +0800 Subject: [PATCH 079/337] added early stopping to dist mode cim --- examples/cim/dqn/dist_learner.py | 15 +++++++++++++-- examples/cim/dqn/single_process_launcher.py | 1 + 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/examples/cim/dqn/dist_learner.py b/examples/cim/dqn/dist_learner.py index 8671649fe..73e2d1be7 100644 --- a/examples/cim/dqn/dist_learner.py +++ b/examples/cim/dqn/dist_learner.py @@ -3,7 +3,7 @@ import os -from maro.rl import ActorProxy, SimpleLearner, AgentManagerMode, TwoPhaseLinearExplorer +from maro.rl import ActorProxy, AgentManagerMode, SimpleEarlyStoppingChecker, SimpleLearner, TwoPhaseLinearExplorer from maro.simulator import Env from maro.utils import Logger @@ -37,15 +37,26 @@ def set_input_dim(): "redis_address": ("localhost", 6379) } + early_stopping_checker = SimpleEarlyStoppingChecker( + last_k=config.general.early_stopping.last_k, + metric_func=lambda x: 1 - x["container_shortage"] / x["order_requirements"], + threshold=config.general.early_stopping.threshold + ) + learner = SimpleLearner( trainable_agents=agent_manager, actor=ActorProxy(proxy_params=proxy_params), explorer=TwoPhaseLinearExplorer(**config.exploration), logger=Logger("distributed_cim_learner", auto_timestamp=False) ) - learner.train(config.general.max_episode) + learner.train( + max_episode=config.general.max_episode, + early_stopping_checker=early_stopping_checker, + early_stopping_check_ep=config.general.early_stopping.start_ep + ) learner.test() learner.dump_models(os.path.join(os.getcwd(), "models")) + learner.exit() if __name__ == "__main__": diff --git a/examples/cim/dqn/single_process_launcher.py b/examples/cim/dqn/single_process_launcher.py index e57b3d212..8e91cd812 100644 --- a/examples/cim/dqn/single_process_launcher.py +++ b/examples/cim/dqn/single_process_launcher.py @@ -77,6 +77,7 @@ def set_input_dim(): ) learner.test() learner.dump_models(os.path.join(os.getcwd(), "models")) + learner.exit() if __name__ == "__main__": From 9a58d51188362b6bb86155c2d6c0e7d99452669b Mon Sep 17 00:00:00 2001 From: ysqyang Date: Thu, 29 Oct 2020 14:13:50 +0800 Subject: [PATCH 080/337] added experience collecting func --- examples/cim/dqn/components/config.py | 23 ++++----- examples/cim/dqn/config.yml | 2 +- examples/cim/dqn/dist_actor.py | 15 ++---- examples/cim/dqn/dist_learner.py | 23 +++------ examples/cim/dqn/single_process_launcher.py | 17 ++----- maro/rl/__init__.py | 6 ++- maro/rl/algorithms/dqn.py | 10 ++-- .../dist_topologies/experience_collection.py | 48 +++++++++++++++++++ .../single_learner_multi_actor_sync_mode.py | 22 ++++----- .../rl_formulation.ipynb | 2 +- 10 files changed, 96 insertions(+), 72 deletions(-) create mode 100644 maro/rl/dist_topologies/experience_collection.py diff --git a/examples/cim/dqn/components/config.py b/examples/cim/dqn/components/config.py index aaebb3ff9..974fcd591 100644 --- a/examples/cim/dqn/components/config.py +++ b/examples/cim/dqn/components/config.py @@ -9,19 +9,20 @@ import os import yaml -from maro.utils import convert_dottable + +def set_input_dim(config): + # obtain model input dimension from state shaping configurations + look_back = config["state_shaping"]["look_back"] + max_ports_downstream = config["state_shaping"]["max_ports_downstream"] + num_port_attributes = len(config["state_shaping"]["port_attributes"]) + num_vessel_attributes = len(config["state_shaping"]["vessel_attributes"]) + + input_dim = (look_back + 1) * (max_ports_downstream + 1) * num_port_attributes + num_vessel_attributes + config["agents"]["algorithm"]["input_dim"] = input_dim + + return config CONFIG_PATH = os.path.join(os.path.split(os.path.realpath(__file__))[0], "../config.yml") with io.open(CONFIG_PATH, "r") as in_file: config = yaml.safe_load(in_file) - -# obtain model input dimension from state shaping configurations -look_back = config["state_shaping"]["look_back"] -max_ports_downstream = config["state_shaping"]["max_ports_downstream"] -num_port_attributes = len(config["state_shaping"]["port_attributes"]) -num_vessel_attributes = len(config["state_shaping"]["vessel_attributes"]) - -input_dim = (look_back + 1) * (max_ports_downstream + 1) * num_port_attributes + num_vessel_attributes -config["agents"]["algorithm"]["input_dim"] = input_dim -config = convert_dottable(config) diff --git a/examples/cim/dqn/config.yml b/examples/cim/dqn/config.yml index 422a93dfa..70245785b 100644 --- a/examples/cim/dqn/config.yml +++ b/examples/cim/dqn/config.yml @@ -53,7 +53,7 @@ agents: lr: 0.05 hyper_parameters: reward_decay: .0 - target_replacement_period: 5 + target_replacement_frequency: 5 tau: 0.1 experience_pool: capacity: -1 diff --git a/examples/cim/dqn/dist_actor.py b/examples/cim/dqn/dist_actor.py index 18da573d9..832cbc6b9 100644 --- a/examples/cim/dqn/dist_actor.py +++ b/examples/cim/dqn/dist_actor.py @@ -7,25 +7,18 @@ from maro.simulator import Env from maro.rl import AgentManagerMode, SimpleActor, ActorWorker, KStepExperienceShaper +from maro.utils import convert_dottable from components.action_shaper import CIMActionShaper from components.agent_manager import create_dqn_agents, DQNAgentManager +from components.config import set_input_dim from components.experience_shaper import TruncatedExperienceShaper from components.state_shaper import CIMStateShaper def launch(config): - def set_input_dim(): - # obtain model input dimension from state shaping configurations - look_back = config["state_shaping"]["look_back"] - max_ports_downstream = config["state_shaping"]["max_ports_downstream"] - num_port_attributes = len(config["state_shaping"]["port_attributes"]) - num_vessel_attributes = len(config["state_shaping"]["vessel_attributes"]) - - input_dim = (look_back + 1) * (max_ports_downstream + 1) * num_port_attributes + num_vessel_attributes - config["agents"]["algorithm"]["input_dim"] = input_dim - - set_input_dim() + set_input_dim(config) + config = convert_dottable(config) env = Env(config.env.scenario, config.env.topology, durations=config.env.durations) agent_id_list = [str(agent_id) for agent_id in env.agent_idx_list] state_shaper = CIMStateShaper(**config.state_shaping) diff --git a/examples/cim/dqn/dist_learner.py b/examples/cim/dqn/dist_learner.py index 73e2d1be7..9e885df18 100644 --- a/examples/cim/dqn/dist_learner.py +++ b/examples/cim/dqn/dist_learner.py @@ -3,26 +3,18 @@ import os -from maro.rl import ActorProxy, AgentManagerMode, SimpleEarlyStoppingChecker, SimpleLearner, TwoPhaseLinearExplorer +from maro.rl import ActorProxy, AgentManagerMode, SimpleEarlyStoppingChecker, SimpleLearner, TwoPhaseLinearExplorer, \ + concat_experiences_by_agent from maro.simulator import Env -from maro.utils import Logger +from maro.utils import Logger, convert_dottable from components.agent_manager import create_dqn_agents, DQNAgentManager +from components.config import set_input_dim def launch(config): - def set_input_dim(): - # obtain model input dimension from state shaping configurations - look_back = config["state_shaping"]["look_back"] - max_ports_downstream = config["state_shaping"]["max_ports_downstream"] - num_port_attributes = len(config["state_shaping"]["port_attributes"]) - num_vessel_attributes = len(config["state_shaping"]["vessel_attributes"]) - - input_dim = (look_back + 1) * (max_ports_downstream + 1) * num_port_attributes + num_vessel_attributes - config["agents"]["algorithm"]["input_dim"] = input_dim - - set_input_dim() - + set_input_dim(config) + config = convert_dottable(config) env = Env(config.env.scenario, config.env.topology, durations=config.env.durations) agent_id_list = [str(agent_id) for agent_id in env.agent_idx_list] agent_manager = DQNAgentManager( @@ -45,7 +37,7 @@ def set_input_dim(): learner = SimpleLearner( trainable_agents=agent_manager, - actor=ActorProxy(proxy_params=proxy_params), + actor=ActorProxy(proxy_params=proxy_params, experience_collecting_func=concat_experiences_by_agent), explorer=TwoPhaseLinearExplorer(**config.exploration), logger=Logger("distributed_cim_learner", auto_timestamp=False) ) @@ -62,4 +54,3 @@ def set_input_dim(): if __name__ == "__main__": from components.config import config launch(config) - diff --git a/examples/cim/dqn/single_process_launcher.py b/examples/cim/dqn/single_process_launcher.py index 8e91cd812..56af163d0 100644 --- a/examples/cim/dqn/single_process_launcher.py +++ b/examples/cim/dqn/single_process_launcher.py @@ -9,28 +9,19 @@ from maro.simulator import Env from maro.rl import AgentManagerMode, SimpleEarlyStoppingChecker, KStepExperienceShaper, SimpleLearner, SimpleActor, \ TwoPhaseLinearExplorer -from maro.utils import Logger +from maro.utils import Logger, convert_dottable from components.action_shaper import CIMActionShaper from components.agent_manager import create_dqn_agents, DQNAgentManager +from components.config import set_input_dim from components.experience_shaper import TruncatedExperienceShaper from components.state_shaper import CIMStateShaper def launch(config): # First determine the input dimension and add it to the config. - def set_input_dim(): - # obtain model input dimension from state shaping configurations - look_back = config["state_shaping"]["look_back"] - max_ports_downstream = config["state_shaping"]["max_ports_downstream"] - num_port_attributes = len(config["state_shaping"]["port_attributes"]) - num_vessel_attributes = len(config["state_shaping"]["vessel_attributes"]) - - input_dim = (look_back + 1) * (max_ports_downstream + 1) * num_port_attributes + num_vessel_attributes - config["agents"]["algorithm"]["input_dim"] = input_dim - - set_input_dim() - + set_input_dim(config) + config = convert_dottable(config) # Step 1: Initialize a CIM environment for using a toy dataset. env = Env(config.env.scenario, config.env.topology, durations=config.env.durations) agent_id_list = [str(agent_id) for agent_id in env.agent_idx_list] diff --git a/maro/rl/__init__.py b/maro/rl/__init__.py index b8e5c4cbe..aa79181aa 100644 --- a/maro/rl/__init__.py +++ b/maro/rl/__init__.py @@ -25,6 +25,8 @@ from maro.rl.early_stopping.abs_early_stopping_checker import AbsEarlyStoppingChecker from maro.rl.early_stopping.simple_early_stopping_checker import SimpleEarlyStoppingChecker from maro.rl.dist_topologies.single_learner_multi_actor_sync_mode import ActorProxy, ActorWorker +from maro.rl.dist_topologies.experience_collection import concat_experiences_by_agent, \ + merge_experiences_with_trajectory_boundaries __all__ = [ @@ -55,5 +57,7 @@ "AbsEarlyStoppingChecker", "SimpleEarlyStoppingChecker", "ActorProxy", - "ActorWorker" + "ActorWorker", + "concat_experiences_by_agent", + "merge_experiences_with_trajectory_boundaries" ] diff --git a/maro/rl/algorithms/dqn.py b/maro/rl/algorithms/dqn.py index 9631ee06c..200c59368 100644 --- a/maro/rl/algorithms/dqn.py +++ b/maro/rl/algorithms/dqn.py @@ -15,17 +15,17 @@ class DQNHyperParams: Args: num_actions (int): number of possible actions reward_decay (float): reward decay as defined in standard RL terminology - target_replacement_period (int): number of training frequency of target model replacement + target_replacement_frequency (int): number of training frequency of target model replacement tau (float): soft update coefficient, e.g., target_model = tau * eval_model + (1-tau) * target_model """ - __slots__ = ["num_actions", "reward_decay", "target_replacement_period", "tau"] + __slots__ = ["num_actions", "reward_decay", "target_replacement_frequency", "tau"] def __init__( - self, num_actions: int, reward_decay: float, target_replacement_period: int, tau: float = 1.0 + self, num_actions: int, reward_decay: float, target_replacement_frequency: int, tau: float = 1.0 ): self.num_actions = num_actions self.reward_decay = reward_decay - self.target_replacement_period = target_replacement_period + self.target_replacement_frequency = target_replacement_frequency self.tau = tau @@ -95,7 +95,7 @@ def train(self, states: np.ndarray, actions: np.ndarray, rewards: np.ndarray, ne loss.backward() self._optimizer.step() self._train_cnt += 1 - if self._train_cnt % self._hyper_params.target_replacement_period == 0: + if self._train_cnt % self._hyper_params.target_replacement_frequency == 0: self._update_target_model() return np.abs((current_q_values - target_q_values).detach().numpy()) diff --git a/maro/rl/dist_topologies/experience_collection.py b/maro/rl/dist_topologies/experience_collection.py new file mode 100644 index 000000000..f4cea35d9 --- /dev/null +++ b/maro/rl/dist_topologies/experience_collection.py @@ -0,0 +1,48 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +from collections import defaultdict + + +def concat_experiences_by_agent(exp_by_source: dict) -> dict: + """Concatenate experiences from multiple sources, by agent ID. + + The experience from each source is expected to be already grouped by agent ID. The result is a single dictionary + of experiences with keys being agent IDs and values being the concatenation of experiences from all sources + for each agent ID. + + Args: + exp_by_source (dict): Experiences from multiple sources. Each value should consist of experiences grouped by + agent ID. + + Returns: + Merged experiences with agent IDs as keys. + """ + merged = {} + for exp_by_agent in exp_by_source.values(): + for agent_id, exp in exp_by_agent.items(): + if agent_id not in merged: + merged[agent_id] = defaultdict(list) + for k, v in exp.items(): + merged[agent_id][k].extend(v) + + return merged + + +def merge_experiences_with_trajectory_boundaries(trajectories_by_source) -> dict: + """Collect each agent's trajectories from multiple sources. + + Args: + trajectories_by_source (dict): Agent's trajectories from multiple sources. + + Returns: + A list of trajectories for each agent. + """ + merged = defaultdict(list) + for exp_by_agent in trajectories_by_source.values(): + for agent_id, trajectory in exp_by_agent.items(): + merged[agent_id].append(trajectory) + + return merged + + diff --git a/maro/rl/dist_topologies/single_learner_multi_actor_sync_mode.py b/maro/rl/dist_topologies/single_learner_multi_actor_sync_mode.py index 1460cc52c..d15c8ab28 100644 --- a/maro/rl/dist_topologies/single_learner_multi_actor_sync_mode.py +++ b/maro/rl/dist_topologies/single_learner_multi_actor_sync_mode.py @@ -2,8 +2,8 @@ # Licensed under the MIT license. from enum import Enum -from collections import defaultdict import sys +from typing import Callable from maro.communication import Proxy, SessionType from maro.communication.registry_table import RegisterTable @@ -21,9 +21,11 @@ class ActorProxy(object): Args: proxy_params: Parameters for instantiating a ``Proxy`` instance. + experience_collecting_func (Callable): A function responsible for collecting experiences from multiple sources. """ - def __init__(self, proxy_params): + def __init__(self, proxy_params, experience_collecting_func: Callable): self._proxy = Proxy(component_type="learner", **proxy_params) + self._experience_collecting_func = experience_collecting_func def roll_out( self, model_dict: dict = None, epsilon_dict: dict = None, done: bool = False, return_details: bool = True @@ -54,7 +56,6 @@ def roll_out( ) return None, None else: - performance, exp_by_agent = {}, {} payloads = [(peer, {PayloadKey.MODEL: model_dict, PayloadKey.EPSILON: epsilon_dict, PayloadKey.RETURN_DETAILS: return_details}) @@ -65,16 +66,11 @@ def roll_out( session_type=SessionType.TASK, destination_payload_list=payloads ) - for msg in replies: - performance[msg.source] = msg.payload[PayloadKey.PERFORMANCE] - if msg.payload[PayloadKey.EXPERIENCE] is not None: - for agent_id, exp_set in msg.payload[PayloadKey.EXPERIENCE].items(): - if agent_id not in exp_by_agent: - exp_by_agent[agent_id] = defaultdict(list) - for k, v in exp_set.items(): - exp_by_agent[agent_id][k].extend(v) - - return performance, exp_by_agent + + performance = {msg.source: msg.payload[PayloadKey.PERFORMANCE] for msg in replies} + experiences_by_source = {msg.source: msg.payload[PayloadKey.EXPERIENCE] for msg in replies} + + return performance, self._experience_collecting_func(experiences_by_source) class ActorWorker(object): diff --git a/notebooks/container_inventory_management/rl_formulation.ipynb b/notebooks/container_inventory_management/rl_formulation.ipynb index 38b273999..13578f091 100644 --- a/notebooks/container_inventory_management/rl_formulation.ipynb +++ b/notebooks/container_inventory_management/rl_formulation.ipynb @@ -248,7 +248,7 @@ " optimizer_opt=(RMSprop, {\"lr\": 0.05}),\n", " loss_func_dict={\"eval\": smooth_l1_loss},\n", " hyper_params=DQNHyperParams(num_actions=num_actions, reward_decay=.0,\n", - " target_replacement_period=5, tau=0.1)\n", + " target_replacement_frequency=5, tau=0.1)\n", " )\n", "\n", " experience_pool = ColumnBasedStore()\n", From c52052909ba89daa77efb4b0bd50b0df91905f7d Mon Sep 17 00:00:00 2001 From: ysqyang Date: Thu, 29 Oct 2020 14:34:11 +0800 Subject: [PATCH 081/337] edited notebook according to changes in CIM example --- examples/cim/dqn/config.yml | 4 +- .../rl_formulation.ipynb | 98 ++++++++++++------- 2 files changed, 62 insertions(+), 40 deletions(-) diff --git a/examples/cim/dqn/config.yml b/examples/cim/dqn/config.yml index 70245785b..3a74c700f 100644 --- a/examples/cim/dqn/config.yml +++ b/examples/cim/dqn/config.yml @@ -3,7 +3,7 @@ env: topology: "toy.4p_ssdd_l0.0" durations: 1120 general: - max_episode: 500 # max episode + max_episode: 500 early_stopping: start_ep: 50 last_k: 5 @@ -59,6 +59,6 @@ agents: capacity: -1 training_loop_parameters: min_experiences_to_train: 1024 - num_batches: 10 # number of times the algorithm's step() method is called + num_batches: 10 batch_size: 128 seed: 1024 # for reproducibility diff --git a/notebooks/container_inventory_management/rl_formulation.ipynb b/notebooks/container_inventory_management/rl_formulation.ipynb index 13578f091..5755b06df 100644 --- a/notebooks/container_inventory_management/rl_formulation.ipynb +++ b/notebooks/container_inventory_management/rl_formulation.ipynb @@ -231,36 +231,64 @@ "from maro.rl import AbsAgentManager, LearningModel, MLPFullyConnectedNet, DQN, DQNHyperParams, ColumnBasedStore\n", "\n", "\n", + "input_dim = 171\n", "num_actions = 21\n", "\n", "\n", - "class DQNAgentManager(AbsAgentManager):\n", - " def _assemble(self, agent_dict):\n", - " for agent_id in self._agent_id_list:\n", - " eval_model = LearningModel(decision_layers=MLPFullyConnectedNet(name=f'{agent_id}.policy',\n", - " input_dim=self._state_shaper.dim,\n", - " output_dim=num_actions,\n", - " hidden_dims=[256, 128, 64],\n", - " dropout_p=.0)\n", - " )\n", - "\n", - " algorithm = DQN(model_dict={\"eval\": eval_model},\n", - " optimizer_opt=(RMSprop, {\"lr\": 0.05}),\n", - " loss_func_dict={\"eval\": smooth_l1_loss},\n", - " hyper_params=DQNHyperParams(num_actions=num_actions, reward_decay=.0,\n", - " target_replacement_frequency=5, tau=0.1)\n", - " )\n", - "\n", - " experience_pool = ColumnBasedStore()\n", - " \n", - " agent_dict[agent_id] = CIMAgent(name=agent_id, algorithm=algorithm, experience_pool=experience_pool,\n", - " min_experiences_to_train=1024, num_batches=10, batch_size=128)\n", - "\n", - " def store_experiences(self, experiences):\n", - " # The output of the experience shaper is organized by the agent ID. \n", - " for agent_id, exp in experiences.items():\n", - " exp.update({\"loss\": [1e8] * len(exp[next(iter(exp))])})\n", - " self._agent_dict[agent_id].store_experiences(exp)" + "\n", + "def create_dqn_agents(agent_id_list):\n", + " agent_dict = {}\n", + " for agent_id in agent_id_list:\n", + " eval_model = LearningModel(\n", + " decision_layers=FullyConnectedNet(\n", + " name=f'{agent_id}.policy',\n", + " input_dim=input_dim,\n", + " output_dim=num_actions,\n", + " activation=nn.LeakyReLU, \n", + " hidden_dims=[256, 128, 64],\n", + " softmax_enabled=False,\n", + " batch_norm_enabled=True,\n", + " dropout_p=.0\n", + " )\n", + " )\n", + "\n", + " algorithm = DQN(\n", + " eval_model=eval_model,\n", + " optimizer_cls=RMSprop,\n", + " optimizer_params={\"lr\": 0.05},\n", + " loss_func=nn.functional.smooth_l1_loss,\n", + " hyper_params=DQNHyperParams(\n", + " num_actions=num_actions,\n", + " reward_decay=.0,\n", + " target_replacement_frequency=5,\n", + " tau=0.1\n", + " )\n", + " )\n", + "\n", + " experience_pool = ColumnBasedStore()\n", + " agent_dict[agent_id] = CIMAgent(\n", + " name=agent_id,\n", + " algorithm=algorithm,\n", + " experience_pool=experience_pool,\n", + " min_experiences_to_train=1024,\n", + " num_batches=10\n", + " batch_size=128\n", + " )\n", + "\n", + " return agent_dict\n", + "\n", + "\n", + "class DQNAgentManager(SimpleAgentManager):\n", + " def train(self, experiences_by_agent, performance=None):\n", + " self._assert_train_mode()\n", + "\n", + " # store experiences for each agent\n", + " for agent_id, exp in experiences_by_agent.items():\n", + " exp.update({\"loss\": [1e8] * len(list(exp.values())[0])})\n", + " self.agent_dict[agent_id].store_experiences(exp)\n", + "\n", + " for agent in self.agent_dict.values():\n", + " agent.train()" ] }, { @@ -417,7 +445,6 @@ "\n", "# Step 1: initialize a CIM environment for using a toy dataset. \n", "env = Env(\"cim\", \"toy.4p_ssdd_l0.0\", durations=1120)\n", - "total_episodes = 100\n", "agent_id_list = [str(agent_id) for agent_id in env.agent_idx_list]\n", "\n", "# Step 2: create state, action and experience shapers. We also need to create an explorer here due to the \n", @@ -433,26 +460,21 @@ "experience_shaper = TruncatedExperienceShaper(time_window=100, fulfillment_factor=1.0, shortage_factor=1.0,\n", " time_decay_factor=0.97)\n", "\n", - "explorer = TwoPhaseLinearExplorer(agent_id_list, total_episodes, \n", - " epsilon_range_dict={\"_all_\": (.0, .4)},\n", - " split_point_dict={\"_all_\": (.5, .8)},\n", - " with_cache=True)\n", - "\n", "# Step 3: create an agent manager.\n", "agent_manager = DQNAgentManager(name=\"cim_learner\",\n", " mode=AgentMode.TRAIN_INFERENCE,\n", - " agent_id_list=agent_id_list,\n", + " agent_dict=create_dqn_agents(agent_id_list),\n", " state_shaper=state_shaper,\n", " action_shaper=action_shaper,\n", - " experience_shaper=experience_shaper,\n", - " explorer=explorer)\n", + " experience_shaper=experience_shaper)\n", "\n", "# Step 4: Create an actor and a learner to start the training process. \n", "actor = SimpleActor(env, agent_manager)\n", - "learner = SimpleLearner(trainable_agents=agent_manager, actor=actor,\n", + "learner = SimpleLearner(trainable_agents=agent_manager, actor=actor, \n", + " explorer=TwoPhaseLinearExplorer(eps_split=0.32, progress_split=0.5, max_eps=0.4, min_eps=.0),\n", " logger=Logger(\"single_host_cim_learner\", format_=LogFormat.simple, auto_timestamp=False))\n", "\n", - "learner.train(total_episodes)" + "learner.train(max_episode=100)" ] } ], From cfe3d89f0f9164a92841daaef8f4e345e95f66d1 Mon Sep 17 00:00:00 2001 From: ysqyang Date: Thu, 29 Oct 2020 06:40:18 +0000 Subject: [PATCH 082/337] fixed bugs in nb --- .../rl_formulation.ipynb | 150 +++--------------- 1 file changed, 23 insertions(+), 127 deletions(-) diff --git a/notebooks/container_inventory_management/rl_formulation.ipynb b/notebooks/container_inventory_management/rl_formulation.ipynb index 5755b06df..8d22f35b4 100644 --- a/notebooks/container_inventory_management/rl_formulation.ipynb +++ b/notebooks/container_inventory_management/rl_formulation.ipynb @@ -218,17 +218,18 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": 11, "metadata": {}, "outputs": [], "source": [ "import io\n", "import yaml\n", "\n", + "import torch.nn as\n", "from torch.nn.functional import smooth_l1_loss\n", "from torch.optim import RMSprop\n", "\n", - "from maro.rl import AbsAgentManager, LearningModel, MLPFullyConnectedNet, DQN, DQNHyperParams, ColumnBasedStore\n", + "from maro.rl import SimpleAgentManager, LearningModel, FullyConnectedNet, DQN, DQNHyperParams, ColumnBasedStore\n", "\n", "\n", "input_dim = 171\n", @@ -271,7 +272,7 @@ " algorithm=algorithm,\n", " experience_pool=experience_pool,\n", " min_experiences_to_train=1024,\n", - " num_batches=10\n", + " num_batches=10,\n", " batch_size=128\n", " )\n", "\n", @@ -310,137 +311,25 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": 12, "metadata": {}, "outputs": [ { - "name": "stdout", - "output_type": "stream", - "text": [ - "10:42:49 | single_host_cim_learner | INFO | ep 1 - performance: {'order_requirements': 2240000, 'container_shortage': 1471903, 'operation_number': 2998694}, epsilons: {'0': 0.4, '1': 0.4, '2': 0.4, '3': 0.4}\n", - "10:42:53 | single_host_cim_learner | INFO | ep 2 - performance: {'order_requirements': 2240000, 'container_shortage': 1749783, 'operation_number': 2682255}, epsilons: {'0': 0.39836734693877884, '1': 0.39836734693877884, '2': 0.39836734693877884, '3': 0.39836734693877884}\n", - "10:42:57 | single_host_cim_learner | INFO | ep 3 - performance: {'order_requirements': 2240000, 'container_shortage': 1602611, 'operation_number': 2882583}, epsilons: {'0': 0.39673469387755766, '1': 0.39673469387755766, '2': 0.39673469387755766, '3': 0.39673469387755766}\n", - "10:43:01 | single_host_cim_learner | INFO | ep 4 - performance: {'order_requirements': 2240000, 'container_shortage': 1685691, 'operation_number': 2560449}, epsilons: {'0': 0.3951020408163365, '1': 0.3951020408163365, '2': 0.3951020408163365, '3': 0.3951020408163365}\n", - "10:43:06 | single_host_cim_learner | INFO | ep 5 - performance: {'order_requirements': 2240000, 'container_shortage': 1642661, 'operation_number': 3391669}, epsilons: {'0': 0.3934693877551153, '1': 0.3934693877551153, '2': 0.3934693877551153, '3': 0.3934693877551153}\n", - "10:43:10 | single_host_cim_learner | INFO | ep 6 - performance: {'order_requirements': 2240000, 'container_shortage': 1417467, 'operation_number': 3304565}, epsilons: {'0': 0.39183673469389413, '1': 0.39183673469389413, '2': 0.39183673469389413, '3': 0.39183673469389413}\n", - "10:43:14 | single_host_cim_learner | INFO | ep 7 - performance: {'order_requirements': 2240000, 'container_shortage': 1334117, 'operation_number': 3217649}, epsilons: {'0': 0.39020408163267295, '1': 0.39020408163267295, '2': 0.39020408163267295, '3': 0.39020408163267295}\n", - "10:43:19 | single_host_cim_learner | INFO | ep 8 - performance: {'order_requirements': 2240000, 'container_shortage': 1970188, 'operation_number': 2342474}, epsilons: {'0': 0.38857142857145177, '1': 0.38857142857145177, '2': 0.38857142857145177, '3': 0.38857142857145177}\n", - "10:43:24 | single_host_cim_learner | INFO | ep 9 - performance: {'order_requirements': 2240000, 'container_shortage': 921116, 'operation_number': 3455650}, epsilons: {'0': 0.3869387755102306, '1': 0.3869387755102306, '2': 0.3869387755102306, '3': 0.3869387755102306}\n", - "10:43:29 | single_host_cim_learner | INFO | ep 10 - performance: {'order_requirements': 2240000, 'container_shortage': 1214701, 'operation_number': 2963197}, epsilons: {'0': 0.3853061224490094, '1': 0.3853061224490094, '2': 0.3853061224490094, '3': 0.3853061224490094}\n", - "10:43:33 | single_host_cim_learner | INFO | ep 11 - performance: {'order_requirements': 2240000, 'container_shortage': 1400335, 'operation_number': 3407082}, epsilons: {'0': 0.38367346938778824, '1': 0.38367346938778824, '2': 0.38367346938778824, '3': 0.38367346938778824}\n", - "10:43:38 | single_host_cim_learner | INFO | ep 12 - performance: {'order_requirements': 2240000, 'container_shortage': 728902, 'operation_number': 3688912}, epsilons: {'0': 0.38204081632656706, '1': 0.38204081632656706, '2': 0.38204081632656706, '3': 0.38204081632656706}\n", - "10:43:43 | single_host_cim_learner | INFO | ep 13 - performance: {'order_requirements': 2240000, 'container_shortage': 899359, 'operation_number': 4382531}, epsilons: {'0': 0.3804081632653459, '1': 0.3804081632653459, '2': 0.3804081632653459, '3': 0.3804081632653459}\n", - "10:43:47 | single_host_cim_learner | INFO | ep 14 - performance: {'order_requirements': 2240000, 'container_shortage': 961579, 'operation_number': 4486164}, epsilons: {'0': 0.3787755102041247, '1': 0.3787755102041247, '2': 0.3787755102041247, '3': 0.3787755102041247}\n", - "10:43:52 | single_host_cim_learner | INFO | ep 15 - performance: {'order_requirements': 2240000, 'container_shortage': 831690, 'operation_number': 4283354}, epsilons: {'0': 0.3771428571429035, '1': 0.3771428571429035, '2': 0.3771428571429035, '3': 0.3771428571429035}\n", - "10:43:56 | single_host_cim_learner | INFO | ep 16 - performance: {'order_requirements': 2240000, 'container_shortage': 996019, 'operation_number': 4626933}, epsilons: {'0': 0.37551020408168234, '1': 0.37551020408168234, '2': 0.37551020408168234, '3': 0.37551020408168234}\n", - "10:44:01 | single_host_cim_learner | INFO | ep 17 - performance: {'order_requirements': 2240000, 'container_shortage': 747960, 'operation_number': 3693008}, epsilons: {'0': 0.37387755102046116, '1': 0.37387755102046116, '2': 0.37387755102046116, '3': 0.37387755102046116}\n", - "10:44:06 | single_host_cim_learner | INFO | ep 18 - performance: {'order_requirements': 2240000, 'container_shortage': 693959, 'operation_number': 3760747}, epsilons: {'0': 0.37224489795924, '1': 0.37224489795924, '2': 0.37224489795924, '3': 0.37224489795924}\n", - "10:44:10 | single_host_cim_learner | INFO | ep 19 - performance: {'order_requirements': 2240000, 'container_shortage': 708174, 'operation_number': 3817824}, epsilons: {'0': 0.3706122448980188, '1': 0.3706122448980188, '2': 0.3706122448980188, '3': 0.3706122448980188}\n", - "10:44:15 | single_host_cim_learner | INFO | ep 20 - performance: {'order_requirements': 2240000, 'container_shortage': 877014, 'operation_number': 3850338}, epsilons: {'0': 0.3689795918367976, '1': 0.3689795918367976, '2': 0.3689795918367976, '3': 0.3689795918367976}\n", - "10:44:19 | single_host_cim_learner | INFO | ep 21 - performance: {'order_requirements': 2240000, 'container_shortage': 737060, 'operation_number': 3896340}, epsilons: {'0': 0.36734693877557645, '1': 0.36734693877557645, '2': 0.36734693877557645, '3': 0.36734693877557645}\n", - "10:44:24 | single_host_cim_learner | INFO | ep 22 - performance: {'order_requirements': 2240000, 'container_shortage': 779835, 'operation_number': 4066950}, epsilons: {'0': 0.36571428571435527, '1': 0.36571428571435527, '2': 0.36571428571435527, '3': 0.36571428571435527}\n", - "10:44:29 | single_host_cim_learner | INFO | ep 23 - performance: {'order_requirements': 2240000, 'container_shortage': 739221, 'operation_number': 3852759}, epsilons: {'0': 0.3640816326531341, '1': 0.3640816326531341, '2': 0.3640816326531341, '3': 0.3640816326531341}\n", - "10:44:33 | single_host_cim_learner | INFO | ep 24 - performance: {'order_requirements': 2240000, 'container_shortage': 599721, 'operation_number': 3997403}, epsilons: {'0': 0.3624489795919129, '1': 0.3624489795919129, '2': 0.3624489795919129, '3': 0.3624489795919129}\n", - "10:44:38 | single_host_cim_learner | INFO | ep 25 - performance: {'order_requirements': 2240000, 'container_shortage': 781556, 'operation_number': 4142961}, epsilons: {'0': 0.36081632653069173, '1': 0.36081632653069173, '2': 0.36081632653069173, '3': 0.36081632653069173}\n", - "10:44:43 | single_host_cim_learner | INFO | ep 26 - performance: {'order_requirements': 2240000, 'container_shortage': 735564, 'operation_number': 3897327}, epsilons: {'0': 0.35918367346947055, '1': 0.35918367346947055, '2': 0.35918367346947055, '3': 0.35918367346947055}\n", - "10:44:48 | single_host_cim_learner | INFO | ep 27 - performance: {'order_requirements': 2240000, 'container_shortage': 806261, 'operation_number': 4010914}, epsilons: {'0': 0.3575510204082494, '1': 0.3575510204082494, '2': 0.3575510204082494, '3': 0.3575510204082494}\n", - "10:44:52 | single_host_cim_learner | INFO | ep 28 - performance: {'order_requirements': 2240000, 'container_shortage': 601312, 'operation_number': 4292785}, epsilons: {'0': 0.3559183673470282, '1': 0.3559183673470282, '2': 0.3559183673470282, '3': 0.3559183673470282}\n", - "10:44:57 | single_host_cim_learner | INFO | ep 29 - performance: {'order_requirements': 2240000, 'container_shortage': 559083, 'operation_number': 4170095}, epsilons: {'0': 0.354285714285807, '1': 0.354285714285807, '2': 0.354285714285807, '3': 0.354285714285807}\n", - "10:45:02 | single_host_cim_learner | INFO | ep 30 - performance: {'order_requirements': 2240000, 'container_shortage': 504573, 'operation_number': 4163412}, epsilons: {'0': 0.35265306122458584, '1': 0.35265306122458584, '2': 0.35265306122458584, '3': 0.35265306122458584}\n", - "10:45:07 | single_host_cim_learner | INFO | ep 31 - performance: {'order_requirements': 2240000, 'container_shortage': 851783, 'operation_number': 4303632}, epsilons: {'0': 0.35102040816336466, '1': 0.35102040816336466, '2': 0.35102040816336466, '3': 0.35102040816336466}\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "10:45:12 | single_host_cim_learner | INFO | ep 32 - performance: {'order_requirements': 2240000, 'container_shortage': 538322, 'operation_number': 3954792}, epsilons: {'0': 0.3493877551021435, '1': 0.3493877551021435, '2': 0.3493877551021435, '3': 0.3493877551021435}\n", - "10:45:16 | single_host_cim_learner | INFO | ep 33 - performance: {'order_requirements': 2240000, 'container_shortage': 622649, 'operation_number': 4035090}, epsilons: {'0': 0.3477551020409223, '1': 0.3477551020409223, '2': 0.3477551020409223, '3': 0.3477551020409223}\n", - "10:45:21 | single_host_cim_learner | INFO | ep 34 - performance: {'order_requirements': 2240000, 'container_shortage': 706875, 'operation_number': 3731018}, epsilons: {'0': 0.3461224489797011, '1': 0.3461224489797011, '2': 0.3461224489797011, '3': 0.3461224489797011}\n", - "10:45:26 | single_host_cim_learner | INFO | ep 35 - performance: {'order_requirements': 2240000, 'container_shortage': 476805, 'operation_number': 4059672}, epsilons: {'0': 0.34448979591847995, '1': 0.34448979591847995, '2': 0.34448979591847995, '3': 0.34448979591847995}\n", - "10:45:30 | single_host_cim_learner | INFO | ep 36 - performance: {'order_requirements': 2240000, 'container_shortage': 792619, 'operation_number': 3501539}, epsilons: {'0': 0.34285714285725877, '1': 0.34285714285725877, '2': 0.34285714285725877, '3': 0.34285714285725877}\n", - "10:45:35 | single_host_cim_learner | INFO | ep 37 - performance: {'order_requirements': 2240000, 'container_shortage': 508840, 'operation_number': 4132005}, epsilons: {'0': 0.3412244897960376, '1': 0.3412244897960376, '2': 0.3412244897960376, '3': 0.3412244897960376}\n", - "10:45:40 | single_host_cim_learner | INFO | ep 38 - performance: {'order_requirements': 2240000, 'container_shortage': 632638, 'operation_number': 3917010}, epsilons: {'0': 0.3395918367348164, '1': 0.3395918367348164, '2': 0.3395918367348164, '3': 0.3395918367348164}\n", - "10:45:45 | single_host_cim_learner | INFO | ep 39 - performance: {'order_requirements': 2240000, 'container_shortage': 558274, 'operation_number': 4168704}, epsilons: {'0': 0.33795918367359523, '1': 0.33795918367359523, '2': 0.33795918367359523, '3': 0.33795918367359523}\n", - "10:45:50 | single_host_cim_learner | INFO | ep 40 - performance: {'order_requirements': 2240000, 'container_shortage': 574387, 'operation_number': 4589579}, epsilons: {'0': 0.33632653061237405, '1': 0.33632653061237405, '2': 0.33632653061237405, '3': 0.33632653061237405}\n", - "10:45:55 | single_host_cim_learner | INFO | ep 41 - performance: {'order_requirements': 2240000, 'container_shortage': 514229, 'operation_number': 4196751}, epsilons: {'0': 0.3346938775511529, '1': 0.3346938775511529, '2': 0.3346938775511529, '3': 0.3346938775511529}\n", - "10:45:59 | single_host_cim_learner | INFO | ep 42 - performance: {'order_requirements': 2240000, 'container_shortage': 546013, 'operation_number': 4070179}, epsilons: {'0': 0.3330612244899317, '1': 0.3330612244899317, '2': 0.3330612244899317, '3': 0.3330612244899317}\n", - "10:46:04 | single_host_cim_learner | INFO | ep 43 - performance: {'order_requirements': 2240000, 'container_shortage': 566994, 'operation_number': 4073685}, epsilons: {'0': 0.3314285714287105, '1': 0.3314285714287105, '2': 0.3314285714287105, '3': 0.3314285714287105}\n", - "10:46:09 | single_host_cim_learner | INFO | ep 44 - performance: {'order_requirements': 2240000, 'container_shortage': 603963, 'operation_number': 4005202}, epsilons: {'0': 0.32979591836748934, '1': 0.32979591836748934, '2': 0.32979591836748934, '3': 0.32979591836748934}\n", - "10:46:13 | single_host_cim_learner | INFO | ep 45 - performance: {'order_requirements': 2240000, 'container_shortage': 404557, 'operation_number': 4628756}, epsilons: {'0': 0.32816326530626816, '1': 0.32816326530626816, '2': 0.32816326530626816, '3': 0.32816326530626816}\n", - "10:46:18 | single_host_cim_learner | INFO | ep 46 - performance: {'order_requirements': 2240000, 'container_shortage': 459178, 'operation_number': 4251896}, epsilons: {'0': 0.326530612245047, '1': 0.326530612245047, '2': 0.326530612245047, '3': 0.326530612245047}\n", - "10:46:23 | single_host_cim_learner | INFO | ep 47 - performance: {'order_requirements': 2240000, 'container_shortage': 680686, 'operation_number': 4587090}, epsilons: {'0': 0.3248979591838258, '1': 0.3248979591838258, '2': 0.3248979591838258, '3': 0.3248979591838258}\n", - "10:46:28 | single_host_cim_learner | INFO | ep 48 - performance: {'order_requirements': 2240000, 'container_shortage': 431512, 'operation_number': 4108741}, epsilons: {'0': 0.3232653061226046, '1': 0.3232653061226046, '2': 0.3232653061226046, '3': 0.3232653061226046}\n", - "10:46:32 | single_host_cim_learner | INFO | ep 49 - performance: {'order_requirements': 2240000, 'container_shortage': 551913, 'operation_number': 4381044}, epsilons: {'0': 0.32163265306138344, '1': 0.32163265306138344, '2': 0.32163265306138344, '3': 0.32163265306138344}\n", - "10:46:37 | single_host_cim_learner | INFO | ep 50 - performance: {'order_requirements': 2240000, 'container_shortage': 731294, 'operation_number': 4311078}, epsilons: {'0': 0.32000000000016227, '1': 0.32000000000016227, '2': 0.32000000000016227, '3': 0.32000000000016227}\n", - "10:46:42 | single_host_cim_learner | INFO | ep 51 - performance: {'order_requirements': 2240000, 'container_shortage': 643775, 'operation_number': 3956599}, epsilons: {'0': 0.3183673469389411, '1': 0.3183673469389411, '2': 0.3183673469389411, '3': 0.3183673469389411}\n", - "10:46:47 | single_host_cim_learner | INFO | ep 52 - performance: {'order_requirements': 2240000, 'container_shortage': 520778, 'operation_number': 4270831}, epsilons: {'0': 0.3119673469389539, '1': 0.3119673469389539, '2': 0.3119673469389539, '3': 0.3119673469389539}\n", - "10:46:51 | single_host_cim_learner | INFO | ep 53 - performance: {'order_requirements': 2240000, 'container_shortage': 424125, 'operation_number': 4409254}, epsilons: {'0': 0.3055673469389667, '1': 0.3055673469389667, '2': 0.3055673469389667, '3': 0.3055673469389667}\n", - "10:46:56 | single_host_cim_learner | INFO | ep 54 - performance: {'order_requirements': 2240000, 'container_shortage': 626297, 'operation_number': 4028701}, epsilons: {'0': 0.2991673469389795, '1': 0.2991673469389795, '2': 0.2991673469389795, '3': 0.2991673469389795}\n", - "10:47:01 | single_host_cim_learner | INFO | ep 55 - performance: {'order_requirements': 2240000, 'container_shortage': 380644, 'operation_number': 4520496}, epsilons: {'0': 0.2927673469389923, '1': 0.2927673469389923, '2': 0.2927673469389923, '3': 0.2927673469389923}\n", - "10:47:06 | single_host_cim_learner | INFO | ep 56 - performance: {'order_requirements': 2240000, 'container_shortage': 301492, 'operation_number': 4504974}, epsilons: {'0': 0.2863673469390051, '1': 0.2863673469390051, '2': 0.2863673469390051, '3': 0.2863673469390051}\n", - "10:47:11 | single_host_cim_learner | INFO | ep 57 - performance: {'order_requirements': 2240000, 'container_shortage': 480587, 'operation_number': 4638024}, epsilons: {'0': 0.2799673469390179, '1': 0.2799673469390179, '2': 0.2799673469390179, '3': 0.2799673469390179}\n", - "10:47:16 | single_host_cim_learner | INFO | ep 58 - performance: {'order_requirements': 2240000, 'container_shortage': 279138, 'operation_number': 4420042}, epsilons: {'0': 0.27356734693903073, '1': 0.27356734693903073, '2': 0.27356734693903073, '3': 0.27356734693903073}\n", - "10:47:21 | single_host_cim_learner | INFO | ep 59 - performance: {'order_requirements': 2240000, 'container_shortage': 309761, 'operation_number': 4384082}, epsilons: {'0': 0.26716734693904354, '1': 0.26716734693904354, '2': 0.26716734693904354, '3': 0.26716734693904354}\n", - "10:47:25 | single_host_cim_learner | INFO | ep 60 - performance: {'order_requirements': 2240000, 'container_shortage': 232598, 'operation_number': 4498145}, epsilons: {'0': 0.26076734693905634, '1': 0.26076734693905634, '2': 0.26076734693905634, '3': 0.26076734693905634}\n", - "10:47:30 | single_host_cim_learner | INFO | ep 61 - performance: {'order_requirements': 2240000, 'container_shortage': 364515, 'operation_number': 4578089}, epsilons: {'0': 0.25436734693906915, '1': 0.25436734693906915, '2': 0.25436734693906915, '3': 0.25436734693906915}\n", - "10:47:35 | single_host_cim_learner | INFO | ep 62 - performance: {'order_requirements': 2240000, 'container_shortage': 276436, 'operation_number': 4315130}, epsilons: {'0': 0.24796734693908196, '1': 0.24796734693908196, '2': 0.24796734693908196, '3': 0.24796734693908196}\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "10:47:40 | single_host_cim_learner | INFO | ep 63 - performance: {'order_requirements': 2240000, 'container_shortage': 377861, 'operation_number': 4280589}, epsilons: {'0': 0.24156734693909476, '1': 0.24156734693909476, '2': 0.24156734693909476, '3': 0.24156734693909476}\n", - "10:47:45 | single_host_cim_learner | INFO | ep 64 - performance: {'order_requirements': 2240000, 'container_shortage': 256642, 'operation_number': 4542020}, epsilons: {'0': 0.23516734693910757, '1': 0.23516734693910757, '2': 0.23516734693910757, '3': 0.23516734693910757}\n", - "10:47:50 | single_host_cim_learner | INFO | ep 65 - performance: {'order_requirements': 2240000, 'container_shortage': 246418, 'operation_number': 4533773}, epsilons: {'0': 0.22876734693912038, '1': 0.22876734693912038, '2': 0.22876734693912038, '3': 0.22876734693912038}\n", - "10:47:55 | single_host_cim_learner | INFO | ep 66 - performance: {'order_requirements': 2240000, 'container_shortage': 251040, 'operation_number': 4303025}, epsilons: {'0': 0.22236734693913318, '1': 0.22236734693913318, '2': 0.22236734693913318, '3': 0.22236734693913318}\n", - "10:48:00 | single_host_cim_learner | INFO | ep 67 - performance: {'order_requirements': 2240000, 'container_shortage': 333194, 'operation_number': 4391156}, epsilons: {'0': 0.215967346939146, '1': 0.215967346939146, '2': 0.215967346939146, '3': 0.215967346939146}\n", - "10:48:05 | single_host_cim_learner | INFO | ep 68 - performance: {'order_requirements': 2240000, 'container_shortage': 316087, 'operation_number': 4189109}, epsilons: {'0': 0.2095673469391588, '1': 0.2095673469391588, '2': 0.2095673469391588, '3': 0.2095673469391588}\n", - "10:48:10 | single_host_cim_learner | INFO | ep 69 - performance: {'order_requirements': 2240000, 'container_shortage': 338686, 'operation_number': 4063866}, epsilons: {'0': 0.2031673469391716, '1': 0.2031673469391716, '2': 0.2031673469391716, '3': 0.2031673469391716}\n", - "10:48:15 | single_host_cim_learner | INFO | ep 70 - performance: {'order_requirements': 2240000, 'container_shortage': 239006, 'operation_number': 4512630}, epsilons: {'0': 0.1967673469391844, '1': 0.1967673469391844, '2': 0.1967673469391844, '3': 0.1967673469391844}\n", - "10:48:20 | single_host_cim_learner | INFO | ep 71 - performance: {'order_requirements': 2240000, 'container_shortage': 239085, 'operation_number': 4375504}, epsilons: {'0': 0.1903673469391972, '1': 0.1903673469391972, '2': 0.1903673469391972, '3': 0.1903673469391972}\n", - "10:48:25 | single_host_cim_learner | INFO | ep 72 - performance: {'order_requirements': 2240000, 'container_shortage': 213239, 'operation_number': 4399888}, epsilons: {'0': 0.18396734693921002, '1': 0.18396734693921002, '2': 0.18396734693921002, '3': 0.18396734693921002}\n", - "10:48:30 | single_host_cim_learner | INFO | ep 73 - performance: {'order_requirements': 2240000, 'container_shortage': 427995, 'operation_number': 4177450}, epsilons: {'0': 0.17756734693922283, '1': 0.17756734693922283, '2': 0.17756734693922283, '3': 0.17756734693922283}\n", - "10:48:35 | single_host_cim_learner | INFO | ep 74 - performance: {'order_requirements': 2240000, 'container_shortage': 585601, 'operation_number': 3969371}, epsilons: {'0': 0.17116734693923563, '1': 0.17116734693923563, '2': 0.17116734693923563, '3': 0.17116734693923563}\n", - "10:48:40 | single_host_cim_learner | INFO | ep 75 - performance: {'order_requirements': 2240000, 'container_shortage': 326819, 'operation_number': 4122134}, epsilons: {'0': 0.16476734693924844, '1': 0.16476734693924844, '2': 0.16476734693924844, '3': 0.16476734693924844}\n", - "10:48:45 | single_host_cim_learner | INFO | ep 76 - performance: {'order_requirements': 2240000, 'container_shortage': 148429, 'operation_number': 4367860}, epsilons: {'0': 0.15836734693926124, '1': 0.15836734693926124, '2': 0.15836734693926124, '3': 0.15836734693926124}\n", - "10:48:50 | single_host_cim_learner | INFO | ep 77 - performance: {'order_requirements': 2240000, 'container_shortage': 207352, 'operation_number': 4388219}, epsilons: {'0': 0.15196734693927405, '1': 0.15196734693927405, '2': 0.15196734693927405, '3': 0.15196734693927405}\n", - "10:48:55 | single_host_cim_learner | INFO | ep 78 - performance: {'order_requirements': 2240000, 'container_shortage': 260726, 'operation_number': 4301982}, epsilons: {'0': 0.14556734693928686, '1': 0.14556734693928686, '2': 0.14556734693928686, '3': 0.14556734693928686}\n", - "10:49:00 | single_host_cim_learner | INFO | ep 79 - performance: {'order_requirements': 2240000, 'container_shortage': 129180, 'operation_number': 4560744}, epsilons: {'0': 0.13916734693929966, '1': 0.13916734693929966, '2': 0.13916734693929966, '3': 0.13916734693929966}\n", - "10:49:05 | single_host_cim_learner | INFO | ep 80 - performance: {'order_requirements': 2240000, 'container_shortage': 60834, 'operation_number': 4562955}, epsilons: {'0': 0.13276734693931247, '1': 0.13276734693931247, '2': 0.13276734693931247, '3': 0.13276734693931247}\n", - "10:49:10 | single_host_cim_learner | INFO | ep 81 - performance: {'order_requirements': 2240000, 'container_shortage': 90745, 'operation_number': 4433685}, epsilons: {'0': 0.12636734693932528, '1': 0.12636734693932528, '2': 0.12636734693932528, '3': 0.12636734693932528}\n", - "10:49:15 | single_host_cim_learner | INFO | ep 82 - performance: {'order_requirements': 2240000, 'container_shortage': 196778, 'operation_number': 4264127}, epsilons: {'0': 0.11996734693933808, '1': 0.11996734693933808, '2': 0.11996734693933808, '3': 0.11996734693933808}\n", - "10:49:20 | single_host_cim_learner | INFO | ep 83 - performance: {'order_requirements': 2240000, 'container_shortage': 174344, 'operation_number': 4506476}, epsilons: {'0': 0.11356734693935089, '1': 0.11356734693935089, '2': 0.11356734693935089, '3': 0.11356734693935089}\n", - "10:49:25 | single_host_cim_learner | INFO | ep 84 - performance: {'order_requirements': 2240000, 'container_shortage': 163431, 'operation_number': 4402926}, epsilons: {'0': 0.1071673469393637, '1': 0.1071673469393637, '2': 0.1071673469393637, '3': 0.1071673469393637}\n", - "10:49:29 | single_host_cim_learner | INFO | ep 85 - performance: {'order_requirements': 2240000, 'container_shortage': 359664, 'operation_number': 3931534}, epsilons: {'0': 0.1007673469393765, '1': 0.1007673469393765, '2': 0.1007673469393765, '3': 0.1007673469393765}\n", - "10:49:34 | single_host_cim_learner | INFO | ep 86 - performance: {'order_requirements': 2240000, 'container_shortage': 105613, 'operation_number': 4407328}, epsilons: {'0': 0.09436734693938931, '1': 0.09436734693938931, '2': 0.09436734693938931, '3': 0.09436734693938931}\n", - "10:49:39 | single_host_cim_learner | INFO | ep 87 - performance: {'order_requirements': 2240000, 'container_shortage': 518503, 'operation_number': 3770252}, epsilons: {'0': 0.08796734693940211, '1': 0.08796734693940211, '2': 0.08796734693940211, '3': 0.08796734693940211}\n", - "10:49:44 | single_host_cim_learner | INFO | ep 88 - performance: {'order_requirements': 2240000, 'container_shortage': 272846, 'operation_number': 4075937}, epsilons: {'0': 0.08156734693941492, '1': 0.08156734693941492, '2': 0.08156734693941492, '3': 0.08156734693941492}\n", - "10:49:49 | single_host_cim_learner | INFO | ep 89 - performance: {'order_requirements': 2240000, 'container_shortage': 302339, 'operation_number': 4011795}, epsilons: {'0': 0.07516734693942773, '1': 0.07516734693942773, '2': 0.07516734693942773, '3': 0.07516734693942773}\n", - "10:49:54 | single_host_cim_learner | INFO | ep 90 - performance: {'order_requirements': 2240000, 'container_shortage': 184906, 'operation_number': 4262003}, epsilons: {'0': 0.06876734693944053, '1': 0.06876734693944053, '2': 0.06876734693944053, '3': 0.06876734693944053}\n", - "10:49:59 | single_host_cim_learner | INFO | ep 91 - performance: {'order_requirements': 2240000, 'container_shortage': 316880, 'operation_number': 3925505}, epsilons: {'0': 0.06236734693945333, '1': 0.06236734693945333, '2': 0.06236734693945333, '3': 0.06236734693945333}\n", - "10:50:04 | single_host_cim_learner | INFO | ep 92 - performance: {'order_requirements': 2240000, 'container_shortage': 182421, 'operation_number': 4191916}, epsilons: {'0': 0.05596734693946613, '1': 0.05596734693946613, '2': 0.05596734693946613, '3': 0.05596734693946613}\n", - "10:50:09 | single_host_cim_learner | INFO | ep 93 - performance: {'order_requirements': 2240000, 'container_shortage': 87904, 'operation_number': 4366189}, epsilons: {'0': 0.04956734693947893, '1': 0.04956734693947893, '2': 0.04956734693947893, '3': 0.04956734693947893}\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "10:50:15 | single_host_cim_learner | INFO | ep 94 - performance: {'order_requirements': 2240000, 'container_shortage': 288975, 'operation_number': 3916529}, epsilons: {'0': 0.04316734693949173, '1': 0.04316734693949173, '2': 0.04316734693949173, '3': 0.04316734693949173}\n", - "10:50:20 | single_host_cim_learner | INFO | ep 95 - performance: {'order_requirements': 2240000, 'container_shortage': 259727, 'operation_number': 4057942}, epsilons: {'0': 0.03676734693950453, '1': 0.03676734693950453, '2': 0.03676734693950453, '3': 0.03676734693950453}\n", - "10:50:25 | single_host_cim_learner | INFO | ep 96 - performance: {'order_requirements': 2240000, 'container_shortage': 284335, 'operation_number': 3962831}, epsilons: {'0': 0.03036734693951733, '1': 0.03036734693951733, '2': 0.03036734693951733, '3': 0.03036734693951733}\n", - "10:50:30 | single_host_cim_learner | INFO | ep 97 - performance: {'order_requirements': 2240000, 'container_shortage': 55845, 'operation_number': 4383305}, epsilons: {'0': 0.023967346939530128, '1': 0.023967346939530128, '2': 0.023967346939530128, '3': 0.023967346939530128}\n", - "10:50:35 | single_host_cim_learner | INFO | ep 98 - performance: {'order_requirements': 2240000, 'container_shortage': 252218, 'operation_number': 4007054}, epsilons: {'0': 0.017567346939542927, '1': 0.017567346939542927, '2': 0.017567346939542927, '3': 0.017567346939542927}\n", - "10:50:40 | single_host_cim_learner | INFO | ep 99 - performance: {'order_requirements': 2240000, 'container_shortage': 45084, 'operation_number': 4339789}, epsilons: {'0': 0.011167346939555726, '1': 0.011167346939555726, '2': 0.011167346939555726, '3': 0.011167346939555726}\n", - "10:50:45 | single_host_cim_learner | INFO | ep 100 - performance: {'order_requirements': 2240000, 'container_shortage': 265096, 'operation_number': 3919248}, epsilons: {'0': 0.004767346939568526, '1': 0.004767346939568526, '2': 0.004767346939568526, '3': 0.004767346939568526}\n" + "ename": "NameError", + "evalue": "name 'nn' is not defined", + "output_type": "error", + "traceback": [ + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mNameError\u001b[0m Traceback (most recent call last)", + "\u001b[0;32m\u001b[0m in \u001b[0;36m\u001b[0;34m\u001b[0m\n\u001b[1;32m 23\u001b[0m agent_manager = DQNAgentManager(name=\"cim_learner\",\n\u001b[1;32m 24\u001b[0m \u001b[0mmode\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mAgentManagerMode\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mTRAIN_INFERENCE\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 25\u001b[0;31m \u001b[0magent_dict\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mcreate_dqn_agents\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0magent_id_list\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 26\u001b[0m \u001b[0mstate_shaper\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mstate_shaper\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 27\u001b[0m \u001b[0maction_shaper\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0maction_shaper\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m\u001b[0m in \u001b[0;36mcreate_dqn_agents\u001b[0;34m(agent_id_list)\u001b[0m\n\u001b[1;32m 21\u001b[0m \u001b[0minput_dim\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0minput_dim\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 22\u001b[0m \u001b[0moutput_dim\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mnum_actions\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 23\u001b[0;31m \u001b[0mactivation\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mnn\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mLeakyReLU\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 24\u001b[0m \u001b[0mhidden_dims\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m256\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m128\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m64\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 25\u001b[0m \u001b[0msoftmax_enabled\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mFalse\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;31mNameError\u001b[0m: name 'nn' is not defined" ] } ], "source": [ "from maro.simulator import Env\n", - "from maro.rl import SimpleLearner, SimpleActor, AgentMode, TwoPhaseLinearExplorer\n", + "from maro.rl import SimpleLearner, SimpleActor, AgentManagerMode, TwoPhaseLinearExplorer\n", "from maro.utils import Logger, LogFormat\n", "\n", "# Step 1: initialize a CIM environment for using a toy dataset. \n", @@ -462,7 +351,7 @@ "\n", "# Step 3: create an agent manager.\n", "agent_manager = DQNAgentManager(name=\"cim_learner\",\n", - " mode=AgentMode.TRAIN_INFERENCE,\n", + " mode=AgentManagerMode.TRAIN_INFERENCE,\n", " agent_dict=create_dqn_agents(agent_id_list),\n", " state_shaper=state_shaper,\n", " action_shaper=action_shaper,\n", @@ -476,6 +365,13 @@ "\n", "learner.train(max_episode=100)" ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] } ], "metadata": { From 040220fb648136f0d010cabba16eef16293cfd61 Mon Sep 17 00:00:00 2001 From: ysqyang Date: Thu, 29 Oct 2020 14:43:36 +0800 Subject: [PATCH 083/337] fixed lint formatting issues --- maro/rl/dist_topologies/experience_collection.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/maro/rl/dist_topologies/experience_collection.py b/maro/rl/dist_topologies/experience_collection.py index f4cea35d9..d490ff781 100644 --- a/maro/rl/dist_topologies/experience_collection.py +++ b/maro/rl/dist_topologies/experience_collection.py @@ -36,7 +36,7 @@ def merge_experiences_with_trajectory_boundaries(trajectories_by_source) -> dict trajectories_by_source (dict): Agent's trajectories from multiple sources. Returns: - A list of trajectories for each agent. + A list of trajectories for each agent. """ merged = defaultdict(list) for exp_by_agent in trajectories_by_source.values(): @@ -44,5 +44,3 @@ def merge_experiences_with_trajectory_boundaries(trajectories_by_source) -> dict merged[agent_id].append(trajectory) return merged - - From bc1d9f4c3b25ecb620bdfa17e30a9b1f80301d9f Mon Sep 17 00:00:00 2001 From: ysqyang Date: Thu, 29 Oct 2020 14:57:48 +0800 Subject: [PATCH 084/337] fixed a typo --- docs/source/examples/multi_agent_dqn_cim.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/examples/multi_agent_dqn_cim.rst b/docs/source/examples/multi_agent_dqn_cim.rst index d54df810b..34f5ea5f5 100644 --- a/docs/source/examples/multi_agent_dqn_cim.rst +++ b/docs/source/examples/multi_agent_dqn_cim.rst @@ -142,7 +142,7 @@ the DQN algorithm and an experience pool for each agent. set_seeds(config.agents.seed) num_actions = config.agents.algorithm.num_actions for agent_id in self._agent_id_list: - eval_model = LearningModel(decision_layers=MLPFullyConnectedNet(name=f'{agent_id}.policy', + eval_model = LearningModel(decision_layers=FullyConnectedNet(name=f'{agent_id}.policy', input_dim=self._state_shaper.dim, output_dim=num_actions, **config.agents.algorithm.model) From 472360f8f6c8a56a0139cdb6b0b1d467dde5f3bf Mon Sep 17 00:00:00 2001 From: ysqyang Date: Thu, 29 Oct 2020 17:03:51 +0800 Subject: [PATCH 085/337] fixed some PR comments --- .../simple_early_stopping_checker.py | 38 ++++++++++++++----- maro/rl/explorer/simple_explorer.py | 30 +++++++-------- maro/rl/learner/simple_learner.py | 24 ++++++------ .../{fully_connected_net.py => fc_net.py} | 0 maro/utils/exception/error_code.py | 4 +- maro/utils/exception/rl_toolkit_exception.py | 12 ++++++ 6 files changed, 71 insertions(+), 37 deletions(-) rename maro/rl/models/{fully_connected_net.py => fc_net.py} (100%) diff --git a/maro/rl/early_stopping/simple_early_stopping_checker.py b/maro/rl/early_stopping/simple_early_stopping_checker.py index 9d7cdef42..7a07a4eb5 100644 --- a/maro/rl/early_stopping/simple_early_stopping_checker.py +++ b/maro/rl/early_stopping/simple_early_stopping_checker.py @@ -1,29 +1,47 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT license. -from typing import Callable - from statistics import mean, stdev from .abs_early_stopping_checker import AbsEarlyStoppingChecker -class SimpleEarlyStoppingChecker(AbsEarlyStoppingChecker): +class RSDEarlyStoppingChecker(AbsEarlyStoppingChecker): + """Simple early stopping checker based on the mean and standard deviation of the last k performance records. + + Args: + last_k (int): Number of the latest performance records to check for early stopping. + threshold (float): The threshold value against which the early stopping metric is compared. The early stopping + condition is satisfied if the metric obtained using the ``performance_metric_func`` is below this threshold. + """ + def __init__(self, last_k: int, threshold: float): + super().__init__() + self._last_k = last_k + self._threshold = threshold + + def __call__(self, metric_series): + if self._last_k > len(metric_series): + return False + else: + return stdev(metric_series) / mean(metric_series) < self._threshold + + +class MaxDeltaEarlyStoppingChecker(AbsEarlyStoppingChecker): """Simple early stopping checker based on the mean and standard deviation of the last k performance records. Args: last_k (int): Number of the latest performance records to check for early stopping. - metric_func (Callable): A function to obtain the metric from a performance record to be evaluated against a - threshold value. threshold (float): The threshold value against which the early stopping metric is compared. The early stopping condition is satisfied if the metric obtained using the ``performance_metric_func`` is below this threshold. """ - def __init__(self, last_k: int, metric_func: Callable, threshold: float): + def __init__(self, last_k: int, threshold: float): super().__init__() self._last_k = last_k - self._metric_func = metric_func self._threshold = threshold - def __call__(self, performance_history): - metrics = list(map(self._metric_func, performance_history[-self._last_k:])) - return stdev(metrics) / mean(metrics) < self._threshold + def __call__(self, metric_series): + if self._last_k > len(metric_series): + return False + else: + max_delta = max(abs(val2 - val1) / val1 for val1, val2 in zip(metric_series, metric_series[1:])) + return max_delta < self._threshold diff --git a/maro/rl/explorer/simple_explorer.py b/maro/rl/explorer/simple_explorer.py index 804925f73..53a9a45e4 100644 --- a/maro/rl/explorer/simple_explorer.py +++ b/maro/rl/explorer/simple_explorer.py @@ -24,26 +24,26 @@ class TwoPhaseLinearExplorer(AbsExplorer): """Exploration schedule that consists of two linear schedules separated by a split point. Args: - progress_split (float): The point where the switch from the first linear schedule to the second occurs. + start_eps (float): Exploration rate for the first episode. + mid_eps (float): Exploration rate for the last episode. + end_eps (float): The exploration rate where the switch from the first linear schedule to the second occurs. + In other words, this is the exploration rate where the first linear schedule ends and the second begins. + split_point (float): The point where the switch from the first linear schedule to the second occurs. Here "point" means the percentage of training loop completion, i.e., current_episode / max_episode, which means it must be a floating point number between 0 and 1.0. - eps_split (float): The exploration rate where the switch from the first linear schedule to the second occurs. - In other words, this is the exploration rate where the first linear schedule ends and the second begins. - max_eps (float): Maximum exploration rate, i.e., the exploration rate for the first episode. - min_eps (float): Minimum exploration rate, i.e., the exploration rate for the last episode. """ - def __init__(self, progress_split: float, eps_split: float, max_eps: float, min_eps: float = 0): + def __init__(self, start_eps: float, mid_eps: float, end_eps: float, split_point: float): super().__init__() - if progress_split > 1.0 or progress_split < 0.0: - raise ValueError("progress_split must be between 0 and 1.0") - self._progress_split = progress_split - self._eps_split = eps_split - self._max_eps = max_eps - self._min_eps = min_eps + if split_point > 1.0 or split_point < 0.0: + raise ValueError("split_point must be between 0 and 1.0") + self._split_point = split_point + self._start_eps = start_eps + self._mid_eps = mid_eps + self._end_eps = end_eps def generate_epsilon(self, current_ep, max_ep, performance_history=None): progress = current_ep / (max_ep - 1) - if progress <= self._progress_split: - return self._max_eps - (self._max_eps - self._eps_split) * progress / self._progress_split + if progress <= self._split_point: + return self._start_eps - (self._start_eps - self._mid_eps) * progress / self._split_point else: - return self._min_eps + (self._eps_split - self._min_eps) * (1 - progress) / (1 - self._progress_split) + return self._end_eps + (self._mid_eps - self._end_eps) * (1 - progress) / (1 - self._split_point) diff --git a/maro/rl/learner/simple_learner.py b/maro/rl/learner/simple_learner.py index b379f2825..84f0e1275 100644 --- a/maro/rl/learner/simple_learner.py +++ b/maro/rl/learner/simple_learner.py @@ -11,6 +11,7 @@ from maro.rl.explorer.abs_explorer import AbsExplorer from maro.rl.dist_topologies.single_learner_multi_actor_sync_mode import ActorProxy from maro.utils import Logger, DummyLogger +from maro.utils.exception.rl_toolkit_exception import InvalidEpisodeError, InfiniteTrainingLoopError class SimpleLearner(AbsLearner): @@ -54,29 +55,30 @@ def _sample(self, ep, max_ep): self._logger.info(f"ep {ep} - performance: {performance}, epsilons: {epsilon_dict}") return performance, exp_by_agent - def train(self, max_episode: int, early_stopping_checker: Callable = None, early_stopping_check_ep: int = None): + def train(self, max_episode: int, early_stopping_checker: Callable = None, warmup_ep: int = None): """Main loop for collecting experiences from the actor and using them to update policies. Args: - max_episode (int): number of episodes to be run. If negative, the training loop will run forever unless + max_episode (int): number of episodes to be run. If -1, the training loop will run forever unless an ``early_stopping_checker`` is provided and the early stopping condition is met. early_stopping_checker (Callable): A Callable object to determine whether the training loop should be terminated based on the latest performances. Defaults to None. - early_stopping_check_ep (int): Episode from which early stopping check is initiated. Defaults to None. + warmup_ep (int): Episode from which early stopping check is initiated. Defaults to None. """ - if max_episode < 0 and early_stopping_checker is None: - warnings.warn( + if max_episode < -1: + raise InvalidEpisodeError("max_episode can only be a non-negative integer or -1.") + if max_episode == -1 and early_stopping_checker is None: + raise InfiniteTrainingLoopError( "The training loop will run forever since neither maximum episode nor early stopping checker " "is provided. " ) episode = 0 - while max_episode < 0 or episode < max_episode: + while max_episode == -1 or episode < max_episode: performance, exp_by_agent = self._sample(episode, max_episode) self._performance_history.append(performance) - if early_stopping_checker is not None and \ - (early_stopping_check_ep is None or episode >= early_stopping_check_ep) and \ + if early_stopping_checker is not None and (warmup_ep is None or episode >= warmup_ep) and \ early_stopping_checker(self._performance_history): - self._logger.info("Early stopping condition satisfied. Training complete.") + self._logger.info("Early stopping condition hit. Training complete.") break self._trainable_agents.train(exp_by_agent) episode += 1 @@ -89,11 +91,11 @@ def test(self): ) self._logger.info(f"test performance: {performance}") - def exit(self): + def exit(self, code: int = 0): """Tell the remote actor to exit""" if isinstance(self._actor, ActorProxy): self._actor.roll_out(done=True) - sys.exit() + sys.exit(code) def dump_models(self, model_dump_dir: str): self._trainable_agents.dump_models_to_files(model_dump_dir) diff --git a/maro/rl/models/fully_connected_net.py b/maro/rl/models/fc_net.py similarity index 100% rename from maro/rl/models/fully_connected_net.py rename to maro/rl/models/fc_net.py diff --git a/maro/utils/exception/error_code.py b/maro/utils/exception/error_code.py index efb728208..f09dcd12f 100644 --- a/maro/utils/exception/error_code.py +++ b/maro/utils/exception/error_code.py @@ -38,5 +38,7 @@ 4001: "Unsupported Agent Mode", 4002: "Missing Shaper", 4003: "Wrong Agent Manager Mode", - 4004: "Store Misalignment Error" + 4004: "Store Misalignment Error", + 4005: "Invalid Episode", + 4006: "Infinite Training Loop" } diff --git a/maro/utils/exception/rl_toolkit_exception.py b/maro/utils/exception/rl_toolkit_exception.py index 93684b8eb..c2d6068fa 100644 --- a/maro/utils/exception/rl_toolkit_exception.py +++ b/maro/utils/exception/rl_toolkit_exception.py @@ -27,3 +27,15 @@ class StoreMisalignmentError(MAROException): sizes.""" def __init__(self, msg: str = None): super().__init__(4004, msg) + + +class InvalidEpisodeError(MAROException): + """Raised when the ``max_episode`` passed to the the ``SimpleLearner``'s ``train`` method is negative and not -1.""" + def __init__(self, msg: str = None): + super().__init__(4005, msg) + + +class InfiniteTrainingLoopError(MAROException): + """Raised when the ``SimpleLearner``'s training loop becomes infinite.""" + def __init__(self, msg: str = None): + super().__init__(4006, msg) From ea1448d508fafbccffd4d10cdc624b02bdc8367b Mon Sep 17 00:00:00 2001 From: ysqyang Date: Fri, 30 Oct 2020 00:25:57 +0800 Subject: [PATCH 086/337] fixed more PR comments --- docs/source/examples/multi_agent_dqn_cim.rst | 205 +++++++++++------- examples/cim/dqn/config.yml | 2 +- examples/cim/dqn/dist_learner.py | 6 +- examples/cim/dqn/single_process_launcher.py | 11 +- maro/rl/__init__.py | 7 +- .../simple_early_stopping_checker.py | 1 + maro/rl/learner/simple_learner.py | 27 ++- 7 files changed, 155 insertions(+), 104 deletions(-) diff --git a/docs/source/examples/multi_agent_dqn_cim.rst b/docs/source/examples/multi_agent_dqn_cim.rst index 34f5ea5f5..c5641f719 100644 --- a/docs/source/examples/multi_agent_dqn_cim.rst +++ b/docs/source/examples/multi_agent_dqn_cim.rst @@ -71,7 +71,6 @@ an episode trajectory to trainable experiences for RL agents. For this specific combination of fulfillment and shortage in a limited time window. .. code-block:: python - class TruncatedExperienceShaper(ExperienceShaper): ... def __call__(self, trajectory, snapshot_list): @@ -85,26 +84,10 @@ combination of fulfillment and shortage in a limited time window. experiences["state"].append(transition["state"]) experiences["action"].append(transition["action"]) experiences["reward"].append(self._compute_reward(transition["event"], snapshot_list)) - experiences["next_state"].append(trajectory[i + 1]["state"]) + experiences["next_state"].append(trajectory[i+1]["state"]) return experiences_by_agent - def _compute_reward(self, decision_event, snapshot_list): - start_tick = decision_event.tick + 1 - end_tick = decision_event.tick + self._time_window - ticks = list(range(start_tick, end_tick)) - - # calculate tc reward - future_fulfillment = snapshot_list["ports"][ticks::"fulfillment"] - future_shortage = snapshot_list["ports"][ticks::"shortage"] - decay_list = [self._time_decay_factor ** i for i in range(end_tick - start_tick) - for _ in range(future_fulfillment.shape[0] // (end_tick - start_tick))] - - tot_fulfillment = np.dot(future_fulfillment, decay_list) - tot_shortage = np.dot(future_shortage, decay_list) - - return np.float(self._fulfillment_factor * tot_fulfillment - self._shortage_factor * tot_shortage) - Agent ----- @@ -131,32 +114,58 @@ abstraction of a port. We choose DQN as our underlying learning algorithm with a Agent Manager ------------- +The complexities of the environment can be isolated from the learning algorithm by using an `Agent manager `_ -is an agent assembler and isolates the complexities of the environment and algorithm. For this scenario, It will load -the DQN algorithm and an experience pool for each agent. +to manage individual agents. We define a function to create the agents and an AgentManager class +that implements the abstract ``train`` method in accordance with the DQN algorithm. . .. code-block:: python + def create_dqn_agents(agent_id_list, config): + num_actions = config.algorithm.num_actions + set_seeds(config.seed) + agent_dict = {} + for agent_id in agent_id_list: + eval_model = LearningModel( + decision_layers=FullyConnectedNet( + name=f'{agent_id}.policy', + input_dim=config.algorithm.input_dim, + output_dim=num_actions, + activation=nn.LeakyReLU, **config.algorithm.model + ) + ) + + algorithm = DQN( + eval_model=eval_model, + optimizer_cls=RMSprop, + optimizer_params=config.algorithm.optimizer, + loss_func=nn.functional.smooth_l1_loss, + hyper_params=DQNHyperParams( + **config.algorithm.hyper_parameters, + num_actions=num_actions + ) + ) + + experience_pool = ColumnBasedStore(**config.experience_pool) + agent_dict[agent_id] = CIMAgent( + name=agent_id, + algorithm=algorithm, + experience_pool=experience_pool, + **config.training_loop_parameters + ) + + return agent_dict class DQNAgentManager(AbsAgentManager): - def _assemble(self, agent_dict): - set_seeds(config.agents.seed) - num_actions = config.agents.algorithm.num_actions - for agent_id in self._agent_id_list: - eval_model = LearningModel(decision_layers=FullyConnectedNet(name=f'{agent_id}.policy', - input_dim=self._state_shaper.dim, - output_dim=num_actions, - **config.agents.algorithm.model) - ) - - algorithm = DQN(model_dict={"eval": eval_model}, - optimizer_opt=(RMSprop, config.agents.algorithm.optimizer), - loss_func_dict={"eval": smooth_l1_loss}, - hyper_params=DQNHyperParams(**config.agents.algorithm.hyper_parameters, - num_actions=num_actions)) - - experience_pool = ColumnBasedStore(**config.agents.experience_pool) - agent_dict[agent_id] = CIMAgent(name=agent_id, algorithm=algorithm, experience_pool=experience_pool, - **config.agents.training_loop_parameters) + def train(self, experiences_by_agent, performance=None): + self._assert_train_mode() + + # store experiences for each agent + for agent_id, exp in experiences_by_agent.items(): + exp.update({"loss": [1e8] * len(list(exp.values())[0])}) + self.agent_dict[agent_id].store_experiences(exp) + + for agent in self.agent_dict.values(): + agent.train() Main Loop with Actor and Learner (Single Process) ------------------------------------------------- @@ -164,38 +173,46 @@ Main Loop with Actor and Learner (Single Process) This single-process workflow of a learning policy's interaction with a MARO environment is comprised of: - Initializing an environment with specific scenario and topology parameters. - Defining scenario-specific components, e.g. shapers. -- Creating an agent manager, which assembles underlying agents. +- Creating agents and an agent manager. - Creating an `actor `_ and a `learner `_ to start the training process in which the agent manager interacts with the environment for collecting experiences and updating policies. .. code-block::python - env = Env(config.env.scenario, config.env.topology, durations=config.env.durations) agent_id_list = [str(agent_id) for agent_id in env.agent_idx_list] + state_shaper = CIMStateShaper(**config.state_shaping) action_shaper = CIMActionShaper(action_space=list(np.linspace(-1.0, 1.0, config.agents.algorithm.num_actions))) experience_shaper = TruncatedExperienceShaper(**config.experience_shaping.truncated) - exploration_config = {"epsilon_range_dict": {"_all_": config.exploration.epsilon_range}, - "split_point_dict": {"_all_": config.exploration.split_point}, - "with_cache": config.exploration.with_cache - } - explorer = TwoPhaseLinearExplorer(agent_id_list, config.general.total_training_episodes, **exploration_config) - - agent_manager = DQNAgentManager(name="cim_learner", - mode=AgentMode.TRAIN_INFERENCE, - agent_id_list=agent_id_list, - state_shaper=state_shaper, - action_shaper=action_shaper, - experience_shaper=experience_shaper, - explorer=explorer) + agent_manager = DQNAgentManager( + name="cim_learner", + mode=AgentManagerMode.TRAIN_INFERENCE, + agent_dict=create_dqn_agents(agent_id_list, config.agents), + state_shaper=state_shaper, + action_shaper=action_shaper, + experience_shaper=experience_shaper + ) + + early_stopping_checker = MaxDeltaEarlyStoppingChecker( + last_k=config.general.early_stopping.last_k, + threshold=config.general.early_stopping.threshold + ) actor = SimpleActor(env=env, inference_agents=agent_manager) - learner = SimpleLearner(trainable_agents=agent_manager, actor=actor, - logger=Logger("single_host_cim_learner", auto_timestamp=False)) - - learner.train(total_episodes=config.general.total_training_episodes) + learner = SimpleLearner( + trainable_agents=agent_manager, + actor=actor, + explorer=TwoPhaseLinearExplorer(**config.exploration), + logger=Logger("single_host_cim_learner", auto_timestamp=False) + ) + learner.train( + max_episode=config.general.max_episode, + early_stopping_checker=early_stopping_checker, + warmup_ep=config.general.early_stopping.warmup_ep, + early_stopping_metric_func=lambda x: 1 - x["container_shortage"] / x["order_requirements"], + ) Main Loop with Actor and Learner (Distributed/Multi-process) @@ -214,20 +231,23 @@ wrap it in an ActorWorker instance. Finally, we launch the worker and it starts learner. The following code snippet shows the creation of an actor worker with a simple (local) actor wrapped inside. .. code-block:: python - - agent_manager = DQNAgentManager(name="cim_remote_actor", - agent_id_list=agent_id_list, - mode=AgentMode.INFERENCE, - state_shaper=state_shaper, - action_shaper=action_shaper, - experience_shaper=experience_shaper, - explorer=explorer) - proxy_params = {"group_name": config.distributed.group_name, - "expected_peers": config.distributed.actor.peer, - "redis_address": (config.distributed.redis.host_name, config.distributed.redis.port) - } - actor_worker = ActorWorker(local_actor=SimpleActor(env=env, inference_agents=agent_manager), - proxy_params=proxy_params) + agent_manager = DQNAgentManager( + name="distributed_cim_actor", + mode=AgentManagerMode.INFERENCE, + agent_dict=create_dqn_agents(agent_id_list, config.agents), + state_shaper=state_shaper, + action_shaper=action_shaper, + experience_shaper=experience_shaper, + ) + proxy_params = { + "group_name": os.environ["GROUP"], + "expected_peers": {"learner": 1}, + "redis_address": ("localhost", 6379) + } + actor_worker = ActorWorker( + local_actor=SimpleActor(env=env, inference_agents=agent_manager), + proxy_params=proxy_params + ) actor_worker.launch() On the learner side, an agent manager in AgentMode.TRAIN mode is required. However, it is not necessary to create shapers for an @@ -239,18 +259,35 @@ roll-out is performed remotely. The code snippet below shows the creation of a l inside. .. code-block:: python - - agent_manager = DQNAgentManager(name="cim_remote_learner", agent_id_list=agent_id_list, mode=AgentMode.TRAIN, - state_shaper=state_shaper, explorer=explorer) - - proxy_params = {"group_name": config.distributed.group_name, - "expected_peers": config.distributed.learner.peer, - "redis_address": (config.distributed.redis.host_name, config.distributed.redis.port) - } - learner = SimpleLearner(trainable_agents=agent_manager, - actor=ActorProxy(proxy_params=proxy_params), - logger=Logger("distributed_cim_learner", auto_timestamp=False)) - learner.train(total_episodes=config.general.total_training_episodes) + agent_manager = DQNAgentManager( + name="distributed_cim_learner", + mode=AgentManagerMode.TRAIN, + agent_dict=create_dqn_agents(agent_id_list, config.agents), + ) + + proxy_params = { + "group_name": os.environ["GROUP"], + "expected_peers": {"actor": int(os.environ["NUM_ACTORS"])}, + "redis_address": ("localhost", 6379) + } + + early_stopping_checker = SimpleEarlyStoppingChecker( + last_k=config.general.early_stopping.last_k, + threshold=config.general.early_stopping.threshold + ) + + learner = SimpleLearner( + trainable_agents=agent_manager, + actor=ActorProxy(proxy_params=proxy_params, experience_collecting_func=concat_experiences_by_agent), + explorer=TwoPhaseLinearExplorer(**config.exploration), + logger=Logger("distributed_cim_learner", auto_timestamp=False) + ) + learner.train( + max_episode=config.general.max_episode, + early_stopping_checker=early_stopping_checker, + warmup_ep=config.general.early_stopping.warmup_ep, + early_stopping_metric_func=lambda x: 1 - x["container_shortage"] / x["order_requirements"], + ) .. note:: diff --git a/examples/cim/dqn/config.yml b/examples/cim/dqn/config.yml index 3a74c700f..d9a632629 100644 --- a/examples/cim/dqn/config.yml +++ b/examples/cim/dqn/config.yml @@ -5,7 +5,7 @@ env: general: max_episode: 500 early_stopping: - start_ep: 50 + warmup_ep: 50 last_k: 5 threshold: 0.2 state_shaping: diff --git a/examples/cim/dqn/dist_learner.py b/examples/cim/dqn/dist_learner.py index 9e885df18..68903c88c 100644 --- a/examples/cim/dqn/dist_learner.py +++ b/examples/cim/dqn/dist_learner.py @@ -3,7 +3,7 @@ import os -from maro.rl import ActorProxy, AgentManagerMode, SimpleEarlyStoppingChecker, SimpleLearner, TwoPhaseLinearExplorer, \ +from maro.rl import ActorProxy, AgentManagerMode, MaxDeltaEarlyStoppingChecker, SimpleLearner, TwoPhaseLinearExplorer, \ concat_experiences_by_agent from maro.simulator import Env from maro.utils import Logger, convert_dottable @@ -31,7 +31,6 @@ def launch(config): early_stopping_checker = SimpleEarlyStoppingChecker( last_k=config.general.early_stopping.last_k, - metric_func=lambda x: 1 - x["container_shortage"] / x["order_requirements"], threshold=config.general.early_stopping.threshold ) @@ -44,7 +43,8 @@ def launch(config): learner.train( max_episode=config.general.max_episode, early_stopping_checker=early_stopping_checker, - early_stopping_check_ep=config.general.early_stopping.start_ep + warmup_ep=config.general.early_stopping.warmup_ep, + early_stopping_metric_func=lambda x: 1 - x["container_shortage"] / x["order_requirements"], ) learner.test() learner.dump_models(os.path.join(os.getcwd(), "models")) diff --git a/examples/cim/dqn/single_process_launcher.py b/examples/cim/dqn/single_process_launcher.py index 56af163d0..ca420f00e 100644 --- a/examples/cim/dqn/single_process_launcher.py +++ b/examples/cim/dqn/single_process_launcher.py @@ -1,13 +1,12 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT license. - import os import numpy as np from maro.simulator import Env -from maro.rl import AgentManagerMode, SimpleEarlyStoppingChecker, KStepExperienceShaper, SimpleLearner, SimpleActor, \ +from maro.rl import AgentManagerMode, MaxDeltaEarlyStoppingChecker, KStepExperienceShaper, SimpleLearner, SimpleActor, \ TwoPhaseLinearExplorer from maro.utils import Logger, convert_dottable @@ -38,7 +37,7 @@ def launch(config): **config.experience_shaping.k_step ) - # Step 3: Create an agent manager. + # Step 3: Create agents and an agent manager. agent_manager = DQNAgentManager( name="cim_learner", mode=AgentManagerMode.TRAIN_INFERENCE, @@ -49,9 +48,8 @@ def launch(config): ) # Step 4: Create an actor and a learner to start the training process. - early_stopping_checker = SimpleEarlyStoppingChecker( + early_stopping_checker = MaxDeltaEarlyStoppingChecker( last_k=config.general.early_stopping.last_k, - metric_func=lambda x: 1 - x["container_shortage"] / x["order_requirements"], threshold=config.general.early_stopping.threshold ) actor = SimpleActor(env=env, inference_agents=agent_manager) @@ -64,7 +62,8 @@ def launch(config): learner.train( max_episode=config.general.max_episode, early_stopping_checker=early_stopping_checker, - early_stopping_check_ep=config.general.early_stopping.start_ep + warmup_ep=config.general.early_stopping.warmup_ep, + early_stopping_metric_func=lambda x: 1 - x["container_shortage"] / x["order_requirements"], ) learner.test() learner.dump_models(os.path.join(os.getcwd(), "models")) diff --git a/maro/rl/__init__.py b/maro/rl/__init__.py index aa79181aa..eea2d84d9 100644 --- a/maro/rl/__init__.py +++ b/maro/rl/__init__.py @@ -11,7 +11,7 @@ from maro.rl.algorithms.abs_algorithm import AbsAlgorithm from maro.rl.algorithms.dqn import DQN, DQNHyperParams from maro.rl.models.learning_model import LearningModel -from maro.rl.models.fully_connected_net import FullyConnectedNet +from maro.rl.models.fc_net import FullyConnectedNet from maro.rl.storage.abs_store import AbsStore from maro.rl.storage.column_based_store import ColumnBasedStore from maro.rl.storage.utils import OverwriteType @@ -23,7 +23,7 @@ from maro.rl.explorer.abs_explorer import AbsExplorer from maro.rl.explorer.simple_explorer import LinearExplorer, TwoPhaseLinearExplorer from maro.rl.early_stopping.abs_early_stopping_checker import AbsEarlyStoppingChecker -from maro.rl.early_stopping.simple_early_stopping_checker import SimpleEarlyStoppingChecker +from maro.rl.early_stopping.simple_early_stopping_checker import RSDEarlyStoppingChecker, MaxDeltaEarlyStoppingChecker from maro.rl.dist_topologies.single_learner_multi_actor_sync_mode import ActorProxy, ActorWorker from maro.rl.dist_topologies.experience_collection import concat_experiences_by_agent, \ merge_experiences_with_trajectory_boundaries @@ -55,7 +55,8 @@ "LinearExplorer", "TwoPhaseLinearExplorer", "AbsEarlyStoppingChecker", - "SimpleEarlyStoppingChecker", + "RSDEarlyStoppingChecker", + "MaxDeltaEarlyStoppingChecker", "ActorProxy", "ActorWorker", "concat_experiences_by_agent", diff --git a/maro/rl/early_stopping/simple_early_stopping_checker.py b/maro/rl/early_stopping/simple_early_stopping_checker.py index 7a07a4eb5..17461ba82 100644 --- a/maro/rl/early_stopping/simple_early_stopping_checker.py +++ b/maro/rl/early_stopping/simple_early_stopping_checker.py @@ -43,5 +43,6 @@ def __call__(self, metric_series): if self._last_k > len(metric_series): return False else: + metric_series = metric_series[-self._last_k:] max_delta = max(abs(val2 - val1) / val1 for val1, val2 in zip(metric_series, metric_series[1:])) return max_delta < self._threshold diff --git a/maro/rl/learner/simple_learner.py b/maro/rl/learner/simple_learner.py index 84f0e1275..9497e6f14 100644 --- a/maro/rl/learner/simple_learner.py +++ b/maro/rl/learner/simple_learner.py @@ -21,14 +21,15 @@ class SimpleLearner(AbsLearner): trainable_agents (AbsAgentManager): An AgentManager instance that manages all agents. actor (SimpleActor or ActorProxy): An SimpleActor or ActorProxy instance responsible for performing roll-outs (environment sampling). - explorer (AbsExplorer): An explorer instance responsible for generating exploration rates. Defaults to None. + explorer (dict or AbsExplorer): An explorer instance responsible for generating exploration rates. + Defaults to None. logger (Logger): Used to log important messages. """ def __init__( self, trainable_agents: SimpleAgentManager, actor: Union[SimpleActor, ActorProxy], - explorer: AbsExplorer = None, + explorer: Union[dict, AbsExplorer] = None, logger: Logger = DummyLogger() ): super().__init__() @@ -39,13 +40,18 @@ def __init__( self._performance_history = [] def _get_epsilons(self, current_ep, max_ep): - if self._explorer is not None: + if self._explorer is None: + return None + elif isinstance(self._explorer, dict): return { - agent_id: self._explorer.generate_epsilon(current_ep, max_ep, self._performance_history) + agent_id: self._explorer[agent_id].generate_epsilon(current_ep, max_ep, self._performance_history) for agent_id in self._trainable_agents.agent_dict } else: - return None + return { + agent_id: self._explorer.generate_epsilon(current_ep, max_ep, self._performance_history) + for agent_id in self._trainable_agents.agent_dict + } def _sample(self, ep, max_ep): """Perform one episode of environment sampling through actor roll-out.""" @@ -55,7 +61,10 @@ def _sample(self, ep, max_ep): self._logger.info(f"ep {ep} - performance: {performance}, epsilons: {epsilon_dict}") return performance, exp_by_agent - def train(self, max_episode: int, early_stopping_checker: Callable = None, warmup_ep: int = None): + def train( + self, max_episode: int, early_stopping_checker: Callable = None, warmup_ep: int = None, + early_stopping_metric_func: Callable = None + ): """Main loop for collecting experiences from the actor and using them to update policies. Args: @@ -64,6 +73,8 @@ def train(self, max_episode: int, early_stopping_checker: Callable = None, warmu early_stopping_checker (Callable): A Callable object to determine whether the training loop should be terminated based on the latest performances. Defaults to None. warmup_ep (int): Episode from which early stopping check is initiated. Defaults to None. + early_stopping_metric_func (Callable): A function to extract the metric from a performance record + for early stopping checking. """ if max_episode < -1: raise InvalidEpisodeError("max_episode can only be a non-negative integer or -1.") @@ -73,11 +84,13 @@ def train(self, max_episode: int, early_stopping_checker: Callable = None, warmu "is provided. " ) episode = 0 + metric_series = [] while max_episode == -1 or episode < max_episode: performance, exp_by_agent = self._sample(episode, max_episode) self._performance_history.append(performance) + metric_series.append(early_stopping_metric_func(performance)) if early_stopping_checker is not None and (warmup_ep is None or episode >= warmup_ep) and \ - early_stopping_checker(self._performance_history): + early_stopping_checker(metric_series): self._logger.info("Early stopping condition hit. Training complete.") break self._trainable_agents.train(exp_by_agent) From b27a27248e79d6ac6e17675788e2c6d2ff00d435 Mon Sep 17 00:00:00 2001 From: ysqyang Date: Fri, 30 Oct 2020 09:27:56 +0800 Subject: [PATCH 087/337] revised docs --- docs/source/examples/multi_agent_dqn_cim.rst | 5 +++-- docs/source/key_components/rl_toolkit.rst | 20 +++---------------- .../rl_formulation.ipynb | 8 ++++---- 3 files changed, 10 insertions(+), 23 deletions(-) diff --git a/docs/source/examples/multi_agent_dqn_cim.rst b/docs/source/examples/multi_agent_dqn_cim.rst index c5641f719..d628605b1 100644 --- a/docs/source/examples/multi_agent_dqn_cim.rst +++ b/docs/source/examples/multi_agent_dqn_cim.rst @@ -116,8 +116,9 @@ Agent Manager The complexities of the environment can be isolated from the learning algorithm by using an `Agent manager `_ -to manage individual agents. We define a function to create the agents and an AgentManager class -that implements the abstract ``train`` method in accordance with the DQN algorithm. . +to manage individual agents. We define a function to create the agents and an agent manager class +that implements the ``train`` method where the newly obtained experiences are stored in the agents' +experience pools before training, in accordance with the DQN algorithm. .. code-block:: python def create_dqn_agents(agent_id_list, config): diff --git a/docs/source/key_components/rl_toolkit.rst b/docs/source/key_components/rl_toolkit.rst index 56ae2f61c..1c526b446 100644 --- a/docs/source/key_components/rl_toolkit.rst +++ b/docs/source/key_components/rl_toolkit.rst @@ -70,23 +70,9 @@ Agent Manager The agent manager provides a unified interactive interface with the environment for RL agent(s). From the actor's perspective, it isolates the complex dependencies of the various homogeneous/heterogeneous agents, so that the whole agent manager -will behave just like a single agent. - -.. code-block:: python - - def assemble_agents(self, config): - # Initialize experience pool instance. - ... - # Construct underlying learning model and related RL algorithm. - ... - for agent_id in self._agent_id_list: - # Assemble your agent here, load experience pool, RL algorithms, etc. - # You can control the experience pool and learning model sharing pattern, based on different assembling strategy. - self._agent_dict[agent_id] = Agent(...) - -Furthermore, to well serve the distributed algorithm (scalable), the agent -manager provides two kinds of working modes, which can be applied in different -distributed components, such as inference mode in actor, training mode in learner. +will behave just like a single agent. Furthermore, to well serve the distributed algorithm +(scalable), the agent manager provides two kinds of working modes, which can be applied in +different distributed components, such as inference mode in actor, training mode in learner. .. image:: ../images/rl/agent_manager.svg :target: ../images/rl/agent_manager.svg diff --git a/notebooks/container_inventory_management/rl_formulation.ipynb b/notebooks/container_inventory_management/rl_formulation.ipynb index 8d22f35b4..251195e60 100644 --- a/notebooks/container_inventory_management/rl_formulation.ipynb +++ b/notebooks/container_inventory_management/rl_formulation.ipynb @@ -17,9 +17,9 @@ "\n", "State shaper converts the environment observation to the model input state which includes temporal and spatial information. For this scenario, the model input state includes: \n", "\n", - "- Temporal information, it includes the past week's information of ports and vessels, such as shortage on port and remaining space on vessel. \n", + "- Temporal information, including the past week's information of ports and vessels, such as shortage on port and remaining space on vessel. \n", "\n", - "- Spatial information, it includes the related downstream port features. " + "- Spatial information, it including the related downstream port features. " ] }, { @@ -213,7 +213,7 @@ "source": [ "## [Agent Manager](https://maro.readthedocs.io/en/latest/key_components/rl_toolkit.html#agent-manager)\n", "\n", - "The agent manager inherits from MARO's `AbsAgentManager` which is an agent assembler and isolates the complexities of the environment and algorithm. It will load the DQN algorithm and an experience pool for each agent." + "The complexities of the environment can be isolated from the learning algorithm by using an AgentManager to manage individual agents. We define a function to create the agents and an agent manager class that implements the ``train`` method where the newly obtained experiences are stored in the agents' experience pools before training, in accordance with the DQN algorithm." ] }, { @@ -304,7 +304,7 @@ "\n", "- Define scenario-specific components, e.g. shapers. \n", "\n", - "- Create an agent manager, which assembles underlying agents. \n", + "- Create agents and an agent manager. \n", "\n", "- Create an actor and a learner to start the training process in which the agent manager interacts with the environment for collecting experiences and updating policies. " ] From 4c95ac87740cb99db6f71e4dbd99cbeea4958a0a Mon Sep 17 00:00:00 2001 From: ysqyang Date: Fri, 30 Oct 2020 09:29:50 +0800 Subject: [PATCH 088/337] removed nb output --- .../rl_formulation.ipynb | 27 +++++-------------- 1 file changed, 7 insertions(+), 20 deletions(-) diff --git a/notebooks/container_inventory_management/rl_formulation.ipynb b/notebooks/container_inventory_management/rl_formulation.ipynb index 251195e60..f906612f8 100644 --- a/notebooks/container_inventory_management/rl_formulation.ipynb +++ b/notebooks/container_inventory_management/rl_formulation.ipynb @@ -24,7 +24,7 @@ }, { "cell_type": "code", - "execution_count": 1, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -66,7 +66,7 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -116,7 +116,7 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -178,7 +178,7 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -218,7 +218,7 @@ }, { "cell_type": "code", - "execution_count": 11, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -311,22 +311,9 @@ }, { "cell_type": "code", - "execution_count": 12, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "ename": "NameError", - "evalue": "name 'nn' is not defined", - "output_type": "error", - "traceback": [ - "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", - "\u001b[0;31mNameError\u001b[0m Traceback (most recent call last)", - "\u001b[0;32m\u001b[0m in \u001b[0;36m\u001b[0;34m\u001b[0m\n\u001b[1;32m 23\u001b[0m agent_manager = DQNAgentManager(name=\"cim_learner\",\n\u001b[1;32m 24\u001b[0m \u001b[0mmode\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mAgentManagerMode\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mTRAIN_INFERENCE\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 25\u001b[0;31m \u001b[0magent_dict\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mcreate_dqn_agents\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0magent_id_list\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 26\u001b[0m \u001b[0mstate_shaper\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mstate_shaper\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 27\u001b[0m \u001b[0maction_shaper\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0maction_shaper\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m\u001b[0m in \u001b[0;36mcreate_dqn_agents\u001b[0;34m(agent_id_list)\u001b[0m\n\u001b[1;32m 21\u001b[0m \u001b[0minput_dim\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0minput_dim\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 22\u001b[0m \u001b[0moutput_dim\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mnum_actions\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 23\u001b[0;31m \u001b[0mactivation\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mnn\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mLeakyReLU\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 24\u001b[0m \u001b[0mhidden_dims\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m256\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m128\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m64\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 25\u001b[0m \u001b[0msoftmax_enabled\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mFalse\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;31mNameError\u001b[0m: name 'nn' is not defined" - ] - } - ], + "outputs": [], "source": [ "from maro.simulator import Env\n", "from maro.rl import SimpleLearner, SimpleActor, AgentManagerMode, TwoPhaseLinearExplorer\n", From 3bf5e8d194c5a094e209d1d7a4e2cb4fbeb88808 Mon Sep 17 00:00:00 2001 From: ysqyang Date: Fri, 30 Oct 2020 09:41:27 +0800 Subject: [PATCH 089/337] fixed a bug in simple_learner --- maro/rl/learner/simple_learner.py | 16 ++++++++++------ .../rl_formulation.ipynb | 2 +- 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/maro/rl/learner/simple_learner.py b/maro/rl/learner/simple_learner.py index 9497e6f14..506874510 100644 --- a/maro/rl/learner/simple_learner.py +++ b/maro/rl/learner/simple_learner.py @@ -74,7 +74,7 @@ def train( terminated based on the latest performances. Defaults to None. warmup_ep (int): Episode from which early stopping check is initiated. Defaults to None. early_stopping_metric_func (Callable): A function to extract the metric from a performance record - for early stopping checking. + for early stopping checking. Defaults to None. """ if max_episode < -1: raise InvalidEpisodeError("max_episode can only be a non-negative integer or -1.") @@ -83,16 +83,20 @@ def train( "The training loop will run forever since neither maximum episode nor early stopping checker " "is provided. " ) + if early_stopping_checker is not None: + assert early_stopping_metric_func is not None, \ + "early_stopping_metric_func cannot be None if early_stopping_checker is provided." + episode = 0 metric_series = [] while max_episode == -1 or episode < max_episode: performance, exp_by_agent = self._sample(episode, max_episode) self._performance_history.append(performance) - metric_series.append(early_stopping_metric_func(performance)) - if early_stopping_checker is not None and (warmup_ep is None or episode >= warmup_ep) and \ - early_stopping_checker(metric_series): - self._logger.info("Early stopping condition hit. Training complete.") - break + if early_stopping_checker is not None: + metric_series.append(early_stopping_metric_func(performance)) + if warmup_ep is None or episode >= warmup_ep and early_stopping_checker(metric_series): + self._logger.info("Early stopping condition hit. Training complete.") + break self._trainable_agents.train(exp_by_agent) episode += 1 diff --git a/notebooks/container_inventory_management/rl_formulation.ipynb b/notebooks/container_inventory_management/rl_formulation.ipynb index f906612f8..60bf25cc0 100644 --- a/notebooks/container_inventory_management/rl_formulation.ipynb +++ b/notebooks/container_inventory_management/rl_formulation.ipynb @@ -347,7 +347,7 @@ "# Step 4: Create an actor and a learner to start the training process. \n", "actor = SimpleActor(env, agent_manager)\n", "learner = SimpleLearner(trainable_agents=agent_manager, actor=actor, \n", - " explorer=TwoPhaseLinearExplorer(eps_split=0.32, progress_split=0.5, max_eps=0.4, min_eps=.0),\n", + " explorer=TwoPhaseLinearExplorer(start_eps=0.4, mid_eps=0.32, end_eps=0.0, split_point=0.5),\n", " logger=Logger(\"single_host_cim_learner\", format_=LogFormat.simple, auto_timestamp=False))\n", "\n", "learner.train(max_episode=100)" From 0d517831a2c847d157ee75e6a3e89ac6fb17c438 Mon Sep 17 00:00:00 2001 From: ysqyang Date: Fri, 30 Oct 2020 09:43:32 +0800 Subject: [PATCH 090/337] fixed a typo in nb --- notebooks/container_inventory_management/rl_formulation.ipynb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/notebooks/container_inventory_management/rl_formulation.ipynb b/notebooks/container_inventory_management/rl_formulation.ipynb index 60bf25cc0..03f742e87 100644 --- a/notebooks/container_inventory_management/rl_formulation.ipynb +++ b/notebooks/container_inventory_management/rl_formulation.ipynb @@ -225,7 +225,7 @@ "import io\n", "import yaml\n", "\n", - "import torch.nn as\n", + "import torch.nn as nn\n", "from torch.nn.functional import smooth_l1_loss\n", "from torch.optim import RMSprop\n", "\n", From bd732495716b7d9932d69014a9591125ebf90241 Mon Sep 17 00:00:00 2001 From: ysqyang Date: Fri, 30 Oct 2020 09:57:57 +0800 Subject: [PATCH 091/337] fixed a bug --- examples/cim/dqn/config.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/examples/cim/dqn/config.yml b/examples/cim/dqn/config.yml index d9a632629..2ffd2a25f 100644 --- a/examples/cim/dqn/config.yml +++ b/examples/cim/dqn/config.yml @@ -34,10 +34,10 @@ experience_shaping: shortage_factor: 1.0 time_decay_factor: 0.97 exploration: - min_eps: .0 - max_eps: .4 - eps_split: 0.32 - progress_split: 0.5 + start_eps: 0.4 + mid_eps: 0.32 + end_eps: 0.0 + split_point: 0.5 agents: algorithm: num_actions: 21 From c3367f1e11f29b52cd8d405241677d56a953e861 Mon Sep 17 00:00:00 2001 From: ysqyang Date: Fri, 30 Oct 2020 10:01:03 +0800 Subject: [PATCH 092/337] fixed a bug --- docs/source/examples/multi_agent_dqn_cim.rst | 2 +- examples/cim/dqn/dist_learner.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/source/examples/multi_agent_dqn_cim.rst b/docs/source/examples/multi_agent_dqn_cim.rst index d628605b1..1d0c29192 100644 --- a/docs/source/examples/multi_agent_dqn_cim.rst +++ b/docs/source/examples/multi_agent_dqn_cim.rst @@ -272,7 +272,7 @@ inside. "redis_address": ("localhost", 6379) } - early_stopping_checker = SimpleEarlyStoppingChecker( + early_stopping_checker = MaxDeltaEarlyStoppingChecker( last_k=config.general.early_stopping.last_k, threshold=config.general.early_stopping.threshold ) diff --git a/examples/cim/dqn/dist_learner.py b/examples/cim/dqn/dist_learner.py index 68903c88c..2efcecfb1 100644 --- a/examples/cim/dqn/dist_learner.py +++ b/examples/cim/dqn/dist_learner.py @@ -29,7 +29,7 @@ def launch(config): "redis_address": ("localhost", 6379) } - early_stopping_checker = SimpleEarlyStoppingChecker( + early_stopping_checker = MaxDeltaEarlyStoppingChecker( last_k=config.general.early_stopping.last_k, threshold=config.general.early_stopping.threshold ) From ac7309d0d5e3bf7b3e5ae5a671c4966d2acb1bb7 Mon Sep 17 00:00:00 2001 From: ysqyang Date: Fri, 30 Oct 2020 10:12:53 +0800 Subject: [PATCH 093/337] fixed a bug --- .../dist_topologies/single_learner_multi_actor_sync_mode.py | 2 +- maro/rl/learner/simple_learner.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/maro/rl/dist_topologies/single_learner_multi_actor_sync_mode.py b/maro/rl/dist_topologies/single_learner_multi_actor_sync_mode.py index d15c8ab28..5c8c0547c 100644 --- a/maro/rl/dist_topologies/single_learner_multi_actor_sync_mode.py +++ b/maro/rl/dist_topologies/single_learner_multi_actor_sync_mode.py @@ -67,7 +67,7 @@ def roll_out( destination_payload_list=payloads ) - performance = {msg.source: msg.payload[PayloadKey.PERFORMANCE] for msg in replies} + performance = [(msg.source, msg.payload[PayloadKey.PERFORMANCE]) for msg in replies] experiences_by_source = {msg.source: msg.payload[PayloadKey.EXPERIENCE] for msg in replies} return performance, self._experience_collecting_func(experiences_by_source) diff --git a/maro/rl/learner/simple_learner.py b/maro/rl/learner/simple_learner.py index 506874510..93a98cf71 100644 --- a/maro/rl/learner/simple_learner.py +++ b/maro/rl/learner/simple_learner.py @@ -91,9 +91,9 @@ def train( metric_series = [] while max_episode == -1 or episode < max_episode: performance, exp_by_agent = self._sample(episode, max_episode) - self._performance_history.append(performance) + latest = [perf for _, perf in performance] if isinstance(performance, list) else [performance] if early_stopping_checker is not None: - metric_series.append(early_stopping_metric_func(performance)) + metric_series.extend(map(early_stopping_metric_func, latest)) if warmup_ep is None or episode >= warmup_ep and early_stopping_checker(metric_series): self._logger.info("Early stopping condition hit. Training complete.") break From 89376276a3ff48466671a73a25f689678f4f583b Mon Sep 17 00:00:00 2001 From: ysqyang Date: Fri, 30 Oct 2020 10:20:11 +0800 Subject: [PATCH 094/337] removed unused import --- maro/rl/learner/simple_learner.py | 1 - 1 file changed, 1 deletion(-) diff --git a/maro/rl/learner/simple_learner.py b/maro/rl/learner/simple_learner.py index 93a98cf71..0758b1032 100644 --- a/maro/rl/learner/simple_learner.py +++ b/maro/rl/learner/simple_learner.py @@ -3,7 +3,6 @@ import sys from typing import Callable, Union -import warnings from .abs_learner import AbsLearner from maro.rl.actor.simple_actor import SimpleActor From bb961531a55d7284d1abc1a9bfb3a18e57ddbe1f Mon Sep 17 00:00:00 2001 From: ysqyang Date: Fri, 30 Oct 2020 10:42:10 +0800 Subject: [PATCH 095/337] fixed a bug --- maro/rl/early_stopping/simple_early_stopping_checker.py | 1 + 1 file changed, 1 insertion(+) diff --git a/maro/rl/early_stopping/simple_early_stopping_checker.py b/maro/rl/early_stopping/simple_early_stopping_checker.py index 17461ba82..17c98b539 100644 --- a/maro/rl/early_stopping/simple_early_stopping_checker.py +++ b/maro/rl/early_stopping/simple_early_stopping_checker.py @@ -23,6 +23,7 @@ def __call__(self, metric_series): if self._last_k > len(metric_series): return False else: + metric_series = metric_series[-self._last_k:] return stdev(metric_series) / mean(metric_series) < self._threshold From 1d5d69547003d86943df2c5b4ea16d21e2be22ea Mon Sep 17 00:00:00 2001 From: ysqyang Date: Fri, 30 Oct 2020 10:58:22 +0800 Subject: [PATCH 096/337] 1. changed early stopping default config; 2. renamed param in early stopping checker and added typing --- examples/cim/dqn/config.yml | 4 ++-- maro/rl/early_stopping/abs_early_stopping_checker.py | 4 ++-- maro/rl/early_stopping/simple_early_stopping_checker.py | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/examples/cim/dqn/config.yml b/examples/cim/dqn/config.yml index 2ffd2a25f..c30c98ae7 100644 --- a/examples/cim/dqn/config.yml +++ b/examples/cim/dqn/config.yml @@ -6,8 +6,8 @@ general: max_episode: 500 early_stopping: warmup_ep: 50 - last_k: 5 - threshold: 0.2 + last_k: 10 + threshold: 0.05 state_shaping: look_back: 7 max_ports_downstream: 2 diff --git a/maro/rl/early_stopping/abs_early_stopping_checker.py b/maro/rl/early_stopping/abs_early_stopping_checker.py index 733a00aa1..5832dd3da 100644 --- a/maro/rl/early_stopping/abs_early_stopping_checker.py +++ b/maro/rl/early_stopping/abs_early_stopping_checker.py @@ -13,11 +13,11 @@ def __init__(self): pass @abstractmethod - def __call__(self, performance_history) -> bool: + def __call__(self, metric_series) -> bool: """Check whether the early stopping condition (defined in the class) is met. Args: - performance_history: History of performances (from actors) used to check whether the early stopping + metric_series: History of performances (from actors) used to check whether the early stopping condition is satisfied. Returns: diff --git a/maro/rl/early_stopping/simple_early_stopping_checker.py b/maro/rl/early_stopping/simple_early_stopping_checker.py index 17c98b539..815dd1caa 100644 --- a/maro/rl/early_stopping/simple_early_stopping_checker.py +++ b/maro/rl/early_stopping/simple_early_stopping_checker.py @@ -19,7 +19,7 @@ def __init__(self, last_k: int, threshold: float): self._last_k = last_k self._threshold = threshold - def __call__(self, metric_series): + def __call__(self, metric_series: list): if self._last_k > len(metric_series): return False else: @@ -40,7 +40,7 @@ def __init__(self, last_k: int, threshold: float): self._last_k = last_k self._threshold = threshold - def __call__(self, metric_series): + def __call__(self, metric_series: list): if self._last_k > len(metric_series): return False else: From 57735570b0ee45b9e8e712a2279ba556896221ec Mon Sep 17 00:00:00 2001 From: ysqyang Date: Fri, 30 Oct 2020 16:13:06 +0800 Subject: [PATCH 097/337] fixed some doc issues --- docs/source/examples/multi_agent_dqn_cim.rst | 6 ++-- examples/cim/dqn/README.md | 4 ++- examples/cim/dqn/config.yml | 2 +- maro/rl/algorithms/dqn.py | 35 ++++++++++++------- .../simple_early_stopping_checker.py | 11 +++--- .../rl_formulation.ipynb | 2 +- 6 files changed, 37 insertions(+), 23 deletions(-) diff --git a/docs/source/examples/multi_agent_dqn_cim.rst b/docs/source/examples/multi_agent_dqn_cim.rst index 1d0c29192..0f803090f 100644 --- a/docs/source/examples/multi_agent_dqn_cim.rst +++ b/docs/source/examples/multi_agent_dqn_cim.rst @@ -226,7 +226,7 @@ occurs on the actor side, we need to create appropriate agent managers on both s On the actor side, the agent manager must be equipped with all shapers as well as an explorer. Thus, The code for creating an environment and an agent manager on the actor side is similar to that for the single-host version, -except that it is necessary to set the AgentMode to AgentMode.INFERENCE. As in the single-process version, the environment +except that it is necessary to set the AgentManagerMode to AgentManagerMode.INFERENCE. As in the single-process version, the environment and the agent manager are wrapped in a SimpleActor instance. To make the actor a distributed worker, we need to further wrap it in an ActorWorker instance. Finally, we launch the worker and it starts to listen to roll-out requests from the learner. The following code snippet shows the creation of an actor worker with a simple (local) actor wrapped inside. @@ -251,8 +251,8 @@ learner. The following code snippet shows the creation of an actor worker with a ) actor_worker.launch() -On the learner side, an agent manager in AgentMode.TRAIN mode is required. However, it is not necessary to create shapers for an -agent manager in AgentMode.TRAIN mode (although a state shaper is created in this example so that the model input dimension can +On the learner side, an agent manager in AgentManagerMode.TRAIN mode is required. However, it is not necessary to create shapers for an +agent manager in AgentManagerMode.TRAIN mode (although a state shaper is created in this example so that the model input dimension can be readily accessed). Instead of creating an actor, we create an actor proxy and wrap it inside the learner. This proxy serves as the communication interface for the learner and is responsible for sending roll-out requests to remote actor processes and receiving results. Calling the train method executes the usual training loop except that the actual diff --git a/examples/cim/dqn/README.md b/examples/cim/dqn/README.md index 816ed5052..3f02df519 100644 --- a/examples/cim/dqn/README.md +++ b/examples/cim/dqn/README.md @@ -19,4 +19,6 @@ the configuration if you want to try out different settings. The examples/cim/dqn/components folder contains dist_learner.py and dist_actor.py for distributed training. For debugging purposes, we provide a script that simulates distributed mode using multi-processing. Simply go to examples/cim/dqn -and run multi_process_launcher.py to start the learner and actor processes. +and execute python3 multi_process_launcher.py \[GROUP_NAME\] \[NUM_ACTORS\], where +GROUP_NAME is the identifier for the current run and NUM_ACTORS is the number of actor +processes to launch. diff --git a/examples/cim/dqn/config.yml b/examples/cim/dqn/config.yml index c30c98ae7..8769eef79 100644 --- a/examples/cim/dqn/config.yml +++ b/examples/cim/dqn/config.yml @@ -53,7 +53,7 @@ agents: lr: 0.05 hyper_parameters: reward_decay: .0 - target_replacement_frequency: 5 + target_update_frequency: 5 tau: 0.1 experience_pool: capacity: -1 diff --git a/maro/rl/algorithms/dqn.py b/maro/rl/algorithms/dqn.py index 200c59368..ac6945f17 100644 --- a/maro/rl/algorithms/dqn.py +++ b/maro/rl/algorithms/dqn.py @@ -13,19 +13,23 @@ class DQNHyperParams: """Hyper-parameter set for the DQN algorithm. Args: - num_actions (int): number of possible actions - reward_decay (float): reward decay as defined in standard RL terminology - target_replacement_frequency (int): number of training frequency of target model replacement - tau (float): soft update coefficient, e.g., target_model = tau * eval_model + (1-tau) * target_model + num_actions (int): Number of possible actions. + reward_decay (float): Reward decay as defined in standard RL terminology. + target_update_frequency (int): Number of training rounds between target model updates. + tau (float): Soft update coefficient, i.e., target_model = tau * eval_model + (1-tau) * target_model. """ - __slots__ = ["num_actions", "reward_decay", "target_replacement_frequency", "tau"] + __slots__ = ["num_actions", "reward_decay", "target_update_frequency", "tau"] def __init__( - self, num_actions: int, reward_decay: float, target_replacement_frequency: int, tau: float = 1.0 + self, + num_actions: int, + reward_decay: float, + target_update_frequency: int, + tau: float = 1.0 ): self.num_actions = num_actions self.reward_decay = reward_decay - self.target_replacement_frequency = target_replacement_frequency + self.target_update_frequency = target_update_frequency self.tau = tau @@ -36,15 +40,20 @@ class DQN(AbsAlgorithm): Args: eval_model (nn.Module): Q-value model for given states and actions. - optimizer_cls: torch optimizer class for the eval model. If this is None, the eval model is not trainable. - optimizer_params: parameters required for the eval optimizer class. - loss_func (Callable): loss function for the value model. - hyper_params: hyper-parameter set for the DQN algorithm. + optimizer_cls: Torch optimizer class for the eval model. If this is None, the eval model is not trainable. + optimizer_params: Parameters required for the eval optimizer class. + loss_func (Callable): Loss function for the value model. + hyper_params: Hyper-parameter set for the DQN algorithm. target_model (nn.Module): Q-value model to train the ``eval_model`` against and to be updated periodically. If it is None, the target model will be initialized as a deep copy of the eval model. """ def __init__( - self, eval_model: nn.Module, optimizer_cls, optimizer_params, loss_func, hyper_params: DQNHyperParams, + self, + eval_model: nn.Module, + optimizer_cls, + optimizer_params, + loss_func, + hyper_params: DQNHyperParams, target_model=None ): super().__init__() @@ -95,7 +104,7 @@ def train(self, states: np.ndarray, actions: np.ndarray, rewards: np.ndarray, ne loss.backward() self._optimizer.step() self._train_cnt += 1 - if self._train_cnt % self._hyper_params.target_replacement_frequency == 0: + if self._train_cnt % self._hyper_params.target_update_frequency == 0: self._update_target_model() return np.abs((current_q_values - target_q_values).detach().numpy()) diff --git a/maro/rl/early_stopping/simple_early_stopping_checker.py b/maro/rl/early_stopping/simple_early_stopping_checker.py index 815dd1caa..c1a80b229 100644 --- a/maro/rl/early_stopping/simple_early_stopping_checker.py +++ b/maro/rl/early_stopping/simple_early_stopping_checker.py @@ -7,12 +7,12 @@ class RSDEarlyStoppingChecker(AbsEarlyStoppingChecker): - """Simple early stopping checker based on the mean and standard deviation of the last k performance records. + """Early stopping checker based on the mean and standard deviation of the last k metric values. Args: last_k (int): Number of the latest performance records to check for early stopping. threshold (float): The threshold value against which the early stopping metric is compared. The early stopping - condition is satisfied if the metric obtained using the ``performance_metric_func`` is below this threshold. + condition is satisfied if the metric is below this threshold. """ def __init__(self, last_k: int, threshold: float): super().__init__() @@ -28,12 +28,15 @@ def __call__(self, metric_series: list): class MaxDeltaEarlyStoppingChecker(AbsEarlyStoppingChecker): - """Simple early stopping checker based on the mean and standard deviation of the last k performance records. + """Early stopping checker based on the maximum relative change over the last k metric values. + + The relative change is defined as |m(i+1) - m(i)| / m[i]. The maximum of the last k-1 changes in the metric series + is compared with the threshold to determine if early stopping should be triggered. Args: last_k (int): Number of the latest performance records to check for early stopping. threshold (float): The threshold value against which the early stopping metric is compared. The early stopping - condition is satisfied if the metric obtained using the ``performance_metric_func`` is below this threshold. + condition is satisfied if the metric is below this threshold. """ def __init__(self, last_k: int, threshold: float): super().__init__() diff --git a/notebooks/container_inventory_management/rl_formulation.ipynb b/notebooks/container_inventory_management/rl_formulation.ipynb index 03f742e87..af6d4f4dc 100644 --- a/notebooks/container_inventory_management/rl_formulation.ipynb +++ b/notebooks/container_inventory_management/rl_formulation.ipynb @@ -261,7 +261,7 @@ " hyper_params=DQNHyperParams(\n", " num_actions=num_actions,\n", " reward_decay=.0,\n", - " target_replacement_frequency=5,\n", + " target_update_frequency=5,\n", " tau=0.1\n", " )\n", " )\n", From fd4aa746bc98a75ffebeb907863f15c9d9559f71 Mon Sep 17 00:00:00 2001 From: ysqyang Date: Fri, 30 Oct 2020 17:17:31 +0800 Subject: [PATCH 098/337] revised ac and pg agent managers' train() to accommodate dist mode --- examples/cim/ac/components/agent_manager.py | 6 +++++- examples/cim/pg/components/agent_manager.py | 6 +++++- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/examples/cim/ac/components/agent_manager.py b/examples/cim/ac/components/agent_manager.py index 38594f740..240e35675 100644 --- a/examples/cim/ac/components/agent_manager.py +++ b/examples/cim/ac/components/agent_manager.py @@ -51,4 +51,8 @@ def create_ac_agents(agent_id_list, config): class ACAgentManager(SimpleAgentManager): def train(self, experiences_by_agent: dict): for agent_id, experiences in experiences_by_agent.items(): - self.agent_dict[agent_id].train(experiences["states"], experiences["actions"], experiences["rewards"]) + if isinstance(experiences, list): + for trajectory in experiences_by_agent: + self.agent_dict[agent_id].train(trajectory["states"], trajectory["actions"], trajectory["rewards"]) + else: + self.agent_dict[agent_id].train(experiences["states"], experiences["actions"], experiences["rewards"]) diff --git a/examples/cim/pg/components/agent_manager.py b/examples/cim/pg/components/agent_manager.py index 9a594d1dc..403cda021 100644 --- a/examples/cim/pg/components/agent_manager.py +++ b/examples/cim/pg/components/agent_manager.py @@ -40,4 +40,8 @@ def create_pg_agents(agent_id_list, config): class PGAgentManager(SimpleAgentManager): def train(self, experiences_by_agent: dict): for agent_id, experiences in experiences_by_agent.items(): - self.agent_dict[agent_id].train(experiences["states"], experiences["actions"], experiences["rewards"]) + if isinstance(experiences, list): + for trajectory in experiences_by_agent: + self.agent_dict[agent_id].train(trajectory["states"], trajectory["actions"], trajectory["rewards"]) + else: + self.agent_dict[agent_id].train(experiences["states"], experiences["actions"], experiences["rewards"]) From 65b2d0bf5a829ca9d4a02e0f9db81dc54fe8dbaa Mon Sep 17 00:00:00 2001 From: ysqyang Date: Fri, 30 Oct 2020 17:18:45 +0800 Subject: [PATCH 099/337] fixed a bug --- examples/cim/ac/multi_process_launcher.py | 23 +++++++++++++---------- examples/cim/pg/multi_process_launcher.py | 23 +++++++++++++---------- 2 files changed, 26 insertions(+), 20 deletions(-) diff --git a/examples/cim/ac/multi_process_launcher.py b/examples/cim/ac/multi_process_launcher.py index 9ec989550..0de79a77c 100644 --- a/examples/cim/ac/multi_process_launcher.py +++ b/examples/cim/ac/multi_process_launcher.py @@ -5,19 +5,22 @@ This script is used to debug distributed algorithm in single host multi-process mode. """ +import argparse import os -from components.config import config +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("group_name", help="group name") + parser.add_argument("num_actors", type=int, help="number of actors") + args = parser.parse_args() -ACTOR_NUM = config.distributed.learner.peer["actor_worker"] # must be same as in config -LEARNER_NUM = config.distributed.actor.peer["actor"] + learner_path = f"{os.path.split(os.path.realpath(__file__))[0]}/dist_learner.py &" + actor_path = f"{os.path.split(os.path.realpath(__file__))[0]}/dist_actor.py &" -learner_path = f"{os.path.split(os.path.realpath(__file__))[0]}/dist_learner.py &" -actor_path = f"{os.path.split(os.path.realpath(__file__))[0]}/dist_actor.py &" + # Launch the learner process + os.system(f"GROUP={args.group_name} NUM_ACTORS={args.num_actors} python " + learner_path) -for l_num in range(LEARNER_NUM): - os.system(f"python " + learner_path) - -for a_num in range(ACTOR_NUM): - os.system(f"python " + actor_path) + # Launch the actor processes + for _ in range(args.num_actors): + os.system(f"GROUP={args.group_name} python " + actor_path) diff --git a/examples/cim/pg/multi_process_launcher.py b/examples/cim/pg/multi_process_launcher.py index 9ec989550..0de79a77c 100644 --- a/examples/cim/pg/multi_process_launcher.py +++ b/examples/cim/pg/multi_process_launcher.py @@ -5,19 +5,22 @@ This script is used to debug distributed algorithm in single host multi-process mode. """ +import argparse import os -from components.config import config +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("group_name", help="group name") + parser.add_argument("num_actors", type=int, help="number of actors") + args = parser.parse_args() -ACTOR_NUM = config.distributed.learner.peer["actor_worker"] # must be same as in config -LEARNER_NUM = config.distributed.actor.peer["actor"] + learner_path = f"{os.path.split(os.path.realpath(__file__))[0]}/dist_learner.py &" + actor_path = f"{os.path.split(os.path.realpath(__file__))[0]}/dist_actor.py &" -learner_path = f"{os.path.split(os.path.realpath(__file__))[0]}/dist_learner.py &" -actor_path = f"{os.path.split(os.path.realpath(__file__))[0]}/dist_actor.py &" + # Launch the learner process + os.system(f"GROUP={args.group_name} NUM_ACTORS={args.num_actors} python " + learner_path) -for l_num in range(LEARNER_NUM): - os.system(f"python " + learner_path) - -for a_num in range(ACTOR_NUM): - os.system(f"python " + actor_path) + # Launch the actor processes + for _ in range(args.num_actors): + os.system(f"GROUP={args.group_name} python " + actor_path) From cf8f612ac2df7cf83c5e1ff9f4846ce7c95cbdd4 Mon Sep 17 00:00:00 2001 From: ysqyang Date: Fri, 30 Oct 2020 17:25:20 +0800 Subject: [PATCH 100/337] fixed a bug --- examples/cim/ac/config.yml | 6 +++++- examples/cim/ac/single_process_launcher.py | 2 +- examples/cim/pg/config.yml | 6 +++++- examples/cim/pg/dist_learner.py | 2 +- examples/cim/pg/single_process_launcher.py | 2 +- maro/rl/__init__.py | 8 +------- maro/rl/agent/abs_agent.py | 6 ------ 7 files changed, 14 insertions(+), 18 deletions(-) diff --git a/examples/cim/ac/config.yml b/examples/cim/ac/config.yml index 09f4184f0..e61809d93 100644 --- a/examples/cim/ac/config.yml +++ b/examples/cim/ac/config.yml @@ -3,7 +3,11 @@ env: topology: "toy.4p_ssdd_l0.0" durations: 1120 general: - total_training_episodes: 500 # max episode + max_episode: 500 + early_stopping: + warmup_ep: 50 + last_k: 10 + threshold: 0.05 state_shaping: look_back: 7 max_ports_downstream: 2 diff --git a/examples/cim/ac/single_process_launcher.py b/examples/cim/ac/single_process_launcher.py index 5c74ee8cd..9e2abc62d 100644 --- a/examples/cim/ac/single_process_launcher.py +++ b/examples/cim/ac/single_process_launcher.py @@ -47,7 +47,7 @@ def launch(config): trainable_agents=agent_manager, actor=actor, logger=Logger("single_host_cim_learner", auto_timestamp=False) ) - learner.train(max_episode=config.general.total_training_episodes) + learner.train(max_episode=config.general.max_episode) learner.test() learner.dump_models(os.path.join(os.getcwd(), "models")) diff --git a/examples/cim/pg/config.yml b/examples/cim/pg/config.yml index 2cdb72567..62baf506a 100644 --- a/examples/cim/pg/config.yml +++ b/examples/cim/pg/config.yml @@ -3,7 +3,11 @@ env: topology: "toy.4p_ssdd_l0.0" durations: 1120 general: - total_training_episodes: 500 # max episode + max_episode: 500 + early_stopping: + warmup_ep: 50 + last_k: 10 + threshold: 0.05 state_shaping: look_back: 7 max_ports_downstream: 2 diff --git a/examples/cim/pg/dist_learner.py b/examples/cim/pg/dist_learner.py index 7bd87eb96..ed6b6149d 100644 --- a/examples/cim/pg/dist_learner.py +++ b/examples/cim/pg/dist_learner.py @@ -34,6 +34,6 @@ ), logger=Logger("distributed_cim_learner", auto_timestamp=False) ) - learner.train(max_episode=config.general.total_training_episodes) + learner.train(max_episode=config.general.max_episode) learner.test() learner.dump_models(os.path.join(os.getcwd(), "models")) diff --git a/examples/cim/pg/single_process_launcher.py b/examples/cim/pg/single_process_launcher.py index d5e5adeeb..a7d604b2a 100644 --- a/examples/cim/pg/single_process_launcher.py +++ b/examples/cim/pg/single_process_launcher.py @@ -47,7 +47,7 @@ def launch(config): trainable_agents=agent_manager, actor=actor, logger=Logger("single_host_cim_learner", auto_timestamp=False) ) - learner.train(max_episode=config.general.total_training_episodes) + learner.train(max_episode=config.general.max_episodes) learner.test() learner.dump_models(os.path.join(os.getcwd(), "models")) diff --git a/maro/rl/__init__.py b/maro/rl/__init__.py index 60a022a1d..defc358a0 100644 --- a/maro/rl/__init__.py +++ b/maro/rl/__init__.py @@ -5,7 +5,7 @@ from maro.rl.actor.simple_actor import SimpleActor from maro.rl.learner.abs_learner import AbsLearner from maro.rl.learner.simple_learner import SimpleLearner -from maro.rl.agent.abs_agent import AbsAgent, AgentMode +from maro.rl.agent.abs_agent import AbsAgent from maro.rl.agent.abs_agent_manager import AbsAgentManager, AgentManagerMode from maro.rl.agent.simple_agent_manager import SimpleAgentManager from maro.rl.algorithms.abs_algorithm import AbsAlgorithm @@ -16,12 +16,6 @@ PPOHyperParametersWithCombinedModel from maro.rl.algorithms.dqn import DQN, DQNHyperParams from maro.rl.models.learning_model import LearningModel -from maro.rl.agent.abs_agent import AbsAgent -from maro.rl.agent.abs_agent_manager import AbsAgentManager, AgentManagerMode -from maro.rl.agent.simple_agent_manager import SimpleAgentManager -from maro.rl.algorithms.abs_algorithm import AbsAlgorithm -from maro.rl.algorithms.dqn import DQN, DQNHyperParams -from maro.rl.models.learning_model import LearningModel from maro.rl.models.fc_net import FullyConnectedNet from maro.rl.storage.abs_store import AbsStore from maro.rl.storage.column_based_store import ColumnBasedStore diff --git a/maro/rl/agent/abs_agent.py b/maro/rl/agent/abs_agent.py index 423c519e5..beded268f 100644 --- a/maro/rl/agent/abs_agent.py +++ b/maro/rl/agent/abs_agent.py @@ -10,12 +10,6 @@ from maro.rl.storage.abs_store import AbsStore -class AgentMode(Enum): - TRAIN = "train" - INFERENCE = "inference" - TRAIN_INFERENCE = "train_inference" - - class AbsAgent(ABC): """Abstract RL agent class. From 0cbe739abf424d8af159124c98f8c697d17260c3 Mon Sep 17 00:00:00 2001 From: ysqyang Date: Fri, 30 Oct 2020 17:26:41 +0800 Subject: [PATCH 101/337] fixed a bug --- examples/cim/ac/dist_actor.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/cim/ac/dist_actor.py b/examples/cim/ac/dist_actor.py index 6905ed8d6..6f43bd8fd 100644 --- a/examples/cim/ac/dist_actor.py +++ b/examples/cim/ac/dist_actor.py @@ -10,7 +10,7 @@ from maro.utils import convert_dottable from components.action_shaper import CIMActionShaper -from components.agent_manager import create_ac_agent, ACAgentManager +from components.agent_manager import create_ac_agents, ACAgentManager from components.config import config, set_input_dim from components.experience_shaper import TruncatedExperienceShaper from components.state_shaper import CIMStateShaper @@ -28,7 +28,7 @@ def launch(config): agent_manager = ACAgentManager( name="cim_remote_actor", mode=AgentManagerMode.INFERENCE, - agent_dict=create_ac_agent(agent_id_list, config.agents), + agent_dict=create_ac_agents(agent_id_list, config.agents), state_shaper=state_shaper, action_shaper=action_shaper, experience_shaper=experience_shaper, From 74b0880eb262ca44dfb6e0d8faa4690ac14934b7 Mon Sep 17 00:00:00 2001 From: ysqyang Date: Fri, 30 Oct 2020 17:31:22 +0800 Subject: [PATCH 102/337] fixed a bug --- examples/cim/ac/components/agent_manager.py | 8 ++++---- examples/cim/pg/components/agent_manager.py | 11 ++++++----- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/examples/cim/ac/components/agent_manager.py b/examples/cim/ac/components/agent_manager.py index 240e35675..fa6b0b5e4 100644 --- a/examples/cim/ac/components/agent_manager.py +++ b/examples/cim/ac/components/agent_manager.py @@ -50,9 +50,9 @@ def create_ac_agents(agent_id_list, config): class ACAgentManager(SimpleAgentManager): def train(self, experiences_by_agent: dict): - for agent_id, experiences in experiences_by_agent.items(): - if isinstance(experiences, list): - for trajectory in experiences_by_agent: + for agent_id, exp in experiences_by_agent.items(): + if isinstance(exp, list): + for trajectory in exp: self.agent_dict[agent_id].train(trajectory["states"], trajectory["actions"], trajectory["rewards"]) else: - self.agent_dict[agent_id].train(experiences["states"], experiences["actions"], experiences["rewards"]) + self.agent_dict[agent_id].train(exp["states"], exp["actions"], exp["rewards"]) diff --git a/examples/cim/pg/components/agent_manager.py b/examples/cim/pg/components/agent_manager.py index 403cda021..38109396f 100644 --- a/examples/cim/pg/components/agent_manager.py +++ b/examples/cim/pg/components/agent_manager.py @@ -39,9 +39,10 @@ def create_pg_agents(agent_id_list, config): class PGAgentManager(SimpleAgentManager): def train(self, experiences_by_agent: dict): - for agent_id, experiences in experiences_by_agent.items(): - if isinstance(experiences, list): - for trajectory in experiences_by_agent: - self.agent_dict[agent_id].train(trajectory["states"], trajectory["actions"], trajectory["rewards"]) + for agent_id, exp in experiences_by_agent.items(): + if isinstance(exp, list): + for trajectory in exp: + self.agent_dict[agent_id].train(trajectory["states"], trajectory["actions"], + trajectory["rewards"]) else: - self.agent_dict[agent_id].train(experiences["states"], experiences["actions"], experiences["rewards"]) + self.agent_dict[agent_id].train(exp["states"], exp["actions"], exp["rewards"]) From 24827b9249e04f0f5f677ec75047a15c1ccdddc3 Mon Sep 17 00:00:00 2001 From: ysqyang Date: Fri, 30 Oct 2020 17:38:15 +0800 Subject: [PATCH 103/337] updated rl toolkit doc and removed an unused import --- docs/source/key_components/rl_toolkit.rst | 8 +++----- maro/rl/agent/abs_agent.py | 1 - 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/docs/source/key_components/rl_toolkit.rst b/docs/source/key_components/rl_toolkit.rst index 1c526b446..72050a365 100644 --- a/docs/source/key_components/rl_toolkit.rst +++ b/docs/source/key_components/rl_toolkit.rst @@ -22,16 +22,14 @@ Learner and Actor .. code-block:: python # Train function of learner. - def train(self, total_episodes): - for current_ep in range(total_episodes): - models = self._trainable_agents.get_models() + def train(self, max_episode: + for current_ep in range(max_episode): + models = self._trainable_agents.dump_models() performance, experiences = self._actor.roll_out(models=models, epsilons=self._trainable_agents.explorer.epsilons, seed=self._seed) - self._trainable_agents.store_experiences(experiences) self._trainable_agents.train() - self._trainable_agents.update_epsilon(performance) * **Actor** is the abstraction of experience collection. It is responsible for interacting with the environment and collecting experience. The experiences diff --git a/maro/rl/agent/abs_agent.py b/maro/rl/agent/abs_agent.py index beded268f..24398d4f8 100644 --- a/maro/rl/agent/abs_agent.py +++ b/maro/rl/agent/abs_agent.py @@ -2,7 +2,6 @@ # Licensed under the MIT license. from abc import ABC, abstractmethod -from enum import Enum import os import pickle From b29d225ad5f76edacc5e00b4e6ff98921d70d9e1 Mon Sep 17 00:00:00 2001 From: ysqyang Date: Fri, 30 Oct 2020 20:37:49 +0800 Subject: [PATCH 104/337] fixed a typo --- examples/cim/pg/single_process_launcher.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/cim/pg/single_process_launcher.py b/examples/cim/pg/single_process_launcher.py index a7d604b2a..2daa0cfe4 100644 --- a/examples/cim/pg/single_process_launcher.py +++ b/examples/cim/pg/single_process_launcher.py @@ -47,7 +47,7 @@ def launch(config): trainable_agents=agent_manager, actor=actor, logger=Logger("single_host_cim_learner", auto_timestamp=False) ) - learner.train(max_episode=config.general.max_episodes) + learner.train(max_episode=config.general.max_episode) learner.test() learner.dump_models(os.path.join(os.getcwd(), "models")) From 9709ca4e11d5018b0517d22f0a0f919495d5df37 Mon Sep 17 00:00:00 2001 From: ysqyang Date: Mon, 2 Nov 2020 16:35:33 +0800 Subject: [PATCH 105/337] replaced IdentityLayers with nn.Identity --- maro/rl/models/learning_model.py | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/maro/rl/models/learning_model.py b/maro/rl/models/learning_model.py index b186c528e..378f59175 100644 --- a/maro/rl/models/learning_model.py +++ b/maro/rl/models/learning_model.py @@ -5,12 +5,6 @@ import torch.nn as nn -class IdentityLayers(nn.Module): - """A convenience dummy model that effects the identity transformation.""" - def forward(self, inputs): - return inputs - - class LearningModel(nn.Module): """A general model abstraction that consists of representation layers and decision layers. @@ -22,8 +16,8 @@ class LearningModel(nn.Module): """ def __init__( self, - representation_layers: nn.Module = IdentityLayers(), - decision_layers: nn.Module = IdentityLayers(), + representation_layers: nn.Module = nn.Identity(), + decision_layers: nn.Module = nn.Identity(), clip_value: float = None ): super().__init__() From 39b2bf5ca8b4c650ebeac331d68056f19c201c87 Mon Sep 17 00:00:00 2001 From: ysqyang Date: Mon, 2 Nov 2020 17:09:31 +0800 Subject: [PATCH 106/337] added truncated_cumulative_reward util function --- maro/rl/algorithms/ac.py | 8 ++-- maro/rl/algorithms/ppo.py | 8 ++-- maro/rl/utils/trajectory_utils.py | 66 ++++++++++++++++++++++--------- tests/test_trajectory_utils.py | 4 +- 4 files changed, 58 insertions(+), 28 deletions(-) diff --git a/maro/rl/algorithms/ac.py b/maro/rl/algorithms/ac.py index 7a8bebc24..bea966432 100644 --- a/maro/rl/algorithms/ac.py +++ b/maro/rl/algorithms/ac.py @@ -89,8 +89,8 @@ def _get_values_and_bootstrapped_returns(self, state_sequence, reward_sequence): state_values = self._model_dict["value"](state_sequence).detach().squeeze() state_values_numpy = state_values.numpy() return_est = get_lambda_returns( - reward_sequence, self._hyper_params.reward_decay, self._hyper_params.lam, - k=self._hyper_params.k, values=state_values_numpy + reward_sequence, state_values_numpy, self._hyper_params.reward_decay, self._hyper_params.lam, + k=self._hyper_params.k ) return_est = torch.from_numpy(return_est) return state_values, return_est @@ -196,8 +196,8 @@ def _get_values_and_bootstrapped_returns(self, state_sequence, reward_sequence): state_values = self._policy_value_model(state_sequence)[0].detach().squeeze() state_values_numpy = state_values.numpy() return_est = get_lambda_returns( - reward_sequence, self._hyper_params.reward_decay, self._hyper_params.lam, - k=self._hyper_params.k, values=state_values_numpy + reward_sequence, state_values_numpy, self._hyper_params.reward_decay, self._hyper_params.lam, + k=self._hyper_params.k ) return_est = torch.from_numpy(return_est) return state_values, return_est diff --git a/maro/rl/algorithms/ppo.py b/maro/rl/algorithms/ppo.py index 2386ffeec..4904e1d44 100644 --- a/maro/rl/algorithms/ppo.py +++ b/maro/rl/algorithms/ppo.py @@ -85,8 +85,8 @@ def _get_values_and_bootstrapped_returns(self, states: torch.tensor, rewards: np state_values = self._model_dict["value"](states).detach().squeeze() state_values_numpy = state_values.numpy() return_est = get_lambda_returns( - rewards, self._hyper_params.reward_decay, self._hyper_params.lam, - k=self._hyper_params.k, values=state_values_numpy + rewards, state_values_numpy, self._hyper_params.reward_decay, self._hyper_params.lam, + k=self._hyper_params.k ) return_est = torch.from_numpy(return_est) return state_values, return_est @@ -204,8 +204,8 @@ def _get_values_and_bootstrapped_returns(self, state_sequence, reward_sequence): state_values = self._policy_value_model(state_sequence)[0].detach().squeeze() state_values_numpy = state_values.numpy() return_est = get_lambda_returns( - reward_sequence, self._hyper_params.reward_decay, self._hyper_params.lam, - k=self._hyper_params.k, values=state_values_numpy + reward_sequence, state_values_numpy, self._hyper_params.reward_decay, self._hyper_params.lam, + k=self._hyper_params.k ) return_est = torch.from_numpy(return_est) return state_values, return_est diff --git a/maro/rl/utils/trajectory_utils.py b/maro/rl/utils/trajectory_utils.py index 2366b4624..a73b51c4b 100644 --- a/maro/rl/utils/trajectory_utils.py +++ b/maro/rl/utils/trajectory_utils.py @@ -2,48 +2,78 @@ # Licensed under the MIT license. from functools import reduce +from typing import Union import numpy as np -def get_k_step_returns(rewards: np.ndarray, discount: float, k: int = -1, values: np.ndarray = None): +def get_truncated_cumulative_reward( + rewards: Union[list, np.ndarray], + discount: float, + k: int = -1 +): + """Compute K-step cumulative rewards from a reward sequence. + Args: + rewards (list or np.ndarray): reward sequence from a trajectory. + discount (float): reward discount as in standard RL. + k (int): number of steps in computing cumulative rewards. If it is -1, returns are computed using the + largest possible number of steps. Defaults to -1. + + Returns: + An ndarray containing the k-step cumulative rewards for each time step. + """ + if k < 0: + k = len(rewards) - 1 + return reduce( + lambda x, y: x * discount + y, + [np.pad(rewards[i:], (0, i)) for i in range(min(k, len(rewards)) - 1, -1, -1)] + ) + + +def get_k_step_returns( + rewards: Union[list, np.ndarray], + values: Union[list, np.ndarray], + discount: float, + k: int = -1 +): """Compute K-step returns given reward and value sequences. Args: - rewards (np.ndarray): reward sequence from a trajectory. + rewards (list or np.ndarray): reward sequence from a trajectory. + values (list or np.ndarray): sequence of values for the traversed states in a trajectory. discount (float): reward discount as in standard RL. k (int): number of steps in computing returns. If it is -1, returns are computed using the largest possible number of steps. Defaults to -1. - values (np.ndarray): sequence of values for the traversed states in a trajectory. If it is None, the state - immediately after the final state in the given sequence is assumed to be terminal with value zero, and the - computed returns for k = -1 are actual full returns. Defaults to None. Returns: An ndarray containing the k-step returns for each time step. """ - if values is not None: - assert len(rewards) == len(values), "rewards and values should have the same length" - assert len(values.shape) == 1, "values should be a one-dimensional array" - rewards[-1] = values[-1] + assert len(rewards) == len(values), "rewards and values should have the same length" + assert len(values.shape) == 1, "values should be a one-dimensional array" + rewards[-1] = values[-1] if k < 0: k = len(rewards) - 1 return reduce( lambda x, y: x * discount + y, [np.pad(rewards[i:], (0, i)) for i in range(min(k, len(rewards)) - 1, -1, -1)], - np.pad(values[k:], (0, k)) if values is not None else np.zeros(len(rewards)) + np.pad(values[k:], (0, k)) ) -def get_lambda_returns(rewards: np.ndarray, discount: float, lam: float, k: int = -1, values: np.ndarray = None): +def get_lambda_returns( + rewards: Union[list, np.ndarray], + values: Union[list, np.ndarray], + discount: float, + lam: float, + k: int = -1 +): """Compute lambda returns given reward and value sequences and a k. Args: rewards (np.ndarray): reward sequence from a trajectory. + values (np.ndarray): sequence of values for the traversed states in a trajectory. discount (float): reward discount as in standard RL. lam (float): the lambda coefficient involved in computing lambda returns. k (int): number of steps where the lambda return series is truncated. If it is -1, no truncating is done and the lambda return is carried out to the end of the sequence. Defaults to -1. - values (np.ndarray): sequence of values for the traversed states in a trajectory. If it is None, the state - immediately after the final state in the given sequence is assumed to be terminal with value zero. - Defaults to None. Returns: An ndarray containing the lambda returns for each time step. @@ -53,17 +83,17 @@ def get_lambda_returns(rewards: np.ndarray, discount: float, lam: float, k: int # If lambda is zero, lambda return reduces to one-step return if lam == .0: - return get_k_step_returns(rewards, discount, k=1, values=values) + return get_k_step_returns(rewards, values, discount, k=1) # If lambda is one, lambda return reduces to k-step return if lam == 1.0: - return get_k_step_returns(rewards, discount, k=k, values=values) + return get_k_step_returns(rewards, values, discount, k=k) k = min(k, len(rewards) - 1) pre_truncate = reduce( lambda x, y: x * lam + y, - [get_k_step_returns(rewards, discount, k=k, values=values) for k in range(k - 1, 0, -1)] + [get_k_step_returns(rewards, values, discount, k=k) for k in range(k - 1, 0, -1)] ) - post_truncate = get_k_step_returns(rewards, discount, k=k, values=values) * lam**(k - 1) + post_truncate = get_k_step_returns(rewards, values, discount, k=k) * lam**(k - 1) return (1 - lam) * pre_truncate + post_truncate diff --git a/tests/test_trajectory_utils.py b/tests/test_trajectory_utils.py index feb9bdc85..df05d8032 100644 --- a/tests/test_trajectory_utils.py +++ b/tests/test_trajectory_utils.py @@ -17,12 +17,12 @@ def setUp(self) -> None: self.k = 4 def test_k_step_return(self): - returns = get_k_step_returns(self.rewards, self.discount, k=self.k, values=self.values) + returns = get_k_step_returns(self.rewards, self.values, self.discount, k=self.k) expected = np.asarray([10.1296, 8.912, 8.64, 5.8, 6.0]) np.testing.assert_allclose(returns, expected, rtol=1e-4) def test_lambda_return(self): - returns = get_lambda_returns(self.rewards, self.discount, self.lam, k=self.k, values=self.values) + returns = get_lambda_returns(self.rewards, self.values, self.discount, self.lam, k=self.k) expected = np.asarray([8.1378176, 6.03712, 7.744, 5.8, 6.0]) np.testing.assert_allclose(returns, expected, rtol=1e-4) From df1555b7fc0eb98f9138d6f5d6ee2f4ec3250035 Mon Sep 17 00:00:00 2001 From: ysqyang Date: Mon, 2 Nov 2020 23:10:28 +0800 Subject: [PATCH 107/337] 1. added skip connection option in FC_net; 2. generalized learning model --- examples/cim/dqn/components/agent_manager.py | 5 ++-- maro/rl/models/fc_net.py | 29 ++++++++++++++++++-- maro/rl/models/learning_model.py | 22 ++++----------- 3 files changed, 35 insertions(+), 21 deletions(-) diff --git a/examples/cim/dqn/components/agent_manager.py b/examples/cim/dqn/components/agent_manager.py index 67367a657..2ac9aa686 100644 --- a/examples/cim/dqn/components/agent_manager.py +++ b/examples/cim/dqn/components/agent_manager.py @@ -15,11 +15,12 @@ def create_dqn_agents(agent_id_list, config): agent_dict = {} for agent_id in agent_id_list: eval_model = LearningModel( - decision_layers=FullyConnectedNet( + FullyConnectedNet( name=f'{agent_id}.policy', input_dim=config.algorithm.input_dim, output_dim=num_actions, - activation=nn.LeakyReLU, **config.algorithm.model + activation=nn.LeakyReLU, + **config.algorithm.model ) ) diff --git a/maro/rl/models/fc_net.py b/maro/rl/models/fc_net.py index ef94cbac0..2ab067f13 100644 --- a/maro/rl/models/fc_net.py +++ b/maro/rl/models/fc_net.py @@ -3,6 +3,7 @@ from collections import OrderedDict +import torch import torch.nn as nn @@ -20,11 +21,24 @@ class FullyConnectedNet(nn.Module): softmax_enabled (bool): If true, the output of the net will be a softmax transformation of the top layer's output. Defaults to False. batch_norm_enabled (bool): If true, batch normalization will be performed at each layer. + skip_connection_enabled (bool): If true, a skip connection will be built between the bottom (input) layer and + top (output) layer. Defaults to False. dropout_p (float): Dropout probability. Defaults to None, in which case there is no drop-out. + clip_value (float): Gradient clipping threshold. Defaults to None, in which case not gradient clipping is + performed. """ def __init__( - self, name: str, input_dim: int, output_dim: int, hidden_dims: [int], activation=nn.LeakyReLU, - softmax_enabled: bool = False, batch_norm_enabled: bool = False, dropout_p: float = None + self, + name: str, + input_dim: int, + output_dim: int, + hidden_dims: [int], + activation=nn.LeakyReLU, + softmax_enabled: bool = False, + batch_norm_enabled: bool = False, + skip_connection_enabled: bool = False, + dropout_p: float = None, + clip_value: float = None ): super().__init__() self._name = name @@ -38,6 +52,11 @@ def __init__( self._batch_norm_enabled = batch_norm_enabled self._dropout_p = dropout_p + if skip_connection_enabled and input_dim != output_dim: + raise ValueError(f"input and output dimensions must match if skip connection is enabled") + + self._skip_connection_enabled = skip_connection_enabled + # build the net self._layers = self._build_layers([input_dim] + self._hidden_dims) if len(self._hidden_dims) == 0: @@ -46,8 +65,14 @@ def __init__( self._top_layer = nn.Linear(hidden_dims[-1], self._output_dim) self._net = nn.Sequential(*self._layers, self._top_layer) + if clip_value is not None: + for param in self._net.parameters(): + param.register_hook(lambda grad: torch.clamp(grad, -clip_value, clip_value)) + def forward(self, x): out = self._net(x).double() + if self._skip_connection_enabled: + out += x return self._softmax(out) if self._softmax else out @property diff --git a/maro/rl/models/learning_model.py b/maro/rl/models/learning_model.py index 378f59175..459e61dcd 100644 --- a/maro/rl/models/learning_model.py +++ b/maro/rl/models/learning_model.py @@ -6,26 +6,14 @@ class LearningModel(nn.Module): - """A general model abstraction that consists of representation layers and decision layers. + """A general NN model that consists of multiple building blocks. - Args: - representation_layers (nn.Module): An NN-based feature extractor. - decision_layers (nn.Module): An NN model that takes the output of the representation layers as input - and outputs values of interest in RL (e.g., state & action values). - clip_value (float): Threshold used to clip gradients. + The building blocks must be chainable, i.e., the output dimension of one block must match the input dimension of + its successors. """ - def __init__( - self, - representation_layers: nn.Module = nn.Identity(), - decision_layers: nn.Module = nn.Identity(), - clip_value: float = None - ): + def __init__(self, *blocks): super().__init__() - self._net = nn.Sequential(representation_layers, decision_layers) - - if clip_value is not None: - for param in self._net.parameters(): - param.register_hook(lambda grad: torch.clamp(grad, -clip_value, clip_value)) + self._net = nn.Sequential(*blocks) def forward(self, inputs): return self._net(inputs) From 6746c07039e0c16f706aad3e81d0d8599888f575 Mon Sep 17 00:00:00 2001 From: ysqyang Date: Mon, 2 Nov 2020 23:24:23 +0800 Subject: [PATCH 108/337] added skip_connection option in config --- maro/rl/models/fc_net.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/maro/rl/models/fc_net.py b/maro/rl/models/fc_net.py index 2ab067f13..54177d7d0 100644 --- a/maro/rl/models/fc_net.py +++ b/maro/rl/models/fc_net.py @@ -53,7 +53,8 @@ def __init__( self._dropout_p = dropout_p if skip_connection_enabled and input_dim != output_dim: - raise ValueError(f"input and output dimensions must match if skip connection is enabled") + raise ValueError(f"input and output dimensions must match if skip connection is enabled, " + f"got {input_dim} and {output_dim}") self._skip_connection_enabled = skip_connection_enabled From 079c5a8d191af90ff2313db95acf6867530fb215 Mon Sep 17 00:00:00 2001 From: ysqyang Date: Mon, 2 Nov 2020 23:48:08 +0800 Subject: [PATCH 109/337] removed type casting in fc_net --- examples/cim/dqn/components/agent.py | 4 ++-- examples/cim/dqn/config.yml | 1 + maro/rl/models/fc_net.py | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/examples/cim/dqn/components/agent.py b/examples/cim/dqn/components/agent.py index 9cf358166..db7f7b5f2 100644 --- a/examples/cim/dqn/components/agent.py +++ b/examples/cim/dqn/components/agent.py @@ -25,7 +25,7 @@ def train(self): """Implementation of the training loop for DQN. Experiences are sampled using their TD errors as weights. After training, the new TD errors are updated - in the experience pool. + in the experience pool. """ if len(self._experience_pool) < self._min_experiences_to_train: return @@ -34,7 +34,7 @@ def train(self): indexes, sample = self._experience_pool.sample_by_key("loss", self._batch_size) state = np.asarray(sample["state"]) action = np.asarray(sample["action"]) - reward = np.asarray(sample["reward"]) + reward = np.asarray(sample["reward"], dtype=np.float32) next_state = np.asarray(sample["next_state"]) loss = self._algorithm.train(state, action, reward, next_state) self._experience_pool.update(indexes, {"loss": loss}) diff --git a/examples/cim/dqn/config.yml b/examples/cim/dqn/config.yml index 8769eef79..5e98d8320 100644 --- a/examples/cim/dqn/config.yml +++ b/examples/cim/dqn/config.yml @@ -48,6 +48,7 @@ agents: - 64 softmax_enabled: false batch_norm_enabled: true + skip_connection_enabled: false dropout_p: 0.0 optimizer: lr: 0.05 diff --git a/maro/rl/models/fc_net.py b/maro/rl/models/fc_net.py index 54177d7d0..2156da998 100644 --- a/maro/rl/models/fc_net.py +++ b/maro/rl/models/fc_net.py @@ -71,7 +71,7 @@ def __init__( param.register_hook(lambda grad: torch.clamp(grad, -clip_value, clip_value)) def forward(self, x): - out = self._net(x).double() + out = self._net(x) if self._skip_connection_enabled: out += x return self._softmax(out) if self._softmax else out From afce3247f413259169aaa31b2f88c9e9d27aa80d Mon Sep 17 00:00:00 2001 From: ysqyang Date: Mon, 2 Nov 2020 23:53:06 +0800 Subject: [PATCH 110/337] fixed lint formatting issues --- maro/rl/models/fc_net.py | 6 ++++-- maro/rl/models/learning_model.py | 1 - 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/maro/rl/models/fc_net.py b/maro/rl/models/fc_net.py index 2156da998..4a4a5543c 100644 --- a/maro/rl/models/fc_net.py +++ b/maro/rl/models/fc_net.py @@ -53,8 +53,10 @@ def __init__( self._dropout_p = dropout_p if skip_connection_enabled and input_dim != output_dim: - raise ValueError(f"input and output dimensions must match if skip connection is enabled, " - f"got {input_dim} and {output_dim}") + raise ValueError( + f"input and output dimensions must match if skip connection is enabled, " + f"got {input_dim} and {output_dim}" + ) self._skip_connection_enabled = skip_connection_enabled diff --git a/maro/rl/models/learning_model.py b/maro/rl/models/learning_model.py index 459e61dcd..0f92f9b12 100644 --- a/maro/rl/models/learning_model.py +++ b/maro/rl/models/learning_model.py @@ -1,7 +1,6 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT license. -import torch import torch.nn as nn From 8249c33c3f8e6e10fd77a6d977840d87a7b07ca4 Mon Sep 17 00:00:00 2001 From: ysqyang Date: Tue, 3 Nov 2020 00:00:44 +0800 Subject: [PATCH 111/337] refined docstring --- maro/rl/models/learning_model.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/maro/rl/models/learning_model.py b/maro/rl/models/learning_model.py index 0f92f9b12..6f95fd7e5 100644 --- a/maro/rl/models/learning_model.py +++ b/maro/rl/models/learning_model.py @@ -5,7 +5,7 @@ class LearningModel(nn.Module): - """A general NN model that consists of multiple building blocks. + """NN model that consists of multiple building blocks. The building blocks must be chainable, i.e., the output dimension of one block must match the input dimension of its successors. From 009793ed758135d678b986bd81a9ef01fc536dc8 Mon Sep 17 00:00:00 2001 From: ysqyang Date: Tue, 3 Nov 2020 00:13:25 +0800 Subject: [PATCH 112/337] mv dueling_actiovalue_model and fixed some bugs --- .../{torch => }/dueling_action_value_model.py | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) rename maro/rl/models/{torch => }/dueling_action_value_model.py (62%) diff --git a/maro/rl/models/torch/dueling_action_value_model.py b/maro/rl/models/dueling_action_value_model.py similarity index 62% rename from maro/rl/models/torch/dueling_action_value_model.py rename to maro/rl/models/dueling_action_value_model.py index 7e986dc23..6e2362cf1 100644 --- a/maro/rl/models/torch/dueling_action_value_model.py +++ b/maro/rl/models/dueling_action_value_model.py @@ -3,25 +3,23 @@ import torch.nn as nn -from maro.rl.models.torch.learning_model import IdentityLayers - class DuelingActionValueModel(nn.Module): - def __init__(self, value_head: nn.Module, advantage_head: nn.Module, shared_layers:nn.Module = IdentityLayers(), - advantage_mode: str = 'mean'): + def __init__(self, value_head: nn.Module, advantage_head: nn.Module, shared_layers:nn.Module = nn.Identity(), + advantage_mode: str = "mean"): super().__init__() self._value_head = value_head self._advantage_head = advantage_head self._shared_layers = shared_layers - if self._advantage_mode not in {'mean', 'max'}: - raise ValueError('Advantage mode must be "mean" or "max"') + if self._advantage_mode not in {"mean", "max"}: + raise ValueError("Advantage mode must be 'mean' or 'max'") self._advantage_mode = advantage_mode def forward(self, inputs): features = self._shared_layers(inputs) - state_values = self._value_layers(features) - advantages = self._advantage_layers(features) + state_values = self._value_head(features) + advantages = self._advantage_head(features) # use mean or max correction to address the identifiability issue - corrections = advantages.mean(1) if self._advantage_mode == 'mean' else advantages.max(1)[0] + corrections = advantages.mean(1) if self._advantage_mode == "mean" else advantages.max(1)[0] q_values = state_values + advantages - corrections.unsqueeze(1) return q_values From a2fef981c2b89ccb1010c0b0f8172a7c1b4068ec Mon Sep 17 00:00:00 2001 From: ysqyang Date: Tue, 3 Nov 2020 00:28:39 +0800 Subject: [PATCH 113/337] added multi-head functionality to LearningModel --- maro/rl/models/learning_model.py | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/maro/rl/models/learning_model.py b/maro/rl/models/learning_model.py index 6f95fd7e5..29e9f4cb0 100644 --- a/maro/rl/models/learning_model.py +++ b/maro/rl/models/learning_model.py @@ -5,14 +5,19 @@ class LearningModel(nn.Module): - """NN model that consists of multiple building blocks. + """NN model that consists of multiple shared blocks and multiple heads. - The building blocks must be chainable, i.e., the output dimension of one block must match the input dimension of + The shared blocks must be chainable, i.e., the output dimension of one block must match the input dimension of its successors. """ - def __init__(self, *blocks): + def __init__(self, *blocks, **heads): super().__init__() - self._net = nn.Sequential(*blocks) + self._shared = nn.Sequential(*blocks) + self._heads = heads def forward(self, inputs): - return self._net(inputs) + if not self._heads: + return self._shared(inputs) + else: + features = self._shared(inputs) + return {name: layers(features) for name, layers in self._heads.items()} From b5302928a65007cc9c4a2149d87e7a756c0186d1 Mon Sep 17 00:00:00 2001 From: ysqyang Date: Tue, 3 Nov 2020 00:33:49 +0800 Subject: [PATCH 114/337] refined learning model docstring --- maro/rl/models/learning_model.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/maro/rl/models/learning_model.py b/maro/rl/models/learning_model.py index 29e9f4cb0..25cbe7a75 100644 --- a/maro/rl/models/learning_model.py +++ b/maro/rl/models/learning_model.py @@ -7,8 +7,10 @@ class LearningModel(nn.Module): """NN model that consists of multiple shared blocks and multiple heads. - The shared blocks must be chainable, i.e., the output dimension of one block must match the input dimension of - its successors. + The shared blocks must be chainable, i.e., the output dimension of a block must match the input dimension of + its successor. Heads must be provided in the form of keyword arguments. If at least one head is provided, the + output of the model will be a dictionary with the names of the heads as keys and the corresponding head outputs + as values. Otherwise, the output will be the output of the last block. """ def __init__(self, *blocks, **heads): super().__init__() From bfc0e0fbef88d5de4205a15315f5f50a51de415d Mon Sep 17 00:00:00 2001 From: ysqyang Date: Tue, 3 Nov 2020 09:15:52 +0800 Subject: [PATCH 115/337] added head_key param in learningModel forward --- maro/rl/models/learning_model.py | 25 ++++++++++++++++++++++--- 1 file changed, 22 insertions(+), 3 deletions(-) diff --git a/maro/rl/models/learning_model.py b/maro/rl/models/learning_model.py index 25cbe7a75..40aea90f5 100644 --- a/maro/rl/models/learning_model.py +++ b/maro/rl/models/learning_model.py @@ -17,9 +17,28 @@ def __init__(self, *blocks, **heads): self._shared = nn.Sequential(*blocks) self._heads = heads - def forward(self, inputs): + def forward(self, inputs, head_key=None): + """Feedforward computations for the given head(s). + + Args: + inputs: Inputs to the model. + head_key: The key(s) to the head(s) from which the output is required. If this is None, the results from + all heads will be returned in the form of a dictionary. If this is a list, the results will be the + outputs from the heads contained in head_key in the form of a dictionary. If this is a single key, + the result will be the output from the corresponding head. + + Returns: + Outputs from the required head(s). + """ if not self._heads: return self._shared(inputs) + + features = self._shared(inputs) + + if head_key is None: + return {head_name: layers(features) for head_name, layers in self._heads.items()} + + if isinstance(head_key, list): + return {head_name: self._heads[head_name](features) for head_name in head_key} else: - features = self._shared(inputs) - return {name: layers(features) for name, layers in self._heads.items()} + return self._heads[head_key](features) From fa568196bdd897fed5f634cbadf0596f03743f9c Mon Sep 17 00:00:00 2001 From: ysqyang Date: Tue, 3 Nov 2020 10:30:38 +0800 Subject: [PATCH 116/337] added double DQN and dueling features to DQN --- examples/cim/dqn/config.yml | 2 ++ maro/rl/algorithms/dqn.py | 37 +++++++++++++++++--- maro/rl/models/dueling_action_value_model.py | 2 +- 3 files changed, 35 insertions(+), 6 deletions(-) diff --git a/examples/cim/dqn/config.yml b/examples/cim/dqn/config.yml index 5e98d8320..61691777a 100644 --- a/examples/cim/dqn/config.yml +++ b/examples/cim/dqn/config.yml @@ -56,6 +56,8 @@ agents: reward_decay: .0 target_update_frequency: 5 tau: 0.1 + is_double: false + advantage_mode: "mean" experience_pool: capacity: -1 training_loop_parameters: diff --git a/maro/rl/algorithms/dqn.py b/maro/rl/algorithms/dqn.py index ac6945f17..249f4ea1a 100644 --- a/maro/rl/algorithms/dqn.py +++ b/maro/rl/algorithms/dqn.py @@ -17,20 +17,28 @@ class DQNHyperParams: reward_decay (float): Reward decay as defined in standard RL terminology. target_update_frequency (int): Number of training rounds between target model updates. tau (float): Soft update coefficient, i.e., target_model = tau * eval_model + (1-tau) * target_model. + is_double (bool): If True, the next Q values will be computed according to the double DQN algorithm. + See https://arxiv.org/pdf/1509.06461.pdf for details. Defaults to False. + advantage_mode (str): advantage mode for the dueling Q-value model. Defaults to None, in which + case it is assumed that the regular Q-value model is used. """ - __slots__ = ["num_actions", "reward_decay", "target_update_frequency", "tau"] + __slots__ = ["num_actions", "reward_decay", "target_update_frequency", "tau", "is_double", "advantage_mode"] def __init__( self, num_actions: int, reward_decay: float, target_update_frequency: int, - tau: float = 1.0 + tau: float = 1.0, + is_double: bool = False, + advantage_mode: str = None ): self.num_actions = num_actions self.reward_decay = reward_decay self.target_update_frequency = target_update_frequency self.tau = tau + self.is_double = is_double + self.advantage_mode = advantage_mode class DQN(AbsAlgorithm): @@ -82,7 +90,7 @@ def choose_action(self, state: np.ndarray, epsilon: float = None): state = torch.from_numpy(state).unsqueeze(0) self._model_dict["eval"].eval() with torch.no_grad(): - q_values = self._model_dict["eval"](state) + q_values = self._get_q_values("eval", state) return q_values.argmax(dim=1).item() return np.random.choice(self._hyper_params.num_actions) @@ -95,8 +103,9 @@ def train(self, states: np.ndarray, actions: np.ndarray, rewards: np.ndarray, ne next_states = torch.from_numpy(next_states).to(self._device) # (N, state_dim) if len(actions.shape) == 1: actions = actions.unsqueeze(1) # (N, 1) - current_q_values = self._model_dict["eval"](states).gather(1, actions).squeeze(1) # (N,) - next_q_values = self._model_dict["target"](next_states).max(dim=1)[0] # (N,) + current_q_values_all = self._get_q_values("eval", states) + current_q_values = current_q_values_all.gather(1, actions).squeeze(1) # (N,) + next_q_values = self._get_next_q_values(current_q_values_all, next_states) # (N,) target_q_values = (rewards + self._hyper_params.reward_decay * next_q_values).detach() # (N,) loss = self._loss_func(current_q_values, target_q_values) self._model_dict["eval"].train() @@ -118,6 +127,24 @@ def _update_target_model(self): self._hyper_params.tau * eval_params.data + (1 - self._hyper_params.tau) * target_params.data ) + def _get_q_values(self, which: str, states): + if self._hyper_params.advantage_mode is None: + return self._model_dict[which](states) + + state_values = self._model_dict[which](states, "state") + advantages = self._model_dict[which](states, "advantage") + # Use mean or max correction to address the identifiability issue + corrections = advantages.mean(1) if self._hyper_params.advantage_mode == "mean" else advantages.max(1)[0] + q_values = state_values + advantages - corrections.unsqueeze(1) + return q_values + + def _get_next_q_values(self, current_q_values_all, states): + if self._hyper_params.is_double: + actions = current_q_values_all.max(dim=1)[1] + return self._get_q_values("target", states).gather(1, actions).squeeze(1) # (N,) + else: + return self._get_q_values("target", states).max(dim=1)[0] # (N,) + def load_models(self, eval_model): """Load the eval model from memory.""" self._model_dict["eval"].load_state_dict(eval_model) diff --git a/maro/rl/models/dueling_action_value_model.py b/maro/rl/models/dueling_action_value_model.py index 6e2362cf1..026337d3e 100644 --- a/maro/rl/models/dueling_action_value_model.py +++ b/maro/rl/models/dueling_action_value_model.py @@ -6,7 +6,7 @@ class DuelingActionValueModel(nn.Module): def __init__(self, value_head: nn.Module, advantage_head: nn.Module, shared_layers:nn.Module = nn.Identity(), - advantage_mode: str = "mean"): + ): super().__init__() self._value_head = value_head self._advantage_head = advantage_head From 551a4a79681668645380b4192143e87d124e14b0 Mon Sep 17 00:00:00 2001 From: ysqyang Date: Tue, 3 Nov 2020 10:34:01 +0800 Subject: [PATCH 117/337] fixed a bug --- maro/rl/algorithms/dqn.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/maro/rl/algorithms/dqn.py b/maro/rl/algorithms/dqn.py index 249f4ea1a..289598ed8 100644 --- a/maro/rl/algorithms/dqn.py +++ b/maro/rl/algorithms/dqn.py @@ -140,7 +140,7 @@ def _get_q_values(self, which: str, states): def _get_next_q_values(self, current_q_values_all, states): if self._hyper_params.is_double: - actions = current_q_values_all.max(dim=1)[1] + actions = current_q_values_all.max(dim=1)[1].unsqueeze(1) return self._get_q_values("target", states).gather(1, actions).squeeze(1) # (N,) else: return self._get_q_values("target", states).max(dim=1)[0] # (N,) From 98501c53695a59e1a4f4775983b9542f4345d871 Mon Sep 17 00:00:00 2001 From: ysqyang Date: Tue, 3 Nov 2020 10:54:03 +0800 Subject: [PATCH 118/337] added DuelingQModelHead enum --- maro/rl/__init__.py | 2 +- maro/rl/algorithms/dqn.py | 11 +++++++++-- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/maro/rl/__init__.py b/maro/rl/__init__.py index eea2d84d9..3c3348d9a 100644 --- a/maro/rl/__init__.py +++ b/maro/rl/__init__.py @@ -9,7 +9,7 @@ from maro.rl.agent.abs_agent_manager import AbsAgentManager, AgentManagerMode from maro.rl.agent.simple_agent_manager import SimpleAgentManager from maro.rl.algorithms.abs_algorithm import AbsAlgorithm -from maro.rl.algorithms.dqn import DQN, DQNHyperParams +from maro.rl.algorithms.dqn import DQN, DQNHyperParams, DuelingQModelHead from maro.rl.models.learning_model import LearningModel from maro.rl.models.fc_net import FullyConnectedNet from maro.rl.storage.abs_store import AbsStore diff --git a/maro/rl/algorithms/dqn.py b/maro/rl/algorithms/dqn.py index 289598ed8..adab4cd5c 100644 --- a/maro/rl/algorithms/dqn.py +++ b/maro/rl/algorithms/dqn.py @@ -1,6 +1,8 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT license. +from enum import Enum + import numpy as np import torch import torch.nn as nn @@ -9,6 +11,11 @@ from maro.utils import clone +class DuelingQModelHead(Enum): + STATE_VALUE = "state_value" + ADVANTAGE = "advantage" + + class DQNHyperParams: """Hyper-parameter set for the DQN algorithm. @@ -131,8 +138,8 @@ def _get_q_values(self, which: str, states): if self._hyper_params.advantage_mode is None: return self._model_dict[which](states) - state_values = self._model_dict[which](states, "state") - advantages = self._model_dict[which](states, "advantage") + state_values = self._model_dict[which](states, head_key=DuelingQModelHead.STATE_VALUE) + advantages = self._model_dict[which](states, head_key=DuelingQModelHead.ADVANTAGE) # Use mean or max correction to address the identifiability issue corrections = advantages.mean(1) if self._hyper_params.advantage_mode == "mean" else advantages.max(1)[0] q_values = state_values + advantages - corrections.unsqueeze(1) From 0be772fff767a16e4be40865ff391aab77ded247 Mon Sep 17 00:00:00 2001 From: ysqyang Date: Tue, 3 Nov 2020 11:08:32 +0800 Subject: [PATCH 119/337] fixed a bug --- maro/rl/algorithms/dqn.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/maro/rl/algorithms/dqn.py b/maro/rl/algorithms/dqn.py index adab4cd5c..ae2a7cdc5 100644 --- a/maro/rl/algorithms/dqn.py +++ b/maro/rl/algorithms/dqn.py @@ -138,8 +138,8 @@ def _get_q_values(self, which: str, states): if self._hyper_params.advantage_mode is None: return self._model_dict[which](states) - state_values = self._model_dict[which](states, head_key=DuelingQModelHead.STATE_VALUE) - advantages = self._model_dict[which](states, head_key=DuelingQModelHead.ADVANTAGE) + state_values = self._model_dict[which](states, head_key=DuelingQModelHead.STATE_VALUE.value) + advantages = self._model_dict[which](states, head_key=DuelingQModelHead.ADVANTAGE.value) # Use mean or max correction to address the identifiability issue corrections = advantages.mean(1) if self._hyper_params.advantage_mode == "mean" else advantages.max(1)[0] q_values = state_values + advantages - corrections.unsqueeze(1) From f93ad36d41191a1d92b59e87f92a832f159f5993 Mon Sep 17 00:00:00 2001 From: ysqyang Date: Tue, 3 Nov 2020 11:54:33 +0800 Subject: [PATCH 120/337] removed unwanted file --- maro/rl/models/dueling_action_value_model.py | 25 -------------------- 1 file changed, 25 deletions(-) delete mode 100644 maro/rl/models/dueling_action_value_model.py diff --git a/maro/rl/models/dueling_action_value_model.py b/maro/rl/models/dueling_action_value_model.py deleted file mode 100644 index 026337d3e..000000000 --- a/maro/rl/models/dueling_action_value_model.py +++ /dev/null @@ -1,25 +0,0 @@ -# Copyright (c) Microsoft Corporation. -# Licensed under the MIT license. - -import torch.nn as nn - - -class DuelingActionValueModel(nn.Module): - def __init__(self, value_head: nn.Module, advantage_head: nn.Module, shared_layers:nn.Module = nn.Identity(), - ): - super().__init__() - self._value_head = value_head - self._advantage_head = advantage_head - self._shared_layers = shared_layers - if self._advantage_mode not in {"mean", "max"}: - raise ValueError("Advantage mode must be 'mean' or 'max'") - self._advantage_mode = advantage_mode - - def forward(self, inputs): - features = self._shared_layers(inputs) - state_values = self._value_head(features) - advantages = self._advantage_head(features) - # use mean or max correction to address the identifiability issue - corrections = advantages.mean(1) if self._advantage_mode == "mean" else advantages.max(1)[0] - q_values = state_values + advantages - corrections.unsqueeze(1) - return q_values From 475f7c1e525c3d75cf38a4ed54525357322e250e Mon Sep 17 00:00:00 2001 From: ysqyang Date: Tue, 3 Nov 2020 22:50:25 +0800 Subject: [PATCH 121/337] fixed PR comments --- maro/rl/models/learning_model.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/maro/rl/models/learning_model.py b/maro/rl/models/learning_model.py index 40aea90f5..b6c03ba2a 100644 --- a/maro/rl/models/learning_model.py +++ b/maro/rl/models/learning_model.py @@ -14,8 +14,8 @@ class LearningModel(nn.Module): """ def __init__(self, *blocks, **heads): super().__init__() - self._shared = nn.Sequential(*blocks) - self._heads = heads + self._representation_stack = nn.Sequential(*blocks) + self._net = {key: nn.Sequential(self._representation_stack, task_head) for key, task_head in heads.items()} def forward(self, inputs, head_key=None): """Feedforward computations for the given head(s). @@ -33,12 +33,10 @@ def forward(self, inputs, head_key=None): if not self._heads: return self._shared(inputs) - features = self._shared(inputs) - if head_key is None: - return {head_name: layers(features) for head_name, layers in self._heads.items()} + return {key: net(inputs) for key, net in self._net.items()} if isinstance(head_key, list): - return {head_name: self._heads[head_name](features) for head_name in head_key} + return {key: self._net[key](inputs) for key in head_key} else: - return self._heads[head_key](features) + return self._net[head_key](inputs) From 5a72609c3044000f1bc21c48e7425f8964ca8a3d Mon Sep 17 00:00:00 2001 From: ysqyang Date: Wed, 4 Nov 2020 00:08:08 +0800 Subject: [PATCH 122/337] added top layer logic and is_top option in fc_net --- maro/rl/models/fc_net.py | 34 +++++++++++++++++++--------------- 1 file changed, 19 insertions(+), 15 deletions(-) diff --git a/maro/rl/models/fc_net.py b/maro/rl/models/fc_net.py index 4a4a5543c..603ae6670 100644 --- a/maro/rl/models/fc_net.py +++ b/maro/rl/models/fc_net.py @@ -18,6 +18,8 @@ class FullyConnectedNet(nn.Module): output_dim (int): Network output dimension. hidden_dims ([int]): Dimensions of hidden layers. Its length is the number of hidden layers. activation: A ``torch.nn`` activation type. If None, there will be no activation. Defaults to LeakyReLU. + is_top (bool): If true, this block will be the top block of the full model and the top layer of this block + will be the final output layer. Defaults to False. softmax_enabled (bool): If true, the output of the net will be a softmax transformation of the top layer's output. Defaults to False. batch_norm_enabled (bool): If true, batch normalization will be performed at each layer. @@ -34,6 +36,7 @@ def __init__( output_dim: int, hidden_dims: [int], activation=nn.LeakyReLU, + is_top: bool = False, softmax_enabled: bool = False, batch_norm_enabled: bool = False, skip_connection_enabled: bool = False, @@ -48,6 +51,7 @@ def __init__( # network features self._activation = activation + self._is_top = is_top self._softmax = nn.Softmax(dim=1) if softmax_enabled else None self._batch_norm_enabled = batch_norm_enabled self._dropout_p = dropout_p @@ -61,12 +65,14 @@ def __init__( self._skip_connection_enabled = skip_connection_enabled # build the net - self._layers = self._build_layers([input_dim] + self._hidden_dims) - if len(self._hidden_dims) == 0: - self._top_layer = nn.Linear(self._input_dim, self._output_dim) + dims = [self._input_dim] + self._hidden_dims + layers = [self._build_basic_layer(in_dim, out_dim) for in_dim, out_dim in zip(dims, dims[1:])] + if is_top: + layers.append(self._build_top_layer(dims[-1], self._output_dim)) else: - self._top_layer = nn.Linear(hidden_dims[-1], self._output_dim) - self._net = nn.Sequential(*self._layers, self._top_layer) + layers.append(self._build_internal_layer(dims[-1], self._output_dim)) + + self._net = nn.Sequential(*layers) if clip_value is not None: for param in self._net.parameters(): @@ -90,7 +96,7 @@ def input_dim(self): def output_dim(self): return self._output_dim - def _build_basic_layer(self, input_dim, output_dim): + def _build_internal_layer(self, input_dim, output_dim): """Build basic layer. BN -> Linear -> Activation -> Dropout @@ -105,12 +111,10 @@ def _build_basic_layer(self, input_dim, output_dim): components.append(("dropout", nn.Dropout(p=self._dropout_p))) return nn.Sequential(OrderedDict(components)) - def _build_layers(self, layer_dims: []): - """Build multi basic layer. - - BasicLayer1 -> BasicLayer2 -> ... - """ - layers = [] - for input_dim, output_dim in zip(layer_dims, layer_dims[1:]): - layers.append(self._build_basic_layer(input_dim, output_dim)) - return layers + def _build_top_layer(self, input_dim, output_dim): + """The output layer may need to be treated differently.""" + components = [] + if self._batch_norm_enabled: + components.append(("batch_norm", nn.BatchNorm1d(input_dim))) + components.append(("linear", nn.Linear(input_dim, output_dim))) + return nn.Sequential(OrderedDict(components)) From 26140c8c5a4e7cfbbde90f96fcbb51f3539d7bf5 Mon Sep 17 00:00:00 2001 From: ysqyang Date: Wed, 4 Nov 2020 00:20:42 +0800 Subject: [PATCH 123/337] fixed a bug --- maro/rl/models/fc_net.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/maro/rl/models/fc_net.py b/maro/rl/models/fc_net.py index 603ae6670..5c35731e3 100644 --- a/maro/rl/models/fc_net.py +++ b/maro/rl/models/fc_net.py @@ -66,12 +66,12 @@ def __init__( # build the net dims = [self._input_dim] + self._hidden_dims - layers = [self._build_basic_layer(in_dim, out_dim) for in_dim, out_dim in zip(dims, dims[1:])] + layers = [self._build_internal_layer(in_dim, out_dim) for in_dim, out_dim in zip(dims, dims[1:])] if is_top: layers.append(self._build_top_layer(dims[-1], self._output_dim)) else: layers.append(self._build_internal_layer(dims[-1], self._output_dim)) - + self._net = nn.Sequential(*layers) if clip_value is not None: From 946546f903811c9b04cc93b50a57a2bec60e86bc Mon Sep 17 00:00:00 2001 From: ysqyang Date: Wed, 4 Nov 2020 00:24:25 +0800 Subject: [PATCH 124/337] fixed a bug --- maro/rl/models/learning_model.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/maro/rl/models/learning_model.py b/maro/rl/models/learning_model.py index b6c03ba2a..b971f9aec 100644 --- a/maro/rl/models/learning_model.py +++ b/maro/rl/models/learning_model.py @@ -30,8 +30,8 @@ def forward(self, inputs, head_key=None): Returns: Outputs from the required head(s). """ - if not self._heads: - return self._shared(inputs) + if not self._net: + return self._representation_stack(inputs) if head_key is None: return {key: net(inputs) for key, net in self._net.items()} From e30f485ce1fcd0773380e7990b7c1c306c15e171 Mon Sep 17 00:00:00 2001 From: ysqyang Date: Wed, 4 Nov 2020 00:42:48 +0800 Subject: [PATCH 125/337] reverted some changes in learning model --- maro/rl/models/learning_model.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/maro/rl/models/learning_model.py b/maro/rl/models/learning_model.py index b971f9aec..3e9c08f7e 100644 --- a/maro/rl/models/learning_model.py +++ b/maro/rl/models/learning_model.py @@ -15,7 +15,7 @@ class LearningModel(nn.Module): def __init__(self, *blocks, **heads): super().__init__() self._representation_stack = nn.Sequential(*blocks) - self._net = {key: nn.Sequential(self._representation_stack, task_head) for key, task_head in heads.items()} + self._task_heads = heads def forward(self, inputs, head_key=None): """Feedforward computations for the given head(s). @@ -30,13 +30,15 @@ def forward(self, inputs, head_key=None): Returns: Outputs from the required head(s). """ - if not self._net: + if not self._task_heads: return self._representation_stack(inputs) + representation_features = self._representation_stack(inputs) + if head_key is None: - return {key: net(inputs) for key, net in self._net.items()} + return {key: task_head(representation_features) for key, task_head in self._task_heads.items()} if isinstance(head_key, list): - return {key: self._net[key](inputs) for key in head_key} + return {key: self._task_heads[key](representation_features) for key in head_key} else: - return self._net[head_key](inputs) + return self._task_heads[head_key](representation_features) From dd1a0797e1b212edf55866b37f83e0d216e9d1d3 Mon Sep 17 00:00:00 2001 From: ysqyang Date: Wed, 4 Nov 2020 00:47:38 +0800 Subject: [PATCH 126/337] reverted some changes in learning model --- maro/rl/models/learning_model.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/maro/rl/models/learning_model.py b/maro/rl/models/learning_model.py index 3e9c08f7e..6187717bb 100644 --- a/maro/rl/models/learning_model.py +++ b/maro/rl/models/learning_model.py @@ -15,7 +15,7 @@ class LearningModel(nn.Module): def __init__(self, *blocks, **heads): super().__init__() self._representation_stack = nn.Sequential(*blocks) - self._task_heads = heads + self._net = {key: nn.Sequential(self._representation_stack, task_head) for key, task_head in heads.items()} def forward(self, inputs, head_key=None): """Feedforward computations for the given head(s). @@ -30,15 +30,17 @@ def forward(self, inputs, head_key=None): Returns: Outputs from the required head(s). """ - if not self._task_heads: + if not self._net: return self._representation_stack(inputs) - representation_features = self._representation_stack(inputs) - if head_key is None: - return {key: task_head(representation_features) for key, task_head in self._task_heads.items()} + return {key: net(inputs) for key, net in self._net.items()} if isinstance(head_key, list): - return {key: self._task_heads[key](representation_features) for key in head_key} + return {key: self._net[key](inputs) for key in head_key} else: - return self._task_heads[head_key](representation_features) + return self._net[head_key](inputs) + + def eval(self): + for net in self._net.values(): + net.eval() From 7be2516945c888001e5d34d39eb5f4bc291e311f Mon Sep 17 00:00:00 2001 From: ysqyang Date: Wed, 4 Nov 2020 09:03:22 +0800 Subject: [PATCH 127/337] added members to learning model to fix the mode issue --- maro/rl/models/learning_model.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/maro/rl/models/learning_model.py b/maro/rl/models/learning_model.py index 6187717bb..33c5b3d94 100644 --- a/maro/rl/models/learning_model.py +++ b/maro/rl/models/learning_model.py @@ -5,17 +5,19 @@ class LearningModel(nn.Module): - """NN model that consists of multiple shared blocks and multiple heads. + """NN model that consists of multiple shared blocks and multiple task heads. The shared blocks must be chainable, i.e., the output dimension of a block must match the input dimension of its successor. Heads must be provided in the form of keyword arguments. If at least one head is provided, the output of the model will be a dictionary with the names of the heads as keys and the corresponding head outputs as values. Otherwise, the output will be the output of the last block. """ - def __init__(self, *blocks, **heads): + def __init__(self, *blocks, **task_heads): super().__init__() self._representation_stack = nn.Sequential(*blocks) - self._net = {key: nn.Sequential(self._representation_stack, task_head) for key, task_head in heads.items()} + for key, task_head in task_heads: + setattr(self, f"_{key}_head", task_head) + self._net = {key: nn.Sequential(self._representation_stack, task_head) for key, task_head in task_heads.items()} def forward(self, inputs, head_key=None): """Feedforward computations for the given head(s). @@ -40,7 +42,3 @@ def forward(self, inputs, head_key=None): return {key: self._net[key](inputs) for key in head_key} else: return self._net[head_key](inputs) - - def eval(self): - for net in self._net.values(): - net.eval() From 8eb1358e677e68d0715f801e7f2f0df5d71304d6 Mon Sep 17 00:00:00 2001 From: ysqyang Date: Wed, 4 Nov 2020 09:06:41 +0800 Subject: [PATCH 128/337] fixed a bug --- maro/rl/models/learning_model.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/maro/rl/models/learning_model.py b/maro/rl/models/learning_model.py index 33c5b3d94..c6fa17e1f 100644 --- a/maro/rl/models/learning_model.py +++ b/maro/rl/models/learning_model.py @@ -15,7 +15,7 @@ class LearningModel(nn.Module): def __init__(self, *blocks, **task_heads): super().__init__() self._representation_stack = nn.Sequential(*blocks) - for key, task_head in task_heads: + for key, task_head in task_heads.items(): setattr(self, f"_{key}_head", task_head) self._net = {key: nn.Sequential(self._representation_stack, task_head) for key, task_head in task_heads.items()} From c440c7a5c2cb07707f783ed36412e5004339ae74 Mon Sep 17 00:00:00 2001 From: ysqyang Date: Wed, 4 Nov 2020 10:13:34 +0800 Subject: [PATCH 129/337] fixed mode setting issue in learning model --- maro/rl/models/learning_model.py | 29 ++++++++++++++++------------- 1 file changed, 16 insertions(+), 13 deletions(-) diff --git a/maro/rl/models/learning_model.py b/maro/rl/models/learning_model.py index c6fa17e1f..7e5fca8e5 100644 --- a/maro/rl/models/learning_model.py +++ b/maro/rl/models/learning_model.py @@ -14,17 +14,20 @@ class LearningModel(nn.Module): """ def __init__(self, *blocks, **task_heads): super().__init__() - self._representation_stack = nn.Sequential(*blocks) - for key, task_head in task_heads.items(): - setattr(self, f"_{key}_head", task_head) - self._net = {key: nn.Sequential(self._representation_stack, task_head) for key, task_head in task_heads.items()} + self._task_head_keys = list(task_heads.keys()) + if not self._task_head_keys: + self.net = nn.Sequential(*blocks) + else: + representation_stack = nn.Sequential(*blocks) + for key, task_head in task_heads.items(): + setattr(self, key, nn.Sequential(representation_stack, task_head)) - def forward(self, inputs, head_key=None): + def forward(self, inputs, key=None): """Feedforward computations for the given head(s). Args: inputs: Inputs to the model. - head_key: The key(s) to the head(s) from which the output is required. If this is None, the results from + key: The key(s) to the head(s) from which the output is required. If this is None, the results from all heads will be returned in the form of a dictionary. If this is a list, the results will be the outputs from the heads contained in head_key in the form of a dictionary. If this is a single key, the result will be the output from the corresponding head. @@ -32,13 +35,13 @@ def forward(self, inputs, head_key=None): Returns: Outputs from the required head(s). """ - if not self._net: - return self._representation_stack(inputs) + if not self._task_head_keys: + return self.net(inputs) - if head_key is None: - return {key: net(inputs) for key, net in self._net.items()} + if key is None: + return {key: getattr(self, key)(inputs) for key in self._task_head_keys} - if isinstance(head_key, list): - return {key: self._net[key](inputs) for key in head_key} + if isinstance(key, list): + return {k: getattr(self, k)(inputs) for k in key} else: - return self._net[head_key](inputs) + return getattr(self, key)(inputs) From ff485af71a66e31af111762bcf80ab3b5cb70436 Mon Sep 17 00:00:00 2001 From: ysqyang Date: Wed, 4 Nov 2020 16:46:51 +0800 Subject: [PATCH 130/337] fixed PR comments --- examples/cim/dqn/components/agent.py | 2 +- .../cim/dqn/components/experience_shaper.py | 2 +- maro/rl/algorithms/dqn.py | 152 +++++++++++------- 3 files changed, 97 insertions(+), 59 deletions(-) diff --git a/examples/cim/dqn/components/agent.py b/examples/cim/dqn/components/agent.py index db7f7b5f2..4fc8e15f1 100644 --- a/examples/cim/dqn/components/agent.py +++ b/examples/cim/dqn/components/agent.py @@ -34,7 +34,7 @@ def train(self): indexes, sample = self._experience_pool.sample_by_key("loss", self._batch_size) state = np.asarray(sample["state"]) action = np.asarray(sample["action"]) - reward = np.asarray(sample["reward"], dtype=np.float32) + reward = np.asarray(sample["reward"]) next_state = np.asarray(sample["next_state"]) loss = self._algorithm.train(state, action, reward, next_state) self._experience_pool.update(indexes, {"loss": loss}) diff --git a/examples/cim/dqn/components/experience_shaper.py b/examples/cim/dqn/components/experience_shaper.py index 2941f2159..4342bad00 100644 --- a/examples/cim/dqn/components/experience_shaper.py +++ b/examples/cim/dqn/components/experience_shaper.py @@ -46,4 +46,4 @@ def _compute_reward(self, decision_event, snapshot_list): tot_fulfillment = np.dot(future_fulfillment, decay_list) tot_shortage = np.dot(future_shortage, decay_list) - return np.float(self._fulfillment_factor * tot_fulfillment - self._shortage_factor * tot_shortage) + return np.float32(self._fulfillment_factor * tot_fulfillment - self._shortage_factor * tot_shortage) diff --git a/maro/rl/algorithms/dqn.py b/maro/rl/algorithms/dqn.py index ae2a7cdc5..594f97dfc 100644 --- a/maro/rl/algorithms/dqn.py +++ b/maro/rl/algorithms/dqn.py @@ -1,7 +1,7 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT license. -from enum import Enum +import warnings import numpy as np import torch @@ -11,11 +11,6 @@ from maro.utils import clone -class DuelingQModelHead(Enum): - STATE_VALUE = "state_value" - ADVANTAGE = "advantage" - - class DQNHyperParams: """Hyper-parameter set for the DQN algorithm. @@ -26,10 +21,13 @@ class DQNHyperParams: tau (float): Soft update coefficient, i.e., target_model = tau * eval_model + (1-tau) * target_model. is_double (bool): If True, the next Q values will be computed according to the double DQN algorithm. See https://arxiv.org/pdf/1509.06461.pdf for details. Defaults to False. + is_dueling (bool): If True, the dueling Q-value model is used to compute Q values. advantage_mode (str): advantage mode for the dueling Q-value model. Defaults to None, in which case it is assumed that the regular Q-value model is used. """ - __slots__ = ["num_actions", "reward_decay", "target_update_frequency", "tau", "is_double", "advantage_mode"] + __slots__ = [ + "num_actions", "reward_decay", "target_update_frequency", "tau", "is_double", "is_dueling", "advantage_mode" + ] def __init__( self, @@ -38,6 +36,7 @@ def __init__( target_update_frequency: int, tau: float = 1.0, is_double: bool = False, + is_dueling: bool = False, advantage_mode: str = None ): self.num_actions = num_actions @@ -45,6 +44,7 @@ def __init__( self.target_update_frequency = target_update_frequency self.tau = tau self.is_double = is_double + self.is_dueling = is_dueling self.advantage_mode = advantage_mode @@ -54,79 +54,117 @@ class DQN(AbsAlgorithm): See https://web.stanford.edu/class/psych209/Readings/MnihEtAlHassibis15NatureControlDeepRL.pdf for details. Args: - eval_model (nn.Module): Q-value model for given states and actions. - optimizer_cls: Torch optimizer class for the eval model. If this is None, the eval model is not trainable. - optimizer_params: Parameters required for the eval optimizer class. + value_model (nn.Module): Q-value or state value model depending on whether ``is_dueling`` is true + in ``hyper-params``. + value_optimizer_cls: Torch optimizer class for the value model. If this is None, the eval model is not + trainable. + value_optimizer_params: Parameters required for the optimizer class for the value model. loss_func (Callable): Loss function for the value model. hyper_params: Hyper-parameter set for the DQN algorithm. - target_model (nn.Module): Q-value model to train the ``eval_model`` against and to be updated periodically. If - it is None, the target model will be initialized as a deep copy of the eval model. + advantage_model (nn.Module): Model that estimates the advantage value. If ``is_dueling`` is true + in ``hyper-params``, Defaults to None. + advantage_optimizer_cls: Torch optimizer class for the advantage model. If this is None, the eval model is + not trainable. + advantage_optimizer_params: Parameters required for the optimizer class for the advantage model """ def __init__( self, - eval_model: nn.Module, - optimizer_cls, - optimizer_params, + value_model: nn.Module, + value_optimizer_cls, + value_optimizer_params, loss_func, hyper_params: DQNHyperParams, - target_model=None + advantage_model: nn.Module = None, + advantage_optimizer_cls=None, + advantage_optimizer_params=None ): super().__init__() self._device = torch.device("cuda" if torch.cuda.is_available() else "cpu") - self._model_dict = {"eval": eval_model.to(self._device)} - if optimizer_cls is not None: - self._optimizer = optimizer_cls(self._model_dict["eval"].parameters(), **optimizer_params) - if target_model is None: - self._model_dict["target"] = clone(eval_model).to(self._device) - else: - self._model_dict["target"] = target_model.to(self._device) - # No gradient computation required for the target model - for param in self._model_dict["target"].parameters(): - param.requires_grad = False + self._hyper_params = hyper_params + self._is_training = value_optimizer_cls is not None + if self._hyper_params.is_dueling: + assert advantage_model is not None, "advantage_model cannot be None under dueling mode." + if self._is_training: + assert advantage_optimizer_cls is not None, \ + "advantage_optimizer_cls cannot be None under dueling mode." + + self._model_dict = { + "state_value": value_model.to(self._device), + "state_value_target": clone(value_model).to(self._device) if self._is_training else None, + "advantage": advantage_model.to(self._device), + "advantage_target": clone(advantage_model).to(self._device) if self._is_training else None + } + self._value_optimizer = value_optimizer_cls( + self._model_dict["state_value"].parameters(), **value_optimizer_params + ) + self._advantage_optimizer = advantage_optimizer_cls( + self._model_dict["advantage"].parameters(), **advantage_optimizer_params + ) + # No gradient computation required for the target models + for param in self._model_dict["state_value_target"].parameters(): + param.requires_grad = False + for param in self._model_dict["advantage_target"].parameters(): + param.requires_grad = False + else: + self._model_dict = { + "q_value": value_model.to(self._device), + "q_value_target": clone(value_model).to(self._device) if self._is_training else None, + } + self._optimizer = value_optimizer_cls( + self._model_dict["q_value"].parameters(), **value_optimizer_params + ) self._loss_func = loss_func - self._hyper_params = hyper_params self._train_cnt = 0 @property def eval_model(self): return self._model_dict["eval"] + @property + def is_training(self): + return self._is_training + def choose_action(self, state: np.ndarray, epsilon: float = None): if epsilon is None or np.random.rand() > epsilon: state = torch.from_numpy(state).unsqueeze(0) self._model_dict["eval"].eval() with torch.no_grad(): - q_values = self._get_q_values("eval", state) + q_values = self._get_q_values(state) return q_values.argmax(dim=1).item() return np.random.choice(self._hyper_params.num_actions) def train(self, states: np.ndarray, actions: np.ndarray, rewards: np.ndarray, next_states: np.ndarray): - if hasattr(self, "_optimizer"): - states = torch.from_numpy(states).to(self._device) # (N, state_dim) - actions = torch.from_numpy(actions).to(self._device) # (N,) - rewards = torch.from_numpy(rewards).to(self._device) # (N,) - next_states = torch.from_numpy(next_states).to(self._device) # (N, state_dim) - if len(actions.shape) == 1: - actions = actions.unsqueeze(1) # (N, 1) - current_q_values_all = self._get_q_values("eval", states) - current_q_values = current_q_values_all.gather(1, actions).squeeze(1) # (N,) - next_q_values = self._get_next_q_values(current_q_values_all, next_states) # (N,) - target_q_values = (rewards + self._hyper_params.reward_decay * next_q_values).detach() # (N,) - loss = self._loss_func(current_q_values, target_q_values) - self._model_dict["eval"].train() - self._optimizer.zero_grad() - loss.backward() - self._optimizer.step() - self._train_cnt += 1 - if self._train_cnt % self._hyper_params.target_update_frequency == 0: - self._update_target_model() - - return np.abs((current_q_values - target_q_values).detach().numpy()) + if not self._is_training(): + warnings.warn( + "DQN is not in training mode since no optimizer is provided. Did you provide optimizer_cls and " + "optimizer_params when instantiating the algorithm?" + ) + + states = torch.from_numpy(states).to(self._device) # (N, state_dim) + actions = torch.from_numpy(actions).to(self._device) # (N,) + rewards = torch.from_numpy(rewards).to(self._device) # (N,) + next_states = torch.from_numpy(next_states).to(self._device) # (N, state_dim) + if len(actions.shape) == 1: + actions = actions.unsqueeze(1) # (N, 1) + current_q_values_all = self._get_q_values(states) + current_q_values = current_q_values_all.gather(1, actions).squeeze(1) # (N,) + next_q_values = self._get_next_q_values(current_q_values_all, next_states) # (N,) + target_q_values = (rewards + self._hyper_params.reward_decay * next_q_values).detach() # (N,) + loss = self._loss_func(current_q_values, target_q_values) + self._model_dict["eval"].train() + self._optimizer.zero_grad() + loss.backward() + self._optimizer.step() + self._train_cnt += 1 + if self._train_cnt % self._hyper_params.target_update_frequency == 0: + self._update_target_model() + + return np.abs((current_q_values - target_q_values).detach().numpy()) def _update_target_model(self): - if hasattr(self, "_optimizer"): + if self._is_training(): for eval_params, target_params in zip( self._model_dict["eval"].parameters(), self._model_dict["target"].parameters() ): @@ -134,12 +172,12 @@ def _update_target_model(self): self._hyper_params.tau * eval_params.data + (1 - self._hyper_params.tau) * target_params.data ) - def _get_q_values(self, which: str, states): - if self._hyper_params.advantage_mode is None: - return self._model_dict[which](states) + def _get_q_values(self, states, is_target: bool = False): + if not self._hyper_params.is_dueling: + return self._model_dict["q_value_target" if is_target else "q_value"](states) - state_values = self._model_dict[which](states, head_key=DuelingQModelHead.STATE_VALUE.value) - advantages = self._model_dict[which](states, head_key=DuelingQModelHead.ADVANTAGE.value) + state_values = self._model_dict["state_value_target" if is_target else "state_value"](states) + advantages = self._model_dict["advantage_target" if is_target else "advantage"](states) # Use mean or max correction to address the identifiability issue corrections = advantages.mean(1) if self._hyper_params.advantage_mode == "mean" else advantages.max(1)[0] q_values = state_values + advantages - corrections.unsqueeze(1) @@ -148,9 +186,9 @@ def _get_q_values(self, which: str, states): def _get_next_q_values(self, current_q_values_all, states): if self._hyper_params.is_double: actions = current_q_values_all.max(dim=1)[1].unsqueeze(1) - return self._get_q_values("target", states).gather(1, actions).squeeze(1) # (N,) + return self._get_q_values(states, is_target=True).gather(1, actions).squeeze(1) # (N,) else: - return self._get_q_values("target", states).max(dim=1)[0] # (N,) + return self._get_q_values(states, is_target=True).max(dim=1)[0] # (N,) def load_models(self, eval_model): """Load the eval model from memory.""" From 8692b893d0204ae9d622efcd02d91201e3b5f5b6 Mon Sep 17 00:00:00 2001 From: ysqyang Date: Wed, 4 Nov 2020 17:00:43 +0800 Subject: [PATCH 131/337] revised cim example according to DQN changes --- examples/cim/dqn/components/agent_manager.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/examples/cim/dqn/components/agent_manager.py b/examples/cim/dqn/components/agent_manager.py index 2ac9aa686..4d35b63e0 100644 --- a/examples/cim/dqn/components/agent_manager.py +++ b/examples/cim/dqn/components/agent_manager.py @@ -25,9 +25,9 @@ def create_dqn_agents(agent_id_list, config): ) algorithm = DQN( - eval_model=eval_model, - optimizer_cls=RMSprop, - optimizer_params=config.algorithm.optimizer, + value_model=eval_model, + value_optimizer_cls=RMSprop, + value_optimizer_params=config.algorithm.optimizer, loss_func=nn.functional.smooth_l1_loss, hyper_params=DQNHyperParams( **config.algorithm.hyper_parameters, From 5d0e2c7606ebb035051cedf147a40fe9646cc0a9 Mon Sep 17 00:00:00 2001 From: ysqyang Date: Wed, 4 Nov 2020 17:10:39 +0800 Subject: [PATCH 132/337] renamed eval_model to q_value_model in cim example --- examples/cim/dqn/components/agent_manager.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/cim/dqn/components/agent_manager.py b/examples/cim/dqn/components/agent_manager.py index 4d35b63e0..03782ffa5 100644 --- a/examples/cim/dqn/components/agent_manager.py +++ b/examples/cim/dqn/components/agent_manager.py @@ -14,7 +14,7 @@ def create_dqn_agents(agent_id_list, config): set_seeds(config.seed) agent_dict = {} for agent_id in agent_id_list: - eval_model = LearningModel( + q_value_model = LearningModel( FullyConnectedNet( name=f'{agent_id}.policy', input_dim=config.algorithm.input_dim, @@ -25,7 +25,7 @@ def create_dqn_agents(agent_id_list, config): ) algorithm = DQN( - value_model=eval_model, + value_model=q_value_model, value_optimizer_cls=RMSprop, value_optimizer_params=config.algorithm.optimizer, loss_func=nn.functional.smooth_l1_loss, From f93049729f24884e8359bd6653f745940d3c2368 Mon Sep 17 00:00:00 2001 From: ysqyang Date: Wed, 4 Nov 2020 17:48:09 +0800 Subject: [PATCH 133/337] more fixes --- maro/rl/__init__.py | 2 +- maro/rl/algorithms/dqn.py | 78 +++++++++++++++++++++++++++------------ 2 files changed, 56 insertions(+), 24 deletions(-) diff --git a/maro/rl/__init__.py b/maro/rl/__init__.py index 3c3348d9a..eea2d84d9 100644 --- a/maro/rl/__init__.py +++ b/maro/rl/__init__.py @@ -9,7 +9,7 @@ from maro.rl.agent.abs_agent_manager import AbsAgentManager, AgentManagerMode from maro.rl.agent.simple_agent_manager import SimpleAgentManager from maro.rl.algorithms.abs_algorithm import AbsAlgorithm -from maro.rl.algorithms.dqn import DQN, DQNHyperParams, DuelingQModelHead +from maro.rl.algorithms.dqn import DQN, DQNHyperParams from maro.rl.models.learning_model import LearningModel from maro.rl.models.fc_net import FullyConnectedNet from maro.rl.storage.abs_store import AbsStore diff --git a/maro/rl/algorithms/dqn.py b/maro/rl/algorithms/dqn.py index 594f97dfc..6536e5a47 100644 --- a/maro/rl/algorithms/dqn.py +++ b/maro/rl/algorithms/dqn.py @@ -117,10 +117,6 @@ def __init__( self._loss_func = loss_func self._train_cnt = 0 - @property - def eval_model(self): - return self._model_dict["eval"] - @property def is_training(self): return self._is_training @@ -128,7 +124,11 @@ def is_training(self): def choose_action(self, state: np.ndarray, epsilon: float = None): if epsilon is None or np.random.rand() > epsilon: state = torch.from_numpy(state).unsqueeze(0) - self._model_dict["eval"].eval() + if self._hyper_params.is_dueling: + self._model_dict["state_value"].eval() + self._model_dict["advantage"].eval() + else: + self._model_dict["q_value"].eval() with torch.no_grad(): q_values = self._get_q_values(state) return q_values.argmax(dim=1).item() @@ -153,24 +153,52 @@ def train(self, states: np.ndarray, actions: np.ndarray, rewards: np.ndarray, ne next_q_values = self._get_next_q_values(current_q_values_all, next_states) # (N,) target_q_values = (rewards + self._hyper_params.reward_decay * next_q_values).detach() # (N,) loss = self._loss_func(current_q_values, target_q_values) - self._model_dict["eval"].train() - self._optimizer.zero_grad() - loss.backward() - self._optimizer.step() + + if self._hyper_params.is_dueling: + self._model_dict["state_value"].train() + self._model_dict["advantage"].train() + self._value_optimizer.zero_grad() + self._advantage_optimizer.zero_grad() + loss.backward() + self._value_optimizer.step() + self._advantage_optimizer.step() + else: + self._model_dict["q_value"].train() + self._optimizer.zero_grad() + loss.backward() + self._optimizer.step() + self._train_cnt += 1 if self._train_cnt % self._hyper_params.target_update_frequency == 0: - self._update_target_model() + self._update_targets() return np.abs((current_q_values - target_q_values).detach().numpy()) - def _update_target_model(self): + def _update_targets(self): if self._is_training(): - for eval_params, target_params in zip( - self._model_dict["eval"].parameters(), self._model_dict["target"].parameters() - ): - target_params.data = ( - self._hyper_params.tau * eval_params.data + (1 - self._hyper_params.tau) * target_params.data - ) + if not self._hyper_params.is_dueling: + for eval_params, target_params in zip( + self._model_dict["q_value"].parameters(), self._model_dict["q_value_target"].parameters() + ): + target_params.data = ( + self._hyper_params.tau * eval_params.data + + (1 - self._hyper_params.tau) * target_params.data + ) + else: + for eval_params, target_params in zip( + self._model_dict["state_value"].parameters(), self._model_dict["state_value_target"].parameters() + ): + target_params.data = ( + self._hyper_params.tau * eval_params.data + + (1 - self._hyper_params.tau) * target_params.data + ) + for eval_params, target_params in zip( + self._model_dict["advantage"].parameters(), self._model_dict["advantage_target"].parameters() + ): + target_params.data = ( + self._hyper_params.tau * eval_params.data + + (1 - self._hyper_params.tau) * target_params.data + ) def _get_q_values(self, states, is_target: bool = False): if not self._hyper_params.is_dueling: @@ -190,18 +218,22 @@ def _get_next_q_values(self, current_q_values_all, states): else: return self._get_q_values(states, is_target=True).max(dim=1)[0] # (N,) - def load_models(self, eval_model): - """Load the eval model from memory.""" - self._model_dict["eval"].load_state_dict(eval_model) + def _get_state_dicts(self): + return {k: model.state_dict() for k, model in self._model_dict.items()} + + def load_models(self, model_dict): + """Load models from memory.""" + for key in self._model_dict: + self._model_dict[key].load_state_dict(model_dict[key]) def dump_models(self): """Return the eval model.""" - return self._model_dict["eval"].state_dict() + return self._get_state_dicts() def load_models_from_file(self, path): """Load the eval model from disk.""" - self._model_dict["eval"] = torch.load(path) + self._model_dict = torch.load(path) def dump_models_to_file(self, path: str): """Dump the eval model to disk.""" - torch.save(self._model_dict["eval"].state_dict(), path) + torch.save(self._get_state_dicts(), path) From 2a91c4b2adb410d292a9aef1736908020ed52593 Mon Sep 17 00:00:00 2001 From: ysqyang Date: Wed, 4 Nov 2020 17:50:27 +0800 Subject: [PATCH 134/337] fixed a bug --- maro/rl/algorithms/dqn.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/maro/rl/algorithms/dqn.py b/maro/rl/algorithms/dqn.py index 6536e5a47..209094f12 100644 --- a/maro/rl/algorithms/dqn.py +++ b/maro/rl/algorithms/dqn.py @@ -136,7 +136,7 @@ def choose_action(self, state: np.ndarray, epsilon: float = None): return np.random.choice(self._hyper_params.num_actions) def train(self, states: np.ndarray, actions: np.ndarray, rewards: np.ndarray, next_states: np.ndarray): - if not self._is_training(): + if not self._is_training: warnings.warn( "DQN is not in training mode since no optimizer is provided. Did you provide optimizer_cls and " "optimizer_params when instantiating the algorithm?" @@ -175,7 +175,7 @@ def train(self, states: np.ndarray, actions: np.ndarray, rewards: np.ndarray, ne return np.abs((current_q_values - target_q_values).detach().numpy()) def _update_targets(self): - if self._is_training(): + if self._is_training: if not self._hyper_params.is_dueling: for eval_params, target_params in zip( self._model_dict["q_value"].parameters(), self._model_dict["q_value_target"].parameters() From 81a89b3f76424fc337820e6ea3484dc10108471b Mon Sep 17 00:00:00 2001 From: ysqyang Date: Wed, 4 Nov 2020 18:00:28 +0800 Subject: [PATCH 135/337] fixed a bug --- maro/rl/algorithms/dqn.py | 1 + 1 file changed, 1 insertion(+) diff --git a/maro/rl/algorithms/dqn.py b/maro/rl/algorithms/dqn.py index 209094f12..10ae29901 100644 --- a/maro/rl/algorithms/dqn.py +++ b/maro/rl/algorithms/dqn.py @@ -141,6 +141,7 @@ def train(self, states: np.ndarray, actions: np.ndarray, rewards: np.ndarray, ne "DQN is not in training mode since no optimizer is provided. Did you provide optimizer_cls and " "optimizer_params when instantiating the algorithm?" ) + return states = torch.from_numpy(states).to(self._device) # (N, state_dim) actions = torch.from_numpy(actions).to(self._device) # (N,) From 2568982e10ccfa3829292259bfc758a6f64ee5f8 Mon Sep 17 00:00:00 2001 From: ysqyang Date: Wed, 4 Nov 2020 22:14:13 +0800 Subject: [PATCH 136/337] added doc per PR comments --- maro/rl/algorithms/dqn.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/maro/rl/algorithms/dqn.py b/maro/rl/algorithms/dqn.py index 10ae29901..bb512c02b 100644 --- a/maro/rl/algorithms/dqn.py +++ b/maro/rl/algorithms/dqn.py @@ -19,10 +19,12 @@ class DQNHyperParams: reward_decay (float): Reward decay as defined in standard RL terminology. target_update_frequency (int): Number of training rounds between target model updates. tau (float): Soft update coefficient, i.e., target_model = tau * eval_model + (1-tau) * target_model. - is_double (bool): If True, the next Q values will be computed according to the double DQN algorithm. - See https://arxiv.org/pdf/1509.06461.pdf for details. Defaults to False. - is_dueling (bool): If True, the dueling Q-value model is used to compute Q values. - advantage_mode (str): advantage mode for the dueling Q-value model. Defaults to None, in which + is_double (bool): If True, the next Q values will be computed according to the double DQN algorithm, + i.e., q_next = Q_target(s, argmax(Q_eval(s, a))). Otherwise, q_next = max(Q_target(s, a)), per ordinary + DQN. See https://arxiv.org/pdf/1509.06461.pdf for details. Defaults to False. + is_dueling (bool): If True, the Q values will be computed using a dueling architecture with a head for + state values and a head for advantages. See https://arxiv.org/pdf/1511.06581.pdf for details. + advantage_mode (str): advantage mode for the dueling architecture. Defaults to None, in which case it is assumed that the regular Q-value model is used. """ __slots__ = [ From 0eaa00d47f9513ebc84f9a04eeb6cbc2ce8f2b5e Mon Sep 17 00:00:00 2001 From: ysqyang Date: Thu, 5 Nov 2020 09:35:47 +0800 Subject: [PATCH 137/337] removed learner.exit() in single_process_launcher --- examples/cim/dqn/single_process_launcher.py | 1 - 1 file changed, 1 deletion(-) diff --git a/examples/cim/dqn/single_process_launcher.py b/examples/cim/dqn/single_process_launcher.py index ca420f00e..a74701459 100644 --- a/examples/cim/dqn/single_process_launcher.py +++ b/examples/cim/dqn/single_process_launcher.py @@ -67,7 +67,6 @@ def launch(config): ) learner.test() learner.dump_models(os.path.join(os.getcwd(), "models")) - learner.exit() if __name__ == "__main__": From cb86fd923af47a7b1ca59ad401693f99e69b3443 Mon Sep 17 00:00:00 2001 From: ysqyang Date: Thu, 5 Nov 2020 09:36:59 +0800 Subject: [PATCH 138/337] removed learner.exit() in single_process_launcher --- examples/cim/dqn/single_process_launcher.py | 1 - 1 file changed, 1 deletion(-) diff --git a/examples/cim/dqn/single_process_launcher.py b/examples/cim/dqn/single_process_launcher.py index ca420f00e..a74701459 100644 --- a/examples/cim/dqn/single_process_launcher.py +++ b/examples/cim/dqn/single_process_launcher.py @@ -67,7 +67,6 @@ def launch(config): ) learner.test() learner.dump_models(os.path.join(os.getcwd(), "models")) - learner.exit() if __name__ == "__main__": From c9f3e66b0cd6777bdfe619d07bcb966fedb0fff1 Mon Sep 17 00:00:00 2001 From: ysqyang Date: Thu, 5 Nov 2020 19:04:36 +0800 Subject: [PATCH 139/337] fixed PR comments --- maro/rl/__init__.py | 9 ++--- maro/rl/models/{fc_net.py => fc_block.py} | 41 ++++++++-------------- maro/rl/models/learning_model.py | 42 ++++++++++++++++------- 3 files changed, 49 insertions(+), 43 deletions(-) rename maro/rl/models/{fc_net.py => fc_block.py} (72%) diff --git a/maro/rl/__init__.py b/maro/rl/__init__.py index eea2d84d9..40dc10a48 100644 --- a/maro/rl/__init__.py +++ b/maro/rl/__init__.py @@ -10,8 +10,8 @@ from maro.rl.agent.simple_agent_manager import SimpleAgentManager from maro.rl.algorithms.abs_algorithm import AbsAlgorithm from maro.rl.algorithms.dqn import DQN, DQNHyperParams -from maro.rl.models.learning_model import LearningModel -from maro.rl.models.fc_net import FullyConnectedNet +from maro.rl.models.learning_model import MultiHeadLearningModel, SingleHeadLearningModel +from maro.rl.models.fc_block import FullyConnectedBlock from maro.rl.storage.abs_store import AbsStore from maro.rl.storage.column_based_store import ColumnBasedStore from maro.rl.storage.utils import OverwriteType @@ -41,8 +41,9 @@ "AbsAlgorithm", "DQN", "DQNHyperParams", - "LearningModel", - "FullyConnectedNet", + "MultiHeadLearningModel", + "SingleHeadLearningModel", + "FullyConnectedBlock", "AbsStore", "ColumnBasedStore", "OverwriteType", diff --git a/maro/rl/models/fc_net.py b/maro/rl/models/fc_block.py similarity index 72% rename from maro/rl/models/fc_net.py rename to maro/rl/models/fc_block.py index 5c35731e3..9cef4e82e 100644 --- a/maro/rl/models/fc_net.py +++ b/maro/rl/models/fc_block.py @@ -7,10 +7,8 @@ import torch.nn as nn -class FullyConnectedNet(nn.Module): - """NN model to compute state or action values. - - Fully connected network with optional batch normalization, activation and dropout components. +class FullyConnectedBlock(nn.Module): + """Fully connected network with optional batch normalization, activation and dropout components. Args: name (str): Network name. @@ -18,7 +16,7 @@ class FullyConnectedNet(nn.Module): output_dim (int): Network output dimension. hidden_dims ([int]): Dimensions of hidden layers. Its length is the number of hidden layers. activation: A ``torch.nn`` activation type. If None, there will be no activation. Defaults to LeakyReLU. - is_top (bool): If true, this block will be the top block of the full model and the top layer of this block + is_head (bool): If true, this block will be the top block of the full model and the top layer of this block will be the final output layer. Defaults to False. softmax_enabled (bool): If true, the output of the net will be a softmax transformation of the top layer's output. Defaults to False. @@ -26,7 +24,7 @@ class FullyConnectedNet(nn.Module): skip_connection_enabled (bool): If true, a skip connection will be built between the bottom (input) layer and top (output) layer. Defaults to False. dropout_p (float): Dropout probability. Defaults to None, in which case there is no drop-out. - clip_value (float): Gradient clipping threshold. Defaults to None, in which case not gradient clipping is + gradient_threshold (float): Gradient clipping threshold. Defaults to None, in which case not gradient clipping is performed. """ def __init__( @@ -36,12 +34,12 @@ def __init__( output_dim: int, hidden_dims: [int], activation=nn.LeakyReLU, - is_top: bool = False, + is_head: bool = False, softmax_enabled: bool = False, batch_norm_enabled: bool = False, skip_connection_enabled: bool = False, dropout_p: float = None, - clip_value: float = None + gradient_threshold: float = None ): super().__init__() self._name = name @@ -51,7 +49,7 @@ def __init__( # network features self._activation = activation - self._is_top = is_top + self._is_head = is_head self._softmax = nn.Softmax(dim=1) if softmax_enabled else None self._batch_norm_enabled = batch_norm_enabled self._dropout_p = dropout_p @@ -67,16 +65,15 @@ def __init__( # build the net dims = [self._input_dim] + self._hidden_dims layers = [self._build_internal_layer(in_dim, out_dim) for in_dim, out_dim in zip(dims, dims[1:])] - if is_top: - layers.append(self._build_top_layer(dims[-1], self._output_dim)) - else: - layers.append(self._build_internal_layer(dims[-1], self._output_dim)) + # top layer + layers.append(self._build_layer(dims[-1], self._output_dim, is_head=self._is_head)) self._net = nn.Sequential(*layers) - if clip_value is not None: + self._gradient_threshold = gradient_threshold + if gradient_threshold is not None: for param in self._net.parameters(): - param.register_hook(lambda grad: torch.clamp(grad, -clip_value, clip_value)) + param.register_hook(lambda grad: torch.clamp(grad, -gradient_threshold, gradient_threshold)) def forward(self, x): out = self._net(x) @@ -96,7 +93,7 @@ def input_dim(self): def output_dim(self): return self._output_dim - def _build_internal_layer(self, input_dim, output_dim): + def _build_layer(self, input_dim, output_dim, is_head: bool = False): """Build basic layer. BN -> Linear -> Activation -> Dropout @@ -105,16 +102,8 @@ def _build_internal_layer(self, input_dim, output_dim): if self._batch_norm_enabled: components.append(("batch_norm", nn.BatchNorm1d(input_dim))) components.append(("linear", nn.Linear(input_dim, output_dim))) - if self._activation is not None: + if not is_head and self._activation is not None: components.append(("activation", self._activation())) - if self._dropout_p: + if not is_head and self._dropout_p: components.append(("dropout", nn.Dropout(p=self._dropout_p))) return nn.Sequential(OrderedDict(components)) - - def _build_top_layer(self, input_dim, output_dim): - """The output layer may need to be treated differently.""" - components = [] - if self._batch_norm_enabled: - components.append(("batch_norm", nn.BatchNorm1d(input_dim))) - components.append(("linear", nn.Linear(input_dim, output_dim))) - return nn.Sequential(OrderedDict(components)) diff --git a/maro/rl/models/learning_model.py b/maro/rl/models/learning_model.py index 7e5fca8e5..30f12edfd 100644 --- a/maro/rl/models/learning_model.py +++ b/maro/rl/models/learning_model.py @@ -4,23 +4,42 @@ import torch.nn as nn -class LearningModel(nn.Module): - """NN model that consists of multiple shared blocks and multiple task heads. +class SingleHeadLearningModel(nn.Module): + """NN model that consists of shared blocks and multiple task heads. + + The shared blocks must be chainable, i.e., the output dimension of a block must match the input dimension of + its successor. + """ + def __init__(self, block_list: list): + super().__init__() + self._net = nn.Sequential(*block_list) + + def forward(self, inputs): + """Feedforward computation. + + Args: + inputs: Inputs to the model. + + Returns: + Outputs from the model. + """ + return self._net(inputs) + + +class MultiHeadLearningModel(nn.Module): + """NN model that consists of shared blocks and multiple task heads. The shared blocks must be chainable, i.e., the output dimension of a block must match the input dimension of its successor. Heads must be provided in the form of keyword arguments. If at least one head is provided, the output of the model will be a dictionary with the names of the heads as keys and the corresponding head outputs as values. Otherwise, the output will be the output of the last block. """ - def __init__(self, *blocks, **task_heads): + def __init__(self, shared_block_list: list, task_head_block_dict: dict): super().__init__() - self._task_head_keys = list(task_heads.keys()) - if not self._task_head_keys: - self.net = nn.Sequential(*blocks) - else: - representation_stack = nn.Sequential(*blocks) - for key, task_head in task_heads.items(): - setattr(self, key, nn.Sequential(representation_stack, task_head)) + self._task_head_keys = list(task_head_block_dict.keys()) + shared_stack = nn.Sequential(*shared_block_list) + for key, head in task_head_block_dict.items(): + setattr(self, key, nn.Sequential(shared_stack, head)) def forward(self, inputs, key=None): """Feedforward computations for the given head(s). @@ -35,9 +54,6 @@ def forward(self, inputs, key=None): Returns: Outputs from the required head(s). """ - if not self._task_head_keys: - return self.net(inputs) - if key is None: return {key: getattr(self, key)(inputs) for key in self._task_head_keys} From 29f601d8ced6c39e617391d84e9553b5770ce05d Mon Sep 17 00:00:00 2001 From: ysqyang Date: Thu, 5 Nov 2020 23:21:45 +0800 Subject: [PATCH 140/337] fixed rl/__init__ --- maro/rl/__init__.py | 68 +++++++++++++++++++++++---------------------- 1 file changed, 35 insertions(+), 33 deletions(-) diff --git a/maro/rl/__init__.py b/maro/rl/__init__.py index 6dda81bfe..c26fb843c 100644 --- a/maro/rl/__init__.py +++ b/maro/rl/__init__.py @@ -13,7 +13,9 @@ ) from maro.rl.dist_topologies.single_learner_multi_actor_sync_mode import ActorProxy, ActorWorker from maro.rl.early_stopping.abs_early_stopping_checker import AbsEarlyStoppingChecker -from maro.rl.early_stopping.simple_early_stopping_checker import MaxDeltaEarlyStoppingChecker, RSDEarlyStoppingChecker +from maro.rl.early_stopping.simple_early_stopping_checker import ( + MaxDeltaEarlyStoppingChecker, RSDEarlyStoppingChecker +) from maro.rl.explorer.abs_explorer import AbsExplorer from maro.rl.explorer.simple_explorer import LinearExplorer, TwoPhaseLinearExplorer from maro.rl.learner.abs_learner import AbsLearner @@ -30,36 +32,36 @@ from maro.rl.storage.utils import OverwriteType __all__ = [ - "AbsActor", - "SimpleActor", - "AbsLearner", - "SimpleLearner", - "AbsAgent", - "AbsAgentManager", - "AgentManagerMode", - "SimpleAgentManager", - "AbsAlgorithm", - "DQN", - "DQNHyperParams", - "MultiHeadLearningModel", - "SingleHeadLearningModel", - "FullyConnectedBlock", - "AbsStore", - "ColumnBasedStore", - "OverwriteType", - "AbsShaper", - "StateShaper", - "ActionShaper", - "ExperienceShaper", - "KStepExperienceShaper", - "AbsExplorer", - "LinearExplorer", - "TwoPhaseLinearExplorer", - "AbsEarlyStoppingChecker", - "RSDEarlyStoppingChecker", - "MaxDeltaEarlyStoppingChecker", - "ActorProxy", - "ActorWorker", - "concat_experiences_by_agent", - "merge_experiences_with_trajectory_boundaries" + 'AbsActor', + 'AbsAgent', + 'AbsAgentManager', + 'AbsAlgorithm', + 'AbsEarlyStoppingChecker', + 'AbsExplorer', + 'AbsLearner', + 'AbsShaper', + 'AbsStore', + 'ActionShaper', + 'ActorProxy', + 'ActorWorker', + 'AgentManagerMode', + 'ColumnBasedStore', + 'DQN', + 'DQNHyperParams', + 'ExperienceShaper', + 'FullyConnectedBlock', + 'KStepExperienceShaper', + 'LinearExplorer', + 'MaxDeltaEarlyStoppingChecker', + 'MultiHeadLearningModel', + 'OverwriteType', + 'RSDEarlyStoppingChecker', + 'SimpleActor', + 'SimpleAgentManager', + 'SimpleLearner', + 'SingleHeadLearningModel', + 'StateShaper', + 'TwoPhaseLinearExplorer', + 'concat_experiences_by_agent', + 'merge_experiences_with_trajectory_boundaries' ] From b0892ac706650e9c5d6ab2211643ed3f09492596 Mon Sep 17 00:00:00 2001 From: ysqyang Date: Thu, 5 Nov 2020 23:26:30 +0800 Subject: [PATCH 141/337] fixed issues in example --- examples/cim/dqn/components/agent_manager.py | 10 ++++++---- examples/cim/dqn/dist_actor.py | 13 +++++++------ examples/cim/dqn/dist_learner.py | 6 +++--- 3 files changed, 16 insertions(+), 13 deletions(-) diff --git a/examples/cim/dqn/components/agent_manager.py b/examples/cim/dqn/components/agent_manager.py index 9a3aac048..57436912a 100644 --- a/examples/cim/dqn/components/agent_manager.py +++ b/examples/cim/dqn/components/agent_manager.py @@ -4,7 +4,9 @@ import torch.nn as nn from torch.optim import RMSprop -from maro.rl import DQN, ColumnBasedStore, DQNHyperParams, FullyConnectedNet, LearningModel, SimpleAgentManager +from maro.rl import ( + DQN, ColumnBasedStore, DQNHyperParams, FullyConnectedBlock, SingleHeadLearningModel, SimpleAgentManager +) from maro.utils import set_seeds from .agent import CIMAgent @@ -15,14 +17,14 @@ def create_dqn_agents(agent_id_list, config): set_seeds(config.seed) agent_dict = {} for agent_id in agent_id_list: - eval_model = LearningModel( - FullyConnectedNet( + eval_model = SingleHeadLearningModel( + [FullyConnectedBlock( name=f'{agent_id}.policy', input_dim=config.algorithm.input_dim, output_dim=num_actions, activation=nn.LeakyReLU, **config.algorithm.model - ) + )] ) algorithm = DQN( diff --git a/examples/cim/dqn/dist_actor.py b/examples/cim/dqn/dist_actor.py index 2213d2af3..d8b5d1eeb 100644 --- a/examples/cim/dqn/dist_actor.py +++ b/examples/cim/dqn/dist_actor.py @@ -5,16 +5,16 @@ import numpy as np +from components.action_shaper import CIMActionShaper +from components.agent_manager import DQNAgentManager, create_dqn_agents +from components.config import set_input_dim +from components.experience_shaper import TruncatedExperienceShaper +from components.state_shaper import CIMStateShaper + from maro.rl import ActorWorker, AgentManagerMode, KStepExperienceShaper, SimpleActor from maro.simulator import Env from maro.utils import convert_dottable -from .components.action_shaper import CIMActionShaper -from .components.agent_manager import DQNAgentManager, create_dqn_agents -from .components.config import config, set_input_dim -from .components.experience_shaper import TruncatedExperienceShaper -from .components.state_shaper import CIMStateShaper - def launch(config): set_input_dim(config) @@ -52,4 +52,5 @@ def launch(config): if __name__ == "__main__": + from components.config import config launch(config) diff --git a/examples/cim/dqn/dist_learner.py b/examples/cim/dqn/dist_learner.py index 78e3761b8..8e970f0b6 100644 --- a/examples/cim/dqn/dist_learner.py +++ b/examples/cim/dqn/dist_learner.py @@ -3,6 +3,9 @@ import os +from components.agent_manager import DQNAgentManager, create_dqn_agents +from components.config import set_input_dim + from maro.rl import ( ActorProxy, AgentManagerMode, MaxDeltaEarlyStoppingChecker, SimpleLearner, TwoPhaseLinearExplorer, concat_experiences_by_agent @@ -10,9 +13,6 @@ from maro.simulator import Env from maro.utils import Logger, convert_dottable -from .components.agent_manager import DQNAgentManager, create_dqn_agents -from .components.config import set_input_dim - def launch(config): set_input_dim(config) From d1d83957f7776bf13b59bb1e483df966e2a7c9bd Mon Sep 17 00:00:00 2001 From: ysqyang Date: Thu, 5 Nov 2020 23:27:41 +0800 Subject: [PATCH 142/337] fixed a bug --- examples/cim/dqn/single_process_launcher.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/examples/cim/dqn/single_process_launcher.py b/examples/cim/dqn/single_process_launcher.py index 96870027b..2eee33b53 100644 --- a/examples/cim/dqn/single_process_launcher.py +++ b/examples/cim/dqn/single_process_launcher.py @@ -5,6 +5,12 @@ import numpy as np +from components.action_shaper import CIMActionShaper +from components.agent_manager import DQNAgentManager, create_dqn_agents +from components.config import set_input_dim +from components.experience_shaper import TruncatedExperienceShaper +from components.state_shaper import CIMStateShaper + from maro.rl import ( AgentManagerMode, KStepExperienceShaper, MaxDeltaEarlyStoppingChecker, SimpleActor, SimpleLearner, TwoPhaseLinearExplorer @@ -12,12 +18,6 @@ from maro.simulator import Env from maro.utils import Logger, convert_dottable -from .components.action_shaper import CIMActionShaper -from .components.agent_manager import DQNAgentManager, create_dqn_agents -from .components.config import set_input_dim -from .components.experience_shaper import TruncatedExperienceShaper -from .components.state_shaper import CIMStateShaper - def launch(config): # First determine the input dimension and add it to the config. From b360f640ae6b0f8a5156b2d07663cba00bf35126 Mon Sep 17 00:00:00 2001 From: ysqyang Date: Thu, 5 Nov 2020 23:30:04 +0800 Subject: [PATCH 143/337] fixed a bug --- examples/cim/dqn/components/agent_manager.py | 1 + maro/rl/models/fc_block.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/examples/cim/dqn/components/agent_manager.py b/examples/cim/dqn/components/agent_manager.py index 57436912a..d41c5c029 100644 --- a/examples/cim/dqn/components/agent_manager.py +++ b/examples/cim/dqn/components/agent_manager.py @@ -23,6 +23,7 @@ def create_dqn_agents(agent_id_list, config): input_dim=config.algorithm.input_dim, output_dim=num_actions, activation=nn.LeakyReLU, + is_head=True, **config.algorithm.model )] ) diff --git a/maro/rl/models/fc_block.py b/maro/rl/models/fc_block.py index 9cef4e82e..598b43ab9 100644 --- a/maro/rl/models/fc_block.py +++ b/maro/rl/models/fc_block.py @@ -64,7 +64,7 @@ def __init__( # build the net dims = [self._input_dim] + self._hidden_dims - layers = [self._build_internal_layer(in_dim, out_dim) for in_dim, out_dim in zip(dims, dims[1:])] + layers = [self._build_layer(in_dim, out_dim) for in_dim, out_dim in zip(dims, dims[1:])] # top layer layers.append(self._build_layer(dims[-1], self._output_dim, is_head=self._is_head)) From 6bace61b15ab365337c68d4f4438e57e3a6f301e Mon Sep 17 00:00:00 2001 From: ysqyang Date: Thu, 5 Nov 2020 23:53:15 +0800 Subject: [PATCH 144/337] fixed lint formatting issues --- maro/rl/__init__.py | 4 +--- maro/rl/models/fc_block.py | 4 ++-- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/maro/rl/__init__.py b/maro/rl/__init__.py index c26fb843c..1349bce48 100644 --- a/maro/rl/__init__.py +++ b/maro/rl/__init__.py @@ -13,9 +13,7 @@ ) from maro.rl.dist_topologies.single_learner_multi_actor_sync_mode import ActorProxy, ActorWorker from maro.rl.early_stopping.abs_early_stopping_checker import AbsEarlyStoppingChecker -from maro.rl.early_stopping.simple_early_stopping_checker import ( - MaxDeltaEarlyStoppingChecker, RSDEarlyStoppingChecker -) +from maro.rl.early_stopping.simple_early_stopping_checker import MaxDeltaEarlyStoppingChecker, RSDEarlyStoppingChecker from maro.rl.explorer.abs_explorer import AbsExplorer from maro.rl.explorer.simple_explorer import LinearExplorer, TwoPhaseLinearExplorer from maro.rl.learner.abs_learner import AbsLearner diff --git a/maro/rl/models/fc_block.py b/maro/rl/models/fc_block.py index 598b43ab9..1753b87c4 100644 --- a/maro/rl/models/fc_block.py +++ b/maro/rl/models/fc_block.py @@ -24,8 +24,8 @@ class FullyConnectedBlock(nn.Module): skip_connection_enabled (bool): If true, a skip connection will be built between the bottom (input) layer and top (output) layer. Defaults to False. dropout_p (float): Dropout probability. Defaults to None, in which case there is no drop-out. - gradient_threshold (float): Gradient clipping threshold. Defaults to None, in which case not gradient clipping is - performed. + gradient_threshold (float): Gradient clipping threshold. Defaults to None, in which case not gradient clipping + is performed. """ def __init__( self, From 7e4b4094180315cc70d0a84e7350bd424b149bc0 Mon Sep 17 00:00:00 2001 From: ysqyang Date: Fri, 6 Nov 2020 10:19:06 +0800 Subject: [PATCH 145/337] double DQN feature --- examples/cim/dqn/components/agent_manager.py | 8 +- examples/cim/dqn/config.yml | 1 - maro/rl/__init__.py | 3 + maro/rl/algorithms/dqn.py | 151 ++++--------------- 4 files changed, 40 insertions(+), 123 deletions(-) diff --git a/examples/cim/dqn/components/agent_manager.py b/examples/cim/dqn/components/agent_manager.py index 787c9b4fa..62b54ace2 100644 --- a/examples/cim/dqn/components/agent_manager.py +++ b/examples/cim/dqn/components/agent_manager.py @@ -15,7 +15,7 @@ def create_dqn_agents(agent_id_list, config): set_seeds(config.seed) agent_dict = {} for agent_id in agent_id_list: - q_value_model = LearningModel( + q_model = LearningModel( FullyConnectedNet( name=f'{agent_id}.policy', input_dim=config.algorithm.input_dim, @@ -26,9 +26,9 @@ def create_dqn_agents(agent_id_list, config): ) algorithm = DQN( - value_model=q_value_model, - value_optimizer_cls=RMSprop, - value_optimizer_params=config.algorithm.optimizer, + q_model=q_model, + optimizer_cls=RMSprop, + optimizer_params=config.algorithm.optimizer, loss_func=nn.functional.smooth_l1_loss, hyper_params=DQNHyperParams( **config.algorithm.hyper_parameters, diff --git a/examples/cim/dqn/config.yml b/examples/cim/dqn/config.yml index 61691777a..f3f613d2c 100644 --- a/examples/cim/dqn/config.yml +++ b/examples/cim/dqn/config.yml @@ -57,7 +57,6 @@ agents: target_update_frequency: 5 tau: 0.1 is_double: false - advantage_mode: "mean" experience_pool: capacity: -1 training_loop_parameters: diff --git a/maro/rl/__init__.py b/maro/rl/__init__.py index 483ca0154..a3affd331 100644 --- a/maro/rl/__init__.py +++ b/maro/rl/__init__.py @@ -16,12 +16,15 @@ from maro.rl.early_stopping.simple_early_stopping_checker import MaxDeltaEarlyStoppingChecker, RSDEarlyStoppingChecker from maro.rl.explorer.abs_explorer import AbsExplorer from maro.rl.explorer.simple_explorer import LinearExplorer, TwoPhaseLinearExplorer +from maro.rl.learner.abs_learner import AbsLearner +from maro.rl.learner.simple_learner import SimpleLearner from maro.rl.models.fc_net import FullyConnectedNet from maro.rl.models.learning_model import LearningModel from maro.rl.shaping.abs_shaper import AbsShaper from maro.rl.shaping.action_shaper import ActionShaper from maro.rl.shaping.experience_shaper import ExperienceShaper from maro.rl.shaping.k_step_experience_shaper import KStepExperienceShaper +from maro.rl.shaping.state_shaper import StateShaper from maro.rl.storage.abs_store import AbsStore from maro.rl.storage.column_based_store import ColumnBasedStore from maro.rl.storage.utils import OverwriteType diff --git a/maro/rl/algorithms/dqn.py b/maro/rl/algorithms/dqn.py index bb512c02b..b9c5285a2 100644 --- a/maro/rl/algorithms/dqn.py +++ b/maro/rl/algorithms/dqn.py @@ -1,8 +1,6 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT license. -import warnings - import numpy as np import torch import torch.nn as nn @@ -22,14 +20,8 @@ class DQNHyperParams: is_double (bool): If True, the next Q values will be computed according to the double DQN algorithm, i.e., q_next = Q_target(s, argmax(Q_eval(s, a))). Otherwise, q_next = max(Q_target(s, a)), per ordinary DQN. See https://arxiv.org/pdf/1509.06461.pdf for details. Defaults to False. - is_dueling (bool): If True, the Q values will be computed using a dueling architecture with a head for - state values and a head for advantages. See https://arxiv.org/pdf/1511.06581.pdf for details. - advantage_mode (str): advantage mode for the dueling architecture. Defaults to None, in which - case it is assumed that the regular Q-value model is used. """ - __slots__ = [ - "num_actions", "reward_decay", "target_update_frequency", "tau", "is_double", "is_dueling", "advantage_mode" - ] + __slots__ = ["num_actions", "reward_decay", "target_update_frequency", "tau", "is_double"] def __init__( self, @@ -37,17 +29,13 @@ def __init__( reward_decay: float, target_update_frequency: int, tau: float = 1.0, - is_double: bool = False, - is_dueling: bool = False, - advantage_mode: str = None + is_double: bool = False ): self.num_actions = num_actions self.reward_decay = reward_decay self.target_update_frequency = target_update_frequency self.tau = tau self.is_double = is_double - self.is_dueling = is_dueling - self.advantage_mode = advantage_mode class DQN(AbsAlgorithm): @@ -56,69 +44,40 @@ class DQN(AbsAlgorithm): See https://web.stanford.edu/class/psych209/Readings/MnihEtAlHassibis15NatureControlDeepRL.pdf for details. Args: - value_model (nn.Module): Q-value or state value model depending on whether ``is_dueling`` is true - in ``hyper-params``. - value_optimizer_cls: Torch optimizer class for the value model. If this is None, the eval model is not + q_model (nn.Module): Q-value model. + optimizer_cls: Torch optimizer class for the value model. If this is None, the eval model is not trainable. - value_optimizer_params: Parameters required for the optimizer class for the value model. + optimizer_params: Parameters required for the optimizer class for the value model. loss_func (Callable): Loss function for the value model. hyper_params: Hyper-parameter set for the DQN algorithm. - advantage_model (nn.Module): Model that estimates the advantage value. If ``is_dueling`` is true - in ``hyper-params``, Defaults to None. - advantage_optimizer_cls: Torch optimizer class for the advantage model. If this is None, the eval model is - not trainable. - advantage_optimizer_params: Parameters required for the optimizer class for the advantage model """ def __init__( self, - value_model: nn.Module, - value_optimizer_cls, - value_optimizer_params, + q_model: nn.Module, + optimizer_cls, + optimizer_params, loss_func, hyper_params: DQNHyperParams, - advantage_model: nn.Module = None, - advantage_optimizer_cls=None, - advantage_optimizer_params=None ): super().__init__() self._device = torch.device("cuda" if torch.cuda.is_available() else "cpu") self._hyper_params = hyper_params - self._is_training = value_optimizer_cls is not None - if self._hyper_params.is_dueling: - assert advantage_model is not None, "advantage_model cannot be None under dueling mode." - if self._is_training: - assert advantage_optimizer_cls is not None, \ - "advantage_optimizer_cls cannot be None under dueling mode." - - self._model_dict = { - "state_value": value_model.to(self._device), - "state_value_target": clone(value_model).to(self._device) if self._is_training else None, - "advantage": advantage_model.to(self._device), - "advantage_target": clone(advantage_model).to(self._device) if self._is_training else None - } - self._value_optimizer = value_optimizer_cls( - self._model_dict["state_value"].parameters(), **value_optimizer_params - ) - self._advantage_optimizer = advantage_optimizer_cls( - self._model_dict["advantage"].parameters(), **advantage_optimizer_params - ) - # No gradient computation required for the target models - for param in self._model_dict["state_value_target"].parameters(): - param.requires_grad = False - for param in self._model_dict["advantage_target"].parameters(): - param.requires_grad = False - else: - self._model_dict = { - "q_value": value_model.to(self._device), - "q_value_target": clone(value_model).to(self._device) if self._is_training else None, - } - self._optimizer = value_optimizer_cls( - self._model_dict["q_value"].parameters(), **value_optimizer_params - ) - + self._is_training = optimizer_cls is not None self._loss_func = loss_func self._train_cnt = 0 + self._model_dict = { + "eval": q_model.to(self._device), + "target": clone(q_model).to(self._device) if self._is_training else None, + } + + if self._model_dict["target"] is not None: + # No gradient computation required for the target model + for param in self._model_dict["target"].parameters(): + param.requires_grad = False + + self._optimizer = optimizer_cls(self._model_dict["eval"].parameters(), **optimizer_params) + @property def is_training(self): return self._is_training @@ -126,11 +85,7 @@ def is_training(self): def choose_action(self, state: np.ndarray, epsilon: float = None): if epsilon is None or np.random.rand() > epsilon: state = torch.from_numpy(state).unsqueeze(0) - if self._hyper_params.is_dueling: - self._model_dict["state_value"].eval() - self._model_dict["advantage"].eval() - else: - self._model_dict["q_value"].eval() + self._model_dict["eval"].eval() with torch.no_grad(): q_values = self._get_q_values(state) return q_values.argmax(dim=1).item() @@ -139,10 +94,6 @@ def choose_action(self, state: np.ndarray, epsilon: float = None): def train(self, states: np.ndarray, actions: np.ndarray, rewards: np.ndarray, next_states: np.ndarray): if not self._is_training: - warnings.warn( - "DQN is not in training mode since no optimizer is provided. Did you provide optimizer_cls and " - "optimizer_params when instantiating the algorithm?" - ) return states = torch.from_numpy(states).to(self._device) # (N, state_dim) @@ -156,20 +107,10 @@ def train(self, states: np.ndarray, actions: np.ndarray, rewards: np.ndarray, ne next_q_values = self._get_next_q_values(current_q_values_all, next_states) # (N,) target_q_values = (rewards + self._hyper_params.reward_decay * next_q_values).detach() # (N,) loss = self._loss_func(current_q_values, target_q_values) - - if self._hyper_params.is_dueling: - self._model_dict["state_value"].train() - self._model_dict["advantage"].train() - self._value_optimizer.zero_grad() - self._advantage_optimizer.zero_grad() - loss.backward() - self._value_optimizer.step() - self._advantage_optimizer.step() - else: - self._model_dict["q_value"].train() - self._optimizer.zero_grad() - loss.backward() - self._optimizer.step() + self._model_dict["eval"].train() + self._optimizer.zero_grad() + loss.backward() + self._optimizer.step() self._train_cnt += 1 if self._train_cnt % self._hyper_params.target_update_frequency == 0: @@ -178,41 +119,15 @@ def train(self, states: np.ndarray, actions: np.ndarray, rewards: np.ndarray, ne return np.abs((current_q_values - target_q_values).detach().numpy()) def _update_targets(self): - if self._is_training: - if not self._hyper_params.is_dueling: - for eval_params, target_params in zip( - self._model_dict["q_value"].parameters(), self._model_dict["q_value_target"].parameters() - ): - target_params.data = ( - self._hyper_params.tau * eval_params.data + - (1 - self._hyper_params.tau) * target_params.data - ) - else: - for eval_params, target_params in zip( - self._model_dict["state_value"].parameters(), self._model_dict["state_value_target"].parameters() - ): - target_params.data = ( - self._hyper_params.tau * eval_params.data + - (1 - self._hyper_params.tau) * target_params.data - ) - for eval_params, target_params in zip( - self._model_dict["advantage"].parameters(), self._model_dict["advantage_target"].parameters() - ): - target_params.data = ( - self._hyper_params.tau * eval_params.data + - (1 - self._hyper_params.tau) * target_params.data - ) + for eval_params, target_params in zip( + self._model_dict["eval"].parameters(), self._model_dict["target"].parameters() + ): + target_params.data = ( + self._hyper_params.tau * eval_params.data + (1 - self._hyper_params.tau) * target_params.data + ) def _get_q_values(self, states, is_target: bool = False): - if not self._hyper_params.is_dueling: - return self._model_dict["q_value_target" if is_target else "q_value"](states) - - state_values = self._model_dict["state_value_target" if is_target else "state_value"](states) - advantages = self._model_dict["advantage_target" if is_target else "advantage"](states) - # Use mean or max correction to address the identifiability issue - corrections = advantages.mean(1) if self._hyper_params.advantage_mode == "mean" else advantages.max(1)[0] - q_values = state_values + advantages - corrections.unsqueeze(1) - return q_values + return self._model_dict["target" if is_target else "eval"](states) def _get_next_q_values(self, current_q_values_all, states): if self._hyper_params.is_double: From 0ea51a435256e9fc972f4a15a3eb9030b922251d Mon Sep 17 00:00:00 2001 From: ysqyang Date: Fri, 6 Nov 2020 10:22:56 +0800 Subject: [PATCH 146/337] fixed a bug --- examples/cim/dqn/dist_actor.py | 12 ++++++------ examples/cim/dqn/dist_learner.py | 6 +++--- examples/cim/dqn/single_process_launcher.py | 12 ++++++------ 3 files changed, 15 insertions(+), 15 deletions(-) diff --git a/examples/cim/dqn/dist_actor.py b/examples/cim/dqn/dist_actor.py index 2213d2af3..80fe71dfc 100644 --- a/examples/cim/dqn/dist_actor.py +++ b/examples/cim/dqn/dist_actor.py @@ -5,16 +5,16 @@ import numpy as np +from components.action_shaper import CIMActionShaper +from components.agent_manager import DQNAgentManager, create_dqn_agents +from components.config import config, set_input_dim +from components.experience_shaper import TruncatedExperienceShaper +from components.state_shaper import CIMStateShaper + from maro.rl import ActorWorker, AgentManagerMode, KStepExperienceShaper, SimpleActor from maro.simulator import Env from maro.utils import convert_dottable -from .components.action_shaper import CIMActionShaper -from .components.agent_manager import DQNAgentManager, create_dqn_agents -from .components.config import config, set_input_dim -from .components.experience_shaper import TruncatedExperienceShaper -from .components.state_shaper import CIMStateShaper - def launch(config): set_input_dim(config) diff --git a/examples/cim/dqn/dist_learner.py b/examples/cim/dqn/dist_learner.py index 78e3761b8..8e970f0b6 100644 --- a/examples/cim/dqn/dist_learner.py +++ b/examples/cim/dqn/dist_learner.py @@ -3,6 +3,9 @@ import os +from components.agent_manager import DQNAgentManager, create_dqn_agents +from components.config import set_input_dim + from maro.rl import ( ActorProxy, AgentManagerMode, MaxDeltaEarlyStoppingChecker, SimpleLearner, TwoPhaseLinearExplorer, concat_experiences_by_agent @@ -10,9 +13,6 @@ from maro.simulator import Env from maro.utils import Logger, convert_dottable -from .components.agent_manager import DQNAgentManager, create_dqn_agents -from .components.config import set_input_dim - def launch(config): set_input_dim(config) diff --git a/examples/cim/dqn/single_process_launcher.py b/examples/cim/dqn/single_process_launcher.py index 96870027b..cac3f311b 100644 --- a/examples/cim/dqn/single_process_launcher.py +++ b/examples/cim/dqn/single_process_launcher.py @@ -5,6 +5,12 @@ import numpy as np +from components.action_shaper import CIMActionShaper +from components.agent_manager import DQNAgentManager, create_dqn_agents +from components.config import set_input_dim +from omponents.experience_shaper import TruncatedExperienceShaper +from components.state_shaper import CIMStateShaper + from maro.rl import ( AgentManagerMode, KStepExperienceShaper, MaxDeltaEarlyStoppingChecker, SimpleActor, SimpleLearner, TwoPhaseLinearExplorer @@ -12,12 +18,6 @@ from maro.simulator import Env from maro.utils import Logger, convert_dottable -from .components.action_shaper import CIMActionShaper -from .components.agent_manager import DQNAgentManager, create_dqn_agents -from .components.config import set_input_dim -from .components.experience_shaper import TruncatedExperienceShaper -from .components.state_shaper import CIMStateShaper - def launch(config): # First determine the input dimension and add it to the config. From 5e14c04ff6c4cd95b780939ccfca4273f4f1f6ff Mon Sep 17 00:00:00 2001 From: ysqyang Date: Fri, 6 Nov 2020 10:23:27 +0800 Subject: [PATCH 147/337] fixed a bug --- examples/cim/dqn/single_process_launcher.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/cim/dqn/single_process_launcher.py b/examples/cim/dqn/single_process_launcher.py index cac3f311b..2eee33b53 100644 --- a/examples/cim/dqn/single_process_launcher.py +++ b/examples/cim/dqn/single_process_launcher.py @@ -8,7 +8,7 @@ from components.action_shaper import CIMActionShaper from components.agent_manager import DQNAgentManager, create_dqn_agents from components.config import set_input_dim -from omponents.experience_shaper import TruncatedExperienceShaper +from components.experience_shaper import TruncatedExperienceShaper from components.state_shaper import CIMStateShaper from maro.rl import ( From a3f340b62419cdfb9f57a8107ce46266b5abc654 Mon Sep 17 00:00:00 2001 From: ysqyang Date: Fri, 6 Nov 2020 20:22:54 +0800 Subject: [PATCH 148/337] fixed PR comments --- maro/rl/algorithms/dqn.py | 82 +++++++++++++++++++-------------------- 1 file changed, 40 insertions(+), 42 deletions(-) diff --git a/maro/rl/algorithms/dqn.py b/maro/rl/algorithms/dqn.py index b9c5285a2..1a67763d9 100644 --- a/maro/rl/algorithms/dqn.py +++ b/maro/rl/algorithms/dqn.py @@ -1,6 +1,8 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT license. +from typing import Callable + import numpy as np import torch import torch.nn as nn @@ -16,10 +18,10 @@ class DQNHyperParams: num_actions (int): Number of possible actions. reward_decay (float): Reward decay as defined in standard RL terminology. target_update_frequency (int): Number of training rounds between target model updates. - tau (float): Soft update coefficient, i.e., target_model = tau * eval_model + (1-tau) * target_model. + tau (float): Soft update coefficient, i.e., target_model = tau * eval_model + (1 - tau) * target_model. is_double (bool): If True, the next Q values will be computed according to the double DQN algorithm, - i.e., q_next = Q_target(s, argmax(Q_eval(s, a))). Otherwise, q_next = max(Q_target(s, a)), per ordinary - DQN. See https://arxiv.org/pdf/1509.06461.pdf for details. Defaults to False. + i.e., q_next = Q_target(s, argmax(Q_eval(s, a))). Otherwise, q_next = max(Q_target(s, a)). + See https://arxiv.org/pdf/1509.06461.pdf for details. Defaults to False. """ __slots__ = ["num_actions", "reward_decay", "target_update_frequency", "tau", "is_double"] @@ -28,8 +30,8 @@ def __init__( num_actions: int, reward_decay: float, target_update_frequency: int, - tau: float = 1.0, - is_double: bool = False + tau: float = 0.1, + is_double: bool = True ): self.num_actions = num_actions self.reward_decay = reward_decay @@ -45,47 +47,45 @@ class DQN(AbsAlgorithm): Args: q_model (nn.Module): Q-value model. - optimizer_cls: Torch optimizer class for the value model. If this is None, the eval model is not + optimizer_cls (torch.optim.Optimizer): Torch optimizer class for the value model. If this is None, the eval model is not trainable. - optimizer_params: Parameters required for the optimizer class for the value model. + optimizer_params (dict): Parameters required for the optimizer class for the value model. loss_func (Callable): Loss function for the value model. hyper_params: Hyper-parameter set for the DQN algorithm. """ def __init__( self, q_model: nn.Module, - optimizer_cls, - optimizer_params, - loss_func, + optimizer_cls: torch.optim.Optimizer, + optimizer_params: dict, + loss_func: Callable, hyper_params: DQNHyperParams, ): super().__init__() self._device = torch.device("cuda" if torch.cuda.is_available() else "cpu") self._hyper_params = hyper_params - self._is_training = optimizer_cls is not None + self._is_trainable = optimizer_cls is not None self._loss_func = loss_func - self._train_cnt = 0 + self._training_counter = 0 - self._model_dict = { - "eval": q_model.to(self._device), - "target": clone(q_model).to(self._device) if self._is_training else None, - } + self._eval_model = q_model.to(self._device) + self._target_model = clone(q_model).to(self._device) if self._is_trainable else None - if self._model_dict["target"] is not None: + if self._target_model is not None: # No gradient computation required for the target model - for param in self._model_dict["target"].parameters(): + for param in self._target_model.parameters(): param.requires_grad = False - self._optimizer = optimizer_cls(self._model_dict["eval"].parameters(), **optimizer_params) + self._optimizer = optimizer_cls(self._eval_model.parameters(), **optimizer_params) @property - def is_training(self): - return self._is_training + def is_trainable(self): + return self._is_trainable def choose_action(self, state: np.ndarray, epsilon: float = None): if epsilon is None or np.random.rand() > epsilon: state = torch.from_numpy(state).unsqueeze(0) - self._model_dict["eval"].eval() + self._eval_model.eval() with torch.no_grad(): q_values = self._get_q_values(state) return q_values.argmax(dim=1).item() @@ -93,7 +93,7 @@ def choose_action(self, state: np.ndarray, epsilon: float = None): return np.random.choice(self._hyper_params.num_actions) def train(self, states: np.ndarray, actions: np.ndarray, rewards: np.ndarray, next_states: np.ndarray): - if not self._is_training: + if not self._is_trainable: return states = torch.from_numpy(states).to(self._device) # (N, state_dim) @@ -102,56 +102,54 @@ def train(self, states: np.ndarray, actions: np.ndarray, rewards: np.ndarray, ne next_states = torch.from_numpy(next_states).to(self._device) # (N, state_dim) if len(actions.shape) == 1: actions = actions.unsqueeze(1) # (N, 1) - current_q_values_all = self._get_q_values(states) - current_q_values = current_q_values_all.gather(1, actions).squeeze(1) # (N,) - next_q_values = self._get_next_q_values(current_q_values_all, next_states) # (N,) + current_q_values_for_all_actions = self._get_q_values(states) + current_q_values = current_q_values_for_all_actions.gather(1, actions).squeeze(1) # (N,) + next_q_values = self._get_next_q_values(current_q_values_for_all_actions, next_states) # (N,) target_q_values = (rewards + self._hyper_params.reward_decay * next_q_values).detach() # (N,) loss = self._loss_func(current_q_values, target_q_values) - self._model_dict["eval"].train() + self._eval_model.train() self._optimizer.zero_grad() loss.backward() self._optimizer.step() - self._train_cnt += 1 - if self._train_cnt % self._hyper_params.target_update_frequency == 0: + self._training_counter += 1 + if self._training_counter % self._hyper_params.target_update_frequency == 0: self._update_targets() return np.abs((current_q_values - target_q_values).detach().numpy()) def _update_targets(self): for eval_params, target_params in zip( - self._model_dict["eval"].parameters(), self._model_dict["target"].parameters() + self._eval_model.parameters(), self._target_model.parameters() ): target_params.data = ( self._hyper_params.tau * eval_params.data + (1 - self._hyper_params.tau) * target_params.data ) def _get_q_values(self, states, is_target: bool = False): - return self._model_dict["target" if is_target else "eval"](states) + model = self._target_model if is_target else self._eval_model + return model(states) - def _get_next_q_values(self, current_q_values_all, states): + def _get_next_q_values(self, current_q_values_all, next_states): if self._hyper_params.is_double: actions = current_q_values_all.max(dim=1)[1].unsqueeze(1) - return self._get_q_values(states, is_target=True).gather(1, actions).squeeze(1) # (N,) + return self._get_q_values(next_states, is_target=True).gather(1, actions).squeeze(1) # (N,) else: - return self._get_q_values(states, is_target=True).max(dim=1)[0] # (N,) - - def _get_state_dicts(self): - return {k: model.state_dict() for k, model in self._model_dict.items()} + return self._get_q_values(next_states, is_target=True).max(dim=1)[0] # (N,) def load_models(self, model_dict): """Load models from memory.""" - for key in self._model_dict: - self._model_dict[key].load_state_dict(model_dict[key]) + self._eval_model.load_state_dict(model_dict["eval"]) + self._target_model.load_state_dict(model_dict["target"]) def dump_models(self): """Return the eval model.""" - return self._get_state_dicts() + return {"eval": self._eval_model.state_dict(), "target": self._target_model.state_dict()} def load_models_from_file(self, path): """Load the eval model from disk.""" - self._model_dict = torch.load(path) + self.load_models(torch.load(path)) def dump_models_to_file(self, path: str): """Dump the eval model to disk.""" - torch.save(self._get_state_dicts(), path) + torch.save(self.dump_models(), path) From 65f230cc98be1073df5991db6c1bc80a716681f1 Mon Sep 17 00:00:00 2001 From: ysqyang Date: Fri, 6 Nov 2020 20:27:21 +0800 Subject: [PATCH 149/337] fixed lint issue --- maro/rl/algorithms/dqn.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/maro/rl/algorithms/dqn.py b/maro/rl/algorithms/dqn.py index 1a67763d9..a668f9f28 100644 --- a/maro/rl/algorithms/dqn.py +++ b/maro/rl/algorithms/dqn.py @@ -47,8 +47,8 @@ class DQN(AbsAlgorithm): Args: q_model (nn.Module): Q-value model. - optimizer_cls (torch.optim.Optimizer): Torch optimizer class for the value model. If this is None, the eval model is not - trainable. + optimizer_cls (torch.optim.Optimizer): Torch optimizer class for the value model. If this is None, the eval + model is not trainable. optimizer_params (dict): Parameters required for the optimizer class for the value model. loss_func (Callable): Loss function for the value model. hyper_params: Hyper-parameter set for the DQN algorithm. From 8597db29b3a26ac9cd67e680e50038ea2b60707c Mon Sep 17 00:00:00 2001 From: ysqyang Date: Mon, 9 Nov 2020 10:44:12 +0800 Subject: [PATCH 150/337] embedded optimizer into SingleHeadLearningModel --- maro/rl/models/learning_model.py | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/maro/rl/models/learning_model.py b/maro/rl/models/learning_model.py index 30f12edfd..89e6eb669 100644 --- a/maro/rl/models/learning_model.py +++ b/maro/rl/models/learning_model.py @@ -1,6 +1,7 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT license. +import torch import torch.nn as nn @@ -10,9 +11,10 @@ class SingleHeadLearningModel(nn.Module): The shared blocks must be chainable, i.e., the output dimension of a block must match the input dimension of its successor. """ - def __init__(self, block_list: list): + def __init__(self, block_list: list, optimizer_cls, optimizer_params: dict): super().__init__() self._net = nn.Sequential(*block_list) + self._optimizer = optimizer_cls(self._net.parameters(), **optimizer_params) def forward(self, inputs): """Feedforward computation. @@ -25,6 +27,19 @@ def forward(self, inputs): """ return self._net(inputs) + def step(self, loss: torch.tensor): + """Feedforward computation. + + Args: + loss: loss tensor + + Returns: + Outputs from the model. + """ + self._optimizer.zero_grad() + loss.backward() + self._optimizer.step() + class MultiHeadLearningModel(nn.Module): """NN model that consists of shared blocks and multiple task heads. From c4f1ff8046e292be77816e11a6c13df41597490d Mon Sep 17 00:00:00 2001 From: ysqyang Date: Mon, 9 Nov 2020 10:49:26 +0800 Subject: [PATCH 151/337] 1. fixed PR comments related to load/dump; 2. removed abstract load/dump methods from AbsAlgorithm --- maro/rl/algorithms/abs_algorithm.py | 20 -------------------- maro/rl/algorithms/dqn.py | 19 +++++++++---------- 2 files changed, 9 insertions(+), 30 deletions(-) diff --git a/maro/rl/algorithms/abs_algorithm.py b/maro/rl/algorithms/abs_algorithm.py index f1f8d7b00..87106999a 100644 --- a/maro/rl/algorithms/abs_algorithm.py +++ b/maro/rl/algorithms/abs_algorithm.py @@ -38,23 +38,3 @@ def train(self, *args, **kwargs): algorithm, this may look like train(self, state_batch, action_batch, reward_batch, next_state_batch). """ return NotImplementedError - - @abstractmethod - def load_models(self, *models, **model_dict): - """Load trainable models from memory.""" - return NotImplementedError - - @abstractmethod - def dump_models(self): - """Return the algorithm's trainable models.""" - return NotImplementedError - - @abstractmethod - def load_models_from_file(self, path): - """Load trainable models from disk.""" - return NotImplementedError - - @abstractmethod - def dump_models_to_file(self, path: str): - """Dump the algorithm's trainable models to disk.""" - return NotImplementedError diff --git a/maro/rl/algorithms/dqn.py b/maro/rl/algorithms/dqn.py index a668f9f28..26984d4a3 100644 --- a/maro/rl/algorithms/dqn.py +++ b/maro/rl/algorithms/dqn.py @@ -137,19 +137,18 @@ def _get_next_q_values(self, current_q_values_all, next_states): else: return self._get_q_values(next_states, is_target=True).max(dim=1)[0] # (N,) - def load_models(self, model_dict): - """Load models from memory.""" - self._eval_model.load_state_dict(model_dict["eval"]) - self._target_model.load_state_dict(model_dict["target"]) + def load_eval_model(self, eval_model): + """Load evaluation model from memory.""" + self._eval_model.load_state_dict(eval_model) - def dump_models(self): + def dump_eval_model(self): """Return the eval model.""" - return {"eval": self._eval_model.state_dict(), "target": self._target_model.state_dict()} + return self._eval_model.state_dict() - def load_models_from_file(self, path): + def load_eval_model_from_file(self, path): """Load the eval model from disk.""" - self.load_models(torch.load(path)) + self._eval_model.load_state_dict(torch.load(path)) - def dump_models_to_file(self, path: str): + def dump_eval_model_to_file(self, path: str): """Dump the eval model to disk.""" - torch.save(self.dump_models(), path) + torch.save(self._eval_model.state_dict(), path) From dbab05ee0fb443d278250a443aad403ab928f316 Mon Sep 17 00:00:00 2001 From: ysqyang Date: Mon, 9 Nov 2020 11:18:33 +0800 Subject: [PATCH 152/337] added load_models in simple_learner --- maro/rl/algorithms/abs_algorithm.py | 20 ++++++++++++++++++++ maro/rl/algorithms/dqn.py | 8 ++++---- maro/rl/learner/simple_learner.py | 7 +++++-- 3 files changed, 29 insertions(+), 6 deletions(-) diff --git a/maro/rl/algorithms/abs_algorithm.py b/maro/rl/algorithms/abs_algorithm.py index 87106999a..f1f8d7b00 100644 --- a/maro/rl/algorithms/abs_algorithm.py +++ b/maro/rl/algorithms/abs_algorithm.py @@ -38,3 +38,23 @@ def train(self, *args, **kwargs): algorithm, this may look like train(self, state_batch, action_batch, reward_batch, next_state_batch). """ return NotImplementedError + + @abstractmethod + def load_models(self, *models, **model_dict): + """Load trainable models from memory.""" + return NotImplementedError + + @abstractmethod + def dump_models(self): + """Return the algorithm's trainable models.""" + return NotImplementedError + + @abstractmethod + def load_models_from_file(self, path): + """Load trainable models from disk.""" + return NotImplementedError + + @abstractmethod + def dump_models_to_file(self, path: str): + """Dump the algorithm's trainable models to disk.""" + return NotImplementedError diff --git a/maro/rl/algorithms/dqn.py b/maro/rl/algorithms/dqn.py index 26984d4a3..64c77895b 100644 --- a/maro/rl/algorithms/dqn.py +++ b/maro/rl/algorithms/dqn.py @@ -137,18 +137,18 @@ def _get_next_q_values(self, current_q_values_all, next_states): else: return self._get_q_values(next_states, is_target=True).max(dim=1)[0] # (N,) - def load_eval_model(self, eval_model): + def load_models(self, eval_model): """Load evaluation model from memory.""" self._eval_model.load_state_dict(eval_model) - def dump_eval_model(self): + def dump_models(self): """Return the eval model.""" return self._eval_model.state_dict() - def load_eval_model_from_file(self, path): + def load_models_from_file(self, path): """Load the eval model from disk.""" self._eval_model.load_state_dict(torch.load(path)) - def dump_eval_model_to_file(self, path: str): + def dump_models_to_file(self, path: str): """Dump the eval model to disk.""" torch.save(self._eval_model.state_dict(), path) diff --git a/maro/rl/learner/simple_learner.py b/maro/rl/learner/simple_learner.py index 90b089cc1..cb8050e6b 100644 --- a/maro/rl/learner/simple_learner.py +++ b/maro/rl/learner/simple_learner.py @@ -114,8 +114,11 @@ def exit(self, code: int = 0): self._actor.roll_out(done=True) sys.exit(code) - def dump_models(self, model_dump_dir: str): - self._trainable_agents.dump_models_to_files(model_dump_dir) + def load_models(self, dir_path: str): + self._trainable_agents.load_models_from_files(dir_path) + + def dump_models(self, dir_path: str): + self._trainable_agents.dump_models_to_files(dir_path) def _is_shared_agent_instance(self): """If true, the set of agents performing inference in actor is the same as self._trainable_agents.""" From f9d34e1152fd6191e7f5b46dbd584257537824fa Mon Sep 17 00:00:00 2001 From: ysqyang Date: Mon, 9 Nov 2020 11:37:02 +0800 Subject: [PATCH 153/337] minor docstring edits --- maro/rl/algorithms/dqn.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/maro/rl/algorithms/dqn.py b/maro/rl/algorithms/dqn.py index 64c77895b..f5a601d10 100644 --- a/maro/rl/algorithms/dqn.py +++ b/maro/rl/algorithms/dqn.py @@ -138,17 +138,17 @@ def _get_next_q_values(self, current_q_values_all, next_states): return self._get_q_values(next_states, is_target=True).max(dim=1)[0] # (N,) def load_models(self, eval_model): - """Load evaluation model from memory.""" + """Load the evaluation model from memory.""" self._eval_model.load_state_dict(eval_model) def dump_models(self): - """Return the eval model.""" + """Return the evaluation model.""" return self._eval_model.state_dict() def load_models_from_file(self, path): - """Load the eval model from disk.""" + """Load the evaluation model from disk.""" self._eval_model.load_state_dict(torch.load(path)) def dump_models_to_file(self, path: str): - """Dump the eval model to disk.""" + """Dump the evaluation model to disk.""" torch.save(self._eval_model.state_dict(), path) From b227509bb1b6b3d3afa7a58a2dca2007cdfb24b7 Mon Sep 17 00:00:00 2001 From: ysqyang Date: Mon, 9 Nov 2020 11:43:19 +0800 Subject: [PATCH 154/337] minor docstring edits --- maro/rl/early_stopping/simple_early_stopping_checker.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/maro/rl/early_stopping/simple_early_stopping_checker.py b/maro/rl/early_stopping/simple_early_stopping_checker.py index 517d5655a..2232d2b46 100644 --- a/maro/rl/early_stopping/simple_early_stopping_checker.py +++ b/maro/rl/early_stopping/simple_early_stopping_checker.py @@ -8,10 +8,10 @@ class SimpleEarlyStoppingChecker(AbsEarlyStoppingChecker): - """Early stopping checker based on the some simple measure obtained by applying a measure function to the last - k metric values. + """Early stopping checker based on the some simple measure. - The measure function must take a list as input and output a single number. + The measure is obtained by applying a user-defined measure function to the last k metric values. The measure + function must take a list as input and output a single number. """ def __init__(self, last_k, threshold, measure_func: Callable[[list], Union[int, float]]): super().__init__(last_k, threshold) From b8bab19cdb093017f680aa001483c7c6e131b17a Mon Sep 17 00:00:00 2001 From: ysqyang Date: Mon, 9 Nov 2020 11:44:08 +0800 Subject: [PATCH 155/337] minor docstring edits --- maro/rl/early_stopping/simple_early_stopping_checker.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/maro/rl/early_stopping/simple_early_stopping_checker.py b/maro/rl/early_stopping/simple_early_stopping_checker.py index 517d5655a..2232d2b46 100644 --- a/maro/rl/early_stopping/simple_early_stopping_checker.py +++ b/maro/rl/early_stopping/simple_early_stopping_checker.py @@ -8,10 +8,10 @@ class SimpleEarlyStoppingChecker(AbsEarlyStoppingChecker): - """Early stopping checker based on the some simple measure obtained by applying a measure function to the last - k metric values. + """Early stopping checker based on the some simple measure. - The measure function must take a list as input and output a single number. + The measure is obtained by applying a user-defined measure function to the last k metric values. The measure + function must take a list as input and output a single number. """ def __init__(self, last_k, threshold, measure_func: Callable[[list], Union[int, float]]): super().__init__(last_k, threshold) From 10e7f87b07e0dadbf8c99744530ca69ad950141b Mon Sep 17 00:00:00 2001 From: ysqyang Date: Mon, 9 Nov 2020 16:16:20 +0800 Subject: [PATCH 156/337] mv optimizer options inside LearningMode --- maro/rl/__init__.py | 2 + maro/rl/algorithms/dqn.py | 21 ++-- maro/rl/models/abs_learning_model.py | 21 ++++ maro/rl/models/learning_model.py | 106 +++++++++++++++---- maro/utils/exception/error_code.py | 3 +- maro/utils/exception/rl_toolkit_exception.py | 6 ++ 6 files changed, 128 insertions(+), 31 deletions(-) create mode 100644 maro/rl/models/abs_learning_model.py diff --git a/maro/rl/__init__.py b/maro/rl/__init__.py index a5ad26742..a0c8453a3 100644 --- a/maro/rl/__init__.py +++ b/maro/rl/__init__.py @@ -20,6 +20,7 @@ from maro.rl.explorer.simple_explorer import LinearExplorer, TwoPhaseLinearExplorer from maro.rl.learner.abs_learner import AbsLearner from maro.rl.learner.simple_learner import SimpleLearner +from maro.rl.models.abs_learning_model import AbsLearningModel from maro.rl.models.fc_block import FullyConnectedBlock from maro.rl.models.learning_model import MultiHeadLearningModel, SingleHeadLearningModel from maro.rl.shaping.abs_shaper import AbsShaper @@ -39,6 +40,7 @@ 'AbsEarlyStoppingChecker', 'AbsExplorer', 'AbsLearner', + 'AbsLearningModel', 'AbsShaper', 'AbsStore', 'ActionShaper', diff --git a/maro/rl/algorithms/dqn.py b/maro/rl/algorithms/dqn.py index ac6945f17..9b491248a 100644 --- a/maro/rl/algorithms/dqn.py +++ b/maro/rl/algorithms/dqn.py @@ -16,7 +16,7 @@ class DQNHyperParams: num_actions (int): Number of possible actions. reward_decay (float): Reward decay as defined in standard RL terminology. target_update_frequency (int): Number of training rounds between target model updates. - tau (float): Soft update coefficient, i.e., target_model = tau * eval_model + (1-tau) * target_model. + tau (float): Soft update coefficient, i.e., target_model = tau * q_model + (1-tau) * target_model. """ __slots__ = ["num_actions", "reward_decay", "target_update_frequency", "tau"] @@ -39,30 +39,27 @@ class DQN(AbsAlgorithm): See https://web.stanford.edu/class/psych209/Readings/MnihEtAlHassibis15NatureControlDeepRL.pdf for details. Args: - eval_model (nn.Module): Q-value model for given states and actions. + q_model (nn.Module): Q-value model for given states and actions. optimizer_cls: Torch optimizer class for the eval model. If this is None, the eval model is not trainable. optimizer_params: Parameters required for the eval optimizer class. loss_func (Callable): Loss function for the value model. hyper_params: Hyper-parameter set for the DQN algorithm. - target_model (nn.Module): Q-value model to train the ``eval_model`` against and to be updated periodically. If + target_model (nn.Module): Q-value model to train the ``q_model`` against and to be updated periodically. If it is None, the target model will be initialized as a deep copy of the eval model. """ def __init__( self, - eval_model: nn.Module, - optimizer_cls, - optimizer_params, + q_model: nn.Module, loss_func, hyper_params: DQNHyperParams, - target_model=None ): super().__init__() self._device = torch.device("cuda" if torch.cuda.is_available() else "cpu") - self._model_dict = {"eval": eval_model.to(self._device)} + self._model_dict = {"eval": q_model.to(self._device)} if optimizer_cls is not None: self._optimizer = optimizer_cls(self._model_dict["eval"].parameters(), **optimizer_params) if target_model is None: - self._model_dict["target"] = clone(eval_model).to(self._device) + self._model_dict["target"] = clone(q_model).to(self._device) else: self._model_dict["target"] = target_model.to(self._device) # No gradient computation required for the target model @@ -74,7 +71,7 @@ def __init__( self._train_cnt = 0 @property - def eval_model(self): + def q_model(self): return self._model_dict["eval"] def choose_action(self, state: np.ndarray, epsilon: float = None): @@ -118,9 +115,9 @@ def _update_target_model(self): self._hyper_params.tau * eval_params.data + (1 - self._hyper_params.tau) * target_params.data ) - def load_models(self, eval_model): + def load_models(self, q_model): """Load the eval model from memory.""" - self._model_dict["eval"].load_state_dict(eval_model) + self._model_dict["eval"].load_state_dict(q_model) def dump_models(self): """Return the eval model.""" diff --git a/maro/rl/models/abs_learning_model.py b/maro/rl/models/abs_learning_model.py new file mode 100644 index 000000000..2e797665d --- /dev/null +++ b/maro/rl/models/abs_learning_model.py @@ -0,0 +1,21 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +from abc import abstractmethod + +import torch.nn as nn + + +class AbsLearningModel(nn.Module): + def __init__(self): + super().__init__() + + @abstractmethod + def forward(self, inputs): + """Feedforward computation""" + return NotImplemented + + @abstractmethod + def step(self, *losses): + """Use losses to back-propagate gradients and apply the gradients to the underlying parameters.""" + return NotImplemented diff --git a/maro/rl/models/learning_model.py b/maro/rl/models/learning_model.py index 89e6eb669..66eed714a 100644 --- a/maro/rl/models/learning_model.py +++ b/maro/rl/models/learning_model.py @@ -1,20 +1,31 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT license. +from typing import Dict, Tuple + import torch import torch.nn as nn +from maro.utils.exception.rl_toolkit_exception import MissingOptimizerError +from .abs_learning_model import AbsLearningModel + -class SingleHeadLearningModel(nn.Module): +class SingleHeadLearningModel(AbsLearningModel): """NN model that consists of shared blocks and multiple task heads. The shared blocks must be chainable, i.e., the output dimension of a block must match the input dimension of its successor. """ - def __init__(self, block_list: list, optimizer_cls, optimizer_params: dict): + def __init__(self, block_list: list, optimizer_opt: Tuple = None): super().__init__() self._net = nn.Sequential(*block_list) - self._optimizer = optimizer_cls(self._net.parameters(), **optimizer_params) + self._is_trainable = optimizer_opt is not None + if self._is_trainable: + self._optimizer = optimizer_opt[0](self._net.parameters(), **optimizer_opt[1]) + + @property + def is_trainable(self): + return self._is_trainable def forward(self, inputs): """Feedforward computation. @@ -28,20 +39,13 @@ def forward(self, inputs): return self._net(inputs) def step(self, loss: torch.tensor): - """Feedforward computation. - - Args: - loss: loss tensor - - Returns: - Outputs from the model. - """ + """Use the loss to back-propagate gradients and apply them to the underlying parameters.""" self._optimizer.zero_grad() loss.backward() self._optimizer.step() -class MultiHeadLearningModel(nn.Module): +class MultiHeadLearningModel(AbsLearningModel): """NN model that consists of shared blocks and multiple task heads. The shared blocks must be chainable, i.e., the output dimension of a block must match the input dimension of @@ -49,12 +53,55 @@ class MultiHeadLearningModel(nn.Module): output of the model will be a dictionary with the names of the heads as keys and the corresponding head outputs as values. Otherwise, the output will be the output of the last block. """ - def __init__(self, shared_block_list: list, task_head_block_dict: dict): + def __init__( + self, + head_block_dict: dict, + head_optimizer_opt_dict: Dict[Tuple[torch.optim.Optimizer, dict]] = None, + shared_block_list: list = None, + shared_stack_optimizer_opt: Tuple[torch.optim.Optimizer, dict] = None + ): super().__init__() - self._task_head_keys = list(task_head_block_dict.keys()) - shared_stack = nn.Sequential(*shared_block_list) - for key, head in task_head_block_dict.items(): - setattr(self, key, nn.Sequential(shared_stack, head)) + self._has_shared_layers = shared_block_list is not None + self._has_trainable_shared_layers = self._has_shared_layers and shared_stack_optimizer_opt is not None + self._has_trainable_heads = head_optimizer_opt_dict is not None + # shared stack + if self._has_shared_layers: + self._shared_stack = nn.Sequential(*shared_block_list) + if self._has_trainable_shared_layers: + self._shared_stack_optimizer = shared_stack_optimizer_opt[0]( + self._shared_stack.parameters(), **shared_stack_optimizer_opt[1] + ) + + # heads + self._head_keys = list(head_block_dict.keys()) + if self._has_shared_layers: + self._net = nn.ModuleDict({ + key: nn.Sequential(self._shared_stack, head) for key, head in head_block_dict.items() + }) + else: + self._net = head_block_dict + + if self._has_trainable_heads: + self._head_optimizer_dict = { + key: head_optimizer_opt_dict[key][0](head.parameters(), **head_optimizer_opt_dict[key][1]) + for key, head in head_block_dict.items() + } + + @property + def has_shared_layers(self): + return self._has_shared_layers + + @property + def has_trainable_shared_layers(self): + return self._has_trainable_shared_layers + + @property + def has_trainable_heads(self): + return self._has_trainable_heads + + @property + def head_keys(self): + return self._head_keys def forward(self, inputs, key=None): """Feedforward computations for the given head(s). @@ -70,9 +117,32 @@ def forward(self, inputs, key=None): Outputs from the required head(s). """ if key is None: - return {key: getattr(self, key)(inputs) for key in self._task_head_keys} + return {key: getattr(self, key)(inputs) for key in self._head_keys} if isinstance(key, list): return {k: getattr(self, k)(inputs) for k in key} else: return getattr(self, key)(inputs) + + def step(self, *losses): + """Use the losses to back-propagate gradients and apply them to the underlying parameters.""" + if not self._has_trainable_shared_layers and not self._has_trainable_heads: + raise MissingOptimizerError("No optimizer registered to the model") + + # Zero all gradients + if self._has_trainable_shared_layers: + self._shared_stack_optimizer.zero_grad() + if self._has_trainable_heads: + for optim in self._head_optimizer_dict.values(): + optim.zero_grad() + + # Accumulate gradients from all losses + for loss in losses: + loss.backward() + + # Apply gradients + if self._has_trainable_shared_layers: + self._shared_stack_optimizer.step() + if self._has_trainable_heads: + for optim in self._head_optimizer_dict.values(): + optim.step() diff --git a/maro/utils/exception/error_code.py b/maro/utils/exception/error_code.py index f09dcd12f..0eb864fa3 100644 --- a/maro/utils/exception/error_code.py +++ b/maro/utils/exception/error_code.py @@ -40,5 +40,6 @@ 4003: "Wrong Agent Manager Mode", 4004: "Store Misalignment Error", 4005: "Invalid Episode", - 4006: "Infinite Training Loop" + 4006: "Infinite Training Loop", + 4007: "Missing Optimizer" } diff --git a/maro/utils/exception/rl_toolkit_exception.py b/maro/utils/exception/rl_toolkit_exception.py index c35e0b897..949100494 100644 --- a/maro/utils/exception/rl_toolkit_exception.py +++ b/maro/utils/exception/rl_toolkit_exception.py @@ -39,3 +39,9 @@ class InfiniteTrainingLoopError(MAROException): """Raised when the ``SimpleLearner``'s training loop becomes infinite.""" def __init__(self, msg: str = None): super().__init__(4006, msg) + + +class MissingOptimizerError(MAROException): + """Raised when the optimizers are missing when calling LearningModel's step() method.""" + def __init__(self, msg: str = None): + super().__init__(4007, msg) From 774e9385a4f6ee3ee11ac8b2f98047d0cc421486 Mon Sep 17 00:00:00 2001 From: ysqyang Date: Tue, 10 Nov 2020 15:23:16 +0800 Subject: [PATCH 157/337] modified example accordingly --- examples/cim/dqn/components/agent_manager.py | 5 ++--- maro/rl/algorithms/dqn.py | 1 - maro/rl/models/learning_model.py | 8 +++----- 3 files changed, 5 insertions(+), 9 deletions(-) diff --git a/examples/cim/dqn/components/agent_manager.py b/examples/cim/dqn/components/agent_manager.py index f668d1779..81aae2dea 100644 --- a/examples/cim/dqn/components/agent_manager.py +++ b/examples/cim/dqn/components/agent_manager.py @@ -25,13 +25,12 @@ def create_dqn_agents(agent_id_list, config): activation=nn.LeakyReLU, is_head=True, **config.algorithm.model - )] + )], + optimizer_opt=(RMSprop, config.algorithm.optimizer), ) algorithm = DQN( q_model=q_model, - optimizer_cls=RMSprop, - optimizer_params=config.algorithm.optimizer, loss_func=nn.functional.smooth_l1_loss, hyper_params=DQNHyperParams( **config.algorithm.hyper_parameters, diff --git a/maro/rl/algorithms/dqn.py b/maro/rl/algorithms/dqn.py index 1b6d52158..ff275a34c 100644 --- a/maro/rl/algorithms/dqn.py +++ b/maro/rl/algorithms/dqn.py @@ -9,7 +9,6 @@ from maro.rl.algorithms.abs_algorithm import AbsAlgorithm from maro.rl.models.abs_learning_model import AbsLearningModel -from maro.rl.models.learning_model import MultiHeadLearningModel class DQNHyperParams: diff --git a/maro/rl/models/learning_model.py b/maro/rl/models/learning_model.py index b34fd45e0..0f83b86bd 100644 --- a/maro/rl/models/learning_model.py +++ b/maro/rl/models/learning_model.py @@ -1,8 +1,6 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT license. -from typing import Dict, Tuple - import torch import torch.nn as nn @@ -17,7 +15,7 @@ class SingleHeadLearningModel(AbsLearningModel): The shared blocks must be chainable, i.e., the output dimension of a block must match the input dimension of its successor. """ - def __init__(self, block_list: list, optimizer_opt: Tuple = None): + def __init__(self, block_list: list, optimizer_opt: tuple = None): super().__init__() self._net = nn.Sequential(*block_list) self._is_trainable = optimizer_opt is not None @@ -72,9 +70,9 @@ class MultiHeadLearningModel(AbsLearningModel): def __init__( self, head_block_dict: dict, - head_optimizer_opt_dict: Dict[Tuple[torch.optim.Optimizer, dict]] = None, + head_optimizer_opt_dict: dict = None, shared_block_list: list = None, - shared_stack_optimizer_opt: Tuple[torch.optim.Optimizer, dict] = None + shared_stack_optimizer_opt: tuple = None ): super().__init__() self._has_shared_layers = shared_block_list is not None From a1d5f4636f2ebbb3c3780d8d2442db8b1e1180be Mon Sep 17 00:00:00 2001 From: ysqyang Date: Tue, 10 Nov 2020 15:34:31 +0800 Subject: [PATCH 158/337] fixed a bug --- maro/rl/models/learning_model.py | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/maro/rl/models/learning_model.py b/maro/rl/models/learning_model.py index 0f83b86bd..259b1c17a 100644 --- a/maro/rl/models/learning_model.py +++ b/maro/rl/models/learning_model.py @@ -26,10 +26,11 @@ def __init__(self, block_list: list, optimizer_opt: tuple = None): param.requires_grad = False def __getstate__(self): - return { - "_net": self.__dict__["_net"], - "_is_trainable": self.__dict__["_is_trainable"], - } + dic = self.__dict__.copy() + if "_optimizer" in dic: + del dic["_optimizer"] + + return dic def __setstate__(self, dic: dict): self.__dict__ = dic @@ -108,13 +109,12 @@ def __init__( param.requires_grad = False def __getstate__(self): - return { - "_net": self.__dict__["_net"], - "_head_keys": self.__dict__["_head_keys"], - "_has_shared_layers": self.__dict__["_has_shared_layers"], - "_has_trainable_shared_layers": self.__dict__["_has_trainable_shared_layers"], - "_has_trainable_heads": self.__dict__["_has_trainable_heads"] - } + dic = self.__dict__.copy() + if "_shared_stack_optimizer" in dic: + del dic["_shared_stack_optimizer"] + if "_head_optimizer_dict" in dic: + del dic["_head_optimizer_dict"] + return dic def __setstate__(self, dic: dict): self.__dict__ = dic From aec8374f42fbc33cd0257daf64ca89dcaa12907b Mon Sep 17 00:00:00 2001 From: ysqyang Date: Tue, 10 Nov 2020 15:47:09 +0800 Subject: [PATCH 159/337] fixed a bug --- maro/rl/algorithms/dqn.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/maro/rl/algorithms/dqn.py b/maro/rl/algorithms/dqn.py index ff275a34c..98eb62e05 100644 --- a/maro/rl/algorithms/dqn.py +++ b/maro/rl/algorithms/dqn.py @@ -47,14 +47,14 @@ class DQN(AbsAlgorithm): Args: q_model (nn.Module): Q-value model. - loss_func (Callable): Loss function for the value model. + loss_cls (Callable): Loss function class for computing TD error. hyper_params: Hyper-parameter set for the DQN algorithm. """ - def __init__(self, q_model: AbsLearningModel, loss_func: Callable, hyper_params: DQNHyperParams): + def __init__(self, q_model: AbsLearningModel, loss_cls: Callable, hyper_params: DQNHyperParams): super().__init__() self._device = torch.device("cuda" if torch.cuda.is_available() else "cpu") self._hyper_params = hyper_params - self._loss_func = loss_func + self._loss_func = loss_cls(reduction="none") self._training_counter = 0 self._eval_model = q_model.to(self._device) @@ -70,7 +70,9 @@ def choose_action(self, state: np.ndarray, epsilon: float = None): return np.random.choice(self._hyper_params.num_actions) - def _compute_loss(self, states: np.ndarray, actions: np.ndarray, rewards: np.ndarray, next_states: np.ndarray): + def _compute_td_errors( + self, states: np.ndarray, actions: np.ndarray, rewards: np.ndarray, next_states: np.ndarray + ): states = torch.from_numpy(states).to(self._device) # (N, state_dim) actions = torch.from_numpy(actions).to(self._device) # (N,) rewards = torch.from_numpy(rewards).to(self._device) # (N,) @@ -84,14 +86,14 @@ def _compute_loss(self, states: np.ndarray, actions: np.ndarray, rewards: np.nda return self._loss_func(current_q_values, target_q_values) def train(self, states: np.ndarray, actions: np.ndarray, rewards: np.ndarray, next_states: np.ndarray): - loss = self._compute_loss(states, actions, rewards, next_states) - self._eval_model.step(loss) + td_errors = self._compute_td_errors(states, actions, rewards, next_states) + self._eval_model.step(td_errors.mean()) self._training_counter += 1 if self._training_counter % self._hyper_params.target_update_frequency == 0: self._update_targets() - return loss.detach().numpy() + return td_errors.detach().numpy() def _update_targets(self): for eval_params, target_params in zip( From e76efa599da21e584a1384618400f6afe87581e5 Mon Sep 17 00:00:00 2001 From: ysqyang Date: Tue, 10 Nov 2020 15:48:33 +0800 Subject: [PATCH 160/337] fixed a bug --- examples/cim/dqn/components/agent_manager.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/cim/dqn/components/agent_manager.py b/examples/cim/dqn/components/agent_manager.py index 81aae2dea..37524124b 100644 --- a/examples/cim/dqn/components/agent_manager.py +++ b/examples/cim/dqn/components/agent_manager.py @@ -31,7 +31,7 @@ def create_dqn_agents(agent_id_list, config): algorithm = DQN( q_model=q_model, - loss_func=nn.functional.smooth_l1_loss, + loss_cls=nn.SmoothL1Loss, hyper_params=DQNHyperParams( **config.algorithm.hyper_parameters, num_actions=num_actions From 7c229d78fd4dfd54a2c9ed3d7db983756a971d34 Mon Sep 17 00:00:00 2001 From: ysqyang Date: Tue, 10 Nov 2020 21:10:58 +0800 Subject: [PATCH 161/337] added dueling DQN feature --- docs/source/examples/cim.rst | 2 +- docs/source/examples/multi_agent_dqn_cim.rst | 2 +- examples/cim/dqn/components/agent_manager.py | 10 +-- examples/cim/dqn/config.yml | 4 +- maro/rl/__init__.py | 11 +-- maro/rl/algorithms/dqn.py | 78 +++++++++++++------ maro/rl/models/learning_model.py | 11 ++- .../rl_formulation.ipynb | 4 +- 8 files changed, 81 insertions(+), 41 deletions(-) diff --git a/docs/source/examples/cim.rst b/docs/source/examples/cim.rst index 17ea1642d..eaecbdf80 100644 --- a/docs/source/examples/cim.rst +++ b/docs/source/examples/cim.rst @@ -152,7 +152,7 @@ the DQN algorithm and an experience pool for each agent. algorithm = DQN(model_dict={"eval": eval_model}, optimizer_opt=(RMSprop, config.agents.algorithm.optimizer), loss_func_dict={"eval": smooth_l1_loss}, - hyper_params=DQNHyperParams(**config.agents.algorithm.hyper_parameters, + hyper_params=DQNConfig(**config.agents.algorithm.hyper_parameters, num_actions=num_actions)) experience_pool = ColumnBasedStore(**config.agents.experience_pool) diff --git a/docs/source/examples/multi_agent_dqn_cim.rst b/docs/source/examples/multi_agent_dqn_cim.rst index 0f803090f..5a81d5fdc 100644 --- a/docs/source/examples/multi_agent_dqn_cim.rst +++ b/docs/source/examples/multi_agent_dqn_cim.rst @@ -140,7 +140,7 @@ experience pools before training, in accordance with the DQN algorithm. optimizer_cls=RMSprop, optimizer_params=config.algorithm.optimizer, loss_func=nn.functional.smooth_l1_loss, - hyper_params=DQNHyperParams( + hyper_params=DQNConfig( **config.algorithm.hyper_parameters, num_actions=num_actions ) diff --git a/examples/cim/dqn/components/agent_manager.py b/examples/cim/dqn/components/agent_manager.py index 37524124b..e1eeceabb 100644 --- a/examples/cim/dqn/components/agent_manager.py +++ b/examples/cim/dqn/components/agent_manager.py @@ -5,7 +5,7 @@ from torch.optim import RMSprop from maro.rl import ( - ColumnBasedStore, DQN, DQNHyperParams, FullyConnectedBlock, SimpleAgentManager, SingleHeadLearningModel + ColumnBasedStore, DQN, DQNConfig, FullyConnectedBlock, SimpleAgentManager, SingleTaskLearningModel ) from maro.utils import set_seeds @@ -17,7 +17,7 @@ def create_dqn_agents(agent_id_list, config): set_seeds(config.seed) agent_dict = {} for agent_id in agent_id_list: - q_model = SingleHeadLearningModel( + q_model = SingleTaskLearningModel( [FullyConnectedBlock( name=f'{agent_id}.policy', input_dim=config.algorithm.input_dim, @@ -30,10 +30,10 @@ def create_dqn_agents(agent_id_list, config): ) algorithm = DQN( - q_model=q_model, + core_model=q_model, loss_cls=nn.SmoothL1Loss, - hyper_params=DQNHyperParams( - **config.algorithm.hyper_parameters, + config=DQNConfig( + **config.algorithm.config, num_actions=num_actions ) ) diff --git a/examples/cim/dqn/config.yml b/examples/cim/dqn/config.yml index a9e2fa856..3333af646 100644 --- a/examples/cim/dqn/config.yml +++ b/examples/cim/dqn/config.yml @@ -55,11 +55,13 @@ agents: dropout_p: 0.0 optimizer: lr: 0.05 - hyper_parameters: + config: reward_decay: .0 target_update_frequency: 5 tau: 0.1 is_double: false + advantage_mode: "mean" + per_sample_td_error_enabled: true experience_pool: capacity: -1 training_loop_parameters: diff --git a/maro/rl/__init__.py b/maro/rl/__init__.py index a0c8453a3..fd9128260 100644 --- a/maro/rl/__init__.py +++ b/maro/rl/__init__.py @@ -7,7 +7,7 @@ from maro.rl.agent.abs_agent_manager import AbsAgentManager, AgentManagerMode from maro.rl.agent.simple_agent_manager import SimpleAgentManager from maro.rl.algorithms.abs_algorithm import AbsAlgorithm -from maro.rl.algorithms.dqn import DQN, DQNHyperParams +from maro.rl.algorithms.dqn import DQN, DQNConfig, DuelingHead from maro.rl.dist_topologies.experience_collection import ( concat_experiences_by_agent, merge_experiences_with_trajectory_boundaries ) @@ -22,7 +22,7 @@ from maro.rl.learner.simple_learner import SimpleLearner from maro.rl.models.abs_learning_model import AbsLearningModel from maro.rl.models.fc_block import FullyConnectedBlock -from maro.rl.models.learning_model import MultiHeadLearningModel, SingleHeadLearningModel +from maro.rl.models.learning_model import MultiTaskLearningModel, SingleTaskLearningModel from maro.rl.shaping.abs_shaper import AbsShaper from maro.rl.shaping.action_shaper import ActionShaper from maro.rl.shaping.experience_shaper import ExperienceShaper @@ -49,20 +49,21 @@ 'AgentManagerMode', 'ColumnBasedStore', 'DQN', - 'DQNHyperParams', + 'DQNConfig', + 'DuelingHead', 'ExperienceShaper', 'FullyConnectedBlock', 'KStepExperienceShaper', 'LinearExplorer', 'MaxDeltaEarlyStoppingChecker', - 'MultiHeadLearningModel', + 'MultiTaskLearningModel', 'OverwriteType', 'RSDEarlyStoppingChecker', 'SimpleActor', 'SimpleAgentManager', 'SimpleEarlyStoppingChecker', 'SimpleLearner', - 'SingleHeadLearningModel', + 'SingleTaskLearningModel', 'StateShaper', 'TwoPhaseLinearExplorer', 'concat_experiences_by_agent', diff --git a/maro/rl/algorithms/dqn.py b/maro/rl/algorithms/dqn.py index 98eb62e05..6f17f8f97 100644 --- a/maro/rl/algorithms/dqn.py +++ b/maro/rl/algorithms/dqn.py @@ -1,6 +1,7 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT license. +from enum import Enum from typing import Callable import numpy as np @@ -9,10 +10,16 @@ from maro.rl.algorithms.abs_algorithm import AbsAlgorithm from maro.rl.models.abs_learning_model import AbsLearningModel +from maro.rl.models.learning_model import MultiTaskLearningModel -class DQNHyperParams: - """Hyper-parameter set for the DQN algorithm. +class DuelingHead(Enum): + STATE_VALUE = "state_value" + ADVANTAGE = "advantage" + + +class DQNConfig: + """Configuration for the DQN algorithm. Args: num_actions (int): Number of possible actions. @@ -22,8 +29,15 @@ class DQNHyperParams: is_double (bool): If True, the next Q values will be computed according to the double DQN algorithm, i.e., q_next = Q_target(s, argmax(Q_eval(s, a))). Otherwise, q_next = max(Q_target(s, a)). See https://arxiv.org/pdf/1509.06461.pdf for details. Defaults to False. + advantage_mode (str): advantage mode for the dueling architecture. Defaults to None, in which + case it is assumed that the regular Q-value model is used. + per_sample_td_error_enabled (bool): If True, per-sample TD errors will be returned by the DQN's train() + method. Defaults to False. """ - __slots__ = ["num_actions", "reward_decay", "target_update_frequency", "tau", "is_double"] + __slots__ = [ + "num_actions", "reward_decay", "target_update_frequency", "tau", "is_double", "advantage_mode", + "per_sample_td_error_enabled" + ] def __init__( self, @@ -31,13 +45,17 @@ def __init__( reward_decay: float, target_update_frequency: int, tau: float = 0.1, - is_double: bool = True + is_double: bool = True, + advantage_mode: str = None, + per_sample_td_error_enabled: bool = False ): self.num_actions = num_actions self.reward_decay = reward_decay self.target_update_frequency = target_update_frequency self.tau = tau self.is_double = is_double + self.advantage_mode = advantage_mode + self.per_sample_td_error_enabled = per_sample_td_error_enabled class DQN(AbsAlgorithm): @@ -46,19 +64,27 @@ class DQN(AbsAlgorithm): See https://web.stanford.edu/class/psych209/Readings/MnihEtAlHassibis15NatureControlDeepRL.pdf for details. Args: - q_model (nn.Module): Q-value model. + core_model (nn.Module): Q-value model. loss_cls (Callable): Loss function class for computing TD error. - hyper_params: Hyper-parameter set for the DQN algorithm. + config: Configuration for DQN algorithm. """ - def __init__(self, q_model: AbsLearningModel, loss_cls: Callable, hyper_params: DQNHyperParams): + def __init__(self, core_model: AbsLearningModel, loss_cls: Callable, config: DQNConfig): super().__init__() self._device = torch.device("cuda" if torch.cuda.is_available() else "cpu") - self._hyper_params = hyper_params - self._loss_func = loss_cls(reduction="none") + self._config = config + self._loss_func = loss_cls(reduction="none" if self._config.per_sample_td_error_enabled else "mean") self._training_counter = 0 - self._eval_model = q_model.to(self._device) - self._target_model = q_model.copy().to(self._device) if self._eval_model.is_trainable else None + if self._config.advantage_mode is not None: + assert isinstance(core_model, MultiTaskLearningModel), \ + "core_model must be a MultiTaskLearningModel if dueling architecture is used." + assert DuelingHead.STATE_VALUE.value in core_model.heads, \ + "core_mode; must have a task head named 'state_value'" + assert DuelingHead.ADVANTAGE.value in core_model.heads, \ + "core_model must have a task head named 'advantage'" + + self._eval_model = core_model.to(self._device) + self._target_model = core_model.copy().to(self._device) if self._eval_model.is_trainable else None def choose_action(self, state: np.ndarray, epsilon: float = None): if epsilon is None or np.random.rand() > epsilon: @@ -68,7 +94,7 @@ def choose_action(self, state: np.ndarray, epsilon: float = None): q_values = self._get_q_values(state) return q_values.argmax(dim=1).item() - return np.random.choice(self._hyper_params.num_actions) + return np.random.choice(self._config.num_actions) def _compute_td_errors( self, states: np.ndarray, actions: np.ndarray, rewards: np.ndarray, next_states: np.ndarray @@ -82,33 +108,41 @@ def _compute_td_errors( current_q_values_for_all_actions = self._get_q_values(states) current_q_values = current_q_values_for_all_actions.gather(1, actions).squeeze(1) # (N,) next_q_values = self._get_next_q_values(current_q_values_for_all_actions, next_states) # (N,) - target_q_values = (rewards + self._hyper_params.reward_decay * next_q_values).detach() # (N,) + target_q_values = (rewards + self._config.reward_decay * next_q_values).detach() # (N,) return self._loss_func(current_q_values, target_q_values) def train(self, states: np.ndarray, actions: np.ndarray, rewards: np.ndarray, next_states: np.ndarray): - td_errors = self._compute_td_errors(states, actions, rewards, next_states) - self._eval_model.step(td_errors.mean()) - + loss = self._compute_td_errors(states, actions, rewards, next_states) + self._eval_model.step(loss.mean() if self._config.per_sample_td_error_enabled else loss) self._training_counter += 1 - if self._training_counter % self._hyper_params.target_update_frequency == 0: + if self._training_counter % self._config.target_update_frequency == 0: self._update_targets() - return td_errors.detach().numpy() + return loss.detach().numpy() def _update_targets(self): for eval_params, target_params in zip( self._eval_model.parameters(), self._target_model.parameters() ): target_params.data = ( - self._hyper_params.tau * eval_params.data + (1 - self._hyper_params.tau) * target_params.data + self._config.tau * eval_params.data + (1 - self._config.tau) * target_params.data ) def _get_q_values(self, states, is_target: bool = False): - model = self._target_model if is_target else self._eval_model - return model(states) + if self._config.advantage_mode is not None: + output = self._target_model(states) if is_target else self._eval_model(states) + state_values = output["state_value"] + advantages = output["advantage"] + # Use mean or max correction to address the identifiability issue + corrections = advantages.mean(1) if self._config.advantage_mode == "mean" else advantages.max(1)[0] + q_values = state_values + advantages - corrections.unsqueeze(1) + return q_values + else: + model = self._target_model if is_target else self._eval_model + return model(states) def _get_next_q_values(self, current_q_values_all, next_states): - if self._hyper_params.is_double: + if self._config.is_double: actions = current_q_values_all.max(dim=1)[1].unsqueeze(1) return self._get_q_values(next_states, is_target=True).gather(1, actions).squeeze(1) # (N,) else: diff --git a/maro/rl/models/learning_model.py b/maro/rl/models/learning_model.py index 259b1c17a..09c758874 100644 --- a/maro/rl/models/learning_model.py +++ b/maro/rl/models/learning_model.py @@ -9,10 +9,10 @@ from .abs_learning_model import AbsLearningModel -class SingleHeadLearningModel(AbsLearningModel): - """NN model that consists of shared blocks and multiple task heads. +class SingleTaskLearningModel(AbsLearningModel): + """NN model that consists of a sequence of chainable blocks. - The shared blocks must be chainable, i.e., the output dimension of a block must match the input dimension of + The blocks must be chainable, i.e., the output dimension of a block must match the input dimension of its successor. """ def __init__(self, block_list: list, optimizer_opt: tuple = None): @@ -60,7 +60,7 @@ def copy(self): return clone(self) -class MultiHeadLearningModel(AbsLearningModel): +class MultiTaskLearningModel(AbsLearningModel): """NN model that consists of shared blocks and multiple task heads. The shared blocks must be chainable, i.e., the output dimension of a block must match the input dimension of @@ -119,6 +119,9 @@ def __getstate__(self): def __setstate__(self, dic: dict): self.__dict__ = dic + def __getitem__(self, head_key): + return self._net[head_key] + @property def has_shared_layers(self): return self._has_shared_layers diff --git a/notebooks/container_inventory_management/rl_formulation.ipynb b/notebooks/container_inventory_management/rl_formulation.ipynb index 903a1984f..d4fa32e43 100644 --- a/notebooks/container_inventory_management/rl_formulation.ipynb +++ b/notebooks/container_inventory_management/rl_formulation.ipynb @@ -229,7 +229,7 @@ "from torch.nn.functional import smooth_l1_loss\n", "from torch.optim import RMSprop\n", "\n", - "from maro.rl import SimpleAgentManager, LearningModel, FullyConnectedNet, DQN, DQNHyperParams, ColumnBasedStore\n", + "from maro.rl import SimpleAgentManager, LearningModel, FullyConnectedNet, DQN, DQNConfig, ColumnBasedStore\n", "\n", "\n", "input_dim = 171\n", @@ -258,7 +258,7 @@ " optimizer_cls=RMSprop,\n", " optimizer_params={\"lr\": 0.05},\n", " loss_func=nn.functional.smooth_l1_loss,\n", - " hyper_params=DQNHyperParams(\n", + " hyper_params=DQNConfig(\n", " num_actions=num_actions,\n", " reward_decay=.0,\n", " target_update_frequency=5,\n", From f42ec3e0a5fc3f220fea161bcf5e30f9fc2f8292 Mon Sep 17 00:00:00 2001 From: ysqyang Date: Tue, 10 Nov 2020 23:39:21 +0800 Subject: [PATCH 162/337] revised and refined docstrings --- maro/rl/algorithms/dqn.py | 6 +-- maro/rl/models/learning_model.py | 80 +++++++++++++++++--------------- 2 files changed, 46 insertions(+), 40 deletions(-) diff --git a/maro/rl/algorithms/dqn.py b/maro/rl/algorithms/dqn.py index 6f17f8f97..fdd2bf9d5 100644 --- a/maro/rl/algorithms/dqn.py +++ b/maro/rl/algorithms/dqn.py @@ -78,9 +78,9 @@ def __init__(self, core_model: AbsLearningModel, loss_cls: Callable, config: DQN if self._config.advantage_mode is not None: assert isinstance(core_model, MultiTaskLearningModel), \ "core_model must be a MultiTaskLearningModel if dueling architecture is used." - assert DuelingHead.STATE_VALUE.value in core_model.heads, \ - "core_mode; must have a task head named 'state_value'" - assert DuelingHead.ADVANTAGE.value in core_model.heads, \ + assert DuelingHead.STATE_VALUE.value in core_model.tasks, \ + "core_model must have a task head named 'state_value'" + assert DuelingHead.ADVANTAGE.value in core_model.tasks, \ "core_model must have a task head named 'advantage'" self._eval_model = core_model.to(self._device) diff --git a/maro/rl/models/learning_model.py b/maro/rl/models/learning_model.py index 09c758874..4d8013cf5 100644 --- a/maro/rl/models/learning_model.py +++ b/maro/rl/models/learning_model.py @@ -12,8 +12,10 @@ class SingleTaskLearningModel(AbsLearningModel): """NN model that consists of a sequence of chainable blocks. - The blocks must be chainable, i.e., the output dimension of a block must match the input dimension of - its successor. + Args: + block_list (list): List of blocks that compose the model. They must be chainable, i.e., the output dimension + of a block must match the input dimension of its successor. + optimizer_opt (tuple): Optimizer option for the model. Default to None. """ def __init__(self, block_list: list, optimizer_opt: tuple = None): super().__init__() @@ -61,57 +63,61 @@ def copy(self): class MultiTaskLearningModel(AbsLearningModel): - """NN model that consists of shared blocks and multiple task heads. - - The shared blocks must be chainable, i.e., the output dimension of a block must match the input dimension of - its successor. Heads must be provided in the form of keyword arguments. If at least one head is provided, the - output of the model will be a dictionary with the names of the heads as keys and the corresponding head outputs - as values. Otherwise, the output will be the output of the last block. + """NN model that consists of multiple task heads and an optional shared stack. + + Args: + task_block_dict (dict): Dictionary of network blocks that perform designated tasks. + task_optimizer_opt_dict (dict): Dictionary of optimizer options for each task block. An optimizer option + is specified in the form of a tuple: (optimizer class, optimizer parameters). Defaults to None. + shared_block_list (list): List of blocks that compose the bottom stack of the model shared by all tasks. + The shared blocks must be chainable, i.e., the output dimension of a block must match the input dimension + of its successor. Defaults to None. + shared_optimizer_opt (tuple): Optimizer option for the shared part of the model. Default to None. """ def __init__( self, - head_block_dict: dict, - head_optimizer_opt_dict: dict = None, + task_block_dict: dict, + task_optimizer_opt_dict: dict = None, shared_block_list: list = None, - shared_stack_optimizer_opt: tuple = None + shared_optimizer_opt: tuple = None ): super().__init__() self._has_shared_layers = shared_block_list is not None - self._has_trainable_shared_layers = self._has_shared_layers and shared_stack_optimizer_opt is not None - self._has_trainable_heads = head_optimizer_opt_dict is not None + self._has_trainable_shared_layers = self._has_shared_layers and shared_optimizer_opt is not None + self._has_trainable_heads = task_optimizer_opt_dict is not None # shared stack if self._has_shared_layers: self._shared_stack = nn.Sequential(*shared_block_list) if self._has_trainable_shared_layers: - self._shared_stack_optimizer = shared_stack_optimizer_opt[0]( - self._shared_stack.parameters(), **shared_stack_optimizer_opt[1] + self._shared_optimizer = shared_optimizer_opt[0]( + self._shared_stack.parameters(), **shared_optimizer_opt[1] ) else: for param in self._shared_stack.parameters(): param.requires_grad = False # heads - self._head_keys = list(head_block_dict.keys()) + self._tasks = list(task_block_dict.keys()) self._net = nn.ModuleDict({ key: nn.Sequential(self._shared_stack, head) if self._has_shared_layers else head - for key, head in head_block_dict.items() + for key, head in task_block_dict.items() }) if self._has_trainable_heads: self._head_optimizer_dict = { - key: head_optimizer_opt_dict[key][0](head.parameters(), **head_optimizer_opt_dict[key][1]) - for key, head in head_block_dict.items() + key: task_optimizer_opt_dict[key][0](head.parameters(), **task_optimizer_opt_dict[key][1]) + for key, head in task_block_dict.items() } else: - for key, head in head_block_dict.items(): + for key, head in task_block_dict.items(): for param in head.parameters(): param.requires_grad = False def __getstate__(self): dic = self.__dict__.copy() - if "_shared_stack_optimizer" in dic: - del dic["_shared_stack_optimizer"] + if "_shared_optimizer" in dic: + del dic["_shared_optimizer"] if "_head_optimizer_dict" in dic: del dic["_head_optimizer_dict"] return dic @@ -119,8 +125,8 @@ def __getstate__(self): def __setstate__(self, dic: dict): self.__dict__ = dic - def __getitem__(self, head_key): - return self._net[head_key] + def __getitem__(self, task): + return self._net[task] @property def has_shared_layers(self): @@ -147,29 +153,29 @@ def is_trainable(self): return self._has_trainable_shared_layers or self._has_trainable_heads @property - def heads(self) -> [str]: - return self._head_keys + def tasks(self) -> [str]: + return self._tasks - def forward(self, inputs, key=None): + def forward(self, inputs, task=None): """Feedforward computations for the given head(s). Args: inputs: Inputs to the model. - key: The key(s) to the head(s) from which the output is required. If this is None, the results from - all heads will be returned in the form of a dictionary. If this is a list, the results will be the - outputs from the heads contained in head_key in the form of a dictionary. If this is a single key, + task: The task for which the network output is required. If this is None, the results from all task + heads will be returned in the form of a dictionary. If this is a list, the results will be the + outputs from the heads contained in task in the form of a dictionary. If this is a single key, the result will be the output from the corresponding head. Returns: Outputs from the required head(s). """ - if key is None: - return {key: self._net[key](inputs) for key in self._head_keys} + if task is None: + return {key: self._net[key](inputs) for key in self._task_keys} - if isinstance(key, list): - return {k: self._net[k](inputs) for k in key} + if isinstance(task, list): + return {k: self._net[k](inputs) for k in task} else: - return self._net[key](inputs) + return self._net[task](inputs) def step(self, *losses): """Use the losses to back-propagate gradients and apply them to the underlying parameters.""" @@ -178,7 +184,7 @@ def step(self, *losses): # Zero all gradients if self._has_trainable_shared_layers: - self._shared_stack_optimizer.zero_grad() + self._shared_optimizer.zero_grad() if self._has_trainable_heads: for optim in self._head_optimizer_dict.values(): optim.zero_grad() @@ -189,7 +195,7 @@ def step(self, *losses): # Apply gradients if self._has_trainable_shared_layers: - self._shared_stack_optimizer.step() + self._shared_optimizer.step() if self._has_trainable_heads: for optim in self._head_optimizer_dict.values(): optim.step() From 55db382660e3be8d2b08ac6269dad794d00c86bf Mon Sep 17 00:00:00 2001 From: ysqyang Date: Tue, 10 Nov 2020 23:43:44 +0800 Subject: [PATCH 163/337] fixed a bug --- maro/rl/models/learning_model.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/maro/rl/models/learning_model.py b/maro/rl/models/learning_model.py index 4d8013cf5..bb3394408 100644 --- a/maro/rl/models/learning_model.py +++ b/maro/rl/models/learning_model.py @@ -170,7 +170,7 @@ def forward(self, inputs, task=None): Outputs from the required head(s). """ if task is None: - return {key: self._net[key](inputs) for key in self._task_keys} + return {key: self._net[key](inputs) for key in self._tasks} if isinstance(task, list): return {k: self._net[k](inputs) for k in task} From 714936292d3174c986b4ed75a07a71c7fc72fa33 Mon Sep 17 00:00:00 2001 From: ysqyang Date: Tue, 10 Nov 2020 23:47:05 +0800 Subject: [PATCH 164/337] fixed lint issues --- maro/rl/algorithms/dqn.py | 3 +-- maro/rl/models/learning_model.py | 1 + 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/maro/rl/algorithms/dqn.py b/maro/rl/algorithms/dqn.py index fdd2bf9d5..eb62a7b90 100644 --- a/maro/rl/algorithms/dqn.py +++ b/maro/rl/algorithms/dqn.py @@ -6,7 +6,6 @@ import numpy as np import torch -import torch.nn as nn from maro.rl.algorithms.abs_algorithm import AbsAlgorithm from maro.rl.models.abs_learning_model import AbsLearningModel @@ -64,7 +63,7 @@ class DQN(AbsAlgorithm): See https://web.stanford.edu/class/psych209/Readings/MnihEtAlHassibis15NatureControlDeepRL.pdf for details. Args: - core_model (nn.Module): Q-value model. + core_model (AbsLearningModel): Q-value model. loss_cls (Callable): Loss function class for computing TD error. config: Configuration for DQN algorithm. """ diff --git a/maro/rl/models/learning_model.py b/maro/rl/models/learning_model.py index bb3394408..62f4e9d7d 100644 --- a/maro/rl/models/learning_model.py +++ b/maro/rl/models/learning_model.py @@ -6,6 +6,7 @@ from maro.utils import clone from maro.utils.exception.rl_toolkit_exception import MissingOptimizerError + from .abs_learning_model import AbsLearningModel From 29175a3c83025e32aa7a104f4b68bb88b2b4b85b Mon Sep 17 00:00:00 2001 From: ysqyang Date: Wed, 11 Nov 2020 16:48:11 +0800 Subject: [PATCH 165/337] added load/dump functions to LearningModel --- examples/cim/dqn/components/agent.py | 9 +++++++ maro/rl/agent/abs_agent.py | 23 ++++++---------- maro/rl/algorithms/abs_algorithm.py | 35 ++++++++++--------------- maro/rl/algorithms/dqn.py | 39 ++++++++++++++-------------- maro/rl/models/abs_learning_model.py | 13 ++++++++++ maro/rl/models/learning_model.py | 7 ----- 6 files changed, 62 insertions(+), 64 deletions(-) diff --git a/examples/cim/dqn/components/agent.py b/examples/cim/dqn/components/agent.py index 4fc8e15f1..deb8f0ff7 100644 --- a/examples/cim/dqn/components/agent.py +++ b/examples/cim/dqn/components/agent.py @@ -1,6 +1,9 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT license. +import os +import pickle + import numpy as np from maro.rl import AbsAgent, ColumnBasedStore @@ -38,3 +41,9 @@ def train(self): next_state = np.asarray(sample["next_state"]) loss = self._algorithm.train(state, action, reward, next_state) self._experience_pool.update(indexes, {"loss": loss}) + + def dump_experience_pool(self, dir_path: str): + """Dump the experience pool to disk.""" + os.makedirs(dir_path, exist_ok=True) + with open(os.path.join(dir_path, self._name), "wb") as fp: + pickle.dump(self._experience_pool, fp) diff --git a/maro/rl/agent/abs_agent.py b/maro/rl/agent/abs_agent.py index 4a68c256a..b77aefa17 100644 --- a/maro/rl/agent/abs_agent.py +++ b/maro/rl/agent/abs_agent.py @@ -66,15 +66,15 @@ def store_experiences(self, experiences): if self._experience_pool is not None: self._experience_pool.put(experiences) - def load_models(self, *models, **model_dict): + def load_model(self, model): """Load models from memory.""" - self._algorithm.load_models(*models, **model_dict) + self._algorithm.model.load(model) - def dump_models(self): + def dump_model(self): """Return the algorithm's trainable models.""" - return self._algorithm.dump_models() + return self._algorithm.model.dump() - def load_models_from_file(self, dir_path: str): + def load_model_from_file(self, dir_path: str): """Load trainable models from disk. Load trainable models from the specified directory. The model file is always prefixed with the agent's name. @@ -82,9 +82,9 @@ def load_models_from_file(self, dir_path: str): Args: dir_path (str): path to the directory where the models are saved. """ - self._algorithm.load_models_from_file(os.path.join(dir_path, self._name)) + self._algorithm.model.load_from_file(os.path.join(dir_path, self._name)) - def dump_models_to_file(self, dir_path: str): + def dump_model_to_file(self, dir_path: str): """Dump the algorithm's trainable models to disk. Dump trainable models to the specified directory. The model file is always prefixed with the agent's name. @@ -92,11 +92,4 @@ def dump_models_to_file(self, dir_path: str): Args: dir_path (str): path to the directory where the models are saved. """ - self._algorithm.dump_models_to_file(os.path.join(dir_path, self._name)) - - def dump_experience_pool(self, dir_path: str): - """Dump the experience pool to disk.""" - if self._experience_pool is not None: - os.makedirs(dir_path, exist_ok=True) - with open(os.path.join(dir_path, self._name), "wb") as fp: - pickle.dump(self._experience_pool, fp) + self._algorithm.model.dump_to_file(os.path.join(dir_path, self._name)) diff --git a/maro/rl/algorithms/abs_algorithm.py b/maro/rl/algorithms/abs_algorithm.py index f1f8d7b00..84e998a1f 100644 --- a/maro/rl/algorithms/abs_algorithm.py +++ b/maro/rl/algorithms/abs_algorithm.py @@ -3,6 +3,8 @@ from abc import ABC, abstractmethod +from maro.rl.models.abs_learning_model import AbsLearningModel + class AbsAlgorithm(ABC): """Abstract RL algorithm class. @@ -10,9 +12,18 @@ class AbsAlgorithm(ABC): The class provides uniform policy interfaces such as ``choose_action`` and ``train``. We also provide some predefined RL algorithm based on it, such DQN, A2C, etc. User can inherit from it to customize their own algorithms. + + Args: + core_model (AbsLearningModel): Task model or container of task models required by the algorithm. + config: Settings for the algorithm. """ - def __init__(self): - pass + def __init__(self, core_model: AbsLearningModel, config): + self._core_model = core_model + self._config = config + + @property + def model(self): + return self._core_model @abstractmethod def choose_action(self, state, epsilon: float = None): @@ -38,23 +49,3 @@ def train(self, *args, **kwargs): algorithm, this may look like train(self, state_batch, action_batch, reward_batch, next_state_batch). """ return NotImplementedError - - @abstractmethod - def load_models(self, *models, **model_dict): - """Load trainable models from memory.""" - return NotImplementedError - - @abstractmethod - def dump_models(self): - """Return the algorithm's trainable models.""" - return NotImplementedError - - @abstractmethod - def load_models_from_file(self, path): - """Load trainable models from disk.""" - return NotImplementedError - - @abstractmethod - def dump_models_to_file(self, path: str): - """Dump the algorithm's trainable models to disk.""" - return NotImplementedError diff --git a/maro/rl/algorithms/dqn.py b/maro/rl/algorithms/dqn.py index eb62a7b90..bd679d42c 100644 --- a/maro/rl/algorithms/dqn.py +++ b/maro/rl/algorithms/dqn.py @@ -2,7 +2,6 @@ # Licensed under the MIT license. from enum import Enum -from typing import Callable import numpy as np import torch @@ -32,16 +31,18 @@ class DQNConfig: case it is assumed that the regular Q-value model is used. per_sample_td_error_enabled (bool): If True, per-sample TD errors will be returned by the DQN's train() method. Defaults to False. + loss_cls: """ __slots__ = [ - "num_actions", "reward_decay", "target_update_frequency", "tau", "is_double", "advantage_mode", - "per_sample_td_error_enabled" + "num_actions", "reward_decay", "loss_func", "target_update_frequency", "tau", "is_double", + "advantage_mode", "per_sample_td_error_enabled" ] def __init__( self, num_actions: int, reward_decay: float, + loss_cls, target_update_frequency: int, tau: float = 0.1, is_double: bool = True, @@ -55,6 +56,7 @@ def __init__( self.is_double = is_double self.advantage_mode = advantage_mode self.per_sample_td_error_enabled = per_sample_td_error_enabled + self.loss_func = loss_cls(reduction="none" if per_sample_td_error_enabled else "mean") class DQN(AbsAlgorithm): @@ -64,14 +66,11 @@ class DQN(AbsAlgorithm): Args: core_model (AbsLearningModel): Q-value model. - loss_cls (Callable): Loss function class for computing TD error. config: Configuration for DQN algorithm. """ - def __init__(self, core_model: AbsLearningModel, loss_cls: Callable, config: DQNConfig): - super().__init__() + def __init__(self, core_model: AbsLearningModel, config: DQNConfig): + super().__init__(core_model, config) self._device = torch.device("cuda" if torch.cuda.is_available() else "cpu") - self._config = config - self._loss_func = loss_cls(reduction="none" if self._config.per_sample_td_error_enabled else "mean") self._training_counter = 0 if self._config.advantage_mode is not None: @@ -82,13 +81,13 @@ def __init__(self, core_model: AbsLearningModel, loss_cls: Callable, config: DQN assert DuelingHead.ADVANTAGE.value in core_model.tasks, \ "core_model must have a task head named 'advantage'" - self._eval_model = core_model.to(self._device) - self._target_model = core_model.copy().to(self._device) if self._eval_model.is_trainable else None + self._core_model.to(self._device) + self._target_model = core_model.copy().to(self._device) if core_model.is_trainable else None def choose_action(self, state: np.ndarray, epsilon: float = None): if epsilon is None or np.random.rand() > epsilon: state = torch.from_numpy(state).unsqueeze(0) - self._eval_model.eval() + self._core_model.eval() with torch.no_grad(): q_values = self._get_q_values(state) return q_values.argmax(dim=1).item() @@ -108,11 +107,11 @@ def _compute_td_errors( current_q_values = current_q_values_for_all_actions.gather(1, actions).squeeze(1) # (N,) next_q_values = self._get_next_q_values(current_q_values_for_all_actions, next_states) # (N,) target_q_values = (rewards + self._config.reward_decay * next_q_values).detach() # (N,) - return self._loss_func(current_q_values, target_q_values) + return self._config.loss_func(current_q_values, target_q_values) def train(self, states: np.ndarray, actions: np.ndarray, rewards: np.ndarray, next_states: np.ndarray): loss = self._compute_td_errors(states, actions, rewards, next_states) - self._eval_model.step(loss.mean() if self._config.per_sample_td_error_enabled else loss) + self._core_model.step(loss.mean() if self._config.per_sample_td_error_enabled else loss) self._training_counter += 1 if self._training_counter % self._config.target_update_frequency == 0: self._update_targets() @@ -121,7 +120,7 @@ def train(self, states: np.ndarray, actions: np.ndarray, rewards: np.ndarray, ne def _update_targets(self): for eval_params, target_params in zip( - self._eval_model.parameters(), self._target_model.parameters() + self._core_model.parameters(), self._target_model.parameters() ): target_params.data = ( self._config.tau * eval_params.data + (1 - self._config.tau) * target_params.data @@ -129,7 +128,7 @@ def _update_targets(self): def _get_q_values(self, states, is_target: bool = False): if self._config.advantage_mode is not None: - output = self._target_model(states) if is_target else self._eval_model(states) + output = self._target_model(states) if is_target else self._core_model(states) state_values = output["state_value"] advantages = output["advantage"] # Use mean or max correction to address the identifiability issue @@ -137,7 +136,7 @@ def _get_q_values(self, states, is_target: bool = False): q_values = state_values + advantages - corrections.unsqueeze(1) return q_values else: - model = self._target_model if is_target else self._eval_model + model = self._target_model if is_target else self._core_model return model(states) def _get_next_q_values(self, current_q_values_all, next_states): @@ -149,16 +148,16 @@ def _get_next_q_values(self, current_q_values_all, next_states): def load_models(self, eval_model): """Load the evaluation model from memory.""" - self._eval_model.load_state_dict(eval_model) + self._core_model.load_state_dict(eval_model) def dump_models(self): """Return the evaluation model.""" - return self._eval_model.state_dict() + return self._core_model.state_dict() def load_models_from_file(self, path): """Load the evaluation model from disk.""" - self._eval_model.load_state_dict(torch.load(path)) + self._core_model.load_state_dict(torch.load(path)) def dump_models_to_file(self, path: str): """Dump the evaluation model to disk.""" - torch.save(self._eval_model.state_dict(), path) + torch.save(self._core_model.state_dict(), path) diff --git a/maro/rl/models/abs_learning_model.py b/maro/rl/models/abs_learning_model.py index f9320799b..50a879548 100644 --- a/maro/rl/models/abs_learning_model.py +++ b/maro/rl/models/abs_learning_model.py @@ -3,6 +3,7 @@ from abc import abstractmethod +import torch import torch.nn as nn from maro.utils import clone @@ -24,3 +25,15 @@ def step(self, *losses): def copy(self): return clone(self) + + def load(self, state_dict): + self.load_state_dict(state_dict) + + def dump(self): + return self.state_dict() + + def load_from_file(self, path: str): + self.load_state_dict(torch.load(path)) + + def dump_to_file(self, path: str): + torch.save(self.state_dict(), path) diff --git a/maro/rl/models/learning_model.py b/maro/rl/models/learning_model.py index 62f4e9d7d..bf4ceb0b5 100644 --- a/maro/rl/models/learning_model.py +++ b/maro/rl/models/learning_model.py @@ -4,7 +4,6 @@ import torch import torch.nn as nn -from maro.utils import clone from maro.utils.exception.rl_toolkit_exception import MissingOptimizerError from .abs_learning_model import AbsLearningModel @@ -59,9 +58,6 @@ def step(self, loss: torch.tensor): loss.backward() self._optimizer.step() - def copy(self): - return clone(self) - class MultiTaskLearningModel(AbsLearningModel): """NN model that consists of multiple task heads and an optional shared stack. @@ -200,6 +196,3 @@ def step(self, *losses): if self._has_trainable_heads: for optim in self._head_optimizer_dict.values(): optim.step() - - def copy(self): - return clone(self) From a8f9087bd4730e664cb43441af788e6c057d35a8 Mon Sep 17 00:00:00 2001 From: ysqyang Date: Wed, 11 Nov 2020 16:49:50 +0800 Subject: [PATCH 166/337] fixed a bug --- examples/cim/dqn/components/agent_manager.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/cim/dqn/components/agent_manager.py b/examples/cim/dqn/components/agent_manager.py index e1eeceabb..8c2434f13 100644 --- a/examples/cim/dqn/components/agent_manager.py +++ b/examples/cim/dqn/components/agent_manager.py @@ -31,9 +31,9 @@ def create_dqn_agents(agent_id_list, config): algorithm = DQN( core_model=q_model, - loss_cls=nn.SmoothL1Loss, config=DQNConfig( **config.algorithm.config, + loss_cls=nn.SmoothL1Loss, num_actions=num_actions ) ) From 9788114e1cf7e88aa899d63e3b1f2848c73439eb Mon Sep 17 00:00:00 2001 From: ysqyang Date: Wed, 11 Nov 2020 16:57:43 +0800 Subject: [PATCH 167/337] fixed a bug --- maro/rl/agent/simple_agent_manager.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/maro/rl/agent/simple_agent_manager.py b/maro/rl/agent/simple_agent_manager.py index c9d1bb875..4849d29b6 100644 --- a/maro/rl/agent/simple_agent_manager.py +++ b/maro/rl/agent/simple_agent_manager.py @@ -88,19 +88,19 @@ def train(self, *args, **kwargs): def load_models(self, agent_model_dict): """Load models from memory for each agent.""" for agent_id, models in agent_model_dict.items(): - self.agent_dict[agent_id].load_models(models) + self.agent_dict[agent_id].load_model(models) def dump_models(self) -> dict: """Get agents' underlying models. This is usually used in distributed mode where models need to be broadcast to remote roll-out actors. """ - return {agent_id: agent.dump_models() for agent_id, agent in self.agent_dict.items()} + return {agent_id: agent.dump_model() for agent_id, agent in self.agent_dict.items()} def load_models_from_files(self, dir_path): """Load models from disk for each agent.""" for agent in self.agent_dict.values(): - agent.load_models_from_file(dir_path) + agent.load_model_from_file(dir_path) def dump_models_to_files(self, dir_path: str): """Dump agents' models to disk. @@ -109,4 +109,4 @@ def dump_models_to_files(self, dir_path: str): """ os.makedirs(dir_path, exist_ok=True) for agent in self.agent_dict.values(): - agent.dump_models_to_file(dir_path) + agent.dump_model_to_file(dir_path) From 0c75673eca179b390d25eeafdf559965328128ef Mon Sep 17 00:00:00 2001 From: ysqyang Date: Wed, 11 Nov 2020 17:08:12 +0800 Subject: [PATCH 168/337] fixed lint issues --- maro/rl/agent/abs_agent.py | 1 - 1 file changed, 1 deletion(-) diff --git a/maro/rl/agent/abs_agent.py b/maro/rl/agent/abs_agent.py index b77aefa17..6ac28047c 100644 --- a/maro/rl/agent/abs_agent.py +++ b/maro/rl/agent/abs_agent.py @@ -2,7 +2,6 @@ # Licensed under the MIT license. import os -import pickle from abc import ABC, abstractmethod from maro.rl.algorithms.abs_algorithm import AbsAlgorithm From 312d12d9abe74d5f9dcc1fefe1ac4a9b8fba0d66 Mon Sep 17 00:00:00 2001 From: ysqyang Date: Wed, 11 Nov 2020 17:12:42 +0800 Subject: [PATCH 169/337] merged with v0.2_embedded_optims --- maro/rl/algorithms/ac.py | 24 +++++------------------- maro/rl/algorithms/dqn.py | 2 +- maro/rl/algorithms/pg.py | 14 -------------- maro/rl/algorithms/ppo.py | 13 ------------- 4 files changed, 6 insertions(+), 47 deletions(-) diff --git a/maro/rl/algorithms/ac.py b/maro/rl/algorithms/ac.py index 88dd9e1f1..8ce251a0b 100644 --- a/maro/rl/algorithms/ac.py +++ b/maro/rl/algorithms/ac.py @@ -22,7 +22,8 @@ class ActorCriticConfig: Args: num_actions (int): number of possible actions - reward_decay (float): reward decay as defined in standard RL terminology + reward_decay (float): reward decay as defined in standard RL terminology. + critic_loss_func (Callable): loss function for the critic model. actor_train_iters (int): number of gradient descent steps for the policy model per call to ``train``. critic_train_iters (int): number of gradient descent steps for the value model per call to ``train``. k (int): number of time steps used in computing returns or return estimates. Defaults to -1, in which case @@ -52,16 +53,13 @@ class ActorCritic(AbsAlgorithm): Args: core_model (MultiTaskLearningModel): Multi-task model that computes action distributions and state values. It may or may not have a shared bottom stack. - critic_loss_func (Callable): loss function for the value model. config: Configuration for the AC algorithm. """ - def __init__(self, core_model: MultiTaskLearningModel, critic_loss_func: Callable, config: ActorCriticConfig): - super().__init__() + def __init__(self, core_model: MultiTaskLearningModel, config: ActorCriticConfig): + super().__init__(core_model, config) self._device = torch.device("cuda" if torch.cuda.is_available() else "cpu") - self._core_model = core_model - self._critic_loss_func = critic_loss_func - self._config = config + self._core_model.to(self._device) @property def model(self): @@ -104,15 +102,3 @@ def train(self, states: np.ndarray, actions: np.ndarray, rewards: np.ndarray): for _ in range(self._config.critic_train_iters): critic_loss = self._critic_loss_func(self._core_model(states, task="critic").squeeze(), return_est) self._core_model.step(critic_loss) - - def load_models(self, model): - self._core_model.load_state_dict(model) - - def dump_models(self): - return self._core_model.state_dict() - - def load_models_from_file(self, path): - self._core_model = torch.load(path) - - def dump_models_to_file(self, path: str): - torch.save(self._core_model.state_dict(), path) diff --git a/maro/rl/algorithms/dqn.py b/maro/rl/algorithms/dqn.py index 1836f423a..95021c47f 100644 --- a/maro/rl/algorithms/dqn.py +++ b/maro/rl/algorithms/dqn.py @@ -31,7 +31,7 @@ class DQNConfig: case it is assumed that the regular Q-value model is used. per_sample_td_error_enabled (bool): If True, per-sample TD errors will be returned by the DQN's train() method. Defaults to False. - loss_cls: + loss_cls: Loss function class for evaluating TD errors. """ __slots__ = [ "num_actions", "reward_decay", "loss_func", "target_update_frequency", "tau", "is_double", diff --git a/maro/rl/algorithms/pg.py b/maro/rl/algorithms/pg.py index 442d06a62..c163e09ac 100644 --- a/maro/rl/algorithms/pg.py +++ b/maro/rl/algorithms/pg.py @@ -66,17 +66,3 @@ def train(self, states: np.ndarray, actions: np.ndarray, returns: np.ndarray): self._optimizer.zero_grad() policy_loss.backward() self._optimizer.step() - - def load_models(self, policy_model): - self._policy_model.load_state_dict(policy_model) - - def dump_models(self): - return self._policy_model.state_dict() - - def load_models_from_file(self, path): - """Load trainable models from disk.""" - self._policy_model = torch.load(path) - - def dump_models_to_file(self, path: str): - """Dump the algorithm's trainable models to disk.""" - torch.save(self._policy_model.state_dict(), path) diff --git a/maro/rl/algorithms/ppo.py b/maro/rl/algorithms/ppo.py index 83d741dc1..7f04176a7 100644 --- a/maro/rl/algorithms/ppo.py +++ b/maro/rl/algorithms/ppo.py @@ -121,16 +121,3 @@ def train( self._value_optimizer.zero_grad() value_loss.backward() self._value_optimizer.step() - - def load_models(self, model_dict): - for name, model in self._model_dict.items(): - model.load_state_dict(model_dict[name]) - - def dump_models(self): - return {name: model.state_dict() for name, model in self._model_dict.items()} - - def load_models_from_file(self, path): - self._model_dict = torch.load(path) - - def dump_models_to_file(self, path: str): - torch.save({name: model.state_dict() for name, model in self._model_dict.items()}, path) From 1baad5ec15373b384f8142275f4fb65e44aae556 Mon Sep 17 00:00:00 2001 From: ysqyang Date: Wed, 11 Nov 2020 17:13:51 +0800 Subject: [PATCH 170/337] refined DQN docstrings --- maro/rl/algorithms/dqn.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/maro/rl/algorithms/dqn.py b/maro/rl/algorithms/dqn.py index bd679d42c..d12236c03 100644 --- a/maro/rl/algorithms/dqn.py +++ b/maro/rl/algorithms/dqn.py @@ -22,6 +22,7 @@ class DQNConfig: Args: num_actions (int): Number of possible actions. reward_decay (float): Reward decay as defined in standard RL terminology. + loss_cls: Loss function class for evaluating TD errors. target_update_frequency (int): Number of training rounds between target model updates. tau (float): Soft update coefficient, i.e., target_model = tau * eval_model + (1 - tau) * target_model. is_double (bool): If True, the next Q values will be computed according to the double DQN algorithm, @@ -31,7 +32,6 @@ class DQNConfig: case it is assumed that the regular Q-value model is used. per_sample_td_error_enabled (bool): If True, per-sample TD errors will be returned by the DQN's train() method. Defaults to False. - loss_cls: """ __slots__ = [ "num_actions", "reward_decay", "loss_func", "target_update_frequency", "tau", "is_double", From e7c8a2a53b1f73ac95ca153c64a152591f7c3df1 Mon Sep 17 00:00:00 2001 From: ysqyang Date: Wed, 11 Nov 2020 04:18:18 -0500 Subject: [PATCH 171/337] removed load/dump functions from DQN --- maro/rl/algorithms/dqn.py | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/maro/rl/algorithms/dqn.py b/maro/rl/algorithms/dqn.py index d12236c03..c58a56150 100644 --- a/maro/rl/algorithms/dqn.py +++ b/maro/rl/algorithms/dqn.py @@ -145,19 +145,3 @@ def _get_next_q_values(self, current_q_values_all, next_states): return self._get_q_values(next_states, is_target=True).gather(1, actions).squeeze(1) # (N,) else: return self._get_q_values(next_states, is_target=True).max(dim=1)[0] # (N,) - - def load_models(self, eval_model): - """Load the evaluation model from memory.""" - self._core_model.load_state_dict(eval_model) - - def dump_models(self): - """Return the evaluation model.""" - return self._core_model.state_dict() - - def load_models_from_file(self, path): - """Load the evaluation model from disk.""" - self._core_model.load_state_dict(torch.load(path)) - - def dump_models_to_file(self, path: str): - """Dump the evaluation model to disk.""" - torch.save(self._core_model.state_dict(), path) From 80f13f6ce24b0fed31cc4f67009f4a2f186d6fcf Mon Sep 17 00:00:00 2001 From: ysqyang Date: Wed, 11 Nov 2020 17:48:04 +0800 Subject: [PATCH 172/337] added task validator --- maro/rl/algorithms/abs_algorithm.py | 3 +++ maro/rl/algorithms/dqn.py | 6 +----- maro/rl/algorithms/task_validator.py | 22 ++++++++++++++++++++ maro/utils/exception/error_code.py | 3 ++- maro/utils/exception/rl_toolkit_exception.py | 6 ++++++ 5 files changed, 34 insertions(+), 6 deletions(-) create mode 100644 maro/rl/algorithms/task_validator.py diff --git a/maro/rl/algorithms/abs_algorithm.py b/maro/rl/algorithms/abs_algorithm.py index 84e998a1f..20a01e0d7 100644 --- a/maro/rl/algorithms/abs_algorithm.py +++ b/maro/rl/algorithms/abs_algorithm.py @@ -5,6 +5,8 @@ from maro.rl.models.abs_learning_model import AbsLearningModel +from .task_validator import validate_tasks + class AbsAlgorithm(ABC): """Abstract RL algorithm class. @@ -17,6 +19,7 @@ class AbsAlgorithm(ABC): core_model (AbsLearningModel): Task model or container of task models required by the algorithm. config: Settings for the algorithm. """ + @validate_tasks def __init__(self, core_model: AbsLearningModel, config): self._core_model = core_model self._config = config diff --git a/maro/rl/algorithms/dqn.py b/maro/rl/algorithms/dqn.py index c58a56150..0dfc23eec 100644 --- a/maro/rl/algorithms/dqn.py +++ b/maro/rl/algorithms/dqn.py @@ -75,11 +75,7 @@ def __init__(self, core_model: AbsLearningModel, config: DQNConfig): if self._config.advantage_mode is not None: assert isinstance(core_model, MultiTaskLearningModel), \ - "core_model must be a MultiTaskLearningModel if dueling architecture is used." - assert DuelingHead.STATE_VALUE.value in core_model.tasks, \ - "core_model must have a task head named 'state_value'" - assert DuelingHead.ADVANTAGE.value in core_model.tasks, \ - "core_model must have a task head named 'advantage'" + f"core_model must be a MultiTaskLearningModel if dueling architecture is used." self._core_model.to(self._device) self._target_model = core_model.copy().to(self._device) if core_model.is_trainable else None diff --git a/maro/rl/algorithms/task_validator.py b/maro/rl/algorithms/task_validator.py new file mode 100644 index 000000000..478009f6b --- /dev/null +++ b/maro/rl/algorithms/task_validator.py @@ -0,0 +1,22 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +from enum import Enum + +from maro.rl.models.learning_model import MultiTaskLearningModel +from maro.utils.exception.rl_toolkit_exception import UnrecognizedTaskError + + +def validate_tasks(task_enum: Enum): + def decorator(init_func): + def wrapper(core_model, config): + recognized_tasks = set(member.value for member in task_enum) + model_tasks = set(core_model.tasks) + if isinstance(core_model, MultiTaskLearningModel) and model_tasks != recognized_tasks: + raise UnrecognizedTaskError(f"Expected task names {recognized_tasks}, got {model_tasks}") + + init_func(core_model, config) + + return wrapper + + return decorator diff --git a/maro/utils/exception/error_code.py b/maro/utils/exception/error_code.py index 0eb864fa3..213918907 100644 --- a/maro/utils/exception/error_code.py +++ b/maro/utils/exception/error_code.py @@ -41,5 +41,6 @@ 4004: "Store Misalignment Error", 4005: "Invalid Episode", 4006: "Infinite Training Loop", - 4007: "Missing Optimizer" + 4007: "Missing Optimizer", + 4008: "Unrecognized Task" } diff --git a/maro/utils/exception/rl_toolkit_exception.py b/maro/utils/exception/rl_toolkit_exception.py index 949100494..a0d6a4a94 100644 --- a/maro/utils/exception/rl_toolkit_exception.py +++ b/maro/utils/exception/rl_toolkit_exception.py @@ -45,3 +45,9 @@ class MissingOptimizerError(MAROException): """Raised when the optimizers are missing when calling LearningModel's step() method.""" def __init__(self, msg: str = None): super().__init__(4007, msg) + + +class UnrecognizedTaskError(MAROException): + """Raised when a MultiTaskLearningModel has task names that are not unrecognized by an algorithm.""" + def __init__(self, msg: str = None): + super().__init__(4008, msg) From 0abc408bb1cbe06077a13d0c7a76bfe23976ec45 Mon Sep 17 00:00:00 2001 From: ysqyang Date: Wed, 11 Nov 2020 17:50:56 +0800 Subject: [PATCH 173/337] fixed decorator use --- maro/rl/algorithms/abs_algorithm.py | 3 --- maro/rl/algorithms/dqn.py | 5 ++++- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/maro/rl/algorithms/abs_algorithm.py b/maro/rl/algorithms/abs_algorithm.py index 20a01e0d7..84e998a1f 100644 --- a/maro/rl/algorithms/abs_algorithm.py +++ b/maro/rl/algorithms/abs_algorithm.py @@ -5,8 +5,6 @@ from maro.rl.models.abs_learning_model import AbsLearningModel -from .task_validator import validate_tasks - class AbsAlgorithm(ABC): """Abstract RL algorithm class. @@ -19,7 +17,6 @@ class AbsAlgorithm(ABC): core_model (AbsLearningModel): Task model or container of task models required by the algorithm. config: Settings for the algorithm. """ - @validate_tasks def __init__(self, core_model: AbsLearningModel, config): self._core_model = core_model self._config = config diff --git a/maro/rl/algorithms/dqn.py b/maro/rl/algorithms/dqn.py index 0dfc23eec..ad473cb43 100644 --- a/maro/rl/algorithms/dqn.py +++ b/maro/rl/algorithms/dqn.py @@ -10,8 +10,10 @@ from maro.rl.models.abs_learning_model import AbsLearningModel from maro.rl.models.learning_model import MultiTaskLearningModel +from .task_validator import validate_tasks -class DuelingHead(Enum): + +class DuelingDQNTask(Enum): STATE_VALUE = "state_value" ADVANTAGE = "advantage" @@ -68,6 +70,7 @@ class DQN(AbsAlgorithm): core_model (AbsLearningModel): Q-value model. config: Configuration for DQN algorithm. """ + @validate_tasks(DuelingDQNTask) def __init__(self, core_model: AbsLearningModel, config: DQNConfig): super().__init__(core_model, config) self._device = torch.device("cuda" if torch.cuda.is_available() else "cpu") From d53885ebf2f9e382fa4e726636cff1af4520153a Mon Sep 17 00:00:00 2001 From: ysqyang Date: Wed, 11 Nov 2020 17:52:25 +0800 Subject: [PATCH 174/337] fixed a typo --- maro/rl/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/maro/rl/__init__.py b/maro/rl/__init__.py index fd9128260..24be78b90 100644 --- a/maro/rl/__init__.py +++ b/maro/rl/__init__.py @@ -7,7 +7,7 @@ from maro.rl.agent.abs_agent_manager import AbsAgentManager, AgentManagerMode from maro.rl.agent.simple_agent_manager import SimpleAgentManager from maro.rl.algorithms.abs_algorithm import AbsAlgorithm -from maro.rl.algorithms.dqn import DQN, DQNConfig, DuelingHead +from maro.rl.algorithms.dqn import DQN, DQNConfig, DuelingDQNTask from maro.rl.dist_topologies.experience_collection import ( concat_experiences_by_agent, merge_experiences_with_trajectory_boundaries ) @@ -50,7 +50,7 @@ 'ColumnBasedStore', 'DQN', 'DQNConfig', - 'DuelingHead', + 'DuelingDQNTask', 'ExperienceShaper', 'FullyConnectedBlock', 'KStepExperienceShaper', From 4fbc37248d108de67512c997dddf34efca21c0ea Mon Sep 17 00:00:00 2001 From: ysqyang Date: Wed, 11 Nov 2020 17:56:24 +0800 Subject: [PATCH 175/337] fixed a bug --- maro/rl/algorithms/task_validator.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/maro/rl/algorithms/task_validator.py b/maro/rl/algorithms/task_validator.py index 478009f6b..4adfb244e 100644 --- a/maro/rl/algorithms/task_validator.py +++ b/maro/rl/algorithms/task_validator.py @@ -2,6 +2,7 @@ # Licensed under the MIT license. from enum import Enum +from functools import wraps from maro.rl.models.learning_model import MultiTaskLearningModel from maro.utils.exception.rl_toolkit_exception import UnrecognizedTaskError @@ -9,13 +10,14 @@ def validate_tasks(task_enum: Enum): def decorator(init_func): - def wrapper(core_model, config): + @wraps(init_func) + def wrapper(self, core_model, config): recognized_tasks = set(member.value for member in task_enum) model_tasks = set(core_model.tasks) if isinstance(core_model, MultiTaskLearningModel) and model_tasks != recognized_tasks: raise UnrecognizedTaskError(f"Expected task names {recognized_tasks}, got {model_tasks}") - init_func(core_model, config) + init_func(self, core_model, config) return wrapper From acc903437932f0e0f311da138d7b87a182d9e4f0 Mon Sep 17 00:00:00 2001 From: ysqyang Date: Wed, 11 Nov 2020 18:21:48 +0800 Subject: [PATCH 176/337] revised --- maro/rl/algorithms/pg.py | 1 - maro/rl/algorithms/ppo.py | 3 ++- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/maro/rl/algorithms/pg.py b/maro/rl/algorithms/pg.py index f5c2be95a..c8365f83e 100644 --- a/maro/rl/algorithms/pg.py +++ b/maro/rl/algorithms/pg.py @@ -31,7 +31,6 @@ class PolicyGradient(AbsAlgorithm): core_model (SingleTaskLearningModel): Policy model. config: Configuration for the PG algorithm. """ - def __init__(self, core_model: SingleTaskLearningModel, config: PolicyGradientConfig): super().__init__(core_model, config) self._device = torch.device("cuda" if torch.cuda.is_available() else "cpu") diff --git a/maro/rl/algorithms/ppo.py b/maro/rl/algorithms/ppo.py index 0dc4fe2de..c512d2a64 100644 --- a/maro/rl/algorithms/ppo.py +++ b/maro/rl/algorithms/ppo.py @@ -67,7 +67,8 @@ class PPO(AbsAlgorithm): See https://arxiv.org/pdf/1707.06347.pdf for details. Args: - core_model (nn.Module): model for generating actions given states. + core_model (MultiTaskLearningModel): Multi-task model that computes action distributions and state values. + It may or may not have a shared bottom stack. config: Configuration for the PPO algorithm. """ @validate_tasks(PPOTask) From 7b875e32aeedbdb4b7074a5c17aed2af7d97a200 Mon Sep 17 00:00:00 2001 From: ysqyang Date: Wed, 11 Nov 2020 18:22:21 +0800 Subject: [PATCH 177/337] fixed lint issues --- maro/rl/algorithms/dqn.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/maro/rl/algorithms/dqn.py b/maro/rl/algorithms/dqn.py index ad473cb43..5ac33f047 100644 --- a/maro/rl/algorithms/dqn.py +++ b/maro/rl/algorithms/dqn.py @@ -30,7 +30,7 @@ class DQNConfig: is_double (bool): If True, the next Q values will be computed according to the double DQN algorithm, i.e., q_next = Q_target(s, argmax(Q_eval(s, a))). Otherwise, q_next = max(Q_target(s, a)). See https://arxiv.org/pdf/1509.06461.pdf for details. Defaults to False. - advantage_mode (str): advantage mode for the dueling architecture. Defaults to None, in which + advantage_mode (str): Advantage mode for the dueling architecture. Defaults to None, in which case it is assumed that the regular Q-value model is used. per_sample_td_error_enabled (bool): If True, per-sample TD errors will be returned by the DQN's train() method. Defaults to False. @@ -78,7 +78,7 @@ def __init__(self, core_model: AbsLearningModel, config: DQNConfig): if self._config.advantage_mode is not None: assert isinstance(core_model, MultiTaskLearningModel), \ - f"core_model must be a MultiTaskLearningModel if dueling architecture is used." + "core_model must be a MultiTaskLearningModel if dueling architecture is used." self._core_model.to(self._device) self._target_model = core_model.copy().to(self._device) if core_model.is_trainable else None From ea8b1f78eb7143021abf87f7205b171a0f8d7386 Mon Sep 17 00:00:00 2001 From: ysqyang Date: Thu, 12 Nov 2020 13:47:19 +0800 Subject: [PATCH 178/337] changed LearningModel's step() to take a single loss --- maro/rl/models/learning_model.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/maro/rl/models/learning_model.py b/maro/rl/models/learning_model.py index bf4ceb0b5..b4646cf4b 100644 --- a/maro/rl/models/learning_model.py +++ b/maro/rl/models/learning_model.py @@ -174,8 +174,8 @@ def forward(self, inputs, task=None): else: return self._net[task](inputs) - def step(self, *losses): - """Use the losses to back-propagate gradients and apply them to the underlying parameters.""" + def step(self, loss): + """Use the loss to back-propagate gradients and apply them to the underlying parameters.""" if not self._has_trainable_shared_layers and not self._has_trainable_heads: raise MissingOptimizerError("No optimizer registered to the model") @@ -186,9 +186,8 @@ def step(self, *losses): for optim in self._head_optimizer_dict.values(): optim.zero_grad() - # Accumulate gradients from all losses - for loss in losses: - loss.backward() + # Obtain gradients through back-propagation + loss.backward() # Apply gradients if self._has_trainable_shared_layers: From 06a0acb1ed8e012a64e1ac07a21ad9e0f030e132 Mon Sep 17 00:00:00 2001 From: ysqyang Date: Thu, 12 Nov 2020 23:07:17 +0800 Subject: [PATCH 179/337] revised learning model design --- examples/cim/dqn/components/agent_manager.py | 7 +- maro/rl/__init__.py | 5 +- maro/rl/algorithms/abs_algorithm.py | 8 +- maro/rl/algorithms/dqn.py | 28 ++-- ...sk_validator.py => task_name_validator.py} | 4 +- maro/rl/models/abs_learning_model.py | 4 +- maro/rl/models/learning_model.py | 153 +++++++----------- 7 files changed, 84 insertions(+), 125 deletions(-) rename maro/rl/algorithms/{task_validator.py => task_name_validator.py} (88%) diff --git a/examples/cim/dqn/components/agent_manager.py b/examples/cim/dqn/components/agent_manager.py index 8c2434f13..5d6dd4201 100644 --- a/examples/cim/dqn/components/agent_manager.py +++ b/examples/cim/dqn/components/agent_manager.py @@ -5,7 +5,8 @@ from torch.optim import RMSprop from maro.rl import ( - ColumnBasedStore, DQN, DQNConfig, FullyConnectedBlock, SimpleAgentManager, SingleTaskLearningModel + ColumnBasedStore, DQN, DQNConfig, FullyConnectedBlock, MultiTaskLearningModel, LearningModule, SimpleAgentManager, + OptimizerOptions ) from maro.utils import set_seeds @@ -17,8 +18,8 @@ def create_dqn_agents(agent_id_list, config): set_seeds(config.seed) agent_dict = {} for agent_id in agent_id_list: - q_model = SingleTaskLearningModel( - [FullyConnectedBlock( + q_model = MultiTaskLearningModel( + {"q_value": "[FullyConnectedBlock( name=f'{agent_id}.policy', input_dim=config.algorithm.input_dim, output_dim=num_actions, diff --git a/maro/rl/__init__.py b/maro/rl/__init__.py index 24be78b90..d9e47b58d 100644 --- a/maro/rl/__init__.py +++ b/maro/rl/__init__.py @@ -22,7 +22,7 @@ from maro.rl.learner.simple_learner import SimpleLearner from maro.rl.models.abs_learning_model import AbsLearningModel from maro.rl.models.fc_block import FullyConnectedBlock -from maro.rl.models.learning_model import MultiTaskLearningModel, SingleTaskLearningModel +from maro.rl.models.learning_model import LearningModule, MultiTaskLearningModel, OptimizerOptions from maro.rl.shaping.abs_shaper import AbsShaper from maro.rl.shaping.action_shaper import ActionShaper from maro.rl.shaping.experience_shaper import ExperienceShaper @@ -54,16 +54,17 @@ 'ExperienceShaper', 'FullyConnectedBlock', 'KStepExperienceShaper', + 'LearningModule', 'LinearExplorer', 'MaxDeltaEarlyStoppingChecker', 'MultiTaskLearningModel', + 'OptimizerOptions', 'OverwriteType', 'RSDEarlyStoppingChecker', 'SimpleActor', 'SimpleAgentManager', 'SimpleEarlyStoppingChecker', 'SimpleLearner', - 'SingleTaskLearningModel', 'StateShaper', 'TwoPhaseLinearExplorer', 'concat_experiences_by_agent', diff --git a/maro/rl/algorithms/abs_algorithm.py b/maro/rl/algorithms/abs_algorithm.py index 84e998a1f..82de7edc1 100644 --- a/maro/rl/algorithms/abs_algorithm.py +++ b/maro/rl/algorithms/abs_algorithm.py @@ -14,16 +14,16 @@ class AbsAlgorithm(ABC): algorithms. Args: - core_model (AbsLearningModel): Task model or container of task models required by the algorithm. + model (AbsLearningModel): Task model or container of task models required by the algorithm. config: Settings for the algorithm. """ - def __init__(self, core_model: AbsLearningModel, config): - self._core_model = core_model + def __init__(self, model: AbsLearningModel, config): + self._model = model self._config = config @property def model(self): - return self._core_model + return self._model @abstractmethod def choose_action(self, state, epsilon: float = None): diff --git a/maro/rl/algorithms/dqn.py b/maro/rl/algorithms/dqn.py index 5ac33f047..ba982984b 100644 --- a/maro/rl/algorithms/dqn.py +++ b/maro/rl/algorithms/dqn.py @@ -10,7 +10,7 @@ from maro.rl.models.abs_learning_model import AbsLearningModel from maro.rl.models.learning_model import MultiTaskLearningModel -from .task_validator import validate_tasks +from .task_name_validator import validate_task_names class DuelingDQNTask(Enum): @@ -67,26 +67,26 @@ class DQN(AbsAlgorithm): See https://web.stanford.edu/class/psych209/Readings/MnihEtAlHassibis15NatureControlDeepRL.pdf for details. Args: - core_model (AbsLearningModel): Q-value model. + model (AbsLearningModel): Q-value model. config: Configuration for DQN algorithm. """ - @validate_tasks(DuelingDQNTask) - def __init__(self, core_model: AbsLearningModel, config: DQNConfig): - super().__init__(core_model, config) + @validate_task_names(DuelingDQNTask) + def __init__(self, model: AbsLearningModel, config: DQNConfig): + super().__init__(model, config) self._device = torch.device("cuda" if torch.cuda.is_available() else "cpu") self._training_counter = 0 if self._config.advantage_mode is not None: - assert isinstance(core_model, MultiTaskLearningModel), \ - "core_model must be a MultiTaskLearningModel if dueling architecture is used." + assert isinstance(model, MultiTaskLearningModel), \ + "model must be a MultiTaskLearningModel if dueling architecture is used." - self._core_model.to(self._device) - self._target_model = core_model.copy().to(self._device) if core_model.is_trainable else None + self._model.to(self._device) + self._target_model = model.copy().to(self._device) if model.is_trainable else None def choose_action(self, state: np.ndarray, epsilon: float = None): if epsilon is None or np.random.rand() > epsilon: state = torch.from_numpy(state).unsqueeze(0) - self._core_model.eval() + self._model.eval() with torch.no_grad(): q_values = self._get_q_values(state) return q_values.argmax(dim=1).item() @@ -110,7 +110,7 @@ def _compute_td_errors( def train(self, states: np.ndarray, actions: np.ndarray, rewards: np.ndarray, next_states: np.ndarray): loss = self._compute_td_errors(states, actions, rewards, next_states) - self._core_model.step(loss.mean() if self._config.per_sample_td_error_enabled else loss) + self._model.learn(loss.mean() if self._config.per_sample_td_error_enabled else loss) self._training_counter += 1 if self._training_counter % self._config.target_update_frequency == 0: self._update_targets() @@ -119,7 +119,7 @@ def train(self, states: np.ndarray, actions: np.ndarray, rewards: np.ndarray, ne def _update_targets(self): for eval_params, target_params in zip( - self._core_model.parameters(), self._target_model.parameters() + self._model.parameters(), self._target_model.parameters() ): target_params.data = ( self._config.tau * eval_params.data + (1 - self._config.tau) * target_params.data @@ -127,7 +127,7 @@ def _update_targets(self): def _get_q_values(self, states, is_target: bool = False): if self._config.advantage_mode is not None: - output = self._target_model(states) if is_target else self._core_model(states) + output = self._target_model(states) if is_target else self._model(states) state_values = output["state_value"] advantages = output["advantage"] # Use mean or max correction to address the identifiability issue @@ -135,7 +135,7 @@ def _get_q_values(self, states, is_target: bool = False): q_values = state_values + advantages - corrections.unsqueeze(1) return q_values else: - model = self._target_model if is_target else self._core_model + model = self._target_model if is_target else self._model return model(states) def _get_next_q_values(self, current_q_values_all, next_states): diff --git a/maro/rl/algorithms/task_validator.py b/maro/rl/algorithms/task_name_validator.py similarity index 88% rename from maro/rl/algorithms/task_validator.py rename to maro/rl/algorithms/task_name_validator.py index 4adfb244e..4a84e5fcf 100644 --- a/maro/rl/algorithms/task_validator.py +++ b/maro/rl/algorithms/task_name_validator.py @@ -8,12 +8,12 @@ from maro.utils.exception.rl_toolkit_exception import UnrecognizedTaskError -def validate_tasks(task_enum: Enum): +def validate_task_names(task_enum: Enum): def decorator(init_func): @wraps(init_func) def wrapper(self, core_model, config): recognized_tasks = set(member.value for member in task_enum) - model_tasks = set(core_model.tasks) + model_tasks = set(core_model.task_names) if isinstance(core_model, MultiTaskLearningModel) and model_tasks != recognized_tasks: raise UnrecognizedTaskError(f"Expected task names {recognized_tasks}, got {model_tasks}") diff --git a/maro/rl/models/abs_learning_model.py b/maro/rl/models/abs_learning_model.py index 50a879548..34ee1f388 100644 --- a/maro/rl/models/abs_learning_model.py +++ b/maro/rl/models/abs_learning_model.py @@ -19,8 +19,8 @@ def forward(self, inputs): return NotImplemented @abstractmethod - def step(self, *losses): - """Use losses to back-propagate gradients and apply the gradients to the underlying parameters.""" + def learn(self, loss): + """Use a loss to back-propagate gradients and apply the gradients to the underlying parameters.""" return NotImplemented def copy(self): diff --git a/maro/rl/models/learning_model.py b/maro/rl/models/learning_model.py index b4646cf4b..b29163f40 100644 --- a/maro/rl/models/learning_model.py +++ b/maro/rl/models/learning_model.py @@ -1,7 +1,9 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT license. -import torch +from collections import namedtuple +from typing import Dict, Union + import torch.nn as nn from maro.utils.exception.rl_toolkit_exception import MissingOptimizerError @@ -9,21 +11,26 @@ from .abs_learning_model import AbsLearningModel -class SingleTaskLearningModel(AbsLearningModel): +OptimizerOptions = namedtuple("OptimizerOptions", ["cls", "params"]) + + +class LearningModule(nn.Module): """NN model that consists of a sequence of chainable blocks. Args: block_list (list): List of blocks that compose the model. They must be chainable, i.e., the output dimension of a block must match the input dimension of its successor. - optimizer_opt (tuple): Optimizer option for the model. Default to None. + optimizer_options (OptimizerOptions): A namedtuple of (optimizer_class, optimizer_parameters). """ - def __init__(self, block_list: list, optimizer_opt: tuple = None): + def __init__(self, name: str, block_list: list, optimizer_options: OptimizerOptions = None): super().__init__() + self._name = name self._net = nn.Sequential(*block_list) - self._is_trainable = optimizer_opt is not None + self._is_trainable = optimizer_options is not None if self._is_trainable: - self._optimizer = optimizer_opt[0](self._net.parameters(), **optimizer_opt[1]) + self._optimizer = optimizer_options.cls(self._net.parameters(), **optimizer_options.params) else: + self._net.eval() for param in self._net.parameters(): param.requires_grad = False @@ -31,12 +38,16 @@ def __getstate__(self): dic = self.__dict__.copy() if "_optimizer" in dic: del dic["_optimizer"] - + dic["is_trainable"] = False return dic def __setstate__(self, dic: dict): self.__dict__ = dic + @property + def name(self): + return self._name + @property def is_trainable(self): return self._is_trainable @@ -52,10 +63,12 @@ def forward(self, inputs): """ return self._net(inputs) - def step(self, loss: torch.tensor): - """Use the loss to back-propagate gradients and apply them to the underlying parameters.""" + def zero_grad(self): + if not self._is_trainable: + raise MissingOptimizerError("No optimizer registered to the model") self._optimizer.zero_grad() - loss.backward() + + def step(self): self._optimizer.step() @@ -63,54 +76,28 @@ class MultiTaskLearningModel(AbsLearningModel): """NN model that consists of multiple task heads and an optional shared stack. Args: - task_block_dict (dict): Dictionary of network blocks that perform designated tasks. - task_optimizer_opt_dict (dict): Dictionary of optimizer options for each task block. An optimizer option - is specified in the form of a tuple: (optimizer class, optimizer parameters). Defaults to None. - shared_block_list (list): List of blocks that compose the bottom stack of the model shared by all tasks. - The shared blocks must be chainable, i.e., the output dimension of a block must match the input dimension - of its successor. Defaults to None. - shared_optimizer_opt (tuple): Optimizer option for the shared part of the model. Default to None. + task_modules (LearningModule): LearningModule instances, each of which performs a designated task. + shared_module (LearningModule): Network module that forms that shared part of the model. Defaults to + None. """ def __init__( self, - task_block_dict: dict, - task_optimizer_opt_dict: dict = None, - shared_block_list: list = None, - shared_optimizer_opt: tuple = None + *task_modules: LearningModule, + shared_module: LearningModule = None ): super().__init__() - self._has_shared_layers = shared_block_list is not None - self._has_trainable_shared_layers = self._has_shared_layers and shared_optimizer_opt is not None - self._has_trainable_heads = task_optimizer_opt_dict is not None + self._task_names = [module.name for module in task_modules] # shared stack - if self._has_shared_layers: - self._shared_stack = nn.Sequential(*shared_block_list) - if self._has_trainable_shared_layers: - self._shared_optimizer = shared_optimizer_opt[0]( - self._shared_stack.parameters(), **shared_optimizer_opt[1] - ) - else: - for param in self._shared_stack.parameters(): - param.requires_grad = False - - # heads - self._tasks = list(task_block_dict.keys()) + self._shared_module = shared_module + + # task_heads + self._task_modules = task_modules self._net = nn.ModuleDict({ - key: nn.Sequential(self._shared_stack, head) if self._has_shared_layers else head - for key, head in task_block_dict.items() + task_module.name: nn.Sequential(self._shared_module, task_module) if self._shared_module else task_module + for task_module in self._task_modules }) - if self._has_trainable_heads: - self._head_optimizer_dict = { - key: task_optimizer_opt_dict[key][0](head.parameters(), **task_optimizer_opt_dict[key][1]) - for key, head in task_block_dict.items() - } - else: - for key, head in task_block_dict.items(): - for param in head.parameters(): - param.requires_grad = False - def __getstate__(self): dic = self.__dict__.copy() if "_shared_optimizer" in dic: @@ -126,72 +113,42 @@ def __getitem__(self, task): return self._net[task] @property - def has_shared_layers(self): - return self._has_shared_layers + def task_names(self) -> [str]: + return self._task_names - @property - def has_trainable_shared_layers(self): - return self._has_trainable_shared_layers - - @has_trainable_shared_layers.setter - def has_trainable_shared_layers(self, value: bool): - self._has_trainable_shared_layers = value - - @property - def has_trainable_heads(self): - return self._has_trainable_heads - - @has_trainable_heads.setter - def has_trainable_heads(self, value: bool): - self._has_trainable_heads = value - - @property - def is_trainable(self): - return self._has_trainable_shared_layers or self._has_trainable_heads - - @property - def tasks(self) -> [str]: - return self._tasks - - def forward(self, inputs, task=None): + def forward(self, inputs, task_name=None): """Feedforward computations for the given head(s). Args: inputs: Inputs to the model. - task: The task for which the network output is required. If this is None, the results from all task - heads will be returned in the form of a dictionary. If this is a list, the results will be the - outputs from the heads contained in task in the form of a dictionary. If this is a single key, - the result will be the output from the corresponding head. + task_name: The name of the task for which the network output is required. If this is None, the results from + all task heads will be returned in the form of a dictionary. If this is a list, the results will be the + outputs from the heads contained in task in the form of a dictionary. If this is a single key, the + result will be the output from the corresponding head. Returns: Outputs from the required head(s). """ - if task is None: + if task_name is None: return {key: self._net[key](inputs) for key in self._tasks} - if isinstance(task, list): - return {k: self._net[k](inputs) for k in task} + if isinstance(task_name, list): + return {k: self._net[k](inputs) for k in task_name} else: - return self._net[task](inputs) + return self._net[task_name](inputs) - def step(self, loss): + def learn(self, loss): """Use the loss to back-propagate gradients and apply them to the underlying parameters.""" - if not self._has_trainable_shared_layers and not self._has_trainable_heads: - raise MissingOptimizerError("No optimizer registered to the model") - - # Zero all gradients - if self._has_trainable_shared_layers: - self._shared_optimizer.zero_grad() - if self._has_trainable_heads: - for optim in self._head_optimizer_dict.values(): - optim.zero_grad() + for task_module in self._task_modules: + task_module.zero_grad() + if self._shared_module is not None: + self._shared_module.zero_grad() # Obtain gradients through back-propagation loss.backward() # Apply gradients - if self._has_trainable_shared_layers: - self._shared_optimizer.step() - if self._has_trainable_heads: - for optim in self._head_optimizer_dict.values(): - optim.step() + for task_module in self._task_modules: + task_module.step() + if self._shared_module is not None: + self._shared_module.step() From 3d07ce8e2a3d07a9dd838201fa72793982cd0f54 Mon Sep 17 00:00:00 2001 From: ysqyang Date: Fri, 13 Nov 2020 00:04:02 +0800 Subject: [PATCH 180/337] revised example --- examples/cim/dqn/components/agent_manager.py | 11 ++++++----- maro/rl/models/fc_block.py | 7 ++++--- maro/rl/models/learning_model.py | 2 +- 3 files changed, 11 insertions(+), 9 deletions(-) diff --git a/examples/cim/dqn/components/agent_manager.py b/examples/cim/dqn/components/agent_manager.py index 5d6dd4201..ad17b4076 100644 --- a/examples/cim/dqn/components/agent_manager.py +++ b/examples/cim/dqn/components/agent_manager.py @@ -18,20 +18,21 @@ def create_dqn_agents(agent_id_list, config): set_seeds(config.seed) agent_dict = {} for agent_id in agent_id_list: - q_model = MultiTaskLearningModel( - {"q_value": "[FullyConnectedBlock( - name=f'{agent_id}.policy', + q_module = LearningModule( + "q_value", + [FullyConnectedBlock( + name=f'{agent_id}.q_value', input_dim=config.algorithm.input_dim, output_dim=num_actions, activation=nn.LeakyReLU, is_head=True, **config.algorithm.model )], - optimizer_opt=(RMSprop, config.algorithm.optimizer), + optimizer_options=OptimizerOptions(cls=RMSprop, params=config.algorithm.optimizer) ) algorithm = DQN( - core_model=q_model, + core_model=MultiTaskLearningModel(q_module), config=DQNConfig( **config.algorithm.config, loss_cls=nn.SmoothL1Loss, diff --git a/maro/rl/models/fc_block.py b/maro/rl/models/fc_block.py index 1753b87c4..ff0bf4043 100644 --- a/maro/rl/models/fc_block.py +++ b/maro/rl/models/fc_block.py @@ -29,7 +29,6 @@ class FullyConnectedBlock(nn.Module): """ def __init__( self, - name: str, input_dim: int, output_dim: int, hidden_dims: [int], @@ -39,10 +38,10 @@ def __init__( batch_norm_enabled: bool = False, skip_connection_enabled: bool = False, dropout_p: float = None, - gradient_threshold: float = None + gradient_threshold: float = None, + name: str = None ): super().__init__() - self._name = name self._input_dim = input_dim self._hidden_dims = hidden_dims if hidden_dims is not None else [] self._output_dim = output_dim @@ -75,6 +74,8 @@ def __init__( for param in self._net.parameters(): param.register_hook(lambda grad: torch.clamp(grad, -gradient_threshold, gradient_threshold)) + self._name = name + def forward(self, x): out = self._net(x) if self._skip_connection_enabled: diff --git a/maro/rl/models/learning_model.py b/maro/rl/models/learning_model.py index b29163f40..ba2a03738 100644 --- a/maro/rl/models/learning_model.py +++ b/maro/rl/models/learning_model.py @@ -130,7 +130,7 @@ def forward(self, inputs, task_name=None): Outputs from the required head(s). """ if task_name is None: - return {key: self._net[key](inputs) for key in self._tasks} + return {key: self._net[key](inputs) for key in self._task_names} if isinstance(task_name, list): return {k: self._net[k](inputs) for k in task_name} From 1369241bce3fe887fac7162e6f73062252e4938b Mon Sep 17 00:00:00 2001 From: ysqyang Date: Fri, 13 Nov 2020 00:09:30 +0800 Subject: [PATCH 181/337] fixed a bug --- maro/rl/algorithms/task_name_validator.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/maro/rl/algorithms/task_name_validator.py b/maro/rl/algorithms/task_name_validator.py index 4a84e5fcf..452dcd8d9 100644 --- a/maro/rl/algorithms/task_name_validator.py +++ b/maro/rl/algorithms/task_name_validator.py @@ -4,7 +4,6 @@ from enum import Enum from functools import wraps -from maro.rl.models.learning_model import MultiTaskLearningModel from maro.utils.exception.rl_toolkit_exception import UnrecognizedTaskError @@ -12,10 +11,10 @@ def validate_task_names(task_enum: Enum): def decorator(init_func): @wraps(init_func) def wrapper(self, core_model, config): - recognized_tasks = set(member.value for member in task_enum) - model_tasks = set(core_model.task_names) - if isinstance(core_model, MultiTaskLearningModel) and model_tasks != recognized_tasks: - raise UnrecognizedTaskError(f"Expected task names {recognized_tasks}, got {model_tasks}") + recognized_task_names = set(member.value for member in task_enum) + model_task_names = set(core_model.task_names) + if len(model_task_names) > 1 and model_task_names != recognized_task_names: + raise UnrecognizedTaskError(f"Expected task names {recognized_task_names}, got {model_task_names}") init_func(self, core_model, config) From ef01678d05960d6863789cdbf5ab005b7671985e Mon Sep 17 00:00:00 2001 From: ysqyang Date: Fri, 13 Nov 2020 00:11:25 +0800 Subject: [PATCH 182/337] fixed a bug --- maro/rl/models/learning_model.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/maro/rl/models/learning_model.py b/maro/rl/models/learning_model.py index ba2a03738..246be8b3c 100644 --- a/maro/rl/models/learning_model.py +++ b/maro/rl/models/learning_model.py @@ -116,6 +116,10 @@ def __getitem__(self, task): def task_names(self) -> [str]: return self._task_names + @property + def is_trainable(self) -> bool: + return any(task_module.is_trainable for task_module in self._task_modules) or self._shared_module.is_trainable + def forward(self, inputs, task_name=None): """Feedforward computations for the given head(s). From 01cbc2d8c3b2cb32c0c75ec1913470b389d326ff Mon Sep 17 00:00:00 2001 From: ysqyang Date: Fri, 13 Nov 2020 00:20:59 +0800 Subject: [PATCH 183/337] fixed a bug --- maro/rl/algorithms/dqn.py | 3 +-- maro/rl/models/learning_model.py | 3 +++ 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/maro/rl/algorithms/dqn.py b/maro/rl/algorithms/dqn.py index ba982984b..b0863393e 100644 --- a/maro/rl/algorithms/dqn.py +++ b/maro/rl/algorithms/dqn.py @@ -135,8 +135,7 @@ def _get_q_values(self, states, is_target: bool = False): q_values = state_values + advantages - corrections.unsqueeze(1) return q_values else: - model = self._target_model if is_target else self._model - return model(states) + return self._target_model(states) if is_target else self._model(states) def _get_next_q_values(self, current_q_values_all, next_states): if self._config.is_double: diff --git a/maro/rl/models/learning_model.py b/maro/rl/models/learning_model.py index 246be8b3c..58d766108 100644 --- a/maro/rl/models/learning_model.py +++ b/maro/rl/models/learning_model.py @@ -133,6 +133,9 @@ def forward(self, inputs, task_name=None): Returns: Outputs from the required head(s). """ + if len(self._task_modules) == 1: + return self._net[task_name](inputs) + if task_name is None: return {key: self._net[key](inputs) for key in self._task_names} From 6dc27d7ea2da4eb6527bdbfc5dcec72d495e803d Mon Sep 17 00:00:00 2001 From: ysqyang Date: Fri, 13 Nov 2020 00:23:18 +0800 Subject: [PATCH 184/337] fixed a bug --- maro/rl/models/learning_model.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/maro/rl/models/learning_model.py b/maro/rl/models/learning_model.py index 58d766108..1ac1c35b8 100644 --- a/maro/rl/models/learning_model.py +++ b/maro/rl/models/learning_model.py @@ -134,8 +134,9 @@ def forward(self, inputs, task_name=None): Outputs from the required head(s). """ if len(self._task_modules) == 1: + task_name = self._task_modules[0].name return self._net[task_name](inputs) - + if task_name is None: return {key: self._net[key](inputs) for key in self._task_names} From a89173219fdc2118ced002469e67304244cf2bea Mon Sep 17 00:00:00 2001 From: ysqyang Date: Fri, 13 Nov 2020 14:29:45 +0800 Subject: [PATCH 185/337] added decorator utils to algorithm --- maro/rl/__init__.py | 6 +- maro/rl/algorithms/dqn.py | 77 ++++++++----------- .../{task_name_validator.py => utils.py} | 29 +++++++ maro/rl/models/abs_learning_model.py | 39 ---------- maro/rl/models/learning_model.py | 57 ++++++++++---- 5 files changed, 102 insertions(+), 106 deletions(-) rename maro/rl/algorithms/{task_name_validator.py => utils.py} (51%) delete mode 100644 maro/rl/models/abs_learning_model.py diff --git a/maro/rl/__init__.py b/maro/rl/__init__.py index d9e47b58d..676b47a7c 100644 --- a/maro/rl/__init__.py +++ b/maro/rl/__init__.py @@ -20,9 +20,8 @@ from maro.rl.explorer.simple_explorer import LinearExplorer, TwoPhaseLinearExplorer from maro.rl.learner.abs_learner import AbsLearner from maro.rl.learner.simple_learner import SimpleLearner -from maro.rl.models.abs_learning_model import AbsLearningModel from maro.rl.models.fc_block import FullyConnectedBlock -from maro.rl.models.learning_model import LearningModule, MultiTaskLearningModel, OptimizerOptions +from maro.rl.models.learning_model import LearningModel, LearningModule, OptimizerOptions from maro.rl.shaping.abs_shaper import AbsShaper from maro.rl.shaping.action_shaper import ActionShaper from maro.rl.shaping.experience_shaper import ExperienceShaper @@ -40,7 +39,6 @@ 'AbsEarlyStoppingChecker', 'AbsExplorer', 'AbsLearner', - 'AbsLearningModel', 'AbsShaper', 'AbsStore', 'ActionShaper', @@ -54,10 +52,10 @@ 'ExperienceShaper', 'FullyConnectedBlock', 'KStepExperienceShaper', + 'LearningModel', 'LearningModule', 'LinearExplorer', 'MaxDeltaEarlyStoppingChecker', - 'MultiTaskLearningModel', 'OptimizerOptions', 'OverwriteType', 'RSDEarlyStoppingChecker', diff --git a/maro/rl/algorithms/dqn.py b/maro/rl/algorithms/dqn.py index c36bd1085..db011a151 100644 --- a/maro/rl/algorithms/dqn.py +++ b/maro/rl/algorithms/dqn.py @@ -4,13 +4,11 @@ from enum import Enum import numpy as np -import torch from maro.rl.algorithms.abs_algorithm import AbsAlgorithm -from maro.rl.models.abs_learning_model import AbsLearningModel -from maro.rl.models.learning_model import MultiTaskLearningModel +from maro.rl.models.learning_model import LearningModel -from .task_name_validator import validate_task_names +from .utils import validate_task_names, to_device, preprocess class DuelingDQNTask(Enum): @@ -67,48 +65,54 @@ class DQN(AbsAlgorithm): See https://web.stanford.edu/class/psych209/Readings/MnihEtAlHassibis15NatureControlDeepRL.pdf for details. Args: -<<<<<<< HEAD - model (AbsLearningModel): Q-value model. + model (LearningModel): Q-value model. config: Configuration for DQN algorithm. """ + @to_device @validate_task_names(DuelingDQNTask) - def __init__(self, model: AbsLearningModel, config: DQNConfig): + def __init__(self, model: LearningModel, config: DQNConfig): super().__init__(model, config) - self._device = torch.device("cuda" if torch.cuda.is_available() else "cpu") self._training_counter = 0 + self._target_model = model.copy() if model.is_trainable else None - if self._config.advantage_mode is not None: - assert isinstance(model, MultiTaskLearningModel), \ - "model must be a MultiTaskLearningModel if dueling architecture is used." - - self._model.to(self._device) - self._target_model = model.copy().to(self._device) if model.is_trainable else None - + @preprocess def choose_action(self, state: np.ndarray, epsilon: float = None): if epsilon is None or np.random.rand() > epsilon: - state = torch.from_numpy(state).unsqueeze(0) - self._model.eval() - with torch.no_grad(): - q_values = self._get_q_values(state) + q_values = self._get_q_values(self._model, state, is_training=False) return q_values.argmax(dim=1).item() return np.random.choice(self._config.num_actions) - def _compute_td_errors( - self, states: np.ndarray, actions: np.ndarray, rewards: np.ndarray, next_states: np.ndarray - ): - states = torch.from_numpy(states).to(self._device) # (N, state_dim) - actions = torch.from_numpy(actions).to(self._device) # (N,) - rewards = torch.from_numpy(rewards).to(self._device) # (N,) - next_states = torch.from_numpy(next_states).to(self._device) # (N, state_dim) + def _get_q_values(self, model, states, is_training: bool = True): + if self._config.advantage_mode is not None: + output = model(states, is_training=is_training) + state_values = output["state_value"] + advantages = output["advantage"] + # Use mean or max correction to address the identifiability issue + corrections = advantages.mean(1) if self._config.advantage_mode == "mean" else advantages.max(1)[0] + q_values = state_values + advantages - corrections.unsqueeze(1) + return q_values + else: + return model(states, is_training=is_training) + + def _get_next_q_values(self, current_q_values_for_all_actions, next_states): + next_q_values_for_all_actions = self._get_q_values(self._target_model, next_states, is_training=False) + if self._config.is_double: + actions = current_q_values_for_all_actions.max(dim=1)[1].unsqueeze(1) + return next_q_values_for_all_actions.gather(1, actions).squeeze(1) # (N,) + else: + return next_q_values_for_all_actions.max(dim=1)[0] # (N,) + + def _compute_td_errors(self, states, actions, rewards, next_states): if len(actions.shape) == 1: actions = actions.unsqueeze(1) # (N, 1) - current_q_values_for_all_actions = self._get_q_values(states) + current_q_values_for_all_actions = self._get_q_values(self._model, states) current_q_values = current_q_values_for_all_actions.gather(1, actions).squeeze(1) # (N,) next_q_values = self._get_next_q_values(current_q_values_for_all_actions, next_states) # (N,) target_q_values = (rewards + self._config.reward_decay * next_q_values).detach() # (N,) return self._config.loss_func(current_q_values, target_q_values) + @preprocess def train(self, states: np.ndarray, actions: np.ndarray, rewards: np.ndarray, next_states: np.ndarray): loss = self._compute_td_errors(states, actions, rewards, next_states) self._model.learn(loss.mean() if self._config.per_sample_td_error_enabled else loss) @@ -125,22 +129,3 @@ def _update_targets(self): target_params.data = ( self._config.tau * eval_params.data + (1 - self._config.tau) * target_params.data ) - - def _get_q_values(self, states, is_target: bool = False): - if self._config.advantage_mode is not None: - output = self._target_model(states) if is_target else self._model(states) - state_values = output["state_value"] - advantages = output["advantage"] - # Use mean or max correction to address the identifiability issue - corrections = advantages.mean(1) if self._config.advantage_mode == "mean" else advantages.max(1)[0] - q_values = state_values + advantages - corrections.unsqueeze(1) - return q_values - else: - return self._target_model(states) if is_target else self._model(states) - - def _get_next_q_values(self, current_q_values_all, next_states): - if self._config.is_double: - actions = current_q_values_all.max(dim=1)[1].unsqueeze(1) - return self._get_q_values(next_states, is_target=True).gather(1, actions).squeeze(1) # (N,) - else: - return self._get_q_values(next_states, is_target=True).max(dim=1)[0] # (N,) diff --git a/maro/rl/algorithms/task_name_validator.py b/maro/rl/algorithms/utils.py similarity index 51% rename from maro/rl/algorithms/task_name_validator.py rename to maro/rl/algorithms/utils.py index 452dcd8d9..0e028f4db 100644 --- a/maro/rl/algorithms/task_name_validator.py +++ b/maro/rl/algorithms/utils.py @@ -4,9 +4,15 @@ from enum import Enum from functools import wraps +import numpy as np +import torch + from maro.utils.exception.rl_toolkit_exception import UnrecognizedTaskError +device = torch.device("cuda" if torch.cuda.is_available() else "cpu") + + def validate_task_names(task_enum: Enum): def decorator(init_func): @wraps(init_func) @@ -21,3 +27,26 @@ def wrapper(self, core_model, config): return wrapper return decorator + + +def to_device(init_func): + @wraps(init_func) + def wrapper(self, model, config): + init_func(self, model.to(device), config) + + return wrapper + + +def preprocess(func): + @wraps(func) + def wrapper(*args, **kwargs): + converted_args = [ + torch.from_numpy(arg).to(device) if isinstance(arg, np.ndarray) else arg for arg in args + ] + converted_kwargs = { + kw: torch.from_numpy(arg).to(device) if isinstance(arg, np.ndarray) else arg + for kw, arg in kwargs.items() + } + return func(*converted_args, **converted_kwargs) + + return wrapper diff --git a/maro/rl/models/abs_learning_model.py b/maro/rl/models/abs_learning_model.py deleted file mode 100644 index 34ee1f388..000000000 --- a/maro/rl/models/abs_learning_model.py +++ /dev/null @@ -1,39 +0,0 @@ -# Copyright (c) Microsoft Corporation. -# Licensed under the MIT license. - -from abc import abstractmethod - -import torch -import torch.nn as nn - -from maro.utils import clone - - -class AbsLearningModel(nn.Module): - def __init__(self): - super().__init__() - - @abstractmethod - def forward(self, inputs): - """Feedforward computation""" - return NotImplemented - - @abstractmethod - def learn(self, loss): - """Use a loss to back-propagate gradients and apply the gradients to the underlying parameters.""" - return NotImplemented - - def copy(self): - return clone(self) - - def load(self, state_dict): - self.load_state_dict(state_dict) - - def dump(self): - return self.state_dict() - - def load_from_file(self, path: str): - self.load_state_dict(torch.load(path)) - - def dump_to_file(self, path: str): - torch.save(self.state_dict(), path) diff --git a/maro/rl/models/learning_model.py b/maro/rl/models/learning_model.py index 1ac1c35b8..09fc81616 100644 --- a/maro/rl/models/learning_model.py +++ b/maro/rl/models/learning_model.py @@ -2,15 +2,13 @@ # Licensed under the MIT license. from collections import namedtuple -from typing import Dict, Union +import torch import torch.nn as nn +from maro.utils import clone from maro.utils.exception.rl_toolkit_exception import MissingOptimizerError -from .abs_learning_model import AbsLearningModel - - OptimizerOptions = namedtuple("OptimizerOptions", ["cls", "params"]) @@ -72,7 +70,7 @@ def step(self): self._optimizer.step() -class MultiTaskLearningModel(AbsLearningModel): +class LearningModel(nn.Module): """NN model that consists of multiple task heads and an optional shared stack. Args: @@ -120,30 +118,40 @@ def task_names(self) -> [str]: def is_trainable(self) -> bool: return any(task_module.is_trainable for task_module in self._task_modules) or self._shared_module.is_trainable - def forward(self, inputs, task_name=None): + def _forward(self, inputs, task_name: str = None): + if len(self._task_modules) == 1: + task_name = self._task_modules[0].name + return self._net[task_name](inputs) + + if task_name is None: + return {key: self._net[key](inputs) for key in self._task_names} + + if isinstance(task_name, list): + return {k: self._net[k](inputs) for k in task_name} + else: + return self._net[task_name](inputs) + + def forward(self, inputs, task_name: str = None, is_training: bool = True): """Feedforward computations for the given head(s). Args: inputs: Inputs to the model. - task_name: The name of the task for which the network output is required. If this is None, the results from + task_name (str): The name of the task for which the network output is required. If this is None, the results from all task heads will be returned in the form of a dictionary. If this is a list, the results will be the outputs from the heads contained in task in the form of a dictionary. If this is a single key, the result will be the output from the corresponding head. + is_training (bool): If true, all torch submodules will be set to training mode, and auto-differentiation + will be turned on. Defaults to True. Returns: Outputs from the required head(s). """ - if len(self._task_modules) == 1: - task_name = self._task_modules[0].name - return self._net[task_name](inputs) + self.train(mode=is_training) + if is_training: + return self._forward(inputs, task_name) - if task_name is None: - return {key: self._net[key](inputs) for key in self._task_names} - - if isinstance(task_name, list): - return {k: self._net[k](inputs) for k in task_name} - else: - return self._net[task_name](inputs) + with torch.no_grad(): + return self._forward(inputs, task_name) def learn(self, loss): """Use the loss to back-propagate gradients and apply them to the underlying parameters.""" @@ -160,3 +168,18 @@ def learn(self, loss): task_module.step() if self._shared_module is not None: self._shared_module.step() + + def copy(self): + return clone(self) + + def load(self, state_dict): + self.load_state_dict(state_dict) + + def dump(self): + return self.state_dict() + + def load_from_file(self, path: str): + self.load_state_dict(torch.load(path)) + + def dump_to_file(self, path: str): + torch.save(self.state_dict(), path) From 13de76e2b009cd275a2e2c9230c90f8102f92167 Mon Sep 17 00:00:00 2001 From: ysqyang Date: Fri, 13 Nov 2020 14:37:46 +0800 Subject: [PATCH 186/337] fixed a bug --- maro/rl/algorithms/abs_algorithm.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/maro/rl/algorithms/abs_algorithm.py b/maro/rl/algorithms/abs_algorithm.py index 82de7edc1..38fe7c908 100644 --- a/maro/rl/algorithms/abs_algorithm.py +++ b/maro/rl/algorithms/abs_algorithm.py @@ -3,7 +3,7 @@ from abc import ABC, abstractmethod -from maro.rl.models.abs_learning_model import AbsLearningModel +from maro.rl.models.learning_model import LearningModel class AbsAlgorithm(ABC): @@ -14,10 +14,10 @@ class AbsAlgorithm(ABC): algorithms. Args: - model (AbsLearningModel): Task model or container of task models required by the algorithm. + model (LearningModel): Task model or container of task models required by the algorithm. config: Settings for the algorithm. """ - def __init__(self, model: AbsLearningModel, config): + def __init__(self, model: LearningModel, config): self._model = model self._config = config From 2dee56cd383d779ae497bee1d7d0c503437a0ecd Mon Sep 17 00:00:00 2001 From: ysqyang Date: Fri, 13 Nov 2020 14:40:46 +0800 Subject: [PATCH 187/337] renamed core_model to model --- examples/cim/dqn/components/agent_manager.py | 3 +-- maro/rl/algorithms/utils.py | 6 +++--- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/examples/cim/dqn/components/agent_manager.py b/examples/cim/dqn/components/agent_manager.py index 85805f935..121c7e196 100644 --- a/examples/cim/dqn/components/agent_manager.py +++ b/examples/cim/dqn/components/agent_manager.py @@ -21,7 +21,6 @@ def create_dqn_agents(agent_id_list, config): q_module = LearningModule( "q_value", [FullyConnectedBlock( - name=f'{agent_id}.q_value', input_dim=config.algorithm.input_dim, output_dim=num_actions, activation=nn.LeakyReLU, @@ -32,7 +31,7 @@ def create_dqn_agents(agent_id_list, config): ) algorithm = DQN( - core_model=LearningModel(q_module), + model=LearningModel(q_module), config=DQNConfig( **config.algorithm.config, loss_cls=nn.SmoothL1Loss, diff --git a/maro/rl/algorithms/utils.py b/maro/rl/algorithms/utils.py index 0e028f4db..952c23c8c 100644 --- a/maro/rl/algorithms/utils.py +++ b/maro/rl/algorithms/utils.py @@ -16,13 +16,13 @@ def validate_task_names(task_enum: Enum): def decorator(init_func): @wraps(init_func) - def wrapper(self, core_model, config): + def wrapper(self, model, config): recognized_task_names = set(member.value for member in task_enum) - model_task_names = set(core_model.task_names) + model_task_names = set(model.task_names) if len(model_task_names) > 1 and model_task_names != recognized_task_names: raise UnrecognizedTaskError(f"Expected task names {recognized_task_names}, got {model_task_names}") - init_func(self, core_model, config) + init_func(self, model, config) return wrapper From aed936bbba5a0bfbc3a5dd9383b49f0643205af3 Mon Sep 17 00:00:00 2001 From: ysqyang Date: Fri, 13 Nov 2020 14:45:45 +0800 Subject: [PATCH 188/337] fixed a bug --- maro/rl/algorithms/dqn.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/maro/rl/algorithms/dqn.py b/maro/rl/algorithms/dqn.py index db011a151..ec39c0afe 100644 --- a/maro/rl/algorithms/dqn.py +++ b/maro/rl/algorithms/dqn.py @@ -84,6 +84,8 @@ def choose_action(self, state: np.ndarray, epsilon: float = None): return np.random.choice(self._config.num_actions) def _get_q_values(self, model, states, is_training: bool = True): + if len(states.shape) == 1: + states = states.unsqueeze(dim=0) if self._config.advantage_mode is not None: output = model(states, is_training=is_training) state_values = output["state_value"] From fc4b2ff4de8f3b833830674d59b1b328b1637b94 Mon Sep 17 00:00:00 2001 From: ysqyang Date: Fri, 13 Nov 2020 14:59:45 +0800 Subject: [PATCH 189/337] 1. fixed lint formatting issues; 2. refined learning model docstrings --- maro/rl/__init__.py | 6 +++++- maro/rl/algorithms/dqn.py | 2 +- maro/rl/algorithms/utils.py | 1 - maro/rl/models/learning_model.py | 10 ++++++---- 4 files changed, 12 insertions(+), 7 deletions(-) diff --git a/maro/rl/__init__.py b/maro/rl/__init__.py index 676b47a7c..3fb556beb 100644 --- a/maro/rl/__init__.py +++ b/maro/rl/__init__.py @@ -8,6 +8,7 @@ from maro.rl.agent.simple_agent_manager import SimpleAgentManager from maro.rl.algorithms.abs_algorithm import AbsAlgorithm from maro.rl.algorithms.dqn import DQN, DQNConfig, DuelingDQNTask +from maro.rl.algorithms.utils import preprocess, to_device, validate_task_names from maro.rl.dist_topologies.experience_collection import ( concat_experiences_by_agent, merge_experiences_with_trajectory_boundaries ) @@ -66,5 +67,8 @@ 'StateShaper', 'TwoPhaseLinearExplorer', 'concat_experiences_by_agent', - 'merge_experiences_with_trajectory_boundaries' + 'merge_experiences_with_trajectory_boundaries', + 'preprocess', + 'to_device', + 'validate_task_names' ] diff --git a/maro/rl/algorithms/dqn.py b/maro/rl/algorithms/dqn.py index ec39c0afe..39b3eb668 100644 --- a/maro/rl/algorithms/dqn.py +++ b/maro/rl/algorithms/dqn.py @@ -8,7 +8,7 @@ from maro.rl.algorithms.abs_algorithm import AbsAlgorithm from maro.rl.models.learning_model import LearningModel -from .utils import validate_task_names, to_device, preprocess +from .utils import preprocess, to_device, validate_task_names class DuelingDQNTask(Enum): diff --git a/maro/rl/algorithms/utils.py b/maro/rl/algorithms/utils.py index 952c23c8c..c2b4314f2 100644 --- a/maro/rl/algorithms/utils.py +++ b/maro/rl/algorithms/utils.py @@ -9,7 +9,6 @@ from maro.utils.exception.rl_toolkit_exception import UnrecognizedTaskError - device = torch.device("cuda" if torch.cuda.is_available() else "cpu") diff --git a/maro/rl/models/learning_model.py b/maro/rl/models/learning_model.py index 09fc81616..9476a7cee 100644 --- a/maro/rl/models/learning_model.py +++ b/maro/rl/models/learning_model.py @@ -136,10 +136,12 @@ def forward(self, inputs, task_name: str = None, is_training: bool = True): Args: inputs: Inputs to the model. - task_name (str): The name of the task for which the network output is required. If this is None, the results from - all task heads will be returned in the form of a dictionary. If this is a list, the results will be the - outputs from the heads contained in task in the form of a dictionary. If this is a single key, the - result will be the output from the corresponding head. + task_name (str): The name of the task for which the network output is required. If the model contains only + one task module, the task_name is ignored and the output of that module will be returned. If the model + contains multiple task modules, then 1) if task_name is None, the output from all task modules will be + returned in the form of a dictionary; 2) if task_name is a list, the outputs from the task modules + specified in the list will be returned in the form of a dictionary; 3) if this is a single string, + the output from the corresponding task module will be returned. is_training (bool): If true, all torch submodules will be set to training mode, and auto-differentiation will be turned on. Defaults to True. From 04d203360903e6f07c6815587d7ed18f8e8edde7 Mon Sep 17 00:00:00 2001 From: ysqyang Date: Fri, 13 Nov 2020 15:12:38 +0800 Subject: [PATCH 190/337] rm trailing whitespaces --- maro/rl/models/learning_model.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/maro/rl/models/learning_model.py b/maro/rl/models/learning_model.py index 9476a7cee..f766e120f 100644 --- a/maro/rl/models/learning_model.py +++ b/maro/rl/models/learning_model.py @@ -141,7 +141,7 @@ def forward(self, inputs, task_name: str = None, is_training: bool = True): contains multiple task modules, then 1) if task_name is None, the output from all task modules will be returned in the form of a dictionary; 2) if task_name is a list, the outputs from the task modules specified in the list will be returned in the form of a dictionary; 3) if this is a single string, - the output from the corresponding task module will be returned. + the output from the corresponding task module will be returned. is_training (bool): If true, all torch submodules will be set to training mode, and auto-differentiation will be turned on. Defaults to True. From 1b364bc5a139e990610cf8113e7d1619c88552d0 Mon Sep 17 00:00:00 2001 From: ysqyang Date: Fri, 13 Nov 2020 17:06:55 +0800 Subject: [PATCH 191/337] added decorator for choose_action --- maro/rl/algorithms/dqn.py | 14 +++++--------- maro/rl/algorithms/utils.py | 10 ++++++++++ 2 files changed, 15 insertions(+), 9 deletions(-) diff --git a/maro/rl/algorithms/dqn.py b/maro/rl/algorithms/dqn.py index 39b3eb668..52f115e9c 100644 --- a/maro/rl/algorithms/dqn.py +++ b/maro/rl/algorithms/dqn.py @@ -8,7 +8,7 @@ from maro.rl.algorithms.abs_algorithm import AbsAlgorithm from maro.rl.models.learning_model import LearningModel -from .utils import preprocess, to_device, validate_task_names +from .utils import add_zeroth_dim, preprocess, to_device, validate_task_names class DuelingDQNTask(Enum): @@ -75,17 +75,13 @@ def __init__(self, model: LearningModel, config: DQNConfig): self._training_counter = 0 self._target_model = model.copy() if model.is_trainable else None + @add_zeroth_dim @preprocess - def choose_action(self, state: np.ndarray, epsilon: float = None): - if epsilon is None or np.random.rand() > epsilon: - q_values = self._get_q_values(self._model, state, is_training=False) - return q_values.argmax(dim=1).item() - - return np.random.choice(self._config.num_actions) + def choose_action(self, state: np.ndarray): + q_values = self._get_q_values(self._model, state, is_training=False) + return q_values.argmax(dim=1).item() def _get_q_values(self, model, states, is_training: bool = True): - if len(states.shape) == 1: - states = states.unsqueeze(dim=0) if self._config.advantage_mode is not None: output = model(states, is_training=is_training) state_values = output["state_value"] diff --git a/maro/rl/algorithms/utils.py b/maro/rl/algorithms/utils.py index c2b4314f2..e3a0e9976 100644 --- a/maro/rl/algorithms/utils.py +++ b/maro/rl/algorithms/utils.py @@ -49,3 +49,13 @@ def wrapper(*args, **kwargs): return func(*converted_args, **converted_kwargs) return wrapper + + +def add_zeroth_dim(func): + @wraps(func) + def wrapper(self, state, **kwargs): + if len(state.dim) == 1: + state = state.unsqueeze(dim=0) + return func(self, state, **kwargs) + + return wrapper From d94d9adec176126c4ec65db7a46664b302bb12c0 Mon Sep 17 00:00:00 2001 From: ysqyang Date: Fri, 13 Nov 2020 17:09:41 +0800 Subject: [PATCH 192/337] fixed a bug --- maro/rl/algorithms/dqn.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/maro/rl/algorithms/dqn.py b/maro/rl/algorithms/dqn.py index 52f115e9c..b28ccd313 100644 --- a/maro/rl/algorithms/dqn.py +++ b/maro/rl/algorithms/dqn.py @@ -77,7 +77,7 @@ def __init__(self, model: LearningModel, config: DQNConfig): @add_zeroth_dim @preprocess - def choose_action(self, state: np.ndarray): + def choose_action(self, state: np.ndarray, epsilon=None): q_values = self._get_q_values(self._model, state, is_training=False) return q_values.argmax(dim=1).item() From 4a0f89be85753189f9b5e961aa0f36dd7551ba21 Mon Sep 17 00:00:00 2001 From: ysqyang Date: Fri, 13 Nov 2020 18:06:18 +0800 Subject: [PATCH 193/337] fixed a bug --- maro/rl/algorithms/dqn.py | 1 - maro/rl/algorithms/utils.py | 4 +++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/maro/rl/algorithms/dqn.py b/maro/rl/algorithms/dqn.py index b28ccd313..18a992831 100644 --- a/maro/rl/algorithms/dqn.py +++ b/maro/rl/algorithms/dqn.py @@ -76,7 +76,6 @@ def __init__(self, model: LearningModel, config: DQNConfig): self._target_model = model.copy() if model.is_trainable else None @add_zeroth_dim - @preprocess def choose_action(self, state: np.ndarray, epsilon=None): q_values = self._get_q_values(self._model, state, is_training=False) return q_values.argmax(dim=1).item() diff --git a/maro/rl/algorithms/utils.py b/maro/rl/algorithms/utils.py index e3a0e9976..d9214591e 100644 --- a/maro/rl/algorithms/utils.py +++ b/maro/rl/algorithms/utils.py @@ -54,7 +54,9 @@ def wrapper(*args, **kwargs): def add_zeroth_dim(func): @wraps(func) def wrapper(self, state, **kwargs): - if len(state.dim) == 1: + if isinstance(state, np.ndarray): + state = torch.from_numpy(state).to(device) + if len(state.shape) == 1: state = state.unsqueeze(dim=0) return func(self, state, **kwargs) From 26d09cf5722385c0a918f2de66fadc8a20aafbcc Mon Sep 17 00:00:00 2001 From: ysqyang Date: Fri, 13 Nov 2020 18:26:29 +0800 Subject: [PATCH 194/337] fixed version-related issues --- maro/rl/agent/abs_agent.py | 4 ++-- maro/rl/agent/simple_agent_manager.py | 2 +- maro/rl/algorithms/dqn.py | 7 +++++-- 3 files changed, 8 insertions(+), 5 deletions(-) diff --git a/maro/rl/agent/abs_agent.py b/maro/rl/agent/abs_agent.py index 6ac28047c..f84bf2c0a 100644 --- a/maro/rl/agent/abs_agent.py +++ b/maro/rl/agent/abs_agent.py @@ -40,7 +40,7 @@ def experience_pool(self): """Underlying experience pool where the agent stores experiences.""" return self._experience_pool - def choose_action(self, model_state, epsilon: float = .0): + def choose_action(self, model_state, epsilon: float = None): """Choose an action using the underlying algorithm based on a preprocessed env state. Args: @@ -49,7 +49,7 @@ def choose_action(self, model_state, epsilon: float = .0): Returns: Action given by the underlying policy model. """ - return self._algorithm.choose_action(model_state, epsilon) + return self._algorithm.choose_action(model_state, epsilon=epsilon) @abstractmethod def train(self, *args, **kwargs): diff --git a/maro/rl/agent/simple_agent_manager.py b/maro/rl/agent/simple_agent_manager.py index 4849d29b6..11e7777a6 100644 --- a/maro/rl/agent/simple_agent_manager.py +++ b/maro/rl/agent/simple_agent_manager.py @@ -46,7 +46,7 @@ def choose_action(self, decision_event, snapshot_list, epsilon_dict: dict = None self._assert_inference_mode() agent_id, model_state = self._state_shaper(decision_event, snapshot_list) model_action = self.agent_dict[agent_id].choose_action( - model_state, epsilon_dict[agent_id] if epsilon_dict else None + model_state, epsilon=epsilon_dict[agent_id] if epsilon_dict else None ) self._transition_cache = { "state": model_state, diff --git a/maro/rl/algorithms/dqn.py b/maro/rl/algorithms/dqn.py index 18a992831..915e0b7b6 100644 --- a/maro/rl/algorithms/dqn.py +++ b/maro/rl/algorithms/dqn.py @@ -77,8 +77,11 @@ def __init__(self, model: LearningModel, config: DQNConfig): @add_zeroth_dim def choose_action(self, state: np.ndarray, epsilon=None): - q_values = self._get_q_values(self._model, state, is_training=False) - return q_values.argmax(dim=1).item() + if epsilon is None or np.random.rand() > epsilon: + q_values = self._get_q_values(self._model, state, is_training=False) + return q_values.argmax(dim=1).item() + + return np.random.choice(self._config.num_actions) def _get_q_values(self, model, states, is_training: bool = True): if self._config.advantage_mode is not None: From b1a77ac33bd47500263c8b0e4ca817083ded1602 Mon Sep 17 00:00:00 2001 From: ysqyang Date: Fri, 13 Nov 2020 18:28:53 +0800 Subject: [PATCH 195/337] renamed add_zeroth_dim decorator to expand_dim --- maro/rl/algorithms/dqn.py | 4 ++-- maro/rl/algorithms/utils.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/maro/rl/algorithms/dqn.py b/maro/rl/algorithms/dqn.py index 915e0b7b6..7d104ca84 100644 --- a/maro/rl/algorithms/dqn.py +++ b/maro/rl/algorithms/dqn.py @@ -8,7 +8,7 @@ from maro.rl.algorithms.abs_algorithm import AbsAlgorithm from maro.rl.models.learning_model import LearningModel -from .utils import add_zeroth_dim, preprocess, to_device, validate_task_names +from .utils import expand_dim, preprocess, to_device, validate_task_names class DuelingDQNTask(Enum): @@ -75,7 +75,7 @@ def __init__(self, model: LearningModel, config: DQNConfig): self._training_counter = 0 self._target_model = model.copy() if model.is_trainable else None - @add_zeroth_dim + @expand_dim def choose_action(self, state: np.ndarray, epsilon=None): if epsilon is None or np.random.rand() > epsilon: q_values = self._get_q_values(self._model, state, is_training=False) diff --git a/maro/rl/algorithms/utils.py b/maro/rl/algorithms/utils.py index d9214591e..e4c7a9faf 100644 --- a/maro/rl/algorithms/utils.py +++ b/maro/rl/algorithms/utils.py @@ -51,7 +51,7 @@ def wrapper(*args, **kwargs): return wrapper -def add_zeroth_dim(func): +def expand_dim(func): @wraps(func) def wrapper(self, state, **kwargs): if isinstance(state, np.ndarray): From 56b3d9d3a53761f6380450cb4357c6e6c440967e Mon Sep 17 00:00:00 2001 From: ysqyang Date: Mon, 16 Nov 2020 15:59:01 +0800 Subject: [PATCH 196/337] overhauled exploration abstraction --- docs/source/examples/cim.rst | 4 +- docs/source/examples/multi_agent_dqn_cim.rst | 4 +- examples/cim/dqn/components/agent.py | 16 ++- examples/cim/dqn/components/agent_manager.py | 6 +- examples/cim/dqn/config.yml | 14 +-- examples/cim/dqn/dist_learner.py | 10 +- examples/cim/dqn/single_process_launcher.py | 25 ++-- examples/cim/gnn/launcher.py | 2 +- maro/rl/__init__.py | 12 +- maro/rl/actor/abs_actor.py | 16 +-- maro/rl/actor/simple_actor.py | 24 ++-- maro/rl/agent/abs_agent.py | 37 +++++- maro/rl/agent/abs_agent_manager.py | 31 ++++- maro/rl/agent/simple_agent_manager.py | 6 +- maro/rl/{explorer => exploration}/__init__.py | 0 maro/rl/exploration/abs_explorer.py | 32 +++++ .../rl/exploration/epsilon_greedy_explorer.py | 45 +++++++ maro/rl/exploration/epsilon_schedule.py | 51 ++++++++ maro/rl/explorer/abs_explorer.py | 19 --- maro/rl/explorer/simple_explorer.py | 49 -------- maro/rl/learner/abs_learner.py | 2 +- maro/rl/learner/simple_learner.py | 114 ++++++++---------- maro/utils/exception/error_code.py | 4 +- maro/utils/exception/rl_toolkit_exception.py | 12 ++ .../rl_formulation.ipynb | 2 +- 25 files changed, 335 insertions(+), 202 deletions(-) rename maro/rl/{explorer => exploration}/__init__.py (100%) create mode 100644 maro/rl/exploration/abs_explorer.py create mode 100644 maro/rl/exploration/epsilon_greedy_explorer.py create mode 100644 maro/rl/exploration/epsilon_schedule.py delete mode 100644 maro/rl/explorer/abs_explorer.py delete mode 100644 maro/rl/explorer/simple_explorer.py diff --git a/docs/source/examples/cim.rst b/docs/source/examples/cim.rst index 17ea1642d..039b0072a 100644 --- a/docs/source/examples/cim.rst +++ b/docs/source/examples/cim.rst @@ -196,7 +196,7 @@ policies. learner = SimpleLearner(trainable_agents=agent_manager, actor=actor, logger=Logger("single_host_cim_learner", auto_timestamp=False)) - learner.train(total_episodes=config.general.total_training_episodes) + learner.learn(total_episodes=config.general.total_training_episodes) Main Loop with Actor and Learner (Distributed / Multi-process) @@ -251,5 +251,5 @@ inside. learner = SimpleLearner(trainable_agents=agent_manager, actor=ActorProxy(proxy_params=proxy_params), logger=Logger("distributed_cim_learner", auto_timestamp=False)) - learner.train(total_episodes=config.general.total_training_episodes) + learner.learn(total_episodes=config.general.total_training_episodes) diff --git a/docs/source/examples/multi_agent_dqn_cim.rst b/docs/source/examples/multi_agent_dqn_cim.rst index 0f803090f..edc715133 100644 --- a/docs/source/examples/multi_agent_dqn_cim.rst +++ b/docs/source/examples/multi_agent_dqn_cim.rst @@ -208,7 +208,7 @@ policies. explorer=TwoPhaseLinearExplorer(**config.exploration), logger=Logger("single_host_cim_learner", auto_timestamp=False) ) - learner.train( + learner.learn( max_episode=config.general.max_episode, early_stopping_checker=early_stopping_checker, warmup_ep=config.general.early_stopping.warmup_ep, @@ -283,7 +283,7 @@ inside. explorer=TwoPhaseLinearExplorer(**config.exploration), logger=Logger("distributed_cim_learner", auto_timestamp=False) ) - learner.train( + learner.learn( max_episode=config.general.max_episode, early_stopping_checker=early_stopping_checker, warmup_ep=config.general.early_stopping.warmup_ep, diff --git a/examples/cim/dqn/components/agent.py b/examples/cim/dqn/components/agent.py index 4fc8e15f1..7a34d0b04 100644 --- a/examples/cim/dqn/components/agent.py +++ b/examples/cim/dqn/components/agent.py @@ -3,7 +3,7 @@ import numpy as np -from maro.rl import AbsAgent, ColumnBasedStore +from maro.rl import AbsAgent, EpsilonGreedyExplorer, ColumnBasedStore class CIMAgent(AbsAgent): @@ -14,9 +14,17 @@ class CIMAgent(AbsAgent): num_batches: number of batches to train the DQN model on per call to ``train``. batch_size: mini-batch size. """ - def __init__(self, name, algorithm, experience_pool: ColumnBasedStore, min_experiences_to_train, - num_batches, batch_size): - super().__init__(name, algorithm, experience_pool) + def __init__( + self, + name: str, + algorithm, + explorer: EpsilonGreedyExplorer, + experience_pool: ColumnBasedStore, + min_experiences_to_train, + num_batches, + batch_size + ): + super().__init__(name, algorithm, explorer, experience_pool) self._min_experiences_to_train = min_experiences_to_train self._num_batches = num_batches self._batch_size = batch_size diff --git a/examples/cim/dqn/components/agent_manager.py b/examples/cim/dqn/components/agent_manager.py index f668d1779..3f7ec3117 100644 --- a/examples/cim/dqn/components/agent_manager.py +++ b/examples/cim/dqn/components/agent_manager.py @@ -1,11 +1,14 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT license. +import numpy as np + import torch.nn as nn from torch.optim import RMSprop from maro.rl import ( - ColumnBasedStore, DQN, DQNHyperParams, FullyConnectedBlock, SimpleAgentManager, SingleHeadLearningModel + ColumnBasedStore, DQN, DQNHyperParams, EpsilonGreedyExplorer, FullyConnectedBlock, SimpleAgentManager, + SingleHeadLearningModel ) from maro.utils import set_seeds @@ -43,6 +46,7 @@ def create_dqn_agents(agent_id_list, config): agent_dict[agent_id] = CIMAgent( name=agent_id, algorithm=algorithm, + explorer=EpsilonGreedyExplorer(num_actions), experience_pool=experience_pool, **config.training_loop_parameters ) diff --git a/examples/cim/dqn/config.yml b/examples/cim/dqn/config.yml index bed3bcccd..a07eca1ba 100644 --- a/examples/cim/dqn/config.yml +++ b/examples/cim/dqn/config.yml @@ -2,8 +2,13 @@ env: scenario: "cim" topology: "toy.4p_ssdd_l0.0" durations: 1120 -general: - max_episode: 500 +main_loop: + exploration: + max_ep: 500 + split_ep: 250 + start_eps: 0.4 + mid_eps: 0.32 + end_eps: 0.0 early_stopping: warmup_ep: 50 last_k: 10 @@ -36,11 +41,6 @@ experience_shaping: fulfillment_factor: 1.0 shortage_factor: 1.0 time_decay_factor: 0.97 -exploration: - start_eps: 0.4 - mid_eps: 0.32 - end_eps: 0.0 - split_point: 0.5 agents: algorithm: num_actions: 21 diff --git a/examples/cim/dqn/dist_learner.py b/examples/cim/dqn/dist_learner.py index 2f40dbe46..9f59e0c8c 100644 --- a/examples/cim/dqn/dist_learner.py +++ b/examples/cim/dqn/dist_learner.py @@ -6,7 +6,9 @@ from components.agent_manager import DQNAgentManager, create_dqn_agents from components.config import set_input_dim -from maro.rl import ActorProxy, AgentManagerMode, SimpleLearner, TwoPhaseLinearExplorer, concat_experiences_by_agent +from maro.rl import ( + ActorProxy, AgentManagerMode, SimpleLearner, concat_experiences_by_agent, two_phase_linear_epsilon_schedule +) from maro.simulator import Env from maro.utils import Logger, convert_dottable @@ -29,13 +31,13 @@ def launch(config): "redis_address": ("localhost", 6379) } + exploration_schedule = two_phase_linear_epsilon_schedule(**config.main_loop.exploration) learner = SimpleLearner( - trainable_agents=agent_manager, + agent_manager=agent_manager, actor=ActorProxy(proxy_params=proxy_params, experience_collecting_func=concat_experiences_by_agent), - explorer=TwoPhaseLinearExplorer(**config.exploration), logger=Logger("distributed_cim_learner", auto_timestamp=False) ) - learner.train(max_episode=config.general.max_episode) + learner.learn(exploration_schedule) learner.test() learner.dump_models(os.path.join(os.getcwd(), "models")) learner.exit() diff --git a/examples/cim/dqn/single_process_launcher.py b/examples/cim/dqn/single_process_launcher.py index 864ce8b32..1b064699c 100644 --- a/examples/cim/dqn/single_process_launcher.py +++ b/examples/cim/dqn/single_process_launcher.py @@ -14,7 +14,7 @@ from maro.rl import ( AgentManagerMode, KStepExperienceShaper, MaxDeltaEarlyStoppingChecker, SimpleActor, SimpleEarlyStoppingChecker, - SimpleLearner, TwoPhaseLinearExplorer + SimpleLearner, two_phase_linear_epsilon_schedule ) from maro.simulator import Env from maro.utils import Logger, convert_dottable @@ -27,11 +27,12 @@ def launch(config): # Step 1: Initialize a CIM environment for using a toy dataset. env = Env(config.env.scenario, config.env.topology, durations=config.env.durations) agent_id_list = [str(agent_id) for agent_id in env.agent_idx_list] + action_space = list(np.linspace(-1.0, 1.0, config.agents.algorithm.num_actions)) # Step 2: Create state, action and experience shapers. We also need to create an explorer here due to the # greedy nature of the DQN algorithm. state_shaper = CIMStateShaper(**config.state_shaping) - action_shaper = CIMActionShaper(action_space=list(np.linspace(-1.0, 1.0, config.agents.algorithm.num_actions))) + action_shaper = CIMActionShaper(action_space=action_space) if config.experience_shaping.type == "truncated": experience_shaper = TruncatedExperienceShaper(**config.experience_shaping.truncated) else: @@ -52,29 +53,29 @@ def launch(config): # Step 4: Create an actor and a learner to start the training process. perf_checker = SimpleEarlyStoppingChecker( - last_k=config.general.early_stopping.last_k, - threshold=config.general.early_stopping.perf_threshold, + last_k=config.main_loop.early_stopping.last_k, + threshold=config.main_loop.early_stopping.perf_threshold, measure_func=lambda vals: mean(vals) ) perf_stability_checker = MaxDeltaEarlyStoppingChecker( - last_k=config.general.early_stopping.last_k, - threshold=config.general.early_stopping.perf_stability_threshold + last_k=config.main_loop.early_stopping.last_k, + threshold=config.main_loop.early_stopping.perf_stability_threshold ) combined_checker = perf_checker & perf_stability_checker - actor = SimpleActor(env=env, inference_agents=agent_manager) + exploration_schedule = two_phase_linear_epsilon_schedule(**config.main_loop.exploration) + actor = SimpleActor(env, agent_manager) learner = SimpleLearner( - trainable_agents=agent_manager, + agent_manager=agent_manager, actor=actor, - explorer=TwoPhaseLinearExplorer(**config.exploration), logger=Logger("single_host_cim_learner", auto_timestamp=False) ) - learner.train( - max_episode=config.general.max_episode, + learner.learn( + exploration_schedule, early_stopping_checker=combined_checker, - warmup_ep=config.general.early_stopping.warmup_ep, + warmup_ep=config.main_loop.early_stopping.warmup_ep, early_stopping_metric_func=lambda x: 1 - x["container_shortage"] / x["order_requirements"], ) learner.test() diff --git a/examples/cim/gnn/launcher.py b/examples/cim/gnn/launcher.py index df450d2dd..42189c70d 100644 --- a/examples/cim/gnn/launcher.py +++ b/examples/cim/gnn/launcher.py @@ -63,7 +63,7 @@ # Learner function for training and testing. learner = GNNLearner(actor, agent_manager, logger=simulation_logger) - learner.train(config.training) + learner.learn(config.training) # Cancel all the child process used for rollout. actor.exit() diff --git a/maro/rl/__init__.py b/maro/rl/__init__.py index a5ad26742..96bf466fc 100644 --- a/maro/rl/__init__.py +++ b/maro/rl/__init__.py @@ -16,8 +16,9 @@ from maro.rl.early_stopping.simple_early_stopping_checker import ( MaxDeltaEarlyStoppingChecker, RSDEarlyStoppingChecker, SimpleEarlyStoppingChecker ) -from maro.rl.explorer.abs_explorer import AbsExplorer -from maro.rl.explorer.simple_explorer import LinearExplorer, TwoPhaseLinearExplorer +from maro.rl.exploration.abs_explorer import AbsExplorer +from maro.rl.exploration.epsilon_greedy_explorer import EpsilonGreedyExplorer +from maro.rl.exploration.epsilon_schedule import linear_epsilon_schedule, two_phase_linear_epsilon_schedule from maro.rl.learner.abs_learner import AbsLearner from maro.rl.learner.simple_learner import SimpleLearner from maro.rl.models.fc_block import FullyConnectedBlock @@ -48,10 +49,10 @@ 'ColumnBasedStore', 'DQN', 'DQNHyperParams', + 'EpsilonGreedyExplorer', 'ExperienceShaper', 'FullyConnectedBlock', 'KStepExperienceShaper', - 'LinearExplorer', 'MaxDeltaEarlyStoppingChecker', 'MultiHeadLearningModel', 'OverwriteType', @@ -62,7 +63,8 @@ 'SimpleLearner', 'SingleHeadLearningModel', 'StateShaper', - 'TwoPhaseLinearExplorer', 'concat_experiences_by_agent', - 'merge_experiences_with_trajectory_boundaries' + 'linear_epsilon_schedule', + 'merge_experiences_with_trajectory_boundaries', + 'two_phase_linear_epsilon_schedule' ] diff --git a/maro/rl/actor/abs_actor.py b/maro/rl/actor/abs_actor.py index eae8c5118..83830bc9f 100644 --- a/maro/rl/actor/abs_actor.py +++ b/maro/rl/actor/abs_actor.py @@ -16,22 +16,22 @@ class AbsActor(ABC): Args: env (Env): An Env instance. - inference_agents (AbsAgentManager or dict): A dict of agents or an AgentManager instance that manages + agents (AbsAgentManager or dict): A dict of agents or an AgentManager instance that manages all agents. """ - def __init__(self, env: Env, inference_agents: Union[AbsAgentManager, dict]): + def __init__(self, env: Env, agents: Union[AbsAgentManager, dict]): self._env = env - self._inference_agents = inference_agents + self._agents = agents @abstractmethod - def roll_out(self, model_dict: dict = None, epsilon_dict: dict = None, done: bool = None, - return_details: bool = True): + def roll_out( + self, model_dict: dict = None, done: bool = None, return_details: bool = True + ): """This method performs a single episode of roll-out. Args: model_dict (dict): If not None, the agents will load the models from model_dict and use these models to perform roll-out. - epsilon_dict (dict): Exploration rate by agent. done (bool): If True, the current call is the last call, i.e., no more roll-outs will be performed. This flag is used to signal remote actor workers to exit. return_details (bool): If True, return episode details (e.g., experiences) as well as performance @@ -43,6 +43,6 @@ def roll_out(self, model_dict: dict = None, epsilon_dict: dict = None, done: boo return NotImplementedError @property - def inference_agents(self): + def agents(self): """Agents performing inference during roll-out.""" - return self._inference_agents + return self._agents diff --git a/maro/rl/actor/simple_actor.py b/maro/rl/actor/simple_actor.py index b54707c02..85e6b6fe3 100644 --- a/maro/rl/actor/simple_actor.py +++ b/maro/rl/actor/simple_actor.py @@ -12,20 +12,20 @@ class SimpleActor(AbsActor): Args: env (Env): An Env instance. - inference_agents (AbsAgentManager): An AgentManager instance that manages all agents. + agent_manager (SimpleAgentManager): An AgentManager instance that manages all agents. """ - def __init__(self, env: Env, inference_agents: SimpleAgentManager): - super().__init__(env, inference_agents) + def __init__(self, env: Env, agent_manager: SimpleAgentManager): + super().__init__(env, agent_manager) def roll_out( - self, model_dict: dict = None, epsilon_dict: dict = None, done: bool = False, return_details: bool = True + self, model_dict: dict = None, exploration_params=None, done: bool = False, return_details: bool = True ): """Perform one episode of roll-out and return performance and experiences. Args: model_dict (dict): If not None, the agents will load the models from model_dict and use these models to perform roll-out. - epsilon_dict (dict): Exploration rate by agent. + exploration_params: Exploration parameters. done (bool): If True, the current call is the last call, i.e., no more roll-outs will be performed. This flag is used to signal remote actor workers to exit. return_details (bool): If True, return experiences as well as performance metrics provided by the env. @@ -40,16 +40,18 @@ def roll_out( # load models if model_dict is not None: - self._inference_agents.load_models(model_dict) + self._agents.load_models(model_dict) + + # load exploration parameters: + if exploration_params is not None: + self._agents.load_exploration_params(exploration_params) metrics, decision_event, is_done = self._env.step(None) while not is_done: - action = self._inference_agents.choose_action( - decision_event, self._env.snapshot_list, epsilon_dict=epsilon_dict - ) + action = self._agents.choose_action(decision_event, self._env.snapshot_list) metrics, decision_event, is_done = self._env.step(action) - self._inference_agents.on_env_feedback(metrics) + self._agents.on_env_feedback(metrics) - details = self._inference_agents.post_process(self._env.snapshot_list) if return_details else None + details = self._agents.post_process(self._env.snapshot_list) if return_details else None return self._env.metrics, details diff --git a/maro/rl/agent/abs_agent.py b/maro/rl/agent/abs_agent.py index 4a68c256a..76a3b69e7 100644 --- a/maro/rl/agent/abs_agent.py +++ b/maro/rl/agent/abs_agent.py @@ -6,7 +6,9 @@ from abc import ABC, abstractmethod from maro.rl.algorithms.abs_algorithm import AbsAlgorithm +from maro.rl.exploration.abs_explorer import AbsExplorer from maro.rl.storage.abs_store import AbsStore +from maro.utils.exception.rl_toolkit_exception import MissingExplorerError class AbsAgent(ABC): @@ -23,12 +25,20 @@ class AbsAgent(ABC): algorithm (AbsAlgorithm): A concrete algorithm instance that inherits from AbstractAlgorithm. This is the centerpiece of the Agent class and is responsible for the most important tasks of an agent: choosing actions and optimizing models. + explorer (AbsExplorer): Explorer instance to generate exploratory actions. Defaults to None. experience_pool (AbsStore): It is used to store experiences processed by the experience shaper, which will be used by some value-based algorithms, such as DQN. Defaults to None. """ - def __init__(self, name: str, algorithm: AbsAlgorithm, experience_pool: AbsStore = None): + def __init__( + self, + name: str, + algorithm: AbsAlgorithm, + explorer: AbsExplorer = None, + experience_pool: AbsStore = None + ): self._name = name self._algorithm = algorithm + self._explorer = explorer self._experience_pool = experience_pool @property @@ -36,21 +46,40 @@ def algorithm(self): """Underlying algorithm employed by the agent.""" return self._algorithm + @property + def explorer(self): + """Explorer used by the agent to generate exploratory actions.""" + return self._explorer + @property def experience_pool(self): """Underlying experience pool where the agent stores experiences.""" return self._experience_pool - def choose_action(self, model_state, epsilon: float = .0): + def choose_action(self, model_state): """Choose an action using the underlying algorithm based on a preprocessed env state. Args: model_state: State vector as accepted by the underlying algorithm. - epsilon (float): Exploration rate. Returns: Action given by the underlying policy model. """ - return self._algorithm.choose_action(model_state, epsilon) + action_from_algorithm = self._algorithm.choose_action(model_state) + return action_from_algorithm if self._explorer is None else self._explorer(action_from_algorithm) + + def load_exploration_params(self, exploration_params): + if self._explorer is None: + raise MissingExplorerError( + "No explorer found. Make sure to pass an explorer instance when creating the agent." + ) + self._explorer.load_exploration_params(exploration_params) + + def update_exploration_params(self): + if self._explorer is None: + raise MissingExplorerError( + "No explorer found. Make sure to pass an explorer instance when creating the agent." + ) + self._explorer.update() @abstractmethod def train(self, *args, **kwargs): diff --git a/maro/rl/agent/abs_agent_manager.py b/maro/rl/agent/abs_agent_manager.py index 2c2299d92..477767cfa 100644 --- a/maro/rl/agent/abs_agent_manager.py +++ b/maro/rl/agent/abs_agent_manager.py @@ -3,6 +3,7 @@ from abc import ABC, abstractmethod from enum import Enum +from typing import Iterator, Union from maro.rl.shaping.action_shaper import ActionShaper from maro.rl.shaping.experience_shaper import ExperienceShaper @@ -54,6 +55,11 @@ def __init__( def __getitem__(self, agent_id): return self.agent_dict[agent_id] + @property + def name(self): + """Agent manager's name.""" + return self._name + @abstractmethod def choose_action(self, *args, **kwargs): """Generate an environment executable action given the current decision event and snapshot list. @@ -82,10 +88,27 @@ def train(self, *args, **kwargs): """Train the agents.""" return NotImplemented - @property - def name(self): - """Agent manager's name.""" - return self._name + def register_exploration_schedule(self, exploration_schedule: Union[Iterator, dict]): + for agent_id, agent in self.agent_dict.values(): + agent.explorer.register_schedule( + exploration_schedule[agent_id] if isinstance(exploration_schedule, dict) else exploration_schedule + ) + + def load_exploration_params(self, exploration_params): + is_per_agent = set(exploration_params).issubset(set(self.agent_dict.keys())) + if is_per_agent: + for agent_id, params in exploration_params.items(): + self.agent_dict[agent_id].load_exploration_params(params) + else: + for agent in self.agent_dict.values(): + agent.load_exploration_params(exploration_params) + + def dump_exploration_params(self): + return {agent_id: agent.dump_exploration_params() for agent_id, agent in self.agent_dict.items()} + + def update_exploration_params(self): + for agent in self.agent_dict.values(): + agent.explorer.update() def _assert_train_mode(self): if self._mode != AgentManagerMode.TRAIN and self._mode != AgentManagerMode.TRAIN_INFERENCE: diff --git a/maro/rl/agent/simple_agent_manager.py b/maro/rl/agent/simple_agent_manager.py index c9d1bb875..6e3574900 100644 --- a/maro/rl/agent/simple_agent_manager.py +++ b/maro/rl/agent/simple_agent_manager.py @@ -42,12 +42,10 @@ def __init__( self._transition_cache = {} self._trajectory = ColumnBasedStore() - def choose_action(self, decision_event, snapshot_list, epsilon_dict: dict = None): + def choose_action(self, decision_event, snapshot_list): self._assert_inference_mode() agent_id, model_state = self._state_shaper(decision_event, snapshot_list) - model_action = self.agent_dict[agent_id].choose_action( - model_state, epsilon_dict[agent_id] if epsilon_dict else None - ) + model_action = self.agent_dict[agent_id].choose_action(model_state) self._transition_cache = { "state": model_state, "action": model_action, diff --git a/maro/rl/explorer/__init__.py b/maro/rl/exploration/__init__.py similarity index 100% rename from maro/rl/explorer/__init__.py rename to maro/rl/exploration/__init__.py diff --git a/maro/rl/exploration/abs_explorer.py b/maro/rl/exploration/abs_explorer.py new file mode 100644 index 000000000..25dbe8ada --- /dev/null +++ b/maro/rl/exploration/abs_explorer.py @@ -0,0 +1,32 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +from abc import ABC, abstractmethod + + +class AbsExplorer(ABC): + """Abstract explorer class for generating exploration rates. + + """ + def __init__(self): + pass + + @abstractmethod + def register_schedule(self, exploration_param_iter): + return NotImplementedError + + @abstractmethod + def load_exploration_params(self, exploration_params): + return NotImplementedError + + @abstractmethod + def dump_exploration_params(self): + return NotImplementedError + + @abstractmethod + def update(self): + return NotImplementedError + + @abstractmethod + def __call__(self, action): + return NotImplementedError diff --git a/maro/rl/exploration/epsilon_greedy_explorer.py b/maro/rl/exploration/epsilon_greedy_explorer.py new file mode 100644 index 000000000..a76dbb58a --- /dev/null +++ b/maro/rl/exploration/epsilon_greedy_explorer.py @@ -0,0 +1,45 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import random +from typing import Generator + +from maro.utils.exception.rl_toolkit_exception import MissingExplorationScheduleError + +from .abs_explorer import AbsExplorer + + +class EpsilonGreedyExplorer(AbsExplorer): + """Epsilon greedy explorer for discrete action spaces. + + Args: + num_actions (int): Number of all possible actions. + """ + def __init__(self, num_actions: int): + super().__init__() + self._epsilon_schedule = None + self._epsilon = None + self._num_actions = num_actions + + def __call__(self, action): + assert action < self._num_actions, f"Invalid action: {action}" + if self._epsilon_schedule is None or random.random() > self._epsilon: + return action + else: + return random.randrange(self._num_actions) + + def register_schedule(self, epsilon_schedule: Generator): + self._epsilon_schedule = epsilon_schedule + + def load_exploration_params(self, epsilon: float): + self._epsilon = epsilon + + def dump_exploration_params(self): + return self._epsilon + + def update(self): + if self._epsilon is None: + raise MissingExplorationScheduleError( + "An iterable epsilon schedule must be registered first by calling register_schedule()." + ) + self._epsilon = next(self._epsilon_schedule) diff --git a/maro/rl/exploration/epsilon_schedule.py b/maro/rl/exploration/epsilon_schedule.py new file mode 100644 index 000000000..4f2b063ce --- /dev/null +++ b/maro/rl/exploration/epsilon_schedule.py @@ -0,0 +1,51 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +from maro.utils.exception.rl_toolkit_exception import InvalidEpisodeError + + +def linear_epsilon_schedule(max_ep: int, start_eps: float, end_eps: float = .0): + """Linear exploration rate generator for epsilon-greedy exploration. + + Args: + max_ep (int): Maximum number of episodes to run. + start_eps (float): The exploration rate for the first episode. + end_eps (float): The exploration rate for the last episode. Defaults to zero. + + """ + if max_ep <= 0: + raise InvalidEpisodeError("max_ep must be a positive integer.") + current_eps = start_eps + eps_delta = (end_eps - start_eps) / (max_ep - 1) + + for ep in range(max_ep): + yield current_eps + current_eps += eps_delta + + +def two_phase_linear_epsilon_schedule( + max_ep: int, split_ep: float, start_eps: float, mid_eps: float, end_eps: float = .0 +): + """Exploration schedule comprised of two linear schedules joined together for epsilon-greedy exploration. + + Args: + max_ep (int): Maximum number of episodes to run. + split_ep (float): The episode where the switch from the first linear schedule to the second occurs. + start_eps (float): Exploration rate for the first episode. + mid_eps (float): The exploration rate where the switch from the first linear schedule to the second occurs. + In other words, this is the exploration rate where the first linear schedule ends and the second begins. + end_eps (float): Exploration rate for the last episode. Defaults to zero. + + Returns: + An iterator over the series of exploration rates from episode 0 to ``max_ep`` - 1. + """ + if max_ep <= 0: + raise InvalidEpisodeError("max_ep must be a positive integer.") + if split_ep <= 0 or split_ep >= max_ep: + raise ValueError("split_ep must be between 0 and max_ep - 1.") + current_eps = start_eps + eps_delta_phase_1 = (mid_eps - start_eps) / split_ep + eps_delta_phase_2 = (end_eps - mid_eps) / (max_ep - split_ep - 1) + for ep in range(max_ep): + yield current_eps + current_eps += eps_delta_phase_1 if ep < split_ep else eps_delta_phase_2 diff --git a/maro/rl/explorer/abs_explorer.py b/maro/rl/explorer/abs_explorer.py deleted file mode 100644 index 00c4e620a..000000000 --- a/maro/rl/explorer/abs_explorer.py +++ /dev/null @@ -1,19 +0,0 @@ -# Copyright (c) Microsoft Corporation. -# Licensed under the MIT license. - -from abc import ABC, abstractmethod - - -class AbsExplorer(ABC): - """Abstract explorer class for generating exploration rates. - - """ - def __init__(self): - pass - - # TODO: performance: summary -> total perf (current version), details -> per-agent perf - @abstractmethod - def generate_epsilon(self, current_ep: int, max_ep: int, performance_history=None): - """Generate an exploration rate based on the performance history. - """ - return NotImplemented diff --git a/maro/rl/explorer/simple_explorer.py b/maro/rl/explorer/simple_explorer.py deleted file mode 100644 index 53a9a45e4..000000000 --- a/maro/rl/explorer/simple_explorer.py +++ /dev/null @@ -1,49 +0,0 @@ -# Copyright (c) Microsoft Corporation. -# Licensed under the MIT license. - -from .abs_explorer import AbsExplorer - - -class LinearExplorer(AbsExplorer): - """Exploration schedule where the exploration rate decreases with the number of episodes in a linear fashion. - - Args: - max_eps (float): Maximum exploration rate, i.e., the exploration rate for the first episode. - min_eps (float): Minimum exploration rate, i.e., the exploration rate for the last episode. - """ - def __init__(self, max_eps: float, min_eps: float = .0): - super().__init__() - self._max_eps = max_eps - self._min_eps = min_eps - - def generate_epsilon(self, current_ep, max_ep, performance_history=None): - return self._min_eps + (self._max_eps - self._min_eps) * (1 - current_ep / (max_ep - 1)) - - -class TwoPhaseLinearExplorer(AbsExplorer): - """Exploration schedule that consists of two linear schedules separated by a split point. - - Args: - start_eps (float): Exploration rate for the first episode. - mid_eps (float): Exploration rate for the last episode. - end_eps (float): The exploration rate where the switch from the first linear schedule to the second occurs. - In other words, this is the exploration rate where the first linear schedule ends and the second begins. - split_point (float): The point where the switch from the first linear schedule to the second occurs. - Here "point" means the percentage of training loop completion, i.e., current_episode / max_episode, - which means it must be a floating point number between 0 and 1.0. - """ - def __init__(self, start_eps: float, mid_eps: float, end_eps: float, split_point: float): - super().__init__() - if split_point > 1.0 or split_point < 0.0: - raise ValueError("split_point must be between 0 and 1.0") - self._split_point = split_point - self._start_eps = start_eps - self._mid_eps = mid_eps - self._end_eps = end_eps - - def generate_epsilon(self, current_ep, max_ep, performance_history=None): - progress = current_ep / (max_ep - 1) - if progress <= self._split_point: - return self._start_eps - (self._start_eps - self._mid_eps) * progress / self._split_point - else: - return self._end_eps + (self._mid_eps - self._end_eps) * (1 - progress) / (1 - self._split_point) diff --git a/maro/rl/learner/abs_learner.py b/maro/rl/learner/abs_learner.py index a58582514..055a09d15 100644 --- a/maro/rl/learner/abs_learner.py +++ b/maro/rl/learner/abs_learner.py @@ -9,7 +9,7 @@ class AbsLearner(ABC): def __init__(self): pass - def train(self, *args, **kwargs): + def learn(self, *args, **kwargs): """The outermost training loop logic is implemented here.""" pass diff --git a/maro/rl/learner/simple_learner.py b/maro/rl/learner/simple_learner.py index cb8050e6b..d1e21f0df 100644 --- a/maro/rl/learner/simple_learner.py +++ b/maro/rl/learner/simple_learner.py @@ -2,14 +2,12 @@ # Licensed under the MIT license. import sys -from typing import Callable, Union +from typing import Callable, Iterable, Union from maro.rl.actor.simple_actor import SimpleActor from maro.rl.agent.simple_agent_manager import SimpleAgentManager from maro.rl.dist_topologies.single_learner_multi_actor_sync_mode import ActorProxy -from maro.rl.explorer.abs_explorer import AbsExplorer from maro.utils import DummyLogger, Logger -from maro.utils.exception.rl_toolkit_exception import InfiniteTrainingLoopError, InvalidEpisodeError from .abs_learner import AbsLearner @@ -18,108 +16,100 @@ class SimpleLearner(AbsLearner): """A simple implementation of ``AbsLearner``. Args: - trainable_agents (AbsAgentManager): An AgentManager instance that manages all agents. + agent_manager (AbsAgentManager): An AgentManager instance that manages all agents. actor (SimpleActor or ActorProxy): An SimpleActor or ActorProxy instance responsible for performing roll-outs (environment sampling). - explorer (dict or AbsExplorer): An explorer instance responsible for generating exploration rates. - Defaults to None. logger (Logger): Used to log important messages. """ def __init__( self, - trainable_agents: SimpleAgentManager, + agent_manager: SimpleAgentManager, actor: Union[SimpleActor, ActorProxy], - explorer: Union[dict, AbsExplorer] = None, logger: Logger = DummyLogger() ): super().__init__() - self._trainable_agents = trainable_agents + self._agent_manager = agent_manager self._actor = actor - self._explorer = explorer self._logger = logger self._performance_history = [] - def _get_epsilons(self, current_ep, max_ep): - if self._explorer is None: - return None - elif isinstance(self._explorer, dict): - return { - agent_id: self._explorer[agent_id].generate_epsilon(current_ep, max_ep, self._performance_history) - for agent_id in self._trainable_agents.agent_dict - } - else: - return { - agent_id: self._explorer.generate_epsilon(current_ep, max_ep, self._performance_history) - for agent_id in self._trainable_agents.agent_dict - } - - def _sample(self, ep, max_ep): - """Perform one episode of environment sampling through actor roll-out.""" - model_dict = None if self._is_shared_agent_instance() else self._trainable_agents.dump_models() - epsilon_dict = self._get_epsilons(ep, max_ep) - performance, exp_by_agent = self._actor.roll_out(model_dict=model_dict, epsilon_dict=epsilon_dict) - self._logger.info(f"ep {ep} - performance: {performance}, epsilons: {epsilon_dict}") - return performance, exp_by_agent - - def train( - self, max_episode: int, early_stopping_checker: Callable = None, warmup_ep: int = None, - early_stopping_metric_func: Callable = None + def learn( + self, + exploration_schedule: Union[Iterable, dict], + early_stopping_checker: Callable = None, + warmup_ep: int = None, + early_stopping_metric_func: Callable = None, ): """Main loop for collecting experiences from the actor and using them to update policies. Args: - max_episode (int): number of episodes to be run. If -1, the training loop will run forever unless - an ``early_stopping_checker`` is provided and the early stopping condition is met. + exploration_schedule (Union[Iterable, dict]): Explorations schedules for the underlying agents. If it is + a dictionary, the exploration schedule will be registered on a per-agent basis based on agent ID's. + If it is a single iterable object, the exploration schedule will be registered for all agents. early_stopping_checker (Callable): A Callable object to determine whether the training loop should be terminated based on the latest performances. Defaults to None. warmup_ep (int): Episode from which early stopping check is initiated. Defaults to None. early_stopping_metric_func (Callable): A function to extract the metric from a performance record for early stopping checking. Defaults to None. """ - if max_episode < -1: - raise InvalidEpisodeError("max_episode can only be a non-negative integer or -1.") - if max_episode == -1 and early_stopping_checker is None: - raise InfiniteTrainingLoopError( - "The training loop will run forever since neither maximum episode nor early stopping checker " - "is provided. " - ) if early_stopping_checker is not None: assert early_stopping_metric_func is not None, \ "early_stopping_metric_func cannot be None if early_stopping_checker is provided." - episode = 0 - metric_series = [] - while max_episode == -1 or episode < max_episode: - performance, exp_by_agent = self._sample(episode, max_episode) - latest = [perf for _, perf in performance] if isinstance(performance, list) else [performance] - if early_stopping_checker is not None: - metric_series.extend(map(early_stopping_metric_func, latest)) - if warmup_ep is None or episode >= warmup_ep and early_stopping_checker(metric_series): - self._logger.info("Early stopping condition hit. Training complete.") - break - self._trainable_agents.train(exp_by_agent) - episode += 1 + self._agent_manager.register_exploration_schedule(exploration_schedule) + ep, metric_series = 0, [] + while True: + try: + performance, exp_by_agent = self._sample() + latest = [perf for _, perf in performance] if isinstance(performance, list) else [performance] + if early_stopping_checker is not None: + metric_series.extend(map(early_stopping_metric_func, latest)) + if warmup_ep is None or ep >= warmup_ep and early_stopping_checker(metric_series): + self._logger.info("Early stopping condition hit. Training complete.") + break + except StopIteration: + self._logger.info(f"Maximum number of episodes {ep + 1} reached. Training complete.") + break + + ep += 1 + self._agent_manager.train(exp_by_agent) def test(self): """Test policy performance.""" performance, _ = self._actor.roll_out( - model_dict=self._trainable_agents.dump_models(), + model_dict=self._agent_manager.dump_models(), return_details=False ) self._logger.info(f"test performance: {performance}") def exit(self, code: int = 0): - """Tell the remote actor to exit""" + """Tell the remote actor to exit.""" if isinstance(self._actor, ActorProxy): self._actor.roll_out(done=True) sys.exit(code) def load_models(self, dir_path: str): - self._trainable_agents.load_models_from_files(dir_path) + self._agent_manager.load_models_from_files(dir_path) def dump_models(self, dir_path: str): - self._trainable_agents.dump_models_to_files(dir_path) + self._agent_manager.dump_models_to_files(dir_path) def _is_shared_agent_instance(self): - """If true, the set of agents performing inference in actor is the same as self._trainable_agents.""" - return isinstance(self._actor, SimpleActor) and id(self._actor.inference_agents) == id(self._trainable_agents) + """If true, the set of agents performing inference in actor is the same as self._agent_manager.""" + return isinstance(self._actor, SimpleActor) and id(self._actor.agents) == id(self._agent_manager) + + def _sample(self): + """Perform one episode of environment sampling through actor roll-out.""" + self._agent_manager.update_exploration_params() + if self._is_shared_agent_instance(): + model_dict, exploration_params = None, None + else: + model_dict = self._agent_manager.dump_models() + exploration_params = self._agent_manager.dump_exploration_params() + + performance, exp_by_agent = self._actor.roll_out( + model_dict=model_dict, exploration_params=exploration_params + ) + + self._logger.info(f"performance: {performance}, exploration_params: {exploration_params}") + return performance, exp_by_agent diff --git a/maro/utils/exception/error_code.py b/maro/utils/exception/error_code.py index f09dcd12f..df4b37c28 100644 --- a/maro/utils/exception/error_code.py +++ b/maro/utils/exception/error_code.py @@ -40,5 +40,7 @@ 4003: "Wrong Agent Manager Mode", 4004: "Store Misalignment Error", 4005: "Invalid Episode", - 4006: "Infinite Training Loop" + 4006: "Infinite Training Loop", + 4009: "Missing Exploration Schedule Error", + 4010: "Missing Explorer Error" } diff --git a/maro/utils/exception/rl_toolkit_exception.py b/maro/utils/exception/rl_toolkit_exception.py index c35e0b897..f602a1a78 100644 --- a/maro/utils/exception/rl_toolkit_exception.py +++ b/maro/utils/exception/rl_toolkit_exception.py @@ -39,3 +39,15 @@ class InfiniteTrainingLoopError(MAROException): """Raised when the ``SimpleLearner``'s training loop becomes infinite.""" def __init__(self, msg: str = None): super().__init__(4006, msg) + + +class MissingExplorationScheduleError(MAROException): + """Raised when calling an explorer's ``update`` method with no exploration schedule registered.""" + def __init__(self, msg: str = None): + super().__init__(4009, msg) + + +class MissingExplorerError(MAROException): + """Raised when a call to an agent's ``update_exploration_params`` is made but there is no explorer present.""" + def __init__(self, msg: str = None): + super().__init__(4009, msg) diff --git a/notebooks/container_inventory_management/rl_formulation.ipynb b/notebooks/container_inventory_management/rl_formulation.ipynb index 903a1984f..88f3b3ab5 100644 --- a/notebooks/container_inventory_management/rl_formulation.ipynb +++ b/notebooks/container_inventory_management/rl_formulation.ipynb @@ -475,7 +475,7 @@ " explorer=TwoPhaseLinearExplorer(start_eps=0.4, mid_eps=0.32, end_eps=0.0, split_point=0.5),\n", " logger=Logger(\"single_host_cim_learner\", format_=LogFormat.simple, auto_timestamp=False))\n", "\n", - "learner.train(max_episode=100)" + "learner.learn(max_episode=100)" ] }, { From 885f9b330f27994e6783addb97d79b6dc343f065 Mon Sep 17 00:00:00 2001 From: ysqyang Date: Mon, 16 Nov 2020 16:05:01 +0800 Subject: [PATCH 197/337] fixed a bug --- maro/rl/agent/abs_agent_manager.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/maro/rl/agent/abs_agent_manager.py b/maro/rl/agent/abs_agent_manager.py index 477767cfa..47144972b 100644 --- a/maro/rl/agent/abs_agent_manager.py +++ b/maro/rl/agent/abs_agent_manager.py @@ -89,7 +89,7 @@ def train(self, *args, **kwargs): return NotImplemented def register_exploration_schedule(self, exploration_schedule: Union[Iterator, dict]): - for agent_id, agent in self.agent_dict.values(): + for agent_id, agent in self.agent_dict.items(): agent.explorer.register_schedule( exploration_schedule[agent_id] if isinstance(exploration_schedule, dict) else exploration_schedule ) From 2c3cdddb77689b9be75f1ff941d71d7f373cbdd0 Mon Sep 17 00:00:00 2001 From: ysqyang Date: Mon, 16 Nov 2020 16:08:54 +0800 Subject: [PATCH 198/337] fixed a bug --- maro/rl/exploration/epsilon_greedy_explorer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/maro/rl/exploration/epsilon_greedy_explorer.py b/maro/rl/exploration/epsilon_greedy_explorer.py index a76dbb58a..2606e1d4f 100644 --- a/maro/rl/exploration/epsilon_greedy_explorer.py +++ b/maro/rl/exploration/epsilon_greedy_explorer.py @@ -38,7 +38,7 @@ def dump_exploration_params(self): return self._epsilon def update(self): - if self._epsilon is None: + if self._epsilon_schedule is None: raise MissingExplorationScheduleError( "An iterable epsilon schedule must be registered first by calling register_schedule()." ) From cca18a15c8a61fab322326f8549d80407e2c018f Mon Sep 17 00:00:00 2001 From: ysqyang Date: Mon, 16 Nov 2020 16:13:41 +0800 Subject: [PATCH 199/337] fixed a bug --- maro/rl/learner/simple_learner.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/maro/rl/learner/simple_learner.py b/maro/rl/learner/simple_learner.py index d1e21f0df..339accd0b 100644 --- a/maro/rl/learner/simple_learner.py +++ b/maro/rl/learner/simple_learner.py @@ -101,15 +101,13 @@ def _is_shared_agent_instance(self): def _sample(self): """Perform one episode of environment sampling through actor roll-out.""" self._agent_manager.update_exploration_params() + exploration_params = self._agent_manager.dump_exploration_params() if self._is_shared_agent_instance(): - model_dict, exploration_params = None, None + performance, exp_by_agent = self._actor.roll_out() else: - model_dict = self._agent_manager.dump_models() - exploration_params = self._agent_manager.dump_exploration_params() - - performance, exp_by_agent = self._actor.roll_out( - model_dict=model_dict, exploration_params=exploration_params - ) + performance, exp_by_agent = self._actor.roll_out( + model_dict=self._agent_manager.dump_models(), exploration_params=exploration_params + ) self._logger.info(f"performance: {performance}, exploration_params: {exploration_params}") return performance, exp_by_agent From 5cde3196e6c576442f1b9b975d0d45c42fe15c78 Mon Sep 17 00:00:00 2001 From: ysqyang Date: Mon, 16 Nov 2020 16:30:17 +0800 Subject: [PATCH 200/337] added exploration related methods to abs_agent --- maro/rl/agent/abs_agent.py | 22 ++++++++++++---------- maro/rl/agent/abs_agent_manager.py | 4 ++-- 2 files changed, 14 insertions(+), 12 deletions(-) diff --git a/maro/rl/agent/abs_agent.py b/maro/rl/agent/abs_agent.py index 76a3b69e7..88960193b 100644 --- a/maro/rl/agent/abs_agent.py +++ b/maro/rl/agent/abs_agent.py @@ -67,19 +67,21 @@ def choose_action(self, model_state): action_from_algorithm = self._algorithm.choose_action(model_state) return action_from_algorithm if self._explorer is None else self._explorer(action_from_algorithm) + def register_exploration_schedule(self, exploration_schedule): + if self._explorer: + self._explorer.register_schedule(exploration_schedule) + def load_exploration_params(self, exploration_params): - if self._explorer is None: - raise MissingExplorerError( - "No explorer found. Make sure to pass an explorer instance when creating the agent." - ) - self._explorer.load_exploration_params(exploration_params) + if self._explorer: + self._explorer.load_exploration_params(exploration_params) + + def dump_exploration_params(self): + if self._explorer: + return self._explorer.dump_exploration_params() def update_exploration_params(self): - if self._explorer is None: - raise MissingExplorerError( - "No explorer found. Make sure to pass an explorer instance when creating the agent." - ) - self._explorer.update() + if self._explorer: + self._explorer.update() @abstractmethod def train(self, *args, **kwargs): diff --git a/maro/rl/agent/abs_agent_manager.py b/maro/rl/agent/abs_agent_manager.py index 47144972b..1c26f89e4 100644 --- a/maro/rl/agent/abs_agent_manager.py +++ b/maro/rl/agent/abs_agent_manager.py @@ -90,7 +90,7 @@ def train(self, *args, **kwargs): def register_exploration_schedule(self, exploration_schedule: Union[Iterator, dict]): for agent_id, agent in self.agent_dict.items(): - agent.explorer.register_schedule( + agent.register_exploration_schedule( exploration_schedule[agent_id] if isinstance(exploration_schedule, dict) else exploration_schedule ) @@ -108,7 +108,7 @@ def dump_exploration_params(self): def update_exploration_params(self): for agent in self.agent_dict.values(): - agent.explorer.update() + agent.update_exploration_params() def _assert_train_mode(self): if self._mode != AgentManagerMode.TRAIN and self._mode != AgentManagerMode.TRAIN_INFERENCE: From a0a497a1a7b167315a2d4561ed5eeb1b449afaef Mon Sep 17 00:00:00 2001 From: ysqyang Date: Mon, 16 Nov 2020 23:01:33 +0800 Subject: [PATCH 201/337] fixed a bug --- maro/rl/agent/abs_agent_manager.py | 44 ++++++++++++-------- maro/utils/exception/rl_toolkit_exception.py | 2 +- 2 files changed, 28 insertions(+), 18 deletions(-) diff --git a/maro/rl/agent/abs_agent_manager.py b/maro/rl/agent/abs_agent_manager.py index 1c26f89e4..137feece5 100644 --- a/maro/rl/agent/abs_agent_manager.py +++ b/maro/rl/agent/abs_agent_manager.py @@ -3,12 +3,13 @@ from abc import ABC, abstractmethod from enum import Enum -from typing import Iterator, Union +from typing import Iterable, Union +from maro.rl.exploration.abs_explorer import AbsExplorer from maro.rl.shaping.action_shaper import ActionShaper from maro.rl.shaping.experience_shaper import ExperienceShaper from maro.rl.shaping.state_shaper import StateShaper -from maro.utils.exception.rl_toolkit_exception import WrongAgentManagerModeError +from maro.utils.exception.rl_toolkit_exception import MissingExplorerError, WrongAgentManagerModeError class AgentManagerMode(Enum): @@ -29,12 +30,14 @@ class AbsAgentManager(ABC): mode (AgentManagerMode): An ``AgentManagerNode`` enum member that indicates the role of the agent manager in the current process. agent_dict (dict): A dictionary of agents to be wrapper by the agent manager. - experience_shaper (ExperienceShaper, optional): It is responsible for processing data in the replay buffer at - the end of an episode. state_shaper (StateShaper, optional): It is responsible for converting the environment observation to model input. action_shaper (ActionShaper, optional): It is responsible for converting an agent's model output to environment executable action. Cannot be None under Inference and TrainInference modes. + experience_shaper (ExperienceShaper, optional): It is responsible for processing data in the replay buffer at + the end of an episode. + explorer (AbsExplorer): Shared explorer for all agents. If this is not None, the underlying agents' explorers + will not be used. Defaults to None. """ def __init__( self, @@ -43,7 +46,8 @@ def __init__( agent_dict: dict, state_shaper: StateShaper = None, action_shaper: ActionShaper = None, - experience_shaper: ExperienceShaper = None + experience_shaper: ExperienceShaper = None, + explorer: AbsExplorer = None ): self._name = name self._mode = mode @@ -51,6 +55,7 @@ def __init__( self._state_shaper = state_shaper self._action_shaper = action_shaper self._experience_shaper = experience_shaper + self._explorer = explorer def __getitem__(self, agent_id): return self.agent_dict[agent_id] @@ -88,27 +93,32 @@ def train(self, *args, **kwargs): """Train the agents.""" return NotImplemented - def register_exploration_schedule(self, exploration_schedule: Union[Iterator, dict]): - for agent_id, agent in self.agent_dict.items(): - agent.register_exploration_schedule( - exploration_schedule[agent_id] if isinstance(exploration_schedule, dict) else exploration_schedule - ) + def register_exploration_schedule(self, exploration_schedule: Union[Iterable, dict]): + if isinstance(exploration_schedule, dict): + for agent_id, agent in self.agent_dict.items(): + agent.register_exploration_schedule(exploration_schedule[agent_id]) + else: + self._explorer.register_schedule(exploration_schedule) def load_exploration_params(self, exploration_params): - is_per_agent = set(exploration_params).issubset(set(self.agent_dict.keys())) - if is_per_agent: + if self._explorer is None: for agent_id, params in exploration_params.items(): self.agent_dict[agent_id].load_exploration_params(params) else: - for agent in self.agent_dict.values(): - agent.load_exploration_params(exploration_params) + self._explorer.load_exploration_params(exploration_params) def dump_exploration_params(self): - return {agent_id: agent.dump_exploration_params() for agent_id, agent in self.agent_dict.items()} + if self._explorer is None: + return {agent_id: agent.dump_exploration_params() for agent_id, agent in self.agent_dict.items()} + else: + return self._explorer.dump_exploration_params() def update_exploration_params(self): - for agent in self.agent_dict.values(): - agent.update_exploration_params() + if self._explorer is None: + for agent in self.agent_dict.values(): + agent.update_exploration_params() + else: + self._explorer.update() def _assert_train_mode(self): if self._mode != AgentManagerMode.TRAIN and self._mode != AgentManagerMode.TRAIN_INFERENCE: diff --git a/maro/utils/exception/rl_toolkit_exception.py b/maro/utils/exception/rl_toolkit_exception.py index f602a1a78..6c77c19b1 100644 --- a/maro/utils/exception/rl_toolkit_exception.py +++ b/maro/utils/exception/rl_toolkit_exception.py @@ -48,6 +48,6 @@ def __init__(self, msg: str = None): class MissingExplorerError(MAROException): - """Raised when a call to an agent's ``update_exploration_params`` is made but there is no explorer present.""" + """Raised when a call to an explorer-related method is made but there is no explorer present.""" def __init__(self, msg: str = None): super().__init__(4009, msg) From ab6934b11b06a53bfdae566fd44454dac558f45d Mon Sep 17 00:00:00 2001 From: ysqyang Date: Mon, 16 Nov 2020 23:09:09 +0800 Subject: [PATCH 202/337] fixed a bug --- examples/cim/dqn/components/agent_manager.py | 1 - examples/cim/dqn/dist_actor.py | 5 +++-- examples/cim/dqn/dist_learner.py | 4 +++- examples/cim/dqn/single_process_launcher.py | 7 ++++--- maro/rl/agent/simple_agent_manager.py | 7 +++++-- 5 files changed, 15 insertions(+), 9 deletions(-) diff --git a/examples/cim/dqn/components/agent_manager.py b/examples/cim/dqn/components/agent_manager.py index 3f7ec3117..db863a20d 100644 --- a/examples/cim/dqn/components/agent_manager.py +++ b/examples/cim/dqn/components/agent_manager.py @@ -46,7 +46,6 @@ def create_dqn_agents(agent_id_list, config): agent_dict[agent_id] = CIMAgent( name=agent_id, algorithm=algorithm, - explorer=EpsilonGreedyExplorer(num_actions), experience_pool=experience_pool, **config.training_loop_parameters ) diff --git a/examples/cim/dqn/dist_actor.py b/examples/cim/dqn/dist_actor.py index 0769b7f2b..655e0f233 100644 --- a/examples/cim/dqn/dist_actor.py +++ b/examples/cim/dqn/dist_actor.py @@ -11,7 +11,7 @@ from components.experience_shaper import TruncatedExperienceShaper from components.state_shaper import CIMStateShaper -from maro.rl import ActorWorker, AgentManagerMode, KStepExperienceShaper, SimpleActor +from maro.rl import ActorWorker, AgentManagerMode, EpsilonGreedyExplorer, KStepExperienceShaper, SimpleActor from maro.simulator import Env from maro.utils import convert_dottable @@ -39,6 +39,7 @@ def launch(config, distributed_config): state_shaper=state_shaper, action_shaper=action_shaper, experience_shaper=experience_shaper, + explorer=EpsilonGreedyExplorer(config.agents.algorithm.num_actions), ) proxy_params = { "group_name": os.environ["GROUP"] if "GROUP" in os.environ else distributed_config.group, @@ -47,7 +48,7 @@ def launch(config, distributed_config): "max_retries": 15 } actor_worker = ActorWorker( - local_actor=SimpleActor(env=env, inference_agents=agent_manager), + local_actor=SimpleActor(env, agent_manager), proxy_params=proxy_params ) actor_worker.launch() diff --git a/examples/cim/dqn/dist_learner.py b/examples/cim/dqn/dist_learner.py index 499f6c72c..e80cde1d5 100644 --- a/examples/cim/dqn/dist_learner.py +++ b/examples/cim/dqn/dist_learner.py @@ -7,7 +7,8 @@ from components.config import set_input_dim from maro.rl import ( - ActorProxy, AgentManagerMode, SimpleLearner, concat_experiences_by_agent, two_phase_linear_epsilon_schedule + ActorProxy, AgentManagerMode, EpsilonGreedyExplorer, SimpleLearner, concat_experiences_by_agent, + two_phase_linear_epsilon_schedule ) from maro.simulator import Env from maro.utils import Logger, convert_dottable @@ -24,6 +25,7 @@ def launch(config, distributed_config): name="distributed_cim_learner", mode=AgentManagerMode.TRAIN, agent_dict=create_dqn_agents(agent_id_list, config.agents), + explorer=EpsilonGreedyExplorer(config.agents.algorithm.num_actions), ) proxy_params = { diff --git a/examples/cim/dqn/single_process_launcher.py b/examples/cim/dqn/single_process_launcher.py index 1b064699c..3c7d5716d 100644 --- a/examples/cim/dqn/single_process_launcher.py +++ b/examples/cim/dqn/single_process_launcher.py @@ -13,8 +13,8 @@ from components.state_shaper import CIMStateShaper from maro.rl import ( - AgentManagerMode, KStepExperienceShaper, MaxDeltaEarlyStoppingChecker, SimpleActor, SimpleEarlyStoppingChecker, - SimpleLearner, two_phase_linear_epsilon_schedule + AgentManagerMode, EpsilonGreedyExplorer, KStepExperienceShaper, MaxDeltaEarlyStoppingChecker, SimpleActor, + SimpleEarlyStoppingChecker, SimpleLearner, two_phase_linear_epsilon_schedule ) from maro.simulator import Env from maro.utils import Logger, convert_dottable @@ -48,7 +48,8 @@ def launch(config): agent_dict=create_dqn_agents(agent_id_list, config.agents), state_shaper=state_shaper, action_shaper=action_shaper, - experience_shaper=experience_shaper + experience_shaper=experience_shaper, + explorer=EpsilonGreedyExplorer(config.agents.algorithm.num_actions), ) # Step 4: Create an actor and a learner to start the training process. diff --git a/maro/rl/agent/simple_agent_manager.py b/maro/rl/agent/simple_agent_manager.py index 6e3574900..6b9c770a2 100644 --- a/maro/rl/agent/simple_agent_manager.py +++ b/maro/rl/agent/simple_agent_manager.py @@ -4,6 +4,7 @@ import os from abc import abstractmethod +from maro.rl.exploration.abs_explorer import AbsExplorer from maro.rl.shaping.action_shaper import ActionShaper from maro.rl.shaping.experience_shaper import ExperienceShaper from maro.rl.shaping.state_shaper import StateShaper @@ -21,7 +22,8 @@ def __init__( agent_dict: dict, state_shaper: StateShaper = None, action_shaper: ActionShaper = None, - experience_shaper: ExperienceShaper = None + experience_shaper: ExperienceShaper = None, + explorer: AbsExplorer = None ): if mode in {AgentManagerMode.INFERENCE, AgentManagerMode.TRAIN_INFERENCE}: if state_shaper is None: @@ -35,7 +37,8 @@ def __init__( name, mode, agent_dict, state_shaper=state_shaper, action_shaper=action_shaper, - experience_shaper=experience_shaper + experience_shaper=experience_shaper, + explorer=explorer ) # Data structures to temporarily store transitions and trajectory From 6df170f035355db841ffb5838d8fd7925307ebe0 Mon Sep 17 00:00:00 2001 From: ysqyang Date: Mon, 16 Nov 2020 23:14:39 +0800 Subject: [PATCH 203/337] fixed a bug --- examples/cim/dqn/components/agent.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/examples/cim/dqn/components/agent.py b/examples/cim/dqn/components/agent.py index 7a34d0b04..836e03541 100644 --- a/examples/cim/dqn/components/agent.py +++ b/examples/cim/dqn/components/agent.py @@ -18,13 +18,13 @@ def __init__( self, name: str, algorithm, - explorer: EpsilonGreedyExplorer, experience_pool: ColumnBasedStore, min_experiences_to_train, num_batches, - batch_size + batch_size, + explorer: EpsilonGreedyExplorer = None ): - super().__init__(name, algorithm, explorer, experience_pool) + super().__init__(name, algorithm, explorer=explorer, experience_pool=experience_pool) self._min_experiences_to_train = min_experiences_to_train self._num_batches = num_batches self._batch_size = batch_size From 09a0122861bfbb18099a0ed0dbda94d2ad6c1ea5 Mon Sep 17 00:00:00 2001 From: ysqyang Date: Mon, 16 Nov 2020 23:17:47 +0800 Subject: [PATCH 204/337] fixed a bug --- maro/rl/agent/simple_agent_manager.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/maro/rl/agent/simple_agent_manager.py b/maro/rl/agent/simple_agent_manager.py index 6b9c770a2..bd67f15bd 100644 --- a/maro/rl/agent/simple_agent_manager.py +++ b/maro/rl/agent/simple_agent_manager.py @@ -47,16 +47,18 @@ def __init__( def choose_action(self, decision_event, snapshot_list): self._assert_inference_mode() - agent_id, model_state = self._state_shaper(decision_event, snapshot_list) - model_action = self.agent_dict[agent_id].choose_action(model_state) + agent_id, state = self._state_shaper(decision_event, snapshot_list) + action = self.agent_dict[agent_id].choose_action(state) self._transition_cache = { - "state": model_state, - "action": model_action, + "state": state, + "action": action, "reward": None, "agent_id": agent_id, "event": decision_event } - return self._action_shaper(model_action, decision_event, snapshot_list) + if self._explorer: + action = self._explorer(action) + return self._action_shaper(action, decision_event, snapshot_list) def on_env_feedback(self, metrics): """This method records the environment-generated metrics as part of the latest transition in the trajectory. From 4200ef3ba43510c9d469a9741253df798c846c56 Mon Sep 17 00:00:00 2001 From: ysqyang Date: Mon, 16 Nov 2020 23:54:17 +0800 Subject: [PATCH 205/337] fixed a bug --- maro/rl/actor/abs_actor.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/maro/rl/actor/abs_actor.py b/maro/rl/actor/abs_actor.py index 83830bc9f..505d359d0 100644 --- a/maro/rl/actor/abs_actor.py +++ b/maro/rl/actor/abs_actor.py @@ -25,13 +25,14 @@ def __init__(self, env: Env, agents: Union[AbsAgentManager, dict]): @abstractmethod def roll_out( - self, model_dict: dict = None, done: bool = None, return_details: bool = True + self, model_dict: dict = None, exploration_params=None, done: bool = None, return_details: bool = True ): """This method performs a single episode of roll-out. Args: model_dict (dict): If not None, the agents will load the models from model_dict and use these models to perform roll-out. + exploration_params: Exploration parameters. done (bool): If True, the current call is the last call, i.e., no more roll-outs will be performed. This flag is used to signal remote actor workers to exit. return_details (bool): If True, return episode details (e.g., experiences) as well as performance From 7f47c038d0ad1576881040b84f821621c344e1c4 Mon Sep 17 00:00:00 2001 From: ysqyang Date: Tue, 17 Nov 2020 00:15:02 +0800 Subject: [PATCH 206/337] fixed a bug --- maro/rl/dist_topologies/common.py | 2 +- .../single_learner_multi_actor_sync_mode.py | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/maro/rl/dist_topologies/common.py b/maro/rl/dist_topologies/common.py index 6053bda2e..2df6dabfd 100644 --- a/maro/rl/dist_topologies/common.py +++ b/maro/rl/dist_topologies/common.py @@ -3,7 +3,7 @@ class PayloadKey(Enum): MODEL = "model" - EPSILON = "epsilon" + EXPLORATION_PARAMS = "exploration_params" PERFORMANCE = "performance" DETAILS = "details" SEED = "seed" diff --git a/maro/rl/dist_topologies/single_learner_multi_actor_sync_mode.py b/maro/rl/dist_topologies/single_learner_multi_actor_sync_mode.py index 048681fa2..dd6514b96 100644 --- a/maro/rl/dist_topologies/single_learner_multi_actor_sync_mode.py +++ b/maro/rl/dist_topologies/single_learner_multi_actor_sync_mode.py @@ -28,7 +28,7 @@ def __init__(self, proxy_params, experience_collecting_func: Callable): self._experience_collecting_func = experience_collecting_func def roll_out( - self, model_dict: dict = None, epsilon_dict: dict = None, done: bool = False, return_details: bool = True + self, model_dict: dict = None, exploration_params=None, done: bool = False, return_details: bool = True ): """Send roll-out requests to remote actors. @@ -40,7 +40,7 @@ def roll_out( Args: model_dict (dict): If not None, the agents will load the models from model_dict and use these models to perform roll-out. - epsilon_dict (dict): Exploration rate by agent. + exploration_params: Exploration parameters. done (bool): If True, the current call is the last call, i.e., no more roll-outs will be performed. This flag is used to signal remote actor workers to exit. return_details (bool): If True, return experiences as well as performance metrics provided by the env. @@ -57,7 +57,7 @@ def roll_out( return None, None else: payloads = [(peer, {PayloadKey.MODEL: model_dict, - PayloadKey.EPSILON: epsilon_dict, + PayloadKey.EXPLORATION_PARAMS: exploration_params, PayloadKey.RETURN_DETAILS: return_details}) for peer in self._proxy.peers["actor"]] # TODO: double check when ack enable @@ -99,7 +99,7 @@ def on_rollout_request(self, message): performance, details = self._local_actor.roll_out( model_dict=data[PayloadKey.MODEL], - epsilon_dict=data[PayloadKey.EPSILON], + exploration_params=data[PayloadKey.EXPLORATION_PARAMS], return_details=data[PayloadKey.RETURN_DETAILS] ) From 07e06e05aa8312a4a95ba2d9354686d4b9b8a025 Mon Sep 17 00:00:00 2001 From: ysqyang Date: Tue, 17 Nov 2020 13:52:54 +0800 Subject: [PATCH 207/337] separated learning with exploration schedule and without --- maro/rl/learner/simple_learner.py | 70 ++++++++++++++++++++++++------- 1 file changed, 55 insertions(+), 15 deletions(-) diff --git a/maro/rl/learner/simple_learner.py b/maro/rl/learner/simple_learner.py index 339accd0b..8f53c44d0 100644 --- a/maro/rl/learner/simple_learner.py +++ b/maro/rl/learner/simple_learner.py @@ -8,6 +8,7 @@ from maro.rl.agent.simple_agent_manager import SimpleAgentManager from maro.rl.dist_topologies.single_learner_multi_actor_sync_mode import ActorProxy from maro.utils import DummyLogger, Logger +from maro.utils.exception.rl_toolkit_exception import InfiniteTrainingLoopError, InvalidEpisodeError from .abs_learner import AbsLearner @@ -34,6 +35,50 @@ def __init__( self._performance_history = [] def learn( + self, + max_episode: int, + early_stopping_checker: Callable = None, + warmup_ep: int = None, + early_stopping_metric_func: Callable = None, + ): + """Main loop for collecting experiences from the actor and using them to update policies. + + Args: + max_episode (int): number of episodes to be run. If -1, the training loop will run forever unless + an ``early_stopping_checker`` is provided and the early stopping condition is met. + early_stopping_checker (Callable): A Callable object to determine whether the training loop should be + terminated based on the latest performances. Defaults to None. + warmup_ep (int): Episode from which early stopping check is initiated. Defaults to None. + early_stopping_metric_func (Callable): A function to extract the metric from a performance record + for early stopping checking. Defaults to None. + """ + if max_episode < -1: + raise InvalidEpisodeError("max_episode can only be a non-negative integer or -1.") + if max_episode == -1 and early_stopping_checker is None: + raise InfiniteTrainingLoopError( + "The training loop will run forever since neither maximum episode nor early stopping checker " + "is provided. " + ) + if early_stopping_checker is not None: + assert early_stopping_metric_func is not None, \ + "early_stopping_metric_func cannot be None if early_stopping_checker is provided." + + episode, metric_series = 0, [] + while max_episode == -1 or episode < max_episode: + performance, exp_by_agent = self._actor.roll_out( + model_dict=None if self._is_shared_agent_instance() else self._agent_manager.dump_models() + ) + self._logger.info(f"ep {episode} - performance: {performance}") + latest = [perf for _, perf in performance] if isinstance(performance, list) else [performance] + if early_stopping_checker is not None: + metric_series.extend(map(early_stopping_metric_func, latest)) + if warmup_ep is None or episode >= warmup_ep and early_stopping_checker(metric_series): + self._logger.info("Early stopping condition hit. Training complete.") + break + self._agent_manager.train(exp_by_agent) + episode += 1 + + def learn_with_exploration_schedule( self, exploration_schedule: Union[Iterable, dict], early_stopping_checker: Callable = None, @@ -60,7 +105,16 @@ def learn( ep, metric_series = 0, [] while True: try: - performance, exp_by_agent = self._sample() + self._agent_manager.update_exploration_params() + exploration_params = self._agent_manager.dump_exploration_params() + if self._is_shared_agent_instance(): + performance, exp_by_agent = self._actor.roll_out() + else: + performance, exp_by_agent = self._actor.roll_out( + model_dict=self._agent_manager.dump_models(), exploration_params=exploration_params + ) + self._logger.info(f"performance: {performance}, exploration_params: {exploration_params}") + # Early stopping checking latest = [perf for _, perf in performance] if isinstance(performance, list) else [performance] if early_stopping_checker is not None: metric_series.extend(map(early_stopping_metric_func, latest)) @@ -97,17 +151,3 @@ def dump_models(self, dir_path: str): def _is_shared_agent_instance(self): """If true, the set of agents performing inference in actor is the same as self._agent_manager.""" return isinstance(self._actor, SimpleActor) and id(self._actor.agents) == id(self._agent_manager) - - def _sample(self): - """Perform one episode of environment sampling through actor roll-out.""" - self._agent_manager.update_exploration_params() - exploration_params = self._agent_manager.dump_exploration_params() - if self._is_shared_agent_instance(): - performance, exp_by_agent = self._actor.roll_out() - else: - performance, exp_by_agent = self._actor.roll_out( - model_dict=self._agent_manager.dump_models(), exploration_params=exploration_params - ) - - self._logger.info(f"performance: {performance}, exploration_params: {exploration_params}") - return performance, exp_by_agent From ca15318c19f664c28697b07fea85e47d9a8e7aaf Mon Sep 17 00:00:00 2001 From: ysqyang Date: Tue, 17 Nov 2020 13:56:57 +0800 Subject: [PATCH 208/337] small fixes --- examples/cim/dqn/dist_learner.py | 2 +- examples/cim/dqn/single_process_launcher.py | 2 +- maro/rl/learner/simple_learner.py | 12 +++++++----- 3 files changed, 9 insertions(+), 7 deletions(-) diff --git a/examples/cim/dqn/dist_learner.py b/examples/cim/dqn/dist_learner.py index e80cde1d5..999198313 100644 --- a/examples/cim/dqn/dist_learner.py +++ b/examples/cim/dqn/dist_learner.py @@ -45,7 +45,7 @@ def launch(config, distributed_config): actor=ActorProxy(proxy_params=proxy_params, experience_collecting_func=concat_experiences_by_agent), logger=Logger("distributed_cim_learner", auto_timestamp=False) ) - learner.learn(exploration_schedule) + learner.learn_with_exploration_schedule(exploration_schedule) learner.test() learner.dump_models(os.path.join(os.getcwd(), "models")) learner.exit() diff --git a/examples/cim/dqn/single_process_launcher.py b/examples/cim/dqn/single_process_launcher.py index 3c7d5716d..533d85caf 100644 --- a/examples/cim/dqn/single_process_launcher.py +++ b/examples/cim/dqn/single_process_launcher.py @@ -73,7 +73,7 @@ def launch(config): actor=actor, logger=Logger("single_host_cim_learner", auto_timestamp=False) ) - learner.learn( + learner.learn_with_exploration_schedule( exploration_schedule, early_stopping_checker=combined_checker, warmup_ep=config.main_loop.early_stopping.warmup_ep, diff --git a/maro/rl/learner/simple_learner.py b/maro/rl/learner/simple_learner.py index 8f53c44d0..c6de6db4c 100644 --- a/maro/rl/learner/simple_learner.py +++ b/maro/rl/learner/simple_learner.py @@ -102,7 +102,7 @@ def learn_with_exploration_schedule( "early_stopping_metric_func cannot be None if early_stopping_checker is provided." self._agent_manager.register_exploration_schedule(exploration_schedule) - ep, metric_series = 0, [] + episode, metric_series = 0, [] while True: try: self._agent_manager.update_exploration_params() @@ -113,19 +113,21 @@ def learn_with_exploration_schedule( performance, exp_by_agent = self._actor.roll_out( model_dict=self._agent_manager.dump_models(), exploration_params=exploration_params ) - self._logger.info(f"performance: {performance}, exploration_params: {exploration_params}") + self._logger.info( + f"ep {episode} - performance: {performance}, exploration_params: {exploration_params}" + ) # Early stopping checking latest = [perf for _, perf in performance] if isinstance(performance, list) else [performance] if early_stopping_checker is not None: metric_series.extend(map(early_stopping_metric_func, latest)) - if warmup_ep is None or ep >= warmup_ep and early_stopping_checker(metric_series): + if warmup_ep is None or episode >= warmup_ep and early_stopping_checker(metric_series): self._logger.info("Early stopping condition hit. Training complete.") break except StopIteration: - self._logger.info(f"Maximum number of episodes {ep + 1} reached. Training complete.") + self._logger.info(f"Maximum number of episodes ({episode + 1}) reached. Training complete.") break - ep += 1 + episode += 1 self._agent_manager.train(exp_by_agent) def test(self): From 0b7ac924bdbc212ad161942ccd231170b5d45307 Mon Sep 17 00:00:00 2001 From: ysqyang Date: Tue, 17 Nov 2020 15:38:26 +0800 Subject: [PATCH 209/337] moved explorer logic to actor side --- examples/cim/dqn/components/agent.py | 4 +-- examples/cim/dqn/components/agent_manager.py | 6 +--- examples/cim/dqn/dist_actor.py | 3 +- examples/cim/dqn/dist_learner.py | 3 +- examples/cim/dqn/single_process_launcher.py | 6 ++-- maro/rl/agent/abs_agent.py | 13 -------- maro/rl/agent/abs_agent_manager.py | 31 ++----------------- maro/rl/agent/simple_agent_manager.py | 3 -- maro/rl/exploration/abs_explorer.py | 12 ------- .../rl/exploration/epsilon_greedy_explorer.py | 16 ---------- maro/rl/learner/simple_learner.py | 28 ++++++++++------- maro/utils/exception/error_code.py | 4 +-- maro/utils/exception/rl_toolkit_exception.py | 12 ------- 13 files changed, 27 insertions(+), 114 deletions(-) diff --git a/examples/cim/dqn/components/agent.py b/examples/cim/dqn/components/agent.py index 836e03541..790a2baa3 100644 --- a/examples/cim/dqn/components/agent.py +++ b/examples/cim/dqn/components/agent.py @@ -18,11 +18,11 @@ def __init__( self, name: str, algorithm, + explorer: EpsilonGreedyExplorer, experience_pool: ColumnBasedStore, min_experiences_to_train, num_batches, - batch_size, - explorer: EpsilonGreedyExplorer = None + batch_size ): super().__init__(name, algorithm, explorer=explorer, experience_pool=experience_pool) self._min_experiences_to_train = min_experiences_to_train diff --git a/examples/cim/dqn/components/agent_manager.py b/examples/cim/dqn/components/agent_manager.py index db863a20d..f47a38d97 100644 --- a/examples/cim/dqn/components/agent_manager.py +++ b/examples/cim/dqn/components/agent_manager.py @@ -1,8 +1,6 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT license. -import numpy as np - import torch.nn as nn from torch.optim import RMSprop @@ -44,9 +42,7 @@ def create_dqn_agents(agent_id_list, config): experience_pool = ColumnBasedStore(**config.experience_pool) agent_dict[agent_id] = CIMAgent( - name=agent_id, - algorithm=algorithm, - experience_pool=experience_pool, + agent_id, algorithm, EpsilonGreedyExplorer(num_actions), experience_pool, **config.training_loop_parameters ) diff --git a/examples/cim/dqn/dist_actor.py b/examples/cim/dqn/dist_actor.py index 655e0f233..0822b2060 100644 --- a/examples/cim/dqn/dist_actor.py +++ b/examples/cim/dqn/dist_actor.py @@ -38,8 +38,7 @@ def launch(config, distributed_config): agent_dict=create_dqn_agents(agent_id_list, config.agents), state_shaper=state_shaper, action_shaper=action_shaper, - experience_shaper=experience_shaper, - explorer=EpsilonGreedyExplorer(config.agents.algorithm.num_actions), + experience_shaper=experience_shaper ) proxy_params = { "group_name": os.environ["GROUP"] if "GROUP" in os.environ else distributed_config.group, diff --git a/examples/cim/dqn/dist_learner.py b/examples/cim/dqn/dist_learner.py index 999198313..283068e8d 100644 --- a/examples/cim/dqn/dist_learner.py +++ b/examples/cim/dqn/dist_learner.py @@ -24,8 +24,7 @@ def launch(config, distributed_config): agent_manager = DQNAgentManager( name="distributed_cim_learner", mode=AgentManagerMode.TRAIN, - agent_dict=create_dqn_agents(agent_id_list, config.agents), - explorer=EpsilonGreedyExplorer(config.agents.algorithm.num_actions), + agent_dict=create_dqn_agents(agent_id_list, config.agents) ) proxy_params = { diff --git a/examples/cim/dqn/single_process_launcher.py b/examples/cim/dqn/single_process_launcher.py index 533d85caf..6d3c678f9 100644 --- a/examples/cim/dqn/single_process_launcher.py +++ b/examples/cim/dqn/single_process_launcher.py @@ -48,8 +48,7 @@ def launch(config): agent_dict=create_dqn_agents(agent_id_list, config.agents), state_shaper=state_shaper, action_shaper=action_shaper, - experience_shaper=experience_shaper, - explorer=EpsilonGreedyExplorer(config.agents.algorithm.num_actions), + experience_shaper=experience_shaper ) # Step 4: Create an actor and a learner to start the training process. @@ -66,7 +65,6 @@ def launch(config): combined_checker = perf_checker & perf_stability_checker - exploration_schedule = two_phase_linear_epsilon_schedule(**config.main_loop.exploration) actor = SimpleActor(env, agent_manager) learner = SimpleLearner( agent_manager=agent_manager, @@ -74,7 +72,7 @@ def launch(config): logger=Logger("single_host_cim_learner", auto_timestamp=False) ) learner.learn_with_exploration_schedule( - exploration_schedule, + two_phase_linear_epsilon_schedule(**config.main_loop.exploration), early_stopping_checker=combined_checker, warmup_ep=config.main_loop.early_stopping.warmup_ep, early_stopping_metric_func=lambda x: 1 - x["container_shortage"] / x["order_requirements"], diff --git a/maro/rl/agent/abs_agent.py b/maro/rl/agent/abs_agent.py index 88960193b..a9cfca7dd 100644 --- a/maro/rl/agent/abs_agent.py +++ b/maro/rl/agent/abs_agent.py @@ -8,7 +8,6 @@ from maro.rl.algorithms.abs_algorithm import AbsAlgorithm from maro.rl.exploration.abs_explorer import AbsExplorer from maro.rl.storage.abs_store import AbsStore -from maro.utils.exception.rl_toolkit_exception import MissingExplorerError class AbsAgent(ABC): @@ -67,22 +66,10 @@ def choose_action(self, model_state): action_from_algorithm = self._algorithm.choose_action(model_state) return action_from_algorithm if self._explorer is None else self._explorer(action_from_algorithm) - def register_exploration_schedule(self, exploration_schedule): - if self._explorer: - self._explorer.register_schedule(exploration_schedule) - def load_exploration_params(self, exploration_params): if self._explorer: self._explorer.load_exploration_params(exploration_params) - def dump_exploration_params(self): - if self._explorer: - return self._explorer.dump_exploration_params() - - def update_exploration_params(self): - if self._explorer: - self._explorer.update() - @abstractmethod def train(self, *args, **kwargs): """Training logic to be implemented by the user. diff --git a/maro/rl/agent/abs_agent_manager.py b/maro/rl/agent/abs_agent_manager.py index 137feece5..c97bd8deb 100644 --- a/maro/rl/agent/abs_agent_manager.py +++ b/maro/rl/agent/abs_agent_manager.py @@ -3,13 +3,11 @@ from abc import ABC, abstractmethod from enum import Enum -from typing import Iterable, Union -from maro.rl.exploration.abs_explorer import AbsExplorer from maro.rl.shaping.action_shaper import ActionShaper from maro.rl.shaping.experience_shaper import ExperienceShaper from maro.rl.shaping.state_shaper import StateShaper -from maro.utils.exception.rl_toolkit_exception import MissingExplorerError, WrongAgentManagerModeError +from maro.utils.exception.rl_toolkit_exception import WrongAgentManagerModeError class AgentManagerMode(Enum): @@ -36,8 +34,6 @@ class AbsAgentManager(ABC): executable action. Cannot be None under Inference and TrainInference modes. experience_shaper (ExperienceShaper, optional): It is responsible for processing data in the replay buffer at the end of an episode. - explorer (AbsExplorer): Shared explorer for all agents. If this is not None, the underlying agents' explorers - will not be used. Defaults to None. """ def __init__( self, @@ -47,7 +43,6 @@ def __init__( state_shaper: StateShaper = None, action_shaper: ActionShaper = None, experience_shaper: ExperienceShaper = None, - explorer: AbsExplorer = None ): self._name = name self._mode = mode @@ -55,7 +50,6 @@ def __init__( self._state_shaper = state_shaper self._action_shaper = action_shaper self._experience_shaper = experience_shaper - self._explorer = explorer def __getitem__(self, agent_id): return self.agent_dict[agent_id] @@ -93,32 +87,13 @@ def train(self, *args, **kwargs): """Train the agents.""" return NotImplemented - def register_exploration_schedule(self, exploration_schedule: Union[Iterable, dict]): - if isinstance(exploration_schedule, dict): - for agent_id, agent in self.agent_dict.items(): - agent.register_exploration_schedule(exploration_schedule[agent_id]) - else: - self._explorer.register_schedule(exploration_schedule) - def load_exploration_params(self, exploration_params): - if self._explorer is None: + if set(exploration_params.keys()).issubset(set(self.agent_dict.keys())): for agent_id, params in exploration_params.items(): self.agent_dict[agent_id].load_exploration_params(params) else: - self._explorer.load_exploration_params(exploration_params) - - def dump_exploration_params(self): - if self._explorer is None: - return {agent_id: agent.dump_exploration_params() for agent_id, agent in self.agent_dict.items()} - else: - return self._explorer.dump_exploration_params() - - def update_exploration_params(self): - if self._explorer is None: for agent in self.agent_dict.values(): - agent.update_exploration_params() - else: - self._explorer.update() + agent.load_exploration_params(exploration_params) def _assert_train_mode(self): if self._mode != AgentManagerMode.TRAIN and self._mode != AgentManagerMode.TRAIN_INFERENCE: diff --git a/maro/rl/agent/simple_agent_manager.py b/maro/rl/agent/simple_agent_manager.py index bd67f15bd..021d0cd40 100644 --- a/maro/rl/agent/simple_agent_manager.py +++ b/maro/rl/agent/simple_agent_manager.py @@ -38,7 +38,6 @@ def __init__( state_shaper=state_shaper, action_shaper=action_shaper, experience_shaper=experience_shaper, - explorer=explorer ) # Data structures to temporarily store transitions and trajectory @@ -56,8 +55,6 @@ def choose_action(self, decision_event, snapshot_list): "agent_id": agent_id, "event": decision_event } - if self._explorer: - action = self._explorer(action) return self._action_shaper(action, decision_event, snapshot_list) def on_env_feedback(self, metrics): diff --git a/maro/rl/exploration/abs_explorer.py b/maro/rl/exploration/abs_explorer.py index 25dbe8ada..43859d1f8 100644 --- a/maro/rl/exploration/abs_explorer.py +++ b/maro/rl/exploration/abs_explorer.py @@ -11,22 +11,10 @@ class AbsExplorer(ABC): def __init__(self): pass - @abstractmethod - def register_schedule(self, exploration_param_iter): - return NotImplementedError - @abstractmethod def load_exploration_params(self, exploration_params): return NotImplementedError - @abstractmethod - def dump_exploration_params(self): - return NotImplementedError - - @abstractmethod - def update(self): - return NotImplementedError - @abstractmethod def __call__(self, action): return NotImplementedError diff --git a/maro/rl/exploration/epsilon_greedy_explorer.py b/maro/rl/exploration/epsilon_greedy_explorer.py index 2606e1d4f..9080144ab 100644 --- a/maro/rl/exploration/epsilon_greedy_explorer.py +++ b/maro/rl/exploration/epsilon_greedy_explorer.py @@ -2,9 +2,6 @@ # Licensed under the MIT license. import random -from typing import Generator - -from maro.utils.exception.rl_toolkit_exception import MissingExplorationScheduleError from .abs_explorer import AbsExplorer @@ -28,18 +25,5 @@ def __call__(self, action): else: return random.randrange(self._num_actions) - def register_schedule(self, epsilon_schedule: Generator): - self._epsilon_schedule = epsilon_schedule - def load_exploration_params(self, epsilon: float): self._epsilon = epsilon - - def dump_exploration_params(self): - return self._epsilon - - def update(self): - if self._epsilon_schedule is None: - raise MissingExplorationScheduleError( - "An iterable epsilon schedule must be registered first by calling register_schedule()." - ) - self._epsilon = next(self._epsilon_schedule) diff --git a/maro/rl/learner/simple_learner.py b/maro/rl/learner/simple_learner.py index c6de6db4c..48f1b9176 100644 --- a/maro/rl/learner/simple_learner.py +++ b/maro/rl/learner/simple_learner.py @@ -2,7 +2,7 @@ # Licensed under the MIT license. import sys -from typing import Callable, Iterable, Union +from typing import Callable, Iterator, Union from maro.rl.actor.simple_actor import SimpleActor from maro.rl.agent.simple_agent_manager import SimpleAgentManager @@ -80,7 +80,7 @@ def learn( def learn_with_exploration_schedule( self, - exploration_schedule: Union[Iterable, dict], + exploration_schedule: Union[Iterator, dict], early_stopping_checker: Callable = None, warmup_ep: int = None, early_stopping_metric_func: Callable = None, @@ -88,9 +88,9 @@ def learn_with_exploration_schedule( """Main loop for collecting experiences from the actor and using them to update policies. Args: - exploration_schedule (Union[Iterable, dict]): Explorations schedules for the underlying agents. If it is + exploration_schedule (Union[Iterator, dict]): Explorations schedules for the underlying agents. If it is a dictionary, the exploration schedule will be registered on a per-agent basis based on agent ID's. - If it is a single iterable object, the exploration schedule will be registered for all agents. + If it is a single iterator object, the exploration schedule will be registered for all agents. early_stopping_checker (Callable): A Callable object to determine whether the training loop should be terminated based on the latest performances. Defaults to None. warmup_ep (int): Episode from which early stopping check is initiated. Defaults to None. @@ -101,21 +101,25 @@ def learn_with_exploration_schedule( assert early_stopping_metric_func is not None, \ "early_stopping_metric_func cannot be None if early_stopping_checker is provided." - self._agent_manager.register_exploration_schedule(exploration_schedule) episode, metric_series = 0, [] while True: try: - self._agent_manager.update_exploration_params() - exploration_params = self._agent_manager.dump_exploration_params() - if self._is_shared_agent_instance(): - performance, exp_by_agent = self._actor.roll_out() + if isinstance(exploration_schedule, dict): + exploration_params = { + agent_id: next(schedule) for agent_id, schedule in exploration_schedule.items() + } else: - performance, exp_by_agent = self._actor.roll_out( - model_dict=self._agent_manager.dump_models(), exploration_params=exploration_params - ) + exploration_params = next(exploration_schedule) + + performance, exp_by_agent = self._actor.roll_out( + model_dict=None if self._is_shared_agent_instance else self._agent_manager.dump_models(), + exploration_params=exploration_params + ) + self._logger.info( f"ep {episode} - performance: {performance}, exploration_params: {exploration_params}" ) + # Early stopping checking latest = [perf for _, perf in performance] if isinstance(performance, list) else [performance] if early_stopping_checker is not None: diff --git a/maro/utils/exception/error_code.py b/maro/utils/exception/error_code.py index df4b37c28..f09dcd12f 100644 --- a/maro/utils/exception/error_code.py +++ b/maro/utils/exception/error_code.py @@ -40,7 +40,5 @@ 4003: "Wrong Agent Manager Mode", 4004: "Store Misalignment Error", 4005: "Invalid Episode", - 4006: "Infinite Training Loop", - 4009: "Missing Exploration Schedule Error", - 4010: "Missing Explorer Error" + 4006: "Infinite Training Loop" } diff --git a/maro/utils/exception/rl_toolkit_exception.py b/maro/utils/exception/rl_toolkit_exception.py index 6c77c19b1..c35e0b897 100644 --- a/maro/utils/exception/rl_toolkit_exception.py +++ b/maro/utils/exception/rl_toolkit_exception.py @@ -39,15 +39,3 @@ class InfiniteTrainingLoopError(MAROException): """Raised when the ``SimpleLearner``'s training loop becomes infinite.""" def __init__(self, msg: str = None): super().__init__(4006, msg) - - -class MissingExplorationScheduleError(MAROException): - """Raised when calling an explorer's ``update`` method with no exploration schedule registered.""" - def __init__(self, msg: str = None): - super().__init__(4009, msg) - - -class MissingExplorerError(MAROException): - """Raised when a call to an explorer-related method is made but there is no explorer present.""" - def __init__(self, msg: str = None): - super().__init__(4009, msg) From d331b48db79a103464d179b36cf8363d27724985 Mon Sep 17 00:00:00 2001 From: ysqyang Date: Tue, 17 Nov 2020 15:43:56 +0800 Subject: [PATCH 210/337] fixed a bug --- maro/rl/agent/abs_agent_manager.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/maro/rl/agent/abs_agent_manager.py b/maro/rl/agent/abs_agent_manager.py index c97bd8deb..968411b0d 100644 --- a/maro/rl/agent/abs_agent_manager.py +++ b/maro/rl/agent/abs_agent_manager.py @@ -88,7 +88,7 @@ def train(self, *args, **kwargs): return NotImplemented def load_exploration_params(self, exploration_params): - if set(exploration_params.keys()).issubset(set(self.agent_dict.keys())): + if isinstance(exploration_params, dict) and exploration_params.keys() <= self.agent_dict.keys(): for agent_id, params in exploration_params.items(): self.agent_dict[agent_id].load_exploration_params(params) else: From 3e811b3ab82b1e272ca3cf0a40bbc37a8f6ad434 Mon Sep 17 00:00:00 2001 From: ysqyang Date: Tue, 17 Nov 2020 15:47:53 +0800 Subject: [PATCH 211/337] fixed a bug --- maro/rl/agent/abs_agent.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/maro/rl/agent/abs_agent.py b/maro/rl/agent/abs_agent.py index a9cfca7dd..42cbf69f8 100644 --- a/maro/rl/agent/abs_agent.py +++ b/maro/rl/agent/abs_agent.py @@ -61,10 +61,11 @@ def choose_action(self, model_state): Args: model_state: State vector as accepted by the underlying algorithm. Returns: - Action given by the underlying policy model. + If the agent's explorer is None, the action given by the underlying model is returned. Otherwise, + an exploratory action is returned. """ - action_from_algorithm = self._algorithm.choose_action(model_state) - return action_from_algorithm if self._explorer is None else self._explorer(action_from_algorithm) + action = self._algorithm.choose_action(model_state) + return action if self._explorer is None else self._explorer(action) def load_exploration_params(self, exploration_params): if self._explorer: From bdd3a7d28376ef314ffec932dbc64201f32f8f98 Mon Sep 17 00:00:00 2001 From: ysqyang Date: Tue, 17 Nov 2020 15:52:52 +0800 Subject: [PATCH 212/337] fixed a bug --- maro/rl/exploration/epsilon_greedy_explorer.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/maro/rl/exploration/epsilon_greedy_explorer.py b/maro/rl/exploration/epsilon_greedy_explorer.py index 9080144ab..01283798b 100644 --- a/maro/rl/exploration/epsilon_greedy_explorer.py +++ b/maro/rl/exploration/epsilon_greedy_explorer.py @@ -14,13 +14,12 @@ class EpsilonGreedyExplorer(AbsExplorer): """ def __init__(self, num_actions: int): super().__init__() - self._epsilon_schedule = None self._epsilon = None self._num_actions = num_actions def __call__(self, action): assert action < self._num_actions, f"Invalid action: {action}" - if self._epsilon_schedule is None or random.random() > self._epsilon: + if random.random() > self._epsilon: return action else: return random.randrange(self._num_actions) From 6d3e7fd99e8749ac4b70289a9780774e68177f33 Mon Sep 17 00:00:00 2001 From: ysqyang Date: Tue, 17 Nov 2020 22:30:06 +0800 Subject: [PATCH 213/337] fixed a bug --- maro/rl/learner/simple_learner.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/maro/rl/learner/simple_learner.py b/maro/rl/learner/simple_learner.py index 48f1b9176..f35c0b987 100644 --- a/maro/rl/learner/simple_learner.py +++ b/maro/rl/learner/simple_learner.py @@ -112,7 +112,7 @@ def learn_with_exploration_schedule( exploration_params = next(exploration_schedule) performance, exp_by_agent = self._actor.roll_out( - model_dict=None if self._is_shared_agent_instance else self._agent_manager.dump_models(), + model_dict=None if self._is_shared_agent_instance() else self._agent_manager.dump_models(), exploration_params=exploration_params ) @@ -128,7 +128,7 @@ def learn_with_exploration_schedule( self._logger.info("Early stopping condition hit. Training complete.") break except StopIteration: - self._logger.info(f"Maximum number of episodes ({episode + 1}) reached. Training complete.") + self._logger.info(f"Maximum number of episodes ({episode}) reached. Training complete.") break episode += 1 From a304f4cd217f092912faa00c52bc8f293331b3c7 Mon Sep 17 00:00:00 2001 From: ysqyang Date: Wed, 18 Nov 2020 17:37:48 +0800 Subject: [PATCH 214/337] removed unwanted param from simple agent manager --- maro/rl/agent/simple_agent_manager.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/maro/rl/agent/simple_agent_manager.py b/maro/rl/agent/simple_agent_manager.py index 021d0cd40..c29134362 100644 --- a/maro/rl/agent/simple_agent_manager.py +++ b/maro/rl/agent/simple_agent_manager.py @@ -22,8 +22,7 @@ def __init__( agent_dict: dict, state_shaper: StateShaper = None, action_shaper: ActionShaper = None, - experience_shaper: ExperienceShaper = None, - explorer: AbsExplorer = None + experience_shaper: ExperienceShaper = None ): if mode in {AgentManagerMode.INFERENCE, AgentManagerMode.TRAIN_INFERENCE}: if state_shaper is None: @@ -37,7 +36,7 @@ def __init__( name, mode, agent_dict, state_shaper=state_shaper, action_shaper=action_shaper, - experience_shaper=experience_shaper, + experience_shaper=experience_shaper ) # Data structures to temporarily store transitions and trajectory From 065edb9bd9bc69c0ff5efea98a84139ee81a21a8 Mon Sep 17 00:00:00 2001 From: ysqyang Date: Wed, 18 Nov 2020 17:45:24 +0800 Subject: [PATCH 215/337] small fixes --- examples/cim/dqn/components/agent_manager.py | 2 +- maro/rl/__init__.py | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/examples/cim/dqn/components/agent_manager.py b/examples/cim/dqn/components/agent_manager.py index 44479bada..7c90b1f9a 100644 --- a/examples/cim/dqn/components/agent_manager.py +++ b/examples/cim/dqn/components/agent_manager.py @@ -6,7 +6,7 @@ from maro.rl import ( ColumnBasedStore, DQN, DQNConfig, EpsilonGreedyExplorer, FullyConnectedBlock, LearningModel, LearningModule, - SimpleAgentManager, OptimizerOptions + OptimizerOptions, SimpleAgentManager ) from maro.utils import set_seeds diff --git a/maro/rl/__init__.py b/maro/rl/__init__.py index 7fade7779..065b633cc 100644 --- a/maro/rl/__init__.py +++ b/maro/rl/__init__.py @@ -57,7 +57,6 @@ 'KStepExperienceShaper', 'LearningModel', 'LearningModule', - 'LinearExplorer', 'MaxDeltaEarlyStoppingChecker', 'OptimizerOptions', 'OverwriteType', From dbdb25c72fd6554b3a0e1b6792382649efe8b200 Mon Sep 17 00:00:00 2001 From: ysqyang Date: Wed, 18 Nov 2020 18:24:53 +0800 Subject: [PATCH 216/337] revised code based on revised abstractions --- docs/source/examples/cim.rst | 6 +-- docs/source/examples/multi_agent_dqn_cim.rst | 16 +++---- examples/cim/ac/components/agent_manager.py | 47 +++++++++++--------- examples/cim/ac/config.yml | 8 ++-- examples/cim/ac/dist_actor.py | 2 +- examples/cim/ac/dist_learner.py | 17 ++----- examples/cim/ac/single_process_launcher.py | 30 +++++++++++-- examples/cim/dqn/components/agent_manager.py | 3 +- examples/cim/dqn/config.yml | 2 +- examples/cim/pg/config.yml | 2 +- examples/cim/pg/dist_learner.py | 4 +- examples/cim/pg/single_process_launcher.py | 30 +++++++++++-- maro/rl/algorithms/ac.py | 9 ++-- maro/rl/algorithms/dqn.py | 4 +- maro/rl/algorithms/pg.py | 10 ++--- maro/rl/algorithms/ppo.py | 13 ++---- 16 files changed, 114 insertions(+), 89 deletions(-) diff --git a/docs/source/examples/cim.rst b/docs/source/examples/cim.rst index c7ce500b9..5d612ef1f 100644 --- a/docs/source/examples/cim.rst +++ b/docs/source/examples/cim.rst @@ -182,7 +182,7 @@ policies. "split_point_dict": {"_all_": config.exploration.split_point}, "with_cache": config.exploration.with_cache } - explorer = TwoPhaseLinearExplorer(agent_id_list, config.general.total_training_episodes, **exploration_config) + explorer = TwoPhaseLinearExplorer(agent_id_list, config.main_loop.total_training_episodes, **exploration_config) agent_manager = DQNAgentManager(name="cim_learner", mode=AgentMode.TRAIN_INFERENCE, @@ -196,7 +196,7 @@ policies. learner = SimpleLearner(trainable_agents=agent_manager, actor=actor, logger=Logger("single_host_cim_learner", auto_timestamp=False)) - learner.learn(total_episodes=config.general.total_training_episodes) + learner.learn(total_episodes=config.main_loop.total_training_episodes) Main Loop with Actor and Learner (Distributed / Multi-process) @@ -251,5 +251,5 @@ inside. learner = SimpleLearner(trainable_agents=agent_manager, actor=ActorProxy(proxy_params=proxy_params), logger=Logger("distributed_cim_learner", auto_timestamp=False)) - learner.learn(total_episodes=config.general.total_training_episodes) + learner.learn(total_episodes=config.main_loop.total_training_episodes) diff --git a/docs/source/examples/multi_agent_dqn_cim.rst b/docs/source/examples/multi_agent_dqn_cim.rst index 2b917646c..3314c179b 100644 --- a/docs/source/examples/multi_agent_dqn_cim.rst +++ b/docs/source/examples/multi_agent_dqn_cim.rst @@ -198,8 +198,8 @@ policies. ) early_stopping_checker = MaxDeltaEarlyStoppingChecker( - last_k=config.general.early_stopping.last_k, - threshold=config.general.early_stopping.threshold + last_k=config.main_loop.early_stopping.last_k, + threshold=config.main_loop.early_stopping.threshold ) actor = SimpleActor(env=env, inference_agents=agent_manager) learner = SimpleLearner( @@ -209,9 +209,9 @@ policies. logger=Logger("single_host_cim_learner", auto_timestamp=False) ) learner.learn( - max_episode=config.general.max_episode, + max_episode=config.main_loop.max_episode, early_stopping_checker=early_stopping_checker, - warmup_ep=config.general.early_stopping.warmup_ep, + warmup_ep=config.main_loop.early_stopping.warmup_ep, early_stopping_metric_func=lambda x: 1 - x["container_shortage"] / x["order_requirements"], ) @@ -273,8 +273,8 @@ inside. } early_stopping_checker = MaxDeltaEarlyStoppingChecker( - last_k=config.general.early_stopping.last_k, - threshold=config.general.early_stopping.threshold + last_k=config.main_loop.early_stopping.last_k, + threshold=config.main_loop.early_stopping.threshold ) learner = SimpleLearner( @@ -284,9 +284,9 @@ inside. logger=Logger("distributed_cim_learner", auto_timestamp=False) ) learner.learn( - max_episode=config.general.max_episode, + max_episode=config.main_loop.max_episode, early_stopping_checker=early_stopping_checker, - warmup_ep=config.general.early_stopping.warmup_ep, + warmup_ep=config.main_loop.early_stopping.warmup_ep, early_stopping_metric_func=lambda x: 1 - x["container_shortage"] / x["order_requirements"], ) diff --git a/examples/cim/ac/components/agent_manager.py b/examples/cim/ac/components/agent_manager.py index 177b3a2b9..aa9af4ba8 100644 --- a/examples/cim/ac/components/agent_manager.py +++ b/examples/cim/ac/components/agent_manager.py @@ -5,7 +5,10 @@ from torch.optim import Adam, RMSprop from .agent import CIMAgent -from maro.rl import SimpleAgentManager, LearningModel, FullyConnectedNet, ActorCritic, ActorCriticConfig +from maro.rl import ( + ActorCritic, ActorCriticConfig, FullyConnectedBlock, LearningModel, LearningModule, + OptimizerOptions, SimpleAgentManager +) from maro.utils import set_seeds @@ -15,30 +18,34 @@ def create_ac_agents(agent_id_list, config): agent_dict = {} for agent_id in agent_id_list: - policy_model = LearningModel( - decision_layers=FullyConnectedNet( - name=f'{agent_id}.policy', input_dim=config.algorithm.input_dim, output_dim=num_actions, - activation=nn.Tanh, **config.algorithm.policy_model - ) + actor_module = LearningModule( + "actor", + [FullyConnectedBlock( + input_dim=config.algorithm.input_dim, + output_dim=num_actions, + activation=nn.Tanh, + is_head=True, + **config.algorithm.actor_model + )], + optimizer_options=OptimizerOptions(cls=Adam, params=config.algorithm.optimizer) ) - value_model = LearningModel( - decision_layers=FullyConnectedNet( - name=f'{agent_id}.value', input_dim=config.algorithm.input_dim, output_dim=1, - activation=nn.LeakyReLU, **config.algorithm.value_model - ) + critic_module = LearningModule( + "critic", + [FullyConnectedBlock( + input_dim=config.algorithm.input_dim, + output_dim=1, + activation=nn.LeakyReLU, + is_head=True, + **config.algorithm.critic_model + )], + optimizer_options=OptimizerOptions(cls=Adam, params=config.algorithm.optimizer) ) algorithm = ActorCritic( - policy_model=policy_model, - value_model=value_model, - value_loss_func=nn.functional.smooth_l1_loss, - policy_optimizer_cls=Adam, - policy_optimizer_params=config.algorithm.policy_optimizer, - value_optimizer_cls=RMSprop, - value_optimizer_params=config.algorithm.value_optimizer, - hyper_params=ActorCriticConfig( - num_actions=num_actions, + LearningModel(actor_module, critic_module), + config=ActorCriticConfig( + critic_loss_func=nn.functional.smooth_l1_loss, **config.algorithm.hyper_parameters, ) ) diff --git a/examples/cim/ac/config.yml b/examples/cim/ac/config.yml index e61809d93..a579d1773 100644 --- a/examples/cim/ac/config.yml +++ b/examples/cim/ac/config.yml @@ -2,7 +2,7 @@ env: scenario: "cim" topology: "toy.4p_ssdd_l0.0" durations: 1120 -general: +main_loop: max_episode: 500 early_stopping: warmup_ep: 50 @@ -31,14 +31,14 @@ experience_shaping: agents: algorithm: num_actions: 21 - policy_model: + actor_model: hidden_dims: - 256 - 128 - 64 softmax_enabled: true batch_norm_enabled: false - value_model: + critic_model: hidden_dims: - 256 - 128 @@ -49,7 +49,7 @@ agents: lr: 0.001 value_optimizer: lr: 0.001 - hyper_parameters: + config: reward_decay: .0 policy_train_iters: 1 value_train_iters: 10 diff --git a/examples/cim/ac/dist_actor.py b/examples/cim/ac/dist_actor.py index 6f43bd8fd..d5af8a5c8 100644 --- a/examples/cim/ac/dist_actor.py +++ b/examples/cim/ac/dist_actor.py @@ -39,7 +39,7 @@ def launch(config): "redis_address": ("localhost", 6379) } actor_worker = ActorWorker( - local_actor=SimpleActor(env=env, inference_agents=agent_manager), + local_actor=SimpleActor(env=env, agent_manager=agent_manager), proxy_params=proxy_params ) actor_worker.launch() diff --git a/examples/cim/ac/dist_learner.py b/examples/cim/ac/dist_learner.py index 76ad87e07..adaa0dc97 100644 --- a/examples/cim/ac/dist_learner.py +++ b/examples/cim/ac/dist_learner.py @@ -3,8 +3,7 @@ import os -from maro.rl import ActorProxy, AgentManagerMode, merge_experiences_with_trajectory_boundaries, \ - MaxDeltaEarlyStoppingChecker, SimpleLearner +from maro.rl import ActorProxy, AgentManagerMode, SimpleLearner, merge_experiences_with_trajectory_boundaries from maro.simulator import Env from maro.utils import Logger, convert_dottable @@ -29,25 +28,15 @@ def launch(config): "redis_address": ("localhost", 6379) } - early_stopping_checker = MaxDeltaEarlyStoppingChecker( - last_k=config.general.early_stopping.last_k, - threshold=config.general.early_stopping.threshold - ) - learner = SimpleLearner( - trainable_agents=agent_manager, + agent_manager=agent_manager, actor=ActorProxy( proxy_params=proxy_params, experience_collecting_func=merge_experiences_with_trajectory_boundaries ), logger=Logger("distributed_cim_learner", auto_timestamp=False) ) - learner.train( - max_episode=config.general.max_episode, - early_stopping_checker=early_stopping_checker, - warmup_ep=config.general.early_stopping.warmup_ep, - early_stopping_metric_func=lambda x: 1 - x["container_shortage"] / x["order_requirements"], - ) + learner.learn(max_episode=config.main_loop.max_episode) learner.test() learner.dump_models(os.path.join(os.getcwd(), "models")) learner.exit() diff --git a/examples/cim/ac/single_process_launcher.py b/examples/cim/ac/single_process_launcher.py index 9e2abc62d..73f1d8299 100644 --- a/examples/cim/ac/single_process_launcher.py +++ b/examples/cim/ac/single_process_launcher.py @@ -2,11 +2,14 @@ # Licensed under the MIT license. import os +from statistics import mean import numpy as np from maro.simulator import Env -from maro.rl import SimpleLearner, SimpleActor, AgentManagerMode +from maro.rl import ( + AgentManagerMode, SimpleActor, SimpleEarlyStoppingChecker, SimpleLearner, MaxDeltaEarlyStoppingChecker +) from maro.utils import Logger, convert_dottable from components.action_shaper import CIMActionShaper @@ -42,12 +45,31 @@ def launch(config): ) # Step 4: Create an actor and a learner to start the training process. - actor = SimpleActor(env=env, inference_agents=agent_manager) + perf_checker = SimpleEarlyStoppingChecker( + last_k=config.main_loop.early_stopping.last_k, + threshold=config.main_loop.early_stopping.perf_threshold, + measure_func=lambda vals: mean(vals) + ) + + perf_stability_checker = MaxDeltaEarlyStoppingChecker( + last_k=config.main_loop.early_stopping.last_k, + threshold=config.main_loop.early_stopping.perf_stability_threshold + ) + + combined_checker = perf_checker & perf_stability_checker + + actor = SimpleActor(env=env, agent_manager=agent_manager) learner = SimpleLearner( - trainable_agents=agent_manager, actor=actor, + agent_manager=agent_manager, + actor=actor, logger=Logger("single_host_cim_learner", auto_timestamp=False) ) - learner.train(max_episode=config.general.max_episode) + learner.learn( + max_episode=config.main_loop.max_episode, + early_stopping_checker=combined_checker, + warmup_ep=config.main_loop.early_stopping.warmup_ep, + early_stopping_metric_func=lambda x: 1 - x["container_shortage"] / x["order_requirements"] + ) learner.test() learner.dump_models(os.path.join(os.getcwd(), "models")) diff --git a/examples/cim/dqn/components/agent_manager.py b/examples/cim/dqn/components/agent_manager.py index 7c90b1f9a..bd9c9bf94 100644 --- a/examples/cim/dqn/components/agent_manager.py +++ b/examples/cim/dqn/components/agent_manager.py @@ -34,8 +34,7 @@ def create_dqn_agents(agent_id_list, config): model=LearningModel(q_module), config=DQNConfig( **config.algorithm.config, - loss_cls=nn.SmoothL1Loss, - num_actions=num_actions + loss_cls=nn.SmoothL1Loss ) ) diff --git a/examples/cim/dqn/config.yml b/examples/cim/dqn/config.yml index 19b7e7c54..53ff0b43c 100644 --- a/examples/cim/dqn/config.yml +++ b/examples/cim/dqn/config.yml @@ -14,7 +14,7 @@ main_loop: last_k: 10 perf_threshold: 0.9 # minimum performance (fulfillment ratio) required to trigger early stopping perf_stability_threshold: 0.1 # stability is measured by the maximum of abs(perf_(i+1) - perf_i) / perf_i - # over the last k eisodes (where perf is short for performance). This value must + # over the last k episodes (where perf is short for performance). This value must # be below this threshold to trigger early stopping state_shaping: look_back: 7 diff --git a/examples/cim/pg/config.yml b/examples/cim/pg/config.yml index 62baf506a..b7274deb6 100644 --- a/examples/cim/pg/config.yml +++ b/examples/cim/pg/config.yml @@ -2,7 +2,7 @@ env: scenario: "cim" topology: "toy.4p_ssdd_l0.0" durations: 1120 -general: +main_loop: max_episode: 500 early_stopping: warmup_ep: 50 diff --git a/examples/cim/pg/dist_learner.py b/examples/cim/pg/dist_learner.py index ed6b6149d..a721dff3f 100644 --- a/examples/cim/pg/dist_learner.py +++ b/examples/cim/pg/dist_learner.py @@ -27,13 +27,13 @@ } learner = SimpleLearner( - trainable_agents=agent_manager, + agent_manager=agent_manager, actor=ActorProxy( proxy_params=proxy_params, experience_collecting_func=merge_experiences_with_trajectory_boundaries ), logger=Logger("distributed_cim_learner", auto_timestamp=False) ) - learner.train(max_episode=config.general.max_episode) + learner.learn(max_episode=config.main_loop.max_episode) learner.test() learner.dump_models(os.path.join(os.getcwd(), "models")) diff --git a/examples/cim/pg/single_process_launcher.py b/examples/cim/pg/single_process_launcher.py index 2daa0cfe4..b6c95579b 100644 --- a/examples/cim/pg/single_process_launcher.py +++ b/examples/cim/pg/single_process_launcher.py @@ -2,11 +2,14 @@ # Licensed under the MIT license. import os +from statistics import mean import numpy as np from maro.simulator import Env -from maro.rl import SimpleLearner, SimpleActor, AgentManagerMode +from maro.rl import ( + AgentManagerMode, SimpleActor, SimpleEarlyStoppingChecker, SimpleLearner, MaxDeltaEarlyStoppingChecker +) from maro.utils import Logger, convert_dottable from components.action_shaper import CIMActionShaper @@ -42,12 +45,31 @@ def launch(config): ) # Step 4: Create an actor and a learner to start the training process. - actor = SimpleActor(env=env, inference_agents=agent_manager) + perf_checker = SimpleEarlyStoppingChecker( + last_k=config.main_loop.early_stopping.last_k, + threshold=config.main_loop.early_stopping.perf_threshold, + measure_func=lambda vals: mean(vals) + ) + + perf_stability_checker = MaxDeltaEarlyStoppingChecker( + last_k=config.main_loop.early_stopping.last_k, + threshold=config.main_loop.early_stopping.perf_stability_threshold + ) + + combined_checker = perf_checker & perf_stability_checker + + actor = SimpleActor(env=env, agent_manager=agent_manager) learner = SimpleLearner( - trainable_agents=agent_manager, actor=actor, + agent_manager=agent_manager, + actor=actor, logger=Logger("single_host_cim_learner", auto_timestamp=False) ) - learner.train(max_episode=config.general.max_episode) + learner.learn( + max_episode=config.main_loop.max_episode, + early_stopping_checker=combined_checker, + warmup_ep=config.main_loop.early_stopping.warmup_ep, + early_stopping_metric_func=lambda x: 1 - x["container_shortage"] / x["order_requirements"] + ) learner.test() learner.dump_models(os.path.join(os.getcwd(), "models")) diff --git a/maro/rl/algorithms/ac.py b/maro/rl/algorithms/ac.py index 22d5203da..9c2e37d02 100644 --- a/maro/rl/algorithms/ac.py +++ b/maro/rl/algorithms/ac.py @@ -23,7 +23,6 @@ class ActorCriticConfig: """Configuration for the Actor-Critic algorithm. Args: - num_actions (int): Number of possible actions reward_decay (float): Reward decay as defined in standard RL terminology. critic_loss_func (Callable): Loss function for the critic model. actor_train_iters (int): Number of gradient descent steps for the policy model per call to ``train``. @@ -34,12 +33,11 @@ class ActorCriticConfig: k-step return is computed. """ __slots__ = [ - "num_actions", "reward_decay", "critic_loss_func", "actor_train_iters", "critic_train_iters", "k", "lam" + "reward_decay", "critic_loss_func", "actor_train_iters", "critic_train_iters", "k", "lam" ] def __init__( self, - num_actions: int, reward_decay: float, critic_loss_func: Callable, actor_train_iters: int, @@ -47,7 +45,6 @@ def __init__( k: int = -1, lam: float = 1.0 ): - self.num_actions = num_actions self.reward_decay = reward_decay self.critic_loss_func = critic_loss_func self.actor_train_iters = actor_train_iters @@ -73,8 +70,8 @@ def __init__(self, model: LearningModel, config: ActorCriticConfig): @expand_dim def choose_action(self, state: np.ndarray): - action_dist = self._model(state, task_name="actor", is_training=False).squeeze().numpy() # (num_actions,) - return np.random.choice(self._config.num_actions, p=action_dist) + action_distribution = self._model(state, task_name="actor", is_training=False).squeeze().numpy() + return np.random.choice(len(action_distribution), p=action_distribution) def _get_values_and_bootstrapped_returns(self, state_sequence, reward_sequence): state_values = self._model(state_sequence, task_name="critic").detach().squeeze() diff --git a/maro/rl/algorithms/dqn.py b/maro/rl/algorithms/dqn.py index 60f6ce19e..9fb25f721 100644 --- a/maro/rl/algorithms/dqn.py +++ b/maro/rl/algorithms/dqn.py @@ -20,7 +20,6 @@ class DQNConfig: """Configuration for the DQN algorithm. Args: - num_actions (int): Number of possible actions. reward_decay (float): Reward decay as defined in standard RL terminology. loss_cls: Loss function class for evaluating TD errors. target_update_frequency (int): Number of training rounds between target model updates. @@ -34,7 +33,7 @@ class DQNConfig: method. Defaults to False. """ __slots__ = [ - "num_actions", "reward_decay", "loss_func", "target_update_frequency", "tau", "is_double", + "reward_decay", "loss_func", "target_update_frequency", "tau", "is_double", "advantage_mode", "per_sample_td_error_enabled" ] @@ -49,7 +48,6 @@ def __init__( advantage_mode: str = None, per_sample_td_error_enabled: bool = False ): - self.num_actions = num_actions self.reward_decay = reward_decay self.target_update_frequency = target_update_frequency self.tau = tau diff --git a/maro/rl/algorithms/pg.py b/maro/rl/algorithms/pg.py index 0ba1ac0cb..3e874aba7 100644 --- a/maro/rl/algorithms/pg.py +++ b/maro/rl/algorithms/pg.py @@ -14,13 +14,11 @@ class PolicyGradientConfig: """Configuration for the Policy Gradient (PG) algorithm. Args: - num_actions (int): Number of possible actions. reward_decay (float): Reward decay as defined in standard RL terminology. """ - __slots__ = ["num_actions", "reward_decay"] + __slots__ = ["reward_decay"] - def __init__(self, num_actions: int, reward_decay: float): - self.num_actions = num_actions + def __init__(self, reward_decay: float): self.reward_decay = reward_decay @@ -41,8 +39,8 @@ def __init__(self, model: LearningModel, config: PolicyGradientConfig): @expand_dim def choose_action(self, state: np.ndarray): - action_distribution = self._model(state, is_training=False) # (num_actions,) - return np.random.choice(self._config.num_actions, p=action_distribution) + action_distribution = self._model(state, is_training=False).squeeze().numpy() # (num_actions,) + return np.random.choice(len(action_distribution), p=action_distribution) @preprocess def train(self, states: np.ndarray, actions: np.ndarray, returns: np.ndarray): diff --git a/maro/rl/algorithms/ppo.py b/maro/rl/algorithms/ppo.py index 8b30f4bc3..4a424fcc3 100644 --- a/maro/rl/algorithms/ppo.py +++ b/maro/rl/algorithms/ppo.py @@ -23,7 +23,6 @@ class PPOConfig: """Configuration for the Proximal Policy Optimization (PPO) algorithm. Args: - num_actions (int): Number of possible actions. reward_decay (float): Reward decay as defined in standard RL terminology. critic_loss_func (Callable): Critic loss function. clip_ratio (float): Clip ratio as defined in PPO's objective function. @@ -35,13 +34,11 @@ class PPOConfig: k-step return is computed. """ __slots__ = [ - "num_actions", "reward_decay", "critic_loss_func", "clip_ratio", "policy_train_iters", "value_train_iters", - "k", "lam" + "reward_decay", "critic_loss_func", "clip_ratio", "policy_train_iters", "value_train_iters", "k", "lam" ] def __init__( self, - num_actions: int, reward_decay: float, critic_loss_func: Callable, clip_ratio: float, @@ -50,7 +47,6 @@ def __init__( k: int = -1, lam: float = 1.0 ): - self.num_actions = num_actions self.reward_decay = reward_decay self.critic_loss_func = critic_loss_func self.clip_ratio = clip_ratio @@ -79,11 +75,8 @@ def __init__(self, model: LearningModel, config: PPOConfig): @expand_dim def choose_action(self, state: np.ndarray): - state = torch.from_numpy(state).unsqueeze(0).to(self._device) # (1, state_dim) - self._model.eval() - with torch.no_grad(): - action_dist = self._model(state, task_name="actor").squeeze().numpy() # (num_actions,) - return np.random.choice(self._config.num_actions, p=action_dist) + action_distribution = self._model(state, task_name="actor", is_training=False).squeeze().numpy() + return np.random.choice(len(action_distribution), p=action_distribution) def _get_values_and_bootstrapped_returns(self, states: torch.tensor, rewards: np.ndarray): state_values = self._model(states, task_name="critic").detach().squeeze() From d33d81e40e5668ad33a954aeadfa884262868d87 Mon Sep 17 00:00:00 2001 From: ysqyang Date: Wed, 18 Nov 2020 18:31:41 +0800 Subject: [PATCH 217/337] fixed some bugs --- examples/cim/ac/components/agent_manager.py | 6 ++--- examples/cim/pg/components/agent_manager.py | 29 +++++++++++---------- 2 files changed, 18 insertions(+), 17 deletions(-) diff --git a/examples/cim/ac/components/agent_manager.py b/examples/cim/ac/components/agent_manager.py index aa9af4ba8..6764bf287 100644 --- a/examples/cim/ac/components/agent_manager.py +++ b/examples/cim/ac/components/agent_manager.py @@ -6,8 +6,8 @@ from .agent import CIMAgent from maro.rl import ( - ActorCritic, ActorCriticConfig, FullyConnectedBlock, LearningModel, LearningModule, - OptimizerOptions, SimpleAgentManager + ActorCritic, ActorCriticConfig, FullyConnectedBlock, LearningModel, LearningModule, OptimizerOptions, + SimpleAgentManager ) from maro.utils import set_seeds @@ -39,7 +39,7 @@ def create_ac_agents(agent_id_list, config): is_head=True, **config.algorithm.critic_model )], - optimizer_options=OptimizerOptions(cls=Adam, params=config.algorithm.optimizer) + optimizer_options=OptimizerOptions(cls=RMSprop, params=config.algorithm.optimizer) ) algorithm = ActorCritic( diff --git a/examples/cim/pg/components/agent_manager.py b/examples/cim/pg/components/agent_manager.py index 1a33b8296..ef7ba1edd 100644 --- a/examples/cim/pg/components/agent_manager.py +++ b/examples/cim/pg/components/agent_manager.py @@ -5,8 +5,10 @@ from torch.optim import Adam from .agent import CIMAgent -from maro.rl import SimpleAgentManager, LearningModel, FullyConnectedNet, PolicyGradient, \ - PolicyGradientConfig +from maro.rl import ( + FullyConnectedBlock, LearningModel, LearningModule, OptimizerOptions, PolicyGradient, PolicyGradientConfig, + SimpleAgentManager +) from maro.utils import set_seeds @@ -15,21 +17,20 @@ def create_pg_agents(agent_id_list, config): set_seeds(config.seed) agent_dict = {} for agent_id in agent_id_list: - policy_model = LearningModel( - decision_layers=FullyConnectedNet( - name=f'{agent_id}.policy', input_dim=config.algorithm.input_dim, output_dim=num_actions, - activation=nn.Tanh, **config.algorithm.policy_model - ) + policy_module = LearningModule( + "policy", + [FullyConnectedBlock( + input_dim=config.algorithm.input_dim, + output_dim=num_actions, + activation=nn.Tanh, + **config.algorithm.policy_model + )], + optimizer_options=OptimizerOptions(cls=Adam, params=config.algorithm.optimizer) ) algorithm = PolicyGradient( - policy_model=policy_model, - optimizer_cls=Adam, - optimizer_params=config.algorithm.optimizer, - hyper_params=PolicyGradientConfig( - num_actions=num_actions, - **config.algorithm.hyper_parameters, - ) + model=LearningModel(policy_module), + hyper_params=PolicyGradientConfig(**config.algorithm.hyper_parameters) ) agent_dict[agent_id] = CIMAgent(name=agent_id, algorithm=algorithm) From 368fa367b1bae2d26df6877cbf5a1f5f611e77bd Mon Sep 17 00:00:00 2001 From: ysqyang Date: Wed, 18 Nov 2020 18:33:48 +0800 Subject: [PATCH 218/337] fixed a bug --- examples/cim/pg/components/agent_manager.py | 2 +- examples/cim/pg/config.yml | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/examples/cim/pg/components/agent_manager.py b/examples/cim/pg/components/agent_manager.py index ef7ba1edd..28314f323 100644 --- a/examples/cim/pg/components/agent_manager.py +++ b/examples/cim/pg/components/agent_manager.py @@ -30,7 +30,7 @@ def create_pg_agents(agent_id_list, config): algorithm = PolicyGradient( model=LearningModel(policy_module), - hyper_params=PolicyGradientConfig(**config.algorithm.hyper_parameters) + config=PolicyGradientConfig(reward_decay=config.algorithm.reward_decay) ) agent_dict[agent_id] = CIMAgent(name=agent_id, algorithm=algorithm) diff --git a/examples/cim/pg/config.yml b/examples/cim/pg/config.yml index b7274deb6..a7182e7b0 100644 --- a/examples/cim/pg/config.yml +++ b/examples/cim/pg/config.yml @@ -30,6 +30,7 @@ experience_shaping: time_decay_factor: 0.97 agents: algorithm: + reward_decay: .0 num_actions: 21 policy_model: hidden_dims: @@ -40,8 +41,6 @@ agents: batch_norm_enabled: false optimizer: lr: 0.001 - hyper_parameters: - reward_decay: .0 seed: 1024 # for reproducibility distributed: group_name: "pg_distributed_test" From d0e3fa373cfe737e33261c8abe58019eea825460 Mon Sep 17 00:00:00 2001 From: ysqyang Date: Wed, 18 Nov 2020 18:35:17 +0800 Subject: [PATCH 219/337] fixed a bug --- examples/cim/ac/config.yml | 5 ++++- examples/cim/pg/config.yml | 5 ++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/examples/cim/ac/config.yml b/examples/cim/ac/config.yml index a579d1773..6b9b08101 100644 --- a/examples/cim/ac/config.yml +++ b/examples/cim/ac/config.yml @@ -7,7 +7,10 @@ main_loop: early_stopping: warmup_ep: 50 last_k: 10 - threshold: 0.05 + perf_threshold: 0.9 # minimum performance (fulfillment ratio) required to trigger early stopping + perf_stability_threshold: 0.1 # stability is measured by the maximum of abs(perf_(i+1) - perf_i) / perf_i + # over the last k episodes (where perf is short for performance). This value must + # be below this threshold to trigger early stopping state_shaping: look_back: 7 max_ports_downstream: 2 diff --git a/examples/cim/pg/config.yml b/examples/cim/pg/config.yml index a7182e7b0..965461bb9 100644 --- a/examples/cim/pg/config.yml +++ b/examples/cim/pg/config.yml @@ -7,7 +7,10 @@ main_loop: early_stopping: warmup_ep: 50 last_k: 10 - threshold: 0.05 + perf_threshold: 0.9 # minimum performance (fulfillment ratio) required to trigger early stopping + perf_stability_threshold: 0.1 # stability is measured by the maximum of abs(perf_(i+1) - perf_i) / perf_i + # over the last k episodes (where perf is short for performance). This value must + # be below this threshold to trigger early stopping state_shaping: look_back: 7 max_ports_downstream: 2 From adcfd472ea64008d8a4c0ea235c348e018d6b49f Mon Sep 17 00:00:00 2001 From: ysqyang Date: Wed, 18 Nov 2020 18:37:29 +0800 Subject: [PATCH 220/337] fixed a bug --- maro/rl/algorithms/ac.py | 4 ++-- maro/rl/algorithms/pg.py | 2 +- maro/rl/algorithms/ppo.py | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/maro/rl/algorithms/ac.py b/maro/rl/algorithms/ac.py index 9c2e37d02..7b7d0f23e 100644 --- a/maro/rl/algorithms/ac.py +++ b/maro/rl/algorithms/ac.py @@ -94,11 +94,11 @@ def train(self, states: np.ndarray, actions: np.ndarray, rewards: np.ndarray): for _ in range(self._config.actor_train_iters): action_prob = self._model(states, task_name="actor").gather(1, actions.unsqueeze(1)).squeeze() # (N,) actor_loss = -(torch.log(action_prob) * advantages).mean() - self._model.step(actor_loss) + self._model.learn(actor_loss) # value model training for _ in range(self._config.critic_train_iters): critic_loss = self._config.critic_loss_func( self._model(states, task_name="critic").squeeze(), return_est ) - self._model.step(critic_loss) + self._model.learn(critic_loss) diff --git a/maro/rl/algorithms/pg.py b/maro/rl/algorithms/pg.py index 3e874aba7..04c35d501 100644 --- a/maro/rl/algorithms/pg.py +++ b/maro/rl/algorithms/pg.py @@ -47,4 +47,4 @@ def train(self, states: np.ndarray, actions: np.ndarray, returns: np.ndarray): action_distributions = self._model(states) action_prob = action_distributions.gather(1, actions.unsqueeze(1)).squeeze() # (N, 1) loss = -(torch.log(action_prob) * returns).mean() - self._model.step(loss) + self._model.learn(loss) diff --git a/maro/rl/algorithms/ppo.py b/maro/rl/algorithms/ppo.py index 4a424fcc3..718e988fe 100644 --- a/maro/rl/algorithms/ppo.py +++ b/maro/rl/algorithms/ppo.py @@ -104,9 +104,9 @@ def train( ratio = torch.exp(torch.log(action_prob) - log_action_prob_old) clipped_ratio = torch.clamp(ratio, 1 - self._config.clip_ratio, 1 + self._config.clip_ratio) loss = -(torch.min(ratio * advantages, clipped_ratio * advantages)).mean() - self._model.step(loss) + self._model.learn(loss) # value model training for _ in range(self._config.value_train_iters): loss = self._config.lovalue_loss_func(self._model(states, task_name="critic"), return_est) - self._model.step(loss) + self._model.learn(loss) From 275a27866e31c37d621b66e4d45e95b23070450b Mon Sep 17 00:00:00 2001 From: ysqyang Date: Wed, 18 Nov 2020 18:44:49 +0800 Subject: [PATCH 221/337] fixed a bug --- examples/cim/ac/components/agent_manager.py | 4 ++-- examples/cim/ac/config.yml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/examples/cim/ac/components/agent_manager.py b/examples/cim/ac/components/agent_manager.py index 6764bf287..266a9464a 100644 --- a/examples/cim/ac/components/agent_manager.py +++ b/examples/cim/ac/components/agent_manager.py @@ -27,7 +27,7 @@ def create_ac_agents(agent_id_list, config): is_head=True, **config.algorithm.actor_model )], - optimizer_options=OptimizerOptions(cls=Adam, params=config.algorithm.optimizer) + optimizer_options=OptimizerOptions(cls=Adam, params=config.algorithm.actor_optimizer) ) critic_module = LearningModule( @@ -39,7 +39,7 @@ def create_ac_agents(agent_id_list, config): is_head=True, **config.algorithm.critic_model )], - optimizer_options=OptimizerOptions(cls=RMSprop, params=config.algorithm.optimizer) + optimizer_options=OptimizerOptions(cls=RMSprop, params=config.algorithm.critic_optimizer) ) algorithm = ActorCritic( diff --git a/examples/cim/ac/config.yml b/examples/cim/ac/config.yml index 6b9b08101..5d6ee518a 100644 --- a/examples/cim/ac/config.yml +++ b/examples/cim/ac/config.yml @@ -48,9 +48,9 @@ agents: - 64 softmax_enabled: false batch_norm_enabled: true - policy_optimizer: + actor_optimizer: lr: 0.001 - value_optimizer: + critic_optimizer: lr: 0.001 config: reward_decay: .0 From e9a6eca39700af77e2fdebb2d68f9f85121d97f5 Mon Sep 17 00:00:00 2001 From: ysqyang Date: Wed, 18 Nov 2020 18:45:41 +0800 Subject: [PATCH 222/337] fixed a bug --- examples/cim/ac/components/agent_manager.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/cim/ac/components/agent_manager.py b/examples/cim/ac/components/agent_manager.py index 266a9464a..ab44f20da 100644 --- a/examples/cim/ac/components/agent_manager.py +++ b/examples/cim/ac/components/agent_manager.py @@ -46,7 +46,7 @@ def create_ac_agents(agent_id_list, config): LearningModel(actor_module, critic_module), config=ActorCriticConfig( critic_loss_func=nn.functional.smooth_l1_loss, - **config.algorithm.hyper_parameters, + **config.algorithm.config, ) ) From 4ae2f87a0e7533ea7810b19737ee58356ddf085b Mon Sep 17 00:00:00 2001 From: ysqyang Date: Wed, 18 Nov 2020 19:18:07 +0800 Subject: [PATCH 223/337] fixed a bug --- examples/cim/ac/config.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/cim/ac/config.yml b/examples/cim/ac/config.yml index 5d6ee518a..e881cce46 100644 --- a/examples/cim/ac/config.yml +++ b/examples/cim/ac/config.yml @@ -54,8 +54,8 @@ agents: lr: 0.001 config: reward_decay: .0 - policy_train_iters: 1 - value_train_iters: 10 + actor_train_iters: 1 + critic_train_iters: 10 k: 1 lam: 0.0 seed: 1024 # for reproducibility From c6c1221d818983cd613d18471c14b009b0dca0f3 Mon Sep 17 00:00:00 2001 From: ysqyang Date: Wed, 18 Nov 2020 19:27:11 +0800 Subject: [PATCH 224/337] added shared_module property to LearningModel --- maro/rl/models/learning_model.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/maro/rl/models/learning_model.py b/maro/rl/models/learning_model.py index f766e120f..9b3427c01 100644 --- a/maro/rl/models/learning_model.py +++ b/maro/rl/models/learning_model.py @@ -114,9 +114,16 @@ def __getitem__(self, task): def task_names(self) -> [str]: return self._task_names + @property + def shared(self): + return self._shared_module + @property def is_trainable(self) -> bool: - return any(task_module.is_trainable for task_module in self._task_modules) or self._shared_module.is_trainable + return ( + any(task_module.is_trainable for task_module in self._task_modules) or + (self._shared_module is not None and self._shared_module.is_trainable) + ) def _forward(self, inputs, task_name: str = None): if len(self._task_modules) == 1: From 4daa253cc358a8c40792613746af373a23ad3430 Mon Sep 17 00:00:00 2001 From: ysqyang Date: Wed, 18 Nov 2020 19:27:39 +0800 Subject: [PATCH 225/337] added shared_module property to LearningModel --- maro/rl/models/learning_model.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/maro/rl/models/learning_model.py b/maro/rl/models/learning_model.py index 9b3427c01..3ca760366 100644 --- a/maro/rl/models/learning_model.py +++ b/maro/rl/models/learning_model.py @@ -115,7 +115,7 @@ def task_names(self) -> [str]: return self._task_names @property - def shared(self): + def shared_module(self): return self._shared_module @property From 6f4ebbeed4a240a7d7ef6bac82a5c2150e66540a Mon Sep 17 00:00:00 2001 From: ysqyang Date: Wed, 18 Nov 2020 19:30:09 +0800 Subject: [PATCH 226/337] fixed a bug with k-step return in AC --- maro/rl/algorithms/ac.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/maro/rl/algorithms/ac.py b/maro/rl/algorithms/ac.py index 7b7d0f23e..303d08ac8 100644 --- a/maro/rl/algorithms/ac.py +++ b/maro/rl/algorithms/ac.py @@ -75,9 +75,8 @@ def choose_action(self, state: np.ndarray): def _get_values_and_bootstrapped_returns(self, state_sequence, reward_sequence): state_values = self._model(state_sequence, task_name="critic").detach().squeeze() - state_values_numpy = state_values.numpy() return_est = get_lambda_returns( - reward_sequence, state_values_numpy, self._config.reward_decay, self._config.lam, + reward_sequence, state_values, self._config.reward_decay, self._config.lam, k=self._config.k ) return_est = torch.from_numpy(return_est) @@ -87,7 +86,7 @@ def _get_values_and_bootstrapped_returns(self, state_sequence, reward_sequence): def train(self, states: np.ndarray, actions: np.ndarray, rewards: np.ndarray): state_values, return_est = self._get_values_and_bootstrapped_returns(states, rewards) advantages = return_est - state_values - if self._model.has_trainable_shared_layers: + if self._model.shared_module is not None and self._model.shared_module.is_trainable: pass else: # policy model training From fe45444a8b7234cf0770f052648385e767f85faa Mon Sep 17 00:00:00 2001 From: ysqyang Date: Wed, 18 Nov 2020 19:31:15 +0800 Subject: [PATCH 227/337] fixed a bug --- maro/rl/algorithms/ac.py | 1 - 1 file changed, 1 deletion(-) diff --git a/maro/rl/algorithms/ac.py b/maro/rl/algorithms/ac.py index 303d08ac8..35d450843 100644 --- a/maro/rl/algorithms/ac.py +++ b/maro/rl/algorithms/ac.py @@ -79,7 +79,6 @@ def _get_values_and_bootstrapped_returns(self, state_sequence, reward_sequence): reward_sequence, state_values, self._config.reward_decay, self._config.lam, k=self._config.k ) - return_est = torch.from_numpy(return_est) return state_values, return_est @preprocess From 926f469eb8c1842e7314d2d546bc14390aefdf34 Mon Sep 17 00:00:00 2001 From: ysqyang Date: Wed, 18 Nov 2020 19:33:13 +0800 Subject: [PATCH 228/337] fixed a bug --- maro/rl/algorithms/ac.py | 3 +-- maro/rl/algorithms/ppo.py | 5 +---- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/maro/rl/algorithms/ac.py b/maro/rl/algorithms/ac.py index 35d450843..182330ed9 100644 --- a/maro/rl/algorithms/ac.py +++ b/maro/rl/algorithms/ac.py @@ -76,8 +76,7 @@ def choose_action(self, state: np.ndarray): def _get_values_and_bootstrapped_returns(self, state_sequence, reward_sequence): state_values = self._model(state_sequence, task_name="critic").detach().squeeze() return_est = get_lambda_returns( - reward_sequence, state_values, self._config.reward_decay, self._config.lam, - k=self._config.k + reward_sequence, state_values, self._config.reward_decay, self._config.lam, k=self._config.k ) return state_values, return_est diff --git a/maro/rl/algorithms/ppo.py b/maro/rl/algorithms/ppo.py index 718e988fe..1f3a2fd89 100644 --- a/maro/rl/algorithms/ppo.py +++ b/maro/rl/algorithms/ppo.py @@ -80,12 +80,9 @@ def choose_action(self, state: np.ndarray): def _get_values_and_bootstrapped_returns(self, states: torch.tensor, rewards: np.ndarray): state_values = self._model(states, task_name="critic").detach().squeeze() - state_values_numpy = state_values.numpy() return_est = get_lambda_returns( - rewards, state_values_numpy, self._config.reward_decay, self._config.lam, - k=self._config.k + rewards, state_values, self._config.reward_decay, self._config.lam, k=self._config.k ) - return_est = torch.from_numpy(return_est) return state_values, return_est @preprocess From 3eb024e83580ce1e2508ffc5ba4c43dca071e616 Mon Sep 17 00:00:00 2001 From: ysqyang Date: Wed, 18 Nov 2020 23:04:29 +0800 Subject: [PATCH 229/337] merged pg, ac and ppo examples --- examples/cim/ac/components/agent.py | 11 --- examples/cim/ac/components/agent_manager.py | 65 --------------- examples/cim/ac/components/config.py | 28 ------- examples/cim/ac/single_process_launcher.py | 79 ------------------- examples/cim/pg/README.md | 22 ------ examples/cim/pg/components/__init__.py | 2 - examples/cim/pg/components/action_shaper.py | 33 -------- examples/cim/pg/components/agent.py | 11 --- examples/cim/pg/components/agent_manager.py | 49 ------------ .../cim/pg/components/experience_shaper.py | 48 ----------- examples/cim/pg/components/state_shaper.py | 29 ------- examples/cim/pg/config.yml | 56 ------------- examples/cim/pg/dist_actor.py | 41 ---------- examples/cim/pg/dist_learner.py | 39 --------- examples/cim/pg/multi_process_launcher.py | 26 ------ .../cim/{ac => policy_optimization}/README.md | 0 .../components/__init__.py | 0 .../components/action_shaper.py | 0 .../components/agent_manager.py | 76 ++++++++++++++++++ .../components/config.py | 2 +- .../components/experience_shaper.py | 0 .../components/state_shaper.py | 0 .../{ac => policy_optimization}/config.yml | 61 ++++++++------ .../{ac => policy_optimization}/dist_actor.py | 6 +- .../dist_learner.py | 6 +- .../multi_process_launcher.py | 0 .../single_process_launcher.py | 6 +- maro/rl/agent/simple_agent_manager.py | 1 - 28 files changed, 121 insertions(+), 576 deletions(-) delete mode 100644 examples/cim/ac/components/agent.py delete mode 100644 examples/cim/ac/components/agent_manager.py delete mode 100644 examples/cim/ac/components/config.py delete mode 100644 examples/cim/ac/single_process_launcher.py delete mode 100644 examples/cim/pg/README.md delete mode 100644 examples/cim/pg/components/__init__.py delete mode 100644 examples/cim/pg/components/action_shaper.py delete mode 100644 examples/cim/pg/components/agent.py delete mode 100644 examples/cim/pg/components/agent_manager.py delete mode 100644 examples/cim/pg/components/experience_shaper.py delete mode 100644 examples/cim/pg/components/state_shaper.py delete mode 100644 examples/cim/pg/config.yml delete mode 100644 examples/cim/pg/dist_actor.py delete mode 100644 examples/cim/pg/dist_learner.py delete mode 100644 examples/cim/pg/multi_process_launcher.py rename examples/cim/{ac => policy_optimization}/README.md (100%) rename examples/cim/{ac => policy_optimization}/components/__init__.py (100%) rename examples/cim/{ac => policy_optimization}/components/action_shaper.py (100%) create mode 100644 examples/cim/policy_optimization/components/agent_manager.py rename examples/cim/{pg => policy_optimization}/components/config.py (93%) rename examples/cim/{ac => policy_optimization}/components/experience_shaper.py (100%) rename examples/cim/{ac => policy_optimization}/components/state_shaper.py (100%) rename examples/cim/{ac => policy_optimization}/config.yml (62%) rename examples/cim/{ac => policy_optimization}/dist_actor.py (90%) rename examples/cim/{ac => policy_optimization}/dist_learner.py (89%) rename examples/cim/{ac => policy_optimization}/multi_process_launcher.py (100%) rename examples/cim/{pg => policy_optimization}/single_process_launcher.py (94%) diff --git a/examples/cim/ac/components/agent.py b/examples/cim/ac/components/agent.py deleted file mode 100644 index 39d0e994d..000000000 --- a/examples/cim/ac/components/agent.py +++ /dev/null @@ -1,11 +0,0 @@ -# Copyright (c) Microsoft Corporation. -# Licensed under the MIT license. - -import numpy as np - -from maro.rl import AbsAgent - - -class CIMAgent(AbsAgent): - def train(self, states: np.ndarray, actions: np.ndarray, rewards: np.ndarray): - self._algorithm.train(states, actions, rewards) diff --git a/examples/cim/ac/components/agent_manager.py b/examples/cim/ac/components/agent_manager.py deleted file mode 100644 index ab44f20da..000000000 --- a/examples/cim/ac/components/agent_manager.py +++ /dev/null @@ -1,65 +0,0 @@ -# Copyright (c) Microsoft Corporation. -# Licensed under the MIT license. - -import torch.nn as nn -from torch.optim import Adam, RMSprop - -from .agent import CIMAgent -from maro.rl import ( - ActorCritic, ActorCriticConfig, FullyConnectedBlock, LearningModel, LearningModule, OptimizerOptions, - SimpleAgentManager -) -from maro.utils import set_seeds - - -def create_ac_agents(agent_id_list, config): - num_actions = config.algorithm.num_actions - set_seeds(config.seed) - agent_dict = {} - - for agent_id in agent_id_list: - actor_module = LearningModule( - "actor", - [FullyConnectedBlock( - input_dim=config.algorithm.input_dim, - output_dim=num_actions, - activation=nn.Tanh, - is_head=True, - **config.algorithm.actor_model - )], - optimizer_options=OptimizerOptions(cls=Adam, params=config.algorithm.actor_optimizer) - ) - - critic_module = LearningModule( - "critic", - [FullyConnectedBlock( - input_dim=config.algorithm.input_dim, - output_dim=1, - activation=nn.LeakyReLU, - is_head=True, - **config.algorithm.critic_model - )], - optimizer_options=OptimizerOptions(cls=RMSprop, params=config.algorithm.critic_optimizer) - ) - - algorithm = ActorCritic( - LearningModel(actor_module, critic_module), - config=ActorCriticConfig( - critic_loss_func=nn.functional.smooth_l1_loss, - **config.algorithm.config, - ) - ) - - agent_dict[agent_id] = CIMAgent(name=agent_id, algorithm=algorithm) - - return agent_dict - - -class ACAgentManager(SimpleAgentManager): - def train(self, experiences_by_agent: dict): - for agent_id, exp in experiences_by_agent.items(): - if isinstance(exp, list): - for trajectory in exp: - self.agent_dict[agent_id].train(trajectory["states"], trajectory["actions"], trajectory["rewards"]) - else: - self.agent_dict[agent_id].train(exp["states"], exp["actions"], exp["rewards"]) diff --git a/examples/cim/ac/components/config.py b/examples/cim/ac/components/config.py deleted file mode 100644 index 974fcd591..000000000 --- a/examples/cim/ac/components/config.py +++ /dev/null @@ -1,28 +0,0 @@ -# Copyright (c) Microsoft Corporation. -# Licensed under the MIT license. - -""" -This file is used to load config and convert it into a dotted dictionary. -""" - -import io -import os -import yaml - - -def set_input_dim(config): - # obtain model input dimension from state shaping configurations - look_back = config["state_shaping"]["look_back"] - max_ports_downstream = config["state_shaping"]["max_ports_downstream"] - num_port_attributes = len(config["state_shaping"]["port_attributes"]) - num_vessel_attributes = len(config["state_shaping"]["vessel_attributes"]) - - input_dim = (look_back + 1) * (max_ports_downstream + 1) * num_port_attributes + num_vessel_attributes - config["agents"]["algorithm"]["input_dim"] = input_dim - - return config - - -CONFIG_PATH = os.path.join(os.path.split(os.path.realpath(__file__))[0], "../config.yml") -with io.open(CONFIG_PATH, "r") as in_file: - config = yaml.safe_load(in_file) diff --git a/examples/cim/ac/single_process_launcher.py b/examples/cim/ac/single_process_launcher.py deleted file mode 100644 index 73f1d8299..000000000 --- a/examples/cim/ac/single_process_launcher.py +++ /dev/null @@ -1,79 +0,0 @@ -# Copyright (c) Microsoft Corporation. -# Licensed under the MIT license. - -import os -from statistics import mean - -import numpy as np - -from maro.simulator import Env -from maro.rl import ( - AgentManagerMode, SimpleActor, SimpleEarlyStoppingChecker, SimpleLearner, MaxDeltaEarlyStoppingChecker -) -from maro.utils import Logger, convert_dottable - -from components.action_shaper import CIMActionShaper -from components.agent_manager import create_ac_agents, ACAgentManager -from components.config import set_input_dim -from components.experience_shaper import TruncatedExperienceShaper -from components.state_shaper import CIMStateShaper - - -def launch(config): - # First determine the input dimension and add it to the config. - set_input_dim(config) - config = convert_dottable(config) - - # Step 1: initialize a CIM environment for using a toy dataset. - env = Env(config.env.scenario, config.env.topology, durations=config.env.durations) - agent_id_list = [str(agent_id) for agent_id in env.agent_idx_list] - - # Step 2: create state, action and experience shapers. We also need to create an explorer here due to the - # greedy nature of the DQN algorithm. - state_shaper = CIMStateShaper(**config.state_shaping) - action_shaper = CIMActionShaper(action_space=list(np.linspace(-1.0, 1.0, config.agents.algorithm.num_actions))) - experience_shaper = TruncatedExperienceShaper(**config.experience_shaping) - - # Step 3: create an agent manager. - agent_manager = ACAgentManager( - name="cim_learner", - mode=AgentManagerMode.TRAIN_INFERENCE, - agent_dict=create_ac_agents(agent_id_list, config.agents), - state_shaper=state_shaper, - action_shaper=action_shaper, - experience_shaper=experience_shaper, - ) - - # Step 4: Create an actor and a learner to start the training process. - perf_checker = SimpleEarlyStoppingChecker( - last_k=config.main_loop.early_stopping.last_k, - threshold=config.main_loop.early_stopping.perf_threshold, - measure_func=lambda vals: mean(vals) - ) - - perf_stability_checker = MaxDeltaEarlyStoppingChecker( - last_k=config.main_loop.early_stopping.last_k, - threshold=config.main_loop.early_stopping.perf_stability_threshold - ) - - combined_checker = perf_checker & perf_stability_checker - - actor = SimpleActor(env=env, agent_manager=agent_manager) - learner = SimpleLearner( - agent_manager=agent_manager, - actor=actor, - logger=Logger("single_host_cim_learner", auto_timestamp=False) - ) - learner.learn( - max_episode=config.main_loop.max_episode, - early_stopping_checker=combined_checker, - warmup_ep=config.main_loop.early_stopping.warmup_ep, - early_stopping_metric_func=lambda x: 1 - x["container_shortage"] / x["order_requirements"] - ) - learner.test() - learner.dump_models(os.path.join(os.getcwd(), "models")) - - -if __name__ == "__main__": - from components.config import config - launch(config) diff --git a/examples/cim/pg/README.md b/examples/cim/pg/README.md deleted file mode 100644 index 816ed5052..000000000 --- a/examples/cim/pg/README.md +++ /dev/null @@ -1,22 +0,0 @@ -# Overview - -The CIM problem is one of the quintessential use cases of MARO. The example can -be run with a set of scenario configurations that can be found under -maro/simulator/scenarios/cim. General experimental parameters (e.g., type of -topology, type of algorithm to use, number of training episodes) can be configured -through config.yml. Each RL formulation has a dedicated folder, e.g., dqn, and -all algorithm-specific parameters can be configured through -the config.py file in that folder. - -## Single-host Single-process Mode - -To run the CIM example using the DQN algorithm under single-host mode, go to -examples/cim/dqn and run single_process_launcher.py. You may play around with -the configuration if you want to try out different settings. - -## Distributed Mode - -The examples/cim/dqn/components folder contains dist_learner.py and dist_actor.py -for distributed training. For debugging purposes, we provide a script that -simulates distributed mode using multi-processing. Simply go to examples/cim/dqn -and run multi_process_launcher.py to start the learner and actor processes. diff --git a/examples/cim/pg/components/__init__.py b/examples/cim/pg/components/__init__.py deleted file mode 100644 index b14b47650..000000000 --- a/examples/cim/pg/components/__init__.py +++ /dev/null @@ -1,2 +0,0 @@ -# Copyright (c) Microsoft Corporation. -# Licensed under the MIT license. diff --git a/examples/cim/pg/components/action_shaper.py b/examples/cim/pg/components/action_shaper.py deleted file mode 100644 index 687d18d88..000000000 --- a/examples/cim/pg/components/action_shaper.py +++ /dev/null @@ -1,33 +0,0 @@ -# Copyright (c) Microsoft Corporation. -# Licensed under the MIT license. - -from maro.rl import ActionShaper -from maro.simulator.scenarios.cim.common import Action - - -class CIMActionShaper(ActionShaper): - def __init__(self, action_space): - super().__init__() - self._action_space = action_space - self._zero_action_index = action_space.index(0) - - def __call__(self, model_action, decision_event, snapshot_list): - scope = decision_event.action_scope - tick = decision_event.tick - port_idx = decision_event.port_idx - vessel_idx = decision_event.vessel_idx - - port_empty = snapshot_list["ports"][tick: port_idx: ["empty", "full", "on_shipper", "on_consignee"]][0] - vessel_remaining_space = snapshot_list["vessels"][tick: vessel_idx: ["empty", "full", "remaining_space"]][2] - early_discharge = snapshot_list["vessels"][tick:vessel_idx: "early_discharge"][0] - assert 0 <= model_action < len(self._action_space) - - if model_action < self._zero_action_index: - actual_action = max(round(self._action_space[model_action] * port_empty), -vessel_remaining_space) - elif model_action > self._zero_action_index: - plan_action = self._action_space[model_action] * (scope.discharge + early_discharge) - early_discharge - actual_action = round(plan_action) if plan_action > 0 else round(self._action_space[model_action] * scope.discharge) - else: - actual_action = 0 - - return Action(vessel_idx, port_idx, actual_action) diff --git a/examples/cim/pg/components/agent.py b/examples/cim/pg/components/agent.py deleted file mode 100644 index 39d0e994d..000000000 --- a/examples/cim/pg/components/agent.py +++ /dev/null @@ -1,11 +0,0 @@ -# Copyright (c) Microsoft Corporation. -# Licensed under the MIT license. - -import numpy as np - -from maro.rl import AbsAgent - - -class CIMAgent(AbsAgent): - def train(self, states: np.ndarray, actions: np.ndarray, rewards: np.ndarray): - self._algorithm.train(states, actions, rewards) diff --git a/examples/cim/pg/components/agent_manager.py b/examples/cim/pg/components/agent_manager.py deleted file mode 100644 index 28314f323..000000000 --- a/examples/cim/pg/components/agent_manager.py +++ /dev/null @@ -1,49 +0,0 @@ -# Copyright (c) Microsoft Corporation. -# Licensed under the MIT license. - -import torch.nn as nn -from torch.optim import Adam - -from .agent import CIMAgent -from maro.rl import ( - FullyConnectedBlock, LearningModel, LearningModule, OptimizerOptions, PolicyGradient, PolicyGradientConfig, - SimpleAgentManager -) -from maro.utils import set_seeds - - -def create_pg_agents(agent_id_list, config): - num_actions = config.algorithm.num_actions - set_seeds(config.seed) - agent_dict = {} - for agent_id in agent_id_list: - policy_module = LearningModule( - "policy", - [FullyConnectedBlock( - input_dim=config.algorithm.input_dim, - output_dim=num_actions, - activation=nn.Tanh, - **config.algorithm.policy_model - )], - optimizer_options=OptimizerOptions(cls=Adam, params=config.algorithm.optimizer) - ) - - algorithm = PolicyGradient( - model=LearningModel(policy_module), - config=PolicyGradientConfig(reward_decay=config.algorithm.reward_decay) - ) - - agent_dict[agent_id] = CIMAgent(name=agent_id, algorithm=algorithm) - - return agent_dict - - -class PGAgentManager(SimpleAgentManager): - def train(self, experiences_by_agent: dict): - for agent_id, exp in experiences_by_agent.items(): - if isinstance(exp, list): - for trajectory in exp: - self.agent_dict[agent_id].train(trajectory["states"], trajectory["actions"], - trajectory["rewards"]) - else: - self.agent_dict[agent_id].train(exp["states"], exp["actions"], exp["rewards"]) diff --git a/examples/cim/pg/components/experience_shaper.py b/examples/cim/pg/components/experience_shaper.py deleted file mode 100644 index 08bccce49..000000000 --- a/examples/cim/pg/components/experience_shaper.py +++ /dev/null @@ -1,48 +0,0 @@ -# Copyright (c) Microsoft Corporation. -# Licensed under the MIT license. - -from collections import defaultdict - -import numpy as np - -from maro.rl import ExperienceShaper - - -class TruncatedExperienceShaper(ExperienceShaper): - def __init__(self, *, time_window: int, time_decay_factor: float, fulfillment_factor: float, - shortage_factor: float): - super().__init__(reward_func=None) - self._time_window = time_window - self._time_decay_factor = time_decay_factor - self._fulfillment_factor = fulfillment_factor - self._shortage_factor = shortage_factor - - def __call__(self, trajectory, snapshot_list): - agent_ids = np.asarray(trajectory.get_by_key("agent_id")) - states = np.asarray(trajectory.get_by_key("state")) - actions = np.asarray(trajectory.get_by_key("action")) - rewards = np.fromiter( - map(self._compute_reward, trajectory.get_by_key("event"), [snapshot_list]*len(trajectory)), - dtype=np.float32 - ) - return {agent_id: {"states": states[agent_ids == agent_id], - "actions": actions[agent_ids == agent_id], - "rewards": rewards[agent_ids == agent_id], - } - for agent_id in set(agent_ids)} - - def _compute_reward(self, decision_event, snapshot_list): - start_tick = decision_event.tick + 1 - end_tick = decision_event.tick + self._time_window - ticks = list(range(start_tick, end_tick)) - - # calculate tc reward - future_fulfillment = snapshot_list["ports"][ticks::"fulfillment"] - future_shortage = snapshot_list["ports"][ticks::"shortage"] - decay_list = [self._time_decay_factor ** i for i in range(end_tick - start_tick) - for _ in range(future_fulfillment.shape[0]//(end_tick-start_tick))] - - tot_fulfillment = np.dot(future_fulfillment, decay_list) - tot_shortage = np.dot(future_shortage, decay_list) - - return np.float(self._fulfillment_factor * tot_fulfillment - self._shortage_factor * tot_shortage) diff --git a/examples/cim/pg/components/state_shaper.py b/examples/cim/pg/components/state_shaper.py deleted file mode 100644 index 0e2af0ab3..000000000 --- a/examples/cim/pg/components/state_shaper.py +++ /dev/null @@ -1,29 +0,0 @@ -# Copyright (c) Microsoft Corporation. -# Licensed under the MIT license. - -import numpy as np - -from maro.rl import StateShaper - - -class CIMStateShaper(StateShaper): - def __init__(self, *, look_back, max_ports_downstream, port_attributes, vessel_attributes): - super().__init__() - self._look_back = look_back - self._max_ports_downstream = max_ports_downstream - self._port_attributes = port_attributes - self._vessel_attributes = vessel_attributes - self._dim = (look_back + 1) * (max_ports_downstream + 1) * len(port_attributes) + len(vessel_attributes) - - def __call__(self, decision_event, snapshot_list): - tick, port_idx, vessel_idx = decision_event.tick, decision_event.port_idx, decision_event.vessel_idx - ticks = [tick - rt for rt in range(self._look_back-1)] - future_port_idx_list = snapshot_list["vessels"][tick: vessel_idx: 'future_stop_list'].astype('int') - port_features = snapshot_list["ports"][ticks: [port_idx] + list(future_port_idx_list): self._port_attributes] - vessel_features = snapshot_list["vessels"][tick: vessel_idx: self._vessel_attributes] - state = np.concatenate((port_features, vessel_features)) - return str(port_idx), state - - @property - def dim(self): - return self._dim diff --git a/examples/cim/pg/config.yml b/examples/cim/pg/config.yml deleted file mode 100644 index 965461bb9..000000000 --- a/examples/cim/pg/config.yml +++ /dev/null @@ -1,56 +0,0 @@ -env: - scenario: "cim" - topology: "toy.4p_ssdd_l0.0" - durations: 1120 -main_loop: - max_episode: 500 - early_stopping: - warmup_ep: 50 - last_k: 10 - perf_threshold: 0.9 # minimum performance (fulfillment ratio) required to trigger early stopping - perf_stability_threshold: 0.1 # stability is measured by the maximum of abs(perf_(i+1) - perf_i) / perf_i - # over the last k episodes (where perf is short for performance). This value must - # be below this threshold to trigger early stopping -state_shaping: - look_back: 7 - max_ports_downstream: 2 - port_attributes: - - "empty" - - "full" - - "on_shipper" - - "on_consignee" - - "booking" - - "shortage" - - "fulfillment" - vessel_attributes: - - "empty" - - "full" - - "remaining_space" -experience_shaping: - time_window: 100 - fulfillment_factor: 1.0 - shortage_factor: 1.0 - time_decay_factor: 0.97 -agents: - algorithm: - reward_decay: .0 - num_actions: 21 - policy_model: - hidden_dims: - - 256 - - 128 - - 64 - softmax_enabled: true - batch_norm_enabled: false - optimizer: - lr: 0.001 - seed: 1024 # for reproducibility -distributed: - group_name: "pg_distributed_test" - actor: - peer: {"actor": 1} - learner: - peer: {"actor_worker": 1} - redis: - host_name: "localhost" - port: 6379 diff --git a/examples/cim/pg/dist_actor.py b/examples/cim/pg/dist_actor.py deleted file mode 100644 index 720b2536e..000000000 --- a/examples/cim/pg/dist_actor.py +++ /dev/null @@ -1,41 +0,0 @@ -# Copyright (c) Microsoft Corporation. -# Licensed under the MIT license. - -import os - -import numpy as np - -from maro.simulator import Env -from maro.rl import AgentManagerMode, SimpleActor, ActorWorker - -from components.action_shaper import CIMActionShaper -from components.agent_manager import create_pg_agents, PGAgentManager -from components.config import config -from components.experience_shaper import TruncatedExperienceShaper -from components.state_shaper import CIMStateShaper - - -if __name__ == "__main__": - env = Env(config.env.scenario, config.env.topology, durations=config.env.durations) - agent_id_list = [str(agent_id) for agent_id in env.agent_idx_list] - state_shaper = CIMStateShaper(**config.state_shaping) - action_shaper = CIMActionShaper(action_space=list(np.linspace(-1.0, 1.0, config.agents.algorithm.num_actions))) - experience_shaper = TruncatedExperienceShaper(**config.experience_shaping) - agent_manager = PGAgentManager( - name="cim_remote_actor", - mode=AgentManagerMode.INFERENCE, - agent_dict=create_pg_agents(agent_id_list, config.agents), - state_shaper=state_shaper, - action_shaper=action_shaper, - experience_shaper=experience_shaper - ) - proxy_params = { - "group_name": os.environ["GROUP"], - "expected_peers": {"learner": 1}, - "redis_address": ("localhost", 6379) - } - actor_worker = ActorWorker( - local_actor=SimpleActor(env=env, inference_agents=agent_manager), - proxy_params=proxy_params - ) - actor_worker.launch() diff --git a/examples/cim/pg/dist_learner.py b/examples/cim/pg/dist_learner.py deleted file mode 100644 index a721dff3f..000000000 --- a/examples/cim/pg/dist_learner.py +++ /dev/null @@ -1,39 +0,0 @@ -# Copyright (c) Microsoft Corporation. -# Licensed under the MIT license. - -import os - -from maro.rl import ActorProxy, AgentManagerMode, merge_experiences_with_trajectory_boundaries, SimpleLearner -from maro.simulator import Env -from maro.utils import Logger - -from components.agent_manager import create_pg_agents, PGAgentManager -from components.config import config - - -if __name__ == "__main__": - env = Env(config.env.scenario, config.env.topology, durations=config.env.durations) - agent_id_list = [str(agent_id) for agent_id in env.agent_idx_list] - agent_manager = PGAgentManager( - name="cim_remote_learner", - mode=AgentManagerMode.TRAIN, - agent_dict=create_pg_agents(agent_id_list, config.agents), - ) - - proxy_params = { - "group_name": os.environ["GROUP"], - "expected_peers": {"actor": int(os.environ["NUM_ACTORS"])}, - "redis_address": ("localhost", 6379) - } - - learner = SimpleLearner( - agent_manager=agent_manager, - actor=ActorProxy( - proxy_params=proxy_params, - experience_collecting_func=merge_experiences_with_trajectory_boundaries - ), - logger=Logger("distributed_cim_learner", auto_timestamp=False) - ) - learner.learn(max_episode=config.main_loop.max_episode) - learner.test() - learner.dump_models(os.path.join(os.getcwd(), "models")) diff --git a/examples/cim/pg/multi_process_launcher.py b/examples/cim/pg/multi_process_launcher.py deleted file mode 100644 index 0de79a77c..000000000 --- a/examples/cim/pg/multi_process_launcher.py +++ /dev/null @@ -1,26 +0,0 @@ -# Copyright (c) Microsoft Corporation. -# Licensed under the MIT license. - -""" -This script is used to debug distributed algorithm in single host multi-process mode. -""" - -import argparse -import os - - -if __name__ == "__main__": - parser = argparse.ArgumentParser() - parser.add_argument("group_name", help="group name") - parser.add_argument("num_actors", type=int, help="number of actors") - args = parser.parse_args() - - learner_path = f"{os.path.split(os.path.realpath(__file__))[0]}/dist_learner.py &" - actor_path = f"{os.path.split(os.path.realpath(__file__))[0]}/dist_actor.py &" - - # Launch the learner process - os.system(f"GROUP={args.group_name} NUM_ACTORS={args.num_actors} python " + learner_path) - - # Launch the actor processes - for _ in range(args.num_actors): - os.system(f"GROUP={args.group_name} python " + actor_path) diff --git a/examples/cim/ac/README.md b/examples/cim/policy_optimization/README.md similarity index 100% rename from examples/cim/ac/README.md rename to examples/cim/policy_optimization/README.md diff --git a/examples/cim/ac/components/__init__.py b/examples/cim/policy_optimization/components/__init__.py similarity index 100% rename from examples/cim/ac/components/__init__.py rename to examples/cim/policy_optimization/components/__init__.py diff --git a/examples/cim/ac/components/action_shaper.py b/examples/cim/policy_optimization/components/action_shaper.py similarity index 100% rename from examples/cim/ac/components/action_shaper.py rename to examples/cim/policy_optimization/components/action_shaper.py diff --git a/examples/cim/policy_optimization/components/agent_manager.py b/examples/cim/policy_optimization/components/agent_manager.py new file mode 100644 index 000000000..1506912df --- /dev/null +++ b/examples/cim/policy_optimization/components/agent_manager.py @@ -0,0 +1,76 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import numpy as np +import torch.nn as nn +from torch.optim import Adam, RMSprop + +from maro.rl import ( + AbsAgent, ActorCritic, ActorCriticConfig, FullyConnectedBlock, LearningModel, LearningModule, OptimizerOptions, + PolicyGradient, PolicyGradientConfig, PPO, PPOConfig, SimpleAgentManager +) +from maro.utils import set_seeds + + +class POAgent(AbsAgent): + def train(self, states: np.ndarray, actions: np.ndarray, rewards: np.ndarray): + self._algorithm.train(states, actions, rewards) + + +def create_po_agents(agent_id_list, config): + algorithm_map = { + "actor_critic": (ActorCritic, ActorCriticConfig), + "ppo": (PPO, PPOConfig), + "policy_gradient": (PolicyGradient, PolicyGradientConfig) + } + input_dim, num_actions = config.input_dim, config.num_actions + set_seeds(config.seed) + config = config[config.algorithm] + algorithm_cls, algorithm_config = algorithm_map[config.algorithm] + agent_dict = {} + for agent_id in agent_id_list: + actor_module = LearningModule( + "actor", + [FullyConnectedBlock( + input_dim=input_dim, + output_dim=num_actions, + activation=nn.Tanh, + is_head=True, + **config.actor_model + )], + optimizer_options=OptimizerOptions(cls=Adam, params=config.actor_optimizer) + ) + + if config.algorithm in {"actor_critic", "ppo"}: + critic_module = LearningModule( + "critic", + [FullyConnectedBlock( + input_dim=config.input_dim, + output_dim=1, + activation=nn.LeakyReLU, + is_head=True, + **config.critic_model + )], + optimizer_options=OptimizerOptions(cls=RMSprop, params=config.critic_optimizer) + ) + + algorithm = algorithm_cls( + LearningModel(actor_module, critic_module), + algorithm_config(critic_loss_func=nn.functional.smooth_l1_loss, **config.config) + ) + else: + algorithm = algorithm_cls(LearningModel(actor_module), algorithm_config(**config.config)) + + agent_dict[agent_id] = POAgent(name=agent_id, algorithm=algorithm) + + return agent_dict + + +class POAgentManager(SimpleAgentManager): + def train(self, experiences_by_agent: dict): + for agent_id, exp in experiences_by_agent.items(): + if isinstance(exp, list): + for trajectory in exp: + self.agent_dict[agent_id].train(trajectory["states"], trajectory["actions"], trajectory["rewards"]) + else: + self.agent_dict[agent_id].train(exp["states"], exp["actions"], exp["rewards"]) diff --git a/examples/cim/pg/components/config.py b/examples/cim/policy_optimization/components/config.py similarity index 93% rename from examples/cim/pg/components/config.py rename to examples/cim/policy_optimization/components/config.py index 974fcd591..d2f329c37 100644 --- a/examples/cim/pg/components/config.py +++ b/examples/cim/policy_optimization/components/config.py @@ -18,7 +18,7 @@ def set_input_dim(config): num_vessel_attributes = len(config["state_shaping"]["vessel_attributes"]) input_dim = (look_back + 1) * (max_ports_downstream + 1) * num_port_attributes + num_vessel_attributes - config["agents"]["algorithm"]["input_dim"] = input_dim + config["agents"]["input_dim"] = input_dim return config diff --git a/examples/cim/ac/components/experience_shaper.py b/examples/cim/policy_optimization/components/experience_shaper.py similarity index 100% rename from examples/cim/ac/components/experience_shaper.py rename to examples/cim/policy_optimization/components/experience_shaper.py diff --git a/examples/cim/ac/components/state_shaper.py b/examples/cim/policy_optimization/components/state_shaper.py similarity index 100% rename from examples/cim/ac/components/state_shaper.py rename to examples/cim/policy_optimization/components/state_shaper.py diff --git a/examples/cim/ac/config.yml b/examples/cim/policy_optimization/config.yml similarity index 62% rename from examples/cim/ac/config.yml rename to examples/cim/policy_optimization/config.yml index e881cce46..8635e3be8 100644 --- a/examples/cim/ac/config.yml +++ b/examples/cim/policy_optimization/config.yml @@ -32,33 +32,42 @@ experience_shaping: shortage_factor: 1.0 time_decay_factor: 0.97 agents: - algorithm: - num_actions: 21 - actor_model: - hidden_dims: - - 256 - - 128 - - 64 - softmax_enabled: true - batch_norm_enabled: false - critic_model: - hidden_dims: - - 256 - - 128 - - 64 - softmax_enabled: false - batch_norm_enabled: true - actor_optimizer: - lr: 0.001 - critic_optimizer: - lr: 0.001 - config: - reward_decay: .0 - actor_train_iters: 1 - critic_train_iters: 10 - k: 1 - lam: 0.0 seed: 1024 # for reproducibility + num_actions: 21 + algorithm: "actor_critic" # "actor_critic", "policy_gradient" or "ppo" + actor_model: + hidden_dims: + - 256 + - 128 + - 64 + softmax_enabled: true + batch_norm_enabled: false + critic_model: + hidden_dims: + - 256 + - 128 + - 64 + softmax_enabled: false + batch_norm_enabled: true + actor_optimizer: + lr: 0.001 + critic_optimizer: + lr: 0.001 + actor_critic: + reward_decay: .0 + actor_train_iters: 1 + critic_train_iters: 10 + k: 1 + lam: 0.0 + policy_gradient: + reward_decay: .0 + ppo: + reward_decay: .0 + clip_ratio: 0.8 + actor_train_iters: 1 + critic_train_iters: 10 + k: 1 + lam: 0.0 distributed: group_name: "ac_distributed_test" actor: diff --git a/examples/cim/ac/dist_actor.py b/examples/cim/policy_optimization/dist_actor.py similarity index 90% rename from examples/cim/ac/dist_actor.py rename to examples/cim/policy_optimization/dist_actor.py index d5af8a5c8..4ce8a27e4 100644 --- a/examples/cim/ac/dist_actor.py +++ b/examples/cim/policy_optimization/dist_actor.py @@ -10,7 +10,7 @@ from maro.utils import convert_dottable from components.action_shaper import CIMActionShaper -from components.agent_manager import create_ac_agents, ACAgentManager +from components.agent_manager import create_po_agents, POAgentManager from components.config import config, set_input_dim from components.experience_shaper import TruncatedExperienceShaper from components.state_shaper import CIMStateShaper @@ -25,10 +25,10 @@ def launch(config): action_shaper = CIMActionShaper(action_space=list(np.linspace(-1.0, 1.0, config.agents.algorithm.num_actions))) experience_shaper = TruncatedExperienceShaper(**config.experience_shaping) - agent_manager = ACAgentManager( + agent_manager = POAgentManager( name="cim_remote_actor", mode=AgentManagerMode.INFERENCE, - agent_dict=create_ac_agents(agent_id_list, config.agents), + agent_dict=create_po_agents(agent_id_list, config.agents), state_shaper=state_shaper, action_shaper=action_shaper, experience_shaper=experience_shaper, diff --git a/examples/cim/ac/dist_learner.py b/examples/cim/policy_optimization/dist_learner.py similarity index 89% rename from examples/cim/ac/dist_learner.py rename to examples/cim/policy_optimization/dist_learner.py index adaa0dc97..c4f955b9b 100644 --- a/examples/cim/ac/dist_learner.py +++ b/examples/cim/policy_optimization/dist_learner.py @@ -7,7 +7,7 @@ from maro.simulator import Env from maro.utils import Logger, convert_dottable -from components.agent_manager import create_ac_agents, ACAgentManager +from components.agent_manager import create_po_agents, POAgentManager from components.config import set_input_dim @@ -16,10 +16,10 @@ def launch(config): config = convert_dottable(config) env = Env(config.env.scenario, config.env.topology, durations=config.env.durations) agent_id_list = [str(agent_id) for agent_id in env.agent_idx_list] - agent_manager = ACAgentManager( + agent_manager = POAgentManager( name="cim_remote_learner", mode=AgentManagerMode.TRAIN, - agent_dict=create_ac_agents(agent_id_list, config.agents) + agent_dict=create_po_agents(agent_id_list, config.agents) ) proxy_params = { diff --git a/examples/cim/ac/multi_process_launcher.py b/examples/cim/policy_optimization/multi_process_launcher.py similarity index 100% rename from examples/cim/ac/multi_process_launcher.py rename to examples/cim/policy_optimization/multi_process_launcher.py diff --git a/examples/cim/pg/single_process_launcher.py b/examples/cim/policy_optimization/single_process_launcher.py similarity index 94% rename from examples/cim/pg/single_process_launcher.py rename to examples/cim/policy_optimization/single_process_launcher.py index b6c95579b..c94e70eac 100644 --- a/examples/cim/pg/single_process_launcher.py +++ b/examples/cim/policy_optimization/single_process_launcher.py @@ -13,7 +13,7 @@ from maro.utils import Logger, convert_dottable from components.action_shaper import CIMActionShaper -from components.agent_manager import create_pg_agents, PGAgentManager +from components.agent_manager import create_po_agents, POAgentManager from components.config import set_input_dim from components.experience_shaper import TruncatedExperienceShaper from components.state_shaper import CIMStateShaper @@ -35,10 +35,10 @@ def launch(config): experience_shaper = TruncatedExperienceShaper(**config.experience_shaping) # Step 3: create an agent manager. - agent_manager = PGAgentManager( + agent_manager = POAgentManager( name="cim_learner", mode=AgentManagerMode.TRAIN_INFERENCE, - agent_dict=create_pg_agents(agent_id_list, config.agents), + agent_dict=create_po_agents(agent_id_list, config.agents), state_shaper=state_shaper, action_shaper=action_shaper, experience_shaper=experience_shaper, diff --git a/maro/rl/agent/simple_agent_manager.py b/maro/rl/agent/simple_agent_manager.py index de66ef33f..deb5919cf 100644 --- a/maro/rl/agent/simple_agent_manager.py +++ b/maro/rl/agent/simple_agent_manager.py @@ -4,7 +4,6 @@ import os from abc import abstractmethod -from maro.rl.exploration.abs_explorer import AbsExplorer from maro.rl.shaping.action_shaper import ActionShaper from maro.rl.shaping.experience_shaper import ExperienceShaper from maro.rl.shaping.state_shaper import StateShaper From 19e272e40eeb706f9b43255b9650661f94914562 Mon Sep 17 00:00:00 2001 From: ysqyang Date: Wed, 18 Nov 2020 23:07:14 +0800 Subject: [PATCH 230/337] fixed a bug --- examples/cim/policy_optimization/dist_actor.py | 2 +- examples/cim/policy_optimization/single_process_launcher.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/cim/policy_optimization/dist_actor.py b/examples/cim/policy_optimization/dist_actor.py index 4ce8a27e4..900b8ebe7 100644 --- a/examples/cim/policy_optimization/dist_actor.py +++ b/examples/cim/policy_optimization/dist_actor.py @@ -22,7 +22,7 @@ def launch(config): env = Env(config.env.scenario, config.env.topology, durations=config.env.durations) agent_id_list = [str(agent_id) for agent_id in env.agent_idx_list] state_shaper = CIMStateShaper(**config.state_shaping) - action_shaper = CIMActionShaper(action_space=list(np.linspace(-1.0, 1.0, config.agents.algorithm.num_actions))) + action_shaper = CIMActionShaper(action_space=list(np.linspace(-1.0, 1.0, config.agents.num_actions))) experience_shaper = TruncatedExperienceShaper(**config.experience_shaping) agent_manager = POAgentManager( diff --git a/examples/cim/policy_optimization/single_process_launcher.py b/examples/cim/policy_optimization/single_process_launcher.py index c94e70eac..b54827747 100644 --- a/examples/cim/policy_optimization/single_process_launcher.py +++ b/examples/cim/policy_optimization/single_process_launcher.py @@ -31,7 +31,7 @@ def launch(config): # Step 2: create state, action and experience shapers. We also need to create an explorer here due to the # greedy nature of the DQN algorithm. state_shaper = CIMStateShaper(**config.state_shaping) - action_shaper = CIMActionShaper(action_space=list(np.linspace(-1.0, 1.0, config.agents.algorithm.num_actions))) + action_shaper = CIMActionShaper(action_space=list(np.linspace(-1.0, 1.0, config.agents.num_actions))) experience_shaper = TruncatedExperienceShaper(**config.experience_shaping) # Step 3: create an agent manager. From 00429e4aec97a1f861463912659cf9b1159bd20e Mon Sep 17 00:00:00 2001 From: ysqyang Date: Wed, 18 Nov 2020 23:09:34 +0800 Subject: [PATCH 231/337] fixed a bug --- examples/cim/policy_optimization/components/agent_manager.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/examples/cim/policy_optimization/components/agent_manager.py b/examples/cim/policy_optimization/components/agent_manager.py index 1506912df..401118cfe 100644 --- a/examples/cim/policy_optimization/components/agent_manager.py +++ b/examples/cim/policy_optimization/components/agent_manager.py @@ -25,7 +25,6 @@ def create_po_agents(agent_id_list, config): } input_dim, num_actions = config.input_dim, config.num_actions set_seeds(config.seed) - config = config[config.algorithm] algorithm_cls, algorithm_config = algorithm_map[config.algorithm] agent_dict = {} for agent_id in agent_id_list: @@ -56,10 +55,10 @@ def create_po_agents(agent_id_list, config): algorithm = algorithm_cls( LearningModel(actor_module, critic_module), - algorithm_config(critic_loss_func=nn.functional.smooth_l1_loss, **config.config) + algorithm_config(critic_loss_func=nn.functional.smooth_l1_loss, **config[config.algorithm]) ) else: - algorithm = algorithm_cls(LearningModel(actor_module), algorithm_config(**config.config)) + algorithm = algorithm_cls(LearningModel(actor_module), algorithm_config(**config[config.algorithm])) agent_dict[agent_id] = POAgent(name=agent_id, algorithm=algorithm) From 9abbe38ca2b6e4eac37aede7e4b1717ff18ec79e Mon Sep 17 00:00:00 2001 From: ysqyang Date: Wed, 18 Nov 2020 23:12:51 +0800 Subject: [PATCH 232/337] fixed naming for ppo --- maro/rl/algorithms/ppo.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/maro/rl/algorithms/ppo.py b/maro/rl/algorithms/ppo.py index 1f3a2fd89..d91aa0d2d 100644 --- a/maro/rl/algorithms/ppo.py +++ b/maro/rl/algorithms/ppo.py @@ -26,15 +26,15 @@ class PPOConfig: reward_decay (float): Reward decay as defined in standard RL terminology. critic_loss_func (Callable): Critic loss function. clip_ratio (float): Clip ratio as defined in PPO's objective function. - policy_train_iters (int): Number of gradient descent steps for the policy model per call to ``train``. - value_train_iters (int): Number of gradient descent steps for the value model per call to ``train``. + actor_train_iters (int): Number of gradient descent steps for the policy model per call to ``train``. + critic_train_iters (int): Number of gradient descent steps for the value model per call to ``train``. k (int): Number of time steps used in computing returns or return estimates. Defaults to -1, in which case rewards are accumulated until the end of the trajectory. lam (float): Lambda coefficient used in computing lambda returns. Defaults to 1.0, in which case the usual k-step return is computed. """ __slots__ = [ - "reward_decay", "critic_loss_func", "clip_ratio", "policy_train_iters", "value_train_iters", "k", "lam" + "reward_decay", "critic_loss_func", "clip_ratio", "actor_train_iters", "critic_train_iters", "k", "lam" ] def __init__( @@ -42,16 +42,16 @@ def __init__( reward_decay: float, critic_loss_func: Callable, clip_ratio: float, - policy_train_iters: int, - value_train_iters: int, + actor_train_iters: int, + critic_train_iters: int, k: int = -1, lam: float = 1.0 ): self.reward_decay = reward_decay self.critic_loss_func = critic_loss_func self.clip_ratio = clip_ratio - self.policy_train_iters = policy_train_iters - self.value_train_iters = value_train_iters + self.actor_train_iters = actor_train_iters + self.critic_train_iters = critic_train_iters self.k = k self.lam = lam From 699e89520d086ef55d1dbfed981b4f0846afd063 Mon Sep 17 00:00:00 2001 From: ysqyang Date: Wed, 18 Nov 2020 23:25:05 +0800 Subject: [PATCH 233/337] renamed some variables in PPO --- maro/rl/algorithms/ac.py | 2 +- maro/rl/algorithms/ppo.py | 34 +++++++++++++++++----------------- 2 files changed, 18 insertions(+), 18 deletions(-) diff --git a/maro/rl/algorithms/ac.py b/maro/rl/algorithms/ac.py index 182330ed9..5a2799e4b 100644 --- a/maro/rl/algorithms/ac.py +++ b/maro/rl/algorithms/ac.py @@ -84,7 +84,7 @@ def _get_values_and_bootstrapped_returns(self, state_sequence, reward_sequence): def train(self, states: np.ndarray, actions: np.ndarray, rewards: np.ndarray): state_values, return_est = self._get_values_and_bootstrapped_returns(states, rewards) advantages = return_est - state_values - if self._model.shared_module is not None and self._model.shared_module.is_trainable: + if self._model.shared_module and self._model.shared_module.is_trainable: pass else: # policy model training diff --git a/maro/rl/algorithms/ppo.py b/maro/rl/algorithms/ppo.py index d91aa0d2d..594c9a7bb 100644 --- a/maro/rl/algorithms/ppo.py +++ b/maro/rl/algorithms/ppo.py @@ -87,23 +87,23 @@ def _get_values_and_bootstrapped_returns(self, states: torch.tensor, rewards: np @preprocess def train( - self, states: np.ndarray, actions: np.ndarray, log_action_prob: np.ndarray, rewards: np.ndarray + self, states: np.ndarray, actions: np.ndarray, log_action_prob_old: np.ndarray, rewards: np.ndarray ): - states = torch.from_numpy(states).to(self._device) # (N, state_dim) state_values, return_est = self._get_values_and_bootstrapped_returns(states, rewards) advantages = return_est - state_values - actions = torch.from_numpy(actions).to(self._device) # (N,) - log_action_prob_old = torch.from_numpy(log_action_prob).to(self._device) - - # policy model training (with the value model fixed) - for _ in range(self._config.policy_train_iters): - action_prob = self._model(states, task_name="actor").gather(1, actions.unsqueeze(1)).squeeze() # (N, 1) - ratio = torch.exp(torch.log(action_prob) - log_action_prob_old) - clipped_ratio = torch.clamp(ratio, 1 - self._config.clip_ratio, 1 + self._config.clip_ratio) - loss = -(torch.min(ratio * advantages, clipped_ratio * advantages)).mean() - self._model.learn(loss) - - # value model training - for _ in range(self._config.value_train_iters): - loss = self._config.lovalue_loss_func(self._model(states, task_name="critic"), return_est) - self._model.learn(loss) + + if self._model.shared_module and self._model.shared_module.is_trainable: + pass + else: + # policy model training (with the value model fixed) + for _ in range(self._config.actor_train_iters): + action_prob = self._model(states, task_name="actor").gather(1, actions.unsqueeze(1)).squeeze() # (N, 1) + ratio = torch.exp(torch.log(action_prob) - log_action_prob_old) + clipped_ratio = torch.clamp(ratio, 1 - self._config.clip_ratio, 1 + self._config.clip_ratio) + loss = -(torch.min(ratio * advantages, clipped_ratio * advantages)).mean() + self._model.learn(loss) + + # value model training + for _ in range(self._config.critic_train_iters): + loss = self._config.critic_loss_func(self._model(states, task_name="critic"), return_est) + self._model.learn(loss) From 41efa70d299bcb6d95068035aa9b0c07fc412eff Mon Sep 17 00:00:00 2001 From: ysqyang Date: Thu, 19 Nov 2020 14:29:56 +0800 Subject: [PATCH 234/337] added ActionWithLogProbability return type for PO-type algorithms --- docs/source/examples/cim.rst | 4 ++-- docs/source/examples/multi_agent_dqn_cim.rst | 4 ++-- examples/cim/dqn/components/agent.py | 2 +- examples/cim/dqn/components/agent_manager.py | 4 ++-- .../components/agent_manager.py | 21 ++++++++++++------- .../components/experience_shaper.py | 11 ++++++---- maro/rl/__init__.py | 4 +++- maro/rl/agent/abs_agent.py | 6 +++++- maro/rl/agent/simple_agent_manager.py | 12 ++++++++--- maro/rl/algorithms/ac.py | 15 ++++++++++--- maro/rl/algorithms/dqn.py | 2 +- maro/rl/algorithms/pg.py | 13 ++++++++++-- maro/rl/algorithms/ppo.py | 15 ++++++++++--- maro/rl/algorithms/utils.py | 3 +++ .../rl_formulation.ipynb | 4 ++-- 15 files changed, 86 insertions(+), 34 deletions(-) diff --git a/docs/source/examples/cim.rst b/docs/source/examples/cim.rst index 5d612ef1f..a111b9d99 100644 --- a/docs/source/examples/cim.rst +++ b/docs/source/examples/cim.rst @@ -114,7 +114,7 @@ algorithm, experience pool, and a set of parameters that governs the training lo abstraction of a port. We choose DQN as our underlying learning algorithm with a TD-error-based sampling mechanism. .. code-block:: python - class CIMAgent(AbsAgent): + class DQNAgent(AbsAgent): ... def train(self): if len(self._experience_pool) < self._min_experiences_to_train: @@ -156,7 +156,7 @@ the DQN algorithm and an experience pool for each agent. num_actions=num_actions)) experience_pool = ColumnBasedStore(**config.agents.experience_pool) - agent_dict[agent_id] = CIMAgent(name=agent_id, algorithm=algorithm, experience_pool=experience_pool, + agent_dict[agent_id] = DQNAgent(name=agent_id, algorithm=algorithm, experience_pool=experience_pool, **config.agents.training_loop_parameters) Main Loop with Actor and Learner (Single Process) diff --git a/docs/source/examples/multi_agent_dqn_cim.rst b/docs/source/examples/multi_agent_dqn_cim.rst index 3314c179b..d21fbce5b 100644 --- a/docs/source/examples/multi_agent_dqn_cim.rst +++ b/docs/source/examples/multi_agent_dqn_cim.rst @@ -96,7 +96,7 @@ algorithm, experience pool, and a set of parameters that governs the training lo abstraction of a port. We choose DQN as our underlying learning algorithm with a TD-error-based sampling mechanism. .. code-block:: python - class CIMAgent(AbsAgent): + class DQNAgent(AbsAgent): ... def train(self): if len(self._experience_pool) < self._min_experiences_to_train: @@ -147,7 +147,7 @@ experience pools before training, in accordance with the DQN algorithm. ) experience_pool = ColumnBasedStore(**config.experience_pool) - agent_dict[agent_id] = CIMAgent( + agent_dict[agent_id] = DQNAgent( name=agent_id, algorithm=algorithm, experience_pool=experience_pool, diff --git a/examples/cim/dqn/components/agent.py b/examples/cim/dqn/components/agent.py index 93c9a4abb..a3ad7dc98 100644 --- a/examples/cim/dqn/components/agent.py +++ b/examples/cim/dqn/components/agent.py @@ -9,7 +9,7 @@ from maro.rl import AbsAgent, EpsilonGreedyExplorer, ColumnBasedStore -class CIMAgent(AbsAgent): +class DQNAgent(AbsAgent): """Implementation of AbsAgent for the DQN algorithm. Args: diff --git a/examples/cim/dqn/components/agent_manager.py b/examples/cim/dqn/components/agent_manager.py index bd9c9bf94..cb35facd7 100644 --- a/examples/cim/dqn/components/agent_manager.py +++ b/examples/cim/dqn/components/agent_manager.py @@ -10,7 +10,7 @@ ) from maro.utils import set_seeds -from .agent import CIMAgent +from .agent import DQNAgent def create_dqn_agents(agent_id_list, config): @@ -39,7 +39,7 @@ def create_dqn_agents(agent_id_list, config): ) experience_pool = ColumnBasedStore(**config.experience_pool) - agent_dict[agent_id] = CIMAgent( + agent_dict[agent_id] = DQNAgent( agent_id, algorithm, EpsilonGreedyExplorer(num_actions), experience_pool, **config.training_loop_parameters ) diff --git a/examples/cim/policy_optimization/components/agent_manager.py b/examples/cim/policy_optimization/components/agent_manager.py index 401118cfe..7edde4524 100644 --- a/examples/cim/policy_optimization/components/agent_manager.py +++ b/examples/cim/policy_optimization/components/agent_manager.py @@ -13,8 +13,11 @@ class POAgent(AbsAgent): - def train(self, states: np.ndarray, actions: np.ndarray, rewards: np.ndarray): - self._algorithm.train(states, actions, rewards) + def train(self, states: np.ndarray, actions: np.ndarray, log_action_prob: np.ndarray, rewards: np.ndarray): + if isinstance(self._algorithm, PPO): + self._algorithm.train(states, actions, log_action_prob, rewards) + else: + self._algorithm.train(states, actions, rewards) def create_po_agents(agent_id_list, config): @@ -68,8 +71,12 @@ def create_po_agents(agent_id_list, config): class POAgentManager(SimpleAgentManager): def train(self, experiences_by_agent: dict): for agent_id, exp in experiences_by_agent.items(): - if isinstance(exp, list): - for trajectory in exp: - self.agent_dict[agent_id].train(trajectory["states"], trajectory["actions"], trajectory["rewards"]) - else: - self.agent_dict[agent_id].train(exp["states"], exp["actions"], exp["rewards"]) + if not isinstance(exp, list): + exp = [exp] + for trajectory in exp: + self.agent_dict[agent_id].train( + trajectory["state"], + trajectory["action"], + trajectory["log_action_probability"], + trajectory["reward"] + ) diff --git a/examples/cim/policy_optimization/components/experience_shaper.py b/examples/cim/policy_optimization/components/experience_shaper.py index a4ac82585..e74a62f7f 100644 --- a/examples/cim/policy_optimization/components/experience_shaper.py +++ b/examples/cim/policy_optimization/components/experience_shaper.py @@ -21,14 +21,17 @@ def __call__(self, trajectory, snapshot_list): agent_ids = np.asarray(trajectory.get_by_key("agent_id")) states = np.asarray(trajectory.get_by_key("state")) actions = np.asarray(trajectory.get_by_key("action")) + log_action_probabilities = np.asarray(trajectory.get_by_key["log_action_probability"]) rewards = np.fromiter( map(self._compute_reward, trajectory.get_by_key("event"), [snapshot_list] * len(trajectory)), dtype=np.float32 ) - return {agent_id: {"states": states[agent_ids == agent_id], - "actions": actions[agent_ids == agent_id], - "rewards": rewards[agent_ids == agent_id], - } + return {agent_id: { + "state": states[agent_ids == agent_id], + "action": actions[agent_ids == agent_id], + "log_action_probability": log_action_probabilities[agent_ids == agent_id], + "reward": rewards[agent_ids == agent_id], + } for agent_id in set(agent_ids)} def _compute_reward(self, decision_event, snapshot_list): diff --git a/maro/rl/__init__.py b/maro/rl/__init__.py index 4770445eb..9a81f57db 100644 --- a/maro/rl/__init__.py +++ b/maro/rl/__init__.py @@ -11,7 +11,7 @@ from maro.rl.algorithms.dqn import DQN, DQNConfig, DuelingDQNTask from maro.rl.algorithms.pg import PolicyGradient, PolicyGradientConfig from maro.rl.algorithms.ppo import PPO, PPOConfig -from maro.rl.algorithms.utils import preprocess, to_device, validate_task_names +from maro.rl.algorithms.utils import ActionWithLogProbability, expand_dim, preprocess, to_device, validate_task_names from maro.rl.dist_topologies.experience_collection import ( concat_experiences_by_agent, merge_experiences_with_trajectory_boundaries ) @@ -47,6 +47,7 @@ 'AbsShaper', 'AbsStore', 'ActionShaper', + 'ActionWithLogProbability', 'ActorCritic', 'ActorCriticConfig', 'ActorProxy', @@ -76,6 +77,7 @@ 'SimpleLearner', 'StateShaper', 'concat_experiences_by_agent', + 'expand_dim', 'linear_epsilon_schedule', 'merge_experiences_with_trajectory_boundaries', 'preprocess', diff --git a/maro/rl/agent/abs_agent.py b/maro/rl/agent/abs_agent.py index dd05fb34a..43d9611c0 100644 --- a/maro/rl/agent/abs_agent.py +++ b/maro/rl/agent/abs_agent.py @@ -5,6 +5,7 @@ from abc import ABC, abstractmethod from maro.rl.algorithms.abs_algorithm import AbsAlgorithm +from maro.rl.algorithms.utils import ActionWithLogProbability from maro.rl.exploration.abs_explorer import AbsExplorer from maro.rl.storage.abs_store import AbsStore @@ -64,7 +65,10 @@ def choose_action(self, model_state): an exploratory action is returned. """ action = self._algorithm.choose_action(model_state) - return action if self._explorer is None else self._explorer(action) + if isinstance(action, ActionWithLogProbability) or self._explorer is None: + return action + else: + return self._explorer(action) def load_exploration_params(self, exploration_params): if self._explorer: diff --git a/maro/rl/agent/simple_agent_manager.py b/maro/rl/agent/simple_agent_manager.py index deb5919cf..20cd18289 100644 --- a/maro/rl/agent/simple_agent_manager.py +++ b/maro/rl/agent/simple_agent_manager.py @@ -4,6 +4,7 @@ import os from abc import abstractmethod +from maro.rl.algorithms.utils import ActionWithLogProbability from maro.rl.shaping.action_shaper import ActionShaper from maro.rl.shaping.experience_shaper import ExperienceShaper from maro.rl.shaping.state_shaper import StateShaper @@ -45,15 +46,20 @@ def __init__( def choose_action(self, decision_event, snapshot_list): self._assert_inference_mode() agent_id, state = self._state_shaper(decision_event, snapshot_list) - action = self.agent_dict[agent_id].choose_action(state) + action_info = self.agent_dict[agent_id].choose_action(state) self._transition_cache = { "state": state, - "action": action, "reward": None, "agent_id": agent_id, "event": decision_event } - return self._action_shaper(action, decision_event, snapshot_list) + if isinstance(action_info, ActionWithLogProbability): + self._transition_cache["action"] = action_info.action + self._transition_cache["log_action_probability"] = action_info.log_probability + else: + self._transition_cache["action"] = action_info + + return self._action_shaper(self._transition_cache["action"], decision_event, snapshot_list) def on_env_feedback(self, metrics): """This method records the environment-generated metrics as part of the latest transition in the trajectory. diff --git a/maro/rl/algorithms/ac.py b/maro/rl/algorithms/ac.py index 5a2799e4b..d18a79d26 100644 --- a/maro/rl/algorithms/ac.py +++ b/maro/rl/algorithms/ac.py @@ -11,7 +11,7 @@ from maro.rl.models.learning_model import LearningModel from maro.rl.utils.trajectory_utils import get_lambda_returns -from .utils import expand_dim, preprocess, to_device, validate_task_names +from .utils import ActionWithLogProbability, expand_dim, preprocess, to_device, validate_task_names class ActorCriticTask(Enum): @@ -63,15 +63,24 @@ class ActorCritic(AbsAlgorithm): It may or may not have a shared bottom stack. config: Configuration for the AC algorithm. """ - @to_device @validate_task_names(ActorCriticTask) + @to_device def __init__(self, model: LearningModel, config: ActorCriticConfig): super().__init__(model, config) @expand_dim def choose_action(self, state: np.ndarray): + """Use the actor (policy) model to generate a stochastic action. + + Args: + state: Input to the actor model. + + Returns: + A ActionWithLogProbability namedtuple instance containing the action index and the corresponding probability. + """ action_distribution = self._model(state, task_name="actor", is_training=False).squeeze().numpy() - return np.random.choice(len(action_distribution), p=action_distribution) + action = np.random.choice(len(action_distribution), p=action_distribution) + return ActionWithLogProbability(action=action, log_probability=np.log(action_distribution[action])) def _get_values_and_bootstrapped_returns(self, state_sequence, reward_sequence): state_values = self._model(state_sequence, task_name="critic").detach().squeeze() diff --git a/maro/rl/algorithms/dqn.py b/maro/rl/algorithms/dqn.py index 9fb25f721..081508b01 100644 --- a/maro/rl/algorithms/dqn.py +++ b/maro/rl/algorithms/dqn.py @@ -66,8 +66,8 @@ class DQN(AbsAlgorithm): model (LearningModel): Q-value model. config: Configuration for DQN algorithm. """ - @to_device @validate_task_names(DuelingDQNTask) + @to_device def __init__(self, model: LearningModel, config: DQNConfig): super().__init__(model, config) self._training_counter = 0 diff --git a/maro/rl/algorithms/pg.py b/maro/rl/algorithms/pg.py index 04c35d501..f798c017b 100644 --- a/maro/rl/algorithms/pg.py +++ b/maro/rl/algorithms/pg.py @@ -7,7 +7,7 @@ from maro.rl.algorithms.abs_algorithm import AbsAlgorithm from maro.rl.models.learning_model import LearningModel -from .utils import expand_dim, preprocess, to_device +from .utils import ActionWithLogProbability, expand_dim, preprocess, to_device class PolicyGradientConfig: @@ -39,8 +39,17 @@ def __init__(self, model: LearningModel, config: PolicyGradientConfig): @expand_dim def choose_action(self, state: np.ndarray): + """Use the actor (policy) model to generate a stochastic action. + + Args: + state: Input to the actor model. + + Returns: + A ActionWithLogProbability namedtuple instance containing the action index and the corresponding probability. + """ action_distribution = self._model(state, is_training=False).squeeze().numpy() # (num_actions,) - return np.random.choice(len(action_distribution), p=action_distribution) + action = np.random.choice(len(action_distribution), p=action_distribution) + return ActionWithLogProbability(action=action, log_probability=np.log(action_distribution[action])) @preprocess def train(self, states: np.ndarray, actions: np.ndarray, returns: np.ndarray): diff --git a/maro/rl/algorithms/ppo.py b/maro/rl/algorithms/ppo.py index 594c9a7bb..08129664d 100644 --- a/maro/rl/algorithms/ppo.py +++ b/maro/rl/algorithms/ppo.py @@ -11,7 +11,7 @@ from maro.rl.models.learning_model import LearningModel from maro.rl.utils.trajectory_utils import get_lambda_returns -from .utils import expand_dim, preprocess, to_device, validate_task_names +from .utils import ActionWithLogProbability, expand_dim, preprocess, to_device, validate_task_names class PPOTask(Enum): @@ -66,8 +66,8 @@ class PPO(AbsAlgorithm): It may or may not have a shared bottom stack. config: Configuration for the PPO algorithm. """ - @to_device @validate_task_names(PPOTask) + @to_device def __init__(self, model: LearningModel, config: PPOConfig): super().__init__(model, config) self._device = torch.device("cuda" if torch.cuda.is_available() else "cpu") @@ -75,8 +75,17 @@ def __init__(self, model: LearningModel, config: PPOConfig): @expand_dim def choose_action(self, state: np.ndarray): + """Use the actor (policy) model to generate a stochastic action. + + Args: + state: Input to the actor model. + + Returns: + A ActionWithLogProbability namedtuple instance containing the action index and the corresponding probability. + """ action_distribution = self._model(state, task_name="actor", is_training=False).squeeze().numpy() - return np.random.choice(len(action_distribution), p=action_distribution) + action = np.random.choice(len(action_distribution), p=action_distribution) + return ActionWithLogProbability(action=action, log_probability=np.log(action_distribution[action])) def _get_values_and_bootstrapped_returns(self, states: torch.tensor, rewards: np.ndarray): state_values = self._model(states, task_name="critic").detach().squeeze() diff --git a/maro/rl/algorithms/utils.py b/maro/rl/algorithms/utils.py index e4c7a9faf..a973d209b 100644 --- a/maro/rl/algorithms/utils.py +++ b/maro/rl/algorithms/utils.py @@ -1,6 +1,7 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT license. +from collections import namedtuple from enum import Enum from functools import wraps @@ -11,6 +12,8 @@ device = torch.device("cuda" if torch.cuda.is_available() else "cpu") +ActionWithLogProbability = namedtuple("action_with_probability", ["action", "log_probability"]) + def validate_task_names(task_enum: Enum): def decorator(init_func): diff --git a/notebooks/container_inventory_management/rl_formulation.ipynb b/notebooks/container_inventory_management/rl_formulation.ipynb index 833a24640..e4a270fd3 100644 --- a/notebooks/container_inventory_management/rl_formulation.ipynb +++ b/notebooks/container_inventory_management/rl_formulation.ipynb @@ -185,7 +185,7 @@ "from maro.rl import AbsAgent, ColumnBasedStore\n", "\n", "\n", - "class CIMAgent(AbsAgent):\n", + "class DQNAgent(AbsAgent):\n", " def __init__(self, name, algorithm, experience_pool: ColumnBasedStore, min_experiences_to_train,\n", " num_batches, batch_size):\n", " super().__init__(name, algorithm, experience_pool)\n", @@ -267,7 +267,7 @@ " )\n", "\n", " experience_pool = ColumnBasedStore()\n", - " agent_dict[agent_id] = CIMAgent(\n", + " agent_dict[agent_id] = DQNAgent(\n", " name=agent_id,\n", " algorithm=algorithm,\n", " experience_pool=experience_pool,\n", From 694d2b0b294769918f2a5a48968fc42a3cf3c746 Mon Sep 17 00:00:00 2001 From: ysqyang Date: Thu, 19 Nov 2020 14:31:11 +0800 Subject: [PATCH 235/337] fixed a bug --- .../cim/policy_optimization/components/experience_shaper.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/cim/policy_optimization/components/experience_shaper.py b/examples/cim/policy_optimization/components/experience_shaper.py index e74a62f7f..0ce38104b 100644 --- a/examples/cim/policy_optimization/components/experience_shaper.py +++ b/examples/cim/policy_optimization/components/experience_shaper.py @@ -21,7 +21,7 @@ def __call__(self, trajectory, snapshot_list): agent_ids = np.asarray(trajectory.get_by_key("agent_id")) states = np.asarray(trajectory.get_by_key("state")) actions = np.asarray(trajectory.get_by_key("action")) - log_action_probabilities = np.asarray(trajectory.get_by_key["log_action_probability"]) + log_action_probabilities = np.asarray(trajectory.get_by_key("log_action_probability")) rewards = np.fromiter( map(self._compute_reward, trajectory.get_by_key("event"), [snapshot_list] * len(trajectory)), dtype=np.float32 From 7db7100716a72d8d69963c9815d5c181a18ff270 Mon Sep 17 00:00:00 2001 From: ysqyang Date: Thu, 19 Nov 2020 14:33:59 +0800 Subject: [PATCH 236/337] fixed a bug --- maro/rl/algorithms/ppo.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/maro/rl/algorithms/ppo.py b/maro/rl/algorithms/ppo.py index 08129664d..819bd1f79 100644 --- a/maro/rl/algorithms/ppo.py +++ b/maro/rl/algorithms/ppo.py @@ -109,10 +109,12 @@ def train( action_prob = self._model(states, task_name="actor").gather(1, actions.unsqueeze(1)).squeeze() # (N, 1) ratio = torch.exp(torch.log(action_prob) - log_action_prob_old) clipped_ratio = torch.clamp(ratio, 1 - self._config.clip_ratio, 1 + self._config.clip_ratio) - loss = -(torch.min(ratio * advantages, clipped_ratio * advantages)).mean() - self._model.learn(loss) + actor_loss = -(torch.min(ratio * advantages, clipped_ratio * advantages)).mean() + self._model.learn(actor_loss) # value model training for _ in range(self._config.critic_train_iters): - loss = self._config.critic_loss_func(self._model(states, task_name="critic"), return_est) - self._model.learn(loss) + critic_loss = self._config.critic_loss_func( + self._model(states, task_name="critic").squeeze(), return_est + ) + self._model.learn(critic_loss) From c499534d94342e2392a57785f4b931769ac1595b Mon Sep 17 00:00:00 2001 From: ysqyang Date: Thu, 19 Nov 2020 14:48:46 +0800 Subject: [PATCH 237/337] fixed lint issues --- maro/rl/algorithms/ac.py | 3 ++- maro/rl/algorithms/pg.py | 3 ++- maro/rl/algorithms/ppo.py | 3 ++- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/maro/rl/algorithms/ac.py b/maro/rl/algorithms/ac.py index d18a79d26..7d8400c2e 100644 --- a/maro/rl/algorithms/ac.py +++ b/maro/rl/algorithms/ac.py @@ -76,7 +76,8 @@ def choose_action(self, state: np.ndarray): state: Input to the actor model. Returns: - A ActionWithLogProbability namedtuple instance containing the action index and the corresponding probability. + A ActionWithLogProbability namedtuple instance containing the action index and the corresponding + log probability. """ action_distribution = self._model(state, task_name="actor", is_training=False).squeeze().numpy() action = np.random.choice(len(action_distribution), p=action_distribution) diff --git a/maro/rl/algorithms/pg.py b/maro/rl/algorithms/pg.py index f798c017b..50a0a3283 100644 --- a/maro/rl/algorithms/pg.py +++ b/maro/rl/algorithms/pg.py @@ -45,7 +45,8 @@ def choose_action(self, state: np.ndarray): state: Input to the actor model. Returns: - A ActionWithLogProbability namedtuple instance containing the action index and the corresponding probability. + A ActionWithLogProbability namedtuple instance containing the action index and the corresponding + log probability. """ action_distribution = self._model(state, is_training=False).squeeze().numpy() # (num_actions,) action = np.random.choice(len(action_distribution), p=action_distribution) diff --git a/maro/rl/algorithms/ppo.py b/maro/rl/algorithms/ppo.py index 819bd1f79..810a13c10 100644 --- a/maro/rl/algorithms/ppo.py +++ b/maro/rl/algorithms/ppo.py @@ -81,7 +81,8 @@ def choose_action(self, state: np.ndarray): state: Input to the actor model. Returns: - A ActionWithLogProbability namedtuple instance containing the action index and the corresponding probability. + A ActionWithLogProbability namedtuple instance containing the action index and the corresponding + log probability. """ action_distribution = self._model(state, task_name="actor", is_training=False).squeeze().numpy() action = np.random.choice(len(action_distribution), p=action_distribution) From c98f5620f394a4f1dc9c071ce92eb56c2a8e15e3 Mon Sep 17 00:00:00 2001 From: ysqyang Date: Thu, 19 Nov 2020 15:58:03 +0800 Subject: [PATCH 238/337] revised __getstate__ for LearningModel --- maro/rl/models/learning_model.py | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/maro/rl/models/learning_model.py b/maro/rl/models/learning_model.py index 3ca760366..0380b8895 100644 --- a/maro/rl/models/learning_model.py +++ b/maro/rl/models/learning_model.py @@ -69,14 +69,16 @@ def zero_grad(self): def step(self): self._optimizer.step() + def copy(self): + return clone(self) + class LearningModel(nn.Module): """NN model that consists of multiple task heads and an optional shared stack. Args: task_modules (LearningModule): LearningModule instances, each of which performs a designated task. - shared_module (LearningModule): Network module that forms that shared part of the model. Defaults to - None. + shared_module (LearningModule): Network module that forms that shared part of the model. Defaults to None. """ def __init__( self, @@ -97,11 +99,16 @@ def __init__( }) def __getstate__(self): + shared_module = self._shared_module.copy() if self._shared_module else None + task_modules = (task_module.copy() for task_module in self._task_modules) + net = nn.ModuleDict({ + task_module.name: nn.Sequential(shared_module, task_module) if shared_module else task_module + for task_module in task_modules + }) dic = self.__dict__.copy() - if "_shared_optimizer" in dic: - del dic["_shared_optimizer"] - if "_head_optimizer_dict" in dic: - del dic["_head_optimizer_dict"] + dic["_shared_module"] = shared_module + dic["_task_modules"] = task_modules + dic["_net"] = net return dic def __setstate__(self, dic: dict): From a4b419b40d3363bd2e67e8b8a0e4e75b441a9e62 Mon Sep 17 00:00:00 2001 From: ysqyang Date: Thu, 19 Nov 2020 16:07:56 +0800 Subject: [PATCH 239/337] fixed a bug --- maro/rl/models/learning_model.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/maro/rl/models/learning_model.py b/maro/rl/models/learning_model.py index 0380b8895..39203e5fc 100644 --- a/maro/rl/models/learning_model.py +++ b/maro/rl/models/learning_model.py @@ -92,7 +92,7 @@ def __init__( self._shared_module = shared_module # task_heads - self._task_modules = task_modules + self._task_modules = list(task_modules) self._net = nn.ModuleDict({ task_module.name: nn.Sequential(self._shared_module, task_module) if self._shared_module else task_module for task_module in self._task_modules @@ -100,7 +100,7 @@ def __init__( def __getstate__(self): shared_module = self._shared_module.copy() if self._shared_module else None - task_modules = (task_module.copy() for task_module in self._task_modules) + task_modules = [task_module.copy() for task_module in self._task_modules] net = nn.ModuleDict({ task_module.name: nn.Sequential(shared_module, task_module) if shared_module else task_module for task_module in task_modules From 3e74a200c1915af4d1d39680fad93dcc477d43db Mon Sep 17 00:00:00 2001 From: ysqyang Date: Thu, 19 Nov 2020 17:23:00 +0800 Subject: [PATCH 240/337] added soft_update function to learningModel --- maro/rl/algorithms/dqn.py | 10 +--------- maro/rl/models/learning_model.py | 5 +++++ 2 files changed, 6 insertions(+), 9 deletions(-) diff --git a/maro/rl/algorithms/dqn.py b/maro/rl/algorithms/dqn.py index 7d104ca84..a67ff993d 100644 --- a/maro/rl/algorithms/dqn.py +++ b/maro/rl/algorithms/dqn.py @@ -118,14 +118,6 @@ def train(self, states: np.ndarray, actions: np.ndarray, rewards: np.ndarray, ne self._model.learn(loss.mean() if self._config.per_sample_td_error_enabled else loss) self._training_counter += 1 if self._training_counter % self._config.target_update_frequency == 0: - self._update_targets() + self._target_model.soft_update(self._model, self._config.tau) return loss.detach().numpy() - - def _update_targets(self): - for eval_params, target_params in zip( - self._model.parameters(), self._target_model.parameters() - ): - target_params.data = ( - self._config.tau * eval_params.data + (1 - self._config.tau) * target_params.data - ) diff --git a/maro/rl/models/learning_model.py b/maro/rl/models/learning_model.py index 39203e5fc..6cac3ffb5 100644 --- a/maro/rl/models/learning_model.py +++ b/maro/rl/models/learning_model.py @@ -185,6 +185,11 @@ def learn(self, loss): if self._shared_module is not None: self._shared_module.step() + def soft_update(self, other_model: nn.Module, tau: float): + for params, other_params in zip(self.parameters(), other_model.parameters()): + params.data = (1 - tau) * params.data + tau * other_params.data + return self + def copy(self): return clone(self) From e972f750c6c225f750b58379de763e42bc2dfec9 Mon Sep 17 00:00:00 2001 From: ysqyang Date: Thu, 19 Nov 2020 19:22:49 +0800 Subject: [PATCH 241/337] fixed a bug --- maro/rl/models/learning_model.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/maro/rl/models/learning_model.py b/maro/rl/models/learning_model.py index 6cac3ffb5..c414a2cfe 100644 --- a/maro/rl/models/learning_model.py +++ b/maro/rl/models/learning_model.py @@ -36,7 +36,7 @@ def __getstate__(self): dic = self.__dict__.copy() if "_optimizer" in dic: del dic["_optimizer"] - dic["is_trainable"] = False + dic["_is_trainable"] = False return dic def __setstate__(self, dic: dict): @@ -188,7 +188,6 @@ def learn(self, loss): def soft_update(self, other_model: nn.Module, tau: float): for params, other_params in zip(self.parameters(), other_model.parameters()): params.data = (1 - tau) * params.data + tau * other_params.data - return self def copy(self): return clone(self) From 0e13ab2d900933a82adb3a3fc2f049a69af7e446 Mon Sep 17 00:00:00 2001 From: ysqyang Date: Thu, 19 Nov 2020 23:00:40 +0800 Subject: [PATCH 242/337] revised learningModel --- maro/rl/__init__.py | 1 + maro/rl/models/abs_block.py | 14 +++++ maro/rl/models/fc_block.py | 4 +- maro/rl/models/learning_model.py | 56 ++++++++++---------- maro/rl/models/utils.py | 17 ++++++ maro/utils/exception/error_code.py | 5 +- maro/utils/exception/rl_toolkit_exception.py | 10 +++- 7 files changed, 76 insertions(+), 31 deletions(-) create mode 100644 maro/rl/models/abs_block.py create mode 100644 maro/rl/models/utils.py diff --git a/maro/rl/__init__.py b/maro/rl/__init__.py index 065b633cc..5a0c50756 100644 --- a/maro/rl/__init__.py +++ b/maro/rl/__init__.py @@ -22,6 +22,7 @@ from maro.rl.exploration.epsilon_schedule import linear_epsilon_schedule, two_phase_linear_epsilon_schedule from maro.rl.learner.abs_learner import AbsLearner from maro.rl.learner.simple_learner import SimpleLearner +from maro.rl.models.abs_block import AbsBlock from maro.rl.models.fc_block import FullyConnectedBlock from maro.rl.models.learning_model import LearningModel, LearningModule, OptimizerOptions from maro.rl.shaping.abs_shaper import AbsShaper diff --git a/maro/rl/models/abs_block.py b/maro/rl/models/abs_block.py new file mode 100644 index 000000000..2f5a1e850 --- /dev/null +++ b/maro/rl/models/abs_block.py @@ -0,0 +1,14 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import torch.nn as nn + + +class AbsBlock(nn.Module): + @property + def input_dim(self): + raise NotImplementedError + + @property + def output_dim(self): + raise NotImplementedError diff --git a/maro/rl/models/fc_block.py b/maro/rl/models/fc_block.py index ff0bf4043..16b759aa3 100644 --- a/maro/rl/models/fc_block.py +++ b/maro/rl/models/fc_block.py @@ -6,8 +6,10 @@ import torch import torch.nn as nn +from .abs_block import AbsBlock -class FullyConnectedBlock(nn.Module): + +class FullyConnectedBlock(AbsBlock): """Fully connected network with optional batch normalization, activation and dropout components. Args: diff --git a/maro/rl/models/learning_model.py b/maro/rl/models/learning_model.py index c414a2cfe..7412fb988 100644 --- a/maro/rl/models/learning_model.py +++ b/maro/rl/models/learning_model.py @@ -9,6 +9,9 @@ from maro.utils import clone from maro.utils.exception.rl_toolkit_exception import MissingOptimizerError +from .abs_block import AbsBlock +from .utils import check_chainability + OptimizerOptions = namedtuple("OptimizerOptions", ["cls", "params"]) @@ -20,9 +23,11 @@ class LearningModule(nn.Module): of a block must match the input dimension of its successor. optimizer_options (OptimizerOptions): A namedtuple of (optimizer_class, optimizer_parameters). """ - def __init__(self, name: str, block_list: list, optimizer_options: OptimizerOptions = None): + def __init__(self, name: str, block_list: [AbsBlock], optimizer_options: OptimizerOptions = None): super().__init__() self._name = name + self._input_dim = block_list[0].input_dim + self._output_dim = block_list[-1].output_dim self._net = nn.Sequential(*block_list) self._is_trainable = optimizer_options is not None if self._is_trainable: @@ -46,6 +51,14 @@ def __setstate__(self, dic: dict): def name(self): return self._name + @property + def input_dim(self): + return self._input_dim + + @property + def output_dim(self): + return self._output_dim + @property def is_trainable(self): return self._is_trainable @@ -80,6 +93,7 @@ class LearningModel(nn.Module): task_modules (LearningModule): LearningModule instances, each of which performs a designated task. shared_module (LearningModule): Network module that forms that shared part of the model. Defaults to None. """ + @check_chainability def __init__( self, *task_modules: LearningModule, @@ -92,31 +106,17 @@ def __init__( self._shared_module = shared_module # task_heads - self._task_modules = list(task_modules) - self._net = nn.ModuleDict({ - task_module.name: nn.Sequential(self._shared_module, task_module) if self._shared_module else task_module - for task_module in self._task_modules - }) + self._task_module_dict = nn.ModuleDict({task_module.name: task_module for task_module in task_modules}) def __getstate__(self): - shared_module = self._shared_module.copy() if self._shared_module else None - task_modules = [task_module.copy() for task_module in self._task_modules] - net = nn.ModuleDict({ - task_module.name: nn.Sequential(shared_module, task_module) if shared_module else task_module - for task_module in task_modules - }) dic = self.__dict__.copy() - dic["_shared_module"] = shared_module - dic["_task_modules"] = task_modules - dic["_net"] = net + dic["_shared_module"] = self._shared_module.copy() if self._shared_module else None + dic["_task_module_dict"] = {name: task_module.copy() for name, task_module in self._task_module_dict.items()} return dic def __setstate__(self, dic: dict): self.__dict__ = dic - def __getitem__(self, task): - return self._net[task] - @property def task_names(self) -> [str]: return self._task_names @@ -128,22 +128,24 @@ def shared_module(self): @property def is_trainable(self) -> bool: return ( - any(task_module.is_trainable for task_module in self._task_modules) or + any(task_module.is_trainable for task_module in self._task_module_dict.values()) or (self._shared_module is not None and self._shared_module.is_trainable) ) def _forward(self, inputs, task_name: str = None): - if len(self._task_modules) == 1: - task_name = self._task_modules[0].name - return self._net[task_name](inputs) + if self._shared_module: + inputs = self._shared_module(inputs) + + if len(self._task_module_dict) == 1: + return list(self._task_module_dict.values())[0](inputs) if task_name is None: - return {key: self._net[key](inputs) for key in self._task_names} + return {name: task_module(inputs) for name, task_module in self._task_module_dict.items()} if isinstance(task_name, list): - return {k: self._net[k](inputs) for k in task_name} + return {name: self._task_module_dict[name](inputs) for name in task_name} else: - return self._net[task_name](inputs) + return self._task_module_dict[task_name](inputs) def forward(self, inputs, task_name: str = None, is_training: bool = True): """Feedforward computations for the given head(s). @@ -171,7 +173,7 @@ def forward(self, inputs, task_name: str = None, is_training: bool = True): def learn(self, loss): """Use the loss to back-propagate gradients and apply them to the underlying parameters.""" - for task_module in self._task_modules: + for task_module in self._task_module_dict.values(): task_module.zero_grad() if self._shared_module is not None: self._shared_module.zero_grad() @@ -180,7 +182,7 @@ def learn(self, loss): loss.backward() # Apply gradients - for task_module in self._task_modules: + for task_module in self._task_module_dict.values(): task_module.step() if self._shared_module is not None: self._shared_module.step() diff --git a/maro/rl/models/utils.py b/maro/rl/models/utils.py new file mode 100644 index 000000000..a6875726e --- /dev/null +++ b/maro/rl/models/utils.py @@ -0,0 +1,17 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +from maro.utils.exception.rl_toolkit_exception import UnchainableModuleError + + +def check_chainability(init_func): + def decorator(self, *task_modules, shared_module=None): + if shared_module is not None: + for task_module in task_modules: + if shared_module.output_dim != task_module.input_dim: + raise UnchainableModuleError( + f"Expected input dimension {shared_module.output_dim} for {task_module.name}, " + f"got {task_module.input_dim}") + init_func(self, *task_modules, shared_module=shared_module) + + return decorator diff --git a/maro/utils/exception/error_code.py b/maro/utils/exception/error_code.py index 213918907..ee045bad5 100644 --- a/maro/utils/exception/error_code.py +++ b/maro/utils/exception/error_code.py @@ -38,9 +38,10 @@ 4001: "Unsupported Agent Mode", 4002: "Missing Shaper", 4003: "Wrong Agent Manager Mode", - 4004: "Store Misalignment Error", + 4004: "Store Misalignment", 4005: "Invalid Episode", 4006: "Infinite Training Loop", 4007: "Missing Optimizer", - 4008: "Unrecognized Task" + 4008: "Unrecognized Task", + 4009: "Unchainable Modules" } diff --git a/maro/utils/exception/rl_toolkit_exception.py b/maro/utils/exception/rl_toolkit_exception.py index a0d6a4a94..951115082 100644 --- a/maro/utils/exception/rl_toolkit_exception.py +++ b/maro/utils/exception/rl_toolkit_exception.py @@ -48,6 +48,14 @@ def __init__(self, msg: str = None): class UnrecognizedTaskError(MAROException): - """Raised when a MultiTaskLearningModel has task names that are not unrecognized by an algorithm.""" + """Raised when a LearningModel has task names that are not unrecognized by an algorithm.""" def __init__(self, msg: str = None): super().__init__(4008, msg) + + +class UnchainableModuleError(MAROException): + """Raised when the modules passed to a LearningModel have incorrect input/output dimensions that make them + unchainable. + """ + def __init__(self, msg: str = None): + super().__init__(4009, msg) From 81a0341ee7e71269e612543e7845a5fab2e8b51d Mon Sep 17 00:00:00 2001 From: ysqyang Date: Thu, 19 Nov 2020 23:28:57 +0800 Subject: [PATCH 243/337] rm __getstate__ and __setstate__ from LearningModel --- maro/rl/models/learning_model.py | 9 --------- 1 file changed, 9 deletions(-) diff --git a/maro/rl/models/learning_model.py b/maro/rl/models/learning_model.py index 7412fb988..bc287def2 100644 --- a/maro/rl/models/learning_model.py +++ b/maro/rl/models/learning_model.py @@ -108,15 +108,6 @@ def __init__( # task_heads self._task_module_dict = nn.ModuleDict({task_module.name: task_module for task_module in task_modules}) - def __getstate__(self): - dic = self.__dict__.copy() - dic["_shared_module"] = self._shared_module.copy() if self._shared_module else None - dic["_task_module_dict"] = {name: task_module.copy() for name, task_module in self._task_module_dict.items()} - return dic - - def __setstate__(self, dic: dict): - self.__dict__ = dic - @property def task_names(self) -> [str]: return self._task_names From f552ae9a8df0bed6f6f0988307898e047f8d3846 Mon Sep 17 00:00:00 2001 From: ysqyang Date: Fri, 20 Nov 2020 12:10:16 +0800 Subject: [PATCH 244/337] added noise explorer --- maro/rl/agent/simple_agent_manager.py | 1 - maro/rl/exploration/abs_explorer.py | 2 +- .../rl/exploration/epsilon_greedy_explorer.py | 11 ++- maro/rl/exploration/noise_explorer.py | 86 +++++++++++++++++++ maro/utils/exception/error_code.py | 3 +- maro/utils/exception/rl_toolkit_exception.py | 6 ++ 6 files changed, 104 insertions(+), 5 deletions(-) create mode 100644 maro/rl/exploration/noise_explorer.py diff --git a/maro/rl/agent/simple_agent_manager.py b/maro/rl/agent/simple_agent_manager.py index c29134362..dc81e7561 100644 --- a/maro/rl/agent/simple_agent_manager.py +++ b/maro/rl/agent/simple_agent_manager.py @@ -4,7 +4,6 @@ import os from abc import abstractmethod -from maro.rl.exploration.abs_explorer import AbsExplorer from maro.rl.shaping.action_shaper import ActionShaper from maro.rl.shaping.experience_shaper import ExperienceShaper from maro.rl.shaping.state_shaper import StateShaper diff --git a/maro/rl/exploration/abs_explorer.py b/maro/rl/exploration/abs_explorer.py index 43859d1f8..70cdeab74 100644 --- a/maro/rl/exploration/abs_explorer.py +++ b/maro/rl/exploration/abs_explorer.py @@ -12,7 +12,7 @@ def __init__(self): pass @abstractmethod - def load_exploration_params(self, exploration_params): + def load_exploration_params(self, **exploration_params): return NotImplementedError @abstractmethod diff --git a/maro/rl/exploration/epsilon_greedy_explorer.py b/maro/rl/exploration/epsilon_greedy_explorer.py index 01283798b..8bbf249d4 100644 --- a/maro/rl/exploration/epsilon_greedy_explorer.py +++ b/maro/rl/exploration/epsilon_greedy_explorer.py @@ -3,6 +3,8 @@ import random +from maro.utils.exception.rl_toolkit_exception import MissingExplorationParametersError + from .abs_explorer import AbsExplorer @@ -14,15 +16,20 @@ class EpsilonGreedyExplorer(AbsExplorer): """ def __init__(self, num_actions: int): super().__init__() - self._epsilon = None self._num_actions = num_actions + self._epsilon = None def __call__(self, action): assert action < self._num_actions, f"Invalid action: {action}" + if self._epsilon is None: + raise MissingExplorationParametersError( + 'Epsilon is not set. Use load_exploration_params with keyword argument "epsilon" to ' + 'load the exploration parameters first.' + ) if random.random() > self._epsilon: return action else: return random.randrange(self._num_actions) - def load_exploration_params(self, epsilon: float): + def load_exploration_params(self, *, epsilon: float): self._epsilon = epsilon diff --git a/maro/rl/exploration/noise_explorer.py b/maro/rl/exploration/noise_explorer.py new file mode 100644 index 000000000..a7ebe8696 --- /dev/null +++ b/maro/rl/exploration/noise_explorer.py @@ -0,0 +1,86 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +from abc import abstractmethod +from typing import Union + +import numpy as np + +from maro.utils.exception.rl_toolkit_exception import MissingExplorationParametersError +from .abs_explorer import AbsExplorer + + +class NoiseExplorer(AbsExplorer): + """Explorer that adds a random noise to a model-generated action.""" + def __init__( + self, + action_dim: int, + min_action: Union[float, np.ndarray] = None, + max_action: Union[float, np.ndarray] = None + ): + super().__init__() + self._action_dim = action_dim + self._min_action = min_action + self._max_action = max_action + + @abstractmethod + def load_exploration_params(self, **exploration_params): + raise NotImplementedError + + @abstractmethod + def __call__(self, action): + return NotImplementedError + + +class UniformNoiseExplorer(NoiseExplorer): + """Explorer that adds a random noise to a model-generated action sampled from a uniform distribution.""" + def __init__( + self, + action_dim: int, + min_action: Union[float, np.ndarray] = None, + max_action: Union[float, np.ndarray] = None + ): + super().__init__(action_dim, min_action, max_action) + self._noise_bound = None + + def load_exploration_params(self, *, noise_bound: float): + self._noise_bound = noise_bound + + def __call__(self, action: np.ndarray): + if self._noise_bound is None: + raise MissingExplorationParametersError( + 'Noise bound is not set. Use load_exploration_params with keyword argument "noise_bound" to ' + 'load the exploration parameters first.' + ) + action += np.random.uniform(-self._noise_bound, self._noise_bound, self._action_dim) + if self._min_action is not None or self._max_action is not None: + return np.clip(action, self._min_action, self._max_action) + else: + return action + + +class GaussianNoiseExplorer(NoiseExplorer): + """Explorer that adds a random noise to a model-generated action sampled from a Gaussian distribution.""" + def __init__( + self, + action_dim: int, + min_action: Union[float, np.ndarray] = None, + max_action: Union[float, np.ndarray] = None + ): + super().__init__(action_dim, min_action, max_action) + self._noise_scale = None + + def load_exploration_params(self, *, noise_scale: float): + self._noise_scale = noise_scale + + def __call__(self, action: np.ndarray): + if self._noise_scale is None: + raise MissingExplorationParametersError( + 'Noise scale is not set. Use load_exploration_params with keyword argument "noise_scale" to ' + 'load the exploration parameters first.' + ) + action += np.random.normal(scale=self._noise_scale, size=self._action_dim) + if self._min_action is not None or self._max_action is not None: + return np.clip(action, self._min_action, self._max_action) + else: + return action diff --git a/maro/utils/exception/error_code.py b/maro/utils/exception/error_code.py index f09dcd12f..8507ca2ac 100644 --- a/maro/utils/exception/error_code.py +++ b/maro/utils/exception/error_code.py @@ -40,5 +40,6 @@ 4003: "Wrong Agent Manager Mode", 4004: "Store Misalignment Error", 4005: "Invalid Episode", - 4006: "Infinite Training Loop" + 4006: "Infinite Training Loop", + 4010: "Missing Exploration Parameters" } diff --git a/maro/utils/exception/rl_toolkit_exception.py b/maro/utils/exception/rl_toolkit_exception.py index c35e0b897..1644ffa4d 100644 --- a/maro/utils/exception/rl_toolkit_exception.py +++ b/maro/utils/exception/rl_toolkit_exception.py @@ -39,3 +39,9 @@ class InfiniteTrainingLoopError(MAROException): """Raised when the ``SimpleLearner``'s training loop becomes infinite.""" def __init__(self, msg: str = None): super().__init__(4006, msg) + + +class MissingExplorationParametersError(MAROException): + """Raised when an explorer is called without loading the exploration parameters first.""" + def __init__(self, msg: str = None): + super().__init__(4010, msg) From f78bee0bcde2cff68132993b50419cf7f2236e4d Mon Sep 17 00:00:00 2001 From: ysqyang Date: Fri, 20 Nov 2020 12:23:10 +0800 Subject: [PATCH 245/337] formatting --- maro/rl/models/learning_model.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/maro/rl/models/learning_model.py b/maro/rl/models/learning_model.py index bc287def2..2f85abab5 100644 --- a/maro/rl/models/learning_model.py +++ b/maro/rl/models/learning_model.py @@ -94,11 +94,7 @@ class LearningModel(nn.Module): shared_module (LearningModule): Network module that forms that shared part of the model. Defaults to None. """ @check_chainability - def __init__( - self, - *task_modules: LearningModule, - shared_module: LearningModule = None - ): + def __init__(self, *task_modules: LearningModule, shared_module: LearningModule = None): super().__init__() self._task_names = [module.name for module in task_modules] From 5d011490fb9653dfa6bd785afd99f09beed17f75 Mon Sep 17 00:00:00 2001 From: ysqyang Date: Fri, 20 Nov 2020 13:22:18 +0800 Subject: [PATCH 246/337] fixed formatting --- maro/rl/exploration/noise_explorer.py | 1 + 1 file changed, 1 insertion(+) diff --git a/maro/rl/exploration/noise_explorer.py b/maro/rl/exploration/noise_explorer.py index a7ebe8696..6db9a126c 100644 --- a/maro/rl/exploration/noise_explorer.py +++ b/maro/rl/exploration/noise_explorer.py @@ -7,6 +7,7 @@ import numpy as np from maro.utils.exception.rl_toolkit_exception import MissingExplorationParametersError + from .abs_explorer import AbsExplorer From 9e010fea46b91fd8db07931bab4b8454f7260577 Mon Sep 17 00:00:00 2001 From: ysqyang Date: Mon, 23 Nov 2020 10:47:11 +0800 Subject: [PATCH 247/337] removed unnecessary comma --- maro/rl/agent/abs_agent_manager.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/maro/rl/agent/abs_agent_manager.py b/maro/rl/agent/abs_agent_manager.py index 968411b0d..9b9c85b52 100644 --- a/maro/rl/agent/abs_agent_manager.py +++ b/maro/rl/agent/abs_agent_manager.py @@ -42,7 +42,7 @@ def __init__( agent_dict: dict, state_shaper: StateShaper = None, action_shaper: ActionShaper = None, - experience_shaper: ExperienceShaper = None, + experience_shaper: ExperienceShaper = None ): self._name = name self._mode = mode From c774441dcfe0a8d85ca8f03afe594c74502256a5 Mon Sep 17 00:00:00 2001 From: ysqyang Date: Mon, 23 Nov 2020 10:47:51 +0800 Subject: [PATCH 248/337] removed unnecessary comma --- maro/rl/agent/abs_agent_manager.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/maro/rl/agent/abs_agent_manager.py b/maro/rl/agent/abs_agent_manager.py index 968411b0d..9b9c85b52 100644 --- a/maro/rl/agent/abs_agent_manager.py +++ b/maro/rl/agent/abs_agent_manager.py @@ -42,7 +42,7 @@ def __init__( agent_dict: dict, state_shaper: StateShaper = None, action_shaper: ActionShaper = None, - experience_shaper: ExperienceShaper = None, + experience_shaper: ExperienceShaper = None ): self._name = name self._mode = mode From e7522fdc9dd18cfcd195716ab92d41fdb640c2bb Mon Sep 17 00:00:00 2001 From: ysqyang Date: Mon, 23 Nov 2020 10:48:20 +0800 Subject: [PATCH 249/337] removed unnecessary comma --- maro/rl/agent/abs_agent_manager.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/maro/rl/agent/abs_agent_manager.py b/maro/rl/agent/abs_agent_manager.py index 968411b0d..9b9c85b52 100644 --- a/maro/rl/agent/abs_agent_manager.py +++ b/maro/rl/agent/abs_agent_manager.py @@ -42,7 +42,7 @@ def __init__( agent_dict: dict, state_shaper: StateShaper = None, action_shaper: ActionShaper = None, - experience_shaper: ExperienceShaper = None, + experience_shaper: ExperienceShaper = None ): self._name = name self._mode = mode From 704b2ad444b3c0f8340665210d476f8698d9454e Mon Sep 17 00:00:00 2001 From: ysqyang Date: Mon, 23 Nov 2020 23:29:56 +0800 Subject: [PATCH 250/337] fixed PR comments --- maro/rl/actor/simple_actor.py | 2 +- maro/rl/agent/abs_agent.py | 4 +- maro/rl/agent/abs_agent_manager.py | 8 ++-- maro/rl/agent/simple_agent_manager.py | 10 ++-- maro/rl/exploration/abs_explorer.py | 2 +- .../rl/exploration/epsilon_greedy_explorer.py | 19 +++----- ...schedule.py => epsilon_greedy_schedule.py} | 0 maro/rl/exploration/noise_explorer.py | 47 ++++++++++--------- maro/utils/exception/error_code.py | 3 +- maro/utils/exception/rl_toolkit_exception.py | 6 --- 10 files changed, 46 insertions(+), 55 deletions(-) rename maro/rl/exploration/{epsilon_schedule.py => epsilon_greedy_schedule.py} (100%) diff --git a/maro/rl/actor/simple_actor.py b/maro/rl/actor/simple_actor.py index 85e6b6fe3..96f78a3c3 100644 --- a/maro/rl/actor/simple_actor.py +++ b/maro/rl/actor/simple_actor.py @@ -44,7 +44,7 @@ def roll_out( # load exploration parameters: if exploration_params is not None: - self._agents.load_exploration_params(exploration_params) + self._agents.update(exploration_params) metrics, decision_event, is_done = self._env.step(None) while not is_done: diff --git a/maro/rl/agent/abs_agent.py b/maro/rl/agent/abs_agent.py index 42cbf69f8..ab03b0f64 100644 --- a/maro/rl/agent/abs_agent.py +++ b/maro/rl/agent/abs_agent.py @@ -67,9 +67,9 @@ def choose_action(self, model_state): action = self._algorithm.choose_action(model_state) return action if self._explorer is None else self._explorer(action) - def load_exploration_params(self, exploration_params): + def update(self, exploration_params): if self._explorer: - self._explorer.load_exploration_params(exploration_params) + self._explorer.update(exploration_params) @abstractmethod def train(self, *args, **kwargs): diff --git a/maro/rl/agent/abs_agent_manager.py b/maro/rl/agent/abs_agent_manager.py index 9b9c85b52..296462519 100644 --- a/maro/rl/agent/abs_agent_manager.py +++ b/maro/rl/agent/abs_agent_manager.py @@ -87,13 +87,15 @@ def train(self, *args, **kwargs): """Train the agents.""" return NotImplemented - def load_exploration_params(self, exploration_params): + def update(self, exploration_params): + # Per-agent exploration parameters if isinstance(exploration_params, dict) and exploration_params.keys() <= self.agent_dict.keys(): for agent_id, params in exploration_params.items(): - self.agent_dict[agent_id].load_exploration_params(params) + self.agent_dict[agent_id].update(params) + # Shared exploration parameters for all agents else: for agent in self.agent_dict.values(): - agent.load_exploration_params(exploration_params) + agent.update(exploration_params) def _assert_train_mode(self): if self._mode != AgentManagerMode.TRAIN and self._mode != AgentManagerMode.TRAIN_INFERENCE: diff --git a/maro/rl/agent/simple_agent_manager.py b/maro/rl/agent/simple_agent_manager.py index dc81e7561..6e3574900 100644 --- a/maro/rl/agent/simple_agent_manager.py +++ b/maro/rl/agent/simple_agent_manager.py @@ -44,16 +44,16 @@ def __init__( def choose_action(self, decision_event, snapshot_list): self._assert_inference_mode() - agent_id, state = self._state_shaper(decision_event, snapshot_list) - action = self.agent_dict[agent_id].choose_action(state) + agent_id, model_state = self._state_shaper(decision_event, snapshot_list) + model_action = self.agent_dict[agent_id].choose_action(model_state) self._transition_cache = { - "state": state, - "action": action, + "state": model_state, + "action": model_action, "reward": None, "agent_id": agent_id, "event": decision_event } - return self._action_shaper(action, decision_event, snapshot_list) + return self._action_shaper(model_action, decision_event, snapshot_list) def on_env_feedback(self, metrics): """This method records the environment-generated metrics as part of the latest transition in the trajectory. diff --git a/maro/rl/exploration/abs_explorer.py b/maro/rl/exploration/abs_explorer.py index 70cdeab74..a50e4be57 100644 --- a/maro/rl/exploration/abs_explorer.py +++ b/maro/rl/exploration/abs_explorer.py @@ -12,7 +12,7 @@ def __init__(self): pass @abstractmethod - def load_exploration_params(self, **exploration_params): + def update(self, **exploration_params): return NotImplementedError @abstractmethod diff --git a/maro/rl/exploration/epsilon_greedy_explorer.py b/maro/rl/exploration/epsilon_greedy_explorer.py index 8bbf249d4..7e846d283 100644 --- a/maro/rl/exploration/epsilon_greedy_explorer.py +++ b/maro/rl/exploration/epsilon_greedy_explorer.py @@ -14,22 +14,17 @@ class EpsilonGreedyExplorer(AbsExplorer): Args: num_actions (int): Number of all possible actions. """ - def __init__(self, num_actions: int): + def __init__(self, num_actions: int, epsilon: float = .0): super().__init__() self._num_actions = num_actions - self._epsilon = None - - def __call__(self, action): - assert action < self._num_actions, f"Invalid action: {action}" - if self._epsilon is None: - raise MissingExplorationParametersError( - 'Epsilon is not set. Use load_exploration_params with keyword argument "epsilon" to ' - 'load the exploration parameters first.' - ) + self._epsilon = epsilon + + def __call__(self, action_index: int): + assert (action_index < self._num_actions), f"Invalid action: {action_index}" if random.random() > self._epsilon: - return action + return action_index else: return random.randrange(self._num_actions) - def load_exploration_params(self, *, epsilon: float): + def update(self, *, epsilon: float): self._epsilon = epsilon diff --git a/maro/rl/exploration/epsilon_schedule.py b/maro/rl/exploration/epsilon_greedy_schedule.py similarity index 100% rename from maro/rl/exploration/epsilon_schedule.py rename to maro/rl/exploration/epsilon_greedy_schedule.py diff --git a/maro/rl/exploration/noise_explorer.py b/maro/rl/exploration/noise_explorer.py index 6db9a126c..e4b4cdf2a 100644 --- a/maro/rl/exploration/noise_explorer.py +++ b/maro/rl/exploration/noise_explorer.py @@ -6,8 +6,6 @@ import numpy as np -from maro.utils.exception.rl_toolkit_exception import MissingExplorationParametersError - from .abs_explorer import AbsExplorer @@ -25,7 +23,7 @@ def __init__( self._max_action = max_action @abstractmethod - def load_exploration_params(self, **exploration_params): + def update(self, **exploration_params): raise NotImplementedError @abstractmethod @@ -39,21 +37,20 @@ def __init__( self, action_dim: int, min_action: Union[float, np.ndarray] = None, - max_action: Union[float, np.ndarray] = None + max_action: Union[float, np.ndarray] = None, + noise_lower_bound: Union[float, np.ndarray] = .0, + noise_upper_bound: Union[float, np.ndarray] = .0 ): super().__init__(action_dim, min_action, max_action) - self._noise_bound = None + self._noise_lower_bound = noise_lower_bound + self._noise_upper_bound = noise_upper_bound - def load_exploration_params(self, *, noise_bound: float): - self._noise_bound = noise_bound + def update(self, *, noise_lower_bound: Union[float, np.ndarray], noise_upper_bound: Union[float, np.ndarray]): + self._noise_lower_bound = noise_lower_bound + self._noise_upper_bound = noise_upper_bound def __call__(self, action: np.ndarray): - if self._noise_bound is None: - raise MissingExplorationParametersError( - 'Noise bound is not set. Use load_exploration_params with keyword argument "noise_bound" to ' - 'load the exploration parameters first.' - ) - action += np.random.uniform(-self._noise_bound, self._noise_bound, self._action_dim) + action += np.random.uniform(self._noise_lower_bound, self._noise_upper_bound, self._action_dim) if self._min_action is not None or self._max_action is not None: return np.clip(action, self._min_action, self._max_action) else: @@ -66,21 +63,25 @@ def __init__( self, action_dim: int, min_action: Union[float, np.ndarray] = None, - max_action: Union[float, np.ndarray] = None + max_action: Union[float, np.ndarray] = None, + noise_mean: Union[float, np.ndarray] = .0, + noise_stddev: Union[float, np.ndarray] = .0, + is_relative_stddev: bool = False ): super().__init__(action_dim, min_action, max_action) - self._noise_scale = None + if is_relative_stddev and noise_mean != .0: + raise ValueError("Standard deviation cannot be relative if noise mean is non-zero.") + self._noise_mean = noise_mean + self._noise_stddev = noise_stddev + self._is_relative_stddev = is_relative_stddev - def load_exploration_params(self, *, noise_scale: float): - self._noise_scale = noise_scale + def update(self, *, noise_mean: Union[float, np.ndarray], noise_stddev: Union[float, np.ndarray]): + self._noise_mean = noise_mean + self._noise_stddev = noise_stddev def __call__(self, action: np.ndarray): - if self._noise_scale is None: - raise MissingExplorationParametersError( - 'Noise scale is not set. Use load_exploration_params with keyword argument "noise_scale" to ' - 'load the exploration parameters first.' - ) - action += np.random.normal(scale=self._noise_scale, size=self._action_dim) + noise = np.random.normal(loc=self._noise_mean, scale=self._noise_stddev, size=self._action_dim) + action += (noise * action) if self._is_relative_stddev else noise if self._min_action is not None or self._max_action is not None: return np.clip(action, self._min_action, self._max_action) else: diff --git a/maro/utils/exception/error_code.py b/maro/utils/exception/error_code.py index 8507ca2ac..f09dcd12f 100644 --- a/maro/utils/exception/error_code.py +++ b/maro/utils/exception/error_code.py @@ -40,6 +40,5 @@ 4003: "Wrong Agent Manager Mode", 4004: "Store Misalignment Error", 4005: "Invalid Episode", - 4006: "Infinite Training Loop", - 4010: "Missing Exploration Parameters" + 4006: "Infinite Training Loop" } diff --git a/maro/utils/exception/rl_toolkit_exception.py b/maro/utils/exception/rl_toolkit_exception.py index 1644ffa4d..c35e0b897 100644 --- a/maro/utils/exception/rl_toolkit_exception.py +++ b/maro/utils/exception/rl_toolkit_exception.py @@ -39,9 +39,3 @@ class InfiniteTrainingLoopError(MAROException): """Raised when the ``SimpleLearner``'s training loop becomes infinite.""" def __init__(self, msg: str = None): super().__init__(4006, msg) - - -class MissingExplorationParametersError(MAROException): - """Raised when an explorer is called without loading the exploration parameters first.""" - def __init__(self, msg: str = None): - super().__init__(4010, msg) From f97a302c095422b0a3bb74e820ed3e29aa3bd31a Mon Sep 17 00:00:00 2001 From: ysqyang Date: Tue, 24 Nov 2020 11:14:57 +0800 Subject: [PATCH 251/337] removed unwanted exception and imports --- maro/rl/exploration/epsilon_greedy_explorer.py | 2 -- maro/utils/exception/rl_toolkit_exception.py | 6 ------ 2 files changed, 8 deletions(-) diff --git a/maro/rl/exploration/epsilon_greedy_explorer.py b/maro/rl/exploration/epsilon_greedy_explorer.py index 7e846d283..2e80c381f 100644 --- a/maro/rl/exploration/epsilon_greedy_explorer.py +++ b/maro/rl/exploration/epsilon_greedy_explorer.py @@ -3,8 +3,6 @@ import random -from maro.utils.exception.rl_toolkit_exception import MissingExplorationParametersError - from .abs_explorer import AbsExplorer diff --git a/maro/utils/exception/rl_toolkit_exception.py b/maro/utils/exception/rl_toolkit_exception.py index c74f49477..951115082 100644 --- a/maro/utils/exception/rl_toolkit_exception.py +++ b/maro/utils/exception/rl_toolkit_exception.py @@ -59,9 +59,3 @@ class UnchainableModuleError(MAROException): """ def __init__(self, msg: str = None): super().__init__(4009, msg) - - -class MissingExplorationParametersError(MAROException): - """Raised when an explorer is called without loading the exploration parameters first.""" - def __init__(self, msg: str = None): - super().__init__(4010, msg) From 18e3366ee942f6d7d1d73b914c2a2d737fa19564 Mon Sep 17 00:00:00 2001 From: ysqyang Date: Tue, 24 Nov 2020 11:17:20 +0800 Subject: [PATCH 252/337] removed unwanted exception and imports --- maro/rl/exploration/epsilon_greedy_explorer.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/maro/rl/exploration/epsilon_greedy_explorer.py b/maro/rl/exploration/epsilon_greedy_explorer.py index 7e846d283..2e80c381f 100644 --- a/maro/rl/exploration/epsilon_greedy_explorer.py +++ b/maro/rl/exploration/epsilon_greedy_explorer.py @@ -3,8 +3,6 @@ import random -from maro.utils.exception.rl_toolkit_exception import MissingExplorationParametersError - from .abs_explorer import AbsExplorer From 4f612f5d15952bbc6cfdb8f8ec786951c04ff10d Mon Sep 17 00:00:00 2001 From: ysqyang Date: Tue, 24 Nov 2020 14:00:55 +0800 Subject: [PATCH 253/337] fixed a bug --- maro/rl/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/maro/rl/__init__.py b/maro/rl/__init__.py index 96bf466fc..2f455c577 100644 --- a/maro/rl/__init__.py +++ b/maro/rl/__init__.py @@ -18,7 +18,7 @@ ) from maro.rl.exploration.abs_explorer import AbsExplorer from maro.rl.exploration.epsilon_greedy_explorer import EpsilonGreedyExplorer -from maro.rl.exploration.epsilon_schedule import linear_epsilon_schedule, two_phase_linear_epsilon_schedule +from maro.rl.exploration.epsilon_greedy_schedule import linear_epsilon_schedule, two_phase_linear_epsilon_schedule from maro.rl.learner.abs_learner import AbsLearner from maro.rl.learner.simple_learner import SimpleLearner from maro.rl.models.fc_block import FullyConnectedBlock From 85557a87b52eae499b078c291d7c60e3953b171d Mon Sep 17 00:00:00 2001 From: ysqyang Date: Wed, 25 Nov 2020 23:14:32 +0800 Subject: [PATCH 254/337] fixed PR comments --- examples/cim/dqn/config.yml | 2 +- examples/cim/dqn/dist_actor.py | 2 +- examples/cim/dqn/dist_learner.py | 9 +- examples/cim/dqn/single_process_launcher.py | 27 ++-- maro/rl/__init__.py | 13 +- maro/rl/agent/abs_agent_manager.py | 2 +- .../abs_early_stopping_checker.py | 54 +++++--- .../simple_early_stopping_checker.py | 36 +++--- .../exploration/abs_exploration_scheduler.py | 34 +++++ .../rl/exploration/epsilon_greedy_schedule.py | 51 -------- .../exploration/epsilon_greedy_scheduler.py | 70 +++++++++++ maro/rl/exploration/noise_explorer.py | 8 +- maro/rl/learner/simple_learner.py | 117 ++++++------------ 13 files changed, 227 insertions(+), 198 deletions(-) create mode 100644 maro/rl/exploration/abs_exploration_scheduler.py delete mode 100644 maro/rl/exploration/epsilon_greedy_schedule.py create mode 100644 maro/rl/exploration/epsilon_greedy_scheduler.py diff --git a/examples/cim/dqn/config.yml b/examples/cim/dqn/config.yml index a07eca1ba..39082c2b0 100644 --- a/examples/cim/dqn/config.yml +++ b/examples/cim/dqn/config.yml @@ -3,8 +3,8 @@ env: topology: "toy.4p_ssdd_l0.0" durations: 1120 main_loop: + max_episode: 500 exploration: - max_ep: 500 split_ep: 250 start_eps: 0.4 mid_eps: 0.32 diff --git a/examples/cim/dqn/dist_actor.py b/examples/cim/dqn/dist_actor.py index 0822b2060..2357f0822 100644 --- a/examples/cim/dqn/dist_actor.py +++ b/examples/cim/dqn/dist_actor.py @@ -11,7 +11,7 @@ from components.experience_shaper import TruncatedExperienceShaper from components.state_shaper import CIMStateShaper -from maro.rl import ActorWorker, AgentManagerMode, EpsilonGreedyExplorer, KStepExperienceShaper, SimpleActor +from maro.rl import ActorWorker, AgentManagerMode, KStepExperienceShaper, SimpleActor from maro.simulator import Env from maro.utils import convert_dottable diff --git a/examples/cim/dqn/dist_learner.py b/examples/cim/dqn/dist_learner.py index 283068e8d..9e8f8ebea 100644 --- a/examples/cim/dqn/dist_learner.py +++ b/examples/cim/dqn/dist_learner.py @@ -7,8 +7,8 @@ from components.config import set_input_dim from maro.rl import ( - ActorProxy, AgentManagerMode, EpsilonGreedyExplorer, SimpleLearner, concat_experiences_by_agent, - two_phase_linear_epsilon_schedule + ActorProxy, AgentManagerMode, ExplorationOptions, SimpleLearner, concat_experiences_by_agent, + TwoPhaseLinearEpsilonScheduler ) from maro.simulator import Env from maro.utils import Logger, convert_dottable @@ -38,13 +38,14 @@ def launch(config, distributed_config): "max_retries": 15 } - exploration_schedule = two_phase_linear_epsilon_schedule(**config.main_loop.exploration) learner = SimpleLearner( agent_manager=agent_manager, actor=ActorProxy(proxy_params=proxy_params, experience_collecting_func=concat_experiences_by_agent), + max_episode=config.main_loop.max_episode, + exploration_options=ExplorationOptions(TwoPhaseLinearEpsilonScheduler, config.main_loop.exploration), logger=Logger("distributed_cim_learner", auto_timestamp=False) ) - learner.learn_with_exploration_schedule(exploration_schedule) + learner.learn() learner.test() learner.dump_models(os.path.join(os.getcwd(), "models")) learner.exit() diff --git a/examples/cim/dqn/single_process_launcher.py b/examples/cim/dqn/single_process_launcher.py index 6d3c678f9..fa842ecb0 100644 --- a/examples/cim/dqn/single_process_launcher.py +++ b/examples/cim/dqn/single_process_launcher.py @@ -13,8 +13,8 @@ from components.state_shaper import CIMStateShaper from maro.rl import ( - AgentManagerMode, EpsilonGreedyExplorer, KStepExperienceShaper, MaxDeltaEarlyStoppingChecker, SimpleActor, - SimpleEarlyStoppingChecker, SimpleLearner, two_phase_linear_epsilon_schedule + AgentManagerMode, ExplorationOptions, KStepExperienceShaper, MaxDeltaEarlyStoppingChecker, + SimpleActor, SimpleEarlyStoppingChecker, SimpleLearner, TwoPhaseLinearEpsilonScheduler, ) from maro.simulator import Env from maro.utils import Logger, convert_dottable @@ -52,31 +52,34 @@ def launch(config): ) # Step 4: Create an actor and a learner to start the training process. + def metric_func(performance): + return 1 - performance["container_shortage"] / performance["order_requirements"] + perf_checker = SimpleEarlyStoppingChecker( last_k=config.main_loop.early_stopping.last_k, threshold=config.main_loop.early_stopping.perf_threshold, - measure_func=lambda vals: mean(vals) + warmup_ep=config.main_loop.early_stopping.warmup_ep, + metric_func=metric_func, + measure_func=lambda metric_series : mean(metric_series) ) perf_stability_checker = MaxDeltaEarlyStoppingChecker( last_k=config.main_loop.early_stopping.last_k, - threshold=config.main_loop.early_stopping.perf_stability_threshold + threshold=config.main_loop.early_stopping.perf_stability_threshold, + warmup_ep=config.main_loop.early_stopping.warmup_ep, + metric_func=metric_func ) - combined_checker = perf_checker & perf_stability_checker - actor = SimpleActor(env, agent_manager) learner = SimpleLearner( agent_manager=agent_manager, actor=actor, + max_episode=config.main_loop.max_episode, + exploration_options=ExplorationOptions(TwoPhaseLinearEpsilonScheduler, config.main_loop.exploration), + early_stopping_checker=perf_checker & perf_stability_checker, logger=Logger("single_host_cim_learner", auto_timestamp=False) ) - learner.learn_with_exploration_schedule( - two_phase_linear_epsilon_schedule(**config.main_loop.exploration), - early_stopping_checker=combined_checker, - warmup_ep=config.main_loop.early_stopping.warmup_ep, - early_stopping_metric_func=lambda x: 1 - x["container_shortage"] / x["order_requirements"], - ) + learner.learn() learner.test() learner.dump_models(os.path.join(os.getcwd(), "models")) diff --git a/maro/rl/__init__.py b/maro/rl/__init__.py index 2f455c577..41c7d56bb 100644 --- a/maro/rl/__init__.py +++ b/maro/rl/__init__.py @@ -16,11 +16,12 @@ from maro.rl.early_stopping.simple_early_stopping_checker import ( MaxDeltaEarlyStoppingChecker, RSDEarlyStoppingChecker, SimpleEarlyStoppingChecker ) +from maro.rl.exploration.abs_exploration_scheduler import AbsExplorationScheduler, NullExplorationScheduler from maro.rl.exploration.abs_explorer import AbsExplorer from maro.rl.exploration.epsilon_greedy_explorer import EpsilonGreedyExplorer -from maro.rl.exploration.epsilon_greedy_schedule import linear_epsilon_schedule, two_phase_linear_epsilon_schedule +from maro.rl.exploration.epsilon_greedy_scheduler import LinearEpsilonScheduler, TwoPhaseLinearEpsilonScheduler from maro.rl.learner.abs_learner import AbsLearner -from maro.rl.learner.simple_learner import SimpleLearner +from maro.rl.learner.simple_learner import ExplorationOptions, SimpleLearner from maro.rl.models.fc_block import FullyConnectedBlock from maro.rl.models.learning_model import MultiHeadLearningModel, SingleHeadLearningModel from maro.rl.shaping.abs_shaper import AbsShaper @@ -51,10 +52,13 @@ 'DQNHyperParams', 'EpsilonGreedyExplorer', 'ExperienceShaper', + 'ExplorationOptions', 'FullyConnectedBlock', 'KStepExperienceShaper', + 'LinearEpsilonScheduler', 'MaxDeltaEarlyStoppingChecker', 'MultiHeadLearningModel', + 'NullExplorationScheduler', 'OverwriteType', 'RSDEarlyStoppingChecker', 'SimpleActor', @@ -63,8 +67,7 @@ 'SimpleLearner', 'SingleHeadLearningModel', 'StateShaper', + 'TwoPhaseLinearEpsilonScheduler', 'concat_experiences_by_agent', - 'linear_epsilon_schedule', - 'merge_experiences_with_trajectory_boundaries', - 'two_phase_linear_epsilon_schedule' + 'merge_experiences_with_trajectory_boundaries' ] diff --git a/maro/rl/agent/abs_agent_manager.py b/maro/rl/agent/abs_agent_manager.py index 296462519..ae076c152 100644 --- a/maro/rl/agent/abs_agent_manager.py +++ b/maro/rl/agent/abs_agent_manager.py @@ -87,7 +87,7 @@ def train(self, *args, **kwargs): """Train the agents.""" return NotImplemented - def update(self, exploration_params): + def update_exploration_params(self, exploration_params): # Per-agent exploration parameters if isinstance(exploration_params, dict) and exploration_params.keys() <= self.agent_dict.keys(): for agent_id, params in exploration_params.items(): diff --git a/maro/rl/early_stopping/abs_early_stopping_checker.py b/maro/rl/early_stopping/abs_early_stopping_checker.py index d5d26d115..cecab94eb 100644 --- a/maro/rl/early_stopping/abs_early_stopping_checker.py +++ b/maro/rl/early_stopping/abs_early_stopping_checker.py @@ -2,6 +2,8 @@ # Licensed under the MIT license. from abc import ABC, abstractmethod +from collections import deque +from typing import Callable class AbsEarlyStoppingChecker(ABC): @@ -11,26 +13,42 @@ class AbsEarlyStoppingChecker(ABC): last_k (int): Number of the latest metric values to check for early stopping. threshold (float): The threshold value against which a user-defined measure is compared to determine whether early-stopping should be triggered. + warmup_ep (int): Number of episodes before early stopping checker takes effect. + metric_func (Callable): Function to extract early stopping metric from a performance record. """ - def __init__(self, last_k, threshold): + def __init__(self, last_k: int, threshold: float, warmup_ep: int, metric_func: Callable): super().__init__() self._last_k = last_k self._threshold = threshold + self._metric_func = metric_func + self._warmup_ep = warmup_ep + self._metric_series = deque() + self._ep_count = 0 - def is_triggered(self, metric_series): - return len(metric_series) >= self._last_k - - @abstractmethod - def __call__(self, metric_series) -> bool: - """Check whether the early stopping condition (defined in the class) is met. + def update(self, performance) -> bool: + """Update with the latest performance record and check whether an early stopping condition is met. Args: - metric_series: History of performances (from actors) used to check whether the early stopping - condition is satisfied. + performance: Performance record from the latest roll-out episode. Returns: A boolean value indicating whether early stopping should be triggered. """ + self._ep_count += 1 + if isinstance(performance, list): + self._metric_series.extend(map(self._metric_func, (perf for _, perf in performance))) + else: + self._metric_series.append(self._metric_func(performance)) + if self._ep_count < self._warmup_ep or len(self._metric_series) < self._last_k: + return False + + while len(self._metric_series) > self._last_k: + self._metric_series.popleft() + + return self.check() + + @abstractmethod + def check(self) -> bool: return NotImplemented def __or__(self, other_checker): @@ -43,8 +61,8 @@ def __init__(self, checker, other): self._checker = checker self._other_checker = other - def __call__(self, metric_series) -> bool: - return self._checker(metric_series) or self._other_checker(metric_series) + def push(self, performance) -> bool: + return self._checker.update(performance) or self._other_checker.update(performance) return OrChecker(self, other_checker) @@ -59,8 +77,10 @@ def __init__(self, checker, other): self._checker = checker self._other_checker = other - def __call__(self, metric_series) -> bool: - return self._checker(metric_series) and self._other_checker(metric_series) + def push(self, performance) -> bool: + result = self._checker.update(performance) + other_result = self._other_checker.update(performance) + return result and other_result return AndChecker(self, other_checker) @@ -74,8 +94,8 @@ def __init__(self, checker, other): self._checker = checker self._other_checker = other - def __call__(self, metric_series) -> bool: - return self._checker(metric_series) ^ self._other_checker(metric_series) + def __call__(self, performance) -> bool: + return self._checker.update(performance) ^ self._other_checker.update(performance) return XorChecker(self, other_checker) @@ -89,7 +109,7 @@ def __init__(self, checker): super().__init__() self._checker = checker - def __call__(self, metric_series) -> bool: - return not self._checker(metric_series) + def __call__(self, performance) -> bool: + return not self._checker.update(performance) return InverseChecker(self) diff --git a/maro/rl/early_stopping/simple_early_stopping_checker.py b/maro/rl/early_stopping/simple_early_stopping_checker.py index 2232d2b46..ab445d79f 100644 --- a/maro/rl/early_stopping/simple_early_stopping_checker.py +++ b/maro/rl/early_stopping/simple_early_stopping_checker.py @@ -13,25 +13,25 @@ class SimpleEarlyStoppingChecker(AbsEarlyStoppingChecker): The measure is obtained by applying a user-defined measure function to the last k metric values. The measure function must take a list as input and output a single number. """ - def __init__(self, last_k, threshold, measure_func: Callable[[list], Union[int, float]]): - super().__init__(last_k, threshold) + def __init__( + self, + last_k: int, + threshold: float, + warmup_ep: int, + metric_func: Callable, + measure_func: Callable[[list], Union[int, float]] + ): + super().__init__(last_k, threshold, warmup_ep, metric_func) self._measure_func = measure_func - def __call__(self, metric_series: list): - if not self.is_triggered(metric_series): - return False - else: - return self._measure_func(metric_series[-self._last_k:]) >= self._threshold + def check(self): + return self._measure_func(self._metric_series) >= self._threshold class RSDEarlyStoppingChecker(AbsEarlyStoppingChecker): """Early stopping checker based on the mean and standard deviation of the last k metric values.""" - def __call__(self, metric_series: list): - if not self.is_triggered(metric_series): - return False - else: - metric_series = metric_series[-self._last_k:] - return stdev(metric_series) / mean(metric_series) < self._threshold + def check(self): + return stdev(self._metric_series) / mean(self._metric_series) < self._threshold class MaxDeltaEarlyStoppingChecker(AbsEarlyStoppingChecker): @@ -40,10 +40,6 @@ class MaxDeltaEarlyStoppingChecker(AbsEarlyStoppingChecker): The relative change is defined as |m(i+1) - m(i)| / m[i]. The maximum of the last k-1 changes in the metric series is compared with the threshold to determine if early stopping should be triggered. """ - def __call__(self, metric_series: list): - if not self.is_triggered(metric_series): - return False - else: - metric_series = metric_series[-self._last_k:] - max_delta = max(abs(val2 - val1) / val1 for val1, val2 in zip(metric_series, metric_series[1:])) - return max_delta < self._threshold + def check(self): + max_delta = max(abs(val2 - val1) / val1 for val1, val2 in zip(self._metric_series, self._metric_series[1:])) + return max_delta < self._threshold diff --git a/maro/rl/exploration/abs_exploration_scheduler.py b/maro/rl/exploration/abs_exploration_scheduler.py new file mode 100644 index 000000000..31f9895de --- /dev/null +++ b/maro/rl/exploration/abs_exploration_scheduler.py @@ -0,0 +1,34 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +from abc import ABC, abstractmethod + + +class AbsExplorationScheduler(ABC): + """Scheduler that generates exploration parameters for each episode. + + Args: + max_ep (int): Maximum number of episodes to be run. + """ + def __init__(self, max_ep: int): + self._max_ep = max_ep + self._current_ep = 0 + + def __iter__(self): + return self + + @abstractmethod + def __next__(self): + raise NotImplementedError + + @property + def current_ep(self): + return self._current_ep + + +class NullExplorationScheduler(AbsExplorationScheduler): + """Dummy scheduler that generates.""" + def __next__(self): + if self._current_ep == self._max_ep: + raise StopIteration + self._current_ep += 1 diff --git a/maro/rl/exploration/epsilon_greedy_schedule.py b/maro/rl/exploration/epsilon_greedy_schedule.py deleted file mode 100644 index 4f2b063ce..000000000 --- a/maro/rl/exploration/epsilon_greedy_schedule.py +++ /dev/null @@ -1,51 +0,0 @@ -# Copyright (c) Microsoft Corporation. -# Licensed under the MIT license. - -from maro.utils.exception.rl_toolkit_exception import InvalidEpisodeError - - -def linear_epsilon_schedule(max_ep: int, start_eps: float, end_eps: float = .0): - """Linear exploration rate generator for epsilon-greedy exploration. - - Args: - max_ep (int): Maximum number of episodes to run. - start_eps (float): The exploration rate for the first episode. - end_eps (float): The exploration rate for the last episode. Defaults to zero. - - """ - if max_ep <= 0: - raise InvalidEpisodeError("max_ep must be a positive integer.") - current_eps = start_eps - eps_delta = (end_eps - start_eps) / (max_ep - 1) - - for ep in range(max_ep): - yield current_eps - current_eps += eps_delta - - -def two_phase_linear_epsilon_schedule( - max_ep: int, split_ep: float, start_eps: float, mid_eps: float, end_eps: float = .0 -): - """Exploration schedule comprised of two linear schedules joined together for epsilon-greedy exploration. - - Args: - max_ep (int): Maximum number of episodes to run. - split_ep (float): The episode where the switch from the first linear schedule to the second occurs. - start_eps (float): Exploration rate for the first episode. - mid_eps (float): The exploration rate where the switch from the first linear schedule to the second occurs. - In other words, this is the exploration rate where the first linear schedule ends and the second begins. - end_eps (float): Exploration rate for the last episode. Defaults to zero. - - Returns: - An iterator over the series of exploration rates from episode 0 to ``max_ep`` - 1. - """ - if max_ep <= 0: - raise InvalidEpisodeError("max_ep must be a positive integer.") - if split_ep <= 0 or split_ep >= max_ep: - raise ValueError("split_ep must be between 0 and max_ep - 1.") - current_eps = start_eps - eps_delta_phase_1 = (mid_eps - start_eps) / split_ep - eps_delta_phase_2 = (end_eps - mid_eps) / (max_ep - split_ep - 1) - for ep in range(max_ep): - yield current_eps - current_eps += eps_delta_phase_1 if ep < split_ep else eps_delta_phase_2 diff --git a/maro/rl/exploration/epsilon_greedy_scheduler.py b/maro/rl/exploration/epsilon_greedy_scheduler.py new file mode 100644 index 000000000..4b73651ea --- /dev/null +++ b/maro/rl/exploration/epsilon_greedy_scheduler.py @@ -0,0 +1,70 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +from maro.utils.exception.rl_toolkit_exception import InvalidEpisodeError + +from .abs_exploration_scheduler import AbsExplorationScheduler + + +class LinearEpsilonScheduler(AbsExplorationScheduler): + """Linear exploration rate generator for epsilon-greedy exploration. + + Args: + max_ep (int): Maximum number of episodes to run. + start_eps (float): The exploration rate for the first episode. + end_eps (float): The exploration rate for the last episode. Defaults to zero. + + """ + def __init__(self, max_ep: int, start_eps: float, end_eps: float = .0): + if max_ep <= 0: + raise InvalidEpisodeError("max_ep must be a positive integer.") + super().__init__(max_ep) + self._start_eps = start_eps + self._end_eps = end_eps + self._current_eps = start_eps + self._eps_delta = (self._end_eps - self._start_eps) / (self._max_ep - 1) + + def __next__(self): + if self._current_ep == self._max_ep: + raise StopIteration + eps = self._current_eps + self._current_ep += 1 + self._current_eps += self._eps_delta + return eps + + +class TwoPhaseLinearEpsilonScheduler(AbsExplorationScheduler): + """Exploration schedule comprised of two linear schedules joined together for epsilon-greedy exploration. + + Args: + max_ep (int): Maximum number of episodes to run. + split_ep (float): The episode where the switch from the first linear schedule to the second occurs. + start_eps (float): Exploration rate for the first episode. + mid_eps (float): The exploration rate where the switch from the first linear schedule to the second occurs. + In other words, this is the exploration rate where the first linear schedule ends and the second begins. + end_eps (float): Exploration rate for the last episode. Defaults to zero. + + Returns: + An iterator over the series of exploration rates from episode 0 to ``max_ep`` - 1. + """ + def __init__(self, max_ep: int, split_ep: float, start_eps: float, mid_eps: float, end_eps: float = .0): + if max_ep <= 0: + raise InvalidEpisodeError("max_ep must be a positive integer.") + if split_ep <= 0 or split_ep >= max_ep: + raise ValueError("split_ep must be between 0 and max_ep - 1.") + super().__init__(max_ep) + self._split_ep = split_ep + self._start_eps = start_eps + self._mid_eps = mid_eps + self._end_eps = end_eps + self._current_eps = start_eps + self._eps_delta_1 = (mid_eps - start_eps) / split_ep + self._eps_delta_2 = (end_eps - mid_eps) / (max_ep - split_ep - 1) + + def __next__(self): + if self._current_ep == self._max_ep: + raise StopIteration + eps = self._current_eps + self._current_ep += 1 + self._current_eps += self._eps_delta_1 if self._current_ep < self._split_ep else self._eps_delta_2 + return eps diff --git a/maro/rl/exploration/noise_explorer.py b/maro/rl/exploration/noise_explorer.py index e4b4cdf2a..91d7623c6 100644 --- a/maro/rl/exploration/noise_explorer.py +++ b/maro/rl/exploration/noise_explorer.py @@ -66,14 +66,14 @@ def __init__( max_action: Union[float, np.ndarray] = None, noise_mean: Union[float, np.ndarray] = .0, noise_stddev: Union[float, np.ndarray] = .0, - is_relative_stddev: bool = False + is_relative: bool = False ): super().__init__(action_dim, min_action, max_action) - if is_relative_stddev and noise_mean != .0: + if is_relative and noise_mean != .0: raise ValueError("Standard deviation cannot be relative if noise mean is non-zero.") self._noise_mean = noise_mean self._noise_stddev = noise_stddev - self._is_relative_stddev = is_relative_stddev + self._is_relative = is_relative def update(self, *, noise_mean: Union[float, np.ndarray], noise_stddev: Union[float, np.ndarray]): self._noise_mean = noise_mean @@ -81,7 +81,7 @@ def update(self, *, noise_mean: Union[float, np.ndarray], noise_stddev: Union[fl def __call__(self, action: np.ndarray): noise = np.random.normal(loc=self._noise_mean, scale=self._noise_stddev, size=self._action_dim) - action += (noise * action) if self._is_relative_stddev else noise + action += (noise * action) if self._is_relative else noise if self._min_action is not None or self._max_action is not None: return np.clip(action, self._min_action, self._max_action) else: diff --git a/maro/rl/learner/simple_learner.py b/maro/rl/learner/simple_learner.py index f35c0b987..ed3f42713 100644 --- a/maro/rl/learner/simple_learner.py +++ b/maro/rl/learner/simple_learner.py @@ -2,17 +2,23 @@ # Licensed under the MIT license. import sys -from typing import Callable, Iterator, Union +from collections import namedtuple +from typing import Union from maro.rl.actor.simple_actor import SimpleActor from maro.rl.agent.simple_agent_manager import SimpleAgentManager from maro.rl.dist_topologies.single_learner_multi_actor_sync_mode import ActorProxy +from maro.rl.early_stopping.abs_early_stopping_checker import AbsEarlyStoppingChecker +from maro.rl.exploration.abs_exploration_scheduler import NullExplorationScheduler from maro.utils import DummyLogger, Logger from maro.utils.exception.rl_toolkit_exception import InfiniteTrainingLoopError, InvalidEpisodeError from .abs_learner import AbsLearner +ExplorationOptions = namedtuple("ExplorationOptions", ["cls", "params"]) + + class SimpleLearner(AbsLearner): """A simple implementation of ``AbsLearner``. @@ -20,38 +26,22 @@ class SimpleLearner(AbsLearner): agent_manager (AbsAgentManager): An AgentManager instance that manages all agents. actor (SimpleActor or ActorProxy): An SimpleActor or ActorProxy instance responsible for performing roll-outs (environment sampling). + max_episode (int): number of episodes to be run. If -1, the training loop will run forever unless + an ``early_stopping_checker`` is provided and the early stopping condition is met. + exploration_options (ExplorationOptions): Exploration scheduler class and parameters. Defaults to None. + early_stopping_checker (EarlyStoppingOptions): Early stopping checker that checks the performance history to + determine whether early stopping condition is satisfied. Defaults to None. logger (Logger): Used to log important messages. """ def __init__( self, agent_manager: SimpleAgentManager, actor: Union[SimpleActor, ActorProxy], - logger: Logger = DummyLogger() - ): - super().__init__() - self._agent_manager = agent_manager - self._actor = actor - self._logger = logger - self._performance_history = [] - - def learn( - self, max_episode: int, - early_stopping_checker: Callable = None, - warmup_ep: int = None, - early_stopping_metric_func: Callable = None, + exploration_options: ExplorationOptions = None, + early_stopping_checker: AbsEarlyStoppingChecker = None, + logger: Logger = DummyLogger() ): - """Main loop for collecting experiences from the actor and using them to update policies. - - Args: - max_episode (int): number of episodes to be run. If -1, the training loop will run forever unless - an ``early_stopping_checker`` is provided and the early stopping condition is met. - early_stopping_checker (Callable): A Callable object to determine whether the training loop should be - terminated based on the latest performances. Defaults to None. - warmup_ep (int): Episode from which early stopping check is initiated. Defaults to None. - early_stopping_metric_func (Callable): A function to extract the metric from a performance record - for early stopping checking. Defaults to None. - """ if max_episode < -1: raise InvalidEpisodeError("max_episode can only be a non-negative integer or -1.") if max_episode == -1 and early_stopping_checker is None: @@ -59,79 +49,42 @@ def learn( "The training loop will run forever since neither maximum episode nor early stopping checker " "is provided. " ) - if early_stopping_checker is not None: - assert early_stopping_metric_func is not None, \ - "early_stopping_metric_func cannot be None if early_stopping_checker is provided." - - episode, metric_series = 0, [] - while max_episode == -1 or episode < max_episode: - performance, exp_by_agent = self._actor.roll_out( - model_dict=None if self._is_shared_agent_instance() else self._agent_manager.dump_models() - ) - self._logger.info(f"ep {episode} - performance: {performance}") - latest = [perf for _, perf in performance] if isinstance(performance, list) else [performance] - if early_stopping_checker is not None: - metric_series.extend(map(early_stopping_metric_func, latest)) - if warmup_ep is None or episode >= warmup_ep and early_stopping_checker(metric_series): - self._logger.info("Early stopping condition hit. Training complete.") - break - self._agent_manager.train(exp_by_agent) - episode += 1 + super().__init__() + self._agent_manager = agent_manager + self._actor = actor + self._max_episode = max_episode + if exploration_options is None: + self._exploration_scheduler = NullExplorationScheduler(max_episode) + else: + self._exploration_scheduler = exploration_options.cls(max_ep=max_episode, **exploration_options.params) + self._early_stopping_checker = early_stopping_checker + self._logger = logger + self._performance_history = [] - def learn_with_exploration_schedule( - self, - exploration_schedule: Union[Iterator, dict], - early_stopping_checker: Callable = None, - warmup_ep: int = None, - early_stopping_metric_func: Callable = None, - ): - """Main loop for collecting experiences from the actor and using them to update policies. - - Args: - exploration_schedule (Union[Iterator, dict]): Explorations schedules for the underlying agents. If it is - a dictionary, the exploration schedule will be registered on a per-agent basis based on agent ID's. - If it is a single iterator object, the exploration schedule will be registered for all agents. - early_stopping_checker (Callable): A Callable object to determine whether the training loop should be - terminated based on the latest performances. Defaults to None. - warmup_ep (int): Episode from which early stopping check is initiated. Defaults to None. - early_stopping_metric_func (Callable): A function to extract the metric from a performance record - for early stopping checking. Defaults to None. - """ - if early_stopping_checker is not None: - assert early_stopping_metric_func is not None, \ - "early_stopping_metric_func cannot be None if early_stopping_checker is provided." - - episode, metric_series = 0, [] + def learn(self): + """Main loop for collecting experiences from the actor and using them to update policies.""" while True: try: - if isinstance(exploration_schedule, dict): - exploration_params = { - agent_id: next(schedule) for agent_id, schedule in exploration_schedule.items() - } - else: - exploration_params = next(exploration_schedule) - + exploration_params = next(self._exploration_scheduler) performance, exp_by_agent = self._actor.roll_out( model_dict=None if self._is_shared_agent_instance() else self._agent_manager.dump_models(), exploration_params=exploration_params ) self._logger.info( - f"ep {episode} - performance: {performance}, exploration_params: {exploration_params}" + f"ep {self._exploration_scheduler.current_episode} - " + f"performance: {performance}, exploration_params: {exploration_params}" ) # Early stopping checking latest = [perf for _, perf in performance] if isinstance(performance, list) else [performance] - if early_stopping_checker is not None: - metric_series.extend(map(early_stopping_metric_func, latest)) - if warmup_ep is None or episode >= warmup_ep and early_stopping_checker(metric_series): - self._logger.info("Early stopping condition hit. Training complete.") - break + if self._early_stopping_checker is not None and self._early_stopping_checker.update(latest): + self._logger.info("Early stopping condition hit. Training complete.") + break except StopIteration: - self._logger.info(f"Maximum number of episodes ({episode}) reached. Training complete.") + self._logger.info(f"Maximum number of episodes ({self._max_episode}) reached. Training complete.") break - episode += 1 self._agent_manager.train(exp_by_agent) def test(self): From e7fde355a8b0eab4bd5a50a08b016e1fbed75d91 Mon Sep 17 00:00:00 2001 From: ysqyang Date: Wed, 25 Nov 2020 23:16:26 +0800 Subject: [PATCH 255/337] fixed a bug --- maro/rl/actor/simple_actor.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/maro/rl/actor/simple_actor.py b/maro/rl/actor/simple_actor.py index 96f78a3c3..935d6c5c6 100644 --- a/maro/rl/actor/simple_actor.py +++ b/maro/rl/actor/simple_actor.py @@ -44,7 +44,7 @@ def roll_out( # load exploration parameters: if exploration_params is not None: - self._agents.update(exploration_params) + self._agents.update_exploration_params(exploration_params) metrics, decision_event, is_done = self._env.step(None) while not is_done: From c3ea72f1a54218c9e8c1eab917cee7f290f1968e Mon Sep 17 00:00:00 2001 From: ysqyang Date: Wed, 25 Nov 2020 23:30:53 +0800 Subject: [PATCH 256/337] fixed a bug --- maro/rl/agent/abs_agent_manager.py | 4 +-- .../single_learner_multi_actor_sync_mode.py | 30 +++++++++---------- .../exploration/epsilon_greedy_scheduler.py | 4 +-- 3 files changed, 19 insertions(+), 19 deletions(-) diff --git a/maro/rl/agent/abs_agent_manager.py b/maro/rl/agent/abs_agent_manager.py index ae076c152..b5bb452cf 100644 --- a/maro/rl/agent/abs_agent_manager.py +++ b/maro/rl/agent/abs_agent_manager.py @@ -91,11 +91,11 @@ def update_exploration_params(self, exploration_params): # Per-agent exploration parameters if isinstance(exploration_params, dict) and exploration_params.keys() <= self.agent_dict.keys(): for agent_id, params in exploration_params.items(): - self.agent_dict[agent_id].update(params) + self.agent_dict[agent_id].update(**params) # Shared exploration parameters for all agents else: for agent in self.agent_dict.values(): - agent.update(exploration_params) + agent.update(**exploration_params) def _assert_train_mode(self): if self._mode != AgentManagerMode.TRAIN and self._mode != AgentManagerMode.TRAIN_INFERENCE: diff --git a/maro/rl/dist_topologies/single_learner_multi_actor_sync_mode.py b/maro/rl/dist_topologies/single_learner_multi_actor_sync_mode.py index 538d95ab3..4b11cebef 100644 --- a/maro/rl/dist_topologies/single_learner_multi_actor_sync_mode.py +++ b/maro/rl/dist_topologies/single_learner_multi_actor_sync_mode.py @@ -55,23 +55,23 @@ def roll_out( payload={PayloadKey.DONE: True} ) return None, None - else: - payloads = [(peer, {PayloadKey.MODEL: model_dict, - PayloadKey.EXPLORATION_PARAMS: exploration_params, - PayloadKey.RETURN_DETAILS: return_details}) - for peer in self._proxy.peers_name["actor"]] - # TODO: double check when ack enable - replies = self._proxy.scatter( - tag=MessageTag.ROLLOUT, - session_type=SessionType.TASK, - destination_payload_list=payloads - ) - performance = [(msg.source, msg.payload[PayloadKey.PERFORMANCE]) for msg in replies] - details_by_source = {msg.source: msg.payload[PayloadKey.DETAILS] for msg in replies} - details = self._experience_collecting_func(details_by_source) if return_details else None + payloads = [(peer, {PayloadKey.MODEL: model_dict, + PayloadKey.EXPLORATION_PARAMS: exploration_params, + PayloadKey.RETURN_DETAILS: return_details}) + for peer in self._proxy.peers_name["actor"]] + # TODO: double check when ack enable + replies = self._proxy.scatter( + tag=MessageTag.ROLLOUT, + session_type=SessionType.TASK, + destination_payload_list=payloads + ) + + performance = [(msg.source, msg.payload[PayloadKey.PERFORMANCE]) for msg in replies] + details_by_source = {msg.source: msg.payload[PayloadKey.DETAILS] for msg in replies} + details = self._experience_collecting_func(details_by_source) if return_details else None - return performance, details + return performance, details class ActorWorker(object): diff --git a/maro/rl/exploration/epsilon_greedy_scheduler.py b/maro/rl/exploration/epsilon_greedy_scheduler.py index 4b73651ea..db625fdf3 100644 --- a/maro/rl/exploration/epsilon_greedy_scheduler.py +++ b/maro/rl/exploration/epsilon_greedy_scheduler.py @@ -30,7 +30,7 @@ def __next__(self): eps = self._current_eps self._current_ep += 1 self._current_eps += self._eps_delta - return eps + return {"epsilon": eps} class TwoPhaseLinearEpsilonScheduler(AbsExplorationScheduler): @@ -67,4 +67,4 @@ def __next__(self): eps = self._current_eps self._current_ep += 1 self._current_eps += self._eps_delta_1 if self._current_ep < self._split_ep else self._eps_delta_2 - return eps + return {"epsilon": eps} From a0dbad79597d29809c1c6c636cc2a466459d3801 Mon Sep 17 00:00:00 2001 From: ysqyang Date: Wed, 25 Nov 2020 23:32:29 +0800 Subject: [PATCH 257/337] fixed a bug --- maro/rl/agent/abs_agent.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/maro/rl/agent/abs_agent.py b/maro/rl/agent/abs_agent.py index ab03b0f64..b50a43da8 100644 --- a/maro/rl/agent/abs_agent.py +++ b/maro/rl/agent/abs_agent.py @@ -67,9 +67,9 @@ def choose_action(self, model_state): action = self._algorithm.choose_action(model_state) return action if self._explorer is None else self._explorer(action) - def update(self, exploration_params): + def update(self, **exploration_params): if self._explorer: - self._explorer.update(exploration_params) + self._explorer.update(**exploration_params) @abstractmethod def train(self, *args, **kwargs): From f719fa8b3e7188ddd6e5160cd5bd1c480b407843 Mon Sep 17 00:00:00 2001 From: ysqyang Date: Wed, 25 Nov 2020 23:33:25 +0800 Subject: [PATCH 258/337] fixed a bug --- maro/rl/learner/simple_learner.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/maro/rl/learner/simple_learner.py b/maro/rl/learner/simple_learner.py index ed3f42713..a6528ccbf 100644 --- a/maro/rl/learner/simple_learner.py +++ b/maro/rl/learner/simple_learner.py @@ -72,7 +72,7 @@ def learn(self): ) self._logger.info( - f"ep {self._exploration_scheduler.current_episode} - " + f"ep {self._exploration_scheduler.current_ep} - " f"performance: {performance}, exploration_params: {exploration_params}" ) From 451f645184518b6ee7740747edd06711c6645bdb Mon Sep 17 00:00:00 2001 From: ysqyang Date: Wed, 25 Nov 2020 23:34:42 +0800 Subject: [PATCH 259/337] fixed a bug --- maro/rl/early_stopping/abs_early_stopping_checker.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/maro/rl/early_stopping/abs_early_stopping_checker.py b/maro/rl/early_stopping/abs_early_stopping_checker.py index cecab94eb..3043314a3 100644 --- a/maro/rl/early_stopping/abs_early_stopping_checker.py +++ b/maro/rl/early_stopping/abs_early_stopping_checker.py @@ -61,7 +61,7 @@ def __init__(self, checker, other): self._checker = checker self._other_checker = other - def push(self, performance) -> bool: + def update(self, performance) -> bool: return self._checker.update(performance) or self._other_checker.update(performance) return OrChecker(self, other_checker) @@ -77,7 +77,7 @@ def __init__(self, checker, other): self._checker = checker self._other_checker = other - def push(self, performance) -> bool: + def update(self, performance) -> bool: result = self._checker.update(performance) other_result = self._other_checker.update(performance) return result and other_result @@ -94,7 +94,7 @@ def __init__(self, checker, other): self._checker = checker self._other_checker = other - def __call__(self, performance) -> bool: + def update(self, performance) -> bool: return self._checker.update(performance) ^ self._other_checker.update(performance) return XorChecker(self, other_checker) @@ -109,7 +109,7 @@ def __init__(self, checker): super().__init__() self._checker = checker - def __call__(self, performance) -> bool: + def update(self, performance) -> bool: return not self._checker.update(performance) return InverseChecker(self) From 16f55d284f5003b9a517472766bac7b5748504f5 Mon Sep 17 00:00:00 2001 From: ysqyang Date: Wed, 25 Nov 2020 23:37:36 +0800 Subject: [PATCH 260/337] fixed a bug --- maro/rl/learner/simple_learner.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/maro/rl/learner/simple_learner.py b/maro/rl/learner/simple_learner.py index a6528ccbf..3b6c73847 100644 --- a/maro/rl/learner/simple_learner.py +++ b/maro/rl/learner/simple_learner.py @@ -77,8 +77,7 @@ def learn(self): ) # Early stopping checking - latest = [perf for _, perf in performance] if isinstance(performance, list) else [performance] - if self._early_stopping_checker is not None and self._early_stopping_checker.update(latest): + if self._early_stopping_checker is not None and self._early_stopping_checker.update(performance): self._logger.info("Early stopping condition hit. Training complete.") break except StopIteration: From f4c7d35099571b6e48a54a773f9ba87d32e4e0e0 Mon Sep 17 00:00:00 2001 From: ysqyang Date: Wed, 25 Nov 2020 23:43:28 +0800 Subject: [PATCH 261/337] fixed lint issue --- maro/rl/learner/simple_learner.py | 1 - 1 file changed, 1 deletion(-) diff --git a/maro/rl/learner/simple_learner.py b/maro/rl/learner/simple_learner.py index 3b6c73847..bf973ba65 100644 --- a/maro/rl/learner/simple_learner.py +++ b/maro/rl/learner/simple_learner.py @@ -15,7 +15,6 @@ from .abs_learner import AbsLearner - ExplorationOptions = namedtuple("ExplorationOptions", ["cls", "params"]) From 1469b775e8105cdb2aa5d1da1395678fb6bb15ad Mon Sep 17 00:00:00 2001 From: ysqyang Date: Wed, 25 Nov 2020 23:47:28 +0800 Subject: [PATCH 262/337] fixed a bug --- maro/rl/early_stopping/simple_early_stopping_checker.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/maro/rl/early_stopping/simple_early_stopping_checker.py b/maro/rl/early_stopping/simple_early_stopping_checker.py index ab445d79f..b28f233c2 100644 --- a/maro/rl/early_stopping/simple_early_stopping_checker.py +++ b/maro/rl/early_stopping/simple_early_stopping_checker.py @@ -41,5 +41,8 @@ class MaxDeltaEarlyStoppingChecker(AbsEarlyStoppingChecker): is compared with the threshold to determine if early stopping should be triggered. """ def check(self): - max_delta = max(abs(val2 - val1) / val1 for val1, val2 in zip(self._metric_series, self._metric_series[1:])) + max_delta = max( + abs(self._metric_series[i] - self._metric_series[i-1]) / self._metric_series[i-1] + for i in range(1, self._last_k) + ) return max_delta < self._threshold From e379f7182dd27aacfb5dce66f07a15bc1635a335 Mon Sep 17 00:00:00 2001 From: ysqyang Date: Wed, 25 Nov 2020 23:56:14 +0800 Subject: [PATCH 263/337] fixed lint issue --- maro/rl/early_stopping/simple_early_stopping_checker.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/maro/rl/early_stopping/simple_early_stopping_checker.py b/maro/rl/early_stopping/simple_early_stopping_checker.py index b28f233c2..61bf2c68f 100644 --- a/maro/rl/early_stopping/simple_early_stopping_checker.py +++ b/maro/rl/early_stopping/simple_early_stopping_checker.py @@ -42,7 +42,7 @@ class MaxDeltaEarlyStoppingChecker(AbsEarlyStoppingChecker): """ def check(self): max_delta = max( - abs(self._metric_series[i] - self._metric_series[i-1]) / self._metric_series[i-1] + abs(self._metric_series[i] - self._metric_series[i - 1]) / self._metric_series[i - 1] for i in range(1, self._last_k) ) return max_delta < self._threshold From 5ce1680708e7a25fe43eda49bd83901946cb5690 Mon Sep 17 00:00:00 2001 From: ysqyang Date: Thu, 26 Nov 2020 00:05:32 +0800 Subject: [PATCH 264/337] fixed naming --- maro/rl/dist_topologies/single_learner_multi_actor_sync_mode.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/maro/rl/dist_topologies/single_learner_multi_actor_sync_mode.py b/maro/rl/dist_topologies/single_learner_multi_actor_sync_mode.py index 4b11cebef..f6f4c0098 100644 --- a/maro/rl/dist_topologies/single_learner_multi_actor_sync_mode.py +++ b/maro/rl/dist_topologies/single_learner_multi_actor_sync_mode.py @@ -84,7 +84,7 @@ class ActorWorker(object): def __init__(self, local_actor: AbsActor, proxy_params): self._local_actor = local_actor self._proxy = Proxy(component_type="actor", **proxy_params) - self._registry_table = RegisterTable(self._proxy.get_peers) + self._registry_table = RegisterTable(self._proxy.peers_name) self._registry_table.register_event_handler("learner:rollout:1", self.on_rollout_request) def on_rollout_request(self, message): From bbf140bebfacbeba273b2d95ac8cca3b98d702dd Mon Sep 17 00:00:00 2001 From: ysqyang Date: Fri, 27 Nov 2020 23:40:15 +0800 Subject: [PATCH 265/337] combined exploration param generation and early stopping in scheduler --- examples/cim/dqn/config.yml | 8 +- examples/cim/dqn/dist_learner.py | 14 +- examples/cim/dqn/single_process_launcher.py | 38 ++--- maro/rl/__init__.py | 26 ++-- .../abs_early_stopping_checker.py | 115 --------------- .../simple_early_stopping_checker.py | 48 ------- .../exploration/abs_exploration_scheduler.py | 34 ----- .../exploration/epsilon_greedy_scheduler.py | 70 --------- maro/rl/learner/simple_learner.py | 60 ++------ .../__init__.py | 0 .../exploration_parameter_generator.py | 136 ++++++++++++++++++ maro/rl/scheduling/scheduler.py | 68 +++++++++ 12 files changed, 264 insertions(+), 353 deletions(-) delete mode 100644 maro/rl/early_stopping/abs_early_stopping_checker.py delete mode 100644 maro/rl/early_stopping/simple_early_stopping_checker.py delete mode 100644 maro/rl/exploration/abs_exploration_scheduler.py delete mode 100644 maro/rl/exploration/epsilon_greedy_scheduler.py rename maro/rl/{early_stopping => scheduling}/__init__.py (100%) create mode 100644 maro/rl/scheduling/exploration_parameter_generator.py create mode 100644 maro/rl/scheduling/scheduler.py diff --git a/examples/cim/dqn/config.yml b/examples/cim/dqn/config.yml index 39082c2b0..e02e814e9 100644 --- a/examples/cim/dqn/config.yml +++ b/examples/cim/dqn/config.yml @@ -5,10 +5,12 @@ env: main_loop: max_episode: 500 exploration: + parameter_names: + - "epsilon" split_ep: 250 - start_eps: 0.4 - mid_eps: 0.32 - end_eps: 0.0 + start_values: 0.4 + mid_values: 0.32 + end_values: 0.0 early_stopping: warmup_ep: 50 last_k: 10 diff --git a/examples/cim/dqn/dist_learner.py b/examples/cim/dqn/dist_learner.py index 9e8f8ebea..94530e818 100644 --- a/examples/cim/dqn/dist_learner.py +++ b/examples/cim/dqn/dist_learner.py @@ -7,8 +7,8 @@ from components.config import set_input_dim from maro.rl import ( - ActorProxy, AgentManagerMode, ExplorationOptions, SimpleLearner, concat_experiences_by_agent, - TwoPhaseLinearEpsilonScheduler + ActorProxy, AgentManagerMode, Scheduler, SimpleLearner, TwoPhaseLinearExplorationParameterGenerator, + concat_experiences_by_agent ) from maro.simulator import Env from maro.utils import Logger, convert_dottable @@ -38,11 +38,17 @@ def launch(config, distributed_config): "max_retries": 15 } + scheduler = Scheduler( + config.main_loop.max_episode, + warmup_ep=config.main_loop.early_stopping.warmup_ep, + exploration_parameter_generator_cls=TwoPhaseLinearExplorationParameterGenerator, + exploration_parameter_generator_config=config.main_loop.exploration, + ) + learner = SimpleLearner( agent_manager=agent_manager, actor=ActorProxy(proxy_params=proxy_params, experience_collecting_func=concat_experiences_by_agent), - max_episode=config.main_loop.max_episode, - exploration_options=ExplorationOptions(TwoPhaseLinearEpsilonScheduler, config.main_loop.exploration), + scheduler=scheduler, logger=Logger("distributed_cim_learner", auto_timestamp=False) ) learner.learn() diff --git a/examples/cim/dqn/single_process_launcher.py b/examples/cim/dqn/single_process_launcher.py index fa842ecb0..dd3788a93 100644 --- a/examples/cim/dqn/single_process_launcher.py +++ b/examples/cim/dqn/single_process_launcher.py @@ -13,8 +13,8 @@ from components.state_shaper import CIMStateShaper from maro.rl import ( - AgentManagerMode, ExplorationOptions, KStepExperienceShaper, MaxDeltaEarlyStoppingChecker, - SimpleActor, SimpleEarlyStoppingChecker, SimpleLearner, TwoPhaseLinearEpsilonScheduler, + AgentManagerMode, KStepExperienceShaper, Scheduler, SimpleActor, SimpleLearner, + StaticExplorationParameterGenerator, TwoPhaseLinearExplorationParameterGenerator ) from maro.simulator import Env from maro.utils import Logger, convert_dottable @@ -52,31 +52,33 @@ def launch(config): ) # Step 4: Create an actor and a learner to start the training process. - def metric_func(performance): - return 1 - performance["container_shortage"] / performance["order_requirements"] + def early_stopping_callback(perf_history): + last_k = config.main_loop.early_stopping.last_k + perf_threshold = config.main_loop.early_stopping.perf_threshold + perf_stability_threshold = config.main_loop.early_stopping.perf_stability_threshold + if len(perf_history) < last_k: + return False - perf_checker = SimpleEarlyStoppingChecker( - last_k=config.main_loop.early_stopping.last_k, - threshold=config.main_loop.early_stopping.perf_threshold, - warmup_ep=config.main_loop.early_stopping.warmup_ep, - metric_func=metric_func, - measure_func=lambda metric_series : mean(metric_series) - ) + metric_series = list( + map(lambda p: 1 - p["container_shortage"] / p["order_requirements"], perf_history[-last_k:]) + ) + mean_perf = mean(metric_series) + max_delta = max(abs(metric_series[i] - metric_series[i - 1]) / metric_series[i - 1] for i in range(1, last_k)) + return mean_perf > perf_threshold and max_delta < perf_stability_threshold - perf_stability_checker = MaxDeltaEarlyStoppingChecker( - last_k=config.main_loop.early_stopping.last_k, - threshold=config.main_loop.early_stopping.perf_stability_threshold, + scheduler = Scheduler( + config.main_loop.max_episode, warmup_ep=config.main_loop.early_stopping.warmup_ep, - metric_func=metric_func + early_stopping_callback=early_stopping_callback, + exploration_parameter_generator_cls=TwoPhaseLinearExplorationParameterGenerator, + exploration_parameter_generator_config=config.main_loop.exploration, ) actor = SimpleActor(env, agent_manager) learner = SimpleLearner( agent_manager=agent_manager, actor=actor, - max_episode=config.main_loop.max_episode, - exploration_options=ExplorationOptions(TwoPhaseLinearEpsilonScheduler, config.main_loop.exploration), - early_stopping_checker=perf_checker & perf_stability_checker, + scheduler=scheduler, logger=Logger("single_host_cim_learner", auto_timestamp=False) ) learner.learn() diff --git a/maro/rl/__init__.py b/maro/rl/__init__.py index 41c7d56bb..01503baea 100644 --- a/maro/rl/__init__.py +++ b/maro/rl/__init__.py @@ -12,18 +12,17 @@ concat_experiences_by_agent, merge_experiences_with_trajectory_boundaries ) from maro.rl.dist_topologies.single_learner_multi_actor_sync_mode import ActorProxy, ActorWorker -from maro.rl.early_stopping.abs_early_stopping_checker import AbsEarlyStoppingChecker -from maro.rl.early_stopping.simple_early_stopping_checker import ( - MaxDeltaEarlyStoppingChecker, RSDEarlyStoppingChecker, SimpleEarlyStoppingChecker -) -from maro.rl.exploration.abs_exploration_scheduler import AbsExplorationScheduler, NullExplorationScheduler from maro.rl.exploration.abs_explorer import AbsExplorer from maro.rl.exploration.epsilon_greedy_explorer import EpsilonGreedyExplorer -from maro.rl.exploration.epsilon_greedy_scheduler import LinearEpsilonScheduler, TwoPhaseLinearEpsilonScheduler from maro.rl.learner.abs_learner import AbsLearner -from maro.rl.learner.simple_learner import ExplorationOptions, SimpleLearner +from maro.rl.learner.simple_learner import SimpleLearner from maro.rl.models.fc_block import FullyConnectedBlock from maro.rl.models.learning_model import MultiHeadLearningModel, SingleHeadLearningModel +from maro.rl.scheduling.scheduler import Scheduler +from maro.rl.scheduling.exploration_parameter_generator import ( + DynamicExplorationParameterGenerator, LinearExplorationParameterGenerator, + StaticExplorationParameterGenerator, TwoPhaseLinearExplorationParameterGenerator +) from maro.rl.shaping.abs_shaper import AbsShaper from maro.rl.shaping.action_shaper import ActionShaper from maro.rl.shaping.experience_shaper import ExperienceShaper @@ -38,7 +37,6 @@ 'AbsAgent', 'AbsAgentManager', 'AbsAlgorithm', - 'AbsEarlyStoppingChecker', 'AbsExplorer', 'AbsLearner', 'AbsShaper', @@ -50,24 +48,22 @@ 'ColumnBasedStore', 'DQN', 'DQNHyperParams', + 'DynamicExplorationParameterGenerator', 'EpsilonGreedyExplorer', 'ExperienceShaper', - 'ExplorationOptions', 'FullyConnectedBlock', 'KStepExperienceShaper', - 'LinearEpsilonScheduler', - 'MaxDeltaEarlyStoppingChecker', + 'LinearExplorationParameterGenerator', 'MultiHeadLearningModel', - 'NullExplorationScheduler', 'OverwriteType', - 'RSDEarlyStoppingChecker', + 'Scheduler', 'SimpleActor', 'SimpleAgentManager', - 'SimpleEarlyStoppingChecker', 'SimpleLearner', 'SingleHeadLearningModel', 'StateShaper', - 'TwoPhaseLinearEpsilonScheduler', + 'StaticExplorationParameterGenerator', + 'TwoPhaseLinearExplorationParameterGenerator', 'concat_experiences_by_agent', 'merge_experiences_with_trajectory_boundaries' ] diff --git a/maro/rl/early_stopping/abs_early_stopping_checker.py b/maro/rl/early_stopping/abs_early_stopping_checker.py deleted file mode 100644 index 3043314a3..000000000 --- a/maro/rl/early_stopping/abs_early_stopping_checker.py +++ /dev/null @@ -1,115 +0,0 @@ -# Copyright (c) Microsoft Corporation. -# Licensed under the MIT license. - -from abc import ABC, abstractmethod -from collections import deque -from typing import Callable - - -class AbsEarlyStoppingChecker(ABC): - """Class that checks for early stopping conditions. - - Args: - last_k (int): Number of the latest metric values to check for early stopping. - threshold (float): The threshold value against which a user-defined measure is compared to determine - whether early-stopping should be triggered. - warmup_ep (int): Number of episodes before early stopping checker takes effect. - metric_func (Callable): Function to extract early stopping metric from a performance record. - """ - def __init__(self, last_k: int, threshold: float, warmup_ep: int, metric_func: Callable): - super().__init__() - self._last_k = last_k - self._threshold = threshold - self._metric_func = metric_func - self._warmup_ep = warmup_ep - self._metric_series = deque() - self._ep_count = 0 - - def update(self, performance) -> bool: - """Update with the latest performance record and check whether an early stopping condition is met. - - Args: - performance: Performance record from the latest roll-out episode. - - Returns: - A boolean value indicating whether early stopping should be triggered. - """ - self._ep_count += 1 - if isinstance(performance, list): - self._metric_series.extend(map(self._metric_func, (perf for _, perf in performance))) - else: - self._metric_series.append(self._metric_func(performance)) - if self._ep_count < self._warmup_ep or len(self._metric_series) < self._last_k: - return False - - while len(self._metric_series) > self._last_k: - self._metric_series.popleft() - - return self.check() - - @abstractmethod - def check(self) -> bool: - return NotImplemented - - def __or__(self, other_checker): - """Return a checker that is the result of logical OR between itself and another checker. - - The resulting checker returns True iff at least one of the checkers returns True. - """ - class OrChecker: - def __init__(self, checker, other): - self._checker = checker - self._other_checker = other - - def update(self, performance) -> bool: - return self._checker.update(performance) or self._other_checker.update(performance) - - return OrChecker(self, other_checker) - - def __and__(self, other_checker): - """Return a checker that is the result of logical AND between itself and another checker. - - The resulting checker returns True iff both checkers return True. - """ - class AndChecker: - def __init__(self, checker, other): - super().__init__() - self._checker = checker - self._other_checker = other - - def update(self, performance) -> bool: - result = self._checker.update(performance) - other_result = self._other_checker.update(performance) - return result and other_result - - return AndChecker(self, other_checker) - - def __xor__(self, other_checker): - """Return a checker that is the result of logical XOR between itself and another checker. - - The resulting checker returns True iff one checker returns True and the other returns False. - """ - class XorChecker: - def __init__(self, checker, other): - self._checker = checker - self._other_checker = other - - def update(self, performance) -> bool: - return self._checker.update(performance) ^ self._other_checker.update(performance) - - return XorChecker(self, other_checker) - - def __invert__(self): - """Return a checker that is the result of logical NOT of itself. - - The resulting checker returns True iff itself returns False. - """ - class InverseChecker: - def __init__(self, checker): - super().__init__() - self._checker = checker - - def update(self, performance) -> bool: - return not self._checker.update(performance) - - return InverseChecker(self) diff --git a/maro/rl/early_stopping/simple_early_stopping_checker.py b/maro/rl/early_stopping/simple_early_stopping_checker.py deleted file mode 100644 index 61bf2c68f..000000000 --- a/maro/rl/early_stopping/simple_early_stopping_checker.py +++ /dev/null @@ -1,48 +0,0 @@ -# Copyright (c) Microsoft Corporation. -# Licensed under the MIT license. - -from statistics import mean, stdev -from typing import Callable, Union - -from .abs_early_stopping_checker import AbsEarlyStoppingChecker - - -class SimpleEarlyStoppingChecker(AbsEarlyStoppingChecker): - """Early stopping checker based on the some simple measure. - - The measure is obtained by applying a user-defined measure function to the last k metric values. The measure - function must take a list as input and output a single number. - """ - def __init__( - self, - last_k: int, - threshold: float, - warmup_ep: int, - metric_func: Callable, - measure_func: Callable[[list], Union[int, float]] - ): - super().__init__(last_k, threshold, warmup_ep, metric_func) - self._measure_func = measure_func - - def check(self): - return self._measure_func(self._metric_series) >= self._threshold - - -class RSDEarlyStoppingChecker(AbsEarlyStoppingChecker): - """Early stopping checker based on the mean and standard deviation of the last k metric values.""" - def check(self): - return stdev(self._metric_series) / mean(self._metric_series) < self._threshold - - -class MaxDeltaEarlyStoppingChecker(AbsEarlyStoppingChecker): - """Early stopping checker based on the maximum relative variation over the last k metric values. - - The relative change is defined as |m(i+1) - m(i)| / m[i]. The maximum of the last k-1 changes in the metric series - is compared with the threshold to determine if early stopping should be triggered. - """ - def check(self): - max_delta = max( - abs(self._metric_series[i] - self._metric_series[i - 1]) / self._metric_series[i - 1] - for i in range(1, self._last_k) - ) - return max_delta < self._threshold diff --git a/maro/rl/exploration/abs_exploration_scheduler.py b/maro/rl/exploration/abs_exploration_scheduler.py deleted file mode 100644 index 31f9895de..000000000 --- a/maro/rl/exploration/abs_exploration_scheduler.py +++ /dev/null @@ -1,34 +0,0 @@ -# Copyright (c) Microsoft Corporation. -# Licensed under the MIT license. - -from abc import ABC, abstractmethod - - -class AbsExplorationScheduler(ABC): - """Scheduler that generates exploration parameters for each episode. - - Args: - max_ep (int): Maximum number of episodes to be run. - """ - def __init__(self, max_ep: int): - self._max_ep = max_ep - self._current_ep = 0 - - def __iter__(self): - return self - - @abstractmethod - def __next__(self): - raise NotImplementedError - - @property - def current_ep(self): - return self._current_ep - - -class NullExplorationScheduler(AbsExplorationScheduler): - """Dummy scheduler that generates.""" - def __next__(self): - if self._current_ep == self._max_ep: - raise StopIteration - self._current_ep += 1 diff --git a/maro/rl/exploration/epsilon_greedy_scheduler.py b/maro/rl/exploration/epsilon_greedy_scheduler.py deleted file mode 100644 index db625fdf3..000000000 --- a/maro/rl/exploration/epsilon_greedy_scheduler.py +++ /dev/null @@ -1,70 +0,0 @@ -# Copyright (c) Microsoft Corporation. -# Licensed under the MIT license. - -from maro.utils.exception.rl_toolkit_exception import InvalidEpisodeError - -from .abs_exploration_scheduler import AbsExplorationScheduler - - -class LinearEpsilonScheduler(AbsExplorationScheduler): - """Linear exploration rate generator for epsilon-greedy exploration. - - Args: - max_ep (int): Maximum number of episodes to run. - start_eps (float): The exploration rate for the first episode. - end_eps (float): The exploration rate for the last episode. Defaults to zero. - - """ - def __init__(self, max_ep: int, start_eps: float, end_eps: float = .0): - if max_ep <= 0: - raise InvalidEpisodeError("max_ep must be a positive integer.") - super().__init__(max_ep) - self._start_eps = start_eps - self._end_eps = end_eps - self._current_eps = start_eps - self._eps_delta = (self._end_eps - self._start_eps) / (self._max_ep - 1) - - def __next__(self): - if self._current_ep == self._max_ep: - raise StopIteration - eps = self._current_eps - self._current_ep += 1 - self._current_eps += self._eps_delta - return {"epsilon": eps} - - -class TwoPhaseLinearEpsilonScheduler(AbsExplorationScheduler): - """Exploration schedule comprised of two linear schedules joined together for epsilon-greedy exploration. - - Args: - max_ep (int): Maximum number of episodes to run. - split_ep (float): The episode where the switch from the first linear schedule to the second occurs. - start_eps (float): Exploration rate for the first episode. - mid_eps (float): The exploration rate where the switch from the first linear schedule to the second occurs. - In other words, this is the exploration rate where the first linear schedule ends and the second begins. - end_eps (float): Exploration rate for the last episode. Defaults to zero. - - Returns: - An iterator over the series of exploration rates from episode 0 to ``max_ep`` - 1. - """ - def __init__(self, max_ep: int, split_ep: float, start_eps: float, mid_eps: float, end_eps: float = .0): - if max_ep <= 0: - raise InvalidEpisodeError("max_ep must be a positive integer.") - if split_ep <= 0 or split_ep >= max_ep: - raise ValueError("split_ep must be between 0 and max_ep - 1.") - super().__init__(max_ep) - self._split_ep = split_ep - self._start_eps = start_eps - self._mid_eps = mid_eps - self._end_eps = end_eps - self._current_eps = start_eps - self._eps_delta_1 = (mid_eps - start_eps) / split_ep - self._eps_delta_2 = (end_eps - mid_eps) / (max_ep - split_ep - 1) - - def __next__(self): - if self._current_ep == self._max_ep: - raise StopIteration - eps = self._current_eps - self._current_ep += 1 - self._current_eps += self._eps_delta_1 if self._current_ep < self._split_ep else self._eps_delta_2 - return {"epsilon": eps} diff --git a/maro/rl/learner/simple_learner.py b/maro/rl/learner/simple_learner.py index bf973ba65..874528f02 100644 --- a/maro/rl/learner/simple_learner.py +++ b/maro/rl/learner/simple_learner.py @@ -2,21 +2,16 @@ # Licensed under the MIT license. import sys -from collections import namedtuple from typing import Union from maro.rl.actor.simple_actor import SimpleActor from maro.rl.agent.simple_agent_manager import SimpleAgentManager from maro.rl.dist_topologies.single_learner_multi_actor_sync_mode import ActorProxy -from maro.rl.early_stopping.abs_early_stopping_checker import AbsEarlyStoppingChecker -from maro.rl.exploration.abs_exploration_scheduler import NullExplorationScheduler +from maro.rl.scheduling.scheduler import Scheduler from maro.utils import DummyLogger, Logger -from maro.utils.exception.rl_toolkit_exception import InfiniteTrainingLoopError, InvalidEpisodeError from .abs_learner import AbsLearner -ExplorationOptions = namedtuple("ExplorationOptions", ["cls", "params"]) - class SimpleLearner(AbsLearner): """A simple implementation of ``AbsLearner``. @@ -25,63 +20,36 @@ class SimpleLearner(AbsLearner): agent_manager (AbsAgentManager): An AgentManager instance that manages all agents. actor (SimpleActor or ActorProxy): An SimpleActor or ActorProxy instance responsible for performing roll-outs (environment sampling). - max_episode (int): number of episodes to be run. If -1, the training loop will run forever unless - an ``early_stopping_checker`` is provided and the early stopping condition is met. - exploration_options (ExplorationOptions): Exploration scheduler class and parameters. Defaults to None. - early_stopping_checker (EarlyStoppingOptions): Early stopping checker that checks the performance history to - determine whether early stopping condition is satisfied. Defaults to None. + scheduler (AbsScheduler): A scheduler responsible for iterating over episodes and generating exploration + parameters if necessary. logger (Logger): Used to log important messages. """ def __init__( self, agent_manager: SimpleAgentManager, actor: Union[SimpleActor, ActorProxy], - max_episode: int, - exploration_options: ExplorationOptions = None, - early_stopping_checker: AbsEarlyStoppingChecker = None, + scheduler: Scheduler, logger: Logger = DummyLogger() ): - if max_episode < -1: - raise InvalidEpisodeError("max_episode can only be a non-negative integer or -1.") - if max_episode == -1 and early_stopping_checker is None: - raise InfiniteTrainingLoopError( - "The training loop will run forever since neither maximum episode nor early stopping checker " - "is provided. " - ) super().__init__() self._agent_manager = agent_manager self._actor = actor - self._max_episode = max_episode - if exploration_options is None: - self._exploration_scheduler = NullExplorationScheduler(max_episode) - else: - self._exploration_scheduler = exploration_options.cls(max_ep=max_episode, **exploration_options.params) - self._early_stopping_checker = early_stopping_checker + self._scheduler = scheduler self._logger = logger self._performance_history = [] def learn(self): """Main loop for collecting experiences from the actor and using them to update policies.""" - while True: - try: - exploration_params = next(self._exploration_scheduler) - performance, exp_by_agent = self._actor.roll_out( - model_dict=None if self._is_shared_agent_instance() else self._agent_manager.dump_models(), - exploration_params=exploration_params - ) - - self._logger.info( - f"ep {self._exploration_scheduler.current_ep} - " - f"performance: {performance}, exploration_params: {exploration_params}" - ) + for exploration_params in self._scheduler: + performance, exp_by_agent = self._actor.roll_out( + model_dict=None if self._is_shared_agent_instance() else self._agent_manager.dump_models(), + exploration_params=exploration_params + ) - # Early stopping checking - if self._early_stopping_checker is not None and self._early_stopping_checker.update(performance): - self._logger.info("Early stopping condition hit. Training complete.") - break - except StopIteration: - self._logger.info(f"Maximum number of episodes ({self._max_episode}) reached. Training complete.") - break + self._logger.info( + f"ep {self._scheduler.current_ep} - performance: {performance}, " + f"exploration_params: {exploration_params}" + ) self._agent_manager.train(exp_by_agent) diff --git a/maro/rl/early_stopping/__init__.py b/maro/rl/scheduling/__init__.py similarity index 100% rename from maro/rl/early_stopping/__init__.py rename to maro/rl/scheduling/__init__.py diff --git a/maro/rl/scheduling/exploration_parameter_generator.py b/maro/rl/scheduling/exploration_parameter_generator.py new file mode 100644 index 000000000..42b15236c --- /dev/null +++ b/maro/rl/scheduling/exploration_parameter_generator.py @@ -0,0 +1,136 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +from abc import abstractmethod +from typing import Union + +import numpy as np + + +class StaticExplorationParameterGenerator(object): + """Exploration parameter generator based on a pre-defined schedule. + + Args: + max_ep (int): Maximum number of episodes to run. + """ + def __init__(self, max_ep: int, **config): + super().__init__() + self._max_ep = max_ep + self._config = config + + @abstractmethod + def next(self): + raise NotImplementedError + + +class DynamicExplorationParameterGenerator(object): + """Dynamic exploration parameter generator based on the performances history.""" + def __init__(self, **config): + super().__init__() + self._config = config + + @abstractmethod + def next(self, performance_history: list): + raise NotImplementedError + + +class LinearExplorationParameterGenerator(StaticExplorationParameterGenerator): + """Static exploration parameter generator based on a linear schedule. + + Args: + max_ep (int): Maximum number of episodes to run. + parameter_names ([str]): List of exploration parameter names. + start_values (Union[float, list, tuple, np.ndarray]): Exploration parameter values for the first episode. + These values must correspond to ``parameter_names``. + end_values (Union[float, list, tuple, np.ndarray]): Exploration parameter values rate for the last episode. + These values must correspond to ``parameter_names``. + """ + def __init__( + self, + max_ep: int, + *, + parameter_names: [str], + start_values: Union[float, list, tuple, np.ndarray], + end_values: Union[float, list, tuple, np.ndarray] + ): + super().__init__(max_ep) + self._parameter_names = parameter_names + if isinstance(start_values, float): + self._current_values = start_values * np.ones(len(self._parameter_names)) + elif isinstance(start_values, (list, tuple)): + self._current_values = np.asarray(start_values) + else: + self._current_values = start_values + + if isinstance(end_values, float): + end_values = end_values * np.ones(len(self._parameter_names)) + elif isinstance(end_values, (list, tuple)): + end_values = np.asarray(end_values) + + self._delta = (end_values - self._current_values) / (max_ep - 1) + + def next(self): + current_values = self._current_values + self._current_values += self._delta + return dict(zip(self._parameter_names, current_values)) + + +class TwoPhaseLinearExplorationParameterGenerator(StaticExplorationParameterGenerator): + """Exploration parameter generator based on two linear schedules joined together. + + Args: + max_ep (int): Maximum number of episodes to run. + parameter_names ([str]): List of exploration parameter names. + split_ep (float): The episode where the switch from the first linear schedule to the second occurs. + start_values (Union[float, list, tuple, np.ndarray]): Exploration parameter values for the first episode. + These values must correspond to ``parameter_names``. + mid_values (Union[float, list, tuple, np.ndarray]): Exploration parameter values where the switch from the + first linear schedule to the second occurs. In other words, this is the exploration rate where the first + linear schedule ends and the second begins. These values must correspond to ``parameter_names``. + end_values (Union[float, list, tuple, np.ndarray]): Exploration parameter values for the last episode. + These values must correspond to ``parameter_names``. + + Returns: + An iterator over the series of exploration rates from episode 0 to ``max_ep`` - 1. + """ + def __init__( + self, + max_ep: int, + *, + parameter_names: [str], + split_ep: float, + start_values: Union[float, list, tuple, np.ndarray], + mid_values: Union[float, list, tuple, np.ndarray], + end_values: Union[float, list, tuple, np.ndarray] + ): + if split_ep <= 0 or split_ep >= max_ep: + raise ValueError("split_ep must be between 0 and max_ep - 1.") + super().__init__(max_ep) + self._parameter_names = parameter_names + self._split_ep = split_ep + if isinstance(start_values, float): + self._current_values = start_values * np.ones(len(self._parameter_names)) + elif isinstance(start_values, (list, tuple)): + self._current_values = np.asarray(start_values) + else: + self._current_values = start_values + + if isinstance(mid_values, float): + mid_values = mid_values * np.ones(len(self._parameter_names)) + elif isinstance(mid_values, (list, tuple)): + mid_values = np.asarray(mid_values) + + if isinstance(end_values, float): + end_values = end_values * np.ones(len(self._parameter_names)) + elif isinstance(end_values, (list, tuple)): + end_values = np.asarray(end_values) + + self._delta_1 = (mid_values - self._current_values) / split_ep + self._delta_2 = (end_values - mid_values) / (max_ep - split_ep - 1) + self._current_ep = 0 + + def next(self): + current_values = self._current_values + self._current_values += self._delta_1 if self._current_ep < self._split_ep else self._delta_2 + self._current_ep += 1 + return dict(zip(self._parameter_names, current_values)) diff --git a/maro/rl/scheduling/scheduler.py b/maro/rl/scheduling/scheduler.py new file mode 100644 index 000000000..a6c61e593 --- /dev/null +++ b/maro/rl/scheduling/scheduler.py @@ -0,0 +1,68 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +from typing import Callable, Union + +from maro.utils.exception.rl_toolkit_exception import InfiniteTrainingLoopError, InvalidEpisodeError + +from .exploration_parameter_generator import ( + DynamicExplorationParameterGenerator, StaticExplorationParameterGenerator +) + + +class Scheduler(object): + """Scheduler that generates exploration parameters for each episode. + + Args: + max_ep (int): Maximum number of episodes to be run. + """ + + def __init__( + self, + max_ep: int, + warmup_ep: int = 0, + early_stopping_callback: Callable = None, + exploration_parameter_generator_cls=None, + exploration_parameter_generator_config: dict = None + ): + if max_ep < -1: + raise InvalidEpisodeError("max_episode can only be a non-negative integer or -1.") + if max_ep == -1 and early_stopping_callback is None: + raise InfiniteTrainingLoopError( + "The training loop will run forever since neither maximum episode nor early stopping checker " + "is provided. " + ) + self._max_ep = max_ep + self._warmup_ep = warmup_ep + self._early_stopping_callback = early_stopping_callback + self._current_ep = 0 + self._performance_history = [] + if exploration_parameter_generator_cls is None: + self._exploration_parameter_generator = None + elif exploration_parameter_generator_cls == StaticExplorationParameterGenerator: + self._exploration_parameter_generator = StaticExplorationParameterGenerator( + max_ep, **exploration_parameter_generator_config + ) + else: + self._exploration_parameter_generator = DynamicExplorationParameterGenerator( + **exploration_parameter_generator_config + ) + + def __iter__(self): + return self + + def __next__(self, performance): + if self._current_ep == self._max_ep: + raise StopIteration + if self._current_ep >= self._warmup_ep: + if self._early_stopping_callback and self._early_stopping_callback(self._performance_history): + raise StopIteration + self._current_ep += 1 + if isinstance(self._exploration_parameter_generator, StaticExplorationParameterGenerator): + return self._exploration_parameter_generator.next() + elif isinstance(self._exploration_parameter_generator, DynamicExplorationParameterGenerator): + return self._exploration_parameter_generator.next(self._performance_history) + + @property + def current_ep(self): + return self._current_ep From 1e4f982e75d80fa15e217b44c8fc986c59496580 Mon Sep 17 00:00:00 2001 From: ysqyang Date: Fri, 27 Nov 2020 23:41:59 +0800 Subject: [PATCH 266/337] fixed a bug --- maro/rl/scheduling/scheduler.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/maro/rl/scheduling/scheduler.py b/maro/rl/scheduling/scheduler.py index a6c61e593..3ffa4a688 100644 --- a/maro/rl/scheduling/scheduler.py +++ b/maro/rl/scheduling/scheduler.py @@ -51,7 +51,7 @@ def __init__( def __iter__(self): return self - def __next__(self, performance): + def __next__(self): if self._current_ep == self._max_ep: raise StopIteration if self._current_ep >= self._warmup_ep: From 9bc84304bc8485e11640b892d7f1114287d1abd6 Mon Sep 17 00:00:00 2001 From: ysqyang Date: Fri, 27 Nov 2020 23:49:10 +0800 Subject: [PATCH 267/337] fixed a bug --- maro/rl/scheduling/scheduler.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/maro/rl/scheduling/scheduler.py b/maro/rl/scheduling/scheduler.py index 3ffa4a688..a8d380b97 100644 --- a/maro/rl/scheduling/scheduler.py +++ b/maro/rl/scheduling/scheduler.py @@ -39,7 +39,7 @@ def __init__( self._performance_history = [] if exploration_parameter_generator_cls is None: self._exploration_parameter_generator = None - elif exploration_parameter_generator_cls == StaticExplorationParameterGenerator: + elif issubclass(exploration_parameter_generator_cls, StaticExplorationParameterGenerator): self._exploration_parameter_generator = StaticExplorationParameterGenerator( max_ep, **exploration_parameter_generator_config ) From d15a76131ed0b101044acf12da79983ddd52b5e9 Mon Sep 17 00:00:00 2001 From: ysqyang Date: Fri, 27 Nov 2020 23:50:16 +0800 Subject: [PATCH 268/337] fixed a bug --- maro/rl/scheduling/scheduler.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/maro/rl/scheduling/scheduler.py b/maro/rl/scheduling/scheduler.py index a8d380b97..85d67df08 100644 --- a/maro/rl/scheduling/scheduler.py +++ b/maro/rl/scheduling/scheduler.py @@ -40,11 +40,11 @@ def __init__( if exploration_parameter_generator_cls is None: self._exploration_parameter_generator = None elif issubclass(exploration_parameter_generator_cls, StaticExplorationParameterGenerator): - self._exploration_parameter_generator = StaticExplorationParameterGenerator( + self._exploration_parameter_generator = exploration_parameter_generator_cls( max_ep, **exploration_parameter_generator_config ) else: - self._exploration_parameter_generator = DynamicExplorationParameterGenerator( + self._exploration_parameter_generator = exploration_parameter_generator_cls( **exploration_parameter_generator_config ) From fa33a91554cfe339c60f249156ed57df71bde1d0 Mon Sep 17 00:00:00 2001 From: ysqyang Date: Sat, 28 Nov 2020 00:00:31 +0800 Subject: [PATCH 269/337] fixed a bug --- maro/rl/scheduling/exploration_parameter_generator.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/maro/rl/scheduling/exploration_parameter_generator.py b/maro/rl/scheduling/exploration_parameter_generator.py index 42b15236c..ac25c5ada 100644 --- a/maro/rl/scheduling/exploration_parameter_generator.py +++ b/maro/rl/scheduling/exploration_parameter_generator.py @@ -70,7 +70,7 @@ def __init__( self._delta = (end_values - self._current_values) / (max_ep - 1) def next(self): - current_values = self._current_values + current_values = self._current_values.copy() self._current_values += self._delta return dict(zip(self._parameter_names, current_values)) @@ -130,7 +130,7 @@ def __init__( self._current_ep = 0 def next(self): - current_values = self._current_values + current_values = self._current_values.copy() self._current_values += self._delta_1 if self._current_ep < self._split_ep else self._delta_2 self._current_ep += 1 return dict(zip(self._parameter_names, current_values)) From fa34480236d192965bb54ac49341354bab92bf8a Mon Sep 17 00:00:00 2001 From: ysqyang Date: Sat, 28 Nov 2020 00:10:11 +0800 Subject: [PATCH 270/337] fixed a bug --- maro/rl/learner/simple_learner.py | 4 ++-- maro/rl/scheduling/scheduler.py | 3 +++ 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/maro/rl/learner/simple_learner.py b/maro/rl/learner/simple_learner.py index 874528f02..dabea5d71 100644 --- a/maro/rl/learner/simple_learner.py +++ b/maro/rl/learner/simple_learner.py @@ -45,9 +45,9 @@ def learn(self): model_dict=None if self._is_shared_agent_instance() else self._agent_manager.dump_models(), exploration_params=exploration_params ) - + self._scheduler.record_performance(performance) self._logger.info( - f"ep {self._scheduler.current_ep} - performance: {performance}, " + f"ep {self._scheduler.current_ep - 1} - performance: {performance}, " f"exploration_params: {exploration_params}" ) diff --git a/maro/rl/scheduling/scheduler.py b/maro/rl/scheduling/scheduler.py index 85d67df08..700947264 100644 --- a/maro/rl/scheduling/scheduler.py +++ b/maro/rl/scheduling/scheduler.py @@ -66,3 +66,6 @@ def __next__(self): @property def current_ep(self): return self._current_ep + + def record_performance(self, performance): + self._performance_history.append(performance) From 8030edd753a078644e9a6dcab2fc46de5f9c055a Mon Sep 17 00:00:00 2001 From: ysqyang Date: Sat, 28 Nov 2020 00:15:48 +0800 Subject: [PATCH 271/337] fixed lint issues --- maro/rl/__init__.py | 4 ++-- maro/rl/scheduling/scheduler.py | 6 ++---- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/maro/rl/__init__.py b/maro/rl/__init__.py index 01503baea..4b0e783aa 100644 --- a/maro/rl/__init__.py +++ b/maro/rl/__init__.py @@ -20,8 +20,8 @@ from maro.rl.models.learning_model import MultiHeadLearningModel, SingleHeadLearningModel from maro.rl.scheduling.scheduler import Scheduler from maro.rl.scheduling.exploration_parameter_generator import ( - DynamicExplorationParameterGenerator, LinearExplorationParameterGenerator, - StaticExplorationParameterGenerator, TwoPhaseLinearExplorationParameterGenerator + DynamicExplorationParameterGenerator, LinearExplorationParameterGenerator, StaticExplorationParameterGenerator, + TwoPhaseLinearExplorationParameterGenerator ) from maro.rl.shaping.abs_shaper import AbsShaper from maro.rl.shaping.action_shaper import ActionShaper diff --git a/maro/rl/scheduling/scheduler.py b/maro/rl/scheduling/scheduler.py index 700947264..6c231ff10 100644 --- a/maro/rl/scheduling/scheduler.py +++ b/maro/rl/scheduling/scheduler.py @@ -1,13 +1,11 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT license. -from typing import Callable, Union +from typing import Callable from maro.utils.exception.rl_toolkit_exception import InfiniteTrainingLoopError, InvalidEpisodeError -from .exploration_parameter_generator import ( - DynamicExplorationParameterGenerator, StaticExplorationParameterGenerator -) +from .exploration_parameter_generator import DynamicExplorationParameterGenerator, StaticExplorationParameterGenerator class Scheduler(object): From 7011d64c079e155f56f4b171d875f4f14ad174d8 Mon Sep 17 00:00:00 2001 From: ysqyang Date: Sat, 28 Nov 2020 00:23:07 +0800 Subject: [PATCH 272/337] fixed lint issue --- maro/rl/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/maro/rl/__init__.py b/maro/rl/__init__.py index 4b0e783aa..6dac4b19c 100644 --- a/maro/rl/__init__.py +++ b/maro/rl/__init__.py @@ -18,11 +18,11 @@ from maro.rl.learner.simple_learner import SimpleLearner from maro.rl.models.fc_block import FullyConnectedBlock from maro.rl.models.learning_model import MultiHeadLearningModel, SingleHeadLearningModel -from maro.rl.scheduling.scheduler import Scheduler from maro.rl.scheduling.exploration_parameter_generator import ( DynamicExplorationParameterGenerator, LinearExplorationParameterGenerator, StaticExplorationParameterGenerator, TwoPhaseLinearExplorationParameterGenerator ) +from maro.rl.scheduling.scheduler import Scheduler from maro.rl.shaping.abs_shaper import AbsShaper from maro.rl.shaping.action_shaper import ActionShaper from maro.rl.shaping.experience_shaper import ExperienceShaper From 39e99a3bcaa3703a102581eb6cffd5a00613a050 Mon Sep 17 00:00:00 2001 From: ysqyang Date: Sat, 28 Nov 2020 09:28:51 +0800 Subject: [PATCH 273/337] moved logger inside scheduler --- maro/rl/learner/simple_learner.py | 14 ++----------- maro/rl/scheduling/scheduler.py | 35 ++++++++++++++++++++++++------- 2 files changed, 29 insertions(+), 20 deletions(-) diff --git a/maro/rl/learner/simple_learner.py b/maro/rl/learner/simple_learner.py index dabea5d71..d7b4bc857 100644 --- a/maro/rl/learner/simple_learner.py +++ b/maro/rl/learner/simple_learner.py @@ -8,7 +8,6 @@ from maro.rl.agent.simple_agent_manager import SimpleAgentManager from maro.rl.dist_topologies.single_learner_multi_actor_sync_mode import ActorProxy from maro.rl.scheduling.scheduler import Scheduler -from maro.utils import DummyLogger, Logger from .abs_learner import AbsLearner @@ -22,21 +21,17 @@ class SimpleLearner(AbsLearner): (environment sampling). scheduler (AbsScheduler): A scheduler responsible for iterating over episodes and generating exploration parameters if necessary. - logger (Logger): Used to log important messages. """ def __init__( self, agent_manager: SimpleAgentManager, actor: Union[SimpleActor, ActorProxy], - scheduler: Scheduler, - logger: Logger = DummyLogger() + scheduler: Scheduler ): super().__init__() self._agent_manager = agent_manager self._actor = actor self._scheduler = scheduler - self._logger = logger - self._performance_history = [] def learn(self): """Main loop for collecting experiences from the actor and using them to update policies.""" @@ -46,11 +41,6 @@ def learn(self): exploration_params=exploration_params ) self._scheduler.record_performance(performance) - self._logger.info( - f"ep {self._scheduler.current_ep - 1} - performance: {performance}, " - f"exploration_params: {exploration_params}" - ) - self._agent_manager.train(exp_by_agent) def test(self): @@ -59,7 +49,7 @@ def test(self): model_dict=self._agent_manager.dump_models(), return_details=False ) - self._logger.info(f"test performance: {performance}") + self._scheduler.record_performance(performance) def exit(self, code: int = 0): """Tell the remote actor to exit.""" diff --git a/maro/rl/scheduling/scheduler.py b/maro/rl/scheduling/scheduler.py index 6c231ff10..971b070a7 100644 --- a/maro/rl/scheduling/scheduler.py +++ b/maro/rl/scheduling/scheduler.py @@ -3,6 +3,7 @@ from typing import Callable +from maro.utils import DummyLogger, Logger from maro.utils.exception.rl_toolkit_exception import InfiniteTrainingLoopError, InvalidEpisodeError from .exploration_parameter_generator import DynamicExplorationParameterGenerator, StaticExplorationParameterGenerator @@ -13,6 +14,14 @@ class Scheduler(object): Args: max_ep (int): Maximum number of episodes to be run. + warmup_ep (int): Episode from which early stopping checking is initiated. + early_stopping_callback (Callable): Function that returns a boolean indicating whether early stopping should + be triggered. Defaults to None, in which case no early stopping check will be performed. + exploration_parameter_generator_cls: Subclass of StaticExplorationParameterGenerator or + DynamicExplorationParameterGenerator. Defaults to None, which means no exploration outside the algorithm. + exploration_parameter_generator_config (dict): Configuration for the exploration parameter generator. + Defaults to None. + logger (Logger): Used to log important messages. """ def __init__( @@ -21,7 +30,8 @@ def __init__( warmup_ep: int = 0, early_stopping_callback: Callable = None, exploration_parameter_generator_cls=None, - exploration_parameter_generator_config: dict = None + exploration_parameter_generator_config: dict = None, + logger: Logger = DummyLogger() ): if max_ep < -1: raise InvalidEpisodeError("max_episode can only be a non-negative integer or -1.") @@ -35,16 +45,20 @@ def __init__( self._early_stopping_callback = early_stopping_callback self._current_ep = 0 self._performance_history = [] - if exploration_parameter_generator_cls is None: - self._exploration_parameter_generator = None - elif issubclass(exploration_parameter_generator_cls, StaticExplorationParameterGenerator): + self._exploration_params = None + + if issubclass(exploration_parameter_generator_cls, StaticExplorationParameterGenerator): self._exploration_parameter_generator = exploration_parameter_generator_cls( max_ep, **exploration_parameter_generator_config ) - else: + elif issubclass(exploration_parameter_generator_cls, DynamicExplorationParameterGenerator): self._exploration_parameter_generator = exploration_parameter_generator_cls( **exploration_parameter_generator_config ) + else: + self._exploration_parameter_generator = None + + self._logger = logger def __iter__(self): return self @@ -55,11 +69,12 @@ def __next__(self): if self._current_ep >= self._warmup_ep: if self._early_stopping_callback and self._early_stopping_callback(self._performance_history): raise StopIteration - self._current_ep += 1 if isinstance(self._exploration_parameter_generator, StaticExplorationParameterGenerator): - return self._exploration_parameter_generator.next() + self._exploration_params = self._exploration_parameter_generator.next() elif isinstance(self._exploration_parameter_generator, DynamicExplorationParameterGenerator): - return self._exploration_parameter_generator.next(self._performance_history) + self._exploration_params = self._exploration_parameter_generator.next(self._performance_history) + + return self._exploration_params @property def current_ep(self): @@ -67,3 +82,7 @@ def current_ep(self): def record_performance(self, performance): self._performance_history.append(performance) + self._logger.info( + f"ep {self._current_ep} - performance: {performance}, exploration_params: {self._exploration_params}" + ) + self._current_ep += 1 From 13f1843ef2a0398df031ac9a0d4552212116f773 Mon Sep 17 00:00:00 2001 From: ysqyang Date: Sat, 28 Nov 2020 09:30:44 +0800 Subject: [PATCH 274/337] fixed a bug --- examples/cim/dqn/single_process_launcher.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/examples/cim/dqn/single_process_launcher.py b/examples/cim/dqn/single_process_launcher.py index dd3788a93..47e6e84df 100644 --- a/examples/cim/dqn/single_process_launcher.py +++ b/examples/cim/dqn/single_process_launcher.py @@ -72,15 +72,11 @@ def early_stopping_callback(perf_history): early_stopping_callback=early_stopping_callback, exploration_parameter_generator_cls=TwoPhaseLinearExplorationParameterGenerator, exploration_parameter_generator_config=config.main_loop.exploration, + logger=Logger("single_host_cim_learner", auto_timestamp=False) ) actor = SimpleActor(env, agent_manager) - learner = SimpleLearner( - agent_manager=agent_manager, - actor=actor, - scheduler=scheduler, - logger=Logger("single_host_cim_learner", auto_timestamp=False) - ) + learner = SimpleLearner(agent_manager, actor, scheduler) learner.learn() learner.test() learner.dump_models(os.path.join(os.getcwd(), "models")) From 27c64fa63f66397d4d7c9f777d2286e707d263ed Mon Sep 17 00:00:00 2001 From: ysqyang Date: Sat, 28 Nov 2020 09:35:16 +0800 Subject: [PATCH 275/337] fixed a bug --- maro/rl/scheduling/scheduler.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/maro/rl/scheduling/scheduler.py b/maro/rl/scheduling/scheduler.py index 971b070a7..409e35c32 100644 --- a/maro/rl/scheduling/scheduler.py +++ b/maro/rl/scheduling/scheduler.py @@ -47,7 +47,9 @@ def __init__( self._performance_history = [] self._exploration_params = None - if issubclass(exploration_parameter_generator_cls, StaticExplorationParameterGenerator): + if exploration_parameter_generator_cls is None: + self._exploration_parameter_generator = None + elif issubclass(exploration_parameter_generator_cls, StaticExplorationParameterGenerator): self._exploration_parameter_generator = exploration_parameter_generator_cls( max_ep, **exploration_parameter_generator_config ) @@ -55,8 +57,6 @@ def __init__( self._exploration_parameter_generator = exploration_parameter_generator_cls( **exploration_parameter_generator_config ) - else: - self._exploration_parameter_generator = None self._logger = logger From 54bb17e87aaa4c2e60cc1d7ee252236d5ab8e3b3 Mon Sep 17 00:00:00 2001 From: ysqyang Date: Sat, 28 Nov 2020 09:40:36 +0800 Subject: [PATCH 276/337] fixed a bug --- maro/rl/scheduling/scheduler.py | 9 ++++++++- maro/utils/exception/error_code.py | 3 ++- maro/utils/exception/rl_toolkit_exception.py | 9 +++++++++ 3 files changed, 19 insertions(+), 2 deletions(-) diff --git a/maro/rl/scheduling/scheduler.py b/maro/rl/scheduling/scheduler.py index 409e35c32..bcbcd9f9e 100644 --- a/maro/rl/scheduling/scheduler.py +++ b/maro/rl/scheduling/scheduler.py @@ -4,7 +4,9 @@ from typing import Callable from maro.utils import DummyLogger, Logger -from maro.utils.exception.rl_toolkit_exception import InfiniteTrainingLoopError, InvalidEpisodeError +from maro.utils.exception.rl_toolkit_exception import ( + InfiniteTrainingLoopError, InvalidEpisodeError, UnrecognizedExplorationParameterGeneratorClass +) from .exploration_parameter_generator import DynamicExplorationParameterGenerator, StaticExplorationParameterGenerator @@ -57,6 +59,11 @@ def __init__( self._exploration_parameter_generator = exploration_parameter_generator_cls( **exploration_parameter_generator_config ) + else: + raise UnrecognizedExplorationParameterGeneratorClass( + f"exploration_parameter_generator_cls must be a subclass of StaticExplorationParameterGenerator " + f"or DynamicExplorationParameterGenerator" + ) self._logger = logger diff --git a/maro/utils/exception/error_code.py b/maro/utils/exception/error_code.py index 8965e7a27..e46aef585 100644 --- a/maro/utils/exception/error_code.py +++ b/maro/utils/exception/error_code.py @@ -44,5 +44,6 @@ 4003: "Wrong Agent Manager Mode", 4004: "Store Misalignment Error", 4005: "Invalid Episode", - 4006: "Infinite Training Loop" + 4006: "Infinite Training Loop", + 4010: "Unrecognized Exploration Parameter Generator Class", } diff --git a/maro/utils/exception/rl_toolkit_exception.py b/maro/utils/exception/rl_toolkit_exception.py index c35e0b897..76364b1fe 100644 --- a/maro/utils/exception/rl_toolkit_exception.py +++ b/maro/utils/exception/rl_toolkit_exception.py @@ -39,3 +39,12 @@ class InfiniteTrainingLoopError(MAROException): """Raised when the ``SimpleLearner``'s training loop becomes infinite.""" def __init__(self, msg: str = None): super().__init__(4006, msg) + + +class UnrecognizedExplorationParameterGeneratorClass(MAROException): + """ + Raised when the ``exploration_parameter_generator_cls`` passed to a ``Scheduler`` is not a subclass of + ``StaticExplorationParameterGenerator`` or ``DynamicExplorationParameterGenerator``. + """ + def __init__(self, msg: str = None): + super().__init__(4009, msg) From c210605a264c1ac95826f08f31ff9c62264e7b83 Mon Sep 17 00:00:00 2001 From: ysqyang Date: Sat, 28 Nov 2020 09:47:24 +0800 Subject: [PATCH 277/337] fixed lint issues --- maro/rl/scheduling/scheduler.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/maro/rl/scheduling/scheduler.py b/maro/rl/scheduling/scheduler.py index bcbcd9f9e..479a8632b 100644 --- a/maro/rl/scheduling/scheduler.py +++ b/maro/rl/scheduling/scheduler.py @@ -61,8 +61,8 @@ def __init__( ) else: raise UnrecognizedExplorationParameterGeneratorClass( - f"exploration_parameter_generator_cls must be a subclass of StaticExplorationParameterGenerator " - f"or DynamicExplorationParameterGenerator" + "exploration_parameter_generator_cls must be a subclass of StaticExplorationParameterGenerator " + "or DynamicExplorationParameterGenerator" ) self._logger = logger From e4b4e65f794350e6d471650f247d698ff39dc200 Mon Sep 17 00:00:00 2001 From: ysqyang Date: Sat, 28 Nov 2020 12:18:09 +0800 Subject: [PATCH 278/337] fixed lint issue --- maro/rl/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/maro/rl/__init__.py b/maro/rl/__init__.py index 90f827c3e..b448a1eaf 100644 --- a/maro/rl/__init__.py +++ b/maro/rl/__init__.py @@ -19,7 +19,7 @@ from maro.rl.learner.simple_learner import SimpleLearner from maro.rl.models.abs_block import AbsBlock from maro.rl.models.fc_block import FullyConnectedBlock -from maro.rl.models.learning_model import LearningModuleManager, LearningModule, OptimizerOptions +from maro.rl.models.learning_model import LearningModule, LearningModuleManager, OptimizerOptions from maro.rl.scheduling.exploration_parameter_generator import ( DynamicExplorationParameterGenerator, LinearExplorationParameterGenerator, StaticExplorationParameterGenerator, TwoPhaseLinearExplorationParameterGenerator From d094903a0f322a6cbc7d6d94672c17e7314f0349 Mon Sep 17 00:00:00 2001 From: ysqyang Date: Mon, 30 Nov 2020 16:10:50 +0800 Subject: [PATCH 279/337] removed epsilon parameter from choose_action --- maro/rl/algorithms/abs_algorithm.py | 5 +---- maro/rl/algorithms/dqn.py | 15 ++++++--------- 2 files changed, 7 insertions(+), 13 deletions(-) diff --git a/maro/rl/algorithms/abs_algorithm.py b/maro/rl/algorithms/abs_algorithm.py index f1f8d7b00..b8ccb539f 100644 --- a/maro/rl/algorithms/abs_algorithm.py +++ b/maro/rl/algorithms/abs_algorithm.py @@ -15,14 +15,11 @@ def __init__(self): pass @abstractmethod - def choose_action(self, state, epsilon: float = None): + def choose_action(self, state): """This method uses the underlying model(s) to compute an action from a shaped state. Args: state: A state object shaped by a ``StateShaper`` to conform to the model input format. - epsilon (float, optional): Exploration rate. For greedy value-based algorithms, this being None means - using the model output without exploration. For algorithms with inherently stochastic policies such - as policy gradient, this is usually ignored. Defaults to None. Returns: The action to be taken given ``state``. It is usually necessary to use an ``ActionShaper`` to convert diff --git a/maro/rl/algorithms/dqn.py b/maro/rl/algorithms/dqn.py index f5a601d10..55b0e37df 100644 --- a/maro/rl/algorithms/dqn.py +++ b/maro/rl/algorithms/dqn.py @@ -82,15 +82,12 @@ def __init__( def is_trainable(self): return self._is_trainable - def choose_action(self, state: np.ndarray, epsilon: float = None): - if epsilon is None or np.random.rand() > epsilon: - state = torch.from_numpy(state).unsqueeze(0) - self._eval_model.eval() - with torch.no_grad(): - q_values = self._get_q_values(state) - return q_values.argmax(dim=1).item() - - return np.random.choice(self._hyper_params.num_actions) + def choose_action(self, state: np.ndarray): + state = torch.from_numpy(state).unsqueeze(0) + self._eval_model.eval() + with torch.no_grad(): + q_values = self._get_q_values(state) + return q_values.argmax(dim=1).item() def train(self, states: np.ndarray, actions: np.ndarray, rewards: np.ndarray, next_states: np.ndarray): if not self._is_trainable: From 4fc1cf68e57f49aa4fc73325846df28a60784716 Mon Sep 17 00:00:00 2001 From: ysqyang Date: Mon, 30 Nov 2020 16:13:31 +0800 Subject: [PATCH 280/337] removed epsilon parameter from choose_action --- maro/rl/algorithms/abs_algorithm.py | 5 +---- maro/rl/algorithms/dqn.py | 8 ++------ maro/rl/algorithms/utils.py | 6 ++++-- 3 files changed, 7 insertions(+), 12 deletions(-) diff --git a/maro/rl/algorithms/abs_algorithm.py b/maro/rl/algorithms/abs_algorithm.py index df4bb21d8..4df4aa373 100644 --- a/maro/rl/algorithms/abs_algorithm.py +++ b/maro/rl/algorithms/abs_algorithm.py @@ -26,14 +26,11 @@ def model(self): return self._model @abstractmethod - def choose_action(self, state, epsilon: float = None): + def choose_action(self, state): """This method uses the underlying model(s) to compute an action from a shaped state. Args: state: A state object shaped by a ``StateShaper`` to conform to the model input format. - epsilon (float, optional): Exploration rate. For greedy value-based algorithms, this being None means - using the model output without exploration. For algorithms with inherently stochastic policies such - as policy gradient, this is usually ignored. Defaults to None. Returns: The action to be taken given ``state``. It is usually necessary to use an ``ActionShaper`` to convert diff --git a/maro/rl/algorithms/dqn.py b/maro/rl/algorithms/dqn.py index 5980dc28a..d12c37e3d 100644 --- a/maro/rl/algorithms/dqn.py +++ b/maro/rl/algorithms/dqn.py @@ -76,12 +76,8 @@ def __init__(self, model: LearningModuleManager, config: DQNConfig): self._target_model = model.copy() if model.is_trainable else None @expand_dim - def choose_action(self, state: np.ndarray, epsilon=None): - if epsilon is None or np.random.rand() > epsilon: - q_values = self._get_q_values(self._model, state, is_training=False) - return q_values.argmax(dim=1).item() - - return np.random.choice(self._config.num_actions) + def choose_action(self, state: np.ndarray): + return self._get_q_values(self._model, state, is_training=False).argmax(dim=1).data def _get_q_values(self, model, states, is_training: bool = True): if self._config.advantage_mode is not None: diff --git a/maro/rl/algorithms/utils.py b/maro/rl/algorithms/utils.py index e4c7a9faf..6160f8329 100644 --- a/maro/rl/algorithms/utils.py +++ b/maro/rl/algorithms/utils.py @@ -56,8 +56,10 @@ def expand_dim(func): def wrapper(self, state, **kwargs): if isinstance(state, np.ndarray): state = torch.from_numpy(state).to(device) - if len(state.shape) == 1: + is_single = len(state.shape) == 1 + if is_single: state = state.unsqueeze(dim=0) - return func(self, state, **kwargs) + result = func(self, state, **kwargs) + return result.item() if is_single else result.numpy() return wrapper From 5b40972eab27aa437674eceefc67b55d487a0aae Mon Sep 17 00:00:00 2001 From: ysqyang Date: Thu, 3 Dec 2020 17:19:45 +0800 Subject: [PATCH 281/337] changed agent manager's train parameter to experience_by_agent --- maro/rl/agent/abs_agent_manager.py | 2 +- maro/rl/agent/simple_agent_manager.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/maro/rl/agent/abs_agent_manager.py b/maro/rl/agent/abs_agent_manager.py index b5bb452cf..0989aaf94 100644 --- a/maro/rl/agent/abs_agent_manager.py +++ b/maro/rl/agent/abs_agent_manager.py @@ -83,7 +83,7 @@ def post_process(self, *args, **kwargs): return NotImplemented @abstractmethod - def train(self, *args, **kwargs): + def train(self, experience_by_agent: dict): """Train the agents.""" return NotImplemented diff --git a/maro/rl/agent/simple_agent_manager.py b/maro/rl/agent/simple_agent_manager.py index fc2a4c12d..057cb3997 100644 --- a/maro/rl/agent/simple_agent_manager.py +++ b/maro/rl/agent/simple_agent_manager.py @@ -79,7 +79,7 @@ def post_process(self, snapshot_list): return experiences @abstractmethod - def train(self, *args, **kwargs): + def train(self, experiences_by_agent: dict): """Train all agents.""" return NotImplementedError From 7ac67c3369a62e97ed2a1f6a2ce0e866fcd4fcc3 Mon Sep 17 00:00:00 2001 From: ysqyang Date: Fri, 11 Dec 2020 13:35:46 +0800 Subject: [PATCH 282/337] fixed some PR comments --- examples/cim/dqn/components/agent.py | 5 +++++ examples/cim/dqn/dist_actor.py | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/examples/cim/dqn/components/agent.py b/examples/cim/dqn/components/agent.py index 790a2baa3..bd0dbaf02 100644 --- a/examples/cim/dqn/components/agent.py +++ b/examples/cim/dqn/components/agent.py @@ -10,6 +10,11 @@ class CIMAgent(AbsAgent): """Implementation of AbsAgent for the DQN algorithm. Args: + name (str): Agent's name. + algorithm (AbsAlgorithm): A concrete algorithm instance that inherits from AbstractAlgorithm. + explorer (AbsExplorer): Explorer instance to generate exploratory actions. + experience_pool (AbsStore): It is used to store experiences processed by the experience shaper, which will be + used by some value-based algorithms, such as DQN. min_experiences_to_train: minimum number of experiences required for training. num_batches: number of batches to train the DQN model on per call to ``train``. batch_size: mini-batch size. diff --git a/examples/cim/dqn/dist_actor.py b/examples/cim/dqn/dist_actor.py index 2357f0822..56a2f9e98 100644 --- a/examples/cim/dqn/dist_actor.py +++ b/examples/cim/dqn/dist_actor.py @@ -47,7 +47,7 @@ def launch(config, distributed_config): "max_retries": 15 } actor_worker = ActorWorker( - local_actor=SimpleActor(env, agent_manager), + local_actor=SimpleActor(env=env, agent_manager=agent_manager), proxy_params=proxy_params ) actor_worker.launch() From 92702a61d742cb3a3fb266e030438c8b8d3de357 Mon Sep 17 00:00:00 2001 From: ysqyang Date: Mon, 14 Dec 2020 12:34:48 +0800 Subject: [PATCH 283/337] renamed zero_grad to zero_gradients in LearningModule --- maro/rl/models/learning_model.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/maro/rl/models/learning_model.py b/maro/rl/models/learning_model.py index 854b9e021..93034cfa4 100644 --- a/maro/rl/models/learning_model.py +++ b/maro/rl/models/learning_model.py @@ -74,7 +74,7 @@ def forward(self, inputs): """ return self._net(inputs) - def zero_grad(self): + def zero_gradients(self): if not self._is_trainable: raise MissingOptimizerError("No optimizer registered to the model") self._optimizer.zero_grad() @@ -165,9 +165,9 @@ def forward(self, inputs, task_name: str = None, is_training: bool = True): def learn(self, loss): """Use the loss to back-propagate gradients and apply them to the underlying parameters.""" for task_module in self._task_module_dict.values(): - task_module.zero_grad() + task_module.zero_gradients() if self._shared_module is not None: - self._shared_module.zero_grad() + self._shared_module.zero_gradients() # Obtain gradients through back-propagation loss.backward() From 4b022785e73fa8d918a851027d13e927ec43baee Mon Sep 17 00:00:00 2001 From: ysqyang Date: Wed, 16 Dec 2020 00:04:59 +0800 Subject: [PATCH 284/337] fixed some PR comments --- examples/cim/dqn/components/agent_manager.py | 2 +- examples/cim/dqn/config.yml | 45 +++++++++---------- examples/cim/dqn/dist_actor.py | 12 ++--- examples/cim/dqn/single_process_launcher.py | 22 +++------ maro/rl/learner/simple_learner.py | 10 ++++- .../exploration_parameter_generator.py | 12 +++-- maro/rl/scheduling/scheduler.py | 14 +++--- 7 files changed, 50 insertions(+), 67 deletions(-) diff --git a/examples/cim/dqn/components/agent_manager.py b/examples/cim/dqn/components/agent_manager.py index f47a38d97..e63771ee1 100644 --- a/examples/cim/dqn/components/agent_manager.py +++ b/examples/cim/dqn/components/agent_manager.py @@ -42,7 +42,7 @@ def create_dqn_agents(agent_id_list, config): experience_pool = ColumnBasedStore(**config.experience_pool) agent_dict[agent_id] = CIMAgent( - agent_id, algorithm, EpsilonGreedyExplorer(num_actions), experience_pool, + agent_id, algorithm, EpsilonGreedyExplorer(num_actions, epsilon=.0), experience_pool, **config.training_loop_parameters ) diff --git a/examples/cim/dqn/config.yml b/examples/cim/dqn/config.yml index e02e814e9..dc05c0055 100644 --- a/examples/cim/dqn/config.yml +++ b/examples/cim/dqn/config.yml @@ -2,6 +2,26 @@ env: scenario: "cim" topology: "toy.4p_ssdd_l0.0" durations: 1120 + state_shaping: + look_back: 7 + max_ports_downstream: 2 + port_attributes: + - "empty" + - "full" + - "on_shipper" + - "on_consignee" + - "booking" + - "shortage" + - "fulfillment" + vessel_attributes: + - "empty" + - "full" + - "remaining_space" + experience_shaping: + time_window: 100 + fulfillment_factor: 1.0 + shortage_factor: 1.0 + time_decay_factor: 0.97 main_loop: max_episode: 500 exploration: @@ -18,31 +38,6 @@ main_loop: perf_stability_threshold: 0.1 # stability is measured by the maximum of abs(perf_(i+1) - perf_i) / perf_i # over the last k eisodes (where perf is short for performance). This value must # be below this threshold to trigger early stopping -state_shaping: - look_back: 7 - max_ports_downstream: 2 - port_attributes: - - "empty" - - "full" - - "on_shipper" - - "on_consignee" - - "booking" - - "shortage" - - "fulfillment" - vessel_attributes: - - "empty" - - "full" - - "remaining_space" -experience_shaping: - type: "truncated" - k_step: - reward_decay: 0.9 - steps: 5 - truncated: - time_window: 100 - fulfillment_factor: 1.0 - shortage_factor: 1.0 - time_decay_factor: 0.97 agents: algorithm: num_actions: 21 diff --git a/examples/cim/dqn/dist_actor.py b/examples/cim/dqn/dist_actor.py index 56a2f9e98..19c9a1cf1 100644 --- a/examples/cim/dqn/dist_actor.py +++ b/examples/cim/dqn/dist_actor.py @@ -11,7 +11,7 @@ from components.experience_shaper import TruncatedExperienceShaper from components.state_shaper import CIMStateShaper -from maro.rl import ActorWorker, AgentManagerMode, KStepExperienceShaper, SimpleActor +from maro.rl import ActorWorker, AgentManagerMode, SimpleActor from maro.simulator import Env from maro.utils import convert_dottable @@ -22,15 +22,9 @@ def launch(config, distributed_config): distributed_config = convert_dottable(distributed_config) env = Env(config.env.scenario, config.env.topology, durations=config.env.durations) agent_id_list = [str(agent_id) for agent_id in env.agent_idx_list] - state_shaper = CIMStateShaper(**config.state_shaping) + state_shaper = CIMStateShaper(**config.env.state_shaping) action_shaper = CIMActionShaper(action_space=list(np.linspace(-1.0, 1.0, config.agents.algorithm.num_actions))) - if config.experience_shaping.type == "truncated": - experience_shaper = TruncatedExperienceShaper(**config.experience_shaping.truncated) - else: - experience_shaper = KStepExperienceShaper( - reward_func=lambda mt: 1 - mt["container_shortage"] / mt["order_requirements"], - **config.experience_shaping.k_step - ) + experience_shaper = TruncatedExperienceShaper(**config.env.experience_shaping) agent_manager = DQNAgentManager( name="distributed_cim_actor", diff --git a/examples/cim/dqn/single_process_launcher.py b/examples/cim/dqn/single_process_launcher.py index 47e6e84df..f9c69b662 100644 --- a/examples/cim/dqn/single_process_launcher.py +++ b/examples/cim/dqn/single_process_launcher.py @@ -12,10 +12,7 @@ from components.experience_shaper import TruncatedExperienceShaper from components.state_shaper import CIMStateShaper -from maro.rl import ( - AgentManagerMode, KStepExperienceShaper, Scheduler, SimpleActor, SimpleLearner, - StaticExplorationParameterGenerator, TwoPhaseLinearExplorationParameterGenerator -) +from maro.rl import AgentManagerMode, Scheduler, SimpleActor, SimpleLearner, TwoPhaseLinearExplorationParameterGenerator from maro.simulator import Env from maro.utils import Logger, convert_dottable @@ -31,15 +28,9 @@ def launch(config): # Step 2: Create state, action and experience shapers. We also need to create an explorer here due to the # greedy nature of the DQN algorithm. - state_shaper = CIMStateShaper(**config.state_shaping) + state_shaper = CIMStateShaper(**config.env.state_shaping) action_shaper = CIMActionShaper(action_space=action_space) - if config.experience_shaping.type == "truncated": - experience_shaper = TruncatedExperienceShaper(**config.experience_shaping.truncated) - else: - experience_shaper = KStepExperienceShaper( - reward_func=lambda mt: 1 - mt["container_shortage"] / mt["order_requirements"], - **config.experience_shaping.k_step - ) + experience_shaper = TruncatedExperienceShaper(**config.env.experience_shaping) # Step 3: Create agents and an agent manager. agent_manager = DQNAgentManager( @@ -71,12 +62,13 @@ def early_stopping_callback(perf_history): warmup_ep=config.main_loop.early_stopping.warmup_ep, early_stopping_callback=early_stopping_callback, exploration_parameter_generator_cls=TwoPhaseLinearExplorationParameterGenerator, - exploration_parameter_generator_config=config.main_loop.exploration, - logger=Logger("single_host_cim_learner", auto_timestamp=False) + exploration_parameter_generator_config=config.main_loop.exploration ) actor = SimpleActor(env, agent_manager) - learner = SimpleLearner(agent_manager, actor, scheduler) + learner = SimpleLearner( + agent_manager, actor, scheduler, logger=Logger("single_host_cim_learner", auto_timestamp=False) + ) learner.learn() learner.test() learner.dump_models(os.path.join(os.getcwd(), "models")) diff --git a/maro/rl/learner/simple_learner.py b/maro/rl/learner/simple_learner.py index d7b4bc857..ec9bd6fe8 100644 --- a/maro/rl/learner/simple_learner.py +++ b/maro/rl/learner/simple_learner.py @@ -8,6 +8,7 @@ from maro.rl.agent.simple_agent_manager import SimpleAgentManager from maro.rl.dist_topologies.single_learner_multi_actor_sync_mode import ActorProxy from maro.rl.scheduling.scheduler import Scheduler +from maro.utils import DummyLogger, Logger from .abs_learner import AbsLearner @@ -21,17 +22,20 @@ class SimpleLearner(AbsLearner): (environment sampling). scheduler (AbsScheduler): A scheduler responsible for iterating over episodes and generating exploration parameters if necessary. + logger (Logger): Used to log important messages. """ def __init__( self, agent_manager: SimpleAgentManager, actor: Union[SimpleActor, ActorProxy], - scheduler: Scheduler + scheduler: Scheduler, + logger: Logger = DummyLogger() ): super().__init__() self._agent_manager = agent_manager self._actor = actor self._scheduler = scheduler + self._logger = logger def learn(self): """Main loop for collecting experiences from the actor and using them to update policies.""" @@ -41,6 +45,10 @@ def learn(self): exploration_params=exploration_params ) self._scheduler.record_performance(performance) + self._logger.info( + f"ep {self._scheduler.current_ep} - performance: {performance}, " + f"exploration_params: {self._scheduler.exploration_params}" + ) self._agent_manager.train(exp_by_agent) def test(self): diff --git a/maro/rl/scheduling/exploration_parameter_generator.py b/maro/rl/scheduling/exploration_parameter_generator.py index ac25c5ada..355decff1 100644 --- a/maro/rl/scheduling/exploration_parameter_generator.py +++ b/maro/rl/scheduling/exploration_parameter_generator.py @@ -1,33 +1,31 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT license. -from abc import abstractmethod +from abc import ABC, abstractmethod from typing import Union import numpy as np -class StaticExplorationParameterGenerator(object): +class StaticExplorationParameterGenerator(ABC): """Exploration parameter generator based on a pre-defined schedule. Args: max_ep (int): Maximum number of episodes to run. """ - def __init__(self, max_ep: int, **config): + def __init__(self, max_ep: int): super().__init__() self._max_ep = max_ep - self._config = config @abstractmethod def next(self): raise NotImplementedError -class DynamicExplorationParameterGenerator(object): +class DynamicExplorationParameterGenerator(ABC): """Dynamic exploration parameter generator based on the performances history.""" - def __init__(self, **config): + def __init__(self): super().__init__() - self._config = config @abstractmethod def next(self, performance_history: list): diff --git a/maro/rl/scheduling/scheduler.py b/maro/rl/scheduling/scheduler.py index 479a8632b..0967bf071 100644 --- a/maro/rl/scheduling/scheduler.py +++ b/maro/rl/scheduling/scheduler.py @@ -3,7 +3,6 @@ from typing import Callable -from maro.utils import DummyLogger, Logger from maro.utils.exception.rl_toolkit_exception import ( InfiniteTrainingLoopError, InvalidEpisodeError, UnrecognizedExplorationParameterGeneratorClass ) @@ -23,7 +22,6 @@ class Scheduler(object): DynamicExplorationParameterGenerator. Defaults to None, which means no exploration outside the algorithm. exploration_parameter_generator_config (dict): Configuration for the exploration parameter generator. Defaults to None. - logger (Logger): Used to log important messages. """ def __init__( @@ -32,8 +30,7 @@ def __init__( warmup_ep: int = 0, early_stopping_callback: Callable = None, exploration_parameter_generator_cls=None, - exploration_parameter_generator_config: dict = None, - logger: Logger = DummyLogger() + exploration_parameter_generator_config: dict = None ): if max_ep < -1: raise InvalidEpisodeError("max_episode can only be a non-negative integer or -1.") @@ -65,8 +62,6 @@ def __init__( "or DynamicExplorationParameterGenerator" ) - self._logger = logger - def __iter__(self): return self @@ -87,9 +82,10 @@ def __next__(self): def current_ep(self): return self._current_ep + @property + def exploration_params(self): + return self._exploration_params + def record_performance(self, performance): self._performance_history.append(performance) - self._logger.info( - f"ep {self._current_ep} - performance: {performance}, exploration_params: {self._exploration_params}" - ) self._current_ep += 1 From 752b8b4f64d1a91bddf780da1f4a3ec735fa5210 Mon Sep 17 00:00:00 2001 From: ysqyang Date: Wed, 16 Dec 2020 00:07:06 +0800 Subject: [PATCH 285/337] bug fix --- examples/cim/dqn/dist_actor.py | 2 +- examples/cim/dqn/dist_learner.py | 2 +- examples/cim/dqn/single_process_launcher.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/examples/cim/dqn/dist_actor.py b/examples/cim/dqn/dist_actor.py index 19c9a1cf1..b1fd4643b 100644 --- a/examples/cim/dqn/dist_actor.py +++ b/examples/cim/dqn/dist_actor.py @@ -17,7 +17,7 @@ def launch(config, distributed_config): - set_input_dim(config) + set_input_dim(config.env) config = convert_dottable(config) distributed_config = convert_dottable(distributed_config) env = Env(config.env.scenario, config.env.topology, durations=config.env.durations) diff --git a/examples/cim/dqn/dist_learner.py b/examples/cim/dqn/dist_learner.py index 94530e818..08cea1c35 100644 --- a/examples/cim/dqn/dist_learner.py +++ b/examples/cim/dqn/dist_learner.py @@ -15,7 +15,7 @@ def launch(config, distributed_config): - set_input_dim(config) + set_input_dim(config.env) config = convert_dottable(config) distributed_config = convert_dottable(distributed_config) env = Env(config.env.scenario, config.env.topology, durations=config.env.durations) diff --git a/examples/cim/dqn/single_process_launcher.py b/examples/cim/dqn/single_process_launcher.py index f9c69b662..abc2bf4d9 100644 --- a/examples/cim/dqn/single_process_launcher.py +++ b/examples/cim/dqn/single_process_launcher.py @@ -19,7 +19,7 @@ def launch(config): # First determine the input dimension and add it to the config. - set_input_dim(config) + set_input_dim(config.env) config = convert_dottable(config) # Step 1: Initialize a CIM environment for using a toy dataset. env = Env(config.env.scenario, config.env.topology, durations=config.env.durations) From 089b857ff2358962f6670b45e26a6902b8779f40 Mon Sep 17 00:00:00 2001 From: ysqyang Date: Wed, 16 Dec 2020 00:08:25 +0800 Subject: [PATCH 286/337] bug fix --- examples/cim/dqn/dist_actor.py | 2 +- examples/cim/dqn/dist_learner.py | 2 +- examples/cim/dqn/single_process_launcher.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/examples/cim/dqn/dist_actor.py b/examples/cim/dqn/dist_actor.py index b1fd4643b..39b9c0fcd 100644 --- a/examples/cim/dqn/dist_actor.py +++ b/examples/cim/dqn/dist_actor.py @@ -17,7 +17,7 @@ def launch(config, distributed_config): - set_input_dim(config.env) + set_input_dim(config["env"]) config = convert_dottable(config) distributed_config = convert_dottable(distributed_config) env = Env(config.env.scenario, config.env.topology, durations=config.env.durations) diff --git a/examples/cim/dqn/dist_learner.py b/examples/cim/dqn/dist_learner.py index 08cea1c35..3c5e71204 100644 --- a/examples/cim/dqn/dist_learner.py +++ b/examples/cim/dqn/dist_learner.py @@ -15,7 +15,7 @@ def launch(config, distributed_config): - set_input_dim(config.env) + set_input_dim(config["env"]) config = convert_dottable(config) distributed_config = convert_dottable(distributed_config) env = Env(config.env.scenario, config.env.topology, durations=config.env.durations) diff --git a/examples/cim/dqn/single_process_launcher.py b/examples/cim/dqn/single_process_launcher.py index abc2bf4d9..fad9bc50e 100644 --- a/examples/cim/dqn/single_process_launcher.py +++ b/examples/cim/dqn/single_process_launcher.py @@ -19,7 +19,7 @@ def launch(config): # First determine the input dimension and add it to the config. - set_input_dim(config.env) + set_input_dim(config["env"]) config = convert_dottable(config) # Step 1: Initialize a CIM environment for using a toy dataset. env = Env(config.env.scenario, config.env.topology, durations=config.env.durations) From d9eb6345c3d0ea2c498c3194fc7c3676ad1c849e Mon Sep 17 00:00:00 2001 From: ysqyang Date: Wed, 16 Dec 2020 00:09:55 +0800 Subject: [PATCH 287/337] bug fix --- examples/cim/dqn/components/config.py | 8 ++++---- examples/cim/dqn/dist_actor.py | 2 +- examples/cim/dqn/dist_learner.py | 2 +- examples/cim/dqn/single_process_launcher.py | 2 +- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/examples/cim/dqn/components/config.py b/examples/cim/dqn/components/config.py index b79b7d31c..63c041c7d 100644 --- a/examples/cim/dqn/components/config.py +++ b/examples/cim/dqn/components/config.py @@ -13,10 +13,10 @@ def set_input_dim(config): # obtain model input dimension from state shaping configurations - look_back = config["state_shaping"]["look_back"] - max_ports_downstream = config["state_shaping"]["max_ports_downstream"] - num_port_attributes = len(config["state_shaping"]["port_attributes"]) - num_vessel_attributes = len(config["state_shaping"]["vessel_attributes"]) + look_back = config["env"]["state_shaping"]["look_back"] + max_ports_downstream = config["env"]["state_shaping"]["max_ports_downstream"] + num_port_attributes = len(config["env"]["state_shaping"]["port_attributes"]) + num_vessel_attributes = len(config["env"]["state_shaping"]["vessel_attributes"]) input_dim = (look_back + 1) * (max_ports_downstream + 1) * num_port_attributes + num_vessel_attributes config["agents"]["algorithm"]["input_dim"] = input_dim diff --git a/examples/cim/dqn/dist_actor.py b/examples/cim/dqn/dist_actor.py index 39b9c0fcd..19c9a1cf1 100644 --- a/examples/cim/dqn/dist_actor.py +++ b/examples/cim/dqn/dist_actor.py @@ -17,7 +17,7 @@ def launch(config, distributed_config): - set_input_dim(config["env"]) + set_input_dim(config) config = convert_dottable(config) distributed_config = convert_dottable(distributed_config) env = Env(config.env.scenario, config.env.topology, durations=config.env.durations) diff --git a/examples/cim/dqn/dist_learner.py b/examples/cim/dqn/dist_learner.py index 3c5e71204..94530e818 100644 --- a/examples/cim/dqn/dist_learner.py +++ b/examples/cim/dqn/dist_learner.py @@ -15,7 +15,7 @@ def launch(config, distributed_config): - set_input_dim(config["env"]) + set_input_dim(config) config = convert_dottable(config) distributed_config = convert_dottable(distributed_config) env = Env(config.env.scenario, config.env.topology, durations=config.env.durations) diff --git a/examples/cim/dqn/single_process_launcher.py b/examples/cim/dqn/single_process_launcher.py index fad9bc50e..f9c69b662 100644 --- a/examples/cim/dqn/single_process_launcher.py +++ b/examples/cim/dqn/single_process_launcher.py @@ -19,7 +19,7 @@ def launch(config): # First determine the input dimension and add it to the config. - set_input_dim(config["env"]) + set_input_dim(config) config = convert_dottable(config) # Step 1: Initialize a CIM environment for using a toy dataset. env = Env(config.env.scenario, config.env.topology, durations=config.env.durations) From fd67efd01c6d92b8ca4a91c9ce7175a79c661d29 Mon Sep 17 00:00:00 2001 From: ysqyang Date: Wed, 23 Dec 2020 00:35:28 +0800 Subject: [PATCH 288/337] removed explorer abstraction from agent --- examples/cim/dqn/components/agent.py | 3 +-- examples/cim/dqn/components/agent_manager.py | 3 +-- maro/rl/actor/simple_actor.py | 2 +- maro/rl/agent/abs_agent.py | 16 +++----------- maro/rl/agent/abs_agent_manager.py | 10 ++++----- maro/rl/algorithms/abs_algorithm.py | 3 +++ maro/rl/algorithms/dqn.py | 21 +++++++++++++------ .../rl/exploration/epsilon_greedy_explorer.py | 16 +++++++++----- 8 files changed, 40 insertions(+), 34 deletions(-) diff --git a/examples/cim/dqn/components/agent.py b/examples/cim/dqn/components/agent.py index bd0dbaf02..e7500774f 100644 --- a/examples/cim/dqn/components/agent.py +++ b/examples/cim/dqn/components/agent.py @@ -23,13 +23,12 @@ def __init__( self, name: str, algorithm, - explorer: EpsilonGreedyExplorer, experience_pool: ColumnBasedStore, min_experiences_to_train, num_batches, batch_size ): - super().__init__(name, algorithm, explorer=explorer, experience_pool=experience_pool) + super().__init__(name, algorithm, experience_pool=experience_pool) self._min_experiences_to_train = min_experiences_to_train self._num_batches = num_batches self._batch_size = batch_size diff --git a/examples/cim/dqn/components/agent_manager.py b/examples/cim/dqn/components/agent_manager.py index e63771ee1..6776408f2 100644 --- a/examples/cim/dqn/components/agent_manager.py +++ b/examples/cim/dqn/components/agent_manager.py @@ -40,9 +40,8 @@ def create_dqn_agents(agent_id_list, config): ) ) - experience_pool = ColumnBasedStore(**config.experience_pool) agent_dict[agent_id] = CIMAgent( - agent_id, algorithm, EpsilonGreedyExplorer(num_actions, epsilon=.0), experience_pool, + agent_id, algorithm, ColumnBasedStore(**config.experience_pool), **config.training_loop_parameters ) diff --git a/maro/rl/actor/simple_actor.py b/maro/rl/actor/simple_actor.py index 935d6c5c6..8c01e3937 100644 --- a/maro/rl/actor/simple_actor.py +++ b/maro/rl/actor/simple_actor.py @@ -44,7 +44,7 @@ def roll_out( # load exploration parameters: if exploration_params is not None: - self._agents.update_exploration_params(exploration_params) + self._agents.set_exploration_params(exploration_params) metrics, decision_event, is_done = self._env.step(None) while not is_done: diff --git a/maro/rl/agent/abs_agent.py b/maro/rl/agent/abs_agent.py index b50a43da8..d0ae99234 100644 --- a/maro/rl/agent/abs_agent.py +++ b/maro/rl/agent/abs_agent.py @@ -24,7 +24,6 @@ class AbsAgent(ABC): algorithm (AbsAlgorithm): A concrete algorithm instance that inherits from AbstractAlgorithm. This is the centerpiece of the Agent class and is responsible for the most important tasks of an agent: choosing actions and optimizing models. - explorer (AbsExplorer): Explorer instance to generate exploratory actions. Defaults to None. experience_pool (AbsStore): It is used to store experiences processed by the experience shaper, which will be used by some value-based algorithms, such as DQN. Defaults to None. """ @@ -32,12 +31,10 @@ def __init__( self, name: str, algorithm: AbsAlgorithm, - explorer: AbsExplorer = None, experience_pool: AbsStore = None ): self._name = name self._algorithm = algorithm - self._explorer = explorer self._experience_pool = experience_pool @property @@ -45,11 +42,6 @@ def algorithm(self): """Underlying algorithm employed by the agent.""" return self._algorithm - @property - def explorer(self): - """Explorer used by the agent to generate exploratory actions.""" - return self._explorer - @property def experience_pool(self): """Underlying experience pool where the agent stores experiences.""" @@ -64,12 +56,10 @@ def choose_action(self, model_state): If the agent's explorer is None, the action given by the underlying model is returned. Otherwise, an exploratory action is returned. """ - action = self._algorithm.choose_action(model_state) - return action if self._explorer is None else self._explorer(action) + return self._algorithm.choose_action(model_state) - def update(self, **exploration_params): - if self._explorer: - self._explorer.update(**exploration_params) + def set_exploration_params(self, **params): + self._algorithm.set_exploration_params(**params) @abstractmethod def train(self, *args, **kwargs): diff --git a/maro/rl/agent/abs_agent_manager.py b/maro/rl/agent/abs_agent_manager.py index b5bb452cf..1fb8a3517 100644 --- a/maro/rl/agent/abs_agent_manager.py +++ b/maro/rl/agent/abs_agent_manager.py @@ -87,15 +87,15 @@ def train(self, *args, **kwargs): """Train the agents.""" return NotImplemented - def update_exploration_params(self, exploration_params): + def set_exploration_params(self, params): # Per-agent exploration parameters - if isinstance(exploration_params, dict) and exploration_params.keys() <= self.agent_dict.keys(): - for agent_id, params in exploration_params.items(): - self.agent_dict[agent_id].update(**params) + if isinstance(params, dict) and params.keys() <= self.agent_dict.keys(): + for agent_id, params in params.items(): + self.agent_dict[agent_id].set_exploration_params(**params) # Shared exploration parameters for all agents else: for agent in self.agent_dict.values(): - agent.update(**exploration_params) + agent.set_exploration_params(**params) def _assert_train_mode(self): if self._mode != AgentManagerMode.TRAIN and self._mode != AgentManagerMode.TRAIN_INFERENCE: diff --git a/maro/rl/algorithms/abs_algorithm.py b/maro/rl/algorithms/abs_algorithm.py index b8ccb539f..8e0271081 100644 --- a/maro/rl/algorithms/abs_algorithm.py +++ b/maro/rl/algorithms/abs_algorithm.py @@ -55,3 +55,6 @@ def load_models_from_file(self, path): def dump_models_to_file(self, path: str): """Dump the algorithm's trainable models to disk.""" return NotImplementedError + + def set_exploration_params(self, **params): + pass diff --git a/maro/rl/algorithms/dqn.py b/maro/rl/algorithms/dqn.py index 55b0e37df..0abea991a 100644 --- a/maro/rl/algorithms/dqn.py +++ b/maro/rl/algorithms/dqn.py @@ -18,24 +18,27 @@ class DQNHyperParams: num_actions (int): Number of possible actions. reward_decay (float): Reward decay as defined in standard RL terminology. target_update_frequency (int): Number of training rounds between target model updates. + epsilon (float): Exploration rate for epsilon-greedy exploration. Defaults to None. tau (float): Soft update coefficient, i.e., target_model = tau * eval_model + (1 - tau) * target_model. is_double (bool): If True, the next Q values will be computed according to the double DQN algorithm, i.e., q_next = Q_target(s, argmax(Q_eval(s, a))). Otherwise, q_next = max(Q_target(s, a)). See https://arxiv.org/pdf/1509.06461.pdf for details. Defaults to False. """ - __slots__ = ["num_actions", "reward_decay", "target_update_frequency", "tau", "is_double"] + __slots__ = ["num_actions", "reward_decay", "target_update_frequency", "epsilon", "tau", "is_double"] def __init__( self, num_actions: int, reward_decay: float, target_update_frequency: int, + epsilon: float = .0, tau: float = 0.1, is_double: bool = True ): self.num_actions = num_actions self.reward_decay = reward_decay self.target_update_frequency = target_update_frequency + self.epsilon = epsilon self.tau = tau self.is_double = is_double @@ -83,11 +86,14 @@ def is_trainable(self): return self._is_trainable def choose_action(self, state: np.ndarray): - state = torch.from_numpy(state).unsqueeze(0) - self._eval_model.eval() - with torch.no_grad(): - q_values = self._get_q_values(state) - return q_values.argmax(dim=1).item() + if np.random.random() < self._hyper_params.epsilon: + return np.random.choice(self._hyper_params.num_actions) + else: + state = torch.from_numpy(state).unsqueeze(0) + self._eval_model.eval() + with torch.no_grad(): + q_values = self._get_q_values(state) + return q_values.argmax(dim=1).item() def train(self, states: np.ndarray, actions: np.ndarray, rewards: np.ndarray, next_states: np.ndarray): if not self._is_trainable: @@ -149,3 +155,6 @@ def load_models_from_file(self, path): def dump_models_to_file(self, path: str): """Dump the evaluation model to disk.""" torch.save(self._eval_model.state_dict(), path) + + def set_exploration_params(self, epsilon): + self._hyper_params.epsilon = epsilon diff --git a/maro/rl/exploration/epsilon_greedy_explorer.py b/maro/rl/exploration/epsilon_greedy_explorer.py index 2e80c381f..becddf94f 100644 --- a/maro/rl/exploration/epsilon_greedy_explorer.py +++ b/maro/rl/exploration/epsilon_greedy_explorer.py @@ -2,6 +2,9 @@ # Licensed under the MIT license. import random +from typing import Union + +import numpy as np from .abs_explorer import AbsExplorer @@ -17,12 +20,15 @@ def __init__(self, num_actions: int, epsilon: float = .0): self._num_actions = num_actions self._epsilon = epsilon - def __call__(self, action_index: int): - assert (action_index < self._num_actions), f"Invalid action: {action_index}" - if random.random() > self._epsilon: - return action_index + def __call__(self, action_index: Union[int, np.ndarray]): + if isinstance(action_index, np.ndarray): + return [self._get_exploration_action(act) for act in action_index] else: - return random.randrange(self._num_actions) + return self._get_exploration_action(action_index) def update(self, *, epsilon: float): self._epsilon = epsilon + + def _get_exploration_action(self, action_index): + assert (action_index < self._num_actions), f"Invalid action: {action_index}" + return action_index if np.random.random() > self._epsilon else np.random.choice(self._num_actions) From b29fd13ddc42068ce686f3ca7965f55e11fc2453 Mon Sep 17 00:00:00 2001 From: ysqyang Date: Wed, 23 Dec 2020 14:38:39 +0800 Subject: [PATCH 289/337] added DEVICE env variable as first choice for torch device --- examples/cim/dqn/config.yml | 1 - maro/rl/algorithms/utils.py | 8 ++++++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/examples/cim/dqn/config.yml b/examples/cim/dqn/config.yml index 7cc857d81..be7b25eb0 100644 --- a/examples/cim/dqn/config.yml +++ b/examples/cim/dqn/config.yml @@ -57,7 +57,6 @@ agents: target_update_frequency: 5 tau: 0.1 is_double: true - advantage_mode: "mean" per_sample_td_error_enabled: true experience_pool: capacity: -1 diff --git a/maro/rl/algorithms/utils.py b/maro/rl/algorithms/utils.py index 6160f8329..7d6952d56 100644 --- a/maro/rl/algorithms/utils.py +++ b/maro/rl/algorithms/utils.py @@ -3,13 +3,14 @@ from enum import Enum from functools import wraps +from os import environ import numpy as np import torch from maro.utils.exception.rl_toolkit_exception import UnrecognizedTaskError -device = torch.device("cuda" if torch.cuda.is_available() else "cpu") +device = environ.get("DEVICE", torch.device("cuda" if torch.cuda.is_available() else "cpu")) def validate_task_names(task_enum: Enum): @@ -60,6 +61,9 @@ def wrapper(self, state, **kwargs): if is_single: state = state.unsqueeze(dim=0) result = func(self, state, **kwargs) - return result.item() if is_single else result.numpy() + if isinstance(result, torch.Tensor): + return result.item() if is_single else result.numpy() + else: + return result return wrapper From 7cb9388072373a8329e78afba424baa9b473cf11 Mon Sep 17 00:00:00 2001 From: ysqyang Date: Wed, 23 Dec 2020 16:41:48 +0800 Subject: [PATCH 290/337] refined dqn example --- examples/cim/dqn/components/__init__.py | 2 - examples/cim/dqn/components/config.py | 13 ----- examples/cim/dqn/components/state_shaper.py | 13 ++--- examples/cim/dqn/config.yml | 12 ----- examples/cim/dqn/dist_actor.py | 2 +- examples/cim/dqn/dist_learner.py | 11 ++-- examples/cim/dqn/single_process_launcher.py | 56 +++++++++++++-------- maro/rl/scheduling/scheduler.py | 8 +-- 8 files changed, 49 insertions(+), 68 deletions(-) diff --git a/examples/cim/dqn/components/__init__.py b/examples/cim/dqn/components/__init__.py index 214159c25..d24def64a 100644 --- a/examples/cim/dqn/components/__init__.py +++ b/examples/cim/dqn/components/__init__.py @@ -3,14 +3,12 @@ from .action_shaper import CIMActionShaper from .agent_manager import DQNAgentManager, create_dqn_agents -from .config import set_input_dim from .experience_shaper import TruncatedExperienceShaper from .state_shaper import CIMStateShaper __all__ = [ "CIMActionShaper", "DQNAgentManager", "create_dqn_agents", - "set_input_dim", "TruncatedExperienceShaper", "CIMStateShaper" ] diff --git a/examples/cim/dqn/components/config.py b/examples/cim/dqn/components/config.py index 63c041c7d..c53868b6e 100644 --- a/examples/cim/dqn/components/config.py +++ b/examples/cim/dqn/components/config.py @@ -11,19 +11,6 @@ import yaml -def set_input_dim(config): - # obtain model input dimension from state shaping configurations - look_back = config["env"]["state_shaping"]["look_back"] - max_ports_downstream = config["env"]["state_shaping"]["max_ports_downstream"] - num_port_attributes = len(config["env"]["state_shaping"]["port_attributes"]) - num_vessel_attributes = len(config["env"]["state_shaping"]["vessel_attributes"]) - - input_dim = (look_back + 1) * (max_ports_downstream + 1) * num_port_attributes + num_vessel_attributes - config["agents"]["algorithm"]["input_dim"] = input_dim - - return config - - CONFIG_PATH = os.path.join(os.path.split(os.path.realpath(__file__))[0], "../config.yml") with io.open(CONFIG_PATH, "r") as in_file: config = yaml.safe_load(in_file) diff --git a/examples/cim/dqn/components/state_shaper.py b/examples/cim/dqn/components/state_shaper.py index b49899af5..93b1bf824 100644 --- a/examples/cim/dqn/components/state_shaper.py +++ b/examples/cim/dqn/components/state_shaper.py @@ -5,22 +5,23 @@ from maro.rl import StateShaper +PORT_ATTRIBUTES = ["empty", "full", "on_shipper", "on_consignee", "booking", "shortage", "fulfillment"] +VESSEL_ATTRIBUTES = ["empty", "full", "remaining_space"] + class CIMStateShaper(StateShaper): - def __init__(self, *, look_back, max_ports_downstream, port_attributes, vessel_attributes): + def __init__(self, *, look_back, max_ports_downstream): super().__init__() self._look_back = look_back self._max_ports_downstream = max_ports_downstream - self._port_attributes = port_attributes - self._vessel_attributes = vessel_attributes - self._dim = (look_back + 1) * (max_ports_downstream + 1) * len(port_attributes) + len(vessel_attributes) + self._dim = (look_back + 1) * (max_ports_downstream + 1) * len(PORT_ATTRIBUTES) + len(VESSEL_ATTRIBUTES) def __call__(self, decision_event, snapshot_list): tick, port_idx, vessel_idx = decision_event.tick, decision_event.port_idx, decision_event.vessel_idx ticks = [tick - rt for rt in range(self._look_back - 1)] future_port_idx_list = snapshot_list["vessels"][tick: vessel_idx: 'future_stop_list'].astype('int') - port_features = snapshot_list["ports"][ticks: [port_idx] + list(future_port_idx_list): self._port_attributes] - vessel_features = snapshot_list["vessels"][tick: vessel_idx: self._vessel_attributes] + port_features = snapshot_list["ports"][ticks: [port_idx] + list(future_port_idx_list): PORT_ATTRIBUTES] + vessel_features = snapshot_list["vessels"][tick: vessel_idx: VESSEL_ATTRIBUTES] state = np.concatenate((port_features, vessel_features)) return str(port_idx), state diff --git a/examples/cim/dqn/config.yml b/examples/cim/dqn/config.yml index dc05c0055..95014e839 100644 --- a/examples/cim/dqn/config.yml +++ b/examples/cim/dqn/config.yml @@ -5,18 +5,6 @@ env: state_shaping: look_back: 7 max_ports_downstream: 2 - port_attributes: - - "empty" - - "full" - - "on_shipper" - - "on_consignee" - - "booking" - - "shortage" - - "fulfillment" - vessel_attributes: - - "empty" - - "full" - - "remaining_space" experience_shaping: time_window: 100 fulfillment_factor: 1.0 diff --git a/examples/cim/dqn/dist_actor.py b/examples/cim/dqn/dist_actor.py index 8e546e137..4376ffae6 100644 --- a/examples/cim/dqn/dist_actor.py +++ b/examples/cim/dqn/dist_actor.py @@ -16,7 +16,6 @@ def launch(config, distributed_config): - set_input_dim(config) config = convert_dottable(config) distributed_config = convert_dottable(distributed_config) env = Env(config.env.scenario, config.env.topology, durations=config.env.durations) @@ -25,6 +24,7 @@ def launch(config, distributed_config): action_shaper = CIMActionShaper(action_space=list(np.linspace(-1.0, 1.0, config.agents.algorithm.num_actions))) experience_shaper = TruncatedExperienceShaper(**config.env.experience_shaping) + config["agents"]["algorithm"]["input_dim"] = state_shaper.dim agent_manager = DQNAgentManager( name="distributed_cim_actor", mode=AgentManagerMode.INFERENCE, diff --git a/examples/cim/dqn/dist_learner.py b/examples/cim/dqn/dist_learner.py index ffd6bf8e1..034b048f4 100644 --- a/examples/cim/dqn/dist_learner.py +++ b/examples/cim/dqn/dist_learner.py @@ -10,16 +10,16 @@ from maro.simulator import Env from maro.utils import Logger, convert_dottable -from components import DQNAgentManager, create_dqn_agents, set_input_dim +from components import CIMStateShaper, DQNAgentManager, create_dqn_agents def launch(config, distributed_config): - set_input_dim(config) config = convert_dottable(config) distributed_config = convert_dottable(distributed_config) env = Env(config.env.scenario, config.env.topology, durations=config.env.durations) agent_id_list = [str(agent_id) for agent_id in env.agent_idx_list] + config["agents"]["algorithm"]["input_dim"] = CIMStateShaper(**config.env.state_shaping).dim agent_manager = DQNAgentManager( name="distributed_cim_learner", mode=AgentManagerMode.TRAIN, @@ -29,17 +29,14 @@ def launch(config, distributed_config): proxy_params = { "group_name": os.environ["GROUP"] if "GROUP" in os.environ else distributed_config.group, "expected_peers": { - "actor": int( - os.environ["NUM_ACTORS"] if "NUM_ACTORS" in os.environ - else distributed_config.num_actors - )}, + "actor": int(os.environ["NUM_ACTORS"] if "NUM_ACTORS" in os.environ else distributed_config.num_actors) + }, "redis_address": (distributed_config.redis.hostname, distributed_config.redis.port), "max_retries": 15 } scheduler = Scheduler( config.main_loop.max_episode, - warmup_ep=config.main_loop.early_stopping.warmup_ep, exploration_parameter_generator_cls=TwoPhaseLinearExplorationParameterGenerator, exploration_parameter_generator_config=config.main_loop.exploration, ) diff --git a/examples/cim/dqn/single_process_launcher.py b/examples/cim/dqn/single_process_launcher.py index 03eed2307..bebb57351 100644 --- a/examples/cim/dqn/single_process_launcher.py +++ b/examples/cim/dqn/single_process_launcher.py @@ -11,14 +11,42 @@ from maro.simulator import Env from maro.utils import Logger, convert_dottable -from components import ( - CIMActionShaper, CIMStateShaper, DQNAgentManager, TruncatedExperienceShaper, create_dqn_agents, set_input_dim -) +from components import CIMActionShaper, CIMStateShaper, DQNAgentManager, TruncatedExperienceShaper, create_dqn_agents + + +class EarlyStopping: + """Callable class that checks the performance history to determine early stopping. + + Args: + warmup_ep (int): Episode from which early stopping checking is initiated. + last_k (int): Number of latest performance records to check for early stopping. + perf_threshold (float): The mean of the ``last_k`` performance metric values must be above this value to + trigger early stopping. + perf_stability_threshold (float): The maximum one-step change over the ``last_k`` performance metrics must be + below this value to trigger early stopping. + """ + def __init__(self, warmup_ep: int, last_k: int, perf_threshold: float, perf_stability_threshold: float): + self._warmup_ep = warmup_ep + self._last_k = last_k + self._perf_threshold = perf_threshold + self._perf_stability_threshold = perf_stability_threshold + + def get_metric(record): + return 1 - record["container_shortage"] / record["order_requirements"] + self._metric_func = get_metric + + def __call__(self, perf_history) -> bool: + if len(perf_history) < max(self._last_k, self._warmup_ep): + return False + + metric_series = list(map(self._metric_func, perf_history[-self._last_k:])) + max_delta = max( + abs(metric_series[i] - metric_series[i - 1]) / metric_series[i - 1] for i in range(1, self._last_k) + ) + return mean(metric_series) > self._perf_threshold and max_delta < self._perf_stability_threshold def launch(config): - # First determine the input dimension and add it to the config. - set_input_dim(config) config = convert_dottable(config) # Step 1: Initialize a CIM environment for using a toy dataset. env = Env(config.env.scenario, config.env.topology, durations=config.env.durations) @@ -32,6 +60,7 @@ def launch(config): experience_shaper = TruncatedExperienceShaper(**config.env.experience_shaping) # Step 3: Create agents and an agent manager. + config["agents"]["algorithm"]["input_dim"] = state_shaper.dim agent_manager = DQNAgentManager( name="cim_learner", mode=AgentManagerMode.TRAIN_INFERENCE, @@ -42,24 +71,9 @@ def launch(config): ) # Step 4: Create an actor and a learner to start the training process. - def early_stopping_callback(perf_history): - last_k = config.main_loop.early_stopping.last_k - perf_threshold = config.main_loop.early_stopping.perf_threshold - perf_stability_threshold = config.main_loop.early_stopping.perf_stability_threshold - if len(perf_history) < last_k: - return False - - metric_series = list( - map(lambda p: 1 - p["container_shortage"] / p["order_requirements"], perf_history[-last_k:]) - ) - mean_perf = mean(metric_series) - max_delta = max(abs(metric_series[i] - metric_series[i - 1]) / metric_series[i - 1] for i in range(1, last_k)) - return mean_perf > perf_threshold and max_delta < perf_stability_threshold - scheduler = Scheduler( config.main_loop.max_episode, - warmup_ep=config.main_loop.early_stopping.warmup_ep, - early_stopping_callback=early_stopping_callback, + early_stopping_callback=EarlyStopping(**config.main_loop.early_stopping), exploration_parameter_generator_cls=TwoPhaseLinearExplorationParameterGenerator, exploration_parameter_generator_config=config.main_loop.exploration ) diff --git a/maro/rl/scheduling/scheduler.py b/maro/rl/scheduling/scheduler.py index 0967bf071..ae6aef853 100644 --- a/maro/rl/scheduling/scheduler.py +++ b/maro/rl/scheduling/scheduler.py @@ -15,7 +15,6 @@ class Scheduler(object): Args: max_ep (int): Maximum number of episodes to be run. - warmup_ep (int): Episode from which early stopping checking is initiated. early_stopping_callback (Callable): Function that returns a boolean indicating whether early stopping should be triggered. Defaults to None, in which case no early stopping check will be performed. exploration_parameter_generator_cls: Subclass of StaticExplorationParameterGenerator or @@ -27,7 +26,6 @@ class Scheduler(object): def __init__( self, max_ep: int, - warmup_ep: int = 0, early_stopping_callback: Callable = None, exploration_parameter_generator_cls=None, exploration_parameter_generator_config: dict = None @@ -40,7 +38,6 @@ def __init__( "is provided. " ) self._max_ep = max_ep - self._warmup_ep = warmup_ep self._early_stopping_callback = early_stopping_callback self._current_ep = 0 self._performance_history = [] @@ -68,9 +65,8 @@ def __iter__(self): def __next__(self): if self._current_ep == self._max_ep: raise StopIteration - if self._current_ep >= self._warmup_ep: - if self._early_stopping_callback and self._early_stopping_callback(self._performance_history): - raise StopIteration + if self._early_stopping_callback and self._early_stopping_callback(self._performance_history): + raise StopIteration if isinstance(self._exploration_parameter_generator, StaticExplorationParameterGenerator): self._exploration_params = self._exploration_parameter_generator.next() elif isinstance(self._exploration_parameter_generator, DynamicExplorationParameterGenerator): From 236c9fcd7c065f11d5fa706a15abd8c343ef6cc2 Mon Sep 17 00:00:00 2001 From: ysqyang Date: Wed, 23 Dec 2020 16:57:12 +0800 Subject: [PATCH 291/337] fixed lint issues --- maro/rl/agent/abs_agent.py | 1 - 1 file changed, 1 deletion(-) diff --git a/maro/rl/agent/abs_agent.py b/maro/rl/agent/abs_agent.py index d0ae99234..725e2e43a 100644 --- a/maro/rl/agent/abs_agent.py +++ b/maro/rl/agent/abs_agent.py @@ -6,7 +6,6 @@ from abc import ABC, abstractmethod from maro.rl.algorithms.abs_algorithm import AbsAlgorithm -from maro.rl.exploration.abs_explorer import AbsExplorer from maro.rl.storage.abs_store import AbsStore From 0209a9874a3807056c02850e898149483a58064c Mon Sep 17 00:00:00 2001 From: ysqyang Date: Wed, 23 Dec 2020 17:53:07 +0800 Subject: [PATCH 292/337] removed unwanted import in cim example --- examples/cim/dqn/components/agent_manager.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/cim/dqn/components/agent_manager.py b/examples/cim/dqn/components/agent_manager.py index dc0b05a71..6517db167 100644 --- a/examples/cim/dqn/components/agent_manager.py +++ b/examples/cim/dqn/components/agent_manager.py @@ -5,8 +5,8 @@ from torch.optim import RMSprop from maro.rl import ( - ColumnBasedStore, DQN, DQNConfig, EpsilonGreedyExplorer, FullyConnectedBlock, LearningModuleManager, LearningModule, - OptimizerOptions, SimpleAgentManager + ColumnBasedStore, DQN, DQNConfig, FullyConnectedBlock, LearningModuleManager, LearningModule, OptimizerOptions, + SimpleAgentManager ) from maro.utils import set_seeds From 804fd08a27a1cea9f8f792028ccb9baa43cb7c3b Mon Sep 17 00:00:00 2001 From: ysqyang Date: Thu, 24 Dec 2020 00:26:19 +0800 Subject: [PATCH 293/337] updated cim-dqn notebook --- .../rl_formulation.ipynb | 298 ++++++------------ 1 file changed, 92 insertions(+), 206 deletions(-) diff --git a/notebooks/container_inventory_management/rl_formulation.ipynb b/notebooks/container_inventory_management/rl_formulation.ipynb index 31a3834aa..8d90b2be7 100644 --- a/notebooks/container_inventory_management/rl_formulation.ipynb +++ b/notebooks/container_inventory_management/rl_formulation.ipynb @@ -24,7 +24,7 @@ }, { "cell_type": "code", - "execution_count": 1, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -32,27 +32,32 @@ "from maro.rl import StateShaper\n", "\n", "\n", + "PORT_ATTRIBUTES = [\"empty\", \"full\", \"on_shipper\", \"on_consignee\", \"booking\", \"shortage\", \"fulfillment\"]\n", + "VESSEL_ATTRIBUTES = [\"empty\", \"full\", \"remaining_space\"]\n", + "\n", + "\n", "class CIMStateShaper(StateShaper):\n", - " def __init__(self, *, look_back, max_ports_downstream, port_attributes, vessel_attributes):\n", + " def __init__(self, *, look_back, max_ports_downstream):\n", " super().__init__()\n", " self._look_back = look_back\n", " self._max_ports_downstream = max_ports_downstream\n", - " self._port_attributes = port_attributes\n", - " self._vessel_attributes = vessel_attributes\n", - " self._dim = (look_back + 1) * (max_ports_downstream + 1) * len(port_attributes) + len(vessel_attributes)\n", + " self._dim = (look_back + 1) * (max_ports_downstream + 1) * len(PORT_ATTRIBUTES) + len(VESSEL_ATTRIBUTES)\n", "\n", " def __call__(self, decision_event, snapshot_list):\n", " tick, port_idx, vessel_idx = decision_event.tick, decision_event.port_idx, decision_event.vessel_idx\n", - " ticks = [tick - rt for rt in range(self._look_back-1)]\n", + " ticks = [tick - rt for rt in range(self._look_back - 1)]\n", " future_port_idx_list = snapshot_list[\"vessels\"][tick: vessel_idx: 'future_stop_list'].astype('int')\n", - " port_features = snapshot_list[\"ports\"][ticks: [port_idx] + list(future_port_idx_list): self._port_attributes]\n", - " vessel_features = snapshot_list[\"vessels\"][tick: vessel_idx: self._vessel_attributes]\n", + " port_features = snapshot_list[\"ports\"][ticks: [port_idx] + list(future_port_idx_list): PORT_ATTRIBUTES]\n", + " vessel_features = snapshot_list[\"vessels\"][tick: vessel_idx: VESSEL_ATTRIBUTES]\n", " state = np.concatenate((port_features, vessel_features))\n", " return str(port_idx), state\n", - " \n", + "\n", " @property\n", " def dim(self):\n", - " return self._dim" + " return self._dim\n", + " \n", + "# Create a state shaper\n", + "state_shaper = CIMStateShaper(look_back=7, max_ports_downstream=2)" ] }, { @@ -66,7 +71,7 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -92,7 +97,7 @@ " early_discharge = snapshot_list[\"vessels\"][tick:vessel_idx: \"early_discharge\"][0]\n", " \n", " if model_action < self._zero_action_index:\n", - " # The number of loaded containers must be less thean the vessel's remaining space.\n", + " # The number of loaded containers must be less than the vessel's remaining space.\n", " actual_action = max(round(self._action_space[model_action] * port_empty), -vessel_remaining_space)\n", " elif model_action > self._zero_action_index:\n", " # In the case of an early discharge event, we need to subtract the early discharge amount from the expected \n", @@ -102,7 +107,11 @@ " else:\n", " actual_action = 0\n", "\n", - " return Action(vessel_idx, port_idx, actual_action)" + " return Action(vessel_idx, port_idx, actual_action)\n", + " \n", + "# Create an action shaper\n", + "NUM_ACTIONS = 21\n", + "action_shaper = CIMActionShaper(action_space=list(np.linspace(-1.0, 1.0, NUM_ACTIONS)))" ] }, { @@ -116,7 +125,7 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -126,8 +135,9 @@ "\n", "\n", "class TruncatedExperienceShaper(ExperienceShaper):\n", - " def __init__(self, *, time_window: int, time_decay_factor: float, fulfillment_factor: float,\n", - " shortage_factor: float):\n", + " def __init__(\n", + " self, *, time_window: int, time_decay_factor: float, fulfillment_factor: float, shortage_factor: float\n", + " ):\n", " super().__init__(reward_func=None)\n", " self._time_window = time_window\n", " self._time_decay_factor = time_decay_factor\n", @@ -141,12 +151,11 @@ " agent_id = transition[\"agent_id\"]\n", " if agent_id not in experiences_by_agent:\n", " experiences_by_agent[agent_id] = defaultdict(list)\n", - " \n", " experiences = experiences_by_agent[agent_id]\n", " experiences[\"state\"].append(transition[\"state\"])\n", " experiences[\"action\"].append(transition[\"action\"])\n", " experiences[\"reward\"].append(self._compute_reward(transition[\"event\"], snapshot_list))\n", - " experiences[\"next_state\"].append(trajectory[i+1][\"state\"])\n", + " experiences[\"next_state\"].append(trajectory[i + 1][\"state\"])\n", "\n", " return experiences_by_agent\n", "\n", @@ -155,16 +164,21 @@ " end_tick = decision_event.tick + self._time_window\n", " ticks = list(range(start_tick, end_tick))\n", "\n", - " # Calculate truncate reward.\n", + " # calculate tc reward\n", " future_fulfillment = snapshot_list[\"ports\"][ticks::\"fulfillment\"]\n", " future_shortage = snapshot_list[\"ports\"][ticks::\"shortage\"]\n", - " decay_list = [self._time_decay_factor ** i for i in range(end_tick - start_tick)\n", - " for _ in range(future_fulfillment.shape[0]//(end_tick-start_tick))]\n", + " decay_list = [\n", + " self._time_decay_factor ** i for i in range(end_tick - start_tick)\n", + " for _ in range(future_fulfillment.shape[0] // (end_tick - start_tick))\n", + " ]\n", "\n", " tot_fulfillment = np.dot(future_fulfillment, decay_list)\n", " tot_shortage = np.dot(future_shortage, decay_list)\n", "\n", - " return np.float(self._fulfillment_factor * tot_fulfillment - self._shortage_factor * tot_shortage)" + " return np.float32(self._fulfillment_factor * tot_fulfillment - self._shortage_factor * tot_shortage)\n", + " \n", + "# Create an experience shaper\n", + "experience_shaper = TruncatedExperienceShaper(time_window=100, fulfillment_factor=1.0, shortage_factor=1.0, time_decay_factor=0.97)" ] }, { @@ -178,7 +192,7 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -186,8 +200,7 @@ "\n", "\n", "class CIMAgent(AbsAgent):\n", - " def __init__(self, name, algorithm, experience_pool: ColumnBasedStore, min_experiences_to_train,\n", - " num_batches, batch_size):\n", + " def __init__(self, name, algorithm, experience_pool: ColumnBasedStore, min_experiences_to_train, num_batches, batch_size):\n", " super().__init__(name, algorithm, experience_pool)\n", " self._min_experiences_to_train = min_experiences_to_train\n", " self._num_batches = num_batches\n", @@ -218,7 +231,7 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -229,51 +242,47 @@ "from torch.nn.functional import smooth_l1_loss\n", "from torch.optim import RMSprop\n", "\n", - "from maro.rl import SimpleAgentManager, LearningModuleManager, FullyConnectedNet, DQN, DQNConfig, ColumnBasedStore\n", - "\n", - "\n", - "input_dim = 171\n", - "num_actions = 21\n", - "\n", + "from maro.rl import (\n", + " ColumnBasedStore, DQN, DQNConfig, FullyConnectedBlock, LearningModuleManager, LearningModule, OptimizerOptions, SimpleAgentManager\n", + ")\n", + "from maro.utils import set_seeds\n", "\n", "\n", "def create_dqn_agents(agent_id_list):\n", + " set_seeds(1024) # for reproducibility\n", " agent_dict = {}\n", " for agent_id in agent_id_list:\n", - " eval_model = LearningModuleManager(\n", - " decision_layers=FullyConnectedNet(\n", - " name=f'{agent_id}.policy',\n", - " input_dim=input_dim,\n", - " output_dim=num_actions,\n", - " activation=nn.LeakyReLU, \n", + " q_module = LearningModule(\n", + " \"q_value\",\n", + " [FullyConnectedBlock(\n", + " input_dim=state_shaper.dim,\n", " hidden_dims=[256, 128, 64],\n", + " output_dim=NUM_ACTIONS,\n", + " activation=nn.LeakyReLU,\n", + " is_head=True,\n", + " batch_norm_enabled=True, \n", " softmax_enabled=False,\n", - " batch_norm_enabled=True,\n", - " dropout_p=.0\n", - " )\n", + " skip_connection_enabled=False,\n", + " dropout_p=.0)\n", + " ],\n", + " optimizer_options=OptimizerOptions(cls=RMSprop, params={\"lr\": 0.05})\n", " )\n", "\n", " algorithm = DQN(\n", - " eval_model=eval_model,\n", - " optimizer_cls=RMSprop,\n", - " optimizer_params={\"lr\": 0.05},\n", - " loss_func=nn.functional.smooth_l1_loss,\n", - " hyper_params=DQNConfig(\n", - " num_actions=num_actions,\n", - " reward_decay=.0,\n", - " target_update_frequency=5,\n", - " tau=0.1\n", + " model=LearningModuleManager(q_module),\n", + " config=DQNConfig(\n", + " reward_decay=.0, \n", + " target_update_frequency=5, \n", + " tau=0.1, \n", + " is_double=True, \n", + " per_sample_td_error_enabled=True,\n", + " loss_cls=nn.SmoothL1Loss,\n", + " num_actions=NUM_ACTIONS\n", " )\n", " )\n", "\n", - " experience_pool = ColumnBasedStore()\n", " agent_dict[agent_id] = CIMAgent(\n", - " name=agent_id,\n", - " algorithm=algorithm,\n", - " experience_pool=experience_pool,\n", - " min_experiences_to_train=1024,\n", - " num_batches=10,\n", - " batch_size=128\n", + " agent_id, algorithm, ColumnBasedStore(), min_experiences_to_train=1024, num_batches=10, batch_size=128\n", " )\n", "\n", " return agent_dict\n", @@ -311,157 +320,19 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "08:19:40 | single_host_cim_learner | INFO | ep 0 - performance: {'order_requirements': 2240000, 'container_shortage': 1449174, 'operation_number': 2877166}, epsilons: {'0': 0.4, '1': 0.4, '2': 0.4, '3': 0.4}\n", - "08:19:44 | single_host_cim_learner | INFO | ep 1 - performance: {'order_requirements': 2240000, 'container_shortage': 1433716, 'operation_number': 2572894}, epsilons: {'0': 0.3983838383838384, '1': 0.3983838383838384, '2': 0.3983838383838384, '3': 0.3983838383838384}\n", - "08:19:48 | single_host_cim_learner | INFO | ep 2 - performance: {'order_requirements': 2240000, 'container_shortage': 1508563, 'operation_number': 2738400}, epsilons: {'0': 0.39676767676767677, '1': 0.39676767676767677, '2': 0.39676767676767677, '3': 0.39676767676767677}\n", - "08:19:53 | single_host_cim_learner | INFO | ep 3 - performance: {'order_requirements': 2240000, 'container_shortage': 1489816, 'operation_number': 2809858}, epsilons: {'0': 0.3951515151515152, '1': 0.3951515151515152, '2': 0.3951515151515152, '3': 0.3951515151515152}\n", - "08:19:57 | single_host_cim_learner | INFO | ep 4 - performance: {'order_requirements': 2240000, 'container_shortage': 1123285, 'operation_number': 3364524}, epsilons: {'0': 0.39353535353535357, '1': 0.39353535353535357, '2': 0.39353535353535357, '3': 0.39353535353535357}\n", - "08:20:01 | single_host_cim_learner | INFO | ep 5 - performance: {'order_requirements': 2240000, 'container_shortage': 1349341, 'operation_number': 2694390}, epsilons: {'0': 0.39191919191919194, '1': 0.39191919191919194, '2': 0.39191919191919194, '3': 0.39191919191919194}\n", - "08:20:06 | single_host_cim_learner | INFO | ep 6 - performance: {'order_requirements': 2240000, 'container_shortage': 1064843, 'operation_number': 3079866}, epsilons: {'0': 0.3903030303030303, '1': 0.3903030303030303, '2': 0.3903030303030303, '3': 0.3903030303030303}\n", - "08:20:10 | single_host_cim_learner | INFO | ep 7 - performance: {'order_requirements': 2240000, 'container_shortage': 1030570, 'operation_number': 3785677}, epsilons: {'0': 0.3886868686868687, '1': 0.3886868686868687, '2': 0.3886868686868687, '3': 0.3886868686868687}\n", - "08:20:15 | single_host_cim_learner | INFO | ep 8 - performance: {'order_requirements': 2240000, 'container_shortage': 1518994, 'operation_number': 2751331}, epsilons: {'0': 0.38707070707070707, '1': 0.38707070707070707, '2': 0.38707070707070707, '3': 0.38707070707070707}\n", - "08:20:20 | single_host_cim_learner | INFO | ep 9 - performance: {'order_requirements': 2240000, 'container_shortage': 1263743, 'operation_number': 3811452}, epsilons: {'0': 0.3854545454545455, '1': 0.3854545454545455, '2': 0.3854545454545455, '3': 0.3854545454545455}\n", - "08:20:24 | single_host_cim_learner | INFO | ep 10 - performance: {'order_requirements': 2240000, 'container_shortage': 1283297, 'operation_number': 2929005}, epsilons: {'0': 0.38383838383838387, '1': 0.38383838383838387, '2': 0.38383838383838387, '3': 0.38383838383838387}\n", - "08:20:29 | single_host_cim_learner | INFO | ep 11 - performance: {'order_requirements': 2240000, 'container_shortage': 1218838, 'operation_number': 3655184}, epsilons: {'0': 0.38222222222222224, '1': 0.38222222222222224, '2': 0.38222222222222224, '3': 0.38222222222222224}\n", - "08:20:34 | single_host_cim_learner | INFO | ep 12 - performance: {'order_requirements': 2240000, 'container_shortage': 1661433, 'operation_number': 2834056}, epsilons: {'0': 0.3806060606060606, '1': 0.3806060606060606, '2': 0.3806060606060606, '3': 0.3806060606060606}\n", - "08:20:38 | single_host_cim_learner | INFO | ep 13 - performance: {'order_requirements': 2240000, 'container_shortage': 998497, 'operation_number': 4331405}, epsilons: {'0': 0.378989898989899, '1': 0.378989898989899, '2': 0.378989898989899, '3': 0.378989898989899}\n", - "08:20:43 | single_host_cim_learner | INFO | ep 14 - performance: {'order_requirements': 2240000, 'container_shortage': 817916, 'operation_number': 4062977}, epsilons: {'0': 0.3773737373737374, '1': 0.3773737373737374, '2': 0.3773737373737374, '3': 0.3773737373737374}\n", - "08:20:47 | single_host_cim_learner | INFO | ep 15 - performance: {'order_requirements': 2240000, 'container_shortage': 1205514, 'operation_number': 3433622}, epsilons: {'0': 0.3757575757575758, '1': 0.3757575757575758, '2': 0.3757575757575758, '3': 0.3757575757575758}\n", - "08:20:52 | single_host_cim_learner | INFO | ep 16 - performance: {'order_requirements': 2240000, 'container_shortage': 1031884, 'operation_number': 3870460}, epsilons: {'0': 0.37414141414141416, '1': 0.37414141414141416, '2': 0.37414141414141416, '3': 0.37414141414141416}\n", - "08:20:56 | single_host_cim_learner | INFO | ep 17 - performance: {'order_requirements': 2240000, 'container_shortage': 810135, 'operation_number': 4242833}, epsilons: {'0': 0.37252525252525254, '1': 0.37252525252525254, '2': 0.37252525252525254, '3': 0.37252525252525254}\n", - "08:21:01 | single_host_cim_learner | INFO | ep 18 - performance: {'order_requirements': 2240000, 'container_shortage': 1169174, 'operation_number': 3537445}, epsilons: {'0': 0.3709090909090909, '1': 0.3709090909090909, '2': 0.3709090909090909, '3': 0.3709090909090909}\n", - "08:21:06 | single_host_cim_learner | INFO | ep 19 - performance: {'order_requirements': 2240000, 'container_shortage': 955743, 'operation_number': 4069342}, epsilons: {'0': 0.36929292929292934, '1': 0.36929292929292934, '2': 0.36929292929292934, '3': 0.36929292929292934}\n", - "08:21:10 | single_host_cim_learner | INFO | ep 20 - performance: {'order_requirements': 2240000, 'container_shortage': 834081, 'operation_number': 4349373}, epsilons: {'0': 0.3676767676767677, '1': 0.3676767676767677, '2': 0.3676767676767677, '3': 0.3676767676767677}\n", - "08:21:15 | single_host_cim_learner | INFO | ep 21 - performance: {'order_requirements': 2240000, 'container_shortage': 887920, 'operation_number': 4085329}, epsilons: {'0': 0.3660606060606061, '1': 0.3660606060606061, '2': 0.3660606060606061, '3': 0.3660606060606061}\n", - "08:21:19 | single_host_cim_learner | INFO | ep 22 - performance: {'order_requirements': 2240000, 'container_shortage': 968113, 'operation_number': 4141775}, epsilons: {'0': 0.36444444444444446, '1': 0.36444444444444446, '2': 0.36444444444444446, '3': 0.36444444444444446}\n", - "08:21:24 | single_host_cim_learner | INFO | ep 23 - performance: {'order_requirements': 2240000, 'container_shortage': 1007433, 'operation_number': 3887778}, epsilons: {'0': 0.36282828282828283, '1': 0.36282828282828283, '2': 0.36282828282828283, '3': 0.36282828282828283}\n", - "08:21:29 | single_host_cim_learner | INFO | ep 24 - performance: {'order_requirements': 2240000, 'container_shortage': 872307, 'operation_number': 4125882}, epsilons: {'0': 0.3612121212121212, '1': 0.3612121212121212, '2': 0.3612121212121212, '3': 0.3612121212121212}\n", - "08:21:34 | single_host_cim_learner | INFO | ep 25 - performance: {'order_requirements': 2240000, 'container_shortage': 982211, 'operation_number': 4188103}, epsilons: {'0': 0.3595959595959596, '1': 0.3595959595959596, '2': 0.3595959595959596, '3': 0.3595959595959596}\n", - "08:21:38 | single_host_cim_learner | INFO | ep 26 - performance: {'order_requirements': 2240000, 'container_shortage': 1044675, 'operation_number': 4238421}, epsilons: {'0': 0.357979797979798, '1': 0.357979797979798, '2': 0.357979797979798, '3': 0.357979797979798}\n", - "08:21:43 | single_host_cim_learner | INFO | ep 27 - performance: {'order_requirements': 2240000, 'container_shortage': 1087284, 'operation_number': 3871621}, epsilons: {'0': 0.3563636363636364, '1': 0.3563636363636364, '2': 0.3563636363636364, '3': 0.3563636363636364}\n", - "08:21:48 | single_host_cim_learner | INFO | ep 28 - performance: {'order_requirements': 2240000, 'container_shortage': 766053, 'operation_number': 3905529}, epsilons: {'0': 0.35474747474747476, '1': 0.35474747474747476, '2': 0.35474747474747476, '3': 0.35474747474747476}\n", - "08:21:52 | single_host_cim_learner | INFO | ep 29 - performance: {'order_requirements': 2240000, 'container_shortage': 866625, 'operation_number': 4174561}, epsilons: {'0': 0.35313131313131313, '1': 0.35313131313131313, '2': 0.35313131313131313, '3': 0.35313131313131313}\n", - "08:21:57 | single_host_cim_learner | INFO | ep 30 - performance: {'order_requirements': 2240000, 'container_shortage': 970164, 'operation_number': 3620473}, epsilons: {'0': 0.3515151515151515, '1': 0.3515151515151515, '2': 0.3515151515151515, '3': 0.3515151515151515}\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "08:22:02 | single_host_cim_learner | INFO | ep 31 - performance: {'order_requirements': 2240000, 'container_shortage': 589462, 'operation_number': 4166597}, epsilons: {'0': 0.34989898989898993, '1': 0.34989898989898993, '2': 0.34989898989898993, '3': 0.34989898989898993}\n", - "08:22:06 | single_host_cim_learner | INFO | ep 32 - performance: {'order_requirements': 2240000, 'container_shortage': 651137, 'operation_number': 3812302}, epsilons: {'0': 0.3482828282828283, '1': 0.3482828282828283, '2': 0.3482828282828283, '3': 0.3482828282828283}\n", - "08:22:11 | single_host_cim_learner | INFO | ep 33 - performance: {'order_requirements': 2240000, 'container_shortage': 506324, 'operation_number': 4250385}, epsilons: {'0': 0.3466666666666667, '1': 0.3466666666666667, '2': 0.3466666666666667, '3': 0.3466666666666667}\n", - "08:22:16 | single_host_cim_learner | INFO | ep 34 - performance: {'order_requirements': 2240000, 'container_shortage': 656772, 'operation_number': 4013839}, epsilons: {'0': 0.34505050505050505, '1': 0.34505050505050505, '2': 0.34505050505050505, '3': 0.34505050505050505}\n", - "08:22:20 | single_host_cim_learner | INFO | ep 35 - performance: {'order_requirements': 2240000, 'container_shortage': 549707, 'operation_number': 4060300}, epsilons: {'0': 0.3434343434343434, '1': 0.3434343434343434, '2': 0.3434343434343434, '3': 0.3434343434343434}\n", - "08:22:25 | single_host_cim_learner | INFO | ep 36 - performance: {'order_requirements': 2240000, 'container_shortage': 608510, 'operation_number': 4065739}, epsilons: {'0': 0.3418181818181818, '1': 0.3418181818181818, '2': 0.3418181818181818, '3': 0.3418181818181818}\n", - "08:22:30 | single_host_cim_learner | INFO | ep 37 - performance: {'order_requirements': 2240000, 'container_shortage': 533948, 'operation_number': 4050012}, epsilons: {'0': 0.3402020202020202, '1': 0.3402020202020202, '2': 0.3402020202020202, '3': 0.3402020202020202}\n", - "08:22:34 | single_host_cim_learner | INFO | ep 38 - performance: {'order_requirements': 2240000, 'container_shortage': 506934, 'operation_number': 4276627}, epsilons: {'0': 0.3385858585858586, '1': 0.3385858585858586, '2': 0.3385858585858586, '3': 0.3385858585858586}\n", - "08:22:39 | single_host_cim_learner | INFO | ep 39 - performance: {'order_requirements': 2240000, 'container_shortage': 516833, 'operation_number': 4413279}, epsilons: {'0': 0.336969696969697, '1': 0.336969696969697, '2': 0.336969696969697, '3': 0.336969696969697}\n", - "08:22:44 | single_host_cim_learner | INFO | ep 40 - performance: {'order_requirements': 2240000, 'container_shortage': 497496, 'operation_number': 4473180}, epsilons: {'0': 0.33535353535353535, '1': 0.33535353535353535, '2': 0.33535353535353535, '3': 0.33535353535353535}\n", - "08:22:48 | single_host_cim_learner | INFO | ep 41 - performance: {'order_requirements': 2240000, 'container_shortage': 866223, 'operation_number': 4156194}, epsilons: {'0': 0.3337373737373738, '1': 0.3337373737373738, '2': 0.3337373737373738, '3': 0.3337373737373738}\n", - "08:22:53 | single_host_cim_learner | INFO | ep 42 - performance: {'order_requirements': 2240000, 'container_shortage': 446579, 'operation_number': 4225675}, epsilons: {'0': 0.33212121212121215, '1': 0.33212121212121215, '2': 0.33212121212121215, '3': 0.33212121212121215}\n", - "08:22:58 | single_host_cim_learner | INFO | ep 43 - performance: {'order_requirements': 2240000, 'container_shortage': 400190, 'operation_number': 4255237}, epsilons: {'0': 0.3305050505050505, '1': 0.3305050505050505, '2': 0.3305050505050505, '3': 0.3305050505050505}\n", - "08:23:02 | single_host_cim_learner | INFO | ep 44 - performance: {'order_requirements': 2240000, 'container_shortage': 466887, 'operation_number': 4216786}, epsilons: {'0': 0.3288888888888889, '1': 0.3288888888888889, '2': 0.3288888888888889, '3': 0.3288888888888889}\n", - "08:23:07 | single_host_cim_learner | INFO | ep 45 - performance: {'order_requirements': 2240000, 'container_shortage': 499424, 'operation_number': 4268237}, epsilons: {'0': 0.32727272727272727, '1': 0.32727272727272727, '2': 0.32727272727272727, '3': 0.32727272727272727}\n", - "08:23:12 | single_host_cim_learner | INFO | ep 46 - performance: {'order_requirements': 2240000, 'container_shortage': 525049, 'operation_number': 4131713}, epsilons: {'0': 0.32565656565656564, '1': 0.32565656565656564, '2': 0.32565656565656564, '3': 0.32565656565656564}\n", - "08:23:17 | single_host_cim_learner | INFO | ep 47 - performance: {'order_requirements': 2240000, 'container_shortage': 510123, 'operation_number': 4232019}, epsilons: {'0': 0.324040404040404, '1': 0.324040404040404, '2': 0.324040404040404, '3': 0.324040404040404}\n", - "08:23:21 | single_host_cim_learner | INFO | ep 48 - performance: {'order_requirements': 2240000, 'container_shortage': 452370, 'operation_number': 4175531}, epsilons: {'0': 0.32242424242424245, '1': 0.32242424242424245, '2': 0.32242424242424245, '3': 0.32242424242424245}\n", - "08:23:26 | single_host_cim_learner | INFO | ep 49 - performance: {'order_requirements': 2240000, 'container_shortage': 424113, 'operation_number': 4204467}, epsilons: {'0': 0.3208080808080808, '1': 0.3208080808080808, '2': 0.3208080808080808, '3': 0.3208080808080808}\n", - "08:23:31 | single_host_cim_learner | INFO | ep 50 - performance: {'order_requirements': 2240000, 'container_shortage': 486622, 'operation_number': 4177974}, epsilons: {'0': 0.31676767676767675, '1': 0.31676767676767675, '2': 0.31676767676767675, '3': 0.31676767676767675}\n", - "08:23:36 | single_host_cim_learner | INFO | ep 51 - performance: {'order_requirements': 2240000, 'container_shortage': 477881, 'operation_number': 4023553}, epsilons: {'0': 0.3103030303030303, '1': 0.3103030303030303, '2': 0.3103030303030303, '3': 0.3103030303030303}\n", - "08:23:41 | single_host_cim_learner | INFO | ep 52 - performance: {'order_requirements': 2240000, 'container_shortage': 546463, 'operation_number': 4020774}, epsilons: {'0': 0.3038383838383838, '1': 0.3038383838383838, '2': 0.3038383838383838, '3': 0.3038383838383838}\n", - "08:23:45 | single_host_cim_learner | INFO | ep 53 - performance: {'order_requirements': 2240000, 'container_shortage': 461636, 'operation_number': 4188035}, epsilons: {'0': 0.2973737373737374, '1': 0.2973737373737374, '2': 0.2973737373737374, '3': 0.2973737373737374}\n", - "08:23:50 | single_host_cim_learner | INFO | ep 54 - performance: {'order_requirements': 2240000, 'container_shortage': 570378, 'operation_number': 3776713}, epsilons: {'0': 0.29090909090909095, '1': 0.29090909090909095, '2': 0.29090909090909095, '3': 0.29090909090909095}\n", - "08:23:55 | single_host_cim_learner | INFO | ep 55 - performance: {'order_requirements': 2240000, 'container_shortage': 538013, 'operation_number': 4023664}, epsilons: {'0': 0.28444444444444444, '1': 0.28444444444444444, '2': 0.28444444444444444, '3': 0.28444444444444444}\n", - "08:24:00 | single_host_cim_learner | INFO | ep 56 - performance: {'order_requirements': 2240000, 'container_shortage': 580577, 'operation_number': 3894939}, epsilons: {'0': 0.277979797979798, '1': 0.277979797979798, '2': 0.277979797979798, '3': 0.277979797979798}\n", - "08:24:05 | single_host_cim_learner | INFO | ep 57 - performance: {'order_requirements': 2240000, 'container_shortage': 470579, 'operation_number': 4114149}, epsilons: {'0': 0.2715151515151515, '1': 0.2715151515151515, '2': 0.2715151515151515, '3': 0.2715151515151515}\n", - "08:24:10 | single_host_cim_learner | INFO | ep 58 - performance: {'order_requirements': 2240000, 'container_shortage': 599047, 'operation_number': 3703140}, epsilons: {'0': 0.26505050505050504, '1': 0.26505050505050504, '2': 0.26505050505050504, '3': 0.26505050505050504}\n", - "08:24:15 | single_host_cim_learner | INFO | ep 59 - performance: {'order_requirements': 2240000, 'container_shortage': 592780, 'operation_number': 3752613}, epsilons: {'0': 0.25858585858585864, '1': 0.25858585858585864, '2': 0.25858585858585864, '3': 0.25858585858585864}\n", - "08:24:20 | single_host_cim_learner | INFO | ep 60 - performance: {'order_requirements': 2240000, 'container_shortage': 626194, 'operation_number': 3682348}, epsilons: {'0': 0.25212121212121213, '1': 0.25212121212121213, '2': 0.25212121212121213, '3': 0.25212121212121213}\n", - "08:24:25 | single_host_cim_learner | INFO | ep 61 - performance: {'order_requirements': 2240000, 'container_shortage': 331137, 'operation_number': 4097250}, epsilons: {'0': 0.24565656565656568, '1': 0.24565656565656568, '2': 0.24565656565656568, '3': 0.24565656565656568}\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "08:24:30 | single_host_cim_learner | INFO | ep 62 - performance: {'order_requirements': 2240000, 'container_shortage': 427942, 'operation_number': 3986817}, epsilons: {'0': 0.23919191919191918, '1': 0.23919191919191918, '2': 0.23919191919191918, '3': 0.23919191919191918}\n", - "08:24:34 | single_host_cim_learner | INFO | ep 63 - performance: {'order_requirements': 2240000, 'container_shortage': 349003, 'operation_number': 4169958}, epsilons: {'0': 0.23272727272727273, '1': 0.23272727272727273, '2': 0.23272727272727273, '3': 0.23272727272727273}\n", - "08:24:39 | single_host_cim_learner | INFO | ep 64 - performance: {'order_requirements': 2240000, 'container_shortage': 443412, 'operation_number': 4040920}, epsilons: {'0': 0.22626262626262622, '1': 0.22626262626262622, '2': 0.22626262626262622, '3': 0.22626262626262622}\n", - "08:24:44 | single_host_cim_learner | INFO | ep 65 - performance: {'order_requirements': 2240000, 'container_shortage': 491950, 'operation_number': 3867277}, epsilons: {'0': 0.2197979797979798, '1': 0.2197979797979798, '2': 0.2197979797979798, '3': 0.2197979797979798}\n", - "08:24:49 | single_host_cim_learner | INFO | ep 66 - performance: {'order_requirements': 2240000, 'container_shortage': 555217, 'operation_number': 4051336}, epsilons: {'0': 0.21333333333333337, '1': 0.21333333333333337, '2': 0.21333333333333337, '3': 0.21333333333333337}\n", - "08:24:54 | single_host_cim_learner | INFO | ep 67 - performance: {'order_requirements': 2240000, 'container_shortage': 453428, 'operation_number': 3853567}, epsilons: {'0': 0.20686868686868687, '1': 0.20686868686868687, '2': 0.20686868686868687, '3': 0.20686868686868687}\n", - "08:24:59 | single_host_cim_learner | INFO | ep 68 - performance: {'order_requirements': 2240000, 'container_shortage': 502834, 'operation_number': 3935391}, epsilons: {'0': 0.20040404040404042, '1': 0.20040404040404042, '2': 0.20040404040404042, '3': 0.20040404040404042}\n", - "08:25:04 | single_host_cim_learner | INFO | ep 69 - performance: {'order_requirements': 2240000, 'container_shortage': 651275, 'operation_number': 4529383}, epsilons: {'0': 0.1939393939393939, '1': 0.1939393939393939, '2': 0.1939393939393939, '3': 0.1939393939393939}\n", - "08:25:09 | single_host_cim_learner | INFO | ep 70 - performance: {'order_requirements': 2240000, 'container_shortage': 450364, 'operation_number': 3752971}, epsilons: {'0': 0.1874747474747475, '1': 0.1874747474747475, '2': 0.1874747474747475, '3': 0.1874747474747475}\n", - "08:25:14 | single_host_cim_learner | INFO | ep 71 - performance: {'order_requirements': 2240000, 'container_shortage': 380907, 'operation_number': 3936188}, epsilons: {'0': 0.18101010101010104, '1': 0.18101010101010104, '2': 0.18101010101010104, '3': 0.18101010101010104}\n", - "08:25:18 | single_host_cim_learner | INFO | ep 72 - performance: {'order_requirements': 2240000, 'container_shortage': 342391, 'operation_number': 4033114}, epsilons: {'0': 0.17454545454545453, '1': 0.17454545454545453, '2': 0.17454545454545453, '3': 0.17454545454545453}\n", - "08:25:23 | single_host_cim_learner | INFO | ep 73 - performance: {'order_requirements': 2240000, 'container_shortage': 391236, 'operation_number': 3948352}, epsilons: {'0': 0.1680808080808081, '1': 0.1680808080808081, '2': 0.1680808080808081, '3': 0.1680808080808081}\n", - "08:25:28 | single_host_cim_learner | INFO | ep 74 - performance: {'order_requirements': 2240000, 'container_shortage': 698458, 'operation_number': 3325779}, epsilons: {'0': 0.1616161616161616, '1': 0.1616161616161616, '2': 0.1616161616161616, '3': 0.1616161616161616}\n", - "08:25:33 | single_host_cim_learner | INFO | ep 75 - performance: {'order_requirements': 2240000, 'container_shortage': 370796, 'operation_number': 4465673}, epsilons: {'0': 0.15515151515151515, '1': 0.15515151515151515, '2': 0.15515151515151515, '3': 0.15515151515151515}\n", - "08:25:38 | single_host_cim_learner | INFO | ep 76 - performance: {'order_requirements': 2240000, 'container_shortage': 140750, 'operation_number': 4611626}, epsilons: {'0': 0.14868686868686873, '1': 0.14868686868686873, '2': 0.14868686868686873, '3': 0.14868686868686873}\n", - "08:25:43 | single_host_cim_learner | INFO | ep 77 - performance: {'order_requirements': 2240000, 'container_shortage': 159430, 'operation_number': 4627419}, epsilons: {'0': 0.14222222222222222, '1': 0.14222222222222222, '2': 0.14222222222222222, '3': 0.14222222222222222}\n", - "08:25:48 | single_host_cim_learner | INFO | ep 78 - performance: {'order_requirements': 2240000, 'container_shortage': 79279, 'operation_number': 4574175}, epsilons: {'0': 0.13575757575757577, '1': 0.13575757575757577, '2': 0.13575757575757577, '3': 0.13575757575757577}\n", - "08:25:53 | single_host_cim_learner | INFO | ep 79 - performance: {'order_requirements': 2240000, 'container_shortage': 157224, 'operation_number': 4481568}, epsilons: {'0': 0.12929292929292927, '1': 0.12929292929292927, '2': 0.12929292929292927, '3': 0.12929292929292927}\n", - "08:25:58 | single_host_cim_learner | INFO | ep 80 - performance: {'order_requirements': 2240000, 'container_shortage': 107323, 'operation_number': 4483160}, epsilons: {'0': 0.12282828282828284, '1': 0.12282828282828284, '2': 0.12282828282828284, '3': 0.12282828282828284}\n", - "08:26:03 | single_host_cim_learner | INFO | ep 81 - performance: {'order_requirements': 2240000, 'container_shortage': 115601, 'operation_number': 4583155}, epsilons: {'0': 0.11636363636363634, '1': 0.11636363636363634, '2': 0.11636363636363634, '3': 0.11636363636363634}\n", - "08:26:08 | single_host_cim_learner | INFO | ep 82 - performance: {'order_requirements': 2240000, 'container_shortage': 132296, 'operation_number': 4547233}, epsilons: {'0': 0.1098989898989899, '1': 0.1098989898989899, '2': 0.1098989898989899, '3': 0.1098989898989899}\n", - "08:26:13 | single_host_cim_learner | INFO | ep 83 - performance: {'order_requirements': 2240000, 'container_shortage': 80712, 'operation_number': 4558938}, epsilons: {'0': 0.10343434343434346, '1': 0.10343434343434346, '2': 0.10343434343434346, '3': 0.10343434343434346}\n", - "08:26:18 | single_host_cim_learner | INFO | ep 84 - performance: {'order_requirements': 2240000, 'container_shortage': 155909, 'operation_number': 4617332}, epsilons: {'0': 0.09696969696969696, '1': 0.09696969696969696, '2': 0.09696969696969696, '3': 0.09696969696969696}\n", - "08:26:23 | single_host_cim_learner | INFO | ep 85 - performance: {'order_requirements': 2240000, 'container_shortage': 99228, 'operation_number': 4594562}, epsilons: {'0': 0.09050505050505052, '1': 0.09050505050505052, '2': 0.09050505050505052, '3': 0.09050505050505052}\n", - "08:26:28 | single_host_cim_learner | INFO | ep 86 - performance: {'order_requirements': 2240000, 'container_shortage': 85741, 'operation_number': 4516614}, epsilons: {'0': 0.08404040404040401, '1': 0.08404040404040401, '2': 0.08404040404040401, '3': 0.08404040404040401}\n", - "08:26:33 | single_host_cim_learner | INFO | ep 87 - performance: {'order_requirements': 2240000, 'container_shortage': 27260, 'operation_number': 4476453}, epsilons: {'0': 0.07757575757575758, '1': 0.07757575757575758, '2': 0.07757575757575758, '3': 0.07757575757575758}\n", - "08:26:38 | single_host_cim_learner | INFO | ep 88 - performance: {'order_requirements': 2240000, 'container_shortage': 81625, 'operation_number': 4424191}, epsilons: {'0': 0.07111111111111114, '1': 0.07111111111111114, '2': 0.07111111111111114, '3': 0.07111111111111114}\n", - "08:26:43 | single_host_cim_learner | INFO | ep 89 - performance: {'order_requirements': 2240000, 'container_shortage': 39858, 'operation_number': 4517449}, epsilons: {'0': 0.06464646464646463, '1': 0.06464646464646463, '2': 0.06464646464646463, '3': 0.06464646464646463}\n", - "08:26:48 | single_host_cim_learner | INFO | ep 90 - performance: {'order_requirements': 2240000, 'container_shortage': 68142, 'operation_number': 4411657}, epsilons: {'0': 0.0581818181818182, '1': 0.0581818181818182, '2': 0.0581818181818182, '3': 0.0581818181818182}\n", - "08:26:53 | single_host_cim_learner | INFO | ep 91 - performance: {'order_requirements': 2240000, 'container_shortage': 35214, 'operation_number': 4514769}, epsilons: {'0': 0.051717171717171696, '1': 0.051717171717171696, '2': 0.051717171717171696, '3': 0.051717171717171696}\n", - "08:26:58 | single_host_cim_learner | INFO | ep 92 - performance: {'order_requirements': 2240000, 'container_shortage': 73510, 'operation_number': 4448161}, epsilons: {'0': 0.04525252525252526, '1': 0.04525252525252526, '2': 0.04525252525252526, '3': 0.04525252525252526}\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "08:27:03 | single_host_cim_learner | INFO | ep 93 - performance: {'order_requirements': 2240000, 'container_shortage': 54462, 'operation_number': 4504207}, epsilons: {'0': 0.03878787878787875, '1': 0.03878787878787875, '2': 0.03878787878787875, '3': 0.03878787878787875}\n", - "08:27:08 | single_host_cim_learner | INFO | ep 94 - performance: {'order_requirements': 2240000, 'container_shortage': 17770, 'operation_number': 4507162}, epsilons: {'0': 0.032323232323232316, '1': 0.032323232323232316, '2': 0.032323232323232316, '3': 0.032323232323232316}\n", - "08:27:13 | single_host_cim_learner | INFO | ep 95 - performance: {'order_requirements': 2240000, 'container_shortage': 31159, 'operation_number': 4457436}, epsilons: {'0': 0.025858585858585883, '1': 0.025858585858585883, '2': 0.025858585858585883, '3': 0.025858585858585883}\n", - "08:27:18 | single_host_cim_learner | INFO | ep 96 - performance: {'order_requirements': 2240000, 'container_shortage': 29988, 'operation_number': 4428244}, epsilons: {'0': 0.019393939393939377, '1': 0.019393939393939377, '2': 0.019393939393939377, '3': 0.019393939393939377}\n", - "08:27:23 | single_host_cim_learner | INFO | ep 97 - performance: {'order_requirements': 2240000, 'container_shortage': 5791, 'operation_number': 4473662}, epsilons: {'0': 0.012929292929292941, '1': 0.012929292929292941, '2': 0.012929292929292941, '3': 0.012929292929292941}\n", - "08:27:28 | single_host_cim_learner | INFO | ep 98 - performance: {'order_requirements': 2240000, 'container_shortage': 8556, 'operation_number': 4451186}, epsilons: {'0': 0.006464646464646435, '1': 0.006464646464646435, '2': 0.006464646464646435, '3': 0.006464646464646435}\n", - "08:27:33 | single_host_cim_learner | INFO | ep 99 - performance: {'order_requirements': 2240000, 'container_shortage': 0, 'operation_number': 4457874}, epsilons: {'0': 0.0, '1': 0.0, '2': 0.0, '3': 0.0}\n" - ] - } - ], + "outputs": [], "source": [ "from maro.simulator import Env\n", - "from maro.rl import SimpleLearner, SimpleActor, AgentManagerMode, TwoPhaseLinearExplorer\n", + "from maro.rl import AgentManagerMode, Scheduler, SimpleActor, SimpleLearner, TwoPhaseLinearExplorationParameterGenerator\n", "from maro.utils import Logger, LogFormat\n", "\n", - "# Step 1: initialize a CIM environment for using a toy dataset. \n", + "# Step 1: initialize a CIM environment for a toy dataset. \n", "env = Env(\"cim\", \"toy.4p_ssdd_l0.0\", durations=1120)\n", "agent_id_list = [str(agent_id) for agent_id in env.agent_idx_list]\n", "\n", - "# Step 2: create state, action and experience shapers. We also need to create an explorer here due to the \n", - "# greedy nature of the DQN algorithm. \n", - "state_shaper = CIMStateShaper(look_back=7, max_ports_downstream=2, \n", - " port_attributes=[\"empty\", \"full\", \"on_shipper\", \"on_consignee\", \n", - " \"booking\", \"shortage\", \"fulfillment\"],\n", - " vessel_attributes=[\"empty\", \"full\", \"remaining_space\"]\n", - " )\n", - "\n", - "action_shaper = CIMActionShaper(action_space=list(np.linspace(-1.0, 1.0, num_actions)))\n", - "\n", - "experience_shaper = TruncatedExperienceShaper(time_window=100, fulfillment_factor=1.0, shortage_factor=1.0,\n", - " time_decay_factor=0.97)\n", - "\n", - "# Step 3: create an agent manager.\n", + "# Step 2: create DQN agents and an agent manager to manage them.\n", "agent_manager = DQNAgentManager(name=\"cim_learner\",\n", " mode=AgentManagerMode.TRAIN_INFERENCE,\n", " agent_dict=create_dqn_agents(agent_id_list),\n", @@ -469,13 +340,28 @@ " action_shaper=action_shaper,\n", " experience_shaper=experience_shaper)\n", "\n", - "# Step 4: Create an actor and a learner to start the training process. \n", + "# Step 3: Create an actor and a learner to start the training process. \n", + "max_episode = 100\n", + "scheduler = Scheduler(\n", + " max_episode, \n", + " exploration_parameter_generator_cls=TwoPhaseLinearExplorationParameterGenerator,\n", + " exploration_parameter_generator_config={\n", + " \"parameter_names\": [\"epsilon\"], \n", + " \"split_ep\": 50, \n", + " \"start_values\": 0.4, \n", + " \"mid_values\": 0.32, \n", + " \"end_values\": .0\n", + " }\n", + ")\n", + "\n", "actor = SimpleActor(env, agent_manager)\n", - "learner = SimpleLearner(trainable_agents=agent_manager, actor=actor, \n", - " explorer=TwoPhaseLinearExplorer(start_eps=0.4, mid_eps=0.32, end_eps=0.0, split_point=0.5),\n", - " logger=Logger(\"single_host_cim_learner\", format_=LogFormat.simple, auto_timestamp=False))\n", + "learner = SimpleLearner(\n", + " agent_manager, actor, scheduler, \n", + " logger=Logger(\"single_host_cim_learner\", format_=LogFormat.simple, auto_timestamp=False)\n", + ")\n", "\n", - "learner.learn(max_episode=100)" + "learner.learn()\n", + "learner.test()" ] }, { @@ -502,7 +388,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.7.7" + "version": "3.8.5" } }, "nbformat": 4, From 9dd71d613b81704729ec9632af1eec7ac7965019 Mon Sep 17 00:00:00 2001 From: ysqyang Date: Thu, 24 Dec 2020 14:34:41 +0800 Subject: [PATCH 294/337] simplified scheduler --- examples/cim/dqn/dist_learner.py | 11 +--- examples/cim/dqn/single_process_launcher.py | 12 ++-- maro/rl/__init__.py | 11 +--- maro/rl/learner/simple_learner.py | 8 +-- maro/rl/scheduling/scheduler.py | 57 +++++-------------- ...rator.py => simple_parameter_scheduler.py} | 50 +++++----------- maro/utils/exception/error_code.py | 3 +- maro/utils/exception/rl_toolkit_exception.py | 9 --- 8 files changed, 46 insertions(+), 115 deletions(-) rename maro/rl/scheduling/{exploration_parameter_generator.py => simple_parameter_scheduler.py} (79%) diff --git a/examples/cim/dqn/dist_learner.py b/examples/cim/dqn/dist_learner.py index 034b048f4..70082ba26 100644 --- a/examples/cim/dqn/dist_learner.py +++ b/examples/cim/dqn/dist_learner.py @@ -4,8 +4,7 @@ import os from maro.rl import ( - ActorProxy, AgentManagerMode, Scheduler, SimpleLearner, TwoPhaseLinearExplorationParameterGenerator, - concat_experiences_by_agent + ActorProxy, AgentManagerMode, SimpleLearner, TwoPhaseLinearParameterScheduler, concat_experiences_by_agent ) from maro.simulator import Env from maro.utils import Logger, convert_dottable @@ -35,16 +34,10 @@ def launch(config, distributed_config): "max_retries": 15 } - scheduler = Scheduler( - config.main_loop.max_episode, - exploration_parameter_generator_cls=TwoPhaseLinearExplorationParameterGenerator, - exploration_parameter_generator_config=config.main_loop.exploration, - ) - learner = SimpleLearner( agent_manager=agent_manager, actor=ActorProxy(proxy_params=proxy_params, experience_collecting_func=concat_experiences_by_agent), - scheduler=scheduler, + scheduler=TwoPhaseLinearParameterScheduler(config.main_loop.max_episode, **config.main_loop.exploration), logger=Logger("distributed_cim_learner", auto_timestamp=False) ) learner.learn() diff --git a/examples/cim/dqn/single_process_launcher.py b/examples/cim/dqn/single_process_launcher.py index bebb57351..3c41d7824 100644 --- a/examples/cim/dqn/single_process_launcher.py +++ b/examples/cim/dqn/single_process_launcher.py @@ -7,9 +7,9 @@ import numpy as np -from maro.rl import AgentManagerMode, Scheduler, SimpleActor, SimpleLearner, TwoPhaseLinearExplorationParameterGenerator +from maro.rl import AgentManagerMode, SimpleActor, SimpleLearner, TwoPhaseLinearParameterScheduler from maro.simulator import Env -from maro.utils import Logger, convert_dottable +from maro.utils import LogFormat, Logger, convert_dottable from components import CIMActionShaper, CIMStateShaper, DQNAgentManager, TruncatedExperienceShaper, create_dqn_agents @@ -71,16 +71,16 @@ def launch(config): ) # Step 4: Create an actor and a learner to start the training process. - scheduler = Scheduler( + scheduler = TwoPhaseLinearParameterScheduler( config.main_loop.max_episode, early_stopping_callback=EarlyStopping(**config.main_loop.early_stopping), - exploration_parameter_generator_cls=TwoPhaseLinearExplorationParameterGenerator, - exploration_parameter_generator_config=config.main_loop.exploration + **config.main_loop.exploration ) actor = SimpleActor(env, agent_manager) learner = SimpleLearner( - agent_manager, actor, scheduler, logger=Logger("single_host_cim_learner", auto_timestamp=False) + agent_manager, actor, scheduler, + logger=Logger("single_host_cim_learner", format_=LogFormat.simple, auto_timestamp=False) ) learner.learn() learner.test() diff --git a/maro/rl/__init__.py b/maro/rl/__init__.py index 6dac4b19c..e863b75e5 100644 --- a/maro/rl/__init__.py +++ b/maro/rl/__init__.py @@ -18,11 +18,8 @@ from maro.rl.learner.simple_learner import SimpleLearner from maro.rl.models.fc_block import FullyConnectedBlock from maro.rl.models.learning_model import MultiHeadLearningModel, SingleHeadLearningModel -from maro.rl.scheduling.exploration_parameter_generator import ( - DynamicExplorationParameterGenerator, LinearExplorationParameterGenerator, StaticExplorationParameterGenerator, - TwoPhaseLinearExplorationParameterGenerator -) from maro.rl.scheduling.scheduler import Scheduler +from maro.rl.scheduling.simple_parameter_scheduler import LinearParameterScheduler, TwoPhaseLinearParameterScheduler from maro.rl.shaping.abs_shaper import AbsShaper from maro.rl.shaping.action_shaper import ActionShaper from maro.rl.shaping.experience_shaper import ExperienceShaper @@ -48,12 +45,11 @@ 'ColumnBasedStore', 'DQN', 'DQNHyperParams', - 'DynamicExplorationParameterGenerator', 'EpsilonGreedyExplorer', 'ExperienceShaper', 'FullyConnectedBlock', 'KStepExperienceShaper', - 'LinearExplorationParameterGenerator', + 'LinearParameterScheduler', 'MultiHeadLearningModel', 'OverwriteType', 'Scheduler', @@ -62,8 +58,7 @@ 'SimpleLearner', 'SingleHeadLearningModel', 'StateShaper', - 'StaticExplorationParameterGenerator', - 'TwoPhaseLinearExplorationParameterGenerator', + 'TwoPhaseLinearParameterScheduler', 'concat_experiences_by_agent', 'merge_experiences_with_trajectory_boundaries' ] diff --git a/maro/rl/learner/simple_learner.py b/maro/rl/learner/simple_learner.py index ec9bd6fe8..dbf43c468 100644 --- a/maro/rl/learner/simple_learner.py +++ b/maro/rl/learner/simple_learner.py @@ -45,10 +45,10 @@ def learn(self): exploration_params=exploration_params ) self._scheduler.record_performance(performance) - self._logger.info( - f"ep {self._scheduler.current_ep} - performance: {performance}, " - f"exploration_params: {self._scheduler.exploration_params}" - ) + ep_summary = f"ep {self._scheduler.current_ep} - performance: {performance}" + if exploration_params: + ep_summary = f"{ep_summary}, exploration_params: {self._scheduler.exploration_params}" + self._logger.info(ep_summary) self._agent_manager.train(exp_by_agent) def test(self): diff --git a/maro/rl/scheduling/scheduler.py b/maro/rl/scheduling/scheduler.py index ae6aef853..3ae49ad4d 100644 --- a/maro/rl/scheduling/scheduler.py +++ b/maro/rl/scheduling/scheduler.py @@ -1,79 +1,53 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT license. +from abc import ABC, abstractmethod from typing import Callable -from maro.utils.exception.rl_toolkit_exception import ( - InfiniteTrainingLoopError, InvalidEpisodeError, UnrecognizedExplorationParameterGeneratorClass -) +from maro.utils.exception.rl_toolkit_exception import InfiniteTrainingLoopError, InvalidEpisodeError -from .exploration_parameter_generator import DynamicExplorationParameterGenerator, StaticExplorationParameterGenerator - -class Scheduler(object): +class Scheduler(ABC): """Scheduler that generates exploration parameters for each episode. Args: - max_ep (int): Maximum number of episodes to be run. + max_ep (int): Maximum number of episodes to be run. If -1, an early stopping callback is expected to prevent + the training loop from running forever. early_stopping_callback (Callable): Function that returns a boolean indicating whether early stopping should be triggered. Defaults to None, in which case no early stopping check will be performed. - exploration_parameter_generator_cls: Subclass of StaticExplorationParameterGenerator or - DynamicExplorationParameterGenerator. Defaults to None, which means no exploration outside the algorithm. - exploration_parameter_generator_config (dict): Configuration for the exploration parameter generator. - Defaults to None. """ - def __init__( - self, - max_ep: int, - early_stopping_callback: Callable = None, - exploration_parameter_generator_cls=None, - exploration_parameter_generator_config: dict = None - ): + def __init__(self, max_ep: int, early_stopping_callback: Callable = None): if max_ep < -1: raise InvalidEpisodeError("max_episode can only be a non-negative integer or -1.") if max_ep == -1 and early_stopping_callback is None: raise InfiniteTrainingLoopError( - "The training loop will run forever since neither maximum episode nor early stopping checker " - "is provided. " + "A positive max_ep or an early stopping checker must be provided to prevent the training loop from " + "running forever." ) self._max_ep = max_ep self._early_stopping_callback = early_stopping_callback - self._current_ep = 0 + self._current_ep = -1 self._performance_history = [] self._exploration_params = None - if exploration_parameter_generator_cls is None: - self._exploration_parameter_generator = None - elif issubclass(exploration_parameter_generator_cls, StaticExplorationParameterGenerator): - self._exploration_parameter_generator = exploration_parameter_generator_cls( - max_ep, **exploration_parameter_generator_config - ) - elif issubclass(exploration_parameter_generator_cls, DynamicExplorationParameterGenerator): - self._exploration_parameter_generator = exploration_parameter_generator_cls( - **exploration_parameter_generator_config - ) - else: - raise UnrecognizedExplorationParameterGeneratorClass( - "exploration_parameter_generator_cls must be a subclass of StaticExplorationParameterGenerator " - "or DynamicExplorationParameterGenerator" - ) - def __iter__(self): return self def __next__(self): + self._current_ep += 1 if self._current_ep == self._max_ep: raise StopIteration if self._early_stopping_callback and self._early_stopping_callback(self._performance_history): raise StopIteration - if isinstance(self._exploration_parameter_generator, StaticExplorationParameterGenerator): - self._exploration_params = self._exploration_parameter_generator.next() - elif isinstance(self._exploration_parameter_generator, DynamicExplorationParameterGenerator): - self._exploration_params = self._exploration_parameter_generator.next(self._performance_history) + self._exploration_params = self._get_next_exploration_params() return self._exploration_params + @abstractmethod + def _get_next_exploration_params(self): + pass + @property def current_ep(self): return self._current_ep @@ -84,4 +58,3 @@ def exploration_params(self): def record_performance(self, performance): self._performance_history.append(performance) - self._current_ep += 1 diff --git a/maro/rl/scheduling/exploration_parameter_generator.py b/maro/rl/scheduling/simple_parameter_scheduler.py similarity index 79% rename from maro/rl/scheduling/exploration_parameter_generator.py rename to maro/rl/scheduling/simple_parameter_scheduler.py index 355decff1..7cdfe2570 100644 --- a/maro/rl/scheduling/exploration_parameter_generator.py +++ b/maro/rl/scheduling/simple_parameter_scheduler.py @@ -1,42 +1,20 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT license. -from abc import ABC, abstractmethod -from typing import Union +from typing import Callable, Union import numpy as np +from .scheduler import Scheduler -class StaticExplorationParameterGenerator(ABC): - """Exploration parameter generator based on a pre-defined schedule. - Args: - max_ep (int): Maximum number of episodes to run. - """ - def __init__(self, max_ep: int): - super().__init__() - self._max_ep = max_ep - - @abstractmethod - def next(self): - raise NotImplementedError - - -class DynamicExplorationParameterGenerator(ABC): - """Dynamic exploration parameter generator based on the performances history.""" - def __init__(self): - super().__init__() - - @abstractmethod - def next(self, performance_history: list): - raise NotImplementedError - - -class LinearExplorationParameterGenerator(StaticExplorationParameterGenerator): +class LinearParameterScheduler(Scheduler): """Static exploration parameter generator based on a linear schedule. Args: max_ep (int): Maximum number of episodes to run. + early_stopping_callback (Callable): Function that returns a boolean indicating whether early stopping should + be triggered. Defaults to None, in which case no early stopping check will be performed. parameter_names ([str]): List of exploration parameter names. start_values (Union[float, list, tuple, np.ndarray]): Exploration parameter values for the first episode. These values must correspond to ``parameter_names``. @@ -46,12 +24,13 @@ class LinearExplorationParameterGenerator(StaticExplorationParameterGenerator): def __init__( self, max_ep: int, + early_stopping_callback: Callable = None, *, parameter_names: [str], start_values: Union[float, list, tuple, np.ndarray], end_values: Union[float, list, tuple, np.ndarray] ): - super().__init__(max_ep) + super().__init__(max_ep, early_stopping_callback=early_stopping_callback) self._parameter_names = parameter_names if isinstance(start_values, float): self._current_values = start_values * np.ones(len(self._parameter_names)) @@ -65,19 +44,21 @@ def __init__( elif isinstance(end_values, (list, tuple)): end_values = np.asarray(end_values) - self._delta = (end_values - self._current_values) / (max_ep - 1) + self._delta = (end_values - self._current_values) / (self._max_ep - 1) - def next(self): + def _get_next_exploration_params(self): current_values = self._current_values.copy() self._current_values += self._delta return dict(zip(self._parameter_names, current_values)) -class TwoPhaseLinearExplorationParameterGenerator(StaticExplorationParameterGenerator): +class TwoPhaseLinearParameterScheduler(Scheduler): """Exploration parameter generator based on two linear schedules joined together. Args: max_ep (int): Maximum number of episodes to run. + early_stopping_callback (Callable): Function that returns a boolean indicating whether early stopping should + be triggered. Defaults to None, in which case no early stopping check will be performed. parameter_names ([str]): List of exploration parameter names. split_ep (float): The episode where the switch from the first linear schedule to the second occurs. start_values (Union[float, list, tuple, np.ndarray]): Exploration parameter values for the first episode. @@ -94,6 +75,7 @@ class TwoPhaseLinearExplorationParameterGenerator(StaticExplorationParameterGene def __init__( self, max_ep: int, + early_stopping_callback: Callable = None, *, parameter_names: [str], split_ep: float, @@ -103,7 +85,7 @@ def __init__( ): if split_ep <= 0 or split_ep >= max_ep: raise ValueError("split_ep must be between 0 and max_ep - 1.") - super().__init__(max_ep) + super().__init__(max_ep, early_stopping_callback=early_stopping_callback) self._parameter_names = parameter_names self._split_ep = split_ep if isinstance(start_values, float): @@ -125,10 +107,8 @@ def __init__( self._delta_1 = (mid_values - self._current_values) / split_ep self._delta_2 = (end_values - mid_values) / (max_ep - split_ep - 1) - self._current_ep = 0 - def next(self): + def _get_next_exploration_params(self): current_values = self._current_values.copy() self._current_values += self._delta_1 if self._current_ep < self._split_ep else self._delta_2 - self._current_ep += 1 return dict(zip(self._parameter_names, current_values)) diff --git a/maro/utils/exception/error_code.py b/maro/utils/exception/error_code.py index e46aef585..8965e7a27 100644 --- a/maro/utils/exception/error_code.py +++ b/maro/utils/exception/error_code.py @@ -44,6 +44,5 @@ 4003: "Wrong Agent Manager Mode", 4004: "Store Misalignment Error", 4005: "Invalid Episode", - 4006: "Infinite Training Loop", - 4010: "Unrecognized Exploration Parameter Generator Class", + 4006: "Infinite Training Loop" } diff --git a/maro/utils/exception/rl_toolkit_exception.py b/maro/utils/exception/rl_toolkit_exception.py index 76364b1fe..c35e0b897 100644 --- a/maro/utils/exception/rl_toolkit_exception.py +++ b/maro/utils/exception/rl_toolkit_exception.py @@ -39,12 +39,3 @@ class InfiniteTrainingLoopError(MAROException): """Raised when the ``SimpleLearner``'s training loop becomes infinite.""" def __init__(self, msg: str = None): super().__init__(4006, msg) - - -class UnrecognizedExplorationParameterGeneratorClass(MAROException): - """ - Raised when the ``exploration_parameter_generator_cls`` passed to a ``Scheduler`` is not a subclass of - ``StaticExplorationParameterGenerator`` or ``DynamicExplorationParameterGenerator``. - """ - def __init__(self, msg: str = None): - super().__init__(4009, msg) From bc7b841e7938906c823a69d38286c9d3c24fe5fe Mon Sep 17 00:00:00 2001 From: ysqyang Date: Thu, 24 Dec 2020 14:51:51 +0800 Subject: [PATCH 295/337] edited notebook according to merged scheduler changes --- examples/cim/dqn/config.yml | 2 +- examples/cim/dqn/single_process_launcher.py | 1 - .../rl_formulation.ipynb | 37 +++++++++---------- 3 files changed, 19 insertions(+), 21 deletions(-) diff --git a/examples/cim/dqn/config.yml b/examples/cim/dqn/config.yml index c893ce04b..69e29ab98 100644 --- a/examples/cim/dqn/config.yml +++ b/examples/cim/dqn/config.yml @@ -52,4 +52,4 @@ agents: min_experiences_to_train: 1024 num_batches: 10 batch_size: 128 - seed: 1024 # for reproducibility + seed: 1 # for reproducibility diff --git a/examples/cim/dqn/single_process_launcher.py b/examples/cim/dqn/single_process_launcher.py index 3c41d7824..81f0d29fa 100644 --- a/examples/cim/dqn/single_process_launcher.py +++ b/examples/cim/dqn/single_process_launcher.py @@ -6,7 +6,6 @@ import numpy as np - from maro.rl import AgentManagerMode, SimpleActor, SimpleLearner, TwoPhaseLinearParameterScheduler from maro.simulator import Env from maro.utils import LogFormat, Logger, convert_dottable diff --git a/notebooks/container_inventory_management/rl_formulation.ipynb b/notebooks/container_inventory_management/rl_formulation.ipynb index 8d90b2be7..97ea8b28d 100644 --- a/notebooks/container_inventory_management/rl_formulation.ipynb +++ b/notebooks/container_inventory_management/rl_formulation.ipynb @@ -249,7 +249,7 @@ "\n", "\n", "def create_dqn_agents(agent_id_list):\n", - " set_seeds(1024) # for reproducibility\n", + " set_seeds(1) # for reproducibility\n", " agent_dict = {}\n", " for agent_id in agent_id_list:\n", " q_module = LearningModule(\n", @@ -325,33 +325,32 @@ "outputs": [], "source": [ "from maro.simulator import Env\n", - "from maro.rl import AgentManagerMode, Scheduler, SimpleActor, SimpleLearner, TwoPhaseLinearExplorationParameterGenerator\n", - "from maro.utils import Logger, LogFormat\n", + "from maro.rl import AgentManagerMode, SimpleActor, SimpleLearner, TwoPhaseLinearParameterScheduler\n", + "from maro.utils import LogFormat, Logger\n", "\n", "# Step 1: initialize a CIM environment for a toy dataset. \n", "env = Env(\"cim\", \"toy.4p_ssdd_l0.0\", durations=1120)\n", "agent_id_list = [str(agent_id) for agent_id in env.agent_idx_list]\n", "\n", "# Step 2: create DQN agents and an agent manager to manage them.\n", - "agent_manager = DQNAgentManager(name=\"cim_learner\",\n", - " mode=AgentManagerMode.TRAIN_INFERENCE,\n", - " agent_dict=create_dqn_agents(agent_id_list),\n", - " state_shaper=state_shaper,\n", - " action_shaper=action_shaper,\n", - " experience_shaper=experience_shaper)\n", + "agent_manager = DQNAgentManager(\n", + " name=\"cim_learner\",\n", + " mode=AgentManagerMode.TRAIN_INFERENCE,\n", + " agent_dict=create_dqn_agents(agent_id_list),\n", + " state_shaper=state_shaper,\n", + " action_shaper=action_shaper,\n", + " experience_shaper=experience_shaper\n", + ")\n", "\n", "# Step 3: Create an actor and a learner to start the training process. \n", "max_episode = 100\n", - "scheduler = Scheduler(\n", - " max_episode, \n", - " exploration_parameter_generator_cls=TwoPhaseLinearExplorationParameterGenerator,\n", - " exploration_parameter_generator_config={\n", - " \"parameter_names\": [\"epsilon\"], \n", - " \"split_ep\": 50, \n", - " \"start_values\": 0.4, \n", - " \"mid_values\": 0.32, \n", - " \"end_values\": .0\n", - " }\n", + "scheduler = TwoPhaseLinearParameterScheduler(\n", + " max_episode,\n", + " parameter_names=[\"epsilon\"],\n", + " split_ep=50,\n", + " start_values=0.4,\n", + " mid_values=0.32,\n", + " end_values=.0\n", ")\n", "\n", "actor = SimpleActor(env, agent_manager)\n", From 8ad6378fc4864df22fbd922ab7524c692c539005 Mon Sep 17 00:00:00 2001 From: ysqyang Date: Thu, 24 Dec 2020 16:17:26 +0800 Subject: [PATCH 296/337] refined dimension check for learning module manager and removed num_actions from DQNConfig --- maro/rl/agent/abs_agent_manager.py | 6 +-- maro/rl/agent/simple_agent_manager.py | 8 ++-- maro/rl/algorithms/dqn.py | 13 +++--- maro/rl/algorithms/utils.py | 4 +- maro/rl/models/learning_model.py | 21 +++++++-- maro/rl/models/utils.py | 17 ++++---- maro/rl/scheduling/scheduler.py | 6 +-- maro/rl/storage/column_based_store.py | 6 +-- maro/rl/storage/utils.py | 4 +- maro/utils/exception/error_code.py | 17 ++++---- maro/utils/exception/rl_toolkit_exception.py | 45 ++++++++------------ 11 files changed, 76 insertions(+), 71 deletions(-) diff --git a/maro/rl/agent/abs_agent_manager.py b/maro/rl/agent/abs_agent_manager.py index 731f2957e..b746ea241 100644 --- a/maro/rl/agent/abs_agent_manager.py +++ b/maro/rl/agent/abs_agent_manager.py @@ -7,7 +7,7 @@ from maro.rl.shaping.action_shaper import ActionShaper from maro.rl.shaping.experience_shaper import ExperienceShaper from maro.rl.shaping.state_shaper import StateShaper -from maro.utils.exception.rl_toolkit_exception import WrongAgentManagerModeError +from maro.utils.exception.rl_toolkit_exception import AgentManagerModeError class AgentManagerMode(Enum): @@ -99,8 +99,8 @@ def set_exploration_params(self, params): def _assert_train_mode(self): if self._mode != AgentManagerMode.TRAIN and self._mode != AgentManagerMode.TRAIN_INFERENCE: - raise WrongAgentManagerModeError(msg=f"this method is unavailable under mode {self._mode}") + raise AgentManagerModeError(msg=f"this method is unavailable under mode {self._mode}") def _assert_inference_mode(self): if self._mode != AgentManagerMode.INFERENCE and self._mode != AgentManagerMode.TRAIN_INFERENCE: - raise WrongAgentManagerModeError(msg=f"this method is unavailable under mode {self._mode}") + raise AgentManagerModeError(msg=f"this method is unavailable under mode {self._mode}") diff --git a/maro/rl/agent/simple_agent_manager.py b/maro/rl/agent/simple_agent_manager.py index 057cb3997..1f1760469 100644 --- a/maro/rl/agent/simple_agent_manager.py +++ b/maro/rl/agent/simple_agent_manager.py @@ -8,7 +8,7 @@ from maro.rl.shaping.experience_shaper import ExperienceShaper from maro.rl.shaping.state_shaper import StateShaper from maro.rl.storage.column_based_store import ColumnBasedStore -from maro.utils.exception.rl_toolkit_exception import MissingShaperError +from maro.utils.exception.rl_toolkit_exception import MissingShaper from .abs_agent_manager import AbsAgentManager, AgentManagerMode @@ -25,11 +25,11 @@ def __init__( ): if mode in {AgentManagerMode.INFERENCE, AgentManagerMode.TRAIN_INFERENCE}: if state_shaper is None: - raise MissingShaperError(msg=f"state shaper cannot be None under mode {self._mode}") + raise MissingShaper(msg=f"state shaper cannot be None under mode {self._mode}") if action_shaper is None: - raise MissingShaperError(msg=f"action_shaper cannot be None under mode {self._mode}") + raise MissingShaper(msg=f"action_shaper cannot be None under mode {self._mode}") if experience_shaper is None: - raise MissingShaperError(msg=f"experience_shaper cannot be None under mode {self._mode}") + raise MissingShaper(msg=f"experience_shaper cannot be None under mode {self._mode}") super().__init__( name, mode, agent_dict, diff --git a/maro/rl/algorithms/dqn.py b/maro/rl/algorithms/dqn.py index 28dd2a5e0..f3a0bb7ee 100644 --- a/maro/rl/algorithms/dqn.py +++ b/maro/rl/algorithms/dqn.py @@ -20,7 +20,6 @@ class DQNConfig: """Configuration for the DQN algorithm. Args: - num_actions (int): Number of possible actions. reward_decay (float): Reward decay as defined in standard RL terminology. loss_cls: Loss function class for evaluating TD errors. target_update_frequency (int): Number of training rounds between target model updates. @@ -35,13 +34,12 @@ class DQNConfig: method. Defaults to False. """ __slots__ = [ - "num_actions", "reward_decay", "loss_func", "target_update_frequency", "epsilon", "tau", "is_double", - "advantage_mode", "per_sample_td_error_enabled" + "reward_decay", "loss_func", "target_update_frequency", "epsilon", "tau", "is_double", "advantage_mode", + "per_sample_td_error_enabled" ] def __init__( self, - num_actions: int, reward_decay: float, loss_cls, target_update_frequency: int, @@ -51,7 +49,6 @@ def __init__( advantage_mode: str = None, per_sample_td_error_enabled: bool = False ): - self.num_actions = num_actions self.reward_decay = reward_decay self.target_update_frequency = target_update_frequency self.epsilon = epsilon @@ -75,13 +72,17 @@ class DQN(AbsAlgorithm): @validate_task_names(DuelingDQNTask) def __init__(self, model: LearningModuleManager, config: DQNConfig): super().__init__(model, config) + if isinstance(self._model.output_dim, int): + self._num_actions = self._model.output_dim + else: + self._num_actions = self._model.output_dim[DuelingDQNTask.ADVANTAGE.value] self._training_counter = 0 self._target_model = model.copy() if model.is_trainable else None @expand_dim def choose_action(self, state: np.ndarray): if np.random.random() < self._config.epsilon: - return np.random.choice(self._config.num_actions) + return np.random.choice(self._num_actions) else: return self._get_q_values(self._model, state, is_training=False).argmax(dim=1).data diff --git a/maro/rl/algorithms/utils.py b/maro/rl/algorithms/utils.py index 7d6952d56..7a2465298 100644 --- a/maro/rl/algorithms/utils.py +++ b/maro/rl/algorithms/utils.py @@ -8,7 +8,7 @@ import numpy as np import torch -from maro.utils.exception.rl_toolkit_exception import UnrecognizedTaskError +from maro.utils.exception.rl_toolkit_exception import UnrecognizedTask device = environ.get("DEVICE", torch.device("cuda" if torch.cuda.is_available() else "cpu")) @@ -20,7 +20,7 @@ def wrapper(self, model, config): recognized_task_names = set(member.value for member in task_enum) model_task_names = set(model.task_names) if len(model_task_names) > 1 and model_task_names != recognized_task_names: - raise UnrecognizedTaskError(f"Expected task names {recognized_task_names}, got {model_task_names}") + raise UnrecognizedTask(f"Expected task names {recognized_task_names}, got {model_task_names}") init_func(self, model, config) diff --git a/maro/rl/models/learning_model.py b/maro/rl/models/learning_model.py index 93034cfa4..fb0f8d3d9 100644 --- a/maro/rl/models/learning_model.py +++ b/maro/rl/models/learning_model.py @@ -7,10 +7,10 @@ import torch.nn as nn from maro.utils import clone -from maro.utils.exception.rl_toolkit_exception import MissingOptimizerError +from maro.utils.exception.rl_toolkit_exception import MissingOptimizer from .abs_block import AbsBlock -from .utils import check_chainability +from .utils import validate_dims OptimizerOptions = namedtuple("OptimizerOptions", ["cls", "params"]) @@ -76,7 +76,7 @@ def forward(self, inputs): def zero_gradients(self): if not self._is_trainable: - raise MissingOptimizerError("No optimizer registered to the model") + raise MissingOptimizer("No optimizer registered to the model") self._optimizer.zero_grad() def step(self): @@ -93,7 +93,7 @@ class LearningModuleManager(nn.Module): task_modules (LearningModule): LearningModule instances, each of which performs a designated task. shared_module (LearningModule): Network module that forms that shared part of the model. Defaults to None. """ - @check_chainability + @validate_dims def __init__( self, *task_modules: LearningModule, @@ -107,6 +107,11 @@ def __init__( # task_heads self._task_module_dict = nn.ModuleDict({task_module.name: task_module for task_module in task_modules}) + self._input_dim = self._shared_module.input_dim if self._shared_module else task_modules[0].input_dim + if len(task_modules) == 1: + self._output_dim = task_modules[0].output_dim + else: + self._output_dim = {task_module.name: task_module.output_dim for task_module in task_modules} @property def task_names(self) -> [str]: @@ -116,6 +121,14 @@ def task_names(self) -> [str]: def shared_module(self): return self._shared_module + @property + def input_dim(self): + return self._input_dim + + @property + def output_dim(self): + return self._output_dim + @property def is_trainable(self) -> bool: return ( diff --git a/maro/rl/models/utils.py b/maro/rl/models/utils.py index a6875726e..10f922ea6 100644 --- a/maro/rl/models/utils.py +++ b/maro/rl/models/utils.py @@ -1,17 +1,18 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT license. -from maro.utils.exception.rl_toolkit_exception import UnchainableModuleError +from maro.utils.exception.rl_toolkit_exception import LearningModuleDimensionError -def check_chainability(init_func): +def validate_dims(init_func): def decorator(self, *task_modules, shared_module=None): - if shared_module is not None: - for task_module in task_modules: - if shared_module.output_dim != task_module.input_dim: - raise UnchainableModuleError( - f"Expected input dimension {shared_module.output_dim} for {task_module.name}, " - f"got {task_module.input_dim}") + expected_dim = shared_module.output_dim if shared_module else task_modules[0].input_dim + for task_module in task_modules: + if task_module.input_dim != expected_dim: + raise LearningModuleDimensionError( + f"Expected input dimension {expected_dim} for {task_module.name}, " + f"got {task_module.input_dim}") + init_func(self, *task_modules, shared_module=shared_module) return decorator diff --git a/maro/rl/scheduling/scheduler.py b/maro/rl/scheduling/scheduler.py index 3ae49ad4d..2d42099ce 100644 --- a/maro/rl/scheduling/scheduler.py +++ b/maro/rl/scheduling/scheduler.py @@ -4,7 +4,7 @@ from abc import ABC, abstractmethod from typing import Callable -from maro.utils.exception.rl_toolkit_exception import InfiniteTrainingLoopError, InvalidEpisodeError +from maro.utils.exception.rl_toolkit_exception import InvalidTrainingLoop, InvalidEpisode class Scheduler(ABC): @@ -19,9 +19,9 @@ class Scheduler(ABC): def __init__(self, max_ep: int, early_stopping_callback: Callable = None): if max_ep < -1: - raise InvalidEpisodeError("max_episode can only be a non-negative integer or -1.") + raise InvalidEpisode("max_episode can only be a non-negative integer or -1.") if max_ep == -1 and early_stopping_callback is None: - raise InfiniteTrainingLoopError( + raise InvalidTrainingLoop( "A positive max_ep or an early stopping checker must be provided to prevent the training loop from " "running forever." ) diff --git a/maro/rl/storage/column_based_store.py b/maro/rl/storage/column_based_store.py index 063a5f42b..3a50fc856 100644 --- a/maro/rl/storage/column_based_store.py +++ b/maro/rl/storage/column_based_store.py @@ -7,7 +7,7 @@ import numpy as np from maro.utils import clone -from maro.utils.exception.rl_toolkit_exception import StoreMisalignmentError +from maro.utils.exception.rl_toolkit_exception import StoreMisalignment from .abs_store import AbsStore from .utils import OverwriteType, check_uniformity, get_update_indexes, normalize @@ -88,7 +88,7 @@ def put(self, contents: dict, overwrite_indexes: Sequence = None) -> List[int]: Args: contents (dict): dictionary of items to add to the store. If the store is not empty, this must have the - same keys as the store itself. Otherwise an ``StoreMisalignmentError`` will be raised. + same keys as the store itself. Otherwise an ``StoreMisalignment`` will be raised. overwrite_indexes (Sequence, optional): indexes where the contents are to be overwritten. This is only used when the store has a fixed capacity and putting ``contents`` in the store would exceed this capacity. If this is None and overwriting is necessary, rolling or random overwriting will be done @@ -97,7 +97,7 @@ def put(self, contents: dict, overwrite_indexes: Sequence = None) -> List[int]: The indexes where the newly added entries reside in the store. """ if len(self._store) > 0 and contents.keys() != self._store.keys(): - raise StoreMisalignmentError(f"expected keys {list(self._store.keys())}, got {list(contents.keys())}") + raise StoreMisalignment(f"expected keys {list(self._store.keys())}, got {list(contents.keys())}") added = contents[next(iter(contents))] added_size = len(added) if isinstance(added, list) else 1 if self._capacity < 0: diff --git a/maro/rl/storage/utils.py b/maro/rl/storage/utils.py index 9c4da220b..279853fd0 100644 --- a/maro/rl/storage/utils.py +++ b/maro/rl/storage/utils.py @@ -6,7 +6,7 @@ import numpy as np -from maro.utils.exception.rl_toolkit_exception import StoreMisalignmentError +from maro.utils.exception.rl_toolkit_exception import StoreMisalignment def check_uniformity(arg_num): @@ -18,7 +18,7 @@ def wrapper(*args, **kwargs): return func(*args, **kwargs) col_length = len(contents[next(iter(contents))]) if any(not isinstance(val, list) or len(val) != col_length for val in contents.values()): - raise StoreMisalignmentError("values of contents should consist of lists of the same length") + raise StoreMisalignment("values of contents should consist of lists of the same length") return func(*args, **kwargs) return wrapper return decorator diff --git a/maro/utils/exception/error_code.py b/maro/utils/exception/error_code.py index 6deb33746..bed5ac4c3 100644 --- a/maro/utils/exception/error_code.py +++ b/maro/utils/exception/error_code.py @@ -39,13 +39,12 @@ 3003: "Deployment Error", # 4000-4999: Error codes for RL toolkit - 4001: "Unsupported Agent Mode", - 4002: "Missing Shaper", - 4003: "Wrong Agent Manager Mode", - 4004: "Store Misalignment", - 4005: "Invalid Episode", - 4006: "Infinite Training Loop", - 4007: "Missing Optimizer", - 4008: "Unrecognized Task", - 4009: "Unchainable Modules", + 4000: "Wrong Agent Manager Mode", + 4001: "Missing Shaper", + 4002: "Store Misalignment", + 4003: "Invalid Episode", + 4004: "Infinite Training Loop", + 4005: "Missing Optimizer", + 4006: "Unrecognized Task", + 4007: "Wrong Learning Module Dimension", } diff --git a/maro/utils/exception/rl_toolkit_exception.py b/maro/utils/exception/rl_toolkit_exception.py index f487997cf..7e347630a 100644 --- a/maro/utils/exception/rl_toolkit_exception.py +++ b/maro/utils/exception/rl_toolkit_exception.py @@ -4,59 +4,50 @@ from .base_exception import MAROException -class UnsupportedAgentManagerModeError(MAROException): - """Unsupported agent mode.""" +class AgentManagerModeError(MAROException): + """Wrong agent manager mode.""" def __init__(self, msg: str = None): - super().__init__(4001, msg) + super().__init__(4000, msg) -class MissingShaperError(MAROException): +class MissingShaper(MAROException): """Missing shaper.""" def __init__(self, msg: str = None): - super().__init__(4002, msg) - - -class WrongAgentManagerModeError(MAROException): - """Wrong agent manager mode.""" - def __init__(self, msg: str = None): - super().__init__(4003, msg) + super().__init__(4001, msg) -class StoreMisalignmentError(MAROException): +class StoreMisalignment(MAROException): """Raised when a ``put`` operation on a ``ColumnBasedStore`` would cause the underlying lists to have different sizes.""" def __init__(self, msg: str = None): - super().__init__(4004, msg) + super().__init__(4002, msg) -class InvalidEpisodeError(MAROException): +class InvalidEpisode(MAROException): """Raised when the ``max_episode`` passed to the the ``SimpleLearner``'s ``train`` method is negative and not -1.""" def __init__(self, msg: str = None): - super().__init__(4005, msg) + super().__init__(4003, msg) -class InfiniteTrainingLoopError(MAROException): +class InvalidTrainingLoop(MAROException): """Raised when the ``SimpleLearner``'s training loop becomes infinite.""" def __init__(self, msg: str = None): - super().__init__(4006, msg) + super().__init__(4004, msg) -class MissingOptimizerError(MAROException): +class MissingOptimizer(MAROException): """Raised when the optimizers are missing when calling LearningModuleManager's step() method.""" def __init__(self, msg: str = None): - super().__init__(4007, msg) + super().__init__(4005, msg) -class UnrecognizedTaskError(MAROException): +class UnrecognizedTask(MAROException): """Raised when a LearningModuleManager has task names that are not unrecognized by an algorithm.""" def __init__(self, msg: str = None): - super().__init__(4008, msg) + super().__init__(4006, msg) -class UnchainableModuleError(MAROException): - """ - Raised when the modules passed to a LearningModuleManager have incorrect input/output dimensions that make them - unchainable. - """ +class LearningModuleDimensionError(MAROException): + """Raised when a learning module's input dimension is incorrect.""" def __init__(self, msg: str = None): - super().__init__(4009, msg) + super().__init__(4007, msg) From d4fe0dbd3ae7083eda42bf1cd8914ff822c81a31 Mon Sep 17 00:00:00 2001 From: ysqyang Date: Thu, 24 Dec 2020 16:18:29 +0800 Subject: [PATCH 297/337] bug fix for cim example --- examples/cim/dqn/components/agent_manager.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/examples/cim/dqn/components/agent_manager.py b/examples/cim/dqn/components/agent_manager.py index 6517db167..1c2cbe9bd 100644 --- a/examples/cim/dqn/components/agent_manager.py +++ b/examples/cim/dqn/components/agent_manager.py @@ -32,11 +32,7 @@ def create_dqn_agents(agent_id_list, config): algorithm = DQN( model=LearningModuleManager(q_module), - config=DQNConfig( - **config.algorithm.config, - loss_cls=nn.SmoothL1Loss, - num_actions=num_actions - ) + config=DQNConfig(**config.algorithm.config, loss_cls=nn.SmoothL1Loss) ) agent_dict[agent_id] = CIMAgent( From 9fe7590cc2b2fc57549af93a407d96f8f9a38da4 Mon Sep 17 00:00:00 2001 From: ysqyang Date: Thu, 24 Dec 2020 16:36:37 +0800 Subject: [PATCH 298/337] added notebook output --- .../rl_formulation.ipynb | 908 ++++++++++-------- 1 file changed, 513 insertions(+), 395 deletions(-) diff --git a/notebooks/container_inventory_management/rl_formulation.ipynb b/notebooks/container_inventory_management/rl_formulation.ipynb index 97ea8b28d..2f1f706e2 100644 --- a/notebooks/container_inventory_management/rl_formulation.ipynb +++ b/notebooks/container_inventory_management/rl_formulation.ipynb @@ -1,395 +1,513 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Quick Start\n", - "\n", - "This notebook demonstrates how to use MARO's reinforcement learning (RL) toolkit to solve the container inventory management ([CIM](https://maro.readthedocs.io/en/latest/scenarios/container_inventory_management.html)) problem. It is formalized as a multi-agent reinforcement learning problem, where each port acts as a decision agent. The agents take actions independently, e.g., loading containers to vessels or discharging containers from vessels. " - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## [State Shaper](https://maro.readthedocs.io/en/latest/key_components/rl_toolkit.html#shapers)\n", - "\n", - "State shaper converts the environment observation to the model input state which includes temporal and spatial information. For this scenario, the model input state includes: \n", - "\n", - "- Temporal information, including the past week's information of ports and vessels, such as shortage on port and remaining space on vessel. \n", - "\n", - "- Spatial information, it including the related downstream port features. " - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "import numpy as np\n", - "from maro.rl import StateShaper\n", - "\n", - "\n", - "PORT_ATTRIBUTES = [\"empty\", \"full\", \"on_shipper\", \"on_consignee\", \"booking\", \"shortage\", \"fulfillment\"]\n", - "VESSEL_ATTRIBUTES = [\"empty\", \"full\", \"remaining_space\"]\n", - "\n", - "\n", - "class CIMStateShaper(StateShaper):\n", - " def __init__(self, *, look_back, max_ports_downstream):\n", - " super().__init__()\n", - " self._look_back = look_back\n", - " self._max_ports_downstream = max_ports_downstream\n", - " self._dim = (look_back + 1) * (max_ports_downstream + 1) * len(PORT_ATTRIBUTES) + len(VESSEL_ATTRIBUTES)\n", - "\n", - " def __call__(self, decision_event, snapshot_list):\n", - " tick, port_idx, vessel_idx = decision_event.tick, decision_event.port_idx, decision_event.vessel_idx\n", - " ticks = [tick - rt for rt in range(self._look_back - 1)]\n", - " future_port_idx_list = snapshot_list[\"vessels\"][tick: vessel_idx: 'future_stop_list'].astype('int')\n", - " port_features = snapshot_list[\"ports\"][ticks: [port_idx] + list(future_port_idx_list): PORT_ATTRIBUTES]\n", - " vessel_features = snapshot_list[\"vessels\"][tick: vessel_idx: VESSEL_ATTRIBUTES]\n", - " state = np.concatenate((port_features, vessel_features))\n", - " return str(port_idx), state\n", - "\n", - " @property\n", - " def dim(self):\n", - " return self._dim\n", - " \n", - "# Create a state shaper\n", - "state_shaper = CIMStateShaper(look_back=7, max_ports_downstream=2)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## [Action Shaper](https://maro.readthedocs.io/en/latest/key_components/rl_toolkit.html#shapers)\n", - "\n", - "Action shaper is used to convert an agent's model output to an environment executable action. For this specific scenario, the output is a discrete index that corresponds to a percentage indicating the fraction of containers to be loaded to or discharged from the arriving vessel." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from maro.rl import ActionShaper\n", - "from maro.simulator.scenarios.cim.common import Action\n", - "\n", - "\n", - "class CIMActionShaper(ActionShaper):\n", - " def __init__(self, action_space):\n", - " super().__init__()\n", - " self._action_space = action_space\n", - " self._zero_action_index = action_space.index(0)\n", - "\n", - " def __call__(self, model_action, decision_event, snapshot_list):\n", - " assert 0 <= model_action < len(self._action_space)\n", - " \n", - " scope = decision_event.action_scope\n", - " tick = decision_event.tick\n", - " port_idx = decision_event.port_idx\n", - " vessel_idx = decision_event.vessel_idx\n", - " port_empty = snapshot_list[\"ports\"][tick: port_idx: [\"empty\", \"full\", \"on_shipper\", \"on_consignee\"]][0]\n", - " vessel_remaining_space = snapshot_list[\"vessels\"][tick: vessel_idx: [\"empty\", \"full\", \"remaining_space\"]][2]\n", - " early_discharge = snapshot_list[\"vessels\"][tick:vessel_idx: \"early_discharge\"][0]\n", - " \n", - " if model_action < self._zero_action_index:\n", - " # The number of loaded containers must be less than the vessel's remaining space.\n", - " actual_action = max(round(self._action_space[model_action] * port_empty), -vessel_remaining_space)\n", - " elif model_action > self._zero_action_index:\n", - " # In the case of an early discharge event, we need to subtract the early discharge amount from the expected \n", - " # discharge quote. \n", - " plan_action = self._action_space[model_action] * (scope.discharge + early_discharge) - early_discharge\n", - " actual_action = round(plan_action) if plan_action > 0 else round(self._action_space[model_action] * scope.discharge)\n", - " else:\n", - " actual_action = 0\n", - "\n", - " return Action(vessel_idx, port_idx, actual_action)\n", - " \n", - "# Create an action shaper\n", - "NUM_ACTIONS = 21\n", - "action_shaper = CIMActionShaper(action_space=list(np.linspace(-1.0, 1.0, NUM_ACTIONS)))" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## [Experience Shaper](https://maro.readthedocs.io/en/latest/key_components/rl_toolkit.html#shapers)\n", - "\n", - "Experience shaper is used to convert an episode trajectory to trainable experiences for RL agents. For this specific scenario, the reward is a linear combination of fulfillment and shortage in a limited time window." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from collections import defaultdict\n", - "\n", - "from maro.rl import ExperienceShaper\n", - "\n", - "\n", - "class TruncatedExperienceShaper(ExperienceShaper):\n", - " def __init__(\n", - " self, *, time_window: int, time_decay_factor: float, fulfillment_factor: float, shortage_factor: float\n", - " ):\n", - " super().__init__(reward_func=None)\n", - " self._time_window = time_window\n", - " self._time_decay_factor = time_decay_factor\n", - " self._fulfillment_factor = fulfillment_factor\n", - " self._shortage_factor = shortage_factor\n", - "\n", - " def __call__(self, trajectory, snapshot_list):\n", - " experiences_by_agent = {}\n", - " for i in range(len(trajectory) - 1):\n", - " transition = trajectory[i]\n", - " agent_id = transition[\"agent_id\"]\n", - " if agent_id not in experiences_by_agent:\n", - " experiences_by_agent[agent_id] = defaultdict(list)\n", - " experiences = experiences_by_agent[agent_id]\n", - " experiences[\"state\"].append(transition[\"state\"])\n", - " experiences[\"action\"].append(transition[\"action\"])\n", - " experiences[\"reward\"].append(self._compute_reward(transition[\"event\"], snapshot_list))\n", - " experiences[\"next_state\"].append(trajectory[i + 1][\"state\"])\n", - "\n", - " return experiences_by_agent\n", - "\n", - " def _compute_reward(self, decision_event, snapshot_list):\n", - " start_tick = decision_event.tick + 1\n", - " end_tick = decision_event.tick + self._time_window\n", - " ticks = list(range(start_tick, end_tick))\n", - "\n", - " # calculate tc reward\n", - " future_fulfillment = snapshot_list[\"ports\"][ticks::\"fulfillment\"]\n", - " future_shortage = snapshot_list[\"ports\"][ticks::\"shortage\"]\n", - " decay_list = [\n", - " self._time_decay_factor ** i for i in range(end_tick - start_tick)\n", - " for _ in range(future_fulfillment.shape[0] // (end_tick - start_tick))\n", - " ]\n", - "\n", - " tot_fulfillment = np.dot(future_fulfillment, decay_list)\n", - " tot_shortage = np.dot(future_shortage, decay_list)\n", - "\n", - " return np.float32(self._fulfillment_factor * tot_fulfillment - self._shortage_factor * tot_shortage)\n", - " \n", - "# Create an experience shaper\n", - "experience_shaper = TruncatedExperienceShaper(time_window=100, fulfillment_factor=1.0, shortage_factor=1.0, time_decay_factor=0.97)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## [Agent](https://maro.readthedocs.io/en/latest/key_components/rl_toolkit.html#agent)\n", - "\n", - "For this scenario, the agent is the abstraction of a port. We choose DQN as our underlying learning algorithm with a TD-error-based sampling mechanism. " - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from maro.rl import AbsAgent, ColumnBasedStore\n", - "\n", - "\n", - "class CIMAgent(AbsAgent):\n", - " def __init__(self, name, algorithm, experience_pool: ColumnBasedStore, min_experiences_to_train, num_batches, batch_size):\n", - " super().__init__(name, algorithm, experience_pool)\n", - " self._min_experiences_to_train = min_experiences_to_train\n", - " self._num_batches = num_batches\n", - " self._batch_size = batch_size\n", - "\n", - " def train(self):\n", - " if len(self._experience_pool) < self._min_experiences_to_train:\n", - " return\n", - "\n", - " for _ in range(self._num_batches):\n", - " indexes, sample = self._experience_pool.sample_by_key(\"loss\", self._batch_size)\n", - " state = np.asarray(sample[\"state\"])\n", - " action = np.asarray(sample[\"action\"])\n", - " reward = np.asarray(sample[\"reward\"])\n", - " next_state = np.asarray(sample[\"next_state\"])\n", - " loss = self._algorithm.train(state, action, reward, next_state)\n", - " self._experience_pool.update(indexes, {\"loss\": loss})" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## [Agent Manager](https://maro.readthedocs.io/en/latest/key_components/rl_toolkit.html#agent-manager)\n", - "\n", - "The complexities of the environment can be isolated from the learning algorithm by using an AgentManager to manage individual agents. We define a function to create the agents and an agent manager class that implements the ``train`` method where the newly obtained experiences are stored in the agents' experience pools before training, in accordance with the DQN algorithm." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "import io\n", - "import yaml\n", - "\n", - "import torch.nn as nn\n", - "from torch.nn.functional import smooth_l1_loss\n", - "from torch.optim import RMSprop\n", - "\n", - "from maro.rl import (\n", - " ColumnBasedStore, DQN, DQNConfig, FullyConnectedBlock, LearningModuleManager, LearningModule, OptimizerOptions, SimpleAgentManager\n", - ")\n", - "from maro.utils import set_seeds\n", - "\n", - "\n", - "def create_dqn_agents(agent_id_list):\n", - " set_seeds(1) # for reproducibility\n", - " agent_dict = {}\n", - " for agent_id in agent_id_list:\n", - " q_module = LearningModule(\n", - " \"q_value\",\n", - " [FullyConnectedBlock(\n", - " input_dim=state_shaper.dim,\n", - " hidden_dims=[256, 128, 64],\n", - " output_dim=NUM_ACTIONS,\n", - " activation=nn.LeakyReLU,\n", - " is_head=True,\n", - " batch_norm_enabled=True, \n", - " softmax_enabled=False,\n", - " skip_connection_enabled=False,\n", - " dropout_p=.0)\n", - " ],\n", - " optimizer_options=OptimizerOptions(cls=RMSprop, params={\"lr\": 0.05})\n", - " )\n", - "\n", - " algorithm = DQN(\n", - " model=LearningModuleManager(q_module),\n", - " config=DQNConfig(\n", - " reward_decay=.0, \n", - " target_update_frequency=5, \n", - " tau=0.1, \n", - " is_double=True, \n", - " per_sample_td_error_enabled=True,\n", - " loss_cls=nn.SmoothL1Loss,\n", - " num_actions=NUM_ACTIONS\n", - " )\n", - " )\n", - "\n", - " agent_dict[agent_id] = CIMAgent(\n", - " agent_id, algorithm, ColumnBasedStore(), min_experiences_to_train=1024, num_batches=10, batch_size=128\n", - " )\n", - "\n", - " return agent_dict\n", - "\n", - "\n", - "class DQNAgentManager(SimpleAgentManager):\n", - " def train(self, experiences_by_agent, performance=None):\n", - " self._assert_train_mode()\n", - "\n", - " # store experiences for each agent\n", - " for agent_id, exp in experiences_by_agent.items():\n", - " exp.update({\"loss\": [1e8] * len(list(exp.values())[0])})\n", - " self.agent_dict[agent_id].store_experiences(exp)\n", - "\n", - " for agent in self.agent_dict.values():\n", - " agent.train()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Main Loop with [Actor and Learner](https://maro.readthedocs.io/en/latest/key_components/rl_toolkit.html#learner-and-actor)\n", - "\n", - "This code cell demonstrates the typical workflow of a learning policy's interaction with a MARO environment. \n", - "\n", - "- Initialize an environment with specific scenario and topology parameters. \n", - "\n", - "- Define scenario-specific components, e.g. shapers. \n", - "\n", - "- Create agents and an agent manager. \n", - "\n", - "- Create an actor and a learner to start the training process in which the agent manager interacts with the environment for collecting experiences and updating policies. " - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from maro.simulator import Env\n", - "from maro.rl import AgentManagerMode, SimpleActor, SimpleLearner, TwoPhaseLinearParameterScheduler\n", - "from maro.utils import LogFormat, Logger\n", - "\n", - "# Step 1: initialize a CIM environment for a toy dataset. \n", - "env = Env(\"cim\", \"toy.4p_ssdd_l0.0\", durations=1120)\n", - "agent_id_list = [str(agent_id) for agent_id in env.agent_idx_list]\n", - "\n", - "# Step 2: create DQN agents and an agent manager to manage them.\n", - "agent_manager = DQNAgentManager(\n", - " name=\"cim_learner\",\n", - " mode=AgentManagerMode.TRAIN_INFERENCE,\n", - " agent_dict=create_dqn_agents(agent_id_list),\n", - " state_shaper=state_shaper,\n", - " action_shaper=action_shaper,\n", - " experience_shaper=experience_shaper\n", - ")\n", - "\n", - "# Step 3: Create an actor and a learner to start the training process. \n", - "max_episode = 100\n", - "scheduler = TwoPhaseLinearParameterScheduler(\n", - " max_episode,\n", - " parameter_names=[\"epsilon\"],\n", - " split_ep=50,\n", - " start_values=0.4,\n", - " mid_values=0.32,\n", - " end_values=.0\n", - ")\n", - "\n", - "actor = SimpleActor(env, agent_manager)\n", - "learner = SimpleLearner(\n", - " agent_manager, actor, scheduler, \n", - " logger=Logger(\"single_host_cim_learner\", format_=LogFormat.simple, auto_timestamp=False)\n", - ")\n", - "\n", - "learner.learn()\n", - "learner.test()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.8.5" - } - }, - "nbformat": 4, - "nbformat_minor": 4 -} +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Quick Start\n", + "\n", + "This notebook demonstrates how to use MARO's reinforcement learning (RL) toolkit to solve the container inventory management ([CIM](https://maro.readthedocs.io/en/latest/scenarios/container_inventory_management.html)) problem. It is formalized as a multi-agent reinforcement learning problem, where each port acts as a decision agent. The agents take actions independently, e.g., loading containers to vessels or discharging containers from vessels. " + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## [State Shaper](https://maro.readthedocs.io/en/latest/key_components/rl_toolkit.html#shapers)\n", + "\n", + "State shaper converts the environment observation to the model input state which includes temporal and spatial information. For this scenario, the model input state includes: \n", + "\n", + "- Temporal information, including the past week's information of ports and vessels, such as shortage on port and remaining space on vessel. \n", + "\n", + "- Spatial information, it including the related downstream port features. " + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "import numpy as np\n", + "from maro.rl import StateShaper\n", + "\n", + "\n", + "PORT_ATTRIBUTES = [\"empty\", \"full\", \"on_shipper\", \"on_consignee\", \"booking\", \"shortage\", \"fulfillment\"]\n", + "VESSEL_ATTRIBUTES = [\"empty\", \"full\", \"remaining_space\"]\n", + "\n", + "\n", + "class CIMStateShaper(StateShaper):\n", + " def __init__(self, *, look_back, max_ports_downstream):\n", + " super().__init__()\n", + " self._look_back = look_back\n", + " self._max_ports_downstream = max_ports_downstream\n", + " self._dim = (look_back + 1) * (max_ports_downstream + 1) * len(PORT_ATTRIBUTES) + len(VESSEL_ATTRIBUTES)\n", + "\n", + " def __call__(self, decision_event, snapshot_list):\n", + " tick, port_idx, vessel_idx = decision_event.tick, decision_event.port_idx, decision_event.vessel_idx\n", + " ticks = [tick - rt for rt in range(self._look_back - 1)]\n", + " future_port_idx_list = snapshot_list[\"vessels\"][tick: vessel_idx: 'future_stop_list'].astype('int')\n", + " port_features = snapshot_list[\"ports\"][ticks: [port_idx] + list(future_port_idx_list): PORT_ATTRIBUTES]\n", + " vessel_features = snapshot_list[\"vessels\"][tick: vessel_idx: VESSEL_ATTRIBUTES]\n", + " state = np.concatenate((port_features, vessel_features))\n", + " return str(port_idx), state\n", + "\n", + " @property\n", + " def dim(self):\n", + " return self._dim\n", + " \n", + "# Create a state shaper\n", + "state_shaper = CIMStateShaper(look_back=7, max_ports_downstream=2)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## [Action Shaper](https://maro.readthedocs.io/en/latest/key_components/rl_toolkit.html#shapers)\n", + "\n", + "Action shaper is used to convert an agent's model output to an environment executable action. For this specific scenario, the output is a discrete index that corresponds to a percentage indicating the fraction of containers to be loaded to or discharged from the arriving vessel." + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "from maro.rl import ActionShaper\n", + "from maro.simulator.scenarios.cim.common import Action\n", + "\n", + "\n", + "class CIMActionShaper(ActionShaper):\n", + " def __init__(self, action_space):\n", + " super().__init__()\n", + " self._action_space = action_space\n", + " self._zero_action_index = action_space.index(0)\n", + "\n", + " def __call__(self, model_action, decision_event, snapshot_list):\n", + " assert 0 <= model_action < len(self._action_space)\n", + " \n", + " scope = decision_event.action_scope\n", + " tick = decision_event.tick\n", + " port_idx = decision_event.port_idx\n", + " vessel_idx = decision_event.vessel_idx\n", + " port_empty = snapshot_list[\"ports\"][tick: port_idx: [\"empty\", \"full\", \"on_shipper\", \"on_consignee\"]][0]\n", + " vessel_remaining_space = snapshot_list[\"vessels\"][tick: vessel_idx: [\"empty\", \"full\", \"remaining_space\"]][2]\n", + " early_discharge = snapshot_list[\"vessels\"][tick:vessel_idx: \"early_discharge\"][0]\n", + " \n", + " if model_action < self._zero_action_index:\n", + " # The number of loaded containers must be less than the vessel's remaining space.\n", + " actual_action = max(round(self._action_space[model_action] * port_empty), -vessel_remaining_space)\n", + " elif model_action > self._zero_action_index:\n", + " # In the case of an early discharge event, we need to subtract the early discharge amount from the expected \n", + " # discharge quote. \n", + " plan_action = self._action_space[model_action] * (scope.discharge + early_discharge) - early_discharge\n", + " actual_action = round(plan_action) if plan_action > 0 else round(self._action_space[model_action] * scope.discharge)\n", + " else:\n", + " actual_action = 0\n", + "\n", + " return Action(vessel_idx, port_idx, actual_action)\n", + " \n", + "# Create an action shaper\n", + "NUM_ACTIONS = 21\n", + "action_shaper = CIMActionShaper(action_space=list(np.linspace(-1.0, 1.0, NUM_ACTIONS)))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## [Experience Shaper](https://maro.readthedocs.io/en/latest/key_components/rl_toolkit.html#shapers)\n", + "\n", + "Experience shaper is used to convert an episode trajectory to trainable experiences for RL agents. For this specific scenario, the reward is a linear combination of fulfillment and shortage in a limited time window." + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [], + "source": [ + "from collections import defaultdict\n", + "\n", + "from maro.rl import ExperienceShaper\n", + "\n", + "\n", + "class TruncatedExperienceShaper(ExperienceShaper):\n", + " def __init__(\n", + " self, *, time_window: int, time_decay_factor: float, fulfillment_factor: float, shortage_factor: float\n", + " ):\n", + " super().__init__(reward_func=None)\n", + " self._time_window = time_window\n", + " self._time_decay_factor = time_decay_factor\n", + " self._fulfillment_factor = fulfillment_factor\n", + " self._shortage_factor = shortage_factor\n", + "\n", + " def __call__(self, trajectory, snapshot_list):\n", + " experiences_by_agent = {}\n", + " for i in range(len(trajectory) - 1):\n", + " transition = trajectory[i]\n", + " agent_id = transition[\"agent_id\"]\n", + " if agent_id not in experiences_by_agent:\n", + " experiences_by_agent[agent_id] = defaultdict(list)\n", + " experiences = experiences_by_agent[agent_id]\n", + " experiences[\"state\"].append(transition[\"state\"])\n", + " experiences[\"action\"].append(transition[\"action\"])\n", + " experiences[\"reward\"].append(self._compute_reward(transition[\"event\"], snapshot_list))\n", + " experiences[\"next_state\"].append(trajectory[i + 1][\"state\"])\n", + "\n", + " return experiences_by_agent\n", + "\n", + " def _compute_reward(self, decision_event, snapshot_list):\n", + " start_tick = decision_event.tick + 1\n", + " end_tick = decision_event.tick + self._time_window\n", + " ticks = list(range(start_tick, end_tick))\n", + "\n", + " # calculate tc reward\n", + " future_fulfillment = snapshot_list[\"ports\"][ticks::\"fulfillment\"]\n", + " future_shortage = snapshot_list[\"ports\"][ticks::\"shortage\"]\n", + " decay_list = [\n", + " self._time_decay_factor ** i for i in range(end_tick - start_tick)\n", + " for _ in range(future_fulfillment.shape[0] // (end_tick - start_tick))\n", + " ]\n", + "\n", + " tot_fulfillment = np.dot(future_fulfillment, decay_list)\n", + " tot_shortage = np.dot(future_shortage, decay_list)\n", + "\n", + " return np.float32(self._fulfillment_factor * tot_fulfillment - self._shortage_factor * tot_shortage)\n", + " \n", + "# Create an experience shaper\n", + "experience_shaper = TruncatedExperienceShaper(time_window=100, fulfillment_factor=1.0, shortage_factor=1.0, time_decay_factor=0.97)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## [Agent](https://maro.readthedocs.io/en/latest/key_components/rl_toolkit.html#agent)\n", + "\n", + "For this scenario, the agent is the abstraction of a port. We choose DQN as our underlying learning algorithm with a TD-error-based sampling mechanism. " + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [], + "source": [ + "from maro.rl import AbsAgent, ColumnBasedStore\n", + "\n", + "\n", + "class CIMAgent(AbsAgent):\n", + " def __init__(self, name, algorithm, experience_pool: ColumnBasedStore, min_experiences_to_train, num_batches, batch_size):\n", + " super().__init__(name, algorithm, experience_pool)\n", + " self._min_experiences_to_train = min_experiences_to_train\n", + " self._num_batches = num_batches\n", + " self._batch_size = batch_size\n", + "\n", + " def train(self):\n", + " if len(self._experience_pool) < self._min_experiences_to_train:\n", + " return\n", + "\n", + " for _ in range(self._num_batches):\n", + " indexes, sample = self._experience_pool.sample_by_key(\"loss\", self._batch_size)\n", + " state = np.asarray(sample[\"state\"])\n", + " action = np.asarray(sample[\"action\"])\n", + " reward = np.asarray(sample[\"reward\"])\n", + " next_state = np.asarray(sample[\"next_state\"])\n", + " loss = self._algorithm.train(state, action, reward, next_state)\n", + " self._experience_pool.update(indexes, {\"loss\": loss})" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## [Agent Manager](https://maro.readthedocs.io/en/latest/key_components/rl_toolkit.html#agent-manager)\n", + "\n", + "The complexities of the environment can be isolated from the learning algorithm by using an AgentManager to manage individual agents. We define a function to create the agents and an agent manager class that implements the ``train`` method where the newly obtained experiences are stored in the agents' experience pools before training, in accordance with the DQN algorithm." + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [], + "source": [ + "import io\n", + "import yaml\n", + "\n", + "import torch.nn as nn\n", + "from torch.nn.functional import smooth_l1_loss\n", + "from torch.optim import RMSprop\n", + "\n", + "from maro.rl import (\n", + " ColumnBasedStore, DQN, DQNConfig, FullyConnectedBlock, LearningModuleManager, LearningModule, OptimizerOptions, SimpleAgentManager\n", + ")\n", + "from maro.utils import set_seeds\n", + "\n", + "\n", + "def create_dqn_agents(agent_id_list):\n", + " set_seeds(64) # for reproducibility\n", + " agent_dict = {}\n", + " for agent_id in agent_id_list:\n", + " q_module = LearningModule(\n", + " \"q_value\",\n", + " [FullyConnectedBlock(\n", + " input_dim=state_shaper.dim,\n", + " hidden_dims=[256, 128, 64],\n", + " output_dim=NUM_ACTIONS,\n", + " activation=nn.LeakyReLU,\n", + " is_head=True,\n", + " batch_norm_enabled=True, \n", + " softmax_enabled=False,\n", + " skip_connection_enabled=False,\n", + " dropout_p=.0)\n", + " ],\n", + " optimizer_options=OptimizerOptions(cls=RMSprop, params={\"lr\": 0.05})\n", + " )\n", + "\n", + " algorithm = DQN(\n", + " model=LearningModuleManager(q_module),\n", + " config=DQNConfig(\n", + " reward_decay=.0, \n", + " target_update_frequency=5, \n", + " tau=0.1, \n", + " is_double=True, \n", + " per_sample_td_error_enabled=True,\n", + " loss_cls=nn.SmoothL1Loss\n", + " )\n", + " )\n", + "\n", + " agent_dict[agent_id] = CIMAgent(\n", + " agent_id, algorithm, ColumnBasedStore(), min_experiences_to_train=1024, num_batches=10, batch_size=128\n", + " )\n", + "\n", + " return agent_dict\n", + "\n", + "\n", + "class DQNAgentManager(SimpleAgentManager):\n", + " def train(self, experiences_by_agent, performance=None):\n", + " self._assert_train_mode()\n", + "\n", + " # store experiences for each agent\n", + " for agent_id, exp in experiences_by_agent.items():\n", + " exp.update({\"loss\": [1e8] * len(list(exp.values())[0])})\n", + " self.agent_dict[agent_id].store_experiences(exp)\n", + "\n", + " for agent in self.agent_dict.values():\n", + " agent.train()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Main Loop with [Actor and Learner](https://maro.readthedocs.io/en/latest/key_components/rl_toolkit.html#learner-and-actor)\n", + "\n", + "This code cell demonstrates the typical workflow of a learning policy's interaction with a MARO environment. \n", + "\n", + "- Initialize an environment with specific scenario and topology parameters. \n", + "\n", + "- Define scenario-specific components, e.g. shapers. \n", + "\n", + "- Create agents and an agent manager. \n", + "\n", + "- Create an actor and a learner to start the training process in which the agent manager interacts with the environment for collecting experiences and updating policies. " + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "08:23:10 | single_host_cim_learner | INFO | ep 0 - performance: {'order_requirements': 2240000, 'container_shortage': 1352136, 'operation_number': 3254760}, exploration_params: {'epsilon': 0.4}\n", + "08:23:15 | single_host_cim_learner | INFO | ep 1 - performance: {'order_requirements': 2240000, 'container_shortage': 1249849, 'operation_number': 3426101}, exploration_params: {'epsilon': 0.39840000000000003}\n", + "08:23:20 | single_host_cim_learner | INFO | ep 2 - performance: {'order_requirements': 2240000, 'container_shortage': 1174857, 'operation_number': 3816050}, exploration_params: {'epsilon': 0.39680000000000004}\n", + "08:23:25 | single_host_cim_learner | INFO | ep 3 - performance: {'order_requirements': 2240000, 'container_shortage': 1168029, 'operation_number': 3783409}, exploration_params: {'epsilon': 0.39520000000000005}\n", + "08:23:30 | single_host_cim_learner | INFO | ep 4 - performance: {'order_requirements': 2240000, 'container_shortage': 1478014, 'operation_number': 3503012}, exploration_params: {'epsilon': 0.39360000000000006}\n", + "08:23:35 | single_host_cim_learner | INFO | ep 5 - performance: {'order_requirements': 2240000, 'container_shortage': 1450497, 'operation_number': 4235336.0}, exploration_params: {'epsilon': 0.39200000000000007}\n", + "08:23:40 | single_host_cim_learner | INFO | ep 6 - performance: {'order_requirements': 2240000, 'container_shortage': 1250538, 'operation_number': 3681417}, exploration_params: {'epsilon': 0.3904000000000001}\n", + "08:23:45 | single_host_cim_learner | INFO | ep 7 - performance: {'order_requirements': 2240000, 'container_shortage': 1923270, 'operation_number': 2793886}, exploration_params: {'epsilon': 0.3888000000000001}\n", + "08:23:50 | single_host_cim_learner | INFO | ep 8 - performance: {'order_requirements': 2240000, 'container_shortage': 1621199, 'operation_number': 2655534}, exploration_params: {'epsilon': 0.3872000000000001}\n", + "08:23:56 | single_host_cim_learner | INFO | ep 9 - performance: {'order_requirements': 2240000, 'container_shortage': 1185373, 'operation_number': 3303864}, exploration_params: {'epsilon': 0.3856000000000001}\n", + "08:24:01 | single_host_cim_learner | INFO | ep 10 - performance: {'order_requirements': 2240000, 'container_shortage': 720509, 'operation_number': 3923254}, exploration_params: {'epsilon': 0.3840000000000001}\n", + "08:24:06 | single_host_cim_learner | INFO | ep 11 - performance: {'order_requirements': 2240000, 'container_shortage': 604046, 'operation_number': 4055816}, exploration_params: {'epsilon': 0.38240000000000013}\n", + "08:24:11 | single_host_cim_learner | INFO | ep 12 - performance: {'order_requirements': 2240000, 'container_shortage': 721395, 'operation_number': 4088165}, exploration_params: {'epsilon': 0.38080000000000014}\n", + "08:24:17 | single_host_cim_learner | INFO | ep 13 - performance: {'order_requirements': 2240000, 'container_shortage': 569133, 'operation_number': 5024948}, exploration_params: {'epsilon': 0.37920000000000015}\n", + "08:24:22 | single_host_cim_learner | INFO | ep 14 - performance: {'order_requirements': 2240000, 'container_shortage': 672845, 'operation_number': 4619933}, exploration_params: {'epsilon': 0.37760000000000016}\n", + "08:24:27 | single_host_cim_learner | INFO | ep 15 - performance: {'order_requirements': 2240000, 'container_shortage': 899736, 'operation_number': 4688569}, exploration_params: {'epsilon': 0.37600000000000017}\n", + "08:24:32 | single_host_cim_learner | INFO | ep 16 - performance: {'order_requirements': 2240000, 'container_shortage': 950907, 'operation_number': 4453843}, exploration_params: {'epsilon': 0.3744000000000002}\n", + "08:24:37 | single_host_cim_learner | INFO | ep 17 - performance: {'order_requirements': 2240000, 'container_shortage': 642622, 'operation_number': 5087980}, exploration_params: {'epsilon': 0.3728000000000002}\n", + "08:24:42 | single_host_cim_learner | INFO | ep 18 - performance: {'order_requirements': 2240000, 'container_shortage': 804926, 'operation_number': 4627842}, exploration_params: {'epsilon': 0.3712000000000002}\n", + "08:24:47 | single_host_cim_learner | INFO | ep 19 - performance: {'order_requirements': 2240000, 'container_shortage': 955385, 'operation_number': 4955868}, exploration_params: {'epsilon': 0.3696000000000002}\n", + "08:24:52 | single_host_cim_learner | INFO | ep 20 - performance: {'order_requirements': 2240000, 'container_shortage': 829930, 'operation_number': 5148070}, exploration_params: {'epsilon': 0.3680000000000002}\n", + "08:24:58 | single_host_cim_learner | INFO | ep 21 - performance: {'order_requirements': 2240000, 'container_shortage': 644572, 'operation_number': 5105741}, exploration_params: {'epsilon': 0.3664000000000002}\n", + "08:25:03 | single_host_cim_learner | INFO | ep 22 - performance: {'order_requirements': 2240000, 'container_shortage': 555530, 'operation_number': 4839572}, exploration_params: {'epsilon': 0.36480000000000024}\n", + "08:25:08 | single_host_cim_learner | INFO | ep 23 - performance: {'order_requirements': 2240000, 'container_shortage': 1031378, 'operation_number': 3728440}, exploration_params: {'epsilon': 0.36320000000000024}\n", + "08:25:13 | single_host_cim_learner | INFO | ep 24 - performance: {'order_requirements': 2240000, 'container_shortage': 723926, 'operation_number': 5235602}, exploration_params: {'epsilon': 0.36160000000000025}\n", + "08:25:19 | single_host_cim_learner | INFO | ep 25 - performance: {'order_requirements': 2240000, 'container_shortage': 676156, 'operation_number': 5142291}, exploration_params: {'epsilon': 0.36000000000000026}\n", + "08:25:24 | single_host_cim_learner | INFO | ep 26 - performance: {'order_requirements': 2240000, 'container_shortage': 842840, 'operation_number': 5028770}, exploration_params: {'epsilon': 0.3584000000000003}\n", + "08:25:29 | single_host_cim_learner | INFO | ep 27 - performance: {'order_requirements': 2240000, 'container_shortage': 865620, 'operation_number': 4766610}, exploration_params: {'epsilon': 0.3568000000000003}\n", + "08:25:34 | single_host_cim_learner | INFO | ep 28 - performance: {'order_requirements': 2240000, 'container_shortage': 794776, 'operation_number': 5052284}, exploration_params: {'epsilon': 0.3552000000000003}\n", + "08:25:39 | single_host_cim_learner | INFO | ep 29 - performance: {'order_requirements': 2240000, 'container_shortage': 699853, 'operation_number': 5152245}, exploration_params: {'epsilon': 0.3536000000000003}\n", + "08:25:45 | single_host_cim_learner | INFO | ep 30 - performance: {'order_requirements': 2240000, 'container_shortage': 616300, 'operation_number': 4678601}, exploration_params: {'epsilon': 0.3520000000000003}\n", + "08:25:50 | single_host_cim_learner | INFO | ep 31 - performance: {'order_requirements': 2240000, 'container_shortage': 728820, 'operation_number': 4974799}, exploration_params: {'epsilon': 0.3504000000000003}\n", + "08:25:55 | single_host_cim_learner | INFO | ep 32 - performance: {'order_requirements': 2240000, 'container_shortage': 727185, 'operation_number': 4755672}, exploration_params: {'epsilon': 0.34880000000000033}\n", + "08:26:00 | single_host_cim_learner | INFO | ep 33 - performance: {'order_requirements': 2240000, 'container_shortage': 727381, 'operation_number': 4873634}, exploration_params: {'epsilon': 0.34720000000000034}\n", + "08:26:06 | single_host_cim_learner | INFO | ep 34 - performance: {'order_requirements': 2240000, 'container_shortage': 679647, 'operation_number': 4624782}, exploration_params: {'epsilon': 0.34560000000000035}\n", + "08:26:11 | single_host_cim_learner | INFO | ep 35 - performance: {'order_requirements': 2240000, 'container_shortage': 625849, 'operation_number': 4947661}, exploration_params: {'epsilon': 0.34400000000000036}\n", + "08:26:16 | single_host_cim_learner | INFO | ep 36 - performance: {'order_requirements': 2240000, 'container_shortage': 706626, 'operation_number': 4546677}, exploration_params: {'epsilon': 0.34240000000000037}\n", + "08:26:21 | single_host_cim_learner | INFO | ep 37 - performance: {'order_requirements': 2240000, 'container_shortage': 560302, 'operation_number': 4578177}, exploration_params: {'epsilon': 0.3408000000000004}\n", + "08:26:26 | single_host_cim_learner | INFO | ep 38 - performance: {'order_requirements': 2240000, 'container_shortage': 774175, 'operation_number': 5005750}, exploration_params: {'epsilon': 0.3392000000000004}\n", + "08:26:31 | single_host_cim_learner | INFO | ep 39 - performance: {'order_requirements': 2240000, 'container_shortage': 584020, 'operation_number': 4787339}, exploration_params: {'epsilon': 0.3376000000000004}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "08:26:37 | single_host_cim_learner | INFO | ep 40 - performance: {'order_requirements': 2240000, 'container_shortage': 576394, 'operation_number': 4720419}, exploration_params: {'epsilon': 0.3360000000000004}\n", + "08:26:42 | single_host_cim_learner | INFO | ep 41 - performance: {'order_requirements': 2240000, 'container_shortage': 702115, 'operation_number': 4928421}, exploration_params: {'epsilon': 0.3344000000000004}\n", + "08:26:47 | single_host_cim_learner | INFO | ep 42 - performance: {'order_requirements': 2240000, 'container_shortage': 822003, 'operation_number': 5127494}, exploration_params: {'epsilon': 0.33280000000000043}\n", + "08:26:52 | single_host_cim_learner | INFO | ep 43 - performance: {'order_requirements': 2240000, 'container_shortage': 676856, 'operation_number': 4361711}, exploration_params: {'epsilon': 0.33120000000000044}\n", + "08:26:58 | single_host_cim_learner | INFO | ep 44 - performance: {'order_requirements': 2240000, 'container_shortage': 669537, 'operation_number': 5291246}, exploration_params: {'epsilon': 0.32960000000000045}\n", + "08:27:03 | single_host_cim_learner | INFO | ep 45 - performance: {'order_requirements': 2240000, 'container_shortage': 569000, 'operation_number': 4652232}, exploration_params: {'epsilon': 0.32800000000000046}\n", + "08:27:08 | single_host_cim_learner | INFO | ep 46 - performance: {'order_requirements': 2240000, 'container_shortage': 604969, 'operation_number': 5123438}, exploration_params: {'epsilon': 0.32640000000000047}\n", + "08:27:14 | single_host_cim_learner | INFO | ep 47 - performance: {'order_requirements': 2240000, 'container_shortage': 557511, 'operation_number': 4832546}, exploration_params: {'epsilon': 0.3248000000000005}\n", + "08:27:19 | single_host_cim_learner | INFO | ep 48 - performance: {'order_requirements': 2240000, 'container_shortage': 661551, 'operation_number': 4916774}, exploration_params: {'epsilon': 0.3232000000000005}\n", + "08:27:24 | single_host_cim_learner | INFO | ep 49 - performance: {'order_requirements': 2240000, 'container_shortage': 678443, 'operation_number': 4797744}, exploration_params: {'epsilon': 0.3216000000000005}\n", + "08:27:29 | single_host_cim_learner | INFO | ep 50 - performance: {'order_requirements': 2240000, 'container_shortage': 647478, 'operation_number': 4983993}, exploration_params: {'epsilon': 0.3200000000000005}\n", + "08:27:35 | single_host_cim_learner | INFO | ep 51 - performance: {'order_requirements': 2240000, 'container_shortage': 436833, 'operation_number': 4411942}, exploration_params: {'epsilon': 0.3134693877551025}\n", + "08:27:40 | single_host_cim_learner | INFO | ep 52 - performance: {'order_requirements': 2240000, 'container_shortage': 626074, 'operation_number': 4618660}, exploration_params: {'epsilon': 0.30693877551020454}\n", + "08:27:45 | single_host_cim_learner | INFO | ep 53 - performance: {'order_requirements': 2240000, 'container_shortage': 708019, 'operation_number': 4538794}, exploration_params: {'epsilon': 0.30040816326530656}\n", + "08:27:50 | single_host_cim_learner | INFO | ep 54 - performance: {'order_requirements': 2240000, 'container_shortage': 704531, 'operation_number': 5035620}, exploration_params: {'epsilon': 0.2938775510204086}\n", + "08:27:56 | single_host_cim_learner | INFO | ep 55 - performance: {'order_requirements': 2240000, 'container_shortage': 675365, 'operation_number': 4744667}, exploration_params: {'epsilon': 0.2873469387755106}\n", + "08:28:01 | single_host_cim_learner | INFO | ep 56 - performance: {'order_requirements': 2240000, 'container_shortage': 536896, 'operation_number': 4664484}, exploration_params: {'epsilon': 0.2808163265306126}\n", + "08:28:06 | single_host_cim_learner | INFO | ep 57 - performance: {'order_requirements': 2240000, 'container_shortage': 517742, 'operation_number': 4515999}, exploration_params: {'epsilon': 0.27428571428571463}\n", + "08:28:11 | single_host_cim_learner | INFO | ep 58 - performance: {'order_requirements': 2240000, 'container_shortage': 495825, 'operation_number': 4592775}, exploration_params: {'epsilon': 0.26775510204081665}\n", + "08:28:17 | single_host_cim_learner | INFO | ep 59 - performance: {'order_requirements': 2240000, 'container_shortage': 485726, 'operation_number': 4586843}, exploration_params: {'epsilon': 0.26122448979591867}\n", + "08:28:22 | single_host_cim_learner | INFO | ep 60 - performance: {'order_requirements': 2240000, 'container_shortage': 350307, 'operation_number': 4856216}, exploration_params: {'epsilon': 0.2546938775510207}\n", + "08:28:27 | single_host_cim_learner | INFO | ep 61 - performance: {'order_requirements': 2240000, 'container_shortage': 440101, 'operation_number': 4511357}, exploration_params: {'epsilon': 0.24816326530612273}\n", + "08:28:33 | single_host_cim_learner | INFO | ep 62 - performance: {'order_requirements': 2240000, 'container_shortage': 366995, 'operation_number': 4489751}, exploration_params: {'epsilon': 0.24163265306122478}\n", + "08:28:38 | single_host_cim_learner | INFO | ep 63 - performance: {'order_requirements': 2240000, 'container_shortage': 407148, 'operation_number': 4229339}, exploration_params: {'epsilon': 0.23510204081632682}\n", + "08:28:43 | single_host_cim_learner | INFO | ep 64 - performance: {'order_requirements': 2240000, 'container_shortage': 490311, 'operation_number': 4231013}, exploration_params: {'epsilon': 0.22857142857142887}\n", + "08:28:49 | single_host_cim_learner | INFO | ep 65 - performance: {'order_requirements': 2240000, 'container_shortage': 495735, 'operation_number': 4084791}, exploration_params: {'epsilon': 0.22204081632653092}\n", + "08:28:54 | single_host_cim_learner | INFO | ep 66 - performance: {'order_requirements': 2240000, 'container_shortage': 531423, 'operation_number': 4125375}, exploration_params: {'epsilon': 0.21551020408163296}\n", + "08:28:59 | single_host_cim_learner | INFO | ep 67 - performance: {'order_requirements': 2240000, 'container_shortage': 530556, 'operation_number': 4015393}, exploration_params: {'epsilon': 0.208979591836735}\n", + "08:29:05 | single_host_cim_learner | INFO | ep 68 - performance: {'order_requirements': 2240000, 'container_shortage': 271291, 'operation_number': 4631212}, exploration_params: {'epsilon': 0.20244897959183705}\n", + "08:29:10 | single_host_cim_learner | INFO | ep 69 - performance: {'order_requirements': 2240000, 'container_shortage': 355445, 'operation_number': 4429670}, exploration_params: {'epsilon': 0.1959183673469391}\n", + "08:29:16 | single_host_cim_learner | INFO | ep 70 - performance: {'order_requirements': 2240000, 'container_shortage': 358595, 'operation_number': 4645877}, exploration_params: {'epsilon': 0.18938775510204114}\n", + "08:29:21 | single_host_cim_learner | INFO | ep 71 - performance: {'order_requirements': 2240000, 'container_shortage': 269327, 'operation_number': 4763005}, exploration_params: {'epsilon': 0.1828571428571432}\n", + "08:29:26 | single_host_cim_learner | INFO | ep 72 - performance: {'order_requirements': 2240000, 'container_shortage': 252523, 'operation_number': 4390522}, exploration_params: {'epsilon': 0.17632653061224524}\n", + "08:29:32 | single_host_cim_learner | INFO | ep 73 - performance: {'order_requirements': 2240000, 'container_shortage': 224338, 'operation_number': 4651297}, exploration_params: {'epsilon': 0.16979591836734728}\n", + "08:29:37 | single_host_cim_learner | INFO | ep 74 - performance: {'order_requirements': 2240000, 'container_shortage': 419598, 'operation_number': 4577744}, exploration_params: {'epsilon': 0.16326530612244933}\n", + "08:29:43 | single_host_cim_learner | INFO | ep 75 - performance: {'order_requirements': 2240000, 'container_shortage': 319832, 'operation_number': 4523035}, exploration_params: {'epsilon': 0.15673469387755137}\n", + "08:29:48 | single_host_cim_learner | INFO | ep 76 - performance: {'order_requirements': 2240000, 'container_shortage': 291227, 'operation_number': 4569509}, exploration_params: {'epsilon': 0.15020408163265342}\n", + "08:29:53 | single_host_cim_learner | INFO | ep 77 - performance: {'order_requirements': 2240000, 'container_shortage': 235975, 'operation_number': 4438580}, exploration_params: {'epsilon': 0.14367346938775546}\n", + "08:29:59 | single_host_cim_learner | INFO | ep 78 - performance: {'order_requirements': 2240000, 'container_shortage': 209720, 'operation_number': 4472226}, exploration_params: {'epsilon': 0.1371428571428575}\n", + "08:30:04 | single_host_cim_learner | INFO | ep 79 - performance: {'order_requirements': 2240000, 'container_shortage': 203721, 'operation_number': 4475482}, exploration_params: {'epsilon': 0.13061224489795956}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "08:30:10 | single_host_cim_learner | INFO | ep 80 - performance: {'order_requirements': 2240000, 'container_shortage': 215189, 'operation_number': 4290320}, exploration_params: {'epsilon': 0.1240816326530616}\n", + "08:30:15 | single_host_cim_learner | INFO | ep 81 - performance: {'order_requirements': 2240000, 'container_shortage': 203791, 'operation_number': 4321789}, exploration_params: {'epsilon': 0.11755102040816365}\n", + "08:30:21 | single_host_cim_learner | INFO | ep 82 - performance: {'order_requirements': 2240000, 'container_shortage': 230472, 'operation_number': 4216193}, exploration_params: {'epsilon': 0.1110204081632657}\n", + "08:30:26 | single_host_cim_learner | INFO | ep 83 - performance: {'order_requirements': 2240000, 'container_shortage': 148354, 'operation_number': 4274694}, exploration_params: {'epsilon': 0.10448979591836774}\n", + "08:30:32 | single_host_cim_learner | INFO | ep 84 - performance: {'order_requirements': 2240000, 'container_shortage': 196658, 'operation_number': 4234519}, exploration_params: {'epsilon': 0.09795918367346979}\n", + "08:30:37 | single_host_cim_learner | INFO | ep 85 - performance: {'order_requirements': 2240000, 'container_shortage': 117587, 'operation_number': 4409388}, exploration_params: {'epsilon': 0.09142857142857183}\n", + "08:30:43 | single_host_cim_learner | INFO | ep 86 - performance: {'order_requirements': 2240000, 'container_shortage': 129900, 'operation_number': 4370451}, exploration_params: {'epsilon': 0.08489795918367388}\n", + "08:30:49 | single_host_cim_learner | INFO | ep 87 - performance: {'order_requirements': 2240000, 'container_shortage': 150391, 'operation_number': 4372565}, exploration_params: {'epsilon': 0.07836734693877592}\n", + "08:30:54 | single_host_cim_learner | INFO | ep 88 - performance: {'order_requirements': 2240000, 'container_shortage': 240991, 'operation_number': 4263189}, exploration_params: {'epsilon': 0.07183673469387797}\n", + "08:31:00 | single_host_cim_learner | INFO | ep 89 - performance: {'order_requirements': 2240000, 'container_shortage': 108313, 'operation_number': 4400742}, exploration_params: {'epsilon': 0.06530612244898001}\n", + "08:31:05 | single_host_cim_learner | INFO | ep 90 - performance: {'order_requirements': 2240000, 'container_shortage': 153466, 'operation_number': 4211634}, exploration_params: {'epsilon': 0.05877551020408205}\n", + "08:31:11 | single_host_cim_learner | INFO | ep 91 - performance: {'order_requirements': 2240000, 'container_shortage': 144307, 'operation_number': 4295363}, exploration_params: {'epsilon': 0.05224489795918409}\n", + "08:31:16 | single_host_cim_learner | INFO | ep 92 - performance: {'order_requirements': 2240000, 'container_shortage': 135912, 'operation_number': 4270395}, exploration_params: {'epsilon': 0.04571428571428613}\n", + "08:31:22 | single_host_cim_learner | INFO | ep 93 - performance: {'order_requirements': 2240000, 'container_shortage': 130515, 'operation_number': 4309926}, exploration_params: {'epsilon': 0.03918367346938817}\n", + "08:31:28 | single_host_cim_learner | INFO | ep 94 - performance: {'order_requirements': 2240000, 'container_shortage': 129339, 'operation_number': 4209056}, exploration_params: {'epsilon': 0.03265306122449021}\n", + "08:31:33 | single_host_cim_learner | INFO | ep 95 - performance: {'order_requirements': 2240000, 'container_shortage': 134269, 'operation_number': 4254431}, exploration_params: {'epsilon': 0.026122448979592247}\n", + "08:31:39 | single_host_cim_learner | INFO | ep 96 - performance: {'order_requirements': 2240000, 'container_shortage': 104881, 'operation_number': 4244267}, exploration_params: {'epsilon': 0.019591836734694286}\n", + "08:31:44 | single_host_cim_learner | INFO | ep 97 - performance: {'order_requirements': 2240000, 'container_shortage': 89076, 'operation_number': 4300194}, exploration_params: {'epsilon': 0.013061224489796327}\n", + "08:31:50 | single_host_cim_learner | INFO | ep 98 - performance: {'order_requirements': 2240000, 'container_shortage': 94324, 'operation_number': 4267081}, exploration_params: {'epsilon': 0.006530612244898367}\n", + "08:31:56 | single_host_cim_learner | INFO | ep 99 - performance: {'order_requirements': 2240000, 'container_shortage': 88931, 'operation_number': 4267048}, exploration_params: {'epsilon': 4.0766001685454967e-16}\n" + ] + } + ], + "source": [ + "from maro.simulator import Env\n", + "from maro.rl import AgentManagerMode, SimpleActor, SimpleLearner, TwoPhaseLinearParameterScheduler\n", + "from maro.utils import LogFormat, Logger\n", + "\n", + "# Step 1: initialize a CIM environment for a toy dataset. \n", + "env = Env(\"cim\", \"toy.4p_ssdd_l0.0\", durations=1120)\n", + "agent_id_list = [str(agent_id) for agent_id in env.agent_idx_list]\n", + "\n", + "# Step 2: create DQN agents and an agent manager to manage them.\n", + "agent_manager = DQNAgentManager(\n", + " name=\"cim_learner\",\n", + " mode=AgentManagerMode.TRAIN_INFERENCE,\n", + " agent_dict=create_dqn_agents(agent_id_list),\n", + " state_shaper=state_shaper,\n", + " action_shaper=action_shaper,\n", + " experience_shaper=experience_shaper\n", + ")\n", + "\n", + "# Step 3: Create an actor and a learner to start the training process. \n", + "max_episode = 100\n", + "scheduler = TwoPhaseLinearParameterScheduler(\n", + " max_episode,\n", + " parameter_names=[\"epsilon\"],\n", + " split_ep=50,\n", + " start_values=0.4,\n", + " mid_values=0.32,\n", + " end_values=.0\n", + ")\n", + "\n", + "actor = SimpleActor(env, agent_manager)\n", + "learner = SimpleLearner(\n", + " agent_manager, actor, scheduler, \n", + " logger=Logger(\"single_host_cim_learner\", format_=LogFormat.simple, auto_timestamp=False)\n", + ")\n", + "\n", + "learner.learn()\n", + "learner.test()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.7.7" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} \ No newline at end of file From b4ef3f2fdfc284b463b51fc0b64d1c2914df23b0 Mon Sep 17 00:00:00 2001 From: ysqyang Date: Thu, 24 Dec 2020 17:21:58 +0800 Subject: [PATCH 299/337] updated cim PO example code according to changes in maro/rl --- examples/cim/dqn/config.yml | 7 -- examples/cim/dqn/dist_actor.py | 2 +- examples/cim/dqn/dist_learner.py | 2 +- examples/cim/dqn/single_process_launcher.py | 42 +--------- .../components/__init__.py | 16 +++- .../components/state_shaper.py | 18 +++-- examples/cim/policy_optimization/config.yml | 39 +++------- .../cim/policy_optimization/dist_actor.py | 14 ++-- .../cim/policy_optimization/dist_learner.py | 17 ++-- .../distributed_config.yml | 6 ++ .../single_process_launcher.py | 78 +++++++++++-------- .../rl_formulation.ipynb | 2 +- 12 files changed, 104 insertions(+), 139 deletions(-) create mode 100644 examples/cim/policy_optimization/distributed_config.yml diff --git a/examples/cim/dqn/config.yml b/examples/cim/dqn/config.yml index 44563602d..671f66ad7 100644 --- a/examples/cim/dqn/config.yml +++ b/examples/cim/dqn/config.yml @@ -19,13 +19,6 @@ main_loop: start_values: 0.4 mid_values: 0.32 end_values: 0.0 - early_stopping: - warmup_ep: 50 - last_k: 10 - perf_threshold: 0.9 # minimum performance (fulfillment ratio) required to trigger early stopping - perf_stability_threshold: 0.1 # stability is measured by the maximum of abs(perf_(i+1) - perf_i) / perf_i - # over the last k episodes (where perf is short for performance). This value must - # be below this threshold to trigger early stopping agents: algorithm: num_actions: 21 diff --git a/examples/cim/dqn/dist_actor.py b/examples/cim/dqn/dist_actor.py index 4376ffae6..095636ca5 100644 --- a/examples/cim/dqn/dist_actor.py +++ b/examples/cim/dqn/dist_actor.py @@ -26,7 +26,7 @@ def launch(config, distributed_config): config["agents"]["algorithm"]["input_dim"] = state_shaper.dim agent_manager = DQNAgentManager( - name="distributed_cim_actor", + name="cim_actor", mode=AgentManagerMode.INFERENCE, agent_dict=create_dqn_agents(agent_id_list, config.agents), state_shaper=state_shaper, diff --git a/examples/cim/dqn/dist_learner.py b/examples/cim/dqn/dist_learner.py index 70082ba26..156de4ef4 100644 --- a/examples/cim/dqn/dist_learner.py +++ b/examples/cim/dqn/dist_learner.py @@ -38,7 +38,7 @@ def launch(config, distributed_config): agent_manager=agent_manager, actor=ActorProxy(proxy_params=proxy_params, experience_collecting_func=concat_experiences_by_agent), scheduler=TwoPhaseLinearParameterScheduler(config.main_loop.max_episode, **config.main_loop.exploration), - logger=Logger("distributed_cim_learner", auto_timestamp=False) + logger=Logger("cim_learner", auto_timestamp=False) ) learner.learn() learner.test() diff --git a/examples/cim/dqn/single_process_launcher.py b/examples/cim/dqn/single_process_launcher.py index 81f0d29fa..0dae3cca8 100644 --- a/examples/cim/dqn/single_process_launcher.py +++ b/examples/cim/dqn/single_process_launcher.py @@ -2,7 +2,6 @@ # Licensed under the MIT license. import os -from statistics import mean import numpy as np @@ -13,38 +12,6 @@ from components import CIMActionShaper, CIMStateShaper, DQNAgentManager, TruncatedExperienceShaper, create_dqn_agents -class EarlyStopping: - """Callable class that checks the performance history to determine early stopping. - - Args: - warmup_ep (int): Episode from which early stopping checking is initiated. - last_k (int): Number of latest performance records to check for early stopping. - perf_threshold (float): The mean of the ``last_k`` performance metric values must be above this value to - trigger early stopping. - perf_stability_threshold (float): The maximum one-step change over the ``last_k`` performance metrics must be - below this value to trigger early stopping. - """ - def __init__(self, warmup_ep: int, last_k: int, perf_threshold: float, perf_stability_threshold: float): - self._warmup_ep = warmup_ep - self._last_k = last_k - self._perf_threshold = perf_threshold - self._perf_stability_threshold = perf_stability_threshold - - def get_metric(record): - return 1 - record["container_shortage"] / record["order_requirements"] - self._metric_func = get_metric - - def __call__(self, perf_history) -> bool: - if len(perf_history) < max(self._last_k, self._warmup_ep): - return False - - metric_series = list(map(self._metric_func, perf_history[-self._last_k:])) - max_delta = max( - abs(metric_series[i] - metric_series[i - 1]) / metric_series[i - 1] for i in range(1, self._last_k) - ) - return mean(metric_series) > self._perf_threshold and max_delta < self._perf_stability_threshold - - def launch(config): config = convert_dottable(config) # Step 1: Initialize a CIM environment for using a toy dataset. @@ -70,16 +37,11 @@ def launch(config): ) # Step 4: Create an actor and a learner to start the training process. - scheduler = TwoPhaseLinearParameterScheduler( - config.main_loop.max_episode, - early_stopping_callback=EarlyStopping(**config.main_loop.early_stopping), - **config.main_loop.exploration - ) - + scheduler = TwoPhaseLinearParameterScheduler(config.main_loop.max_episode, **config.main_loop.exploration) actor = SimpleActor(env, agent_manager) learner = SimpleLearner( agent_manager, actor, scheduler, - logger=Logger("single_host_cim_learner", format_=LogFormat.simple, auto_timestamp=False) + logger=Logger("cim_learner", format_=LogFormat.simple, auto_timestamp=False) ) learner.learn() learner.test() diff --git a/examples/cim/policy_optimization/components/__init__.py b/examples/cim/policy_optimization/components/__init__.py index b14b47650..b096bd2c5 100644 --- a/examples/cim/policy_optimization/components/__init__.py +++ b/examples/cim/policy_optimization/components/__init__.py @@ -1,2 +1,14 @@ -# Copyright (c) Microsoft Corporation. -# Licensed under the MIT license. +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +from .action_shaper import CIMActionShaper +from .agent_manager import POAgentManager, create_po_agents +from .experience_shaper import TruncatedExperienceShaper +from .state_shaper import CIMStateShaper + +__all__ = [ + "CIMActionShaper", + "POAgentManager", "create_po_agents", + "TruncatedExperienceShaper", + "CIMStateShaper" +] diff --git a/examples/cim/policy_optimization/components/state_shaper.py b/examples/cim/policy_optimization/components/state_shaper.py index 58889baca..93b1bf824 100644 --- a/examples/cim/policy_optimization/components/state_shaper.py +++ b/examples/cim/policy_optimization/components/state_shaper.py @@ -5,20 +5,26 @@ from maro.rl import StateShaper +PORT_ATTRIBUTES = ["empty", "full", "on_shipper", "on_consignee", "booking", "shortage", "fulfillment"] +VESSEL_ATTRIBUTES = ["empty", "full", "remaining_space"] + class CIMStateShaper(StateShaper): - def __init__(self, *, look_back, max_ports_downstream, port_attributes, vessel_attributes): + def __init__(self, *, look_back, max_ports_downstream): super().__init__() self._look_back = look_back self._max_ports_downstream = max_ports_downstream - self._port_attributes = port_attributes - self._vessel_attributes = vessel_attributes + self._dim = (look_back + 1) * (max_ports_downstream + 1) * len(PORT_ATTRIBUTES) + len(VESSEL_ATTRIBUTES) def __call__(self, decision_event, snapshot_list): tick, port_idx, vessel_idx = decision_event.tick, decision_event.port_idx, decision_event.vessel_idx - ticks = [tick - rt for rt in range(self._look_back-1)] + ticks = [tick - rt for rt in range(self._look_back - 1)] future_port_idx_list = snapshot_list["vessels"][tick: vessel_idx: 'future_stop_list'].astype('int') - port_features = snapshot_list["ports"][ticks: [port_idx] + list(future_port_idx_list): self._port_attributes] - vessel_features = snapshot_list["vessels"][tick: vessel_idx: self._vessel_attributes] + port_features = snapshot_list["ports"][ticks: [port_idx] + list(future_port_idx_list): PORT_ATTRIBUTES] + vessel_features = snapshot_list["vessels"][tick: vessel_idx: VESSEL_ATTRIBUTES] state = np.concatenate((port_features, vessel_features)) return str(port_idx), state + + @property + def dim(self): + return self._dim diff --git a/examples/cim/policy_optimization/config.yml b/examples/cim/policy_optimization/config.yml index 8635e3be8..e6ff56ef7 100644 --- a/examples/cim/policy_optimization/config.yml +++ b/examples/cim/policy_optimization/config.yml @@ -2,35 +2,23 @@ env: scenario: "cim" topology: "toy.4p_ssdd_l0.0" durations: 1120 + state_shaping: + look_back: 7 + max_ports_downstream: 2 + experience_shaping: + time_window: 100 + fulfillment_factor: 1.0 + shortage_factor: 1.0 + time_decay_factor: 0.97 main_loop: max_episode: 500 early_stopping: warmup_ep: 50 last_k: 10 - perf_threshold: 0.9 # minimum performance (fulfillment ratio) required to trigger early stopping + perf_threshold: 0.95 # minimum performance (fulfillment ratio) required to trigger early stopping perf_stability_threshold: 0.1 # stability is measured by the maximum of abs(perf_(i+1) - perf_i) / perf_i # over the last k episodes (where perf is short for performance). This value must # be below this threshold to trigger early stopping -state_shaping: - look_back: 7 - max_ports_downstream: 2 - port_attributes: - - "empty" - - "full" - - "on_shipper" - - "on_consignee" - - "booking" - - "shortage" - - "fulfillment" - vessel_attributes: - - "empty" - - "full" - - "remaining_space" -experience_shaping: - time_window: 100 - fulfillment_factor: 1.0 - shortage_factor: 1.0 - time_decay_factor: 0.97 agents: seed: 1024 # for reproducibility num_actions: 21 @@ -68,12 +56,3 @@ agents: critic_train_iters: 10 k: 1 lam: 0.0 -distributed: - group_name: "ac_distributed_test" - actor: - peer: {"actor": 1} - learner: - peer: {"actor_worker": 1} - redis: - host_name: "localhost" - port: 6379 diff --git a/examples/cim/policy_optimization/dist_actor.py b/examples/cim/policy_optimization/dist_actor.py index 900b8ebe7..832142dfd 100644 --- a/examples/cim/policy_optimization/dist_actor.py +++ b/examples/cim/policy_optimization/dist_actor.py @@ -9,24 +9,20 @@ from maro.rl import AgentManagerMode, SimpleActor, ActorWorker from maro.utils import convert_dottable -from components.action_shaper import CIMActionShaper -from components.agent_manager import create_po_agents, POAgentManager -from components.config import config, set_input_dim -from components.experience_shaper import TruncatedExperienceShaper -from components.state_shaper import CIMStateShaper +from components import CIMActionShaper, CIMStateShaper, POAgentManager, TruncatedExperienceShaper, create_po_agents def launch(config): - set_input_dim(config) config = convert_dottable(config) env = Env(config.env.scenario, config.env.topology, durations=config.env.durations) agent_id_list = [str(agent_id) for agent_id in env.agent_idx_list] - state_shaper = CIMStateShaper(**config.state_shaping) + state_shaper = CIMStateShaper(**config.env.state_shaping) action_shaper = CIMActionShaper(action_space=list(np.linspace(-1.0, 1.0, config.agents.num_actions))) - experience_shaper = TruncatedExperienceShaper(**config.experience_shaping) + experience_shaper = TruncatedExperienceShaper(**config.env.experience_shaping) + config["agents"]["input_dim"] = state_shaper.dim agent_manager = POAgentManager( - name="cim_remote_actor", + name="cim_actor", mode=AgentManagerMode.INFERENCE, agent_dict=create_po_agents(agent_id_list, config.agents), state_shaper=state_shaper, diff --git a/examples/cim/policy_optimization/dist_learner.py b/examples/cim/policy_optimization/dist_learner.py index c4f955b9b..1dd9ba82c 100644 --- a/examples/cim/policy_optimization/dist_learner.py +++ b/examples/cim/policy_optimization/dist_learner.py @@ -3,21 +3,20 @@ import os -from maro.rl import ActorProxy, AgentManagerMode, SimpleLearner, merge_experiences_with_trajectory_boundaries +from maro.rl import ActorProxy, AgentManagerMode, Scheduler, SimpleLearner, merge_experiences_with_trajectory_boundaries from maro.simulator import Env from maro.utils import Logger, convert_dottable -from components.agent_manager import create_po_agents, POAgentManager -from components.config import set_input_dim +from components import CIMStateShaper, POAgentManager, create_po_agents def launch(config): - set_input_dim(config) config = convert_dottable(config) env = Env(config.env.scenario, config.env.topology, durations=config.env.durations) agent_id_list = [str(agent_id) for agent_id in env.agent_idx_list] + config["agents"]["input_dim"] = CIMStateShaper(**config.env.state_shaping).dim agent_manager = POAgentManager( - name="cim_remote_learner", + name="cim_learner", mode=AgentManagerMode.TRAIN, agent_dict=create_po_agents(agent_id_list, config.agents) ) @@ -31,12 +30,12 @@ def launch(config): learner = SimpleLearner( agent_manager=agent_manager, actor=ActorProxy( - proxy_params=proxy_params, - experience_collecting_func=merge_experiences_with_trajectory_boundaries + proxy_params=proxy_params, experience_collecting_func=merge_experiences_with_trajectory_boundaries ), - logger=Logger("distributed_cim_learner", auto_timestamp=False) + scheduler=Scheduler(config.main_loop.max_episode), + logger=Logger("cim_learner", auto_timestamp=False) ) - learner.learn(max_episode=config.main_loop.max_episode) + learner.learn() learner.test() learner.dump_models(os.path.join(os.getcwd(), "models")) learner.exit() diff --git a/examples/cim/policy_optimization/distributed_config.yml b/examples/cim/policy_optimization/distributed_config.yml new file mode 100644 index 000000000..5b7c18b13 --- /dev/null +++ b/examples/cim/policy_optimization/distributed_config.yml @@ -0,0 +1,6 @@ +redis: + hostname: "localhost" + port: 6379 +group: test_group +num_actors: 1 +num_learners: 1 diff --git a/examples/cim/policy_optimization/single_process_launcher.py b/examples/cim/policy_optimization/single_process_launcher.py index b54827747..32d27e23e 100644 --- a/examples/cim/policy_optimization/single_process_launcher.py +++ b/examples/cim/policy_optimization/single_process_launcher.py @@ -7,21 +7,46 @@ import numpy as np from maro.simulator import Env -from maro.rl import ( - AgentManagerMode, SimpleActor, SimpleEarlyStoppingChecker, SimpleLearner, MaxDeltaEarlyStoppingChecker -) -from maro.utils import Logger, convert_dottable +from maro.rl import AgentManagerMode, Scheduler, SimpleActor, SimpleLearner +from maro.utils import LogFormat, Logger, convert_dottable -from components.action_shaper import CIMActionShaper -from components.agent_manager import create_po_agents, POAgentManager -from components.config import set_input_dim -from components.experience_shaper import TruncatedExperienceShaper -from components.state_shaper import CIMStateShaper +from components import CIMActionShaper, CIMStateShaper, POAgentManager, TruncatedExperienceShaper, create_po_agents + + +class EarlyStopping: + """Callable class that checks the performance history to determine early stopping. + + Args: + warmup_ep (int): Episode from which early stopping checking is initiated. + last_k (int): Number of latest performance records to check for early stopping. + perf_threshold (float): The mean of the ``last_k`` performance metric values must be above this value to + trigger early stopping. + perf_stability_threshold (float): The maximum one-step change over the ``last_k`` performance metrics must be + below this value to trigger early stopping. + """ + def __init__(self, warmup_ep: int, last_k: int, perf_threshold: float, perf_stability_threshold: float): + self._warmup_ep = warmup_ep + self._last_k = last_k + self._perf_threshold = perf_threshold + self._perf_stability_threshold = perf_stability_threshold + + def get_metric(record): + return 1 - record["container_shortage"] / record["order_requirements"] + self._metric_func = get_metric + + def __call__(self, perf_history) -> bool: + if len(perf_history) < max(self._last_k, self._warmup_ep): + return False + + metric_series = list(map(self._metric_func, perf_history[-self._last_k:])) + max_delta = max( + abs(metric_series[i] - metric_series[i - 1]) / metric_series[i - 1] for i in range(1, self._last_k) + ) + return mean(metric_series) > self._perf_threshold and max_delta < self._perf_stability_threshold def launch(config): # First determine the input dimension and add it to the config. - set_input_dim(config) config = convert_dottable(config) # Step 1: initialize a CIM environment for using a toy dataset. @@ -30,11 +55,12 @@ def launch(config): # Step 2: create state, action and experience shapers. We also need to create an explorer here due to the # greedy nature of the DQN algorithm. - state_shaper = CIMStateShaper(**config.state_shaping) + state_shaper = CIMStateShaper(**config.env.state_shaping) action_shaper = CIMActionShaper(action_space=list(np.linspace(-1.0, 1.0, config.agents.num_actions))) - experience_shaper = TruncatedExperienceShaper(**config.experience_shaping) + experience_shaper = TruncatedExperienceShaper(**config.env.experience_shaping) # Step 3: create an agent manager. + config["agents"]["input_dim"] = state_shaper.dim agent_manager = POAgentManager( name="cim_learner", mode=AgentManagerMode.TRAIN_INFERENCE, @@ -45,31 +71,17 @@ def launch(config): ) # Step 4: Create an actor and a learner to start the training process. - perf_checker = SimpleEarlyStoppingChecker( - last_k=config.main_loop.early_stopping.last_k, - threshold=config.main_loop.early_stopping.perf_threshold, - measure_func=lambda vals: mean(vals) - ) - - perf_stability_checker = MaxDeltaEarlyStoppingChecker( - last_k=config.main_loop.early_stopping.last_k, - threshold=config.main_loop.early_stopping.perf_stability_threshold + scheduler = Scheduler( + config.main_loop.max_episode, + early_stopping_callback=EarlyStopping(**config.main_loop.early_stopping) ) - combined_checker = perf_checker & perf_stability_checker - - actor = SimpleActor(env=env, agent_manager=agent_manager) + actor = SimpleActor(env, agent_manager) learner = SimpleLearner( - agent_manager=agent_manager, - actor=actor, - logger=Logger("single_host_cim_learner", auto_timestamp=False) - ) - learner.learn( - max_episode=config.main_loop.max_episode, - early_stopping_checker=combined_checker, - warmup_ep=config.main_loop.early_stopping.warmup_ep, - early_stopping_metric_func=lambda x: 1 - x["container_shortage"] / x["order_requirements"] + agent_manager, actor, scheduler, + logger=Logger("cim_learner", format_=LogFormat.simple, auto_timestamp=False) ) + learner.learn() learner.test() learner.dump_models(os.path.join(os.getcwd(), "models")) diff --git a/notebooks/container_inventory_management/rl_formulation.ipynb b/notebooks/container_inventory_management/rl_formulation.ipynb index 2f1f706e2..195871c52 100644 --- a/notebooks/container_inventory_management/rl_formulation.ipynb +++ b/notebooks/container_inventory_management/rl_formulation.ipynb @@ -505,7 +505,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.7.7" + "version": "3.8.5" } }, "nbformat": 4, From ccf14c08dcbd401eaa4ea9173bfd1a07c7838f87 Mon Sep 17 00:00:00 2001 From: ysqyang Date: Thu, 24 Dec 2020 18:58:32 +0800 Subject: [PATCH 300/337] removed early stopping from CIM dqn example --- examples/cim/dqn/config.yml | 2 +- examples/cim/dqn/single_process_launcher.py | 40 +------------------ maro/rl/scheduling/scheduler.py | 8 ++-- .../scheduling/simple_parameter_scheduler.py | 4 +- 4 files changed, 7 insertions(+), 47 deletions(-) diff --git a/examples/cim/dqn/config.yml b/examples/cim/dqn/config.yml index 95014e839..951c5b575 100644 --- a/examples/cim/dqn/config.yml +++ b/examples/cim/dqn/config.yml @@ -51,4 +51,4 @@ agents: min_experiences_to_train: 1024 num_batches: 10 batch_size: 128 - seed: 1024 # for reproducibility + seed: 32 # for reproducibility diff --git a/examples/cim/dqn/single_process_launcher.py b/examples/cim/dqn/single_process_launcher.py index 3c41d7824..91ed1cbb4 100644 --- a/examples/cim/dqn/single_process_launcher.py +++ b/examples/cim/dqn/single_process_launcher.py @@ -2,11 +2,9 @@ # Licensed under the MIT license. import os -from statistics import mean import numpy as np - from maro.rl import AgentManagerMode, SimpleActor, SimpleLearner, TwoPhaseLinearParameterScheduler from maro.simulator import Env from maro.utils import LogFormat, Logger, convert_dottable @@ -14,38 +12,6 @@ from components import CIMActionShaper, CIMStateShaper, DQNAgentManager, TruncatedExperienceShaper, create_dqn_agents -class EarlyStopping: - """Callable class that checks the performance history to determine early stopping. - - Args: - warmup_ep (int): Episode from which early stopping checking is initiated. - last_k (int): Number of latest performance records to check for early stopping. - perf_threshold (float): The mean of the ``last_k`` performance metric values must be above this value to - trigger early stopping. - perf_stability_threshold (float): The maximum one-step change over the ``last_k`` performance metrics must be - below this value to trigger early stopping. - """ - def __init__(self, warmup_ep: int, last_k: int, perf_threshold: float, perf_stability_threshold: float): - self._warmup_ep = warmup_ep - self._last_k = last_k - self._perf_threshold = perf_threshold - self._perf_stability_threshold = perf_stability_threshold - - def get_metric(record): - return 1 - record["container_shortage"] / record["order_requirements"] - self._metric_func = get_metric - - def __call__(self, perf_history) -> bool: - if len(perf_history) < max(self._last_k, self._warmup_ep): - return False - - metric_series = list(map(self._metric_func, perf_history[-self._last_k:])) - max_delta = max( - abs(metric_series[i] - metric_series[i - 1]) / metric_series[i - 1] for i in range(1, self._last_k) - ) - return mean(metric_series) > self._perf_threshold and max_delta < self._perf_stability_threshold - - def launch(config): config = convert_dottable(config) # Step 1: Initialize a CIM environment for using a toy dataset. @@ -71,11 +37,7 @@ def launch(config): ) # Step 4: Create an actor and a learner to start the training process. - scheduler = TwoPhaseLinearParameterScheduler( - config.main_loop.max_episode, - early_stopping_callback=EarlyStopping(**config.main_loop.early_stopping), - **config.main_loop.exploration - ) + scheduler = TwoPhaseLinearParameterScheduler(config.main_loop.max_episode, **config.main_loop.exploration) actor = SimpleActor(env, agent_manager) learner = SimpleLearner( diff --git a/maro/rl/scheduling/scheduler.py b/maro/rl/scheduling/scheduler.py index 3ae49ad4d..2bd6387b8 100644 --- a/maro/rl/scheduling/scheduler.py +++ b/maro/rl/scheduling/scheduler.py @@ -1,13 +1,12 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT license. -from abc import ABC, abstractmethod from typing import Callable from maro.utils.exception.rl_toolkit_exception import InfiniteTrainingLoopError, InvalidEpisodeError -class Scheduler(ABC): +class Scheduler(object): """Scheduler that generates exploration parameters for each episode. Args: @@ -41,11 +40,10 @@ def __next__(self): if self._early_stopping_callback and self._early_stopping_callback(self._performance_history): raise StopIteration - self._exploration_params = self._get_next_exploration_params() + self._exploration_params = self.get_next_exploration_params() return self._exploration_params - @abstractmethod - def _get_next_exploration_params(self): + def get_next_exploration_params(self): pass @property diff --git a/maro/rl/scheduling/simple_parameter_scheduler.py b/maro/rl/scheduling/simple_parameter_scheduler.py index 7cdfe2570..1ba1f02b7 100644 --- a/maro/rl/scheduling/simple_parameter_scheduler.py +++ b/maro/rl/scheduling/simple_parameter_scheduler.py @@ -46,7 +46,7 @@ def __init__( self._delta = (end_values - self._current_values) / (self._max_ep - 1) - def _get_next_exploration_params(self): + def get_next_exploration_params(self): current_values = self._current_values.copy() self._current_values += self._delta return dict(zip(self._parameter_names, current_values)) @@ -108,7 +108,7 @@ def __init__( self._delta_1 = (mid_values - self._current_values) / split_ep self._delta_2 = (end_values - mid_values) / (max_ep - split_ep - 1) - def _get_next_exploration_params(self): + def get_next_exploration_params(self): current_values = self._current_values.copy() self._current_values += self._delta_1 if self._current_ep < self._split_ep else self._delta_2 return dict(zip(self._parameter_names, current_values)) From a6629147b22d133714977e6ba70c5a410be8c356 Mon Sep 17 00:00:00 2001 From: ysqyang Date: Thu, 24 Dec 2020 23:46:24 +0800 Subject: [PATCH 301/337] combined ac and ppo and simplified example code and config --- examples/cim/dqn/components/config.py | 2 +- .../components/agent_manager.py | 25 ++-- .../policy_optimization/components/config.py | 19 +-- examples/cim/policy_optimization/config.yml | 32 ++--- .../single_process_launcher.py | 2 + maro/rl/__init__.py | 3 - maro/rl/algorithms/ac.py | 57 +++++---- maro/rl/algorithms/ppo.py | 121 ------------------ 8 files changed, 63 insertions(+), 198 deletions(-) delete mode 100644 maro/rl/algorithms/ppo.py diff --git a/examples/cim/dqn/components/config.py b/examples/cim/dqn/components/config.py index c53868b6e..e8712e0cf 100644 --- a/examples/cim/dqn/components/config.py +++ b/examples/cim/dqn/components/config.py @@ -2,7 +2,7 @@ # Licensed under the MIT license. """ -This file is used to load config and convert it into a dotted dictionary. +This file is used to load the configuration and convert it into a dotted dictionary. """ import io diff --git a/examples/cim/policy_optimization/components/agent_manager.py b/examples/cim/policy_optimization/components/agent_manager.py index c7aa40a3e..eab327622 100644 --- a/examples/cim/policy_optimization/components/agent_manager.py +++ b/examples/cim/policy_optimization/components/agent_manager.py @@ -6,29 +6,20 @@ from torch.optim import Adam, RMSprop from maro.rl import ( - AbsAgent, ActorCritic, ActorCriticConfig, FullyConnectedBlock, LearningModuleManager, LearningModule, OptimizerOptions, - PolicyGradient, PolicyGradientConfig, PPO, PPOConfig, SimpleAgentManager + AbsAgent, ActorCritic, ActorCriticConfig, FullyConnectedBlock, LearningModuleManager, LearningModule, + OptimizerOptions, PolicyGradient, PolicyGradientConfig, SimpleAgentManager ) from maro.utils import set_seeds class POAgent(AbsAgent): def train(self, states: np.ndarray, actions: np.ndarray, log_action_prob: np.ndarray, rewards: np.ndarray): - if isinstance(self._algorithm, PPO): - self._algorithm.train(states, actions, log_action_prob, rewards) - else: - self._algorithm.train(states, actions, rewards) + self._algorithm.train(states, actions, log_action_prob, rewards) def create_po_agents(agent_id_list, config): - algorithm_map = { - "actor_critic": (ActorCritic, ActorCriticConfig), - "ppo": (PPO, PPOConfig), - "policy_gradient": (PolicyGradient, PolicyGradientConfig) - } input_dim, num_actions = config.input_dim, config.num_actions set_seeds(config.seed) - algorithm_cls, algorithm_config = algorithm_map[config.algorithm] agent_dict = {} for agent_id in agent_id_list: actor_module = LearningModule( @@ -43,7 +34,7 @@ def create_po_agents(agent_id_list, config): optimizer_options=OptimizerOptions(cls=Adam, params=config.actor_optimizer) ) - if config.algorithm in {"actor_critic", "ppo"}: + if config.type == "actor_critic": critic_module = LearningModule( "critic", [FullyConnectedBlock( @@ -56,12 +47,14 @@ def create_po_agents(agent_id_list, config): optimizer_options=OptimizerOptions(cls=RMSprop, params=config.critic_optimizer) ) - algorithm = algorithm_cls( + hyper_params = config.actor_critic_hyper_parameters + hyper_params.update({"reward_decay": config.reward_decay}) + algorithm = ActorCritic( LearningModuleManager(actor_module, critic_module), - algorithm_config(critic_loss_func=nn.functional.smooth_l1_loss, **config[config.algorithm]) + ActorCriticConfig(critic_loss_func=nn.functional.smooth_l1_loss, **hyper_params) ) else: - algorithm = algorithm_cls(LearningModuleManager(actor_module), algorithm_config(**config[config.algorithm])) + algorithm = PolicyGradient(LearningModuleManager(actor_module), PolicyGradientConfig(config.reward_decay)) agent_dict[agent_id] = POAgent(name=agent_id, algorithm=algorithm) diff --git a/examples/cim/policy_optimization/components/config.py b/examples/cim/policy_optimization/components/config.py index d2f329c37..ee93a8814 100644 --- a/examples/cim/policy_optimization/components/config.py +++ b/examples/cim/policy_optimization/components/config.py @@ -2,7 +2,7 @@ # Licensed under the MIT license. """ -This file is used to load config and convert it into a dotted dictionary. +This file is used to load the configuration and convert it into a dotted dictionary. """ import io @@ -10,19 +10,10 @@ import yaml -def set_input_dim(config): - # obtain model input dimension from state shaping configurations - look_back = config["state_shaping"]["look_back"] - max_ports_downstream = config["state_shaping"]["max_ports_downstream"] - num_port_attributes = len(config["state_shaping"]["port_attributes"]) - num_vessel_attributes = len(config["state_shaping"]["vessel_attributes"]) - - input_dim = (look_back + 1) * (max_ports_downstream + 1) * num_port_attributes + num_vessel_attributes - config["agents"]["input_dim"] = input_dim - - return config - - CONFIG_PATH = os.path.join(os.path.split(os.path.realpath(__file__))[0], "../config.yml") with io.open(CONFIG_PATH, "r") as in_file: config = yaml.safe_load(in_file) + +DISTRIBUTED_CONFIG_PATH = os.path.join(os.path.split(os.path.realpath(__file__))[0], "../distributed_config.yml") +with io.open(DISTRIBUTED_CONFIG_PATH, "r") as in_file: + distributed_config = yaml.safe_load(in_file) diff --git a/examples/cim/policy_optimization/config.yml b/examples/cim/policy_optimization/config.yml index e6ff56ef7..ec4000239 100644 --- a/examples/cim/policy_optimization/config.yml +++ b/examples/cim/policy_optimization/config.yml @@ -13,16 +13,16 @@ env: main_loop: max_episode: 500 early_stopping: - warmup_ep: 50 - last_k: 10 + warmup_ep: 20 + last_k: 5 perf_threshold: 0.95 # minimum performance (fulfillment ratio) required to trigger early stopping perf_stability_threshold: 0.1 # stability is measured by the maximum of abs(perf_(i+1) - perf_i) / perf_i - # over the last k episodes (where perf is short for performance). This value must - # be below this threshold to trigger early stopping + # over the last k episodes (where perf is short for performance). This value must + # be below this threshold to trigger early stopping agents: seed: 1024 # for reproducibility + type: "actor_critic" # "actor_critic" or "policy_gradient" num_actions: 21 - algorithm: "actor_critic" # "actor_critic", "policy_gradient" or "ppo" actor_model: hidden_dims: - 256 @@ -30,6 +30,8 @@ agents: - 64 softmax_enabled: true batch_norm_enabled: false + actor_optimizer: + lr: 0.001 critic_model: hidden_dims: - 256 @@ -37,22 +39,12 @@ agents: - 64 softmax_enabled: false batch_norm_enabled: true - actor_optimizer: - lr: 0.001 critic_optimizer: lr: 0.001 - actor_critic: - reward_decay: .0 - actor_train_iters: 1 - critic_train_iters: 10 - k: 1 - lam: 0.0 - policy_gradient: - reward_decay: .0 - ppo: - reward_decay: .0 - clip_ratio: 0.8 - actor_train_iters: 1 - critic_train_iters: 10 + reward_decay: .0 + actor_critic_hyper_parameters: + train_iters: 10 + actor_loss_coefficient: 0.1 k: 1 lam: 0.0 + # clip_ratio: 0.8 diff --git a/examples/cim/policy_optimization/single_process_launcher.py b/examples/cim/policy_optimization/single_process_launcher.py index e21e6097d..74c72e2f3 100644 --- a/examples/cim/policy_optimization/single_process_launcher.py +++ b/examples/cim/policy_optimization/single_process_launcher.py @@ -29,6 +29,7 @@ def __init__(self, warmup_ep: int, last_k: int, perf_threshold: float, perf_stab self._last_k = last_k self._perf_threshold = perf_threshold self._perf_stability_threshold = perf_stability_threshold + print(f"perf threshold: {self._perf_threshold}, perf stability threshold: {self._perf_stability_threshold}") def get_metric(record): return 1 - record["container_shortage"] / record["order_requirements"] @@ -42,6 +43,7 @@ def __call__(self, perf_history) -> bool: max_delta = max( abs(metric_series[i] - metric_series[i - 1]) / metric_series[i - 1] for i in range(1, self._last_k) ) + print(f"mean_metric: {mean(metric_series)}, max_delta: {max_delta}") return mean(metric_series) > self._perf_threshold and max_delta < self._perf_stability_threshold diff --git a/maro/rl/__init__.py b/maro/rl/__init__.py index a6bfce048..4597fb161 100644 --- a/maro/rl/__init__.py +++ b/maro/rl/__init__.py @@ -10,7 +10,6 @@ from maro.rl.algorithms.ac import ActorCritic, ActorCriticConfig from maro.rl.algorithms.dqn import DQN, DQNConfig, DuelingDQNTask from maro.rl.algorithms.pg import PolicyGradient, PolicyGradientConfig -from maro.rl.algorithms.ppo import PPO, PPOConfig from maro.rl.algorithms.utils import ActionWithLogProbability, expand_dim, preprocess, to_device, validate_task_names from maro.rl.dist_topologies.experience_collection import ( concat_experiences_by_agent, merge_experiences_with_trajectory_boundaries @@ -65,8 +64,6 @@ 'OverwriteType', 'PolicyGradient', 'PolicyGradientConfig', - 'PPO', - 'PPOConfig', 'Scheduler', 'SimpleActor', 'SimpleAgentManager', diff --git a/maro/rl/algorithms/ac.py b/maro/rl/algorithms/ac.py index 7da0255b4..f4c9fbf65 100644 --- a/maro/rl/algorithms/ac.py +++ b/maro/rl/algorithms/ac.py @@ -25,32 +25,37 @@ class ActorCriticConfig: Args: reward_decay (float): Reward decay as defined in standard RL terminology. critic_loss_func (Callable): Loss function for the critic model. - actor_train_iters (int): Number of gradient descent steps for the policy model per call to ``train``. - critic_train_iters (int): Number of gradient descent steps for the value model per call to ``train``. + train_iters (int): Number of gradient descent steps per call to ``train``. + actor_loss_coefficient (float): The coefficient for actor loss in the total loss function, e.g., + loss = critic_loss + ``actor_loss_coefficient`` * actor_loss. Defaults to 1.0. k (int): Number of time steps used in computing returns or return estimates. Defaults to -1, in which case rewards are accumulated until the end of the trajectory. lam (float): Lambda coefficient used in computing lambda returns. Defaults to 1.0, in which case the usual k-step return is computed. + clip_ratio (float): Clip ratio in the PPO algorithm (https://arxiv.org/pdf/1707.06347.pdf). Defaults to None, + in which case the actor loss is calculated using the usual policy gradient theorem. """ __slots__ = [ - "reward_decay", "critic_loss_func", "actor_train_iters", "critic_train_iters", "k", "lam" + "reward_decay", "critic_loss_func", "train_iters", "actor_loss_coefficient", "k", "lam", "clip_ratio" ] def __init__( self, reward_decay: float, critic_loss_func: Callable, - actor_train_iters: int, - critic_train_iters: int, + train_iters: int, + actor_loss_coefficient: float = 1.0, k: int = -1, - lam: float = 1.0 + lam: float = 1.0, + clip_ratio: float = None ): self.reward_decay = reward_decay self.critic_loss_func = critic_loss_func - self.actor_train_iters = actor_train_iters - self.critic_train_iters = critic_train_iters + self.train_iters = train_iters + self.actor_loss_coefficient = actor_loss_coefficient self.k = k self.lam = lam + self.clip_ratio = clip_ratio class ActorCritic(AbsAlgorithm): @@ -91,21 +96,27 @@ def _get_values_and_bootstrapped_returns(self, state_sequence, reward_sequence): return state_values, return_est @preprocess - def train(self, states: np.ndarray, actions: np.ndarray, rewards: np.ndarray): + def train( + self, states: np.ndarray, actions: np.ndarray, log_action_prob: np.ndarray, rewards: np.ndarray + ): state_values, return_est = self._get_values_and_bootstrapped_returns(states, rewards) advantages = return_est - state_values - if self._model.shared_module and self._model.shared_module.is_trainable: - pass + for _ in range(self._config.train_iters): + critic_loss = self._config.critic_loss_func( + self._model(states, task_name="critic").squeeze(), return_est + ) + action_prob = self._model(states, task_name="actor").gather(1, actions.unsqueeze(1)).squeeze() # (N,) + log_action_prob_new = torch.log(action_prob) + actor_loss = self._actor_loss(log_action_prob_new, log_action_prob, advantages) + loss = critic_loss + self._config.actor_loss_coefficient * actor_loss + self._model.learn(loss) + + def _actor_loss(self, log_action_prob_new, log_action_prob_old, advantages): + if self._config.clip_ratio is not None: + ratio = torch.exp(log_action_prob_new - log_action_prob_old) + clip_ratio = torch.clamp(ratio, 1 - self._config.clip_ratio, 1 + self._config.clip_ratio) + actor_loss = -(torch.min(ratio * advantages, clip_ratio * advantages)).mean() else: - # policy model training - for _ in range(self._config.actor_train_iters): - action_prob = self._model(states, task_name="actor").gather(1, actions.unsqueeze(1)).squeeze() # (N,) - actor_loss = -(torch.log(action_prob) * advantages).mean() - self._model.learn(actor_loss) - - # value model training - for _ in range(self._config.critic_train_iters): - critic_loss = self._config.critic_loss_func( - self._model(states, task_name="critic").squeeze(), return_est - ) - self._model.learn(critic_loss) + actor_loss = -(log_action_prob_new * advantages).mean() + + return actor_loss diff --git a/maro/rl/algorithms/ppo.py b/maro/rl/algorithms/ppo.py deleted file mode 100644 index d0eb6d75b..000000000 --- a/maro/rl/algorithms/ppo.py +++ /dev/null @@ -1,121 +0,0 @@ -# Copyright (c) Microsoft Corporation. -# Licensed under the MIT license. - -from enum import Enum -from typing import Callable - -import numpy as np -import torch - -from maro.rl.algorithms.abs_algorithm import AbsAlgorithm -from maro.rl.models.learning_model import LearningModuleManager -from maro.rl.utils.trajectory_utils import get_lambda_returns - -from .utils import ActionWithLogProbability, expand_dim, preprocess, to_device, validate_task_names - - -class PPOTask(Enum): - ACTOR = "actor" - CRITIC = "critic" - - -class PPOConfig: - """Configuration for the Proximal Policy Optimization (PPO) algorithm. - - Args: - reward_decay (float): Reward decay as defined in standard RL terminology. - critic_loss_func (Callable): Critic loss function. - clip_ratio (float): Clip ratio as defined in PPO's objective function. - actor_train_iters (int): Number of gradient descent steps for the policy model per call to ``train``. - critic_train_iters (int): Number of gradient descent steps for the value model per call to ``train``. - k (int): Number of time steps used in computing returns or return estimates. Defaults to -1, in which case - rewards are accumulated until the end of the trajectory. - lam (float): Lambda coefficient used in computing lambda returns. Defaults to 1.0, in which case the usual - k-step return is computed. - """ - __slots__ = [ - "reward_decay", "critic_loss_func", "clip_ratio", "actor_train_iters", "critic_train_iters", "k", "lam" - ] - - def __init__( - self, - reward_decay: float, - critic_loss_func: Callable, - clip_ratio: float, - actor_train_iters: int, - critic_train_iters: int, - k: int = -1, - lam: float = 1.0 - ): - self.reward_decay = reward_decay - self.critic_loss_func = critic_loss_func - self.clip_ratio = clip_ratio - self.actor_train_iters = actor_train_iters - self.critic_train_iters = critic_train_iters - self.k = k - self.lam = lam - - -class PPO(AbsAlgorithm): - """Proximal Policy Optimization (PPO) algorithm. - - See https://arxiv.org/pdf/1707.06347.pdf for details. - - Args: - model (LearningModuleManager): Multi-task model that computes action distributions and state values. - It may or may not have a shared bottom stack. - config: Configuration for the PPO algorithm. - """ - @validate_task_names(PPOTask) - @to_device - def __init__(self, model: LearningModuleManager, config: PPOConfig): - super().__init__(model, config) - self._device = torch.device("cuda" if torch.cuda.is_available() else "cpu") - self._model.to(self._device) - - @expand_dim - def choose_action(self, state: np.ndarray): - """Use the actor (policy) model to generate a stochastic action. - - Args: - state: Input to the actor model. - - Returns: - A ActionWithLogProbability namedtuple instance containing the action index and the corresponding - log probability. - """ - action_distribution = self._model(state, task_name="actor", is_training=False).squeeze().numpy() - action = np.random.choice(len(action_distribution), p=action_distribution) - return ActionWithLogProbability(action=action, log_probability=np.log(action_distribution[action])) - - def _get_values_and_bootstrapped_returns(self, states: torch.tensor, rewards: np.ndarray): - state_values = self._model(states, task_name="critic").detach().squeeze() - return_est = get_lambda_returns( - rewards, state_values, self._config.reward_decay, self._config.lam, k=self._config.k - ) - return state_values, return_est - - @preprocess - def train( - self, states: np.ndarray, actions: np.ndarray, log_action_prob_old: np.ndarray, rewards: np.ndarray - ): - state_values, return_est = self._get_values_and_bootstrapped_returns(states, rewards) - advantages = return_est - state_values - - if self._model.shared_module and self._model.shared_module.is_trainable: - pass - else: - # policy model training (with the value model fixed) - for _ in range(self._config.actor_train_iters): - action_prob = self._model(states, task_name="actor").gather(1, actions.unsqueeze(1)).squeeze() # (N, 1) - ratio = torch.exp(torch.log(action_prob) - log_action_prob_old) - clipped_ratio = torch.clamp(ratio, 1 - self._config.clip_ratio, 1 + self._config.clip_ratio) - actor_loss = -(torch.min(ratio * advantages, clipped_ratio * advantages)).mean() - self._model.learn(actor_loss) - - # value model training - for _ in range(self._config.critic_train_iters): - critic_loss = self._config.critic_loss_func( - self._model(states, task_name="critic").squeeze(), return_est - ) - self._model.learn(critic_loss) From 382108e0dd1cbf809bf9de1541bedf4bb94a078c Mon Sep 17 00:00:00 2001 From: ysqyang Date: Fri, 25 Dec 2020 00:09:05 +0800 Subject: [PATCH 302/337] removed early stopping from cim example config --- examples/cim/dqn/config.yml | 7 ------- 1 file changed, 7 deletions(-) diff --git a/examples/cim/dqn/config.yml b/examples/cim/dqn/config.yml index 951c5b575..3e3db11f2 100644 --- a/examples/cim/dqn/config.yml +++ b/examples/cim/dqn/config.yml @@ -19,13 +19,6 @@ main_loop: start_values: 0.4 mid_values: 0.32 end_values: 0.0 - early_stopping: - warmup_ep: 50 - last_k: 10 - perf_threshold: 0.9 # minimum performance (fulfillment ratio) required to trigger early stopping - perf_stability_threshold: 0.1 # stability is measured by the maximum of abs(perf_(i+1) - perf_i) / perf_i - # over the last k eisodes (where perf is short for performance). This value must - # be below this threshold to trigger early stopping agents: algorithm: num_actions: 21 From 407fb9acb67c42c42d5e38a0b36523f31c0052dc Mon Sep 17 00:00:00 2001 From: ysqyang Date: Fri, 25 Dec 2020 12:51:23 +0800 Subject: [PATCH 303/337] moved decorator logic inside algorithms --- maro/rl/__init__.py | 8 +--- maro/rl/algorithms/abs_algorithm.py | 11 ++++- maro/rl/algorithms/dqn.py | 44 ++++++++++-------- maro/rl/algorithms/utils.py | 69 ----------------------------- 4 files changed, 37 insertions(+), 95 deletions(-) delete mode 100644 maro/rl/algorithms/utils.py diff --git a/maro/rl/__init__.py b/maro/rl/__init__.py index dd4c798e8..badba1f2d 100644 --- a/maro/rl/__init__.py +++ b/maro/rl/__init__.py @@ -7,8 +7,7 @@ from maro.rl.agent.abs_agent_manager import AbsAgentManager, AgentManagerMode from maro.rl.agent.simple_agent_manager import SimpleAgentManager from maro.rl.algorithms.abs_algorithm import AbsAlgorithm -from maro.rl.algorithms.dqn import DQN, DQNConfig, DuelingDQNTask -from maro.rl.algorithms.utils import preprocess, to_device, validate_task_names +from maro.rl.algorithms.dqn import DQN, DQNConfig from maro.rl.dist_topologies.experience_collection import ( concat_experiences_by_agent, merge_experiences_with_trajectory_boundaries ) @@ -47,7 +46,6 @@ 'ColumnBasedStore', 'DQN', 'DQNConfig', - 'DuelingDQNTask', 'EpsilonGreedyExplorer', 'ExperienceShaper', 'FullyConnectedBlock', @@ -64,9 +62,5 @@ 'StateShaper', 'TwoPhaseLinearParameterScheduler', 'concat_experiences_by_agent', - 'merge_experiences_with_trajectory_boundaries', - 'preprocess', - 'to_device', - 'validate_task_names', 'merge_experiences_with_trajectory_boundaries' ] diff --git a/maro/rl/algorithms/abs_algorithm.py b/maro/rl/algorithms/abs_algorithm.py index 681989741..3c5fc39d9 100644 --- a/maro/rl/algorithms/abs_algorithm.py +++ b/maro/rl/algorithms/abs_algorithm.py @@ -3,7 +3,10 @@ from abc import ABC, abstractmethod +import torch + from maro.rl.models.learning_model import LearningModuleManager +from maro.utils.exception.rl_toolkit_exception import UnrecognizedTask class AbsAlgorithm(ABC): @@ -18,7 +21,8 @@ class AbsAlgorithm(ABC): config: Settings for the algorithm. """ def __init__(self, model: LearningModuleManager, config): - self._model = model + self._device = torch.device("cuda" if torch.cuda.is_available() else "cpu") + self._model = model.to(self._device) self._config = config @property @@ -49,3 +53,8 @@ def train(self, *args, **kwargs): def set_exploration_params(self, **params): pass + + @staticmethod + def validate_task_names(model_task_names, expected_task_names): + if len(model_task_names) > 1 and set(model_task_names) != set(expected_task_names): + raise UnrecognizedTask(f"Expected task names {expected_task_names}, got {model_task_names}") diff --git a/maro/rl/algorithms/dqn.py b/maro/rl/algorithms/dqn.py index f3a0bb7ee..52b34b91b 100644 --- a/maro/rl/algorithms/dqn.py +++ b/maro/rl/algorithms/dqn.py @@ -1,20 +1,14 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT license. -from enum import Enum +from typing import Union import numpy as np +import torch from maro.rl.algorithms.abs_algorithm import AbsAlgorithm from maro.rl.models.learning_model import LearningModuleManager -from .utils import expand_dim, preprocess, to_device, validate_task_names - - -class DuelingDQNTask(Enum): - STATE_VALUE = "state_value" - ADVANTAGE = "advantage" - class DQNConfig: """Configuration for the DQN algorithm. @@ -68,23 +62,34 @@ class DQN(AbsAlgorithm): model (LearningModuleManager): Q-value model. config: Configuration for DQN algorithm. """ - @to_device - @validate_task_names(DuelingDQNTask) def __init__(self, model: LearningModuleManager, config: DQNConfig): + self.validate_task_names(model.task_names, {"state_value", "advantage"}) super().__init__(model, config) if isinstance(self._model.output_dim, int): self._num_actions = self._model.output_dim else: - self._num_actions = self._model.output_dim[DuelingDQNTask.ADVANTAGE.value] + self._num_actions = self._model.output_dim["advantage"] self._training_counter = 0 self._target_model = model.copy() if model.is_trainable else None - @expand_dim - def choose_action(self, state: np.ndarray): - if np.random.random() < self._config.epsilon: - return np.random.choice(self._num_actions) - else: - return self._get_q_values(self._model, state, is_training=False).argmax(dim=1).data + def choose_action(self, state: np.ndarray) -> Union[int, np.ndarray]: + state = torch.from_numpy(state).to(self._device) + is_single = len(state.shape) == 1 + if is_single: + state = state.unsqueeze(dim=0) + + greedy_action = self._get_q_values(self._model, state, is_training=False).argmax(dim=1).data + # No exploration + if self._config.epsilon == .0: + return greedy_action.item() if is_single else greedy_action.numpy() + + if is_single: + return greedy_action if np.random.random() > self._config.epsilon else np.random.choice(self._num_actions) + + return np.array([ + act if np.random.random() > self._config.epsilon else np.random.choice(self._num_actions) + for act in greedy_action + ]) def _get_q_values(self, model, states, is_training: bool = True): if self._config.advantage_mode is not None: @@ -115,8 +120,11 @@ def _compute_td_errors(self, states, actions, rewards, next_states): target_q_values = (rewards + self._config.reward_decay * next_q_values).detach() # (N,) return self._config.loss_func(current_q_values, target_q_values) - @preprocess def train(self, states: np.ndarray, actions: np.ndarray, rewards: np.ndarray, next_states: np.ndarray): + states = torch.from_numpy(states).to(self._device) + actions = torch.from_numpy(actions).to(self._device) + rewards = torch.from_numpy(rewards).to(self._device) + next_states = torch.from_numpy(next_states).to(self._device) loss = self._compute_td_errors(states, actions, rewards, next_states) self._model.learn(loss.mean() if self._config.per_sample_td_error_enabled else loss) self._training_counter += 1 diff --git a/maro/rl/algorithms/utils.py b/maro/rl/algorithms/utils.py deleted file mode 100644 index 7a2465298..000000000 --- a/maro/rl/algorithms/utils.py +++ /dev/null @@ -1,69 +0,0 @@ -# Copyright (c) Microsoft Corporation. -# Licensed under the MIT license. - -from enum import Enum -from functools import wraps -from os import environ - -import numpy as np -import torch - -from maro.utils.exception.rl_toolkit_exception import UnrecognizedTask - -device = environ.get("DEVICE", torch.device("cuda" if torch.cuda.is_available() else "cpu")) - - -def validate_task_names(task_enum: Enum): - def decorator(init_func): - @wraps(init_func) - def wrapper(self, model, config): - recognized_task_names = set(member.value for member in task_enum) - model_task_names = set(model.task_names) - if len(model_task_names) > 1 and model_task_names != recognized_task_names: - raise UnrecognizedTask(f"Expected task names {recognized_task_names}, got {model_task_names}") - - init_func(self, model, config) - - return wrapper - - return decorator - - -def to_device(init_func): - @wraps(init_func) - def wrapper(self, model, config): - init_func(self, model.to(device), config) - - return wrapper - - -def preprocess(func): - @wraps(func) - def wrapper(*args, **kwargs): - converted_args = [ - torch.from_numpy(arg).to(device) if isinstance(arg, np.ndarray) else arg for arg in args - ] - converted_kwargs = { - kw: torch.from_numpy(arg).to(device) if isinstance(arg, np.ndarray) else arg - for kw, arg in kwargs.items() - } - return func(*converted_args, **converted_kwargs) - - return wrapper - - -def expand_dim(func): - @wraps(func) - def wrapper(self, state, **kwargs): - if isinstance(state, np.ndarray): - state = torch.from_numpy(state).to(device) - is_single = len(state.shape) == 1 - if is_single: - state = state.unsqueeze(dim=0) - result = func(self, state, **kwargs) - if isinstance(result, torch.Tensor): - return result.item() if is_single else result.numpy() - else: - return result - - return wrapper From 61e869ed46404e588eebc270bd01bfd401b84ec0 Mon Sep 17 00:00:00 2001 From: ysqyang Date: Fri, 25 Dec 2020 12:56:23 +0800 Subject: [PATCH 304/337] renamed early_stopping_callback to early_stopping_checker --- maro/rl/scheduling/scheduler.py | 10 +++++----- maro/rl/scheduling/simple_parameter_scheduler.py | 12 ++++++------ 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/maro/rl/scheduling/scheduler.py b/maro/rl/scheduling/scheduler.py index 2bd6387b8..aa5ccf1bd 100644 --- a/maro/rl/scheduling/scheduler.py +++ b/maro/rl/scheduling/scheduler.py @@ -12,20 +12,20 @@ class Scheduler(object): Args: max_ep (int): Maximum number of episodes to be run. If -1, an early stopping callback is expected to prevent the training loop from running forever. - early_stopping_callback (Callable): Function that returns a boolean indicating whether early stopping should + early_stopping_checker (Callable): Function that returns a boolean indicating whether early stopping should be triggered. Defaults to None, in which case no early stopping check will be performed. """ - def __init__(self, max_ep: int, early_stopping_callback: Callable = None): + def __init__(self, max_ep: int, early_stopping_checker: Callable = None): if max_ep < -1: raise InvalidEpisodeError("max_episode can only be a non-negative integer or -1.") - if max_ep == -1 and early_stopping_callback is None: + if max_ep == -1 and early_stopping_checker is None: raise InfiniteTrainingLoopError( "A positive max_ep or an early stopping checker must be provided to prevent the training loop from " "running forever." ) self._max_ep = max_ep - self._early_stopping_callback = early_stopping_callback + self._early_stopping_checker = early_stopping_checker self._current_ep = -1 self._performance_history = [] self._exploration_params = None @@ -37,7 +37,7 @@ def __next__(self): self._current_ep += 1 if self._current_ep == self._max_ep: raise StopIteration - if self._early_stopping_callback and self._early_stopping_callback(self._performance_history): + if self._early_stopping_checker and self._early_stopping_checker(self._performance_history): raise StopIteration self._exploration_params = self.get_next_exploration_params() diff --git a/maro/rl/scheduling/simple_parameter_scheduler.py b/maro/rl/scheduling/simple_parameter_scheduler.py index 1ba1f02b7..cb280f515 100644 --- a/maro/rl/scheduling/simple_parameter_scheduler.py +++ b/maro/rl/scheduling/simple_parameter_scheduler.py @@ -13,7 +13,7 @@ class LinearParameterScheduler(Scheduler): Args: max_ep (int): Maximum number of episodes to run. - early_stopping_callback (Callable): Function that returns a boolean indicating whether early stopping should + early_stopping_checker (Callable): Function that returns a boolean indicating whether early stopping should be triggered. Defaults to None, in which case no early stopping check will be performed. parameter_names ([str]): List of exploration parameter names. start_values (Union[float, list, tuple, np.ndarray]): Exploration parameter values for the first episode. @@ -24,13 +24,13 @@ class LinearParameterScheduler(Scheduler): def __init__( self, max_ep: int, - early_stopping_callback: Callable = None, + early_stopping_checker: Callable = None, *, parameter_names: [str], start_values: Union[float, list, tuple, np.ndarray], end_values: Union[float, list, tuple, np.ndarray] ): - super().__init__(max_ep, early_stopping_callback=early_stopping_callback) + super().__init__(max_ep, early_stopping_checker=early_stopping_checker) self._parameter_names = parameter_names if isinstance(start_values, float): self._current_values = start_values * np.ones(len(self._parameter_names)) @@ -57,7 +57,7 @@ class TwoPhaseLinearParameterScheduler(Scheduler): Args: max_ep (int): Maximum number of episodes to run. - early_stopping_callback (Callable): Function that returns a boolean indicating whether early stopping should + early_stopping_checker (Callable): Function that returns a boolean indicating whether early stopping should be triggered. Defaults to None, in which case no early stopping check will be performed. parameter_names ([str]): List of exploration parameter names. split_ep (float): The episode where the switch from the first linear schedule to the second occurs. @@ -75,7 +75,7 @@ class TwoPhaseLinearParameterScheduler(Scheduler): def __init__( self, max_ep: int, - early_stopping_callback: Callable = None, + early_stopping_checker: Callable = None, *, parameter_names: [str], split_ep: float, @@ -85,7 +85,7 @@ def __init__( ): if split_ep <= 0 or split_ep >= max_ep: raise ValueError("split_ep must be between 0 and max_ep - 1.") - super().__init__(max_ep, early_stopping_callback=early_stopping_callback) + super().__init__(max_ep, early_stopping_checker=early_stopping_checker) self._parameter_names = parameter_names self._split_ep = split_ep if isinstance(start_values, float): From d70074a3fc1fe43d22e35e7d9a1082a3b4e823a3 Mon Sep 17 00:00:00 2001 From: ysqyang Date: Fri, 25 Dec 2020 15:26:43 +0800 Subject: [PATCH 305/337] put PG and AC under PolicyOptimization class and refined examples accordingly --- examples/cim/dqn/config.yml | 2 +- .../components/agent_manager.py | 8 +- examples/cim/policy_optimization/config.yml | 2 +- .../single_process_launcher.py | 5 +- maro/rl/__init__.py | 10 +- maro/rl/agent/simple_agent_manager.py | 2 +- maro/rl/algorithms/common.py | 6 - maro/rl/algorithms/dqn.py | 17 +-- maro/rl/algorithms/pg.py | 60 ---------- .../{ac.py => policy_optimization.py} | 106 ++++++++++++------ maro/rl/shaping/k_step_experience_shaper.py | 14 +-- maro/rl/utils/trajectory_utils.py | 24 ++-- .../rl_formulation.ipynb | 2 +- 13 files changed, 119 insertions(+), 139 deletions(-) delete mode 100644 maro/rl/algorithms/common.py delete mode 100644 maro/rl/algorithms/pg.py rename maro/rl/algorithms/{ac.py => policy_optimization.py} (55%) diff --git a/examples/cim/dqn/config.yml b/examples/cim/dqn/config.yml index e3564c570..52d996a73 100644 --- a/examples/cim/dqn/config.yml +++ b/examples/cim/dqn/config.yml @@ -34,7 +34,7 @@ agents: optimizer: lr: 0.05 config: - reward_decay: .0 + reward_discount: .0 target_update_frequency: 5 tau: 0.1 is_double: true diff --git a/examples/cim/policy_optimization/components/agent_manager.py b/examples/cim/policy_optimization/components/agent_manager.py index eab327622..db235ac00 100644 --- a/examples/cim/policy_optimization/components/agent_manager.py +++ b/examples/cim/policy_optimization/components/agent_manager.py @@ -7,7 +7,7 @@ from maro.rl import ( AbsAgent, ActorCritic, ActorCriticConfig, FullyConnectedBlock, LearningModuleManager, LearningModule, - OptimizerOptions, PolicyGradient, PolicyGradientConfig, SimpleAgentManager + OptimizerOptions, PolicyGradient, PolicyOptimizationConfig, SimpleAgentManager ) from maro.utils import set_seeds @@ -48,13 +48,15 @@ def create_po_agents(agent_id_list, config): ) hyper_params = config.actor_critic_hyper_parameters - hyper_params.update({"reward_decay": config.reward_decay}) + hyper_params.update({"reward_discount": config.reward_discount}) algorithm = ActorCritic( LearningModuleManager(actor_module, critic_module), ActorCriticConfig(critic_loss_func=nn.functional.smooth_l1_loss, **hyper_params) ) else: - algorithm = PolicyGradient(LearningModuleManager(actor_module), PolicyGradientConfig(config.reward_decay)) + algorithm = PolicyGradient( + LearningModuleManager(actor_module), PolicyOptimizationConfig(config.reward_discount) + ) agent_dict[agent_id] = POAgent(name=agent_id, algorithm=algorithm) diff --git a/examples/cim/policy_optimization/config.yml b/examples/cim/policy_optimization/config.yml index ec4000239..6243af934 100644 --- a/examples/cim/policy_optimization/config.yml +++ b/examples/cim/policy_optimization/config.yml @@ -41,7 +41,7 @@ agents: batch_norm_enabled: true critic_optimizer: lr: 0.001 - reward_decay: .0 + reward_discount: .0 actor_critic_hyper_parameters: train_iters: 10 actor_loss_coefficient: 0.1 diff --git a/examples/cim/policy_optimization/single_process_launcher.py b/examples/cim/policy_optimization/single_process_launcher.py index 74c72e2f3..64f90e26e 100644 --- a/examples/cim/policy_optimization/single_process_launcher.py +++ b/examples/cim/policy_optimization/single_process_launcher.py @@ -13,7 +13,7 @@ from components import CIMActionShaper, CIMStateShaper, POAgentManager, TruncatedExperienceShaper, create_po_agents -class EarlyStopping: +class EarlyStoppingChecker: """Callable class that checks the performance history to determine early stopping. Args: @@ -29,7 +29,6 @@ def __init__(self, warmup_ep: int, last_k: int, perf_threshold: float, perf_stab self._last_k = last_k self._perf_threshold = perf_threshold self._perf_stability_threshold = perf_stability_threshold - print(f"perf threshold: {self._perf_threshold}, perf stability threshold: {self._perf_stability_threshold}") def get_metric(record): return 1 - record["container_shortage"] / record["order_requirements"] @@ -75,7 +74,7 @@ def launch(config): # Step 4: Create an actor and a learner to start the training process. scheduler = Scheduler( config.main_loop.max_episode, - early_stopping_callback=EarlyStopping(**config.main_loop.early_stopping) + early_stopping_checker=EarlyStoppingChecker(**config.main_loop.early_stopping) ) actor = SimpleActor(env, agent_manager) learner = SimpleLearner( diff --git a/maro/rl/__init__.py b/maro/rl/__init__.py index 2e422c28e..3d434b540 100644 --- a/maro/rl/__init__.py +++ b/maro/rl/__init__.py @@ -7,9 +7,10 @@ from maro.rl.agent.abs_agent_manager import AbsAgentManager, AgentManagerMode from maro.rl.agent.simple_agent_manager import SimpleAgentManager from maro.rl.algorithms.abs_algorithm import AbsAlgorithm -from maro.rl.algorithms.ac import ActorCritic, ActorCriticConfig -from maro.rl.algorithms.pg import PolicyGradient, PolicyGradientConfig -from maro.rl.algorithms.common import ActionWithLogProbability +from maro.rl.algorithms.policy_optimization import ( + ActionWithLogProbability, ActorCritic, ActorCriticConfig, PolicyGradient, PolicyOptimization, + PolicyOptimizationConfig +) from maro.rl.algorithms.dqn import DQN, DQNConfig from maro.rl.dist_topologies.experience_collection import ( concat_experiences_by_agent, merge_experiences_with_trajectory_boundaries @@ -62,7 +63,8 @@ 'OptimizerOptions', 'OverwriteType', 'PolicyGradient', - 'PolicyGradientConfig', + 'PolicyOptimization', + 'PolicyOptimizationConfig', 'Scheduler', 'SimpleActor', 'SimpleAgentManager', diff --git a/maro/rl/agent/simple_agent_manager.py b/maro/rl/agent/simple_agent_manager.py index 854941b81..91248eb14 100644 --- a/maro/rl/agent/simple_agent_manager.py +++ b/maro/rl/agent/simple_agent_manager.py @@ -4,7 +4,7 @@ import os from abc import abstractmethod -from maro.rl.algorithms.utils import ActionWithLogProbability +from maro.rl.algorithms.policy_optimization import ActionWithLogProbability from maro.rl.shaping.action_shaper import ActionShaper from maro.rl.shaping.experience_shaper import ExperienceShaper from maro.rl.shaping.state_shaper import StateShaper diff --git a/maro/rl/algorithms/common.py b/maro/rl/algorithms/common.py deleted file mode 100644 index a4731095e..000000000 --- a/maro/rl/algorithms/common.py +++ /dev/null @@ -1,6 +0,0 @@ -# Copyright (c) Microsoft Corporation. -# Licensed under the MIT license. - -from collections import namedtuple - -ActionWithLogProbability = namedtuple("action_with_probability", ["action", "log_probability"]) diff --git a/maro/rl/algorithms/dqn.py b/maro/rl/algorithms/dqn.py index 52b34b91b..524323c9e 100644 --- a/maro/rl/algorithms/dqn.py +++ b/maro/rl/algorithms/dqn.py @@ -14,7 +14,7 @@ class DQNConfig: """Configuration for the DQN algorithm. Args: - reward_decay (float): Reward decay as defined in standard RL terminology. + reward_discount (float): Reward decay as defined in standard RL terminology. loss_cls: Loss function class for evaluating TD errors. target_update_frequency (int): Number of training rounds between target model updates. epsilon (float): Exploration rate for epsilon-greedy exploration. Defaults to None. @@ -28,13 +28,13 @@ class DQNConfig: method. Defaults to False. """ __slots__ = [ - "reward_decay", "loss_func", "target_update_frequency", "epsilon", "tau", "is_double", "advantage_mode", + "reward_discount", "loss_func", "target_update_frequency", "epsilon", "tau", "is_double", "advantage_mode", "per_sample_td_error_enabled" ] def __init__( self, - reward_decay: float, + reward_discount: float, loss_cls, target_update_frequency: int, epsilon: float = .0, @@ -43,7 +43,7 @@ def __init__( advantage_mode: str = None, per_sample_td_error_enabled: bool = False ): - self.reward_decay = reward_decay + self.reward_discount = reward_discount self.target_update_frequency = target_update_frequency self.epsilon = epsilon self.tau = tau @@ -72,7 +72,7 @@ def __init__(self, model: LearningModuleManager, config: DQNConfig): self._training_counter = 0 self._target_model = model.copy() if model.is_trainable else None - def choose_action(self, state: np.ndarray) -> Union[int, np.ndarray]: + def choose_action(self, state: np.ndarray) -> Union[int, np.ndarray, list]: state = torch.from_numpy(state).to(self._device) is_single = len(state.shape) == 1 if is_single: @@ -86,10 +86,11 @@ def choose_action(self, state: np.ndarray) -> Union[int, np.ndarray]: if is_single: return greedy_action if np.random.random() > self._config.epsilon else np.random.choice(self._num_actions) - return np.array([ + # batch inference + return [ act if np.random.random() > self._config.epsilon else np.random.choice(self._num_actions) for act in greedy_action - ]) + ] def _get_q_values(self, model, states, is_training: bool = True): if self._config.advantage_mode is not None: @@ -117,7 +118,7 @@ def _compute_td_errors(self, states, actions, rewards, next_states): current_q_values_for_all_actions = self._get_q_values(self._model, states) current_q_values = current_q_values_for_all_actions.gather(1, actions).squeeze(1) # (N,) next_q_values = self._get_next_q_values(current_q_values_for_all_actions, next_states) # (N,) - target_q_values = (rewards + self._config.reward_decay * next_q_values).detach() # (N,) + target_q_values = (rewards + self._config.reward_discount * next_q_values).detach() # (N,) return self._config.loss_func(current_q_values, target_q_values) def train(self, states: np.ndarray, actions: np.ndarray, rewards: np.ndarray, next_states: np.ndarray): diff --git a/maro/rl/algorithms/pg.py b/maro/rl/algorithms/pg.py deleted file mode 100644 index 602d58420..000000000 --- a/maro/rl/algorithms/pg.py +++ /dev/null @@ -1,60 +0,0 @@ -# Copyright (c) Microsoft Corporation. -# Licensed under the MIT license. - -import numpy as np -import torch - -from maro.rl.algorithms.abs_algorithm import AbsAlgorithm -from maro.rl.models.learning_model import LearningModuleManager - -from .utils import ActionWithLogProbability, expand_dim, preprocess, to_device - - -class PolicyGradientConfig: - """Configuration for the Policy Gradient (PG) algorithm. - - Args: - reward_decay (float): Reward decay as defined in standard RL terminology. - """ - __slots__ = ["reward_decay"] - - def __init__(self, reward_decay: float): - self.reward_decay = reward_decay - - -class PolicyGradient(AbsAlgorithm): - """Policy Gradient (PG) algorithm. - - The Policy Gradient algorithm base on the policy gradient theorem, a.k.a. REINFORCE. - - Args: - model (LearningModuleManager): Policy model. - config: Configuration for the PG algorithm. - """ - @to_device - def __init__(self, model: LearningModuleManager, config: PolicyGradientConfig): - super().__init__(model, config) - self._device = torch.device("cuda" if torch.cuda.is_available() else "cpu") - self._model.to(self._device) - - @expand_dim - def choose_action(self, state: np.ndarray): - """Use the actor (policy) model to generate a stochastic action. - - Args: - state: Input to the actor model. - - Returns: - A ActionWithLogProbability namedtuple instance containing the action index and the corresponding - log probability. - """ - action_distribution = self._model(state, is_training=False).squeeze().numpy() # (num_actions,) - action = np.random.choice(len(action_distribution), p=action_distribution) - return ActionWithLogProbability(action=action, log_probability=np.log(action_distribution[action])) - - @preprocess - def train(self, states: np.ndarray, actions: np.ndarray, returns: np.ndarray): - action_distributions = self._model(states) - action_prob = action_distributions.gather(1, actions.unsqueeze(1)).squeeze() # (N, 1) - loss = -(torch.log(action_prob) * returns).mean() - self._model.learn(loss) diff --git a/maro/rl/algorithms/ac.py b/maro/rl/algorithms/policy_optimization.py similarity index 55% rename from maro/rl/algorithms/ac.py rename to maro/rl/algorithms/policy_optimization.py index f4c9fbf65..95d968677 100644 --- a/maro/rl/algorithms/ac.py +++ b/maro/rl/algorithms/policy_optimization.py @@ -1,29 +1,84 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT license. -from enum import Enum -from typing import Callable +from collections import namedtuple +from typing import Callable, List, Union import numpy as np import torch from maro.rl.algorithms.abs_algorithm import AbsAlgorithm from maro.rl.models.learning_model import LearningModuleManager -from maro.rl.utils.trajectory_utils import get_lambda_returns +from maro.rl.utils.trajectory_utils import get_lambda_returns, get_truncated_cumulative_reward -from .utils import ActionWithLogProbability, expand_dim, preprocess, to_device, validate_task_names +ActionWithLogProbability = namedtuple("ActionWithLogProbability", ["action", "log_probability"]) -class ActorCriticTask(Enum): - ACTOR = "actor" - CRITIC = "critic" +class PolicyOptimizationConfig: + """Configuration for the policy optimization algorithm family.""" + __slots__ = ["reward_discount"] + def __init__(self, reward_discount): + self.reward_discount = reward_discount -class ActorCriticConfig: + +class PolicyOptimization(AbsAlgorithm): + """Policy optimization algorithm family. + + The algorithm family includes policy gradient (e.g. REINFORCE), actor-critic, PPO, etc. + """ + def choose_action(self, state: np.ndarray) -> Union[ActionWithLogProbability, List[ActionWithLogProbability]]: + """Use the actor (policy) model to generate stochastic actions. + + Args: + state: Input to the actor model. + + Returns: + A single ActionWithLogProbability namedtuple or a list of ActionWithLogProbability namedtuples. + """ + state = torch.from_numpy(state).to(self._device) + is_single = len(state.shape) == 1 + if is_single: + state = state.unsqueeze(dim=0) + + action_distribution = self._model(state, task_name="actor", is_training=False).squeeze().numpy() + if is_single: + action = np.random.choice(len(action_distribution), p=action_distribution) + return ActionWithLogProbability(action=action, log_probability=np.log(action_distribution[action])) + + # batch inference + batch_results = [] + for distribution in action_distribution: + action = np.random.choice(len(distribution), p=distribution) + batch_results.append(ActionWithLogProbability(action=action, log_probability=np.log(distribution[action]))) + + return batch_results + + def train( + self, states: np.ndarray, actions: np.ndarray, log_action_prob: np.ndarray, rewards: np.ndarray + ): + raise NotImplementedError + + +class PolicyGradient(PolicyOptimization): + def train( + self, states: np.ndarray, actions: np.ndarray, log_action_prob: np.ndarray, rewards: np.ndarray + ): + states = torch.from_numpy(states).to(self._device) + actions = torch.from_numpy(actions).to(self._device) + returns = get_truncated_cumulative_reward(rewards, self._config.reward_discount) + returns = torch.from_numpy(returns).to(self._device) + action_distributions = self._model(states) + action_prob = action_distributions.gather(1, actions.unsqueeze(1)).squeeze() # (N, 1) + loss = -(torch.log(action_prob) * returns).mean() + self._model.learn(loss) + + +class ActorCriticConfig(PolicyOptimizationConfig): """Configuration for the Actor-Critic algorithm. Args: - reward_decay (float): Reward decay as defined in standard RL terminology. + reward_discount (float): Reward decay as defined in standard RL terminology. critic_loss_func (Callable): Loss function for the critic model. train_iters (int): Number of gradient descent steps per call to ``train``. actor_loss_coefficient (float): The coefficient for actor loss in the total loss function, e.g., @@ -36,12 +91,12 @@ class ActorCriticConfig: in which case the actor loss is calculated using the usual policy gradient theorem. """ __slots__ = [ - "reward_decay", "critic_loss_func", "train_iters", "actor_loss_coefficient", "k", "lam", "clip_ratio" + "reward_discount", "critic_loss_func", "train_iters", "actor_loss_coefficient", "k", "lam", "clip_ratio" ] def __init__( self, - reward_decay: float, + reward_discount: float, critic_loss_func: Callable, train_iters: int, actor_loss_coefficient: float = 1.0, @@ -49,7 +104,7 @@ def __init__( lam: float = 1.0, clip_ratio: float = None ): - self.reward_decay = reward_decay + super().__init__(reward_discount) self.critic_loss_func = critic_loss_func self.train_iters = train_iters self.actor_loss_coefficient = actor_loss_coefficient @@ -58,7 +113,7 @@ def __init__( self.clip_ratio = clip_ratio -class ActorCritic(AbsAlgorithm): +class ActorCritic(PolicyOptimization): """Actor Critic algorithm with separate policy and value models (no shared layers). The Actor-Critic algorithm base on the policy gradient theorem. @@ -68,37 +123,24 @@ class ActorCritic(AbsAlgorithm): It may or may not have a shared bottom stack. config: Configuration for the AC algorithm. """ - @validate_task_names(ActorCriticTask) - @to_device def __init__(self, model: LearningModuleManager, config: ActorCriticConfig): + self.validate_task_names(model.task_names, {"actor", "critic"}) super().__init__(model, config) - @expand_dim - def choose_action(self, state: np.ndarray): - """Use the actor (policy) model to generate a stochastic action. - - Args: - state: Input to the actor model. - - Returns: - A ActionWithLogProbability namedtuple instance containing the action index and the corresponding - log probability. - """ - action_distribution = self._model(state, task_name="actor", is_training=False).squeeze().numpy() - action = np.random.choice(len(action_distribution), p=action_distribution) - return ActionWithLogProbability(action=action, log_probability=np.log(action_distribution[action])) - def _get_values_and_bootstrapped_returns(self, state_sequence, reward_sequence): state_values = self._model(state_sequence, task_name="critic").detach().squeeze() return_est = get_lambda_returns( - reward_sequence, state_values, self._config.reward_decay, self._config.lam, k=self._config.k + reward_sequence, state_values, self._config.reward_discount, self._config.lam, k=self._config.k ) return state_values, return_est - @preprocess def train( self, states: np.ndarray, actions: np.ndarray, log_action_prob: np.ndarray, rewards: np.ndarray ): + states = torch.from_numpy(states).to(self._device) + actions = torch.from_numpy(actions).to(self._device) + log_action_prob = torch.from_numpy(log_action_prob).to(self._device) + rewards = torch.from_numpy(rewards).to(self._device) state_values, return_est = self._get_values_and_bootstrapped_returns(states, rewards) advantages = return_est - state_values for _ in range(self._config.train_iters): diff --git a/maro/rl/shaping/k_step_experience_shaper.py b/maro/rl/shaping/k_step_experience_shaper.py index fbf892b5e..04649ab9c 100644 --- a/maro/rl/shaping/k_step_experience_shaper.py +++ b/maro/rl/shaping/k_step_experience_shaper.py @@ -23,13 +23,13 @@ class KStepExperienceShaper(ExperienceShaper): Args: reward_func (Callable): a function used to compute immediate rewards from metrics given by the env. - reward_decay (float): decay factor used to evaluate multi-step returns. + reward_discount (float): decay factor used to evaluate multi-step returns. num_steps (int): number of time steps used in computing returns is_per_agent (bool): if True, the generated experiences will be bucketed by agent ID. """ - def __init__(self, reward_func: Callable, reward_decay: float, num_steps: int, is_per_agent: bool = True): + def __init__(self, reward_func: Callable, reward_discount: float, num_steps: int, is_per_agent: bool = True): super().__init__(reward_func) - self._reward_decay = reward_decay + self._reward_discount = reward_discount self._num_steps = num_steps self._is_per_agent = is_per_agent @@ -42,11 +42,11 @@ def __call__(self, trajectory, snapshot_list): next_transition = trajectory[min(len(trajectory) - 1, i + self._num_steps)] reward_list.appendleft(self._reward_func(trajectory[i]["metrics"])) # compute the full return - full_return = full_return * self._reward_decay + reward_list[0] + full_return = full_return * self._reward_discount + reward_list[0] # compute the partial return - partial_return = partial_return * self._reward_decay + reward_list[0] + partial_return = partial_return * self._reward_discount + reward_list[0] if len(reward_list) > self._num_steps: - partial_return -= reward_list.pop() * self._reward_decay ** (self._num_steps - 1) + partial_return -= reward_list.pop() * self._reward_discount ** (self._num_steps - 1) agent_exp = experiences[transition["agent_id"]] if self._is_per_agent else experiences agent_exp[KStepExperienceKeys.STATE.value].appendleft(transition["state"]) agent_exp[KStepExperienceKeys.ACTION.value].appendleft(transition["action"]) @@ -55,7 +55,7 @@ def __call__(self, trajectory, snapshot_list): agent_exp[KStepExperienceKeys.NEXT_STATE.value].appendleft(next_transition["state"]) agent_exp[KStepExperienceKeys.NEXT_ACTION.value].appendleft(next_transition["action"]) agent_exp[KStepExperienceKeys.DISCOUNT.value].appendleft( - self._reward_decay ** (min(self._num_steps, len(trajectory) - 1 - i)) + self._reward_discount ** (min(self._num_steps, len(trajectory) - 1 - i)) ) return dict(experiences) diff --git a/maro/rl/utils/trajectory_utils.py b/maro/rl/utils/trajectory_utils.py index d6a87b02c..97936b1bd 100644 --- a/maro/rl/utils/trajectory_utils.py +++ b/maro/rl/utils/trajectory_utils.py @@ -16,9 +16,9 @@ def get_truncated_cumulative_reward( ): """Compute K-step cumulative rewards from a reward sequence. Args: - rewards (Union[list, np.ndarray, torch.tensor]): reward sequence from a trajectory. - discount (float): reward discount as in standard RL. - k (int): number of steps in computing cumulative rewards. If it is -1, returns are computed using the + rewards (Union[list, np.ndarray, torch.tensor]): Reward sequence from a trajectory. + discount (float): Reward discount as in standard RL. + k (int): Number of steps in computing cumulative rewards. If it is -1, returns are computed using the largest possible number of steps. Defaults to -1. Returns: @@ -41,10 +41,10 @@ def get_k_step_returns( ): """Compute K-step returns given reward and value sequences. Args: - rewards (Union[list, np.ndarray, torch.tensor]): reward sequence from a trajectory. - values (Union[list, np.ndarray, torch.tensor]): sequence of values for the traversed states in a trajectory. - discount (float): reward discount as in standard RL. - k (int): number of steps in computing returns. If it is -1, returns are computed using the largest possible + rewards (Union[list, np.ndarray, torch.tensor]): Reward sequence from a trajectory. + values (Union[list, np.ndarray, torch.tensor]): Sequence of values for the traversed states in a trajectory. + discount (float): Reward discount as in standard RL. + k (int): Number of steps in computing returns. If it is -1, returns are computed using the largest possible number of steps. Defaults to -1. Returns: @@ -72,11 +72,11 @@ def get_lambda_returns( ): """Compute lambda returns given reward and value sequences and a k. Args: - rewards (Union[list, np.ndarray, torch.tensor]): reward sequence from a trajectory. - values (Union[list, np.ndarray, torch.tensor]): sequence of values for the traversed states in a trajectory. - discount (float): reward discount as in standard RL. - lam (float): the lambda coefficient involved in computing lambda returns. - k (int): number of steps where the lambda return series is truncated. If it is -1, no truncating is done and + rewards (Union[list, np.ndarray, torch.tensor]): Reward sequence from a trajectory. + values (Union[list, np.ndarray, torch.tensor]): Sequence of values for the traversed states in a trajectory. + discount (float): Reward discount as in standard RL. + lam (float): Lambda coefficient involved in computing lambda returns. + k (int): Number of steps where the lambda return series is truncated. If it is -1, no truncating is done and the lambda return is carried out to the end of the sequence. Defaults to -1. Returns: diff --git a/notebooks/container_inventory_management/rl_formulation.ipynb b/notebooks/container_inventory_management/rl_formulation.ipynb index 195871c52..00c10be40 100644 --- a/notebooks/container_inventory_management/rl_formulation.ipynb +++ b/notebooks/container_inventory_management/rl_formulation.ipynb @@ -271,7 +271,7 @@ " algorithm = DQN(\n", " model=LearningModuleManager(q_module),\n", " config=DQNConfig(\n", - " reward_decay=.0, \n", + " reward_discount=.0, \n", " target_update_frequency=5, \n", " tau=0.1, \n", " is_double=True, \n", From afdd054e01f39f31bc36e9e662a129a9e8283b3b Mon Sep 17 00:00:00 2001 From: ysqyang Date: Fri, 25 Dec 2020 17:59:49 +0800 Subject: [PATCH 306/337] fixed lint issues --- maro/rl/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/maro/rl/__init__.py b/maro/rl/__init__.py index 3d434b540..bcb7c0586 100644 --- a/maro/rl/__init__.py +++ b/maro/rl/__init__.py @@ -7,11 +7,11 @@ from maro.rl.agent.abs_agent_manager import AbsAgentManager, AgentManagerMode from maro.rl.agent.simple_agent_manager import SimpleAgentManager from maro.rl.algorithms.abs_algorithm import AbsAlgorithm +from maro.rl.algorithms.dqn import DQN, DQNConfig from maro.rl.algorithms.policy_optimization import ( ActionWithLogProbability, ActorCritic, ActorCriticConfig, PolicyGradient, PolicyOptimization, PolicyOptimizationConfig ) -from maro.rl.algorithms.dqn import DQN, DQNConfig from maro.rl.dist_topologies.experience_collection import ( concat_experiences_by_agent, merge_experiences_with_trajectory_boundaries ) From 53d2ee45b69e38e5defb9ba75a5b54dc50851094 Mon Sep 17 00:00:00 2001 From: ysqyang Date: Sun, 27 Dec 2020 23:32:13 +0800 Subject: [PATCH 307/337] removed action_dim from noise explorer classes and added some shape checks --- maro/rl/exploration/noise_explorer.py | 39 ++++++++++++++------------- 1 file changed, 21 insertions(+), 18 deletions(-) diff --git a/maro/rl/exploration/noise_explorer.py b/maro/rl/exploration/noise_explorer.py index 91d7623c6..d075dc153 100644 --- a/maro/rl/exploration/noise_explorer.py +++ b/maro/rl/exploration/noise_explorer.py @@ -13,12 +13,12 @@ class NoiseExplorer(AbsExplorer): """Explorer that adds a random noise to a model-generated action.""" def __init__( self, - action_dim: int, - min_action: Union[float, np.ndarray] = None, - max_action: Union[float, np.ndarray] = None + min_action: Union[float, list, np.ndarray] = None, + max_action: Union[float, list, np.ndarray] = None ): + if isinstance(min_action, (list, np.ndarray)) and isinstance(max_action, (list, np.ndarray)): + assert len(min_action) == len(max_action), "min_action and max_action should have the same dimension." super().__init__() - self._action_dim = action_dim self._min_action = min_action self._max_action = max_action @@ -35,13 +35,15 @@ class UniformNoiseExplorer(NoiseExplorer): """Explorer that adds a random noise to a model-generated action sampled from a uniform distribution.""" def __init__( self, - action_dim: int, - min_action: Union[float, np.ndarray] = None, - max_action: Union[float, np.ndarray] = None, - noise_lower_bound: Union[float, np.ndarray] = .0, - noise_upper_bound: Union[float, np.ndarray] = .0 + min_action: Union[float, list, np.ndarray] = None, + max_action: Union[float, list, np.ndarray] = None, + noise_lower_bound: Union[float, list, np.ndarray] = .0, + noise_upper_bound: Union[float, list, np.ndarray] = .0 ): - super().__init__(action_dim, min_action, max_action) + if isinstance(noise_upper_bound, (list, np.ndarray)) and isinstance(noise_upper_bound, (list, np.ndarray)): + assert len(noise_lower_bound) == len(noise_upper_bound), \ + "noise_lower_bound and noise_upper_bound should have the same dimension." + super().__init__(min_action, max_action) self._noise_lower_bound = noise_lower_bound self._noise_upper_bound = noise_upper_bound @@ -50,7 +52,7 @@ def update(self, *, noise_lower_bound: Union[float, np.ndarray], noise_upper_bou self._noise_upper_bound = noise_upper_bound def __call__(self, action: np.ndarray): - action += np.random.uniform(self._noise_lower_bound, self._noise_upper_bound, self._action_dim) + action += np.random.uniform(self._noise_lower_bound, self._noise_upper_bound) if self._min_action is not None or self._max_action is not None: return np.clip(action, self._min_action, self._max_action) else: @@ -61,16 +63,17 @@ class GaussianNoiseExplorer(NoiseExplorer): """Explorer that adds a random noise to a model-generated action sampled from a Gaussian distribution.""" def __init__( self, - action_dim: int, - min_action: Union[float, np.ndarray] = None, - max_action: Union[float, np.ndarray] = None, - noise_mean: Union[float, np.ndarray] = .0, - noise_stddev: Union[float, np.ndarray] = .0, + min_action: Union[float, list, np.ndarray] = None, + max_action: Union[float, list, np.ndarray] = None, + noise_mean: Union[float, list, np.ndarray] = .0, + noise_stddev: Union[float, list, np.ndarray] = .0, is_relative: bool = False ): - super().__init__(action_dim, min_action, max_action) + if isinstance(noise_mean, (list, np.ndarray)) and isinstance(noise_stddev, (list, np.ndarray)): + assert len(noise_mean) == len(noise_stddev), "noise_mean and noise_stddev should have the same dimension." if is_relative and noise_mean != .0: raise ValueError("Standard deviation cannot be relative if noise mean is non-zero.") + super().__init__(min_action, max_action) self._noise_mean = noise_mean self._noise_stddev = noise_stddev self._is_relative = is_relative @@ -80,7 +83,7 @@ def update(self, *, noise_mean: Union[float, np.ndarray], noise_stddev: Union[fl self._noise_stddev = noise_stddev def __call__(self, action: np.ndarray): - noise = np.random.normal(loc=self._noise_mean, scale=self._noise_stddev, size=self._action_dim) + noise = np.random.normal(loc=self._noise_mean, scale=self._noise_stddev) action += (noise * action) if self._is_relative else noise if self._min_action is not None or self._max_action is not None: return np.clip(action, self._min_action, self._max_action) From ee3743597a35af5299bafcaeb62379fd5c8307fa Mon Sep 17 00:00:00 2001 From: ysqyang Date: Mon, 28 Dec 2020 00:07:40 +0800 Subject: [PATCH 308/337] modified NoiseExplorer's __call__ logic to batch processing --- maro/rl/exploration/noise_explorer.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/maro/rl/exploration/noise_explorer.py b/maro/rl/exploration/noise_explorer.py index d075dc153..999622924 100644 --- a/maro/rl/exploration/noise_explorer.py +++ b/maro/rl/exploration/noise_explorer.py @@ -52,6 +52,9 @@ def update(self, *, noise_lower_bound: Union[float, np.ndarray], noise_upper_bou self._noise_upper_bound = noise_upper_bound def __call__(self, action: np.ndarray): + return [self._get_exploration_action(act) for act in action] + + def _get_exploration_action(self, action): action += np.random.uniform(self._noise_lower_bound, self._noise_upper_bound) if self._min_action is not None or self._max_action is not None: return np.clip(action, self._min_action, self._max_action) @@ -83,6 +86,9 @@ def update(self, *, noise_mean: Union[float, np.ndarray], noise_stddev: Union[fl self._noise_stddev = noise_stddev def __call__(self, action: np.ndarray): + return [self._get_exploration_action(act) for act in action] + + def _get_exploration_action(self, action): noise = np.random.normal(loc=self._noise_mean, scale=self._noise_stddev) action += (noise * action) if self._is_relative else noise if self._min_action is not None or self._max_action is not None: From 56cacfdbcd110e26c1e157c7d55efe405b08d67f Mon Sep 17 00:00:00 2001 From: ysqyang Date: Mon, 28 Dec 2020 00:36:00 +0800 Subject: [PATCH 309/337] made NoiseExplorer's __call__ return type np array --- maro/rl/exploration/noise_explorer.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/maro/rl/exploration/noise_explorer.py b/maro/rl/exploration/noise_explorer.py index 999622924..370d3eefe 100644 --- a/maro/rl/exploration/noise_explorer.py +++ b/maro/rl/exploration/noise_explorer.py @@ -27,8 +27,8 @@ def update(self, **exploration_params): raise NotImplementedError @abstractmethod - def __call__(self, action): - return NotImplementedError + def __call__(self, action) -> np.ndarray: + raise NotImplementedError class UniformNoiseExplorer(NoiseExplorer): @@ -51,8 +51,8 @@ def update(self, *, noise_lower_bound: Union[float, np.ndarray], noise_upper_bou self._noise_lower_bound = noise_lower_bound self._noise_upper_bound = noise_upper_bound - def __call__(self, action: np.ndarray): - return [self._get_exploration_action(act) for act in action] + def __call__(self, action: np.ndarray) -> np.ndarray: + return np.array([self._get_exploration_action(act) for act in action]) def _get_exploration_action(self, action): action += np.random.uniform(self._noise_lower_bound, self._noise_upper_bound) @@ -85,8 +85,8 @@ def update(self, *, noise_mean: Union[float, np.ndarray], noise_stddev: Union[fl self._noise_mean = noise_mean self._noise_stddev = noise_stddev - def __call__(self, action: np.ndarray): - return [self._get_exploration_action(act) for act in action] + def __call__(self, action: np.ndarray) -> np.ndarray: + return np.array([self._get_exploration_action(act) for act in action]) def _get_exploration_action(self, action): noise = np.random.normal(loc=self._noise_mean, scale=self._noise_stddev) From c981ffe00b01e3bd8e7df160c9342d6d583f6c92 Mon Sep 17 00:00:00 2001 From: ysqyang Date: Mon, 28 Dec 2020 15:25:58 +0800 Subject: [PATCH 310/337] renamed update to set_parameters in explorer --- maro/rl/exploration/abs_explorer.py | 2 +- maro/rl/exploration/noise_explorer.py | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/maro/rl/exploration/abs_explorer.py b/maro/rl/exploration/abs_explorer.py index a50e4be57..40558b263 100644 --- a/maro/rl/exploration/abs_explorer.py +++ b/maro/rl/exploration/abs_explorer.py @@ -12,7 +12,7 @@ def __init__(self): pass @abstractmethod - def update(self, **exploration_params): + def set_parameters(self, **exploration_params): return NotImplementedError @abstractmethod diff --git a/maro/rl/exploration/noise_explorer.py b/maro/rl/exploration/noise_explorer.py index 370d3eefe..065cf4d83 100644 --- a/maro/rl/exploration/noise_explorer.py +++ b/maro/rl/exploration/noise_explorer.py @@ -23,7 +23,7 @@ def __init__( self._max_action = max_action @abstractmethod - def update(self, **exploration_params): + def set_parameters(self, **parameters): raise NotImplementedError @abstractmethod @@ -47,7 +47,7 @@ def __init__( self._noise_lower_bound = noise_lower_bound self._noise_upper_bound = noise_upper_bound - def update(self, *, noise_lower_bound: Union[float, np.ndarray], noise_upper_bound: Union[float, np.ndarray]): + def set_parameters(self, *, noise_lower_bound, noise_upper_bound): self._noise_lower_bound = noise_lower_bound self._noise_upper_bound = noise_upper_bound @@ -81,7 +81,7 @@ def __init__( self._noise_stddev = noise_stddev self._is_relative = is_relative - def update(self, *, noise_mean: Union[float, np.ndarray], noise_stddev: Union[float, np.ndarray]): + def set_parameters(self, *, noise_mean, noise_stddev): self._noise_mean = noise_mean self._noise_stddev = noise_stddev From 39a29461eaef9f3d47901d41c403a74c10894d21 Mon Sep 17 00:00:00 2001 From: ysqyang Date: Mon, 28 Dec 2020 15:27:45 +0800 Subject: [PATCH 311/337] fixed old naming in test_grass --- maro/rl/exploration/epsilon_greedy_explorer.py | 2 +- tests/cli/grass/test_grass.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/maro/rl/exploration/epsilon_greedy_explorer.py b/maro/rl/exploration/epsilon_greedy_explorer.py index becddf94f..6df487ac0 100644 --- a/maro/rl/exploration/epsilon_greedy_explorer.py +++ b/maro/rl/exploration/epsilon_greedy_explorer.py @@ -26,7 +26,7 @@ def __call__(self, action_index: Union[int, np.ndarray]): else: return self._get_exploration_action(action_index) - def update(self, *, epsilon: float): + def set_parameters(self, *, epsilon: float): self._epsilon = epsilon def _get_exploration_action(self, action_index): diff --git a/tests/cli/grass/test_grass.py b/tests/cli/grass/test_grass.py index 55b5d714c..ef34297cc 100644 --- a/tests/cli/grass/test_grass.py +++ b/tests/cli/grass/test_grass.py @@ -357,7 +357,7 @@ def test21_train_dqn(self) -> None: with open(f"{dqn_target_dir}/distributed_config.yml", "r") as fr: distributed_config = yaml.safe_load(fr) with open(f"{dqn_target_dir}/config.yml", "w") as fw: - config["general"]["max_episode"] = 30 + config["main_loop"]["max_episode"] = 30 yaml.safe_dump(config, fw) with open(f"{dqn_target_dir}/distributed_config.yml", "w") as fw: distributed_config["redis"]["hostname"] = master_details["private_ip_address"] From 48e284adf76506770814495bd7dd68f993954847 Mon Sep 17 00:00:00 2001 From: ysqyang Date: Wed, 30 Dec 2020 18:04:43 +0800 Subject: [PATCH 312/337] moved optimizer options to LearningModel --- docs/source/examples/cim.rst | 2 +- docs/source/examples/multi_agent_dqn_cim.rst | 2 +- examples/cim/dqn/components/agent_manager.py | 14 +- maro/rl/__init__.py | 4 +- maro/rl/algorithms/abs_algorithm.py | 6 +- maro/rl/algorithms/dqn.py | 6 +- maro/rl/models/__init__.py | 4 +- maro/rl/models/learning_model.py | 168 +++++++++--------- maro/utils/exception/rl_toolkit_exception.py | 6 +- .../rl_formulation.ipynb | 6 +- 10 files changed, 109 insertions(+), 109 deletions(-) diff --git a/docs/source/examples/cim.rst b/docs/source/examples/cim.rst index f503801fb..c7ce500b9 100644 --- a/docs/source/examples/cim.rst +++ b/docs/source/examples/cim.rst @@ -143,7 +143,7 @@ the DQN algorithm and an experience pool for each agent. set_seeds(config.agents.seed) num_actions = config.agents.algorithm.num_actions for agent_id in self._agent_id_list: - eval_model = LearningModuleManager(decision_layers=MLPDecisionLayers(name=f'{agent_id}.policy', + eval_model = LearningModel(decision_layers=MLPDecisionLayers(name=f'{agent_id}.policy', input_dim=self._state_shaper.dim, output_dim=num_actions, **config.agents.algorithm.model) diff --git a/docs/source/examples/multi_agent_dqn_cim.rst b/docs/source/examples/multi_agent_dqn_cim.rst index 787fe4cee..2b917646c 100644 --- a/docs/source/examples/multi_agent_dqn_cim.rst +++ b/docs/source/examples/multi_agent_dqn_cim.rst @@ -126,7 +126,7 @@ experience pools before training, in accordance with the DQN algorithm. set_seeds(config.seed) agent_dict = {} for agent_id in agent_id_list: - eval_model = LearningModuleManager( + eval_model = LearningModel( decision_layers=FullyConnectedNet( name=f'{agent_id}.policy', input_dim=config.algorithm.input_dim, diff --git a/examples/cim/dqn/components/agent_manager.py b/examples/cim/dqn/components/agent_manager.py index 1c2cbe9bd..94afef408 100644 --- a/examples/cim/dqn/components/agent_manager.py +++ b/examples/cim/dqn/components/agent_manager.py @@ -5,7 +5,7 @@ from torch.optim import RMSprop from maro.rl import ( - ColumnBasedStore, DQN, DQNConfig, FullyConnectedBlock, LearningModuleManager, LearningModule, OptimizerOptions, + ColumnBasedStore, DQN, DQNConfig, FullyConnectedBlock, LearningModel, NNStack, OptimizerOptions, SimpleAgentManager ) from maro.utils import set_seeds @@ -18,23 +18,21 @@ def create_dqn_agents(agent_id_list, config): set_seeds(config.seed) agent_dict = {} for agent_id in agent_id_list: - q_module = LearningModule( + q_net = NNStack( "q_value", - [FullyConnectedBlock( + FullyConnectedBlock( input_dim=config.algorithm.input_dim, output_dim=num_actions, activation=nn.LeakyReLU, is_head=True, **config.algorithm.model - )], - optimizer_options=OptimizerOptions(cls=RMSprop, params=config.algorithm.optimizer) + ) ) algorithm = DQN( - model=LearningModuleManager(q_module), - config=DQNConfig(**config.algorithm.config, loss_cls=nn.SmoothL1Loss) + LearningModel(q_net, optimizer_options=OptimizerOptions(cls=RMSprop, params=config.algorithm.optimizer)), + DQNConfig(**config.algorithm.config, loss_cls=nn.SmoothL1Loss) ) - agent_dict[agent_id] = CIMAgent( agent_id, algorithm, ColumnBasedStore(**config.experience_pool), **config.training_loop_parameters diff --git a/maro/rl/__init__.py b/maro/rl/__init__.py index b2d1b5269..e2fe1729a 100644 --- a/maro/rl/__init__.py +++ b/maro/rl/__init__.py @@ -11,7 +11,7 @@ AbsExplorer, EpsilonGreedyExplorer, GaussianNoiseExplorer, NoiseExplorer, UniformNoiseExplorer ) from maro.rl.learner import AbsLearner, SimpleLearner -from maro.rl.models import AbsBlock, FullyConnectedBlock, LearningModule, LearningModuleManager, OptimizerOptions +from maro.rl.models import AbsBlock, FullyConnectedBlock, NNStack, LearningModel, OptimizerOptions from maro.rl.scheduling import LinearParameterScheduler, Scheduler, TwoPhaseLinearParameterScheduler from maro.rl.shaping import AbsShaper, ActionShaper, ExperienceShaper, KStepExperienceShaper, StateShaper from maro.rl.storage import AbsStore, ColumnBasedStore, OverwriteType @@ -23,7 +23,7 @@ "ActorProxy", "ActorWorker", "concat_experiences_by_agent", "merge_experiences_with_trajectory_boundaries", "AbsExplorer", "EpsilonGreedyExplorer", "GaussianNoiseExplorer", "NoiseExplorer", "UniformNoiseExplorer", "AbsLearner", "SimpleLearner", - "AbsBlock", "FullyConnectedBlock", "LearningModule", "LearningModuleManager", "OptimizerOptions", + "AbsBlock", "FullyConnectedBlock", "NNStack", "LearningModel", "OptimizerOptions", "LinearParameterScheduler", "Scheduler", "TwoPhaseLinearParameterScheduler", "AbsShaper", "ActionShaper", "ExperienceShaper", "KStepExperienceShaper", "StateShaper", "AbsStore", "ColumnBasedStore", "OverwriteType" diff --git a/maro/rl/algorithms/abs_algorithm.py b/maro/rl/algorithms/abs_algorithm.py index be75832db..923a64ba8 100644 --- a/maro/rl/algorithms/abs_algorithm.py +++ b/maro/rl/algorithms/abs_algorithm.py @@ -5,7 +5,7 @@ import torch -from maro.rl.models.learning_model import LearningModuleManager +from maro.rl.models.learning_model import LearningModel from maro.utils.exception.rl_toolkit_exception import UnrecognizedTask @@ -17,10 +17,10 @@ class AbsAlgorithm(ABC): algorithms. Args: - model (LearningModuleManager): Task model or container of task models required by the algorithm. + model (LearningModel): Task model or container of task models required by the algorithm. config: Settings for the algorithm. """ - def __init__(self, model: LearningModuleManager, config): + def __init__(self, model: LearningModel, config): self._device = torch.device("cuda" if torch.cuda.is_available() else "cpu") self._model = model.to(self._device) self._config = config diff --git a/maro/rl/algorithms/dqn.py b/maro/rl/algorithms/dqn.py index 52b34b91b..ea675fb43 100644 --- a/maro/rl/algorithms/dqn.py +++ b/maro/rl/algorithms/dqn.py @@ -7,7 +7,7 @@ import torch from maro.rl.algorithms.abs_algorithm import AbsAlgorithm -from maro.rl.models.learning_model import LearningModuleManager +from maro.rl.models.learning_model import LearningModel class DQNConfig: @@ -59,10 +59,10 @@ class DQN(AbsAlgorithm): See https://web.stanford.edu/class/psych209/Readings/MnihEtAlHassibis15NatureControlDeepRL.pdf for details. Args: - model (LearningModuleManager): Q-value model. + model (LearningModel): Q-value model. config: Configuration for DQN algorithm. """ - def __init__(self, model: LearningModuleManager, config: DQNConfig): + def __init__(self, model: LearningModel, config: DQNConfig): self.validate_task_names(model.task_names, {"state_value", "advantage"}) super().__init__(model, config) if isinstance(self._model.output_dim, int): diff --git a/maro/rl/models/__init__.py b/maro/rl/models/__init__.py index 16f84d6fa..5eb309515 100644 --- a/maro/rl/models/__init__.py +++ b/maro/rl/models/__init__.py @@ -3,6 +3,6 @@ from .abs_block import AbsBlock from .fc_block import FullyConnectedBlock -from .learning_model import LearningModule, LearningModuleManager, OptimizerOptions +from .learning_model import NNStack, LearningModel, OptimizerOptions -__all__ = ["AbsBlock", "FullyConnectedBlock", "LearningModule", "LearningModuleManager", "OptimizerOptions"] +__all__ = ["AbsBlock", "FullyConnectedBlock", "NNStack", "LearningModel", "OptimizerOptions"] diff --git a/maro/rl/models/learning_model.py b/maro/rl/models/learning_model.py index b07eade3c..8f3387c61 100644 --- a/maro/rl/models/learning_model.py +++ b/maro/rl/models/learning_model.py @@ -2,49 +2,34 @@ # Licensed under the MIT license. from collections import namedtuple +from itertools import chain +from typing import Dict, Union import torch import torch.nn as nn from maro.utils import clone -from maro.utils.exception.rl_toolkit_exception import LearningModuleDimensionError, MissingOptimizer +from maro.utils.exception.rl_toolkit_exception import NNStackDimensionError, MissingOptimizer from .abs_block import AbsBlock OptimizerOptions = namedtuple("OptimizerOptions", ["cls", "params"]) -class LearningModule(nn.Module): - """NN model that consists of a sequence of chainable blocks. +class NNStack(nn.Module): + """An NN stack that consists of a sequence of chainable blocks. Args: - block_list (list): List of blocks that compose the model. They must be chainable, i.e., the output dimension + name (str): Name of the stack. + blocks (AbsBlock): Blocks that comprise the model. They must be chainable, i.e., the output dimension of a block must match the input dimension of its successor. - optimizer_options (OptimizerOptions): A namedtuple of (optimizer_class, optimizer_parameters). """ - def __init__(self, name: str, block_list: [AbsBlock], optimizer_options: OptimizerOptions = None): + def __init__(self, name: str, *blocks: [AbsBlock]): super().__init__() self._name = name self._input_dim = block_list[0].input_dim self._output_dim = block_list[-1].output_dim self._net = nn.Sequential(*block_list) - self._is_trainable = optimizer_options is not None - if self._is_trainable: - self._optimizer = optimizer_options.cls(self._net.parameters(), **optimizer_options.params) - else: - self._net.eval() - for param in self._net.parameters(): - param.requires_grad = False - - def __getstate__(self): - dic = self.__dict__.copy() - if "_optimizer" in dic: - del dic["_optimizer"] - dic["_is_trainable"] = False - return dic - - def __setstate__(self, dic: dict): - self.__dict__ = dic @property def name(self): @@ -58,10 +43,6 @@ def input_dim(self): def output_dim(self): return self._output_dim - @property - def is_trainable(self): - return self._is_trainable - def forward(self, inputs): """Feedforward computation. @@ -73,52 +54,72 @@ def forward(self, inputs): """ return self._net(inputs) - def zero_gradients(self): - if not self._is_trainable: - raise MissingOptimizer("No optimizer registered to the model") - self._optimizer.zero_grad() - - def step(self): - self._optimizer.step() - - def copy(self): - return clone(self) - -class LearningModuleManager(nn.Module): +class LearningModel(nn.Module): """NN model that consists of multiple task heads and an optional shared stack. Args: - task_modules (LearningModule): LearningModule instances, each of which performs a designated task. - shared_module (LearningModule): Network module that forms that shared part of the model. Defaults to None. + task_stacks (NNStack): NNStack instances, each of which performs a designated task. + shared_stack (NNStack): Network module that forms that shared part of the model. Defaults to None. + optimizer_options (Union[OptimizerOptions, Dict[str, OptimizerOptions]]): Optimizer options for + the internal stacks. If none, no optimizer will be created for the model and the model will not + be trainable. If it is a single OptimizerOptions instance, an optimizer will be created to jointly + optimize all parameters of the model. If it is a dictionary, for each `(key, value)` pair, an optimizer + specified by `value` will be created for the internal stack named `key`. Defaults to None. """ def __init__( - self, - *task_modules: LearningModule, - shared_module: LearningModule = None + self, + *task_stacks: NNStack, + shared_stack: NNStack = None, + optimizer_options: Union[OptimizerOptions, Dict[str, OptimizerOptions]] = None ): - self.validate_dims(*task_modules, shared_module=shared_module) + self.validate_dims(*task_stacks, shared_stack=shared_stack) super().__init__() - self._task_names = [module.name for module in task_modules] - + self._stack_dict = {stack.name: stack for stack in task_stacks] # shared stack - self._shared_module = shared_module + self._shared_stack = shared_stack + if self._shared_stack: + self._stack_dict[self._shared_stack.name] = self._shared_stack # task_heads - self._task_module_dict = nn.ModuleDict({task_module.name: task_module for task_module in task_modules}) - self._input_dim = self._shared_module.input_dim if self._shared_module else task_modules[0].input_dim - if len(task_modules) == 1: - self._output_dim = task_modules[0].output_dim + self._task_stack_dict = nn.ModuleDict({task_stack.name: task_stack for task_stack in task_stacks}) + self._input_dim = self._shared_stack.input_dim if self._shared_stack else task_stacks[0].input_dim + if len(task_stacks) == 1: + self._output_dim = task_stacks[0].output_dim + else: + self._output_dim = {task_stack.name: task_stack.output_dim for task_stack in task_stacks} + + self._is_trainable = optimizer_options is not None + if self._is_trainable: + if isinstance(optimizer_options, OptimizerOptions): + self._optimizer = optimizer_options.cls(self.parameters(), **optimizer_options.params) + else: + self._optimizer = { + stack_name: opt.cls(self._stack_dict[stack_name].parameters(), **opt.params) + for stack_name, opt in optimizer_options.items() + } else: - self._output_dim = {task_module.name: task_module.output_dim for task_module in task_modules} + self.eval() + for param in self.parameters(): + param.requires_grad = False + + def __getstate__(self): + dic = self.__dict__.copy() + if "_optimizer" in dic: + del dic["_optimizer"] + dic["_is_trainable"] = False + return dic + def __setstate__(self, dic: dict): + self.__dict__ = dic + @property def task_names(self) -> [str]: - return self._task_names + return list(self._task_stack_dict.keys()) @property - def shared_module(self): - return self._shared_module + def shared_stack(self): + return self._shared_stack @property def input_dim(self): @@ -130,25 +131,22 @@ def output_dim(self): @property def is_trainable(self) -> bool: - return ( - any(task_module.is_trainable for task_module in self._task_module_dict.values()) or - (self._shared_module is not None and self._shared_module.is_trainable) - ) + return self._is_trainable def _forward(self, inputs, task_name: str = None): - if self._shared_module: - inputs = self._shared_module(inputs) + if self._shared_stack: + inputs = self._shared_stack(inputs) - if len(self._task_module_dict) == 1: - return list(self._task_module_dict.values())[0](inputs) + if len(self._task_stack_dict) == 1: + return list(self._task_stack_dict.values())[0](inputs) if task_name is None: - return {name: task_module(inputs) for name, task_module in self._task_module_dict.items()} + return {name: task_stack(inputs) for name, task_stack in self._task_stack_dict.items()} if isinstance(task_name, list): - return {name: self._task_module_dict[name](inputs) for name in task_name} + return {name: self._task_stack_dict[name](inputs) for name in task_name} else: - return self._task_module_dict[task_name](inputs) + return self._task_stack_dict[task_name](inputs) def forward(self, inputs, task_name: str = None, is_training: bool = True): """Feedforward computations for the given head(s). @@ -173,22 +171,26 @@ def forward(self, inputs, task_name: str = None, is_training: bool = True): with torch.no_grad(): return self._forward(inputs, task_name) - + def learn(self, loss): """Use the loss to back-propagate gradients and apply them to the underlying parameters.""" - for task_module in self._task_module_dict.values(): - task_module.zero_gradients() - if self._shared_module is not None: - self._shared_module.zero_gradients() + if not self._is_trainable: + raise MissingOptimizer("No optimizer registered to the model") + if isinstance(self._optimizer, dict): + for optimizer in self._optimizer.values(): + optimizer.zero_grad() + else: + self._optimizer.zero_grad() # Obtain gradients through back-propagation loss.backward() # Apply gradients - for task_module in self._task_module_dict.values(): - task_module.step() - if self._shared_module is not None: - self._shared_module.step() + if isinstance(self._optimizer, dict): + for optimizer in self._optimizer.values(): + optimizer.step() + else: + self._optimizer.step() def soft_update(self, other_model: nn.Module, tau: float): for params, other_params in zip(self.parameters(), other_model.parameters()): @@ -210,10 +212,10 @@ def dump_to_file(self, path: str): torch.save(self.state_dict(), path) @staticmethod - def validate_dims(*task_modules, shared_module=None): - expected_dim = shared_module.output_dim if shared_module else task_modules[0].input_dim - for task_module in task_modules: - if task_module.input_dim != expected_dim: - raise LearningModuleDimensionError( - f"Expected input dimension {expected_dim} for task module: {task_module.name}, " - f"got {task_module.input_dim}") + def validate_dims(*task_stacks, shared_stack=None): + expected_dim = shared_stack.output_dim if shared_stack else task_stacks[0].input_dim + for task_stack in task_stacks: + if task_stack.input_dim != expected_dim: + raise NNStackDimensionError( + f"Expected input dimension {expected_dim} for task module: {task_stack.name}, " + f"got {task_stack.input_dim}") diff --git a/maro/utils/exception/rl_toolkit_exception.py b/maro/utils/exception/rl_toolkit_exception.py index 55ecde19f..f24506b88 100644 --- a/maro/utils/exception/rl_toolkit_exception.py +++ b/maro/utils/exception/rl_toolkit_exception.py @@ -36,18 +36,18 @@ def __init__(self, msg: str = None): class MissingOptimizer(MAROException): - """Raised when the optimizers are missing when calling LearningModuleManager's step() method.""" + """Raised when the optimizers are missing when calling LearningModel's step() method.""" def __init__(self, msg: str = None): super().__init__(4005, msg) class UnrecognizedTask(MAROException): - """Raised when a LearningModuleManager has task names that are not unrecognized by an algorithm.""" + """Raised when a LearningModel has task names that are not unrecognized by an algorithm.""" def __init__(self, msg: str = None): super().__init__(4006, msg) -class LearningModuleDimensionError(MAROException): +class NNStackDimensionError(MAROException): """Raised when a learning module's input dimension is incorrect.""" def __init__(self, msg: str = None): super().__init__(4007, msg) diff --git a/notebooks/container_inventory_management/rl_formulation.ipynb b/notebooks/container_inventory_management/rl_formulation.ipynb index 60eff91ca..66783e3bd 100644 --- a/notebooks/container_inventory_management/rl_formulation.ipynb +++ b/notebooks/container_inventory_management/rl_formulation.ipynb @@ -243,7 +243,7 @@ "from torch.optim import RMSprop\n", "\n", "from maro.rl import (\n", - " ColumnBasedStore, DQN, DQNConfig, FullyConnectedBlock, LearningModuleManager, LearningModule, OptimizerOptions, SimpleAgentManager\n", + " ColumnBasedStore, DQN, DQNConfig, FullyConnectedBlock, LearningModel, NNStack, OptimizerOptions, SimpleAgentManager\n", ")\n", "from maro.utils import set_seeds\n", "\n", @@ -252,7 +252,7 @@ " set_seeds(64) # for reproducibility\n", " agent_dict = {}\n", " for agent_id in agent_id_list:\n", - " q_module = LearningModule(\n", + " q_module = NNStack(\n", " \"q_value\",\n", " [FullyConnectedBlock(\n", " input_dim=state_shaper.dim,\n", @@ -269,7 +269,7 @@ " )\n", "\n", " algorithm = DQN(\n", - " model=LearningModuleManager(q_module),\n", + " model=LearningModel(q_module),\n", " config=DQNConfig(\n", " reward_decay=.0, \n", " target_update_frequency=5, \n", From 0d3da6b5d739a5b3cf4939ad86166096947a33a2 Mon Sep 17 00:00:00 2001 From: ysqyang Date: Wed, 30 Dec 2020 18:06:54 +0800 Subject: [PATCH 313/337] typo fix --- maro/rl/models/learning_model.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/maro/rl/models/learning_model.py b/maro/rl/models/learning_model.py index 8f3387c61..0f46a171b 100644 --- a/maro/rl/models/learning_model.py +++ b/maro/rl/models/learning_model.py @@ -27,9 +27,9 @@ class NNStack(nn.Module): def __init__(self, name: str, *blocks: [AbsBlock]): super().__init__() self._name = name - self._input_dim = block_list[0].input_dim - self._output_dim = block_list[-1].output_dim - self._net = nn.Sequential(*block_list) + self._input_dim = blocks[0].input_dim + self._output_dim = blocks[-1].output_dim + self._net = nn.Sequential(*blocks) @property def name(self): @@ -75,7 +75,7 @@ def __init__( ): self.validate_dims(*task_stacks, shared_stack=shared_stack) super().__init__() - self._stack_dict = {stack.name: stack for stack in task_stacks] + self._stack_dict = {stack.name: stack for stack in task_stacks} # shared stack self._shared_stack = shared_stack if self._shared_stack: From b22fd44fa3a2c9ee4e55bd729ac4dd61648b5842 Mon Sep 17 00:00:00 2001 From: ysqyang Date: Thu, 31 Dec 2020 00:01:26 +0800 Subject: [PATCH 314/337] fixed lint issues --- maro/rl/__init__.py | 4 ++-- maro/rl/models/__init__.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/maro/rl/__init__.py b/maro/rl/__init__.py index e2fe1729a..e65c530ff 100644 --- a/maro/rl/__init__.py +++ b/maro/rl/__init__.py @@ -11,7 +11,7 @@ AbsExplorer, EpsilonGreedyExplorer, GaussianNoiseExplorer, NoiseExplorer, UniformNoiseExplorer ) from maro.rl.learner import AbsLearner, SimpleLearner -from maro.rl.models import AbsBlock, FullyConnectedBlock, NNStack, LearningModel, OptimizerOptions +from maro.rl.models import AbsBlock, FullyConnectedBlock, LearningModel, NNStack, OptimizerOptions from maro.rl.scheduling import LinearParameterScheduler, Scheduler, TwoPhaseLinearParameterScheduler from maro.rl.shaping import AbsShaper, ActionShaper, ExperienceShaper, KStepExperienceShaper, StateShaper from maro.rl.storage import AbsStore, ColumnBasedStore, OverwriteType @@ -23,7 +23,7 @@ "ActorProxy", "ActorWorker", "concat_experiences_by_agent", "merge_experiences_with_trajectory_boundaries", "AbsExplorer", "EpsilonGreedyExplorer", "GaussianNoiseExplorer", "NoiseExplorer", "UniformNoiseExplorer", "AbsLearner", "SimpleLearner", - "AbsBlock", "FullyConnectedBlock", "NNStack", "LearningModel", "OptimizerOptions", + "AbsBlock", "FullyConnectedBlock", "LearningModel", "NNStack", "OptimizerOptions", "LinearParameterScheduler", "Scheduler", "TwoPhaseLinearParameterScheduler", "AbsShaper", "ActionShaper", "ExperienceShaper", "KStepExperienceShaper", "StateShaper", "AbsStore", "ColumnBasedStore", "OverwriteType" diff --git a/maro/rl/models/__init__.py b/maro/rl/models/__init__.py index 5eb309515..e56123c23 100644 --- a/maro/rl/models/__init__.py +++ b/maro/rl/models/__init__.py @@ -3,6 +3,6 @@ from .abs_block import AbsBlock from .fc_block import FullyConnectedBlock -from .learning_model import NNStack, LearningModel, OptimizerOptions +from .learning_model import LearningModel, NNStack, OptimizerOptions -__all__ = ["AbsBlock", "FullyConnectedBlock", "NNStack", "LearningModel", "OptimizerOptions"] +__all__ = ["AbsBlock", "FullyConnectedBlock", "LearningModel", "NNStack", "OptimizerOptions"] From ebf49e21ffec0cf1da69596e86baa0b07799db15 Mon Sep 17 00:00:00 2001 From: ysqyang Date: Thu, 31 Dec 2020 00:08:59 +0800 Subject: [PATCH 315/337] updated notebook --- docs/source/examples/cim.rst | 255 ------------------ .../rl_formulation.ipynb | 10 +- 2 files changed, 4 insertions(+), 261 deletions(-) delete mode 100644 docs/source/examples/cim.rst diff --git a/docs/source/examples/cim.rst b/docs/source/examples/cim.rst deleted file mode 100644 index c7ce500b9..000000000 --- a/docs/source/examples/cim.rst +++ /dev/null @@ -1,255 +0,0 @@ -Example Scenario: Container Inventory Management -================================================ - -This example demonstrates how to use MARO's reinforcement learning (RL) toolkit to solve the -`container inventory management `_ -(CIM) problem. It is formalized as a multi-agent reinforcement learning problem, where each port acts as a decision -agent. The agents take actions independently, e.g., loading containers to vessels or discharging containers from vessels. - -State Shaper ------------- - -`State shaper `_ converts the environment -observation to the model input state which includes temporal and spatial information. For this scenario, the model input -state includes: - -- Temporal information, including the past week's information of ports and vessels, such as shortage on port and -remaining space on vessel. -- Spatial information, including related downstream port features. - -.. code-block:: python - - class CIMStateShaper(StateShaper): - ... - def __call__(self, decision_event, snapshot_list): - tick, port_idx, vessel_idx = decision_event.tick, decision_event.port_idx, decision_event.vessel_idx - ticks = [tick - rt for rt in range(self._look_back - 1)] - future_port_idx_list = snapshot_list["vessels"][tick : vessel_idx : 'future_stop_list'].astype('int') - port_features = snapshot_list["ports"][ticks : [port_idx] + list(future_port_idx_list) : self._port_attributes] - vessel_features = snapshot_list["vessels"][tick : vessel_idx : self._vessel_attributes] - state = np.concatenate((port_features, vessel_features)) - return str(port_idx), state - - -Action Shaper -------------- - -`Action shaper `_ is used to convert an -agent's model output to an environment executable action. For this specific scenario, the action space consists of -integers from -10 to 10, with -10 indicating loading 100% of the containers in the current inventory to the vessel and -10 indicating discharging 100% of the containers on the vessel to the port. - -.. code-block:: python - - class CIMActionShaper(ActionShaper): - ... - def __call__(self, model_action, decision_event, snapshot_list): - scope = decision_event.action_scope - tick = decision_event.tick - port_idx = decision_event.port_idx - vessel_idx = decision_event.vessel_idx - - port_empty = snapshot_list["ports"][tick: port_idx: ["empty", "full", "on_shipper", "on_consignee"]][0] - vessel_remaining_space = snapshot_list["vessels"][tick: vessel_idx: ["empty", "full", "remaining_space"]][2] - early_discharge = snapshot_list["vessels"][tick:vessel_idx: "early_discharge"][0] - assert 0 <= model_action < len(self._action_space) - - if model_action < self._zero_action_index: - actual_action = max(round(self._action_space[model_action] * port_empty), -vessel_remaining_space) - elif model_action > self._zero_action_index: - plan_action = self._action_space[model_action] * (scope.discharge + early_discharge) - early_discharge - actual_action = round(plan_action) if plan_action > 0 else round(self._action_space[model_action] * scope.discharge) - else: - actual_action = 0 - - return Action(vessel_idx, port_idx, actual_action) - -Experience Shaper ------------------ - -`Experience shaper `_ is used to convert -an episode trajectory to trainable experiences for RL agents. For this specific scenario, the reward is a linear -combination of fulfillment and shortage in a limited time window. - -.. code-block:: python - - class TruncatedExperienceShaper(ExperienceShaper): - ... - def __call__(self, trajectory, snapshot_list): - experiences_by_agent = {} - for i in range(len(trajectory) - 1): - transition = trajectory[i] - agent_id = transition["agent_id"] - if agent_id not in experiences_by_agent: - experiences_by_agent[agent_id] = defaultdict(list) - experiences = experiences_by_agent[agent_id] - experiences["state"].append(transition["state"]) - experiences["action"].append(transition["action"]) - experiences["reward"].append(self._compute_reward(transition["event"], snapshot_list)) - experiences["next_state"].append(trajectory[i + 1]["state"]) - - return experiences_by_agent - - def _compute_reward(self, decision_event, snapshot_list): - start_tick = decision_event.tick + 1 - end_tick = decision_event.tick + self._time_window - ticks = list(range(start_tick, end_tick)) - - # calculate tc reward - future_fulfillment = snapshot_list["ports"][ticks::"fulfillment"] - future_shortage = snapshot_list["ports"][ticks::"shortage"] - decay_list = [self._time_decay_factor ** i for i in range(end_tick - start_tick) - for _ in range(future_fulfillment.shape[0] // (end_tick - start_tick))] - - tot_fulfillment = np.dot(future_fulfillment, decay_list) - tot_shortage = np.dot(future_shortage, decay_list) - - return np.float(self._fulfillment_factor * tot_fulfillment - self._shortage_factor * tot_shortage) - -Agent ------ - -`Agent `_ is a combination of (RL) -algorithm, experience pool, and a set of parameters that governs the training loop. For this scenario, the agent is the -abstraction of a port. We choose DQN as our underlying learning algorithm with a TD-error-based sampling mechanism. - -.. code-block:: python - class CIMAgent(AbsAgent): - ... - def train(self): - if len(self._experience_pool) < self._min_experiences_to_train: - return - - for _ in range(self._num_batches): - indexes, sample = self._experience_pool.sample_by_key("loss", self._batch_size) - state = np.asarray(sample["state"]) - action = np.asarray(sample["action"]) - reward = np.asarray(sample["reward"]) - next_state = np.asarray(sample["next_state"]) - loss = self._algorithm.train(state, action, reward, next_state) - self._experience_pool.update(indexes, {"loss": loss}) - -Agent Manager -------------- - -`Agent manager `_ -is an agent assembler and isolates the complexities of the environment and algorithm. For this scenario, It will load -the DQN algorithm and an experience pool for each agent. - -.. code-block:: python - - class DQNAgentManager(AbsAgentManager): - def _assemble(self, agent_dict): - set_seeds(config.agents.seed) - num_actions = config.agents.algorithm.num_actions - for agent_id in self._agent_id_list: - eval_model = LearningModel(decision_layers=MLPDecisionLayers(name=f'{agent_id}.policy', - input_dim=self._state_shaper.dim, - output_dim=num_actions, - **config.agents.algorithm.model) - ) - - algorithm = DQN(model_dict={"eval": eval_model}, - optimizer_opt=(RMSprop, config.agents.algorithm.optimizer), - loss_func_dict={"eval": smooth_l1_loss}, - hyper_params=DQNConfig(**config.agents.algorithm.hyper_parameters, - num_actions=num_actions)) - - experience_pool = ColumnBasedStore(**config.agents.experience_pool) - agent_dict[agent_id] = CIMAgent(name=agent_id, algorithm=algorithm, experience_pool=experience_pool, - **config.agents.training_loop_parameters) - -Main Loop with Actor and Learner (Single Process) -------------------------------------------------- - -This single-process workflow of a learning policy's interaction with a MARO environment is comprised of: -- Initializing an environment with specific scenario and topology parameters. -- Defining scenario-specific components, e.g. shapers. -- Creating an agent manager, which assembles underlying agents. -- Creating an `actor `_ and a -`learner `_ to start the -training process in which the agent manager interacts with the environment for collecting experiences and updating -policies. - -.. code-block::python - - env = Env(config.env.scenario, config.env.topology, durations=config.env.durations) - agent_id_list = [str(agent_id) for agent_id in env.agent_idx_list] - state_shaper = CIMStateShaper(**config.state_shaping) - action_shaper = CIMActionShaper(action_space=list(np.linspace(-1.0, 1.0, config.agents.algorithm.num_actions))) - experience_shaper = TruncatedExperienceShaper(**config.experience_shaping.truncated) - exploration_config = {"epsilon_range_dict": {"_all_": config.exploration.epsilon_range}, - "split_point_dict": {"_all_": config.exploration.split_point}, - "with_cache": config.exploration.with_cache - } - explorer = TwoPhaseLinearExplorer(agent_id_list, config.general.total_training_episodes, **exploration_config) - - agent_manager = DQNAgentManager(name="cim_learner", - mode=AgentMode.TRAIN_INFERENCE, - agent_id_list=agent_id_list, - state_shaper=state_shaper, - action_shaper=action_shaper, - experience_shaper=experience_shaper, - explorer=explorer) - - actor = SimpleActor(env=env, inference_agents=agent_manager) - learner = SimpleLearner(trainable_agents=agent_manager, actor=actor, - logger=Logger("single_host_cim_learner", auto_timestamp=False)) - - learner.learn(total_episodes=config.general.total_training_episodes) - - -Main Loop with Actor and Learner (Distributed / Multi-process) --------------------------------------------------------------- - -We demonstrate a single-learner and multi-actor topology where the learner drives the program by telling remote actors -to perform roll-out tasks and using the results they sent back to improve the policies. The workflow usually involves -launching a learner process and an actor process separately. Because training occurs on the learner side and inference -occurs on the actor side, we need to create appropriate agent managers on both sides. - -On the actor side, the agent manager must be equipped with all shapers as well as an explorer. Thus, The code for -creating an environment and an agent manager on the actor side is similar to that for the single-host version, -except that it is necessary to set the AgentMode to AgentMode.INFERENCE. As in the single-process version, the environment -and the agent manager are wrapped in a SimpleActor instance. To make the actor a distributed worker, we need to further -wrap it in an ActorWorker instance. Finally, we launch the worker and it starts to listen to roll-out requests from the -learner. The following code snippet shows the creation of an actor worker with a simple (local) actor wrapped inside. - -.. code-block:: python - - agent_manager = DQNAgentManager(name="cim_remote_actor", - agent_id_list=agent_id_list, - mode=AgentMode.INFERENCE, - state_shaper=state_shaper, - action_shaper=action_shaper, - experience_shaper=experience_shaper, - explorer=explorer) - proxy_params = {"group_name": config.distributed.group_name, - "expected_peers": config.distributed.actor.peer, - "redis_address": (config.distributed.redis.host_name, config.distributed.redis.port) - } - actor_worker = ActorWorker(local_actor=SimpleActor(env=env, inference_agents=agent_manager), - proxy_params=proxy_params) - actor_worker.launch() - -On the learner side, an agent manager in AgentMode.TRAIN mode is required. However, it is not necessary to create shapers for an -agent manager in AgentMode.TRAIN mode (although a state shaper is created in this example so that the model input dimension can -be readily accessed). Instead of creating an actor, we create an actor proxy and wrap it inside the learner. This proxy -serves as the communication interface for the learner and is responsible for sending roll-out requests to remote actor -processes and receiving results. Calling the train method executes the usual training loop except that the actual -roll-out is performed remotely. The code snippet below shows the creation of a learner with an actor proxy wrapped -inside. - -.. code-block:: python - - agent_manager = DQNAgentManager(name="cim_remote_learner", agent_id_list=agent_id_list, mode=AgentMode.TRAIN, - state_shaper=state_shaper, explorer=explorer) - - proxy_params = {"group_name": config.distributed.group_name, - "expected_peers": config.distributed.learner.peer, - "redis_address": (config.distributed.redis.host_name, config.distributed.redis.port) - } - learner = SimpleLearner(trainable_agents=agent_manager, - actor=ActorProxy(proxy_params=proxy_params), - logger=Logger("distributed_cim_learner", auto_timestamp=False)) - learner.learn(total_episodes=config.general.total_training_episodes) - diff --git a/notebooks/container_inventory_management/rl_formulation.ipynb b/notebooks/container_inventory_management/rl_formulation.ipynb index 66783e3bd..705072c24 100644 --- a/notebooks/container_inventory_management/rl_formulation.ipynb +++ b/notebooks/container_inventory_management/rl_formulation.ipynb @@ -252,9 +252,9 @@ " set_seeds(64) # for reproducibility\n", " agent_dict = {}\n", " for agent_id in agent_id_list:\n", - " q_module = NNStack(\n", + " q_net = NNStack(\n", " \"q_value\",\n", - " [FullyConnectedBlock(\n", + " FullyConnectedBlock(\n", " input_dim=state_shaper.dim,\n", " hidden_dims=[256, 128, 64],\n", " output_dim=NUM_ACTIONS,\n", @@ -264,12 +264,10 @@ " softmax_enabled=False,\n", " skip_connection_enabled=False,\n", " dropout_p=.0)\n", - " ],\n", - " optimizer_options=OptimizerOptions(cls=RMSprop, params={\"lr\": 0.05})\n", " )\n", "\n", " algorithm = DQN(\n", - " model=LearningModel(q_module),\n", + " model=LearningModel(q_net, optimizer_options=OptimizerOptions(cls=RMSprop, params={\"lr\": 0.05})),\n", " config=DQNConfig(\n", " reward_decay=.0, \n", " target_update_frequency=5, \n", @@ -510,4 +508,4 @@ }, "nbformat": 4, "nbformat_minor": 4 -} +} \ No newline at end of file From d70814396eb4673cc5cde8508d02df1625727c27 Mon Sep 17 00:00:00 2001 From: ysqyang Date: Thu, 31 Dec 2020 20:15:06 +0800 Subject: [PATCH 316/337] updated cim example for policy optimization --- .../components/agent_manager.py | 31 +++++++++++-------- maro/rl/algorithms/policy_optimization.py | 6 ++-- 2 files changed, 21 insertions(+), 16 deletions(-) diff --git a/examples/cim/policy_optimization/components/agent_manager.py b/examples/cim/policy_optimization/components/agent_manager.py index db235ac00..669e8344d 100644 --- a/examples/cim/policy_optimization/components/agent_manager.py +++ b/examples/cim/policy_optimization/components/agent_manager.py @@ -6,7 +6,7 @@ from torch.optim import Adam, RMSprop from maro.rl import ( - AbsAgent, ActorCritic, ActorCriticConfig, FullyConnectedBlock, LearningModuleManager, LearningModule, + AbsAgent, ActorCritic, ActorCriticConfig, FullyConnectedBlock, LearningModel, NNStack, OptimizerOptions, PolicyGradient, PolicyOptimizationConfig, SimpleAgentManager ) from maro.utils import set_seeds @@ -22,41 +22,46 @@ def create_po_agents(agent_id_list, config): set_seeds(config.seed) agent_dict = {} for agent_id in agent_id_list: - actor_module = LearningModule( + actor_net = NNStack( "actor", - [FullyConnectedBlock( + FullyConnectedBlock( input_dim=input_dim, output_dim=num_actions, activation=nn.Tanh, is_head=True, **config.actor_model - )], - optimizer_options=OptimizerOptions(cls=Adam, params=config.actor_optimizer) + ) ) if config.type == "actor_critic": - critic_module = LearningModule( + critic_net = NNStack( "critic", - [FullyConnectedBlock( + FullyConnectedBlock( input_dim=config.input_dim, output_dim=1, activation=nn.LeakyReLU, is_head=True, **config.critic_model - )], - optimizer_options=OptimizerOptions(cls=RMSprop, params=config.critic_optimizer) + ) ) hyper_params = config.actor_critic_hyper_parameters hyper_params.update({"reward_discount": config.reward_discount}) + learning_model = LearningModel( + actor_net, critic_net, + optimizer_options={ + "actor": OptimizerOptions(cls=Adam, params=config.actor_optimizer) + "critic": OptimizerOptions(cls=RMSprop, params=config.critic_optimizer) + ) algorithm = ActorCritic( - LearningModuleManager(actor_module, critic_module), - ActorCriticConfig(critic_loss_func=nn.functional.smooth_l1_loss, **hyper_params) + learning_model, ActorCriticConfig(critic_loss_func=nn.SmoothL1Loss, **hyper_params) ) else: - algorithm = PolicyGradient( - LearningModuleManager(actor_module), PolicyOptimizationConfig(config.reward_discount) + learning_model = LearningModel( + actor_net, + optimizer_options=OptimizerOptions(cls=Adam, params=config.actor_optimizer) ) + algorithm = PolicyGradient(learning_model, PolicyOptimizationConfig(config.reward_discount)) agent_dict[agent_id] = POAgent(name=agent_id, algorithm=algorithm) diff --git a/maro/rl/algorithms/policy_optimization.py b/maro/rl/algorithms/policy_optimization.py index f4a36d118..b265459d5 100644 --- a/maro/rl/algorithms/policy_optimization.py +++ b/maro/rl/algorithms/policy_optimization.py @@ -8,7 +8,7 @@ import torch from maro.rl.algorithms.abs_algorithm import AbsAlgorithm -from maro.rl.models.learning_model import LearningModuleManager +from maro.rl.models.learning_model import LearningModel from maro.rl.utils.trajectory_utils import get_lambda_returns, get_truncated_cumulative_reward ActionInfo = namedtuple("ActionInfo", ["action", "log_probability"]) @@ -119,11 +119,11 @@ class ActorCritic(PolicyOptimization): The Actor-Critic algorithm base on the policy gradient theorem. Args: - model (LearningModuleManager): Multi-task model that computes action distributions and state values. + model (LearningModel): Multi-task model that computes action distributions and state values. It may or may not have a shared bottom stack. config: Configuration for the AC algorithm. """ - def __init__(self, model: LearningModuleManager, config: ActorCriticConfig): + def __init__(self, model: LearningModel, config: ActorCriticConfig): self.validate_task_names(model.task_names, {"actor", "critic"}) super().__init__(model, config) From da171e2593328032d2d047aaeac1be9b71d5f178 Mon Sep 17 00:00:00 2001 From: ysqyang Date: Thu, 31 Dec 2020 20:21:32 +0800 Subject: [PATCH 317/337] typo fix --- maro/rl/models/learning_model.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/maro/rl/models/learning_model.py b/maro/rl/models/learning_model.py index ac448a91f..0f46a171b 100644 --- a/maro/rl/models/learning_model.py +++ b/maro/rl/models/learning_model.py @@ -67,10 +67,6 @@ class LearningModel(nn.Module): optimize all parameters of the model. If it is a dictionary, for each `(key, value)` pair, an optimizer specified by `value` will be created for the internal stack named `key`. Defaults to None. """ -<<<<<<< HEAD - def __init__(self, *task_modules: LearningModule, shared_module: LearningModule = None): - self.validate_dims(*task_modules, shared_module=shared_module) -======= def __init__( self, *task_stacks: NNStack, @@ -78,7 +74,6 @@ def __init__( optimizer_options: Union[OptimizerOptions, Dict[str, OptimizerOptions]] = None ): self.validate_dims(*task_stacks, shared_stack=shared_stack) ->>>>>>> v0.2_learning_model_refinement super().__init__() self._stack_dict = {stack.name: stack for stack in task_stacks} # shared stack From 97c9bf5f69f179e2bc3e7bae90a87180465084e3 Mon Sep 17 00:00:00 2001 From: ysqyang Date: Thu, 31 Dec 2020 20:23:44 +0800 Subject: [PATCH 318/337] typo fix --- examples/cim/policy_optimization/components/agent_manager.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/examples/cim/policy_optimization/components/agent_manager.py b/examples/cim/policy_optimization/components/agent_manager.py index 669e8344d..7e651408e 100644 --- a/examples/cim/policy_optimization/components/agent_manager.py +++ b/examples/cim/policy_optimization/components/agent_manager.py @@ -51,7 +51,8 @@ def create_po_agents(agent_id_list, config): actor_net, critic_net, optimizer_options={ "actor": OptimizerOptions(cls=Adam, params=config.actor_optimizer) - "critic": OptimizerOptions(cls=RMSprop, params=config.critic_optimizer) + "critic": OptimizerOptions(cls=RMSprop, params=config.critic_optimizer) + } ) algorithm = ActorCritic( learning_model, ActorCriticConfig(critic_loss_func=nn.SmoothL1Loss, **hyper_params) From 4f5df18cc648c7671d985eac31e758cf0e4fa79c Mon Sep 17 00:00:00 2001 From: ysqyang Date: Thu, 31 Dec 2020 20:26:35 +0800 Subject: [PATCH 319/337] typo fix --- examples/cim/policy_optimization/components/agent_manager.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/cim/policy_optimization/components/agent_manager.py b/examples/cim/policy_optimization/components/agent_manager.py index 7e651408e..a8eb7de1d 100644 --- a/examples/cim/policy_optimization/components/agent_manager.py +++ b/examples/cim/policy_optimization/components/agent_manager.py @@ -50,7 +50,7 @@ def create_po_agents(agent_id_list, config): learning_model = LearningModel( actor_net, critic_net, optimizer_options={ - "actor": OptimizerOptions(cls=Adam, params=config.actor_optimizer) + "actor": OptimizerOptions(cls=Adam, params=config.actor_optimizer), "critic": OptimizerOptions(cls=RMSprop, params=config.critic_optimizer) } ) From 4472d7cee950b9c6a3b5b66a2e4db8795a080b75 Mon Sep 17 00:00:00 2001 From: ysqyang Date: Thu, 31 Dec 2020 20:35:13 +0800 Subject: [PATCH 320/337] typo fix --- examples/cim/policy_optimization/components/agent_manager.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/cim/policy_optimization/components/agent_manager.py b/examples/cim/policy_optimization/components/agent_manager.py index a8eb7de1d..6ea9cda8e 100644 --- a/examples/cim/policy_optimization/components/agent_manager.py +++ b/examples/cim/policy_optimization/components/agent_manager.py @@ -55,7 +55,7 @@ def create_po_agents(agent_id_list, config): } ) algorithm = ActorCritic( - learning_model, ActorCriticConfig(critic_loss_func=nn.SmoothL1Loss, **hyper_params) + learning_model, ActorCriticConfig(critic_loss_func=nn.SmoothL1Loss(), **hyper_params) ) else: learning_model = LearningModel( From c65e2b319a90e34b0ca140c866c519d937cba4bc Mon Sep 17 00:00:00 2001 From: ysqyang Date: Fri, 1 Jan 2021 00:06:11 +0800 Subject: [PATCH 321/337] misc edits --- examples/cim/dqn/components/agent.py | 3 +-- examples/cim/dqn/components/agent_manager.py | 9 ++++++--- examples/cim/dqn/config.yml | 4 ++-- examples/cim/dqn/dist_actor.py | 2 +- examples/cim/dqn/dist_learner.py | 4 ++-- examples/cim/dqn/single_process_launcher.py | 2 +- maro/rl/algorithms/dqn.py | 10 +++++----- maro/rl/shaping/k_step_experience_shaper.py | 14 +++++++------- .../rl_formulation.ipynb | 2 +- 9 files changed, 26 insertions(+), 24 deletions(-) diff --git a/examples/cim/dqn/components/agent.py b/examples/cim/dqn/components/agent.py index b73768b8e..4b08a98a9 100644 --- a/examples/cim/dqn/components/agent.py +++ b/examples/cim/dqn/components/agent.py @@ -6,7 +6,7 @@ import numpy as np -from maro.rl import AbsAgent, EpsilonGreedyExplorer, ColumnBasedStore +from maro.rl import AbsAgent, ColumnBasedStore class CIMAgent(AbsAgent): @@ -15,7 +15,6 @@ class CIMAgent(AbsAgent): Args: name (str): Agent's name. algorithm (AbsAlgorithm): A concrete algorithm instance that inherits from AbstractAlgorithm. - explorer (AbsExplorer): Explorer instance to generate exploratory actions. experience_pool (AbsStore): It is used to store experiences processed by the experience shaper, which will be used by some value-based algorithms, such as DQN. min_experiences_to_train: minimum number of experiences required for training. diff --git a/examples/cim/dqn/components/agent_manager.py b/examples/cim/dqn/components/agent_manager.py index 94afef408..23cf6b49f 100644 --- a/examples/cim/dqn/components/agent_manager.py +++ b/examples/cim/dqn/components/agent_manager.py @@ -28,10 +28,13 @@ def create_dqn_agents(agent_id_list, config): **config.algorithm.model ) ) - + learning_model = LearningModel( + q_net, + optimizer_options=OptimizerOptions(cls=RMSprop, params=config.algorithm.optimizer) + ) algorithm = DQN( - LearningModel(q_net, optimizer_options=OptimizerOptions(cls=RMSprop, params=config.algorithm.optimizer)), - DQNConfig(**config.algorithm.config, loss_cls=nn.SmoothL1Loss) + learning_model, + DQNConfig(**config.algorithm.hyper_params, loss_cls=nn.SmoothL1Loss) ) agent_dict[agent_id] = CIMAgent( agent_id, algorithm, ColumnBasedStore(**config.experience_pool), diff --git a/examples/cim/dqn/config.yml b/examples/cim/dqn/config.yml index e3564c570..db91e4faf 100644 --- a/examples/cim/dqn/config.yml +++ b/examples/cim/dqn/config.yml @@ -33,8 +33,8 @@ agents: dropout_p: 0.0 optimizer: lr: 0.05 - config: - reward_decay: .0 + hyper_params: + reward_discount: .0 target_update_frequency: 5 tau: 0.1 is_double: true diff --git a/examples/cim/dqn/dist_actor.py b/examples/cim/dqn/dist_actor.py index 4376ffae6..095636ca5 100644 --- a/examples/cim/dqn/dist_actor.py +++ b/examples/cim/dqn/dist_actor.py @@ -26,7 +26,7 @@ def launch(config, distributed_config): config["agents"]["algorithm"]["input_dim"] = state_shaper.dim agent_manager = DQNAgentManager( - name="distributed_cim_actor", + name="cim_actor", mode=AgentManagerMode.INFERENCE, agent_dict=create_dqn_agents(agent_id_list, config.agents), state_shaper=state_shaper, diff --git a/examples/cim/dqn/dist_learner.py b/examples/cim/dqn/dist_learner.py index 70082ba26..29c27a21a 100644 --- a/examples/cim/dqn/dist_learner.py +++ b/examples/cim/dqn/dist_learner.py @@ -20,7 +20,7 @@ def launch(config, distributed_config): config["agents"]["algorithm"]["input_dim"] = CIMStateShaper(**config.env.state_shaping).dim agent_manager = DQNAgentManager( - name="distributed_cim_learner", + name="cim_learner", mode=AgentManagerMode.TRAIN, agent_dict=create_dqn_agents(agent_id_list, config.agents) ) @@ -38,7 +38,7 @@ def launch(config, distributed_config): agent_manager=agent_manager, actor=ActorProxy(proxy_params=proxy_params, experience_collecting_func=concat_experiences_by_agent), scheduler=TwoPhaseLinearParameterScheduler(config.main_loop.max_episode, **config.main_loop.exploration), - logger=Logger("distributed_cim_learner", auto_timestamp=False) + logger=Logger("cim_learner", auto_timestamp=False) ) learner.learn() learner.test() diff --git a/examples/cim/dqn/single_process_launcher.py b/examples/cim/dqn/single_process_launcher.py index 91ed1cbb4..aca2cadfd 100644 --- a/examples/cim/dqn/single_process_launcher.py +++ b/examples/cim/dqn/single_process_launcher.py @@ -42,7 +42,7 @@ def launch(config): actor = SimpleActor(env, agent_manager) learner = SimpleLearner( agent_manager, actor, scheduler, - logger=Logger("single_host_cim_learner", format_=LogFormat.simple, auto_timestamp=False) + logger=Logger("cim_learner", format_=LogFormat.simple, auto_timestamp=False) ) learner.learn() learner.test() diff --git a/maro/rl/algorithms/dqn.py b/maro/rl/algorithms/dqn.py index ea675fb43..fa004c9f6 100644 --- a/maro/rl/algorithms/dqn.py +++ b/maro/rl/algorithms/dqn.py @@ -14,7 +14,7 @@ class DQNConfig: """Configuration for the DQN algorithm. Args: - reward_decay (float): Reward decay as defined in standard RL terminology. + reward_discount (float): Reward decay as defined in standard RL terminology. loss_cls: Loss function class for evaluating TD errors. target_update_frequency (int): Number of training rounds between target model updates. epsilon (float): Exploration rate for epsilon-greedy exploration. Defaults to None. @@ -28,13 +28,13 @@ class DQNConfig: method. Defaults to False. """ __slots__ = [ - "reward_decay", "loss_func", "target_update_frequency", "epsilon", "tau", "is_double", "advantage_mode", + "reward_discount", "loss_func", "target_update_frequency", "epsilon", "tau", "is_double", "advantage_mode", "per_sample_td_error_enabled" ] def __init__( self, - reward_decay: float, + reward_discount: float, loss_cls, target_update_frequency: int, epsilon: float = .0, @@ -43,7 +43,7 @@ def __init__( advantage_mode: str = None, per_sample_td_error_enabled: bool = False ): - self.reward_decay = reward_decay + self.reward_discount = reward_discount self.target_update_frequency = target_update_frequency self.epsilon = epsilon self.tau = tau @@ -117,7 +117,7 @@ def _compute_td_errors(self, states, actions, rewards, next_states): current_q_values_for_all_actions = self._get_q_values(self._model, states) current_q_values = current_q_values_for_all_actions.gather(1, actions).squeeze(1) # (N,) next_q_values = self._get_next_q_values(current_q_values_for_all_actions, next_states) # (N,) - target_q_values = (rewards + self._config.reward_decay * next_q_values).detach() # (N,) + target_q_values = (rewards + self._config.reward_discount * next_q_values).detach() # (N,) return self._config.loss_func(current_q_values, target_q_values) def train(self, states: np.ndarray, actions: np.ndarray, rewards: np.ndarray, next_states: np.ndarray): diff --git a/maro/rl/shaping/k_step_experience_shaper.py b/maro/rl/shaping/k_step_experience_shaper.py index 1ffa65722..332cf0418 100644 --- a/maro/rl/shaping/k_step_experience_shaper.py +++ b/maro/rl/shaping/k_step_experience_shaper.py @@ -23,13 +23,13 @@ class KStepExperienceShaper(ExperienceShaper): Args: reward_func (Callable): a function used to compute immediate rewards from metrics given by the env. - reward_decay (float): decay factor used to evaluate multi-step returns. + reward_discount (float): decay factor used to evaluate multi-step returns. steps (int): number of time steps used in computing returns is_per_agent (bool): if True, the generated experiences will be bucketed by agent ID. """ - def __init__(self, reward_func: Callable, reward_decay: float, steps: int, is_per_agent: bool = True): + def __init__(self, reward_func: Callable, reward_discount: float, steps: int, is_per_agent: bool = True): super().__init__(reward_func) - self._reward_decay = reward_decay + self._reward_discount = reward_discount self._steps = steps self._is_per_agent = is_per_agent @@ -42,11 +42,11 @@ def __call__(self, trajectory, snapshot_list): next_transition = trajectory[min(len(trajectory) - 1, i + self._steps)] reward_list.appendleft(self._reward_func(trajectory[i]["metrics"])) # compute the full return - full_return = full_return * self._reward_decay + reward_list[0] + full_return = full_return * self._reward_discount + reward_list[0] # compute the partial return - partial_return = partial_return * self._reward_decay + reward_list[0] + partial_return = partial_return * self._reward_discount + reward_list[0] if len(reward_list) > self._steps: - partial_return -= reward_list.pop() * self._reward_decay ** (self._steps - 1) + partial_return -= reward_list.pop() * self._reward_discount ** (self._steps - 1) agent_exp = experiences[transition["agent_id"]] if self._is_per_agent else experiences agent_exp[KStepExperienceKeys.STATE.value].appendleft(transition["state"]) agent_exp[KStepExperienceKeys.ACTION.value].appendleft(transition["action"]) @@ -55,7 +55,7 @@ def __call__(self, trajectory, snapshot_list): agent_exp[KStepExperienceKeys.NEXT_STATE.value].appendleft(next_transition["state"]) agent_exp[KStepExperienceKeys.NEXT_ACTION.value].appendleft(next_transition["action"]) agent_exp[KStepExperienceKeys.DISCOUNT.value].appendleft( - self._reward_decay ** (min(self._steps, len(trajectory) - 1 - i)) + self._reward_discount ** (min(self._steps, len(trajectory) - 1 - i)) ) return dict(experiences) diff --git a/notebooks/container_inventory_management/rl_formulation.ipynb b/notebooks/container_inventory_management/rl_formulation.ipynb index 705072c24..f564699aa 100644 --- a/notebooks/container_inventory_management/rl_formulation.ipynb +++ b/notebooks/container_inventory_management/rl_formulation.ipynb @@ -269,7 +269,7 @@ " algorithm = DQN(\n", " model=LearningModel(q_net, optimizer_options=OptimizerOptions(cls=RMSprop, params={\"lr\": 0.05})),\n", " config=DQNConfig(\n", - " reward_decay=.0, \n", + " reward_discount=.0, \n", " target_update_frequency=5, \n", " tau=0.1, \n", " is_double=True, \n", From 238d43287719691ca3c4b398a7d0cc2aefc0c46a Mon Sep 17 00:00:00 2001 From: ysqyang Date: Tue, 5 Jan 2021 14:40:25 +0800 Subject: [PATCH 322/337] minor edits to rl_toolkit.rst --- docs/source/key_components/rl_toolkit.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/key_components/rl_toolkit.rst b/docs/source/key_components/rl_toolkit.rst index ce3824bb2..e4c809be9 100644 --- a/docs/source/key_components/rl_toolkit.rst +++ b/docs/source/key_components/rl_toolkit.rst @@ -25,7 +25,7 @@ Learner and Actor def learn(self, total_episodes): for exploration_params in self._scheduler: performance, exp_by_agent = self._actor.roll_out( - model_dict=None if self._is_shared_agent_instance() else self._agent_manager.dump_models(), + self._agent_manager.dump_models(), exploration_params=exploration_params ) self._scheduler.record_performance(performance) From 806ec1d53b31c02e32caef9910c6444b84cf2184 Mon Sep 17 00:00:00 2001 From: ysqyang Date: Tue, 5 Jan 2021 14:41:32 +0800 Subject: [PATCH 323/337] checked out docs from master --- docs/source/key_components/rl_toolkit.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/key_components/rl_toolkit.rst b/docs/source/key_components/rl_toolkit.rst index e4c809be9..57b64f182 100644 --- a/docs/source/key_components/rl_toolkit.rst +++ b/docs/source/key_components/rl_toolkit.rst @@ -22,7 +22,7 @@ Learner and Actor .. code-block:: python # Train function of learner. - def learn(self, total_episodes): + def learn(self): for exploration_params in self._scheduler: performance, exp_by_agent = self._actor.roll_out( self._agent_manager.dump_models(), From 89f1d7ceede294b6ffee8b5a79b28f08d8671a7e Mon Sep 17 00:00:00 2001 From: ysqyang Date: Tue, 5 Jan 2021 14:44:09 +0800 Subject: [PATCH 324/337] fixed typo in k-step shaper --- maro/rl/shaping/k_step_experience_shaper.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/maro/rl/shaping/k_step_experience_shaper.py b/maro/rl/shaping/k_step_experience_shaper.py index 826f85b64..332cf0418 100644 --- a/maro/rl/shaping/k_step_experience_shaper.py +++ b/maro/rl/shaping/k_step_experience_shaper.py @@ -39,7 +39,7 @@ def __call__(self, trajectory, snapshot_list): full_return = partial_return = 0 for i in range(len(trajectory) - 2, -1, -1): transition = trajectory[i] - next_transition = trajectory[min(len(trajectory) - 1, i + self._num_steps)] + next_transition = trajectory[min(len(trajectory) - 1, i + self._steps)] reward_list.appendleft(self._reward_func(trajectory[i]["metrics"])) # compute the full return full_return = full_return * self._reward_discount + reward_list[0] From 3ec712c8f8398640e73a2246c049ea92125f2aa9 Mon Sep 17 00:00:00 2001 From: ysqyang Date: Tue, 5 Jan 2021 14:53:17 +0800 Subject: [PATCH 325/337] fixed lint issues --- maro/rl/__init__.py | 4 ++-- maro/rl/algorithms/__init__.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/maro/rl/__init__.py b/maro/rl/__init__.py index afdcc77e2..75b348374 100644 --- a/maro/rl/__init__.py +++ b/maro/rl/__init__.py @@ -4,7 +4,7 @@ from maro.rl.actor import AbsActor, SimpleActor from maro.rl.agent import AbsAgent, AbsAgentManager, AgentManagerMode, SimpleAgentManager from maro.rl.algorithms import ( - AbsAlgorithm, ActionInfo, ActorCritic, ActorCriticConfig, DQN, DQNConfig, PolicyGradient, PolicyOptimization, + DQN, AbsAlgorithm, ActionInfo, ActorCritic, ActorCriticConfig, DQN, DQNConfig, PolicyGradient, PolicyOptimization, PolicyOptimizationConfig ) from maro.rl.dist_topologies import ( @@ -22,7 +22,7 @@ __all__ = [ "AbsActor", "SimpleActor", "AbsAgent", "AbsAgentManager", "AgentManagerMode", "SimpleAgentManager", - "AbsAlgorithm", "ActionInfo", "ActorCritic", "ActorCriticConfig", "DQN", "DQNConfig", "PolicyGradient", + "AbsAlgorithm", "ActionInfo", "ActorCritic", "ActorCriticConfig", "DQN", "DQNConfig", "PolicyGradient", "PolicyOptimization", "PolicyOptimizationConfig", "ActorProxy", "ActorWorker", "concat_experiences_by_agent", "merge_experiences_with_trajectory_boundaries", "AbsExplorer", "EpsilonGreedyExplorer", "GaussianNoiseExplorer", "NoiseExplorer", "UniformNoiseExplorer", diff --git a/maro/rl/algorithms/__init__.py b/maro/rl/algorithms/__init__.py index fd9989446..d508eb20c 100644 --- a/maro/rl/algorithms/__init__.py +++ b/maro/rl/algorithms/__init__.py @@ -8,8 +8,8 @@ ) __all__ = [ - "AbsAlgorithm", + "AbsAlgorithm", "DQN", "DQNConfig", - "ActionInfo", "ActorCritic", "ActorCriticConfig", "PolicyGradient", "PolicyOptimization", + "ActionInfo", "ActorCritic", "ActorCriticConfig", "PolicyGradient", "PolicyOptimization", "PolicyOptimizationConfig" ] From 3d4758aaf833934054c9de5906a113b986923047 Mon Sep 17 00:00:00 2001 From: ysqyang Date: Tue, 5 Jan 2021 14:56:46 +0800 Subject: [PATCH 326/337] bug fix in store --- maro/rl/storage/column_based_store.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/maro/rl/storage/column_based_store.py b/maro/rl/storage/column_based_store.py index ed4c29491..20d29c12a 100644 --- a/maro/rl/storage/column_based_store.py +++ b/maro/rl/storage/column_based_store.py @@ -192,7 +192,7 @@ def sample(self, size, weights: Union[list, np.ndarray] = None, replace: bool = """ if weights is not None: weights = np.asarray(weights) - weights /= np.sum(weights) + weights = weights / np.sum(weights) indexes = np.random.choice(self._size, size=size, replace=replace, p=weights) return indexes, self.get(indexes) From 8f8ee611608ca827deeb81ff8e78867f8aec0fd1 Mon Sep 17 00:00:00 2001 From: ysqyang Date: Tue, 5 Jan 2021 14:57:55 +0800 Subject: [PATCH 327/337] lint issue fix --- maro/rl/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/maro/rl/__init__.py b/maro/rl/__init__.py index 75b348374..17a3aacb3 100644 --- a/maro/rl/__init__.py +++ b/maro/rl/__init__.py @@ -4,7 +4,7 @@ from maro.rl.actor import AbsActor, SimpleActor from maro.rl.agent import AbsAgent, AbsAgentManager, AgentManagerMode, SimpleAgentManager from maro.rl.algorithms import ( - DQN, AbsAlgorithm, ActionInfo, ActorCritic, ActorCriticConfig, DQN, DQNConfig, PolicyGradient, PolicyOptimization, + DQN, AbsAlgorithm, ActionInfo, ActorCritic, ActorCriticConfig, DQNConfig, PolicyGradient, PolicyOptimization, PolicyOptimizationConfig ) from maro.rl.dist_topologies import ( From 0d669f5cc3a90207ea9dd848156139c4d9b93980 Mon Sep 17 00:00:00 2001 From: ysqyang Date: Tue, 5 Jan 2021 07:04:50 +0000 Subject: [PATCH 328/337] changed default max_ep to 100 for policy_optimization algos --- examples/cim/policy_optimization/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/cim/policy_optimization/config.yml b/examples/cim/policy_optimization/config.yml index 6243af934..267633cd4 100644 --- a/examples/cim/policy_optimization/config.yml +++ b/examples/cim/policy_optimization/config.yml @@ -11,7 +11,7 @@ env: shortage_factor: 1.0 time_decay_factor: 0.97 main_loop: - max_episode: 500 + max_episode: 100 early_stopping: warmup_ep: 20 last_k: 5 From d4bb9d55acecb87ac693a067804a38914243025b Mon Sep 17 00:00:00 2001 From: Meroy Chen <39452768+Meroy9819@users.noreply.github.com> Date: Tue, 5 Jan 2021 21:29:29 +0800 Subject: [PATCH 329/337] vis doc update to master (#244) * refine readme * feat: refine data push/pull (#138) * feat: refine data push/pull * test: add cli provision testing * fix: style fix * fix: add necessary comments * fix: from code review * add fall back function in weather download (#112) * fix deployment issue in multi envs * fix typo * fix ~/.maro not exist issue in build * skip deploy when build * update for comments * temporarily disable weather info * replace ecr with cim in setup.py * replace ecr in manifest * remove weather check when read data * fix station id issue * fix format * add TODO in comments * add noaa weather source * fix weather reset and weather comment * add comment for weather data url * some format update * add fall back function in weather download * update comment * update for comments * update comment * add period * fix for pylint * update for pylint check * added example docs (#136) * added example docs * added citibike greedy example doc * modified citibike doc * fixed PR comments * fixed more PR comments * fixed small formatting issue Co-authored-by: ysqyang * switch the key and value of handler_dict in decorator (#144) * switch the key and value of handler_dict in decorator * add dist decorator UT and fixed multithreading conflict in maro test suite * pr comments update. * resolved comments about decorator UT * rename handler_fun in dist decorator * change self.attr into class_name.attr * update UT tests comments * V0.1 annotation (#147) * refine the annotation of simulator core * remove reward from env(be) * format refined * white spaces test * left-padding spaces refined * format modifed * update the left-padding spaces of docstrings * code format updated * update according to comments * update according to PR comments Co-authored-by: Jinyu Wang * Event payload details for env.summary (#156) * key_list of events added for env.summary * code refined according to lint * 2 kinds of Payload added for CIM scenario; citi bike summary refined according to comments * code format refined * try trigger the git tests * update github workflow Co-authored-by: Jinyu Wang * Implemented dump snapshots and convert to CSV. * Let BE supports params when dump snapshot. * Refactor dump code to core.py * Implemented decision event dump. * V0.2 online lp for citi bike (#159) * key_list of events added for env.summary * code refined according to lint * 2 kinds of Payload added for CIM scenario; citi bike summary refined according to comments * code format refined * try trigger the git tests * update github workflow * online LP example added for citi bike * infeasible solution * infeasible solution fixed: call snapshot before any env.step() * experiment results of toy topos added * experiment results of toy topos added * experiment result update: better than naive baseline * PuLP version added * greedy experiment results update * citibike result update * modified according to PR comments * update experiment results and forecasting comparison * citi bike lp README updated * README updated * modified according to PR comments * update according to PR comments Co-authored-by: Jinyu Wang Co-authored-by: Jinyu Wang * V0.2 rl toolkit refinement (#165) * refined rl abstractions * fixed formattin issues * checked out error-code related code from v0.2_pg * fixed a bug * fixed a bug * fixed a bug * fixed a bug * fixed a bug * fixed a bug * fixed a bug * fixed a bug * fixed a bug * fixed a bug * fixed a bug * fixed a bug * renamed save_models to dump_models * 1. set default batch_norm_enabled to True; 2. used state_dict in dqn model saving * renamed dump_experience_store to dump_experience_pool * fixed a bug in the dump_experience_pool method * fixed some PR comments * fixed more PR comments * 1.fixed some PR comments; 2.added early_stopping_checker; 3.revised explorer class * fixed cim example according to rl toolkit changes * fixed some more PR comments * rewrote multi_process_launcher to eliminate the distributed section in config * 1. fixed a typo; 2. added logging before early stopping * fixed a bug * fixed a bug * fixed a bug * added early stopping feature to CIM exmaple * fixed a typo * fixed some issues with early stopping * changed early stopping metric func * fixed a bug * fixed a bug * added early stopping to dist mode cim * added experience collecting func * edited notebook according to changes in CIM example * fixed bugs in nb * fixed lint formatting issues * fixed a typo * fixed some PR comments * fixed more PR comments * revised docs * removed nb output * fixed a bug in simple_learner * fixed a typo in nb * fixed a bug * fixed a bug * fixed a bug * removed unused import * fixed a bug * 1. changed early stopping default config; 2. renamed param in early stopping checker and added typing * fixed some doc issues * added output to nb Co-authored-by: ysqyang * replace is not '' with !='' * Fixed issues that code review mentioned. * removed path from hello.py * Changed import sort. * Fix import sorting in citi_bike/business_engine * visualization 0.1 * Updated lint configurations. * Fixed formatting error that caused lint errors. * render html title function * Try to fix lint errors. * flake-8 style fix * remove space around 18,35 * dump_csv_converter.py re-formatting. * files re-formatting. * style fixed * tab delete * white space fix * white space fix-2 * vis redundant function delete * refine * update according to flake8 * re-formatting after merged upstream. * Updated import section. * Updated import section. * V0.2 Logical operator overloading for EarlyStoppingChecker (#178) * 1. added logical operator overloading for early stopping checker; 2. added mean value checker * fixed PR comments * removed learner.exit() in single_process_launcher * added another early stopping checker in example * fixed PR comments and lint issues * lint issue fix * fixed lint issues * fixed a bug * fixed a bug Co-authored-by: ysqyang * V0.2 skip connection (#176) * replaced IdentityLayers with nn.Identity * 1. added skip connection option in FC_net; 2. generalized learning model * added skip_connection option in config * removed type casting in fc_net * fixed lint formatting issues * refined docstring * added multi-head functionality to LearningModel * refined learning model docstring * added head_key param in learningModel forward * fixed PR comments * added top layer logic and is_top option in fc_net * fixed a bug * fixed a bug * reverted some changes in learning model * reverted some changes in learning model * added members to learning model to fix the mode issue * fixed a bug * fixed mode setting issue in learning model * removed learner.exit() in single_process_launcher * fixed PR comments * fixed rl/__init__ * fixed issues in example * fixed a bug * fixed a bug * fixed lint formatting issues * moved reward type casting to exp shaper Co-authored-by: ysqyang * pr refine * isort fix * white space * lint error * \n error * test continuation * indent * continuation of indent * indent 0.3 * comment update * comment update 0.2 * f-string update * f-string 0.2 * lint 0.3 * lint 0.4 * lint 0.4 * lint 0.5 * lint 0.6 * docstring update * data version deploy update * condition update * add whitespace * V0.2 vis dump feature enhancement. (#190) * Dumps added manifest file. * Code updated format by flake8 * Changed manifest file format for easy reading. * deploy info update; docs update * weird white space * Update dashboard_visualization.md * new endline? * delete dependency * delete irrelevant file * change scenario to enum, divide file path into a separated class * fixed a bug in learner's test() (#193) Co-authored-by: ysqyang * V0.2 double dqn (#188) * added dueling action value model * renamed params in dueling_action_value_model * renamed shared_features to features * replaced IdentityLayers with nn.Identity * 1. added skip connection option in FC_net; 2. generalized learning model * added skip_connection option in config * removed type casting in fc_net * fixed lint formatting issues * refined docstring * mv dueling_actiovalue_model and fixed some bugs * added multi-head functionality to LearningModel * refined learning model docstring * added head_key param in learningModel forward * added double DQN and dueling features to DQN * fixed a bug * added DuelingQModelHead enum * fixed a bug * removed unwanted file * fixed PR comments * added top layer logic and is_top option in fc_net * fixed a bug * fixed a bug * reverted some changes in learning model * reverted some changes in learning model * added members to learning model to fix the mode issue * fixed a bug * fixed mode setting issue in learning model * fixed PR comments * revised cim example according to DQN changes * renamed eval_model to q_value_model in cim example * more fixes * fixed a bug * fixed a bug * added doc per PR comments * removed learner.exit() in single_process_launcher * removed learner.exit() in single_process_launcher * fixed PR comments * fixed rl/__init__ * fixed issues in example * fixed a bug * fixed a bug * fixed lint formatting issues * double DQN feature * fixed a bug * fixed a bug * fixed PR comments * fixed lint issue * 1. fixed PR comments related to load/dump; 2. removed abstract load/dump methods from AbsAlgorithm * added load_models in simple_learner * minor docstring edits * minor docstring edits * set is_double to true in DQN config Co-authored-by: ysqyang Co-authored-by: Arthur Jiang * V0.2 feature predefined image (#183) * feat: support predefined image provision * style: fix linting errors * style: fix linting errors * style: fix linting errors * style: fix linting errors * fix: error scripts invocation after using relative import * fix: missing init.py * fixed a bug in learner's test() * feat: add distributed_config for dqn example * test: update test for grass * test: update test for k8s * feat: add promptings for steps * fix: change relative imports to absolute imports Co-authored-by: ysqyang Co-authored-by: Arthur Jiang * doc refine * doc update * params type * data structure update * doc&enum, formula refine * refine * add ut, refine doc * style refine * isort * strong type fix * os._exit delete * revert datalib * import new line * change test case * change file name & doc * change deploy path * delete params * revert file * delete duplicate file * delete single process * update naming * manually change import order * delete blank * edit error * requirement txt * style fix & refine * comments&docstring refine * add parameter name * test & dump * comments update * V0.2 feature proxy rejoin (#158) * update dist decorator * replace proxy.get_peers by proxy.peers * update proxy rejoin (draft, not runable for proxy rejoin) * fix bugs in proxy * add message cache, and redesign rejoin parameter * feat: add checkpoint with test * update proxy.rejoin * fixed rejoin bug, rename func * add test example(temp) * feat: add FaultToleranceAgent, refine other MasterAgents and NodeAgents. * capital env vari name * rm json.dumps; change retries to 10; temp add warning level for rejoin * fix: unable to load FaultToleranceAgent, missing params * fix: delete mapping in StopJob if FaultTolerance is activated, add exception handler for FaultToleranceAgent * feat: add node_id to node_details * fix: add a new dependency for tests * style: meet linting requirements * style: remaining linting problems * lint fixed; rm temp test folder. * fixed lint f-string without placeholder * fix: add a flag for "remove_container", refine restart logic and Redis keys naming * proxy rejoin update. * variable rename. * fixed lint issues * fixed lint issues * add exit code for different error * feat: add special errors handler * add max rejoin times * remove unused import * add rejoin UT; resolve rejoin comments * lint fixed * fixed UT import problem * rm MessageCache in proxy * fix: refine key naming * update proxy rejoin; add topic for broadcast * feat: support predefined image provision * update UT for communication * add docstring for rejoin * fixed isort and zmq driver import * fixed isort and UT test * fix isort issue * proxy rejoin update (comments v2) * fixed isort error * style: fix linting errors * style: fix linting errors * style: fix linting errors * style: fix linting errors * feat: add exists method for checkpoint * fix: error scripts invocation after using relative import * fix: missing init.py * fixed a bug in learner's test() * add driver close and socket SUB disconnect for rejoin * feat: add distributed_config for dqn example * test: update test for grass * test: update test for k8s * feat: add promptings for steps * fix: change relative imports to absolute imports * fixed comments and update logger level * mv driver in proxy.__init__ for issue temp fixed. * Update docstring and comments * style: fix code reviews problems * fix code format Co-authored-by: Lyuchun Huang Co-authored-by: ysqyang * V0.2 feature cli windows (#203) * fix: change local mkdir to os.makedirs * fix: add utf8 encoding for logger * fix: add powershell.exe prefix to subprocess functions * feat: add debug_green * fix: use fsutil to create fix-size files in Windows * fix: use universal_newlines=True to handle encoding problem in different operating systems * fix: use temp file to do copy when the operating system is not Linux * fix: linting error * fix: use fsutil in test_k8s.py * feat: dynamic init ABS_PATH in GlobalParams * fix: use -Command to execute Powershell command * fix: refine code style in k8s_azure_executor.py, add Windows support for k8s mode * fix: problems in code review * EventBuffer refine (#197) * merge uniform event changes back * 1st step: move executing events into stack for better removing performance * flush event pool * typo * add option for env to enable event pool * refine stack functions * fix comment issues, add typings * lint fixing * lint fix * add missing fix * linting * lint * use linked list instead original event list and execute stack * add missing file * linting, and fixes * add missing file * linting fix * fixing comments * add missing file * rename event_list to event_linked_list * correct import path * change enable_event_pool to disable_finished_events * add missing file * V0.2 merge master (#214) * fix the visualization of docs/key_components/distributed_toolkit * add examples into isort ignore * refine import path for examples (#195) * refine import path for examples * refine indents * fixed formatting issues * update code style * add editorconfig-checker, add editorconfig path into lint, change super-linter version * change path for code saving in cim.gnn Co-authored-by: Jinyu Wang Co-authored-by: ysqyang Co-authored-by: Wenlei Shi * fix issue that sometimes there is conflict between distutils and setuptools (#208) * fix issue that cython and setuptools conflict * follow the accepted temp workaround * update comment, it should be conflict between setuptools and distutils * fixed bugs related to proxy interface changes Co-authored-by: Jinyu Wang Co-authored-by: Jinyu-W <53509467+Jinyu-W@users.noreply.github.com> Co-authored-by: ysqyang Co-authored-by: Wenlei Shi Co-authored-by: Chaos Yu * typo fix * Bug fix: event buffer issue that cause Actions cannot be passed into business engine (#215) * bug fix * clear the reference after extract sub events, update ut to cover this issue Co-authored-by: Jinyu-W <53509467+Jinyu-W@users.noreply.github.com> * fix flake8 style problem * V0.2 feature refine mode namings (#212) * feat: refine cli exception * feat: refine mode namings * EventBuffer refine (#197) * merge uniform event changes back * 1st step: move executing events into stack for better removing performance * flush event pool * typo * add option for env to enable event pool * refine stack functions * fix comment issues, add typings * lint fixing * lint fix * add missing fix * linting * lint * use linked list instead original event list and execute stack * add missing file * linting, and fixes * add missing file * linting fix * fixing comments * add missing file * rename event_list to event_linked_list * correct import path * change enable_event_pool to disable_finished_events * add missing file * fixed bugs in dist rl * feat: rename files * tests: set longer gracefully wait time * style: fix linting errors * style: fix linting errors * style: fix linting errors * fix: rm redundant variables * fix: refine error message Co-authored-by: Chaos Yu Co-authored-by: ysqyang * V0.2 vis new (#210) Co-authored-by: Wenlei Shi Co-authored-by: Chaos Yu * V0.2 local host process (#221) * Update local process (not ready) * update cli process mode * add setup/clear/template for maro process * fix process stop * add logger and rename parameters * add logger for setup/clear * fixed close not exist pid when given pid list. * Fixed comments and rename setup/clear with create/delete * update ProcessInternalError * comments fix * delete toolkit change * doc update * citi bike update * deploy path * datalib update * revert datalib * revert * maro file format * comments update * doc update * V0.2 grass on premises (#220) * feat: refine cli exception * commit on v0.2_grass_on_premises Co-authored-by: Lyuchun Huang Co-authored-by: Chaos Yu Co-authored-by: ysqyang * V0.2 vm scheduling scenario (#189) * Initialize * Data center scenario init * Code style modification * V0.2 event buffer subevents expand (#180) * V0.2 rl toolkit refinement (#165) * refined rl abstractions * fixed formattin issues * checked out error-code related code from v0.2_pg * fixed a bug * fixed a bug * fixed a bug * fixed a bug * fixed a bug * fixed a bug * fixed a bug * fixed a bug * fixed a bug * fixed a bug * fixed a bug * fixed a bug * renamed save_models to dump_models * 1. set default batch_norm_enabled to True; 2. used state_dict in dqn model saving * renamed dump_experience_store to dump_experience_pool * fixed a bug in the dump_experience_pool method * fixed some PR comments * fixed more PR comments * 1.fixed some PR comments; 2.added early_stopping_checker; 3.revised explorer class * fixed cim example according to rl toolkit changes * fixed some more PR comments * rewrote multi_process_launcher to eliminate the distributed section in config * 1. fixed a typo; 2. added logging before early stopping * fixed a bug * fixed a bug * fixed a bug * added early stopping feature to CIM exmaple * fixed a typo * fixed some issues with early stopping * changed early stopping metric func * fixed a bug * fixed a bug * added early stopping to dist mode cim * added experience collecting func * edited notebook according to changes in CIM example * fixed bugs in nb * fixed lint formatting issues * fixed a typo * fixed some PR comments * fixed more PR comments * revised docs * removed nb output * fixed a bug in simple_learner * fixed a typo in nb * fixed a bug * fixed a bug * fixed a bug * removed unused import * fixed a bug * 1. changed early stopping default config; 2. renamed param in early stopping checker and added typing * fixed some doc issues * added output to nb Co-authored-by: ysqyang * unfold sub-events, insert after parent * remove event category, use different class instead, add helper functions to gen decision and action event * add a method to support add immediate event to cascade event with tick validation * fix ut issue * add action as 1st sub event to ensure the executing order Co-authored-by: ysqyang Co-authored-by: ysqyang * Data center scenario update * Code style update * Data scenario business engine update * Isort update * Fix lint code check * Fix based on PR comments. * Update based on PR comments. * Add decision payload * Add config file * Update utilization series logic * Update based on PR comment * Update based on PR * Update * Update * Add the ValidPm class * Update docs string and naming * Add energy consumption * Lint code fixed * Refining postpone function * Lint style update * Init data pipeline * Update based on PR comment * Add data pipeline download * Lint style update * Code style fix * Temp update * Data pipeline update * Add aria2p download function * Update based on PR comment * Update based on PR comment * Update based on PR comment * Update naming of variables * Rename topology * Renaming * Fix valid pm list * Pylint fix * Update comment * Update docstring and comment * Fix init import * Update tick issue * fix merge problem * update style * V0.2 datacenter data pipeline (#199) * Data pipeline update * Data pipeline update * Lint update * Update pipeline * Add vmid mapping * Update lint style * Add VM data analytics * Update notebook * Add binary converter * Modift vmtable yaml * Update binary meta file * Add cpu reader * random example added for data center * Fix bugs * Fix pylint * Add launcher * Fix pylint * best fit policy added * Add reset * Add config * Add config * Modify action object * Modify config * Fix naming * Modify config * Add snapshot list * Modify a spelling typo * Update based on PR comments. * Rename scenario to vm scheduling * Rename scenario * Update print messages * Lint fix * Lint fix * Rename scenario * Modify the calculation of cpu utilization * Add comment * Modify data pipeline path * Fix typo * Modify naming * Add unittest * Add comment * Unify naming * Fix data path typo * Update comments * Update snapshot features * Add take snapshot * Add summary keys * Update cpu reader * Update naming * Add unit test * Rename snapshot node * Add processed data pipeline * Modify config * Add comment * Lint style fix Co-authored-by: Jinyu Wang * Add package used in vm_scheduling * add aria2p to test requirement * best fit example: update the usage of snapshot * Add aria2p to test requriement * Remove finish event * Fix unittest * Add test dataset * Update based on PR comment * Refine cpu reader and unittest * Lint update * Refine based on PR comment * Add agent index * Add node maping * Refine based on PR comments * Renaming postpone_step * Renaming and refine based on PR comments * Rename config * Update Co-authored-by: Jinyu Wang Co-authored-by: Chaos Yu Co-authored-by: ysqyang Co-authored-by: ysqyang Co-authored-by: Jinyu-W <53509467+Jinyu-W@users.noreply.github.com> * Resolve none action problem (#224) * V0.2 vm_scheduling notebook (#223) * Initialize * Data center scenario init * Code style modification * V0.2 event buffer subevents expand (#180) * V0.2 rl toolkit refinement (#165) * refined rl abstractions * fixed formattin issues * checked out error-code related code from v0.2_pg * fixed a bug * fixed a bug * fixed a bug * fixed a bug * fixed a bug * fixed a bug * fixed a bug * fixed a bug * fixed a bug * fixed a bug * fixed a bug * fixed a bug * renamed save_models to dump_models * 1. set default batch_norm_enabled to True; 2. used state_dict in dqn model saving * renamed dump_experience_store to dump_experience_pool * fixed a bug in the dump_experience_pool method * fixed some PR comments * fixed more PR comments * 1.fixed some PR comments; 2.added early_stopping_checker; 3.revised explorer class * fixed cim example according to rl toolkit changes * fixed some more PR comments * rewrote multi_process_launcher to eliminate the distributed section in config * 1. fixed a typo; 2. added logging before early stopping * fixed a bug * fixed a bug * fixed a bug * added early stopping feature to CIM exmaple * fixed a typo * fixed some issues with early stopping * changed early stopping metric func * fixed a bug * fixed a bug * added early stopping to dist mode cim * added experience collecting func * edited notebook according to changes in CIM example * fixed bugs in nb * fixed lint formatting issues * fixed a typo * fixed some PR comments * fixed more PR comments * revised docs * removed nb output * fixed a bug in simple_learner * fixed a typo in nb * fixed a bug * fixed a bug * fixed a bug * removed unused import * fixed a bug * 1. changed early stopping default config; 2. renamed param in early stopping checker and added typing * fixed some doc issues * added output to nb Co-authored-by: ysqyang * unfold sub-events, insert after parent * remove event category, use different class instead, add helper functions to gen decision and action event * add a method to support add immediate event to cascade event with tick validation * fix ut issue * add action as 1st sub event to ensure the executing order Co-authored-by: ysqyang Co-authored-by: ysqyang * Data center scenario update * Code style update * Data scenario business engine update * Isort update * Fix lint code check * Fix based on PR comments. * Update based on PR comments. * Add decision payload * Add config file * Update utilization series logic * Update based on PR comment * Update based on PR * Update * Update * Add the ValidPm class * Update docs string and naming * Add energy consumption * Lint code fixed * Refining postpone function * Lint style update * Init data pipeline * Update based on PR comment * Add data pipeline download * Lint style update * Code style fix * Temp update * Data pipeline update * Add aria2p download function * Update based on PR comment * Update based on PR comment * Update based on PR comment * Update naming of variables * Rename topology * Renaming * Fix valid pm list * Pylint fix * Update comment * Update docstring and comment * Fix init import * Update tick issue * fix merge problem * update style * V0.2 datacenter data pipeline (#199) * Data pipeline update * Data pipeline update * Lint update * Update pipeline * Add vmid mapping * Update lint style * Add VM data analytics * Update notebook * Add binary converter * Modift vmtable yaml * Update binary meta file * Add cpu reader * random example added for data center * Fix bugs * Fix pylint * Add launcher * Fix pylint * best fit policy added * Add reset * Add config * Add config * Modify action object * Modify config * Fix naming * Modify config * Add snapshot list * Modify a spelling typo * Update based on PR comments. * Rename scenario to vm scheduling * Rename scenario * Update print messages * Lint fix * Lint fix * Rename scenario * Modify the calculation of cpu utilization * Add comment * Modify data pipeline path * Fix typo * Modify naming * Add unittest * Add comment * Unify naming * Fix data path typo * Update comments * Update snapshot features * Add take snapshot * Add summary keys * Update cpu reader * Update naming * Add unit test * Rename snapshot node * Add processed data pipeline * Modify config * Add comment * Lint style fix Co-authored-by: Jinyu Wang * Add package used in vm_scheduling * add aria2p to test requirement * best fit example: update the usage of snapshot * Add aria2p to test requriement * Remove finish event * Fix unittest * Add test dataset * Update based on PR comment * Refine cpu reader and unittest * Lint update * Refine based on PR comment * Add agent index * Add node maping * Init vm shceduling notebook * Add notebook * Refine based on PR comments * Renaming postpone_step * Renaming and refine based on PR comments * Rename config * Update based on the v0.2_datacenter * Update notebook * Update * update filepath * notebook updated Co-authored-by: Jinyu Wang Co-authored-by: Chaos Yu Co-authored-by: ysqyang Co-authored-by: ysqyang Co-authored-by: Jinyu-W <53509467+Jinyu-W@users.noreply.github.com> * Update process mode docs and fixed on premises (#226) * V0.2 Add github workflow integration (#222) * test: add github workflow integration * fix: split procedures && bug fixed * test: add training only restriction * fix: add 'approved' restriction * fix: change default ssh port to 22 * style: in one line * feat: add timeout for Subprocess.run * test: change default node_size to Standard_D2s_v3 * style: refine style * fix: add ssh_port param to on-premises mode * fix: add missing init.py * update param name * V0.2 explorer (#198) * overhauled exploration abstraction * fixed a bug * fixed a bug * fixed a bug * added exploration related methods to abs_agent * fixed a bug * fixed a bug * fixed a bug * fixed a bug * fixed a bug * fixed a bug * separated learning with exploration schedule and without * small fixes * moved explorer logic to actor side * fixed a bug * fixed a bug * fixed a bug * fixed a bug * removed unwanted param from simple agent manager * added noise explorer * fixed formatting * removed unnecessary comma * fixed PR comments * removed unwanted exception and imports * fixed a bug * fixed PR comments * fixed a bug * fixed a bug * fixed a bug * fixed a bug * fixed a bug * fixed a bug * fixed lint issue * fixed a bug * fixed lint issue * fixed naming * combined exploration param generation and early stopping in scheduler * fixed a bug * fixed a bug * fixed a bug * fixed a bug * fixed a bug * fixed lint issues * fixed lint issue * moved logger inside scheduler * fixed a bug * fixed a bug * fixed a bug * fixed lint issues * removed epsilon parameter from choose_action * fixed some PR comments * fixed some PR comments * bug fix * bug fix * bug fix * removed explorer abstraction from agent * refined dqn example * fixed lint issues * simplified scheduler * removed early stopping from CIM dqn example * removed early stopping from cim example config * renamed early_stopping_callback to early_stopping_checker * removed action_dim from noise explorer classes and added some shape checks * modified NoiseExplorer's __call__ logic to batch processing * made NoiseExplorer's __call__ return type np array * renamed update to set_parameters in explorer * fixed old naming in test_grass Co-authored-by: ysqyang * V0.2 embedded optim (#191) * added dueling action value model * renamed params in dueling_action_value_model * renamed shared_features to features * replaced IdentityLayers with nn.Identity * 1. added skip connection option in FC_net; 2. generalized learning model * added skip_connection option in config * removed type casting in fc_net * fixed lint formatting issues * refined docstring * mv dueling_actiovalue_model and fixed some bugs * added multi-head functionality to LearningModel * refined learning model docstring * added head_key param in learningModel forward * added double DQN and dueling features to DQN * fixed a bug * added DuelingQModelHead enum * fixed a bug * removed unwanted file * fixed PR comments * added top layer logic and is_top option in fc_net * fixed a bug * fixed a bug * reverted some changes in learning model * reverted some changes in learning model * added members to learning model to fix the mode issue * fixed a bug * fixed mode setting issue in learning model * fixed PR comments * revised cim example according to DQN changes * renamed eval_model to q_value_model in cim example * more fixes * fixed a bug * fixed a bug * added doc per PR comments * removed learner.exit() in single_process_launcher * removed learner.exit() in single_process_launcher * fixed PR comments * fixed rl/__init__ * fixed issues in example * fixed a bug * fixed a bug * fixed lint formatting issues * double DQN feature * fixed a bug * fixed a bug * fixed PR comments * fixed lint issue * embedded optimizer into SingleHeadLearningModel * 1. fixed PR comments related to load/dump; 2. removed abstract load/dump methods from AbsAlgorithm * added load_models in simple_learner * minor docstring edits * minor docstring edits * minor docstring edits * mv optimizer options inside LearningMode * modified example accordingly * fixed a bug * fixed a bug * fixed a bug * added dueling DQN feature * revised and refined docstrings * fixed a bug * fixed lint issues * added load/dump functions to LearningModel * fixed a bug * fixed a bug * fixed lint issues * refined DQN docstrings * removed load/dump functions from DQN * added task validator * fixed decorator use * fixed a typo * fixed a bug * fixed lint issues * changed LearningModel's step() to take a single loss * revised learning model design * revised example * fixed a bug * fixed a bug * fixed a bug * fixed a bug * added decorator utils to algorithm * fixed a bug * renamed core_model to model * fixed a bug * 1. fixed lint formatting issues; 2. refined learning model docstrings * rm trailing whitespaces * added decorator for choose_action * fixed a bug * fixed a bug * fixed version-related issues * renamed add_zeroth_dim decorator to expand_dim * overhauled exploration abstraction * fixed a bug * fixed a bug * fixed a bug * added exploration related methods to abs_agent * fixed a bug * fixed a bug * fixed a bug * fixed a bug * fixed a bug * fixed a bug * separated learning with exploration schedule and without * small fixes * moved explorer logic to actor side * fixed a bug * fixed a bug * fixed a bug * fixed a bug * removed unwanted param from simple agent manager * small fixes * added shared_module property to LearningModel * added shared_module property to LearningModel * revised __getstate__ for LearningModel * fixed a bug * added soft_update function to learningModel * fixed a bug * revised learningModel * rm __getstate__ and __setstate__ from LearningModel * added noise explorer * fixed formatting * removed unnecessary comma * removed unnecessary comma * fixed PR comments * removed unwanted exception and imports * removed unwanted exception and imports * fixed a bug * fixed PR comments * fixed a bug * fixed a bug * fixed a bug * fixed a bug * fixed a bug * fixed a bug * fixed lint issue * fixed a bug * fixed lint issue * fixed naming * combined exploration param generation and early stopping in scheduler * fixed a bug * fixed a bug * fixed a bug * fixed a bug * fixed a bug * fixed lint issues * fixed lint issue * moved logger inside scheduler * fixed a bug * fixed a bug * fixed a bug * fixed lint issues * fixed lint issue * removed epsilon parameter from choose_action * removed epsilon parameter from choose_action * changed agent manager's train parameter to experience_by_agent * fixed some PR comments * renamed zero_grad to zero_gradients in LearningModule * fixed some PR comments * bug fix * bug fix * bug fix * removed explorer abstraction from agent * added DEVICE env variable as first choice for torch device * refined dqn example * fixed lint issues * removed unwanted import in cim example * updated cim-dqn notebook * simplified scheduler * edited notebook according to merged scheduler changes * refined dimension check for learning module manager and removed num_actions from DQNConfig * bug fix for cim example * added notebook output * removed early stopping from CIM dqn example * removed early stopping from cim example config * moved decorator logic inside algorithms * renamed early_stopping_callback to early_stopping_checker * removed action_dim from noise explorer classes and added some shape checks * modified NoiseExplorer's __call__ logic to batch processing * made NoiseExplorer's __call__ return type np array * renamed update to set_parameters in explorer * fixed old naming in test_grass Co-authored-by: ysqyang * V0.2 VM scheduling docs (#228) * Initialize * Data center scenario init * Code style modification * V0.2 event buffer subevents expand (#180) * V0.2 rl toolkit refinement (#165) * refined rl abstractions * fixed formattin issues * checked out error-code related code from v0.2_pg * fixed a bug * fixed a bug * fixed a bug * fixed a bug * fixed a bug * fixed a bug * fixed a bug * fixed a bug * fixed a bug * fixed a bug * fixed a bug * fixed a bug * renamed save_models to dump_models * 1. set default batch_norm_enabled to True; 2. used state_dict in dqn model saving * renamed dump_experience_store to dump_experience_pool * fixed a bug in the dump_experience_pool method * fixed some PR comments * fixed more PR comments * 1.fixed some PR comments; 2.added early_stopping_checker; 3.revised explorer class * fixed cim example according to rl toolkit changes * fixed some more PR comments * rewrote multi_process_launcher to eliminate the distributed section in config * 1. fixed a typo; 2. added logging before early stopping * fixed a bug * fixed a bug * fixed a bug * added early stopping feature to CIM exmaple * fixed a typo * fixed some issues with early stopping * changed early stopping metric func * fixed a bug * fixed a bug * added early stopping to dist mode cim * added experience collecting func * edited notebook according to changes in CIM example * fixed bugs in nb * fixed lint formatting issues * fixed a typo * fixed some PR comments * fixed more PR comments * revised docs * removed nb output * fixed a bug in simple_learner * fixed a typo in nb * fixed a bug * fixed a bug * fixed a bug * removed unused import * fixed a bug * 1. changed early stopping default config; 2. renamed param in early stopping checker and added typing * fixed some doc issues * added output to nb Co-authored-by: ysqyang * unfold sub-events, insert after parent * remove event category, use different class instead, add helper functions to gen decision and action event * add a method to support add immediate event to cascade event with tick validation * fix ut issue * add action as 1st sub event to ensure the executing order Co-authored-by: ysqyang Co-authored-by: ysqyang * Data center scenario update * Code style update * Data scenario business engine update * Isort update * Fix lint code check * Fix based on PR comments. * Update based on PR comments. * Add decision payload * Add config file * Update utilization series logic * Update based on PR comment * Update based on PR * Update * Update * Add the ValidPm class * Update docs string and naming * Add energy consumption * Lint code fixed * Refining postpone function * Lint style update * Init data pipeline * Update based on PR comment * Add data pipeline download * Lint style update * Code style fix * Temp update * Data pipeline update * Add aria2p download function * Update based on PR comment * Update based on PR comment * Update based on PR comment * Update naming of variables * Rename topology * Renaming * Fix valid pm list * Pylint fix * Update comment * Update docstring and comment * Fix init import * Update tick issue * fix merge problem * update style * V0.2 datacenter data pipeline (#199) * Data pipeline update * Data pipeline update * Lint update * Update pipeline * Add vmid mapping * Update lint style * Add VM data analytics * Update notebook * Add binary converter * Modift vmtable yaml * Update binary meta file * Add cpu reader * random example added for data center * Fix bugs * Fix pylint * Add launcher * Fix pylint * best fit policy added * Add reset * Add config * Add config * Modify action object * Modify config * Fix naming * Modify config * Add snapshot list * Modify a spelling typo * Update based on PR comments. * Rename scenario to vm scheduling * Rename scenario * Update print messages * Lint fix * Lint fix * Rename scenario * Modify the calculation of cpu utilization * Add comment * Modify data pipeline path * Fix typo * Modify naming * Add unittest * Add comment * Unify naming * Fix data path typo * Update comments * Update snapshot features * Add take snapshot * Add summary keys * Update cpu reader * Update naming * Add unit test * Rename snapshot node * Add processed data pipeline * Modify config * Add comment * Lint style fix Co-authored-by: Jinyu Wang * Add package used in vm_scheduling * add aria2p to test requirement * best fit example: update the usage of snapshot * Add aria2p to test requriement * Remove finish event * Fix unittest * Add test dataset * Update based on PR comment * vm doc init * Update docs * Update docs * Update docs * Update docs * Remove old notebook * Update docs * Update docs * Add figure * Update docs Co-authored-by: Jinyu Wang Co-authored-by: Chaos Yu Co-authored-by: ysqyang Co-authored-by: ysqyang Co-authored-by: Jinyu-W <53509467+Jinyu-W@users.noreply.github.com> * doc update * new link * image update * v0.2 VM Scheduling docs refinement (#231) * Fix typo * Refining vm scheduling docs * image change * V0.2 store refinement (#234) * updated docs and images for rl toolkit * 1. fixed import formats for maro/rl; 2. changed decorators to hypers in store * fixed lint issues Co-authored-by: ysqyang * Fix bug (#237) vm scenario: fix the event type bug of the postpone event * V0.2 rl toolkit doc (#235) * updated docs and images for rl toolkit * updated cim example doc * updated cim exmaple docs * updated cim example rst * updated rl_toolkit and cim example docs * replaced q_module with q_net in example rst * refined doc * refined doc * updated figures * updated figures Co-authored-by: ysqyang * Merge V0.2 vis into V0.2 (#233) * Implemented dump snapshots and convert to CSV. * Let BE supports params when dump snapshot. * Refactor dump code to core.py * Implemented decision event dump. * replace is not '' with !='' * Fixed issues that code review mentioned. * removed path from hello.py * Changed import sort. * Fix import sorting in citi_bike/business_engine * visualization 0.1 * Updated lint configurations. * Fixed formatting error that caused lint errors. * render html title function * Try to fix lint errors. * flake-8 style fix * remove space around 18,35 * dump_csv_converter.py re-formatting. * files re-formatting. * style fixed * tab delete * white space fix * white space fix-2 * vis redundant function delete * refine * re-formatting after merged upstream. * Updated import section. * Updated import section. * pr refine * isort fix * white space * lint error * \n error * test continuation * indent * continuation of indent * indent 0.3 * comment update * comment update 0.2 * f-string update * f-string 0.2 * lint 0.3 * lint 0.4 * lint 0.4 * lint 0.5 * lint 0.6 * docstring update * data version deploy update * condition update * add whitespace * V0.2 vis dump feature enhancement. (#190) * Dumps added manifest file. * Code updated format by flake8 * Changed manifest file format for easy reading. * deploy info update; docs update * weird white space * Update dashboard_visualization.md * new endline? * delete dependency * delete irrelevant file * change scenario to enum, divide file path into a separated class * doc refine * doc update * params type * data structure update * doc&enum, formula refine * refine * add ut, refine doc * style refine * isort * strong type fix * os._exit delete * revert datalib * import new line * change test case * change file name & doc * change deploy path * delete params * revert file * delete duplicate file * delete single process * update naming * manually change import order * delete blank * edit error * requirement txt * style fix & refine * comments&docstring refine * add parameter name * test & dump * comments update * Added manifest file. (#201) Only a few changes that need to meet requirements of manifest file format. * comments fix * delete toolkit change * doc update * citi bike update * deploy path * datalib update * revert datalib * revert * maro file format * comments update * doc update * update param name * doc update * new link * image update * V0.2 visualization-0.1 (#181) * visualization 0.1 * render html title function * flake-8 style fix * style fixed * tab delete * white space fix * white space fix-2 * vis redundant function delete * refine * pr refine * isort fix * white space * lint error * \n error * test continuation * indent * continuation of indent * indent 0.3 * comment update * comment update 0.2 * f-string update * f-string 0.2 * lint 0.3 * lint 0.4 * lint 0.4 * lint 0.5 * lint 0.6 * docstring update * data version deploy update * condition update * add whitespace * deploy info update; docs update * weird white space * Update dashboard_visualization.md * new endline? * delete dependency * delete irrelevant file * change scenario to enum, divide file path into a separated class * fix the visualization of docs/key_components/distributed_toolkit * doc refine * doc update * params type * add examples into isort ignore * data structure update * doc&enum, formula refine * refine * add ut, refine doc * style refine * isort * strong type fix * os._exit delete * revert datalib * import new line * change test case * change file name & doc * change deploy path * delete params * revert file * delete duplicate file * delete single process * update naming * manually change import order * delete blank * edit error * requirement txt * style fix & refine * comments&docstring refine * add parameter name * test & dump * comments update * comments fix * delete toolkit change * doc update * citi bike update * deploy path * datalib update * revert datalib * revert * maro file format * comments update * doc update * update param name * doc update * new link * image update Co-authored-by: Jinyu Wang Co-authored-by: Miaoran Chen (Wicresoft) * image change * add reset snapshot * delete dump * add new line * add next steps * import change * relative import * add init file * import change * change utils file * change cliexpcetion to clierror * dashboard test * change result * change assertation * move not * unit test change * core change * unit test delete name_mapping_file * update cim business engine * doc update * change relative path * doc update * doc update * doc update * doc update * doc update * doc update * doc update * doc update * doc update * doc update * duc update * duc update * duc update * doc update * doc update * doc update * doc update * doc update * doc update * doc update * doc update * change import sequence * comments update * doc add pic * add dependency * doc update * doc update * doc update * doc update * doc update * doc update * doc update * doc update * doc update * doc update * doc update * doc update * Update dashboard_visualization.rst * doc update * doc update * doc update * doc update * doc update * doc update * doc update * doc update * doc update * delete white space * doc update * doc update * update doc * update doc * update doc Co-authored-by: Michael Li Co-authored-by: Miaoran Chen (Wicresoft) Co-authored-by: Jinyu Wang Co-authored-by: Jinyu-W <53509467+Jinyu-W@users.noreply.github.com> * V0.2 docs process mode (#230) * Update process mode docs and fixed on premises * Update orchestration docs * Update process mode docs add JOB_NAME as env variable * fixed bugs * fixed isort issue * update docs index Co-authored-by: kaiqli * V0.2 learning model refinement (#236) * moved optimizer options to LearningModel * typo fix * fixed lint issues * updated notebook * misc edits * 1. renamed CIMAgent to DQNAgent; 2. moved create_dqn_agents to Agent section in notebook * renamed single_host_cim_learner ot cim_learner in notebook * updated notebook output * typo fix * removed dimension check in absence of shared stack * fixed a typo * fixed lint issues Co-authored-by: ysqyang * Update vm docs (#241) Co-authored-by: Jinyu-W <53509467+Jinyu-W@users.noreply.github.com> * V0.2 info update (#240) * update readme * update version * refine reademe format * add vis gif * add citation * update citation * update badge Co-authored-by: Arthur Jiang * Fix typo (#242) * Fix typo * fix typo * fix * doc update * doc update * doc update * doc update * doc update * doc update * doc update * doc update * doc update * doc update * doc update * doc update * doc update * doc update * doc update * doc update Co-authored-by: Arthur Jiang Co-authored-by: Arthur Jiang Co-authored-by: Romic Huang Co-authored-by: zhanyu wang Co-authored-by: ysqyang Co-authored-by: ysqyang Co-authored-by: kaiqli <59279714+kaiqli@users.noreply.github.com> Co-authored-by: Jinyu-W <53509467+Jinyu-W@users.noreply.github.com> Co-authored-by: Jinyu Wang Co-authored-by: Jinyu Wang Co-authored-by: Michael Li Co-authored-by: Miaoran Chen (Wicresoft) Co-authored-by: Chaos Yu Co-authored-by: Wenlei Shi Co-authored-by: kyu-kuanwei <72911362+kyu-kuanwei@users.noreply.github.com> Co-authored-by: kaiqli --- .../dashboard_visualization.rst | 20 +++++++++---------- docs/source/key_components/rl_toolkit.rst | 2 +- docs/source/scenarios/citi_bike.rst | 16 +++++++-------- .../container_inventory_management.rst | 18 ++++++++--------- maro/data_lib/dump_csv_converter.py | 5 ++--- requirements.dev.txt | 2 +- 6 files changed, 31 insertions(+), 32 deletions(-) diff --git a/docs/source/key_components/dashboard_visualization.rst b/docs/source/key_components/dashboard_visualization.rst index 98099d33d..7906b7de7 100644 --- a/docs/source/key_components/dashboard_visualization.rst +++ b/docs/source/key_components/dashboard_visualization.rst @@ -62,7 +62,7 @@ path of a local file folder, data would be dumped to this folder. Data would be dumped automatically when the Env object is initialized. For more details about Environment, please refer to -`Environment <../simulation_toolkit.html#Environment>`_. +`Environment `_. Launch Visualization Tool ~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -95,7 +95,7 @@ Folder Structure epoch_# # folders to restore data of each epoch. {resource_holder}.csv # attributes of current epoch. manifest.yml # basic info like scenario name, number of epoches. - index\_name\_mapping file # relationship between an index and its name of resource holders. + index_name_mapping file # relationship between an index and its name of resource holders. {resource_holder}_summary.csv # cross-epoch summary information. @@ -144,8 +144,8 @@ To view the details of a resource holder or a tick, user could select the specific index of epoch/snapshot/resource holder by sliding the slider on the left side of page. -.. figure:: ..\images\visualization\dashboard\epoch_resource_holder_index_selection.gif - :alt: epoch\_resource\_holder\_index\_selection +.. image:: ../images/visualization/dashboard/epoch_resource_holder_index_selection.gif + :alt: epoch_resource_holder_index_selection Snapshot/Resource Holder Sampling Ratio Selection ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -154,8 +154,8 @@ To view trends in the data, or to weed out excess information, user could select the sampling ratio of snapshot/resource holder by sliding to change the number of data to be displayed. -.. figure:: ..\images\visualization\dashboard\snapshot_sampling_ratio_selection.gif - :alt: snapshot\_sampling\_ratio\_selection +.. image:: ../images/visualization/dashboard/snapshot_sampling_ratio_selection.gif + :alt: snapshot_sampling_ratio_selection Formula Calculation ^^^^^^^^^^^^^^^^^^^ @@ -164,8 +164,8 @@ User could generate their own attributes by using pre-defined formulas. The results of the formula calculation could be reused as the input parameter of formula. -.. figure:: ..\images\visualization\dashboard\formula_calculation.gif - :alt: formula\_calculation +.. image:: ../images/visualization/dashboard/formula_calculation.gif + :alt: formula_calculation Inter-epoch view ~~~~~~~~~~~~~~~~ @@ -187,8 +187,8 @@ To view trends in the data, or to weed out excess information, user could select the sampling ratio of epoch by sliding to change the number of data to be displayed. -.. figure:: ..\images\visualization\dashboard\epoch_sampling_ratio.gif - :alt: epoch\_sampling\_ratio +.. image:: ../images/visualization/dashboard/epoch_sampling_ratio.gif + :alt: epoch_sampling_ratio Formula Calculation ^^^^^^^^^^^^^^^^^^^ diff --git a/docs/source/key_components/rl_toolkit.rst b/docs/source/key_components/rl_toolkit.rst index 57b64f182..61d430415 100644 --- a/docs/source/key_components/rl_toolkit.rst +++ b/docs/source/key_components/rl_toolkit.rst @@ -208,4 +208,4 @@ As an example, the exploration for DQN may be carried out with the aid of an ``E explorer = EpsilonGreedyExplorer(num_actions=10) greedy_action = learning_model(state, is_training=False).argmax(dim=1).data - exploration_action = explorer(greedy_action) + exploration_action = explorer(greedy_action) \ No newline at end of file diff --git a/docs/source/scenarios/citi_bike.rst b/docs/source/scenarios/citi_bike.rst index b77300072..bec912069 100644 --- a/docs/source/scenarios/citi_bike.rst +++ b/docs/source/scenarios/citi_bike.rst @@ -783,8 +783,8 @@ freely adjust the sampling rate. For example, if there are 100 snapshots and user selected 0.3 as sampling ratio, 30 snapshots data would be selected to render the chart. -.. figure:: ..\images\visualization\dashboard\citi_bike_intra_epoch_by_station.gif - :alt: citi\_bike\_intra\_epoch\_by\_station +.. image:: ../images/visualization/dashboard/citi_bike_intra_epoch_by_station.gif + :alt: citi_bike_intra_epoch_by_station To be specific, the line chart could be customized with operations in the following example. @@ -796,16 +796,16 @@ will be provided with the option to quickly select a set of data. e.g. In this scenario, item "Requirement Info" refers to [trip\_requirement, shortage, fulfillment]. -.. figure:: ..\images\visualization\dashboard\citi_bike_intra_epoch_by_station_2.gif - :alt: citi\_bike\_intra\_epoch\_by\_station\_2 +.. image:: ../images/visualization/dashboard/citi_bike_intra_epoch_by_station_2.gif + :alt: citi_bike_intra_epoch_by_station_2 Moreover, to improve the flexibility of visualizing data, user could use pre-defined formula and selected attributes to generate new attributes. Generated attributes would be treated in the same way as original attributes. -.. figure:: ..\images\visualization\dashboard\citi_bike_intra_epoch_by_station_3.gif - :alt: citi\_bike\_intra\_epoch\_by\_station\_3 +.. image:: ../images/visualization/dashboard/citi_bike_intra_epoch_by_station_3.gif + :alt: citi_bike_intra_epoch_by_station_3 If user choose to view information by snapshot, it means attributes of all stations within a selected snapshot would be displayed. By changing @@ -816,5 +816,5 @@ of sampled data. Particularly, if user want to check the name of a specific station, just hovering on the according bar. -.. figure:: ..\images\visualization\dashboard\citi_bike_intra_epoch_by_snapshot.gif - :alt: citi\_bike\_intra\_epoch\_by\_snapshot \ No newline at end of file +.. image:: ../images/visualization/dashboard/citi_bike_intra_epoch_by_snapshot.gif + :alt: citi_bike_intra_epoch_by_snapshot \ No newline at end of file diff --git a/docs/source/scenarios/container_inventory_management.rst b/docs/source/scenarios/container_inventory_management.rst index 3eeaa64eb..3e5e8ab98 100644 --- a/docs/source/scenarios/container_inventory_management.rst +++ b/docs/source/scenarios/container_inventory_management.rst @@ -658,8 +658,8 @@ To change "Start Epoch" and "End Epoch", user could specify the selected data range. To change "Epoch Sampling Ratio", user could change the sampling rate of selected data. -.. figure:: ..\images\visualization\dashboard\cim_inter_epoch.gif - :alt: cim\_inter\_epoch +.. image:: ../images/visualization/dashboard/cim_inter_epoch.gif + :alt: cim_inter_epoch Intra-epoch view ^^^^^^^^^^^^^^^^ @@ -669,13 +669,13 @@ slider, users can select different epochs. Furthermore, this part of data is divided into two dimensions: by snapshot and by port according to time and space. In terms of data display, according to the different types of attributes, it is divided into two levels: accumulated data - (accumulated attributes. e.g. acc\_fulfillment) and detail data. +(accumulated attributes. e.g. acc\_fulfillment) and detail data. If user choose to view information by ports, attributes of the selected port would be displayed. -.. figure:: ..\images\visualization\dashboard\cim_intra_epoch_by_ports.gif - :alt: cim\_intra\_epoch\_by\_ports +.. image:: ../images/visualization/dashboard/cim_intra_epoch_by_ports.gif + :alt: cim_intra_epoch_by_ports If user choose to view data by snapshots, attributes of selected snapshot would be displayed. The charts and data involved in this part @@ -697,8 +697,8 @@ over time in the current epoch. The bar chart of Port Accumulated Attributes displays the global change of ports. -.. figure:: ..\images\visualization\dashboard\cim_intra_epoch_by_snapshot_acc_data.gif - :alt: cim\_intra\_epoch\_by\_snapshot\_acc\_data +.. image:: ../images/visualization/dashboard/cim_intra_epoch_by_snapshot_acc_data.gif + :alt: cim_intra_epoch_by_snapshot_acc_data Detail Data ~~~~~~~~~~~ @@ -706,5 +706,5 @@ Detail Data Since the cargoes is transported through vessels, information of vessels could be viewed by snapshot. Same as ports, user could change the sampling rate of vessels. -.. figure:: ..\images\visualization\dashboard\cim_intra_epoch_by_snapshot_detail_data.gif - :alt: cim\_intra\_epoch\_by\_snapshot\_detail\_data +.. image:: ../images/visualization/dashboard/cim_intra_epoch_by_snapshot_detail_data.gif + :alt: cim_intra_epoch_by_snapshot_detail_data diff --git a/maro/data_lib/dump_csv_converter.py b/maro/data_lib/dump_csv_converter.py index 97a777c8d..04d6b1117 100644 --- a/maro/data_lib/dump_csv_converter.py +++ b/maro/data_lib/dump_csv_converter.py @@ -1,5 +1,5 @@ -# Copyright (c) Microsoft Corporation. -# Licensed under the MIT license. +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. import os @@ -7,7 +7,6 @@ from datetime import datetime from math import floor from pathlib import Path -from shutil import copyfile import numpy as np import pandas as pd diff --git a/requirements.dev.txt b/requirements.dev.txt index cee7b6bb3..ad3abcc75 100644 --- a/requirements.dev.txt +++ b/requirements.dev.txt @@ -36,4 +36,4 @@ streamlit==0.69.1 altair==4.1.0 tqdm==4.51.0 editorconfig-checker -aria2p==0.9.1 +aria2p==0.9.1 \ No newline at end of file From 166ae35bf96230e35fccf181b0d063847d5d4974 Mon Sep 17 00:00:00 2001 From: ysqyang Date: Wed, 6 Jan 2021 11:55:56 +0800 Subject: [PATCH 330/337] bug fix related to np array divide (#245) Co-authored-by: ysqyang --- maro/rl/storage/column_based_store.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/maro/rl/storage/column_based_store.py b/maro/rl/storage/column_based_store.py index ed4c29491..20d29c12a 100644 --- a/maro/rl/storage/column_based_store.py +++ b/maro/rl/storage/column_based_store.py @@ -192,7 +192,7 @@ def sample(self, size, weights: Union[list, np.ndarray] = None, replace: bool = """ if weights is not None: weights = np.asarray(weights) - weights /= np.sum(weights) + weights = weights / np.sum(weights) indexes = np.random.choice(self._size, size=size, replace=replace, p=weights) return indexes, self.get(indexes) From 97fab8c6d3b3299ff3d70485b7bfe4117943daf6 Mon Sep 17 00:00:00 2001 From: Jinyu-W <53509467+Jinyu-W@users.noreply.github.com> Date: Mon, 11 Jan 2021 15:22:08 +0800 Subject: [PATCH 331/337] Master.simple bike (#250) * notebook for simple bike repositioning added * add simple rule-based algorithms * unify input * add policy based on statistics * update be for simple bike scenario to fit latest event buffer changes (#247) * change rendered graph * figures updated * change notebook * matplot updated * figures updated Co-authored-by: Jinyu Wang Co-authored-by: wesley Co-authored-by: Chaos Yu --- .../articles/simple_bike_repositioning.ipynb | 721 ++++++++++++++++++ .../DecisionLogic.png | Bin 0 -> 143047 bytes .../simple_bike_repositioning/TripLogic.png | Bin 0 -> 131866 bytes .../TripRequirements.png | Bin 0 -> 141914 bytes 4 files changed, 721 insertions(+) create mode 100644 notebooks/articles/simple_bike_repositioning.ipynb create mode 100644 notebooks/articles/simple_bike_repositioning/DecisionLogic.png create mode 100644 notebooks/articles/simple_bike_repositioning/TripLogic.png create mode 100644 notebooks/articles/simple_bike_repositioning/TripRequirements.png diff --git a/notebooks/articles/simple_bike_repositioning.ipynb b/notebooks/articles/simple_bike_repositioning.ipynb new file mode 100644 index 000000000..3155e6265 --- /dev/null +++ b/notebooks/articles/simple_bike_repositioning.ipynb @@ -0,0 +1,721 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# The Simple Bike Repositioning Scenario\n" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "import matplotlib\n", + "import matplotlib.pyplot as plt\n", + "import numpy as np\n", + "import random\n", + "from math import pi, sin\n", + "from typing import List\n", + "\n", + "from maro.backends.frame import FrameBase, FrameNode, NodeAttribute, NodeBase, node\n", + "from maro.event_buffer import MaroEvents\n", + "from maro.simulator.scenarios import AbsBusinessEngine\n", + "from maro.simulator import Env" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 1. Problem Setting" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "To simplify this problem, we consider a situation where are only 4 bike stations. Let's denote the station set as $\\{s_i|i=0,1,2,3\\}$." + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "TOTAL_STATIONS = 4" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The detailed trip requirement distribution among these stations are shown in the figure below. Here we use $F_{ij}(t)$ to present the trip requirements from station $s_i$ to station $s_j$.\n", + "\n", + "\n", + "\n", + "The right half of this figure shows the trip requirement curve of each station pair $F_{ij}(t)$ over time $t$. We can find that:\n", + "- $F_{02}(t)$ and $F_{20}(t)$ follow a symmetric and periodic transition pattern. In the first half of the period, $F_{02}(t)$ first increases from 0 to the maximum and then decreases to 0, while $F_{20}(t)$ remains at 0. In the second half of the period, $F_{20}(t)$ starts to increase and then decreases, while $F_{02}(t)$ remains at 0. \n", + "- Generally speaking, $F_{13}(t)$ and $F_{31}(t)$ remains at the same level over time, with slight fluctuations only in a few times.\n", + "- As for $F_{01}(t)$ and $F_{32}(t)$, they show stable and equal trip requirements.\n", + "\n", + "Specifically, we use sine functions to model these periodic changes, that we defined:\n", + "\n", + "$F_{02}(t) = \\max(0, 8 \\sin(0.1 t))$\n", + "\n", + "$F_{20}(t) = \\max(0, -8 \\sin(0.1 t))$\n", + "\n", + "$F_{13}(t) = 5 + \\max(0, 5 \\sin(0.2\\pi t) - 4) + \\min(0, 5 \\sin(0.2\\pi t) + 4)$\n", + "\n", + "$F_{31}(t) = 5 + \\max(0, 5 \\sin(0.4 t) - 4) + \\min(0, 5 \\sin(0.4 t) + 4)$\n", + "\n", + "$F_{01}(t) = 3$\n", + "\n", + "$F_{32}(t) = 3$" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [], + "source": [ + "max_0_2 = 8 # The maximum trip requirements between station 0 and station 2.\n", + "exp_1_3 = 5 # The expected trip requirements between station 1 and station 3.\n", + "const_num = 3 # The number of trip requirements from station 0 to station 1, from station 3 to station 2.\n", + "\n", + "def generate_station_requirements(tick: int):\n", + " # The number of trip requirements between station 0 and station 2 follows a sine function.\n", + " # Specifically, it is the F20(t) and F02(t) in the figure above.\n", + " num_between_0_2 = round(sin(0.1 * tick) * max_0_2)\n", + " requirements_between_0_2 = [(0, 2)] * num_between_0_2 if num_between_0_2 >= 0 else [(2, 0)] * -num_between_0_2\n", + "\n", + " # The number of trip requirements between station 1 and station 3 are almostly equal (with only slight difference).\n", + " # Specifically, it is the F13(t) and F31(t) in the figure above.\n", + " num_1_3 = round(exp_1_3 + max(0, exp_1_3 * (sin(0.2 * pi * tick) - 1) + 1) + min(0, exp_1_3 * (sin(0.2 * pi * tick) + 1) - 1))\n", + " requirements_1_3 = [(1, 3)] * num_1_3\n", + " num_3_1 = round(exp_1_3 + max(0, exp_1_3 * (sin(0.4 * tick) - 1) + 1) + min(0, exp_1_3 * (sin(0.4 * tick) + 1) - 1))\n", + " requirements_3_1 = [(3, 1)] * num_3_1\n", + " \n", + " # The number of trip requirements from station 0 to station 1 and the ones from station 3 to station 2 are equal.\n", + " # Specifically, it is the F01(t) and F32(t) in the figure above.\n", + " const_requirements = [(0, 1)] * const_num + [(3, 2)] * const_num\n", + "\n", + " # Randomly shuffle the order of trip requirements.\n", + " requirements = (requirements_between_0_2 + requirements_1_3 + requirements_3_1 + const_requirements)\n", + " random.shuffle(requirements)\n", + " return requirements" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Morever, for each trip and the bike repositioning operation, we assume that they can both be done no longer than 3 ticks. That:" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [], + "source": [ + "# The maximum trip duration. Unit: tick.\n", + "maximum_duration = 3\n", + "\n", + "def generate_duration():\n", + " return random.randrange(1, maximum_duration)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "To ensure the bike number in this simple scenario is almost enough for trip requirements, we should take the time for trips and repositioning operations into consideration. Along with the peak requirement and the expectation of each station pair, we can roughly figure out the total bike demand in this problem. And then distribute these bikes equally to each station." + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Total bikes needed in this scenario: 96, intial station inventory: 24.\n" + ] + } + ], + "source": [ + "bike_needed_between_0_2 = max_0_2 * maximum_duration * 2\n", + "bike_needed_between_1_3 = exp_1_3 * maximum_duration * 2\n", + "bike_needed_const = const_num * maximum_duration * 2\n", + "\n", + "total_bike_needed = bike_needed_between_0_2 + bike_needed_between_1_3 + bike_needed_const\n", + "\n", + "# The initial bike number in each station\n", + "INITIAL_BIKES = total_bike_needed // TOTAL_STATIONS\n", + "\n", + "print(f\"Total bikes needed in this scenario: {total_bike_needed}, intial station inventory: {INITIAL_BIKES}.\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 2. Simulation Logics and Implementation" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "To implement the simple bike repositioning scenario, we first define the class of the bike station. Here each station only contains 3 attributes:\n", + "- `bikes`: The real-time available bikes in this station.\n", + "- `shortage`: How many trip requirements is failed due to the lack of bikes till now.\n", + "- `requirements`: How many trip requirements received till now." + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [], + "source": [ + "# Definition of the bike station.\n", + "# The attribute name can be used to access the environment snapshot lists.\n", + "@node(\"station\")\n", + "class Station(NodeBase):\n", + " bikes = NodeAttribute(\"i\")\n", + " shortage = NodeAttribute(\"i\")\n", + " requirements = NodeAttribute(\"i\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "In MARO, we use the environment frame to save the environment status. To initialize the frame of this scenario, we need to specify how many stations we have and how many snapshots we need." + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [], + "source": [ + "MAX_TICKS = 100\n", + "\n", + "# Definition of the environment frame. The number of stations is given here.\n", + "class SimpleCitibikeFrame(FrameBase):\n", + " stations = FrameNode(Station, TOTAL_STATIONS)\n", + "\n", + " def __init__(self):\n", + " super().__init__(enable_snapshot=True, total_snapshot=MAX_TICKS)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "In MARO, the simulator is driven by the generation and the processing of various events. In this simple scenario, we only take the trip requirement logic and the bike repositioning logic into consideration. Then only 3 scenario-dependent event types here:\n", + "\n", + "- `TRIP_REQUIREMENT_EVENT`: When trip requirement generated, a event with this type will be inserted into the event buffer to trigger the trip requirement handling logic.\n", + "- `PENDING_DECISION_EVENT`: When the simulator want the agent to do a bike repositioning, it will throw a pending decision event. \n", + "- `BIKE_ARRIVAL_EVENT`: Both a trip requirement fulfillment and a bike repositioning operation are paired with a bike arrival event. It means the trip is finished and the bike turns to be available in the destination station, and the bikes are successfully transported to the destination station respectively." + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [], + "source": [ + "# Predefined event types\n", + "TRIP_REQUIREMENT_EVENT = \"trip_requirement_event\"\n", + "PENDING_DECISION_EVENT = \"pending_decision_event\"\n", + "BIKE_ARRIVAL_EVENT = \"bike_arrival_event\"" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [], + "source": [ + "# The event thrown by the environment to the agent, which is used to trigger the agent's decision.\n", + "class PendingDecisionEvent():\n", + " def __init__(self, station_index: int):\n", + " self.station_index = station_index\n", + " \n", + " def __repr__(self):\n", + " return f\"\"" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### 2.1. The Trip Fulfillment Logic" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\n", + "\n", + "The figure above illustrate the trip fulfillment logic in this simple scenario:\n", + "1. Each time, the simulator will generate several trip requirements according to the pattern we defined in the **Problem Setting** part. These trip requirements will then be inserted into the event buffer.\n", + "2. Once the simulator get a trip requirement event from the event buffer, it will call the corresponding callback function to handle it.\n", + "3. If there is available bike in the source station, the remaining bikes state of the source station will be updated and a future bike arrival event for the destination station will be generated and inserted into the event buffer. Else, a shortage of the source station is recorded.\n", + "4. Once the simulator get a bike arrival event from the event buffer, it will call the corresponding callback function to handle it.\n", + "5. The remaining bikes state of the station bike arrives will be updated." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### 2.2. The Bike Repositioning Decision Logic" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\n", + "\n", + "The figure above illustrate the bike repositioning logic in this simple scenario:\n", + "1. Each time, the simulator will check the state of each station.\n", + "2. When the station lack of bikes, the simulator will create an pending decision event and insert it into the event buffer.\n", + "3. Once the simulator gets a pending decision event from the event buffer, it will throw it out to the agent to trigger the agent's bike repositioning actions.\n", + "4. The agent can query the data model to get the information it needs, and make bike repositioning decision and reply with the repositioning action to the simulator.\n", + "5. Once the simulator get a repositioning action sent by the agent from the event buffer, it will call the corresponding handler function.\n", + "6. The handler function of the repositioning action will update the state of the source station and create a corresponding bike arrival event for it." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The overall logic of this simple bike repositioning scenario and the hanlder functions are all defined as a `BusinessEngine`." + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [], + "source": [ + "class SimpleCitibikeBusinessEngine(AbsBusinessEngine):\n", + " def __init__(self, **kwargs):\n", + " super().__init__(scenario_name=\"SimpleCitibike\", **kwargs)\n", + "\n", + " self._frame = SimpleCitibikeFrame()\n", + " \n", + " # A number of station instances will be created after the initialization of the frame.\n", + " self._stations: List[Station] = self._frame.stations\n", + " self._intialize_stations()\n", + "\n", + " # Register the event types and their handler functions.\n", + " self._event_buffer.register_event_handler(TRIP_REQUIREMENT_EVENT, self._on_requirement)\n", + " self._event_buffer.register_event_handler(BIKE_ARRIVAL_EVENT, self._on_arrive_station)\n", + " self._event_buffer.register_event_handler(MaroEvents.TAKE_ACTION, self._on_action_recieved)\n", + "\n", + " @property\n", + " def snapshots(self):\n", + " # The interface for the agents to access.\n", + " return self._frame.snapshots\n", + "\n", + " @property\n", + " def frame(self):\n", + " # The interface for the agents to access.\n", + " return self._frame\n", + "\n", + " def step(self, tick: int):\n", + " for station in self._stations:\n", + " if station.bikes == 0:\n", + " # For station without any bikes left, create a PENDING_DECISION_EVENT and insert it into the event buffer.\n", + " pending_decision_event = self._event_buffer.gen_decision_event(tick, PendingDecisionEvent(station.index))\n", + " self._event_buffer.insert_event(pending_decision_event)\n", + "\n", + " # Generate the trip requirements and insert them into the event buffer.\n", + " # A trip: (source station, destination station)\n", + " for source_station, destination_station in generate_station_requirements(tick):\n", + " requirement_event = self._event_buffer.gen_atom_event(tick, TRIP_REQUIREMENT_EVENT, (source_station, destination_station))\n", + " self._event_buffer.insert_event(requirement_event)\n", + "\n", + " def reset(self):\n", + " self._frame.reset()\n", + " self._frame.snapshots.reset()\n", + " self._intialize_stations()\n", + " random.seed(1234)\n", + "\n", + " def post_step(self, tick: int):\n", + " # Take and save the snapshot of the environment state at the end of each tick.\n", + " self._frame.take_snapshot(tick)\n", + "\n", + " # Clear requirement and shortage after taking snapshot,\n", + " # so that they will only be value at specific tick\n", + " for station in self._stations:\n", + " station.requirements = 0\n", + " station.shortage = 0\n", + "\n", + " # To end the simulation or not.\n", + " return tick == self._max_tick - 1\n", + "\n", + " def _intialize_stations(self):\n", + " for station in self._stations:\n", + " station.bikes = INITIAL_BIKES\n", + "\n", + " def _on_requirement(self, event):\n", + " src_station_index, dest_station_index = event.payload\n", + " station: Station = self._stations[src_station_index]\n", + " station.requirements += 1\n", + "\n", + " if station.bikes < 1:\n", + " # No bike left, a shortage is recorded.\n", + " station.shortage += 1\n", + " else:\n", + " # Fulfilled the trip requirement.\n", + " station.bikes -= 1\n", + "\n", + " # Generate the BIKE_ARRIVAL_EVENT and insert it into the event buffer.\n", + " bike_arrive_event = self._event_buffer.gen_atom_event(event.tick + generate_duration(), BIKE_ARRIVAL_EVENT, (dest_station_index, 1))\n", + " self._event_buffer.insert_event(bike_arrive_event)\n", + "\n", + " def _on_arrive_station(self, event):\n", + " # The handler for BIKE_ARRIVAL_EVENT, which includes both the ending of a trip and the repositioning of the bikes.\n", + " station_index, number = event.payload\n", + " station = self._stations[station_index]\n", + " station.bikes += number\n", + "\n", + " def _on_action_recieved(self, event):\n", + " # The repositioning action from agent.\n", + " action = event.payload\n", + " \n", + " if action:\n", + " src_station_index, target_station_index, number = action\n", + " station = self._stations[src_station_index]\n", + " station.bikes -= number\n", + "\n", + " # Generate the BIKE_ARRIVAL_EVENT and insert it into the event buffer.\n", + " arrive_event = self._event_buffer.gen_atom_event(event.tick + generate_duration(), BIKE_ARRIVAL_EVENT, (target_station_index, number))\n", + " self._event_buffer.insert_event(arrive_event)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 3. Rule-Based Solution to This Problem" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### 3.1 Problem Analysis" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### 3.1.1 Oracle Policy\n", + "\n", + "I would like to divide-and-conquer the four-node system, i.e, the left subsystem and the right subsystem. The insight is that the two subsystems exchange bikes of the same amount, controlled by the functions $F_{32}$ and $F_{01}$. This indicates that, if the initial bikes can already support the inner requirement loops in each subsystem, we can balance each subsystem separately. \n", + "\n", + "For the left subsystem $\\left\\{s_1, s_3\\right\\}$, the *outer system* gives bikes to $s_1$ and ask out from $s_3$. Therefore, the straight-forward approach is to move bikes from $s_1$ to $s_3$ with the same amount as that $s_1$ receives by the function $F_{01}$.\n", + "\n", + "For the right part, as the requirement curves $F_{20}$ and $F_{02}$ are with the period $20*\\pi$, we consider the policy in each half period. The first half period is really simple because only $s_0$ has requirements. Therefore, the right policy is to simply move all bikes from $s_2$ to $s_0$. In the second half period, as $s_0$ also gives bikes to $s_1$ by the function $F_{01}$, $s_0$ has to reserve a certain amount $R_{01}$ when sending its bikes to $s_2$, instead of send all its bikes to $s_2$ like $s_2$ does in the first half period. For simplicity, the reserved amount $R_{01}$ can be set to be constant.\n", + "\n", + "The code implementation can be seen in `RuleBasedPolicy` class below.\n", + "\n", + "#### 3.1.2 Policy Guided by Statistics\n", + "\n", + "Although the oracle policy solves the problem in simple rules and gains obvious improvement in performance, it is binded to the bike requirement functions. Here, I would like to give a more general solution without building upon the specific settings. \n", + "\n", + "For human beings, the most straight-forward strategy when they knows nothing about the latent requirement curves is to run the system for a while and see which station needs additional bikes. Similarly, I maintain a statistical variable to describe the extent to which one station is in lack of bikes. Each time a station needs bikes, I provide bikes from the station of the least shortage.\n", + "\n", + "As you can see in the implementation, the statistical variable I use is the exponential moving average of the shortage of each station along the ticks. In addition, to prevent the same station being chosen for multiple times in a single tick, I add a penalty to the selected station, which will reduce its ranking in the latter selections." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### 3.2 Implementation" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": {}, + "outputs": [], + "source": [ + "class RuleBasedPolicy:\n", + " def __init__(self, env):\n", + " self.env = env\n", + " self.half_period = 10 * pi\n", + " \n", + " def __call__(self, evt):\n", + " # Determine which half-period current environment is in.\n", + " phase = int(self.env.tick / self.half_period) % 2\n", + " # Get the amount of current bikes in that station.\n", + " cur_bikes = self.env.snapshot_list[\"station\"][self.env.tick::\"bikes\"]\n", + " if evt.station_index == 0 or evt.station_index == 2:\n", + " if phase == 0:\n", + " # Give all the bikes in 2 to 0.\n", + " payload = (2, 0, cur_bikes[2])\n", + " else:\n", + " # Reserve 5 bikes in 0 and give all the remainings to 2.\n", + " payload = (0, 2, max(cur_bikes[0] - 5, 0))\n", + " else:\n", + " # Always reserve 5 bikes in 1 and give all the remainings to 3.\n", + " payload = (1, 3, max(cur_bikes[1] - 5, 0))\n", + " return payload\n", + " \n", + "class NoActionPolicy:\n", + " def __init__(self, env):\n", + " self.env = env\n", + " \n", + " def __call__(self, evt):\n", + " return None" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": {}, + "outputs": [], + "source": [ + "class StatisticsBasedPolicy:\n", + " def __init__(self, env, decay_factor=0.98, weight=0.6):\n", + " self.env = env\n", + " self.factor = decay_factor\n", + " self.weight = weight\n", + " self.tick = env.tick\n", + " self.shortage_emv = np.zeros(TOTAL_STATIONS, np.double)\n", + " self.selected_cnts = np.zeros(TOTAL_STATIONS, np.double)\n", + " \n", + " def __call__(self, evt):\n", + " src_station = evt.station_index\n", + " # Get the tick range from the last action time.\n", + " past_ticks = list(range(self.tick, self.env.tick))\n", + " tick_len = len(past_ticks)\n", + " if tick_len != 0:\n", + " # If it is the start of a new tick.\n", + " past_shortage = self.env.snapshot_list[\"station\"][past_ticks::\"shortage\"].reshape(tick_len, TOTAL_STATIONS)\n", + " decay_vec = np.logspace(0, tick_len - 1, tick_len, base=self.factor)\n", + " self.shortage_emv = np.matmul(decay_vec, past_shortage) + (self.factor**tick_len) * self.shortage_emv\n", + " self.tick = self.env.tick\n", + " self.selected_cnts = np.zeros(TOTAL_STATIONS, np.double)\n", + " \n", + " # Sort the station and get the one of the least shortage and the least selected count.\n", + " candidate_dest_station = np.argmin(self.shortage_emv + self.weight * self.selected_cnts)\n", + " if candidate_dest_station == src_station:\n", + " return None\n", + " else:\n", + " self.selected_cnts[candidate_dest_station] += 1\n", + " dest_sh = self.shortage_emv[candidate_dest_station] + 1e-8\n", + " src_sh = self.shortage_emv[src_station] + 1e-8\n", + " dest_inventory = self.env.snapshot_list[\"station\"][self.tick:candidate_dest_station:\"bikes\"]\n", + " dest_ratio = src_sh / (src_sh + dest_sh)\n", + " payoff = (candidate_dest_station, src_station, int(dest_ratio * dest_inventory))\n", + " \n", + " return payoff" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### 3.3 Interaction with the Environment" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "metadata": {}, + "outputs": [], + "source": [ + "def run_policy(PolicyClass, args=()):\n", + " # Initialize an environment instance with the predefined SimpleCitibikeBusinessEngine.\n", + " env = Env(durations=MAX_TICKS, business_engine_cls=SimpleCitibikeBusinessEngine)\n", + "\n", + " # Get the station attributes from the snapshot_list.\n", + " station_snapshots = env.snapshot_list[\"station\"]\n", + " policy = PolicyClass(env, *args)\n", + "\n", + " is_done = False\n", + " action = None\n", + " while not is_done:\n", + " # Looping until the environment ends.\n", + " metrics, decision_evt, is_done = env.step(action)\n", + " if not is_done:\n", + " # Get the action by calling the policy.\n", + " action = policy(decision_evt)\n", + "\n", + " # Using [tick(s) : node index(s) : attribute(s)] to access the environment snapshots.\n", + " # The return is a Numpy array with shape (ticks * nodes * attributes * slots, )\n", + " all_shortage = station_snapshots[::\"shortage\"]\n", + " all_requirement = station_snapshots[::\"requirements\"]\n", + "\n", + " return {\n", + " \"requirement\": all_requirement.reshape(MAX_TICKS, TOTAL_STATIONS),\n", + " \"shortage\": all_shortage.reshape(MAX_TICKS, TOTAL_STATIONS)\n", + " }\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### Test baseline with no actions" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "metadata": {}, + "outputs": [], + "source": [ + "no_action_result = run_policy(NoActionPolicy)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### Test rule-based policy" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "metadata": {}, + "outputs": [], + "source": [ + "rule_based_result = run_policy(RuleBasedPolicy)" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "metadata": {}, + "outputs": [], + "source": [ + "def grid_search_on_statistic_hyperparameters():\n", + " # simple grid search on the optimal hyper-parameters\n", + " min_rlt, min_decay, min_weight = None, 0, 0\n", + " # decay: 0~0.95\n", + " for i in range(0, 20):\n", + " decay = i * 0.05\n", + " # weight: 0~0.9\n", + " for j in range(0, 10):\n", + " weight = j * 0.1\n", + " rlt = run_policy(StatisticsBasedPolicy, args=(decay, weight))\n", + " if min_rlt is None or rlt[\"shortage\"].sum() < min_rlt[\"shortage\"].sum():\n", + " min_rlt = rlt\n", + " min_decay = decay\n", + " min_weight = weight\n", + " return {\"best_result\": min_rlt, \"decay_factor\": min_decay, \"selection_penalty_weight\": min_weight}" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "metadata": {}, + "outputs": [], + "source": [ + "searched_result = grid_search_on_statistic_hyperparameters()\n", + "statistics_result = searched_result[\"best_result\"]" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### Display the algorithm results" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "" + ] + }, + "execution_count": 18, + "metadata": {}, + "output_type": "execute_result" + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAABCUAAAHoCAYAAAB+RlVfAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8li6FKAAAgAElEQVR4nOzdd1RU194+8OfQQUENGESxxV6iRFCiUWLFgjM3TRNjidf4S8KIgLF3rKioYGFM1cSoyRsTk8wosaBRY6cYoyZi7xUVFOnM/v2Ri9er9DnDYYbns9as974z++z95K4b5pzv7CIJIUBEREREREREVN6slA5ARERERERERJUTixJEREREREREpAgWJYiIiIiIiIhIESxKEBEREREREZEiWJQgIiIiIiIiIkWwKEFEREREREREirBROoBc3NzcRIMGDZSOQURERERERERPSEhISBZC1CzoM4spSjRo0ADx8fFKxyAiIiIiIiKiJ0iSdKmwz7h8g4iIiIiIiIgUwaIEERERERERESmCRQkiIiIiIiIiUgSLEkRERERERESkCBYliIiIiIiIiEgRLEoQERERERERkSJYlCAiIiIiIiIiRbAoQURERERERESKYFGCiIiIiIiIiBTBogQRERERERERKYJFCSIiIiIiIiJSBIsSRERERERERKQIFiWIiIiIiIiISBEsShARERERERGRIliUICIiIiIiIiJFsChBRERERERERIpgUYKIiIiIiKgMhBAwGAxKxyAyayxKEBERERERlUHnzp3x3nvvKR2DyKyxKEFERERERFRK58+fx4EDB7Bu3Tps375d6ThEZotFCSIiIiIiolLS6/UAgNq1ayMoKAhZWVkKJyIyTyxKEBERERERlZJOp0PLli2xZs0anDlzBhEREUpHIjJLLEoQERERERGVQkpKCvbu3Qu1Wg1/f38MGDAA8+bNw4ULF5SORmR2WJQgIiIiIiIqha1btyI3NxcqlQoAsHTpUlhbWyM4OFjhZETmh0UJIiIiIiKiUtDpdKhZsyZ8fX0BAJ6enggLC8PmzZuh0+kUTkdkXliUICIiIiIiKqGcnBzExMSgf//+sLa2fvx+SEgIWrVqheDgYKSnpyuYkMi8sChBRERERERUQvv27UNqaurjpRv5bG1todVqcenSJcyfP1+hdETmh0UJIiIiIiKiEtLpdLC3t0evXr2e+czPzw9Dhw7FokWLkJSUpEA6IvPDogQREREREVEJCCGg0+nQvXt3VK1atcA2ERERcHJyQlBQEIQQ5ZyQyPywKEFERERERFQCf//9N86fPw+1Wl1oG3d3d8ybNw+xsbHYuHFjOaYjMk/lVpSQJGmyJElxkiQ9kCTpjiRJekmSWj/V5itJksRTr0PllZGIiIiIiKgw+Sdr9O/fv8h2H330Edq1a4cxY8bg4cOH5RGNyGyV50yJrgC0ADoB6A4gF0CsJEnPPdUuFoDHE69+5ZiRiIiIiIioQDqdDu3atYOnp2eR7aytraHVanHjxg2EhYWVTzgiM1VuRQkhRG8hxBohxAkhxHEAQwHUBPDKU02zhBA3n3jdK6+MREREREREBbl9+zYOHTpU5NKNJ/n6+mLkyJFYtmwZjh8/buJ0ROZLyT0lnP8z/v2n3u8sSdJtSZJOS5L0uSRJzyuQjcgoV65cgZ+fH/7++2+lo5jE8OHD8emnnyodg4gqiczMTLz11lvYtGmT0X19++23eOedd5CTkyNDMiKqTLZs2QIhRImLEgAQHh6O6tWrIzQ01ITJiMybkkWJZQD+AHDwife2AhgGoAeAsQA6ANglSZJ9QR1IkvSBJEnxkiTF37lzx9R5iUpsw4YN+P333/HRRx9Z3K7LFy5cwNdff43Q0FBcuHBB6ThEVAlERETgxx9/xMiRI2HM9/3169fx4Ycf4v/+7/+wbNkyGRMSUWWg1+vh6ekJLy+vEl/j6uqK0NBQ7Nq1C9evXzdhOiLzpUhRQpKkpQA6A3hTCJGX/74Q4jshhE4IcVwIoQfQF0AzAAEF9SOE+EwI4SOE8KlZs2a5ZCcqCb1eDwcHB+zduxfr169XOo6s9Hr94/8cEhKiYBIiqgwuXLiA+fPno0uXLnj48CEmTZpU5r7GjRuH7OxsdOrUCWFhYbh69aqMSYnIkmVmZmLbtm1QqVSQJKlU1/7rX/8C8M9MCyJ6VrkXJSRJigQwCEB3IcT5otoKIa4DuAqgSXlkI5LDnTt3cODAAYwfPx7t27fHuHHjkJKSonQs2eh0OrRo0QKzZ8+GXq9/vAs1EZEpBAcHw9raGhs2bMCYMWOwevVqHDhwoNT97Nq1C99++y0mTpyIdevWIS8vDx9//LEJEhORJdq1axfS09OhUqlKfW3r1q3RoEED3jMRFaJcixKSJC3DfwsSp0rQ3g1AHQA3TJ2NSC756w1ff/11rFq1Crdv38aMGTOUjiWL1NRU7NmzB2q1GqGhoWjZsiWCg4ORnp6udDQiskA6nQ6bN2/GrFmz4OnpiRkzZsDT0xMajQa5ubkl7ic7OxujRo3CCy+8gEmTJqFhw4aYOnUqNm7ciO3bt5vwn4CILIVer0eVKlXQrVu3Ul8rSRLUajViY2N5z0RUgHIrSkiSFA3g3wDeBXBfkqRa/3lV/c/nVSVJWixJUkdJkhpIktQVgB7AbQA/lVdOImM9ud7Q29sbgYGBiI6ORmJiotLRjLZ161bk5uZCpVLB1tYWWq0Wly5dwvz585WORkQWJj09HcHBwWjVqhWCg4MBAFWrVkVUVBSOHTuG6OjoEve1dOlSnDp1CitWrICjoyMAYPz48WjSpAmCgoKQlZVlkn8GIrIMQgjodDr07t0bDg4OZepDpVIhMzMTsbGxMqcjMn/lOVNCg39O3NiJf2Y+5L/G/efzPAAvAvgFwGkAXwNIAtBRCPGwHHMSlVlB6w3nzp0LV1dXaDQaGAwGhRMaR6fTwc3NDS+//DIA4NVXX8WQIUMQERGB06dPK5yOiCzJvHnzcOnSJWi1Wtja2j5+/4033kDv3r0xffp03LhR/ETKy5cvY86cOXjttdfQr1+/x+/b29tj5cqVOHPmDCIiIkzyz0BEliExMRHXr18v09KNfH5+fnBxceESDqIClFtRQgghFfIK+8/nGUKI3kKI54UQdkKI+kKI4UKIK+WVkchYu3fvxqNHj/7nS6tGjRpYvHgxDh8+jNWrVyuYzjg5OTmIiYlBQEAArK2tH78fEREBR0dHjBo1yuJOGiEiZSQlJSEiIgJDhw6Fn5/f/3wmSRJWrFiBrKwsjBs3rpAe/is0NBRCCERFRT3zmb+/PwYMGIB58+bxNCEiKpRer4ckSQgIKHDv/RKxs7NDnz59sHnzZrP/kYpIbkoeCUpkcXQ6XYHrDYcOHYouXbpg4sSJSE5OViidcfbv34+UlJRnzuauVasW5s6di9jYWGzcuFGhdERkKYQQGDVqFJycnAqdwdCkSRNMmjQJGzZswK5duwrtKyYmBj/99BNmzJiB+vXrF9hm6dKlsLa25mlCRFQonU6HTp06wdjT/tRqNW7duoW4uDiZkhFZBhYliGQihIBery9wvaEkSYiOjkZqaiomT56sUELj6HQ62NnZwd/f/5nPAgMD8dJLL2HMmDF4+JCrrYio7L7//nvs3LkT8+bNg7u7e6Ht8jesHDVqFLKzs5/5PCMjA6NHj0bz5s2LPGXD09MTYWFhPE2IiAp05coVHD161KilG/n69u0La2tr/q0hegqLEkQy+eOPP3D16tVCv7RefPFFhISE4IsvvsChQ4fKOZ1x8jd46t69O6pWrfrM59bW1li1ahVu3LiBWbNmKZCQiCzBw4cP8fHHH6Ndu3b46KOPimzr6OiIFStW4NSpU4iMjHzm84ULF+L8+fOIjo6GnZ1dkX2FhIQ83lCTO+MT0ZM2b94MAM/MFC2L5557Dp07d4Zerze6LyJLwqIEkUx0Ol2x6w3DwsJQu3btUh9np7RTp07h3LlzRX4h+/r6YuTIkYiKisLx48fLMR0RWYqwsDDcuHEDWq32f/auKUxAQABee+01zJ49G5cvX378/tmzZ7FgwQIMGjQI3bt3L7YfniZERIXR6XRo1KgRmjdvLkt/arUax48f5z42RE9gUYJIJiVZb+js7IzIyEgcPXoUq1atKsd0xsmv6Bc3dTE8PBzVq1fnppdEVGrHjx/HsmXL8P/+3/+Dr69via+LioqCEAKhoaEA/pnZNXr0aNjZ2WHJkiUl7sfPzw9Dhw5FREQEkpKSSp2fiCzPw4cPsWvXLqjV6senqhkr/16KsyWI/otFCSIZXL16FYmJiSVabzhgwAD07NkT06ZNw82bN8shnfF0Oh1eeukleHp6FtnO1dUVCxYswO+//45vvvmmnNIRkbkTQkCj0aB69eqlnqlQv359TJ8+HT/99BN+/fVX/PTTT9i6dStmz54NDw+PUvWVf5pQUFAQC6tEhB07diA7O1uWpRv5mjRpgubNm7MoQfQEFiWIZFCa9YaSJGHlypXIyMjA+PHjTR3NaHfu3MGBAwdK/IU8YsQIvPzyyxg3bhzu379v4nREZAnWrl2Lffv2YeHChXB1dS319WPHjkWzZs0QFBSE0NBQtGnTBkFBQaXux93dHfPmzeNpQkQE4J8fZapXr45XXnlF1n7VajV2796N1NRUWfslMleSpfwS4OPjI+Lj45WOQZVUv379cPr0aZw5c6bE0/umTZuGefPm4c0334Stra3RGTp06IAxY8YY3c/Tvv76awwfPhzx8fHw9vYu0TV//PEHvL298dFHHyE6Olr2TESW5LvvvoODgwNee+01k4+VlZWFGTNm/M/+CwWRJAkffPABunbtavJM9+/fR7NmzdC4cWPs27cPVlZl+71k586d6NmzJwBg3759ZX6IyMvLQ4cOHXDz5k2cPn0aVapUKVM/RBWZVqtF48aNCzxRS24PHz7EokWLoNFoSj17qSwSExOxdOlS5OXlGd1XTEwM+vfvj/Xr18uQ7L/27duHLl264LvvvsPbb78ta99EFZUkSQlCCJ8CP2NRgsg4aWlpcHNzg0ajwdKlS0t8XXp6Ot5++22cPn3a6AyZmZm4fPky9Ho9+vfvb3R/T3rzzTdx6NAhXL16tVTrKYODg7Fy5UrExcWVuJhBVNmcPHkSXl5esLGxwcmTJ/HCCy+YdLz58+dj6tSpaNKkSZH/PicnJ0OSJCQlJZVp5kJpaDQafPrpp0hISICXl5dRfU2bNg329vaYPn26Uf3s2rULPXr0wP/93/9h4MCBRvVFVNHkPxC3a9cOCQkJJh9vzJgxiIqKwuuvv45NmzaZdKyMjAy0bt0aycnJqFWrltH92djYQKvV4tVXX5Uh3X/l5eXB3d0dffr0wbp162Ttm6iiKqooASGERby8vb0FkRI2bdokAIjffvtNsQxZWVmiRYsWomHDhiI9PV22fjMyMkSVKlXERx99VOprU1JShLu7u2jfvr3Izc2VLRORpTAYDMLPz0/UqFFDVK1aVQQEBAiDwWCy8S5cuCAcHR3Fm2++WWzbY8eOCWtra/HBBx+YLI8QQsTFxQlJkkRwcLBJxymt3Nxc4ebmJgYPHqx0FCJZ5eTkiBdffFEAEADElStXTDreH3/8IaysrET9+vUFALFlyxaTjjdz5kwBQMTGxpp0HDm89957onr16iI7O1vpKETlAkC8KORZnntKEBnJVOsNS8POzg5arRYXLlxAeHi4bP3u3r0bjx49KtEGnk+rVq0alixZgri4OHzxxReyZSKyFOvXr8fevXuxcOFCzJo1C1u2bIFOpzPZeCEhIbCyskJkZGSxbdu0aYPg4GB8/vnnOHLkiEny5OXlQaPRwN3dHbNnzzbJGGVlbW2NgIAAxMTEmNXxzUTFWbFiBY4fP465c+cC+O+eWKZgMBig0Wjg6uqKw4cPo3nz5hg9ejQyMjJMMt65c+ewYMECvPPOO+jRo4dJxpCTSqVCSkoK9u/fr3QUIuUVVq0wtxdnSpAScnNzRc2aNcW7776rdBQhhBCDBw8WdnZ24vTp07L0FxgYKJycnERGRkaZrjcYDKJr166iRo0a4vbt27JkIrIE9+/fF+7u7sLX11fk5eWJ7Oxs0bp1a1G/fn2RlpYm+3g6nU4AEIsWLSrxNampqcLDw0O0a9fOJLOdVq1aJQCI9evXy963HH788UcBQOzevVvpKESyuHr1qqhataro16+fMBgMolGjRqJv374mG2/16tUCgFizZo0QQohdu3YJAGLmzJmyj2UwGESfPn2Es7OzuHbtmuz9m8KDBw+EnZ2d+Pjjj5WOQlQuUMRMCcWLCXK9WJQgJezfv18AEN99953SUYQQQty4cUO4uLgIf39/o6eBGwwG4enpKV5//XWj+jl58qSwsbERI0aMMKofIksSFBQkrKysREJCwuP39u7dKwCIyZMnyzrWo0ePRIMGDUTLli1LPU3422+/FQDEypUrZc10+/ZtUaNGDdGtWzeTLlkxxsOHD4WdnZ0YO3as0lGIZPH2228Le3t7cfbsWSGEEKGhocLe3l48fPhQ9rHu3r0r3NzcxCuvvCLy8vIevz9o0CBhb28vzpw5I+t4+UXEpUuXytqvqfXp00c0bty4wv4dJJITixJEJjJp0iRhY2MjUlJSlI7y2PLlywUAsXHjRqP6SUxMFADE6tWrjc40YcIEAUDs37/f6L6IzF1CQoKwsrISQUFBz3z23nvvCVtbW/H333/LNt60adPK/Iu/wWAQPXr0ENWqVRM3b96ULdO///1vYWNjI06ePClbn6bABwayFLGxsQKACAsLe/xe/syFTZs2yT7ehx9+KKytrcWxY8f+5/3r168LZ2dn0adPH9n+vUpLSxN169YVbdq0ETk5ObL0WV6io6MFAFn/5hNVVCxKEJlIy5YtRY8ePZSO8T9ycnKEl5eXqFOnjlG/foSFhQlJksStW7eMzvTw4UPh6elpljcMRHLKy8sTvr6+wt3dXdy/f/+Zz2/duiWqV68uevToIcsN++nTp4WdnZ0YMmRImfs4deqUsLW1FcOGDTM6jxBC7Nu3TwAQEydOlKU/U+IDA1mCzMxM0axZM9GoUaP/WY6ZnZ0tqlWrJoYPHy7reEeOHBGSJIkxY8YU+HlUVJQAIH788UdZxps4caIAIPbt2ydLf+Xp8uXLAoBYuHCh0lGITI5FCSITOHv2rAAgoqKilI7yjAMHDggAYvz48WXuw9vbW3Ts2FG2TD/88EOF/e+LqLx8/vnnAoBYu3ZtoW3yH4SNXRZmMBiEv7+/cHFxETdu3DCqr8mTJwsAYs+ePUb1k5OTI9q0aSPq1q1rkr0z5Jb/wFCavTiIKpr58+cLAOLXX3995rNBgwaJmjVryrZvTG5urvD29hYeHh4iNTW1wDY5OTmibdu2om7dukYvHclfIip3YaU8eXl5iVdeeUXpGEQmx6IEkQlERkYKAOLcuXNKRynQ+++/L2xsbMSJEydKfe3Vq1cFABEeHi5bnic3obp+/bps/RKZizt37ojnnntO+Pn5FTkLoiQ39SXx/fffCwBi+fLlZe4jX1pamqhXr55o1aqVUcfX5f/dlOsX0vLw0ksvic6dOysdg6hM8o8CfuONNwr8PH/fGLmWV+YXVb/99tsi2+XvyTVhwoQyj5W/mXb16tVlmdWplBkzZggrKytuCE4Wj0UJIhPo1q2baN26tdIxClXSB6CCfPLJJwKA7Ou9z5w5I+zt7SvMaSVE5WnkyJHCxsZGHD9+vNi2hw8fLnL6c3EePHgg6tSpI1566SXZlkz9/PPPAoBYvHhxma6/du2acHZ2Fn379jWrPRryHxju3LmjdBSiUvvXv/4lqlSpIi5fvlzg5/fv3xc2NjayLKcq7fKzESNGGLW3zPr16wUAsWrVqjJdX1HEx8cLAOKrr75SOgqRSbEoQSSze/fuCWtra9l3yZfbZ599JgCIb775plTX9evXT7zwwgsmeXCYMWOGACB27dole99EFdXBgwcFADFu3LgSX1PYRnElMW7cOAFAHDx4sNTXFsZgMIiAgABRtWpVceXKlVJfn7/rfv7O/+Yi/4Hh66+/VjoKUano9foS7VfQvXt30bJlS6PHy9+o99SpUyVqn38KT9euXUt9v5GSkiJq1aolfHx8THJkcXkyGAyidu3ahc5mIbIULEoQyWzDhg2y3/CbQv6mes8//3yBm+oVJC0tTdjb24vQ0FCTZEpPTxcNGzYUzZs3F1lZWSYZg6giKevms4UdqVec48ePC2trazFy5MiyxC3SuXPnhIODgxgwYECprsvf+X/mzJmyZzK1/AeGN998U+koRCWW/13bokWLYr9r8zeeNKZgWNYjjfNnZq5bt65U1wUHBwtJkkRcXFyprquoPvzwQ1GlSpX/2YiUyNKwKEEks3feeUc8//zzpXpQUEpRxw8W5KeffjL5TIYtW7YIAGLBggUmG4OoojDmmN7Vq1cLAGLNmjUlam8wGISfn59wdXUVycnJpR6vJGbPni0AiG3btpWofVZWlmjevLlo1KiRSE9PN0kmU/vwww9F1apVRWZmptJRiEpk+vTpAoD47bffim177tw5AUBERkaWaazs7GzRunVrUb9+ffHo0aNSXZubmys6dOgg3N3dS3y8+tGjR4WVlZUIDAwsS9wKKf++qKDNSIksBYsSRDLKP0JrxIgRSkcpsaCgIGFlZSUSEhKKbfvvf/9bVKtWzajN7EritddeE05OTuLSpUsmHYdISTdu3BAuLi7C39+/TMuh8vLyRKdOnUTNmjXFvXv3im2/du1aAUB8/vnnZYlbIhkZGaJx48aiSZMmJXpIDw8PFwBETEyMyTKZWv4Dw9atW5WOQlSs/KOABw8eXOJrWrVqJbp161am8ZYsWSIAiJ9//rlM18fHxwtJkkRwcHCxbfPy8kTHjh1L/DfRXGRkZAgnJyeLKrQQPa2oooT0z+fmz8fHR8THxysdgyqw33//HQ0bNoSnp6dR/ezatQs9evTAzz//jH/9618ypTOtlJQUNG/eHB4eHhg6dGiRbcPDw9GrVy9s2LDBpJkuXbqEFi1awNfXFyqVyuj+rKys8M4776BWrVoypCOSx5AhQ7Bx40acOHECTZo0KVMfx44dQ7t27dCvXz9069atyLYLFy7ECy+8gP3798PKyqpM45XEtm3b0KdPHwwePBjt2rUrtF1eXh7CwsLQu3dvbNq0yWR5TC0zMxOurq4YPnw4oqOjlY5DVCghBPr06YNDhw4hKSmpxN+JU6ZMwaJFi3Dnzh3UqFGjxONdu3YNzZs3x6uvvgq9Xg9JksqUe9SoUfjkk08wa9YsODk5Fdru3Llz0Gq1WLNmDYYPH16msSqq119/HfHx8bh8+XKZ/3skqsgkSUoQQvgU+GFh1Qpze3GmBBXl2LFjwtraWrRp08aonejzj7V0cXERaWlpMiY0ve+//17Y2toKAEW+JEkSOp2uXDItX75cSJJUbKaSvkaNGlUuuYlK4rfffhMAxLRp04zuK38qdnGvatWqicTERBnSF2/EiBElylSrVi2LmBH12muvibp165rVySFU+WzcuFEAEMuWLSvVdQcOHBAAxIYNG0p13cCBA4WDg4PRx6Pfu3dPNG7cuER/U/r27WsWy2dLa926dSU6TpXIXIEzJagyMxgM8PPzQ0JCAjIzMxEZGYnQ0NAy9bVp0ya8+eabWLZsGYKDg2VOanrp6enIzc0tso2NjU2Rv1LI7dGjR8jLyzO6n8GDB+PPP//ExYsX+QsDKS4nJwdeXl7IyMjAyZMn4ejoaHSfaWlpMBgMRbaxt7eHvb290WOV1IMHD4pt4+DgADs7u3JIY1pr1qzBiBEjcPToUXh5eSkdh+gZaWlpaN68OWrWrIm4uDjY2NiU+Nq8vDzUrl0b3bt3x7fffluia2JjY9GrVy/Mnj0b06dPL2vsx3Jzc5Genl5sO2dnZ4v8ns/Ly4Ovry+uX7+OU6dOwcXFRelIRLIqaqZEyf9aEZmptWvXYv/+/fjyyy/xww8/YMaMGRg4cCBq165dqn7S0tIQEhICLy8vaDQaE6U1rfIsNpRUlSpVZOnn9ddfx+bNm/Hnn3+ibdu2svRJVFZRUVH466+/oNfrZSlIAEDVqlVl6UdOlemmOSAgAJIkQafTsShBFdLs2bNx7do1bNy4sVQFCQCwtrZGQEAANm3ahJycHNja2hbZPisrC6NGjULjxo0xfvx4Y2I/ZmNjU6n+pjzN2toaq1atgq+vL2bOnInIyEilIxGVG9MtOCWqAO7du4fx48ejU6dOGD58OFasWIHs7GyMHTu21H3NmTMHV69ehVarLfWXPZnekw8MREq6cuUKwsLCoFar0b9/f6XjkEyef/55vPzyy/wbQxXSyZMnERkZiffffx8dO3YsUx9qtRqpqan4/fffi227ePFinD59GitXroSDg0OZxqNntW/fHh988AFWrFiBP//8U+k4ROWGRQmyaFOnTsW9e/eg1WphZWWFRo0aYdKkSfjuu++wc+fOEvfz119/YenSpRgxYkSZv+zJtNzd3eHr6wu9Xq90FKrkxowZAyEEli1bpnQUkplarUZCQgKuXbumdBSix4QQ0Gg0cHFxwYIFC8rcT69evWBvb19s4e3ixYuYN28e3nzzTfTu3bvM41HB5s+fjxo1akCj0RS7ZI/IUrAoQRYrLi4On376KUaPHv0/0/knTpyIF154AaNGjUJ2dnax/Tz5Zb9w4UJTRiYjqVQqxMXF4fr160pHoUpq69at+PHHHzFt2jQ0aNBA6Tgks/yTgjZv3qxwEqL/Wr9+Pfbu3YsFCxbAzc2tzP1UqVIFPXr0gE6nQ1F7zoWEhMDKyorLC0zkueeew6JFi7B//36sXbtW6ThE5YJFCbJIeXl50Gg0qFWrFmbPnv0/nzk6OmLlypVISkrCkiVLiu1rw4YN2LNnD8LDw436sifTU6vVAPjAQMrIzMxEUFAQmjZtWqYlYlTxtWzZEi+88AJnZFGFkZKSgrFjx8LX1xfvv/++0f2p1WpcuHABf/31V4Gf6/V66HQ6zJw5E3Xr1jV6PCrYe++9h06dOmH8+PG4d++e0oy0HywAACAASURBVHGITI5FCbJIn332GeLj47FkyZICN03q27cvXn/9dcyZMweXLl0qtJ/U1FSMHTsWHTp0wMiRI00ZmWTQqlUrNGzYkA8MpIhFixbh3LlziI6OLtcTMKj8SJIEtVqN2NhYPHr0SOk4RJg+fTqSk5MfL1M1Vv4+OAUt4UhPT0dwcDBatmxZ5lPMqGSsrKyg1Wpx7949TJ06Vek4RCbHogRZnNu3b2PKlCno1q0b3nnnnULbRUVFQZKkIr9Yp0+fjtu3b8v2ZU+mJUkSVCoVHxio3J0/fx7h4eEYOHAgevbsqXQcMiGVSoWsrCzs2LFD6ShUySUmJkKr1SIwMBDt2rWTpc86derA29u7wOJ+eHg4Ll68iOjo6GJP5yDjtW3bFqNHj8ann36KuLg4peMQmRSfssjiTJw4EY8ePUJ0dHSR51jXq1cPM2bMwM8//4wtW7Y88/nRo0cRHR0NjUYDb29vU0YmGanVamRmZiI2NlbpKFRJCCEwevRo2NjYYOnSpUrHIRPr0qULqlWrxhlZpCiDwQCNRgM3NzfMnTtX1r7VajUOHTqEW7duPX7v9OnTWLRoEYYMGYKuXbvKOh4Vbvbs2ahVqxYCAwORl5endBwik2FRgizKvn378NVXX2Hs2LFo0aJFse3HjBmDFi1aYPTo0cjIyHj8vim/7Mm0/Pz8+MBA5eqXX35BTEwMZs2ahTp16igdh0zM1tYW/fr1g16v50MCKebLL7/E4cOHsXjxYlSvXl3WvtVqNYQQj3+wEUIgKCgIDg4OiIiIkHUsKpqLiwuWLFmChIQEfPbZZ0rHITIZFiXIYuTm5kKj0aBevXqYNm1aia6xs7NDdHQ0Lly48D/HaK1evRqHDh1CRESE7F/2ZFq2trbo06cP9Ho9j9Iik3v06BFCQkLQunVrjB49Wuk4VE5UKhXu3LmDI0eOKB2FKqHk5GRMmjQJfn5+GDJkiOz9t23bFnXr1n1c3P/hhx+wY8cOzJ07F7Vq1ZJ9PCraO++8g27dumHKlCm4ffu20nGITIJFCbIYK1aswPHjxxEVFYUqVaqU+Lpu3bph0KBBWLhwIc6cOYO7d+9i0qRJ6NKlC4YOHWrCxGQqarUat2/f5gMDmdy8efNw+fJlaLVarrGuRPr06QMbGxvOyCJFTJo0CampqcUuUy2r/P2Ztm/fjjt37mDMmDHw8vJCYGCg7GNR8SRJQnR0NB49eoSJEycqHYfIJFiUIItw/fp1zJw5E/369cNrr71W6uuXLFkCOzs7jB49GpMmTUJKSgq0Wq1JvuzJ9Pr27Qtra2s+MJBJnTp1CosXL8Z7772HLl26KB2HylGNGjXQpUuXAk8oIDKlgwcP4ssvv8SYMWPQunVrk42jUqmQnp6OgIAAXLt2DatWrYKNjY3JxqOitWjRAmPHjsVXX32Fffv2KR2HSHaSEELpDLLw8fER8fHxSscghQwaNAg//fQTTp48iUaNGpWpj2XLlj0+iWPs2LFYvHixnBGpnHXr1g3Jyck4fvy40lHIzGRnZyM+Ph65ublFtps5cyb++OMPJCUl4fnnny+ndFRRREVFYcyYMdi0aRNcXV0LbSdJEjp06MBjYsloubm5aN++Pe7cuYNTp06hatWqJhsrKysLbm5uSEtLw8iRI/H555+bbCwqmUePHqFly5aoVq0aVq5cWWRbSZLg7e0NJyenckpHVDxJkhKEED4FfsaiBJm7nTt3omfPnggLC8PMmTPL3E/+l31ycjL++usvODs7y5iSyltkZCQ+/vhjnD9/Hg0bNlQ6DpmRiIgITJgwoURt84/jo8rn/PnzaNy4MUpyH9WtWzfs3LmTs+/IKMuXL0dISAi+//57DBgwwOTjvfPOO4iNjcWpU6fg5uZm8vGoeD///DNef/31ErUNDQ1FZGSkiRMRlRyLEmSxsrKy0LZtW+Tm5uLEiRNwcHAwqr+0tDRkZWUV+asXmYdz586hcePGWLZsGYKDg5WOQ2akY8eOSE9PL/Zmrlq1ajwuuJL7888/kZycXGSbvXv3YtasWVi3bh0GDx5cTsnI0ty4cQPNmzfHyy+/jK1bt5ZLgSs1NRVpaWk8VaiCOXHiRLEbXs6dOxfnz5/HhQsXWAylCoNFCbJY4eHhmDJlCn799Vf06dNH6ThUwbRs2RK1a9dGbGys0lHITNy8eRO1a9fG7NmzS3yKD1FR8vLy0LFjR1y+fBlJSUmoVq2a0pHIDA0ZMgQbN27EiRMn0KRJE6XjUAX35ZdfYuTIkTh27BjatGmjdBwiAEUXJbjRJZmtS5cuYc6cOXjjjTdYkKACqdVq7NmzB6mpqUpHITOxZcsWCCGgUqmUjkIWwtraGlqtFrdv38aMGTOUjkNm6LfffsP69esxYcIEFiSoRAICAgCAm/GS2WBRgsxWSEgIJElCVFSU0lGoglKr1cjNzcXWrVuVjkJmQq/Xo169evxliWTl4+ODwMBArFy5EkePHlU6DpmR7OxsjBo1Cg0aNMDkyZOVjkNmolatWvD19eUpZGQ2WJQgs7Rlyxb88ssvmDlzJurWrat0HKqgfH194ebmxl8KqEQyMjKwfft2qNVqrsEl2c2dOxeurq7QaDQwGAxKxyEzERUVhb///hsrVqzgSQpUKmq1GkeOHMGNGzeUjkJULBYlyOxkZGRg9OjRaNGixeMjPIkKYm1tjf79+yMmJgY5OTlKx6EKbufOncjIyODSDTKJGjVqICIiAocOHcKaNWuUjkNm4MqVK5g1axbUajX69++vdBwyM/nfZZs3b1Y4CVHxWJQgs7NgwQJcuHABWq0WdnZ2SsehCk6lUiElJQX79+9XOgpVcHq9Hs7Oznj11VeVjkIWatiwYejSpQsmTpyIu3fvKh2HKrjQ0FAIIbBs2TKlo5AZat26NRo0aMAlHGQWWJQgs3LmzBksWLAA7777Lrp27ap0HDID/v7+sLOz4xIOKpLBYIBer0efPn1gb2+vdByyUJIkITo6GikpKdwfgIq0detWbNq0CVOnTkWDBg2UjkNmSJIkqNVq7NixA+np6UrHISoSixJkNoQQGD16NBwcHLB48WKl45CZqFq1Knr06AGdTgdLOQKZ5JeYmIgbN25w6QaZ3IsvvoiQkBB88cUXOHz4sNJxqALKzMxEUFAQmjZtinHjxikdh8yYSqVCZmYmj0anCo9FCTIbmzZtwrZt2zBnzhx4eHgoHYfMiEqlwrlz53Dq1Cmlo1AFpdPpYGVlhX79+ikdhSqBsLAweHh4IDAwEHl5eUrHoQpm4cKFOHfuHKKjozlzi4zi5+cHFxcXLuGgCo9FCTILaWlpCAkJgZeXFzQajdJxyMzk//rNJRxUGJ1Oh86dO8PV1VXpKFQJODs7IzIyEkePHsWqVauUjkMVyLlz5xAeHo6BAweiZ8+eSschM2dnZ4e+fftCr9fz1B+q0FiUILMwe/ZsXLt2DVqtFjY2NkrHITPj6emJdu3a8ZcCKtDly5dx7NgxLt2gcjVgwAD07NkT06ZNw61bt5SOQxWAEALBwcGwtbXF0qVLlY5DFkKlUuHWrVuIi4tTOgpRoViUoArv5MmTiIyMxPvvv4+OHTsqHYfMlEqlwoEDB3Dnzh2lo1AFk1+sUqvVCiehykSSJKxcuRLp6ekYP3680nGoAvjll18QExODWbNmoU6dOkrHIQvRt29fWFtb84cZqtAkS9n4zcfHR8THxysdo1LIy8tDdnZ2se0cHBwgSZJRYwkh0K1bNxw/fhxJSUlwc3Mzqj+qvBITE+Ht7Y3PPvsMQ4YMKbKtvb09rKxYs61IhBBG/z0pTO/evXHp0iXuOUKKmDZtGubNm4fdu3fzOFoLJYRAZmZmkW0yMjLw0ksvwcXFBYmJibC1tS2ndFQZdOvWDXfv3sWff/6pdBSSwa1bt+Du7q50jFKTJClBCOFT0Ge866ZSuXnzJho1agQnJ6diX8OHDzd6vPXr12PPnj0IDw9nQYKM8tJLL8HT0xMffPBBsf/bbdu2LdLS0pSOTP9x9uxZPP/889i8ebPsfT948AC//fYbl26QYqZMmYL69evzlAULNmjQoGK/d1xdXXH58mVotVoWJEh2KpUKx48fx8WLF5WOQkZ68OAB6tWrh0WLFikdRVZcnE+lMn78eNy4cQNz5swp8kvzxIkTWLt2LQYMGID+/fuXaayUlBSMGzcOHTp0wMiRI8samQjAP1Olv/vuO+zbt6/IdmlpaZg7dy7mzJmDhQsXllM6KowQAkFBQUhOTsY333xT5r8nhdm+fTtycnK4dIMU4+TkhNGjR2PcuHG4dOkS6tevr3QkktHDhw/x008/wd/fH927dy+ybevWrdGlS5dySkaViUqlwtixY6HX6zF69Gil45ARtm/fjuzsbItb0s7lG1Rie/bsQdeuXTFt2jTMmTOnyLbZ2dnw8vJCZmYmTp48CUdHx1KPFxwcjOjoaBw5cgTe3t5ljU1Uau+//z7Wrl2LY8eOoWXLlkrHqdR+/PFHvPXWW3B3d0dGRgbu3LkDOzs72fofNmwYtmzZglu3bnETXVLMmTNn0LRpU6xYsQJBQUFKxyEZ5f8N27NnD/z8/JSOQ5VYixYt4OnpiR07digdhYwwbNgwxMTE4ObNm2Z338LlG2S0nJwcaDQaNGjQAJMnTy62vZ2dHbRaLS5cuIDw8PBSj3f06FFER0cjMDCQBQkqdwsWLICzszNGjRoFSyncmqO0tDSEhobCy8sLWq0WDx48wN69e2XrPzc3FzExMQgICDC7L3ayLE2aNEHz5s15bLEF0ul0qFGjBjp16qR0FKrk1Go19uzZg9TUVKWjUBnl37f069fP4u5bWJSgEomKisJff/2F5cuXw8nJqUTXdO3aFe+++y4WLlyIM2fOlHgsg8GAwMBAuLm5Ye7cuWWNTFRmNWvWRHh4OHbv3o0NGzYoHafSmj17Nq5evQqtVos+ffrAwcFB1t3DDx48iLt373LpBlUIKpUKu3fvxoMHD5SOQjLJy8vDli1bWPikCkGtViMnJwfbtm1TOgqVkSXft7AoQcW6evUqZs2aBbVaXerN4BYvXgwHBwcEBQWV+Bfn1atX4/Dhw1i8eDGqV69elshERhs5ciTat2+PsWPH8lcFBTx9FLCTkxN69uwJnU4n2+wVnU4HW1tb+Pv7y9IfkTH4wGB5LPkBgszPyy+/DDc3N87IMmM6nQ52dnbo3bu30lFkx6IEFWvMmDEwGAxYtmxZqa/18PDAnDlzsH37dvz444/Ftk9OTsbEiRPh5+dX7LGNRKZkbW2NVatW4fbt25gxY4bScSoVIQRGjRoFFxcXLFiw4PH7arUaFy9exIkTJ2QZR6/Xo1u3bnBxcZGlPyJjdOzYEa6urnxgsCD5hU9LfIAg82NtbY2AgADExMQgNzdX6ThUBnq9Hl27doWzs7PSUWTHogQVadu2bfjhhx8wdepUNGjQoEx9aDQaeHl5ITQ0FA8fPiyy7eTJk5Gamoro6GhIklSm8Yjk4u3tjcDAQKxcuRJHjx5VOk6lUdhRwPknb8ixhOP06dNISkriL5hUYeQ/MGzZsoUPDBZCp9Oha9euLHxShaFWq3H//n3s379f6ShUSklJSRZ938KiBBUqMzMTQUFBaNq0qVHnp9vY2ECr1eLatWuYPXt2oe0OHTqEL774AqGhoWjdunWZxyOS09y5c+Hq6gqNRgODwaB0HItX1FHAHh4eaN++vSy/JOcXNuQ+YpTIGPkPDAcOHFA6ChmJhU+qiPz9/WFnZ8cZWWbI0u9bWJSgQkVERODs2bOIjo6Gvb29UX117NgR77//PqKionDy5MlnPs/Ly4NGo0GdOnUwc+ZMo8YiklONGjUQERGBQ4cOYc2aNUrHsXgzZszA7du3odVqYWX17FeUWq3G4cOHcfPmTaPG0el0aNu2LerXr29UP0Ry4gOD5ch/gCjtXlxEplS1alV0795d1v2ZqHzo9XqLvm9hUYIKdP78ecyfPx8DBw5Ez549ZelzwYIFcHFxgUajeeYP4apVq3D06FFERkZa5DopMm/Dhg1Dly5dMHHiRNy9e1fpOBYrMTER0dHR0Gg0hR4FnP+r45YtW8o8zt27d7Fv3z7+gkkVjrOzM7p168aihAXQ6/Vo06aNxT5AkPlSq9U4e/YskpKSlI5CJVQZ7ltYlKBnCCEQHBwMGxsbLF26VLZ+3dzcEB4ejr1792LdunWP37958yamTp2KXr164a233pJtPCK5SJKE6OhopKSkYPLkyUrHsUgGgwEajabYo4BffPFF1KtXz6iHtl9//RUGg4G/YFKFpFarcebMGT4wmLHK8ABB5it/+j+Ln+YjJibG4u9byq0oIUnSZEmS4iRJeiBJ0h1JkvSSJLV+qo0kSVKYJEnXJUnKkCRptyRJrcorI/1Dp9Nhy5YtmDVrFurUqSNr3yNHjkSHDh0wbtw4pKSkAADGjx+PzMxMrFy5kptbUoX14osvIiQkBF988QUOHz6sdByLk38UcERERJFHAUuSBLVajR07diAjI6NMY+l0Onh4eBQ6G4NISXxgMH+//vor8vLyLPoBgsxX3bp18dJLL8myaTSVD71eb/H3LeU5U6IrAC2ATgC6A8gFECtJ0nNPtJkAYCyA0QDaA7gNYIckSZzPX04ePXqEkJAQtG7dGqNHj5a9fysrK6xatQrJycmYPn069uzZg3Xr1mHChAlo2rSp7OMRySksLAweHh4IDAxEXl6e0nEsRv5RwF26dMHQoUOLba9Wq5GRkYGdO3eWeqzs7Gxs3boVKpWqwD0riJRWr149eHl5sShhxvR6PWrVqgUfHx+loxAVSK1W48CBA7hz547SUagYWVlZ2Lp1K/r372/R9y3l9k8mhOgthFgjhDghhDgOYCiAmgBeAf6ZJQEgFMACIcSPQogTAN4D4Azg3fLKWdnNmzcPly5dglarha2trUnGaNeuHQIDA6HVavHee++hQYMGnBJPZsHZ2RlLly7F0aNHsWrVKqXjWIz8o4C1Wm2JZku9+uqrcHZ2LtND2549e/Dw4UP+gkkVWv4DQ3JystJRqJSys7Px66+/WvwDBJk3lUoFg8GAmJgYpaNQMfLvWyx9OZik1M6rkiR5ALgOoIsQYp8kSS8AOAeggxAi7ol2WwAkCyHeK6o/Hx8fER8fb9LM5mzbtm2YNGkScnJyimyXlJSEwYMH46uvvjJpnpSUFDRr1gy3b9+GXq+32ONtyPIIIeDv74+4uDhcvHixyKUGVLzExER4e3tj7NixWLx4cYmvGzhwIPbt24erV6+W6sb/ww8/xDfffIO7d+/C0dGxLJGJTC4+Ph7t27fH119/jWHDhikdh0phx44d8Pf3h06nY/GTKiwhBDw9PZGZmQkPDw+j+3N2dsZPP/2EWrVqyZCOnjR69Gh8+eWXFnHfIklSghCiwClkNuUd5gnLAPwB4OB//v/8/xXfeqrdLQAFbmwgSdIHAD4A/pnuSAVLTU3F8OHD4eDgUOxapE6dOmH+/Pkmz1S9enVs3LgRcXFxLEiQWZEkCVOmTEH37t1x8OBB9O3bV+lIZm39+vWws7PD9OnTS3WdWq3Gxo0bkZCQgPbt25fompMnT2L16tX497//bfZf7GTZ2rVrh9q1a0On07EoYWb0ej0cHR3Ro0cPpaMQFUqSJCxevBg//vij0X3l5ubil19+wcaNG02y9LsyE0JAp9OhV69eFn/fokhRQpKkpQA6A+gshCjzwmwhxGcAPgP+mSkhUzyLM2PGDNy6dQtHjhypUOsb/fz84Ofnp3QMolLz9vaGJEmIj49nUcII+V+23bp1Q7Vq1Up1bd++fWFlZQWdTleiooQQAhqNBi4uLuVSeCUyhpWVFVQqFdavX4+srCzY29srHYlKIP9vWs+ePeHk5KR0HKIiDRo0CIMGDZKlr+bNm0Ov17MoIbPjx4/j8uXLmDFjhtJRTK7cF7tJkhQJYBCA7kKI8098dPM//9f9qUvcn/iMSumPP/7AypUr8dFHH1WoggSROXNxcUGzZs3AJWPGSUpKwtmzZ8u0TtLV1RWdO3cu8e7h69evx969exEeHg43N7dSj0dU3lQqFdLS0rB7926lo1AJHT9+HJcuXbL4td9ET1OpVNi9ezcePHigdBSLkr93VkBAgMJJTK9cixKSJC3DfwsSp576+AL+KT70eqK9A4AuAA6UW0gLYjAYoNFo4Orqinnz5ikdh8iieHt7syhhpPyCQlmXcKnVahw7dgyXLl0qsl1KSgrGjh2LDh06YOTIkWUai6i8de/eHU5OTjyFw4wY+zeNyFyp1Wrk5ORg27ZtSkexKDqdDr6+vpVir45yK0pIkhQN4N/45ySN+5Ik1frPqyoAiH923IwCMFGSpDckSWoN4CsAaQA2lFdOS/LVV1/h4MGDiIiIQI0aNZSOQ2RRfHx8cP36ddy4cUPpKGZLp9PBy8urzHsC5W8iV9xsienTpyM5ORmrVq3ibvhkNhwdHdGrVy/o9XootSk5lY5Op0OHDh0qxQME0ZM6duwIV1dXFlFldOPGDcTFxVWamVfleXemwT/He+4EcOOJ17gn2iwCEAkgGkA8AA8A/kKIh+WY0yLcvXsXEyZMQOfOnblJFpEJ5C+HSkhIUDiJeUpOTsaBAweM2p2+adOmaNasWZFFicTERGi1WgQGBqJdu3ZlHotICWq1GleuXMGxY8eUjkLFuHHjBo4cOVJpHiCInmRjY4N+/fphy5YtyM3NVTqORdi8eTMAVJpTfMqtKCGEkAp5hT3RRgghwoQQHkIIByHEq0KIE+WV0ZJMmTIFKSkp0Gq1kCRJ6ThEFsfLywtWVlZcwlFGMTExMBgMRt/Aq9Vq/PbbbwWuY81fwubm5oa5c+caNQ6REgICAiBJEn99NANbtmwBABYlqNJSq9W4f/8+Dhzgqns56HQ6NGjQAK1bt1Y6SrngPFYLdOTIEXz++ecIDg7Giy++qHQcIotUtWpVNG/enEWJMtLpdKhdu7bRsxdUKhVycnKwffv2Zz778ssvcfjwYSxevBjVq1c3ahwiJbi7u8PX17fEG7qScnQ6HerXr19pHiCInta7d2/Y2dmxiCqD9PR0xMbGQq1WV5ofl1mUsDB5eXkIDAyEh4cHwsLClI5DZNF8fHyQkJDA9d6llJWVhW3btqF///5G7/FQ2DrW5ORkTJo0CX5+fhgyZIhRYxApSa1WIz4+HteuXVM6ChUiPT0dO3bsqFQPEERPc3Z2RteuXVlElUFsbCwyMzMrzdINgEUJi/PJJ58gMTERS5cuhYuLi9JxiCyaj48Pbt68ievXrysdxazs3r0baWlpskxzLmwd6+TJk5Gamoro6Gg+JJBZy//3JH99MVU8O3fuRGZmJpduUKWnVqtx+vRpJCUlKR3FrOl0Ori4uMDPz0/pKOWGRQkLcuvWLUydOhU9e/bEwIEDlY5DZPHyN7vkEo7S0el0cHJyQvfu3WXpT61W4969ezh48CAA4ODBg/jiiy8QGhrKqdRk9lq2bImGDRvy18cKrDI+QBAVJP84XC7hKDuDwYDNmzejT58+sLOzUzpOuWFRwoJMmDAB6enpWLlyJX8ZJCoHbdu25WaXpSSEgF6vR69eveDo6ChLn0+uY83NzYVGo0GdOnUwc+ZMWfonUpIkSVCr1YiNjcWjR4+UjkNPqawPEEQFqV+/Ptq2bcsiqhHi4uJw69atSjfzikUJC7F3716sXbsW48ePR7NmzZSOQ1QpODk5oVWrVjwWtBSOHTuGK1euyPplm7+OVafTYdWqVfjjjz8QGRkJZ2dn2cYgUpJarUZWVhZ27NihdBR6Snx8PG7evFnpHiCICqNWq7F//34kJycrHcUs6XQ6WFtbo2/fvkpHKVcsSliAnJwcaDQa1K9fH1OnTlU6DlGl4uPjg/j4eG52WUI6nQ6SJCEgIEDWfvPXsU6cOBG9evXCW2+9JWv/RErq0qULqlWrxl8fK6DK+gBBVBiVSgWDwYCYmBilo5glvV6Pzp0747nnnlM6SrmyUToAGW/lypU4efIkfv75Zzg5OSkdh6hS8fHxwZo1a3DlyhXUq1dP6TgVnl6vh6+vL9zd3WXtV6VSISgoCHl5eVzCRhbH1tYWffv2hV6vR25uLmxsePtmjAsXLmDBggXIysoyuq/t27dXygcIosJ4e3vDw8MDer0ew4YNk73//fv3IykpCSNGjJC9b6VdvHgRx48fx5IlS5SOUu74rWYBPvnkE7z66qucOkikgCc3u2RRomjXr19HfHw85s+fL3vf9erVw7Bhw9CuXTs0bdpU9v6JlPbuu+/iu+++g1arRXBwsNJxzJbBYMCQIUOQmJgoS3HUwcEBGo1GhmRElsHKygoqlQobNmxAVlYW7O3tZes7JSUFb7zxBm7fvo0mTZqgS5cusvVdEeTPhqtMR4HmY1HCzCUlJeH06dMIDg7mL4NECmjTpg1sbGyQkJCAN954Q+k4FVr+kYam+rL9+uuvTdIvUUXQv39/+Pv7Y/r06RgwYAA8PDyUjmSWvv76axw4cABr1qzB8OHDlY5DZJFUKhU+++wz7NmzB/7+/rL1O23aNCQnJ8Pd3R0ajQaJiYmwtbWVrX+l6XQ6NG/eHE2aNFE6SrnjnhJmLv/IncpYUSOqCBwcHNC6dWuewFECOp0ODRs2RKtWrZSOQmR2JEnCypUrkZmZifHjxysdxyzdu3cPEyZMwCuvvGKSaeVE9I8ePXrA0dFR1qNBExISoNVqMWrUKHz66ac4ceIEli9fLlv/SktNTcXu3bsr7cx3FiXMnF6vR9u2bTltnEhB3OyyeI8ePUJsbCzUajVndRGVUZMmTTBhwgSsX78ev/32m9JxzM6UKVNw//59aLVaWFnxFpjIVBwdHeHv7w+dTifLvZHBYIBGo8Hzzz+POXPmQK1WIyAgAGFhYbh27ZoMiZW3bds2ARrKiAAAIABJREFU5ObmVtofmvkX2YwlJydj//79lbaiRlRR+Pj44N69e7h48aLSUSqs2NhYZGVlVdovWyK5TJkyBQ0bNsSoUaOQnZ2tdByzceTIEXz22WcIDg5GmzZtlI5DZPFUKhWuXLmCP//80+i+vvjiCxw5cgSLFy9GtWrVIEkSli9fjtzcXHz88ccypFWeTqeDq6srOnbsqHQURbAoYcZiYmJgMBhYlCBSmLe3N4B/phZSwXQ6HapVqwY/Pz+loxCZNUdHRyxfvhx///03oqKilI5jFvLy8hAYGIhatWohLCxM6ThElUL//v0hSZLRSzju3LmDSZMm4dVXX8XgwYMfv//CCy9gypQp+P7777F9+3Zj4yoqNzcXMTEx6N+/P6ytrZWOowgWJcyYXq+Hh4cH2rVrp3QUokrtxRdfhK2tLfeVKITBYMDmzZvRt29fi9qQikgp/fv3h1qtxqxZs3DlyhWl41R4n376KRITE7F06VK4uLgoHYeoUnB3d4evr+/jEyXKatKkSXj48CG0Wu0zyz/Hjx+Pxo0bIygoSJYjfpWyf/9+3L9/v1LPJmVRwkxlZWVh69atUKlUXBdJpDB7e3u0adOGRYlCHDlyBLdv367UX7ZEclu2bBmEEAgNDVU6SoV269YtTJkyBT169MDbb7+tdByiSkWlUiEuLg7Xr18v0/UHDhzA6tWrMWbMGLRs2fKZzx0cHBAdHY0zZ85g8eLFxsZVjE6ng52dnawnlZgbPs2aqd27dyMtLY1LN4gqCB8fHyQkJHCzywLodDpYW1ujb9++SkchshgNGjTAtGnTsGnTJmzdulXpOBXWhAkTkJ6ejujoaG6yS1TO8p9T8o8EL43c3FxoNBp4enpixowZhbbz9/fHW2+9hblz5+LChQtlzqoUIQR0Oh26d+8OZ2dnpeMohkUJM6XX6+Ho6Iju3bsrHYWI8M++EikpKTh//rzSUSocvV4PPz8/1KhRQ+koRBZl7NixaNq0KYKCgpCZmal0nArn999/x9q1azFu3Dg0a9ZM6ThElU6rVq3QoEGDMi3hiI6OxrFjxxAVFYWqVasW2TYyMhLW1tYICQkpa1TFJCUl4ezZs5V+NimLEmYov6Lm7+8PR0dHpeMQEf6ZKQGASziecuHCBZw4caLSf9kSmYK9vT2io6Nx7tw5LFq0SOk4FUpOTg40Gg3q1auHqVOnKh2HqFKSJAlqtRqxsbFIT08v8XU3btzA9OnT0bt3b7zxxhvFtvf09ERYWBj0er3Re1iUt/yNQCv7fRKLEmbozz//xJUrVyr9/3iJKpJWrVrB3t6eRYmn5N8ccKkZkWn07NkTb7/9NubPn49z584pHafCWLFiBU6cOIHly5ejSpUqSschqrTUajUyMzMRGxtb4mvGjRuHrKwsrFixosTLrkJCQtCqVSsEBweXqgCiNJ1OBy8vL9StW1fpKIpiUcIM6XQ6SJKE/v37Kx2FiP7Dzs4Obdu2ZVHiKTqdDi1atECjRo2UjkJksZYsWQJbW1sEBwdzXxsA165dw8yZMxEQEMCCKJHCunTpAhcXlxIfDbpr1y5s2LABkyZNQpMmTUo8jq2tLaKjo3Hx4kXMnz+/rHHL1Z07d3Dw4P9n777DoyqzB45/76ROElIgBUgoSWgJCdIWAakSENSg/hDdtexaYelFQV26NDGsiBiwoay46gprCUWQoIC6CtIJCTUhkAFSIL3PzP39MZmBkEkyNZPA+3meeSQztxxDyNw597zn/CZ+TyGSEk1SQkICd999N0FBQY4ORRCEm+ibXWq1WkeH0ijk5+ezd+9e8WYrCHYWHBzMokWL2L59O999952jw3G4mTNnolareeedd0RzS0FwMFdXV0aNGsXWrVvrvT6qqKhg0qRJhIaG8uqrr5p9rsGDB/P0008TFxfHmTNnLA25wWzfvh2tViuukwBnRwcgmOfy5cscPHiQpUuXOjoUoYkpV2v43/lrDO0c6OhQblu9evVi7dq1nDt3jk6dOtnsuGlpaeTn59O9e3ebHdNaJ0+eZM+ePXVuk5ycjFqtFm+2gtAApkyZwieffMK0adNQqVR1buvk5MTYsWNp0aKF3eMqLy9nz549DB8+vEFGmO/atYuvvvqKRYsWERYWZvfzCYJQv9GjR/Of//yHhQsX1nlT9dChQ5w6dYqtW7da3DcvLi6OhIQEJk+ezM6dOxt1YjIhIYHWrVvTs2dPR4fieLIs3xaPXr16yXeC999/XwbkEydOODoUoYn576FLcrtXtspnMwsdHcpt69ixYzIg//vf/7bZMYuLi+V27drJnp6e8qVLl2x2XGtkZmbKvr6+MlDvo3379rJarXZ0yIJwR/jll19kDw8Pk/5tDh48WNZqtXaP6aWXXpIBec2aNXY/V1lZmdyxY0e5Q4cOcmlpqd3PJwiCaa5fvy77+PiY9LvpySeftPp8b731lgzIhw8ftkH09lFaWip7enrK48ePd3QoDQY4KNfyWV5USjQxCQkJhIaG0rVrV0eHIjQxF67pmv5cvF5Mh8C6RysJlomMjMTd3Z2DBw/yxBNP2OSYy5YtIz09HRcXF2bOnMlXX31lk+Na45VXXqGoqIj9+/cTGhpa57be3t44OTk1UGSCcGe75557yMzMpLS0tM7tvvjiC6ZNm8bnn3/Ok08+abd4kpKSePvtt3F1dWXu3LmMHTvWrktPV65cydmzZ9mxYwfu7u52O48gCObx8/Pj8uXLFBcX17utv7+/1ed78skneemll9iyZQs9evSw+nj2sGfPHoqLi0U1aRVJvk0aIvXu3Vu+3RvMFRcX4+/vz7hx41i9erWjwxGamJe+OsZ/D2ew+KGuPN2vvaPDuW31798fZ2dn9u3bZ/Wxzpw5Q3R0NI899hgdO3ZkwYIF7Ny5kxEjRtggUsv8+uuvDBgwgFdeeYU33njDYXEIgmA5jUZDv379uHjxIqdPn8bHx8fm55BlmSFDhpCUlMSWLVsYMmQIf/7zn/n0009tfi6ACxcuEBkZyf3338/mzZvtcg5BEJqOe+65h/Ly8kbbgHzSpEls2LCBa9eu3TFJVEmSDsmy3NvYa6LRZROSmJhIWVmZyKgJFlHl6SolMvLqvoMmWKdXr14cOXIEjUZj1XFkWWbSpEkolUri4uKYPXs2HTp0YPLkyZSXl9soWvOo1WomTJhAmzZtmDdvnkNiEATBek5OTqxbt46srCzmz59vl3N89tln7Nu3jxUrVtC/f39mzZrFxo0b2bt3r13ON23aNBQKBatWrbLL8QVBaFpiY2M5dOhQvT12HEGWZRISEhgxYsQdk5Coj0hKNCEJCQn4+PgwaNAgR4ciNEGqqmSEKlckJeypd+/eFBUVWd31edOmTSQmJrJkyRJatmyJu7s77777LmfPniUuLs5G0ZpnzZo1nDhxgtWrV+Pp6emQGARBsI1evXoxYcIE3n33XY4cOWLTY+fl5fHyyy/Tt29fnnvuOQDmzJlDu3btmDRpEpWVlTY935YtW0hISGDBggW0adPGpscWBKFp0t/E3bp1q4Mjqeno0aNkZGSIG803EUmJJkKr1bJ161ZGjhyJi4uLo8MRmhiNVuZKXhlwIzkh2Efv3rqqNGvKBQsLC5kxYwY9evRgwoQJhufvu+8+xowZw9KlS0lLS7M6VnNcvnyZBQsWMGrUKB5++OEGPbcgCPaxZMkSWrRowcSJE206ynju3Lnk5OSwdu1aw8QNDw8PVq9ezcmTJ226BLWkpISpU6cSGRnJ9OnTbXZcQRCatoiICMLDw0lISHB0KDVs2bIFSZJ44IEHHB1KoyGSEk3EgQMHyMrKEhk1wSJZhWWotTLOCklUSthZly5d8PDwsCopsWjRIi5fvszatWtrNIlctWoVTk5OTJs2zdpQzfLSSy9RUVHBmjVrGvV4LUEQTOfn50dcXBy///47n3zyiU2OefjwYdatW8fEiRNrNJgbPXo0DzzwAAsXLiQjI8Mm51u+fDkXLlwgPj5e3LQRBMFAkiRiY2PZvXu3SQ02G1JCQgJ9+/YlMDDQ0aE0GiIp0UQkJCTg5OTEqFGjHB2K0ATpExHdQnzIKiynXG1dvwOhdk5OTvTs2ZNDhw5ZtL++W/2LL75I3759a7zepk0bFixYYChXbgi7d+/myy+/5LXXXiM8PLxBzikIQsP461//ysCBA3nllVe4du2aVcfSarVMmDCBgIAAFi9eXON1SZJ455130Gg0zJw506pzga4Z8JtvvslTTz3FkCFDrD6eIAi3l9GjR1NeXs6uXbscHYqBSqXi0KFDxMbGOjqURkUkJZqILVu2MHDgQPz8/BwditAE6Zds9AltAcDlqqUcgn3om12q1Wqz9pNlmYkTJ+Lr68vy5ctr3W769OlERkYydepUSkpKrA23TuXl5UyaNInw8HBeeeUVu55LEISGJ0kS8fHx5OXl8dprr1l1rI8++ogDBw6wcuVKfH19jW4TFhbGP/7xDzZt2sQPP/xg8blkWWby5Mm4u7s7rM+OIAiN24ABA/D19W1USzj0PS5E9Xt1IinRBKSlpZGUlCR+eAWLZeTqkxK6pJZYwmFfvXv3pqSkhFOnTpm138aNG/n555954403aNGiRa3bubi4EB8fT3p6OsuWLbM23Dq99dZbnD59mjVr1ogO0YJwm4qOjmbatGl89NFH7N+/36Jj5OTk8NprrzF48GCefPLJOredNWuW1dOENm/ezK5duwzNgAVBEG7l4uLCqFGj2Lp1q9VT0WwlISGBsLAwIiMjHR1KoyKSEk3Ali1bAESZj2AxVV4pfh4udAxsVvW1fe+u3+ksaXaZl5fHrFmzqnWrr8uQIUN46qmniIuLs3rSR23S09NZvHgx//d//yeWjgnCbW7hwoW0atWKiRMnWnTx/uqrr1JQUEB8fHy9fWfc3d2Jj4+3eJpQbc2ABUEQbjV69Giys7M5cOCAo0OhuLiY3bt3ExsbK/pz3UIkJZqAhIQEIiIi6NChg6NDEZqojNxSQvw8aOnjjkISlRL21qlTJ7y8vMzqK6HvVr9u3TpDt/r6xMXF4e7uzqRJk5Bl2dJwazV9+nQkSeLtt9+2+bEFQWhcmjVrxqpVqzh8+DDvvfeeWfv+9ttvrF+/nhkzZtC1a1eT9hkxYgSPPvqoRdOEXn/9dVQqFWvXrsXZ2dmsfQVBuLOMHDkSZ2fnRrGEY9euXZSXl4vqdyNEUqKRy8/PZ+/eveKHV7CKKreEYF8lLk4KgrzdyRBjQe1KoVDQs2dPkyslDh06xNq1a5k0aRLdu3c3+TwtW7ZkyZIlJCYmsmnTJkvDNWrbtm18++23zJ8/nzZt2tj02IIgNE5jx44lJiaGOXPmkJmZadI+arWaCRMmEBISwvz58806n36a0NSpU03eJykpiVWrVvHCCy8YbQYsCIJwM19fXwYNGmSoPHekhIQEfHx8GDhwoKNDaXREUqKR27FjB2q1WizdECwmyzKqvFKC/ZQAhPgpRaVEA+jduzdHjx6lsrKyzu20Wi0TJ04kMDDQaLf6+kyYMIHu3bszY8YMCgsLLQ23mtLSUqZMmUJERAQzZsywyTEFQWj8JEni3XffpaSkhFmzZpm0z9q1azl27BirVq3Cy8vLrPOFhISwYMECtm7datJdTFmWmTRpEj4+PnU2AxYEQbjZ6NGjOXnyJKmpqQ6LQaPRsHXrVkaNGiXGFxshat4auYSEBPz9/cXdAMFi14srKKvUEuyrS0oE+yo5mJ7r4Khuf71796asrIyvv/6a0NDQWrdLTEzkwIEDfPbZZ/j4+BjdpqRCTaVGxkdZ803M2dmZdevW0a9fPxYtWsTKlSutjv2NN94gLS2NH3/8EVdXV6uPJwhC09G5c2dmz57N0qVLue++++jYsWOt25aUlDBv3jzuu+8+xowZY9H5pk+fzoYNG5g6dSqBgYF1Ll/75Zdf2LdvHx9++CH+/v4WnU9ovK4VldPM3QVXZ3HPVLCt2NhYpk+fzpYtW5g2bZpZ+2ZnZ5u9xMyY06dPk52dLarfayHZYx2yI/Tu3Vs2p6lcU3D16lU6duzIo48+yieffOLocIQm6nhGHqPf/ZX3n+7FfV1bErfzFO/vTeX0klE4KUSTHXtJTU0lPDzcpG2HDBnCjz/+WGvTo5e+OkZqThHfTLyn1mO8+OKLfPLJJxw5coTo6GiLYgY4e/Ys0dHRjBkzhn//+98WH0cQhKarpKSEqKgoky7E3dzcOHHiRJ3Ji/rs27ePwYMHm7Rt3759+fXXX03uvSM0DbIs86eliTx5dztmDO/k6HCE21BUVBRBQUHs3r3b5H0yMjKIiooiPz/fJjG4urpy9epV/Pz8bHK8pkaSpEOyLPc29pqolGjEZs2aRUVFhdVzw4U7m36pxo1KCQ/UWpnMgjJaVz0n2F5YWBj79+8nJyenzu0kSWLw4MF1dmE+npFH+vUSNFq51kTS8uXL+frrr5k0aRJ79+61qKuzLMtMmTIFNzc3m1RcCILQNHl4eHDgwAGTutV37tzZ5ARsbQYNGsSJEye4ePGiSduKhMTtJ6eogpyiCvanXXN0KMJtavTo0cTFxZGXl4evr69J+8ycOZPy8nK++uorPD09rY6hTZs2d2xCoj4iKdFI7d27l88++4y5c+fSqZPIGAuWU1U1tQyp6imh7y2hyisVSQk769Onj9XH0Ghl0q+VUKHRcjmvlDbNPYxu5+/vz4oVK3jxxRfZuHEjf/3rX80+19dff83OnTtZvXo1rVq1sjZ0QRCaMH9/f+6///4GO19UVBRRUVENdj6hcdFfqySpCtBqZRSiklOwsdjYWJYvX86OHTv485//XO/2P/zwA5s2bWLRokWMHTu2ASK8s4lUcyNUWVnJxIkTad++vaiSEKyWkVuKp6uToR+BvmJCNLtsGlS5pVRotACczy6qc9vnnnuOvn37MmvWLHJzzesbUlRUxPTp0+nevTsTJ060OF5BEARBMJf+mqSoXE3atWIHRyPcjvr06UNgYKBJTXXLy8uZPHky4eHhzJ49uwGiE0RSohF6++23SU5O5p133sHDw/hdUUEwlX7yhr6c35CUEGNBm4TzOTcSEanZdV+oKRQK1q5dS05ODnPnzjXrPK+//joZGRmsXbsWZ2dRRCcIgiA0HFVeieHPSSrbrN8XhJs5OTnx4IMPsn379nono8XFxXH27Fneffdd3N3dGyjCO5tISjQyGRkZLFq0iNjYWDEGVLAJVW6pIREBoHR1ooWnKxmiUqJJ0CciXJ0VpObUXSkB0KNHDyZOnMi6des4dOiQSec4efIkq1at4rnnnqNfv35WxSsIgiAI5lJVVXW6uyg4niGSEoJ9xMbGkp+fzy+//FLrNmlpaSxdupQxY8YwcuTIBozuziaSEo3MjBkz0Gq1rF692tGhCLcJfaXEzYL9lKJSoolIzS7CR+lCRCvveisl9BYvXkxgYCATJ05Eq9XWua0sy0yaNAlvb29WrFhhi5AFQRAEwSyqqp5Jka28OSGSEoKdDB8+HDc3tzqXcEybNg0nJydWrVrVgJEJIinRiOzcuZPNmzczZ84cQkNDHR2OcBsoKleTX1pJsG/1ZUDBvkpUuSW17CU0JqnZxYQFeBLu72lyUsLX15eVK1dy4MABPvroozq3/fzzz9m7dy/Lly/H39/fFiELgiAIglkyqqo6o4N9SLqcj0YrOzok4Tbk6enJsGHD2LJlC7Jc82csISGBLVu2sGDBAtq0aeOACO9cIinRSJSVlTF58mQ6derEyy+/7OhwhNuEYRzorZUSvrpKCWO/kIXGJS2nmDB/L8ICPLlaUEZxudqk/Z588kkGDx7Mq6++SnZ2ttFt8vLyeOmll+jTpw8vvPCCLcMWBEEQBJPpqzqjQ3wpqdCQZsJyRUGwxOjRozl//jwpKSnVni8pKWHq1KlERkYyffp0B0V35xJJiUYiLi6Oc+fO8e677+Lm5ubocITbhL5xVLBvzeUbZZVarhVXOCIswUTF5WquFpQRFuBJWIAXoEtSmEKSJOLj4yksLOTVV181us38+fPJyspi7dq1KBTi7UAQBEFoeAVllRSWqQn2VdItxAdA9JUQ7ObBBx8EqLGEY9myZaSnpxMfH4+Li4sjQrujiavQRiA1NZVly5bx2GOPMXz4cEeHI9xG9JUSIUYqJW5+XWic9AmI8ABPwgI8AUg1MSkB0LVrV2bMmMHHH3/M//73v2qvHTlyhPj4eCZOnEivXr1sF7QgCIIgmOHmqs7wAC+ULk4iKSHYTXBwML169WLLli2G586cOUNcXBxPPvkkQ4YMcVxwdzCRlHAwWZaZOnUqzs7OvPXWW44OR7jNZOSV4uqkIMCrevWNfjmHaHbZuJ3P1pWvhvp70b6FJ5Kka3xpjvnz5xMSEsKECRNQq3VLP7RaLRMmTMDf358lS5bYPG5BEARBMJUhKeGrxEkh0bW1NyfEWFDBjkaPHs1vv/1GVlaWoeG3u7s7K1eudHRodyyRlHCwhIQEtm3bxqJFiwgODnZ0OMJtRpVbSmtfdxQKqdrzIVWNL0WlROOWml2MJEG7Fh64uzjR2kdpcrNLPS8vL1atWsXx48eJj48H4OOPP2b//v3ExcXh6+trj9AFQRAEwST6GyQhfrprk+gQH5IvF6DW1D09ShAsNXr0aGRZZtu2bWzatInExESWLFlCy5YtHR3aHUskJRyouLiYadOmERUVxZQpUxwdjnAbMjYOFMBb6YyXm7OolGjkUnOKCfFT4u7iBEBYgCepFjT/GjNmDCNGjGDevHkkJSXx6quvMnDgQJ5++mlbhywIgiAIZlHlleLmrMDfyxWAbiE+lFZqOG9mEl4QTHXXXXfRpk0bvvjiC2bMmEH37t2ZMGGCo8O6o4mkhAMtXbqU9PR01q5dKxqqCHahqhqxdStJkgj2VZIhKiUatbScIsL8vQxfhwd4kZZdbPbUFEmSePfddykvL6dfv37k5eWxdu1aJEmqf2dBEARBsCP9tYr+PSk6WFfBdzwjz5FhCbcxSZKIjY1l165dXL58mXXr1uHs7OzosO5oIinhIKdPn2blypX89a9/ZeDAgY4OR7gNlas1ZBWWE1y1VONWwX5KUSnRiMmyTFp2saHBJegqJYordH+v5urYsSOvvPIKRUVFTJ8+naioKFuGKwiCIAgWycgtqVbVGebviaerk8V9JT7ff5GRb+8Tyz+EOo0ePRqAF154gb59+zo4GkGkhBykVatWzJgxg5kzZzo6FOE2dSWvDMDo8g3QNZQ6eOF6Q4YkmCGzoJziCo1hFChgqJo4n11EkLe72cecM2cOXbp04ZFHHrFZnIIgCIJgDVVeKRGtvA1fKxQSXYN9LE5KfHXwEqeuFnIoPZe7w1rYKkzhNjN8+HA++ugjxo4d6+hQBERSwmG8vb1ZsWKFo8MQbmMZN3WzNibYT0lBmZrCskqauYvlQ42NfspGmH/1Sgnda8X0D/c3+5hubm488cQTtglQqFdBQQFZWVlUVlY6OhThDuPi4kJgYCDe3t71bywIDlRWqSGnqKLGtUq3YB82/p5OpUaLi5Pphd1ZhWUcvaRb9pGYkimSEkKtFAoFzz//vKPDEKqYlJSQJOljYJosy4W3PO8JrJFl+Tl7BCcIguVUeSUAhNRSKRFy01jQLi1FUqKxOZ+ja/B18/KNlt7uuLsozJ7AITS8goICMjMzCQ4ORqlUiv4dQoORZZnS0lJUKhWASEwIjZp+GemtVZ3RIT6Uq7WczSwisrXpP8M/ncoCoG1zD3anZDHngUjbBSsIgt2Ymnr8G2Dsk40S+KvtwhEEwVZUuaUoJGjpY7zMX39XQowFbZxSs4vwcHWi5U3LNBQKiVB/L9IsmMAhNKysrCyCg4Px8PAQCQmhQUmShIeHB8HBwWRlZTk6HEGok6qWqs7oYB8ATqjMa3a5KzmLYF8lLw4MJTWnmPPZ4v1SEJqCOpMSkiQ1lySpBSABflVf6x8BwINAZkMEKgiCeTLySgnydq+17DH4pkoJofFJyykm1N+zxgda3VhQUSnR2FVWVqJUGq9SEoSGoFQqxdIhodGrrVKifQtPmrk5m9VXoqxSwy/nsomJCGRYRBAAicniY4ogNAX1VUrkAFmADCQD2Tc9rgIfAWvtGaAgCJapbRyonr+nG67OClEp0UilZhdXa3KpF+7vyaXrJZSrNQ6ISjCHqJAQHEn8/AlNgSq3FCeFVK0qEHSVgVHBPpzIMD0p8eu5HMoqtcREBtHaV0nX1t7sThHVQoLQFNSXlBgKDENXKfEocO9NjwFAW1mWl9o1QkEQLKLKK6118gbo3vCDfZVkiEqJRqdcrSEjt4TQm5pc6oUFeKGV4eK1EgdEJgiCIAi2o8orpaW3O85Gqjq7hfiQcqWQCrVpoz0TUzLxcnPm7lBdc8thEUEcTL/O9eIKm8YsCILt1ZmUkGV5ryzLe4BQ4Luqr/WP32RZvtwgUQqCYBaNVuZqflmdlRKgW8MpKiUan/RrJWhlCA8wlpTQPXdeNLsU7gAbNmzAy6tmxZAgCLeHuqo6o4J9qNBoOZNZaPT1m2m1MokpWQzuFICrs+7jzfCIILTyjeaXgiA0XiY1upRlOR1wlySpvyRJD0uS9H83P+wcoyAIZsosKEOtleuslABdUiJDJCUanRvjQGt+GNNXT6SKZpeCnTzzzDNIkoQkSTg7O9O2bVsmTJhAbm5ug8fy+OOPk5qa2uDntTVJkti8ebOjwxCERqeuqs5uIfpml/Uv4Tihyie7sJyYyEDDc1HB3gR5u7H7lOgrIQiNnakjQWOALwBjw35lwMmWQQmCYB1D46j6KiX8lOQUlVNWqcHdRfwzbiz0VRChRiolmrm7ENDMjTRRKSHYUUxMDBs3bkStVpOcnMxzzz1HXl4eX3zxRYPGoVQq62wYqlarcXJyEv0TBKEJUmu0XC2ovaqzbXOa9+r6AAAgAElEQVQPvN2dOZ6Rz1/61H2sxJRMFBIM6XQjKSFJEsMigvjuiIpytQY3Z3GdIwiNlakjQVcD24AQWZYVtzzEv3BBaGT0SzJCTKiUALgs+ko0Kmk5xQR5u+HlZjxvHOYvJnAI9uXm5kbLli0JCQlhxIgRPP744/zwww+G1/Pz8xk3bhyBgYE0a9aMwYMHc/DgwWrH+PTTT2nXrh0eHh48+OCDxMfHV0seLFy4kKioqGr73Lpc49av9fts2LCB8PBw3NzcKC4uRpZl3nzzTcLDw1EqlURHR/PZZ58Z9rtw4QKSJPHll18yePBglEolPXr04Pjx4yQlJdG/f388PT0ZMGAAaWlp1WLasmULvXr1wt3dndDQUObMmUNFxY016u3bt2fJkiWMHz8eb29vQkJCiIuLq/Y6wNixY5EkyfD1pUuXeOihh2jevDkeHh506dKFL7/80tS/IkFo8q4WlKGpo6pTkiS6hfiaNBZ0V3Imvds3x8/TtdrzwyOCKK7Q8HvqdZvELAiCfZialGgPLLa2h4QkSYMkSUqQJEklSZIsSdIzt7y+oer5mx+/W3NOQbgT6SslWptQKXHz9kLjkJpdZLTJpV5YgJdhiYcg2Ftqaio7duzAxcUFAFmWeeCBB1CpVGzdupUjR44waNAg7r33Xq5cuQLA/v37eeaZZxg3bhxHjx4lNjaW+fPn2ySetLQ0Pv/8czZt2sSxY8dwd3dn7ty5rF+/nvj4eJKTk3nttdcYP34827Ztq7bvggULeOWVVzhy5Ai+vr785S9/YcqUKSxdupQDBw5QVlbG1KlTDdvv3LmTJ598ksmTJ3Py5Ek+/vhjNm/ezD/+8Y9qx121ahXR0dEcPnyYV155hdmzZ/Pbb78B8McffwDw4YcfcuXKFcPXEydOpKSkhJ9++omTJ0/y9ttv4+vra5PvkSA0BfobKHVVdUYF+3D6amGdE6cycks4dbWQ4VVjQG/WL7wFShcndqeIJRyC0JiZtHwD+BXoDJy38nxeQBLwadXDmETg6Zu+Fi1zBcFMGbmltPB0xcO17n/i+gsB0eyycUnNKeb+6Fa1vh4e4EluSSW5xRU17goJjdf06dM5evRog56ze/fuvP3222bvt2PHDry8vNBoNJSVlQHw1ltvAfDTTz9x9OhRsrOzDUsrFi9ezJYtW9i4cSOzZ89m9erVDBs2jDlz5gDQqVMn/vjjD9avX2/1/1NFRQUbN24kKEj3AaS4uJi33nqLH374gYEDBwIQGhrKgQMHiI+P54EHHjDsO3PmTO6//34AXnrpJWJjY1m8eDFDhw4FYPLkyUyePNmw/dKlS5k1axbPPvssAOHh4axYsYKnnnqKuLg4Q+XHiBEjDPtNmTKFd955h927d9OvXz8CAgIA8PX1pWXLloZjp6enM2bMGO666y5DzIJwJzEsNa2jqrNbiA+VGpnTVwvpFmI8aacf+zksIrDGa+4uTgzs6E9iciaLRncVS70EoZEyNSnxHrBSkqTWwAmg8uYXZVk+bMpBZFneDmwHXVVELZuVy7J81cS4BEEwor5xoHotfdxRSKJSojG5XlxBXkklYXVWStxodtnLs3lDhSbcQQYNGsQHH3xAaWkpH374IefPnzdUEBw6dIiSkhLDh229srIyzp/X3btISUkhNja22uv9+vWzSVIiJCTEkJAASE5OpqysjJEjR1b7wFFZWWlYKqHXrVs3w5/1x4iOjq72XHFxMSUlJXh4eHDo0CEOHDjAihUrDNtotVpKS0u5evUqrVq1qnFcgNatW5OVVXfH/2nTpvH3v/+dHTt2MGzYMB555BF69epl4ndBEJo+UyolooN1zS6PZ+TXmpRITMkkLMCTsADjk3piIoP4ITmTlCuFRLb2tjJqQRDswdSkhL5l9AdGXrN1o8sBkiRlAXnAXmCOLMtilk8T83HSx/i5+fFIx0ccHUqjpdXKvLz5GI/3bsPdYcZ6yFpOlVtCp6Bm9W7n4qSgpbe7XSolPvk1jdySSmYO72TzY99Kq5WZ8uUR0q/V32dhTM8Qnr3HujuSyZcLeGvXaVY93p1m7i5WHetW+mUZ4bVcXAGEVk3lOJ9dTK92tk1KVKi1zN58jHM2Wh4S0dKbuLF32eRYTZ0lFQuO4uHhQYcOHQB45513GDp0KIsXL2bhwoVotVqCgoL4+eefa+zn7W36Bb9CoUCW5WrPVVZW1rL1DZ6e1RN2Wq0W0PV+aNu2bbXX9EtOjH2tT2AYe05/TK1Wy4IFCxg7dmyNOG5Oytx6HkmSDMeozfPPP899993H9u3bSUxMpH///rz22mssXLiwzv0E4XaRkVuKv5dbnY22Q/yU+Hm4cCLD+ASOwrJKfk+9xnN1vK/f2yUQSdIlL5p6UiK/tJLZm48x78FIQvw8HB2OINiMqUmJhqop3AF8DaSh62OxBPhRkqResiyX37qxJEnjgHFAjQsRwbG+Ov0VLT1biqREHdKuFfP1YRUl5RqbJiVkWUaVV8rQzjXLGI0J9lOSYeNKiTOZhSzdloJCkhg/KAzPWho22kr69RK2Hb9CdLAPgc3cat3ucn4Zi7cm0ye0OV1b+1h0Lo1WZvZ/j5GkKmB3ShYP9wi2NGyjUqumaoQZmbyh18ZPiYuTRJodml1+9Esq3x69zMCO/rg6mdp2yLiswnI2Hcpg4tAOdfbIEBq/BQsWMGrUKMaNG0fPnj3JzMxEoVAQFhZmdPuIiAh+/716S6hbvw4ICCAzMxNZlg3JAEuWt0RGRuLm5kZ6ejr33nuv2fvXpWfPnpw6dcqQoLGUi4sLGk3NNfEhISGMGzeOcePGsWLFClavXi2SEsIdw5SqTkmSiAr24XgtY0H3ncmhUiMzzEg/CT1/Lzd6tPElMSWTqcM6WhWzo+08eZWdJzMZFhHEY71FUkK4fZj0SUGW5XR7B1J1npvbTp+QJOkQkA48gC5Zcev2H1BVvdG7d2/51tcFx5BlmaySLJwkMZilLvqs/76z2TYdyXm9uIKySq1JyzdAVzb5x4Vcm5wbdH//c79NQpKgQqPl57PZjIyqvT+CLRzP0HXmfmNMdJ3JhvySSu795x7mfZvE5r/3R6Ewf23p5/vTSVIV4OIksSsl0/ZJiZxiXJykOu+AODspaNvcw+bNLjNyS1iz+xwjIoP44K+9rT7epeslDHzzJ3anZPLCQOMfXoWmYciQIURGRrJkyRLi4+O55557eOihh3jzzTfp0qULV69eZceOHcTExDBw4ECmTp1K//79Wb58OY8++ih79uzhm2++qXHM69evs2zZMv785z+zZ88eNm/eXEsEtWvWrBkvv/wyL7/8MrIsM2jQIIqKivj9999RKBSMGzfO4v/v+fPn8+CDD9KuXTsee+wxnJ2dSUpK4sCBA7z55psmH6d9+/bs3r2bwYMH4+bmhp+fH9OmTWPUqFF06tSJgoICduzYQWRkpMWxCkJTo8orJbJV/ZUL3UJ8eG9vqtFrpd0pmfh5uNCzbd1NYodFBBG38zSZBWUEebtbFbcjJSbrGnZmFZQ5OBJBsC2TboNJkvR/dT3sFVzVtI8MoGmnNe8wBRUFVGoryS7NrlGaK9xwoirrX1Kh4bfUazY7rqFxVD2TN/SC/ZRcLShDram71NhU3xxRcSDtOvNju+KjdGFXsv1XXyWp8nF1VtS7ZMXHw4XX7o/g8MU8Nh26ZPZ5sgvLeXPnaQZ08Of/eoSw93Q2FWrbfN/0UrOLaNfCE6d6Eia6CRy2rZR4fUsyAPNjbfPBqE1zD7q0bMauZNH1/Hbw0ksvsX79ei5evMj27du59957efHFF+ncuTOPPfYYp0+fpnXr1gD07duX9evXs27dOrp168bXX39dowIgIiKCdevW8cEHH9CtWzd27dpVY6qFqfRLS1auXEnXrl0ZPnw4//3vf61uHnnfffexbds2fvrpJ/r06UOfPn144403zK7O/Oc//8lPP/1EmzZt6NGjB6BbGjJlyhQiIyMZPnw4QUFB/Otf/7IqXkFoKrRa2eT+V9HBvmi0MilXCqo9r9Zo+fF0FkO7BOJcT2Xf8EhdJYW+KWZTVFap4eezOQBkFtQoIBeEJs3cnhK30n/itMstcUmS/IFg4Io9ji/YR1aJ7hd+qbqU4spivFxrXxt/JzuRkU/X1t6k5RSzOyXT5OUW9TE0jjK5UsIDjVYms7Dc5ERGbfJLK1m2PYXubXx5sk9bDl64zk+ns9Bo5Xo/ZFvjeEY+Ea28cTFhucGYnsH854+LvPH9KUZEtjRresXy71Moq9Sw6KGupGYX85+DlziQdp0BHf2tCb+a1JziOptc6oUFeLL3dLbNvrc/ncrih+RMZo/sbNN1qjERQazbe568kgp8PcSkkKZgw4YNRp9/4okneOKJJwxfr169mtWrV9d6nGeffdYwtQIwWgUxfvx4xo8fX+25adOmGf78zDPP8Mwzzxi+XrhwodHlDZIkMWXKFKZMmWI0lvbt29dIkvfu3bvGcyNHjqzx3IgRIxgxYoTR4wJcuHChxnN79uyp9nVsbGyNxp9r1qyp9ZiCcLvLKS6nQq016bqjW4iuAvKEKp8ebf0Mzx9KzyWvpJKYOpZu6HUM9KJtcw8SUzJ54u6mueT7t/PXKK3UoJAgq1BUSgi3F5MqJWRZVtz8AFyBu4GfgUGmnkySJC9JkrpLktS96txtq75uW/XaSkmS+kmS1F6SpCHAFiAL+Kau4wqNS3ZJtuHPWaVNNyNtTxqtTNLlfHq382NQxwASk7NsVlWir5QI8TXtg6U+eWGLZpdv/XCa68UVLHk4CoVCIiYiiOvFFRy5aLvlIbfSamWSVPl0CzatR4QkSSx+OIqCMjVv7jxt8nkOpF3n68MqXhwYRniAFwM6+OPmrCDRhrPP1Rot6deKa+0gfrNwfy8qNFoyckusPm9ZpYYFCScJD/DkhQG2XWYxLCIQjVZmz+ns+jcWBEEQ7gimTN7Qa+XjTgtPV47f0uxy96ksXJ0UDOoUUMueN0iSxLCIQH49l0NJhdqyoB1sV0omnq5O/Kl9c1EpIdx2LOpiJsuyWpblP4B/AGvN2LU3cKTqoQQWVf35dUADRAPfAWeAfwGngX6yLBdaEqfgGDcnIm5OUAg3pGYXUVKhITrEl2ERgVwtKOPk5YL6dzRBRm4pXm7OeCtNK4TSXxCo8qz7cJukymfj7+k83bcdUVUJgsGdA3BW6Hov2EtqTjHFFRqiQ0xvXNmlpTfP9m/Pl39cNClhUqnRMu/bJIJ9lUy5V7eaTOmqm32+KznTZgmljNxSKjVynU0u9UINY0GtX8Kxds95Ll4vYfFDUbg6W9fc8lZ3hfji7+Vm158BQRAEoWkxLDU1oapTkiSiQ3xqTOBITM6kb3gLvExspj08IohytZZfqpZANCWyLLM7JZNBnQJo09xD9JQQbjvWXn3mAeGmbizL8h5ZliUjj2dkWS6VZfk+WZYDZVl2lWW5XdXz5i/8FhyqWqVEiaiUMEbfT6JbiE+1UVW2oMorJdhXaehmXx9DUsKKSgmtVtfcsrmnGzNHdDY87+3uwt1hze26hjPppu+lOaYP70RgMzfmfZeERlt3UmHDrxc4nVnIgthIlK43VqsNiwhClVfK6Uzb5E1Tc/TjQE1YvlG1xMPavhIXcop5b+95Rt/Vmv4dbLcMRU9XMRPIPjv03xCalkcffVT0GRIEATB/qWm3YB/OZhVSWqGbYnM+u4jUnGJiIkxf+vqn0OY0c3e2aYVjQ0lSFZBZUE5MRBBB3m5kFZajrefaRRCaElMbXfa85dFLkqQHgffRVToIgkFWSRZuTrqxjNmlolLCmOMZ+ShdnAgP8KKFlxu92vrZ7E0yI9e0xlF6SlcnWni6Gu5aWOI/By9x9FIe/7i/Cz5Kl2qvxUQEcS6ryC7jK0H3vXR3UdDBhCUPN/Nyc2buA5EkqQr49/7aBwxdzS/j7cQz3Nsl0NAoS29YF93FUKKNGjnqEwyh/vX/vzT3dMVH6WLVBA5ZlpmfcBJXJwVzH4iw+Dj1GRYRRGG5mgNp1+12DkEQBKHpUOWV0szdGW93l/o3BqJDfNHKkHxFdyNid9U1U12jQG/l4qRgSOdAfjyV1eQ+0O9KyUQhwdAugQQ2c0etlbleUuHosATBZkytlDgI/FH1X/2fE9A1uHzBPqEJTVV2aTYhXiF4uniK5Ru1OKHSNbnUNygcFhFEkqqAK/nW93VQ5ZaY3bAyxE9JhoWVEteLK1ix4xR9QpvziJHxmPoGVLvtdGfihCqPyFbe9XbeNubBbq0Y0MGfuJ2nyS40vj5z8bZk1FqZhbFda1SfBHq7c1eID7tsVAmSmlOMr4cLzU1ovilJEmEBnlZVSuxIusq+M9nMGN6JQDuOSLNH/w1BEASh6VLllpp1rRJdtSxU31ciMTmLyFbeZl/vxEQEklNUwdGqUeJNRWJyJr3a+dHc05Ugb92Nv0yxhEO4jZh6FR8KhFX9NxRoB3jIstxflmXTO8UJd4TskmwCPAIIUAaI5RtGqDVaTl7Or9YDYXik7o67tcscCssqKShTm1UpAbrySUuXb7y54xRFZWqWPBxldMlIm+YedA5qZpcPpBqtTJKqgG4hdc8nr40kSSx6qCtllRqWb0+p8fq+M9lsO36FSUM70LaF8cahMRFBHLuUZ5NO2KnZRSZN3tAL8/cyLPkwV3G5mte3JtOlZTP+1q+dRccwldLViQEd/ElMsV3/DUEQBKHpUuWVEmLGtUqQtxsBzdw4kZFPbnEFB9Ovm7V0Q29Ip0CcFZLNKhwbwuW8UpKvFBhu8uhvImSJZpfCbcTU6RvptzwuybIs0nOCUdml2QR6BBLgESCWbxhxPruYskpttR4I4QFetG/hYfUHd0PjKDPvHAT7KlHllZr9gfFQei5f/nGJ5waE0imoWa3bxUQG8seFXPJLKs06fn1Ss4sordQY7qBYIjzAi3GDwvj6iIr9qdcMz5erdRMp2rfwYNyg2idSxFQt6fjRBtUSqdmmTd7QCwvwJLOgnKJy8zuJv/PjWa7kl7H0kSiLqkzMFRMZREau7fpvCIIgCE2XuZUSkiTRLdiHE6p8fjqdhVa+8f5rDh8PF/7U3r69rmzt1qUqQVVJCVEpIdxOTL4SlSSpmyRJn0qSdFCSpD8kSfqXJElR9gxOaHq0spbs0mz8lf4EKAPE8g0jjleVDEYH37i7rxtVFcT/zl2j2IIPmHrmNo7SC/ZVUq7WklNk+vpEddVEipbe7kwb1rHObWMignRjIc/Y9iJAX8ZpzuQNYyYP7Uiwr5J53yVRqdE1Y/xwXyppOcUseigKdxenWvft0rIZwb5KqxNKhWWVZBWWmzR5Q09fVXHBzH4dZzILWf9zGmN7hdCrXXOz9rWUvv9GU7oQFARBEGwvv7SSwnLzqzqjgn04l13Ed0cvE+TtRlRry977YyKDOJ1ZyKXr1o/Ubgi7UrII9fc0NMEO8NIt38iqZdmpIDRFpja6HA0cBtoA3wM7gLbAEUmSYu0XntDU5JXnodaqCfQIJNAjkOzSbFGufYsTqnw8XZ1qlOnHRARRodHysxWjqvSVEiHmVkr4eVTb3xSf/Z5O8pUC5sdG4lnPOC7DWEgbl0ueUN1oGGoNpasTC0d35UxmERt+vcCl6yWs+fEc90e3ZHA9888lSTdd4pdzOYau4JbQNwINM6HJpZ6+quK8Gc0uZVlm3rdJeLo58+qoLuYFaQVD/40mVDIrCIIg2J7hBoqv8WWRtekW4oMsw94z2dzbJQiFwrQpY7fSL/toCn2OisrV/H7+GjERgYYlsq7OClp4uopKCeG2YmqlxBJgqSzLQ2VZnlf1GAosr3pNEIAb40ADlLqeEuWacgoqChwcVeNyPCOfrsE+Nd5Me7f3w0fpYtWbpCq3FFdnBf5VWXRTmTsWNKuwjH/+cIaBHf0ZFdWy3u0VColhXQLZa+OxkCdU+UQF32gYao3hkUEM6xLIqsQzvLzpGE4KiXkPRpq0b0xkEGWVWn49Z3lCyZCUMKNSol0LDyTJvLGg3x29zP6068we2ZkWZv6cWCsmIoijNuq/IQh6GzZswMvLusSkIAgNx7DU1MxKiZuXaup7cVmiXQtPOgZ6NYmkxM9nsqnQaGtMGQn0didT9JQQbiOmJiU6ARuNPL8R6Gy7cISmTt/YUl8pATSaJRybDl4iSZXv0BgqNVpSrhTQzUgPBN2oqgB+OpWFxsJRVRl5ujWa5t490F8YqPJMK2Vcti2FcrWW1x8y3tzSmJhI3VjIPy7YZiykoWFosGVNLo1ZOLorGq3M/rTrTBvWkVY+pl0w3R3aAi83Z3afsvwC53x2MQpJl2gwlbuLEyF+SlJNXL5RUFbJkm0p3BXiw5//1NbSUC2mv6j66ZRYwtFYPfPMM0iSxOLFi6s9v2fPHiRJIifH8sSbLUiSxObNm6s99/jjj5OamuqgiARBMFdGru5aw9z+V4He7rT0dkfp4kT/cH+rYoiJDGJ/6nWb9Lracuwyh9LtM/I6MSULH6ULvdv5VXs+yNtNJPgbyL/+d4F/fHOi3seOpCuODrVJMzUpkQX0MvJ8L6DxpxmFBqNvbBngEUCAh67sPavU8R9A/ncuh1mbjzPu04OUVFjes8FaZzOLKFdra+2BEBMRxLXiCo5eyrXo+OY2jtLzUbrQzM3ZpEqJ385f49ujlxk/OIxQMyZF6MdC2qp8/1x2EWWVWqJDvG1yPNBNClkQ25WYiCCeGxBq8n6uzgoGdwogMcXy2eep2UWE+Hng5lx7/wpjwvy9SDVx+cZbP5zhWnE5ix+Oskl1ibkiWun6b+xKdvzvBKF27u7uxMXFkZ3dOBLK9VEqlQQGWn7XVBCEhqXKLcXNWYG/V/3jr2/1WO8Q/tq/XZ29nkzxYLdWqLUya/ecs+o4JzLymfrlEV789BB5Jab35TKFRivz46lM7u0SWKMhdWAzN7F8owFkFZaxIOEkW45e5oeTmbU+vjmsYu63Jy2+BhRMT0p8CLwvSdIcSZKGVj3mAu8BH9gvPKGp0VdKBCgDCFQ2jkqJCrWWed8l4e/lyuX8Mtb8aN0bkDVOqPRNLo0nJQZ3DsBZIVn8oU2VZ1lSAqrGgtbTU6JSo2X+d0m0aa5k0tAOZh3f1mMhDU0ubVgpAfDE3W356G+9cTFzIsWwiECyC8s5bmE1jm7yhulJHr2wAE/Scorr/Z4mqfL59LcLPHV3O4tHqFrrRv+NbMoqLe+/IdjX0KFDad++fY1qiVvt27ePu+++G3d3d4KCgpgxYwYVFbVflGs0Gp5//nlCQ0NRKpV07NiRN998E622+pKuf/3rX0RHR+Pm5kZQUBB/+9vfAGjfvj0AY8eORZIkw9fGlm+8//77dOjQAVdXVzp06MCHH35Y7XVJkvjggw8YO3Ysnp6ehIWF8dlnn5ny7REEwUqqvFKC/ZQmV1rebOaIzrw2KsLqGLq29uHPf2rD+l/SOGPhVCitVmbud0n4Kl3IL63kzZ2nrY7rZocv5pJbUskwI6NPg7zdyS4st7iyVjCNfrLaV3/vx8G5MbU+lv9fNDlF5RyramYvmM+cnhKLgAnA7qrH34EFwDL7hCY0Rdkl2fi6+eLq5Iq/h660ztFjQdf/ksb57GLiHr2LR3uF8NHPqZzLcsxYwuMZ+TRzc6Z9C+MfPr3dXbg7rLlh/JM5yio1ZBeWm71GUy/YV0lGPZUSH/+SxtmsIhbGdrXoLoV+LOSZTNMbM9YmqZaGoY4ytHMgCgmL/u60Wpm0nGKzmlzqhfl7UlKhqXNtqVYrM++7JPw8XHl5hGNX3A2LsL7/hmBfCoWCN954g/fee4/z588b3UalUjFq1Ch69OjBkSNHWL9+PV988QWvvfZarcfVarUEBwfz1VdfkZKSwtKlS1m2bBmffPKJYZv333+f8ePH8+yzz3L8+HG2b99OVJRu0Ncff/wBwIcffsiVK1cMX9/qm2++YfLkyUyfPp2kpCSmTZvGxIkT2bJlS7XtXn/9dR566CGOHTvG448/znPPPcfFixfN+l4JgmA+a26g2NLskV3wcndm7rdJFt0s+fKPSxy7lMeC2K480789Xxy4yNFLtvtQmpiSiYuTxCAjDbcDvd3RynCtSPSVsKfElEyCfZV0aVn72HuAIZ0DcFJITaJPSWNVd8v8KrLuX+oqYJUkSc2qnhPD5oUaskuzDcs2lM5Kmrk0M1RPOIIqr5R3dp9lRGQQQ7sEEh3iww8nrzLv25N8/uLdFmXprZGkyifKSJPLm8VEBLFoSzIXcoppb8YH7iv5ujI+ayolDtTR7+FyXilvJ54lJiKoRsMlU+nHQiamZNK5nl/w9TmeUf/3siH5ebrSu31zdiVn8pKZH/yvFpRRWqkh1KJKCV0iIzW7iJY+7ka3+ergJY5czGPl2Lvw8XAx+xy2dHdYc7zcnElMybT456ipWrTlJMmXG7bxb2RrbxbEdjV7v/vvv5977rmHOXPm8OWXX9Z4fe3atbRu3Zq1a9eiUCiIiIjgjTfeYPz48SxevBgPj5q9UVxcXHj99dcNX7dv357Dhw/zxRdf8PzzzwOwePFipk+fzsyZMw3b9eqlWz0aEKB7b/H19aVly9ob7K5cuZKnn36ayZMnA9CpUycOHTrEihUriI29MTDs6aef5qmnnjKcd/Xq1ezbt8/wnCAI9qHKLaVra9stvbRUc09XXhnZhde+PsG3R1U80iPE5H2vFZWzYscp+oY156HurRkWEciWY5eZ++0Jvps0wCZLJBOTM+kb1gJv95rv20HNdI2qMwvKCfQ2/t4vWKe0QsMv53L485/a1vt5wdfDlT+192N3Shaz7mu4yWa3E/Pqk9ElI0RCQqhNdkk2AcobGd0AjwBySh13R/T1LSeRkZkfq5ui4O/lxqyRXfgt9RoJxy43aCwVai0pVwrpVks/Cb2Yqg9q5mZbDSO2rKiUKH3WkVMAACAASURBVCxTU1BmvOnT4q3JyMgsiDVtIoUxthoLWanRknyloN7vZUOLiQjk1NVCQxMvU+knb4RbUPWhX/JxvpZml7nFFazYcYo/tfdjTM9gs49va27OTgzq5G9V/w2hYaxYsYJNmzZx6NChGq+lpKTQt29fFIoblxEDBgygoqKCc+dqXyL33nvv0bt3bwICAvDy8mLVqlWG6oSsrCxUKhXDhg2zKu6UlBTuueeeas8NGDCA5OTkas9169bN8GdnZ2cCAgLIyhL9TgTBnkorNFwrrmgUlRIAj/duQ/c2vizdlkJ+qelNL1fsOEVxuZrFVQ2/m7m7MPfBSJJUBXy+P93quNJyijmfXWy4mXOroKpEhOgrYT+/nsuhrFJrdPmMMTERQZy6Wsil6+ZdAwo6JlVKSJLkBywEhgKB3JLMkGVZdJgSAF1Ty3DfcMPXAR4BDquU+OlUFjtPZjLrvs6E+N24a/dEn7ZsOniJpdtSuLdLIM2MZKDt4UxmIRUaLVG19JPQa9Pcg85BzdidksULA8NMPr5+coY1lRKgS254t6r+Pdl7Jpvvk64y677OtGlu3lzxW8VEBPFW4hmyCssIbGZZdv9MZiEV6vq/lw0tJiKIZdtPsTsli7/1b2/yfvpGlfqqB3O09HbHw9Wp1maXb+48RUGZmsUPmz4pxd5iIoLYfuIqJ1T53NXGMf0tHMGSigVH6tOnD2PGjGH27NnMmzfP5P1q+zn7z3/+w/Tp01m5ciX9+/fH29ub+Ph4vvnmG1uFbFZcLi4uNV6/tb+FIAi2Zek4UHtRKCSWPBzF6Hd/4a0fTrPooah69zmUfp2vDmYwflAYHYNuVH3GdmvFlwcu8ubO04yMakVAM8vHbuuXgtZWUahPSmQViuUb9pKYkomXmzN3h7YwafthEUEs2ZZCYkomz95jerN0QcfUSolPgVjgv+iSE/NueQgCGq2Ga6XXDKNAAQKVgQ5pdFlWqWFBwknCAzx58ZYP9k5Vb0DZReWs2nW2wWLSN2Y05e5+TGQgBy6YN6pKlVuKQqLWEv766JMZt07gKKvUsOC7JML8PXlhoPW/ZIdFBCHL1o2F1I92dVTDxtqEBXgRFuBpdpXL+exiPF2dCPI2/wJGkiRC/T1Jza5ZKXH4Yi5f/nGJZ/u3p0tLx5fK6un7b4i1l43fsmXL+Pnnn9mxY0e15yMiIvj999+rfYj/5ZdfcHV1JTw8/NbDGF6/++67mTx5Mj179qRDhw7VelYEBgYSHBzM7t27a43HxcUFjabuJqkRERH8+uuvNc4dGWl5lZcgCLZhSEr4WneDw5aign14um87Nv6eXu/oeLVGy5xvkmjl487UYR2rvSZJEq8/FEVZpYbl36dYFdOu5Ey6tGxW640gfy9XJElUStiLViuz+1QWgzsH4Ops2sflUH9POgR6sTtFVNxZwtSkxBDgUVmWF8my/J4sy+/f/LBjfEITkluei0bWGHpKQFWlRGmWTaYtmGPdnvNcvF7C4oeijP4y6RbiyxN92rLhf2kNtsb7hCofb3dn2ppQaTAsIgiNVmbPGdN/sWXkldLS293sqRF6hkqJWyZwvL83lQvXSnj9oSizx1Uaox8LmWjFL+3jGfk0c3emnZVVG/YQExHE76nXKKxlGYwxqTnFhAZ4WlzJEOrvaVgCoqfRysz7NonAZm5MH97JouPai5+nK73bNbfqZ0BoGB06dGDcuHGsXr262vMTJ07k8uXLTJw4kZSUFLZt28arr77K5MmTjfaTAF1vh8OHD/P9999z9uxZFi9ezN69e6ttM2fOHN5++21WrVrFmTNnOHr0KP/85z8Nr7dv357du3dz9epVcnONj06eNWsWGzduJD4+nrNnz7JmzRr+/e9/M3v2bCu/G4IgWMvapab2MnNEZ5p7ujH326Q6lxZ++ls6p64WMv/BSDzdahacdwj04sWBYXx9WMWBtNr7dNUlr6SCg+m5huW8xjg7KfD3ciOrUCQl7OG4Kp/swnJiTFy6oTcsIpDfU6/VuhRaqJ2pn17Om7GtcIfSL9PQjwIFCPQIRK1Vk1fecCNy0q8Vs27veWLvak3/Dv61bjf7vi74ebgy77u634Bs5YQqj24hviZ98Owe4ou/l6tZH9pUuaVWvcn7e7rh6qyo1g/h4rUS1u45x4PdWjGgY+3fS3Pox0L+fNbysZAnVPlEN6ImlzeLiQiiUiOz74zpvVRSs4sItWDyhl5YgBcZuSWUq298Pz/7PZ2TlwuY92AkXkYunBwtJjKQlCsFZvffEBre/PnzcXau/jMUHBzM999/z5EjR+jevTvPPfccf/nLX1i2rPaBXOPHj+exxx7jiSee4E9/+hMXLlzgpZdeqrbNhAkTiI+P58MPPyQqKoqRI0dy8uRJw+v//Oc/+emnn2jTpg09evQwep6HH36YNWvWsGrVKiIjI1m9ejVr166t1uRSEATHUOWV4KSQDI0aGwsfpQtzHujC0Ut5/OfgJaPbZBWU8dauMwzqFMDIqNqb7U6+twPBvkrmfZtEpcb8JWF7Tmej0crERNbdDDqwmVudk7cEy+1OycRJITG0s3lJieERQai1MntPO3byYFNkaqJhGrBckqS7JEmy/lapcFvSL9OoVilR1fSyofpKyLLM/O9O4uqkYO4Ddc+x9vFw4dVRXTiUnsvmwxl2jausUsPpq4Um90BQKCTu7RLIntNZVKhNe0OzdsSWQiER7Ks0VErIssyChCScFRJzH7Bt2bM1YyEr1FpOXSkkupH1k9Dr2dYXXw8Xk5cmlFVqUOWVWjXaNDzAE60M6dd0H/CzC8tZ+cNpBnTw54HoVhYf1570d4BEmWPjsmHDBrZu3VrtucDAQAoLC5FlGX//G8nJQYMGsX//fsrLy8nMzGTVqlW4udX+QcPV1ZX169eTm5tLXl4e69evZ/78+Vy4cKHads8//zzJyclUVFRw9epVPv74Y8NrsbGxnD17lsrKSsN+zzzzDEVF1Xuq/P3vf+fcuXNUVlZy7tw5XnzxxWqvy7LMo48+Wu25Cxcu8PLLL9f7PRIEwXKqXF1Vp7OFVZ329HD3YO4Obc6KHae4XlxR4/Wl21Oo0Gh5fXTXOm8webg6syA2ktOZhfzrfxfMjmNXSiYBzdzoVs91TpC3u1i+YSe7kjPp3c4PXw9Xs/br0daP5p6uFo2Hv9OZ+hvhHKAEDgMVkiRpbn7YLzyhKckqraqUuKmnhD5BkV3aMBnDnSevsvdMNjOGdzI0AarLmJ4h9G7nxxvfnyKvpOYbkK2cvlpIpUY2a1pETEQQhWVq/qhjTKeeRitzNb/M6nLIYF+lobTyh+RMfjqt+15a2qeiNjfGQpr/gVTfMDS6kU3e0HN2UnBv50B+PJWF2oQ7JOnXSpDlG1M0LBHmf2MsKMDy7SmUVWpY9FDdF06OFBbgRZi/+f03BEEQhKZLlWddVac9SZLE4oejKCpTs+L7U9Ve+9+5HL47epm/Dw43aVz78Mgg7u0SyKpdZ7iab3rioEKtZe/pbIZ1Cay3GjTIW1RK2ENGbgmnrhbWuXymNvrqih9PZVlUJXMnMzUp8QXgA0wFHgceu+UhCOSU6O56t1De6FKrr5RoiGaXJRVqXt+STJeWzfhbv3Ym7aNQ6N6A8ksreXPnabvFdqKqcZI5d/cHdPTHzVlh0oe2zIIy1FrZ6sZR+koJ/feyc1Azs6ZImMrN2YnBnQLYnZJp9tIZQ8PQ4MbV5PJmMZFB5JdWcijd+Jr3m+kTCeEWTN7QC9WPBc0u5vfUa3x9RMW4QWFWHbMhxESa339DEARBaLpUuaWENJJxoMZ0CmrGcwNC+c/BS4b38Aq1lnnfJdGmuZKJQ4w38r2VJEksjO2KWiuzeFty/TtUOZB2naJytUkfiAObuXOtuFx8+LUxfQVnfctnajM8MpCCMjUHL9R/DSjcYGpSojfwF1mW42VZ3izL8n9vftgzQKHpyCrNorl7c1wUN8asNWSlxDu7z3E5v4wlD0eZVRYY0cqbZ/q354sDFzl2yT69L05k5OPn4UKIGXcHPFyduaeDP4kpmfU2CrXViK1gPyU5RRWs3HkGVV4pSx6JsrhxZn2GRQTy/+y9eXxU9dn3/z4zk8xktoQkMxOSsBggkABhUVmksoSgpWj1dutdW1u9b7V3e6u11qUVREW72Ralz9O7z9NKXdqf1tYHF1BUlqi4IKJgEgh7CCSEzCQhk5lMJrOd3x+TGRKyzEySSTLwfb9eeTEzZ7synJzzPdf3c30uq6M9nLCJlvLaZlJTkhiTPnIHNZdPyiRJKUWVUDrWYVB50QDKN/RqFRajmkP1Dla/UUFOWgp3LZkUecNhZukUc8z+GwKBQCBITLz+AKdbBq7qjDc/XjqJLKOGR16vwOcPsP6jKo7aWnn8m1PRJEVfxT42Q8uPFk/krbI6dhyObhy8tbIetUrBgj480UJYjBpkGRqcQi0xmGytrGeCSdfvcdnlk0wkKxWihCNGon3a2A+MnH5yghGJzWULKyNCqJVqUtWpcfeUOFzv4Nkdx7jx4lwuGZ8e8/b3lkzCpA+6LvvjYHpZVmtnWk5qzFL6kgILJ5vaOFTv7HO9kFngQDwlgHDS5K8fV3H97Fwu7cd3GS2htpCxXrTLaoImlyO1LAHAoEliXl5GVOUpR21OsoyaHl28Y+GiTB0bvzrFoXonj31zKinJI9/+5+Jxo0jTJokbt0AgEFwAnLa7CcgDH6vEG51axeqrC9lf18JT7x7kD9sOc0WhheIpsc+c/2BRHuMztKx+Y18XM+qekGWZLfvruXxSZlT38FAbcVHCMXg43F52HmvsV+lGCJ1axfwJGWyJYlJRcJZoR8GrgLWSJK0CyoEuWltZlvvX80ZwXmF1WcPKiCNWB2qVkjHpWkwppn6Xb7i9ft4ur4to9vjP3SfRqVX8bPmUfh3HoEnikasKufvlPbz0WTW3zB/fr/30hNvr51C9g+IpeTFvu7TADK/B/7x/hPl5Gb2u92FHBj4WJUZPhAYKRo2Kn3+jf99ltIzSJXPJ+HS2VFq574rJUW0T+i5vvzz273KoWVZoYfUb+/jLh8cwaHq/1O490TwglUSIPJOenceaWDrFzLJ+Sg6HmrD/xsGg/8ZIND4TCASCocDf3Izn+HFSZs4c7lBiorXdx96TzVHN7NeM0HagPbF8WhaXT8rkzx8eQ5OkYPXV/TP81iQpefyaaXz/r7t45PUKZo8d1eu6zW1eapvbuLt4YlT7DnmnWeNodtns8nDU1srF43qPe6ixOdrZfqCeSM/72WkpXD4pM6ZJrA8PNeD1R+58EomSQguPvF7BUZuTiWZD1Nv5/AHe219PS1vkstaZY9OYknX+aAaiTUq83fHve0DnU0DqeD/yp+QEccfWZqMgI9jx4p6X95KhT+Zv/zkXs9bc7/KN1/bU8vMN5RHXkyT4zfVFZOj732LqqqLR/O3Tap77+PigJiUq61rwB2Sm98MDwWLUMOeidN7Ye4o39p7qc928TF1MssKemGjWo0lS8PA3CsgcwHcZLV+fmsWaTfv55EhDn+1bQ4QNQ0do543OLCu08ORblfzi7cqI614xtffWYtEya0waG/ee4rFvTh3wvoaSK6dlsWFPLRv21HLTJWOGOxyBQCAYFqxrn8a+cSOTv/xiRCsBz+XlXSd48q1Knv3eJREf5MKlpiNcKQFBT4g110zj2j9+zN3FE8kd1X/PrkX5Jq6blcM/d9fwz919d3vTJCkoLoiuDaU5pJRwxE8p8T/vH2X9R1V8vrKEdF1snSjigc8f4Jb1n3HgtCOq9f/Pdy/us33ruWytrGeUNqnP5FE0LJ1i5hFga6U1pqTEH0uP8vTWQ1Gtu2pFwQWZlFgS1ygECY8v4KOxrTFcvlHb3MbJMy5kWcaUYuJo89F+7ferk82M0ibx1j2X09c9Wq1SDvhiKUkSiyab+O27B2lxezFqkiJvFAUhz4RYOm905u//OZfG1sg3nFExti3qiQy9mq8evQK1amjyjDfPHctzn1TxyBsVbP7xQpJVfc+Ul4UMQ0do543OjE5NYfeqElrbfX2uJyFhHoR+7TdeMoarZ2QPODE11CwrsHBxRwecKwotMbffEggEgkRHDgRwlG5HbmtD9niQ+mitO9LY2+HF9djGfSyY2HfZQai7V3YCJCUgWBb5+cqSiGOTaPj9TTN48OtTkOl7el+nVkU9/szQqVFI8VVK7D3RjD8gU3rAyvUX58btONHywqfVHDjtYO1NM5g/oXcFsSzDfzz/OWs27mNhfiba5MiPvD5/gO0HrCwtMKOM0PkkEtlpKUzNNrJ1fz3/tSg6c9QTjS7+5/0jfGN6Fo9cFVmZYxik55SRQlRJCVmWP4h3IILEprGtERkZs9aM2+vH3iE7qm50YdaaaWhrICAHUEixXdjLaoJeDEN1Awt1x6iotXPZhMgz99FQVmMnQ5fM6H621UxWKRidOnQ38KFKSECHrPGbU/mP53ez/qMqfhjB1bq8JpikSoRZFgCjJmnQklvRkGgJCejogHPNNK76Xzv47bsH+cW/TR/ukAQCgWBIcVdU4LcFDX8Dra0oEigpUV5rZ3yGluMdD1Q/7aMcs7bZRaZenVD3qsFISEBw4muw26srFRImg5r6OCUl/AGZilPByaCtlfXDnpSob3Hz9JZDLJ5s4t9m5URUFD157TRu+D+f8odtR6Iq7/6i+gz2Ni/LBuAn0ZmSAgt/2H6YRmd7RCW3LMs8+mYFKoXE6qumDvq5kgjE9JcmSVK2JEnzJEla2PknXsEJEodQeYYpxYS1k+FOWa2dzJRM/LKfJnds1iMh/4D+Kgz6QygpUV4TW0eIviivsTM9d2QbMw4nxVMsLCu08Idth8PSzt4or21hem6a+C7PMwqzjXz/svG8FMcOOAKBQDBScZSWhl8HWluHMZLYsLu8VDe6uOnSMfzbrBz+7wfHwm2ue6K2uS0h/CQSCYtREzejy2M2Jy6Pn3RdMh8eskU06ow3T75Viccf4PFvTo1qHHjJ+HRuuDiXZ3cc44g1crnH1sp6kpUKLs83RVw3GkoKLMgylB6MXML+3v56Sg/a+Mmy/AsyIQFRJiU6khHvAzXAx8D7QGmnH8EFTsjI0qw1U+84m7Etr2nGrDV3WSdaKuta8PXTi6G/jNIlkzsqJVwmMFDaPH4OWx0J4YEwnDx6dSEyMms27ut1nXCSSnyX5yX3LcuPawccwdAzfvx4fve73w37PuLBq6++KpKjgkHDub0UlEH1QCIlJcLlqTlp/PwbU1AnKXj0zX29dhyoPdM2YENuQVfMBk3clBJlHRN0P1iYR6vHz85jw9fX4OMjDWz86hQ/WjyBcRnRm4P/fPkUdGoVj7ze+3kZYlullXkTMtAPsBtaiGk5RixGNVv3991hzOXxsWbjfqZkGfj+ZeMH5diJSLRKiWcAP1AIuIDLgRuBSuDr8QlNkEiElRJaU/jiaFCrKK+1hztyxGp2WTFAL4b+UpSbGj72QNlfZycgw/TcoUusJCK5o7TcXTyJd/fVU3qw5zaa+0OGoQngJyGIHYMmiZUrCiivtfPSrhPDHc4Fy6233ookSUiShEqlYuzYsfzwhz/kzJkzwx2aQHBe4q2tpf3gQXTz5wOJmZSYlmPEbNBw/xWT2XG4gbfLT3dbNxCQOdXsJjdByi8TBYtRjTVORpfltXZSkpTcMn8cKUnKiA/X8cLjC/DIGxWMy9BG7c8QIkOv5oErJ/PpsUbe/Kp3w/ijNifHGlpZFqXJaDRIkkRJgYUPD9twe3tXmfzv7UeobW7jiWunkXQBdyGL9jdfBDwky/IBgt02bLIsbwAeAp6IV3CCxMHqsiIhka5JD5dvLJpsoqK2hUxNR1IiRqXEQL0Y+sv0nDSqG13YXZHb8UQilGWeLmb3I3LH5XnkmXQ8+sa+Hi/e5eK7PO/55oxs5udl8Nt3DtDgFH3Xh4uSkhLq6uo4fvw4zz77LBs3buRHP/rRcIclEJyXOErfByD16quAREtKNDM2XRs2KP7uvHFMzTbyxKb9OM8xeW5wtuPxB0T5xiBjMWpoavXEpbSivNbO1Gwj2mQVl0/KZFtlfUS1QTz4y45jHLO18tg3p/bLj+Tbc8ZSlJvKk29V0uLueWwfSrgUD5KfRIiSAgsuj5+dxxp7XH7E6uQvO45x/excLh2fPqjHTjSiTUqkAA0dr5uAUBppP1A02EEJEg9bm42MlAxUChX1DjfJSgULJ5lwtvtwuII3IGtbzzPgvVFeOzxeDCFlRvkgqCXKa+yYDGosxsQxrRouklUKnrhmGieaXPzp/e7dWspr7WTqhz5JJRg6JEniiWun0ub18+vNB4Y7nAsWtVpNVlYWubm5XHHFFXzrW9/ivffe67KOJEm8+uqrXT6LVGpht9u58847MZvNGAwGFi1axO7duyPG43Q6+e53v4terycrK6vbMdauXUtRURE6nY6cnBxuv/12mpvPepPY7XZuueUWzGYzGo2GvLw8nnnmmZjievHFFxk3bhxarZarrrqK+vrhmTEUnH84t28nOS8PdUGwpXoiJSXKOjyzQigVEk9eO416h5t157Q1rEmgdqCJRGh8aRtktYTPH2DfqbP/vyWFFk7Z3eyvaxnU40Si5oyL/7X9MF+fmsWSyf1TMYTOywZnO09v6bnd5rZKK4WjjYN+fs6fkBFUmVR2v2fIsszqNypISVLy829ENuI834k2KXEACH1be4H/kiRpHPDfQG08AhMkFlaXNdwO1NrSjtmoDl/IDpxyka5Jj0kp0eYZPv+AadnBY5bVDtxwr7zWTlGOMLmMlgUTM7l6RjZ/+uAo1Y1dB2blNXami+/yvGei2cDtl+fx6hc1fH58+OpXBUGOHTvGO++8Q1LSwLrIyLLMihUrqK2tZdOmTezZs4eFCxdSXFxMXV1dn9uuXbuWgoICvvzySx5//HEefvhhNmzYEF6uUCh45pln2LdvHy+99BK7du3i7rvvDi9ftWoV5eXlbNq0iYMHD/LXv/6VnJycqOP67LPPuPXWW7nzzjvZu3cvV199NatXrx7Q9yEQAPidTlo//xz9ksUodcE6+URJSpxp9VBzpq3bOG3W2FH8+6Vj+OvHxzl4+qy5YKgdqFBKDC5mQ3CiZrBLOI7YnLi9gfBEXfEUM5IEW/fHNsE4UNZs3I+ExOqrI7fI7Iui3DS+M3csL3xynH2nuk46NrV62F3dREnh4KokINgVbWF+Jtsqrd1UJhvL6vjkaCMPfH0KmRG6c1wIROvksQ7I6ni9BngH+DbQDnw/DnEJEgyby0aWLniK1Le4MRvUTDLrUasUQV+JFFNMSYn9dS3D5sWQqk1iXIZ2wB04Wtt9HLE5WVE0epAiuzBYtaKA0gNWVr+xj+dvuxRJknB5fBy2OrhyWlbkHQgSnruLJ/Lm3lM88noFm+7+GqrzpMby9C9/SXvl0CpA1AVTyHr44Zi2eeedd9Dr9fj9ftzuoEfQ2rVrBxRHaWkpe/fuxWazkZISfCh54okn2LhxI3/729948MEHe9127ty5rFy5EoD8/Hw+//xz1q5dy3XXXQfAvffeG153/PjxPPXUU1xzzTW88MILKBQKqqurmT17NnPmzAFg3LhxMcW1bt06li5d2i2G9evXD+g7EQhaP/oIvF4MxcUoEiwpEVKT9lRS+eCVU3in4jSPvFHBK3fOQ5KkcHctoZQYXMwdSgnrIJtdni2ZDY7DM/VqZo1JY9uBen5cMmlQj9Ub2w/U897+eh76+hSyB+G8eeCKKWwuP80jr1fw6n9dhkIRnOQqPWAlIEPJIPpJdGZpgYV399Wz71QL0zr+XhxuL09u2k9Rbio3zxkbl+MmGlGN9GRZ/v9kWX6+4/WXwHjgUmCsLMv/ilt0goTB1mYLG1rWt7ixGDWolAqmZhuDJQxaU0zlG+U1QZXCcPkHTM9JHXD5xr5TLciy8ECIFYtRw70lk/jgkI139wXNsvaf6khSie/ygkCbrOKRqwo5cNrB858cH+5wLjgWLlzI3r17w4qDb3zjG9xzzz0D2ucXX3yBy+XCZDKh1+vDPxUVFRw92r1cqzPzOwwAO7/fv39/+P327dtZtmwZubm5GAwGrrvuOjweD6dPB68fP/zhD3nllVeYMWMG999/Px988EFMcVVWVvYYg0AwUBzbt6NMSyNl5sxwUsKfYEmJqT3cl0fpknno61PYVdXEa3uCguraM20YNSoMmoGprgRdsRiDSonBbgtaXmtHl6wkL/Nsp4uSQgtlNXZO2+PT7aMzbq+fR9/cx0Sznv/82kWDss9UbRI/Wz6FL080868vToY/33agHotRHVZKDzZhlUmnEo6ntxzG5mzniWumoVQIBTBEr5TogizLLuDLQY5FkKB4/V6a3E2YU4IZRmtLO5dPCiYoinLT+Ofuk1w/zcTBpoNR77Osdni9GIpyU9lUVkdTq4d0XXK/9lE2zImVRObWy8bz6hc1rNm4n4X5prNtx0TnjQuGK6daWDLZxDNbD3P1jOzwwCuRiVWxMFxotVomTpwIwB/+8AeWLFnCE088wWOPPRZeR5KkblJUr7d3c+BAIIDFYmHHjh3dlhmNxn7HWl1dzYoVK7jjjjtYs2YNGRkZfPnll3z729/G4/EAsHz5cqqrq9m8eTPbtm1jxYoV3HjjjTz33HNxi0sgiITs89H6wYfoFy9GUipBqURKTk4YpURZTTMXZepITek5yXDTJWN4ZfdJfvl2JUsLLNQ2t5EzSjvEUZ7/pGuTUSmkQW8LWlZjZ2pOalhNAEHTxqfeOci2A/V8Z+64PrYeOP/z/lFONrXx0h1zSVYNnlry+tm5/HP3SX69+QBXFGahVSv54KCNa2bldPldB5NMvZrZY0extbKee0vy2X+qhRc+Pc7Nc8YyY4zozhfi/NDECoaVRnfQUdakNdHa7sPRYVjyUgAAIABJREFU7gvLyablpOLy+FGRSqO7EX8gOnfg8prh9WIIyasGopaoqLWTZdRgPg8epoYalVLBk9dO45TdzR+2HaG8xo7ZoD4vHkwF0SFJEo99cyoef4An36oc7nAuaB599FF+85vfcOrU2XZqJpOpixdEfX19n94Qs2fPpr6+HoVCwcSJE7v8mM19S2Z37tzZ7X1Bhyng7t278Xg8PP3008yfP5/8/PwucYbIzMzklltu4fnnn2f9+vW88MILtLe3RxVXQUFBjzEIBAOhbc8e/HY7+uLi8GcKnS5hkhIhn6feUHSYCza1evj9ewepPdMmSjfigEIhYTaoB1Up4fUH2F/X0s0vZJJZz9h0Ldsq4+srUdXQyv/54CjXzMzmsgmZg7pvhULiiWun0eL28dS7B9l5rIlWjz9upRshlhaYqaht4VRzG4+8UUFqShIPXDk5rsdMNERSQjBgrK7gxcmsNYeNdiwdxjuhme22Nh0BOUCTO7JxXWu7j6M2ZxdH56EmnJSo6b/ZZVmtfVh/h0TnkvHp3HBxLs/uOMaHh21CJXEBMi5Dx48WT2DjV6f4+EhD5A0EcWHx4sUUFhby5JNPhj8rLi7mj3/8I7t372bPnj3ceuutaDS9Jw1LSkpYsGAB11xzDZs3b6aqqopPP/2URx99tEeVQmd27tzJr371Kw4fPsxf/vIXXnzxRX7yk58AMGnSJAKBAM888wxVVVW8/PLLXTprAKxevZrXX3+dw4cPU1lZyYYNG8jLy0OtVkcV1z333MPWrVu7xPDaa6/19+sUCABwbC9FSkpCt2BB+LNgUsI1jFFFR4OznVN2d0Ql6NTsVL43fzx/31nNsQYnucLkMi6YjRqsjsFTShyud+LxBbqNYSVJoqTAwkdHGnB5fL1sPTBkWebRN/ehVipY+Y2CuBxjSpaRWy8bzz8+P8H/3n6YlCTloCc/zmVZR6vRe1/ZyxfVZ/jZ8inhVrqCICIpIRgwIQNLU4opbLQTmtGeYNKTkqSkyR59W9CwyeUwlj0YNUnkZer6rZRwuL0cs7UOS/eQ84mfLZ+CNllJg9MTThQJLiz+a9EExmVoWfnmZ7i98RkECSLz05/+lPXr11NdXQ3A73//e/Ly8li8eDE33HADt99+e5+KB0mSePvttykuLuaOO+5g8uTJ3HTTTRw8eJDs7Ow+j33fffdRVlbGrFmzWLVqFWvWrOGGG24AoKioiHXr1rF27VoKCwt59tlnu7UMVavVrFy5khkzZrBgwQIcDgcbN26MOq558+axfv16/vSnP1FUVMSGDRu6lLIIBP3BWVqKdu5clPqzNfsKnY6Aa2iSEvY2L25vdOrVcwmbXEYxWXDfFfmk69R4/bJQSsQJi1E9qOUb5R3d54p6MJsvKTDj8QXYcTg+EwXvVJzmw0M2frIsP65K43tLJmHSq/n8+Bkun5SJJkkZt2MBTDTrGZehZVdVExePG8UNs3PjerxEpF+eEgJBZ0KJBpPWxOGaDqVER/mGUiExLcdIbWMTaDoSGBl976+spndH56Fkem4qn1f1ryXh3pPN4X0I+k+mXs2DX5/CqtcrmD121HCHIxgGNElKHvzGGB7adSO/+dDJo0u/Ndwhndc8//zzPX5+8803c/PNN4ffZ2dns3nz5i7rXH/99V3eHz9+vMt7g8HAunXrWLduXdTxnLuPnrjnnnu6GXHedNNN4dcrV64Md87oiWjiuu2227jtttu6fHbXXXdFjE0g6In2Y1V4jh9n1Pdu6fL5UJVvONt9LH/mQ+ZNyGDtTTNj3r68xo4kwdTsyL4rRk0Sq1YUcO8re7mok2miYPCwGDXsPDZ4LbTLauwY1CrGpXf3ALn0onQMGhXbKuu5curgdkRrbfexZtN+CkYb+d78+HpWGDRJPHJVIXe/vIcrBvn36AlJkrii0MJfPz7OE9dMi5t/RSITdVJCkiQLcAswAXhEluUGSZIWAKdkWa6KV4CCkY/NZUMpKRmlHoW1JTiL1jm7OS0nlZe/UJB00dlSj74or2keEV4M03NSeWPvKWyOdkyG2Aw3tx+wolYpmHNRepyiu3D4ztyxTM02MlOYAV2wTBodQFL4+OzkkeEORSAQCAaMs3Q7AIYlS7p8rtDp8Df3v2w0Wp7ZcohTdjdb9tXj8QViNhIsq7GTl6mLupPGNTOzuShTJxSPccJsUIeVL4Mx419ea2faOSaXIZKUChZPNrOt0oo/IA9q54g/bD9Mnd3N/7559pC0Ar96RjZj0rVDNgl6b0k+N1w8hslZhiE5XqIR1f+4JEkXAweB7wD/CYRSo8uAX8QnNEGiYHVZyUjJQKlQUt/iRq1SYNSczXcV5abidmuRkLC12SLub6R4MYQuUhUxlnDIsszWynoWTMxEmyzESANFkiRmjR01bKanguGn1RecOaxqPBO3OlaBQCAYKhylpagLCkgaPbrL50OhlDhwuoXnPjnOJLMeR7uPXf1QhFbU2nuU9veGJEnMGJMmWh/GidAkns0xcLNLjy/AgTpHnz5eJQVmGls9YVXwYHCo3sH6HVXcdEkuF48bOmXszCE8L3VqlUhI9EG0aajfAetkWZ4FdD7j3wUW9LyJ4ELB1mYLtwOtb2nHYtR0eYCcnpMGKNGq0sL+E73hcHupahgZXgxTc1KRpLPlJNFyqN7JyaY2SjpMbQQCwcBweBwA+HHzUZzqWAUCgWAo8J05Q9uXe7qpJAAUOm1ckxKyLPPI6xUYNSpe/M85qFUKtlbWx7QPa4ub0y1uoXoYQYR83AbDV+JQvQOPv7vJZWcW55tRKSS2xXju9EbovNRrVPxseXzMLQUjn2iTEhcDL/TweR0gnrwucKwuKyatCQheEEN+EiHyMnXokpWo5NSI5Rv7TrUgyzBtBCgl9GoVE0z6mM0uQzf4pXFuLyQQXCi0eFoAUCd7Yh5ACwQCwUjC+cEHEAh0aQUaIt5Kif/3ZS2fHz/Dz5cXMDo1ha9NzGRrZT2yLEe9j9CYSHTEGjmExt2D0RY0Gl+3VG0Sl45PH7T78Rt7T/FZVRMPXjmFdJ3oSHGhEm1Sog3oSUszBYhvs1rBiKehrQGzNvgAbnW0d/OCUCgkpuak4m03RCzfKB8hJpchpuekhl2Io2VrZT1FuanhzLVAIBgYTo8TAEuaxPYDVgKB6AfQAoFAMJJwlr6PymxGM7Ww27JQ941YkgTRYnd5+dXblcwem8YNFwed/0sKLdScaeNgvSPq/ZTV2FFIUDg6ssmlYGiwGAZPKVFe24xRo2JsDyaXnSkptHCo3smJxoF1i7G3eXnyrUpmjEnj3y8dM6B9CRKbaJMSbwCPSpIUmgKXJUkaD/wG+H9xiEuQIHj8HprbmzGldFJKGLo/jBflpOJoTYlYvlFWaycnLYVMfWzGkvFiek4q9S3tUV/obY529p5sFqUbAsEgEirfSNfLNDg97K2JvxGcQCAQDDYBj4fWHTvQL1nSo0+SQquFQAC5rW3Qj/3b9w5wxuXhiWvPOv8vnRKcUNpWGf38YkWtnYlmPTq18MwaKaRpk0hWKqh3DEZSIugXEsnHq6RDDTxQtcTTWw7R2NrOk6IjxQVPtEmJ+4F0wAZogY+AI0AzsCo+oQkSgZDywaw142z34fL4u5VvQLA1ps9joMndhDfg7XV/FbX2EaOSgLPyxPIofSVKD1iRZURSQiAYRBzeYFJCq/GhUkhs3S9KOAQCQeLh2vU5AZcLQ3F3PwkIKiWAQS/hKKtp5v/77ATfmz+eqdlnx1hmo4YZualsifKaKssyZR2dGQQjB0mSMBvVWAdYvuH2+jl42hGV2fy4DB2TzPoBJSUqau28+Olxbpk3bkQY3AuGl6iSErIst8iy/DXgWuAhYB3wdVmWF8myHP+GyoIRS0j5kJmSGVYT9FS2UJSbhuwzIiPT2NbY477sbUGTy5F0YSrMNqKQiNpXYktlPdmpGgpGC3ddgWCwCCkl2v1tg1rHKhAIBEOJc/t2pJQUtPPm9bhcGYekhD8QNBHM1Ku574r8bstLCizsPdmMNYpZ9vqWdmyO9hFhRi7oisWoGXD5xsHTDrx+OerJwZJCC7uqmrC39T7Z2BuBgMwjb1SQrkvmp1dMjnl7wflHtC1BJwDIsrxdluXfybL8lCzLWzuWLY1ngIKRTci40qw1hy+G5h6UEuPStWikoC1JbyUc+2pHlp8EgDZZxURzdGaXbq+fjw43UFJoEe0rBYJBJOQp0eprHbQ6VoFAIBhKZFnGUVqKbsFlKNQ9l6iGlBL+QUxKvLzrBF/V2Fm1ogCjJqnb8qUdys7SA5FLOMo6Suemx9AOVDA0mA3qASclymIch5cUWPAFZD441Hdpdk/8c/dJ9pxo5ufLC0hN6X5eCi48oi3feE+SpG56dEmSSoDXBzckQSIRKt8waU1h2Zi5B08JhUJiQno2ANa2nm98sV4Mh4rpOWmU1dgjGk99crSBNq9flG4IBINMSCnR6m0dtDpWwcjg+PHjSJLE7t27+72P999/H0mSaGiIT7vY8ePH87vf/S4u+x4Ir776qkiAJxDtBw/iq6vDsKR7140Qg12+0ehs57fvHmR+XgbfnJHd4zoFow3kpKWwZX/kpERFrR2lQhImlyMQi1GD1TGw8o2KGjujtEnkjkqJav2ZY9LI0CXHXFJ5ptXDb945wJzx6Vw3O6c/oQrOQ6JNSrwDbJEkKfy02JGQeI2g34TgAsXqsqJSqEhTp3Uq3+h5BqAoK+iqW+fs+cZXXmtnTHoKo0ZYO6Ci3FQanO2cjpCB3rLfii5Zydy89CGKTCC4MAh5Sri8rkGpYxX0js1m40c/+hHjx49HrVZjsVhYunQpW7ZsCa/T34f0xYsXc9ddd3X5bMyYMdTV1TFz5syo9tHTsS+77DLq6urIyMiIOSaBYKhwbN8OkoR+8aJe1xnspMSvNx+gtd3HE9dO7TWBJUkSSwvMfHTEhtvr73N/ZbV2Jpn1pCQrByU+weBhNqpxuH24PL5+76Os1s70KEwuQygVEsVTzLx/0IrXH4j6OE+9e4AWt48nrp0mEquCMNEmJe4C9gGbJUlKkSRpGcGOHD+VZfn/xi06wYjH5rJhSjGhkBTUt7SjTVai78WR+ZKxY5BliYO2mh6Xl9eMLJPLECGPi7I+zC4DAZntB+pZNNmEWiVu1gLBYBJSSrh8wVZ5A6ljFfTN9ddfz65du1i/fj2HDh1i06ZNLF++nMbGnr2ABopSqSQrKwuVqv9O/snJyWRlZYnBrWBE49xeSsqMGaj6SJ6dTUoMvDxt9/Em/vVFDbdfnsdEc98+VyUFFtzeAB8f6V1tJMvyiB2nCc62Be2v2aXb6+dQvYPpObGpYJYWWGhx+/j8eFNU63954gwv7zrJfywYz+Qs4b8mOEu0RpcycAtgB0oJKiTulWX5z3GMTZAA2NpsmLQd7UAdbixGTa8Dwxm56cg+A8fO1HVb1uzycKLJxfSckVenWDjaiFIhUdGHr0TFKTv1Le0snSJKNwSCwSbkKRGQA7T52igpMPe7jlXQO83NzezYsYNf//rXLF26lHHjxnHppZdy//338+///u9AUO1QXV3NAw88gCRJ4et9Y2Mj3/72t8nNzSUlJYWpU6fy3HPPhfd966238sEHH/DHP/4xvN3x48e7lW94vV7uuecesrOzUavVjBkzhp/97Gd9Hrun8o2dO3dSXFyMTqcjNTWV4uJiTp06BcCHH37IvHnz0Ov1pKamMmfOHCoqKvr8bpxOJ9/97nfR6/VkZWV1U2usXbuWoqIidDodOTk53H777TQ3n21da7fbueWWWzCbzWg0GvLy8njmmWe6LL/zzjsxm80YDAYWLVrUraTlxRdfZNy4cWi1Wq666irq64VaKFHw1ltxV1SgX9Jz140Qg6WU8PkDrHq9guxUDfcsnRhx/bl56ejVqj4VaKfsbhpbPeGuZIKRRchkvr++EvvrWvAH5JjH4ZdPyiRZpWBrFOU/IdNVi1HNj0u6m64KLmx6TUpIkjS78w9QBPwKyAZeBL7otExwgWJz2TCnBGu8rS1uzIaeSzcAxqZrUQSM1Dm73/RCRpIj8WanSVIyyazvUymxtdKKQoIlHT2/BQLB4OHwOEhRBWtcXT4XM8eMIkOXzDZRwjGo6PV69Ho9b775Jm53zwPbDRs2kJuby+rVq6mrq6OuLphkdrvdzJ49m02bNrFv3z5+/OMf84Mf/IBt27YBsG7dOubPn89tt90W3m7MmDHd9v+HP/yB1157jX/84x8cPnyYV155hcmTJ/d57HP56quvWLJkCRMnTuTjjz9m586dfOtb38Ln8+Hz+bjmmmv42te+xldffcVnn33Gvffei1LZt8Jt7dq1FBQU8OWXX/L444/z8MMPs2HDhvByhULBM888w759+3jppZfYtWsXd999d3j5qlWrKC8vZ9OmTRw8eJC//vWv5OQEa6llWWbFihXU1tayadMm9uzZw8KFCykuLg7/jp999hm33nord955J3v37uXqq69m9erVfcYsGDk4338foNdWoCEGKynxwqfVHDjtYPXVhWiTI6uQ1ColC/Mz2VZpJRDo2T8r1BpdmFyOTEKl0/X99JWo6Oc4XKdWsWBCBtsO1Ef0Xvv7zmr2nWrhkasKe1VVCy5c+jojdgMy0HnaO/T+v4AfdLyWAaFXv0CxtlmZM3pO8LWjnRkdN6vaBx5EZTZheeCB8LqSJGFMyqTZ0312M5SUmJY98pISELxIb620Istyj0qQrfvruWRcOukdfhi1zlrueO8O/nLFX8jRJ46Jj9Pj5Ltvf5c1C9ZQZCoa7nAEArx+L26/m/HG8RxvOU6rt5XMlEyKp5h5d99pvP4AScpoKxGHl9/s+g0Hmg4M6TGnpE/hoTkPRbWuSqXi+eef54477uDPf/4zs2bNYsGCBdx4443MnTsXgPT0dJRKJQaDgaysrPC2OTk5PNDpen/nnXeyfft2Xn75ZZYuXUpqairJyclotdou251LdXU1+fn5XH755UiSxNixY7nsssv6PPa5PPXUU8ycOZM///msmLOgoACApqYmmpubufrqq5kwYULwO5oyJeJ3M3fuXFauXAlAfn4+n3/+OWvXruW6664D4N577w2vO378eJ566imuueYaXnjhBRQKBdXV1cyePZs5c4L3y3HjxoXXLy0tZe/evdhsNlJSgsm3J554go0bN/K3v/2NBx98kHXr1rF06dJuMaxfvz5i7ILh5+njz+G7xsgvJvatWlBotcDAkhIuj4+ntxxi8WQTV07t/e/kXEoKLLxdfpryWjszxnRPPJTXNqNSSEzph+S+yd3E9zZ/j7WL15I/SsyQxwOzMVS+0T+lRFmNnUx9MqNTu5vVR2JpgYXS1yuYuHIzfRXR+QIyl0/KZMX00f2KUXB+09dI7iIgr+Pfi3p4n9fpX8EFSJuvDYfHgVlrRpZl6lvcWIxqfGfO0PLWWzT/61Vkb9eab4vOhEdu7mamVF5jZ1yGllTtyGwLND03jaZWD7XNbd2W1Ta3sb+uhaUFZ1US+xv3c9JxkqPNR4cyzAFT66zlqP0olY2Vwx2KQACcNbm06IKlUa3e4GA91jpWQXRcf/31nDp1io0bN7J8+XI++eQT5s2bxy9/+cs+t/P7/fziF7+gqKiIjIwM9Ho9GzZs4MSJEzEd/9Zbb2Xv3r3k5+fz3//937z11lsEAtEbqAHs2bOH4uKeOxykp6dz6623cuWVV7JixQrWrl0bVYzz58/v9n7//v3h99u3b2fZsmXk5uZiMBi47rrr8Hg8nD59GoAf/vCHvPLKK8yYMYP777+fDz74ILztF198gcvlwmQyhdUqer2eiooKjh4N3kMqKyt7jEGQGOzV2CgbR0TfE0mpREpJIeDqv6fEEasTZ7uPf790TEw+K0smm1FIvXc2Kquxk28xoEmKfR6yyl5FdUs1e617Y95WEB1GjQpNkqLf5RvlNXam5aT2y5vn32blcN+yfP5rUR4/6OPnJyX5/P6mGcL/R9AjvSolZFmuHspABIlHgytYv5uZkkmL24fbG8Bi1OD84AMIBAi0tOD6cg+6uXPC24xPG83htlYqTjVyybizD/FlNXZmjR25ksCQsVNFrZ3cUdouy7Z33MBLCs/6SVhdwdq6kEFfouD0Bmv3Qw+CAsFwE/KTyNIGZ/xCSYlQHeu2SiuXTcgctvhiIVrFwnCj0WhYtmwZy5YtY/Xq1dx+++089thj3H///SQn99wd6Xe/+x2///3vWbduHdOnT0ev1/Pwww9jtUauM+7M7NmzOX78OO+++y7btm3j+9//PjNmzGDLli0oFIOjiHnuuee49957eeedd3jzzTdZuXIlr7/+OldeeWW/9lddXc2KFSu44447WLNmDRkZGXz55Zd8+9vfxuPxALB8+XKqq6vZvHkz27ZtY8WKFdx4440899xzBAIBLBYLO3bs6LZvo1G0XjwfcCR5aVdHl1xT6HQDUkpUNQS3zTPpY9pulC6ZS8als7XSyk+vmNxlmSzLlNfa+XoMyovO2NuDathTzlP92l4QGUmSMBs01PfD6NLl8XHY6uDKqf3zRdOpVdyzdFK/thUIQvTlKXGdJElJnV73+jN04QpGEta24GDTnGIOy8VMBjXO0vdRZmYiJSfjLC3tsk2BKReAnSfO5rxCCoSR6CcRYkqWAZVC6tFXYkullbxMHRM6DQBsrmCJSugBKlEIJVESLW7B+UvonAwpJVze4AxiqI51a2XkOlbBwCgsLMTn84V9JpKTk/H7u6rdPvroI66++mpuueUWZs6cyYQJEzh06FCXdXraricMBgM33HADf/rTn3jrrbfYvn07R44ciXofs2bNYvv27X2uM2PGDB566CHef/99Fi9ezAsvvNDn+jt37uz2PlQSsnv3bjweD08//TTz588nPz8/bKrZmczMTG655Raef/551q9fzwsvvEB7ezuzZ8+mvr4ehULBxIkTu/yYzcHkfUFBQY8xCBKDluQADpWXdn/kB0aFTjugpMRRWysKCcZlaCOvfA4lhWYq61qoOdNVqVFzpo1mlzfcjSxWWjwtgEhKxBuLUd0vpURlXQsBWfiFCIaXvqYdXgVGdXrd28+/4hmgYOQSevA2aU3hzKwlRUHrjh0YiovRzpuLY/v2Lg8MkzOD/gplp87KZUN+EiOx80YITZKSyVmGcKwhnO0+dh5t7FK6AcGuJHBWeZAohB4AQ7PTAsFwE1LtZOmCM3Qu39nB8tICC9WNLo5Yxfk6GDQ2NlJcXMzf//53ysrKqKqq4l//+hdPPfUUS5cuDc/ajx8/nh07dlBbWxvueJGfn8+2bdv46KOPOHDgAHfddRdVVVVd9j9+/Hh27drF8ePHaWho6LEsY+3atbz88stUVlZy5MgRXnrpJYxGI7m5ub0e+1weeOAB9uzZw5133slXX33FwYMHefbZZzlx4gRVVVX87Gc/45NPPqG6uprS0lLKysooLCzs87vZuXMnv/rVrzh8+DB/+ctfePHFF/nJT34CwKRJkwgEAjzzzDNUVVXx8ssvd+msAbB69Wpef/11Dh8+TGVlJRs2bCAvLw+1Wk1JSQkLFizgmmuuYfPmzVRVVfHpp5/y6KOPhtUT99xzD1u3bu0Sw2uvvRbpv1QwAnC1O/F0VKaGxk19MVClxDGbk9xR2n61J19aEEz+bj/QVeEUNiPv5zitpb0jKdEqkhLxxGzUYOuH0WVowm0kTw4Kzn96TUrIsqyQZdna6XVvP8Lk8gIlVKJg1prDmdnMI/sIuFzoi5dgKC7Ge+IEnmPHwtuYdcGH94ONteHPymuCbdOmxtgbeagpyk2lrMbeJcmy45ANjz9ASUFXyVvou0m0h3uhlBCMNELn5LnlG0A4Gbi1MrYSAUHP6PV65s2bx7p161i0aBFTp07l4Ycf5uabb+aVV14Jr7dmzRpOnjzJhAkTMJmCLaFXrVrFnDlzWL58OQsXLkSn0/Gd73yny/5D5R+FhYWYTKYevRwMBgO//e1vmTNnDrNnz2bv3r1s3rwZbYcBYE/HPpeZM2eydetWDhw4wLx585g7dy7/+Mc/SEpKQqvVcujQIW688Uby8/P5/ve/z3e+8x0eeqjv0pr77ruPsrIyZs2axapVq1izZg033HADAEVFRaxbt461a9dSWFjIs88+261lqFqtZuXKlcyYMYMFCxbgcDjYuHEjEJRdv/322xQXF3PHHXcwefJkbrrpJg4ePEh2djYA8+bNY/369fzpT3+iqKiIDRs28Nhjj/UZs2BkcKbp7HgnNGHRF0rtQJMSreSZdP3adoJJT16mji37u/pKlNXYSVJK5GfFVhISIqSUqHP23DFHMDhYDJp+KSXKa+yYDOpwW1GBYDgQ/VgE/aahrYFkRTLGZCP1juCNVr3rY3wpKejmzcPf3Aw8jrO0FHWHy7kpJTiIPO2sx+31o0lSUlZjJy9Th1EzMk0uQ0zLSeXlXSc52dTG2A5Z5JbKetK0SVw8blSXdRO1fCOk7Eg0hYfg/OXc8o3Of1OjU1OYlmNka2U9P1w8YVjiO59Qq9X88pe/jGhqOW/ePL766qsun40aNapLi8yeyM/P59NPP+32eedE7x133MEdd9wR07EXL17crYTna1/7Gh9++GGP+4gU57kcP3484jr33HMP99xzT5fPbrrppvDrlStXhjtn9ITBYGDdunWsW7eu13Vuu+02brvtti6f3XXXXRFjEwwvTY1n1QGhCYu+UOh0+GyRkxc9EQjIVDW0MjcvvV/bQ9Af67mPq3C4vRg6xmXltc1MyTL2S30BZz0lbG02vH4vScqRPd5LVCxGNa0eP852X0wtN8tq7RTlCJWEYHiJyjVKkiR1p9c5kiQ9LknSbyVJWhi/0AQjHWubFZPWhCRJWFvaMSQrcX/4AbrLLkOh0ZCUlYWmsBDH9rO+EqM0o1CgRFZhYEnoAAAgAElEQVS2sL8umDkvr7X3u05xKAnJFkMyRn9ApvSAlSWTzajOaUkY8ttItId7oZQQjDTCSQltV0+JECUFFr48cYYGZ/96swsEAkE8abKfVQfEu3yj3uGmzeuP2eSyM0unmPH6ZXYcDpZHybJMec3AxmkhpYSMzOnW0/3ej6BvQkqHWNQSre0+jtqcCTEOF5zf9JmUkCRpsiRJ+wCXJEl7JEkqBHYB9wF3AtslSbp2COIUjEBsLhtmbVA+bXW4meWz4aurw1C8JLyOvriYtj178DUF2/YpJAUZKZkoVC2U19ixOdqps7vD3S1GMvlZepKVCspqg+UmX544wxmXt1vphtvnTtiH+7CnRIIlUwTnLw6PAwkJQ7KBFFVKt7+pkgILsgylB0QJh0AgGHmccZxNRIQmLPpCodPhd/Vv7HDMFtxuQmb/yjcALh43ijRtEls7SjhONLlocfsGNJMeSkqA8JWIJ2ZjcA45lqTEvlMtyLLwkxAMP5GUEr8D6oBvAhXA28A7QCpBE8z/C/wsngEKRi5WlzVcjlHf0s7805UgSegXLQqvo1+yGGQZ5wdnZbRZOjPJGifltXYqwiaXI/9iqFYpmTLaQHmHIdDW/fUkKSUW5ndtR9i5ZjTRHu4TNZkiOH9xep3ok/QoJAValZZWX9dzc2q2kdGpGrZW1veyB4FAIBg+mluDigONQh2dUkKrJdDqirheTxyzBcccA1FKqJQKiieb2X7Qis8fCJsgThtgUmKMYQwgOnDEk5BSwhpDW9CyDl+3gfz/CgSDQaSkxDzgflmW3wJ+BIwF/keW5YAsywHgfwFT4hyjYIRia7Nh0oaSEm4Kj39FSlERqsyzD+mawkJUFgvOTu3ZzFozarWT8ho7ZTV2JAmmJsjFcFpOKuW1QbPLrZX1zMvLCNdchggNOpIUSbR6EuvhPuwpkWAGnYLzF4fHgSHZAIAuSdctYSZJEksLzOw43IDbG7ndpEAgEAwlZ9oaAZhgGB+V0aVCp0N2uZB76E4TiaO2VnTJSixGdeSV+2BpgYVml5cvTzRTXmsnWaUg32Lo9/5a2luYlDYJCYm6VmF2GS/MhtiVEuW1drKMGswGYXIpGF4iJSUygFMAsiw7gFbgTKflZ4D+X6UECUurt5VWbyumFBOyLOOrt2I+dQx9cXGX9SRJQl+8BOfHHxNoD2ZuM1MyCShaOGx18FlVIxNM+pgMeYaTopxUHG4f7x+0cdTW2q10A87KM8cZxyWsUiLR4hacvzg8DvTJwVk/XZKum6cEBEs4XB4/nx5rHOrwBAKBoE+a2+2kuGWyjblRe0oABFyxqyWONbRykUmHJEkxb9uZhfmZJCkltlbWU15jp2C0kWRVVDZ0PdLiaSEjJQNTikkoJeKIXq1Cm6ykPgalRKL4ugnOf6K5wsgR3keNJEkLJUl6U5KkWkmSZEmSbj1nuSRJ0mOSJJ2SJKlNkqT3JUma2t/jCeJH6MZq1pppdnmZdWofQBc/iRCGJUuQXS5cu3aFt/HITgJ4+eRoY0I5/oYu3E9vPQScbUnYmdB3c1HqRQn3cB9KSrT72/H6vcMcjUDQVSmhTdL2WFo0Ly8DbbIyXAM9Ugj0Y6ZTIBgsxPk3Mmj2OjC6Jcz6rNiSEv0wuzxmc3JRZv9LN0IYNEnMy8tgy/56KgbYmUGWZVraW0hVpzJaP1ooJeKIJElYjBrqHdEpJRxuL8dsrQk1Dhecv0STlPh7RyLhTUAD/KXT+xdjPJ6eoDfFj4G2HpY/CPwUuBu4FLACWyRJEmqMEUZIgmjSmqh3uJlbtw+veTTJEyd2W1c7dy6SVoujo4Qj5EMhqYLGR4mUoc23GEhWKSirsTMly0DuKG23dWwuG8mKZEbrRiecN0MoKQHCV0IwMnB6nRiSei/fANAkKVk4ycS2Smu31pDDhU6no7a2Fo/HM2JiElwYyLKMx+OhtrYWna7/hoeCwcEeaMXgVZKZkonD6+hR7dWZ/iYl3F4/tc1t5A3A5LIzJQUWqhpacbT7BuT71eZrwyf7MCYbydZlC6VEnDEb1NiiVEpU1CbeOFxw/hJJM//COe//3sM6UScmZFl+m6BZJpIkPd95mRTUmt0L/FqW5f/X8dn3CSYmbiZoqikYIYSVEilmTp5oZqbtMPI3/61HyaBCrUa/YAHO0veRV68Od+xIN7ppbEwMk8sQSUoFBaONfHWymWWF3Us34GyrVH2yPngzDvhQKRKjPMXhcZCuSafJ3YTT6yRNkzbcIQkucBweB5PSJgGgU+mo9lX3uN7SAjPv7DvNvlMtI8KwKzc3l4aGBqqrq/H5fMMdjuACQ6VSkZqaSmZmZuSVBXGlhTaM/uTw2KehrYGxSWN7Xb+/SYnqRheyDHmmwUlKLC0w8+ibQRXsQB5a7e1Bo0xjspHR+tFsObGFgBxAIfW/HETQOxajhq86zCsjUd7RTS6RxuGC85c+n5RkWb5tqAIBLgKygPc6Hb9NkqQPgcsQSYkRRWelxMFP38Yc8KE5x0+iM/olS3Bs2UJ7ZSWm0UGlxBiTlzNNUJhtHJKYB4uinFS+OtnM0h78JCCYsDGlmNAnBSWUrd5WUtUj/4Lf7m/HE/Bg0VpocjcJpYRgRNDZU6K38g2A4ilmJAm27K8fEUkJhUKB2WzGbO5e4iUY+ZxscvGbdw7wm+uL0CWI55FgZNKiaCcnoAurRK0uK2ONg5+UCHXemBBl541ny59lnHEcy8Yt63F57igtU7IMVDW0Msnc/5KQUDtQo9pIdiAbX8CHzWXDout5DCUYGBajmvoWN7IsR/QW2XOimZy0FDL0AzNGFQgGg5GUpszq+PfcouD6Tsu6IEnSnZIk7ZYkabfNFrlOTzB4WF1WUlQp6JP0JH/2MU6Vhqyvzet1ff3iRSBJOLaXYk4JDtJnX6TkvmX5aJMTa8D3rUvHcNuC8b3W4FldHUqJTkmJRCBUujFaNxoQZpeC4UeW5WD5Rh/dN0Jk6NVMzTbyRfWZHpcLBLHwz90n2VRWx5cnxPkkGBgtKi+pkjaslIjUgaPfSYmG4PoXRVm+8bf9f2PT0U19rnPfsnx+ekU+KuXATC4BUpODnhKA8JWIIxajBrc3QIu7b4VeRa2dd/ed5oqpIjkkGBmMpKREzMiy/GdZli+RZfkSk8k03OFcUITUAMgy6WW7+Cq7kBRt7+2EVOnppMyahbO0lFR1KkmKJPR6F3cVTxrCqAeHaTmpPHr1VBSKnjPQDW0NmLXm8Oxuojzch9qAhgYNiZJMEZy/uHwuAnIg7CmhTdLS5msjIPds4Dc9J42ymmbh4SAYMFs6TFOrGsR1UNB/vH4vbUkyaUp9uIW61WXtcxuFLuhVFWv3jaM2J1lGTVTKHq/fS5O7ieb2vmX+V0zN4s6FE2KK41xa2s8qJXL0OQDCVyKOmDraglr7aAsaCMiser2CdF0y95bkD1VoAkGfjKSkxOmOf89N2Vk6LROMEEK+Ce6yMjROO0cmzYq4jX7JYtz79uGrr8esNUflQp1ouLwunF4nphQTuqTgbEWiPNx3U0p4EiOZIjh/CZ2TYaWEKvg31ZtR3PScVFrcPk40xd5KTyAIUXPGxYHTwXPvmC0xrt+CkUnooT812YghyYBGqYk49umvUqKqoTVqlURDW0OX+OKJ3dPJU6JjfHGqVSQl4oXFGJwg7Kst6Cu7T7L3ZDM/X15AakrSUIUmEPTJSEpKVBFMPoSL2yRJ0gCXA58MV1CCngkpJRzbS/FLChoKZkfcxtDhOeF8/30yUzLPy6RESJZp1prD5RudO1qMZBzeYJxZumC1VKIoPATnL6G/nc6eEtB7oq+ow4ytrMY+BNEJzle2VQZnsjP1yRy1ieugoP+caQ0+/Kdp0pAkCZPWFLF8Q9mPpIQsyxyztUZtcmltC57jQ5GUCCslko1ok7SkqdOoc4ryjXhxNinRs1KiqdXDb945wJzx6Vw3O2coQxMI+mRIkxKSJOklSZopSdLMjmOP7Xg/Vg7qbZ8BHpIk6TpJkqYBzwNO4KWhjFPQN7IsY2uzYdKacJaWcsgyEaM5PeJ2yXl5JI0di6O0FLPWHL4pnk+EZJmJ7CkhkhKCkUI3pURIfeTr+W8q32IgWamgolYkJQT9Z2tlPXkmHQsmZgqlhGBANJ0JKgJGaTOAYEv0SEkJSasFSYopKdHU6sHe5iUvSpPL0KSQvd0e93K3Fk8LSkkZvn6P1o0WSok4Yu4o36h39JyUeOqdAzjcPp64dlpEI0yBYCgZaqXEJcCejp8U4PGO12s6lj8FPA38EdgNjAaukGU5MaaaLxCcXidtvjbSPcm0Hz7MR+aCcGa2LyRJwrBkCa5Pd5KZNOr8VEp0apUaugEnysN9qFzDnGJGISlE+YZg2An97YQ8JUJ/U23eth7XT1YpKBhtEEoJQb9xuL3sPNbIsgILeZl6TtnbcHv9wx2WIEFpag5WH6fpg34SJq0p4thHkiQUWm1MSYmQyWW0SolQYsQv+8MqyXjR4mnBmGwMPwBn67OFUiKO6NQqDGoV1h7KN748cYZ/fH6S/1gwnslZhmGITiDonSFNSsiy/L4sy1IPP7d2LJdlWX5MluXRsixrZFleJMtyxVDGKIhM6GZmOBI0AvvUUoDFEF07IX1xMbLHQ6q1FafX2WtteKLSuVVqSHLe6kmMmbbOs9J9dTkQCIaKkGt7N6VEH+fmtJxUKmrtBALC7FIQOx8easDrlykptJBn0iHLwuxS0H/OOILjpPTUoALRlGKKaHQJQV8JfyxJiVA70MzYlBIAdnd8k7j2djtG9dnW7yGlhDAkjh9moxrrOUoJnz/AqtcqyDJq+LEwtxSMQEaSp4QgQQjdzLR7DyGPu4g6XWZUSgkA7exZKIxG9AeD0r1IMsZEw+qyolFq0CfpSVGlICEljFLC4XWgkBRok7Tok/QJE7fg/OXc8o1InhIQ9JVwtPs43igeJAWxs7WynlHaJGaPHRWedRZJCUF/CXlKpKcFkxJmrRmXzxUx6a/Q6WJTSthaSVYqyBmVEtX6nRMjZ9rj2/Y2pJQIka3Pps3XNiR+FhcqFqOmm9Hl33dWs7+uhUeuKkQfRYcWgWCoEUkJQcyEbmba3YdoveQyAMxRJiWkpCT0Cxei/eIgwHlXwmFzBb02JElCISkSSnHg8DjQJekSLm7B+UuohOjc7hu9eUpAsC0oQLnwlRDEiM8fYPsBK0ummFEqpHAng2PC7FLQT5pdTag9MilpZ8s3IJq2oDEmJRpaGZehRdlLq/JzsbXZUEnBB9N4Jwda2lu6KCWyddmA6MART4JJibNKCavDze/fO8TlkzL5xvSsYYxMIOgdkZQQxExI3TDK7uP01EsAsBijK98AMBQvIe1US5d9nS9Y26yYUkzh9/rkxFEcOD3O8GyGUEoIRgIOjwO1Uk2yMhk4W77RV9nXJIsetUpBufCVEMTIF9VnsLd5KSkIdibXJqsYnaoRZpeCftPssWNoA2Vq8N5qTjEDkSdkgkmJ6Mtbj9mcUftJQDApclHaRcEY452UOEcpMVofbAsqfCXih9mgxtrSHi6R+dXbB2j3BXj8m1OFuaVgxCKSEoKYsblsaP1K9Pp0qkzjATBF6SkBoLv8ctLcwQx9NLWViURDWwNmrTn8Xp+kTxjFgcPjCHcM0SXrEsYLQ3D+4vCePSchOk+JJKWCgtFGyoRSQhAjWyvrSVYqWJh/NrGcZ9JxVJRvCPqJ3esIJiWMwYfysFIiQvexWIwuff4AJ5pcUXfegOCE0KS0SQA0u+OblLB77F3LN0JKCadQSsQLs1GDxx+g2eXl06ONvLanlh8syovpHBEIhhqRlBDEjLW1nrSWAPrFi6l3ehmlTUKtUka9vdJgwFR0Mcl+6bwq35BlGavLGh50QPAhKlG6WDi8jrBM3pBkEEoJwbDj8Jw9J4GwT0ukRF9Rbir7au34hdmlIAa2VlqZNyGjS711XqaeYzanMOUT9IvmgBNDuwIpKQkgPGkRnVIiuqTEyTNteP0yeZnRKSXa/e3Y2+1clHoRSkkZV6VEQA7g8Di6JCVS1amkqFKoaxVKiXgRUi/XNrex+o0Kckel8KPFE4c5KoGgb0RSQhAzp21VjGrxYyheQn1Le9Qml50xLlnKqJYApxuOD36Aw0Srt5U2X1tYngmJVQbh8DjCHUN0SbqEiVtw/uL0OLskJSRJQpukjZiUmJ6TSqvHT1WDOIcF0XHU5qSqoZWSAnOXz/NMOhxuH42tnmGKTJDItODG6E8Ov9cl6dCqtBFLV2NJSoQ8T6KdBQ8lRCxaC6nq1LgmJVq9rQTkAKnq1PBnkiSRrcsWSok4EhqX/3rzAQ5bnTx29VRSkqOfPBQIhgORlBDEjNVxmlEuBbrLLsPqcEdtctkZffESRjmh3no0DhEODyE5ZjelRII83J/rKZEoZSeC85dzlRIQNLt0+fqutS7KFWaXgtjYuj/YunFph59EiLNml+J6KIidFkU7qXLX8laT1jSoSolQd5gJUXpKhBIiZq057kmJUFvnzkoJCPpKCKVE/LAYguPyj440UFJgoaTQEmELgWD4EUkJQUwEAgGacGI2ZqPQaqlvcWOJwU8iRHJuLhnosZ5H5RuhQUYXT4lkfcJ4M5zrKdHma8MX8A1zVIILmXM9JYColBITTDpSkpT/P3vvHuVIdt/3fasKhWfh1QD6ge6Z6Znd2ce8+BDJ5VIb7tK7dCRFkakofhw6kkPajKQTJ4rlhKFIKbYOKTmMaUUydSTLOXEiiceWePSgQ4mSxV1xlg9ptVxSy3nu7nB6enYa6MYbKKBQQFWhKn8UbjW6G48qPLoL3fdzzhwue9DomhkAde/3fn/fL67RsEuKTZ6/ncOFlQhWY3srFR/qnj5P0sDxn27u4AYVyE4cmq6h4dEQYYJ7vp4KpGy0bwRhtNswtL334KbaxG/e/E109I71tbsFCfEgj1jQu/9p+kLWKqlgCnFffKaiRK1tvu572zcAM1eCtm/MjsXu+IafZ/HP/ssLR3w1FIo9qChBcUS9VoDiARaXz6GjGyjUxxvfAIDFaBplT3v0A+cEsshIBpLW1+bFKaEbOhrqrlWebASpW4JylPR1Stioq/VwLC6kI7SBg2KLsqTgW/crB0Y3ACAdC8DrYbExZtilrhv46d99Ff/2qxuTXiZlziAugahnr7CaCqZsjW8AOOCWeOHNF/CZVz6DVwuvWl8zmzechVwCZhPIUTolau3a0CYlyvj4eQ7vfSSFT/zA4zi1EBz9DRSKC6CiBMUR1dIWACAWTKDUaEM3nNWB9rLgi6PlBeT28dj49loiCQIvoKk195xquBFJlWDAoKIExVXsz5QATFHCzkL28moUN7MiDbukjOQrr+WhG+hrceZYBmcTobGdEhtFCZLSQaVJMylOGqTVIsbv3ZAvBhZRaBaGhqcOEiU2xU0Ae5srNoqS7ZBLwDxA4VkeUV8UcX98pu0bYrsrzPRkSgC0geMw+K0Pvws/+uT6UV8GhWIbKkpQHFGt7AAAoqEF5Oumy2GcTAnAFCUAoFB8czoXd8QUmgUEPUGrthDY3dyPmoE/aupKHQCsDSD5M8yDy4NyPFE7Klqd1gFRws74BmA2cMhqB3cnsN1TTgbP385hKeLDpXS07++fS4XGdkpcz5gbvjINyjxxEAdCrLvWIaSCKbQ6LdTV+sDv5QaJErVNALub+XpLRaHeduaUaBawGFwEwzCWU2JW7TKDnBJpoStK0BEOCoXShYoSFEdURXNEISIkkRNbADD2+EYiZI45lMpb07m4I6YgF/a4JABYbRZudxwQUYKIKNQpQTlqyIL9QKaEx74oAYDmSlCG0tY6+OobBfyNx5bAskzfx5xLhfBmqQm1ozt+fvL6qzbVia6TMn+UJdM9GQ8m9nzdTi2o5ZRo7j3QIE4JEhJJAljP2Qy5BMxQbjJmGvfFoeoqZE22/f1OsDIlBogS2w0adkmhUEyoKEFxRLVRAgDEo0vIiV2nxBhBlwCQEEyrbLF2PG5KhWZhT/MG0OM4UNx9WnvAKeGdj+umHF/2vyYJIX50+wYAnE0KCHo5XN+anTWZMv+8tFGGpHTw/gsH8yQIZ5MCNN3Ag7JzxxvJNaFOiZNHpWo6S+PhveuCVMD8/8PCLvuNb+iGjjdF01lKnBKkecPJ+AZxSgBAzGc2FVXaFdvf7wRREeFhPQh49gbIJgNJ8CxPnRIUCsWCihIUR9SapigRi69YTonUmKJEMrYKACiJuelc3BGTb+atxQaBnPK6fQyCXB85zQjz5kaQOiUoRwURxMYJugTMLIBL6Siu0dYDyhCev5VDgOfwnoeSAx9DTqGd1oJ2dAM3syI4loGsdtBS3Z0tRJkulbopOsQiewUvcngxLOySiBKdHlFiR9pBq9MCA6bHKdEAywCnE/bDDAvNgrVWIVkPswq7FBURUW8UDLPXhcQyLJZDy9QpQaFQLKgoQXGEKJtqejyRRr7eQlLwgufGexklF7qihDS8GmseMAyj7/jGvGQzWOMb3XETct3DZl4plFlCZpH7ZUq0O21bdbWX16K4lRWhjWG7pxx/DMPAC7dzeOp8En6eG/i4h5LdWtCis8/xu4UGZLWDt5/unkbTsMsTRaVRBK8ZECL9nRK2xjd6RAkyunExcRHb0jYMw8DdooRTC0H4PINfv7001Sbqat0SRuJ+M+9iVmGXYls8UAdKoLWgFAqlFypKUBwhtkVwHSAoLCAntrEYHi9PAgCE+CJ8ioFyqzzFKzwaREVEu9OeW6fEfqu8lYWhUKcE5Wgg75n9mRIhj7lYtzPCcWUtiram407e3e8/ytFwa1tEttbC+x8/2LrRSzTIIxHyOnZKkDyJ95437wsVieZKnCSqchmCDHjisT1fD/JBCLxgyymxR5Tohlw+mX4S7U4bpVYJGwVnzRtFuQhgN9di1k6JmlI7kCdBWBFWqFOCQqFYUFGC4ghRa0BoM2BZFjmxNXYdKACw0SgiTaCszP/MNznx2J8pMS+be0uU6I5tBDwBMGBcL6ZQji/kNbl/QUtcPHZqQS+tmgvu6zTsktKH52/lwTDA+x4bnCdBGKeB4/pWFUEvh+850z2Npk6JE0W1XUWkCXCRg5vyVDBlM1Ni93NuU9yEwAu4nLwMAMjUs7hXbDhr3ugKIeQAJd5tBpnZ+EZbHChKpENpFOQC1A4V6ygUChUlKA4ROxIEzbQJ5sT22M0bAMB6vYjJLCqd+R8RyMvm4mK/U2JexjcaagN+zg+e4wGY8552Z/cplFmwf6SIQN5Tdl6bZxMhCD4PrmXmX/ikTJ8XXsvhraditnKRziUF506JTA2X0lEkBPP5y1SUOFHUtDoE2QDbR5RYDCwOHd9gfD6A4w44Jc5EzljNFbcL99FSdUfNG+RnEqdExBsBA2a2mRK+/lW7K8IKDBjYkXZm8rMpFMp8QUUJiiPqaEHo8NA6OkpSG4sTiBIAEFU9qBjzv/Hdf6MnBD1m+JTbN/d1pd538+d2MYVyfKkrdTBgLBGCEOTtv6dYlsGl1QiuZ8SZXCNlftmptXBtq4bnRoxuEM6mQig22hBb9k51tY6OW1kRl9eiiIdMsbdCa0FPFNWOhLA82CkxbHyDYRiwodCBTIn16DpWhBUAwO2C2cRx1sH4BnFnEFcnx3IIe8Ozy5RQhjslACAjZWbysykUynxBRQmKIxpMG2H4UGwoMAxMNL4BALGOD1W2NaWrOzrI4oJ0fxM4lpuLzX1dqR8IFBR4wfViCuX4QoQyltl7m3LilACAK2sx3N4WoWg07JKyywuvma1PdkUJMrdv1y1xJ99AW9NxZS2KWMALAKjQWtAThQgZEY0Hwx0MoSTjG4ZhDPz+XlFC1mRsS9tYj6wj4o1A4AXcq20BAB5yOL7h43zWqCZghl3OwinR0TuoK/WBQZdEXKG5EhQKBaCiBMUhdU5DmAladaCTBF0CQBxB1DwKdGO+NwyFZgFhPmyd4vYyD2MQdaW+Z5ECACFvyKplpFAOm4baOPCaBJxlSgBmroSi6XgjN/9jYpTp8cLtPE4tBPDIkr0NHZnb3yjY+0wkOSaXVqPwelgIPg9t3zhB6IaOOttGxOi/RloMLELVVdTag/Nu2FDQEiXeFE1XxHp0HYC5od+RthHyclh0UMtOqst7KzqjvuhMRIn9VeP7WQ4ugwFDGzgoFAoAKkpQHNLgO4hwIUuUmNQpEWfD6LC78+PzSkEuHAi5JAi84PrNfUNtUKcExVWIinjgNQnstm9Imk2nBAm7zNCwS4pJU9Hw9e8W8dzjS3s2Z8M4vRAExzK4ZzPs8lqmCsHnwdmE+XqNh3hU6fjGiaGu1KEzQJQJ9P39ZNB0VZI8qn6wwV2nBKkDPRs5C8AcfagoOZxLCbZfwwD6VpfHfbNxSohtc2xuUKYEz/FIBVPINqgoQaFQqChBcUCno6HpMxDhw8jV2wAwUdAlAMR5U0EvyaWJr+8oyTfzw0WJORjf2J8pMQ/X7ZQH4gMaqjUnNJTGgdck4CxTAgDOJIII+z1UlKBYfP1OEYqm2x7dAACvh8XphaDt8Y3rGRGXViNgWXPDGA96UabjG3PHy/fK0PXBIxaDIJv8KHdQWAVMpwSAoWGXvU4JUgd6OnIaALASWkHLKDkKuSQ/b/9aZVZOCVExRYlBTgnAFFe2JTq+4WZERcSN4o2jvgxKD4ZhoPp7v4f2xsZRX8pUoaIExTZieRsGwyDijyAvtsAyQCLkneg5E94FAECpNd+iRKFZsBYZ+5nbTAmv+x0eTvnoVz+KT770yaO+DIoN+r0mAeeZEgzD4MpalNaCUixe2igjwHN45/qCo+87mwzhro3xDUXTcXtbxJW1mPW1eNBLK0HnjFtZEX/nN/4SL94ZLBwMotKqAABiAzbkRBgYFnbZmymxKW5iObSMgMd0XiwGlgFWxtqCfZcEsDu+0UvMFyhhekMAACAASURBVBs6RjIu5DmHiRIrwgp1Sricz936HD74xx/EzdLNo74USpdOtYrtn/05SF/72lFfylShogTFNpWKqWZH/QvIiS0kBR883GQvoYVgAgBQlpzf9N2CYRjIy0OcEl4BkuLuMYi+mRJzIKY45X79vjWbS3E3gzIlfJwPHMPZzpQAzLn+13ZEtLXONC+RMqdsVZo4tRCA1+Ps/nUuGcJmSRp5cv5Grg5F03Fpdde2Hg/ytBJ0znhQMT9jslXZ8feSDXnMF+v7+0QYGOaU4HpFidom1iPr1u95DHPtFI3Yv0dLqoSm1jw4vuGPQ9ZktLTpho7bdUrkpBw6Ov1sdis70g4MGPiFl35h7vPfjgtq1hTyPOn0EV/JdKGiBMU2lXJXlBASyIntiUc3ACARNu2zxer8KuXVdhWarh04fSC4fXPf7rSh6ErfTImm1jw2i4WG0kBdqWNb2h6aeE5xB4MyJRiGQZAPOso7ubIag9ox8PrOfGfXUKZDpipjNdZ/1n8Y51ICWqqObG34JpWMCl3pESViQS+qEs2UmCfy3eysUsO5mFRpm06JeLC/G8fv8SPijVgVnf0gTgnDMHBfvL9HlFBa5mvL57fvcCACSL/xDQBTH+EgosSgTAkASAtpaIY21DFCOVrKrTI8rAfXi9fx+3d+/6gvh4JdUYKnogTlpFKrmzfPaDiJfL09ccglAMQjS2AMAyUxN/FzHRX7e7/34/bASBIyun9+32o50OyfSLsZkvDd7rTnflzouKMbOiRV6pspAThvtLmyRsMuKbtkqjJW4+OIEuZn4qiwy+uZGsJ+D84kdtuYFkJe1Nsa1A49aZwXcqKZnVVqtB1/b6VRBADEhP5jnQCwGFwcOb7RaTZRapVQV+tW8wYA1OrmZ6PhKdu+JvKz9o+axn1xALMTJQZVggJmNgYAmivhYsqtMt659E68Y+kd+OVv/TLKLfuvOcpsUDMZAFSUoJxgag1zIxeLLCIvtrA4BaeELxaHIANFafBpgduxbvTB/osPwWuKEm61vRFRop9TArA/u+92eudWaS+6u2mqTeiGPtD2G/KEHIlla/EAYkGe5kpQ0GhrqDZVrMYO1jePgogSo8Iur2/VcGUtuqcVIR7kAYDWgs4RpGWsOEZAaaWeB9cxEIn0P6wAzBGO4UGXIUBVca/0XQC7zRsAsF3yAAaPomz/QIccoJDmD8KsnBK1dg0+zgcfN/gAa1VYBQCaK+Fiyq0yEoEEPvHEJ9BUm/jlb/3yUV/SiUfNZsEEg+Bi/cfD5hUqSlBsU2ua6qgQWUZJUhx1Yw+Ci0QQk4DyHLdvWJbIAeMbAi/AgAFZcz6XehiQMMsDmRLe0J7fn3d6Fz0ZKXOEV0IZBRl3IsLYfpyObzAMg8urUVyjosSJJ1MxP4fHcUqkBB/CPg82hoRdtrUOXtsR9+RJAEC8GwpNa0HnB9IyNp5TogBBBjyxwaMLqWBqZCUoANwrvgEAOBM9Y/3evVITXixYDkA7kLXKfqcEyb2YhVMi6h385weA5dAyAOqUcDPlVhkJfwIPxx/Gj174Ufzhd/8Qr+ZfPerLOtGo2Sz49IqjOuB5gIoSFNvUZPOGpflMlX0amRJsNIqIZKCsTL+O6rAgTolB4xtkDMKtm/tRTgk352E4YVvaBsdw5n9Tp4SrIbbffpkSgHNRAgAur0bxRq6Olno8MlIo45Gpmg6bcTIlGIbB2VQIG0PGN17fqUPtGLiyuvcEKx40RQlaCzo/kEyJ4hiZElW5grBsHrwMIhVIodgsDnRRsqGuKFHZgI/zWaMOhmFgo9BAlF90dC/Ly3kEPAFrTUKI+7vjG60pixJtcejoBmB+lsd9ceqUcClNtQlZk7EQMLNRfuItP4Gl4BI+9dKnoOnaEV/dycUUJY7X6AZARQmKA0RFBK8BtY7pkJhGpgQXjSLaBCqaOPFzHRX5Zh5RX3SgRdHtYxB1tX+mhNuv2ynZRharwirCfJgugFwOEfAGZkp4nGVKAGauhKYbuL09v581lMkhTom1MZwSgNnAMWx8wwq5XNt7Qhzrjm/QWtD5IWcFXTp3SlTbVYRlgI0Md0pohmbVh+6HiBKb4iZOR06DZcwle0lSILY0LAVWHDslFoOLB05XiZthFk6JYc0bhBXB2Z+DcniQ/K0FvylKBPkgPvrOj+L1yuv4ndd+5ygv7USjZagoQTnh1LUGQgpjnR4shid3SnDhMCJNoGLM78a30CwMHN0Adp0SZPPvNohTYv/igYgSbr1up2xL20gLaawIK9Qq6nIGvSYJIT7kqBIUAC6vmSfXN2jY5YlmqyrDy7FICeOJ6udSAjJVGbLS33FzfauGWJA/IHosdMc3KnR8Yy5oax1Umiq8HhaVpgrNYUBpTasjLBvghoxvkByqQWGXRJR4s5nZ07xBRLEz0VWUW2XbVZ75Zr7vWoXneAi8MJNMCTuiRDqUpu5Fl0JCLYkoAQDvP/N+fG/6e/Grr/7q0EwUymzQJQmdWg18evWoL2XqUFGCYhtRbyKseqxE6mmMbzA8j5jCQ2IUKJ35PEEqyIWBIZfA7mmvpLhTeLEyJfaPb7j8up2SbWSRFtJIC2l6KuNyLPfOkEwJp60w6agfCyEvzZU44WQqMlZifrDseLO4JOxys9T/c/HaVg2XV6MHTqPp+MZ8UejmSTy2bN4Xyw4dLrWOBMHG+AaAgRs7NhSCxgIZJb9PlDDv2Y8lTwOwn8dQkAsDx0yjvuhsnBIjxjcAWAcFtKrbfZRlU5RI+BPW1xiGwc888TNQOgo+88pnjurSTizHtQ4UoKIExQF1tCDoPHJiCxzLINE9+ZmUGMwTpXmtGco380gGkgN/38qUcGk2g6iIYBkWQc/eNHq3X7cTSA3oSmiFnsrMAYNyTghOK0GB3bBLWgt6sslU5bHyJAjnkqZQ1m+Eo6V28EaujsurB0/H/TyHAM/R8Y05gRy+PL5sbqpLDnIlDMNADU1EZIAV+gurgD2nRD4GdKDjbHS3eWOjKMHrYXFh0Qy+tHM/MwwDRbl4IOSSEPfFp58pYXN8Ix1KQ9bkqYsilMkh6/JEILHn62ciZ/DhSx/Gl+59CS9vv3wUl3ZioaIEhQKgzigIw498vY3FsG/sk6b9LLDmTZvMrs0TuqGbN/phTgmXZzM01AYEXjhwskdECrdetxPIoo04JRpqwwpTpLgPO6KEqqtQO86s8FfWzLDLQdZ7yvEnU5lMlDibJLWgB8Xa29siNN04kCdBiAd5lCU6vjEPkDHVC2lzU110kCshqRI6jIGw4QfDDl5mk8MMUtW5HzYUQnbBvC+fiew2b2wUJJxNhLAW7tZp2nD+NdQGZE0e7JTwT9cpoeoqJFWy7ZQA7P05KIcLESVIGGov/+jyP8KqsIpP/dWnHN+LKeNjiRKrVJSgnGAaHhVhJoCc2MLiFEY3CHHOvGkRm9g8UW6V0TE6tjIl3Oo4qCv1vps/juUQ9ARde91OIIudldCKlWBO3RLupaE04ON88HL93VjkPTVOA4duALdo2OWJpK11kK+3x6oDJQS8HNJRf98GDpJXQvJL9hMLeqlTYk7I7RMlnDglKm0zuDLKBIc+zst5EffFh4xvBJHtHlCvR9etr28UGziXCmExuAiO4WwFN4+qLo/5YlMVJUblAvWSDpmbK3pPdh+lVgkCL/QNcvd7/Pj4Ex/Hvdo9/Nat3zqCqzuZqNkswPPwpAbvO+YVKkpQbCPxOiKeEPKi6ZSYFgtdBXYexzeKchEAbDkl3Lq5byiNgSfSAi8cS6cEANrA4WJERRz4mgR6XDyaQ1Gie4J9fYvahE8i21VzozmJUwIwwy77OSWubdWwEPIiHe0v2i+EvKhQUWIuyNXb4DkGD6fM+7cTp0StbYpTMS404pFmA0de7u+U4LpOibgRsDb3akfHm6UmziZD8LAeLAYXbWVKkJ8xyCkR98WnKkqIbVP4jfoGB30S6D3ZvZTl8p6Qy/28d+29eN+p9+E3rv0GFZUOCTWTBb+8PNSFNa8cvz8RZSaoSgtNHxDhI8jVW1OpAyUsdAN05lGUILbLQTd6wHQcBDwB1wZGioo4MFAw5A1ZQZjzTFbKgmVYLAYXLacEtYq6FzJSNIhxnRLLET+Sgg8vbZTx3Xxj6C8nm5CTitpRbSf/u4FM1awDncQpAZhhlxtF6UAw3/VM/5BLQizIu6Z9o95S0dFpsOAgcmILi2E/YkEePMeg5CCglGzuo/xol0AqkBrolGC8XmwnWKxpu8/zoNyEphs41xVLVkIrjpwSgw5Qor4oJFWamg2fjEfacUpEvBEEPUF6T3Yh5dZwUQIAPvauj8EwDHz6m58+pKs62ajZLPjV49e8AQCeo74AynxQLZsKqOCLoNpUp1IHShAiCfAaUJLnL1PCutEPCI8iCLzgXqeE2sCq0P8D7jg5JRaDi+BZHgv+Bfg5P1X1XUxdqQ9dzBJRwmktKMMweOupGP705g7+9ObO0Mf6eRYvf+I5RPy8o59xkvj0Nz+NNypv4Le+fz6su5mKKUqcig+31Y/iXDKEektDsaEg1XUNyooZcvn+C0sDv88tTomdWgt/8/98Ef/9+x7Gjz/90FFfjivJi20sRnxgGAaJkA8lByJlpWWOb8T8/cd4ekkFU7hTuTPw97MJBu9p7b5eScAqaYFJC2l8K/etkT/HOkAZML4R95mO1Wq7OvSQxS5ORAmGYcxWLOqUcB2lVgmnw6eHPiYtpPGRKx/BZ//6s7hTuYPz8fOHdHUnEzWbReipp476MmYCFSUotqh1RQlvN/8hNcXxDU8kiljDQKlZnNpzHhbEEjmsfQMYry3gsBiUKQGY1+1WMcUJWSlrza0yDIPl0DI9lXExDaVhVdL2Y1ynBAB86gOX8ENvHR4Q9d18A//6hTu4kanhPQ8Nf2+fZG6Xb+NO5Q4MwxjoDnATW1UZLAMsDxivsAs5pd4oNKx74a1tEbqBvs0bhFjQi5psOhS4KQVFj8Mn//gWxJaG13bqR3YNbicntvBQ9985IXgdZUqQ8Y14MDHikaZIUGwV0dE74Fhuz++Jioha0MBqbne9tVE078cPJXedEvlmHpquwcMOXtIX5AIEXkCQ7y/IRf3m63ZaogT5O7ATdAmYfw671aaUw6PcKuNti28b+bh3r7wbn/3rz2Jb2qaixAwxFAVaoXAsmzcAKkpQbFKt5QAAHo+p/E+rDhQAuGgEkU2g3Og/V+lmCs0CFvwL4Lnhp6kCL6CuunMBOCxTIuwND7SWzhPbjW28bWn3xpoWaC2omxEV0Upk7wdZWI8jSixH/fihtwy/oZclBf/6hTu4vkVFiWFsN7atKr9+6exuI1ORsRTxg+cmm1y1GjiKEp44Z248SU7J5QHNG4DZvmEYQE1WsTDFe6gTvnangD++tg2G2XWOUA6Sr7fxnofMf9uE4HM0zlVpVcDoQEQY/dmxGFyEbugot8oHxIDN2iYAIC3uihX3ihISIS+iQXPNsSqsomN0kG/mrWyGvn+eZn6o2BDzmWu7aeVKEKdE1Ds6UwIw78nfKXxnKj+bMh06egfVdnXk+AYA6zHz6HieJ9SdHcAwjq0oQTMlKLaoiKYoYRBRQpieU4KNRBBtGnP5YVZoFoY2bxBC3pArMyV0Qx86v38cnBKariHXzFlOCaA7h0udEq5lVpkSdlkIebEWD+Bat02BchClo6Agm4LlvLyXMtXmxCGXgBmU6fOwe8Iur2VqSAo+LA9ppiJCRNlBPsE0aWsd/G//8SbWE0H8wKUVK2ODspeW2kFNVq2WsWTIi6IDp0RVKkJoGfBG7Y1vALDeS71sipsAgHR511VztyBZoxtAT53miNGHQrMwdMy0d3xjGpCgSydOCVERXesoPYlU21Xohu5MlGjN3zp+nlAzGQCgogTlZFNrmB80Cmeq3qkpihJcNIaIBJS7NVrzRF7OIxkcfRri1kwJSZVgwDjW7RuFZgEdo7Pn5D0tpFFulecqpO8kMTJTwtPNlNCcZUo44cpa1Kp4pBxkR9rN5JgX19FWRZ445BIAWJbB2WTImu8HgOtbNVxZGxxyCZjjGwCOrBb0//rqBu4VJfz837qEs8kQdsQWtI5+JNfiZvKi6YogLWMJwYuS1D4QbDqIilSEIJsHLqMgQkE/R+JmbROczmCx1LG+tlGQLKcO0FOnOWL0oSAXhq5VSEvGNJ0SAU8APGsvk4c2cLgPEj6/EBgtSgT5IAKewFwG1s8TatZ8f/CrVJSgnGBqTfODRmZNNT0hTHd8I9oEKppo+6bvFkadPhDcmilBusSHZUpIqgTdmN+Fa6ZhKsv7nRLA6IUc5fBROgranfbMMiXscnk1hvulJmouaUtwG73uiHnYSHR0Azu11lScEoAZNHivaL7+pLaGu4XG0DwJAFjoihJH0cDxoNzEZ//8u/iBy8t4+pEUVuMB8+9EpMLsfnJ18+9kiTglBB9aqo6m0hn2bRYVuYyIDHDR0aMLxCnRrxZ0U9zEshIA2zDFV7GlothoW5kmgD2nhGEYI9cq1vhGazqiRK1dsxVySaD3ZPdBBIaEf3Q2CmC6JagoMVvUTBZgGPBLgwOV5xkqSlBsUe+GFtWMKPw8i6CXG/Ed9uEiEUQlAxo61hziPNDROyi1SrZCocLesCudEqNECYEXYMCArM2vzZcscvY7JYD5OeE9SYx6TQIAz/HgWX7GooS5obhO3RJ96X3vzMNGIie2oOnGVJwSAHAuKeDNchNqR7cVcgmYlaAAUDmC8Y2f/+JNcCyDn/vBCwBgiTM0V+IgOXGvKEHGVe2GXdbaNQiyAS4yWpRIBBJgwPR3SoibOKWGoUvm55zVvNHjlPBxPiT8iaEjVKIiQtGVoWsVv8ePgCcwVacEcV/YgTSAzYPAeVJwKkok/AmUZSpKzBI1m4VncRGM92gyiWYNFSUotqi1RfhUoNL2IBHyTTVp3cyUMP97nlTWcqsM3dAdOSXc5gQhG8CBmRJec/HTUNwnqNiFLHLISQyw65qYl1n4kwQR74ZlSgCmXfQwRIlrmeks0o8bWSkLBgzWI+tzsZEg+QnTdEpouoE3y01c2zKFq2Ehl8BupsRh14I+fyuH52/n8VPPnsdK1PzzE3GG5kocJNcd31iK7I5vAEDBZthlVRURbpou0FHwLI+4P25VdhJ0Q8eb4ptYM2LQm+YC6V63eaPXKQFgZJ2mVQc64gAl6otOVZRw4pRIBBLgWZ7ek12ENb5hI1OCPG6e1vDziJrNHts8CYCKEhSbiJ0GQgqLQqON5BRHNwDilDD/e54+0Ijd0o5TQuAF6IbuOscBESUGLR7IxtCNoyd22Za2seBfQMCzuxlJBVPgGG4uNlMnjVGvSULIE0JTnV2mRDTI40wiiOtb1CnRj2wji1QwhVPhU3PhlCCOgLUpOSWsBo6ChOtbVSxFfNbJ+iCCXg5ejkX5EEUJWengn3/xJs4vCvjwU2etr1OnxGDyYgteD4towHS2JEPEKTFalDAMAzVdQlg21zZ2WAwuHgi63Ja20e60cZpLQpfMA42NggSOZXB6YW+t56g6TeLCGHWAEvfFpyZKOB3fYBnW/HNQ96JrKMklcAxnO6w0EUjM1Rp+HqGiBIUCoK43EdY4lBrKVJs3AIDhOMRg3mTn6QPNutEH7TklAPdt7q1T6QHz++S63Th6YpdsI7snTwIAPKwHS8EleirjQiz3zpBMCWD2TgnAdEvQ8Y3+bEvbSIfS5intHLyPiCMgPTWnhPn63Cg0cD1Tw+XV0U0LDMMgFuRRlQ4vU+LXrn4XWxUZn/zApT1VqH6eQ1LwUqdEH3JiC0uRXUcocUqUbIzdyJoMBRrCsgHWRvsGAKQCqQPjG/dr9wEAp/lloNOB0W5joyDhVDwAr2fv0p1UXA/KfrJ7gDJ1p4TNzSxhRaCtWG6i3Coj7o+DZextFYlTYp4zyNyM0elA3dmhogSFUkcbYd1MoJ62UwIA4h5zfnye5tEsS6SNSlDiOKir9Zlek1NIhseg+X3y9XkWJbal7T15EoQVgZ7KuBE7mRJAdyRKm70osVWRj6zC0c1kG1msCCtYCa2g1q65TnDdz1ZFxkLIi6DXM5XniwZ4JAUvrmVq2ChKI/MkCAsh76GNb2wUGviNFzfww29bxbvPHZwLX40FqCjRh5zYxlJ41/ViiRI2nBK1bv5WuM2CDQVHPNpkMbh4YHzjnngPAHAmYG5AdEnC3ULjwOgGYDolFF0ZeKhDBI9Ra5W4Lz61oMu6UkfUaz9TAjDHKuk92T2UWiXboxuAKUp0jI5VB0uZLlqhAGjasW3eAKgoQbFJnVUgwD8TpwSwm/w8V04JuQAGDBKB0SFA5NRXUty1cCdZEWF+cPtG7+PmDcMwrBPd/aRD83HCe9IgAtig1yQhxIcgq7PdUJGMAOqW2EtH7yAn5SynBOD+gLpMVZ5angThXFLAn9/OwzDMClk7xIL8oYgShmHgn/1/N+HzsPiZH3is72NW4wE6vtGHfL2FxcjuOsfn4RD2e1C0EXRJnAZRBGxnb6WCKZRbZWi6Zn1ts7aJMB9GImQKCVq9gc2StCfkkjDqPZhv5hHxRuD3DB8vmpZTQukokDV5LKdEQS5A6VAR2A2UW2XHogT5Psr0sepAqVOCctJpeDSEEICmG0iEpu+U8EViCCscSq3S1J97VhSaBSz4F+BhR5+8uXUMoq7U4ef84Ln+XeLznilRapXQ7rQHOiXyzTxUnVY+uglHTokZvy4vkQaOLRp22UtBLkAzNKSF9NxU+WUqzemLEqkQZNWsibzkyCkx+8+cL13fwdfuFPE//+ePYjHcfzNKnBJuC2A+avJi+8DfWVLwoWjDKVFpVwAAUW74+FkvqUAKBgyU5N31z6a4iTORM+AE83lyuTJaqj7QKQEMDm4uyAVbjs64Pw5REfeII+NAHJhOMiWA3QDqHWlnop9PmQ5luWzr0I2wEDBFiXlax88TaoaKEhQKAKDB6wgwphUxOQOnBBeJIiqzc6Ww5pt5W3kSgHs39w21MXTz51YxxS7ECjrIKaEb+gHbLOVoERURDBgE+eHW56AnOPPxjYifx7lkiDol9kFOZNPCfDglDMMwnRJTCrkknEuZn4/pqB+psL37YizonXklaKOt4ZN/dAsX0xH8N+8+M/Bxq7EA2ppuywFwUpDaGupt7UBoaSLktVUJSsY3nNRhknVEb9jlpriJ9eg6uJD5Gstumxs98prrZVTFdUEu2ArkJtc8aTU7se87FiUE2orlJpyOb5Dq0Hlax88T1ClBoQBQZAltL+BjzJthYgaZElwkgmhT33NS4Hbs3ugB927uRUUcGijo1uu2C1nckMVOL/OwmTqJNJQGBK8wMlzrMJwSgHkCThs49mK9r0JpJANJ11f5lSUFLVWfulPibNL87LTrkgCAeJBHVVZn6k74leffwI7Ywic/cAkcO3iEYDVuCn9bldm12Mwb+freOlBCQjAztUZRaZlOCTKSagfiYiACeVNtYkfawXpkHWxXlMjtmBu9fqJE2BtGmA8PvJcVmgVbByjkmicd4SCihhNhBth1fNBciaOnqTYha/JY4xvztI6fJ9RsFlwsBjZoL6tmHqGiBGUklXIGAOBhzQXYTJwS0QgiYmeuFNZC054lEti1orvOKaEMd0p4WA8CnoDrsjDsQhY3/cY3rNOlGdnOJVXC3/7i38a1wrWZPP9xpa7UR+ZJAKYo0VSbM7eeX1mLIltroVAfvSHZj9bR8V//+l/gj68dr0U2eV8th5anUuW3U2vh2X91Fa/vzCYImIQ5Ttsp8VB3g2g3TwIA4kEvOroBsTWZRX4Qr+/U8e++sYm/985TePvp+NDHWrWgNOzSIie2AOCgU0LwOXJKxAJ7N3P/4ku38ak/utX3e8jhBgmkfFB/AABYj+6KEsV8BYLPg9SA9deK0L8WVDd0++MbPvP1MmnY5bjjG0uhJbAMi63G1kQ/3y5vVN7AD33hh6hbsg9kDIm4H+wQ88XAgJmrdfw8cdzrQAEqSlBsUK2Y830MY24UZuGUYKNRRBr63HyYqbqKcqtse3yDWNHdFhhpZwMY4kNz7ZQQeKHv4mg5tGw+ZkZOibvVu3it/Bq+nfv2TJ7/uFJX6yPzJADzPdUxOmh3nIsFTiCtCjfGGOF45X4Fr9yv4Orrx2vRm5WyiPvi1ufapFV+336zgrsFCX/w17PZjJAwx+k7JUL4hR++hA8+MXhEYj/xoHn/nMUIh2EY+Ln/eAMRvwcf/b7+4Za9EJGGhl3usitK7N38JwUfyk0FHX24CFptVxFsA759daBfvp3DF17N9hVRF/wLYBnWqu4kzRu9TolqqYZzqdDA8MxBwc3VdhWartkb3/BHre+ZBCLMOA265FkeZyJn8EbljYl+vl2+ePeLuFe7Rw8O+kCa8Jw4JTiWQ9wfn5t1/LyhZrPHunkDoKIExQbVqilKaDA3CgvBWYxvRBGVDIiKCLXj/uDBklyCAcP2+AbP8vBzfvc5JUZkSgBmHobbrtsu243+daAA4ON8SAaSM3NKkAUiWWhS7FFX7IkSZLRo1q/Ni6tRMMx4DRzP38oBADaK8/n+GcT+99WkVX4bBVP0JH9f04Y4Adam7JRgGAZ//4kzWHAQ/kweO4sGjj/86wxevlfG//p9j9m6pmiAR9jvoU6JHvKiKXIuRvYHXXphGKP/3SqtCsJNA2xkd0NuGAayVRnFRhs7XdGjFw/rQcKfsJwSm7VNAMDpyGlLlGiUxb7NG4RBFdfkOe0coFhOiSmNbzh1SgDAxcRF3Cr2d5RMm6sPrgIw8zsoeyHCghNRgjyeihLTxzAM6pSgUACgVi8CAFqIIB7k4eGm/7LhohFEu+v2efhAs270AXtOCcCdjoNRmRKAO6/bLlkp2zfkkpAOpWfmlCALxGKzOJPnP66QTIlREFGiqc52Hl7weXAuGcI1h7kSh3WIhwAAIABJREFUhmHg+dtdUaIwn++fQex/X01a5bdRMD/87xYk3JuBgLNVkRHycogG+rcMHSaxoHkN0xYlarKKX/zSbbztdAx/5x2nbH/faozWgvaSE1sI8BzCvr2tWomQ6ZwY1cBRbZYQlgGuxylR6maaABj4OZIKpiwBe1PcxEpoBQFPwJofV+r1vs0bhFVhFQ21cSCkkowm2BnfmFqmRDfo0o64vJ+LiYvIy3lrjTUr7tXuWWLEvdq9mf6seYSsw520bwBUlJgVnUoFhiyDX1096kuZKVSUoIyk2jA3VfVOGIkZ5EkAJOjS/O95+EAjiwe7TgkAELyC68Y3RmVKAOZ1z7VTItTfKQEMnsOdBkTsoE4JZ9SVuq0TtpCn65SYcQMHAFxZi+F6xtlC/W5BwmapibPJECpNdeaNC4eFYRh9nRLA+FV+d4sSznZPgV+4PX23xFZFxlo8OND6fpjsjm9M1xH4r/7sdZQlBZ/8W5fADgm33M9aPECdEj3k620sRnwHXitkbHVUrkRFLiPcNMD1OCV6RZ9BobmLgcU9Ton1yDoAgPF4AK8PAa3dN+SSMCgkkjR62FmrBDwB8Cw/FaeEwAu26tL3czF5EQBwqzRbt8SLD14EAJyJnKFOiT6QWs+4f3guzX4S/sRcrOHnjZNQBwpQUYJig5psBt6U9QgSDmyqTmCjUUQkc9ZyHj7QyOLBbtAl4D7HQbvThqIrIzMlBF5AXZlNAN0sqSt11NV63+YNArGd64Y+9Z9PxI5Zn/gcN+pq3arQHQbJMzgMwezyahQ5sY18H+v1IIhL4h8+dRYAsFF0z3t/EirtClqd1h6nxCRVfoZhYKPQwFMPJ/HoUhhfnsEIxyzqQMclPoPxjRuZGj730n382JPrjppAAOqU2E9ObGEp7D/w9WRXlBjllKi1q12nRI8o0RV9/Dw7cAwsFUyh0CzAMAzcF+9jPbpu/V7HH0BAa1vCXT8GtUk5cUowDIO4Lz5x0GWtXRtrdAMAHo0/CpZhcbN0c6JrGMVXHnwFj8YfxbuW34XN2ubMA5PnjZJcQtATRMDj7HNzIbBA2zdmwEmoAwWoKEGxgdi9QeWUMJI2u9idwkWj8+WUaObBMqyjeTu3ZTMQoWGUU+KwqhenDVmcDcqUIL+n6MpMXnNkg1aQC3TBYxPd0G25d4DDy5QAdtsVnORKvHA7h4vpCL734SSA3RGFeadfo80kVX7FhoJ6S8O5VAjPXVjEK/crqE55tCFTaU495HJcIn4POJaZmiih6wY+8YUbWAj58NN/8xHH378aD6De1lCT3Z/ldBgQp8R+SOvYKKdETa1DkLEnU4KIPk8/ksL1TK3v/SAVTKHSrmBH2kFDbVhOCQBoe/0IjhAlyHtwvzBYaBYQ88Xg5ewdKEX90ak4JZyGXBKCfBDnoudmKkpUW1W8WngVz5x6BuuRdYiKaLVNUEzKrbLjPAnAHN9oqI2ZB1CfNKgoQaF0EdU6AgpQaBpIzsgpMW/jGwW5gKQ/CY7lbH+P25wSRJQYNb8v8IKrrtsuRJQYlSnR+9hpst3YBsdwkDV5Lv/+joKm2oQBw5EoMetMCQC4kI6AZQbPg++nLCn41v0Knn18CafiAfAcc2zCLsmmp/d9Rar8xnFKkAyJcykBzz6+hI5u4Orr03MX1VsqxJbmGqcEwzCIB3lUmtMRAX7nmw/wnQdVfOK/eAwRv/PMjNWY6TiibgnTtZMTWwfqQAEg4ufhYRmUpMGbLaWjoGm0EZGNPZkSmaoMwefBUw8nUZaUvuMyxMnwSu4VANjjlGh6fIhDRdA7eBxiwb8AP+fvO77hZMw07otPRZSIep05dnq5kLiAm8WbMxPzv5b5GnRDN0WJ7t/zffH+TH7WvFJulbEQGE+UAMzAV8r0ULNZsMEg2Oj476t5gIoSlJGIWgMhhYXY0maWKcEKAgIKA97g5sL65fRGD5iOBDc5Dki+xSibJXFKzNtpv7V5GjK+QU57J6kz7IeoiGioDTwSN08u6QiHPey6d4DDdUoEvR48vCjYdkp85bU8dAN4/+NL8HAsTi8Ej03YpSX29byveJZHKpAaS9wjfy/nkiG8dS2GpOC1Rl+mAdkAusUpAQCxoHcqGSNlScH/8Z9ewxNnF/CBt44XgGbVgtJcCTTaGppK50AdKACwLIOFkBfF+uB/N7KZF/aNb2xVZKzGAriyZgoV/XIlSDvGyzsvA8Aep0Sd4RFjOkOvnWEYLIeW+zolnARyR31TcEq0x3dKAGbYZalVskZPps3VB1eRCqRwIXEBZyPmeB1pPKGYlFtlJPzOQi6BXVGCZFJQpgOpA3VDLtIsoaIEZSR1XYagmQo9CXuaNgzLwhOJINbxzcWHWaHpXJRwrVNixPy+4BWgGzpkbb4WrduNbXhZ71ALIjntnaTOcNDPBoC3pN4CgIZd2qWu2ntNArBmXQ9L6Lu8GsO1rf7W6/08fzuHpYgPl1bNhfnZpHB8xjekbQQ9wQNiZlpIjxUau1GU4PWwSMcCYFkGzz62hBdfL0DRppPzQhwAbnFKAOg6JSYXJT79J6+h0dLwyQ9cGnuxSsSaTGX2jiO3k+vWgfZzSgBAQvANdUqQ0+FIE3uDLruZJo8uh+Fhmb7iJnFKfHPnm/BzfiyHlgGY7o0KeISN0a+XtHCwmjcv5x2tVWK+2OSZEsr4mRKA6ZQAMJMRDqWj4BvZb+DpU0+DZVisCCvwsB7cE2kDRy8luTTW+AZp6yjL7nc8zxNqNgvPMR/dAKgoQbFBnWkj2DFtoaQWaxaw0Shiimc+xjccnj4A3UwJxT2OA7IBHNm+0d0gusnlYYeslEVaSINlBn/MCV4BYW946uMb5PneuvhWANQpYZexnBKH0L4BmLkSxUbb2rgMoq118NU3Cnj28SVro/hQKoT7pSY6ujve+5OQbZjvq/2b4JXQythOibOJELhuY8Szjy+i3tbw8r3p3AeIA2DNRU6JeNCL6oTjG9+6X8HvvvIA//Cps3hkyXn1IiEpeOHzsNQpAVhBtot9gi4B8++qOCRTotY2xQZB5cAEdl9vJNPEz3N4dDncX5ToCgeZRganI6et+1axoaDOehHURs/or4RW9jglOnoHJbnkKJA75ouhptQmCn8W2+JEosSjC4+CY7iZiBKv7LwCSZXwzNozAAAP68Hp8GnqlOhBN3RU2pWxMyUA6pSYNmo2e+zzJAAqSlBsUGcVBHVTjEiFZ+OUALq5Ei3W9aKE0lFQaVfGckpohuaaACAnQZcAXOXysMOoOlBCOjTeCe8wyMLQckrMyIZ63CAjRXZECQ/rgY/zHUqmBACr1eDa1vBTxJc2ypCUDt7/+JL1tXOpEJSOjq1jcBq9LfV/X6WFNHJSDh19uM18PxsFaU/V4VPnk/B52KmNcGQqMrwcawUVuoF40IvyBOMbWkfHz37hBlaifvyPz56f6FoYhjEbOKgogVzdFCX6jW8AZtjlUKdENywx6glbot3+TJMra9G+jqsF/wI4xsyo6h3d2Cg0IHt88Cqjm3/SQhrlVtlyNVbaFXSMjjUaYoeYLwbd0Mdu3GppLSi6MtH4RsATwEOxh2YiSlzdugo/58cTK09YX1uPrNNa0B5qbVOUIq4HJ5CRD7ev4+eJTkOCXqtRUeKwYRjmnzMMY+z7NV7xOWVqNDwafLp5cjBLpwQXiSDSNFz/YVaUiwDg6EYP7DoO3LK5tytKzLtTYhQrwgoyjcxUf/Z2Yxt+zo9VYRUCL1ivGcpwREUEYE+UAEzB7LBEiQsrEXADrNe9PH8rhwDP4cmHdhd051Lme+g4hF0Sp8R+VkIr0AwNBdm+K0jt6Hiz3NwjSgS9ZiDg87dzU3GVbVVlpGN+sKx7ZnHjIdMpMe6f77dfuo/b2yJ+7gcvIOQbHH5ol9U4rQUFdsc3FgeNb4S8Q9s3iFMi7tn9/NqfaXJ5NYaarOJBee/fN8uwSAbMpp7ekMuNogTZ4wPXGv3vY7XgdEV2J3WghLg/DgBj50qQz/BJnBKAmStxu3R7qs5SwzBw9cFVPJl+En7P7r/xenQdD+oPoOna1H7WPEPW4OM4JQKeAPycn45vTBE1a65PqShxNLwOYKXn1+WjvRxKw2eAN8wb6qwyJQCAi0URqXdQlsuuGXHoxzg3egAIebuOA8U9ogTLsAh6gkMfR9o5xj05OQpaWgvlVtmRU2Kar7mslMVyaBkMwyAZSFKnhE3s5pwQgp7goY1vBLwczi8KQxs4DMPAC7dz+M/OJ+Hnd5t5znWr/OY9V0JSJYiKONApAThrsnlQbkLTDZxN7v33fvbxJWxVZLyem/wzJ1ORXZUnAZiZEkpHh6Q4c5UA5ojBL/3ZG3jvIyl8/6XlqVwPdUqY5MU2Ql4OwgChJyH40FQ6aCr9N69kIx/z9zRv7Ms0ubw6uF6YHHTsd0ooXj8gN0feo8h7kORKkLFBJ67OqC+658/iFCLMTOKUAMxciXKrjB1peueSb1TewLa0jWdOPbPn6+uRdWi6NpMWrnlkElGCYRgs+Bdcf7g4T5yUOlDAnaKEZhjGTs8vOox9hDSlGlQPwBpBeD3swJv1NGAjEUSrKhRdcY2boB/kJNDp+IbbHAd1pQ6BF0YGpLntuu1ATorsOCXSQtrabE3t5ze2rZ+9GFx0dHp8kiHveydOicN8XV5Zi+JGZnDY5a1tEdlaC89dWNrz9YWQFxG/Z+4bOMiifVU42PRg1es6aLIhIk2vUwIwcyUA4IXbk4t5marsquYNwBzfADBWA8cvfuk22pqOn/+hi1NLYl+NBVBsKGipzkWS40Su3r8OlEAOZQa5JSqtCgIaA194bx0oAKx1RYlHlgV4ORbXMgc3/eSgY68oISEQDQOGAaM53BW2/z1IApadjm8AGDvscppOCWC6YZdXH1wFAwbvXXvvnq8TZwod4TAheRDjtG8AZtglFSWmx64oMV7D0jzhRlHiHMMwWYZh7jEM8zsMw5w76gs6yVRLpm3I0INIhrwzraPhIlGEy6Z90s0faGM7JVyWzdBQG44CBXuvu/Brv4bG174+8TXcLN3Ep176lOM59FGQkyI7Ton9ltdpkJWy1vOmginqlLBJXanDx/ng5ew5sg5zfAMALq/FUJIUZGv957ufv5UHwwB/47G9mwCGYXAuNf8NHESUIFW6vZC2gP3p/61bt/Dmhz+M+z/2Dw78Cn3sf8D//vVfR/TjP4U3f/zHoebM98lSxI+3rEXx5Vu7uRIdvYNPv/xpfHPnm7avt6V2UKi3sRob7gY7bOKhrijR08AhfvnLKP3f/27o931zs4wvvJrFTzx9DmeToaGPdcKsa0F3ai389OdfRaPtbnt8XmxhcUCeBGAGXQJAsdE/V6LWrkFoMeCiUetrmYoMr4dFsjv66vNweGwl3LcWlBx07B/fCC+Yz9eRhn9+pIIpcAx3wCnhJBsg7ptwfKNtihLEcTEujyw8Ag/jmboocTl12RqTIRAR6F6NNnAAZvMGACwEnDslAFCnxJTRslkwPA9PKjn6wXOO20SJvwLw3wL4PgAfAbAM4C8Yhun7icowzH/HMMwrDMO8UijQk8hZUK2YNzelIyAx46AwLhpBpG5uTt38gZaTcuBZ3pq9tIsbMyXsiBL7nRKGqqL4a7+O6h/8/sTX8IU7X8Dvvv67uF68PvFz9UJOiuw6JQBntvNhyJqMcqu865QILKLQLLh6JMkt2H1NEoJ88FCdEpb1ekDY5fO3c3jbqVjfUMVzqRA2iu5474+L9b4KHXxfBfkg4r74AadE/fkXIP3lS4CuH/jVUjR4WYBV2pBe/CqaL79sfd+zjy/h1QdV5Lvhg59/4/P43O3P4WNf+5jtf/PtrnjkxvENAKj0NHBUf/fzKPzqr8LoDBZoX3y9AI5l8JPPPDzV61mLm6LNrHIl/uTGNv7g2xn85V13J/LnxPZwp0RXWBjolGhXEG4a4CK7G/KtrlOnN9Pk8moU1/s4rr7/7PfjQxc/ZH0GksyVWMJ8Pn2EKOFhPVgKLlnvwYJcwIJ/ATzLD/2+XqL+ycY3puWU8HE+nI+fx63SrYmeh5Bv5nGjdMNq3egl7o8j6otSp0SXcqsMlmER9Y4nLC34FyxhgzI5ajYLz8oKGNZtW/bp46o/oWEYf2IYxucNw7hmGMbzAH4Q5jX+gwGP/7eGYbzDMIx3pFLOTq0p9qjWzJOrph6yTglmBReNIto0b9JuDskhp+DDqib7QbIZ3DIGQcY3RmE5JbpZGGomA2iaZSmbBLLguPrg6sTP1Uu2kQXHcLZsq9N2SpDn6XVKKLoy1fGQ44rd1yThsMc3HlsOw8MyfXMldmotXM/UDoxuEB5KCciJbUguPy0exnZjGzzLDzx5XRFWDjgl1GwWnsVFnPncbx/49esf+F/wHz74cZz5f/8f67GE57rtJV95LY+iXMRnv/1ZPBx7GPlmHv/mO//G1vVa8/xuG9/oOiWqPU4JNZuFIctQNjYGft+9ooRT8QACXm7gY8Zh1k4J4goYJOa5AcMwkBOHj28kw11RYkADR7VVRbjRARfZ3ZBvVQ6OD11ejaLe0nC/tNfl9T1L34OffsdPW///zbJZI5xImQcgujTaFdb7Hiw0C44DucN8GBzDTZ4pMaEoAZi5EjdLN6ci6H9166sAcCBPgrAeWcd98f7EP+c4UG6VEfPFwLHjfc4QpwQ9iJkOauZk1IECLhMl9mMYRgPATQCTdV5RxqZWNx0oohqauVOCjUQQ7e4v3NxxvN3Y7mtfHoXllHBL0KVq71Sa53j4OJ+1+WvfMy2Ok4oSqq7i9crrAIAXt16c6Ln2sy1tYzG4CA87OgNlwb8AP+efmlOCLAiJU4JYcukIx2gaasPRYvawxzf8PIdHl8N9Q+peeM0cNXju8f6iBAm7vDfHDRyjBNl0KH3AKTGsX32jKOFsMgQ2GAQXj+/5THl8JYx01I8v38rjl175JcgdGb/0zC/hR87/CH771m/jTuXOyOvNVM3XxprrnBKmKEFqQQ3DsP7s8s3BdvW7hYbV5DJNlsI+cCwzM6fEte77ZVRzzVEiyhramo7F8OB1TiJExjf6OyVqrQoE2XR9EjL9RIm1br3wiL8PMu61vGza6Ec5JYC978F8M+94zJRhGER90YmcEgwYR463QVxIXECtXZtKO9bVB1exKqzi4Vh/l9F6ZB2btc2Jf85xoCyXxwq5JCz4F6AZGj2ImRJKNkNFCTfAMIwfwGMApjfsTXFEVTLFgUJbmGnzBmBmSkS6+ws3ixJZKdvXvjwK4jhwi1OiodjLlADMaydjJ8q9TQBAp1CE3h7c2T6KjeoG2p02riSv4LvV7+KB+GDs59pPtpG1lScBmIuw5dDy1JwS+y3uiwHzpIrM91IG43h84xDbNwhX1vpbr5+/lcPphSDOL/bfNJLN5N05DrscJciSU9rev5tBooTYUlFstK2/Fz6d3iNKMAyD5y4s4Rtbf4UvbnwRH7r4IZyNnsVPvf2nIHgF/MJf/cLIk7itigyWAZajg0+/j4JogAfD7I5vdCoVGC1z1KR1o78ooesGNkuSJW5NEw/HYjnin4lTQmpruFtogGHQ933jFnLdMaFhTgk/bzZzDMqUqLSrCMsA282UaKkdFBvtA+NDjyyF4fWwI50jJBg3nTZnye2IEivCCvLNPFRdRUF27pQAzFyJSYIuw96wYydpPy4mpxN2KWsyXtp+Cc+cemZgLtp6dB0FueCaQ6OjpNwqO8oh2Q/JonDzGPa8oLfb6BSKVJQ4ChiG+QzDME8zDHOWYZgnAPwegBCA3zziSzuxiHIFAFBDzApqmhVcNAKPDoSZgGvHN9qdNopycSynhJfzwst65y5TAjBdHruixG4YlLY9/kaeLDR+8q0/CQC4unV17Ofaz7a0bStPgrAqrE7VKcExnOWQIP9LGzhGU1fq1piTHYhT4jA3OpdWo6g2VWz1nCo3FQ3fuFvCc48vDVz0nkkEwTDzXQs6SpBNh9JodVqotM37htHpQN3Z6bugspo3upvs/aIEALzv0QSY5B8i7l3CR658BIA5//1P3v5P8K3ct/BHG3809HozFRlLET94zlVLHXAsg2iAt9o31Ez3z82yaN240fd7sjUZLVXH2dT0RQnAHOHYqkzfdXQzK8IwgPeeT6HYUKycD7eRE83rGuaUAMwGjn6ZEqquoqFJCMu7mRLZav/xIZ5jcWElMrReGDDfI0nBi8iCvUwJwLyX6YaO7cY2SnLpQKijHSZ1SkxjdAMAzsfOg2f5iXMlXsq+hHanPXB0A9gNu6QjHOah4CROCdLaQUWJySFrbCpKHA1rAP4DgNcB/AGANoB3G4ZBPyWOCLFl3jSbbHT2Tonu6UIcQdd+mJHO7HGcEoCZK+EGJV43dEiqNFb1onLvHsCbwVmTjHDcLN6EwAt4T/o9eDj2MF58MJ0RDk3XkG/mbTslgO4J7xSdEkvBJWt0hNhnqSgxGqdOiRAfggEDsjYb23k/rqyalXm9G4qv3SlC0XQ89/jgU0k/z2E1FsDGnI5v2BFkiRBIRpi0QgHQNPCr/UQJ83Nwv1OiV2DaUP8UnD+Hh7i/j4Bnd2P3w+d/GFdSV/CZVz4z1CK85cI6UEI86LXaN8jnaPCd70TrtddgaAdzR3ZFnOmPbwDAWiwwk/GNa103wAefON39/+4c4ciLpvthmFMCMEc4+mVKkCyFcM/4BnGe9Atavbwaxc2sCF0fLKhuFBs4lxTAhswgUltOie5973rxOgwYYzklYr7YRJkSEd90RAkv58X5+PmJnRJXt65C4AV8z+L3DHyM1cAh0gaOcqs8dh0oAEvQoGGXk7NbB0pFiUPHMIy/ZxhG2jAMr2EYq4Zh/IhhGNOJ3qWMhajWEWoBOuPpmyg/Tdju6UJM97tWlCCn6U5O4XvpHYM4ShpqAwYM26GCYW/YElPa9zcRfPvbAUwmStwq3cKFxAWwDItnTj2DV3KvWAu7Scg38+gYHUf/RulQGuVWeSqb2/0Wd7/Hj7A3TDMlbNBQGwjzzkQJ4HBHoh5ZFuDl2D3z8S/cziHs9+CdZ4efLp1LCbg3pw0cdgRZq8mmO8KkZsxZcH71YL/6RkECxzI4vRDsPiYNQ5bRqVSsn/cb134dC8xbcPPOqT2bN5Zh8bNP/Cyq7So+++3PDryeTEV2XfMGIR7kUe2Ob5C/p/D73w+j1UL77sGwSyLiPDRDp8SO2ILa0af6vNczNaxE/Xj6kRQ4lsENl+ZKkPGNYZWgAJAUfH2dEpYo0YQVdDksaPXyWhSNtoZ7pcGfXfdI5krI/De3lSnRfQ9+p/AdAM6rywHTjeQGpwQAXExcxK3irbHdcLqh48UHL+Kp1afAc4NbSE5HToNl2BOfK9HSWpBUaTKnRIA6JaaFJUqsHbyHHkdcJUpQ3EetIyGomC+TWTsl2FAQ4DhEVd61H2b7mxWcIvCCKzIliMBgd/FAxJROo4FOoYjQk+8GWBZKZrwAKrVjhlxeTJgzo0+vPY2O0cE3Mt8Y6/l6IaFYTtwsRESYhluin8Wd1IJSBqN0FLQ7bceVoMDhihI+D4fHVsK4njEX7R3dwAu383jfo4sjxwTOJUO4V5BcO1c/DDuCLPlcJI8ddsqzUWzgVDwAr4fd8xgyyvAvv/kvoRs6fuyR/wn5evtASOLjicfxdx/9u/j8G5/va+/WOjp2xJbrQi4J8aDXCrpUs1mwoRBC73kSAPqOcNwrShB8HqRGjBeMy2osAN0wW2SmyfVMDZdWo/DzHM4vCiPDHY+KvNhG2O9B0Ds8HDkh+PoGXVZappjWmymRqQ7ONLmyRuqF+/991GQVxYaCcykzCBawJ0osh5YBANcK1wBgLKcEGd8Y53NKbIuI+sarkuzHxcRF1NU6HtTHy5y6UbyBUqs0dHQDMF0Z6VD6xI9vkNfxJKJEzGe6Cd26jp8n1GwWYFnwS/0DtI8bVJSgDKVuNBHUzJv0rJ0SDMOAi0QQbbGu/TDLNrJgGRZLofE+INzilKgrdQCwPb9PxBSSJ+F7+GF4lpagjemUuFO9A1VXcSF5AQBwOXkZC/6FqeRKWMKRg9wPIiLsrzN0iqqr5ujIvp+dCqaQl6lTYhhOX5MAEPJ0nRKHHHZ5aTWKa1tmaN+rD6ooSQqeHTK6QXgoFYKkdJATxw+IPSrsCLIRbwQhPmQ9lggM/MrB79koSHuaJIibQs1m8ReZv8Cf3f8zfOTyR/BfXb4ClgGev5078Bz/+G3/GHFfHJ966VPQjb0n/Ll6Gx3dwGos6PBPejjEQ16rEpSEgXrX18GGQmjdPChKbBQlnEuFBmaWTMosakHrLRUbBQlXVs1N6pW1KK5vjbfZnTWj6kAJScGLsmS+tnrZHd8wrFHUTEXG8oBMk4dTAvw8O3CcpXe8iWFZMMGgLVHCx/mQDCTxetlstiKZRk6I++LQdG0ssXfqTolu2OW4uRJXH1wFx3B4avWpkY9dj65jU9wc6+ccF8jaexJRwsN6EPPFXLuOnyfUjFmpzfCDXT7HCSpKUIbSQBt+zXwzkBqzWcJFo4g1gWq7ClVXZ/7znLItbSMVSIFnx/uAELzucEqQDaDT9g0iSnjX180Z8Mx4ogSZEb24YC44OJbD02tP4+tbX5/4352c0jpxs+y3nY9LvpmHbugHnRJB6pQYhdPXJLA7vnGYtaAAcGU1inpLw/1SEy/czsHDMnjmkdGixNluHsDGHDZw2BFkGYbBSmhlj1OCi8etk15CvyYJ4pSQsm/iF1/+RZyJnMGHLn0I8ZAX7zizgOdvHxT1It4I/uk7/imuF6/j9+/8/p7fs6zzrnVK8CjvEyUYloX/4kXIfRo4Ngqzad4gkBGDaeZK3MiYeR+kAvPyWgyVfSGxbsEUJUYfvCRCXugGLEGJQMYdwh0vWJ/5PFvVweNDnm7Y5aBxFitDpDuuw4aC0Jv2PufSoTQ0QwPL9rrXAAAgAElEQVTLsGNtLonTwekIh2EYENvTFSUeij0EL+sdO1fi6tZVvH3p7bbcG+uRddwX7x8QOE8SpPlukvYNwAy7pKLE5Ayr1D6OUFGCMpQGp8KneREN8JbNdpaw0QgidfOGMG4l1SzJNrJj50kA3RYLFwRdWhtAm/P7gleApEho3dswrWSnT/dNy7fLzeJNhL1hrIXXrK89fepp1NU6vp379ljPSdiWtpHwJ+D32K8BTAVS8DCeiZ0SliCy3ykRSKEgF070YmcUxEHkJFPiKMY3gN1N1rVMDc/fzuFdZxcQDY4WKskG4+4chl3aFWTTQnqPKNFvQUWaJHqdEmwkAjYUwr+v/znui/fx8Xd9HF7OFMKfu7CI29ti33aIHzz3g3jH0jvwK9/+Fct6DACZqvlYtwZdxoJetFQdLbVj/j11w0D9Fy+i/dprMNRdcVZWOshUZUvUmgXp2PSdEmTE6TJxSnT/1425EjmxjaXw6HtGousYLUl7RQnSOBP37G7IM5XhQatX1mK4ka0dcF0A5nhTb+YKFwzZckoAu/efhD9hBS47gdjvnYoSsiZDM7SpBV0CAM/yeHTh0bFEia36Fu5U7uCZtWdsPX49sg5Zk090/hMJp5zEKQGYtaA06HJyqChBofRQ93Tg6fhmnidB4CJRRGrmYsyNKuu2tD12ngSwt8XiKLE2gA6cEpqhobG5AX5tDazXa4oSuVzfpPhR3CrdwsXExT1W5CdXnoSX9eLqg6uOn6+XcYQjjuWwFFqa2ClBbOv7nRKpYAqaro0dHnYSIC0Kbs+UAIBHlsLwelh86do23sg18Ozj9sa5liN+BHgO9+awFtTu+2oltLIbdDlgQbX/FBgwXRblh1P49/8/e28a3cZ5mAs/M4PBvhEguABcQErWQlKLLVurZZJKnCbXcZymN0m/fHHSmzRt2iS9bZo6m3Pdxs7S2k3SNm2TNPem6ZYv556kbeq4dp2IpGR5kW1ZlkRSK0FxAVeAxA4MMDPfj+EMCWLhDIiNEp5zeI4OMAtIYWbe93mfxTSM+9vvx1HXUek98e/7yyxqCYIg8MVDX0SEieBb574lvZ4vZLAaYDMIz1T/vB9cMCj9nbQ93eAZBonr16VtPYuZf69iQ0tTqDdqiqqUuDgdhMuqkybyu5pNoCmi6nIleJ7HfCiOBln2DeF3WQynW7ACiQDUHAm9SZjQi5km+ZQ6e1wWRBk2q3JqbCGCNptesn6QBvmkhPj8KcS6AQhBl4ByUkK8h1vUxcuUAIAuexdGfCOKSf2hKaHRa6M8CRFuixsA4Ancvg0cxbBviPtX4xh+K4FnWSTn5mqkRA01AADHcQhreCClLXmehAjKbIbJLwRtiTKyakGKS2E2Mrt5pUQVZEqIgwclmRIAEPB6oO5wA1iRW7MsUvPKVhUSbALXlq9JIZci9LQeh5oPYWByYFOe40KJo2ZDc8mUEmLYWM3CkRuigqigTIkykxI0RWJ3sxnPDguNFPmqQNeCJAl01BswtgUbOOReV06jEyEmhFAilIeUWPHLr7MjfP9wBCTH45F7Hkl7fZvDiM56Q9ZcCQDYXrcdH+z6IH567ac4P38egLDibzeooVNTsn6/cqNuRVmz5JkAsGpf0fX0AABia8Iuy0FKAILVpahKiallSSUBCCGxOxpNOcMdK4WlaBJJlkeDjBDR+pUFmvUNHEvxJZgYSmoRk5NpIoVdZiFpPIvpdh0lpIT4/CmkeQMo3L4h5moUUykBCGGXkWQEE8EJRfsNTg6i09KJNnObrO3FWtDbOezSH/dDp9JJhH+hsGltVTeG32pIzc8Lldo1UqKGGoBoeAksBXCsTnoQlxqUxQLToiC7rTaWdSG6oLhqcj2MaiOSXBIMm5neXU6IE0C5UnnRux+Ym4TG3QFgTVq+QgvHtaVrSHEpdNm7Mt7ra+3DdHgaN5ZvKDqmCI7nMBOeKej/yGl0FkUpUa+rh4ZKH9yKg8OFWI2UyAXRUqTEjyx+L4tR5aoUohR9R6MR7Xb5k8VOh0FSCmwVsByLucicrOtKXKWdmrkMPh6XbAlrMZalSWJgYgBnbUt471laahBYi7d2NeLlMR9C8eyZM7+z73fQoG/AEy8/gRSXwlQV14ECqxlNkYkpAKv3U7qtDaTJhPiaXAmRxOkoYaYEALRYi0dKBKJJjPuiktVJxN4WCy5OB6oq7HIuKCyEyAm6lOwbWZQS5jiRWQea5zvY6TBCr6Yywi45jhdICUc6KcFGy6SU0KwoJRRaaMXFjmJmSgCQxgpKLBwhJoTXZl+TrZIAhMUDvUp/W4dd+uP+TaskAIGUCDEhJNnqy4bbKpDaq7I8Q29VKDeb1XDbYNkvXBCJpA5uQ3mUEqTFDNN8BABZdX40ccKqpGpyPcRJVIgJbTpIaDMIMSFoKW3e3u61EJUSUSRWlRKuwkiJ4cWVkMv67oz3elt68Tgex+DUILbXbVd0XEB4oDIcU7BSYj46jySXLDjI1BvOrAMFVgeHNaVEbkjtG7R8pcR6+0bw2WcRu3hx4/0O3A3Tif4CPuUqxMmWXOuGiE6HEc9cnEEixUKjqs5V/PVYiC0gxafk2TdWVmknp0bRiux1oJ51TRLxVBxfP/t1uHk73nF6Dmw4AsqYPgF/y64GfO/UGE5dXcQDezOvbz2tx2fv+Sz+cOgP8eMrP8b0sgs7G+VbgcqNuhX7RnxqGiYAqpW/E0EQ0HZ3p9WCji1G4LRoN6yr3CxcdTo8PzoHjuNBkvJaPiaCExjxjeDtHW9Pe/2SV5ho711HSuxxWfGjs5OY9MfQZq+OZpT5kEAwyAm6tOpokAQyakGXE8swxvhVUkJGpglFEuh2mjOUEtPLMSRS6zJXClBKNOiU14ECgoWOAKHcvpEoDSmxzboNGkqDYd8wHuh8QNY+Z6bPIMWnFJESBEGg3dyO8cB4YR/0FoA/7oddu/mxqTi+9cf9BbfVVROeHX8Wu2270W5uL/m54qk4fnLtJ/iVaeH+WFNK1FADgCW/IGWPJnVlzZTQxzioCFXVKSVySfOVQJxwVTpXIpQMKfLui5L6qIaAWlRKrNT8KSUlRvwjsGqsWSfvjYZGdNm7Cs6VEP+PClVKcDyHuUh2ibjc82f7fohKids5QGsjhJIhkASpSDZKEiR0Kh0iyQh4nsfMo1+C/4f/gKV//pecP/6//yFmv/zlTX/eY9vr0VlvwK/e6VK0X2e9ARwP3PSVtzFkM5CuKxmErLjN9LyQiZArU2KtNH1wahDeiBe/Z30IKg5Ieqcz9jnQXodmixZ/M3g9azAgANzffj8ONB7AP4/+M7zL0arNkwAA64p9g52ZAUHTUNXXS+/peroRv3oVHCNMfMcWwugosXUDAFrqdGBSHBYj8itr/+qNv8JnT38WCTZ9H3Gi3ePMVEoAwIXp6snXUaKUIEkCNoMGvnV/o+XEMkwhFpR1tQ4UEP6m+bDHZcWwN4AUu5qXMCbadTLsG/LuGW2mNnTZu3Cg8YCs7deDIimYNebCMyVkNF0ogYpUYZdtl7SgsRFSXArfv/h9NBmasLd+r6Jzuc23dy1oMZUS4vG2OvxxPx4ZegQ/uPSDspzvlxO/xNfPfh0n514AkL1S+1ZFjZSoISeWA8LkjIFZkiyWGpTZDAKATW2pupuZGGK42aBLABXPlQgxIUXefUkpoQbUHQIpQep0oGw2xbWgw4vDGSGXa9HX2ocLCxcKUsqIapZClRLA6v+zUnA8h5nITNaJm5pSw6qx1uwbeRBiQjDQBpCEsseSGB7LLi6CC4fR+LnPYdf5N3L+OD75CaRmZ2XX6+WCy6rDyc/0YYfC1XhRkr2VLBzSdSWDkLXr7KBJGt7gJIBMUkJskli7Cjw4OQirxooj7b0AshOdKorEFx/YjWFvEP/0cnbPN0EQeLv77ZgMTYIh5raEfQNzs1A5m0GQq997bU8PkEwicfUaeJ5fIXFK17whQmktaJJN4oXpF8DxHCZX/r9FXJwKoNWmkxQhInY0mqCmyKrKlZhfISUcMjIlACFXIqtSIpwCKSklYqg3qqGl86uh9rZYEE9yuLHmfiBlrhSolNCqtPjxO3+Mg80HZW2fDXWauoJJiWIrJQAhV+Ky/zJYjt1w2x9f+TGuLF3BH939R6BIZWo0t8UNb9iLeCpe6Efd0vDFfLDpNk9KiGqLWyFX4tTUKfDgy0ZWiTalFxIjWSu1b2XUSIkaciIQXgQAxGGGo1xKiZVVhjrKVHWkhDfshU1rg05V+EC3apQSjEKlxMrnjpvUUDWs+lSV1oLGU3FcX76eNU9CRH9rP3jwODV1SvZxRYhBlYUqJYDVVWGl8MV8SHLJnBM3h95RU0rkQZgJFzSYNdAGRJNRJDxCYrpoL8oFkVRjblYmzEzMBdhKYZeSSkwG2UcSpBAaG5sHqdeDtKSvmq4PbUxxKZyeOo37Wu6DxiVUBOe6pzywpxn3bq/HU/91BQuh7Kv5olxbZRytaqUETZEwaVSgFzPT1bUrYZfxS5ewEE4glEiVPOQSWM0/kJsr8fr86xLBvn7AfmF6GXtd1ox91CoSu5pNGTkKlcRcMAGrnt6QQBBRb9SkZUqwHItAIgBjTFB7AhAyTWR8/3pWsmkuTK0SAGMLEZg0qrQsL9JgAB+LgWc3npQXAxaNpaCgS4qgpMWXYqLL3oVoKrphCOVCdAHffuPbOOo8ivvb71d8HrfZDR48JkLKQjVvBXA8h6X4Uk0psQ5Dk0KLS7lsPaIi6Kx2BoTr9lFJADVSooY8CESEm0kU1rIqJQCgDgb4Y9V1M9tsHSiwaoMQgyYrhTATlh1yCQAGtTDIYJrtaQoHpaTElaUrYHk2o3ljLXbW7USToakgC4c37IWJNikiXESI4XqFhl1ulDnSoGuoZUrkQYgJKcqTEKFX6RFJRcB4xgEAmhXSIRckUsJTmdo3k5ZGg0mztZQSCgnZZmMzZrkl0C5nhiJKJGNEcuaN+TcQZILoa+2Dqr4eBE0jleOeQhAEvvxQNxJJDl97ZjTrNk2GJjh120GZRqpaKQEIuRJa/0IGKUG7XKAsFsSHL0n1sWtXzUsFpUqJwclBqElh4ryWlFiKMJj0x6QJ93rscVlwyRsAl8OGU27MBeNoNG1s3RBhN6rhi6wqJUJMCDx4mKM8KMtq0KWc719nvQEGNZWWK7E+cwUQSAkAm1Z4yUWdpq6goEuT2pRTBbkZiGOGjcIun3rtKSTYBL5w6AsFfQ6xFvR2bOAIMSGk+FRxSYkqG8crRYJN4Iz3DDSUBr64T8q+KhVYjsWofxQuowsRmsXVHbePSgKokRI15EEgtgQAiJJW2A3lUUqIdVpWTlt1DKs37N1U8wawqjiouH1DaabEyudmGtIHmbTTieTMjOwk9RHfCIDsIZciCIJAb0svXpp5KcOnvBFmIjMFZ35oKA3qdfUF14KK++VVSsRqSolcUPqdFCHaNxiPB4RWC1VTZnPDWqjbhaCqRIVICUBs4Ng6SgmlhKzT4MS8KiqFN66FOMkWSYnByUHQJI2jzqMgSBIqZ3NeorPTYcRv3deJn74xjZfHskuDXZoDoHQTMOiU3T/KDYcaMISXM0gJMewyNjycNV+gVDBpaZi1KllKCZ7nMTg5iCPOI2jQNcATWL2ecoVcitjbYkEonsJNf3XkqsyFEmiQEXIpwm7QYHGNUkdUFBhjQoMYz/OYXpanlCBJAj0uS5pyZGwhnEFCkQZhciLXwrFZFKKUCCaCRc+TENFh6YBOpctLSpydOYtnPM/gIz0fKTiQUKwFvR3DLkWrRTFICQNtgIbSVN04XinOzpxFLBXDu7a9C0Dpvxc3gzcRS8Xw4a4Pg04BZ123l42oRkrUkBPBxDIIjkeMNJVPKbGyymBNquGL+6qmNoznecxGZjetlBBljdVg31CSKaFKclCleMTr0gfGtNMJPh4H65f34BleHIZNa0OjPn8ac39rP2KpGF6ZeUX2ZwQEtcJm2lGchsJrQTdSSjh0DvhiPlme2NsRSr+TIkT7BuPxQN3enubNzwZSq4XK2SwpKyqBjnqjNNncClBKyDYbm7GkZcG7Mq/ztU0S4sT2YNNB6d5IO50b5tR8on87XFYd/te/X0JyTUCgCGNqLwiCx3nfi7I/cyXQmhJW3WhXZliqtqcHiavXMO71Q60i4SyTFcVVp5ellLi+fB3T4Wn0tfbBbUkPBxQn2OtDLkVksyxUEvPBOBoUKiUiDIsYI9zLxcm7OQaQZjMWwwwSKU62fWiPy4LRmSCSLIcok4I3EM8goSSlRJlICavGWlCmRCnyJAAhfHO3bbe0sLEeSTaJr7zyFbiMLvzmnt8s+Dx6Wo8GXcNtGXYpqhqK0QxHEARsWtuWz5QYmhqCTqXD+3e+H0CmTa3YEEm3u3Q70HOTwyvG2aqZB5UDNVKihpwIpsIwJAioVMLqSTlArfiPLQkSCTaBaKo6VlL8cT/ibHzzSgl1lSglFGZKMDdvQscAcUv6wE1pLeiwL3/IpYh7mu6BXqVXbOGYCReulACEyVShSglv2AuT2pRzYu3QO8DyLJYSSwV/vlsZhWZK6Gm9oJQYH5esGRtB4+4AMz6u+FzFwjaHAcvRJPwRZuONK4xCCNkmlbDSttSceY9ZuwrsCXowEZpIq+2TYwnTqSn8ybu6cXUujB+cyVS8RMJNIDmL5AWuVrQwwuQ9W0OJtqcbSKUQHrmMDrsBlMyKzs3CZdXJUkqI9+bell6hsSAwLg2eL04F4LbrYdFnr1be0WiCWlUdYZccx2M+lJBVBypCzHoQGzhWlRI8KItF+vu56uRJr/e0WJBIcbg2F16TubJeKVFmUkJrRYJNIJaSZ+UBhEyJUpESgJArcdl/GSkulfHeP47+I8YCY/j8wc9Dq5JPMGWD2+KuKSWKgK1OSvA8j4HJARx1HkWnpRMUQZWFlNCpdHAFVThwjcc0ljEWGCvpOasJNVKihpwIpSLQMyTsBk1JPILZQGi1IGgalphwvmrxoxWjeQMA1KQaKlJV0UyJBJtAkksqGjwwHg/0CSCuTyenxMG0nAaOaDKKscBY3pBLEWpKjWOuYxiaHJLNEgeZIMLJ8KaVEjORGXB85urrRsjVvCFC7Iyv5Upkh1KiTIRk35ia2jDkUoS6owOMx1OxFQgxtNCzBcIuCyFkGyLCZHSxPn1SKjVJrPz+ImmwnpRILSyAS+S3Xry1qxFv3d2Ab/3iGmYC6ROn6aU47MSdOOM9o9gCVk40RIXnG+3MVEroVsIuVdevlCXkUkRLnU6WUmJwchA99h449A60m9sRZILS5PzidCBnngQghHx2NZvTchQqBV+EAcvxsupARdSvKEd9Kw0cS3GBaDbFhFws8e8nVymxt0UIBL04vSxlzXSsV0roy2vfsGqEz6QkV6KUSglAICViqViaVQgQFiO+8+Z30N/aj97W3k2fx212wxOs3POhUhCtFsUkJaplDF8IRv2jmI/Oo6+1DzRFw2V0lZysGl4cxm7bbnDeWRy4Lnz/CslX26qokRI15EQQMWgZFepN5cmTAATJF2mxwBwWJoXVwrKK6fObVUoQBAEjbayoUkIM6lESKsiMj0OXAKLrFpNE2bEcpcTVpavgeC5vyOVa9LX2YT42jxF/drnmekgNAZtUSiS5ZGF1pGFv3nM79EJrSa0WNBMczyGcDBcUdGlQGRBhwgDLbhhyKULd0QEuEkFqoTL/F2K9440tEHZZCCFbHxDu34vr5idik8TaPIldtl1SyCywOkFPzWysWHrswW6wHI/Hn06/R0wvx7DNcBCxVAyvzr4q+3OXG/awHywI8Pb6jPdUzc0g6+pgmx4rKynhsuoQSqQQiCVzbrMYW8TFxYsSmSSGA44Hx+ELJzC9HMuZJyFib4sFl6YrH3Y5HxI820qUEqKdVVRKBBICuWKKrpASy4LCU27QartND5NWhQtTgdykRJmVEnWaOgBQZOEIMkGYNaUjJcQsqvW5En/26p+B53l87uDninIet8WNEBO67VSN/rgfBAiJkNosbFrbls6UGJocAgEC97XcBwAZNrViI8WlcGXpCrrsXUh6vbCHgN2WHTVSooYaACCMBNRJFeyG8uRJiKDMZpiXhQFRtdzQiqWUAFZXdisFkZRQZN/weGDgaUS49BU0ymwGaTTKIiXEgUS+kMu1OO46DpIgZd+QJeJok0oJQHkDB8/zGysl9IJSolYLmolIMgIefMFKiTiXAEdAtn1DVFRUKleipU4HmiK2RANHIYRs3VwUBMdjXpOuUhhb0ySxFF/C+YXzaSoJYI36SsY9pdWmxyf7t+OZi7M4dVUgmILxJELxFPbV3w2dSlfVAzpz0AefzoJAMnNiThAE+B27sH1pUiKxygGpFjSPWuLU1Cnw4KX/uw6zcN2NB8Yl9cOeLHWga9HjsiDCsBXPVpkPCt/RBgVKCTH4ezG0opRILIHiCehVOhBqNaaXYjBpVLDosttX1oMkCfQ4BZLGsxiGy6qDTp1eT0qVmZQQAyvlkhIcz5VcKeE2u6FX6dNyJU5PncYvJn6B39r7W5teNFp7HuD2C7v0x/ywaqxQkcWxa9t0AimxVRUnA5MD2OfYJylH3GY3JoITBSlp5cAT8CCWiqG7vhtJrxekXo8+91vw5sKbBS2UbUXUSIkaciJMJUEl1bAby6eUAIRcCfOyMFCoFlLCG/bCQBuK8sCtFqWEkglgwjOek0yRWws6vDgMh84hTc43Qp22Dvsd+2X7wiXiaBNKCXFQozRXIsgEEUlG8g6KxPComn0jE6KdqRBSQk8Lsua4GlC73bL20axsV6laUBVFot2+NRo4CiFk+Zk52MIEZvj0CY1nTZPE6enT4Hguk5RQmFPzW72d6Kg34LGfDSORYqXJtNtuwZHmIxicHKzaQbFhaQHz+josRbOrEgKt29EemkOnicr6fikgWg6mlnLnOQ1MDqDZ0IwddTsACPdNmqThCXqknIgeV/5npaikuDhd2bDLuaColFAWdAkAi2uUEhZWDdVKe9j0srw60LXY22LB6EwIl2dDWZUxolKCLZdSQqtMKRFJRsDxXMnaNwCAJEjstu+WFjgSbAJfO/s1uM1ufLj7w0U7z1rlz+0Ef9xfNOsGANi1diS5ZMUz1ArBbGQWo/7RtOeT2+JGnI1jNjJbknOK32tRKUG7XOhr7QMPHqenT5fknNWGGilRQ06EaQ5EUi35J8sFymyGaVEYEFULO+iNeNFsaC5KtoZRbayoUkLpBJDneTAeD4xaS9YsDNmkxErIpRL0tvZi1D8q6yHgDXuhoTSwawtPjhZJBaVKCTkTN5qkYdPaarWgWRBkggAKIyXE1gamyQbKJG9/VXMzCK22YqQEIMizK71KLAeFELJJrxcNcbV0XYgYWwhDoyLhsgoKhgZdA7ps6RkzdGMjQJKySQmNisKXH+qGZzGC7w2Npfn5+1r7MBedw2X/ZdmfvZxQ++Yxr6vLGXg66WgHxXNo8U2X7TNJSokcYZfxVBwve19GX2uf9DykSAptpjaMB8ZxYTqAznoDTNr8KoHtDiO0NImLU8Hi/gIKMbeilHAoGOfo1Sro1ZSUKbGcWIaJoUCZhWtkakleHeha7GmxgGE5gZTIUv8qkhJ8tDzh30qVEuI9vJRKCQDotnfjiv8KklwS/+fi/8FkaBJfPPxFqKniLZ45DQLJdtspJeL+ojRviBAJjmpZXFSCU1OnAAhNcCJKraAZXhyGXqWH2+wWSAmnE7ttu9Ggb6hqxV8xUSMlasgKjuMQ1nDgU1pJqlgukBYzyOUwTLSpam5mM+GZokkDjbSxokGXwWRQ+hxywPp84EIhmIy2gpUS0WQUnoBHVsjlWogstZwb8kxkZtPEkTj5EiXrciFX4t6gb6gpJbJAXEkpJFNCrxKUEqxb/vVJkCTU7e0VbeDodBhw0xcBW2FP/UYohJBNer1o5I0ZiqOxhQg66g1I8UmcmT6D3tbejOMSNA1VQ4Os8FwRx+9w4IE9zfj2wHW8PCYQ2a46He5ruQ8EiKoc0PGpFMhFQSmxHM1OSgyv3E+oG1fK9rnsBjW0NJnTvvHKzCuIs3H0tfSlvS76rS9NB7BngzwJQFALdTstlVdKhOKwG9RQq5QNh+uNGvjCAqGxFF+CKU5I7WEFKSXW2F3WN28AAKHTASRZNqWERErIDLoMJlZIiRJmSgACKZFgEzg1eQrfv/h9vN39dhxuPlzUc4gkmydYOdK6EvDFfUVXSgDVs7ioBAOTA2g1taLDsmoJFf9dKgXNiG8EXfYukAS5opRwgiAI9Lf240Xvi1Ud2lws1EiJGrIiFFgARxLgWV0FlBIWsIGA5EerBogD82Kg0pkSSpUS4mqy2dKQVYZHu5zggkGwoVDOY4z6R8GDl50nIaLD3IF2czsGpwY33NYb9haFOHIanRkrvBtBrsTdoXPUMiWyQLQUFbLKJiolkq2NivZTd3QgMV65Qee2eiOSLJ9XJl8NKISQTXq9aFLZMRedS6vvG1sUSIlXZ19FNBXNsG6IkKu+WotH37kbFEngf5/xQK0iUW/QwK6zY69jr6z7R7mRmp8HwbF57RuXGDVCejPily6V7XMRBAFnnlrQgckBGGgD7m66O+31dnM7JoOTmAlEsCdP88Za7HFZcGk6WFFibj4Yh8OkfIxjN6rhW1G4BBIBmKI8SItZyjRRqpRotemkDIr1IZfASgi4Xl+2TAmapGGiTVWnlBAXNh498yhUpAqfufszJTmP2+LGzeDNkhy7WuGPFde+YdNtTaVENBnF2ZmzaWowQCBZDLShJKREkkviytIVdNu7wYbD4IJBKV+pt6UXsVQMZ2fOFv281YYaKVFDViyvyEVTnKEimRJcKASbpq4qbmZhJowQEyquUqICHjs+lULszTcVZ0okVlaTzfVCMwXDpq/qyQmmG15c9copAUEQ6G3pxdmZs3hm7Alyg7kAACAASURBVBk8O/5szp/J0GRRiKNmQ3NBSgktpd3wge7QO26r9g02GETi+vUNt5MaYdTKlRJaRpjQJJ0ORfupO9xITk2DZ7KvUpcaom9cbthlJJHC9fny3Tc8ixE8fcGLm8FpMHELnr7gzfoz4UsnVbhEAuzCIpyGZrA8KymDmBSHCX8UnQ4DBiYHoFPpcKj5UNZzF0JKNFt0+IO37gDPC9YNkhQGk32tfRjxjZTMB1woxN9vTleHpRxKibHFKJZbtyM+XHxS4rL/ck5y3JWDlOB4DqemTuGY81iGXN5tdiPFp0DQS1LF5UbY47IglmRxo4LZKnPBhKI8CRF2gwaL4dWgS2OEBWW2rNqHFColCIKQyJxcbSukwVBUUoIZH0fKl3sV26KxyCYlxAaSUpMSbeY2aQz1if2fQKNBGRktF26zG5OhyTRSdbMIMkFMBieLcqzY+fPgueIFLjIsg1AyVFxSYovaN17yvgSGY9KsG4BwjbrN7pLYN8aWx5BgE0KexIpKUBxbH2w+WPWhzcVCcSJWa7jlsLw8BwBI8cbyKyUswkPNprLgZqx8XtpcEPMFNtPqsBYGdWWUEoGnn8bM5z6Ppb94L0iClGTvG4HxjINQq2G2NQM3BKm9jVp9cEmkxLQX2p07sx5j2DeMRn0j6nWZ1Xcb4f72+/EPI/+Az57+7IbbbrduV3z89WgzteFF74uIp+LQquQNVmciM2gyNG0ocXfoHPDH/UhxqaIlXFcz5v7szxB85j+x48wLIHW5B+mFhK+KUM8Jg+Zko7IaM01HB8CyYCYnodm2TfF5NwtRon1jIYz+XfnDX3mex2//4+t4xePDf/7P+7C9obRtDN7lGB74y9OIpiIw7QxjcDiF50+/kXVbk1aFgc/0Sc8JscrTaXMD0VOCyszYjAl/FCzHo8NuwHfHh3Ck+Qg0VPZnC+1yIfjss+BZFgQlP+TxN4658a9vTKPNtnpv62vpw1+c+wucmjqF9+18n+xjlRoiKRGy2LGUJVMiEE3CF2HA3bETiWd+DC4aBamXd8/eCPFUHB985oM43HwY337LtzPeb6nTYcSbmfUw4hvBQmwhq8JFlDZTmkV0O+VNTKWwy6kAdjQqv/aLgblgHLublZ+73qjGm1PLuLZ0DUvxJdh9BKgOc1qmiVIc2WbHlbkQnJbs+wqkRHGUVTzL4ubDH4Lu7gNo+eY3s25Tp61TrJQoZdAlIIRdHmg8gLnoHD6w+wMlO4/b4kaKS2E6PI12c/umj8dyLD72Xx/DeGAcP3v3zzZFpkTPvYGbH/gAXN/8BszveMemPxuwShyI6oZiQAxL9cW3ln1jYHIAJrUJ+xv2Z7zntrhxbu5c0c+5tp0ucUogoenWNgCAhtLgmPMYBqcG8Sj/aFGy7aoVt/6ouIaCsBwQVpUSvLHsSglyJSzKSuhxrgq8aOKq+WZaHdbCSBuRYBNIsknQlLzKsGIgdv48AMB/8wqMJqPsGxvj8UDd3gajRhi4RZhIGpsuRykx4htRHHIpYn/Dfjz7a88inorn3Y4kyKIMHo46j+KHIz/E2dmzUj/1RpgOT8tS0jToG8DxHPxxv+wWkq0KnmUR/uVJ8NEoIi+9BNOJEzm3lUgJWvnkQOUV7hGMXdm+6jUNHJUgJWwGNax6WlbY5dMXZvDC9UUQBPDYzy7hnz56qKQDk8efHgHL8fjGB9rw2OvAF992FMeaM6+FhVACH/7BWXztmcv48/ftA7B6H2hp3AF4hPvngcYDUvOGSjeD2cgsfnff7+Y8P+10AqkUUvPzoJvl33dpisRPfuco1v5ptlm3ocXYgsHJwaokJVL1jfBnUUqMLQrqAf2ePcDTP0L88mXo77qrKOe+snQFCTaBoakhDEwMoL8tfUWwpU4PX4RBjGHTqikHJgdAEiSOu45nHFMMgXPYAjBo5A0tOx1G6NUULk4H8GsHWgr/hQoEy/FYDBeolDCq4Y8k8MTLT8CiNuP+sz5Qd1okhYlSpQQA/PZ9nfjwUbek8lmPYiolYhcuILWwgNibb+bcxqKxyF7lLpd9AwCe6n0KPPiSEvtrQw2LMa74v1f/L0Z8IyBA4MnXnsRTvU8VfKzQL34BAIidf7PopMRmgsLXgyZp4TsU2zpKCZZjcXr6NI67joMmM8fnbrMbPx/7OWKpGHQq5dd4LgwvDsNEm9BqasXM4LdB1dVB27Vber+3tRe/mPgFRv2jihXHWwk1+0YNWbEcFgb6CcICW5mDLqmVWq06ToflxHJR5XOFQAoxLJZSYsUDX24LR3xY6PYOzE0qWpFmPB6o3R3S5w4l07MjKLsdhFqdk5QIM2GMB8c3dSN1GV3YZt2W96fD0gGS2Pwt7e6mu6FX6RVJ5eT67h06wWJwO4Rdxt68AHZpCQAQHhjIu204GYaW0hZE0tGTKwSqwomFukNY2U1UuIHDs4F9I5xI4Ymfj6DHZcb/emcXzlz34ekLyjJPlGDo6gL+89IsPnViO+wWYVX2Llcn7mg0Zfwc3V6Pjx3vxE/OTeGsRxh4SqREu0BCinkrYv3peOxVECDyEn5yiM5c0KkpaOnViTRBEOhr7cMrM68gmqye/I7ktBeUzQa92YDlLJkSoq2n+Z47AaCouRIjPuFZ0GxoxtfPfh2xVLpVQ1zlX2/hGJocwp0Nd8KqzVQlWbVWgDXAYpYfXEmRBHqcFlyYqkzYpS+cAMcDDQXaNwjT6zg3fw6f2vmbMMWEBZXp5ZiUaaIUKoqEMQ+hU0xSIjwwCABIeWeQ8mefNFo1VkVBlypSVdSJWi5oVdqSn0ciJYqQH+CL+fCXb/wlDjUdwu/s/x08N/4cXvK+VPDxxOdpfHh4059NhBhGWUz7hni8raSUuLh4Ef64P8O6IUKsi50IThT1vGLIJZFiET51Csbe3jSVYDWHNhcTNVKihqwIRlduIho7NKrydaQDAGUVSAlLSg0evGz5YKkwE5mBmlQXrSpJbBgoJynBMwwSly+DNJsRTARghLxBGJ9MgpmagrqjQ1rFXm89IUgSdHNzzgnEqH8UABSHXFYKakqNY65jGJocAsdv7NmMJqNYSizJIq1EdcTtEHYZHjgJqFQw3HsvQgODef2vISZUUJ4EAKhuCpPeKJdfSbMelMkEqr6+sg0c9UZpRTwXvvX8VcyHEnj8oR586IgbPS4znvj5CMKJ4pO18SSLx/79EjrrDfjYfZ2r1rU8hNsnT2yHy6rDl/7tEpIsJ9wHSBJmZztsWptE6o4tRGA3qPHy7GnsdezNez+lXYWTEtnQ19oHhmM2NREoNsTKN5tBnTVTYmwxDIok0L6jDaqGBsSKSEoMLw7DprXhq/d+Fd6IF3934e/S3s9WC+oNe3Fl6UpG64aIuWAcbMIOglZGuPa4LBiZCSLFFs8fLxdiHWhjAUGXBh0DTcN/Yoe1Gw+ajwGAlCnRsibTpJgoLilxUlKl5prcWjVW+ZkSTABmtfmWkZZbtVZYNVZ4Apsnrb/x+jcQS8XwhcNfwEd6PoJWUyu++spXM/K55CAx5gHj8YA0mxEfGQHPspv+fEBplBKAQEpspUyJgckBqAgVjrmOZX2/mGSViCQrhFx22bsQPfcGuGAQxhPppIhNa8M+x74aKVHD7YlgTHgQ6c1NZT+32PVtZQQypNI3NG9Y8EQXYxUeWCUlypkrEb92DXwyCdtvfBhRLQFdJHva+3owU1NAKgV1RwcM6hWFR5Y6U9qVO5iu0JDLSqK/tR/zsXmM+kY33FYM0JNj73HoV5QSt0HYZejkAPT33A3LQ+8Cu7iI+MWLubdlQgXlSQAAceMmSL6w60njdoPxjBd03mKg02HAXDCRk2C4PBvED14cx6/f04Y72+pAkQQef6gH86EEvvX81aJ/nu+dGsO4L4o/eagbGhWFmbBAyOZbPdOrVfhfD3bhylwIP3xxHMlpL1QNDSBoGs2G5lWlxGIYrY4khn3DOVs3RIiWDSW1oPlwV+NdMNGmqmrhEEkJq16dNVNibCGCNpseahUJbU8P4peKtyo67BtGt70bdzfdjQc7H8TfD/992uRLUkqsqQUVB8O5/u8uTAXAMQ6EOWUqnr0tFsSTHK5XIOxyLigQmYXYN04v/iMIKoL3d/w++KDw2SmLGVNL0YKsG3JAGorTvsFMTiJx7TpsH/4QgNwqHKvGimgqKmvyHEwEy2LdKCfc5s03cLw+9zp+duNn+I3u30CnpRMaSoMvHPoCxoPj+OHwDxUfT1RJ2D78IXDRaNFI9VJkSgBbj5QYmhzCgaYDOccjbSYh56GYYZfXlq8hySXRVd+F8MmTIGgaxmOZpEhfax9G/aNVF9pcTNRIiRqyIpAIgGJ5mCzFZU3lQGTvV5TDFb+hzURmilYHCqw2DGSb3JcK4oDW8uCDiJnU0C7kru9cC3HCpulw51V4qPKk5Y/4RuA0OIsuCywljruOgyRIeVWkCoJQbVobSIK85ZUSzM2bYG7cgKm/H8bjxwGKQiiPhSPEhArKk+BZFsmbE9DydEHSfHVHh1R5WwlsW0nZz2bh4HkeX/q3SzBrVXjkV1YDZO9sq8Ov39OGH7w4jsuzmWGEhWLCF8VfD1zHA3uacfwOgTybDk/LImTf1tWIE7sa8M3nryIyOSXZL5xGZ5pSQmcRiJRcq+0iSJ0OlM1WNKUETdK4t+VenJo6BZYrzsriZsDzPJIzM4JSQk9nrQT1LEbQuVINqe3pBuPxgA1vfkIaTUYxFhiTlGufvvvT0FJafPWVr4LnhSabRrMWKpLA9PLqNTU0NQS32S3Jl9fj4tQyeMaBZcan6Nm2ZyXs8sJUoMDfqHDMhQojJYZ9wzg1+zMkl45AjzawAWERh7IImRKFhFzKQbGUEuLE1vLgg1B3dCCWg/ASgwrlqCWCTLDkIZflhtvi3tSKeJJL4omXn0CzoRkf2/Mx6fV7Xffi/vb78b0L38N0WFmYe3hgAJpdu2B+29sAFM/W5Y/7oaE0sgPQ5cKutVd8DC8XE8EJ3AjcyPt80tN6NBmaiqqUkEIubd0IDQxAf/gwSENmA49oKTk1dapo56421EiJGrIimApDnyBQbyy9P3A9KMuKfWNlXFPpkBxv2Fu0OlCgQkqJ4WGQFgvolhZETTQ0s0tgwxsPHMUJm9rtljIlsn1u2ukEu7gILpHIeG/YN7ylVBKAIN3c79gvSyonZY7I+I6oSBXsWvstr5QQCQhjfz8oqxX6u+5C+GRuUiKcDBeklEjOzIBnGBgobUHXk7qjA+zSEtjlyljExAaObBaOn5ybxqvjS/jcO3ahbl2uzyO/shNmrQpf+rdL0kRyM+B5Hn/8H8OgSAKPvnM1XEsuIUsQBP74wW6kOB7+sQmJlGg2NGM2MovlCANfhEGIehMtxhZss24cLFpILWg+9LX0wR/34+JibsVOucD6/eDjcUkpEYgl0+wLHMfDsxhBxwopoevuBngeidGRTZ/76tJVcDyHLptwT67X1eNTd30KL8+8jOduPgdAyHposmglpUSYCePs7Nm8CpeL0wE06loBQNHqcofdAKNGhYuVICWCCRCE0KQhFxzP4SsvfwVWjQ2JhbfBF2bABQVyMKU3YjHMlIyUoIpESoRODkC9fRvUbW2CCieHfUMkGeSSEreiUmIxtljwAtK/jP4Lri9fx2cPfhZ6On2y/8g9j4AgCHz97NdlHy+1tITouXMwneiHurMThE6HWJFyJfxxP2xaW9HtNzadDYFEAElOnjq3khDHe72tvXm3K3Yt6IhvBGa1GY75BJITEzCdyJ5n0WHpQKupFQOT+TO6tjJqpEQNWRFio9AmyLI3bwAAqdGA0GphCgqS5kqG5CTYBHxxX1GVEpUIuoxfugRddxcIgkCU5qGPcYi8cGbD/ZhxD6i6OlBWa16lRK5gukAigInQxJbJk1iLvtY+XPZfxkw4vxx5JjIDFaGSQiw3Qr2u/pZXSoQHBqG54w6oW4VJivHECSSuXgUzlX1VqNBMCZE0M9DGwkiJlQaOSoVdttn0IIjVUEMRgWgSX3tmFHe1WfHeA60Z+9UZ1PjcO3bh1fEl/OTc5muTnx+Zw8nL8/iDt+5A85o6QiWEbJtdj9+9rwP6gA+zOiEI0Wl0Is7G8ebMFEAwmI6/ib7WPlkD32KTEsdcx6AiVFXhyRV/L9rlRJ1eCHcNxFYH7dPLMSRSnERaabuF+2euFW0lWFs9J+J9O96H3bbdePLsk9J15LLqpEyJM94zSHGpnKQEz/O4OB1Al0OoZPYE5V9PJEmg22nGhenykxLzwTjsBg1UlPyh8E+u/QQXFy/iM3f/IUheC184ATYgkBJzvDBeKp19wwCeYcAnC5/gscEgoq+9BlO/0Iak7e5CanYWqYVMotyqEa5jOWGXgUQAZs2tR0oAheUHzEXm8Dfn/wbHXcdxojWzearJ0ISP7/s4BicHMTQ5JOuYkVOnAI6Dsb8fBEVBu3t30Wxdvriv6HkSwGpGhdzA1EpiaGoI263b0WrKfOauhdssKGiKsSAACBbnLnuXpGAy9vVl3a5aQ5uLiRopUUNWhPgYNAkKdqPyAKhigDKboQvEoCJUFZV+iRPSoiol1OVVSnCJBOLXrkHb3QOWYxHhYjBCIwQRboCExyO1FGgoDVSEKqdSAsgkJcSQy62mlABWvdNDU/kHDN6wF42GRlCkvEDYBn3DLd2+wQYCiL72Goz9q2y/qb8PQO4WjkIzJUR7kUFnQSRViFLCLRxnfHO+4UKhpSm01OkyakGf/K/LWIoyePzdPTkD8957oBV3tVnxtWdGEcgi/5eLKJPCn/zHCHY0GvEbx9zS64UQsh/dbYSK5/DvXg6JFCvt++aMByrDNaT45IZ5EiJEUqJYAz+LxoK7Gu+qDlJiWiQlXJIKZq2FQ/w+dK7Ye1T19VA1NxdFqj28OAyHzpFWSUyRFB49/CgWYgv42/N/C0CYWItKiaHJIVg0Fuxz7Mt6zJlAHIthBve47gBJkIpXEfe2WDA6E0SyzGGXc8E4Gs3yxzj+uB/fev1buLvxbjy47Z2wGdRYjDBgV5QS3pTwDCilfQPAptQS4dOngVRKuj/renoAIOuKu0RK3K5KiRWrUiFhl0++9iRYnsXnD30+Jwn78O6Hsc2yDV87+7WMBpxsCA0MQuVwSCSltqcb8dFR8KnNhx77Y/6i50kAq20e1d7AEUgE8Prc67KeT26LG+FkuCi/U4JN4NryNXTbu4XFnK7deWuw+1v7keSSVRXaXEzUSIkasiJIJEAnaTgqoJQAhMAoBIKo09ZVlJQQ8wK2slIicfUqkExC29MjTdzqWrchPDi04cOM8YxLEzeCIGBQG7IHXTpdADJJCbF6rtu+9ZQSHZYOuM3uDScxSjNHHHrHLW3fCJ86DbBsmgRR7XZD3dmZkwgrNFOCGRdSyA06c2GZEi0tgEpV0VyJznqjVJcJABemlvHPr0zgQ0fc6Hbm9miTJIHH392DpSiDJ//rcsHn//bJ65hejuHxh3pAr1kxLoSQJeeEAK5hTo/vn/ZI+171T4A2jcJIm3BX412yjkU7neDjcbA56goLQV9rH24EbmAyOFm0YxaC5LSgbqGdTtTpRVJiNUxQ/D6IpAQA6Hq6i1IBKIZcrsdex16854734J9G/wlXl66ixarDbDCOWJLBqelTuM91H1Rk9rpKMQ/izrZ6OA1OxeGAe1qsYFIcrs7JyzoqFuaCCUV5Et96/VuIJqN49PCjIAgCdoNmRSkRAGk0YjokEEulVEoAmyQlTg6Astmg27cXAKDdvRsgiKwr7nJJCY7nEGbCt1ymRKupFSRBKv4+v+R9Cc+NP4eP7vlo3lV3mqLxxcNfxHR4Gt+/+P28x+QYBpHTpwWVBCncp3U9PeBjMSTGxhR9vmzwxX0lyf0Sj1lpG/ZGODN9BizPyiMlRAVNESwc15auIcWlsEvTjtgbb0gKplzY37AfJrXplrVw1EiJGrIirEpBlaIrppQgzRawgaCQ3FvBm1kplBJaSguKoMoWdCkOZHU93QgxwqDPvmMP2EAAsfPnc+7HBoNgfT5oVpQSgJCHkVUp0dQIUFQGKTG8OIwWY8uWHaz0tvTi7OzZvKoWpZkjDboG+OP+LeGxLAThgQFQdju0e/emvW460Y/Iq6+BDaVPPBJsAgzHFKSUEJQ8Qt5JIcojgqahbm2tLCnhMMCzGAHP82A5Idyy3qjBp9+2Y8N9u50WfOiIG//8ygQuTCmXx16fD+PvTo/hPXe6cKgzXbpbCCErXv879+3AX528Bo4RJjU3A1OgzVdwvOVe0CQt61jFrgUFVgM2K93CkfR6QRqNoMxm2ESlRGQtKRGBSaOCY83zV9vdA2Z8POP6UYJoMgpPwJNTufb7d/0+TGoTvvLyV+C0asHxwMD4qwgkAnkH65emA6BIAl3N5oLCAfe6hOdDuXMl5kMJ2UqJ8/Pn8a/X/xUPdz0sZaLYjWoshhlwwQAosxnTSzEhj6OANg85IPVCLgFbICnBJ5MInz4NY18fCEpQdZAGA9TbOrMSXlatPFIixITAg7/llBJqSg2X0aXo+8ywDL76ylfRamrFR3o+suH29zTdgwc6H8APLv0gL/kRPfsquEgExhXVIQBoV1Qu8eHNZc3wPC9lShQbYvVztSslBicHYdPasKd+z4bbigqaYoRdiu107svLAM+nKUyzgSZpHHcdx+np01UR2lxs1EiJGrIiTLMgklrYDZVSSljABoOw6yqb3OuNeEESZJrUdbMgCAIG2lA2pUTs0iVQVitUTqdEhNR33QXQNEJ5wgfFqinRdw8IpEQomTkoJlQqqBobkFpPSmzBkMu16GvtQ5JL4kXvi1nfT3JJLMQWFCslAMAXq+6HdCHgGWZl0NsrreaIMPb3A8kkIi+8kPa6SJQVat/QuAsnJYCVBo7xSpISRkQZFrPBOH50dgJvTgXw6AO7YdbKm7x/+m07UG/U4Ev/dgksJ9/qwPM8HvvZJWhpCp//b7sz3i+EkBUJhE/8+jGQBIE/f24SBtqA6eTL4MnQhq0bayFZwopUCwoAreZWbLNsq7iFQ6wDBQDrSqbEWqWEZzGCTochTfYtSrY3MwEZ9Y+CB58z48eqteIPDvwBzs2fw1RSuE5/eXMAKlKFo86jOY97YTqAOxqM0NKUVKPI8fKtGO12PUxaVVlzJZIsB18kgQbTxgRCikvhiZefQKO+ER/f93HpdbtRI2VKkCvNG01mraKMCiXYrFIi+vo5cMFg2sQWEIJUs1mDNJQGOpVuQ1IimBDsK7caKQEoDzX84fAPMR4cxxcOfQEaSh7h9Zm7PwMNpUlrwFmP8MAACK0WhiNHpNfUbjdIvX7Ttq5QMoQUlyqtUqKKGziSXBIvTL+A3pbeDZumAIGo11CaoiglRvwjsGqsMAydg6qhAdrujcfL/a39VRPaXGzUSIkaMsCyKUQ1PAhWW9FMCTYYgE1rqyjDOhOeQYO+QfbqnlzkUhyUAvFLw9D29IAgCASZlcGDuR6GgwdzevyBNc0ba5QS+SZ/tNOZNoFYji9jOjy9JUMuRexv2A+LxpJzEjMXmQPHc8qUEisE160Ydhl9/XVwoRBMJzIliLr9+0FZrRlEmEiUKQ265KJRpGZnoe7ogIE2FBz8pO5wg7k5AZ6tzKrDtpWGhdfGl/Dkc1dwpNOOd+2T/30ya2k8+sBuvDkVwI/OTsje7z8uzODMdR/+6Fd2wmHKvM8XQsgmp72gbDa4mmz4n2+5A78cnYdJ5UBSNQECFI65MrvXcyFXTs1m0dfah9fnXkcgUf5gRRFrSYlV+8aaTImFsNS8IULbI5IShU9AxFW5fETxu7e/G/sc+/BvE98FyCheX3gBB5sO5rw+eZ7Hxall7F2p9uywdCCWiim6vxEEgT0uCy6VkZRYDCfA8/LqQH985ce4snQlo0XBblDDFxYyJUSlRKnyJIC1pERh97rwwEkQajWMR9MJJm13D1ILC0jOZf6fWTXWDUMKpXHFrUhKWOSTbNPhaXzvwvdwf/v9uNd1r+xz1Ovq8ck7P4kXvS/i+ZvPZ7zP8zxCAydhOHYMpHb1+0qQJLRdXZsmJUQ1sqhqKCaMtBE0SVc1KXFu7hxCyZDsvCOSINFmbiuaUqKrbjciZ16E8US/rBBoMbT5VrRw1EiJGjIQ9M+AJwhwnF5RVVYxQVnM4JYFUqLSSgmnoXjWDRFGtbEs9g0uHkfi2jVpQCue06Q2wdjfD8bjQWIs+ypxwuMBKEpqUNjoc69Pyx/xb908CREqUoXjruM4NXUqq1RuJiKsJitSSqy0dNyKYZehgQEQanXaao4IgqJg7O1F+NSptCwTUSmhdEC7quTpgJ7WF66UcLvBM0zRJ79y0bGSG/DYz4YRSaTw5Ye6FdeyvWufE0c67XjyuSvwhTNredcjFE/iiadH0OMy4/891J51m0II2bWT7Y/c24E7GoyY8wuTuDZ9jyIbF2k2gzQYSkJKsDyLM9Mbtw+VCmv/Tno1BbWKlJQSUSYFbyAuNW+IUNXVgXa5NpUrMewbRqO+EfW6+pzbkASJRw8/ilAyAJ3r/4OPmc47WJ9aimEpmsSeFkHqL/qtlYYD7lkJu0ykykMOzgWF62Qj+8ZCdAF/9cZf4ZjzGN7a9ta09xwmDUKJFFLLy6BWlBKlypMA1pASUeX3OmFiOwj9kcPScUSs2gAyJ7dWjXVDpUSAEcikrWrTzAe32Y04G8dcZG7Dbb9+9usgCAKP3POI4vO8f+f7scu2C3/66p9mPMsSV64g5Z2RAqPXQtvTg/jly5sKuxQX/kqhlCAIQlhcrGJl6ODkIDSUBoebD8veR2zg2AziqTiuL1/HjrgFfDQK0wbWDREmtQkHmg7Ibm3ZSqiREjVkYGlJmGjxvF62hLjYIM1mcNEobGorYqlYxepvZsIzaDYWL+RSRLmUEokrVwCWlRK2ReuFiTZt2IjAeMZBYb1BVQAAIABJREFUt7hAqFeJqQ2VEnNz0sNRDLncbc+Uhm8l9LX2YTmxjDcX3sx4zxsWJkxKlBKifWM+dmspJXieR/jkAAxHjkje5/UwnjgBLhBA9Nw56TXxOylWzsqFREp0dECv0oPhmIJyOsTMFPF45UaTWQu9moI/wuCjxztwR6NyGwtBEPjyQ92IJFL4ys9HMemP5v158rkrWAgn8PhDPaBytHsUQsiunWzTFIkvP9SDREwgmw43yV85FH+nYteCAsCe+j2waW0Vs3CwoRC4UEjKzCAIAnV6WsqU8Kxr3lgLbU/PpmpBR3wjsux0u2y78Os7fx0q41UAwB3Ggzm/S6euCeSqmAtRqN96r8uKJMvj6uzmyXqO4ze8BkZnhNX9jZQST732FBiWydqiINpbU4EgCJMJs8F41SolmBs3kJyYyDrx0e7eBZBkzrDLhdgCpkJTOX/ELIRbUimxQrK9Pv963r/B02NPY3ByEB/f93E0GZoUn0dFqvDFQ1/EfHQe33nzO2nvhQcGAILIWhWp7ekBn0ggceNGIb8egFVrRSkqQQFUfHExH3iex8DkAA41H0pTQW0Et9mNqdDUprLBrixdAcuzaL8aAKHTQX9YPinS11Idoc3FRvYo5RpuaywvCYywirbmrKMrNSiLsOpSB+EhPBuZRae1s6yfIcWlMBedK4lSwkAbynKTjq3I+kQ/8lr/Pu2qg2bXLoQHBmD/aGYgEzM+Do27I+01I23MmYVBO50AyyI1Pw/a6cSbC2+izdS25Qcqx5zHoCJVGJwczGgOEMMAlQxCbFobKIK65ZQSiWvXkJyagv1jH8u5jeHYMRA0LZAXBw8CWM3WUPo9SXg8AEFA3d4Gw5hwn4gmo4pX60R7EuPxAMePK9q3GCAIAnc0GLEQSuD3TtxR8HHuaDThN4934jtDN/DTN6Y33P7/OdiGO9vqsr7H8zwmghM41HxI9vl5nkdyZgbG3l7ptSPb7Ohp2IarqZfx4B33yz6WiFKQEhRJ4b6W+/D8zedLFu6WD+LvI5I3gGDhEO0bYwsrpER9Jkmn7elG6LnnkPL5oLIrm0CEmTDGg+N4Z+c7ZW3/yTs/iR+PPg0mYcB///ZlALkbXtQqEjubBDLNoXNAr9IrbiwQ7R8nL89jT8vmVtw/9aM38POLM7K2bbLkJiWu+K/gGc8z+O29v412c6aiyG7UgGZT4ALLiOuNYEN8eZQSBWRKiLa5bBNbUqeDZts2xLIoJRx6B16aeQnv+Ok7NjxHnTb7/WQrQxx3fv705zfcdptlGx7e/XDB59rfsF9owBn5Jzzc9bBknQudHIBu716o6jMVTmIGQfzSJWh37izovKJ9o1T3QpuuekmJG8s3MB2elhVKuhZuixssz2IqNIUOS8fGO2SBuHDnGrwM473HQGrk2+V7W3vxp6/+KQanBvFwV+HfuWpDjZSoIQOBkLCCq9GUd7C2FpRFmKDso4WL/Yz3TNlJiYXoAlieLZlSYjJUeoYzfmkYlN0OVZMwaRZJCdEfbOzvg++730NqaQmqutUBBc9xYG7ezJDh51N4rK0F5RrteGXmFbxr27uK/juVG0a1Efc03oOByQF8+u5Pp703E55Bva5edqAVIMij7Tr7LZcpER4YBJB90CuCMhqgP3QIoYGTaPjsIyAIAi96X4RVY1X8YGc846Cbm0FqtVLNbiQZUUxKUDYbSLNZIDkqhG++fz8okoBBs7lH8h++bQf2tVgQYfJL4LU0ifu7GnO+f3XpKhZiCzjYdFD2uVm/H3w8njbZBoDv/eon8Ny1o9jXtE32sUTQLieib7yheL+N8D+6/weevvE0vvn6N/H4sceLfvx8EHN3MkiJFaWESEqsz5QAAMPRo1j4828gfOo0rL/6bkXnHfWPAoDsjB+T2oS/7P07XJ4Jw3Z3fmLebddDSwttDgRBoN3crjgErtWmx/1djfjO0A3897tbClYcPD8yh59fnMEHD7dhf2v+SXKDSYP6PLlZv5z4JQgQ+MDuD2R9325UY8/iDRCpFJa3dQPnUSalhHJSIjwwAG13N+im7AS6tqdHsNbxfJoi5Pfu/D1Z9wGHzlGSTIJKo15Xj+/e/11ZiwjHXMdAU5tTF39w9wfx02s/xdDUEN67471Izs0jfvEiHL//+1m3V7e3gzQaEbt0CdZf+7WCzikSBmLbSrFh19oxtrz52tJSQGxikpsnIWJtLWihpMTw4jBsKgssNxZg/Ig864aIVlMrvnT4SzjSnGmV3cqokRI1ZGA5JNx89YbcvtNSgzILpIST0WG7dTuGJofKzgaKq+AlUUqoy9O+Eb90CdqeVY96mAlDp9JJPnHTiRPw/e13EDl9GpZ3rRIIqZkZ8PF4WvMGICg8EmwCSTaZ8fBdG0z3WlMUsVRM8Y2+WtHX2oevnf0axgPjkjwZWKkDLeD70aBrwGJssYifsPIInzwJbU8P6Mb8wYjGE/2Y+/LjYDweUO42nJo6hb7WPlAkpeh8jMcjqRxE2WUhNi+CIISwS8+44n2LhfX5AYWCpki8Y8/mSdSByQEQIHC8Rb5yRFIAuNKvB7vehA/su6+gz0E7neCCQbDhMChjcf5GgLD6+XD3w/jBpR/gPXe8B3c23Fm0Y2+ErEoJA40rswJh7FkMw2nRQqfOvB60XV1QNTYifPKkYlJCTsjletzX0YX7Chhvuy1uXFi4oHi/xx7swlu/MYQv/8cwvvvw3Yr3jzEs/vhnw9jRaMRjD3aD3mQLxuDkIPY59uVcQa43aHBodhicWoMJdxdw/nJJlRKEWg2oVIpJiZTPh9j586j/xCdybqPt6UbgX/8VqdlZ0M2r95BGQyMe2v5QwZ/5VkC+5pliY7t1O1xGFwYnB/HeHe9FeGgQgPDczAaCJKHt7s5qvZELX9wHi8ZS9EB3EXat0KK3nvCqBgxMDqDb3q24Ya8YtaDDvmHcEbeAIPww9vVuvMM6vG/n+wo+d7WililRQwYCUYE1NVqU++KKBXKFlGCDQfS19uG1udfKnpYu5gWUSilR6qBLLhZD4sYN6Lp7pNdCyVCad1/b3Q3KUZ/RiJBYmaCtbd4AVhUW2dQStFP4OyW9XgxNDkGv0itaaa1miOTK0FR6sJA34i3o++HQO26pTInU4iJiFy7kHDithehpDp88ifPz5xFkgorJK57n00gJg2pFKZEqLKdF4+6Q2mZqAIYmh7DHsSdvIOJ6ZFMAbBalqAUV8fG9gvf7iZefQIorPCROKZJeLwi1GtQa+0WdXo1l0b6xGMlJUhEEAWN/H8JnzoBLbBxouhYjvhE0G5rLYlfpMHfAG/Yinoor2q+lTo9PnbgDzw3PYeCK8vvjXw9cx/RyDI8/1LNpQmI2MotR/2jee5PNQOPw7Aj8u+/EVERoZyilUoIgCJAGg2JSIjw4BPA8THnuzzop7LLwyW0NmwdBEOhr7cMrM68gmowifHIAtMsFzR25rX3anm4krlwBzzA5t8kHf9xfsjwJQLCFJNgEoqnKZMPlwmJsERcXLha0eGZWm2HT2gomJaLJKMYCY+i4HoZu/37FdrxbFTVSooYMBFfqn6z24isE5ELMlGADQfS29FYkLb2QZgW5MNAGxNn4pkJyNkJ89DLAcVKyNiDYN0zq1SA9giRh6utH5PTptAfaah2oO+NzA6vhhGtBarWg7HYw01MYnBzEUedRqKnKtLcUG06jEzvqdqRVMHE8h9nIbGFKCX3DLZUpER5aGfTKSI+mm5uh2b0boYFBDE4OgiZpxStRqfkFcNGo9P1ca98oBOoON1JzcwXJom81zEfnccl3CX0tfYr2y6YA2CxW1VcbZ2QohZ7W47P3fBZXl67iR5d/VPTj50LS6wXd3AyCXB1+CZkSDDiOx9hCJGvIpQhTfz/4aBTRs2cVnXfYN1y2JiS3xQ0ePCZC8itqRXzseCc6HQY89u/DiCflN3HcWAjje6fG8J47XTjUufkBvphs39+a+56mGr+Bhtgybu68E9PLMdQb1ZKNpVQgDfoCSIkBqJqaoNmdO3Ras3MnoFJJOVQ1VA59rX1IsAm8dPMUIi+9BOOJE3kVBrqeHvAMg8T16wWdzxfzlZSstOls0nmqCaenToMHX7Ci1212K7apibi6dBUcz6Ht4gKMMls3bgfUSIkaMrAUD4JOAQ5bBe0bFlEpEVhNS1/xfpUL3rAXNq0NOlXxVz5EtUIpW0Xi60IuAYGUWN83bzzRDy4SQeTVV6XXGI8HpMEAlcORtq2JFgiNfA0co4HrmI/N3zLWDRF9rX04P39e6mxfjC0iySUVNW+IcOgcWE4sg2ELW9moNoRODkDV3AzNrl2ytjf19yP2xhsYuPlLHGw6KJEKciE2ZYjNGeL+hV5P6pVAV+amsnC+WxGiGkjp9Zv0ekEaDJLKrRhQrbGElQJvaXsLjrmO4a/P/3XZMl6SXm+GxaXOoAbHCxPrcCKFzix5EiL0hw+D0OkQOnlS9jkDiQAmQhOy8yQ2CzEUspABu1pF4vGHejDhj+JvB+U1CvA8j8f+fRgamsTn/1tx2p4GpwbRamrN6xcPnTwJDgRG2/cKdaAlVEmIoBQqJbhEAuEXzsDY35d3YktqtdBs374pG0ANxcGBhgMw0kb88vxPwCcSeRUuwOoYr1BCqdSBv+Kxqy3scnByEE2GJuysKywg1G0pvBZ02CdcZ52z+RVMtxtqpEQNGVhmgtDFkTcAqtQQMyXYQAAUSaG3pRcvTL1QUmXBenjD3pKoJIBVG0QpcyXiw5egcjjSPP5hJpymlAAAw5EjILRahNdYOJjxcag7OjIGMQa1QTpONtBOJ15ST4AkSEV+9K2AvpY+sDyL09OnARRWBypC9C8uxLa+WoKLxxF58UWYNhj0roWxvx9eK4eJyFRB5NWqkic9U6JwpYRwnEqGXVYLhiaH4DK6sN26XdF+Yh1oMT3Dqvp6EGp1yUgJgiDwhYNfQJJN4qnXnirJOdbj/2/vzsPjKsvGj3+f2TKTfd/TpqWllKR0h0IpTVFeN9Si+IKi4Au+LggiKoIIVFZBUEFRQRSXuuD2Iig/FKVJy1ZKC1ialkLbtE2arUmzTJKZzPb8/phMmjSZyUxmS8L9ua5ekJkz5zydPjM55z73c9/u5ubhYEtATqp/LfeOQ11A6BojhpQU0s9eTV9tHVrrsI4ZKHIZST2JaASKwEXagSNg9bx8Pri4lJ9s3s+hzok/00+90cLz+zq47j0LKMiI/rxlwD3Ayy0vU1MR+jutb1MtjcVzOIyNpi4H5TnhtxScLENqZEGJga1b0Q4HGeeeO+G21uoqnLt2hT2vRHyYjWbOLjub53peg/Q0UpcvD719RQWGzMxJB5TiHZQILA3pdE6dTAmnx8lLLS9RUx7+ecuJKjMrOeY8Rq+rN+LX1nfUk+syU5RTgeWkyItAz1QSlBBj9PoGsLoMSQ1KKLMZlZqKr8f/YV9bsRa7286rba8mbAwt/S2TuuAMRyBTIp51JRz19aOWboB/2UUg2yHAYLWSdtZZ9NXWDp+MDB5sGFPkEo6PO1SmxLZCO4vzgxcHm66q8qvIt+UP30mOZnlPQao/A2UmLOHoHzrpTV838UlvgLXqVF5d6p+Ha8sjL/DkamhAWa2YivwdJKJevjF7FiiV1GKXU8GAe4CtLVsnvBgbTyAoEUvKYMBcUhK3oATArMxZXLHoCp5ueJqtLVvjdhzw37X2dnSMeZ9y0vzL3LYPBSXG67wxUnrNOjytrQzu2RPWcQOt5xK1fCPVnEphamFUReBu+sBCLEYDG56sD3mR3Dfo4fa/76a6LJNLzhjbtnMyXmx+EbfPHXIJk7utHeeuXRw+eRlH7YP+TIk4FrkMiLSmhL22FkNqKqlnTNze11Zdjbe7G08cP28iPGvL19JtGuTIe0/zFzgNQSmFrbpqUvVA3F43va7e4SUW8TAVMyW2tW6Luhj7yA4ckao/+gZzGt1knLtuyhX/TCYJSogx+hgkxWUiLz259QCMWVl4e/1BiTNLzsRisFDXWJeQY2ut/UGJOHTegOgvoibi6+/Htf8A1urRJ6En1pQIyDh3He7mZgbfegufw4GnuWVMPQk4Pu5gGR5dJek0FME5eSuj/jtMNQZl8GfsHHket9cdVaZEgc0flJgJbUH7NgVOesMvaqoMBl5dlEplu6LIEvn670DQLLAuf3j5xiQLaRmsVsylpe/4YpdbW7Yy6B2c1InaeMsSYsFcVhrXoATA5dWXU55ezl0v34XbG79svGB1N3JS/b9rXz3URYrJMOEygPSataDUmALFwdR31FOWXhZxu9xozMmcM+n11gBFmVauPe9k6vYe5Z/1bUG3e+Dfb9FuH+T2D1djNMTm5L6usY4MSwZLi4J3ZemrqwOgY/EZHOjox+XxJWT5hiEtDd9AeOcNWmv6NtWStno1hgkubAGsQ0WxHbKEI+lWdudi8GleXRTe0kZrVTXOt97CF2Gxy65BfyA03oUuAY45pk5Qoq6xjlRTKiuLJ3+uOtkOHP3ufhrsh5jb7I3oZs47gQQlxBj9Bhdml5m8JGZKgH8JRyAokWpOZVXpKmobaxOSWtjp7GTQOxiXzhswIlMiTss3nHv2gNaj6klorcetKQGQvtZ/t7pv06bhdfUpc8aupZ0oU+LlbH+by9UqeKXo6aymooZ+dz+vtL1CS38LmZbMiOshwIhMiWm+fEP7fPTV1ZF29tlhnfQGdDm72G07xoq3vAy8HFnBPgBXw8FRQTOLwYJJmaIK8lnmSAeOzU2byTBnsLwodLrwibx2O77e3phnSoC/rkS8gxJWk5VvnPENGnoa+NXuX8XtOMGDEv7lGwc6+pmTn4ZhgotrU14etiVL6KsNMyiRwCKXAZVZlTT0NkT1+/qyM2dzSnEGt/2tngHX2A4pb7b28ugLB7l4ZQVLZ+VEM9xhXp+XLU1bWFO2JmSLxL5NmzCXl2OaexIuT/w7bwQY0tLwhpkp4azfjae9nfQwlm4ApCw4Gczm4XpUInnUlm0sbIKXLOEVi7VWVYHbzeDetyI6TqD4ZDyDEmajmQxLxpRZvuHTPjY3bmZ12eqoirGXZ5RjUqaIg69vHnsTjWZet43U5csmffyZSIISYox+swejx0JeWpIzJTIz8fZ0D/+8tnwtR/qOsL87vOJX0Wjp86fmxy1TwhLfTIlAwSPbiKDEoHcQt89NpmVsITpTQQHWxadhr60bs15/pIkyJV7wvU3xMU1p58xck3pGyRlYjVY2N26mua950st7slOyMRlM0z5T4vhJb2SFmp478hw+NCsOm+mrDb9gH4DP5cLd1DQqaKaUItWcGl1QorIS18GD79j11CNP1CLtVx+PzhsB5tJSvEc7Im6BGalzys/hXbPexU93/nQ4CyrW3Ef8XUTMpWWjHs8Z8bs2VOeNkdLXrcNZX4+7tTXkdt3Obo70HUlYkcuAysxK7C57VCnbJqOBO9ZX09zj5AfPju4soLXmlr/Wk2k18fX3hFdgNxxvdLxB12BXyGwh38AA/Vu3kr5uHXkZ1uHHE7d8I7yMsL7aWjAYSF97Tnj7tliwzp+Ps16CEsnWt2kTZzrL2dd7gCZ704TbW4dbukb2bxf4fMZz+Qb4gx5TZfnGns49MSnGbjaYKc8ojzhTov7oGwAsOukslDmy37UznQQlxBj9Fh8mrzXura0mYsjKHK4pAcfXnieiC0dz/+RT88MR90yJXfWYiotHdc8IHCtw7BNlrDsX586d9A+1mrPMHrs+12ayYVCGcWth9Lv72W6vZ8XbGk9zSyz+GlOOzWRjVekq6hrraOlvmXQhVIMyUGAroMPREeMRJlZf7aahk97I6kLUNdZRaCukev5q7BEU7ANwNzaCzzcmaJZmTosyU6IS38AAnvbpnb0yWbs6dtHp7Jzc0o0j8Q1KQPw6cIx0/crrAbhn2z1x2b+7uRmMRszFRaMez0gxYRrKjpibH7zI5ajXDAUCA8sIgkl0PYmA4Q4cUdSVAFhRmcvHlpfzs+cOsK/9eCvq/3v1CNsOHuOG950yKqgTrdrGWkzKxOqy1UG36X/ppeGuCPkjlrkmsqZEON+Z9tpN2JYswZQb/gWntboaR/3ud2xwdipwNTUx+Pbb1Mx/D3C8I1Io5rJSjNnZEXfgGA5KxLkGWK41d8oEJeqa6vzF2MuiL8Y+O3N2xN9xO/e9SF6vZtY57436+DONBCXEKB63C0cKWIh/FemJjKwpAVCUVkRVXlVC6koEMiXivnwjToUunfX1Y+pJBCoEj1dTAhjuldzz+F8xlZRgSB07B5RSQS/+Xmp+CbfPzcoj1oRcQCRLTXkNzf3N7O/eH1XQqiC1YNpnSthr67AtXYopJ/zUaZfXxQtHXmBtxVqy1p2Lp6WFwTffDP/1gUyeEwqxppnTomqxG8i8eKcu4ahrrMOojJxddnbErx3OlCgrm2DLyCUyKFGSXsJnT/ssmxo3saVpS8z3725uxlRUiDKZRj2ulCJ7qK5EuJkSlpNOwjxrFvYJlnDsPuYPSizMi02rzHAF1ltPtgPHSDe87xTSUkzc/Fd/0cueATfffnoPy2Zl87HlFVHvf6S6xjqWFy8fN6MwwF5biyEjg9QVK8hL8y9zzbCayLTG/66nITUVPB70BLUD3K2tDO7eE3G7QWt1Fb6eHtxNE9+dF/ER6IS2cN1HmJs1l9rGiZdpKaWwVlfjrN8d0bESFZTIs+VNmZoSdY11LClYQo41+iVflZmVHO49jE/7wn5NfWc9c9oU6WtmVoe6WJCghBil+5j/xM9iHP/CNZGMmaODEuDvwrHz6M7hdXDx0tzfTLo5PeSJSTRsJhsKFZdMCW9fH66GhlFLN+B4ACRYUCLl5PmYy8rQTieWyuBVzNPN6eOOu7axlkxLJotU+YwOSqyt8GcFaHRULWMLbYXTuvuGu7mZwT2Rn/S+0voKA54Baipq/BkWSmHfFP4SjsEgy4uiXr4RCEocfGcGJWoba1lWtGxSxRDdzc0oiwVjXuzXJQeWOiTqO+WyUy9jTtYcvv3yt3F6nDHdd6gOJYG6EhN13ghQSpGxroaBl7biGwgejKvvqGdWxqy4/S4LpjStFIvBElWxy4C89BSue88CXjrQyZP/aea+Z/ZyrN/F7eurJ6y/EYnDvYc50HMgZNcN7fPRV1tH+pqzUWbzcEHwRNSTAH+mBDBhB45AvZFw60kEBOpQSV2J5LHXbsJy0klYZs+mpqKGHa07sLvsE77OWlXF4Ntv43OG/73V6ezEbDAHzaCNlamSKdHS18Kbx96MeulGQGVWJYPeweFubBPpc/XRZOrlFGMpxszEfidPBxKUEKN0d/pP/GwJPoEZjzErE+1wjKomvK5iHRodl7tYI7X0tcQtSwL8J5Tp5vS41JQIRMrHtAMd+qUWLCihlBo+gRmvyGVAuiV9TIaH1+fluabnWFO+BmtJ2YwOSuTb8jkt/zQguuU9BakFtDumb6ZE4A5tpNWj6xrrsJlsnF58Oqb8fGyLFw/fGQqHq+EgxoJ8jOmjT6LSTGn0eyb/eTIVFaFstndkpkSTvYl93ftCXoyF4m5uxlxSMtwNJZbMRYVgMCTsO8VsNHPTGTfR1NfEo7sejem+QwclApkS4V8cpK87F+1y0f/ii0G3SUaRSwCjwciszFk09Mbm8/Tx02exuDyLDU/W85uXD3HpmZVUlca2m0ggCzMQeB6Pc+dOvJ2dw997gdbp5QlYugEjghIhAlEA9k21mGfPGrc2VCjW+fNRZnPEywBEbHjtdgZe2T4c7K+pqMGjPbxw5IUJX2utrgKPh8G9e8M+3jHHMfJseXFvS5lrzaV7sBuPb2zB2kQKLIWJWVAiwragO/fUAXDa3LNicvyZxjTxJuKdpLvH33orNc5Fb8JhGIoi+np6MAzVRliQs4DitGLqGuu4YP4FcTt2c39z3IpcBox3cR8LgTsc1hMyJQJLBTLMwbNgMtbV0LVxI5bKEEGJcYIpOzt2DhcHM5duZ2D79skOfwzn3rc48qUv4e3pCb2hyUTR9deT9cHzY3bsYNZWrGVnx86o5khhaiF2lx2Hx4HNlJgT2ljRWmN/+h9YZs8mZW74J71aa+qa6jiz5EysJn+BuPR16zj6/e/jOnRo3DomJ3I1NJAyzvxMM6dF1c1EGQxYKiuHMzEmMrhvH803fpPim2/Gtqh64hcEobWmdcO3sD/zzITbWk9dSPlDD0XU6SQcgRO1dRWRZb0ExKsdKIAymzEVFeFJYKDz9JLTed+c9/HzN37O+XPPZ1bmrKj3qT0ePG3tQYMSuWkW8tMtZNnCXwKQunwZhsxM7JtqyXj3u8c8f8x5jJb+Fi5ZeMmkxx2NysxK9nXvm3jDMBgNitvXV/PhH71AfnoKX/mvk2Oy35E2N21mXvY8KjKCLwmx19aB0Uj6Of7U65xUMwaVwEyJdH9QouGCj6CMwet+eXt6yL3ssogvNpXFQsopp+CUtqBJYf/Xv8HjGV5Oe1r+aeSk5FDbWMt754SuQWCrDrR03YVt8eKwjtfU1xT3pRvgD0poNN2D3eTb8uN+vGDqGuuozKxkTlZkwbpgRrYFDVWHJuClVx4HMyw7+6MxOf5MI0EJMUpXj//CNSu9MMkj8deUAPD29g4XbFRKsbZ8LU/uf5JB7yApxvi0LW3pa2FZYXxb9URbmC8YZ3095tLSUcWtBr2DPPLGI1RmVjI7K/iFX+oZZ1B4/fVkhriwTzOn0eXsGvXYcHGw0tW4Spvx2e147XaMGdEtA9I+H60bNuDt6SHz/e8Pue3Ajh203nYbaWeuwpQf3196Fy24CIvBEtU67cAv5o6BDioyY7suOt7s//oXA9u3U3TjNyJ63d6uvbT2t3Ll4iuHH8tav56Ohx+m7dt3U/HQTybch+vgQTLOO2/M49Eu3wCwVM4Oa02u1prWb92Kc+dOWjbcwpw//SnkBUJmL5zuAAAgAElEQVQofc8+S/cf/0j6unWYS4JnZ/kcDnoef5xjP/85+V/4wqSOFUxtYy1zs+ZOeh66m5tJqYms2GkkzKWlw8U0E+W6FdexpWkLd227i5+86ydR30n0tLWB1xs0KPGFmpNot5dHtE9lNpO+Zg19dXVor3fMHAwUuTw179TJDTpKlVmV1DXW4fa5I+7oMp7TyrN58OPLKM+xxbx+Q89gDzvadvA/1f8Tcru+TZtIXb58+PzEZDRw38cWx6wl6UTSVq0i94rL0Y4JUvRNRnIv/dSkjmGtrqL370+hfb64ZD+J8Xn7+jl6//2knLpwOKhgNBg5p/wcNjVumvBzZCouxpiXF3ZdiRePvMiOth1cvfTqmIw/lEDgo9PRmbSgRL+7n22t2/jEKZ+I2T7zrHlkmDPCypRoPryHP+htLD+WTdFJk7+RMZNJUEKM0tbtD0rkZBdNsGX8GTOHghI9o+tKrKtYxx/2/oGXW17mnPLwWl1Fwu6yY3fb49Z5IyBYbYZoOep3jcmSePSNR2m0N/LIfz0S8peaMhrJ+59Ph9x/ujl9TIuqzY2bWVG8ggxLBr1lxwvTGRcsmNxfYkjP44/jeP11Su68k+yPfiTktoMHGjjw4Q/Tfu99lN5zd1THnUhWShafrv50VPsotPkDf+2O9mkVlPD199P27btJWbCAnE9E9su9rrEOhWJN+fECT+aiQgquuor273wH+6ZNZIRYA+3t7sbb1TWmyCVAqin6oETKnDnY//kMPpcrZDZC75NPMrB9O+nvfhd9/36Wrt8/Ru4nI78b7RsYoPWuu0iZP5/yHzwwYXsw38AAHQ89TOYHP4ilPLIL2GDsLjs7WndwadWlk3q9b3AQb0dHXDpvBJhLS3Hs2BG3/Y+nILWAq5ZcxT2v3MOzh5/l3bPHZiJE4njb1PGLgS6uyJ7UftPPXUfvU0/h2LmT1KVLRz1X3+G/270wN7FFLgNmZ87Goz0csR8ZvqMYrQ+cFp9llc8feR6v9oZM6w50RSi84fpRj39kWWw+i+EwZmZSdN11cT2GraqK7t8/hvvw4XG/a0V8dPzoR3iOHqX8hz8YFWCsqajhif1P8Hr766wsXhn09UoprFWnhlUPxOV1cde2u5idOZvLqi6LyfhDybP56w0ls67Ei80v4va5Y7Z0A/zvebgdOL79+DV40uHm934nZsefaSQEKkbptPtbFBbkx/eCPBzGLP/yDW9P96jHVxavJNWUGrcuHIEe9fGsKQGQZol9poS3pwf3ocOj6kk09jbyszd+xvsq38eqklVRHyPNnDYqmHKo95C/ONjQF/1wtfwo72x6urpov/c+bMuWkXXB+gm3T5k7h7zLL6fniScYeOWVqI6dCAWp/uyf6VbssuMnP8HT0kLxhg1jughMpK6xjkUFi8bcKcn91CdJmT+PtjvuxOdwBH398SKXlWOeC3TfiKaVnWXOHPD5cB8+HHQbb28vbd+5F+vi0yj/wQ9IO+ssjj7wAJ6jkf87djz0MJ7mFoq/tSGsfuVF37gBjEba7rwr4mMF88KRF/Boz6RP1I5fbMc3KOFua0N7Erse+eJTLmZBzgLu3nZ3VJ1dIH7vU/qaNWAyjVuXpb6znsrMStIt8S1iF0xgvXUsOnDE2+bGzeRac1mUvyjoNoH3OGPd5JY5TRfW4WUAsoQjUZx73+LYr39N9oUXjll6cVbpWZgN5rC6cNiqqxncty/k71GAX+z6BYd6D3Hj6TfGLeN4pECmRDKDEnWNdWRaMllSuCSm+63MqpwwKLH52V9Ql93Cxz3LOGnhmTE9/kwiQQkxSpfDHwAoLk7+ndtAZVrfCR04LEYLq8tWs7lxc1x6aQeq6Ma9pkQcMiWcuwNFLv2ZElpr7tp2F2ajma+t/FpMjnFiTYnh4mDl/vTtWLXwO/r9+/Ha7RRvuCXsFNL8z38Oc2kprbfdhna7ozp+vBWm+jMloqmDkGiD+/bR+ctfkfWRj5C6bOnELxihfaCd+s76cesWKLOZ4ltuwd3cTMdDDwfdh6vhIDB+IdY0cxoe7cHtm/y/e6CWSqi6EkfvfwBvVxclGzagDAaKbr4J7XTSdu+9ER1r8MABOn/xC7LWryd1+fKwXmMuLqbgi1+kr7Y2oo4lodQ21pKTkjNcvDVSiQpK4PXiaU9sYViTwcRNq26ibaCNh3Y+FNW+jr9PsQ12GzMzSV2xAnvt2PlQ31lPVX7ii1wGBNZth3MXMZncXjfPH3meteVrMajgv2tGdkWYyVJOOgmVkiIdOBJEa03r7bdhzMig4CvXjnk+1ZzKGSVnUNdYN+E5r7W6Gnw+nHuCt9lusjfxyBuPcN7s8zirLDEFF5MdlPD6vGxp2sI55edgMsR2kUBlZiWt/a1BA9cuRz937/4BRXYjX7zkgZgee6aRoIQYpcfVi8UFJXlToNBl1vjLN8CfztbuaB/uwR5LgUyJRCzf6HfFNlPCWe+/sxFoB7rp8CaeP/I8Vy6+cvgiOFppljQcHsdwFeXNTZuZnzOf8gx/CqsxLw+VkhJVUMKxcyfdf/oTuZ+8BGsES0AMNhtF37yRwbf3cWzjbyZ9/ETItGRiMVimTaaE1prW227HkJZG4de+GvHrh6teB+nwkLpyJVkf/hCdjz7K4IHxgwKuhgYwmzGPs3Qh1ZwKEGVb0Mqh4xwc93lHfT1djz1Gzsc/jvVU/zr9lDlzyL3icnqf/Bv927aFdZzh99Jmo/C6yIKFuZd+KqysknC4fW6eO/Ic55Sfg9EwuZoYEy1LiIVYBTonY0nhEtbPW8/G+o3s794/6f24m5sx5uVhsFpjODq/jHPX4dq3H9eIDJ8ORwftA+1J6bwRkJWSRU5KDg09U7ujzY72Hdjd9pDZQid2RZjJlNmM9ZRThs8nRHz1PPEEju07KPjqVzDljF+bpKa8hkZ744SfJWuVP8sl1L/d3dvuxqAMfH3l1yc/6AhlWjIxKROdjs6EHXOk/xz9D92D3TFduhEQWJp22D5+huXDv/sKTZkevjb3M6SmT26Z3juFBCXEKP3eflIH1XCbq2QKZEp4e8cGJdaUrcGgDHFZwtHS34LFYIl7ReI0cxp298S9pyPh2FWPubwcY3Y2A+4B7n7lbk7OOZlPLIxdYZ9AP+t+dz89gz282vbqqAtNpRTmkpJJX0Bor5fWb92KKT+f/KsjL8CUfu65pK9dS8eDD+JubZ3UGBJBKTWt2oL2/v0pBrZto/Daa0cVUQ1XXWMdZellnJR9UtBtCq+7DoPVSuvtt417R8h1sAFLRcW4y0bSzP6q9NEEJYzp6ZgKCsZtC6p9PlpvvQ1jbi4F13xp1HP5n4ssQ6f3//0/BrZupeDL12DKy4tojMpspujmm/1ZJQ8HzyoJx+vtr2N3hb4Ym4i7uRkMBn/rzjgxlyUvKAFw7fJrSTWncsfWOyadnec+ErwdaLQClfr7ao+ndye7yGVAOKnNyba5cTMWgyXk8sb+554b1RVhprNWVeGsr0f7fMkeyozm7enxL1NdvJjsjwbvyBBoUzvREg5zUSHGgvygWS61h2vZ3LSZKxdfSXFa8eQHHiGlFLnW3KRlStQ11mEy+Iuxx1qotqCH97/Or/SLnNmVx3vff1XMjz3TSFBCjNKPA6vLEFFbsnhRRiOG9PRxW0HmWHNYUrCEzY2bY37c5r5mStJLQqZxxkK6OR2Hx4HX543ZPp27dg2vB31458O09rdy06qbYpquFghK9Ln7eO7Ic+MWBzOXlk76AqLrscdw7t5N4Q3XY0yPfC20Uoqim76J9nppu/ueSY0hUQpTC6dFpoTXbqftnnuwLlpE9scujPj1A+4BtjZvZV3FupBdDEz5+RR8+RoGXtqK/emnxzzvOnjQX/dhHLEISgBYKitxHTw45vHuP/0Z586dFH39uuGAaYDBZqPopptw7dvPsV//OuT+vX19tN99D9aqKnIuumhSY0w7/XR/VsnPg2eVhKO2sRazwcxZpZNP4fU0N2MqKgqrJsZkBbqSJCsokWvN5Zpl17C9bTtPNTw1qX24m+MXlLBUVJAyfx72EXUl6jvqUaikFbkMmJ05O6zK9Mmitaa2sZZVpauGs63GY99UizEnJ+xWi9Odtboa38DAuN+FInaOPuBfDlj8rQ0hl6kWpxWzMHfhcMZhKLaqahz1Y4MSDo+Du7fdzbzseVxyauLbBOfZ8pIXlGiqY2XRyrjU1wm0jB4v+Hrn365FA988/3sxP+5MJEEJMcqAYZAUtwmDIbr2Z7FizMzE1zs2KAH+JRx7ju2htT+2d8Nb+lsoSYtvkUtg+Mux3xObJRyeri7cTU3Yqqs40H2AX9f/mg+f9GGWFka29n8igXH3ufrY3LiZPGse1fmj2xuZyyYXlPB0dHD0/gdIXbVqwhagoVgqKsj73Gex/+Mf9D3/wqT3E28FtgLaB6Z+psTRH/wQb2cnxbfcMqnWl1tbtuLyucK6I59z8cVYTz2Vtrvvwdt3/LOhvV5chw5jqRx/PXeayR+UGPBEV5DQMmfOmEwJT1cXR7/3PVJXriTzgx8c93UZ564jfd06jv7oxyEzdDp++EM8HR3+k9BJthGF41klbXfcPqm791pr6hrrOKPkjJAXYxOJZwZAgMFmw5ibm/C2oCN9dP5Hqc6r5r5X7sPuiizDTWuNu6Ulru9T+rpzGdi+fTiIX99Zz9ysuVH928ZCZWYlnc7OiN+zRNnfvZ8jfUdCfjdpt5u+LVtIr6mJ6jM7nQTqUkldifhx7Kqn6/ePkfOJT2BdOHHwsKaihtfbX5/wwt5aXY1r/wF8/aPPLR/Z+QjN/c3ceMaNMWnRG6lkZUoc7DlIQ09DXJZuANhMNkrSSsYEJZ75x495MaeDSzmT2fOWxeXYM40EJcQoDqObFE/wVniJZsjOGremBBxPZ4v1Eo7mvua415OAEcsgYlRXIlDkMqWqijtfvhOb2ca1y8cWTYpW4I5092C3vzhYxdjiYObSUrwdHficE/RSP0H7vffhczopvuXmkHfUw5F3xRWYZ8+i7fbb8blcUe0rXgpSC6Z8oUvnnj10/fa3ZF98EbZFk+utXddYR4Y5g2VFE/9iVkYjxRtuwXP0KB0PPjj8uLu5Ge1yjVvkEmJTUwL8QQlvdzeerq7hx9q/+128/f0Tzsuib94IXi9t3x6/Ja1z716O/ea3ZP/3f2NbFLzKfzhM+fkUXHMN/S++hP0f/4j49Q09DTTaG4PW+AhXPDMARoom+yoWjAYjN626iWPOYzz42oMTv2AEb2cnenAwzkGJGvB66XvuecC/fCOZRS4DAuutp2oHjrqmOuB4oebxDLz6Gr7eXv97/A6RMncuymaTuhJxon0+Wm+7DWNe3pjlgMHUVNSg0Wxp2hJyO2t1FWiN883jxS4behr4Rf0vOH/u+SHbisZTsoISw/Ws4hSUAH/wdWRG2EB/D9/Z91PKek184ZLvx+24M40EJcQoAxYvVl/sC3FNljEza9yaEgBzMucwO3P28ElFLDg9TjqdnQnJlAhc3MeqA4dzqH1XXWYz21q38eVlXx7uDR1LgWDK5qbN9Ln7xr2oOV6YriXs/Q688go9TzxB3v/8Dylz50Y9TkNKCsU334Lr0CGOPfpo1PuLhwJbAf3u/pi3ho2V4ToK2dkUfvnLk9qHT/vY3LSZs8vODvvujG3xYrI/9jGObdyIc+9bAMPZC3FfvjGncuh4BwH/BUnPn/9C7mWXkjJ/fujXlpeT/4XPY//nP4cvDgO0z0frt27FmJlJ4bWTey9PlPPxi0k5dSFt3757VFZJOIYvxiqCX4xNRHs8uNva3hFBCYCq/Cr+e8F/89jex9jTuSfs1w0XAy2L3/tkO+00jHl59NXW0j7QzlHH0aTXkwD/72lgyha7rG2spSqvKmQh6L5Nm1BmM+mrY78efapSJhPWU06RtqBxMrwc8PqvY8zICOs1C3MXUphaOOGy5UCh80CWi9aau16+C5vRxldXRF6kOlZyrbl0Ojrj0jUvlLrGOk7OOTmuNxsDtXMCf7cf/fYa2jK83HDq1VhsaXE77kwjQQkxykCKxqpsyR7GMGNm5rg1JcBfO6CmvIZtLdtidlE33A40kZkSMRq7c9cuXPPK+W79g1TlVfHR+cGLJkUjMO6nG54mxZjCqtKxxcEirZav3W5ab7sNU2kJ+Z//XOzGevZqMt7zHjp+8hCupqaY7TdWhtuCTtG6Ej3/9384Xn+dwuuuwzjUDSdSb3S8wTHnsYjvUhRc+2WMGRn+4pFahx2UCNaWK1yBTAxXQwPa4/HPy+JiCq68MqzX515+OZbKSlrvuB3f4ODw4z2P/xXHa69R+LWvYcyOTQVuZTRSsmHDmKyScNQ11rEwd2FUxc48bW3g9SY0KJHoE9oTXb30arJTsrnj5Tvw6fCKALqPHAHi2zZVGY2k16ylb8sWdrXtBEhq542AiowKjMo4JYtddjg6eOPoG6GXbmiNvbaW1DNXYUh7Z11cWKurce7ejfbGru6VAM+xY7R/73uknn46meefH/brAue8LzS/wKB3MOh2poICTEVFwwGlfx76J1tbtnLV0qvIt+VHPf7JyrXl4vQ6cXii6xoViW5nN6+1vxbXLAnwZ0r0u/vpcHSwf+/L/N64g7VdxdS86/K4HnemmZJBCaXUlUqpBqWUUym1Qym1JtljeidwDQ4waIFUY+wLwUyWMSsTb5CaEuC/y+f2uXmx+cWYHK+lzx+USEimhCW2mRKO+l38ucZCp6OTm1fdPOkWfxMJXPx1ODpYVbIKm2lsEOt4UOJIWPs8tvE3DL69j+Ibb8SQGts10EXfuAGMRtruvCum+42FgtQCgCm5hMPT1UX7fd/Ftnw5Wes/POn91DXWYVRGVpdFdpfRlJND4de+imPHDnr++gSDDQ0YsrIwBmmZFqtMCXNZGZjNuA420PW73zP45psU3XBD2BckBouFoptvwn3oMJ0//zkA3u5u2u+7D9vSpWRdsD6q8Z3Itngx2RdeOCqrZCLHnMd4vf31qE/UjrcDTUxQQg8O4u1MTku5gKyULL6y/CvsPLqTx99+PKzXJOp9yli3Dp/dzn/qn8WgDCzIDb+dcryYjWbK0sumZLHL55qeQ6NDfg5cBw7gPnyYjHdI142RbNVVaIcD14EDyR7KjNL+3e/iC2M54HhqKmpweBxsawndftpaXY1z1y763f3cu+1eFuYu5KIFkyusHCt5Vn/mbqczcd/hw8XYo1ymOJHAMrWGngZuf/prmHxw4wUPxPWYM9GUC0oopS4CHgDuApYCLwJPK6VmJXVg7wAtLf4eu+mWqdNH15iVhS9ITQmApYVLybRkxqyuRHO//+SxLL0sJvsLZWQXi2h5jh1jn6eFJ/Mb+djJH4vrWuKR1YuDncyZiorAaAwrU8Ld2krHgw+StvYc0t/1rlgNc5i5uJiCL15JX20t9k2bYr7/aBTapm6mxNHv34/XbvcXt4yivkddYx3Li5aTlRJ5pkXWRz6CbckS2u+9F+fON7BUzg46lljVlFAmE5aKCgZe2c7RBx4gbfVqMt7zXxHtI331ajLe+146H/4prsZG2u+/H29PD8UbbglZYX2yCr5y7aiskomEczEWjkQsSwgwl5eNOmYyfeikD7GscBn3v3o/3c7uCbd3H2nGkJExpmtLrKWddRbKYmHXkVc5KfukcQPGyTA7c/aUzJSoa6yjOK2YBTnBgzeBNqvvlFagIwU6eTmkrkTMDLz6Gj1/+T//csB58yJ+/eklp2Mz2SbswmGrrsJ18CA/euUB2h3tfHPVN+N2oypcuVZ/K/FE1pXY3LSZfFt+3OvrBNqCPrL5XnbkdHO5uYbS2clfPjfdTLmgBPAV4Jda60e01nu01lcDLcAXkjyuGa+pxZ/enmkb/05kMhgys9AuV9CCiSaDiTXla9jStCUmrTWb+5oxKmPI9aWxMlxTwhV9UGJg1xv87D1GskzpfGlZeEWTJstmsqHwXxgGKw6mTCbMRUVhXUC03X0P2uul+Kaboi5uGUzupZdimXcSbXfcic+RuNTBiUzVTAnHf/5D95/+RO6nPoV1wcmT3k+jvZF93ftCFpELRRkMFG+4BW93N87du0mpHH/pBoDZYMZisMSkm41lzhwcr7+Odrkovnly87LoGzegjEaarrmG7j/8kdxPfRLrKadEPbbxnJhVMpG6xjoKUwujbheZ6EyJkcdMJqUU31z1TewuO/e/ev+E2yeqGKghNRXbmWewV7dyau7UOSGuzKrkcO/hsJe7JILT4+SllpdYW7425OfbvqmWlFMXYi6e/DKn6cpSWYlKTR2uVyWioz0eWm+9NaLlgCdKMaZwVulZ1DbWhgxAW6uqOJyv+d3bf+Cj8z/K4oLkt7LNtQ0FJRyJCUq4vW5/MfbyscXYY604rRirMYWtg29S2W3hik/cG9fjzVSmZA9gJKWUBVgO3HfCU88Ak2+kPgXt3PUqt/59cl9K8TJodEMh5GTEvjjiZAXuLDV+7vMo8/hF8hbld/LUwm4++eAaLN7ovngOpznI1UaaPxv/GNiA0QNnwc+fvYe/PTXxiW0oDq+Tt8oVty65ZlJ3pCNhUAbSzGlUZlYOX1SPx1xaSv+W5zj8mf8Nuo32ehh4aSv5V1+FpaIiHsMFQJnNFN9yC4cvvYyDl1yCKXdqzHGNJuUsAxuf+yGbnnkk2cMZ5hsYgE9aSV2yD/Wvz096Px2ODgDWVUz+LqN14UJyLrmEro0bg9aTCEgzp/GPhn/w1rHwljEE41rSgCvfgKWiDMvbd8Pbk9zPl0pwNexFLUshden+qN7LiegsjfMz2fje2IDh4Im/QkfbldXLu9sLaPzfz0Z1zMED+zHm5WGwxr84cuCi/ugPH6T7z3+J+/EmYgU+NKeQv+i/cGDbv1EhElR8pXaMC7OwxvHfP2BwdSs92kfJ069x+JfBv3sTKau4Ded8J5f+YA1G39RoNz5g8uLIdFD15/9w+GfB3yfHa6+RP8kLyOlOGY1YT11I71NP4Tp4MNnDmfa89l4G9+6l7IEHoqpPUlNRw7OHn+WKZ67AYhi/W552u3j7QiOpg3Dhrw9y+NHkfxc4LYNwBnz379fzy8H4d/lzGn30Z/VT9dddCfn7F5+mOZgFNy67DrNl6jQMmE6mVFACyAeMQNsJj7cB7z5xY6XUZ4HPAsyaNb1Wd3h9HpxGd7KHMca8llTWXfiBZA9jWOrpK0ldsQKf0wHO8e9wLx0wsiIvnV6zB1eU5zu5A0ZOb0/Haw++ZCRWLGhqjmTRnOqiT0XZstJk4MP2eayv+lhsBjeBj5/y8QnT4bIuWE/XH/844XuZ+cEPkveZz8RyeONKO/10Cq69FvumZxPy7xuu9x3KZneOI/o5EEMq3YypuIg+7wBEkYCUYkzhwpMvpCIzuoBTwZeuxtvZQca7zg253fp569nRtgO7yx7V8XzZqXg8OfiKshmMYl86PwP3YB7G7Gz6vI6o3stw+OaW4j5yBO0NPZfm9Vg5b39a1J8DU0EB6avPjmof4TJmZpJ1wQUMHtg/ZT6/F9Vn0m7s56jVBaF+92SmYMpPxx3lvAyHzraxqCGDFU0WvM6p8T6d5jGyKDcVh8Ed+n1KJC+sas2g6gh4fcHfp9QVK6KqqTPd5Vx0Mcd+s3HKfOamu9wrLifjv86Lah/nzjqXpxueps/VxyDBC14WpOTx0Z1ppHU58JL8DNEsNKtbMjlqdSfmfMcHp7els6hRhfyMx8r7D+cxeGolZ665OO7HmqlUsitZj6SUKgWOAGu11ltGPH4LcInWOujCvxUrVujt27cnYJRCCCGEEEIIIYQIl1Jqh9Z6xXjPTbWaEh347yUVnfB4EdCa+OEIIYQQQgghhBAiXqZUUEJr7QJ2ACfmNp2HvwuHEEIIIYQQQgghZoipVlMC4HvARqXUNuAF4PNAKfBQUkclhBBCCCGEEEKImJpyQQmt9R+UUnnATUAJsAt4v9b6UHJHJoQQQgghhBBCiFiackEJAK31j4EfJ3scQgghhBBCCCGEiJ8pVVNCCCGEEEIIIYQQ7xwSlBBCCCGEEEIIIURSSFBCCCGEEEIIIYQQSSFBCSGEEEIIIYQQQiSFBCWEEEIIIYQQQgiRFBKUEEIIIYQQQgghRFJIUEIIIYQQQgghhBBJIUEJIYQQQgghhBBCJIUEJYQQQgghhBBCCJEUEpQQQgghhBBCCCFEUkhQQgghhBBCCCGEEEkhQQkhhBBCCCGEEEIkhQQlhBBCCCGEEEIIkRRKa53sMcSEUuoocCjZ45iEfKAj2YMQIk5kfouZTua4mMlkfouZTOa3mOmm2hyfrbUuGO+JGROUmK6UUtu11iuSPQ4h4kHmt5jpZI6LmUzmt5jJZH6LmW46zXFZviGEEEIIIYQQQoikkKCEEEIIIYQQQgghkkKCEsn302QPQIg4kvktZjqZ42Imk/ktZjKZ32KmmzZzXGpKCCGEEEIIIYQQIikkU0IIIYQQQgghhBBJIUEJIYQQQgghhBBCJIUEJZJEKXWlUqpBKeVUSu1QSq1J9piEiJRS6htKqVeUUr1KqaNKqb8ppapP2EYppb6llGpWSjmUUnVKqapkjVmIyRqa71op9eCIx2R+i2lNKVWilPrV0He4Uym1Wym1dsTzMsfFtKWUMiqlbh9xzt2glLpDKWUasY3McTEtKKXOUUo9qZQ6MnQ+8ukTnp9wLiulcpRSG5VSPUN/NiqlshP6FxmHBCWSQCl1EfAAcBewFHgReFopNSupAxMicjXAj4GzgHMBD/BvpVTuiG2+DnwVuBpYCbQD/1JKZSR2qEJMnlJqFfBZYOcJT8n8FtPW0InoC4ACPgAsxD+X20dsJnNcTGfXA18EvgScAlwz9PM3Rmwjc1xMF+nALvzz2DHO8+HM5d8By4D3Dv1ZBmyM45jDIoUuk0Ap9TKwU2v9vyMee2OJa6IAAAbuSURBVBv4s9b6G8FfKcTUppRKB3qA9VrrvymlFNAMPKi1vnNoGxv+L8mvaa0fTt5ohQiPUioLeBX4DLAB2KW1vkrmt5julFJ3AWu11quDPC9zXExrSqm/A51a68tGPPYrIE9rfb7McTFdKaX6gKu01r8c+nnCuayUWgjsBs7WWr8wtM3ZwHPAKVrrvYn/m/hJpkSCKaUswHLgmROeegb/3WYhprMM/N8rXUM/zwGKGTHftdYOYAsy38X08VP8QePaEx6X+S2mu/XAy0qpPyil2pVSryulAgE3kDkupr/ngXVKqVMAlFKn4s/s/H9Dz8scFzNFOHP5TKAPf5Z+wAtAP0me76aJNxExlg8YgbYTHm8D3p344QgRUw8ArwMvDf1cPPTf8eZ7WaIGJcRkKaX+F5gHfHKcp2V+i+luLnAl8H3gbmAJ8MOh5x5E5riY/u7Bf8Nkt1LKi//a506t9Y+Hnpc5LmaKcOZyMXBUj1gqobXWSqn2Ea9PCglKCCFiQin1PeBs/Clh3mSPR4hoKaUW4K/9c7bW2p3s8QgRBwZg+4ilo68ppebjX3P/YPCXCTFtXARcCnwCqMcfeHtAKdWgtf55UkcmhBgmyzcSrwPwAkUnPF4EtCZ+OEJETyn1feDjwLla6wMjngrMaZnvYjo6E392W71SyqOU8gBrgSuH/r9zaDuZ32K6asG/vnikPUCg8LZ8h4vp7l7gPq31Y1rrN7TWG4HvcbzQpcxxMVOEM5dbgYIRS/QCtSgKSfJ8l6BEgmmtXcAO4LwTnjqP0et7hJgWlFIPcDwg8eYJTzfg/5I7b8T2VmANMt/F1PdXYBH+O2uBP9uBx4b+/y1kfovp7QVgwQmPnQwcGvp/+Q4X010q/puBI3k5fg0kc1zMFOHM5Zfwd/A4c8TrzgTSSPJ8l+UbyfE9YKNSahv+E4LPA6XAQ0kdlRARUkr9CPgU/mJpXUqpwHq0Pq1139A6tfuBG5VSb+K/iLsJf5Gd3yVl0EKESWvdDXSPfEwp1Q8c01rvGvpZ5reYzr4PvKiU+ibwB/xtyr8E3AjDa41ljovp7G/ADUqpBvzLN5YCXwF+DTLHxfQy1OVu3tCPBmCWUmoJ/vOSwxPNZa31HqXUP4CHlVKfHdrPw8Dfk9l5A6QlaNIopa7E30u2BH+/2Wu11luSOyohIqOUCvYFcqvW+ltD2yj8bRQ/B+QALwNfDFzUCTGdKKXqGGoJOvSzzG8xrSmlPoC/dsoC4DD+WhI/DBRCkzkupjOlVAZwO3AB/hT1FvzZbrdprZ1D28gcF9OCUqoGOLETGMCvtNafDmcuK6Vy8Bc0/tDQQ0/iby3aTRJJUEIIIYQQQgghhBBJITUlhBBCCCGEEEIIkRQSlBBCCCGEEEIIIURSSFBCCCGEEEIIIYQQSSFBCSGEEEIIIYQQQiSFBCWEEEIIIYQQQgiRFBKUEEIIIYQQQgghRFJIUEIIIYQQCaeU0kqpC8Pc9ltKqV0TbymEEEKI6UaCEkIIIYSImaFgQ6g/vxzatAT4WxKHKoQQQogpwJTsAQghhBBiRikZ8f/nA4+c8JgDQGvdmshBCSGEEGJqkkwJIYQQQsSM1ro18AfoPvExrXUPjF2+oZQqVUr9VinVqZQaUEq9rpRaN94xlFKzlFJvKqV+pZQyKaWylFIblVLtSimnUuqAUurLCfkLCyGEECIqkikhhBBCiKRSSqUBm4F2YD3QDCwOsu1C4BngT8BXtdZaKXUHsAh/ZkYbMAcoSMDQhRBCCBElCUoIIYQQItk+ARQDZ2qtO4Ye23/iRkqpM4CngO9rre8c8dRs4FWt9bahnw/Fc7BCCCGEiB1ZviGEEEKIZFsK7BwRkBhPGfBv4J4TAhIAPwEuUkr9Ryl1n1JqbbwGKoQQQojYkqCEEEIIIaaDDmArcLFSKmfkE1rrp/FnS9wH5ANPKaV+kfghCiGEECJSEpQQQgghRLK9BpymlMoPsc0g8CGgC/iXUip75JNa6w6t9Uat9aeBK4DLlFIp8RqwEEIIIWJDghJCCCGESLbf4S9y+YRSao1Saq5S6kMndt/QWjuADwI9jAhMKKVuU0qtV0rNHyqE+RHggNZ6MMF/DyGEEEJESIISQgghhEgqrXU/sBZoAv4G7AJuBfQ42zrwd9no5XhgYhC4E/gP8AKQgT94IYQQQogpTmk95ve9EEIIIYQQQgghRNxJpoQQQgghhBBCCCGSQoISQgghhBBCCCGESAoJSgghhBBCCCGEECIpJCghhBBCCCGEEEKIpJCghBBCCCGEEEIIIZJCghJCCCGEEEIIIYRICglKCCGEEEIIIYQQIikkKCGEEEIIIYQQQoikkKCEEEIIIYQQQgghkuL/A6LpQ6u0ZGw3AAAAAElFTkSuQmCC\n", + "text/plain": [ + "
" + ] + }, + "metadata": { + "needs_background": "light" + }, + "output_type": "display_data" + } + ], + "source": [ + "font = {'size': 14}\n", + "\n", + "matplotlib.rc('font', **font)\n", + "\n", + "tot_requirement = no_action_result[\"requirement\"].sum(axis=1)\n", + "no_action_shortage = no_action_result[\"shortage\"].sum(axis=1)\n", + "rule_based_shortage = rule_based_result[\"shortage\"].sum(axis=1)\n", + "searched_statistics_shortage = statistics_result[\"shortage\"].sum(axis=1)\n", + "\n", + "plt.figure(figsize=(18,8))\n", + "ticks = np.arange(tot_requirement.shape[0])\n", + "\n", + "plt.plot(ticks, tot_requirement, color=\"black\", label=f\"Requirements\")\n", + "plt.plot(ticks, no_action_shortage, color=\"tab:blue\", label=f\"No action\")\n", + "plt.plot(ticks, rule_based_shortage, color=\"tab:red\", label=f\"Rule based\")\n", + "plt.plot(ticks, searched_statistics_shortage, color=\"tab:green\", label=f\"Statistics based\")\n", + "\n", + "plt.xlabel('Ticks')\n", + "plt.ylabel('Bike amount')\n", + "plt.legend()" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.7.7" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} diff --git a/notebooks/articles/simple_bike_repositioning/DecisionLogic.png b/notebooks/articles/simple_bike_repositioning/DecisionLogic.png new file mode 100644 index 0000000000000000000000000000000000000000..6e058861dd8edbec87e894e7335f1b85f3567767 GIT binary patch literal 143047 zcmeFZ2T+q+yEYs}=?EwS1_UW89Ym0#v_u3%I!JFSn*h>5dQqe(0@4+vgkGd6(wibh zPzhZrB1BqfQbHg=_*bxQ-*>;~%=u=%`DXr;8G}!rC#&3TUDtiDm3x}%N;H&EN(cl( zqoRCK8v-E>hd@YH$w|Q{j^g4G;Lko+Z6z3__&w_^xHw>~pr!zUltoc(TOI`06i&*9 zt`NwfX5znnZH{?2AduZ`m5U0xo@VnqWKT@=&2nj3)-{+z4${I|6Iw_}YK27}B))yt zOld&+h{wRfBG8Z|mL!wneMiL;`;RV>F7LHfpWQjxd9~cCJ@V7}Mq?G)(YYS$$ z!GYhdMILYn?EmeWxs{G_-*4B{*U5oUe!W7@+*SN_1%Wg(zvBGuI?(dU%hH)IZ(_rK zdr6Y3QCiY-{(4Q&iSgc5{OyS_I4X;dk>l4BoQD)*VFq`9dqncU|FjXw2SjCkdj-B! z%^Mw*AokvD@Z&=Wgt_kSke?s1$Nzo!e<}@KEBz8Pd)Mi=_Xso*O7j}OtjOZ)VOtzl zts|W}To(7SYlwa4B|G=Z^?-HjmsZpVj?!&fofLU>nd+B1f)X`rOB}UHtqi`938+vD zUO$n?rsyw)^+8C#CLI3t9H}t!mibr~HNJGF`1~TzOb~92-K16CsV(yOxfPwX`xWqA z3=ZWcxv?`K(RgfO`u0FC&ABaYzZj5_^6JC?6F`5hfxO^zM$5Y>E(XW(F=?98qgf~TKhyJ z0>5zMh@Jn~#yRC8Io-i!+_~VKxlW(FsrQe23*X`mo5h60qRyIE?|g0(yxRh!G< zztM30>EVBw-rjIG`@fc;nrL+=_|n5d_X#OKY4ue9Df;yR`82$fLhxFaW2%r5%c zXJ7Sr5BuT++_mmTzT6J37dPdy!pJTY&Zz`=X|IOv=aLgxNz4^R)EiyBaN*5hbGvzN zqkw*~=VP%8Z`^`m(gC~qZv?uW`q_PliaX9=sJ# z_wVdNIbwZNV9ir3;F$J4`MT~;0S_L*_60my^I3WgKau;2MIz!==!U0+dTZMoPx%<( z5of%W_M>QF@)h`@emw+Kq5#~<{jsoz#YK=T8e)N&y$ z?73`w4kkRx_cSu;xMV*?#4bDc&>mX}hQ9YyKAhA#i+j-Hf<~h)wese*Czo@n)XIe_KjNyc3^c`{uA#-7- z!QRv6DD+g~USE{hxEMm_;7aiM27lo!&nRX&0Y2TRX9jc=QBi|PN2#1uoJyICUGiu- zrSAkfHleMA|My<<7`{31mIvce<4ed=WqiE>EQ-e!y@EA!L0chwdzxUU7PQc7&DoxG zlVm4aV2RfhlzZ%X+T8rB^``Ol4;_Y3=S~SmHM0JjX6YD@8eS?ZZ4Zbutn;|P>-o&r zGKfVOj*cZ|eD>kl1bR%#o<&5*UhNCk^=fD=cnWKL~=y=P^^UGaB^kLUd6@EM$) zJdBK+OY`^S?!T9r5**{gWF_l|w4SCBxKU=+a{Rt!bfcPIbQ|hd@N2T*`u)3aQ0S4h z5*9{E&LDtZwxc$(ZU?XBMl=TTWX)}Iqpj9R=mZDZnJNwFV%%6d&ZO+7 z=3Sc<`s+VUQ~5BX%3z_=F1|5f@p;;Ucu|RQ1$Bf5)IfRHPp0KgOm3?0%BymuD1M)E znbpxf@l^xy1+yJtW>4$Onp~daQ!&Vr=ND5G{{=UGt-gv0(P9ECK8H8u_~R(lmfU*Z z>$`p@?a-#aROR{2^0K<>;xa&x30vz^%|PW9!~7xG207$0g;Z%;;hMTNeR=2 z(ssF1D{4A4za;*1xuFg3`)c-9)8Ml(m+X7})!JFggQNs>mv8vNql9wgvI+@R?pUod zM$46w&%`v3DO#>UGyO}B)-hPG7LKQ3L-a<^ z3gSt!`W%HC8uul}Aq{bSb1MJtFP|0hqc$Ks2IzB2Y1r68A$*LDpb!}jZ- z8t&D#%75)hGemI{I=Y(Q_I%R#QbT&vESYf2Cvh~6t=om_Aho*wp@VA3WlFb^75qvu zi~aMdh{1UO!LOPr8~CVlenxMbI{xd3x?xg71WSziG}~-Kmik(lOuR7LmwYxclPw8z zQ*Obi*6P{vI+U39zJ#%46~2dj6*N1%zee&SOvM45m!S0P^Zx}_|L?-g7{0$D|3C~~ zhl4M`zy~0D)9ky;?DXGXWZ7w3Bh$|WSs{@4({ZC4r-A{aC{UldMD_gvXAvSg>gWF7 zd;cq!|6kaIs%u9a??87q{iyYeiVD))S=#BYH1;B$`H||gk~Ey06Pj^E7g6D32Bk zR=P5gH5(XGghSGk?mlA~1H5{ld_Tn~0&d z#-xTu)|-}E+t2T_X)z_`J|du}w3Dcs#6HA0YYR73a=;=uPj7rifG+pM}fJ^DYZY7Z?No!_D zky5E|OS#^5`*SnW61w}&(f_PuZ_Ln(9^p;^bPW zEa@;Db+2goE*^RjNd9SH)GP}(RKb(Son}qiB;&~ zoE=?8(yTWN60q%*6wF`DC6ev)v#-Aw&ru7Gv~Z-0X=p2_3!pe)SCf`uZbLl z9+`f!Z=7c|FAQWi6LBu%(@ zYPoZ|qd)P)#kqT#Ry7N>2W7q6v=N5t@m#u&MrPa34YzTql_!MSl)F=UpR*SF8<(Fk zQdZ!#m5BOQ*;9c~$s6lGOwrbYRxP&6vn1M6lj|r?ezZBw)~5HuYj`etanO@=k1D|&r!C_G6Gu1Ti`uLc;ip%c!t&?f8MoPYq31XWV zqQ&w+X~Qo)V75#&_peq6T_Te~daQ6$!0yL$q!DW1tDARWNmalG8zppj z_Y#$6Vr+~NQ1K3->Mt?MFfPc9h5>59*x{5pS6ZCg6+r_`GQ6*to~KTfnpMWe)QQ0F6ct(UD1@7aR*8nZML zBRzDVe87>PAnoUzX6+2V;9&H~OIsUalnx9DuXEK7^C`<6Pyw?lk-MjI2O7+Js1f&Y zx0pi>-G!d-D$Rq+g!|&Xsi(fjmgUmCva&LLgD>1-`vYD6Z5Qx*cF&pA3$n6=N7D7D zd1ND=%e~4V7z7-SJs|G?2$_0OsBT4W_>0eJ%@|-MWIE=r4mZwI4osl^@aPmpKN-^1 z$61lY)gxoMRN2IPHv461y9(j}Wb#$lP1x^{(wS$jH!;Nfps+0RMC;N?F3`djmjEV; z=*8C?(mB~~5WE4N=*x6xld&rErF**UhaaQ9J@%fa(1-wlH@ zoPjhLmQj^xdg~aH9+JiR2TlbY>InEct3MV!d0Mxf>E>GY%ZBfB12oX6ggs{FY{VE# zdw`hf2={Qd-aek1m)8Bol-0*O|uVeBG^QO8^#!#vn@(eR*0h zop{pjUV>=yEsvaX_`^C{UNPn4b~Y5EQKhKz<*yv(8pmj&#(G0@V*Tlkcwb3!40sBZ z*PP+G=?hbcI7mjl=dCfId_Lk3nO#@wHV;1VWFwD-SHNO3U+TGKEf9O0p#CJTV zWcPz8(#PZ7X*7D9jK#fQVk&UA3zKX`Qc;YA*D3bi@ikH=dQ*fFD- z3vTXHj?Xhc;L#j$njf+zTjY{$cA$&of*O2xDgBC9JE44zz5(aAzAYnfG!3%YU`Pv| z;!pOv=H_WBRFL$gnbd}de~(WIcc5ddSbSI^oqR77Y+vAh%77ZwvnR;F7J6D*{Pmrd zwZzOcc&{Y7K20|P<<{Uw1KXaZiY}SfQtsF1FiEi;%EA`-=^@5KfIWSSwc-qn6!oDz z40KXn^Q@4#AkACozX$f4F|A&sj)hE-Y>LxzQ$@lBTIQW!eM0i+%_H!X6k-u`v$5PF z_d?GnDXmUZm37n;V0iZeFo=yB?0wi0#04+oLn%Xw(%(|ETAsVYbD{^o8Y~ptC;GyF zm;RRY)pE97$yvMlDgHu~h@i$+k)}odf-`r~Ua+=kC{?!9*f4)NambZM|e%j);x`@;MAiXZVL9W``(>i;LfSb29Ow8HjKnkr0R< z*B{YlPdLsZk8-S|VVqK6LNgwaz=p&Bem+-L-)9UB8NoOq%8Gw1(|a13unU!nUU4F8Z(Fo@iXpL?=YPrV>@ zo&l0B`S11m(&a8u>`bpRqPdj*i#EYBw+u6Io6S!AFCJ7jx%JVCX@4djkDu3s1it!{ zN_}^T&ZCT`&MiloUU7ahJ^)eqlf3Qeft5OeUjQoMFcMrEh67|w`yZ@lk}}-Z)>i1Q zVqGOUB<3G1LO|c3@52XA4uKf_aY#DzKS?B5abl%YxDoXn(Vd}4w8PDR8roR_ZFvx# zRZD=4UHM^S-x7Gb4~VnZAYS~kFbE`&TFeAwTi4|9IdyDhd)eJdjvrPIfrQBH-PR3p zMz<<1O`sb0zMuNT3PE{s9pwZnvZ&pe&SPLr{usvT`zi z@3eA~ShNdV`=J6*)GiGP;RhcP+IC$+VufvpJ!JhfBrpWQ??c!~Jp05~g#>cqZH8j_ z&6O{2&e`_m&C@{wMSchl0ui~190z*<(swERq?+F(ia{W+X7Dq`@mMUDD7g9`f&=|o zQqbswuHreyzPFn$!FB0DSaCG|rnZhw>~Ts6D*3m0CVZbK1OEFc*K#T<&S?*7K>}gF zPbO$@G98-poyXK0vVdqIn$X|IH=q7(d@3uG)ThmRh?C`s_^q3(d);Uri}C+Hez4@L z|I#EiKcWg0{bmC2zrH_n{`uZ-Ko1v+0LLNr_Hsq8C%?D!efAs4R1lPeP~-Lr z)!;kDa7HT-TugqhH^0o1lrXg)vPM(y(QhhoS|jxdP?}TpKepEz!QwGqBjxg?ofrke zf7e>*H?1{|nSl7MrK95to&(|hKR?$Ee0RLY>wEn9`}n-Io*n@*!PA0uDzG8*VY})1 zqip*i8^1aQcOmkMCqUYE<~$%eo56Oc&7DXBf5t`PePBHF#4Q{6_kH;~_5gyO^IEx4 zSW?n;=paN=<+F`7MQm{|1i#T-YGgJnBy*EeB0KO#2YKOaaSGACZ$* zxW&=+{u2Lw|INgJO_+Q-?LgYjy!ApblUcj^!Db2%R%K#)(lejbadFE&vZJ&)zh*bNm)fr8hr5djiRod&P;nZCX$!Klc#- z(Yyr4FWH61&UH4(^_6B=*{0DR7->iZorr97*N-f1zh5%h*jilN9I4M)r0D&;$QwS& zUOn4WEKK2G+$)M&?7ds7FU%w~U-;U89}iPFqu{enp;`gyTS}3vkAbnt6YGj_>cM1X z_5)jm*3ny~Lh!N0KC){#h4K%T7ekK}B)Tlo3aWN;6SMQoW_h=JKc-drL}&_g?xe1V zE5M}$pa`5zhRN_$-WOsI-i-z4nS|=sdVu9YfayS(ug!$$vrl1P*2}%35T~qSnS(_o zxz;1T$m0V|)b_UXq?_nYI2PA>Xmw=}?v&;|i?bVMy{jm~+?_eiP#(o1@g6Kh&*1wY z#D^`y#sioaqMTk~uU_s!q~ng+812mI%sU^&#B;-W?rJknc?->&3(Bq4`34&%U7G9l zU3BKf>(5v}UNYQT?B;Ntdi&in&UW)B)HSH7!99v-7 zl3;jf+XV<&epl;WWA&n0vE1S@*2UDEzS8#5l3BB^(bE`@{H+u~gc2qfY27vQ$J~Xd zrlNYyaW*wkZZBhWADbA)93H!6ALZ}S`8APU!c3{gL+C0SPKxUW!aU(R3ka+DU%{E# zX-&!Ci|m(R*_l>n%LrL&JmM3q6D4`9O;Cf~AFE)xT1(pk&7plG>mZYK37yT~=(HNb zmYqvm3_Tt07v?^mxP_YQm~^Zhn1eE!NIf|ed=)yKFxkbmgV#1i)FqqsR!?ryF!I$J zhqT%pD>yG`-aMgTW@KVW44Z50@f*o=i)ZoA6~@5iWV7iUq<8(55wbKMFB59JTyWB& zYc1>&X=}N-rFvgHT2`&f#qP0Dh+AiWqYB0h*OX*7=Zb4LDo&VPj1o7Q8+7Z5SHj~^ z4x`0v=v7HEAOh?Z=Ax~s?Bs__#5lZH*wOyFC{S);4z3C5z~Tip81V@?&AYA0DA^OX zj|epMNlNUTSfpO7&;DvU-i8zh8PPR`zIVCK&Lfh+vMo+D`&wjY7>C)RCC^^Z4Fqc$ zX|M{aqE{AZ^~siL7x)Ud9;P~e(gqnkFhf)T(Y}lu7LSf3I*y3!m__=Kdz5I;`-_k5 zWY#jdSgkCzZ9ja~HaczNvz>ks8PgacJ2c=jmTObhYFD%2m&Nbn>jOvBire!rglEp) zkS#yHEZ-Td)l(uI?bs;U@q)NFFa87(bYy=XHAK8@0M|RWcgLTaAiPA!Le1DK{bzON z;Z_JXoWg}ZD&;CialAr|ag^*@2%C5=?bBj~6SANDW+DR&4k2f6Y?C-W5!>P#lfEww zLu|n!m|ls;%>;Q@J59#zO=b-@X0j{$K7Z@b>|)xDn7|CWMgO7?J6a7}N*=;lK9!O+ zjxLjH@Iw+BGYz6+J9Zo+Yku__uDIl7$fcQ2y66c7ee%jw08=;HD!6E_!!M>ev))UtO< zjEoOtJ_~h2hmYs9? zo1G-5-dS^Y4`BxOMe>NjB~kX^*2FpBGFS#tbJ}IeVzMY*uRc4za`O?f7R|>y_HXsO z@G@dxYWnS2!(eM9YHV+rW7t+@F)9@X5h&oZ>=&l3ucJ&RJTXhoPO?I7-4|q4VBEkmmbmp?Qe|lnR{t`%mZ`(T{1~)k%u048LU`IQ$!Rs7s;G@N!p$5?``~_& z4(>VOIw`t!?&CtEFLr5)gt@(MaL_tnf8h8Yu)F-S(HD7e3#<=oWIObXRkRot@60ON zcD%&`T2{9;^t_llzRD`td;UxFx8V}-gW?aeHbXKoqkiq{oyIoC4uJ52DDnPypZn&Zcv+aDK1&vofyRM)_n zp${J(fkm+S@lRO5>{LbTi%&Na8WFW>plx`SCPWz5WG@xJ(8rVBN-mKWMdgNnCb-ya zSLQV$eY9XEsLWp-PhR7DS5~rC`VG@323yT`iT5jS@-&6G0pgRd%p*$s#%8~V z?onBtg9h|Uy*6gMER_Xqz7ZtQQ*d_F=!IozEQ{d8B;WeJQS1SoW9YfQCFoM(aWKKA z@kIr*Pg>S0fOH8W;<@MVL}VVg@3t(%96a5P{V8_*ElsTKA{_Z3Ah0u( zJ>+%blB}|{9vcoh+UDn(nA8=ow_B5juA)uuG0$aT+`W`Pd=B(|P}4!mZ!_en?1kD* zU5Ca)Yd*fire3C-*plXWh2}L5(@BrsTIqXBT)2IsYFsJOTtp%Zo<{enlPGHl~{n5`eNWnw$Xi@lEnY#*l>;gNJ zbJl9pQ~l+O&AGDjww~mEk0MHRdgtJO)olxz+z-eUz1yINNnqMNXLZKp@yK@ZAs9B_ zl2DHSsARP+cfqbMVM+Ew=$+&5&={lL)<(kucU%8}>LhjS1e6TS9s;Y{0t+jLy9RPA zif)m>&&JHHuxcw?$yWKqckZx*cf1x(et9d;v&%NLLKO!i)aOwNj?^2m#%GS;Y&aW6 z`C=1H5w(6Mj?R^Wc>TuE4z6t$AwDej7Rx?-5J>4! zwSkNt*K_zj3(23Q0tj8RA308r;n0nd&?(;tO3f#i>*R=3wrM<65v0KsmUvy#^pqYj zNB`aZ124=Zhb<0Ak{tCb)M7u}_ZgB6)(92w#Xtga=lbPrkg<7Xov12YF9vczA!mOv zjUZ;6CmmaG^dX$3Dv@xmNq`dRL(<*V@K;{q#)#6hh| zEJn8>J!S#%L=)lp*UfCe%!%#u|7u@k-kH6eoA@+4k$);jebURC%VUG*dNA*6`g-bL zGah@?GD|5c6}{{V`TWyMK;D4%Ti)PyV>M_FGI{bJz6*iGCH@SIKk_7+$B5Jx&^oi9 z%mrfc;d_4Mv!f8WCGYi%z6Dx#0p$o2_x$gW?UC%??}ASuf%o?~$ln?yexA-r-1;9E z3dr+EB6l3Aa=vP8oJ`c$Zv>h6WF9$?G-T2euuG=m+}P6^?++ae99RB>T;2uQBdYx% znVhE-O%=X&7D8?QYX-z-z>AbVA|j$2J-sFjp?3V2yMJev-@XCfgC71aqzBArChq*W z_jkTo#ngQH`<=j-uwTQY_Ss`W#9J03KND@>FasNrKLvJZ0yeKeysP{hVgCLlK9T^} zeUJ_%KR-W)294AU9~lLW;)pW(Jtgxk!}U#bnz9TG)pnTg9#i!*b4~0HS`0e*-Xc+1 zVZTr8x5S2~EGPhasMXIu>jbpS;-4E)x_%uUY=;53FX=yy$&BG}^;`kbPyB!RNmHpc zrZ1mJ&;toU!1vPYO#C+|nTiWvym$dpVIog{nKV&2-|R@!43u4+$7^c7uZrqF+LP~p z^vwT;ZPbf}H$p{{FWUY(m>B)qruLF`V$6fnmHxa{Pq^6-qj9E}ohwp(&sII)ZqURn z%xlLWXW^YH86A%q>;n+TRCMcc2NwoWE-`s%US4T+c&DAF~;9 zt5*?D$5a+1eWYu%CIn>GJi(Q&-&!zAG^RjWhdhwg83`dMt` zdr!!bzWDln@?8{JQ&x|YeDiI`lPsbX;pE{HhZ*S^HxHGUDo3ft1Z>g<(3xC1%>kw6+Rj=G1iT%&p2XB!%BIC zEa@$ym*lC!S#9>FOYEg(GBwIacC5iH3Qr>A?EvEVt(K$uUk_akw7mSoKXjjHatU!> z3;4&8?lp8$U?Npm2REW!oqe0zukZ| zY5uH?@Tup~Gtrkd0*Jr&Vz&KhLqDeX{SE~F)64QCk(W#^z9uk!zh#m4)3c7A?h6YK z`!Gga6o@Zs@gI>Tec~?EnVx>jc<=K-WIKp+3q(}c*98%&`-fR{yel`&bbSi&c*Jv6 zp=xezIg+NYzx4XI!ShJy3`D&xg9;dx^I-JET)rrKGw!2?|3m*SXX!w|`L$HZ|9!fX z#09WG?N7ZB7Xz>yIDisrCsLB2&Z2tVQujX&t z7)PMsV35|!1pY$*Lux_6*g27VB=v2F+AjKkGu4iF@)Y5H&)b6)xs+BUF9eQX{9Wu0 zC*5ggcW+In4X5;$ywQ?5Lztb4GJACC&=;rAgpX+9I32mroR_cgNB&9`2NfpjCPAk(82H9#Bg+0}*mX4>f^ z)N#b^8NcdQO-_GE@f5}FwFXX!6l%2~IQ&({IM<$q$X64$5!(KQXhgR#RxuUPeA@Of zSv{l`je1y$*dPK0qBRkNaGPz$!}f!hD5D@2ou^4RK?Ytd`Lo#hZHmJ8VOV zVE5d_YKZRf0wHu8OxeCx&av$Jt0nR2NMo8_myuh_{i;YmdWnTIS`^>386xVf4={(V zogERS=KUdpoz|}?&smAQ!hJGg5qAzssFl$*(Q?DkN8L7PupIC?RB09(oQINc+F=uA z5ZWj(yMK_Gl8Q2znvxc_y4gCdwQ-pC-UQ=NTJTz;v8}jg zw1RG4Zj`yTL$cTvjP>Bzld@@z#|hrqtu{UrNEPj!n!y!VFGKIUrNJ`)t0{TDgk^5M zCFEq@;*J$Ux-td-k$oB?)X=zH)6+$e%=a4c4DcFaHwrD(kT98|^6qfHqN}}UEP=?k zpo;6vLG(RaqRYAe;c|9WSA(kT!aNF)!;U<{eTA^z+F4k{N`wCidiLB_e~Drg9>z#T zy1lGDE?+lpPD3D^3x)B{N^DQ_=VH3lGVLRyaEZ~vag?+BDRx+(X_o@V@_Y-PKitG9 z)_|RMp%e9?+cn+(U)!XP4piB|=2oMsea9blU~;;B%?OFm`(S$FqM_| zKw2jZ-OG9;o?%aP0C4_&odOP->8=f@S$t+<89g>*#)G0T6kz5#_i0d#iSowwCythc zOtv!)L!QE?8mF%yHBT0^Kl?%@arwoFr52*DcZ?lb+8)8raQ~=u!i_S&MfS?z0WPYd zLX)H8+0OF+&nytm6SqL&U*-*&ssu! z>dn2~PLjEmd8w8=etqvn*F9l^`DMBJS^ityHM)(KCPb`SK zz0J+ne^+QHKjar%!3x6`pt)&mN)$6#2d}>K`TAMG&ffmrq1p43HKcj!FVt*M4Qj?E z*6*LFm>x1Uj$)HZ@*c~Q!SyU20hOZ%Kle2O!~#Ne#K0!BqyN#OoS7=4otyB0f!WjB zW3*aX18Ll1e2ml#EK74aRQ=(Ntm4~t!>hAoB+Kpv`uFyqkua0dW{`f({*iHTX_H&8 z)g|~4p+RdC4tzz}7znwf4OOGd4%N?3Ror~P&l-uY_%7bOXnh(*8-r4d+kWFz=B4#M zxW7Wzj>p3zF4}0KOj&q320jcs_^USMC}JCfgtw3|y1G}i7{8FJNVQyfB3L4G6n-4xvYWNC>JKebbo{I1piu? zXnzp9n95GEI|EfR9f!bO)ge-&gA@gWVr{-s zu)>`!-0;-9WSvk&n=YAt>mj?qNa1*{cg{IjyT&YFw_EGOj=7&Cq<1@=?fac$Ldpj@ z$BXyb;FfCK=d?o=C#!(f>Q#QTTH9n*V_>wE_QMI%0+(GEXRC;nTb`7kd)bMI7{q%c zCOe+QV77p(i>1nZ0$LVyw2Hcsni`qcUdop(H#pR2)^AKMg$H}@=E?5h-5L47&gO`y zP_=>*Z67GuwY^Zv^J(?+zKon$5Wo{lp0=!4Pr0JQQ%Hg%hY6CW@Wz42Fz6@J%LZ{3 zTIC?fG+3qNyzCU`&at_9N&piv*BzT`Gt!2a{s{_4x6GX>FF*T*n$(;nq67G z_)Z@8;gM}ou}_9i*hUUUfylPh%4R!;cQzO zt*2<_3T-Wt&RR$MEr**wfZjJjyPNmb&>oj&5Jx2#9m?-w@^-ur4OcadoYHHqgzi^0 z44LXiC+U_f;x@!u`YzZ$GOcp5TsuV0RKBJssI=ON+PG%x-ygId(Qzc{)IFm}D5PJ~ z__Ck!yIgNQ!}2JwjOr-78YnWhpX`Xw$MnU+-S?hTAG&U~TJ22}p^<@6SaR%kxhxo( zfOIvFTIa*oIoG%fixodID#7%clE8YowdVAtfCpQ3d+FQO7pT7wtRbDLNyt&xixeC5 ze3vU)h1-rR=!l*&(39G>v*WcrHdNA?C>+I>e6_>9Nmwb#E>Jfm=VN1%##Tk2U0t8( zAy&zUc%!V;V7tNM2ux+)B;u3UAmlViDuobE9TnX>KJne;oQNjpA4c!!DXgFA!_gnx zO8a1P^KyMd)xf%DvaGmmhHx6;tB?$2VK<+`fkP8+Ai^Ybom?e5(blj{9$D z-DH8|Z!3#8a|WCTq7kWy>-KgI`fzHyAMgGt8-%>% z^)qh8tTrXfH5C|7wV;mZ3XU-uf*=S8ZP#ad%+SFYu?6(lh`6j;N}j!ty_6@lsdvs( z_Q%&>RBx|zF2tw`-QyP7&&>Ao6aX(Q?euO|@>_lSbhiB!2NDl#C-A-JXl-%pDbK1?T_0LuC!v-_UH#gTR z3;Q0nz5~a*Q~V9lb1Th>+%jP+MEm3yxF02qeUz64fzyjK&;l~WN|Q869^P}#ppexT zb*1;AL2)UMJxNgo5^#=n`yqR+lso|&Onhmc1bD2EMLr4%$GK z4v;D?l2-3|xNgKf0yIw$96TT1V@9Hz3wH7Kq*s8=j+nEkR~ex0WkL@2fb69R;gA0e z{SN#yS|lDO;(fJkx53X4ur8ayDaFgF>zmtYW47DwwIm{EL4Ci^*Y<6qQNn%6(9l%$kT1N^*B;IRza5%v-rBZl>o1Yg7@;2EGlB8cHq1_l6~3}Eh)hGHckt8; zT;`1qFmxvo_y;1%EwzM_62ZxkAyTj;kcWH24dTKFGz{WYhwP$H2w-R|yA|C`=osUp zl9Hgw7rb27O4dPE*YD3YqBP~AxYIP%{U^cD4e>@8ed9hk*?_v2GFh*3H0iXzV_*F| z1#>GbycEWFBCN|iulPxUZ~&WqC3{7%?oKf52{*9)!*J|OR8-U|7N^7`&{M&9p3xWW zelmPv6&#BZ1yoRFUkK-2#cBjtz`_wM)@>ULHjE@jd` zH|Xyb;g^Z0+k;P?@#G zo3$R>`1V=|=Ho)8@7CaKB*`(_%5p%3qHw70hQ5kclFAL2`^S$IptH7&(Bh_L3zvPa z=wANs>j9+l)qkEIA|}gTj~BBW1;g0JuGNQ%2sCNlE#+Z)r)uCvZ%+awa#yjQL!d)( z?zBM1Nf5~B7_lKh*V017O!?*0KTm=YbAuC3gG=H^p-Xc(Hco-dLjPwi(S}pj2dG53 ztmf)DF(e)Gp%P&rj$`#OKpDgG260rDT8=<@-FS*{7Zo4D!+Ybb35%oxnX|y2dp}P2 z?B)C|`wpY9XEO7@5$y~;`<@MxF2V!(9wbU-xHY7m~m5txYg0YqDFJj^(m@h!s8 zHD`drSl`|uBnUJ9Zy|Os4I9F_34@9cAgb)jyBjHk`j{CUbvc1}Cr^O4tps2JV7wj9 zi}Z}xOM5X5%;%e*c$0b?#H~ZY4`QYLEou={xJ{be=E|$BERG-iaHAebq|3t4i(BXj zi6QqvQ|pqtmCNcu)IUsW&*vs3aUfu1uBH*gM+ulvo_%C#9-jJj=?rn3G9BYGk8~hX z<8S6l%tE@;R&N z69V_)ds_Ge^r0Q;8U-9&dY2+xC}RhJ-|(R+^6-K#d1~S@F_613Do1D$a~FDli%-N< zI&qos+>D5}y=nUVuZVXk=OrkE`^d8|oxqAp`))Xiy<6>)bA0~SyZio(j`i3CznwYr zDCh&#lV*|Hnk127T-}wt13)to`?qUdMKfsro3RVoTkOLRGr1>xTI54g~Mo40n%V=kEA6a z@mG9kzyTYno<;NHOT;EKnj{w*w4^$m?e&ynN#nT|#N`JSQ+#YtWo7<5-sf-48vstv z+a_-Ii<;!Br(7ly>PpA4R7@RjTXo{OrovOSj`&9Av?jIF002`Y|PXMUzTvwK@ZZpV#*nL=Fq z-4m(ahOfcG7iFmW0;dvUb&HJaW7gCh8(IDIA_B;0*&aX}!4I?KdEw?i9`U5|y&=-$T((VA5gG-NW$@5L2is2SS!o(xF48}qjM2*? z-4JsxaBL`K1voI@YUash9mbTiKuB{}xg3*RHCAUw0)WM4O7pr)Esp3kyCZxL>!lS$ zX-CoLV{-*V0q0T2LZRkY`jJy9asmFBIWUY>VbHBnWwBDtrO?*X;vhH{fQrgK?Y2Dc zH;W_H&%b!P4{nIHh3Tf)8rlF-X;(N<1kxLJ(Tm5f78xDwSIO^G@d4})RfF^e`zSCl z12M0aFT^rV!HnY#j=ckld*SPwY}aTe`U~}y3iMGiCsi}j`ccnaCOh)RRl2|d+bq)c z(OM5Bnee{SY{v!Jt@-C#@>!0ah0CUB!n)(`IgEXlnTnCRd4J|SVDh(+tl3q*n=3PC zcx}rM0g+aib3}M!?*gD56{#&%{uHpBf0Okb? z$~uSgMe_u*2MRycW}~kvSSLyM15@i^wblk0n+2Mv+dnta5*~z($@5nBS+tQ&*%EHp zxTbqLTPT-xI<*3h)xwf5vH__>lqlafVf+B!5{`Z)PuU@#R^j_mut-afJvZrpaktX`Tn z%f}XMuJK&pVFwwJS~|vAYp}`pU+W?8gEG=eQIgt-EXN~r#&sZ-JaWB^fpOG(u1M=d zmgBta_Sa^u>-7ySPaP96#;;-H(a_O`mH3j2V|@c$H6t!Dm=2`l!L7-eLj0vNZ@-x` z@LMV>eRcTtTDFBFfJXEi5;;pbwKxQGRtH#VLp)#(x-y6{9x?EPIRzc*_7oAS;Lo!- zqY~+J9`hgQ4<^}QbSE_(A=m<7#~A}9Rx=}ehKg~HpE7W*HwTD*s<*ntF2tKNp@!8y z=nh5BCc3T}FgwS;CoaZ2cB0@L^qdA-bZ0$NXaQe9ctn#!O!dJUD0bal=?Fa%y zFeL`5Yg=Dq=bLo>Vu3o=gMWe0UzRWh{p`(@%d2E+YU*a5PILxhw&V8Q6{J#EHK;;n zkpvpQ@gUyDtmL_3z>b1n>nT;cr+g@`cXvwq3zqtf0QVZ?MI z*$}|c=Wf~(Zm+!05HzmHl2#w7PFzn>v?uAG^7|SWrXPU^q?ZHT+_U$+gmr~?gus;F$F9l8jHMHQ)c}NwSIRDHD z=7a9$a|y>g}(He$J)W`M{SK4-TYiZkTWgqN4} z+@%;I6-N{}i_v^bFd5|L4}yIUGu`)WoIo~;<>U)Nn;;1BX(hMCY?>ns40H90#FNg` zshG1sX>Ex>p_0A$7v?>Eg;V|L_dSa&DZ7(Z6&OhsIEW!mgMsB+9-}qxT2VFH#2`-t z(y=r)<&M1rg+o(43Gl(Ies8$dQb1DF%_M8*8p)T8Ho=-w#OpBne~R9U9rQgv5z9ZX zl_%Jx)q7KSAtI7Lk5T`MQi^?GBfA^@{{%iR%7aw6MLLl^Oq~>{39GohheC)0$-o|A zoQwgKYL8vG(I00b?lZ|g(q%ufcS4v?o#)Z9Fi=vXcKdIFFR+Qh?Z*O7Ou$ovGr0dP z)b@Y-i`PU#6`#*JzNd$z_xylyV2j1G=p0bNz#jr=>c8e*)KjI14t%uKAqGG=Z-A3E zFF@5*2~?|5Cx75~2!uPMxcGf* zoNB!+%I_`VGcBRX|BI6UYyX|WfKkFs6rtUJt1{p`-pOgH9`{%7;(7enjpWA7CfQ%|Aw6?i{Yk0ZcC%bLlbI(6-fy-i zd%d%9yYtrPrrR?fe&joajd*_J7q)t9KD^sojk`WK!ZZT#z2|ItvgRF`x`{WDJ+-mg+TmmjgJ&?I5R|tInxZJWk z|E<3YixEB1FG2RZI%oESL4&WCA$Y*@PO*{=%Y8l4{(Bhg%`U^a#o3DD=cWT61Fka)?(U_bW%qitN4;-iGzHb2VU#96u__u0+pRw}JetDuFJa z7Tvg}+IZvaUvJwFNp-ypB3l6GeqP|}rey%-$)8{!Y20WL9wJKpMgU2m3zGDxxKKXF zDhdhLLO<}o6`}5@HApH`qdh&hzA&5_@($$|>^7KR zL!e{5ad7u^D*=-#R3s=z6{trZKhG%0B_^a)#Fgm{QLLy`s`#t*ct>B`o44Y%n@#?9 zZ|BiR<_3nAk1l=CGOZl*?S-19S%?1B^(^RY&hTxUk`K2N&7bWE%Q|kQ2DGlr?8Ghj zVJL>NAGj)3@3r0BrRuV3t+yaE^Wr_bP;EI-)>Bl1lClms@t`?g-203QOL+rAw;*0>tYRwIT zS{(R~-gGhH7N&{@poxy?^^sm*TW+&@Z+9YwYXd9-5RB*?*{t`&<%5tS^XZiJZxC*X z(Uio^l>dc#cJV$UXf&#f^g4c8s<~f>=WOx9mE;%J#m5%x7Y3rWb}LdHUc{=MUikWk zs{;4If7s2UP;jLD^;D5Lf~rE`nG_4thX+Ga>m5VK)+Y*V*0($K=O25Rm!Hf0*Dyr0`Ai?mimBwn|-TLZ^*+M+6o(4vv`pDH5XNaAZA3=ZYY z6<4SYnz{Q|^@$}}KbfkYskn!}IjkmOF#6^E&B07s4x713K){8nZsL2YDa*=tbvd`PO5qFvve4C z5~w+S^2z(g$5eGnh9bqV9PXcT>PZnh(n}`fgZg@*-%L$aCzmgD3ntHNJ6gjwE7{$Z zvT!__@@BGe(!#}Fis4aB$3l<6A(mvukA;QSA9WlXEocS2M;y`uN=8MdZ`Jxw{eNtI zdpy(oAHPngoGx-YmCChFom7-^jTx1sj@+W$Nn)5$VJpKH&av8bb&23w(O;J|4 zU$)uCmKMg0#I~66d#`iO_xpYPe!o9@q{qWU5Bt16@AvEVyuFgdWLGLn;_N6x!HL_K56sxllI#Q{^)q4n8(r+I?1FBsAib0+JUK zkPDCb4IdVJL&wh4FbCA2wY^Iq^<%?d__W27P=3opB2(vctOvLO}??eMSI+$@qn5*ulIeVeEm^eO^x zoe$Bf%hF#Yv8(YW4uwW-YGsK9xEi59uC8(ul{tl;s+FZ!h9vmLGVOI^TI|VngBkRS z+SDXLWfa>OQimquwT@iS03$hKklbcJHOPd&9EEeLIre0-`F^XGIalE)0|zE2VTw3} z0$zi#Z8`g>`12;CW^Bz>r%g_d95D{BGQ#2T;&{c$P%TwO#a>8a{eybnJKd)F{<}L4 z`vQh?_&(*D{;z-26r73wr70ke9QXbq8lpJUfz8Ygg9llJz$KN$22W0P@p(!RiY)re zE(?|54H2xCcGmYG13HeVnT)BBgh~6vuY0M&1{!W)urhq$W=CS^3}J*Y_4*`xu(|`O z$wo-+HR|J&6430YM#eE+J!AqUE}s^dS`*5+U$VZ>xSgPFF!H?i`$p3BH$zAn{7K%` z7}UZxQXG9ZX{R$mng$KSRpkWO8q9_wO6qP;zY0)P+Ymskntq8=DKtHK>n2P_J{7<2 zG@uO{>PJ3dt;cf_D}nCsKWLchpyP_CX#hnUNC7nk^q`Xcw6|{}JurI98KSe;HD!5j}6sHW&#jSF>y-_@a^! zS`k=Uk{W_E^^&&1Ty$x3t6`;2zyUUaStDduc;^hA4NZwYjI|%+#~zICavGcQM+SC< z!I>|6><{>cB!m<(lhFnN=l?4B8Z?I|@LFfiW0uKCe&Lw#%eO!1aTl7f9ThgyPHPW8 z1c7ee>VNueoCZSz9VgR&-ts%bYK(ah9uiIvNljRZI>l_Owj$OoQz8>pl%wolm2sG} zr;{|K>9bVOF>r{%;gEZcwK##``Bgi^GV)OCOb7En^}gsX!~Y=xc|EV+l)&s@4rvq|$+%Q)G#clp~?Q z@@-NGdYJjPt*Q-kJ4GXDz652=_ZDN+)L_tas}{^CEN;&P{!9KzeX>qzQPCbqfk}p> zIBudikLnLhvH%!Q9M#bGa|)ClNZ7!v5#f=ON%Ms1M+u>01^1l7ciujMo@!{@U?F*+ zftgi;Ghcl$Q&p{iL?F&PJ}JE$v~d9wF{;YN);hUJjS*jI%%0$4eT=OcXixhfsDTgf z5+}QD^#(YK({)%AZV;QAlpOuLFgvDLBU}9E=KFXd7Y0RaU75Z>Mz2Y{2u3&ktd_j& z2q=AkeuySYSd7GP zshrKOgO+w}_ue{`mJ*32&yQvb`*M~dBP-YE3dhYE0evv~z_bjg31l?o6qB)95O;VE6^IdX`smpTtm8@4!_E@gcQym?s_KPP2$9f%$#hQFL zTe9`Z9l@%W-#0rB#Yu%??h+f5NLN&l$$69vRQDb0^xAUQq~ke7cz8u_V21FOw2Ch~ z3fW^eC+l>WRBJJSk z*^knV&f~uFi_}7;0|LZYf!ZWEMNg8G&6G82hdc=bdz$mq!Q^zB4amu*H@~UiQF!)7 zs7Kgxbmng;<+w%` zy~-DchnNgxtW>KAl&NRde)m(e98;0{bPTi;xWCfg%qi6&c)69xZ*!$SVACcy28n`Xv*P$Z(u?;hK}T3_w0`kVjzeEQ#P8l z@ezUSag8W?)WlQKT+J~@Q7&R1$}N)Ly%RlAS{yAh6FF({!@Dw!;CbA0t7UfkTySHJ zq{?#B2;Fgw(`~7&;WVH6p(oST8YcGDLqtxfVESc;vH5I0wu&9*o$hbnC)v`R_c!{X z++TglsEN907qBfCmb zw(ojo7GJMa9=NemvDWHfE#M(a)Hm%pQ=K|G>w9*>f{o}9cJ%NBbD9NP>1Ps;*UsKx z?k3H4Nb_wb5l3E{I%{cHkcuJE>e&P)rrigm;sNAluBPViX|Vs*C`!ZXyuPtNE-Ms zK<)mvkUCDa5Z7e2s>Qx$e2ZHe!&t1v&JB0lOrKmB@m&h=Y`7W#F0(mBIor#5177`0 zRWuD7)y^gi|Mpm@M1yFZa-lDOWb?x;kC|k=u$(xWZ{uvVJ49`B$kV!GvtXXCvs}ox zto00QD7;|xk}mtGgB!51T_SZk)ZZV%hGN+<@TBDCspD>&G53G||2(Kwj3dsI)3p{)m&bFK z#&3t;D)ak3QI1QFLw;G8*k2+}z&!)Wf|=DERiD^*;MF1tpC36maV+wC;nqBxt|(3DMJQH-^)M*g0D(X@f&^8l z?>ra)P2zYalU7;_js=w1=|#Qek){*ew+`{*$||9pdDGNu1mxf3^UwUl&P~hGrf^rE z>jC|jeZDOTrwC9ugT(?b-7cfP(5|N+oL(2%&^4}oTLTe^^#$h0{?4eaQ}ajv;cJ&} z{Hju_uB4qZ@X;GSNn^Pq*=o*te+3O@c}{*nA&Ni4Kr3GyYEDyF_MeluY4#dSoys-DnKgBW&Cb9(FFYK?{<8Et)Hw7!6g znRwB`CA<&wy^M=Rv9^(rbfq6{d`IIS$QLt?b5(f-o7W|7=1*k(I$<~GZz=Xi_SuZNsC8T|t<-H9heW>=;V;=u{mO7W*XaHYAA`)^G=1?^u8R5zM9d|O@dH#4Hw ze@MQNf8pG}%7D_u(S7cX@AUrOUFLHKA|!rcEq9c9rWcBeIR*t1&D$0g<12;r!F?4~ zRX~(&vi1c)B~r_B6%`&kCqKLc3ZnLwbvm36u}yVr`S=XUZ-9kT3)T?q{93(H12ea0 zD-m2D!T4mmJvQgK-y;rphG)6N>+v6c3zF9+Xo+_0zkSi~QP<=4 zDBEeB-uzoz=!z(zQ&GawC?l;C^E8BmmSLx5)kcCV(~dHKEAJ*neGjl zwFs=*cDp(6NXQ^JS=5#p~c%98y=A1`oXEz6ur&g})5T00w<*({C$Vl1``G9LB|o%1xsxsV;FmHSS%GQxWy- zLT*j69`vR?dBCb9jTEh?!t!~>+od&ss($jUlXB^`>EpMN37IJac^Yk#oIg@nGy`oG zpAlB8ecl{aE}^kX4eZFtwsNgeHoD9g+z@_p=PtOO(pM_n*%KGI>2K=R=!eg__n3TI zti0qFNf183R+ZP?$p|J0Qcn|jcdj5m&DaR}pREZoo$BxoU2UaUV1rXAxU&P=dU`_R z*fLNRsH{2}JYh9CP}35}Pl(&Q_QvrNlP8>rJWlDnqjHJX(y*4jXsMvU?~zsRk8wX_ zUY{-DxAc(mCc+xuh|Q(-_|Tz?(0$2Zypd~-lt3*5&HyqvcwwC&6}-n57Z_a`mr#LA z*akn)xiM)1rwFrUoh>TC&}hs^8fC5!a5F(!;*Pp53pLTvt+r_g@j3J9IrplRkCT%O zo07F>iZKhtrYZ0GYi0oPqq{c$_#(|$cCJU$DkPzoXAIilY2eJjI=X%7ri6#U2~mp1 zth#dFnl%w&b#Rz{Jvz<&VjBwg;3o@I3{Y1Ag~sL5(o!J6Dja{NsIXGavTjZKj}&fI zKuA8b)2XMv1BHr%3wAfHy5a;z<2mTYCQu} zPiGc^ejrmQpQ<}E!|NOG@g`sQIadaEgh1H+m$2U6;?Db9oZ=%?AK$T_SoK13tpt`r z@Lln!|J7O}ua#LGh(72u_)wd;{47nIG8;SST(^~&pP%9rG#SMoLQZU%>vwhtNcfT` z*-}b9n(Kem?=6}u!=S6f z7Usr#rY&;eJpx=)*s`c;iNW{SHT7z?bMKC;QJ5AblM^YpxrV7mL7rr1u%$Fe4Kpr&l$N>SM~746LKk+(SIk6pegf=T?)I$A|PWpO>X^k611(r=?FZ3_>U=mlktu9oW8<2-eb- zj_uOV3&zsUB4f3t%Au+E6|*&Dtk}bfNJdXoAVN&xQg(F3AI|nS_s9+QKL*gP;5d&$ zxeEvfke|l2^HCd1i$CUW&Bmv8WLA*QCFP8@Rkfa4WdKh7x#VBTuoKf;XPjd9_WugV zg)?)>yP&J52s$UmAXsX15Kln0>uB!=!>(7aLUnZ)vPc zj&$=Kv7}tNkA4hP89TcDakkGk&)yosa1W)AVqZCRx z(-di#^sTnV#`T(?8NX{RW5Dzl1{7Gn$oKP-yzfdqDV9qwWs60ElqlwF8W4f{X>uU31ok9)Zxz^^2fS*#l@is5phGFoo8|X>a6znHM6)z0ruO|H% zHiFeEXn&;v6XSaEeOq`Gr)Fts#CKCuA!jxc*?2Kjwt>*QRg#v94TXF9sQ z6uVlCNC^Yo6GkgL$J{yAV_dR6WE;(7QM6YzH+IPuFrZt-4VpS5A3a=Mx2|%Su={f4 z+SeDdFAhruNt#eIa5qkzdqL%lpif|dP?K}_r{Z^sTS1ehA&NPrrV@q9!Kbzpkg#Ni z0YsoLoq5Nfxe>*q4&2brsU8*K?{N+nn%SwT1TO($;@TBJcH-9~w~8fL0~6Y6JW$`5 zN&0+N*2iH{a>7s_-J_1aLW@+hlo$gb$PIu(8_~o#JidNRT(Qp4SQQk?PS}jrJ&op;z#jgT zJHuuRr_W<+EZurlMcoWRH=lgtj zf<@#o`#Xy<4X&0v1)HEI(#b~coEZJ+>AMp>N(}ua2>=SkCYV{1o{AI<~e?f6+Dzao*#Ts(6>O_n<0Mg+R~8x2$+LHhCt)rQJfuY zc5-v_(?)d-xRR1mkG|Yn1pjMdSBKHjF^XI__o-8j1`mpPKUEbbFfEVZRcOFX!}=KH zKws>FhIG`HF+fh##2bkOmE(~Sh8@gy!wif1te5)}zA()Bi0^V*z4pw5O9YASX zQKhbm8F}W)qPhBN$aDwwr(W_lNOP8FF+RT#!(PpRbqrKX&1Y|bIjl_3t+&{$HvmSv zym7-Odaekv2*0Hk*9Daf;AAWu6Oeg%0_Rd{S=EysqZcXEyL42X7K%;Tq&Qg`pf~Ug zmyDl2qlQ2=i5^z6o8GX@Sd(5wKJ_V9ZFkGf&JEpZ{lh81=~Hpz0lY;4$D}yBP|h*R zEq${ffG7ChGn)uAk>mDuB3jLOP}WkB4~)1^vxrClz*V*)*X25Q`Y zc}qssLk}Y5S%v1FLZFa*bQ=d4}QgK5;*(MSH(b6F!-H;rCniy=g z#RP)P(;B&Or)RaTr1o*%5}?E#yGd54;(`zM4J{itmZ!l zvYl=qRgBEO-AYA-^w(VZh}0Yi=yRn6`olPvxYvgFOU3>0$&RE%A^)B~y|+KcuaQzR zYZ7dF9K1a-{a1biUb<&iW1V0_bbf?8W9@B=wUsna$DO%%N~w7qT7lqRk@H_)s{R14 zR#Yn}9bVG`YI@6U&N2HT`)C5eeeN(Wj>W7V2kCDybX&A=l0 zdj~O2wy^iRD;69b9py)x`lD&ri=$S87l&`=@na1F@KC1r!Md1YdY!TKlBzY zdKoLTQfyV15pn!c*2CSZ$it5FZ`$kEp4t3!Y@~a+tS`d=*^!Dw7f|{F4I`P4Hw_Ue z^A$qVA%8pXvCtQh%}$5svh!MmP(o<~^?a5&d^Q=$ej+tPG>ef^Mij#M;`}_7FB=qe zh+CYnn}2!oD*9|mk!0WaBjnnL+0gD_wue4D$6f37<(Ak!4<((DyPH78ziYm|E}z<6 zI&6EZNTH_U|b6F^}H*NR?n6V@8WlR=%yP zwfIRf-^cWL5+S-Gt9e%f$9B#JGiM_wZLFK5ZB7m=I%XfG#O#hfMz-SJN-YnUb!9^# z3+Edz&#onVNmpupe|}Eu*Ie6I29XhJ>jwMv*79HcUyfQq!x-2@o>gT#;%QE!`c1>;28%7g+nFfvoSa- zSzFAu6$B1EXPPjh>cXnW&XX@flBa5ur#efr%+7;cVv3SC@5%mtAfbU^Mv3@VAv7P_bO61#H+5dOQF11w(maYjsy(KGc77^A7SMtQ$s;*Pvc*$^w(dM*2 zO*!sgOtRAMtA6djTy--4Y|pEz+?Tj41fr%D#jP2Va1fqVB-X@q=0)wa12t8xUm@me zbC2e!PwSqw%3HV6_Ki`rfGud6QES?G+w?f}bliw*+#kT;<+zfpdX1KUs^ks#w#9Mk zbHZt|!`u-P*o+6GZnhEhCgQ^zMpA*@wGIOsy;#^I)m?b4-2NQmfcfd{_AV zP^b7zgLHA`p-g@7sp)y|m59<33t%lS1)eJ%VCwu6DpCKZiuNb0+x_P;0pq%Bg%5-T zLCg*d#jk6r!m{nWN%|_uz#@<)&d@i}IrK0cPMo$#?v(K1c>`0<){vLqj#!P=i*s3t zpQW!?;e#c(=6zdLp56R7eEfr9kf#eV8R#bo!c$l%{@W@b9Z52d>#Q_J$l!$O{$m2u ztv+cJhdO|Tr%HD9>CLLj=!)QsSrfqG@P6+wDq$I-G&|(V;s3elH;$Nqsdw_tLT3rJ z3p||~t^1n9q8+=WI0Loy`@TCm(s7o z&Fc$*lh(9siwct2WFKr!aV0rL7+c{H{V*bO42}=&Gk9h zq!f{sm!x{P41-mJbhfLQHtJK6y!)uo=@#XIWVKc{1(3I+c(|}17?IE^y@!CFL<9oW z%u;}s=^Gf0J5Of=pg13qzFh4sF@PxH>ge}J|86umirFmpv1YHeFB@&UlbgJ|NA=b zKDy$LrVJ^hBc**r2OH?kyuX@7S4%&vR#||~&N{Rs2waay3bk#BWR?fw?o9jqhYE?_ z>(O(RJZmZ`UlVLl;0894U((`2vbihwk(6J_@~DIRUPx1t5XQ?vL@eK&3Q4(e>Icqe zH+TV?PT-x2*GPx9Ax{a6qiE;E42+B+24MC6wf9dZt6v|kxpjPYft?r7nQ4Oi1Up{onUdrD>(0%+L6oy< zzgZhK|DMY$6TDpm;OZJ-g8<1s99J)l{(KnDqS&crV9TcE+K_F{d98#!XgF2>heS#) z*%*JO#{y!FbSY`Du@AT?8`h^)zwha%hqj?As{!$Si7x+v@6flEv?(FJTuEu|xFJ}GtY zgIYN`k>~8xXVxHC1XC}1dKa7 z&?tc zPPlz}AKKC*!V?5b=zLxf0N>sFSVr_G%vK5zG`0N|4RaC%3xX`U3YhBgYUupbIx#up ziZQaY!#G^fT*5fD69B0O>Mg%hnP(bMT3ooY9;TOKfT$n(zTiy=m8^yIME0_}3Lq!@ zjcwdqVu6L#FrC0o=-~ZY5VRi3cX}Q9o$sWjE(8@Vw_Q#iqVw8WEoHAIoK0U|i z4EgL5lD3@$ceODjkH+6fj7B2hLU~Crw+K216n(;LmcKMN9ZlIK+Tn@3DO(%A?ni}C zR>Pqu&ybJKandz5QQfUN4B?i&4>iymf9N|zML;uA?0~Iz2BMHDNI0>vhQWR_mR&MEvY zjRqJsB(9dtd4AEA{s1o*riMtu>Z|GsbI>7~yL8JeRzh{p9*qP#QYO%sX^QlJ`YYGw zzpBD%<5v%N9W4x=vVe5~n%@#5ttXG+LodHj*tvFNMjo|ImNtJtJda0+Kju~eOZA~^ zrpzb`qsPRgo{jMwxX2w-v~k)y5M+6%?!(eGW^iULsUiWE4d{h4>WjLhCN75OuUBY*T$$nzP~yn ze=*i1nsp(W8Bjb@(yS;f%a}oOKigD_U!N54?KwLfesK7S>Il=oI^)i|p1QDFLEzv8 zBel_i8aj_z6KnO48v`T|ADa6g3QBE%iAn+sokxAT@6h)+M$21S<_v7qh;503`s=Ar z*82Z^!s>BRRfKVQE(Ea0v$LKDrwg*V-@J1e zpczov8h`{XmoPlgTE|c4{_uobu~#d&Wd2*^j=(>0^G|0?5w$gZx_Y_s#k+&)+3fr; zas>14NU<mZZ{@lM$5E@2s4%Z_>QQTnp^Y-1KZ$pXN>uTQuI7yefC`-krl z>3+T*+(26gq^Gv~>*|wQA83g|lY|Y@Znd?W>!_!V4Hsri)BOtx9mXcba4&_u+Tgj) zWeEvZaK->dJLhgRYBD<}0UJH00bwVcwEz2k-s9`GyAvy{=_4VTHvq5FIFpZNV$HSzhD&4eaPdjNyCCvAtVu}_a*<5O}=I@UxNnAN;%J7$~ zRDn|Yf@{~daH8I;F3u2XPfxxw?Z-(L#8v=Fj{niftu}G#=0?tC)g=fZ!nFP;K^_Ec zr_Bn6EL;=wB5Hc7K3-Uexo|r-QPWxSonkM2pGWt}?J11NKztT&fRB$3+?cR(i+nT% zbfw}2bGCk5Jwh!uNh5CF8g3c+HRiWLMR3sO$J&DT805?K46!b1ryeAD6FHZFEZn1I zE|?y*_!)f~oc^VeKB=6>iMprIc?Yd)KnTh?%aN+TRHlv*Yamy-MPg`Q15 zY|x~2Og%Fm7m+5*wr`pQqD?oeh+)Jzwz{7aP}cq5vgc!%+OnAn*$AXKLT$ldd8ri) z+@G>L`91y$?}CIDFo)^sdaj51x}->5GA#sPO4ly!bOCa@*(@fxO>A+k2|->!6GmFs z&|e1H;e6Cj|NRz)KfYz1Ojs!uzTT9=CctsEP@vCCRZ`Q3j8))K$X`{iyXfnh^~SUV z!B$x;F|Z;j2c3CJVgq6`jnL!=ss_{nzvzl%x3^9C^)SUq2>vscFFO=HIXFCy z^JqzKEP)*@{UJ`waCwl%8-@}VQp zw{pl3Q|s++(3$NspC=4Af3AVntn{TqfbZ>}EQ@cqt@HBBc` zxgmh+bK=J#oAARe^RLzIh1|q2Qj{yplF23vOv#pUmjml$vl+(hh@!dHeG1eM;BbvY zC&z)M;5AWZFIy@ucjEIenD!1eA8Ax@u>db4U;EfHT}2V8z(?Pe(%UC7t_QBSZ%4a5q0P4P%Ri`1mfU#;TK$>GV+Q zd0cIFRp;c-vBh3LxQjb6ZjtpP&%ucjIA_1j=(XMuzM0h+J#7U{P8xsz;%`BAa<5n7 zEkdCsXgO}QTFEs(lV$B!&;m8*&=F1)eM*D)_dyh&i|_|&=zoXM{-Ut{kAItgAbMPH z;&SNPz?DX79OhnhtYLQe>m1IdCgMvP=Lo&Szxl;vXov-Tc}zIzzoCxG#vKf;v8Ck; zM^2KEwpwbxu2{IM8ri=iAAQy6vj7Hy&OQVBrdlp3aPoiji=FHHt52=UoX#m_gy^n4 zVAIn7Gi?ajY7Zp&=&AHjpxY+I1DY2s5=xBTv~Rar^jy&9U*3)Z)A?}PEW`UU9FVS1j;>jA3fRR?HUZ}b2 zY+H|h|9f_E?WH2x;Xzt&C{=|V%nqZx0k|pLW~{TObCY)7m7Xi=87432vA-MSsAl!` z-|LhCQJZ71f zFKDdP7)xmgy|7tkUfDx&V@tw+lQ8C2SScHQd3mQDz|1F-461hGaAi9q=oRkb+j=oO zL-w9s2@7gG4%m;jouZ^KAPvJh_XKVxeS~m{Ao)_|O_(kmL(BrzxB#e%ZmgOWy0rMg z&&PWPssfbrSkex1>LbTzyiaf9hp+B-jC#>ZTD4B)gc1lRH~|9i{*GLmV;(THC|yu# z659oNojs?$-xh=}_&eXY6CGMu>g>AVUn}9^t3V&p`|+`!>GiM7PZ%8B90y`qF4GH3+@DTO#}vn=Q+tiSJV^;8l%BD@T14QO*1m}8Shii zmxm9)^(xD1;30bk_~}nV>5UD@uLHS3UO{oC0qPbpN6e5>L&0rUm<%p zwjALg(}Z0AeCfkTcg#dh>96)tXD6=K`))B$ z^T1{J3N$O$-ytY9|Nd9*|G0JJWzgFOIQ$Kzu7WD~=uLRtkS%=-$LlrU+`v78zb z+;Yox?ww_LO4!mXC|L+MhIc6uCf>KvFGpuai8qH|=5p!&P52VgaVwc}S7@y#GV%ns za~0o#W6DHe@So@2Rlhf_{IC=lP;1j_h%%lNfKu5Mo9V%&(1<7+@Wd`IxP$n;o5esQ z{CAo~QutIRIo650=e%iuw{r-vqN(1S+ipd^us9MKkjm|$2WrzRjG+WLn-3QP)W#hi zOO6UG61G1;^@aEty&BKtg}M3mM{j8^J?zxB*3S71@xdRqsVAf-MVm4X-plY!n$ErQ z&c)P4nFaPrTLF}^t%nlDeX9(8&dS#OG@e^YAKsKxh0xY@&5)WfbYfUOTMZq79snoHd1E zDK5`n1l(%f;w(59e}%95(4`|^7F{rW7Y|FViCf;2`ReO2=~Vq;@u%KW0@=^T(ED-Y z)m+ygufzD54iYeYyMV|R(@2dUyho+rQ70Ls3xZ@9+=~4PotfR=cQ?w)FF5--?4!QY z0bz^4Gn#>17LTanM3JZ{_6NGN7@$|7larZ3*7i-p(!wtG8(u=iSZGRAROU3I zhtA4U8=++m;56~_#INUD{@70G=Y#-KXCPPE^1t##CGSRbV`zArN2WLC3AA?HH}tAd zyEd+Os!CQrRJUYMBU?^yuXcd7#_B_csQJTPh-GEZs4rXKLbGh|s|Sb1+QG=0z7Bey zV^Gf295T74<*N{%z$&T}znMRP{Gh|6YA)D^-{<%}#rb=hcBRm4W^=R-nLF;H05RzL zIh0D2i3z)=p$)utdq5rDl(2`@Pl$V=cHFi z#o|7)I7;7glTkJoKZTpD=~(XKAg;Z!M`1r!jhlCA^3AyAyLgg-S1^-f8(oq`cCh?x zYc9^en@rlPrXp+gtVE86PAM*DM-0g#NxB@lIg_D3vX6v(JYAjVoAO->Kl zCD9!XItVEZoyei+Cq@)fbF|sIX%g5p7o9~1Ikt)?+%~U$AU!U-7mFNQlnOm@UU2zr zVXVPPotLtS{`LY!Tq)9RcjQtUkM&O!b2F&dNndZvj~i8<@j&_x2!3_he?V9%{`_%V zjCHlK8Rln3*;ZE8O;gqVdcIx&Q6oSVXZ8Hw-6bWeGp!EW1_35rcO5-R{hv~i0EE1%wsCPFE=zBniHgQ5WR zcA6oxi^2)D^I_zTcIH1bwajo0>`d-e^Ski%@a~qp_AXdg7GYB&r&hz{0TByqsx^!; zYv_afPRXm`|DY}vevcT};;Kk&;Bu`2Mx49H^ILGQE@7$3fmAWbxHA`e3kkr=$?x&X z_i3d2_?WcVXC18UIQ~CTczY0TyXUr%&Y|f{`NYk|TC`@n&14BJqYgogx8JgEi3i>& zpI_wro8|jiO8yg`e{L7&)0+6S*aLX+9398>7R+6%_-1sIO7nkNzlc({3`fyqF2ch=NOt)r@QQx&+dpgOIL>lUvTXG}Zj-B&1wzbwwr(C%ZkY?Unny`kf( zhy1=CDrnB@oK0CB0=f2j1Np1h+>t6W@b9MDspYSp+x;~EO~&7&^f(Soahh1gF3+Lo zf{G;yx+02A=U<7s@=IcSGG3jS#^|grhn=a;+S>mwLjaT0oGH05fJ6 z1=ck>KeM##ZwqRjT0^H}r({1LIuaBdRx2h%D&0%ucm(+_vs{I*mwH?BMoZMkRjnZo zL{<-qLx`1d{7HL~@%ctL57!%-TXyyjL{5CU=W^o&%C#69W+&OJd%*wQeUyIN>8;fu zH;CE9x|3SsWaz!p3ysdiV^1?)Tqlrp_Sm&*i zt1->vuGV_STQss5GCCT?qNM{tHU;6}j!SZKi=?P)3<(J_U293fZ@i78#Yx6+o);vl z=)3=YiLjE`C}bOq*^Ni#HIcL(3Z&A^(S;T7z)p$(rfex3{Saz4(>O#L4xgM76;e=z zhFXJPuc$SASs|@qOb8PKvDK%e#wmxD9PKk2JHnVu|CFgLm@=Ybaen8PZREyTGe{qH zD)a53L#FJA&SXP8yJvI*yJq=0%_Qr79KD(noriBL>TnP8{l0(-9ZK~^zADr?;=Qvg zjaMRs_PTkPWFOoK>*WEdr_Na8_0sqeC0K(YU(~$F@DUU?yW+h#^Ak5<)>kc+OHsb3E z_~n?hQ4fTsPnlXbKp=38VhZs@K#v9Jf7*HJ{!zkFEa93p;sX_3k1+rt_|TZzdYKMh z9nnVFWtW~~tC!fBK9pcT@W?#ss$oQ!`iG|xl8+vqQ4BKS;$RYCenb`Gh}j##i98{4 zBu<|yQvV|xy*kJBQS1|cq2$z>K|jdQW;deINm+xq9ktPFbViw-yQq^v{t1WFBUqI@)- zHgh~nE>l-@N#4x3Ry+~8T+=JTeSF;LV<4zvvo8-HfuuGB@)zCD6F-Dy&EAcb-Bt_z zn5j3=7%S>oIL^>f@+Ai+9Pq8%SGLJ}(`R}7^-xB`>Ze|~^mzxv;&mr;5B)dB&Eww~ zH-)38L!tA5=WhN!1{Ex`rrab9aW{Ppvbyp+T32EJhh*pQz~r8$4{q35D^9}{E_9)) zhR(rO!B(y}&*+R~CgH+r@9Blt3v#^u{j+_y%DnEIP->YJ^Xu8Mla`v*<(e33*tLep@g~QJbncCoP>92ql`hm_>w?9uHFVo|MH##1^TSf$p=Q%!f*LTxihtCfj$< z|GG`9l&2j*Nx6Qh)Z28)S}J~)!laql5wAE<;3lsc|A?5XOiHf%^lAQYD<|jLSSDu~ZKG-S_&xIdd5&r(#kU=@3!ZlgVhqKvRuSwc}RhoeI zscCvSa#$KV>MN_7F-sV%@0^BG1}U=lk6ogGiB#>$yxPp*V<;1%!B+-5T=F^A6X)vx z*)O*Lzon65+TGZr$@;F0LY&*_!E1eVWm+#Y51De7kfqQ!}G!*DRVF`z6U6NsGs^wVSS|4Snkk zK_Br5yT&?h-}#isVR}C9T5b(PU-rQcrMpX#TrDq{_=x@*idzmP6$g@`FL%^E_>DOB zh9Q`&8cz?nl&YcX+ut5Nompp8#!%F6TWjy!enw-^1eWP7;UCZZmLawaj#_XGlsWsb zBjdKenRyp`nImoo3JpoukP4`jXT#aRTKoIGZvyvnbzV5)dCQ}DV`ukhHw1YtPwEhzH9(>8$=#KvUszg-RfZGD$Qfe*_ zKH2{cNv)46l#%a$d_-<7ZdiS(A8*8J0%xyz6NWhO)V*-pmv%#3>6;wno-XXao{Q18 z!!VXc-oPfGpg1t!YjXi4|4}ov;jD=pZ_wPN>1A)&H8J)Kw!k!ePC_(}*pPZ2C+V!L zW0oslnvMip4{`rfP)CNMdb5X?>@vNi_b%`v3&}7UEU){|l(#EOjK{iC!q=C~^0HYD zu+Efu=5Ni8M{%=!{zOgZR0MMI%RYEWaMF(}%eby$)w-=LSBf}_?cKy^M0e_u^n7%w z1 z*=G6y!S%_%3C>YRJH64NLkmjubkoaVFrPVtPQaSew$bc@V)VPL*}Yd6&IB&sY1OW4 z5Y-z6CQG|tzUXP}^yurv5~Obw5p5y{lWwC~(;A;}z`0=ZJEip7bSW36W{l|UPfC8$ zJ0V_dKAY*NS@TKKy8rdgBIw-OMhEPBHjjzh5#A1bV3|>sry7 zQPqd05|<``^@Vm#GS0#2tT!!6e=y3ctjBOO!O4%Z&tS9LL!CWGU)k1R45)4y(X<)N z-1mDyS_uBmhoMutyBSg> z1cvS*{s+%{-gCb1|JGtH7BKKI_r33Z@4c^UUwb9X?()pD-m6EM{CLv@AcRX0Jq$*?1G5$$W(52;%P-p) z8Az+{3>TaEa8DUy5HF8iRyNG>n(cLb?< zgg3w4n*FDI6Y|6)pfQ9`8!cI9)a$~&4*Gj@GxSzG^K_@U8Yrw9c`@SLjy3`GJ^@zI zD^n9ItG-F|#fbBrDBA6fBUNZ$rRr=deH2n={^anjEfxiL*G z*HKs;8KG)GPc}2JP@NhoFEkFXu3xBItm3p78dr1HJhr_$occT(!B%|JpD<~16CJ$f zv5u@(Viw!Dly`*=1eTcAN#;I7R!VC%phtU#O?eg{RHY-I!kdssA&VT~gtSrA8?A{0T>~H8jECwcU8>l~s;zc)5KSJMz$8bnos>Q; zp?A&bcmGY*f65vtZv{tjT|&n-pN-RL%kFBN+Gv*T?pVCN%Ye6zq3qk)3FO08fTHIK z2OLiIyzp&|d#Py7~}1+VV|(8Wnhb?a)6Qlp~A z8lyT&07Q2?9Y<6_5q}Q=N;Frf9D$^H+`Yr_-|FTa>297RO|L>(9j-E0S?&qp(twM$ z+-l>#M+$ETXOi!Fc)IiQ3Xu(3dt!5-zgHX2*1?DC!T~k}VyDMRZKWV)a?IwMd;k~7 z&A&s_Cpf=jg#bPAkhJTgTqw{+|A4%4!WrdtvwKtdHtOYS3P5S$Dkx;vj~uRbvfTpl z?fqH-nilsb_cr?`M~1&rCWQio8wnvm@?=z-(c)(V0@^x!MHp=9O9)3tCca74`C3-Lua1ke@8;{vp5w^X2Z+0j`*Yx|m&TiuQR zNX|DKK9~TVf3T30f!kFvbcaN=;T|ndhQ`*IN?)G6C zAj>l*bQSeSjA!vYWf8_$QbT=!ce96=rH`oA-4@A%DrA}Hk}nb2UmeNcR4}WRJ-yII zOoDb|qR=D2PXakxl}*R%1D`e^1={OMzWK5@_RApzxQNu|3+D zXWuOWgj?n<>Ma|409KRH`r~aB0LV=Lw*iX%hm)iIY-gk9ws&@f)UJw)DKspFd%oHc z133lIkD(!eYB5UnkO8PvhUQV2qR|2mZfOR^yaQa}Y$-5YSCLzr;XjVC;?D#UR{@7j z<(}1z9UgOYJc|k#n^l#SpjYKYQ5s|bx$bs)90N2bo1!(CJ#OAbzmv!QcP;adLUh%R zk(=#STvq4yS=gTN?RG$~_Gf0kp&bdWatrMYQdG-Bugs3}y3SEM8KY{imKsm9A zj3fE?dI>ziu!0>iGPOQiWs?O(wA(mZoWI%Pw(I0&Gu-J``pB{Hn=OD$^-qHm-mL)T zkJ|>NyAfm!fWYCVaGzTFr?(XX**oO<9)$tzfR;zc^{o>C<}Kimb55fcZ>}HdhpRO> z!#sY`fWNHSZH;pR^4eeb;*#VCXrH_bqi|`0z3)mmDJFaF;GQP~Fv>A9-(XRSS$ggR zdL@!8TbrAil44wPe+k7GyLE+ARyiVP^JFqizvbDQ+~n{`DogR6j_;)y-P8fon}9}$ zVqS;U;q!?E(uEp_5M;c2F0^sSj8BVgp7+hx#<=IhXca(9UborP!@ek{PGDzl4Cpp` zM5;aXW=k7pUKy6)5~Z!g)&jCn|HN#pk!9mSA9Vrb{NwcTE13$bM)zWY8v6nyAHck*8#*yf0A?;!$fGmvyX|=Q8pdfG#p$@3KEH#_ z{5R@wC!G$ZVWz~PX0CJmjeD0C?wP6EN_*TLujKW<>erns`PoCU^)?@mQ#CiI5DrI* zDzV&ZH2~a9>AO6=Sg8MSs&=q?NkUfPRE0S4a8N{xqUy`mxm(Ajx}vJ90&xIhp({Xi zM3YF^B;k*sy3x&PJbvR9R~j!Ba9oXq4rqiP3+xO&VALp$E$#g2 zw%a)6c^2q3ZP|2g>~HylV_J@*s1PEbE5p7*$Gvx?RLvSyP^2p;XIg z_IsrX(^;U*hanvQ>RI9?r*L(i+8IKE-q8SiPG z3jO*=`Fs|HBu1wF(d*~fovXD|0@Ku?cmGe*N$0&4ng_6|6>ZdUarHtu6j2qURmvcB zYj>4$w?nE_rjv3g60KG)aH{lr$T=Nm;3r|krBx$rFK5b&dGX|53gQ8xoV+YY)t|~9 zH?LFW(LJxbB~KnZJ-HKrTbD}wA6bDT?&rEBH4TV8ccPtkE1#%5$s;$e%gRQFrDuH3 zuJ9C-!s;wz$Jx|1m6O@jw~q42T;Ic(g4oluSg*>e9s3Bx9qHcSfc1BRLPYU`b>}zg z=2QE0&p*g#@=aZ3%Pw+s@(!iKNb9>|8n(Xzd6(D9CCRU3{xOqNCeP{3@SU=p97%Jl z_ouRRwhi#S{Mwn>Swu0^suAgS2I!P{6jQn`yKHeMUN$~{C~}0{V2_|1Z7{QbcSuJb zBi49#l>eR7VkZm0?LT6Max~@JjP377c+CLLq#QbTzczWNMFb}s-*eUf2y zxG^GWH$Zh+UC5;3GRnl( zlRE{OY1?P|_f{UQY0mq>s@8Y-iVepk$5x)Laq6Mlip468cCCj>O z5g;7P~|ofQS7?JE>~6zLlsPSpL0^AK{)C=b)2y$3us=&RZg|LOth_(0_28p z6UsYRvWbU4Gq93ovxT+*w5r+A!)W?k$C>O@blz#(JIo7=6EdV)2kaqDGT1jU_T(cV zx;c3x`9o>&DIMnQ{Zvil-Vni$0qpOR9t$Mc;1{PT2Lv~5!J=3I( zcrW6%8^0h33=~ZjpB;!sJUx!;k*4A!d*F8ZV>wHK+CcGr1CPl$-*&8~WeZo+KF*xi z&DA^Z%m)u1{FY*GJ6NhbqXvbMu->~JI-|lvMuIDw)3tj<L z))NB?k&bdqG0aT?@@lXUNForob$KL*4%E@n;ZUG9K0O}v{ZhFfTj#S=m1mIb2*Mi1 zjW-c9TdDR67lCWuYOddCPYdhb!Msem0yVr%Mn(iMX?#BxF{->vv_VTX56iOIt#x=x2*h3=^m(Y~U}kSGK?S{cO8-y|c%cLI*{`i7=x_;i^Hpm`GFn>L{cwkP17R3Dmk8f7(Y zXbO_`M@d`a9!i4AwhN*5i;)^s`OAzjM62(4H4|Q6L~&carcirp(5f`ES?c^IPQ6D2 zQnZ~~1poe)^CoAi5r`n*XduyF#20-zMpZ}AVJiW>&v<<> zMOs>VSS*p0*FH}}XHbbLv)f?X*#7d&Mxc`q4ITRjAekt@MOz&t!Tg&YCY@WAeuvnX zllr$WL3g|mam@PT8Pl%`F%zZ+is}O|wu5Fu2oPQRl6&`_YsvUzjkw_UDYa8OO*;Wc zBH)<8Ad>HBKpP7dq*XxQm3rn&QfYVZB{I!Bia2v5{Zv(c`$VAX(e0Yz7p>$YoD znI7NI==+MnaFbraDaGt>=z!++P|Q4PLHsSW>fNr^Tg=#Yfznp(hy?H$@z%G6){9^I zYDB0Ckdenp`w<*Z-|1IUw+tn!XrU_x7mz->+ztjdlK|FDIawGau`|qN%mps~BJq7e zu~a-03zf5cgt z)~?(e|9Mg+<>qN8eKN8nimHyxd<{>$v};EEOZtIpF#aJp{UZ57uH}ZVSvmauYMeFG z3Q^~FqR+Bsm!9#qpz>s*6Uzkyh+}dpi~ADuTbGyLFBnAOF(~+31HUrDK6z~B1z@K3 z>`L)GPJ=U2E*?Ph8F&xK7z&0KmU1S5;Z+saSkBA-#KUl7`C~wmcc$DhC_MaQUk`Ml zLEXQ;T)z$7{bDTbBbzMt#_%VdRF^f+5&W$RiX1!}VCMa@%L1W{%FZw)L&>x-LVN7T7egFH`XkYkicSPXylVF?rs-;suEG6Opljj>(;cIre*f;l(1J5{FHW?Usv>dVJs-xZlE? zS{Ig^AaU!z>abnX>s)Kc6~d6j^69%hrq0*nf$`w6LJ5xqR^Ac%`-&EskmSsf{gAD3 zCyM^$mAPiRL{#p5nykx;$AR2!)XlsSU!-K(dPBn!Y$EiplKJrW_l)tc+llcfw|JAO z4jw5RP20W3x|g3{B&8+&@+0=6w$c40R<4rV$sKpA+-*G6tA63)EK#J}(c5t^WjA>` zLBGx%bPk%Kc>S)X*2XpxOI4=ClH;!KZcT5oS%_YlQ6LB330gX}Mzd&3nnL(bhFrEc zJVFn~aEIEy%I-ef+c5hTSI=jw(MAmGR#fzEfNFl_{;6H5@99#+S;EPYaWtBZYhx0u z_6wKN7*J?ad0%JK!yU};wBTY~NR5-BaI_5{+D!ZGwCJIlvhwgjl$GaUl1>Np^&ko; z60TLfBA{Gg&p}O;da7W_`C{O8UY;Kw(OgBVC&Np>Aj_ASV>K+fRLq+iY20Q=f{a(USKv~KC zfh6!5>~77~p77E11-`3A9dX)oW+}b7F~JZ{qQH;OPsHBJfLYuz4}o=xW#IJC@nK9L}*jHUg0HpCo?e0}`;)G@P zhV3{$_PopoI9C~232qo3XUl-P_^q-)OjZRme@7-T*yavUdMZ7Zmz#YYf*Dtb>Uiau zQQ})>sm=n-Vfg|4?A7I_YO|=GUiwrW8J{DQW#jS3RU$`;_RQEa&-xEc)Ht&a--|T7 z34Jl1I$S76cXeg-h|V}~xdAihZX+=(t7~DNSbCFc|Nqa;8g)>ASkIH{KF4$~v!m|Z zIVb5Fm8VynCP6C#Zoc3E4L69H>d*5AW=x;qng1DK6l~=~furQ|QjoWrs+s2v_Dv;{ z^=R6?*$6MaD;8tF*-hui%&2pbjQhZV4YYFubJ6))`&gjIaQWS9ji{rSQQTTkA5f$zU|-(c+gwDO4R!+s}>k`ZKjMWgG4zs`&i?mkFQVw0n&C6An{ zJHqp7O_EkpQ4PEL(n3-T;g>XZq@yQ_(NHt5papS^)Ox-I#Qg+6q6lq_6g^D8P;K4% zK;Cyuwhlyxv;ENnpYNg#=gNbx2=`{qOP9~r)24)YOy!vv=;(d`b@A{ogtP@i0eK@- z$>;6r3VTViJoZg zxVhiWetwX5-E*_&zR;whUT2@52$iu*5n;mgx%qdPvloX^u+u+RF)!y$w;`A-Tyd;F zKxH$#hO*K9#m!B&RVL^XKa`1+GCvo$;jVtack7XYqOM=>g~2b?l&-4rh-kYMpV8Ry z5r$VSGlwl6nkE2?W{^JkJ%6X>MKPkyELO_aH2$5h7C%W)P*?M3OqRD(>*&Yo__H(? z+SCEpwmcV$WS{R=V;JlW#p!pa$dOxX-T~oo-#BaGbCDs6&=%#2b_rZhDu zX+f3S;O+!GKlgWRNZ41lP+Q^+0w1B_MClb1`z0AUCP5x{j9m-Ltk|&SA{r~;g0h+6 z;SC>(Kw+-ahDq6olaMuXcLQE8m!^YSm-0)!BeE4*Js*&nNnIH_52)b3uZ&J%Y>?bh zudrE>6!`u^YjcOI`5Z?wj4Yet3v}Yu7^2+pk>?F+5i*h~;hM+yCwM0ac>^!UCnonT zkB0*hD#p5GQ(*G3`S=?Lqh$^v;wM>hjEfuFBr-hQ(bmnFCAIz<2y4NTtGwDyQ{rnBnhK|hp znVfJ$606D=?o@}#R8lz2TE~$}-vD!{$78#}lZI{Vb=~#tte3|MvKZXW#;O;ejvGH3 zfkp@h1-{knMl$w1y2x(~X+=*N!2 zm7){8LrIa}f)n8J#}34*`7bDonLMU6ZiF*WAd#X3#v~G+czIAA9hsUsQ~B>b3OL_2tc;)% zAbZt#^f&+yTyhQQ_2tAL-3Ck^{?}K#UXa0Lo=w-r7T48n>W9Kmo<^Rm4WJO!3Z#0+ zUt@v9?Q!hyiKiggZdVs=-j%4?;VXs=I@58???Xu|TCsyNDW-8>U05HXe&XpLmApMW zTVfJ7Oi}4m0wNi}Bj1Fmjbm@A0RPEG(iX#QX2H_6;3q}`^Pmu%79@x?rB$3-(rT>` zI6HW2qel1PWw_LV6hy(ET`Pj1gvcG%h95%ZBJTd$XE(e9<^lk%wKAckuAE>S5P7#R&|#pNhlgjAWXjpx7di&R@d$FD=`quzAvh#Gzq#cv4; zdG~r2AVq0yqBT%tD^ev0nai*$VeFtDT{8LcyB~h0hY;uJGjZ$JIo=)t;bgz)MpQjn z7bNi>&j%N-T5dtZ^F>j&X~#UV{CwwFKea*RQ%voETtcM{G6NRFB~3S7<$IBKq`U*1 zLZ9nJ7eqax4l%?m4KqJVMVPIjWCE)C@xwp#h36;Wb{1uA2dh1gy^mrd8lmGv*1wir zMMTYcgy1i%OZuHQVs+_W37Q>z0JdaLPRj6>vhmu$F84tn2jHG9OjZ6zDOAuNF%v^kSx=q&2V5X5h&N1k67R zg^yhXL~9aR8Atb!*M!8G(Q54jYrdIIOCMxwgW1}tStR^(B%@RK!w;iq^GGEgpibgv z>u+zjIt`{bdM?8}@4ee`1M>7SfTc8uE==L{deKfj`aMS8Wm~;cl~iJ7jyXxeD_ZWn97AEhpuim%{i!qg0|=8R}> z-CH$tgmamdzlkKRAz5GW!bo@z%6QpT`xDKW>!E7w($Z3&!N|PRVsYR@pJB@X@1!hG zY`B>9@C1uHq#DY=o%`$l3F9??{QRl+9J>?|>Btb&ajXE=E4<6bo6+|QiOZ0sc% zOdx#!Gqw6TXg}HR#WUBzdDA>Pkt04Y>p_;ybM;T?JaP4eev@ja`u8ucoWIPLnw)_@ zLN>JfCf2p>GLLAIuYBLPYk#>G$1F#3=|b3Jz&yXPABb|WGI8k*a%$0kctztz}rtv#?_vD;!LAE9Ju6A zzcLv<8K!c@mq0lM$ADlf=X;GZ^ZV-ae%yx2RWzA7AwpK3eG~SQ2oFXIM}Zg7|H;G%b4O`UZ4FQ z@_k)Ru!yyOk!N%igm{Vpqtdw1!EeM{;8hOZZBoH`YVoQsd@p;yG`%3>7~Ed#My$0^ zzj2i0l+n~69YJv4DY%^R^?}vacoFq%{(Sy)^I4HbFw5#Mi7PtVNa86i*!76w+BUc} zvo7q~yGVV&jhM3eL0;UoO{G|uW=zCj_qWaW)A)@zZB=0{<1~k?R#&3f9={)m55)&W zkI+Q&c(Is`x+l53U3ZhM6zJlibdGrvQ$_KGKdxD6n>(2PrE)aFPK_s?#hS6+RJQQ+<~6(>pL?;YA{d}jiF=nF(B zqNnZus_Xzo;VmQ4bv8x^T33rEU1q*hc$=?ve#Se2C9D1~!ip(&F4Vr?W-a<7Tttz4 zzXk}PQ}HY*GZhIVld*@1R1Vmp1zzmPo^L1`;-<-TzKrpzWUsUx(YJk?J(J=Jk;gM44y;;8EGsoRZ@iG9aC9# zRy0tIUSRie^lLU?L3p-91WgKeB}}HuS2u8rcxvuutW%FyR&u{Qe%|`Hwajjw!h&QT zFR2r+Gg$?^Jn*9>9>jgm_!KI+4}(Y}n(<|$q&12(|%##_`!>Yc25fyHoy=0**#&q$MxH<2y8z*PJ=C5+F zNR4E^xy27~IQCRWCW@8C)R$_Ew`Jzh_y@HGiV}Pq85FZp`17AM&d@!tVY`jRk;6vR z{~2WyZ-^%uW~Nwi;TX?PGmc4B9yqrf2~t67&0O}$j9XphT+s2tk-jLQ?cW>= zMI2M5r_S(`?)$}-+p&d{a@*rp@&nGyju%v8&L&#A#aeEKya$fh#p_)|OECJ`8zJZW z@=Z?QRIhcaP_@CIqaQgzyXeP&|3+8tx1y5dc9{vNveNCuHH)3eE2IS zw3-bo{HvttXC9r^@ukMn?N#OFTo@mUy5!Oau#D`=nOi`>mfy`z+9_K$q6r*Hth&Vm zP3jFL5f{9{xsCfGhdj`p(B}eipY=D#mtd=h@~(zwQ5*)er)0Ij68IiREEb~Y;A&w70apo})L*;ciEp?hES6~mLpl?;rX&+g z7KhlUjezG=e4anqjDv<8_cg5po5dQYPQGT)+iR*esna}C;$%@qA3@0xv`I6R1IhC3++JvGz%t(lFCvkM|u=RVu1ep6|I17%lkpNiZ z%Iq9->Y#=9z*EMpij20>ZJ~-j3cuXiyTf+k0QR9zYUy!I>?x$5wD0IF*idJsf5uFu zx(G5^c?y~3j1`&w*5NA1G&wMk zku#vEo|nXpVS~WFG*QKV zXasNMlkJJHeT(C&^U2!R9fl;Tt2>j=aZKG_f$)C5y{#_h8I&kC0*BPoZrJS|>fs`T~otbjCtHOA0 zH$6T54i3(-&I*Ff^VKN{{aA0z`JSVUTj!g+iBU{EJShpDIQz%R%WYxqyCZM^v*li#~`cDv#V8&#p0jwYcHAv z#vUpn_n&}3fF{H{)HbrlaUdh?u7)5(rOglX2CNk)J0YI%k=q;j^qgT`M_1=(KUvqu z6Y{NN#P<^lK8MZcEIRf(l)tZPs=`+663EbsstL&YDgsV*3y%v)w_+W(XVUt~n(OoO zx<3j|rzus!I0T=ysJ{e_nZkEXp|$xf2=1=^&N4|3bS%|WX3eUn+}vu&n-`d(r!7g2 zD`tPd_WR3Z-!poa%kR&l#Yfg@N0_EbBIl5%5LV`k&FTI$IQLrTQK$a_n=9@>B-}jw zF;K&q`wZV*l-;FT5fL){Zjov|$jADaDN{a>z?bH+aWO(ewKI z?HX$ZF}Ig3mb9m*p&I}hyZwh$dqgE0z2W2V_>DdUQv$G_!Z(n9L3y* zTRPWI8yXrwxcb2oR~N@h;m(Ee15%p9Glr_)muvm~{i`S^%^ZCt0*U+L2D|ai6!#ex+7|Q&VwjT3YFBVB)ik7@EU2+vABy z%b<^rDc>4E^`dtUEmbM>7va<|%TnN%p>GCydp|(q26bX9OIsDCjs#+J0X7ISLQlebbNYMFI#T;tS<=xI{Y%nmaSvS{g*y(z2v30%l3l zd0D3A!3ik?vRPY^wB5y#DKIvK&ly=cRR*A{T!EngQ1%-9v4ffyu{M+zvC#6Gi#Mefc1WuUji7S0|qQ;vEBpaSftaa0TjX@ z050!*bv&iQCkV`c^!&~MBu(OESk?tR-0)wD>&Obmg8bN7`b?3%FH1 z{&`!vvNL+#_4%1tfL?z*Yh2vvufrd}8&j^`0a%c^b>)cDGwo|8kc(i*BcG@Cod3YJ ze&(S8Ig*!bKYf8JjNf-z*U;Ths_hJc&c?55IS%{9da=OCm3MH$99%8kX^n^Y{f=u} zYKi9ayfS;1n}rno!ZSujzTc;Q0h*@@Lc7a)`|A!aH)LcpOtE&~W z_E70I$c7%D>rq}J>)w7BnC4|pEi?qEi681qF#Xirje037TB_4Hu5;(t-g4W!@pRv# zAKqwcm1Z&WxF+?u8aimr2;GiMyY)f(Tt4G^g{PTHQTj?2+gdd`26oA>qIkbB9QZ6~~WSYn3 z=Y2q?;a5~y%e5DCh(r-au!z=V)H>WlQ9mOtorD${e|rf?jXb z7ILmu38~rcnHC$H6Ppd;zp(=SpDZ5y64o~b5BIXQ{x+$wq}t#I?!;}LW}sU5>Xn4kc866K1EV5PQC11IW7u2Jodg_t zz(OU3I+lcMC`VnD*YstJb_%wi&bn4G%l4sgcB6-=AEPg6WB_hDkKGy_tM48&wCUXP zDL_2{%zJP!otqufiCqa3SWZINn>^;7zjyCmw4O2Oc8Pt3LL>s>MY*H&0C}Bv8|iju zzuvANz$}^>RZG~BpNL@H>S_Fe&A)l5h86osUlF~TUWf+=C@rzS2Rw%tamaKvpyckY z4mZl@>?zlqDPJTEPGy4V8j+L>^8RJV)QEHu-@O(LaGvLoP4s!XbU*P^X>sxUu}tcj zqg-v3EFc@~@9{lvem%lYKJVN=o`yjRC=x(jN>zp{I!5UKTHqQp&V~LzNXSzyh($M+ zR75deR@WMaHKOG%U2*j(nt$}v69cK)`<2@hp&J;Q{7g=MHtNNuil_QEb8u-Un@XhPj3e4*g8zc*L z4uuz9uC8?&ZYy$Kn7MO0K&nJ3917C?&p{9)4mnXRI^QsVAgNq26c6H6%HBc z-84^jP@j0PNYxtlfjAKU@Jc#+>l{!zsCV zCU(^YDZt!RXVfH!|BIRf1gES_mZYxt=L=^%oS&IhQJH~vLTsz7xd56}Zt^7MDv=X! z%dV=|sZMJf#~Mt(Co1rn$p6wC{&+Chrtc5|#wsMj#?v z;QGrFucgZ$MliZ%dI5!+#Md;V)1g4@Z?}2j zL4#bult!?v^i__tdbM^=5#u$0q&#tx5Kx#EkBkdi(6&R!e!h~_VdvTcgb|;{uxb~kQI7kkj)of_@8Ty z8svcM?&1%kq$~~u@b>>v)P7C`wq;xHRYIsm2n954&6S$;iD5SAEnw-zzIQke>ekLM z;O zom(tXA={*g-=AZt)E!ArW9is0Zi0$i1u%{YN$S9^y}zLe2c6V(XKzq9;f^CTW92uT z`-9l|1HzxOAS9$HPbVS;{wy>NNU?j#g1D6w-JLLX`BEsws);)t$!p?*Q)K7$B;>4d z4j$x_G}7eEd@0lQk}|};X?L+%wVCYkn79tH&( znBMt8Zq^S>Ev$%wfewo7v=PCnIyLbcc~WCDf`_6+ztl^JB5RCEJBH|$a`NJ2`wgmh zVcGx`!Glfu=DvXIkwT}9F23K6ccIbBz~2aukbr^%@izur>#;%FaZib<-Hnj;VBA#y zrPk~^{^Z`pP=_8Xh-d%Gk`61odzcEh+Y##I$;J1=b?iJnpmtvn96(T|xe-AGw17WH&Zpc>ofMoE30 z6j~^s6T&qb{CMRHk1Zk55F|L)&Q-Jj)m#xMc1SM$C+s5rZocLM6RcdrK{5L)5%) zObKb_+JSj#;R$TU1s89Cz+`ZBm=3*hzmuGV)xgiJv|r5H`t^eQp{&Y+eLLz~Y50a4 z5GmA(>I>4*NgtE1`9kh>=~OKP5z&@Zg6~x`P3Old!rQnz=m{*y;m3&m>5g2T8FniuvrpUE#smGu1Ra&?L5#D3ZR&^POqiBu&2&5J0I5g=DN@0- ztnz{ZIT-KL9a1>K^o1-?AUFEeb2&E+W-;Q)LwNVY#G=55Q)G&6W57nES>m1MUv@-r4j(zZf%>p8esQV z_Ln0qJ>@xvNvo|lhq6%JKi+4UexuZYpqYz?Mr>ekNpm;ZP%j!|D`9mrL_r)_gx01` z+f@sa>om8-5yigM$LfPkRSCcX&jS$;N$X*)5NTgDGz}U9DJnfTaN7-nO#pFa>m^+F zgN3n+k&#isLAHD_+lqT5C*aLUAb0$^Vrfx0C;?+aSP>9r|8*N?4z|R3_|CwQMi%@?ZaPVO=bUvy1~O9iq2y*$_()+-h`yi419 zQzCuld&aPCLLm3U=DP(2c^6lH{x0dN*V8+fL{)X)?E$}(RYS2=U zyWS7JD9%(Ke#QwTE@u1=}nL`%4U!t-pn_Yk$)+--B z7ky!Qu6iS(^ydcyKOkdd)M6;=M0sh4h&bK#?? z7X|~cYg76%eBfuL#Sxjep;BK9m%>M@-9Td}8kW#vzL&L=ec1ec9=>$UOnzB-r(Oq z;9Jta78o5nsOvM)$CiqG!tq14zqgp{%}+lG*5D2xM9OW9=#zJPL#OlRPL zf#OfNYq~Xuu+(!(UB@LvyZK}h?ATpCw_x6V_^o(g9Hms|@~_GjIwL`~632rU_L{df z^VN;&IgY(CQ$NgMTSSIb+d7X9^{Tag6AcUwP70~}p6l_!$0RXJJ4!I82v&QW0Os^I z3Sdc8&AtUfJ{^p{HaT+E1AQ)>wKlohgSqHvB2810^9{gb?7V-_A2?q%dz+OLcgO}8 zePlftA%NT&EB9qZpQa~#DCgL9C%l;Wl}?4q%LX8V9tm<;juzX`bN8>IRCzc-)Swu$nGU>xQ3gR&BN=CO~vJRbep*|B|M6* zdDaQ7PoIjrS^WB7PiuuiF?w5Cx&81CvJ%7hE9v}i0qak<2yXgQ0=*gI{SQ^udqSz^~P-x1r<3fQ>%?HX#G zuz3hn%;jKo+mh*%7q*6gpSb-1STGM6CJ6Q-~cs~ZY@7p=ckW6XA)q0^75tQ5H^DWQAHFS8MexR@w<)t3t4V+5kVv)FG zCjUysXSOeLH>0)K+TFPof$iF^et$josTM-3pw9_jn6R;7aWbnydqtSD)MRop;Vdq2qO%*(1RU7TK+?i2zSwZ>ZK7uUSu`L{eNfm~iNnJ*G6 zRCA$;58d3*>@^>!S^{ntQb9%c=bZu%ZvL8@>Z#6DcfOW%tRRAaN%;?2QHM z_|O{KWN=>`-8^j=!U>`}apNvvJR-^9gFDLHzqtAIr5rgX|2sqU(L21mQrCU-pp*tm z>*y_l|Lo$^Gqk^; zW3EZ>!km91B}TK>R4zZjKz0Pr=B}lYh#=5VC_D}QfE|dEkjldH3b!eN! z#&goRy~$ElgJDW<=iMQ51zXsM2!e*ki@dMo2dvLO$*SqRTrD~^wdB_DhyKUYjIwvcBL?sa1VUU>1kzl>KoA}-9 z@1+y6Sx<4&0fl5{W&N>x(tq~M&vLME= zCZb$9Z{cncp5Kk0*pSwJeuR-iAX+#cN4$JixG3ekn1p{m#|8MZ0>BExUKaZ@ZB*+H zF#O!kglG!9ozO$_B7%Y)7b$5dPza9kh`pzl+oB!pufF~6-Jrv^Ozc0YK7y-l) z{$l<>d2N_1%z=X&bv=){Wrs$cdinU>5Y;HeZ%0yAf->i5A>eNHgQa5K(jVth#K$BV zDrAM;O3n5xV#EhG6@eWd|9zxZ5=DK@(QZmMw6tnk|Fy)?pF`)z@jMb&fxj5K57RhDtI z2Y=bd^6aO{1gSY0;I4u1|AVt{bc)Ah112c-#~YtC#VKg$=67L2Bb#jhA8BtLmF3p; z3#%w4At{Z52uO*9G%5lLh=jt8ASv9WbeD9)HjoAZ6_geUl~7Wt8>BIph4X$B;4DT(#C*bIxDP;}tF*lzOES0q@5ukqGrT0Tp`Iky};1Icy7qeMh}* ziuYRF%Mv`g?+jNVwZMxHP0Q)#P8$@ecb9ZBvK-QWz~6;z7K zk^QxSDhdMr^DWs?*OO}7^CgcsPj(%@&-syfrF8>#k1PMP(?eAml2>3=Tdlsyc_Fe< z0!K3Ia{`H%RZRD)YHZ=S;$HK+Ql%cR-}3n2*xRRTxW6Jo@%rNBJT?fc&2tvNX9 zJ$UdtBs3dt_F_erE6=4@T1F74?ojNNXGQ4uR+GWvW(Zp=5E31>#9jEcxDAM4+W^Vy z_0iVE{x|C!E-R<&PdT1bjUefHl}9P$9%W1Y@l>PS`akD~)!dL}Sr`z2QxVd%KLK^h ze?+8Tmw&9I>A%3}bU)~@{5N2^30LR7b?e6N=uIet6dSqgMBm?uIQ8))MYRBxtLG`~ zK88R?k0N~PJU{qNiCb4@eGWl0o72-nf3Fba(uANfOx_v#2h_b5HDn2zHko^O8IXv+ z+O>P6tYpUIrJ8n|!EwHqoYdaA+DTF<;4)G3Yd~l}Dcwzhgmk3)SaJ|mz*cvm)8M%3 zxzb_s5Mye_-^+1sD58vzTQUk_E^M!ZKhNY!M#l8rplEsO$-FfZ7D{)=0giNswrO5( zYPM5;YrCPOp(@Av+ob&FuK^X+zwZM9)&g!-z{bPt?iM}_%+XBl1Xedrhdj|~{k@H) z+-U#3w~#`|-mp8;kyN};EWIj4$Dcr2>Zw!&NyA@`#vgj}10hL%pN8zJwE0bg1TwXhb*^A)TVk3SGbFV*mB)J%4tqpYKrHey0RJ_64-wtgGgZ#uZRTdS zJ*l5ZeX$_PfIl#N|JcFz{l%2-Ysj3>R=%bTnZ=^qs+Rbji}KEO>_FhSC4evt%dfjT zLrrFfxqZk z!N5}a@Vi8PP4PeR?q~7&8^lE9!qas&o}(z(#yI({4v>s#(3oXdnGW8LM3FDGMJY$d zAPJI}9~KcDq7@x1)5+c@bQD)c9pm+tzotE^Ly2Gel^11m5j|+K;X4$B{W}-qkd;zp z-ltnHgT2Q^tZ(5E4U6X0H|2h0d+7(Pp*u8yx@}OtErv4ubTB<6AL)KYJV4lE3HAp(C zXPs(&RCbEj+a#8TL~@B2HY~$X#DK5&BB6`#;6mG-4g)ecoBS(xX*LW**^Pb8;i@DiD#@-&p>d;1OFwcX6v z`_J+mLK!Y>idG9#P;C|21L^Th|G~^X@&{Vq4UQ+>pvH$|k%gV=RcXfG$D+5D05kB~ z?IizPCf7MX+MIia934FU#0HY*BEyQDxeGSZAH5i9&1A1l)Zny zQI4QQ{1oG^nfL{Hvo)$Ote05f!GbXVM58lWK>r3JJ@N0O>tkr!JA;Sk?dTq}+?>oC zJIg}3x2fyNZ{LU8TdGqg%a7dpr(*rWXhjhs)Cz*ugWpvvauFuJ z&37m}*jM6r79-#JTG6?$&zL&2#5f)tm$m=Ez`V)F>i9BU0~(AcKc{%%K;?{<^YLoN z2vw)3O5)JH>wER9R@}tW@=!z)a*5w@?u?xjq{w!`(pNWzzNxmZDt%F}Q&l{>yk*Ih zi6Io)8^52&(iaZ*Vq{qIBm?{CiY z`I$r}+KheII_+w3fHeo766$Ex$%g;r1$f zlu5C~<0SA=0u6XPa0Hx9U~^#16I@h+r1w1D9T!C>bV!{@deeB6sSN+PS;SMl%l}G` z)%haCmIFyc=Sx-;ugl6pp9cI2e(xOIAAe0KV(uV!eM^DpyVMCC>$W?fbQp5CtZGRs z3!Toqjy#!K1ios1F!25Hh`7UG%O+Vr!;&qZA`c07!%q+HWd7B*80yjcUsG29RWaUf zc1nwnIFKJrCVS`b3$$gF!ZrOQ_niv@9uFQIFQdq;O1bRu{`e3r20|}}zx`;f=Omol zW3|6iBL_q={bySQ7T9t`pz?M@%1-WunLy;zsHS3}Wp{7QXeVI;II-KdrERh z`7p@3TVA_bB_n=q@pEjjg{5B$baj@_RL8+PWlZXjO$5@6L z%qT-_C?~xdA3S)VJ6siHnwcWL_!;Y=h5mHgo9)LPP6vA%Eg9-Llxwp$_4LjnD9!3r zYi-6??qB)5+Mx~0znW~!1+&p{TUY@^^w(hUHDU)bL_4sFEwS+dGX&N%%ImLq-0_e} zxJpV&FBs=UT{q_Mn$*^QvLs)y68bAPw&z-4Pq8&K41CfVg4y0KxJqg|L83&2aI{Sn zZpb)UdowVj&YE~lG0pdK#uxOTjn1-}Hleat?(%{GHH#DC9-<$FpgMutRzbf8Fsvo7%s zjtFm!eBu4|#|Jy*2bJLD_p+xdMLJ~hY0~cTg_4zUkM|3?jeGkW!|9YYHhrq+1RiqvD{`!^bSK*URg1+$+NuCS<8GCO_Y7{+8a8d2QU}$J4$F@Aa zI!^FIkhn;S|L*AY9RY~8q6qFs- zyP{jIQVCVn)kAO65-VV-gup~3wu#HqOd=IqTcAh{)BLfCf8q?jG^5+17sGdg-sAMc zVl|P#5FA8H?r5YnT}*kgCV8+TDdfETa+{8v+;_o!!S!LicV2=^@Qbo5kB6qxk~$;> zq>hGpPlZzRt^Q>MBO|P8=^f9UIdg^hC>-a!`$LV8Dot!;Se?yQiXh>9ycrL?o%_)* zaE?(>P;3qndpbD_Qlqt1E`lX&=(YkI(_AmwB8j<)XnPCeneEGZyD335AjfTtL&$pA z7jANA7ID*Ik@!`ZUy~h#vIvYJ7=MTC!*{w?Lu;8nR)Aak+0t{T7d-ibwiSvo&3OhL z=o`xBDLx51B-B=KvY^$yCMcHBZ}me;SEgrq2zO=;Hui_3F|=_BL%#L#VYHVO_mXx-tc<$X=X%@GmmKG{iUfx_plz?b zHs5oefURtIM(5zW7oH!N9{~8KamVia(>#W z8)^~tVoQ>pnl#Bhlalowv*?R@-uFnqMqzKy4c)%C!ig8tn(uV7a!YaJ@^l2q=q zsw9k(bUYrPT%p09ZMAiYyP#?;ZXbLtb(e;#;9phF&MJmpE(S|DK^C#X0`vQ_m9Iz%J(farz;p$J55Pv9)`+_w-}Buxb;(QJYQe7N<`=ah$)9gn=Zos0 zW>cE+>TFNM-K1@FDhlZk(cQ`~ZnXq}^HpmECm_ht}MkxavXEo2eYg zhe^KCmMTkZs3oI?B$9uaX&`kuL2+g~c)5A9=jjf9_A>2g8LA9eLz(T)JC>^wndt&c&AOf5+IjhE$2vZ9t}rVaZ? z#=S||I_f6(jN%6`W;pS6hRm6Q&t%J|lk7hl4vhuDdp$T47KwygR8cftO0}xiKqnAJ zm5&f`j7NA=NHfeERvCWFd}q>nULfS$<)T-M-^x0lk9dNukZDlK$^<3@LmnWrYtIGQ z=Zov9Px|C2xKZca$q%~(Km2ZC%~P+z{<()2Z%#I_JAQcy99Y{sRDyC!mV-NE>}MI{ z+iOZI?)mJT+gnYOY?s`UEHMf{O75C@ZE3z;UB_ZY!T`HNAPV>Wg})Z>38`JA7j0q+ z)+Vo?3)JDftsQbV?Hp#LLAR;+{Zr!Cm2O=m>g*}+#Kinmb#&c;i?;Bjc1VIN zbFT>cFG|WD&1=t`Cj$%XmUFRYqBg8W?JKv(>Pa282^Jo^vDYcm_(~kZj9I&TjeUvI z&s-P9>If;)=q~$wsbFQc!fenI%ToTv&awKZW2OaJr0e%FXlWh2UR@f8ZJv7HF;ld{ zXuYJCQHP>L=zhIT9{!Ba$lC;0o;=r_p@KybCvmkMo?a@QuX6J8`Rz?2!~OLx1+5Do zNj+CB6!8Zpj~qEzy?OoWoluaq>+BAnjtqjbX+Pp$>=X$l>SpC{-C|nfh?rf4c8gZt zie&%BSnrD4=7#SGYxVI?92XJ1VoU?%Tx)+Bu@N*cCB^q`_e&k@PDn2G*<#)tBxQW9 znQ)W{Vu)k>e0ifRu#ZN=sGdkzbmGkW33OYT^u$DYazNo|+0MLbeX{$;r-8doi#q$u zprxS9Xnwp8DZ)*>Yl&9Ef+9b*4MgO{ju#31jGW(-ZX#AzuNqDw^)O7cwbuk1=$nHYwX)C$zKGArgX<Z6 zJ`xfW*MdO~uk-RKrT%U{Mj`}G-GS{iqseCmntG*0*tHhnMPTtOKk1e~BJ{~>2%&em zJNv)Ai^{Ph@3#KDQufU*Q=}A^UdPU~2lO;z&RlBrPGS+56Md}piV|I?XM??DoG)@^ zFz*LrD~MTT$P%MRbq zkh{#{#&P0EFl){^VrQNnuO>1#>rWhMtT5{WOQ#Hssga)Z^1WH1rEbT$V6-`{!-7Dj zPT^fq11251;T5k0>#Bl-dloC1I?rW6>`NDtS?D~Tv{pZp*CT^s1V991=Fk&kos*|p zj%3GYPyriVg+Dov(AHeRqM#3BmI-Z!b1ZB63 z?*X}X?RF!lRvy`LsastfRgGPjqxXA)*<2`pVxM zPOmCCZZhu!`?#i-(KC}y&knVM^YW+ASl`wOJ2S_M-rl?`)Lvv1QPfl zb$`uV#e^T{Cj3p2=6Y3%krSa?$C4mN&uQTL1YnApCe{i#o7k5by{G8Jtp*xurT0SDo#b z4AY9h4v7_R>3L_VMHss6`OEf@VMP94lXuUQ9YpU2KzlxG8)gYq94@1C0vL8nU>(=W8oP`Zp-?dE&k4H2YBDx z(qRIjk-+>Ju>>{n&Q={cTj?y(8_qf_8BgDHUst%-9%T>p@I(TA07Z@;a7jW!Us=rx zWug+nQKHJN3-e579A~q&VL_jR>DC2MNW7cEt)}IyeR*eW+Fie#;gh ztzErR&5Jp83`QJyYd-uSy%%i}Q8eJ8*?5Twie#m@Y+B+W#9%L7IkoKo4^6WN5xY}T zV5M1#n4QFnxJm0{{izg!$+nrzwoYje7yh-FC*oO z+QyUw4g_Gye^9bloGcaMs;GR5Nvm)XQhv3AiNupN5e))n9fO6v4^oOQ>t6?%b@P^&O&krJ&TbNcz#k#4o@&H4!iTZG8-%KF)k3 z(IU+Kas5@B=jWy6F=I2pE8F}*kv}g&DiR<>+5ANzji8oPBzQ9hR9&sZ5+`OkLMXXZ zlCoY(ji9P_SavdtHVYeyDHAbE7~B=HZ1E9ca%NQLuG!i>p+r8y@RqXIUq&>r>5@LY zk7##Wr1Owm`Lc_;bHJrYX zseKY-plPV>#8vj%fDThwcWEi~vYA4_n%S(i@tBJf(M%MEMlHurM)_&|4z*rlZ2`JQ z*lv_hQms|jSxGDJl0TJ^(2_>BLaUs1Xkm>=p$k`L#QjeWZ%#XHW{%DkuZK=q?dl?O znQ=e&@J>Y7O5It>pV@EbMAR!hQ#~F1)n@ckquW%ld-NLoKJuWVdr>BEP%)_08daha z9p(jgXG8=wZ~C}D*XKp=!#|8(R!)#`7(B8Tn~9ujXIx@MFpwW){k=Vxet9kq-6p={ z7P5vIAf3Mz`%+MLBBC6Im}+bcc$RM~UoT$p=dBN1yyaprB~(F8sihJ^JdkFu1$3u>blddKAd1leZon8-5Bw$(Wkih z_}``ChBWucmnM(!eGNUGmoD{9vjYx_z8oA8zY?g^-tw-1w!p=BW)XEeFEE^}F!Ir( zBP`q>gOkYl89X-i2pGgr#Mal+2+_;p2fHh#ee1p>EKwwD0vwVkjhB2a zxGU8tBhCp3%y-uKJjVB`dh_O}={N00d2ja*gY2<(B__PcDPIPMf&M+R3{i=0)IN%0 zmP;1YXuzGP=P%qv?F(BH+UcBmihq>-acC~Cj!y5*gudw=I6Eub9 zDF~TGZ{@{e*E6%QRNIs`3zg+-qq5<;E2!6W5t%Q)*c!VDCSxT!FFRyot}BC8n3?CHab9gffWOeT<7Cb^O*hvX^h1f>rOT8ndIefxu3-uL_1hQcOVxS; z_2_!D`pt|>C_7zqa-4^jX7l5-s?nD@OsG0YHnhVVYb?>8l}U$S;%z5O)8FT!xYAPX zkqykn_ znD>`ByWm=7p1X-B{hpD2wsV;7%VqqE%V(f>F2@<1)TT;ok5AI)860l%PiToha_A56p9}_oKg6R-3BY2&<(2o3`kS6 zj3$=yBkhdsR;@Bqn&cH|Oa@S589iw&EElUxw|Hs0MY?#Rihxr~`=8zp^N!gH^bGEr9c{7Qx3 zgqcXqlKG4q(aO&lRzD%1VP9#BD!0pqU!v)~pp$vALM*}z<3v=R6s|p9Yf8gOEK-&g zde%QLAs1H9A5m<#-koe~w+YRMF=(J_)pLHQ1?y#{LZWtN352%M*fX7{j%AHk8ZW5T zUVqN)M?7 z84)*zyi3@NPBb%-og)$8_xd5`4DL0wfmlY--#*lp1654<&0bq=s<+v_0v0WtMAkPQ z#L+T~I7f#V7!nl~MNSII`-Q7Y7#E%JbhC-T<1QMVUJSP*3Gr~tm8_z87*UZLhnO{` zq4<}S#oNT(cDnB_Zf%Z4w3kU>Ydx{3@WtCsdv1kSFp=o0C~VdES=?p5Z0F$^6{ux5 ztEq47dNoqth#wmIh#^~Pptd7dE4_-QLX!|uUA!Vpd(Wp{BF|C@mBRcb62ZE31EnL( z0fML>&1t!yb~>bKUz2#15K6iS@fk`@;+flPGifzf!|lnP*B1tIY6UA}qWc!Y_JXIJ zeB9A6#N71eZBHGKCR)0KDJ{M7clfnw873H}*9JHu@dP-DcTtH7sRgHx_V`RJIe(r{ zlvW7XdVz(xupwT5#+k+K`G%hV+_#oo8GrKN!4s6gA*WI$)y$<*)3jj`qp=%BIpN zFd?kWDsDwc^P>dvy5OjSzwtH~}kUjSNID67LnPhxxw6MIEgg>#kfq)0&7p3}te(+}R+$Wi~pc)#Y*A7gAjO{HyP%W(do^ z4fK@V1Vzuq6ST45N9@^&)hKl~kdIS1!|$3f`g99>4R1K#e3$lSQPwNTspgT{vX1zZ zvp(eGANb`QUoz7fMv^@|V+Ur&Pv8eIHq-%-~NoH*(+42IN^Q0WfLY%}MbbE7M zbKnzY!9;Mtw!iUesGEcL#;|vPII3yONyi@iiIZ|~ zHFw8@!Fz$6ejt3Yw){m^N>?7Sae1BF0ZBEZPTy0%YsoVW*!UjtdNL!4DDs;nd#DGZ zgPC;mtj#p$=y-9xiQ1KdD|K%W_~2VBOW?C5KPjLh(n{r+{4MlB z9xWT&`=kL+h_s}n*mYuxOcgiN%Lr=@;T}d4FI;_E`B6%C(xoZMoHFF~YB$gqpR@O1 z`%daBuO>Y@KOx{$sL#-;XR<1qL48UjiasZZ5sJ8r$)AP0lIr z)ZXYDeic@5H9Mud^68Ly@btuhGpG-H&-7W!n+KETA{AcXUndWbR%>+wp0vt113n&@ z*YlNNgQXqfBKq^^iHb-Lg1kW2exNA~{N7ZjJ_2*V9)ytgJl8p=b>Tmo|Nc)RoHje2 z-xUn*4}$LebOX4k>seP=^YB-N>4!ce?tI9K5;_N_n)>hW{=6@0Z4mLyBNC6FKieW; z)W_5Q+O4s_j8h)UJh=JYFmJLJXQ$p_MUY+M!%33cfQq9?LUt1&62cn5`xyzzO%0ti zIJf!W_TB#b+b5-`cLx?UD7k;4(7!+awaX*XA*j_k~Krua?CB^8M8FCIfJ^ zWfKCW3)#eIe1TCfg1lwmo0LDHhLslZr2Mr8r^pVExYDFTiSXj`^71pO#`Y;G*5;l9 zvpj$@DHWxDJ)ggnN);2Uj3#143)+_gLXCIY8H}ADVj5ky!jL_)%4CiE|?YAqxcHq!o5EO!1)L@Ciri9)Mtk}wJL(Kxd z^N9883TB%kFMTG)ew^g;3Qm`QlIOm&p_D%XoWoZa)&JZAp?p!_m!{^#BT;rM`|8U5 zK*>rtLVn#GVg&M+79J#V)HHumW6PUN-4RSKlr!q@X0nWg)drmTNd*E~68c-p%F0e# zV^K>{<+~>Uzg1IY6q)GA)bROWJ8^-P;6&)Rr>q>cxn95GN@_yT|Je7x!a6)mz!rKh z7}d76nTlBC|9hzga+E)saaaWi%yGRmG(}`a((c3=NjolxZ-ATlvbzJFMCY5hPOx72 z{U9*l9j`Lf7Kmn4X!5WcyOCrWd2lrBAY=S9=t*fTsCzl{3Zm zxjxB>6QWf5FWSv`doI*h{qDAqAA^KKPxDomIV+*QI;XxfIVI)2c}HvjTlOe^+~U+9 z$iwROzi(Y-yx>q{6f8|bLQ;4=16nVc#GkH79QN6%4D#CrD|mhNmF47uvm9#dW`m_U ziGNOa(3=#<`YCC-I%yRDvkjhHm0~&XPIB68w)Cb`mhF=6nZIdH6R#yx1Mv6dJY1;# z53CDFLu3wIvFbt|j3;o}jAI4Z<+084f+qQU^P6W+pZd(snzO8)V`b&Q;Yi@LtK!iv zp`sNuSE}W8>v?Y?WHU4m{7>b?Sf2EPLkzAER15 zKPU{xm<0sd%}S_e0+)5&m6UY%Pzdrd?qiwb8e)k-`7bRkb$zg91ykISHLCm?v5Pzo zz3}Q4H4R0B!omvfUN)yCXUWn&8D?P*%z(mdzZx?=*ZiJ-4FI01cwX;olAG{u0Yq1c_54DDE+Ll<9C}6T zAE%tx@z9mXY$MEK-sYKMK6~~{M_0qbxcx=?@U{MM6HGLAuV_xK9IYPpJBiI>cxdzP zDOZJnPJOi4NLRzk9?9Xr6VpE_OVlQJoj0cwCqT%uw;8)0m2_febT-)R?>*g!d)1(m1jSr#zNQf?LD}1AgHj4D}4(I`sG{( zc(I%XTb?nb{x#B)H#Sc1bjo)Cj5U(@&`)TR-++evdr5VXA*W&Jj=g|p=zMuxTD$R=3=AxdaMI=;u ziMIDrnldf$KT+?7dP7T-;|D%I2bTz+ze&i8sev9w0`H>{YWRNTyggKCmK7FKP}Qx~ zndRLp=hr?t)6Mw55K-6+U&g^yJcnD4!Hz}0CCWkEZ*y4zISj|gzk$$117Y8=%Rd;o z?U}||j&HD^uN)-3uxCif{kli<`B*HgfkHm&+3H&cHZ7A-L$tA$T!1CED*3RDG(jr# z*rm04f|u-2H1*BOXNDV?MINx9Z`qWw0S}Rjmi;C5`A^40uYomFao)6Pb?#@7e&AOX)+;40QDs6+Ur4mqZjRhJ=WIkta)sxulMLt#P72_f? z6k{K2pi#lYF79aJjvVAw8GeP5dT=amqCcG1Y?h}XS3^#h$7 znr-4$G`kN)VlykuhPD)HiTrOEp&6;3v##jdA7CfeYG`UbscqV)gW@$eALn@az(eO# zA`~WSyz6**`L#IoZ^;K|MaW`s0AsT=ho`RCer3=;?ZM&;UHe?OtcpUpFJHM=NxDoj zIB`S<=APl`{b;tfz=8F=6x#Wj+RARY*K6#X89-^~X|tjZgByH=pIlY)wB|ykot>Rc zDZ>-2eRChqg`?q}nk@KE*mAyW9Vg`dZb~~O?tP8KPe^nw6JBSHJkNgM|E2=AH)PAs z_LK!}RVPNRc!DDjXIs&;jkq$(dT92$rkdRSSU(7|w6ea!tQb)PbAp(%Ye3 z-87}^@=??&_m9$x+LcisJ4tWc_cV|%h#-Kl`UN$gk#6>5kyK(ZRO%HH(C9!LSu- zi4>kT@nsx4nd29JUR06i8e(7M5btz|pyM!LMm(tcHmVdQPyr>BqT$M^VzWYvD8|gw zBB-nm!rl}0Z2)pA+H&8mx;!8GZ<2*zpUaCW03?N?QDyZcYd1`h*g7@u9f|Bk-06No7a?B4>Bl0zVJE&97_91wB#*YID)KNl}G z|MF^rAaAnbIFhP~(L33#Ha~7c+gIY%l-r?fc#avwP^`(V1DltTqc7Y7s`G>>{x&9V{?{8nRFHi-Q%l)&K?0J*rh2s8LC(y`k zg2g_<9+%^+>lTnDhD`WlEVkQn6%h+1A;Jv|@3tM*S~({%K_2WaP*2KR!13RgptqqC z=8ijqyC0U^`Tn2gm=0rlwgXKZ%~D$P1_#?zgwls|G{sn;`o%;HS~89BC-FXJ^&SJ5;R+`H*%-s-2w?v z`R|f-^3v{_R~_S~dX;e+>Sg!aOlE}4+(VpOj2Dr<7YxG(d@E9~0+%1t;A&A$8e*L0 zm{6D9W7FHaGC3Eq9Iam;}!sGI|A$q5{anxN)zsN$lwzEV^r$v(i zWbD>EICSOJAWP$>|9`*y7hZWJuY&TI$Kw_3mEhFZq`(7zU6qo0+x zA?!ZNM;?+I-U<1Nz`Vu?y=Po`JYa_}Eh};24$CCTf4v@{Ui4Ox`S`PS`%jXd47Q$= zbxX?MT@dM+WxU3V0Q@o)_1FNioB{7eiOa)X@&nSh5@#)2>AQ*+fe0n^$#}%~1`zRM zK5CsA&Ol_~D_3s4WjJeh{=aKfETNH+RmmBDMifNo5&it#+&k3j=o=GcJqVmuwc3q5 zP<5qDnB5?2qnH5kt0Y;;B_c2pxuf4ww;_eF{NFx#w8$f}!6~v$_(D)V;*<#Z3lAbA zKcFGPOZNmq25-;XpFI+S98lsq{t7LKSi)PE{&&0LzCVPM@UfVH7K0Tmxr;7?T|$V~ zpl+2tkr4^BnFcW zJDz^2G9D#%IKLzna1Y!0pWZX}z;2~|h2zDX1oGB24?JMRH&6Y(3!)hxA~DgXyT}Z_ zzQa=m3{C#&9Q|tL5rJl}hB-nkg;QMoJ~74dFO2{BjfAwHgfeI{zz5*M>3@{^ZQJLd zKw;_I{XnN>d5~Pv>r0jp-y#I3+d<5ODoU8yA-JF1*h6gK+LUMZyn+iHkl1d5=n?~> z%ZThp;R)bOu?>{aF$4+P0#lY0ED1ru4mAnH5F|gwq^(nPAvKd&WO4GZ_xCPm>Wo)r zqwD}0%DBdvZ0Y+)D5c2x1U-VG9Wl2T8=$ zMcF?)wbz!;UM3R>FJ4w~GjintVV~=OJX;2^V5aY#mwZ4`vEBD5SG)79o*W~r9Y^lh z&k#!yCHWRm&dS_TM;$1;-m>NW4DmyMz7xx@=~2^J5mf>lWe0~soA1kfmF~PEZztKn zFpzb;DIr5*CpO$#jDFKCB+>cZ6Kr@t3bnrwbvxtlSL@5wzQ4#5i#aV)mZ^1;{xLK> zOesTuLh3nwP5K!oi#yI-V17{!*;Jjf8I(EaODKbnsx4ogxY+9g>{pkaZaEtM#9Am* z0WOjF7N#w-buu{Cg4g8ez?%hj)pbDv!=*@Y1jMg8e9pn+#HH z-GQH=RPX8fs_9by397jt)6iH~xOwx77Z%PP#=@Sa%_w`b0w{hu0)W{UHeqc(fqJl} zU0I%~1F**o^QB8Fcrx(#)KjR^jb$gA`lt3Bb__7yZNo5+4NAnrrTg1j3u-ra>P<@msZ=+ahKS^vW9tA1P0q z&MX&pyY*CMCLR0Fs5TUeni#lbg8>U zmbHsJggA-lcnt&hW@;0mbEV+HOTA`RkhQ!G$TQ`uSFfNzjU5sz>QU3chBR{Qs==NL zZmud6r*@w;0X3{n$<4i}*;!%BzKWs{G~p!f$cpqpQMY=UXLnW{?0hFK z0*=h+=mYr_>@btO1Da#8MFvQrx_t4tG*YB_NY+=G9=xK4&(qalV@DC9HB)oiKp1lJ zRh>d*DX|sA!}o_+k^x?Q4f4#b09oD3d{2@ZMlbF{j&6!qp3>k_f87eWb8NHop*#&I zyxijN-jF|e+x-eIoB>GB?M1f-jhBQGVPB{~Dhc>TnPO5x+ZjRSQ?9D4lKMOenOEh% z_mJf~rY=H7{$*J!P`Qa(;cIw%lQg1e0u}k08%{GrVMG0~I^=W)<|he2-0p?7GlS*Q z9Cc2$mBZ2j)QSdLg+bw$5BmkE#1cwwLIlDeaSjR!3JeNT-palAT1zC{d5Ct+uA=BU zQF3$bGT+5vs;1?^?^m)XHuW=!J5JIEsm;`&;LqGZ{_>Ee626sPxQl2_CNCK4{{YtO zVb`EMQ~l>v%8|Cgsls;EG{x>|M}CpA7{+0QHXy9@HNR)UxXP-+6)I3)i^j>@GE^(I z)?QAD26UDCrmJL~hLa`q1BoQzjCP5=V6k)mpE4BT0w6_b9G)}-e%x$jRj>72%pvg9 zE#tf})-?;9fh9L6LVLxa>pHfEGGX>zyP6zW@w(kX#z2JLdcm+1Hvv(iH@gqIfBcm) zgO1BomfpktK{RQ|{y?E@9>>Bn(AHIK`C+wXra&&hgs$a8BHOeX&Be7h35dY+bp7K~ z&$DWAC9LO9q$!bSYIKl8VsAdFm26vM)qKy(GDLEJbEI$s=D0n%=hayOjSTw9zg~_s zG?n4`FJ7ySp+I5Zym>P)BqTqQ7;r}tv85_9WEv{NFulk&DuR4((usd?W96P!rJJ9< zL5W4vf#w&h<{pr+qCdQ6gcc{_W`ruGOwFPpGRw$yU-|%j_^n z^;x74tG=z>q%r0+MtmbaE`2@>T#JTVWErc2DL8UH4?F}$Dv3C6KhJ2lj-(n+r|%Ms z_D`@8=nz^t%^1aJ(}kjdN>tp2z8-vG5d(0s*Ar{*6$*yvTe3r479NIM>w zrAz&;3wo2y$#wlMGh8K58bRa!nyKkqi#%g62L#!xhZnSxRf*KNTUG~sh=!f zPyv-rneQLilRtjKVcK8+SZi)8I-3&gDs|O3Q-nl3_ubkjx=sfW(mH%#B%;1_-CfXV z4fW+N=e>L*LJy!9XNU(jv{BT9xM7!dzWl(W*P;NW{br-Hw- zQy6)cmzU?X{EhXlGh)KGId$7J*wSr;_HaV)ay?H5^_7FYfdjH{*PH(5s`}SkS(rTrQz99`5;Vkd*ibDrhrgq95Jq>N>few7*QgWE zl0w=HeCI#cEIn4tn*;|fcYn^-Nxm?F&r6IVekrFK>nhf8lk=y3;;T#NDO-ZvXR?a) zf#5LBH?j}RNCS{aOy?g(pSgbhdit%Yxj<=**h;9)BR1V@`j%LMtsCUAo}YA|Qj&;KA{2&H-<WheCaByHBg4J~?D4?(da!nP3T zpB3!Jr%650Z$^}03h_cDFm+P)&YforBfevsFdUB&43$-q4#6-c$K9Kp!=$z#vl>{v zt)xE};jzlxUPw^9OSh9jk1Fc(oRrtuR>>f016*nZnA>;up2@ehbkEnly_?aN$%?N9 zI>W)At1(_9? zVe7;N@GXepMfcB9j*X4o*VYcNwY|-QiIJ4q>+#wP=i87t*uoS|j%4mwkJJot=xhbY zwpVrg7X5Ib1wu&<-y{$guyPhx7zBGzVH(coxgYh+I{*>;I@pJf3V{Y0v#@fzvEI0{#pfyg|rEb>6u(P7lel|aR$EJXMT|rIgMz)dTxtBp| zxDU$448yd^U4wntF65pIeU zKOaOTv^jpcJjiwagDh9?CPY!$#u9_QE-#A*U?#fRzfqRflf}x(F(1aS`x&~o^x9G= zR5Zf?K z3z7K(Mq(#3mU=YHLq$d9*^3#GxUUr2Sk);A$|t$`g_yI*75)UWSANSjKnZYIXMd#m zquiHKlgxA2b4X^CPWiCq1_irJ^EMmhjVI&51!4E(A6L+ar zlEO>}Qih=O9)HeN@tIlpEHwg_56BXefAsztJ)XdLx4u3tw>dK%RU%W%#S+M`4NJ~< z&ggB?NqT4nDS%G;?oO zG8_OSIr=S#;A1-|k2B%ujGu*{XE!KYN6Fk)TV!zw3ELyA0u21n^tq*yYi+8g-()R2 z-#f1uA@U$4!wEv<-5`SC#znHc8&yn}Klcj=ygUA^?2&%%tM_H3lHy`F<}_BP=1Q${ z2Q$G`zaD= z&*tR!MAY@P70-hWPv&#yt`nPl1NYm-{hfsj!)cT(wpQ#sOt{aF^&=-E^Fb7Aa$VMj z-(TJ|%^?p~v5Ws8a*mzb9khyf;x>z<%RTn2;8?mK%h!sqk$x_bKz*W}ClgDVr`Lz1 zLnz-g_VQi2)K)Wx9LhhoIm;u4v}7)Uw4V`!E?S1?e7VSeW$TV^*fAf@%B|7MQ&R*i0V}vO$;y0v$B#{GIJkOcm@55!(0<0 zud+93dF6$_(-O%U^{SL?*`LNlCAEcyhf`7xO(wI&;4lVf2f-gc>=Q-cx8-W(nX(xq zj`YzEGd3A>Br~acOQI*oiCFCjm53=4Q}p=lkR|M?1LjyR1J+N!_7v%JRw&;FLDzP{ z%c$-dd)?4Y5+ji3UudG;&9VcJfHCv^q=PL+5RE)Kh5f zPeWC?-$UF0fOoFHJo?Q=fzOWHw$+onReE#p??Z`+QhD+N_&Gj~jD#^Q+mIH%T_0Wc zU8^Xe%FXVIVsM@1ZI8lRiHV9z`=0WGjBa*cg#t4)dj# zRq}%ovRvKM+lVufer>2KG$!_;7i`bS0Kev7edlN`7=lWlJs>U~?Q7PVq=BI9C=et~ zK{1y8_U$Mu`o#&6I=!(Al#jt+0uo!TOAlu{vMfY4P}p|*w$z-K5NksDm=3)LpcGIy zocN|`SqhFCOA&R-vm&-^OMSv(+g%2mupBbz2K#-n@o&POK1jRHI{tKEfW9i_y9!B& z*oWs;OwTvKh#?>?$%95U?N;^H$~Tu#QL&~|A;qPHImi|RuA)-{hOo;)E} z7xK4y^9lYRbz%$uV+$Y%b1Z0h0J9`OJO#qnB}f?tZWS+6{2kAAs!}qretJ~>*)K&L zm-5Bl}xx*2$5e_WfmDoc{U=Rl|CmS2WYM;D$|)=kQV5k5-Q zu79wjc(Ff`W}2Rji@oyIB`(5%wH*pSieS=2`TVV8^{uL$%O?`S6|Da)>cQuQK~5HK z=?T1zb17u5Fw(c?#kRfWqp5LM7MQr@~sMi^vr7krw+{bkmQ(sQX;73pO4^t?QL z1qgT-mm@cu)$r;cS^S9mI&t!sU}`e@)hXQq?XQs}A@AR#UWd&FjDI(D-h789$*(UQ zGB#E@xSXDuX*`rOxUWmDB-~pYSy;tX_!N#JfjARL?p;EKi_KZk4Aum>Qu(NtI=ZQ9 zf^W4Ni>M1#@o5oGE7mH>5&tOkr}2HWQ0X4tsM;g8 z8w}dOxSR(%p?rEqXYcv?WgATk4dQ-XX!f*y5ChT1WJ_wmyI1mH1__Qa%)fbw9|=6& zUo5j$AUWAgWX~<-xEpY#bnWu%6K2iA*U_}}4R%dd8Z3XSlt2nBl_cT* zO!X~e(F2Kr^zZn+zt8!?k(Wx^g$y%+JB|Kybxk6e_hxz-2PaRNvZ@ys85w!KzxC>J;Q5ODMOljb6OB|I z-eXLBn6D|XcUZ1|y^!P5<{XOlpEl{i#M-n+RctJH@f?)pwf{R&yZjgs-rxanZoHzr zyu0i4%7kG`{*G-vRyjZ#&)qXcUb4tJ0#Zb$qO?ZU@j}KK;C39x~gwcN^$T04R5FxqtOIf4$~U%%guE(g-Kr&FPz? z(~lmT2NX)6u5?_~m;r2`3kSab-Ay!G7@(+#R;UN5L@Qy!ufO`Z<-7X4KsaJpNUEY~ zmY(_Re`GRo%YXj{B2s$*Q=jDVUNtF))P65P9%fqYtyhHk+oh%_4CTh?Z|d&{xmpSi&ylwglf3R=#0ZP1E{j>-M#6 zjaS|wV$m^bI#%=k^JxJ<+vn1n@@`?SbQ2E~Bc0V4EC(ssk0yc6Rgm2iJ!QWT*S?zX4ZN zPK^mw@s$wRcK-yqS)gviS=HD34WxnVce4F4pT6F0kgg-@0! zjdNj)+(8K}yH0tFT}!@MQs6kpSZZ|?;?q>U9pAq=)0pI~AelHIv+Md>R$AITISQO5 z(Cs_jDnT{qI`@YOD==PTCLpo}Q+n6&p9P2NY{LbusqSz}At)xLLEy7g-g}~%k)=Bm&J~kjYo*E7K2GghFHLn@8KTS!^9Z?Dm|#7^!?f^VSYB;d zI%S2OaKyvmeO!YF>+arOy}w@M-ir4<|3~y2G&$|F@$JtyKCtuQy0Qd8HE!L4Bq4>< zbd4(6hwLi~+`9z_-_+({89jz(Ly<2_C$@}J`8)F~4Zty0yZvIW4KhG+Dh#YeEcr$% zB>AG@>y3>Ma0KA0#ar}Jc&tw@fT0LvTy_q`f-zo=6(Z1mgOaycTlSCa@ z!CEQepa6YYceyFxAK~NrHpl;d(*2pb$sL|wTFunJv|7@cKjOX(9nBB?=j6M|EPNlk z;nV-su?_IaoP+j{nTZxSetqDtgeF@>YQRC%LH1UZ((3$RB`Y#gJu9g*S)>e7j727v)V3V#TWmz!)mL9 zSaTqV*4DcYT@OD=CPYBh^ON8K%f0L-GF%T{s*n8oj(x&36UveotXC7NDdHG616^aM zv^_pyOA{AklE!u}hJ@r88BdmaIF}GzA&BDgE2+Qk=DCeOpWqKCBbm=DD0bR9nNwuK zLRrcYKb>^pXQ2~Wl!_2d1RRPg>=1g*j0ZS=O0rUKj;8(*6cbYbzS}=qAV0=`b6Ri% zxM{vVCEY)^2Gt>#NfjdalLV|*YQgzmwF%a=O2kiNUDSGzLDZnKW*CF(MmAj70(u73 z@~~3R#`2dVq%<|Tzn>4?HW69lEH0Cx?7%gZ)CPyjDEQ|XLze!;*}Iw3$GnY?w^=UL z;gAx@^rEh~dSJVm_PHFUnJ%PsTaIbzujMBDzbXXqv#M}p+!1g(s^y>726pj{=~Z>= zwWIKq25^q-5S_4ux-DSarq*ls29Qd7;0|30X6arxy3FNX0OlCFZ7y#av>COkj3CIc zo|97Zojb1LcV7*qLvRw6B))iL|F+`-fdugTWKy+4;-CP%mTn;Os74Ctz6%QrJbLnk z&Xhh#;-UXbwC4-9And9RvDN#Ed1}vN6EG#{APe+o*nV^7%UjVsExUQp&vmQ1K<_69 z;KTGn*-S5&vwIUT|9Kcg+>LKP3yrEljs@;^@)NFywG3pc^t#7%i*iW1L#uxc(BNN< z&VF}Zqk+#8qpH};ZsRSWKV8>bCEE8bwpo{BA&9%llB%3DC%4OuoE^$ZhsPfh5kiDB zHO8C%;olZuR-mCB!_vua@)4n;o%7d&t9#;|4Q#g*jP3pxW&WQMO7Dfe~7g78*=|VeQ!+F<(}+X?YD`(AX@r*JCy^D*Q&cO7QgV* z6roFQl38AlxGZJ7x>!rL;EDteWxJ~kiFd#GhlWp72GYd!DmnMdM-}79h{^`!6Vx3E zGm-dvI+v6s0f07NlA7n}>Z8RvcRFsV-SJ%}GxlJe%<<3%9hnTa( z%oKDy2OH2KVX=yDzNT0yZk({v-L;fbt@@{3_2o5x7v~`Lxv4X^b~XzplDv|~7Sd`| ztX&Ycm-)R3{gx*3O;zWdb5bB7fw&J zTZ~{)4(&ok9n;BU`K})~)eL`)#wLJj-uP3;G55c4RO9N82J7e>#ve=FWLRfRNHzb| zaWn3hiOGYz9@;M#pY;fm=I&Ky`H_`$)ajVc4{P)kD~mYfD$gMUbm6Bi67Ng$xWs8* zn^(b<^!?~F>hga|$ULN|s7JOeSrY$pp8a+N%GjDZjm_@|T?>;y%ZjAFQa<^H4QMxx z^~XbMGqiq@Je)7AkwcRO=i zKZ*JjJhyuV*jUDP&O=we_#!VU1tq)Ew>O(bxib2R-nAI(j@{hX%evL{cG5@N7VtMR z5&0?6erTUpKYCaTM4n0^$u?33rgVGZ8mc$e595>!U$Ba7o?p^*v2iZv9V1g?yy`lD zG58YO-P`4Ca>m4jlNj}N&pqhkJGq4C+NJ`kuh$IJPb%Aw9fF-Q5oFgZh%cjRLMKgm z=~&CQC^Az3YVs$y-;OCuda6(?3quxB0nD2G(61c*z0Rs5f$@T{Ax zk`fz zKU5=3-s=6o49n6{?%7k>WtRUml(&FH-2Z^hPR6{h&2;gRe2Tgq;@gK+m_0$EaXe_1_G0XQ-zWum3lXENpDcV30+)fhi} zzFxodD}d{!MMw9!vzkEO!E7dL3QWSL#wT# zA`Xe5^=o9id&$Co71VC~SF3*ah4AJIOEEMcG|QOz;Ljn2ahesO1I4 z?|hTE8ux|X*<^>BDY*Mv+3)u(KR-IWSgEuud5XUsZj3KuY=w@9RAR|%Ie>TcHST95 z#hH#OkSKB)me0YkKAOaGxhI89?YXNA+K7>9HbQYrUf;mtQ+dDHE2iG>?Vq(Sdkxf? zJn3=%`Eb@rQ@-&b17qeFpS)Ya;ytf-3hmMktZrciWIn{&ler$he7fpdTUsW=3eAJ- zG;H2BfY9Fsxic~5uxu#!m5CWuzUnAnS)3NTqiI@F|Kf0II)FHgeT{Eq0Hjaw^t97? zqY$CfnvBplGJmhAk(H0GI-CD~jWINq_T(2~V4Tl*U#6|y)t<-WWvhM16)bbiN;$0V zNv2$pt7O;tp61t60AYNbee5ctTs6LBEb#5k_-m0Y`TXNxa!$`~;^?;V*=U-t3hL6~ zw%y6DaV=-X2M%_0isTe!3t{?lF_}@-}FzA5&_*_xN6ep=1U{AfIm_W{0p= zzYJ;>ZS~P?$xu~z_Gj%6FX=|pvq%Zs9=4fDTivCjdm@#g^a#eA-%dAKCX5h*ABtKT zGzHB^za_hLU>nR@u6rc)Og^Tehlui^?^HPEw(?$j`ic-#rER?WM-Fg*G&}Em3;(Re z(${(Fb>HLm-9e2oNV)0iE5vH`akp%7bmrh;~ zxaaGCy3+c(UMXlKq+?<;z5soQ<$^|?d=on2(kPwC(GWh z)^9%L{9KoAY^&(PM?&({HUzE*Mf9rh8!2EWezKXFOGoV0MyP2BDW*m%`5CJ5`d7ZW zi(tI*ZQtUfkr?DRs`w5RL@V7j^dWIF_K9(Gj7d_a*X?(?OkF1e9^LCrC0}wZ@{86! zH0r{V!P6Gh!{L`to@sUL|2%oVQlP>#lN=rGt^H1)XM2`T6!papYzORB-uGYrQ(sHE zsL7X;R;KEaI97t-iHM^%`-rtay+8dv=OiER64AZ`_ChFJc;WNkofI&5R< zpJw(x?u2IU;Vz9UW0k;w51)>C7Sv84)Od(n>fl#^j(OTY&Ot&c+J){EUIg@o%Cxs$ zC}tyAH>U0&(qBw#{tKE0&n+NnrE-DZkb|S6DOF4clXI+92~!dUe&MCV8)|+-BTIj= z$tm%2j=aXGTg+{2s4*Q=qw9s&I%c6E0)cn|?s~X2w^_0-H1N zu*11UO5txhYcsQ_wz@7)nNT1XyWe}IlC?J+I3h~H&MRh|?vW#tIm|gP?53F@_rNS5 zAhhP%%bK|VC2IgZ5nxiWwaQRkpsSv}aSOUvwobU*!q@*x(F>sVmZvLWI>K&( zk|l4ZEaqBhaCMEMQtTF)-_eocNu|y9M9364a!m1i!i7J-Vh_DCg#*AcHXPM`1xjR& zGBh%J3eadbP$o&C3~@sr7EMm_y*rJk3<@kHgEErd`Ur_o0dy=_o{lK$(?BGbtAOOf z`;~bPHerkb(bsm2*38Q4U4QwyST@!$nRII|v)GYmh@R@*pHP=lZ2O&c#zrSy`t^;f z!<7si!`zW0z&YCcnmEEHCpX*>A~NyaE2PXvY#@)znX|9ED4JWJgirFJiL3JXQM+y` z(&avRK0Mdep=Ces~$JfwLVjlTnY~|Dz9X=nsBeKPCxu-exZKq{vSxaGZ#b>aK_ct#0OrzqF zZ(exw7h&&NUcP*<{uDP0FRi`b#ZhE7X99LAWbe!`EcE#lEmd_k5+%Y!a#owII&OOq z8m*^eV3EH&&M)fVOqR3GJ5@lQG;L~%dHcaU@%b825wIemdoWD>{r8h^GzJDn!ZFo&%*_%tbN4ZBk;DM!GN2_2un7xXcla=QWqJ%+RNP$uaoYSW2!Ejrv5sqMq2UU5I}U1x}nqAT2BKH&uC=r96rR<9A76iF13Q7O|_*G;dczno=se2nhE~y%99c$ zdnU%x`TnrM<`|Fi3Y=55_~gATK=^Cg8I@xYZWyNB#m+8f$*0=Yl3rW*SEf_R=`qP7 z!K&IUP(drxA+g%jD&$J^Fvnwgxx2bI4}Sj6v23KH{SdwXQ4FR^Kk&#V-seDl^pS|A z#Jd|&inn=-J6{a%M=wPD;()Qqz7`qlh<81bjoMQV%KEgE;USz-oor*<>5?@bdtv;O zmptV&XeLP5Fw*QOmrT23H@#<~?lmJ#wo;fsmrf1=EhD4Bqhg^Z6+L}*LzjuRTn^RK zlP~z}Mj4xpiFnY>;=42NM8_2qd`0fkuQML@&uXE%fk++)C+3bp&Uue9Ze-#N&}H%oe>W=U{K0`x>pz7lx?FU%sNkg=HE% zF}pHV+(YEC^PTxm-8rq(@{`qLt;7UNHR7j^QNif3X^+Q9Vn-eY9x?J}4(U{R^2tuc zQtoz{&-|<7tw;7xmVXYG-O1AK?TMg9kJLMIIjD@|R+^gF?+FuSJKbkKhgN94?v_Rg z?p0|@p!RC8BiVk#VLyg;GGljLdUSF=^v~K;mw<^1(l=8@ct(_6jKuDG$VN%pwF@40 zcruZ$l|R9nM=bvG3o$E1ea3z^MwdL|+Ar)|a#Xl9HWe;C)S73DDQ&Hs{?Yi&p5!i^tUb z0uIvj&)w?}l8grwn?bTuh*EIHvk@Jbe&lhU*o=?hiw84*1)uI3*JpP zTJGXt62L~L3aXb22@z+)(e?ZPFl}>KSu$iU%yl&tr^l$OP|AIrX!77vmk-gYmdXO4>1Nvd|0{UEgX*0c; z=uzO>ftQd2{1woG6ysK-yP_%ssVQw;S65zR-p;HG$P15L?nq(xJY~BEB;h6BA1I?Z z`7J@CDOXje4S^*e$pW5^F zrI}=4AKcWAL0O6$Six@(#o0W`@BpAzj)=ch#p@J$?|uA&WGpFW__1`sm0qawQfqum zYDAP|{ecc5>=8eI5Y5Al;cXyUS=X^2;#I&aYwm_y7Jd zx>i%=de-G!krPBQ>b6!-Wg;xD#+T3#=EaNEoPi=dw7PLGmue5PXUxuqyUKs=)wtR} z!p51p%!tKVzDwLIDR^@zA+p6v9%V!W7^TWqSRC%T#?aW_7thq5Chw`%H8Z_qcygNn zJGyl9>2T+mxcPwx65!!z_hcEDxaJP+hi71?ZVYyQHGcMZaY*S_*1gL0!1+cU-W;Zx zd<}Ctd`FTBFCCIa*g7kcenh+D*@%lDY2Ei#EMpw*TLYpsf2txSyl0DY;sMw(mS6(+ zq2@-mZa+A`f3N8tq3rf8d}id)Qd~Jf5Irdm;et7YaHeR+cSc449?zhSl~=ZNjT9y) z^H(O@@C`>Mf4W~)27-%IB|OBe;Z0`JnfVBKtwf#g&Hk8dxQIrK5(9NF(~t3qc?*in z`mlV`=M2pch#l|OKJu6oHNR2RryOY|sBY?sA5XV`e)r^MpNx+BW=cA=C66#9YodS9 z+_agY=k3u_M;7E`57-X!J#FrmGa;-9x6J4xc-A#DEf&H=W_)+5`)ydS>0bcOrPz_W zQ7=MFdqm8sTc?-sZ;2{CeIgnj_lr!4=9jbcD?N?N*ukaEB6i%Fh#unIK#-Fh=$cK} zcU2SVSHhy*V7u=%rY5o$6>u6qmwBT$VUW6$7`Cwx?moy}nftI9fr1YUsxl2Ecec{Z6aX^6TNGSfJJNZhXyN!OS}Sk6`?0 z`-x`9SL3IT7KskGCbrVmi?sJ6o8fhr(`3f@0#9!0#iHxH-q`TC&i!HwR9jYRp5$x0zHO2sY;CP@!BhG4?ZN=iIJCkd)lF4sTd$ z!KI0sNE4r9`6LRmOn97fb>Pa_VRZX61UkMpq}XU(Tu++| zmvcNM|2~mQV+$b11%YQboux**{fP8SvmNh*AB!~lG95CyYV-d3YS!oL?`k()A8^3- z2kmEODYe(>ZIYMVlR;@z-wQLfo2h21)kx=d;;Rz%(Co-~Y1R~H3ib0r984KCxf1P$ zTE@#Y8eq{cjg+rv@H-SFOttK~hHFghWof$aJ3)w??){n_CojBVi zpu89k!s-kN9>m5+l_$R+VTYuPcE>E*1~Z+k3MRf94;1GZ3`5AssK2Dd&OkY~D_bnu z$Zra?=fCqwR#~gi?KB?^XORmE(m<#C4;Cwx)P@dQxt#+J%{4en!gBD zlXfX5M(3#%u-!WEAZxveS3OsG(#M_ zFzNhHf>?3f#PM+O@Q7Ku81dp3;bp;nNdh|{J`r^u<(apRHe@Gx`6Y?b z+{um>{qW3Y?(dLUQ|&OmIw0<{dI`u@^d!$e*F@S2<==MvN?j3U?XnhhGAxP*b%x{mueY}5ON?LW*WKp_I`9ssi`rx{xEU&l4@TQ60{Y~h&qTZ$t~ ziGY%a3cwnNTMlUv*17=HsN9!+2IN;}&$RvJ1=1Ye{>u_u!~Xyv?<*h<&-E3c!27L_ z`|!&h0KMJ%)m$gcQ5cYqPGIeovc9I?R-O0lg?qh}8Idj5VR^5&^{BJ7U%q@pLpCO4 zj`&J0`3WlXF7-qX+BNP}@I;t4f|ymnE2=Nee^W`)fpf58tnxv+%XJm$a7!x zu*POW2eb+ZN_aY>$&p^#tM+7++8r&c-((Sjx!snm?O*KU`jrYd` zGBlm9*up0{EhhRos!iYtS8Rj2=mEF7u;Io3hEu;SA6+f&cZ^7?r2~^>E*sx zl?FmYc0jRVu<=e|?$8TYU+#AQ2sV4rEiG29@8xUv(*)fF`~_~mba55Z`2lt(9CMQ+ z-g`+sPd1qt9(=rY{h`leTJ@5g@B8;bSspphoc$7qWgOTJEy`0$u32bkffKCZDX?T> zM5DdMCr!Nz0hIye9~jwQ^S?KC>+2loeYV+Pgfi0vqxmB<)a!H_8l!Jztv58ErR3Rk zC3=p2&Sb2Zbcj3LeO}4S=s^3+bGcWGx+n`H7c9{B6m5}NkJL@*D4MU+O|z^d6exI9 z81`xiD45m3tLj?kb_ZIP4IT*>QLm|TsAyyP1XZ$_A>4*w+MzR)4w$t*F{b0u1Tn;^^XDO%TvN5L9v@_D`W{s zk7s(0luE9P5`_`W=-Inw@H^dLB-`r(?1eV?Z2A4vQcH&a{Ei}*iO^A+8NWcF(*y6i zJw(4>-0fn|iK_&hzm4|b#fJ?5hzL&@<=Q4Eq!wfs+;}EB;d6>C) zcZutgivN3mYa2$%+_+Q4jV}$Qth-pKl8xF8GIH0JloFeunN&|XK}K4ie%;c!++DG~ zFp|{nA^-T;c=xj^SaT?>U!R8C8@}HHw)60&)?%4`zaFz<&V04SbiBwy>zQ%ak|yut z({NKS3bK~p*(uq2flXWbBZZF-oO!=$J$13MTE`UboHiEV0`Dci_eCb!VN@vSVv;UK z=7Yb_cuhK8yr)*XCFt?!-N(lYyOAlc1pYi&q&VVv;p|RE$H-_mKT)};L#?5?bO;}~|A!LG+9BXJIgcLpO;R?Yc`HXWcOzl7N zWsbQ#nDc4z;|-KDcB=3u2wP(g$mLQl;8q{9ClUt`iPCaI-HEjb` zg$y%&2X6mLXaaxFsNPuLX!5b1GCa#%W%z61k)e%Xaa2`xI!Up|VyM zB^~b4x&{?@TSNmG0|wb1V5fQY&I^K8*v z(z!t#=~i2XN2XHI5P#&zqKHs&G5pqS+x=jlmHo||m1o23K!06$5Sz4xBKzJ}3(553 z)H&N^^*&*72sYmm+fX^MIXH;(H*fUJL!a&rAnkhi!Im``cT`9-RKn-IY}ob{e52e9 z2&0~^v`!AVj@2;oXW6D^%g*P#d7THbXy9u@CeEP(lt_8?V{q}~)*dXxJpUqodfqYk znMF=T<~DwwtEm-U@b~7vsIUUDg7Wuvs6!=U!N-DnVghuOSXT>yqecA6t4w(m`DFV) ztUp=z^?UNo4Pz5DvowzAIWeIu$Z(}r}e+6P8exd zB6m|B*gv&io2juu6LHC5v1L>I)s_wOI6T#GKG$2hFKzSj#=qB#E#Z7y5;$CLQ}2O$ z!jWDU6v9{lep?j`TwPW;0CJW<%pT8@&TG!Kj&L4HCiDqv<2EX+Mg&yj?kt+>8{rTu zn!|_%iC~~Nhc~lNapXd}%{fAv4A8)^`ke0LK^+_?W zF?&2xHZ-)9mknOZJ%dej>T?xB6VNa+BNG(kJ0?D=x&2l@j!1$^Ad=_|Nw#{WG9VO;5gdpE%PE)!>QtU%u{7pjceQXGi1Qd zp6+)pE?8@Gd+W*EHV!K8C>*oT&2%08ppBm^l6;XXyZ*8aU_|PUPpY6g!*|7rfx{0J z@ILsTSh+1B{7LuvY4me^z*gN5*ci7(>k(sRU1Usp)i$B6l4>e)NErTZS}g$^v~-%v z7N6liCzu4U9yA>sS7a6(@F)wyolmfqn2Pi2bK(=!t~eFsbV)tmn7Vrg0S1I9rh|_W zQGlLTV8^(P3B6u%HcwS)hk+}dSW;We)eRx}a~+8BIrq_F)dhr{y5se3C?l~b)v8Vu z{Ix;p)^%e;n9E7Ij%BG<-DqvYTuSS1^IUMp9n>nVU6o6xNT#gUXJX*byBm!sbl~uH zN+3I_?i6d|zkGZY32i!;UAe!}xL4|YV~9sv-Y&pGbWS--aDEGAQ(R>-X0)GWR`dFu zX09CCKAbnAwTgX@n{T}U5P!_kN@%Ov(^PGa(p+Z=YgPz21h zXOA5+!rasQ*zIjEhIvI-hLm_1Vjw_QBhFcD;LlZhStPSNBO zIQ5m*`>?_Wxe_)9>t5FS_(2%+T*_If|ESlt^g$yOz6XgxMV3(CM*HZ#AVB`2S zTFX~%A?(Ni^|yW{-}q!GeJ2B58}%^qP+R#}-Cw)gQc9E;(O9ZqC@`GMvDNjWM!;HFG4~ zM_@*q?kpCG{gn0q&I6I~&cj{xy!n|XNWA6lTokcTd)BSvoJf{ zY}SK7tANl=1M6>S;Lm#{vDMa8ZPWJ10-26IAik_h7+(6~yX)BlaQJFHXBu|iJ9lQQ z{begN9y!gZ55jNh%Gb*ZEt-{ZemFuD=`LzJVrH9sP`emR>lDq9DJ`0xpKRAsR#xcC zET~UW0?x_u*1FBH0!okmwt+X)&#_Dy@R3+tUoQ|J%wIMBOzzdk+)P zxU6NKT(SN8%sRB7#5BonN(`Lqi6AzSJi*~1jYP{swJ;@Z5j(e~uQd3hRmp5euXba; zV_?WC#EN51l5CTnbGT*9oBgBJFw(#`0;^DWWT116ntwI;`PxYLy zDWAQ`Kl@!EuDrm<;B!`as<+zR!hV>pG!T|)eQUDVu0sfa@Ye(4G;V|8y@nsnuuja1 z6pZn9#wJ@mL#(tWRxS~G3SKx-;sR!>BkUk}*=M*M-5#@^tf--bB-%^HivsR6lKZ%C z!PSu3`z7&kM*cKgs-AjWw;%q4gS*Eq1#e!+8fltOY{0f zQbim>u#M5iIExf(=|B7=_*c#m(nhoH?Hk23CEG*)jWn--trivr^p#q>nIwycH{z1< z?i>3UBoVu9-h=MJ&-a>e!(gR%OotxV+MQ@+3c2TrLIeBwhqCt?EaiqAHQo#%-RG0R zuv6uT-l~OGh_?Ghk^fN867j2b90KoiB9MXR7lIAE3}ttwAxxucCl6%2mF?+>LCaV0 z6&|;*uR0G9PJUYBtBVz-o!Z??0{b~4&nOeVBW5|C(iLx{cCu$cxVPVHA@Ir zEwTI{czqBKnG`N4@Tf^q5T3G^kPX~m0_L;H$(%N8ZPa##yZIHkexlyyd}flFSj;jc z5Si(Eeem(yW47OOf7Qqw?_YZH;ZKy(VZbmi!cUyE*RUew)=I*-tl z3XwM#5AqG85V@i~CxO>G{;Qc_>bp16vz)3xOEe!U4sRakWJn#)K<)8-3 z5)TC1;LPI8t|%5cp2NezVLt~wF1X4M>FbMiHx}X7E-PW)PHsW4f+sKWKHKq-5!oK4 z1;JYAKmnon#cYZwR&e_^-fy7~xr8jJ3{}&gfm?sP5*{)5I4?eU!%+TB#H@o8v_aQ+ zq{d~*8ZpTuhEExiF7WuYIX~x)M{dG zPtZ3cJrM0c^BK)(R%Hp(Wblhc$RB_=OoV>q&pa<5&u$9E^4A}H4zFi!2pjRA3$G(u z3duF-ADtEIIVN2BTAo<{&UVV!@R_NOJjMqB&dip^LAnbcVm9ITo#Z^kPuJZUeXyRv zglK)plCU`bAXyzUh2=x?1jKDlH_nR zwRVMny9!B{3tpWtm_?9gmK5X0uDcz90W!;393gEx$MDws8^nlyzygW1* z&l^2BG&7Wa$djJQcbI{wnht2gA+ciG!R_7W=MDZtwX`#zSQl&>EMx?aFSkb*K`?jn zvhe|+GX;6hFKlzjuqeK;AT+eegvm|zbD z*BRbs+vP9r> zuS%Rl9P!QJp7h6QmgbPdMjp4RxgLC+d0Y+hB;3Vgkd-x*?oH3IrS2cDGU~!GKytkK zMe%MeAmkA{Hvcg{iYzL8ao+5=k^frTIrM=`xUADhEJwmFKh5XtnB`STmepuxYxYiu%*~B|X_*3*Afn^>StQ*HmP2 z#tw;AnNIlJrV0P~uzF(i1_*hQ9wWWs`^yiBI-*TDLatROZbH-JL9ip91{n9HwJTjh z62bKl1s8de;iPIt>v|lIz9Q+qb*_OL=*N08kFh@jiu1-}iAd)y?qK@~H`H+dT6!w~^wMVM!v<$lKovEL^LiD?+DKZ0A0 z6rs@eZC{dZs)qUjGHT0#UwDp5;0@5oZ9h;W)!KGNub$qn(tb>$vQ#vH{RmkN!3N20 zx#ELXYLRd) zIqX2bs;rgD8Ky8dZJcBnLJm%;G9305_W9T?h22~JUJw3*oM{4g<*#N49$<{~+dLXQ zz1`2Iol%VZ3?3sgGI%VEm{rv1rs%3E??q$aA(tqrry^yLWMw!rh9u7ghW-sS;M@l0|XJu76d zXYVP_?r;!s-N!mc<8RQOb=>+>NYE_$sqct7w`C?+6#L2a+NDY5sS(@%oGJdIAqx1EuVF7?FJSLw^GIEVcHFDQXUSU5nNRej zgt7p)be8K^gOilPJ?3pQr>g7u;4E-r`~}v3)~|)wxI~BdI?=&bQ_`^B4}o&xJEN=|AI#9$;G4I^?z3%6P}> zUB=x22h0CieSqD4!D}y^hyklBUWE9o@CBlptzX?abV~H~Pz9!987UuPwACUnwqWct zMq2FK+a!HHoZc8n#VzxhcdSt-R7?S_P7oZKR5I#cR4C$m&Tm@xL~46}dN&d>JL*Vo z9X8Z0C$+G3ykvk{KBV%ViuJObicN5WA_jcIFLqjPV~%&U`RgV5VhVhotuu1rm3xdC zgj5tZ3f>Hz0ac;~UhLcq?{%SlXlYE^4t<7CnLTqQRC+a?SA%Vj{2uH3Ve*y40Oj8* zla@Jb%PNs@YH008NnZ!2yg(pSw?9sK$DRSCbeQz9?_d+s0MqKHxB=SsxXL2& zo!|V_GmPTX!y#JG0(XXIFW&9EcY7sOpr2#t%=#pU)i=_))3!^K0ZX} zH!Tbcd@RqSj2;PYmogeg%9@Menr9nNp?u~ArY{LocBt8kusRw1U6Z~XP zXxuX`=nY}gNRyt7>sYyP4X5nkplKcq) zm@ROSCM~7&_t2p~;Z3b?!ndSG-I%KqtL(-%bgl8z*rPW!i|0y(p$4pg#Mjy0Vq-EZ zkEo`)wt8xaZkGC7P2c1{-(=TNWeuN98MIDCIab!w0&>C6+I+Qr+6`s{zo*>R+DVvW zSrne#&h+wWC9y)pe)J<2q-R-X)j(J$l4h{yyLMPP_5g~&qn+6WAGa084aktxkk;nR zLZS+(bOt;ke-Yw)w}j}#t&Ro4B0t`2YSMUV z+wzuiFqW@YlM!o*FCJCtGDX`}8uRi})PeeBZpAr&Z4*mqWw$BCWs^Um^pE)$YeyXtJV!|zT8 z^9kX-|EBqNw68CNZ1zIJH1+k2v3%<*hb0~~T+@{o+A;6T$;8$c?0?5iw!-)D3`Z>+ zBo%(T^czIu%}k{(oteYUtxv7OKrzS<#y4^uUaVO|%a$s5$vTUag7i?FHR+9z2@uH= zdN7wAzI=es0$Y<3>p8!hmwC{X#TOEta3iEi-qPIFCMg%Z@1{$`93Ht+KXkCUxk-gO z+(aTZA!P%P*LO0N`qga@mg8tih6FVHu9&-93{HZNTalyHwD8!#G}_X5(oHBL-(&pu z1ZE!1JgM#A=J+yH(Ped3HMaUP0ExiE2Q>7;Q`r(|=$+m?bV%7y;TX0?z}DFZ(8m8D zi?4@A7Qm&4LrGI;W?hfD4Aj$W+vs)LV5w~my_1}+v-Z5;&Pr2wzVTQwI-DE<4mzCA zDV6g*$38~5(p-LVwiqnN#&I2f>r#1lLDfnC)xI2M>v=wU?FXCK;=42A_)?g=IUX*% zI6)qabgNnGIM?1MYwiQJ;>CuTQ?0K;J2zKS@HHZZC8%3mR$)HU3c7rvP$-9vyl1|J8&`o>zg8U1-14$z^j-xiwPjO-t1%4`Y{i*iFI{5`DxJ(`cL=DyvuASGDd31MZs z1~DMEGPN()Hym$V;T<-eLhm?op;}@paun)_np!q~5&rGLWf|b9{Hq^-7$$S5zZjdm z>4=eg9@G56bb<;PO{vFg8loaG#&m-DKV$jNNWA|4BXR0+KM7ybj(4P?aF*+J5B75l z)7-8pu}eiaJEUlao!v!UBnxjwHi7Vy@Vm7GW_Rln{y)avJRa)(jT=@Sb<(26a*(D{ z*(po1P9=MG(kP~~FO%#=gS1Gt5VA949WiAM&1ko8V;M$FC1Dt17MU?K&u8j)?)!J& z&;2Zap4Y1*eLvgvUat40&TQJNj^8_9o4JLG$tE}Yc|mS8^+E?fDM5Tx18$K{Eq96S zH1Em@Y?}D6%=Mu6qMF-u2bhgchYtxmdiB?YwJy%Jqq2?4H~i8qks;t0TDDP~jjL{X zIV49=ob%*n!8CU&%9$S9eCwH!{#zwV@($DMVxiwy3ZAy%ClZ*&YFEWVx7Ah1J~Nu+ z)3Bx~G(Vx=xX(uO9~;Lw^&gW3yc>gB)D9mRnuTKb=%iG&$D~ZyY^?oaS2p?a<|&!W z%%<+kyKB3>4(9m)@Rg(A8($PamGssc4KpCIJpyC*68~B)5tg@CrH44JndBO zPioVph5Xs?;G$Wb6CFYah~@u!nY)-=7dhK%^p!Rp6BQXXwjr~I#CmtCuI?y?79Xcvwa<*HquAdzlQJES5_;WR2!*EaEqar` znmQu|(ZN$>EcV#QJ<2*}^IXT)p!V5zwOo*}k~#jj&B5BOM@66a*+I@B(vI0EeGWfI zU^*7#Nd!U~Pf=g(Xf54wNcgaP4eITypVsd_z=EO*4OeqfIVc7l_FN~fK`w__vjC}%z zGr|Q<&tzm>J(i-Lu=kSXH@9$s82s_2i4VVpo(jpG6`1#iRj##K&|gq+Cu~`KZ!I0L z5LU_?anH-APWX?uv{OfK%Z_;{v%@9N-+dEqCx5Hv49l?&&}d4nW264qjSDII6Q^L^ zK81Dhze%ZYE{)enZtI!@k9s87m|bjW(|Ih4J;`007;W@_`*&dW_I*(2A^!uV`<)9Nx-*p%F602n_l0%T4vA6f&QcYwQ9V9V=;@aP zfld{GC__1j&w@j{6O7P^Etyu_^`{g;h$%aZ$RpSoD~Iv~an;>5s>eo(SuD2{eKC%Z z0V3bL%PIDDoLc||Ue&l4jATq@gLGCChLkh>0>;n`=~#F5bM3D%v(lL0QLX`q zCv)%!BwRSykU-I_H;8{4JyJ5yKrHD_eMl zfdk`9#~s196FU6IU%c<*Vq2+CqnpRAIMAlCT3E)sFqSyCel6gFdL*}ZYV9#=ZY11B zBiM3JP5ffmX6+73Huf=n^~Q1Z7`m!Ly=>bN3&$vSRM;Iw>%JL7V#l*DD&YH>(yJ=P(f zWfQqEX99ivv=rIUM$!TwKUtr7N%#8`!^LNA$=hqJG0%~XL5#i5Sq{GN40!}JeYnP4 z#sZdn`rF^S^Z~~13ED-S+3&a5@{g^!qdyX=$aQ6R91wKrRY z>p^^5S&eWfBegyh<<`nQ=nPM;bxo*chJ&TzHsKl7{LSVVT3PGfEGEATT>3~;*s^9< zT&q|qRYu1v6id)oL`d2lThi3wfhule!@IokQA)p`V0N(DwbJM6YMY8$J|J^nrNshN z?ufp)&4KYI+_mLI#IAh2OUP1eE6rl)rPdmGkzqX+8l3zhRhhgvSd%e46Cp@U3?La= zM0*7+I%oBtJ>?qttJeEss;i+EFYP_UHGV_V+aV{);8et}NwdON*joaoF{r8C5pkDy5(<+y-S{dTJ;t4e za%F_*_Zuxx%h5cxtt&u(EW0jeFgTg%t%oR zMb)}j3ulp&f`DqU3?Q%^GQlE+9)17-H0iuY7Xe%_M=A>s-ud}9 zn=pX)%~GOPMc7EfRu5OYIADuILA6`gxJ<+|8|41RF8G7p?})Ne7&Iw)K8sB*ECnaY zv5>vo7G36tDyen&7Lln5$#`X;C=BaD>FtzI`+%qAFS}D@YLsBCj<0c$8vuUah^n{0 z{GolO;T_1aM!bpHl={28{Z~(n!Pr)9!9;v%89h1ZH|H+Knf;~i?_|;vZV|pnePMB3Yr(Li3BliR3-u`WFc z5>0D|2AG(8U^35#oZMXdSwr?*7M{Nxq_U?^Khfuh&<3b_!uR)PR1|Ypwc;@+S%n0a z8Hx1_K;KC)UYWT;>n~SHVCL$C`Z~yFP^x@N?o>q2k68Om7_%#o8OAxa-qB`A1_%PD zI*XB2;_J!xx9>-7fCJ4aZtNwfL~!z0&sNh;LCg(9F{iB3A2L&Ga7^b&=C~J{er_To zjWED4#BRS3tejS#!m8}cA7GW(<;v~K*YyFbztcCXo_HSj?gmJ+3!6YRmG;FkZDAqG z-9cQ1UE#}M_bE^dnXJXe{-9+?NgXefl!KFu$Nk`fafpw^wnB%GtgPIsj`?q;_&+Oq zweN=qGudQ!f~(OQ^K+YO+lp_LrS@V&JeHipTk1NiPpWGwl@Mv_(VZF%u!ig*H*+;3 z=YK^%c0H@L&R%|L8QeZgJR?J6DN0f6`zQC}9xr7Tw!OcoJpL4(exO&56dqN=Q?Ydp z&4<8tqJgdiu~Y=)$()2c%;Q+rBzxDj^*qV@@|6 z>lSK@@ED!WbH`X1%6Z5}KglV>I8%723afA_KLaSko&Z^+AFG4X+&A|**4{4Pv$>#h zr`k7$^eZJkeF}~p?c%JJSjYw_$w$maxp>dsoQ@|QX~L!r(Jbt6SfikzJ}Za@0T!D;h!#@b!J{5YHA6a7cL-u>U_#(=a6lb8xJ# zX_D)n!(jj_DdjTa+Lp?z{`Fb&53=Eq+* z=2GNdpwC>-JcKtGR<4@^IoCa)>f2l;WLmb1-ty(LSh}&?SV@d~=AnWeHD$YeeWA#k zSJZ`SzZ;&_0++eX(1=NMEHDEn8^M@ zyn{8i%B&l9#U@RoPE~zMM7Vn5L-atQbX%xA3G7O+0uyZ>hp)Zh=Sd=66oN<}uB-_BajUH0^MzP^0x`#>$+ zs41$wDs1|*8;dTbRH}iR$z{I7=+d+=e5%={$cJl6IgT3Z9c_(`ICciap19F0;>3fzj6+1204g{_nrBHzWe-kf|d|k&UQH zE=LE%&ABZLeGxO9UE~L4_mF?R`SY(E*5ZR6=0^a8T8ZvH0MH7=(EsH`VlaD!z0Z+r zy#nHpf$#86gPa?0_185YjOR|wz8u}*|2>ftSr69?z1@8_toQly7zVLpRa?22B`<7D zRsj1JId*8Gu6kpcaE3zwb`cfmk9c5anbG*c`nE~8m^M8)N@K6AzdTVm^w)D-(~w-6 z>qgvXL79mrNYPrwc>ydm#|T#8@CF0THiB*bSz}Q9kCe)xaqv4z2*GR)cFk7_*+4;( zIdi~J*vQx_IJhoJQnL%>>Z?S8WM1@_(3~i26l&I&c*=bGd0k&EZ=;`AZWu-T(%Og|WNx7k{-$YBhNoC1*1dS&S`OD!%%hoDfjpVeGQFe;BZ&i2;*fqH zR=$Mq);u%vXq!gya`X3$Z8R)O_YIfYTuAz%{}J*!^F~BxjkoqI4EdUTNvbk?;+{oc zLo%-OVB-&$jmbWz0qNC?m+uVquUc$ku6(WzMmr7WZ^Y+PQOkPnx{YGOB*q$Tx_EBG z^q;?;AXdR*EmwDL{~0$y5-H}L3sDt9%nQHj(q*q@MLR6zy7DIhq1CTCBl7+6DvB%mCJOe-$5}IZ*pP4R;sEVq+G1*e{g#xaL0LXfW ziGOF>u|cwmesN1y@${Nt`vY-T@4{anWLpxq^G<*KzWN-mP}ixP5Rm1d7BNgFlohiE zp1;%}?vX9VV_pM=B6U^uo-q-v;W22S@P5Cw-ZdF1YR0X2`1q|8#5eX#N8kaynQ-=4ktbxE`Ds+mkfTurafGR_M8A$%c-_%s5~0G&5e zR!kjfcXoD0K~9&L83xT)SP@*e)TdAki8=xTa*)aLg#YjX-`-xBi_8pw`$f|}3=US! zl?*DvJFxR+>?wWbn{RdKqTtt5R-8x?bj(@Hx_9=KZsgixEsmPs_Ia}LI^I&7d2g@> z)k%&{LF8zibl&}?ZiS&7#*Cgsly6wr({);4{Yu^7$3;V-d3Q%zn%}7QGc3`#+v@%c zSL(J$k7!rTtv5Yu&?h$^VC@pZ$ek_`D2i23y!iZ4Z2a$CvbkSKgg-N(v8tv8v*3g) zMp*a&LAWt!rmN=A740C8L9V%Mz2&~5cw*=KP zeu47E(!(}J69E&UUz9Wl5Fl-*q#0EN0LdDwr^4BriV@~!J{8y$DJ>N>dqSoWTVA-! z;$^Lzw|2;!jV`4E?LSW_wcW$$QSYzszw<-gw(W_?3lJ{N4Pb+qVu)ah;k-!NYO#j< zr~;(Ax;j-N6jW-V`O{YBt(BFX+^-PsVM?x0qhGWc$AYm6lBG#jr@!8pn_FDId_iRw z!%^_dBp=B(_>b3Cw3YG;wFuMTU4eGMdX0*(%oV1VOJu1XCzKQ;OST=n^@b0g8Ku7^ zABVykE5Z-pEna=qHa&4Pl~qA7jl`yg?6PYu!5zmO6jyaU(7QUM{k(BE!pU4Afpn2r zHvq7Dq~PJ-fK@~@Gv74um$UgF51-8n(7Q9jn0Hwtou_14;8`>kZW?hGAtWkpWpvQ_*1 zQm%Mn4Q>X~dDyA&0?S{06M)(Zj-rK5Or%nCx7f6Oj{0%mJ1jM1+ImMc5wmZ$oliBn zN)Sth3jJh)X4_>1`-Cq?-5swbhjMv@U}mp^?>Y+hZaiWje2&;8l-4Xz600smmL@Gq|J-6e9S&q0<9HlCP znop6j#Q8#~it1-M%9fI8eOfcnUd<0?h;pVQuB%80ZRqAJ2HSU-LEpxW)) z;kXh`s4}6U=yOOtZWYU|SuL5DA-T>z_w@HY?oaZI0fl@L$uSmhNS*mhpdo2_{QlEF zCT6bK%_2beeVH)ZL8Hpla}ZNri$rgsw``e^|FE-$K8xlk%9rZ*z$ z%4krGi{`?f)@w_)pcX+>TbfNsnsvg}71zG^rG&nuy=N|+%w2h473{rY-CMTdHv9Yt z2e+EDVLU*7a_ze-4Ylo#U@w4iN6(R!g!rNzUGASbS6Ta-`a-6Y4qwZ0l9vDZ!?iWj z7=X5~t{jKt`%L>Y(SYK!ErQ=~R6bUUz9VNU2@tdno^TVZZ7$Q06#GcXxC{Z8Q0g)v zF-3)peH$~`Gq=54kJ=rwzb$`JY{rQ!n|Av!om0tYD#F^O!_P-oA?lOPf*PM$4iMWYd-DV9`ljS^@kO>KW zcxPki+QSR8afzsHVufy1jeRKu!g?bbnNU5>eLPt=CqBp;i1I0+xsh@P*Yjq_b=v*k zMX`6H0=4IP<##&#y=I%lig+jk%I1qc+p~>)e)gg?DwKrwX{#Sr247qZaKxxLFvvDEJ{Oy{M8zNlgWo zL26dC_zx>sZ}C1{pL-eEL-qthNB{=qZ3oiQ>|czSF0fR0dwRw zn)Zv&*4YcP^I%VBUhh}i45(~|ZEu=s2`_SU#MIbP?4F zcPvn<2yn-NVo=niE66f2*T}~P25xs{K(NInZYc4UF9P%S@9KfT+}qF&6s{w6au(3w zwH$yyxurC5*6!a%$$x2Vffw7HaNeL*;W2$Q4v*-s&drApb$+u{9rKV#JR;b->_WH5 z-C!Y3g)%tJ(LgZCf?w0>lXeb#Z~FG+0prfBpPrl_;zMHIC-c~!( zuQAgkAlI|_vKx%!reiIcwEoYY*w^(6ZDr|z#(r}(TIm?IoOZQ)DYP++eZAH1Yh@U% zyPd=*L$hk6Rw)V0TVU2@TG*aMke>^uYJDpFqUAC6vxQ)F0D?y70h!~!JS*84_WtP=lj5L zP@iQmpbAY~{+OCw6yccvn#WmAx$ry)$S{1QHLOMHRq6`T51v&4QWN;Z7VPU{4E^7| z9-o`y73H_A^0sb2wzxw;fO3zClV+C2dria)@{a`mG zO{o(GzhFQ^l^0UBDL+zl^6g;N_uAJveff}y*R=OP4vY;c5qeO9?Kz=0qv+xPjV^0# zdygR2GO%|(?$)OdY^3#_VzS$}2Xu3KQvQ1 z>etu79`H(<6SWeez$lgV%D)?^nebf7Mm! zQl!Ar*?^B4+CJ%5Td(SAOTNdal75r6YKM3MPi)@d1q6zs2AaUg@t@&3UjOL-v}E_j zwxVYugL(6!FrV13LKk>}%xjI;+N&CUC*H|*AUo+;=b_ir{!py z>S~?Mq{5d|H$K0bTh0x@k52(sK5T~8rHHNXR%umYMI+CLfW&9_~kamNJ~TmYsq=>T!huTqD$+8O z53VEm7Sg!i+|kzWyj}au$HWB$hQ-T!aUZuPa)EmrH&YxiF0EM5uYZ5>g|2ws0m1Uf ztpy1d_tX34Yj=6MW3x( zxl;yb7qs-SkM@3aYG_i9!n!e-+w01DHU90IXLGPPOEu%~FW3I;JOJBGJyW|ViNew} zFy_TCOX54rx45VbA8?dr(*6kjrfsx2+e9Iq)veXGZpkKO;YE~bFo1HYP2{M-wa51< z1J+Kr9VRa1>u9pd?1F=Xqb!5KofHe?XU47`6!L#U^`^|;BRCzB(;t@ff8W~pu~(4^ z9C9GEX%o6lLc%y8`uhpp=(o~gGkJ8f7$Nb|cEx~r0=#Y7G17ZJ01f*m|1G2MOjGPk zR?~q4y?L*EN1M>0Fnp>;&?ha#>OzxA`;|K!`BU7>(Fbu4t;z=L?YSdj%vLtO_q%dE#TW)DQag!*IOdw(u${>Kr2(zEZ1V_Su2ovFvL;w%8>Z@Ozf>Bm zn!dkKwNg~pS6WtfoXsM(O?fDN9Na0BeXgdkb(WH#iXSte7Yj2^X#eQRb_)P#674j` zmjfe?k4*4#s^u)&yZ!pU4D+0Aee-E<(d5RDxyQjC?VnJlMY(60JhVGKd7ZG( z`l}tPs=gHlO z6*{l+$H({R*Lp7(zd3l?hD5y%L&Ms`>ckDaPpx2kenfTqGZ3Dw+ECx5|6Ut^kvda7 zFu9W%&0FR*1ovsnswIUKoW252+NRl_DB6z<+h(i1tWOO~Lr!8fx@h zx$U3R*Z1|MNS6I^vw*v~UGS(>r_w$JZ0yhtXCT~0Qo&8ETd z2lqYqc8=NVMAU>KWZcfw-P^4*S9BhmZVw%5N;Ck&uuR&mcl>#W(fGTtdf^68d4&Hj z_1(RHVQ__HZg}Y9fX4C#!!p9<7-4ex6&7=-kgE}q^qCJjXY<2d>#g!4D3wg$5GZF_ z^XS&h)Ra3PxZhN{?`m3?TDs+W4&ymXqFV`XC z5hO+nb!qIzv}@MuqBsSF?Y#5>8QB>=+mh zos7SHG`+pF8o^lOtEf(cX49XN5(>&2tauMe#2REx98C`yVs_ zDkcE}I@15-A4edt#*xk#_am`+gQCw>tha%SKDlWyq5q}H<^HC)Ebi2e!szQM^&&gl zd`AyA-WuA`=t0zJ4drgHY;yws`+=HzfrwlHg6L0YO?)~XkcYy#bBrXlhbq}Cj#3R* zFKOrotYRi)g5~Ss*8-!4rT)8i$v*Z96J^`)X)F0#9xFu*RqMS^ zm!9T|e)Y8c;B{5*CUT~S_fk;mlQH@FpkFhcrH#m?jJP%gLKLtKHLlZUz0@{~ys|d{MwDbM zt7G3^BEYI}_qzCS&py+q<45gfs`eJ)b2JaZ;`uV{J~=Up6eg=4=_qHa2c&oqZjgF7 zwG@j&ND8fUyZktg{HQX^XjE;Jd9?>2#9F z9wKc&-|fg=JzQJRmQf*G%T$58+n5w3kILgKC$)ea zxPJ}t?T;L|YpX-UJy+PsWp!B*cc(pfx|bMX3mcFLLHlhcSaf6{V>0ahXLQ%@hYO;Y z!&QD&Qhlu<1DoaIru3k|NF)U}_ylNH-N>NvzoqK#+}D4~=J|O_OIGio+Fx*)HQ6?6 zqKrP1=($B1j@M)TjgQsK`G{)k&H(s)YPa6cGp)bAB_O~wI2WHR#a^p{1%49&%g(vc zA6DPCQ;ZW0T0PRt&=UGAzM+TDIL{cp1=R6%JC0A=OSg6s-~TitgUIK%9Q?;kqPE!q zVh&91@xupcKY9}YFEy;{kO{ta=xH#~nlEV~J|#+2E;!i19r54e+GBnhlaiU4IZQn=Ki2ZF0t~W9?_g*)1m@JfLf&ZOgaHc# zrux!LR6B6k1LXumtPlN0k@15J{$@HCfMC*#Y4;f_8wY7k=sP7oQf3~u6P`d7(+4>N zHN9!McXODMMH^KITEt4{WolA<#6JI8nMTtIqg8~?h~<$6Sg38nhxX3`0!Yh$JB|4c zKHs>|iOXo>htlkM`wKZ9B~E=*@96haW4KYVM#~pYrz~qVCtP3d(sgZE_;Cm8UuXJb zC|I$kbM`>d8mtNFv$XLm>|?*MZ1y>sV2*QL;&7Wu&m7kh9rR({7UF%NMtHvP*0*>* zLL2ALiPi-+tVJ-@ESalfQl1y|c$c(}2J0s`3r{coZQLr+I{?O@xvZO`r&FK?cEB-J zxjSk#z3}|{K${QcJbU9;JJiZWjEdzX>#dsVetRUNHe)z12kEl*&b70Xj6&XTG{Yz% z>?><4un^GD5C#^Y3}Y`#bTV8>!|VgAxB4E!Jz+oL4F~2u;Kzbcs^X->E)mK6CNH)J z;L!~cRncH^OxJDL3aWrUEzxi&hp{`(Zj*NBPvx~ER`UJXgwgn_l;<9)wV(ZtW}Nk@ zxZpz=Fu?R1?$$oqlI&jT1k`;RG2@t@!A7JdDP;IKSR!BHUKRdCWyt5_l~0qq(i)M+ zM&^Ou_6$ND7W+S+r@JFkWRK?OGesRrc0VH)AN!$+2yOk*h_fOwM6HYG*bhh6^)pLz z;XVB2n6IXxSAoi4v7TpGjdCJi3GUV8w?K_&IJrvn{w=8;~<}YMM9yz2mzhZwpnc9&^eBQ$ICc8C0sYF4oHBgC|k%l|j6NSZUx#QK!F% z;zJ;+K$;rfr#(9c^U4>UG3n;FSam%qMILj4{61vEr+meKdm@LS4o!TrRmVZwN|&mk z3nnSr6Dz-yfWUPWd>?RlYWH!8vb&fyQ1XM_WXBVk4L)tA^Fjte69 z$LnCi%Mm<=$meGzG4>6Cq*_xofj-^k*EI{F&4ibZ;LHGqFAB5DZ=6cTYSdogyXi)z zA>5za*yH_CV&H^NR4Bpv;Tgvk8ejwwyfG}E#&qmb(g}8DbFh$w6 z0uq+rs8d=3TqTas?O-{99ks>a29|iVwA$VbnCDBBElOD!RwsY%TthJMyUj-$vF4(4 zYAbewmKAyXvJk%F3*-P-x60pMt0JeQQrtsRoko(;ZGKy2Dfr1!Ch=V(6Er- z3R}Wi@x`33Z$STLJz8*6#nJ8Pz6PaP|FcnqO29$f z1^Tc0>l=Dzq{4p^cbFCS*i~9Qpp|>1ckvm%QM=GB27iyX1@j;f;j2W2;jP=YhY8EFKp zxO+?T-wlY-g=}@>x+2zagPcVnQQsbEfPf6@zeDOOTR^Km;EWl?wZMuN7+!tCyY&oD zY$GkuHx*^3Bl5-er&9G>RIS{=(_?eTL|)t5s9Wr@W>nUe?D`_l!#E)~^ehZOB?GJd z%h(0!zWqz3p0BvzrJ=?o)&q)$Ma)IEuh@l)bsd8Jb<7bZ&sZBEdWMdE!4Nuv`n0x)e`^o(d!Z{wP$8M?&s{rQTeV`&~2!$U3CUU3M#lmBUpP3mdXM}&31o<p{k38NMd@>;-?W2<28HbqxuF#!@0P!zZT0(pr@Fm{xjM7vv{VtDS zcWN`f26^7pY=M;ob;n$KeJhZ-@apr>M6$UaE;RH^E9dz?IAIaMv&Q-JV7pICp!dHjZdxbsRYul`F`K zD;RsAappA=8Z%z4dE#m=Cft@Ppq4qGrB<)R_aFe#(~z+A^5buTvqEwu=J@c|1k&-b z8Rcz8kgk-bYYluFjr#W3dxW(KD{g;``9O4k3K>+h@@AWFu^z@496mn?pt)nFv1Wq1`;}-8 zBkbLF*|C*&&_n;lmE+i1=kTI(|$zNww6g_Th05g(wy9L&52*?9AZA@ggd zOoP9*9{5WJlMPsNb8fU6e0mIQc)ULp4-e`*HrT8x@&vm4nzmShPOdHF(#ucz5K2CH z_=nVagEoLu{$nR}!99guC}&me)EXg!At=wIyU8J9}2Abg05}5Fk^p zHZa>9El_sh{z6S_-x3&-HwUAW{IE$STx`IzylC#at+Z@0*pD-j!xgrJ3lHlHW8bQ2 z4bOVK$wxBBPXU8OB(;HvV&QmP_I|0ql5U7q?g~MSFdWLBu%5AUC~FU;Zahk41B35T zp-)v&=>v0~gX;ZyZb0`>^lY9R0~e(c*0PmOut0o%6*#z4LZXvGG^2Dvqtgqt0*6if zMwAF^@!;XFpVM26BE`z>)?E2!w~)Dm|C%%Q z3j3a;zNkYc0Aj((k!R2i%oMeXUdV1xmv+qVEMhJClYQ5eN7q-9uC`GX1EVyymTtZ! zbjT+&Ve^@DcMtRq+X(atW&&Co#Zn?FMa-9Gp$`2t5_b`i5GTRsEAY#B_EOCu7Qdbz zH!{1BZhR^sf2*Q6zfT9An2$u=BXBduU^4xaT?A*4FaKBQmCa6&=0n}VJ=2D`=e0L4o>yL zef-{x`{3^z7Ixp=eFviMdBMR;A>kk0u;(sO>oYEO@!;V&cr4z2eKvqVJD$9HF#()( zr+kcXmj(f8kv3QZc{=C(r3US*I`T5=X4+Az`DEltK0+^!y!s!6ZmjoWdxOzLACbxw zlpTMh&|744Xyb7<&n;L>Ked1F8Q#_Yn%$RapVV4|zsfg9MO;n6Ply8^`{AW-Qrv2D zY(y#8scj;o&yR?2*zPH-td!4j{DNNrEOxqvXKF!;LmR8KRXyP-Zu3vzd0)Z=^eWg1 zsF?k{_*N&QbZ;-}wA$$?9-C~Ue$9PfK4_hBQICIG!)k~vGFv5fc%~YzJGoFq9dckr z+}mJGt4ZDTXU2C>3_pr(H*0en1b-rV9aspALvIsZ!i{EMWmYcHyP#UR_UnX2t86JV z(sQt@r7bwim&vS)8q=28BKt`Nh_&9zd3X1guma#G51Guk<|M@1P+of-IE;anARGeJ zzJ%cYe*6#=zp_WdL~@7s>muS&q0(v)$-p{FzF#$r(^?1>6)3VD)Ng$D=Y5bl`1w9b zMlJD2;qa`woJ#KPA{YJHJ(1U$4PoORSrb2cXWHx!7B#(E#z#^iysw3eG7!!aB<(|V zBj}%dq3HF19TZ>+l>1=`zk@xqe^avUHt7kvif#-pyuKfMLWt`f2B^4ezm0MN1&P{% zagJiZe38%Z2hKWN1n;P_Ow+UWfn=0|t;G4}a`(ER=14in82!8Wl1A-$D2f7XR z%?Q_31ZNiG;bqz9b23VcS}8Vn-HBukDPyd_(87FLk%r`;k{y(=Q@QsL!Q>lbQVoeV z#pb7XRxZ5VSf`;88LB|0*VH!S`MKAbPFUuiQqD0ty5AdkL{gg%8(JBcFEpL=Or_|> zznm{@n<>yr2B||ec$Tm3Y+kTGnBF?Lu!q4ikU&J_mwO&)hTSy3{4kB8ZI%n3UjlO% z$Wjw{a&Xe^Pq8NOB)+$GbZiS?i}os$=W==nawW%W+c<&DaKuzuIcm*^k5C$|krexQ zxc-*$fqO4EAAWGqSMP~Y3BJ-eGLqj?v0Fdrkv@~Mj1KsRF|&R$QeS;Z@&W$? z-}vn34nD71g8QXcw0{7Z(7kxAy=|amShX| z)nBUjw$<;EvHd}vC*6bW_JIb<1M2shT>CImvqgG=I3OxCb&qzbVkLZ`<;l@# z5J^yvG~~yVmb3%g@9$?^prdCgnR@D)N4kslLNw}XsNAD`N1=K}s_l-r@viE8?5fSp zIXI62M;C4I9T;qW50zksq-ciArmMKEMp!+&n|6eSGP=OCnN9Gyvl}4ECf?wd%iLWD z;dmj6bl``*gH6XJcO|88*9&>g-AXOY6ryoAo3il=*cF;-ZmuC0ymyN9$cw$PKVf?I zu!NWpbB;L9a%_!yKn14ecwpR+*NQ2>qxh)uO)klZy%1^3y781?Um$YYiq)baJ{I6m zXS$wwyBOZO@N2gq-Z7=6eLF7Msx7e)IQQt`@{;39&bWwB_lvK_AGRE?SfcwvIscpo zp<|0XXUwo0E!4&dRPFI%+ukD#nf&FPO|tXOn26!irbt&JJfC)s=*n>EJ^gzT6q
^=$UBMv&wr%8J*3*G&eU5hZ^^92dM@Kpi`Tqke8f!N zr=49XeIJLcbbHG#WbZ>b358?LNch7i5?BYb?^#iXK&k!jvn3Go_~k~D81QoRqlh<> zdgMihRh)!k{YD88&^l{>aneJVw5ES%ZqowaC)cu>Mu-E7-rQN)Zo^Pzl(52rcr~26 znuYlC<}R})9M_Hi`9SqNgN9$j^&5bi$YfJO{oL7AKyZi99}7VQQH++yw|<_vo7Rt9 z3bwVuqz)*iU4d`3wy|4pU=8OvwCSZAv)eSbg8EIUMDfjWnKj_Neq19SwMPV;3Xmp6 ze_isW!j*=0Le5L+=18L(O?B)I*;PvXo!t<`w=bZb27JWsCdwf9o39<@#C+wGHx!0= z+b<(Jrlf>13~p-HJ=f_{o%F!x#EAKvqFOmf4ue?*=P0ze&N(^?b;*|6g6ANDTH>lX zzZomk*sW&=*JljdqaiCttINR=tH)ii%;!<6SoqAHM;frJfoqfD;mB@NUd!^ijCauJ zA=mytl>}2~mO)h-DX$UCK|a^($<=TSQz93qD@@rO8c4TU z0i~PXFH#C*4SrjiR|3mI7%lgqcXn-vC9ji z)!nL8Ulj_uv}kmB$ub*Ch;eb~6giq8NI`lEvg!1HjP=#X@FQuF*w z+O?9XjSiWVrx`c+{|`ee@Y@IPu;Q9x65R78#n2;dr3{bpGqT&DHRW-s=sP1A*inbD z-Y|Qd0j)=UP}7!**3dAt55>&J1$~RkXyGG`bGLU&YE&$Y8`W8C0$*LT_0RJufQya& zW}(&QZmrTfd?R^CLeEhz2NHCHDBd-vyQyn!JdTH;v7cyf4eQM|gMpDXfGw`8Z=+K_EzP#`o(4wUQC(Tz z#7Nok#bf&=$1Wwi7nPr5GWh?gnIK}?Bl*G(S*DqAfR*{MCK@yze#!4jJ0^eyX%Oa$ z$|{Hc2ocFJhrb2)*9}+R^t?){{e{%rL4miJfK-B!2eCh3eG-#c`b+k6cg+L`2;-{Q z_J;O%4|cRB*V|l91fP+Zm%;X&ehMo9$Cy%BBq+>wNcQpWgN=cGc|=Uu=A39 zcXPR)?Xe{s5Q%sE7K#6D!V#eB=uR{llWnt`nS0Oye&N-G8lt8^Z`Mb{D5dc!CkS7B)e$>`mD75;rk_Ni!wQ?#g{oWcXoZ9Zk0 zDF=@;azt^fR>m3l)DpNndy}gEK({-A{bNBwA}wO1a3ON}c7G%aj+ZT+U0=z4vmWS! zp>3h;7Yq|Q<8a3Kt|)G4P|&;n^?~^-qfP3-m$rOfJ>_?6p`eqdpV`|FfW+ck-Xr?{ zMx>uNHz6W>!yJAp@m=pi&bn?#>MalZL<*|S_GIrDbbVT=u|8lRv{?btQcwmc+Os~u znCN2)xQ6NxD}JKWG)5&uSn!hJ#M89RJ-$wyJ!jAM)_V*_b=_M{YgA%20pgc)o-E}s zalcXiSZOgc<79TDQktt~Ig!=uu|iEVFSF02SpoLUlJ8W9+P2F5W_~xStVTZhbbVtE zHign<4$jEqE%(VwkCa!X*Wb29&KZ3DU%MIrXHPtzbdr&zyoSiAXE^h%#fC(5J@A}{ zV{OUS>B(K+;JMwFh7(o#^n0qkXF$JVI_UFho~?AL>x-=tQ*7*+2XMVVXi4HzH1d?d z@WB>{h&{jcTA-+GrETxD1=gi|)nd}wXc-a>yd(Z_79nfV56;d(GOFN7v7$bq0rQN|V&cn@Ekc-u47KH3?Yb?jNnP@K-;KH!fru@fh)bJ} zo81)^?9aRAPy<+=m^pq?JzU6WN{r{H*%dV<`UysyTSzc#cL0J4VUQOG;>SwSS2H{< z=E?OQwnzolmcESKhmFK2))N2-gv`r2L8F3zjfbs`8Un`mk@CfnpDyM@r@C4JH{|g4 z8=s)m5V=JZ-cOgVsb?IP7sjM(fYxe|oU<1+<5OgX`aFE38n6V@36(TZEtOJ$t!Zt;lq$GR#%UknHDqu`zlIPR%~vkqvRKjB=t?nc4f!Nk76A?Px<4rCuz)?NrOfH%gl;AR(&qWxvzo z&sd^~$9-pT63(?1W#)f@Wx>-4a!rGaJ-5u&b|y7g9_ZdWdMZ&Y(GuWQpiL96g8#B zSHT^+_YhSBMW5LYTcB<8vX{mt3ByEO1 zgF2o$wT9PX8NEJO#$&hfTK(QA`i!=ZdlgFS4~*zWn6g1OfUk^kVu$K{a%?8+SwmtW z@IGU^zuUf97`+_OK<07R!bm>lMr~subHyusl_eh-iGX?K70_3ypiAotxY;xm=U+`H zck>+C70FmUi&D|Q&RK*5V+%^+F9BpykF?-p5pVN^*lY%{y1S%{_$!1V6dq9Q+Y&E6 z(b`3_F00*%R$8|a0F6LUXATC?G@G7+26vDzs1WQr@v$#b*LD@fg;V!>P>I!4-uHs(_rv0QObQ`$vN( z()}!-?8VgxG3+O0LHBR5M#xf$5$6iWrR1 zBYy&weGq#EJo{p{x~1{ODb z!6^q;A|b)b&^@Ec?{DNsz+P0_LV2rXhh)kE&p1fgF%dI-CwO!LP`w8oWI<@yXi*s} zzMu+7yi2(MkvX&l_-88=&6mw(LIrW;b->=hbmW7oHC_*4uP!uUe6h6MBw8vTr!ojGxNM7cr@O#$AE% zP6odzlbpNes2+dr_}~PY%dAMQ-5MjHaCfs&%c#fpFZ1Xbu!DHN6*VATgjjY;#$;s| zg%Gy>ybm;kXQ6 zOb0P&TZIoEz&*Bl2i_9Io&Mwh>20kT*(5rAFlPnj^n&vdzG2i<)JA`qf>dHfrFaex zD@53YLZB)S4sNFJxQ^4I4%=Jzs#3p$coR2zOY^{%U$!RFqPQv5H>uCT4fc!)gWyGQ2eQ;rh8LY><=|2R2moas=qi7n?OBI^ zA421qG0)f&fIx9Fc%}hz8cBEhi^d>Mu=B&%Gj!?q1Gi-NU9fAu zkMRk!kk5T-!}NUVk`4r5(Mfm?fD%#~SW_FCw2v>1I}SFfLaOMksPbzlxb(`Isi^vl zOGg%{lApfnRWB|e+S*(YCDZDp<}h7>Lw@1eqI8lAK=uu>`!b zMTvhLwOwjo28V7NNwuC&S|}BQ3*F6!j2VV=ye!A3D@K!TO7C%E7hbZWu$=`a5+}xQ z?HA%PMi4JOtGUV*DAY`4iAuIr#`|{f0%g4q@p?p{AE! zV4xoN-g(Oc_IY}b-Ez=$sFk{<`4kFHsAgsOEz1#LPcCj%t&QMN@GXZ7%DYjFIY{3o zSC9zB->VNN*hk#|P<{b8?Ny7;4=(0=?54d9g`q_{yFxe< z5d8x83)@LVcvFbVbn;gKpww!iuNvewnR>N!sR|C#z({&8jJ+Rvy-|Na!_D7^0i|4Y zUc4Uztlpvr$ixDQtc}2n)fUH65X)G*VdirOX`lN5imqIEiGG3h2{Y2SeFIGx*R=-r z0GMi}sTD(Tq*@{fqh!3V4uUhE-SC?FMuXVeqoq4w-M{3~N=4xBok%zUvdq+01LNlb z)&m`;gL6!4pQsTmev_lG_X`(8IHiXYc5}RE93LAiiiYIZlD*|f-|D7PD61*^TREc^ zl5&Lo#x*1_zqvCeaf28~ztNha$?qfCwQp)_suTq1j>#d2$!$=W)><@TH9GakvGtEB z>j{3JsNl~Ed5eB`lH__h{a5~G1ER&@`RV2onzDl4T`SKu zUZqD_Ls1;pAY4Nk&LvC`c!dM^ZqUGYPOw3;udffRM}J2yT;B~V**5)3q*HeuWHr)y zDEID(|y9;k_VjZOzCZAcR)@N%e3qw6x^oTt-`u620yFxKh4FG4ogXH{i)-A?7M!F|jw!^h ztgj%y!IWsEh3DtprodthE*xy(jIc3itoQSL$_mtw%d`$8HTPJfqQ71gm(v+oZJje!>~S3(MC(p z+kjj>?jX4UjRKetLxMDs+zH|j_&2g#L0i-XZ|T^g z5B%%C&?FDFqRm!7gB)8`6FIrpLBxK$0GE~Zcfw(cCk}7tq4+-Hxrlr8Eee&uBRKWJ z5?unZ4gF+|0zlhWuiU!*J`yCb>ze_{qZB^?>$-Q3xq5IKGMyty!_4h892}z{kxDTL z6BHJGAa1HQbn&5>b;QP1RZgFke88N8kcjOCO%{HO-8|?hs}TVJ!8a*LKtLG1r9`mh zymE`T+GnP^mkf|%ef7-9>$T*?f~kyX&F4^psdgGTVU#{)<*Ybfbv0&{kxx~`wkHq$QX|`GGD_@v&+U0%M~7ff5aNF1q;w^M z1hQDALt(9I3#_Fs8Anc|$fGC+C%E2&HgZTgXD#1Ee#Bp@lChP0v<2`*6p5?jyQ0-n~`F>D5;+ z)zAUZq}=M4XnO$rS_0X~&JsFlxMP|1wz}=BG#Fm+J-Y@=?c5WGodH+IO58W#tlrXo zMXF-X0EY7w7}%jMeXk0gj~m?Vl))}O9}wdKY_UlpD2C#v?z&sQYZr-sOR zkammU!mEu$WPDV3XrHmzr&$zUJVXtkbz^eM-a&)$wWO)jH zFs_7-y7T;F0KRU`in?by!ohrj`>w(DglMfy@p;M9Y~ZD76VGc3u~%%?v}M&Yd{3zm z=rSYE<}ABtR|R*d9be4Kz@!9soX>H=t}?5ixa&H|t_ihA}OtPE(L|%JB#+F*6iKfYpn0vNiDDPDd6Qz z;hogG4h$c63NU71wmM{`V@9}7U=1^V^FUar!S~+kFdub;??C;70?7tnT|p=f7jl!e z0sgyh1|}bOQvCFi`6ks)YUe}JV@WSx`7IChI_>M$68W3yZ>IX7-)aJG)Tn#xGS8a@ zSS7C;`SZfM__mTCBF}tE&DzU;Q2HB0+ZnT62fW+Bl{I5FEPpe(?JL?_!~=+`78|j% zw{M{gDX`4DF)CuHpL1D8ho48vr@Bu90Ho%j&up8WhC250AB_UxjZCXeM z!Zu|95)YB62P%e0zY=gdLKNwwpWEHr8wCob7O*JzdNMrfss(Rx;7T4$iYWRW+|@v!V}t@pxUR|zqYpQu~E+_YVMOeV_>lEG>{S`_yET>{y>TpEv+q)k@x?LFTm+w3n^!~B# zDeaD_IJ9hQjSKm#<3|@ZH=N;De&oB!a-C9Kf6cQ~eXB^B)^mb0&+d+*>Lb`nPCvV7 z3z6nXpK|HTo3F<~#R1nhr2+guxRK6e$DPYoI=ZWC9-VVK?vEVzjL(2D-`125eu+Bm zLu>C-^W6r%6vThlzux~E9Kr$yxx{}Gxh0(&L>BKnsMc~Em6*}-^s%2zn|+_QbEH+F{(b6;o5hY+x;B0veEF_0sBKgh^Kh#& z*Iz3$*m38nRpkA@#X;U7hw)u4k8g>EW!yia|8-_~NO0c~mBUak_o)5sCC4wl47>z8 zo@@N2Su1PweTqx1am<&Q%OiS5P@`;IN#7d{sDW-Emh}Ag={!g_^AGhH5Nr)}ruoi; z;3B=0WGkJ&M%v!YryoHK(4-U6uj?;(Bv|RZ>+K5)0wNXrWf&;Xp9Nl@V^_3d&VlnR*(wNNdaNmN44(xluS9gR;yh%9@6^Lv9{agu4u1wf#- zZPO-!gQ|!qOUUiAZGM6393QIKT2o>SW@i`~q0!cV2v%Q}0vSHT_j9{JgFNV8=JBwV zPXAXqK5HgI#zz%fD;T4mI2G^JhoGsAPl?(uy1}^!^We%}Mm?Jp@+km|8si7+L**{0 z*}_N20KUva%iP`0K+wS1g?<$ zk>VAS&LiSB+VQa4@1xH_8Q&l?4^T43KbJMdI^|FMKtab($@^j2<|d}5_FxDj!8!;M zyTM8>{3O7}bFUd#Qx|A)H+H~sG{C3V?ST|Otc!Ykd&isrzpzqHZM$I@A#lIRROJCW z&ePJh46thnKY57#cCZ}KI>uH$x)UIOp%e}$elF7Dr+4K8GKqVpD}YwYtf;7%lzln_ z^ucx^VCCFt2nzWH`qxi`PWfr+>4A|`Oggk(aRPQjADdmiMK(Vc?B?yWf*SA*`mpZ! zjf5YAiQNk=A(J1i!D$B)=%23#_`d^3#_Ir3xLV7K_5g9Zk;JHz8gXYAYy8)Vq^LOz zL9$cTA_=O0A_49D<%G_CBg@*FFVE@@ghlkn4>L4I$o5RKIJGxL)0pVYt^6$zu8SC~?{LGWed-(2Awa+1FeiQ9W~kAd#R zfOKkod4URQ1GUMb)CJ#h7NMz?N*7=rnUv3L?-jEn(&uxo9cETkqvuI$0Ub-NO&7^- z;CC=%H$3Tofsnp{d!U))O0;!ewcS;8XI2mf6W_Td9uq7_#DRLMy{uI5m(!C7PT!82 zsAIcoYjWAUPJp@;ua4Uly*|RoimpPfQ8_&N2-)<(p=4{SrtWaAjmll9W3|#Dq;i68 zr*a+dRD_^?Kte<4tSL!;nU4_(;;@Dm4zvRPjSOY(NQS*aSWK7JEC)gP0Yqu#~|SSm3=p8gHykc;>IO*Y{l*>xPqOOU%ayt>pFo^2y|9k!9n2YqJ*EVLHX>xhLG3cf&0P_NldqUujh)yA;|6J zybG)n%T#}cH=p%2%eGv@ON8`jb@GUpMTjt z_Np&ifu-8&jkXhyyH*ScG~toTUhcLPBE>@nL_-u643=?nbDAkq4Rfw=QEu|^d;jS& z&h*+oJw#^NDT`W6-mHQ;S82RF#9{dx8|tWmEIIJ6ivG!0Ep5@jCFqcXr7DiAaw8h9 zar=Gh6v2^+C=d2{YFL5G)s;wri8 zV5BcU_-+pB1+A2e|Y>b{WfnKN2A3VPB=TscyHGb8!(^WN01$Lj8RZ!LY<1nXZ zsjU!ub=1rr#%hAZ9k5Eh7N2~!-rE_@A&#f7q}@NFbKT6~I_U(ihAfSRqk`t9{fp^S zTD~m(5nelNX)g7|pho?3$KIA?KJn=r?!q{U7+?31SF6+a*>L3W<>6XO;q&+&>g z-B??(lv@sDKO##mZ$F|*Hp)BbAYlCVjq~I>o=E7SG!mBDDRK*CG48&T6Dn5)7fuN1 zVEhN3E8#tRTV+erwM`<8GJjhOqb)r(hFcK=4IpKJT&%WbtW)agoe zQpP-s2f3bK$NPt~px)96G83-Pz>6LX*Lu4)H~T%ivBr43cu@XqT-5#x<;ZL5YJ24` z?6)2}zfW>(hcwxV^I8!$(1o2tqkpB*wRztZGeYyEldXZQIq3M9ChEoLwF_f&`qw1g zcyTNIS5B`s-bI^ggrh)4_wd~Pc^=Kf_+IQi4(p8idA8T#C-;YkDNmYLsJ}~)jb$CG zbk51Jrb@UxD;hi@{BCx@pHO@$yOIjS1r9%7C(u7On%~!DUT+WD?>(a);&HK3%H;jt zDXZmjYFYV(3hED7JFcdWgB+bGjK-~{$fh4XtHFi5<#L^S_mmo4 z=qU9QUG{i*QJ4QLUVmr2Rc0qp5&_c-y*>_w&e}3I_p)8Z$@&kSE1t{E*Xdj%(VHNz zJvFqcHhr?FFzxkfUApPW3ALf~pPHDpdJmb*G{qJVYQ)M!$XW6DB`AAaXq(z5uG|1K zTPy0%OwoflDc>6&IDAU8q8xrIR@e8v@ih|u0uO~+D{VF& zZKY;w?kg~|bQg`3uyCDRpiDndo6P7k>dsq6Pc}`V`8Cp}GS$N!VCVB<= zU45nO-X%CZ);A5B0LI&+yc^qr(1=#0Kh}1_0?258j!}QX-o$f}kN|c#;5oUXmhg-&W?_+Cmwx1+>{bm;p&H@@<}_Yw3qqWd0jd+&L=^3a0EO}@}@$u}{#CE=@1J zme+26dl$hl>6nK`+^O=N@FaZNxiiO_Vswo={>c|SHM3Ifx2_s;!NnlB=QngZxDUd= z7ECsN)U3yH$j%)>ecij+huw1URrqlvJbrY=MRiQt*m-lVMOdJ|bj91?(hI{7oQH`0 z>vy=tmz?l*TvmboJ@6AjPL@YexRC~gt}pIOPkrMkc|~KAG>fSfQqUAEHj|!jIC4Cw z!SWc%l89HOB%BR}_i|_~GQrF2Oi{rRPK%N5zPVRtmlm3*zBS|1Ep3cnv<9(pGu!1J z6iS(00KACW*sNvRoaOq#i}H{I@=wPo%6tpoVn_bY>{7n}_IbF^N~i{5uRJNfs+x1m z?3rQ*#m;mY#t$%iq3~uzMBkgdFnpq#e^n)Di-x0dNP+x`9XZ1JHC8Im!>h}sR%l;I zLc~nR{ozK#K@zp?XJTsU{?AK!T${RfoeFhtvcZ$g!Yh92kb_^E^IbIC{F}xwXPjrN z)G0G^@%zkl%gVy|FNsHA!%-E_$B?yV$M8*g{NnzLjSjPxaY|qpNAzm1omM?Ni*IP* ziZyjFD-G2-dy;3+Zb_@>1`RzdznD)yU*^rs0M(gnpNXZ5*IZyBvz} ztq76eU3h6mGsULfsk$e;bQT_gzg_BFyA+M1N{BYi$IDV-a8n$7MVzPBL(UPD>1xtK zq?TcDE$%;L<2o)W;GA*yj`g}9i=oT&SX1MRU!pBZ;E!(3&6s<4VZO|RZkeq7yooTX zLz$Rap>o2su*(KVFd}N^7@7+9r(*6MGqd2k+idt;^WU+M!z~ zKy7;IXzm4UE_KZc(WNY;;hf*AKY;k87C@N@SFChSwyzl3Hm!i40?1lhXHd)^k1FUX z^(iN6YH(9va}R^%sGvl{aQE1(z)2C-wP)+7k$TFh;8bTW+XBC;ytW2;vW0|1#8t^d zuPBy`30p<7DKmR7vckF~!5Ot=eW|}~&jEIux?tL7$Y;a7xk-j~(Rd$?K;^1Z3PP?? zyFQ90y5%dd@WJYM)i=sYjTXgdUM_|Q)Ic}RsG1}N-+4*1?1M^V=IQc%LPEmm&4 zjpc?y3G~q*?%BgUa1q?T2LD+_K(K0or4!Ka0fQTc0&h^Zi+hV#z)GQdQm8WFql_P)@MH(*`+eiHY+64=BeK>NAjD(7F!!8RpLi7Uzc z%~N1831}J~+VBY;mHXM{)X(Q-tL_*B6BI1AzP1fW6HL*4TX&Rouel#83I#v&p-}~3 zn*bJ;PyWeWW%`sKof)jF1x6;tp|nwUk9B(ohgLB8ytlc8 z#KmVfo0i{0%i47k&>#od?&G%$ZuyTgpj+NoKu`in@j$BP_8EXqG=HP~-Amtf0@|92 zh5=*RJWvIkTh^URqjye3DDF+mnS>B>fz5}M_8z-^Pze)5lmq6bMU72Ghmj3|`%8a+ z2uf&Q%wc-}7$~U%kl`&6#I*VUtdUzByH)OB&hGA8#$WCo|6`luLEBAD z6K#}K*g~I-HN&`@D{PyOC5N8X1|sHrx$axUDv;&2wW1db4Dns?O*u%NxV^{Zdl9H` zNveq_ZBpP2<9{t83Va+tOrH<`8=uPunFO;my0COfG1oQ-ncwM`#jWPdHW7ijXFEd%jCpJJG5jJtYzn5u7S zQg&|B0y&yCsX?2Ab4-7K4s?~{PYM0!ARmNm*gu`!{}6%yQ%|)~lF*#U13(;Ipy?om zPyZhUi+r64SSfope63)Ub8wB ztmr!bu$O>`52PHf>H+=%Y$IUx4;%8t-K}eTe&`16Y97I~VpcVP#DCX&Tjj|`n?&&_ z9bF+&lbpo~NseCCk@Q+}D1ZJ~T$xLmDo`fc?u*tNQLw-=T~b{9C4Q%({MP!nDf!)D zzBn4RZcc`bZf?i2J9|zvR??OM=rS|<0+bY0qiTOp$D}1yo(G3FX^CpD=HdtnN?jE^&C~Yc3#IoVT z9HF_&FH3dJCZQ*2>25Peb6ZM;jCJcHhw*FG6N+N)r1d1y^Ejo|{h2!Igoi3MtGe$6 z=H~NVLP}=|iQ0X=NJCBeF*h%Q!wkGxZ87z9eaie5Y2I}u`wN^7sZ-oyh1qG_YY9^Jip z`!mPc#4^vz**!5UdVA#P;}0V1FSy?&I;Or^Q;}IrtrYU`p%{tYB#AO9*+b|R*M2FX z+6y_oo6jiTo<;%P1i#RhVqJgsCZr7; zvajKgum&LNbX--nT*L%|Y?ftjhR`-`=+ArbcZMIwL{`xGes^fJP>j%M(U0~r=t3bq zo9iMNUvB&RIH|v`>;Z?nP&W2sVsGHhlN99w7Cl+AyXem0e(4}=jkD!(2d;YSiWOaR znNW@@vy&2G#iuPmoXex8mIBA843o@+C#J1SHEAdGc*Ohp2Oau^x{zG87Y^9ZT#%f~ zq6kaq(E=gMkytb=0=UL^P!yvYg$}lOH~G@{2#8I+2gK(@y6TR_%llezSkaWruiPn& zuE#9jp(GykFz+L2$VQ+Y$t$5JZX7F4$`F-ZxNxZN+yp6_Pv1%4qfK;UF3wFgq}VUC5TlktbC90(MXx-_(<|n+a%t5}_{?bdQSS z_E)`y$|$;1xqo#stlFUEofK?e%dR=zt2QfEv3u=L#M(u1UBU@~=m_%>!(-|1vvDNi zq6wxY)1~ubWi=uf^Ppn%OIB_#LvZPAGgoN!^8*kQxEDtgR065#N^eq81D{WqNR2i9 zKL!P3w(p22IrpWKSa61Jk;?F-MoV0zZ*=PY>`!`QB@UX-MW>3x_4C!D^uas;Z~iU{N%W*hDiF#v??Xe-jMzFxY|~Sr2|F_xP$$jC z7A-O@&M-@<$E4`a(FQ1u1ZwdXouA{qVl@6Cg6ou;ZvAAcwtopO#ShPO=Nj!WbHY+}jWOEMAXNLIS~H}2yAwC2!Efsem#qvE)(^DOD(obz$J=zw07q+?9+!X^NC$Xyix;P{f*&`;D zLljc?i;#yNbLw{qdYg3^FnLd;52Ly7NHYLp;w%xo|~{VKqKj(aMo zO}^yriXe}3F#hv@Dv!fuUgJ46{OziSW+N+pgWqfqyS=PL_>zJ zSD-^cu-1P+;bgKyn)@9#mB&W4=dY2aPYx(J1iW*iu{VtP6VlHTImC41u8xk&RC{Rd zh(quvaMD%%8T9Z6ZSh9eBegPjj)@*;f5f)LRLx z#)u-hQk41|TjI+TB744sW|<1qnlPiLd3%Cqy~DHeL|6KyLo*Y)NX~rLYn~swS|{LX z_5(DMz(~dR{x}%F@Eo!Yv@!C_i7Y(Z8Qpzjyn>-@N*=*Jmr@o{mCsJU+u zSS{l;kMXYGKc>6ao=shKjyKlvUDk)CS;I`g3Z;E&jT*|j^UA@xA`9%92{JX=dID1} z@8Q|Fso41Y$Mk8em>`)Q=!UmE21j6_B7lh889@ArINqAWp{Nr)e>|Mh87pzp6qP3# zAG2+tZxM}}Wsr)%eLYM-_A@l2`wdss=Delj9oJ3YdOGRK#MXKmH;Xh3tIr9fG5XYyree;4RTFF=GEO&-h4zN$Qs?nsVy@8{iD!f~v&W)p zk*zI&;G81%wn9g@`*pPb^YTQ2o(j1P|N|!4gOzFHrhbI7bd{|eLNN%Uu%EU zhS1&%=*n9_B*h`?4Rnfw%uo}N|M^59{*N8)e=RDWcphB9w|TVxkLOmx(jIdZYFN19KTiq}U%|!)9CZ^^>O$zsz)|-D-T!~_fI&2{g*h?KlR!MjleE(^j~-1|F=GsoH{~=!Gv%Gc$qi2dFnl_iwXb z{wOj5YY~dF6?gyofpi##@%W8k;6*t48!<50wfmduW3wsnEph&z?E~u|E7z9uV5{iA zpv{44c+c<5zylY-#}9>;Cw3^}w_s13)uaa7+S`{v7A96{r@uh=wn3{l%UpW^(h>_5 zfkpTXu$W-?{u}$MAeDF+CVt>Q_cLK*G5xNrzK?VlA{Jbw_FV`3m_~ckWc9(>XY;Qg z5;38z_DW#OGWPvpjxrRO^HXG?|6vaN?^T$I*}yVl_tLQZnW_Q$kCmf=E4=8Jhj31c z?%Bvl@nnDnZ#o=u}_b4?@R>HPi*cePU2EQ1{Lf))gcJdY2!%>UGo1yXyM>m??FOP_bZngZ`YXhreU{x1Py6UL{S{R1O=9_`pvR$4j|zY`Wiy;;{^ z8+8S9ZPvxvBH+?%YVNZG)}gpblhFoE$HoCoR?L8YnEp>j@qs_zYRde09k>+a@7w`D zK`8JX@O@N{(E7cr;5WCU$02_J0DNbF?|`q+WozI_+M!IAmn!y$fR_y5F)UU1*k(Bh z{TVPA2;4>Bb;|AbBjEcN7ZVJ&PJhlC{Q8T}2>1oGSdt)4{{mYzUdWTO1)GOeV7njx z%N2?d`^6)5b+#b)J5B^fs2igJk6Ae>J0K^7HywmAjZM$XYXMfR>p)rE1AO28|F&g7 z9^&uc+WY^YOaDt1$EKB(bOU~02Cl^ta2SUKAtU5>p2nsDm_P-mT_ZA+&69|FMK$?pt%MRmecH z^Is|mKBgS(YOb}YE(;5I`foJQuuDIi^zH*0WB+3dLju=8WQOo=*m=5iF))t5fRmq} zxbfCpW4Yh47Oa^8*MD!~sGRrIqpG+Atl~UiYj~9HUMr~L7cc1B2&;|t_uJ>X6Tq+f zFWvZeQT$&Tw@n$>j(&=)9R&8rzck_m-U;)f6U3~~tx@g$4sfPg#tm{eOG{0%R2^CY zIjPtehrmr2{I$Datg-5ij4^{C+PB9oh1Azu6Nl_9cS25>LQ|yc&r0KDuaN}kf{YBi zKW9NEy9ObE8747()Fwi)W0f6CnP!XiP&Q**? zSQz{%#E+I=GAes6ZTt1NV>iDZXZDZfu<)7R-sM8jUzZpV|;#w9e%nKeN`gDHj&v&lFd{fyXUcR!Ai*3-;e zedrw`Vop08**(6)tg#m+L7P9#>i6jqKPJfx@s-vL6$cH^GNwVkw`G?zzE*wLqS-ff zTO$672aTHL$hCVCMfYwLMHI5_C@Go%{*@dRQ`5u}pb#k&rdMvQi6PAdUMDeY692Y6 zq5ZD>R#+`YGk4K4@Y<(oxuE=PTs)BpS{@`q_Z()uUk`U52(&x7N1(=-LDec2asG=0 z4K}g72BR>n%zScKuRk`J;>RVWO@z{*=PX>jl z$qiTYpQefEJ)AZ3eZwI?XQPPVK>v;Q_$W}+3L)`<0Dku7koZkJTo z^Ku$ZjWe8ktZDP`RQ`^H zT;=uL7Zo65W%2vQ5#Wg6rr?YkFCjc(^%8vS`OJbJF0k8mU>z-6w#2nJeCV! zx|BDK46+Wsq^LdN#xVxjvv=6)Yt*$5Bc#hhwrkzoWaiNS2dj}hEX8c;hL@L9yG&xnX(Ss^@B~`V*!fp0N zMe*=jh08UsjvXN>1|(^vMZ$y-`BTnRNeYttHJpXGI5qKSG1{(k#U;efw3d9^G`YCN ze0`-^DW*xwAf^<;87y%6xwW}9gdq%g#r@Tk(>`5#gmsT)Z?i?^&3CrDMVuHPR)E<*0Jz2 zL(G+V5bnZF%iYpa`4sa$B{ybpmCuX#3p$T#jmdtEI5gCV6luqcYItEJ$y2o-Guw(tISM1Lk?A5 zw2}=edmZ0)M2Ie}RNK75l2~4;pUj@$R!$<_lUT>jG)c!p3~{fS52VE6zpmAu@0S3Efw_-dV&!?ChDn3|Fcd<;mQI75mo zhZO3zso{k2;O9Orf4Ga+azVC2P}4{suLb(C@U+I_K5!5SU~Q;PSnUWe=nDMnFzKeU zh*JNml7)hg)ePeU)rw%oNsL zn9>2Zk)T6YsVn>>DzWUtUoZ6i zpFiGI5@A014f3q?4UJ3=xX);0$K%<19efg!f7p-C=}dbHjTa=9t5~ zI0l%!O0v{F?CO7;SR;p08hs>pE^WRGk>yvhs1Veum!kftX!?UcYvx3b_69H4^hFJh zsS`>*RBEp?)|Fg(adDrZNBN?Eb-AzMJecI-?&CXP6Ew63ae>;TCl7*ksi;HPw8ztM zpB-LjexZaGd=OS1W~h8;hm4QNAnwf^?OB_QVjr3srB`qMJeN=`7@Bz!ceF4DS8yu^ z+>Im7!dG0=isHS^FQU?R&;Mu!7E2BB;BNw5#KDPUJ;LY4Zt683>%L z<+yEXfzi6%!%t^25`XFTVAQANmVLWDcU;>|Vb@SfyU_#W?M5rdi5j$=gUD*(UY)Vi zi`j2vCL?Czv-Hgwa{`qJ%+-f-4`$ApTsJQ)WiZJndhm6f0{(H_=bp924mc1hb<228 zSgNM8)XSBCY%_IQcJ?$jhp`5;eI8Gh0%=n&OL@sJT&rHo3}b+Q4T@kiVQRF;>s_yP znM^Y>Q6dPIDnyh5==mPypq}_X;TW__^F;d6?8OtPmqO|UZvN@=!y9_7+ON$Gj|e|9 z?D9%7TxP)|E@8z1m+l&xw%FGShgV!BOXsktp$XX1ppITqfYvu(3D)_Jp`UFwzJ}P~ zNnWv~N_>ct{UMVAOPd6w!RJBZCq!U{g5lS2YFJ4C@tGX_QKUHy@M|GaBdsY)z1(?W zDUH+R1nnPenH!03j8)#JgE_D-7FoV9ytYzq@2fs#$X8I`!^jtsJY_B)%-~4x;V0Z> z;82YePP^)?G#=NdHJNelNudTEOgn2uigY>fu5Jvgv@}k+0%7@P;5PmY<#PF}bA7$o zAg5aCNK&tA`Bne*7}=V{zA3s>Dm;R!HUYZ#F5CVQ-SAwX<6UYBFHkLeft}Pe zQy}?1v7MJzZ=)4h7iDU)&eZOMFpgNCg(dl3N})FIs%5W>nkbXo&GYZ*@}F?DF|x+j zJ2x&g+B%ay4v;?x>gRI-pS=E+c};oD*SXDg*4hp%xycm+UhIgrk_&SwqZ&G2PcOdn zZmoQ=+I!IO0PpQ)Q&OcKhVehnsuD0%YrHU7inq0YY_K%u<}?8}&{hx>-WKQ>}vfWyzSzcWbxzh!Ev!omyAdh2eW%Z2T3y)%`K4g@ z88W9JV@$QG#7sq+$loS7&&&@G-tW#C?vIg6p1+vI^-8zQKHH^oHZ}#VU7^l_#4ChT ziU+0Y)w?4##2`h~m`^A!=;vl28Kmk>4&${5XPVawbL!;TZ0}7qd{n!5CV-0uAdn^u zb&+hU)29U%UyPRSm#?arMEW*FI*n?PPJ0Avjy>W(TR7jvJt>nrb&^tv&(G7{8(+3k z!smL?z5K=a(82uH=DU~f=z0m33h5t+M{8Fzn~cWA^CcpM*FSHnr)w+&IM?*BAr`k2 zt~W_5wAz!dbr)FQ3rjj-$q=Ow=!V{@8)!Du36rm*OE7-%oj8J)C{zLl4zZW&XH1xP7}m*YM4 z&QqgVqszj)$!#uuW+=N)GjxN+=w`zZ5Za)dH%bbeWTg7_zsIJ+{AruWUX8EUCt5z*VP+ysb zvWz(X_^#SMHI>{ik4~Wmo8y=2C9wM&f)z%5^9a8ayRLS2+^wWZN*A z*$1}hnH$r*uAD7BBQf2DxT}gr{SVsfk9h4}8N@iGc<7Tc=a=f+PFT zODBR$|0IP2jKiLbU}r3_Tt)%w>71whid)93(2j2{k=n)smD)*_3k%Rl5S#+9s=3h% z=Msa_LkApumdkP-1;g;YS$6y5gC|k5zHyx+Icd^pV zc^VIer5bMF4|Ah%7)q}v_t}#TH^>ig!g@9d{Za}w-7W}pnn)?O{Cy@+^Q*Ms1N%U$ zYp2rmO_Um!C*On%uS?!73#rW`yDj2{g~d+^iq|ZsbPl&2g&E(M`9kagKA?DHU)dP@ zTpO=P>&j@j-WYrP>K7!6bffq~Q8t|0sd1rgx%}qvoE}iE?_63d*GZFko4TYYJUHbi3?^{Z+SYHK?j52WWmsolT)_(I z%)L1b^5IG?`I}A!>1N8(C024>lF!Qsx{=|vO5z=GfNKi$Pr%c$T(Vd8 zqf7V4o@pl8FH&M(X98S5FdOKAz{UnSSqB@^a6?>B!&zq%(q*l~=$s`(reUZ{;|+&< zjD3UL;>q`BCL7#lD|y5ln3|#+Y-dH18-mhEmBstQKbl!YX$v3~pj!>S@f^kXIpb|Iz)xz%^;QVg8=-akapJa|}03#o~WDynJ%g z;skzAT)4uYK=Y2-myar;UN6=w-Ru$P8Os)De6b6|1>m*je(oF5`Rnva)HQ{A?_t@J zkBVSx8$pPttw6b5ual+tn3ifphve$g{)r|6}j|SJ)PCrMR@T4@3Z6hdA*a za*Or(o#{u9h~B|1x_pNXx&AHpaRNUSFb-(eHLP4S&we~w8U0F!#+7{ZCsq)m7g zNC00T3J6fZ>#>jZ=E8{&9|Dxcw&KT{>2y!V4$>b0B|py26aYWY0|1kAHy1Vq`?_sK zwO8oO)ryDw*Nja~P5m7~@wc~7Xi)K=-sEou@u$%sd&tZ0auVud%`B1&Tv4emE6X6m zu5u zqFdqJlH&rGnLDoY-glm&nsVp(IsN!<>YV$7! zkz8t>J>=wAE<(McILYxUjcSxDqfC-oRECQC;MDQ5P%gEHHA^0OT)YMA9T%F&O5ew6SmBoPe~L`)#Mz%wGW9+S&8bXaRSl zEe{@>k-MGtTg_;d-U1|oEo?Y0l54SPTeogChpSAjF`8Gox2!R$O+17cITr|A%rM2> z@oZsz%lzIW3D!Mmb={QhT`9*IZLpuI-iKLDx9uN~x@wT^Npc>)zqMk-fXDvHz`@Cx z4`}q?&Q7P*Wr5K6lq$cqCE_8qiTHmI!ey(uz4ZS&^Z#U0%SqW@DPZ3s5X1=!e=Ot< z#{9KhI0jOx0WRisL!xs??V;*+7Jrpw8mRX1fCguxuf%Rr^`tH5c9Nd08M&?#7ds1i z99Yyk9*J(N`qSkZV^?2aUs?9;5BR*MlHv{E>&tbEj?3K>pH(R)>tD6no&W{8a;2is zdol$zeJ<+B+s)C!d?DANYt`nf*Mzg$2!$-&k6HTs7klAZZ98BiVDe=9Z<4FXSi91^ ze{yI-Yw1&6S?B1yQNJUZ+_@kFdx--i|CDRYaQL!?F~tCx2Mi8_L@jjaG2S(HPFpb$ z<#g#62Ia6=s0)TVj|Ea*RFSi~$M47Z9Y7Z!0a$XrUnU0Qo3*brw8C<|gW{pbY)LYw zd#nKPIA^YK42@b_pQsvIYn?Cz21?>=_W2HmE&eeU3(I-Tkejp6;X`#idopK7lelYo z)f=*c<|5()6zk>j$kmu=BiZ1$w zFJ1lWKGm{}yjW{-x<(^W0fqHXTDd`U0Y7Vtds4fs;?lnt#g?-({#}|>poNTD@GjdGfjv%T5Z~gXM{>qEl-q1F<|8(injMxW44?PUqA8o1g zf`p5&oy#mr;~}%7OWG2Ej*r}w=|4TDrqY%&KPu56rcy=8o=-Tg4Bse<|I3a#l|`=s z4lh8Z9Jrj#KCVFlD_t`a!h?nW4h^zuHFJ&J)_N zGFWt8H)yM0GGQV%Hl0U14#?pXRTGw;v^{aQKCNW$t(CH57fzj2=)Tam(BX_PwCQ}n zczMCLUoxfyl8+p=tP|~8C~|Z+bIr;sApOl6(^aarnC&c+u|rxpbiNd=8K76zT6%$v z(9&6YbOy)6+7z6+wANrFPF^vRo4(Fk(_q=|Ri`Oua8+1E%4S1Kr3{d3M8g!8yTj+& zFa1B`adkw>Ff%rP`&+r~?wp%n+s^+m^8=b_eJ91)r+RVb?cV}hZ*BU{nYI7$>=&y#7b2)vGt%C`P?yD7klbZbkh3 zZEvofTTrXN$oKF`z4wt<>i!t{+KS~?*UbhxXVtEWeHP)a$o^UVZRyiT+X8wNZ*R$! zo_?!Qb>F)mwtbK0zqhxEv`_xLcG+^xeb0XUW=j@i*Z`^$;k8eLWtZdQ%v;|!mA*ZF z+qKYH<6gZ++jNPUwqXd{gUtZ^Uo2v#= zS&vdYJWKmi=Q!JKzE8EbhTFoA$8Uc(>zY2R`=QFb>vj7pGI#Ar-14@z-|2Ty>24nD z-3Gvo&9hMoq8YPyoHZ6v2F8(Z(i6$9^RoN(1FYA@q_mvc^YNbc>-Aq&d_J}`R-&7s z;VdvcBC6RlzSYU)vqadoyAoOXMy?+%XS1#NVPzP1 z^!%2W@3*ess9|R&Qh)s0vCuy;kEJ6R4#=Q-Ve+@7clCY=l{9~BFqW%MT9J8wamMsV z+1uii1(dz-1W1X;+q2C&_~ZA{6O(KA*7mwSo_52Y0rF=@Nqj_r1mHt!C$=dZt4^Z%-`A=lRTxgxb6=9ic6>SSTq zV2_e2r>RNzY%Bf$uyY{?!wg20ddmAt_x?Q_svhqR_sbXU%@;iCeg5cUL#?BqH=So^ z*inUI46t#4628FJ0xg<{D4`5$e+@TB99;-n3AkWY&XJv3lT2KJ-s{%2P5?Kil{|xi zjomFbHZg;no~onLLo-+=0lRxZw1gGd5rQyPJfjMLF+E@sIjgV!XGr99=-7OH?Q&r9 OWAJqKb6Mw<&;$T(S9)>) literal 0 HcmV?d00001 diff --git a/notebooks/articles/simple_bike_repositioning/TripLogic.png b/notebooks/articles/simple_bike_repositioning/TripLogic.png new file mode 100644 index 0000000000000000000000000000000000000000..68171d8ef247372da1268d46240ae3157079b8a7 GIT binary patch literal 131866 zcmeEu2UJsAw{Aoc1q2I-AVmeFNKph7kQS5@1VRWRT?M3r1OiG&2-ql6l^*H+&^xH8 zlnBzJ2}LOai2*60g!Xn&l=GkS|M$Lo-+SY}G5$Sw4nu~`}ZM6gR z9P}U%=zxa$Wf%ywHyQ-m&9`qa@JU5plOOP7mn%#S3@Ys4oC0q4SSxEOgFwaM3~M)O zf&2Y;)D2xhAf_hjzg^AF*_I&CW`@RPW%ymQZ|ii4T%Sy{n2#^&oIRa?u66HG#^6D= zv$wU(iVRj}(T=A_**~ zc3eyF4?W*Y!+-hr_2zMl#4T z{#0Vqg9Fh{wd1ibhCDvVzml+fl_ckZ>G(!P8Vi+domEfWZ| zE!C~v8J*s5jOHJwY_cXigWDuXt95 z`eIifFLZ2;c>>e_AwJf`poUD=P6YHs7w2Nv5IbKv;fB|{x8XLSZ(!o$D@QRUyY-~^ zGdr4N_G6}j7|t`Z2ranKCPmg zM(#mg_G49lD96%Ylx*xQajIF!l3V|NYxc#JSDvQ>4~b6D38ENDae7uY<1BZVR}aES zhlJHmFTG-Z5LqKDR3}_vBJoT!S7OZq-g4_^nE4Eh+~v~MutJQpJ1?*^Xr-UrR+N*E zVrth1`jfq{ieA3fio*qqG`uH zrN6i24Fq~c+wbyK2`KFPm#?fGqNA#etn`5c6bTs}it^>+Lk8V`?~T!*7A#MDG^gPx z9po)eHIm)fL-F>!hbWyxnX@8-y@pEU|A^B@+5^#8_BF z$b%=hIMMHt@VT<^@&0D#Gyk$hE^uq}P_M%Oa; zUGFIJK;OwS!4&Ao;6?gWRdzsuUn>iBl;e9KzIVrlV$%Hv%Wi!MGgb}$8h6}I-S5+w z+*5GPA-II0nj)WW^+sKK#_LGL5}E_EGE*R|KLevGuV5zxxbernAFG?t&1$RYBZ3+i zWYsq*jeZopls4mjY>em+i8nV=VK>`G!{yz5KS8K*fa6sPK2==+=5Qp2#D!Ho_!zSZd99-a(7@^v{Bd!eW#M{A0?h#LZv z&4ji}^ySiQ7q4LMGtzdPp2@-~trl7(*$I$py}p^3FdA-koka?YomC6c@VHtQ3MMlC zHO5i}7(Yh88}NyH4TNvv*TRZ6wNK@AokNd%ua9r3(>n?cJ6V);_GDGVg395(lVwkO ziY#(g-Y%l!F+%XUxHFlm_NX?&N(EnIR)&zsn`SaZ#a2oX4 z=Y^){u$hKgHFWmCt&>4LeJVHMHVt4N12g`ap)LjcK}+K)4=v{Mheeyqb+A{@zShbH zf35Y20JCw|6Tzs$0le3DX!}zT=-)`{FJb+E2>9E5cn72cfnIhVY;*Dk@bMmy6h?7#UXk}F zo@UZiT%XceTDk)QF)EyAY(p^5yMsS_3<(6}I~yHTa-0g_cPX_WrkoA_+2ch{ zK0rP+U-aNI6~=??Vmm|5{p|3~mlJ?|jQsx|;=gC{KWnIO+_aM~eUg=v8~tEdP*Cv7 zYVYfn`H5ycORYPJUM%pzgNuiO4C3z$?&`#7xZ=>JP+oYa#O^QgQudTeoBqzugC=_q z9dhBLrWb$p8QxFr2^W?&4gNUGVfaG(9$fu z7f(N2&KU zw3WRFp7~qv3bXWiL@Vjpo1`e)g0@AE)3tp$D#+iMH|cU0jf6}o5<0|Fl!Qd6*}Yl* zRz!$=?-;q$dEY$E*OA#3dOu^#%Uja(blEqhivu+L>)+^HJUPv5l5PnHAtq}-+@aS5 z2!UVfh^*G?-WeT$fe2klWmoj|?pXC+D9uF07F0BygF=FZV=7|IlZTF9Nrf9xnTxvX ze>Cc%?+ru!P>~p5#>Ibg_y5>NlS-%ve}NZsV6U5+*L5=^G9P5;Z@# z0Y53fMvi45C(O3#FiO`haBScC81$%tZD&SJ_TbvOK3DkxS`DMqTB+%t*$TeZ;NjJ_ z0wA6Gmx|z54p~qPIg`FmG>S3#5&A4RyjL^!DdW*cPX|Km92#Nkl$x_DuO6pCjZtv_ zkvnM9jGGU^NtO@b`zk|W!Q4L2E?eny2(GMNdjgxV%Wwp^n~Sg?A@ez&?@s}o*DwW^ zw4edN)}YtVph~GszFA;Sp7mtTfEN;A%$d&bm`M7FNaE?)IC1OMm|O%G#jr;*zF92| z6-ngpll5IUsTba_S(d~n+(YscWa*Ym0b9xFheK;~>c%`u$7axTi5s<dB}yoyePE@X7QON=ch2)lI&@ zoLkBPuXm}*kP8+UF={;k zYH|&)B{up!ZeA;l?+i;7e|QXm+bE1mOo?CHTJ1ExN^AUBEq^RbBkKKG-AWpibi?7C zgUz}kpznc?MDUtM(_3AdUz3m&k~-YR^K_I>WIULE;{gv!<1P0@Ru^<*Nm!J}NZV?e zg&{eTH|(;F%+!p1u|x*XdsYsuD^&W7EpN;D_*@gxcdoB4e8lt70kG;B+-y_4-Cjsc z12O9viiUrRPT;L7c!k_0yHSUDeHf*$fKpvsKE1oB^TBz=yPExHEG6LdqNC20i4yA- zLM+~g0RC;{t?)l{&wbHi2aPgK9@9d6J9PyapKb^yj`>ooSb$uwd3=AU61`{~9Ek{` z5#OveC{}uJs9j4z>&aL_QaaU(5V(E6!p2laLJ zCUlZZc|3?eg8)_c!=uUGU}f!Y1P#9*98RU@fa%}DGY?Zf1yMQ2Uw8&^2gvRW%j*XT zAF&TkC@KlXS;}ZD3JPU4^bPsS#L~M)&Ir$`oyckMeHke&BBUPzjuplWz$?>RUHQtc z9$o0*FE=c?`X5A?jDSeR_$xuOi%}TC0o~L{lZwR8$`oadAC_I5cvemMK@$&ZtZ`Kx zvrN25CjsSnbO@pllN^ZR0|?VMkM;k_T0nv0p3j7wqFhZ*O!4MR9x{$Lc_(u)pj*kI zkrT7ub?v1JZ=}h~alcBTg|HWZO>HJYWA+I94zf1zPpmS&k&70Vr(-bGJ`Nzlu*+NS zF?&8C;vbAvPs6(5HG2y5`e=B?q1Kg#-G>AZyrFuB1^O&6yL(QdFV|Xxkgd$khmBA6 zi5}_p8!*fESwEL^gLi)(>qc~sDgfqqb^qB&qUmiCqXcw1_H)ytcw*cL z%$HLo58}nu8pGaJ5b1;{+N0sZb3B1@Wr^$E=jMd0C3PkI_1Igj)@0c{`BI7+bxxIC zl}KC|7K-VIgvp;eHKHE5g*7dmq9k`BUtr;a&`QE)c;=e;fQ8E{YS$I?p$qZJYgYkh zrrgw;eCp7!yLcTTasdkI{~*Bf^&;kg=33WfN_FRkX8U}cINYhE<1IJuGsY;u)#Lm7{oF8H zb9NxwQX=t4M!(Pn5#7#-w3q7564i;BD>owrS!{AbqtbdJ175V9S>S3n?=T4eFyH0N znj09FcVCoN2laZV6+{kHadN~J}(e!;Neza9mZfzQT1+5Fl>0obfD9Aw^71`nYX<@?RD}z_@SU0uo zOF5~+3-ujE_Sp@sqon-wP{7O7BDX@`CC%o)u8Q&~wPQ?Mi9CI-$EgK%!iD-xcV(r{pLCO0<>A1XOg<9nf)udbt{cj(G z1zCLmb>zcLy~Iv9hJNj`m|0V6(|;kmp&zXH`ZCu#Vc8%e)oS1Y8Uc>61f?U(e9@_S zIJ|h~a3)2j>`4*6P;gQERF)z*`y(%%4{`v`*4Q2d=5uU>p4Gb=lS5h!$MjUcAT3X5 zWRQ%q3&B0`DgpRB5c9m}_7T6LO*Rn7-yw0@sdU(QA!%cn z+Mv=lbpZVS^0S0}{-?MX;b;o*{?ufMs(>ju&nKX}a?^G5 zGSIG@jazq-*fNY$aD751Wh!NhF59LTfY;C7t?JUyRu$R>QrbH($tRGuwINH{_mf8f zp74M5iYIBzG8)+UO@Fif8%T*~XCoK+)C6SMGk^_AA@wkAgam}Tujv1SZ4W+!-tGYv zr26wQ&`-+0DF&PtmG}fk1M;UAl$3346{M2PRUfx_oCgi{2HG~G4ZtTL(1eim&#wie zZvA`*){@vuc@h3lJg+M#D$Nu@& zY3d}NqIj#T>y^ zZB3fe+iej+F~9UBruy@_Z@-@Nm)~?O^%Bn+XjZ_Q17mwh+fcu9sMd0v%(_~Qj9S&C z4!G*>j@hDqK)sGB|H*>pC`;n|K%j`HGdy?6b<_ts)wby{;D%9bV>J@^DKXM0zO^O% zX9L*U<{Gxp0$eM9l4h$)KtE{9 zK2D@;GibnL$4scZ0jZCElnIO-+376|)b8PjsI7why|?)~;O531YWM!aKWYymbpA9h zYWL79KlT{rw%I{y3$Mmw6&p_g<5Au4Udf)P&hwva8o1xOcpvPL@8@U!hc55vADx?Gro`%fJ|7|CSLG~+;|8XTet2k2owEXd{?5wmr%jf) ztP@$sj8rj9lFT!H{6qb#Y%$9-)QzBC$t_Rqy3pCS-*EtlAb`f8F3R10YnRPmj?igR zsyWb+43Co6V4FcSRpTtnjC_5J(Y;Q>dDrfw73H$WRx8~}4QBv7xgMK1fKceox&8S? zN&5Z7TUFzGsVQG!fgSAyN^bnk=l8F~7!sBNu9;DKtIM1QsoVfb zhRx6pt}F9oOsTb8uK?QaKS33=GHHn zAq%=bWoC1;!L3j2WX*ip8_*Uxwi#`E{NrDvj_zhP`Q~K*heyMj^Xw>o!L@)Ek{jn$ z8F{~Gscx=s9z$?(@PwQ1BR!^(Sh_At8hu9kz%r;PFQHAeG4wm$T2mXrzuh62w|PyX zmkGxclEB@qMqbJ4s~jt{C=vOhnv3eK)$H!D0v6gBJu`aUTIdwnZ$xXboxJK+nt)(@(W%fc_TFX7u`Ca}hiq{^H&=xS9(&j#T6jC+2%NZKzKNj*{6 zW6s3lBpF(PR#IxTzH&0KZNBNzL8oeP0IY;ju|+-js1uLm@GxD$Ok4M&x1_k_h(=FY z-Gt+0y4@pLWW|`UnHx4jUwBf$ zM4FcoBeJVj<+`80+y|)@?1gkQUs?l74UDS3Ys{A?st{{SB3HBSp%3YZy`OD59&#q{ zATIdpnfy5y<){TB+iNaflBN)ZORn8OrqyDW)E@qZn4_REpui&QTd2i)YGt`OtGAdA z1X5R_hAowaf@}o%_!2EO6bL?${hHKw`O9PQZZ=LqcxM^nei-7;`0WGH;Ixc^>=%{? zNBU>g7-UR9FGw+ggtT$iFg_Rjezc>#nc#!NVhh67(UExVex;mas0!W5s*Amg=o1i| z;W8(Q@4NlumAK|Q>_%VH_`$Vupf6x1gPJIMr)oZ@8eo&w39c)#&xN?WI(}!DdiClA zj+bS!iWGE58TzAPDM-%8#VAV|cO#wCy9;3fkqqy^K*M5GgmUA=){^(2fmQz(K%jNu z+9-*yQj1yUyZKA5^&#y_Yf}ICjSH#8y;Y+aj zzP%hkp#!heaSeJcwhhm>A`|lZxlvx;iSR)c-UJ9KK_x9hvIaA(#wKzaU)||)$jK{1 zL=wUP3T8pJ$mO9Jge)aeTMk2l?jV_I4=bP+i+d@pD;WfHyN3HhC`aG3BNb26JKeq% z8NBao$>Yl{Wr#_?9}o44#$5;rt5)?`S)ClZ7cF97h00{8tcM-t$Nc~ohWqNF*)U)?_^1IEd|F7-v`Q(b*Vo`am{0LK zL;+IR+rUsyY$@Q8_|#FT71?t|1`pWM+nHxYN1fav!Fy=;HcZ!Fpg+6d)qnP7pFE5D zMV67RX;G9JMsenPJpj+Mme|Cf5x0y4_n;V}7o4S_>uaCYPV>YW*bSppWva<4r}3Z4 zH4rs|)?R=U?7s5TFLHMH#qgBku_{Na1D8$ycyS`_Aie9rV&u$smmZY>RU3IW1f0?$ z$ChX#sU>mQ6|WAnskyT%e{hDmTcJwVazi}>wktd@3M>=~ofBf=WQ`XaSfnqP0@h+* zYrpb40WcZi>uvWL_#{td9-KB+e>X9b>@${>mqk-HRvR}U9mx_e zW&+ShK-1Z$D;<}z6?K97A0$jpAUp)WX(`7RF9rcx2EA1J8Q|@A1o#(WUK2{AbW-RO z=))`~*_^Z|TK{N4H#gV z`wSs#FN@$W^+Y&@Sp`wt#Rly_&EZ}y}wp-2ILIK)PDls ztG|G+lJe0Z-&c02Cr(LnsGSU8I0U}=fWdO@L>kMqi#z?3$S;rYvt7WQma$W{sp(;k zfrp*BV8~|Rw@$9iKdVDI&HYjh3g{lg@xz{ zOhJ@`HQW1E%d=zm&yZC1VHMC8dBakQM2jQ4ZYWjxEalHL?G=kw-z&zn)KZSvy=N2X z*){UZqvKL(pZQtdS0OWagl!EC;>Ik-P4>O;TPIbYZKZY{MV`lM{OB9Ker+{o{-QD| zA$A^%ZK(AS=EuM9lj@BvY-sJ$*mzJr<{~Q9-Bup1D$ie;19UDk0^~5M99y{L^6)cZ zfif2=?g#lRH-*wi(krk?SG`mg9@|g5%c&t&lr3fy%$xIFnOOVs%27|g&Al7F2pJiIiYlyR}T`B8SPnaEL*MO5xX=Hn*)MEF-2DCzWttJ z#~>qZh;dsP#fVwrm)Ke23qQ>-ibEX*Grr8IOHK`K?s!C1iKJHAE4W`}o&B(7>KKo! zC~tYPisQBfR4x>>_x@i(&e?215opAS#|F_FbpU-x>~*O6tO^PN{ho?k`>y;dxFUt| zocBP9kt(sarUuH#o_&$!R#k+$U9FRxn8U0Kc^-pp$mNQB^(35-de7-NN_A@4F@O9~ zxyxaX!~x3mG1Q`I&zcpvG+dQ;rTJxfr1U5j#VO zWxVYTr)zh`3ra$ohn+>U$d5>s;jb$!ZH{>EN$vDqUQN0b)>vv8cTmuUn{WS!~^_Vb$3UO1>A34|B#E0 zaIc>RWRMu)UZMU3rH1dwu>N0Wfd3=XT1=}C8xR9qS^$!$V~fD>*{SBADtvNcN5cE_ zGmz@RE!g@a!`}YzznHK7eEr6rKha4T%G3|x7I>YLw^GMH69eO#U20IE;9N;{NMbbd zuSxX(XnvrVGwOm#fCSL#J34(M9-)|1Hjpg>`yRIseD=&N!u3DX|)po^^OE`_wdZxCaF?#Oq zjd|<)G~k1?aERMAdc+yrLF;-ii2_Qfm9zB zoP91X6mS@bIE>EQHl2EVA9CT;DWGV<)t@zim!qSV9x4e51;8k)T{ng- zwnW+*SwqZF`1TsGwKK#H=Bj`2V7KT`Gk&SgyG=!Hi@N1nz}IKHx7jlP_>;X|j~`s@ z+~W9vubxQ$MWCPUfp|l=V^sG4-LCp>6Q*@#XtpU7z_+32gJeB{q7kDlS^VwJ-)rLk zmk+3<0);zT+XPiFBRb6N@|%F8Td$|$0BY?e^cS%vrIMY-cE}}lD;ET1O9TC#?cI() zO6>;;lUFWt+HA1}pxgu0Nt4=+IsYiv|JiQeE@+$1LLF?l%@PsXLH}GluB(ln*k;m< zz}ijaM%PV699i5O6YJn5ROKrwv)C%0QdH^7BpUd(uXr)sPiUgh>-2T)f)*Lw>p!*m zSE+|y^)k~zR=(dq9%$A3h`!OplbugJbsvvIr{)Z_+9RXJHM)vl4w`EOo z&uTgr{Tj(dq<&k2ui1yVQmQusC*q!#bDkJD?=`UENto!AxNf58-!-d~d_)7wbN#4D z*0r%jL6I$Sl*$1n(;vM>KyQ@>0&X^QgCL7%2pxjn_^fy|T$h{ISP?o>ufl8mF9uMS zn5sUbgTSqMN_-B+Ra=B3)(4Sb}u&<_ctL#!Eotw z2(+V(QT044lc40C#t=r@gAI0Zu#+J)s9VZGG{K#o4XsF)PQJkO^tWNw=hhKDdm+AD z@L--?KK)hHYvn9l;YW&w>4R|*q&D9NV-#e1DWdhIu&q?b!&g-9oO<0z|FyzWozLSe zGHKu~%|2vz?KnM31Ylm*RtPT*CTiQTjr2EIq*1EMG4#wp_4ZTV_=&c~I3daS%O}+l zdmiK7^_^$GhEio*as0Zd(Ub1$&TA4$Tv`3%Y+#qXD8QZ{oj1G)6iZ(f8l?ChCt8<3 zBuu2}z)0M}YK}8;=wky_f25wz7{M^jJEjZQ#gOZ*0S?@e0qZ8ntVBp|&QWHhD z;hh6I$<8Fm@_k8$#Y969pqO;ZQ;bzt6BB3RbOo#Er@##B$KzA3Y<|r4wJVK4;jYw> zlYpYFw1~BeE|>2UGOUwTR$%=6hidSycmg4|>qF~`FZ*SjQZRQD$a%j9of?locP}+0YHwnUSWDeBw z&|gdE8JMbKi#N{?9h5+Ub6^vffs-hoqfAsO%_c_^QpW$Keu_;C(c1S63zS-Co{1l& z1c|V6N5{_)V;NA=8-h`=aU07}V`&+VHl6@Ik-HiMx^pO+h_Y*PQ2KTcN;5BuVD-JX z_e*b0M2aBaug_O}bhj*a)F)DVR6Ru^k^1}k`ZkSa-KPg^#T%AiNJ&dC>f|Ka^`>QI zH3Oxfq=9PJbvM*lR%uz@qH^=sZ=Xun z^}5lrmV<14%E;F4LJE`TSj*Fm;cm3&FMcBQ7$7zqD+x=yHR*gmX5KC zib|$*PJg*UftOOhgjn6C^yUD*t3p($`oT1xf``guap zQ&IZUk>Melgz(hJT#NGi`vD`{KwCkku%eQ~QF-VE4>U8)z5r(=f}o7<$ea zlg?lQ#$aGylyg=ZoT{O<%i$td+>FlQT009NcsQoPPv~FW{5bh6 zI94dbJgd_?%kL5xKNt8OdgE(~D{9CdPI$Iz6ei)4$sd6&AMwg(hEBJ2RIy)845cgTV3(%sGqhfk3*&^Av} zj(q9RZa3c?Zo`iAj^@q$z^j8;b3gg{;x4jzs8?P`l!#?L+kOw`7Ef<%fpMG+H20mn zl4EW~`~5J-es}$0lYY<1p&Fmn#rFZQSbLqVeJLf+-_^VTXaZK%X7`*9H^Xnb_I*sz z=`^e;`qs~v$@@aD3vb7P^2g?z5E1D|>ZGa4qSEW~tem#K5)m-346fp_89bB@(t5~* zp1nM=_rpdV7%7T;U!I&`PT&x9A7XkVbFP!-$>m-dgEK&(VpxwIEreoE5=pvMSb9z~ zL^1%T;A9Dnb?1k+WpQAW3w=Vhj|l}tPVd?pG1fHJy{`s<;#FSu{iDVONfZjY36HU(y zX@5(95&45Dj)VWM7@`= zYgcsEG~NBS10Mib1H}L7eOc4q{RLry^*XBI&vZkgGxc+2(O=&`w=iEDVH@q>!@e?o zh+F*96f&wkP)3fR_@bApvQ=apWsD4xMhPNU^@97m($Kwy(@SF#8NITEI@785ZPNvRfTl%QB4wU{(*F!FPzEpraJw=nl{n6Axpn6Sgg3de*3lWGaQ zgc{FmM=&Njt*zCw!(JX+h+YA5z0OrjxHir+o6&UD^T~A_@!eEmePDFJR@J7#2=l860pQ`s+$>t19SRtAPU{>U08_%v z^w`l>kEVDM&`i+-`}jyvi*I>eWz47cTI{mQu`nx`uedh!82%8%OG^tOsO!(BqThSgAS9X4Ph)G7}j)ALPa0uQ6#j$!+`E z#nnh59i<{mATEPGUQKpbGe1dQi)aJPK?@>GeQ>v>jhT!cz9wr5#{hI%uGa6WPwqrU zjSuCH<~=p{oH2;^%F9#h-|SZ}&uQZzj8c~~ucxm)L)T!dnWD{uyGsU)_21peXh6DB zfpI9tp8ih@6?F=d8C@gOZUcLhdIvv$nD4VWT$#uys`viZ$;&z4Q+vJ5D^0RZSu9x- z%td<~nwZrgBnx##N%l6S*5Po#q%*>*83D|PDiRx-gLEm4tRoub~Rh_7r_=) z6cNR!F)ly6&XwHa8IG;jpRhj)56@3ppVoCX40o0SULX9M1!1bCW2oT|ybJ z17Y(%g9soh6shHul!>D~6ol@cS-2>tIQbn9>tU!@flyYjf788#T39;c7%|!(2ORfp zENpdL*`+1J`;^|0@XX*06w;yNrOlWL;yr{sjfJGbL$yEH?SUA|mFqJjJ#SAr(+EmCchi!O1qV*rA^RrYRvG_estB}oE? zEuP|sr@14@%h!lp5W|^WcGNg(2p#4O=S~M ziw`C~85QU|S23uzS!~}%HMwlq_f!1W{;m&BHDRG)sLUzz_35v+>tM9WIeoT9Z<5M6 z74!pR^V1V6c0QGfjp5pUevt%km}y3v5X;3Cd?ytWdj*F*EgF_auk>9M*sJINbYB2W z%qUK2iUvnVlL1uK%$IVhKRZ`t{C9zSd!>Qk=2!7@x!{gc7-@!K{ zORw)sC9@_jC#|)Tqmt%MvZ|wAM{Xm5TS*(ac|6gqdMCk%^;?ePaIY1jP2;+fSUOIe zWg$9^CqCLDn`@nV>~v#xA1wrjg-zUdE-A1C!b`i7#%=lkrkJ7viBPo~RikLRktq*zDsM9$mJoE{P&S9MzIGec21S(VZ`I{=fid8#+>oWcemmYfhxWvlQdk3Gj80_uH%+#fBL>T?3xhEfvZ z0cN1g2l%gK=%6B7urv|Lzbd-7NFH(b^&4kc_N>Hmpn`4!Kp$A7ZA@u0 z7;Jd=+4L)#c^y7Yh05B{D020w;cmv{fE$)v-v;r=* z2eNWd&{XN~V9}Mjk7OQ;tfhlfhwgeVByDo@i>58PR*$oYpxR#l0k}9-QDVTT%9w$n zZ&_*d;|iVBXiXpv1G)?W>{(WvTFliQ_3H%I78Tk5DJI-|(fc|aD7NT=pb=+9UaAKH zvK43oc23$2IK!4$<1a`@1ji%EW_(*7QRCmi0MwKDFZ@2%vPB8{*zt9L6lhGGzb==R zX*(7fyT9P^TiXA|!~*P{|5J7J2{kU+_-R@9g(4bWJy&16U2PlsG#R zFF?>qO3}Oe7+|nlLR*CZs4O`C&!)27;zW#rS#j-5!>YISS6sP031?qao>$a>%e?wL zvd2~>HJV||;D9L^hlSdQ1IB2&Nwq&0pu;OZu6rOV%v-G9pCUY!iUsjLwTws$OQdBu z`Y}s!-FCIZqv43h?cC9CujqKnGBo4faWz*CARapo)m8f)pvmC-p|^EKWG{tUjZB|^wr~fu8Mnmyp*)Z^uWc7Ftr%jN0LTUEl+u8 zB|co5e*%1SX2Y;<5of#Iu>L19)YW?q@pC>VxFM}_Em6IF_6A^tCQZ*WW=cch~us-Fxlbz_Qz)O;i zoaNI)^UnnH?zaDBrtRScUg3%ZQcjkpgj*`8M6vRM*Q%-oOL7uHU|YX?k^U zJXbbb%oeXdk*O+>6Q!V;KoFnQ4;Ptu0B5hsEa|<1y$a+L6@7;i$8kNLHH*XE^X*ky z-La#E@<^u;0083jNKY3zpW=YcUz^pM>0SUxXTXo!4FZQ$tsBI1xobASK)3k*b@io5 z;7?h!dx}Tqs>OQw_sZmURRCvLA%lg}+$rOnPwn(#Vt_rB`bq%9(E%yi)cK}d{k#!r z4P)(oAPqxR9N5CeypMl_vY+bodsyd5CR`5{Gb*vvSSytcyjEz!?x9^^-l)J7-Mhch z`a|;Fph(%q1#F{^JQ>b;YD)ZVpK{SXzs)5-o^Ye?$_#PQ3EU zKAyWKIf-%t};fS&>5>+`kD z34wFLv$?6TIt9O#p3?#_;Uu0z1(09K?dxo~|V*-OkR zs%#uEo*v(9`R-(1*lMb$rH~4d;BDXop4ZHg?yaezo8;8MXG#DTo697t18B9JgM0^T z7Hg3;%iS&^4I3Pk zMZ9zPCWoQm4OsJlLBm|kvUmge;#mXj>Kf0s zw{&yEChCApj60P?^v2EtX-EbRy7~fu{H}a}2C4Rl1sbfrT@V77cN?^nZ=mOv>=tsQ z>Mz#ya;_G6xFEmTV8RG6)#Dk78Qe%Da(05ITmy$FG4rz9{Ub@!*^#9EFviPCG3X`n z$s%tuc9w97jo_akX&V{JEssm~rA$@Lze6{cbWoW1`&5$D4WRqed5qa-48x z8E-y_g@HMiZfwtjRa>(}MToB_*r*^mG zDtmDvfPewg!vtVH%6v>c3~~CzIi?JgTo>j}7yad&vjU>irrJ~pthd>iVY|i7iA`c? zW`NEAV;g_n4pKcuxXOf7%{kID^HEo=SoP$j)z?-g2U>{vDh@lMHjsmOK>GtgPY_<}TO;}boGz^SIn@W6l_d);O{FF}8^lB6DX@k7e5Hke-vdQ|C2-Y&qF>RTd!F^ofKnJHznM z!cDsYV`Q@BNdV$j0TVe|>oiz%*V4=g9&R_V8^~`0IRRCXavQ1v#;so8??GgS7BE+` z;w2liyQ!?$--P0H4rQia=;5C(Za@wE-@N-@2OwL^A{7?^JPli?0lA}`~A0lWe4-3S7Z?0<|V-`-)0zCBZ` zpaxX#pWX($K+(!YL)rbHRK$NuSI0U6y2lR0jqL-S{kV-ofP$H;y?uS3o}IH85uZcd zoof{!HLOl!24?qx#{W~2QOnYQ(|^D&@{L~QireiR2{Nvk?aA`^Oq~>`o=mOBeUd&J z?_by5oBvj{K`0tQJVrW9J{}9&f8-l4&&mTSNy!BrTK@oGV*e#*?V>X0galqFhv(Mz zGn6)0Wdw+WIU|L)($kPS&q|I9l&3*bg- z8nOlOu;5g;Owv_{qXXq}kYQ*7f9%we)SQcnCdRNCKzw-~gpEyz=IH?(%hLb`hxsd=b1 zcFlB2wqp^FB_aqOY*?xn-}qDv0MlM`Niu33>r28qynDjFu1C4(HW@K^ zcQkadzp_DU!$A4k9A=%Zarw$>mLDYxI7yajl-+ZM3rhHQ$$R)0X)zhy+SVeKil`D>6JJ>wd9_Rd%3)CKVxCe0hy>|Ms!? zsv}};dA90FXB}sGsCp>hBhusp;BYEe7;Y5q_Q9!8(IzX+!W%JYBv(9gKrVB2F|Wv| z2{Zrg`!_>(n|G3VDY9<~=68pf6qcTF-TkT|_x;Ui#pbM<<5Ckdu27AqBJhI(SG!j~ zA?p&4=;{kDU)xwNA4fEo-{=9k>YIT{Lx$E`tBZ#7UsU=zl9+Sx&nG$cOtmjuz<6jY2md-jvZw0c0 zc&YVy;UTrIy2bOP{zHT3WpQ%(=V6-_73H{`Og;CpUR}K3TB>8=@NI?*;>GulAI04- zc^(2RWm3RJ- z@7a}Xbnb1kug{})d`V|4$-Ak48h-=Ng#9#>wwk~3=A@i2C8)ft;xc*i3GpO(n6FP4 zsh@=O70NdNMOK)W4vi*JVKaF-78x#{T2Y{Q3gemVI9MaKp^g3loIjW@HL+5pfl3CK zds-)d9CZF+hD_2v``!GK^3{fXfIlDd#IJC#W~@lDN8KpbH3I{zVn#TSOf~az43P^x zK1Jy$*y?3zw;vz8^Fe;;qu0=U@_Rp-1E6X_C@dsg7S}738Kw!~#hBbK4%e^mi^YAW zG+k2D{Q)}oT)88%khrRh^HfK^ukIM5nxA#Fsxg3W{gBBS+yIzHKQ$*rbabCKGUp0I z-y?O^m&?ogl4NfaBJ7(21@Z^g@%qHW~Mf)ncBq#%tOv{(=|Qr19$5kc@%LK z;ZYV2#%s=n_;;ZtjuyPlCKh;AwXho5IbpmvnYCgi7qgnFDVm0SXP3m_lybJe!u&!P z!Kv}ByW)aB$!nTj?xJ;tR9(YBSiZ46UD~EudG>DUTxXM|mWylj-^^i22|#78pp~s& zs0bpdm6{5W@}A^R)D@Zne0ID!#8Hk|o<(*e5#^uQ0Vo~-WB?(nSW*_!i%bBp42f9d zs%)B(d!Sq#!0-*R8Cv?xOKQ7q0XtLdzV{-fR#w)6l^|UyiEqt}s>r&TQZQ6HWcI19 z#mqz=!|k_jSnJWn>ps6$>oAg-K#G5k#Vwg}S_~&hyeoXBFjV-Yx6vo86q6NNipdNO z<+O;?;emp(r&MY{BUP9A8aKHWWXDc|pCuy?92tg}-=IebI->kH#Dctf~$NF)F zC{S)ai_J>*6lMxt7?*3E>`Ei~ZO&|Rc{qhY1;{#^3p&kTj(`{$Yoz-QMWO8E=fp7e zs)A>s)^2Doh>VRXlE(U;LpTB5c>lv(iwG@p_c{{*&AH(bm%eX8GD;$r!!zIf(mj;GkvQ#tkex)tu{gz|+x|5duHA771f&Lnw z+N4jV&9Vey!Tk3xsaZ?kD)egI$F;DY?Z+=%A0=W&WhJkAbur1WRFN7tx$&;}_6paL zs%&lp!@S4q)8p9D<+|$rx|{V#&fLmUR<{%V-Ez@z)qcfiol9$?P0d}anpn3^uG_$n=j$6Fj|Z+0LB9FX=Po>^RO zoA_3Noi5)$?^K+3TdQnFo69Ori^tw#V&KMwdWlcdrj!Tr^`4K~2NxfSek^G@>TK>r zBwY9u8%(MoE_2Iu-fdw@QiyccYkWB66;7%=uqyqDlKsT0*5D2DS&d7D{Lwy(xe+=N zh1TPvl(^O-qRUiSmwS7@hV;2yJYb8(Y!}{_WOT{mIS3A4ju(AG2RE&I8OTfn{CdYB z^c|zk`j-0)u#!|N0{_ef`Gj`fy_4B;{f=fxt;xS${MvV|*>B*qcA`zV0wr4>NGoUe z@rE>H19c7OeJ`0!KF*TOSQ!5}v9UT+>CrHsbgxIhe|W8Z_@h;vX}=AXcQ_!VIOHnJ#44x~$#d`pRtL@;SMBy9_ShFEV(!6{fu7 z7jW(XLr*&9Q{ETKWmW53-9_N>6Jc_QzgAQdOTv?u!$|$;87KBdwKO!YZTgrZTcbw~ z@r{dIsYiIZ1I`lQ2XJ4e^c*Klb(2^ngl=#kV@P%GiKN;IF}bz1^X_&lhl$yWHr8pR z$$IURc0$l@KQ=X3D)1Lf#DGvx#WA$R!7s~P_~emA zrj?HSaLoG1*Fgtn5x@7h9TM$}W(;kO!yGDihm!m6W(xUYpEk+6i!snw7#SmG9)=K@ z-}1WEc-E3G6E*N&h(^bYWp87@c@Abe-ff*Wreyo9ymH3J@vK*v$}i!nhBjx%#@+Gv zy}re8Jx&{E{`kDmoj6M{qGuDds;8U8@j`1r0f&8#gU*(1V`%^QKxR`=-#C>RG+&7^ ze<)`G9zbGE7wczyKB$IrSeb9k#eMi+bbWbTl4-ktO;gTf>9ks!I%Q>L>P)4kxHMT& zRxVjtu9fA6NG@Q4XuFvyQ>IQTnwk5)fD4M1C7Fpk2#A*C#v+jdDgwWo=AHLF=XcKe z&%gZW^E~%;U)OiLBJWNm@YPM5E(x(h419X1L3AK4U%A+&jJ95Ahw28RBl~8j`eX%P za*?qT|5)>w^=jaRs~~XpnsvL5vJjThgGq=B2#U`$gqmK{2E<~NW*S!vE-ZA43Um7WU(6Jc zny+R^vjm;dos;BA#i0{Y2Ct%J*cLRpZ>1l8VP=C=V+1DBG16`ef4=tW39xs z3%Pa+xpU7%+*7hvyT#UnU)SvS`&j9p71aj%$(YWK#_sE#!h?tEdxjN~I@W@iCo7oE z>2XdkPurH@`~W%AX~Bw>S18Sf_5OJ+mvwd>Mbam?EVn-mqhDy9Eh6%gSfkPng3JT2 zC2VAYZq(-}hF44f9b9zN(`Rr9L~R#()~hdl02MJF z!78{p)fi9o{b)vn^&WxDe(@De=9Ra`!MkW>#Akwk36W$U8O;;JC^ zuk*{ZuSM={94XI2mU+$e5>S0_Nc{MeE>?$!H{l$K#%w3LF2A!;w{EN>jWt;1%ZcQI zzIx=LQV#Sz*M~byTdeyOljK59$v(x1QWlG9-;wrKxXmc;j+zEjxlp6L=H?l5uZ&$u zr3skomt%y|n69i?#k|6@!Fv8nvdDltq;(XzDw?A+@*o!Fvlt_d zjJ24)AaRLfAmlUzqtD5XZI`EMEG5=8cWyNC+!W_x!H#fMW~xVjOU|w}jS(ftTb8~$ zbV!B4TM^XEoF8&aekrmElC_O>@W1YNVq|$B9zwSX$Wtp=GoR$B0BSIdb`F84digLfrdguGoPbJ=|B|F2Puot*CBF>_@9&;^$a)LJ{h}Ul} zz)JL|(pIQGI&Vzl^#8{@Y1l?Ay=w2XP~P;-K?aVRhYHAJM~+~Zw>)5@d1aCwT+D2= zVWQ&OY+tq_VzCB@`hJ!ZxN)|#Ihh)I$R7%ud=}sNJi4lBc^}`FHdeFD-`H5#o~&I|OKx;z-KpRu@QF_5Iyl ztbCZ^U@;=L73E{`d(4lYL}?T6a9upOsdC8RUwmWB`5$&PlL~8^`>0twf^QiGKA<2- zX1Tjj@Q)x(-J5t`!iUSIep#^JVROcE{jPXec|%(a$0Ei+FI4}9EV$0>fPRfwySX-$ zc-njMd14l}*=s+KdE-m($I~;xoY%&vFLRGiO*{kjcn9dLIsYG*s%j&Cg_JKV=#Nbk zzmuCPznOL^uN7J0D(Xic&iO+@gq1D=49KzCk-yEYUx0<5R+3s|5SgT|g{^w{E5XAb zvZ>m*EHB(8O8Z=2b?6a52wkA#cuf0U(_v7+LAXno;O86R*bFSjWKG<0WI`5R zfJG({ebjF~;r3kT7M2Y)w|RTob%da=g>BiEh!4MGBN^XMydn&O^!Hy9uyv2JO3yxj zX{Q~Q&4|zl*#1Nj@E$tx;Mh*TYu~=Ki)3aSE^T+I{Kn{tp~ZaGirK2Sxnj*%XwsIc zY!(HOEd02b$+%!3#ge!6}sFIL##Tfa*T{*%O}>J;u!0LwP&*k&K6@y=(Xa@17uX1p_lZda$d+`+VEF-(bQ z01CRl$(QqcBoA_tJlZ?_(h=FPbYJB4rH2B)qkU%Uc9mVYT*lq?h}$1}sRe)GH@g_k zGwi=$qrbkr_p2^6X?^rlyp?tW(wy7ltenFs18_Ey3qw7{vRcKGL&C!@-L)!-=dTW`oe5)C=b2`bKtT=dOc8N@(8+hIp}{>k(4X&rC#(UK6uWC#-f)0SO-GZlC`jOs zFOOv8xk6dzYIZW4PwH2>@lr3yr&>j%&kC6eCX_*W z{06f=IKNb96ovmyx)XyIClD3gez4V#ruW=uJ_Q7`7UuaZa4CKFllBtrcnIbD(=63myjr|+Z!ghpW9IQn$n z6xJA?xAo8U_wgTz5ozn#@T@SQg1s#46(8%hyH|tI{_Gt+*pou1Uf+Zbli)=x#$&%2 z=*KS$T0VBMdp_^mO19ei;N5%CUmimjK)))>Ip)hHr>0T^>?CG<9P|d8H0vT8o~=Tu zrXyB~bkK;U#fqmT(=g?;^C}Vnuo!eUK(s+Nx%R*1&X(YUS)qk>VYS4@4*fh8%jlE+ zdEoX5T?g&>ry4J%O>2zX6 z-__;~;|dx=e%`ux-`mCc8G&KUs9~^t0o?K5ddQ0kp_1$sc49%8#P(v-Xv|0-0e`C6 ziy~z{Mec`>cGR`K?gf1Yqjt6FyFLhJrJ7k#Un$^rfP^Mw&3~8BY*E#$C5*yjf2uwk zu1nzamp#Y)B?5wPxE~{n6+#r`RuJi?E&Up&GlKk4!aZ}xNoUrpKYdcT>;f(U4LW6x zxpP5BcxhQ@5Zkc!)S&= zQ}Ic&?X{b`!*U0WLKZ&r#nl#D9OLKVS+}auF@p z=PxNtd@!u`8fm1K<m3A2HaJxlW8V|Y~_45PN;b}H?@{bH58@A&Zk-8KJ_<<04cLeqx5j}T&uo` z<+tNhI) zsV!Nt_Ipox=|_iVe`YC?w-JC5HALP5tSscsg_77&8LifjfQJWFmrsXaeBV8+cePR< zTj8L6`nmLBz}-J3K3kvkH?FkxG{g_z*%$Tg%Utux7z$nwTTF3h$13HLZhK9<@E6pi z7e?D_7R(P)r}m?ugucIXW#Z-!33bf^c<6j!(E;agQ-N0SP9B}{+EA|s?h9p5c{`b( zC$g$i3%&haGf80L2Qmkp_FGSGzg*~xgAdd0p*r4>M*qB`nz%&ebU{EzC@+3IZqwwU z6S%{uTYtvT3?}kVXKYkXuWQt5Ui{V}@lkZd#tY3xX~`|WheIzz`t`YBW4;sG~` z(7!iS(qpTdC?3(*SQ*;W8;>0!i^~uMHT~$yScfrilB2B}A@Htvta*5!XT|~LdD1>B zzev-lgE7{h!Qc-I2_jAxmhKH`>1d#6*{f7kAOq#!g0!UZa$n0I4$aq!Gi&Zs!_U)c zuoA~?f}7Lq;L}z%YJ_Y;3Uw<;WYR8=4EBd8t0j-Hi=F1Onrmee#k(@eFx_1cxlPF` zF_vVgd+;r`#UlSmkf+USMMuucDq4Cilu9jQDeJMj5c^o^8<-*Q)PYO7VAMe^i&|Es zMIEbNpo=B67lV_dY?7DgDj;eA*1Zk;1rHUgwc#^X7 z%D28E(&Gc8V_$2WE_S4L()x-uddZC?t!F-WUIG`ifBZ1!g%m$mXtO^0fnj#^zL<$8 zdX`evEM^Xfp&N~JAN_58M*wgDpBe8AJvbW|I~N!F{)@9=Ik<+Xd*ZE@DEVEAy&5$S zbEGc{4d$i_vZsNm=Hd@KpwNCjs9Ra>LP7SUq>&pXI(BW`?C6i_79o`Y8s&q}9_ZE( zFPA$Y$RfXJAM5MI!}*9z-{kR;5I5=;M5J-d`}2Gcc%r|_Bh>D@-z5OxuF{^$ja*yA z7S?k#WRp-k?Nx?Pq}%XH6}k)uBU+?#eS&aztYRW|Sed~vz*%2unM-K-E8Kf`{rP}A zSuT$0@ws1xINRv`Tm)qTLqK)x^zs9T9Sn5cD!joE9`<4N(=9g5Hlp9pRvx%K+=-fh zR3&-nUW}Pu(;T^aJpTkrZ&R=c*X{*3p|8w)i3+0MPd=0*=S+Y=6BYE?A`r!73 z-uo!oV1_?`SqF3UIrNY2u8(k{epFHW@IIR`?o@Qa(LS{gMf)0>`y{P3>HLE611wYW zP_PnyIYIt<^t_B`!Q^+i$Sc@Aflj)7;yc4GW@@T8gvNler)1GfOwj#BZmK!qF~QXX zAFcA$l;8eyFTTAP>Ro`E4afA`LJOP-UdcNL%i}rx_v&5ltaAKoslhoTV6QQrmtyL)sR_1FJA$ujx%L7-XejupcH@c)|@Xd2LhPTgLy6|l0aY2=vw^P zHCX79yHMbHD0kiLgXY7y*6^lpAB;kG5rDW!u{$Mq&3a%Y&C05wtrftWBFI^cb6UET ziBaeVTA~hIjI#;q)S;z`sYZlAAQ|EL3r?MP{*$1$@+mQdoo?yeOGq>O*ooHcCS?Cj zp{UrqG)BVsV0cMBqaNJ!(tKQ1dxCzgg|v*Anl#j7`OevZq+wdq!rtv%y?&@ZKmChU#ESdh<6M}lzmHU+2Sjc?hXIKZy>w)?KkfvRE39BDsL$EkT?Xj(d(6E z``O4+Uk|Dcd6Sjjz{;o9o)+NFsUEUhvcv@SkMDfRlAuS_Nm=SKQNZ2DhjT+AR($>Q z_$s@Lz1p}fXD-a%*}vG`rXKFFInJh66Z2wZgLQl1kP_kNx<%0>$n%r6Gq8#yUp#V0 zVh?UiRg~eiS2B1G9K3tvRNcBB>R- z%NVn|4oeI+BT>cnr|>O6;Ds!m@;_a27P+|hLgla1r_L5WAF=P)09N_vMg;P<4xA`T z`W+bqmObUtY|yg-Lk$}N)yn;3mxn$gd+{}g6L3fOnI)UMGpPnn_Y9Qj%g6-&P$0X< z9Q{J#FkKVlwi(^>z)DtA8=q7*TTo<85O-RPY>8a5GI3S+u4Twm3#@S6_LXsfOO$3R zOJa0+sx~V0y+Oo~7j)PcXS130=sSnV4R&-?2U0_!s;>AGnsQo!V{K!%VW>`cy z5OKV{b{9O4>7_w9;6w3}eOEG)LR6y1t(ZnkskF2_u?E#NcK5W0^a)lh8zCTUWAoUA z$*mqWh&yHkGfGEhVrtK~oUkXwdfzpCE)-X(0p2iYe!7uvwp(3p@oK<+#+1b0M;d)- zG@5sII3?e}1E;{F|w=`ZV6k%-fL%(_xZZTcGt* zk;73108f;s0_C3p*)Gq<4VnE&xO7970n>Zr_vxx-hCG4jS{L)0X;PJnL5gfbwUj~X zdzcWGz_Z`Psx4>Vlj73@#GI(H!p2?e&$9C2nnlu)z`MmNz7KrWwI(1GruCoj@FkZX zbkjfOXN41s=1ZGzykxZn3oB0!Tl==O6b{6Q#!?T~?>2fWx}B;2SI<>DTM{GYvUm7m zK;uZlhlP$YxCB>19h{&OSalYWZ|5a2R1I-w znD5z&PUGgPjC2ef+jr4JtLQoGDCeSwo^gqI<1t=Wz+{!728!Y3(m)sTYfGA;JnW65 z8FJAR+Kscl6(14t(B1Oxy#w;Y&39yDI)v8P^x>;^kUB<;lyF{l{Oq`t7%`YEW)pue z?(3lr-}q7M)w@SlqV-LMoOAe@I8FVeSX!;axrr>RiP`#niHpt?=K~t91hE!)eu-yK zcxV=zxa#45yTYd*OfWYM4L}EVA zXZadCS&f}ei}DEWRj)y9KZ7UnPzK30W4_)Awefl|`oYAltN?E8L$-F!;0{hUx~O5S zjrZ^(YV`|V5Um#nD{(B{2tG65Hdt<`Uw6MVqts=>wSPJ>QHxjt+`@iE3FVu0 z6TXWnYE(K5x&r97Ra)L8l7w0ZTS?H@HckQJ%y3(k7?U2_gWQ;&T{~4vO+3ar*TTv# zi*9fG*hD3Z_+t{h&?m4W#66hes8^SoKR(lO?nU`X7;nuPiYq;rnhc)yIhzQ

f6< zB9vn8-QK#KrHw57)ur#iHy2@2ulz{I?yF(awQic7eVCzPr9#RAs;p7i2`S-=c^l%Y5OX2^450Yi=~fG=Be z$SOkL*bl6E02Anncw)^&FqC&O16vD;N$ZIl38)!C6M%yFe7NKFC+l9m?K8P+VzoAp z0x7057ZRl1i&rFW!|s~8)yryps@ZkyD@cG}01lVXS90FniFc1ezekH)DNi9-o!jvB zL3h|#q4-)@qjLfN*s>}1aqi;A_OX5qE4%e?n`CTZ5>)yVlFwa(TKU%&v{SE|?boFk z#ry_iv^boHJ(2NN_#lmjuoG>%=B?+G)@~Km=E5A@1@VFV^&sKKZK6I8G$)N@a z5>&>`klA?dlvCtvM6E~1_-PSGIJ>EdZQ+BUjJE{qu3n8QkKKO1n=otZDg`tZm7we;rR_tGU$u}07dqd? zeT$ksl!isYNpk7YGKqOJ{k`U~Xj$vJ#C^R6*P+h|>dohiCaY-qvEJQl+@5EQBJvFOjq3ADU|BR z7BPlp?j_g|FOp4Vg^`{YvKVufxcSgeNGcE!)t=?(I>bz%-TT`Ms7U1AyraBE>K6w~ z7V}EMNOY@Y2pI=&URcSBlwO4pg5#0d8ZE9bH?1@-7HRB2nm4}sZFRY?XY6vN3ds-s zWN={Hm<8Y`Mxi4=BS%ZtT`)+#dw8kVs-N+`OvA)0J-IVr{G{0x9AMIf@?jxkwZ+gc z*#;MrDt^c-6#@ZNo}{kdGR&MgF;0N-KLN2D71Vg)7svuNP9c1Kx~G9_725Ygavk3u zSkwj*6(V=#HA}Yq>pLorQ#bBaX@rah%Drs7h-2?HNRbQ!)x?3~00K<~!UJZX7a*+M zP|@=E@0b9Q$@})(>b9QP%p90I#aVu|-S^X*A)r@ZnyUV^_c^-$(>`^k7NIN`kS%aK zcYj<(G_Cv@vMezpJohyF>l57o>#NNZZ&+t^{2D0g1cDFlz~pc-w9v0129#@{m>#-w zKmGfh3zCXr{MM<~V}puy?(k+OFOLd#9e5#@vtW|tLr0;c-TX-vt9Pxkjs~UweZT3jly+rj7FOyHtV%4#jn9&3S#x0}nE9|g?`Q@?gZPW%^4Glkd zy1zIvP7sV3QGj9Vea2&;B5Tl!cUON}omp}iuDvph*+?C1Tu1~05DNd{W{x1$O-LNW zxVgjpV`Ni>wCO;JZ%)&;S^ZnifofqjmbCkS3|zh>9ZcsDkX6c^ zTLeka^>PUSYVNEw8LiybtNv%qbGt(etvkw4Q((55c8*-Ew_4G*YA+XeaBx_sB{%R* zlB2Af2bZYNxbu~LyYij8wRBs2vPP~bn-(6~gjRMWMP!Co*e%&a{I0z2nChPv@GJX_ z+y;jsh&o-m=LS0(!0u9S)ShvR#F;_G?|Zj;!N8W=r+c?A%R5j{{jyhJb@moe4bGI* ze14#dw`3EO8b5H$b%Aas3&t*Rl$bh$@biZLu;(vd1+?aLqUfV>{OUkofrLI?W2R@>aU#iR74W$N z|HnNmg82Qu#GO5u>nO)fu+@D9?v$SUiUzH}*~RSfLK~*G(?w2o9&v(}Q5N#i3^h4r zScV-LK6rhPN-q<%!@1Yzui83%m4Ts-O49}B1JTO2&Ax*8b6qTjs~X3}Fc18)2_S6R zg1703)sd7|XLvsg2D(At$Y#m&vJtTPt}DMq(r*F%=OFyM%mgJ_=wx-Sxl|cAZRzi% z$0xSLG2_as&+P#5%<^7!1%>);D*mZ&E25#`W_|1}Z7a;|&1 z{kzXe!*h=VR&I#tPCOf`Jf>J|@{^?z{P_%fHY)R%z$3l=SgN1d?AzQYF`+RX_c;1$ zC%@IoM>~uq`WZ~P$H@i9-8XOD2i0l@&*XMLXLbJdb2@t^8ea z5v-9iYhOFPmhht3m+s02QD1nvphSlWH+sFc|JU!5zGR>|8W;*rtT3%>jvNYtP;gbP zUAAxmM#^m1XVvldt2Nxb%d#=Uk=0K3P8{hX@ET!)!660>*(6-Pcmf;`keg9dQqTct zlUe}uRT}vv;q;FZKffC9$8Gytz5+x%y!Y+75+-#FC0JzfxfkgwK9GOi)t(`E`G5Yi zdy`NS<~f!XKg~O2S_eCNEcXa`f4U@ZwSYK#7KePS<=H<8BONnHWDExiqmJhIDx-S! z_saV-Cp|BYgm|RaAP+4$yAHe!beClCGidr=`QqAgk4T1M63ob=`}nfw22Z~`xT04s zcNiUFc*9uDEN=FVk6a*7mrL1jASQN4j2;A+z&=pTALr*v78^IC#yRpJ~{uI=gdlRLcuQ}w}`X9hs{O)Yey8t?+ zKzLR1NiEj2PDM?l$hY(|^DNIc^s^06h&8N2J%1UwqqzZ*$gaFp?PUe8wP_p;WJP{^ zE~(07y+S{lDMZZR#ZC<7twWz))0>h;@-1sJA=rYg zbhd=|NLhwl62ERE(feRUz8ORFV?cF9a6p|OqPBNxatq|mdy7rmO(cNsgvObb-!O>r zNz@CCyvj_VRo0m3UH)_9ck$x4qV3LWn*o^=#5lu3wjxCh=!muhFsmD{+;fSY-+WYt zKL=z`WVV24rbc5SXApHA`8a|~TA}8o&&_7V5i4f!ptSz$-vl)0B!8N(Qw9h%*>t~) za~m^HGjewPU3kD2)!jR-5G+l8oxPLYnm7m2bQ3$N!uwh? z+e@x~))B0h0GVn@bBCrbbcAM9hxjcfX(M}0+0Ad5q#0Zeh}=o~`~QAH`yp?ybXyK~ z^ZP^XF0l)hD)qgW9=^_kypM5y+Q(maII86TDno@kAuO~2mliPlev0g<2s|x;CU^)^ zI5yT#Ruxmn$7>`lKEl|SSCsSLnL<0*)gko)L2=mz)V?3iKCr^}2j+Ht;ZD-K($5!- zs&XPKV#Fn(aZdnB^D(I3z8HX(SzPCpDY=2%d~TqfZ(@lX<9nM4NZA}f$P(|Ujk=sz z`9uXf#;s@=^45TSrakt$oQ_mu$cUw$d-)auc~z%T9ybToUFG8M8}KY_pbB}} zIUr`YSH=Nz^ZXFEsz{+XtMan_Ij$-QU=Q?Kp+WtN#v>ivEz==^L{5S?K|pt^Edwr$ zs)q1V-k0_p-qCt-wAF?1>_;@$f{0Y}9zqL{^w;Owz%8xH8zpCtf!|K>BKW7moPeR+ zCuOtO&Cq~a$){r^@c>4E>1!Kot*{Fu;RNv$*}T{D`hC^2^cEW!ShJ6yaE_@n;}auv9OKcLE(R-RsNy3gOL zYw!uag7q@0jybU3keR^qXMpl7ng=n6sp$7?QY0m(S_SHTx05x-u5z1XExDOQ|_sYdJR?>lIu?m-~kF!0r zuz_PD)NeN@`Hg4~{}D^N{+@SsPBw5KPoH_8?VA1Kr3RWvNj;Uiu&_|SjDnPa)HH~H zXZkgtxi?y))a}UI_fw7XiFktEnOH{j+}F)&VSr?A!vghgzU;MGjmVCwJmK_ah$ zu&{iltm^Xvvl3SmkdnVA!nq*Ad-Jto^cJg#U7omoXLSniAYR{I9$MfJo(RZOtJpH2 zVP(EB4S1h&?bXkH*QL9a2dx{}0Vi>X&Q1c?)EUb`?3TC^*LLO4t3div?K`d>rr+>#uxPhdGI^d}X4%0lZfvn}i zrPsLJo}u+@0}pq&PiqV&W*~wTws1qoEc{8Qy9HfC*KSLfdYS3Dw#Nq0(HUD+8iT!{ z1L%eNvJK?i1t);Q!+BZxGuu^h0-A&qsyG1<&bE}P#qI+!i`r_xf~*Tb-Lza(K(-F# zGYf#Ri9*c8G*RjKlOwA!7L4W!o8UC^!4Dzc*VCFCU<3iXj_!!JPH6-=0|s^*`insN zMJ}iKx)q!hgoR`35C4y{Mt}E5INgQSQ~%c6@KBCOpA~u{HW}Cb{f@6rOs7EFJ1A( z!cjaJ=T__W)6-1X48HJus`LgxY3?s=w!L)qMau`8kBD1wbKqhTRp+}G?>cuZ&4;Lx ztcI%LW$K?D`h9kk=y6v$cmV%a(snGRJ3zZ7bGmSmDnUUS{0LKPF0-Is*Thb?A!Wdb z=~f%;c3{M~1UXoO6K&G-U&*5#^6myaVN!VBPl;VKr$#<4}Yy)Qt#U zzXb$Dg7`gOp5lca$fqS_{jl}>H{;1Bz|vG|HKZzhZ{0zG<+gK4yyK4Sojta@eza+b zZPCSBuuzZmVJyQ(H3l=|?wAPO2%h_17a~#c=H*^^wJONtOWHBZmu&r_SAPI1F69jC zTtV2Ro^!c7lp+pjb>ThuFnIlX4V(PcmK4DaAR(rtaA+&_Mnf3$!*bu8sJyfD?vbL= zch9J6iqMm`zofe*F^SXAM#*77D}WdnO=n=oQc-*m@=Ci6Kjm3Yc-d4B0heW8DW&cS zris|s$M!97*Aa}tD;R$<=&W=5cMft+W>?u8YIq14-n?lCx3tE_#)D@;rW{+JPykXH zr_1($oj|>x|3Meb)Nb{@&nu{7N$N@o3|V=NnuW52O;+T}V?It_Q2tB`Z#7&;Q6~4) zIvts@rL_z}N2i>GbDAE=h=&AJ_N1w0T2{DUYzz0}{Nob{{6Mg#?j|zM;pXSoh5Q=H z^t`bob+;UMVHQtY-{M--y`&+?TfYRifo%I}^pLK6rFyk94>-&MLx`Mke-<^~7cK?( zebKd({9vFrZ{+Se`&;TpWKfab%HMilfH*o3K;g0c*T5dz`sb$V#ZVg71*DsN5tZG& ztklT+QP8shcVT5EaC^_!wMs@XX@X726kYV9j8@L8dPFR4BNV_|&JX9joY@O<{>)44 z(C^1XqTU_rB2&v_0-toUiQVZ|AN1!{2q>WVRZ$pEJ#DIoIY`97eERYMgO29oz`Fr6 z?GO#vlxc(X3p7p(yp5#~NEA2b&SDI^$_qt*5+~qwd4)upG>)M3M%Z?BVK?H|E_wC! z-x~{ql7=)~f5>}$NZ3$=s=RjRG|Og>ZGK&u@&UU+u`Y_zXl=JSvT}qt-75K-s`N*g zTf^Y&M7al-Q?xKm?%Dd-?w6boRA!@)j zw5nSyYWO5xs-?y~RSxD-ZG2Yyp()U3qCVY{1g^9a8jsHu?oe}U< z9#Mt#8kjf-A%yDA3$a2Vrvk75GpWbzdvo0fIK$!cQA)Y?ttWbH!nKan^q-y{&I#MW zheW*!!N=mTun*@@GjB-nna6^j*jOXTL@ib~4Fo~{t)BT$fJQ2(L z`(~a;HkNP}MFO8Ho70G69uR*)j}&j`)vGKd6`huJmDFG_s~x{{tlcK;c}L9A+?0F? z?mVDhTfTU(5`Lr<4~S1qIcXa$2B&k3gAtwJr1 zMTx-jrL(!N8Gp}L#oD`~bomo#bZMYKn-wlI4y}y1`eBBKT)dzh0uG&p$;s(TmA?_U zR3@2mW2gg4q@5Un-S}UUHnM?(zuREuyui!9t_h9%XFh=p-YAR>iBjdSt&06e!SQ_j zIO^e>uVaTg(3mSMI-iI{uXht$eMsg$_SvKCG{6nM;FL+S{mdzNshuT>VQ%g(JM+_@ z%bPoRB|0nt)K9WJ)6T4bp=~9zI<-)atI%MIt7f}fY3z)$4lEZIO)FFx<@%49Bn-F??Gw5^Q|Yj24sz_sz&fxkV(Q?fEkQJIXkvG?}H47 zz45*PX)XWj_u+D%>m=i$`uA3(KAb!KzeKp51$ZF)x8_S>)O0rugwUBLp^#;j|^KcT&f9@&dYO zjQUB!JrMkb9pGx_Yd%fqps(@{3W8t+Uqn-gn#$+xw8@u}-x9z+dd%;xbKF@@QS!6H z30c!7Fj%K#&^V>)$~zqwOUB_%x-4*;%euirA3xqp-ef?XWIw~orb`k*QJpC@iLlN% zqh`5-KL%ZDtYO@nZE#rDJMhdHJo3p($u@jD5HX_AQ`=Y=&UO~2m4HHvN>C^J%6G?!nyzX$v4p3+!{(AmPVMqeaKSRE(eUR=j7 zI@jfa;p;gqZ+n}g>4HZvy@06}0Wml4Mgsjs7!a@6Lg1vJTkIpx%C8RBO36CV*~D zitYR4qkkr+H-S`6+;78BY}*`^T+=+k4{(jsg-zGu)o?8G{Wg$oR??qb#x_S%56j-r zTVOomG;y6V`}NpEh8xb8sGOXN5C=?G8$fm79hq*y^K-o5cXxTgvLQGA@z&iCTJHw$ zf}5)gSa8{}0qUjb)b*I%xT7?EL={_1DccI)?4ncn$WlFxF>Fx4zwyD>KzbBjX}ZS% zbgl$V(hJ}4$*ZvIAUkpk3vp=@+_#zxBNcAG#c(>3$fr*aJ z1$pRtmMPP74k@`<%j>nf7c8+}5uJR?=-#A_0m;5SgwO-U6Im%d_y9bT@vw@{Ox4Qw zkds?`kbRr$?Af<+U2M*HV;)n|A;D}Sn;1N>6B1o)jI_kkhd=iViX{iHpAv+=_+lAm z9W-NK;0_-iPccW+yd?PD7K6z~zX*2yz?9^%lRsj*Bclt*Pe}8+|VAqd*%@3W;ro? zE!3=oe=F%?HZGHm3hqt2rIXP!jvi#c@S&ZJhTGG2L*bDg5t_%@zIQ9?2?cS-;rkiR@wBqk-8GYt~yayS>4a3AwO4mycKmJ zYD@HdHYjy~jLkRO3z7T6>e_B!&$Dgxoh@fBW^7Mn2F&)s4Cr?!(mu~kn77IHO*gE5 ztxC!}h5WhGh#uM%Qt9H9KcYq8T*Ebmj5H!bRUg-&=ehxVIG8cZgXmQ%ApIqbE^5^K zVa6A!I6{|As--cnak6TJo{~zhQ1$^GpanG-y({Y>)S26M-R7lEUBi(>M#!;#_(Zk2F7qbF<9M8||XSIc|K8g`M8~U99L<8ayImZ+RlqL0oSI6lgp* zZaql5`MqmWuku%NHt1f*<}rteur78V2#G^}-0y<1#tW=Vj@d*c#mr>U`5z`1H9Jiy z^?G1nwuUZDFWsDk_%hG&;OjZRjeqQB*T8JMA^vOb&G=kQ)D)*fkUgDwMLBUlPT<

Z7fOy+?QX+}%UAp9^!#_JyKyh&yX>6Y!nRP-@ z7I12iYPD99dGtOBRjl$w8P2?i6mA|8hfe(BtxsjW;Biv!B+Fnc4DQjA^f1>ICM4yU z7mD)nw0V?dBPmuvos@pSeobMc=CP(M3OOo+{dF-2^Q(zPQ}FKy95t*_s7q{YT z%%K9EET7&&FJV0MOVVZ?SN#mNjeH_6+vgfb-zSxP`W&v%c$A|JK5veb_;j%6`@={fd;@b4CX!_wNv&mhd%_}N@>TFK-~RG- zS)N%%r_XkI4(z>Ad4jCi3DsNXp}_Y=v_J2KDZ$3ZB&!c z2eWc$w5bE5+y%lV-=I7?Jtf_-$9{tTm(Y~xkzamNI#N=vCzsCcmEJtg2ESA1FSjHt zC9>pnR*GV-cy&au9R2p%QCSi1u+lMIk}v0BotTu6W)}Htw>qlhBU0i#J94Qroad<= za>$tL*Kx_X+7L23G9vCV`>WhbnC{_U<3^YMk8i1)hW(Jm6rCnv<6*tbULQCdkG-w0v% z2kUzIxAIbRXz+umH%tiT_Dn~W_*5L;iv6@v6=PEsG0&rI68JNH4UoYZ=^l4xK2+tF zP#lj@^2?fy4gx$PuJ5!lNZe%7hBpt4)gtqQ-S8PCl#qbbF?V;17zJ}PWxp;SRV<1a z0#VOnQ6|G_sl@#~qSc+IUE?%!h%R256ZHoNjwj@Mk%GIhQU(nMGEFpX0}3>Cs|p?D zGe)lB5t8sGe5h$hkPhB(ib)hQA$ndIHmGBTl72Hzwt?4Uw3G5#xe3rJ-AhDIp-E`a z^Iejifh8tgtCkbwWb{=_vOkCOg0n8m$J1-3vxmIbOM(Uo?T=SUZ-UQ76U^QIBdR>l zuZ!QTbyd0P30HZ~CiQ;t!UT-~%#HsbiC1g}6Ldem53W6o%ZQC_hh?u0VX7z~o0-cq zX%!_>t3XQQjB+qOS@gRGA#R*PpNqeP>Utt%)%k1zid4ySfi<}cO-)Bac>(094eXz(JtQ;((0Oo(NjAF}xy(!!@j(SIp-|dULz&3Lmd>uDkC$S-o zP66{HO}k~6O@}(Givf0Rl-60u6PihlR%Lm4mVN3hf~gVMSi#H=#${TE=Np?FGvmfV zM?2L{J3;U1xeSLhBzYu~QTa)beb^G44U%+$oiXD33X4q8dA>lkABI9Pv83}cRB5JF zASJ&!z@wt^qr{QSruGXaPiD9}`+$x!guRw<+6k<938fD+R+$b6`TU=Tmua({SP43Q zz?MZ}z$@6kEN#>g1op`Nd+J`YFZ?B(#I6wuIWC$x zNxYm{B2vF-;iu4BR zK}TP+0`&72S?93fW3=3o0u`-C3%PJT4vSdWgbC?90HK8e2{GHBRfhnNT4lS|x;XuZ zeWKp)0`>ozb$IhpW=(@Eq?Sb=piQb5Scw40{1Oh0#V%cu9By8aUx9OaONX{@F2_A? zyZ-ajMzi!Ql9R;nHwRUNGYUQ6z!UIg9RX^9Z@dTzC1_s!An%_evYVOFS$Cv6uhK{`Rk2wu1uX3WA;uPOP#0ZOkw4G?`83 z)PphBhj+`7$v}!9^?T;Byf6NU{}vH>^+hVw=b-G^<)L98-Xd`BM0*)Q8|nPoS+Ra0kZY zexkM@@@$UaZ7AxK0#jNRf{A#RLZ=bd?n0s>n9peYRkncUO4NNXv+%=1j56pD-gw5v z6M#VXB&uleFU`Av$ZlSB>VW7guLye5zoc_Skq*)k@SxDcfxr}{0^ z2sJLZdg(0+alNahaWS--Gdsl4gSpqWff$73fyB+%DB~ZAS=)pnXG0$v6IRYR! z8{mJJPF~yfpj~^{)Li?qYf0 z4Q7(A*0F6tV5g6JpdtQEWwxv#y9OkWzir_SF5t63P|X7fGI`gfc~yqZ!6Apy+nL~c zfC^RNqn7A``PIfsf`^qSFb)C{BGuG94IxR`548lz!f5*CYXRt4aNKYs~g z*bK35{%LS#%K%8mI=+~dHbHl+ONOV2A(Qp03_za<_BBhgyBo-z4L>4&ukFzAtD(B0 zd6zLbmf)EFKeoOyAj&&3~M;8tyyS`eaNaX23eeo|uY#TmWTZfpN zZ$6!fjv1SJno!l0ZWRr>j<~0wF{hD?>5hK3VtDDXee)Wz#ou|OYY(~M5Rj}WXKyE; z?ZE9STY+y18+UwCw%9J!D{TF(XV$4scEZJHreTJ6U0; zgy2pS({cm$zA`u{`W@dgfS>Na9d%g=au9deW>HXc*j*R!<$o=9(51LZ!DstuI~9yh zk@R`CaaURkx9mfwI%ud}D{Kl29xhTbUPLt;X+TsU9fepXD{WFxw+eVEPyJg1iKamB zhI(g6g)IDINU$h&!?dUwBx`;nRQjC1#Xf$0JbOrK)oHr{Kdhf`X_N?zHG3o2 zke|=Ct!{tt3IkOjFq;x^+Ab|KA8T{K0*dWyzYr?`)0U~`@^OnmY->LD>pH+ccp2d(}dtLZ<>x^sfhsjJd60(%F zHqA=K6DT)NQf?3^m>Ranx<5S}BYQpH;p)ETs%||mx4C9oG!!SfWj&<|`q&iP4Ip)A zJ+)T8$k%IA(tHfbzcXc%$hPf0Nb{Ym`dLEHh=^*|Mt{7Jdo|*s0SP5j%Ixm9Z5=K@ z*`6`?yF*#tmRXxp7EARQQF*O-)j$gKqLLBky#Yafm{!!LBlOJP}97 z!^-llG(}uGU0H0Cv+S2m?Bk&2m{a|tYPOC;{2FnUX+&l*_Y?i~!L*o3uFbuOUp+v- zrLk??RjWz2B(&7B{HhFF)6=nPcZ3&1aD`32C2v}{>o`p=^nBY24^d98*Ob=}l^9yn zmaY-ro&*Dv&U^&PN7jD~-27H;7U{lO#SH&yIySivXWn|nRj~JJ4=VA3d@T3m8ayy0 zFa(u+8S&Kw^9-tD`tbi5K18`Lh?;mbB4SAZilzBr2$>W4*qAP~r`&2TzFJ2GHBcZ6 z8ux4b&@*0XNa<`k4yk2hs2VtUY zFsP0IVRC%Y187@1n}Ffm(cTV#pIBvMxv4lH#HuzB$je$UbtpBy2%I&m70b%1nwS*e z&vi!%3G9BaQ+>L@1d`XBqb|9ZC)x}Q19}?9E@YvjDjBaL+gCCvNK`&A7eUT*w$19q zUV6gw?(jRj=NH=l0vku32_rKM$+`8YzFdViXm2+lJ)l!lv-h1Y`CnnUBWo6XNw*+7 zF0eY(b?&(<_*}VD!^O<;{72klc83qB3ru?q^ps!}W+*5o->em5Bto}jqHmcnIS&A( z?{}BPQ_%Lj(PEddb;a zdZ0LcKTk_~M_~|EkrFd%MBTL5;m;p&^;7yqB!|oY9C=v%)Hn|{31ZVx)7UA4b+T$2 zBxHN@V<3!IRWx5!@BBmEWcN9r0y+0Tf@749$rqRNMCQdzRvte7hV__9U@-m4_~DE} zuduE8$N*6`|9P@}@ZB>lbKVlp*_+>=g)N^eBp#lY10keU4Y|bxp6n)RPlik1sQcLV z%pq*AbjfucULzD@>weo@lV1}h3$@&u26xHN&ql{%;q0zC_zZDEE=?h9I&Yxwss>C1lSFD@Z`avQ4{$u3E@jQ*l^Iv zR4K6()UHHuQ0?|KXNn+>tH~GeeIS8vJ>tw;7*;g53OuHDw=>)xzCTGs9tEWH>w+Dk zYr3x(%kHCE6=uqUri!VGdS-L8Yo`{Lh;$&ng6ac z1_|xnVTe`hj_yl}65CqK8Na-Ghp@N}USdB<5_g+bU-|L$FuTQ*mg)L@*A(_x5}zXY zusH&H?Xk6eyk%fM4Ee&{y|U0`Gt0-urRLjgVS9x;ZN2ItgCJ7T6)S-Yf+#-^Du7Ak z!xn=0bwMG%A7(Q;CLb9B)hQrQM_h5`ABWN>%gytPAr-S0vN>uRR%&X< zY1A+$`r&{aNkTL4&$Cd63wXs;@Y7in)XnzZJ!OG4MXilTuQH5j;d?z+9z&?kVvyAV zv5muUCTWq#a;@8t&>d7~ip9lsz)p=H|B)7*d5Pk}AN(jwtBX=lm%z2giVP5(7{Z`= z^m^&oNp7k6>Wtaq>a3x(V;zP3=Ehd18}5(EEY@FxjYLn9rNBF7jQ&B}AREF(Ue-S6 z)|(&8F-zf$u$i*kZ(^9h`(@jxTOz*=oI+VOM&1nlqU4V>cdocU(sVNPV3G_y!q(7f zMz#z7F81)Ogqe$o!FQX;zFCd;?O$To=aAFrk$-%XwvAc}t(jM{ZmxRF8fF?ZV6o#` zXFh)P6|Tei(a@-?`;o?LjdGTWHyv+^)r*NGT#<-Tdo9kCFI+>c*bNB@J6|%gYtpMT zq4gCxosWBW=byNHS*I*0IheB?;*#{~*t%R}yGNmb@a`J0;d^n6ey`vN2kWJFHe2#* z_K@^G70YREa`~QF;qnc&hlj7mRi8*bDH8lXRv+N_nLQL+NrrDD7gLHm=E0BIN{WHS zHfo2~C$G{>Tpq8>5Qt>aZpT)Bdur7Ndf%*+K8j-wywBR?k597;uJ2}5JIH=qw%eT- zX=!a;r|JoMz$wA{c9n@i(Vyd34GSB4$Y7e&V*Ih&#nhamt#@GJaHt$%I2dn+Wj$g= zPaquq{%~1ZF$S+s?qhduF8EQWS^mPaVMUp7VI1sBwTIhg)E2?VcIxre41CS?%w9^I zxoY^~u}N0j5|~m;nzDQ$E%7a>A2}ayp8lcz!~glmSv{Si$&+!m=L!U9B9CD??-tFB z@I+qXdB5`$v_HBnWo|cItQEpZTg&lDQj1VBC6z}jElPplJvRq?wspHfq$EyTc}qnR z#FRxCb78{SwLa@^If~?WC$pC5`I1#*Yia1#S>XUdvki&;)sw!tIyYUuL+x`g-bY z#@T94GTO#MsA-Tbxv__W(NbSR!OYA;RNR7P@IL#i-u3X{O~o(YQrf?Yp`}Gu+R@>X z^S^={41c{bFgHheG7BxRbsQ8yHd9{V2?+@?g)9@2kp+RtDGyM=OQA80=Vs+&%SkS` zvo1Z=Z?&Y4aamxXD_C}naAK4k3DA_9Py&ZZQfY&xq?{$rc`gCuDF7&Z#`emnTf0d z^-;Mi?V}!E9^AzfdF4Np-|BSnn3nAn>G&r;!t{Dgp*GVt(*4jY^daWV9hbF@2ilCV z=AT{iCJu&53e_zWFL2*DbYuIo3MCD87f7j=u&&8uL_hz)m?EguQ;*qqA|~R1#X}!b zD3qC8V6U76rMOSQKmA4_`4mgShX>zei?pU3fklZg{_e zb`&StmvX7emmi_mTymK(Rsl9k=tL1XWJe+iQo0y7AC$zDv-eQ1$FJpH8*? zOW~#%J>>ZqY6QfQyd&6)TV16k=(J55-uL!wUc{Z;3Kfd>@5)i5-hRVN!ZIuib!Qi1 z$G4MXYpQgoTu|7uc{N;ap9~z50Qi%B-KWH*+wOL%2Lq9bhM#p~1KxfRBCa$4V>E8c z>+;iz`?3E>eY+zSYU8|Iv>iy*I6q=ac|EijSW3#du+2*%xhRNlpT|DfeRC?QPQjfh z?(Xu&zEqhd`JwnKpHEt0&eX6|Y`6BCIeEs=i>gzay31sO&%DYs4g4CnWMrDVyJ0>Y z@~_l(f=vNGOE0NEOrdv)siye}qws?RIre5y_GmuYGFplf5P)hnlZtexDt@SShvEq@ zK*~BiT}pZKdAO4fC(G{GxT2OT&KyjvQR!;8apA`j5k{=bYu&TOY)xHF@YF@3)dJt# zAd_Usc0+9HIWaTrInAmyoU3HGhpyet;Fv+F*rdWqK#Us6u9r_a&>g|7W(^L;oU}7C zlq<3f<>Q6lHY-@6?lR`4mbfD}d|8#*ThvGQRLtBRRzm6(+P8pkSKe(*ioukv>E8&| zpk^q04lGW+0ELtsHJoU)O|-k^dea{VJP02n4{aY2@$BRnqFj+nJvS`B<;X<~4=t(0 zX4qmn<^c*VlcuO}b!Gz1v{-xZw!MB;in}cYAwwJsd3n@qx6=n~E|4DNjy2suPsQ02 zQ&0-?3EmTn5W5TuL^h>+<8NI2QLJX3fDDKj=u2!<4e8I(8s<2lrhI0-&Cg9};6D9r zjC^Bu+^Y*VUx~!jjzK=@6*{@cEzM{$-&4uEqC0#q#wilhJ?fl1m z;?kLQ_B@_F!~*;KhIfYSt%SSbq6P1i*6m_yS_U6#=Sg|zWnDM6dzvL=NY>?1$Z8|g zjw$s8Hm}`;_s|i~dZnZ%S!o>iy1ND>Y4w?3WJ(D~L+!P1y;kkfGlU$Qr=l%ObSCh0 z%hiw!FC|%3x+m^xG#;0{@Eww2L!%82)&u>U_~DwO##|>)&O!5);h=mC4BHxjl=X&La@a!yJT5HdO#`ZLAay$QlF&&x&1(^htm>w0>66%O1E8?UBa zR^vKtw)`=rSoIN(X^GECN;hB3mR?_#yd&0mS;YVg76}U|wi!E-%I+-Wd_y@qf%sr6 zqv;Jxq{ad{Ni)kJq28)KqfmkeU1vDKQ2IBrJ6p%(XW%0}CU z|J~vdOYNT9)C{-0`r)7b^7AvwizqmjW1>R~$s71N?<321ZXHe`5{+<^=HmVK5+rq- zmIu`>f;^i$s^9?w8ubUA&<$v4r&mN~DXk4xF;j9WCAG?Uodj1!KO{({JTxKg6+&{?WOB{h`6ryPd@ZGfc=c1@IPhr8vb7dP zL~-oId?ZF391%Na;QG!j2I7Bp-yMtl@uKJZdCOScHl-u1|a%(Uso8zh^}@Rpb>qD!lJNZ#(Nc-@Uq(Okm8?dN}4WAdOsmSxIP{y^(DrI7>;0 zE7Sg{M~?PpvJ$p=@G)8%9sfMLIJLuUyII4dUWIIS)&T>!f8vsiWrdrnj z9-~#gcf&C^AJaY6?fJA)F9n)AC07>Tr4?^Yzo5`lV!Q>(@!W8!6tcnw;LDR=9h?&* zf<`NXS53a(KXSw(3$Y;ja!FAb_yS5f0#d#WtZ?6xwj=9lT5aIH|{vqEpz{8LLJ$(q_5-Sp6&0+*IEsx)8?un*+zkn zB5r1J{TK5Xmdzvg;YCmX&{ef`&LSW)h#P!eAKuHGrbF0l)0MWdAfjuxO6iuy zNh~YF34`V3SQ~4g^f|O~KSO$#nTOKnJ9F0=?%WbhA*HVd?881aaVf84 zuRlqw67(Ncc=akZvikxYllhT7cHUz@AmNVN(^-C{-d!7r4Yh`*)R$r5B}IAXZ(X6s zTa8wp7g}I)vlOm_1b88PA`wPTW^9fDMTWrYAizsMKi+d+vY1PbY&p2AD`um}u~E7o z+#S*s8y2{?(5N9bou7Te;t7D)8=;4mX7mes#pYs1+?<&i9vDU~zjhzzsjb+d$-S-V z=g%Bfy91rvyEh3UY2EkTN9IT0<0W5*z`}0*-<`}4kxM#Y%q#f*{-DHx<0Vn(?SmT1 zgV2+#MHG`4g_*~d8MmVLrY;-p1N^2D#CRftBy|-ndqH?OHcSe0I zTgU4t-WW8WrOo0nl00tmZmg~+!lA5hKp1%@&!SQMT&F-ly&yL2`Ik)Ac3xw;I>hl*rSo1g z#UNlSnoUpOcf1HLJS;3?CmT|Ekl6%b1oX1cCW~p0X?F#)IN=0O3XB%iw;0`=P3{W` zRbk?5M2#qc7{W_QdK@XPVCH+o1mN_r#~~u|dA>^2qRBKPD&U~pN(ks5#?CY^WRIGp za1%-6Wx(s;%%CytG`t&!Girb)f`?{h^BbC4JJ@6Ch`klVxwR3oa6*?cLSVW^Er;EKlM5}Jn5h~8zeakq!-y= zJC>q#p>2y}?F~Mf-%1Dz@|Us+LKzrr)@w?82#<(GOya6HZnAis`F~9^^nXd?S$luy z%8{^`5Z))M?=mHOf-4_Hoh!d3joDe$P!{6~7aVP^3t3kDSmwNQ!eciktnlrIw?~eP za9X(2A!bs|V!&r*RXZGvWLUQBUe%n)iwzepE}pfbZS=J4;a8~z+8?}ME`BMow5#C+ zZM!6){(vZ!$Z4-Yo<1eg(9p=1;ZyE-zua8CG8Y?5<`861D-i8hp6|0ft6LWUg1l{N zhPOsW)O2(!t^)J6PCgm(RHhq}FGh{&CMRvwiNcBWk796$YgWZ_F?C_3)FJe%nLaaP0o4V-a2iriI+|Z&u#A6q&*pDLc^`T z*Z9#&Ys>D(;*i0~UaVKwy66bfvCmGS*BLm&cb(sL!8yfKn zz25clj`k-8)yLkr%&478(atM9Bc)>aLYL>(nZlk^H^jiQx;^1@ZvNIam$L&sE85eQ z&)167kDTZvxpWd?1E>1iH&93j)_%l`K)+5Rt?jjVzh-S6lp35`Y9*;NieV7}GIJv@ zh>>hh#&ShvL)@1!dqeL6&*)yro~r!%7n5ArSi4*8J4O?gwG3x?G~>9-=q3lxvZthL zvmAu=dpL8e8b~)WSW^;>xt5sO`xgX4#R6Uoh)YJ4#Bx3!ZK*YpQuxVti}t!}Z=7i+ zgUC`%LrI^~22#WDd?IR;(0g?JSP|3r(mHrj*mQb_$Dq4zb7WOan);At3-PgQV8;JS zBqj4X)sJ!Gj{32|8|1UX{fUq%oyvALhPXi%_fHg5_Qq0Oowc?+>37rhlaNwD<7X6+ z7CgZE#B*spvjFM%L(=yzSXaw4F3}IVLV#Bf^#H% zam<*TVAhi;CP&pIq2!PlUOr%`EP5mw6tXEO~7+@ za&Jj-kD}AEQj*g#n8HV}Q2)61nIRng6H{l*E$&ZZ>c==nCmJTntbN-A@VJ?tTKS6g zL&e45_1e1C8{LBSc#3>QCS3V2cV?dZSqH1s`c2RuQ#GVG)e1e11G5$y0Ns)tevj?6 z$>T&Na%B)|m_yE5k-4Nl8f?uDQ>QOYbFxj6JjI&UO}+6mDf@?pOR#tIjxfcxd19ynQX9 zyU|TOPp`?mXrm=BFRP5@et3kYlbPK_^J!(J>Qq2pEtwC-$>Jv-p;V72q|5k3in=P(xDw(}}oIBEX4U128!`Hpx$4}z#Rt3Z7y1_oS%T-j9#IIHC zmrivpIfD5u1%K8xVRideoRFw(l+{a8wM+G-JZ^+c?PFsd;_c`QF=*=&@yRqj$tz%LOgkcHPBk`sNX0Rr)WC60abYEvg41ZKA8mU`DAV8OzZh_xft_8%)EZC8 zNJea}EXf8TV#@8e|F-0r85J#U!7&8FSZ2jADb1KRRgjhTg5(izd*#weXN38i1lK7u7j>oP z-pxv{XTyM-(pN^kY^+kaYZmFT|ET!J

g6w6EOKxG!0uB-Bp-$1SY)bLBq76=uA; zS4%EI9*ZItWEa%OY**JZ#Biw}ag#Nw^-}S65MnnM{ZQs5C-nc!tpDyOgcCXbFt*Z- zL?j^T-6Dh|9(wxI28}M>fpo6khw334aeEBxQP^33=fj6W#5pGgtsQHek)2Dg?61j5 zpjQSTwER)F&+ED#-U+L5mJdGT#b4gx>7^O!T8fS&DmjAkoc(brdTcNhV7b!us;}LK zm2zdu-$-+AzAQEb$DVR0z|@5=?dURrXH0*;_Ob>*V~R>kR^&azx=2Eu}5wt}>ga#svyp?|=n5mDy+%q7o04V3$4-SWp-QA(H_c6NH|fF) zE(`=qbUx-SeN10o&JyW7^*m`WwS<-yxpW!8Uc;*I+nL%w7c%Mon9$0_&qA3U94&89 z;Mg#A-jXuTZSwwjLWW;H+ilB8pd=I*t{vGPpAi!R+#}auBkFCnmbul+kTZe6O`#9) z8x(3tC(j_`mDT=}V^dQbi(U8QTS4UK*W%rGgao;Dp}qZWyH}LveGew1oA%lZ{nBxp zN_8kV<@vop_OYUJ=pEUg>5PY%ag2?NsWlL3_7Je1D##O!_`0Wk*@0N zKoJr(+rb&-Z>H8Qd2J1udw2TV;#t4hD3(gO7vR|aw z<+no;5ecl7GtzY2*WulwU`bE*JB{{SbCkidd`#rdIdY56{g(I5`b`HlS-dbV5zA3s zuYKaoAY;!HqxBD>PEF&>@4tE372q}*ZI}qQad(gW z=upns1r6b;SRu0RD?*sF#8F->jEst;fd|O#kiC{{^K3=OyxTllA0tC93?@0Z8x#~2 z-uwExR_wtaG(sy{y| zwG^$)q3j)OZ@7|ZxXkul`i@XNWYJ}M)z3xoXmcg{I4CtebskI>&JLO(ABxX%GkL41 zscKdo+<%HlOG`6D(lEPZHlk10&jDV`G#kOLOC*g3yo;3=Ej2 z4irkj?L0(;NTPkkDxbD(&U;wMbOF0FGB;5$xQ46_JZU*aENjl?{fu6rtA`9>DZ=^g z-@R)%MM3a;JPra&6IDjX3f6{JMg zK(9W7;AMQKp0|Wg4rZFrmVR?<6CH6QCSe?%wpR%Y$(MKpO@fLpG;gF|Wyr-f^;3y` ziTRS{=Zt%xXlY}x$N%n!e7k8Fm4gdzUz%NT2xG}(N_#r34oNZh8=_)jBS-2V9L&=^ zTyX6BhhCbXkFuhR{mpEHf^N68w4j)6@oR*pDUJ7-C`Q#i=lT)bw2!l9LQgn3RjPop zF?UQfZ%1R|RYG#g2*>*)Gl$#rCu(Mh{7KO&Te4f&0#;S;#*uj;d1acqyb@ZC8|u$D z_+Q_iui_{wVqjny7OV9$P?QY1&Z1nd!Cd$*FVfuRWx%pqYDNYPn-B#L_S->x_rA_k zDon5bF`JAT-alhU5xjgt)llzmcWw_x)j$dM?on*z{CuV{qmm0^M9cl1-hdg6^4-A{ zb-Xh?5K#}G`*`d^ zk|ssRc6;_@o z)}8&i*$x{Un^mA}gymM497iNOd8CbvO{d0-_9tC4K{OF>KHh8mF!n4XW2B;9qe8Wn zBl5<0gw{ZLW>1k~TN`sllL1+3^u3AFflD~Ffr5UTcX2kYwOmv5z zjn$jD{!`C#m19cop&woQzslvcDAiT9y|UtG%k&Ts3kpD?8q5Eh`eN(^dkD}4zCMhr!|f84mi@ot(| zWfX|$53=+5lXs48iUiEncp^P4W@|Ncu)7Rad*htJXdD%RJGi)Gu*2KO$Pi*JXm(Tl;&U8f7l(Rer8e+*i0n{1|AjW#R?T&XtngD2N3d zbD73yBtx84)Ko7h8gl5drSkiJ0x}a9{Et2XO+Q_DA6mbrIpz*2;N{+vj5 z5X&uw9}pt%3#!(n9v&V{@?b)F8@OI_#5pO6MqBf})?rgM!ZVV*D{NUB!~M;hpAL;7 zZ{dvO11GK@6I-5edh-JT1ud`LicHyR-`T-xk=Y1SK@09NvYv&Zh=PIG=9Xf7-?8|> z2ZSfsEhhQ9a7GVju&dv}RJO-X+uAC3rgwJc8$X&+(jH$755FoyqgXuil>eV15U|Lp zYWL=){o+-+W)M^L^mw{obK9e7DxOe~{+w&7^}o;(d@+Z)NT7>NLn*G%t2zEPm+yz! zzEVCr)Mo)V z=#n@%Sx?&(+(<~}CzlMmi!d~P+_ki-7qJ5J2Tow$z59vpQ8V4j9tbax&fB{Z@UOVJ zxrJx0GOET^*80Y?Fg)2>021;b zDhkPXPv5j)y6rY%WY5`=y%l13bU=HHfPlaYG+WO+gd`PUOOf4RbS}Pq7J9?D-^LB% zv>nSNnclh=ZaGmI<>K|5Ab^aU)X4~xN>K^&){xlOhf@~}=Wq{9r#Ld-L05C(H!oat zsP3fpDo9(f`ny%O-f)m4j@tc7w0BLlT@ZFkaPe(U)ik$2WEjI!t!oz47_QcFT!Z@6 z0t6#{L2A`+!Rg}7$|6)j=bgSUOm6iUM#P`%nOJnw|J3q;B`wDsB(wz<0zMN`QWlWK zP7{9UALy94brV$xcZ-e=NGXH(LiCK@JTL4)0q`OvLYKnT!Ny=A1-03Y+;yU)vxEh-`>`i-53A;YHkdj_Sl`xy~?7b zU*@fb@XEi?5$x2ZU{q*9%D0U)YQVP|GYhkVoccj zt#e7@-IE{+277uzd|O^I(~07@W5x`MGNBKo^1zDT3wLHSrs;nvx%R>_Ht9JAIXb>P zNRBOYAKHrqSbE**OOuYuMsngv8b6t0k2{IyrxsTzu3%n?^VA5ryS65?qUT!_Qu!$I zsq`&JV5?y9Il_$A`jb#%n8()h&z?(4E`q79tVRs9^w4_#Ynv(T*isVT9@gIJy_RO( zUNjP&)Y8CWkiK)vT}R|praWC1C=%);Ipg|x)=?xx6 zw=0&|82?J){1V_f2nF2ZtAx)hn;}!b;og>T6UXQ?vYyw}p=dgAGSC_nP^UP0R9O+Y z@GOJVO@E1|^F9M~oPzS~J-;tp-*VcV_e&M#gJt~D6ED+`=f!_`!%jK8Fy=m$^tnBt zjS|$G6j{?*%Ft`BoU>HAps&05Ci_F4dB}jD0*c%nI&Ys!zi991Zq4?zJ|4mm>!KHJ zcuN%Cd-?6ydhINIOcbRMq3@OG6(#023wO1fRsYoS3@B=%ahbgcPU3<5rS^X0E%F~= zaPLhz`S_sJ)JKRHeTBaa-%upr;v6*b=A-N=B(ZkD+<7r zW}!QfqoD7uN#os)8|tI+Z_0M|F2w(rD$+q{sw5Z@M$yuDqb)JH@heRWA;~uqMeT1s z(a8q9ab+?+>0tQ7ebrV$7)Y{ZL%x8@V!n4q^ST|Lt3%0GOGSf{mCAkJQ!A-s!pRFI^v2nLDK{%k~_09uh9McSSxsTs)@Tku!}H634w; zjUA80XTPiP_-D`!{vn`%>UglyoSWFPlVSH+P9c(;e2InVD#6MP8u*L#g*3-AG3-Hy zhK5F1aO)v`6Ic>1h4C8sv;HrE21;PdoJFlSdjVP1P7+t8{q@W8mgTo~|N1d4G};g< z!Gw#nBAJQf301x}*Kh3QCx7y~=h2?!FR}Jn`ntPiRO%03+W5v*jL~_8k{jGkQMW*ry=oJF4^*|aPe4v-*0kWp7PmT1wi5>>DTT@mUgI>G++#>NkQbrqcZ34 zPTJkz#ubU+%Q@k7{oEG<+3lFiFZD59&W|@Q7`R`#)|c1y3lws5z3Ir2EZ9k~cT`KJ zph;vw!#wo_ytLN?$K>&vjCQQsx5=j++@DEWY>W$mbUJo0z(`zg{#e+z!F>5pDww(%Fw)#WeK(deLA!qpu z7`{RRuD6Hsoa8#^vxx^le{u)NfNuAp#0BJ$PsBQ6Lh!=Z4ud{@8WfH}$!}3VZD^oD zqy5xL&A?3q)*EPdeE-)G^FvH+)i!(n$(7~K;#$Q8l^b$Niy^1tc(JqsU-;y~g^<(@ zA-~)@JCvJzc4i|=@hqok-Z^ePlypo*Tr zHtndDByU*WCv?HN?DtAaN^kfm?}k-LMyQB%jQe%4)P{tR+{i8E^bnq^i71K~BLF#< zs&~=2IR?z_G$6oyOsU#_ZRI1PO$t#wr(0>=Se!VlXz*S|v5KZ`6r@kO&gRD_@ug=q z*8j!*f08AC;yxSDuP}22T+rjx?-o%o5gTsDe@y)Flv#w)bb{11vtMm@jyR_!HOVQr z@!aXgp~3?HIwd!@)L!2v$d0}5mEv#D{az6YLa#ULXYnCwIcJG90dgG7y%Np%7OgSe zZiU()Z*6LAV3%VRTq2Vf&Rp{gpo`p4vh3;g|IB7DI$)#vpWq~m6JbvR6i zEIjg9s&5AIdhW0W^o#jy-aWs5eRAHJQLBvlQC}Z+ zvbMq+x5pb9n|LmQvM~I7Wp{MU+731b^ck$-uX-f1cqo1^g6B&3a|)D-bN)@*{F08w zNjl2nzZ1SBJBPavCrk>ScYj-+-`I23nda?v8}4 zPgxz7E2~W(&rVACRT2fmU&{zqQdI4yT#XIa-y4ON)@$I=c^!x&3ypB+ulgAaxsRrC zLOmfV$NHaC8rT&trB#JFljb=udSGog)+$!< zbNdd*4A9if0*mp_2cO~zp7MCtpE?ZRHw3t8)jHbmpH@H)>hdV%EgRX3Q3LP$!)6Go zwfo=opt-keA7s89qxsVc{`?-M-@oUSR^A>EYlT;8`Ny(9uK*$oKZ9pHs%%%)@{!&_ zDuU#=%Zg*THh)q(W?e+*pr#CT92W(90u9se!~F#z5+)z=2r!XZ?!HRjGfFX9XP}F0Uv(<>!~kGF8bsYOTsFb;^f<2LcRN$j+VtSp$GvKxQObVvJWq%p{y$9<>o_X zUoy3lFkb2P=~-qF*)tuXKih>niZJfdCGurRkA8IXaMkp7^b%cqmew9k?4o)0Z>xQjhnPEC1CB zKuE1eEY1`nH{L0ggzw9BKKW1=F<2`lbJ@jnyJ3oG2Qg=K*C=w6L-bpVpgvJJuQa1m z+CRJQ9wr_|VfRD?&%3wnpHmSn370@I@{IMZLjqwRrwO{IH#jC%Cj&E_eB{DqEd?!2&9r~({qLg)QB-ct>|dwE zJAYVX*9U~%lEa}+4cKnDII_SxO#(qtV&pb0mQ+BX?jU$X+p_XH@!n#YSx1czJB36_ za81Z9c<51pj!beZ&gkDBl4J!K95Axr-_5{KH8eqUJ0HLcv9(7CFk#8PRvJu~l`dey zoXK=^er@L@fPl%y9tJCQ&ESB9gL(T+kR7hM3oqG8crL=Yk9;q&PWh*ci_VGvQZth{ zO$Mj@NvyqveiUY^NXcxhItkAPYh?Ig!4+fhgZrcEL*_*)el``>DC(&u0ze7S;p}7N zF#Ul$_8NI<4Jf3;U`PlbcAc=Us!wy`HR&2gJhx=fYiKW$x?AS{h4IkZ*?M8 ziRdqRkyQ(VhWO95Mt8k#y4^uJYxyuKyu3SG{b}euyF&XAZi%~R2~cL_ zUzGWE5reerQ)!Bn%ufm_+_-P)epvBJwtEbs@n^-8l8k2i?k2ef1HcHL{pfwpEnqsX zBcu-b8BW3EEHmH%0D8+ysb%NGZ;tanS>R^d6Aq6+g0S;5$onzssEs7AjBt8Cn#0zD z91S4ILy`f+2FOXuTlz>e;HB8kVdF6)_^?KU{J0w+kBu#*KX+pT`v~(d2D7H+ZzYE3 z{U6r>*D7b!bz$1+EvX(JCyULed9IoBjAh~)m{7PU@Ep}x=vjlJ)@Rh2{{gNn8XnEH z|`(iKP?G-3Fz{K4gPEUst%n)@La z4B}+6BL-+N)9f2fp6~Sb*`Aez68`fv)PWQC+sQn02S5BB*)q<^X%^CimciX@j*vH$ z5-bqK(6Ik_jQm5uEddGOxr?^bPvadgC;Jw%a(l7>Ep0$p4|e*Q3|KpHp6C3&@GlCp zY&rTLN+$C!`mX_TsQu!P0@h|H`PwYBG_5FVU5_L_a4Qa}ZHCV9D;idKn zc3sXC7JWBDX>RE6lLqy)wg2x!{F(cjR*!K3mW{#cy_@)LqQ6l=k$<2G?L9ypm!Xk$ zS$jhF z)F1f0qSqj<{2lQdCj$7jbUTNx+yW@KpUwbopAqI*b&bZORP~k=C&k=H39T?-BjVdk zHPO!lzeN4C#bOKcFhzvv2$IJTnH#zYSRSy$dcGS$N{D|who*m*Vlq324?r{AQCTH-;iDAlzj zuZmZzx+3G|$dBNRp9dNmMYY(Jc1%bNO&9gZ)*~z_feb$YW@E2tuQlqGVoN3IVVITt z15>Dn@MHhy{iit09YoeWYKdyVnO&Qfzr@traORFu^{?t)SplL7IG()G&NQ}^0*P%q z37NLNBKW;@9t_MgtoeffN7`FJRk^iY!zw8ff^;541f_%x(jX!rA)!bL2na|C8@6;v ziG|TIBEp!ngi2nxMn2w)GSXmVdgBTe*WOn*Q(aZ`46w@Olfn%U?#NdO(N2Lnr!2&q{ zvM>nf)^TKW{d&Rb6ojA{RDS>*v;$W9mw|u1#R;4FOTD`Py)dOciHkEO3AUQ>nm3f# zldn~eBwrbrlq3vxjXSFteI>bcd{ng#PKlpR`RKL$cfd~=N3MLz-y8wmU)%7%D-(?W ztW0j7+;+Ydk=!M3cP;9BC$##UT%+co9_go+A`F&di|f^O&rQzAxasHTcaj$RsEQ@D z!b;|*)Q>IE2jH;hdVy2ZXuodHs38B4H&p^5w7GlZ)1>2?^Dcfb92l z{TgHE-u?R7vuAJTeg&4;6zo?9fbqniyzemF<>yz&GfeFnbO&SxU`~F~7sP@UyYV(9V8#pN!ahDewo8Wz=z|;(kPUS+Gb?Lf{$F<;>6Iz=A1sH+ z(qEaGnYU?Z0)G8^{!C2l0S32U9CCJ6UR^zl#?T)H19!Sdf~K+TvroxyBX zo3(;w^Wz;o!G7709O6Ut3O}cEnuhAl)qo(h&TUs~04uJ8I_&9IygGb8IS4J2+U#+dZqI_{nOYddEA6>^2K6^y%dhYhp=EAm#>q#j{<`7lkWX{`~`-(?JY)Z$T+RYPo55&trat z-SvS^?0e#13#BEO4xr3|oQ+1=iGm;^PS#WhgaMrdE|3a437}(>b|=PaJYZ&S01XTF z_uYT}d|L6R91Q<&C3_haaMMI*GnlXRKiJ#6iYZ{bSgpRe6?ZzmxOKkitThr=wMXxO zq?mdlBI4rok@5pjHw@Wz5QJ5KI$i6j`=B z#kgC-10YcQ*OI0ARo9mrJ)yX}fQ6*j6xi>7)vEep@qjKsAkjCmRf5JbITXnut)@m! z?XV>>ADouoPyX@a$Ket<-NjL}@Le7r6})wU$7Yk+xp8$Q>seao4=0^39kaEnseMS; zl;Xw+B*)xz)-|jr<_|`B=vsz;rVL9IxZWTXd+WBc!`Hr2t$b5T*SQEkBXGl@2$PzC zzeH8ORWlu9l)Xv1SmPk@@_g#DktwetakL@xb(VQEmG$(5_0&MX0d;Zd)WyNiOuRA< z#}mX)Q;+jt*!Z85Jt65o-xl`|5_0xSv)`o0DspND|oaW;5?oI3t_Z7Z+1SzV0}{(FLZi*JTo?RQQSxH&Ll1OX$H6a zXPbi&gMGOU`FBa_WkByBm>dxqNj`4bCAhV{-6Q%5;(?qIKM8^%_3gU`LTFRYYUSUO z`r%7v_m{kaf(tb9cE}(Fj?N&~C>_R{(dGWMuRwRfxq1SvZ%ZhT(@7>{X6u zT%d}Z>F)bn7Mfd!W|^Ez;}w5BPP98-{cCCAh_-A=^5_r4zW&QnEj)wt^{Hwt$NS*- z_-I5o`CYG#`1Pj0Y2BwA9l{sec;`W|H)Z5J^Q6?CSK_R@qwnvM+$h#iz~ljGyiZzrWkrFg@XUMF$53G_gx z&cm+bOe}3mTdqv?QD`YY^k;(FEQCO_5)%_sK5nr+!tD~A8CIk9zoeq3*BG!Ab5GyF zLl-xlFYU*z#!QWP1->XzMo}H_=Q*3&W(fW{A0I6lR^b3LnC5R?LK&6ddV-HV5j>9E z;$Tg6+@g(TNZ1-Rdbw_d#os)GlXbNrCGi4$j5e5BC1qm~9r6@|1xY5+n}xL%C!q!E zI%i5JYUPJ|0i;aHX2;hciVZXg=o|}oWoSEJ>NZ4+vc*NuG$rcw$Z(tv3gaFqcz0p(^76J) zA}c*$aU~7OpV#JDFrpPQ61G^>tjEewui!5UPr%jk#ru*MNPTsJn2~{Ga}ba z@IW^Tjs9ZldNbeR%dgHL)*+;%zAW+fJH2XNtkUS7<+A8|&ElQQlCyj_PS1YG-gAiC4itSj@k6INq|9l0O*@Bnhdx7tWx&h>f~`nN1M<(qXDcbe-oS ze*U|)@KG&ZaDDXYd${e><`L4#I#o$RDatToy0gH>2;QGlPW-quLfQVh6ma*#0mcvT z12u#ClmnhuKF3MvVzi&_ncz1$3)e=Aq&vi0I!qt)`BtL+h#S4` zpkbF^?iie}_)R^d4lOA8#SPhI*O%W4j|^)Bv!V2bhxoy!v#VN{d>7y~P*n@vE*jFb zhFFG-xF#p2~f=w3skEtH?O2PrIL63{Kp6mu6)LJn5w1u)U^9j{dsdHyDx$}io*a(d}264vF z@)}z!15IOK3b$_Muqd|*AfBn~*{Zngx1E-Rlxd&ucwjL(dLBNn;?bkWc}nB5`m$oO zW@p+5rhgi9eYJ67?St9d2sK-)8Lq9LF6Hx?l`iuw7F=UWH&1g{3lo3ln?Ax}IuFyU zvnb-Yzk^20YG_PPBYvq@8t$=Q3m<5$3{$Bb+ETNjs-P}B4>R>;GfhcNRWmZ6f^X_o zsx4bawu&tP>Yj={7INM)1f7xFRZe?)bA!LW zX)68VEkShw6wHA_+4nOR!5{Q^0$ZQd?AFz%PC_lhL5!NP@@Z4JN9O^X~%$9r0Ajck7G{PyJZG)q-1T9t^9 zFu4wA7_a1A#On8t(RvMfR2pHd$Zcn^fggd44Z&pkH{^cvq(BKKw3rq$Q#oY@np%$A zGs0HWY6-Sp?(u=g@WS|p^WgC(67wY0M`HBYz00+aXI9+jnK_$_JIiozahrhHogL%@ z5WAl5c7Y_1i`}@BH2tL#U3KlspXkk|{y!_xCPA;gaQ02xuU|DaE+~0j2(DYJA)WV_ zWbG!oBm7mHNfRu6{pFbaO|N!SP{87L>8DdG)fS9X*Vnb(jCiSf#Df^CjbHkD&1=-) zvoG&WpEtq-gkYq&R#~P4naH4ZciqI0vV`GZoY{M1lXSeRlTzgFI$w#lgXU`t-W8sA ze%i0AP;oKsn6Ru!+amX;6j^{V^)LzKB=`7~r}Sgq3!7<*Gh~xrpDJ^Qd!8^Y^j=QXF zrg6@W))1dt3!`AkM7umwIB1W69f8bPubP@Cg2bfkI;c({@T}OpCZ9#V|Ko&u z3x4dF(&e%t!aXS9W4&Nr_S&&Hq#anKzkeS@30)f#lfaaUjsX%57$lH5o4X~W)cY3@ z8Ppq37y6fv3`|0zQHLfX;ErE_6bPO2gd#2Ef+JTa{-?v?;`ToQFH0GC8VJQ+kq*_n zxmyWNpq5)nh&Eq9HNeB?TPq>FGljb7#zB?NAR8?YpQ}3Ecx5(GUiXOa1bbX?X$l0? zgmdV9Ik{~7PIy*b8Nej6%-=lqB;*CSkqRFbciBDa$P8)pyG$|-37@>?iYRpSr%-mLUUdxYP+%mB6QYz;KB6? z_!kP2n!3VxE&(u-JD(!rV=sM1Cq^w17~6>{8l1AblqP(hFbtSh!wNcTumJT8S1~_&QAq5s5yA2A2DEy{Q0bAeM#RN9 zsoG-l9HZqS^gY-jEmA^!iI+QS*^~A+yGX#Z4h4)Pz|aei_96%W{e&g)G$?nm3*JH% z_{Cv$p3_!wGyi5ql0H))4#neizga(w{aQB+2sf@=y2F5THYjc>F4A@`P$Iay#><%X zj{fJ@gkCO=*REaeNj?)%L`R@ncyozHm&r%^f{xT{*WN?f3hy%UUExvGzl7fg5d`sw zKG;C&Yj*{LIOEKb#~V)9dX^gn^BKwUSp$#O3cE5dE%7jsUQ3}Y>gFKUHskWn#b3y5 zUm%KKrL{VuJFC7ttsZFSs^jWfCbV=fbSg3XYOG$0pD$P#b9!=;Ns|bYyYmLtz74*u zjsTRv=%G(x@tb3SBL<(!C&D~j>&8nh_iyu8bVhwBlk`U6Op>9Txp+aifz6@&<vd0tiF;x zwbAj;u(DE)T8dDVq*qhH=R@fORqJij2sn#C95N1n!S0QHlO>}{e(v$mT9LVc0Xbz- z?x) z_wJ?CL$PFw-6LIR;UJSy@alF=0eR8P5ZloEmvoh z`(g3lXB61M4ipC<4~;C*?cwZ3?E*kTPd&;f zFbLH5mLN8#xmn`oSN|Ii!~y_SK?oco{i18#1@&YuAWl4y{HMl@O$)uWhS+c{`j`;a zbI&{8(tN!$@7-;6nPH*3$N=Ho6Efjlr`e6+17BGuzSD39Ysl1xJeFzJs7MoMpou9t zGd@sLiFMF+^A`dem8?2wQDJMYjZuHM*3xG4tBm_OW6KFI zB@U(aGYyfoqif#>zEr)sk>>V*5l$6qAfZZK^=R6T*{`$Uio%O+=9ss6cONHK6j^vP zX0JWyY9H2ueP~oH6OP`jctB;n9ZS9%rl)hDDXPnFP`MR3-uTnN0Zfpla@J+1(Qg|7 z70PmTHXd@)IMt7|sbFHQ(zD4JUU6s_MMB*r)TXNCU(PleuJLkWO(-@Q;SfaSQT~?RaBmb(7S|B&KTU;J%YU15iI|?Tv2!KQYqSw- z!%J+Di(A8r=mdM|D6yoT=8JnLTtdWq5ia>?!)NYGb#=AcppP&C)lE#yCt$z_yW`0o zu|GUjBEj>9zO|C4YJYDZgnty7D7G%Yc4hvXh}4J*Qd7T=CI@OqVW&Jh$1z>{xy?2@GF|{V z0drf&ty9W8u=jzJS*6-j<^5wCaeHq5``;-aRBiEwAHNkW<GrON7*grM*spM6}#y%JMnE5-WtEte=_}>LF zr%21Ir8x({n+0P&J5myiadwc7gE1c^xtPo0D@!m^VMVV$U3^H>H0CNSU%H4EW@Ul+ z;>Cw4B`m;8ILrD|d}c%gG~lu{W5wt_D9=;t8XOsUgt109W{^WAKKIry6lezRj*xIt$_($NYoUpr9q1Nx|ywuk(U z&KF0w**h^f{>d7wHIGE)?MwowABNv|_E9P*i$@&C*~MFT2iD9lQ#K+tG9Vs>3)dn$ z={Dl*2rgJm*10ryVY#L((@eF-Pr>~`vCZpXFJ)uAb~_H2^nj_*++ju5o{^o3sr!K* z&v84nM{+8eyQu{Nx=@r3<#qVlF=8C_XAWUFL)+aHD%^-0g>rmCmNqD3XuH}ePRsA- z))1tIoPQPv*ft`n;CVPZ0WGb(pyVeMGc{EU8kREb917 zLSf`jm83$gv{C84xJ&(^xMLUKVDKkO(RX;j@8KRBkTCQ^|BvrL!i*@mL}q0D=Tt5w zavWvxXZ88|$Hkje$%H;U_U@O~he=VwUrB}Gvkgi#E62J~PYJ@zECfLgl7Gt5xF)Lb zu`PJ3W$d5b+RlA;j1p*X0Kbpxe6J^?5)e#8KY_h%+5z8)oftr|c{kaa|F;A`&}a-w(ooi7)tIQCxK z1emt{3H*z1d74Ct&Xw-C-YtHiXs|bhhejhWAjwT zg|hg)q`uBmAI!;DgvB%<;8*0PRL?7#UZ+yqOHm);?h*;?#%IlB1cYe3ul$+>%NUbO zPDNU*YDNUdr{cMWJ9%!^x8ki;J+U|%Sat1PbFtEv%YRZUiy^YyrS^%-AbDDlk_JfX zQP~gk_x=U z;Ed2}5`QEi!E^j40FqQz8b5fA)bur2LH5o;6-3fqlYJjbQ`Pv7CQrl<5$68;8+Q^Bl+(25(7x)!}`G9{PW|UeJ_Dq zLtLHc>JuVoxPZVOtUe25|M<;6=q_>P{7$D8S)y|fy{AbM>||q+ZW3q~ID535w#}N562O9`LoIhbo#mN1Sm%zMJME<0#W5z)(jNb8 zZV?}Z5;s66LEz3>OR@wJBfI2Hsep~p3+)hBs!yQ#u5pqDY;*Tb>Y-dV1z8sS;7Dyp zAXE13Q34^^U?uf!aKOL%Z5;rAzVfF1y{T@WV3J4$0kw7#U`e3Lc1$G?G;0|7es+n- z+o{|tSTMma3vD01>wDIsM$h;H$Y*i(7o2%+k;}UFYC%qs{~ynMnh}yvN2T8pX(8}1 zI0w>TrE;*Oa3I*qc-I7eiG(>bL*ii8OHn*RAOVrBs;Cf{M_3}TdlY_vLy(dWk?x_Z z@Y^QgJM#m)Hva$H-#Qi8p99fOV`QNCVfKFm+rRek@3y|&%5eNQ>(FtAXMdd$bJ8VV z75K60mqwO^?PTS)p} z^%_=;_l!&;?BU(JH}a}Z9zq^f?bp zm8IAZhA7w}#};BrRx#x23+Gtg4yNu#@#u8#yvLWkB+ut`RzmO%GUCVZTR+4gije`N zTmu+})4zSTwz`_-&qb*M?u(vUgT@R{8D#TdNrE9(1gu9x%*=ft7m}SuDSGeBDYb_J zm7dT10iyyU-<6gWREbGB-(|9Etm>u)KuFrYbV1Ep(e2{bSiOJ=0|uc`32p6!Wha9g zE2H1^$pIBpzNre4yzjvWJ(JXULIMIE$>EjH7i}l`4-f7D=2O#${c)5j9i=EEFp@H; z8yUzn1ONWcZ#yler)FQ|zZ9lVfAiT7Ko{?PsjWm!cipd{suRk(1z|3jHSV##e3fbkBTLA;r~Z`#6TV|I)ql%|W{a>4hZ^+LT# zL~2@EmP2xKa*5wz17D(m3JoEZiZH0M*IkdN)wjTp!$P%9Oqg-01wMloWT{}&D_!bv zk}SY2=%nO)j{)9W^-PLl9IPHE>g zdMTUiRGK~^+rGN6u1)W&cu!^Izs0BEl6jufTCeP1@d?saC>K?qF^4lO*4#xsNQ+0@ z`4(E0090WiiE9>DdhPNyj60WQIE^EMyDYyxh%T(@i#)hc6_hXO*;=0hJ2^dqTw7Zk zYK{5}RgtxG+lPUG_5;aT^=Z#|#B9S&`bXP2r+^02*_^DDdXq?J1h5-;c`p(UG!w5L z-YqOFR{-u>gUXK%+Dk$KBm8qab`%7MzZNec4p?S-1s>4nj040nr zU3KJd7_R=~eIUU~U48uo;E&jGI-9ghBguT;%ec+g9&7+sKsP2|hNrU}g!9i-wm0HVD#PPV#MyRiV1KnHzxgWmFElz=4)3183ld@T$ zY0IL&8rEd2EqM6sKV>xuV>4j_JuUA3wB@PKSw2dLEQ5Ro5ffV=+X<(&JTw8FM`lPN zH^-F({2Xa7L$y3wE=Pfy@$j+)P5D(e%k{oR$+&T5M*6B;3@mPPU2h8Vu=N>=CPM!b zPJ((6p$pvLJvst0kxk#M2?KfDyk*s~WI3xw1m4zMV^%L-J7H#^b2Rg9`wI6ADhU4i zQ=Hv*5k_?}HNhTBYM`k74pJ^PKw)d5!P0nDKs-958G~bZBW+ zs%7c{JNITpX<6}c6uRG{1v=KQZc3$Ps^;Fo__YuU!S(Cu7?V5Gmr|{}A3Y+2Jq7EYXRXsPaq5B=y7l3gV>8 zNK;^Cp0v*i#_-eSZ@ycodu_LCb$UPCkWQQ;X}CGUT))k25FfOTm~t5YDE^c5k)40a zct}>xJ~t3L3t9v6t^{FKKFrcUUPikr?9X#?HU=HBHQcpE@}U|&7uyM}5T zTktG%kO5odf$a~dL?R4Z8s<1)NvJmkBhQTVvC-%^ylO@;#wEP{9axDTgJ6%M?BMHE z4&$ONfF;9Y$Ec3#WGn>jPO!sVX_Hk)!aV0&D#9n^To-{4I_$FGyA~$Ys9lq+-TbLW z?d~rTsp{(}oxR2j8a85lS-27lF17_XV&arq-2ma>z*X*yr9JkRoLQ?dKj)?5cPvL# zrO}6NF1U6~^CAGPeo%MOGtv`4a+%+!qB>W4YYSNUezvg(r~X7o=Q8yx)T+$NOu7o1 znz4}`x_Xqf+3HzHvdv1RQT~}@N}uc>wxao*W8yXe@!`)3I;HqmKim6l>&VFcQEC5E zGqKbCQI|0{tA>if>%nh{j@q|$NTM^bRW&LMM5j#$hCtB%D&t{Rqi^#8kU;&`=XW{W z`aWi^mkc}_DJ?R6xt}^|_?EEoJ!Kzn1U8UKdcN{UHR8vPCt0h!!&xsFk7}vhfCQ z#pGXIBzi`T-qOu{VPdls$4ZYV*ObOGLH<#G%$9Z0YYgVT{9Xed=|KB)6-hUF?C*DZ z@+lb|)Zw{z25;@^pcQj5CDP@&v=zdeJc(3dD~JnCL-IG(h=zwBQ*LtUzdiq+qegb; zK{q^tCrjsR&!1*xeP({+E-!I}>2AC5YS@r&v0uK>(YKoheX52AO_h)>)lrp7xg6`a z%>M6s`nTmGp(24?SV&OnrW7!wT{HNivqxHs-C`^b%%{MK{cDt88M@lJLiWR9=5auV zbUcrKPb*lkiMdv91MF8_PiVj9$_nwhEtxD9md!9CwfMM7xm^pA{pkLHA{80{g>-+s z&hnQlzH{&z6ltJdh6bx#IoUxt{D#YK`|>~F@YQ>yIJRPLRx<_)il2^h)6Z(@1;rnU zf~z+do*Ah>C~oZ5Rao7e6{V4H2Y@t$+X_h;toSbTLY3|DSs zizaimFY)i1-)H?s8L$eF$^yQ`RrN^sdwR>?bs;ppWwoh$xc~FzEeD$OGwLVRf&p)* z1|C6QqV5d9ov*IFIVE;=qZfCd4)yg916d_?Q#Zfkqn&1Be5$V*&6ug5Z4-gyO*It* zD@O8vhPAE?!V^3f_&W(2^1UV?Vpvddi+o)|t=+e|{{qs`iq5DC19x@{vpuqeAox8A zUK-H;9&%m}yBuU@oHy&M_vlRip8?e0#udtd*eGDz0>WCNe+g?%O9Zy_j7rO=8H59R zr5S)nhoMlv{{j8|%6&I3%#nz6pvvy+zxL)#tT0~DLNVYrUy#4~4oE(S9y)t;D2R<= zAI$^+0_+3qJTRgGg7na|B&d1;bC5*y3_JdDp290g3!cX%AcXblTe8UuIl20gv*B8Q z*=*%V$v>3)HjiSD62*W9Fb{AyP0x+K2P+hr%JTBSo0>?(xUpS*m% zwv5RT_lM7@H+%rXB>pSG1o!|bQvgsNP>=U-DBQh%TV%PZKxmf}geX1t{$mfCd>LVN z6R6eigJQ_nC&}6W#28pKx`TMDFR?tQ*0b~<jFyXma#6aP-)--`i2 z_Z5+Z+3;_y4c=4K6bcCyNdjOlSGg%QCA>{+@?yc6a!>CI8|72T1KM2^b^E*r<-Jk* zkgj{I4DhdkqoR*yBqdL8$Tpuu%rT=uaT!XBhU~RATNQ_W^0<2B-n$RrOj6aP1C5?zcZ)%-TfA@gA zWaaqa$3rsYoS_W6PZz@@j1vku?%sXck;)FbdG(u%McoM|k8zxY^BSduj=0&_^D5dKM8XQ4-~->VHU zXt6bHkz#ReqFkAG8Hb_tXA@%B4939$5NMaSZT@BnsISA($4J^5K%a1tbbVq?rNKf* zR<;Q^1{7%?k98h}eKan<ex-({Z?;k$5Wlv24MUJ8grxXHHS^3>ohrW7Y%(b8NJN>dLn zGDG3X7ayO(dyG9x?unNEo%a@6Ccs)iU0$!{QvRkp$W21NlPhGYkBN)eDfKaHQ*jaq zfC=$LfxMVkshn6i6~ug0W0tWy{Nq&ho7~Y3xgaEX)eTp=-f$T-dE3*qKovIBTr4Z{ zfQxGY5z&i{Q9c41GSfvW5AZ-IR>uX%8OP#-CfAC(85z;g&{WHaW2M=DWgkiEf+y0k z$6tRmK`E+|3t%WaK!Pdnmq0dEHTStWkf=PDlH#Cu=nf}7&7=KZdWG+q@k9{_YhMr* zYY-8>Ud@?Q8ATPP>hJIRO?SdZmlE!Ilz%qhbZKp-zL&}U5GuD}K& zdJ>9<-+x&Mob-DgG{A){mzJ0;&hn8PTx-7P9pcFvp)|Rv&uxFI%1e=Dftm(Qj6Sje z8=uNlJe*s6X&U*>8pIna#d(+Oj=?}Z_zwB<3B;*r`pn!${-$WW8Tl5ko<`HoT#-51 z-K&zJ>(WWuS= z-9(^(P%85=>%w$)dzqYsG629RITXom`fupC-&zf=&mbEF_uWmo1O&!kRwwoFcAnk7 z!ksQUaU)I=be>6*5I7wk^u2iJ&hAV}@_CiFiG3|kTp0dlr*i@z)R`j@jB?PknVR1R zuS&m-Ab_Hkp87wm3w2=$x1VhXIfj>j%x2509?ux#W+i15l)5VrryDr(xa+LY$yQJw zOrLO+mq|P_ERN|3mz5Z|S7n2|KZjdF6QTcN*+75-ZnXdmxaKa+rjR_rB*64LX|6Z_ zh>IT-;yJ%~^C1?JZ=VY%{Ip$!p3(1O#I6BNZx!+gR=gt#I^$}KOajjYhw2exnpE}; zY%6D%Aw)(eI{aSsS$!Gv1^DuL!>K^1-fuah3Ydi^xTh%!emVKQhz$~L1ONwY!P z`K3AJWe2F8SVYau7w2>@AfoW*o%?UcOW-z_=ZEaTU7%-Hef}7V7Xz9XT6+4%@9_x0 zNS&=22)pIw zZ?M=>lu!Ir=5TB=usBT8>`!H1N#jq>CFS^7A0wg`kfh)BZb1UJbiqJ@45rYIb$uevOjf1|?NC!{Ey zT?MzK$~WrI=FTCz@(aJm&zup$uILl4P{lvHB?`HsRZjh@34$TNUZ;saC^7>Kjjr8w zo?>Ps+5ndM&eY$SBbiI&0UsZ^obdUNhj~!CoA60bz?RnKzLw4LeArcJQ^YgKVc`$b zO|w!^b1h|a3t7V0h}kr=y~0JWpa@73BY{#SMgID`?sXSx`r5_>%=;)HGXz8}gp6=a z_Nr*P`RHzlZB45~`9$)uw}GR-(j}%(bltU=$~fQIh56<(mmk-Y;lnkZm!I_YYJoLx z1gKo)Ks8r#CwOV8YPXZW0FD&jkAPokv>lGj_eVa#$RI7J_=yr~%3`d&EN@E$-VmpX zIGCAfmepz0`SqlUyDH(ap8@EdLFIAh-aSz2e{%vO6}3p71KQJX>_hpB_3OG%@DIEh zV)N%H4NgoL-c)^6%-U0Jtm+Uc88G&?PthouMXy{6kZQ2 zLmn#~$m_(MEjnK?fZ*5um)5zm(pchg=W7P+AUbeus+tBLAK%WGDijHsy4oYTq#cQ` zI-}9cl7M$BJZ;muNOX23Ksax?(WDpa)ue0;Yvt6G%{x5@NxxzFAaYc&KLQZXNRT7{ z`aHO8riXU(SL%zX{2_NzxKv<^G8LO^J;^- ze+kN=24u9#)|4t&dHgo!mdh*an7-0Wu*ZkrBZ3fsJtK?bdH8`FNF@?TdPs~})=@R5 zvBdqU?9G2&6+){eb6BrNwNs_V{Z~8|9*e5@!6g;MR6U6IFgDM=Z_BwV1vkLUez?6X z=d8w3dI&}w%iomt&p#`1ZiA-8eZ!1%b!a>BFf1ngHwz!;RRF9BDY1$I6U;PY9P7#=$fq?5RE<&7A`4Tx5Pz~bLq z1ZZ#VLHivF)cAML9kj*a#XQs?Zzxg!fg=cXOE)!fhM6Tqd=G~i>AwW{0}h5!X=fQZ z?Y>?*PS$K~{!EkeIOFD#M5AZ8P|?sb;4{Ds;jz5^{Qt59`0BCV&Qn7)fGzONqq4p3@$v3w8;GDOvVPh9 zDe5-toSl;^@-%?~A(xvPtOF)<0d4NG*_gb7Q3q}J;;$*ADahqLUyTB+>}6m6kS5@` zfQA_#0dHskDC=})>O55P-(G(zx_-L8Er2EU#>xj4RiOtip`+2Es??nt4md(>2WB7& z?64wu@GdacQIVXCj4V?IO!vDwHZ$W9J9CmndvqS-K)3n!;_O(Z+$u-S>KihVXm~Qt z`NEc~{+Gu2Jq8A*o*}r>H@XUREGfuXTXj0I8TsMxia+}X`NeutlyL>EiF9)6Du>Lhc~8eB_$~f)$iQA zb?epB{{DvPU5AHFGZlgKvm3CKO=TlKD_|^P??o!3IIk_=Xd`2yoI}ou7&# z%Rn!xbl#wHqYD^EitDP^KN(Qfwox-{Vl0w4})-TGfH=9VR zw$NT%ZA^2?%i96~nu~KsFA+q(>!2Cxr0V)ohE9+V&PhN(vU7sU*V1|;05Ls?D7nEQ67o!Pgt}caN z96DcGqoSfN(Xx~xmI;~yX!1fEbg9V9$tr};X7G$2zfx2rblT6-da&Ibs9LC>QK`=O zrl)ja=|}Rlec&L*2aZ3x&-R|!WUfFa|B5#$!V^e*z|A59(sUV*^4UtMyX>@SG>eRg zfJ%!UvOz=A`oW)7^Zf?Lj@XOeLTG7e?b4}=@mXyTJItJ3X@oIF)BZYdQ_JFCSca@0 zNSe#f(17n${fGZM*7rXt*WIo{nnaM8WHeHa6LA3wkekX%xFB`yDz?nc2ei4k3)VVx zN$t`C)DAX&S8bo&!`5`ZRqLzLfJj8;?%(Xz>_bL&VM14IhJ{2ipWvj@qN!(y_R6!J zwpN7>ztadB6u!??Xn{aJcs)S)qU55b6YQ6M=6ig{5hN06^d5=jn^o$(=@G)z$dzn; zs;zP`)L3LcVn|2q$D@C6nB(t7yme#P={~Oqx-K_(-x)BFP+>D+GBM+&(g3XY*_}dp!~Zmm|bWU#;CuSAPZ4Ng-yf zA?Nr%ujJq1GM@i%vw(5M6kWR{B%^348i(4QAwb^KO+C)`z)Ig46{ms>8vE`+%MaLD z?gGQ7K;3Yat&kT0;!Lb&oFv08dmw(eht54W{&%hB4$0r59Xvcdq#7b%2L2XbUZzLz zTjzeU#BgTv@bY5v+H*c@cILS$g-+-K>WmzBn$CCT?y#VXUE#}DEE&znH=t2P0ePEz z$zbRpLw`^=G6CfTI1tO!2z<9d!0zwk=S%!E_9f?wMdzIra4KESl}0G=(< z{M_NNSYVAA2)nr(5f+x#o4ADB+&r5l>Y8B|{qh0~6(10dN{HX{Gjl+R;x=mn{_)uZ z($4#HGNS${|MlAbKde~EFpg0R{}-t|an=D*kAwHwS>g?2)$)ZwP})dqP$x(#Gn=h( zH7}Z*0I7m+$0VRIBMp<+pp0ABd0BIblIyX8f3?Y~HtsRLRt!$c%={L&=5l|DF)cJH z{@E3nA%NSKRoWjGFJF5{*4@)V{}a?Y*|5&+E7Lyrl*6l#7$yiQI9;AjF&TVEVkaND zqY7L!+&KacU@$LwqXF~F3*Za6%yNIQ5HvMKftu3$5foBg-eq%rdHCda!qDIK&TlGQHEuZoihWA#6@Gd@ zsi5U7z`g33sRD6l<%uaOUKLIPnjvHuSt(HH2q z&(HwtiSQZJR;!|7MNlK|_qgabW;Al|@dG!Dn;dVEZ*Kqawc(W_nc_`G{x)`=mNyA#wqXws zPTX{v{FlzAQ~e8?{;J0~sfeVskKX7nrzO=gW;`=cnOcoJHos zm&pK!w>LDt!RlLZaI(Ib>3E4kz&18(U<;WHB}#TTGvhMJaC(T}L*=e9?6C)pcvk|Q z;6DPLhVxvtHZP+)qv!RLtUuuO9fcZ}mt0ME*TYwJKN%cSYeG*4M9~(Y}Rj(F+ZmkUL2arwS3eNDO( ziA%{k^UQ!i7&cC!s|K(HDd+pI77jZ=C2!No#e&x;=8?|UA);$XYNBAT?K6z4K8`=U zL(nbnh_jFLc-REzeRp@l-32XLgFf%>(A^)1=* z*Y0llvpoah<{!&2`A{;iriP$=v5imb-)Kx8)aHli8|y0jDdqaD@E4lt&F{Y#cjY_0K)d#seiR)L~B zrCejiIs0(o01aHhFr7K?_44qTr=^-HS(mmt%R^R>vV4Pc;hCZ7#{i{VHFZBhO!Wi$ z;fHH?6<&utSBecZBSK;ds3Kz<| zJ4I)~Neaec#dCk<$XFYhek3uGyvQd?jZa8@>32PMvG1so8Lu-_R~_E&)pNTvj*SQB8$9)0+V5-4*w2j ziK=V0BifW~6cnZ_nlID*$ml&@|B_)9+4Fc4Etv50R*s`Ph8`_%@oluy`BA~C zP$Sb6(&Fz~`6-5nS#2kzx2*|_&+ZLpw(fD5M#wG1nD!glZV432F&&a$=z%7ABozvf zb_FAm$QLOhNZt@NK95mSEe%cArbNYI0=ZS)Yn#fWHu8}v6M4fw)r()p%IVq`AKVbP zmiT?@{f7VnDVf#UoP!Yu$Wn;CRw{Cwut_k*>N=sF%V5hC3{FfFa&JP%pS7cF93dh- zEC&iW^2_EN#sJhR~D7dC~NTouW22 zsIj&Ss$3}bHo7r}-GS}K+Oh4S7=EzS_a>H*GS7*PDi(oWZk&}p)XR~>lD0s*w@0+5 zN#YU$bA4T|@+fd2@4`u7gGY$G3PU>W1le8O_8D7R^n;a4XzT&i`VSnyt1DX;8JALi zY7N>|C|qYzG;=5@ee)AfEag(?jkHi6Wv3piE!R#g2)AynXYZuYIpGJJN7_dq%)$VSY~&zlSfQ@qJ!?g;oDSpyqJGf4^!c3c~4@6T%3R9man@n7A64ipz>)?a=whOHJfO zHosQu@W)T@-%uXnNK{bDEE40k)uSj<0}huPQJ{~v>H+)#t(Q@fzz(UZ1*n=-s73US z;mxzZBSzJ=;vTeV|$A|G7bZ1 zeChZrNc?QBf6L=0xxwRZS}}&N_zQm9FC6&HF>onA6EP{JcR7fTOcdm(X$TrIwaQah zVa%(0#(zNGN*CS|TIo7Tq;+6gK_O0bT9ofMb-KyxvH{+;x}d z2N!V~-o4p$El&T9FHIL6>?*>SH;9e5cP9^(W#^`m%3s4?WzOdj@9%%uNuY@FXTHM+ z2fYFvgh`Kr97aL;QL(R?dSAgoEd~u^Ge^wm_@Q1~D9b~CTlHd-2!gG7K_!iw7kC0Vm*uLs_W#!Z>6m;pC!wVCBmvM z3WMvI-R-2Ta;>3j8Lp-$xWz!zbmq;#?(A?a^aW0n3yVStK~&nnYpxZFQNwaI^_+~% znxgPmjALBHMXPZl=dnL~Zhs%0RB{V7j%$8zdSu(bMiUUpB$A$!QT!9`g-}7y)&+X9 zq%8@9M_2O=mwxF1$r(`Qk%qZ}yo=~#@TEiN5`Q^!J>;%(KY0;J6_Z%hx1WwT_W9&a zVbS9=J|phR&nurA(_7@kOe-Bjy^;z*YDTAVM&GDdky6?NZ#14Ne%vhmm_~uuiz35i zdv|fo0>o-+nKDhIg}%BwQY6jnZu2rn6Q;W_f4tSHQ`n^-6C?0yzlhgzfl=YJE@ick z<2xWrw%M#a?9ZmZm^Tp_L_2*DrldNolpwF?nNq(h}!xt!fgxsO0O=S&U9g2YbvYWh@6R+>Ey-Txy4*Fn*AkkbRW=Xlv=GU{zv{jTGacnooEv#ohT5?fp~yrs9}0Z;g$V4l1H`{x!cW)n zZR+oSmX}W`&GnfQ-?)WA(tzgA8xmMKX(0qkQg@Z5V0oRr44s!;J~h9kF)}rfbi@s% z7eD`penc%6&&`+GepRp--f7EHoOCojv zh|m)Xvh@%7kALw5NTHF8Dlw!E@1#GXaq?OXi@r0j4e~kVuBySGpkvA?%Q(Y$Mc&Yv zlv*F9^&1|hkfo?ehF;HHdKkFKwO>D=aSV}!9EDQvgzy?EfpTM?4MM8~mX$Suj-jkl|AA-za1EK)ZoJZxdbI9)c1 zI9K@bbKwWdFh5xJRz)Kd*wG>S_s1}^?)RAUTeQ{$%pm4w%I)kNkKT;X>6hW`3|)3L z;kG{ni*;t>kp-X|^eS4{kRcKFwZ5)5MzU3u?A$m9BEgzEdT~TaI@(RT_*9cG6YL{@ zOf{dY?`nkYvRkH|m|qd)yVuu~$D^ClBxGG*{zz?>N~>vCX0j@nQuM;tVIbz#jQ$pQ zT-PkDr0VUA`lv!T8;qC9M?*^XwOlhz%GK;zp2ys^fq%W>mjXS=t6+rrmlc_$#(is( zsuE`36BG1BQ(-xeDtv+s4X<>FlnM!Jw;rq2;y(J$`qFmL=zs@++HikjNb9I-tOk*H zc+wtz(G!w)^{~H2&QF!cQKQZ4YiI!0ffiQxhJn#(@!9&32Ulrk+77ST#mRis6Z-Km zfQ6(u6dQN*KbDALlPOo6Rj#87QY+Ylt0Lf5{^H(k2NlxSI=1opnrLcGlh~3F7U8gJ z#qL$A_Y=JzOFmnpshYO0LrU8;0P&Ykj+%`xNVZaO+0ZE-3OL~;`^ULjYMqr|S&P`V zu{#Zwe0@AHSqHO<82dZaURIvE`jh1Za;o?^X3SEZl^ykJ3^ttCT2^tgZ#Qpsh%4{u zE2is>ZZNJo{#n06X6SvN2Z+Rm4?nnxO=8_dxZf@-pmfkM`@dhmAnRLG;g>(4*Q|?I zj8~_hdaFguE9+AM9HSSWcvhi2SA!_DTze=ofl#L7*PM1cwej7Po%mmGV9fs^P+W>t zZ_?+qe!FsMh9Ju9ntCaQuTw}`*Kn(BW#>}&u`RLVaws<|{OqCpuHmO=a;Bdi%K~e6 zV_d&$WjAG8)0S(DcHDp|0~auRvDecCwa`X|$yjT4i?zrHj2UX>_05#mIKO`9!zVf< z9_d?C^}Qrt$JKF;)^u=WYQKi5s0&~V?qwBdrL2iWvu5{^N`>VmglPY`+w_Btc5^I= z@){;bOGOvd_DzqDKC4GzCtOF@^h?En@#n0!eFN!U@vfsS4lD|ei%)(ZsmXnNLQ9wV zHBswxVQXQn-;~6VzuXm!f%$7cas{!or)2A(SLNl>Ofes$G@Eufb#XMeO`Pu2`m#n*T z_q2pshW}wg8Zf5<1QGBIsPdrzK@4CYlDy7oW|;mXe+a;hZXPSFmjb==JBFX2Ov+|W zQ804x;8e4Bk_Vi5JeqM2{~K)>ktbECyqA*Dk-&0m49P3JqNJab$Kyq_$bS8-FSi#6 ze*NMBk6i5U7^Z@t!NHe9<}1;n3-B>c#kI6b>rAF&!ZBb}0aCoj&_ng9VKQ^Qn)+Jg zlTup)5|tQ(Yh$%|y~!Q=u>CsED*AF|Dv; z)MLmA1Jl&0Ls7{k3Galn(msaaHM$kU-JlKM-d~bS8|#0yd-;FmZK9^@D6xV1V$%s?P8-`;M9!=Q@QhF=uOQeB?kFGC>``O^duoyzoh0n zBA7no9y$Jwl#J1(b=o@U{|n^^_wm76&)Q~ymCDbaiIG9eh8e<1u> zGO9&8dj`RdYMO#&Rb>VZhT?^xBX;-hnoj+SZz1}@BP1^}dRp=(9n3XbQb;oOa}Yqn z;tSxeezHhqcsR9ns_B|HC8CS$7nupg z`7BND{JuI)4b*I7S4gU+TsmHRMtYJE@?g-yEgDGW01;dGF`_%MQD9$TGHX7hKH>GG zNTWM1#mi5rpwR~!`2r3RqF+Sf3jgng>Hm1HBhL`hk1slET<t3TQC`a$C7j-tawOvcrLqR1Gm0Vkoe1!Fe{fq@AZgll& zYr#TkCdFLHiUlX9Gj~Tt;36%bMzR3mJ62ZKr=p_0tpUQ5-zgDOvfQha^ukFtOU4Sx zN_b3?I}M`U88~@gVTE1PwfmMD+;MJTz~C?`M7#JVAeW+5q#8Ngv)V(Iyw~7wCVcNe z##=KrDdpMr=}q`(EccMUw#TC>{CuhF=NzjHl4Z($wk3zJqtgbzO&Ih_U zcur~}lktbF^1SAAet`|elhm@)MpZSMjMFvrAb$HE>lb}};e}=Fn<2k73jJZ}@NcR& z_gn{6maso$JenglIz)$}CRm2n9XLd!h;;qd!t}DzWq4x?c6keRnC~oue7M* zR`2cTa8%j*f@x-o?DbMVO&Ryd8zY6{sV3>N`2rSU#sCZO+UU%#fuG4rd-ZEod4N&0 z$7nFui~8TY4Udu}!$VpG15W;d0T(5Ji21CoaizEZwUBS=RK=|FrK65Nr>h2Q{*ktp z_NTa=`G`xe-|ZT)=R(zhr2OxMh99z*!uykFc@28&v=x<`@!3J7ju8r9B7VKJ8q%{d z{-Vu%`h(8bs;O68We1=g zGIheR`%ChC&tD0hTZA_77_#x%u8ru@fL53ovm)^%qvfo>ysBEFPq(%6qz40izoqGa z{g&cGInItK6lc>6^oosBv{t9kz7|poC0re1@VsKOdi(ib$bICI^XvTr%eQmV7ReEy zO5?K<))wc-Mx4bQzhEz2EqNo6?wYmseESA%u_5x@TGakDH~*sX7A-|{LgZR$Iks4& zpypTdPho$AMY@a1-*HFnOT)NisGiN;RZ;tWqw|FkcPd9aq(C#BvJ&sT=2u)d|9M*~ zOh<|U&c(MtM-7jjt$nQzqSQ`_RzzFM74@wxYWql5|6E~aHrf?}8M)DD!wKoN9@h$1 z+vC<{Zo5Jze0hA1UQl+llcbX6Oc9&gM`roIOL7+%?lPKmQSinK`U2WZ13D zatsgjZnWPu6qKM=mAls~aS^S5(7(rlGuD1wQ&5yg+jFePF;Y2+e$999F}Y?LD%eDy zf~kgqDx8^anpr^nPEQ0bSLc`eA8W0LwRAUSyS19_pKB(BwfNCRMn+1zA5I$-{6+Ge zxLJkEzO;7xiYEDD4N9N#ray@qx2FZRbL4+d)IKHmSGk}svI z#_@AyPnf0eiG%gE?#6wMg)^?%jn@0tm}t`IuBl`D7tV)4^oAaH{<4$(+5bCSrcB~2 zaO;~GF0Ja0a2Ao8lrBeMDLvGoitm-LLNfVhI;a|G3fF!8vAw7Q`+(-}C00@p) z7@G!8&)5)XPg_Q5tzTA_BwLy(-XI@!2o&Fb0YOs_cy-No+ZtHuF%*S1Rb?LjdQ)_q z`+5M`lw6!GTaE(DTJhO#;4!__dH9RNqSqo$MLe{zNy(4DQc_gKlq*7#uU%p8p4`05ltOA!7hk;Wqu_ zG_D#ha7pU^H*WkH_qYqLme6+vpqFrNr~&j6$R-$iNBnCvDansTp*4E_p*ue+jO-Ur z`hD>w@KL4kx>YDs6-{%a*#QmwK6e<;ccrtwrNp(;ZtrEmralRb;J^D8(IU=6y&3r= z3qmVp#GPaz`Np;z-AaSqf5<*1Eu{=jzzr3@%xotA8?RmB6^8he0!6vj$%XDN^~a2C zX4FN@3xg$|tdg_cJW!w1olKd#mDr4MUw+49+WZyO!THTa)mqSu6+SgL-9EQ&YfwXx2sQ$My%ULskOiS2si|n=MQzFv#POW%4B=c566j1b6`N6- zyk-g+KM%GcS}jAeq7`}PhdNugv=MkcxQ*UY53p*9;(XOVLE~^JP@-gSIiHlg7Z}*0 zFLnUmvhZq`o{AnD=u*p(5h<}OD*Vw{zem$a(&0WsKyw_ zncaBu(usvOPR-SVlFx1eEsr8Bers9v)rHVi$To@FF%2Fu6u|<>q~>$V;ZdxHgx=@> z+5{eh!-hh4Z~HHHBP#Gzg|L=Ylu`ryV8j~0R^asW1l0J9=-LZJY)%Y}xx{*_H408N zi&R@mv7C5g0m*I~Z^UJ_T%}4GRl+YiI!Mbt5ry%H3&r?5NbK-WQS#nkb^6K#4ASHi z8|$eH31S2D`&ZS6OuxVjO$g422wUfozJ%#DP_E3zg)pp!)Z~ju&FJURb-lgp#hOi@ z*~XVuISlB6cP7-B?kvqJ>Q)94i=6PA{3G4${0^RYV!-Ejbm7LG-i;jv)0HW~m_@Ql zH>Vux!MfE27Ts-z-G{O_0Ykq1BynH>+_Ud!kJ8TC*~~?xObg09+*%E{>Kjl6F_qv{ zU)Mp=4%$VVS-F)2T#!pn>^Zs8FWgQd4;F?ZsodyRAZ!xtw8&Q##TO(|CAgHd(PzlABB=u2a+%71Ox zmzdn4UGci_;D{CoLd)1rxYsiHFsU6!fEXEXQr{E!2{=anEV~9V>NV1|Z#;1<16`j! z7zt3{>keCNsP#;;14U2weyEG8_07_^=iNlQV@`L-d9!!vYJHvB;luPj zokKW!y?3u7Uy9z@;o+FQo;lz)+2Krjg7ws49O)ITnwOzH?f~HCZg2&3uWbmg#bqhD6+d)c&XoV%Y2m9VVULsr+bI2!B?RpTnxi+qZAY zUF*Gg)sx|JIl=q?n8H6KzTh`C{Q?+R70y?myby3|VC4G7Y}s`H^3n#ZIpY0&ngRNr zov6lrz>8*T0mf9_wOAYv5Vum&&Vd!ohNOC`bzpNx$6P;3f$4_j?Bqj;8RJJ!lGk;v z{Wl>K=bfIbvJjdx0Yr3049vqT_?vP=42+h+0Ne-b{m0&GxdU!M*?adj!~0EYJ+Y1u z6IW%47EwOg8oOMGSRtJ(8huLm~uX*c@EZs(Tb4c?>RNYiK6e9KEu)Tc(gsv4{H*^+g)n8HKQsu87@_&jp)h(X)+A13x2ij%Y1*=X?1`bYhXf_r->%;V z`c}LD;7EVaY}92U;5^e42YC9x+=aCQ#Y<7yVkH*f(6wzY5Jpl$rY-|pm`%N-%Rek*9^NEapAidAUOmBIw{L(WN2`DEQ zY!|r~$Q{N9gf(AJg_s?)(_*U^`^8Q?LB{2Q+Kv6khiE9B!7F9x*G~IJV{Fah!VR06 z+~~mF(cV)7gZHR)Pp2*KVJYrfw;}B&S17vCY^LSKTa?To(i>|8F}C8jE*12&M+>y- zoBcdzRjS)vg3idFjhoLDKE01@KFcO z6HZ3S=k(fBS%B|`t#u#BKWA{RY)O8ZBff)O)*fJ}p$F${q313)(JP z!uMnUQDt)X+?fC`6z+vB(2}Zi36TqlEpc25d=jZDO_fw8ENrLr!^V z+va>xmFzh2my_e@RGexUK0bHph>1DNEDe>M->K&BzAzh@vhB7PAEldHUZF~RKet7B-3s*uf0&TcFCu13gLSt&M2YkEojA$=-;Sz4V=$8!ji}9uK%nvh^CI zyg^6PmFWc5=FD&O<K5nYj@CtD`%jqKbng_a{^=+UtC+4a#lPP^gaefT*yQrgzrqn;{ z3E7+I{}yvxP*Bt1|75tI9$J>L?lg{S?gQhWZo94g5N-sT%hc~84s=oU6)=SAFJ8JN zkV$k91;gfWpMdS0_c$ql2s(Dng!SS1V}L%k$BBFOw-#9J7PVy+*~HENYLMV)Gya%V zWNKrdl=tHAz5TP*XU;uz9O5mMHw_ZBzNUnROb+S8SeS2*=db?N4wK+Cf6#b>2G4Y+Ut!G?Np{ z&f}0gPp~D)wgvg(Bt107IdEwK=jMp=w=t1A!RdV=Wp|dHGd@=uH!q*;D#oW6B(GJX zExb#5>`SSh@k)^S&f0R^kqN5=D?SfC^XK?-e_}vabpkH!F&M3^gkJF|dQ*YpQSJ6IHW`A_E@`7x@Qrj4T?JT105UB+WlmC#w@9I*7;O~?xua*Q|A?vUlRi< zwnOb-f2Wk%DH!{%fpH*%skpqyVU?NwWR9fK30y zq0&(l=w%V4=GQFh2rYabJ~&`o_@dQtP$?9vU->m%|MMh^zlm+2)Jm%FaQRDMeuX4g z{hCr_5+I8#Fzj@E+nyEq?d#wdvX(E6Bm(9pem?gTMtlB@p&S0$3+L;_akPT_ z1>Yq4Htxs9(;E!3PP@}+`@R-u)XW3HMxyY59(QZeH=bpilnSC3*MxpmV;QYp@wBF% z%M(-viqbAme1#kWoF_v)qsT;cIVZn4?$K1zV^sAku&Q4FnkFrk1KCo&xodd-8-!W+ zcoI#RBy5zz(K@MBdOWfbk(H44>EAki5i7-W{LCvBe#TD}^Nw#xCH>nO8tpcsry=2^ z>aYuoNupX9X6bi^m-$S&O;viJI`8mW@QX(hv&Y6O=)N0qwW#YWXo>B|#&YG8y_5Ba z)G%j<+#1JRAsv^gU7i#*@sOU!8qfc|kH*#jyO5<0OK-ihyeur~niIlKbueF>KzH^L zY6_NalJ`hJz`0ub&Q80q%F-FQ4``Lm+Up?++@$e_o#FM803N3!LFNR`Nxg@Zn z&}1(8M~M$JK55CBl>(5o%zOUL3K+z6KB`>pc8C*vRQ+BHOHo;T4)euOgQ%TKE}D#1DR@|mmfc}ts~Uvo~xv@ zESfjoFuV+n5oKXxdoC(UDI((eR+XbGER;&XIpE=g-M255NI>A)w)^ipMGgXpZL&s9sOI%Gzd79G-#SbY0%dM(mv)soSWtzta=gMBg>+Fq~CO;%#r z+o&6Pz+G!E&m!x~e+4MxnK?`DsLjSZ{1&4n4FUp&wu)hjFhs%1k!&V!4P~Vrt$tmo z&txjfj|&j?BL1{G|NI_eOdc`$W-TyNxci_XP!rQlF~QPC0f%;7cQ-sP2tTvNj&C8d-JTnRN8^KoetMRbIftjidOyX{!O zEGMd0*6Rv?wP#d@`#eziGq~nfYeh7IonLdAh^j8JE8&Jm4gs1Cc{`UPffG6mUgdARkAEFuEG(RCPn^WL-uUBbIm z)Zuy!6mw{hkf&?DI+ASCCCBG+ z5K~(MUyhZOId`-|`!94neyoDD}qs>FOp=_goDbt?|DzkR9O$5`m_pSCKK z&JnC}D;v$D5)T{Md9RI1QM;Mb0A^*?vJ0TU;yPaZ{s3Wgkjamk3Xg@C^r<6OeG3N% zyMMhf)shSs8})&EbV?2*WJ^v^ACM7W=RX(%l6<5zf-Dl@)>=liQ<~&-`o;$zDvfsx zK2_I2w^C9K8uJd0v0%lSC-!Yw@0i6c4+H$c$Iroxp3bTlJHf7k`yBjqQ_l6JjDf>^ z7fTKO9Mk7i_~bhQ(jPN?3M zB`lnFt%xiL)$Hnt{`-E!va6Z#dx>hLcnme$7*uI?d=TWdpSs&PalXVyclLW|pVpx# zE6R9?#qm$jO6b@)2L-}!$`G@+>^oIt!P+@B2(gBitOG6{t!ep5k0y;}yEAgcneToR ziSeC;sCqqhSt7y-Q(v%%m5~&#LG#-^zwA*cY~k*IIV_WePi0<2OXG zepz^xee>D~W~$^(so)EPc?U2@@=O~x=L895U32wwk&wgO&SUxZsPHRf>A zmp9C9#E~J8wntO@q(Zi$eZHC&Z82RS8YtUrahAVwo3xOz7FvEn{7qAta71Ai)O)3M z1^J9XJc~p3hO3MEpDx>M4rz0mh*tdwgPTD1+=aH0^K#(eV_XB_$C3_oc;K6$E!H2L zXImr`Mt=$T`-F%7j7C| z0d5zA&w_^h_O!(}8Q(bV3zNR&)Ysf>42W*cjI*`<*Xs@$215w^=;^E|UQ(Q9_&v8> z^7L{khFXPH*iTi*02+8`0}fhBXQ4)i0jUnKWH-6kQ%2xFT37kLmTQ2)Z}vZ-e@CbC zKuzm#Z`iu3Fa0_Wp&E|)dfot8sVD6c6cd5+q9PPc;6?f6wTFx`C(hJRgLBusIX@t# z{eDSf>N%yZ>~0pC-7h95;seMvJC16X&txzu>f^|iWX54V}y12g-`5izmZFi#W6f{rs9Pj$ZzJVFOK^CKYA#ZIxJ*uX|8-Pz0XXYz`#8=%=Dzp;MH z9wt6#0#g?Sj9xeiel%*06mHLy1L00tCTZ~|kynPyz8@}R;>w;@Vw>O<9c47H`0>3? z1p}i)VW#HKRKY>ZH!vw_8gTrR@L#&P*IOp8dS zDozi}5rrg@ZZ)M4avjCY1%Fkac3)|2YP@LtJ4pVim2y%nBuqO3%56V8#XhR?t{BE| z4#FHd69Q{M{>gnFpzkaqwu(&GuH@1-f{pc)@`$@R-(n=Lov@Y8Rd# zz6}}-!}HaA-1&ac6xWp4>7iuBF?Za0b)~pAyTsG3uw%-Kjg-f+Uw-Au*fWkR2v@cJ z>g1Ahs@Yzt2lc~eBbsFpH*sN`6Bpk~LVZvNfei=Ls8dMII=Zup<41h5snC z>9BeP&;;UaIe;^;GfFIR@LS*j`w=y!q^)aN3qiW2nJOnd*&sD}r}`N0iyxwLUXjpD zfwD#&+*y%%SVBljMdHl+`mM@}#OGzX>;7BQ16{pb zh}}Gwi6n4xP;(&C0D`F<$Oyw)ne0kQ`)cSN{A7n>(6+pRa1VwV1gKghuC3GK`~oS8 z^)3>LUG zmsEW-Q%q#GrAZWn3{?rG4s6FZR=v4q7_$DV+<4|Db-@#u%ok2kjsj5aMQ1^*7C zKRqIB#$Q0T*0!jo#>~>BJvy=p!8+{`$ll4&tt3Xy+wB-g z;8yCJZgsl@NsQp0G3IpaF5+*sJQQUeT-(G(rebc_?mDr`@-o@ro=qpg^do zza=X`|D|{%RD^S;6Ea)R@|40jR|#z~FZ6nA|E+JTR;|z9Kg-UU^$@HT-8@&Z-2NlX zzp)6!Mou?GwNnT>Sm`}8^@waHWsMIpe8k|mwIeg2-|mqOvkuEi4rfAo>YKLluNt<`=@e&BYY!T8?8zEU_-H`wyY%*>N9k6daZwe&;xz84K7|eT%#_T;CqT964)qr`c3Ujg5GS= zaU!HU%3!v{KdyF|jM3i2yse;EJ^Z(o3FAee)fg;j&&D6TE!3YB0w?I*#O9ER z80N&4c2r14Y*g-i6O`3u+Q9o3dr*v-*S_x}Ahb-XEt{YZyMD z{*-$vN^$C^MC*llHc6du0`hL7fb6AIv!|YJ|#e% zT5P_^obA2I6m$);n$*ZRRn!)%TX8+g7?a_z@qq4B)hPmX@w$V@Syd7Bi=-=8S|p-G zVfAFP2dbErzwO%$8r#l=`okPGSL;BhSLldQ7q;>b{BmQ?&%~-6nZr!Ht%mDKy$Baa zdD+BdDV(=OXWy#Yszm7%?p!gte+WwX4LpCEh(KqcQUPbLsH{sBG+>#+L~Oy&M11Fo zNxe@U%=|!~JgY+Co|PlTaL@_Pe%c~9keo$FlLLszjnselZgB$vX`w7xa?KrWvGEzv z&s@x`0|Ioel)u-i86El#8=EwEe1tI^H9~z>M-hL!9z&J_m1cYW;~Ha*n80*7;)D7R z{J&GSNsQ}Pnd+$&BK%MScr(YJUmg)QD{I(}mDj9?Gf*Yv+lLqF6O^+H=M&p?Qc4)! zU}@G&VhB+C_rvVZnGO%*1qXdCs$p~B}v9hCr__n4-VUt~u{uE)Pvq`l{7e&d_G+r|T zG(Ggr-e!b?ITv)h*PQD#2Bn0r1;=hXPBXg4%(?7Cy|$X$arv3I&|jVjB?dOuAot=D zS(6oLCESWgX@wn5>JPnqMF7VLmOT2*uuXX=-2|WdJ-aR1g0|07&Yib-(lg(IoAT!+ ziVx1y+$?yNr$4Rn)rHD4B0}Bm&>8Tp9~-(~E2pvT{_KUFNu`%pTiD}#Bu#)>Vi#+C zk^(cW8z|`zn7<`0GRxTJ_<7Ff?*dZ9dVv5|9Cg1nBT_snW+o_ZB)8sNE^0Y+xcpn0 zsAz*(|ED6)rs5E&T~plYJWLDTx;+P>V>%g*-Ksd3^^1h@0&iP1o#Ezg3)_@AyP$cg zTBaQ!?x-=cIOFGel<|D6hFDPBpWqlPgH)SKo z=+}Wm`jcUbM{~UJhe0bMPgiAv6!v$Dx^A~&*p5zZeK|d-GBPu8y^o2Qev39{n&QKO zv*_=Rq1g0gY_72D1<(z=I3zz*9yLA)`HS$xshH^d@dbl%EjAunF!FnCj2>gJytZIX zY&usRH?zkOsG=(g*Sd%(ABRAZjp;SMuT>Y}K*XH=Fuv-SZk>;h07~O2-b|R|iZnPw z;Ackf5xhho&Uhm8dX?ksJCjzwP+p7K{lSBU^Cv~bi6G~;zhQvUsc65Fe)cAV@^JA9 zuta47K}F($IwrYaSigoW&00P;*YVSdM$@aaQuJ~i31p+739{5X+)qIBCz-#M?C5AM zH^0p4xO^|IT7g!mimh(DlPP#uwZw2@`JC5NGCx;j6d<@5@N%sHV(|DW^!E+`-_dmm z1SI8@pNb=LRW>U(nH5ezE5tIsv*#AKqR&gFMMRno3htdb{GZ4=Vg!^%$`YdG1tX?GX4JwK2sz z)d>11GD@FuI8oG@vSVP9I$Vq5PMNJhfmJY$7#Lxyg0w%$JA)06H!$8|mA3#y@ox>o z%a>AT3VbV}bLbdlIS_U#>rf!3B>A|U_yuO*T{AvEMwE7Av$Vef^eB?F^9^@7FCmpH zwGHfZ_LmKBL7}3Vj`=;SeZ6oNy98kXQvq7tY@f{*EiWXMQiaiAu6kgVc={#oioxuWjdBiY0t^dwNMTp{M~=@kO6TcllB!&%LRh0_k!yiqoMv(al1? zQKgBLl(^!vsU)<)84Kmjk6ZW$toZnuppr9F3*Hc7M2Eo|G;+xBgI`^8?Y?ULdI=-`!w^ zGlZx(?jM6T2!#ImS&~tQzs3+>ZzjaTP zsO7PHL`fCBgFBYinpN!tIu7Z{$F3Bb3l0#{AuK!A_s2& zAJ%L1_tg~%TqRu#JdNnju|!P+Qrz4V6*awUkl35FSTa6Y976 zcAUlH!k@`qoGde_JmyCaB1Al2jxkDR!7?+q zH<@$P6*;nryX}ylIunCnbohu{&`bfwWQax0Ta0jG$6yJ1#fHDLZ^;3_DPJza=Ma%U z8M0MLtpe-PM1LIk0DRH?V#Iq6_$?qiJY`&)w)rnRIEeeKZ&HtBcsL)+sI+1%nf+5z zr&2vl^wDXy8t5|}b@*2APU?*w|Jd2?YMO>7Ga;{BZGTl5#$vFBrq!EVdRM^ih$Sy* z@UsO=*FoqOTdIx|6M(0Mkjn1OiTYf4qpMCvx%oqPe(6lds$laykW{<1CnM0@BaYC~ zI2k&^X>ETv&LW!$z8Te^(0^*@acXzpsYtJ zN|F>_@JjV&W7K50W+ug5Hj*Q3!b}I(q(NUajm|km+`FjffqM8naryC%3WQc|=beg) z=hH&Ti-5>a6Cf3ln))-JsB0A2SVJry$$k!$K)`6L7k~78`b~O2V4DTX-}=JgcU~bI ziS-XfH?8B1j@~7+BjUGy-t-ngI-}A;8I*~$3|HRUCla8@|5|O@ z2aGL}VeFLcPR_e0LYK7&VPj3B);4n~I56E#XTp!JX`80i9~t3=Pm?izSM<_Fx`yJt zLG2mQ^#s$ z&`Sa)O%;aUf0VAA`m`{N#?h=WqvBzeL*^H5QR%{aP`kmC6oZ#a1$o1OX9qd61Gddq z)Dg*&DPXO*->lC$Q~rnvaG7i}^Ib{qw@>wgX=@rYGXTXU5rhn)MHPAgWdsONwTOrp z0#8*f5T!3OFi^tYt*&7~bCT^4_T1@m!l^01j)M`D!LRXnXXFpa40q?XRMC zXFfi;NHS^CKqRA=SJgSy;GPN5d(%GXhm%! zk8>)CU=*XmmV81);g$t2vKZJb>ua33cC;_Oc_Hmc_@bW3B%UvS*f->@srOuz0(gwmJV`quE;=mD`gMiQFcR%!A8!L%rMx3~<*uN1pNxXGT;+FLCD?>V844S z6fsz6alW6bZ2n&iav4*y-*iciJrA2&8-JukAD>kE)`&14e8IA3#KMF&IOfo1=TaRf zYWe~R(X$|cL+ zm?<0+(b6{Bbd}-Gi1RbeO%8u)(xLUToQ6;gmsxfnCG~H@}0@Pq|})J4pLM(TAYm($2=EsFib$I^0RJ)A}ZO6 z^S8oF{vXcXJRa({4Ifr_T9mYi5GuREovk9Hl6~J2V%&DaP}T-RsVMut?_(FT7lyQu zCCk`nLS-KfS!O0PmiPMV?s@L#_dK89=Y8M)vSj8v*Lt4Ec^t>dAO4z69k04&G^ew> z2;zf#GLrq(L?7b?D5*sYtNV|fwxWLa(1jkqGuPQPX3fOul^;0iES+{n7pn=)AF(D* z^Xa7VMk;0DmJ(q`UttNmwB5&{t{C;-Q=0tJz7D(G@i@+DdBTuJZ!`Dkaz^}>#%%h) zXG7snzj~)$N}`JOXPJe4=FR6|j?GRR{(f6edIbi?ndvhjNb>s|PIT7>oz14pZ9NaxA0rQ?$K*I|X{H||j<&Qq@qImA?X@-H~t zXLy*QjjvbVEvoOvcvtFNovW6Q$X(#b;kS@2^Fe6`1WWk#`9_XaD+SHyFr`k)&+waL z0r`*?+sXos@ztMb*;$s!s#)!fwZlgSlV%I-8)rW{#2(iW3fwsAS7}0OiFe6cv(cX3 z_$zGLi6ezy$?1KP=BalZ{m+DdVytXpuD4i@#`ky8&OjMZmS#vPNS*@E=a_fCVS)46vyzzd{C5}grK zcZ`C8!FLav_}x2}CqjsZb^yG)Ial6kxQ`r>OXaMspdn(MzX9{o*nS)3a=z^iF2dYU zb4gTgK@~;9x=LgR0sj@yNlU7?xyJ;|F3Bi^bhUKes}}R%9&w>;AGqM}2WTh$y5RY^ zH=ks52^jK@Z}a5uxc#_Vmw$DKJw@cg0YT*OalSa5LEq*I2LK*F)gJh>z|!Xh2$mH( zJbk2=Vbx8nZ@zC3+G?9r7c_lFz1m?8+U9G4t(l*#=2Ry+=2SCyWf$f^PNOM+MVX<_ zNpt>p%ujL>7I7O|E!@gKKbP@F4{DH@k0g!%48ym;Sogxa+EHiXemnHiT6HiT-J?R;PY;~VbUJd%c zq-H-ewbgzlnK4Iw-}cxo*4fjSzHkprSV+%&T}^eVOQ4{9hx3O$)_IWl3O=t-$yid` z{R8-1vSH`dvYyqi=r|vt4n0mUya{rVhM$c4!vd^xr1vNN-a0Km6HxoVaDe=zC#ukX ztM*ZERPjvutknXan0@Rz-a&&s+*g-%q@t+eyjAK-cLh6niEE{;^dPkf17phBOYgjaD& z_vpDw!7ihFgO75XkTk?pAzSFXf4P6P8w%g3KvJ~ajwl;5+vpnW9;zVJ+E5ny zsBdkH=aLm(EQcR1Lzy>A*v6IrM0I)k9BAq)m|P*O8a(7~@~6JH#Wxx@`xjaSd^uTn z-(u#CB_@kkmw!&R=X&c;1#qs$6>OC>9pgO2h-OuD-{ZtMvToYt9Fq*(ZMpq_w0eW* z7s#@~#jx&kZ&P+e3%ss!49q`?7mSLJ-20MgeZkGe)F?|s$*x$287aYMwL0A)8?gCv z-rtdzah7%TcC?r*Gvh^LHgH7-$L&q*u{~T?!Ov~tRqX#B0CakAUYP%b6Go9@VHA%& z6b-xMh(;a--_fXmbv`<1X5f@X8!Y_pwnK3v9fto5LeE@)S4sq|gi$Ej6EqvObffEiKUuioB3{+*0cFeq3EaQcGj9@%<8b;Q zExI&vUY&mB@h7y7Tb$~WCB{QYmYM89rj9!2>hbb|%R9&am0yI^U)Fh)zS^sliN)z9 z86y4CLeWKV@MCsHQiJPEc)6tPwO09_B}?-sMH2ZboY`n2-iF~l4A)8XFbnnHJLAFI z9*3Ubd`eU=PmGqI5ZUqv0;O%9T8FHK4fJa79ayMyJ|gn0CF+xzRBrmzunVDKtWKtO zbMJ$#rViQ7WLa@O7!6NrcN`wu_@YMWxE+7yqs*v0pJhW)f)Al}j_SOy6?? zFQ|#P^veFS<6MDd%U~g4zNB~No^^3CWj$@esc}J;SzRzP&zlLX6BD7%kRy+G1FgXk zU_+ODTD;P(KcMsRszLEZCKnzMGwo(Sz(PCj`Xcb)M?x;}aRKKoHBF;F&>8TkH zzCBmB!cW%6-sL2gGT|ES7D%g5daNE8rmaNI@qP9>_)4+1C$m1bZB~iyG+odgmMmQp zHCU+2n-1ARiM5USgSp$UW7+MkT8FoaiVSllOb;a|N_Jx5$Aq=C} z(b6&{u(mpL>xRe|K`=VVqr8OGX{c+TaOO`?CY7$v@6;CLkcN1G-+nc4Bvi@&3RaZz z5(;5*h=Bp2Ix?y83yz5V8ph)yFn{_eua_4p9=B9S{h`^ZNs#pT^vwNU!-@+03i%AS zk(`FAL|)j^l-lViMY?8Ph~4Q$ub<%Z|I!$DT;rtp&*w;Iamvws^9`T*sKN!v$2X`zkFO$*kZ$0zb~cCROt`rC*m=WQiKnkP<_r`ALT zB{G$ZTxW+uZz)|sN+FaUT@PD6&Zlx~PKhq;SX>429^`5f5Bcj}G}B$;P7YGEW9a#6 z4gW6q|I)5dntIwsa@_I1eHy{+F0WPu87eI6-{!vLj zM+PuNii#GB32CCOp+tHC%YoJft-KS$ED)3DshsK{OM8xGD~5M{MuSBgx2UFtZI~)S z8cSRnky0m}<0}FgFd*8Uw>XT@RwZ26AufyRYVr0{dsGkEKerDOAM^{3Sa0&QeBt3g=VZX52kP(GO}!%yrD(M1WJB zRM~ijgy%fWS3$pqzYMAC;$xIgNYb#=4Bsmngsod|-mwOdSgWA&zZ{Ldubwq*A$|c(2q*UBTcV zQHJVou}UI!$PCtl;`qs-x{@eR**O7@kin4Qxfs31d0+mU1fy6zj`gDgDM!!OSjZe2 zDgAsO9;yi0n4<^%wQd6q3W7{Zgc9T!?CV?%c;0#ygbftTb`5i?! zgDzGd#T`E`_aAhZ`E!kefvTCSZgnW-E_U?8V>EZa@%~6iRl zfj@;l$<{W}_i(b4_1OnQ1#E>seBsi#RCJMv|LVJQY?FGS${^noe#Q2fGC&u$-Ot() z5)>{Pb{NbjdyBSM;dVLRdWAW^=^DT;I_R;sOLyHn z1iXh$9R!xeMOU4D$tM%QChXU3=k~b^24*lI_1oh3azMSpdY@8R_nsWI*Tnp_?Joxu zlfykrPB{ply1M+NI|#tfXd{f6b+3E~Q4J*$M0}-YWo{RplXsj;Sr5btl_SRgQH8mm zcYfr3O%Chh&TmUHl|^V{UP*db_Li4eC}~YJtTSizJ>__9`>Xp?CzOiS6g{B0uyi|r z4N$ESLL@@oDX1L9V2ktVcdqN2@)PnMIY}pUb+sjp)clDimwJ2cky78YqFS)k8Z6e- zu5vNp#X1${_g7CXNC@~AT}4H#CVbkS=1bg>Z$2g#u&`t+jq6iV?8{a17F*zTSqZ~- znlso+1-wa5qzZ5ONzlhfmMbsA;YOANH@h|}O+z z^yPXM^t@7sQJ|CUl_E=HAzu1u`w2|x?LXu3Al2X`M}Xk^)a%UG9_&g^2bL@i^U#+7 z?dF?0UDoNtOU(^8?I%Z<4g(&}er-Z3>3~vCt%dJ$Ot=wkDZAPQJrMcuP4rVA0TpfG z>lWJhQhssVK)l@-U5wt+5IV%+d z{e|z^u;B}orvH~e8V(Rr>&M6-0Tc>ciL}kb$*96?%w?Aj9z1F~TKZ ztmaavzG4**hWawCjn`U>y(2oGdGtyul+W~aVlon>5Py-ZqCO_PtkCw8)H@PtcNi}%gg z9qq2Uv1jnWb0~*cH(_Q4PZPfxhd>)cU!K3Txspq{L@_=hTiaYzK}@`@sDjVhs4XgH zO$m2UKs$*#Jeyxv>{O;8Pw%uf?Tz(~|1_M~>nAvi`*iZC*p|XsQ#OZ-U}UXh0|{5! zT}D;SkMK|G#adomi1?#B)okM2o(T0{I@8F+uwKdQp26tYGvd@?N1U|05~Xh47$Yi_Spj^PczM z3=Dm>S8Tpn)M+kf2^QewvqpQD(x@5>VVx?Cft@1i42{Tp4OQzp9E@*FkY>^I!_Ql4 zcx(bv_q~<*G!ikzB8p^J#GlvfTzo&%D`aOTySb7`736Mkw&w4+!|DH~F>WX7>QtKS z;zYH}TBkLflH>Q|R@CubqQT@QI1@-COLh9Xy;Ws(6H95n?t>d&^F&C#EvU${v8F3m zd!|oG5RfOM)Ovt?kUD)5*4U)E_xv!l}f1yAOS7H~ef^aaTy z-wh9WQ}a%0X6>|FCi@|)K1o%J>r%7}O|`}5+R^HqvNKnzlR5=syYs$gNvD~a)e-G% zYz^kJ|9P^6?Xwz^kPFMnlSKG79u|Lfc4HG@=VMS8J)wcYLn9^wk90FMvA*Og2v;Y3 zDwO6d=AZgV_Ml>Ep%=VnW486FL~KaToMtYz9ZH1%Utg$5=?elA9T&EEC}$imnG z*pEkF{1QtZ{cor!V@yv&(G3Ei{d)y}u_({7+nDrLMJf1ru8QYd(3avhx8?`vcoZL{ zcCo!SK(B`{B0KuV`nS)EE?dLK9hZW|n=YFcc#TBKzp;dGKKm5l93OEPPbjG1u{XTk zZ0(~h*Hf;pRd}xte{bh-({NQ2sm`pV?BA|CqN*%sEjI#qjK}*1=Ph z!kvICDHI*P1f6YM1|>wHqPCQ_^R_%qkKlMNMXz4_+Gn0jZxxg@F7vtl3)Cl&t=IVp z?#NxTm8q&QS@4hU@cmGkfLN?T@_v zqCBWmn=G2PK3z?FPPY`}Qwxq(xxDdF@5<^&J`!k@7Fe|m>Na|o13`Eb;I+FPIC)fY z^T#R<|G$C9U&sBwr{OZ6iFUn2(}P&j>ZmRhRF0MFE;$Es)K*wz`RbJyut_=fPA?l{ z0#o;WNODQGq^%CPL}5|4SGV)b1<%;0r4EmMVe~QU?f2uq;fc;{a=*TMHQrTb;f{pF ztD!paROc{w(}1j4iZQM2)DGjJHx5f&T_g$mHP@EyO_Dc|8dOc9m+cp?&RaEl`y+!5 zdM`B+`kwj_V##iT!Lc;bDNKON$wX>>0n$9yHW(D)l3CEjM5GB)+U~NoyvN!1(e4+8qL;CAGt-8KM2t2(kzxi zChaApep`A}TuAVmqFg`@oVWsS_$yexd$J|cz6~s)62_c=im9KuNBcr)_csbqX_e1 zv2bj&_9S$yeh*&ZL=V!iT9Qs~X98bh8|DL?5fiam=((O8Gf+@DIH#JF^Iu13LrG3v z!^V+6|B+l*!t zIuf%pdB2@|S5Lv^)}(LAS)dxl1I2HnT@o#KEl(T>HZTV)C`|LCoSHvsB|yR8GqymD zZVieA@BfwjF&|*D$au_bZbCrE>8b^dEy$MgGd5(Uqn;yO&D^vJb@*6jDHrnW)4F`r zW9IZg&9v*U4kuoC+3Y`crg16)quFJ4U~o-?@3M*n$y|KTb$SjpXk5W~`Esb=WD+H= zbxO0X#7)nMi(1aVg>_i1aB*I!U`FO;4{Zn<_fg0xlnDpT72KV(S(j$>V z0nBEwSi|K@**EESVQZLD{iw6Nzb0TRC{cOuOCr0)m|p0Oty`0&VKsns=frx|;s5sF z|3T)u*7PSe_1PP*u+Q^Mmucq}VwGA+i*4K81usrVH*u*LK6EOPM{>zT+;Pg{YU7R$ zci%h8aSi@uvq&&St)jbS99%S&kJj93CkA>eQ=Bd9U%N%rnhR~>{CW*wAlzjHi}U8A zJTddSc*UWf&5gAfxd<)996z)ctxR?=uw)vvw|*ob{qJZNJt1ODWTgO`mYD$&Nm(bv4c zkFFLzur&h&AR;DIij`D~;9>+BueXT1f+0sHXrpP~^x!^9{Y;52AbHScMXHuhsLA&kZ&!hl}O7vtAnYj7t9Vub@xqSF814fT9Ij%aZl; zE&_^H3fF&eKv+_FO0g7N>g>yM3Ox29L8Hk5ahA+cyo6EHLf^w zx$dr?hv|M$ZSpjlw-FVY8p^>uPi z%`WNtk`pfPRwWV>kW=-ZVm$Qq=gL$W=l~1RqvqujcB|QRh;s$27}S-P%vK`+xIsd> zhl4y+m=(~V$B8#7FpAkOr=13+2+u|}xw`UmB_j|M8Lf>xW5`zDi=|FD^QnZt1dxC2 z7*y~f8j?B9g+P&va-3v7ZOQj~o5v4%TOsnyRh%z%DPx+i00|IlB={AEq6cWWvYM{^ z&fjIOZAYs!zXRAvDVBZ=O| z%5;y16o;|kl>9ZDorWx49mz=FL|U!Nh2lW!>2VsjX6>^zjC1zfJ?!RAUYt%9Aa#sI z$}e4!UwMC$6mO`b4jajZm{yV3R@LaA=U@(??DcaORD{y+-OTN4 zxMsk;30mXotmd>_^2$X+?L(E5l_>xdED}??nWvaYqZwAxuSyP9z}Q+(cr)6y6qS9eo=Yt>!h>wfPNa7qEZEEjg6$1O}l$(O$Kt{73dlz16|^c zFMbhvJfPrvH_88-5o|JdV*pv(?q^-jQ|)67WUdNU6Q4Q;Ep>Ez8>A267BeJr(C-i^ z#qMsIiUg0NecLr78~TB-*QUtsJvr66PTsD7_t&^hTZGaxO%*os_c5uByQT3x%Ip@l zjikCXh?Dj*&9y<~l?4xZzW-f$(`qNbZVq{9y3>8g*aVQ!hK5Yw8F?O`WJdOl%*Ax( zN|$3P_%Ae#hOfa~v0ET4z9v7H?h(7`XzEBn7;pgk=QL3%PzeYnkQnb6Yl4Ya?siTC zb6JJpP);X+NgPI@vJn`;yY#k1WdqYtu$!I-6)Mw8Xl*k|a2^Y`Aq_ zm?$0)tUx&u{vBNX$k-?#l`sv>Yytc(=W2lL>cqg{0OO^NrZ{9e&1D4!fQfMjK;VcJ z36LMDNm<@<>En~i>fKKY~TM^CFS>-5RXty?ky=zSsMK&s^8n?G1;(%hP_ zrs(J{KRJjr*Y*tzwJfTjdOaC7iR-KaD$Z#?6+j#6xHsS@yDlCA&I^bt>YJnlYQx3b zHCYA8xIvdrRTrRY5#L=+pczqZ1YUVr9JixHFp8r+B^qtx39*& za^x5o|3=~6s;#t9Az~`I%8_@d9c=6rEv14*gkvMZKu!)=qBuc2qi>faDjJ3Bqo?m4 zp{3ve6T{KHq{UJT9F1Vsc$E?$A@HdGh#lG8LZ+W8i~EHuC%iiNaLoLkHQ zV6QFh902SnA83F4miX-w-aIN@yF#Ze#8V5z$Gs!YM;zm*U0-pJ+~duY&fBN|@vZ@b z$)n^utZ%k9dIMd3KHpabw;z+eh~8QxLCgba+AH!OWvpT~p&+`I|yYUih zw&5{pHmJ(+hCM72UITlwH`CxE0zRX(r|S-@i_sUu)-Qmt(5VT$P?#dwi&?-ikrG-H zK??@_Og6G{N|v&Ic(ximp|$MIt8KO$NWOuW2TKey{3?r%^up*a)>^+QOk_h2cICd6 z4AhA*Vjdu5qL1=id)4Q4s|Xop*_)0s+c#8LuB=RJT1p|VRibZE4^qYJ&u_ipp>l$n z1Mc^b{O6Hj^E3BAuF6r;5s~Q0#R!|YH?ZrfeLv>f^QNvSHk@VJ76#O!C;g5r92$W7 z=?4wSUXeF!&Tj$WuX5^8_#wuhtUoWFm^XwF^&7AmIJtzqj=dvfuZO5+%$MTQsPi{q z392Nhh)gJggL84;{n`qZ0als#yAQFBa53vE2gwrX3USVdfYa5(5n*sf24R#V{iHh^ zP|b-qFfyV0MxNr80e-J_ruP0tqPb05xJWt+AOJpt=ti#kkG0Vlpa>$BHk(bS7I~Op znnM9_(o%_@NU>3l5QD@fKzWKcnR!m)!|N7Q9RUQMOo*<-MXtyl8Hzl=^_}@Yh(>@( z+|y1gEB#)sF2rx>kWZ+vsnv}wTMoVVIyqzi_3zSNxzHAZh3MqW~>e(?>SG220d7fSYiZBaK@mNrNHI9EXO- znFSKe1wX7W4Akpx&8Xmtc)T+D$T!IrhfiyW&HEcG4s8=bR5nyeG&gJ>!cB~i>)~Lj zW;$L%Q-*U5jXa{<+Q_12GtG_7wfe-hHdSK1W1QufQ!GusQY--UOp(HZxfk;urM zy?^Mja8fs40s3rMtkB0oAI^EZ5mj!lc!=Um>6YrPVWL?J*9r{l8B9Q2$?2JSrhB3+ zh%C^8ngH(bS9#3RwN?|?YlDixNXW$7dVls^Eb|75DE^R8dUR7fe9QQwmE7Ws>e%K| zuU^F+|0PLp2PJialV}@M9rgyaWdbAn-Dv+|_n~&s*waBM!!wrq{`=z{hgU7ob9t8YE&FI%N*kS?f`krfR-FZvvKHoNiNf zQpGIQ_4<$bJ69B~91O%h7BUQ+)V=nXg73PT3U?C)@2s9}xW^XC9IcNCr0`xI_lzTf zIKms$JKb1Lbk6I&G`ul0R_#ClS91KtuE%>(8ikFV8WJs=RnVcO6O=B=2{KeRKQLx- z%ekN-rpI}5Al7gO^i6g)Xh+TW1jnv!ftzF$fUTL|3m;YYfL+eV*8#?y`Mo4%~%-D;7jKa*#wad$>I2CkFH75XPa2yWlG3j$9y)ZP5 zA8kbaGh?|#)53#nDU(7uDWr38*)+v>+;6u$)TdClD=mE(%*XxnNzc22f?5bN@cv1+ zF6U^hx47)82x88AR~;C*dkYG@DD$2fyo$^eLWV$3)Y6UGvspn}oMM=>F?ugm`CzPh z9<4$E?J0se%fwp(UJSpU|He#`&}=ZpM6P5xgvZy>FB|K#JkcSXf2~aeY+27U9QRU* z8apG~CGCw%4Hr;RvBW!jbaOned(qw=mf+7S&H2Tg>~Rz>mMGRScDTuR3JQ4gV7A_O zy~pf2c0Pa(-?8iP%iNYS4UIYc14@rJKp$F#g)BVWOGwGI)i0;FL1#Y0cz0rGkB-=K ztV7CPqL$t9KlUGa>0JKWo?gUHHs;VmeKXqLNTW8XW175Fz~%t5ivxBS7n%h+!bhTX zJjL#f=+`C@?A$YOGCH$OG4&8Ms&RLb>$K8(V82NYqjpd%f6ETG5=kFPXeu zam^E)2JPPo^=<}9EU=$e?vc(ZM4f@d4f;8qNh;4f89D~+$H(+=m4iEg7TaPwonFvr zI4;V&dwF^{c=G^$xBGZPu?h3eW*NEZzUOT)enk_uxCQ@_gMFG3Sm{ei>ld@<7_~cu z!POBaAX#Tb#8)_Ao;2Epl$dfnyy$1TaYJ1cmuAOYy;Ry=> zXmaqb(;s6x9QiqvPYPiFv_50n8v8<`nThB`3_rgozbA4wZPG5u;sm$n zXftK~b6A{@D5-nRX473|$&0;&af9{qeZ%2{j0judj=_6#m!PwiSzw01p@E0n^*>|b zcp$FG^lcF>iVCs^*RI6#BvjsUrk>Gcz!sAePYDp*`AN1^rpBaw=qFZIsZ-3QoY1_ERwAC4~paMq+&W$9*Qz zghgZf!tPh#gcdJ0=XjuDV$d?|bxwEtGgi=z9t;WZpSAzKbr|pwVxPEN0Bq_#{whmg z4@ob(v4}O?1ka+S@QLk(bBgS(J$ZpU)Ii<6)a3wbBWa{%+v~&wVtLVbrW4Mcr?e~b z7z*-=%hm-d)s9`C@1pHFa3;DbPkvFev{_B`htGUYh~RSbyzBhN{94w|rnl0sCQkhWp%h>itOwP-D*%GTn55K^G7<2 zbA#e|_;Ll#??rs(YTg(3iQ=)lKU%(h#=oZ4V-ADLJx$mHYjw4Zi>umsZ`_3W2 zQQ)-l?Zfk&uPS21)a-@<=^$F`vx2ucVU`*7fv|B99;lnH^;s|rkohu}In{+0%u0L* zJ)&t%^#f9mS1}$d;Zi*>LFOYwFn%}gyjtgWaJS~}8CCYH z7M4ApvNn9$?;g=DPNdm4k(J4RM3k{phPF_!5;6+G_iY8-aLILn@A8uyJ(Mrl^$y(t z$qh-P=`bIMTN00kC2RtJzT?Fg!pI`@HwR!=liQT~INHX%sv;t5d%Z9&Tje6HYNNIS zEtYchooXvWJ0j-jpTfzjpWFhZ4&C(Fyc^6)ihk%W+OAtHXxiz8vJ0wo4{z%~lm08<4qSrw^0% zU5X0av~kDw)!bqFJeLpV)=Bz9enIVEt*P)?iO1?_GC0( zqBq-~FGeRsL3c8}F=2fKTUhe1t??o_wr=EVg%7G2<|WoAY(MiCE>B$@O@s+ZsS9S* z^(ukePr0HLxYC{a4L&oBinusUIL6&>PT$vaN*c&(eHwkkCl;!7)2tpOes&Dw!T8jt z32nZ$O!avTkgIzCC&S;Xdtqfq1cXv9KtYMyP;LLSa~$g8_bQj0>4L3iLjs(NMDAGk zvv;_m7tPiGl!z$!@DrER%$jaJJ&%d1n~Ib#-z;9_8_-tCTznkBbtQJqMj6{1#U<6t zGEbXkUKQypaDKXX9q^+sNFYH7D^?jC8*Q4$p6(o28$nc>ts_nu#sW3)lp8%HeodFk?@MZXcu1bqFh4DAl$c0Ve64{2n@J z{AGwet zkuDk6Odnsy@A5X*dKI&pc;D>YeDS5D9Y(244Jyny6@jCHp6|`rWspW_@Ew!Zwq0DC za*#>qZTW}}xIWU(xDmRq7IK3-YwWDo1%1n@!6eJZj3|cLFbjlCka8B`Kt1yYQqSNP ze**Ge=m$oz-Fh10Qhb%(n@Y6|CWq5)i653rmGyRT3Jn0p6=%I8;odVJkk7u(o?n!c zN1JB?t<0P4{bH8=9KfRUU-=Rvs{To(9EPfn_g_#`R*`8vRKQcD&L+Kvp>J!K?N9UQkPU zJ!XOjr^{%Kv69P&HdMaE<&9=mC*rZmTkg0bj;)f{^8O_qRoM0~2k z0Tf12&G!%*?AjGVQku&yxQdLe?BwS4u4lMSss%?%V(RTaGL{7#NF5UMYdFg0tF!k4 z?w$e%%E+x&UPg{nHTpl44K*9iMkbD&&*<_^YJYTqCpp{-H<#?m&uAq>Q{U6xN7Hb( z>))nj9^cg`l+!t7iW$ryfdZ3@{Mz{%?Zg^vPzDrWOsHcv-%kTlv0jn?ye}tU96!eC znN>QeA=nw~hIAlm@>Gd<0t@3`5M~wBhcMZ?bK_C(XqBVl%nj6Yv9Rr70NQ5FZ-pSz ze(}?rB_(XS&O4dOYoD7>1nyfGI1H70$L2%FAtl>Wwm zbtT>z5v(?k2=`E7Y20X25zI5|-Y&ujRLGy7Yb$NtC3E^|4@wnGY~=Gwi||&G&4A(+ zuQY-g^{a?$94T6E&%;#x9GNv6=%XN5wuH6#QfzSdAheKrt=74}-;?`8Q|(E!L!cyr zq2Ks>?^vAJ$&hK5{$L&h-}JsM%}XOpZhJ$#8LOQ&#T^^P?UkswhbkAD?mDyLiVHK% zEUW%dIK0`_vZ?t&-zrs9LYi!Da{ZByHhis_igojA@!O$KP9i){U-!#~pSy5SK;&mG zLz+|qB4MvT(|Ho4tM%jSx+*u6=YHu_*pCrZHMfK4zIf@1=HY>e~y$#wb@kiv{fIHax^Xdu5d z1vF(-gFXz0gFB$eN&)M%gwhfR=SBvicBJ!u+)-$}@ zmIu-JE2-DiNhNZm@VvsJ{S2dH^Ogp?*%ZU|q+5p4BGEgS1J^EhtRo~1@7~)195nsa znIid0>VzKw0*@OP%Nt|aohLy+^0Ya5fjv33nFLhEhNzCCtb=hmc@l3;{g?eFH46m3 zBOLdQ0*0Win;(HtN39Q}6$WZpB98P;e4HPWsJ2U8WXCd12)`__tJQE4Irw7xg?HDeGN<+;C^m1IWBm(z-&W{bpSJe7yNC2`;VluAS<@kkDkJY-P{111E&vT zRf9lMP|2J3-ry%4v-iPs`%!i`AtpF^KKQN;Vv_stWSmImq&}k9w z+KQ!qP}D*XUS^Gt6bZ{i+lsb;YC%78F|yR^o=e~etb`^1IAHkGA#;FAr}+bXC59GJ z)?K-h^9K}|US7aLhFQR#uHmAKN3?X<}PO72aIFnHh_T}I7Vhn=q1-W4SmxP2Ao72-vbHVz2`@c1JE~j9b$0H z7XYa|j-wU00z|4%gMJAx7#jGa6e!8)X+B={0MiPI&p?8*gH?DV;k5qo^;~9}|H2}1 zKI}7l^AE;FK!rl7JF|`s9)4wa)B>s(M4V;(u+&Ut z>9ZQk4Q2ZjDE1l?JRIRF2MSVMS#pXcQ(UgE&MkNZDAMZ#>Gih!d*Ae^?2isoX9BK2 z4;@T=Lt@L-nbeFMWQ~=`#qc{9j-7c-iCej7SgZzPwKX(l6%JQ!H!+RBeG$P@8`d2L z$`LJkut8nx{wwhYadGv;c~m2})!uM))y31i<|nmnZyFP&7@o7Cf!^Rd=fGzm<)e=IDM zER$NG1|#0zEFkG*{+4f9jIPdY!i4Q!I-^Q?Oa`kWK1H@0fuxF49jT)AA1qhh)1-|PvP+E1RP6h_93cQfQ_}tq1 z5u&KoeU3x1`8eyhepTLD1?uO4qgSV+t;S>&zCcX3)j~Gbrh&#->Ajt*ZRv^5gOMs% z^ZOl(dIKocPan&1KWgqn+%ov6=G*Hx5jUvcfA;uw1-(DjM;bus0PDErph!=FWc4!w z_vcd$q}3$e-B@_Pf8$AzDR8A>Mw~jd>ZE8PC8U+CC(IjRD=hM{Fxw8QH3;)615>!8 za{fPm9K9TW3ryiSUl^wb@R_-aX_m8o|LmsIL2Ze6kwXQ z6=84%B1>&=zL$#XfruXm<>H=SVC7Waoiof)yv4g9V*3oJ4@~4sTLz zM%TsL+20`bmWuDz_~e#FX>)EbAUK$;N;*DNxv_^oqgr`FJ%=G{f>{>ucL1ftlNm=o zD_fMmRTCKVutv{H2M|Mq%N{1Lp5VGW5FnW`*P8|Zcu|i7xc!hyr`XFtjc{)kB5jmn z4Ez(6gCZL09aGI2b;E$(KGb&|)MGo1Ak!G-y;C=Pk8v8`{$^rfjzgna-%5fCFgx(dtzunRmW12-vtFedCFagmbQ-Kr zT&QVTHOT8N1up<9S!gXOo{N@Euk;iR8%uOTKd%k8gr@s&AF5;qE}sANv^eZ7YttTC z3o!M5NWf4p$;2OFP5U4b5EHH;!g&^9WF$^i*2jcNZ)NQj{052yIx6Gyn(57h1uaL2 zBbA(Eo}eOdyNq|M;m-2y*ykqGnln8&;Q=z}(C(Tj|7kuXtqmmXIs{IKE2kzVw@H(Nn8cndQ%C#&=qyjFn8wYsM@yoB zDNmf1>G`$kK|@{b;bItUYeAO)pc(nbZ=n}XtO2lQyF-q5tCO<1)>{wYV8OMKeA-M= zxCqj4+T;Z(Vzr=n(Hc=XBxU~M797aO{n%J%Eo#Q(>$sYx5CO5~0_ zOM@=42})gk?IqUq*Ll>FpbGFJhdK5dsCh_vw=%MiK4rH|Zc^;j4m>}lWj_EppoHTf ztDCrDx0^*1qT1NptM0a|mSWmYR6wX})K1gfWexh`1|Np20l^ak!$XZ@ z1W+USm*+QR(v+>=C!EJ_`QRK!#Ofm!|Fe#gO^X)G8r}I20fYyLPo~FD_x*H){RA3_ zw>`i%#Wu8s+BB!o3S?NzAVnW;^jUtpJiGfrA1?jz-wT41IYOnsC?7(sXWt$GtdZ=E zA3&l!zP-Y`(+?>SN4-WY96*zs+2+x+^BH;-SW4=+dzV;;V`&unyNx<u9u z=*s?*RG1trHkAKB&)j{(?8>usdl(2C^`{d4t=qG1+k%dF)B>qKfeyRO2q-p{ z9Kye2QFH;DLtCZy`ZZslf%nf(LYyS;?TtBtr60*IYwNp%)Z|nC1%*31^uQfA$VrA& z-ndl7A!ToMe)kcskN^6-|0FluaIhOHF_s0ED!aT%r)3zqSXZD;`D5QbUyyxdoz}-& z+CQ6kCyf{s22D&Xt?#rRl_sZ8=;zb==}|_#K&2eGdh~*}A$tiNo*MZBXphP>P_ro= zj+F@)OcD+E?|$|o3BU$#-Ys1In~P*6e8a?w065$aR;;-sZJ${HG_Pa*0YrH(<ft|L@t4kK$dWu{2_-ED~R1i*G8YS@Nbh`X=_(c%)X9gQ|}=Aavn#LU<6+ zbsZ}=XcUy=yO!@e>=!G|`5iE1YaKFCf*jgy^yu9OVqnOYkN&sY*b#yG&?}FffT<0H zQ;&hVrWHh7IXR_f`1Pq)Ow9yJ8P#Uq*xU~o{bdOzP!FWLr>1|EulCO2m<#}&^T^=J$n zp!pIA6N%w412j*-&#&H~0Sn*Kh7t?<>|eFt{llG?xL?6gLoMW4K%ZMh@sfU${% z9podmH5a(;AZa`SNwbAi)IVPD03cfc@7BC3=>mL1!@9>&6pM+J>kI%>D#VBOOb2zA zaUS$~|NAGnA|Y(M)7vM9DBuzr1>1uJ+{J5v$^-wxwAy9q^J{e;fruX|Ftl%#CEqha z?bP`1mt8-e~eGNmC;su^$re4+XZ5ZqB-_oBr)GWZ7yCkhM|@{SuxmI2`f z3%^RjYN`Kn?TDp!r%2(QC?16ux>+#w4QT_7nGaWfzg5w{n!Gq(*)R4AlfZD6H}$3) zFh%Gv^7IXfR#{&s4t0q)|Fo6|$2kci)&UaS#Y^`Bemv$Kz1+OQOb5b|EUii^(9&K@ z3DpT6RNNX+91DT1=iHJkv}I7|l*!ZAG^-#+}K}Ci6)X7aa0WU|{exl%^HFemy4Z zGHM0ZdL7v6cwiNC!cb#b7v~qhUY^$xCnSWM7shI;FR}yY0)UfhQU=t{rLJ<~jnrInt|$h2WGKqY^25rxq(t)Sw^j^^DQ#{gJ$>#|zW_0Nd#- zHgdp)CIvZ7_dhn&l_-p5LGo${MaB0v6>Wk)S0h^T60x*@M}slwS5;|7&acAEe+@ zifaqkUXVEtGw8AW9=L47>-TcX4vHVU`$PHOqc1nj_ObsNk9({4R!^gpE%dIXVkNs}(1-_F{BA0fH(-0efFfd%dbQ@{v0OtleA%5_yhvhoI*8%w5e!%qG z_P)WrU{caMr5Tk&14323=rtil$Hw*X%NyW31_rTztVP?GfTQWcK6ofXM{dy&*a|r{ zCbEMa`#z^%JzD+1Kn(W3BN-|@XbdCrDE=Tn7H)Q`TH0c82|MZL*LDR zKF(G#6Ghxpn1-(N92WcxjQ;H4zr=#@w<^sDQH8vy0!LEP^bWq|sd0sCUZ zR=pBF@I1eM&j}vY-8vrVml+r!6*I#d9pHBwY5V)9gV%@W`Gr(Oz{dT5+WYRPCbOF0e5g|?au=kTEmO&T->-y`c^Z=M~-iHn`NwxN$|M(s^05kpX$N8 zEY&-JS75|P^$bdLvnAq>eZRHj0AuL&mU^~Z!jiYu&XXa?t;VxdLz%h)>rN{_2&

2G#-hH6=hq8N96A4W2Lb7khxdboYOZ$Dj9x&;>;IC8KjCtT6 z<9=h55g(bwn-Q)W&|mU9#dq8vY=j<=$`fL24bjtNq&;9^Au-9a!4!|5OV#{cYPs~UwhcM5ph)6%_G1t ztR;h(({hvUR2Ys#E3*ctwUt4TC|tmNm^DcFvY_Mj_Y?jY3Qi*9$%&w)>MPBorw?gq zFN<4G-5npL2tz{9kRCs)7-bL52 zIq5>oHR1S1N@x7Lho)NTx4Z&Jj0|tLKnXw zarZQCf*?^3F~d7%Y5TLS>-|xMHkS@+G5k)0&M03E`;>9j@V&M|!_jL7kfc`yxr`n0 zwY#b>$m~1h=%Z>*jlreVc^IQYf<>EXZ&M`>UyzzqsuzY(6e4AVF};Avsn>eLH6A`=9gB02F;M~FfCeu&wM|EqXc;+@7jyCXXn zc4rt4LIvB{p*MduR%i8x2*t@uc2@g6;uDKG0=#Gzf-f0~5Ck@qQ@W#z2=i2Ux=I?uUsc_(f-xGe4q*(Q^+Obts#P<}Ehux~VEXxfqaNsbVJ%B=rJr*g z{61#a{=b(=))j?=l`6|^FCaK2%_H5M4x$%spA-a9tQq1z@H>Nn!ifi6Y0SV2nfe|= zq5Vd8XupV7>C3~=kA#{)NbB*vpfPXQOt5~LDrFj9z6yofPBvK(&)X6XUAS8s+&`CX z@T|7S#sS-%#wFYoF#(Mv0tIk#$5F1tahaDxEgKjj93rm}E)W8(P9d}A(f zJVbnRrH)%z{z6iA{@hIz*iBiHA{hhRyeCmjnM9cU;baY^st$c*&VIf4Ym!(hA#pYc zyZ?nUdT@PYe!8OfPkae7S?Y_RK@(RV7Os{jiO+ZorwtPgP{gS}3+8@CmiYMyk@0mz zUStm5zW{8t39wL7uvm*HM2#D_#^d>XV|Yfv$} zPkVXMPP;d)%_Y$uNnMg|DKIW1aE?FVEJDy8^4A$gosb5CvgbsQFQn|W?!3|?=AFhw~@SvZ5xD- zXtFz8a=crCQ>b^B>G6{Y@Ud&RZn!2TDLJE`j4Ww5g}+Xp<~}Hxn6Q+Bo}hnnH;p3n z5Tv-ZmTT%KNu*-VxmDy0ue0>}dd$3U(>23tDWyb_W^#!@<*>G%ro|0oXZP{LuZqFG zD>)alMsE#MKXHN{5D!_&3Y>-=wQMkdB=)dB?zP@+!@bT3Im*Yn4DkR7jIVJOe&8ZZ zqo?T{c69tIu^#z({A^molO?ak-mwdU3vh8-u517711c@fSMm zqlxbmqsDbw>7UmOROl%Ila&CYe!XXDSi-0d9<8(??5|f#Z@KPoV836to^}sUgA1{AjlCausc^ijRZtBIv+1hJ$Ojz4CCfKYlcD zun?jgmmu3{o8H9U33D*HWT;&!Q-dF{UI@}!&ei8@%ha!y3aF59b=%u;VyU4}aQ$bL z=iD%n-Z?*^iqdr-<=1waX~xEOOT{mxy~GZ+V22%(pzflOBrCVH`9eZnlldOEs)lQ{ zEW4HzGVp*_^KV#9BE6)&kp81ADp>|o&;H;na!6w9_PFnbjlC#s;h~&HfQDp8i|=Fm zm{v8U@%B>c{^)&#<@-To$NiJX3>=Lf5_;>YtNjm7P=q53akFsrlO^^cqUYk(D8zyD zjkjBk-GT&78>Ey3E2N-gH)TQPb@-Ym5#>?odb5u-WmqDa<}Y;BdpsdmbF@1-!t3th zMAt%pwHK*E}x>rPahdBR0b3xPN^2zz1M4xjw6=3~%gRB(Z(q_+O})8R$b7ad{j z6wcT6RP#!Hc!qKXE=VHc(UV!9z>l4-mM$4>pkg!yO`|_;pw{Q#u35#qmeb}wVsUfS z(#6#a__uSawAkzX&M(-KuztftZ&p_=Re!dozMa0ci2DJ`f0Sh2eZXK6^AeRB>)m9P zy-C8S4n6CgceV(O%Q^!&q0(0^V@Km;Y2P%GmFjvntaQG3zdhjt><_)}4e}6{4bBZ{W+~tmZGW}Z zx%-(&-?@&K5c)|K0RKW3&j#f=Llv&SeZW)UPGZTk7k%BO$xhC^n&N^!?JY=ZOyXAn z(2RMG&I!$E!oE4N-p0;l_TF?VtmFbuAWGdk#XH@&u+Q%kU04<3dEYo+N&m4`qh4l( zsF~`d#G#*i2!UTd3=xSJ;qf~@n>Dtzn3^D^tf?mh*%QO#4u6?%jj@^vrciAzcdAZ6 zNTwPO6s-IN6Q}&Ta<#STGoOYDT2bSkXAvZeFMX;LEB;HA7=NLrIsI;}WaP|aH|e0C zBHb;rJ(sZ)wR{+#L*g|J*D|~<8m|A;vqYCiOA0j^Xe|g!TaDaloCph25j8nmGS zToD~$Tg^i;hS?+SY$Wq~BU$z>W%I>pa@6LE^(e-ZVR>~$6>Zx1A$aKcCREZRS8 zbtnDgVM}>0iPH?I#;bk)P0tK(Wyd-6h=qKN-gnCKXo&!E`BFcOZ1kWx1d1D1aMk;Y zx;i|s-HQ^oyy4mWvNCiSrI`sYIF|*N#s6G>lVGVWt ziI(?Ewm-0bS^E#yJ7vG;uz=nu9=40*{q1#pmcb3>)>E=KQNSp78h)Ffr))0AtPMdd}4MTueRvf$PRK~RL-G1`i`83T2{rIX6OxuDcEDH zy=<{V9uqTsHzz;9-OSQq9*wcj!;(O5b0&TePBgr=W%=d%-uS>)UC4$n+ws}WOc<-_ zS4qLJ1c^SI6s(CMPQCDKRd1Z`5Cw6xI?kt+YQ!1Ih=eNG3)@BdPfxd|hGp0%tk_h^ z1qi(oaS6!N@$diBzmz^jEJ%oOF2EK;%gyp`okul^3h{ICv0AOn24A%6uU8rj?L-$RMToIFmqCug$yz83m@rG=$9$iWv{Rb3b3f3 z(9h@|e@50_*M;JPOcXAO7vIUUUMUrGl`ItIS!(DL3GLBx>JJP=c6%(3B=O-6l$;3s zST6RtmlW+Ex5ftDs*sqv@CFhUtwjswglj8s$`>aXz1K%by+hnzb!H9N zk7!KdGI-!T(K+^&bTKaK*zjp+T@l{)WaX@_gp-$EwPZnjGnO{mNXbfG>A31ZwYKf(?liqi;Bm;7s;_Cf|1C>TZN~zL4aa~^y#}mDu*w}ng(FfN6)aRI56Qb z%olo?a#UPU`NP&T+bp3~jMqWU9XV;xc(r%>s$42msCq7z+ z`AR=I){$OGzK_Z3fqjV@&1q>(F5v|psr^hpkY7ykhLcH7d~P3?i!EAPb|UTKY3Pk~ z2eO=726E?$nPIt|KK0TIh<4{9mW+wDW7%MQ$+`V6;>}u~GRUr30e8)=^lR}u8s8eB zYqDmEG_%(!SF|5OV^=$koXh;?Dwx`e7ZzCOx!BUe|I=SP3JvO}-(|PU-TKp(t*VSY zY1h6)6Bw_h*o}1Wzl;3suoOCWt~)PIP;=}_?4`@Eu z*8P75^p`3MMXs#m?XyS=9dux0X&+#^O+=MmZ_gp~>|8YqLR4H%5NGt^bH2+x2Y7v{ zBivu0y7BYPPf76jN8v@@Zi`_3{IeA?85o3^VR4Aq@5m@huT4{Ei=T3|EX2E34!B1vxBcQ<2@=sCpNn{2hAcu;?i?(06N%u8!9&k)$&yycen!#0aHo3KhngJ$-A zb89F9X6k&iEG4ba#IP3ICv7y3T=qI5#jWfrSTU%hklWBU*74^GoJJoJ?2|0?Zc(6( z4i25^?)*CWW;S)uy~gK=TZ;l4ZsH+lwkgNorkrEH>%rP=AFG{xl5$oO7yWeDWtkz` zV`xEfYZE_I3)Er}U-we_4m+3%1RaX4Io%X#skC-aHP*nPxBDp4uHj>4J%uOW z@n|87$I42u=6Er@`-Pn+bKq)@)`<}kH_?=qq6X5Nq4D108U592??6j2*IxHC8AU5K zOWJ}ja|c~%EhiYHZGe}5#vt$7QJ&LGCms|bJs6TB$IH=DeZ!;f-I-{HN+*ghA0?6`N9UE0~KyOjrcQ zg$teK=_Wy;JZ=rhw@~>5=ep2NYLq*nJ>8_-G7qOv4c0LF6q+BQjK3ZptDqsE-t9Wk zGQNbTKdy-UC0EGiit0E%KQpsNS>{Y6XZYyewp`EqUl7aoNju;eui2mqi^cayQ|bpe zte(I_5qeYEgpiI)pif2J+pjF>X;*65u^;iG97E(N4DxobSOTqKH zf(HS-v8mgFw{#l^hZpo1ZDhyGwbG(zKCr7+d$B{0M71>TYm?=&(Ylw1#V7#^F zXJuO&C2_*fq-S308m{+xt|<08i(ZbsM(Ol&D=8^|3xT$>c=TPR*7hI;Ppu3a0qvKM zwzyk;(I>+aiA>O#`f%H6snNqbbA8#CLN0e7JBIeFD4&0aG{vo%6xRWJZMbsDmnPDE zkZG}n@YL=%(+N-Wu+4BgrO7P}eZGbI3u48efPz@0Aih}xXUIkd>z>{dqXQ6mrN5e{Qh@ndnqM2pPdzrCvNk5< zDza$jO{K%J;jej8C)Q(R`Oc6lJ-+VNfH`LDsc^?W3lzPVel`xBA@6W$47c2HNZjy< z)R^nMj#bLyuNeaC15-x4gs2mJV$5?I;4+dcezP-YVqX`x zs*6ej>!3S?aHp}|Pckw3X>HYNd4D4|w!|jUC(jbtE~&?}hnY zdkH063?8tJZ>9a4TQj$IGY)ZqRYV(iDq0?or& zs}}7RL=eSUNll}21*^k4bS>TMp0()L5=vkunIdUq;+s0leo&C|URM`of^2hSr|2(N z;^#mNp>nVsqkP0#MJt_U5!dRG^iM4*C3_-3pQl<96~gCz=Qswqj&>Q}GhgQsgMLll_r;zeD)V3yJa#Etal_6iCujDy$+HKJE>+yEm2X>C=dYgF8Kr^ig!A4S4G_#&d)%B zx=vl`E`twDi+A65H>j37Q*kXQU{mgitAF$;8o5X(Ugsm9xJ`?W57*6uSf+e!^b~6H z;!2We$t85l^y;pZ_A=E#<=PN^EMek9{oav<^RBtWVht_637CS2oG+^(xj#zCQBDNFwgg^5uBab44ECqV4 zS{h`th$F>26EnQ54M`;y)Q(%g`OCyc!ilji8%qFS6zMmE-O@g9l6d2@GmE1*cWDRe zUFHO6b?yO(J1YhJxMFpXwzNBuo)FI6!zAcKE zM8;xu=z9(pFv#3Cm) zut-1WP3_3y1=6DmObTG*b= zHzL2Q%j+&kICWU=QGpZ4c27C9T}H&WGVUEHM;qMqGGKk9Sg#IVU_<#2efihUAi7|P zhaFmMm+EKP#{q<^_ZUP_t+8a3_z(C$sWvO_Seg;7F=%s;BZ_7k%YZ5FfL8^EvWp49q(rPXdXMlqX%k}1nHj~G}t zcPrTF_ZzPoq+}{!>h5z2`>9Y~HTF|RIxH-ZZSj|rh&XYZ9o%L<3)Ra#1p4cBW(!eo z|07i34dbhg$FLBfOfrSGB<^0g@a(zp?Z;imLqW9=tu1+#lfXor-}n%`gK0YnMsw`y zmL|yjWv|ks&2qtG@7=$EjK7-O$The2*e_h$kMm^Nf$o9DY%!eU&uTn2Tdm?%OH&2f zt37&abW~m*@H?iy{dk<~1>=|u)Ba~DHNJ%Qm&dmIr02l!}hs}ZE?UHVZ;t0fSk7USVMDgB5P8uPY8%n z-jg4<(fhE@lXDpV#?~l}m{6kHoPPS0%ufc z8$!CB;%`%u^i1_7F!<`&rG+W$lD*oG(%LVck4Z9s%Gki1lP^JfHFskh)<`Zm*Z2x# zg(i?KJ1)Ek=rlU zJ8w2AjG;FkZl^on)Ejw)O`2!59QnIU6k7F&qwPL?&9Q53vcG=$^oc+dgLGn6LJ8Bx z6Y~r~7nfpEGu1j4%gp?_MNP{q@MDO@?hX*NC4UI)iZ&W7tzT>^L{4Dp#<>~f`1oVp z&}BRlf{C*Rxz4qc1KUsC5yR`;yt1NW0!96yiTih z?D&NNUkH_c8q}R;$yCDM66-+H@#9SF^6^k9LBA->W$D$wMxPuA8FyZ{hJ8x5Siz4+c=h zb@*x`Ho;F1@bn-=b1q}eD%aIQSQ_NspTxx6HKSF7xIIkS$(tqY)eC)Cmyd(;0k$W?G$OHmI(=bFzmFs~V`ypnb602AorMOO zGv8SzbIw{~Hv>IwPMh9`7n`^Y+UsJ}gjDcva}87mGbW{M;ftM*odOarpQK)b*Q<^| zBe@>5`^2=2wfp|oev#NF6BoCqVNEzFF6!~LL4JC6o4qJDvJnx0uv?fo(zhVN#-a7*q7fe5>=HW%VC zM%>*sT%{7S@A7+AmgbVXKN}QNKMxEFrWrM}c+oq%;~rpKS7aU{I)|BcYW*j5;-_;V zoRNw#1(7f<9PZ+{20b!VV)(g;3&-VLJm%nLm4N#)5W6iH1U?EDj~WFR25S43h0ye4 z%7JxubIJ|ZP}js6tX}wIlR(cjT32{DGnv@=J}l=z;l%+<(l!0Sf@b->rCb~t&fzg5 zv3AbX4`@XVgcWw;B(woJ8TFT^Es&+py*SfvH8!nu1jdi>C!db9AC&oyp8sNG0RH}M z_s3Oz;qII=&PXphckVGpO-OER5zW|?Sr6qxT&ivVVkftKWLy4)2GqMmWaqoG08;9s zQ*mzpC_OtM1;3FB^c#;H;x1hlvM0XL@PpN~D+KwMoeEc+(i;a|i7TmS2Yy6(+@gU? zHHx6yt6DsQiZH)NN zUFdJ6N9BhowwRw~l727K8pFHCnlN*Key3&pc70qO^#~#<1fkZ~R?TU{aTgD$^}T8m ziMl)#TOLphsQx-x436-aUrs=1N8y zfiyZDR)MSb6yY>C6NT(|XY})}M=m@_rMqQZY_>&&6AG^!Log^*`0ptMZIUW`f$0!N zE57+%uxLIE=cQSrxze^~#vpCMWSUYGczbl;T zr;}SO^R@4iI5w&GyA9azlj1)%TEH|b6LiQ6m*d>o z)4Z}b+LuFt{FvS$Ti>cah61~EMhu%XJ?Z)Q0FuAwV(0BuZknMEEIrEF-nw3YG|!|# z#)9}2Ae(0sF6WeXSM%d_cEVIuYzUa+NgPRt7m+>L(3pQ`$&z!*c zF57BRBd*0t;9W%_&AG^Wphfy}bM0pG?GtRZ8h<~9Q$5|8e_%-fyB}UZZ74=Va-Gia zp~WXYb~4(lA<7W&Vx}3Q^#jXjDeVfXJukXQY%JCG(bp{Po7_htO=2#`Uj+E)VTyGh zaMSgfE;h>^PHU}LvrjsBT|AebDCHvg*$j!QZyoy;hSq=>x7r{viW6n)R>EWnZQaq) zR`PYH4Fl$SOWE5$yub$3_G)MEp%kY;GhiTMN;XWIe>D&pZT+wH5?J8=*Anr^Qy^du z29NxIy((r4_MTy3*|Dbeld2Ke<+~f`CC8s-M*X2x$TPCzW0wk_1kyH)c&7uCh4`_2GcP)kNGDuAFRZ- zhl8QBz~mJTjGO=P; f|6=tw=GS*d>nK|lz?=XfHVf(J%E68cMQ@<*U-Fs zxX*L+JkOW^dHrDQnZ5V5uGQ|n~hQx}3*H&c>Nj6`M4z>J0i4yzB3e(t=lNO&*5f;EIeb3{ujl1@U!7wsPE{YVrn4(xAO-%uM_Xyim5Bsrnn z?%ch{vv)ByC(GT+E7qci*gZsGyIiLcN@3Ca|Nbj7ND3PG`x5!Tdo8LNf8GLo7R+_W z{68Kelw!cep!i>}!jk@fe^scGtZZOUGQU!#^(4_r8znY2_QrW!TpaZ2pX>Ob;nDj? zx|rM9B|H_seBOv?*!j^x>n&yi1*7%9r}fJ3^E-H!{|B+gxVQL#-OVwuu*e19r}fx{ z#Rep);?s!b%TS~eud@8!xpD3dwd`;9mlRIdpL*XHKYh4B23uaGTK)a0* zA016}GD1t&1D|6Z`jbn@je4w3mYJy&nUpWo?Vz(;1upm4<9YR_Vq;_y^hJ=1{9ekx z$C!11P%0=we+(BCM0e$fILO%=_VRDP0e+@CN~ z_rj+^WoCz_R}-A=9E2&m)ImzNgJtH$BjkT@ixhtN; zH@t&yQa~S=39V)Zrl!tJ*#v`>bO#J0zvYblA)ch|8%0SUKnt^|;q=@*pTG8k$E<&6 zN%zPsB=DX^xvelyj9d1;es?`D{ashYZw5wc_#q{G0*YsD-6X&Gz8S4g*CP?h)mxZ^ zzedhcX4-qpg9mb^%gob5mWXqqB_9j! z7URU=eMkai_#D={8wKELNDPt4GQ+ss zeJ_sd@e}VxqyM$c|7h+0=46?m6v*_WE5`hI%3dp7(B@o&sJ)XR*8|KlN9Jh+vuiMK zmHPl(B+~11rur@opU%(F-~8{9xzV1-z6of46F)Js-#7a*4Hbuy;re3r1(35ZLdCpy z{b=idxbUc{sm*0OKNwg$RKNWZWq)whM?&WRZdryC1N`axgLgX=)}2LAc|XU6163F% zuL+=bydruN#vo3?6x6-hIedS?CL8RShOcJpaj+S zl*fKK{k{p}!)QU2kP|q1+*q#{mxf4hcZ~{Y#~mFS^m|8gk97Xtki#e99h zNDF1KdNTJy(wNY>@7-x=n=tOqrN`^R)0o1w;Epln@x`Fv5L-wtE~nTgk?NJvZ|_0+ zq{bc7cKqM!M_+=4qzs$5AxRvBiS>YxmFok@5JToe!R9bp3Fc~wdBq5DO4?(#NKJNj2AR&A;gLRjgfnxxr$5iqi z7Jm#hiI*@vlEzzQT$n~<40nEUll+gTV{V=vCgeC)zBiy;=|nNqT)#UgM_aI?PwY(z zuNM-Rq3cWggK%DQr0VhDLAlWB(WO*a6B~SuKDp2N`|@#dK7aYL@l;ndC$Nf1o$JZD$?SEZWnHuc{$m^>1z6yetT;%x=#JKctfyaovG(C6h4!D zgqFFrpTrvxKUmW94}McYDAjml&dNrB&je9$RKlC~l7j18b{Z7~)93-`_3vAkF8_@Mn&+{F1LPpy)XGLuXN|* zFV~&GU(kgq`;S~YyoJ;Vl-5VEa>DEY+TW@s6?CIHKW}r?n!_**aoTQakp+Zd2;g7+2{ z7uGT3S_E8N1|yIdLZz=<@mS6>6vlhC@A~TphSlL=T-?cSK z@HY>PYmwOx(Wp7SY=7XOR4AJ(Z5UXRZGy{BhH_h-a$9@nz>vpNM|;=BufO1c`?V4u z<#tlbAHq+1g5;qL-X~r|I@Uzrz2P5pZ_VjtJ}?p9Dz3uYMn9jC&>5m!S)?%_B&35h z*d0dOzta@bVZj{_wd#1(Z2y#t1I41G<2m#252*mk-|UcLzyNHZv;3Y$D72dUqwi() zT`WQ^C+xf0<()jQLGkMn#nr_1J5P%fjqa}^OsZ)@1b*+TO#sQJ9cnCctx(D4E6)&a zEf4H8P1J?Q{nnq>3<+Yt-Ca4{q~iFQ=s}|3NA~_KMt7-mts2<5kLlP_+tj|`G%+X~=gw+Ih4OpAo z1T+*?`Wpv-h85^e@GRwFP>%iS(a4dMHfCZZ$#c9iO~#X^4+Igj&t!zlC{_=;PwLN8 zkIDUBgyZ2Kzt%Jb1VEItZ&}CwQ1iP0SCSSc2ugW5GcdUNL^GpK3u}Ds0nun?Rl5^v%oTAtY9j_=k)%8br*DIqraC{wgVz6k+g&il#95} znw+D{p3y;SPW&BPx&3#FzR!OMzzYz-J+K6vKb7v`V{)ln@kCAH?XN=raSsG}PYCd; zC(SQE@OYG^MXsS4stCD{>BmzC{4Pf#5jcRRWD;5riN7!amsuamUi(L`u~3x+O8v2c zbEJQN*7jWRYFsIC;asOBG;#vOyTHY0dtZtAr|2Ea-+W=Xd7qEHKY3p~z3~e7s3!k^ ze2ob#m&={Fg=WxxNB(0cmo^wx3wfuhpC&({MIOVpT0XtMa@+21alOZ>nylQ`mJwoO z^uc7gMa;#1LQ`yLXlQfZ)&AL$Rl&D!aaUJY%`9)p$jDR?eoX@F+uQMhfq~6U7iPxB zG9n@(&mjVy5n|CXF*$j8EW5$i%?M9V&n`|szqe;%vA4H#%F5K2&2W~6r=~t{cbG!* zn|gbDKVME~R#$7-&edB~e8a^PUd%pz`=ZHu%;+Suv2jl860I|O{%(_y7j7CbIeg2O>oC@_Y`L` zh@5+gW)s7x_b1&2LEXoX1LAmEE3)dRrlv;9=9$;j;iS_?4~Q;bIXhR3o?+<4#>FYT zeOp;en*v8f^Y`q)K8zzhYmBOWy_pkrs8Q8$?h9iTM)}-2&U( zF`qFKZO-{5u}E{0h88BM>diJ#joFw1T;JlG^VtEeSkx{_J)S_@ALYLS?nX~n&)7p| zBb~9$4yITk$-X(v!`+G8L5f&!TA%w<1T_IK#{HWY2eH&YG?O&|A_=|*c<_U2t4Ti= zJxqT<79gP#U5DZp-!>GW%N6}70WHiOjssi+^ah8Glok`M^I#AyKRRYFoe zi24v@Wm8GAM9c;S!&p+F9r9RwBFz{Et)^}ZGf_n5_TmD5763&k&cCiM&&Mz*Qm0L? z0}d9+gPSXR&rwPk2JYOs(-te?=QWXwB)IjhUQuwigi+mf<_&xO+fWEKAwRbH!IIK= zEx4u=s)_Xe6993olYBr)%4I(mi0W)&l0%`7;k})Jit8X~JwYdaecHd=9!A;jv_Jt) zC-XN{wGr8VPfw6a%S*6JdM%K1&`PKb3#G%IQt4>rKcPl{jho9&$Dgy;$`uK5KmJ-v zJN!l**GnR!u%JKzn=+Sgfuc(u6Kq{ULbh*VZT(y-^4nR?FK)Dmd`#qb8ys`lO>ei7 zZ$aogO%zEB*Wt`)5dxRKj^iUE`DZ5GoSo3cPz7xpPp)ys%h?`7x~86M^TsO z^M(hfhIQ916GH7=a=fpRl)vUTP0+%>kU+2fzXKt+_GJW~81|XUu;vDPd6Dpob z8TVm0IbwadvAGG0z61rV`mQ(}_Qck zbw9U1En;4L#Ela6IFgduUeHhVT&)-g z6A_y5h2+8QJ%Ozn$JOVN-7g=tAK>(pPA>;IO53+IBmyf=A*u;x2)4W18=497^*Yh7 zE2A*`f@_`kem-`)x~TbVRJmU=>Ai4K{Z@=|kE(2$*t5@kr^(69VCBx#qcqgYE+z4Z z1xoEB9B_Kp>|)V+H7!^#Ha1pcy-pvtYyt?b0WLTf!}9cN1!?(hu~t=GK(u6qluep& zQtiuIeZf2-GS>mC#ot6itz?@{)&?1C9F{{qn+cq`CvIIS&3~W;OmFFt4@!OtQ*R{L z0-OF`w0&yCO0MOX| z0lO^*+KZ|%o>t-8cZA&bv%5NO_)cfyS>UZ!dl*_*3GUMAsE$NbZofIlaw2#Pm!v<(<5=i^Pj)N|g0gKWR+?R;NLZCNIRFj^d z@*YHDUfD1bq!K%mDN)lmTL2ngcHMINW(&(UYj%1+7-Dwsob!Soc(0S&BhRzv74A*v zZZ;ychkCnV-_9`n`UkgdPHDh;84d7B!?JpYLGJ}k@6&ygBZuH`JkpUgo(sN@{5rf|E|Q&XUlnE|!memc0g0WezQ07me7D5%kNiTta)hxxl1rdu`&v-gak}GUn5!q;5904zM$>QNgoYg?%@I% z*p^IxQm9HGmDbM!pd8Wd1fOhgKw)==HWTi{O7WXx1*p);Hkp~23gA9S3$6VCIvZqA^4$NusYv8ICH} zwCf>>GL)-lZ(A-m!a2NWk4Hx^gSm}~zyO}-&X^w&phmQ!psi|bgFSIJ$hdOwI#Bv- zzFZHjcJLA*U5B|epn{3LV%EocC?@_A98(SLNb)*6-dBfT9mm`<)R4tk@G zJlGTml?=WFT#2Mu$q{=X>V+v(&-w}y^87l$?A|^PHO@GS2kIxW2M?5H34rCseE?Oy z*2N|T{D=$|h;w#Ruywfyi+&hSusbb`Wl7bX`JMobZfb?O8wI^mB}Jftao|LySi%Sg zcg|ki3-4z;It;ZbUW*Hwf$q!SDF=93A7r-i0;T0{ejU~n2($7MY~t@;((r?4Uvd{4 zVZ=*2G7pEx;1&0f_&N(rhv1J$4U}r?3RT@Tw}y(@(^k_`&y=I0>4*fG;w(XP3J0nt zz8VP$?iU~7ww~^V%G6c4(V~>0Wj0((kxOPs)>UEe2LpEcHt4P0Fk)AD>O-#pZNU}J z(XFPv;+{A_$=v8A?7kF@!v%3J4C}lW+#>3&&zLk=t57|lIm9Jj*|s~FPdtswuJ^x0 z1sR6Q&LwOmR(#q;eSv!)LcsNUZOqVh5{mnU|5n(a4l1@0xQ}=SVE3 z$zxOlpvuIp47vNw0Uiu&0~5GYGEy0m`tq37$drje4GnLjM5ReaF$0E6F^v|sb%iq? zu1R5z5b#8sSI>C!H2aM6Bo&Cuszuhf_^OXJNX)HQq=E2`RAnNdJZHhPluA`7sG8OU zU}4(o*%VpIcm=?z(2$is%Xky9^-olXy8{W0#3euy&51AYHdY{mXv{+rc%+9+6c8D@ zz}*DMOotM9b4gpXe8h;;s=^b;&oc$noC+p;4yCS@rFI8iSK-w6Pn2^MzUVM&;<1b( zb}!}OrYpGN(ro}lVvfRV=!-~7>aqUNKRCidNTl>PnM)gs(=HyAkJ^6vc#~KUS5gFY z*dR-;@MKVu6T?6TXxf(O9p@~TK&xyjc+;O3lvl(Fp~RF_S;ie zrxo}xc`8E{ZSI%>i=nlV4?FgJ3pl9%5C?%qYJmI%fPAln8a||o6xLHq5b7QaFvALF~y&Oqzp-Lr&tH7JKp*zA(aiuFuRN^HRP(2H_BI!`oFXjxEm9> zDiUL^M(G|?9;egEf9+n=`$E+=GpCmjzLdrbD9Ez@QO0UcAEeKTR_DWadt3 zDw5TztT|q{%Yk-F8-sQiIJCb(pE5#YhZrl^gv&e&5cD+aLp-E@1JbXDo+%*( z24Fr2RY38PO7iHYvI2BLvH_WUBi)H0vXlnJ7s1-Qa(!NwYpZG2*NXQJ=F*75SuYk=?ACp80D6u|j6)oI>AWK- zOd4TmH2YHHCl zwa&l3!En%C-2L%NM`)1z9W~Gj+Pe_EoWRU=bq2%FFPbFk8@gz-* zt?6>hZn-U(_|c%*#L5q=pT z*~0V!Q9$5PR&8WH4u$9D<_a1j6MgV)_(*~JhL>{3CD#dtm)xavb01LlOOpe|Q!#f1 zNfYiVo$3d=_uo969tCCuztdMFF8nmNvb_9bxjW&7cF8F7-i4IAOmjPs?{9zpeBbH_ zClE+z^E;|J)q#@POY%}GDJlHiV3&Vm?3;E~Hc<0xQTwTkc%K>|OIV;%n#1e?^7=BM zsI7e_D=aMhd3JiEPQR0d=#q3F$aL4;Pe3+9FH_!s3LPMw-fptzd{wCP>}Net3RYx= zN${1Ovdg}1x4V1t-SiSj_%Zg{UQ)|y!bmYNFdPuuubss!(+pg5n-K!%oHIAotAXFL zmNJZGYC5n16By+fGf0Q`0rECq8p=_!W9Q&d38xay6u2O`qI!FI=1h{>tdE1%3t&N` za16UM1G-9&QwO(QIF#bVh)`QyJu6;H)Nv)0Vr@-uZEwf>G{HNn>7u(S7Qi>Yz*Wq3 z7pA^{AYGm)F+ySui*K56S354Vb8{&o&P^uzF{Kj1N_}DTX8ma|%Cwam5#2<@#7q!~ zF3?jBMGAz*OW5BzZ9zLFIgAZ+v5K(JQyVi#QbaRir zIy6jhu~fi6QX|mQR}iV`c=fZmt<%uN9B6|nUEx}P#excd{lfd4+lROm^#Pds2M41P zROh{L8y!_OH3ncjz%E`8mJ;sTO`?>B^g~yx*B9yc<$Vqqb_On5Dp;PT*=vmJ^ zIcw{0;oyv30=qXcf1l{)E2yi|amh2Hx5*fA&O=Qwu4>2KjSs!kgIoNp$^MC0(*Vu0 zCe^o<8{B=DhwJLjoJde*zu6P+Wv?f&je#_R>8ckmg*Z3v%E%saRHktX0UeW9)qw|# zQ#aM!GO-Pi%+xafx+UA3YFa>0%|i9B+M3@IsUusD15w*GPGJdE-?5-kC1C zJWxjHvGQu01i0@3!Z7Z_4L!6cr;G^P-=MO-6-x&sWB8lI^_1rzA{%`$F77WkqaP@^ zL?w5~uf9dBFVX=+6C4Bj!BD`KKlc|~gN2;dB9mYE(V+~?cc|T_6ts=iyZA|}SPxtM z*d zLjhUZ&QyPzx`l)iA|rG|z`WkUdGVN zuq&DU2iSb~tyW;ytb@ld;OLrz6)`3X2m{I#7GW$#w>v3IMod1zW zuIVPRe7P@Fbna_wvoU}ow+)#jVt>M*@CX!KBIVG^jxTPR?-#$;;UFLsX_lnJ=$=J( zO`hA0Podd0o$^;x5a7yiXSD03=r@Nquv(*D(`)#7aB)>&>^JjR5>i=z!W`Yj@g)?H z0DKG~VQT(2f1Lfn^ZOG$m`9{g{1J3*_|{v{FY8JYB)T&Qwb6#pxVn>~@@xMK+A+Zy z;VdUfktlESzHZ#mwjSe7buYboKxOasFCXtiNVs)L+J#kCcF3-!O_wfzsAv8j2?r3Q zXEjV%9{>zgE!w3d;I*)hBk#gt@cGuL>o?C1aPEu*a5W5-y+B7qe~j}uyX|tK?d|pQ zE83^fKxORuy>7vrJX;T=WS}Z9eu~wPbmc%n394qM@Hd19Riu}Ily)=7&^0#I{gU_| zy!@(a<@c*<)fcAURVm?TmHjU?gK0owz(>y-V}OwaE(M9YV{Q$$VW1y?yZQS>roA!7 z`8T9Nw`R`Uj1k$<;Ewo;!)JR|WZzt2bex`6VDL}yXBU^bsU(5u9s$Jpx7Wy~#U1lb z_W(P^_h$cxPC?X%xFOv3*wiyd65-G$U4ifa|MS~6_8GHg+C|D?z_S85gm^myDnm(D zK8b6Oikk~SBYsFg;J-plT*peZ>WCh>zvO8yWAf$1S53-(`>clVfU)T`#n zo@c7`R&(>w-%AFFEkY6t`YU4X*(klkl%@!!GLzN^Sg6=fNZsb5A8w%^@ra!rHfy-5teT=0GS$ogGyYjOC=oNz=QR*H#7_7upm=Tn z3Le4?mI_;^8tz3J4(eYZ3cYg^2no@|Jt60$eTr=s<%(2r4^aA5=^kT(z-W{)g(~G7$`bW*{x^dU=bxWW_ zYE!=hLFF~%BM#;yw3Fc5l*|ePpu@jmDcc-E7kvV*={Ly>g9(9VebJriPU;5rS@nR4 z7L7{AO!ez=)7d?XEIAXkH30#fBIF2zMLL zK{jgyQje1t2BG=sDy@IL3{*dgE$9v#j_%K|o*9r8ndF%7W#}$)Kx-i#G0WmlVcO^a z(^-##Jt?pe+-;H^E;od=xjcYUj{Sgcs1`;@hbKmUL7_;Z?{0>4OG}AEB53He(`r{!B97d)A zxR*6}n1N|vrpbH0R>IG5`mXuzKV1_aAm&9r1*-I24BtM1>5E^3<3p2!Y_C7Q?h{9& z)j~lSs%M~<+#i_OY`OuE@Pb*f?k4%lM+wx^Q`gUsH#hCouz40sc1prIPVlTQ1R*K- z3yx{;pp`tmQXX;C8+IkU{R}0lGbN&4U=zJG6TwHuo8d3t1`V9 z!8#I&zz6%j;|hY3fbGcqH-L7u4=h$8gFA8H2 zscE90<=KSc<8|dbs$`*~OOA0eKDj*YJ2P&p@tRP5TE!=NRDJ#;acXbN zLK~NETL&AzV+}pW5*X2Y22hQfy8VJZZf<%gUv<=Y;UW5Nh(nXH23Q-gt^_DMIscERAZgks+F;y1{B0G>?WuT@ueMQo6D9|l>@#2NS_`)7;GigpvPIO$H zNrRo_HO4hM0AloV?j@t)z4`8h(~f~Tj3N*~v%_IQ1|s9=xzkX4v{DAAn?1j4LF2u* zYKP4?#Z_inT_ydR1(;^SoHI-JS?<)&HJV~3)+s(r+Q!k>SW)J+0VRJWn7?aYZeZG5 zJX;ltMNWW3dI&=R#Q=bwa}zQjL*`BZfm|{ZKTp!Bad?ck!iB{HsLAdaGn$1!-#0we zcOdJ?h?}ynL)(|^dDd5Dixz&8JC@eZ{a#;G0l?1K$jCjQ4t|a>0!B$bZ8KA{JoFG;sCFS&r3j}+^S`E_3XCvN6?$43YSJH4XA!(;hZ zc2!QtP73ZWbt4u0^0r4fHP;N<4)@cQo;VkZefGFd&_%0gy6#!?!s%5Z>0I;!PTQ{K z3rYv>o#gCg-Kde(gr%7>Cxt$Rn6|tEhvOe^M}nuC$p@#8ovGFzuIRZR%#L&=*wGUK(3cdLv!%ylO<=$UUZh!!jX!`-$`GYKTsrt@;_{&=X_cs+b;cV-x`3a*B%w z6y0I`pThJj$Q?WEiu4Iv(50njYUr~lq(B5VW@~Xe2(J}o}=qp6Z}>( zqOw~eKP5PM-OAc$-fB$|!Uv2YDCmu6=uE{ivcwxVjzc7kL~`-aLYXy!$oq3sBo1u) zyahk-PI09KGT^xp=yPGUrr5elb8&L+^qP55f;X|mzG;+5U2X4NJyBCvf6>ULerAu0 zE(1!p2Cy4&psVO}g3mT&g|slo9tjDqmRM%0*3;V)QqAPu=<7_3NWYj$4EqKk9be8% z)=5vOy(9#ON5Ez99!l-8ZAe>iGUpysyW zAG1Q;W{+!{XBS zAttqA%YZwDdW_$-1r+weZPaSbs$ywH=tx+?6-(v)Z#)Os6j}_VW)5X1t=<64GABWXlyeoXjaqdKpE0RhVZ`QXlII zDGv*`iOeamGTA#FEoC(cV3*gu?-)a5{N$(M>4tTE4}_Sd>YP9 z**`jZz2DpGS&Ha;Cj9m~w6;*Gzv#!umh=6uVz0U^yK))}bZ3=}9Aj#$;>SwJaqVWk zrziAAz8tLfKZxpk>5jeq3yHVs2zr&l0mp&3jl|Ouyzo=P1FWh|4?jFj6N>`c4h^@H zftbiY>rGFf4S_YBKzX}ZmPF~gvPaB*$dQ*V`yHltBlx|D`JdA zB5^#Fr5=Q{)7fXl znskwEt1qZjrlW~K#3n@FJx=*{&}#2BLonyT2TTERsmknPcdH=>&s??DF+c8(WdNZg zFr#^WI1hk70o*>n&Xt!6-Kw6+Qjs?ePY3~l$9_kdYkAu^y{cpO>P1dD#FGI}uuw_G zA@ey6gf3^^I_ARJ>Yn$}EjFzRom&YU@hv9-YQ^?xcOGhGT-9+wp4=i8tu&4*&GbLv z)~*%OSDLFd0Xq&Yi>$hrOq@a*4^|HJ9U(4#U-l)vs>7>KjRLJC=emirUAOMDO;+nF zR@p}F&x+=3tlchXh`b+G#gh#YRl1)p7M1qOhQ~+n<3}E{!##4JcPYR;pJ~shZxfa= zXDaVNw$_hY^@Rj|4|RVfkV?~36O0&@KmDge;X`aQ?=(Z;SF{&@um#|>T)krr&Gxn_ zK%NrjD$C$%`LEFNVeBSwwDbsO{tAr8zvbE7{UhRW>!4lu8kflJL`^lA0gbJRLtMPYW)HN!ePP zqHLoaJ-dg-x|6%LM=I?4CmwW^_H_oVCM?_Tq5t^87Fql1$g&b|o4U+T_3p0t2C3Jg zFb1N+nwlAk-cWz$jQImujNP9(8s{sNCt`4`%9i!5)V{*05DdcjhOcu4>Y^w z5g~?b4*~5HAx00LVJK_NV04~88Alh05|o0t9XgE43(S;_-wj=N-Q@vhMThCRT$VQV z$Q~!D`pE;wu>_#3>Sy_q*xXvN*aBDrJ4zXD6RslmaYpbYYW8M$wQe7ggb%^hm8-xk zD7jqj`a641~sl&{&M&&s)C` z>Tx)=Dj-4g>s`wOCJaR=Pe^pQ@!2a(-7LRa&2Vq1IrQFp~a=NhM(@ zXZeiZqRvris%%@ytdDrTt2-tw({TOE*GR8CTYNq~kuk1DegQ@A*A0Ak?#j!VET{DQ z4&ySo>b>B2ed_iELXf@O&Tpaoh{c&DTzI@x`+lW$ERjx?qvwVQ{TO?AOy)_zMw3!J zi~Nrb6195Moa6xYz23JL@`|y?=ZKF7FMpWp6-`NZ6x{jzwFOmDWK&MR>>KU?m|ybH+)6>{R&C`>8Sv_u7#v!Onvzj~ZPkr6-Ul@zhm_AGKD~#>>kp z(UFZW@8%T7p037srN%Ptzu@stVB*=)@ikIndgvVRwc)H{v&KcH5Auj*)#0Jyein{H z*;=9T3rKt(XS$NEC-jbBoWcw3Fg_=NKnoSMRUNy9ZbLV*$5S& zkK%Q-*SD6|^Sqq(U}3a>?zTy}WHHpo5ts_&4l*7lyoXMV(Y>2n4(|S#G_+O_j1Dj_C@&dVi_89G5@enq$lP^6GvGHX}9PZ(<7J7_I6rM$uXrG!NP>jKZjq6nv_0iY%Eb4OP#z&N~;$aUpQ8BzWa-r zN|m{UbpROO{RccqB8qEJvD71HH!vkKmfDKha5M3< zAowH?O256>FP0QHa?4?r4D2o9Z0mT<)Il))`$#O{&FQB|ibm@XiFbit(s>}N&=j)#-sc-46vp<($`Xmn z89x1ky4GVw1&4aownySDmjj)}d`^5_Wm7sehd*mKe~7Vv7=GEL&a82NYq+5)N{{wj zjU%q|`sz)M`mscET}NB8J*)yC0yrBB%!tt!!f+G)Mg`}lUbTNSaO1Vlvu#3}v@g2d zi5>gvtO$CEJhKUuo@nkZUbw*8-PDBjY8$05RKzX5Ny^#l1zmXA_xuSF`e5bb)1j*4 zxHncBbGY>p5Lm8C9aA5!XG=DNipvu^dip5XTOS;Ku7AyRpwAH&aR80~PB_Dl;|n zlSD@Bps8SqbdJu9+-sGV=Gj~dnB>t&`?kIGrr_jVnbw?2FMErZpw@%~F2l>wwAmq# zqYx)j(t=45LK`k^);q|dSQ=n9$l&HwN|-sZx#~6{Emu}fZo?DX1mi9kW{1i8_an=8 zESdN7>8V9m?gkqUnjQuf*40;Gsb0|xF5NWwlg!kYuPs+_`f_?s(@E#5B>eC$&3coW zZVu+Gd=mJs@)7oR^Nr#^NWu@cXt4?I)5k9w+6ce0)qTL30O zUi36F^J(u-+|cvE|1^mhAG)YTKl;@RPoTMbc8#3`vYAhg-$%Ow>{TvQ_U{q{Mu>l7^`XyU_JB20>F6gWz9<7FIy!&rDr zz|1Kn(Ban+1vECJKDSQ7Z<3qPSnHiuPR1t>C| za_yqB^GKKWy3K}0Mj50DNpvwer0+5VSryEYxTOb^)`KqHV?UV4vfg1AE_u4!O0b+2 z0f}aLnl3N*G%vLwmOVIWbxGqmsieW?q0RGd*e4zOF*s+uthT%Jhg6T)?Bm%2n<5vF zp)3#H+tqIYT|yot14aeZ=mrw~Nvm=b1ksnNH36Sy0|YkfeOy$g=fyBHjTO`S<$nD^ z;DA*5EY{N1!wJXsdjrj%iSg_-iKT*2vG|x5zJFU^?rlEbAhw=#A~t86iZjOaE4#uc3rbdy8P@&y};s2Ojp7S z9v{y}y3XpA$Bz5FqsjGI(KW?M<+cB@o~x%*1$8yulBSIE%9@lyQo5Gj4cZ8&da6~<%&4|zXiB}e`e}~9yo~H5B>RpQi)ZJTnI-bkH(15 zqE@>cix2Ba&sDO2jDj&n6oPLds!Q+xi?>A1&=`t#8Pb0dSyI) zaYe=TRes{KAD~AyU)lIiwEyAy(1g6cH?lRnh#T5l`^1-*(Y3KNDD4yg+UVmf|LSYim{(`7fpq=uOgpZPFmV>jOaMaqRw5L zBAO`Y-3|@gbn$VogM)@_;r_4Q{gJ&N4Ugu#zFdfrTUxxl+NJ!rg3cvALBoshv)r(Z zC&hd0kabc?)dgYtQe^$C%m~OH+HvD&r`*h%v`X2_Hlfce8lLkC9}a1|Hu8-Kx_)l$ zPkR)Tes!1!{HDdj<>lp#GT?^{?xQk3+^+MCIF>o$_CHs>?$rP$-~S!Apz^$NCVhwdrpds$1Y6vDDrT)iW)#+} zHj3!2?L#Q;gNfnO0w7VA%&lk1&oZDSwpWkoNC3DQUnaSL-4QU)VHu-zt~4OoG%bZl zYN_?0;jv%GNH9ok8vaji5SH|CfG%+!QK~gXYpS$(y2Skq%qxIKIg#*9n{o8~Sbnj_ z*rG0h&4zP7C+D*wIMw5V&{zr^&wdi^G!aArdQ_Z>>#H9ZHwW^lXA8@*_HA0#%)plH zEhMKA1%=24YMF!T83V#)xNE`!=c3*VRoszeR(CQR{Q4T2#xsJte1GNO5`$^uu z=S9FUZ=>c|4<7P|jF#SuUl_pk7X7X;5PPKFd!zt(57P$|nYQEN6MWX>$?08os5?wc zb$j>Z?aUmD5@%ryXUt-p25~~Int;yy!#o<60=|hIz&f?TA8V(tK7T|D!~7=;-1C1G zwUQM&3AOW*118Y9Bl%`k{4{xT7rA<67{WP3H7{&6Q+x=Gh|KjLy{Lgc2S%TTbss0= ziL2mw9EbvkJ-9TBCHzeIRV)}xY|lYVZ2-$Pst`vR*n1)W_K~KU5`Gz%5=L20Vx9~t zSjRi_j3&QcK9B^r;2uZAyjXf)af(``eQq!dtg9OV2GILX7f7`wQ{X7CitT?2cCtLr zfX$&0n(DhQPFkUW=HJWv!IS2&M0B7l9I3zBlyeO79v+R4$eZ=4ThurxI8<3jPdA;1 z-O*nc?(k*;@EF5x*@8!2GFrAe32D)WOuPQ*7!*6m^BRQ!BivPWfwQzCq9Qp}W}Bx_ zywz^4`}eYz3aPxEUJA@k(7!2o@H~UIJJF`TQTho);bq6c#;p-$d=E@sQRa7Pt*g2q ziXd{=KCs`aFvQP*S%sC*yg1@R?-|LAw`3x$qWY90P*Xzw9QeT!Oh{59?d6)>{Y&6< zs!kla>9~0bxV!(+*LMsSZm*1Y8W>A2hQLX2WO@GlJ*B52MhQd?wj?HsUM5g=U0?=R z>Sj78$c*ULQ;P3oG4wX96Xg?kl#{muHMvpTClbS1;=Ke-4&9V$xx-W_U3k|BnYA`H zCtGMS)NV@!i543tDTH!s?gGYYzD4Yd=bk8;I(P|^(1Tv~Gf&O;*dBrLk>qFio64=;+3~A zcL!25$*_u!guUMD)#Df5=91o3z#N+)L%Ut+!wK{agC3mOtlFc~n!8tfZIZ&jND%uc zYc8fbj_f@fY*4N#@lFStGznTAMT*wi9j z!)4r^@z~CfL+1`>79rMk6>?jQ2QRt|Nhb8(zG;uj4sBj_WHt2z>96Y=n;7^=dVl7!1+AN4&+ZYl1S;^Oz zD%I4>ZOSAB@CCq%a)n;%UceFBMh(w(Ugy?(&h_KSXE4TJcY{-_&NnATLxb-?X&I>} z|6@hVPBQL1C6Rw}EX%4(q*rH}hUjprP%4z(@Mjsfx4C!26|yee_Vc5R3j;JG;yo4> zuhPH_p>_7xSb?~#zLSMc(n9Ifxw$7d^Js7n7b>-u<>zfW?BKMWi6Uo6vhn&9AF z>el>hd>0`#QTNa;n&wTFShZzokxGE`9dJ~xc>+HO zq1F}hp_GD$1b`(&ZARjB=P|HvXrG%V+1%=pa`Ql!AW5B#299J;+REbzi7=-96T$11 zTd0zwmgI6v1zAm)g|`ii9pIaz&5G&b;<4_tAq@pS>?NC1ZxV413YoiKAh=>fd|(BZ z-36XOjE-JT3eFE}2}DEvFn1cSOoJ!2A_*>>8rt%T%0J3LAS?5g%X%{$gG17Yo(_|q z2iEj)$F2SibgEbGxW6Y~I#GT+5%Vl{G9jMzrMSHYZMn{n-=Iej*24fgJ9evGH*nFza>1A%0;3{qDq& zTY^-%X&pra{)7_AV2>U8OHxaU6EMur9IGuSzhcKskG0wXcuBP0W+QDJ`E+K1d$69ubmqA`+;m2dmjAv z1oouw`fokV98l_g)(d`x)MSY5z2-2fqG{=7vlt=>u??pa#8$p=%lci8wan3wE{?XI`J^9LI0yz@cP^d+s4l}JJ-`JbBNU5nB-BhCC@G?o9k(ky|nZI zN9#XqkRC7heXq{>RcqHdjo6RKCX=OXt~ZiQJT4AdAM45tNHTI2bvqgJ9rMdgk)Lnc z{7R$gqyI=wM)p8C^C(X8h%}*u8ZL({<$GId0ol8z9mQ~he>?M?}1(1Aj6eTXt6x^sFEHF2^4>ifzMuSYF6{~k&r@4ZM+3+~zt zjLkM;`r^5&I4=c=9mgYBOidR-$GEMTq$kzKw2S5Ww$*?IxN!BpV|eyT$x&lM4DzaK zeK&_grBohcAh*PRj_x+l+kSF6e)iOQz#_2OK;R42{}ga6hTz!emsLG7+e!*wc)&i@ zAnkO4l2kZvC>d_MNSLTr)bdSnZ|q3Ss?rtV`PGw;fqr*NmkA)|St$5QX0mNEAI;2* zc&YbT@m*cp8E1w`l%QPsesqAmXjC-ke2(3KE0FT0$#kW#KpR?6aIo`C>l&lOtiDV;_H@triP}mpeQna?T7DR{jto-P zH6CXxkDD8LMY(=Alu|(Nnv=YOTdn(VPkBA-#NT`S295{6d_*9G$5i*NSq)jZVxcw_ zQu9k*NpBM({ao-@M_{L4Vft%E{i^Dwj*gCsWVpGNA5?bM@R=5U>I`?B&V9JxRqD4y zf`q|UOQDZPzSyn7zWJzU{>hQpokdF8|jnZT+{$|2#lz0Ghj`! zZhyFDHGrdiSBCHvA}p!V)FmhTfPH5 zCyy@k%ugEQ<+ORpckL+4AL26w%^KI9A6aYG24z(IXe>h9rS{zUO?o)BMk|o@^UT`p za7*1in3Dw31KbqD9{z{35pXqDY$!Y?@L=-?T#bblSb+kTiGmJgXGkxS0rHa4y!w-FC(MRNpsE!11-p;KW44i1Wv*|5*{8Ov>A| zv4~9}AN3>o>-R-?w#jun$*sw|CgYhEvqfV(jGYpv;$D2?5HyE0&;~dzVuh;D?7Co5 z-H+Q&6XZ_K2bh~(IqN ztdo8gmORF+OiNbCY~iAk;rh?AfJXGnnwC;LD53PgNsK7lv(I)JfUq09rPl$Y>Mku- zrajP9=%Q?3{ON>oC}9kjE2$=eW?>k=A-3twkSr<2Q2JUUz4g?L1kpC%oeW(S3)&S9 zA_e!!&o^uYO465yq|8Iz7w#v7FEHg#);)ZY$fG+4I|}#E3^~>9yq^!`2+~h2&1DIiEvO^?Pe77A{ zmV9MTk@qPgKA!AwSh8Q0h15nM55@}{={kE6;d-j^?qhwG$x?<9OL^>^#n{)qvV(Jt zS_9Wd91{gPj>QXwT)(Q6oDAL;w8j&C zf5rZCD^3K%z7UE`xIeaI1>~i($#elBhFxIknyId1liZl2```9uRfjLYC@wzuGAGYz zIyR*>BHLlZj@?*9jaxyd(W`n~Df>g7lp!Z4Gy7c)U#?@UCUCAM+tl)5BEJ=zP~U}H zo$POLOh-$ei~L$Sq}R_X8N$~qG1ZxVDyC%3amdi$(89hey=+)$^Tyc$S}R``Qm^a7p~wXBmxwU*#44QnCQwm-=Ro1wPvf{OB)OI$@=-V z6tRmYO>eE8)oa1Pg{#Z1u=_IWuBM)xTz>CRC+hzBCW z1YLZYtRpn=2zk`wJiN_q#PT+wqy%I60J1uQJgdIY4H#`Zx3Xb%!lLRy&pA9Q(F0e7 zp2m;+Ip{{b;XP-Cal0|Mt>=?mO@*r#rRskgc~f5}c=<_mep;erdV2V+;#!1rTg#W4 zUPvi~x0i=W@?R4;;yk|>6GAN0){jDjrKDPRvxEl7crWQ?&__%1;|uh$5PiEb_b&zk zYJ!#WW|L=yn=|2C>geJkyipSr%w*jej5(l^52`jsa){?JgK9Z{(Cgl^nyRYqRa~Ry zgV}2W$FCQ+5qLvai$Sde8qNwcL)$GAgYsenB==a}sb=Cm`Mxb1T@WgqecUX;u3x9W zL*AV*D9E|E*m5>F=Raj+TKwPvGWg$jLZ95m7uzqVB1fipE$74FXy14N`Ste0wed(jxZLje5$q8xFbo@Z#+_u&^SVCo51#45JCc|11eL#vbrY_W=2`{rs%f#m5Y(A*6NV6m zv=Ix2U??IjKAw>Ws0 z*n3)tJm%!wpxv+l%f#0h8erE(@GRv=G7efKkfQPaS<1v>kr_CY^#ROA8K;&07rnQouf0A_)o;_gd|Wi< z<529Q+oG-ya-NpsyuO~Dj6fzo2Uf6xtut_A^Z~~-dN_0+X(}kFXr%C|Vya8sqs(Ga zliiq4ZzELuY(v$y$=9H#2G4p0dda`JB-TXVgy)uKN;&=LRxc-hL8Zhjeo^3`{>can z{EwO~b!UByvvZC?Z0aWTau%PI@WK0j;0kzymVa;-OiD7?8hVVQl|kHX0U-pi_jeiB zU$_QYW49-`O$J>{^CCUumiGIk4K`OMX+RbY7C>3_aBhV6eqZGP-CSS_?%#{pE7(9 z9iec4m&{$*GC|)#h@N3XRQ5~O8^`2uBY8YeSIaaq6~=z?#eAj%D+C2A6yF0jp=46z zp4@Z2O9Rl`WTt+Cs!GN7`A3>=>mA?MpVb`MDL^q>U_VgtEeNa1$dd;TLZ#-dl1pc< zknaqkP(OuV-Kdcv?s31K-Flb1gPvOg>e&3s|E)woL6q7)0xEScNnsuEK&gfDw$A1y z&&|QAM8-=EZ60ENZu{9ELm(iixCXd%`Q@=Xs2rDBV|LZ&i3RCm@#;fH8a(f{YA#VC ze^^4%Sa3PhmkZLtbTSX9$v{-@XgBrWy5{YaVmLXD*#^Cf%B*(e+XXLvgjA=>J(*63_j6QvGk zBg0%8uzpNL_4|>~xcGQv2IMtp%&K!cdaiu+U}otRtJV&5vn%daGi3oS98}{A2iuQSvZAwuZb-__4Zdpvd^~P)xGe;f;GE_R zSO_AH29$yF5!Q~rk=~}jIOGy&7BY1qJkr+I&NDBK z454wdI)BwJdD<$Ox5aHx#sm20lJm~yo()D(9xW-q6r5w z1(~(PTh?uYH_XzV2_3R40Qc7+v6&T{MccX7!Ic$45pxe4CfYT|5+rz_Aa^ zNuRp`ZUOSxdkfjsQDNP%*dx(!Y(O7|H1oNo^(%5e{V`=~iqpl{yO;R?SMA=YRPGCf zC*T8sVlnA(fM>k)>N}*x!4X#?0f8vE`sCMgASJdKuFFT0dfmYN@{G;|PQBqP69BI! zYetIkfMC)g4MvTfQA2kLiTfqT9da8tI~mAB4X@bQIwqKYSpQ`(U8px`V9Zt-X`SuT zd8hMa2cP{pjExycn;pTe^V?`=b4MwGh{T4ML7R$-itFky`%Nx>V7U0bPKhaUBbYyD z0E9-|x!3@8kg#)M>-lcbc^^lqwz~Q+U0_f;k@67~7A`!Epf#Hv^98lx1L6m0 zD1y4Po15F>&LWC~&!$jiHQ5T_*(6)lAK`>Y6NJra2pI`@k-V~BZ=+>r6)yUX&6 z#m);OL#%^qhvjs8bfdy8g73QfX`fzt`itp+)1@UK8K~=j;jfa9zW@#lY?!P7TtrV( z^W9s)T=zhLNd%}fGzp=~AM(=T0&DtYAc*ic5!>44#+$k--%hRodf4=L=8NTK=f&lE z$+((b&c)PMXR!vc%rqu6cUnsl3~JB?oNSTlr#Mn|UXn_6S}A3Z29hQQE-0NI{8CQ_x~R zHJ#$3;IW5(GB9Xk!A0*ml+kD9&Qw@b$V6fM@PN->tm@Tf_^rR(qKH{*xrbe+uZb7R zaaNlVe&yO!Hmu4a`E+CYyy)5!MpzPifJnHdU}=2lYd8DW)aQSnO5%^Q!PFta06ZAJ z({iOcJ6pR;Nhv3{;z2=KS@zGLH+-5zRNwX-r!R1)@rF^4DtsLq(o9Tb?QwJ*TsRzb zZ;B*9nVKow>;}e>gI0B~*#m%T5%d<$83?yDA(sCP!+aA(Cs};{T5wloI+9nQVfXWMiapA;kLX*O|u z*8HG35hln$lYnK((};`AaU{D+O}Ot%=Hl4cSnbj&xzsPBPtRGZ0VAYi-djka$ur`VwN8LJ_392g9mJx-xY&8}+F-9h0m z%*_{;ZO?fgZA8}Y9Br2@jN(|K(qvyQ43^t#PN=hP6m1MnF2E#n&q9{(`kNO9wxBu2 zMjJN(TD0-!{ck0(lDB=NdkTDwY6OF8eZMB#t>e#bbP+1h&&w^y zo3nicwXI2FxtS_lAa3=7WG?4)O+_=qUAOMVi?+?Xuk4$<5nsMM@Q?%)5&crwucch} z*Fprp$a#$OLi9_KC-umXce;^Z94q&sNpEwK=PL@4m(!=H%M%yA%RZ=M%!CmY z^yoI-vp_lzf~(REkL?S8%pX6c^EMs-4fe;bZeV7VrPW_00-(#xp) zC71q)utjf(+|%|MnT?rSgD26+$G5bQA%U1@cuf&O7ssPW-RaSbZ@DMDjRN|=KJ}yu z7Y1UV$^2T5qdaEz-)P-D;Xf3Dh^T(AI>f)-ot_B`F`=pzY$J={(L$vAsoJIxA ziv}qU4WDFhr5Uln(oEP;JjIE?(t1oV}*KCT2SgMt}Qh)OcnoCOD z%!@1Y{R)N`rrizs7JgdXkg!dI(BO&{H^Qly(K!ML0ZR-~6odSgyQIq}APolcs1?WW zI(^>$d%81utXkrRPgql~U&ZNhE%EM-9Qw4j^B4T z_1RER*0Xx-I_8olDIh+NWnQt5=OYbzoZwL3sK;}2GN$009Xl6W%&SyNb66RsKOF2$ z7<6Nh4dSy>mFZS;o45{6Z}W5Y)+zohDJH!0>&J+G@&4dVqZ;2pf*t}687P|k^L8c|SXw?cE< zkD-N7YXzALv7#AU*P6la<_GE@u>1a%k;|ulA=xtbDL1P*X!f!Sh=vo>qm>(A=jiBO z4B&FUMR4+Li)%0-gyae65rDiV3Ai*h#8@cwHifb+P=0$+o4FKDyHqQHJAKI`Ig89s z(&PMF@*|t+u}HJxva;7}*_*?7E8aW$a&>J&H}>%{!+_xWFSaBU!x(ow31VfKX;AHC zG84Du!q|HSJ-wVvobo=JX$ggsY2m<09j?Dl76su2ECoH8uYtL`>LzBm3oH+2n(??K z-EWJBEhgNpQ}dI@OEO*qriV&%IpD@T=zD@77#``cnHj%v? zQtU6?^}2>`-*73GzhpXAO4b{OeA|*Z6qYbrsBgF59I}hD+dcKU;@@l*2bU zuI#~5E>lOb*vX<5=+A8AC1wqDmJmr~fW8pcWvgFyYBqoe{M-=r+?ioze$2Tpu{(N8 z!KkzwG9f-0s8p<0iP^K?m$mU9gl4i`Fp9J3nn09uM15U0<8uD+TM-Vl$c5gisX@Q$ z0mt?AL_Gr{i&DaO@C$gVKABAn9fCA;hJ?VXS4tU9K}rC*CNG&Rf)S%uDuqEpLNe^b z@J;ztBm z1-J|jV9%*AC3jvG#hkk~?u4m$r|eaAw*h(#g5H=Q|9<9i#;-syj!G0Q93zpJ>KJah z(_|2r&83tg^}(rWOAvQBqhCEB*2S5(J>QK!c7yDnW{sGTM7l zs!fW3ifyxLY*gL_*LK6+^`EWKSiJ)oFd4_lM`HX-;>%Elyv}VZ*Vv4oN_5^_j|*GY z>u8A2DVsg^R6y0=@z@}=!%o5b%NUJ$4pKxM<>1(;w~a(rPoy>*vFC2L87+a)I7qa3 z`a~gDBeSKG9wputg^bZgD3qAKC_^! zC)@#Gi+T6=52b3)`M}lmFZuT=>C8?j7&NV?4kRQNB5XJeYK^(ao}CG$AfQG8mgUCWpy`G=ECD4A-ucV)Y^hWOz2R9h z$7yxsc-)HoKD&YY;{-cd{qyyS`iShjPukd_Rzv?#MIXecMO_1e^ul$=YuUb%0tY%B zSs{I|u71@2WP9T|KHn6-wMum~jwYbbrjNf90-v-m^z*WX%YT+&_>6wlWDlty#U&G7 zj@_Y8gHe*XO#LDfw;decw7GTKEZ{%Mg2V+)xI`Ha$ zV(zyoJ?J)5DmUCGfT%DaaE%f4Kp>Q``B{c??^9FD^dBR@=gK}Zcy4ySvgg!~lBwfI z>OL(;H(gH$4DxM?d^z*(-}ujxSKP9Jbv9#Zd>}s^FOT9Qi0lYR-!dGexg8`?SlHQgB&JZjuAAw%BeloO36${W7 z|MYFgehm$S&*Psw!_^d8GkJ=QxKNj(yRh2m-nba}FyQh>vVFG(GvV&82|D?^;vIG(wgfg0 z3xDKKlt5k&MOHnsm4maK=uNr>gUxh5`7WLSJ2fq~rZUlY)(61|vT~`oBWd(kZ~dQ? z+6ldNHP=knZYBC5bx=T2$ZpW2?dULNq?3=ZFc(pG<=Lx7->vY{mOp=ay}YbI`!n=Ircvv*(0~w2^nVm%jd@PI1=vJ?SY+n>aO}_c zQvJ3HM%9Kozey{!kM>E=1(}9*u0p5IO;A0^bm`CMs zhA4B6yZS^BGl(yluoHP zn5l^#&O94m@GtxSzOybI+f2?-cf@?m(>|KSj_*ASk7Za|kf+A#c)G8U-G{emjyx9C zPWbkI26{L#C?U`@wLl1Qrb>mF42_kRT$wt2>**{X5rdp7c@0tVm`jnmCL{lmf9f2H zf3b$QvQ`E*7;qrm^4b4nkeRQH%S8W03>`ER$_}<#3cbvyn{GAbviICvvwi!V*bVCR zug#OAtqOzwq9(w|*Pbp&s+?@_rFM*nr{`V^+KuYv|%j)nd{o^QH69qikxV@>uR)M1}anb-61A1+#8ut-bw7A$8T!@9ZyeR!6wS&>VvKX)e2Mxk-mMi;#R1S9W zU}BlGn(C;U`YLYjnYT}^bM!c=-S>Q`lk{Kr)91?e5GhbVvWg1U?ri*!-O&N00e;(0 zAM5LMcp7Vpg}CVUS9${C(YxuTc3FHJi;56)pkW*$n&cxh*H;o-5<6$9a7-n7Ch)M! zki){}3-ScbCch@IKsdi48K>^eAM@lGnjY$^aq}5I-vGZgM$eFQB%{ppAn?m{=a>wX zr`!>u|fu!l z0B!m28vra=?_HuAJtel-*%9H{ub{jO(V0h0H68Z0<+mPE7z(zX4 z8c=1&tG6n%2T!weTxZUob{6?D{J;=@+?Rrf{hycqCLEjKI-%l2*#zUhz8FW!>8=Jh zJUd=CAYnG~gpK>2`KY3S+E`TYnn|qm(Q|zZ^sM0_(GW9Rgr)r;$LIqXG0*cqV>;Wq zfcs!fM`CY=2j#M~BC%1VQ)#mO?G84BXo~=wTQ+D(0P!3lAiM{+kbxhc4W65CfE84~ zvzxv7TY8P9(={2`gl0g)r3%F52dk2so@)(79G1VCeA zOrV5zGA=7B5<~AlKf_HMKKmupud-w5KJvG6pOhqjEnC_db)8*dt{gTH@#}eo(}jMv zz$=9_N~;7R1a0`WgdyR0AszXyU3Ud}80R061R)Tay(1I2g_?2kK2ZE48;9IMk9ciJ zHx>XSO*lcNg`Yb7%IzK5rlCN|tK{0Wuc6lj!cv27$V=l!3etZDfEAT)x#MJ*dM9&8n>W+^ z=ipD)g{TB;EK_D9?%WSjRU9pjeRqhZ88s%LV4YLaK){Nh?N9wbhne4RJb{v&hDy!e z`{Jyeyw2Lr3w@H++UN_m^@xw=kWER(10Udd0wG`rGvnr6DbW8y`W zC;kq|MRKi%CfS;}xy2p9R*T-00dD`Hz?S5E*iIBzCHyzK?txe2Q}sqvzr!DQ`XYCQ z=lSIU7=8k{{PhoZ&!MJf8GKlVno~Ee%_bwlm2X^P_*j7i>ZcS^;74CP`9u?~5s&Qv zq!Q0ZYgH8OuODd2?@7U9(Y(q7HcBZ8pdW?OU{M7#&q z{;iJZS>p5{q;IP9N6_r}dmstIp08HiLhVoZSpx6P(VeGRADhY^ZtC5SZ7a4~S)ne3 zRgE+K=Mi`bA>0`Zq`Xb3jUvfiGL5w)9|aSkhpSry5g&jr&s*{Q&-?%22L4H(H&QSzx(gH$ck0?K{(aF}I|&xL-8*W>-6$?-$J;YYoYN7rq!=0ETN1 znpQb%MYbwd5>VOwwtYEkx=KGf=fP1(B`=7*gZ38!s!_2x6%cT-a~Kvb9KL^0DL}HH z(gIXWqRZLuaul({afVHyz<~Il{ZAJwWyaqngC-z-XJHXK=5dfY^p+eo>4sOf-FCg! z&_!#5=ATT&-CC&+BcHsUb2=^K{%`d5G1n_A>;x1U=xJ zxMf?Pux%wo_`fd~0E9%}?Uz)?0><3usgqLy2-?Ln>m#zt1dS804$`9^p{P3MCH!}F zB}g%;dVc+U4yJLaqe-PQ5ywg}iGvu#*J2m0sS778_+Rj z7XE-}enVEP?xY%hE!NLM`{R1eE|Qyg#NezXmanRb0M8DN`K$)#5AX5D%;)Z;Z>hrB z4)gcW!+&mD-P1jmd{+tTjSa%=okReqN00uGz^S9AjVo}qblm_{HUdnnik+Yi5($cm zFE-}c10%NT24%UYlRkm7feyz&mJFohLk5{Gn)2V}{T0TkD$~Z^;d26(mj%0Xe*D1b z79#TXIVZ)x3mU+JCqTtza#*^Wzh2X*$L%A;c~=!UsW|lTp;I>=t6b{7sqWwuI#Xb! zL__B>*gKRXz}w32)*%72BeMZBd{=s`Ehj6BA!ab>zw0@`)Nd1BUpMSfTkvB#kD1`*t#*ZxVwypOkKR4% z)pks(mscNyf)ZT29>H_1Gxe|>s;7>19a$v6l^tz9*+huP=rzvc$e*nvl{5dX(V-fd zR~g?kYL4t{0<&!A;LEjm`SM?P76Se=;W2|7p1+tQp1Vhhq>AP-y<&GJwEX=;Q{YqeIu`}lDGR5W2 z`XNp#^u+(DK(|+1Epd@tIFFt?q%uiRYf_sv=xi0)_TR2VKIlr6Belvn=upLC zI2Kd2Is9Vmrd3w!h>4to9LL*F|CC|Go077|Ede0Hl+xvba_-uutgJr-4MBFeK7QA1 z=u%(`6z4?fZet7m9Zx6YSd;jZa0xHaB zXqtl!$V9s~4p56Bl|mEOb_O7?Rn&uS_?J<74?|va4-kOA>^lMU*31VWPIU@mM)$H% zjnf=&s^}$D6+O4&_uGjqYYL$;T`&+_bY=D1*~vi>zJN$IfUs@c=#vM)@+lnP{cq8f zi-t+W1sp_SL&{%jo9$a|(}0^Mu*4?0oRoA*QQrJFb#WHILLv34JHS!_%AfN`{WD9r}Puj8Fx z(l=k}P={?D%NaUZ%LkczOX1gq7#kM%vCmN(vcwzD9DO8j#0Y|@Y zUjZZgRZ1(s3~U+gmMBYV{odC<)BW*-9E3?vHH#FdouY zk&Jdc02y8A;r7RjuQt3@$2>_qyx_=M9R>CX^mx>7V9~O?WHkH_1w8B=-^ZsDrw{i& zwPF3lf@i1r7-GAqTgiZ9w~sr{d5cyP>M-tWDoW_GZlGykV7qY?W-pD@2XO&JwD;8y z%nOk?mx*U7@=OOI&^xRHxK^D>Uy`Hwtw3vrAKR)6wCcfYBflPOz|Zp3s#Q%3gB=$y zIGLXXj;>zF0lZFDofN|pQ*#fEt{VdAI@3l&w7nX9%wzvb_xQsvb5`5>MdM0vl>49x0wsPsuA z_gEXRv@dXmM>`zT`F#H+-W$CSw`KxDhfP@{mkX@5mCZM8RV+U0$Ljl*v(V&uQv*(Dib~B#;X$494MWY8$;9ym|9h{ynpMY9M>zJ38(50)bHc0F`=F*S{r$E7g2Gw7=gxkRpT{cX*}wJ(|e| z|2^>j(H|z;FZ|OexESQYWzH+^Vi`+cj9G*Df|4h}xG>oB zgR$aj7t_X=jzN%RDv9edGjm5&s^6m1Y=FMC5qe&y!;FU-REP%pgsTq7|_cJLI1^5 zEVVT5`&HBfn^yiC6u-=T!e;u;_=B(Zg%eT_lUP$UKc>72WB>%C(bCkI4AjdY#J8~l z`#j3Xsw2qzf$!RC_V%hJJFo>_110|U4racnuYTRmu4TtjlOnl1weNMzJ0R&-^F~(t zDPLm+_D;(VD;ny#BLY%>sSZ=nK{^=No4D6knDb7B;>kmZaqo>?o(EUiFhv~DaWLuh zY|l>kFO^R>MB2HoRopak)ZCg5KDI2Uecw$l^aPL2D3H>%_a!AiH}0TVUs*1Y4OiJV zHY6S7kd+v-@J7=lM1_!>_eHTwkk61m5AqHN`8}2$7)ohb|2+!czWH+$5Xq*?n3W?- zV8;>IOeIz7*Cj>r5&Ws?6ugDJ&WMTz&Yc~dB4DZST+`S(e?(h+T*61N7&b>?!A)EWqE&IPk3+7C-G>oVRIHF^VsRVIpcNL>RDqOVrv&;k6!w~&bG)CX znd9f!Enl|{a#0IU_Y~AY$2JQ||BxxY9`Lf~9uQv0_o6y$FqJTnj0ShBSGR=e@E#3(w zic)&2l~3^c7}hs~&()clp6&=J%SP}E$5}nPtygUd6<2SdOS18Z_3KaM-8Iu|+Wws! z7{YPQK^$g)$vySFx*;o`$s4tFYj^ z5PErl7SV}j?U_CMmygNJFBcZ4ehluNCcpBnsRCWmmcS*8_I<{+#U z))3|ZWy`~&GHNjkZYGAvea{#6r7z=b6Ukfvfigl+LpI zdN%!n<2Y>&C05EhEPw+2=r#JBMiVHq>6xDWJC=z1gA;q<18?gL%FL3c(&|&v zkZB;}Nn!DBh2W)9IiPW%-H%{Ivxz*k7UjeNZdvbg}rKStQ^jIisyI z-#F`;Bs>yGY;R>3$1E(bXijMGwo3HmW}!=ao3sbiMqQc!d)P z=7H8+m|5D2m0MGZRidybj<(YMiMK}xq^Dkl@hOR;e{3UoKW=|)0`0|)`upc|ec-HW ze_4-iP}o>ptf!&}PYvnPRFO9>o$?!=Au~+-r8MXy->-lk63m96v+Jp0_=+MOjwFG_@!24Wv-*0>Z8r;9Q--dyBgojbnl zo;t~9u28Qg&gJrtUCh*sTQ3zZd@U34Nk~K0ve5*b^GK2@yIfde{M(!Wq!6!PufA-J zd~PEVow@G@MvPr(N8*kW^`h?pH@E<5glq%fGgwu5u&RIhO=c@lnYQ;kHfjHJgOI%b zGg@vLv<7C{F|L2M?nn?Ri~g=DQbW^Y`H**e)~bbs4KHsO9{R>r7=uY-$v7nc0~#0i z=h5|5221ektNRU`=ASb$?o0RIq=cU2xG(avCmkecZ;WS2P|MNK&>%N2M+HCB;7gUe zlDK?eAZUr|`N7X>841Q^+Ro)LA6MC9;3{F?$UNZ%N!kB)*+Jf4`s%73B_-|dBYU;f z>}6OS;b;{qZGoBpVv=9jMuPaC(1H5YQN#upqNG6OrrizMP%>(@qu| zU#)vRM?lbyOG%D<7mVxx;s+wbB8O9BdQyS+36CJx0bih(@WF%3m94cnL;;7bj|;58 zYfJlb(zdfRBeBx$!|`4Miy5rlBP>S<9z?*E-Y zUz6$nt-VO~@4hCaNGC|~&%S1HyUwi}qOybo+H?*0*f8HKLJI`r1sGR(fchA_jnqbY z<%2^B;PSt+{am5(;bUDI+{w~4maN2x5p6KA*i*C>N#qJprLz_p%G?jpXFoAQ~*xq^EkMH+x$E0eu|-IkJ0gwAm&mGewloZOVH@Mw@=*x{#`16 zxYS0l7INmie}0y#83dcPJ!|`9mcs5>3eVcVZi{MQylGUO$J|#sc?WTS4NQt|wD)DL zXdr>7=`aa~tzOL+Du;!Q0XIaKshQ{FtH>+kj8^^bA{(YSA}Y45i?Qjz~+Do{qHrbAa76G zFrcH&WW_*g`F5K2l87|EfT0UtE`DpX{>4bv>ZhJ_C47O!n>QaaOXFd283rK^5Co3YQ(v?wCNurO8L;ei`>UKgO@N zMSa>$9SnO+s4A+{#P;SxglfmeSJdL>XXYK3Gb?s^Bui!QS-!pc=xlVtXqBRqLZyJ$ zb*ktH%U+Lfm3xYvC>Hh-`!~LaQ|hu^JJs!JJ<&kkf`Y^p z1kc6p)7}Vv4v((FkGX(cvSu2i_V{7CwqOGlsy&9q*xqH`QY(Tie&pU>W_uC~ms=P1 z-u`)g=*9_7c7Q(~xCzr>6;8$a}BbTZNYEo}MKTI%*n^ z(0dd`!6v7aK4?`r{?Oe2m3)7!61K@}z;KXeJskJrBsXT8!eEQCU!&>h5oVLzuJ`O z*4b#PMb#1+1x7!(SkzD7_3z})*ZdQmm|fbSDvokU(0g#q+kR1#7_TI}Ft9t<4DFt- zdu3nZm{i@?q z9=nr~q0ds@R~(mKO~PE*pl_FEJ!&EGMf6ua=ZQg>PC7>vN@G zWS`|x4uE*i{yK>-SiBKBCBT=Wwsh?J_C$6L}dAyPod7J%`45 zPPdd_1eVYI4BT#=SLvBjr@NKh-}_Qa7oXVZ|LuK4n!ivQN|+WMnMqF570+7*^BlK$ z>mZc9-P$iSdr+x*R%Q@AOJ4EF2n5Hj!PX9G4nmj9*n1vS4fqrni@N9`ibXlM@eexh zEE^=0&OQ4$G_x+r36%7$#-vN_3hJGA6f6-V%TnZ)I^m!+MQX|NiZazG6hz z%;sgGs#qWSa>6 z6X{TS^Skc{(|}L&FSsOjng`D^OWL73kzJ^?HXOINIX|_~Jq!C0$`e49e={$RtjfOC zS&)7EsI{eV$7>?06Xrb58C33rJRRC3i*$RdwU%a`wBpwi9{6AeTh}PRV}vF3G71&K`BAX=Y4t+iy2EaaY3emsuI&r4~pEI z3}*Z@wwib^Lhhf|B&y5$hz4F%x;b~WBetIQ!nW#oFYJ$ej(Rq6zU?&cT`W!D?39Q> z7pr>%j^0LhwX*Xro2r{0mwOF{*~9&g^xAoA=r{xM(ug(|V8~AGgFRJ{%+ranaem>S z&T!$^^RS&&qOw!nZ3m9jFwGF#Lm7%+|8A}7r@e=A;c{>*HX1Qs25fY#f_C}nU6JL( zy#wM?F;&t0LogwqY?c6}J>21O~Qq$CIFkX9)PNof^nkdls(l8tmr z2&gm&(#-%$cQ;53NW;(r1K%3<-tT+P?>qm=fH1S375BRCE9@CORa5XHC5{c(yssJ# z5-*?=tUXtaOEnkpk7kHCp4XoMNewV!NZ8BDDgt41Q-nrkgwiv&%=_1`M9gvvw(}S z0*Zct&Afw?tcw9vsujKZop7jczq`{8blOYBuXM5d#lq`Nn$xXQQ${a-Hr^W99!mMrhaDE6$=1Q zs>S^3oqKn*yNAO0&)AOg@6{Xc&#g&Kch!%T?IMWI#)eUdtBW3feYf8QX*HYoToA(9 z83lCpT1@r4E1ipXS)K3Cx-6epcf$v3JId_aZcOn*Q|R3cpuY5qtz8!4@I$>n-m5Wr!xOcv zHYe%x?ZvT4e-g8r@3ADqnoma>^2e^Se;vaL`t%$~5A$^X<5hZ)ob97wBBHV~bNEPy ztk=4gZ!P7yLp6&<*RZeNN@t&DuIhAtheF!s!rJI^pc+bZfR77~1N(i3=*>6HZeMqC<$S))S|ltTgVU{YWEu%jGmVnOZ5Zc_CcUA}Nxut;>*+0|m&(BF zPcCy{yv6*wTrcfQ8V0YPe!W6Il?-7o`Dyi2v3hmZ{w%i|s*vm&j+FzxecwdO7jyH$755qDaSEW4B{KjF2qA3qZSego3M0F+@Cc1S0Lfzr zv(DF8$6YKX!QxYCXf{Chog?wxh4ImGzv`|srU)*VCJBDN2li9?nhhYD2ZTQahfyiD z0nUY?F{|(7o7(TH$aAbzZo{(vowxZV4*0t!Y(Ks%-mDa}x714_j+)$;B)%)g&c>>u zr&qmS!yqh~v4YH&0wSWJ}r(-N!N+e;&20!Pl@V7PJ#9GJXx*|=#YE)BBP zgKc-Www$MZLkg>grzTjboJ+dDj_M#lv{7K?>zlJN5$N9j4#peC?aZ^vvf zxL(1%CEbee!V|8Rf{kE)Bh$Xk_s5_lq-2F|QLvol#p%iN8dJ#wJiP6d8(Jv6dK6l` z{C2)K8i63>@+B&!v(L(|95XO2>CLYwZgu5CM3cD3Je7`R&csY3D@bvZjQ5A(COXJi$Gdo+y;4{AfkZoQI z`oMH?{DJ%c$VyFuK{c;xs((HhY<3JMza{x510qByIGOZ;Tn|_le^3RM937yLdS&_J z2mx0SVk~dtvgE6@9$`C!L!cZqH~iZj$S(@Me^0o)EXA0JckF*SI?PS7rkV9dKLD85 zG6IMs>-g70njM_Ws zTE)R3Y78WoZQjhQ)P`Xo{%mjS98sUZ??y(=fasC7iPS6r_CCrGFyuVmO}I=+k-@wI zlFt}`HRd<9SYSi=Nj(x9(F)I0S86E*@BPZY5c2#>q9a1sJ|`z@z&AA-&C4*j#|}>wQ8rwq^Q60N1@J9krtC!K zqYjwEc`$TZBo0W6WaPUYib3Ubvl5WzfTSQ_(nkY9)gRYz=8}a--5x$W3=KEf8+dlh z3i$yE*SAl%>zd>-WrZZxjNU0@KZw?@(yhN`iGlv5lGnoout3>Hz(A)z^${l$%(Trk ze48*EcLd}V0L($?48J~G?f2ub;Z9f_Ds|CBLlxlD;`GLYE$frM{9lHDmuhuk64yH$ z#&!F-w}|PNka;=4DEbmXvETc0P-_8 zw<0`UgUiHS zK*I*0U!l3wz}{mRkmS$?X*`gm2`Tv9O~5zYd=4m6u@q5pZ(7da0DjY)^4c`4H9ga) zLpDlpqI0r$LbXA23XS7#+(CncYd{o$5$|iS(3tf|Jt>=NqO0hasD4V58k$te8ey(! zq{%{7&Y+n|@H@O&NpMhV)C?eZAQ1#uWIa=x`6l>HCJwN}(L*ruxv5z8S_*P^X7@e0 zUPaa?oD@q2yj$e2aRaN8fkT=b?ki)@T21ej==k6Eh#-&r=}LJM^SPwO{_2SdQeeEU zf|fq)`2w*Zz7^VXLh52q{_J!)N|H#>u*xoG*LT+&(Z$~HF`TUt2-H*HXPM7+^En(+P)?Jbk&y!OlJ|^czvzr$slp%r8J^zW zOt6vasU^R1`0n41YhCMU++A&_dHz;M6lz#(n(#%>-hWh!c^q^MF6*=Fb$FXABmoK~JZJcGDqq%1 z0@G*zJ+P5F@-{?%=;cP!s&KVNK!#4N1<6`$wjIibBRmyiZmXwq05UokG01Nr!x8P`UX{jN_rU;PntqmI_qHaVbBZ|K&K z9EaWQSy@?T(T}Zq+NAtZ&mnqkbfrL&x?xMi46_LU;pw2gpJxhYPoNiiBJFt!5(>Ut zIm#P#7w!UdB6EGD*V)z{cd{?Hnwqq7Oipp44CfUPo%mckL}ex?R7(RS@8pr;Df9KQ zQSbkN=?&6(bCL)wv$93>tfKe07^(-imN_+Fb~`fc_iDOXnJq`K*zBJ zqfI}?`-b;Vhk{g6f`Li?-G5aph^^yPv_v^@K2531zU&8urydt_IP)eZ}!J{!`@I2xv8B*2m>L=!kxRZcv6 zKKm$Nvc?q2m3AYmLzXK5vxZ@&ND;N+y0&|id(48Vw}MI2jXNO!&^pGdw>b=Z0R zYdo*<*F^Fg7~+;=8FNN-}o;B9dFGQrQn`({!t>dTyF)N=F@& z;{pp76^=?$KOO6k_~vkn@p|p zx}EpmKKD~Cvp9b2Ao5u+EZK24> zmpL#uIeeew4qVey>ItE4NxY@0AJf394+qZqiSpRN+VFetV-Gq}=^Z==7QibppTuJo z5zz}31f@Og{P znB${U7m*MAt>Mwb!_7)a7ENpBA4R#go)R^j4Q66S9*FnF_0KGsb5U%*2H=~n& zH>eOEYaY!CLTYpnF{oO42L$+}T&N+Ecw)Loa#06{zbwqIeWGld*prVJ%73zT>n!P~siM&^p;*#qwrm~q;mO}n2FhvZ6d!=tK#*<; z+UCa1p>Q_&>|9ttpj<-of-B8L@wrzj zJa`jy?N8fuJzKz?NzxPX%Yg2w{-V^5r2DH=a{q`v@?ZRkDFm*^Z%d&@`u;R^cmRJzW49k zRbzQ-t@Myc@C7nFnz0x@7eB7|eRyQ<+x*j?FR2{%r}((%06S@8tPs9?b-BwbDQVo; z7FsQ)y8iPab`BH1tAwYUm?xTi*4DKgoAKj?Gc1MmNnxT2D*N4fUSVAE{5*#-R$_w; zIc7N-`VkmW;eN!=$wbFZKTKnA$b794k%=+?v|^ErIsmgHl1}TyjEju=f)w^UVuDMr zxD$$0zf$T?mps>{{fqL)-~c-?hmp$!6w*ReSH?wXU1CE+L$y>0gt4FT&wdj0EXkpG z>J9!qT&T!oy95kPagwWnjO;rC&X8MFriOt*iVc^rPn65jS@0TBH<7ue@>E;ijB2e1V~dJQ)-;yKp01?m z#1v$3_gP#8Z+<9Yko3x$mQ24RWsJ=q9TwjEny56zperpAS1YHs)t-mkAZ?0^X zTyqJSb~CO;$AS7nLC-=L-Tu{^ngN4sB_M%g+xDg77w7+qyCa;m*W9eJE9(M`dYW@D z1Am`kB))ap$p!+14fT6*!{#bkRzsNi)3x#-N%Kt>dqbnfA?;;V($^N5+cz)`AKQ>X zi9MK0@kkOs1emrP)+y{tWBr7BoCMts1>pA&WqJZ`gLHCbcv=TcnTvW^l zz~e09*TW6jk~$cIPMH*_pP06&JU|j$_`h`I-j)q?b0{YT79tSgnT8$MAj6E0+LDRg zjx5$|Co)>VcYLf5?#4~ccEY`B8}Bysfjk@}TgF!ERTpG4t$prjGR_@q67*>QfQ5uS z`f0Z~v3tkzP&G&5MLST-`J~Y>fX&{WvHjEI6n_ien8%&(aC<%_W2XQ!_Im*Ay8WKo z?3Oi>3yT2%Ci((R38F698Q4{xxFl%n=;1_8-U#I28JsjqZ7TjE>m@Or&9xTVFnhk8 z+E;8Fc`tE?>WqKmXLAp@7*KEl3DaLeGFY-^_x+#ls$iuO`r+36U?(g7S#d>-AfUAR z(Mve&cXhr%mgT4*Cl`R6&&}|oJzcTafHJcVPTFzB+_u`QYEmsF@O&(n3D`_3?`?q_ zNy;!pacX+Z-`Bb+Q7_ruJkR&S<*`BL{xZ$@)B>w}H+?5^bH?0l_`bfIHMk(FJt%hc?kjQiaI?9~c<0HRidD3XfFu^Nrc6hiZe|M_%;h z`?EV~XI#qKu!!i<+C3cFrGI!mMGYtb0D9o{01M)Oxtb#}h8-s06h$WwfSwuVwP|$4 z%-oNJ$5hlTeX2@m8S|T@gN2Fo`S|o3-Z$P6=KwC(DCHgHx$H2_vm-W8HM3{i>MF@A z#~$kFcH3qI;QVu~kbNmnB)CjcJn`M|&|eIqQa8bf@)hEorTn5;q2L~TsU(wy z+K+92nbRKNv{3v=1zb%;Sc>S>Woi3=%y7q@j{=lV9h$YNPd0!%uG7%J3a|Fo-55 z)Z3c_Ku zKQq_w(#HaiBeebvq}kmEVORW*#NGM;Pm!ilH3LnDnxz2XpA=ZB^rKwpyWfCifDBe0 zo)jNMJRH0A6(P; z>GUt-9)o+D?gSdxzioxCSxDjr*V4r#Ci{=CGw z|L{{8(;%@spf%u~WT7MfZi|?v_hm4Tg^($$KmP(b@B8%MF`^(4^kSg;M8&vzu@@q( zy%5-ZFmQD-pcx8Bv=cZoWeiAP`3%E50msw8kGKh+((2H4dDa9-B_Vq_I4KWJB;d2A zrtE#|)|ypsD{qSq%prJEv*1V_8avuBTPy$ezs~6 z6`YCe(n7K5*Tny%IRZix(BEGty)+2WVbk&no0V#-<w#VLxMLdOQ!{=}&lb>^qTrFnHMR0z?f)#nN!P!b5PDpD z18qA4Kp(A4&%3=Ec8x9|%-WX+=b$Nec#I3cY&7doeV)FE2swcEaieknj(z)&*$g0f0;?oo#efTyw~Whz8$Ta-xut^TfAMsP zjx|jksFRn7#OUrkb;2#T!6yh=oG1Mzg9XZAb(^qh(4MpyBfmxYv!Bc?4!rsc19ynJ z;gxlHoO&$FWvfl{=Nd&_ajYCbt^9mrclABVfa3c>&k& z8sOf>{!66~7CoP^Q-E6mrd$(NU{y*IWUQIWf!vg*0i8S# zUO!X9dlu*463+l`CFAQH+|fyR$4Wy>!J4;WG6ils(Zvl$G<4L_bx5!EYHDIM0E0&9 zWaq{(`_2y55;FZ|8=gY6R?U-tXuQHUf4AC8b94qIMpzk5*6Rb_r7A+{D;G8&)w;)tlb zD?%nv(mZ`)6$GhDjNYAj(SL9lcYCb*%WMbMfD-zx{8#hNG3a3_PAF*Bkl0X1RkqoG z!?O1#1YLwQo{*|MH=9cJZI(l$Pp{Da|7^mzj1xkYIHmpeAb)ivK^7np;_gYu;M4AvV&Tz){Kee z#!{Ob8?b^6p(Y+Tb)-(eXtn!CQ_tO4qc)5vQB*LsWNbVAEv5t@z5hB2U9W3Xe~a+d z9ZIoG3t?WZ0W17P8nZJXdqb#=9cvVNzd;I~V*Hg>S& z86#SurJ&TV^{RUL`qFy)S=>i~4<}#Votboz z#$&OXTRel!$re?D^3A&DIL_EOeZBa`NHOOr{h3o`yT^x{ie#<5|Sr)$pW)-;o`Na3ja;u^;^is*My zVBNTvQmv0h34U95a#R#gf8+6N zZqiu@<@!FYOJs%Zx^wQg^>kqs{|`Z}E4{(G{ucId0thZssaCDo7aerD6TM>A*Q;G9 z6IYx2D@<1LUi?5gk>F`s^sAS{lzlBab4G>L{w=cUpZFRR5s~wl+q$;CquY3PFZu6; z$zX+;=;)ZicDg-!2{M?8s`6Nnqw|kDbyf zV5{-;DM)tVzRR3-+LA^Q7aP0e>kN~jxGk#SSVdALdbjI&z%l4~NaqoOp`zYmh@N(_ zWOccei2c^S%~b-sud_L>YzT5_^W5mzhIULvwQ?$&jGDx&O-$xMLo2y)>5$fUKAh7E zGXCdfT9(W*10b!ZyadL4=y*$2$97N)fwCZtSCU;tQ?HB5W;VTi*LTYI?7g3lCF1oTOh>e4gUtvE|YTpW>=t?aV2n)6_I&oAbR)d-UlU z&Mm7^mV8*b)^u3dO8az4_8(O{YhgW3fcLF#ant;0K!t~b#dpQh-6BMZLo15$fr#e; zGxH#^bRb(SBPPi`8gfm>JnCHBtzVpjj1gUN?3;&AH!ZEe0e=x#2Pz|m$jt^6D9y4ao9?CMNz=JRj1g-0!(K;mE7x~CcybjJ-0>u!3F&!ULT&BopG?Q64_ zv2QH1fIp6w>|=vs2#Suq3lPx7rhR`0c{Zl^rmr@F-y5{4kUO4b!e-@0`Hh@&D9L{8 z6xlJ8QqHUK7OZ*p{Vx`CEXkp4gYoV@Fd;0a(qWSr%R5&5xl&tgj6p0EL)a|AkUaBr-)s&QkIq}_iJgXP5L{6tM~(2oNwDOVuskpAJcbK%L+ z*BzVU>%D@^4LdxJ*IOG%Fx^ zRa9UTFSU_6=$BH4muMA%Abt=rR$SE(Hto-&4}?>(jau{bFpV%l20D~cq^Ts&(RXI` z*AJ5s%!9v4s&=Mn^QE0%C`%hWXmhbp>S{fKk6-$pq=}eeQMiM*t`r|=TSO>`M|yAY zbr0}|M#1)13v|4YAzON5th$isDi9NILYK-*M7>*2qa zp0oVV35nK1&gK;-?&L@e$2+NY>v1P6Ra>g;&rUhJ{1msY*UZyXZV#-59vz=__pQ=S zCD*I3(w%CUs3)`csQGcw!1bH-yr!E)!4V~mKB8LC(;QUoDZj(PQxF8B5GT>qr^i{2{D#IsCH^db79}x;J)7zO@P*~BcrmnX!viSWl zXlEOIH=%cZd|*^PEGG~QOP}2Y-@^X;TM6Db@o6yQgkhK@HFNV16v9SU0-3`q6|pQQ zt5oWF81f7 z#lTVX{aB%%Z7EJGf$qpl91`6IVt2<^ZNy8HnEmDZM8hTx9!@X)Ew-mCn}TVurXlF% z$P0G+4tzmJV$wVxlH(I4?=GR*{cYM>WdDqp55%|1$8|68r~+shUTUH_cg!K+`<3dckN5kLJ`S}f%RfrJ7NgTzo93+C&x@6h_Y#&Gl*utV_FnjcrdT~ zDsuM4LAKvdujVzDd;giGK>rm3_DURy(sWr0?Ef6J`%yQo934X6jk=jdT{U`VtpuKA zdGx1hBKPI;CEm%@Im``{mS?_u=iYr;^t0^GDZeiy#rwB>x#s2@! zHgkX6L$br32nnF|3f@L^M^>97$Mq5Gp?93?4jU@oReF_P1b&bA-q_4?n;@+{p077Z z<1jvLnw9d?p0>4ZcmejtSr&bL&AJKB!QM{OW%Ly67U``b6dc3*dk!{tMS$a)&3p(4 ztav63@nt_J+%GnvRQ}OSeAYyqgBT6TM&LBD9dJn07foU_FK%#O(GoLE%gFL7q z$TTx=c-WrxpB-pct&G%LhU+*?vCzTI`9*(2dmT34#x;6|tc=+^Y>uT7kwQ7GS=Z&) z`hrGX(Z|F8Z#n@`!+@v#*m&xECF1jLXAd|6>+z%$<9qNb765}WXVYBix?h%U}d zdQkLD^w~0SaG#tXo6Wuy2;DSL^>cm-Zt$I?+H zoNae^cG;B}njl9ZWE%s4;y1}bWDE2b9C%Vv5KjU)1~_xJcn*S||DVl-s;^wm%Y|d| z=a+@=1#Ap>*O27axNu$D&qZR@&rQ@l6-;Ii>wG*`TsVq2K6oY;u;NHoz#D-eSrulh<0m^ku>M5^07z&?;s^VbXZ|C| z#~{URgMSMU@=q0QoP*RqP%_h%@AMu2P1Tw3zr-T+&pCk85$=n_ z6%+8aBnbyVL?L|EVS1SlZ);O^#g2|4`0rVQ8K=;Z*xgppqY14yp@i1GAQH^)h*UE4 zOONL{d;F14Ddjv-yU2jo{z;n&XQ}_c1sD4H{Qx8#X|Oipo?(*wvA0e{nuTN^ek1#E z&k~%IeN{Ze-nm0z4<9S2=caM>#Ga5r#?@-8j~m(V@>eVCKS?gH>j0bT|68Lk1K%SO z`-rl0#0^)7hiRCg3mJEOV6ehi<5ziT2@Q38`UkJtO3P}gg-2O&x+wVp*ryp%frbZ zj)NcevRWB}!0NlVuFe%0Ob^Eeru4n)lM5XN5|^qil;Bi_S&e@_&Xa!s()gV8_Psby zJgIm+_r3(ONZuH&?6s`=YQwkMI%0+Aeq#&|DXa7RcFRiM2N4Js_}12`e{CQvF;WVb z`L@GDaMNnkchY>${S~*a@347>$Vo|JqHa<(WL+0UU6^G|ASgp5NVw$NHE@!@1gGnk#r45^}lGK!xLT?GvRIV%5M`_0&AlfP)N<>a!} zB^hV1Qw~EK@Za1Y390sP92}c`knrTma3Nc0OhyMS($D~|W?Xr%bC!8`$IsMcYCo|7 zF=uzyCjpC{xLkIclCUo@N;&zk;&&Puzca-aQEi)DTdtE*uiZTGNj~g$l4(L+4lbyg z%Y7Ks@*LhqcTX!H9aJ5i^T~yAhNa91X=>^huw`tIi|%7*Elkd{=(y@=&S`MDPp2MP zR@Ll(NIe=Wvp#1_U?@qa3u4nbaqTR2Ts71}!e`34$dUzaz2lu1O;G}Q2)X*kPtfPb43e1}D^;97_s&^1;BK3AwN9HK zM(1l;Rq1v%vX%PEi*<}fwtJ3!sc%e}&UmOqKt!~!8kV!vznXOGoEK=DuNDk^EAG*r z5qH}NGU90bI&+v;wbw`+VuL7ycc0`07 z!$Hk1p8ojaoTuweF6^V{6MKVmqbz+wLe6Wes`fpU>_r-`M=2E(6Dm;KLi7lE`vwZm z)-ch%D1CJ%mHvKn$GVO=SYJQ3?&>{pMWz07@J$V>2pyfew_c1^21`LV>*7DBva+ge z(!2THS6L+$;HK(*r ze>-0|$I+cTb-l<@BeOp)V>`5)aLs*BeWy3U4NWK>PtS9^vz5*@=i@D@F2l>vrX-`Z ztHh@2-}EE1l&D&~lT$tofw!^YFB8I5(^?8>;OJUDfWTBEHT58|g_}(={2?b0I&Eb3CM(lgDjn~H%zNR86Y`EM`(=Gfq+DC`|LHVNnWyef<bGA_CtCKN?9%#d{Su$eSv}eooLQz-!GSnyJJgm-?>335lApbC+MlPg_ z97QC}$%Ky@d6v$91a$QzLN(4D-!SttLd2yvPWbGTD}*J%lDn#wPPD%|BvV5K2;OYc zf(te91kpT8sWZ8M$)H=~J-4CXl|X`RwpI_N26AXOc~WuJ1gQN9JR(|obC+F{6!rg)e=2ltAy-!#D8XmjGVyE*L^^47|CvWCL z9C|8}rQ=}G9d<;oH!LyPw1GoF55^=QM+!4W#5Pnc>9AXq1y8`U6H2-(vIQ#Xywgsx zim%*zOk>uhQ+HS(;(aNjaucF* z@F3tYXA?mBc?}H%A3BJbn@JJ-5*3i6dLFfl8{4Nsp7d~dGj2ETtBtQzEBR-z;VPr} zq++TBVdS^Brv2Tv4>G0e9+cF}W=HriN*5zphJ9YG6VII(XGLY&=T#jM35`ovpSsue zzAimNP&ssNK^EeJxfY_w7+`D?^M0zwurIBfC8WV3CJDYJjrV7uBHQN!t;C8~M|y8g zdIVUW;7qn!j@K{+(Lov?c$@f6gj2Pm(82FW zWlDV8(-LJqP$*zN5uTJbPSCoeXOZXSsZ*=1MJNpbx#S5Im}NnV z>*42ZG(iQdCfNMXYh6)wqX`L0zMje2N%5WVgS8diWMompEIns8Jdw~f;m*x3h;v7* z#deK6xxBi&w6-=Y(+P6=dzTUj2iU2|RE=%i?S+~gxzgJ@AMUDIX2kTcdU9?Tj_>}& zPf|Hx5s+3vC?mcvgxul116M?5mpfN(q#T{s1aO|WBDi98KJie6@hZJ)yMsy=dYi^{ zu3W`^Wf5$|*R>;~VGWr*M3bhnFc81`ZJy>+Cip_{kCs*<2r zTtMXx53U$6dd6U5nh4(Q4}YJRKC(?l6%T>8ZclPu@5k#0J+bC2uu|VR!@_eAza#N5 z;>1c}W9_KyetwYzKRj45(9O|?ujw~ML4McSjy7J-wgi^BqrCEAZhf?na14p;DgA=C zyzAm+Z3N)hBI?h%o|%Mgq5D=7#Y~yv0A-jf4#FMNPszqCYX5c*`=qxy<eb zn*yZIgFgQj|Gj{^OTb|UTC~A+!{uLh<0Ja})oSNF48`lkx1Hn&d3bm^DD zx?vvW1si#%U|?XFRZ^nv3u-0@D;t|wIth10+rNw3D;e0kckep4XR5R7>m?2wG8;6W zx@$`St>gHyQ3BQ0D5!(Iz3Z#Y!YI%sgR$}Fw|L{CPT27z4>&nvmqCk1PcJD>D%5Lz zef{~e%D0NjN+l?)-b;2`q_8l%uwEr8gGa65hH!Kg2=88B-*$?ndVdTeKD!BBBdQY8 z(|PaQxidU7QMlPn=|1|tO!w6fQ=E%%;(1YBef@{pCqH3PBAE+?hYu-53Z00n_RuTj zhG#?Qc@k|${7u_54=lTP`F;y|?MfFFZK3SH>zr!cQ) zqgfl+ydXaE^}g`Hv}#7FM7*ln4Z*8$t)SU#Y@^k_tBbC=(FftPej}g@_;EzT^h(R! zCQc`l!{DNVDv%t(s4pNe#yR+MQ$S8G*X_G;Cda3(Cyb8|f1%nV?s}E|ArxJgMgh-= zHi#b+yL$Se6bXZBP^10lJo3un}Zsws>aVvPn&55hiC6^+i!beZW7|t z*1PQt2L!4xV#>()Yjxs0U|zwIi>uk>oF)xo6WulnaGM<*%$)J(`WO}l{}oCxO#mm{ z_RwA?!WK_5qn3dN#r52&@ms+<0~>`uc$B=dGve+$X-(C{hxK<&2mOra0q^qPOE|_X zj4!el8+933;H`I}6#rcI)t7sEsCm{OBeTO_H{0PTA{04l-hA#G?;yMJv8T(GoP?X0 z@_h*PK}k}A7?;4riI;pmZP>%e7VBL?-z&|dB_#SXu9##`?fTrnfHOHcxqp}E4_6*k zPbQuOOK66z3%@_56lAmr(EvcC_1i;R*xyaNXgsp(Z+PY>fN#iUF}3jy{$W?;t(_@5_mgy^ zErHr`qGfD&X*32?;%TOnHQqKA7l)BJPESuyW!$`ZMvxgkDN_X`X&byzEv;p3y8sf= zjloFcaiX+HUOFQT;U57xWo5(pW`jp4q6yOHlwcS>XYNq24T?5Fzw|HWxhj3#G0k&1 z^hSTy$jhZNU)GJwVCPms&PQRj4+M-Px{0_jJ`)LUq<_Vs?bznJc>+09O4=r}HU){L z{QUfB3x-BSg0JS5$LvyBn8_G?`#C`iZq)N3K0(wDQW>;*#g5QL)`~3u{{)TxdY3SR zbB#p=X=D-*f~G0swpOm1*mtT?N$>>=pp(+S{F$963vl94ax0A!U?@6vw0mpf1QH$x zfHHrC+~*pW=O>U+G0>`DvM`ZKt^9H1{8G&%lGJOrjSqkE_f4P+qmJn~5iSrGa%JbfZK# zw9p`)!QU6Tkfu|)iOmspX`V1DNRtF5Bg2Dg;B4I!P9yLxO~`iq4zo}0%gBDjCKg^4HpPO`Y`#@zcg34y0eo_0qYV|DYU zUhAx4z$&<@qz&u3Pu)ssiXqCmm@?D59oJ{!PQQ>KLec}d;!uj$6YfL65uHF z3ovvKvC!>7M)D=Xza9$5;4YCct+W6~#_N}e|9y$@+4O5C$5;2=4*xv&-27g>@-0eg z7_%32^&m4<_9Jn9ihRjv6kIgDJw+z)-S7Ai`5n`%BhyMPN>V0KF%9WFZ&Al2ZvFcz zNxTVSE~|gG2bNqXne4%E;cI2e!p>Jy9ab7o=0`|tLGp7K5RnvbPL2|!Rqr;P7yDf7 zcTATafFJ1wJ;Nn8AE35U`vhpR3;{$N6qs|}30!E>(CAW;)42U~h0=+u58mQV8V8;#@1X?(Nu*;<#%14>u2yN=ygt0I8dLhl;o#Kg+Vy5mz< z1D%rjtd&{Ia5xuCM77b>@Vbpq@>1R>&}@E`1SvTUv;jTvKr*R8vt~eNS}u!%Wdo$1 z4~MoW{FlJh)>Qe?r>G`Wzr^mJHHnoQReLlW^aJlfl<n~-S11pRg%oVXr`j3aZ;-8K`@j5076ArAdwo8s1aJl z$Bx1v)f0VHVpRL}%AxtqM?mF34?7~{CqpD{1M8JL^u)c)ZQEdbHfV%~lup!z@0Cbo z0G))JRVnT^Yr5@w^tj8Mlc;3YzFUu_v~ZjcU+j!wM)v8aJ@FUWSnTv?CbPfsTh+TL z$6rgF9;_SoXa2a$cxf&ksq`<6rhdE%d`tER0>;arLCoj$$0fbb`F8cRe~5hcGVAzx zl4&&PgfN5m^B_!ze;0Puw8M^ZVs>!_rpv^Y!Wn2wst|dIW^mHCHbI|YO+52o^>d8! z0>9!&)M0t4Eo}OsLz4pi^W7SMw9oR$B$YCf*sC$x=AZ zImqqhTmnu_ycM-FB0<%+X)ABG{FMy3NsD`uAe$l&m};mk$=Ke!cpgY#!1C!CXu`L@ zpsMh8&;Cp-zGwEjX`xElWFquY(=^o48~ z8~AF*4ZdWH!ha91s~UQEOFBMUk4hj#u?=SNFuDo3R(^aP7Tv$m4#h8j;eV40Z9&;W zppq$DD{m!l+HoM2zUEZYQGPES*y?yZ#9Nk5e}%C_;K76yK#!mSTase$|GJM$JqgPI z{YTJY0w)C%S-{^d+q2Q#1x^;0cZ6)l`FT40-d2sF0w()W_BR6fc4bTY%H%_7U)lD;~SUMR04R5L8fY!Mc6cdoQkp&N zPiuXFabuvuvZ$ctSj3}V%A~`+=mY}$4V)||WfvrTkrnf;gj{BP^ULn#0c=1vmGdn5 zD~`c?@csc?Lml#b-gyx7u8brC{<19PE)qU>6{GOoM+2fnAi%4--_{PjSE;tg>_@#rpqf zd-HfI_qJ`EwTuai70WzFlUb}X&y)+37O~28cdlb zM2JX+_q-Om+s||Ve(&?%`+fh|`|}BFUDx%!zQcJQ=W!h8Tc(yhgdkkljXJ?=g(d!) zfrlTfxGS5eMvW^wnTEJ>?|0y0EDb~<_8%9J{M}$I8j9jn zX;?KLF8JwYR*xt8^uMl*gchF|N~iNr;W6GIz1%AgJeV!=2I5H$Kr zLK8^qye>oqZK&kZS7{YCEj|=k_tiudZC~oxigC;hq|z_;oH~H)pU8An4p3)Zx||{% zl~wdG}HDsiF2Vx9Qo6SDXg-5ROpeo~EC;9!8D$dbf(M zjXw9*32%i0=_+ZJQ4hOJ_pGAYq;t7}q^fcb1&5P!*we;Y>!?4y&6HP^YsnY9oZKVa z&o{9J<)W^{+>wp+*l{WqS0=S)}711ft~%ey6> z!Zn%Txquu-Cy0a~+FmXzO3F#@=^f<^wL2Cz>Yp+;v&&#E4HJNVSGmiAEyCepCST9`RRG23lb?jredUpy*|Y2Ocdk8o!n-~j zHea@}wDI1To8dBkGM9z9Koz~`=vze!d8arQMzODF;}X5^xM~$!X3@&%&^fcLi5(ZJ zcyqW_>1S5|Bxe6-zUr~Dv5zmp-o;}k=Tg}A>i~b!rIOjf@ayL?ynS1MH><|!1@ElP zy2+%P2;7t;YA`h%C;6%Q?&FbK(nL&<#vuSSm|yJY$oqbR$S*{@nXe)FL`r)s1h9#jTCf}p#&3LnjNMuB*^w_dN=F^>K(V^F7FcA+VfultZlG+f zx9nry;T@H9!7GuGJ4N4HnhN4)_0}wRvX!nGWmR9>n5Uym{!DagLA2}Gy-8m^d6dsfHRT(`qp^x%bwCJf!(y)^^&8N<~UGvS7ZS6X= z*-Uak9XoO?6u7vv5F+L6ka?BIvl%GC8ZLReEW9^3QpYsfoY>&^HbO=CaH31i@L;6| zd%~qJ^3mvuT->1oxeC)4RU$20hd9n*FjMtzfP1Ij=ul1#=LgLE?Cssc)85~|w7U+I zUon94y+fH5t&9m}kaKg60M%kR%VjpsoR8tI3f9lAU4PlNPQk$A6vFl@UDy~2j9@H@ z+*T}HD+eceDml4w5fh4wXUPPVG+3W2ru2lEY{J8$JY~$xgmgj7Bv+TiV zoZM-~6>9@LPw5XKd_d&tH*ei?Y-h1iu&6hA<@T^N&1O|@N!Q6g@+Vmj)$s5-=KZ$= z$VRxC%ZYvMn6H%9zFuU{;XQ}Y`vPg3xD9{tj~ z=7u4c6NCiVM=CY}Xx3iS`xkR5873*tg}73YN1HADEL)B!i*3Fu7iJti*H^Y002`b1 z;lh}d!&i0coF*T{^AI821$fKmgKFkP3~Liw4$#<4M?eN0Dk;pj1+*0b}je zWf$^anP?oJGk(VGJ>6s~MDT5XRcBPCImCK_b39mxaR?1^t0Z(dybL%+5w*cO@byDpu~XKSihvon ze1Yuep6bF1iO_#=juFnS@}Uy%rHhXfJbtC>N;%v&j%tIU@*tA9e;jFZ!Bd$ZerY}c z-V_w*9|8EFqN}vG8be$fQiWDK(fpmPAgAYo61pHS@zv08`&z*($lA=K)lS59R zT4KG^;ta?JA_)T)G+!HWp!gTIK{Q8(ebNV>>fytO*C^>Jllk8sxGRKpr2hdd$Kxwp zaOIgjrxly`wY|O9Gm5aCrT>`k@CIh_A+4a!?ozwQd*sBg`pUkqdfS1BSX z4nCG45$kG@2UxBsXS&0A3lNI6z(e7xeQf|Iey-uG<{sm;gN$i1Xg`cftVaBuGmb0{ z@0%{m6Zkbw9Ig^;`$Ke2AU1rburqwoNKeH%gg#4}F~l`QPcRt>R5)aMzzzz${USi4 zkQBNKn*;(;JWs;=&71Y)OM5=Iu{Q7U+3AmCluP=kbhjOtgU zOfbnBIn;E9553B5gNF16iUyHO!LrQC;>Z5OD|u;}W|8d74&pBW6&N|u;X?WQ>h>X4 zW+_2=q>2BCKNzD%gZxjYFY~mluJC3^1$SlEnfPL0U)wHGzn%zH8eO5T#wOu5%X9Sn zEsi?5bTX6)K*rkoC`~x^!8gJU8Zrbbd$8C-l3Ji{0bv#%Qbzztjihj*^QIln!A-=A ziF7&Sf6R>MZy$>fZ%Cqhx^r>aS|3YS%5#*6i$jVm13Fpw`R`S~gSq@eSH=uy`P1@3Q z(gD$dzJF@*WC%;#KIgB{*x;1et8=1Unl1yY{O#zBU{-WGl+X z`qf2>?@h*`H|qTAx7Igl8$Z&fp3^WFICT#{jQ2EL`ecE5bH3g|<+E!4V9KJp#rcOP ziAdBAnK_C^mB=1pLtg75%|KzkCd=&UO8w);V*mC_w)MGh*EJ+%%~xv5Vxj7&Y%B?{gaX^xW97Ap zkGYDsUDj5vh`*?2p12wHc;*39cY&6myFvI#Vz+|aN91ncts!(~4g0KU%0kA(TzNg$ zT1(?Ibw$VTZY{t1W!2`KDyswXqXGlBJ?3N|Fga?u*BcRm3LKp1Ny)J=)}3cxCnKyf z=xa#a;C#%b(VGKDz06v63fr5}PoElj*N1o;c{k>GFSK)S$f=ZTA5tlQnR{*V{F>tW zcU7{-2WlsXi!rOimsb0$^f)=4>40u4v^M~Q4N{T5U566PpkjKT<}6 zd3&U&K9>(36JtA8Z9=K`nw@j*r;9|(NU?)J%B!;qd*T=t8Gd36+jv%82FuCU6~pczM9)iQZG@KCCUc8 zvsRtWJ05|e*WKar9%7Nb(WUb?2Vy95vz*xXmI+z~duOL_&HC8zqd7?#A-M{)b-fX} zPWGdX{BqA6B1pCz^NC@ir!SM&puHG6I(+}UPYC0I&$%CZ3$APM$%uFiRvxN4-Spy z9EXDuInu(@N!2w{(Q?F3pr)(-mNeB!35VlDF5>}YpgxIy;(KB*Q1_CL?Y?Y%HgD*{1Rr(*mpPMyuw*A_Cl+D{=dGM_5o6z+oi!PUN>oCNr4S+FOZ8U zq1K>#@H9VqSMY2)*{^Za?#mm={Sqs!-)-w!e+-beeQf||4#BynIJZNl>Qt4cUzn>v zcRbu`5^0-d(6Laj8$5I?5W>N0M<0L);eu24s-0 zkdkSNsubm**hqzXxko`Mz6WntywZ47qP2r@_<$6dRf`$g82_WDB$&)fidDRWBJpI&^4FeIt=^pTQ6((pUL2Pj;7L+# z7IC=0RM3xEdVHQ|XGmYt8+m+ox%RSzKoDII~V}1dmip>LqemJ^a4)uvd zQ4tw}aCaTbonDn9*)xr<`6vqO+}&%mG}w|v2|Y`j9Ff5h5X44gr>wgxQc!72|lx0^?#ip}PZs&ZC4 z?Y`RP@Eu2-tc&GA9;kd*pPs}AfgPWqxE@F)HoPS(bFIHi{`;!*cQE8mu z*R!+M>Fqm*UZ>jUfm9ztmgo9Lw2FAikZtty1t?QYR8WSP)S=L+uOn4>=&LIxog%Sz zSy}-^r5+Q@;g#3C8cMj~-Ki#t4om3m;aOt#4doOG_o%mucPcopRz47OSyn^p@F zx8A$nVVvUN!Gj;8rk2e368y5F?JSKi-B;{F1QkzI+#y^c9gn)i@GA$%KL0FCOSC__ zQ09Xp+s*ph1QTyJCnnor^j!fRIfAHJ(jDdVk1`N z^%|ojn49^j$GJ=85-wItV&@zf-r8QSSM*sp1ZvuTqj6-_!WJG(ELRQv&J*!%hcB;h zGk!FdY6Y*v%b*kidmQNQ1`-*xY`uP*UbL22LiZ|GM}DoEVqLWA+1Yc$QT_p~Ou5;E z0T0Vz>MJtZ)-%-Y{qNIA`rJ;FkDowaMGl!f28Ee23CQIhIhr0Qw+~H9thLa6An?@!ww>DPkG?_$R4*Rg=qG$etJ#UCh(@Z(KJ8p)$-ag-3UkE&`uXdKduBiPz zEN7-DkOpm%iTOx^L@lawsqJNub#2hl(k-3YN`~R;{cYv7I^tAp0%>$7jvp^E*nS2* zp|@WS`&4?=6C5t-#mEfF4rqGkNw!1fzw{V~Y#3Nc#UBXfZ>uhfXHapf7@)7C5o2Rq z#dKeKU{%dr_T;qQwf0&T<;Xg@M@H{6@66s~`=x6i(stNdR--DlzEQ$6|q{@5E zu2B)!Lxwy7g^B8j+W~kfY?ioQ>*OhR=agF3{73%LRP#6P1Ymf{%YOO_YYc?Zuii_0 zclhrEp3Uv1!^3#D#9CVl{g}OGgNxSG%Vh>Dna9Ptb*(?-{6fxHrmRoC!asEC+b z@E8ZO_4lobm{%;jjZcRj8^wyaHq-};^_5m{NSEUWoh~)pmHVmf{icTxeUeB< zwy~hAq58+7Mt4>RK}R}Xy#*nR#5)A;kg>8h9*7if)9^+g&cxfeh`tT1&kd-KK1!gX zU@{5}xa^vyYWgkj^*vHPJ)|c=penp3t5KwbC8#NGCv?blXW{H~rAo}>b$VjkpAKbt+YFypHQ;-pN!QWreQTCcXNl)a?I3w|_Bx2oQ? z%3T}mDKSFUQ6<#|13`j&uXbYGnCT$LolnP5+D zH<$9{JYg-*`1S7lNowPljmtCvxJG{|Mc)4F;H|Ri=DbH#^#lrMl{+) zm;KXP$o7)%(t_uq)+>!t7d?~nn#EE0%W`-F9wj{NcWOvO%13~Q&T~JrKV6tbteluL zY1(~K-62z+GAFe<=-c@rOunmae5n2VKsD-z^3 zKXvxo3ykzsI+$AYl>73(FRB_yr>`E4eL)(W6zV-~Y(^d#7+Tmbaf$Bj7#GI!HNnr) zLoUPYRj$hSCANbsTl`YXd0q)W;i2}F3(E>_Qqc`wC0JMdJ8N?6tVw~y^UcAgh8pbG z%KNN)vm*0-le#BeG6qi($*WH3R}cSDC2MQ~y)IpHojKSCNGdvJeda9U_MB6z6+G{8 zES~VF-{T4XTN~jqL`Fi<@P%a8W$bp-qbRx@P*209k6)*xIhdO&Pi91I+FdR1p1GsD z$A-n;Zp|_e2^XP`w>7#nu7Szs3AO={%L(sbMbrz)bZgJl_=BsV`DFYbo5Yjcr!vCZ zJ5xOdxjs4U6iI;3)(3^aC-;F-&i3TfH_W>bzZ2P-C;&|20VhA5;{dB}|Nghj!H4E8 zk<@g@4w>z5zm9$TC0PiKMB(tESZb(B#{U6dw*C7QqTL4}cDQgnEqqA91h@zpfhhXz z-w#t9EEaA+_FX;r&}aobrx437hhNB+&@v3Q&$EV!H-r(lzgX&0oipzf;f&-iH?$}IL9qiBEnyCQY z4|8m;WGDlQwdId_{)T+g{ZGFZvB>yWq(gd2#2y1u0s9Lv$h-Y-gAL~Gx~#}x4{}ApezW{S ztjN3j-+i6u@@IDdgG(N?Kmcw{nnLy@SoN6AmAn5h?%z|!`4OZ!j$#1I#HSpHtL4e`_Afo_E39b#u zW2%_`z9>joG4zEMVh4zCwB=e8QX!x~mOkd#X3=~5_F_$pq@CgqH0|%PCgC*U>91xZ zfgWx$8;sPpXc(S0!ll8UDY*T=mk0JEkTmHs@XmDHkoQFS+v>I_-TW^Oh|rDNtTmWw z4s)Gw$2tP##&&?Co2D_vOK=>Y!P#SW2!rLx_SV5n`tp!Xiw|&TKp^?YGjD|V${9qbqdXHG~- z2z^J{wZ*?utZd+?uo-h4&dl4?^sdaogA*Q~DyahQegE!p?@nqekG+?p zv-3MI0zHmz`pXw5ZfNuvd*}9TumSh=_3eK=gyO7MhlW1uA6Xpgayq-ZK4-L|&6I1! zM{6{D)siz~)WaBbt*q|n`H*!L0jIe0d&tzY?Ckfx9S7M2+oR11^MC#ISBLKa%bv#w zIT$+blo<2{fS^J`xY(hNY=m}_XUX!EEZK@sYDFyW)CgVekD{W(_N`ijz$|e?P|ePF zCXv^4s@-`yWLv$mCwV?Z7lh39kGb=#2ssrt(q*p!Id#7xbn5={Mi7aJRrJ|=JsO6EtnxbRZgU(G5XLL*33ca6C~W| zuify1FNY#881W#Jm5@ZfF9RW8kh~M>;c(g*TtO;`ACP(?)DMG{z`2JvlNz~NGIB5L z+6224*pZPVjYgLN*+$@~WryQ)oj|X*6F!Tc0BahmFGy{4UJbsb&<}oBv51rkk#f2L z@|cqZcsL-ML*a14h{$fgNgUOka7WhmW4AgrDABai@`pdzB#lCa3jsIwq}k&D3Qd|t zzT3&tS_f`TCk=}DmSM=(0UcoLI@yRr2$V&seLHAmvR9DDDyQn;F@Q|bUJ!8~@+PfW zpk+XaxJg%aWi)OiojnMWvdvSCr3kns1&e;y)^J@`bk#1>8PL-(Kh_sl@FrY-k{lJ4 zR9NhGQBWLac_FibfXGa?27N$DB#!V2D77t=95+9BLdq8l^XNf7)H z0B`u$7UwgZ2=)=+N8|4khMY8p3V_8Tm7~~j3gJU>{mthVe&Fan_S{)qL2+wcUt(v?? z+u7Dt( zv5@sk5!g#Aht8cl*8r66g|7v8D~eIdmR+Ni`Z}hd2kOvVadE3i71d`WcLbS6zh;_t z0Da`y%!Hg?))mUH13EHW;K;1^R=x7Hp)k8P`+-5FXk zcolap)U^~;Hx2GQsBCuU3j?LTZX>ny<{o16glpxBJk|+M#ud6$dbHZyPA(OCX; zg{~uzUavn=nsS4&lPWMbXn{YaA~m9f@7T?RTqX~TSk{iP1lC2yI?JvfD%YFnxb}%n zQaV=b4c3)U>2`eu$Y{=^#<8#* zk`US6gaBhb_6wpaHC)OGnyb-#EeKnlwnb$D;d_Kkd%<94&22fcl@i(<$}qmhol6|Z z3e4t(wUhyCX^&u+s=SAhh48VGiRcwqD>Q@t%}(fbvRho-pcf|8TSd6e^N3Ou=jSOC z@*6)Ni0Ye=dmsy`a8jIYy#16*nub@1Biix)C6$}KMOU7zhbz3Qr$)_0VD6eJjPnzH zVr89#<`1=oaNmrXf<4$vB}{4JuM)Fy zN^*EPoSPwx01)lIIYK<#-N_64*vCagrl*q*k@BDq@Fmo0t02yJ(g(y)sl>tZ(_yp< zTnDUzUS|%GZ2J#@pex;P>K$cH5G`klf!(v|9MvC|C~D5-iS!b(jvCutEn=rYh1N(b zYxyRGJ|kvYAEL>}nku*}sMw3P7HzDRRI+?_<*EUsppkn)6HhxR;bKJ{EuD8S47M8) zC;RIG0}$l}=23Zp^fBOa95ue(?1fc;5{-POH*Sq-a7EIuB0A!l_GYX&Ava+@hAwAK zxeGKv(wZo}&}2H4t>RUo8&mpqN2K{#|FtW9mxwV`^MUhf7G^Ib7rfsp&Qyr4+#vTD z#AtE4A1t*@38U{5F~5b=cm(>V&uQ7U)^E`>b1<@+7M5toyWUith@EbUv< zx8Ns8Bx2k$c!+nc=HySQR;^F3x{$xLIa0E;3!%1D16K}`V14%_h}B=(>idHBieI-f zU1SMDy1O+q!dAh3>MQCHp8UbrHW+1pC*dPCe;Y)f!l5}s=bzKXign0gjZR^&W=tP{ ztDiF~#g^CxP1S%M7KJi-Gp(6_6m(}3I9koByvmt7^a{)Mh zAqqmriLM6=%Bhgp8s0_t3Lb<|H|5D`JM&}PPPH+&Yd^=8hC6!o~Tzx0fj%#+eAf1=O+c~GY z>ESI{0mw#g9`iuXy8HACm|$Es2=Hc<$RZ;qQao~!2)h&?_$=rynxqyA`a;tS)dU(s z?Kukw!NN^cgGwT=5bR2_hy?mkD}|>-SJAe}mn>-LUpbr%QZeIqKx;C9pNikw z-=l;2e1b6vxSmhW(-{5FgVT6mS{ef3e>F%3iMiKeP|bK-py>hkV$<_?rd$@u@Wwmf zTCJa6O@MnC&`)4GgjsST1tIWT`KOlyk_V6+GcF^ltp7qy69Okg02SP9?ju(u8I*m| zeobSWj`*zQ5W@LEL&=B8Yy!Ap6jeb+vz0fHB7UG|axEZUsw#@eh;%@z!MmUg=324; z;_dxbjtBOmo5M*Uxq-nk{_);sv9?kDd5B{nDTmlyI21;Suz+xyJ&?J%G>Em z7%)`Zfabl7UPL&HItdW|w{7fci&X!l$>fmpJQVXm#%}uM4jB+keFwcCvo6#T_g|oK z)mOio2(nykBWl#w+5lFciDX&HBD?ft{d6jStp|xfp9Y{JZ#uw8lubTagN5ltO31G~zrV_^-)Dj|e!WTx_gNN+1&C(TY^ z%QCEhEweun)P~E{jtFO$w4dBKMnst6NW7^8BuNy13qYA#NLKux58GZ6?>wg{6m!Sc zz#o{sz%(UsOsTesn8P53J#V|t2Z)w^o$|8un9vEGq1 z;6!zB%p;K2CgjM&^|zzK3OOp~ARw`ndg;I62b#4eUwuKn?{hznioX_8{?TACnq;3E z2QiR|$iYMnM92l%sGY!MB}0eCp^DHa;qw3INQ^$>X9Li|-j9e0u}{tsP;*-=qEeBGWG1u4VL|* z(=P{ZGb$~sM1z>v;4oRj9@umNWkZ07F4g30P5S=;!(l!~Ogfcc0}1Fdg1D+A$WKYa zhBc}R0Oz6n8{+|(u+5*ZWg-r?A{4@xFy|%6_wC=G@T_#@$okmXb5{cmr-i1pxqTQ+sez?A3x z|35u$Zk`^&oN8w6W}YzIyAF6(!qoG^s9Xm^GPF=~)jE+3!lZiuYma(UiwMgnRwMFs z9{*L>A7TE>)ITN+{P#h-yb^62knIB4v#dTF?^xX0cyb$J2CTkYX!Ph}M zbR0+nG=%c$Who8JChz*DDq9iKVbnpp|*?;)OntKo^l< zWQES6yuv-@O^MJMqU$1LuR6UCGsz*03*$ZoNr|Vd+mDp6%h=s*i4aSAh5N|fpg!0s z+@J+QE$~uGc?FY8XBpHBwk(^hNJQ5T78(#u0LL6&EkmaH1)Rf;$55qVPmV3G2v1A2 zwYrdp2-NvMMalKIJ#ieQAy35IQ}R9YsVV8fOQnbu!$*%EotLD>Ue-%*8l_dPd$WTh zBRo9+=s1~5QDT7g@q3x!#dMrHyn$TCAY=`Z{e;TkOu~J@UOgp%3E;*b8p?Ow-B#{J zB_i7R<`(_UPS4Y-L6_sg=JErnJj~w8cW}k*cW7HOM~@3n_OrlH6$Dawxwc1>ulNd3 zaC8wo17$y*_o=`uTNt+u@_dB=QZ}~0tHm8bWHP>`6D3>X4xU3N&!9z_S?%0eC2Y$i7K-c2#-D3`V?%tKeW3 zNc2n&XkbWjbh$jZkyt(?!bRCN`K4KI0A@=?^#$nQBjvyDBRWbpVyJWUXy=pjvKRX@ zxswu!ReJ;S`T7w;px@8lfywE%SpCr}!Awon)1a7N(c{*8j$mc=bi&hsuPv8}T0qKf ztwygMwqJe^?@g@m+Z>hxdOc#CQn1q7jm4=CFOK#12e{5fMj&{( zGh!iT0qUFLsZ2(Bl|rpULuQ9dt`Avf$&n)f+tXJLHiaGwKjY(>eHlI4Pw544)10wm zi9#+vqOfw~$vEvzT;qB8f}%1$1F=coMN0^`$0%iEWoxZYwPI-sJ-T#9-4S%eJaRD+ zO*9bT6`SmbB(#an$Yj-%T^&u?st&juk1aKG4`10+e^oKpbhIl@;6t+%{;(s!(_?(Tteyfh%>lwXHq? z=}^5W`x}bWM3G8_q9^Hxk7u7J07Ty-N(Q6;`1WJAFqg^(J+93!UaMtNVYa^`)TG#| zifYXp&;+Vc$`Ra(P%R0(>Cp7$FLVEX2^ zXo7QZK}bI?EbM`H%w$Qqs>4sE{(z~v^Lz`sr(`jPT5N(w8)A)bGQwFH*y_x>3c;B| zOe8DkJ9O<`K;^YzU50GgWX^)P>;npZ8tWuQSMz80M z(=O2GKLy>c-W~Qvu`fP}0YAlSC2(GnC69$wGtUWg0>u3!KlcNLxKg%8;7@;Up`M9EgCuzZlXQ2@kuZRQMY9HvNtIBt?`=~_2{%(kk8=hmfrLE4J;f@BMaJ_@ zyOVd0U2}3VieA|5j`-Qq@xzrguri(DKNA;B*KiHi5}jbwRj$Fb@v7rPz5}=Dv6gk`&Z>G%50st#5Cix(I%Q#x2#pNbQoN|8 z-Onff$igh&=Iz0i!%l@I9|Va}_qiWfm@VQWzX6mDHxp72a`G+ayrGBK3LQYFnvqtp zg#toR6$%TLB7^gX!i0dhPA9yp^L08n8%2w1*jrCp2#d1`n!}=PLK5~QFi=CnRsiVC z457CQ)bBw{g&f0%2*Kbwh}7>}J~t;i>LBtp8si2HDkrAEpuXI5!QIwq(_5BLP=alV zW9ufGLE-)KUrQdGYP9s0&Pta|tJ)iO>!2`w-fL8UpU;=vy3ua?w0BrO8kMSNg8c!a z0y*!Pf_FQylNSR-3dpw!CBR0oCw6Ei-4(rL2GJ0l(3W&wc{LFSRfy^2pMjz)7mV!SH#{_BAp=NB zp=+K3V7smNREq=KCR3)Q>1Zb~qt(GKl9Ti5Tj6Z((B_1kOx`11Vqys}dHLeV0qzA~ zSYSZ%XaCSX}_9aBZj6gl}BF{87j>O^+xw^&@{QuZXJZ#xNFbxg3OAf*EZ%7WFDpnyUO&+&mI zMOJSb({L~e_RdtXtf;0=4?lDnTS+IpM~g_neh84ZMTGVDB)aT5z*`|0QQTj+Pc^J3 zlvuL1T&H{+ch<*9p4N;o-7V3eW|7k);$*0V4}-9XyKqM&x1O6(Ex_?Iak5zRARq41>d`GChXwmf>jp#>(; z$l{YiR*q^>J$in$kqX)OBE_?7S#RP1={5_P((+Zg8Isp>GEJ_0?#n)LvL6*)A*EkA z`s|_Ti=q~ptlO&iMyO<~dRcL~zdP0T!D#8zp*ofvbe%+F%SrzVX$}qSXw6A4Ron7O zc6_UObV6ZW)uFGG-85rC`Pj9T@k)lqIt7EB{$h-DqvY0mElSlkS4W|C%l$4!%1!1-~B?oL`qlW=t z9k?TAz7O$XV!c4t-x6erhIVBJTKX-Lr;kFnS$Vr*o}@bx09a&L3C3ZsZll%^n_~91 z6}t8CiQ5g&oB{Of1B(6&(ZaIsqy<=v>Bt+Q=ovD!ysQqmlQ+_Nxrny$0djqYSOFL` z#1Vi%heNgUr8qtX%im@hsUh#7n+{^lbT`@__C^F8E4n(SAXm;n!2yHhkNj=71d86T zh$F&?aRfBZzelRg9<|6Hbp47{>oXP({ld0}tRY9+B(Eqf<^cPn@5C^qO|}Tn5|s1g z!ZR)m8aFr+Gc0>UCORJ@c~E{rC0Ur*_vZE4Dn_J)d8XQH;^Ccq%b+T&F!}Xa?+le7 zQAaZ}`EXXaDwz&$!sfW*ZIKxrXqi`V_(-tIh?F=C6%v`=1JSTchk&>kv{04KrsNns z2mNKlRusw?MbwBDughPkpiPlX9@?#p@J5OtUl;{-4uDs$$)b>dDUI`QE) z|97Ba6R2R}d37fcUGvbEyO$b|0!W~Mc9FnI2l;C-4bl$?+~z3&=u@8Sh9$Ph&Riw3 zKap>U*s?U>f7Sx9XJ6swR7LNFY9IXntz;9J2 zvx!hpCb$1rWKi`Gl5hVGi=>lC0VH5V@AquRQwEm`1qRrq0Pg{3*x$1m{*GUC8g?j1 zU|IjKjSR?a0V<4n)C_jC%skND1t6#Rcb%lc(FJ)5h!GcrUcMG$$UFm@@XdVcp8;Ny z$6ilEXnP-R0C){;41~4-{Qn5;2GqMgC?oF$D4LsAD(qku@gG-e6>GcMGK;iO`A5*G z4N~S@j*gY6I0D%};%Nye?>rx%HhhEfYHF1GLA-lx`V2^s)&CiyO^qBk$Go)fA^YGS zk|@@(spl(5>=hhH)jWvck;e`XY=E*6Kft5)n12M1wp4gYBdB>NhFN&CLIsUCNJB|A z-VEVMih;UL=D!5RJ4t=R{$n^hWs{e-`=P?FcEJg{`OG18=`p(bc41V$$!AKj*K?4u>1NNFQQ`CqgBiGbn1VZ{}I4LN2 z{*}ut|EW~i0b2b_iPNO#{5_qaCH$gi#eV%q;4&>)HQQYs^jv7HL^-mygKiy!o{^}n zt`ItVNbXXIEyF*1h1dXKPQlg)*8i|E&4E!SEOY3#2>%c|y{Sq&5q}baLiI_SS|k^M zZh`-3X_F@cmh%f@ReJtd8Az8!Q~MF9-qz~J+bY;aM{CvK7F^?h(RqJevj_@>|KC6! z)OEu}koX{-O2q=aP~7K+HOIm3@&beg5My@f1gp3&&w{%hIn)wCShDVgK}9`TM<{MN zarNU0A5c+VvThKA-`zu?@~ZIs`j>$g%(S3n*92QcOAWINRu|0nu8Ap!f*T-Gh#eK& zEU`i13x1A=BH}HQVGJDv+%`4uY;pVdK-1DfmyC3urs|LHf;sri^{z?&>2Z#F4135G zGRp}Z4mY(D_TbGecW+M%GlM?!a1}C-HXOii=crqx@s36o*&2MmU^fj~8Md>)MJx2n zV3Ze|*(egFRaV~*#ojyH>RNSz9?CXW^*|SY@9}P}`)NR`}En1eo9A@O-hj1$Lb5Af^alcw0NW^YWX-wE|0Y zS#dobm#Ge~huMNAw(X)7^+-enSV>nb3i{%C&2O?~b27-_&wrbrPq~9jsn{cp8`*)% zE_?p`_!16s!C2X9?1DEiGy*R3rx5*|AQdF5%?3}ur22Foe#lh0+l1(R(N7YBjsM={) zZ{hMqKD1)%)&2ZR{aUoQGWV}mog{9-j+q$X6mItTL@Y~&DaV_T6Y;1z)S)0Cz~2px zEvz_1IzBrv=>hIk(YrT$>5jEl`RpuuFQGhNu0txf{}3!btF6tCuFEB=2YGgnR{WP`%mzB?rl&lV44o+NJa#LvV;LA;*GsTM!3CTm&>A4@P*d;`nJ{|pAi<6FB%xf-ng-l z0FMhKNUTU#>TkzL*VZwTl?p0ZXv_NhF%lj`Iz~t|?#r`OXKc%b>hv@uy9o*BJX1{2 z{ccr4Ys+&Z^Hsd-hN<2&hhC2{_2=Z(#2g$zJuaY{!4=Pnn3!9o^X85(kJ>hj(9eDUD}2rpV2KVd#fnQ#?PX^Eb0+U7(}f4EK# zZ|*VuB#%BL3D=}ZU`XM5Y>;*vOsXfOB38w+kau5CS1W}qK`6lGpuLf`25hq(K)ymS zG7|G-w-aY3V+O5A4?xiQcutR>Q9E^tGcq*pu+JPs7*sd!xH=U-Cj>|Za_37Ul)@9# zF2wY&#hKWw+N!P>K0n+eoI`b){pMw7+%FdRkF*Nd}^O?$XJ# zja#9Y8gkyyAuKkD-csLUG3n90U)HU&*_S*?D4C&>jy%wEvr{V!l>qaDWU%suO;(+| z(n`h?-~~yP0;%PsxWmg&hs1f_X5AH%$JjUt&;v*|Os)p55`Xqph%V6R8uQh5X>b5W zR!tcUO|MTVwIYmwE>=FN#HZpChbrdRhX(4fIW?5kY)-BrqAg%v?Qu0i+7vA6zgW^y z>uDXc$T>$4y_iKVE5RR3kLRrv69-~vM_n+-cc85yg;jJxENp8g=xp4C_C?oFXM@TX z>cFWIzQLX$KClna&H&CQKptoy)G`1G&T8HoP^$oS2{*V5iC9QZITmuu3mM$J8uob# z(rd}L=^}ii-GZ6f%3@)s;&Dt@0^>LTR#tnUKT>rCvIR@Ip973>6RD)gaOR33(bX;v zUe+2wUMLxmseI-_Skac9nn_6|iPvb0xq+#;qR7=-NL`#`h3ZLUvL~UHm`{LY4V`AJ zeBM30^W3-qZ)TVVPzkrwZs4ggeu&QHY4ytV?s73&s*ixv_ZOoGxKn`j|6xg;-5O1&cz>=hLKHTUo8U`;=+ZbMpS<`bx{iqRd>+M|>V zK{*6^JdY(>+59F>!edY;DBu%pXjn%ZsmXofnbajl;VWV2MolHMju6pNO6PJ-i$RwF zn|h$15lYHQBwJuWbKLPF2vR&8#FWzYm8rs)G#vE#(j~`Y zT$@gwRXq!kz0yP@6%^v`8F@f`F{K-`VfKmAdT&drTGA_y2-n|;e+l89EkWF;76a!i z@|bE_V6!VSfg1TYvs-r>=m1}ww@!^3e*GBfuk9ghF9uF7M~`@YA`6e%R8k>AO9=eB zO-S7kQu<85-b_eKiU;@-&W9{0)MhBLSMk(lH#p(|jHz%T{{kj=6&gJ>GGb0-KwCD4 z`X>pgX@2ECzEJ&fGoHa8YOdyPAtxa=K=uy-VtE3q7gQAaoVQ>bFmwY62s-Rugc766 z8e5s!4jGdC;}Filk`Czpp#0z!Ck1h|GJC}Dpy+}3wL_*p;>miTD-VQk_JWoa9IJYO z4MA;jo1`bF7W-yg0pgo|;)UNKW)c=hXkh7HEfLfQr-U0?Henr$c2;=eT!d<{9f59# zb1e5h3+Y3GrnU*{{T|Kx3BYU{2$KEYrp`Dq5G3I>Z<_>`NnrEz{^uydUEAfLMeT?v zD0(Cj4F$Cm!d7eC4|=DB1VL%Lrr0oEKm2T0cv;1A((T(?1YxQ+ktmgs^FE}jsPurM zfIz=P<^f5m5g24&Eja%}gAS3%Yl zPIx3!tdc;YBK+JMP8TpEG%aw76G`(qzCO#iVJJVWMd!a-=bzPS5Ui5tIcF`}qOh_0 zt;dS9wVL}KEOyR zh63j~#M;=bcK0+^@B!f6v67(t0}J85%DurR45g`+r zw!ZNAnv*KGMqn;W+D1F?THKQEGO$<$`puv{)D3|ax)$0@cSO1})c_~=$JMHT%zbu^atEbCm zbhZ~V)ZbVvTR+7u_kOx#HGsrJBY|iCl5Zi*6Jc@S;D&G`EJf_~-;>*lzrj{Q`!`um zN1mUD)P^F1ROn)JlVG+E+Tw3XBe~foXGH7^Q2sl0(nDPvHA~4E_<9E2OSqM|1lug{ z@??Aks>r2)jZWpY&P*@zPep4^{K2$`hm)VS{M!e9pkfm|I5Z+?{C3biD;*x$#V-QN zB3eYrac8TDyml3D)jo&H<3f8xQq+T%F|W)C|Dz`br*4ozg69`Z&cHZORYEn-{4BV| z1q^3FAbzhkRv4H zQ^b-;05ZY~S)Io5%$U5~nunswP#i)UTOby>AD+0w35x)g|MtB?>2gtb^SzSzLtEB# ztDPU(omlKKv)uebuQr_{vyze~;p8IoowVuzF)Ki!=?JA1=&tuc=#_)eae!5gbV&hz zdit@xA)R3NmJ~&F(++lun^ih|zIRYjOomP8xlJu1^JLDji79CYy6?hJU%IQU4~^s_ z89E+;!f!zrXYK8+uAuuUxl~E{+d}=tT8;n`qMn6>R?%bvLZB4JKueupREaszyF%{W01Yny;f-(F zB#3SjCYS>XxQg)!iusC+{Ot${6Bv#E?Uhn+stV71T1`?0mRhOv)Pwq`67W=w`LzZbnf z=X}n&zUTK(f4RCY<2ldealhT~ch%Qym>>W_irN*#{OOCGsAe4_WdH^vV&_Ml-2drF z=nG*U^!~anb3EyJ(3RI`U1sm~1lXOA7pMh70+!!?IWDWHdn{tknWgEG`aVIt@8ko- zXuvIa3`p1=M3zk`aI^d$fac#u?Z0o<@LN`)z}f|3YyhU`?16)IpO*e#mP&mGVAtgT zK4kRj2%#OQuy&HKW^(@Wr%(5^Na4pEA12=aJ`7eb{5=f1->BOU72kRF;Qf&-cC}qY za12c5Nv#gE5B>PYhvTwN6u~c^TnD7*Ul1BNN&YYao%4&f?-Mqz5T0GI=Wy*;CIWiF zpF+Th`Pa&5&$M6a^0)2zm4S})_&z7wd;H(gx?X~xx>CG%>9A&?{ z@v1$)edF*Ta5TPWygfYl>wi3W?f!Cl>a||t|Keiap(VfxHsRt7ckU*7613v3FNQ>o0e96 z%3r_|$bJO?OM`=3r=Ay^%hkDkLg9I4KK z&0nOqA3xr+?|~Jll`{kM&g74ed>0>*gF}ExSi|#f;Cb`l(Hm=SrL+GpOYHNyqJu`m z0}8MA*M4&}&|0__f(kb1ee?quNdk~PU~&h(fCA+5W7>e?_YJgm|MGq&#R1;V0RlGh zYkzEBKPL-*`(UB$Jd|?=ee;?9v8AbZ4Np$z0XS%xSU{XQ)4D@r#|06gYPI7@OVjuM zYfAxMX9rsf4{+^O>N)!O3UJ2f&j~)`-_>he=UzA;%p4C}?TPa&T0JK8?Q5>GIU`r# z%r(vf*X+S&t`6Mfb|ZmAZ$Bxa1oD;gzwYw=xZ4cia$lhs1p>y%bxa2|HZUhTjRzb7 zUHex@hyU1PbIK0ueOvWP+yk7i3!S?e4BxTuT&!WooQr&p{M7T?%me_pb-D(-imaP$ zoTu7U{r!4}zO(coZ|MjB^Dn}hGkQVu0Y|gYQJybU<%{oCZwhCFZUMPrK%>&=`lZ_m zFY1(%Yxhsn-Q~``iajf}sj{xa2d?9Tn}^oa?WHz&sI~GWCvZ1<*I^gu54;coW5B~3 z2lh;WyAD5?cKsXW+xsa0Kx_kQnRs(#fTiz?^N_gH&GrAf#@Kx(eW9ZnyXdB2R{aZV zyMy@M_}vv}wG@D;vN0Dq~JjMZe{?FrfKBkRRF;c z$DXr)-0p>E1CKE9P#pArz{4)&`Edv^Xwt&x$2nR30?i#lYUMER;(RR;{^h%SBnFD} znKhR6zy9U`q5GQWAHb3S_e%G_kN!l!Uk(ID15xN-*#C|L9$9D{=eGJApuBHg3p3h} z-rdj3Ec|=uYkS!j?f@+K{*ezLxYtC+0U$HAz~_23V7QBjZt47S=~TVm1q}Q3Zy)4y zZr8G(Im$?Qps)W{pTkvifaQi2V|Fs>B8D|t{#DNYUpjYjQMm2tI`Gx}C;vYe_WkFQ z`3qqUm_ee{-V%h5UnBOuN507sMecO-!?csN2k&erF#bk6bQ1Hf&TVGW^GWWXz z`~!_-ovX(5-{0WcA2cO)&;R>6a7kuk4AAcYUxvV6CFZaE8h759(+2R704*%w1DDG1 ztdsi*Y1F$5hvTz)A4`{t_}3W_6P+|QOJ!xJ1LX^twG@DYJz%On1-!}?6=}0BQ6CO1 z&ALFIu^$7FXl4pYGyk>q>XYVG?m@xVz^zAs?eP49>iRu8%5l>vH&1&kp<$$QNE;a5 zG7@rH+t14Vvl6Ei`M~C~qzM>K<#1K|eO@++>-@3W$OGe<5$PZKZ>0dX62mSK{_JqP zIq6o!`oftpfqVIPCbA?3PkYxg2*b{6wnMmq`Wx2yuN`mSn@%!oy%5Rl7!8sw^z?Ij z5u74f?F!ugBQ``L-kW~|YFUmFXC%Mf|!r|+4;y0R8Mgn4#4A7O-aK3Eg=DF(x)90o#uIGlQ^h5 z_wr9i<^pq#_3;n*Q{)QN_hQ-u_nO8ga@pRus&KZK#B7T1-t^JF{;$a;$|r!*rBUdV zVT2|!XvS#nnHviosewOsgz$c*XZR$~xX<_I#6>2nJB7=(O^4#vOVIW!Ju3frUXI@V zN&8WxiDy5&c0$DliuwK)^}B}pV@9092BaK$^_8ei5Mo-@Dm@!FXG zJ#<%o>V15pw8*K%j8Gt7QdO=wNIo>6hNenHwU|J;ta!urWw-UKnMbnjU=?T~SQ-7L zZ&0CnNMkmt0HxXSBmbgzLg!9SOh)*Tue4l_; zncgzl;)$-zeOug%X_h0Iz0Y2=@$q}ZAD>NlP5G@@$NVtxHwOf_M*%aM_OCq2Fc<{+ z_UZGs#Xz}o6v$jeLNBPObUXdjG~zOoxk)htV@5zvUZQ8>br=gamcJfttHXlf{R`pBnBu6;zA*`uv2w1{0qg2;$b z+bsy3K3cb)jo*F1VC8$=OgfV|1IRrAx4%H#UDIdMZxL!l6ZgboVu&j4^l#{0D15EK zF~7}{hL|)(R=9Hqs#sW9(5f~xF9zRJ9*kpERJh>^adG*ke|R%^9^cIi_>y_<{^Rjk zf1L7UZ~>{R1dS2{>$6J zJkJ9RyB-r*s_VM0=ZfNpz+bqS$kty`xBm8e4Kso0lCoFg#$DEzvG<$~@2ug6Q&$Gz z5`4*f*-99^wGb3Xv^0eQYvMZ)YvAa}7kHpI$PxM9GaWFw`5kWV?g8O|Yo4Z_<;}bX zFt(Cl6ko^=|Hau;Uah!mOi&yqqEv?&E(yP>dY>j$rkdcru1t_pt)6_r!*01;RHKQ7 z1%Wp>3~kfW(h#KVor&Mib_;BRf`IQCMzm#l601_6E$M8k`@rpG&MiauPV|}p9F}v- z-$pXGDjD;MxxC6Psid2YKv+wG!+GJh#KQCp*!pSw+_SwOuxwdD7v7s3jEMc(KK#B( zi25brYEel^|G}sZOvi%zighy@<6VMZi(CTwy?H?(Oi_2gZGZsDoVpqL<{<%%QyN(U z7)zD4ovR{PzYib+#61;di=EjyR_V&dUK@$BH8Lt7B`h}l*!Qe+c%DMrx!C^|z0kql zdF`!ZLK0-wRP2EcnaZ!qOY!Xjd;{2fAbwqG{cEq@8Y8%EN&5hiTRW zH5pm`POb=C0fzY9_0ornYq3pDEhC#R;h~@!Cd!y&J~@uOmOcrppfsNW{~vKA$au3&MVT zya+XScaJ4E;nE|cjR{gJRWt+~VncQN+H=7$&`f16LpMURVQ)%RARyuNw#PYkH%|R- zo_(R_U#qq9|3Q4!G4Bt({2Ko3DG-jg#TP+-F59%ySlohpf&b`S{%CckTAKyOPIV3LderC z5vWjG#8!u(O^;gA*1)l%Rj0Z}d3;iOb6}fIgl09kA}w}lVByQm<%v3B%~8pr#2VGx zVgq;;30No8n6oqBE+;bBH2teK76vUxP1UYEUztc06JfVa`=kr~>|Y$)~D&GLEul;#&L z;4XQ`r2}{6d~rZ>wnf2Ya9vR)It`NHK-}tlvd3+4iqq$>OrSO@6vLMrG;+nZC6PxP(b@#1a&Z}+7n*=9dAUk zGUXlLdhfp)YIO@8ChB=#q1k?l3H5o9dN+9Ed>S(`d|vtk1z}wJ3&5G*_O#i$8eEk|Lny=1ExoJVQ^tYFd&C+iZL85h z!mVa4L=X~A+-vzv+;aq~<_&MvH|w}}Il8*?oxS^jW#2$WgT*=v?Z?}s1NN}TJ@Z|_ z%g?1pi~8v$i|_6q9~Cpib*bA+*s_l4R$K9wSai86J=dR~Ziomh50OXzIevl3*fvcc zD|yQ|PpqD&v9r#=wy#1=FNm7(E^-+1m{MbGqy?rawH@$xl$ zW4uBhAVzny{o$lrCNqg!e)#@Xu}ot#Ytae9m?J)Ypx(TlvMP&#n&ch=2YR}d=9oxW{ruYlKweCH( z|2WGMKLR5x76CwU|MdHHdn`hecG?B?=O+tP_)yL>&9{7QQj&ac|E!I5?on(?uc&_3 zNe8CWYDlR)m4fs7`BTQ`St4@I&ArFas*(}E8G8}h>+d5{s{;%w`fd?zvC28kyn7}H zRIu@_F}m?26(v!{on2gf+uUI0huw=CQ4*FOfwPrS-qN;5coYR6aJU%81o33gKM2m9 zcS!9kSlzDoxnEKlMrsr{UZqEI!y#XFu=?_3R>e`X;KzMA9MX_UuT1)i&@>&Bk!I-x2H$*(+y3KNglIapq ziR>}Tcv7~euxB|*?G&rxYStcz7sFsi6ycw@?f`=1)YzZXE+g-1Xwcvx*K(!xH2<=_ z>9!x{As(d3vdIr-AmIOqqc7(f&(R~adeWDQR=@3Fe0^GEHFECXDjZR%7SPuuYpc8s zg)O5BCP{j7-Amxg)H%1W_c55;+gh(KJOdT7&rm?cL4%c_cP$i+Be*E z(AgrHFAw}CJz%+@aHclxYQO75y`$Gr9?;fc5Kpn}@{19E#@$Ji8B=<=i^tedFph+! zU;KHSXxsGqZPjqm>Ms#Ya;SQi`+z@>(CdaW{Q8w|vw+#sTE*aC@V@uFz8y4m?WK~! zl>o;13Dn~IFv_|GR>3#`|ACkR2qKZ@%A?D#}T<1xIZL~OE;kVVw>L|4p2>JivDeU zL`B{v;OO9vC=BWDL58oQi|xlMFo8rC*R~KdOJW5w9HJX>Z9jA_GyiKZ7=TVotSy0o z{}=tgvD1I~MTg42j_rSIUd|m16!&fafQE>EZf6gDD%^Q>6>B>_dGe3lqL} z1Lu+FHEG|MeTm;dN?FMwt~+nA=qD3MNw9oao5ly;0j8r&f12NZl*=RF;RYKV|WhlTv7u^lkf%J=EG7wM8){IC> zQ$?h&Ckayux*-Z`LP$82ypkh`<6bT*MHq%43^liQ3$1JKpAsgW*8AEl@;IIAc{$fy zYh-Tiau)zt$j*0p6b3^0C?+q{OV~s^Cocg{0AfpKMrY^)O1$iL*pR3NeNSnkR@Q3E z^LipjXmvTtNO;L070(*eywsNgVfA#){t0H#f=$ zO(Uy+jGa-rRDBX>gR!s?mskQeiRMfUeHC34G<=SLqa(<FYkmt|y1kl{w1fI=y>QM69^f^MCpWn5?nSz~EiCNl@>3evA$!1Mx_2F*v8_bli zYSY8OszDI_(9n{s#rw5Gz_lSQK)2ojO$$?Mu6Tq=uI*HP_zAr5}#nJd>I^(EUB!JlQCu+ZnZ}?V6@k){}us)5^zsHGd1&T@R=Mtz4VRx zxhkdLnapKFeMPBK0Kl3$j$>;UCAyghfUi-GsU0Ad47E*^Mjfp z$oq6i=2K2==t&xLzZ>AYhOZ6GJ1uzj4R(lzDl80e!Q+=vH?1h~+i*x&`qg$6Qaq-G z3+IQw0i8>vg-w~aZ48QV*L%#kZqEbD<{{1xW#fB?%@ZlE6r(>7pW1~%5L)2+5rR;w z2r*>LOd4tvUxvWn1QN+cb@A&7MO^rDReVD*++P5=B*f}#8HhCpHI5-gxki+_AK#AI ze=x`R+o&ylj0RYdqL=CA&(R;q#}}`ZD0{iZe-1E=aRUmx*xOkGNtaXu|F6{`vuU5 zoqwU>{je=@;Laf=)dY~_;vvZu*ZYo>kkEytJe?nAp`(Ya&6KYtr~ zv~T>3l_Dyv+$}BDW3GlJ<%XcrNBki|K&f z{b(`_EHs$yq&4&@!*R)7pr1%^ckHV;=19mfnT+&)8d_(q(!k|~$ZnJk(6w@J znQp^fSAa|>Pf;X7)F@f~oNej&d^C^Nw!8uADI-Tb6WEbLmm9;1Od!NE;>#)1A|G@o%|`A%$)t?Ii;rx(21kjboov&msk^ zm5*?|5_#9#etF}P^|+6b6u|LneBU^nlFN@YPK4@id5NdisNSmS(tIHoE8S$538h7= zYRs8|Dr6M`bwb?}_IUXxLn-nRu9`BA_@UwCz+Z1xaE4_xjc87{*|Mk)4G@!pbGDV( zn0v>Zyw$N5@qu6L#ds?de@*Sikp&o9%pm>&emSYTxfKY3l~$psmYM;(?`H@G^n0}C zy$4vjlqn9Kcu4NSD;)q<0G=L-h@8g9>?0~GAMVV*pE=~Au71bXdK?%@j6d!=HU#s% z8AxDaWW$Tp=6DI?#I>rlH}d5VbWEbI;gQ9UWrk0SJ#S*W^Q6uV+ zXE z290H3Vcf0a(`^%8RzoK{ZVvT6p;$MCjS5cej*j^faTdnN4JE?NcGf9}aG|I)3tD(S zd#omX8DTz$UdY9?P~|{WlOo7~DM78-P)+P-=1rb5_qnnlAhzQmumgw+E%O8Z(c%)@ z{OXsNvO!!*C$t9ZVqYKw8#>?2CJa`-#*45%{G!38w62 zfd2_l?#`wG*p%X5Z0h^RY*%kKAh+$K{DsFdkHYn7Y{5{lLPy5m&fKu`YH8m#a$cQ5t8X$=*na%Tig{*1!vvw#g?wESixL= z#dm?~X#@FtCaLVgm=<2IW!N7!Z7PLL%SyM!)Ld5A)+v6)GP{)j`=i$lKXLxsUZA%Q zCAh??>w0P**o2V)^#kOEEFIRXxt{(0GbYHcnp_!bd29W_!g_*Z{$&mB&O+((mCRc` z2Bu~P?Wac1SM=>5+Ar$?r!x5-SUI4}c68sh@#(~~T%a{ze`Zcg@||ei{h4U*u_bCj znX!;4%ZK6-6D7H>bZC?<{sK`TDdSDUvHFJkt1>dWZeF9hgYlfu8|1ygmk0y9;t5}h z-yFt6$*=688>Erwwh<2(Q^bW*2e+E#v*7sr=u5G;P*zKeA~eIG>{Byelfo;Dq%RK+ z;a!uQYSj?GqBnFJ)#u&=k<<~ukRHgQZsTyVba$M^$V=2WMo^?MP4q6oq$bq8gjEDg zbCjMBZx|Bhu2htwyTWt?W)LyMUepR{!?1^RJ+cq?n1JXv|DdhDE^>DL>VP%wE#(4Z zC~=efxhfyJm5WRbrm~V|e5l6$7l;Z>#J%s>?_vRRo|az>EFz3lVcFLOIHi$}_ zusixIC}9PiY_f{5tzA$hi|lDU^Y7-}bmwCJsR_?eBU*z(pPhXsO5hh_l& zU*D|&4KHg5=BES1fMK{JP!1{kN01z76BDkzxbj`cqTzR3*zUBjh4g~o_C{O6!iSFg zzVr`XglbU`rLr3`Owv=wwJ7w@HgRA&Bbteu1{FSxHbKD_ny;y*O7)?I7d{vutx_G~ zVg;<7znErV>0!N2cu!UsL2bLuAF`X~|C2j{!6z3eaQ}!Vr@pDBhZY3NdCL`7+JCTLD^-@O9WZza8@N zrp0Z~Sn+Jc!)Cn*^Ln*hv8M9&;)ZHV<>S#&9sU6UL=8936dT2bT4QE?gkMiTMP4D+ zR(BX;ZVwvD{Ossk8t$LOOu3PFgvU?*PT9~Z>LPb#$0fQZKErl|9?eMIV%jWTq6Kq* zv1$~G?Auvg)_9A`l_Kl8q$$SWxH0rq}S?`Ys}1r;3KvaHR!-nhAXDCvF;yJzSo zr6<$(iNWg;I4SJ=#O0_lv=*bFErccIEd`DjU_7KPSHlsj8<+1Fw3slvvlI(h<=-i? zVECF|sUKH_w#?LTuj z1YgY!-^)vH4q%AdaLM<5$mJyt7||So9Y;N18D&V_Mv?MB_8+9VEBg{?E$nxJTuf9e zr%MQSJTDl2Z?UsOPc?kC%&lEyVXDg_#99jJYh{^*c(TTO+J9}NMf+n=wj;@X)1gZy zc76^lnUPlGk){I9O{-|b5|!k2Cd(9FZJQ?$K-kj@JIw|uk(=((e3Wz`wOSw0t3tF= zfog7h<9zO4=5$nu3wARIO-PgBZ^x(Refq3XAy61M#S@_R0S^DzagJaPEN>;q5~&ER zLS=2nZ(YW*9N?Q@|Jv`hMOzvL1OLW|1NGBDl8T^x8jys97a6+rm)mx@&}9NJ0EAP3 zU$s!Cn7J)cv3N0s zDRy)Kn+DMy$O3EQBb6>M5a4!eT*ir94<8JEffjAJwGCg*ka25lV6AWkW>rO%bn^3@ z+lJQzu-;$pF|3MoBb|X5^IPGkjaNCuAkk-(d<1F^>jhkWG4RAi$n^71h;|Jtm@ftO1QzlJJk63a7rXC^~Cucu_)CaBeEL9B+`3Fep( zmjYR?TqAJ&<)I6g*0ASJ6-WB%)@J?uwF6g4wd7X?Vsaw)c_%&R!>nKADqk1|6t{>Q zL$|gf6N;*S)^mM426d}!4|e?(A5vNL(d~_l82OPM(-_zsEVfYGGCvY2z5SEEX{q?} z9cs10x=$&{B+JV_?ro>IySrO40|;4B4Q zUAh4qtI2*F-7;2=KL#|(+YVg%Gl{gxtzSR2|6ztW=#J$D8*fjTYD1Siw>K8#HrC@b z>Q{0b>X)Yu1EM2#s_1@kp%HTp--St{*XLo&dERT|r7d@P*@!#~_x5Al0)fwhCf-~= zwbwz0hb&LiqZ=Jp6W5e$=L$Q#B_!me_jR++Z7(itnmF~V6GU^jy!SIZAKLu|y-Xw} zC{IrNO8s?k{~O0xr_=wQ1&CAtdO!j8{_{bzmu%5~pjqxsv2Ulyed>p?g-c!u?{*DS_K;%mWB zNJ59y;I`%hFvKAr?a41TUNRE1vzf`DW8tai;s4WC zmapCf-@euW!%0@CDoRW^d$|azp1YMzo_Ne;6<$oEOBIJNPx<<8p4p~pIvN-f2^o|I3(^n) z6CNHVJC<7BG0Ximb}8!P)B^3RG}6sXMZs>NXy&Zng#&W-#sN7Sz4YxgFVK`@1|S>z z$(Hwzb65QJOyp^3RFcbMH2(weIrw*1st*V1(%4Wc^N)InG4=vFWAOta#wu67g~7P2 zKM&SBO{`NK=@06aiuhubTAH3%<*0IZWQx%>)eH%A9Z1Ask>8itG7|nF*hv%3^KZ(I z2KBDv%wvYc-@W`iYB10Ty-Le>nYzLO;PWSC1DPQ z_E%JZ-!kw5;S+@&r8t>~pvC6je)-IQ@ZjzU*GV|We9j=$y~TqeY^9o=QgZ*^E7{ZA zHv?l>yL%sCNvw)dLOT!BW-f9}9r}riTxA}pnc)sjvp8p{U~iCr2dALTXp_UVtR5v; z?LaUQ-fvGenpSMU&Fp$rvZzqviVw4t{XEGifU%pHm?v$5sBbPqbV^t__d9;Zh)aB%MMZil z#Y&CoI?H}PCnm{L&HB3Ik<#`mnmlSTzGfN}$UY(mT#aWZZH~)^?ED-aXjB|-*&f4$ z`ert!fs;!{-@c>G+YK%VCUN;iLl_#&zJXk(HckJbX<} z(VY25W&Ar?`@7@i!fh=OcD+C-r0RVu2q)~iqw9zbU1}`!oS61k`eT!XYW~s1rhJ-b ztxWs!%F((xt~ovzH-r!l_x1=+n(7fA86Oh+$3q3|ucQ4~*(igZ$&o2A?FhX{e}Bqx zwvvRd6F0|Ai_HyA0l6?18IPMerrHRe{NZyPd91LWROs>ljvS^zDuCPr5jSG)#a^SKxqxc9iCfys{y zhie(yo;c%T+Kv?aruZx2pD~hYzR{t>9{@96@o$}ie$C_5bCpla5!Zm5I4=>VY?~U{ zVIw_0bFIY-OYHb^Z%TsucDpfcU7nr}9#Eew1x7I-$0ud{a&m$FKUVJpvR(Pc_X-j6 zIMja(QA)2*u|+ihAD#JJwQ~09;=e_o1)J0#=Gw`XQmVG&Hn) zjY(l72u~%cV$IfGQFcwP)RQ2*AXC8go$hO~dn33Y>Z&L2C} z4mgSOl-vDu#nR;ueiyq_-;qQwy2?nWL@8pk^?i=yOsNwuH1U3MxnC1t-r5muyS)J78SSYg@n?IIw6J-SAPXE9E}Kl za*~M%0j3WNE1AO^!=2l_nX0Sz+mbVCYCdvCjO16>)D+r;=%RJ0YZjQMJ#!3lotrR7 z)2CT_>1vwSjErG*-QBbMtvdUyV5#R7h^sE%qeo{pzI>!BD&&{CzhgQCK|kR=(osjn zHE0Y?j&xmCeAU?L+*>|8CX~Bc<=eiO?q3dz0wAVMTMGT!W5cw^kd_e;F3B%OcYex1 z#^FSBRi>Yg@u+S3@=FS4>g6u$OIx{Z4|mt3XsZ^00@Dt;)gFzKS@<>;0pSPSteC?7 zvkN1J_-m`4{2{vKJ493|b!J0AkX(Kvzxs^hi)Ud$jk0}Mv7^4K^u`fWP?=(`#)8}p zGBp1g(+3~6L-%Bli+WeChJ#o{={V0kb&ff=OMLgN%*B~BxwIJRV8Uixh`wY7syka_ z+hpMz?z`Y%e7vt!Q_b5^GhB5{#+~UqXc@fK1ca8aMSmN{i3NEn z45Ur#OtgqXT$W45Vum7>psf#Yk>zi~)jk+`Rq3All011^Be|s!Y<=tIJLmR>Ey?$a(RQqTvEaN-u~9TMXZd zNAtgk+*5URb4z^y3e~#UG6j?DE z_DiH0agyhr8@!@PDBw?!q8;7WDc*yA*<3YZCr;c5%2eT2ua7gNDuN!QhvB%qJ4yVB zZ#H;CUJgS)q`LVD+s}M^SOCnRqi`4Vv7NhD2 zbsGaKzfxex{(Vm72&D^B)c00Os_!+F%Ym^5y)Ko8Sn+4nrlrB2-8>`aXb>(;>nIOU zGw+m2Iz+K(3EBI2f!T;@l|f5;Fwt3VnBM94pyt!=&!|n`Ev+`hvGRJ;)xDjtpFvpm z$*dykhAp7=X&}Z_B#@!bed>MM++Q-U^lv>2F(Fg7hywSI1}FP$#f1!2@b!ZoLhkHD!wY9B zN{d3#Rv5}4x_GCZi!2k4Ud_f}Mz6B6#>4SJW$WTvorVE5(QdbgDLM29r4zsp7KJjD zF;>22(1g9WsPSUj;}qHJyg;PKoI~Z=RlY31+y>Z=P8|~L7XD_sw(r4y5f|`2SHsT* zi-h={egE=5c>}NbSHR%VLQrR}ZK{dLQLc)sb0eU>!OB!Ir4};hnfq zBG2W1O@80Va(%XxVHhTDRbAFO#h`EoFOnE~iw^-h*=kFNFk$V!YZRqZ^Xr#rTYYfV zxo}$jIq@vXmtd&)eUTwuBiq7xXM8G)vyQ**8>unvwQXYiu_Sx02O4iAlyBTus_?!lXCf(}A~tahGry7KPiKS@XRVXY*>sdI zmtAX-^j7CDtoD zL7|Srq3sTRWSb2_oJxK0cOTH&b<(kff%Dk002o+av)W_o+I{Y^qTj|<*0;?CEu3|l zhP|&PY;_3DPA4wkZ=rCy$)!mchc@WC>9`f&T770(l_v-ivrUAvvdk}j=SQ#>O5l5M zOLY;J<}DYmmiKpF$C~RDA#Us%TA5y!JDM>+)Fh5fwy*t>rP^W3=&-pSC4mizdlPf! zXx5%8w?{?FN8*4j{S?KZT(PtpwTZEQdZ!<4Z{696BR;FsZ5q}EWZW%B_xUd+xHIfbrCX-;_hDYf;sX_Ta=g4lfj%acewks-I|`6rWvg_@pR zlPZimy0|mkvw=l!l!P;vfmQ{e`48x}CNtNt_ZN(zqUJ5zkM6D(>-wsw*o;Q(Ce2^G zRko8r(1_I7y5E-RV@CPBYD`(3+LxqUC(73bt~#4)y!>#h3_qG+k$;B=uEuH&+%bZl zw0{Nv3T3hV=Yk9nW`eGiwGk~OqoF7qXRdg?_;)=gf9e>V{`gBM+!%RFw26AtD8B4u>4vY}(dRn3w z(AN;kkjr|8g0+cgG(+>Q$2za2p`F)~Z3IY(701}Lv6;Eab#$3>^knG(o8!}n%xxB0 z*o_?SdiyNSp`D;kxT-nY3aWy^4KJUN)tEZ-q`kBL=Ttq!z5tc*<9?2ufOCHzv^d1N zbt&B2f)YG$v(V&?yDqxtQIaYFlLT3*rDSI=nYQz~)i^F;iWMb5{pj2jcqK~^%>6FQ zbrm2^BG#ZpG}O76c=rWAId|Zwu}O$~S;E`XUZ5r&casdx%h59R>V@Gwqe(5*Wp!U~ zr%Wu{7d_iE3ipkj7x-gI%e^Zl#R8G z{h;V%zaj9&_{SHEPy6L2wj8THY<;vL85@H?8|uSdOFV3YjhZX_lLoJQSR`G|=HgEp zBzdIkHK0s?#C~|(K;0bRQcG$PK_{|@%j=GQ{qwN7gMx+n(2oZ2*GOgP$*895b%uf& z)4LySCpvbYBNnYETE%r27IOLrj;1B(PDVC)tZ~!RMcAzL)r^JX0}4HafVVXh7x4P7 zCTLW+1fq)A&yv)3tI7N!ng}X7sX0}lI21ocV=VfqJ)Z4t4{?u=uG(B6#|ZS(GGPPB zN9lB?K)i?T3IU1mVo40f)}HdM z>iLe%7&?h0rQ73-o}l9Jbv>gKHld!KJT*P-Y+|mm$>J7l%50c8VPg(OB~9PMB53b- zenW@8DlO&s9XXL~clgfXhnIZAW@x5CNg0DIvw_sA2eX?>OM)NPVne(ZtSCs(`MA(Z zrwv*<>~SP14!u?~6L!m{WXlS&Ar{-A5TOiTaw1GYDlIld7ce(b|G+4v*Waci7j zuSj|&o*QnO{k zsV_OQVXoy)gX4DML_Ku+sZyt1nXCtoFl%;<$lxx@v1pJ>c)0$IYZhSBQh8F**)617 z^jJ^8L_rTKvwfGp%3`A}_3NHlGb{h~P1Sa6wSsdhzCenc`n@I-5n(lz7P(G^uxriJ zD@v@x6m2(X)q0Jf6262rWZxq6C*w!(Ol4;050Ws|R?L2eTS_C3Af}PGb~P$*O^Tk! za~+z;BNzn}xZW_ceOEtTHx)fK8H#bCv~2OkS_P+N%6V~1j-`i-Er{)8G4OfRcq`3U zd|6U*OB(>!>?iF!eWyLm1Uy{BO801p*L`oR0Fi_~AX`f{bgv{W$ZY6M1O=L=>rV^n zMoT7NMRwyoI)|IC;^XQ)_j)tb(kE0N+MH|?M#Lv~f!XRP-s_HgW0Rl_&2$ zEwvHY9+V%hscJxn2bUn(D|mAjWETdurRV68(xlY{J#Lv*l*5 zv)BqsZZ1qW1HRxPs^}y4$(jP=&fVDG9WEa|VEPO@Vo$MXP5(q4dV=;TD1IJUt}(V0 zA*9*EF6;UEcqV6)*Y7xV0~gv3vXxX0LqAkqUTc_3^(jFbq|S{_Ax#H(QRqFri*s2D zwBFQ=xCHY~d34Yscj1m!rUxzKwSa!=YXLodVPmIv-&-+8-~C)tugM1OoiYPw;x4{^ z^S)7l%*S)}>|m~16xKPYwg9u8DQlOZ!njfgqlXZtJafZ#hI9z@y{zmdHK14b0j*Qj zoTlQ={L^5|;88{-w-s75bRdR-(NNuPkt!OqhZZv+$6nfUO@zCz5FTSlX@u9Ibf603 z#>7Kw+H~AN2)7F}!9J9fI4E{et9>GSS8yp2s*cn3K&bbYX+ zLngwiIUF~Z83vX>;;kfvP?O>ni~JcF-4d^nR9%xjUst>OWSQ{F3!LffRaV$m&4rD` zV^yGddx7oW%bzMerW@)>Q$Y@k+2Fnpr75oJw!4Ae@25LeHEG$3N9OJ-1qPtx=5Cpd zeBZkIb=Pb1GYTS+riy-hlb|-f*ePd{9-1wUTdoVoOuds}DiQB->iZqSyR-FuPkO5j z?XWCT(-(hJSYkY+FRL&1b%Kt{(p3+BiBA^Xef2QO!4tB2Im?bjQ~GYu*@|PC7jidV zm}cGz@%p`4U(mB7y^=Yz7+(IlrQiLXkymHNo6B_Y1ra*z;$=FF$HgP@)bgakV&_igIWpAjdXL)Usnx0e z>#J%3^G>Rc&gTmwu7UvJV|#j@{wJn5QlfQhD0Q-et9aJ%wRn#2y=Z8i(p5sVJw;s;-GuiR zC+hGOGON1(K=b%gueta6X1Hu}pv4sip13FteoIEVP&vYXvE;yt42VA=G1@TmrDL>U!<%S9Wq70x1=r8G1U*^w_~r zqE2p|SRqh!2JUnPm;7u?XnmPh>J;zGz&NBBl;jvk>!FOoS7pS|epB?@fc=h9RYG>+E7;DN)s8jzZ6k+0h_->Bi9D+SXVQ>5X_WXA*>4lV=^$s6$ zl&j;tWE>uMTGV_0eqtf}H~|+LmedCMDGm7CF zd(W_@((em&MnyyfR8*vih=kso2m}@B35-An4!=nfIAZhlZWW)UJTd$4`wkmi@V z=yyl>gn;1nCeiBcO}^KA;h7pG>jLe01Ob~L_^&4$iO<_FsL!!ck`E`5u^g%dFX9>X z_NwM zxhh8SJ8FGF);jgsNohk{AgjCkk13!5GCGnP)&O&#b(?PkPG`f2wsS2DTzGa ze}LG#e2m!JaYme9Dk1v0YwC)CBlKfC)5R$#{ob6?1~I;Xr}PNkd>ss*?aEMDsZhTU zw-$jLTnq;4rVw-^v@DK<6~4b3x1N;(75Ho+lxyh)lvW7nTuW_PNikFg@5RS|@4zgo z4q@M3O6yq$h(R9_TPwFgUwwtO6@upRwSIwU7_6;=M zlY%hToUL0S9y`qoWCZ*(H8jJ&)h&?dk+a@E`|iMe>sZe<;PNq--d>be@uu17=lF)Q zfL9;MhAPk~)9}80$WTkvWqACMzasZ~i&NJ%l3vGW5-Pw~w;On?1vh<`SclyJHXB!7Q+-GwYb{&#_M@qccsOIXPLwj_T zR}$~_H9-%zJhpE3R?C~rf?HXuOC_#kWt&TjHY9&e%*T%HhPVjttx*VHMA}r;VH)l^ z9zdB9CzIv~v)PNcCQe`pSvr5Q?!&G{(Cm3M5Go?&*L}jy)mn=V(j**KJrM(y_cEZ8 z)?4GBOM4KPt2}yp!}%4z;IF#{V8iL8$5mu7%f8Jlk=4#^3XN*ZFVN`jsG!x&c7fI2 z?@2>w0f*UrjDt;L$-CB?Xm}nac}S(+zfF#6nJ$LXfpRq^U_MGvwV}i#9CLzCj#uxY zvFf1PMwNlZ@Z&Y_b8^}tG*_+6c&AOU>hVe=`9<=X;Kh*EI0wY1I3GxGRI##Z;Q(O zi3^q6_LAO^rL-&0!*w|$#`Ka;_w%GW=^^N>caCw3?1q_=`@V#0@-|P7O02@KkV829 z<)oFmajcvl<(l~=yQGOXG=h44;G_$_w*9?!ZSJSl+IbaNqsjun3c;pmSJ#2kL`*plRyLqo3GZ1Bln)61%;aPv~BZ z3rpXLSR_PxfEIV_`S`PlSTIMlQG5V>gn zv&2Tv`h_Wc`d}@sFSRnrB*qQi8h5~`i*OAT6Ad(|sLU@>X^W-Qh%tGt%b!H4X_(7H z`_U8CJ&})J8uT|B{g4`Y{ z4V4c)3)w=s3@8<3biZxxOC?^%bIbbObxWDMZ(TC~<2Ab?%GAMx_DFL~u(?pQc*~AP z4P0&0$jjKQ9~xMublOz?K9uyHixIag;xO)Fu-MTrutq!{z7Y1B)50{FCpZjLlya=^ z9-x&*3&x}fgr7*wxWK(M8BO+MBP||b9se4LN~BY7)W9_A{7vlK3$I!Iu9G+8;><~^ zVG^oc8?ZbGvQAr#Yas8U*1$X+V}oYxo7mQ)&aYMOEBV7nj!3nVn%Vuvuf9t4Y^cg= zyI-w7Kxfl-bIsPKS~+OXd)^cT`?9sb+_UymZP;354MvibY;c7$(eFnw?OdcV(G$v< z(h(ddj7Rf2mpQZh_#TVJS(xzg0c(XzTla|FSanYKQK>A560`$1&2Xx48WQ1ZcDtaL z^dJ;zX~n0yrdci`Zxe5|UAboUVc#;!8)%#OYw69$-px)RPIQ z{<>4%JAvsE{(dH`C+Sf8U|Sm0r8v&D8!mpew{DadC>g)KHME%O>lT2cmXz=-OJlzD z%~nTZ7HfjSrJO+-XZ?+`f znHWUKWgC^$$9$&=cb8=C0u@gUJuw^qTD4M{N@S#DU^nXUv=eEy#_k5yl5=-EL8M~5eTZQhMJrp7`nsVbHf-8auu~_e5f?uKL_z zvP5-s|DaDe8huT&)~Yparb+Kxsfp9Lfac&A!;M~1FY2KnEOF0^{cj$N_8SZ`fePftqYq_^VQh2Z04pq!>O2I*lmhR;oS|hRYe6^#tgQ0<^y?AgoBLBx5j=a?2@jW zZ{07$N76M-03W9XDQS29+x?X8h>C zU2o{C{9{P{(y5lAVlm$)YKP?Ox91=jKopzQ@X3Mjn>27fw3xY<4%JWnhRMk}G z38lH7u5<7?4R!Q5?S(eCx@GNtk~g`uG2dtyoiLqTy-LLo|bZKOr6!Uca*61H4rSnu7D*i6rM zH0z*q6IP+T{!QZgvRqPbNgjXfLczuZ(Ns%TT4DNIqS^@_N59@LyuvY&9~H4oC-p_W zseiABrFC`%01BMyppU$o>noR zD}#dcqoxT5dNwLl2-VEvYZ^;`d*| z)u$oS{Hd;n26iP>{&cof(7bI46$g*z39U)hY9W%yzF+!;)>N`^8fVXW5ockHDo)@r zPn^n1VP26mxbYqTK0;8)pP7eY04?7G9MBO?5Pz+(LlBaf%V%G-op}eC-OHRbg znTOJ*b6u{OVk-`Hbrx$rhtfcvNI0?pG7ly(p#|!*2_lnW)BWMi`4$`Q!2`v4jzvka zCQb>&NXp#Y?}PEma276yM6m<<`v9PRv^=q%BPCJ;Ri$tdu8!7VjIRg>`WZTHjYL1+*a%;#a=eO>`NF{T?}} zj|55$q4Gkl5~in<99~#qpTcv++S3%@dJR{31!_hvGDXV!#NH_sWN0qvUH7dcjaIOL;?9L8R*>A-~WFqBOO4PR_S2#XTFlOxx2 z%5E&CRiD9bNNf(O1eUOxoD|bAv`2Dq7orv>MnezIP>%S&&hEUEBI*?^zBFdwe>msj zG(=BquR+)FJ4--a7jh@>Ywjvn`npkN%op)&%nr4H-%AW#L7r5DMqad!u};31)VJ%$ z4+&fK8)*9^7>c0z>4uF29WWkeSKIwAU)V1UTm7LCXgbEg>D*0)vFP&60d|dQn>a0q zj~;ioH2=c`UnT9!^|x1A^VVlYDVsM7cuV02A=4I{;%7|j$1WfbqfOjoq%SYD|K zc=p(ZdgMHu93^;F5nk&f9yv$1H)l5MKg)MOPHv;baOx)O@pjRX$=&TJgv*j<8i*^Xj`qP^QMs#)lHQqjzRg0k!lyvkZAMd{6_0 zFR)Y@)fz`Eh~|8igD^HApn6$;zMAhc{v2@Pw%qg7`|*Q&=vB+fC%-(wKuv8Uz9c;M zi{(FFXLGOZ@AAU${&7LpbpGYAXvUN4a%nIoxHLT`Nb<0e7R0Jm2H>Y-95yL zEe(UD2XN^T>z(=Tu2of|9~q(gHZQA{8D8}Ij^&J%eNPo-*0bmE8qdlqZ*!z_$2Wwl zq6$boIvh2lDSM0n8XqSbN76~_V`^Wh_N(2Vt;SyNP!iUIrVU(#`qLgr<@lh{WOe(5XZZQi>?X4Loub#YhD^$hKQ!m zCEq0<`<3^~Eypm$7TRoOSBWW|B}~-)VlJ3_X>{8Y8FK=s85j|$NofH7x%5iQHVUrh z_xHGtBp?|58hZp2*fn>uA(;;lpRf7|7x*46Psgt$FYoKYOUwVjB%Wt_=d0>C@nEdZ zO8;YOBY;0UOr}H}r8$M? z^n6ys-BOBeZ%+%(c|5T{{${Z+gY}dNOb;AY=Wp9muuMpZ&Nk(v!)8+yB#oS*iRhP( zG}*gXOE>BFi8^}17vEuQ8OnNmX|rhAB} zIv9VH&o*K%=K-zW2r5;9I(XWQF$71moX>v?*A2IkO$xM~U02L?ITabJbIDXXNzayE z`$p*TYzdOZnCo71yGernwNf!l2Z5 z`Ip#U%upm~$$EeS?o@EdB6|x%=JB1dm+;MCJ)8CxUzFAlekQmtEpq6rPhF*hfX3&3 zSf-j9WM%orECm8A1dJ6Eoe^HQ*%ij3y#zhSP{y5)Txr`Ue02lj?v zOF6GWyP_B*s~X>fXlD7Y%Pw$UfM-5*WzRFB`=YjrxK)jluaO+(sc!IT4(k(Qsg6xm z?=OG;<=x0~;79bZnG+WW%wFO}i*^vnMaAmdqby|pb05aKS7AdTY z>BG1@f@&X+NSHfd!;~ZzKq7PD;^C>>2$|Ww?thm&_^$wxJX}=8rcWDF_0mLv_?Ah} zsn$K^rG3UW##!GnO`4aub#u13WbdMiS-7RyvEqa>dO`Px@_xcO%=SNOc8DNIwdc0@q(b6*C%5}14U$C~_fD`sv&u zYuNN(D}_VjXU&bpRUT$DI0JKH(~5?F z$zc43_gM6y3tAyJDcV2lvL4rDFPP(s${C-N6unU6RWcqecY!-Wa_cE-Kb<}K+LfGZ z_GWjL=%DPB9+}N9Cr6f&17lJB4}s4C5cnx5IISt$EC=@HQQ8(A{|J+>pwFl5+%Q;W z#qT1mZ4*w9NN||}(SK^#d!NQC7C$~*xFPA!5 zSM$X=O8cqDl*v*KGnw4PA*1z?I&aONB@6ebp01qk#8y<H$hEg&%|tI z#Pm{rC@osA#h$eq>ab3$?Z9u&(t!M-@3kpq;n&^hlo!gHg<~4_7hY%Gd&FF!@)}igOC48Wz35;dcAeg%#$|DEHEHx|^YBBX_zKNN=%SF)CWlPQ%(AtS ztW85aOZ|m4#o{$8EG_bd2Dh271H5!m-|_U@r027g$?IJ>$LFOHVWFYEa1PM!Mh|sj zF93r(1a?h9i!zsv1r8RIqL(*OWpz zp3rH?E}0f&T&9WcBtO01G4MDOc#_V$fbR3pfftQ=$qlvySicjLdDO|-ld*pVdw%pakH7sbRS}+XGOpRfKM=oU zfa)1=b4X`8?^DpLy|pwJkA0kkfP=Y5I3;s>Mw|Zml*g7Kw*va*-61*LS37kFP%BJU zD}?`hmlm)RoCDBFNj=+}y{6Zoir_GXv*6@cUnpp;?>atUzrJZV_?HX%enA?kg}zGm z8F3t$&^^>4UxE~4%iiW-qkmiQ3jQP*oAT;96z7Ri_BiugJH(+}aOO4*0sJQp1A>kv zE&9CEUc-qgdQQ&?Naa>$TKRs}aC<`kpp9~-=?C=X1{t}k`J1bB!ceN+c4|gW>paNr z(F3Kvkl(b~x$1V;sLrYcG7<#XT9u0+AMX3^)LnPAzhdVZu2OLRwUBp>YC>Ciy!_!< z*~o<6+Z2SrEMLs_TZ0>$(3ppG>U~i|w7HfFAFQ9P&C;~6M-O@0F>LlsWcjWB8ve*7psLP`YWndCP7gfie~dq#S%RkU zp1}tTaouv1a1(`wmR>TB#c*CP~rvcwV}?Q`zN06i25#Ttiu!nTzEL+S)fnQbA9*-YXIYe%AY~P?!t%`s^5V} zIOkV*Evl{MYo*P5f<>Bn%R@)1hc@qvT{q6D^uSV!(&`Q|NT*UU^h-O7{%N-8|7EDe zm?9YiyA^=EP<4AThgZz19^uyv)qaXu&5@JF-z}XZ@aQ>O?vXMg0!Sg?Knl@zT*8(G zd-K)#axiZCZ6|hs!p0kN-Cv8RaugRV(w% zcxY)oAr)vp?2}_#`4pX-rBMmfxuC$Pc=Q&3HLW-Nl-_1W_)Q8w^MLKhlU(^vAd0Z2 zCm26v%Wn$;*THiiIC3zj_M;PQuI6ZYuEwd;8$uf$81g$m>@<&nu%fh*cBtuMFyaR2 z;8@bB{k5deVTFvn#O=0EeyNnAp?O>K;ci@((oVX~VuLU_2E_MLPN?yyG?hhCuJ1IY zAbI~s-+wTZY&-vwCz?W8VOxfXJos|D;Oitt^qb;;@klP@FyMP zwIBiAYDeOOKJz8*HvNRAf~WO&t?!HWE#S$70|{WP$)T@R@tw``cb15N-~>ccL_g|p z_Lp+Dc8rj#f4D0-w&SxhoZ=Qh=K=Fm1FNqa%_R!^pB<{c%YxcKGfk@@01-+9efvjv zH2d2W3!oynQ4HeJcY-shsPz*8-rrCs$Mz;7G-MtUSU&Ssw8a(Z3<9>zcQ+V}Q>=wO z=1Pz&7ECqO^MbwcVL@~DglsGqS4j*}RSi>V?b{-3aw;wJ2Go2UVt;zvn_f&9jt|IvV6*0rId6S6*@S&3N= z?@V{cx)(v#Ih?Vk|Ckju4DQI8v=ZzJ7%q4UOydGJIL6%F>o55EAHO9dOF2J3UzOO8 zEBxdKSd54{bIbt_Q(Ey#D^u^nJ38*!%w*f+p4)KCuxbo^B5?%{{P-R*0H~4`Pw=0; z`2ms!RokbR=hcetgFNmO<}s;dvRDD*Ue8 z?pDdj7L;?v?tIGSfbDs#^=ltg_LqR`^fkCbK!bdR>I2`MYLs`qre~cc_xv{wb9B;_ zaldAN{bXM1q-#_Dl6ZrmaT0?%@6>xMVgi9o@#9V98T=C8${9T~WjY2zuIE)Lm%M3e~By{m6nP)^d;DG zy)**()uz=N$iq$;u-}Vgf!aYL#*C)7rC44=I~LRF={A!0;!|GN>Bmi%l`OLG1o$d! zVZ2XU@C=VJ_ga;?$F{3VnQMCXwt(q^MMLEB-r3g?^Zw6o4ILO0hM$kTSb9!}_9+Gq z7>}Wnc!<4=HDO_5@up^A>daKldME{N^@4=5PoD+mrCagWK&#wCzlzdBfA@dy1bD{q zGE^y1J95VWdc~-*cnf5ZJyU7{qkm6-8evP!b!m{wPv=*5Q^y$T!sAHN;b+*E`H0nk z#q5lShBNPi@WbFkRvis zftdHWNhSyOS@E_Xuz>?PWE}NFl;^eD{oeb)VG}#??W|hsn;*n>z1fPxF*SMBrBm>) zk9WInM;Ml7?hD2C|20Y>T1@@D^5=7oIV390?DNl0)qu zdfGHt-W+7&j{L3T(xo;f`b7UZF%le~Qcj~h*~x+O990CENI{}qoP&RpzCBgE6zQW! za)f^otq@ZjrD|-Zce0?q6m_}0iRFVbRH=#!$g-1#sQ{v-jIPyH8vP|YSc!8Cvgq8m z+OWYHs`k(@rv0;qWCJO?2pMSDO~mjwFX)+tAXq_7TtQyg>u5{>+kBf0t(xO6WaL9p z$gJoV$BCXMtXR}o`r~3=@gIU^W+klz8!fGbMOV1YdSNN{dRJ;hw9?ALhK9thLjePc zu^{a*cY^QCkU@Z{iuTy$&C4}cGO7*vCtc|R!HRVg^2xvqK(d(;{3d9cr^^Q`Om zj>J4?4Uav8cyvrdpL&1PyDL_0&5YPkXh-hG$6prVq6hVoO8M_{6nyBcd+Ejsv$Oxm z4Og^NUGQ57$X{Nreur8s6Ix_Qrag zK60K|KEZpIzAcq+6apIo0#092`-(2ar)3hCZy=EpBp!18T)B~x7~+86r1OOP))cIq z*P}902xS^6{23JjCQ1nf9SM0%<^6;~YQHJ1`CN`%*KIZpk&Z)J`w(CD%X3Mu9YeJn z3~!^x#x$I+QJLQNEI}T8KN#v+)1kK$zVKdIKxoKb8ENi97u2_a<`ny7OccH!g2eiw zeYACfxBMMcdOMcW+H=7S6d<|5|+o}ILW}GExx0|$sjgO#$^ahi?R9< z-|!m7RsRpam3qphe}^PO(zey8pxfJ!si4K*cyp667Z4z7Y0fAXnpwhiM=`GT$X=E=9WO_^?)`f0$5JqDY32d2%J#Y$ zC!02gS8L9sso8kj&G$lS0=n_n-2#$V-Ca!IJe#1&z@Q6JM`IKP^TGYfx(#ECIMNFP0?1nERG*Kd&Ny%8k16?vC^@s3)yGg zttSEUUVWidjH`nP5%DJ_kQ-5T?8`YE%ROu zMm&OP!175oysuF_tvc+|$;3;4idUMuvU?}a1y(!jka>#OI%{gxISs5cR)96iKT?x< z+C%+7wJYj})OoGtTq^D0TxGLti{=btNDAE&X3enYpNec{4@7ytJGq_w3RKt^rp=IEB9Zd zUtZODfYpRPt0KuHrfZz0Zw8Dt{~2bNi{5fWIZclW?`>QPZ>*J?j{@Bnr{L=rxOhy3 z+m0X&pd5Hi z)jFLxxR8I+7rc@!A88Kg11`)ek)+`5tI+q2aZ&7JtnXUxGJZdEIdfk3A61CE+PN~F z))yhY)75!at(d4f)J_9|1Jf{$}4iFvX*xWBx3=CX=Ka*y9DHtV>wQF{+37*I9}-OtFdp3sX&hopwO5a=#7}* z-*&{F&W1;;&JXJ5I%*4?`RfKcn`W1qNW9v#Y~}Uz5ZjA{uk}NXp1En=lxlu#D?V%+MRMI&vH5Fl^)$ zYqV0qpOf$+NZ2qNU?35DDl75`pUdwgCq!Z$cXX5#OoR5G26#Bt_LhL<51K zp@VzG?C1iPc+xwW+Wt8T^6f+msi}132`aM*xY$ATK#|bdQ00sc{hpAOU~x4u+~EpH z?YL5P(qcsN`TH-glbes{L2IZE5{!2+UyhLOZW;N;sp0pi*O3Rmw<_Dd4kir0QpJrt z(5Wev@W(WOc4ru~XpNY=>aOB`X!bp>mVFTjf~2+Csti9b)p&ra92=7q)r~lB?MtGb zs_d_twUz&@R7MMe_WHE+Ilc^qUIQ1ZU~ zO~7#nOMfLA%p>KwU3R_h&+@xjdv5N!HBq1`vG~?AqsKJ3>u1z!De!UM)ym-~)Txaa z|GL}R{N#Y^#p;v#a-SxxL6|N#2N=+?9lr)^T-<$xVUpxe3NGJgUAb zqeo<}`;brj;nCg)NWaB;w|ricmbsF&XTS!N%i!Yd!dvgI_l-qTfH4s_FgV(1K2e)$ z@w3Mcg)`t;!`A95FYee`{8ZdB-{XLY+sAy8nL?m-RME9b1G-MUNluG;7A+<`X1-2| zlC($t>ytgXd!1zUhAF+JnWdcF)nEQess%%w9THxZ7Lf<7QCP{xr-@;y)P#_@3=DdQ{s8?Jn~XWO3I`3 zj5_U#NBiX@v#;*@wT|h3%#JU2z%*y6kiif|U!ixZzOjlsjT2iULUhzi_sKyS$MYv@ z&0tNprFXmga?LnkgBdol>3b;YasJuf$jxwFZH5g%b2zUz@+CSos>K+q--!>14&=Gs z=nwSnFmlQnKYk{MWO){pu_17g@teJw1<$wWNzdJEocvVIk2euv=Gnil-Gh4C+B+rb zAD>|HM1th?1ypG{RI!{^Dn51*09Xd3T2}G6oU-zMCCFpt=cBP1k)` zvI{*euF5FrCRN;Za_YISx--?7WJ>rXVm0z?+jZliB&_1Sr{haj%7ddny;abwvf(3o znJ8jFVAP?N6zDX{d{O9okM9PlO@D*9V|&tlC;)2!!haLT_zgC%zReJFi}8kL*V73} zgKq0bPH~INU$?nT7Bso@n?GQQn-tL)T;rzn_`>)!oMaZj zmjK%JxtC+Trc|$0rdwjn1zm-?1&oRABsI9K_TEDXnT1{`r=@IdiHIff-(uiUM;@N0 zDb5`CcDX(AcoN8Wy5wdyJ+wQGUK@xqsziOsv1+N|x{db_}ZJOC2QlwQB_ zM~3NF@fd@OOt}`;`7{TfVb$&;*y!Ae$XUaMpic;Z4y7B@ z@Bz2}4^PLBf>{-T-kiY_KR6wyil}W~>r*ke#l(~~6V=HYWerdJ$}~yOfB20X%*pXh zh(}4~*&(k_e`pu3D$1V5y5naj9ep}eZB3`G%#K}X3n2C)Gq)T089T_Gc9()O&Ske7 z?1xTVoU{}pRDJ5_>H@U6BfhClF0E-ayVOnap1!xWYzB#qBa;|()F6JtjuyI=K>j)h z*5+ryJjbaqOSB}Hg-h^5rd)Q@erLN}=VoAyPB(G>S!EACA>rU#e+e`nwRg;3 z@4o*dUN;`=EC*8%+)3|y0Lt)b9MfHplKn#M7{{~SfGuijU%hw;C!J_`d?=qNVty({ zDO-ibOspcL(Ob3wfy%b)UhtI3i(G#aQ@y);YMLx;OwBhq6oHCWD@xOE4QRfQ*-m-! z_3-0ZcQcg56I!EC*ZFsc$&8rGV`7^3i#FD6z(a8vAZr!63SVV!&-&}@q&5h7P3RyF zC-2jbJEVN|H!Hfc^HM2P8f$V>R_aw~%L^@qW*K7;OkC0O&U^Ms#nL1XRT`zX*pKyE zX+?~|cALgOTHT}0K(gzD2r~}+7tKNsuw2JJ1R$vj?a9LbyLm#7xBUloj@RtkFhs`X?$bhPCT&U$21A>twq6P z@iGdj%f_c)F6TZj`>thcA!oU?rSg@D5l;8f9;+8xa8lKo+k&9aRk)TD1D{2h&H#d~ z5o*G61qkL;KKV@F&MRvp0$Y_iQM=kQ{_q89JW9n992M<(l4jNau?r=l;o}eY3C-XX z8)y;rxybADBc)jd5Uwmtl-AaTTrI9txL$@^dnyBTFOZd5LhmOdD~lDvwZ;nX{um=q z`Sn|x67P*xHoIfY=YKUenKo`$GwMF*U4r3IznMM$?y0c$J0BtEv$SL# zCbu72-{$a8|F0TmwNcg#+@CBl^}N(t3P>}APK?^gQfNy#fHcE;VmF1GN-;@!rCvrO zkk|ysRQHE`ta-0wPuEuSz_C8CrW-kG&vLg0knATSQm`r=4ldn4`F84Fu-sYRYx^ty zs%Ha8CT?ZC_S1JMa4OR=@sI<+MWv4Ls4?dcNYU+-XH-%krHFd z|6$18Y)^jS(i2iv=QW)5^a|l^85!{%Tvw7+zHnK@S(+HP%;?H}Rlzjp)8^PwZgaxb zXf!k%(qln9Hv^)@dwP{$WG2x{OZx*yr~S8wooyhYv^yLTE;NpQRG=_UE7NHUyJOZ; z#COwc*JNW4y4GM_+BS9GS`6wbO;)X^%y7s8BUYqi^}^Ps&iTDh>OyVb?|yuW`Nbe*KL} znonLnDHmA@sVHK6%@$biL}07yLPYy}bz_HDsbJ6<5{7|`2NTd%z>SH(4Ruq_0G9YD`MC%iE+AcaItcd$0y^iDqC%b4TGshck%-e{@FBL3G43GAk0W0 zF%;+zS`bJGogW9#w9dDwf!4924;vJ#J4FPVaj1BM-chkSj^F;rIwAQA~TC9;oOcF%}060TM(cPWqrpaUo-# zy&4tbdeP5*kmfZB#$j4=3fhjB{ShjExcuLC8lMBFXi{+@r336eu+)~XVLonlFYw{k zj@wjSFAu9zu^+^hE9Qk1nBEKFYF+?gb#AnuczZq#&^%hjb;yPJpnwCqxKIwm$0fxd zuu1!h4WVl09*MH)p}(AFwh@*Qj{x9Uem>MYlzeEjeEf4Hxv&aXt`;*g4jRJ zK}%M7pPKZ^S4vv0BIL&|rU3ufzC!i#KV4On%AeH-WNH2Ax28?a?VC6+oSw1W^Cgrnhn!gw>;V{9Cyh8tO>TNW!GE8O zgpPQ+nd>wsuBc~vdi2)MY4d_gbJwmPk~ha)i5t$SJz1d~vpM`d>X|*o5ie5vgG~trS$xUznzGw`>7y>YPJ901HMsCp!w$LBuSyG(aR|T{ zET=w6EQozsO12)25pKhAm5(NWHmc^_a43sCE+mCYT3`nbQGEEaxSok*1Daw8F)j@;Ff61EZsxv>>!%< z9q(&H>32aM_km?zMNL#EV_2hJH=cRKJ>qY*pVP4?7Ce`QzLM0+96hjl zu(q@EGesop8nl)L0yU0htWvj@iv^T$maau1Wpg_Jy)8Q0Mt)rm_=eZV^$)E5yV+kj z>we1fxMlDkZDU1|EXDelDp|_!7-Q~{x4_lmKc7G{bzc36z$cwuvZ4;PISK*+qJh^4 zkiFN>u5UYLd~ut4MC(dSm5tn_@=O8(^YS)re36M&ac{ zh$D*HBEHjL%UTJnr#jGp)kppR^UMEh9$3VtO^RwMEtE^V;mF(3VGPVIgeDofRPCB& zU5j#b%H#ABD2fM275+}Z2(+gsZh>D@%RJ^E`F&)m0)VZw%>IIcTAx4+!GJcl?=N7S zf8YJ{*#Gy#cT9k5ph^#(PzGLm4gPa~J8<{Q3E79%@&C-)rdG;_n1rb*&Ev%Zf?wYQ z0wp%pA#fcDfW7_0^8bxL*1FJPxJvz2ts#-Z)2|Y`NI_?G&1(x>TUX zX^~&{Vncl`LDx8Tr6-z0Hi`vA_W1pu&6xT4)eM1RNn~`x?i%wgMwXW4H{b^k9?YCu zx%P4#KjUu!h5oDjH%J1eHg=tstFtSg1(fOE9a2WxnYSCl1q%w=$oZb6JifO4vop$R zFUWhDc2*$6g;iFk=soMn$w{C3`8$36&1QerYwtKCZuEs~kW~m1mGJZe=isK^|D|!V-l@_NzSn!2^*m0Qjf3 z&+Q4m8GY#b-+JBvOk1Y0X?Q-pyD#sd5(US4B!(QELV-eU|2(v$R&pGiWicS1F|vJR z`9(Y;j`dFF=Wgt`ri*UI(mO?(t?2G#X|vBkwcLSwK`g!F>au)*y?sqkSJ_sj(77O2 zXKrRBr4uUDD79c|1%R2S$YoCg=fHyPGJZ!z%!j8Dl&1H^KhwAD(R&)@bzj^E7h3gR z0rDfTmE zZKntCa6{0$`)rLs((CdJn!jccYgL@c&K3 zJPrt@X13)B^Y0|<=!s#KE205XrLwIGFra9#?4Kg z*VzmSbEu@~x?g4^fZzByC1rO=!$t>sZ>htnyDKUB@9Q^v!&m}1S^W1t{a>og@WMGW zigKTjVBGolfp(}+dp9{g5>1dlKHgsn66}kfHui&pFAg_@5rPRVaV zn=(Gjh3YP$EdAxBxJO?-pZr;*^T$V6r?p94lWN&;%^43MbXU6?5aDszk2OAiYVUu% zy_L?g*;LxZ0U=2*@$6EN0OS*^1F$IVI}@rNwdbpvObP# zY*Zz=e-<{CD`M<^{oFd_EZBOzQhfw?C?@9}vgaKRN0BDy?MRvfQs5QI%Rj03f{@f& zC3|(`bfpupOA1iG{6%y5>OZ9Rk8S0=iv!cqLB`cwcSwfmqLmpxe=b~rO^vGE3C<*VV3v4H)8wc->lRx^ZIzw_vj*N_!cJcUDcC}f0*I*Dc{qbBlnS>#D@6pM+^?oN8rbN^MDP@1s~^wl}KnB{VpZOAaW1!*AYf8zkW zol-HO%^v9Q{)2%fYd}=K31^K}PLZr)HwrFovfx(b4mJFZW<!b4ZX}hL zteuxDvX?m*GRH=S>&tJB!OP2oCqMm9e%?y;{$RS;V>%o*0Dr-kOoBLHEMA=jI-Sjh zjh6fZ27PQxxAiokSOC28dggdPfwZ%51^lHIOT*=F>Ofn8{mhpNV9i{2GmRr1QwBUr zx`!O_V}V78(B%O$eXuL!kT5CxaaA(=rtChB?a6tr$>1NfbJPmC2M1;kJhJZYqouK_ zgEV#e|7sLb_nuhXHmM@DAlvp{r+$tG2)!PtO5y*iqc?eq1~2dCrG(y|?*>8Hn)_{6 zWdGV4!Cf%{$rT>hhHf*xcLQlexeM6r=t3VwYw&%*!T+vaV6m90~y&%t_M zJeQ{VRFPiAhqljBE$Hq?eEQ}c=BY8efmdgq?TN+z4{2W>4s{#%TakTjl4Kc@p=1f! z4HA*Po~RHiA?qaj*kxa`WZy|Cp@^}@*s_f!l`Z=cS;sQQdhXxUbKd8D&-1?5IoJ8G ztE;Q|E%*Jszu(XHlS~jXF<5LIL5W$Z&3sgW&UNwqptc&g#xEH@iy(?@qMMu*Y zYF9L{9MO%QrJYwbM#o=TPOZ8!y6YxMbH*E?n_>F}mk1vcMQ?j#aMubLDR^v@SO+PZ zy6a}n{7U|s?1=+&?&*oy1(|5czVA+46YIv^2?6~Y(5bMX0mg2to2xPOkIIUk8Vt-v zh+AUGQ@F0J#~NDT$|q}jsQEu8*v$<3wVjLz{xjB8NZmZy+#qS0CIhss7dgwN+;7bS z14DB24}SH^whmDg#h0b7_=}(cAF^wiDB}2F0%Y-wMD2Yfd$sp|m2s`dSFV#d1H*3Q zZsFpMXMx2!g1XLI;=DrcfvkMn9VTxh0L^xCecO2jyFzv;#>Ng`KOcb=If z7Q_$&KY?#fryg=FcoOjzyY2(vxS6=&?bUp|k&;r#Xs6yvkk%^5DIOZ96g*m3RjIBDbMQz_!l{uu7l1Me|v*(_p`uk|vXmCE5Zm-3x z1ZsGT#+2I)OFFF8ug)+G2n!0*)*_7l_vll+BD=$H=n5t%NYOozE18O@ZnjA zc1ny35}$!y-A@Z<#JY65*Woif`O!n3rYIiL;UOHTY4qvI%=KY|NpQx`ofYm^9Xh!a zPug7K>pKmXvX9YYwIx?uitEm&S2hD)B>9eMaF}@Yz1=#fLvFY0T>8lSSC%rSjbaJ* zq4c(4@5wDHR?G4qx!|OE|E0hGA*C$ve*qH$rGAi*gIj4rD3z&%gv5Yfb~mQ~>(@K* z&CN~FNh)n?xlMGl=E0zzWR>~wvrpiW%(v`q7_-Mn>c z<}2lVnZp=zu-dhF{|Enz7cV4*kvt46EG%t@L^4G!mz}_^4$F+^@Q@E$S-U~hm&azp z%fqWy*b%*6a|*Cj(53dcL~_b~3!|^&sYTDxy04Tnx`+}N#f8h(w^xa_fsP++2gP>3 z@NcZuWv)}n0ohi(&_YeOm|A*Ma52Vlg=z-O|4!X%$S)lVa~1(??*-G@40?h0cmN~* zHqP*6G;MT2z!|?2|>>zVM1hudrm2*QC^&toAarHd7aKImni%-F4xM9h7?f?i9SjU>+WF zM|68BupiAC0oIl!T~oVDL?C68|9ersvh5w+MdNaHWRLb-Gsd_YXs@N|rf(zzn)y^&Cl&(x>WvPON7&-8|Jj7pjNy+FB4aDUH|0F zsesHoVA9_35GW=4^-(vZwePNKqvWGB@i-czq-UYxx1G(wAREf~|2IoglN{%_%`J8` z+cvc4-6EbwgblaYnpVIOiWp0IO|Y47Oq z-^vQ5AVQ!8)lD#Qg0C-R>7RA#4l4rqL88LE!&|Wf_3u4xd>=O=q;R<>PTps-9vFfq z{EAR9Wkbqow$YL!!2**KW&tVePKaouxNQH8Jw67Jx|@d=m(xB*7l1 zPrp%u(PIJXp|A?B;Au!WKouqqz-9ctw7;@6;0t-+@3`U6zc&Z}nsXtOM&kNIzz{NB z=rNlw%#m^TGWyvRd-svc^z~nJN8Z=od2=K{{maM#8!k_|UjL-3b(C`)V=#@xZP~Y0 zV0`$VvjGjje;uWW$*=q>o~9Atn=8NtPz~8_GqcwKU7!Nf_C|d)_*EWZ*Epp{g3b4$ z7pSQgKE+U-XfB0^n+snr^yD2SJv-Q2yEY?OQ4q%aW%Cy|w~pV6p{y3CXH5 zr>m^mdh~emWgf^$sOrZjyspw`zkq}8!kWaMHFM9=GN+;^rLw3$!V3JV9Ln~#W$Xre1 zbeoQYjhEt|8-8~wP_=3fAn$M1Uu6+Dfoar8=PKKRv@Sbz^ zzIL#@a-hS&AwB-mmKi~O4@wsRUYaVD09}cY0EaL!5$b6JAp41# zzF}{fbEfa=jj`KtHK_^8P1Llx3U~7CaMHR6tq^K}R4^9^YaL|gG?8A37DdqZ>g6m7 zY}Yd-7`U@lfLq;GR>3r2wzxQ?Yk=lV(~A}KoE(1VSFz=bj4~6SElVE+G&%@+ZY^qfpMl4}#9{n{ zK(!xI$B(k+8Ct@uLc*iBXrJ))j8m=!ip{=_Uer29?}fHxvVveS2frZMP)1YK#Av#b zir3nUdkvPnrU0{Bo_PUILf0Al?3cGB80{UrzN%_`E$E@<`?$DhOl){aGe&Lnf~P7l z^pbhYV|K7O1j@?79(A(Y#2Q}KH6l=J>JoHDL#+kbZS0Ns8oU2md$do~cMa_2diN69 zV1Fy<=LtkhAlqBJlYDAS5B*z|fcA3!Y!N1>3XYe-x-==6ISVHd!;7Xo1xNOaS)5~@s%A$oSn&x56w zOXSLFNKsZ+{7CJyMPA_%IT+ctSu8iikT*YTJBmN9ST~6cjD$UEEb8bVsZB^oXx~~I zwiqtA`=+~EEgiXW|7_$&ju35iKA)hlaDwUnf}xl@{-GeEvYYAg0bb;=?;vyIEJKqu z7_$ws+?Olo>}mNyQv^h=JV5V8Qh#*5Ew1wz-b_A?WYT7~rYggeGwaS;;*}82M;vR;}h`V=HOUuii zY49{cEo@t!@laIwmn9bvO;quzw&8HFeb$!fEoaTTRk8F{{rl{A$)P9@qVu*jIP_x~ zgx=+xAGniNb0MvK85SlO&9`#3@HEVYJ|`lV*A^A@v7djS@t_fGwFfUFbDHSLI3ko; z6qB{&-0^uV)=AusK>b9Qvt9Z4>jh$OwD&oaYL^$l^f}~rYvJSIX+meMxTE03i)Cas z6n9W9%|tW&N%(TtlC+l@jxoRmAq%RDC^b8L8xPvt?mR*mi{N6ODAFW3q~i%QEB5}4 zI1LE|w}9nu(IYGC!dX=4FyjOUBYNkWaZ-rm*vosFp)zo)C%?V}uK(A!JBrKeC7;@w zjNS5B-yTyJnVhN9tVMPv6&{3t+NK3FOs8FD^}G5E?;Y!1A272tg-SIM?Lk*wv9B!& zCd|ra#wFTM>zn}E={_{}J|y^zc_;2v6l?|iYzIr~6l7LQ@YIG#y&Vc^g@xp90F#0D%TZ+yFxMWmKPE|fD zQ5WfZ43rC!rXg|BvTeFMszainEW82}&zig+GuH$~<~rC~qSUE%yvrMBIU*cg7YBF* z4U5X$R|IQEN|{kSKSt(&T=u@OaJ&gh_Z@Dd!#zd$LOEj7hmzsI6uv_*ZpMAu3Ktz6%Vt+R># zb|drox2;D5ps!Z^E8fkKjM-8ri<*IHN=!Cj>SI40$_b2<-B z9D5&%(L3}4$diE%Pv5F13CEmiLQJx_XgdXgW4poqd_y<0^!6UWS3e%|8)NyoG!rhq zAA2u*Zd(d<+L=iausEfJF`u!WDa(b;f|hB4?AEzf#`a2R>V=>k+4&AiwdxQ!UU6|3s9COLWiRhq1*a>CyRD9=9*bmD zJdFZN1{C=4-TYDU2`$C>#$fS zF}PMhGqJFO`F)7Rv|BAOT&d9s=^o7pch^;DYX&5fWIJj9OWsqIhVTOq4!eV{ALa5k zzPH2~ycvsPCkW#Nt)b(G<-E6@7Cn}Bv#)4r=AQ*k)x2oM_b5F-3L{xt2mBj3>tE}Z z{>SLw3N{wazy5Y{c$u2@{#`K^>-f{k91XW|-MZya4xwGO)vCzMT!lrCjHgC2GI`k%3(sy3?nNbtoWbnxRyP5rmC_5as&Y@dvCqEXq z^{Aj_>_S-1Rli#jg$aN?Dp@DGa>)*H;m*<8qtwQFddn+p+ps%Q07cW_))VEaC-Kaz zc)yCYHlI<3N4g$xCRX$%w^wxAA+!Ws6BF`uCm8U!Q!t^eQ2;C)t@W%hQEpzTv8~Q)H#RdQ{o#{i^qU-@px0kbEEc0{I~q9n>$ua^qPgsz(P!%6M&2OccIWiIw|VXWwET#o78L-$HlMXwHYu8R!+ei4oqE2f zRUs5bLwx>rPJU<#% zLZp?WjPG#DhQ`ygFeA4y)>7|B4CQiQv{IuxD==P$Xcm#5frEPle@@DCMy(w+ z=fwfSJVZ19nD%K7$0)0{A1mHcL=7-~e}1XdxUzgP?YpUug1hQ^iRz67ui%D#IlrI} z&fN-xfln%QVPx2IiV4Cd3*ZykiO`$_am@@%8kWpfR&zftsF)&U;{ZXwEYocWszAKZ z*s;j}N|FENr`8rmS>eM8hA@J|j;Q;sV~B7pBIb#?vR{79@{^yJIe%aop5Y#PidzOF z4=vB%6_a3#h0dY~_Xpaql>+F!AKk&fA-5akCy%@(t{g-0lRu|0e`4eexmWq;hsUD= zR<|SS_!n26n45cDQ5#N`8Fu~jxzfAM_axtM%KXpl*!@vyN>I4LuUdqe>h}z(nFdV# zqGxaK?ae9bddC6??Fsnc{Cr>Ys3sWSVhts9@7>F&O<++LYS^bdp0d00Y3zXWz8(it zK8-{&m~@|pXj%d${G$k%cPQb3Ka$DMptr+l2}Q@cmXh?VpHyHkBtxM`poaUO?P@P2 zQ{e}<-5`X^;OalIJVa zUH`e>BWE==-=3ep#u0L9LmZ9k9{*--_jN39i~2^Ll(Cz2o6C>X+s=)k2=M>=zzp3i zmd;-_X@3HKkbTh@BkFFkrx&Fm$loKsoyFG8{%NY^W1UAi*n3-d%k6By=_Sx3&8&Vr z#gW9U|F?7hhy0iCaH9NuqIXV8hQf|4dL)l+>Cw?1n+_9H5>lYG5nGs*h6`i8P z#KLl#)HIFK&2{pue<67~U5wMbTOXf6(pGON5&QuOo^!VtltV&Mt6vJhgsztWb7jV)KUUb2X zG81%Np6Q;OY4>-yET{9rj~{y|ik zLM2{i!|Y0Y0YR!aOThf!>HS0uaF4O^_mDSGqO7`)MOHfu#KA_oAlL0QT>*5!Q7mP9DzFXms&~EagP;7_XWzdZ8UQg7-Kbd zxr{#!&K{%B(M%GU!o2|R$6DpjlB@EoW`rwY0#_(T0I$qCy>eUnko?!oxa>PmpEO7P6bxvq$f4<^et=KT+J+(XFB6#d{=QMUo=3V z@zkTse3r4$xw;*^Zsnm+#{E}3_0abRrc`#_=rd*I9%2G5mVM>Cqbg2`&U3kH zCMm}QKrkMur#}QFtU!%J5oABU+(-!y(Q3cfjEaF8vJ;Kp>(l=sUOADd1R`RDx49V# zjzz-Mn5?W|zMp2qucmMpWI8GMv|3V0Dao`%!>n@b6=$&s=sB#VU+C#fmy>c=b@`8C z1VLiZ;{Z)FB==gW{6Dof@4pOPPcf_jEBoWK_ZJZAPLx;Vune9Ib=E%;V5p>Avu#a} zBJkpu{J$CWZKQ2!S$WQOFd1c4mOe;3ULm6r!C0Ly<1}o2cI4G=1!_neU;VRE?0WTY zn8z=h9iS{nK^n6?_d+A1d8`h=UmW?-02Crq1n(#q)5O<i@) zeZz2qJBP`ei9pa{%Hf!5rIxU@YaxUP2>Jbl(Y@ls_$+^?mLQBcMeh$|v2ORy@My7E zi2*G3x9kPN0#}v);{XDWd}8tn`}|v-8xSU0vkI{ZlXTy{&GqEnT{S)fD9D!H5lO2C zsdq+07rH9WgO77uBk8hmW|hcb1UP=DO)iVntXA^AS;dVpl^e;Z`1B?0AI%rwJoyP-g;^s`i=aI_uMXoBW>-@Iu7R__j;0{vv7fH z{2){i1t62?{c6vZS}CI4mfk&}QKJLDNWx#Ez^iSb{9B9me=PB}meF6YrZZSf9AK6Q zna3yM)ayDq`#84J#R-y^%eK^`#rz8Ygzd-L*2CKFx7J1u?LV}FY8kNO{^j>DBqAMQ;R_Y@mk3;@M{ zRAUprpq*a^9bobu@ow3Alm-tdiJ<|4HO*sPZE^fd1{MW#?PcUx8YIyui}a$IES}M# z#?JtmGM16h(ES(2XHwUAXKTsku;`;uf1e-z%`xpqP)#K1yGS*LYFJ^8*zUQU{s9CX zNf;Yj@p^sLfPmgG*KY}2WrB|nhVed$hab+NPqZb z8s>u(n^6Fb*c$dfJ=9ioCzw(MmRlf&Huv5qSlJJE%ti=_5L3sA;!F3##Z|3a{}kn? zWUg?euOsys{Z+4R9<4rVcpb&^KSWrGS_VDbVVD>E%Vs|O@p->jcmTJG^l5Cg+|7Kx zk@ANPOZ@_^+tf4?_Nk@9CRGE6!OEV~xYCBZ;k4gjiBNk{gbTIt6we|h!jN_h^y8#u z_aBr+0VwjUtV8MmLWMMKvEulsT4q_l=`SNjJwntX4Q*9V`KL;x4aS?yr=p)~j{*zK z_1#xOJLHxjh{BS*E|>dZWDV)JBtwMEeb}3C%S>Tk!XJmfkQm|@;5`0+zr^DhK*17v zmv!_iGk;CV*;g_nmkX{i%$!zK_M;f^kN7pkbOYlvR9yV&U?_+PKvVutp_J`!|8AWA zQLDgDHq^dVa*_kuz?dYj$;KXybEYkUx0wQy##1}@xK3Io#nT>m{pIx5a%e}?3L(d$ zsxM1ZxeR+^M|Hr^lvQGbPT8WqehoVJAeb9$74Co3oHACwmWHkSA#igEe%ujoeFab- zBxo`zEf}tc-$2a`=S||m ztnFzf+An$EIDTfJ+I2-c29f$g0&-AHc0!Vj*8Dpo|5^@!)RBJKcK@iDVkV8s;S{8x+NZ4>$EszRc}8<#k0^NT8>ztSevS%l~|h zBZJ~pA`JZ4JbM(7{2_N`d|-hAdor|?&@u-3AuH11L5_E2#mXBhtez=D#V$|A#~4iJG>G>d$jGu9apFLrCGz&h?2_~Ml$boy%S*o1vp-9G7$Tk) zd}^=S7^qhN+pPdCM$+&7&n@G0{2)J)E?s?rs^uIz4AnE*lw9+`VPN+Lcm(QCr;$hX zgKmix%%vX9oG~@|ju5mIU4q3&1*VofsGkZ4X7e^(pPifTKpVOkSS!MmiUvZ@II`{W zSGq}hKvIv{_TAtCda{%}{yiWa1s@0!7T!wW2MV${GOZy|qL$41aa6XjnDTf>bm~v; zS~QV6eQR-$7i^4jz#ebhzw!*Toiu(7puAePyB8W6_O>(f_|ci}SG*xtt>m@5t zzI|A_2{E9J+0Ev08k`O7zl4XSML{s00t`hZZHGcca#6e!xPC2|V zDUi)ujLAZ;**CE5F2S9ng*!bnRnjh&lFLo$p2s|&%+sGeb_C{NR5xHwIP9@e4OVG@ zdr2C>QYw-VxL|4)#4|4l7-omlsRPT`gt&(0fw(!Vt2~KyF$6vp>Z1sXk<@Au&i>xa zc;t;Fgs~;304Pq<{zeaE2}Y<%pPny!(8?fZ4d4La+J#aGYLnD?YeN!uw*g8^s+h#%_hSJU(gN5RwMqnJ5nHZcF?G8Lw75+x$%6B4+>j_kL%5hId5w z_}a$Vi+#U4}4fbmAU*~D~2;Y;h6AfkulSuZL%!*2I} zcT?h(9n2gx9K~}Hs{oMNE}-6@pP9S@pBMZa8`lHHva=E&kR7M7^n=MH>t!kV$mL(% z(KP#`ujcQNh2=cGD*HIIXB67l3XOs8JzFbd4wMEp^HFZGJ6@74M$WcQ;?4_y5r2I; zuyNn$y|&cE87Dxw!;i~wpUe>0wWpG6XJ_i zT9GyU=+W1Fhu=ZSychnr@+Wq#ZqonK(FPZj?W{E7s$V5WujO@W-Q))#5@KgXLwZl1 z&d!v}nvZa4*1zRHwM_m1IRl6|uvq4?tgX(F@fNSC7^4$J$~iV(@Q+AYglp_Pqh-D8 zWB$F_e0!+GJRnlj+uQr+WuWf-WdB3fh~P!6w_>&CZQCsYQrrY|QqU1r>% zOvx7_53ERp#7|&;ECxkM1ktwcuJr z3xc;lQvl6>*==_hv!8i4JTK#z)q{;`CBkVw>3A4`QttM&44GY<3eF49;G^n2r+;GF zj!M1OsYTp$GMDk2)jizMUCQG;roHeHbsA-^x5O)q8>vqR7H507uzaRBRrm%^dJ@O# zOn@UBChkycbo+Chano_yb>@o8tce(UxRw$*f6$~PRxJ&6OdH%^QaG*iGayiF`$m;0 zZ3Y-tT=WWtsg5MC-80LP6zCx@4*%kxqXzi~=YdB*4c!8;a+T3QNOA7z3a*5Jx~-Sx zJ{@zdP#11Ji+xH28-t11~&jo;S;OO3ZRyfQ+*P?|Ff+tqsFAhjoZ@C+WH@6dw0Qc)KA{iz^e*JZ_q z_?v7%Z@k1=ewJ#e#(n)^d;ED9xwBi0xnLt1e!6cwwfpHyQZCHFy42U;> z#R1QEcZ)x&;Z-ymQgZ$%vKcC~zPCQ=T4&Z|(GtNXDcqk7n5vKSUtVc~)RdP%flpdQ zr{|xE^o9R#sXyZ$NzW;Xv@%7%5CKZtJfO@9)ELP0-YKBtlzV%~b?q|=lbd}&qH-LUX* z2d*0nfV2W1WXf6Vd2gkvsDf3$b47B4a zUO$c=ASbcPm^{|tYt^JU0%K{zBIEa#jU|b4)~EJ_g`H3nJx-I@qn{w-F2F#Vs6o|j zGM~O|%_1*=w9iHNdNU_tyq<%{?=33RQOSw|)JUVDzJ83h_)sduI^t6UiVGn5Y#y`S zId@2eIg1|$S<2gGIFB6bC}%Ojsi*oZ@-r5hP3w_&XC z&1b7tb8E_qY$TH^YPOS2&RQp>_fd<9>1t!`+@wr&iCl+M#)>8Ha(bzeq{FD~ue{=o zv%+N}!nNq_Y~d;pS_}?Ija0j4_v|?E9@yqzgKFFhQWLnKVIf5HCn~Oi&Sf%4S(Xxh zIL)sZlW6Ka$i=ll2COaPh#sgPN&Z<6iC&LjRiz)~d`bMH7KDQ$6_6`qo7l!fQIF&v zF*6pRMFlV-jwg*gjnhTeGXgizv$kV!@1y|`n*EBgFSOC+63E7$Z_cAf@!W$Q-O?M!*ph=BeuMKhXfY}cSk+UPut#|=bsNeM; zgyVMKK{WL{rfC>35>movPQNBhD|nTNz?IUk(q%;uADe>bHt}na3YV$q@a&HF&g1Yl z${n}NZg-OV!6V}cz#LT53BqHqABF+^@ToCyt&f7nPxb;`o;zmHzIfaIMf`Epnh=msAtZsnvc!p}=lMM#W9kc+p1zT_ z_O*OGU@IK6y{ep4dBTd@@EE;wpA@7~$b`I)MjcozcADQiD-9wv>lSF7fsv73q>Tw8 z!&p&hT8Vg}a!IGR0%#@V^ny(HtJe^g2mSADfD9w(7_HoE0x~6GVW5X!IOnW?{LIU) zo`CF9aRsWo`ti8HE$Tti^r319(IIqn=}J>1+*ADZ@@Fzv#Npu9!M&+IM zsS9t>N6mLnSy|}&f@(rhrQBt>(?eDucZuqIqZ~SEclAK(39?S}sS9b$D zNtG`yRH?D)cz_p!-z@hkFfR>;(oy&ph=$w%T1cwI)gMazzxBAxb6^2cC5lr~4HF$< zSUPVRC675=dimmNi%`IX?cQ#f=JWA-2rl4R3UIITIrNr8>Bw7|8WU$^x87I^o^<6T zJl7^7ppiXzH(E1mf~x?wyks5mL>Z;5Bg^`=}8c=|x~ z_*(&D28(l(c&hXXh=<|P8Y%)j2bPzypmqAZg|A06;{vTaS=prx+J-s#9benMYD|yX z4tQtU?T+PMVL;m10-n}_VEvp{HfEma7&S3zR)RBo751Ivpy;ylp7W z3j!P{^ntFC^0yEY^yEPMJ((70=2-2rbk}_wM_X&dAp1S+Ts=LZ!^B6oXHcc6He4;xF&)-mCR!+b4{P zKn=(R#ofW~JcqG5mm5;4RaYVTt`6Y-;_wc2#Qb+11GR)vL7#&;A7QAOUL36*4vOY! z_uBkwD7jJ@Rp$*vQ{}ivG{42YAov)(-@&Vn>@rXC_8`wr7h!j~T6ffoU?TVpQ*z!0 z$dKZJU__au$J^S3^i{SM$j*FD=iip!&|FVk2h@lb4j6sle%hxYG*Zqo$9Gc)Y_)aZ zf%bs5TCav-mbtHl-H-%vI!^Z?e!9)&SWtlZr#-wTr0wGL)LRD-+`RHXn5AfG1kbi7 z9#|m@TuvOA9?Y0NTpp<`uhDDy%#LuVJJ_+2GjBb&yf)Qp@v|q}#kc!ovW`wOXQHS- znJbIc2~oec)Us~*^@xq-AH#J#UF{*4ZN3d`nT=pjTZh|IOAB)uo;o`l^AGs9x8>@5 ze!Z)BNW{-b9*X%vI2Ho|4jxPORj&hJqgu-Qm7(CdINX3081o)ANTdu_x_&&*hG=wfk@eH{R+vs4#Cbuas02bu$>?NMmnehiUt#gI}M9SR{(H zC+_&VqzR9HiN7)0(wJL33a_Zb;|>d#ck6W!!m)#c&5P49-wjb}>(|$9A-4uuio# zPixf!G2|9*-fPzO*_OWN7_leU;!At$8 zj=9eOWSVhs$GlakD_k$y5TCHvJT-G6!X&x`3kvkiz5 zuiSGgr!yD-X`$5pkn^qW01fYF(CIIQT^z-9{(15`GV2p$t6%iVQ$)d+LX5-__n8(2 zXhWpa?$l2kuXb54`-x$%vzM#2lmCuCIFu$&;T{om53O&1sBl(@h?~Z#?6*zAsAo5lQjhVxWk|8SAa`HYhn95; zpVo51{0zkdy8dmp zKxJ0gI~YO>V~3B<{Fh^$`yxKIqCmzl=uq6c@tdSpr)W&$@VYY7?sim#op^(xu|GNG z@`WA=#yH;^bL|6NFbnHs@earz?yXB^5=R>pxN+~R9<5C_x9NTf_;YgW_yvh3m8m9K zvWu5X=P^1Cv)BXE{OmpeChhi*9t5dOCih>kIB-d@{z(hRB%OsX ztJCevtruhvGiw3d9dm^k27jf-WWK^T?f-nzdkRLq;OS14f65;N$$#&IN-bXhlAl@= zqnIK)$lOT_whG8-9eiwx#DhWNT6IA)ji$}mSy7BPXu!O|7d^QCse=e{I^kae=sk*Bx@q_ISRtHr;#sOVFWb-; zRh-xV`ullxrR(u0Wx4;%4lW$Wx)|EuNR_e~^WL?SGw-NmVOaK)DqwaJ`!O2zdnIPp zglT3SU4BvX@L1A1RZ^Bp$V*(Qhrfmb9*JpY8@mJ|u&CUUm879|exuzWE4$Xrf3uB0 zGahQ-ymyDkhJ+Tq*TPLdLk(fHJU+yPT7!81>q$XgclX8@mlp(<#l(r5`X?y^l(4Z<$+UU??P1o+0Yd z35o!rF4wiZaj7OEUe()T$h1w|Gd?@?RuB)M$BLY&Ijlfj3sAjJ!nI4CKS0m!tkVzy3m z+~^@Pyhpw7QG>MHB}?@49r;#quJ4@kNrT~0U|5dyDcI})s13l>XJe%`Hn9I1Za35 z9NkO5QA^EV`E4!)-E`yCr84U;(_Kgm^qqtr8 zoM8l*`+7q)v&=ih5R=xW>EkqoVP+PW-@Y!0E{wXLlku*|c>kP`j8ew0k)S`ZlRWu zVs2Fl>ZU|O!T=qRPcZEYjC1xwhr7Fnro_Yj?L*$;r~}39B}+>Q7Rz{?4rr&zyl|0b zijT|Rn@v8`U@Mna%F{^CviSX7uYYLBcySt3Sel|NbEPis^d~I`(}g8wZ^gF?`0Wed zjK4D)EdvP~C~6W`5$Efe^l+x$>Z%cdf5?~G2!J&U!5%%9%is#D~+8UJ6IZ9Y?AlCI6SV$ z@Kfcsd7O316G(>`yb(4CG}ZfA-9Pi~mOp-8dG* zRW?|}_)2R{zI&a0#vE7`x9(=F3owSr$isWSa}Rx5#nOHN{&G)k1m-tr(62~HOQZ(^ zlY5n^S5wAMNsbY5u8V=^G%ta1W?3ys3y5-X8S{isD#JavM-512eOWa_tN{@0Rdu*Ur!iR$&uE9He z_KOd{+DA6h(BXw|{$1_0k1fYZPY;)-nr;}HY#9wFSk1KBKX%7v=IY?dF0wVPzpFV5e(Qm(R#MxTU;WJ)H^nbVg z<85v^-*dz%r^Zb=1Al^H6Hr}>?qqxL-bbS~?&V#Osp*^D>wkF{e|-jRG?W4P zl=q&+$BGqCuR{PAT{_=ZZW-3cY5Z6B<05N^5vnR@aoMlI<3sR_90BghZTm+uB(Fd9^*-oW35oI19R*Z1ZO{Bei;OuLZxYw@}T-g(t+FkW=| zDRt!4>cXKf-)*0;^2gXkqQ7~zv&^v*Pg=eHRe8z^{gUkp+p)-QjNY}|s$J`O#V)a) zt{`9{BNB)}QDJAyfy@?UVbwiH%Cq+t0X@fNdU>`j?Oo?Q!aQ905O2xii}T) zL?axS8Y7RI#{qT#(s)Qxl<|X*SP(H|NeW2}FW$jAaox}N%%G}9u#=Ii8L0NnR)6g` z2S%}Jl_FqLoKYY|!3j5F0IQh~3{M$6x6e1fQC~QRdSS3$|Zxy?jvLXR>AZsJN@-Lr?5`g1wPn2=>&iLbni`8CvtI_2kZf&e5TN%z=pir`{iy=smkj`7axp zldteHOl4HMQW$6rtf5mf_Z_5Nrhr2~OZ&%Vc5GozjMX>zB4c$|std#fw zjU52+H{WA@Z{|kD0EBCh-_&+~io6~;(Js%0L-7|#&01l0r-@6#e1DWQ2tAEg*($+9 zwnt%W76<;9N@Q&3fivv(eZWq5W65p(#JI)KpQU*0l*{1wl3;{?enxAcf5L?nM zoiW$9F9Q~{ z?dK;O<{*7eV7Q1Wsv)WW;R$+pym-4alyU~7X0Zeiu)~V@1r%tUNVKSXyZBj4opF9r zi20-zh#PTh=48e4LYcDL3X4_J5l_NF6p}ZAnHBPqhDW1;4gAy`jJt>UIvls!%%)Cy zai~KM7oGHtnMO9dyzZ(i9`83fmTp7R2H0HQmwG?4C7B0+AGT@WW-4dT(>h?Ksx>Mu!9*90b5@Fex^UA0Kv2CejC}_p~CPpYL00mpz-`Ujk z9uf=DH-_K(>wZty<~!{HK~BbYI(PO*=5H-;%omb2q}v0osmSZzeL>={9k+-u`3F7SMK7o<^jD74r>7Zu9HgdGL^joN+tD+pQn36WD5zx1tUROtEG>Bqz(_PQHjLfS7qiFjnN^G@w$Yhbo za>ftH9=s|J6l{Sj>oP#00lX~ps-#`_#bD>U2uL%%GBzV^SnqIslkGzA38)*B%;?; zcPShOydFLn5%)e_c**y{_$gQ8G)4;4_Bj7xGziSwn4;*14mxo4ke-D)GgKCzDV08` zhP~#&%-Sx3%BE`q8)5oj|@)9)zDE)gxA4 z|C(0V&m}2`B1O+X2@X;Y43Qp{m~D?o&A^}UbsYjZ$^?fx@;HXSUA(X-JEeh1z?oUKC7q1+ z)I85ufngCX%+RpdUY(Fym&Lq6PzA&$)DfuDY36X_iVItwPp{YuN;SQFEqZ6A?~37b zXY}XB*(SD&?F-4GvYMG$0_iLid~|L}bhFR3;3Z$XbiMH~cj8PUpQ557(RxqRcWUYS zBz9V9y~Ek^Xm z=y|t!ntG4q(ag_ufB-BI^tt;;=`W)V1(Fp68Ju3c2meobX}@~z#(0X4`J^#z|} zNDwl5R3VQ~zUMmtvR0LTfBy|NytUo*E)9QtUczI;N={>iy)WFQ)6gV)G>!o3UjrlT zXI?sJH6BX(lc9j#AYN!ziAme*W3v*xd*s>wD(uR`po#vHQc&>Ty4 zMPnJVWGG|FI*zd%^%ah>G-T;CLo*snL#4u@#E`_;LZZo1=#)L3Mxsz!zUS4cM%VRy zXRi6 z=iZp#B?f6B%Ybl)_FLy)XN4AQC(7G*Z)=5Z{WN=?cU8^)&#A!O%=+ z%;FbZ>@uhHLu&0?x$S39OSitb5El?opzN1hmj#Rg)JsQ0tmt*4zyn}n!_NR@W*Y|}M^3(zJ1D(ZaD#A*PS~PuAmAAc zdu7)W!v>ga!7ji~85|;8Db-C$co34_eyKq3j6z zWr&DP{W~yM7Ho*;T0ZKGFhzk^=nF#@h}}sBTsLaE0Fh%&I01jF(yJKTJcD9tXi~zT z3rW8z#J3LIkix_#=hqK<&nd}*rH2Q0$zTUJlc)%?UV5$by`>j3OvXUY$0|N7kR3EF z$*imIQcyI4NS;zEvFWqN)ZX9C>Z`D8z-c$3R14*(j9ON~7^mO2ByU6|H$pNIIfn^} zvOUp2G_XR3Y>+*dorW-1kC$wZ5;3!BE7Bz;*G=TGrvsq3hs!_7{{IFhgdij%V8aWF zuUq-b`d=JS9iDLNQS5ziG|UbB)7fMlYET`IssFgu9|EYpjPmGynyY5~Jmy+XcGWUm z=gD6s5z>ZnsNB+q@Raib_WnYy2k^Bwt!7J52xS8WK=DMZ$Q`@oQ5h(I5&ATu05QH1 zV?!>sY*Dg5$HFLX-LOp#*5Y!3J!&~CW9<<4Cd%yZ{-{*$JHF%;ei|ys1@na) z9O@esiYcaodkJ3xkWe;8aU5`i*cc=eSsavl)4c&b6Px7=0Q7s|cz#exWuU0!{$V2v z&$p5uh-P!=+Mf8DI|J<>GS(cQthyB3{+~C`|M(yZMSwx0P>v568$Ti!0s?;Vg{}wz z)FBb&2oVcLc zaTVFjD8`Qg5F&8L{_}@}G)Niy@~+JDZ0wUc73aFZ`*gX!JzN6-J!y~(=EhIdizwQO zHA3uEkGtLWo?k!3fgpFGS0=@0+m_-EOf_rO%#@|8_OSoHA(iVP2WTyp$<8{s`#FsP zmm~Y~zjg$Aw1JKp851-uE~|D)>q|zlLz` zR?xhgvlME1*y{KfbatPngwY#^IcfkV9_1`xdW_`9f$tID>vI#UYS#;27N`406Rdt2 zfAVBY_D`m`{nc$jCs?8W$w{SR0E_cY^Ej?K)UGg_*Ta&NE>!J}+)DPg@tEp{HThcF z$}kJKOp~<`eDpTT=3Nlrg!yY$YNDPzaoCk|Q+`@RM{X)GFqVG;d;xh_D%Z|Scf*iS zk}XBgU$wVVbNI2Q=_ALqy7faDyPTL9&Q&@fHtBbNza@H7WV=@|OlY%;VMWfVc@pK? z(BV>LC@pSB(-i$%Y?AEkeV%P{iZG{k5cl;j<+uv z*>!&%m}tc@j1)03nBi4D)kk-R0TpKCp{;{s&4h@)CnIzCbSmXo>>AbP0_ce_k`u$_OllS#SWf*c4V_RjxG=A_uum)bJR@?MK_!~&URT7^RleoZfd zBi=2i1r)2;^ZHzElwZ`@oAZr&ej1s0cWY`HW-M$JK3X7wrY3d=$$P$@5FQkpleo~5 zm4%*6M0UyYnSjh^$c*Jh_DpZqs~F6w_{b=`I&H`%cIFJZ>~bQlqcfw`G%(UX*`X6C z0`$;ageIdbyYV3XyAjD)wi`j?5#IT*v|9`H>sVf-Yq&iCt^%?zUNnQElTol=dO*)t zM8J+oP}5aDaaYAc6XwS4>zglbRYf6=@>}E}NOzLCxR#pkkWfL)N!8`aHlKqyR}Be12z~zk^%Ed8d2woH+Kw{2q!fBUP2=q_R*R<4zzD=C&w1G0_PPX0?}s z$4{+aWkAH0xrN_#DQ-Mi5*Vl0b7oIT_Q){u*%W|3_!@&0%Ud6J-39xZXncM0p<2aa zEk>rFAV=Jr92%G{Pua}~B}|INW0R148l)}6Lk83CHK+Tq`bDl%|MN`;iQV8#R^FlA zS6HHeLx|uw^8Q&AA^A~Z*Aae?-eZM!uWAAWS5mpwBgEwp*FG=o%d2jM=$akk zxOETv0PVwf>*Rsl0GSC5#SCX%IkjMs1KpB1+LDOEoBUFAQxG!okf%D}FZ))W`w3cE zXSTb+rQ5D<7Hd2O_d{hPEUhJAGR<}{q(xS_{-(aA6F*dX4R$HG8OKqXV6Ey}D-{Z2 z!fsfVfvM7nNJg6qEBAD~SD|#Ljeqkw#RRE5h~MJ7?71M80!Sq{HRoVl`!POqgT$0e z)cZF3V5Z4g{(E6a4mWFcc|f-*W0_(Rwp%~M@fW;#`SFi$2THd+*<>4N&+gL}Nu@b$ zi)u*8rm&V)t*rXcqnLVD536{q{Y82EBs*zz&co27SI`5k-(>H^>(ze zsZ{G?zi$^Zwnq4eYS@)(dZ)6~8+n&GsHTWOyM~ci^0i;zC`R8oQ(tsrC?l;hTKMf$ zxH6Vr)2BTw?VXk_#HSM(Wjv&mi=LyX>7mmB(myu~&ioXD$dHbp!R#VQYDe~~RKHJp z6Woc=>S7oew5CL;wtk;*F!_5|BoI#+8eESnx}Bm&XtlV5KR(jDwOX|;McqixlqyGjsOXic*@l?;EIff_FIDd~rACJRm`O5TH; zm@1*~4KFs{Y4M=D^|+ppbz`|ZP)zr;>YnTQ@f)BG9)Om?0f=V{!kUT<%`6^`-*%s> z=tXF&Hm?$zhlC8T=5H@GHPkCcbM)v2H7gG4W8Yw=Qo|YhaalF;U1Jz~p;nj4XWf7} zArX2W&})V}w%m6!6)+KTSfjEhS%6wAj}%NZ$loU<$3LKp;A|Nnj7<(ks*QMIlT5gz ztR#{HquhGnXerBWhig*2RO?rX6|Dx@HS3t5QXBE|q{{@xRQL~${BUOZSaa^eF^utI zQw%B2F2EXrCzB62MB+yHyR*b`AY+xeD4%c_aDz$QgDaV?L8D Date: Mon, 11 Jan 2021 07:26:22 +0000 Subject: [PATCH 332/337] simple bike repositioning article: formula updated --- notebooks/articles/simple_bike_repositioning.ipynb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/notebooks/articles/simple_bike_repositioning.ipynb b/notebooks/articles/simple_bike_repositioning.ipynb index 3155e6265..ef7d27a9d 100644 --- a/notebooks/articles/simple_bike_repositioning.ipynb +++ b/notebooks/articles/simple_bike_repositioning.ipynb @@ -37,7 +37,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "To simplify this problem, we consider a situation where are only 4 bike stations. Let's denote the station set as $\\{s_i|i=0,1,2,3\\}$." + "To simplify this problem, we consider a situation where are only 4 bike stations. Let's denote the station set as $\\{s_i \\mid i=0,1,2,3\\}$." ] }, { From 6e761797d2a31414ddfaef3b3f039d81d54e69ea Mon Sep 17 00:00:00 2001 From: ysqyang Date: Tue, 12 Jan 2021 07:30:13 +0000 Subject: [PATCH 333/337] checked out docs/source from v0.2 --- .../dashboard_visualization.rst | 20 +++++++++---------- docs/source/key_components/rl_toolkit.rst | 2 +- docs/source/scenarios/citi_bike.rst | 16 +++++++-------- .../container_inventory_management.rst | 18 ++++++++--------- 4 files changed, 28 insertions(+), 28 deletions(-) diff --git a/docs/source/key_components/dashboard_visualization.rst b/docs/source/key_components/dashboard_visualization.rst index 7906b7de7..98099d33d 100644 --- a/docs/source/key_components/dashboard_visualization.rst +++ b/docs/source/key_components/dashboard_visualization.rst @@ -62,7 +62,7 @@ path of a local file folder, data would be dumped to this folder. Data would be dumped automatically when the Env object is initialized. For more details about Environment, please refer to -`Environment `_. +`Environment <../simulation_toolkit.html#Environment>`_. Launch Visualization Tool ~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -95,7 +95,7 @@ Folder Structure epoch_# # folders to restore data of each epoch. {resource_holder}.csv # attributes of current epoch. manifest.yml # basic info like scenario name, number of epoches. - index_name_mapping file # relationship between an index and its name of resource holders. + index\_name\_mapping file # relationship between an index and its name of resource holders. {resource_holder}_summary.csv # cross-epoch summary information. @@ -144,8 +144,8 @@ To view the details of a resource holder or a tick, user could select the specific index of epoch/snapshot/resource holder by sliding the slider on the left side of page. -.. image:: ../images/visualization/dashboard/epoch_resource_holder_index_selection.gif - :alt: epoch_resource_holder_index_selection +.. figure:: ..\images\visualization\dashboard\epoch_resource_holder_index_selection.gif + :alt: epoch\_resource\_holder\_index\_selection Snapshot/Resource Holder Sampling Ratio Selection ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -154,8 +154,8 @@ To view trends in the data, or to weed out excess information, user could select the sampling ratio of snapshot/resource holder by sliding to change the number of data to be displayed. -.. image:: ../images/visualization/dashboard/snapshot_sampling_ratio_selection.gif - :alt: snapshot_sampling_ratio_selection +.. figure:: ..\images\visualization\dashboard\snapshot_sampling_ratio_selection.gif + :alt: snapshot\_sampling\_ratio\_selection Formula Calculation ^^^^^^^^^^^^^^^^^^^ @@ -164,8 +164,8 @@ User could generate their own attributes by using pre-defined formulas. The results of the formula calculation could be reused as the input parameter of formula. -.. image:: ../images/visualization/dashboard/formula_calculation.gif - :alt: formula_calculation +.. figure:: ..\images\visualization\dashboard\formula_calculation.gif + :alt: formula\_calculation Inter-epoch view ~~~~~~~~~~~~~~~~ @@ -187,8 +187,8 @@ To view trends in the data, or to weed out excess information, user could select the sampling ratio of epoch by sliding to change the number of data to be displayed. -.. image:: ../images/visualization/dashboard/epoch_sampling_ratio.gif - :alt: epoch_sampling_ratio +.. figure:: ..\images\visualization\dashboard\epoch_sampling_ratio.gif + :alt: epoch\_sampling\_ratio Formula Calculation ^^^^^^^^^^^^^^^^^^^ diff --git a/docs/source/key_components/rl_toolkit.rst b/docs/source/key_components/rl_toolkit.rst index daf5caa59..15152d423 100644 --- a/docs/source/key_components/rl_toolkit.rst +++ b/docs/source/key_components/rl_toolkit.rst @@ -208,4 +208,4 @@ As an example, the exploration for DQN may be carried out with the aid of an ``E explorer = EpsilonGreedyExplorer(num_actions=10) greedy_action = learning_model(state, is_training=False).argmax(dim=1).data - exploration_action = explorer(greedy_action) \ No newline at end of file + exploration_action = explorer(greedy_action) diff --git a/docs/source/scenarios/citi_bike.rst b/docs/source/scenarios/citi_bike.rst index bec912069..b77300072 100644 --- a/docs/source/scenarios/citi_bike.rst +++ b/docs/source/scenarios/citi_bike.rst @@ -783,8 +783,8 @@ freely adjust the sampling rate. For example, if there are 100 snapshots and user selected 0.3 as sampling ratio, 30 snapshots data would be selected to render the chart. -.. image:: ../images/visualization/dashboard/citi_bike_intra_epoch_by_station.gif - :alt: citi_bike_intra_epoch_by_station +.. figure:: ..\images\visualization\dashboard\citi_bike_intra_epoch_by_station.gif + :alt: citi\_bike\_intra\_epoch\_by\_station To be specific, the line chart could be customized with operations in the following example. @@ -796,16 +796,16 @@ will be provided with the option to quickly select a set of data. e.g. In this scenario, item "Requirement Info" refers to [trip\_requirement, shortage, fulfillment]. -.. image:: ../images/visualization/dashboard/citi_bike_intra_epoch_by_station_2.gif - :alt: citi_bike_intra_epoch_by_station_2 +.. figure:: ..\images\visualization\dashboard\citi_bike_intra_epoch_by_station_2.gif + :alt: citi\_bike\_intra\_epoch\_by\_station\_2 Moreover, to improve the flexibility of visualizing data, user could use pre-defined formula and selected attributes to generate new attributes. Generated attributes would be treated in the same way as original attributes. -.. image:: ../images/visualization/dashboard/citi_bike_intra_epoch_by_station_3.gif - :alt: citi_bike_intra_epoch_by_station_3 +.. figure:: ..\images\visualization\dashboard\citi_bike_intra_epoch_by_station_3.gif + :alt: citi\_bike\_intra\_epoch\_by\_station\_3 If user choose to view information by snapshot, it means attributes of all stations within a selected snapshot would be displayed. By changing @@ -816,5 +816,5 @@ of sampled data. Particularly, if user want to check the name of a specific station, just hovering on the according bar. -.. image:: ../images/visualization/dashboard/citi_bike_intra_epoch_by_snapshot.gif - :alt: citi_bike_intra_epoch_by_snapshot \ No newline at end of file +.. figure:: ..\images\visualization\dashboard\citi_bike_intra_epoch_by_snapshot.gif + :alt: citi\_bike\_intra\_epoch\_by\_snapshot \ No newline at end of file diff --git a/docs/source/scenarios/container_inventory_management.rst b/docs/source/scenarios/container_inventory_management.rst index 3e5e8ab98..3eeaa64eb 100644 --- a/docs/source/scenarios/container_inventory_management.rst +++ b/docs/source/scenarios/container_inventory_management.rst @@ -658,8 +658,8 @@ To change "Start Epoch" and "End Epoch", user could specify the selected data range. To change "Epoch Sampling Ratio", user could change the sampling rate of selected data. -.. image:: ../images/visualization/dashboard/cim_inter_epoch.gif - :alt: cim_inter_epoch +.. figure:: ..\images\visualization\dashboard\cim_inter_epoch.gif + :alt: cim\_inter\_epoch Intra-epoch view ^^^^^^^^^^^^^^^^ @@ -669,13 +669,13 @@ slider, users can select different epochs. Furthermore, this part of data is divided into two dimensions: by snapshot and by port according to time and space. In terms of data display, according to the different types of attributes, it is divided into two levels: accumulated data -(accumulated attributes. e.g. acc\_fulfillment) and detail data. + (accumulated attributes. e.g. acc\_fulfillment) and detail data. If user choose to view information by ports, attributes of the selected port would be displayed. -.. image:: ../images/visualization/dashboard/cim_intra_epoch_by_ports.gif - :alt: cim_intra_epoch_by_ports +.. figure:: ..\images\visualization\dashboard\cim_intra_epoch_by_ports.gif + :alt: cim\_intra\_epoch\_by\_ports If user choose to view data by snapshots, attributes of selected snapshot would be displayed. The charts and data involved in this part @@ -697,8 +697,8 @@ over time in the current epoch. The bar chart of Port Accumulated Attributes displays the global change of ports. -.. image:: ../images/visualization/dashboard/cim_intra_epoch_by_snapshot_acc_data.gif - :alt: cim_intra_epoch_by_snapshot_acc_data +.. figure:: ..\images\visualization\dashboard\cim_intra_epoch_by_snapshot_acc_data.gif + :alt: cim\_intra\_epoch\_by\_snapshot\_acc\_data Detail Data ~~~~~~~~~~~ @@ -706,5 +706,5 @@ Detail Data Since the cargoes is transported through vessels, information of vessels could be viewed by snapshot. Same as ports, user could change the sampling rate of vessels. -.. image:: ../images/visualization/dashboard/cim_intra_epoch_by_snapshot_detail_data.gif - :alt: cim_intra_epoch_by_snapshot_detail_data +.. figure:: ..\images\visualization\dashboard\cim_intra_epoch_by_snapshot_detail_data.gif + :alt: cim\_intra\_epoch\_by\_snapshot\_detail\_data From 1f82cdece11b4b30a20fa40fae25dcb9015bd596 Mon Sep 17 00:00:00 2001 From: ysqyang Date: Tue, 12 Jan 2021 10:57:13 +0000 Subject: [PATCH 334/337] aligned with v0.2 --- maro/data_lib/dump_csv_converter.py | 5 +- .../articles/simple_bike_repositioning.ipynb | 721 ------------------ .../DecisionLogic.png | Bin 143047 -> 0 bytes .../simple_bike_repositioning/TripLogic.png | Bin 131866 -> 0 bytes .../TripRequirements.png | Bin 141914 -> 0 bytes requirements.dev.txt | 2 +- 6 files changed, 4 insertions(+), 724 deletions(-) delete mode 100644 notebooks/articles/simple_bike_repositioning.ipynb delete mode 100644 notebooks/articles/simple_bike_repositioning/DecisionLogic.png delete mode 100644 notebooks/articles/simple_bike_repositioning/TripLogic.png delete mode 100644 notebooks/articles/simple_bike_repositioning/TripRequirements.png diff --git a/maro/data_lib/dump_csv_converter.py b/maro/data_lib/dump_csv_converter.py index 04d6b1117..97a777c8d 100644 --- a/maro/data_lib/dump_csv_converter.py +++ b/maro/data_lib/dump_csv_converter.py @@ -1,5 +1,5 @@ -# Copyright (c) Microsoft Corporation. -# Licensed under the MIT license. +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. import os @@ -7,6 +7,7 @@ from datetime import datetime from math import floor from pathlib import Path +from shutil import copyfile import numpy as np import pandas as pd diff --git a/notebooks/articles/simple_bike_repositioning.ipynb b/notebooks/articles/simple_bike_repositioning.ipynb deleted file mode 100644 index ef7d27a9d..000000000 --- a/notebooks/articles/simple_bike_repositioning.ipynb +++ /dev/null @@ -1,721 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# The Simple Bike Repositioning Scenario\n" - ] - }, - { - "cell_type": "code", - "execution_count": 1, - "metadata": {}, - "outputs": [], - "source": [ - "import matplotlib\n", - "import matplotlib.pyplot as plt\n", - "import numpy as np\n", - "import random\n", - "from math import pi, sin\n", - "from typing import List\n", - "\n", - "from maro.backends.frame import FrameBase, FrameNode, NodeAttribute, NodeBase, node\n", - "from maro.event_buffer import MaroEvents\n", - "from maro.simulator.scenarios import AbsBusinessEngine\n", - "from maro.simulator import Env" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## 1. Problem Setting" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "To simplify this problem, we consider a situation where are only 4 bike stations. Let's denote the station set as $\\{s_i \\mid i=0,1,2,3\\}$." - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": {}, - "outputs": [], - "source": [ - "TOTAL_STATIONS = 4" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "The detailed trip requirement distribution among these stations are shown in the figure below. Here we use $F_{ij}(t)$ to present the trip requirements from station $s_i$ to station $s_j$.\n", - "\n", - "\n", - "\n", - "The right half of this figure shows the trip requirement curve of each station pair $F_{ij}(t)$ over time $t$. We can find that:\n", - "- $F_{02}(t)$ and $F_{20}(t)$ follow a symmetric and periodic transition pattern. In the first half of the period, $F_{02}(t)$ first increases from 0 to the maximum and then decreases to 0, while $F_{20}(t)$ remains at 0. In the second half of the period, $F_{20}(t)$ starts to increase and then decreases, while $F_{02}(t)$ remains at 0. \n", - "- Generally speaking, $F_{13}(t)$ and $F_{31}(t)$ remains at the same level over time, with slight fluctuations only in a few times.\n", - "- As for $F_{01}(t)$ and $F_{32}(t)$, they show stable and equal trip requirements.\n", - "\n", - "Specifically, we use sine functions to model these periodic changes, that we defined:\n", - "\n", - "$F_{02}(t) = \\max(0, 8 \\sin(0.1 t))$\n", - "\n", - "$F_{20}(t) = \\max(0, -8 \\sin(0.1 t))$\n", - "\n", - "$F_{13}(t) = 5 + \\max(0, 5 \\sin(0.2\\pi t) - 4) + \\min(0, 5 \\sin(0.2\\pi t) + 4)$\n", - "\n", - "$F_{31}(t) = 5 + \\max(0, 5 \\sin(0.4 t) - 4) + \\min(0, 5 \\sin(0.4 t) + 4)$\n", - "\n", - "$F_{01}(t) = 3$\n", - "\n", - "$F_{32}(t) = 3$" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": {}, - "outputs": [], - "source": [ - "max_0_2 = 8 # The maximum trip requirements between station 0 and station 2.\n", - "exp_1_3 = 5 # The expected trip requirements between station 1 and station 3.\n", - "const_num = 3 # The number of trip requirements from station 0 to station 1, from station 3 to station 2.\n", - "\n", - "def generate_station_requirements(tick: int):\n", - " # The number of trip requirements between station 0 and station 2 follows a sine function.\n", - " # Specifically, it is the F20(t) and F02(t) in the figure above.\n", - " num_between_0_2 = round(sin(0.1 * tick) * max_0_2)\n", - " requirements_between_0_2 = [(0, 2)] * num_between_0_2 if num_between_0_2 >= 0 else [(2, 0)] * -num_between_0_2\n", - "\n", - " # The number of trip requirements between station 1 and station 3 are almostly equal (with only slight difference).\n", - " # Specifically, it is the F13(t) and F31(t) in the figure above.\n", - " num_1_3 = round(exp_1_3 + max(0, exp_1_3 * (sin(0.2 * pi * tick) - 1) + 1) + min(0, exp_1_3 * (sin(0.2 * pi * tick) + 1) - 1))\n", - " requirements_1_3 = [(1, 3)] * num_1_3\n", - " num_3_1 = round(exp_1_3 + max(0, exp_1_3 * (sin(0.4 * tick) - 1) + 1) + min(0, exp_1_3 * (sin(0.4 * tick) + 1) - 1))\n", - " requirements_3_1 = [(3, 1)] * num_3_1\n", - " \n", - " # The number of trip requirements from station 0 to station 1 and the ones from station 3 to station 2 are equal.\n", - " # Specifically, it is the F01(t) and F32(t) in the figure above.\n", - " const_requirements = [(0, 1)] * const_num + [(3, 2)] * const_num\n", - "\n", - " # Randomly shuffle the order of trip requirements.\n", - " requirements = (requirements_between_0_2 + requirements_1_3 + requirements_3_1 + const_requirements)\n", - " random.shuffle(requirements)\n", - " return requirements" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Morever, for each trip and the bike repositioning operation, we assume that they can both be done no longer than 3 ticks. That:" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "metadata": {}, - "outputs": [], - "source": [ - "# The maximum trip duration. Unit: tick.\n", - "maximum_duration = 3\n", - "\n", - "def generate_duration():\n", - " return random.randrange(1, maximum_duration)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "To ensure the bike number in this simple scenario is almost enough for trip requirements, we should take the time for trips and repositioning operations into consideration. Along with the peak requirement and the expectation of each station pair, we can roughly figure out the total bike demand in this problem. And then distribute these bikes equally to each station." - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Total bikes needed in this scenario: 96, intial station inventory: 24.\n" - ] - } - ], - "source": [ - "bike_needed_between_0_2 = max_0_2 * maximum_duration * 2\n", - "bike_needed_between_1_3 = exp_1_3 * maximum_duration * 2\n", - "bike_needed_const = const_num * maximum_duration * 2\n", - "\n", - "total_bike_needed = bike_needed_between_0_2 + bike_needed_between_1_3 + bike_needed_const\n", - "\n", - "# The initial bike number in each station\n", - "INITIAL_BIKES = total_bike_needed // TOTAL_STATIONS\n", - "\n", - "print(f\"Total bikes needed in this scenario: {total_bike_needed}, intial station inventory: {INITIAL_BIKES}.\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## 2. Simulation Logics and Implementation" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "To implement the simple bike repositioning scenario, we first define the class of the bike station. Here each station only contains 3 attributes:\n", - "- `bikes`: The real-time available bikes in this station.\n", - "- `shortage`: How many trip requirements is failed due to the lack of bikes till now.\n", - "- `requirements`: How many trip requirements received till now." - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "metadata": {}, - "outputs": [], - "source": [ - "# Definition of the bike station.\n", - "# The attribute name can be used to access the environment snapshot lists.\n", - "@node(\"station\")\n", - "class Station(NodeBase):\n", - " bikes = NodeAttribute(\"i\")\n", - " shortage = NodeAttribute(\"i\")\n", - " requirements = NodeAttribute(\"i\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "In MARO, we use the environment frame to save the environment status. To initialize the frame of this scenario, we need to specify how many stations we have and how many snapshots we need." - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "metadata": {}, - "outputs": [], - "source": [ - "MAX_TICKS = 100\n", - "\n", - "# Definition of the environment frame. The number of stations is given here.\n", - "class SimpleCitibikeFrame(FrameBase):\n", - " stations = FrameNode(Station, TOTAL_STATIONS)\n", - "\n", - " def __init__(self):\n", - " super().__init__(enable_snapshot=True, total_snapshot=MAX_TICKS)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "In MARO, the simulator is driven by the generation and the processing of various events. In this simple scenario, we only take the trip requirement logic and the bike repositioning logic into consideration. Then only 3 scenario-dependent event types here:\n", - "\n", - "- `TRIP_REQUIREMENT_EVENT`: When trip requirement generated, a event with this type will be inserted into the event buffer to trigger the trip requirement handling logic.\n", - "- `PENDING_DECISION_EVENT`: When the simulator want the agent to do a bike repositioning, it will throw a pending decision event. \n", - "- `BIKE_ARRIVAL_EVENT`: Both a trip requirement fulfillment and a bike repositioning operation are paired with a bike arrival event. It means the trip is finished and the bike turns to be available in the destination station, and the bikes are successfully transported to the destination station respectively." - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "metadata": {}, - "outputs": [], - "source": [ - "# Predefined event types\n", - "TRIP_REQUIREMENT_EVENT = \"trip_requirement_event\"\n", - "PENDING_DECISION_EVENT = \"pending_decision_event\"\n", - "BIKE_ARRIVAL_EVENT = \"bike_arrival_event\"" - ] - }, - { - "cell_type": "code", - "execution_count": 9, - "metadata": {}, - "outputs": [], - "source": [ - "# The event thrown by the environment to the agent, which is used to trigger the agent's decision.\n", - "class PendingDecisionEvent():\n", - " def __init__(self, station_index: int):\n", - " self.station_index = station_index\n", - " \n", - " def __repr__(self):\n", - " return f\"\"" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### 2.1. The Trip Fulfillment Logic" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "\n", - "\n", - "The figure above illustrate the trip fulfillment logic in this simple scenario:\n", - "1. Each time, the simulator will generate several trip requirements according to the pattern we defined in the **Problem Setting** part. These trip requirements will then be inserted into the event buffer.\n", - "2. Once the simulator get a trip requirement event from the event buffer, it will call the corresponding callback function to handle it.\n", - "3. If there is available bike in the source station, the remaining bikes state of the source station will be updated and a future bike arrival event for the destination station will be generated and inserted into the event buffer. Else, a shortage of the source station is recorded.\n", - "4. Once the simulator get a bike arrival event from the event buffer, it will call the corresponding callback function to handle it.\n", - "5. The remaining bikes state of the station bike arrives will be updated." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### 2.2. The Bike Repositioning Decision Logic" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "\n", - "\n", - "The figure above illustrate the bike repositioning logic in this simple scenario:\n", - "1. Each time, the simulator will check the state of each station.\n", - "2. When the station lack of bikes, the simulator will create an pending decision event and insert it into the event buffer.\n", - "3. Once the simulator gets a pending decision event from the event buffer, it will throw it out to the agent to trigger the agent's bike repositioning actions.\n", - "4. The agent can query the data model to get the information it needs, and make bike repositioning decision and reply with the repositioning action to the simulator.\n", - "5. Once the simulator get a repositioning action sent by the agent from the event buffer, it will call the corresponding handler function.\n", - "6. The handler function of the repositioning action will update the state of the source station and create a corresponding bike arrival event for it." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "The overall logic of this simple bike repositioning scenario and the hanlder functions are all defined as a `BusinessEngine`." - ] - }, - { - "cell_type": "code", - "execution_count": 10, - "metadata": {}, - "outputs": [], - "source": [ - "class SimpleCitibikeBusinessEngine(AbsBusinessEngine):\n", - " def __init__(self, **kwargs):\n", - " super().__init__(scenario_name=\"SimpleCitibike\", **kwargs)\n", - "\n", - " self._frame = SimpleCitibikeFrame()\n", - " \n", - " # A number of station instances will be created after the initialization of the frame.\n", - " self._stations: List[Station] = self._frame.stations\n", - " self._intialize_stations()\n", - "\n", - " # Register the event types and their handler functions.\n", - " self._event_buffer.register_event_handler(TRIP_REQUIREMENT_EVENT, self._on_requirement)\n", - " self._event_buffer.register_event_handler(BIKE_ARRIVAL_EVENT, self._on_arrive_station)\n", - " self._event_buffer.register_event_handler(MaroEvents.TAKE_ACTION, self._on_action_recieved)\n", - "\n", - " @property\n", - " def snapshots(self):\n", - " # The interface for the agents to access.\n", - " return self._frame.snapshots\n", - "\n", - " @property\n", - " def frame(self):\n", - " # The interface for the agents to access.\n", - " return self._frame\n", - "\n", - " def step(self, tick: int):\n", - " for station in self._stations:\n", - " if station.bikes == 0:\n", - " # For station without any bikes left, create a PENDING_DECISION_EVENT and insert it into the event buffer.\n", - " pending_decision_event = self._event_buffer.gen_decision_event(tick, PendingDecisionEvent(station.index))\n", - " self._event_buffer.insert_event(pending_decision_event)\n", - "\n", - " # Generate the trip requirements and insert them into the event buffer.\n", - " # A trip: (source station, destination station)\n", - " for source_station, destination_station in generate_station_requirements(tick):\n", - " requirement_event = self._event_buffer.gen_atom_event(tick, TRIP_REQUIREMENT_EVENT, (source_station, destination_station))\n", - " self._event_buffer.insert_event(requirement_event)\n", - "\n", - " def reset(self):\n", - " self._frame.reset()\n", - " self._frame.snapshots.reset()\n", - " self._intialize_stations()\n", - " random.seed(1234)\n", - "\n", - " def post_step(self, tick: int):\n", - " # Take and save the snapshot of the environment state at the end of each tick.\n", - " self._frame.take_snapshot(tick)\n", - "\n", - " # Clear requirement and shortage after taking snapshot,\n", - " # so that they will only be value at specific tick\n", - " for station in self._stations:\n", - " station.requirements = 0\n", - " station.shortage = 0\n", - "\n", - " # To end the simulation or not.\n", - " return tick == self._max_tick - 1\n", - "\n", - " def _intialize_stations(self):\n", - " for station in self._stations:\n", - " station.bikes = INITIAL_BIKES\n", - "\n", - " def _on_requirement(self, event):\n", - " src_station_index, dest_station_index = event.payload\n", - " station: Station = self._stations[src_station_index]\n", - " station.requirements += 1\n", - "\n", - " if station.bikes < 1:\n", - " # No bike left, a shortage is recorded.\n", - " station.shortage += 1\n", - " else:\n", - " # Fulfilled the trip requirement.\n", - " station.bikes -= 1\n", - "\n", - " # Generate the BIKE_ARRIVAL_EVENT and insert it into the event buffer.\n", - " bike_arrive_event = self._event_buffer.gen_atom_event(event.tick + generate_duration(), BIKE_ARRIVAL_EVENT, (dest_station_index, 1))\n", - " self._event_buffer.insert_event(bike_arrive_event)\n", - "\n", - " def _on_arrive_station(self, event):\n", - " # The handler for BIKE_ARRIVAL_EVENT, which includes both the ending of a trip and the repositioning of the bikes.\n", - " station_index, number = event.payload\n", - " station = self._stations[station_index]\n", - " station.bikes += number\n", - "\n", - " def _on_action_recieved(self, event):\n", - " # The repositioning action from agent.\n", - " action = event.payload\n", - " \n", - " if action:\n", - " src_station_index, target_station_index, number = action\n", - " station = self._stations[src_station_index]\n", - " station.bikes -= number\n", - "\n", - " # Generate the BIKE_ARRIVAL_EVENT and insert it into the event buffer.\n", - " arrive_event = self._event_buffer.gen_atom_event(event.tick + generate_duration(), BIKE_ARRIVAL_EVENT, (target_station_index, number))\n", - " self._event_buffer.insert_event(arrive_event)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## 3. Rule-Based Solution to This Problem" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### 3.1 Problem Analysis" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### 3.1.1 Oracle Policy\n", - "\n", - "I would like to divide-and-conquer the four-node system, i.e, the left subsystem and the right subsystem. The insight is that the two subsystems exchange bikes of the same amount, controlled by the functions $F_{32}$ and $F_{01}$. This indicates that, if the initial bikes can already support the inner requirement loops in each subsystem, we can balance each subsystem separately. \n", - "\n", - "For the left subsystem $\\left\\{s_1, s_3\\right\\}$, the *outer system* gives bikes to $s_1$ and ask out from $s_3$. Therefore, the straight-forward approach is to move bikes from $s_1$ to $s_3$ with the same amount as that $s_1$ receives by the function $F_{01}$.\n", - "\n", - "For the right part, as the requirement curves $F_{20}$ and $F_{02}$ are with the period $20*\\pi$, we consider the policy in each half period. The first half period is really simple because only $s_0$ has requirements. Therefore, the right policy is to simply move all bikes from $s_2$ to $s_0$. In the second half period, as $s_0$ also gives bikes to $s_1$ by the function $F_{01}$, $s_0$ has to reserve a certain amount $R_{01}$ when sending its bikes to $s_2$, instead of send all its bikes to $s_2$ like $s_2$ does in the first half period. For simplicity, the reserved amount $R_{01}$ can be set to be constant.\n", - "\n", - "The code implementation can be seen in `RuleBasedPolicy` class below.\n", - "\n", - "#### 3.1.2 Policy Guided by Statistics\n", - "\n", - "Although the oracle policy solves the problem in simple rules and gains obvious improvement in performance, it is binded to the bike requirement functions. Here, I would like to give a more general solution without building upon the specific settings. \n", - "\n", - "For human beings, the most straight-forward strategy when they knows nothing about the latent requirement curves is to run the system for a while and see which station needs additional bikes. Similarly, I maintain a statistical variable to describe the extent to which one station is in lack of bikes. Each time a station needs bikes, I provide bikes from the station of the least shortage.\n", - "\n", - "As you can see in the implementation, the statistical variable I use is the exponential moving average of the shortage of each station along the ticks. In addition, to prevent the same station being chosen for multiple times in a single tick, I add a penalty to the selected station, which will reduce its ranking in the latter selections." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### 3.2 Implementation" - ] - }, - { - "cell_type": "code", - "execution_count": 11, - "metadata": {}, - "outputs": [], - "source": [ - "class RuleBasedPolicy:\n", - " def __init__(self, env):\n", - " self.env = env\n", - " self.half_period = 10 * pi\n", - " \n", - " def __call__(self, evt):\n", - " # Determine which half-period current environment is in.\n", - " phase = int(self.env.tick / self.half_period) % 2\n", - " # Get the amount of current bikes in that station.\n", - " cur_bikes = self.env.snapshot_list[\"station\"][self.env.tick::\"bikes\"]\n", - " if evt.station_index == 0 or evt.station_index == 2:\n", - " if phase == 0:\n", - " # Give all the bikes in 2 to 0.\n", - " payload = (2, 0, cur_bikes[2])\n", - " else:\n", - " # Reserve 5 bikes in 0 and give all the remainings to 2.\n", - " payload = (0, 2, max(cur_bikes[0] - 5, 0))\n", - " else:\n", - " # Always reserve 5 bikes in 1 and give all the remainings to 3.\n", - " payload = (1, 3, max(cur_bikes[1] - 5, 0))\n", - " return payload\n", - " \n", - "class NoActionPolicy:\n", - " def __init__(self, env):\n", - " self.env = env\n", - " \n", - " def __call__(self, evt):\n", - " return None" - ] - }, - { - "cell_type": "code", - "execution_count": 12, - "metadata": {}, - "outputs": [], - "source": [ - "class StatisticsBasedPolicy:\n", - " def __init__(self, env, decay_factor=0.98, weight=0.6):\n", - " self.env = env\n", - " self.factor = decay_factor\n", - " self.weight = weight\n", - " self.tick = env.tick\n", - " self.shortage_emv = np.zeros(TOTAL_STATIONS, np.double)\n", - " self.selected_cnts = np.zeros(TOTAL_STATIONS, np.double)\n", - " \n", - " def __call__(self, evt):\n", - " src_station = evt.station_index\n", - " # Get the tick range from the last action time.\n", - " past_ticks = list(range(self.tick, self.env.tick))\n", - " tick_len = len(past_ticks)\n", - " if tick_len != 0:\n", - " # If it is the start of a new tick.\n", - " past_shortage = self.env.snapshot_list[\"station\"][past_ticks::\"shortage\"].reshape(tick_len, TOTAL_STATIONS)\n", - " decay_vec = np.logspace(0, tick_len - 1, tick_len, base=self.factor)\n", - " self.shortage_emv = np.matmul(decay_vec, past_shortage) + (self.factor**tick_len) * self.shortage_emv\n", - " self.tick = self.env.tick\n", - " self.selected_cnts = np.zeros(TOTAL_STATIONS, np.double)\n", - " \n", - " # Sort the station and get the one of the least shortage and the least selected count.\n", - " candidate_dest_station = np.argmin(self.shortage_emv + self.weight * self.selected_cnts)\n", - " if candidate_dest_station == src_station:\n", - " return None\n", - " else:\n", - " self.selected_cnts[candidate_dest_station] += 1\n", - " dest_sh = self.shortage_emv[candidate_dest_station] + 1e-8\n", - " src_sh = self.shortage_emv[src_station] + 1e-8\n", - " dest_inventory = self.env.snapshot_list[\"station\"][self.tick:candidate_dest_station:\"bikes\"]\n", - " dest_ratio = src_sh / (src_sh + dest_sh)\n", - " payoff = (candidate_dest_station, src_station, int(dest_ratio * dest_inventory))\n", - " \n", - " return payoff" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### 3.3 Interaction with the Environment" - ] - }, - { - "cell_type": "code", - "execution_count": 13, - "metadata": {}, - "outputs": [], - "source": [ - "def run_policy(PolicyClass, args=()):\n", - " # Initialize an environment instance with the predefined SimpleCitibikeBusinessEngine.\n", - " env = Env(durations=MAX_TICKS, business_engine_cls=SimpleCitibikeBusinessEngine)\n", - "\n", - " # Get the station attributes from the snapshot_list.\n", - " station_snapshots = env.snapshot_list[\"station\"]\n", - " policy = PolicyClass(env, *args)\n", - "\n", - " is_done = False\n", - " action = None\n", - " while not is_done:\n", - " # Looping until the environment ends.\n", - " metrics, decision_evt, is_done = env.step(action)\n", - " if not is_done:\n", - " # Get the action by calling the policy.\n", - " action = policy(decision_evt)\n", - "\n", - " # Using [tick(s) : node index(s) : attribute(s)] to access the environment snapshots.\n", - " # The return is a Numpy array with shape (ticks * nodes * attributes * slots, )\n", - " all_shortage = station_snapshots[::\"shortage\"]\n", - " all_requirement = station_snapshots[::\"requirements\"]\n", - "\n", - " return {\n", - " \"requirement\": all_requirement.reshape(MAX_TICKS, TOTAL_STATIONS),\n", - " \"shortage\": all_shortage.reshape(MAX_TICKS, TOTAL_STATIONS)\n", - " }\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### Test baseline with no actions" - ] - }, - { - "cell_type": "code", - "execution_count": 14, - "metadata": {}, - "outputs": [], - "source": [ - "no_action_result = run_policy(NoActionPolicy)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### Test rule-based policy" - ] - }, - { - "cell_type": "code", - "execution_count": 15, - "metadata": {}, - "outputs": [], - "source": [ - "rule_based_result = run_policy(RuleBasedPolicy)" - ] - }, - { - "cell_type": "code", - "execution_count": 16, - "metadata": {}, - "outputs": [], - "source": [ - "def grid_search_on_statistic_hyperparameters():\n", - " # simple grid search on the optimal hyper-parameters\n", - " min_rlt, min_decay, min_weight = None, 0, 0\n", - " # decay: 0~0.95\n", - " for i in range(0, 20):\n", - " decay = i * 0.05\n", - " # weight: 0~0.9\n", - " for j in range(0, 10):\n", - " weight = j * 0.1\n", - " rlt = run_policy(StatisticsBasedPolicy, args=(decay, weight))\n", - " if min_rlt is None or rlt[\"shortage\"].sum() < min_rlt[\"shortage\"].sum():\n", - " min_rlt = rlt\n", - " min_decay = decay\n", - " min_weight = weight\n", - " return {\"best_result\": min_rlt, \"decay_factor\": min_decay, \"selection_penalty_weight\": min_weight}" - ] - }, - { - "cell_type": "code", - "execution_count": 17, - "metadata": {}, - "outputs": [], - "source": [ - "searched_result = grid_search_on_statistic_hyperparameters()\n", - "statistics_result = searched_result[\"best_result\"]" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### Display the algorithm results" - ] - }, - { - "cell_type": "code", - "execution_count": 18, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "" - ] - }, - "execution_count": 18, - "metadata": {}, - "output_type": "execute_result" - }, - { - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAABCUAAAHoCAYAAAB+RlVfAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8li6FKAAAgAElEQVR4nOzdd1RU194+8OfQQUENGESxxV6iRFCiUWLFgjM3TRNjidf4S8KIgLF3rKioYGFM1cSoyRsTk8wosaBRY6cYoyZi7xUVFOnM/v2Ri9er9DnDYYbns9as974z++z95K4b5pzv7CIJIUBEREREREREVN6slA5ARERERERERJUTixJEREREREREpAgWJYiIiIiIiIhIESxKEBEREREREZEiWJQgIiIiIiIiIkWwKEFEREREREREirBROoBc3NzcRIMGDZSOQURERERERERPSEhISBZC1CzoM4spSjRo0ADx8fFKxyAiIiIiIiKiJ0iSdKmwz7h8g4iIiIiIiIgUwaIEERERERERESmCRQkiIiIiIiIiUgSLEkRERERERESkCBYliIiIiIiIiEgRLEoQERERERERkSJYlCAiIiIiIiIiRbAoQURERERERESKYFGCiIiIiIiIiBTBogQRERERERERKYJFCSIiIiIiIiJSBIsSRERERERERKQIFiWIiIiIiIiISBEsShARERERERGRIliUICIiIiIiIiJFsChBRERERERERIpgUYKIiIiIiKgMhBAwGAxKxyAyayxKEBERERERlUHnzp3x3nvvKR2DyKyxKEFERERERFRK58+fx4EDB7Bu3Tps375d6ThEZotFCSIiIiIiolLS6/UAgNq1ayMoKAhZWVkKJyIyTyxKEBERERERlZJOp0PLli2xZs0anDlzBhEREUpHIjJLLEoQERERERGVQkpKCvbu3Qu1Wg1/f38MGDAA8+bNw4ULF5SORmR2WJQgIiIiIiIqha1btyI3NxcqlQoAsHTpUlhbWyM4OFjhZETmh0UJIiIiIiKiUtDpdKhZsyZ8fX0BAJ6enggLC8PmzZuh0+kUTkdkXliUICIiIiIiKqGcnBzExMSgf//+sLa2fvx+SEgIWrVqheDgYKSnpyuYkMi8sChBRERERERUQvv27UNqaurjpRv5bG1todVqcenSJcyfP1+hdETmh0UJIiIiIiKiEtLpdLC3t0evXr2e+czPzw9Dhw7FokWLkJSUpEA6IvPDogQREREREVEJCCGg0+nQvXt3VK1atcA2ERERcHJyQlBQEIQQ5ZyQyPywKEFERERERFQCf//9N86fPw+1Wl1oG3d3d8ybNw+xsbHYuHFjOaYjMk/lVpSQJGmyJElxkiQ9kCTpjiRJekmSWj/V5itJksRTr0PllZGIiIiIiKgw+Sdr9O/fv8h2H330Edq1a4cxY8bg4cOH5RGNyGyV50yJrgC0ADoB6A4gF0CsJEnPPdUuFoDHE69+5ZiRiIiIiIioQDqdDu3atYOnp2eR7aytraHVanHjxg2EhYWVTzgiM1VuRQkhRG8hxBohxAkhxHEAQwHUBPDKU02zhBA3n3jdK6+MREREREREBbl9+zYOHTpU5NKNJ/n6+mLkyJFYtmwZjh8/buJ0ROZLyT0lnP8z/v2n3u8sSdJtSZJOS5L0uSRJzyuQjcgoV65cgZ+fH/7++2+lo5jE8OHD8emnnyodg4gqiczMTLz11lvYtGmT0X19++23eOedd5CTkyNDMiKqTLZs2QIhRImLEgAQHh6O6tWrIzQ01ITJiMybkkWJZQD+AHDwife2AhgGoAeAsQA6ANglSZJ9QR1IkvSBJEnxkiTF37lzx9R5iUpsw4YN+P333/HRRx9Z3K7LFy5cwNdff43Q0FBcuHBB6ThEVAlERETgxx9/xMiRI2HM9/3169fx4Ycf4v/+7/+wbNkyGRMSUWWg1+vh6ekJLy+vEl/j6uqK0NBQ7Nq1C9evXzdhOiLzpUhRQpKkpQA6A3hTCJGX/74Q4jshhE4IcVwIoQfQF0AzAAEF9SOE+EwI4SOE8KlZs2a5ZCcqCb1eDwcHB+zduxfr169XOo6s9Hr94/8cEhKiYBIiqgwuXLiA+fPno0uXLnj48CEmTZpU5r7GjRuH7OxsdOrUCWFhYbh69aqMSYnIkmVmZmLbtm1QqVSQJKlU1/7rX/8C8M9MCyJ6VrkXJSRJigQwCEB3IcT5otoKIa4DuAqgSXlkI5LDnTt3cODAAYwfPx7t27fHuHHjkJKSonQs2eh0OrRo0QKzZ8+GXq9/vAs1EZEpBAcHw9raGhs2bMCYMWOwevVqHDhwoNT97Nq1C99++y0mTpyIdevWIS8vDx9//LEJEhORJdq1axfS09OhUqlKfW3r1q3RoEED3jMRFaJcixKSJC3DfwsSp0rQ3g1AHQA3TJ2NSC756w1ff/11rFq1Crdv38aMGTOUjiWL1NRU7NmzB2q1GqGhoWjZsiWCg4ORnp6udDQiskA6nQ6bN2/GrFmz4OnpiRkzZsDT0xMajQa5ubkl7ic7OxujRo3CCy+8gEmTJqFhw4aYOnUqNm7ciO3bt5vwn4CILIVer0eVKlXQrVu3Ul8rSRLUajViY2N5z0RUgHIrSkiSFA3g3wDeBXBfkqRa/3lV/c/nVSVJWixJUkdJkhpIktQVgB7AbQA/lVdOImM9ud7Q29sbgYGBiI6ORmJiotLRjLZ161bk5uZCpVLB1tYWWq0Wly5dwvz585WORkQWJj09HcHBwWjVqhWCg4MBAFWrVkVUVBSOHTuG6OjoEve1dOlSnDp1CitWrICjoyMAYPz48WjSpAmCgoKQlZVlkn8GIrIMQgjodDr07t0bDg4OZepDpVIhMzMTsbGxMqcjMn/lOVNCg39O3NiJf2Y+5L/G/efzPAAvAvgFwGkAXwNIAtBRCPGwHHMSlVlB6w3nzp0LV1dXaDQaGAwGhRMaR6fTwc3NDS+//DIA4NVXX8WQIUMQERGB06dPK5yOiCzJvHnzcOnSJWi1Wtja2j5+/4033kDv3r0xffp03LhR/ETKy5cvY86cOXjttdfQr1+/x+/b29tj5cqVOHPmDCIiIkzyz0BEliExMRHXr18v09KNfH5+fnBxceESDqIClFtRQgghFfIK+8/nGUKI3kKI54UQdkKI+kKI4UKIK+WVkchYu3fvxqNHj/7nS6tGjRpYvHgxDh8+jNWrVyuYzjg5OTmIiYlBQEAArK2tH78fEREBR0dHjBo1yuJOGiEiZSQlJSEiIgJDhw6Fn5/f/3wmSRJWrFiBrKwsjBs3rpAe/is0NBRCCERFRT3zmb+/PwYMGIB58+bxNCEiKpRer4ckSQgIKHDv/RKxs7NDnz59sHnzZrP/kYpIbkoeCUpkcXQ6XYHrDYcOHYouXbpg4sSJSE5OViidcfbv34+UlJRnzuauVasW5s6di9jYWGzcuFGhdERkKYQQGDVqFJycnAqdwdCkSRNMmjQJGzZswK5duwrtKyYmBj/99BNmzJiB+vXrF9hm6dKlsLa25mlCRFQonU6HTp06wdjT/tRqNW7duoW4uDiZkhFZBhYliGQihIBery9wvaEkSYiOjkZqaiomT56sUELj6HQ62NnZwd/f/5nPAgMD8dJLL2HMmDF4+JCrrYio7L7//nvs3LkT8+bNg7u7e6Ht8jesHDVqFLKzs5/5PCMjA6NHj0bz5s2LPGXD09MTYWFhPE2IiAp05coVHD161KilG/n69u0La2tr/q0hegqLEkQy+eOPP3D16tVCv7RefPFFhISE4IsvvsChQ4fKOZ1x8jd46t69O6pWrfrM59bW1li1ahVu3LiBWbNmKZCQiCzBw4cP8fHHH6Ndu3b46KOPimzr6OiIFStW4NSpU4iMjHzm84ULF+L8+fOIjo6GnZ1dkX2FhIQ83lCTO+MT0ZM2b94MAM/MFC2L5557Dp07d4Zerze6LyJLwqIEkUx0Ol2x6w3DwsJQu3btUh9np7RTp07h3LlzRX4h+/r6YuTIkYiKisLx48fLMR0RWYqwsDDcuHEDWq32f/auKUxAQABee+01zJ49G5cvX378/tmzZ7FgwQIMGjQI3bt3L7YfniZERIXR6XRo1KgRmjdvLkt/arUax48f5z42RE9gUYJIJiVZb+js7IzIyEgcPXoUq1atKsd0xsmv6Bc3dTE8PBzVq1fnppdEVGrHjx/HsmXL8P/+3/+Dr69via+LioqCEAKhoaEA/pnZNXr0aNjZ2WHJkiUl7sfPzw9Dhw5FREQEkpKSSp2fiCzPw4cPsWvXLqjV6senqhkr/16KsyWI/otFCSIZXL16FYmJiSVabzhgwAD07NkT06ZNw82bN8shnfF0Oh1eeukleHp6FtnO1dUVCxYswO+//45vvvmmnNIRkbkTQkCj0aB69eqlnqlQv359TJ8+HT/99BN+/fVX/PTTT9i6dStmz54NDw+PUvWVf5pQUFAQC6tEhB07diA7O1uWpRv5mjRpgubNm7MoQfQEFiWIZFCa9YaSJGHlypXIyMjA+PHjTR3NaHfu3MGBAwdK/IU8YsQIvPzyyxg3bhzu379v4nREZAnWrl2Lffv2YeHChXB1dS319WPHjkWzZs0QFBSE0NBQtGnTBkFBQaXux93dHfPmzeNpQkQE4J8fZapXr45XXnlF1n7VajV2796N1NRUWfslMleSpfwS4OPjI+Lj45WOQZVUv379cPr0aZw5c6bE0/umTZuGefPm4c0334Stra3RGTp06IAxY8YY3c/Tvv76awwfPhzx8fHw9vYu0TV//PEHvL298dFHHyE6Olr2TESW5LvvvoODgwNee+01k4+VlZWFGTNm/M/+CwWRJAkffPABunbtavJM9+/fR7NmzdC4cWPs27cPVlZl+71k586d6NmzJwBg3759ZX6IyMvLQ4cOHXDz5k2cPn0aVapUKVM/RBWZVqtF48aNCzxRS24PHz7EokWLoNFoSj17qSwSExOxdOlS5OXlGd1XTEwM+vfvj/Xr18uQ7L/27duHLl264LvvvsPbb78ta99EFZUkSQlCCJ8CP2NRgsg4aWlpcHNzg0ajwdKlS0t8XXp6Ot5++22cPn3a6AyZmZm4fPky9Ho9+vfvb3R/T3rzzTdx6NAhXL16tVTrKYODg7Fy5UrExcWVuJhBVNmcPHkSXl5esLGxwcmTJ/HCCy+YdLz58+dj6tSpaNKkSZH/PicnJ0OSJCQlJZVp5kJpaDQafPrpp0hISICXl5dRfU2bNg329vaYPn26Uf3s2rULPXr0wP/93/9h4MCBRvVFVNHkPxC3a9cOCQkJJh9vzJgxiIqKwuuvv45NmzaZdKyMjAy0bt0aycnJqFWrltH92djYQKvV4tVXX5Uh3X/l5eXB3d0dffr0wbp162Ttm6iiKqooASGERby8vb0FkRI2bdokAIjffvtNsQxZWVmiRYsWomHDhiI9PV22fjMyMkSVKlXERx99VOprU1JShLu7u2jfvr3Izc2VLRORpTAYDMLPz0/UqFFDVK1aVQQEBAiDwWCy8S5cuCAcHR3Fm2++WWzbY8eOCWtra/HBBx+YLI8QQsTFxQlJkkRwcLBJxymt3Nxc4ebmJgYPHqx0FCJZ5eTkiBdffFEAEADElStXTDreH3/8IaysrET9+vUFALFlyxaTjjdz5kwBQMTGxpp0HDm89957onr16iI7O1vpKETlAkC8KORZnntKEBnJVOsNS8POzg5arRYXLlxAeHi4bP3u3r0bjx49KtEGnk+rVq0alixZgri4OHzxxReyZSKyFOvXr8fevXuxcOFCzJo1C1u2bIFOpzPZeCEhIbCyskJkZGSxbdu0aYPg4GB8/vnnOHLkiEny5OXlQaPRwN3dHbNnzzbJGGVlbW2NgIAAxMTEmNXxzUTFWbFiBY4fP465c+cC+O+eWKZgMBig0Wjg6uqKw4cPo3nz5hg9ejQyMjJMMt65c+ewYMECvPPOO+jRo4dJxpCTSqVCSkoK9u/fr3QUIuUVVq0wtxdnSpAScnNzRc2aNcW7776rdBQhhBCDBw8WdnZ24vTp07L0FxgYKJycnERGRkaZrjcYDKJr166iRo0a4vbt27JkIrIE9+/fF+7u7sLX11fk5eWJ7Oxs0bp1a1G/fn2RlpYm+3g6nU4AEIsWLSrxNampqcLDw0O0a9fOJLOdVq1aJQCI9evXy963HH788UcBQOzevVvpKESyuHr1qqhataro16+fMBgMolGjRqJv374mG2/16tUCgFizZo0QQohdu3YJAGLmzJmyj2UwGESfPn2Es7OzuHbtmuz9m8KDBw+EnZ2d+Pjjj5WOQlQuUMRMCcWLCXK9WJQgJezfv18AEN99953SUYQQQty4cUO4uLgIf39/o6eBGwwG4enpKV5//XWj+jl58qSwsbERI0aMMKofIksSFBQkrKysREJCwuP39u7dKwCIyZMnyzrWo0ePRIMGDUTLli1LPU3422+/FQDEypUrZc10+/ZtUaNGDdGtWzeTLlkxxsOHD4WdnZ0YO3as0lGIZPH2228Le3t7cfbsWSGEEKGhocLe3l48fPhQ9rHu3r0r3NzcxCuvvCLy8vIevz9o0CBhb28vzpw5I+t4+UXEpUuXytqvqfXp00c0bty4wv4dJJITixJEJjJp0iRhY2MjUlJSlI7y2PLlywUAsXHjRqP6SUxMFADE6tWrjc40YcIEAUDs37/f6L6IzF1CQoKwsrISQUFBz3z23nvvCVtbW/H333/LNt60adPK/Iu/wWAQPXr0ENWqVRM3b96ULdO///1vYWNjI06ePClbn6bABwayFLGxsQKACAsLe/xe/syFTZs2yT7ehx9+KKytrcWxY8f+5/3r168LZ2dn0adPH9n+vUpLSxN169YVbdq0ETk5ObL0WV6io6MFAFn/5hNVVCxKEJlIy5YtRY8ePZSO8T9ycnKEl5eXqFOnjlG/foSFhQlJksStW7eMzvTw4UPh6elpljcMRHLKy8sTvr6+wt3dXdy/f/+Zz2/duiWqV68uevToIcsN++nTp4WdnZ0YMmRImfs4deqUsLW1FcOGDTM6jxBC7Nu3TwAQEydOlKU/U+IDA1mCzMxM0axZM9GoUaP/WY6ZnZ0tqlWrJoYPHy7reEeOHBGSJIkxY8YU+HlUVJQAIH788UdZxps4caIAIPbt2ydLf+Xp8uXLAoBYuHCh0lGITI5FCSITOHv2rAAgoqKilI7yjAMHDggAYvz48WXuw9vbW3Ts2FG2TD/88EOF/e+LqLx8/vnnAoBYu3ZtoW3yH4SNXRZmMBiEv7+/cHFxETdu3DCqr8mTJwsAYs+ePUb1k5OTI9q0aSPq1q1rkr0z5Jb/wFCavTiIKpr58+cLAOLXX3995rNBgwaJmjVryrZvTG5urvD29hYeHh4iNTW1wDY5OTmibdu2om7dukYvHclfIip3YaU8eXl5iVdeeUXpGEQmx6IEkQlERkYKAOLcuXNKRynQ+++/L2xsbMSJEydKfe3Vq1cFABEeHi5bnic3obp+/bps/RKZizt37ojnnntO+Pn5FTkLoiQ39SXx/fffCwBi+fLlZe4jX1pamqhXr55o1aqVUcfX5f/dlOsX0vLw0ksvic6dOysdg6hM8o8CfuONNwr8PH/fGLmWV+YXVb/99tsi2+XvyTVhwoQyj5W/mXb16tVlmdWplBkzZggrKytuCE4Wj0UJIhPo1q2baN26tdIxClXSB6CCfPLJJwKA7Ou9z5w5I+zt7SvMaSVE5WnkyJHCxsZGHD9+vNi2hw8fLnL6c3EePHgg6tSpI1566SXZlkz9/PPPAoBYvHhxma6/du2acHZ2Fn379jWrPRryHxju3LmjdBSiUvvXv/4lqlSpIi5fvlzg5/fv3xc2NjayLKcq7fKzESNGGLW3zPr16wUAsWrVqjJdX1HEx8cLAOKrr75SOgqRSbEoQSSze/fuCWtra9l3yZfbZ599JgCIb775plTX9evXT7zwwgsmeXCYMWOGACB27dole99EFdXBgwcFADFu3LgSX1PYRnElMW7cOAFAHDx4sNTXFsZgMIiAgABRtWpVceXKlVJfn7/rfv7O/+Yi/4Hh66+/VjoKUano9foS7VfQvXt30bJlS6PHy9+o99SpUyVqn38KT9euXUt9v5GSkiJq1aolfHx8THJkcXkyGAyidu3ahc5mIbIULEoQyWzDhg2y3/CbQv6mes8//3yBm+oVJC0tTdjb24vQ0FCTZEpPTxcNGzYUzZs3F1lZWSYZg6giKevms4UdqVec48ePC2trazFy5MiyxC3SuXPnhIODgxgwYECprsvf+X/mzJmyZzK1/AeGN998U+koRCWW/13bokWLYr9r8zeeNKZgWNYjjfNnZq5bt65U1wUHBwtJkkRcXFyprquoPvzwQ1GlSpX/2YiUyNKwKEEks3feeUc8//zzpXpQUEpRxw8W5KeffjL5TIYtW7YIAGLBggUmG4OoojDmmN7Vq1cLAGLNmjUlam8wGISfn59wdXUVycnJpR6vJGbPni0AiG3btpWofVZWlmjevLlo1KiRSE9PN0kmU/vwww9F1apVRWZmptJRiEpk+vTpAoD47bffim177tw5AUBERkaWaazs7GzRunVrUb9+ffHo0aNSXZubmys6dOgg3N3dS3y8+tGjR4WVlZUIDAwsS9wKKf++qKDNSIksBYsSRDLKP0JrxIgRSkcpsaCgIGFlZSUSEhKKbfvvf/9bVKtWzajN7EritddeE05OTuLSpUsmHYdISTdu3BAuLi7C39+/TMuh8vLyRKdOnUTNmjXFvXv3im2/du1aAUB8/vnnZYlbIhkZGaJx48aiSZMmJXpIDw8PFwBETEyMyTKZWv4Dw9atW5WOQlSs/KOABw8eXOJrWrVqJbp161am8ZYsWSIAiJ9//rlM18fHxwtJkkRwcHCxbfPy8kTHjh1L/DfRXGRkZAgnJyeLKrQQPa2oooT0z+fmz8fHR8THxysdgyqw33//HQ0bNoSnp6dR/ezatQs9evTAzz//jH/9618ypTOtlJQUNG/eHB4eHhg6dGiRbcPDw9GrVy9s2LDBpJkuXbqEFi1awNfXFyqVyuj+rKys8M4776BWrVoypCOSx5AhQ7Bx40acOHECTZo0KVMfx44dQ7t27dCvXz9069atyLYLFy7ECy+8gP3798PKyqpM45XEtm3b0KdPHwwePBjt2rUrtF1eXh7CwsLQu3dvbNq0yWR5TC0zMxOurq4YPnw4oqOjlY5DVCghBPr06YNDhw4hKSmpxN+JU6ZMwaJFi3Dnzh3UqFGjxONdu3YNzZs3x6uvvgq9Xg9JksqUe9SoUfjkk08wa9YsODk5Fdru3Llz0Gq1WLNmDYYPH16msSqq119/HfHx8bh8+XKZ/3skqsgkSUoQQvgU+GFh1Qpze3GmBBXl2LFjwtraWrRp08aonejzj7V0cXERaWlpMiY0ve+//17Y2toKAEW+JEkSOp2uXDItX75cSJJUbKaSvkaNGlUuuYlK4rfffhMAxLRp04zuK38qdnGvatWqicTERBnSF2/EiBElylSrVi2LmBH12muvibp165rVySFU+WzcuFEAEMuWLSvVdQcOHBAAxIYNG0p13cCBA4WDg4PRx6Pfu3dPNG7cuER/U/r27WsWy2dLa926dSU6TpXIXIEzJagyMxgM8PPzQ0JCAjIzMxEZGYnQ0NAy9bVp0ya8+eabWLZsGYKDg2VOanrp6enIzc0tso2NjU2Rv1LI7dGjR8jLyzO6n8GDB+PPP//ExYsX+QsDKS4nJwdeXl7IyMjAyZMn4ejoaHSfaWlpMBgMRbaxt7eHvb290WOV1IMHD4pt4+DgADs7u3JIY1pr1qzBiBEjcPToUXh5eSkdh+gZaWlpaN68OWrWrIm4uDjY2NiU+Nq8vDzUrl0b3bt3x7fffluia2JjY9GrVy/Mnj0b06dPL2vsx3Jzc5Genl5sO2dnZ4v8ns/Ly4Ovry+uX7+OU6dOwcXFRelIRLIqaqZEyf9aEZmptWvXYv/+/fjyyy/xww8/YMaMGRg4cCBq165dqn7S0tIQEhICLy8vaDQaE6U1rfIsNpRUlSpVZOnn9ddfx+bNm/Hnn3+ibdu2svRJVFZRUVH466+/oNfrZSlIAEDVqlVl6UdOlemmOSAgAJIkQafTsShBFdLs2bNx7do1bNy4sVQFCQCwtrZGQEAANm3ahJycHNja2hbZPisrC6NGjULjxo0xfvx4Y2I/ZmNjU6n+pjzN2toaq1atgq+vL2bOnInIyEilIxGVG9MtOCWqAO7du4fx48ejU6dOGD58OFasWIHs7GyMHTu21H3NmTMHV69ehVarLfWXPZnekw8MREq6cuUKwsLCoFar0b9/f6XjkEyef/55vPzyy/wbQxXSyZMnERkZiffffx8dO3YsUx9qtRqpqan4/fffi227ePFinD59GitXroSDg0OZxqNntW/fHh988AFWrFiBP//8U+k4ROWGRQmyaFOnTsW9e/eg1WphZWWFRo0aYdKkSfjuu++wc+fOEvfz119/YenSpRgxYkSZv+zJtNzd3eHr6wu9Xq90FKrkxowZAyEEli1bpnQUkplarUZCQgKuXbumdBSix4QQ0Gg0cHFxwYIFC8rcT69evWBvb19s4e3ixYuYN28e3nzzTfTu3bvM41HB5s+fjxo1akCj0RS7ZI/IUrAoQRYrLi4On376KUaPHv0/0/knTpyIF154AaNGjUJ2dnax/Tz5Zb9w4UJTRiYjqVQqxMXF4fr160pHoUpq69at+PHHHzFt2jQ0aNBA6Tgks/yTgjZv3qxwEqL/Wr9+Pfbu3YsFCxbAzc2tzP1UqVIFPXr0gE6nQ1F7zoWEhMDKyorLC0zkueeew6JFi7B//36sXbtW6ThE5YJFCbJIeXl50Gg0qFWrFmbPnv0/nzk6OmLlypVISkrCkiVLiu1rw4YN2LNnD8LDw436sifTU6vVAPjAQMrIzMxEUFAQmjZtWqYlYlTxtWzZEi+88AJnZFGFkZKSgrFjx8LX1xfvv/++0f2p1WpcuHABf/31V4Gf6/V66HQ6zJw5E3Xr1jV6PCrYe++9h06dOmH8+PG4d++e0oy0HywAACAASURBVHGITI5FCbJIn332GeLj47FkyZICN03q27cvXn/9dcyZMweXLl0qtJ/U1FSMHTsWHTp0wMiRI00ZmWTQqlUrNGzYkA8MpIhFixbh3LlziI6OLtcTMKj8SJIEtVqN2NhYPHr0SOk4RJg+fTqSk5MfL1M1Vv4+OAUt4UhPT0dwcDBatmxZ5lPMqGSsrKyg1Wpx7949TJ06Vek4RCbHogRZnNu3b2PKlCno1q0b3nnnnULbRUVFQZKkIr9Yp0+fjtu3b8v2ZU+mJUkSVCoVHxio3J0/fx7h4eEYOHAgevbsqXQcMiGVSoWsrCzs2LFD6ShUySUmJkKr1SIwMBDt2rWTpc86derA29u7wOJ+eHg4Ll68iOjo6GJP5yDjtW3bFqNHj8ann36KuLg4peMQmRSfssjiTJw4EY8ePUJ0dHSR51jXq1cPM2bMwM8//4wtW7Y88/nRo0cRHR0NjUYDb29vU0YmGanVamRmZiI2NlbpKFRJCCEwevRo2NjYYOnSpUrHIRPr0qULqlWrxhlZpCiDwQCNRgM3NzfMnTtX1r7VajUOHTqEW7duPX7v9OnTWLRoEYYMGYKuXbvKOh4Vbvbs2ahVqxYCAwORl5endBwik2FRgizKvn378NVXX2Hs2LFo0aJFse3HjBmDFi1aYPTo0cjIyHj8vim/7Mm0/Pz8+MBA5eqXX35BTEwMZs2ahTp16igdh0zM1tYW/fr1g16v50MCKebLL7/E4cOHsXjxYlSvXl3WvtVqNYQQj3+wEUIgKCgIDg4OiIiIkHUsKpqLiwuWLFmChIQEfPbZZ0rHITIZFiXIYuTm5kKj0aBevXqYNm1aia6xs7NDdHQ0Lly48D/HaK1evRqHDh1CRESE7F/2ZFq2trbo06cP9Ho9j9Iik3v06BFCQkLQunVrjB49Wuk4VE5UKhXu3LmDI0eOKB2FKqHk5GRMmjQJfn5+GDJkiOz9t23bFnXr1n1c3P/hhx+wY8cOzJ07F7Vq1ZJ9PCraO++8g27dumHKlCm4ffu20nGITIJFCbIYK1aswPHjxxEVFYUqVaqU+Lpu3bph0KBBWLhwIc6cOYO7d+9i0qRJ6NKlC4YOHWrCxGQqarUat2/f5gMDmdy8efNw+fJlaLVarrGuRPr06QMbGxvOyCJFTJo0CampqcUuUy2r/P2Ztm/fjjt37mDMmDHw8vJCYGCg7GNR8SRJQnR0NB49eoSJEycqHYfIJFiUIItw/fp1zJw5E/369cNrr71W6uuXLFkCOzs7jB49GpMmTUJKSgq0Wq1JvuzJ9Pr27Qtra2s+MJBJnTp1CosXL8Z7772HLl26KB2HylGNGjXQpUuXAk8oIDKlgwcP4ssvv8SYMWPQunVrk42jUqmQnp6OgIAAXLt2DatWrYKNjY3JxqOitWjRAmPHjsVXX32Fffv2KR2HSHaSEELpDLLw8fER8fHxSscghQwaNAg//fQTTp48iUaNGpWpj2XLlj0+iWPs2LFYvHixnBGpnHXr1g3Jyck4fvy40lHIzGRnZyM+Ph65ublFtps5cyb++OMPJCUl4fnnny+ndFRRREVFYcyYMdi0aRNcXV0LbSdJEjp06MBjYsloubm5aN++Pe7cuYNTp06hatWqJhsrKysLbm5uSEtLw8iRI/H555+bbCwqmUePHqFly5aoVq0aVq5cWWRbSZLg7e0NJyenckpHVDxJkhKEED4FfsaiBJm7nTt3omfPnggLC8PMmTPL3E/+l31ycjL++usvODs7y5iSyltkZCQ+/vhjnD9/Hg0bNlQ6DpmRiIgITJgwoURt84/jo8rn/PnzaNy4MUpyH9WtWzfs3LmTs+/IKMuXL0dISAi+//57DBgwwOTjvfPOO4iNjcWpU6fg5uZm8vGoeD///DNef/31ErUNDQ1FZGSkiRMRlRyLEmSxsrKy0LZtW+Tm5uLEiRNwcHAwqr+0tDRkZWUV+asXmYdz586hcePGWLZsGYKDg5WOQ2akY8eOSE9PL/Zmrlq1ajwuuJL7888/kZycXGSbvXv3YtasWVi3bh0GDx5cTsnI0ty4cQPNmzfHyy+/jK1bt5ZLgSs1NRVpaWk8VaiCOXHiRLEbXs6dOxfnz5/HhQsXWAylCoNFCbJY4eHhmDJlCn799Vf06dNH6ThUwbRs2RK1a9dGbGys0lHITNy8eRO1a9fG7NmzS3yKD1FR8vLy0LFjR1y+fBlJSUmoVq2a0pHIDA0ZMgQbN27EiRMn0KRJE6XjUAX35ZdfYuTIkTh27BjatGmjdBwiAEUXJbjRJZmtS5cuYc6cOXjjjTdYkKACqdVq7NmzB6mpqUpHITOxZcsWCCGgUqmUjkIWwtraGlqtFrdv38aMGTOUjkNm6LfffsP69esxYcIEFiSoRAICAgCAm/GS2WBRgsxWSEgIJElCVFSU0lGoglKr1cjNzcXWrVuVjkJmQq/Xo169evxliWTl4+ODwMBArFy5EkePHlU6DpmR7OxsjBo1Cg0aNMDkyZOVjkNmolatWvD19eUpZGQ2WJQgs7Rlyxb88ssvmDlzJurWrat0HKqgfH194ebmxl8KqEQyMjKwfft2qNVqrsEl2c2dOxeurq7QaDQwGAxKxyEzERUVhb///hsrVqzgSQpUKmq1GkeOHMGNGzeUjkJULBYlyOxkZGRg9OjRaNGixeMjPIkKYm1tjf79+yMmJgY5OTlKx6EKbufOncjIyODSDTKJGjVqICIiAocOHcKaNWuUjkNm4MqVK5g1axbUajX69++vdBwyM/nfZZs3b1Y4CVHxWJQgs7NgwQJcuHABWq0WdnZ2SsehCk6lUiElJQX79+9XOgpVcHq9Hs7Oznj11VeVjkIWatiwYejSpQsmTpyIu3fvKh2HKrjQ0FAIIbBs2TKlo5AZat26NRo0aMAlHGQWWJQgs3LmzBksWLAA7777Lrp27ap0HDID/v7+sLOz4xIOKpLBYIBer0efPn1gb2+vdByyUJIkITo6GikpKdwfgIq0detWbNq0CVOnTkWDBg2UjkNmSJIkqNVq7NixA+np6UrHISoSixJkNoQQGD16NBwcHLB48WKl45CZqFq1Knr06AGdTgdLOQKZ5JeYmIgbN25w6QaZ3IsvvoiQkBB88cUXOHz4sNJxqALKzMxEUFAQmjZtinHjxikdh8yYSqVCZmYmj0anCo9FCTIbmzZtwrZt2zBnzhx4eHgoHYfMiEqlwrlz53Dq1Cmlo1AFpdPpYGVlhX79+ikdhSqBsLAweHh4IDAwEHl5eUrHoQpm4cKFOHfuHKKjozlzi4zi5+cHFxcXLuGgCo9FCTILaWlpCAkJgZeXFzQajdJxyMzk//rNJRxUGJ1Oh86dO8PV1VXpKFQJODs7IzIyEkePHsWqVauUjkMVyLlz5xAeHo6BAweiZ8+eSschM2dnZ4e+fftCr9fz1B+q0FiUILMwe/ZsXLt2DVqtFjY2NkrHITPj6emJdu3a8ZcCKtDly5dx7NgxLt2gcjVgwAD07NkT06ZNw61bt5SOQxWAEALBwcGwtbXF0qVLlY5DFkKlUuHWrVuIi4tTOgpRoViUoArv5MmTiIyMxPvvv4+OHTsqHYfMlEqlwoEDB3Dnzh2lo1AFk1+sUqvVCiehykSSJKxcuRLp6ekYP3680nGoAvjll18QExODWbNmoU6dOkrHIQvRt29fWFtb84cZqtAkS9n4zcfHR8THxysdo1LIy8tDdnZ2se0cHBwgSZJRYwkh0K1bNxw/fhxJSUlwc3Mzqj+qvBITE+Ht7Y3PPvsMQ4YMKbKtvb09rKxYs61IhBBG/z0pTO/evXHp0iXuOUKKmDZtGubNm4fdu3fzOFoLJYRAZmZmkW0yMjLw0ksvwcXFBYmJibC1tS2ndFQZdOvWDXfv3sWff/6pdBSSwa1bt+Du7q50jFKTJClBCOFT0Ge866ZSuXnzJho1agQnJ6diX8OHDzd6vPXr12PPnj0IDw9nQYKM8tJLL8HT0xMffPBBsf/bbdu2LdLS0pSOTP9x9uxZPP/889i8ebPsfT948AC//fYbl26QYqZMmYL69evzlAULNmjQoGK/d1xdXXH58mVotVoWJEh2KpUKx48fx8WLF5WOQkZ68OAB6tWrh0WLFikdRVZcnE+lMn78eNy4cQNz5swp8kvzxIkTWLt2LQYMGID+/fuXaayUlBSMGzcOHTp0wMiRI8samQjAP1Olv/vuO+zbt6/IdmlpaZg7dy7mzJmDhQsXllM6KowQAkFBQUhOTsY333xT5r8nhdm+fTtycnK4dIMU4+TkhNGjR2PcuHG4dOkS6tevr3QkktHDhw/x008/wd/fH927dy+ybevWrdGlS5dySkaViUqlwtixY6HX6zF69Gil45ARtm/fjuzsbItb0s7lG1Rie/bsQdeuXTFt2jTMmTOnyLbZ2dnw8vJCZmYmTp48CUdHx1KPFxwcjOjoaBw5cgTe3t5ljU1Uau+//z7Wrl2LY8eOoWXLlkrHqdR+/PFHvPXWW3B3d0dGRgbu3LkDOzs72fofNmwYtmzZglu3bnETXVLMmTNn0LRpU6xYsQJBQUFKxyEZ5f8N27NnD/z8/JSOQ5VYixYt4OnpiR07digdhYwwbNgwxMTE4ObNm2Z338LlG2S0nJwcaDQaNGjQAJMnTy62vZ2dHbRaLS5cuIDw8PBSj3f06FFER0cjMDCQBQkqdwsWLICzszNGjRoFSyncmqO0tDSEhobCy8sLWq0WDx48wN69e2XrPzc3FzExMQgICDC7L3ayLE2aNEHz5s15bLEF0ul0qFGjBjp16qR0FKrk1Go19uzZg9TUVKWjUBnl37f069fP4u5bWJSgEomKisJff/2F5cuXw8nJqUTXdO3aFe+++y4WLlyIM2fOlHgsg8GAwMBAuLm5Ye7cuWWNTFRmNWvWRHh4OHbv3o0NGzYoHafSmj17Nq5evQqtVos+ffrAwcFB1t3DDx48iLt373LpBlUIKpUKu3fvxoMHD5SOQjLJy8vDli1bWPikCkGtViMnJwfbtm1TOgqVkSXft7AoQcW6evUqZs2aBbVaXerN4BYvXgwHBwcEBQWV+Bfn1atX4/Dhw1i8eDGqV69elshERhs5ciTat2+PsWPH8lcFBTx9FLCTkxN69uwJnU4n2+wVnU4HW1tb+Pv7y9IfkTH4wGB5LPkBgszPyy+/DDc3N87IMmM6nQ52dnbo3bu30lFkx6IEFWvMmDEwGAxYtmxZqa/18PDAnDlzsH37dvz444/Ftk9OTsbEiRPh5+dX7LGNRKZkbW2NVatW4fbt25gxY4bScSoVIQRGjRoFFxcXLFiw4PH7arUaFy9exIkTJ2QZR6/Xo1u3bnBxcZGlPyJjdOzYEa6urnxgsCD5hU9LfIAg82NtbY2AgADExMQgNzdX6ThUBnq9Hl27doWzs7PSUWTHogQVadu2bfjhhx8wdepUNGjQoEx9aDQaeHl5ITQ0FA8fPiyy7eTJk5Gamoro6GhIklSm8Yjk4u3tjcDAQKxcuRJHjx5VOk6lUdhRwPknb8ixhOP06dNISkriL5hUYeQ/MGzZsoUPDBZCp9Oha9euLHxShaFWq3H//n3s379f6ShUSklJSRZ938KiBBUqMzMTQUFBaNq0qVHnp9vY2ECr1eLatWuYPXt2oe0OHTqEL774AqGhoWjdunWZxyOS09y5c+Hq6gqNRgODwaB0HItX1FHAHh4eaN++vSy/JOcXNuQ+YpTIGPkPDAcOHFA6ChmJhU+qiPz9/WFnZ8cZWWbI0u9bWJSgQkVERODs2bOIjo6Gvb29UX117NgR77//PqKionDy5MlnPs/Ly4NGo0GdOnUwc+ZMo8YiklONGjUQERGBQ4cOYc2aNUrHsXgzZszA7du3odVqYWX17FeUWq3G4cOHcfPmTaPG0el0aNu2LerXr29UP0Ry4gOD5ch/gCjtXlxEplS1alV0795d1v2ZqHzo9XqLvm9hUYIKdP78ecyfPx8DBw5Ez549ZelzwYIFcHFxgUajeeYP4apVq3D06FFERkZa5DopMm/Dhg1Dly5dMHHiRNy9e1fpOBYrMTER0dHR0Gg0hR4FnP+r45YtW8o8zt27d7Fv3z7+gkkVjrOzM7p168aihAXQ6/Vo06aNxT5AkPlSq9U4e/YskpKSlI5CJVQZ7ltYlKBnCCEQHBwMGxsbLF26VLZ+3dzcEB4ejr1792LdunWP37958yamTp2KXr164a233pJtPCK5SJKE6OhopKSkYPLkyUrHsUgGgwEajabYo4BffPFF1KtXz6iHtl9//RUGg4G/YFKFpFarcebMGT4wmLHK8ABB5it/+j+Ln+YjJibG4u9byq0oIUnSZEmS4iRJeiBJ0h1JkvSSJLV+qo0kSVKYJEnXJUnKkCRptyRJrcorI/1Dp9Nhy5YtmDVrFurUqSNr3yNHjkSHDh0wbtw4pKSkAADGjx+PzMxMrFy5kptbUoX14osvIiQkBF988QUOHz6sdByLk38UcERERJFHAUuSBLVajR07diAjI6NMY+l0Onh4eBQ6G4NISXxgMH+//vor8vLyLPoBgsxX3bp18dJLL8myaTSVD71eb/H3LeU5U6IrAC2ATgC6A8gFECtJ0nNPtJkAYCyA0QDaA7gNYIckSZzPX04ePXqEkJAQtG7dGqNHj5a9fysrK6xatQrJycmYPn069uzZg3Xr1mHChAlo2rSp7OMRySksLAweHh4IDAxEXl6e0nEsRv5RwF26dMHQoUOLba9Wq5GRkYGdO3eWeqzs7Gxs3boVKpWqwD0riJRWr149eHl5sShhxvR6PWrVqgUfHx+loxAVSK1W48CBA7hz547SUagYWVlZ2Lp1K/r372/R9y3l9k8mhOgthFgjhDghhDgOYCiAmgBeAf6ZJQEgFMACIcSPQogTAN4D4Azg3fLKWdnNmzcPly5dglarha2trUnGaNeuHQIDA6HVavHee++hQYMGnBJPZsHZ2RlLly7F0aNHsWrVKqXjWIz8o4C1Wm2JZku9+uqrcHZ2LtND2549e/Dw4UP+gkkVWv4DQ3JystJRqJSys7Px66+/WvwDBJk3lUoFg8GAmJgYpaNQMfLvWyx9OZik1M6rkiR5ALgOoIsQYp8kSS8AOAeggxAi7ol2WwAkCyHeK6o/Hx8fER8fb9LM5mzbtm2YNGkScnJyimyXlJSEwYMH46uvvjJpnpSUFDRr1gy3b9+GXq+32ONtyPIIIeDv74+4uDhcvHixyKUGVLzExER4e3tj7NixWLx4cYmvGzhwIPbt24erV6+W6sb/ww8/xDfffIO7d+/C0dGxLJGJTC4+Ph7t27fH119/jWHDhikdh0phx44d8Pf3h06nY/GTKiwhBDw9PZGZmQkPDw+j+3N2dsZPP/2EWrVqyZCOnjR69Gh8+eWXFnHfIklSghCiwClkNuUd5gnLAPwB4OB//v/8/xXfeqrdLQAFbmwgSdIHAD4A/pnuSAVLTU3F8OHD4eDgUOxapE6dOmH+/Pkmz1S9enVs3LgRcXFxLEiQWZEkCVOmTEH37t1x8OBB9O3bV+lIZm39+vWws7PD9OnTS3WdWq3Gxo0bkZCQgPbt25fompMnT2L16tX497//bfZf7GTZ2rVrh9q1a0On07EoYWb0ej0cHR3Ro0cPpaMQFUqSJCxevBg//vij0X3l5ubil19+wcaNG02y9LsyE0JAp9OhV69eFn/fokhRQpKkpQA6A+gshCjzwmwhxGcAPgP+mSkhUzyLM2PGDNy6dQtHjhypUOsb/fz84Ofnp3QMolLz9vaGJEmIj49nUcII+V+23bp1Q7Vq1Up1bd++fWFlZQWdTleiooQQAhqNBi4uLuVSeCUyhpWVFVQqFdavX4+srCzY29srHYlKIP9vWs+ePeHk5KR0HKIiDRo0CIMGDZKlr+bNm0Ov17MoIbPjx4/j8uXLmDFjhtJRTK7cF7tJkhQJYBCA7kKI8098dPM//9f9qUvcn/iMSumPP/7AypUr8dFHH1WoggSROXNxcUGzZs3AJWPGSUpKwtmzZ8u0TtLV1RWdO3cu8e7h69evx969exEeHg43N7dSj0dU3lQqFdLS0rB7926lo1AJHT9+HJcuXbL4td9ET1OpVNi9ezcePHigdBSLkr93VkBAgMJJTK9cixKSJC3DfwsSp576+AL+KT70eqK9A4AuAA6UW0gLYjAYoNFo4Orqinnz5ikdh8iieHt7syhhpPyCQlmXcKnVahw7dgyXLl0qsl1KSgrGjh2LDh06YOTIkWUai6i8de/eHU5OTjyFw4wY+zeNyFyp1Wrk5ORg27ZtSkexKDqdDr6+vpVir45yK0pIkhQN4N/45ySN+5Ik1frPqyoAiH923IwCMFGSpDckSWoN4CsAaQA2lFdOS/LVV1/h4MGDiIiIQI0aNZSOQ2RRfHx8cP36ddy4cUPpKGZLp9PBy8urzHsC5W8iV9xsienTpyM5ORmrVq3ibvhkNhwdHdGrVy/o9XootSk5lY5Op0OHDh0qxQME0ZM6duwIV1dXFlFldOPGDcTFxVWamVfleXemwT/He+4EcOOJ17gn2iwCEAkgGkA8AA8A/kKIh+WY0yLcvXsXEyZMQOfOnblJFpEJ5C+HSkhIUDiJeUpOTsaBAweM2p2+adOmaNasWZFFicTERGi1WgQGBqJdu3ZlHotICWq1GleuXMGxY8eUjkLFuHHjBo4cOVJpHiCInmRjY4N+/fphy5YtyM3NVTqORdi8eTMAVJpTfMqtKCGEkAp5hT3RRgghwoQQHkIIByHEq0KIE+WV0ZJMmTIFKSkp0Gq1kCRJ6ThEFsfLywtWVlZcwlFGMTExMBgMRt/Aq9Vq/PbbbwWuY81fwubm5oa5c+caNQ6REgICAiBJEn99NANbtmwBABYlqNJSq9W4f/8+Dhzgqns56HQ6NGjQAK1bt1Y6SrngPFYLdOTIEXz++ecIDg7Giy++qHQcIotUtWpVNG/enEWJMtLpdKhdu7bRsxdUKhVycnKwffv2Zz778ssvcfjwYSxevBjVq1c3ahwiJbi7u8PX17fEG7qScnQ6HerXr19pHiCInta7d2/Y2dmxiCqD9PR0xMbGQq1WV5ofl1mUsDB5eXkIDAyEh4cHwsLClI5DZNF8fHyQkJDA9d6llJWVhW3btqF///5G7/FQ2DrW5ORkTJo0CX5+fhgyZIhRYxApSa1WIz4+HteuXVM6ChUiPT0dO3bsqFQPEERPc3Z2RteuXVlElUFsbCwyMzMrzdINgEUJi/PJJ58gMTERS5cuhYuLi9JxiCyaj48Pbt68ievXrysdxazs3r0baWlpskxzLmwd6+TJk5Gamoro6Gg+JJBZy//3JH99MVU8O3fuRGZmJpduUKWnVqtx+vRpJCUlKR3FrOl0Ori4uMDPz0/pKOWGRQkLcuvWLUydOhU9e/bEwIEDlY5DZPHyN7vkEo7S0el0cHJyQvfu3WXpT61W4969ezh48CAA4ODBg/jiiy8QGhrKqdRk9lq2bImGDRvy18cKrDI+QBAVJP84XC7hKDuDwYDNmzejT58+sLOzUzpOuWFRwoJMmDAB6enpWLlyJX8ZJCoHbdu25WaXpSSEgF6vR69eveDo6ChLn0+uY83NzYVGo0GdOnUwc+ZMWfonUpIkSVCr1YiNjcWjR4+UjkNPqawPEEQFqV+/Ptq2bcsiqhHi4uJw69atSjfzikUJC7F3716sXbsW48ePR7NmzZSOQ1QpODk5oVWrVjwWtBSOHTuGK1euyPplm7+OVafTYdWqVfjjjz8QGRkJZ2dn2cYgUpJarUZWVhZ27NihdBR6Snx8PG7evFnpHiCICqNWq7F//34kJycrHcUs6XQ6WFtbo2/fvkpHKVcsSliAnJwcaDQa1K9fH1OnTlU6DlGl4uPjg/j4eG52WUI6nQ6SJCEgIEDWfvPXsU6cOBG9evXCW2+9JWv/RErq0qULqlWrxl8fK6DK+gBBVBiVSgWDwYCYmBilo5glvV6Pzp0747nnnlM6SrmyUToAGW/lypU4efIkfv75Zzg5OSkdh6hS8fHxwZo1a3DlyhXUq1dP6TgVnl6vh6+vL9zd3WXtV6VSISgoCHl5eVzCRhbH1tYWffv2hV6vR25uLmxsePtmjAsXLmDBggXIysoyuq/t27dXygcIosJ4e3vDw8MDer0ew4YNk73//fv3IykpCSNGjJC9b6VdvHgRx48fx5IlS5SOUu74rWYBPvnkE7z66qucOkikgCc3u2RRomjXr19HfHw85s+fL3vf9erVw7Bhw9CuXTs0bdpU9v6JlPbuu+/iu+++g1arRXBwsNJxzJbBYMCQIUOQmJgoS3HUwcEBGo1GhmRElsHKygoqlQobNmxAVlYW7O3tZes7JSUFb7zxBm7fvo0mTZqgS5cusvVdEeTPhqtMR4HmY1HCzCUlJeH06dMIDg7mL4NECmjTpg1sbGyQkJCAN954Q+k4FVr+kYam+rL9+uuvTdIvUUXQv39/+Pv7Y/r06RgwYAA8PDyUjmSWvv76axw4cABr1qzB8OHDlY5DZJFUKhU+++wz7NmzB/7+/rL1O23aNCQnJ8Pd3R0ajQaJiYmwtbWVrX+l6XQ6NG/eHE2aNFE6SrnjnhJmLv/IncpYUSOqCBwcHNC6dWuewFECOp0ODRs2RKtWrZSOQmR2JEnCypUrkZmZifHjxysdxyzdu3cPEyZMwCuvvGKSaeVE9I8ePXrA0dFR1qNBExISoNVqMWrUKHz66ac4ceIEli9fLlv/SktNTcXu3bsr7cx3FiXMnF6vR9u2bTltnEhB3OyyeI8ePUJsbCzUajVndRGVUZMmTTBhwgSsX78ev/32m9JxzM6UKVNw//59aLVaWFnxFpjIVBwdHeHv7w+dTifLvZHBYIBGo8Hzzz+POXPmQK1WIyAgAGFhYbh27ZoMiZW3bds2ARrKiAAAIABJREFU5ObmVtofmvkX2YwlJydj//79lbaiRlRR+Pj44N69e7h48aLSUSqs2NhYZGVlVdovWyK5TJkyBQ0bNsSoUaOQnZ2tdByzceTIEXz22WcIDg5GmzZtlI5DZPFUKhWuXLmCP//80+i+vvjiCxw5cgSLFy9GtWrVIEkSli9fjtzcXHz88ccypFWeTqeDq6srOnbsqHQURbAoYcZiYmJgMBhYlCBSmLe3N4B/phZSwXQ6HapVqwY/Pz+loxCZNUdHRyxfvhx///03oqKilI5jFvLy8hAYGIhatWohLCxM6ThElUL//v0hSZLRSzju3LmDSZMm4dVXX8XgwYMfv//CCy9gypQp+P7777F9+3Zj4yoqNzcXMTEx6N+/P6ytrZWOowgWJcyYXq+Hh4cH2rVrp3QUokrtxRdfhK2tLfeVKITBYMDmzZvRt29fi9qQikgp/fv3h1qtxqxZs3DlyhWl41R4n376KRITE7F06VK4uLgoHYeoUnB3d4evr+/jEyXKatKkSXj48CG0Wu0zyz/Hjx+Pxo0bIygoSJYjfpWyf/9+3L9/v1LPJmVRwkxlZWVh69atUKlUXBdJpDB7e3u0adOGRYlCHDlyBLdv367UX7ZEclu2bBmEEAgNDVU6SoV269YtTJkyBT169MDbb7+tdByiSkWlUiEuLg7Xr18v0/UHDhzA6tWrMWbMGLRs2fKZzx0cHBAdHY0zZ85g8eLFxsZVjE6ng52dnawnlZgbPs2aqd27dyMtLY1LN4gqCB8fHyQkJHCzywLodDpYW1ujb9++SkchshgNGjTAtGnTsGnTJmzdulXpOBXWhAkTkJ6ejujoaG6yS1TO8p9T8o8EL43c3FxoNBp4enpixowZhbbz9/fHW2+9hblz5+LChQtlzqoUIQR0Oh26d+8OZ2dnpeMohkUJM6XX6+Ho6Iju3bsrHYWI8M++EikpKTh//rzSUSocvV4PPz8/1KhRQ+koRBZl7NixaNq0KYKCgpCZmal0nArn999/x9q1azFu3Dg0a9ZM6ThElU6rVq3QoEGDMi3hiI6OxrFjxxAVFYWqVasW2TYyMhLW1tYICQkpa1TFJCUl4ezZs5V+NimLEmYov6Lm7+8PR0dHpeMQEf6ZKQGASziecuHCBZw4caLSf9kSmYK9vT2io6Nx7tw5LFq0SOk4FUpOTg40Gg3q1auHqVOnKh2HqFKSJAlqtRqxsbFIT08v8XU3btzA9OnT0bt3b7zxxhvFtvf09ERYWBj0er3Re1iUt/yNQCv7fRKLEmbozz//xJUrVyr9/3iJKpJWrVrB3t6eRYmn5N8ccKkZkWn07NkTb7/9NubPn49z584pHafCWLFiBU6cOIHly5ejSpUqSschqrTUajUyMzMRGxtb4mvGjRuHrKwsrFixosTLrkJCQtCqVSsEBweXqgCiNJ1OBy8vL9StW1fpKIpiUcIM6XQ6SJKE/v37Kx2FiP7Dzs4Obdu2ZVHiKTqdDi1atECjRo2UjkJksZYsWQJbW1sEBwdzXxsA165dw8yZMxEQEMCCKJHCunTpAhcXlxIfDbpr1y5s2LABkyZNQpMmTUo8jq2tLaKjo3Hx4kXMnz+/rHHL1Z07d3Dw4P9n777DoyqzB45/76ROElIgBUgoSWgJCdIWAakSENSg/hDdtexaYelFQV26NDGsiBiwoay46gprCUWQoIC6CtIJCTUhkAFSIL3PzP39MZmBkEkyNZPA+3meeSQztxxDyNw597zn/CZ+TyGSEk1SQkICd999N0FBQY4ORRCEm+ibXWq1WkeH0ijk5+ezd+9e8WYrCHYWHBzMokWL2L59O999952jw3G4mTNnolareeedd0RzS0FwMFdXV0aNGsXWrVvrvT6qqKhg0qRJhIaG8uqrr5p9rsGDB/P0008TFxfHmTNnLA25wWzfvh2tViuukwBnRwcgmOfy5cscPHiQpUuXOjoUoYkpV2v43/lrDO0c6OhQblu9evVi7dq1nDt3jk6dOtnsuGlpaeTn59O9e3ebHdNaJ0+eZM+ePXVuk5ycjFqtFm+2gtAApkyZwieffMK0adNQqVR1buvk5MTYsWNp0aKF3eMqLy9nz549DB8+vEFGmO/atYuvvvqKRYsWERYWZvfzCYJQv9GjR/Of//yHhQsX1nlT9dChQ5w6dYqtW7da3DcvLi6OhIQEJk+ezM6dOxt1YjIhIYHWrVvTs2dPR4fieLIs3xaPXr16yXeC999/XwbkEydOODoUoYn576FLcrtXtspnMwsdHcpt69ixYzIg//vf/7bZMYuLi+V27drJnp6e8qVLl2x2XGtkZmbKvr6+MlDvo3379rJarXZ0yIJwR/jll19kDw8Pk/5tDh48WNZqtXaP6aWXXpIBec2aNXY/V1lZmdyxY0e5Q4cOcmlpqd3PJwiCaa5fvy77+PiY9LvpySeftPp8b731lgzIhw8ftkH09lFaWip7enrK48ePd3QoDQY4KNfyWV5USjQxCQkJhIaG0rVrV0eHIjQxF67pmv5cvF5Mh8C6RysJlomMjMTd3Z2DBw/yxBNP2OSYy5YtIz09HRcXF2bOnMlXX31lk+Na45VXXqGoqIj9+/cTGhpa57be3t44OTk1UGSCcGe75557yMzMpLS0tM7tvvjiC6ZNm8bnn3/Ok08+abd4kpKSePvtt3F1dWXu3LmMHTvWrktPV65cydmzZ9mxYwfu7u52O48gCObx8/Pj8uXLFBcX17utv7+/1ed78skneemll9iyZQs9evSw+nj2sGfPHoqLi0U1aRVJvk0aIvXu3Vu+3RvMFRcX4+/vz7hx41i9erWjwxGamJe+OsZ/D2ew+KGuPN2vvaPDuW31798fZ2dn9u3bZ/Wxzpw5Q3R0NI899hgdO3ZkwYIF7Ny5kxEjRtggUsv8+uuvDBgwgFdeeYU33njDYXEIgmA5jUZDv379uHjxIqdPn8bHx8fm55BlmSFDhpCUlMSWLVsYMmQIf/7zn/n0009tfi6ACxcuEBkZyf3338/mzZvtcg5BEJqOe+65h/Ly8kbbgHzSpEls2LCBa9eu3TFJVEmSDsmy3NvYa6LRZROSmJhIWVmZyKgJFlHl6SolMvLqvoMmWKdXr14cOXIEjUZj1XFkWWbSpEkolUri4uKYPXs2HTp0YPLkyZSXl9soWvOo1WomTJhAmzZtmDdvnkNiEATBek5OTqxbt46srCzmz59vl3N89tln7Nu3jxUrVtC/f39mzZrFxo0b2bt3r13ON23aNBQKBatWrbLL8QVBaFpiY2M5dOhQvT12HEGWZRISEhgxYsQdk5Coj0hKNCEJCQn4+PgwaNAgR4ciNEGqqmSEKlckJeypd+/eFBUVWd31edOmTSQmJrJkyRJatmyJu7s77777LmfPniUuLs5G0ZpnzZo1nDhxgtWrV+Pp6emQGARBsI1evXoxYcIE3n33XY4cOWLTY+fl5fHyyy/Tt29fnnvuOQDmzJlDu3btmDRpEpWVlTY935YtW0hISGDBggW0adPGpscWBKFp0t/E3bp1q4Mjqeno0aNkZGSIG803EUmJJkKr1bJ161ZGjhyJi4uLo8MRmhiNVuZKXhlwIzkh2Efv3rqqNGvKBQsLC5kxYwY9evRgwoQJhufvu+8+xowZw9KlS0lLS7M6VnNcvnyZBQsWMGrUKB5++OEGPbcgCPaxZMkSWrRowcSJE206ynju3Lnk5OSwdu1aw8QNDw8PVq9ezcmTJ226BLWkpISpU6cSGRnJ9OnTbXZcQRCatoiICMLDw0lISHB0KDVs2bIFSZJ44IEHHB1KoyGSEk3EgQMHyMrKEhk1wSJZhWWotTLOCklUSthZly5d8PDwsCopsWjRIi5fvszatWtrNIlctWoVTk5OTJs2zdpQzfLSSy9RUVHBmjVrGvV4LUEQTOfn50dcXBy///47n3zyiU2OefjwYdatW8fEiRNrNJgbPXo0DzzwAAsXLiQjI8Mm51u+fDkXLlwgPj5e3LQRBMFAkiRiY2PZvXu3SQ02G1JCQgJ9+/YlMDDQ0aE0GiIp0UQkJCTg5OTEqFGjHB2K0ATpExHdQnzIKiynXG1dvwOhdk5OTvTs2ZNDhw5ZtL++W/2LL75I3759a7zepk0bFixYYChXbgi7d+/myy+/5LXXXiM8PLxBzikIQsP461//ysCBA3nllVe4du2aVcfSarVMmDCBgIAAFi9eXON1SZJ455130Gg0zJw506pzga4Z8JtvvslTTz3FkCFDrD6eIAi3l9GjR1NeXs6uXbscHYqBSqXi0KFDxMbGOjqURkUkJZqILVu2MHDgQPz8/BwditAE6Zds9AltAcDlqqUcgn3om12q1Wqz9pNlmYkTJ+Lr68vy5ctr3W769OlERkYydepUSkpKrA23TuXl5UyaNInw8HBeeeUVu55LEISGJ0kS8fHx5OXl8dprr1l1rI8++ogDBw6wcuVKfH19jW4TFhbGP/7xDzZt2sQPP/xg8blkWWby5Mm4u7s7rM+OIAiN24ABA/D19W1USzj0PS5E9Xt1IinRBKSlpZGUlCR+eAWLZeTqkxK6pJZYwmFfvXv3pqSkhFOnTpm138aNG/n555954403aNGiRa3bubi4EB8fT3p6OsuWLbM23Dq99dZbnD59mjVr1ogO0YJwm4qOjmbatGl89NFH7N+/36Jj5OTk8NprrzF48GCefPLJOredNWuW1dOENm/ezK5duwzNgAVBEG7l4uLCqFGj2Lp1q9VT0WwlISGBsLAwIiMjHR1KoyKSEk3Ali1bAESZj2AxVV4pfh4udAxsVvW1fe+u3+ksaXaZl5fHrFmzqnWrr8uQIUN46qmniIuLs3rSR23S09NZvHgx//d//yeWjgnCbW7hwoW0atWKiRMnWnTx/uqrr1JQUEB8fHy9fWfc3d2Jj4+3eJpQbc2ABUEQbjV69Giys7M5cOCAo0OhuLiY3bt3ExsbK/pz3UIkJZqAhIQEIiIi6NChg6NDEZqojNxSQvw8aOnjjkISlRL21qlTJ7y8vMzqK6HvVr9u3TpDt/r6xMXF4e7uzqRJk5Bl2dJwazV9+nQkSeLtt9+2+bEFQWhcmjVrxqpVqzh8+DDvvfeeWfv+9ttvrF+/nhkzZtC1a1eT9hkxYgSPPvqoRdOEXn/9dVQqFWvXrsXZ2dmsfQVBuLOMHDkSZ2fnRrGEY9euXZSXl4vqdyNEUqKRy8/PZ+/eveKHV7CKKreEYF8lLk4KgrzdyRBjQe1KoVDQs2dPkyslDh06xNq1a5k0aRLdu3c3+TwtW7ZkyZIlJCYmsmnTJkvDNWrbtm18++23zJ8/nzZt2tj02IIgNE5jx44lJiaGOXPmkJmZadI+arWaCRMmEBISwvz58806n36a0NSpU03eJykpiVWrVvHCCy8YbQYsCIJwM19fXwYNGmSoPHekhIQEfHx8GDhwoKNDaXREUqKR27FjB2q1WizdECwmyzKqvFKC/ZQAhPgpRaVEA+jduzdHjx6lsrKyzu20Wi0TJ04kMDDQaLf6+kyYMIHu3bszY8YMCgsLLQ23mtLSUqZMmUJERAQzZsywyTEFQWj8JEni3XffpaSkhFmzZpm0z9q1azl27BirVq3Cy8vLrPOFhISwYMECtm7datJdTFmWmTRpEj4+PnU2AxYEQbjZ6NGjOXnyJKmpqQ6LQaPRsHXrVkaNGiXGFxshat4auYSEBPz9/cXdAMFi14srKKvUEuyrS0oE+yo5mJ7r4Khuf71796asrIyvv/6a0NDQWrdLTEzkwIEDfPbZZ/j4+BjdpqRCTaVGxkdZ803M2dmZdevW0a9fPxYtWsTKlSutjv2NN94gLS2NH3/8EVdXV6uPJwhC09G5c2dmz57N0qVLue++++jYsWOt25aUlDBv3jzuu+8+xowZY9H5pk+fzoYNG5g6dSqBgYF1Ll/75Zdf2LdvHx9++CH+/v4WnU9ovK4VldPM3QVXZ3HPVLCt2NhYpk+fzpYtW5g2bZpZ+2ZnZ5u9xMyY06dPk52dLarfayHZYx2yI/Tu3Vs2p6lcU3D16lU6duzIo48+yieffOLocIQm6nhGHqPf/ZX3n+7FfV1bErfzFO/vTeX0klE4KUSTHXtJTU0lPDzcpG2HDBnCjz/+WGvTo5e+OkZqThHfTLyn1mO8+OKLfPLJJxw5coTo6GiLYgY4e/Ys0dHRjBkzhn//+98WH0cQhKarpKSEqKgoky7E3dzcOHHiRJ3Ji/rs27ePwYMHm7Rt3759+fXXX03uvSM0DbIs86eliTx5dztmDO/k6HCE21BUVBRBQUHs3r3b5H0yMjKIiooiPz/fJjG4urpy9epV/Pz8bHK8pkaSpEOyLPc29pqolGjEZs2aRUVFhdVzw4U7m36pxo1KCQ/UWpnMgjJaVz0n2F5YWBj79+8nJyenzu0kSWLw4MF1dmE+npFH+vUSNFq51kTS8uXL+frrr5k0aRJ79+61qKuzLMtMmTIFNzc3m1RcCILQNHl4eHDgwAGTutV37tzZ5ARsbQYNGsSJEye4ePGiSduKhMTtJ6eogpyiCvanXXN0KMJtavTo0cTFxZGXl4evr69J+8ycOZPy8nK++uorPD09rY6hTZs2d2xCoj4iKdFI7d27l88++4y5c+fSqZPIGAuWU1U1tQyp6imh7y2hyisVSQk769Onj9XH0Ghl0q+VUKHRcjmvlDbNPYxu5+/vz4oVK3jxxRfZuHEjf/3rX80+19dff83OnTtZvXo1rVq1sjZ0QRCaMH9/f+6///4GO19UVBRRUVENdj6hcdFfqySpCtBqZRSiklOwsdjYWJYvX86OHTv485//XO/2P/zwA5s2bWLRokWMHTu2ASK8s4lUcyNUWVnJxIkTad++vaiSEKyWkVuKp6uToR+BvmJCNLtsGlS5pVRotACczy6qc9vnnnuOvn37MmvWLHJzzesbUlRUxPTp0+nevTsTJ060OF5BEARBMJf+mqSoXE3atWIHRyPcjvr06UNgYKBJTXXLy8uZPHky4eHhzJ49uwGiE0RSohF6++23SU5O5p133sHDw/hdUUEwlX7yhr6c35CUEGNBm4TzOTcSEanZdV+oKRQK1q5dS05ODnPnzjXrPK+//joZGRmsXbsWZ2dRRCcIgiA0HFVeieHPSSrbrN8XhJs5OTnx4IMPsn379nono8XFxXH27Fneffdd3N3dGyjCO5tISjQyGRkZLFq0iNjYWDEGVLAJVW6pIREBoHR1ooWnKxmiUqJJ0CciXJ0VpObUXSkB0KNHDyZOnMi6des4dOiQSec4efIkq1at4rnnnqNfv35WxSsIgiAI5lJVVXW6uyg4niGSEoJ9xMbGkp+fzy+//FLrNmlpaSxdupQxY8YwcuTIBozuziaSEo3MjBkz0Gq1rF692tGhCLcJfaXEzYL9lKJSoolIzS7CR+lCRCvveisl9BYvXkxgYCATJ05Eq9XWua0sy0yaNAlvb29WrFhhi5AFQRAEwSyqqp5Jka28OSGSEoKdDB8+HDc3tzqXcEybNg0nJydWrVrVgJEJIinRiOzcuZPNmzczZ84cQkNDHR2OcBsoKleTX1pJsG/1ZUDBvkpUuSW17CU0JqnZxYQFeBLu72lyUsLX15eVK1dy4MABPvroozq3/fzzz9m7dy/Lly/H39/fFiELgiAIglkyqqo6o4N9SLqcj0YrOzok4Tbk6enJsGHD2LJlC7Jc82csISGBLVu2sGDBAtq0aeOACO9cIinRSJSVlTF58mQ6derEyy+/7OhwhNuEYRzorZUSvrpKCWO/kIXGJS2nmDB/L8ICPLlaUEZxudqk/Z588kkGDx7Mq6++SnZ2ttFt8vLyeOmll+jTpw8vvPCCLcMWBEEQBJPpqzqjQ3wpqdCQZsJyRUGwxOjRozl//jwpKSnVni8pKWHq1KlERkYyffp0B0V35xJJiUYiLi6Oc+fO8e677+Lm5ubocITbhL5xVLBvzeUbZZVarhVXOCIswUTF5WquFpQRFuBJWIAXoEtSmEKSJOLj4yksLOTVV181us38+fPJyspi7dq1KBTi7UAQBEFoeAVllRSWqQn2VdItxAdA9JUQ7ObBBx8EqLGEY9myZaSnpxMfH4+Li4sjQrujiavQRiA1NZVly5bx2GOPMXz4cEeHI9xG9JUSIUYqJW5+XWic9AmI8ABPwgI8AUg1MSkB0LVrV2bMmMHHH3/M//73v2qvHTlyhPj4eCZOnEivXr1sF7QgCIIgmOHmqs7wAC+ULk4iKSHYTXBwML169WLLli2G586cOUNcXBxPPvkkQ4YMcVxwdzCRlHAwWZaZOnUqzs7OvPXWW44OR7jNZOSV4uqkIMCrevWNfjmHaHbZuJ3P1pWvhvp70b6FJ5Kka3xpjvnz5xMSEsKECRNQq3VLP7RaLRMmTMDf358lS5bYPG5BEARBMJUhKeGrxEkh0bW1NyfEWFDBjkaPHs1vv/1GVlaWoeG3u7s7K1eudHRodyyRlHCwhIQEtm3bxqJFiwgODnZ0OMJtRpVbSmtfdxQKqdrzIVWNL0WlROOWml2MJEG7Fh64uzjR2kdpcrNLPS8vL1atWsXx48eJj48H4OOPP2b//v3ExcXh6+trj9AFQRAEwST6GyQhfrprk+gQH5IvF6DW1D09ShAsNXr0aGRZZtu2bWzatInExESWLFlCy5YtHR3aHUskJRyouLiYadOmERUVxZQpUxwdjnAbMjYOFMBb6YyXm7OolGjkUnOKCfFT4u7iBEBYgCepFjT/GjNmDCNGjGDevHkkJSXx6quvMnDgQJ5++mlbhywIgiAIZlHlleLmrMDfyxWAbiE+lFZqOG9mEl4QTHXXXXfRpk0bvvjiC2bMmEH37t2ZMGGCo8O6o4mkhAMtXbqU9PR01q5dKxqqCHahqhqxdStJkgj2VZIhKiUatbScIsL8vQxfhwd4kZZdbPbUFEmSePfddykvL6dfv37k5eWxdu1aJEmqf2dBEARBsCP9tYr+PSk6WFfBdzwjz5FhCbcxSZKIjY1l165dXL58mXXr1uHs7OzosO5oIinhIKdPn2blypX89a9/ZeDAgY4OR7gNlas1ZBWWE1y1VONWwX5KUSnRiMmyTFp2saHBJegqJYordH+v5urYsSOvvPIKRUVFTJ8+naioKFuGKwiCIAgWycgtqVbVGebviaerk8V9JT7ff5GRb+8Tyz+EOo0ePRqAF154gb59+zo4GkGkhBykVatWzJgxg5kzZzo6FOE2dSWvDMDo8g3QNZQ6eOF6Q4YkmCGzoJziCo1hFChgqJo4n11EkLe72cecM2cOXbp04ZFHHrFZnIIgCIJgDVVeKRGtvA1fKxQSXYN9LE5KfHXwEqeuFnIoPZe7w1rYKkzhNjN8+HA++ugjxo4d6+hQBERSwmG8vb1ZsWKFo8MQbmMZN3WzNibYT0lBmZrCskqauYvlQ42NfspGmH/1Sgnda8X0D/c3+5hubm488cQTtglQqFdBQQFZWVlUVlY6OhThDuPi4kJgYCDe3t71bywIDlRWqSGnqKLGtUq3YB82/p5OpUaLi5Pphd1ZhWUcvaRb9pGYkimSEkKtFAoFzz//vKPDEKqYlJSQJOljYJosy4W3PO8JrJFl+Tl7BCcIguVUeSUAhNRSKRFy01jQLi1FUqKxOZ+ja/B18/KNlt7uuLsozJ7AITS8goICMjMzCQ4ORqlUiv4dQoORZZnS0lJUKhWASEwIjZp+GemtVZ3RIT6Uq7WczSwisrXpP8M/ncoCoG1zD3anZDHngUjbBSsIgt2Ymnr8G2Dsk40S+KvtwhEEwVZUuaUoJGjpY7zMX39XQowFbZxSs4vwcHWi5U3LNBQKiVB/L9IsmMAhNKysrCyCg4Px8PAQCQmhQUmShIeHB8HBwWRlZTk6HEGok6qWqs7oYB8ATqjMa3a5KzmLYF8lLw4MJTWnmPPZ4v1SEJqCOpMSkiQ1lySpBSABflVf6x8BwINAZkMEKgiCeTLySgnydq+17DH4pkoJofFJyykm1N+zxgda3VhQUSnR2FVWVqJUGq9SEoSGoFQqxdIhodGrrVKifQtPmrk5m9VXoqxSwy/nsomJCGRYRBAAicniY4ogNAX1VUrkAFmADCQD2Tc9rgIfAWvtGaAgCJapbRyonr+nG67OClEp0UilZhdXa3KpF+7vyaXrJZSrNQ6ISjCHqJAQHEn8/AlNgSq3FCeFVK0qEHSVgVHBPpzIMD0p8eu5HMoqtcREBtHaV0nX1t7sThHVQoLQFNSXlBgKDENXKfEocO9NjwFAW1mWl9o1QkEQLKLKK6118gbo3vCDfZVkiEqJRqdcrSEjt4TQm5pc6oUFeKGV4eK1EgdEJgiCIAi2o8orpaW3O85Gqjq7hfiQcqWQCrVpoz0TUzLxcnPm7lBdc8thEUEcTL/O9eIKm8YsCILt1ZmUkGV5ryzLe4BQ4Luqr/WP32RZvtwgUQqCYBaNVuZqflmdlRKgW8MpKiUan/RrJWhlCA8wlpTQPXdeNLsU7gAbNmzAy6tmxZAgCLeHuqo6o4J9qNBoOZNZaPT1m2m1MokpWQzuFICrs+7jzfCIILTyjeaXgiA0XiY1upRlOR1wlySpvyRJD0uS9H83P+wcoyAIZsosKEOtleuslABdUiJDJCUanRvjQGt+GNNXT6SKZpeCnTzzzDNIkoQkSTg7O9O2bVsmTJhAbm5ug8fy+OOPk5qa2uDntTVJkti8ebOjwxCERqeuqs5uIfpml/Uv4Tihyie7sJyYyEDDc1HB3gR5u7H7lOgrIQiNnakjQWOALwBjw35lwMmWQQmCYB1D46j6KiX8lOQUlVNWqcHdRfwzbiz0VRChRiolmrm7ENDMjTRRKSHYUUxMDBs3bkStVpOcnMxzzz1HXl4eX3zxRYPGoVQq62wYqlarcXJyEv0TBKEJUmu0XC2ovaqzbXOa9+r6AAAgAElEQVQPvN2dOZ6Rz1/61H2sxJRMFBIM6XQjKSFJEsMigvjuiIpytQY3Z3GdIwiNlakjQVcD24AQWZYVtzzEv3BBaGT0SzJCTKiUALgs+ko0Kmk5xQR5u+HlZjxvHOYvJnAI9uXm5kbLli0JCQlhxIgRPP744/zwww+G1/Pz8xk3bhyBgYE0a9aMwYMHc/DgwWrH+PTTT2nXrh0eHh48+OCDxMfHV0seLFy4kKioqGr73Lpc49av9fts2LCB8PBw3NzcKC4uRpZl3nzzTcLDw1EqlURHR/PZZ58Z9rtw4QKSJPHll18yePBglEolPXr04Pjx4yQlJdG/f388PT0ZMGAAaWlp1WLasmULvXr1wt3dndDQUObMmUNFxY016u3bt2fJkiWMHz8eb29vQkJCiIuLq/Y6wNixY5EkyfD1pUuXeOihh2jevDkeHh506dKFL7/80tS/IkFo8q4WlKGpo6pTkiS6hfiaNBZ0V3Imvds3x8/TtdrzwyOCKK7Q8HvqdZvELAiCfZialGgPLLa2h4QkSYMkSUqQJEklSZIsSdIzt7y+oer5mx+/W3NOQbgT6SslWptQKXHz9kLjkJpdZLTJpV5YgJdhiYcg2Ftqaio7duzAxcUFAFmWeeCBB1CpVGzdupUjR44waNAg7r33Xq5cuQLA/v37eeaZZxg3bhxHjx4lNjaW+fPn2ySetLQ0Pv/8czZt2sSxY8dwd3dn7ty5rF+/nvj4eJKTk3nttdcYP34827Ztq7bvggULeOWVVzhy5Ai+vr785S9/YcqUKSxdupQDBw5QVlbG1KlTDdvv3LmTJ598ksmTJ3Py5Ek+/vhjNm/ezD/+8Y9qx121ahXR0dEcPnyYV155hdmzZ/Pbb78B8McffwDw4YcfcuXKFcPXEydOpKSkhJ9++omTJ0/y9ttv4+vra5PvkSA0BfobKHVVdUYF+3D6amGdE6cycks4dbWQ4VVjQG/WL7wFShcndqeIJRyC0JiZtHwD+BXoDJy38nxeQBLwadXDmETg6Zu+Fi1zBcFMGbmltPB0xcO17n/i+gsB0eyycUnNKeb+6Fa1vh4e4EluSSW5xRU17goJjdf06dM5evRog56ze/fuvP3222bvt2PHDry8vNBoNJSVlQHw1ltvAfDTTz9x9OhRsrOzDUsrFi9ezJYtW9i4cSOzZ89m9erVDBs2jDlz5gDQqVMn/vjjD9avX2/1/1NFRQUbN24kKEj3AaS4uJi33nqLH374gYEDBwIQGhrKgQMHiI+P54EHHjDsO3PmTO6//34AXnrpJWJjY1m8eDFDhw4FYPLkyUyePNmw/dKlS5k1axbPPvssAOHh4axYsYKnnnqKuLg4Q+XHiBEjDPtNmTKFd955h927d9OvXz8CAgIA8PX1pWXLloZjp6enM2bMGO666y5DzIJwJzEsNa2jqrNbiA+VGpnTVwvpFmI8aacf+zksIrDGa+4uTgzs6E9iciaLRncVS70EoZEyNSnxHrBSkqTWwAmg8uYXZVk+bMpBZFneDmwHXVVELZuVy7J81cS4BEEwor5xoHotfdxRSKJSojG5XlxBXkklYXVWStxodtnLs3lDhSbcQQYNGsQHH3xAaWkpH374IefPnzdUEBw6dIiSkhLDh229srIyzp/X3btISUkhNja22uv9+vWzSVIiJCTEkJAASE5OpqysjJEjR1b7wFFZWWlYKqHXrVs3w5/1x4iOjq72XHFxMSUlJXh4eHDo0CEOHDjAihUrDNtotVpKS0u5evUqrVq1qnFcgNatW5OVVXfH/2nTpvH3v/+dHTt2MGzYMB555BF69epl4ndBEJo+UyolooN1zS6PZ+TXmpRITMkkLMCTsADjk3piIoP4ITmTlCuFRLb2tjJqQRDswdSkhL5l9AdGXrN1o8sBkiRlAXnAXmCOLMtilk8T83HSx/i5+fFIx0ccHUqjpdXKvLz5GI/3bsPdYcZ6yFpOlVtCp6Bm9W7n4qSgpbe7XSolPvk1jdySSmYO72TzY99Kq5WZ8uUR0q/V32dhTM8Qnr3HujuSyZcLeGvXaVY93p1m7i5WHetW+mUZ4bVcXAGEVk3lOJ9dTK92tk1KVKi1zN58jHM2Wh4S0dKbuLF32eRYTZ0lFQuO4uHhQYcOHQB45513GDp0KIsXL2bhwoVotVqCgoL4+eefa+zn7W36Bb9CoUCW5WrPVVZW1rL1DZ6e1RN2Wq0W0PV+aNu2bbXX9EtOjH2tT2AYe05/TK1Wy4IFCxg7dmyNOG5Oytx6HkmSDMeozfPPP899993H9u3bSUxMpH///rz22mssXLiwzv0E4XaRkVuKv5dbnY22Q/yU+Hm4cCLD+ASOwrJKfk+9xnN1vK/f2yUQSdIlL5p6UiK/tJLZm48x78FIQvw8HB2OINiMqUmJhqop3AF8DaSh62OxBPhRkqResiyX37qxJEnjgHFAjQsRwbG+Ov0VLT1biqREHdKuFfP1YRUl5RqbJiVkWUaVV8rQzjXLGI0J9lOSYeNKiTOZhSzdloJCkhg/KAzPWho22kr69RK2Hb9CdLAPgc3cat3ucn4Zi7cm0ye0OV1b+1h0Lo1WZvZ/j5GkKmB3ShYP9wi2NGyjUqumaoQZmbyh18ZPiYuTRJodml1+9Esq3x69zMCO/rg6mdp2yLiswnI2Hcpg4tAOdfbIEBq/BQsWMGrUKMaNG0fPnj3JzMxEoVAQFhZmdPuIiAh+/716S6hbvw4ICCAzMxNZlg3JAEuWt0RGRuLm5kZ6ejr33nuv2fvXpWfPnpw6dcqQoLGUi4sLGk3NNfEhISGMGzeOcePGsWLFClavXi2SEsIdw5SqTkmSiAr24XgtY0H3ncmhUiMzzEg/CT1/Lzd6tPElMSWTqcM6WhWzo+08eZWdJzMZFhHEY71FUkK4fZj0SUGW5XR7B1J1npvbTp+QJOkQkA48gC5Zcev2H1BVvdG7d2/51tcFx5BlmaySLJwkMZilLvqs/76z2TYdyXm9uIKySq1JyzdAVzb5x4Vcm5wbdH//c79NQpKgQqPl57PZjIyqvT+CLRzP0HXmfmNMdJ3JhvySSu795x7mfZvE5r/3R6Ewf23p5/vTSVIV4OIksSsl0/ZJiZxiXJykOu+AODspaNvcw+bNLjNyS1iz+xwjIoP44K+9rT7epeslDHzzJ3anZPLCQOMfXoWmYciQIURGRrJkyRLi4+O55557eOihh3jzzTfp0qULV69eZceOHcTExDBw4ECmTp1K//79Wb58OY8++ih79uzhm2++qXHM69evs2zZMv785z+zZ88eNm/eXEsEtWvWrBkvv/wyL7/8MrIsM2jQIIqKivj9999RKBSMGzfO4v/v+fPn8+CDD9KuXTsee+wxnJ2dSUpK4sCBA7z55psmH6d9+/bs3r2bwYMH4+bmhp+fH9OmTWPUqFF06tSJgoICduzYQWRkpMWxCkJTo8orJbJV/ZUL3UJ8eG9vqtFrpd0pmfh5uNCzbd1NYodFBBG38zSZBWUEebtbFbcjJSbrGnZmFZQ5OBJBsC2TboNJkvR/dT3sFVzVtI8MoGmnNe8wBRUFVGoryS7NrlGaK9xwoirrX1Kh4bfUazY7rqFxVD2TN/SC/ZRcLShDram71NhU3xxRcSDtOvNju+KjdGFXsv1XXyWp8nF1VtS7ZMXHw4XX7o/g8MU8Nh26ZPZ5sgvLeXPnaQZ08Of/eoSw93Q2FWrbfN/0UrOLaNfCE6d6Eia6CRy2rZR4fUsyAPNjbfPBqE1zD7q0bMauZNH1/Hbw0ksvsX79ei5evMj27du59957efHFF+ncuTOPPfYYp0+fpnXr1gD07duX9evXs27dOrp168bXX39dowIgIiKCdevW8cEHH9CtWzd27dpVY6qFqfRLS1auXEnXrl0ZPnw4//3vf61uHnnfffexbds2fvrpJ/r06UOfPn144403zK7O/Oc//8lPP/1EmzZt6NGjB6BbGjJlyhQiIyMZPnw4QUFB/Otf/7IqXkFoKrRa2eT+V9HBvmi0MilXCqo9r9Zo+fF0FkO7BOJcT2Xf8EhdJYW+KWZTVFap4eezOQBkFtQoIBeEJs3cnhK30n/itMstcUmS/IFg4Io9ji/YR1aJ7hd+qbqU4spivFxrXxt/JzuRkU/X1t6k5RSzOyXT5OUW9TE0jjK5UsIDjVYms7Dc5ERGbfJLK1m2PYXubXx5sk9bDl64zk+ns9Bo5Xo/ZFvjeEY+Ea28cTFhucGYnsH854+LvPH9KUZEtjRresXy71Moq9Sw6KGupGYX85+DlziQdp0BHf2tCb+a1JziOptc6oUFeLL3dLbNvrc/ncrih+RMZo/sbNN1qjERQazbe568kgp8PcSkkKZgw4YNRp9/4okneOKJJwxfr169mtWrV9d6nGeffdYwtQIwWgUxfvx4xo8fX+25adOmGf78zDPP8Mwzzxi+XrhwodHlDZIkMWXKFKZMmWI0lvbt29dIkvfu3bvGcyNHjqzx3IgRIxgxYoTR4wJcuHChxnN79uyp9nVsbGyNxp9r1qyp9ZiCcLvLKS6nQq016bqjW4iuAvKEKp8ebf0Mzx9KzyWvpJKYOpZu6HUM9KJtcw8SUzJ54u6mueT7t/PXKK3UoJAgq1BUSgi3F5MqJWRZVtz8AFyBu4GfgUGmnkySJC9JkrpLktS96txtq75uW/XaSkmS+kmS1F6SpCHAFiAL+Kau4wqNS3ZJtuHPWaVNNyNtTxqtTNLlfHq382NQxwASk7NsVlWir5QI8TXtg6U+eWGLZpdv/XCa68UVLHk4CoVCIiYiiOvFFRy5aLvlIbfSamWSVPl0CzatR4QkSSx+OIqCMjVv7jxt8nkOpF3n68MqXhwYRniAFwM6+OPmrCDRhrPP1Rot6deKa+0gfrNwfy8qNFoyckusPm9ZpYYFCScJD/DkhQG2XWYxLCIQjVZmz+ns+jcWBEEQ7gimTN7Qa+XjTgtPV47f0uxy96ksXJ0UDOoUUMueN0iSxLCIQH49l0NJhdqyoB1sV0omnq5O/Kl9c1EpIdx2LOpiJsuyWpblP4B/AGvN2LU3cKTqoQQWVf35dUADRAPfAWeAfwGngX6yLBdaEqfgGDcnIm5OUAg3pGYXUVKhITrEl2ERgVwtKOPk5YL6dzRBRm4pXm7OeCtNK4TSXxCo8qz7cJukymfj7+k83bcdUVUJgsGdA3BW6Hov2EtqTjHFFRqiQ0xvXNmlpTfP9m/Pl39cNClhUqnRMu/bJIJ9lUy5V7eaTOmqm32+KznTZgmljNxSKjVynU0u9UINY0GtX8Kxds95Ll4vYfFDUbg6W9fc8lZ3hfji7+Vm158BQRAEoWkxLDU1oapTkiSiQ3xqTOBITM6kb3gLvExspj08IohytZZfqpZANCWyLLM7JZNBnQJo09xD9JQQbjvWXn3mAeGmbizL8h5ZliUjj2dkWS6VZfk+WZYDZVl2lWW5XdXz5i/8FhyqWqVEiaiUMEbfT6JbiE+1UVW2oMorJdhXaehmXx9DUsKKSgmtVtfcsrmnGzNHdDY87+3uwt1hze26hjPppu+lOaYP70RgMzfmfZeERlt3UmHDrxc4nVnIgthIlK43VqsNiwhClVfK6Uzb5E1Tc/TjQE1YvlG1xMPavhIXcop5b+95Rt/Vmv4dbLcMRU9XMRPIPjv03xCalkcffVT0GRIEATB/qWm3YB/OZhVSWqGbYnM+u4jUnGJiIkxf+vqn0OY0c3e2aYVjQ0lSFZBZUE5MRBBB3m5kFZajrefaRRCaElMbXfa85dFLkqQHgffRVToIgkFWSRZuTrqxjNmlolLCmOMZ+ShdnAgP8KKFlxu92vrZ7E0yI9e0xlF6SlcnWni6Gu5aWOI/By9x9FIe/7i/Cz5Kl2qvxUQEcS6ryC7jK0H3vXR3UdDBhCUPN/Nyc2buA5EkqQr49/7aBwxdzS/j7cQz3Nsl0NAoS29YF93FUKKNGjnqEwyh/vX/vzT3dMVH6WLVBA5ZlpmfcBJXJwVzH4iw+Dj1GRYRRGG5mgNp1+12DkEQBKHpUOWV0szdGW93l/o3BqJDfNHKkHxFdyNid9U1U12jQG/l4qRgSOdAfjyV1eQ+0O9KyUQhwdAugQQ2c0etlbleUuHosATBZkytlDgI/FH1X/2fE9A1uHzBPqEJTVV2aTYhXiF4uniK5Ru1OKHSNbnUNygcFhFEkqqAK/nW93VQ5ZaY3bAyxE9JhoWVEteLK1ix4xR9QpvziJHxmPoGVLvtdGfihCqPyFbe9XbeNubBbq0Y0MGfuJ2nyS40vj5z8bZk1FqZhbFda1SfBHq7c1eID7tsVAmSmlOMr4cLzU1ovilJEmEBnlZVSuxIusq+M9nMGN6JQDuOSLNH/w1BEASh6VLllpp1rRJdtSxU31ciMTmLyFbeZl/vxEQEklNUwdGqUeJNRWJyJr3a+dHc05Ugb92Nv0yxhEO4jZh6FR8KhFX9NxRoB3jIstxflmXTO8UJd4TskmwCPAIIUAaI5RtGqDVaTl7Or9YDYXik7o67tcscCssqKShTm1UpAbrySUuXb7y54xRFZWqWPBxldMlIm+YedA5qZpcPpBqtTJKqgG4hdc8nr40kSSx6qCtllRqWb0+p8fq+M9lsO36FSUM70LaF8cahMRFBHLuUZ5NO2KnZRSZN3tAL8/cyLPkwV3G5mte3JtOlZTP+1q+dRccwldLViQEd/ElMsV3/DUEQBKHpUuWVEmLGtUqQtxsBzdw4kZFPbnEFB9Ovm7V0Q29Ip0CcFZLNKhwbwuW8UpKvFBhu8uhvImSJZpfCbcTU6RvptzwuybIs0nOCUdml2QR6BBLgESCWbxhxPruYskpttR4I4QFetG/hYfUHd0PjKDPvHAT7KlHllZr9gfFQei5f/nGJ5waE0imoWa3bxUQG8seFXPJLKs06fn1Ss4sordQY7qBYIjzAi3GDwvj6iIr9qdcMz5erdRMp2rfwYNyg2idSxFQt6fjRBtUSqdmmTd7QCwvwJLOgnKJy8zuJv/PjWa7kl7H0kSiLqkzMFRMZREau7fpvCIIgCE2XuZUSkiTRLdiHE6p8fjqdhVa+8f5rDh8PF/7U3r69rmzt1qUqQVVJCVEpIdxOTL4SlSSpmyRJn0qSdFCSpD8kSfqXJElR9gxOaHq0spbs0mz8lf4EKAPE8g0jjleVDEYH37i7rxtVFcT/zl2j2IIPmHrmNo7SC/ZVUq7WklNk+vpEddVEipbe7kwb1rHObWMignRjIc/Y9iJAX8ZpzuQNYyYP7Uiwr5J53yVRqdE1Y/xwXyppOcUseigKdxenWvft0rIZwb5KqxNKhWWVZBWWmzR5Q09fVXHBzH4dZzILWf9zGmN7hdCrXXOz9rWUvv9GU7oQFARBEGwvv7SSwnLzqzqjgn04l13Ed0cvE+TtRlRry977YyKDOJ1ZyKXr1o/Ubgi7UrII9fc0NMEO8NIt38iqZdmpIDRFpja6HA0cBtoA3wM7gLbAEUmSYu0XntDU5JXnodaqCfQIJNAjkOzSbFGufYsTqnw8XZ1qlOnHRARRodHysxWjqvSVEiHmVkr4eVTb3xSf/Z5O8pUC5sdG4lnPOC7DWEgbl0ueUN1oGGoNpasTC0d35UxmERt+vcCl6yWs+fEc90e3ZHA9888lSTdd4pdzOYau4JbQNwINM6HJpZ6+quK8Gc0uZVlm3rdJeLo58+qoLuYFaQVD/40mVDIrCIIg2J7hBoqv8WWRtekW4oMsw94z2dzbJQiFwrQpY7fSL/toCn2OisrV/H7+GjERgYYlsq7OClp4uopKCeG2YmqlxBJgqSzLQ2VZnlf1GAosr3pNEIAb40ADlLqeEuWacgoqChwcVeNyPCOfrsE+Nd5Me7f3w0fpYtWbpCq3FFdnBf5VWXRTmTsWNKuwjH/+cIaBHf0ZFdWy3u0VColhXQLZa+OxkCdU+UQF32gYao3hkUEM6xLIqsQzvLzpGE4KiXkPRpq0b0xkEGWVWn49Z3lCyZCUMKNSol0LDyTJvLGg3x29zP6068we2ZkWZv6cWCsmIoijNuq/IQh6GzZswMvLusSkIAgNx7DU1MxKiZuXaup7cVmiXQtPOgZ6NYmkxM9nsqnQaGtMGQn0didT9JQQbiOmJiU6ARuNPL8R6Gy7cISmTt/YUl8pATSaJRybDl4iSZXv0BgqNVpSrhTQzUgPBN2oqgB+OpWFxsJRVRl5ujWa5t490F8YqPJMK2Vcti2FcrWW1x8y3tzSmJhI3VjIPy7YZiykoWFosGVNLo1ZOLorGq3M/rTrTBvWkVY+pl0w3R3aAi83Z3afsvwC53x2MQpJl2gwlbuLEyF+SlJNXL5RUFbJkm0p3BXiw5//1NbSUC2mv6j66ZRYwtFYPfPMM0iSxOLFi6s9v2fPHiRJIifH8sSbLUiSxObNm6s99/jjj5OamuqgiARBMFdGru5aw9z+V4He7rT0dkfp4kT/cH+rYoiJDGJ/6nWb9Lracuwyh9LtM/I6MSULH6ULvdv5VXs+yNtNJPgbyL/+d4F/fHOi3seOpCuODrVJMzUpkQX0MvJ8L6DxpxmFBqNvbBngEUCAh67sPavU8R9A/ncuh1mbjzPu04OUVFjes8FaZzOLKFdra+2BEBMRxLXiCo5eyrXo+OY2jtLzUbrQzM3ZpEqJ385f49ujlxk/OIxQMyZF6MdC2qp8/1x2EWWVWqJDvG1yPNBNClkQ25WYiCCeGxBq8n6uzgoGdwogMcXy2eep2UWE+Hng5lx7/wpjwvy9SDVx+cZbP5zhWnE5ix+Oskl1ibkiWun6b+xKdvzvBKF27u7uxMXFkZ3dOBLK9VEqlQQGWn7XVBCEhqXKLcXNWYG/V/3jr2/1WO8Q/tq/XZ29nkzxYLdWqLUya/ecs+o4JzLymfrlEV789BB5Jab35TKFRivz46lM7u0SWKMhdWAzN7F8owFkFZaxIOEkW45e5oeTmbU+vjmsYu63Jy2+BhRMT0p8CLwvSdIcSZKGVj3mAu8BH9gvPKGp0VdKBCgDCFQ2jkqJCrWWed8l4e/lyuX8Mtb8aN0bkDVOqPRNLo0nJQZ3DsBZIVn8oU2VZ1lSAqrGgtbTU6JSo2X+d0m0aa5k0tAOZh3f1mMhDU0ubVgpAfDE3W356G+9cTFzIsWwiECyC8s5bmE1jm7yhulJHr2wAE/Scorr/Z4mqfL59LcLPHV3O4tHqFrrRv+NbMoqLe+/IdjX0KFDad++fY1qiVvt27ePu+++G3d3d4KCgpgxYwYVFbVflGs0Gp5//nlCQ0NRKpV07NiRN998E622+pKuf/3rX0RHR+Pm5kZQUBB/+9vfAGjfvj0AY8eORZIkw9fGlm+8//77dOjQAVdXVzp06MCHH35Y7XVJkvjggw8YO3Ysnp6ehIWF8dlnn5ny7REEwUqqvFKC/ZQmV1rebOaIzrw2KsLqGLq29uHPf2rD+l/SOGPhVCitVmbud0n4Kl3IL63kzZ2nrY7rZocv5pJbUskwI6NPg7zdyS4st7iyVjCNfrLaV3/vx8G5MbU+lv9fNDlF5RyramYvmM+cnhKLgAnA7qrH34EFwDL7hCY0Rdkl2fi6+eLq5Iq/h660ztFjQdf/ksb57GLiHr2LR3uF8NHPqZzLcsxYwuMZ+TRzc6Z9C+MfPr3dXbg7rLlh/JM5yio1ZBeWm71GUy/YV0lGPZUSH/+SxtmsIhbGdrXoLoV+LOSZTNMbM9YmqZaGoY4ytHMgCgmL/u60Wpm0nGKzmlzqhfl7UlKhqXNtqVYrM++7JPw8XHl5hGNX3A2LsL7/hmBfCoWCN954g/fee4/z588b3UalUjFq1Ch69OjBkSNHWL9+PV988QWvvfZarcfVarUEBwfz1VdfkZKSwtKlS1m2bBmffPKJYZv333+f8ePH8+yzz3L8+HG2b99OVJRu0Ncff/wBwIcffsiVK1cMX9/qm2++YfLkyUyfPp2kpCSmTZvGxIkT2bJlS7XtXn/9dR566CGOHTvG448/znPPPcfFixfN+l4JgmA+a26g2NLskV3wcndm7rdJFt0s+fKPSxy7lMeC2K480789Xxy4yNFLtvtQmpiSiYuTxCAjDbcDvd3RynCtSPSVsKfElEyCfZV0aVn72HuAIZ0DcFJITaJPSWNVd8v8KrLuX+oqYJUkSc2qnhPD5oUaskuzDcs2lM5Kmrk0M1RPOIIqr5R3dp9lRGQQQ7sEEh3iww8nrzLv25N8/uLdFmXprZGkyifKSJPLm8VEBLFoSzIXcoppb8YH7iv5ujI+ayolDtTR7+FyXilvJ54lJiKoRsMlU+nHQiamZNK5nl/w9TmeUf/3siH5ebrSu31zdiVn8pKZH/yvFpRRWqkh1KJKCV0iIzW7iJY+7ka3+ergJY5czGPl2Lvw8XAx+xy2dHdYc7zcnElMybT456ipWrTlJMmXG7bxb2RrbxbEdjV7v/vvv5977rmHOXPm8OWXX9Z4fe3atbRu3Zq1a9eiUCiIiIjgjTfeYPz48SxevBgPj5q9UVxcXHj99dcNX7dv357Dhw/zxRdf8PzzzwOwePFipk+fzsyZMw3b9eqlWz0aEKB7b/H19aVly9ob7K5cuZKnn36ayZMnA9CpUycOHTrEihUriI29MTDs6aef5qmnnjKcd/Xq1ezbt8/wnCAI9qHKLaVra9stvbRUc09XXhnZhde+PsG3R1U80iPE5H2vFZWzYscp+oY156HurRkWEciWY5eZ++0Jvps0wCZLJBOTM+kb1gJv95rv20HNdI2qMwvKCfQ2/t4vWKe0QsMv53L485/a1vt5wdfDlT+192N3Shaz7mu4yWa3E/Pqk9ElI0RCQqhNdkk2AcobGd0AjwBySh13R/T1LSeRkZkfq5ui4O/lxqyRXfgt9RoJxy43aCwVai0pVwrpVks/Cb2Yqg9q5mZbDSO2rKiUKH3WkVMAACAASURBVCxTU1BmvOnT4q3JyMgsiDVtIoUxthoLWanRknyloN7vZUOLiQjk1NVCQxMvU+knb4RbUPWhX/JxvpZml7nFFazYcYo/tfdjTM9gs49va27OTgzq5G9V/w2hYaxYsYJNmzZx6NChGq+lpKTQt29fFIoblxEDBgygoqKCc+dqXyL33nvv0bt3bwICAvDy8mLVqlWG6oSsrCxUKhXDhg2zKu6UlBTuueeeas8NGDCA5OTkas9169bN8GdnZ2cCAgLIyhL9TgTBnkorNFwrrmgUlRIAj/duQ/c2vizdlkJ+qelNL1fsOEVxuZrFVQ2/m7m7MPfBSJJUBXy+P93quNJyijmfXWy4mXOroKpEhOgrYT+/nsuhrFJrdPmMMTERQZy6Wsil6+ZdAwo6JlVKSJLkBywEhgKB3JLMkGVZdJgSAF1Ty3DfcMPXAR4BDquU+OlUFjtPZjLrvs6E+N24a/dEn7ZsOniJpdtSuLdLIM2MZKDt4UxmIRUaLVG19JPQa9Pcg85BzdidksULA8NMPr5+coY1lRKgS254t6r+Pdl7Jpvvk64y677OtGlu3lzxW8VEBPFW4hmyCssIbGZZdv9MZiEV6vq/lw0tJiKIZdtPsTsli7/1b2/yfvpGlfqqB3O09HbHw9Wp1maXb+48RUGZmsUPmz4pxd5iIoLYfuIqJ1T53NXGMf0tHMGSigVH6tOnD2PGjGH27NnMmzfP5P1q+zn7z3/+w/Tp01m5ciX9+/fH29ub+Ph4vvnmG1uFbFZcLi4uNV6/tb+FIAi2Zek4UHtRKCSWPBzF6Hd/4a0fTrPooah69zmUfp2vDmYwflAYHYNuVH3GdmvFlwcu8ubO04yMakVAM8vHbuuXgtZWUahPSmQViuUb9pKYkomXmzN3h7YwafthEUEs2ZZCYkomz95jerN0QcfUSolPgVjgv+iSE/NueQgCGq2Ga6XXDKNAAQKVgQ5pdFlWqWFBwknCAzx58ZYP9k5Vb0DZReWs2nW2wWLSN2Y05e5+TGQgBy6YN6pKlVuKQqLWEv766JMZt07gKKvUsOC7JML8PXlhoPW/ZIdFBCHL1o2F1I92dVTDxtqEBXgRFuBpdpXL+exiPF2dCPI2/wJGkiRC/T1Jza5ZKXH4Yi5f/nGJZ/u3p0tLx5fK6un7b4i1l43fsmXL+Pnnn9mxY0e15yMiIvj999+rfYj/5ZdfcHV1JTw8/NbDGF6/++67mTx5Mj179qRDhw7VelYEBgYSHBzM7t27a43HxcUFjabuJqkRERH8+uuvNc4dGWl5lZcgCLZhSEr4WneDw5aign14um87Nv6eXu/oeLVGy5xvkmjl487UYR2rvSZJEq8/FEVZpYbl36dYFdOu5Ey6tGxW640gfy9XJElUStiLViuz+1QWgzsH4Ops2sflUH9POgR6sTtFVNxZwtSkxBDgUVmWF8my/J4sy+/f/LBjfEITkluei0bWGHpKQFWlRGmWTaYtmGPdnvNcvF7C4oeijP4y6RbiyxN92rLhf2kNtsb7hCofb3dn2ppQaTAsIgiNVmbPGdN/sWXkldLS293sqRF6hkqJWyZwvL83lQvXSnj9oSizx1Uaox8LmWjFL+3jGfk0c3emnZVVG/YQExHE76nXKKxlGYwxqTnFhAZ4WlzJEOrvaVgCoqfRysz7NonAZm5MH97JouPai5+nK73bNbfqZ0BoGB06dGDcuHGsXr262vMTJ07k8uXLTJw4kZSUFLZt28arr77K5MmTjfaTAF1vh8OHD/P9999z9uxZFi9ezN69e6ttM2fOHN5++21WrVrFmTNnOHr0KP/85z8Nr7dv357du3dz9epVcnONj06eNWsWGzduJD4+nrNnz7JmzRr+/e9/M3v2bCu/G4IgWMvapab2MnNEZ5p7ujH326Q6lxZ++ls6p64WMv/BSDzdahacdwj04sWBYXx9WMWBtNr7dNUlr6SCg+m5huW8xjg7KfD3ciOrUCQl7OG4Kp/swnJiTFy6oTcsIpDfU6/VuhRaqJ2pn17Om7GtcIfSL9PQjwIFCPQIRK1Vk1fecCNy0q8Vs27veWLvak3/Dv61bjf7vi74ebgy77u634Bs5YQqj24hviZ98Owe4ou/l6tZH9pUuaVWvcn7e7rh6qyo1g/h4rUS1u45x4PdWjGgY+3fS3Pox0L+fNbysZAnVPlEN6ImlzeLiQiiUiOz74zpvVRSs4sItWDyhl5YgBcZuSWUq298Pz/7PZ2TlwuY92AkXkYunBwtJjKQlCsFZvffEBre/PnzcXau/jMUHBzM999/z5EjR+jevTvPPfccf/nLX1i2rPaBXOPHj+exxx7jiSee4E9/+hMXLlzgpZdeqrbNhAkTiI+P58MPPyQqKoqRI0dy8uRJw+v//Oc/+emnn2jTpg09evQwep6HH36YNWvWsGrVKiIjI1m9ejVr166t1uRSEATHUOWV4KSQDI0aGwsfpQtzHujC0Ut5/OfgJaPbZBWU8dauMwzqFMDIqNqb7U6+twPBvkrmfZtEpcb8JWF7Tmej0crERNbdDDqwmVudk7cEy+1OycRJITG0s3lJieERQai1MntPO3byYFNkaqJhGrBckqS7JEmy/lapcFvSL9OoVilR1fSyofpKyLLM/O9O4uqkYO4Ddc+x9vFw4dVRXTiUnsvmwxl2jausUsPpq4Um90BQKCTu7RLIntNZVKhNe0OzdsSWQiER7Ks0VErIssyChCScFRJzH7Bt2bM1YyEr1FpOXSkkupH1k9Dr2dYXXw8Xk5cmlFVqUOWVWjXaNDzAE60M6dd0H/CzC8tZ+cNpBnTw54HoVhYf1570d4BEmWPjsmHDBrZu3VrtucDAQAoLC5FlGX//G8nJQYMGsX//fsrLy8nMzGTVqlW4udX+QcPV1ZX169eTm5tLXl4e69evZ/78+Vy4cKHads8//zzJyclUVFRw9epVPv74Y8NrsbGxnD17lsrKSsN+zzzzDEVF1Xuq/P3vf+fcuXNUVlZy7tw5XnzxxWqvy7LMo48+Wu25Cxcu8PLLL9f7PRIEwXKqXF1Vp7OFVZ329HD3YO4Obc6KHae4XlxR4/Wl21Oo0Gh5fXTXOm8webg6syA2ktOZhfzrfxfMjmNXSiYBzdzoVs91TpC3u1i+YSe7kjPp3c4PXw9Xs/br0daP5p6uFo2Hv9OZ+hvhHKAEDgMVkiRpbn7YLzyhKckqraqUuKmnhD5BkV3aMBnDnSevsvdMNjOGdzI0AarLmJ4h9G7nxxvfnyKvpOYbkK2cvlpIpUY2a1pETEQQhWVq/qhjTKeeRitzNb/M6nLIYF+lobTyh+RMfjqt+15a2qeiNjfGQpr/gVTfMDS6kU3e0HN2UnBv50B+PJWF2oQ7JOnXSpDlG1M0LBHmf2MsKMDy7SmUVWpY9FDdF06OFBbgRZi/+f03BEEQhKZLlWddVac9SZLE4oejKCpTs+L7U9Ve+9+5HL47epm/Dw43aVz78Mgg7u0SyKpdZ7iab3rioEKtZe/pbIZ1Cay3GjTIW1RK2ENGbgmnrhbWuXymNvrqih9PZVlUJXMnMzUp8QXgA0wFHgceu+UhCOSU6O56t1De6FKrr5RoiGaXJRVqXt+STJeWzfhbv3Ym7aNQ6N6A8ksreXPnabvFdqKqcZI5d/cHdPTHzVlh0oe2zIIy1FrZ6sZR+koJ/feyc1Azs6ZImMrN2YnBnQLYnZJp9tIZQ8PQ4MbV5PJmMZFB5JdWcijd+Jr3m+kTCeEWTN7QC9WPBc0u5vfUa3x9RMW4QWFWHbMhxESa339DEARBaLpUuaWENJJxoMZ0CmrGcwNC+c/BS4b38Aq1lnnfJdGmuZKJQ4w38r2VJEksjO2KWiuzeFty/TtUOZB2naJytUkfiAObuXOtuFx8+LUxfQVnfctnajM8MpCCMjUHL9R/DSjcYGpSojfwF1mW42VZ3izL8n9vftgzQKHpyCrNorl7c1wUN8asNWSlxDu7z3E5v4wlD0eZVRYY0cqbZ/q354sDFzl2yT69L05k5OPn4UKIGXcHPFyduaeDP4kpmfU2CrXViK1gPyU5RRWs3HkGVV4pSx6JsrhxZn2GRQTy/+y9eXxU9dn3/z4zk8xktoQkMxOSsBggkABhUVmksoSgpWj1dutdW1u9b7V3e6u11qUVREW72Ralz9O7z9NKXdqf1tYHF1BUlqi4IKJgEgh7CCSEzCQhk5lMJrOd3x+TGRKyzEySSTLwfb9eeTEzZ7synJzzPdf3c30uq6M9nLCJlvLaZlJTkhiTPnIHNZdPyiRJKUWVUDrWYVB50QDKN/RqFRajmkP1Dla/UUFOWgp3LZkUecNhZukUc8z+GwKBQCBITLz+AKdbBq7qjDc/XjqJLKOGR16vwOcPsP6jKo7aWnn8m1PRJEVfxT42Q8uPFk/krbI6dhyObhy8tbIetUrBgj480UJYjBpkGRqcQi0xmGytrGeCSdfvcdnlk0wkKxWihCNGon3a2A+MnH5yghGJzWULKyNCqJVqUtWpcfeUOFzv4Nkdx7jx4lwuGZ8e8/b3lkzCpA+6LvvjYHpZVmtnWk5qzFL6kgILJ5vaOFTv7HO9kFngQDwlgHDS5K8fV3H97Fwu7cd3GS2htpCxXrTLaoImlyO1LAHAoEliXl5GVOUpR21OsoyaHl28Y+GiTB0bvzrFoXonj31zKinJI9/+5+Jxo0jTJokbt0AgEFwAnLa7CcgDH6vEG51axeqrC9lf18JT7x7kD9sOc0WhheIpsc+c/2BRHuMztKx+Y18XM+qekGWZLfvruXxSZlT38FAbcVHCMXg43F52HmvsV+lGCJ1axfwJGWyJYlJRcJZoR8GrgLWSJK0CyoEuWltZlvvX80ZwXmF1WcPKiCNWB2qVkjHpWkwppn6Xb7i9ft4ur4to9vjP3SfRqVX8bPmUfh3HoEnikasKufvlPbz0WTW3zB/fr/30hNvr51C9g+IpeTFvu7TADK/B/7x/hPl5Gb2u92FHBj4WJUZPhAYKRo2Kn3+jf99ltIzSJXPJ+HS2VFq574rJUW0T+i5vvzz273KoWVZoYfUb+/jLh8cwaHq/1O490TwglUSIPJOenceaWDrFzLJ+Sg6HmrD/xsGg/8ZIND4TCASCocDf3Izn+HFSZs4c7lBiorXdx96TzVHN7NeM0HagPbF8WhaXT8rkzx8eQ5OkYPXV/TP81iQpefyaaXz/r7t45PUKZo8d1eu6zW1eapvbuLt4YlT7DnmnWeNodtns8nDU1srF43qPe6ixOdrZfqCeSM/72WkpXD4pM6ZJrA8PNeD1R+58EomSQguPvF7BUZuTiWZD1Nv5/AHe219PS1vkstaZY9OYknX+aAaiTUq83fHve0DnU0DqeD/yp+QEccfWZqMgI9jx4p6X95KhT+Zv/zkXs9bc7/KN1/bU8vMN5RHXkyT4zfVFZOj732LqqqLR/O3Tap77+PigJiUq61rwB2Sm98MDwWLUMOeidN7Ye4o39p7qc928TF1MssKemGjWo0lS8PA3CsgcwHcZLV+fmsWaTfv55EhDn+1bQ4QNQ0do543OLCu08ORblfzi7cqI614xtffWYtEya0waG/ee4rFvTh3wvoaSK6dlsWFPLRv21HLTJWOGOxyBQCAYFqxrn8a+cSOTv/xiRCsBz+XlXSd48q1Knv3eJREf5MKlpiNcKQFBT4g110zj2j9+zN3FE8kd1X/PrkX5Jq6blcM/d9fwz919d3vTJCkoLoiuDaU5pJRwxE8p8T/vH2X9R1V8vrKEdF1snSjigc8f4Jb1n3HgtCOq9f/Pdy/us33ruWytrGeUNqnP5FE0LJ1i5hFga6U1pqTEH0uP8vTWQ1Gtu2pFwQWZlFgS1ygECY8v4KOxrTFcvlHb3MbJMy5kWcaUYuJo89F+7ferk82M0ibx1j2X09c9Wq1SDvhiKUkSiyab+O27B2lxezFqkiJvFAUhz4RYOm905u//OZfG1sg3nFExti3qiQy9mq8evQK1amjyjDfPHctzn1TxyBsVbP7xQpJVfc+Ul4UMQ0do543OjE5NYfeqElrbfX2uJyFhHoR+7TdeMoarZ2QPODE11CwrsHBxRwecKwotMbffEggEgkRHDgRwlG5HbmtD9niQ+mitO9LY2+HF9djGfSyY2HfZQai7V3YCJCUgWBb5+cqSiGOTaPj9TTN48OtTkOl7el+nVkU9/szQqVFI8VVK7D3RjD8gU3rAyvUX58btONHywqfVHDjtYO1NM5g/oXcFsSzDfzz/OWs27mNhfiba5MiPvD5/gO0HrCwtMKOM0PkkEtlpKUzNNrJ1fz3/tSg6c9QTjS7+5/0jfGN6Fo9cFVmZYxik55SRQlRJCVmWP4h3IILEprGtERkZs9aM2+vH3iE7qm50YdaaaWhrICAHUEixXdjLaoJeDEN1Awt1x6iotXPZhMgz99FQVmMnQ5fM6H621UxWKRidOnQ38KFKSECHrPGbU/mP53ez/qMqfhjB1bq8JpikSoRZFgCjJmnQklvRkGgJCejogHPNNK76Xzv47bsH+cW/TR/ukAQCgWBIcVdU4LcFDX8Dra0oEigpUV5rZ3yGluMdD1Q/7aMcs7bZRaZenVD3qsFISEBw4muw26srFRImg5r6OCUl/AGZilPByaCtlfXDnpSob3Hz9JZDLJ5s4t9m5URUFD157TRu+D+f8odtR6Iq7/6i+gz2Ni/LBuAn0ZmSAgt/2H6YRmd7RCW3LMs8+mYFKoXE6qumDvq5kgjE9JcmSVK2JEnzJEla2PknXsEJEodQeYYpxYS1k+FOWa2dzJRM/LKfJnds1iMh/4D+Kgz6QygpUV4TW0eIviivsTM9d2QbMw4nxVMsLCu08Idth8PSzt4or21hem6a+C7PMwqzjXz/svG8FMcOOAKBQDBScZSWhl8HWluHMZLYsLu8VDe6uOnSMfzbrBz+7wfHwm2ue6K2uS0h/CQSCYtREzejy2M2Jy6Pn3RdMh8eskU06ow3T75Viccf4PFvTo1qHHjJ+HRuuDiXZ3cc44g1crnH1sp6kpUKLs83RVw3GkoKLMgylB6MXML+3v56Sg/a+Mmy/AsyIQFRJiU6khHvAzXAx8D7QGmnH8EFTsjI0qw1U+84m7Etr2nGrDV3WSdaKuta8PXTi6G/jNIlkzsqJVwmMFDaPH4OWx0J4YEwnDx6dSEyMms27ut1nXCSSnyX5yX3LcuPawccwdAzfvx4fve73w37PuLBq6++KpKjgkHDub0UlEH1QCIlJcLlqTlp/PwbU1AnKXj0zX29dhyoPdM2YENuQVfMBk3clBJlHRN0P1iYR6vHz85jw9fX4OMjDWz86hQ/WjyBcRnRm4P/fPkUdGoVj7ze+3kZYlullXkTMtAPsBtaiGk5RixGNVv3991hzOXxsWbjfqZkGfj+ZeMH5diJSLRKiWcAP1AIuIDLgRuBSuDr8QlNkEiElRJaU/jiaFCrKK+1hztyxGp2WTFAL4b+UpSbGj72QNlfZycgw/TcoUusJCK5o7TcXTyJd/fVU3qw5zaa+0OGoQngJyGIHYMmiZUrCiivtfPSrhPDHc4Fy6233ookSUiShEqlYuzYsfzwhz/kzJkzwx2aQHBe4q2tpf3gQXTz5wOJmZSYlmPEbNBw/xWT2XG4gbfLT3dbNxCQOdXsJjdByi8TBYtRjTVORpfltXZSkpTcMn8cKUnKiA/X8cLjC/DIGxWMy9BG7c8QIkOv5oErJ/PpsUbe/Kp3w/ijNifHGlpZFqXJaDRIkkRJgYUPD9twe3tXmfzv7UeobW7jiWunkXQBdyGL9jdfBDwky/IBgt02bLIsbwAeAp6IV3CCxMHqsiIhka5JD5dvLJpsoqK2hUxNR1IiRqXEQL0Y+sv0nDSqG13YXZHb8UQilGWeLmb3I3LH5XnkmXQ8+sa+Hi/e5eK7PO/55oxs5udl8Nt3DtDgFH3Xh4uSkhLq6uo4fvw4zz77LBs3buRHP/rRcIclEJyXOErfByD16quAREtKNDM2XRs2KP7uvHFMzTbyxKb9OM8xeW5wtuPxB0T5xiBjMWpoavXEpbSivNbO1Gwj2mQVl0/KZFtlfUS1QTz4y45jHLO18tg3p/bLj+Tbc8ZSlJvKk29V0uLueWwfSrgUD5KfRIiSAgsuj5+dxxp7XH7E6uQvO45x/excLh2fPqjHTjSiTUqkAA0dr5uAUBppP1A02EEJEg9bm42MlAxUChX1DjfJSgULJ5lwtvtwuII3IGtbzzPgvVFeOzxeDCFlRvkgqCXKa+yYDGosxsQxrRouklUKnrhmGieaXPzp/e7dWspr7WTqhz5JJRg6JEniiWun0ub18+vNB4Y7nAsWtVpNVlYWubm5XHHFFXzrW9/ivffe67KOJEm8+uqrXT6LVGpht9u58847MZvNGAwGFi1axO7duyPG43Q6+e53v4terycrK6vbMdauXUtRURE6nY6cnBxuv/12mpvPepPY7XZuueUWzGYzGo2GvLw8nnnmmZjievHFFxk3bhxarZarrrqK+vrhmTEUnH84t28nOS8PdUGwpXoiJSXKOjyzQigVEk9eO416h5t157Q1rEmgdqCJRGh8aRtktYTPH2DfqbP/vyWFFk7Z3eyvaxnU40Si5oyL/7X9MF+fmsWSyf1TMYTOywZnO09v6bnd5rZKK4WjjYN+fs6fkBFUmVR2v2fIsszqNypISVLy829ENuI834k2KXEACH1be4H/kiRpHPDfQG08AhMkFlaXNdwO1NrSjtmoDl/IDpxyka5Jj0kp0eYZPv+AadnBY5bVDtxwr7zWTlGOMLmMlgUTM7l6RjZ/+uAo1Y1dB2blNXami+/yvGei2cDtl+fx6hc1fH58+OpXBUGOHTvGO++8Q1LSwLrIyLLMihUrqK2tZdOmTezZs4eFCxdSXFxMXV1dn9uuXbuWgoICvvzySx5//HEefvhhNmzYEF6uUCh45pln2LdvHy+99BK7du3i7rvvDi9ftWoV5eXlbNq0iYMHD/LXv/6VnJycqOP67LPPuPXWW7nzzjvZu3cvV199NatXrx7Q9yEQAPidTlo//xz9ksUodcE6+URJSpxp9VBzpq3bOG3W2FH8+6Vj+OvHxzl4+qy5YKgdqFBKDC5mQ3CiZrBLOI7YnLi9gfBEXfEUM5IEW/fHNsE4UNZs3I+ExOqrI7fI7Iui3DS+M3csL3xynH2nuk46NrV62F3dREnh4KokINgVbWF+Jtsqrd1UJhvL6vjkaCMPfH0KmRG6c1wIROvksQ7I6ni9BngH+DbQDnw/DnEJEgyby0aWLniK1Le4MRvUTDLrUasUQV+JFFNMSYn9dS3D5sWQqk1iXIZ2wB04Wtt9HLE5WVE0epAiuzBYtaKA0gNWVr+xj+dvuxRJknB5fBy2OrhyWlbkHQgSnruLJ/Lm3lM88noFm+7+GqrzpMby9C9/SXvl0CpA1AVTyHr44Zi2eeedd9Dr9fj9ftzuoEfQ2rVrBxRHaWkpe/fuxWazkZISfCh54okn2LhxI3/729948MEHe9127ty5rFy5EoD8/Hw+//xz1q5dy3XXXQfAvffeG153/PjxPPXUU1xzzTW88MILKBQKqqurmT17NnPmzAFg3LhxMcW1bt06li5d2i2G9evXD+g7EQhaP/oIvF4MxcUoEiwpEVKT9lRS+eCVU3in4jSPvFHBK3fOQ5KkcHctoZQYXMwdSgnrIJtdni2ZDY7DM/VqZo1JY9uBen5cMmlQj9Ub2w/U897+eh76+hSyB+G8eeCKKWwuP80jr1fw6n9dhkIRnOQqPWAlIEPJIPpJdGZpgYV399Wz71QL0zr+XhxuL09u2k9Rbio3zxkbl+MmGlGN9GRZ/v9kWX6+4/WXwHjgUmCsLMv/ilt0goTB1mYLG1rWt7ixGDWolAqmZhuDJQxaU0zlG+U1QZXCcPkHTM9JHXD5xr5TLciy8ECIFYtRw70lk/jgkI139wXNsvaf6khSie/ygkCbrOKRqwo5cNrB858cH+5wLjgWLlzI3r17w4qDb3zjG9xzzz0D2ucXX3yBy+XCZDKh1+vDPxUVFRw92r1cqzPzOwwAO7/fv39/+P327dtZtmwZubm5GAwGrrvuOjweD6dPB68fP/zhD3nllVeYMWMG999/Px988EFMcVVWVvYYg0AwUBzbt6NMSyNl5sxwUsKfYEmJqT3cl0fpknno61PYVdXEa3uCguraM20YNSoMmoGprgRdsRiDSonBbgtaXmtHl6wkL/Nsp4uSQgtlNXZO2+PT7aMzbq+fR9/cx0Sznv/82kWDss9UbRI/Wz6FL080868vToY/33agHotRHVZKDzZhlUmnEo6ntxzG5mzniWumoVQIBTBEr5TogizLLuDLQY5FkKB4/V6a3E2YU4IZRmtLO5dPCiYoinLT+Ofuk1w/zcTBpoNR77Osdni9GIpyU9lUVkdTq4d0XXK/9lE2zImVRObWy8bz6hc1rNm4n4X5prNtx0TnjQuGK6daWDLZxDNbD3P1jOzwwCuRiVWxMFxotVomTpwIwB/+8AeWLFnCE088wWOPPRZeR5KkblJUr7d3c+BAIIDFYmHHjh3dlhmNxn7HWl1dzYoVK7jjjjtYs2YNGRkZfPnll3z729/G4/EAsHz5cqqrq9m8eTPbtm1jxYoV3HjjjTz33HNxi0sgiITs89H6wYfoFy9GUipBqURKTk4YpURZTTMXZepITek5yXDTJWN4ZfdJfvl2JUsLLNQ2t5EzSjvEUZ7/pGuTUSmkQW8LWlZjZ2pOalhNAEHTxqfeOci2A/V8Z+64PrYeOP/z/lFONrXx0h1zSVYNnlry+tm5/HP3SX69+QBXFGahVSv54KCNa2bldPldB5NMvZrZY0extbKee0vy2X+qhRc+Pc7Nc8YyY4zozhfi/NDECoaVRnfQUdakNdHa7sPRYVjyUgAAIABJREFU7gvLyablpOLy+FGRSqO7EX8gOnfg8prh9WIIyasGopaoqLWTZdRgPg8epoYalVLBk9dO45TdzR+2HaG8xo7ZoD4vHkwF0SFJEo99cyoef4An36oc7nAuaB599FF+85vfcOrU2XZqJpOpixdEfX19n94Qs2fPpr6+HoVCwcSJE7v8mM19S2Z37tzZ7X1Bhyng7t278Xg8PP3008yfP5/8/PwucYbIzMzklltu4fnnn2f9+vW88MILtLe3RxVXQUFBjzEIBAOhbc8e/HY7+uLi8GcKnS5hkhIhn6feUHSYCza1evj9ewepPdMmSjfigEIhYTaoB1Up4fUH2F/X0s0vZJJZz9h0Ldsq4+srUdXQyv/54CjXzMzmsgmZg7pvhULiiWun0eL28dS7B9l5rIlWjz9upRshlhaYqaht4VRzG4+8UUFqShIPXDk5rsdMNERSQjBgrK7gxcmsNYeNdiwdxjuhme22Nh0BOUCTO7JxXWu7j6M2ZxdH56EmnJSo6b/ZZVmtfVh/h0TnkvHp3HBxLs/uOMaHh21CJXEBMi5Dx48WT2DjV6f4+EhD5A0EcWHx4sUUFhby5JNPhj8rLi7mj3/8I7t372bPnj3ceuutaDS9Jw1LSkpYsGAB11xzDZs3b6aqqopPP/2URx99tEeVQmd27tzJr371Kw4fPsxf/vIXXnzxRX7yk58AMGnSJAKBAM888wxVVVW8/PLLXTprAKxevZrXX3+dw4cPU1lZyYYNG8jLy0OtVkcV1z333MPWrVu7xPDaa6/19+sUCABwbC9FSkpCt2BB+LNgUsI1jFFFR4OznVN2d0Ql6NTsVL43fzx/31nNsQYnucLkMi6YjRqsjsFTShyud+LxBbqNYSVJoqTAwkdHGnB5fL1sPTBkWebRN/ehVipY+Y2CuBxjSpaRWy8bzz8+P8H/3n6YlCTloCc/zmVZR6vRe1/ZyxfVZ/jZ8inhVrqCICIpIRgwIQNLU4opbLQTmtGeYNKTkqSkyR59W9CwyeUwlj0YNUnkZer6rZRwuL0cs7UOS/eQ84mfLZ+CNllJg9MTThQJLiz+a9EExmVoWfnmZ7i98RkECSLz05/+lPXr11NdXQ3A73//e/Ly8li8eDE33HADt99+e5+KB0mSePvttykuLuaOO+5g8uTJ3HTTTRw8eJDs7Ow+j33fffdRVlbGrFmzWLVqFWvWrOGGG24AoKioiHXr1rF27VoKCwt59tlnu7UMVavVrFy5khkzZrBgwQIcDgcbN26MOq558+axfv16/vSnP1FUVMSGDRu6lLIIBP3BWVqKdu5clPqzNfsKnY6Aa2iSEvY2L25vdOrVcwmbXEYxWXDfFfmk69R4/bJQSsQJi1E9qOUb5R3d54p6MJsvKTDj8QXYcTg+EwXvVJzmw0M2frIsP65K43tLJmHSq/n8+Bkun5SJJkkZt2MBTDTrGZehZVdVExePG8UNs3PjerxEpF+eEgJBZ0KJBpPWxOGaDqVER/mGUiExLcdIbWMTaDoSGBl976+spndH56Fkem4qn1f1ryXh3pPN4X0I+k+mXs2DX5/CqtcrmD121HCHIxgGNElKHvzGGB7adSO/+dDJo0u/Ndwhndc8//zzPX5+8803c/PNN4ffZ2dns3nz5i7rXH/99V3eHz9+vMt7g8HAunXrWLduXdTxnLuPnrjnnnu6GXHedNNN4dcrV64Md87oiWjiuu2227jtttu6fHbXXXdFjE0g6In2Y1V4jh9n1Pdu6fL5UJVvONt9LH/mQ+ZNyGDtTTNj3r68xo4kwdTsyL4rRk0Sq1YUcO8re7mok2miYPCwGDXsPDZ4LbTLauwY1CrGpXf3ALn0onQMGhXbKuu5curgdkRrbfexZtN+CkYb+d78+HpWGDRJPHJVIXe/vIcrBvn36AlJkrii0MJfPz7OE9dMi5t/RSITdVJCkiQLcAswAXhEluUGSZIWAKdkWa6KV4CCkY/NZUMpKRmlHoW1JTiL1jm7OS0nlZe/UJB00dlSj74or2keEV4M03NSeWPvKWyOdkyG2Aw3tx+wolYpmHNRepyiu3D4ztyxTM02MlOYAV2wTBodQFL4+OzkkeEORSAQCAaMs3Q7AIYlS7p8rtDp8Df3v2w0Wp7ZcohTdjdb9tXj8QViNhIsq7GTl6mLupPGNTOzuShTJxSPccJsUIeVL4Mx419ea2faOSaXIZKUChZPNrOt0oo/IA9q54g/bD9Mnd3N/7559pC0Ar96RjZj0rVDNgl6b0k+N1w8hslZhiE5XqIR1f+4JEkXAweB7wD/CYRSo8uAX8QnNEGiYHVZyUjJQKlQUt/iRq1SYNSczXcV5abidmuRkLC12SLub6R4MYQuUhUxlnDIsszWynoWTMxEmyzESANFkiRmjR01bKanguGn1RecOaxqPBO3OlaBQCAYKhylpagLCkgaPbrL50OhlDhwuoXnPjnOJLMeR7uPXf1QhFbU2nuU9veGJEnMGJMmWh/GidAkns0xcLNLjy/AgTpHnz5eJQVmGls9YVXwYHCo3sH6HVXcdEkuF48bOmXszCE8L3VqlUhI9EG0aajfAetkWZ4FdD7j3wUW9LyJ4ELB1mYLtwOtb2nHYtR0eYCcnpMGKNGq0sL+E73hcHupahgZXgxTc1KRpLPlJNFyqN7JyaY2SjpMbQQCwcBweBwA+HHzUZzqWAUCgWAo8J05Q9uXe7qpJAAUOm1ckxKyLPPI6xUYNSpe/M85qFUKtlbWx7QPa4ub0y1uoXoYQYR83AbDV+JQvQOPv7vJZWcW55tRKSS2xXju9EbovNRrVPxseXzMLQUjn2iTEhcDL/TweR0gnrwucKwuKyatCQheEEN+EiHyMnXokpWo5NSI5Rv7TrUgyzBtBCgl9GoVE0z6mM0uQzf4pXFuLyQQXCi0eFoAUCd7Yh5ACwQCwUjC+cEHEAh0aQUaIt5Kif/3ZS2fHz/Dz5cXMDo1ha9NzGRrZT2yLEe9j9CYSHTEGjmExt2D0RY0Gl+3VG0Sl45PH7T78Rt7T/FZVRMPXjmFdJ3oSHGhEm1Sog3oSUszBYhvs1rBiKehrQGzNvgAbnW0d/OCUCgkpuak4m03RCzfKB8hJpchpuekhl2Io2VrZT1FuanhzLVAIBgYTo8TAEuaxPYDVgKB6AfQAoFAMJJwlr6PymxGM7Ww27JQ941YkgTRYnd5+dXblcwem8YNFwed/0sKLdScaeNgvSPq/ZTV2FFIUDg6ssmlYGiwGAZPKVFe24xRo2JsDyaXnSkptHCo3smJxoF1i7G3eXnyrUpmjEnj3y8dM6B9CRKbaJMSbwCPSpIUmgKXJUkaD/wG+H9xiEuQIHj8HprbmzGldFJKGLo/jBflpOJoTYlYvlFWaycnLYVMfWzGkvFiek4q9S3tUV/obY529p5sFqUbAsEgEirfSNfLNDg97K2JvxGcQCAQDDYBj4fWHTvQL1nSo0+SQquFQAC5rW3Qj/3b9w5wxuXhiWvPOv8vnRKcUNpWGf38YkWtnYlmPTq18MwaKaRpk0hWKqh3DEZSIugXEsnHq6RDDTxQtcTTWw7R2NrOk6IjxQVPtEmJ+4F0wAZogY+AI0AzsCo+oQkSgZDywaw142z34fL4u5VvQLA1ps9joMndhDfg7XV/FbX2EaOSgLPyxPIofSVKD1iRZURSQiAYRBzeYFJCq/GhUkhs3S9KOAQCQeLh2vU5AZcLQ3F3PwkIKiWAQS/hKKtp5v/77ATfmz+eqdlnx1hmo4YZualsifKaKssyZR2dGQQjB0mSMBvVWAdYvuH2+jl42hGV2fy4DB2TzPoBJSUqau28+Olxbpk3bkQY3AuGl6iSErIst8iy/DXgWuAhYB3wdVmWF8myHP+GyoIRS0j5kJmSGVYT9FS2UJSbhuwzIiPT2NbY477sbUGTy5F0YSrMNqKQiNpXYktlPdmpGgpGC3ddgWCwCCkl2v1tg1rHKhAIBEOJc/t2pJQUtPPm9bhcGYekhD8QNBHM1Ku574r8bstLCizsPdmMNYpZ9vqWdmyO9hFhRi7oisWoGXD5xsHTDrx+OerJwZJCC7uqmrC39T7Z2BuBgMwjb1SQrkvmp1dMjnl7wflHtC1BJwDIsrxdluXfybL8lCzLWzuWLY1ngIKRTci40qw1hy+G5h6UEuPStWikoC1JbyUc+2pHlp8EgDZZxURzdGaXbq+fjw43UFJoEe0rBYJBJOQp0eprHbQ6VoFAIBhKZFnGUVqKbsFlKNQ9l6iGlBL+QUxKvLzrBF/V2Fm1ogCjJqnb8qUdys7SA5FLOMo6Suemx9AOVDA0mA3qASclymIch5cUWPAFZD441Hdpdk/8c/dJ9pxo5ufLC0hN6X5eCi48oi3feE+SpG56dEmSSoDXBzckQSIRKt8waU1h2Zi5B08JhUJiQno2ANa2nm98sV4Mh4rpOWmU1dgjGk99crSBNq9flG4IBINMSCnR6m0dtDpWwcjg+PHjSJLE7t27+72P999/H0mSaGiIT7vY8ePH87vf/S4u+x4Ir776qkiAJxDtBw/iq6vDsKR7140Qg12+0ehs57fvHmR+XgbfnJHd4zoFow3kpKWwZX/kpERFrR2lQhImlyMQi1GD1TGw8o2KGjujtEnkjkqJav2ZY9LI0CXHXFJ5ptXDb945wJzx6Vw3O6c/oQrOQ6JNSrwDbJEkKfy02JGQeI2g34TgAsXqsqJSqEhTp3Uq3+h5BqAoK+iqW+fs+cZXXmtnTHoKo0ZYO6Ci3FQanO2cjpCB3rLfii5Zydy89CGKTCC4MAh5Sri8rkGpYxX0js1m40c/+hHjx49HrVZjsVhYunQpW7ZsCa/T34f0xYsXc9ddd3X5bMyYMdTV1TFz5syo9tHTsS+77DLq6urIyMiIOSaBYKhwbN8OkoR+8aJe1xnspMSvNx+gtd3HE9dO7TWBJUkSSwvMfHTEhtvr73N/ZbV2Jpn1pCQrByU+weBhNqpxuH24PL5+76Os1s70KEwuQygVEsVTzLx/0IrXH4j6OE+9e4AWt48nrp0mEquCMNEmJe4C9gGbJUlKkSRpGcGOHD+VZfn/xi06wYjH5rJhSjGhkBTUt7SjTVai78WR+ZKxY5BliYO2mh6Xl9eMLJPLECGPi7I+zC4DAZntB+pZNNmEWiVu1gLBYBJSSrh8wVZ5A6ljFfTN9ddfz65du1i/fj2HDh1i06ZNLF++nMbGnr2ABopSqSQrKwuVqv9O/snJyWRlZYnBrWBE49xeSsqMGaj6SJ6dTUoMvDxt9/Em/vVFDbdfnsdEc98+VyUFFtzeAB8f6V1tJMvyiB2nCc62Be2v2aXb6+dQvYPpObGpYJYWWGhx+/j8eFNU63954gwv7zrJfywYz+Qs4b8mOEu0RpcycAtgB0oJKiTulWX5z3GMTZAA2NpsmLQd7UAdbixGTa8Dwxm56cg+A8fO1HVb1uzycKLJxfSckVenWDjaiFIhUdGHr0TFKTv1Le0snSJKNwSCwSbkKRGQA7T52igpMPe7jlXQO83NzezYsYNf//rXLF26lHHjxnHppZdy//338+///u9AUO1QXV3NAw88gCRJ4et9Y2Mj3/72t8nNzSUlJYWpU6fy3HPPhfd966238sEHH/DHP/4xvN3x48e7lW94vV7uuecesrOzUavVjBkzhp/97Gd9Hrun8o2dO3dSXFyMTqcjNTWV4uJiTp06BcCHH37IvHnz0Ov1pKamMmfOHCoqKvr8bpxOJ9/97nfR6/VkZWV1U2usXbuWoqIidDodOTk53H777TQ3n21da7fbueWWWzCbzWg0GvLy8njmmWe6LL/zzjsxm80YDAYWLVrUraTlxRdfZNy4cWi1Wq666irq64VaKFHw1ltxV1SgX9Jz140Qg6WU8PkDrHq9guxUDfcsnRhx/bl56ejVqj4VaKfsbhpbPeGuZIKRRchkvr++EvvrWvAH5JjH4ZdPyiRZpWBrFOU/IdNVi1HNj0u6m64KLmx6TUpIkjS78w9QBPwKyAZeBL7otExwgWJz2TCnBGu8rS1uzIaeSzcAxqZrUQSM1Dm73/RCRpIj8WanSVIyyazvUymxtdKKQoIlHT2/BQLB4OHwOEhRBWtcXT4XM8eMIkOXzDZRwjGo6PV69Ho9b775Jm53zwPbDRs2kJuby+rVq6mrq6OuLphkdrvdzJ49m02bNrFv3z5+/OMf84Mf/IBt27YBsG7dOubPn89tt90W3m7MmDHd9v+HP/yB1157jX/84x8cPnyYV155hcmTJ/d57HP56quvWLJkCRMnTuTjjz9m586dfOtb38Ln8+Hz+bjmmmv42te+xldffcVnn33Gvffei1LZt8Jt7dq1FBQU8OWXX/L444/z8MMPs2HDhvByhULBM888w759+3jppZfYtWsXd999d3j5qlWrKC8vZ9OmTRw8eJC//vWv5OQEa6llWWbFihXU1tayadMm9uzZw8KFCykuLg7/jp999hm33nord955J3v37uXqq69m9erVfcYsGDk4338foNdWoCEGKynxwqfVHDjtYPXVhWiTI6uQ1ColC/Mz2VZpJRDo2T8r1BpdmFyOTEKl0/X99JWo6Oc4XKdWsWBCBtsO1Ef0Xvv7zmr2nWrhkasKe1VVCy5c+jojdgMy0HnaO/T+v4AfdLyWAaFXv0CxtlmZM3pO8LWjnRkdN6vaBx5EZTZheeCB8LqSJGFMyqTZ0312M5SUmJY98pISELxIb620Istyj0qQrfvruWRcOukdfhi1zlrueO8O/nLFX8jRJ46Jj9Pj5Ltvf5c1C9ZQZCoa7nAEArx+L26/m/HG8RxvOU6rt5XMlEyKp5h5d99pvP4AScpoKxGHl9/s+g0Hmg4M6TGnpE/hoTkPRbWuSqXi+eef54477uDPf/4zs2bNYsGCBdx4443MnTsXgPT0dJRKJQaDgaysrPC2OTk5PNDpen/nnXeyfft2Xn75ZZYuXUpqairJyclotdou251LdXU1+fn5XH755UiSxNixY7nsssv6PPa5PPXUU8ycOZM///msmLOgoACApqYmmpubufrqq5kwYULwO5oyJeJ3M3fuXFauXAlAfn4+n3/+OWvXruW6664D4N577w2vO378eJ566imuueYaXnjhBRQKBdXV1cyePZs5c4L3y3HjxoXXLy0tZe/evdhsNlJSgsm3J554go0bN/K3v/2NBx98kHXr1rF06dJuMaxfvz5i7ILh5+njz+G7xsgvJvatWlBotcDAkhIuj4+ntxxi8WQTV07t/e/kXEoKLLxdfpryWjszxnRPPJTXNqNSSEzph+S+yd3E9zZ/j7WL15I/SsyQxwOzMVS+0T+lRFmNnUx9MqNTu5vVR2JpgYXS1yuYuHIzfRXR+QIyl0/KZMX00f2KUXB+09dI7iIgr+Pfi3p4n9fpX8EFSJuvDYfHgVlrRpZl6lvcWIxqfGfO0PLWWzT/61Vkb9eab4vOhEdu7mamVF5jZ1yGllTtyGwLND03jaZWD7XNbd2W1Ta3sb+uhaUFZ1US+xv3c9JxkqPNR4cyzAFT66zlqP0olY2Vwx2KQACcNbm06IKlUa3e4GA91jpWQXRcf/31nDp1io0bN7J8+XI++eQT5s2bxy9/+cs+t/P7/fziF7+gqKiIjIwM9Ho9GzZs4MSJEzEd/9Zbb2Xv3r3k5+fz3//937z11lsEAtEbqAHs2bOH4uKeOxykp6dz6623cuWVV7JixQrWrl0bVYzz58/v9n7//v3h99u3b2fZsmXk5uZiMBi47rrr8Hg8nD59GoAf/vCHvPLKK8yYMYP777+fDz74ILztF198gcvlwmQyhdUqer2eiooKjh4N3kMqKyt7jEGQGOzV2CgbR0TfE0mpREpJIeDqv6fEEasTZ7uPf790TEw+K0smm1FIvXc2Kquxk28xoEmKfR6yyl5FdUs1e617Y95WEB1GjQpNkqLf5RvlNXam5aT2y5vn32blcN+yfP5rUR4/6OPnJyX5/P6mGcL/R9AjvSolZFmuHspABIlHgytYv5uZkkmL24fbG8Bi1OD84AMIBAi0tOD6cg+6uXPC24xPG83htlYqTjVyybizD/FlNXZmjR25ksCQsVNFrZ3cUdouy7Z33MBLCs/6SVhdwdq6kEFfouD0Bmv3Qw+CAsFwE/KTyNIGZ/xCSYlQHeu2SiuXTcgctvhiIVrFwnCj0WhYtmwZy5YtY/Xq1dx+++089thj3H///SQn99wd6Xe/+x2///3vWbduHdOnT0ev1/Pwww9jtUauM+7M7NmzOX78OO+++y7btm3j+9//PjNmzGDLli0oFIOjiHnuuee49957eeedd3jzzTdZuXIlr7/+OldeeWW/9lddXc2KFSu44447WLNmDRkZGXz55Zd8+9vfxuPxALB8+XKqq6vZvHkz27ZtY8WKFdx4440899xzBAIBLBYLO3bs6LZvo1G0XjwfcCR5aVdHl1xT6HQDUkpUNQS3zTPpY9pulC6ZS8als7XSyk+vmNxlmSzLlNfa+XoMyovO2NuDathTzlP92l4QGUmSMBs01PfD6NLl8XHY6uDKqf3zRdOpVdyzdFK/thUIQvTlKXGdJElJnV73+jN04QpGEta24GDTnGIOy8VMBjXO0vdRZmYiJSfjLC3tsk2BKReAnSfO5rxCCoSR6CcRYkqWAZVC6tFXYkullbxMHRM6DQBsrmCJSugBKlEIJVESLW7B+UvonAwpJVze4AxiqI51a2XkOlbBwCgsLMTn84V9JpKTk/H7u6rdPvroI66++mpuueUWZs6cyYQJEzh06FCXdXraricMBgM33HADf/rTn3jrrbfYvn07R44ciXofs2bNYvv27X2uM2PGDB566CHef/99Fi9ezAsvvNDn+jt37uz2PlQSsnv3bjweD08//TTz588nPz8/bKrZmczMTG655Raef/551q9fzwsvvEB7ezuzZ8+mvr4ehULBxIkTu/yYzcHkfUFBQY8xCBKDluQADpWXdn/kB0aFTjugpMRRWysKCcZlaCOvfA4lhWYq61qoOdNVqVFzpo1mlzfcjSxWWjwtgEhKxBuLUd0vpURlXQsBWfiFCIaXvqYdXgVGdXrd28+/4hmgYOQSevA2aU3hzKwlRUHrjh0YiovRzpuLY/v2Lg8MkzOD/gplp87KZUN+EiOx80YITZKSyVmGcKwhnO0+dh5t7FK6AcGuJHBWeZAohB4AQ7PTAsFwE1LtZOmCM3Qu39nB8tICC9WNLo5Yxfk6GDQ2NlJcXMzf//53ysrKqKqq4l//+hdPPfUUS5cuDc/ajx8/nh07dlBbWxvueJGfn8+2bdv46KOPOHDgAHfddRdVVVVd9j9+/Hh27drF8ePHaWho6LEsY+3atbz88stUVlZy5MgRXnrpJYxGI7m5ub0e+1weeOAB9uzZw5133slXX33FwYMHefbZZzlx4gRVVVX87Gc/45NPPqG6uprS0lLKysooLCzs87vZuXMnv/rVrzh8+DB/+ctfePHFF/nJT34CwKRJkwgEAjzzzDNUVVXx8ssvd+msAbB69Wpef/11Dh8+TGVlJRs2bCAvLw+1Wk1JSQkLFizgmmuuYfPmzVRVVfHpp5/y6KOPhtUT99xzD1u3bu0Sw2uvvRbpv1QwAnC1O/F0VKaGxk19MVClxDGbk9xR2n61J19aEEz+bj/QVeEUNiPv5zitpb0jKdEqkhLxxGzUYOuH0WVowm0kTw4Kzn96TUrIsqyQZdna6XVvP8Lk8gIlVKJg1prDmdnMI/sIuFzoi5dgKC7Ge+IEnmPHwtuYdcGH94ONteHPymuCbdOmxtgbeagpyk2lrMbeJcmy45ANjz9ASUFXyVvou0m0h3uhlBCMNELn5LnlG0A4Gbi1MrYSAUHP6PV65s2bx7p161i0aBFTp07l4Ycf5uabb+aVV14Jr7dmzRpOnjzJhAkTMJmCLaFXrVrFnDlzWL58OQsXLkSn0/Gd73yny/5D5R+FhYWYTKYevRwMBgO//e1vmTNnDrNnz2bv3r1s3rwZbYcBYE/HPpeZM2eydetWDhw4wLx585g7dy7/+Mc/SEpKQqvVcujQIW688Uby8/P5/ve/z3e+8x0eeqjv0pr77ruPsrIyZs2axapVq1izZg033HADAEVFRaxbt461a9dSWFjIs88+261lqFqtZuXKlcyYMYMFCxbgcDjYuHEjEJRdv/322xQXF3PHHXcwefJkbrrpJg4ePEh2djYA8+bNY/369fzpT3+iqKiIDRs28Nhjj/UZs2BkcKbp7HgnNGHRF0rtQJMSreSZdP3adoJJT16mji37u/pKlNXYSVJK5GfFVhISIqSUqHP23DFHMDhYDJp+KSXKa+yYDOpwW1GBYDgQ/VgE/aahrYFkRTLGZCP1juCNVr3rY3wpKejmzcPf3Aw8jrO0FHWHy7kpJTiIPO2sx+31o0lSUlZjJy9Th1EzMk0uQ0zLSeXlXSc52dTG2A5Z5JbKetK0SVw8blSXdRO1fCOk7Eg0hYfg/OXc8o3Of1OjU1OYlmNka2U9P1w8YVjiO59Qq9X88pe/jGhqOW/ePL766qsun40aNapLi8yeyM/P59NPP+32eedE7x133MEdd9wR07EXL17crYTna1/7Gh9++GGP+4gU57kcP3484jr33HMP99xzT5fPbrrppvDrlStXhjtn9ITBYGDdunWsW7eu13Vuu+02brvtti6f3XXXXRFjEwwvTY1n1QGhCYu+UOh0+GyRkxc9EQjIVDW0MjcvvV/bQ9Af67mPq3C4vRg6xmXltc1MyTL2S30BZz0lbG02vH4vScqRPd5LVCxGNa0eP852X0wtN8tq7RTlCJWEYHiJyjVKkiR1p9c5kiQ9LknSbyVJWhi/0AQjHWubFZPWhCRJWFvaMSQrcX/4AbrLLkOh0ZCUlYWmsBDH9rO+EqM0o1CgRFZhYEnoAAAgAElEQVS2sL8umDkvr7X3u05xKAnJFkMyRn9ApvSAlSWTzajOaUkY8ttItId7oZQQjDTCSQltV0+JECUFFr48cYYGZ/96swsEAkE8abKfVQfEu3yj3uGmzeuP2eSyM0unmPH6ZXYcDpZHybJMec3AxmkhpYSMzOnW0/3ej6BvQkqHWNQSre0+jtqcCTEOF5zf9JmUkCRpsiRJ+wCXJEl7JEkqBHYB9wF3AtslSbp2COIUjEBsLhtmbVA+bXW4meWz4aurw1C8JLyOvriYtj178DUF2/YpJAUZKZkoVC2U19ixOdqps7vD3S1GMvlZepKVCspqg+UmX544wxmXt1vphtvnTtiH+7CnRIIlUwTnLw6PAwkJQ7KBFFVKt7+pkgILsgylB0QJh0AgGHmccZxNRIQmLPpCodPhd/Vv7HDMFtxuQmb/yjcALh43ijRtEls7SjhONLlocfsGNJMeSkqA8JWIJ2ZjcA45lqTEvlMtyLLwkxAMP5GUEr8D6oBvAhXA28A7QCpBE8z/C/wsngEKRi5WlzVcjlHf0s7805UgSegXLQqvo1+yGGQZ5wdnZbRZOjPJGifltXYqwiaXI/9iqFYpmTLaQHmHIdDW/fUkKSUW5ndtR9i5ZjTRHu4TNZkiOH9xep3ok/QoJAValZZWX9dzc2q2kdGpGrZW1veyB4FAIBg+mluDigONQh2dUkKrJdDqirheTxyzBcccA1FKqJQKiieb2X7Qis8fCJsgThtgUmKMYQwgOnDEk5BSwhpDW9CyDl+3gfz/CgSDQaSkxDzgflmW3wJ+BIwF/keW5YAsywHgfwFT4hyjYIRia7Nh0oaSEm4Kj39FSlERqsyzD+mawkJUFgvOTu3ZzFozarWT8ho7ZTV2JAmmJsjFcFpOKuW1QbPLrZX1zMvLCNdchggNOpIUSbR6EuvhPuwpkWAGnYLzF4fHgSHZAIAuSdctYSZJEksLzOw43IDbG7ndpEAgEAwlZ9oaAZhgGB+V0aVCp0N2uZB76E4TiaO2VnTJSixGdeSV+2BpgYVml5cvTzRTXmsnWaUg32Lo9/5a2luYlDYJCYm6VmF2GS/MhtiVEuW1drKMGswGYXIpGF4iJSUygFMAsiw7gFbgTKflZ4D+X6UECUurt5VWbyumFBOyLOOrt2I+dQx9cXGX9SRJQl+8BOfHHxNoD2ZuM1MyCShaOGx18FlVIxNM+pgMeYaTopxUHG4f7x+0cdTW2q10A87KM8cZxyWsUiLR4hacvzg8DvTJwVk/XZKum6cEBEs4XB4/nx5rHOrwBAKBoE+a2+2kuGWyjblRe0oABFyxqyWONbRykUmHJEkxb9uZhfmZJCkltlbWU15jp2C0kWRVVDZ0PdLiaSEjJQNTikkoJeKIXq1Cm6ykPgalRKL4ugnOf6K5wsgR3keNJEkLJUl6U5KkWkmSZEmSbj1nuSRJ0mOSJJ2SJKlNkqT3JUma2t/jCeJH6MZq1pppdnmZdWofQBc/iRCGJUuQXS5cu3aFt/HITgJ4+eRoY0I5/oYu3E9vPQScbUnYmdB3c1HqRQn3cB9KSrT72/H6vcMcjUDQVSmhTdL2WFo0Ly8DbbIyXAM9Ugj0Y6ZTIBgsxPk3Mmj2OjC6Jcz6rNiSEv0wuzxmc3JRZv9LN0IYNEnMy8tgy/56KgbYmUGWZVraW0hVpzJaP1ooJeKIJElYjBrqHdEpJRxuL8dsrQk1Dhecv0STlPh7RyLhTUAD/KXT+xdjPJ6eoDfFj4G2HpY/CPwUuBu4FLACWyRJEmqMEUZIgmjSmqh3uJlbtw+veTTJEyd2W1c7dy6SVoujo4Qj5EMhqYLGR4mUoc23GEhWKSirsTMly0DuKG23dWwuG8mKZEbrRiecN0MoKQHCV0IwMnB6nRiSei/fANAkKVk4ycS2Smu31pDDhU6no7a2Fo/HM2JiElwYyLKMx+OhtrYWna7/hoeCwcEeaMXgVZKZkonD6+hR7dWZ/iYl3F4/tc1t5A3A5LIzJQUWqhpacbT7BuT71eZrwyf7MCYbydZlC6VEnDEb1NiiVEpU1CbeOFxw/hJJM//COe//3sM6UScmZFl+m6BZJpIkPd95mRTUmt0L/FqW5f/X8dn3CSYmbiZoqikYIYSVEilmTp5oZqbtMPI3/61HyaBCrUa/YAHO0veRV68Od+xIN7ppbEwMk8sQSUoFBaONfHWymWWF3Us34GyrVH2yPngzDvhQKRKjPMXhcZCuSafJ3YTT6yRNkzbcIQkucBweB5PSJgGgU+mo9lX3uN7SAjPv7DvNvlMtI8KwKzc3l4aGBqqrq/H5fMMdjuACQ6VSkZqaSmZmZuSVBXGlhTaM/uTw2KehrYGxSWN7Xb+/SYnqRheyDHmmwUlKLC0w8+ibQRXsQB5a7e1Bo0xjspHR+tFsObGFgBxAIfW/HETQOxajhq86zCsjUd7RTS6RxuGC85c+n5RkWb5tqAIBLgKygPc6Hb9NkqQPgcsQSYkRRWelxMFP38Yc8KE5x0+iM/olS3Bs2UJ7ZSWm0UGlxBiTlzNNUJhtHJKYB4uinFS+OtnM0h78JCCYsDGlmNAnBSWUrd5WUtUj/4Lf7m/HE/Bg0VpocjcJpYRgRNDZU6K38g2A4ilmJAm27K8fEUkJhUKB2WzGbO5e4iUY+ZxscvGbdw7wm+uL0CWI55FgZNKiaCcnoAurRK0uK2ONg5+UCHXemBBl541ny59lnHEcy8Yt63F57igtU7IMVDW0Msnc/5KQUDtQo9pIdiAbX8CHzWXDout5DCUYGBajmvoWN7IsR/QW2XOimZy0FDL0AzNGFQgGg5GUpszq+PfcouD6Tsu6IEnSnZIk7ZYkabfNFrlOTzB4WF1WUlQp6JP0JH/2MU6Vhqyvzet1ff3iRSBJOLaXYk4JDtJnX6TkvmX5aJMTa8D3rUvHcNuC8b3W4FldHUqJTkmJRCBUujFaNxoQZpeC4UeW5WD5Rh/dN0Jk6NVMzTbyRfWZHpcLBLHwz90n2VRWx5cnxPkkGBgtKi+pkjaslIjUgaPfSYmG4PoXRVm+8bf9f2PT0U19rnPfsnx+ekU+KuXATC4BUpODnhKA8JWIIxajBrc3QIu7b4VeRa2dd/ed5oqpIjkkGBmMpKREzMiy/GdZli+RZfkSk8k03OFcUITUAMgy6WW7+Cq7kBRt7+2EVOnppMyahbO0lFR1KkmKJPR6F3cVTxrCqAeHaTmpPHr1VBSKnjPQDW0NmLXm8Oxuojzch9qAhgYNiZJMEZy/uHwuAnIg7CmhTdLS5msjIPds4Dc9J42ymmbh4SAYMFs6TFOrGsR1UNB/vH4vbUkyaUp9uIW61WXtcxuFLuhVFWv3jaM2J1lGTVTKHq/fS5O7ieb2vmX+V0zN4s6FE2KK41xa2s8qJXL0OQDCVyKOmDraglr7aAsaCMiser2CdF0y95bkD1VoAkGfjKSkxOmOf89N2Vk6LROMEEK+Ce6yMjROO0cmzYq4jX7JYtz79uGrr8esNUflQp1ouLwunF4nphQTuqTgbEWiPNx3U0p4EiOZIjh/CZ2TYaWEKvg31ZtR3PScVFrcPk40xd5KTyAIUXPGxYHTwXPvmC0xrt+CkUnooT812YghyYBGqYk49umvUqKqoTVqlURDW0OX+OKJ3dPJU6JjfHGqVSQl4oXFGJwg7Kst6Cu7T7L3ZDM/X15AakrSUIUmEPTJSEpKVBFMPoSL2yRJ0gCXA58MV1CCngkpJRzbS/FLChoKZkfcxtDhOeF8/30yUzLPy6RESJZp1prD5RudO1qMZBzeYJxZumC1VKIoPATnL6G/nc6eEtB7oq+ow4ytrMY+BNEJzle2VQZnsjP1yRy1ieugoP+caQ0+/Kdp0pAkCZPWFLF8Q9mPpIQsyxyztUZtcmltC57jQ5GUCCslko1ok7SkqdOoc4ryjXhxNinRs1KiqdXDb945wJzx6Vw3O2coQxMI+mRIkxKSJOklSZopSdLMjmOP7Xg/Vg7qbZ8BHpIk6TpJkqYBzwNO4KWhjFPQN7IsY2uzYdKacJaWcsgyEaM5PeJ2yXl5JI0di6O0FLPWHL4pnk+EZJmJ7CkhkhKCkUI3pURIfeTr+W8q32IgWamgolYkJQT9Z2tlPXkmHQsmZgqlhGBANJ0JKgJGaTOAYEv0SEkJSasFSYopKdHU6sHe5iUvSpPL0KSQvd0e93K3Fk8LSkkZvn6P1o0WSok4Yu4o36h39JyUeOqdAzjcPp64dlpEI0yBYCgZaqXEJcCejp8U4PGO12s6lj8FPA38EdgNjAaukGU5MaaaLxCcXidtvjbSPcm0Hz7MR+aCcGa2LyRJwrBkCa5Pd5KZNOr8VEp0apUaugEnysN9qFzDnGJGISlE+YZg2An97YQ8JUJ/U23eth7XT1YpKBhtEEoJQb9xuL3sPNbIsgILeZl6TtnbcHv9wx2WIEFpag5WH6fpg34SJq0p4thHkiQUWm1MSYmQyWW0SolQYsQv+8MqyXjR4mnBmGwMPwBn67OFUiKO6NQqDGoV1h7KN748cYZ/fH6S/1gwnslZhmGITiDonSFNSsiy/L4sy1IPP7d2LJdlWX5MluXRsixrZFleJMtyxVDGKIhM6GZmOBI0AvvUUoDFEF07IX1xMbLHQ6q1FafX2WtteKLSuVVqSHLe6kmMmbbOs9J9dTkQCIaKkGt7N6VEH+fmtJxUKmrtBALC7FIQOx8easDrlykptJBn0iHLwuxS0H/OOILjpPTUoALRlGKKaHQJQV8JfyxJiVA70MzYlBIAdnd8k7j2djtG9dnW7yGlhDAkjh9moxrrOUoJnz/AqtcqyDJq+LEwtxSMQEaSp4QgQQjdzLR7DyGPu4g6XWZUSgkA7exZKIxG9AeD0r1IMsZEw+qyolFq0CfpSVGlICEljFLC4XWgkBRok7Tok/QJE7fg/OXc8o1InhIQ9JVwtPs43igeJAWxs7WynlHaJGaPHRWedRZJCUF/CXlKpKcFkxJmrRmXzxUx6a/Q6WJTSthaSVYqyBmVEtX6nRMjZ9rj2/Y2pJQIka3Pps3XNiR+FhcqFqOmm9Hl33dWs7+uhUeuKkQfRYcWgWCoEUkJQcyEbmba3YdoveQyAMxRJiWkpCT0Cxei/eIgwHlXwmFzBb02JElCISkSSnHg8DjQJekSLm7B+UuohOjc7hu9eUpAsC0oQLnwlRDEiM8fYPsBK0ummFEqpHAng2PC7FLQT5pdTag9MilpZ8s3IJq2oDEmJRpaGZehRdlLq/JzsbXZUEnBB9N4Jwda2lu6KCWyddmA6MART4JJibNKCavDze/fO8TlkzL5xvSsYYxMIOgdkZQQxExI3TDK7uP01EsAsBijK98AMBQvIe1US5d9nS9Y26yYUkzh9/rkxFEcOD3O8GyGUEoIRgIOjwO1Uk2yMhk4W77RV9nXJIsetUpBufCVEMTIF9VnsLd5KSkIdibXJqsYnaoRZpeCftPssWNoA2Vq8N5qTjEDkSdkgkmJ6Mtbj9mcUftJQDApclHaRcEY452UOEcpMVofbAsqfCXih9mgxtrSHi6R+dXbB2j3BXj8m1OFuaVgxCKSEoKYsblsaP1K9Pp0qkzjATBF6SkBoLv8ctLcwQx9NLWViURDWwNmrTn8Xp+kTxjFgcPjCHcM0SXrEsYLQ3D+4vCePSchOk+JJKWCgtFGyoRSQhAjWyvrSVYqWJh/NrGcZ9JxVJRvCPqJ3esIJiWMwYfysFIiQvexWIwuff4AJ5pcUXfegOCE0KS0SQA0u+OblLB77F3LN0JKCadQSsQLs1GDxx+g2eXl06ONvLanlh8syovpHBEIhhqRlBDEjLW1nrSWAPrFi6l3ehmlTUKtUka9vdJgwFR0Mcl+6bwq35BlGavLGh50QPAhKlG6WDi8jrBM3pBkEEoJwbDj8Jw9J4GwT0ukRF9Rbir7au34hdmlIAa2VlqZNyGjS711XqaeYzanMOUT9IvmgBNDuwIpKQkgPGkRnVIiuqTEyTNteP0yeZnRKSXa/e3Y2+1clHoRSkkZV6VEQA7g8Di6JCVS1amkqFKoaxVKiXgRUi/XNrex+o0Kckel8KPFE4c5KoGgb0RSQhAzp21VjGrxYyheQn1Le9Qml50xLlnKqJYApxuOD36Aw0Srt5U2X1tYngmJVQbh8DjCHUN0SbqEiVtw/uL0OLskJSRJQpukjZiUmJ6TSqvHT1WDOIcF0XHU5qSqoZWSAnOXz/NMOhxuH42tnmGKTJDItODG6E8Ov9cl6dCqtBFLV2NJSoQ8T6KdBQ8lRCxaC6nq1LgmJVq9rQTkAKnq1PBnkiSRrcsWSok4EhqX/3rzAQ5bnTx29VRSkqOfPBQIhgORlBDEjNVxmlEuBbrLLsPqcEdtctkZffESRjmh3no0DhEODyE5ZjelRII83J/rKZEoZSeC85dzlRIQNLt0+fqutS7KFWaXgtjYuj/YunFph59EiLNml+J6KIidFkU7qXLX8laT1jSoSolQd5gJUXpKhBIiZq057kmJUFvnzkoJCPpKCKVE/LAYguPyj440UFJgoaTQEmELgWD4EUkJQUwEAgGacGI2ZqPQaqlvcWOJwU8iRHJuLhnosZ5H5RuhQUYXT4lkfcJ4M5zrKdHma8MX8A1zVIILmXM9JYColBITTDpSkpT/P3vvHuVIdt/3fasKhWfh1QD6ge6Z6Znd2ce8+BDJ5VIb7tK7dCRFkakofhw6kkPajKQTJ4rlhKFIKbYOKTmMaUUydSTLOXEiiceWePSgQ4mSxV1xlg9ptVxSy3nu7nB6enYa6MYbKKBQQFWhKn8UbjW6G48qPLoL3fdzzhwue9DomhkAde/3fn/fL67RsEuKTZ6/ncOFlQhWY3srFR/qnj5P0sDxn27u4AYVyE4cmq6h4dEQYYJ7vp4KpGy0bwRhtNswtL334KbaxG/e/E109I71tbsFCfEgj1jQu/9p+kLWKqlgCnFffKaiRK1tvu572zcAM1eCtm/MjsXu+IafZ/HP/ssLR3w1FIo9qChBcUS9VoDiARaXz6GjGyjUxxvfAIDFaBplT3v0A+cEsshIBpLW1+bFKaEbOhrqrlWebASpW4JylPR1Stioq/VwLC6kI7SBg2KLsqTgW/crB0Y3ACAdC8DrYbExZtilrhv46d99Ff/2qxuTXiZlziAugahnr7CaCqZsjW8AOOCWeOHNF/CZVz6DVwuvWl8zmzechVwCZhPIUTolau3a0CYlyvj4eQ7vfSSFT/zA4zi1EBz9DRSKC6CiBMUR1dIWACAWTKDUaEM3nNWB9rLgi6PlBeT28dj49loiCQIvoKk195xquBFJlWDAoKIExVXsz5QATFHCzkL28moUN7MiDbukjOQrr+WhG+hrceZYBmcTobGdEhtFCZLSQaVJMylOGqTVIsbv3ZAvBhZRaBaGhqcOEiU2xU0Ae5srNoqS7ZBLwDxA4VkeUV8UcX98pu0bYrsrzPRkSgC0geMw+K0Pvws/+uT6UV8GhWIbKkpQHFGt7AAAoqEF5Oumy2GcTAnAFCUAoFB8czoXd8QUmgUEPUGrthDY3dyPmoE/aupKHQCsDSD5M8yDy4NyPFE7Klqd1gFRws74BmA2cMhqB3cnsN1TTgbP385hKeLDpXS07++fS4XGdkpcz5gbvjINyjxxEAdCrLvWIaSCKbQ6LdTV+sDv5QaJErVNALub+XpLRaHeduaUaBawGFwEwzCWU2JW7TKDnBJpoStK0BEOCoXShYoSFEdURXNEISIkkRNbADD2+EYiZI45lMpb07m4I6YgF/a4JABYbRZudxwQUYKIKNQpQTlqyIL9QKaEx74oAYDmSlCG0tY6+OobBfyNx5bAskzfx5xLhfBmqQm1ozt+fvL6qzbVia6TMn+UJdM9GQ8m9nzdTi2o5ZRo7j3QIE4JEhJJAljP2Qy5BMxQbjJmGvfFoeoqZE22/f1OsDIlBogS2w0adkmhUEyoKEFxRLVRAgDEo0vIiV2nxBhBlwCQEEyrbLF2PG5KhWZhT/MG0OM4UNx9WnvAKeGdj+umHF/2vyYJIX50+wYAnE0KCHo5XN+anTWZMv+8tFGGpHTw/gsH8yQIZ5MCNN3Ag7JzxxvJNaFOiZNHpWo6S+PhveuCVMD8/8PCLvuNb+iGjjdF01lKnBKkecPJ+AZxSgBAzGc2FVXaFdvf7wRREeFhPQh49gbIJgNJ8CxPnRIUCsWCihIUR9SapigRi69YTonUmKJEMrYKACiJuelc3BGTb+atxQaBnPK6fQyCXB85zQjz5kaQOiUoRwURxMYJugTMLIBL6Siu0dYDyhCev5VDgOfwnoeSAx9DTqGd1oJ2dAM3syI4loGsdtBS3Z0tRJkulbopOsQiewUvcngxLOySiBKdHlFiR9pBq9MCA6bHKdEAywCnE/bDDAvNgrVWIVkPswq7FBURUW8UDLPXhcQyLJZDy9QpQaFQLKgoQXGEKJtqejyRRr7eQlLwgufGexklF7qihDS8GmseMAyj7/jGvGQzWOMb3XETct3DZl4plFlCZpH7ZUq0O21bdbWX16K4lRWhjWG7pxx/DMPAC7dzeOp8En6eG/i4h5LdWtCis8/xu4UGZLWDt5/unkbTsMsTRaVRBK8ZECL9nRK2xjd6RAkyunExcRHb0jYMw8DdooRTC0H4PINfv7001Sbqat0SRuJ+M+9iVmGXYls8UAdKoLWgFAqlFypKUBwhtkVwHSAoLCAntrEYHi9PAgCE+CJ8ioFyqzzFKzwaREVEu9OeW6fEfqu8lYWhUKcE5Wgg75n9mRIhj7lYtzPCcWUtiram407e3e8/ytFwa1tEttbC+x8/2LrRSzTIIxHyOnZKkDyJ95437wsVieZKnCSqchmCDHjisT1fD/JBCLxgyymxR5Tohlw+mX4S7U4bpVYJGwVnzRtFuQhgN9di1k6JmlI7kCdBWBFWqFOCQqFYUFGC4ghRa0BoM2BZFjmxNXYdKACw0SgiTaCszP/MNznx2J8pMS+be0uU6I5tBDwBMGBcL6ZQji/kNbl/QUtcPHZqQS+tmgvu6zTsktKH52/lwTDA+x4bnCdBGKeB4/pWFUEvh+850z2Npk6JE0W1XUWkCXCRg5vyVDBlM1Ni93NuU9yEwAu4nLwMAMjUs7hXbDhr3ugKIeQAJd5tBpnZ+EZbHChKpENpFOQC1A4V6ygUChUlKA4ROxIEzbQJ5sT22M0bAMB6vYjJLCqd+R8RyMvm4mK/U2JexjcaagN+zg+e4wGY8552Z/cplFmwf6SIQN5Tdl6bZxMhCD4PrmXmX/ikTJ8XXsvhraditnKRziUF506JTA2X0lEkBPP5y1SUOFHUtDoE2QDbR5RYDCwOHd9gfD6A4w44Jc5EzljNFbcL99FSdUfNG+RnEqdExBsBA2a2mRK+/lW7K8IKDBjYkXZm8rMpFMp8QUUJiiPqaEHo8NA6OkpSG4sTiBIAEFU9qBjzv/Hdf6MnBD1m+JTbN/d1pd538+d2MYVyfKkrdTBgLBGCEOTtv6dYlsGl1QiuZ8SZXCNlftmptXBtq4bnRoxuEM6mQig22hBb9k51tY6OW1kRl9eiiIdMsbdCa0FPFNWOhLA82CkxbHyDYRiwodCBTIn16DpWhBUAwO2C2cRx1sH4BnFnEFcnx3IIe8Ozy5RQhjslACAjZWbysykUynxBRQmKIxpMG2H4UGwoMAxMNL4BALGOD1W2NaWrOzrI4oJ0fxM4lpuLzX1dqR8IFBR4wfViCuX4QoQyltl7m3LilACAK2sx3N4WoWg07JKyywuvma1PdkUJMrdv1y1xJ99AW9NxZS2KWMALAKjQWtAThQgZEY0Hwx0MoSTjG4ZhDPz+XlFC1mRsS9tYj6wj4o1A4AXcq20BAB5yOL7h43zWqCZghl3OwinR0TuoK/WBQZdEXKG5EhQKBaCiBMUhdU5DmAladaCTBF0CQBxB1DwKdGO+NwyFZgFhPmyd4vYyD2MQdaW+Z5ECACFvyKplpFAOm4baOPCaBJxlSgBmroSi6XgjN/9jYpTp8cLtPE4tBPDIkr0NHZnb3yjY+0wkOSaXVqPwelgIPg9t3zhB6IaOOttGxOi/RloMLELVVdTag/Nu2FDQEiXeFE1XxHp0HYC5od+RthHyclh0UMtOqst7KzqjvuhMRIn9VeP7WQ4ugwFDGzgoFAoAKkpQHNLgO4hwIUuUmNQpEWfD6LC78+PzSkEuHAi5JAi84PrNfUNtUKcExVWIinjgNQnstm9Imk2nBAm7zNCwS4pJU9Hw9e8W8dzjS3s2Z8M4vRAExzK4ZzPs8lqmCsHnwdmE+XqNh3hU6fjGiaGu1KEzQJQJ9P39ZNB0VZI8qn6wwV2nBKkDPRs5C8AcfagoOZxLCbZfwwD6VpfHfbNxSohtc2xuUKYEz/FIBVPINqgoQaFQqChBcUCno6HpMxDhw8jV2wAwUdAlAMR5U0EvyaWJr+8oyTfzw0WJORjf2J8pMQ/X7ZQH4gMaqjUnNJTGgdck4CxTAgDOJIII+z1UlKBYfP1OEYqm2x7dAACvh8XphaDt8Y3rGRGXViNgWXPDGA96UabjG3PHy/fK0PXBIxaDIJv8KHdQWAVMpwSAoWGXvU4JUgd6OnIaALASWkHLKDkKuSQ/b/9aZVZOCVExRYlBTgnAFFe2JTq+4WZERcSN4o2jvgxKD4ZhoPp7v4f2xsZRX8pUoaIExTZieRsGwyDijyAvtsAyQCLkneg5E94FAECpNd+iRKFZsBYZ+5nbTAmv+x0eTvnoVz+KT770yaO+DIoN+r0mAeeZEgzD4MpalNaCUixe2igjwHN45/qCo+87mwzhro3xDUXTcXtbxJW1mPW1eNBLK0HnjFtZEX/nN/4SL94ZLBwMotKqAABiAzbkRBgYFnbZmymxKW5iObSMgMd0XiwGlgFWxtqCfZcEsDu+0UvMFyhhekMAACAASURBVBs6RjIu5DmHiRIrwgp1Sricz936HD74xx/EzdLNo74USpdOtYrtn/05SF/72lFfylShogTFNpWKqWZH/QvIiS0kBR883GQvoYVgAgBQlpzf9N2CYRjIy0OcEl4BkuLuMYi+mRJzIKY45X79vjWbS3E3gzIlfJwPHMPZzpQAzLn+13ZEtLXONC+RMqdsVZo4tRCA1+Ps/nUuGcJmSRp5cv5Grg5F03Fpdde2Hg/ytBJ0znhQMT9jslXZ8feSDXnMF+v7+0QYGOaU4HpFidom1iPr1u95DHPtFI3Yv0dLqoSm1jw4vuGPQ9ZktLTpho7bdUrkpBw6Ov1sdis70g4MGPiFl35h7vPfjgtq1hTyPOn0EV/JdKGiBMU2lXJXlBASyIntiUc3ACARNu2zxer8KuXVdhWarh04fSC4fXPf7rSh6ErfTImm1jw2i4WG0kBdqWNb2h6aeE5xB4MyJRiGQZAPOso7ubIag9ox8PrOfGfXUKZDpipjNdZ/1n8Y51ICWqqObG34JpWMCl3pESViQS+qEs2UmCfy3eysUsO5mFRpm06JeLC/G8fv8SPijVgVnf0gTgnDMHBfvL9HlFBa5mvL57fvcCACSL/xDQBTH+EgosSgTAkASAtpaIY21DFCOVrKrTI8rAfXi9fx+3d+/6gvh4JdUYKnogTlpFKrmzfPaDiJfL09ccglAMQjS2AMAyUxN/FzHRX7e7/34/bASBIyun9+32o50OyfSLsZkvDd7rTnflzouKMbOiRV6pspAThvtLmyRsMuKbtkqjJW4+OIEuZn4qiwy+uZGsJ+D84kdtuYFkJe1Nsa1A49aZwXcqKZnVVqtB1/b6VRBADEhP5jnQCwGFwcOb7RaTZRapVQV+tW8wYA1OrmZ6PhKdu+JvKz9o+axn1xALMTJQZVggJmNgYAmivhYsqtMt659E68Y+kd+OVv/TLKLfuvOcpsUDMZAFSUoJxgag1zIxeLLCIvtrA4BaeELxaHIANFafBpgduxbvTB/osPwWuKEm61vRFRop9TArA/u+92eudWaS+6u2mqTeiGPtD2G/KEHIlla/EAYkGe5kpQ0GhrqDZVrMYO1jePgogSo8Iur2/VcGUtuqcVIR7kAYDWgs4RpGWsOEZAaaWeB9cxEIn0P6wAzBGO4UGXIUBVca/0XQC7zRsAsF3yAAaPomz/QIccoJDmD8KsnBK1dg0+zgcfN/gAa1VYBQCaK+Fiyq0yEoEEPvHEJ9BUm/jlb/3yUV/SiUfNZsEEg+Bi/cfD5hUqSlBsU2ua6qgQWUZJUhx1Yw+Ci0QQk4DyHLdvWJbIAeMbAi/AgAFZcz6XehiQMMsDmRLe0J7fn3d6Fz0ZKXOEV0IZBRl3IsLYfpyObzAMg8urUVyjosSJJ1MxP4fHcUqkBB/CPg82hoRdtrUOXtsR9+RJAEC8GwpNa0HnB9IyNp5TogBBBjyxwaMLqWBqZCUoANwrvgEAOBM9Y/3evVITXixYDkA7kLXKfqcEyb2YhVMi6h385weA5dAyAOqUcDPlVhkJfwIPxx/Gj174Ufzhd/8Qr+ZfPerLOtGo2Sz49IqjOuB5gIoSFNvUZPOGpflMlX0amRJsNIqIZKCsTL+O6rAgTolB4xtkDMKtm/tRTgk352E4YVvaBsdw5n9Tp4SrIbbffpkSgHNRAgAur0bxRq6Olno8MlIo45Gpmg6bcTIlGIbB2VQIG0PGN17fqUPtGLiyuvcEKx40RQlaCzo/kEyJ4hiZElW5grBsHrwMIhVIodgsDnRRsqGuKFHZgI/zWaMOhmFgo9BAlF90dC/Ly3kEPAFrTUKI+7vjG60pixJtcejoBmB+lsd9ceqUcClNtQlZk7EQMLNRfuItP4Gl4BI+9dKnoOnaEV/dycUUJY7X6AZARQmKA0RFBK8BtY7pkJhGpgQXjSLaBCqaOPFzHRX5Zh5RX3SgRdHtYxB1tX+mhNuv2ynZRharwirCfJgugFwOEfAGZkp4nGVKAGauhKYbuL09v581lMkhTom1MZwSgNnAMWx8wwq5XNt7Qhzrjm/QWtD5IWcFXTp3SlTbVYRlgI0Md0pohmbVh+6HiBKb4iZOR06DZcwle0lSILY0LAVWHDslFoOLB05XiZthFk6JYc0bhBXB2Z+DcniQ/K0FvylKBPkgPvrOj+L1yuv4ndd+5ygv7USjZagoQTnh1LUGQgpjnR4shid3SnDhMCJNoGLM78a30CwMHN0Adp0SZPPvNohTYv/igYgSbr1up2xL20gLaawIK9Qq6nIGvSYJIT7kqBIUAC6vmSfXN2jY5YlmqyrDy7FICeOJ6udSAjJVGbLS33FzfauGWJA/IHosdMc3KnR8Yy5oax1Umiq8HhaVpgrNYUBpTasjLBvghoxvkByqQWGXRJR4s5nZ07xBRLEz0VWUW2XbVZ75Zr7vWoXneAi8MJNMCTuiRDqUpu5Fl0JCLYkoAQDvP/N+fG/6e/Grr/7q0EwUymzQJQmdWg18evWoL2XqUFGCYhtRbyKseqxE6mmMbzA8j5jCQ2IUKJ35PEEqyIWBIZfA7mmvpLhTeLEyJfaPb7j8up2SbWSRFtJIC2l6KuNyLPfOkEwJp60w6agfCyEvzZU44WQqMlZifrDseLO4JOxys9T/c/HaVg2XV6MHTqPp+MZ8UejmSTy2bN4Xyw4dLrWOBMHG+AaAgRs7NhSCxgIZJb9PlDDv2Y8lTwOwn8dQkAsDx0yjvuhsnBIjxjcAWAcFtKrbfZRlU5RI+BPW1xiGwc888TNQOgo+88pnjurSTizHtQ4UoKIExQF1tCDoPHJiCxzLINE9+ZmUGMwTpXmtGco380gGkgN/38qUcGk2g6iIYBkWQc/eNHq3X7cTSA3oSmiFnsrMAYNyTghOK0GB3bBLWgt6sslU5bHyJAjnkqZQ1m+Eo6V28EaujsurB0/H/TyHAM/R8Y05gRy+PL5sbqpLDnIlDMNADU1EZIAV+gurgD2nRD4GdKDjbHS3eWOjKMHrYXFh0Qy+tHM/MwwDRbl4IOSSEPfFp58pYXN8Ix1KQ9bkqYsilMkh6/JEILHn62ciZ/DhSx/Gl+59CS9vv3wUl3ZioaIEhQKgzigIw498vY3FsG/sk6b9LLDmTZvMrs0TuqGbN/phTgmXZzM01AYEXjhwskdECrdetxPIoo04JRpqwwpTpLgPO6KEqqtQO86s8FfWzLDLQdZ7yvEnU5lMlDibJLWgB8Xa29siNN04kCdBiAd5lCU6vjEPkDHVC2lzU110kCshqRI6jIGw4QfDDl5mk8MMUtW5HzYUQnbBvC+fiew2b2wUJJxNhLAW7tZp2nD+NdQGZE0e7JTwT9cpoeoqJFWy7ZQA7P05KIcLESVIGGov/+jyP8KqsIpP/dWnHN+LKeNjiRKrVJSgnGAaHhVhJoCc2MLiFEY3CHHOvGkRm9g8UW6V0TE6tjIl3Oo4qCv1vps/juUQ9ARde91OIIudldCKlWBO3RLupaE04ON88HL93VjkPTVOA4duALdo2OWJpK11kK+3x6oDJQS8HNJRf98GDpJXQvJL9hMLeqlTYk7I7RMlnDglKm0zuDLKBIc+zst5EffFh4xvBJHtHlCvR9etr28UGziXCmExuAiO4WwFN4+qLo/5YlMVJUblAvWSDpmbK3pPdh+lVgkCL/QNcvd7/Pj4Ex/Hvdo9/Nat3zqCqzuZqNkswPPwpAbvO+YVKkpQbCPxOiKeEPKi6ZSYFgtdBXYexzeKchEAbDkl3Lq5byiNgSfSAi8cS6cEANrA4WJERRz4mgR6XDyaQ1Gie4J9fYvahE8i21VzozmJUwIwwy77OSWubdWwEPIiHe0v2i+EvKhQUWIuyNXb4DkGD6fM+7cTp0StbYpTMS404pFmA0de7u+U4LpOibgRsDb3akfHm6UmziZD8LAeLAYXbWVKkJ8xyCkR98WnKkqIbVP4jfoGB30S6D3ZvZTl8p6Qy/28d+29eN+p9+E3rv0GFZUOCTWTBb+8PNSFNa8cvz8RZSaoSgtNHxDhI8jVW1OpAyUsdAN05lGUILbLQTd6wHQcBDwB1wZGioo4MFAw5A1ZQZjzTFbKgmVYLAYXLacEtYq6FzJSNIhxnRLLET+Sgg8vbZTx3Xxj6C8nm5CTitpRbSf/u4FM1awDncQpAZhhlxtF6UAw3/VM/5BLQizIu6Z9o95S0dFpsOAgcmILi2E/YkEePMeg5CCglGzuo/xol0AqkBrolGC8XmwnWKxpu8/zoNyEphs41xVLVkIrjpwSgw5Qor4oJFWamg2fjEfacUpEvBEEPUF6T3Yh5dZwUQIAPvauj8EwDHz6m58+pKs62ajZLPjV49e8AQCeo74AynxQLZsKqOCLoNpUp1IHShAiCfAaUJLnL1PCutEPCI8iCLzgXqeE2sCq0P8D7jg5JRaDi+BZHgv+Bfg5P1X1XUxdqQ9dzBJRwmktKMMweOupGP705g7+9ObO0Mf6eRYvf+I5RPy8o59xkvj0Nz+NNypv4Le+fz6su5mKKUqcig+31Y/iXDKEektDsaEg1XUNyooZcvn+C0sDv88tTomdWgt/8/98Ef/9+x7Gjz/90FFfjivJi20sRnxgGAaJkA8lByJlpWWOb8T8/cd4ekkFU7hTuTPw97MJBu9p7b5eScAqaYFJC2l8K/etkT/HOkAZML4R95mO1Wq7OvSQxS5ORAmGYcxWLOqUcB2lVgmnw6eHPiYtpPGRKx/BZ//6s7hTuYPz8fOHdHUnEzWbReipp476MmYCFSUotqh1RQlvN/8hNcXxDU8kiljDQKlZnNpzHhbEEjmsfQMYry3gsBiUKQGY1+1WMcUJWSlrza0yDIPl0DI9lXExDaVhVdL2Y1ynBAB86gOX8ENvHR4Q9d18A//6hTu4kanhPQ8Nf2+fZG6Xb+NO5Q4MwxjoDnATW1UZLAMsDxivsAs5pd4oNKx74a1tEbqBvs0bhFjQi5psOhS4KQVFj8Mn//gWxJaG13bqR3YNbicntvBQ9985IXgdZUqQ8Y14MDHikaZIUGwV0dE74Fhuz++Jioha0MBqbne9tVE078cPJXedEvlmHpquwcMOXtIX5AIEXkCQ7y/IRf3m63ZaogT5O7ATdAmYfw671aaUw6PcKuNti28b+bh3r7wbn/3rz2Jb2qaixAwxFAVaoXAsmzcAKkpQbFKt5QAAHo+p/E+rDhQAuGgEkU2g3Og/V+lmCs0CFvwL4Lnhp6kCL6CuunMBOCxTIuwND7SWzhPbjW28bWn3xpoWaC2omxEV0Upk7wdZWI8jSixH/fihtwy/oZclBf/6hTu4vkVFiWFsN7atKr9+6exuI1ORsRTxg+cmm1y1GjiKEp44Z248SU7J5QHNG4DZvmEYQE1WsTDFe6gTvnangD++tg2G2XWOUA6Sr7fxnofMf9uE4HM0zlVpVcDoQEQY/dmxGFyEbugot8oHxIDN2iYAIC3uihX3ihISIS+iQXPNsSqsomN0kG/mrWyGvn+eZn6o2BDzmWu7aeVKEKdE1Ds6UwIw78nfKXxnKj+bMh06egfVdnXk+AYA6zHz6HieJ9SdHcAwjq0oQTMlKLaoiKYoYRBRQpieU4KNRBBtGnP5YVZoFoY2bxBC3pArMyV0Qx86v38cnBKariHXzFlOCaA7h0udEq5lVpkSdlkIebEWD+Bat02BchClo6Agm4LlvLyXMtXmxCGXgBmU6fOwe8Iur2VqSAo+LA9ppiJCRNlBPsE0aWsd/G//8SbWE0H8wKUVK2ODspeW2kFNVq2WsWTIi6IDp0RVKkJoGfBG7Y1vALDeS71sipsAgHR511VztyBZoxtAT53miNGHQrMwdMy0d3xjGpCgSydOCVERXesoPYlU21Xohu5MlGjN3zp+nlAzGQCgogTlZFNrmB80Cmeq3qkpihJcNIaIBJS7NVrzRF7OIxkcfRri1kwJSZVgwDjW7RuFZgEdo7Pn5D0tpFFulecqpO8kMTJTwtPNlNCcZUo44cpa1Kp4pBxkR9rN5JgX19FWRZ445BIAWJbB2WTImu8HgOtbNVxZGxxyCZjjGwCOrBb0//rqBu4VJfz837qEs8kQdsQWtI5+JNfiZvKi6YogLWMJwYuS1D4QbDqIilSEIJsHLqMgQkE/R+JmbROczmCx1LG+tlGQLKcO0FOnOWL0oSAXhq5VSEvGNJ0SAU8APGsvk4c2cLgPEj6/EBgtSgT5IAKewFwG1s8TatZ8f/CrVJSgnGBqTfODRmZNNT0hTHd8I9oEKppo+6bvFkadPhDcmilBusSHZUpIqgTdmN+Fa6ZhKsv7nRLA6IUc5fBROgranfbMMiXscnk1hvulJmouaUtwG73uiHnYSHR0Azu11lScEoAZNHivaL7+pLaGu4XG0DwJAFjoihJH0cDxoNzEZ//8u/iBy8t4+pEUVuMB8+9EpMLsfnJ18+9kiTglBB9aqo6m0hn2bRYVuYyIDHDR0aMLxCnRrxZ0U9zEshIA2zDFV7GlothoW5kmgD2nhGEYI9cq1vhGazqiRK1dsxVySaD3ZPdBBIaEf3Q2CmC6JagoMVvUTBZgGPBLgwOV5xkqSlBsUe+GFtWMKPw8i6CXG/Ed9uEiEUQlAxo61hziPNDROyi1SrZCocLesCudEqNECYEXYMCArM2vzZcscvY7JYD5OeE9SYx6TQIAz/HgWX7GooS5obhO3RJ96X3vzMNGIie2oOnGVJwSAHAuKeDNchNqR7cVcgmYlaAAUDmC8Y2f/+JNcCyDn/vBCwBgiTM0V+IgOXGvKEHGVe2GXdbaNQiyAS4yWpRIBBJgwPR3SoibOKWGoUvm55zVvNHjlPBxPiT8iaEjVKIiQtGVoWsVv8ePgCcwVacEcV/YgTSAzYPAeVJwKkok/AmUZSpKzBI1m4VncRGM92gyiWYNFSUotqi1RfhUoNL2IBHyTTVp3cyUMP97nlTWcqsM3dAdOSXc5gQhG8CBmRJec/HTUNwnqNiFLHLISQyw65qYl1n4kwQR74ZlSgCmXfQwRIlrmeks0o8bWSkLBgzWI+tzsZEg+QnTdEpouoE3y01c2zKFq2Ehl8BupsRh14I+fyuH52/n8VPPnsdK1PzzE3GG5kocJNcd31iK7I5vAEDBZthlVRURbpou0FHwLI+4P25VdhJ0Q8eb4ptYM2LQm+YC6V63eaPXKQFgZJ2mVQc64gAl6otOVZRw4pRIBBLgWZ7ek12ENb5hI1OCPG6e1vDziJrNHts8CYCKEhSbiJ0GQgqLQqON5BRHNwDilDD/e54+0Ijd0o5TQuAF6IbuOscBESUGLR7IxtCNoyd22Za2seBfQMCzuxlJBVPgGG4uNlMnjVGvSULIE0JTnV2mRDTI40wiiOtb1CnRj2wji1QwhVPhU3PhlCCOgLUpOSWsBo6ChOtbVSxFfNbJ+iCCXg5ejkX5EEUJWengn3/xJs4vCvjwU2etr1OnxGDyYgteD4towHS2JEPEKTFalDAMAzVdQlg21zZ2WAwuHgi63Ja20e60cZpLQpfMA42NggSOZXB6YW+t56g6TeLCGHWAEvfFpyZKOB3fYBnW/HNQ96JrKMklcAxnO6w0EUjM1Rp+HqGiBIUCoK43EdY4lBrKVJs3AIDhOMRg3mTn6QPNutEH7TklAPdt7q1T6QHz++S63Th6YpdsI7snTwIAPKwHS8EleirjQiz3zpBMCWD2TgnAdEvQ8Y3+bEvbSIfS5intHLyPiCMgPTWnhPn63Cg0cD1Tw+XV0U0LDMMgFuRRlQ4vU+LXrn4XWxUZn/zApT1VqH6eQ1LwUqdEH3JiC0uRXUcocUqUbIzdyJoMBRrCsgHWRvsGAKQCqQPjG/dr9wEAp/lloNOB0W5joyDhVDwAr2fv0p1UXA/KfrJ7gDJ1p4TNzSxhRaCtWG6i3Coj7o+DZextFYlTYp4zyNyM0elA3dmhogSFUkcbYd1MoJ62UwIA4h5zfnye5tEsS6SNSlDiOKir9Zlek1NIhseg+X3y9XkWJbal7T15EoQVgZ7KuBE7mRJAdyRKm70osVWRj6zC0c1kG1msCCtYCa2g1q65TnDdz1ZFxkLIi6DXM5XniwZ4JAUvrmVq2ChKI/MkCAsh76GNb2wUGviNFzfww29bxbvPHZwLX40FqCjRh5zYxlJ41/ViiRI2nBK1bv5WuM2CDQVHPNpkMbh4YHzjnngPAHAmYG5AdEnC3ULjwOgGYDolFF0ZeKhDBI9Ra5W4Lz61oMu6UkfUaz9TAjDHKuk92T2UWiXboxuAKUp0jI5VB0uZLlqhAGjasW3eAKgoQbFJnVUgwD8TpwSwm/w8V04JuQAGDBKB0SFA5NRXUty1cCdZEWF+cPtG7+PmDcMwrBPd/aRD83HCe9IgAtig1yQhxIcgq7PdUJGMAOqW2EtH7yAn5SynBOD+gLpMVZ5angThXFLAn9/OwzDMClk7xIL8oYgShmHgn/1/N+HzsPiZH3is72NW4wE6vtGHfL2FxcjuOsfn4RD2e1C0EXRJnAZRBGxnb6WCKZRbZWi6Zn1ts7aJMB9GImQKCVq9gc2StCfkkjDqPZhv5hHxRuD3DB8vmpZTQukokDV5LKdEQS5A6VAR2A2UW2XHogT5Psr0sepAqVOCctJpeDSEEICmG0iEpu+U8EViCCscSq3S1J97VhSaBSz4F+BhR5+8uXUMoq7U4ef84Ln+XeLznilRapXQ7rQHOiXyzTxUnVY+uglHTokZvy4vkQaOLRp22UtBLkAzNKSF9NxU+WUqzemLEqkQZNWsibzkyCkx+8+cL13fwdfuFPE//+ePYjHcfzNKnBJuC2A+avJi+8DfWVLwoWjDKVFpVwAAUW74+FkvqUAKBgyU5N31z6a4iTORM+AE83lyuTJaqj7QKQEMDm4uyAVbjs64Pw5REfeII+NAHJhOMiWA3QDqHWlnop9PmQ5luWzr0I2wEDBFiXlax88TaoaKEhQKAKDB6wgwphUxOQOnBBeJIiqzc6Ww5pt5W3kSgHs39w21MXTz51YxxS7ECjrIKaEb+gHbLOVoERURDBgE+eHW56AnOPPxjYifx7lkiDol9kFOZNPCfDglDMMwnRJTCrkknEuZn4/pqB+psL37YizonXklaKOt4ZN/dAsX0xH8N+8+M/Bxq7EA2ppuywFwUpDaGupt7UBoaSLktVUJSsY3nNRhknVEb9jlpriJ9eg6uJD5Gstumxs98prrZVTFdUEu2ArkJtc8aTU7se87FiUE2orlJpyOb5Dq0Hlax88T1ClBoQBQZAltL+BjzJthYgaZElwkgmhT33NS4Hbs3ugB927uRUUcGijo1uu2C1nckMVOL/OwmTqJNJQGBK8wMlzrMJwSgHkCThs49mK9r0JpJANJ11f5lSUFLVWfulPibNL87LTrkgCAeJBHVVZn6k74leffwI7Ywic/cAkcO3iEYDVuCn9bldm12Mwb+freOlBCQjAztUZRaZlOCTKSagfiYiACeVNtYkfawXpkHWxXlMjtmBu9fqJE2BtGmA8PvJcVmgVbByjkmicd4SCihhNhBth1fNBciaOnqTYha/JY4xvztI6fJ9RsFlwsBjZoL6tmHqGiBGUklXIGAOBhzQXYTJwS0QgiYmeuFNZC054lEti1orvOKaEMd0p4WA8CnoDrsjDsQhY3/cY3rNOlGdnOJVXC3/7i38a1wrWZPP9xpa7UR+ZJAKYo0VSbM7eeX1mLIltroVAfvSHZj9bR8V//+l/gj68dr0U2eV8th5anUuW3U2vh2X91Fa/vzCYImIQ5Ttsp8VB3g2g3TwIA4kEvOroBsTWZRX4Qr+/U8e++sYm/985TePvp+NDHWrWgNOzSIie2AOCgU0LwOXJKxAJ7N3P/4ku38ak/utX3e8jhBgmkfFB/AABYj+6KEsV8BYLPg9SA9deK0L8WVDd0++MbPvP1MmnY5bjjG0uhJbAMi63G1kQ/3y5vVN7AD33hh6hbsg9kDIm4H+wQ88XAgJmrdfw8cdzrQAEqSlBsUK2Y830MY24UZuGUYKNRRBr63HyYqbqKcqtse3yDWNHdFhhpZwMY4kNz7ZQQeKHv4mg5tGw+ZkZOibvVu3it/Bq+nfv2TJ7/uFJX6yPzJADzPdUxOmh3nIsFTiCtCjfGGOF45X4Fr9yv4Orrx2vRm5WyiPvi1ufapFV+336zgrsFCX/w17PZjJAwx+k7JUL4hR++hA8+MXhEYj/xoHn/nMUIh2EY+Ln/eAMRvwcf/b7+4Za9EJGGhl3usitK7N38JwUfyk0FHX24CFptVxFsA759daBfvp3DF17N9hVRF/wLYBnWqu4kzRu9TolqqYZzqdDA8MxBwc3VdhWartkb3/BHre+ZBCLMOA265FkeZyJn8EbljYl+vl2+ePeLuFe7Rw8O+kCa8Jw4JTiWQ9wfn5t1/LyhZrPHunkDoKIExQbVqilKaDA3CgvBWYxvRBGVDIiKCLXj/uDBklyCAcP2+AbP8vBzfvc5JUZkSgBmHobbrtsu243+daAA4ON8SAaSM3NKkAUiWWhS7FFX7IkSZLRo1q/Ni6tRMMx4DRzP38oBADaK8/n+GcT+99WkVX4bBVP0JH9f04Y4Adam7JRgGAZ//4kzWHAQ/kweO4sGjj/86wxevlfG//p9j9m6pmiAR9jvoU6JHvKiKXIuRvYHXXphGKP/3SqtCsJNA2xkd0NuGAayVRnFRhs7XdGjFw/rQcKfsJwSm7VNAMDpyGlLlGiUxb7NG4RBFdfkOe0coFhOiSmNbzh1SgDAxcRF3Cr2d5RMm6sPrgIw8zsoeyHCghNRgjyeihLTxzAM6pSgUACgVi8CAFqIIB7k4eGm/7LhohFEu+v2efhAs270AXtOCcCdjoNRmRKAO6/bLlkp2zfkkpAOpWfmlCALxGKzOJPnP66QTIlREFGiqc52Hl7weXAuGcI1h7kSh3WIhwAAIABJREFUhmHg+dtdUaIwn++fQex/X01a5bdRMD/87xYk3JuBgLNVkRHycogG+rcMHSaxoHkN0xYlarKKX/zSbbztdAx/5x2nbH/faozWgvaSE1sI8BzCvr2tWomQ6ZwY1cBRbZYQlgGuxylR6maaABj4OZIKpiwBe1PcxEpoBQFPwJofV+r1vs0bhFVhFQ21cSCkkowm2BnfmFqmRDfo0o64vJ+LiYvIy3lrjTUr7tXuWWLEvdq9mf6seYSsw520bwBUlJgVnUoFhiyDX1096kuZKVSUoIyk2jA3VfVOGIkZ5EkAJOjS/O95+EAjiwe7TgkAELyC68Y3RmVKAOZ1z7VTItTfKQEMnsOdBkTsoE4JZ9SVuq0TtpCn65SYcQMHAFxZi+F6xtlC/W5BwmapibPJECpNdeaNC4eFYRh9nRLA+FV+d4sSznZPgV+4PX23xFZFxlo8OND6fpjsjm9M1xH4r/7sdZQlBZ/8W5fADgm33M9aPECdEj3k620sRnwHXitkbHVUrkRFLiPcNMD1OCV6RZ9BobmLgcU9Ton1yDoAgPF4AK8PAa3dN+SSMCgkkjR62FmrBDwB8Cw/FaeEwAu26tL3czF5EQBwqzRbt8SLD14EAJyJnKFOiT6QWs+4f3guzX4S/sRcrOHnjZNQBwpQUYJig5psBt6U9QgSDmyqTmCjUUQkc9ZyHj7QyOLBbtAl4D7HQbvThqIrIzMlBF5AXZlNAN0sqSt11NV63+YNArGd64Y+9Z9PxI5Zn/gcN+pq3arQHQbJMzgMwezyahQ5sY18H+v1IIhL4h8+dRYAsFF0z3t/EirtClqd1h6nxCRVfoZhYKPQwFMPJ/HoUhhfnsEIxyzqQMclPoPxjRuZGj730n382JPrjppAAOqU2E9ObGEp7D/w9WRXlBjllKi1q12nRI8o0RV9/Dw7cAwsFUyh0CzAMAzcF+9jPbpu/V7HH0BAa1vCXT8GtUk5cUowDIO4Lz5x0GWtXRtrdAMAHo0/CpZhcbN0c6JrGMVXHnwFj8YfxbuW34XN2ubMA5PnjZJcQtATRMDj7HNzIbBA2zdmwEmoAwWoKEGxgdi9QeWUMJI2u9idwkWj8+WUaObBMqyjeTu3ZTMQoWGUU+KwqhenDVmcDcqUIL+n6MpMXnNkg1aQC3TBYxPd0G25d4DDy5QAdtsVnORKvHA7h4vpCL734SSA3RGFeadfo80kVX7FhoJ6S8O5VAjPXVjEK/crqE55tCFTaU495HJcIn4POJaZmiih6wY+8YUbWAj58NN/8xHH378aD6De1lCT3Z/ldBgQp8R+SOvYKKdETa1DkLEnU4KIPk8/ksL1TK3v/SAVTKHSrmBH2kFDbVhOCQBoe/0IjhAlyHtwvzBYaBYQ88Xg5ewdKEX90ak4JZyGXBKCfBDnoudmKkpUW1W8WngVz5x6BuuRdYiKaLVNUEzKrbLjPAnAHN9oqI2ZB1CfNKgoQaF0EdU6AgpQaBpIzsgpMW/jGwW5gKQ/CY7lbH+P25wSRJQYNb8v8IKrrtsuRJQYlSnR+9hpst3YBsdwkDV5Lv/+joKm2oQBw5EoMetMCQC4kI6AZQbPg++nLCn41v0Knn18CafiAfAcc2zCLsmmp/d9Rar8xnFKkAyJcykBzz6+hI5u4Orr03MX1VsqxJbmGqcEwzCIB3lUmtMRAX7nmw/wnQdVfOK/eAwRv/PMjNWY6TiibgnTtZMTWwfqQAEg4ufhYRmUpMGbLaWjoGm0EZGNPZkSmaoMwefBUw8nUZaUvuMyxMnwSu4VANjjlGh6fIhDRdA7eBxiwb8AP+fvO77hZMw07otPRZSIep05dnq5kLiAm8WbMxPzv5b5GnRDN0WJ7t/zffH+TH7WvFJulbEQGE+UAMzAV8r0ULNZsMEg2Oj476t5gIoSlJGIWgMhhYXY0maWKcEKAgIKA97g5sL65fRGD5iOBDc5Dki+xSibJXFKzNtpv7V5GjK+QU57J6kz7IeoiGioDTwSN08u6QiHPey6d4DDdUoEvR48vCjYdkp85bU8dAN4/+NL8HAsTi8Ej03YpSX29byveJZHKpAaS9wjfy/nkiG8dS2GpOC1Rl+mAdkAusUpAQCxoHcqGSNlScH/8Z9ewxNnF/CBt44XgGbVgtJcCTTaGppK50AdKACwLIOFkBfF+uB/N7KZF/aNb2xVZKzGAriyZgoV/XIlSDvGyzsvA8Aep0Sd4RFjOkOvnWEYLIeW+zolnARyR31TcEq0x3dKAGbYZalVskZPps3VB1eRCqRwIXEBZyPmeB1pPKGYlFtlJPzOQi6BXVGCZFJQpgOpA3VDLtIsoaIEZSR1XYagmQo9CXuaNgzLwhOJINbxzcWHWaHpXJRwrVNixPy+4BWgGzpkbb4WrduNbXhZ71ALIjntnaTOcNDPBoC3pN4CgIZd2qWu2ntNArBmXQ9L6Lu8GsO1rf7W6/08fzuHpYgPl1bNhfnZpHB8xjekbQQ9wQNiZlpIjxUau1GU4PWwSMcCYFkGzz62hBdfL0DRppPzQhwAbnFKAOg6JSYXJT79J6+h0dLwyQ9cGnuxSsSaTGX2jiO3k+vWgfZzSgBAQvANdUqQ0+FIE3uDLruZJo8uh+Fhmb7iJnFKfHPnm/BzfiyHlgGY7o0KeISN0a+XtHCwmjcv5x2tVWK+2OSZEsr4mRKA6ZQAMJMRDqWj4BvZb+DpU0+DZVisCCvwsB7cE2kDRy8luTTW+AZp6yjL7nc8zxNqNgvPMR/dAKgoQbFBnWkj2DFtoaQWaxaw0Shiimc+xjccnj4A3UwJxT2OA7IBHNm+0d0gusnlYYeslEVaSINlBn/MCV4BYW946uMb5PneuvhWANQpYZexnBKH0L4BmLkSxUbb2rgMoq118NU3Cnj28SVro/hQKoT7pSY6ujve+5OQbZjvq/2b4JXQythOibOJELhuY8Szjy+i3tbw8r3p3AeIA2DNRU6JeNCL6oTjG9+6X8HvvvIA//Cps3hkyXn1IiEpeOHzsNQpAVhBtot9gi4B8++qOCRTotY2xQZB5cAEdl9vJNPEz3N4dDncX5ToCgeZRganI6et+1axoaDOehHURs/or4RW9jglOnoHJbnkKJA75ouhptQmCn8W2+JEosSjC4+CY7iZiBKv7LwCSZXwzNozAAAP68Hp8GnqlOhBN3RU2pWxMyUA6pSYNmo2e+zzJAAqSlBsUGcVBHVTjEiFZ+OUALq5Ei3W9aKE0lFQaVfGckpohuaaACAnQZcAXOXysMOoOlBCOjTeCe8wyMLQckrMyIZ63CAjRXZECQ/rgY/zHUqmBACr1eDa1vBTxJc2ypCUDt7/+JL1tXOpEJSOjq1jcBq9LfV/X6WFNHJSDh19uM18PxsFaU/V4VPnk/B52KmNcGQqMrwcawUVuoF40IvyBOMbWkfHz37hBlaifvyPz56f6FoYhjEbOKgogVzdFCX6jW8AZtjlUKdENywx6glbot3+TJMra9G+jqsF/wI4xsyo6h3d2Cg0IHt88Cqjm3/SQhrlVtlyNVbaFXSMjjUaYoeYLwbd0Mdu3GppLSi6MtH4RsATwEOxh2YiSlzdugo/58cTK09YX1uPrNNa0B5qbVOUIq4HJ5CRD7ev4+eJTkOCXqtRUeKwYRjmnzMMY+z7NV7xOWVqNDwafLp5cjBLpwQXiSDSNFz/YVaUiwDg6EYP7DoO3LK5tytKzLtTYhQrwgoyjcxUf/Z2Yxt+zo9VYRUCL1ivGcpwREUEYE+UAEzB7LBEiQsrEXADrNe9PH8rhwDP4cmHdhd051Lme+g4hF0Sp8R+VkIr0AwNBdm+K0jt6Hiz3NwjSgS9ZiDg87dzU3GVbVVlpGN+sKx7ZnHjIdMpMe6f77dfuo/b2yJ+7gcvIOQbHH5ol9U4rQUFdsc3FgeNb4S8Q9s3iFMi7tn9/NqfaXJ5NYaarOJBee/fN8uwSAbMpp7ekMuNogTZ4wPXGv3vY7XgdEV2J3WghLg/DgBj50qQz/BJnBKAmStxu3R7qs5SwzBw9cFVPJl+En7P7r/xenQdD+oPoOna1H7WPEPW4OM4JQKeAPycn45vTBE1a65PqShxNLwOYKXn1+WjvRxKw2eAN8wb6qwyJQCAi0URqXdQlsuuGXHoxzg3egAIebuOA8U9ogTLsAh6gkMfR9o5xj05OQpaWgvlVtmRU2Kar7mslMVyaBkMwyAZSFKnhE3s5pwQgp7goY1vBLwczi8KQxs4DMPAC7dz+M/OJ+Hnd5t5znWr/OY9V0JSJYiKONApAThrsnlQbkLTDZxN7v33fvbxJWxVZLyem/wzJ1ORXZUnAZiZEkpHh6Q4c5UA5ojBL/3ZG3jvIyl8/6XlqVwPdUqY5MU2Ql4OwgChJyH40FQ6aCr9N69kIx/z9zRv7Ms0ubw6uF6YHHTsd0ooXj8gN0feo8h7kORKkLFBJ67OqC+658/iFCLMTOKUAMxciXKrjB1peueSb1TewLa0jWdOPbPn6+uRdWi6NpMWrnlkElGCYRgs+Bdcf7g4T5yUOlDAnaKEZhjGTs8vOox9hDSlGlQPwBpBeD3swJv1NGAjEUSrKhRdcY2boB/kJNDp+IbbHAd1pQ6BF0YGpLntuu1ATorsOCXSQtrabE3t5ze2rZ+9GFx0dHp8kiHveydOicN8XV5Zi+JGZnDY5a1tEdlaC89dWNrz9YWQFxG/Z+4bOMiifVU42PRg1es6aLIhIk2vUwIwcyUA4IXbk4t5marsquYNwBzfADBWA8cvfuk22pqOn/+hi1NLYl+NBVBsKGipzkWS40Su3r8OlEAOZQa5JSqtCgIaA194bx0oAKx1RYlHlgV4ORbXMgc3/eSgY68oISEQDQOGAaM53BW2/z1IApadjm8AGDvscppOCWC6YZdXH1wFAwbvXXvvnq8TZwod4TAheRDjtG8AZtglFSWmx64oMV7D0jzhRlHiHMMwWYZh7jEM8zsMw5w76gs6yVRLpm3I0INIhrwzraPhIlGEy6Z90s0faGM7JVyWzdBQG44CBXuvu/Brv4bG174+8TXcLN3Ep176lOM59FGQkyI7Ton9ltdpkJWy1vOmginqlLBJXanDx/ng5ew5sg5zfAMALq/FUJIUZGv957ufv5UHwwB/47G9mwCGYXAuNf8NHESUIFW6vZC2gP3p/61bt/Dmhz+M+z/2Dw78Cn3sf8D//vVfR/TjP4U3f/zHoebM98lSxI+3rEXx5Vu7uRIdvYNPv/xpfHPnm7avt6V2UKi3sRob7gY7bOKhrijR08AhfvnLKP3f/27o931zs4wvvJrFTzx9DmeToaGPdcKsa0F3ai389OdfRaPtbnt8XmxhcUCeBGAGXQJAsdE/V6LWrkFoMeCiUetrmYoMr4dFsjv66vNweGwl3LcWlBx07B/fCC+Yz9eRhn9+pIIpcAx3wCnhJBsg7ptwfKNtihLEcTEujyw8Ag/jmboocTl12RqTIRAR6F6NNnAAZvMGACwEnDslAFCnxJTRslkwPA9PKjn6wXOO20SJvwLw3wL4PgAfAbAM4C8Yhun7icowzH/HMMwrDMO8UijQk8hZUK2YNzelIyAx46AwLhpBpG5uTt38gZaTcuBZ3pq9tIsbMyXsiBL7nRKGqqL4a7+O6h/8/sTX8IU7X8Dvvv67uF68PvFz9UJOiuw6JQBntvNhyJqMcqu865QILKLQLLh6JMkt2H1NEoJ88FCdEpb1ekDY5fO3c3jbqVjfUMVzqRA2iu5474+L9b4KHXxfBfkg4r74AadE/fkXIP3lS4CuH/jVUjR4WYBV2pBe/CqaL79sfd+zjy/h1QdV5Lvhg59/4/P43O3P4WNf+5jtf/PtrnjkxvENAKj0NHBUf/fzKPzqr8LoDBZoX3y9AI5l8JPPPDzV61mLm6LNrHIl/uTGNv7g2xn85V13J/LnxPZwp0RXWBjolGhXEG4a4CK7G/KtrlOnN9Pk8moU1/s4rr7/7PfjQxc/ZH0GksyVWMJ8Pn2EKOFhPVgKLlnvwYJcwIJ/ATzLD/2+XqL+ycY3puWU8HE+nI+fx63SrYmeh5Bv5nGjdMNq3egl7o8j6otSp0SXcqsMlmER9Y4nLC34FyxhgzI5ajYLz8oKGNZtW/bp46o/oWEYf2IYxucNw7hmGMbzAH4Q5jX+gwGP/7eGYbzDMIx3pFLOTq0p9qjWzJOrph6yTglmBReNIto0b9JuDskhp+DDqib7QbIZ3DIGQcY3RmE5JbpZGGomA2iaZSmbBLLguPrg6sTP1Uu2kQXHcLZsq9N2SpDn6XVKKLoy1fGQ44rd1yThsMc3HlsOw8MyfXMldmotXM/UDoxuEB5KCciJbUguPy0exnZjGzzLDzx5XRFWDjgl1GwWnsVFnPncbx/49esf+F/wHz74cZz5f/8f67GE57rtJV95LY+iXMRnv/1ZPBx7GPlmHv/mO//G1vVa8/xuG9/oOiWqPU4JNZuFIctQNjYGft+9ooRT8QACXm7gY8Zh1k4J4goYJOa5AcMwkBOHj28kw11RYkADR7VVRbjRARfZ3ZBvVQ6OD11ejaLe0nC/tNfl9T1L34OffsdPW///zbJZI5xImQcgujTaFdb7Hiw0C44DucN8GBzDTZ4pMaEoAZi5EjdLN6ci6H9166sAcCBPgrAeWcd98f7EP+c4UG6VEfPFwLHjfc4QpwQ9iJkOauZk1IECLhMl9mMYRgPATQCTdV5RxqZWNx0oohqauVOCjUQQ7e4v3NxxvN3Y7mtfHoXllHBL0KVq71Sa53j4OJ+1+WvfMy2Ok4oSqq7i9crrAIAXt16c6Ln2sy1tYzG4CA87OgNlwb8AP+efmlOCLAiJU4JYcukIx2gaasPRYvawxzf8PIdHl8N9Q+peeM0cNXju8f6iBAm7vDfHDRyjBNl0KH3AKTGsX32jKOFsMgQ2GAQXj+/5THl8JYx01I8v38rjl175JcgdGb/0zC/hR87/CH771m/jTuXOyOvNVM3XxprrnBKmKEFqQQ3DsP7s8s3BdvW7hYbV5DJNlsI+cCwzM6fEte77ZVRzzVEiyhramo7F8OB1TiJExjf6OyVqrQoE2XR9EjL9RIm1br3wiL8PMu61vGza6Ec5JYC978F8M+94zJRhGER90YmcEgwYR463QVxIXECtXZtKO9bVB1exKqzi4Vh/l9F6ZB2btc2Jf85xoCyXxwq5JCz4F6AZGj2ImRJKNkNFCTfAMIwfwGMApjfsTXFEVTLFgUJbmGnzBmBmSkS6+ws3ixJZKdvXvjwK4jhwi1OiodjLlADMaydjJ8q9TQBAp1CE3h7c2T6KjeoG2p02riSv4LvV7+KB+GDs59pPtpG1lScBmIuw5dDy1JwS+y3uiwHzpIrM91IG43h84xDbNwhX1vpbr5+/lcPphSDOL/bfNJLN5N05DrscJciSU9rev5tBooTYUlFstK2/Fz6d3iNKMAyD5y4s4Rtbf4UvbnwRH7r4IZyNnsVPvf2nIHgF/MJf/cLIk7itigyWAZajg0+/j4JogAfD7I5vdCoVGC1z1KR1o78ooesGNkuSJW5NEw/HYjnin4lTQmpruFtogGHQ933jFnLdMaFhTgk/bzZzDMqUqLSrCMsA282UaKkdFBvtA+NDjyyF4fWwI50jJBg3nTZnye2IEivCCvLNPFRdRUF27pQAzFyJSYIuw96wYydpPy4mpxN2KWsyXtp+Cc+cemZgLtp6dB0FueCaQ6OjpNwqO8oh2Q/JonDzGPa8oLfb6BSKVJQ4ChiG+QzDME8zDHOWYZgnAPwegBCA3zziSzuxiHIFAFBDzApqmhVcNAKPDoSZgGvHN9qdNopycSynhJfzwst65y5TAjBdHruixG4YlLY9/kaeLDR+8q0/CQC4unV17Ofaz7a0bStPgrAqrE7VKcExnOWQIP9LGzhGU1fq1piTHYhT4jA3OpdWo6g2VWz1nCo3FQ3fuFvCc48vDVz0nkkEwTDzXQs6SpBNh9JodVqotM37htHpQN3Z6bugspo3upvs/aIEALzv0QSY5B8i7l3CR658BIA5//1P3v5P8K3ct/BHG3809HozFRlLET94zlVLHXAsg2iAt9o31Ez3z82yaN240fd7sjUZLVXH2dT0RQnAHOHYqkzfdXQzK8IwgPeeT6HYUKycD7eRE83rGuaUAMwGjn6ZEqquoqFJCMu7mRLZav/xIZ5jcWElMrReGDDfI0nBi8iCvUwJwLyX6YaO7cY2SnLpQKijHSZ1SkxjdAMAzsfOg2f5iXMlXsq+hHanPXB0A9gNu6QjHOah4CROCdLaQUWJySFrbCpKHA1rAP4DgNcB/AGANoB3G4ZBPyWOCLFl3jSbbHT2Tonu6UIcQdd+mJHO7HGcEoCZK+EGJV43dEiqNFb1onLvHsCbwVmTjHDcLN6EwAt4T/o9eDj2MF58MJ0RDk3XkG/mbTslgO4J7xSdEkvBJWt0hNhnqSgxGqdOiRAfggEDsjYb23k/rqyalXm9G4qv3SlC0XQ89/jgU0k/z2E1FsDGnI5v2BFkiRBIRpi0QgHQNPCr/UQJ83Nwv1OiV2DaUP8UnD+Hh7i/j4Bnd2P3w+d/GFdSV/CZVz4z1CK85cI6UEI86LXaN8jnaPCd70TrtddgaAdzR3ZFnOmPbwDAWiwwk/GNa103wAefON39/+4c4ciLpvthmFMCMEc4+mVKkCyFcM/4BnGe9Atavbwaxc2sCF0fLKhuFBs4lxTAhswgUltOie5973rxOgwYYzklYr7YRJkSEd90RAkv58X5+PmJnRJXt65C4AV8z+L3DHyM1cAh0gaOcqs8dh0oAEvQoGGXk7NbB0pFiUPHMIy/ZxhG2jAMr2EYq4Zh/IhhGNOJ3qWMhajWEWoBOuPpmyg/Tdju6UJM97tWlCCn6U5O4XvpHYM4ShpqAwYM26GCYW/YElPa9zcRfPvbAUwmStwq3cKFxAWwDItnTj2DV3KvWAu7Scg38+gYHUf/RulQGuVWeSqb2/0Wd7/Hj7A3TDMlbNBQGwjzzkQJ4HBHoh5ZFuDl2D3z8S/cziHs9+CdZ4efLp1LCbg3pw0cdgRZq8mmO8KkZsxZcH71YL/6RkECxzI4vRDsPiYNQ5bRqVSsn/cb134dC8xbcPPOqT2bN5Zh8bNP/Cyq7So+++3PDryeTEV2XfMGIR7kUe2Ob5C/p/D73w+j1UL77sGwSyLiPDRDp8SO2ILa0af6vNczNaxE/Xj6kRQ4lsENl+ZKkPGNYZWgAJAUfH2dEpYo0YQVdDksaPXyWhSNtoZ7pcGfXfdI5krI/De3lSnRfQ9+p/AdAM6rywHTjeQGpwQAXExcxK3irbHdcLqh48UHL+Kp1afAc4NbSE5HToNl2BOfK9HSWpBUaTKnRIA6JaaFJUqsHbyHHkdcJUpQ3EetIyGomC+TWTsl2FAQ4DhEVd61H2b7mxWcIvCCKzIliMBgd/FAxJROo4FOoYjQk+8GWBZKZrwAKrVjhlxeTJgzo0+vPY2O0cE3Mt8Y6/l6IaFYTtwsRESYhluin8Wd1IJSBqN0FLQ7bceVoMDhihI+D4fHVsK4njEX7R3dwAu383jfo4sjxwTOJUO4V5BcO1c/DDuCLPlcJI8ddsqzUWzgVDwAr4fd8xgyyvAvv/kvoRs6fuyR/wn5evtASOLjicfxdx/9u/j8G5/va+/WOjp2xJbrQi4J8aDXCrpUs1mwoRBC73kSAPqOcNwrShB8HqRGjBeMy2osAN0wW2SmyfVMDZdWo/DzHM4vCiPDHY+KvNhG2O9B0Ds8HDkh+PoGXVZappjWmymRqQ7ONLmyRuqF+/991GQVxYaCcykzCBawJ0osh5YBANcK1wBgLKcEGd8Y53NKbIuI+sarkuzHxcRF1NU6HtTHy5y6UbyBUqs0dHQDMF0Z6VD6xI9vkNfxJKJEzGe6Cd26jp8n1GwWYFnwS/0DtI8bVJSgDKVuNBHUzJv0rJ0SDMOAi0QQbbGu/TDLNrJgGRZLofE+INzilKgrdQCwPb9PxBSSJ+F7+GF4lpagjemUuFO9A1VXcSF5AQBwOXkZC/6FqeRKWMKRg9wPIiLsrzN0iqqr5ujIvp+dCqaQl6lTYhhOX5MAEPJ0nRKHHHZ5aTWKa1tmaN+rD6ooSQqeHTK6QXgoFYKkdJATxw+IPSrsCLIRbwQhPmQ9lggM/MrB79koSHuaJIibQs1m8ReZv8Cf3f8zfOTyR/BfXb4ClgGev5078Bz/+G3/GHFfHJ966VPQjb0n/Ll6Gx3dwGos6PBPejjEQ16rEpSEgXrX18GGQmjdPChKbBQlnEuFBmaWTMosakHrLRUbBQlXVs1N6pW1KK5vjbfZnTWj6kAJScGLsmS+tnrZHd8wrFHUTEXG8oBMk4dTAvw8O3CcpXe8iWFZMMGgLVHCx/mQDCTxetlstiKZRk6I++LQdG0ssXfqTolu2OW4uRJXH1wFx3B4avWpkY9dj65jU9wc6+ccF8jaexJRwsN6EPPFXLuOnyfUjFmpzfCDXT7HCSpKUIbSQBt+zXwzkBqzWcJFo4g1gWq7ClVXZ/7znLItbSMVSIFnx/uAELzucEqQDaDT9g0iSnjX180Z8Mx4ogSZEb24YC44OJbD02tP4+tbX5/4352c0jpxs+y3nY9LvpmHbugHnRJB6pQYhdPXJLA7vnGYtaAAcGU1inpLw/1SEy/czsHDMnjmkdGixNluHsDGHDZw2BFkGYbBSmhlj1OCi8etk15CvyYJ4pSQsm/iF1/+RZyJnMGHLn0I8ZAX7zizgOdvHxT1It4I/uk7/imuF6/j9+/8/p7fs6zzrnVK8CjvEyUYloX/4kXIfRo4Ngqzad4gkBGDaeZK3MiYeR+kAvPyWgyVfSGxbsEUJUYfvCRCXugGLEGJQMYdwh0vWJ/5PFvVweNDnm7Y5aBxFitDpDuuw4aC0Jv2PufSoTQ0QwPL9rrXAAAgAElEQVTLsGNtLonTwekIh2EYENvTFSUeij0EL+sdO1fi6tZVvH3p7bbcG+uRddwX7x8QOE8SpPlukvYNwAy7pKLE5Ayr1D6OUFGCMpQGp8KneREN8JbNdpaw0QgidfOGMG4l1SzJNrJj50kA3RYLFwRdWhtAm/P7gleApEho3dswrWSnT/dNy7fLzeJNhL1hrIXXrK89fepp1NU6vp379ljPSdiWtpHwJ+D32K8BTAVS8DCeiZ0SliCy3ykRSKEgF070YmcUxEHkJFPiKMY3gN1N1rVMDc/fzuFdZxcQDY4WKskG4+4chl3aFWTTQnqPKNFvQUWaJHqdEmwkAjYUwr+v/znui/fx8Xd9HF7OFMKfu7CI29ti33aIHzz3g3jH0jvwK9/+Fct6DACZqvlYtwZdxoJetFQdLbVj/j11w0D9Fy+i/dprMNRdcVZWOshUZUvUmgXp2PSdEmTE6TJxSnT/1425EjmxjaXw6HtGousYLUl7RQnSOBP37G7IM5XhQatX1mK4ka0dcF0A5nhTb+YKFwzZckoAu/efhD9hBS47gdjvnYoSsiZDM7SpBV0CAM/yeHTh0bFEia36Fu5U7uCZtWdsPX49sg5Zk090/hMJp5zEKQGYtaA06HJyqChBofRQ93Tg6fhmnidB4CJRRGrmYsyNKuu2tD12ngSwt8XiKLE2gA6cEpqhobG5AX5tDazXa4oSuVzfpPhR3CrdwsXExT1W5CdXnoSX9eLqg6uOn6+XcYQjjuWwFFqa2ClBbOv7nRKpYAqaro0dHnYSIC0Kbs+UAIBHlsLwelh86do23sg18Ozj9sa5liN+BHgO9+awFtTu+2oltLIbdDlgQbX/FBgwXRblh1P49/8/e28a3cZ5mAs/M4PBvhEguABcQErWQlKLLVurZZJKnCbXcZymN0m/fHHSmzRt2iS9bZo6m3Pdxs7S2k3SNm2TNPem6ZYv556kbeq4dp2IpGR5kW1ZlkRSK0FxAVeAxA4MMDPfj+EMCWLhDIiNEp5zeI4OMAtIYWbe93mfxTSM+9vvx1HXUek98e/7yyxqCYIg8MVDX0SEieBb574lvZ4vZLAaYDMIz1T/vB9cMCj9nbQ93eAZBonr16VtPYuZf69iQ0tTqDdqiqqUuDgdhMuqkybyu5pNoCmi6nIleJ7HfCiOBln2DeF3WQynW7ACiQDUHAm9SZjQi5km+ZQ6e1wWRBk2q3JqbCGCNptesn6QBvmkhPj8KcS6AQhBl4ByUkK8h1vUxcuUAIAuexdGfCOKSf2hKaHRa6M8CRFuixsA4Ancvg0cxbBviPtX4xh+K4FnWSTn5mqkRA01AADHcQhreCClLXmehAjKbIbJLwRtiTKyakGKS2E2Mrt5pUQVZEqIgwclmRIAEPB6oO5wA1iRW7MsUvPKVhUSbALXlq9JIZci9LQeh5oPYWByYFOe40KJo2ZDc8mUEmLYWM3CkRuigqigTIkykxI0RWJ3sxnPDguNFPmqQNeCJAl01BswtgUbOOReV06jEyEmhFAilIeUWPHLr7MjfP9wBCTH45F7Hkl7fZvDiM56Q9ZcCQDYXrcdH+z6IH567ac4P38egLDibzeooVNTsn6/cqNuRVmz5JkAsGpf0fX0AABia8Iuy0FKAILVpahKiallSSUBCCGxOxpNOcMdK4WlaBJJlkeDjBDR+pUFmvUNHEvxJZgYSmoRk5NpIoVdZiFpPIvpdh0lpIT4/CmkeQMo3L4h5moUUykBCGGXkWQEE8EJRfsNTg6i09KJNnObrO3FWtDbOezSH/dDp9JJhH+hsGltVTeG32pIzc8Lldo1UqKGGoBoeAksBXCsTnoQlxqUxQLToiC7rTaWdSG6oLhqcj2MaiOSXBIMm5neXU6IE0C5UnnRux+Ym4TG3QFgTVq+QgvHtaVrSHEpdNm7Mt7ra+3DdHgaN5ZvKDqmCI7nMBOeKej/yGl0FkUpUa+rh4ZKH9yKg8OFWI2UyAXRUqTEjyx+L4tR5aoUohR9R6MR7Xb5k8VOh0FSCmwVsByLucicrOtKXKWdmrkMPh6XbAlrMZalSWJgYgBnbUt471laahBYi7d2NeLlMR9C8eyZM7+z73fQoG/AEy8/gRSXwlQV14ECqxlNkYkpAKv3U7qtDaTJhPiaXAmRxOkoYaYEALRYi0dKBKJJjPuiktVJxN4WCy5OB6oq7HIuKCyEyAm6lOwbWZQS5jiRWQea5zvY6TBCr6Yywi45jhdICUc6KcFGy6SU0KwoJRRaaMXFjmJmSgCQxgpKLBwhJoTXZl+TrZIAhMUDvUp/W4dd+uP+TaskAIGUCDEhJNnqy4bbKpDaq7I8Q29VKDeb1XDbYNkvXBCJpA5uQ3mUEqTFDNN8BABZdX40ccKqpGpyPcRJVIgJbTpIaDMIMSFoKW3e3u61EJUSUSRWlRKuwkiJ4cWVkMv67oz3elt68Tgex+DUILbXbVd0XEB4oDIcU7BSYj46jySXLDjI1BvOrAMFVgeHNaVEbkjtG7R8pcR6+0bw2WcRu3hx4/0O3A3Tif4CPuUqxMmWXOuGiE6HEc9cnEEixUKjqs5V/PVYiC0gxafk2TdWVmknp0bRiux1oJ51TRLxVBxfP/t1uHk73nF6Dmw4AsqYPgF/y64GfO/UGE5dXcQDezOvbz2tx2fv+Sz+cOgP8eMrP8b0sgs7G+VbgcqNuhX7RnxqGiYAqpW/E0EQ0HZ3p9WCji1G4LRoN6yr3CxcdTo8PzoHjuNBkvJaPiaCExjxjeDtHW9Pe/2SV5ho711HSuxxWfGjs5OY9MfQZq+OZpT5kEAwyAm6tOpokAQyakGXE8swxvhVUkJGpglFEuh2mjOUEtPLMSRS6zJXClBKNOiU14ECgoWOAKHcvpEoDSmxzboNGkqDYd8wHuh8QNY+Z6bPIMWnFJESBEGg3dyO8cB4YR/0FoA/7oddu/mxqTi+9cf9BbfVVROeHX8Wu2270W5uL/m54qk4fnLtJ/iVaeH+WFNK1FADgCW/IGWPJnVlzZTQxzioCFXVKSVySfOVQJxwVTpXIpQMKfLui5L6qIaAWlRKrNT8KSUlRvwjsGqsWSfvjYZGdNm7Cs6VEP+PClVKcDyHuUh2ibjc82f7fohKids5QGsjhJIhkASpSDZKEiR0Kh0iyQh4nsfMo1+C/4f/gKV//pecP/6//yFmv/zlTX/eY9vr0VlvwK/e6VK0X2e9ARwP3PSVtzFkM5CuKxmErLjN9LyQiZArU2KtNH1wahDeiBe/Z30IKg5Ieqcz9jnQXodmixZ/M3g9azAgANzffj8ONB7AP4/+M7zL0arNkwAA64p9g52ZAUHTUNXXS+/peroRv3oVHCNMfMcWwugosXUDAFrqdGBSHBYj8itr/+qNv8JnT38WCTZ9H3Gi3ePMVEoAwIXp6snXUaKUIEkCNoMGvnV/o+XEMkwhFpR1tQ4UEP6m+bDHZcWwN4AUu5qXMCbadTLsG/LuGW2mNnTZu3Cg8YCs7deDIimYNebCMyVkNF0ogYpUYZdtl7SgsRFSXArfv/h9NBmasLd+r6Jzuc23dy1oMZUS4vG2OvxxPx4ZegQ/uPSDspzvlxO/xNfPfh0n514AkL1S+1ZFjZSoISeWA8LkjIFZkiyWGpTZDAKATW2pupuZGGK42aBLABXPlQgxIUXefUkpoQbUHQIpQep0oGw2xbWgw4vDGSGXa9HX2ocLCxcKUsqIapZClRLA6v+zUnA8h5nITNaJm5pSw6qx1uwbeRBiQjDQBpCEsseSGB7LLi6CC4fR+LnPYdf5N3L+OD75CaRmZ2XX6+WCy6rDyc/0YYfC1XhRkr2VLBzSdSWDkLXr7KBJGt7gJIBMUkJskli7Cjw4OQirxooj7b0AshOdKorEFx/YjWFvEP/0cnbPN0EQeLv77ZgMTYIh5raEfQNzs1A5m0GQq997bU8PkEwicfUaeJ5fIXFK17whQmktaJJN4oXpF8DxHCZX/r9FXJwKoNWmkxQhInY0mqCmyKrKlZhfISUcMjIlACFXIqtSIpwCKSklYqg3qqGl86uh9rZYEE9yuLHmfiBlrhSolNCqtPjxO3+Mg80HZW2fDXWauoJJiWIrJQAhV+Ky/zJYjt1w2x9f+TGuLF3BH939R6BIZWo0t8UNb9iLeCpe6Efd0vDFfLDpNk9KiGqLWyFX4tTUKfDgy0ZWiTalFxIjWSu1b2XUSIkaciIQXgQAxGGGo1xKiZVVhjrKVHWkhDfshU1rg05V+EC3apQSjEKlxMrnjpvUUDWs+lSV1oLGU3FcX76eNU9CRH9rP3jwODV1SvZxRYhBlYUqJYDVVWGl8MV8SHLJnBM3h95RU0rkQZgJFzSYNdAGRJNRJDxCYrpoL8oFkVRjblYmzEzMBdhKYZeSSkwG2UcSpBAaG5sHqdeDtKSvmq4PbUxxKZyeOo37Wu6DxiVUBOe6pzywpxn3bq/HU/91BQuh7Kv5olxbZRytaqUETZEwaVSgFzPT1bUrYZfxS5ewEE4glEiVPOQSWM0/kJsr8fr86xLBvn7AfmF6GXtd1ox91CoSu5pNGTkKlcRcMAGrnt6QQBBRb9SkZUqwHItAIgBjTFB7AhAyTWR8/3pWsmkuTK0SAGMLEZg0qrQsL9JgAB+LgWc3npQXAxaNpaCgS4qgpMWXYqLL3oVoKrphCOVCdAHffuPbOOo8ivvb71d8HrfZDR48JkLKQjVvBXA8h6X4Uk0psQ5Dk0KLS7lsPaIi6Kx2BoTr9lFJADVSooY8CESEm0kU1rIqJQCgDgb4Y9V1M9tsHSiwaoMQgyYrhTATlh1yCQAGtTDIYJrtaQoHpaTElaUrYHk2o3ljLXbW7USToakgC4c37IWJNikiXESI4XqFhl1ulDnSoGuoZUrkQYgJKcqTEKFX6RFJRcB4xgEAmhXSIRckUsJTmdo3k5ZGg0mztZQSCgnZZmMzZrkl0C5nhiJKJGNEcuaN+TcQZILoa+2Dqr4eBE0jleOeQhAEvvxQNxJJDl97ZjTrNk2GJjh120GZRqpaKQEIuRJa/0IGKUG7XKAsFsSHL0n1sWtXzUsFpUqJwclBqElh4ryWlFiKMJj0x6QJ93rscVlwyRsAl8OGU27MBeNoNG1s3RBhN6rhi6wqJUJMCDx4mKM8KMtq0KWc719nvQEGNZWWK7E+cwUQSAkAm1Z4yUWdpq6goEuT2pRTBbkZiGOGjcIun3rtKSTYBL5w6AsFfQ6xFvR2bOAIMSGk+FRxSYkqG8crRYJN4Iz3DDSUBr64T8q+KhVYjsWofxQuowsRmsXVHbePSgKokRI15EEgtgQAiJJW2A3lUUqIdVpWTlt1DKs37N1U8wawqjiouH1DaabEyudmGtIHmbTTieTMjOwk9RHfCIDsIZciCIJAb0svXpp5KcOnvBFmIjMFZ35oKA3qdfUF14KK++VVSsRqSolcUPqdFCHaNxiPB4RWC1VTZnPDWqjbhaCqRIVICUBs4Ng6SgmlhKzT4MS8KiqFN66FOMkWSYnByUHQJI2jzqMgSBIqZ3NeorPTYcRv3deJn74xjZfHskuDXZoDoHQTMOiU3T/KDYcaMISXM0gJMewyNjycNV+gVDBpaZi1KllKCZ7nMTg5iCPOI2jQNcATWL2ecoVcitjbYkEonsJNf3XkqsyFEmiQEXIpwm7QYHGNUkdUFBhjQoMYz/OYXpanlCBJAj0uS5pyZGwhnEFCkQZhciLXwrFZFKKUCCaCRc+TENFh6YBOpctLSpydOYtnPM/gIz0fKTiQUKwFvR3DLkWrRTFICQNtgIbSVN04XinOzpxFLBXDu7a9C0Dpvxc3gzcRS8Xw4a4Pg04BZ123l42oRkrUkBPBxDIIjkeMNJVPKbGyymBNquGL+6qmNoznecxGZjetlBBljdVg31CSKaFKclCleMTr0gfGtNMJPh4H65f34BleHIZNa0OjPn8ac39rP2KpGF6ZeUX2ZwQEtcJm2lGchsJrQTdSSjh0DvhiPlme2NsRSr+TIkT7BuPxQN3enubNzwZSq4XK2SwpKyqBjnqjNNncClBKyDYbm7GkZcG7Mq/ztU0S4sT2YNNB6d5IO50b5tR8on87XFYd/te/X0JyTUCgCGNqLwiCx3nfi7I/cyXQmhJW3WhXZliqtqcHiavXMO71Q60i4SyTFcVVp5ellLi+fB3T4Wn0tfbBbUkPBxQn2OtDLkVksyxUEvPBOBoUKiUiDIsYI9zLxcm7OQaQZjMWwwwSKU62fWiPy4LRmSCSLIcok4I3EM8goSSlRJlICavGWlCmRCnyJAAhfHO3bbe0sLEeSTaJr7zyFbiMLvzmnt8s+Dx6Wo8GXcNtGXYpqhqK0QxHEARsWtuWz5QYmhqCTqXD+3e+H0CmTa3YEEm3u3Q70HOTwyvG2aqZB5UDNVKihpwIpsIwJAioVMLqSTlArfiPLQkSCTaBaKo6VlL8cT/ibHzzSgl1lSglFGZKMDdvQscAcUv6wE1pLeiwL3/IpYh7mu6BXqVXbOGYCReulACEyVShSglv2AuT2pRzYu3QO8DyLJYSSwV/vlsZhWZK6Gm9oJQYH5esGRtB4+4AMz6u+FzFwjaHAcvRJPwRZuONK4xCCNkmlbDSttSceY9ZuwrsCXowEZpIq+2TYwnTqSn8ybu6cXUujB+cyVS8RMJNIDmL5AWuVrQwwuQ9W0OJtqcbSKUQHrmMDrsBlMyKzs3CZdXJUkqI9+bell6hsSAwLg2eL04F4LbrYdFnr1be0WiCWlUdYZccx2M+lJBVBypCzHoQGzhWlRI8KItF+vu56uRJr/e0WJBIcbg2F16TubJeKVFmUkJrRYJNIJaSZ+UBhEyJUpESgJArcdl/GSkulfHeP47+I8YCY/j8wc9Dq5JPMGWD2+KuKSWKgK1OSvA8j4HJARx1HkWnpRMUQZWFlNCpdHAFVThwjcc0ljEWGCvpOasJNVKihpwIpSLQMyTsBk1JPILZQGi1IGgalphwvmrxoxWjeQMA1KQaKlJV0UyJBJtAkksqGjwwHg/0CSCuTyenxMG0nAaOaDKKscBY3pBLEWpKjWOuYxiaHJLNEgeZIMLJ8KaVEjORGXB85urrRsjVvCFC7Iyv5Upkh1KiTIRk35ia2jDkUoS6owOMx1OxFQgxtNCzBcIuCyFkGyLCZHSxPn1SKjVJrPz+ImmwnpRILSyAS+S3Xry1qxFv3d2Ab/3iGmYC6ROn6aU47MSdOOM9o9gCVk40RIXnG+3MVEroVsIuVdevlCXkUkRLnU6WUmJwchA99h449A60m9sRZILS5PzidCBnngQghHx2NZvTchQqBV+EAcvxsupARdSvKEd9Kw0cS3GBaDbFhFws8e8nVymxt0UIBL04vSxlzXSsV0roy2vfsGqEz6QkV6KUSglAICViqViaVQgQFiO+8+Z30N/aj97W3k2fx212wxOs3POhUhCtFsUkJaplDF8IRv2jmI/Oo6+1DzRFw2V0lZysGl4cxm7bbnDeWRy4Lnz/CslX26qokRI15EQQMWgZFepN5cmTAATJF2mxwBwWJoXVwrKK6fObVUoQBAEjbayoUkIM6lESKsiMj0OXAKLrFpNE2bEcpcTVpavgeC5vyOVa9LX2YT42jxF/drnmekgNAZtUSiS5ZGF1pGFv3nM79EJrSa0WNBMczyGcDBcUdGlQGRBhwgDLbhhyKULd0QEuEkFqoTL/F2K9440tEHZZCCFbHxDu34vr5idik8TaPIldtl1SyCywOkFPzWysWHrswW6wHI/Hn06/R0wvx7DNcBCxVAyvzr4q+3OXG/awHywI8Pb6jPdUzc0g6+pgmx4rKynhsuoQSqQQiCVzbrMYW8TFxYsSmSSGA44Hx+ELJzC9HMuZJyFib4sFl6YrH3Y5HxI820qUEqKdVVRKBBICuWKKrpASy4LCU27QartND5NWhQtTgdykRJmVEnWaOgBQZOEIMkGYNaUjJcQsqvW5En/26p+B53l87uDninIet8WNEBO67VSN/rgfBAiJkNosbFrbls6UGJocAgEC97XcBwAZNrViI8WlcGXpCrrsXUh6vbCHgN2WHTVSooYaACCMBNRJFeyG8uRJiKDMZpiXhQFRtdzQiqWUAFZXdisFkZRQZN/weGDgaUS49BU0ymwGaTTKIiXEgUS+kMu1OO46DpIgZd+QJeJok0oJQHkDB8/zGysl9IJSolYLmolIMgIefMFKiTiXAEdAtn1DVFRUKleipU4HmiK2RANHIYRs3VwUBMdjXpOuUhhb0ySxFF/C+YXzaSoJYI36SsY9pdWmxyf7t+OZi7M4dVUgmILxJELxFPbV3w2dSlfVAzpz0AefzoJAMnNiThAE+B27sH1pUiKxygGpFjSPWuLU1Cnw4KX/uw6zcN2NB8Yl9cOeLHWga9HjsiDCsBXPVpkPCt/RBgVKCTH4ezG0opRILIHiCehVOhBqNaaXYjBpVLDosttX1oMkCfQ4BZLGsxiGy6qDTp1eT0qVmZQQAyvlkhIcz5VcKeE2u6FX6dNyJU5PncYvJn6B39r7W5teNFp7HuD2C7v0x/ywaqxQkcWxa9t0AimxVRUnA5MD2OfYJylH3GY3JoITBSlp5cAT8CCWiqG7vhtJrxekXo8+91vw5sKbBS2UbUXUSIkaciJMJUEl1bAby6eUAIRcCfOyMFCoFlLCG/bCQBuK8sCtFqWEkglgwjOek0yRWws6vDgMh84hTc43Qp22Dvsd+2X7wiXiaBNKCXFQozRXIsgEEUlG8g6KxPComn0jE6KdqRBSQk8Lsua4GlC73bL20axsV6laUBVFot2+NRo4CiFk+Zk52MIEZvj0CY1nTZPE6enT4Hguk5RQmFPzW72d6Kg34LGfDSORYqXJtNtuwZHmIxicHKzaQbFhaQHz+josRbOrEgKt29EemkOnicr6fikgWg6mlnLnOQ1MDqDZ0IwddTsACPdNmqThCXqknIgeV/5npaikuDhd2bDLuaColFAWdAkAi2uUEhZWDdVKe9j0srw60LXY22LB6EwIl2dDWZUxolKCLZdSQqtMKRFJRsDxXMnaNwCAJEjstu+WFjgSbAJfO/s1uM1ufLj7w0U7z1rlz+0Ef9xfNOsGANi1diS5ZMUz1ArBbGQWo/7RtOeT2+JGnI1jNjJbknOK32tRKUG7XOhr7QMPHqenT5fknNWGGilRQ06EaQ5EUi35J8sFymyGaVEYEFULO+iNeNFsaC5KtoZRbayoUkLpBJDneTAeD4xaS9YsDNmkxErIpRL0tvZi1D8q6yHgDXuhoTSwawtPjhZJBaVKCTkTN5qkYdPaarWgWRBkggAKIyXE1gamyQbKJG9/VXMzCK22YqQEIMizK71KLAeFELJJrxcNcbV0XYgYWwhDoyLhsgoKhgZdA7ps6RkzdGMjQJKySQmNisKXH+qGZzGC7w2Npfn5+1r7MBedw2X/ZdmfvZxQ++Yxr6vLGXg66WgHxXNo8U2X7TNJSokcYZfxVBwve19GX2uf9DykSAptpjaMB8ZxYTqAznoDTNr8KoHtDiO0NImLU8Hi/gIKMbeilHAoGOfo1Sro1ZSUKbGcWIaJoUCZhWtkakleHeha7GmxgGE5gZTIUv8qkhJ8tDzh30qVEuI9vJRKCQDotnfjiv8KklwS/+fi/8FkaBJfPPxFqKniLZ45DQLJdtspJeL+ojRviBAJjmpZXFSCU1OnAAhNcCJKraAZXhyGXqWH2+wWSAmnE7ttu9Ggb6hqxV8xUSMlasgKjuMQ1nDgU1pJqlgukBYzyOUwTLSpam5mM+GZokkDjbSxokGXwWRQ+hxywPp84EIhmIy2gpUS0WQUnoBHVsjlWogstZwb8kxkZtPEkTj5EiXrciFX4t6gb6gpJbJAXEkpJFNCrxKUEqxb/vVJkCTU7e0VbeDodBhw0xcBW2FP/UYohJBNer1o5I0ZiqOxhQg66g1I8UmcmT6D3tbejOMSNA1VQ4Os8FwRx+9w4IE9zfj2wHW8PCYQ2a46He5ruQ8EiKoc0PGpFMhFQSmxHM1OSgyv3E+oG1fK9rnsBjW0NJnTvvHKzCuIs3H0tfSlvS76rS9NB7BngzwJQFALdTstlVdKhOKwG9RQq5QNh+uNGvjCAqGxFF+CKU5I7WEFKSXW2F3WN28AAKHTASRZNqWERErIDLoMJlZIiRJmSgACKZFgEzg1eQrfv/h9vN39dhxuPlzUc4gkmydYOdK6EvDFfUVXSgDVs7ioBAOTA2g1taLDsmoJFf9dKgXNiG8EXfYukAS5opRwgiAI9Lf240Xvi1Ud2lws1EiJGrIiFFgARxLgWV0FlBIWsIGA5EerBogD82Kg0pkSSpUS4mqy2dKQVYZHu5zggkGwoVDOY4z6R8GDl50nIaLD3IF2czsGpwY33NYb9haFOHIanRkrvBtBrsTdoXPUMiWyQLQUFbLKJiolkq2NivZTd3QgMV65Qee2eiOSLJ9XJl8NKISQTXq9aFLZMRedS6vvG1sUSIlXZ19FNBXNsG6IkKu+WotH37kbFEngf5/xQK0iUW/QwK6zY69jr6z7R7mRmp8HwbF57RuXGDVCejPily6V7XMRBAFnnlrQgckBGGgD7m66O+31dnM7JoOTmAlEsCdP88Za7HFZcGk6WFFibj4Yh8OkfIxjN6rhW1G4BBIBmKI8SItZyjRRqpRotemkDIr1IZfASgi4Xl+2TAmapGGiTVWnlBAXNh498yhUpAqfufszJTmP2+LGzeDNkhy7WuGPFde+YdNtTaVENBnF2ZmzaWowQCBZDLShJKREkkviytIVdNu7wYbD4IJBKV+pt6UXsVQMZ2fOFv281YYaKVFDViyvyEVTnKEimRJcKASbpq4qbmZhJowQEyquUqICHjs+lULszTcVZ0okVlaTzfVCMwXDpq/qyQmmG15c9copAUEQ6G3pxdmZs3hm7Alyg7kAACAASURBVBk8O/5szp/J0GRRiKNmQ3NBSgktpd3wge7QO26r9g02GETi+vUNt5MaYdTKlRJaRpjQJJ0ORfupO9xITk2DZ7KvUpcaom9cbthlJJHC9fny3Tc8ixE8fcGLm8FpMHELnr7gzfoz4UsnVbhEAuzCIpyGZrA8KymDmBSHCX8UnQ4DBiYHoFPpcKj5UNZzF0JKNFt0+IO37gDPC9YNkhQGk32tfRjxjZTMB1woxN9vTleHpRxKibHFKJZbtyM+XHxS4rL/ck5y3JWDlOB4DqemTuGY81iGXN5tdiPFp0DQS1LF5UbY47IglmRxo4LZKnPBhKI8CRF2gwaL4dWgS2OEBWW2rNqHFColCIKQyJxcbSukwVBUUoIZH0fKl3sV26KxyCYlxAaSUpMSbeY2aQz1if2fQKNBGRktF26zG5OhyTRSdbMIMkFMBieLcqzY+fPgueIFLjIsg1AyVFxSYovaN17yvgSGY9KsG4BwjbrN7pLYN8aWx5BgE0KexIpKUBxbH2w+WPWhzcVCcSJWa7jlsLw8BwBI8cbyKyUswkPNprLgZqx8XtpcEPMFNtPqsBYGdWWUEoGnn8bM5z6Ppb94L0iClGTvG4HxjINQq2G2NQM3BKm9jVp9cEmkxLQX2p07sx5j2DeMRn0j6nWZ1Xcb4f72+/EPI/+Az57+7IbbbrduV3z89WgzteFF74uIp+LQquQNVmciM2gyNG0ocXfoHPDH/UhxqaIlXFcz5v7szxB85j+x48wLIHW5B+mFhK+KUM8Jg+Zko7IaM01HB8CyYCYnodm2TfF5NwtRon1jIYz+XfnDX3mex2//4+t4xePDf/7P+7C9obRtDN7lGB74y9OIpiIw7QxjcDiF50+/kXVbk1aFgc/0Sc8JscrTaXMD0VOCyszYjAl/FCzHo8NuwHfHh3Ck+Qg0VPZnC+1yIfjss+BZFgQlP+TxN4658a9vTKPNtnpv62vpw1+c+wucmjqF9+18n+xjlRoiKRGy2LGUJVMiEE3CF2HA3bETiWd+DC4aBamXd8/eCPFUHB985oM43HwY337LtzPeb6nTYcSbmfUw4hvBQmwhq8JFlDZTmkV0O+VNTKWwy6kAdjQqv/aLgblgHLublZ+73qjGm1PLuLZ0DUvxJdh9BKgOc1qmiVIc2WbHlbkQnJbs+wqkRHGUVTzL4ubDH4Lu7gNo+eY3s25Tp61TrJQoZdAlIIRdHmg8gLnoHD6w+wMlO4/b4kaKS2E6PI12c/umj8dyLD72Xx/DeGAcP3v3zzZFpkTPvYGbH/gAXN/8BszveMemPxuwShyI6oZiQAxL9cW3ln1jYHIAJrUJ+xv2Z7zntrhxbu5c0c+5tp0ucUogoenWNgCAhtLgmPMYBqcG8Sj/aFGy7aoVt/6ouIaCsBwQVpUSvLHsSglyJSzKSuhxrgq8aOKq+WZaHdbCSBuRYBNIsknQlLzKsGIgdv48AMB/8wqMJqPsGxvj8UDd3gajRhi4RZhIGpsuRykx4htRHHIpYn/Dfjz7a88inorn3Y4kyKIMHo46j+KHIz/E2dmzUj/1RpgOT8tS0jToG8DxHPxxv+wWkq0KnmUR/uVJ8NEoIi+9BNOJEzm3lUgJWvnkQOUV7hGMXdm+6jUNHJUgJWwGNax6WlbY5dMXZvDC9UUQBPDYzy7hnz56qKQDk8efHgHL8fjGB9rw2OvAF992FMeaM6+FhVACH/7BWXztmcv48/ftA7B6H2hp3AF4hPvngcYDUvOGSjeD2cgsfnff7+Y8P+10AqkUUvPzoJvl33dpisRPfuco1v5ptlm3ocXYgsHJwaokJVL1jfBnUUqMLQrqAf2ePcDTP0L88mXo77qrKOe+snQFCTaBoakhDEwMoL8tfUWwpU4PX4RBjGHTqikHJgdAEiSOu45nHFMMgXPYAjBo5A0tOx1G6NUULk4H8GsHWgr/hQoEy/FYDBeolDCq4Y8k8MTLT8CiNuP+sz5Qd1okhYlSpQQA/PZ9nfjwUbek8lmPYiolYhcuILWwgNibb+bcxqKxyF7lLpd9AwCe6n0KPPiSEvtrQw2LMa74v1f/L0Z8IyBA4MnXnsRTvU8VfKzQL34BAIidf7PopMRmgsLXgyZp4TsU2zpKCZZjcXr6NI67joMmM8fnbrMbPx/7OWKpGHQq5dd4LgwvDsNEm9BqasXM4LdB1dVB27Vber+3tRe/mPgFRv2jihXHWwk1+0YNWbEcFgb6CcICW5mDLqmVWq06ToflxHJR5XOFQAoxLJZSYsUDX24LR3xY6PYOzE0qWpFmPB6o3R3S5w4l07MjKLsdhFqdk5QIM2GMB8c3dSN1GV3YZt2W96fD0gGS2Pwt7e6mu6FX6RVJ5eT67h06wWJwO4Rdxt68AHZpCQAQHhjIu204GYaW0hZE0tGTKwSqwomFukNY2U1UuIHDs4F9I5xI4Ymfj6DHZcb/emcXzlz34ekLyjJPlGDo6gL+89IsPnViO+wWYVX2Llcn7mg0Zfwc3V6Pjx3vxE/OTeGsRxh4SqREu0BCinkrYv3peOxVECDyEn5yiM5c0KkpaOnViTRBEOhr7cMrM68gmqye/I7ktBeUzQa92YDlLJkSoq2n+Z47AaCouRIjPuFZ0GxoxtfPfh2xVLpVQ1zlX2/hGJocwp0Nd8KqzVQlWbVWgDXAYpYfXEmRBHqcFlyYqkzYpS+cAMcDDQXaNwjT6zg3fw6f2vmbMMWEBZXp5ZiUaaIUKoqEMQ+hU0xSIjwwCABIeWeQ8mefNFo1VkVBlypSVdSJWi5oVdqSn0ciJYqQH+CL+fCXb/wlDjUdwu/s/x08N/4cXvK+VPDxxOdpfHh4059NhBhGWUz7hni8raSUuLh4Ef64P8O6IUKsi50IThT1vGLIJZFiET51Csbe3jSVYDWHNhcTNVKihqwIRlduIho7NKrydaQDAGUVSAlLSg0evGz5YKkwE5mBmlQXrSpJbBgoJynBMwwSly+DNJsRTARghLxBGJ9MgpmagrqjQ1rFXm89IUgSdHNzzgnEqH8UABSHXFYKakqNY65jGJocAsdv7NmMJqNYSizJIq1EdcTtEHYZHjgJqFQw3HsvQgODef2vISZUUJ4EAKhuCpPeKJdfSbMelMkEqr6+sg0c9UZpRTwXvvX8VcyHEnj8oR586IgbPS4znvj5CMKJ4pO18SSLx/79EjrrDfjYfZ2r1rU8hNsnT2yHy6rDl/7tEpIsJ9wHSBJmZztsWptE6o4tRGA3qPHy7GnsdezNez+lXYWTEtnQ19oHhmM2NREoNsTKN5tBnTVTYmwxDIok0L6jDaqGBsSKSEoMLw7DprXhq/d+Fd6IF3934e/S3s9WC+oNe3Fl6UpG64aIuWAcbMIOglZGuPa4LBiZCSLFFs8fLxdiHWhjAUGXBh0DTcN/Yoe1Gw+ajwGAlCnRsibTpJgoLilxUlKl5prcWjVW+ZkSTABmtfmWkZZbtVZYNVZ4Apsnrb/x+jcQS8XwhcNfwEd6PoJWUyu++spXM/K55CAx5gHj8YA0mxEfGQHPspv+fEBplBKAQEpspUyJgckBqAgVjrmOZX2/mGSViCQrhFx22bsQPfcGuGAQxhPppIhNa8M+x74aKVHD7YlgTHgQ6c1NZT+32PVtZQQypNI3NG9Y8EQXYxUeWCUlypkrEb92DXwyCdtvfBhRLQFdJHva+3owU1NAKgV1RwcM6hWFR5Y6U9qVO5iu0JDLSqK/tR/zsXmM+kY33FYM0JNj73HoV5QSt0HYZejkAPT33A3LQ+8Cu7iI+MWLubdlQgXlSQAAceMmSL6w60njdoPxjBd03mKg02HAXDCRk2C4PBvED14cx6/f04Y72+pAkQQef6gH86EEvvX81aJ/nu+dGsO4L4o/eagbGhWFmbBAyOZbPdOrVfhfD3bhylwIP3xxHMlpL1QNDSBoGs2G5lWlxGIYrY4khn3DOVs3RIiWDSW1oPlwV+NdMNGmqmrhEEkJq16dNVNibCGCNpseahUJbU8P4peKtyo67BtGt70bdzfdjQc7H8TfD/992uRLUkqsqQUVB8O5/u8uTAXAMQ6EOWUqnr0tFsSTHK5XIOxyLigQmYXYN04v/iMIKoL3d/w++KDw2SmLGVNL0YKsG3JAGorTvsFMTiJx7TpsH/4QgNwqHKvGimgqKmvyHEwEy2LdKCfc5s03cLw+9zp+duNn+I3u30CnpRMaSoMvHPoCxoPj+OHwDxUfT1RJ2D78IXDRaNFI9VJkSgBbj5QYmhzCgaYDOccjbSYh56GYYZfXlq8hySXRVd+F8MmTIGgaxmOZpEhfax9G/aNVF9pcTNRIiRqyIpAIgGJ5mCzFZU3lQGTvV5TDFb+hzURmilYHCqw2DGSb3JcK4oDW8uCDiJnU0C7kru9cC3HCpulw51V4qPKk5Y/4RuA0OIsuCywljruOgyRIeVWkCoJQbVobSIK85ZUSzM2bYG7cgKm/H8bjxwGKQiiPhSPEhArKk+BZFsmbE9DydEHSfHVHh1R5WwlsW0nZz2bh4HkeX/q3SzBrVXjkV1YDZO9sq8Ov39OGH7w4jsuzmWGEhWLCF8VfD1zHA3uacfwOgTybDk/LImTf1tWIE7sa8M3nryIyOSXZL5xGZ5pSQmcRiJRcq+0iSJ0OlM1WNKUETdK4t+VenJo6BZYrzsriZsDzPJIzM4JSQk9nrQT1LEbQuVINqe3pBuPxgA1vfkIaTUYxFhiTlGufvvvT0FJafPWVr4LnhSabRrMWKpLA9PLqNTU0NQS32S3Jl9fj4tQyeMaBZcan6Nm2ZyXs8sJUoMDfqHDMhQojJYZ9wzg1+zMkl45AjzawAWERh7IImRKFhFzKQbGUEuLE1vLgg1B3dCCWg/ASgwrlqCWCTLDkIZflhtvi3tSKeJJL4omXn0CzoRkf2/Mx6fV7Xffi/vb78b0L38N0WFmYe3hgAJpdu2B+29sAFM/W5Y/7oaE0sgPQ5cKutVd8DC8XE8EJ3AjcyPt80tN6NBmaiqqUkEIubd0IDQxAf/gwSENmA49oKTk1dapo56421EiJGrIimApDnyBQbyy9P3A9KMuKfWNlXFPpkBxv2Fu0OlCgQkqJ4WGQFgvolhZETTQ0s0tgwxsPHMUJm9rtljIlsn1u2ukEu7gILpHIeG/YN7ylVBKAIN3c79gvSyonZY7I+I6oSBXsWvstr5QQCQhjfz8oqxX6u+5C+GRuUiKcDBeklEjOzIBnGBgobUHXk7qjA+zSEtjlyljExAaObBaOn5ybxqvjS/jcO3ahbl2uzyO/shNmrQpf+rdL0kRyM+B5Hn/8H8OgSAKPvnM1XEsuIUsQBP74wW6kOB7+sQmJlGg2NGM2MovlCANfhEGIehMtxhZss24cLFpILWg+9LX0wR/34+JibsVOucD6/eDjcUkpEYgl0+wLHMfDsxhBxwopoevuBngeidGRTZ/76tJVcDyHLptwT67X1eNTd30KL8+8jOduPgdAyHposmglpUSYCePs7Nm8CpeL0wE06loBQNHqcofdAKNGhYuVICWCCRCE0KQhFxzP4SsvfwVWjQ2JhbfBF2bABQVyMKU3YjHMlIyUoIpESoRODkC9fRvUbW2CCieHfUMkGeSSEreiUmIxtljwAtK/jP4Lri9fx2cPfhZ6On2y/8g9j4AgCHz97NdlHy+1tITouXMwneiHurMThE6HWJFyJfxxP2xaW9HtNzadDYFEAElOnjq3khDHe72tvXm3K3Yt6IhvBGa1GY75BJITEzCdyJ5n0WHpQKupFQOT+TO6tjJqpEQNWRFio9AmyLI3bwAAqdGA0GphCgqS5kqG5CTYBHxxX1GVEpUIuoxfugRddxcIgkCU5qGPcYi8cGbD/ZhxD6i6OlBWa16lRK5gukAigInQxJbJk1iLvtY+XPZfxkw4vxx5JjIDFaGSQiw3Qr2u/pZXSoQHBqG54w6oW4VJivHECSSuXgUzlX1VqNBMCZE0M9DGwkiJlQaOSoVdttn0IIjVUEMRgWgSX3tmFHe1WfHeA60Z+9UZ1PjcO3bh1fEl/OTc5muTnx+Zw8nL8/iDt+5A85o6QiWEbJtdj9+9rwP6gA+zOiEI0Wl0Is7G8ebMFEAwmI6/ib7WPlkD32KTEsdcx6AiVFXhyRV/L9rlRJ1eCHcNxFYH7dPLMSRSnERaabuF+2euFW0lWFs9J+J9O96H3bbdePLsk9J15LLqpEyJM94zSHGpnKQEz/O4OB1Al0OoZPYE5V9PJEmg22nGhenykxLzwTjsBg1UlPyh8E+u/QQXFy/iM3f/IUheC184ATYgkBJzvDBeKp19wwCeYcAnC5/gscEgoq+9BlO/0Iak7e5CanYWqYVMotyqEa5jOWGXgUQAZs2tR0oAheUHzEXm8Dfn/wbHXcdxojWzearJ0ISP7/s4BicHMTQ5JOuYkVOnAI6Dsb8fBEVBu3t30Wxdvriv6HkSwGpGhdzA1EpiaGoI263b0WrKfOauhdssKGiKsSAACBbnLnuXpGAy9vVl3a5aQ5uLiRopUUNWhPgYNAkKdqPyAKhigDKboQvEoCJUFZV+iRPSoiol1OVVSnCJBOLXrkHb3QOWYxHhYjBCIwQRboCExyO1FGgoDVSEKqdSAsgkJcSQy62mlABWvdNDU/kHDN6wF42GRlCkvEDYBn3DLd2+wQYCiL72Goz9q2y/qb8PQO4WjkIzJUR7kUFnQSRViFLCLRxnfHO+4UKhpSm01OkyakGf/K/LWIoyePzdPTkD8957oBV3tVnxtWdGEcgi/5eLKJPCn/zHCHY0GvEbx9zS64UQsh/dbYSK5/DvXg6JFCvt++aMByrDNaT45IZ5EiJEUqJYAz+LxoK7Gu+qDlJiWiQlXJIKZq2FQ/w+dK7Ye1T19VA1NxdFqj28OAyHzpFWSUyRFB49/CgWYgv42/N/C0CYWItKiaHJIVg0Fuxz7Mt6zJlAHIthBve47gBJkIpXEfe2WDA6E0SyzGGXc8E4Gs3yxzj+uB/fev1buLvxbjy47Z2wGdRYjDBgV5QS3pTwDCilfQPAptQS4dOngVRKuj/renoAIOuKu0RK3K5KiRWrUiFhl0++9iRYnsXnD30+Jwn78O6Hsc2yDV87+7WMBpxsCA0MQuVwSCSltqcb8dFR8KnNhx77Y/6i50kAq20e1d7AEUgE8Prc67KeT26LG+FkuCi/U4JN4NryNXTbu4XFnK7deWuw+1v7keSSVRXaXEzUSIkasiJIJEAnaTgqoJQAhMAoBIKo09ZVlJQQ8wK2slIicfUqkExC29MjTdzqWrchPDi04cOM8YxLEzeCIGBQG7IHXTpdADJJCbF6rtu+9ZQSHZYOuM3uDScxSjNHHHrHLW3fCJ86DbBsmgRR7XZD3dmZkwgrNFOCGRdSyA06c2GZEi0tgEpV0VyJznqjVJcJABemlvHPr0zgQ0fc6Hbm9miTJIHH392DpSiDJ//rcsHn//bJ65hejuHxh3pAr1kxLoSQJeeEAK5hTo/vn/ZI+171T4A2jcJIm3BX412yjkU7neDjcbA56goLQV9rH24EbmAyOFm0YxaC5LSgbqGdTtTpRVJiNUxQ/D6IpAQA6Hq6i1IBKIZcrsdex16854734J9G/wlXl66ixarDbDCOWJLBqelTuM91H1Rk9rpKMQ/izrZ6OA1OxeGAe1qsYFIcrs7JyzoqFuaCCUV5Et96/VuIJqN49PCjIAgCdoNmRSkRAGk0YjokEEulVEoAmyQlTg6Astmg27cXAKDdvRsgiKwr7nJJCY7nEGbCt1ymRKupFSRBKv4+v+R9Cc+NP4eP7vlo3lV3mqLxxcNfxHR4Gt+/+P28x+QYBpHTpwWVBCncp3U9PeBjMSTGxhR9vmzwxX0lyf0Sj1lpG/ZGODN9BizPyiMlRAVNESwc15auIcWlsEvTjtgbb0gKplzY37AfJrXplrVw1EiJGrIirEpBlaIrppQgzRawgaCQ3FvBm1kplBJaSguKoMoWdCkOZHU93QgxwqDPvmMP2EAAsfPnc+7HBoNgfT5oVpQSgJCHkVUp0dQIUFQGKTG8OIwWY8uWHaz0tvTi7OzZvKoWpZkjDboG+OP+LeGxLAThgQFQdju0e/emvW460Y/Iq6+BDaVPPBJsAgzHFKSUEJQ8Qt5JIcojgqahbm2tLCnhMMCzGAHP82A5Idyy3qjBp9+2Y8N9u50WfOiIG//8ygQuTCmXx16fD+PvTo/hPXe6cKgzXbpbCCErXv879+3AX528Bo4RJjU3A1OgzVdwvOVe0CQt61jFrgUFVgM2K93CkfR6QRqNoMxm2ESlRGQtKRGBSaOCY83zV9vdA2Z8POP6UYJoMgpPwJNTufb7d/0+TGoTvvLyV+C0asHxwMD4qwgkAnkH65emA6BIAl3N5oLCAfe6hOdDuXMl5kMJ2UqJ8/Pn8a/X/xUPdz0sZaLYjWoshhlwwQAosxnTSzEhj6OANg85IPVCLgFbICnBJ5MInz4NY18fCEpQdZAGA9TbOrMSXlatPFIixITAg7/llBJqSg2X0aXo+8ywDL76ylfRamrFR3o+suH29zTdgwc6H8APLv0gL/kRPfsquEgExhXVIQBoV1Qu8eHNZc3wPC9lShQbYvVztSslBicHYdPasKd+z4bbigqaYoRdiu107svLAM+nKUyzgSZpHHcdx+np01UR2lxs1EiJGrIiTLMgklrYDZVSSljABoOw6yqb3OuNeEESZJrUdbMgCAIG2lA2pUTs0iVQVitUTqdEhNR33QXQNEJ5wgfFqinRdw8IpEQomTkoJlQqqBobkFpPSmzBkMu16GvtQ5JL4kXvi1nfT3JJLMQWFCslAMAXq+6HdCHgGWZl0NsrreaIMPb3A8kkIi+8kPa6SJQVat/QuAsnJYCVBo7xSpISRkQZFrPBOH50dgJvTgXw6AO7YdbKm7x/+m07UG/U4Ev/dgksJ9/qwPM8HvvZJWhpCp//b7sz3i+EkBUJhE/8+jGQBIE/f24SBtqA6eTL4MnQhq0bayFZwopUCwoAreZWbLNsq7iFQ6wDBQDrSqbEWqWEZzGCTochTfYtSrY3MwEZ9Y+CB58z48eqteIPDvwBzs2fw1RSuE5/eXMAKlKFo86jOY97YTqAOxqM0NKUVKPI8fKtGO12PUxaVVlzJZIsB18kgQbTxgRCikvhiZefQKO+ER/f93HpdbtRI2VKkCvNG01mraKMCiXYrFIi+vo5cMFg2sQWEIJUs1mDNJQGOpVuQ1IimBDsK7caKQEoDzX84fAPMR4cxxcOfQEaSh7h9Zm7PwMNpUlrwFmP8MAACK0WhiNHpNfUbjdIvX7Ttq5QMoQUlyqtUqKKGziSXBIvTL+A3pbeDZumAIGo11CaoiglRvwjsGqsMAydg6qhAdrujcfL/a39VRPaXGzUSIkaMsCyKUQ1PAhWW9FMCTYYgE1rqyjDOhOeQYO+QfbqnlzkUhyUAvFLw9D29IAgCASZlcGDuR6GgwdzevyBNc0ba5QS+SZ/tNOZNoFYji9jOjy9JUMuRexv2A+LxpJzEjMXmQPHc8qUEisE160Ydhl9/XVwoRBMJzIliLr9+0FZrRlEmEiUKQ265KJRpGZnoe7ogIE2FBz8pO5wg7k5AZ6tzKrDtpWGhdfGl/Dkc1dwpNOOd+2T/30ya2k8+sBuvDkVwI/OTsje7z8uzODMdR/+6Fd2wmHKvM8XQsgmp72gbDa4mmz4n2+5A78cnYdJ5UBSNQECFI65MrvXcyFXTs1m0dfah9fnXkcgUf5gRRFrSYlV+8aaTImFsNS8IULbI5IShU9AxFW5fETxu7e/G/sc+/BvE98FyCheX3gBB5sO5rw+eZ7Hxall7F2p9uywdCCWiim6vxEEgT0uCy6VkZRYDCfA8/LqQH985ce4snQlo0XBblDDFxYyJUSlRKnyJIC1pERh97rwwEkQajWMR9MJJm13D1ILC0jOZf6fWTXWDUMKpXHFrUhKWOSTbNPhaXzvwvdwf/v9uNd1r+xz1Ovq8ck7P4kXvS/i+ZvPZ7zP8zxCAydhOHYMpHb1+0qQJLRdXZsmJUQ1sqhqKCaMtBE0SVc1KXFu7hxCyZDsvCOSINFmbiuaUqKrbjciZ16E8US/rBBoMbT5VrRw1EiJGjIQ9M+AJwhwnF5RVVYxQVnM4JYFUqLSSgmnoXjWDRFGtbEs9g0uHkfi2jVpQCue06Q2wdjfD8bjQWIs+ypxwuMBKEpqUNjoc69Pyx/xb908CREqUoXjruM4NXUqq1RuJiKsJitSSqy0dNyKYZehgQEQanXaao4IgqJg7O1F+NSptCwTUSmhdEC7quTpgJ7WF66UcLvBM0zRJ79y0bGSG/DYz4YRSaTw5Ye6FdeyvWufE0c67XjyuSvwhTNredcjFE/iiadH0OMy4/891J51m0II2bWT7Y/c24E7GoyY8wuTuDZ9jyIbF2k2gzQYSkJKsDyLM9Mbtw+VCmv/Tno1BbWKlJQSUSYFbyAuNW+IUNXVgXa5NpUrMewbRqO+EfW6+pzbkASJRw8/ilAyAJ3r/4OPmc47WJ9aimEpmsSeFkHqL/qtlYYD7lkJu0ykykMOzgWF62Qj+8ZCdAF/9cZf4ZjzGN7a9ta09xwmDUKJFFLLy6BWlBKlypMA1pASUeX3OmFiOwj9kcPScUSs2gAyJ7dWjXVDpUSAEcikrWrTzAe32Y04G8dcZG7Dbb9+9usgCAKP3POI4vO8f+f7scu2C3/66p9mPMsSV64g5Z2RAqPXQtvTg/jly5sKuxQX/kqhlCAIQlhcrGJl6ODkIDSUBoebD8veR2zg2AziqTiuL1/HjrgFfDQK0wbWDREmtQkHmg7Ibm3ZSqiREjVkYGlJmGjxvF62hLjYIM1mcNEobGorYqlYxepvZsIzaDYWL+RSRLmUEokrVwCWlRK2ReuFiTZt2IjAeMZBYb1BVQAAIABJREFUt7hAqFeJqQ2VEnNz0sNRDLncbc+Uhm8l9LX2YTmxjDcX3sx4zxsWJkxKlBKifWM+dmspJXieR/jkAAxHjkje5/UwnjgBLhBA9Nw56TXxOylWzsqFREp0dECv0oPhmIJyOsTMFPF45UaTWQu9moI/wuCjxztwR6NyGwtBEPjyQ92IJFL4ys9HMemP5v158rkrWAgn8PhDPaBytHsUQsiunWzTFIkvP9SDREwgmw43yV85FH+nYteCAsCe+j2waW0Vs3CwoRC4UEjKzCAIAnV6WsqU8Kxr3lgLbU/PpmpBR3wjsux0u2y78Os7fx0q41UAwB3Ggzm/S6euCeSqmAtRqN96r8uKJMvj6uzmyXqO4ze8BkZnhNX9jZQST732FBiWydqiINpbU4EgCJMJs8F41SolmBs3kJyYyDrx0e7eBZBkzrDLhdgCpkJTOX/ELIRbUimxQrK9Pv963r/B02NPY3ByEB/f93E0GZoUn0dFqvDFQ1/EfHQe33nzO2nvhQcGAILIWhWp7ekBn0ggceNGIb8egFVrRSkqQQFUfHExH3iex8DkAA41H0pTQW0Et9mNqdDUprLBrixdAcuzaL8aAKHTQX9YPinS11Idoc3FRvYo5RpuaywvCYywirbmrKMrNSiLsOpSB+EhPBuZRae1s6yfIcWlMBedK4lSwkAbynKTjq3I+kQ/8lr/Pu2qg2bXLoQHBmD/aGYgEzM+Do27I+01I23MmYVBO50AyyI1Pw/a6cSbC2+izdS25Qcqx5zHoCJVGJwczGgOEMMAlQxCbFobKIK65ZQSiWvXkJyagv1jH8u5jeHYMRA0LZAXBw8CWM3WUPo9SXg8AEFA3d4Gw5hwn4gmo4pX60R7EuPxAMePK9q3GCAIAnc0GLEQSuD3TtxR8HHuaDThN4934jtDN/DTN6Y33P7/OdiGO9vqsr7H8zwmghM41HxI9vl5nkdyZgbG3l7ptSPb7Ohp2IarqZfx4B33yz6WiFKQEhRJ4b6W+/D8zedLFu6WD+LvI5I3gGDhEO0bYwsrpER9Jkmn7elG6LnnkPL5oLIrm0CEmTDGg+N4Z+c7ZW3/yTs/iR+PPg0mYcB///ZlALkbXtQqEjubBDLNoXNAr9IrbiwQ7R8nL89jT8vmVtw/9aM38POLM7K2bbLkJiWu+K/gGc8z+O29v412c6aiyG7UgGZT4ALLiOuNYEN8eZQSBWRKiLa5bBNbUqeDZts2xLIoJRx6B16aeQnv+Ok7NjxHnTb7/WQrQxx3fv705zfcdptlGx7e/XDB59rfsF9owBn5Jzzc9bBknQudHIBu716o6jMVTmIGQfzSJWh37izovKJ9o1T3QpuuekmJG8s3MB2elhVKuhZuixssz2IqNIUOS8fGO2SBuHDnGrwM473HQGrk2+V7W3vxp6/+KQanBvFwV+HfuWpDjZSoIQOBkLCCq9GUd7C2FpRFmKDso4WL/Yz3TNlJiYXoAlieLZlSYjJUeoYzfmkYlN0OVZMwaRZJCdEfbOzvg++730NqaQmqutUBBc9xYG7ezJDh51N4rK0F5RrteGXmFbxr27uK/juVG0a1Efc03oOByQF8+u5Pp703E55Bva5edqAVIMij7Tr7LZcpER4YBJB90CuCMhqgP3QIoYGTaPjsIyAIAi96X4RVY1X8YGc846Cbm0FqtVLNbiQZUUxKUDYbSLNZIDkqhG++fz8okoBBs7lH8h++bQf2tVgQYfJL4LU0ifu7GnO+f3XpKhZiCzjYdFD2uVm/H3w8njbZBoDv/eon8Ny1o9jXtE32sUTQLieib7yheL+N8D+6/weevvE0vvn6N/H4sceLfvx8EHN3MkiJFaWESEqsz5QAAMPRo1j4828gfOo0rL/6bkXnHfWPAoDsjB+T2oS/7P07XJ4Jw3Z3fmLebddDSwttDgRBoN3crjgErtWmx/1djfjO0A3897tbClYcPD8yh59fnMEHD7dhf2v+SXKDSYP6PLlZv5z4JQgQ+MDuD2R9325UY8/iDRCpFJa3dQPnUSalhHJSIjwwAG13N+im7AS6tqdHsNbxfJoi5Pfu/D1Z9wGHzlGSTIJKo15Xj+/e/11ZiwjHXMdAU5tTF39w9wfx02s/xdDUEN67471Izs0jfvEiHL//+1m3V7e3gzQaEbt0CdZf+7WCzikSBmLbSrFh19oxtrz52tJSQGxikpsnIWJtLWihpMTw4jBsKgssNxZg/Ig864aIVlMrvnT4SzjSnGmV3cqokRI1ZGA5JNx89YbcvtNSgzILpIST0WG7dTuGJofKzgaKq+AlUUqoy9O+Eb90CdqeVY96mAlDp9JJPnHTiRPw/e13EDl9GpZ3rRIIqZkZ8PF4WvMGICg8EmwCSTaZ8fBdG0z3WlMUsVRM8Y2+WtHX2oevnf0axgPjkjwZWKkDLeD70aBrwGJssYifsPIInzwJbU8P6Mb8wYjGE/2Y+/LjYDweUO42nJo6hb7WPlAkpeh8jMcjqRxE2WUhNi+CIISwS8+44n2LhfX5AYWCpki8Y8/mSdSByQEQIHC8Rb5yRFIAuNKvB7vehA/su6+gz0E7neCCQbDhMChjcf5GgLD6+XD3w/jBpR/gPXe8B3c23Fm0Y2+ErEoJA40rswJh7FkMw2nRQqfOvB60XV1QNTYifPKkYlJCTsjletzX0YX7Chhvuy1uXFi4oHi/xx7swlu/MYQv/8cwvvvw3Yr3jzEs/vhnw9jRaMRjD3aD3mQLxuDkIPY59uVcQa43aHBodhicWoMJdxdw/nJJlRKEWg2oVIpJiZTPh9j586j/xCdybqPt6UbgX/8VqdlZ0M2r95BGQyMe2v5QwZ/5VkC+5pliY7t1O1xGFwYnB/HeHe9FeGgQgPDczAaCJKHt7s5qvZELX9wHi8ZS9EB3EXat0KK3nvCqBgxMDqDb3q24Ya8YtaDDvmHcEbeAIPww9vVuvMM6vG/n+wo+d7WililRQwYCUYE1NVqU++KKBXKFlGCDQfS19uG1udfKnpYu5gWUSilR6qBLLhZD4sYN6Lp7pNdCyVCad1/b3Q3KUZ/RiJBYmaCtbd4AVhUW2dQStFP4OyW9XgxNDkGv0itaaa1miOTK0FR6sJA34i3o++HQO26pTInU4iJiFy7kHDithehpDp88ifPz5xFkgorJK57n00gJg2pFKZEqLKdF4+6Q2mZqAIYmh7DHsSdvIOJ6ZFMAbBalqAUV8fG9gvf7iZefQIorPCROKZJeLwi1GtQa+0WdXo1l0b6xGMlJUhEEAWN/H8JnzoBLbBxouhYjvhE0G5rLYlfpMHfAG/Yinoor2q+lTo9PnbgDzw3PYeCK8vvjXw9cx/RyDI8/1LNpQmI2MotR/2jee5PNQOPw7Aj8u+/EVERoZyilUoIgCJAGg2JSIjw4BPA8THnuzzop7LLwyW0NmwdBEOhr7cMrM68gmowifHIAtMsFzR25rX3anm4krlwBzzA5t8kHf9xfsjwJQLCFJNgEoqnKZMPlwmJsERcXLha0eGZWm2HT2gomJaLJKMYCY+i4HoZu/37FdrxbFTVSooYMBFfqn6z24isE5ELMlGADQfS29FYkLb2QZgW5MNAGxNn4pkJyNkJ89DLAcVKyNiDYN0zq1SA9giRh6utH5PTptAfaah2oO+NzA6vhhGtBarWg7HYw01MYnBzEUedRqKnKtLcUG06jEzvqdqRVMHE8h9nIbGFKCX3DLZUpER5aGfTKSI+mm5uh2b0boYFBDE4OgiZpxStRqfkFcNGo9P1ca98oBOoON1JzcwXJom81zEfnccl3CX0tfYr2y6YA2CxW1VcbZ2QohZ7W47P3fBZXl67iR5d/VPTj50LS6wXd3AyCXB1+CZkSDDiOx9hCJGvIpQhTfz/4aBTRs2cVnXfYN1y2JiS3xQ0ePCZC8itqRXzseCc6HQY89u/DiCflN3HcWAjje6fG8J47XTjUufkBvphs39+a+56mGr+Bhtgybu68E9PLMdQb1ZKNpVQgDfoCSIkBqJqaoNmdO3Ras3MnoFJJOVQ1VA59rX1IsAm8dPMUIi+9BOOJE3kVBrqeHvAMg8T16wWdzxfzlZSstOls0nmqCaenToMHX7Ci1212K7apibi6dBUcz6Ht4gKMMls3bgfUSIkaMrAUD4JOAQ5bBe0bFlEpEVhNS1/xfpUL3rAXNq0NOlXxVz5EtUIpW0Xi60IuAYGUWN83bzzRDy4SQeTVV6XXGI8HpMEAlcORtq2JFgiNfA0co4HrmI/N3zLWDRF9rX04P39e6mxfjC0iySUVNW+IcOgcWE4sg2ELW9moNoRODkDV3AzNrl2ytjf19yP2xhsYuPlLHGw6KJEKciE2ZYjNGeL+hV5P6pVAV+amsnC+WxGiGkjp9Zv0ekEaDJLKrRhQrbGElQJvaXsLjrmO4a/P/3XZMl6SXm+GxaXOoAbHCxPrcCKFzix5EiL0hw+D0OkQOnlS9jkDiQAmQhOy8yQ2CzEUspABu1pF4vGHejDhj+JvB+U1CvA8j8f+fRgamsTn/1tx2p4GpwbRamrN6xcPnTwJDgRG2/cKdaAlVEmIoBQqJbhEAuEXzsDY35d3YktqtdBs374pG0ANxcGBhgMw0kb88vxPwCcSeRUuwOoYr1BCqdSBv+Kxqy3scnByEE2GJuysKywg1G0pvBZ02CdcZ52z+RVMtxtqpEQNGVhmgtDFkTcAqtQQMyXYQAAUSaG3pRcvTL1QUmXBenjD3pKoJIBVG0QpcyXiw5egcjjSPP5hJpymlAAAw5EjILRahNdYOJjxcag7OjIGMQa1QTpONtBOJ15ST4AkSEV+9K2AvpY+sDyL09OnARRWBypC9C8uxLa+WoKLxxF58UWYNhj0roWxvx9eK4eJyFRB5NWqkic9U6JwpYRwnEqGXVYLhiaH4DK6sN26XdF+Yh1oMT3Dqvp6EGp1yUgJgiDwhYNfQJJN4qnXnirJOdbj/2/vzsPjKsvGj3+f2TKTfd/TpqWllKR0h0IpTVFeN9Si+IKi4Au+LggiKoIIVFZBUEFRQRSXuuD2Iig/FKVJy1ZKC1ialkLbtE2arUmzTJKZzPb8/phMmjSZyUxmS8L9ua5ekJkz5zydPjM55z73c9/u5ubhYEtATqp/LfeOQ11A6BojhpQU0s9eTV9tHVrrsI4ZKHIZST2JaASKwEXagSNg9bx8Pri4lJ9s3s+hzok/00+90cLz+zq47j0LKMiI/rxlwD3Ayy0vU1MR+jutb1MtjcVzOIyNpi4H5TnhtxScLENqZEGJga1b0Q4HGeeeO+G21uoqnLt2hT2vRHyYjWbOLjub53peg/Q0UpcvD719RQWGzMxJB5TiHZQILA3pdE6dTAmnx8lLLS9RUx7+ecuJKjMrOeY8Rq+rN+LX1nfUk+syU5RTgeWkyItAz1QSlBBj9PoGsLoMSQ1KKLMZlZqKr8f/YV9bsRa7286rba8mbAwt/S2TuuAMRyBTIp51JRz19aOWboB/2UUg2yHAYLWSdtZZ9NXWDp+MDB5sGFPkEo6PO1SmxLZCO4vzgxcHm66q8qvIt+UP30mOZnlPQao/A2UmLOHoHzrpTV838UlvgLXqVF5d6p+Ha8sjL/DkamhAWa2YivwdJKJevjF7FiiV1GKXU8GAe4CtLVsnvBgbTyAoEUvKYMBcUhK3oATArMxZXLHoCp5ueJqtLVvjdhzw37X2dnSMeZ9y0vzL3LYPBSXG67wxUnrNOjytrQzu2RPWcQOt5xK1fCPVnEphamFUReBu+sBCLEYDG56sD3mR3Dfo4fa/76a6LJNLzhjbtnMyXmx+EbfPHXIJk7utHeeuXRw+eRlH7YP+TIk4FrkMiLSmhL22FkNqKqlnTNze11Zdjbe7G08cP28iPGvL19JtGuTIe0/zFzgNQSmFrbpqUvVA3F43va7e4SUW8TAVMyW2tW6Luhj7yA4ckao/+gZzGt1knLtuyhX/TCYJSogx+hgkxWUiLz259QCMWVl4e/1BiTNLzsRisFDXWJeQY2ut/UGJOHTegOgvoibi6+/Htf8A1urRJ6En1pQIyDh3He7mZgbfegufw4GnuWVMPQk4Pu5gGR5dJek0FME5eSuj/jtMNQZl8GfsHHket9cdVaZEgc0flJgJbUH7NgVOesMvaqoMBl5dlEplu6LIEvn670DQLLAuf3j5xiQLaRmsVsylpe/4YpdbW7Yy6B2c1InaeMsSYsFcVhrXoATA5dWXU55ezl0v34XbG79svGB1N3JS/b9rXz3URYrJMOEygPSataDUmALFwdR31FOWXhZxu9xozMmcM+n11gBFmVauPe9k6vYe5Z/1bUG3e+Dfb9FuH+T2D1djNMTm5L6usY4MSwZLi4J3ZemrqwOgY/EZHOjox+XxJWT5hiEtDd9AeOcNWmv6NtWStno1hgkubAGsQ0WxHbKEI+lWdudi8GleXRTe0kZrVTXOt97CF2Gxy65BfyA03oUuAY45pk5Qoq6xjlRTKiuLJ3+uOtkOHP3ufhrsh5jb7I3oZs47gQQlxBj9Bhdml5m8JGZKgH8JRyAokWpOZVXpKmobaxOSWtjp7GTQOxiXzhswIlMiTss3nHv2gNaj6klorcetKQGQvtZ/t7pv06bhdfUpc8aupZ0oU+LlbH+by9UqeKXo6aymooZ+dz+vtL1CS38LmZbMiOshwIhMiWm+fEP7fPTV1ZF29tlhnfQGdDm72G07xoq3vAy8HFnBPgBXw8FRQTOLwYJJmaIK8lnmSAeOzU2byTBnsLwodLrwibx2O77e3phnSoC/rkS8gxJWk5VvnPENGnoa+NXuX8XtOMGDEv7lGwc6+pmTn4ZhgotrU14etiVL6KsNMyiRwCKXAZVZlTT0NkT1+/qyM2dzSnEGt/2tngHX2A4pb7b28ugLB7l4ZQVLZ+VEM9xhXp+XLU1bWFO2JmSLxL5NmzCXl2OaexIuT/w7bwQY0tLwhpkp4azfjae9nfQwlm4ApCw4Gczm4XpUInnUlm0sbIKXLOEVi7VWVYHbzeDetyI6TqD4ZDyDEmajmQxLxpRZvuHTPjY3bmZ12eqoirGXZ5RjUqaIg69vHnsTjWZet43U5csmffyZSIISYox+swejx0JeWpIzJTIz8fZ0D/+8tnwtR/qOsL87vOJX0Wjp86fmxy1TwhLfTIlAwSPbiKDEoHcQt89NpmVsITpTQQHWxadhr60bs15/pIkyJV7wvU3xMU1p58xck3pGyRlYjVY2N26mua950st7slOyMRlM0z5T4vhJb2SFmp478hw+NCsOm+mrDb9gH4DP5cLd1DQqaKaUItWcGl1QorIS18GD79j11CNP1CLtVx+PzhsB5tJSvEc7Im6BGalzys/hXbPexU93/nQ4CyrW3Ef8XUTMpWWjHs8Z8bs2VOeNkdLXrcNZX4+7tTXkdt3Obo70HUlYkcuAysxK7C57VCnbJqOBO9ZX09zj5AfPju4soLXmlr/Wk2k18fX3hFdgNxxvdLxB12BXyGwh38AA/Vu3kr5uHXkZ1uHHE7d8I7yMsL7aWjAYSF97Tnj7tliwzp+Ps16CEsnWt2kTZzrL2dd7gCZ704TbW4dbukb2bxf4fMZz+Qb4gx5TZfnGns49MSnGbjaYKc8ojzhTov7oGwAsOukslDmy37UznQQlxBj9Fh8mrzXura0mYsjKHK4pAcfXnieiC0dz/+RT88MR90yJXfWYiotHdc8IHCtw7BNlrDsX586d9A+1mrPMHrs+12ayYVCGcWth9Lv72W6vZ8XbGk9zSyz+GlOOzWRjVekq6hrraOlvmXQhVIMyUGAroMPREeMRJlZf7aahk97I6kLUNdZRaCukev5q7BEU7ANwNzaCzzcmaJZmTosyU6IS38AAnvbpnb0yWbs6dtHp7Jzc0o0j8Q1KQPw6cIx0/crrAbhn2z1x2b+7uRmMRszFRaMez0gxYRrKjpibH7zI5ajXDAUCA8sIgkl0PYmA4Q4cUdSVAFhRmcvHlpfzs+cOsK/9eCvq/3v1CNsOHuOG950yKqgTrdrGWkzKxOqy1UG36X/ppeGuCPkjlrkmsqZEON+Z9tpN2JYswZQb/gWntboaR/3ud2xwdipwNTUx+Pbb1Mx/D3C8I1Io5rJSjNnZEXfgGA5KxLkGWK41d8oEJeqa6vzF2MuiL8Y+O3N2xN9xO/e9SF6vZtY57436+DONBCXEKB63C0cKWIh/FemJjKwpAVCUVkRVXlVC6koEMiXivnwjToUunfX1Y+pJBCoEj1dTAhjuldzz+F8xlZRgSB07B5RSQS/+Xmp+CbfPzcoj1oRcQCRLTXkNzf3N7O/eH1XQqiC1YNpnSthr67AtXYopJ/zUaZfXxQtHXmBtxVqy1p2Lp6WFwTffDP/1gUyeEwqxppnTomqxG8i8eKcu4ahrrMOojJxddnbErx3OlCgrm2DLyCUyKFGSXsJnT/ssmxo3saVpS8z3725uxlRUiDKZRj2ulCJ7qK5EuJkSlpNOwjxrFvYJlnDsPuYPSizMi02rzHAF1ltPtgPHSDe87xTSUkzc/Fd/0cueATfffnoPy2Zl87HlFVHvf6S6xjqWFy8fN6MwwF5biyEjg9QVK8hL8y9zzbCayLTG/66nITUVPB70BLUD3K2tDO7eE3G7QWt1Fb6eHtxNE9+dF/ER6IS2cN1HmJs1l9rGiZdpKaWwVlfjrN8d0bESFZTIs+VNmZoSdY11LClYQo41+iVflZmVHO49jE/7wn5NfWc9c9oU6WtmVoe6WJCghBil+5j/xM9iHP/CNZGMmaODEuDvwrHz6M7hdXDx0tzfTLo5PeSJSTRsJhsKFZdMCW9fH66GhlFLN+B4ACRYUCLl5PmYy8rQTieWyuBVzNPN6eOOu7axlkxLJotU+YwOSqyt8GcFaHRULWMLbYXTuvuGu7mZwT2Rn/S+0voKA54Baipq/BkWSmHfFP4SjsEgy4uiXr4RCEocfGcGJWoba1lWtGxSxRDdzc0oiwVjXuzXJQeWOiTqO+WyUy9jTtYcvv3yt3F6nDHdd6gOJYG6EhN13ghQSpGxroaBl7biGwgejKvvqGdWxqy4/S4LpjStFIvBElWxy4C89BSue88CXjrQyZP/aea+Z/ZyrN/F7eurJ6y/EYnDvYc50HMgZNcN7fPRV1tH+pqzUWbzcEHwRNSTAH+mBDBhB45AvZFw60kEBOpQSV2J5LHXbsJy0klYZs+mpqKGHa07sLvsE77OWlXF4Ntv43OG/73V6ezEbDAHzaCNlamSKdHS18Kbx96MeulGQGVWJYPeweFubBPpc/XRZOrlFGMpxszEfidPBxKUEKN0d/pP/GwJPoEZjzErE+1wjKomvK5iHRodl7tYI7X0tcQtSwL8J5Tp5vS41JQIRMrHtAMd+qUWLCihlBo+gRmvyGVAuiV9TIaH1+fluabnWFO+BmtJ2YwOSuTb8jkt/zQguuU9BakFtDumb6ZE4A5tpNWj6xrrsJlsnF58Oqb8fGyLFw/fGQqHq+EgxoJ8jOmjT6LSTGn0eyb/eTIVFaFstndkpkSTvYl93ftCXoyF4m5uxlxSMtwNJZbMRYVgMCTsO8VsNHPTGTfR1NfEo7sejem+QwclApkS4V8cpK87F+1y0f/ii0G3SUaRSwCjwciszFk09Mbm8/Tx02exuDyLDU/W85uXD3HpmZVUlca2m0ggCzMQeB6Pc+dOvJ2dw997gdbp5QlYugEjghIhAlEA9k21mGfPGrc2VCjW+fNRZnPEywBEbHjtdgZe2T4c7K+pqMGjPbxw5IUJX2utrgKPh8G9e8M+3jHHMfJseXFvS5lrzaV7sBuPb2zB2kQKLIWJWVAiwragO/fUAXDa3LNicvyZxjTxJuKdpLvH33orNc5Fb8JhGIoi+np6MAzVRliQs4DitGLqGuu4YP4FcTt2c39z3IpcBox3cR8LgTsc1hMyJQJLBTLMwbNgMtbV0LVxI5bKEEGJcYIpOzt2DhcHM5duZ2D79skOfwzn3rc48qUv4e3pCb2hyUTR9deT9cHzY3bsYNZWrGVnx86o5khhaiF2lx2Hx4HNlJgT2ljRWmN/+h9YZs8mZW74J71aa+qa6jiz5EysJn+BuPR16zj6/e/jOnRo3DomJ3I1NJAyzvxMM6dF1c1EGQxYKiuHMzEmMrhvH803fpPim2/Gtqh64hcEobWmdcO3sD/zzITbWk9dSPlDD0XU6SQcgRO1dRWRZb0ExKsdKIAymzEVFeFJYKDz9JLTed+c9/HzN37O+XPPZ1bmrKj3qT0ePG3tQYMSuWkW8tMtZNnCXwKQunwZhsxM7JtqyXj3u8c8f8x5jJb+Fi5ZeMmkxx2NysxK9nXvm3jDMBgNitvXV/PhH71AfnoKX/mvk2Oy35E2N21mXvY8KjKCLwmx19aB0Uj6Of7U65xUMwaVwEyJdH9QouGCj6CMwet+eXt6yL3ssogvNpXFQsopp+CUtqBJYf/Xv8HjGV5Oe1r+aeSk5FDbWMt754SuQWCrDrR03YVt8eKwjtfU1xT3pRvgD0poNN2D3eTb8uN+vGDqGuuozKxkTlZkwbpgRrYFDVWHJuClVx4HMyw7+6MxOf5MI0EJMUpXj//CNSu9MMkj8deUAPD29g4XbFRKsbZ8LU/uf5JB7yApxvi0LW3pa2FZYXxb9URbmC8YZ3095tLSUcWtBr2DPPLGI1RmVjI7K/iFX+oZZ1B4/fVkhriwTzOn0eXsGvXYcHGw0tW4Spvx2e147XaMGdEtA9I+H60bNuDt6SHz/e8Pue3Ajh203nYbaWeuwpQf3196Fy24CIvBEtU67cAv5o6BDioyY7suOt7s//oXA9u3U3TjNyJ63d6uvbT2t3Ll4iuHH8tav56Ohx+m7dt3U/HQTybch+vgQTLOO2/M49Eu3wCwVM4Oa02u1prWb92Kc+dOWjbcwpw//SnkBUJmL5zuAAAgAElEQVQofc8+S/cf/0j6unWYS4JnZ/kcDnoef5xjP/85+V/4wqSOFUxtYy1zs+ZOeh66m5tJqYms2GkkzKWlw8U0E+W6FdexpWkLd227i5+86ydR30n0tLWB1xs0KPGFmpNot5dHtE9lNpO+Zg19dXVor3fMHAwUuTw179TJDTpKlVmV1DXW4fa5I+7oMp7TyrN58OPLKM+xxbx+Q89gDzvadvA/1f8Tcru+TZtIXb58+PzEZDRw38cWx6wl6UTSVq0i94rL0Y4JUvRNRnIv/dSkjmGtrqL370+hfb64ZD+J8Xn7+jl6//2knLpwOKhgNBg5p/wcNjVumvBzZCouxpiXF3ZdiRePvMiOth1cvfTqmIw/lEDgo9PRmbSgRL+7n22t2/jEKZ+I2T7zrHlkmDPCypRoPryHP+htLD+WTdFJk7+RMZNJUEKM0tbtD0rkZBdNsGX8GTOHghI9o+tKrKtYxx/2/oGXW17mnPLwWl1Fwu6yY3fb49Z5IyBYbYZoOep3jcmSePSNR2m0N/LIfz0S8peaMhrJ+59Ph9x/ujl9TIuqzY2bWVG8ggxLBr1lxwvTGRcsmNxfYkjP44/jeP11Su68k+yPfiTktoMHGjjw4Q/Tfu99lN5zd1THnUhWShafrv50VPsotPkDf+2O9mkVlPD199P27btJWbCAnE9E9su9rrEOhWJN+fECT+aiQgquuor273wH+6ZNZIRYA+3t7sbb1TWmyCVAqin6oETKnDnY//kMPpcrZDZC75NPMrB9O+nvfhd9/36Wrt8/Ru4nI78b7RsYoPWuu0iZP5/yHzwwYXsw38AAHQ89TOYHP4ilPLIL2GDsLjs7WndwadWlk3q9b3AQb0dHXDpvBJhLS3Hs2BG3/Y+nILWAq5ZcxT2v3MOzh5/l3bPHZiJE4njb1PGLgS6uyJ7UftPPXUfvU0/h2LmT1KVLRz1X3+G/270wN7FFLgNmZ87Goz0csR8ZvqMYrQ+cFp9llc8feR6v9oZM6w50RSi84fpRj39kWWw+i+EwZmZSdN11cT2GraqK7t8/hvvw4XG/a0V8dPzoR3iOHqX8hz8YFWCsqajhif1P8Hr766wsXhn09UoprFWnhlUPxOV1cde2u5idOZvLqi6LyfhDybP56w0ls67Ei80v4va5Y7Z0A/zvebgdOL79+DV40uHm934nZsefaSQEKkbptPtbFBbkx/eCPBzGLP/yDW9P96jHVxavJNWUGrcuHIEe9fGsKQGQZol9poS3pwf3ocOj6kk09jbyszd+xvsq38eqklVRHyPNnDYqmHKo95C/ONjQF/1wtfwo72x6urpov/c+bMuWkXXB+gm3T5k7h7zLL6fniScYeOWVqI6dCAWp/uyf6VbssuMnP8HT0kLxhg1jughMpK6xjkUFi8bcKcn91CdJmT+PtjvuxOdwBH398SKXlWOeC3TfiKaVnWXOHPD5cB8+HHQbb28vbd+5F+vi0yj/wQ9IO+ssjj7wAJ6jkf87djz0MJ7mFoq/tSGsfuVF37gBjEba7rwr4mMF88KRF/Boz6RP1I5fbMc3KOFua0N7Erse+eJTLmZBzgLu3nZ3VJ1dIH7vU/qaNWAyjVuXpb6znsrMStIt8S1iF0xgvXUsOnDE2+bGzeRac1mUvyjoNoH3OGPd5JY5TRfW4WUAsoQjUZx73+LYr39N9oUXjll6cVbpWZgN5rC6cNiqqxncty/k71GAX+z6BYd6D3Hj6TfGLeN4pECmRDKDEnWNdWRaMllSuCSm+63MqpwwKLH52V9Ql93Cxz3LOGnhmTE9/kwiQQkxSpfDHwAoLk7+ndtAZVrfCR04LEYLq8tWs7lxc1x6aQeq6Ma9pkQcMiWcuwNFLv2ZElpr7tp2F2ajma+t/FpMjnFiTYnh4mDl/vTtWLXwO/r9+/Ha7RRvuCXsFNL8z38Oc2kprbfdhna7ozp+vBWm+jMloqmDkGiD+/bR+ctfkfWRj5C6bOnELxihfaCd+s76cesWKLOZ4ltuwd3cTMdDDwfdh6vhIDB+IdY0cxoe7cHtm/y/e6CWSqi6EkfvfwBvVxclGzagDAaKbr4J7XTSdu+9ER1r8MABOn/xC7LWryd1+fKwXmMuLqbgi1+kr7Y2oo4lodQ21pKTkjNcvDVSiQpK4PXiaU9sYViTwcRNq26ibaCNh3Y+FNW+jr9PsQ12GzMzSV2xAnvt2PlQ31lPVX7ii1wGBNZth3MXMZncXjfPH3meteVrMajgv2tGdkWYyVJOOgmVkiIdOBJEa03r7bdhzMig4CvXjnk+1ZzKGSVnUNdYN+E5r7W6Gnw+nHuCt9lusjfxyBuPcN7s8zirLDEFF5MdlPD6vGxp2sI55edgMsR2kUBlZiWt/a1BA9cuRz937/4BRXYjX7zkgZgee6aRoIQYpcfVi8UFJXlToNBl1vjLN8CfztbuaB/uwR5LgUyJRCzf6HfFNlPCWe+/sxFoB7rp8CaeP/I8Vy6+cvgiOFppljQcHsdwFeXNTZuZnzOf8gx/CqsxLw+VkhJVUMKxcyfdf/oTuZ+8BGsES0AMNhtF37yRwbf3cWzjbyZ9/ETItGRiMVimTaaE1prW227HkJZG4de+GvHrh6teB+nwkLpyJVkf/hCdjz7K4IHxgwKuhgYwmzGPs3Qh1ZwKEGVb0Mqh4xwc93lHfT1djz1Gzsc/jvVU/zr9lDlzyL3icnqf/Bv927aFdZzh99Jmo/C6yIKFuZd+KqysknC4fW6eO/Ic55Sfg9EwuZoYEy1LiIVYBTonY0nhEtbPW8/G+o3s794/6f24m5sx5uVhsFpjODq/jHPX4dq3H9eIDJ8ORwftA+1J6bwRkJWSRU5KDg09U7ujzY72Hdjd9pDZQid2RZjJlNmM9ZRThs8nRHz1PPEEju07KPjqVzDljF+bpKa8hkZ744SfJWuVP8sl1L/d3dvuxqAMfH3l1yc/6AhlWjIxKROdjs6EHXOk/xz9D92D3TFduhEQWJp22D5+huXDv/sKTZkevjb3M6SmT26Z3juFBCXEKP3eflIH1XCbq2QKZEp4e8cGJdaUrcGgDHFZwtHS34LFYIl7ReI0cxp298S9pyPh2FWPubwcY3Y2A+4B7n7lbk7OOZlPLIxdYZ9AP+t+dz89gz282vbqqAtNpRTmkpJJX0Bor5fWb92KKT+f/KsjL8CUfu65pK9dS8eDD+JubZ3UGBJBKTWt2oL2/v0pBrZto/Daa0cVUQ1XXWMdZellnJR9UtBtCq+7DoPVSuvtt417R8h1sAFLRcW4y0bSzP6q9NEEJYzp6ZgKCsZtC6p9PlpvvQ1jbi4F13xp1HP5n4ssQ6f3//0/BrZupeDL12DKy4tojMpspujmm/1ZJQ8HzyoJx+vtr2N3hb4Ym4i7uRkMBn/rzjgxlyUvKAFw7fJrSTWncsfWOyadnec+ErwdaLQClfr7ao+ndye7yGVAOKnNyba5cTMWgyXk8sb+554b1RVhprNWVeGsr0f7fMkeyozm7enxL1NdvJjsjwbvyBBoUzvREg5zUSHGgvygWS61h2vZ3LSZKxdfSXFa8eQHHiGlFLnW3KRlStQ11mEy+Iuxx1qotqCH97/Or/SLnNmVx3vff1XMjz3TSFBCjNKPA6vLEFFbsnhRRiOG9PRxW0HmWHNYUrCEzY2bY37c5r5mStJLQqZxxkK6OR2Hx4HX543ZPp27dg2vB31458O09rdy06qbYpquFghK9Ln7eO7Ic+MWBzOXlk76AqLrscdw7t5N4Q3XY0yPfC20Uoqim76J9nppu/ueSY0hUQpTC6dFpoTXbqftnnuwLlpE9scujPj1A+4BtjZvZV3FupBdDEz5+RR8+RoGXtqK/emnxzzvOnjQX/dhHLEISgBYKitxHTw45vHuP/0Z586dFH39uuGAaYDBZqPopptw7dvPsV//OuT+vX19tN99D9aqKnIuumhSY0w7/XR/VsnPg2eVhKO2sRazwcxZpZNP4fU0N2MqKgqrJsZkBbqSJCsokWvN5Zpl17C9bTtPNTw1qX24m+MXlLBUVJAyfx72EXUl6jvqUaikFbkMmJ05O6zK9Mmitaa2sZZVpauGs63GY99UizEnJ+xWi9Odtboa38DAuN+FInaOPuBfDlj8rQ0hl6kWpxWzMHfhcMZhKLaqahz1Y4MSDo+Du7fdzbzseVxyauLbBOfZ8pIXlGiqY2XRyrjU1wm0jB4v+Hrn365FA988/3sxP+5MJEEJMcqAYZAUtwmDIbr2Z7FizMzE1zs2KAH+JRx7ju2htT+2d8Nb+lsoSYtvkUtg+Mux3xObJRyeri7cTU3Yqqs40H2AX9f/mg+f9GGWFka29n8igXH3ufrY3LiZPGse1fmj2xuZyyYXlPB0dHD0/gdIXbVqwhagoVgqKsj73Gex/+Mf9D3/wqT3E28FtgLaB6Z+psTRH/wQb2cnxbfcMqnWl1tbtuLyucK6I59z8cVYTz2Vtrvvwdt3/LOhvV5chw5jqRx/PXeayR+UGPBEV5DQMmfOmEwJT1cXR7/3PVJXriTzgx8c93UZ564jfd06jv7oxyEzdDp++EM8HR3+k9BJthGF41klbXfcPqm791pr6hrrOKPkjJAXYxOJZwZAgMFmw5ibm/C2oCN9dP5Hqc6r5r5X7sPuiizDTWuNu6Ulru9T+rpzGdi+fTiIX99Zz9ysuVH928ZCZWYlnc7OiN+zRNnfvZ8jfUdCfjdpt5u+LVtIr6mJ6jM7nQTqUkldifhx7Kqn6/ePkfOJT2BdOHHwsKaihtfbX5/wwt5aXY1r/wF8/aPPLR/Z+QjN/c3ceMaNMWnRG6lkZUoc7DlIQ09DXJZuANhMNkrSSsYEJZ75x495MaeDSzmT2fOWxeXYM40EJcQoDqObFE/wVniJZsjOGremBBxPZ4v1Eo7mvua415OAEcsgYlRXIlDkMqWqijtfvhOb2ca1y8cWTYpW4I5092C3vzhYxdjiYObSUrwdHficE/RSP0H7vffhczopvuXmkHfUw5F3xRWYZ8+i7fbb8blcUe0rXgpSC6Z8oUvnnj10/fa3ZF98EbZFk+utXddYR4Y5g2VFE/9iVkYjxRtuwXP0KB0PPjj8uLu5Ge1yjVvkEmJTUwL8QQlvdzeerq7hx9q/+128/f0Tzsuib94IXi9t3x6/Ja1z716O/ea3ZP/3f2NbFLzKfzhM+fkUXHMN/S++hP0f/4j49Q09DTTaG4PW+AhXPDMARoom+yoWjAYjN626iWPOYzz42oMTv2AEb2cnenAwzkGJGvB66XvuecC/fCOZRS4DAuutp2oHjrqmOuB4oebxDLz6Gr7eXv97/A6RMncuymaTuhJxon0+Wm+7DWNe3pjlgMHUVNSg0Wxp2hJyO2t1FWiN883jxS4behr4Rf0vOH/u+SHbisZTsoISw/Ws4hSUAH/wdWRG2EB/D9/Z91PKek184ZLvx+24M40EJcQoAxYvVl/sC3FNljEza9yaEgBzMucwO3P28ElFLDg9TjqdnQnJlAhc3MeqA4dzqH1XXWYz21q38eVlXx7uDR1LgWDK5qbN9Ln7xr2oOV6YriXs/Q688go9TzxB3v/8Dylz50Y9TkNKCsU334Lr0CGOPfpo1PuLhwJbAf3u/pi3ho2V4ToK2dkUfvnLk9qHT/vY3LSZs8vODvvujG3xYrI/9jGObdyIc+9bAMPZC3FfvjGncuh4BwH/BUnPn/9C7mWXkjJ/fujXlpeT/4XPY//nP4cvDgO0z0frt27FmJlJ4bWTey9PlPPxi0k5dSFt3757VFZJOIYvxiqCX4xNRHs8uNva3hFBCYCq/Cr+e8F/89jex9jTuSfs1w0XAy2L3/tkO+00jHl59NXW0j7QzlHH0aTXkwD/72lgyha7rG2spSqvKmQh6L5Nm1BmM+mrY78efapSJhPWU06RtqBxMrwc8PqvY8zICOs1C3MXUphaOOGy5UCh80CWi9aau16+C5vRxldXRF6kOlZyrbl0Ojrj0jUvlLrGOk7OOTmuNxsDtXMCf7cf/fYa2jK83HDq1VhsaXE77kwjQQkxykCKxqpsyR7GMGNm5rg1JcBfO6CmvIZtLdtidlE33A40kZkSMRq7c9cuXPPK+W79g1TlVfHR+cGLJkUjMO6nG54mxZjCqtKxxcEirZav3W5ab7sNU2kJ+Z//XOzGevZqMt7zHjp+8hCupqaY7TdWhtuCTtG6Ej3/9384Xn+dwuuuwzjUDSdSb3S8wTHnsYjvUhRc+2WMGRn+4pFahx2UCNaWK1yBTAxXQwPa4/HPy+JiCq68MqzX515+OZbKSlrvuB3f4ODw4z2P/xXHa69R+LWvYcyOTQVuZTRSsmHDmKyScNQ11rEwd2FUxc48bW3g9SY0KJHoE9oTXb30arJTsrnj5Tvw6fCKALqPHAHi2zZVGY2k16ylb8sWdrXtBEhq542AiowKjMo4JYtddjg6eOPoG6GXbmiNvbaW1DNXYUh7Z11cWKurce7ejfbGru6VAM+xY7R/73uknn46meefH/brAue8LzS/wKB3MOh2poICTEVFwwGlfx76J1tbtnLV0qvIt+VHPf7JyrXl4vQ6cXii6xoViW5nN6+1vxbXLAnwZ0r0u/vpcHSwf+/L/N64g7VdxdS86/K4HnemmZJBCaXUlUqpBqWUUym1Qym1JtljeidwDQ4waIFUY+wLwUyWMSsTb5CaEuC/y+f2uXmx+cWYHK+lzx+USEimhCW2mRKO+l38ucZCp6OTm1fdPOkWfxMJXPx1ODpYVbIKm2lsEOt4UOJIWPs8tvE3DL69j+Ibb8SQGts10EXfuAGMRtruvCum+42FgtQCgCm5hMPT1UX7fd/Ftnw5Wes/POn91DXWYVRGVpdFdpfRlJND4de+imPHDnr++gSDDQ0YsrIwBmmZFqtMCXNZGZjNuA420PW73zP45psU3XBD2BckBouFoptvwn3oMJ0//zkA3u5u2u+7D9vSpWRdsD6q8Z3Itngx2RdeOCqrZCLHnMd4vf31qE/UjrcDTUxQQg8O4u1MTku5gKyULL6y/CvsPLqTx99+PKzXJOp9yli3Dp/dzn/qn8WgDCzIDb+dcryYjWbK0sumZLHL55qeQ6NDfg5cBw7gPnyYjHdI142RbNVVaIcD14EDyR7KjNL+3e/iC2M54HhqKmpweBxsawndftpaXY1z1y763f3cu+1eFuYu5KIFkyusHCt5Vn/mbqczcd/hw8XYo1ymOJHAMrWGngZuf/prmHxw4wUPxPWYM9GUC0oopS4CHgDuApYCLwJPK6VmJXVg7wAtLf4eu+mWqdNH15iVhS9ITQmApYVLybRkxqyuRHO//+SxLL0sJvsLZWQXi2h5jh1jn6eFJ/Mb+djJH4vrWuKR1YuDncyZiorAaAwrU8Ld2krHgw+StvYc0t/1rlgNc5i5uJiCL15JX20t9k2bYr7/aBTapm6mxNHv34/XbvcXt4yivkddYx3Li5aTlRJ5pkXWRz6CbckS2u+9F+fON7BUzg46lljVlFAmE5aKCgZe2c7RBx4gbfVqMt7zXxHtI331ajLe+146H/4prsZG2u+/H29PD8UbbglZYX2yCr5y7aiskomEczEWjkQsSwgwl5eNOmYyfeikD7GscBn3v3o/3c7uCbd3H2nGkJExpmtLrKWddRbKYmHXkVc5KfukcQPGyTA7c/aUzJSoa6yjOK2YBTnBgzeBNqvvlFagIwU6eTmkrkTMDLz6Gj1/+T//csB58yJ+/eklp2Mz2SbswmGrrsJ18CA/euUB2h3tfHPVN+N2oypcuVZ/K/FE1pXY3LSZfFt+3OvrBNqCPrL5XnbkdHO5uYbS2clfPjfdTLmgBPAV4Jda60e01nu01lcDLcAXkjyuGa+pxZ/enmkb/05kMhgys9AuV9CCiSaDiTXla9jStCUmrTWb+5oxKmPI9aWxMlxTwhV9UGJg1xv87D1GskzpfGlZeEWTJstmsqHwXxgGKw6mTCbMRUVhXUC03X0P2uul+Kaboi5uGUzupZdimXcSbXfcic+RuNTBiUzVTAnHf/5D95/+RO6nPoV1wcmT3k+jvZF93ftCFpELRRkMFG+4BW93N87du0mpHH/pBoDZYMZisMSkm41lzhwcr7+Odrkovnly87LoGzegjEaarrmG7j/8kdxPfRLrKadEPbbxnJhVMpG6xjoKUwujbheZ6EyJkcdMJqUU31z1TewuO/e/ev+E2yeqGKghNRXbmWewV7dyau7UOSGuzKrkcO/hsJe7JILT4+SllpdYW7425OfbvqmWlFMXYi6e/DKn6cpSWYlKTR2uVyWioz0eWm+9NaLlgCdKMaZwVulZ1DbWhgxAW6uqOJyv+d3bf+Cj8z/K4oLkt7LNtQ0FJRyJCUq4vW5/MfbyscXYY604rRirMYWtg29S2W3hik/cG9fjzVSmZA9gJKWUBVgO3HfCU88Ak2+kPgXt3PUqt/59cl9K8TJodEMh5GTEvjjiZAXuLDV+7vMo8/hF8hbld/LUwm4++eAaLN7ovngOpznI1UaaPxv/GNiA0QNnwc+fvYe/PTXxiW0oDq+Tt8oVty65ZlJ3pCNhUAbSzGlUZlYOX1SPx1xaSv+W5zj8mf8Nuo32ehh4aSv5V1+FpaIiHsMFQJnNFN9yC4cvvYyDl1yCKXdqzHGNJuUsAxuf+yGbnnkk2cMZ5hsYgE9aSV2yD/Wvz096Px2ODgDWVUz+LqN14UJyLrmEro0bg9aTCEgzp/GPhn/w1rHwljEE41rSgCvfgKWiDMvbd8Pbk9zPl0pwNexFLUshden+qN7LiegsjfMz2fje2IDh4Im/QkfbldXLu9sLaPzfz0Z1zMED+zHm5WGwxr84cuCi/ugPH6T7z3+J+/EmYgU+NKeQv+i/cGDbv1EhElR8pXaMC7OwxvHfP2BwdSs92kfJ069x+JfBv3sTKau4Ded8J5f+YA1G39RoNz5g8uLIdFD15/9w+GfB3yfHa6+RP8kLyOlOGY1YT11I71NP4Tp4MNnDmfa89l4G9+6l7IEHoqpPUlNRw7OHn+WKZ67AYhi/W552u3j7QiOpg3Dhrw9y+NHkfxc4LYNwBnz379fzy8H4d/lzGn30Z/VT9dddCfn7F5+mOZgFNy67DrNl6jQMmE6mVFACyAeMQNsJj7cB7z5xY6XUZ4HPAsyaNb1Wd3h9HpxGd7KHMca8llTWXfiBZA9jWOrpK0ldsQKf0wHO8e9wLx0wsiIvnV6zB1eU5zu5A0ZOb0/Haw++ZCRWLGhqjmTRnOqiT0XZstJk4MP2eayv+lhsBjeBj5/y8QnT4bIuWE/XH/844XuZ+cEPkveZz8RyeONKO/10Cq69FvumZxPy7xuu9x3KZneOI/o5EEMq3YypuIg+7wBEkYCUYkzhwpMvpCIzuoBTwZeuxtvZQca7zg253fp569nRtgO7yx7V8XzZqXg8OfiKshmMYl86PwP3YB7G7Gz6vI6o3stw+OaW4j5yBO0NPZfm9Vg5b39a1J8DU0EB6avPjmof4TJmZpJ1wQUMHtg/ZT6/F9Vn0m7s56jVBaF+92SmYMpPxx3lvAyHzraxqCGDFU0WvM6p8T6d5jGyKDcVh8Ed+n1KJC+sas2g6gh4fcHfp9QVK6KqqTPd5Vx0Mcd+s3HKfOamu9wrLifjv86Lah/nzjqXpxueps/VxyDBC14WpOTx0Z1ppHU58JL8DNEsNKtbMjlqdSfmfMcHp7els6hRhfyMx8r7D+cxeGolZ665OO7HmqlUsitZj6SUKgWOAGu11ltGPH4LcInWOujCvxUrVujt27cnYJRCCCGEEEIIIYQIl1Jqh9Z6xXjPTbWaEh347yUVnfB4EdCa+OEIIYQQQgghhBAiXqZUUEJr7QJ2ACfmNp2HvwuHEEIIIYQQQgghZoipVlMC4HvARqXUNuAF4PNAKfBQUkclhBBCCCGEEEKImJpyQQmt9R+UUnnATUAJsAt4v9b6UHJHJoQQQgghhBBCiFiackEJAK31j4EfJ3scQgghhBBCCCGEiJ8pVVNCCCGEEEIIIYQQ7xwSlBBCCCGEEEIIIURSSFBCCCGEEEIIIYQQSSFBCSGEEEIIIYQQQiSFBCWEEEIIIYQQQgiRFBKUEEIIIYQQQgghRFJIUEIIIYQQQgghhBBJIUEJIYQQQgghhBBCJIUEJYQQQgghhBBCCJEUEpQQQgghhBBCCCFEUkhQQgghhBBCCCGEEEkhQQkhhBBCCCGEEEIkhQQlhBBCCCGEEEIIkRRKa53sMcSEUuoocCjZ45iEfKAj2YMQIk5kfouZTua4mMlkfouZTOa3mOmm2hyfrbUuGO+JGROUmK6UUtu11iuSPQ4h4kHmt5jpZI6LmUzmt5jJZH6LmW46zXFZviGEEEIIIYQQQoikkKCEEEIIIYQQQgghkkKCEsn302QPQIg4kvktZjqZ42Imk/ktZjKZ32KmmzZzXGpKCCGEEEIIIYQQIikkU0IIIYQQQgghhBBJIUEJIYQQQgghhBBCJIUEJZJEKXWlUqpBKeVUSu1QSq1J9piEiJRS6htKqVeUUr1KqaNKqb8ppapP2EYppb6llGpWSjmUUnVKqapkjVmIyRqa71op9eCIx2R+i2lNKVWilPrV0He4Uym1Wym1dsTzMsfFtKWUMiqlbh9xzt2glLpDKWUasY3McTEtKKXOUUo9qZQ6MnQ+8ukTnp9wLiulcpRSG5VSPUN/NiqlshP6FxmHBCWSQCl1EfAAcBewFHgReFopNSupAxMicjXAj4GzgHMBD/BvpVTuiG2+DnwVuBpYCbQD/1JKZSR2qEJMnlJqFfBZYOcJT8n8FtPW0InoC4ACPgAsxD+X20dsJnNcTGfXA18EvgScAlwz9PM3Rmwjc1xMF+nALvzz2DHO8+HM5d8By4D3Dv1ZBmyM45jDIoUuk0Ap9TKwU2v9vyMee2OJa6IAAAbuSURBVBv4s9b6G8FfKcTUppRKB3qA9VrrvymlFNAMPKi1vnNoGxv+L8mvaa0fTt5ohQiPUioLeBX4DLAB2KW1vkrmt5julFJ3AWu11quDPC9zXExrSqm/A51a68tGPPYrIE9rfb7McTFdKaX6gKu01r8c+nnCuayUWgjsBs7WWr8wtM3ZwHPAKVrrvYn/m/hJpkSCKaUswHLgmROeegb/3WYhprMM/N8rXUM/zwGKGTHftdYOYAsy38X08VP8QePaEx6X+S2mu/XAy0qpPyil2pVSryulAgE3kDkupr/ngXVKqVMAlFKn4s/s/H9Dz8scFzNFOHP5TKAPf5Z+wAtAP0me76aJNxExlg8YgbYTHm8D3p344QgRUw8ArwMvDf1cPPTf8eZ7WaIGJcRkKaX+F5gHfHKcp2V+i+luLnAl8H3gbmAJ8MOh5x5E5riY/u7Bf8Nkt1LKi//a506t9Y+Hnpc5LmaKcOZyMXBUj1gqobXWSqn2Ea9PCglKCCFiQin1PeBs/Clh3mSPR4hoKaUW4K/9c7bW2p3s8QgRBwZg+4ilo68ppebjX3P/YPCXCTFtXARcCnwCqMcfeHtAKdWgtf55UkcmhBgmyzcSrwPwAkUnPF4EtCZ+OEJETyn1feDjwLla6wMjngrMaZnvYjo6E392W71SyqOU8gBrgSuH/r9zaDuZ32K6asG/vnikPUCg8LZ8h4vp7l7gPq31Y1rrN7TWG4HvcbzQpcxxMVOEM5dbgYIRS/QCtSgKSfJ8l6BEgmmtXcAO4LwTnjqP0et7hJgWlFIPcDwg8eYJTzfg/5I7b8T2VmANMt/F1PdXYBH+O2uBP9uBx4b+/y1kfovp7QVgwQmPnQwcGvp/+Q4X010q/puBI3k5fg0kc1zMFOHM5Zfwd/A4c8TrzgTSSPJ8l+UbyfE9YKNSahv+E4LPA6XAQ0kdlRARUkr9CPgU/mJpXUqpwHq0Pq1139A6tfuBG5VSb+K/iLsJf5Gd3yVl0EKESWvdDXSPfEwp1Q8c01rvGvpZ5reYzr4PvKiU+ibwB/xtyr8E3AjDa41ljovp7G/ADUqpBvzLN5YCXwF+DTLHxfQy1OVu3tCPBmCWUmoJ/vOSwxPNZa31HqXUP4CHlVKfHdrPw8Dfk9l5A6QlaNIopa7E30u2BH+/2Wu11luSOyohIqOUCvYFcqvW+ltD2yj8bRQ/B+QALwNfDFzUCTGdKKXqGGoJOvSzzG8xrSmlPoC/dsoC4DD+WhI/DBRCkzkupjOlVAZwO3AB/hT1FvzZbrdprZ1D28gcF9OCUqoGOLETGMCvtNafDmcuK6Vy8Bc0/tDQQ0/iby3aTRJJUEIIIYQQQgghhBBJITUlhBBCCCGEEEIIkRQSlBBCCCGEEEIIIURSSFBCCCGEEEIIIYQQSSFBCSGEEEIIIYQQQiSFBCWEEEIIIYQQQgiRFBKUEEIIIYQQQgghRFJIUEIIIYQQCaeU0kqpC8Pc9ltKqV0TbymEEEKI6UaCEkIIIYSImaFgQ6g/vxzatAT4WxKHKoQQQogpwJTsAQghhBBiRikZ8f/nA4+c8JgDQGvdmshBCSGEEGJqkkwJIYQQQsSM1ro18AfoPvExrXUPjF2+oZQqVUr9VinVqZQaUEq9rpRaN94xlFKzlFJvKqV+pZQyKaWylFIblVLtSimnUuqAUurLCfkLCyGEECIqkikhhBBCiKRSSqUBm4F2YD3QDCwOsu1C4BngT8BXtdZaKXUHsAh/ZkYbMAcoSMDQhRBCCBElCUoIIYQQItk+ARQDZ2qtO4Ye23/iRkqpM4CngO9rre8c8dRs4FWt9bahnw/Fc7BCCCGEiB1ZviGEEEKIZFsK7BwRkBhPGfBv4J4TAhIAPwEuUkr9Ryl1n1JqbbwGKoQQQojYkqCEEEIIIaaDDmArcLFSKmfkE1rrp/FnS9wH5ANPKaV+kfghCiGEECJSEpQQQgghRLK9BpymlMoPsc0g8CGgC/iXUip75JNa6w6t9Uat9aeBK4DLlFIp8RqwEEIIIWJDghJCCCGESLbf4S9y+YRSao1Saq5S6kMndt/QWjuADwI9jAhMKKVuU0qtV0rNHyqE+RHggNZ6MMF/DyGEEEJESIISQgghhEgqrXU/sBZoAv4G7AJuBfQ42zrwd9no5XhgYhC4E/gP8AKQgT94IYQQQogpTmk95ve9EEIIIYQQQgghRNxJpoQQQgghhBBCCCGSQoISQgghhBBCCCGESAoJSgghhBBCCCGEECIpJCghhBBCCCGEEEKIpJCghBBCCCGEEEIIIZJCghJCCCGEEEIIIYRICglKCCGEEEIIIYQQIikkKCGEEEIIIYQQQoikkKCEEEIIIYQQQgghkuL/A6LpQ6u0ZGw3AAAAAElFTkSuQmCC\n", - "text/plain": [ - "
" - ] - }, - "metadata": { - "needs_background": "light" - }, - "output_type": "display_data" - } - ], - "source": [ - "font = {'size': 14}\n", - "\n", - "matplotlib.rc('font', **font)\n", - "\n", - "tot_requirement = no_action_result[\"requirement\"].sum(axis=1)\n", - "no_action_shortage = no_action_result[\"shortage\"].sum(axis=1)\n", - "rule_based_shortage = rule_based_result[\"shortage\"].sum(axis=1)\n", - "searched_statistics_shortage = statistics_result[\"shortage\"].sum(axis=1)\n", - "\n", - "plt.figure(figsize=(18,8))\n", - "ticks = np.arange(tot_requirement.shape[0])\n", - "\n", - "plt.plot(ticks, tot_requirement, color=\"black\", label=f\"Requirements\")\n", - "plt.plot(ticks, no_action_shortage, color=\"tab:blue\", label=f\"No action\")\n", - "plt.plot(ticks, rule_based_shortage, color=\"tab:red\", label=f\"Rule based\")\n", - "plt.plot(ticks, searched_statistics_shortage, color=\"tab:green\", label=f\"Statistics based\")\n", - "\n", - "plt.xlabel('Ticks')\n", - "plt.ylabel('Bike amount')\n", - "plt.legend()" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.7.7" - } - }, - "nbformat": 4, - "nbformat_minor": 4 -} diff --git a/notebooks/articles/simple_bike_repositioning/DecisionLogic.png b/notebooks/articles/simple_bike_repositioning/DecisionLogic.png deleted file mode 100644 index 6e058861dd8edbec87e894e7335f1b85f3567767..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 143047 zcmeFZ2T+q+yEYs}=?EwS1_UW89Ym0#v_u3%I!JFSn*h>5dQqe(0@4+vgkGd6(wibh zPzhZrB1BqfQbHg=_*bxQ-*>;~%=u=%`DXr;8G}!rC#&3TUDtiDm3x}%N;H&EN(cl( zqoRCK8v-E>hd@YH$w|Q{j^g4G;Lko+Z6z3__&w_^xHw>~pr!zUltoc(TOI`06i&*9 zt`NwfX5znnZH{?2AduZ`m5U0xo@VnqWKT@=&2nj3)-{+z4${I|6Iw_}YK27}B))yt zOld&+h{wRfBG8Z|mL!wneMiL;`;RV>F7LHfpWQjxd9~cCJ@V7}Mq?G)(YYS$$ z!GYhdMILYn?EmeWxs{G_-*4B{*U5oUe!W7@+*SN_1%Wg(zvBGuI?(dU%hH)IZ(_rK zdr6Y3QCiY-{(4Q&iSgc5{OyS_I4X;dk>l4BoQD)*VFq`9dqncU|FjXw2SjCkdj-B! z%^Mw*AokvD@Z&=Wgt_kSke?s1$Nzo!e<}@KEBz8Pd)Mi=_Xso*O7j}OtjOZ)VOtzl zts|W}To(7SYlwa4B|G=Z^?-HjmsZpVj?!&fofLU>nd+B1f)X`rOB}UHtqi`938+vD zUO$n?rsyw)^+8C#CLI3t9H}t!mibr~HNJGF`1~TzOb~92-K16CsV(yOxfPwX`xWqA z3=ZWcxv?`K(RgfO`u0FC&ABaYzZj5_^6JC?6F`5hfxO^zM$5Y>E(XW(F=?98qgf~TKhyJ z0>5zMh@Jn~#yRC8Io-i!+_~VKxlW(FsrQe23*X`mo5h60qRyIE?|g0(yxRh!G< zztM30>EVBw-rjIG`@fc;nrL+=_|n5d_X#OKY4ue9Df;yR`82$fLhxFaW2%r5%c zXJ7Sr5BuT++_mmTzT6J37dPdy!pJTY&Zz`=X|IOv=aLgxNz4^R)EiyBaN*5hbGvzN zqkw*~=VP%8Z`^`m(gC~qZv?uW`q_PliaX9=sJ# z_wVdNIbwZNV9ir3;F$J4`MT~;0S_L*_60my^I3WgKau;2MIz!==!U0+dTZMoPx%<( z5of%W_M>QF@)h`@emw+Kq5#~<{jsoz#YK=T8e)N&y$ z?73`w4kkRx_cSu;xMV*?#4bDc&>mX}hQ9YyKAhA#i+j-Hf<~h)wese*Czo@n)XIe_KjNyc3^c`{uA#-7- z!QRv6DD+g~USE{hxEMm_;7aiM27lo!&nRX&0Y2TRX9jc=QBi|PN2#1uoJyICUGiu- zrSAkfHleMA|My<<7`{31mIvce<4ed=WqiE>EQ-e!y@EA!L0chwdzxUU7PQc7&DoxG zlVm4aV2RfhlzZ%X+T8rB^``Ol4;_Y3=S~SmHM0JjX6YD@8eS?ZZ4Zbutn;|P>-o&r zGKfVOj*cZ|eD>kl1bR%#o<&5*UhNCk^=fD=cnWKL~=y=P^^UGaB^kLUd6@EM$) zJdBK+OY`^S?!T9r5**{gWF_l|w4SCBxKU=+a{Rt!bfcPIbQ|hd@N2T*`u)3aQ0S4h z5*9{E&LDtZwxc$(ZU?XBMl=TTWX)}Iqpj9R=mZDZnJNwFV%%6d&ZO+7 z=3Sc<`s+VUQ~5BX%3z_=F1|5f@p;;Ucu|RQ1$Bf5)IfRHPp0KgOm3?0%BymuD1M)E znbpxf@l^xy1+yJtW>4$Onp~daQ!&Vr=ND5G{{=UGt-gv0(P9ECK8H8u_~R(lmfU*Z z>$`p@?a-#aROR{2^0K<>;xa&x30vz^%|PW9!~7xG207$0g;Z%;;hMTNeR=2 z(ssF1D{4A4za;*1xuFg3`)c-9)8Ml(m+X7})!JFggQNs>mv8vNql9wgvI+@R?pUod zM$46w&%`v3DO#>UGyO}B)-hPG7LKQ3L-a<^ z3gSt!`W%HC8uul}Aq{bSb1MJtFP|0hqc$Ks2IzB2Y1r68A$*LDpb!}jZ- z8t&D#%75)hGemI{I=Y(Q_I%R#QbT&vESYf2Cvh~6t=om_Aho*wp@VA3WlFb^75qvu zi~aMdh{1UO!LOPr8~CVlenxMbI{xd3x?xg71WSziG}~-Kmik(lOuR7LmwYxclPw8z zQ*Obi*6P{vI+U39zJ#%46~2dj6*N1%zee&SOvM45m!S0P^Zx}_|L?-g7{0$D|3C~~ zhl4M`zy~0D)9ky;?DXGXWZ7w3Bh$|WSs{@4({ZC4r-A{aC{UldMD_gvXAvSg>gWF7 zd;cq!|6kaIs%u9a??87q{iyYeiVD))S=#BYH1;B$`H||gk~Ey06Pj^E7g6D32Bk zR=P5gH5(XGghSGk?mlA~1H5{ld_Tn~0&d z#-xTu)|-}E+t2T_X)z_`J|du}w3Dcs#6HA0YYR73a=;=uPj7rifG+pM}fJ^DYZY7Z?No!_D zky5E|OS#^5`*SnW61w}&(f_PuZ_Ln(9^p;^bPW zEa@;Db+2goE*^RjNd9SH)GP}(RKb(Son}qiB;&~ zoE=?8(yTWN60q%*6wF`DC6ev)v#-Aw&ru7Gv~Z-0X=p2_3!pe)SCf`uZbLl z9+`f!Z=7c|FAQWi6LBu%(@ zYPoZ|qd)P)#kqT#Ry7N>2W7q6v=N5t@m#u&MrPa34YzTql_!MSl)F=UpR*SF8<(Fk zQdZ!#m5BOQ*;9c~$s6lGOwrbYRxP&6vn1M6lj|r?ezZBw)~5HuYj`etanO@=k1D|&r!C_G6Gu1Ti`uLc;ip%c!t&?f8MoPYq31XWV zqQ&w+X~Qo)V75#&_peq6T_Te~daQ6$!0yL$q!DW1tDARWNmalG8zppj z_Y#$6Vr+~NQ1K3->Mt?MFfPc9h5>59*x{5pS6ZCg6+r_`GQ6*to~KTfnpMWe)QQ0F6ct(UD1@7aR*8nZML zBRzDVe87>PAnoUzX6+2V;9&H~OIsUalnx9DuXEK7^C`<6Pyw?lk-MjI2O7+Js1f&Y zx0pi>-G!d-D$Rq+g!|&Xsi(fjmgUmCva&LLgD>1-`vYD6Z5Qx*cF&pA3$n6=N7D7D zd1ND=%e~4V7z7-SJs|G?2$_0OsBT4W_>0eJ%@|-MWIE=r4mZwI4osl^@aPmpKN-^1 z$61lY)gxoMRN2IPHv461y9(j}Wb#$lP1x^{(wS$jH!;Nfps+0RMC;N?F3`djmjEV; z=*8C?(mB~~5WE4N=*x6xld&rErF**UhaaQ9J@%fa(1-wlH@ zoPjhLmQj^xdg~aH9+JiR2TlbY>InEct3MV!d0Mxf>E>GY%ZBfB12oX6ggs{FY{VE# zdw`hf2={Qd-aek1m)8Bol-0*O|uVeBG^QO8^#!#vn@(eR*0h zop{pjUV>=yEsvaX_`^C{UNPn4b~Y5EQKhKz<*yv(8pmj&#(G0@V*Tlkcwb3!40sBZ z*PP+G=?hbcI7mjl=dCfId_Lk3nO#@wHV;1VWFwD-SHNO3U+TGKEf9O0p#CJTV zWcPz8(#PZ7X*7D9jK#fQVk&UA3zKX`Qc;YA*D3bi@ikH=dQ*fFD- z3vTXHj?Xhc;L#j$njf+zTjY{$cA$&of*O2xDgBC9JE44zz5(aAzAYnfG!3%YU`Pv| z;!pOv=H_WBRFL$gnbd}de~(WIcc5ddSbSI^oqR77Y+vAh%77ZwvnR;F7J6D*{Pmrd zwZzOcc&{Y7K20|P<<{Uw1KXaZiY}SfQtsF1FiEi;%EA`-=^@5KfIWSSwc-qn6!oDz z40KXn^Q@4#AkACozX$f4F|A&sj)hE-Y>LxzQ$@lBTIQW!eM0i+%_H!X6k-u`v$5PF z_d?GnDXmUZm37n;V0iZeFo=yB?0wi0#04+oLn%Xw(%(|ETAsVYbD{^o8Y~ptC;GyF zm;RRY)pE97$yvMlDgHu~h@i$+k)}odf-`r~Ua+=kC{?!9*f4)NambZM|e%j);x`@;MAiXZVL9W``(>i;LfSb29Ow8HjKnkr0R< z*B{YlPdLsZk8-S|VVqK6LNgwaz=p&Bem+-L-)9UB8NoOq%8Gw1(|a13unU!nUU4F8Z(Fo@iXpL?=YPrV>@ zo&l0B`S11m(&a8u>`bpRqPdj*i#EYBw+u6Io6S!AFCJ7jx%JVCX@4djkDu3s1it!{ zN_}^T&ZCT`&MiloUU7ahJ^)eqlf3Qeft5OeUjQoMFcMrEh67|w`yZ@lk}}-Z)>i1Q zVqGOUB<3G1LO|c3@52XA4uKf_aY#DzKS?B5abl%YxDoXn(Vd}4w8PDR8roR_ZFvx# zRZD=4UHM^S-x7Gb4~VnZAYS~kFbE`&TFeAwTi4|9IdyDhd)eJdjvrPIfrQBH-PR3p zMz<<1O`sb0zMuNT3PE{s9pwZnvZ&pe&SPLr{usvT`zi z@3eA~ShNdV`=J6*)GiGP;RhcP+IC$+VufvpJ!JhfBrpWQ??c!~Jp05~g#>cqZH8j_ z&6O{2&e`_m&C@{wMSchl0ui~190z*<(swERq?+F(ia{W+X7Dq`@mMUDD7g9`f&=|o zQqbswuHreyzPFn$!FB0DSaCG|rnZhw>~Ts6D*3m0CVZbK1OEFc*K#T<&S?*7K>}gF zPbO$@G98-poyXK0vVdqIn$X|IH=q7(d@3uG)ThmRh?C`s_^q3(d);Uri}C+Hez4@L z|I#EiKcWg0{bmC2zrH_n{`uZ-Ko1v+0LLNr_Hsq8C%?D!efAs4R1lPeP~-Lr z)!;kDa7HT-TugqhH^0o1lrXg)vPM(y(QhhoS|jxdP?}TpKepEz!QwGqBjxg?ofrke zf7e>*H?1{|nSl7MrK95to&(|hKR?$Ee0RLY>wEn9`}n-Io*n@*!PA0uDzG8*VY})1 zqip*i8^1aQcOmkMCqUYE<~$%eo56Oc&7DXBf5t`PePBHF#4Q{6_kH;~_5gyO^IEx4 zSW?n;=paN=<+F`7MQm{|1i#T-YGgJnBy*EeB0KO#2YKOaaSGACZ$* zxW&=+{u2Lw|INgJO_+Q-?LgYjy!ApblUcj^!Db2%R%K#)(lejbadFE&vZJ&)zh*bNm)fr8hr5djiRod&P;nZCX$!Klc#- z(Yyr4FWH61&UH4(^_6B=*{0DR7->iZorr97*N-f1zh5%h*jilN9I4M)r0D&;$QwS& zUOn4WEKK2G+$)M&?7ds7FU%w~U-;U89}iPFqu{enp;`gyTS}3vkAbnt6YGj_>cM1X z_5)jm*3ny~Lh!N0KC){#h4K%T7ekK}B)Tlo3aWN;6SMQoW_h=JKc-drL}&_g?xe1V zE5M}$pa`5zhRN_$-WOsI-i-z4nS|=sdVu9YfayS(ug!$$vrl1P*2}%35T~qSnS(_o zxz;1T$m0V|)b_UXq?_nYI2PA>Xmw=}?v&;|i?bVMy{jm~+?_eiP#(o1@g6Kh&*1wY z#D^`y#sioaqMTk~uU_s!q~ng+812mI%sU^&#B;-W?rJknc?->&3(Bq4`34&%U7G9l zU3BKf>(5v}UNYQT?B;Ntdi&in&UW)B)HSH7!99v-7 zl3;jf+XV<&epl;WWA&n0vE1S@*2UDEzS8#5l3BB^(bE`@{H+u~gc2qfY27vQ$J~Xd zrlNYyaW*wkZZBhWADbA)93H!6ALZ}S`8APU!c3{gL+C0SPKxUW!aU(R3ka+DU%{E# zX-&!Ci|m(R*_l>n%LrL&JmM3q6D4`9O;Cf~AFE)xT1(pk&7plG>mZYK37yT~=(HNb zmYqvm3_Tt07v?^mxP_YQm~^Zhn1eE!NIf|ed=)yKFxkbmgV#1i)FqqsR!?ryF!I$J zhqT%pD>yG`-aMgTW@KVW44Z50@f*o=i)ZoA6~@5iWV7iUq<8(55wbKMFB59JTyWB& zYc1>&X=}N-rFvgHT2`&f#qP0Dh+AiWqYB0h*OX*7=Zb4LDo&VPj1o7Q8+7Z5SHj~^ z4x`0v=v7HEAOh?Z=Ax~s?Bs__#5lZH*wOyFC{S);4z3C5z~Tip81V@?&AYA0DA^OX zj|epMNlNUTSfpO7&;DvU-i8zh8PPR`zIVCK&Lfh+vMo+D`&wjY7>C)RCC^^Z4Fqc$ zX|M{aqE{AZ^~siL7x)Ud9;P~e(gqnkFhf)T(Y}lu7LSf3I*y3!m__=Kdz5I;`-_k5 zWY#jdSgkCzZ9ja~HaczNvz>ks8PgacJ2c=jmTObhYFD%2m&Nbn>jOvBire!rglEp) zkS#yHEZ-Td)l(uI?bs;U@q)NFFa87(bYy=XHAK8@0M|RWcgLTaAiPA!Le1DK{bzON z;Z_JXoWg}ZD&;CialAr|ag^*@2%C5=?bBj~6SANDW+DR&4k2f6Y?C-W5!>P#lfEww zLu|n!m|ls;%>;Q@J59#zO=b-@X0j{$K7Z@b>|)xDn7|CWMgO7?J6a7}N*=;lK9!O+ zjxLjH@Iw+BGYz6+J9Zo+Yku__uDIl7$fcQ2y66c7ee%jw08=;HD!6E_!!M>ev))UtO< zjEoOtJ_~h2hmYs9? zo1G-5-dS^Y4`BxOMe>NjB~kX^*2FpBGFS#tbJ}IeVzMY*uRc4za`O?f7R|>y_HXsO z@G@dxYWnS2!(eM9YHV+rW7t+@F)9@X5h&oZ>=&l3ucJ&RJTXhoPO?I7-4|q4VBEkmmbmp?Qe|lnR{t`%mZ`(T{1~)k%u048LU`IQ$!Rs7s;G@N!p$5?``~_& z4(>VOIw`t!?&CtEFLr5)gt@(MaL_tnf8h8Yu)F-S(HD7e3#<=oWIObXRkRot@60ON zcD%&`T2{9;^t_llzRD`td;UxFx8V}-gW?aeHbXKoqkiq{oyIoC4uJ52DDnPypZn&Zcv+aDK1&vofyRM)_n zp${J(fkm+S@lRO5>{LbTi%&Na8WFW>plx`SCPWz5WG@xJ(8rVBN-mKWMdgNnCb-ya zSLQV$eY9XEsLWp-PhR7DS5~rC`VG@323yT`iT5jS@-&6G0pgRd%p*$s#%8~V z?onBtg9h|Uy*6gMER_Xqz7ZtQQ*d_F=!IozEQ{d8B;WeJQS1SoW9YfQCFoM(aWKKA z@kIr*Pg>S0fOH8W;<@MVL}VVg@3t(%96a5P{V8_*ElsTKA{_Z3Ah0u( zJ>+%blB}|{9vcoh+UDn(nA8=ow_B5juA)uuG0$aT+`W`Pd=B(|P}4!mZ!_en?1kD* zU5Ca)Yd*fire3C-*plXWh2}L5(@BrsTIqXBT)2IsYFsJOTtp%Zo<{enlPGHl~{n5`eNWnw$Xi@lEnY#*l>;gNJ zbJl9pQ~l+O&AGDjww~mEk0MHRdgtJO)olxz+z-eUz1yINNnqMNXLZKp@yK@ZAs9B_ zl2DHSsARP+cfqbMVM+Ew=$+&5&={lL)<(kucU%8}>LhjS1e6TS9s;Y{0t+jLy9RPA zif)m>&&JHHuxcw?$yWKqckZx*cf1x(et9d;v&%NLLKO!i)aOwNj?^2m#%GS;Y&aW6 z`C=1H5w(6Mj?R^Wc>TuE4z6t$AwDej7Rx?-5J>4! zwSkNt*K_zj3(23Q0tj8RA308r;n0nd&?(;tO3f#i>*R=3wrM<65v0KsmUvy#^pqYj zNB`aZ124=Zhb<0Ak{tCb)M7u}_ZgB6)(92w#Xtga=lbPrkg<7Xov12YF9vczA!mOv zjUZ;6CmmaG^dX$3Dv@xmNq`dRL(<*V@K;{q#)#6hh| zEJn8>J!S#%L=)lp*UfCe%!%#u|7u@k-kH6eoA@+4k$);jebURC%VUG*dNA*6`g-bL zGah@?GD|5c6}{{V`TWyMK;D4%Ti)PyV>M_FGI{bJz6*iGCH@SIKk_7+$B5Jx&^oi9 z%mrfc;d_4Mv!f8WCGYi%z6Dx#0p$o2_x$gW?UC%??}ASuf%o?~$ln?yexA-r-1;9E z3dr+EB6l3Aa=vP8oJ`c$Zv>h6WF9$?G-T2euuG=m+}P6^?++ae99RB>T;2uQBdYx% znVhE-O%=X&7D8?QYX-z-z>AbVA|j$2J-sFjp?3V2yMJev-@XCfgC71aqzBArChq*W z_jkTo#ngQH`<=j-uwTQY_Ss`W#9J03KND@>FasNrKLvJZ0yeKeysP{hVgCLlK9T^} zeUJ_%KR-W)294AU9~lLW;)pW(Jtgxk!}U#bnz9TG)pnTg9#i!*b4~0HS`0e*-Xc+1 zVZTr8x5S2~EGPhasMXIu>jbpS;-4E)x_%uUY=;53FX=yy$&BG}^;`kbPyB!RNmHpc zrZ1mJ&;toU!1vPYO#C+|nTiWvym$dpVIog{nKV&2-|R@!43u4+$7^c7uZrqF+LP~p z^vwT;ZPbf}H$p{{FWUY(m>B)qruLF`V$6fnmHxa{Pq^6-qj9E}ohwp(&sII)ZqURn z%xlLWXW^YH86A%q>;n+TRCMcc2NwoWE-`s%US4T+c&DAF~;9 zt5*?D$5a+1eWYu%CIn>GJi(Q&-&!zAG^RjWhdhwg83`dMt` zdr!!bzWDln@?8{JQ&x|YeDiI`lPsbX;pE{HhZ*S^HxHGUDo3ft1Z>g<(3xC1%>kw6+Rj=G1iT%&p2XB!%BIC zEa@$ym*lC!S#9>FOYEg(GBwIacC5iH3Qr>A?EvEVt(K$uUk_akw7mSoKXjjHatU!> z3;4&8?lp8$U?Npm2REW!oqe0zukZ| zY5uH?@Tup~Gtrkd0*Jr&Vz&KhLqDeX{SE~F)64QCk(W#^z9uk!zh#m4)3c7A?h6YK z`!Gga6o@Zs@gI>Tec~?EnVx>jc<=K-WIKp+3q(}c*98%&`-fR{yel`&bbSi&c*Jv6 zp=xezIg+NYzx4XI!ShJy3`D&xg9;dx^I-JET)rrKGw!2?|3m*SXX!w|`L$HZ|9!fX z#09WG?N7ZB7Xz>yIDisrCsLB2&Z2tVQujX&t z7)PMsV35|!1pY$*Lux_6*g27VB=v2F+AjKkGu4iF@)Y5H&)b6)xs+BUF9eQX{9Wu0 zC*5ggcW+In4X5;$ywQ?5Lztb4GJACC&=;rAgpX+9I32mroR_cgNB&9`2NfpjCPAk(82H9#Bg+0}*mX4>f^ z)N#b^8NcdQO-_GE@f5}FwFXX!6l%2~IQ&({IM<$q$X64$5!(KQXhgR#RxuUPeA@Of zSv{l`je1y$*dPK0qBRkNaGPz$!}f!hD5D@2ou^4RK?Ytd`Lo#hZHmJ8VOV zVE5d_YKZRf0wHu8OxeCx&av$Jt0nR2NMo8_myuh_{i;YmdWnTIS`^>386xVf4={(V zogERS=KUdpoz|}?&smAQ!hJGg5qAzssFl$*(Q?DkN8L7PupIC?RB09(oQINc+F=uA z5ZWj(yMK_Gl8Q2znvxc_y4gCdwQ-pC-UQ=NTJTz;v8}jg zw1RG4Zj`yTL$cTvjP>Bzld@@z#|hrqtu{UrNEPj!n!y!VFGKIUrNJ`)t0{TDgk^5M zCFEq@;*J$Ux-td-k$oB?)X=zH)6+$e%=a4c4DcFaHwrD(kT98|^6qfHqN}}UEP=?k zpo;6vLG(RaqRYAe;c|9WSA(kT!aNF)!;U<{eTA^z+F4k{N`wCidiLB_e~Drg9>z#T zy1lGDE?+lpPD3D^3x)B{N^DQ_=VH3lGVLRyaEZ~vag?+BDRx+(X_o@V@_Y-PKitG9 z)_|RMp%e9?+cn+(U)!XP4piB|=2oMsea9blU~;;B%?OFm`(S$FqM_| zKw2jZ-OG9;o?%aP0C4_&odOP->8=f@S$t+<89g>*#)G0T6kz5#_i0d#iSowwCythc zOtv!)L!QE?8mF%yHBT0^Kl?%@arwoFr52*DcZ?lb+8)8raQ~=u!i_S&MfS?z0WPYd zLX)H8+0OF+&nytm6SqL&U*-*&ssu! z>dn2~PLjEmd8w8=etqvn*F9l^`DMBJS^ityHM)(KCPb`SK zz0J+ne^+QHKjar%!3x6`pt)&mN)$6#2d}>K`TAMG&ffmrq1p43HKcj!FVt*M4Qj?E z*6*LFm>x1Uj$)HZ@*c~Q!SyU20hOZ%Kle2O!~#Ne#K0!BqyN#OoS7=4otyB0f!WjB zW3*aX18Ll1e2ml#EK74aRQ=(Ntm4~t!>hAoB+Kpv`uFyqkua0dW{`f({*iHTX_H&8 z)g|~4p+RdC4tzz}7znwf4OOGd4%N?3Ror~P&l-uY_%7bOXnh(*8-r4d+kWFz=B4#M zxW7Wzj>p3zF4}0KOj&q320jcs_^USMC}JCfgtw3|y1G}i7{8FJNVQyfB3L4G6n-4xvYWNC>JKebbo{I1piu? zXnzp9n95GEI|EfR9f!bO)ge-&gA@gWVr{-s zu)>`!-0;-9WSvk&n=YAt>mj?qNa1*{cg{IjyT&YFw_EGOj=7&Cq<1@=?fac$Ldpj@ z$BXyb;FfCK=d?o=C#!(f>Q#QTTH9n*V_>wE_QMI%0+(GEXRC;nTb`7kd)bMI7{q%c zCOe+QV77p(i>1nZ0$LVyw2Hcsni`qcUdop(H#pR2)^AKMg$H}@=E?5h-5L47&gO`y zP_=>*Z67GuwY^Zv^J(?+zKon$5Wo{lp0=!4Pr0JQQ%Hg%hY6CW@Wz42Fz6@J%LZ{3 zTIC?fG+3qNyzCU`&at_9N&piv*BzT`Gt!2a{s{_4x6GX>FF*T*n$(;nq67G z_)Z@8;gM}ou}_9i*hUUUfylPh%4R!;cQzO zt*2<_3T-Wt&RR$MEr**wfZjJjyPNmb&>oj&5Jx2#9m?-w@^-ur4OcadoYHHqgzi^0 z44LXiC+U_f;x@!u`YzZ$GOcp5TsuV0RKBJssI=ON+PG%x-ygId(Qzc{)IFm}D5PJ~ z__Ck!yIgNQ!}2JwjOr-78YnWhpX`Xw$MnU+-S?hTAG&U~TJ22}p^<@6SaR%kxhxo( zfOIvFTIa*oIoG%fixodID#7%clE8YowdVAtfCpQ3d+FQO7pT7wtRbDLNyt&xixeC5 ze3vU)h1-rR=!l*&(39G>v*WcrHdNA?C>+I>e6_>9Nmwb#E>Jfm=VN1%##Tk2U0t8( zAy&zUc%!V;V7tNM2ux+)B;u3UAmlViDuobE9TnX>KJne;oQNjpA4c!!DXgFA!_gnx zO8a1P^KyMd)xf%DvaGmmhHx6;tB?$2VK<+`fkP8+Ai^Ybom?e5(blj{9$D z-DH8|Z!3#8a|WCTq7kWy>-KgI`fzHyAMgGt8-%>% z^)qh8tTrXfH5C|7wV;mZ3XU-uf*=S8ZP#ad%+SFYu?6(lh`6j;N}j!ty_6@lsdvs( z_Q%&>RBx|zF2tw`-QyP7&&>Ao6aX(Q?euO|@>_lSbhiB!2NDl#C-A-JXl-%pDbK1?T_0LuC!v-_UH#gTR z3;Q0nz5~a*Q~V9lb1Th>+%jP+MEm3yxF02qeUz64fzyjK&;l~WN|Q869^P}#ppexT zb*1;AL2)UMJxNgo5^#=n`yqR+lso|&Onhmc1bD2EMLr4%$GK z4v;D?l2-3|xNgKf0yIw$96TT1V@9Hz3wH7Kq*s8=j+nEkR~ex0WkL@2fb69R;gA0e z{SN#yS|lDO;(fJkx53X4ur8ayDaFgF>zmtYW47DwwIm{EL4Ci^*Y<6qQNn%6(9l%$kT1N^*B;IRza5%v-rBZl>o1Yg7@;2EGlB8cHq1_l6~3}Eh)hGHckt8; zT;`1qFmxvo_y;1%EwzM_62ZxkAyTj;kcWH24dTKFGz{WYhwP$H2w-R|yA|C`=osUp zl9Hgw7rb27O4dPE*YD3YqBP~AxYIP%{U^cD4e>@8ed9hk*?_v2GFh*3H0iXzV_*F| z1#>GbycEWFBCN|iulPxUZ~&WqC3{7%?oKf52{*9)!*J|OR8-U|7N^7`&{M&9p3xWW zelmPv6&#BZ1yoRFUkK-2#cBjtz`_wM)@>ULHjE@jd` zH|Xyb;g^Z0+k;P?@#G zo3$R>`1V=|=Ho)8@7CaKB*`(_%5p%3qHw70hQ5kclFAL2`^S$IptH7&(Bh_L3zvPa z=wANs>j9+l)qkEIA|}gTj~BBW1;g0JuGNQ%2sCNlE#+Z)r)uCvZ%+awa#yjQL!d)( z?zBM1Nf5~B7_lKh*V017O!?*0KTm=YbAuC3gG=H^p-Xc(Hco-dLjPwi(S}pj2dG53 ztmf)DF(e)Gp%P&rj$`#OKpDgG260rDT8=<@-FS*{7Zo4D!+Ybb35%oxnX|y2dp}P2 z?B)C|`wpY9XEO7@5$y~;`<@MxF2V!(9wbU-xHY7m~m5txYg0YqDFJj^(m@h!s8 zHD`drSl`|uBnUJ9Zy|Os4I9F_34@9cAgb)jyBjHk`j{CUbvc1}Cr^O4tps2JV7wj9 zi}Z}xOM5X5%;%e*c$0b?#H~ZY4`QYLEou={xJ{be=E|$BERG-iaHAebq|3t4i(BXj zi6QqvQ|pqtmCNcu)IUsW&*vs3aUfu1uBH*gM+ulvo_%C#9-jJj=?rn3G9BYGk8~hX z<8S6l%tE@;R&N z69V_)ds_Ge^r0Q;8U-9&dY2+xC}RhJ-|(R+^6-K#d1~S@F_613Do1D$a~FDli%-N< zI&qos+>D5}y=nUVuZVXk=OrkE`^d8|oxqAp`))Xiy<6>)bA0~SyZio(j`i3CznwYr zDCh&#lV*|Hnk127T-}wt13)to`?qUdMKfsro3RVoTkOLRGr1>xTI54g~Mo40n%V=kEA6a z@mG9kzyTYno<;NHOT;EKnj{w*w4^$m?e&ynN#nT|#N`JSQ+#YtWo7<5-sf-48vstv z+a_-Ii<;!Br(7ly>PpA4R7@RjTXo{OrovOSj`&9Av?jIF002`Y|PXMUzTvwK@ZZpV#*nL=Fq z-4m(ahOfcG7iFmW0;dvUb&HJaW7gCh8(IDIA_B;0*&aX}!4I?KdEw?i9`U5|y&=-$T((VA5gG-NW$@5L2is2SS!o(xF48}qjM2*? z-4JsxaBL`K1voI@YUash9mbTiKuB{}xg3*RHCAUw0)WM4O7pr)Esp3kyCZxL>!lS$ zX-CoLV{-*V0q0T2LZRkY`jJy9asmFBIWUY>VbHBnWwBDtrO?*X;vhH{fQrgK?Y2Dc zH;W_H&%b!P4{nIHh3Tf)8rlF-X;(N<1kxLJ(Tm5f78xDwSIO^G@d4})RfF^e`zSCl z12M0aFT^rV!HnY#j=ckld*SPwY}aTe`U~}y3iMGiCsi}j`ccnaCOh)RRl2|d+bq)c z(OM5Bnee{SY{v!Jt@-C#@>!0ah0CUB!n)(`IgEXlnTnCRd4J|SVDh(+tl3q*n=3PC zcx}rM0g+aib3}M!?*gD56{#&%{uHpBf0Okb? z$~uSgMe_u*2MRycW}~kvSSLyM15@i^wblk0n+2Mv+dnta5*~z($@5nBS+tQ&*%EHp zxTbqLTPT-xI<*3h)xwf5vH__>lqlafVf+B!5{`Z)PuU@#R^j_mut-afJvZrpaktX`Tn z%f}XMuJK&pVFwwJS~|vAYp}`pU+W?8gEG=eQIgt-EXN~r#&sZ-JaWB^fpOG(u1M=d zmgBta_Sa^u>-7ySPaP96#;;-H(a_O`mH3j2V|@c$H6t!Dm=2`l!L7-eLj0vNZ@-x` z@LMV>eRcTtTDFBFfJXEi5;;pbwKxQGRtH#VLp)#(x-y6{9x?EPIRzc*_7oAS;Lo!- zqY~+J9`hgQ4<^}QbSE_(A=m<7#~A}9Rx=}ehKg~HpE7W*HwTD*s<*ntF2tKNp@!8y z=nh5BCc3T}FgwS;CoaZ2cB0@L^qdA-bZ0$NXaQe9ctn#!O!dJUD0bal=?Fa%y zFeL`5Yg=Dq=bLo>Vu3o=gMWe0UzRWh{p`(@%d2E+YU*a5PILxhw&V8Q6{J#EHK;;n zkpvpQ@gUyDtmL_3z>b1n>nT;cr+g@`cXvwq3zqtf0QVZ?MI z*$}|c=Wf~(Zm+!05HzmHl2#w7PFzn>v?uAG^7|SWrXPU^q?ZHT+_U$+gmr~?gus;F$F9l8jHMHQ)c}NwSIRDHD z=7a9$a|y>g}(He$J)W`M{SK4-TYiZkTWgqN4} z+@%;I6-N{}i_v^bFd5|L4}yIUGu`)WoIo~;<>U)Nn;;1BX(hMCY?>ns40H90#FNg` zshG1sX>Ex>p_0A$7v?>Eg;V|L_dSa&DZ7(Z6&OhsIEW!mgMsB+9-}qxT2VFH#2`-t z(y=r)<&M1rg+o(43Gl(Ies8$dQb1DF%_M8*8p)T8Ho=-w#OpBne~R9U9rQgv5z9ZX zl_%Jx)q7KSAtI7Lk5T`MQi^?GBfA^@{{%iR%7aw6MLLl^Oq~>{39GohheC)0$-o|A zoQwgKYL8vG(I00b?lZ|g(q%ufcS4v?o#)Z9Fi=vXcKdIFFR+Qh?Z*O7Ou$ovGr0dP z)b@Y-i`PU#6`#*JzNd$z_xylyV2j1G=p0bNz#jr=>c8e*)KjI14t%uKAqGG=Z-A3E zFF@5*2~?|5Cx75~2!uPMxcGf* zoNB!+%I_`VGcBRX|BI6UYyX|WfKkFs6rtUJt1{p`-pOgH9`{%7;(7enjpWA7CfQ%|Aw6?i{Yk0ZcC%bLlbI(6-fy-i zd%d%9yYtrPrrR?fe&joajd*_J7q)t9KD^sojk`WK!ZZT#z2|ItvgRF`x`{WDJ+-mg+TmmjgJ&?I5R|tInxZJWk z|E<3YixEB1FG2RZI%oESL4&WCA$Y*@PO*{=%Y8l4{(Bhg%`U^a#o3DD=cWT61Fka)?(U_bW%qitN4;-iGzHb2VU#96u__u0+pRw}JetDuFJa z7Tvg}+IZvaUvJwFNp-ypB3l6GeqP|}rey%-$)8{!Y20WL9wJKpMgU2m3zGDxxKKXF zDhdhLLO<}o6`}5@HApH`qdh&hzA&5_@($$|>^7KR zL!e{5ad7u^D*=-#R3s=z6{trZKhG%0B_^a)#Fgm{QLLy`s`#t*ct>B`o44Y%n@#?9 zZ|BiR<_3nAk1l=CGOZl*?S-19S%?1B^(^RY&hTxUk`K2N&7bWE%Q|kQ2DGlr?8Ghj zVJL>NAGj)3@3r0BrRuV3t+yaE^Wr_bP;EI-)>Bl1lClms@t`?g-203QOL+rAw;*0>tYRwIT zS{(R~-gGhH7N&{@poxy?^^sm*TW+&@Z+9YwYXd9-5RB*?*{t`&<%5tS^XZiJZxC*X z(Uio^l>dc#cJV$UXf&#f^g4c8s<~f>=WOx9mE;%J#m5%x7Y3rWb}LdHUc{=MUikWk zs{;4If7s2UP;jLD^;D5Lf~rE`nG_4thX+Ga>m5VK)+Y*V*0($K=O25Rm!Hf0*Dyr0`Ai?mimBwn|-TLZ^*+M+6o(4vv`pDH5XNaAZA3=ZYY z6<4SYnz{Q|^@$}}KbfkYskn!}IjkmOF#6^E&B07s4x713K){8nZsL2YDa*=tbvd`PO5qFvve4C z5~w+S^2z(g$5eGnh9bqV9PXcT>PZnh(n}`fgZg@*-%L$aCzmgD3ntHNJ6gjwE7{$Z zvT!__@@BGe(!#}Fis4aB$3l<6A(mvukA;QSA9WlXEocS2M;y`uN=8MdZ`Jxw{eNtI zdpy(oAHPngoGx-YmCChFom7-^jTx1sj@+W$Nn)5$VJpKH&av8bb&23w(O;J|4 zU$)uCmKMg0#I~66d#`iO_xpYPe!o9@q{qWU5Bt16@AvEVyuFgdWLGLn;_N6x!HL_K56sxllI#Q{^)q4n8(r+I?1FBsAib0+JUK zkPDCb4IdVJL&wh4FbCA2wY^Iq^<%?d__W27P=3opB2(vctOvLO}??eMSI+$@qn5*ulIeVeEm^eO^x zoe$Bf%hF#Yv8(YW4uwW-YGsK9xEi59uC8(ul{tl;s+FZ!h9vmLGVOI^TI|VngBkRS z+SDXLWfa>OQimquwT@iS03$hKklbcJHOPd&9EEeLIre0-`F^XGIalE)0|zE2VTw3} z0$zi#Z8`g>`12;CW^Bz>r%g_d95D{BGQ#2T;&{c$P%TwO#a>8a{eybnJKd)F{<}L4 z`vQh?_&(*D{;z-26r73wr70ke9QXbq8lpJUfz8Ygg9llJz$KN$22W0P@p(!RiY)re zE(?|54H2xCcGmYG13HeVnT)BBgh~6vuY0M&1{!W)urhq$W=CS^3}J*Y_4*`xu(|`O z$wo-+HR|J&6430YM#eE+J!AqUE}s^dS`*5+U$VZ>xSgPFF!H?i`$p3BH$zAn{7K%` z7}UZxQXG9ZX{R$mng$KSRpkWO8q9_wO6qP;zY0)P+Ymskntq8=DKtHK>n2P_J{7<2 zG@uO{>PJ3dt;cf_D}nCsKWLchpyP_CX#hnUNC7nk^q`Xcw6|{}JurI98KSe;HD!5j}6sHW&#jSF>y-_@a^! zS`k=Uk{W_E^^&&1Ty$x3t6`;2zyUUaStDduc;^hA4NZwYjI|%+#~zICavGcQM+SC< z!I>|6><{>cB!m<(lhFnN=l?4B8Z?I|@LFfiW0uKCe&Lw#%eO!1aTl7f9ThgyPHPW8 z1c7ee>VNueoCZSz9VgR&-ts%bYK(ah9uiIvNljRZI>l_Owj$OoQz8>pl%wolm2sG} zr;{|K>9bVOF>r{%;gEZcwK##``Bgi^GV)OCOb7En^}gsX!~Y=xc|EV+l)&s@4rvq|$+%Q)G#clp~?Q z@@-NGdYJjPt*Q-kJ4GXDz652=_ZDN+)L_tas}{^CEN;&P{!9KzeX>qzQPCbqfk}p> zIBudikLnLhvH%!Q9M#bGa|)ClNZ7!v5#f=ON%Ms1M+u>01^1l7ciujMo@!{@U?F*+ zftgi;Ghcl$Q&p{iL?F&PJ}JE$v~d9wF{;YN);hUJjS*jI%%0$4eT=OcXixhfsDTgf z5+}QD^#(YK({)%AZV;QAlpOuLFgvDLBU}9E=KFXd7Y0RaU75Z>Mz2Y{2u3&ktd_j& z2q=AkeuySYSd7GP zshrKOgO+w}_ue{`mJ*32&yQvb`*M~dBP-YE3dhYE0evv~z_bjg31l?o6qB)95O;VE6^IdX`smpTtm8@4!_E@gcQym?s_KPP2$9f%$#hQFL zTe9`Z9l@%W-#0rB#Yu%??h+f5NLN&l$$69vRQDb0^xAUQq~ke7cz8u_V21FOw2Ch~ z3fW^eC+l>WRBJJSk z*^knV&f~uFi_}7;0|LZYf!ZWEMNg8G&6G82hdc=bdz$mq!Q^zB4amu*H@~UiQF!)7 zs7Kgxbmng;<+w%` zy~-DchnNgxtW>KAl&NRde)m(e98;0{bPTi;xWCfg%qi6&c)69xZ*!$SVACcy28n`Xv*P$Z(u?;hK}T3_w0`kVjzeEQ#P8l z@ezUSag8W?)WlQKT+J~@Q7&R1$}N)Ly%RlAS{yAh6FF({!@Dw!;CbA0t7UfkTySHJ zq{?#B2;Fgw(`~7&;WVH6p(oST8YcGDLqtxfVESc;vH5I0wu&9*o$hbnC)v`R_c!{X z++TglsEN907qBfCmb zw(ojo7GJMa9=NemvDWHfE#M(a)Hm%pQ=K|G>w9*>f{o}9cJ%NBbD9NP>1Ps;*UsKx z?k3H4Nb_wb5l3E{I%{cHkcuJE>e&P)rrigm;sNAluBPViX|Vs*C`!ZXyuPtNE-Ms zK<)mvkUCDa5Z7e2s>Qx$e2ZHe!&t1v&JB0lOrKmB@m&h=Y`7W#F0(mBIor#5177`0 zRWuD7)y^gi|Mpm@M1yFZa-lDOWb?x;kC|k=u$(xWZ{uvVJ49`B$kV!GvtXXCvs}ox zto00QD7;|xk}mtGgB!51T_SZk)ZZV%hGN+<@TBDCspD>&G53G||2(Kwj3dsI)3p{)m&bFK z#&3t;D)ak3QI1QFLw;G8*k2+}z&!)Wf|=DERiD^*;MF1tpC36maV+wC;nqBxt|(3DMJQH-^)M*g0D(X@f&^8l z?>ra)P2zYalU7;_js=w1=|#Qek){*ew+`{*$||9pdDGNu1mxf3^UwUl&P~hGrf^rE z>jC|jeZDOTrwC9ugT(?b-7cfP(5|N+oL(2%&^4}oTLTe^^#$h0{?4eaQ}ajv;cJ&} z{Hju_uB4qZ@X;GSNn^Pq*=o*te+3O@c}{*nA&Ni4Kr3GyYEDyF_MeluY4#dSoys-DnKgBW&Cb9(FFYK?{<8Et)Hw7!6g znRwB`CA<&wy^M=Rv9^(rbfq6{d`IIS$QLt?b5(f-o7W|7=1*k(I$<~GZz=Xi_SuZNsC8T|t<-H9heW>=;V;=u{mO7W*XaHYAA`)^G=1?^u8R5zM9d|O@dH#4Hw ze@MQNf8pG}%7D_u(S7cX@AUrOUFLHKA|!rcEq9c9rWcBeIR*t1&D$0g<12;r!F?4~ zRX~(&vi1c)B~r_B6%`&kCqKLc3ZnLwbvm36u}yVr`S=XUZ-9kT3)T?q{93(H12ea0 zD-m2D!T4mmJvQgK-y;rphG)6N>+v6c3zF9+Xo+_0zkSi~QP<=4 zDBEeB-uzoz=!z(zQ&GawC?l;C^E8BmmSLx5)kcCV(~dHKEAJ*neGjl zwFs=*cDp(6NXQ^JS=5#p~c%98y=A1`oXEz6ur&g})5T00w<*({C$Vl1``G9LB|o%1xsxsV;FmHSS%GQxWy- zLT*j69`vR?dBCb9jTEh?!t!~>+od&ss($jUlXB^`>EpMN37IJac^Yk#oIg@nGy`oG zpAlB8ecl{aE}^kX4eZFtwsNgeHoD9g+z@_p=PtOO(pM_n*%KGI>2K=R=!eg__n3TI zti0qFNf183R+ZP?$p|J0Qcn|jcdj5m&DaR}pREZoo$BxoU2UaUV1rXAxU&P=dU`_R z*fLNRsH{2}JYh9CP}35}Pl(&Q_QvrNlP8>rJWlDnqjHJX(y*4jXsMvU?~zsRk8wX_ zUY{-DxAc(mCc+xuh|Q(-_|Tz?(0$2Zypd~-lt3*5&HyqvcwwC&6}-n57Z_a`mr#LA z*akn)xiM)1rwFrUoh>TC&}hs^8fC5!a5F(!;*Pp53pLTvt+r_g@j3J9IrplRkCT%O zo07F>iZKhtrYZ0GYi0oPqq{c$_#(|$cCJU$DkPzoXAIilY2eJjI=X%7ri6#U2~mp1 zth#dFnl%w&b#Rz{Jvz<&VjBwg;3o@I3{Y1Ag~sL5(o!J6Dja{NsIXGavTjZKj}&fI zKuA8b)2XMv1BHr%3wAfHy5a;z<2mTYCQu} zPiGc^ejrmQpQ<}E!|NOG@g`sQIadaEgh1H+m$2U6;?Db9oZ=%?AK$T_SoK13tpt`r z@Lln!|J7O}ua#LGh(72u_)wd;{47nIG8;SST(^~&pP%9rG#SMoLQZU%>vwhtNcfT` z*-}b9n(Kem?=6}u!=S6f z7Usr#rY&;eJpx=)*s`c;iNW{SHT7z?bMKC;QJ5AblM^YpxrV7mL7rr1u%$Fe4Kpr&l$N>SM~746LKk+(SIk6pegf=T?)I$A|PWpO>X^k611(r=?FZ3_>U=mlktu9oW8<2-eb- zj_uOV3&zsUB4f3t%Au+E6|*&Dtk}bfNJdXoAVN&xQg(F3AI|nS_s9+QKL*gP;5d&$ zxeEvfke|l2^HCd1i$CUW&Bmv8WLA*QCFP8@Rkfa4WdKh7x#VBTuoKf;XPjd9_WugV zg)?)>yP&J52s$UmAXsX15Kln0>uB!=!>(7aLUnZ)vPc zj&$=Kv7}tNkA4hP89TcDakkGk&)yosa1W)AVqZCRx z(-di#^sTnV#`T(?8NX{RW5Dzl1{7Gn$oKP-yzfdqDV9qwWs60ElqlwF8W4f{X>uU31ok9)Zxz^^2fS*#l@is5phGFoo8|X>a6znHM6)z0ruO|H% zHiFeEXn&;v6XSaEeOq`Gr)Fts#CKCuA!jxc*?2Kjwt>*QRg#v94TXF9sQ z6uVlCNC^Yo6GkgL$J{yAV_dR6WE;(7QM6YzH+IPuFrZt-4VpS5A3a=Mx2|%Su={f4 z+SeDdFAhruNt#eIa5qkzdqL%lpif|dP?K}_r{Z^sTS1ehA&NPrrV@q9!Kbzpkg#Ni z0YsoLoq5Nfxe>*q4&2brsU8*K?{N+nn%SwT1TO($;@TBJcH-9~w~8fL0~6Y6JW$`5 zN&0+N*2iH{a>7s_-J_1aLW@+hlo$gb$PIu(8_~o#JidNRT(Qp4SQQk?PS}jrJ&op;z#jgT zJHuuRr_W<+EZurlMcoWRH=lgtj zf<@#o`#Xy<4X&0v1)HEI(#b~coEZJ+>AMp>N(}ua2>=SkCYV{1o{AI<~e?f6+Dzao*#Ts(6>O_n<0Mg+R~8x2$+LHhCt)rQJfuY zc5-v_(?)d-xRR1mkG|Yn1pjMdSBKHjF^XI__o-8j1`mpPKUEbbFfEVZRcOFX!}=KH zKws>FhIG`HF+fh##2bkOmE(~Sh8@gy!wif1te5)}zA()Bi0^V*z4pw5O9YASX zQKhbm8F}W)qPhBN$aDwwr(W_lNOP8FF+RT#!(PpRbqrKX&1Y|bIjl_3t+&{$HvmSv zym7-Odaekv2*0Hk*9Daf;AAWu6Oeg%0_Rd{S=EysqZcXEyL42X7K%;Tq&Qg`pf~Ug zmyDl2qlQ2=i5^z6o8GX@Sd(5wKJ_V9ZFkGf&JEpZ{lh81=~Hpz0lY;4$D}yBP|h*R zEq${ffG7ChGn)uAk>mDuB3jLOP}WkB4~)1^vxrClz*V*)*X25Q`Y zc}qssLk}Y5S%v1FLZFa*bQ=d4}QgK5;*(MSH(b6F!-H;rCniy=g z#RP)P(;B&Or)RaTr1o*%5}?E#yGd54;(`zM4J{itmZ!l zvYl=qRgBEO-AYA-^w(VZh}0Yi=yRn6`olPvxYvgFOU3>0$&RE%A^)B~y|+KcuaQzR zYZ7dF9K1a-{a1biUb<&iW1V0_bbf?8W9@B=wUsna$DO%%N~w7qT7lqRk@H_)s{R14 zR#Yn}9bVG`YI@6U&N2HT`)C5eeeN(Wj>W7V2kCDybX&A=l0 zdj~O2wy^iRD;69b9py)x`lD&ri=$S87l&`=@na1F@KC1r!Md1YdY!TKlBzY zdKoLTQfyV15pn!c*2CSZ$it5FZ`$kEp4t3!Y@~a+tS`d=*^!Dw7f|{F4I`P4Hw_Ue z^A$qVA%8pXvCtQh%}$5svh!MmP(o<~^?a5&d^Q=$ej+tPG>ef^Mij#M;`}_7FB=qe zh+CYnn}2!oD*9|mk!0WaBjnnL+0gD_wue4D$6f37<(Ak!4<((DyPH78ziYm|E}z<6 zI&6EZNTH_U|b6F^}H*NR?n6V@8WlR=%yP zwfIRf-^cWL5+S-Gt9e%f$9B#JGiM_wZLFK5ZB7m=I%XfG#O#hfMz-SJN-YnUb!9^# z3+Edz&#onVNmpupe|}Eu*Ie6I29XhJ>jwMv*79HcUyfQq!x-2@o>gT#;%QE!`c1>;28%7g+nFfvoSa- zSzFAu6$B1EXPPjh>cXnW&XX@flBa5ur#efr%+7;cVv3SC@5%mtAfbU^Mv3@VAv7P_bO61#H+5dOQF11w(maYjsy(KGc77^A7SMtQ$s;*Pvc*$^w(dM*2 zO*!sgOtRAMtA6djTy--4Y|pEz+?Tj41fr%D#jP2Va1fqVB-X@q=0)wa12t8xUm@me zbC2e!PwSqw%3HV6_Ki`rfGud6QES?G+w?f}bliw*+#kT;<+zfpdX1KUs^ks#w#9Mk zbHZt|!`u-P*o+6GZnhEhCgQ^zMpA*@wGIOsy;#^I)m?b4-2NQmfcfd{_AV zP^b7zgLHA`p-g@7sp)y|m59<33t%lS1)eJ%VCwu6DpCKZiuNb0+x_P;0pq%Bg%5-T zLCg*d#jk6r!m{nWN%|_uz#@<)&d@i}IrK0cPMo$#?v(K1c>`0<){vLqj#!P=i*s3t zpQW!?;e#c(=6zdLp56R7eEfr9kf#eV8R#bo!c$l%{@W@b9Z52d>#Q_J$l!$O{$m2u ztv+cJhdO|Tr%HD9>CLLj=!)QsSrfqG@P6+wDq$I-G&|(V;s3elH;$Nqsdw_tLT3rJ z3p||~t^1n9q8+=WI0Loy`@TCm(s7o z&Fc$*lh(9siwct2WFKr!aV0rL7+c{H{V*bO42}=&Gk9h zq!f{sm!x{P41-mJbhfLQHtJK6y!)uo=@#XIWVKc{1(3I+c(|}17?IE^y@!CFL<9oW z%u;}s=^Gf0J5Of=pg13qzFh4sF@PxH>ge}J|86umirFmpv1YHeFB@&UlbgJ|NA=b zKDy$LrVJ^hBc**r2OH?kyuX@7S4%&vR#||~&N{Rs2waay3bk#BWR?fw?o9jqhYE?_ z>(O(RJZmZ`UlVLl;0894U((`2vbihwk(6J_@~DIRUPx1t5XQ?vL@eK&3Q4(e>Icqe zH+TV?PT-x2*GPx9Ax{a6qiE;E42+B+24MC6wf9dZt6v|kxpjPYft?r7nQ4Oi1Up{onUdrD>(0%+L6oy< zzgZhK|DMY$6TDpm;OZJ-g8<1s99J)l{(KnDqS&crV9TcE+K_F{d98#!XgF2>heS#) z*%*JO#{y!FbSY`Du@AT?8`h^)zwha%hqj?As{!$Si7x+v@6flEv?(FJTuEu|xFJ}GtY zgIYN`k>~8xXVxHC1XC}1dKa7 z&?tc zPPlz}AKKC*!V?5b=zLxf0N>sFSVr_G%vK5zG`0N|4RaC%3xX`U3YhBgYUupbIx#up ziZQaY!#G^fT*5fD69B0O>Mg%hnP(bMT3ooY9;TOKfT$n(zTiy=m8^yIME0_}3Lq!@ zjcwdqVu6L#FrC0o=-~ZY5VRi3cX}Q9o$sWjE(8@Vw_Q#iqVw8WEoHAIoK0U|i z4EgL5lD3@$ceODjkH+6fj7B2hLU~Crw+K216n(;LmcKMN9ZlIK+Tn@3DO(%A?ni}C zR>Pqu&ybJKandz5QQfUN4B?i&4>iymf9N|zML;uA?0~Iz2BMHDNI0>vhQWR_mR&MEvY zjRqJsB(9dtd4AEA{s1o*riMtu>Z|GsbI>7~yL8JeRzh{p9*qP#QYO%sX^QlJ`YYGw zzpBD%<5v%N9W4x=vVe5~n%@#5ttXG+LodHj*tvFNMjo|ImNtJtJda0+Kju~eOZA~^ zrpzb`qsPRgo{jMwxX2w-v~k)y5M+6%?!(eGW^iULsUiWE4d{h4>WjLhCN75OuUBY*T$$nzP~yn ze=*i1nsp(W8Bjb@(yS;f%a}oOKigD_U!N54?KwLfesK7S>Il=oI^)i|p1QDFLEzv8 zBel_i8aj_z6KnO48v`T|ADa6g3QBE%iAn+sokxAT@6h)+M$21S<_v7qh;503`s=Ar z*82Z^!s>BRRfKVQE(Ea0v$LKDrwg*V-@J1e zpczov8h`{XmoPlgTE|c4{_uobu~#d&Wd2*^j=(>0^G|0?5w$gZx_Y_s#k+&)+3fr; zas>14NU<mZZ{@lM$5E@2s4%Z_>QQTnp^Y-1KZ$pXN>uTQuI7yefC`-krl z>3+T*+(26gq^Gv~>*|wQA83g|lY|Y@Znd?W>!_!V4Hsri)BOtx9mXcba4&_u+Tgj) zWeEvZaK->dJLhgRYBD<}0UJH00bwVcwEz2k-s9`GyAvy{=_4VTHvq5FIFpZNV$HSzhD&4eaPdjNyCCvAtVu}_a*<5O}=I@UxNnAN;%J7$~ zRDn|Yf@{~daH8I;F3u2XPfxxw?Z-(L#8v=Fj{niftu}G#=0?tC)g=fZ!nFP;K^_Ec zr_Bn6EL;=wB5Hc7K3-Uexo|r-QPWxSonkM2pGWt}?J11NKztT&fRB$3+?cR(i+nT% zbfw}2bGCk5Jwh!uNh5CF8g3c+HRiWLMR3sO$J&DT805?K46!b1ryeAD6FHZFEZn1I zE|?y*_!)f~oc^VeKB=6>iMprIc?Yd)KnTh?%aN+TRHlv*Yamy-MPg`Q15 zY|x~2Og%Fm7m+5*wr`pQqD?oeh+)Jzwz{7aP}cq5vgc!%+OnAn*$AXKLT$ldd8ri) z+@G>L`91y$?}CIDFo)^sdaj51x}->5GA#sPO4ly!bOCa@*(@fxO>A+k2|->!6GmFs z&|e1H;e6Cj|NRz)KfYz1Ojs!uzTT9=CctsEP@vCCRZ`Q3j8))K$X`{iyXfnh^~SUV z!B$x;F|Z;j2c3CJVgq6`jnL!=ss_{nzvzl%x3^9C^)SUq2>vscFFO=HIXFCy z^JqzKEP)*@{UJ`waCwl%8-@}VQp zw{pl3Q|s++(3$NspC=4Af3AVntn{TqfbZ>}EQ@cqt@HBBc` zxgmh+bK=J#oAARe^RLzIh1|q2Qj{yplF23vOv#pUmjml$vl+(hh@!dHeG1eM;BbvY zC&z)M;5AWZFIy@ucjEIenD!1eA8Ax@u>db4U;EfHT}2V8z(?Pe(%UC7t_QBSZ%4a5q0P4P%Ri`1mfU#;TK$>GV+Q zd0cIFRp;c-vBh3LxQjb6ZjtpP&%ucjIA_1j=(XMuzM0h+J#7U{P8xsz;%`BAa<5n7 zEkdCsXgO}QTFEs(lV$B!&;m8*&=F1)eM*D)_dyh&i|_|&=zoXM{-Ut{kAItgAbMPH z;&SNPz?DX79OhnhtYLQe>m1IdCgMvP=Lo&Szxl;vXov-Tc}zIzzoCxG#vKf;v8Ck; zM^2KEwpwbxu2{IM8ri=iAAQy6vj7Hy&OQVBrdlp3aPoiji=FHHt52=UoX#m_gy^n4 zVAIn7Gi?ajY7Zp&=&AHjpxY+I1DY2s5=xBTv~Rar^jy&9U*3)Z)A?}PEW`UU9FVS1j;>jA3fRR?HUZ}b2 zY+H|h|9f_E?WH2x;Xzt&C{=|V%nqZx0k|pLW~{TObCY)7m7Xi=87432vA-MSsAl!` z-|LhCQJZ71f zFKDdP7)xmgy|7tkUfDx&V@tw+lQ8C2SScHQd3mQDz|1F-461hGaAi9q=oRkb+j=oO zL-w9s2@7gG4%m;jouZ^KAPvJh_XKVxeS~m{Ao)_|O_(kmL(BrzxB#e%ZmgOWy0rMg z&&PWPssfbrSkex1>LbTzyiaf9hp+B-jC#>ZTD4B)gc1lRH~|9i{*GLmV;(THC|yu# z659oNojs?$-xh=}_&eXY6CGMu>g>AVUn}9^t3V&p`|+`!>GiM7PZ%8B90y`qF4GH3+@DTO#}vn=Q+tiSJV^;8l%BD@T14QO*1m}8Shii zmxm9)^(xD1;30bk_~}nV>5UD@uLHS3UO{oC0qPbpN6e5>L&0rUm<%p zwjALg(}Z0AeCfkTcg#dh>96)tXD6=K`))B$ z^T1{J3N$O$-ytY9|Nd9*|G0JJWzgFOIQ$Kzu7WD~=uLRtkS%=-$LlrU+`v78zb z+;Yox?ww_LO4!mXC|L+MhIc6uCf>KvFGpuai8qH|=5p!&P52VgaVwc}S7@y#GV%ns za~0o#W6DHe@So@2Rlhf_{IC=lP;1j_h%%lNfKu5Mo9V%&(1<7+@Wd`IxP$n;o5esQ z{CAo~QutIRIo650=e%iuw{r-vqN(1S+ipd^us9MKkjm|$2WrzRjG+WLn-3QP)W#hi zOO6UG61G1;^@aEty&BKtg}M3mM{j8^J?zxB*3S71@xdRqsVAf-MVm4X-plY!n$ErQ z&c)P4nFaPrTLF}^t%nlDeX9(8&dS#OG@e^YAKsKxh0xY@&5)WfbYfUOTMZq79snoHd1E zDK5`n1l(%f;w(59e}%95(4`|^7F{rW7Y|FViCf;2`ReO2=~Vq;@u%KW0@=^T(ED-Y z)m+ygufzD54iYeYyMV|R(@2dUyho+rQ70Ls3xZ@9+=~4PotfR=cQ?w)FF5--?4!QY z0bz^4Gn#>17LTanM3JZ{_6NGN7@$|7larZ3*7i-p(!wtG8(u=iSZGRAROU3I zhtA4U8=++m;56~_#INUD{@70G=Y#-KXCPPE^1t##CGSRbV`zArN2WLC3AA?HH}tAd zyEd+Os!CQrRJUYMBU?^yuXcd7#_B_csQJTPh-GEZs4rXKLbGh|s|Sb1+QG=0z7Bey zV^Gf295T74<*N{%z$&T}znMRP{Gh|6YA)D^-{<%}#rb=hcBRm4W^=R-nLF;H05RzL zIh0D2i3z)=p$)utdq5rDl(2`@Pl$V=cHFi z#o|7)I7;7glTkJoKZTpD=~(XKAg;Z!M`1r!jhlCA^3AyAyLgg-S1^-f8(oq`cCh?x zYc9^en@rlPrXp+gtVE86PAM*DM-0g#NxB@lIg_D3vX6v(JYAjVoAO->Kl zCD9!XItVEZoyei+Cq@)fbF|sIX%g5p7o9~1Ikt)?+%~U$AU!U-7mFNQlnOm@UU2zr zVXVPPotLtS{`LY!Tq)9RcjQtUkM&O!b2F&dNndZvj~i8<@j&_x2!3_he?V9%{`_%V zjCHlK8Rln3*;ZE8O;gqVdcIx&Q6oSVXZ8Hw-6bWeGp!EW1_35rcO5-R{hv~i0EE1%wsCPFE=zBniHgQ5WR zcA6oxi^2)D^I_zTcIH1bwajo0>`d-e^Ski%@a~qp_AXdg7GYB&r&hz{0TByqsx^!; zYv_afPRXm`|DY}vevcT};;Kk&;Bu`2Mx49H^ILGQE@7$3fmAWbxHA`e3kkr=$?x&X z_i3d2_?WcVXC18UIQ~CTczY0TyXUr%&Y|f{`NYk|TC`@n&14BJqYgogx8JgEi3i>& zpI_wro8|jiO8yg`e{L7&)0+6S*aLX+9398>7R+6%_-1sIO7nkNzlc({3`fyqF2ch=NOt)r@QQx&+dpgOIL>lUvTXG}Zj-B&1wzbwwr(C%ZkY?Unny`kf( zhy1=CDrnB@oK0CB0=f2j1Np1h+>t6W@b9MDspYSp+x;~EO~&7&^f(Soahh1gF3+Lo zf{G;yx+02A=U<7s@=IcSGG3jS#^|grhn=a;+S>mwLjaT0oGH05fJ6 z1=ck>KeM##ZwqRjT0^H}r({1LIuaBdRx2h%D&0%ucm(+_vs{I*mwH?BMoZMkRjnZo zL{<-qLx`1d{7HL~@%ctL57!%-TXyyjL{5CU=W^o&%C#69W+&OJd%*wQeUyIN>8;fu zH;CE9x|3SsWaz!p3ysdiV^1?)Tqlrp_Sm&*i zt1->vuGV_STQss5GCCT?qNM{tHU;6}j!SZKi=?P)3<(J_U293fZ@i78#Yx6+o);vl z=)3=YiLjE`C}bOq*^Ni#HIcL(3Z&A^(S;T7z)p$(rfex3{Saz4(>O#L4xgM76;e=z zhFXJPuc$SASs|@qOb8PKvDK%e#wmxD9PKk2JHnVu|CFgLm@=Ybaen8PZREyTGe{qH zD)a53L#FJA&SXP8yJvI*yJq=0%_Qr79KD(noriBL>TnP8{l0(-9ZK~^zADr?;=Qvg zjaMRs_PTkPWFOoK>*WEdr_Na8_0sqeC0K(YU(~$F@DUU?yW+h#^Ak5<)>kc+OHsb3E z_~n?hQ4fTsPnlXbKp=38VhZs@K#v9Jf7*HJ{!zkFEa93p;sX_3k1+rt_|TZzdYKMh z9nnVFWtW~~tC!fBK9pcT@W?#ss$oQ!`iG|xl8+vqQ4BKS;$RYCenb`Gh}j##i98{4 zBu<|yQvV|xy*kJBQS1|cq2$z>K|jdQW;deINm+xq9ktPFbViw-yQq^v{t1WFBUqI@)- zHgh~nE>l-@N#4x3Ry+~8T+=JTeSF;LV<4zvvo8-HfuuGB@)zCD6F-Dy&EAcb-Bt_z zn5j3=7%S>oIL^>f@+Ai+9Pq8%SGLJ}(`R}7^-xB`>Ze|~^mzxv;&mr;5B)dB&Eww~ zH-)38L!tA5=WhN!1{Ex`rrab9aW{Ppvbyp+T32EJhh*pQz~r8$4{q35D^9}{E_9)) zhR(rO!B(y}&*+R~CgH+r@9Blt3v#^u{j+_y%DnEIP->YJ^Xu8Mla`v*<(e33*tLep@g~QJbncCoP>92ql`hm_>w?9uHFVo|MH##1^TSf$p=Q%!f*LTxihtCfj$< z|GG`9l&2j*Nx6Qh)Z28)S}J~)!laql5wAE<;3lsc|A?5XOiHf%^lAQYD<|jLSSDu~ZKG-S_&xIdd5&r(#kU=@3!ZlgVhqKvRuSwc}RhoeI zscCvSa#$KV>MN_7F-sV%@0^BG1}U=lk6ogGiB#>$yxPp*V<;1%!B+-5T=F^A6X)vx z*)O*Lzon65+TGZr$@;F0LY&*_!E1eVWm+#Y51De7kfqQ!}G!*DRVF`z6U6NsGs^wVSS|4Snkk zK_Br5yT&?h-}#isVR}C9T5b(PU-rQcrMpX#TrDq{_=x@*idzmP6$g@`FL%^E_>DOB zh9Q`&8cz?nl&YcX+ut5Nompp8#!%F6TWjy!enw-^1eWP7;UCZZmLawaj#_XGlsWsb zBjdKenRyp`nImoo3JpoukP4`jXT#aRTKoIGZvyvnbzV5)dCQ}DV`ukhHw1YtPwEhzH9(>8$=#KvUszg-RfZGD$Qfe*_ zKH2{cNv)46l#%a$d_-<7ZdiS(A8*8J0%xyz6NWhO)V*-pmv%#3>6;wno-XXao{Q18 z!!VXc-oPfGpg1t!YjXi4|4}ov;jD=pZ_wPN>1A)&H8J)Kw!k!ePC_(}*pPZ2C+V!L zW0oslnvMip4{`rfP)CNMdb5X?>@vNi_b%`v3&}7UEU){|l(#EOjK{iC!q=C~^0HYD zu+Efu=5Ni8M{%=!{zOgZR0MMI%RYEWaMF(}%eby$)w-=LSBf}_?cKy^M0e_u^n7%w z1 z*=G6y!S%_%3C>YRJH64NLkmjubkoaVFrPVtPQaSew$bc@V)VPL*}Yd6&IB&sY1OW4 z5Y-z6CQG|tzUXP}^yurv5~Obw5p5y{lWwC~(;A;}z`0=ZJEip7bSW36W{l|UPfC8$ zJ0V_dKAY*NS@TKKy8rdgBIw-OMhEPBHjjzh5#A1bV3|>sry7 zQPqd05|<``^@Vm#GS0#2tT!!6e=y3ctjBOO!O4%Z&tS9LL!CWGU)k1R45)4y(X<)N z-1mDyS_uBmhoMutyBSg> z1cvS*{s+%{-gCb1|JGtH7BKKI_r33Z@4c^UUwb9X?()pD-m6EM{CLv@AcRX0Jq$*?1G5$$W(52;%P-p) z8Az+{3>TaEa8DUy5HF8iRyNG>n(cLb?< zgg3w4n*FDI6Y|6)pfQ9`8!cI9)a$~&4*Gj@GxSzG^K_@U8Yrw9c`@SLjy3`GJ^@zI zD^n9ItG-F|#fbBrDBA6fBUNZ$rRr=deH2n={^anjEfxiL*G z*HKs;8KG)GPc}2JP@NhoFEkFXu3xBItm3p78dr1HJhr_$occT(!B%|JpD<~16CJ$f zv5u@(Viw!Dly`*=1eTcAN#;I7R!VC%phtU#O?eg{RHY-I!kdssA&VT~gtSrA8?A{0T>~H8jECwcU8>l~s;zc)5KSJMz$8bnos>Q; zp?A&bcmGY*f65vtZv{tjT|&n-pN-RL%kFBN+Gv*T?pVCN%Ye6zq3qk)3FO08fTHIK z2OLiIyzp&|d#Py7~}1+VV|(8Wnhb?a)6Qlp~A z8lyT&07Q2?9Y<6_5q}Q=N;Frf9D$^H+`Yr_-|FTa>297RO|L>(9j-E0S?&qp(twM$ z+-l>#M+$ETXOi!Fc)IiQ3Xu(3dt!5-zgHX2*1?DC!T~k}VyDMRZKWV)a?IwMd;k~7 z&A&s_Cpf=jg#bPAkhJTgTqw{+|A4%4!WrdtvwKtdHtOYS3P5S$Dkx;vj~uRbvfTpl z?fqH-nilsb_cr?`M~1&rCWQio8wnvm@?=z-(c)(V0@^x!MHp=9O9)3tCca74`C3-Lua1ke@8;{vp5w^X2Z+0j`*Yx|m&TiuQR zNX|DKK9~TVf3T30f!kFvbcaN=;T|ndhQ`*IN?)G6C zAj>l*bQSeSjA!vYWf8_$QbT=!ce96=rH`oA-4@A%DrA}Hk}nb2UmeNcR4}WRJ-yII zOoDb|qR=D2PXakxl}*R%1D`e^1={OMzWK5@_RApzxQNu|3+D zXWuOWgj?n<>Ma|409KRH`r~aB0LV=Lw*iX%hm)iIY-gk9ws&@f)UJw)DKspFd%oHc z133lIkD(!eYB5UnkO8PvhUQV2qR|2mZfOR^yaQa}Y$-5YSCLzr;XjVC;?D#UR{@7j z<(}1z9UgOYJc|k#n^l#SpjYKYQ5s|bx$bs)90N2bo1!(CJ#OAbzmv!QcP;adLUh%R zk(=#STvq4yS=gTN?RG$~_Gf0kp&bdWatrMYQdG-Bugs3}y3SEM8KY{imKsm9A zj3fE?dI>ziu!0>iGPOQiWs?O(wA(mZoWI%Pw(I0&Gu-J``pB{Hn=OD$^-qHm-mL)T zkJ|>NyAfm!fWYCVaGzTFr?(XX**oO<9)$tzfR;zc^{o>C<}Kimb55fcZ>}HdhpRO> z!#sY`fWNHSZH;pR^4eeb;*#VCXrH_bqi|`0z3)mmDJFaF;GQP~Fv>A9-(XRSS$ggR zdL@!8TbrAil44wPe+k7GyLE+ARyiVP^JFqizvbDQ+~n{`DogR6j_;)y-P8fon}9}$ zVqS;U;q!?E(uEp_5M;c2F0^sSj8BVgp7+hx#<=IhXca(9UborP!@ek{PGDzl4Cpp` zM5;aXW=k7pUKy6)5~Z!g)&jCn|HN#pk!9mSA9Vrb{NwcTE13$bM)zWY8v6nyAHck*8#*yf0A?;!$fGmvyX|=Q8pdfG#p$@3KEH#_ z{5R@wC!G$ZVWz~PX0CJmjeD0C?wP6EN_*TLujKW<>erns`PoCU^)?@mQ#CiI5DrI* zDzV&ZH2~a9>AO6=Sg8MSs&=q?NkUfPRE0S4a8N{xqUy`mxm(Ajx}vJ90&xIhp({Xi zM3YF^B;k*sy3x&PJbvR9R~j!Ba9oXq4rqiP3+xO&VALp$E$#g2 zw%a)6c^2q3ZP|2g>~HylV_J@*s1PEbE5p7*$Gvx?RLvSyP^2p;XIg z_IsrX(^;U*hanvQ>RI9?r*L(i+8IKE-q8SiPG z3jO*=`Fs|HBu1wF(d*~fovXD|0@Ku?cmGe*N$0&4ng_6|6>ZdUarHtu6j2qURmvcB zYj>4$w?nE_rjv3g60KG)aH{lr$T=Nm;3r|krBx$rFK5b&dGX|53gQ8xoV+YY)t|~9 zH?LFW(LJxbB~KnZJ-HKrTbD}wA6bDT?&rEBH4TV8ccPtkE1#%5$s;$e%gRQFrDuH3 zuJ9C-!s;wz$Jx|1m6O@jw~q42T;Ic(g4oluSg*>e9s3Bx9qHcSfc1BRLPYU`b>}zg z=2QE0&p*g#@=aZ3%Pw+s@(!iKNb9>|8n(Xzd6(D9CCRU3{xOqNCeP{3@SU=p97%Jl z_ouRRwhi#S{Mwn>Swu0^suAgS2I!P{6jQn`yKHeMUN$~{C~}0{V2_|1Z7{QbcSuJb zBi49#l>eR7VkZm0?LT6Max~@JjP377c+CLLq#QbTzczWNMFb}s-*eUf2y zxG^GWH$Zh+UC5;3GRnl( zlRE{OY1?P|_f{UQY0mq>s@8Y-iVepk$5x)Laq6Mlip468cCCj>O z5g;7P~|ofQS7?JE>~6zLlsPSpL0^AK{)C=b)2y$3us=&RZg|LOth_(0_28p z6UsYRvWbU4Gq93ovxT+*w5r+A!)W?k$C>O@blz#(JIo7=6EdV)2kaqDGT1jU_T(cV zx;c3x`9o>&DIMnQ{Zvil-Vni$0qpOR9t$Mc;1{PT2Lv~5!J=3I( zcrW6%8^0h33=~ZjpB;!sJUx!;k*4A!d*F8ZV>wHK+CcGr1CPl$-*&8~WeZo+KF*xi z&DA^Z%m)u1{FY*GJ6NhbqXvbMu->~JI-|lvMuIDw)3tj<L z))NB?k&bdqG0aT?@@lXUNForob$KL*4%E@n;ZUG9K0O}v{ZhFfTj#S=m1mIb2*Mi1 zjW-c9TdDR67lCWuYOddCPYdhb!Msem0yVr%Mn(iMX?#BxF{->vv_VTX56iOIt#x=x2*h3=^m(Y~U}kSGK?S{cO8-y|c%cLI*{`i7=x_;i^Hpm`GFn>L{cwkP17R3Dmk8f7(Y zXbO_`M@d`a9!i4AwhN*5i;)^s`OAzjM62(4H4|Q6L~&carcirp(5f`ES?c^IPQ6D2 zQnZ~~1poe)^CoAi5r`n*XduyF#20-zMpZ}AVJiW>&v<<> zMOs>VSS*p0*FH}}XHbbLv)f?X*#7d&Mxc`q4ITRjAekt@MOz&t!Tg&YCY@WAeuvnX zllr$WL3g|mam@PT8Pl%`F%zZ+is}O|wu5Fu2oPQRl6&`_YsvUzjkw_UDYa8OO*;Wc zBH)<8Ad>HBKpP7dq*XxQm3rn&QfYVZB{I!Bia2v5{Zv(c`$VAX(e0Yz7p>$YoD znI7NI==+MnaFbraDaGt>=z!++P|Q4PLHsSW>fNr^Tg=#Yfznp(hy?H$@z%G6){9^I zYDB0Ckdenp`w<*Z-|1IUw+tn!XrU_x7mz->+ztjdlK|FDIawGau`|qN%mps~BJq7e zu~a-03zf5cgt z)~?(e|9Mg+<>qN8eKN8nimHyxd<{>$v};EEOZtIpF#aJp{UZ57uH}ZVSvmauYMeFG z3Q^~FqR+Bsm!9#qpz>s*6Uzkyh+}dpi~ADuTbGyLFBnAOF(~+31HUrDK6z~B1z@K3 z>`L)GPJ=U2E*?Ph8F&xK7z&0KmU1S5;Z+saSkBA-#KUl7`C~wmcc$DhC_MaQUk`Ml zLEXQ;T)z$7{bDTbBbzMt#_%VdRF^f+5&W$RiX1!}VCMa@%L1W{%FZw)L&>x-LVN7T7egFH`XkYkicSPXylVF?rs-;suEG6Opljj>(;cIre*f;l(1J5{FHW?Usv>dVJs-xZlE? zS{Ig^AaU!z>abnX>s)Kc6~d6j^69%hrq0*nf$`w6LJ5xqR^Ac%`-&EskmSsf{gAD3 zCyM^$mAPiRL{#p5nykx;$AR2!)XlsSU!-K(dPBn!Y$EiplKJrW_l)tc+llcfw|JAO z4jw5RP20W3x|g3{B&8+&@+0=6w$c40R<4rV$sKpA+-*G6tA63)EK#J}(c5t^WjA>` zLBGx%bPk%Kc>S)X*2XpxOI4=ClH;!KZcT5oS%_YlQ6LB330gX}Mzd&3nnL(bhFrEc zJVFn~aEIEy%I-ef+c5hTSI=jw(MAmGR#fzEfNFl_{;6H5@99#+S;EPYaWtBZYhx0u z_6wKN7*J?ad0%JK!yU};wBTY~NR5-BaI_5{+D!ZGwCJIlvhwgjl$GaUl1>Np^&ko; z60TLfBA{Gg&p}O;da7W_`C{O8UY;Kw(OgBVC&Np>Aj_ASV>K+fRLq+iY20Q=f{a(USKv~KC zfh6!5>~77~p77E11-`3A9dX)oW+}b7F~JZ{qQH;OPsHBJfLYuz4}o=xW#IJC@nK9L}*jHUg0HpCo?e0}`;)G@P zhV3{$_PopoI9C~232qo3XUl-P_^q-)OjZRme@7-T*yavUdMZ7Zmz#YYf*Dtb>Uiau zQQ})>sm=n-Vfg|4?A7I_YO|=GUiwrW8J{DQW#jS3RU$`;_RQEa&-xEc)Ht&a--|T7 z34Jl1I$S76cXeg-h|V}~xdAihZX+=(t7~DNSbCFc|Nqa;8g)>ASkIH{KF4$~v!m|Z zIVb5Fm8VynCP6C#Zoc3E4L69H>d*5AW=x;qng1DK6l~=~furQ|QjoWrs+s2v_Dv;{ z^=R6?*$6MaD;8tF*-hui%&2pbjQhZV4YYFubJ6))`&gjIaQWS9ji{rSQQTTkA5f$zU|-(c+gwDO4R!+s}>k`ZKjMWgG4zs`&i?mkFQVw0n&C6An{ zJHqp7O_EkpQ4PEL(n3-T;g>XZq@yQ_(NHt5papS^)Ox-I#Qg+6q6lq_6g^D8P;K4% zK;Cyuwhlyxv;ENnpYNg#=gNbx2=`{qOP9~r)24)YOy!vv=;(d`b@A{ogtP@i0eK@- z$>;6r3VTViJoZg zxVhiWetwX5-E*_&zR;whUT2@52$iu*5n;mgx%qdPvloX^u+u+RF)!y$w;`A-Tyd;F zKxH$#hO*K9#m!B&RVL^XKa`1+GCvo$;jVtack7XYqOM=>g~2b?l&-4rh-kYMpV8Ry z5r$VSGlwl6nkE2?W{^JkJ%6X>MKPkyELO_aH2$5h7C%W)P*?M3OqRD(>*&Yo__H(? z+SCEpwmcV$WS{R=V;JlW#p!pa$dOxX-T~oo-#BaGbCDs6&=%#2b_rZhDu zX+f3S;O+!GKlgWRNZ41lP+Q^+0w1B_MClb1`z0AUCP5x{j9m-Ltk|&SA{r~;g0h+6 z;SC>(Kw+-ahDq6olaMuXcLQE8m!^YSm-0)!BeE4*Js*&nNnIH_52)b3uZ&J%Y>?bh zudrE>6!`u^YjcOI`5Z?wj4Yet3v}Yu7^2+pk>?F+5i*h~;hM+yCwM0ac>^!UCnonT zkB0*hD#p5GQ(*G3`S=?Lqh$^v;wM>hjEfuFBr-hQ(bmnFCAIz<2y4NTtGwDyQ{rnBnhK|hp znVfJ$606D=?o@}#R8lz2TE~$}-vD!{$78#}lZI{Vb=~#tte3|MvKZXW#;O;ejvGH3 zfkp@h1-{knMl$w1y2x(~X+=*N!2 zm7){8LrIa}f)n8J#}34*`7bDonLMU6ZiF*WAd#X3#v~G+czIAA9hsUsQ~B>b3OL_2tc;)% zAbZt#^f&+yTyhQQ_2tAL-3Ck^{?}K#UXa0Lo=w-r7T48n>W9Kmo<^Rm4WJO!3Z#0+ zUt@v9?Q!hyiKiggZdVs=-j%4?;VXs=I@58???Xu|TCsyNDW-8>U05HXe&XpLmApMW zTVfJ7Oi}4m0wNi}Bj1Fmjbm@A0RPEG(iX#QX2H_6;3q}`^Pmu%79@x?rB$3-(rT>` zI6HW2qel1PWw_LV6hy(ET`Pj1gvcG%h95%ZBJTd$XE(e9<^lk%wKAckuAE>S5P7#R&|#pNhlgjAWXjpx7di&R@d$FD=`quzAvh#Gzq#cv4; zdG~r2AVq0yqBT%tD^ev0nai*$VeFtDT{8LcyB~h0hY;uJGjZ$JIo=)t;bgz)MpQjn z7bNi>&j%N-T5dtZ^F>j&X~#UV{CwwFKea*RQ%voETtcM{G6NRFB~3S7<$IBKq`U*1 zLZ9nJ7eqax4l%?m4KqJVMVPIjWCE)C@xwp#h36;Wb{1uA2dh1gy^mrd8lmGv*1wir zMMTYcgy1i%OZuHQVs+_W37Q>z0JdaLPRj6>vhmu$F84tn2jHG9OjZ6zDOAuNF%v^kSx=q&2V5X5h&N1k67R zg^yhXL~9aR8Atb!*M!8G(Q54jYrdIIOCMxwgW1}tStR^(B%@RK!w;iq^GGEgpibgv z>u+zjIt`{bdM?8}@4ee`1M>7SfTc8uE==L{deKfj`aMS8Wm~;cl~iJ7jyXxeD_ZWn97AEhpuim%{i!qg0|=8R}> z-CH$tgmamdzlkKRAz5GW!bo@z%6QpT`xDKW>!E7w($Z3&!N|PRVsYR@pJB@X@1!hG zY`B>9@C1uHq#DY=o%`$l3F9??{QRl+9J>?|>Btb&ajXE=E4<6bo6+|QiOZ0sc% zOdx#!Gqw6TXg}HR#WUBzdDA>Pkt04Y>p_;ybM;T?JaP4eev@ja`u8ucoWIPLnw)_@ zLN>JfCf2p>GLLAIuYBLPYk#>G$1F#3=|b3Jz&yXPABb|WGI8k*a%$0kctztz}rtv#?_vD;!LAE9Ju6A zzcLv<8K!c@mq0lM$ADlf=X;GZ^ZV-ae%yx2RWzA7AwpK3eG~SQ2oFXIM}Zg7|H;G%b4O`UZ4FQ z@_k)Ru!yyOk!N%igm{Vpqtdw1!EeM{;8hOZZBoH`YVoQsd@p;yG`%3>7~Ed#My$0^ zzj2i0l+n~69YJv4DY%^R^?}vacoFq%{(Sy)^I4HbFw5#Mi7PtVNa86i*!76w+BUc} zvo7q~yGVV&jhM3eL0;UoO{G|uW=zCj_qWaW)A)@zZB=0{<1~k?R#&3f9={)m55)&W zkI+Q&c(Is`x+l53U3ZhM6zJlibdGrvQ$_KGKdxD6n>(2PrE)aFPK_s?#hS6+RJQQ+<~6(>pL?;YA{d}jiF=nF(B zqNnZus_Xzo;VmQ4bv8x^T33rEU1q*hc$=?ve#Se2C9D1~!ip(&F4Vr?W-a<7Tttz4 zzXk}PQ}HY*GZhIVld*@1R1Vmp1zzmPo^L1`;-<-TzKrpzWUsUx(YJk?J(J=Jk;gM44y;;8EGsoRZ@iG9aC9# zRy0tIUSRie^lLU?L3p-91WgKeB}}HuS2u8rcxvuutW%FyR&u{Qe%|`Hwajjw!h&QT zFR2r+Gg$?^Jn*9>9>jgm_!KI+4}(Y}n(<|$q&12(|%##_`!>Yc25fyHoy=0**#&q$MxH<2y8z*PJ=C5+F zNR4E^xy27~IQCRWCW@8C)R$_Ew`Jzh_y@HGiV}Pq85FZp`17AM&d@!tVY`jRk;6vR z{~2WyZ-^%uW~Nwi;TX?PGmc4B9yqrf2~t67&0O}$j9XphT+s2tk-jLQ?cW>= zMI2M5r_S(`?)$}-+p&d{a@*rp@&nGyju%v8&L&#A#aeEKya$fh#p_)|OECJ`8zJZW z@=Z?QRIhcaP_@CIqaQgzyXeP&|3+8tx1y5dc9{vNveNCuHH)3eE2IS zw3-bo{HvttXC9r^@ukMn?N#OFTo@mUy5!Oau#D`=nOi`>mfy`z+9_K$q6r*Hth&Vm zP3jFL5f{9{xsCfGhdj`p(B}eipY=D#mtd=h@~(zwQ5*)er)0Ij68IiREEb~Y;A&w70apo})L*;ciEp?hES6~mLpl?;rX&+g z7KhlUjezG=e4anqjDv<8_cg5po5dQYPQGT)+iR*esna}C;$%@qA3@0xv`I6R1IhC3++JvGz%t(lFCvkM|u=RVu1ep6|I17%lkpNiZ z%Iq9->Y#=9z*EMpij20>ZJ~-j3cuXiyTf+k0QR9zYUy!I>?x$5wD0IF*idJsf5uFu zx(G5^c?y~3j1`&w*5NA1G&wMk zku#vEo|nXpVS~WFG*QKV zXasNMlkJJHeT(C&^U2!R9fl;Tt2>j=aZKG_f$)C5y{#_h8I&kC0*BPoZrJS|>fs`T~otbjCtHOA0 zH$6T54i3(-&I*Ff^VKN{{aA0z`JSVUTj!g+iBU{EJShpDIQz%R%WYxqyCZM^v*li#~`cDv#V8&#p0jwYcHAv z#vUpn_n&}3fF{H{)HbrlaUdh?u7)5(rOglX2CNk)J0YI%k=q;j^qgT`M_1=(KUvqu z6Y{NN#P<^lK8MZcEIRf(l)tZPs=`+663EbsstL&YDgsV*3y%v)w_+W(XVUt~n(OoO zx<3j|rzus!I0T=ysJ{e_nZkEXp|$xf2=1=^&N4|3bS%|WX3eUn+}vu&n-`d(r!7g2 zD`tPd_WR3Z-!poa%kR&l#Yfg@N0_EbBIl5%5LV`k&FTI$IQLrTQK$a_n=9@>B-}jw zF;K&q`wZV*l-;FT5fL){Zjov|$jADaDN{a>z?bH+aWO(ewKI z?HX$ZF}Ig3mb9m*p&I}hyZwh$dqgE0z2W2V_>DdUQv$G_!Z(n9L3y* zTRPWI8yXrwxcb2oR~N@h;m(Ee15%p9Glr_)muvm~{i`S^%^ZCt0*U+L2D|ai6!#ex+7|Q&VwjT3YFBVB)ik7@EU2+vABy z%b<^rDc>4E^`dtUEmbM>7va<|%TnN%p>GCydp|(q26bX9OIsDCjs#+J0X7ISLQlebbNYMFI#T;tS<=xI{Y%nmaSvS{g*y(z2v30%l3l zd0D3A!3ik?vRPY^wB5y#DKIvK&ly=cRR*A{T!EngQ1%-9v4ffyu{M+zvC#6Gi#Mefc1WuUji7S0|qQ;vEBpaSftaa0TjX@ z050!*bv&iQCkV`c^!&~MBu(OESk?tR-0)wD>&Obmg8bN7`b?3%FH1 z{&`!vvNL+#_4%1tfL?z*Yh2vvufrd}8&j^`0a%c^b>)cDGwo|8kc(i*BcG@Cod3YJ ze&(S8Ig*!bKYf8JjNf-z*U;Ths_hJc&c?55IS%{9da=OCm3MH$99%8kX^n^Y{f=u} zYKi9ayfS;1n}rno!ZSujzTc;Q0h*@@Lc7a)`|A!aH)LcpOtE&~W z_E70I$c7%D>rq}J>)w7BnC4|pEi?qEi681qF#Xirje037TB_4Hu5;(t-g4W!@pRv# zAKqwcm1Z&WxF+?u8aimr2;GiMyY)f(Tt4G^g{PTHQTj?2+gdd`26oA>qIkbB9QZ6~~WSYn3 z=Y2q?;a5~y%e5DCh(r-au!z=V)H>WlQ9mOtorD${e|rf?jXb z7ILmu38~rcnHC$H6Ppd;zp(=SpDZ5y64o~b5BIXQ{x+$wq}t#I?!;}LW}sU5>Xn4kc866K1EV5PQC11IW7u2Jodg_t zz(OU3I+lcMC`VnD*YstJb_%wi&bn4G%l4sgcB6-=AEPg6WB_hDkKGy_tM48&wCUXP zDL_2{%zJP!otqufiCqa3SWZINn>^;7zjyCmw4O2Oc8Pt3LL>s>MY*H&0C}Bv8|iju zzuvANz$}^>RZG~BpNL@H>S_Fe&A)l5h86osUlF~TUWf+=C@rzS2Rw%tamaKvpyckY z4mZl@>?zlqDPJTEPGy4V8j+L>^8RJV)QEHu-@O(LaGvLoP4s!XbU*P^X>sxUu}tcj zqg-v3EFc@~@9{lvem%lYKJVN=o`yjRC=x(jN>zp{I!5UKTHqQp&V~LzNXSzyh($M+ zR75deR@WMaHKOG%U2*j(nt$}v69cK)`<2@hp&J;Q{7g=MHtNNuil_QEb8u-Un@XhPj3e4*g8zc*L z4uuz9uC8?&ZYy$Kn7MO0K&nJ3917C?&p{9)4mnXRI^QsVAgNq26c6H6%HBc z-84^jP@j0PNYxtlfjAKU@Jc#+>l{!zsCV zCU(^YDZt!RXVfH!|BIRf1gES_mZYxt=L=^%oS&IhQJH~vLTsz7xd56}Zt^7MDv=X! z%dV=|sZMJf#~Mt(Co1rn$p6wC{&+Chrtc5|#wsMj#?v z;QGrFucgZ$MliZ%dI5!+#Md;V)1g4@Z?}2j zL4#bult!?v^i__tdbM^=5#u$0q&#tx5Kx#EkBkdi(6&R!e!h~_VdvTcgb|;{uxb~kQI7kkj)of_@8Ty z8svcM?&1%kq$~~u@b>>v)P7C`wq;xHRYIsm2n954&6S$;iD5SAEnw-zzIQke>ekLM z;O zom(tXA={*g-=AZt)E!ArW9is0Zi0$i1u%{YN$S9^y}zLe2c6V(XKzq9;f^CTW92uT z`-9l|1HzxOAS9$HPbVS;{wy>NNU?j#g1D6w-JLLX`BEsws);)t$!p?*Q)K7$B;>4d z4j$x_G}7eEd@0lQk}|};X?L+%wVCYkn79tH&( znBMt8Zq^S>Ev$%wfewo7v=PCnIyLbcc~WCDf`_6+ztl^JB5RCEJBH|$a`NJ2`wgmh zVcGx`!Glfu=DvXIkwT}9F23K6ccIbBz~2aukbr^%@izur>#;%FaZib<-Hnj;VBA#y zrPk~^{^Z`pP=_8Xh-d%Gk`61odzcEh+Y##I$;J1=b?iJnpmtvn96(T|xe-AGw17WH&Zpc>ofMoE30 z6j~^s6T&qb{CMRHk1Zk55F|L)&Q-Jj)m#xMc1SM$C+s5rZocLM6RcdrK{5L)5%) zObKb_+JSj#;R$TU1s89Cz+`ZBm=3*hzmuGV)xgiJv|r5H`t^eQp{&Y+eLLz~Y50a4 z5GmA(>I>4*NgtE1`9kh>=~OKP5z&@Zg6~x`P3Old!rQnz=m{*y;m3&m>5g2T8FniuvrpUE#smGu1Ra&?L5#D3ZR&^POqiBu&2&5J0I5g=DN@0- ztnz{ZIT-KL9a1>K^o1-?AUFEeb2&E+W-;Q)LwNVY#G=55Q)G&6W57nES>m1MUv@-r4j(zZf%>p8esQV z_Ln0qJ>@xvNvo|lhq6%JKi+4UexuZYpqYz?Mr>ekNpm;ZP%j!|D`9mrL_r)_gx01` z+f@sa>om8-5yigM$LfPkRSCcX&jS$;N$X*)5NTgDGz}U9DJnfTaN7-nO#pFa>m^+F zgN3n+k&#isLAHD_+lqT5C*aLUAb0$^Vrfx0C;?+aSP>9r|8*N?4z|R3_|CwQMi%@?ZaPVO=bUvy1~O9iq2y*$_()+-h`yi419 zQzCuld&aPCLLm3U=DP(2c^6lH{x0dN*V8+fL{)X)?E$}(RYS2=U zyWS7JD9%(Ke#QwTE@u1=}nL`%4U!t-pn_Yk$)+--B z7ky!Qu6iS(^ydcyKOkdd)M6;=M0sh4h&bK#?? z7X|~cYg76%eBfuL#Sxjep;BK9m%>M@-9Td}8kW#vzL&L=ec1ec9=>$UOnzB-r(Oq z;9Jta78o5nsOvM)$CiqG!tq14zqgp{%}+lG*5D2xM9OW9=#zJPL#OlRPL zf#OfNYq~Xuu+(!(UB@LvyZK}h?ATpCw_x6V_^o(g9Hms|@~_GjIwL`~632rU_L{df z^VN;&IgY(CQ$NgMTSSIb+d7X9^{Tag6AcUwP70~}p6l_!$0RXJJ4!I82v&QW0Os^I z3Sdc8&AtUfJ{^p{HaT+E1AQ)>wKlohgSqHvB2810^9{gb?7V-_A2?q%dz+OLcgO}8 zePlftA%NT&EB9qZpQa~#DCgL9C%l;Wl}?4q%LX8V9tm<;juzX`bN8>IRCzc-)Swu$nGU>xQ3gR&BN=CO~vJRbep*|B|M6* zdDaQ7PoIjrS^WB7PiuuiF?w5Cx&81CvJ%7hE9v}i0qak<2yXgQ0=*gI{SQ^udqSz^~P-x1r<3fQ>%?HX#G zuz3hn%;jKo+mh*%7q*6gpSb-1STGM6CJ6Q-~cs~ZY@7p=ckW6XA)q0^75tQ5H^DWQAHFS8MexR@w<)t3t4V+5kVv)FG zCjUysXSOeLH>0)K+TFPof$iF^et$josTM-3pw9_jn6R;7aWbnydqtSD)MRop;Vdq2qO%*(1RU7TK+?i2zSwZ>ZK7uUSu`L{eNfm~iNnJ*G6 zRCA$;58d3*>@^>!S^{ntQb9%c=bZu%ZvL8@>Z#6DcfOW%tRRAaN%;?2QHM z_|O{KWN=>`-8^j=!U>`}apNvvJR-^9gFDLHzqtAIr5rgX|2sqU(L21mQrCU-pp*tm z>*y_l|Lo$^Gqk^; zW3EZ>!km91B}TK>R4zZjKz0Pr=B}lYh#=5VC_D}QfE|dEkjldH3b!eN! z#&goRy~$ElgJDW<=iMQ51zXsM2!e*ki@dMo2dvLO$*SqRTrD~^wdB_DhyKUYjIwvcBL?sa1VUU>1kzl>KoA}-9 z@1+y6Sx<4&0fl5{W&N>x(tq~M&vLME= zCZb$9Z{cncp5Kk0*pSwJeuR-iAX+#cN4$JixG3ekn1p{m#|8MZ0>BExUKaZ@ZB*+H zF#O!kglG!9ozO$_B7%Y)7b$5dPza9kh`pzl+oB!pufF~6-Jrv^Ozc0YK7y-l) z{$l<>d2N_1%z=X&bv=){Wrs$cdinU>5Y;HeZ%0yAf->i5A>eNHgQa5K(jVth#K$BV zDrAM;O3n5xV#EhG6@eWd|9zxZ5=DK@(QZmMw6tnk|Fy)?pF`)z@jMb&fxj5K57RhDtI z2Y=bd^6aO{1gSY0;I4u1|AVt{bc)Ah112c-#~YtC#VKg$=67L2Bb#jhA8BtLmF3p; z3#%w4At{Z52uO*9G%5lLh=jt8ASv9WbeD9)HjoAZ6_geUl~7Wt8>BIph4X$B;4DT(#C*bIxDP;}tF*lzOES0q@5ukqGrT0Tp`Iky};1Icy7qeMh}* ziuYRF%Mv`g?+jNVwZMxHP0Q)#P8$@ecb9ZBvK-QWz~6;z7K zk^QxSDhdMr^DWs?*OO}7^CgcsPj(%@&-syfrF8>#k1PMP(?eAml2>3=Tdlsyc_Fe< z0!K3Ia{`H%RZRD)YHZ=S;$HK+Ql%cR-}3n2*xRRTxW6Jo@%rNBJT?fc&2tvNX9 zJ$UdtBs3dt_F_erE6=4@T1F74?ojNNXGQ4uR+GWvW(Zp=5E31>#9jEcxDAM4+W^Vy z_0iVE{x|C!E-R<&PdT1bjUefHl}9P$9%W1Y@l>PS`akD~)!dL}Sr`z2QxVd%KLK^h ze?+8Tmw&9I>A%3}bU)~@{5N2^30LR7b?e6N=uIet6dSqgMBm?uIQ8))MYRBxtLG`~ zK88R?k0N~PJU{qNiCb4@eGWl0o72-nf3Fba(uANfOx_v#2h_b5HDn2zHko^O8IXv+ z+O>P6tYpUIrJ8n|!EwHqoYdaA+DTF<;4)G3Yd~l}Dcwzhgmk3)SaJ|mz*cvm)8M%3 zxzb_s5Mye_-^+1sD58vzTQUk_E^M!ZKhNY!M#l8rplEsO$-FfZ7D{)=0giNswrO5( zYPM5;YrCPOp(@Av+ob&FuK^X+zwZM9)&g!-z{bPt?iM}_%+XBl1Xedrhdj|~{k@H) z+-U#3w~#`|-mp8;kyN};EWIj4$Dcr2>Zw!&NyA@`#vgj}10hL%pN8zJwE0bg1TwXhb*^A)TVk3SGbFV*mB)J%4tqpYKrHey0RJ_64-wtgGgZ#uZRTdS zJ*l5ZeX$_PfIl#N|JcFz{l%2-Ysj3>R=%bTnZ=^qs+Rbji}KEO>_FhSC4evt%dfjT zLrrFfxqZk z!N5}a@Vi8PP4PeR?q~7&8^lE9!qas&o}(z(#yI({4v>s#(3oXdnGW8LM3FDGMJY$d zAPJI}9~KcDq7@x1)5+c@bQD)c9pm+tzotE^Ly2Gel^11m5j|+K;X4$B{W}-qkd;zp z-ltnHgT2Q^tZ(5E4U6X0H|2h0d+7(Pp*u8yx@}OtErv4ubTB<6AL)KYJV4lE3HAp(C zXPs(&RCbEj+a#8TL~@B2HY~$X#DK5&BB6`#;6mG-4g)ecoBS(xX*LW**^Pb8;i@DiD#@-&p>d;1OFwcX6v z`_J+mLK!Y>idG9#P;C|21L^Th|G~^X@&{Vq4UQ+>pvH$|k%gV=RcXfG$D+5D05kB~ z?IizPCf7MX+MIia934FU#0HY*BEyQDxeGSZAH5i9&1A1l)Zny zQI4QQ{1oG^nfL{Hvo)$Ote05f!GbXVM58lWK>r3JJ@N0O>tkr!JA;Sk?dTq}+?>oC zJIg}3x2fyNZ{LU8TdGqg%a7dpr(*rWXhjhs)Cz*ugWpvvauFuJ z&37m}*jM6r79-#JTG6?$&zL&2#5f)tm$m=Ez`V)F>i9BU0~(AcKc{%%K;?{<^YLoN z2vw)3O5)JH>wER9R@}tW@=!z)a*5w@?u?xjq{w!`(pNWzzNxmZDt%F}Q&l{>yk*Ih zi6Io)8^52&(iaZ*Vq{qIBm?{CiY z`I$r}+KheII_+w3fHeo766$Ex$%g;r1$f zlu5C~<0SA=0u6XPa0Hx9U~^#16I@h+r1w1D9T!C>bV!{@deeB6sSN+PS;SMl%l}G` z)%haCmIFyc=Sx-;ugl6pp9cI2e(xOIAAe0KV(uV!eM^DpyVMCC>$W?fbQp5CtZGRs z3!Toqjy#!K1ios1F!25Hh`7UG%O+Vr!;&qZA`c07!%q+HWd7B*80yjcUsG29RWaUf zc1nwnIFKJrCVS`b3$$gF!ZrOQ_niv@9uFQIFQdq;O1bRu{`e3r20|}}zx`;f=Omol zW3|6iBL_q={bySQ7T9t`pz?M@%1-WunLy;zsHS3}Wp{7QXeVI;II-KdrERh z`7p@3TVA_bB_n=q@pEjjg{5B$baj@_RL8+PWlZXjO$5@6L z%qT-_C?~xdA3S)VJ6siHnwcWL_!;Y=h5mHgo9)LPP6vA%Eg9-Llxwp$_4LjnD9!3r zYi-6??qB)5+Mx~0znW~!1+&p{TUY@^^w(hUHDU)bL_4sFEwS+dGX&N%%ImLq-0_e} zxJpV&FBs=UT{q_Mn$*^QvLs)y68bAPw&z-4Pq8&K41CfVg4y0KxJqg|L83&2aI{Sn zZpb)UdowVj&YE~lG0pdK#uxOTjn1-}Hleat?(%{GHH#DC9-<$FpgMutRzbf8Fsvo7%s zjtFm!eBu4|#|Jy*2bJLD_p+xdMLJ~hY0~cTg_4zUkM|3?jeGkW!|9YYHhrq+1RiqvD{`!^bSK*URg1+$+NuCS<8GCO_Y7{+8a8d2QU}$J4$F@Aa zI!^FIkhn;S|L*AY9RY~8q6qFs- zyP{jIQVCVn)kAO65-VV-gup~3wu#HqOd=IqTcAh{)BLfCf8q?jG^5+17sGdg-sAMc zVl|P#5FA8H?r5YnT}*kgCV8+TDdfETa+{8v+;_o!!S!LicV2=^@Qbo5kB6qxk~$;> zq>hGpPlZzRt^Q>MBO|P8=^f9UIdg^hC>-a!`$LV8Dot!;Se?yQiXh>9ycrL?o%_)* zaE?(>P;3qndpbD_Qlqt1E`lX&=(YkI(_AmwB8j<)XnPCeneEGZyD335AjfTtL&$pA z7jANA7ID*Ik@!`ZUy~h#vIvYJ7=MTC!*{w?Lu;8nR)Aak+0t{T7d-ibwiSvo&3OhL z=o`xBDLx51B-B=KvY^$yCMcHBZ}me;SEgrq2zO=;Hui_3F|=_BL%#L#VYHVO_mXx-tc<$X=X%@GmmKG{iUfx_plz?b zHs5oefURtIM(5zW7oH!N9{~8KamVia(>#W z8)^~tVoQ>pnl#Bhlalowv*?R@-uFnqMqzKy4c)%C!ig8tn(uV7a!YaJ@^l2q=q zsw9k(bUYrPT%p09ZMAiYyP#?;ZXbLtb(e;#;9phF&MJmpE(S|DK^C#X0`vQ_m9Iz%J(farz;p$J55Pv9)`+_w-}Buxb;(QJYQe7N<`=ah$)9gn=Zos0 zW>cE+>TFNM-K1@FDhlZk(cQ`~ZnXq}^HpmECm_ht}MkxavXEo2eYg zhe^KCmMTkZs3oI?B$9uaX&`kuL2+g~c)5A9=jjf9_A>2g8LA9eLz(T)JC>^wndt&c&AOf5+IjhE$2vZ9t}rVaZ? z#=S||I_f6(jN%6`W;pS6hRm6Q&t%J|lk7hl4vhuDdp$T47KwygR8cftO0}xiKqnAJ zm5&f`j7NA=NHfeERvCWFd}q>nULfS$<)T-M-^x0lk9dNukZDlK$^<3@LmnWrYtIGQ z=Zov9Px|C2xKZca$q%~(Km2ZC%~P+z{<()2Z%#I_JAQcy99Y{sRDyC!mV-NE>}MI{ z+iOZI?)mJT+gnYOY?s`UEHMf{O75C@ZE3z;UB_ZY!T`HNAPV>Wg})Z>38`JA7j0q+ z)+Vo?3)JDftsQbV?Hp#LLAR;+{Zr!Cm2O=m>g*}+#Kinmb#&c;i?;Bjc1VIN zbFT>cFG|WD&1=t`Cj$%XmUFRYqBg8W?JKv(>Pa282^Jo^vDYcm_(~kZj9I&TjeUvI z&s-P9>If;)=q~$wsbFQc!fenI%ToTv&awKZW2OaJr0e%FXlWh2UR@f8ZJv7HF;ld{ zXuYJCQHP>L=zhIT9{!Ba$lC;0o;=r_p@KybCvmkMo?a@QuX6J8`Rz?2!~OLx1+5Do zNj+CB6!8Zpj~qEzy?OoWoluaq>+BAnjtqjbX+Pp$>=X$l>SpC{-C|nfh?rf4c8gZt zie&%BSnrD4=7#SGYxVI?92XJ1VoU?%Tx)+Bu@N*cCB^q`_e&k@PDn2G*<#)tBxQW9 znQ)W{Vu)k>e0ifRu#ZN=sGdkzbmGkW33OYT^u$DYazNo|+0MLbeX{$;r-8doi#q$u zprxS9Xnwp8DZ)*>Yl&9Ef+9b*4MgO{ju#31jGW(-ZX#AzuNqDw^)O7cwbuk1=$nHYwX)C$zKGArgX<Z6 zJ`xfW*MdO~uk-RKrT%U{Mj`}G-GS{iqseCmntG*0*tHhnMPTtOKk1e~BJ{~>2%&em zJNv)Ai^{Ph@3#KDQufU*Q=}A^UdPU~2lO;z&RlBrPGS+56Md}piV|I?XM??DoG)@^ zFz*LrD~MTT$P%MRbq zkh{#{#&P0EFl){^VrQNnuO>1#>rWhMtT5{WOQ#Hssga)Z^1WH1rEbT$V6-`{!-7Dj zPT^fq11251;T5k0>#Bl-dloC1I?rW6>`NDtS?D~Tv{pZp*CT^s1V991=Fk&kos*|p zj%3GYPyriVg+Dov(AHeRqM#3BmI-Z!b1ZB63 z?*X}X?RF!lRvy`LsastfRgGPjqxXA)*<2`pVxM zPOmCCZZhu!`?#i-(KC}y&knVM^YW+ASl`wOJ2S_M-rl?`)Lvv1QPfl zb$`uV#e^T{Cj3p2=6Y3%krSa?$C4mN&uQTL1YnApCe{i#o7k5by{G8Jtp*xurT0SDo#b z4AY9h4v7_R>3L_VMHss6`OEf@VMP94lXuUQ9YpU2KzlxG8)gYq94@1C0vL8nU>(=W8oP`Zp-?dE&k4H2YBDx z(qRIjk-+>Ju>>{n&Q={cTj?y(8_qf_8BgDHUst%-9%T>p@I(TA07Z@;a7jW!Us=rx zWug+nQKHJN3-e579A~q&VL_jR>DC2MNW7cEt)}IyeR*eW+Fie#;gh ztzErR&5Jp83`QJyYd-uSy%%i}Q8eJ8*?5Twie#m@Y+B+W#9%L7IkoKo4^6WN5xY}T zV5M1#n4QFnxJm0{{izg!$+nrzwoYje7yh-FC*oO z+QyUw4g_Gye^9bloGcaMs;GR5Nvm)XQhv3AiNupN5e))n9fO6v4^oOQ>t6?%b@P^&O&krJ&TbNcz#k#4o@&H4!iTZG8-%KF)k3 z(IU+Kas5@B=jWy6F=I2pE8F}*kv}g&DiR<>+5ANzji8oPBzQ9hR9&sZ5+`OkLMXXZ zlCoY(ji9P_SavdtHVYeyDHAbE7~B=HZ1E9ca%NQLuG!i>p+r8y@RqXIUq&>r>5@LY zk7##Wr1Owm`Lc_;bHJrYX zseKY-plPV>#8vj%fDThwcWEi~vYA4_n%S(i@tBJf(M%MEMlHurM)_&|4z*rlZ2`JQ z*lv_hQms|jSxGDJl0TJ^(2_>BLaUs1Xkm>=p$k`L#QjeWZ%#XHW{%DkuZK=q?dl?O znQ=e&@J>Y7O5It>pV@EbMAR!hQ#~F1)n@ckquW%ld-NLoKJuWVdr>BEP%)_08daha z9p(jgXG8=wZ~C}D*XKp=!#|8(R!)#`7(B8Tn~9ujXIx@MFpwW){k=Vxet9kq-6p={ z7P5vIAf3Mz`%+MLBBC6Im}+bcc$RM~UoT$p=dBN1yyaprB~(F8sihJ^JdkFu1$3u>blddKAd1leZon8-5Bw$(Wkih z_}``ChBWucmnM(!eGNUGmoD{9vjYx_z8oA8zY?g^-tw-1w!p=BW)XEeFEE^}F!Ir( zBP`q>gOkYl89X-i2pGgr#Mal+2+_;p2fHh#ee1p>EKwwD0vwVkjhB2a zxGU8tBhCp3%y-uKJjVB`dh_O}={N00d2ja*gY2<(B__PcDPIPMf&M+R3{i=0)IN%0 zmP;1YXuzGP=P%qv?F(BH+UcBmihq>-acC~Cj!y5*gudw=I6Eub9 zDF~TGZ{@{e*E6%QRNIs`3zg+-qq5<;E2!6W5t%Q)*c!VDCSxT!FFRyot}BC8n3?CHab9gffWOeT<7Cb^O*hvX^h1f>rOT8ndIefxu3-uL_1hQcOVxS; z_2_!D`pt|>C_7zqa-4^jX7l5-s?nD@OsG0YHnhVVYb?>8l}U$S;%z5O)8FT!xYAPX zkqykn_ znD>`ByWm=7p1X-B{hpD2wsV;7%VqqE%V(f>F2@<1)TT;ok5AI)860l%PiToha_A56p9}_oKg6R-3BY2&<(2o3`kS6 zj3$=yBkhdsR;@Bqn&cH|Oa@S589iw&EElUxw|Hs0MY?#Rihxr~`=8zp^N!gH^bGEr9c{7Qx3 zgqcXqlKG4q(aO&lRzD%1VP9#BD!0pqU!v)~pp$vALM*}z<3v=R6s|p9Yf8gOEK-&g zde%QLAs1H9A5m<#-koe~w+YRMF=(J_)pLHQ1?y#{LZWtN352%M*fX7{j%AHk8ZW5T zUVqN)M?7 z84)*zyi3@NPBb%-og)$8_xd5`4DL0wfmlY--#*lp1654<&0bq=s<+v_0v0WtMAkPQ z#L+T~I7f#V7!nl~MNSII`-Q7Y7#E%JbhC-T<1QMVUJSP*3Gr~tm8_z87*UZLhnO{` zq4<}S#oNT(cDnB_Zf%Z4w3kU>Ydx{3@WtCsdv1kSFp=o0C~VdES=?p5Z0F$^6{ux5 ztEq47dNoqth#wmIh#^~Pptd7dE4_-QLX!|uUA!Vpd(Wp{BF|C@mBRcb62ZE31EnL( z0fML>&1t!yb~>bKUz2#15K6iS@fk`@;+flPGifzf!|lnP*B1tIY6UA}qWc!Y_JXIJ zeB9A6#N71eZBHGKCR)0KDJ{M7clfnw873H}*9JHu@dP-DcTtH7sRgHx_V`RJIe(r{ zlvW7XdVz(xupwT5#+k+K`G%hV+_#oo8GrKN!4s6gA*WI$)y$<*)3jj`qp=%BIpN zFd?kWDsDwc^P>dvy5OjSzwtH~}kUjSNID67LnPhxxw6MIEgg>#kfq)0&7p3}te(+}R+$Wi~pc)#Y*A7gAjO{HyP%W(do^ z4fK@V1Vzuq6ST45N9@^&)hKl~kdIS1!|$3f`g99>4R1K#e3$lSQPwNTspgT{vX1zZ zvp(eGANb`QUoz7fMv^@|V+Ur&Pv8eIHq-%-~NoH*(+42IN^Q0WfLY%}MbbE7M zbKnzY!9;Mtw!iUesGEcL#;|vPII3yONyi@iiIZ|~ zHFw8@!Fz$6ejt3Yw){m^N>?7Sae1BF0ZBEZPTy0%YsoVW*!UjtdNL!4DDs;nd#DGZ zgPC;mtj#p$=y-9xiQ1KdD|K%W_~2VBOW?C5KPjLh(n{r+{4MlB z9xWT&`=kL+h_s}n*mYuxOcgiN%Lr=@;T}d4FI;_E`B6%C(xoZMoHFF~YB$gqpR@O1 z`%daBuO>Y@KOx{$sL#-;XR<1qL48UjiasZZ5sJ8r$)AP0lIr z)ZXYDeic@5H9Mud^68Ly@btuhGpG-H&-7W!n+KETA{AcXUndWbR%>+wp0vt113n&@ z*YlNNgQXqfBKq^^iHb-Lg1kW2exNA~{N7ZjJ_2*V9)ytgJl8p=b>Tmo|Nc)RoHje2 z-xUn*4}$LebOX4k>seP=^YB-N>4!ce?tI9K5;_N_n)>hW{=6@0Z4mLyBNC6FKieW; z)W_5Q+O4s_j8h)UJh=JYFmJLJXQ$p_MUY+M!%33cfQq9?LUt1&62cn5`xyzzO%0ti zIJf!W_TB#b+b5-`cLx?UD7k;4(7!+awaX*XA*j_k~Krua?CB^8M8FCIfJ^ zWfKCW3)#eIe1TCfg1lwmo0LDHhLslZr2Mr8r^pVExYDFTiSXj`^71pO#`Y;G*5;l9 zvpj$@DHWxDJ)ggnN);2Uj3#143)+_gLXCIY8H}ADVj5ky!jL_)%4CiE|?YAqxcHq!o5EO!1)L@Ciri9)Mtk}wJL(Kxd z^N9883TB%kFMTG)ew^g;3Qm`QlIOm&p_D%XoWoZa)&JZAp?p!_m!{^#BT;rM`|8U5 zK*>rtLVn#GVg&M+79J#V)HHumW6PUN-4RSKlr!q@X0nWg)drmTNd*E~68c-p%F0e# zV^K>{<+~>Uzg1IY6q)GA)bROWJ8^-P;6&)Rr>q>cxn95GN@_yT|Je7x!a6)mz!rKh z7}d76nTlBC|9hzga+E)saaaWi%yGRmG(}`a((c3=NjolxZ-ATlvbzJFMCY5hPOx72 z{U9*l9j`Lf7Kmn4X!5WcyOCrWd2lrBAY=S9=t*fTsCzl{3Zm zxjxB>6QWf5FWSv`doI*h{qDAqAA^KKPxDomIV+*QI;XxfIVI)2c}HvjTlOe^+~U+9 z$iwROzi(Y-yx>q{6f8|bLQ;4=16nVc#GkH79QN6%4D#CrD|mhNmF47uvm9#dW`m_U ziGNOa(3=#<`YCC-I%yRDvkjhHm0~&XPIB68w)Cb`mhF=6nZIdH6R#yx1Mv6dJY1;# z53CDFLu3wIvFbt|j3;o}jAI4Z<+084f+qQU^P6W+pZd(snzO8)V`b&Q;Yi@LtK!iv zp`sNuSE}W8>v?Y?WHU4m{7>b?Sf2EPLkzAER15 zKPU{xm<0sd%}S_e0+)5&m6UY%Pzdrd?qiwb8e)k-`7bRkb$zg91ykISHLCm?v5Pzo zz3}Q4H4R0B!omvfUN)yCXUWn&8D?P*%z(mdzZx?=*ZiJ-4FI01cwX;olAG{u0Yq1c_54DDE+Ll<9C}6T zAE%tx@z9mXY$MEK-sYKMK6~~{M_0qbxcx=?@U{MM6HGLAuV_xK9IYPpJBiI>cxdzP zDOZJnPJOi4NLRzk9?9Xr6VpE_OVlQJoj0cwCqT%uw;8)0m2_febT-)R?>*g!d)1(m1jSr#zNQf?LD}1AgHj4D}4(I`sG{( zc(I%XTb?nb{x#B)H#Sc1bjo)Cj5U(@&`)TR-++evdr5VXA*W&Jj=g|p=zMuxTD$R=3=AxdaMI=;u ziMIDrnldf$KT+?7dP7T-;|D%I2bTz+ze&i8sev9w0`H>{YWRNTyggKCmK7FKP}Qx~ zndRLp=hr?t)6Mw55K-6+U&g^yJcnD4!Hz}0CCWkEZ*y4zISj|gzk$$117Y8=%Rd;o z?U}||j&HD^uN)-3uxCif{kli<`B*HgfkHm&+3H&cHZ7A-L$tA$T!1CED*3RDG(jr# z*rm04f|u-2H1*BOXNDV?MINx9Z`qWw0S}Rjmi;C5`A^40uYomFao)6Pb?#@7e&AOX)+;40QDs6+Ur4mqZjRhJ=WIkta)sxulMLt#P72_f? z6k{K2pi#lYF79aJjvVAw8GeP5dT=amqCcG1Y?h}XS3^#h$7 znr-4$G`kN)VlykuhPD)HiTrOEp&6;3v##jdA7CfeYG`UbscqV)gW@$eALn@az(eO# zA`~WSyz6**`L#IoZ^;K|MaW`s0AsT=ho`RCer3=;?ZM&;UHe?OtcpUpFJHM=NxDoj zIB`S<=APl`{b;tfz=8F=6x#Wj+RARY*K6#X89-^~X|tjZgByH=pIlY)wB|ykot>Rc zDZ>-2eRChqg`?q}nk@KE*mAyW9Vg`dZb~~O?tP8KPe^nw6JBSHJkNgM|E2=AH)PAs z_LK!}RVPNRc!DDjXIs&;jkq$(dT92$rkdRSSU(7|w6ea!tQb)PbAp(%Ye3 z-87}^@=??&_m9$x+LcisJ4tWc_cV|%h#-Kl`UN$gk#6>5kyK(ZRO%HH(C9!LSu- zi4>kT@nsx4nd29JUR06i8e(7M5btz|pyM!LMm(tcHmVdQPyr>BqT$M^VzWYvD8|gw zBB-nm!rl}0Z2)pA+H&8mx;!8GZ<2*zpUaCW03?N?QDyZcYd1`h*g7@u9f|Bk-06No7a?B4>Bl0zVJE&97_91wB#*YID)KNl}G z|MF^rAaAnbIFhP~(L33#Ha~7c+gIY%l-r?fc#avwP^`(V1DltTqc7Y7s`G>>{x&9V{?{8nRFHi-Q%l)&K?0J*rh2s8LC(y`k zg2g_<9+%^+>lTnDhD`WlEVkQn6%h+1A;Jv|@3tM*S~({%K_2WaP*2KR!13RgptqqC z=8ijqyC0U^`Tn2gm=0rlwgXKZ%~D$P1_#?zgwls|G{sn;`o%;HS~89BC-FXJ^&SJ5;R+`H*%-s-2w?v z`R|f-^3v{_R~_S~dX;e+>Sg!aOlE}4+(VpOj2Dr<7YxG(d@E9~0+%1t;A&A$8e*L0 zm{6D9W7FHaGC3Eq9Iam;}!sGI|A$q5{anxN)zsN$lwzEV^r$v(i zWbD>EICSOJAWP$>|9`*y7hZWJuY&TI$Kw_3mEhFZq`(7zU6qo0+x zA?!ZNM;?+I-U<1Nz`Vu?y=Po`JYa_}Eh};24$CCTf4v@{Ui4Ox`S`PS`%jXd47Q$= zbxX?MT@dM+WxU3V0Q@o)_1FNioB{7eiOa)X@&nSh5@#)2>AQ*+fe0n^$#}%~1`zRM zK5CsA&Ol_~D_3s4WjJeh{=aKfETNH+RmmBDMifNo5&it#+&k3j=o=GcJqVmuwc3q5 zP<5qDnB5?2qnH5kt0Y;;B_c2pxuf4ww;_eF{NFx#w8$f}!6~v$_(D)V;*<#Z3lAbA zKcFGPOZNmq25-;XpFI+S98lsq{t7LKSi)PE{&&0LzCVPM@UfVH7K0Tmxr;7?T|$V~ zpl+2tkr4^BnFcW zJDz^2G9D#%IKLzna1Y!0pWZX}z;2~|h2zDX1oGB24?JMRH&6Y(3!)hxA~DgXyT}Z_ zzQa=m3{C#&9Q|tL5rJl}hB-nkg;QMoJ~74dFO2{BjfAwHgfeI{zz5*M>3@{^ZQJLd zKw;_I{XnN>d5~Pv>r0jp-y#I3+d<5ODoU8yA-JF1*h6gK+LUMZyn+iHkl1d5=n?~> z%ZThp;R)bOu?>{aF$4+P0#lY0ED1ru4mAnH5F|gwq^(nPAvKd&WO4GZ_xCPm>Wo)r zqwD}0%DBdvZ0Y+)D5c2x1U-VG9Wl2T8=$ zMcF?)wbz!;UM3R>FJ4w~GjintVV~=OJX;2^V5aY#mwZ4`vEBD5SG)79o*W~r9Y^lh z&k#!yCHWRm&dS_TM;$1;-m>NW4DmyMz7xx@=~2^J5mf>lWe0~soA1kfmF~PEZztKn zFpzb;DIr5*CpO$#jDFKCB+>cZ6Kr@t3bnrwbvxtlSL@5wzQ4#5i#aV)mZ^1;{xLK> zOesTuLh3nwP5K!oi#yI-V17{!*;Jjf8I(EaODKbnsx4ogxY+9g>{pkaZaEtM#9Am* z0WOjF7N#w-buu{Cg4g8ez?%hj)pbDv!=*@Y1jMg8e9pn+#HH z-GQH=RPX8fs_9by397jt)6iH~xOwx77Z%PP#=@Sa%_w`b0w{hu0)W{UHeqc(fqJl} zU0I%~1F**o^QB8Fcrx(#)KjR^jb$gA`lt3Bb__7yZNo5+4NAnrrTg1j3u-ra>P<@msZ=+ahKS^vW9tA1P0q z&MX&pyY*CMCLR0Fs5TUeni#lbg8>U zmbHsJggA-lcnt&hW@;0mbEV+HOTA`RkhQ!G$TQ`uSFfNzjU5sz>QU3chBR{Qs==NL zZmud6r*@w;0X3{n$<4i}*;!%BzKWs{G~p!f$cpqpQMY=UXLnW{?0hFK z0*=h+=mYr_>@btO1Da#8MFvQrx_t4tG*YB_NY+=G9=xK4&(qalV@DC9HB)oiKp1lJ zRh>d*DX|sA!}o_+k^x?Q4f4#b09oD3d{2@ZMlbF{j&6!qp3>k_f87eWb8NHop*#&I zyxijN-jF|e+x-eIoB>GB?M1f-jhBQGVPB{~Dhc>TnPO5x+ZjRSQ?9D4lKMOenOEh% z_mJf~rY=H7{$*J!P`Qa(;cIw%lQg1e0u}k08%{GrVMG0~I^=W)<|he2-0p?7GlS*Q z9Cc2$mBZ2j)QSdLg+bw$5BmkE#1cwwLIlDeaSjR!3JeNT-palAT1zC{d5Ct+uA=BU zQF3$bGT+5vs;1?^?^m)XHuW=!J5JIEsm;`&;LqGZ{_>Ee626sPxQl2_CNCK4{{YtO zVb`EMQ~l>v%8|Cgsls;EG{x>|M}CpA7{+0QHXy9@HNR)UxXP-+6)I3)i^j>@GE^(I z)?QAD26UDCrmJL~hLa`q1BoQzjCP5=V6k)mpE4BT0w6_b9G)}-e%x$jRj>72%pvg9 zE#tf})-?;9fh9L6LVLxa>pHfEGGX>zyP6zW@w(kX#z2JLdcm+1Hvv(iH@gqIfBcm) zgO1BomfpktK{RQ|{y?E@9>>Bn(AHIK`C+wXra&&hgs$a8BHOeX&Be7h35dY+bp7K~ z&$DWAC9LO9q$!bSYIKl8VsAdFm26vM)qKy(GDLEJbEI$s=D0n%=hayOjSTw9zg~_s zG?n4`FJ7ySp+I5Zym>P)BqTqQ7;r}tv85_9WEv{NFulk&DuR4((usd?W96P!rJJ9< zL5W4vf#w&h<{pr+qCdQ6gcc{_W`ruGOwFPpGRw$yU-|%j_^n z^;x74tG=z>q%r0+MtmbaE`2@>T#JTVWErc2DL8UH4?F}$Dv3C6KhJ2lj-(n+r|%Ms z_D`@8=nz^t%^1aJ(}kjdN>tp2z8-vG5d(0s*Ar{*6$*yvTe3r479NIM>w zrAz&;3wo2y$#wlMGh8K58bRa!nyKkqi#%g62L#!xhZnSxRf*KNTUG~sh=!f zPyv-rneQLilRtjKVcK8+SZi)8I-3&gDs|O3Q-nl3_ubkjx=sfW(mH%#B%;1_-CfXV z4fW+N=e>L*LJy!9XNU(jv{BT9xM7!dzWl(W*P;NW{br-Hw- zQy6)cmzU?X{EhXlGh)KGId$7J*wSr;_HaV)ay?H5^_7FYfdjH{*PH(5s`}SkS(rTrQz99`5;Vkd*ibDrhrgq95Jq>N>few7*QgWE zl0w=HeCI#cEIn4tn*;|fcYn^-Nxm?F&r6IVekrFK>nhf8lk=y3;;T#NDO-ZvXR?a) zf#5LBH?j}RNCS{aOy?g(pSgbhdit%Yxj<=**h;9)BR1V@`j%LMtsCUAo}YA|Qj&;KA{2&H-<WheCaByHBg4J~?D4?(da!nP3T zpB3!Jr%650Z$^}03h_cDFm+P)&YforBfevsFdUB&43$-q4#6-c$K9Kp!=$z#vl>{v zt)xE};jzlxUPw^9OSh9jk1Fc(oRrtuR>>f016*nZnA>;up2@ehbkEnly_?aN$%?N9 zI>W)At1(_9? zVe7;N@GXepMfcB9j*X4o*VYcNwY|-QiIJ4q>+#wP=i87t*uoS|j%4mwkJJot=xhbY zwpVrg7X5Ib1wu&<-y{$guyPhx7zBGzVH(coxgYh+I{*>;I@pJf3V{Y0v#@fzvEI0{#pfyg|rEb>6u(P7lel|aR$EJXMT|rIgMz)dTxtBp| zxDU$448yd^U4wntF65pIeU zKOaOTv^jpcJjiwagDh9?CPY!$#u9_QE-#A*U?#fRzfqRflf}x(F(1aS`x&~o^x9G= zR5Zf?K z3z7K(Mq(#3mU=YHLq$d9*^3#GxUUr2Sk);A$|t$`g_yI*75)UWSANSjKnZYIXMd#m zquiHKlgxA2b4X^CPWiCq1_irJ^EMmhjVI&51!4E(A6L+ar zlEO>}Qih=O9)HeN@tIlpEHwg_56BXefAsztJ)XdLx4u3tw>dK%RU%W%#S+M`4NJ~< z&ggB?NqT4nDS%G;?oO zG8_OSIr=S#;A1-|k2B%ujGu*{XE!KYN6Fk)TV!zw3ELyA0u21n^tq*yYi+8g-()R2 z-#f1uA@U$4!wEv<-5`SC#znHc8&yn}Klcj=ygUA^?2&%%tM_H3lHy`F<}_BP=1Q${ z2Q$G`zaD= z&*tR!MAY@P70-hWPv&#yt`nPl1NYm-{hfsj!)cT(wpQ#sOt{aF^&=-E^Fb7Aa$VMj z-(TJ|%^?p~v5Ws8a*mzb9khyf;x>z<%RTn2;8?mK%h!sqk$x_bKz*W}ClgDVr`Lz1 zLnz-g_VQi2)K)Wx9LhhoIm;u4v}7)Uw4V`!E?S1?e7VSeW$TV^*fAf@%B|7MQ&R*i0V}vO$;y0v$B#{GIJkOcm@55!(0<0 zud+93dF6$_(-O%U^{SL?*`LNlCAEcyhf`7xO(wI&;4lVf2f-gc>=Q-cx8-W(nX(xq zj`YzEGd3A>Br~acOQI*oiCFCjm53=4Q}p=lkR|M?1LjyR1J+N!_7v%JRw&;FLDzP{ z%c$-dd)?4Y5+ji3UudG;&9VcJfHCv^q=PL+5RE)Kh5f zPeWC?-$UF0fOoFHJo?Q=fzOWHw$+onReE#p??Z`+QhD+N_&Gj~jD#^Q+mIH%T_0Wc zU8^Xe%FXVIVsM@1ZI8lRiHV9z`=0WGjBa*cg#t4)dj# zRq}%ovRvKM+lVufer>2KG$!_;7i`bS0Kev7edlN`7=lWlJs>U~?Q7PVq=BI9C=et~ zK{1y8_U$Mu`o#&6I=!(Al#jt+0uo!TOAlu{vMfY4P}p|*w$z-K5NksDm=3)LpcGIy zocN|`SqhFCOA&R-vm&-^OMSv(+g%2mupBbz2K#-n@o&POK1jRHI{tKEfW9i_y9!B& z*oWs;OwTvKh#?>?$%95U?N;^H$~Tu#QL&~|A;qPHImi|RuA)-{hOo;)E} z7xK4y^9lYRbz%$uV+$Y%b1Z0h0J9`OJO#qnB}f?tZWS+6{2kAAs!}qretJ~>*)K&L zm-5Bl}xx*2$5e_WfmDoc{U=Rl|CmS2WYM;D$|)=kQV5k5-Q zu79wjc(Ff`W}2Rji@oyIB`(5%wH*pSieS=2`TVV8^{uL$%O?`S6|Da)>cQuQK~5HK z=?T1zb17u5Fw(c?#kRfWqp5LM7MQr@~sMi^vr7krw+{bkmQ(sQX;73pO4^t?QL z1qgT-mm@cu)$r;cS^S9mI&t!sU}`e@)hXQq?XQs}A@AR#UWd&FjDI(D-h789$*(UQ zGB#E@xSXDuX*`rOxUWmDB-~pYSy;tX_!N#JfjARL?p;EKi_KZk4Aum>Qu(NtI=ZQ9 zf^W4Ni>M1#@o5oGE7mH>5&tOkr}2HWQ0X4tsM;g8 z8w}dOxSR(%p?rEqXYcv?WgATk4dQ-XX!f*y5ChT1WJ_wmyI1mH1__Qa%)fbw9|=6& zUo5j$AUWAgWX~<-xEpY#bnWu%6K2iA*U_}}4R%dd8Z3XSlt2nBl_cT* zO!X~e(F2Kr^zZn+zt8!?k(Wx^g$y%+JB|Kybxk6e_hxz-2PaRNvZ@ys85w!KzxC>J;Q5ODMOljb6OB|I z-eXLBn6D|XcUZ1|y^!P5<{XOlpEl{i#M-n+RctJH@f?)pwf{R&yZjgs-rxanZoHzr zyu0i4%7kG`{*G-vRyjZ#&)qXcUb4tJ0#Zb$qO?ZU@j}KK;C39x~gwcN^$T04R5FxqtOIf4$~U%%guE(g-Kr&FPz? z(~lmT2NX)6u5?_~m;r2`3kSab-Ay!G7@(+#R;UN5L@Qy!ufO`Z<-7X4KsaJpNUEY~ zmY(_Re`GRo%YXj{B2s$*Q=jDVUNtF))P65P9%fqYtyhHk+oh%_4CTh?Z|d&{xmpSi&ylwglf3R=#0ZP1E{j>-M#6 zjaS|wV$m^bI#%=k^JxJ<+vn1n@@`?SbQ2E~Bc0V4EC(ssk0yc6Rgm2iJ!QWT*S?zX4ZN zPK^mw@s$wRcK-yqS)gviS=HD34WxnVce4F4pT6F0kgg-@0! zjdNj)+(8K}yH0tFT}!@MQs6kpSZZ|?;?q>U9pAq=)0pI~AelHIv+Md>R$AITISQO5 z(Cs_jDnT{qI`@YOD==PTCLpo}Q+n6&p9P2NY{LbusqSz}At)xLLEy7g-g}~%k)=Bm&J~kjYo*E7K2GghFHLn@8KTS!^9Z?Dm|#7^!?f^VSYB;d zI%S2OaKyvmeO!YF>+arOy}w@M-ir4<|3~y2G&$|F@$JtyKCtuQy0Qd8HE!L4Bq4>< zbd4(6hwLi~+`9z_-_+({89jz(Ly<2_C$@}J`8)F~4Zty0yZvIW4KhG+Dh#YeEcr$% zB>AG@>y3>Ma0KA0#ar}Jc&tw@fT0LvTy_q`f-zo=6(Z1mgOaycTlSCa@ z!CEQepa6YYceyFxAK~NrHpl;d(*2pb$sL|wTFunJv|7@cKjOX(9nBB?=j6M|EPNlk z;nV-su?_IaoP+j{nTZxSetqDtgeF@>YQRC%LH1UZ((3$RB`Y#gJu9g*S)>e7j727v)V3V#TWmz!)mL9 zSaTqV*4DcYT@OD=CPYBh^ON8K%f0L-GF%T{s*n8oj(x&36UveotXC7NDdHG616^aM zv^_pyOA{AklE!u}hJ@r88BdmaIF}GzA&BDgE2+Qk=DCeOpWqKCBbm=DD0bR9nNwuK zLRrcYKb>^pXQ2~Wl!_2d1RRPg>=1g*j0ZS=O0rUKj;8(*6cbYbzS}=qAV0=`b6Ri% zxM{vVCEY)^2Gt>#NfjdalLV|*YQgzmwF%a=O2kiNUDSGzLDZnKW*CF(MmAj70(u73 z@~~3R#`2dVq%<|Tzn>4?HW69lEH0Cx?7%gZ)CPyjDEQ|XLze!;*}Iw3$GnY?w^=UL z;gAx@^rEh~dSJVm_PHFUnJ%PsTaIbzujMBDzbXXqv#M}p+!1g(s^y>726pj{=~Z>= zwWIKq25^q-5S_4ux-DSarq*ls29Qd7;0|30X6arxy3FNX0OlCFZ7y#av>COkj3CIc zo|97Zojb1LcV7*qLvRw6B))iL|F+`-fdugTWKy+4;-CP%mTn;Os74Ctz6%QrJbLnk z&Xhh#;-UXbwC4-9And9RvDN#Ed1}vN6EG#{APe+o*nV^7%UjVsExUQp&vmQ1K<_69 z;KTGn*-S5&vwIUT|9Kcg+>LKP3yrEljs@;^@)NFywG3pc^t#7%i*iW1L#uxc(BNN< z&VF}Zqk+#8qpH};ZsRSWKV8>bCEE8bwpo{BA&9%llB%3DC%4OuoE^$ZhsPfh5kiDB zHO8C%;olZuR-mCB!_vua@)4n;o%7d&t9#;|4Q#g*jP3pxW&WQMO7Dfe~7g78*=|VeQ!+F<(}+X?YD`(AX@r*JCy^D*Q&cO7QgV* z6roFQl38AlxGZJ7x>!rL;EDteWxJ~kiFd#GhlWp72GYd!DmnMdM-}79h{^`!6Vx3E zGm-dvI+v6s0f07NlA7n}>Z8RvcRFsV-SJ%}GxlJe%<<3%9hnTa( z%oKDy2OH2KVX=yDzNT0yZk({v-L;fbt@@{3_2o5x7v~`Lxv4X^b~XzplDv|~7Sd`| ztX&Ycm-)R3{gx*3O;zWdb5bB7fw&J zTZ~{)4(&ok9n;BU`K})~)eL`)#wLJj-uP3;G55c4RO9N82J7e>#ve=FWLRfRNHzb| zaWn3hiOGYz9@;M#pY;fm=I&Ky`H_`$)ajVc4{P)kD~mYfD$gMUbm6Bi67Ng$xWs8* zn^(b<^!?~F>hga|$ULN|s7JOeSrY$pp8a+N%GjDZjm_@|T?>;y%ZjAFQa<^H4QMxx z^~XbMGqiq@Je)7AkwcRO=i zKZ*JjJhyuV*jUDP&O=we_#!VU1tq)Ew>O(bxib2R-nAI(j@{hX%evL{cG5@N7VtMR z5&0?6erTUpKYCaTM4n0^$u?33rgVGZ8mc$e595>!U$Ba7o?p^*v2iZv9V1g?yy`lD zG58YO-P`4Ca>m4jlNj}N&pqhkJGq4C+NJ`kuh$IJPb%Aw9fF-Q5oFgZh%cjRLMKgm z=~&CQC^Az3YVs$y-;OCuda6(?3quxB0nD2G(61c*z0Rs5f$@T{Ax zk`fz zKU5=3-s=6o49n6{?%7k>WtRUml(&FH-2Z^hPR6{h&2;gRe2Tgq;@gK+m_0$EaXe_1_G0XQ-zWum3lXENpDcV30+)fhi} zzFxodD}d{!MMw9!vzkEO!E7dL3QWSL#wT# zA`Xe5^=o9id&$Co71VC~SF3*ah4AJIOEEMcG|QOz;Ljn2ahesO1I4 z?|hTE8ux|X*<^>BDY*Mv+3)u(KR-IWSgEuud5XUsZj3KuY=w@9RAR|%Ie>TcHST95 z#hH#OkSKB)me0YkKAOaGxhI89?YXNA+K7>9HbQYrUf;mtQ+dDHE2iG>?Vq(Sdkxf? zJn3=%`Eb@rQ@-&b17qeFpS)Ya;ytf-3hmMktZrciWIn{&ler$he7fpdTUsW=3eAJ- zG;H2BfY9Fsxic~5uxu#!m5CWuzUnAnS)3NTqiI@F|Kf0II)FHgeT{Eq0Hjaw^t97? zqY$CfnvBplGJmhAk(H0GI-CD~jWINq_T(2~V4Tl*U#6|y)t<-WWvhM16)bbiN;$0V zNv2$pt7O;tp61t60AYNbee5ctTs6LBEb#5k_-m0Y`TXNxa!$`~;^?;V*=U-t3hL6~ zw%y6DaV=-X2M%_0isTe!3t{?lF_}@-}FzA5&_*_xN6ep=1U{AfIm_W{0p= zzYJ;>ZS~P?$xu~z_Gj%6FX=|pvq%Zs9=4fDTivCjdm@#g^a#eA-%dAKCX5h*ABtKT zGzHB^za_hLU>nR@u6rc)Og^Tehlui^?^HPEw(?$j`ic-#rER?WM-Fg*G&}Em3;(Re z(${(Fb>HLm-9e2oNV)0iE5vH`akp%7bmrh;~ zxaaGCy3+c(UMXlKq+?<;z5soQ<$^|?d=on2(kPwC(GWh z)^9%L{9KoAY^&(PM?&({HUzE*Mf9rh8!2EWezKXFOGoV0MyP2BDW*m%`5CJ5`d7ZW zi(tI*ZQtUfkr?DRs`w5RL@V7j^dWIF_K9(Gj7d_a*X?(?OkF1e9^LCrC0}wZ@{86! zH0r{V!P6Gh!{L`to@sUL|2%oVQlP>#lN=rGt^H1)XM2`T6!papYzORB-uGYrQ(sHE zsL7X;R;KEaI97t-iHM^%`-rtay+8dv=OiER64AZ`_ChFJc;WNkofI&5R< zpJw(x?u2IU;Vz9UW0k;w51)>C7Sv84)Od(n>fl#^j(OTY&Ot&c+J){EUIg@o%Cxs$ zC}tyAH>U0&(qBw#{tKE0&n+NnrE-DZkb|S6DOF4clXI+92~!dUe&MCV8)|+-BTIj= z$tm%2j=aXGTg+{2s4*Q=qw9s&I%c6E0)cn|?s~X2w^_0-H1N zu*11UO5txhYcsQ_wz@7)nNT1XyWe}IlC?J+I3h~H&MRh|?vW#tIm|gP?53F@_rNS5 zAhhP%%bK|VC2IgZ5nxiWwaQRkpsSv}aSOUvwobU*!q@*x(F>sVmZvLWI>K&( zk|l4ZEaqBhaCMEMQtTF)-_eocNu|y9M9364a!m1i!i7J-Vh_DCg#*AcHXPM`1xjR& zGBh%J3eadbP$o&C3~@sr7EMm_y*rJk3<@kHgEErd`Ur_o0dy=_o{lK$(?BGbtAOOf z`;~bPHerkb(bsm2*38Q4U4QwyST@!$nRII|v)GYmh@R@*pHP=lZ2O&c#zrSy`t^;f z!<7si!`zW0z&YCcnmEEHCpX*>A~NyaE2PXvY#@)znX|9ED4JWJgirFJiL3JXQM+y` z(&avRK0Mdep=Ces~$JfwLVjlTnY~|Dz9X=nsBeKPCxu-exZKq{vSxaGZ#b>aK_ct#0OrzqF zZ(exw7h&&NUcP*<{uDP0FRi`b#ZhE7X99LAWbe!`EcE#lEmd_k5+%Y!a#owII&OOq z8m*^eV3EH&&M)fVOqR3GJ5@lQG;L~%dHcaU@%b825wIemdoWD>{r8h^GzJDn!ZFo&%*_%tbN4ZBk;DM!GN2_2un7xXcla=QWqJ%+RNP$uaoYSW2!Ejrv5sqMq2UU5I}U1x}nqAT2BKH&uC=r96rR<9A76iF13Q7O|_*G;dczno=se2nhE~y%99c$ zdnU%x`TnrM<`|Fi3Y=55_~gATK=^Cg8I@xYZWyNB#m+8f$*0=Yl3rW*SEf_R=`qP7 z!K&IUP(drxA+g%jD&$J^Fvnwgxx2bI4}Sj6v23KH{SdwXQ4FR^Kk&#V-seDl^pS|A z#Jd|&inn=-J6{a%M=wPD;()Qqz7`qlh<81bjoMQV%KEgE;USz-oor*<>5?@bdtv;O zmptV&XeLP5Fw*QOmrT23H@#<~?lmJ#wo;fsmrf1=EhD4Bqhg^Z6+L}*LzjuRTn^RK zlP~z}Mj4xpiFnY>;=42NM8_2qd`0fkuQML@&uXE%fk++)C+3bp&Uue9Ze-#N&}H%oe>W=U{K0`x>pz7lx?FU%sNkg=HE% zF}pHV+(YEC^PTxm-8rq(@{`qLt;7UNHR7j^QNif3X^+Q9Vn-eY9x?J}4(U{R^2tuc zQtoz{&-|<7tw;7xmVXYG-O1AK?TMg9kJLMIIjD@|R+^gF?+FuSJKbkKhgN94?v_Rg z?p0|@p!RC8BiVk#VLyg;GGljLdUSF=^v~K;mw<^1(l=8@ct(_6jKuDG$VN%pwF@40 zcruZ$l|R9nM=bvG3o$E1ea3z^MwdL|+Ar)|a#Xl9HWe;C)S73DDQ&Hs{?Yi&p5!i^tUb z0uIvj&)w?}l8grwn?bTuh*EIHvk@Jbe&lhU*o=?hiw84*1)uI3*JpP zTJGXt62L~L3aXb22@z+)(e?ZPFl}>KSu$iU%yl&tr^l$OP|AIrX!77vmk-gYmdXO4>1Nvd|0{UEgX*0c; z=uzO>ftQd2{1woG6ysK-yP_%ssVQw;S65zR-p;HG$P15L?nq(xJY~BEB;h6BA1I?Z z`7J@CDOXje4S^*e$pW5^F zrI}=4AKcWAL0O6$Six@(#o0W`@BpAzj)=ch#p@J$?|uA&WGpFW__1`sm0qawQfqum zYDAP|{ecc5>=8eI5Y5Al;cXyUS=X^2;#I&aYwm_y7Jd zx>i%=de-G!krPBQ>b6!-Wg;xD#+T3#=EaNEoPi=dw7PLGmue5PXUxuqyUKs=)wtR} z!p51p%!tKVzDwLIDR^@zA+p6v9%V!W7^TWqSRC%T#?aW_7thq5Chw`%H8Z_qcygNn zJGyl9>2T+mxcPwx65!!z_hcEDxaJP+hi71?ZVYyQHGcMZaY*S_*1gL0!1+cU-W;Zx zd<}Ctd`FTBFCCIa*g7kcenh+D*@%lDY2Ei#EMpw*TLYpsf2txSyl0DY;sMw(mS6(+ zq2@-mZa+A`f3N8tq3rf8d}id)Qd~Jf5Irdm;et7YaHeR+cSc449?zhSl~=ZNjT9y) z^H(O@@C`>Mf4W~)27-%IB|OBe;Z0`JnfVBKtwf#g&Hk8dxQIrK5(9NF(~t3qc?*in z`mlV`=M2pch#l|OKJu6oHNR2RryOY|sBY?sA5XV`e)r^MpNx+BW=cA=C66#9YodS9 z+_agY=k3u_M;7E`57-X!J#FrmGa;-9x6J4xc-A#DEf&H=W_)+5`)ydS>0bcOrPz_W zQ7=MFdqm8sTc?-sZ;2{CeIgnj_lr!4=9jbcD?N?N*ukaEB6i%Fh#unIK#-Fh=$cK} zcU2SVSHhy*V7u=%rY5o$6>u6qmwBT$VUW6$7`Cwx?moy}nftI9fr1YUsxl2Ecec{Z6aX^6TNGSfJJNZhXyN!OS}Sk6`?0 z`-x`9SL3IT7KskGCbrVmi?sJ6o8fhr(`3f@0#9!0#iHxH-q`TC&i!HwR9jYRp5$x0zHO2sY;CP@!BhG4?ZN=iIJCkd)lF4sTd$ z!KI0sNE4r9`6LRmOn97fb>Pa_VRZX61UkMpq}XU(Tu++| zmvcNM|2~mQV+$b11%YQboux**{fP8SvmNh*AB!~lG95CyYV-d3YS!oL?`k()A8^3- z2kmEODYe(>ZIYMVlR;@z-wQLfo2h21)kx=d;;Rz%(Co-~Y1R~H3ib0r984KCxf1P$ zTE@#Y8eq{cjg+rv@H-SFOttK~hHFghWof$aJ3)w??){n_CojBVi zpu89k!s-kN9>m5+l_$R+VTYuPcE>E*1~Z+k3MRf94;1GZ3`5AssK2Dd&OkY~D_bnu z$Zra?=fCqwR#~gi?KB?^XORmE(m<#C4;Cwx)P@dQxt#+J%{4en!gBD zlXfX5M(3#%u-!WEAZxveS3OsG(#M_ zFzNhHf>?3f#PM+O@Q7Ku81dp3;bp;nNdh|{J`r^u<(apRHe@Gx`6Y?b z+{um>{qW3Y?(dLUQ|&OmIw0<{dI`u@^d!$e*F@S2<==MvN?j3U?XnhhGAxP*b%x{mueY}5ON?LW*WKp_I`9ssi`rx{xEU&l4@TQ60{Y~h&qTZ$t~ ziGY%a3cwnNTMlUv*17=HsN9!+2IN;}&$RvJ1=1Ye{>u_u!~Xyv?<*h<&-E3c!27L_ z`|!&h0KMJ%)m$gcQ5cYqPGIeovc9I?R-O0lg?qh}8Idj5VR^5&^{BJ7U%q@pLpCO4 zj`&J0`3WlXF7-qX+BNP}@I;t4f|ymnE2=Nee^W`)fpf58tnxv+%XJm$a7!x zu*POW2eb+ZN_aY>$&p^#tM+7++8r&c-((Sjx!snm?O*KU`jrYd` zGBlm9*up0{EhhRos!iYtS8Rj2=mEF7u;Io3hEu;SA6+f&cZ^7?r2~^>E*sx zl?FmYc0jRVu<=e|?$8TYU+#AQ2sV4rEiG29@8xUv(*)fF`~_~mba55Z`2lt(9CMQ+ z-g`+sPd1qt9(=rY{h`leTJ@5g@B8;bSspphoc$7qWgOTJEy`0$u32bkffKCZDX?T> zM5DdMCr!Nz0hIye9~jwQ^S?KC>+2loeYV+Pgfi0vqxmB<)a!H_8l!Jztv58ErR3Rk zC3=p2&Sb2Zbcj3LeO}4S=s^3+bGcWGx+n`H7c9{B6m5}NkJL@*D4MU+O|z^d6exI9 z81`xiD45m3tLj?kb_ZIP4IT*>QLm|TsAyyP1XZ$_A>4*w+MzR)4w$t*F{b0u1Tn;^^XDO%TvN5L9v@_D`W{s zk7s(0luE9P5`_`W=-Inw@H^dLB-`r(?1eV?Z2A4vQcH&a{Ei}*iO^A+8NWcF(*y6i zJw(4>-0fn|iK_&hzm4|b#fJ?5hzL&@<=Q4Eq!wfs+;}EB;d6>C) zcZutgivN3mYa2$%+_+Q4jV}$Qth-pKl8xF8GIH0JloFeunN&|XK}K4ie%;c!++DG~ zFp|{nA^-T;c=xj^SaT?>U!R8C8@}HHw)60&)?%4`zaFz<&V04SbiBwy>zQ%ak|yut z({NKS3bK~p*(uq2flXWbBZZF-oO!=$J$13MTE`UboHiEV0`Dci_eCb!VN@vSVv;UK z=7Yb_cuhK8yr)*XCFt?!-N(lYyOAlc1pYi&q&VVv;p|RE$H-_mKT)};L#?5?bO;}~|A!LG+9BXJIgcLpO;R?Yc`HXWcOzl7N zWsbQ#nDc4z;|-KDcB=3u2wP(g$mLQl;8q{9ClUt`iPCaI-HEjb` zg$y%&2X6mLXaaxFsNPuLX!5b1GCa#%W%z61k)e%Xaa2`xI!Up|VyM zB^~b4x&{?@TSNmG0|wb1V5fQY&I^K8*v z(z!t#=~i2XN2XHI5P#&zqKHs&G5pqS+x=jlmHo||m1o23K!06$5Sz4xBKzJ}3(553 z)H&N^^*&*72sYmm+fX^MIXH;(H*fUJL!a&rAnkhi!Im``cT`9-RKn-IY}ob{e52e9 z2&0~^v`!AVj@2;oXW6D^%g*P#d7THbXy9u@CeEP(lt_8?V{q}~)*dXxJpUqodfqYk znMF=T<~DwwtEm-U@b~7vsIUUDg7Wuvs6!=U!N-DnVghuOSXT>yqecA6t4w(m`DFV) ztUp=z^?UNo4Pz5DvowzAIWeIu$Z(}r}e+6P8exd zB6m|B*gv&io2juu6LHC5v1L>I)s_wOI6T#GKG$2hFKzSj#=qB#E#Z7y5;$CLQ}2O$ z!jWDU6v9{lep?j`TwPW;0CJW<%pT8@&TG!Kj&L4HCiDqv<2EX+Mg&yj?kt+>8{rTu zn!|_%iC~~Nhc~lNapXd}%{fAv4A8)^`ke0LK^+_?W zF?&2xHZ-)9mknOZJ%dej>T?xB6VNa+BNG(kJ0?D=x&2l@j!1$^Ad=_|Nw#{WG9VO;5gdpE%PE)!>QtU%u{7pjceQXGi1Qd zp6+)pE?8@Gd+W*EHV!K8C>*oT&2%08ppBm^l6;XXyZ*8aU_|PUPpY6g!*|7rfx{0J z@ILsTSh+1B{7LuvY4me^z*gN5*ci7(>k(sRU1Usp)i$B6l4>e)NErTZS}g$^v~-%v z7N6liCzu4U9yA>sS7a6(@F)wyolmfqn2Pi2bK(=!t~eFsbV)tmn7Vrg0S1I9rh|_W zQGlLTV8^(P3B6u%HcwS)hk+}dSW;We)eRx}a~+8BIrq_F)dhr{y5se3C?l~b)v8Vu z{Ix;p)^%e;n9E7Ij%BG<-DqvYTuSS1^IUMp9n>nVU6o6xNT#gUXJX*byBm!sbl~uH zN+3I_?i6d|zkGZY32i!;UAe!}xL4|YV~9sv-Y&pGbWS--aDEGAQ(R>-X0)GWR`dFu zX09CCKAbnAwTgX@n{T}U5P!_kN@%Ov(^PGa(p+Z=YgPz21h zXOA5+!rasQ*zIjEhIvI-hLm_1Vjw_QBhFcD;LlZhStPSNBO zIQ5m*`>?_Wxe_)9>t5FS_(2%+T*_If|ESlt^g$yOz6XgxMV3(CM*HZ#AVB`2S zTFX~%A?(Ni^|yW{-}q!GeJ2B58}%^qP+R#}-Cw)gQc9E;(O9ZqC@`GMvDNjWM!;HFG4~ zM_@*q?kpCG{gn0q&I6I~&cj{xy!n|XNWA6lTokcTd)BSvoJf{ zY}SK7tANl=1M6>S;Lm#{vDMa8ZPWJ10-26IAik_h7+(6~yX)BlaQJFHXBu|iJ9lQQ z{begN9y!gZ55jNh%Gb*ZEt-{ZemFuD=`LzJVrH9sP`emR>lDq9DJ`0xpKRAsR#xcC zET~UW0?x_u*1FBH0!okmwt+X)&#_Dy@R3+tUoQ|J%wIMBOzzdk+)P zxU6NKT(SN8%sRB7#5BonN(`Lqi6AzSJi*~1jYP{swJ;@Z5j(e~uQd3hRmp5euXba; zV_?WC#EN51l5CTnbGT*9oBgBJFw(#`0;^DWWT116ntwI;`PxYLy zDWAQ`Kl@!EuDrm<;B!`as<+zR!hV>pG!T|)eQUDVu0sfa@Ye(4G;V|8y@nsnuuja1 z6pZn9#wJ@mL#(tWRxS~G3SKx-;sR!>BkUk}*=M*M-5#@^tf--bB-%^HivsR6lKZ%C z!PSu3`z7&kM*cKgs-AjWw;%q4gS*Eq1#e!+8fltOY{0f zQbim>u#M5iIExf(=|B7=_*c#m(nhoH?Hk23CEG*)jWn--trivr^p#q>nIwycH{z1< z?i>3UBoVu9-h=MJ&-a>e!(gR%OotxV+MQ@+3c2TrLIeBwhqCt?EaiqAHQo#%-RG0R zuv6uT-l~OGh_?Ghk^fN867j2b90KoiB9MXR7lIAE3}ttwAxxucCl6%2mF?+>LCaV0 z6&|;*uR0G9PJUYBtBVz-o!Z??0{b~4&nOeVBW5|C(iLx{cCu$cxVPVHA@Ir zEwTI{czqBKnG`N4@Tf^q5T3G^kPX~m0_L;H$(%N8ZPa##yZIHkexlyyd}flFSj;jc z5Si(Eeem(yW47OOf7Qqw?_YZH;ZKy(VZbmi!cUyE*RUew)=I*-tl z3XwM#5AqG85V@i~CxO>G{;Qc_>bp16vz)3xOEe!U4sRakWJn#)K<)8-3 z5)TC1;LPI8t|%5cp2NezVLt~wF1X4M>FbMiHx}X7E-PW)PHsW4f+sKWKHKq-5!oK4 z1;JYAKmnon#cYZwR&e_^-fy7~xr8jJ3{}&gfm?sP5*{)5I4?eU!%+TB#H@o8v_aQ+ zq{d~*8ZpTuhEExiF7WuYIX~x)M{dG zPtZ3cJrM0c^BK)(R%Hp(Wblhc$RB_=OoV>q&pa<5&u$9E^4A}H4zFi!2pjRA3$G(u z3duF-ADtEIIVN2BTAo<{&UVV!@R_NOJjMqB&dip^LAnbcVm9ITo#Z^kPuJZUeXyRv zglK)plCU`bAXyzUh2=x?1jKDlH_nR zwRVMny9!B{3tpWtm_?9gmK5X0uDcz90W!;393gEx$MDws8^nlyzygW1* z&l^2BG&7Wa$djJQcbI{wnht2gA+ciG!R_7W=MDZtwX`#zSQl&>EMx?aFSkb*K`?jn zvhe|+GX;6hFKlzjuqeK;AT+eegvm|zbD z*BRbs+vP9r> zuS%Rl9P!QJp7h6QmgbPdMjp4RxgLC+d0Y+hB;3Vgkd-x*?oH3IrS2cDGU~!GKytkK zMe%MeAmkA{Hvcg{iYzL8ao+5=k^frTIrM=`xUADhEJwmFKh5XtnB`STmepuxYxYiu%*~B|X_*3*Afn^>StQ*HmP2 z#tw;AnNIlJrV0P~uzF(i1_*hQ9wWWs`^yiBI-*TDLatROZbH-JL9ip91{n9HwJTjh z62bKl1s8de;iPIt>v|lIz9Q+qb*_OL=*N08kFh@jiu1-}iAd)y?qK@~H`H+dT6!w~^wMVM!v<$lKovEL^LiD?+DKZ0A0 z6rs@eZC{dZs)qUjGHT0#UwDp5;0@5oZ9h;W)!KGNub$qn(tb>$vQ#vH{RmkN!3N20 zx#ELXYLRd) zIqX2bs;rgD8Ky8dZJcBnLJm%;G9305_W9T?h22~JUJw3*oM{4g<*#N49$<{~+dLXQ zz1`2Iol%VZ3?3sgGI%VEm{rv1rs%3E??q$aA(tqrry^yLWMw!rh9u7ghW-sS;M@l0|XJu76d zXYVP_?r;!s-N!mc<8RQOb=>+>NYE_$sqct7w`C?+6#L2a+NDY5sS(@%oGJdIAqx1EuVF7?FJSLw^GIEVcHFDQXUSU5nNRej zgt7p)be8K^gOilPJ?3pQr>g7u;4E-r`~}v3)~|)wxI~BdI?=&bQ_`^B4}o&xJEN=|AI#9$;G4I^?z3%6P}> zUB=x22h0CieSqD4!D}y^hyklBUWE9o@CBlptzX?abV~H~Pz9!987UuPwACUnwqWct zMq2FK+a!HHoZc8n#VzxhcdSt-R7?S_P7oZKR5I#cR4C$m&Tm@xL~46}dN&d>JL*Vo z9X8Z0C$+G3ykvk{KBV%ViuJObicN5WA_jcIFLqjPV~%&U`RgV5VhVhotuu1rm3xdC zgj5tZ3f>Hz0ac;~UhLcq?{%SlXlYE^4t<7CnLTqQRC+a?SA%Vj{2uH3Ve*y40Oj8* zla@Jb%PNs@YH008NnZ!2yg(pSw?9sK$DRSCbeQz9?_d+s0MqKHxB=SsxXL2& zo!|V_GmPTX!y#JG0(XXIFW&9EcY7sOpr2#t%=#pU)i=_))3!^K0ZX} zH!Tbcd@RqSj2;PYmogeg%9@Menr9nNp?u~ArY{LocBt8kusRw1U6Z~XP zXxuX`=nY}gNRyt7>sYyP4X5nkplKcq) zm@ROSCM~7&_t2p~;Z3b?!ndSG-I%KqtL(-%bgl8z*rPW!i|0y(p$4pg#Mjy0Vq-EZ zkEo`)wt8xaZkGC7P2c1{-(=TNWeuN98MIDCIab!w0&>C6+I+Qr+6`s{zo*>R+DVvW zSrne#&h+wWC9y)pe)J<2q-R-X)j(J$l4h{yyLMPP_5g~&qn+6WAGa084aktxkk;nR zLZS+(bOt;ke-Yw)w}j}#t&Ro4B0t`2YSMUV z+wzuiFqW@YlM!o*FCJCtGDX`}8uRi})PeeBZpAr&Z4*mqWw$BCWs^Um^pE)$YeyXtJV!|zT8 z^9kX-|EBqNw68CNZ1zIJH1+k2v3%<*hb0~~T+@{o+A;6T$;8$c?0?5iw!-)D3`Z>+ zBo%(T^czIu%}k{(oteYUtxv7OKrzS<#y4^uUaVO|%a$s5$vTUag7i?FHR+9z2@uH= zdN7wAzI=es0$Y<3>p8!hmwC{X#TOEta3iEi-qPIFCMg%Z@1{$`93Ht+KXkCUxk-gO z+(aTZA!P%P*LO0N`qga@mg8tih6FVHu9&-93{HZNTalyHwD8!#G}_X5(oHBL-(&pu z1ZE!1JgM#A=J+yH(Ped3HMaUP0ExiE2Q>7;Q`r(|=$+m?bV%7y;TX0?z}DFZ(8m8D zi?4@A7Qm&4LrGI;W?hfD4Aj$W+vs)LV5w~my_1}+v-Z5;&Pr2wzVTQwI-DE<4mzCA zDV6g*$38~5(p-LVwiqnN#&I2f>r#1lLDfnC)xI2M>v=wU?FXCK;=42A_)?g=IUX*% zI6)qabgNnGIM?1MYwiQJ;>CuTQ?0K;J2zKS@HHZZC8%3mR$)HU3c7rvP$-9vyl1|J8&`o>zg8U1-14$z^j-xiwPjO-t1%4`Y{i*iFI{5`DxJ(`cL=DyvuASGDd31MZs z1~DMEGPN()Hym$V;T<-eLhm?op;}@paun)_np!q~5&rGLWf|b9{Hq^-7$$S5zZjdm z>4=eg9@G56bb<;PO{vFg8loaG#&m-DKV$jNNWA|4BXR0+KM7ybj(4P?aF*+J5B75l z)7-8pu}eiaJEUlao!v!UBnxjwHi7Vy@Vm7GW_Rln{y)avJRa)(jT=@Sb<(26a*(D{ z*(po1P9=MG(kP~~FO%#=gS1Gt5VA949WiAM&1ko8V;M$FC1Dt17MU?K&u8j)?)!J& z&;2Zap4Y1*eLvgvUat40&TQJNj^8_9o4JLG$tE}Yc|mS8^+E?fDM5Tx18$K{Eq96S zH1Em@Y?}D6%=Mu6qMF-u2bhgchYtxmdiB?YwJy%Jqq2?4H~i8qks;t0TDDP~jjL{X zIV49=ob%*n!8CU&%9$S9eCwH!{#zwV@($DMVxiwy3ZAy%ClZ*&YFEWVx7Ah1J~Nu+ z)3Bx~G(Vx=xX(uO9~;Lw^&gW3yc>gB)D9mRnuTKb=%iG&$D~ZyY^?oaS2p?a<|&!W z%%<+kyKB3>4(9m)@Rg(A8($PamGssc4KpCIJpyC*68~B)5tg@CrH44JndBO zPioVph5Xs?;G$Wb6CFYah~@u!nY)-=7dhK%^p!Rp6BQXXwjr~I#CmtCuI?y?79Xcvwa<*HquAdzlQJES5_;WR2!*EaEqar` znmQu|(ZN$>EcV#QJ<2*}^IXT)p!V5zwOo*}k~#jj&B5BOM@66a*+I@B(vI0EeGWfI zU^*7#Nd!U~Pf=g(Xf54wNcgaP4eITypVsd_z=EO*4OeqfIVc7l_FN~fK`w__vjC}%z zGr|Q<&tzm>J(i-Lu=kSXH@9$s82s_2i4VVpo(jpG6`1#iRj##K&|gq+Cu~`KZ!I0L z5LU_?anH-APWX?uv{OfK%Z_;{v%@9N-+dEqCx5Hv49l?&&}d4nW264qjSDII6Q^L^ zK81Dhze%ZYE{)enZtI!@k9s87m|bjW(|Ih4J;`007;W@_`*&dW_I*(2A^!uV`<)9Nx-*p%F602n_l0%T4vA6f&QcYwQ9V9V=;@aP zfld{GC__1j&w@j{6O7P^Etyu_^`{g;h$%aZ$RpSoD~Iv~an;>5s>eo(SuD2{eKC%Z z0V3bL%PIDDoLc||Ue&l4jATq@gLGCChLkh>0>;n`=~#F5bM3D%v(lL0QLX`q zCv)%!BwRSykU-I_H;8{4JyJ5yKrHD_eMl zfdk`9#~s196FU6IU%c<*Vq2+CqnpRAIMAlCT3E)sFqSyCel6gFdL*}ZYV9#=ZY11B zBiM3JP5ffmX6+73Huf=n^~Q1Z7`m!Ly=>bN3&$vSRM;Iw>%JL7V#l*DD&YH>(yJ=P(f zWfQqEX99ivv=rIUM$!TwKUtr7N%#8`!^LNA$=hqJG0%~XL5#i5Sq{GN40!}JeYnP4 z#sZdn`rF^S^Z~~13ED-S+3&a5@{g^!qdyX=$aQ6R91wKrRY z>p^^5S&eWfBegyh<<`nQ=nPM;bxo*chJ&TzHsKl7{LSVVT3PGfEGEATT>3~;*s^9< zT&q|qRYu1v6id)oL`d2lThi3wfhule!@IokQA)p`V0N(DwbJM6YMY8$J|J^nrNshN z?ufp)&4KYI+_mLI#IAh2OUP1eE6rl)rPdmGkzqX+8l3zhRhhgvSd%e46Cp@U3?La= zM0*7+I%oBtJ>?qttJeEss;i+EFYP_UHGV_V+aV{);8et}NwdON*joaoF{r8C5pkDy5(<+y-S{dTJ;t4e za%F_*_Zuxx%h5cxtt&u(EW0jeFgTg%t%oR zMb)}j3ulp&f`DqU3?Q%^GQlE+9)17-H0iuY7Xe%_M=A>s-ud}9 zn=pX)%~GOPMc7EfRu5OYIADuILA6`gxJ<+|8|41RF8G7p?})Ne7&Iw)K8sB*ECnaY zv5>vo7G36tDyen&7Lln5$#`X;C=BaD>FtzI`+%qAFS}D@YLsBCj<0c$8vuUah^n{0 z{GolO;T_1aM!bpHl={28{Z~(n!Pr)9!9;v%89h1ZH|H+Knf;~i?_|;vZV|pnePMB3Yr(Li3BliR3-u`WFc z5>0D|2AG(8U^35#oZMXdSwr?*7M{Nxq_U?^Khfuh&<3b_!uR)PR1|Ypwc;@+S%n0a z8Hx1_K;KC)UYWT;>n~SHVCL$C`Z~yFP^x@N?o>q2k68Om7_%#o8OAxa-qB`A1_%PD zI*XB2;_J!xx9>-7fCJ4aZtNwfL~!z0&sNh;LCg(9F{iB3A2L&Ga7^b&=C~J{er_To zjWED4#BRS3tejS#!m8}cA7GW(<;v~K*YyFbztcCXo_HSj?gmJ+3!6YRmG;FkZDAqG z-9cQ1UE#}M_bE^dnXJXe{-9+?NgXefl!KFu$Nk`fafpw^wnB%GtgPIsj`?q;_&+Oq zweN=qGudQ!f~(OQ^K+YO+lp_LrS@V&JeHipTk1NiPpWGwl@Mv_(VZF%u!ig*H*+;3 z=YK^%c0H@L&R%|L8QeZgJR?J6DN0f6`zQC}9xr7Tw!OcoJpL4(exO&56dqN=Q?Ydp z&4<8tqJgdiu~Y=)$()2c%;Q+rBzxDj^*qV@@|6 z>lSK@@ED!WbH`X1%6Z5}KglV>I8%723afA_KLaSko&Z^+AFG4X+&A|**4{4Pv$>#h zr`k7$^eZJkeF}~p?c%JJSjYw_$w$maxp>dsoQ@|QX~L!r(Jbt6SfikzJ}Za@0T!D;h!#@b!J{5YHA6a7cL-u>U_#(=a6lb8xJ# zX_D)n!(jj_DdjTa+Lp?z{`Fb&53=Eq+* z=2GNdpwC>-JcKtGR<4@^IoCa)>f2l;WLmb1-ty(LSh}&?SV@d~=AnWeHD$YeeWA#k zSJZ`SzZ;&_0++eX(1=NMEHDEn8^M@ zyn{8i%B&l9#U@RoPE~zMM7Vn5L-atQbX%xA3G7O+0uyZ>hp)Zh=Sd=66oN<}uB-_BajUH0^MzP^0x`#>$+ zs41$wDs1|*8;dTbRH}iR$z{I7=+d+=e5%={$cJl6IgT3Z9c_(`ICciap19F0;>3fzj6+1204g{_nrBHzWe-kf|d|k&UQH zE=LE%&ABZLeGxO9UE~L4_mF?R`SY(E*5ZR6=0^a8T8ZvH0MH7=(EsH`VlaD!z0Z+r zy#nHpf$#86gPa?0_185YjOR|wz8u}*|2>ftSr69?z1@8_toQly7zVLpRa?22B`<7D zRsj1JId*8Gu6kpcaE3zwb`cfmk9c5anbG*c`nE~8m^M8)N@K6AzdTVm^w)D-(~w-6 z>qgvXL79mrNYPrwc>ydm#|T#8@CF0THiB*bSz}Q9kCe)xaqv4z2*GR)cFk7_*+4;( zIdi~J*vQx_IJhoJQnL%>>Z?S8WM1@_(3~i26l&I&c*=bGd0k&EZ=;`AZWu-T(%Og|WNx7k{-$YBhNoC1*1dS&S`OD!%%hoDfjpVeGQFe;BZ&i2;*fqH zR=$Mq);u%vXq!gya`X3$Z8R)O_YIfYTuAz%{}J*!^F~BxjkoqI4EdUTNvbk?;+{oc zLo%-OVB-&$jmbWz0qNC?m+uVquUc$ku6(WzMmr7WZ^Y+PQOkPnx{YGOB*q$Tx_EBG z^q;?;AXdR*EmwDL{~0$y5-H}L3sDt9%nQHj(q*q@MLR6zy7DIhq1CTCBl7+6DvB%mCJOe-$5}IZ*pP4R;sEVq+G1*e{g#xaL0LXfW ziGOF>u|cwmesN1y@${Nt`vY-T@4{anWLpxq^G<*KzWN-mP}ixP5Rm1d7BNgFlohiE zp1;%}?vX9VV_pM=B6U^uo-q-v;W22S@P5Cw-ZdF1YR0X2`1q|8#5eX#N8kaynQ-=4ktbxE`Ds+mkfTurafGR_M8A$%c-_%s5~0G&5e zR!kjfcXoD0K~9&L83xT)SP@*e)TdAki8=xTa*)aLg#YjX-`-xBi_8pw`$f|}3=US! zl?*DvJFxR+>?wWbn{RdKqTtt5R-8x?bj(@Hx_9=KZsgixEsmPs_Ia}LI^I&7d2g@> z)k%&{LF8zibl&}?ZiS&7#*Cgsly6wr({);4{Yu^7$3;V-d3Q%zn%}7QGc3`#+v@%c zSL(J$k7!rTtv5Yu&?h$^VC@pZ$ek_`D2i23y!iZ4Z2a$CvbkSKgg-N(v8tv8v*3g) zMp*a&LAWt!rmN=A740C8L9V%Mz2&~5cw*=KP zeu47E(!(}J69E&UUz9Wl5Fl-*q#0EN0LdDwr^4BriV@~!J{8y$DJ>N>dqSoWTVA-! z;$^Lzw|2;!jV`4E?LSW_wcW$$QSYzszw<-gw(W_?3lJ{N4Pb+qVu)ah;k-!NYO#j< zr~;(Ax;j-N6jW-V`O{YBt(BFX+^-PsVM?x0qhGWc$AYm6lBG#jr@!8pn_FDId_iRw z!%^_dBp=B(_>b3Cw3YG;wFuMTU4eGMdX0*(%oV1VOJu1XCzKQ;OST=n^@b0g8Ku7^ zABVykE5Z-pEna=qHa&4Pl~qA7jl`yg?6PYu!5zmO6jyaU(7QUM{k(BE!pU4Afpn2r zHvq7Dq~PJ-fK@~@Gv74um$UgF51-8n(7Q9jn0Hwtou_14;8`>kZW?hGAtWkpWpvQ_*1 zQm%Mn4Q>X~dDyA&0?S{06M)(Zj-rK5Or%nCx7f6Oj{0%mJ1jM1+ImMc5wmZ$oliBn zN)Sth3jJh)X4_>1`-Cq?-5swbhjMv@U}mp^?>Y+hZaiWje2&;8l-4Xz600smmL@Gq|J-6e9S&q0<9HlCP znop6j#Q8#~it1-M%9fI8eOfcnUd<0?h;pVQuB%80ZRqAJ2HSU-LEpxW)) z;kXh`s4}6U=yOOtZWYU|SuL5DA-T>z_w@HY?oaZI0fl@L$uSmhNS*mhpdo2_{QlEF zCT6bK%_2beeVH)ZL8Hpla}ZNri$rgsw``e^|FE-$K8xlk%9rZ*z$ z%4krGi{`?f)@w_)pcX+>TbfNsnsvg}71zG^rG&nuy=N|+%w2h473{rY-CMTdHv9Yt z2e+EDVLU*7a_ze-4Ylo#U@w4iN6(R!g!rNzUGASbS6Ta-`a-6Y4qwZ0l9vDZ!?iWj z7=X5~t{jKt`%L>Y(SYK!ErQ=~R6bUUz9VNU2@tdno^TVZZ7$Q06#GcXxC{Z8Q0g)v zF-3)peH$~`Gq=54kJ=rwzb$`JY{rQ!n|Av!om0tYD#F^O!_P-oA?lOPf*PM$4iMWYd-DV9`ljS^@kO>KW zcxPki+QSR8afzsHVufy1jeRKu!g?bbnNU5>eLPt=CqBp;i1I0+xsh@P*Yjq_b=v*k zMX`6H0=4IP<##&#y=I%lig+jk%I1qc+p~>)e)gg?DwKrwX{#Sr247qZaKxxLFvvDEJ{Oy{M8zNlgWo zL26dC_zx>sZ}C1{pL-eEL-qthNB{=qZ3oiQ>|czSF0fR0dwRw zn)Zv&*4YcP^I%VBUhh}i45(~|ZEu=s2`_SU#MIbP?4F zcPvn<2yn-NVo=niE66f2*T}~P25xs{K(NInZYc4UF9P%S@9KfT+}qF&6s{w6au(3w zwH$yyxurC5*6!a%$$x2Vffw7HaNeL*;W2$Q4v*-s&drApb$+u{9rKV#JR;b->_WH5 z-C!Y3g)%tJ(LgZCf?w0>lXeb#Z~FG+0prfBpPrl_;zMHIC-c~!( zuQAgkAlI|_vKx%!reiIcwEoYY*w^(6ZDr|z#(r}(TIm?IoOZQ)DYP++eZAH1Yh@U% zyPd=*L$hk6Rw)V0TVU2@TG*aMke>^uYJDpFqUAC6vxQ)F0D?y70h!~!JS*84_WtP=lj5L zP@iQmpbAY~{+OCw6yccvn#WmAx$ry)$S{1QHLOMHRq6`T51v&4QWN;Z7VPU{4E^7| z9-o`y73H_A^0sb2wzxw;fO3zClV+C2dria)@{a`mG zO{o(GzhFQ^l^0UBDL+zl^6g;N_uAJveff}y*R=OP4vY;c5qeO9?Kz=0qv+xPjV^0# zdygR2GO%|(?$)OdY^3#_VzS$}2Xu3KQvQ1 z>etu79`H(<6SWeez$lgV%D)?^nebf7Mm! zQl!Ar*?^B4+CJ%5Td(SAOTNdal75r6YKM3MPi)@d1q6zs2AaUg@t@&3UjOL-v}E_j zwxVYugL(6!FrV13LKk>}%xjI;+N&CUC*H|*AUo+;=b_ir{!py z>S~?Mq{5d|H$K0bTh0x@k52(sK5T~8rHHNXR%umYMI+CLfW&9_~kamNJ~TmYsq=>T!huTqD$+8O z53VEm7Sg!i+|kzWyj}au$HWB$hQ-T!aUZuPa)EmrH&YxiF0EM5uYZ5>g|2ws0m1Uf ztpy1d_tX34Yj=6MW3x( zxl;yb7qs-SkM@3aYG_i9!n!e-+w01DHU90IXLGPPOEu%~FW3I;JOJBGJyW|ViNew} zFy_TCOX54rx45VbA8?dr(*6kjrfsx2+e9Iq)veXGZpkKO;YE~bFo1HYP2{M-wa51< z1J+Kr9VRa1>u9pd?1F=Xqb!5KofHe?XU47`6!L#U^`^|;BRCzB(;t@ff8W~pu~(4^ z9C9GEX%o6lLc%y8`uhpp=(o~gGkJ8f7$Nb|cEx~r0=#Y7G17ZJ01f*m|1G2MOjGPk zR?~q4y?L*EN1M>0Fnp>;&?ha#>OzxA`;|K!`BU7>(Fbu4t;z=L?YSdj%vLtO_q%dE#TW)DQag!*IOdw(u${>Kr2(zEZ1V_Su2ovFvL;w%8>Z@Ozf>Bm zn!dkKwNg~pS6WtfoXsM(O?fDN9Na0BeXgdkb(WH#iXSte7Yj2^X#eQRb_)P#674j` zmjfe?k4*4#s^u)&yZ!pU4D+0Aee-E<(d5RDxyQjC?VnJlMY(60JhVGKd7ZG( z`l}tPs=gHlO z6*{l+$H({R*Lp7(zd3l?hD5y%L&Ms`>ckDaPpx2kenfTqGZ3Dw+ECx5|6Ut^kvda7 zFu9W%&0FR*1ovsnswIUKoW252+NRl_DB6z<+h(i1tWOO~Lr!8fx@h zx$U3R*Z1|MNS6I^vw*v~UGS(>r_w$JZ0yhtXCT~0Qo&8ETd z2lqYqc8=NVMAU>KWZcfw-P^4*S9BhmZVw%5N;Ck&uuR&mcl>#W(fGTtdf^68d4&Hj z_1(RHVQ__HZg}Y9fX4C#!!p9<7-4ex6&7=-kgE}q^qCJjXY<2d>#g!4D3wg$5GZF_ z^XS&h)Ra3PxZhN{?`m3?TDs+W4&ymXqFV`XC z5hO+nb!qIzv}@MuqBsSF?Y#5>8QB>=+mh zos7SHG`+pF8o^lOtEf(cX49XN5(>&2tauMe#2REx98C`yVs_ zDkcE}I@15-A4edt#*xk#_am`+gQCw>tha%SKDlWyq5q}H<^HC)Ebi2e!szQM^&&gl zd`AyA-WuA`=t0zJ4drgHY;yws`+=HzfrwlHg6L0YO?)~XkcYy#bBrXlhbq}Cj#3R* zFKOrotYRi)g5~Ss*8-!4rT)8i$v*Z96J^`)X)F0#9xFu*RqMS^ zm!9T|e)Y8c;B{5*CUT~S_fk;mlQH@FpkFhcrH#m?jJP%gLKLtKHLlZUz0@{~ys|d{MwDbM zt7G3^BEYI}_qzCS&py+q<45gfs`eJ)b2JaZ;`uV{J~=Up6eg=4=_qHa2c&oqZjgF7 zwG@j&ND8fUyZktg{HQX^XjE;Jd9?>2#9F z9wKc&-|fg=JzQJRmQf*G%T$58+n5w3kILgKC$)ea zxPJ}t?T;L|YpX-UJy+PsWp!B*cc(pfx|bMX3mcFLLHlhcSaf6{V>0ahXLQ%@hYO;Y z!&QD&Qhlu<1DoaIru3k|NF)U}_ylNH-N>NvzoqK#+}D4~=J|O_OIGio+Fx*)HQ6?6 zqKrP1=($B1j@M)TjgQsK`G{)k&H(s)YPa6cGp)bAB_O~wI2WHR#a^p{1%49&%g(vc zA6DPCQ;ZW0T0PRt&=UGAzM+TDIL{cp1=R6%JC0A=OSg6s-~TitgUIK%9Q?;kqPE!q zVh&91@xupcKY9}YFEy;{kO{ta=xH#~nlEV~J|#+2E;!i19r54e+GBnhlaiU4IZQn=Ki2ZF0t~W9?_g*)1m@JfLf&ZOgaHc# zrux!LR6B6k1LXumtPlN0k@15J{$@HCfMC*#Y4;f_8wY7k=sP7oQf3~u6P`d7(+4>N zHN9!McXODMMH^KITEt4{WolA<#6JI8nMTtIqg8~?h~<$6Sg38nhxX3`0!Yh$JB|4c zKHs>|iOXo>htlkM`wKZ9B~E=*@96haW4KYVM#~pYrz~qVCtP3d(sgZE_;Cm8UuXJb zC|I$kbM`>d8mtNFv$XLm>|?*MZ1y>sV2*QL;&7Wu&m7kh9rR({7UF%NMtHvP*0*>* zLL2ALiPi-+tVJ-@ESalfQl1y|c$c(}2J0s`3r{coZQLr+I{?O@xvZO`r&FK?cEB-J zxjSk#z3}|{K${QcJbU9;JJiZWjEdzX>#dsVetRUNHe)z12kEl*&b70Xj6&XTG{Yz% z>?><4un^GD5C#^Y3}Y`#bTV8>!|VgAxB4E!Jz+oL4F~2u;Kzbcs^X->E)mK6CNH)J z;L!~cRncH^OxJDL3aWrUEzxi&hp{`(Zj*NBPvx~ER`UJXgwgn_l;<9)wV(ZtW}Nk@ zxZpz=Fu?R1?$$oqlI&jT1k`;RG2@t@!A7JdDP;IKSR!BHUKRdCWyt5_l~0qq(i)M+ zM&^Ou_6$ND7W+S+r@JFkWRK?OGesRrc0VH)AN!$+2yOk*h_fOwM6HYG*bhh6^)pLz z;XVB2n6IXxSAoi4v7TpGjdCJi3GUV8w?K_&IJrvn{w=8;~<}YMM9yz2mzhZwpnc9&^eBQ$ICc8C0sYF4oHBgC|k%l|j6NSZUx#QK!F% z;zJ;+K$;rfr#(9c^U4>UG3n;FSam%qMILj4{61vEr+meKdm@LS4o!TrRmVZwN|&mk z3nnSr6Dz-yfWUPWd>?RlYWH!8vb&fyQ1XM_WXBVk4L)tA^Fjte69 z$LnCi%Mm<=$meGzG4>6Cq*_xofj-^k*EI{F&4ibZ;LHGqFAB5DZ=6cTYSdogyXi)z zA>5za*yH_CV&H^NR4Bpv;Tgvk8ejwwyfG}E#&qmb(g}8DbFh$w6 z0uq+rs8d=3TqTas?O-{99ks>a29|iVwA$VbnCDBBElOD!RwsY%TthJMyUj-$vF4(4 zYAbewmKAyXvJk%F3*-P-x60pMt0JeQQrtsRoko(;ZGKy2Dfr1!Ch=V(6Er- z3R}Wi@x`33Z$STLJz8*6#nJ8Pz6PaP|FcnqO29$f z1^Tc0>l=Dzq{4p^cbFCS*i~9Qpp|>1ckvm%QM=GB27iyX1@j;f;j2W2;jP=YhY8EFKp zxO+?T-wlY-g=}@>x+2zagPcVnQQsbEfPf6@zeDOOTR^Km;EWl?wZMuN7+!tCyY&oD zY$GkuHx*^3Bl5-er&9G>RIS{=(_?eTL|)t5s9Wr@W>nUe?D`_l!#E)~^ehZOB?GJd z%h(0!zWqz3p0BvzrJ=?o)&q)$Ma)IEuh@l)bsd8Jb<7bZ&sZBEdWMdE!4Nuv`n0x)e`^o(d!Z{wP$8M?&s{rQTeV`&~2!$U3CUU3M#lmBUpP3mdXM}&31o<p{k38NMd@>;-?W2<28HbqxuF#!@0P!zZT0(pr@Fm{xjM7vv{VtDS zcWN`f26^7pY=M;ob;n$KeJhZ-@apr>M6$UaE;RH^E9dz?IAIaMv&Q-JV7pICp!dHjZdxbsRYul`F`K zD;RsAappA=8Z%z4dE#m=Cft@Ppq4qGrB<)R_aFe#(~z+A^5buTvqEwu=J@c|1k&-b z8Rcz8kgk-bYYluFjr#W3dxW(KD{g;``9O4k3K>+h@@AWFu^z@496mn?pt)nFv1Wq1`;}-8 zBkbLF*|C*&&_n;lmE+i1=kTI(|$zNww6g_Th05g(wy9L&52*?9AZA@ggd zOoP9*9{5WJlMPsNb8fU6e0mIQc)ULp4-e`*HrT8x@&vm4nzmShPOdHF(#ucz5K2CH z_=nVagEoLu{$nR}!99guC}&me)EXg!At=wIyU8J9}2Abg05}5Fk^p zHZa>9El_sh{z6S_-x3&-HwUAW{IE$STx`IzylC#at+Z@0*pD-j!xgrJ3lHlHW8bQ2 z4bOVK$wxBBPXU8OB(;HvV&QmP_I|0ql5U7q?g~MSFdWLBu%5AUC~FU;Zahk41B35T zp-)v&=>v0~gX;ZyZb0`>^lY9R0~e(c*0PmOut0o%6*#z4LZXvGG^2Dvqtgqt0*6if zMwAF^@!;XFpVM26BE`z>)?E2!w~)Dm|C%%Q z3j3a;zNkYc0Aj((k!R2i%oMeXUdV1xmv+qVEMhJClYQ5eN7q-9uC`GX1EVyymTtZ! zbjT+&Ve^@DcMtRq+X(atW&&Co#Zn?FMa-9Gp$`2t5_b`i5GTRsEAY#B_EOCu7Qdbz zH!{1BZhR^sf2*Q6zfT9An2$u=BXBduU^4xaT?A*4FaKBQmCa6&=0n}VJ=2D`=e0L4o>yL zef-{x`{3^z7Ixp=eFviMdBMR;A>kk0u;(sO>oYEO@!;V&cr4z2eKvqVJD$9HF#()( zr+kcXmj(f8kv3QZc{=C(r3US*I`T5=X4+Az`DEltK0+^!y!s!6ZmjoWdxOzLACbxw zlpTMh&|744Xyb7<&n;L>Ked1F8Q#_Yn%$RapVV4|zsfg9MO;n6Ply8^`{AW-Qrv2D zY(y#8scj;o&yR?2*zPH-td!4j{DNNrEOxqvXKF!;LmR8KRXyP-Zu3vzd0)Z=^eWg1 zsF?k{_*N&QbZ;-}wA$$?9-C~Ue$9PfK4_hBQICIG!)k~vGFv5fc%~YzJGoFq9dckr z+}mJGt4ZDTXU2C>3_pr(H*0en1b-rV9aspALvIsZ!i{EMWmYcHyP#UR_UnX2t86JV z(sQt@r7bwim&vS)8q=28BKt`Nh_&9zd3X1guma#G51Guk<|M@1P+of-IE;anARGeJ zzJ%cYe*6#=zp_WdL~@7s>muS&q0(v)$-p{FzF#$r(^?1>6)3VD)Ng$D=Y5bl`1w9b zMlJD2;qa`woJ#KPA{YJHJ(1U$4PoORSrb2cXWHx!7B#(E#z#^iysw3eG7!!aB<(|V zBj}%dq3HF19TZ>+l>1=`zk@xqe^avUHt7kvif#-pyuKfMLWt`f2B^4ezm0MN1&P{% zagJiZe38%Z2hKWN1n;P_Ow+UWfn=0|t;G4}a`(ER=14in82!8Wl1A-$D2f7XR z%?Q_31ZNiG;bqz9b23VcS}8Vn-HBukDPyd_(87FLk%r`;k{y(=Q@QsL!Q>lbQVoeV z#pb7XRxZ5VSf`;88LB|0*VH!S`MKAbPFUuiQqD0ty5AdkL{gg%8(JBcFEpL=Or_|> zznm{@n<>yr2B||ec$Tm3Y+kTGnBF?Lu!q4ikU&J_mwO&)hTSy3{4kB8ZI%n3UjlO% z$Wjw{a&Xe^Pq8NOB)+$GbZiS?i}os$=W==nawW%W+c<&DaKuzuIcm*^k5C$|krexQ zxc-*$fqO4EAAWGqSMP~Y3BJ-eGLqj?v0Fdrkv@~Mj1KsRF|&R$QeS;Z@&W$? z-}vn34nD71g8QXcw0{7Z(7kxAy=|amShX| z)nBUjw$<;EvHd}vC*6bW_JIb<1M2shT>CImvqgG=I3OxCb&qzbVkLZ`<;l@# z5J^yvG~~yVmb3%g@9$?^prdCgnR@D)N4kslLNw}XsNAD`N1=K}s_l-r@viE8?5fSp zIXI62M;C4I9T;qW50zksq-ciArmMKEMp!+&n|6eSGP=OCnN9Gyvl}4ECf?wd%iLWD z;dmj6bl``*gH6XJcO|88*9&>g-AXOY6ryoAo3il=*cF;-ZmuC0ymyN9$cw$PKVf?I zu!NWpbB;L9a%_!yKn14ecwpR+*NQ2>qxh)uO)klZy%1^3y781?Um$YYiq)baJ{I6m zXS$wwyBOZO@N2gq-Z7=6eLF7Msx7e)IQQt`@{;39&bWwB_lvK_AGRE?SfcwvIscpo zp<|0XXUwo0E!4&dRPFI%+ukD#nf&FPO|tXOn26!irbt&JJfC)s=*n>EJ^gzT6q
^=$UBMv&wr%8J*3*G&eU5hZ^^92dM@Kpi`Tqke8f!N zr=49XeIJLcbbHG#WbZ>b358?LNch7i5?BYb?^#iXK&k!jvn3Go_~k~D81QoRqlh<> zdgMihRh)!k{YD88&^l{>aneJVw5ES%ZqowaC)cu>Mu-E7-rQN)Zo^Pzl(52rcr~26 znuYlC<}R})9M_Hi`9SqNgN9$j^&5bi$YfJO{oL7AKyZi99}7VQQH++yw|<_vo7Rt9 z3bwVuqz)*iU4d`3wy|4pU=8OvwCSZAv)eSbg8EIUMDfjWnKj_Neq19SwMPV;3Xmp6 ze_isW!j*=0Le5L+=18L(O?B)I*;PvXo!t<`w=bZb27JWsCdwf9o39<@#C+wGHx!0= z+b<(Jrlf>13~p-HJ=f_{o%F!x#EAKvqFOmf4ue?*=P0ze&N(^?b;*|6g6ANDTH>lX zzZomk*sW&=*JljdqaiCttINR=tH)ii%;!<6SoqAHM;frJfoqfD;mB@NUd!^ijCauJ zA=mytl>}2~mO)h-DX$UCK|a^($<=TSQz93qD@@rO8c4TU z0i~PXFH#C*4SrjiR|3mI7%lgqcXn-vC9ji z)!nL8Ulj_uv}kmB$ub*Ch;eb~6giq8NI`lEvg!1HjP=#X@FQuF*w z+O?9XjSiWVrx`c+{|`ee@Y@IPu;Q9x65R78#n2;dr3{bpGqT&DHRW-s=sP1A*inbD z-Y|Qd0j)=UP}7!**3dAt55>&J1$~RkXyGG`bGLU&YE&$Y8`W8C0$*LT_0RJufQya& zW}(&QZmrTfd?R^CLeEhz2NHCHDBd-vyQyn!JdTH;v7cyf4eQM|gMpDXfGw`8Z=+K_EzP#`o(4wUQC(Tz z#7Nok#bf&=$1Wwi7nPr5GWh?gnIK}?Bl*G(S*DqAfR*{MCK@yze#!4jJ0^eyX%Oa$ z$|{Hc2ocFJhrb2)*9}+R^t?){{e{%rL4miJfK-B!2eCh3eG-#c`b+k6cg+L`2;-{Q z_J;O%4|cRB*V|l91fP+Zm%;X&ehMo9$Cy%BBq+>wNcQpWgN=cGc|=Uu=A39 zcXPR)?Xe{s5Q%sE7K#6D!V#eB=uR{llWnt`nS0Oye&N-G8lt8^Z`Mb{D5dc!CkS7B)e$>`mD75;rk_Ni!wQ?#g{oWcXoZ9Zk0 zDF=@;azt^fR>m3l)DpNndy}gEK({-A{bNBwA}wO1a3ON}c7G%aj+ZT+U0=z4vmWS! zp>3h;7Yq|Q<8a3Kt|)G4P|&;n^?~^-qfP3-m$rOfJ>_?6p`eqdpV`|FfW+ck-Xr?{ zMx>uNHz6W>!yJAp@m=pi&bn?#>MalZL<*|S_GIrDbbVT=u|8lRv{?btQcwmc+Os~u znCN2)xQ6NxD}JKWG)5&uSn!hJ#M89RJ-$wyJ!jAM)_V*_b=_M{YgA%20pgc)o-E}s zalcXiSZOgc<79TDQktt~Ig!=uu|iEVFSF02SpoLUlJ8W9+P2F5W_~xStVTZhbbVtE zHign<4$jEqE%(VwkCa!X*Wb29&KZ3DU%MIrXHPtzbdr&zyoSiAXE^h%#fC(5J@A}{ zV{OUS>B(K+;JMwFh7(o#^n0qkXF$JVI_UFho~?AL>x-=tQ*7*+2XMVVXi4HzH1d?d z@WB>{h&{jcTA-+GrETxD1=gi|)nd}wXc-a>yd(Z_79nfV56;d(GOFN7v7$bq0rQN|V&cn@Ekc-u47KH3?Yb?jNnP@K-;KH!fru@fh)bJ} zo81)^?9aRAPy<+=m^pq?JzU6WN{r{H*%dV<`UysyTSzc#cL0J4VUQOG;>SwSS2H{< z=E?OQwnzolmcESKhmFK2))N2-gv`r2L8F3zjfbs`8Un`mk@CfnpDyM@r@C4JH{|g4 z8=s)m5V=JZ-cOgVsb?IP7sjM(fYxe|oU<1+<5OgX`aFE38n6V@36(TZEtOJ$t!Zt;lq$GR#%UknHDqu`zlIPR%~vkqvRKjB=t?nc4f!Nk76A?Px<4rCuz)?NrOfH%gl;AR(&qWxvzo z&sd^~$9-pT63(?1W#)f@Wx>-4a!rGaJ-5u&b|y7g9_ZdWdMZ&Y(GuWQpiL96g8#B zSHT^+_YhSBMW5LYTcB<8vX{mt3ByEO1 zgF2o$wT9PX8NEJO#$&hfTK(QA`i!=ZdlgFS4~*zWn6g1OfUk^kVu$K{a%?8+SwmtW z@IGU^zuUf97`+_OK<07R!bm>lMr~subHyusl_eh-iGX?K70_3ypiAotxY;xm=U+`H zck>+C70FmUi&D|Q&RK*5V+%^+F9BpykF?-p5pVN^*lY%{y1S%{_$!1V6dq9Q+Y&E6 z(b`3_F00*%R$8|a0F6LUXATC?G@G7+26vDzs1WQr@v$#b*LD@fg;V!>P>I!4-uHs(_rv0QObQ`$vN( z()}!-?8VgxG3+O0LHBR5M#xf$5$6iWrR1 zBYy&weGq#EJo{p{x~1{ODb z!6^q;A|b)b&^@Ec?{DNsz+P0_LV2rXhh)kE&p1fgF%dI-CwO!LP`w8oWI<@yXi*s} zzMu+7yi2(MkvX&l_-88=&6mw(LIrW;b->=hbmW7oHC_*4uP!uUe6h6MBw8vTr!ojGxNM7cr@O#$AE% zP6odzlbpNes2+dr_}~PY%dAMQ-5MjHaCfs&%c#fpFZ1Xbu!DHN6*VATgjjY;#$;s| zg%Gy>ybm;kXQ6 zOb0P&TZIoEz&*Bl2i_9Io&Mwh>20kT*(5rAFlPnj^n&vdzG2i<)JA`qf>dHfrFaex zD@53YLZB)S4sNFJxQ^4I4%=Jzs#3p$coR2zOY^{%U$!RFqPQv5H>uCT4fc!)gWyGQ2eQ;rh8LY><=|2R2moas=qi7n?OBI^ zA421qG0)f&fIx9Fc%}hz8cBEhi^d>Mu=B&%Gj!?q1Gi-NU9fAu zkMRk!kk5T-!}NUVk`4r5(Mfm?fD%#~SW_FCw2v>1I}SFfLaOMksPbzlxb(`Isi^vl zOGg%{lApfnRWB|e+S*(YCDZDp<}h7>Lw@1eqI8lAK=uu>`!b zMTvhLwOwjo28V7NNwuC&S|}BQ3*F6!j2VV=ye!A3D@K!TO7C%E7hbZWu$=`a5+}xQ z?HA%PMi4JOtGUV*DAY`4iAuIr#`|{f0%g4q@p?p{AE! zV4xoN-g(Oc_IY}b-Ez=$sFk{<`4kFHsAgsOEz1#LPcCj%t&QMN@GXZ7%DYjFIY{3o zSC9zB->VNN*hk#|P<{b8?Ny7;4=(0=?54d9g`q_{yFxe< z5d8x83)@LVcvFbVbn;gKpww!iuNvewnR>N!sR|C#z({&8jJ+Rvy-|Na!_D7^0i|4Y zUc4Uztlpvr$ixDQtc}2n)fUH65X)G*VdirOX`lN5imqIEiGG3h2{Y2SeFIGx*R=-r z0GMi}sTD(Tq*@{fqh!3V4uUhE-SC?FMuXVeqoq4w-M{3~N=4xBok%zUvdq+01LNlb z)&m`;gL6!4pQsTmev_lG_X`(8IHiXYc5}RE93LAiiiYIZlD*|f-|D7PD61*^TREc^ zl5&Lo#x*1_zqvCeaf28~ztNha$?qfCwQp)_suTq1j>#d2$!$=W)><@TH9GakvGtEB z>j{3JsNl~Ed5eB`lH__h{a5~G1ER&@`RV2onzDl4T`SKu zUZqD_Ls1;pAY4Nk&LvC`c!dM^ZqUGYPOw3;udffRM}J2yT;B~V**5)3q*HeuWHr)y zDEID(|y9;k_VjZOzCZAcR)@N%e3qw6x^oTt-`u620yFxKh4FG4ogXH{i)-A?7M!F|jw!^h ztgj%y!IWsEh3DtprodthE*xy(jIc3itoQSL$_mtw%d`$8HTPJfqQ71gm(v+oZJje!>~S3(MC(p z+kjj>?jX4UjRKetLxMDs+zH|j_&2g#L0i-XZ|T^g z5B%%C&?FDFqRm!7gB)8`6FIrpLBxK$0GE~Zcfw(cCk}7tq4+-Hxrlr8Eee&uBRKWJ z5?unZ4gF+|0zlhWuiU!*J`yCb>ze_{qZB^?>$-Q3xq5IKGMyty!_4h892}z{kxDTL z6BHJGAa1HQbn&5>b;QP1RZgFke88N8kcjOCO%{HO-8|?hs}TVJ!8a*LKtLG1r9`mh zymE`T+GnP^mkf|%ef7-9>$T*?f~kyX&F4^psdgGTVU#{)<*Ybfbv0&{kxx~`wkHq$QX|`GGD_@v&+U0%M~7ff5aNF1q;w^M z1hQDALt(9I3#_Fs8Anc|$fGC+C%E2&HgZTgXD#1Ee#Bp@lChP0v<2`*6p5?jyQ0-n~`F>D5;+ z)zAUZq}=M4XnO$rS_0X~&JsFlxMP|1wz}=BG#Fm+J-Y@=?c5WGodH+IO58W#tlrXo zMXF-X0EY7w7}%jMeXk0gj~m?Vl))}O9}wdKY_UlpD2C#v?z&sQYZr-sOR zkammU!mEu$WPDV3XrHmzr&$zUJVXtkbz^eM-a&)$wWO)jH zFs_7-y7T;F0KRU`in?by!ohrj`>w(DglMfy@p;M9Y~ZD76VGc3u~%%?v}M&Yd{3zm z=rSYE<}ABtR|R*d9be4Kz@!9soX>H=t}?5ixa&H|t_ihA}OtPE(L|%JB#+F*6iKfYpn0vNiDDPDd6Qz z;hogG4h$c63NU71wmM{`V@9}7U=1^V^FUar!S~+kFdub;??C;70?7tnT|p=f7jl!e z0sgyh1|}bOQvCFi`6ks)YUe}JV@WSx`7IChI_>M$68W3yZ>IX7-)aJG)Tn#xGS8a@ zSS7C;`SZfM__mTCBF}tE&DzU;Q2HB0+ZnT62fW+Bl{I5FEPpe(?JL?_!~=+`78|j% zw{M{gDX`4DF)CuHpL1D8ho48vr@Bu90Ho%j&up8WhC250AB_UxjZCXeM z!Zu|95)YB62P%e0zY=gdLKNwwpWEHr8wCob7O*JzdNMrfss(Rx;7T4$iYWRW+|@v!V}t@pxUR|zqYpQu~E+_YVMOeV_>lEG>{S`_yET>{y>TpEv+q)k@x?LFTm+w3n^!~B# zDeaD_IJ9hQjSKm#<3|@ZH=N;De&oB!a-C9Kf6cQ~eXB^B)^mb0&+d+*>Lb`nPCvV7 z3z6nXpK|HTo3F<~#R1nhr2+guxRK6e$DPYoI=ZWC9-VVK?vEVzjL(2D-`125eu+Bm zLu>C-^W6r%6vThlzux~E9Kr$yxx{}Gxh0(&L>BKnsMc~Em6*}-^s%2zn|+_QbEH+F{(b6;o5hY+x;B0veEF_0sBKgh^Kh#& z*Iz3$*m38nRpkA@#X;U7hw)u4k8g>EW!yia|8-_~NO0c~mBUak_o)5sCC4wl47>z8 zo@@N2Su1PweTqx1am<&Q%OiS5P@`;IN#7d{sDW-Emh}Ag={!g_^AGhH5Nr)}ruoi; z;3B=0WGkJ&M%v!YryoHK(4-U6uj?;(Bv|RZ>+K5)0wNXrWf&;Xp9Nl@V^_3d&VlnR*(wNNdaNmN44(xluS9gR;yh%9@6^Lv9{agu4u1wf#- zZPO-!gQ|!qOUUiAZGM6393QIKT2o>SW@i`~q0!cV2v%Q}0vSHT_j9{JgFNV8=JBwV zPXAXqK5HgI#zz%fD;T4mI2G^JhoGsAPl?(uy1}^!^We%}Mm?Jp@+km|8si7+L**{0 z*}_N20KUva%iP`0K+wS1g?<$ zk>VAS&LiSB+VQa4@1xH_8Q&l?4^T43KbJMdI^|FMKtab($@^j2<|d}5_FxDj!8!;M zyTM8>{3O7}bFUd#Qx|A)H+H~sG{C3V?ST|Otc!Ykd&isrzpzqHZM$I@A#lIRROJCW z&ePJh46thnKY57#cCZ}KI>uH$x)UIOp%e}$elF7Dr+4K8GKqVpD}YwYtf;7%lzln_ z^ucx^VCCFt2nzWH`qxi`PWfr+>4A|`Oggk(aRPQjADdmiMK(Vc?B?yWf*SA*`mpZ! zjf5YAiQNk=A(J1i!D$B)=%23#_`d^3#_Ir3xLV7K_5g9Zk;JHz8gXYAYy8)Vq^LOz zL9$cTA_=O0A_49D<%G_CBg@*FFVE@@ghlkn4>L4I$o5RKIJGxL)0pVYt^6$zu8SC~?{LGWed-(2Awa+1FeiQ9W~kAd#R zfOKkod4URQ1GUMb)CJ#h7NMz?N*7=rnUv3L?-jEn(&uxo9cETkqvuI$0Ub-NO&7^- z;CC=%H$3Tofsnp{d!U))O0;!ewcS;8XI2mf6W_Td9uq7_#DRLMy{uI5m(!C7PT!82 zsAIcoYjWAUPJp@;ua4Uly*|RoimpPfQ8_&N2-)<(p=4{SrtWaAjmll9W3|#Dq;i68 zr*a+dRD_^?Kte<4tSL!;nU4_(;;@Dm4zvRPjSOY(NQS*aSWK7JEC)gP0Yqu#~|SSm3=p8gHykc;>IO*Y{l*>xPqOOU%ayt>pFo^2y|9k!9n2YqJ*EVLHX>xhLG3cf&0P_NldqUujh)yA;|6J zybG)n%T#}cH=p%2%eGv@ON8`jb@GUpMTjt z_Np&ifu-8&jkXhyyH*ScG~toTUhcLPBE>@nL_-u643=?nbDAkq4Rfw=QEu|^d;jS& z&h*+oJw#^NDT`W6-mHQ;S82RF#9{dx8|tWmEIIJ6ivG!0Ep5@jCFqcXr7DiAaw8h9 zar=Gh6v2^+C=d2{YFL5G)s;wri8 zV5BcU_-+pB1+A2e|Y>b{WfnKN2A3VPB=TscyHGb8!(^WN01$Lj8RZ!LY<1nXZ zsjU!ub=1rr#%hAZ9k5Eh7N2~!-rE_@A&#f7q}@NFbKT6~I_U(ihAfSRqk`t9{fp^S zTD~m(5nelNX)g7|pho?3$KIA?KJn=r?!q{U7+?31SF6+a*>L3W<>6XO;q&+&>g z-B??(lv@sDKO##mZ$F|*Hp)BbAYlCVjq~I>o=E7SG!mBDDRK*CG48&T6Dn5)7fuN1 zVEhN3E8#tRTV+erwM`<8GJjhOqb)r(hFcK=4IpKJT&%WbtW)agoe zQpP-s2f3bK$NPt~px)96G83-Pz>6LX*Lu4)H~T%ivBr43cu@XqT-5#x<;ZL5YJ24` z?6)2}zfW>(hcwxV^I8!$(1o2tqkpB*wRztZGeYyEldXZQIq3M9ChEoLwF_f&`qw1g zcyTNIS5B`s-bI^ggrh)4_wd~Pc^=Kf_+IQi4(p8idA8T#C-;YkDNmYLsJ}~)jb$CG zbk51Jrb@UxD;hi@{BCx@pHO@$yOIjS1r9%7C(u7On%~!DUT+WD?>(a);&HK3%H;jt zDXZmjYFYV(3hED7JFcdWgB+bGjK-~{$fh4XtHFi5<#L^S_mmo4 z=qU9QUG{i*QJ4QLUVmr2Rc0qp5&_c-y*>_w&e}3I_p)8Z$@&kSE1t{E*Xdj%(VHNz zJvFqcHhr?FFzxkfUApPW3ALf~pPHDpdJmb*G{qJVYQ)M!$XW6DB`AAaXq(z5uG|1K zTPy0%OwoflDc>6&IDAU8q8xrIR@e8v@ih|u0uO~+D{VF& zZKY;w?kg~|bQg`3uyCDRpiDndo6P7k>dsq6Pc}`V`8Cp}GS$N!VCVB<= zU45nO-X%CZ);A5B0LI&+yc^qr(1=#0Kh}1_0?258j!}QX-o$f}kN|c#;5oUXmhg-&W?_+Cmwx1+>{bm;p&H@@<}_Yw3qqWd0jd+&L=^3a0EO}@}@$u}{#CE=@1J zme+26dl$hl>6nK`+^O=N@FaZNxiiO_Vswo={>c|SHM3Ifx2_s;!NnlB=QngZxDUd= z7ECsN)U3yH$j%)>ecij+huw1URrqlvJbrY=MRiQt*m-lVMOdJ|bj91?(hI{7oQH`0 z>vy=tmz?l*TvmboJ@6AjPL@YexRC~gt}pIOPkrMkc|~KAG>fSfQqUAEHj|!jIC4Cw z!SWc%l89HOB%BR}_i|_~GQrF2Oi{rRPK%N5zPVRtmlm3*zBS|1Ep3cnv<9(pGu!1J z6iS(00KACW*sNvRoaOq#i}H{I@=wPo%6tpoVn_bY>{7n}_IbF^N~i{5uRJNfs+x1m z?3rQ*#m;mY#t$%iq3~uzMBkgdFnpq#e^n)Di-x0dNP+x`9XZ1JHC8Im!>h}sR%l;I zLc~nR{ozK#K@zp?XJTsU{?AK!T${RfoeFhtvcZ$g!Yh92kb_^E^IbIC{F}xwXPjrN z)G0G^@%zkl%gVy|FNsHA!%-E_$B?yV$M8*g{NnzLjSjPxaY|qpNAzm1omM?Ni*IP* ziZyjFD-G2-dy;3+Zb_@>1`RzdznD)yU*^rs0M(gnpNXZ5*IZyBvz} ztq76eU3h6mGsULfsk$e;bQT_gzg_BFyA+M1N{BYi$IDV-a8n$7MVzPBL(UPD>1xtK zq?TcDE$%;L<2o)W;GA*yj`g}9i=oT&SX1MRU!pBZ;E!(3&6s<4VZO|RZkeq7yooTX zLz$Rap>o2su*(KVFd}N^7@7+9r(*6MGqd2k+idt;^WU+M!z~ zKy7;IXzm4UE_KZc(WNY;;hf*AKY;k87C@N@SFChSwyzl3Hm!i40?1lhXHd)^k1FUX z^(iN6YH(9va}R^%sGvl{aQE1(z)2C-wP)+7k$TFh;8bTW+XBC;ytW2;vW0|1#8t^d zuPBy`30p<7DKmR7vckF~!5Ot=eW|}~&jEIux?tL7$Y;a7xk-j~(Rd$?K;^1Z3PP?? zyFQ90y5%dd@WJYM)i=sYjTXgdUM_|Q)Ic}RsG1}N-+4*1?1M^V=IQc%LPEmm&4 zjpc?y3G~q*?%BgUa1q?T2LD+_K(K0or4!Ka0fQTc0&h^Zi+hV#z)GQdQm8WFql_P)@MH(*`+eiHY+64=BeK>NAjD(7F!!8RpLi7Uzc z%~N1831}J~+VBY;mHXM{)X(Q-tL_*B6BI1AzP1fW6HL*4TX&Rouel#83I#v&p-}~3 zn*bJ;PyWeWW%`sKof)jF1x6;tp|nwUk9B(ohgLB8ytlc8 z#KmVfo0i{0%i47k&>#od?&G%$ZuyTgpj+NoKu`in@j$BP_8EXqG=HP~-Amtf0@|92 zh5=*RJWvIkTh^URqjye3DDF+mnS>B>fz5}M_8z-^Pze)5lmq6bMU72Ghmj3|`%8a+ z2uf&Q%wc-}7$~U%kl`&6#I*VUtdUzByH)OB&hGA8#$WCo|6`luLEBAD z6K#}K*g~I-HN&`@D{PyOC5N8X1|sHrx$axUDv;&2wW1db4Dns?O*u%NxV^{Zdl9H` zNveq_ZBpP2<9{t83Va+tOrH<`8=uPunFO;my0COfG1oQ-ncwM`#jWPdHW7ijXFEd%jCpJJG5jJtYzn5u7S zQg&|B0y&yCsX?2Ab4-7K4s?~{PYM0!ARmNm*gu`!{}6%yQ%|)~lF*#U13(;Ipy?om zPyZhUi+r64SSfope63)Ub8wB ztmr!bu$O>`52PHf>H+=%Y$IUx4;%8t-K}eTe&`16Y97I~VpcVP#DCX&Tjj|`n?&&_ z9bF+&lbpo~NseCCk@Q+}D1ZJ~T$xLmDo`fc?u*tNQLw-=T~b{9C4Q%({MP!nDf!)D zzBn4RZcc`bZf?i2J9|zvR??OM=rS|<0+bY0qiTOp$D}1yo(G3FX^CpD=HdtnN?jE^&C~Yc3#IoVT z9HF_&FH3dJCZQ*2>25Peb6ZM;jCJcHhw*FG6N+N)r1d1y^Ejo|{h2!Igoi3MtGe$6 z=H~NVLP}=|iQ0X=NJCBeF*h%Q!wkGxZ87z9eaie5Y2I}u`wN^7sZ-oyh1qG_YY9^Jip z`!mPc#4^vz**!5UdVA#P;}0V1FSy?&I;Or^Q;}IrtrYU`p%{tYB#AO9*+b|R*M2FX z+6y_oo6jiTo<;%P1i#RhVqJgsCZr7; zvajKgum&LNbX--nT*L%|Y?ftjhR`-`=+ArbcZMIwL{`xGes^fJP>j%M(U0~r=t3bq zo9iMNUvB&RIH|v`>;Z?nP&W2sVsGHhlN99w7Cl+AyXem0e(4}=jkD!(2d;YSiWOaR znNW@@vy&2G#iuPmoXex8mIBA843o@+C#J1SHEAdGc*Ohp2Oau^x{zG87Y^9ZT#%f~ zq6kaq(E=gMkytb=0=UL^P!yvYg$}lOH~G@{2#8I+2gK(@y6TR_%llezSkaWruiPn& zuE#9jp(GykFz+L2$VQ+Y$t$5JZX7F4$`F-ZxNxZN+yp6_Pv1%4qfK;UF3wFgq}VUC5TlktbC90(MXx-_(<|n+a%t5}_{?bdQSS z_E)`y$|$;1xqo#stlFUEofK?e%dR=zt2QfEv3u=L#M(u1UBU@~=m_%>!(-|1vvDNi zq6wxY)1~ubWi=uf^Ppn%OIB_#LvZPAGgoN!^8*kQxEDtgR065#N^eq81D{WqNR2i9 zKL!P3w(p22IrpWKSa61Jk;?F-MoV0zZ*=PY>`!`QB@UX-MW>3x_4C!D^uas;Z~iU{N%W*hDiF#v??Xe-jMzFxY|~Sr2|F_xP$$jC z7A-O@&M-@<$E4`a(FQ1u1ZwdXouA{qVl@6Cg6ou;ZvAAcwtopO#ShPO=Nj!WbHY+}jWOEMAXNLIS~H}2yAwC2!Efsem#qvE)(^DOD(obz$J=zw07q+?9+!X^NC$Xyix;P{f*&`;D zLljc?i;#yNbLw{qdYg3^FnLd;52Ly7NHYLp;w%xo|~{VKqKj(aMo zO}^yriXe}3F#hv@Dv!fuUgJ46{OziSW+N+pgWqfqyS=PL_>zJ zSD-^cu-1P+;bgKyn)@9#mB&W4=dY2aPYx(J1iW*iu{VtP6VlHTImC41u8xk&RC{Rd zh(quvaMD%%8T9Z6ZSh9eBegPjj)@*;f5f)LRLx z#)u-hQk41|TjI+TB744sW|<1qnlPiLd3%Cqy~DHeL|6KyLo*Y)NX~rLYn~swS|{LX z_5(DMz(~dR{x}%F@Eo!Yv@!C_i7Y(Z8Qpzjyn>-@N*=*Jmr@o{mCsJU+u zSS{l;kMXYGKc>6ao=shKjyKlvUDk)CS;I`g3Z;E&jT*|j^UA@xA`9%92{JX=dID1} z@8Q|Fso41Y$Mk8em>`)Q=!UmE21j6_B7lh889@ArINqAWp{Nr)e>|Mh87pzp6qP3# zAG2+tZxM}}Wsr)%eLYM-_A@l2`wdss=Delj9oJ3YdOGRK#MXKmH;Xh3tIr9fG5XYyree;4RTFF=GEO&-h4zN$Qs?nsVy@8{iD!f~v&W)p zk*zI&;G81%wn9g@`*pPb^YTQ2o(j1P|N|!4gOzFHrhbI7bd{|eLNN%Uu%EU zhS1&%=*n9_B*h`?4Rnfw%uo}N|M^59{*N8)e=RDWcphB9w|TVxkLOmx(jIdZYFN19KTiq}U%|!)9CZ^^>O$zsz)|-D-T!~_fI&2{g*h?KlR!MjleE(^j~-1|F=GsoH{~=!Gv%Gc$qi2dFnl_iwXb z{wOj5YY~dF6?gyofpi##@%W8k;6*t48!<50wfmduW3wsnEph&z?E~u|E7z9uV5{iA zpv{44c+c<5zylY-#}9>;Cw3^}w_s13)uaa7+S`{v7A96{r@uh=wn3{l%UpW^(h>_5 zfkpTXu$W-?{u}$MAeDF+CVt>Q_cLK*G5xNrzK?VlA{Jbw_FV`3m_~ckWc9(>XY;Qg z5;38z_DW#OGWPvpjxrRO^HXG?|6vaN?^T$I*}yVl_tLQZnW_Q$kCmf=E4=8Jhj31c z?%Bvl@nnDnZ#o=u}_b4?@R>HPi*cePU2EQ1{Lf))gcJdY2!%>UGo1yXyM>m??FOP_bZngZ`YXhreU{x1Py6UL{S{R1O=9_`pvR$4j|zY`Wiy;;{^ z8+8S9ZPvxvBH+?%YVNZG)}gpblhFoE$HoCoR?L8YnEp>j@qs_zYRde09k>+a@7w`D zK`8JX@O@N{(E7cr;5WCU$02_J0DNbF?|`q+WozI_+M!IAmn!y$fR_y5F)UU1*k(Bh z{TVPA2;4>Bb;|AbBjEcN7ZVJ&PJhlC{Q8T}2>1oGSdt)4{{mYzUdWTO1)GOeV7njx z%N2?d`^6)5b+#b)J5B^fs2igJk6Ae>J0K^7HywmAjZM$XYXMfR>p)rE1AO28|F&g7 z9^&uc+WY^YOaDt1$EKB(bOU~02Cl^ta2SUKAtU5>p2nsDm_P-mT_ZA+&69|FMK$?pt%MRmecH z^Is|mKBgS(YOb}YE(;5I`foJQuuDIi^zH*0WB+3dLju=8WQOo=*m=5iF))t5fRmq} zxbfCpW4Yh47Oa^8*MD!~sGRrIqpG+Atl~UiYj~9HUMr~L7cc1B2&;|t_uJ>X6Tq+f zFWvZeQT$&Tw@n$>j(&=)9R&8rzck_m-U;)f6U3~~tx@g$4sfPg#tm{eOG{0%R2^CY zIjPtehrmr2{I$Datg-5ij4^{C+PB9oh1Azu6Nl_9cS25>LQ|yc&r0KDuaN}kf{YBi zKW9NEy9ObE8747()Fwi)W0f6CnP!XiP&Q**? zSQz{%#E+I=GAes6ZTt1NV>iDZXZDZfu<)7R-sM8jUzZpV|;#w9e%nKeN`gDHj&v&lFd{fyXUcR!Ai*3-;e zedrw`Vop08**(6)tg#m+L7P9#>i6jqKPJfx@s-vL6$cH^GNwVkw`G?zzE*wLqS-ff zTO$672aTHL$hCVCMfYwLMHI5_C@Go%{*@dRQ`5u}pb#k&rdMvQi6PAdUMDeY692Y6 zq5ZD>R#+`YGk4K4@Y<(oxuE=PTs)BpS{@`q_Z()uUk`U52(&x7N1(=-LDec2asG=0 z4K}g72BR>n%zScKuRk`J;>RVWO@z{*=PX>jl z$qiTYpQefEJ)AZ3eZwI?XQPPVK>v;Q_$W}+3L)`<0Dku7koZkJTo z^Ku$ZjWe8ktZDP`RQ`^H zT;=uL7Zo65W%2vQ5#Wg6rr?YkFCjc(^%8vS`OJbJF0k8mU>z-6w#2nJeCV! zx|BDK46+Wsq^LdN#xVxjvv=6)Yt*$5Bc#hhwrkzoWaiNS2dj}hEX8c;hL@L9yG&xnX(Ss^@B~`V*!fp0N zMe*=jh08UsjvXN>1|(^vMZ$y-`BTnRNeYttHJpXGI5qKSG1{(k#U;efw3d9^G`YCN ze0`-^DW*xwAf^<;87y%6xwW}9gdq%g#r@Tk(>`5#gmsT)Z?i?^&3CrDMVuHPR)E<*0Jz2 zL(G+V5bnZF%iYpa`4sa$B{ybpmCuX#3p$T#jmdtEI5gCV6luqcYItEJ$y2o-Guw(tISM1Lk?A5 zw2}=edmZ0)M2Ie}RNK75l2~4;pUj@$R!$<_lUT>jG)c!p3~{fS52VE6zpmAu@0S3Efw_-dV&!?ChDn3|Fcd<;mQI75mo zhZO3zso{k2;O9Orf4Ga+azVC2P}4{suLb(C@U+I_K5!5SU~Q;PSnUWe=nDMnFzKeU zh*JNml7)hg)ePeU)rw%oNsL zn9>2Zk)T6YsVn>>DzWUtUoZ6i zpFiGI5@A014f3q?4UJ3=xX);0$K%<19efg!f7p-C=}dbHjTa=9t5~ zI0l%!O0v{F?CO7;SR;p08hs>pE^WRGk>yvhs1Veum!kftX!?UcYvx3b_69H4^hFJh zsS`>*RBEp?)|Fg(adDrZNBN?Eb-AzMJecI-?&CXP6Ew63ae>;TCl7*ksi;HPw8ztM zpB-LjexZaGd=OS1W~h8;hm4QNAnwf^?OB_QVjr3srB`qMJeN=`7@Bz!ceF4DS8yu^ z+>Im7!dG0=isHS^FQU?R&;Mu!7E2BB;BNw5#KDPUJ;LY4Zt683>%L z<+yEXfzi6%!%t^25`XFTVAQANmVLWDcU;>|Vb@SfyU_#W?M5rdi5j$=gUD*(UY)Vi zi`j2vCL?Czv-Hgwa{`qJ%+-f-4`$ApTsJQ)WiZJndhm6f0{(H_=bp924mc1hb<228 zSgNM8)XSBCY%_IQcJ?$jhp`5;eI8Gh0%=n&OL@sJT&rHo3}b+Q4T@kiVQRF;>s_yP znM^Y>Q6dPIDnyh5==mPypq}_X;TW__^F;d6?8OtPmqO|UZvN@=!y9_7+ON$Gj|e|9 z?D9%7TxP)|E@8z1m+l&xw%FGShgV!BOXsktp$XX1ppITqfYvu(3D)_Jp`UFwzJ}P~ zNnWv~N_>ct{UMVAOPd6w!RJBZCq!U{g5lS2YFJ4C@tGX_QKUHy@M|GaBdsY)z1(?W zDUH+R1nnPenH!03j8)#JgE_D-7FoV9ytYzq@2fs#$X8I`!^jtsJY_B)%-~4x;V0Z> z;82YePP^)?G#=NdHJNelNudTEOgn2uigY>fu5Jvgv@}k+0%7@P;5PmY<#PF}bA7$o zAg5aCNK&tA`Bne*7}=V{zA3s>Dm;R!HUYZ#F5CVQ-SAwX<6UYBFHkLeft}Pe zQy}?1v7MJzZ=)4h7iDU)&eZOMFpgNCg(dl3N})FIs%5W>nkbXo&GYZ*@}F?DF|x+j zJ2x&g+B%ay4v;?x>gRI-pS=E+c};oD*SXDg*4hp%xycm+UhIgrk_&SwqZ&G2PcOdn zZmoQ=+I!IO0PpQ)Q&OcKhVehnsuD0%YrHU7inq0YY_K%u<}?8}&{hx>-WKQ>}vfWyzSzcWbxzh!Ev!omyAdh2eW%Z2T3y)%`K4g@ z88W9JV@$QG#7sq+$loS7&&&@G-tW#C?vIg6p1+vI^-8zQKHH^oHZ}#VU7^l_#4ChT ziU+0Y)w?4##2`h~m`^A!=;vl28Kmk>4&${5XPVawbL!;TZ0}7qd{n!5CV-0uAdn^u zb&+hU)29U%UyPRSm#?arMEW*FI*n?PPJ0Avjy>W(TR7jvJt>nrb&^tv&(G7{8(+3k z!smL?z5K=a(82uH=DU~f=z0m33h5t+M{8Fzn~cWA^CcpM*FSHnr)w+&IM?*BAr`k2 zt~W_5wAz!dbr)FQ3rjj-$q=Ow=!V{@8)!Du36rm*OE7-%oj8J)C{zLl4zZW&XH1xP7}m*YM4 z&QqgVqszj)$!#uuW+=N)GjxN+=w`zZ5Za)dH%bbeWTg7_zsIJ+{AruWUX8EUCt5z*VP+ysb zvWz(X_^#SMHI>{ik4~Wmo8y=2C9wM&f)z%5^9a8ayRLS2+^wWZN*A z*$1}hnH$r*uAD7BBQf2DxT}gr{SVsfk9h4}8N@iGc<7Tc=a=f+PFT zODBR$|0IP2jKiLbU}r3_Tt)%w>71whid)93(2j2{k=n)smD)*_3k%Rl5S#+9s=3h% z=Msa_LkApumdkP-1;g;YS$6y5gC|k5zHyx+Icd^pV zc^VIer5bMF4|Ah%7)q}v_t}#TH^>ig!g@9d{Za}w-7W}pnn)?O{Cy@+^Q*Ms1N%U$ zYp2rmO_Um!C*On%uS?!73#rW`yDj2{g~d+^iq|ZsbPl&2g&E(M`9kagKA?DHU)dP@ zTpO=P>&j@j-WYrP>K7!6bffq~Q8t|0sd1rgx%}qvoE}iE?_63d*GZFko4TYYJUHbi3?^{Z+SYHK?j52WWmsolT)_(I z%)L1b^5IG?`I}A!>1N8(C024>lF!Qsx{=|vO5z=GfNKi$Pr%c$T(Vd8 zqf7V4o@pl8FH&M(X98S5FdOKAz{UnSSqB@^a6?>B!&zq%(q*l~=$s`(reUZ{;|+&< zjD3UL;>q`BCL7#lD|y5ln3|#+Y-dH18-mhEmBstQKbl!YX$v3~pj!>S@f^kXIpb|Iz)xz%^;QVg8=-akapJa|}03#o~WDynJ%g z;skzAT)4uYK=Y2-myar;UN6=w-Ru$P8Os)De6b6|1>m*je(oF5`Rnva)HQ{A?_t@J zkBVSx8$pPttw6b5ual+tn3ifphve$g{)r|6}j|SJ)PCrMR@T4@3Z6hdA*a za*Or(o#{u9h~B|1x_pNXx&AHpaRNUSFb-(eHLP4S&we~w8U0F!#+7{ZCsq)m7g zNC00T3J6fZ>#>jZ=E8{&9|Dxcw&KT{>2y!V4$>b0B|py26aYWY0|1kAHy1Vq`?_sK zwO8oO)ryDw*Nja~P5m7~@wc~7Xi)K=-sEou@u$%sd&tZ0auVud%`B1&Tv4emE6X6m zu5u zqFdqJlH&rGnLDoY-glm&nsVp(IsN!<>YV$7! zkz8t>J>=wAE<(McILYxUjcSxDqfC-oRECQC;MDQ5P%gEHHA^0OT)YMA9T%F&O5ew6SmBoPe~L`)#Mz%wGW9+S&8bXaRSl zEe{@>k-MGtTg_;d-U1|oEo?Y0l54SPTeogChpSAjF`8Gox2!R$O+17cITr|A%rM2> z@oZsz%lzIW3D!Mmb={QhT`9*IZLpuI-iKLDx9uN~x@wT^Npc>)zqMk-fXDvHz`@Cx z4`}q?&Q7P*Wr5K6lq$cqCE_8qiTHmI!ey(uz4ZS&^Z#U0%SqW@DPZ3s5X1=!e=Ot< z#{9KhI0jOx0WRisL!xs??V;*+7Jrpw8mRX1fCguxuf%Rr^`tH5c9Nd08M&?#7ds1i z99Yyk9*J(N`qSkZV^?2aUs?9;5BR*MlHv{E>&tbEj?3K>pH(R)>tD6no&W{8a;2is zdol$zeJ<+B+s)C!d?DANYt`nf*Mzg$2!$-&k6HTs7klAZZ98BiVDe=9Z<4FXSi91^ ze{yI-Yw1&6S?B1yQNJUZ+_@kFdx--i|CDRYaQL!?F~tCx2Mi8_L@jjaG2S(HPFpb$ z<#g#62Ia6=s0)TVj|Ea*RFSi~$M47Z9Y7Z!0a$XrUnU0Qo3*brw8C<|gW{pbY)LYw zd#nKPIA^YK42@b_pQsvIYn?Cz21?>=_W2HmE&eeU3(I-Tkejp6;X`#idopK7lelYo z)f=*c<|5()6zk>j$kmu=BiZ1$w zFJ1lWKGm{}yjW{-x<(^W0fqHXTDd`U0Y7Vtds4fs;?lnt#g?-({#}|>poNTD@GjdGfjv%T5Z~gXM{>qEl-q1F<|8(injMxW44?PUqA8o1g zf`p5&oy#mr;~}%7OWG2Ej*r}w=|4TDrqY%&KPu56rcy=8o=-Tg4Bse<|I3a#l|`=s z4lh8Z9Jrj#KCVFlD_t`a!h?nW4h^zuHFJ&J)_N zGFWt8H)yM0GGQV%Hl0U14#?pXRTGw;v^{aQKCNW$t(CH57fzj2=)Tam(BX_PwCQ}n zczMCLUoxfyl8+p=tP|~8C~|Z+bIr;sApOl6(^aarnC&c+u|rxpbiNd=8K76zT6%$v z(9&6YbOy)6+7z6+wANrFPF^vRo4(Fk(_q=|Ri`Oua8+1E%4S1Kr3{d3M8g!8yTj+& zFa1B`adkw>Ff%rP`&+r~?wp%n+s^+m^8=b_eJ91)r+RVb?cV}hZ*BU{nYI7$>=&y#7b2)vGt%C`P?yD7klbZbkh3 zZEvofTTrXN$oKF`z4wt<>i!t{+KS~?*UbhxXVtEWeHP)a$o^UVZRyiT+X8wNZ*R$! zo_?!Qb>F)mwtbK0zqhxEv`_xLcG+^xeb0XUW=j@i*Z`^$;k8eLWtZdQ%v;|!mA*ZF z+qKYH<6gZ++jNPUwqXd{gUtZ^Uo2v#= zS&vdYJWKmi=Q!JKzE8EbhTFoA$8Uc(>zY2R`=QFb>vj7pGI#Ar-14@z-|2Ty>24nD z-3Gvo&9hMoq8YPyoHZ6v2F8(Z(i6$9^RoN(1FYA@q_mvc^YNbc>-Aq&d_J}`R-&7s z;VdvcBC6RlzSYU)vqadoyAoOXMy?+%XS1#NVPzP1 z^!%2W@3*ess9|R&Qh)s0vCuy;kEJ6R4#=Q-Ve+@7clCY=l{9~BFqW%MT9J8wamMsV z+1uii1(dz-1W1X;+q2C&_~ZA{6O(KA*7mwSo_52Y0rF=@Nqj_r1mHt!C$=dZt4^Z%-`A=lRTxgxb6=9ic6>SSTq zV2_e2r>RNzY%Bf$uyY{?!wg20ddmAt_x?Q_svhqR_sbXU%@;iCeg5cUL#?BqH=So^ z*inUI46t#4628FJ0xg<{D4`5$e+@TB99;-n3AkWY&XJv3lT2KJ-s{%2P5?Kil{|xi zjomFbHZg;no~onLLo-+=0lRxZw1gGd5rQyPJfjMLF+E@sIjgV!XGr99=-7OH?Q&r9 OWAJqKb6Mw<&;$T(S9)>) diff --git a/notebooks/articles/simple_bike_repositioning/TripLogic.png b/notebooks/articles/simple_bike_repositioning/TripLogic.png deleted file mode 100644 index 68171d8ef247372da1268d46240ae3157079b8a7..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 131866 zcmeEu2UJsAw{Aoc1q2I-AVmeFNKph7kQS5@1VRWRT?M3r1OiG&2-ql6l^*H+&^xH8 zlnBzJ2}LOai2*60g!Xn&l=GkS|M$Lo-+SY}G5$Sw4nu~`}ZM6gR z9P}U%=zxa$Wf%ywHyQ-m&9`qa@JU5plOOP7mn%#S3@Ys4oC0q4SSxEOgFwaM3~M)O zf&2Y;)D2xhAf_hjzg^AF*_I&CW`@RPW%ymQZ|ii4T%Sy{n2#^&oIRa?u66HG#^6D= zv$wU(iVRj}(T=A_**~ zc3eyF4?W*Y!+-hr_2zMl#4T z{#0Vqg9Fh{wd1ibhCDvVzml+fl_ckZ>G(!P8Vi+domEfWZ| zE!C~v8J*s5jOHJwY_cXigWDuXt95 z`eIifFLZ2;c>>e_AwJf`poUD=P6YHs7w2Nv5IbKv;fB|{x8XLSZ(!o$D@QRUyY-~^ zGdr4N_G6}j7|t`Z2ranKCPmg zM(#mg_G49lD96%Ylx*xQajIF!l3V|NYxc#JSDvQ>4~b6D38ENDae7uY<1BZVR}aES zhlJHmFTG-Z5LqKDR3}_vBJoT!S7OZq-g4_^nE4Eh+~v~MutJQpJ1?*^Xr-UrR+N*E zVrth1`jfq{ieA3fio*qqG`uH zrN6i24Fq~c+wbyK2`KFPm#?fGqNA#etn`5c6bTs}it^>+Lk8V`?~T!*7A#MDG^gPx z9po)eHIm)fL-F>!hbWyxnX@8-y@pEU|A^B@+5^#8_BF z$b%=hIMMHt@VT<^@&0D#Gyk$hE^uq}P_M%Oa; zUGFIJK;OwS!4&Ao;6?gWRdzsuUn>iBl;e9KzIVrlV$%Hv%Wi!MGgb}$8h6}I-S5+w z+*5GPA-II0nj)WW^+sKK#_LGL5}E_EGE*R|KLevGuV5zxxbernAFG?t&1$RYBZ3+i zWYsq*jeZopls4mjY>em+i8nV=VK>`G!{yz5KS8K*fa6sPK2==+=5Qp2#D!Ho_!zSZd99-a(7@^v{Bd!eW#M{A0?h#LZv z&4ji}^ySiQ7q4LMGtzdPp2@-~trl7(*$I$py}p^3FdA-koka?YomC6c@VHtQ3MMlC zHO5i}7(Yh88}NyH4TNvv*TRZ6wNK@AokNd%ua9r3(>n?cJ6V);_GDGVg395(lVwkO ziY#(g-Y%l!F+%XUxHFlm_NX?&N(EnIR)&zsn`SaZ#a2oX4 z=Y^){u$hKgHFWmCt&>4LeJVHMHVt4N12g`ap)LjcK}+K)4=v{Mheeyqb+A{@zShbH zf35Y20JCw|6Tzs$0le3DX!}zT=-)`{FJb+E2>9E5cn72cfnIhVY;*Dk@bMmy6h?7#UXk}F zo@UZiT%XceTDk)QF)EyAY(p^5yMsS_3<(6}I~yHTa-0g_cPX_WrkoA_+2ch{ zK0rP+U-aNI6~=??Vmm|5{p|3~mlJ?|jQsx|;=gC{KWnIO+_aM~eUg=v8~tEdP*Cv7 zYVYfn`H5ycORYPJUM%pzgNuiO4C3z$?&`#7xZ=>JP+oYa#O^QgQudTeoBqzugC=_q z9dhBLrWb$p8QxFr2^W?&4gNUGVfaG(9$fu z7f(N2&KU zw3WRFp7~qv3bXWiL@Vjpo1`e)g0@AE)3tp$D#+iMH|cU0jf6}o5<0|Fl!Qd6*}Yl* zRz!$=?-;q$dEY$E*OA#3dOu^#%Uja(blEqhivu+L>)+^HJUPv5l5PnHAtq}-+@aS5 z2!UVfh^*G?-WeT$fe2klWmoj|?pXC+D9uF07F0BygF=FZV=7|IlZTF9Nrf9xnTxvX ze>Cc%?+ru!P>~p5#>Ibg_y5>NlS-%ve}NZsV6U5+*L5=^G9P5;Z@# z0Y53fMvi45C(O3#FiO`haBScC81$%tZD&SJ_TbvOK3DkxS`DMqTB+%t*$TeZ;NjJ_ z0wA6Gmx|z54p~qPIg`FmG>S3#5&A4RyjL^!DdW*cPX|Km92#Nkl$x_DuO6pCjZtv_ zkvnM9jGGU^NtO@b`zk|W!Q4L2E?eny2(GMNdjgxV%Wwp^n~Sg?A@ez&?@s}o*DwW^ zw4edN)}YtVph~GszFA;Sp7mtTfEN;A%$d&bm`M7FNaE?)IC1OMm|O%G#jr;*zF92| z6-ngpll5IUsTba_S(d~n+(YscWa*Ym0b9xFheK;~>c%`u$7axTi5s<dB}yoyePE@X7QON=ch2)lI&@ zoLkBPuXm}*kP8+UF={;k zYH|&)B{up!ZeA;l?+i;7e|QXm+bE1mOo?CHTJ1ExN^AUBEq^RbBkKKG-AWpibi?7C zgUz}kpznc?MDUtM(_3AdUz3m&k~-YR^K_I>WIULE;{gv!<1P0@Ru^<*Nm!J}NZV?e zg&{eTH|(;F%+!p1u|x*XdsYsuD^&W7EpN;D_*@gxcdoB4e8lt70kG;B+-y_4-Cjsc z12O9viiUrRPT;L7c!k_0yHSUDeHf*$fKpvsKE1oB^TBz=yPExHEG6LdqNC20i4yA- zLM+~g0RC;{t?)l{&wbHi2aPgK9@9d6J9PyapKb^yj`>ooSb$uwd3=AU61`{~9Ek{` z5#OveC{}uJs9j4z>&aL_QaaU(5V(E6!p2laLJ zCUlZZc|3?eg8)_c!=uUGU}f!Y1P#9*98RU@fa%}DGY?Zf1yMQ2Uw8&^2gvRW%j*XT zAF&TkC@KlXS;}ZD3JPU4^bPsS#L~M)&Ir$`oyckMeHke&BBUPzjuplWz$?>RUHQtc z9$o0*FE=c?`X5A?jDSeR_$xuOi%}TC0o~L{lZwR8$`oadAC_I5cvemMK@$&ZtZ`Kx zvrN25CjsSnbO@pllN^ZR0|?VMkM;k_T0nv0p3j7wqFhZ*O!4MR9x{$Lc_(u)pj*kI zkrT7ub?v1JZ=}h~alcBTg|HWZO>HJYWA+I94zf1zPpmS&k&70Vr(-bGJ`Nzlu*+NS zF?&8C;vbAvPs6(5HG2y5`e=B?q1Kg#-G>AZyrFuB1^O&6yL(QdFV|Xxkgd$khmBA6 zi5}_p8!*fESwEL^gLi)(>qc~sDgfqqb^qB&qUmiCqXcw1_H)ytcw*cL z%$HLo58}nu8pGaJ5b1;{+N0sZb3B1@Wr^$E=jMd0C3PkI_1Igj)@0c{`BI7+bxxIC zl}KC|7K-VIgvp;eHKHE5g*7dmq9k`BUtr;a&`QE)c;=e;fQ8E{YS$I?p$qZJYgYkh zrrgw;eCp7!yLcTTasdkI{~*Bf^&;kg=33WfN_FRkX8U}cINYhE<1IJuGsY;u)#Lm7{oF8H zb9NxwQX=t4M!(Pn5#7#-w3q7564i;BD>owrS!{AbqtbdJ175V9S>S3n?=T4eFyH0N znj09FcVCoN2laZV6+{kHadN~J}(e!;Neza9mZfzQT1+5Fl>0obfD9Aw^71`nYX<@?RD}z_@SU0uo zOF5~+3-ujE_Sp@sqon-wP{7O7BDX@`CC%o)u8Q&~wPQ?Mi9CI-$EgK%!iD-xcV(r{pLCO0<>A1XOg<9nf)udbt{cj(G z1zCLmb>zcLy~Iv9hJNj`m|0V6(|;kmp&zXH`ZCu#Vc8%e)oS1Y8Uc>61f?U(e9@_S zIJ|h~a3)2j>`4*6P;gQERF)z*`y(%%4{`v`*4Q2d=5uU>p4Gb=lS5h!$MjUcAT3X5 zWRQ%q3&B0`DgpRB5c9m}_7T6LO*Rn7-yw0@sdU(QA!%cn z+Mv=lbpZVS^0S0}{-?MX;b;o*{?ufMs(>ju&nKX}a?^G5 zGSIG@jazq-*fNY$aD751Wh!NhF59LTfY;C7t?JUyRu$R>QrbH($tRGuwINH{_mf8f zp74M5iYIBzG8)+UO@Fif8%T*~XCoK+)C6SMGk^_AA@wkAgam}Tujv1SZ4W+!-tGYv zr26wQ&`-+0DF&PtmG}fk1M;UAl$3346{M2PRUfx_oCgi{2HG~G4ZtTL(1eim&#wie zZvA`*){@vuc@h3lJg+M#D$Nu@& zY3d}NqIj#T>y^ zZB3fe+iej+F~9UBruy@_Z@-@Nm)~?O^%Bn+XjZ_Q17mwh+fcu9sMd0v%(_~Qj9S&C z4!G*>j@hDqK)sGB|H*>pC`;n|K%j`HGdy?6b<_ts)wby{;D%9bV>J@^DKXM0zO^O% zX9L*U<{Gxp0$eM9l4h$)KtE{9 zK2D@;GibnL$4scZ0jZCElnIO-+376|)b8PjsI7why|?)~;O531YWM!aKWYymbpA9h zYWL79KlT{rw%I{y3$Mmw6&p_g<5Au4Udf)P&hwva8o1xOcpvPL@8@U!hc55vADx?Gro`%fJ|7|CSLG~+;|8XTet2k2owEXd{?5wmr%jf) ztP@$sj8rj9lFT!H{6qb#Y%$9-)QzBC$t_Rqy3pCS-*EtlAb`f8F3R10YnRPmj?igR zsyWb+43Co6V4FcSRpTtnjC_5J(Y;Q>dDrfw73H$WRx8~}4QBv7xgMK1fKceox&8S? zN&5Z7TUFzGsVQG!fgSAyN^bnk=l8F~7!sBNu9;DKtIM1QsoVfb zhRx6pt}F9oOsTb8uK?QaKS33=GHHn zAq%=bWoC1;!L3j2WX*ip8_*Uxwi#`E{NrDvj_zhP`Q~K*heyMj^Xw>o!L@)Ek{jn$ z8F{~Gscx=s9z$?(@PwQ1BR!^(Sh_At8hu9kz%r;PFQHAeG4wm$T2mXrzuh62w|PyX zmkGxclEB@qMqbJ4s~jt{C=vOhnv3eK)$H!D0v6gBJu`aUTIdwnZ$xXboxJK+nt)(@(W%fc_TFX7u`Ca}hiq{^H&=xS9(&j#T6jC+2%NZKzKNj*{6 zW6s3lBpF(PR#IxTzH&0KZNBNzL8oeP0IY;ju|+-js1uLm@GxD$Ok4M&x1_k_h(=FY z-Gt+0y4@pLWW|`UnHx4jUwBf$ zM4FcoBeJVj<+`80+y|)@?1gkQUs?l74UDS3Ys{A?st{{SB3HBSp%3YZy`OD59&#q{ zATIdpnfy5y<){TB+iNaflBN)ZORn8OrqyDW)E@qZn4_REpui&QTd2i)YGt`OtGAdA z1X5R_hAowaf@}o%_!2EO6bL?${hHKw`O9PQZZ=LqcxM^nei-7;`0WGH;Ixc^>=%{? zNBU>g7-UR9FGw+ggtT$iFg_Rjezc>#nc#!NVhh67(UExVex;mas0!W5s*Amg=o1i| z;W8(Q@4NlumAK|Q>_%VH_`$Vupf6x1gPJIMr)oZ@8eo&w39c)#&xN?WI(}!DdiClA zj+bS!iWGE58TzAPDM-%8#VAV|cO#wCy9;3fkqqy^K*M5GgmUA=){^(2fmQz(K%jNu z+9-*yQj1yUyZKA5^&#y_Yf}ICjSH#8y;Y+aj zzP%hkp#!heaSeJcwhhm>A`|lZxlvx;iSR)c-UJ9KK_x9hvIaA(#wKzaU)||)$jK{1 zL=wUP3T8pJ$mO9Jge)aeTMk2l?jV_I4=bP+i+d@pD;WfHyN3HhC`aG3BNb26JKeq% z8NBao$>Yl{Wr#_?9}o44#$5;rt5)?`S)ClZ7cF97h00{8tcM-t$Nc~ohWqNF*)U)?_^1IEd|F7-v`Q(b*Vo`am{0LK zL;+IR+rUsyY$@Q8_|#FT71?t|1`pWM+nHxYN1fav!Fy=;HcZ!Fpg+6d)qnP7pFE5D zMV67RX;G9JMsenPJpj+Mme|Cf5x0y4_n;V}7o4S_>uaCYPV>YW*bSppWva<4r}3Z4 zH4rs|)?R=U?7s5TFLHMH#qgBku_{Na1D8$ycyS`_Aie9rV&u$smmZY>RU3IW1f0?$ z$ChX#sU>mQ6|WAnskyT%e{hDmTcJwVazi}>wktd@3M>=~ofBf=WQ`XaSfnqP0@h+* zYrpb40WcZi>uvWL_#{td9-KB+e>X9b>@${>mqk-HRvR}U9mx_e zW&+ShK-1Z$D;<}z6?K97A0$jpAUp)WX(`7RF9rcx2EA1J8Q|@A1o#(WUK2{AbW-RO z=))`~*_^Z|TK{N4H#gV z`wSs#FN@$W^+Y&@Sp`wt#Rly_&EZ}y}wp-2ILIK)PDls ztG|G+lJe0Z-&c02Cr(LnsGSU8I0U}=fWdO@L>kMqi#z?3$S;rYvt7WQma$W{sp(;k zfrp*BV8~|Rw@$9iKdVDI&HYjh3g{lg@xz{ zOhJ@`HQW1E%d=zm&yZC1VHMC8dBakQM2jQ4ZYWjxEalHL?G=kw-z&zn)KZSvy=N2X z*){UZqvKL(pZQtdS0OWagl!EC;>Ik-P4>O;TPIbYZKZY{MV`lM{OB9Ker+{o{-QD| zA$A^%ZK(AS=EuM9lj@BvY-sJ$*mzJr<{~Q9-Bup1D$ie;19UDk0^~5M99y{L^6)cZ zfif2=?g#lRH-*wi(krk?SG`mg9@|g5%c&t&lr3fy%$xIFnOOVs%27|g&Al7F2pJiIiYlyR}T`B8SPnaEL*MO5xX=Hn*)MEF-2DCzWttJ z#~>qZh;dsP#fVwrm)Ke23qQ>-ibEX*Grr8IOHK`K?s!C1iKJHAE4W`}o&B(7>KKo! zC~tYPisQBfR4x>>_x@i(&e?215opAS#|F_FbpU-x>~*O6tO^PN{ho?k`>y;dxFUt| zocBP9kt(sarUuH#o_&$!R#k+$U9FRxn8U0Kc^-pp$mNQB^(35-de7-NN_A@4F@O9~ zxyxaX!~x3mG1Q`I&zcpvG+dQ;rTJxfr1U5j#VO zWxVYTr)zh`3ra$ohn+>U$d5>s;jb$!ZH{>EN$vDqUQN0b)>vv8cTmuUn{WS!~^_Vb$3UO1>A34|B#E0 zaIc>RWRMu)UZMU3rH1dwu>N0Wfd3=XT1=}C8xR9qS^$!$V~fD>*{SBADtvNcN5cE_ zGmz@RE!g@a!`}YzznHK7eEr6rKha4T%G3|x7I>YLw^GMH69eO#U20IE;9N;{NMbbd zuSxX(XnvrVGwOm#fCSL#J34(M9-)|1Hjpg>`yRIseD=&N!u3DX|)po^^OE`_wdZxCaF?#Oq zjd|<)G~k1?aERMAdc+yrLF;-ii2_Qfm9zB zoP91X6mS@bIE>EQHl2EVA9CT;DWGV<)t@zim!qSV9x4e51;8k)T{ng- zwnW+*SwqZF`1TsGwKK#H=Bj`2V7KT`Gk&SgyG=!Hi@N1nz}IKHx7jlP_>;X|j~`s@ z+~W9vubxQ$MWCPUfp|l=V^sG4-LCp>6Q*@#XtpU7z_+32gJeB{q7kDlS^VwJ-)rLk zmk+3<0);zT+XPiFBRb6N@|%F8Td$|$0BY?e^cS%vrIMY-cE}}lD;ET1O9TC#?cI() zO6>;;lUFWt+HA1}pxgu0Nt4=+IsYiv|JiQeE@+$1LLF?l%@PsXLH}GluB(ln*k;m< zz}ijaM%PV699i5O6YJn5ROKrwv)C%0QdH^7BpUd(uXr)sPiUgh>-2T)f)*Lw>p!*m zSE+|y^)k~zR=(dq9%$A3h`!OplbugJbsvvIr{)Z_+9RXJHM)vl4w`EOo z&uTgr{Tj(dq<&k2ui1yVQmQusC*q!#bDkJD?=`UENto!AxNf58-!-d~d_)7wbN#4D z*0r%jL6I$Sl*$1n(;vM>KyQ@>0&X^QgCL7%2pxjn_^fy|T$h{ISP?o>ufl8mF9uMS zn5sUbgTSqMN_-B+Ra=B3)(4Sb}u&<_ctL#!Eotw z2(+V(QT044lc40C#t=r@gAI0Zu#+J)s9VZGG{K#o4XsF)PQJkO^tWNw=hhKDdm+AD z@L--?KK)hHYvn9l;YW&w>4R|*q&D9NV-#e1DWdhIu&q?b!&g-9oO<0z|FyzWozLSe zGHKu~%|2vz?KnM31Ylm*RtPT*CTiQTjr2EIq*1EMG4#wp_4ZTV_=&c~I3daS%O}+l zdmiK7^_^$GhEio*as0Zd(Ub1$&TA4$Tv`3%Y+#qXD8QZ{oj1G)6iZ(f8l?ChCt8<3 zBuu2}z)0M}YK}8;=wky_f25wz7{M^jJEjZQ#gOZ*0S?@e0qZ8ntVBp|&QWHhD z;hh6I$<8Fm@_k8$#Y969pqO;ZQ;bzt6BB3RbOo#Er@##B$KzA3Y<|r4wJVK4;jYw> zlYpYFw1~BeE|>2UGOUwTR$%=6hidSycmg4|>qF~`FZ*SjQZRQD$a%j9of?locP}+0YHwnUSWDeBw z&|gdE8JMbKi#N{?9h5+Ub6^vffs-hoqfAsO%_c_^QpW$Keu_;C(c1S63zS-Co{1l& z1c|V6N5{_)V;NA=8-h`=aU07}V`&+VHl6@Ik-HiMx^pO+h_Y*PQ2KTcN;5BuVD-JX z_e*b0M2aBaug_O}bhj*a)F)DVR6Ru^k^1}k`ZkSa-KPg^#T%AiNJ&dC>f|Ka^`>QI zH3Oxfq=9PJbvM*lR%uz@qH^=sZ=Xun z^}5lrmV<14%E;F4LJE`TSj*Fm;cm3&FMcBQ7$7zqD+x=yHR*gmX5KC zib|$*PJg*UftOOhgjn6C^yUD*t3p($`oT1xf``guap zQ&IZUk>Melgz(hJT#NGi`vD`{KwCkku%eQ~QF-VE4>U8)z5r(=f}o7<$ea zlg?lQ#$aGylyg=ZoT{O<%i$td+>FlQT009NcsQoPPv~FW{5bh6 zI94dbJgd_?%kL5xKNt8OdgE(~D{9CdPI$Iz6ei)4$sd6&AMwg(hEBJ2RIy)845cgTV3(%sGqhfk3*&^Av} zj(q9RZa3c?Zo`iAj^@q$z^j8;b3gg{;x4jzs8?P`l!#?L+kOw`7Ef<%fpMG+H20mn zl4EW~`~5J-es}$0lYY<1p&Fmn#rFZQSbLqVeJLf+-_^VTXaZK%X7`*9H^Xnb_I*sz z=`^e;`qs~v$@@aD3vb7P^2g?z5E1D|>ZGa4qSEW~tem#K5)m-346fp_89bB@(t5~* zp1nM=_rpdV7%7T;U!I&`PT&x9A7XkVbFP!-$>m-dgEK&(VpxwIEreoE5=pvMSb9z~ zL^1%T;A9Dnb?1k+WpQAW3w=Vhj|l}tPVd?pG1fHJy{`s<;#FSu{iDVONfZjY36HU(y zX@5(95&45Dj)VWM7@`= zYgcsEG~NBS10Mib1H}L7eOc4q{RLry^*XBI&vZkgGxc+2(O=&`w=iEDVH@q>!@e?o zh+F*96f&wkP)3fR_@bApvQ=apWsD4xMhPNU^@97m($Kwy(@SF#8NITEI@785ZPNvRfTl%QB4wU{(*F!FPzEpraJw=nl{n6Axpn6Sgg3de*3lWGaQ zgc{FmM=&Njt*zCw!(JX+h+YA5z0OrjxHir+o6&UD^T~A_@!eEmePDFJR@J7#2=l860pQ`s+$>t19SRtAPU{>U08_%v z^w`l>kEVDM&`i+-`}jyvi*I>eWz47cTI{mQu`nx`uedh!82%8%OG^tOsO!(BqThSgAS9X4Ph)G7}j)ALPa0uQ6#j$!+`E z#nnh59i<{mATEPGUQKpbGe1dQi)aJPK?@>GeQ>v>jhT!cz9wr5#{hI%uGa6WPwqrU zjSuCH<~=p{oH2;^%F9#h-|SZ}&uQZzj8c~~ucxm)L)T!dnWD{uyGsU)_21peXh6DB zfpI9tp8ih@6?F=d8C@gOZUcLhdIvv$nD4VWT$#uys`viZ$;&z4Q+vJ5D^0RZSu9x- z%td<~nwZrgBnx##N%l6S*5Po#q%*>*83D|PDiRx-gLEm4tRoub~Rh_7r_=) z6cNR!F)ly6&XwHa8IG;jpRhj)56@3ppVoCX40o0SULX9M1!1bCW2oT|ybJ z17Y(%g9soh6shHul!>D~6ol@cS-2>tIQbn9>tU!@flyYjf788#T39;c7%|!(2ORfp zENpdL*`+1J`;^|0@XX*06w;yNrOlWL;yr{sjfJGbL$yEH?SUA|mFqJjJ#SAr(+EmCchi!O1qV*rA^RrYRvG_estB}oE? zEuP|sr@14@%h!lp5W|^WcGNg(2p#4O=S~M ziw`C~85QU|S23uzS!~}%HMwlq_f!1W{;m&BHDRG)sLUzz_35v+>tM9WIeoT9Z<5M6 z74!pR^V1V6c0QGfjp5pUevt%km}y3v5X;3Cd?ytWdj*F*EgF_auk>9M*sJINbYB2W z%qUK2iUvnVlL1uK%$IVhKRZ`t{C9zSd!>Qk=2!7@x!{gc7-@!K{ zORw)sC9@_jC#|)Tqmt%MvZ|wAM{Xm5TS*(ac|6gqdMCk%^;?ePaIY1jP2;+fSUOIe zWg$9^CqCLDn`@nV>~v#xA1wrjg-zUdE-A1C!b`i7#%=lkrkJ7viBPo~RikLRktq*zDsM9$mJoE{P&S9MzIGec21S(VZ`I{=fid8#+>oWcemmYfhxWvlQdk3Gj80_uH%+#fBL>T?3xhEfvZ z0cN1g2l%gK=%6B7urv|Lzbd-7NFH(b^&4kc_N>Hmpn`4!Kp$A7ZA@u0 z7;Jd=+4L)#c^y7Yh05B{D020w;cmv{fE$)v-v;r=* z2eNWd&{XN~V9}Mjk7OQ;tfhlfhwgeVByDo@i>58PR*$oYpxR#l0k}9-QDVTT%9w$n zZ&_*d;|iVBXiXpv1G)?W>{(WvTFliQ_3H%I78Tk5DJI-|(fc|aD7NT=pb=+9UaAKH zvK43oc23$2IK!4$<1a`@1ji%EW_(*7QRCmi0MwKDFZ@2%vPB8{*zt9L6lhGGzb==R zX*(7fyT9P^TiXA|!~*P{|5J7J2{kU+_-R@9g(4bWJy&16U2PlsG#R zFF?>qO3}Oe7+|nlLR*CZs4O`C&!)27;zW#rS#j-5!>YISS6sP031?qao>$a>%e?wL zvd2~>HJV||;D9L^hlSdQ1IB2&Nwq&0pu;OZu6rOV%v-G9pCUY!iUsjLwTws$OQdBu z`Y}s!-FCIZqv43h?cC9CujqKnGBo4faWz*CARapo)m8f)pvmC-p|^EKWG{tUjZB|^wr~fu8Mnmyp*)Z^uWc7Ftr%jN0LTUEl+u8 zB|co5e*%1SX2Y;<5of#Iu>L19)YW?q@pC>VxFM}_Em6IF_6A^tCQZ*WW=cch~us-Fxlbz_Qz)O;i zoaNI)^UnnH?zaDBrtRScUg3%ZQcjkpgj*`8M6vRM*Q%-oOL7uHU|YX?k^U zJXbbb%oeXdk*O+>6Q!V;KoFnQ4;Ptu0B5hsEa|<1y$a+L6@7;i$8kNLHH*XE^X*ky z-La#E@<^u;0083jNKY3zpW=YcUz^pM>0SUxXTXo!4FZQ$tsBI1xobASK)3k*b@io5 z;7?h!dx}Tqs>OQw_sZmURRCvLA%lg}+$rOnPwn(#Vt_rB`bq%9(E%yi)cK}d{k#!r z4P)(oAPqxR9N5CeypMl_vY+bodsyd5CR`5{Gb*vvSSytcyjEz!?x9^^-l)J7-Mhch z`a|;Fph(%q1#F{^JQ>b;YD)ZVpK{SXzs)5-o^Ye?$_#PQ3EU zKAyWKIf-%t};fS&>5>+`kD z34wFLv$?6TIt9O#p3?#_;Uu0z1(09K?dxo~|V*-OkR zs%#uEo*v(9`R-(1*lMb$rH~4d;BDXop4ZHg?yaezo8;8MXG#DTo697t18B9JgM0^T z7Hg3;%iS&^4I3Pk zMZ9zPCWoQm4OsJlLBm|kvUmge;#mXj>Kf0s zw{&yEChCApj60P?^v2EtX-EbRy7~fu{H}a}2C4Rl1sbfrT@V77cN?^nZ=mOv>=tsQ z>Mz#ya;_G6xFEmTV8RG6)#Dk78Qe%Da(05ITmy$FG4rz9{Ub@!*^#9EFviPCG3X`n z$s%tuc9w97jo_akX&V{JEssm~rA$@Lze6{cbWoW1`&5$D4WRqed5qa-48x z8E-y_g@HMiZfwtjRa>(}MToB_*r*^mG zDtmDvfPewg!vtVH%6v>c3~~CzIi?JgTo>j}7yad&vjU>irrJ~pthd>iVY|i7iA`c? zW`NEAV;g_n4pKcuxXOf7%{kID^HEo=SoP$j)z?-g2U>{vDh@lMHjsmOK>GtgPY_<}TO;}boGz^SIn@W6l_d);O{FF}8^lB6DX@k7e5Hke-vdQ|C2-Y&qF>RTd!F^ofKnJHznM z!cDsYV`Q@BNdV$j0TVe|>oiz%*V4=g9&R_V8^~`0IRRCXavQ1v#;so8??GgS7BE+` z;w2liyQ!?$--P0H4rQia=;5C(Za@wE-@N-@2OwL^A{7?^JPli?0lA}`~A0lWe4-3S7Z?0<|V-`-)0zCBZ` zpaxX#pWX($K+(!YL)rbHRK$NuSI0U6y2lR0jqL-S{kV-ofP$H;y?uS3o}IH85uZcd zoof{!HLOl!24?qx#{W~2QOnYQ(|^D&@{L~QireiR2{Nvk?aA`^Oq~>`o=mOBeUd&J z?_by5oBvj{K`0tQJVrW9J{}9&f8-l4&&mTSNy!BrTK@oGV*e#*?V>X0galqFhv(Mz zGn6)0Wdw+WIU|L)($kPS&q|I9l&3*bg- z8nOlOu;5g;Owv_{qXXq}kYQ*7f9%we)SQcnCdRNCKzw-~gpEyz=IH?(%hLb`hxsd=b1 zcFlB2wqp^FB_aqOY*?xn-}qDv0MlM`Niu33>r28qynDjFu1C4(HW@K^ zcQkadzp_DU!$A4k9A=%Zarw$>mLDYxI7yajl-+ZM3rhHQ$$R)0X)zhy+SVeKil`D>6JJ>wd9_Rd%3)CKVxCe0hy>|Ms!? zsv}};dA90FXB}sGsCp>hBhusp;BYEe7;Y5q_Q9!8(IzX+!W%JYBv(9gKrVB2F|Wv| z2{Zrg`!_>(n|G3VDY9<~=68pf6qcTF-TkT|_x;Ui#pbM<<5Ckdu27AqBJhI(SG!j~ zA?p&4=;{kDU)xwNA4fEo-{=9k>YIT{Lx$E`tBZ#7UsU=zl9+Sx&nG$cOtmjuz<6jY2md-jvZw0c0 zc&YVy;UTrIy2bOP{zHT3WpQ%(=V6-_73H{`Og;CpUR}K3TB>8=@NI?*;>GulAI04- zc^(2RWm3RJ- z@7a}Xbnb1kug{})d`V|4$-Ak48h-=Ng#9#>wwk~3=A@i2C8)ft;xc*i3GpO(n6FP4 zsh@=O70NdNMOK)W4vi*JVKaF-78x#{T2Y{Q3gemVI9MaKp^g3loIjW@HL+5pfl3CK zds-)d9CZF+hD_2v``!GK^3{fXfIlDd#IJC#W~@lDN8KpbH3I{zVn#TSOf~az43P^x zK1Jy$*y?3zw;vz8^Fe;;qu0=U@_Rp-1E6X_C@dsg7S}738Kw!~#hBbK4%e^mi^YAW zG+k2D{Q)}oT)88%khrRh^HfK^ukIM5nxA#Fsxg3W{gBBS+yIzHKQ$*rbabCKGUp0I z-y?O^m&?ogl4NfaBJ7(21@Z^g@%qHW~Mf)ncBq#%tOv{(=|Qr19$5kc@%LK z;ZYV2#%s=n_;;ZtjuyPlCKh;AwXho5IbpmvnYCgi7qgnFDVm0SXP3m_lybJe!u&!P z!Kv}ByW)aB$!nTj?xJ;tR9(YBSiZ46UD~EudG>DUTxXM|mWylj-^^i22|#78pp~s& zs0bpdm6{5W@}A^R)D@Zne0ID!#8Hk|o<(*e5#^uQ0Vo~-WB?(nSW*_!i%bBp42f9d zs%)B(d!Sq#!0-*R8Cv?xOKQ7q0XtLdzV{-fR#w)6l^|UyiEqt}s>r&TQZQ6HWcI19 z#mqz=!|k_jSnJWn>ps6$>oAg-K#G5k#Vwg}S_~&hyeoXBFjV-Yx6vo86q6NNipdNO z<+O;?;emp(r&MY{BUP9A8aKHWWXDc|pCuy?92tg}-=IebI->kH#Dctf~$NF)F zC{S)ai_J>*6lMxt7?*3E>`Ei~ZO&|Rc{qhY1;{#^3p&kTj(`{$Yoz-QMWO8E=fp7e zs)A>s)^2Doh>VRXlE(U;LpTB5c>lv(iwG@p_c{{*&AH(bm%eX8GD;$r!!zIf(mj;GkvQ#tkex)tu{gz|+x|5duHA771f&Lnw z+N4jV&9Vey!Tk3xsaZ?kD)egI$F;DY?Z+=%A0=W&WhJkAbur1WRFN7tx$&;}_6paL zs%&lp!@S4q)8p9D<+|$rx|{V#&fLmUR<{%V-Ez@z)qcfiol9$?P0d}anpn3^uG_$n=j$6Fj|Z+0LB9FX=Po>^RO zoA_3Noi5)$?^K+3TdQnFo69Ori^tw#V&KMwdWlcdrj!Tr^`4K~2NxfSek^G@>TK>r zBwY9u8%(MoE_2Iu-fdw@QiyccYkWB66;7%=uqyqDlKsT0*5D2DS&d7D{Lwy(xe+=N zh1TPvl(^O-qRUiSmwS7@hV;2yJYb8(Y!}{_WOT{mIS3A4ju(AG2RE&I8OTfn{CdYB z^c|zk`j-0)u#!|N0{_ef`Gj`fy_4B;{f=fxt;xS${MvV|*>B*qcA`zV0wr4>NGoUe z@rE>H19c7OeJ`0!KF*TOSQ!5}v9UT+>CrHsbgxIhe|W8Z_@h;vX}=AXcQ_!VIOHnJ#44x~$#d`pRtL@;SMBy9_ShFEV(!6{fu7 z7jW(XLr*&9Q{ETKWmW53-9_N>6Jc_QzgAQdOTv?u!$|$;87KBdwKO!YZTgrZTcbw~ z@r{dIsYiIZ1I`lQ2XJ4e^c*Klb(2^ngl=#kV@P%GiKN;IF}bz1^X_&lhl$yWHr8pR z$$IURc0$l@KQ=X3D)1Lf#DGvx#WA$R!7s~P_~emA zrj?HSaLoG1*Fgtn5x@7h9TM$}W(;kO!yGDihm!m6W(xUYpEk+6i!snw7#SmG9)=K@ z-}1WEc-E3G6E*N&h(^bYWp87@c@Abe-ff*Wreyo9ymH3J@vK*v$}i!nhBjx%#@+Gv zy}re8Jx&{E{`kDmoj6M{qGuDds;8U8@j`1r0f&8#gU*(1V`%^QKxR`=-#C>RG+&7^ ze<)`G9zbGE7wczyKB$IrSeb9k#eMi+bbWbTl4-ktO;gTf>9ks!I%Q>L>P)4kxHMT& zRxVjtu9fA6NG@Q4XuFvyQ>IQTnwk5)fD4M1C7Fpk2#A*C#v+jdDgwWo=AHLF=XcKe z&%gZW^E~%;U)OiLBJWNm@YPM5E(x(h419X1L3AK4U%A+&jJ95Ahw28RBl~8j`eX%P za*?qT|5)>w^=jaRs~~XpnsvL5vJjThgGq=B2#U`$gqmK{2E<~NW*S!vE-ZA43Um7WU(6Jc zny+R^vjm;dos;BA#i0{Y2Ct%J*cLRpZ>1l8VP=C=V+1DBG16`ef4=tW39xs z3%Pa+xpU7%+*7hvyT#UnU)SvS`&j9p71aj%$(YWK#_sE#!h?tEdxjN~I@W@iCo7oE z>2XdkPurH@`~W%AX~Bw>S18Sf_5OJ+mvwd>Mbam?EVn-mqhDy9Eh6%gSfkPng3JT2 zC2VAYZq(-}hF44f9b9zN(`Rr9L~R#()~hdl02MJF z!78{p)fi9o{b)vn^&WxDe(@De=9Ra`!MkW>#Akwk36W$U8O;;JC^ zuk*{ZuSM={94XI2mU+$e5>S0_Nc{MeE>?$!H{l$K#%w3LF2A!;w{EN>jWt;1%ZcQI zzIx=LQV#Sz*M~byTdeyOljK59$v(x1QWlG9-;wrKxXmc;j+zEjxlp6L=H?l5uZ&$u zr3skomt%y|n69i?#k|6@!Fv8nvdDltq;(XzDw?A+@*o!Fvlt_d zjJ24)AaRLfAmlUzqtD5XZI`EMEG5=8cWyNC+!W_x!H#fMW~xVjOU|w}jS(ftTb8~$ zbV!B4TM^XEoF8&aekrmElC_O>@W1YNVq|$B9zwSX$Wtp=GoR$B0BSIdb`F84digLfrdguGoPbJ=|B|F2Puot*CBF>_@9&;^$a)LJ{h}Ul} zz)JL|(pIQGI&Vzl^#8{@Y1l?Ay=w2XP~P;-K?aVRhYHAJM~+~Zw>)5@d1aCwT+D2= zVWQ&OY+tq_VzCB@`hJ!ZxN)|#Ihh)I$R7%ud=}sNJi4lBc^}`FHdeFD-`H5#o~&I|OKx;z-KpRu@QF_5Iyl ztbCZ^U@;=L73E{`d(4lYL}?T6a9upOsdC8RUwmWB`5$&PlL~8^`>0twf^QiGKA<2- zX1Tjj@Q)x(-J5t`!iUSIep#^JVROcE{jPXec|%(a$0Ei+FI4}9EV$0>fPRfwySX-$ zc-njMd14l}*=s+KdE-m($I~;xoY%&vFLRGiO*{kjcn9dLIsYG*s%j&Cg_JKV=#Nbk zzmuCPznOL^uN7J0D(Xic&iO+@gq1D=49KzCk-yEYUx0<5R+3s|5SgT|g{^w{E5XAb zvZ>m*EHB(8O8Z=2b?6a52wkA#cuf0U(_v7+LAXno;O86R*bFSjWKG<0WI`5R zfJG({ebjF~;r3kT7M2Y)w|RTob%da=g>BiEh!4MGBN^XMydn&O^!Hy9uyv2JO3yxj zX{Q~Q&4|zl*#1Nj@E$tx;Mh*TYu~=Ki)3aSE^T+I{Kn{tp~ZaGirK2Sxnj*%XwsIc zY!(HOEd02b$+%!3#ge!6}sFIL##Tfa*T{*%O}>J;u!0LwP&*k&K6@y=(Xa@17uX1p_lZda$d+`+VEF-(bQ z01CRl$(QqcBoA_tJlZ?_(h=FPbYJB4rH2B)qkU%Uc9mVYT*lq?h}$1}sRe)GH@g_k zGwi=$qrbkr_p2^6X?^rlyp?tW(wy7ltenFs18_Ey3qw7{vRcKGL&C!@-L)!-=dTW`oe5)C=b2`bKtT=dOc8N@(8+hIp}{>k(4X&rC#(UK6uWC#-f)0SO-GZlC`jOs zFOOv8xk6dzYIZW4PwH2>@lr3yr&>j%&kC6eCX_*W z{06f=IKNb96ovmyx)XyIClD3gez4V#ruW=uJ_Q7`7UuaZa4CKFllBtrcnIbD(=63myjr|+Z!ghpW9IQn$n z6xJA?xAo8U_wgTz5ozn#@T@SQg1s#46(8%hyH|tI{_Gt+*pou1Uf+Zbli)=x#$&%2 z=*KS$T0VBMdp_^mO19ei;N5%CUmimjK)))>Ip)hHr>0T^>?CG<9P|d8H0vT8o~=Tu zrXyB~bkK;U#fqmT(=g?;^C}Vnuo!eUK(s+Nx%R*1&X(YUS)qk>VYS4@4*fh8%jlE+ zdEoX5T?g&>ry4J%O>2zX6 z-__;~;|dx=e%`ux-`mCc8G&KUs9~^t0o?K5ddQ0kp_1$sc49%8#P(v-Xv|0-0e`C6 ziy~z{Mec`>cGR`K?gf1Yqjt6FyFLhJrJ7k#Un$^rfP^Mw&3~8BY*E#$C5*yjf2uwk zu1nzamp#Y)B?5wPxE~{n6+#r`RuJi?E&Up&GlKk4!aZ}xNoUrpKYdcT>;f(U4LW6x zxpP5BcxhQ@5Zkc!)S&= zQ}Ic&?X{b`!*U0WLKZ&r#nl#D9OLKVS+}auF@p z=PxNtd@!u`8fm1K<m3A2HaJxlW8V|Y~_45PN;b}H?@{bH58@A&Zk-8KJ_<<04cLeqx5j}T&uo` z<+tNhI) zsV!Nt_Ipox=|_iVe`YC?w-JC5HALP5tSscsg_77&8LifjfQJWFmrsXaeBV8+cePR< zTj8L6`nmLBz}-J3K3kvkH?FkxG{g_z*%$Tg%Utux7z$nwTTF3h$13HLZhK9<@E6pi z7e?D_7R(P)r}m?ugucIXW#Z-!33bf^c<6j!(E;agQ-N0SP9B}{+EA|s?h9p5c{`b( zC$g$i3%&haGf80L2Qmkp_FGSGzg*~xgAdd0p*r4>M*qB`nz%&ebU{EzC@+3IZqwwU z6S%{uTYtvT3?}kVXKYkXuWQt5Ui{V}@lkZd#tY3xX~`|WheIzz`t`YBW4;sG~` z(7!iS(qpTdC?3(*SQ*;W8;>0!i^~uMHT~$yScfrilB2B}A@Htvta*5!XT|~LdD1>B zzev-lgE7{h!Qc-I2_jAxmhKH`>1d#6*{f7kAOq#!g0!UZa$n0I4$aq!Gi&Zs!_U)c zuoA~?f}7Lq;L}z%YJ_Y;3Uw<;WYR8=4EBd8t0j-Hi=F1Onrmee#k(@eFx_1cxlPF` zF_vVgd+;r`#UlSmkf+USMMuucDq4Cilu9jQDeJMj5c^o^8<-*Q)PYO7VAMe^i&|Es zMIEbNpo=B67lV_dY?7DgDj;eA*1Zk;1rHUgwc#^X7 z%D28E(&Gc8V_$2WE_S4L()x-uddZC?t!F-WUIG`ifBZ1!g%m$mXtO^0fnj#^zL<$8 zdX`evEM^Xfp&N~JAN_58M*wgDpBe8AJvbW|I~N!F{)@9=Ik<+Xd*ZE@DEVEAy&5$S zbEGc{4d$i_vZsNm=Hd@KpwNCjs9Ra>LP7SUq>&pXI(BW`?C6i_79o`Y8s&q}9_ZE( zFPA$Y$RfXJAM5MI!}*9z-{kR;5I5=;M5J-d`}2Gcc%r|_Bh>D@-z5OxuF{^$ja*yA z7S?k#WRp-k?Nx?Pq}%XH6}k)uBU+?#eS&aztYRW|Sed~vz*%2unM-K-E8Kf`{rP}A zSuT$0@ws1xINRv`Tm)qTLqK)x^zs9T9Sn5cD!joE9`<4N(=9g5Hlp9pRvx%K+=-fh zR3&-nUW}Pu(;T^aJpTkrZ&R=c*X{*3p|8w)i3+0MPd=0*=S+Y=6BYE?A`r!73 z-uo!oV1_?`SqF3UIrNY2u8(k{epFHW@IIR`?o@Qa(LS{gMf)0>`y{P3>HLE611wYW zP_PnyIYIt<^t_B`!Q^+i$Sc@Aflj)7;yc4GW@@T8gvNler)1GfOwj#BZmK!qF~QXX zAFcA$l;8eyFTTAP>Ro`E4afA`LJOP-UdcNL%i}rx_v&5ltaAKoslhoTV6QQrmtyL)sR_1FJA$ujx%L7-XejupcH@c)|@Xd2LhPTgLy6|l0aY2=vw^P zHCX79yHMbHD0kiLgXY7y*6^lpAB;kG5rDW!u{$Mq&3a%Y&C05wtrftWBFI^cb6UET ziBaeVTA~hIjI#;q)S;z`sYZlAAQ|EL3r?MP{*$1$@+mQdoo?yeOGq>O*ooHcCS?Cj zp{UrqG)BVsV0cMBqaNJ!(tKQ1dxCzgg|v*Anl#j7`OevZq+wdq!rtv%y?&@ZKmChU#ESdh<6M}lzmHU+2Sjc?hXIKZy>w)?KkfvRE39BDsL$EkT?Xj(d(6E z``O4+Uk|Dcd6Sjjz{;o9o)+NFsUEUhvcv@SkMDfRlAuS_Nm=SKQNZ2DhjT+AR($>Q z_$s@Lz1p}fXD-a%*}vG`rXKFFInJh66Z2wZgLQl1kP_kNx<%0>$n%r6Gq8#yUp#V0 zVh?UiRg~eiS2B1G9K3tvRNcBB>R- z%NVn|4oeI+BT>cnr|>O6;Ds!m@;_a27P+|hLgla1r_L5WAF=P)09N_vMg;P<4xA`T z`W+bqmObUtY|yg-Lk$}N)yn;3mxn$gd+{}g6L3fOnI)UMGpPnn_Y9Qj%g6-&P$0X< z9Q{J#FkKVlwi(^>z)DtA8=q7*TTo<85O-RPY>8a5GI3S+u4Twm3#@S6_LXsfOO$3R zOJa0+sx~V0y+Oo~7j)PcXS130=sSnV4R&-?2U0_!s;>AGnsQo!V{K!%VW>`cy z5OKV{b{9O4>7_w9;6w3}eOEG)LR6y1t(ZnkskF2_u?E#NcK5W0^a)lh8zCTUWAoUA z$*mqWh&yHkGfGEhVrtK~oUkXwdfzpCE)-X(0p2iYe!7uvwp(3p@oK<+#+1b0M;d)- zG@5sII3?e}1E;{F|w=`ZV6k%-fL%(_xZZTcGt* zk;73108f;s0_C3p*)Gq<4VnE&xO7970n>Zr_vxx-hCG4jS{L)0X;PJnL5gfbwUj~X zdzcWGz_Z`Psx4>Vlj73@#GI(H!p2?e&$9C2nnlu)z`MmNz7KrWwI(1GruCoj@FkZX zbkjfOXN41s=1ZGzykxZn3oB0!Tl==O6b{6Q#!?T~?>2fWx}B;2SI<>DTM{GYvUm7m zK;uZlhlP$YxCB>19h{&OSalYWZ|5a2R1I-w znD5z&PUGgPjC2ef+jr4JtLQoGDCeSwo^gqI<1t=Wz+{!728!Y3(m)sTYfGA;JnW65 z8FJAR+Kscl6(14t(B1Oxy#w;Y&39yDI)v8P^x>;^kUB<;lyF{l{Oq`t7%`YEW)pue z?(3lr-}q7M)w@SlqV-LMoOAe@I8FVeSX!;axrr>RiP`#niHpt?=K~t91hE!)eu-yK zcxV=zxa#45yTYd*OfWYM4L}EVA zXZadCS&f}ei}DEWRj)y9KZ7UnPzK30W4_)Awefl|`oYAltN?E8L$-F!;0{hUx~O5S zjrZ^(YV`|V5Um#nD{(B{2tG65Hdt<`Uw6MVqts=>wSPJ>QHxjt+`@iE3FVu0 z6TXWnYE(K5x&r97Ra)L8l7w0ZTS?H@HckQJ%y3(k7?U2_gWQ;&T{~4vO+3ar*TTv# zi*9fG*hD3Z_+t{h&?m4W#66hes8^SoKR(lO?nU`X7;nuPiYq;rnhc)yIhzQ

f6< zB9vn8-QK#KrHw57)ur#iHy2@2ulz{I?yF(awQic7eVCzPr9#RAs;p7i2`S-=c^l%Y5OX2^450Yi=~fG=Be z$SOkL*bl6E02Anncw)^&FqC&O16vD;N$ZIl38)!C6M%yFe7NKFC+l9m?K8P+VzoAp z0x7057ZRl1i&rFW!|s~8)yryps@ZkyD@cG}01lVXS90FniFc1ezekH)DNi9-o!jvB zL3h|#q4-)@qjLfN*s>}1aqi;A_OX5qE4%e?n`CTZ5>)yVlFwa(TKU%&v{SE|?boFk z#ry_iv^boHJ(2NN_#lmjuoG>%=B?+G)@~Km=E5A@1@VFV^&sKKZK6I8G$)N@a z5>&>`klA?dlvCtvM6E~1_-PSGIJ>EdZQ+BUjJE{qu3n8QkKKO1n=otZDg`tZm7we;rR_tGU$u}07dqd? zeT$ksl!isYNpk7YGKqOJ{k`U~Xj$vJ#C^R6*P+h|>dohiCaY-qvEJQl+@5EQBJvFOjq3ADU|BR z7BPlp?j_g|FOp4Vg^`{YvKVufxcSgeNGcE!)t=?(I>bz%-TT`Ms7U1AyraBE>K6w~ z7V}EMNOY@Y2pI=&URcSBlwO4pg5#0d8ZE9bH?1@-7HRB2nm4}sZFRY?XY6vN3ds-s zWN={Hm<8Y`Mxi4=BS%ZtT`)+#dw8kVs-N+`OvA)0J-IVr{G{0x9AMIf@?jxkwZ+gc z*#;MrDt^c-6#@ZNo}{kdGR&MgF;0N-KLN2D71Vg)7svuNP9c1Kx~G9_725Ygavk3u zSkwj*6(V=#HA}Yq>pLorQ#bBaX@rah%Drs7h-2?HNRbQ!)x?3~00K<~!UJZX7a*+M zP|@=E@0b9Q$@})(>b9QP%p90I#aVu|-S^X*A)r@ZnyUV^_c^-$(>`^k7NIN`kS%aK zcYj<(G_Cv@vMezpJohyF>l57o>#NNZZ&+t^{2D0g1cDFlz~pc-w9v0129#@{m>#-w zKmGfh3zCXr{MM<~V}puy?(k+OFOLd#9e5#@vtW|tLr0;c-TX-vt9Pxkjs~UweZT3jly+rj7FOyHtV%4#jn9&3S#x0}nE9|g?`Q@?gZPW%^4Glkd zy1zIvP7sV3QGj9Vea2&;B5Tl!cUON}omp}iuDvph*+?C1Tu1~05DNd{W{x1$O-LNW zxVgjpV`Ni>wCO;JZ%)&;S^ZnifofqjmbCkS3|zh>9ZcsDkX6c^ zTLeka^>PUSYVNEw8LiybtNv%qbGt(etvkw4Q((55c8*-Ew_4G*YA+XeaBx_sB{%R* zlB2Af2bZYNxbu~LyYij8wRBs2vPP~bn-(6~gjRMWMP!Co*e%&a{I0z2nChPv@GJX_ z+y;jsh&o-m=LS0(!0u9S)ShvR#F;_G?|Zj;!N8W=r+c?A%R5j{{jyhJb@moe4bGI* ze14#dw`3EO8b5H$b%Aas3&t*Rl$bh$@biZLu;(vd1+?aLqUfV>{OUkofrLI?W2R@>aU#iR74W$N z|HnNmg82Qu#GO5u>nO)fu+@D9?v$SUiUzH}*~RSfLK~*G(?w2o9&v(}Q5N#i3^h4r zScV-LK6rhPN-q<%!@1Yzui83%m4Ts-O49}B1JTO2&Ax*8b6qTjs~X3}Fc18)2_S6R zg1703)sd7|XLvsg2D(At$Y#m&vJtTPt}DMq(r*F%=OFyM%mgJ_=wx-Sxl|cAZRzi% z$0xSLG2_as&+P#5%<^7!1%>);D*mZ&E25#`W_|1}Z7a;|&1 z{kzXe!*h=VR&I#tPCOf`Jf>J|@{^?z{P_%fHY)R%z$3l=SgN1d?AzQYF`+RX_c;1$ zC%@IoM>~uq`WZ~P$H@i9-8XOD2i0l@&*XMLXLbJdb2@t^8ea z5v-9iYhOFPmhht3m+s02QD1nvphSlWH+sFc|JU!5zGR>|8W;*rtT3%>jvNYtP;gbP zUAAxmM#^m1XVvldt2Nxb%d#=Uk=0K3P8{hX@ET!)!660>*(6-Pcmf;`keg9dQqTct zlUe}uRT}vv;q;FZKffC9$8Gytz5+x%y!Y+75+-#FC0JzfxfkgwK9GOi)t(`E`G5Yi zdy`NS<~f!XKg~O2S_eCNEcXa`f4U@ZwSYK#7KePS<=H<8BONnHWDExiqmJhIDx-S! z_saV-Cp|BYgm|RaAP+4$yAHe!beClCGidr=`QqAgk4T1M63ob=`}nfw22Z~`xT04s zcNiUFc*9uDEN=FVk6a*7mrL1jASQN4j2;A+z&=pTALr*v78^IC#yRpJ~{uI=gdlRLcuQ}w}`X9hs{O)Yey8t?+ zKzLR1NiEj2PDM?l$hY(|^DNIc^s^06h&8N2J%1UwqqzZ*$gaFp?PUe8wP_p;WJP{^ zE~(07y+S{lDMZZR#ZC<7twWz))0>h;@-1sJA=rYg zbhd=|NLhwl62ERE(feRUz8ORFV?cF9a6p|OqPBNxatq|mdy7rmO(cNsgvObb-!O>r zNz@CCyvj_VRo0m3UH)_9ck$x4qV3LWn*o^=#5lu3wjxCh=!muhFsmD{+;fSY-+WYt zKL=z`WVV24rbc5SXApHA`8a|~TA}8o&&_7V5i4f!ptSz$-vl)0B!8N(Qw9h%*>t~) za~m^HGjewPU3kD2)!jR-5G+l8oxPLYnm7m2bQ3$N!uwh? z+e@x~))B0h0GVn@bBCrbbcAM9hxjcfX(M}0+0Ad5q#0Zeh}=o~`~QAH`yp?ybXyK~ z^ZP^XF0l)hD)qgW9=^_kypM5y+Q(maII86TDno@kAuO~2mliPlev0g<2s|x;CU^)^ zI5yT#Ruxmn$7>`lKEl|SSCsSLnL<0*)gko)L2=mz)V?3iKCr^}2j+Ht;ZD-K($5!- zs&XPKV#Fn(aZdnB^D(I3z8HX(SzPCpDY=2%d~TqfZ(@lX<9nM4NZA}f$P(|Ujk=sz z`9uXf#;s@=^45TSrakt$oQ_mu$cUw$d-)auc~z%T9ybToUFG8M8}KY_pbB}} zIUr`YSH=Nz^ZXFEsz{+XtMan_Ij$-QU=Q?Kp+WtN#v>ivEz==^L{5S?K|pt^Edwr$ zs)q1V-k0_p-qCt-wAF?1>_;@$f{0Y}9zqL{^w;Owz%8xH8zpCtf!|K>BKW7moPeR+ zCuOtO&Cq~a$){r^@c>4E>1!Kot*{Fu;RNv$*}T{D`hC^2^cEW!ShJ6yaE_@n;}auv9OKcLE(R-RsNy3gOL zYw!uag7q@0jybU3keR^qXMpl7ng=n6sp$7?QY0m(S_SHTx05x-u5z1XExDOQ|_sYdJR?>lIu?m-~kF!0r zuz_PD)NeN@`Hg4~{}D^N{+@SsPBw5KPoH_8?VA1Kr3RWvNj;Uiu&_|SjDnPa)HH~H zXZkgtxi?y))a}UI_fw7XiFktEnOH{j+}F)&VSr?A!vghgzU;MGjmVCwJmK_ah$ zu&{iltm^Xvvl3SmkdnVA!nq*Ad-Jto^cJg#U7omoXLSniAYR{I9$MfJo(RZOtJpH2 zVP(EB4S1h&?bXkH*QL9a2dx{}0Vi>X&Q1c?)EUb`?3TC^*LLO4t3div?K`d>rr+>#uxPhdGI^d}X4%0lZfvn}i zrPsLJo}u+@0}pq&PiqV&W*~wTws1qoEc{8Qy9HfC*KSLfdYS3Dw#Nq0(HUD+8iT!{ z1L%eNvJK?i1t);Q!+BZxGuu^h0-A&qsyG1<&bE}P#qI+!i`r_xf~*Tb-Lza(K(-F# zGYf#Ri9*c8G*RjKlOwA!7L4W!o8UC^!4Dzc*VCFCU<3iXj_!!JPH6-=0|s^*`insN zMJ}iKx)q!hgoR`35C4y{Mt}E5INgQSQ~%c6@KBCOpA~u{HW}Cb{f@6rOs7EFJ1A( z!cjaJ=T__W)6-1X48HJus`LgxY3?s=w!L)qMau`8kBD1wbKqhTRp+}G?>cuZ&4;Lx ztcI%LW$K?D`h9kk=y6v$cmV%a(snGRJ3zZ7bGmSmDnUUS{0LKPF0-Is*Thb?A!Wdb z=~f%;c3{M~1UXoO6K&G-U&*5#^6myaVN!VBPl;VKr$#<4}Yy)Qt#U zzXb$Dg7`gOp5lca$fqS_{jl}>H{;1Bz|vG|HKZzhZ{0zG<+gK4yyK4Sojta@eza+b zZPCSBuuzZmVJyQ(H3l=|?wAPO2%h_17a~#c=H*^^wJONtOWHBZmu&r_SAPI1F69jC zTtV2Ro^!c7lp+pjb>ThuFnIlX4V(PcmK4DaAR(rtaA+&_Mnf3$!*bu8sJyfD?vbL= zch9J6iqMm`zofe*F^SXAM#*77D}WdnO=n=oQc-*m@=Ci6Kjm3Yc-d4B0heW8DW&cS zris|s$M!97*Aa}tD;R$<=&W=5cMft+W>?u8YIq14-n?lCx3tE_#)D@;rW{+JPykXH zr_1($oj|>x|3Meb)Nb{@&nu{7N$N@o3|V=NnuW52O;+T}V?It_Q2tB`Z#7&;Q6~4) zIvts@rL_z}N2i>GbDAE=h=&AJ_N1w0T2{DUYzz0}{Nob{{6Mg#?j|zM;pXSoh5Q=H z^t`bob+;UMVHQtY-{M--y`&+?TfYRifo%I}^pLK6rFyk94>-&MLx`Mke-<^~7cK?( zebKd({9vFrZ{+Se`&;TpWKfab%HMilfH*o3K;g0c*T5dz`sb$V#ZVg71*DsN5tZG& ztklT+QP8shcVT5EaC^_!wMs@XX@X726kYV9j8@L8dPFR4BNV_|&JX9joY@O<{>)44 z(C^1XqTU_rB2&v_0-toUiQVZ|AN1!{2q>WVRZ$pEJ#DIoIY`97eERYMgO29oz`Fr6 z?GO#vlxc(X3p7p(yp5#~NEA2b&SDI^$_qt*5+~qwd4)upG>)M3M%Z?BVK?H|E_wC! z-x~{ql7=)~f5>}$NZ3$=s=RjRG|Og>ZGK&u@&UU+u`Y_zXl=JSvT}qt-75K-s`N*g zTf^Y&M7al-Q?xKm?%Dd-?w6boRA!@)j zw5nSyYWO5xs-?y~RSxD-ZG2Yyp()U3qCVY{1g^9a8jsHu?oe}U< z9#Mt#8kjf-A%yDA3$a2Vrvk75GpWbzdvo0fIK$!cQA)Y?ttWbH!nKan^q-y{&I#MW zheW*!!N=mTun*@@GjB-nna6^j*jOXTL@ib~4Fo~{t)BT$fJQ2(L z`(~a;HkNP}MFO8Ho70G69uR*)j}&j`)vGKd6`huJmDFG_s~x{{tlcK;c}L9A+?0F? z?mVDhTfTU(5`Lr<4~S1qIcXa$2B&k3gAtwJr1 zMTx-jrL(!N8Gp}L#oD`~bomo#bZMYKn-wlI4y}y1`eBBKT)dzh0uG&p$;s(TmA?_U zR3@2mW2gg4q@5Un-S}UUHnM?(zuREuyui!9t_h9%XFh=p-YAR>iBjdSt&06e!SQ_j zIO^e>uVaTg(3mSMI-iI{uXht$eMsg$_SvKCG{6nM;FL+S{mdzNshuT>VQ%g(JM+_@ z%bPoRB|0nt)K9WJ)6T4bp=~9zI<-)atI%MIt7f}fY3z)$4lEZIO)FFx<@%49Bn-F??Gw5^Q|Yj24sz_sz&fxkV(Q?fEkQJIXkvG?}H47 zz45*PX)XWj_u+D%>m=i$`uA3(KAb!KzeKp51$ZF)x8_S>)O0rugwUBLp^#;j|^KcT&f9@&dYO zjQUB!JrMkb9pGx_Yd%fqps(@{3W8t+Uqn-gn#$+xw8@u}-x9z+dd%;xbKF@@QS!6H z30c!7Fj%K#&^V>)$~zqwOUB_%x-4*;%euirA3xqp-ef?XWIw~orb`k*QJpC@iLlN% zqh`5-KL%ZDtYO@nZE#rDJMhdHJo3p($u@jD5HX_AQ`=Y=&UO~2m4HHvN>C^J%6G?!nyzX$v4p3+!{(AmPVMqeaKSRE(eUR=j7 zI@jfa;p;gqZ+n}g>4HZvy@06}0Wml4Mgsjs7!a@6Lg1vJTkIpx%C8RBO36CV*~D zitYR4qkkr+H-S`6+;78BY}*`^T+=+k4{(jsg-zGu)o?8G{Wg$oR??qb#x_S%56j-r zTVOomG;y6V`}NpEh8xb8sGOXN5C=?G8$fm79hq*y^K-o5cXxTgvLQGA@z&iCTJHw$ zf}5)gSa8{}0qUjb)b*I%xT7?EL={_1DccI)?4ncn$WlFxF>Fx4zwyD>KzbBjX}ZS% zbgl$V(hJ}4$*ZvIAUkpk3vp=@+_#zxBNcAG#c(>3$fr*aJ z1$pRtmMPP74k@`<%j>nf7c8+}5uJR?=-#A_0m;5SgwO-U6Im%d_y9bT@vw@{Ox4Qw zkds?`kbRr$?Af<+U2M*HV;)n|A;D}Sn;1N>6B1o)jI_kkhd=iViX{iHpAv+=_+lAm z9W-NK;0_-iPccW+yd?PD7K6z~zX*2yz?9^%lRsj*Bclt*Pe}8+|VAqd*%@3W;ro? zE!3=oe=F%?HZGHm3hqt2rIXP!jvi#c@S&ZJhTGG2L*bDg5t_%@zIQ9?2?cS-;rkiR@wBqk-8GYt~yayS>4a3AwO4mycKmJ zYD@HdHYjy~jLkRO3z7T6>e_B!&$Dgxoh@fBW^7Mn2F&)s4Cr?!(mu~kn77IHO*gE5 ztxC!}h5WhGh#uM%Qt9H9KcYq8T*Ebmj5H!bRUg-&=ehxVIG8cZgXmQ%ApIqbE^5^K zVa6A!I6{|As--cnak6TJo{~zhQ1$^GpanG-y({Y>)S26M-R7lEUBi(>M#!;#_(Zk2F7qbF<9M8||XSIc|K8g`M8~U99L<8ayImZ+RlqL0oSI6lgp* zZaql5`MqmWuku%NHt1f*<}rteur78V2#G^}-0y<1#tW=Vj@d*c#mr>U`5z`1H9Jiy z^?G1nwuUZDFWsDk_%hG&;OjZRjeqQB*T8JMA^vOb&G=kQ)D)*fkUgDwMLBUlPT<

Z7fOy+?QX+}%UAp9^!#_JyKyh&yX>6Y!nRP-@ z7I12iYPD99dGtOBRjl$w8P2?i6mA|8hfe(BtxsjW;Biv!B+Fnc4DQjA^f1>ICM4yU z7mD)nw0V?dBPmuvos@pSeobMc=CP(M3OOo+{dF-2^Q(zPQ}FKy95t*_s7q{YT z%%K9EET7&&FJV0MOVVZ?SN#mNjeH_6+vgfb-zSxP`W&v%c$A|JK5veb_;j%6`@={fd;@b4CX!_wNv&mhd%_}N@>TFK-~RG- zS)N%%r_XkI4(z>Ad4jCi3DsNXp}_Y=v_J2KDZ$3ZB&!c z2eWc$w5bE5+y%lV-=I7?Jtf_-$9{tTm(Y~xkzamNI#N=vCzsCcmEJtg2ESA1FSjHt zC9>pnR*GV-cy&au9R2p%QCSi1u+lMIk}v0BotTu6W)}Htw>qlhBU0i#J94Qroad<= za>$tL*Kx_X+7L23G9vCV`>WhbnC{_U<3^YMk8i1)hW(Jm6rCnv<6*tbULQCdkG-w0v% z2kUzIxAIbRXz+umH%tiT_Dn~W_*5L;iv6@v6=PEsG0&rI68JNH4UoYZ=^l4xK2+tF zP#lj@^2?fy4gx$PuJ5!lNZe%7hBpt4)gtqQ-S8PCl#qbbF?V;17zJ}PWxp;SRV<1a z0#VOnQ6|G_sl@#~qSc+IUE?%!h%R256ZHoNjwj@Mk%GIhQU(nMGEFpX0}3>Cs|p?D zGe)lB5t8sGe5h$hkPhB(ib)hQA$ndIHmGBTl72Hzwt?4Uw3G5#xe3rJ-AhDIp-E`a z^Iejifh8tgtCkbwWb{=_vOkCOg0n8m$J1-3vxmIbOM(Uo?T=SUZ-UQ76U^QIBdR>l zuZ!QTbyd0P30HZ~CiQ;t!UT-~%#HsbiC1g}6Ldem53W6o%ZQC_hh?u0VX7z~o0-cq zX%!_>t3XQQjB+qOS@gRGA#R*PpNqeP>Utt%)%k1zid4ySfi<}cO-)Bac>(094eXz(JtQ;((0Oo(NjAF}xy(!!@j(SIp-|dULz&3Lmd>uDkC$S-o zP66{HO}k~6O@}(Givf0Rl-60u6PihlR%Lm4mVN3hf~gVMSi#H=#${TE=Np?FGvmfV zM?2L{J3;U1xeSLhBzYu~QTa)beb^G44U%+$oiXD33X4q8dA>lkABI9Pv83}cRB5JF zASJ&!z@wt^qr{QSruGXaPiD9}`+$x!guRw<+6k<938fD+R+$b6`TU=Tmua({SP43Q zz?MZ}z$@6kEN#>g1op`Nd+J`YFZ?B(#I6wuIWC$x zNxYm{B2vF-;iu4BR zK}TP+0`&72S?93fW3=3o0u`-C3%PJT4vSdWgbC?90HK8e2{GHBRfhnNT4lS|x;XuZ zeWKp)0`>ozb$IhpW=(@Eq?Sb=piQb5Scw40{1Oh0#V%cu9By8aUx9OaONX{@F2_A? zyZ-ajMzi!Ql9R;nHwRUNGYUQ6z!UIg9RX^9Z@dTzC1_s!An%_evYVOFS$Cv6uhK{`Rk2wu1uX3WA;uPOP#0ZOkw4G?`83 z)PphBhj+`7$v}!9^?T;Byf6NU{}vH>^+hVw=b-G^<)L98-Xd`BM0*)Q8|nPoS+Ra0kZY zexkM@@@$UaZ7AxK0#jNRf{A#RLZ=bd?n0s>n9peYRkncUO4NNXv+%=1j56pD-gw5v z6M#VXB&uleFU`Av$ZlSB>VW7guLye5zoc_Skq*)k@SxDcfxr}{0^ z2sJLZdg(0+alNahaWS--Gdsl4gSpqWff$73fyB+%DB~ZAS=)pnXG0$v6IRYR! z8{mJJPF~yfpj~^{)Li?qYf0 z4Q7(A*0F6tV5g6JpdtQEWwxv#y9OkWzir_SF5t63P|X7fGI`gfc~yqZ!6Apy+nL~c zfC^RNqn7A``PIfsf`^qSFb)C{BGuG94IxR`548lz!f5*CYXRt4aNKYs~g z*bK35{%LS#%K%8mI=+~dHbHl+ONOV2A(Qp03_za<_BBhgyBo-z4L>4&ukFzAtD(B0 zd6zLbmf)EFKeoOyAj&&3~M;8tyyS`eaNaX23eeo|uY#TmWTZfpN zZ$6!fjv1SJno!l0ZWRr>j<~0wF{hD?>5hK3VtDDXee)Wz#ou|OYY(~M5Rj}WXKyE; z?ZE9STY+y18+UwCw%9J!D{TF(XV$4scEZJHreTJ6U0; zgy2pS({cm$zA`u{`W@dgfS>Na9d%g=au9deW>HXc*j*R!<$o=9(51LZ!DstuI~9yh zk@R`CaaURkx9mfwI%ud}D{Kl29xhTbUPLt;X+TsU9fepXD{WFxw+eVEPyJg1iKamB zhI(g6g)IDINU$h&!?dUwBx`;nRQjC1#Xf$0JbOrK)oHr{Kdhf`X_N?zHG3o2 zke|=Ct!{tt3IkOjFq;x^+Ab|KA8T{K0*dWyzYr?`)0U~`@^OnmY->LD>pH+ccp2d(}dtLZ<>x^sfhsjJd60(%F zHqA=K6DT)NQf?3^m>Ranx<5S}BYQpH;p)ETs%||mx4C9oG!!SfWj&<|`q&iP4Ip)A zJ+)T8$k%IA(tHfbzcXc%$hPf0Nb{Ym`dLEHh=^*|Mt{7Jdo|*s0SP5j%Ixm9Z5=K@ z*`6`?yF*#tmRXxp7EARQQF*O-)j$gKqLLBky#Yafm{!!LBlOJP}97 z!^-llG(}uGU0H0Cv+S2m?Bk&2m{a|tYPOC;{2FnUX+&l*_Y?i~!L*o3uFbuOUp+v- zrLk??RjWz2B(&7B{HhFF)6=nPcZ3&1aD`32C2v}{>o`p=^nBY24^d98*Ob=}l^9yn zmaY-ro&*Dv&U^&PN7jD~-27H;7U{lO#SH&yIySivXWn|nRj~JJ4=VA3d@T3m8ayy0 zFa(u+8S&Kw^9-tD`tbi5K18`Lh?;mbB4SAZilzBr2$>W4*qAP~r`&2TzFJ2GHBcZ6 z8ux4b&@*0XNa<`k4yk2hs2VtUY zFsP0IVRC%Y187@1n}Ffm(cTV#pIBvMxv4lH#HuzB$je$UbtpBy2%I&m70b%1nwS*e z&vi!%3G9BaQ+>L@1d`XBqb|9ZC)x}Q19}?9E@YvjDjBaL+gCCvNK`&A7eUT*w$19q zUV6gw?(jRj=NH=l0vku32_rKM$+`8YzFdViXm2+lJ)l!lv-h1Y`CnnUBWo6XNw*+7 zF0eY(b?&(<_*}VD!^O<;{72klc83qB3ru?q^ps!}W+*5o->em5Bto}jqHmcnIS&A( z?{}BPQ_%Lj(PEddb;a zdZ0LcKTk_~M_~|EkrFd%MBTL5;m;p&^;7yqB!|oY9C=v%)Hn|{31ZVx)7UA4b+T$2 zBxHN@V<3!IRWx5!@BBmEWcN9r0y+0Tf@749$rqRNMCQdzRvte7hV__9U@-m4_~DE} zuduE8$N*6`|9P@}@ZB>lbKVlp*_+>=g)N^eBp#lY10keU4Y|bxp6n)RPlik1sQcLV z%pq*AbjfucULzD@>weo@lV1}h3$@&u26xHN&ql{%;q0zC_zZDEE=?h9I&Yxwss>C1lSFD@Z`avQ4{$u3E@jQ*l^Iv zR4K6()UHHuQ0?|KXNn+>tH~GeeIS8vJ>tw;7*;g53OuHDw=>)xzCTGs9tEWH>w+Dk zYr3x(%kHCE6=uqUri!VGdS-L8Yo`{Lh;$&ng6ac z1_|xnVTe`hj_yl}65CqK8Na-Ghp@N}USdB<5_g+bU-|L$FuTQ*mg)L@*A(_x5}zXY zusH&H?Xk6eyk%fM4Ee&{y|U0`Gt0-urRLjgVS9x;ZN2ItgCJ7T6)S-Yf+#-^Du7Ak z!xn=0bwMG%A7(Q;CLb9B)hQrQM_h5`ABWN>%gytPAr-S0vN>uRR%&X< zY1A+$`r&{aNkTL4&$Cd63wXs;@Y7in)XnzZJ!OG4MXilTuQH5j;d?z+9z&?kVvyAV zv5muUCTWq#a;@8t&>d7~ip9lsz)p=H|B)7*d5Pk}AN(jwtBX=lm%z2giVP5(7{Z`= z^m^&oNp7k6>Wtaq>a3x(V;zP3=Ehd18}5(EEY@FxjYLn9rNBF7jQ&B}AREF(Ue-S6 z)|(&8F-zf$u$i*kZ(^9h`(@jxTOz*=oI+VOM&1nlqU4V>cdocU(sVNPV3G_y!q(7f zMz#z7F81)Ogqe$o!FQX;zFCd;?O$To=aAFrk$-%XwvAc}t(jM{ZmxRF8fF?ZV6o#` zXFh)P6|Tei(a@-?`;o?LjdGTWHyv+^)r*NGT#<-Tdo9kCFI+>c*bNB@J6|%gYtpMT zq4gCxosWBW=byNHS*I*0IheB?;*#{~*t%R}yGNmb@a`J0;d^n6ey`vN2kWJFHe2#* z_K@^G70YREa`~QF;qnc&hlj7mRi8*bDH8lXRv+N_nLQL+NrrDD7gLHm=E0BIN{WHS zHfo2~C$G{>Tpq8>5Qt>aZpT)Bdur7Ndf%*+K8j-wywBR?k597;uJ2}5JIH=qw%eT- zX=!a;r|JoMz$wA{c9n@i(Vyd34GSB4$Y7e&V*Ih&#nhamt#@GJaHt$%I2dn+Wj$g= zPaquq{%~1ZF$S+s?qhduF8EQWS^mPaVMUp7VI1sBwTIhg)E2?VcIxre41CS?%w9^I zxoY^~u}N0j5|~m;nzDQ$E%7a>A2}ayp8lcz!~glmSv{Si$&+!m=L!U9B9CD??-tFB z@I+qXdB5`$v_HBnWo|cItQEpZTg&lDQj1VBC6z}jElPplJvRq?wspHfq$EyTc}qnR z#FRxCb78{SwLa@^If~?WC$pC5`I1#*Yia1#S>XUdvki&;)sw!tIyYUuL+x`g-bY z#@T94GTO#MsA-Tbxv__W(NbSR!OYA;RNR7P@IL#i-u3X{O~o(YQrf?Yp`}Gu+R@>X z^S^={41c{bFgHheG7BxRbsQ8yHd9{V2?+@?g)9@2kp+RtDGyM=OQA80=Vs+&%SkS` zvo1Z=Z?&Y4aamxXD_C}naAK4k3DA_9Py&ZZQfY&xq?{$rc`gCuDF7&Z#`emnTf0d z^-;Mi?V}!E9^AzfdF4Np-|BSnn3nAn>G&r;!t{Dgp*GVt(*4jY^daWV9hbF@2ilCV z=AT{iCJu&53e_zWFL2*DbYuIo3MCD87f7j=u&&8uL_hz)m?EguQ;*qqA|~R1#X}!b zD3qC8V6U76rMOSQKmA4_`4mgShX>zei?pU3fklZg{_e zb`&StmvX7emmi_mTymK(Rsl9k=tL1XWJe+iQo0y7AC$zDv-eQ1$FJpH8*? zOW~#%J>>ZqY6QfQyd&6)TV16k=(J55-uL!wUc{Z;3Kfd>@5)i5-hRVN!ZIuib!Qi1 z$G4MXYpQgoTu|7uc{N;ap9~z50Qi%B-KWH*+wOL%2Lq9bhM#p~1KxfRBCa$4V>E8c z>+;iz`?3E>eY+zSYU8|Iv>iy*I6q=ac|EijSW3#du+2*%xhRNlpT|DfeRC?QPQjfh z?(Xu&zEqhd`JwnKpHEt0&eX6|Y`6BCIeEs=i>gzay31sO&%DYs4g4CnWMrDVyJ0>Y z@~_l(f=vNGOE0NEOrdv)siye}qws?RIre5y_GmuYGFplf5P)hnlZtexDt@SShvEq@ zK*~BiT}pZKdAO4fC(G{GxT2OT&KyjvQR!;8apA`j5k{=bYu&TOY)xHF@YF@3)dJt# zAd_Usc0+9HIWaTrInAmyoU3HGhpyet;Fv+F*rdWqK#Us6u9r_a&>g|7W(^L;oU}7C zlq<3f<>Q6lHY-@6?lR`4mbfD}d|8#*ThvGQRLtBRRzm6(+P8pkSKe(*ioukv>E8&| zpk^q04lGW+0ELtsHJoU)O|-k^dea{VJP02n4{aY2@$BRnqFj+nJvS`B<;X<~4=t(0 zX4qmn<^c*VlcuO}b!Gz1v{-xZw!MB;in}cYAwwJsd3n@qx6=n~E|4DNjy2suPsQ02 zQ&0-?3EmTn5W5TuL^h>+<8NI2QLJX3fDDKj=u2!<4e8I(8s<2lrhI0-&Cg9};6D9r zjC^Bu+^Y*VUx~!jjzK=@6*{@cEzM{$-&4uEqC0#q#wilhJ?fl1m z;?kLQ_B@_F!~*;KhIfYSt%SSbq6P1i*6m_yS_U6#=Sg|zWnDM6dzvL=NY>?1$Z8|g zjw$s8Hm}`;_s|i~dZnZ%S!o>iy1ND>Y4w?3WJ(D~L+!P1y;kkfGlU$Qr=l%ObSCh0 z%hiw!FC|%3x+m^xG#;0{@Eww2L!%82)&u>U_~DwO##|>)&O!5);h=mC4BHxjl=X&La@a!yJT5HdO#`ZLAay$QlF&&x&1(^htm>w0>66%O1E8?UBa zR^vKtw)`=rSoIN(X^GECN;hB3mR?_#yd&0mS;YVg76}U|wi!E-%I+-Wd_y@qf%sr6 zqv;Jxq{ad{Ni)kJq28)KqfmkeU1vDKQ2IBrJ6p%(XW%0}CU z|J~vdOYNT9)C{-0`r)7b^7AvwizqmjW1>R~$s71N?<321ZXHe`5{+<^=HmVK5+rq- zmIu`>f;^i$s^9?w8ubUA&<$v4r&mN~DXk4xF;j9WCAG?Uodj1!KO{({JTxKg6+&{?WOB{h`6ryPd@ZGfc=c1@IPhr8vb7dP zL~-oId?ZF391%Na;QG!j2I7Bp-yMtl@uKJZdCOScHl-u1|a%(Uso8zh^}@Rpb>qD!lJNZ#(Nc-@Uq(Okm8?dN}4WAdOsmSxIP{y^(DrI7>;0 zE7Sg{M~?PpvJ$p=@G)8%9sfMLIJLuUyII4dUWIIS)&T>!f8vsiWrdrnj z9-~#gcf&C^AJaY6?fJA)F9n)AC07>Tr4?^Yzo5`lV!Q>(@!W8!6tcnw;LDR=9h?&* zf<`NXS53a(KXSw(3$Y;ja!FAb_yS5f0#d#WtZ?6xwj=9lT5aIH|{vqEpz{8LLJ$(q_5-Sp6&0+*IEsx)8?un*+zkn zB5r1J{TK5Xmdzvg;YCmX&{ef`&LSW)h#P!eAKuHGrbF0l)0MWdAfjuxO6iuy zNh~YF34`V3SQ~4g^f|O~KSO$#nTOKnJ9F0=?%WbhA*HVd?881aaVf84 zuRlqw67(Ncc=akZvikxYllhT7cHUz@AmNVN(^-C{-d!7r4Yh`*)R$r5B}IAXZ(X6s zTa8wp7g}I)vlOm_1b88PA`wPTW^9fDMTWrYAizsMKi+d+vY1PbY&p2AD`um}u~E7o z+#S*s8y2{?(5N9bou7Te;t7D)8=;4mX7mes#pYs1+?<&i9vDU~zjhzzsjb+d$-S-V z=g%Bfy91rvyEh3UY2EkTN9IT0<0W5*z`}0*-<`}4kxM#Y%q#f*{-DHx<0Vn(?SmT1 zgV2+#MHG`4g_*~d8MmVLrY;-p1N^2D#CRftBy|-ndqH?OHcSe0I zTgU4t-WW8WrOo0nl00tmZmg~+!lA5hKp1%@&!SQMT&F-ly&yL2`Ik)Ac3xw;I>hl*rSo1g z#UNlSnoUpOcf1HLJS;3?CmT|Ekl6%b1oX1cCW~p0X?F#)IN=0O3XB%iw;0`=P3{W` zRbk?5M2#qc7{W_QdK@XPVCH+o1mN_r#~~u|dA>^2qRBKPD&U~pN(ks5#?CY^WRIGp za1%-6Wx(s;%%CytG`t&!Girb)f`?{h^BbC4JJ@6Ch`klVxwR3oa6*?cLSVW^Er;EKlM5}Jn5h~8zeakq!-y= zJC>q#p>2y}?F~Mf-%1Dz@|Us+LKzrr)@w?82#<(GOya6HZnAis`F~9^^nXd?S$luy z%8{^`5Z))M?=mHOf-4_Hoh!d3joDe$P!{6~7aVP^3t3kDSmwNQ!eciktnlrIw?~eP za9X(2A!bs|V!&r*RXZGvWLUQBUe%n)iwzepE}pfbZS=J4;a8~z+8?}ME`BMow5#C+ zZM!6){(vZ!$Z4-Yo<1eg(9p=1;ZyE-zua8CG8Y?5<`861D-i8hp6|0ft6LWUg1l{N zhPOsW)O2(!t^)J6PCgm(RHhq}FGh{&CMRvwiNcBWk796$YgWZ_F?C_3)FJe%nLaaP0o4V-a2iriI+|Z&u#A6q&*pDLc^`T z*Z9#&Ys>D(;*i0~UaVKwy66bfvCmGS*BLm&cb(sL!8yfKn zz25clj`k-8)yLkr%&478(atM9Bc)>aLYL>(nZlk^H^jiQx;^1@ZvNIam$L&sE85eQ z&)167kDTZvxpWd?1E>1iH&93j)_%l`K)+5Rt?jjVzh-S6lp35`Y9*;NieV7}GIJv@ zh>>hh#&ShvL)@1!dqeL6&*)yro~r!%7n5ArSi4*8J4O?gwG3x?G~>9-=q3lxvZthL zvmAu=dpL8e8b~)WSW^;>xt5sO`xgX4#R6Uoh)YJ4#Bx3!ZK*YpQuxVti}t!}Z=7i+ zgUC`%LrI^~22#WDd?IR;(0g?JSP|3r(mHrj*mQb_$Dq4zb7WOan);At3-PgQV8;JS zBqj4X)sJ!Gj{32|8|1UX{fUq%oyvALhPXi%_fHg5_Qq0Oowc?+>37rhlaNwD<7X6+ z7CgZE#B*spvjFM%L(=yzSXaw4F3}IVLV#Bf^#H% zam<*TVAhi;CP&pIq2!PlUOr%`EP5mw6tXEO~7+@ za&Jj-kD}AEQj*g#n8HV}Q2)61nIRng6H{l*E$&ZZ>c==nCmJTntbN-A@VJ?tTKS6g zL&e45_1e1C8{LBSc#3>QCS3V2cV?dZSqH1s`c2RuQ#GVG)e1e11G5$y0Ns)tevj?6 z$>T&Na%B)|m_yE5k-4Nl8f?uDQ>QOYbFxj6JjI&UO}+6mDf@?pOR#tIjxfcxd19ynQX9 zyU|TOPp`?mXrm=BFRP5@et3kYlbPK_^J!(J>Qq2pEtwC-$>Jv-p;V72q|5k3in=P(xDw(}}oIBEX4U128!`Hpx$4}z#Rt3Z7y1_oS%T-j9#IIHC zmrivpIfD5u1%K8xVRideoRFw(l+{a8wM+G-JZ^+c?PFsd;_c`QF=*=&@yRqj$tz%LOgkcHPBk`sNX0Rr)WC60abYEvg41ZKA8mU`DAV8OzZh_xft_8%)EZC8 zNJea}EXf8TV#@8e|F-0r85J#U!7&8FSZ2jADb1KRRgjhTg5(izd*#weXN38i1lK7u7j>oP z-pxv{XTyM-(pN^kY^+kaYZmFT|ET!J

g6w6EOKxG!0uB-Bp-$1SY)bLBq76=uA; zS4%EI9*ZItWEa%OY**JZ#Biw}ag#Nw^-}S65MnnM{ZQs5C-nc!tpDyOgcCXbFt*Z- zL?j^T-6Dh|9(wxI28}M>fpo6khw334aeEBxQP^33=fj6W#5pGgtsQHek)2Dg?61j5 zpjQSTwER)F&+ED#-U+L5mJdGT#b4gx>7^O!T8fS&DmjAkoc(brdTcNhV7b!us;}LK zm2zdu-$-+AzAQEb$DVR0z|@5=?dURrXH0*;_Ob>*V~R>kR^&azx=2Eu}5wt}>ga#svyp?|=n5mDy+%q7o04V3$4-SWp-QA(H_c6NH|fF) zE(`=qbUx-SeN10o&JyW7^*m`WwS<-yxpW!8Uc;*I+nL%w7c%Mon9$0_&qA3U94&89 z;Mg#A-jXuTZSwwjLWW;H+ilB8pd=I*t{vGPpAi!R+#}auBkFCnmbul+kTZe6O`#9) z8x(3tC(j_`mDT=}V^dQbi(U8QTS4UK*W%rGgao;Dp}qZWyH}LveGew1oA%lZ{nBxp zN_8kV<@vop_OYUJ=pEUg>5PY%ag2?NsWlL3_7Je1D##O!_`0Wk*@0N zKoJr(+rb&-Z>H8Qd2J1udw2TV;#t4hD3(gO7vR|aw z<+no;5ecl7GtzY2*WulwU`bE*JB{{SbCkidd`#rdIdY56{g(I5`b`HlS-dbV5zA3s zuYKaoAY;!HqxBD>PEF&>@4tE372q}*ZI}qQad(gW z=upns1r6b;SRu0RD?*sF#8F->jEst;fd|O#kiC{{^K3=OyxTllA0tC93?@0Z8x#~2 z-uwExR_wtaG(sy{y| zwG^$)q3j)OZ@7|ZxXkul`i@XNWYJ}M)z3xoXmcg{I4CtebskI>&JLO(ABxX%GkL41 zscKdo+<%HlOG`6D(lEPZHlk10&jDV`G#kOLOC*g3yo;3=Ej2 z4irkj?L0(;NTPkkDxbD(&U;wMbOF0FGB;5$xQ46_JZU*aENjl?{fu6rtA`9>DZ=^g z-@R)%MM3a;JPra&6IDjX3f6{JMg zK(9W7;AMQKp0|Wg4rZFrmVR?<6CH6QCSe?%wpR%Y$(MKpO@fLpG;gF|Wyr-f^;3y` ziTRS{=Zt%xXlY}x$N%n!e7k8Fm4gdzUz%NT2xG}(N_#r34oNZh8=_)jBS-2V9L&=^ zTyX6BhhCbXkFuhR{mpEHf^N68w4j)6@oR*pDUJ7-C`Q#i=lT)bw2!l9LQgn3RjPop zF?UQfZ%1R|RYG#g2*>*)Gl$#rCu(Mh{7KO&Te4f&0#;S;#*uj;d1acqyb@ZC8|u$D z_+Q_iui_{wVqjny7OV9$P?QY1&Z1nd!Cd$*FVfuRWx%pqYDNYPn-B#L_S->x_rA_k zDon5bF`JAT-alhU5xjgt)llzmcWw_x)j$dM?on*z{CuV{qmm0^M9cl1-hdg6^4-A{ zb-Xh?5K#}G`*`d^ zk|ssRc6;_@o z)}8&i*$x{Un^mA}gymM497iNOd8CbvO{d0-_9tC4K{OF>KHh8mF!n4XW2B;9qe8Wn zBl5<0gw{ZLW>1k~TN`sllL1+3^u3AFflD~Ffr5UTcX2kYwOmv5z zjn$jD{!`C#m19cop&woQzslvcDAiT9y|UtG%k&Ts3kpD?8q5Eh`eN(^dkD}4zCMhr!|f84mi@ot(| zWfX|$53=+5lXs48iUiEncp^P4W@|Ncu)7Rad*htJXdD%RJGi)Gu*2KO$Pi*JXm(Tl;&U8f7l(Rer8e+*i0n{1|AjW#R?T&XtngD2N3d zbD73yBtx84)Ko7h8gl5drSkiJ0x}a9{Et2XO+Q_DA6mbrIpz*2;N{+vj5 z5X&uw9}pt%3#!(n9v&V{@?b)F8@OI_#5pO6MqBf})?rgM!ZVV*D{NUB!~M;hpAL;7 zZ{dvO11GK@6I-5edh-JT1ud`LicHyR-`T-xk=Y1SK@09NvYv&Zh=PIG=9Xf7-?8|> z2ZSfsEhhQ9a7GVju&dv}RJO-X+uAC3rgwJc8$X&+(jH$755FoyqgXuil>eV15U|Lp zYWL=){o+-+W)M^L^mw{obK9e7DxOe~{+w&7^}o;(d@+Z)NT7>NLn*G%t2zEPm+yz! zzEVCr)Mo)V z=#n@%Sx?&(+(<~}CzlMmi!d~P+_ki-7qJ5J2Tow$z59vpQ8V4j9tbax&fB{Z@UOVJ zxrJx0GOET^*80Y?Fg)2>021;b zDhkPXPv5j)y6rY%WY5`=y%l13bU=HHfPlaYG+WO+gd`PUOOf4RbS}Pq7J9?D-^LB% zv>nSNnclh=ZaGmI<>K|5Ab^aU)X4~xN>K^&){xlOhf@~}=Wq{9r#Ld-L05C(H!oat zsP3fpDo9(f`ny%O-f)m4j@tc7w0BLlT@ZFkaPe(U)ik$2WEjI!t!oz47_QcFT!Z@6 z0t6#{L2A`+!Rg}7$|6)j=bgSUOm6iUM#P`%nOJnw|J3q;B`wDsB(wz<0zMN`QWlWK zP7{9UALy94brV$xcZ-e=NGXH(LiCK@JTL4)0q`OvLYKnT!Ny=A1-03Y+;yU)vxEh-`>`i-53A;YHkdj_Sl`xy~?7b zU*@fb@XEi?5$x2ZU{q*9%D0U)YQVP|GYhkVoccj zt#e7@-IE{+277uzd|O^I(~07@W5x`MGNBKo^1zDT3wLHSrs;nvx%R>_Ht9JAIXb>P zNRBOYAKHrqSbE**OOuYuMsngv8b6t0k2{IyrxsTzu3%n?^VA5ryS65?qUT!_Qu!$I zsq`&JV5?y9Il_$A`jb#%n8()h&z?(4E`q79tVRs9^w4_#Ynv(T*isVT9@gIJy_RO( zUNjP&)Y8CWkiK)vT}R|praWC1C=%);Ipg|x)=?xx6 zw=0&|82?J){1V_f2nF2ZtAx)hn;}!b;og>T6UXQ?vYyw}p=dgAGSC_nP^UP0R9O+Y z@GOJVO@E1|^F9M~oPzS~J-;tp-*VcV_e&M#gJt~D6ED+`=f!_`!%jK8Fy=m$^tnBt zjS|$G6j{?*%Ft`BoU>HAps&05Ci_F4dB}jD0*c%nI&Ys!zi991Zq4?zJ|4mm>!KHJ zcuN%Cd-?6ydhINIOcbRMq3@OG6(#023wO1fRsYoS3@B=%ahbgcPU3<5rS^X0E%F~= zaPLhz`S_sJ)JKRHeTBaa-%upr;v6*b=A-N=B(ZkD+<7r zW}!QfqoD7uN#os)8|tI+Z_0M|F2w(rD$+q{sw5Z@M$yuDqb)JH@heRWA;~uqMeT1s z(a8q9ab+?+>0tQ7ebrV$7)Y{ZL%x8@V!n4q^ST|Lt3%0GOGSf{mCAkJQ!A-s!pRFI^v2nLDK{%k~_09uh9McSSxsTs)@Tku!}H634w; zjUA80XTPiP_-D`!{vn`%>UglyoSWFPlVSH+P9c(;e2InVD#6MP8u*L#g*3-AG3-Hy zhK5F1aO)v`6Ic>1h4C8sv;HrE21;PdoJFlSdjVP1P7+t8{q@W8mgTo~|N1d4G};g< z!Gw#nBAJQf301x}*Kh3QCx7y~=h2?!FR}Jn`ntPiRO%03+W5v*jL~_8k{jGkQMW*ry=oJF4^*|aPe4v-*0kWp7PmT1wi5>>DTT@mUgI>G++#>NkQbrqcZ34 zPTJkz#ubU+%Q@k7{oEG<+3lFiFZD59&W|@Q7`R`#)|c1y3lws5z3Ir2EZ9k~cT`KJ zph;vw!#wo_ytLN?$K>&vjCQQsx5=j++@DEWY>W$mbUJo0z(`zg{#e+z!F>5pDww(%Fw)#WeK(deLA!qpu z7`{RRuD6Hsoa8#^vxx^le{u)NfNuAp#0BJ$PsBQ6Lh!=Z4ud{@8WfH}$!}3VZD^oD zqy5xL&A?3q)*EPdeE-)G^FvH+)i!(n$(7~K;#$Q8l^b$Niy^1tc(JqsU-;y~g^<(@ zA-~)@JCvJzc4i|=@hqok-Z^ePlypo*Tr zHtndDByU*WCv?HN?DtAaN^kfm?}k-LMyQB%jQe%4)P{tR+{i8E^bnq^i71K~BLF#< zs&~=2IR?z_G$6oyOsU#_ZRI1PO$t#wr(0>=Se!VlXz*S|v5KZ`6r@kO&gRD_@ug=q z*8j!*f08AC;yxSDuP}22T+rjx?-o%o5gTsDe@y)Flv#w)bb{11vtMm@jyR_!HOVQr z@!aXgp~3?HIwd!@)L!2v$d0}5mEv#D{az6YLa#ULXYnCwIcJG90dgG7y%Np%7OgSe zZiU()Z*6LAV3%VRTq2Vf&Rp{gpo`p4vh3;g|IB7DI$)#vpWq~m6JbvR6i zEIjg9s&5AIdhW0W^o#jy-aWs5eRAHJQLBvlQC}Z+ zvbMq+x5pb9n|LmQvM~I7Wp{MU+731b^ck$-uX-f1cqo1^g6B&3a|)D-bN)@*{F08w zNjl2nzZ1SBJBPavCrk>ScYj-+-`I23nda?v8}4 zPgxz7E2~W(&rVACRT2fmU&{zqQdI4yT#XIa-y4ON)@$I=c^!x&3ypB+ulgAaxsRrC zLOmfV$NHaC8rT&trB#JFljb=udSGog)+$!< zbNdd*4A9if0*mp_2cO~zp7MCtpE?ZRHw3t8)jHbmpH@H)>hdV%EgRX3Q3LP$!)6Go zwfo=opt-keA7s89qxsVc{`?-M-@oUSR^A>EYlT;8`Ny(9uK*$oKZ9pHs%%%)@{!&_ zDuU#=%Zg*THh)q(W?e+*pr#CT92W(90u9se!~F#z5+)z=2r!XZ?!HRjGfFX9XP}F0Uv(<>!~kGF8bsYOTsFb;^f<2LcRN$j+VtSp$GvKxQObVvJWq%p{y$9<>o_X zUoy3lFkb2P=~-qF*)tuXKih>niZJfdCGurRkA8IXaMkp7^b%cqmew9k?4o)0Z>xQjhnPEC1CB zKuE1eEY1`nH{L0ggzw9BKKW1=F<2`lbJ@jnyJ3oG2Qg=K*C=w6L-bpVpgvJJuQa1m z+CRJQ9wr_|VfRD?&%3wnpHmSn370@I@{IMZLjqwRrwO{IH#jC%Cj&E_eB{DqEd?!2&9r~({qLg)QB-ct>|dwE zJAYVX*9U~%lEa}+4cKnDII_SxO#(qtV&pb0mQ+BX?jU$X+p_XH@!n#YSx1czJB36_ za81Z9c<51pj!beZ&gkDBl4J!K95Axr-_5{KH8eqUJ0HLcv9(7CFk#8PRvJu~l`dey zoXK=^er@L@fPl%y9tJCQ&ESB9gL(T+kR7hM3oqG8crL=Yk9;q&PWh*ci_VGvQZth{ zO$Mj@NvyqveiUY^NXcxhItkAPYh?Ig!4+fhgZrcEL*_*)el``>DC(&u0ze7S;p}7N zF#Ul$_8NI<4Jf3;U`PlbcAc=Us!wy`HR&2gJhx=fYiKW$x?AS{h4IkZ*?M8 ziRdqRkyQ(VhWO95Mt8k#y4^uJYxyuKyu3SG{b}euyF&XAZi%~R2~cL_ zUzGWE5reerQ)!Bn%ufm_+_-P)epvBJwtEbs@n^-8l8k2i?k2ef1HcHL{pfwpEnqsX zBcu-b8BW3EEHmH%0D8+ysb%NGZ;tanS>R^d6Aq6+g0S;5$onzssEs7AjBt8Cn#0zD z91S4ILy`f+2FOXuTlz>e;HB8kVdF6)_^?KU{J0w+kBu#*KX+pT`v~(d2D7H+ZzYE3 z{U6r>*D7b!bz$1+EvX(JCyULed9IoBjAh~)m{7PU@Ep}x=vjlJ)@Rh2{{gNn8XnEH z|`(iKP?G-3Fz{K4gPEUst%n)@La z4B}+6BL-+N)9f2fp6~Sb*`Aez68`fv)PWQC+sQn02S5BB*)q<^X%^CimciX@j*vH$ z5-bqK(6Ik_jQm5uEddGOxr?^bPvadgC;Jw%a(l7>Ep0$p4|e*Q3|KpHp6C3&@GlCp zY&rTLN+$C!`mX_TsQu!P0@h|H`PwYBG_5FVU5_L_a4Qa}ZHCV9D;idKn zc3sXC7JWBDX>RE6lLqy)wg2x!{F(cjR*!K3mW{#cy_@)LqQ6l=k$<2G?L9ypm!Xk$ zS$jhF z)F1f0qSqj<{2lQdCj$7jbUTNx+yW@KpUwbopAqI*b&bZORP~k=C&k=H39T?-BjVdk zHPO!lzeN4C#bOKcFhzvv2$IJTnH#zYSRSy$dcGS$N{D|who*m*Vlq324?r{AQCTH-;iDAlzj zuZmZzx+3G|$dBNRp9dNmMYY(Jc1%bNO&9gZ)*~z_feb$YW@E2tuQlqGVoN3IVVITt z15>Dn@MHhy{iit09YoeWYKdyVnO&Qfzr@traORFu^{?t)SplL7IG()G&NQ}^0*P%q z37NLNBKW;@9t_MgtoeffN7`FJRk^iY!zw8ff^;541f_%x(jX!rA)!bL2na|C8@6;v ziG|TIBEp!ngi2nxMn2w)GSXmVdgBTe*WOn*Q(aZ`46w@Olfn%U?#NdO(N2Lnr!2&q{ zvM>nf)^TKW{d&Rb6ojA{RDS>*v;$W9mw|u1#R;4FOTD`Py)dOciHkEO3AUQ>nm3f# zldn~eBwrbrlq3vxjXSFteI>bcd{ng#PKlpR`RKL$cfd~=N3MLz-y8wmU)%7%D-(?W ztW0j7+;+Ydk=!M3cP;9BC$##UT%+co9_go+A`F&di|f^O&rQzAxasHTcaj$RsEQ@D z!b;|*)Q>IE2jH;hdVy2ZXuodHs38B4H&p^5w7GlZ)1>2?^Dcfb92l z{TgHE-u?R7vuAJTeg&4;6zo?9fbqniyzemF<>yz&GfeFnbO&SxU`~F~7sP@UyYV(9V8#pN!ahDewo8Wz=z|;(kPUS+Gb?Lf{$F<;>6Iz=A1sH+ z(qEaGnYU?Z0)G8^{!C2l0S32U9CCJ6UR^zl#?T)H19!Sdf~K+TvroxyBX zo3(;w^Wz;o!G7709O6Ut3O}cEnuhAl)qo(h&TUs~04uJ8I_&9IygGb8IS4J2+U#+dZqI_{nOYddEA6>^2K6^y%dhYhp=EAm#>q#j{<`7lkWX{`~`-(?JY)Z$T+RYPo55&trat z-SvS^?0e#13#BEO4xr3|oQ+1=iGm;^PS#WhgaMrdE|3a437}(>b|=PaJYZ&S01XTF z_uYT}d|L6R91Q<&C3_haaMMI*GnlXRKiJ#6iYZ{bSgpRe6?ZzmxOKkitThr=wMXxO zq?mdlBI4rok@5pjHw@Wz5QJ5KI$i6j`=B z#kgC-10YcQ*OI0ARo9mrJ)yX}fQ6*j6xi>7)vEep@qjKsAkjCmRf5JbITXnut)@m! z?XV>>ADouoPyX@a$Ket<-NjL}@Le7r6})wU$7Yk+xp8$Q>seao4=0^39kaEnseMS; zl;Xw+B*)xz)-|jr<_|`B=vsz;rVL9IxZWTXd+WBc!`Hr2t$b5T*SQEkBXGl@2$PzC zzeH8ORWlu9l)Xv1SmPk@@_g#DktwetakL@xb(VQEmG$(5_0&MX0d;Zd)WyNiOuRA< z#}mX)Q;+jt*!Z85Jt65o-xl`|5_0xSv)`o0DspND|oaW;5?oI3t_Z7Z+1SzV0}{(FLZi*JTo?RQQSxH&Ll1OX$H6a zXPbi&gMGOU`FBa_WkByBm>dxqNj`4bCAhV{-6Q%5;(?qIKM8^%_3gU`LTFRYYUSUO z`r%7v_m{kaf(tb9cE}(Fj?N&~C>_R{(dGWMuRwRfxq1SvZ%ZhT(@7>{X6u zT%d}Z>F)bn7Mfd!W|^Ez;}w5BPP98-{cCCAh_-A=^5_r4zW&QnEj)wt^{Hwt$NS*- z_-I5o`CYG#`1Pj0Y2BwA9l{sec;`W|H)Z5J^Q6?CSK_R@qwnvM+$h#iz~ljGyiZzrWkrFg@XUMF$53G_gx z&cm+bOe}3mTdqv?QD`YY^k;(FEQCO_5)%_sK5nr+!tD~A8CIk9zoeq3*BG!Ab5GyF zLl-xlFYU*z#!QWP1->XzMo}H_=Q*3&W(fW{A0I6lR^b3LnC5R?LK&6ddV-HV5j>9E z;$Tg6+@g(TNZ1-Rdbw_d#os)GlXbNrCGi4$j5e5BC1qm~9r6@|1xY5+n}xL%C!q!E zI%i5JYUPJ|0i;aHX2;hciVZXg=o|}oWoSEJ>NZ4+vc*NuG$rcw$Z(tv3gaFqcz0p(^76J) zA}c*$aU~7OpV#JDFrpPQ61G^>tjEewui!5UPr%jk#ru*MNPTsJn2~{Ga}ba z@IW^Tjs9ZldNbeR%dgHL)*+;%zAW+fJH2XNtkUS7<+A8|&ElQQlCyj_PS1YG-gAiC4itSj@k6INq|9l0O*@Bnhdx7tWx&h>f~`nN1M<(qXDcbe-oS ze*U|)@KG&ZaDDXYd${e><`L4#I#o$RDatToy0gH>2;QGlPW-quLfQVh6ma*#0mcvT z12u#ClmnhuKF3MvVzi&_ncz1$3)e=Aq&vi0I!qt)`BtL+h#S4` zpkbF^?iie}_)R^d4lOA8#SPhI*O%W4j|^)Bv!V2bhxoy!v#VN{d>7y~P*n@vE*jFb zhFFG-xF#p2~f=w3skEtH?O2PrIL63{Kp6mu6)LJn5w1u)U^9j{dsdHyDx$}io*a(d}264vF z@)}z!15IOK3b$_Muqd|*AfBn~*{Zngx1E-Rlxd&ucwjL(dLBNn;?bkWc}nB5`m$oO zW@p+5rhgi9eYJ67?St9d2sK-)8Lq9LF6Hx?l`iuw7F=UWH&1g{3lo3ln?Ax}IuFyU zvnb-Yzk^20YG_PPBYvq@8t$=Q3m<5$3{$Bb+ETNjs-P}B4>R>;GfhcNRWmZ6f^X_o zsx4bawu&tP>Yj={7INM)1f7xFRZe?)bA!LW zX)68VEkShw6wHA_+4nOR!5{Q^0$ZQd?AFz%PC_lhL5!NP@@Z4JN9O^X~%$9r0Ajck7G{PyJZG)q-1T9t^9 zFu4wA7_a1A#On8t(RvMfR2pHd$Zcn^fggd44Z&pkH{^cvq(BKKw3rq$Q#oY@np%$A zGs0HWY6-Sp?(u=g@WS|p^WgC(67wY0M`HBYz00+aXI9+jnK_$_JIiozahrhHogL%@ z5WAl5c7Y_1i`}@BH2tL#U3KlspXkk|{y!_xCPA;gaQ02xuU|DaE+~0j2(DYJA)WV_ zWbG!oBm7mHNfRu6{pFbaO|N!SP{87L>8DdG)fS9X*Vnb(jCiSf#Df^CjbHkD&1=-) zvoG&WpEtq-gkYq&R#~P4naH4ZciqI0vV`GZoY{M1lXSeRlTzgFI$w#lgXU`t-W8sA ze%i0AP;oKsn6Ru!+amX;6j^{V^)LzKB=`7~r}Sgq3!7<*Gh~xrpDJ^Qd!8^Y^j=QXF zrg6@W))1dt3!`AkM7umwIB1W69f8bPubP@Cg2bfkI;c({@T}OpCZ9#V|Ko&u z3x4dF(&e%t!aXS9W4&Nr_S&&Hq#anKzkeS@30)f#lfaaUjsX%57$lH5o4X~W)cY3@ z8Ppq37y6fv3`|0zQHLfX;ErE_6bPO2gd#2Ef+JTa{-?v?;`ToQFH0GC8VJQ+kq*_n zxmyWNpq5)nh&Eq9HNeB?TPq>FGljb7#zB?NAR8?YpQ}3Ecx5(GUiXOa1bbX?X$l0? zgmdV9Ik{~7PIy*b8Nej6%-=lqB;*CSkqRFbciBDa$P8)pyG$|-37@>?iYRpSr%-mLUUdxYP+%mB6QYz;KB6? z_!kP2n!3VxE&(u-JD(!rV=sM1Cq^w17~6>{8l1AblqP(hFbtSh!wNcTumJT8S1~_&QAq5s5yA2A2DEy{Q0bAeM#RN9 zsoG-l9HZqS^gY-jEmA^!iI+QS*^~A+yGX#Z4h4)Pz|aei_96%W{e&g)G$?nm3*JH% z_{Cv$p3_!wGyi5ql0H))4#neizga(w{aQB+2sf@=y2F5THYjc>F4A@`P$Iay#><%X zj{fJ@gkCO=*REaeNj?)%L`R@ncyozHm&r%^f{xT{*WN?f3hy%UUExvGzl7fg5d`sw zKG;C&Yj*{LIOEKb#~V)9dX^gn^BKwUSp$#O3cE5dE%7jsUQ3}Y>gFKUHskWn#b3y5 zUm%KKrL{VuJFC7ttsZFSs^jWfCbV=fbSg3XYOG$0pD$P#b9!=;Ns|bYyYmLtz74*u zjsTRv=%G(x@tb3SBL<(!C&D~j>&8nh_iyu8bVhwBlk`U6Op>9Txp+aifz6@&<vd0tiF;x zwbAj;u(DE)T8dDVq*qhH=R@fORqJij2sn#C95N1n!S0QHlO>}{e(v$mT9LVc0Xbz- z?x) z_wJ?CL$PFw-6LIR;UJSy@alF=0eR8P5ZloEmvoh z`(g3lXB61M4ipC<4~;C*?cwZ3?E*kTPd&;f zFbLH5mLN8#xmn`oSN|Ii!~y_SK?oco{i18#1@&YuAWl4y{HMl@O$)uWhS+c{`j`;a zbI&{8(tN!$@7-;6nPH*3$N=Ho6Efjlr`e6+17BGuzSD39Ysl1xJeFzJs7MoMpou9t zGd@sLiFMF+^A`dem8?2wQDJMYjZuHM*3xG4tBm_OW6KFI zB@U(aGYyfoqif#>zEr)sk>>V*5l$6qAfZZK^=R6T*{`$Uio%O+=9ss6cONHK6j^vP zX0JWyY9H2ueP~oH6OP`jctB;n9ZS9%rl)hDDXPnFP`MR3-uTnN0Zfpla@J+1(Qg|7 z70PmTHXd@)IMt7|sbFHQ(zD4JUU6s_MMB*r)TXNCU(PleuJLkWO(-@Q;SfaSQT~?RaBmb(7S|B&KTU;J%YU15iI|?Tv2!KQYqSw- z!%J+Di(A8r=mdM|D6yoT=8JnLTtdWq5ia>?!)NYGb#=AcppP&C)lE#yCt$z_yW`0o zu|GUjBEj>9zO|C4YJYDZgnty7D7G%Yc4hvXh}4J*Qd7T=CI@OqVW&Jh$1z>{xy?2@GF|{V z0drf&ty9W8u=jzJS*6-j<^5wCaeHq5``;-aRBiEwAHNkW<GrON7*grM*spM6}#y%JMnE5-WtEte=_}>LF zr%21Ir8x({n+0P&J5myiadwc7gE1c^xtPo0D@!m^VMVV$U3^H>H0CNSU%H4EW@Ul+ z;>Cw4B`m;8ILrD|d}c%gG~lu{W5wt_D9=;t8XOsUgt109W{^WAKKIry6lezRj*xIt$_($NYoUpr9q1Nx|ywuk(U z&KF0w**h^f{>d7wHIGE)?MwowABNv|_E9P*i$@&C*~MFT2iD9lQ#K+tG9Vs>3)dn$ z={Dl*2rgJm*10ryVY#L((@eF-Pr>~`vCZpXFJ)uAb~_H2^nj_*++ju5o{^o3sr!K* z&v84nM{+8eyQu{Nx=@r3<#qVlF=8C_XAWUFL)+aHD%^-0g>rmCmNqD3XuH}ePRsA- z))1tIoPQPv*ft`n;CVPZ0WGb(pyVeMGc{EU8kREb917 zLSf`jm83$gv{C84xJ&(^xMLUKVDKkO(RX;j@8KRBkTCQ^|BvrL!i*@mL}q0D=Tt5w zavWvxXZ88|$Hkje$%H;U_U@O~he=VwUrB}Gvkgi#E62J~PYJ@zECfLgl7Gt5xF)Lb zu`PJ3W$d5b+RlA;j1p*X0Kbpxe6J^?5)e#8KY_h%+5z8)oftr|c{kaa|F;A`&}a-w(ooi7)tIQCxK z1emt{3H*z1d74Ct&Xw-C-YtHiXs|bhhejhWAjwT zg|hg)q`uBmAI!;DgvB%<;8*0PRL?7#UZ+yqOHm);?h*;?#%IlB1cYe3ul$+>%NUbO zPDNU*YDNUdr{cMWJ9%!^x8ki;J+U|%Sat1PbFtEv%YRZUiy^YyrS^%-AbDDlk_JfX zQP~gk_x=U z;Ed2}5`QEi!E^j40FqQz8b5fA)bur2LH5o;6-3fqlYJjbQ`Pv7CQrl<5$68;8+Q^Bl+(25(7x)!}`G9{PW|UeJ_Dq zLtLHc>JuVoxPZVOtUe25|M<;6=q_>P{7$D8S)y|fy{AbM>||q+ZW3q~ID535w#}N562O9`LoIhbo#mN1Sm%zMJME<0#W5z)(jNb8 zZV?}Z5;s66LEz3>OR@wJBfI2Hsep~p3+)hBs!yQ#u5pqDY;*Tb>Y-dV1z8sS;7Dyp zAXE13Q34^^U?uf!aKOL%Z5;rAzVfF1y{T@WV3J4$0kw7#U`e3Lc1$G?G;0|7es+n- z+o{|tSTMma3vD01>wDIsM$h;H$Y*i(7o2%+k;}UFYC%qs{~ynMnh}yvN2T8pX(8}1 zI0w>TrE;*Oa3I*qc-I7eiG(>bL*ii8OHn*RAOVrBs;Cf{M_3}TdlY_vLy(dWk?x_Z z@Y^QgJM#m)Hva$H-#Qi8p99fOV`QNCVfKFm+rRek@3y|&%5eNQ>(FtAXMdd$bJ8VV z75K60mqwO^?PTS)p} z^%_=;_l!&;?BU(JH}a}Z9zq^f?bp zm8IAZhA7w}#};BrRx#x23+Gtg4yNu#@#u8#yvLWkB+ut`RzmO%GUCVZTR+4gije`N zTmu+})4zSTwz`_-&qb*M?u(vUgT@R{8D#TdNrE9(1gu9x%*=ft7m}SuDSGeBDYb_J zm7dT10iyyU-<6gWREbGB-(|9Etm>u)KuFrYbV1Ep(e2{bSiOJ=0|uc`32p6!Wha9g zE2H1^$pIBpzNre4yzjvWJ(JXULIMIE$>EjH7i}l`4-f7D=2O#${c)5j9i=EEFp@H; z8yUzn1ONWcZ#yler)FQ|zZ9lVfAiT7Ko{?PsjWm!cipd{suRk(1z|3jHSV##e3fbkBTLA;r~Z`#6TV|I)ql%|W{a>4hZ^+LT# zL~2@EmP2xKa*5wz17D(m3JoEZiZH0M*IkdN)wjTp!$P%9Oqg-01wMloWT{}&D_!bv zk}SY2=%nO)j{)9W^-PLl9IPHE>g zdMTUiRGK~^+rGN6u1)W&cu!^Izs0BEl6jufTCeP1@d?saC>K?qF^4lO*4#xsNQ+0@ z`4(E0090WiiE9>DdhPNyj60WQIE^EMyDYyxh%T(@i#)hc6_hXO*;=0hJ2^dqTw7Zk zYK{5}RgtxG+lPUG_5;aT^=Z#|#B9S&`bXP2r+^02*_^DDdXq?J1h5-;c`p(UG!w5L z-YqOFR{-u>gUXK%+Dk$KBm8qab`%7MzZNec4p?S-1s>4nj040nr zU3KJd7_R=~eIUU~U48uo;E&jGI-9ghBguT;%ec+g9&7+sKsP2|hNrU}g!9i-wm0HVD#PPV#MyRiV1KnHzxgWmFElz=4)3183ld@T$ zY0IL&8rEd2EqM6sKV>xuV>4j_JuUA3wB@PKSw2dLEQ5Ro5ffV=+X<(&JTw8FM`lPN zH^-F({2Xa7L$y3wE=Pfy@$j+)P5D(e%k{oR$+&T5M*6B;3@mPPU2h8Vu=N>=CPM!b zPJ((6p$pvLJvst0kxk#M2?KfDyk*s~WI3xw1m4zMV^%L-J7H#^b2Rg9`wI6ADhU4i zQ=Hv*5k_?}HNhTBYM`k74pJ^PKw)d5!P0nDKs-958G~bZBW+ zs%7c{JNITpX<6}c6uRG{1v=KQZc3$Ps^;Fo__YuU!S(Cu7?V5Gmr|{}A3Y+2Jq7EYXRXsPaq5B=y7l3gV>8 zNK;^Cp0v*i#_-eSZ@ycodu_LCb$UPCkWQQ;X}CGUT))k25FfOTm~t5YDE^c5k)40a zct}>xJ~t3L3t9v6t^{FKKFrcUUPikr?9X#?HU=HBHQcpE@}U|&7uyM}5T zTktG%kO5odf$a~dL?R4Z8s<1)NvJmkBhQTVvC-%^ylO@;#wEP{9axDTgJ6%M?BMHE z4&$ONfF;9Y$Ec3#WGn>jPO!sVX_Hk)!aV0&D#9n^To-{4I_$FGyA~$Ys9lq+-TbLW z?d~rTsp{(}oxR2j8a85lS-27lF17_XV&arq-2ma>z*X*yr9JkRoLQ?dKj)?5cPvL# zrO}6NF1U6~^CAGPeo%MOGtv`4a+%+!qB>W4YYSNUezvg(r~X7o=Q8yx)T+$NOu7o1 znz4}`x_Xqf+3HzHvdv1RQT~}@N}uc>wxao*W8yXe@!`)3I;HqmKim6l>&VFcQEC5E zGqKbCQI|0{tA>if>%nh{j@q|$NTM^bRW&LMM5j#$hCtB%D&t{Rqi^#8kU;&`=XW{W z`aWi^mkc}_DJ?R6xt}^|_?EEoJ!Kzn1U8UKdcN{UHR8vPCt0h!!&xsFk7}vhfCQ z#pGXIBzi`T-qOu{VPdls$4ZYV*ObOGLH<#G%$9Z0YYgVT{9Xed=|KB)6-hUF?C*DZ z@+lb|)Zw{z25;@^pcQj5CDP@&v=zdeJc(3dD~JnCL-IG(h=zwBQ*LtUzdiq+qegb; zK{q^tCrjsR&!1*xeP({+E-!I}>2AC5YS@r&v0uK>(YKoheX52AO_h)>)lrp7xg6`a z%>M6s`nTmGp(24?SV&OnrW7!wT{HNivqxHs-C`^b%%{MK{cDt88M@lJLiWR9=5auV zbUcrKPb*lkiMdv91MF8_PiVj9$_nwhEtxD9md!9CwfMM7xm^pA{pkLHA{80{g>-+s z&hnQlzH{&z6ltJdh6bx#IoUxt{D#YK`|>~F@YQ>yIJRPLRx<_)il2^h)6Z(@1;rnU zf~z+do*Ah>C~oZ5Rao7e6{V4H2Y@t$+X_h;toSbTLY3|DSs zizaimFY)i1-)H?s8L$eF$^yQ`RrN^sdwR>?bs;ppWwoh$xc~FzEeD$OGwLVRf&p)* z1|C6QqV5d9ov*IFIVE;=qZfCd4)yg916d_?Q#Zfkqn&1Be5$V*&6ug5Z4-gyO*It* zD@O8vhPAE?!V^3f_&W(2^1UV?Vpvddi+o)|t=+e|{{qs`iq5DC19x@{vpuqeAox8A zUK-H;9&%m}yBuU@oHy&M_vlRip8?e0#udtd*eGDz0>WCNe+g?%O9Zy_j7rO=8H59R zr5S)nhoMlv{{j8|%6&I3%#nz6pvvy+zxL)#tT0~DLNVYrUy#4~4oE(S9y)t;D2R<= zAI$^+0_+3qJTRgGg7na|B&d1;bC5*y3_JdDp290g3!cX%AcXblTe8UuIl20gv*B8Q z*=*%V$v>3)HjiSD62*W9Fb{AyP0x+K2P+hr%JTBSo0>?(xUpS*m% zwv5RT_lM7@H+%rXB>pSG1o!|bQvgsNP>=U-DBQh%TV%PZKxmf}geX1t{$mfCd>LVN z6R6eigJQ_nC&}6W#28pKx`TMDFR?tQ*0b~<jFyXma#6aP-)--`i2 z_Z5+Z+3;_y4c=4K6bcCyNdjOlSGg%QCA>{+@?yc6a!>CI8|72T1KM2^b^E*r<-Jk* zkgj{I4DhdkqoR*yBqdL8$Tpuu%rT=uaT!XBhU~RATNQ_W^0<2B-n$RrOj6aP1C5?zcZ)%-TfA@gA zWaaqa$3rsYoS_W6PZz@@j1vku?%sXck;)FbdG(u%McoM|k8zxY^BSduj=0&_^D5dKM8XQ4-~->VHU zXt6bHkz#ReqFkAG8Hb_tXA@%B4939$5NMaSZT@BnsISA($4J^5K%a1tbbVq?rNKf* zR<;Q^1{7%?k98h}eKan<ex-({Z?;k$5Wlv24MUJ8grxXHHS^3>ohrW7Y%(b8NJN>dLn zGDG3X7ayO(dyG9x?unNEo%a@6Ccs)iU0$!{QvRkp$W21NlPhGYkBN)eDfKaHQ*jaq zfC=$LfxMVkshn6i6~ug0W0tWy{Nq&ho7~Y3xgaEX)eTp=-f$T-dE3*qKovIBTr4Z{ zfQxGY5z&i{Q9c41GSfvW5AZ-IR>uX%8OP#-CfAC(85z;g&{WHaW2M=DWgkiEf+y0k z$6tRmK`E+|3t%WaK!Pdnmq0dEHTStWkf=PDlH#Cu=nf}7&7=KZdWG+q@k9{_YhMr* zYY-8>Ud@?Q8ATPP>hJIRO?SdZmlE!Ilz%qhbZKp-zL&}U5GuD}K& zdJ>9<-+x&Mob-DgG{A){mzJ0;&hn8PTx-7P9pcFvp)|Rv&uxFI%1e=Dftm(Qj6Sje z8=uNlJe*s6X&U*>8pIna#d(+Oj=?}Z_zwB<3B;*r`pn!${-$WW8Tl5ko<`HoT#-51 z-K&zJ>(WWuS= z-9(^(P%85=>%w$)dzqYsG629RITXom`fupC-&zf=&mbEF_uWmo1O&!kRwwoFcAnk7 z!ksQUaU)I=be>6*5I7wk^u2iJ&hAV}@_CiFiG3|kTp0dlr*i@z)R`j@jB?PknVR1R zuS&m-Ab_Hkp87wm3w2=$x1VhXIfj>j%x2509?ux#W+i15l)5VrryDr(xa+LY$yQJw zOrLO+mq|P_ERN|3mz5Z|S7n2|KZjdF6QTcN*+75-ZnXdmxaKa+rjR_rB*64LX|6Z_ zh>IT-;yJ%~^C1?JZ=VY%{Ip$!p3(1O#I6BNZx!+gR=gt#I^$}KOajjYhw2exnpE}; zY%6D%Aw)(eI{aSsS$!Gv1^DuL!>K^1-fuah3Ydi^xTh%!emVKQhz$~L1ONwY!P z`K3AJWe2F8SVYau7w2>@AfoW*o%?UcOW-z_=ZEaTU7%-Hef}7V7Xz9XT6+4%@9_x0 zNS&=22)pIw zZ?M=>lu!Ir=5TB=usBT8>`!H1N#jq>CFS^7A0wg`kfh)BZb1UJbiqJ@45rYIb$uevOjf1|?NC!{Ey zT?MzK$~WrI=FTCz@(aJm&zup$uILl4P{lvHB?`HsRZjh@34$TNUZ;saC^7>Kjjr8w zo?>Ps+5ndM&eY$SBbiI&0UsZ^obdUNhj~!CoA60bz?RnKzLw4LeArcJQ^YgKVc`$b zO|w!^b1h|a3t7V0h}kr=y~0JWpa@73BY{#SMgID`?sXSx`r5_>%=;)HGXz8}gp6=a z_Nr*P`RHzlZB45~`9$)uw}GR-(j}%(bltU=$~fQIh56<(mmk-Y;lnkZm!I_YYJoLx z1gKo)Ks8r#CwOV8YPXZW0FD&jkAPokv>lGj_eVa#$RI7J_=yr~%3`d&EN@E$-VmpX zIGCAfmepz0`SqlUyDH(ap8@EdLFIAh-aSz2e{%vO6}3p71KQJX>_hpB_3OG%@DIEh zV)N%H4NgoL-c)^6%-U0Jtm+Uc88G&?PthouMXy{6kZQ2 zLmn#~$m_(MEjnK?fZ*5um)5zm(pchg=W7P+AUbeus+tBLAK%WGDijHsy4oYTq#cQ` zI-}9cl7M$BJZ;muNOX23Ksax?(WDpa)ue0;Yvt6G%{x5@NxxzFAaYc&KLQZXNRT7{ z`aHO8riXU(SL%zX{2_NzxKv<^G8LO^J;^- ze+kN=24u9#)|4t&dHgo!mdh*an7-0Wu*ZkrBZ3fsJtK?bdH8`FNF@?TdPs~})=@R5 zvBdqU?9G2&6+){eb6BrNwNs_V{Z~8|9*e5@!6g;MR6U6IFgDM=Z_BwV1vkLUez?6X z=d8w3dI&}w%iomt&p#`1ZiA-8eZ!1%b!a>BFf1ngHwz!;RRF9BDY1$I6U;PY9P7#=$fq?5RE<&7A`4Tx5Pz~bLq z1ZZ#VLHivF)cAML9kj*a#XQs?Zzxg!fg=cXOE)!fhM6Tqd=G~i>AwW{0}h5!X=fQZ z?Y>?*PS$K~{!EkeIOFD#M5AZ8P|?sb;4{Ds;jz5^{Qt59`0BCV&Qn7)fGzONqq4p3@$v3w8;GDOvVPh9 zDe5-toSl;^@-%?~A(xvPtOF)<0d4NG*_gb7Q3q}J;;$*ADahqLUyTB+>}6m6kS5@` zfQA_#0dHskDC=})>O55P-(G(zx_-L8Er2EU#>xj4RiOtip`+2Es??nt4md(>2WB7& z?64wu@GdacQIVXCj4V?IO!vDwHZ$W9J9CmndvqS-K)3n!;_O(Z+$u-S>KihVXm~Qt z`NEc~{+Gu2Jq8A*o*}r>H@XUREGfuXTXj0I8TsMxia+}X`NeutlyL>EiF9)6Du>Lhc~8eB_$~f)$iQA zb?epB{{DvPU5AHFGZlgKvm3CKO=TlKD_|^P??o!3IIk_=Xd`2yoI}ou7&# z%Rn!xbl#wHqYD^EitDP^KN(Qfwox-{Vl0w4})-TGfH=9VR zw$NT%ZA^2?%i96~nu~KsFA+q(>!2Cxr0V)ohE9+V&PhN(vU7sU*V1|;05Ls?D7nEQ67o!Pgt}caN z96DcGqoSfN(Xx~xmI;~yX!1fEbg9V9$tr};X7G$2zfx2rblT6-da&Ibs9LC>QK`=O zrl)ja=|}Rlec&L*2aZ3x&-R|!WUfFa|B5#$!V^e*z|A59(sUV*^4UtMyX>@SG>eRg zfJ%!UvOz=A`oW)7^Zf?Lj@XOeLTG7e?b4}=@mXyTJItJ3X@oIF)BZYdQ_JFCSca@0 zNSe#f(17n${fGZM*7rXt*WIo{nnaM8WHeHa6LA3wkekX%xFB`yDz?nc2ei4k3)VVx zN$t`C)DAX&S8bo&!`5`ZRqLzLfJj8;?%(Xz>_bL&VM14IhJ{2ipWvj@qN!(y_R6!J zwpN7>ztadB6u!??Xn{aJcs)S)qU55b6YQ6M=6ig{5hN06^d5=jn^o$(=@G)z$dzn; zs;zP`)L3LcVn|2q$D@C6nB(t7yme#P={~Oqx-K_(-x)BFP+>D+GBM+&(g3XY*_}dp!~Zmm|bWU#;CuSAPZ4Ng-yf zA?Nr%ujJq1GM@i%vw(5M6kWR{B%^348i(4QAwb^KO+C)`z)Ig46{ms>8vE`+%MaLD z?gGQ7K;3Yat&kT0;!Lb&oFv08dmw(eht54W{&%hB4$0r59Xvcdq#7b%2L2XbUZzLz zTjzeU#BgTv@bY5v+H*c@cILS$g-+-K>WmzBn$CCT?y#VXUE#}DEE&znH=t2P0ePEz z$zbRpLw`^=G6CfTI1tO!2z<9d!0zwk=S%!E_9f?wMdzIra4KESl}0G=(< z{M_NNSYVAA2)nr(5f+x#o4ADB+&r5l>Y8B|{qh0~6(10dN{HX{Gjl+R;x=mn{_)uZ z($4#HGNS${|MlAbKde~EFpg0R{}-t|an=D*kAwHwS>g?2)$)ZwP})dqP$x(#Gn=h( zH7}Z*0I7m+$0VRIBMp<+pp0ABd0BIblIyX8f3?Y~HtsRLRt!$c%={L&=5l|DF)cJH z{@E3nA%NSKRoWjGFJF5{*4@)V{}a?Y*|5&+E7Lyrl*6l#7$yiQI9;AjF&TVEVkaND zqY7L!+&KacU@$LwqXF~F3*Za6%yNIQ5HvMKftu3$5foBg-eq%rdHCda!qDIK&TlGQHEuZoihWA#6@Gd@ zsi5U7z`g33sRD6l<%uaOUKLIPnjvHuSt(HH2q z&(HwtiSQZJR;!|7MNlK|_qgabW;Al|@dG!Dn;dVEZ*Kqawc(W_nc_`G{x)`=mNyA#wqXws zPTX{v{FlzAQ~e8?{;J0~sfeVskKX7nrzO=gW;`=cnOcoJHos zm&pK!w>LDt!RlLZaI(Ib>3E4kz&18(U<;WHB}#TTGvhMJaC(T}L*=e9?6C)pcvk|Q z;6DPLhVxvtHZP+)qv!RLtUuuO9fcZ}mt0ME*TYwJKN%cSYeG*4M9~(Y}Rj(F+ZmkUL2arwS3eNDO( ziA%{k^UQ!i7&cC!s|K(HDd+pI77jZ=C2!No#e&x;=8?|UA);$XYNBAT?K6z4K8`=U zL(nbnh_jFLc-REzeRp@l-32XLgFf%>(A^)1=* z*Y0llvpoah<{!&2`A{;iriP$=v5imb-)Kx8)aHli8|y0jDdqaD@E4lt&F{Y#cjY_0K)d#seiR)L~B zrCejiIs0(o01aHhFr7K?_44qTr=^-HS(mmt%R^R>vV4Pc;hCZ7#{i{VHFZBhO!Wi$ z;fHH?6<&utSBecZBSK;ds3Kz<| zJ4I)~Neaec#dCk<$XFYhek3uGyvQd?jZa8@>32PMvG1so8Lu-_R~_E&)pNTvj*SQB8$9)0+V5-4*w2j ziK=V0BifW~6cnZ_nlID*$ml&@|B_)9+4Fc4Etv50R*s`Ph8`_%@oluy`BA~C zP$Sb6(&Fz~`6-5nS#2kzx2*|_&+ZLpw(fD5M#wG1nD!glZV432F&&a$=z%7ABozvf zb_FAm$QLOhNZt@NK95mSEe%cArbNYI0=ZS)Yn#fWHu8}v6M4fw)r()p%IVq`AKVbP zmiT?@{f7VnDVf#UoP!Yu$Wn;CRw{Cwut_k*>N=sF%V5hC3{FfFa&JP%pS7cF93dh- zEC&iW^2_EN#sJhR~D7dC~NTouW22 zsIj&Ss$3}bHo7r}-GS}K+Oh4S7=EzS_a>H*GS7*PDi(oWZk&}p)XR~>lD0s*w@0+5 zN#YU$bA4T|@+fd2@4`u7gGY$G3PU>W1le8O_8D7R^n;a4XzT&i`VSnyt1DX;8JALi zY7N>|C|qYzG;=5@ee)AfEag(?jkHi6Wv3piE!R#g2)AynXYZuYIpGJJN7_dq%)$VSY~&zlSfQ@qJ!?g;oDSpyqJGf4^!c3c~4@6T%3R9man@n7A64ipz>)?a=whOHJfO zHosQu@W)T@-%uXnNK{bDEE40k)uSj<0}huPQJ{~v>H+)#t(Q@fzz(UZ1*n=-s73US z;mxzZBSzJ=;vTeV|$A|G7bZ1 zeChZrNc?QBf6L=0xxwRZS}}&N_zQm9FC6&HF>onA6EP{JcR7fTOcdm(X$TrIwaQah zVa%(0#(zNGN*CS|TIo7Tq;+6gK_O0bT9ofMb-KyxvH{+;x}d z2N!V~-o4p$El&T9FHIL6>?*>SH;9e5cP9^(W#^`m%3s4?WzOdj@9%%uNuY@FXTHM+ z2fYFvgh`Kr97aL;QL(R?dSAgoEd~u^Ge^wm_@Q1~D9b~CTlHd-2!gG7K_!iw7kC0Vm*uLs_W#!Z>6m;pC!wVCBmvM z3WMvI-R-2Ta;>3j8Lp-$xWz!zbmq;#?(A?a^aW0n3yVStK~&nnYpxZFQNwaI^_+~% znxgPmjALBHMXPZl=dnL~Zhs%0RB{V7j%$8zdSu(bMiUUpB$A$!QT!9`g-}7y)&+X9 zq%8@9M_2O=mwxF1$r(`Qk%qZ}yo=~#@TEiN5`Q^!J>;%(KY0;J6_Z%hx1WwT_W9&a zVbS9=J|phR&nurA(_7@kOe-Bjy^;z*YDTAVM&GDdky6?NZ#14Ne%vhmm_~uuiz35i zdv|fo0>o-+nKDhIg}%BwQY6jnZu2rn6Q;W_f4tSHQ`n^-6C?0yzlhgzfl=YJE@ick z<2xWrw%M#a?9ZmZm^Tp_L_2*DrldNolpwF?nNq(h}!xt!fgxsO0O=S&U9g2YbvYWh@6R+>Ey-Txy4*Fn*AkkbRW=Xlv=GU{zv{jTGacnooEv#ohT5?fp~yrs9}0Z;g$V4l1H`{x!cW)n zZR+oSmX}W`&GnfQ-?)WA(tzgA8xmMKX(0qkQg@Z5V0oRr44s!;J~h9kF)}rfbi@s% z7eD`penc%6&&`+GepRp--f7EHoOCojv zh|m)Xvh@%7kALw5NTHF8Dlw!E@1#GXaq?OXi@r0j4e~kVuBySGpkvA?%Q(Y$Mc&Yv zlv*F9^&1|hkfo?ehF;HHdKkFKwO>D=aSV}!9EDQvgzy?EfpTM?4MM8~mX$Suj-jkl|AA-za1EK)ZoJZxdbI9)c1 zI9K@bbKwWdFh5xJRz)Kd*wG>S_s1}^?)RAUTeQ{$%pm4w%I)kNkKT;X>6hW`3|)3L z;kG{ni*;t>kp-X|^eS4{kRcKFwZ5)5MzU3u?A$m9BEgzEdT~TaI@(RT_*9cG6YL{@ zOf{dY?`nkYvRkH|m|qd)yVuu~$D^ClBxGG*{zz?>N~>vCX0j@nQuM;tVIbz#jQ$pQ zT-PkDr0VUA`lv!T8;qC9M?*^XwOlhz%GK;zp2ys^fq%W>mjXS=t6+rrmlc_$#(is( zsuE`36BG1BQ(-xeDtv+s4X<>FlnM!Jw;rq2;y(J$`qFmL=zs@++HikjNb9I-tOk*H zc+wtz(G!w)^{~H2&QF!cQKQZ4YiI!0ffiQxhJn#(@!9&32Ulrk+77ST#mRis6Z-Km zfQ6(u6dQN*KbDALlPOo6Rj#87QY+Ylt0Lf5{^H(k2NlxSI=1opnrLcGlh~3F7U8gJ z#qL$A_Y=JzOFmnpshYO0LrU8;0P&Ykj+%`xNVZaO+0ZE-3OL~;`^ULjYMqr|S&P`V zu{#Zwe0@AHSqHO<82dZaURIvE`jh1Za;o?^X3SEZl^ykJ3^ttCT2^tgZ#Qpsh%4{u zE2is>ZZNJo{#n06X6SvN2Z+Rm4?nnxO=8_dxZf@-pmfkM`@dhmAnRLG;g>(4*Q|?I zj8~_hdaFguE9+AM9HSSWcvhi2SA!_DTze=ofl#L7*PM1cwej7Po%mmGV9fs^P+W>t zZ_?+qe!FsMh9Ju9ntCaQuTw}`*Kn(BW#>}&u`RLVaws<|{OqCpuHmO=a;Bdi%K~e6 zV_d&$WjAG8)0S(DcHDp|0~auRvDecCwa`X|$yjT4i?zrHj2UX>_05#mIKO`9!zVf< z9_d?C^}Qrt$JKF;)^u=WYQKi5s0&~V?qwBdrL2iWvu5{^N`>VmglPY`+w_Btc5^I= z@){;bOGOvd_DzqDKC4GzCtOF@^h?En@#n0!eFN!U@vfsS4lD|ei%)(ZsmXnNLQ9wV zHBswxVQXQn-;~6VzuXm!f%$7cas{!or)2A(SLNl>Ofes$G@Eufb#XMeO`Pu2`m#n*T z_q2pshW}wg8Zf5<1QGBIsPdrzK@4CYlDy7oW|;mXe+a;hZXPSFmjb==JBFX2Ov+|W zQ804x;8e4Bk_Vi5JeqM2{~K)>ktbECyqA*Dk-&0m49P3JqNJab$Kyq_$bS8-FSi#6 ze*NMBk6i5U7^Z@t!NHe9<}1;n3-B>c#kI6b>rAF&!ZBb}0aCoj&_ng9VKQ^Qn)+Jg zlTup)5|tQ(Yh$%|y~!Q=u>CsED*AF|Dv; z)MLmA1Jl&0Ls7{k3Galn(msaaHM$kU-JlKM-d~bS8|#0yd-;FmZK9^@D6xV1V$%s?P8-`;M9!=Q@QhF=uOQeB?kFGC>``O^duoyzoh0n zBA7no9y$Jwl#J1(b=o@U{|n^^_wm76&)Q~ymCDbaiIG9eh8e<1u> zGO9&8dj`RdYMO#&Rb>VZhT?^xBX;-hnoj+SZz1}@BP1^}dRp=(9n3XbQb;oOa}Yqn z;tSxeezHhqcsR9ns_B|HC8CS$7nupg z`7BND{JuI)4b*I7S4gU+TsmHRMtYJE@?g-yEgDGW01;dGF`_%MQD9$TGHX7hKH>GG zNTWM1#mi5rpwR~!`2r3RqF+Sf3jgng>Hm1HBhL`hk1slET<t3TQC`a$C7j-tawOvcrLqR1Gm0Vkoe1!Fe{fq@AZgll& zYr#TkCdFLHiUlX9Gj~Tt;36%bMzR3mJ62ZKr=p_0tpUQ5-zgDOvfQha^ukFtOU4Sx zN_b3?I}M`U88~@gVTE1PwfmMD+;MJTz~C?`M7#JVAeW+5q#8Ngv)V(Iyw~7wCVcNe z##=KrDdpMr=}q`(EccMUw#TC>{CuhF=NzjHl4Z($wk3zJqtgbzO&Ih_U zcur~}lktbF^1SAAet`|elhm@)MpZSMjMFvrAb$HE>lb}};e}=Fn<2k73jJZ}@NcR& z_gn{6maso$JenglIz)$}CRm2n9XLd!h;;qd!t}DzWq4x?c6keRnC~oue7M* zR`2cTa8%j*f@x-o?DbMVO&Ryd8zY6{sV3>N`2rSU#sCZO+UU%#fuG4rd-ZEod4N&0 z$7nFui~8TY4Udu}!$VpG15W;d0T(5Ji21CoaizEZwUBS=RK=|FrK65Nr>h2Q{*ktp z_NTa=`G`xe-|ZT)=R(zhr2OxMh99z*!uykFc@28&v=x<`@!3J7ju8r9B7VKJ8q%{d z{-Vu%`h(8bs;O68We1=g zGIheR`%ChC&tD0hTZA_77_#x%u8ru@fL53ovm)^%qvfo>ysBEFPq(%6qz40izoqGa z{g&cGInItK6lc>6^oosBv{t9kz7|poC0re1@VsKOdi(ib$bICI^XvTr%eQmV7ReEy zO5?K<))wc-Mx4bQzhEz2EqNo6?wYmseESA%u_5x@TGakDH~*sX7A-|{LgZR$Iks4& zpypTdPho$AMY@a1-*HFnOT)NisGiN;RZ;tWqw|FkcPd9aq(C#BvJ&sT=2u)d|9M*~ zOh<|U&c(MtM-7jjt$nQzqSQ`_RzzFM74@wxYWql5|6E~aHrf?}8M)DD!wKoN9@h$1 z+vC<{Zo5Jze0hA1UQl+llcbX6Oc9&gM`roIOL7+%?lPKmQSinK`U2WZ13D zatsgjZnWPu6qKM=mAls~aS^S5(7(rlGuD1wQ&5yg+jFePF;Y2+e$999F}Y?LD%eDy zf~kgqDx8^anpr^nPEQ0bSLc`eA8W0LwRAUSyS19_pKB(BwfNCRMn+1zA5I$-{6+Ge zxLJkEzO;7xiYEDD4N9N#ray@qx2FZRbL4+d)IKHmSGk}svI z#_@AyPnf0eiG%gE?#6wMg)^?%jn@0tm}t`IuBl`D7tV)4^oAaH{<4$(+5bCSrcB~2 zaO;~GF0Ja0a2Ao8lrBeMDLvGoitm-LLNfVhI;a|G3fF!8vAw7Q`+(-}C00@p) z7@G!8&)5)XPg_Q5tzTA_BwLy(-XI@!2o&Fb0YOs_cy-No+ZtHuF%*S1Rb?LjdQ)_q z`+5M`lw6!GTaE(DTJhO#;4!__dH9RNqSqo$MLe{zNy(4DQc_gKlq*7#uU%p8p4`05ltOA!7hk;Wqu_ zG_D#ha7pU^H*WkH_qYqLme6+vpqFrNr~&j6$R-$iNBnCvDansTp*4E_p*ue+jO-Ur z`hD>w@KL4kx>YDs6-{%a*#QmwK6e<;ccrtwrNp(;ZtrEmralRb;J^D8(IU=6y&3r= z3qmVp#GPaz`Np;z-AaSqf5<*1Eu{=jzzr3@%xotA8?RmB6^8he0!6vj$%XDN^~a2C zX4FN@3xg$|tdg_cJW!w1olKd#mDr4MUw+49+WZyO!THTa)mqSu6+SgL-9EQ&YfwXx2sQ$My%ULskOiS2si|n=MQzFv#POW%4B=c566j1b6`N6- zyk-g+KM%GcS}jAeq7`}PhdNugv=MkcxQ*UY53p*9;(XOVLE~^JP@-gSIiHlg7Z}*0 zFLnUmvhZq`o{AnD=u*p(5h<}OD*Vw{zem$a(&0WsKyw_ zncaBu(usvOPR-SVlFx1eEsr8Bers9v)rHVi$To@FF%2Fu6u|<>q~>$V;ZdxHgx=@> z+5{eh!-hh4Z~HHHBP#Gzg|L=Ylu`ryV8j~0R^asW1l0J9=-LZJY)%Y}xx{*_H408N zi&R@mv7C5g0m*I~Z^UJ_T%}4GRl+YiI!Mbt5ry%H3&r?5NbK-WQS#nkb^6K#4ASHi z8|$eH31S2D`&ZS6OuxVjO$g422wUfozJ%#DP_E3zg)pp!)Z~ju&FJURb-lgp#hOi@ z*~XVuISlB6cP7-B?kvqJ>Q)94i=6PA{3G4${0^RYV!-Ejbm7LG-i;jv)0HW~m_@Ql zH>Vux!MfE27Ts-z-G{O_0Ykq1BynH>+_Ud!kJ8TC*~~?xObg09+*%E{>Kjl6F_qv{ zU)Mp=4%$VVS-F)2T#!pn>^Zs8FWgQd4;F?ZsodyRAZ!xtw8&Q##TO(|CAgHd(PzlABB=u2a+%71Ox zmzdn4UGci_;D{CoLd)1rxYsiHFsU6!fEXEXQr{E!2{=anEV~9V>NV1|Z#;1<16`j! z7zt3{>keCNsP#;;14U2weyEG8_07_^=iNlQV@`L-d9!!vYJHvB;luPj zokKW!y?3u7Uy9z@;o+FQo;lz)+2Krjg7ws49O)ITnwOzH?f~HCZg2&3uWbmg#bqhD6+d)c&XoV%Y2m9VVULsr+bI2!B?RpTnxi+qZAY zUF*Gg)sx|JIl=q?n8H6KzTh`C{Q?+R70y?myby3|VC4G7Y}s`H^3n#ZIpY0&ngRNr zov6lrz>8*T0mf9_wOAYv5Vum&&Vd!ohNOC`bzpNx$6P;3f$4_j?Bqj;8RJJ!lGk;v z{Wl>K=bfIbvJjdx0Yr3049vqT_?vP=42+h+0Ne-b{m0&GxdU!M*?adj!~0EYJ+Y1u z6IW%47EwOg8oOMGSRtJ(8huLm~uX*c@EZs(Tb4c?>RNYiK6e9KEu)Tc(gsv4{H*^+g)n8HKQsu87@_&jp)h(X)+A13x2ij%Y1*=X?1`bYhXf_r->%;V z`c}LD;7EVaY}92U;5^e42YC9x+=aCQ#Y<7yVkH*f(6wzY5Jpl$rY-|pm`%N-%Rek*9^NEapAidAUOmBIw{L(WN2`DEQ zY!|r~$Q{N9gf(AJg_s?)(_*U^`^8Q?LB{2Q+Kv6khiE9B!7F9x*G~IJV{Fah!VR06 z+~~mF(cV)7gZHR)Pp2*KVJYrfw;}B&S17vCY^LSKTa?To(i>|8F}C8jE*12&M+>y- zoBcdzRjS)vg3idFjhoLDKE01@KFcO z6HZ3S=k(fBS%B|`t#u#BKWA{RY)O8ZBff)O)*fJ}p$F${q313)(JP z!uMnUQDt)X+?fC`6z+vB(2}Zi36TqlEpc25d=jZDO_fw8ENrLr!^V z+va>xmFzh2my_e@RGexUK0bHph>1DNEDe>M->K&BzAzh@vhB7PAEldHUZF~RKet7B-3s*uf0&TcFCu13gLSt&M2YkEojA$=-;Sz4V=$8!ji}9uK%nvh^CI zyg^6PmFWc5=FD&O<K5nYj@CtD`%jqKbng_a{^=+UtC+4a#lPP^gaefT*yQrgzrqn;{ z3E7+I{}yvxP*Bt1|75tI9$J>L?lg{S?gQhWZo94g5N-sT%hc~84s=oU6)=SAFJ8JN zkV$k91;gfWpMdS0_c$ql2s(Dng!SS1V}L%k$BBFOw-#9J7PVy+*~HENYLMV)Gya%V zWNKrdl=tHAz5TP*XU;uz9O5mMHw_ZBzNUnROb+S8SeS2*=db?N4wK+Cf6#b>2G4Y+Ut!G?Np{ z&f}0gPp~D)wgvg(Bt107IdEwK=jMp=w=t1A!RdV=Wp|dHGd@=uH!q*;D#oW6B(GJX zExb#5>`SSh@k)^S&f0R^kqN5=D?SfC^XK?-e_}vabpkH!F&M3^gkJF|dQ*YpQSJ6IHW`A_E@`7x@Qrj4T?JT105UB+WlmC#w@9I*7;O~?xua*Q|A?vUlRi< zwnOb-f2Wk%DH!{%fpH*%skpqyVU?NwWR9fK30y zq0&(l=w%V4=GQFh2rYabJ~&`o_@dQtP$?9vU->m%|MMh^zlm+2)Jm%FaQRDMeuX4g z{hCr_5+I8#Fzj@E+nyEq?d#wdvX(E6Bm(9pem?gTMtlB@p&S0$3+L;_akPT_ z1>Yq4Htxs9(;E!3PP@}+`@R-u)XW3HMxyY59(QZeH=bpilnSC3*MxpmV;QYp@wBF% z%M(-viqbAme1#kWoF_v)qsT;cIVZn4?$K1zV^sAku&Q4FnkFrk1KCo&xodd-8-!W+ zcoI#RBy5zz(K@MBdOWfbk(H44>EAki5i7-W{LCvBe#TD}^Nw#xCH>nO8tpcsry=2^ z>aYuoNupX9X6bi^m-$S&O;viJI`8mW@QX(hv&Y6O=)N0qwW#YWXo>B|#&YG8y_5Ba z)G%j<+#1JRAsv^gU7i#*@sOU!8qfc|kH*#jyO5<0OK-ihyeur~niIlKbueF>KzH^L zY6_NalJ`hJz`0ub&Q80q%F-FQ4``Lm+Up?++@$e_o#FM803N3!LFNR`Nxg@Zn z&}1(8M~M$JK55CBl>(5o%zOUL3K+z6KB`>pc8C*vRQ+BHOHo;T4)euOgQ%TKE}D#1DR@|mmfc}ts~Uvo~xv@ zESfjoFuV+n5oKXxdoC(UDI((eR+XbGER;&XIpE=g-M255NI>A)w)^ipMGgXpZL&s9sOI%Gzd79G-#SbY0%dM(mv)soSWtzta=gMBg>+Fq~CO;%#r z+o&6Pz+G!E&m!x~e+4MxnK?`DsLjSZ{1&4n4FUp&wu)hjFhs%1k!&V!4P~Vrt$tmo z&txjfj|&j?BL1{G|NI_eOdc`$W-TyNxci_XP!rQlF~QPC0f%;7cQ-sP2tTvNj&C8d-JTnRN8^KoetMRbIftjidOyX{!O zEGMd0*6Rv?wP#d@`#eziGq~nfYeh7IonLdAh^j8JE8&Jm4gs1Cc{`UPffG6mUgdARkAEFuEG(RCPn^WL-uUBbIm z)Zuy!6mw{hkf&?DI+ASCCCBG+ z5K~(MUyhZOId`-|`!94neyoDD}qs>FOp=_goDbt?|DzkR9O$5`m_pSCKK z&JnC}D;v$D5)T{Md9RI1QM;Mb0A^*?vJ0TU;yPaZ{s3Wgkjamk3Xg@C^r<6OeG3N% zyMMhf)shSs8})&EbV?2*WJ^v^ACM7W=RX(%l6<5zf-Dl@)>=liQ<~&-`o;$zDvfsx zK2_I2w^C9K8uJd0v0%lSC-!Yw@0i6c4+H$c$Iroxp3bTlJHf7k`yBjqQ_l6JjDf>^ z7fTKO9Mk7i_~bhQ(jPN?3M zB`lnFt%xiL)$Hnt{`-E!va6Z#dx>hLcnme$7*uI?d=TWdpSs&PalXVyclLW|pVpx# zE6R9?#qm$jO6b@)2L-}!$`G@+>^oIt!P+@B2(gBitOG6{t!ep5k0y;}yEAgcneToR ziSeC;sCqqhSt7y-Q(v%%m5~&#LG#-^zwA*cY~k*IIV_WePi0<2OXG zepz^xee>D~W~$^(so)EPc?U2@@=O~x=L895U32wwk&wgO&SUxZsPHRf>A zmp9C9#E~J8wntO@q(Zi$eZHC&Z82RS8YtUrahAVwo3xOz7FvEn{7qAta71Ai)O)3M z1^J9XJc~p3hO3MEpDx>M4rz0mh*tdwgPTD1+=aH0^K#(eV_XB_$C3_oc;K6$E!H2L zXImr`Mt=$T`-F%7j7C| z0d5zA&w_^h_O!(}8Q(bV3zNR&)Ysf>42W*cjI*`<*Xs@$215w^=;^E|UQ(Q9_&v8> z^7L{khFXPH*iTi*02+8`0}fhBXQ4)i0jUnKWH-6kQ%2xFT37kLmTQ2)Z}vZ-e@CbC zKuzm#Z`iu3Fa0_Wp&E|)dfot8sVD6c6cd5+q9PPc;6?f6wTFx`C(hJRgLBusIX@t# z{eDSf>N%yZ>~0pC-7h95;seMvJC16X&txzu>f^|iWX54V}y12g-`5izmZFi#W6f{rs9Pj$ZzJVFOK^CKYA#ZIxJ*uX|8-Pz0XXYz`#8=%=Dzp;MH z9wt6#0#g?Sj9xeiel%*06mHLy1L00tCTZ~|kynPyz8@}R;>w;@Vw>O<9c47H`0>3? z1p}i)VW#HKRKY>ZH!vw_8gTrR@L#&P*IOp8dS zDozi}5rrg@ZZ)M4avjCY1%Fkac3)|2YP@LtJ4pVim2y%nBuqO3%56V8#XhR?t{BE| z4#FHd69Q{M{>gnFpzkaqwu(&GuH@1-f{pc)@`$@R-(n=Lov@Y8Rd# zz6}}-!}HaA-1&ac6xWp4>7iuBF?Za0b)~pAyTsG3uw%-Kjg-f+Uw-Au*fWkR2v@cJ z>g1Ahs@Yzt2lc~eBbsFpH*sN`6Bpk~LVZvNfei=Ls8dMII=Zup<41h5snC z>9BeP&;;UaIe;^;GfFIR@LS*j`w=y!q^)aN3qiW2nJOnd*&sD}r}`N0iyxwLUXjpD zfwD#&+*y%%SVBljMdHl+`mM@}#OGzX>;7BQ16{pb zh}}Gwi6n4xP;(&C0D`F<$Oyw)ne0kQ`)cSN{A7n>(6+pRa1VwV1gKghuC3GK`~oS8 z^)3>LUG zmsEW-Q%q#GrAZWn3{?rG4s6FZR=v4q7_$DV+<4|Db-@#u%ok2kjsj5aMQ1^*7C zKRqIB#$Q0T*0!jo#>~>BJvy=p!8+{`$ll4&tt3Xy+wB-g z;8yCJZgsl@NsQp0G3IpaF5+*sJQQUeT-(G(rebc_?mDr`@-o@ro=qpg^do zza=X`|D|{%RD^S;6Ea)R@|40jR|#z~FZ6nA|E+JTR;|z9Kg-UU^$@HT-8@&Z-2NlX zzp)6!Mou?GwNnT>Sm`}8^@waHWsMIpe8k|mwIeg2-|mqOvkuEi4rfAo>YKLluNt<`=@e&BYY!T8?8zEU_-H`wyY%*>N9k6daZwe&;xz84K7|eT%#_T;CqT964)qr`c3Ujg5GS= zaU!HU%3!v{KdyF|jM3i2yse;EJ^Z(o3FAee)fg;j&&D6TE!3YB0w?I*#O9ER z80N&4c2r14Y*g-i6O`3u+Q9o3dr*v-*S_x}Ahb-XEt{YZyMD z{*-$vN^$C^MC*llHc6du0`hL7fb6AIv!|YJ|#e% zT5P_^obA2I6m$);n$*ZRRn!)%TX8+g7?a_z@qq4B)hPmX@w$V@Syd7Bi=-=8S|p-G zVfAFP2dbErzwO%$8r#l=`okPGSL;BhSLldQ7q;>b{BmQ?&%~-6nZr!Ht%mDKy$Baa zdD+BdDV(=OXWy#Yszm7%?p!gte+WwX4LpCEh(KqcQUPbLsH{sBG+>#+L~Oy&M11Fo zNxe@U%=|!~JgY+Co|PlTaL@_Pe%c~9keo$FlLLszjnselZgB$vX`w7xa?KrWvGEzv z&s@x`0|Ioel)u-i86El#8=EwEe1tI^H9~z>M-hL!9z&J_m1cYW;~Ha*n80*7;)D7R z{J&GSNsQ}Pnd+$&BK%MScr(YJUmg)QD{I(}mDj9?Gf*Yv+lLqF6O^+H=M&p?Qc4)! zU}@G&VhB+C_rvVZnGO%*1qXdCs$p~B}v9hCr__n4-VUt~u{uE)Pvq`l{7e&d_G+r|T zG(Ggr-e!b?ITv)h*PQD#2Bn0r1;=hXPBXg4%(?7Cy|$X$arv3I&|jVjB?dOuAot=D zS(6oLCESWgX@wn5>JPnqMF7VLmOT2*uuXX=-2|WdJ-aR1g0|07&Yib-(lg(IoAT!+ ziVx1y+$?yNr$4Rn)rHD4B0}Bm&>8Tp9~-(~E2pvT{_KUFNu`%pTiD}#Bu#)>Vi#+C zk^(cW8z|`zn7<`0GRxTJ_<7Ff?*dZ9dVv5|9Cg1nBT_snW+o_ZB)8sNE^0Y+xcpn0 zsAz*(|ED6)rs5E&T~plYJWLDTx;+P>V>%g*-Ksd3^^1h@0&iP1o#Ezg3)_@AyP$cg zTBaQ!?x-=cIOFGel<|D6hFDPBpWqlPgH)SKo z=+}Wm`jcUbM{~UJhe0bMPgiAv6!v$Dx^A~&*p5zZeK|d-GBPu8y^o2Qev39{n&QKO zv*_=Rq1g0gY_72D1<(z=I3zz*9yLA)`HS$xshH^d@dbl%EjAunF!FnCj2>gJytZIX zY&usRH?zkOsG=(g*Sd%(ABRAZjp;SMuT>Y}K*XH=Fuv-SZk>;h07~O2-b|R|iZnPw z;Ackf5xhho&Uhm8dX?ksJCjzwP+p7K{lSBU^Cv~bi6G~;zhQvUsc65Fe)cAV@^JA9 zuta47K}F($IwrYaSigoW&00P;*YVSdM$@aaQuJ~i31p+739{5X+)qIBCz-#M?C5AM zH^0p4xO^|IT7g!mimh(DlPP#uwZw2@`JC5NGCx;j6d<@5@N%sHV(|DW^!E+`-_dmm z1SI8@pNb=LRW>U(nH5ezE5tIsv*#AKqR&gFMMRno3htdb{GZ4=Vg!^%$`YdG1tX?GX4JwK2sz z)d>11GD@FuI8oG@vSVP9I$Vq5PMNJhfmJY$7#Lxyg0w%$JA)06H!$8|mA3#y@ox>o z%a>AT3VbV}bLbdlIS_U#>rf!3B>A|U_yuO*T{AvEMwE7Av$Vef^eB?F^9^@7FCmpH zwGHfZ_LmKBL7}3Vj`=;SeZ6oNy98kXQvq7tY@f{*EiWXMQiaiAu6kgVc={#oioxuWjdBiY0t^dwNMTp{M~=@kO6TcllB!&%LRh0_k!yiqoMv(al1? zQKgBLl(^!vsU)<)84Kmjk6ZW$toZnuppr9F3*Hc7M2Eo|G;+xBgI`^8?Y?ULdI=-`!w^ zGlZx(?jM6T2!#ImS&~tQzs3+>ZzjaTP zsO7PHL`fCBgFBYinpN!tIu7Z{$F3Bb3l0#{AuK!A_s2& zAJ%L1_tg~%TqRu#JdNnju|!P+Qrz4V6*awUkl35FSTa6Y976 zcAUlH!k@`qoGde_JmyCaB1Al2jxkDR!7?+q zH<@$P6*;nryX}ylIunCnbohu{&`bfwWQax0Ta0jG$6yJ1#fHDLZ^;3_DPJza=Ma%U z8M0MLtpe-PM1LIk0DRH?V#Iq6_$?qiJY`&)w)rnRIEeeKZ&HtBcsL)+sI+1%nf+5z zr&2vl^wDXy8t5|}b@*2APU?*w|Jd2?YMO>7Ga;{BZGTl5#$vFBrq!EVdRM^ih$Sy* z@UsO=*FoqOTdIx|6M(0Mkjn1OiTYf4qpMCvx%oqPe(6lds$laykW{<1CnM0@BaYC~ zI2k&^X>ETv&LW!$z8Te^(0^*@acXzpsYtJ zN|F>_@JjV&W7K50W+ug5Hj*Q3!b}I(q(NUajm|km+`FjffqM8naryC%3WQc|=beg) z=hH&Ti-5>a6Cf3ln))-JsB0A2SVJry$$k!$K)`6L7k~78`b~O2V4DTX-}=JgcU~bI ziS-XfH?8B1j@~7+BjUGy-t-ngI-}A;8I*~$3|HRUCla8@|5|O@ z2aGL}VeFLcPR_e0LYK7&VPj3B);4n~I56E#XTp!JX`80i9~t3=Pm?izSM<_Fx`yJt zLG2mQ^#s$ z&`Sa)O%;aUf0VAA`m`{N#?h=WqvBzeL*^H5QR%{aP`kmC6oZ#a1$o1OX9qd61Gddq z)Dg*&DPXO*->lC$Q~rnvaG7i}^Ib{qw@>wgX=@rYGXTXU5rhn)MHPAgWdsONwTOrp z0#8*f5T!3OFi^tYt*&7~bCT^4_T1@m!l^01j)M`D!LRXnXXFpa40q?XRMC zXFfi;NHS^CKqRA=SJgSy;GPN5d(%GXhm%! zk8>)CU=*XmmV81);g$t2vKZJb>ua33cC;_Oc_Hmc_@bW3B%UvS*f->@srOuz0(gwmJV`quE;=mD`gMiQFcR%!A8!L%rMx3~<*uN1pNxXGT;+FLCD?>V844S z6fsz6alW6bZ2n&iav4*y-*iciJrA2&8-JukAD>kE)`&14e8IA3#KMF&IOfo1=TaRf zYWe~R(X$|cL+ zm?<0+(b6{Bbd}-Gi1RbeO%8u)(xLUToQ6;gmsxfnCG~H@}0@Pq|})J4pLM(TAYm($2=EsFib$I^0RJ)A}ZO6 z^S8oF{vXcXJRa({4Ifr_T9mYi5GuREovk9Hl6~J2V%&DaP}T-RsVMut?_(FT7lyQu zCCk`nLS-KfS!O0PmiPMV?s@L#_dK89=Y8M)vSj8v*Lt4Ec^t>dAO4z69k04&G^ew> z2;zf#GLrq(L?7b?D5*sYtNV|fwxWLa(1jkqGuPQPX3fOul^;0iES+{n7pn=)AF(D* z^Xa7VMk;0DmJ(q`UttNmwB5&{t{C;-Q=0tJz7D(G@i@+DdBTuJZ!`Dkaz^}>#%%h) zXG7snzj~)$N}`JOXPJe4=FR6|j?GRR{(f6edIbi?ndvhjNb>s|PIT7>oz14pZ9NaxA0rQ?$K*I|X{H||j<&Qq@qImA?X@-H~t zXLy*QjjvbVEvoOvcvtFNovW6Q$X(#b;kS@2^Fe6`1WWk#`9_XaD+SHyFr`k)&+waL z0r`*?+sXos@ztMb*;$s!s#)!fwZlgSlV%I-8)rW{#2(iW3fwsAS7}0OiFe6cv(cX3 z_$zGLi6ezy$?1KP=BalZ{m+DdVytXpuD4i@#`ky8&OjMZmS#vPNS*@E=a_fCVS)46vyzzd{C5}grK zcZ`C8!FLav_}x2}CqjsZb^yG)Ial6kxQ`r>OXaMspdn(MzX9{o*nS)3a=z^iF2dYU zb4gTgK@~;9x=LgR0sj@yNlU7?xyJ;|F3Bi^bhUKes}}R%9&w>;AGqM}2WTh$y5RY^ zH=ks52^jK@Z}a5uxc#_Vmw$DKJw@cg0YT*OalSa5LEq*I2LK*F)gJh>z|!Xh2$mH( zJbk2=Vbx8nZ@zC3+G?9r7c_lFz1m?8+U9G4t(l*#=2Ry+=2SCyWf$f^PNOM+MVX<_ zNpt>p%ujL>7I7O|E!@gKKbP@F4{DH@k0g!%48ym;Sogxa+EHiXemnHiT6HiT-J?R;PY;~VbUJd%c zq-H-ewbgzlnK4Iw-}cxo*4fjSzHkprSV+%&T}^eVOQ4{9hx3O$)_IWl3O=t-$yid` z{R8-1vSH`dvYyqi=r|vt4n0mUya{rVhM$c4!vd^xr1vNN-a0Km6HxoVaDe=zC#ukX ztM*ZERPjvutknXan0@Rz-a&&s+*g-%q@t+eyjAK-cLh6niEE{;^dPkf17phBOYgjaD& z_vpDw!7ihFgO75XkTk?pAzSFXf4P6P8w%g3KvJ~ajwl;5+vpnW9;zVJ+E5ny zsBdkH=aLm(EQcR1Lzy>A*v6IrM0I)k9BAq)m|P*O8a(7~@~6JH#Wxx@`xjaSd^uTn z-(u#CB_@kkmw!&R=X&c;1#qs$6>OC>9pgO2h-OuD-{ZtMvToYt9Fq*(ZMpq_w0eW* z7s#@~#jx&kZ&P+e3%ss!49q`?7mSLJ-20MgeZkGe)F?|s$*x$287aYMwL0A)8?gCv z-rtdzah7%TcC?r*Gvh^LHgH7-$L&q*u{~T?!Ov~tRqX#B0CakAUYP%b6Go9@VHA%& z6b-xMh(;a--_fXmbv`<1X5f@X8!Y_pwnK3v9fto5LeE@)S4sq|gi$Ej6EqvObffEiKUuioB3{+*0cFeq3EaQcGj9@%<8b;Q zExI&vUY&mB@h7y7Tb$~WCB{QYmYM89rj9!2>hbb|%R9&am0yI^U)Fh)zS^sliN)z9 z86y4CLeWKV@MCsHQiJPEc)6tPwO09_B}?-sMH2ZboY`n2-iF~l4A)8XFbnnHJLAFI z9*3Ubd`eU=PmGqI5ZUqv0;O%9T8FHK4fJa79ayMyJ|gn0CF+xzRBrmzunVDKtWKtO zbMJ$#rViQ7WLa@O7!6NrcN`wu_@YMWxE+7yqs*v0pJhW)f)Al}j_SOy6?? zFQ|#P^veFS<6MDd%U~g4zNB~No^^3CWj$@esc}J;SzRzP&zlLX6BD7%kRy+G1FgXk zU_+ODTD;P(KcMsRszLEZCKnzMGwo(Sz(PCj`Xcb)M?x;}aRKKoHBF;F&>8TkH zzCBmB!cW%6-sL2gGT|ES7D%g5daNE8rmaNI@qP9>_)4+1C$m1bZB~iyG+odgmMmQp zHCU+2n-1ARiM5USgSp$UW7+MkT8FoaiVSllOb;a|N_Jx5$Aq=C} z(b6&{u(mpL>xRe|K`=VVqr8OGX{c+TaOO`?CY7$v@6;CLkcN1G-+nc4Bvi@&3RaZz z5(;5*h=Bp2Ix?y83yz5V8ph)yFn{_eua_4p9=B9S{h`^ZNs#pT^vwNU!-@+03i%AS zk(`FAL|)j^l-lViMY?8Ph~4Q$ub<%Z|I!$DT;rtp&*w;Iamvws^9`T*sKN!v$2X`zkFO$*kZ$0zb~cCROt`rC*m=WQiKnkP<_r`ALT zB{G$ZTxW+uZz)|sN+FaUT@PD6&Zlx~PKhq;SX>429^`5f5Bcj}G}B$;P7YGEW9a#6 z4gW6q|I)5dntIwsa@_I1eHy{+F0WPu87eI6-{!vLj zM+PuNii#GB32CCOp+tHC%YoJft-KS$ED)3DshsK{OM8xGD~5M{MuSBgx2UFtZI~)S z8cSRnky0m}<0}FgFd*8Uw>XT@RwZ26AufyRYVr0{dsGkEKerDOAM^{3Sa0&QeBt3g=VZX52kP(GO}!%yrD(M1WJB zRM~ijgy%fWS3$pqzYMAC;$xIgNYb#=4Bsmngsod|-mwOdSgWA&zZ{Ldubwq*A$|c(2q*UBTcV zQHJVou}UI!$PCtl;`qs-x{@eR**O7@kin4Qxfs31d0+mU1fy6zj`gDgDM!!OSjZe2 zDgAsO9;yi0n4<^%wQd6q3W7{Zgc9T!?CV?%c;0#ygbftTb`5i?! zgDzGd#T`E`_aAhZ`E!kefvTCSZgnW-E_U?8V>EZa@%~6iRl zfj@;l$<{W}_i(b4_1OnQ1#E>seBsi#RCJMv|LVJQY?FGS${^noe#Q2fGC&u$-Ot() z5)>{Pb{NbjdyBSM;dVLRdWAW^=^DT;I_R;sOLyHn z1iXh$9R!xeMOU4D$tM%QChXU3=k~b^24*lI_1oh3azMSpdY@8R_nsWI*Tnp_?Joxu zlfykrPB{ply1M+NI|#tfXd{f6b+3E~Q4J*$M0}-YWo{RplXsj;Sr5btl_SRgQH8mm zcYfr3O%Chh&TmUHl|^V{UP*db_Li4eC}~YJtTSizJ>__9`>Xp?CzOiS6g{B0uyi|r z4N$ESLL@@oDX1L9V2ktVcdqN2@)PnMIY}pUb+sjp)clDimwJ2cky78YqFS)k8Z6e- zu5vNp#X1${_g7CXNC@~AT}4H#CVbkS=1bg>Z$2g#u&`t+jq6iV?8{a17F*zTSqZ~- znlso+1-wa5qzZ5ONzlhfmMbsA;YOANH@h|}O+z z^yPXM^t@7sQJ|CUl_E=HAzu1u`w2|x?LXu3Al2X`M}Xk^)a%UG9_&g^2bL@i^U#+7 z?dF?0UDoNtOU(^8?I%Z<4g(&}er-Z3>3~vCt%dJ$Ot=wkDZAPQJrMcuP4rVA0TpfG z>lWJhQhssVK)l@-U5wt+5IV%+d z{e|z^u;B}orvH~e8V(Rr>&M6-0Tc>ciL}kb$*96?%w?Aj9z1F~TKZ ztmaavzG4**hWawCjn`U>y(2oGdGtyul+W~aVlon>5Py-ZqCO_PtkCw8)H@PtcNi}%gg z9qq2Uv1jnWb0~*cH(_Q4PZPfxhd>)cU!K3Txspq{L@_=hTiaYzK}@`@sDjVhs4XgH zO$m2UKs$*#Jeyxv>{O;8Pw%uf?Tz(~|1_M~>nAvi`*iZC*p|XsQ#OZ-U}UXh0|{5! zT}D;SkMK|G#adomi1?#B)okM2o(T0{I@8F+uwKdQp26tYGvd@?N1U|05~Xh47$Yi_Spj^PczM z3=Dm>S8Tpn)M+kf2^QewvqpQD(x@5>VVx?Cft@1i42{Tp4OQzp9E@*FkY>^I!_Ql4 zcx(bv_q~<*G!ikzB8p^J#GlvfTzo&%D`aOTySb7`736Mkw&w4+!|DH~F>WX7>QtKS z;zYH}TBkLflH>Q|R@CubqQT@QI1@-COLh9Xy;Ws(6H95n?t>d&^F&C#EvU${v8F3m zd!|oG5RfOM)Ovt?kUD)5*4U)E_xv!l}f1yAOS7H~ef^aaTy z-wh9WQ}a%0X6>|FCi@|)K1o%J>r%7}O|`}5+R^HqvNKnzlR5=syYs$gNvD~a)e-G% zYz^kJ|9P^6?Xwz^kPFMnlSKG79u|Lfc4HG@=VMS8J)wcYLn9^wk90FMvA*Og2v;Y3 zDwO6d=AZgV_Ml>Ep%=VnW486FL~KaToMtYz9ZH1%Utg$5=?elA9T&EEC}$imnG z*pEkF{1QtZ{cor!V@yv&(G3Ei{d)y}u_({7+nDrLMJf1ru8QYd(3avhx8?`vcoZL{ zcCo!SK(B`{B0KuV`nS)EE?dLK9hZW|n=YFcc#TBKzp;dGKKm5l93OEPPbjG1u{XTk zZ0(~h*Hf;pRd}xte{bh-({NQ2sm`pV?BA|CqN*%sEjI#qjK}*1=Ph z!kvICDHI*P1f6YM1|>wHqPCQ_^R_%qkKlMNMXz4_+Gn0jZxxg@F7vtl3)Cl&t=IVp z?#NxTm8q&QS@4hU@cmGkfLN?T@_v zqCBWmn=G2PK3z?FPPY`}Qwxq(xxDdF@5<^&J`!k@7Fe|m>Na|o13`Eb;I+FPIC)fY z^T#R<|G$C9U&sBwr{OZ6iFUn2(}P&j>ZmRhRF0MFE;$Es)K*wz`RbJyut_=fPA?l{ z0#o;WNODQGq^%CPL}5|4SGV)b1<%;0r4EmMVe~QU?f2uq;fc;{a=*TMHQrTb;f{pF ztD!paROc{w(}1j4iZQM2)DGjJHx5f&T_g$mHP@EyO_Dc|8dOc9m+cp?&RaEl`y+!5 zdM`B+`kwj_V##iT!Lc;bDNKON$wX>>0n$9yHW(D)l3CEjM5GB)+U~NoyvN!1(e4+8qL;CAGt-8KM2t2(kzxi zChaApep`A}TuAVmqFg`@oVWsS_$yexd$J|cz6~s)62_c=im9KuNBcr)_csbqX_e1 zv2bj&_9S$yeh*&ZL=V!iT9Qs~X98bh8|DL?5fiam=((O8Gf+@DIH#JF^Iu13LrG3v z!^V+6|B+l*!t zIuf%pdB2@|S5Lv^)}(LAS)dxl1I2HnT@o#KEl(T>HZTV)C`|LCoSHvsB|yR8GqymD zZVieA@BfwjF&|*D$au_bZbCrE>8b^dEy$MgGd5(Uqn;yO&D^vJb@*6jDHrnW)4F`r zW9IZg&9v*U4kuoC+3Y`crg16)quFJ4U~o-?@3M*n$y|KTb$SjpXk5W~`Esb=WD+H= zbxO0X#7)nMi(1aVg>_i1aB*I!U`FO;4{Zn<_fg0xlnDpT72KV(S(j$>V z0nBEwSi|K@**EESVQZLD{iw6Nzb0TRC{cOuOCr0)m|p0Oty`0&VKsns=frx|;s5sF z|3T)u*7PSe_1PP*u+Q^Mmucq}VwGA+i*4K81usrVH*u*LK6EOPM{>zT+;Pg{YU7R$ zci%h8aSi@uvq&&St)jbS99%S&kJj93CkA>eQ=Bd9U%N%rnhR~>{CW*wAlzjHi}U8A zJTddSc*UWf&5gAfxd<)996z)ctxR?=uw)vvw|*ob{qJZNJt1ODWTgO`mYD$&Nm(bv4c zkFFLzur&h&AR;DIij`D~;9>+BueXT1f+0sHXrpP~^x!^9{Y;52AbHScMXHuhsLA&kZ&!hl}O7vtAnYj7t9Vub@xqSF814fT9Ij%aZl; zE&_^H3fF&eKv+_FO0g7N>g>yM3Ox29L8Hk5ahA+cyo6EHLf^w zx$dr?hv|M$ZSpjlw-FVY8p^>uPi z%`WNtk`pfPRwWV>kW=-ZVm$Qq=gL$W=l~1RqvqujcB|QRh;s$27}S-P%vK`+xIsd> zhl4y+m=(~V$B8#7FpAkOr=13+2+u|}xw`UmB_j|M8Lf>xW5`zDi=|FD^QnZt1dxC2 z7*y~f8j?B9g+P&va-3v7ZOQj~o5v4%TOsnyRh%z%DPx+i00|IlB={AEq6cWWvYM{^ z&fjIOZAYs!zXRAvDVBZ=O| z%5;y16o;|kl>9ZDorWx49mz=FL|U!Nh2lW!>2VsjX6>^zjC1zfJ?!RAUYt%9Aa#sI z$}e4!UwMC$6mO`b4jajZm{yV3R@LaA=U@(??DcaORD{y+-OTN4 zxMsk;30mXotmd>_^2$X+?L(E5l_>xdED}??nWvaYqZwAxuSyP9z}Q+(cr)6y6qS9eo=Yt>!h>wfPNa7qEZEEjg6$1O}l$(O$Kt{73dlz16|^c zFMbhvJfPrvH_88-5o|JdV*pv(?q^-jQ|)67WUdNU6Q4Q;Ep>Ez8>A267BeJr(C-i^ z#qMsIiUg0NecLr78~TB-*QUtsJvr66PTsD7_t&^hTZGaxO%*os_c5uByQT3x%Ip@l zjikCXh?Dj*&9y<~l?4xZzW-f$(`qNbZVq{9y3>8g*aVQ!hK5Yw8F?O`WJdOl%*Ax( zN|$3P_%Ae#hOfa~v0ET4z9v7H?h(7`XzEBn7;pgk=QL3%PzeYnkQnb6Yl4Ya?siTC zb6JJpP);X+NgPI@vJn`;yY#k1WdqYtu$!I-6)Mw8Xl*k|a2^Y`Aq_ zm?$0)tUx&u{vBNX$k-?#l`sv>Yytc(=W2lL>cqg{0OO^NrZ{9e&1D4!fQfMjK;VcJ z36LMDNm<@<>En~i>fKKY~TM^CFS>-5RXty?ky=zSsMK&s^8n?G1;(%hP_ zrs(J{KRJjr*Y*tzwJfTjdOaC7iR-KaD$Z#?6+j#6xHsS@yDlCA&I^bt>YJnlYQx3b zHCYA8xIvdrRTrRY5#L=+pczqZ1YUVr9JixHFp8r+B^qtx39*& za^x5o|3=~6s;#t9Az~`I%8_@d9c=6rEv14*gkvMZKu!)=qBuc2qi>faDjJ3Bqo?m4 zp{3ve6T{KHq{UJT9F1Vsc$E?$A@HdGh#lG8LZ+W8i~EHuC%iiNaLoLkHQ zV6QFh902SnA83F4miX-w-aIN@yF#Ze#8V5z$Gs!YM;zm*U0-pJ+~duY&fBN|@vZ@b z$)n^utZ%k9dIMd3KHpabw;z+eh~8QxLCgba+AH!OWvpT~p&+`I|yYUih zw&5{pHmJ(+hCM72UITlwH`CxE0zRX(r|S-@i_sUu)-Qmt(5VT$P?#dwi&?-ikrG-H zK??@_Og6G{N|v&Ic(ximp|$MIt8KO$NWOuW2TKey{3?r%^up*a)>^+QOk_h2cICd6 z4AhA*Vjdu5qL1=id)4Q4s|Xop*_)0s+c#8LuB=RJT1p|VRibZE4^qYJ&u_ipp>l$n z1Mc^b{O6Hj^E3BAuF6r;5s~Q0#R!|YH?ZrfeLv>f^QNvSHk@VJ76#O!C;g5r92$W7 z=?4wSUXeF!&Tj$WuX5^8_#wuhtUoWFm^XwF^&7AmIJtzqj=dvfuZO5+%$MTQsPi{q z392Nhh)gJggL84;{n`qZ0als#yAQFBa53vE2gwrX3USVdfYa5(5n*sf24R#V{iHh^ zP|b-qFfyV0MxNr80e-J_ruP0tqPb05xJWt+AOJpt=ti#kkG0Vlpa>$BHk(bS7I~Op znnM9_(o%_@NU>3l5QD@fKzWKcnR!m)!|N7Q9RUQMOo*<-MXtyl8Hzl=^_}@Yh(>@( z+|y1gEB#)sF2rx>kWZ+vsnv}wTMoVVIyqzi_3zSNxzHAZh3MqW~>e(?>SG220d7fSYiZBaK@mNrNHI9EXO- znFSKe1wX7W4Akpx&8Xmtc)T+D$T!IrhfiyW&HEcG4s8=bR5nyeG&gJ>!cB~i>)~Lj zW;$L%Q-*U5jXa{<+Q_12GtG_7wfe-hHdSK1W1QufQ!GusQY--UOp(HZxfk;urM zy?^Mja8fs40s3rMtkB0oAI^EZ5mj!lc!=Um>6YrPVWL?J*9r{l8B9Q2$?2JSrhB3+ zh%C^8ngH(bS9#3RwN?|?YlDixNXW$7dVls^Eb|75DE^R8dUR7fe9QQwmE7Ws>e%K| zuU^F+|0PLp2PJialV}@M9rgyaWdbAn-Dv+|_n~&s*waBM!!wrq{`=z{hgU7ob9t8YE&FI%N*kS?f`krfR-FZvvKHoNiNf zQpGIQ_4<$bJ69B~91O%h7BUQ+)V=nXg73PT3U?C)@2s9}xW^XC9IcNCr0`xI_lzTf zIKms$JKb1Lbk6I&G`ul0R_#ClS91KtuE%>(8ikFV8WJs=RnVcO6O=B=2{KeRKQLx- z%ekN-rpI}5Al7gO^i6g)Xh+TW1jnv!ftzF$fUTL|3m;YYfL+eV*8#?y`Mo4%~%-D;7jKa*#wad$>I2CkFH75XPa2yWlG3j$9y)ZP5 zA8kbaGh?|#)53#nDU(7uDWr38*)+v>+;6u$)TdClD=mE(%*XxnNzc22f?5bN@cv1+ zF6U^hx47)82x88AR~;C*dkYG@DD$2fyo$^eLWV$3)Y6UGvspn}oMM=>F?ugm`CzPh z9<4$E?J0se%fwp(UJSpU|He#`&}=ZpM6P5xgvZy>FB|K#JkcSXf2~aeY+27U9QRU* z8apG~CGCw%4Hr;RvBW!jbaOned(qw=mf+7S&H2Tg>~Rz>mMGRScDTuR3JQ4gV7A_O zy~pf2c0Pa(-?8iP%iNYS4UIYc14@rJKp$F#g)BVWOGwGI)i0;FL1#Y0cz0rGkB-=K ztV7CPqL$t9KlUGa>0JKWo?gUHHs;VmeKXqLNTW8XW175Fz~%t5ivxBS7n%h+!bhTX zJjL#f=+`C@?A$YOGCH$OG4&8Ms&RLb>$K8(V82NYqjpd%f6ETG5=kFPXeu zam^E)2JPPo^=<}9EU=$e?vc(ZM4f@d4f;8qNh;4f89D~+$H(+=m4iEg7TaPwonFvr zI4;V&dwF^{c=G^$xBGZPu?h3eW*NEZzUOT)enk_uxCQ@_gMFG3Sm{ei>ld@<7_~cu z!POBaAX#Tb#8)_Ao;2Epl$dfnyy$1TaYJ1cmuAOYy;Ry=> zXmaqb(;s6x9QiqvPYPiFv_50n8v8<`nThB`3_rgozbA4wZPG5u;sm$n zXftK~b6A{@D5-nRX473|$&0;&af9{qeZ%2{j0judj=_6#m!PwiSzw01p@E0n^*>|b zcp$FG^lcF>iVCs^*RI6#BvjsUrk>Gcz!sAePYDp*`AN1^rpBaw=qFZIsZ-3QoY1_ERwAC4~paMq+&W$9*Qz zghgZf!tPh#gcdJ0=XjuDV$d?|bxwEtGgi=z9t;WZpSAzKbr|pwVxPEN0Bq_#{whmg z4@ob(v4}O?1ka+S@QLk(bBgS(J$ZpU)Ii<6)a3wbBWa{%+v~&wVtLVbrW4Mcr?e~b z7z*-=%hm-d)s9`C@1pHFa3;DbPkvFev{_B`htGUYh~RSbyzBhN{94w|rnl0sCQkhWp%h>itOwP-D*%GTn55K^G7<2 zbA#e|_;Ll#??rs(YTg(3iQ=)lKU%(h#=oZ4V-ADLJx$mHYjw4Zi>umsZ`_3W2 zQQ)-l?Zfk&uPS21)a-@<=^$F`vx2ucVU`*7fv|B99;lnH^;s|rkohu}In{+0%u0L* zJ)&t%^#f9mS1}$d;Zi*>LFOYwFn%}gyjtgWaJS~}8CCYH z7M4ApvNn9$?;g=DPNdm4k(J4RM3k{phPF_!5;6+G_iY8-aLILn@A8uyJ(Mrl^$y(t z$qh-P=`bIMTN00kC2RtJzT?Fg!pI`@HwR!=liQT~INHX%sv;t5d%Z9&Tje6HYNNIS zEtYchooXvWJ0j-jpTfzjpWFhZ4&C(Fyc^6)ihk%W+OAtHXxiz8vJ0wo4{z%~lm08<4qSrw^0% zU5X0av~kDw)!bqFJeLpV)=Bz9enIVEt*P)?iO1?_GC0( zqBq-~FGeRsL3c8}F=2fKTUhe1t??o_wr=EVg%7G2<|WoAY(MiCE>B$@O@s+ZsS9S* z^(ukePr0HLxYC{a4L&oBinusUIL6&>PT$vaN*c&(eHwkkCl;!7)2tpOes&Dw!T8jt z32nZ$O!avTkgIzCC&S;Xdtqfq1cXv9KtYMyP;LLSa~$g8_bQj0>4L3iLjs(NMDAGk zvv;_m7tPiGl!z$!@DrER%$jaJJ&%d1n~Ib#-z;9_8_-tCTznkBbtQJqMj6{1#U<6t zGEbXkUKQypaDKXX9q^+sNFYH7D^?jC8*Q4$p6(o28$nc>ts_nu#sW3)lp8%HeodFk?@MZXcu1bqFh4DAl$c0Ve64{2n@J z{AGwet zkuDk6Odnsy@A5X*dKI&pc;D>YeDS5D9Y(244Jyny6@jCHp6|`rWspW_@Ew!Zwq0DC za*#>qZTW}}xIWU(xDmRq7IK3-YwWDo1%1n@!6eJZj3|cLFbjlCka8B`Kt1yYQqSNP ze**Ge=m$oz-Fh10Qhb%(n@Y6|CWq5)i653rmGyRT3Jn0p6=%I8;odVJkk7u(o?n!c zN1JB?t<0P4{bH8=9KfRUU-=Rvs{To(9EPfn_g_#`R*`8vRKQcD&L+Kvp>J!K?N9UQkPU zJ!XOjr^{%Kv69P&HdMaE<&9=mC*rZmTkg0bj;)f{^8O_qRoM0~2k z0Tf12&G!%*?AjGVQku&yxQdLe?BwS4u4lMSss%?%V(RTaGL{7#NF5UMYdFg0tF!k4 z?w$e%%E+x&UPg{nHTpl44K*9iMkbD&&*<_^YJYTqCpp{-H<#?m&uAq>Q{U6xN7Hb( z>))nj9^cg`l+!t7iW$ryfdZ3@{Mz{%?Zg^vPzDrWOsHcv-%kTlv0jn?ye}tU96!eC znN>QeA=nw~hIAlm@>Gd<0t@3`5M~wBhcMZ?bK_C(XqBVl%nj6Yv9Rr70NQ5FZ-pSz ze(}?rB_(XS&O4dOYoD7>1nyfGI1H70$L2%FAtl>Wwm zbtT>z5v(?k2=`E7Y20X25zI5|-Y&ujRLGy7Yb$NtC3E^|4@wnGY~=Gwi||&G&4A(+ zuQY-g^{a?$94T6E&%;#x9GNv6=%XN5wuH6#QfzSdAheKrt=74}-;?`8Q|(E!L!cyr zq2Ks>?^vAJ$&hK5{$L&h-}JsM%}XOpZhJ$#8LOQ&#T^^P?UkswhbkAD?mDyLiVHK% zEUW%dIK0`_vZ?t&-zrs9LYi!Da{ZByHhis_igojA@!O$KP9i){U-!#~pSy5SK;&mG zLz+|qB4MvT(|Ho4tM%jSx+*u6=YHu_*pCrZHMfK4zIf@1=HY>e~y$#wb@kiv{fIHax^Xdu5d z1vF(-gFXz0gFB$eN&)M%gwhfR=SBvicBJ!u+)-$}@ zmIu-JE2-DiNhNZm@VvsJ{S2dH^Ogp?*%ZU|q+5p4BGEgS1J^EhtRo~1@7~)195nsa znIid0>VzKw0*@OP%Nt|aohLy+^0Ya5fjv33nFLhEhNzCCtb=hmc@l3;{g?eFH46m3 zBOLdQ0*0Win;(HtN39Q}6$WZpB98P;e4HPWsJ2U8WXCd12)`__tJQE4Irw7xg?HDeGN<+;C^m1IWBm(z-&W{bpSJe7yNC2`;VluAS<@kkDkJY-P{111E&vT zRf9lMP|2J3-ry%4v-iPs`%!i`AtpF^KKQN;Vv_stWSmImq&}k9w z+KQ!qP}D*XUS^Gt6bZ{i+lsb;YC%78F|yR^o=e~etb`^1IAHkGA#;FAr}+bXC59GJ z)?K-h^9K}|US7aLhFQR#uHmAKN3?X<}PO72aIFnHh_T}I7Vhn=q1-W4SmxP2Ao72-vbHVz2`@c1JE~j9b$0H z7XYa|j-wU00z|4%gMJAx7#jGa6e!8)X+B={0MiPI&p?8*gH?DV;k5qo^;~9}|H2}1 zKI}7l^AE;FK!rl7JF|`s9)4wa)B>s(M4V;(u+&Ut z>9ZQk4Q2ZjDE1l?JRIRF2MSVMS#pXcQ(UgE&MkNZDAMZ#>Gih!d*Ae^?2isoX9BK2 z4;@T=Lt@L-nbeFMWQ~=`#qc{9j-7c-iCej7SgZzPwKX(l6%JQ!H!+RBeG$P@8`d2L z$`LJkut8nx{wwhYadGv;c~m2})!uM))y31i<|nmnZyFP&7@o7Cf!^Rd=fGzm<)e=IDM zER$NG1|#0zEFkG*{+4f9jIPdY!i4Q!I-^Q?Oa`kWK1H@0fuxF49jT)AA1qhh)1-|PvP+E1RP6h_93cQfQ_}tq1 z5u&KoeU3x1`8eyhepTLD1?uO4qgSV+t;S>&zCcX3)j~Gbrh&#->Ajt*ZRv^5gOMs% z^ZOl(dIKocPan&1KWgqn+%ov6=G*Hx5jUvcfA;uw1-(DjM;bus0PDErph!=FWc4!w z_vcd$q}3$e-B@_Pf8$AzDR8A>Mw~jd>ZE8PC8U+CC(IjRD=hM{Fxw8QH3;)615>!8 za{fPm9K9TW3ryiSUl^wb@R_-aX_m8o|LmsIL2Ze6kwXQ z6=84%B1>&=zL$#XfruXm<>H=SVC7Waoiof)yv4g9V*3oJ4@~4sTLz zM%TsL+20`bmWuDz_~e#FX>)EbAUK$;N;*DNxv_^oqgr`FJ%=G{f>{>ucL1ftlNm=o zD_fMmRTCKVutv{H2M|Mq%N{1Lp5VGW5FnW`*P8|Zcu|i7xc!hyr`XFtjc{)kB5jmn z4Ez(6gCZL09aGI2b;E$(KGb&|)MGo1Ak!G-y;C=Pk8v8`{$^rfjzgna-%5fCFgx(dtzunRmW12-vtFedCFagmbQ-Kr zT&QVTHOT8N1up<9S!gXOo{N@Euk;iR8%uOTKd%k8gr@s&AF5;qE}sANv^eZ7YttTC z3o!M5NWf4p$;2OFP5U4b5EHH;!g&^9WF$^i*2jcNZ)NQj{052yIx6Gyn(57h1uaL2 zBbA(Eo}eOdyNq|M;m-2y*ykqGnln8&;Q=z}(C(Tj|7kuXtqmmXIs{IKE2kzVw@H(Nn8cndQ%C#&=qyjFn8wYsM@yoB zDNmf1>G`$kK|@{b;bItUYeAO)pc(nbZ=n}XtO2lQyF-q5tCO<1)>{wYV8OMKeA-M= zxCqj4+T;Z(Vzr=n(Hc=XBxU~M797aO{n%J%Eo#Q(>$sYx5CO5~0_ zOM@=42})gk?IqUq*Ll>FpbGFJhdK5dsCh_vw=%MiK4rH|Zc^;j4m>}lWj_EppoHTf ztDCrDx0^*1qT1NptM0a|mSWmYR6wX})K1gfWexh`1|Np20l^ak!$XZ@ z1W+USm*+QR(v+>=C!EJ_`QRK!#Ofm!|Fe#gO^X)G8r}I20fYyLPo~FD_x*H){RA3_ zw>`i%#Wu8s+BB!o3S?NzAVnW;^jUtpJiGfrA1?jz-wT41IYOnsC?7(sXWt$GtdZ=E zA3&l!zP-Y`(+?>SN4-WY96*zs+2+x+^BH;-SW4=+dzV;;V`&unyNx<u9u z=*s?*RG1trHkAKB&)j{(?8>usdl(2C^`{d4t=qG1+k%dF)B>qKfeyRO2q-p{ z9Kye2QFH;DLtCZy`ZZslf%nf(LYyS;?TtBtr60*IYwNp%)Z|nC1%*31^uQfA$VrA& z-ndl7A!ToMe)kcskN^6-|0FluaIhOHF_s0ED!aT%r)3zqSXZD;`D5QbUyyxdoz}-& z+CQ6kCyf{s22D&Xt?#rRl_sZ8=;zb==}|_#K&2eGdh~*}A$tiNo*MZBXphP>P_ro= zj+F@)OcD+E?|$|o3BU$#-Ys1In~P*6e8a?w065$aR;;-sZJ${HG_Pa*0YrH(<ft|L@t4kK$dWu{2_-ED~R1i*G8YS@Nbh`X=_(c%)X9gQ|}=Aavn#LU<6+ zbsZ}=XcUy=yO!@e>=!G|`5iE1YaKFCf*jgy^yu9OVqnOYkN&sY*b#yG&?}FffT<0H zQ;&hVrWHh7IXR_f`1Pq)Ow9yJ8P#Uq*xU~o{bdOzP!FWLr>1|EulCO2m<#}&^T^=J$n zp!pIA6N%w412j*-&#&H~0Sn*Kh7t?<>|eFt{llG?xL?6gLoMW4K%ZMh@sfU${% z9podmH5a(;AZa`SNwbAi)IVPD03cfc@7BC3=>mL1!@9>&6pM+J>kI%>D#VBOOb2zA zaUS$~|NAGnA|Y(M)7vM9DBuzr1>1uJ+{J5v$^-wxwAy9q^J{e;fruX|Ftl%#CEqha z?bP`1mt8-e~eGNmC;su^$re4+XZ5ZqB-_oBr)GWZ7yCkhM|@{SuxmI2`f z3%^RjYN`Kn?TDp!r%2(QC?16ux>+#w4QT_7nGaWfzg5w{n!Gq(*)R4AlfZD6H}$3) zFh%Gv^7IXfR#{&s4t0q)|Fo6|$2kci)&UaS#Y^`Bemv$Kz1+OQOb5b|EUii^(9&K@ z3DpT6RNNX+91DT1=iHJkv}I7|l*!ZAG^-#+}K}Ci6)X7aa0WU|{exl%^HFemy4Z zGHM0ZdL7v6cwiNC!cb#b7v~qhUY^$xCnSWM7shI;FR}yY0)UfhQU=t{rLJ<~jnrInt|$h2WGKqY^25rxq(t)Sw^j^^DQ#{gJ$>#|zW_0Nd#- zHgdp)CIvZ7_dhn&l_-p5LGo${MaB0v6>Wk)S0h^T60x*@M}slwS5;|7&acAEe+@ zifaqkUXVEtGw8AW9=L47>-TcX4vHVU`$PHOqc1nj_ObsNk9({4R!^gpE%dIXVkNs}(1-_F{BA0fH(-0efFfd%dbQ@{v0OtleA%5_yhvhoI*8%w5e!%qG z_P)WrU{caMr5Tk&14323=rtil$Hw*X%NyW31_rTztVP?GfTQWcK6ofXM{dy&*a|r{ zCbEMa`#z^%JzD+1Kn(W3BN-|@XbdCrDE=Tn7H)Q`TH0c82|MZL*LDR zKF(G#6Ghxpn1-(N92WcxjQ;H4zr=#@w<^sDQH8vy0!LEP^bWq|sd0sCUZ zR=pBF@I1eM&j}vY-8vrVml+r!6*I#d9pHBwY5V)9gV%@W`Gr(Oz{dT5+WYRPCbOF0e5g|?au=kTEmO&T->-y`c^Z=M~-iHn`NwxN$|M(s^05kpX$N8 zEY&-JS75|P^$bdLvnAq>eZRHj0AuL&mU^~Z!jiYu&XXa?t;VxdLz%h)>rN{_2&

2G#-hH6=hq8N96A4W2Lb7khxdboYOZ$Dj9x&;>;IC8KjCtT6 z<9=h55g(bwn-Q)W&|mU9#dq8vY=j<=$`fL24bjtNq&;9^Au-9a!4!|5OV#{cYPs~UwhcM5ph)6%_G1t ztR;h(({hvUR2Ys#E3*ctwUt4TC|tmNm^DcFvY_Mj_Y?jY3Qi*9$%&w)>MPBorw?gq zFN<4G-5npL2tz{9kRCs)7-bL52 zIq5>oHR1S1N@x7Lho)NTx4Z&Jj0|tLKnXw zarZQCf*?^3F~d7%Y5TLS>-|xMHkS@+G5k)0&M03E`;>9j@V&M|!_jL7kfc`yxr`n0 zwY#b>$m~1h=%Z>*jlreVc^IQYf<>EXZ&M`>UyzzqsuzY(6e4AVF};Avsn>eLH6A`=9gB02F;M~FfCeu&wM|EqXc;+@7jyCXXn zc4rt4LIvB{p*MduR%i8x2*t@uc2@g6;uDKG0=#Gzf-f0~5Ck@qQ@W#z2=i2Ux=I?uUsc_(f-xGe4q*(Q^+Obts#P<}Ehux~VEXxfqaNsbVJ%B=rJr*g z{61#a{=b(=))j?=l`6|^FCaK2%_H5M4x$%spA-a9tQq1z@H>Nn!ifi6Y0SV2nfe|= zq5Vd8XupV7>C3~=kA#{)NbB*vpfPXQOt5~LDrFj9z6yofPBvK(&)X6XUAS8s+&`CX z@T|7S#sS-%#wFYoF#(Mv0tIk#$5F1tahaDxEgKjj93rm}E)W8(P9d}A(f zJVbnRrH)%z{z6iA{@hIz*iBiHA{hhRyeCmjnM9cU;baY^st$c*&VIf4Ym!(hA#pYc zyZ?nUdT@PYe!8OfPkae7S?Y_RK@(RV7Os{jiO+ZorwtPgP{gS}3+8@CmiYMyk@0mz zUStm5zW{8t39wL7uvm*HM2#D_#^d>XV|Yfv$} zPkVXMPP;d)%_Y$uNnMg|DKIW1aE?FVEJDy8^4A$gosb5CvgbsQFQn|W?!3|?=AFhw~@SvZ5xD- zXtFz8a=crCQ>b^B>G6{Y@Ud&RZn!2TDLJE`j4Ww5g}+Xp<~}Hxn6Q+Bo}hnnH;p3n z5Tv-ZmTT%KNu*-VxmDy0ue0>}dd$3U(>23tDWyb_W^#!@<*>G%ro|0oXZP{LuZqFG zD>)alMsE#MKXHN{5D!_&3Y>-=wQMkdB=)dB?zP@+!@bT3Im*Yn4DkR7jIVJOe&8ZZ zqo?T{c69tIu^#z({A^molO?ak-mwdU3vh8-u517711c@fSMm zqlxbmqsDbw>7UmOROl%Ila&CYe!XXDSi-0d9<8(??5|f#Z@KPoV836to^}sUgA1{AjlCausc^ijRZtBIv+1hJ$Ojz4CCfKYlcD zun?jgmmu3{o8H9U33D*HWT;&!Q-dF{UI@}!&ei8@%ha!y3aF59b=%u;VyU4}aQ$bL z=iD%n-Z?*^iqdr-<=1waX~xEOOT{mxy~GZ+V22%(pzflOBrCVH`9eZnlldOEs)lQ{ zEW4HzGVp*_^KV#9BE6)&kp81ADp>|o&;H;na!6w9_PFnbjlC#s;h~&HfQDp8i|=Fm zm{v8U@%B>c{^)&#<@-To$NiJX3>=Lf5_;>YtNjm7P=q53akFsrlO^^cqUYk(D8zyD zjkjBk-GT&78>Ey3E2N-gH)TQPb@-Ym5#>?odb5u-WmqDa<}Y;BdpsdmbF@1-!t3th zMAt%pwHK*E}x>rPahdBR0b3xPN^2zz1M4xjw6=3~%gRB(Z(q_+O})8R$b7ad{j z6wcT6RP#!Hc!qKXE=VHc(UV!9z>l4-mM$4>pkg!yO`|_;pw{Q#u35#qmeb}wVsUfS z(#6#a__uSawAkzX&M(-KuztftZ&p_=Re!dozMa0ci2DJ`f0Sh2eZXK6^AeRB>)m9P zy-C8S4n6CgceV(O%Q^!&q0(0^V@Km;Y2P%GmFjvntaQG3zdhjt><_)}4e}6{4bBZ{W+~tmZGW}Z zx%-(&-?@&K5c)|K0RKW3&j#f=Llv&SeZW)UPGZTk7k%BO$xhC^n&N^!?JY=ZOyXAn z(2RMG&I!$E!oE4N-p0;l_TF?VtmFbuAWGdk#XH@&u+Q%kU04<3dEYo+N&m4`qh4l( zsF~`d#G#*i2!UTd3=xSJ;qf~@n>Dtzn3^D^tf?mh*%QO#4u6?%jj@^vrciAzcdAZ6 zNTwPO6s-IN6Q}&Ta<#STGoOYDT2bSkXAvZeFMX;LEB;HA7=NLrIsI;}WaP|aH|e0C zBHb;rJ(sZ)wR{+#L*g|J*D|~<8m|A;vqYCiOA0j^Xe|g!TaDaloCph25j8nmGS zToD~$Tg^i;hS?+SY$Wq~BU$z>W%I>pa@6LE^(e-ZVR>~$6>Zx1A$aKcCREZRS8 zbtnDgVM}>0iPH?I#;bk)P0tK(Wyd-6h=qKN-gnCKXo&!E`BFcOZ1kWx1d1D1aMk;Y zx;i|s-HQ^oyy4mWvNCiSrI`sYIF|*N#s6G>lVGVWt ziI(?Ewm-0bS^E#yJ7vG;uz=nu9=40*{q1#pmcb3>)>E=KQNSp78h)Ffr))0AtPMdd}4MTueRvf$PRK~RL-G1`i`83T2{rIX6OxuDcEDH zy=<{V9uqTsHzz;9-OSQq9*wcj!;(O5b0&TePBgr=W%=d%-uS>)UC4$n+ws}WOc<-_ zS4qLJ1c^SI6s(CMPQCDKRd1Z`5Cw6xI?kt+YQ!1Ih=eNG3)@BdPfxd|hGp0%tk_h^ z1qi(oaS6!N@$diBzmz^jEJ%oOF2EK;%gyp`okul^3h{ICv0AOn24A%6uU8rj?L-$RMToIFmqCug$yz83m@rG=$9$iWv{Rb3b3f3 z(9h@|e@50_*M;JPOcXAO7vIUUUMUrGl`ItIS!(DL3GLBx>JJP=c6%(3B=O-6l$;3s zST6RtmlW+Ex5ftDs*sqv@CFhUtwjswglj8s$`>aXz1K%by+hnzb!H9N zk7!KdGI-!T(K+^&bTKaK*zjp+T@l{)WaX@_gp-$EwPZnjGnO{mNXbfG>A31ZwYKf(?liqi;Bm;7s;_Cf|1C>TZN~zL4aa~^y#}mDu*w}ng(FfN6)aRI56Qb z%olo?a#UPU`NP&T+bp3~jMqWU9XV;xc(r%>s$42msCq7z+ z`AR=I){$OGzK_Z3fqjV@&1q>(F5v|psr^hpkY7ykhLcH7d~P3?i!EAPb|UTKY3Pk~ z2eO=726E?$nPIt|KK0TIh<4{9mW+wDW7%MQ$+`V6;>}u~GRUr30e8)=^lR}u8s8eB zYqDmEG_%(!SF|5OV^=$koXh;?Dwx`e7ZzCOx!BUe|I=SP3JvO}-(|PU-TKp(t*VSY zY1h6)6Bw_h*o}1Wzl;3suoOCWt~)PIP;=}_?4`@Eu z*8P75^p`3MMXs#m?XyS=9dux0X&+#^O+=MmZ_gp~>|8YqLR4H%5NGt^bH2+x2Y7v{ zBivu0y7BYPPf76jN8v@@Zi`_3{IeA?85o3^VR4Aq@5m@huT4{Ei=T3|EX2E34!B1vxBcQ<2@=sCpNn{2hAcu;?i?(06N%u8!9&k)$&yycen!#0aHo3KhngJ$-A zb89F9X6k&iEG4ba#IP3ICv7y3T=qI5#jWfrSTU%hklWBU*74^GoJJoJ?2|0?Zc(6( z4i25^?)*CWW;S)uy~gK=TZ;l4ZsH+lwkgNorkrEH>%rP=AFG{xl5$oO7yWeDWtkz` zV`xEfYZE_I3)Er}U-we_4m+3%1RaX4Io%X#skC-aHP*nPxBDp4uHj>4J%uOW z@n|87$I42u=6Er@`-Pn+bKq)@)`<}kH_?=qq6X5Nq4D108U592??6j2*IxHC8AU5K zOWJ}ja|c~%EhiYHZGe}5#vt$7QJ&LGCms|bJs6TB$IH=DeZ!;f-I-{HN+*ghA0?6`N9UE0~KyOjrcQ zg$teK=_Wy;JZ=rhw@~>5=ep2NYLq*nJ>8_-G7qOv4c0LF6q+BQjK3ZptDqsE-t9Wk zGQNbTKdy-UC0EGiit0E%KQpsNS>{Y6XZYyewp`EqUl7aoNju;eui2mqi^cayQ|bpe zte(I_5qeYEgpiI)pif2J+pjF>X;*65u^;iG97E(N4DxobSOTqKH zf(HS-v8mgFw{#l^hZpo1ZDhyGwbG(zKCr7+d$B{0M71>TYm?=&(Ylw1#V7#^F zXJuO&C2_*fq-S308m{+xt|<08i(ZbsM(Ol&D=8^|3xT$>c=TPR*7hI;Ppu3a0qvKM zwzyk;(I>+aiA>O#`f%H6snNqbbA8#CLN0e7JBIeFD4&0aG{vo%6xRWJZMbsDmnPDE zkZG}n@YL=%(+N-Wu+4BgrO7P}eZGbI3u48efPz@0Aih}xXUIkd>z>{dqXQ6mrN5e{Qh@ndnqM2pPdzrCvNk5< zDza$jO{K%J;jej8C)Q(R`Oc6lJ-+VNfH`LDsc^?W3lzPVel`xBA@6W$47c2HNZjy< z)R^nMj#bLyuNeaC15-x4gs2mJV$5?I;4+dcezP-YVqX`x zs*6ej>!3S?aHp}|Pckw3X>HYNd4D4|w!|jUC(jbtE~&?}hnY zdkH063?8tJZ>9a4TQj$IGY)ZqRYV(iDq0?or& zs}}7RL=eSUNll}21*^k4bS>TMp0()L5=vkunIdUq;+s0leo&C|URM`of^2hSr|2(N z;^#mNp>nVsqkP0#MJt_U5!dRG^iM4*C3_-3pQl<96~gCz=Qswqj&>Q}GhgQsgMLll_r;zeD)V3yJa#Etal_6iCujDy$+HKJE>+yEm2X>C=dYgF8Kr^ig!A4S4G_#&d)%B zx=vl`E`twDi+A65H>j37Q*kXQU{mgitAF$;8o5X(Ugsm9xJ`?W57*6uSf+e!^b~6H z;!2We$t85l^y;pZ_A=E#<=PN^EMek9{oav<^RBtWVht_637CS2oG+^(xj#zCQBDNFwgg^5uBab44ECqV4 zS{h`th$F>26EnQ54M`;y)Q(%g`OCyc!ilji8%qFS6zMmE-O@g9l6d2@GmE1*cWDRe zUFHO6b?yO(J1YhJxMFpXwzNBuo)FI6!zAcKE zM8;xu=z9(pFv#3Cm) zut-1WP3_3y1=6DmObTG*b= zHzL2Q%j+&kICWU=QGpZ4c27C9T}H&WGVUEHM;qMqGGKk9Sg#IVU_<#2efihUAi7|P zhaFmMm+EKP#{q<^_ZUP_t+8a3_z(C$sWvO_Seg;7F=%s;BZ_7k%YZ5FfL8^EvWp49q(rPXdXMlqX%k}1nHj~G}t zcPrTF_ZzPoq+}{!>h5z2`>9Y~HTF|RIxH-ZZSj|rh&XYZ9o%L<3)Ra#1p4cBW(!eo z|07i34dbhg$FLBfOfrSGB<^0g@a(zp?Z;imLqW9=tu1+#lfXor-}n%`gK0YnMsw`y zmL|yjWv|ks&2qtG@7=$EjK7-O$The2*e_h$kMm^Nf$o9DY%!eU&uTn2Tdm?%OH&2f zt37&abW~m*@H?iy{dk<~1>=|u)Ba~DHNJ%Qm&dmIr02l!}hs}ZE?UHVZ;t0fSk7USVMDgB5P8uPY8%n z-jg4<(fhE@lXDpV#?~l}m{6kHoPPS0%ufc z8$!CB;%`%u^i1_7F!<`&rG+W$lD*oG(%LVck4Z9s%Gki1lP^JfHFskh)<`Zm*Z2x# zg(i?KJ1)Ek=rlU zJ8w2AjG;FkZl^on)Ejw)O`2!59QnIU6k7F&qwPL?&9Q53vcG=$^oc+dgLGn6LJ8Bx z6Y~r~7nfpEGu1j4%gp?_MNP{q@MDO@?hX*NC4UI)iZ&W7tzT>^L{4Dp#<>~f`1oVp z&}BRlf{C*Rxz4qc1KUsC5yR`;yt1NW0!96yiTih z?D&NNUkH_c8q}R;$yCDM66-+H@#9SF^6^k9LBA->W$D$wMxPuA8FyZ{hJ8x5Siz4+c=h zb@*x`Ho;F1@bn-=b1q}eD%aIQSQ_NspTxx6HKSF7xIIkS$(tqY)eC)Cmyd(;0k$W?G$OHmI(=bFzmFs~V`ypnb602AorMOO zGv8SzbIw{~Hv>IwPMh9`7n`^Y+UsJ}gjDcva}87mGbW{M;ftM*odOarpQK)b*Q<^| zBe@>5`^2=2wfp|oev#NF6BoCqVNEzFF6!~LL4JC6o4qJDvJnx0uv?fo(zhVN#-a7*q7fe5>=HW%VC zM%>*sT%{7S@A7+AmgbVXKN}QNKMxEFrWrM}c+oq%;~rpKS7aU{I)|BcYW*j5;-_;V zoRNw#1(7f<9PZ+{20b!VV)(g;3&-VLJm%nLm4N#)5W6iH1U?EDj~WFR25S43h0ye4 z%7JxubIJ|ZP}js6tX}wIlR(cjT32{DGnv@=J}l=z;l%+<(l!0Sf@b->rCb~t&fzg5 zv3AbX4`@XVgcWw;B(woJ8TFT^Es&+py*SfvH8!nu1jdi>C!db9AC&oyp8sNG0RH}M z_s3Oz;qII=&PXphckVGpO-OER5zW|?Sr6qxT&ivVVkftKWLy4)2GqMmWaqoG08;9s zQ*mzpC_OtM1;3FB^c#;H;x1hlvM0XL@PpN~D+KwMoeEc+(i;a|i7TmS2Yy6(+@gU? zHHx6yt6DsQiZH)NN zUFdJ6N9BhowwRw~l727K8pFHCnlN*Key3&pc70qO^#~#<1fkZ~R?TU{aTgD$^}T8m ziMl)#TOLphsQx-x436-aUrs=1N8y zfiyZDR)MSb6yY>C6NT(|XY})}M=m@_rMqQZY_>&&6AG^!Log^*`0ptMZIUW`f$0!N zE57+%uxLIE=cQSrxze^~#vpCMWSUYGczbl;T zr;}SO^R@4iI5w&GyA9azlj1)%TEH|b6LiQ6m*d>o z)4Z}b+LuFt{FvS$Ti>cah61~EMhu%XJ?Z)Q0FuAwV(0BuZknMEEIrEF-nw3YG|!|# z#)9}2Ae(0sF6WeXSM%d_cEVIuYzUa+NgPRt7m+>L(3pQ`$&z!*c zF57BRBd*0t;9W%_&AG^Wphfy}bM0pG?GtRZ8h<~9Q$5|8e_%-fyB}UZZ74=Va-Gia zp~WXYb~4(lA<7W&Vx}3Q^#jXjDeVfXJukXQY%JCG(bp{Po7_htO=2#`Uj+E)VTyGh zaMSgfE;h>^PHU}LvrjsBT|AebDCHvg*$j!QZyoy;hSq=>x7r{viW6n)R>EWnZQaq) zR`PYH4Fl$SOWE5$yub$3_G)MEp%kY;GhiTMN;XWIe>D&pZT+wH5?J8=*Anr^Qy^du z29NxIy((r4_MTy3*|Dbeld2Ke<+~f`CC8s-M*X2x$TPCzW0wk_1kyH)c&7uCh4`_2GcP)kNGDuAFRZ- zhl8QBz~mJTjGO=P; f|6=tw=GS*d>nK|lz?=XfHVf(J%E68cMQ@<*U-Fs zxX*L+JkOW^dHrDQnZ5V5uGQ|n~hQx}3*H&c>Nj6`M4z>J0i4yzB3e(t=lNO&*5f;EIeb3{ujl1@U!7wsPE{YVrn4(xAO-%uM_Xyim5Bsrnn z?%ch{vv)ByC(GT+E7qci*gZsGyIiLcN@3Ca|Nbj7ND3PG`x5!Tdo8LNf8GLo7R+_W z{68Kelw!cep!i>}!jk@fe^scGtZZOUGQU!#^(4_r8znY2_QrW!TpaZ2pX>Ob;nDj? zx|rM9B|H_seBOv?*!j^x>n&yi1*7%9r}fJ3^E-H!{|B+gxVQL#-OVwuu*e19r}fx{ z#Rep);?s!b%TS~eud@8!xpD3dwd`;9mlRIdpL*XHKYh4B23uaGTK)a0* zA016}GD1t&1D|6Z`jbn@je4w3mYJy&nUpWo?Vz(;1upm4<9YR_Vq;_y^hJ=1{9ekx z$C!11P%0=we+(BCM0e$fILO%=_VRDP0e+@CN~ z_rj+^WoCz_R}-A=9E2&m)ImzNgJtH$BjkT@ixhtN; zH@t&yQa~S=39V)Zrl!tJ*#v`>bO#J0zvYblA)ch|8%0SUKnt^|;q=@*pTG8k$E<&6 zN%zPsB=DX^xvelyj9d1;es?`D{ashYZw5wc_#q{G0*YsD-6X&Gz8S4g*CP?h)mxZ^ zzedhcX4-qpg9mb^%gob5mWXqqB_9j! z7URU=eMkai_#D={8wKELNDPt4GQ+ss zeJ_sd@e}VxqyM$c|7h+0=46?m6v*_WE5`hI%3dp7(B@o&sJ)XR*8|KlN9Jh+vuiMK zmHPl(B+~11rur@opU%(F-~8{9xzV1-z6of46F)Js-#7a*4Hbuy;re3r1(35ZLdCpy z{b=idxbUc{sm*0OKNwg$RKNWZWq)whM?&WRZdryC1N`axgLgX=)}2LAc|XU6163F% zuL+=bydruN#vo3?6x6-hIedS?CL8RShOcJpaj+S zl*fKK{k{p}!)QU2kP|q1+*q#{mxf4hcZ~{Y#~mFS^m|8gk97Xtki#e99h zNDF1KdNTJy(wNY>@7-x=n=tOqrN`^R)0o1w;Epln@x`Fv5L-wtE~nTgk?NJvZ|_0+ zq{bc7cKqM!M_+=4qzs$5AxRvBiS>YxmFok@5JToe!R9bp3Fc~wdBq5DO4?(#NKJNj2AR&A;gLRjgfnxxr$5iqi z7Jm#hiI*@vlEzzQT$n~<40nEUll+gTV{V=vCgeC)zBiy;=|nNqT)#UgM_aI?PwY(z zuNM-Rq3cWggK%DQr0VhDLAlWB(WO*a6B~SuKDp2N`|@#dK7aYL@l;ndC$Nf1o$JZD$?SEZWnHuc{$m^>1z6yetT;%x=#JKctfyaovG(C6h4!D zgqFFrpTrvxKUmW94}McYDAjml&dNrB&je9$RKlC~l7j18b{Z7~)93-`_3vAkF8_@Mn&+{F1LPpy)XGLuXN|* zFV~&GU(kgq`;S~YyoJ;Vl-5VEa>DEY+TW@s6?CIHKW}r?n!_**aoTQakp+Zd2;g7+2{ z7uGT3S_E8N1|yIdLZz=<@mS6>6vlhC@A~TphSlL=T-?cSK z@HY>PYmwOx(Wp7SY=7XOR4AJ(Z5UXRZGy{BhH_h-a$9@nz>vpNM|;=BufO1c`?V4u z<#tlbAHq+1g5;qL-X~r|I@Uzrz2P5pZ_VjtJ}?p9Dz3uYMn9jC&>5m!S)?%_B&35h z*d0dOzta@bVZj{_wd#1(Z2y#t1I41G<2m#252*mk-|UcLzyNHZv;3Y$D72dUqwi() zT`WQ^C+xf0<()jQLGkMn#nr_1J5P%fjqa}^OsZ)@1b*+TO#sQJ9cnCctx(D4E6)&a zEf4H8P1J?Q{nnq>3<+Yt-Ca4{q~iFQ=s}|3NA~_KMt7-mts2<5kLlP_+tj|`G%+X~=gw+Ih4OpAo z1T+*?`Wpv-h85^e@GRwFP>%iS(a4dMHfCZZ$#c9iO~#X^4+Igj&t!zlC{_=;PwLN8 zkIDUBgyZ2Kzt%Jb1VEItZ&}CwQ1iP0SCSSc2ugW5GcdUNL^GpK3u}Ds0nun?Rl5^v%oTAtY9j_=k)%8br*DIqraC{wgVz6k+g&il#95} znw+D{p3y;SPW&BPx&3#FzR!OMzzYz-J+K6vKb7v`V{)ln@kCAH?XN=raSsG}PYCd; zC(SQE@OYG^MXsS4stCD{>BmzC{4Pf#5jcRRWD;5riN7!amsuamUi(L`u~3x+O8v2c zbEJQN*7jWRYFsIC;asOBG;#vOyTHY0dtZtAr|2Ea-+W=Xd7qEHKY3p~z3~e7s3!k^ ze2ob#m&={Fg=WxxNB(0cmo^wx3wfuhpC&({MIOVpT0XtMa@+21alOZ>nylQ`mJwoO z^uc7gMa;#1LQ`yLXlQfZ)&AL$Rl&D!aaUJY%`9)p$jDR?eoX@F+uQMhfq~6U7iPxB zG9n@(&mjVy5n|CXF*$j8EW5$i%?M9V&n`|szqe;%vA4H#%F5K2&2W~6r=~t{cbG!* zn|gbDKVME~R#$7-&edB~e8a^PUd%pz`=ZHu%;+Suv2jl860I|O{%(_y7j7CbIeg2O>oC@_Y`L` zh@5+gW)s7x_b1&2LEXoX1LAmEE3)dRrlv;9=9$;j;iS_?4~Q;bIXhR3o?+<4#>FYT zeOp;en*v8f^Y`q)K8zzhYmBOWy_pkrs8Q8$?h9iTM)}-2&U( zF`qFKZO-{5u}E{0h88BM>diJ#joFw1T;JlG^VtEeSkx{_J)S_@ALYLS?nX~n&)7p| zBb~9$4yITk$-X(v!`+G8L5f&!TA%w<1T_IK#{HWY2eH&YG?O&|A_=|*c<_U2t4Ti= zJxqT<79gP#U5DZp-!>GW%N6}70WHiOjssi+^ah8Glok`M^I#AyKRRYFoe zi24v@Wm8GAM9c;S!&p+F9r9RwBFz{Et)^}ZGf_n5_TmD5763&k&cCiM&&Mz*Qm0L? z0}d9+gPSXR&rwPk2JYOs(-te?=QWXwB)IjhUQuwigi+mf<_&xO+fWEKAwRbH!IIK= zEx4u=s)_Xe6993olYBr)%4I(mi0W)&l0%`7;k})Jit8X~JwYdaecHd=9!A;jv_Jt) zC-XN{wGr8VPfw6a%S*6JdM%K1&`PKb3#G%IQt4>rKcPl{jho9&$Dgy;$`uK5KmJ-v zJN!l**GnR!u%JKzn=+Sgfuc(u6Kq{ULbh*VZT(y-^4nR?FK)Dmd`#qb8ys`lO>ei7 zZ$aogO%zEB*Wt`)5dxRKj^iUE`DZ5GoSo3cPz7xpPp)ys%h?`7x~86M^TsO z^M(hfhIQ916GH7=a=fpRl)vUTP0+%>kU+2fzXKt+_GJW~81|XUu;vDPd6Dpob z8TVm0IbwadvAGG0z61rV`mQ(}_Qck zbw9U1En;4L#Ela6IFgduUeHhVT&)-g z6A_y5h2+8QJ%Ozn$JOVN-7g=tAK>(pPA>;IO53+IBmyf=A*u;x2)4W18=497^*Yh7 zE2A*`f@_`kem-`)x~TbVRJmU=>Ai4K{Z@=|kE(2$*t5@kr^(69VCBx#qcqgYE+z4Z z1xoEB9B_Kp>|)V+H7!^#Ha1pcy-pvtYyt?b0WLTf!}9cN1!?(hu~t=GK(u6qluep& zQtiuIeZf2-GS>mC#ot6itz?@{)&?1C9F{{qn+cq`CvIIS&3~W;OmFFt4@!OtQ*R{L z0-OF`w0&yCO0MOX| z0lO^*+KZ|%o>t-8cZA&bv%5NO_)cfyS>UZ!dl*_*3GUMAsE$NbZofIlaw2#Pm!v<(<5=i^Pj)N|g0gKWR+?R;NLZCNIRFj^d z@*YHDUfD1bq!K%mDN)lmTL2ngcHMINW(&(UYj%1+7-Dwsob!Soc(0S&BhRzv74A*v zZZ;ychkCnV-_9`n`UkgdPHDh;84d7B!?JpYLGJ}k@6&ygBZuH`JkpUgo(sN@{5rf|E|Q&XUlnE|!memc0g0WezQ07me7D5%kNiTta)hxxl1rdu`&v-gak}GUn5!q;5904zM$>QNgoYg?%@I% z*p^IxQm9HGmDbM!pd8Wd1fOhgKw)==HWTi{O7WXx1*p);Hkp~23gA9S3$6VCIvZqA^4$NusYv8ICH} zwCf>>GL)-lZ(A-m!a2NWk4Hx^gSm}~zyO}-&X^w&phmQ!psi|bgFSIJ$hdOwI#Bv- zzFZHjcJLA*U5B|epn{3LV%EocC?@_A98(SLNb)*6-dBfT9mm`<)R4tk@G zJlGTml?=WFT#2Mu$q{=X>V+v(&-w}y^87l$?A|^PHO@GS2kIxW2M?5H34rCseE?Oy z*2N|T{D=$|h;w#Ruywfyi+&hSusbb`Wl7bX`JMobZfb?O8wI^mB}Jftao|LySi%Sg zcg|ki3-4z;It;ZbUW*Hwf$q!SDF=93A7r-i0;T0{ejU~n2($7MY~t@;((r?4Uvd{4 zVZ=*2G7pEx;1&0f_&N(rhv1J$4U}r?3RT@Tw}y(@(^k_`&y=I0>4*fG;w(XP3J0nt zz8VP$?iU~7ww~^V%G6c4(V~>0Wj0((kxOPs)>UEe2LpEcHt4P0Fk)AD>O-#pZNU}J z(XFPv;+{A_$=v8A?7kF@!v%3J4C}lW+#>3&&zLk=t57|lIm9Jj*|s~FPdtswuJ^x0 z1sR6Q&LwOmR(#q;eSv!)LcsNUZOqVh5{mnU|5n(a4l1@0xQ}=SVE3 z$zxOlpvuIp47vNw0Uiu&0~5GYGEy0m`tq37$drje4GnLjM5ReaF$0E6F^v|sb%iq? zu1R5z5b#8sSI>C!H2aM6Bo&Cuszuhf_^OXJNX)HQq=E2`RAnNdJZHhPluA`7sG8OU zU}4(o*%VpIcm=?z(2$is%Xky9^-olXy8{W0#3euy&51AYHdY{mXv{+rc%+9+6c8D@ zz}*DMOotM9b4gpXe8h;;s=^b;&oc$noC+p;4yCS@rFI8iSK-w6Pn2^MzUVM&;<1b( zb}!}OrYpGN(ro}lVvfRV=!-~7>aqUNKRCidNTl>PnM)gs(=HyAkJ^6vc#~KUS5gFY z*dR-;@MKVu6T?6TXxf(O9p@~TK&xyjc+;O3lvl(Fp~RF_S;ie zrxo}xc`8E{ZSI%>i=nlV4?FgJ3pl9%5C?%qYJmI%fPAln8a||o6xLHq5b7QaFvALF~y&Oqzp-Lr&tH7JKp*zA(aiuFuRN^HRP(2H_BI!`oFXjxEm9> zDiUL^M(G|?9;egEf9+n=`$E+=GpCmjzLdrbD9Ez@QO0UcAEeKTR_DWadt3 zDw5TztT|q{%Yk-F8-sQiIJCb(pE5#YhZrl^gv&e&5cD+aLp-E@1JbXDo+%*( z24Fr2RY38PO7iHYvI2BLvH_WUBi)H0vXlnJ7s1-Qa(!NwYpZG2*NXQJ=F*75SuYk=?ACp80D6u|j6)oI>AWK- zOd4TmH2YHHCl zwa&l3!En%C-2L%NM`)1z9W~Gj+Pe_EoWRU=bq2%FFPbFk8@gz-* zt?6>hZn-U(_|c%*#L5q=pT z*~0V!Q9$5PR&8WH4u$9D<_a1j6MgV)_(*~JhL>{3CD#dtm)xavb01LlOOpe|Q!#f1 zNfYiVo$3d=_uo969tCCuztdMFF8nmNvb_9bxjW&7cF8F7-i4IAOmjPs?{9zpeBbH_ zClE+z^E;|J)q#@POY%}GDJlHiV3&Vm?3;E~Hc<0xQTwTkc%K>|OIV;%n#1e?^7=BM zsI7e_D=aMhd3JiEPQR0d=#q3F$aL4;Pe3+9FH_!s3LPMw-fptzd{wCP>}Net3RYx= zN${1Ovdg}1x4V1t-SiSj_%Zg{UQ)|y!bmYNFdPuuubss!(+pg5n-K!%oHIAotAXFL zmNJZGYC5n16By+fGf0Q`0rECq8p=_!W9Q&d38xay6u2O`qI!FI=1h{>tdE1%3t&N` za16UM1G-9&QwO(QIF#bVh)`QyJu6;H)Nv)0Vr@-uZEwf>G{HNn>7u(S7Qi>Yz*Wq3 z7pA^{AYGm)F+ySui*K56S354Vb8{&o&P^uzF{Kj1N_}DTX8ma|%Cwam5#2<@#7q!~ zF3?jBMGAz*OW5BzZ9zLFIgAZ+v5K(JQyVi#QbaRir zIy6jhu~fi6QX|mQR}iV`c=fZmt<%uN9B6|nUEx}P#excd{lfd4+lROm^#Pds2M41P zROh{L8y!_OH3ncjz%E`8mJ;sTO`?>B^g~yx*B9yc<$Vqqb_On5Dp;PT*=vmJ^ zIcw{0;oyv30=qXcf1l{)E2yi|amh2Hx5*fA&O=Qwu4>2KjSs!kgIoNp$^MC0(*Vu0 zCe^o<8{B=DhwJLjoJde*zu6P+Wv?f&je#_R>8ckmg*Z3v%E%saRHktX0UeW9)qw|# zQ#aM!GO-Pi%+xafx+UA3YFa>0%|i9B+M3@IsUusD15w*GPGJdE-?5-kC1C zJWxjHvGQu01i0@3!Z7Z_4L!6cr;G^P-=MO-6-x&sWB8lI^_1rzA{%`$F77WkqaP@^ zL?w5~uf9dBFVX=+6C4Bj!BD`KKlc|~gN2;dB9mYE(V+~?cc|T_6ts=iyZA|}SPxtM z*d zLjhUZ&QyPzx`l)iA|rG|z`WkUdGVN zuq&DU2iSb~tyW;ytb@ld;OLrz6)`3X2m{I#7GW$#w>v3IMod1zW zuIVPRe7P@Fbna_wvoU}ow+)#jVt>M*@CX!KBIVG^jxTPR?-#$;;UFLsX_lnJ=$=J( zO`hA0Podd0o$^;x5a7yiXSD03=r@Nquv(*D(`)#7aB)>&>^JjR5>i=z!W`Yj@g)?H z0DKG~VQT(2f1Lfn^ZOG$m`9{g{1J3*_|{v{FY8JYB)T&Qwb6#pxVn>~@@xMK+A+Zy z;VdUfktlESzHZ#mwjSe7buYboKxOasFCXtiNVs)L+J#kCcF3-!O_wfzsAv8j2?r3Q zXEjV%9{>zgE!w3d;I*)hBk#gt@cGuL>o?C1aPEu*a5W5-y+B7qe~j}uyX|tK?d|pQ zE83^fKxORuy>7vrJX;T=WS}Z9eu~wPbmc%n394qM@Hd19Riu}Ily)=7&^0#I{gU_| zy!@(a<@c*<)fcAURVm?TmHjU?gK0owz(>y-V}OwaE(M9YV{Q$$VW1y?yZQS>roA!7 z`8T9Nw`R`Uj1k$<;Ewo;!)JR|WZzt2bex`6VDL}yXBU^bsU(5u9s$Jpx7Wy~#U1lb z_W(P^_h$cxPC?X%xFOv3*wiyd65-G$U4ifa|MS~6_8GHg+C|D?z_S85gm^myDnm(D zK8b6Oikk~SBYsFg;J-plT*peZ>WCh>zvO8yWAf$1S53-(`>clVfU)T`#n zo@c7`R&(>w-%AFFEkY6t`YU4X*(klkl%@!!GLzN^Sg6=fNZsb5A8w%^@ra!rHfy-5teT=0GS$ogGyYjOC=oNz=QR*H#7_7upm=Tn z3Le4?mI_;^8tz3J4(eYZ3cYg^2no@|Jt60$eTr=s<%(2r4^aA5=^kT(z-W{)g(~G7$`bW*{x^dU=bxWW_ zYE!=hLFF~%BM#;yw3Fc5l*|ePpu@jmDcc-E7kvV*={Ly>g9(9VebJriPU;5rS@nR4 z7L7{AO!ez=)7d?XEIAXkH30#fBIF2zMLL zK{jgyQje1t2BG=sDy@IL3{*dgE$9v#j_%K|o*9r8ndF%7W#}$)Kx-i#G0WmlVcO^a z(^-##Jt?pe+-;H^E;od=xjcYUj{Sgcs1`;@hbKmUL7_;Z?{0>4OG}AEB53He(`r{!B97d)A zxR*6}n1N|vrpbH0R>IG5`mXuzKV1_aAm&9r1*-I24BtM1>5E^3<3p2!Y_C7Q?h{9& z)j~lSs%M~<+#i_OY`OuE@Pb*f?k4%lM+wx^Q`gUsH#hCouz40sc1prIPVlTQ1R*K- z3yx{;pp`tmQXX;C8+IkU{R}0lGbN&4U=zJG6TwHuo8d3t1`V9 z!8#I&zz6%j;|hY3fbGcqH-L7u4=h$8gFA8H2 zscE90<=KSc<8|dbs$`*~OOA0eKDj*YJ2P&p@tRP5TE!=NRDJ#;acXbN zLK~NETL&AzV+}pW5*X2Y22hQfy8VJZZf<%gUv<=Y;UW5Nh(nXH23Q-gt^_DMIscERAZgks+F;y1{B0G>?WuT@ueMQo6D9|l>@#2NS_`)7;GigpvPIO$H zNrRo_HO4hM0AloV?j@t)z4`8h(~f~Tj3N*~v%_IQ1|s9=xzkX4v{DAAn?1j4LF2u* zYKP4?#Z_inT_ydR1(;^SoHI-JS?<)&HJV~3)+s(r+Q!k>SW)J+0VRJWn7?aYZeZG5 zJX;ltMNWW3dI&=R#Q=bwa}zQjL*`BZfm|{ZKTp!Bad?ck!iB{HsLAdaGn$1!-#0we zcOdJ?h?}ynL)(|^dDd5Dixz&8JC@eZ{a#;G0l?1K$jCjQ4t|a>0!B$bZ8KA{JoFG;sCFS&r3j}+^S`E_3XCvN6?$43YSJH4XA!(;hZ zc2!QtP73ZWbt4u0^0r4fHP;N<4)@cQo;VkZefGFd&_%0gy6#!?!s%5Z>0I;!PTQ{K z3rYv>o#gCg-Kde(gr%7>Cxt$Rn6|tEhvOe^M}nuC$p@#8ovGFzuIRZR%#L&=*wGUK(3cdLv!%ylO<=$UUZh!!jX!`-$`GYKTsrt@;_{&=X_cs+b;cV-x`3a*B%w z6y0I`pThJj$Q?WEiu4Iv(50njYUr~lq(B5VW@~Xe2(J}o}=qp6Z}>( zqOw~eKP5PM-OAc$-fB$|!Uv2YDCmu6=uE{ivcwxVjzc7kL~`-aLYXy!$oq3sBo1u) zyahk-PI09KGT^xp=yPGUrr5elb8&L+^qP55f;X|mzG;+5U2X4NJyBCvf6>ULerAu0 zE(1!p2Cy4&psVO}g3mT&g|slo9tjDqmRM%0*3;V)QqAPu=<7_3NWYj$4EqKk9be8% z)=5vOy(9#ON5Ez99!l-8ZAe>iGUpysyW zAG1Q;W{+!{XBS zAttqA%YZwDdW_$-1r+weZPaSbs$ywH=tx+?6-(v)Z#)Os6j}_VW)5X1t=<64GABWXlyeoXjaqdKpE0RhVZ`QXlII zDGv*`iOeamGTA#FEoC(cV3*gu?-)a5{N$(M>4tTE4}_Sd>YP9 z**`jZz2DpGS&Ha;Cj9m~w6;*Gzv#!umh=6uVz0U^yK))}bZ3=}9Aj#$;>SwJaqVWk zrziAAz8tLfKZxpk>5jeq3yHVs2zr&l0mp&3jl|Ouyzo=P1FWh|4?jFj6N>`c4h^@H zftbiY>rGFf4S_YBKzX}ZmPF~gvPaB*$dQ*V`yHltBlx|D`JdA zB5^#Fr5=Q{)7fXl znskwEt1qZjrlW~K#3n@FJx=*{&}#2BLonyT2TTERsmknPcdH=>&s??DF+c8(WdNZg zFr#^WI1hk70o*>n&Xt!6-Kw6+Qjs?ePY3~l$9_kdYkAu^y{cpO>P1dD#FGI}uuw_G zA@ey6gf3^^I_ARJ>Yn$}EjFzRom&YU@hv9-YQ^?xcOGhGT-9+wp4=i8tu&4*&GbLv z)~*%OSDLFd0Xq&Yi>$hrOq@a*4^|HJ9U(4#U-l)vs>7>KjRLJC=emirUAOMDO;+nF zR@p}F&x+=3tlchXh`b+G#gh#YRl1)p7M1qOhQ~+n<3}E{!##4JcPYR;pJ~shZxfa= zXDaVNw$_hY^@Rj|4|RVfkV?~36O0&@KmDge;X`aQ?=(Z;SF{&@um#|>T)krr&Gxn_ zK%NrjD$C$%`LEFNVeBSwwDbsO{tAr8zvbE7{UhRW>!4lu8kflJL`^lA0gbJRLtMPYW)HN!ePP zqHLoaJ-dg-x|6%LM=I?4CmwW^_H_oVCM?_Tq5t^87Fql1$g&b|o4U+T_3p0t2C3Jg zFb1N+nwlAk-cWz$jQImujNP9(8s{sNCt`4`%9i!5)V{*05DdcjhOcu4>Y^w z5g~?b4*~5HAx00LVJK_NV04~88Alh05|o0t9XgE43(S;_-wj=N-Q@vhMThCRT$VQV z$Q~!D`pE;wu>_#3>Sy_q*xXvN*aBDrJ4zXD6RslmaYpbYYW8M$wQe7ggb%^hm8-xk zD7jqj`a641~sl&{&M&&s)C` z>Tx)=Dj-4g>s`wOCJaR=Pe^pQ@!2a(-7LRa&2Vq1IrQFp~a=NhM(@ zXZeiZqRvris%%@ytdDrTt2-tw({TOE*GR8CTYNq~kuk1DegQ@A*A0Ak?#j!VET{DQ z4&ySo>b>B2ed_iELXf@O&Tpaoh{c&DTzI@x`+lW$ERjx?qvwVQ{TO?AOy)_zMw3!J zi~Nrb6195Moa6xYz23JL@`|y?=ZKF7FMpWp6-`NZ6x{jzwFOmDWK&MR>>KU?m|ybH+)6>{R&C`>8Sv_u7#v!Onvzj~ZPkr6-Ul@zhm_AGKD~#>>kp z(UFZW@8%T7p037srN%Ptzu@stVB*=)@ikIndgvVRwc)H{v&KcH5Auj*)#0Jyein{H z*;=9T3rKt(XS$NEC-jbBoWcw3Fg_=NKnoSMRUNy9ZbLV*$5S& zkK%Q-*SD6|^Sqq(U}3a>?zTy}WHHpo5ts_&4l*7lyoXMV(Y>2n4(|S#G_+O_j1Dj_C@&dVi_89G5@enq$lP^6GvGHX}9PZ(<7J7_I6rM$uXrG!NP>jKZjq6nv_0iY%Eb4OP#z&N~;$aUpQ8BzWa-r zN|m{UbpROO{RccqB8qEJvD71HH!vkKmfDKha5M3< zAowH?O256>FP0QHa?4?r4D2o9Z0mT<)Il))`$#O{&FQB|ibm@XiFbit(s>}N&=j)#-sc-46vp<($`Xmn z89x1ky4GVw1&4aownySDmjj)}d`^5_Wm7sehd*mKe~7Vv7=GEL&a82NYq+5)N{{wj zjU%q|`sz)M`mscET}NB8J*)yC0yrBB%!tt!!f+G)Mg`}lUbTNSaO1Vlvu#3}v@g2d zi5>gvtO$CEJhKUuo@nkZUbw*8-PDBjY8$05RKzX5Ny^#l1zmXA_xuSF`e5bb)1j*4 zxHncBbGY>p5Lm8C9aA5!XG=DNipvu^dip5XTOS;Ku7AyRpwAH&aR80~PB_Dl;|n zlSD@Bps8SqbdJu9+-sGV=Gj~dnB>t&`?kIGrr_jVnbw?2FMErZpw@%~F2l>wwAmq# zqYx)j(t=45LK`k^);q|dSQ=n9$l&HwN|-sZx#~6{Emu}fZo?DX1mi9kW{1i8_an=8 zESdN7>8V9m?gkqUnjQuf*40;Gsb0|xF5NWwlg!kYuPs+_`f_?s(@E#5B>eC$&3coW zZVu+Gd=mJs@)7oR^Nr#^NWu@cXt4?I)5k9w+6ce0)qTL30O zUi36F^J(u-+|cvE|1^mhAG)YTKl;@RPoTMbc8#3`vYAhg-$%Ow>{TvQ_U{q{Mu>l7^`XyU_JB20>F6gWz9<7FIy!&rDr zz|1Kn(Ban+1vECJKDSQ7Z<3qPSnHiuPR1t>C| za_yqB^GKKWy3K}0Mj50DNpvwer0+5VSryEYxTOb^)`KqHV?UV4vfg1AE_u4!O0b+2 z0f}aLnl3N*G%vLwmOVIWbxGqmsieW?q0RGd*e4zOF*s+uthT%Jhg6T)?Bm%2n<5vF zp)3#H+tqIYT|yot14aeZ=mrw~Nvm=b1ksnNH36Sy0|YkfeOy$g=fyBHjTO`S<$nD^ z;DA*5EY{N1!wJXsdjrj%iSg_-iKT*2vG|x5zJFU^?rlEbAhw=#A~t86iZjOaE4#uc3rbdy8P@&y};s2Ojp7S z9v{y}y3XpA$Bz5FqsjGI(KW?M<+cB@o~x%*1$8yulBSIE%9@lyQo5Gj4cZ8&da6~<%&4|zXiB}e`e}~9yo~H5B>RpQi)ZJTnI-bkH(15 zqE@>cix2Ba&sDO2jDj&n6oPLds!Q+xi?>A1&=`t#8Pb0dSyI) zaYe=TRes{KAD~AyU)lIiwEyAy(1g6cH?lRnh#T5l`^1-*(Y3KNDD4yg+UVmf|LSYim{(`7fpq=uOgpZPFmV>jOaMaqRw5L zBAO`Y-3|@gbn$VogM)@_;r_4Q{gJ&N4Ugu#zFdfrTUxxl+NJ!rg3cvALBoshv)r(Z zC&hd0kabc?)dgYtQe^$C%m~OH+HvD&r`*h%v`X2_Hlfce8lLkC9}a1|Hu8-Kx_)l$ zPkR)Tes!1!{HDdj<>lp#GT?^{?xQk3+^+MCIF>o$_CHs>?$rP$-~S!Apz^$NCVhwdrpds$1Y6vDDrT)iW)#+} zHj3!2?L#Q;gNfnO0w7VA%&lk1&oZDSwpWkoNC3DQUnaSL-4QU)VHu-zt~4OoG%bZl zYN_?0;jv%GNH9ok8vaji5SH|CfG%+!QK~gXYpS$(y2Skq%qxIKIg#*9n{o8~Sbnj_ z*rG0h&4zP7C+D*wIMw5V&{zr^&wdi^G!aArdQ_Z>>#H9ZHwW^lXA8@*_HA0#%)plH zEhMKA1%=24YMF!T83V#)xNE`!=c3*VRoszeR(CQR{Q4T2#xsJte1GNO5`$^uu z=S9FUZ=>c|4<7P|jF#SuUl_pk7X7X;5PPKFd!zt(57P$|nYQEN6MWX>$?08os5?wc zb$j>Z?aUmD5@%ryXUt-p25~~Int;yy!#o<60=|hIz&f?TA8V(tK7T|D!~7=;-1C1G zwUQM&3AOW*118Y9Bl%`k{4{xT7rA<67{WP3H7{&6Q+x=Gh|KjLy{Lgc2S%TTbss0= ziL2mw9EbvkJ-9TBCHzeIRV)}xY|lYVZ2-$Pst`vR*n1)W_K~KU5`Gz%5=L20Vx9~t zSjRi_j3&QcK9B^r;2uZAyjXf)af(``eQq!dtg9OV2GILX7f7`wQ{X7CitT?2cCtLr zfX$&0n(DhQPFkUW=HJWv!IS2&M0B7l9I3zBlyeO79v+R4$eZ=4ThurxI8<3jPdA;1 z-O*nc?(k*;@EF5x*@8!2GFrAe32D)WOuPQ*7!*6m^BRQ!BivPWfwQzCq9Qp}W}Bx_ zywz^4`}eYz3aPxEUJA@k(7!2o@H~UIJJF`TQTho);bq6c#;p-$d=E@sQRa7Pt*g2q ziXd{=KCs`aFvQP*S%sC*yg1@R?-|LAw`3x$qWY90P*Xzw9QeT!Oh{59?d6)>{Y&6< zs!kla>9~0bxV!(+*LMsSZm*1Y8W>A2hQLX2WO@GlJ*B52MhQd?wj?HsUM5g=U0?=R z>Sj78$c*ULQ;P3oG4wX96Xg?kl#{muHMvpTClbS1;=Ke-4&9V$xx-W_U3k|BnYA`H zCtGMS)NV@!i543tDTH!s?gGYYzD4Yd=bk8;I(P|^(1Tv~Gf&O;*dBrLk>qFio64=;+3~A zcL!25$*_u!guUMD)#Df5=91o3z#N+)L%Ut+!wK{agC3mOtlFc~n!8tfZIZ&jND%uc zYc8fbj_f@fY*4N#@lFStGznTAMT*wi9j z!)4r^@z~CfL+1`>79rMk6>?jQ2QRt|Nhb8(zG;uj4sBj_WHt2z>96Y=n;7^=dVl7!1+AN4&+ZYl1S;^Oz zD%I4>ZOSAB@CCq%a)n;%UceFBMh(w(Ugy?(&h_KSXE4TJcY{-_&NnATLxb-?X&I>} z|6@hVPBQL1C6Rw}EX%4(q*rH}hUjprP%4z(@Mjsfx4C!26|yee_Vc5R3j;JG;yo4> zuhPH_p>_7xSb?~#zLSMc(n9Ifxw$7d^Js7n7b>-u<>zfW?BKMWi6Uo6vhn&9AF z>el>hd>0`#QTNa;n&wTFShZzokxGE`9dJ~xc>+HO zq1F}hp_GD$1b`(&ZARjB=P|HvXrG%V+1%=pa`Ql!AW5B#299J;+REbzi7=-96T$11 zTd0zwmgI6v1zAm)g|`ii9pIaz&5G&b;<4_tAq@pS>?NC1ZxV413YoiKAh=>fd|(BZ z-36XOjE-JT3eFE}2}DEvFn1cSOoJ!2A_*>>8rt%T%0J3LAS?5g%X%{$gG17Yo(_|q z2iEj)$F2SibgEbGxW6Y~I#GT+5%Vl{G9jMzrMSHYZMn{n-=Iej*24fgJ9evGH*nFza>1A%0;3{qDq& zTY^-%X&pra{)7_AV2>U8OHxaU6EMur9IGuSzhcKskG0wXcuBP0W+QDJ`E+K1d$69ubmqA`+;m2dmjAv z1oouw`fokV98l_g)(d`x)MSY5z2-2fqG{=7vlt=>u??pa#8$p=%lci8wan3wE{?XI`J^9LI0yz@cP^d+s4l}JJ-`JbBNU5nB-BhCC@G?o9k(ky|nZI zN9#XqkRC7heXq{>RcqHdjo6RKCX=OXt~ZiQJT4AdAM45tNHTI2bvqgJ9rMdgk)Lnc z{7R$gqyI=wM)p8C^C(X8h%}*u8ZL({<$GId0ol8z9mQ~he>?M?}1(1Aj6eTXt6x^sFEHF2^4>ifzMuSYF6{~k&r@4ZM+3+~zt zjLkM;`r^5&I4=c=9mgYBOidR-$GEMTq$kzKw2S5Ww$*?IxN!BpV|eyT$x&lM4DzaK zeK&_grBohcAh*PRj_x+l+kSF6e)iOQz#_2OK;R42{}ga6hTz!emsLG7+e!*wc)&i@ zAnkO4l2kZvC>d_MNSLTr)bdSnZ|q3Ss?rtV`PGw;fqr*NmkA)|St$5QX0mNEAI;2* zc&YbT@m*cp8E1w`l%QPsesqAmXjC-ke2(3KE0FT0$#kW#KpR?6aIo`C>l&lOtiDV;_H@triP}mpeQna?T7DR{jto-P zH6CXxkDD8LMY(=Alu|(Nnv=YOTdn(VPkBA-#NT`S295{6d_*9G$5i*NSq)jZVxcw_ zQu9k*NpBM({ao-@M_{L4Vft%E{i^Dwj*gCsWVpGNA5?bM@R=5U>I`?B&V9JxRqD4y zf`q|UOQDZPzSyn7zWJzU{>hQpokdF8|jnZT+{$|2#lz0Ghj`! zZhyFDHGrdiSBCHvA}p!V)FmhTfPH5 zCyy@k%ugEQ<+ORpckL+4AL26w%^KI9A6aYG24z(IXe>h9rS{zUO?o)BMk|o@^UT`p za7*1in3Dw31KbqD9{z{35pXqDY$!Y?@L=-?T#bblSb+kTiGmJgXGkxS0rHa4y!w-FC(MRNpsE!11-p;KW44i1Wv*|5*{8Ov>A| zv4~9}AN3>o>-R-?w#jun$*sw|CgYhEvqfV(jGYpv;$D2?5HyE0&;~dzVuh;D?7Co5 z-H+Q&6XZ_K2bh~(IqN ztdo8gmORF+OiNbCY~iAk;rh?AfJXGnnwC;LD53PgNsK7lv(I)JfUq09rPl$Y>Mku- zrajP9=%Q?3{ON>oC}9kjE2$=eW?>k=A-3twkSr<2Q2JUUz4g?L1kpC%oeW(S3)&S9 zA_e!!&o^uYO465yq|8Iz7w#v7FEHg#);)ZY$fG+4I|}#E3^~>9yq^!`2+~h2&1DIiEvO^?Pe77A{ zmV9MTk@qPgKA!AwSh8Q0h15nM55@}{={kE6;d-j^?qhwG$x?<9OL^>^#n{)qvV(Jt zS_9Wd91{gPj>QXwT)(Q6oDAL;w8j&C zf5rZCD^3K%z7UE`xIeaI1>~i($#elBhFxIknyId1liZl2```9uRfjLYC@wzuGAGYz zIyR*>BHLlZj@?*9jaxyd(W`n~Df>g7lp!Z4Gy7c)U#?@UCUCAM+tl)5BEJ=zP~U}H zo$POLOh-$ei~L$Sq}R_X8N$~qG1ZxVDyC%3amdi$(89hey=+)$^Tyc$S}R``Qm^a7p~wXBmxwU*#44QnCQwm-=Ro1wPvf{OB)OI$@=-V z6tRmYO>eE8)oa1Pg{#Z1u=_IWuBM)xTz>CRC+hzBCW z1YLZYtRpn=2zk`wJiN_q#PT+wqy%I60J1uQJgdIY4H#`Zx3Xb%!lLRy&pA9Q(F0e7 zp2m;+Ip{{b;XP-Cal0|Mt>=?mO@*r#rRskgc~f5}c=<_mep;erdV2V+;#!1rTg#W4 zUPvi~x0i=W@?R4;;yk|>6GAN0){jDjrKDPRvxEl7crWQ?&__%1;|uh$5PiEb_b&zk zYJ!#WW|L=yn=|2C>geJkyipSr%w*jej5(l^52`jsa){?JgK9Z{(Cgl^nyRYqRa~Ry zgV}2W$FCQ+5qLvai$Sde8qNwcL)$GAgYsenB==a}sb=Cm`Mxb1T@WgqecUX;u3x9W zL*AV*D9E|E*m5>F=Raj+TKwPvGWg$jLZ95m7uzqVB1fipE$74FXy14N`Ste0wed(jxZLje5$q8xFbo@Z#+_u&^SVCo51#45JCc|11eL#vbrY_W=2`{rs%f#m5Y(A*6NV6m zv=Ix2U??IjKAw>Ws0 z*n3)tJm%!wpxv+l%f#0h8erE(@GRv=G7efKkfQPaS<1v>kr_CY^#ROA8K;&07rnQouf0A_)o;_gd|Wi< z<529Q+oG-ya-NpsyuO~Dj6fzo2Uf6xtut_A^Z~~-dN_0+X(}kFXr%C|Vya8sqs(Ga zliiq4ZzELuY(v$y$=9H#2G4p0dda`JB-TXVgy)uKN;&=LRxc-hL8Zhjeo^3`{>can z{EwO~b!UByvvZC?Z0aWTau%PI@WK0j;0kzymVa;-OiD7?8hVVQl|kHX0U-pi_jeiB zU$_QYW49-`O$J>{^CCUumiGIk4K`OMX+RbY7C>3_aBhV6eqZGP-CSS_?%#{pE7(9 z9iec4m&{$*GC|)#h@N3XRQ5~O8^`2uBY8YeSIaaq6~=z?#eAj%D+C2A6yF0jp=46z zp4@Z2O9Rl`WTt+Cs!GN7`A3>=>mA?MpVb`MDL^q>U_VgtEeNa1$dd;TLZ#-dl1pc< zknaqkP(OuV-Kdcv?s31K-Flb1gPvOg>e&3s|E)woL6q7)0xEScNnsuEK&gfDw$A1y z&&|QAM8-=EZ60ENZu{9ELm(iixCXd%`Q@=Xs2rDBV|LZ&i3RCm@#;fH8a(f{YA#VC ze^^4%Sa3PhmkZLtbTSX9$v{-@XgBrWy5{YaVmLXD*#^Cf%B*(e+XXLvgjA=>J(*63_j6QvGk zBg0%8uzpNL_4|>~xcGQv2IMtp%&K!cdaiu+U}otRtJV&5vn%daGi3oS98}{A2iuQSvZAwuZb-__4Zdpvd^~P)xGe;f;GE_R zSO_AH29$yF5!Q~rk=~}jIOGy&7BY1qJkr+I&NDBK z454wdI)BwJdD<$Ox5aHx#sm20lJm~yo()D(9xW-q6r5w z1(~(PTh?uYH_XzV2_3R40Qc7+v6&T{MccX7!Ic$45pxe4CfYT|5+rz_Aa^ zNuRp`ZUOSxdkfjsQDNP%*dx(!Y(O7|H1oNo^(%5e{V`=~iqpl{yO;R?SMA=YRPGCf zC*T8sVlnA(fM>k)>N}*x!4X#?0f8vE`sCMgASJdKuFFT0dfmYN@{G;|PQBqP69BI! zYetIkfMC)g4MvTfQA2kLiTfqT9da8tI~mAB4X@bQIwqKYSpQ`(U8px`V9Zt-X`SuT zd8hMa2cP{pjExycn;pTe^V?`=b4MwGh{T4ML7R$-itFky`%Nx>V7U0bPKhaUBbYyD z0E9-|x!3@8kg#)M>-lcbc^^lqwz~Q+U0_f;k@67~7A`!Epf#Hv^98lx1L6m0 zD1y4Po15F>&LWC~&!$jiHQ5T_*(6)lAK`>Y6NJra2pI`@k-V~BZ=+>r6)yUX&6 z#m);OL#%^qhvjs8bfdy8g73QfX`fzt`itp+)1@UK8K~=j;jfa9zW@#lY?!P7TtrV( z^W9s)T=zhLNd%}fGzp=~AM(=T0&DtYAc*ic5!>44#+$k--%hRodf4=L=8NTK=f&lE z$+((b&c)PMXR!vc%rqu6cUnsl3~JB?oNSTlr#Mn|UXn_6S}A3Z29hQQE-0NI{8CQ_x~R zHJ#$3;IW5(GB9Xk!A0*ml+kD9&Qw@b$V6fM@PN->tm@Tf_^rR(qKH{*xrbe+uZb7R zaaNlVe&yO!Hmu4a`E+CYyy)5!MpzPifJnHdU}=2lYd8DW)aQSnO5%^Q!PFta06ZAJ z({iOcJ6pR;Nhv3{;z2=KS@zGLH+-5zRNwX-r!R1)@rF^4DtsLq(o9Tb?QwJ*TsRzb zZ;B*9nVKow>;}e>gI0B~*#m%T5%d<$83?yDA(sCP!+aA(Cs};{T5wloI+9nQVfXWMiapA;kLX*O|u z*8HG35hln$lYnK((};`AaU{D+O}Ot%=Hl4cSnbj&xzsPBPtRGZ0VAYi-djka$ur`VwN8LJ_392g9mJx-xY&8}+F-9h0m z%*_{;ZO?fgZA8}Y9Br2@jN(|K(qvyQ43^t#PN=hP6m1MnF2E#n&q9{(`kNO9wxBu2 zMjJN(TD0-!{ck0(lDB=NdkTDwY6OF8eZMB#t>e#bbP+1h&&w^y zo3nicwXI2FxtS_lAa3=7WG?4)O+_=qUAOMVi?+?Xuk4$<5nsMM@Q?%)5&crwucch} z*Fprp$a#$OLi9_KC-umXce;^Z94q&sNpEwK=PL@4m(!=H%M%yA%RZ=M%!CmY z^yoI-vp_lzf~(REkL?S8%pX6c^EMs-4fe;bZeV7VrPW_00-(#xp) zC71q)utjf(+|%|MnT?rSgD26+$G5bQA%U1@cuf&O7ssPW-RaSbZ@DMDjRN|=KJ}yu z7Y1UV$^2T5qdaEz-)P-D;Xf3Dh^T(AI>f)-ot_B`F`=pzY$J={(L$vAsoJIxA ziv}qU4WDFhr5Uln(oEP;JjIE?(t1oV}*KCT2SgMt}Qh)OcnoCOD z%!@1Y{R)N`rrizs7JgdXkg!dI(BO&{H^Qly(K!ML0ZR-~6odSgyQIq}APolcs1?WW zI(^>$d%81utXkrRPgql~U&ZNhE%EM-9Qw4j^B4T z_1RER*0Xx-I_8olDIh+NWnQt5=OYbzoZwL3sK;}2GN$009Xl6W%&SyNb66RsKOF2$ z7<6Nh4dSy>mFZS;o45{6Z}W5Y)+zohDJH!0>&J+G@&4dVqZ;2pf*t}687P|k^L8c|SXw?cE< zkD-N7YXzALv7#AU*P6la<_GE@u>1a%k;|ulA=xtbDL1P*X!f!Sh=vo>qm>(A=jiBO z4B&FUMR4+Li)%0-gyae65rDiV3Ai*h#8@cwHifb+P=0$+o4FKDyHqQHJAKI`Ig89s z(&PMF@*|t+u}HJxva;7}*_*?7E8aW$a&>J&H}>%{!+_xWFSaBU!x(ow31VfKX;AHC zG84Du!q|HSJ-wVvobo=JX$ggsY2m<09j?Dl76su2ECoH8uYtL`>LzBm3oH+2n(??K z-EWJBEhgNpQ}dI@OEO*qriV&%IpD@T=zD@77#``cnHj%v? zQtU6?^}2>`-*73GzhpXAO4b{OeA|*Z6qYbrsBgF59I}hD+dcKU;@@l*2bU zuI#~5E>lOb*vX<5=+A8AC1wqDmJmr~fW8pcWvgFyYBqoe{M-=r+?ioze$2Tpu{(N8 z!KkzwG9f-0s8p<0iP^K?m$mU9gl4i`Fp9J3nn09uM15U0<8uD+TM-Vl$c5gisX@Q$ z0mt?AL_Gr{i&DaO@C$gVKABAn9fCA;hJ?VXS4tU9K}rC*CNG&Rf)S%uDuqEpLNe^b z@J;ztBm z1-J|jV9%*AC3jvG#hkk~?u4m$r|eaAw*h(#g5H=Q|9<9i#;-syj!G0Q93zpJ>KJah z(_|2r&83tg^}(rWOAvQBqhCEB*2S5(J>QK!c7yDnW{sGTM7l zs!fW3ifyxLY*gL_*LK6+^`EWKSiJ)oFd4_lM`HX-;>%Elyv}VZ*Vv4oN_5^_j|*GY z>u8A2DVsg^R6y0=@z@}=!%o5b%NUJ$4pKxM<>1(;w~a(rPoy>*vFC2L87+a)I7qa3 z`a~gDBeSKG9wputg^bZgD3qAKC_^! zC)@#Gi+T6=52b3)`M}lmFZuT=>C8?j7&NV?4kRQNB5XJeYK^(ao}CG$AfQG8mgUCWpy`G=ECD4A-ucV)Y^hWOz2R9h z$7yxsc-)HoKD&YY;{-cd{qyyS`iShjPukd_Rzv?#MIXecMO_1e^ul$=YuUb%0tY%B zSs{I|u71@2WP9T|KHn6-wMum~jwYbbrjNf90-v-m^z*WX%YT+&_>6wlWDlty#U&G7 zj@_Y8gHe*XO#LDfw;decw7GTKEZ{%Mg2V+)xI`Ha$ zV(zyoJ?J)5DmUCGfT%DaaE%f4Kp>Q``B{c??^9FD^dBR@=gK}Zcy4ySvgg!~lBwfI z>OL(;H(gH$4DxM?d^z*(-}ujxSKP9Jbv9#Zd>}s^FOT9Qi0lYR-!dGexg8`?SlHQgB&JZjuAAw%BeloO36${W7 z|MYFgehm$S&*Psw!_^d8GkJ=QxKNj(yRh2m-nba}FyQh>vVFG(GvV&82|D?^;vIG(wgfg0 z3xDKKlt5k&MOHnsm4maK=uNr>gUxh5`7WLSJ2fq~rZUlY)(61|vT~`oBWd(kZ~dQ? z+6ldNHP=knZYBC5bx=T2$ZpW2?dULNq?3=ZFc(pG<=Lx7->vY{mOp=ay}YbI`!n=Ircvv*(0~w2^nVm%jd@PI1=vJ?SY+n>aO}_c zQvJ3HM%9Kozey{!kM>E=1(}9*u0p5IO;A0^bm`CMs zhA4B6yZS^BGl(yluoHP zn5l^#&O94m@GtxSzOybI+f2?-cf@?m(>|KSj_*ASk7Za|kf+A#c)G8U-G{emjyx9C zPWbkI26{L#C?U`@wLl1Qrb>mF42_kRT$wt2>**{X5rdp7c@0tVm`jnmCL{lmf9f2H zf3b$QvQ`E*7;qrm^4b4nkeRQH%S8W03>`ER$_}<#3cbvyn{GAbviICvvwi!V*bVCR zug#OAtqOzwq9(w|*Pbp&s+?@_rFM*nr{`V^+KuYv|%j)nd{o^QH69qikxV@>uR)M1}anb-61A1+#8ut-bw7A$8T!@9ZyeR!6wS&>VvKX)e2Mxk-mMi;#R1S9W zU}BlGn(C;U`YLYjnYT}^bM!c=-S>Q`lk{Kr)91?e5GhbVvWg1U?ri*!-O&N00e;(0 zAM5LMcp7Vpg}CVUS9${C(YxuTc3FHJi;56)pkW*$n&cxh*H;o-5<6$9a7-n7Ch)M! zki){}3-ScbCch@IKsdi48K>^eAM@lGnjY$^aq}5I-vGZgM$eFQB%{ppAn?m{=a>wX zr`!>u|fu!l z0B!m28vra=?_HuAJtel-*%9H{ub{jO(V0h0H68Z0<+mPE7z(zX4 z8c=1&tG6n%2T!weTxZUob{6?D{J;=@+?Rrf{hycqCLEjKI-%l2*#zUhz8FW!>8=Jh zJUd=CAYnG~gpK>2`KY3S+E`TYnn|qm(Q|zZ^sM0_(GW9Rgr)r;$LIqXG0*cqV>;Wq zfcs!fM`CY=2j#M~BC%1VQ)#mO?G84BXo~=wTQ+D(0P!3lAiM{+kbxhc4W65CfE84~ zvzxv7TY8P9(={2`gl0g)r3%F52dk2so@)(79G1VCeA zOrV5zGA=7B5<~AlKf_HMKKmupud-w5KJvG6pOhqjEnC_db)8*dt{gTH@#}eo(}jMv zz$=9_N~;7R1a0`WgdyR0AszXyU3Ud}80R061R)Tay(1I2g_?2kK2ZE48;9IMk9ciJ zHx>XSO*lcNg`Yb7%IzK5rlCN|tK{0Wuc6lj!cv27$V=l!3etZDfEAT)x#MJ*dM9&8n>W+^ z=ipD)g{TB;EK_D9?%WSjRU9pjeRqhZ88s%LV4YLaK){Nh?N9wbhne4RJb{v&hDy!e z`{Jyeyw2Lr3w@H++UN_m^@xw=kWER(10Udd0wG`rGvnr6DbW8y`W zC;kq|MRKi%CfS;}xy2p9R*T-00dD`Hz?S5E*iIBzCHyzK?txe2Q}sqvzr!DQ`XYCQ z=lSIU7=8k{{PhoZ&!MJf8GKlVno~Ee%_bwlm2X^P_*j7i>ZcS^;74CP`9u?~5s&Qv zq!Q0ZYgH8OuODd2?@7U9(Y(q7HcBZ8pdW?OU{M7#&q z{;iJZS>p5{q;IP9N6_r}dmstIp08HiLhVoZSpx6P(VeGRADhY^ZtC5SZ7a4~S)ne3 zRgE+K=Mi`bA>0`Zq`Xb3jUvfiGL5w)9|aSkhpSry5g&jr&s*{Q&-?%22L4H(H&QSzx(gH$ck0?K{(aF}I|&xL-8*W>-6$?-$J;YYoYN7rq!=0ETN1 znpQb%MYbwd5>VOwwtYEkx=KGf=fP1(B`=7*gZ38!s!_2x6%cT-a~Kvb9KL^0DL}HH z(gIXWqRZLuaul({afVHyz<~Il{ZAJwWyaqngC-z-XJHXK=5dfY^p+eo>4sOf-FCg! z&_!#5=ATT&-CC&+BcHsUb2=^K{%`d5G1n_A>;x1U=xJ zxMf?Pux%wo_`fd~0E9%}?Uz)?0><3usgqLy2-?Ln>m#zt1dS804$`9^p{P3MCH!}F zB}g%;dVc+U4yJLaqe-PQ5ywg}iGvu#*J2m0sS778_+Rj z7XE-}enVEP?xY%hE!NLM`{R1eE|Qyg#NezXmanRb0M8DN`K$)#5AX5D%;)Z;Z>hrB z4)gcW!+&mD-P1jmd{+tTjSa%=okReqN00uGz^S9AjVo}qblm_{HUdnnik+Yi5($cm zFE-}c10%NT24%UYlRkm7feyz&mJFohLk5{Gn)2V}{T0TkD$~Z^;d26(mj%0Xe*D1b z79#TXIVZ)x3mU+JCqTtza#*^Wzh2X*$L%A;c~=!UsW|lTp;I>=t6b{7sqWwuI#Xb! zL__B>*gKRXz}w32)*%72BeMZBd{=s`Ehj6BA!ab>zw0@`)Nd1BUpMSfTkvB#kD1`*t#*ZxVwypOkKR4% z)pks(mscNyf)ZT29>H_1Gxe|>s;7>19a$v6l^tz9*+huP=rzvc$e*nvl{5dX(V-fd zR~g?kYL4t{0<&!A;LEjm`SM?P76Se=;W2|7p1+tQp1Vhhq>AP-y<&GJwEX=;Q{YqeIu`}lDGR5W2 z`XNp#^u+(DK(|+1Epd@tIFFt?q%uiRYf_sv=xi0)_TR2VKIlr6Belvn=upLC zI2Kd2Is9Vmrd3w!h>4to9LL*F|CC|Go077|Ede0Hl+xvba_-uutgJr-4MBFeK7QA1 z=u%(`6z4?fZet7m9Zx6YSd;jZa0xHaB zXqtl!$V9s~4p56Bl|mEOb_O7?Rn&uS_?J<74?|va4-kOA>^lMU*31VWPIU@mM)$H% zjnf=&s^}$D6+O4&_uGjqYYL$;T`&+_bY=D1*~vi>zJN$IfUs@c=#vM)@+lnP{cq8f zi-t+W1sp_SL&{%jo9$a|(}0^Mu*4?0oRoA*QQrJFb#WHILLv34JHS!_%AfN`{WD9r}Puj8Fx z(l=k}P={?D%NaUZ%LkczOX1gq7#kM%vCmN(vcwzD9DO8j#0Y|@Y zUjZZgRZ1(s3~U+gmMBYV{odC<)BW*-9E3?vHH#FdouY zk&Jdc02y8A;r7RjuQt3@$2>_qyx_=M9R>CX^mx>7V9~O?WHkH_1w8B=-^ZsDrw{i& zwPF3lf@i1r7-GAqTgiZ9w~sr{d5cyP>M-tWDoW_GZlGykV7qY?W-pD@2XO&JwD;8y z%nOk?mx*U7@=OOI&^xRHxK^D>Uy`Hwtw3vrAKR)6wCcfYBflPOz|Zp3s#Q%3gB=$y zIGLXXj;>zF0lZFDofN|pQ*#fEt{VdAI@3l&w7nX9%wzvb_xQsvb5`5>MdM0vl>49x0wsPsuA z_gEXRv@dXmM>`zT`F#H+-W$CSw`KxDhfP@{mkX@5mCZM8RV+U0$Ljl*v(V&uQv*(Dib~B#;X$494MWY8$;9ym|9h{ynpMY9M>zJ38(50)bHc0F`=F*S{r$E7g2Gw7=gxkRpT{cX*}wJ(|e| z|2^>j(H|z;FZ|OexESQYWzH+^Vi`+cj9G*Df|4h}xG>oB zgR$aj7t_X=jzN%RDv9edGjm5&s^6m1Y=FMC5qe&y!;FU-REP%pgsTq7|_cJLI1^5 zEVVT5`&HBfn^yiC6u-=T!e;u;_=B(Zg%eT_lUP$UKc>72WB>%C(bCkI4AjdY#J8~l z`#j3Xsw2qzf$!RC_V%hJJFo>_110|U4racnuYTRmu4TtjlOnl1weNMzJ0R&-^F~(t zDPLm+_D;(VD;ny#BLY%>sSZ=nK{^=No4D6knDb7B;>kmZaqo>?o(EUiFhv~DaWLuh zY|l>kFO^R>MB2HoRopak)ZCg5KDI2Uecw$l^aPL2D3H>%_a!AiH}0TVUs*1Y4OiJV zHY6S7kd+v-@J7=lM1_!>_eHTwkk61m5AqHN`8}2$7)ohb|2+!czWH+$5Xq*?n3W?- zV8;>IOeIz7*Cj>r5&Ws?6ugDJ&WMTz&Yc~dB4DZST+`S(e?(h+T*61N7&b>?!A)EWqE&IPk3+7C-G>oVRIHF^VsRVIpcNL>RDqOVrv&;k6!w~&bG)CX znd9f!Enl|{a#0IU_Y~AY$2JQ||BxxY9`Lf~9uQv0_o6y$FqJTnj0ShBSGR=e@E#3(w zic)&2l~3^c7}hs~&()clp6&=J%SP}E$5}nPtygUd6<2SdOS18Z_3KaM-8Iu|+Wws! z7{YPQK^$g)$vySFx*;o`$s4tFYj^ z5PErl7SV}j?U_CMmygNJFBcZ4ehluNCcpBnsRCWmmcS*8_I<{+#U z))3|ZWy`~&GHNjkZYGAvea{#6r7z=b6Ukfvfigl+LpI zdN%!n<2Y>&C05EhEPw+2=r#JBMiVHq>6xDWJC=z1gA;q<18?gL%FL3c(&|&v zkZB;}Nn!DBh2W)9IiPW%-H%{Ivxz*k7UjeNZdvbg}rKStQ^jIisyI z-#F`;Bs>yGY;R>3$1E(bXijMGwo3HmW}!=ao3sbiMqQc!d)P z=7H8+m|5D2m0MGZRidybj<(YMiMK}xq^Dkl@hOR;e{3UoKW=|)0`0|)`upc|ec-HW ze_4-iP}o>ptf!&}PYvnPRFO9>o$?!=Au~+-r8MXy->-lk63m96v+Jp0_=+MOjwFG_@!24Wv-*0>Z8r;9Q--dyBgojbnl zo;t~9u28Qg&gJrtUCh*sTQ3zZd@U34Nk~K0ve5*b^GK2@yIfde{M(!Wq!6!PufA-J zd~PEVow@G@MvPr(N8*kW^`h?pH@E<5glq%fGgwu5u&RIhO=c@lnYQ;kHfjHJgOI%b zGg@vLv<7C{F|L2M?nn?Ri~g=DQbW^Y`H**e)~bbs4KHsO9{R>r7=uY-$v7nc0~#0i z=h5|5221ektNRU`=ASb$?o0RIq=cU2xG(avCmkecZ;WS2P|MNK&>%N2M+HCB;7gUe zlDK?eAZUr|`N7X>841Q^+Ro)LA6MC9;3{F?$UNZ%N!kB)*+Jf4`s%73B_-|dBYU;f z>}6OS;b;{qZGoBpVv=9jMuPaC(1H5YQN#upqNG6OrrizMP%>(@qu| zU#)vRM?lbyOG%D<7mVxx;s+wbB8O9BdQyS+36CJx0bih(@WF%3m94cnL;;7bj|;58 zYfJlb(zdfRBeBx$!|`4Miy5rlBP>S<9z?*E-Y zUz6$nt-VO~@4hCaNGC|~&%S1HyUwi}qOybo+H?*0*f8HKLJI`r1sGR(fchA_jnqbY z<%2^B;PSt+{am5(;bUDI+{w~4maN2x5p6KA*i*C>N#qJprLz_p%G?jpXFoAQ~*xq^EkMH+x$E0eu|-IkJ0gwAm&mGewloZOVH@Mw@=*x{#`16 zxYS0l7INmie}0y#83dcPJ!|`9mcs5>3eVcVZi{MQylGUO$J|#sc?WTS4NQt|wD)DL zXdr>7=`aa~tzOL+Du;!Q0XIaKshQ{FtH>+kj8^^bA{(YSA}Y45i?Qjz~+Do{qHrbAa76G zFrcH&WW_*g`F5K2l87|EfT0UtE`DpX{>4bv>ZhJ_C47O!n>QaaOXFd283rK^5Co3YQ(v?wCNurO8L;ei`>UKgO@N zMSa>$9SnO+s4A+{#P;SxglfmeSJdL>XXYK3Gb?s^Bui!QS-!pc=xlVtXqBRqLZyJ$ zb*ktH%U+Lfm3xYvC>Hh-`!~LaQ|hu^JJs!JJ<&kkf`Y^p z1kc6p)7}Vv4v((FkGX(cvSu2i_V{7CwqOGlsy&9q*xqH`QY(Tie&pU>W_uC~ms=P1 z-u`)g=*9_7c7Q(~xCzr>6;8$a}BbTZNYEo}MKTI%*n^ z(0dd`!6v7aK4?`r{?Oe2m3)7!61K@}z;KXeJskJrBsXT8!eEQCU!&>h5oVLzuJ`O z*4b#PMb#1+1x7!(SkzD7_3z})*ZdQmm|fbSDvokU(0g#q+kR1#7_TI}Ft9t<4DFt- zdu3nZm{i@?q z9=nr~q0ds@R~(mKO~PE*pl_FEJ!&EGMf6ua=ZQg>PC7>vN@G zWS`|x4uE*i{yK>-SiBKBCBT=Wwsh?J_C$6L}dAyPod7J%`45 zPPdd_1eVYI4BT#=SLvBjr@NKh-}_Qa7oXVZ|LuK4n!ivQN|+WMnMqF570+7*^BlK$ z>mZc9-P$iSdr+x*R%Q@AOJ4EF2n5Hj!PX9G4nmj9*n1vS4fqrni@N9`ibXlM@eexh zEE^=0&OQ4$G_x+r36%7$#-vN_3hJGA6f6-V%TnZ)I^m!+MQX|NiZazG6hz z%;sgGs#qWSa>6 z6X{TS^Skc{(|}L&FSsOjng`D^OWL73kzJ^?HXOINIX|_~Jq!C0$`e49e={$RtjfOC zS&)7EsI{eV$7>?06Xrb58C33rJRRC3i*$RdwU%a`wBpwi9{6AeTh}PRV}vF3G71&K`BAX=Y4t+iy2EaaY3emsuI&r4~pEI z3}*Z@wwib^Lhhf|B&y5$hz4F%x;b~WBetIQ!nW#oFYJ$ej(Rq6zU?&cT`W!D?39Q> z7pr>%j^0LhwX*Xro2r{0mwOF{*~9&g^xAoA=r{xM(ug(|V8~AGgFRJ{%+ranaem>S z&T!$^^RS&&qOw!nZ3m9jFwGF#Lm7%+|8A}7r@e=A;c{>*HX1Qs25fY#f_C}nU6JL( zy#wM?F;&t0LogwqY?c6}J>21O~Qq$CIFkX9)PNof^nkdls(l8tmr z2&gm&(#-%$cQ;53NW;(r1K%3<-tT+P?>qm=fH1S375BRCE9@CORa5XHC5{c(yssJ# z5-*?=tUXtaOEnkpk7kHCp4XoMNewV!NZ8BDDgt41Q-nrkgwiv&%=_1`M9gvvw(}S z0*Zct&Afw?tcw9vsujKZop7jczq`{8blOYBuXM5d#lq`Nn$xXQQ${a-Hr^W99!mMrhaDE6$=1Q zs>S^3oqKn*yNAO0&)AOg@6{Xc&#g&Kch!%T?IMWI#)eUdtBW3feYf8QX*HYoToA(9 z83lCpT1@r4E1ipXS)K3Cx-6epcf$v3JId_aZcOn*Q|R3cpuY5qtz8!4@I$>n-m5Wr!xOcv zHYe%x?ZvT4e-g8r@3ADqnoma>^2e^Se;vaL`t%$~5A$^X<5hZ)ob97wBBHV~bNEPy ztk=4gZ!P7yLp6&<*RZeNN@t&DuIhAtheF!s!rJI^pc+bZfR77~1N(i3=*>6HZeMqC<$S))S|ltTgVU{YWEu%jGmVnOZ5Zc_CcUA}Nxut;>*+0|m&(BF zPcCy{yv6*wTrcfQ8V0YPe!W6Il?-7o`Dyi2v3hmZ{w%i|s*vm&j+FzxecwdO7jyH$755qDaSEW4B{KjF2qA3qZSego3M0F+@Cc1S0Lfzr zv(DF8$6YKX!QxYCXf{Chog?wxh4ImGzv`|srU)*VCJBDN2li9?nhhYD2ZTQahfyiD z0nUY?F{|(7o7(TH$aAbzZo{(vowxZV4*0t!Y(Ks%-mDa}x714_j+)$;B)%)g&c>>u zr&qmS!yqh~v4YH&0wSWJ}r(-N!N+e;&20!Pl@V7PJ#9GJXx*|=#YE)BBP zgKc-Www$MZLkg>grzTjboJ+dDj_M#lv{7K?>zlJN5$N9j4#peC?aZ^vvf zxL(1%CEbee!V|8Rf{kE)Bh$Xk_s5_lq-2F|QLvol#p%iN8dJ#wJiP6d8(Jv6dK6l` z{C2)K8i63>@+B&!v(L(|95XO2>CLYwZgu5CM3cD3Je7`R&csY3D@bvZjQ5A(COXJi$Gdo+y;4{AfkZoQI z`oMH?{DJ%c$VyFuK{c;xs((HhY<3JMza{x510qByIGOZ;Tn|_le^3RM937yLdS&_J z2mx0SVk~dtvgE6@9$`C!L!cZqH~iZj$S(@Me^0o)EXA0JckF*SI?PS7rkV9dKLD85 zG6IMs>-g70njM_Ws zTE)R3Y78WoZQjhQ)P`Xo{%mjS98sUZ??y(=fasC7iPS6r_CCrGFyuVmO}I=+k-@wI zlFt}`HRd<9SYSi=Nj(x9(F)I0S86E*@BPZY5c2#>q9a1sJ|`z@z&AA-&C4*j#|}>wQ8rwq^Q60N1@J9krtC!K zqYjwEc`$TZBo0W6WaPUYib3Ubvl5WzfTSQ_(nkY9)gRYz=8}a--5x$W3=KEf8+dlh z3i$yE*SAl%>zd>-WrZZxjNU0@KZw?@(yhN`iGlv5lGnoout3>Hz(A)z^${l$%(Trk ze48*EcLd}V0L($?48J~G?f2ub;Z9f_Ds|CBLlxlD;`GLYE$frM{9lHDmuhuk64yH$ z#&!F-w}|PNka;=4DEbmXvETc0P-_8 zw<0`UgUiHS zK*I*0U!l3wz}{mRkmS$?X*`gm2`Tv9O~5zYd=4m6u@q5pZ(7da0DjY)^4c`4H9ga) zLpDlpqI0r$LbXA23XS7#+(CncYd{o$5$|iS(3tf|Jt>=NqO0hasD4V58k$te8ey(! zq{%{7&Y+n|@H@O&NpMhV)C?eZAQ1#uWIa=x`6l>HCJwN}(L*ruxv5z8S_*P^X7@e0 zUPaa?oD@q2yj$e2aRaN8fkT=b?ki)@T21ej==k6Eh#-&r=}LJM^SPwO{_2SdQeeEU zf|fq)`2w*Zz7^VXLh52q{_J!)N|H#>u*xoG*LT+&(Z$~HF`TUt2-H*HXPM7+^En(+P)?Jbk&y!OlJ|^czvzr$slp%r8J^zW zOt6vasU^R1`0n41YhCMU++A&_dHz;M6lz#(n(#%>-hWh!c^q^MF6*=Fb$FXABmoK~JZJcGDqq%1 z0@G*zJ+P5F@-{?%=;cP!s&KVNK!#4N1<6`$wjIibBRmyiZmXwq05UokG01Nr!x8P`UX{jN_rU;PntqmI_qHaVbBZ|K&K z9EaWQSy@?T(T}Zq+NAtZ&mnqkbfrL&x?xMi46_LU;pw2gpJxhYPoNiiBJFt!5(>Ut zIm#P#7w!UdB6EGD*V)z{cd{?Hnwqq7Oipp44CfUPo%mckL}ex?R7(RS@8pr;Df9KQ zQSbkN=?&6(bCL)wv$93>tfKe07^(-imN_+Fb~`fc_iDOXnJq`K*zBJ zqfI}?`-b;Vhk{g6f`Li?-G5aph^^yPv_v^@K2531zU&8urydt_IP)eZ}!J{!`@I2xv8B*2m>L=!kxRZcv6 zKKm$Nvc?q2m3AYmLzXK5vxZ@&ND;N+y0&|id(48Vw}MI2jXNO!&^pGdw>b=Z0R zYdo*<*F^Fg7~+;=8FNN-}o;B9dFGQrQn`({!t>dTyF)N=F@& z;{pp76^=?$KOO6k_~vkn@p|p zx}EpmKKD~Cvp9b2Ao5u+EZK24> zmpL#uIeeew4qVey>ItE4NxY@0AJf394+qZqiSpRN+VFetV-Gq}=^Z==7QibppTuJo z5zz}31f@Og{P znB${U7m*MAt>Mwb!_7)a7ENpBA4R#go)R^j4Q66S9*FnF_0KGsb5U%*2H=~n& zH>eOEYaY!CLTYpnF{oO42L$+}T&N+Ecw)Loa#06{zbwqIeWGld*prVJ%73zT>n!P~siM&^p;*#qwrm~q;mO}n2FhvZ6d!=tK#*<; z+UCa1p>Q_&>|9ttpj<-of-B8L@wrzj zJa`jy?N8fuJzKz?NzxPX%Yg2w{-V^5r2DH=a{q`v@?ZRkDFm*^Z%d&@`u;R^cmRJzW49k zRbzQ-t@Myc@C7nFnz0x@7eB7|eRyQ<+x*j?FR2{%r}((%06S@8tPs9?b-BwbDQVo; z7FsQ)y8iPab`BH1tAwYUm?xTi*4DKgoAKj?Gc1MmNnxT2D*N4fUSVAE{5*#-R$_w; zIc7N-`VkmW;eN!=$wbFZKTKnA$b794k%=+?v|^ErIsmgHl1}TyjEju=f)w^UVuDMr zxD$$0zf$T?mps>{{fqL)-~c-?hmp$!6w*ReSH?wXU1CE+L$y>0gt4FT&wdj0EXkpG z>J9!qT&T!oy95kPagwWnjO;rC&X8MFriOt*iVc^rPn65jS@0TBH<7ue@>E;ijB2e1V~dJQ)-;yKp01?m z#1v$3_gP#8Z+<9Yko3x$mQ24RWsJ=q9TwjEny56zperpAS1YHs)t-mkAZ?0^X zTyqJSb~CO;$AS7nLC-=L-Tu{^ngN4sB_M%g+xDg77w7+qyCa;m*W9eJE9(M`dYW@D z1Am`kB))ap$p!+14fT6*!{#bkRzsNi)3x#-N%Kt>dqbnfA?;;V($^N5+cz)`AKQ>X zi9MK0@kkOs1emrP)+y{tWBr7BoCMts1>pA&WqJZ`gLHCbcv=TcnTvW^l zz~e09*TW6jk~$cIPMH*_pP06&JU|j$_`h`I-j)q?b0{YT79tSgnT8$MAj6E0+LDRg zjx5$|Co)>VcYLf5?#4~ccEY`B8}Bysfjk@}TgF!ERTpG4t$prjGR_@q67*>QfQ5uS z`f0Z~v3tkzP&G&5MLST-`J~Y>fX&{WvHjEI6n_ien8%&(aC<%_W2XQ!_Im*Ay8WKo z?3Oi>3yT2%Ci((R38F698Q4{xxFl%n=;1_8-U#I28JsjqZ7TjE>m@Or&9xTVFnhk8 z+E;8Fc`tE?>WqKmXLAp@7*KEl3DaLeGFY-^_x+#ls$iuO`r+36U?(g7S#d>-AfUAR z(Mve&cXhr%mgT4*Cl`R6&&}|oJzcTafHJcVPTFzB+_u`QYEmsF@O&(n3D`_3?`?q_ zNy;!pacX+Z-`Bb+Q7_ruJkR&S<*`BL{xZ$@)B>w}H+?5^bH?0l_`bfIHMk(FJt%hc?kjQiaI?9~c<0HRidD3XfFu^Nrc6hiZe|M_%;h z`?EV~XI#qKu!!i<+C3cFrGI!mMGYtb0D9o{01M)Oxtb#}h8-s06h$WwfSwuVwP|$4 z%-oNJ$5hlTeX2@m8S|T@gN2Fo`S|o3-Z$P6=KwC(DCHgHx$H2_vm-W8HM3{i>MF@A z#~$kFcH3qI;QVu~kbNmnB)CjcJn`M|&|eIqQa8bf@)hEorTn5;q2L~TsU(wy z+K+92nbRKNv{3v=1zb%;Sc>S>Woi3=%y7q@j{=lV9h$YNPd0!%uG7%J3a|Fo-55 z)Z3c_Ku zKQq_w(#HaiBeebvq}kmEVORW*#NGM;Pm!ilH3LnDnxz2XpA=ZB^rKwpyWfCifDBe0 zo)jNMJRH0A6(P; z>GUt-9)o+D?gSdxzioxCSxDjr*V4r#Ci{=CGw z|L{{8(;%@spf%u~WT7MfZi|?v_hm4Tg^($$KmP(b@B8%MF`^(4^kSg;M8&vzu@@q( zy%5-ZFmQD-pcx8Bv=cZoWeiAP`3%E50msw8kGKh+((2H4dDa9-B_Vq_I4KWJB;d2A zrtE#|)|ypsD{qSq%prJEv*1V_8avuBTPy$ezs~6 z6`YCe(n7K5*Tny%IRZix(BEGty)+2WVbk&no0V#-<w#VLxMLdOQ!{=}&lb>^qTrFnHMR0z?f)#nN!P!b5PDpD z18qA4Kp(A4&%3=Ec8x9|%-WX+=b$Nec#I3cY&7doeV)FE2swcEaieknj(z)&*$g0f0;?oo#efTyw~Whz8$Ta-xut^TfAMsP zjx|jksFRn7#OUrkb;2#T!6yh=oG1Mzg9XZAb(^qh(4MpyBfmxYv!Bc?4!rsc19ynJ z;gxlHoO&$FWvfl{=Nd&_ajYCbt^9mrclABVfa3c>&k& z8sOf>{!66~7CoP^Q-E6mrd$(NU{y*IWUQIWf!vg*0i8S# zUO!X9dlu*463+l`CFAQH+|fyR$4Wy>!J4;WG6ils(Zvl$G<4L_bx5!EYHDIM0E0&9 zWaq{(`_2y55;FZ|8=gY6R?U-tXuQHUf4AC8b94qIMpzk5*6Rb_r7A+{D;G8&)w;)tlb zD?%nv(mZ`)6$GhDjNYAj(SL9lcYCb*%WMbMfD-zx{8#hNG3a3_PAF*Bkl0X1RkqoG z!?O1#1YLwQo{*|MH=9cJZI(l$Pp{Da|7^mzj1xkYIHmpeAb)ivK^7np;_gYu;M4AvV&Tz){Kee z#!{Ob8?b^6p(Y+Tb)-(eXtn!CQ_tO4qc)5vQB*LsWNbVAEv5t@z5hB2U9W3Xe~a+d z9ZIoG3t?WZ0W17P8nZJXdqb#=9cvVNzd;I~V*Hg>S& z86#SurJ&TV^{RUL`qFy)S=>i~4<}#Votboz z#$&OXTRel!$re?D^3A&DIL_EOeZBa`NHOOr{h3o`yT^x{ie#<5|Sr)$pW)-;o`Na3ja;u^;^is*My zVBNTvQmv0h34U95a#R#gf8+6N zZqiu@<@!FYOJs%Zx^wQg^>kqs{|`Z}E4{(G{ucId0thZssaCDo7aerD6TM>A*Q;G9 z6IYx2D@<1LUi?5gk>F`s^sAS{lzlBab4G>L{w=cUpZFRR5s~wl+q$;CquY3PFZu6; z$zX+;=;)ZicDg-!2{M?8s`6Nnqw|kDbyf zV5{-;DM)tVzRR3-+LA^Q7aP0e>kN~jxGk#SSVdALdbjI&z%l4~NaqoOp`zYmh@N(_ zWOccei2c^S%~b-sud_L>YzT5_^W5mzhIULvwQ?$&jGDx&O-$xMLo2y)>5$fUKAh7E zGXCdfT9(W*10b!ZyadL4=y*$2$97N)fwCZtSCU;tQ?HB5W;VTi*LTYI?7g3lCF1oTOh>e4gUtvE|YTpW>=t?aV2n)6_I&oAbR)d-UlU z&Mm7^mV8*b)^u3dO8az4_8(O{YhgW3fcLF#ant;0K!t~b#dpQh-6BMZLo15$fr#e; zGxH#^bRb(SBPPi`8gfm>JnCHBtzVpjj1gUN?3;&AH!ZEe0e=x#2Pz|m$jt^6D9y4ao9?CMNz=JRj1g-0!(K;mE7x~CcybjJ-0>u!3F&!ULT&BopG?Q64_ zv2QH1fIp6w>|=vs2#Suq3lPx7rhR`0c{Zl^rmr@F-y5{4kUO4b!e-@0`Hh@&D9L{8 z6xlJ8QqHUK7OZ*p{Vx`CEXkp4gYoV@Fd;0a(qWSr%R5&5xl&tgj6p0EL)a|AkUaBr-)s&QkIq}_iJgXP5L{6tM~(2oNwDOVuskpAJcbK%L+ z*BzVU>%D@^4LdxJ*IOG%Fx^ zRa9UTFSU_6=$BH4muMA%Abt=rR$SE(Hto-&4}?>(jau{bFpV%l20D~cq^Ts&(RXI` z*AJ5s%!9v4s&=Mn^QE0%C`%hWXmhbp>S{fKk6-$pq=}eeQMiM*t`r|=TSO>`M|yAY zbr0}|M#1)13v|4YAzON5th$isDi9NILYK-*M7>*2qa zp0oVV35nK1&gK;-?&L@e$2+NY>v1P6Ra>g;&rUhJ{1msY*UZyXZV#-59vz=__pQ=S zCD*I3(w%CUs3)`csQGcw!1bH-yr!E)!4V~mKB8LC(;QUoDZj(PQxF8B5GT>qr^i{2{D#IsCH^db79}x;J)7zO@P*~BcrmnX!viSWl zXlEOIH=%cZd|*^PEGG~QOP}2Y-@^X;TM6Db@o6yQgkhK@HFNV16v9SU0-3`q6|pQQ zt5oWF81f7 z#lTVX{aB%%Z7EJGf$qpl91`6IVt2<^ZNy8HnEmDZM8hTx9!@X)Ew-mCn}TVurXlF% z$P0G+4tzmJV$wVxlH(I4?=GR*{cYM>WdDqp55%|1$8|68r~+shUTUH_cg!K+`<3dckN5kLJ`S}f%RfrJ7NgTzo93+C&x@6h_Y#&Gl*utV_FnjcrdT~ zDsuM4LAKvdujVzDd;giGK>rm3_DURy(sWr0?Ef6J`%yQo934X6jk=jdT{U`VtpuKA zdGx1hBKPI;CEm%@Im``{mS?_u=iYr;^t0^GDZeiy#rwB>x#s2@! zHgkX6L$br32nnF|3f@L^M^>97$Mq5Gp?93?4jU@oReF_P1b&bA-q_4?n;@+{p077Z z<1jvLnw9d?p0>4ZcmejtSr&bL&AJKB!QM{OW%Ly67U``b6dc3*dk!{tMS$a)&3p(4 ztav63@nt_J+%GnvRQ}OSeAYyqgBT6TM&LBD9dJn07foU_FK%#O(GoLE%gFL7q z$TTx=c-WrxpB-pct&G%LhU+*?vCzTI`9*(2dmT34#x;6|tc=+^Y>uT7kwQ7GS=Z&) z`hrGX(Z|F8Z#n@`!+@v#*m&xECF1jLXAd|6>+z%$<9qNb765}WXVYBix?h%U}d zdQkLD^w~0SaG#tXo6Wuy2;DSL^>cm-Zt$I?+H zoNae^cG;B}njl9ZWE%s4;y1}bWDE2b9C%Vv5KjU)1~_xJcn*S||DVl-s;^wm%Y|d| z=a+@=1#Ap>*O27axNu$D&qZR@&rQ@l6-;Ii>wG*`TsVq2K6oY;u;NHoz#D-eSrulh<0m^ku>M5^07z&?;s^VbXZ|C| z#~{URgMSMU@=q0QoP*RqP%_h%@AMu2P1Tw3zr-T+&pCk85$=n_ z6%+8aBnbyVL?L|EVS1SlZ);O^#g2|4`0rVQ8K=;Z*xgppqY14yp@i1GAQH^)h*UE4 zOONL{d;F14Ddjv-yU2jo{z;n&XQ}_c1sD4H{Qx8#X|Oipo?(*wvA0e{nuTN^ek1#E z&k~%IeN{Ze-nm0z4<9S2=caM>#Ga5r#?@-8j~m(V@>eVCKS?gH>j0bT|68Lk1K%SO z`-rl0#0^)7hiRCg3mJEOV6ehi<5ziT2@Q38`UkJtO3P}gg-2O&x+wVp*ryp%frbZ zj)NcevRWB}!0NlVuFe%0Ob^Eeru4n)lM5XN5|^qil;Bi_S&e@_&Xa!s()gV8_Psby zJgIm+_r3(ONZuH&?6s`=YQwkMI%0+Aeq#&|DXa7RcFRiM2N4Js_}12`e{CQvF;WVb z`L@GDaMNnkchY>${S~*a@347>$Vo|JqHa<(WL+0UU6^G|ASgp5NVw$NHE@!@1gGnk#r45^}lGK!xLT?GvRIV%5M`_0&AlfP)N<>a!} zB^hV1Qw~EK@Za1Y390sP92}c`knrTma3Nc0OhyMS($D~|W?Xr%bC!8`$IsMcYCo|7 zF=uzyCjpC{xLkIclCUo@N;&zk;&&Puzca-aQEi)DTdtE*uiZTGNj~g$l4(L+4lbyg z%Y7Ks@*LhqcTX!H9aJ5i^T~yAhNa91X=>^huw`tIi|%7*Elkd{=(y@=&S`MDPp2MP zR@Ll(NIe=Wvp#1_U?@qa3u4nbaqTR2Ts71}!e`34$dUzaz2lu1O;G}Q2)X*kPtfPb43e1}D^;97_s&^1;BK3AwN9HK zM(1l;Rq1v%vX%PEi*<}fwtJ3!sc%e}&UmOqKt!~!8kV!vznXOGoEK=DuNDk^EAG*r z5qH}NGU90bI&+v;wbw`+VuL7ycc0`07 z!$Hk1p8ojaoTuweF6^V{6MKVmqbz+wLe6Wes`fpU>_r-`M=2E(6Dm;KLi7lE`vwZm z)-ch%D1CJ%mHvKn$GVO=SYJQ3?&>{pMWz07@J$V>2pyfew_c1^21`LV>*7DBva+ge z(!2THS6L+$;HK(*r ze>-0|$I+cTb-l<@BeOp)V>`5)aLs*BeWy3U4NWK>PtS9^vz5*@=i@D@F2l>vrX-`Z ztHh@2-}EE1l&D&~lT$tofw!^YFB8I5(^?8>;OJUDfWTBEHT58|g_}(={2?b0I&Eb3CM(lgDjn~H%zNR86Y`EM`(=Gfq+DC`|LHVNnWyef<bGA_CtCKN?9%#d{Su$eSv}eooLQz-!GSnyJJgm-?>335lApbC+MlPg_ z97QC}$%Ky@d6v$91a$QzLN(4D-!SttLd2yvPWbGTD}*J%lDn#wPPD%|BvV5K2;OYc zf(te91kpT8sWZ8M$)H=~J-4CXl|X`RwpI_N26AXOc~WuJ1gQN9JR(|obC+F{6!rg)e=2ltAy-!#D8XmjGVyE*L^^47|CvWCL z9C|8}rQ=}G9d<;oH!LyPw1GoF55^=QM+!4W#5Pnc>9AXq1y8`U6H2-(vIQ#Xywgsx zim%*zOk>uhQ+HS(;(aNjaucF* z@F3tYXA?mBc?}H%A3BJbn@JJ-5*3i6dLFfl8{4Nsp7d~dGj2ETtBtQzEBR-z;VPr} zq++TBVdS^Brv2Tv4>G0e9+cF}W=HriN*5zphJ9YG6VII(XGLY&=T#jM35`ovpSsue zzAimNP&ssNK^EeJxfY_w7+`D?^M0zwurIBfC8WV3CJDYJjrV7uBHQN!t;C8~M|y8g zdIVUW;7qn!j@K{+(Lov?c$@f6gj2Pm(82FW zWlDV8(-LJqP$*zN5uTJbPSCoeXOZXSsZ*=1MJNpbx#S5Im}NnV z>*42ZG(iQdCfNMXYh6)wqX`L0zMje2N%5WVgS8diWMompEIns8Jdw~f;m*x3h;v7* z#deK6xxBi&w6-=Y(+P6=dzTUj2iU2|RE=%i?S+~gxzgJ@AMUDIX2kTcdU9?Tj_>}& zPf|Hx5s+3vC?mcvgxul116M?5mpfN(q#T{s1aO|WBDi98KJie6@hZJ)yMsy=dYi^{ zu3W`^Wf5$|*R>;~VGWr*M3bhnFc81`ZJy>+Cip_{kCs*<2r zTtMXx53U$6dd6U5nh4(Q4}YJRKC(?l6%T>8ZclPu@5k#0J+bC2uu|VR!@_eAza#N5 z;>1c}W9_KyetwYzKRj45(9O|?ujw~ML4McSjy7J-wgi^BqrCEAZhf?na14p;DgA=C zyzAm+Z3N)hBI?h%o|%Mgq5D=7#Y~yv0A-jf4#FMNPszqCYX5c*`=qxy<eb zn*yZIgFgQj|Gj{^OTb|UTC~A+!{uLh<0Ja})oSNF48`lkx1Hn&d3bm^DD zx?vvW1si#%U|?XFRZ^nv3u-0@D;t|wIth10+rNw3D;e0kckep4XR5R7>m?2wG8;6W zx@$`St>gHyQ3BQ0D5!(Iz3Z#Y!YI%sgR$}Fw|L{CPT27z4>&nvmqCk1PcJD>D%5Lz zef{~e%D0NjN+l?)-b;2`q_8l%uwEr8gGa65hH!Kg2=88B-*$?ndVdTeKD!BBBdQY8 z(|PaQxidU7QMlPn=|1|tO!w6fQ=E%%;(1YBef@{pCqH3PBAE+?hYu-53Z00n_RuTj zhG#?Qc@k|${7u_54=lTP`F;y|?MfFFZK3SH>zr!cQ) zqgfl+ydXaE^}g`Hv}#7FM7*ln4Z*8$t)SU#Y@^k_tBbC=(FftPej}g@_;EzT^h(R! zCQc`l!{DNVDv%t(s4pNe#yR+MQ$S8G*X_G;Cda3(Cyb8|f1%nV?s}E|ArxJgMgh-= zHi#b+yL$Se6bXZBP^10lJo3un}Zsws>aVvPn&55hiC6^+i!beZW7|t z*1PQt2L!4xV#>()Yjxs0U|zwIi>uk>oF)xo6WulnaGM<*%$)J(`WO}l{}oCxO#mm{ z_RwA?!WK_5qn3dN#r52&@ms+<0~>`uc$B=dGve+$X-(C{hxK<&2mOra0q^qPOE|_X zj4!el8+933;H`I}6#rcI)t7sEsCm{OBeTO_H{0PTA{04l-hA#G?;yMJv8T(GoP?X0 z@_h*PK}k}A7?;4riI;pmZP>%e7VBL?-z&|dB_#SXu9##`?fTrnfHOHcxqp}E4_6*k zPbQuOOK66z3%@_56lAmr(EvcC_1i;R*xyaNXgsp(Z+PY>fN#iUF}3jy{$W?;t(_@5_mgy^ zErHr`qGfD&X*32?;%TOnHQqKA7l)BJPESuyW!$`ZMvxgkDN_X`X&byzEv;p3y8sf= zjloFcaiX+HUOFQT;U57xWo5(pW`jp4q6yOHlwcS>XYNq24T?5Fzw|HWxhj3#G0k&1 z^hSTy$jhZNU)GJwVCPms&PQRj4+M-Px{0_jJ`)LUq<_Vs?bznJc>+09O4=r}HU){L z{QUfB3x-BSg0JS5$LvyBn8_G?`#C`iZq)N3K0(wDQW>;*#g5QL)`~3u{{)TxdY3SR zbB#p=X=D-*f~G0swpOm1*mtT?N$>>=pp(+S{F$963vl94ax0A!U?@6vw0mpf1QH$x zfHHrC+~*pW=O>U+G0>`DvM`ZKt^9H1{8G&%lGJOrjSqkE_f4P+qmJn~5iSrGa%JbfZK# zw9p`)!QU6Tkfu|)iOmspX`V1DNRtF5Bg2Dg;B4I!P9yLxO~`iq4zo}0%gBDjCKg^4HpPO`Y`#@zcg34y0eo_0qYV|DYU zUhAx4z$&<@qz&u3Pu)ssiXqCmm@?D59oJ{!PQQ>KLec}d;!uj$6YfL65uHF z3ovvKvC!>7M)D=Xza9$5;4YCct+W6~#_N}e|9y$@+4O5C$5;2=4*xv&-27g>@-0eg z7_%32^&m4<_9Jn9ihRjv6kIgDJw+z)-S7Ai`5n`%BhyMPN>V0KF%9WFZ&Al2ZvFcz zNxTVSE~|gG2bNqXne4%E;cI2e!p>Jy9ab7o=0`|tLGp7K5RnvbPL2|!Rqr;P7yDf7 zcTATafFJ1wJ;Nn8AE35U`vhpR3;{$N6qs|}30!E>(CAW;)42U~h0=+u58mQV8V8;#@1X?(Nu*;<#%14>u2yN=ygt0I8dLhl;o#Kg+Vy5mz< z1D%rjtd&{Ia5xuCM77b>@Vbpq@>1R>&}@E`1SvTUv;jTvKr*R8vt~eNS}u!%Wdo$1 z4~MoW{FlJh)>Qe?r>G`Wzr^mJHHnoQReLlW^aJlfl<n~-S11pRg%oVXr`j3aZ;-8K`@j5076ArAdwo8s1aJl z$Bx1v)f0VHVpRL}%AxtqM?mF34?7~{CqpD{1M8JL^u)c)ZQEdbHfV%~lup!z@0Cbo z0G))JRVnT^Yr5@w^tj8Mlc;3YzFUu_v~ZjcU+j!wM)v8aJ@FUWSnTv?CbPfsTh+TL z$6rgF9;_SoXa2a$cxf&ksq`<6rhdE%d`tER0>;arLCoj$$0fbb`F8cRe~5hcGVAzx zl4&&PgfN5m^B_!ze;0Puw8M^ZVs>!_rpv^Y!Wn2wst|dIW^mHCHbI|YO+52o^>d8! z0>9!&)M0t4Eo}OsLz4pi^W7SMw9oR$B$YCf*sC$x=AZ zImqqhTmnu_ycM-FB0<%+X)ABG{FMy3NsD`uAe$l&m};mk$=Ke!cpgY#!1C!CXu`L@ zpsMh8&;Cp-zGwEjX`xElWFquY(=^o48~ z8~AF*4ZdWH!ha91s~UQEOFBMUk4hj#u?=SNFuDo3R(^aP7Tv$m4#h8j;eV40Z9&;W zppq$DD{m!l+HoM2zUEZYQGPES*y?yZ#9Nk5e}%C_;K76yK#!mSTase$|GJM$JqgPI z{YTJY0w)C%S-{^d+q2Q#1x^;0cZ6)l`FT40-d2sF0w()W_BR6fc4bTY%H%_7U)lD;~SUMR04R5L8fY!Mc6cdoQkp&N zPiuXFabuvuvZ$ctSj3}V%A~`+=mY}$4V)||WfvrTkrnf;gj{BP^ULn#0c=1vmGdn5 zD~`c?@csc?Lml#b-gyx7u8brC{<19PE)qU>6{GOoM+2fnAi%4--_{PjSE;tg>_@#rpqf zd-HfI_qJ`EwTuai70WzFlUb}X&y)+37O~28cdlb zM2JX+_q-Om+s||Ve(&?%`+fh|`|}BFUDx%!zQcJQ=W!h8Tc(yhgdkkljXJ?=g(d!) zfrlTfxGS5eMvW^wnTEJ>?|0y0EDb~<_8%9J{M}$I8j9jn zX;?KLF8JwYR*xt8^uMl*gchF|N~iNr;W6GIz1%AgJeV!=2I5H$Kr zLK8^qye>oqZK&kZS7{YCEj|=k_tiudZC~oxigC;hq|z_;oH~H)pU8An4p3)Zx||{% zl~wdG}HDsiF2Vx9Qo6SDXg-5ROpeo~EC;9!8D$dbf(M zjXw9*32%i0=_+ZJQ4hOJ_pGAYq;t7}q^fcb1&5P!*we;Y>!?4y&6HP^YsnY9oZKVa z&o{9J<)W^{+>wp+*l{WqS0=S)}711ft~%ey6> z!Zn%Txquu-Cy0a~+FmXzO3F#@=^f<^wL2Cz>Yp+;v&&#E4HJNVSGmiAEyCepCST9`RRG23lb?jredUpy*|Y2Ocdk8o!n-~j zHea@}wDI1To8dBkGM9z9Koz~`=vze!d8arQMzODF;}X5^xM~$!X3@&%&^fcLi5(ZJ zcyqW_>1S5|Bxe6-zUr~Dv5zmp-o;}k=Tg}A>i~b!rIOjf@ayL?ynS1MH><|!1@ElP zy2+%P2;7t;YA`h%C;6%Q?&FbK(nL&<#vuSSm|yJY$oqbR$S*{@nXe)FL`r)s1h9#jTCf}p#&3LnjNMuB*^w_dN=F^>K(V^F7FcA+VfultZlG+f zx9nry;T@H9!7GuGJ4N4HnhN4)_0}wRvX!nGWmR9>n5Uym{!DagLA2}Gy-8m^d6dsfHRT(`qp^x%bwCJf!(y)^^&8N<~UGvS7ZS6X= z*-Uak9XoO?6u7vv5F+L6ka?BIvl%GC8ZLReEW9^3QpYsfoY>&^HbO=CaH31i@L;6| zd%~qJ^3mvuT->1oxeC)4RU$20hd9n*FjMtzfP1Ij=ul1#=LgLE?Cssc)85~|w7U+I zUon94y+fH5t&9m}kaKg60M%kR%VjpsoR8tI3f9lAU4PlNPQk$A6vFl@UDy~2j9@H@ z+*T}HD+eceDml4w5fh4wXUPPVG+3W2ru2lEY{J8$JY~$xgmgj7Bv+TiV zoZM-~6>9@LPw5XKd_d&tH*ei?Y-h1iu&6hA<@T^N&1O|@N!Q6g@+Vmj)$s5-=KZ$= z$VRxC%ZYvMn6H%9zFuU{;XQ}Y`vPg3xD9{tj~ z=7u4c6NCiVM=CY}Xx3iS`xkR5873*tg}73YN1HADEL)B!i*3Fu7iJti*H^Y002`b1 z;lh}d!&i0coF*T{^AI821$fKmgKFkP3~Liw4$#<4M?eN0Dk;pj1+*0b}je zWf$^anP?oJGk(VGJ>6s~MDT5XRcBPCImCK_b39mxaR?1^t0Z(dybL%+5w*cO@byDpu~XKSihvon ze1Yuep6bF1iO_#=juFnS@}Uy%rHhXfJbtC>N;%v&j%tIU@*tA9e;jFZ!Bd$ZerY}c z-V_w*9|8EFqN}vG8be$fQiWDK(fpmPAgAYo61pHS@zv08`&z*($lA=K)lS59R zT4KG^;ta?JA_)T)G+!HWp!gTIK{Q8(ebNV>>fytO*C^>Jllk8sxGRKpr2hdd$Kxwp zaOIgjrxly`wY|O9Gm5aCrT>`k@CIh_A+4a!?ozwQd*sBg`pUkqdfS1BSX z4nCG45$kG@2UxBsXS&0A3lNI6z(e7xeQf|Iey-uG<{sm;gN$i1Xg`cftVaBuGmb0{ z@0%{m6Zkbw9Ig^;`$Ke2AU1rburqwoNKeH%gg#4}F~l`QPcRt>R5)aMzzzz${USi4 zkQBNKn*;(;JWs;=&71Y)OM5=Iu{Q7U+3AmCluP=kbhjOtgU zOfbnBIn;E9553B5gNF16iUyHO!LrQC;>Z5OD|u;}W|8d74&pBW6&N|u;X?WQ>h>X4 zW+_2=q>2BCKNzD%gZxjYFY~mluJC3^1$SlEnfPL0U)wHGzn%zH8eO5T#wOu5%X9Sn zEsi?5bTX6)K*rkoC`~x^!8gJU8Zrbbd$8C-l3Ji{0bv#%Qbzztjihj*^QIln!A-=A ziF7&Sf6R>MZy$>fZ%Cqhx^r>aS|3YS%5#*6i$jVm13Fpw`R`S~gSq@eSH=uy`P1@3Q z(gD$dzJF@*WC%;#KIgB{*x;1et8=1Unl1yY{O#zBU{-WGl+X z`qf2>?@h*`H|qTAx7Igl8$Z&fp3^WFICT#{jQ2EL`ecE5bH3g|<+E!4V9KJp#rcOP ziAdBAnK_C^mB=1pLtg75%|KzkCd=&UO8w);V*mC_w)MGh*EJ+%%~xv5Vxj7&Y%B?{gaX^xW97Ap zkGYDsUDj5vh`*?2p12wHc;*39cY&6myFvI#Vz+|aN91ncts!(~4g0KU%0kA(TzNg$ zT1(?Ibw$VTZY{t1W!2`KDyswXqXGlBJ?3N|Fga?u*BcRm3LKp1Ny)J=)}3cxCnKyf z=xa#a;C#%b(VGKDz06v63fr5}PoElj*N1o;c{k>GFSK)S$f=ZTA5tlQnR{*V{F>tW zcU7{-2WlsXi!rOimsb0$^f)=4>40u4v^M~Q4N{T5U566PpkjKT<}6 zd3&U&K9>(36JtA8Z9=K`nw@j*r;9|(NU?)J%B!;qd*T=t8Gd36+jv%82FuCU6~pczM9)iQZG@KCCUc8 zvsRtWJ05|e*WKar9%7Nb(WUb?2Vy95vz*xXmI+z~duOL_&HC8zqd7?#A-M{)b-fX} zPWGdX{BqA6B1pCz^NC@ir!SM&puHG6I(+}UPYC0I&$%CZ3$APM$%uFiRvxN4-Spy z9EXDuInu(@N!2w{(Q?F3pr)(-mNeB!35VlDF5>}YpgxIy;(KB*Q1_CL?Y?Y%HgD*{1Rr(*mpPMyuw*A_Cl+D{=dGM_5o6z+oi!PUN>oCNr4S+FOZ8U zq1K>#@H9VqSMY2)*{^Za?#mm={Sqs!-)-w!e+-beeQf||4#BynIJZNl>Qt4cUzn>v zcRbu`5^0-d(6Laj8$5I?5W>N0M<0L);eu24s-0 zkdkSNsubm**hqzXxko`Mz6WntywZ47qP2r@_<$6dRf`$g82_WDB$&)fidDRWBJpI&^4FeIt=^pTQ6((pUL2Pj;7L+# z7IC=0RM3xEdVHQ|XGmYt8+m+ox%RSzKoDII~V}1dmip>LqemJ^a4)uvd zQ4tw}aCaTbonDn9*)xr<`6vqO+}&%mG}w|v2|Y`j9Ff5h5X44gr>wgxQc!72|lx0^?#ip}PZs&ZC4 z?Y`RP@Eu2-tc&GA9;kd*pPs}AfgPWqxE@F)HoPS(bFIHi{`;!*cQE8mu z*R!+M>Fqm*UZ>jUfm9ztmgo9Lw2FAikZtty1t?QYR8WSP)S=L+uOn4>=&LIxog%Sz zSy}-^r5+Q@;g#3C8cMj~-Ki#t4om3m;aOt#4doOG_o%mucPcopRz47OSyn^p@F zx8A$nVVvUN!Gj;8rk2e368y5F?JSKi-B;{F1QkzI+#y^c9gn)i@GA$%KL0FCOSC__ zQ09Xp+s*ph1QTyJCnnor^j!fRIfAHJ(jDdVk1`N z^%|ojn49^j$GJ=85-wItV&@zf-r8QSSM*sp1ZvuTqj6-_!WJG(ELRQv&J*!%hcB;h zGk!FdY6Y*v%b*kidmQNQ1`-*xY`uP*UbL22LiZ|GM}DoEVqLWA+1Yc$QT_p~Ou5;E z0T0Vz>MJtZ)-%-Y{qNIA`rJ;FkDowaMGl!f28Ee23CQIhIhr0Qw+~H9thLa6An?@!ww>DPkG?_$R4*Rg=qG$etJ#UCh(@Z(KJ8p)$-ag-3UkE&`uXdKduBiPz zEN7-DkOpm%iTOx^L@lawsqJNub#2hl(k-3YN`~R;{cYv7I^tAp0%>$7jvp^E*nS2* zp|@WS`&4?=6C5t-#mEfF4rqGkNw!1fzw{V~Y#3Nc#UBXfZ>uhfXHapf7@)7C5o2Rq z#dKeKU{%dr_T;qQwf0&T<;Xg@M@H{6@66s~`=x6i(stNdR--DlzEQ$6|q{@5E zu2B)!Lxwy7g^B8j+W~kfY?ioQ>*OhR=agF3{73%LRP#6P1Ymf{%YOO_YYc?Zuii_0 zclhrEp3Uv1!^3#D#9CVl{g}OGgNxSG%Vh>Dna9Ptb*(?-{6fxHrmRoC!asEC+b z@E8ZO_4lobm{%;jjZcRj8^wyaHq-};^_5m{NSEUWoh~)pmHVmf{icTxeUeB< zwy~hAq58+7Mt4>RK}R}Xy#*nR#5)A;kg>8h9*7if)9^+g&cxfeh`tT1&kd-KK1!gX zU@{5}xa^vyYWgkj^*vHPJ)|c=penp3t5KwbC8#NGCv?blXW{H~rAo}>b$VjkpAKbt+YFypHQ;-pN!QWreQTCcXNl)a?I3w|_Bx2oQ? z%3T}mDKSFUQ6<#|13`j&uXbYGnCT$LolnP5+D zH<$9{JYg-*`1S7lNowPljmtCvxJG{|Mc)4F;H|Ri=DbH#^#lrMl{+) zm;KXP$o7)%(t_uq)+>!t7d?~nn#EE0%W`-F9wj{NcWOvO%13~Q&T~JrKV6tbteluL zY1(~K-62z+GAFe<=-c@rOunmae5n2VKsD-z^3 zKXvxo3ykzsI+$AYl>73(FRB_yr>`E4eL)(W6zV-~Y(^d#7+Tmbaf$Bj7#GI!HNnr) zLoUPYRj$hSCANbsTl`YXd0q)W;i2}F3(E>_Qqc`wC0JMdJ8N?6tVw~y^UcAgh8pbG z%KNN)vm*0-le#BeG6qi($*WH3R}cSDC2MQ~y)IpHojKSCNGdvJeda9U_MB6z6+G{8 zES~VF-{T4XTN~jqL`Fi<@P%a8W$bp-qbRx@P*209k6)*xIhdO&Pi91I+FdR1p1GsD z$A-n;Zp|_e2^XP`w>7#nu7Szs3AO={%L(sbMbrz)bZgJl_=BsV`DFYbo5Yjcr!vCZ zJ5xOdxjs4U6iI;3)(3^aC-;F-&i3TfH_W>bzZ2P-C;&|20VhA5;{dB}|Nghj!H4E8 zk<@g@4w>z5zm9$TC0PiKMB(tESZb(B#{U6dw*C7QqTL4}cDQgnEqqA91h@zpfhhXz z-w#t9EEaA+_FX;r&}aobrx437hhNB+&@v3Q&$EV!H-r(lzgX&0oipzf;f&-iH?$}IL9qiBEnyCQY z4|8m;WGDlQwdId_{)T+g{ZGFZvB>yWq(gd2#2y1u0s9Lv$h-Y-gAL~Gx~#}x4{}ApezW{S ztjN3j-+i6u@@IDdgG(N?Kmcw{nnLy@SoN6AmAn5h?%z|!`4OZ!j$#1I#HSpHtL4e`_Afo_E39b#u zW2%_`z9>joG4zEMVh4zCwB=e8QX!x~mOkd#X3=~5_F_$pq@CgqH0|%PCgC*U>91xZ zfgWx$8;sPpXc(S0!ll8UDY*T=mk0JEkTmHs@XmDHkoQFS+v>I_-TW^Oh|rDNtTmWw z4s)Gw$2tP##&&?Co2D_vOK=>Y!P#SW2!rLx_SV5n`tp!Xiw|&TKp^?YGjD|V${9qbqdXHG~- z2z^J{wZ*?utZd+?uo-h4&dl4?^sdaogA*Q~DyahQegE!p?@nqekG+?p zv-3MI0zHmz`pXw5ZfNuvd*}9TumSh=_3eK=gyO7MhlW1uA6Xpgayq-ZK4-L|&6I1! zM{6{D)siz~)WaBbt*q|n`H*!L0jIe0d&tzY?Ckfx9S7M2+oR11^MC#ISBLKa%bv#w zIT$+blo<2{fS^J`xY(hNY=m}_XUX!EEZK@sYDFyW)CgVekD{W(_N`ijz$|e?P|ePF zCXv^4s@-`yWLv$mCwV?Z7lh39kGb=#2ssrt(q*p!Id#7xbn5={Mi7aJRrJ|=JsO6EtnxbRZgU(G5XLL*33ca6C~W| zuify1FNY#881W#Jm5@ZfF9RW8kh~M>;c(g*TtO;`ACP(?)DMG{z`2JvlNz~NGIB5L z+6224*pZPVjYgLN*+$@~WryQ)oj|X*6F!Tc0BahmFGy{4UJbsb&<}oBv51rkk#f2L z@|cqZcsL-ML*a14h{$fgNgUOka7WhmW4AgrDABai@`pdzB#lCa3jsIwq}k&D3Qd|t zzT3&tS_f`TCk=}DmSM=(0UcoLI@yRr2$V&seLHAmvR9DDDyQn;F@Q|bUJ!8~@+PfW zpk+XaxJg%aWi)OiojnMWvdvSCr3kns1&e;y)^J@`bk#1>8PL-(Kh_sl@FrY-k{lJ4 zR9NhGQBWLac_FibfXGa?27N$DB#!V2D77t=95+9BLdq8l^XNf7)H z0B`u$7UwgZ2=)=+N8|4khMY8p3V_8Tm7~~j3gJU>{mthVe&Fan_S{)qL2+wcUt(v?? z+u7Dt( zv5@sk5!g#Aht8cl*8r66g|7v8D~eIdmR+Ni`Z}hd2kOvVadE3i71d`WcLbS6zh;_t z0Da`y%!Hg?))mUH13EHW;K;1^R=x7Hp)k8P`+-5FXk zcolap)U^~;Hx2GQsBCuU3j?LTZX>ny<{o16glpxBJk|+M#ud6$dbHZyPA(OCX; zg{~uzUavn=nsS4&lPWMbXn{YaA~m9f@7T?RTqX~TSk{iP1lC2yI?JvfD%YFnxb}%n zQaV=b4c3)U>2`eu$Y{=^#<8#* zk`US6gaBhb_6wpaHC)OGnyb-#EeKnlwnb$D;d_Kkd%<94&22fcl@i(<$}qmhol6|Z z3e4t(wUhyCX^&u+s=SAhh48VGiRcwqD>Q@t%}(fbvRho-pcf|8TSd6e^N3Ou=jSOC z@*6)Ni0Ye=dmsy`a8jIYy#16*nub@1Biix)C6$}KMOU7zhbz3Qr$)_0VD6eJjPnzH zVr89#<`1=oaNmrXf<4$vB}{4JuM)Fy zN^*EPoSPwx01)lIIYK<#-N_64*vCagrl*q*k@BDq@Fmo0t02yJ(g(y)sl>tZ(_yp< zTnDUzUS|%GZ2J#@pex;P>K$cH5G`klf!(v|9MvC|C~D5-iS!b(jvCutEn=rYh1N(b zYxyRGJ|kvYAEL>}nku*}sMw3P7HzDRRI+?_<*EUsppkn)6HhxR;bKJ{EuD8S47M8) zC;RIG0}$l}=23Zp^fBOa95ue(?1fc;5{-POH*Sq-a7EIuB0A!l_GYX&Ava+@hAwAK zxeGKv(wZo}&}2H4t>RUo8&mpqN2K{#|FtW9mxwV`^MUhf7G^Ib7rfsp&Qyr4+#vTD z#AtE4A1t*@38U{5F~5b=cm(>V&uQ7U)^E`>b1<@+7M5toyWUith@EbUv< zx8Ns8Bx2k$c!+nc=HySQR;^F3x{$xLIa0E;3!%1D16K}`V14%_h}B=(>idHBieI-f zU1SMDy1O+q!dAh3>MQCHp8UbrHW+1pC*dPCe;Y)f!l5}s=bzKXign0gjZR^&W=tP{ ztDiF~#g^CxP1S%M7KJi-Gp(6_6m(}3I9koByvmt7^a{)Mh zAqqmriLM6=%Bhgp8s0_t3Lb<|H|5D`JM&}PPPH+&Yd^=8hC6!o~Tzx0fj%#+eAf1=O+c~GY z>ESI{0mw#g9`iuXy8HACm|$Es2=Hc<$RZ;qQao~!2)h&?_$=rynxqyA`a;tS)dU(s z?Kukw!NN^cgGwT=5bR2_hy?mkD}|>-SJAe}mn>-LUpbr%QZeIqKx;C9pNikw z-=l;2e1b6vxSmhW(-{5FgVT6mS{ef3e>F%3iMiKeP|bK-py>hkV$<_?rd$@u@Wwmf zTCJa6O@MnC&`)4GgjsST1tIWT`KOlyk_V6+GcF^ltp7qy69Okg02SP9?ju(u8I*m| zeobSWj`*zQ5W@LEL&=B8Yy!Ap6jeb+vz0fHB7UG|axEZUsw#@eh;%@z!MmUg=324; z;_dxbjtBOmo5M*Uxq-nk{_);sv9?kDd5B{nDTmlyI21;Suz+xyJ&?J%G>Em z7%)`Zfabl7UPL&HItdW|w{7fci&X!l$>fmpJQVXm#%}uM4jB+keFwcCvo6#T_g|oK z)mOio2(nykBWl#w+5lFciDX&HBD?ft{d6jStp|xfp9Y{JZ#uw8lubTagN5ltO31G~zrV_^-)Dj|e!WTx_gNN+1&C(TY^ z%QCEhEweun)P~E{jtFO$w4dBKMnst6NW7^8BuNy13qYA#NLKux58GZ6?>wg{6m!Sc zz#o{sz%(UsOsTesn8P53J#V|t2Z)w^o$|8un9vEGq1 z;6!zB%p;K2CgjM&^|zzK3OOp~ARw`ndg;I62b#4eUwuKn?{hznioX_8{?TACnq;3E z2QiR|$iYMnM92l%sGY!MB}0eCp^DHa;qw3INQ^$>X9Li|-j9e0u}{tsP;*-=qEeBGWG1u4VL|* z(=P{ZGb$~sM1z>v;4oRj9@umNWkZ07F4g30P5S=;!(l!~Ogfcc0}1Fdg1D+A$WKYa zhBc}R0Oz6n8{+|(u+5*ZWg-r?A{4@xFy|%6_wC=G@T_#@$okmXb5{cmr-i1pxqTQ+sez?A3x z|35u$Zk`^&oN8w6W}YzIyAF6(!qoG^s9Xm^GPF=~)jE+3!lZiuYma(UiwMgnRwMFs z9{*L>A7TE>)ITN+{P#h-yb^62knIB4v#dTF?^xX0cyb$J2CTkYX!Ph}M zbR0+nG=%c$Who8JChz*DDq9iKVbnpp|*?;)OntKo^l< zWQES6yuv-@O^MJMqU$1LuR6UCGsz*03*$ZoNr|Vd+mDp6%h=s*i4aSAh5N|fpg!0s z+@J+QE$~uGc?FY8XBpHBwk(^hNJQ5T78(#u0LL6&EkmaH1)Rf;$55qVPmV3G2v1A2 zwYrdp2-NvMMalKIJ#ieQAy35IQ}R9YsVV8fOQnbu!$*%EotLD>Ue-%*8l_dPd$WTh zBRo9+=s1~5QDT7g@q3x!#dMrHyn$TCAY=`Z{e;TkOu~J@UOgp%3E;*b8p?Ow-B#{J zB_i7R<`(_UPS4Y-L6_sg=JErnJj~w8cW}k*cW7HOM~@3n_OrlH6$Dawxwc1>ulNd3 zaC8wo17$y*_o=`uTNt+u@_dB=QZ}~0tHm8bWHP>`6D3>X4xU3N&!9z_S?%0eC2Y$i7K-c2#-D3`V?%tKeW3 zNc2n&XkbWjbh$jZkyt(?!bRCN`K4KI0A@=?^#$nQBjvyDBRWbpVyJWUXy=pjvKRX@ zxswu!ReJ;S`T7w;px@8lfywE%SpCr}!Awon)1a7N(c{*8j$mc=bi&hsuPv8}T0qKf ztwygMwqJe^?@g@m+Z>hxdOc#CQn1q7jm4=CFOK#12e{5fMj&{( zGh!iT0qUFLsZ2(Bl|rpULuQ9dt`Avf$&n)f+tXJLHiaGwKjY(>eHlI4Pw544)10wm zi9#+vqOfw~$vEvzT;qB8f}%1$1F=coMN0^`$0%iEWoxZYwPI-sJ-T#9-4S%eJaRD+ zO*9bT6`SmbB(#an$Yj-%T^&u?st&juk1aKG4`10+e^oKpbhIl@;6t+%{;(s!(_?(Tteyfh%>lwXHq? z=}^5W`x}bWM3G8_q9^Hxk7u7J07Ty-N(Q6;`1WJAFqg^(J+93!UaMtNVYa^`)TG#| zifYXp&;+Vc$`Ra(P%R0(>Cp7$FLVEX2^ zXo7QZK}bI?EbM`H%w$Qqs>4sE{(z~v^Lz`sr(`jPT5N(w8)A)bGQwFH*y_x>3c;B| zOe8DkJ9O<`K;^YzU50GgWX^)P>;npZ8tWuQSMz80M z(=O2GKLy>c-W~Qvu`fP}0YAlSC2(GnC69$wGtUWg0>u3!KlcNLxKg%8;7@;Up`M9EgCuzZlXQ2@kuZRQMY9HvNtIBt?`=~_2{%(kk8=hmfrLE4J;f@BMaJ_@ zyOVd0U2}3VieA|5j`-Qq@xzrguri(DKNA;B*KiHi5}jbwRj$Fb@v7rPz5}=Dv6gk`&Z>G%50st#5Cix(I%Q#x2#pNbQoN|8 z-Onff$igh&=Iz0i!%l@I9|Va}_qiWfm@VQWzX6mDHxp72a`G+ayrGBK3LQYFnvqtp zg#toR6$%TLB7^gX!i0dhPA9yp^L08n8%2w1*jrCp2#d1`n!}=PLK5~QFi=CnRsiVC z457CQ)bBw{g&f0%2*Kbwh}7>}J~t;i>LBtp8si2HDkrAEpuXI5!QIwq(_5BLP=alV zW9ufGLE-)KUrQdGYP9s0&Pta|tJ)iO>!2`w-fL8UpU;=vy3ua?w0BrO8kMSNg8c!a z0y*!Pf_FQylNSR-3dpw!CBR0oCw6Ei-4(rL2GJ0l(3W&wc{LFSRfy^2pMjz)7mV!SH#{_BAp=NB zp=+K3V7smNREq=KCR3)Q>1Zb~qt(GKl9Ti5Tj6Z((B_1kOx`11Vqys}dHLeV0qzA~ zSYSZ%XaCSX}_9aBZj6gl}BF{87j>O^+xw^&@{QuZXJZ#xNFbxg3OAf*EZ%7WFDpnyUO&+&mI zMOJSb({L~e_RdtXtf;0=4?lDnTS+IpM~g_neh84ZMTGVDB)aT5z*`|0QQTj+Pc^J3 zlvuL1T&H{+ch<*9p4N;o-7V3eW|7k);$*0V4}-9XyKqM&x1O6(Ex_?Iak5zRARq41>d`GChXwmf>jp#>(; z$l{YiR*q^>J$in$kqX)OBE_?7S#RP1={5_P((+Zg8Isp>GEJ_0?#n)LvL6*)A*EkA z`s|_Ti=q~ptlO&iMyO<~dRcL~zdP0T!D#8zp*ofvbe%+F%SrzVX$}qSXw6A4Ron7O zc6_UObV6ZW)uFGG-85rC`Pj9T@k)lqIt7EB{$h-DqvY0mElSlkS4W|C%l$4!%1!1-~B?oL`qlW=t z9k?TAz7O$XV!c4t-x6erhIVBJTKX-Lr;kFnS$Vr*o}@bx09a&L3C3ZsZll%^n_~91 z6}t8CiQ5g&oB{Of1B(6&(ZaIsqy<=v>Bt+Q=ovD!ysQqmlQ+_Nxrny$0djqYSOFL` z#1Vi%heNgUr8qtX%im@hsUh#7n+{^lbT`@__C^F8E4n(SAXm;n!2yHhkNj=71d86T zh$F&?aRfBZzelRg9<|6Hbp47{>oXP({ld0}tRY9+B(Eqf<^cPn@5C^qO|}Tn5|s1g z!ZR)m8aFr+Gc0>UCORJ@c~E{rC0Ur*_vZE4Dn_J)d8XQH;^Ccq%b+T&F!}Xa?+le7 zQAaZ}`EXXaDwz&$!sfW*ZIKxrXqi`V_(-tIh?F=C6%v`=1JSTchk&>kv{04KrsNns z2mNKlRusw?MbwBDughPkpiPlX9@?#p@J5OtUl;{-4uDs$$)b>dDUI`QE) z|97Ba6R2R}d37fcUGvbEyO$b|0!W~Mc9FnI2l;C-4bl$?+~z3&=u@8Sh9$Ph&Riw3 zKap>U*s?U>f7Sx9XJ6swR7LNFY9IXntz;9J2 zvx!hpCb$1rWKi`Gl5hVGi=>lC0VH5V@AquRQwEm`1qRrq0Pg{3*x$1m{*GUC8g?j1 zU|IjKjSR?a0V<4n)C_jC%skND1t6#Rcb%lc(FJ)5h!GcrUcMG$$UFm@@XdVcp8;Ny z$6ilEXnP-R0C){;41~4-{Qn5;2GqMgC?oF$D4LsAD(qku@gG-e6>GcMGK;iO`A5*G z4N~S@j*gY6I0D%};%Nye?>rx%HhhEfYHF1GLA-lx`V2^s)&CiyO^qBk$Go)fA^YGS zk|@@(spl(5>=hhH)jWvck;e`XY=E*6Kft5)n12M1wp4gYBdB>NhFN&CLIsUCNJB|A z-VEVMih;UL=D!5RJ4t=R{$n^hWs{e-`=P?FcEJg{`OG18=`p(bc41V$$!AKj*K?4u>1NNFQQ`CqgBiGbn1VZ{}I4LN2 z{*}ut|EW~i0b2b_iPNO#{5_qaCH$gi#eV%q;4&>)HQQYs^jv7HL^-mygKiy!o{^}n zt`ItVNbXXIEyF*1h1dXKPQlg)*8i|E&4E!SEOY3#2>%c|y{Sq&5q}baLiI_SS|k^M zZh`-3X_F@cmh%f@ReJtd8Az8!Q~MF9-qz~J+bY;aM{CvK7F^?h(RqJevj_@>|KC6! z)OEu}koX{-O2q=aP~7K+HOIm3@&beg5My@f1gp3&&w{%hIn)wCShDVgK}9`TM<{MN zarNU0A5c+VvThKA-`zu?@~ZIs`j>$g%(S3n*92QcOAWINRu|0nu8Ap!f*T-Gh#eK& zEU`i13x1A=BH}HQVGJDv+%`4uY;pVdK-1DfmyC3urs|LHf;sri^{z?&>2Z#F4135G zGRp}Z4mY(D_TbGecW+M%GlM?!a1}C-HXOii=crqx@s36o*&2MmU^fj~8Md>)MJx2n zV3Ze|*(egFRaV~*#ojyH>RNSz9?CXW^*|SY@9}P}`)NR`}En1eo9A@O-hj1$Lb5Af^alcw0NW^YWX-wE|0Y zS#dobm#Ge~huMNAw(X)7^+-enSV>nb3i{%C&2O?~b27-_&wrbrPq~9jsn{cp8`*)% zE_?p`_!16s!C2X9?1DEiGy*R3rx5*|AQdF5%?3}ur22Foe#lh0+l1(R(N7YBjsM={) zZ{hMqKD1)%)&2ZR{aUoQGWV}mog{9-j+q$X6mItTL@Y~&DaV_T6Y;1z)S)0Cz~2px zEvz_1IzBrv=>hIk(YrT$>5jEl`RpuuFQGhNu0txf{}3!btF6tCuFEB=2YGgnR{WP`%mzB?rl&lV44o+NJa#LvV;LA;*GsTM!3CTm&>A4@P*d;`nJ{|pAi<6FB%xf-ng-l z0FMhKNUTU#>TkzL*VZwTl?p0ZXv_NhF%lj`Iz~t|?#r`OXKc%b>hv@uy9o*BJX1{2 z{ccr4Ys+&Z^Hsd-hN<2&hhC2{_2=Z(#2g$zJuaY{!4=Pnn3!9o^X85(kJ>hj(9eDUD}2rpV2KVd#fnQ#?PX^Eb0+U7(}f4EK# zZ|*VuB#%BL3D=}ZU`XM5Y>;*vOsXfOB38w+kau5CS1W}qK`6lGpuLf`25hq(K)ymS zG7|G-w-aY3V+O5A4?xiQcutR>Q9E^tGcq*pu+JPs7*sd!xH=U-Cj>|Za_37Ul)@9# zF2wY&#hKWw+N!P>K0n+eoI`b){pMw7+%FdRkF*Nd}^O?$XJ# zja#9Y8gkyyAuKkD-csLUG3n90U)HU&*_S*?D4C&>jy%wEvr{V!l>qaDWU%suO;(+| z(n`h?-~~yP0;%PsxWmg&hs1f_X5AH%$JjUt&;v*|Os)p55`Xqph%V6R8uQh5X>b5W zR!tcUO|MTVwIYmwE>=FN#HZpChbrdRhX(4fIW?5kY)-BrqAg%v?Qu0i+7vA6zgW^y z>uDXc$T>$4y_iKVE5RR3kLRrv69-~vM_n+-cc85yg;jJxENp8g=xp4C_C?oFXM@TX z>cFWIzQLX$KClna&H&CQKptoy)G`1G&T8HoP^$oS2{*V5iC9QZITmuu3mM$J8uob# z(rd}L=^}ii-GZ6f%3@)s;&Dt@0^>LTR#tnUKT>rCvIR@Ip973>6RD)gaOR33(bX;v zUe+2wUMLxmseI-_Skac9nn_6|iPvb0xq+#;qR7=-NL`#`h3ZLUvL~UHm`{LY4V`AJ zeBM30^W3-qZ)TVVPzkrwZs4ggeu&QHY4ytV?s73&s*ixv_ZOoGxKn`j|6xg;-5O1&cz>=hLKHTUo8U`;=+ZbMpS<`bx{iqRd>+M|>V zK{*6^JdY(>+59F>!edY;DBu%pXjn%ZsmXofnbajl;VWV2MolHMju6pNO6PJ-i$RwF zn|h$15lYHQBwJuWbKLPF2vR&8#FWzYm8rs)G#vE#(j~`Y zT$@gwRXq!kz0yP@6%^v`8F@f`F{K-`VfKmAdT&drTGA_y2-n|;e+l89EkWF;76a!i z@|bE_V6!VSfg1TYvs-r>=m1}ww@!^3e*GBfuk9ghF9uF7M~`@YA`6e%R8k>AO9=eB zO-S7kQu<85-b_eKiU;@-&W9{0)MhBLSMk(lH#p(|jHz%T{{kj=6&gJ>GGb0-KwCD4 z`X>pgX@2ECzEJ&fGoHa8YOdyPAtxa=K=uy-VtE3q7gQAaoVQ>bFmwY62s-Rugc766 z8e5s!4jGdC;}Filk`Czpp#0z!Ck1h|GJC}Dpy+}3wL_*p;>miTD-VQk_JWoa9IJYO z4MA;jo1`bF7W-yg0pgo|;)UNKW)c=hXkh7HEfLfQr-U0?Henr$c2;=eT!d<{9f59# zb1e5h3+Y3GrnU*{{T|Kx3BYU{2$KEYrp`Dq5G3I>Z<_>`NnrEz{^uydUEAfLMeT?v zD0(Cj4F$Cm!d7eC4|=DB1VL%Lrr0oEKm2T0cv;1A((T(?1YxQ+ktmgs^FE}jsPurM zfIz=P<^f5m5g24&Eja%}gAS3%Yl zPIx3!tdc;YBK+JMP8TpEG%aw76G`(qzCO#iVJJVWMd!a-=bzPS5Ui5tIcF`}qOh_0 zt;dS9wVL}KEOyR zh63j~#M;=bcK0+^@B!f6v67(t0}J85%DurR45g`+r zw!ZNAnv*KGMqn;W+D1F?THKQEGO$<$`puv{)D3|ax)$0@cSO1})c_~=$JMHT%zbu^atEbCm zbhZ~V)ZbVvTR+7u_kOx#HGsrJBY|iCl5Zi*6Jc@S;D&G`EJf_~-;>*lzrj{Q`!`um zN1mUD)P^F1ROn)JlVG+E+Tw3XBe~foXGH7^Q2sl0(nDPvHA~4E_<9E2OSqM|1lug{ z@??Aks>r2)jZWpY&P*@zPep4^{K2$`hm)VS{M!e9pkfm|I5Z+?{C3biD;*x$#V-QN zB3eYrac8TDyml3D)jo&H<3f8xQq+T%F|W)C|Dz`br*4ozg69`Z&cHZORYEn-{4BV| z1q^3FAbzhkRv4H zQ^b-;05ZY~S)Io5%$U5~nunswP#i)UTOby>AD+0w35x)g|MtB?>2gtb^SzSzLtEB# ztDPU(omlKKv)uebuQr_{vyze~;p8IoowVuzF)Ki!=?JA1=&tuc=#_)eae!5gbV&hz zdit@xA)R3NmJ~&F(++lun^ih|zIRYjOomP8xlJu1^JLDji79CYy6?hJU%IQU4~^s_ z89E+;!f!zrXYK8+uAuuUxl~E{+d}=tT8;n`qMn6>R?%bvLZB4JKueupREaszyF%{W01Yny;f-(F zB#3SjCYS>XxQg)!iusC+{Ot${6Bv#E?Uhn+stV71T1`?0mRhOv)Pwq`67W=w`LzZbnf z=X}n&zUTK(f4RCY<2ldealhT~ch%Qym>>W_irN*#{OOCGsAe4_WdH^vV&_Ml-2drF z=nG*U^!~anb3EyJ(3RI`U1sm~1lXOA7pMh70+!!?IWDWHdn{tknWgEG`aVIt@8ko- zXuvIa3`p1=M3zk`aI^d$fac#u?Z0o<@LN`)z}f|3YyhU`?16)IpO*e#mP&mGVAtgT zK4kRj2%#OQuy&HKW^(@Wr%(5^Na4pEA12=aJ`7eb{5=f1->BOU72kRF;Qf&-cC}qY za12c5Nv#gE5B>PYhvTwN6u~c^TnD7*Ul1BNN&YYao%4&f?-Mqz5T0GI=Wy*;CIWiF zpF+Th`Pa&5&$M6a^0)2zm4S})_&z7wd;H(gx?X~xx>CG%>9A&?{ z@v1$)edF*Ta5TPWygfYl>wi3W?f!Cl>a||t|Keiap(VfxHsRt7ckU*7613v3FNQ>o0e96 z%3r_|$bJO?OM`=3r=Ay^%hkDkLg9I4KK z&0nOqA3xr+?|~Jll`{kM&g74ed>0>*gF}ExSi|#f;Cb`l(Hm=SrL+GpOYHNyqJu`m z0}8MA*M4&}&|0__f(kb1ee?quNdk~PU~&h(fCA+5W7>e?_YJgm|MGq&#R1;V0RlGh zYkzEBKPL-*`(UB$Jd|?=ee;?9v8AbZ4Np$z0XS%xSU{XQ)4D@r#|06gYPI7@OVjuM zYfAxMX9rsf4{+^O>N)!O3UJ2f&j~)`-_>he=UzA;%p4C}?TPa&T0JK8?Q5>GIU`r# z%r(vf*X+S&t`6Mfb|ZmAZ$Bxa1oD;gzwYw=xZ4cia$lhs1p>y%bxa2|HZUhTjRzb7 zUHex@hyU1PbIK0ueOvWP+yk7i3!S?e4BxTuT&!WooQr&p{M7T?%me_pb-D(-imaP$ zoTu7U{r!4}zO(coZ|MjB^Dn}hGkQVu0Y|gYQJybU<%{oCZwhCFZUMPrK%>&=`lZ_m zFY1(%Yxhsn-Q~``iajf}sj{xa2d?9Tn}^oa?WHz&sI~GWCvZ1<*I^gu54;coW5B~3 z2lh;WyAD5?cKsXW+xsa0Kx_kQnRs(#fTiz?^N_gH&GrAf#@Kx(eW9ZnyXdB2R{aZV zyMy@M_}vv}wG@D;vN0Dq~JjMZe{?FrfKBkRRF;c z$DXr)-0p>E1CKE9P#pArz{4)&`Edv^Xwt&x$2nR30?i#lYUMER;(RR;{^h%SBnFD} znKhR6zy9U`q5GQWAHb3S_e%G_kN!l!Uk(ID15xN-*#C|L9$9D{=eGJApuBHg3p3h} z-rdj3Ec|=uYkS!j?f@+K{*ezLxYtC+0U$HAz~_23V7QBjZt47S=~TVm1q}Q3Zy)4y zZr8G(Im$?Qps)W{pTkvifaQi2V|Fs>B8D|t{#DNYUpjYjQMm2tI`Gx}C;vYe_WkFQ z`3qqUm_ee{-V%h5UnBOuN507sMecO-!?csN2k&erF#bk6bQ1Hf&TVGW^GWWXz z`~!_-ovX(5-{0WcA2cO)&;R>6a7kuk4AAcYUxvV6CFZaE8h759(+2R704*%w1DDG1 ztdsi*Y1F$5hvTz)A4`{t_}3W_6P+|QOJ!xJ1LX^twG@DYJz%On1-!}?6=}0BQ6CO1 z&ALFIu^$7FXl4pYGyk>q>XYVG?m@xVz^zAs?eP49>iRu8%5l>vH&1&kp<$$QNE;a5 zG7@rH+t14Vvl6Ei`M~C~qzM>K<#1K|eO@++>-@3W$OGe<5$PZKZ>0dX62mSK{_JqP zIq6o!`oftpfqVIPCbA?3PkYxg2*b{6wnMmq`Wx2yuN`mSn@%!oy%5Rl7!8sw^z?Ij z5u74f?F!ugBQ``L-kW~|YFUmFXC%Mf|!r|+4;y0R8Mgn4#4A7O-aK3Eg=DF(x)90o#uIGlQ^h5 z_wr9i<^pq#_3;n*Q{)QN_hQ-u_nO8ga@pRus&KZK#B7T1-t^JF{;$a;$|r!*rBUdV zVT2|!XvS#nnHviosewOsgz$c*XZR$~xX<_I#6>2nJB7=(O^4#vOVIW!Ju3frUXI@V zN&8WxiDy5&c0$DliuwK)^}B}pV@9092BaK$^_8ei5Mo-@Dm@!FXG zJ#<%o>V15pw8*K%j8Gt7QdO=wNIo>6hNenHwU|J;ta!urWw-UKnMbnjU=?T~SQ-7L zZ&0CnNMkmt0HxXSBmbgzLg!9SOh)*Tue4l_; zncgzl;)$-zeOug%X_h0Iz0Y2=@$q}ZAD>NlP5G@@$NVtxHwOf_M*%aM_OCq2Fc<{+ z_UZGs#Xz}o6v$jeLNBPObUXdjG~zOoxk)htV@5zvUZQ8>br=gamcJfttHXlf{R`pBnBu6;zA*`uv2w1{0qg2;$b z+bsy3K3cb)jo*F1VC8$=OgfV|1IRrAx4%H#UDIdMZxL!l6ZgboVu&j4^l#{0D15EK zF~7}{hL|)(R=9Hqs#sW9(5f~xF9zRJ9*kpERJh>^adG*ke|R%^9^cIi_>y_<{^Rjk zf1L7UZ~>{R1dS2{>$6J zJkJ9RyB-r*s_VM0=ZfNpz+bqS$kty`xBm8e4Kso0lCoFg#$DEzvG<$~@2ug6Q&$Gz z5`4*f*-99^wGb3Xv^0eQYvMZ)YvAa}7kHpI$PxM9GaWFw`5kWV?g8O|Yo4Z_<;}bX zFt(Cl6ko^=|Hau;Uah!mOi&yqqEv?&E(yP>dY>j$rkdcru1t_pt)6_r!*01;RHKQ7 z1%Wp>3~kfW(h#KVor&Mib_;BRf`IQCMzm#l601_6E$M8k`@rpG&MiauPV|}p9F}v- z-$pXGDjD;MxxC6Psid2YKv+wG!+GJh#KQCp*!pSw+_SwOuxwdD7v7s3jEMc(KK#B( zi25brYEel^|G}sZOvi%zighy@<6VMZi(CTwy?H?(Oi_2gZGZsDoVpqL<{<%%QyN(U z7)zD4ovR{PzYib+#61;di=EjyR_V&dUK@$BH8Lt7B`h}l*!Qe+c%DMrx!C^|z0kql zdF`!ZLK0-wRP2EcnaZ!qOY!Xjd;{2fAbwqG{cEq@8Y8%EN&5hiTRW zH5pm`POb=C0fzY9_0ornYq3pDEhC#R;h~@!Cd!y&J~@uOmOcrppfsNW{~vKA$au3&MVT zya+XScaJ4E;nE|cjR{gJRWt+~VncQN+H=7$&`f16LpMURVQ)%RARyuNw#PYkH%|R- zo_(R_U#qq9|3Q4!G4Bt({2Ko3DG-jg#TP+-F59%ySlohpf&b`S{%CckTAKyOPIV3LderC z5vWjG#8!u(O^;gA*1)l%Rj0Z}d3;iOb6}fIgl09kA}w}lVByQm<%v3B%~8pr#2VGx zVgq;;30No8n6oqBE+;bBH2teK76vUxP1UYEUztc06JfVa`=kr~>|Y$)~D&GLEul;#&L z;4XQ`r2}{6d~rZ>wnf2Ya9vR)It`NHK-}tlvd3+4iqq$>OrSO@6vLMrG;+nZC6PxP(b@#1a&Z}+7n*=9dAUk zGUXlLdhfp)YIO@8ChB=#q1k?l3H5o9dN+9Ed>S(`d|vtk1z}wJ3&5G*_O#i$8eEk|Lny=1ExoJVQ^tYFd&C+iZL85h z!mVa4L=X~A+-vzv+;aq~<_&MvH|w}}Il8*?oxS^jW#2$WgT*=v?Z?}s1NN}TJ@Z|_ z%g?1pi~8v$i|_6q9~Cpib*bA+*s_l4R$K9wSai86J=dR~Ziomh50OXzIevl3*fvcc zD|yQ|PpqD&v9r#=wy#1=FNm7(E^-+1m{MbGqy?rawH@$xl$ zW4uBhAVzny{o$lrCNqg!e)#@Xu}ot#Ytae9m?J)Ypx(TlvMP&#n&ch=2YR}d=9oxW{ruYlKweCH( z|2WGMKLR5x76CwU|MdHHdn`hecG?B?=O+tP_)yL>&9{7QQj&ac|E!I5?on(?uc&_3 zNe8CWYDlR)m4fs7`BTQ`St4@I&ArFas*(}E8G8}h>+d5{s{;%w`fd?zvC28kyn7}H zRIu@_F}m?26(v!{on2gf+uUI0huw=CQ4*FOfwPrS-qN;5coYR6aJU%81o33gKM2m9 zcS!9kSlzDoxnEKlMrsr{UZqEI!y#XFu=?_3R>e`X;KzMA9MX_UuT1)i&@>&Bk!I-x2H$*(+y3KNglIapq ziR>}Tcv7~euxB|*?G&rxYStcz7sFsi6ycw@?f`=1)YzZXE+g-1Xwcvx*K(!xH2<=_ z>9!x{As(d3vdIr-AmIOqqc7(f&(R~adeWDQR=@3Fe0^GEHFECXDjZR%7SPuuYpc8s zg)O5BCP{j7-Amxg)H%1W_c55;+gh(KJOdT7&rm?cL4%c_cP$i+Be*E z(AgrHFAw}CJz%+@aHclxYQO75y`$Gr9?;fc5Kpn}@{19E#@$Ji8B=<=i^tedFph+! zU;KHSXxsGqZPjqm>Ms#Ya;SQi`+z@>(CdaW{Q8w|vw+#sTE*aC@V@uFz8y4m?WK~! zl>o;13Dn~IFv_|GR>3#`|ACkR2qKZ@%A?D#}T<1xIZL~OE;kVVw>L|4p2>JivDeU zL`B{v;OO9vC=BWDL58oQi|xlMFo8rC*R~KdOJW5w9HJX>Z9jA_GyiKZ7=TVotSy0o z{}=tgvD1I~MTg42j_rSIUd|m16!&fafQE>EZf6gDD%^Q>6>B>_dGe3lqL} z1Lu+FHEG|MeTm;dN?FMwt~+nA=qD3MNw9oao5ly;0j8r&f12NZl*=RF;RYKV|WhlTv7u^lkf%J=EG7wM8){IC> zQ$?h&Ckayux*-Z`LP$82ypkh`<6bT*MHq%43^liQ3$1JKpAsgW*8AEl@;IIAc{$fy zYh-Tiau)zt$j*0p6b3^0C?+q{OV~s^Cocg{0AfpKMrY^)O1$iL*pR3NeNSnkR@Q3E z^LipjXmvTtNO;L070(*eywsNgVfA#){t0H#f=$ zO(Uy+jGa-rRDBX>gR!s?mskQeiRMfUeHC34G<=SLqa(<FYkmt|y1kl{w1fI=y>QM69^f^MCpWn5?nSz~EiCNl@>3evA$!1Mx_2F*v8_bli zYSY8OszDI_(9n{s#rw5Gz_lSQK)2ojO$$?Mu6Tq=uI*HP_zAr5}#nJd>I^(EUB!JlQCu+ZnZ}?V6@k){}us)5^zsHGd1&T@R=Mtz4VRx zxhkdLnapKFeMPBK0Kl3$j$>;UCAyghfUi-GsU0Ad47E*^Mjfp z$oq6i=2K2==t&xLzZ>AYhOZ6GJ1uzj4R(lzDl80e!Q+=vH?1h~+i*x&`qg$6Qaq-G z3+IQw0i8>vg-w~aZ48QV*L%#kZqEbD<{{1xW#fB?%@ZlE6r(>7pW1~%5L)2+5rR;w z2r*>LOd4tvUxvWn1QN+cb@A&7MO^rDReVD*++P5=B*f}#8HhCpHI5-gxki+_AK#AI ze=x`R+o&ylj0RYdqL=CA&(R;q#}}`ZD0{iZe-1E=aRUmx*xOkGNtaXu|F6{`vuU5 zoqwU>{je=@;Laf=)dY~_;vvZu*ZYo>kkEytJe?nAp`(Ya&6KYtr~ zv~T>3l_Dyv+$}BDW3GlJ<%XcrNBki|K&f z{b(`_EHs$yq&4&@!*R)7pr1%^ckHV;=19mfnT+&)8d_(q(!k|~$ZnJk(6w@J znQp^fSAa|>Pf;X7)F@f~oNej&d^C^Nw!8uADI-Tb6WEbLmm9;1Od!NE;>#)1A|G@o%|`A%$)t?Ii;rx(21kjboov&msk^ zm5*?|5_#9#etF}P^|+6b6u|LneBU^nlFN@YPK4@id5NdisNSmS(tIHoE8S$538h7= zYRs8|Dr6M`bwb?}_IUXxLn-nRu9`BA_@UwCz+Z1xaE4_xjc87{*|Mk)4G@!pbGDV( zn0v>Zyw$N5@qu6L#ds?de@*Sikp&o9%pm>&emSYTxfKY3l~$psmYM;(?`H@G^n0}C zy$4vjlqn9Kcu4NSD;)q<0G=L-h@8g9>?0~GAMVV*pE=~Au71bXdK?%@j6d!=HU#s% z8AxDaWW$Tp=6DI?#I>rlH}d5VbWEbI;gQ9UWrk0SJ#S*W^Q6uV+ zXE z290H3Vcf0a(`^%8RzoK{ZVvT6p;$MCjS5cej*j^faTdnN4JE?NcGf9}aG|I)3tD(S zd#omX8DTz$UdY9?P~|{WlOo7~DM78-P)+P-=1rb5_qnnlAhzQmumgw+E%O8Z(c%)@ z{OXsNvO!!*C$t9ZVqYKw8#>?2CJa`-#*45%{G!38w62 zfd2_l?#`wG*p%X5Z0h^RY*%kKAh+$K{DsFdkHYn7Y{5{lLPy5m&fKu`YH8m#a$cQ5t8X$=*na%Tig{*1!vvw#g?wESixL= z#dm?~X#@FtCaLVgm=<2IW!N7!Z7PLL%SyM!)Ld5A)+v6)GP{)j`=i$lKXLxsUZA%Q zCAh??>w0P**o2V)^#kOEEFIRXxt{(0GbYHcnp_!bd29W_!g_*Z{$&mB&O+((mCRc` z2Bu~P?Wac1SM=>5+Ar$?r!x5-SUI4}c68sh@#(~~T%a{ze`Zcg@||ei{h4U*u_bCj znX!;4%ZK6-6D7H>bZC?<{sK`TDdSDUvHFJkt1>dWZeF9hgYlfu8|1ygmk0y9;t5}h z-yFt6$*=688>Erwwh<2(Q^bW*2e+E#v*7sr=u5G;P*zKeA~eIG>{Byelfo;Dq%RK+ z;a!uQYSj?GqBnFJ)#u&=k<<~ukRHgQZsTyVba$M^$V=2WMo^?MP4q6oq$bq8gjEDg zbCjMBZx|Bhu2htwyTWt?W)LyMUepR{!?1^RJ+cq?n1JXv|DdhDE^>DL>VP%wE#(4Z zC~=efxhfyJm5WRbrm~V|e5l6$7l;Z>#J%s>?_vRRo|az>EFz3lVcFLOIHi$}_ zusixIC}9PiY_f{5tzA$hi|lDU^Y7-}bmwCJsR_?eBU*z(pPhXsO5hh_l& zU*D|&4KHg5=BES1fMK{JP!1{kN01z76BDkzxbj`cqTzR3*zUBjh4g~o_C{O6!iSFg zzVr`XglbU`rLr3`Owv=wwJ7w@HgRA&Bbteu1{FSxHbKD_ny;y*O7)?I7d{vutx_G~ zVg;<7znErV>0!N2cu!UsL2bLuAF`X~|C2j{!6z3eaQ}!Vr@pDBhZY3NdCL`7+JCTLD^-@O9WZza8@N zrp0Z~Sn+Jc!)Cn*^Ln*hv8M9&;)ZHV<>S#&9sU6UL=8936dT2bT4QE?gkMiTMP4D+ zR(BX;ZVwvD{Ossk8t$LOOu3PFgvU?*PT9~Z>LPb#$0fQZKErl|9?eMIV%jWTq6Kq* zv1$~G?Auvg)_9A`l_Kl8q$$SWxH0rq}S?`Ys}1r;3KvaHR!-nhAXDCvF;yJzSo zr6<$(iNWg;I4SJ=#O0_lv=*bFErccIEd`DjU_7KPSHlsj8<+1Fw3slvvlI(h<=-i? zVECF|sUKH_w#?LTuj z1YgY!-^)vH4q%AdaLM<5$mJyt7||So9Y;N18D&V_Mv?MB_8+9VEBg{?E$nxJTuf9e zr%MQSJTDl2Z?UsOPc?kC%&lEyVXDg_#99jJYh{^*c(TTO+J9}NMf+n=wj;@X)1gZy zc76^lnUPlGk){I9O{-|b5|!k2Cd(9FZJQ?$K-kj@JIw|uk(=((e3Wz`wOSw0t3tF= zfog7h<9zO4=5$nu3wARIO-PgBZ^x(Refq3XAy61M#S@_R0S^DzagJaPEN>;q5~&ER zLS=2nZ(YW*9N?Q@|Jv`hMOzvL1OLW|1NGBDl8T^x8jys97a6+rm)mx@&}9NJ0EAP3 zU$s!Cn7J)cv3N0s zDRy)Kn+DMy$O3EQBb6>M5a4!eT*ir94<8JEffjAJwGCg*ka25lV6AWkW>rO%bn^3@ z+lJQzu-;$pF|3MoBb|X5^IPGkjaNCuAkk-(d<1F^>jhkWG4RAi$n^71h;|Jtm@ftO1QzlJJk63a7rXC^~Cucu_)CaBeEL9B+`3Fep( zmjYR?TqAJ&<)I6g*0ASJ6-WB%)@J?uwF6g4wd7X?Vsaw)c_%&R!>nKADqk1|6t{>Q zL$|gf6N;*S)^mM426d}!4|e?(A5vNL(d~_l82OPM(-_zsEVfYGGCvY2z5SEEX{q?} z9cs10x=$&{B+JV_?ro>IySrO40|;4B4Q zUAh4qtI2*F-7;2=KL#|(+YVg%Gl{gxtzSR2|6ztW=#J$D8*fjTYD1Siw>K8#HrC@b z>Q{0b>X)Yu1EM2#s_1@kp%HTp--St{*XLo&dERT|r7d@P*@!#~_x5Al0)fwhCf-~= zwbwz0hb&LiqZ=Jp6W5e$=L$Q#B_!me_jR++Z7(itnmF~V6GU^jy!SIZAKLu|y-Xw} zC{IrNO8s?k{~O0xr_=wQ1&CAtdO!j8{_{bzmu%5~pjqxsv2Ulyed>p?g-c!u?{*DS_K;%mWB zNJ59y;I`%hFvKAr?a41TUNRE1vzf`DW8tai;s4WC zmapCf-@euW!%0@CDoRW^d$|azp1YMzo_Ne;6<$oEOBIJNPx<<8p4p~pIvN-f2^o|I3(^n) z6CNHVJC<7BG0Ximb}8!P)B^3RG}6sXMZs>NXy&Zng#&W-#sN7Sz4YxgFVK`@1|S>z z$(Hwzb65QJOyp^3RFcbMH2(weIrw*1st*V1(%4Wc^N)InG4=vFWAOta#wu67g~7P2 zKM&SBO{`NK=@06aiuhubTAH3%<*0IZWQx%>)eH%A9Z1Ask>8itG7|nF*hv%3^KZ(I z2KBDv%wvYc-@W`iYB10Ty-Le>nYzLO;PWSC1DPQ z_E%JZ-!kw5;S+@&r8t>~pvC6je)-IQ@ZjzU*GV|We9j=$y~TqeY^9o=QgZ*^E7{ZA zHv?l>yL%sCNvw)dLOT!BW-f9}9r}riTxA}pnc)sjvp8p{U~iCr2dALTXp_UVtR5v; z?LaUQ-fvGenpSMU&Fp$rvZzqviVw4t{XEGifU%pHm?v$5sBbPqbV^t__d9;Zh)aB%MMZil z#Y&CoI?H}PCnm{L&HB3Ik<#`mnmlSTzGfN}$UY(mT#aWZZH~)^?ED-aXjB|-*&f4$ z`ert!fs;!{-@c>G+YK%VCUN;iLl_#&zJXk(HckJbX<} z(VY25W&Ar?`@7@i!fh=OcD+C-r0RVu2q)~iqw9zbU1}`!oS61k`eT!XYW~s1rhJ-b ztxWs!%F((xt~ovzH-r!l_x1=+n(7fA86Oh+$3q3|ucQ4~*(igZ$&o2A?FhX{e}Bqx zwvvRd6F0|Ai_HyA0l6?18IPMerrHRe{NZyPd91LWROs>ljvS^zDuCPr5jSG)#a^SKxqxc9iCfys{y zhie(yo;c%T+Kv?aruZx2pD~hYzR{t>9{@96@o$}ie$C_5bCpla5!Zm5I4=>VY?~U{ zVIw_0bFIY-OYHb^Z%TsucDpfcU7nr}9#Eew1x7I-$0ud{a&m$FKUVJpvR(Pc_X-j6 zIMja(QA)2*u|+ihAD#JJwQ~09;=e_o1)J0#=Gw`XQmVG&Hn) zjY(l72u~%cV$IfGQFcwP)RQ2*AXC8go$hO~dn33Y>Z&L2C} z4mgSOl-vDu#nR;ueiyq_-;qQwy2?nWL@8pk^?i=yOsNwuH1U3MxnC1t-r5muyS)J78SSYg@n?IIw6J-SAPXE9E}Kl za*~M%0j3WNE1AO^!=2l_nX0Sz+mbVCYCdvCjO16>)D+r;=%RJ0YZjQMJ#!3lotrR7 z)2CT_>1vwSjErG*-QBbMtvdUyV5#R7h^sE%qeo{pzI>!BD&&{CzhgQCK|kR=(osjn zHE0Y?j&xmCeAU?L+*>|8CX~Bc<=eiO?q3dz0wAVMTMGT!W5cw^kd_e;F3B%OcYex1 z#^FSBRi>Yg@u+S3@=FS4>g6u$OIx{Z4|mt3XsZ^00@Dt;)gFzKS@<>;0pSPSteC?7 zvkN1J_-m`4{2{vKJ493|b!J0AkX(Kvzxs^hi)Ud$jk0}Mv7^4K^u`fWP?=(`#)8}p zGBp1g(+3~6L-%Bli+WeChJ#o{={V0kb&ff=OMLgN%*B~BxwIJRV8Uixh`wY7syka_ z+hpMz?z`Y%e7vt!Q_b5^GhB5{#+~UqXc@fK1ca8aMSmN{i3NEn z45Ur#OtgqXT$W45Vum7>psf#Yk>zi~)jk+`Rq3All011^Be|s!Y<=tIJLmR>Ey?$a(RQqTvEaN-u~9TMXZd zNAtgk+*5URb4z^y3e~#UG6j?DE z_DiH0agyhr8@!@PDBw?!q8;7WDc*yA*<3YZCr;c5%2eT2ua7gNDuN!QhvB%qJ4yVB zZ#H;CUJgS)q`LVD+s}M^SOCnRqi`4Vv7NhD2 zbsGaKzfxex{(Vm72&D^B)c00Os_!+F%Ym^5y)Ko8Sn+4nrlrB2-8>`aXb>(;>nIOU zGw+m2Iz+K(3EBI2f!T;@l|f5;Fwt3VnBM94pyt!=&!|n`Ev+`hvGRJ;)xDjtpFvpm z$*dykhAp7=X&}Z_B#@!bed>MM++Q-U^lv>2F(Fg7hywSI1}FP$#f1!2@b!ZoLhkHD!wY9B zN{d3#Rv5}4x_GCZi!2k4Ud_f}Mz6B6#>4SJW$WTvorVE5(QdbgDLM29r4zsp7KJjD zF;>22(1g9WsPSUj;}qHJyg;PKoI~Z=RlY31+y>Z=P8|~L7XD_sw(r4y5f|`2SHsT* zi-h={egE=5c>}NbSHR%VLQrR}ZK{dLQLc)sb0eU>!OB!Ir4};hnfq zBG2W1O@80Va(%XxVHhTDRbAFO#h`EoFOnE~iw^-h*=kFNFk$V!YZRqZ^Xr#rTYYfV zxo}$jIq@vXmtd&)eUTwuBiq7xXM8G)vyQ**8>unvwQXYiu_Sx02O4iAlyBTus_?!lXCf(}A~tahGry7KPiKS@XRVXY*>sdI zmtAX-^j7CDtoD zL7|Srq3sTRWSb2_oJxK0cOTH&b<(kff%Dk002o+av)W_o+I{Y^qTj|<*0;?CEu3|l zhP|&PY;_3DPA4wkZ=rCy$)!mchc@WC>9`f&T770(l_v-ivrUAvvdk}j=SQ#>O5l5M zOLY;J<}DYmmiKpF$C~RDA#Us%TA5y!JDM>+)Fh5fwy*t>rP^W3=&-pSC4mizdlPf! zXx5%8w?{?FN8*4j{S?KZT(PtpwTZEQdZ!<4Z{696BR;FsZ5q}EWZW%B_xUd+xHIfbrCX-;_hDYf;sX_Ta=g4lfj%acewks-I|`6rWvg_@pR zlPZimy0|mkvw=l!l!P;vfmQ{e`48x}CNtNt_ZN(zqUJ5zkM6D(>-wsw*o;Q(Ce2^G zRko8r(1_I7y5E-RV@CPBYD`(3+LxqUC(73bt~#4)y!>#h3_qG+k$;B=uEuH&+%bZl zw0{Nv3T3hV=Yk9nW`eGiwGk~OqoF7qXRdg?_;)=gf9e>V{`gBM+!%RFw26AtD8B4u>4vY}(dRn3w z(AN;kkjr|8g0+cgG(+>Q$2za2p`F)~Z3IY(701}Lv6;Eab#$3>^knG(o8!}n%xxB0 z*o_?SdiyNSp`D;kxT-nY3aWy^4KJUN)tEZ-q`kBL=Ttq!z5tc*<9?2ufOCHzv^d1N zbt&B2f)YG$v(V&?yDqxtQIaYFlLT3*rDSI=nYQz~)i^F;iWMb5{pj2jcqK~^%>6FQ zbrm2^BG#ZpG}O76c=rWAId|Zwu}O$~S;E`XUZ5r&casdx%h59R>V@Gwqe(5*Wp!U~ zr%Wu{7d_iE3ipkj7x-gI%e^Zl#R8G z{h;V%zaj9&_{SHEPy6L2wj8THY<;vL85@H?8|uSdOFV3YjhZX_lLoJQSR`G|=HgEp zBzdIkHK0s?#C~|(K;0bRQcG$PK_{|@%j=GQ{qwN7gMx+n(2oZ2*GOgP$*895b%uf& z)4LySCpvbYBNnYETE%r27IOLrj;1B(PDVC)tZ~!RMcAzL)r^JX0}4HafVVXh7x4P7 zCTLW+1fq)A&yv)3tI7N!ng}X7sX0}lI21ocV=VfqJ)Z4t4{?u=uG(B6#|ZS(GGPPB zN9lB?K)i?T3IU1mVo40f)}HdM z>iLe%7&?h0rQ73-o}l9Jbv>gKHld!KJT*P-Y+|mm$>J7l%50c8VPg(OB~9PMB53b- zenW@8DlO&s9XXL~clgfXhnIZAW@x5CNg0DIvw_sA2eX?>OM)NPVne(ZtSCs(`MA(Z zrwv*<>~SP14!u?~6L!m{WXlS&Ar{-A5TOiTaw1GYDlIld7ce(b|G+4v*Waci7j zuSj|&o*QnO{k zsV_OQVXoy)gX4DML_Ku+sZyt1nXCtoFl%;<$lxx@v1pJ>c)0$IYZhSBQh8F**)617 z^jJ^8L_rTKvwfGp%3`A}_3NHlGb{h~P1Sa6wSsdhzCenc`n@I-5n(lz7P(G^uxriJ zD@v@x6m2(X)q0Jf6262rWZxq6C*w!(Ol4;050Ws|R?L2eTS_C3Af}PGb~P$*O^Tk! za~+z;BNzn}xZW_ceOEtTHx)fK8H#bCv~2OkS_P+N%6V~1j-`i-Er{)8G4OfRcq`3U zd|6U*OB(>!>?iF!eWyLm1Uy{BO801p*L`oR0Fi_~AX`f{bgv{W$ZY6M1O=L=>rV^n zMoT7NMRwyoI)|IC;^XQ)_j)tb(kE0N+MH|?M#Lv~f!XRP-s_HgW0Rl_&2$ zEwvHY9+V%hscJxn2bUn(D|mAjWETdurRV68(xlY{J#Lv*l*5 zv)BqsZZ1qW1HRxPs^}y4$(jP=&fVDG9WEa|VEPO@Vo$MXP5(q4dV=;TD1IJUt}(V0 zA*9*EF6;UEcqV6)*Y7xV0~gv3vXxX0LqAkqUTc_3^(jFbq|S{_Ax#H(QRqFri*s2D zwBFQ=xCHY~d34Yscj1m!rUxzKwSa!=YXLodVPmIv-&-+8-~C)tugM1OoiYPw;x4{^ z^S)7l%*S)}>|m~16xKPYwg9u8DQlOZ!njfgqlXZtJafZ#hI9z@y{zmdHK14b0j*Qj zoTlQ={L^5|;88{-w-s75bRdR-(NNuPkt!OqhZZv+$6nfUO@zCz5FTSlX@u9Ibf603 z#>7Kw+H~AN2)7F}!9J9fI4E{et9>GSS8yp2s*cn3K&bbYX+ zLngwiIUF~Z83vX>;;kfvP?O>ni~JcF-4d^nR9%xjUst>OWSQ{F3!LffRaV$m&4rD` zV^yGddx7oW%bzMerW@)>Q$Y@k+2Fnpr75oJw!4Ae@25LeHEG$3N9OJ-1qPtx=5Cpd zeBZkIb=Pb1GYTS+riy-hlb|-f*ePd{9-1wUTdoVoOuds}DiQB->iZqSyR-FuPkO5j z?XWCT(-(hJSYkY+FRL&1b%Kt{(p3+BiBA^Xef2QO!4tB2Im?bjQ~GYu*@|PC7jidV zm}cGz@%p`4U(mB7y^=Yz7+(IlrQiLXkymHNo6B_Y1ra*z;$=FF$HgP@)bgakV&_igIWpAjdXL)Usnx0e z>#J%3^G>Rc&gTmwu7UvJV|#j@{wJn5QlfQhD0Q-et9aJ%wRn#2y=Z8i(p5sVJw;s;-GuiR zC+hGOGON1(K=b%gueta6X1Hu}pv4sip13FteoIEVP&vYXvE;yt42VA=G1@TmrDL>U!<%S9Wq70x1=r8G1U*^w_~r zqE2p|SRqh!2JUnPm;7u?XnmPh>J;zGz&NBBl;jvk>!FOoS7pS|epB?@fc=h9RYG>+E7;DN)s8jzZ6k+0h_->Bi9D+SXVQ>5X_WXA*>4lV=^$s6$ zl&j;tWE>uMTGV_0eqtf}H~|+LmedCMDGm7CF zd(W_@((em&MnyyfR8*vih=kso2m}@B35-An4!=nfIAZhlZWW)UJTd$4`wkmi@V z=yyl>gn;1nCeiBcO}^KA;h7pG>jLe01Ob~L_^&4$iO<_FsL!!ck`E`5u^g%dFX9>X z_NwM zxhh8SJ8FGF);jgsNohk{AgjCkk13!5GCGnP)&O&#b(?PkPG`f2wsS2DTzGa ze}LG#e2m!JaYme9Dk1v0YwC)CBlKfC)5R$#{ob6?1~I;Xr}PNkd>ss*?aEMDsZhTU zw-$jLTnq;4rVw-^v@DK<6~4b3x1N;(75Ho+lxyh)lvW7nTuW_PNikFg@5RS|@4zgo z4q@M3O6yq$h(R9_TPwFgUwwtO6@upRwSIwU7_6;=M zlY%hToUL0S9y`qoWCZ*(H8jJ&)h&?dk+a@E`|iMe>sZe<;PNq--d>be@uu17=lF)Q zfL9;MhAPk~)9}80$WTkvWqACMzasZ~i&NJ%l3vGW5-Pw~w;On?1vh<`SclyJHXB!7Q+-GwYb{&#_M@qccsOIXPLwj_T zR}$~_H9-%zJhpE3R?C~rf?HXuOC_#kWt&TjHY9&e%*T%HhPVjttx*VHMA}r;VH)l^ z9zdB9CzIv~v)PNcCQe`pSvr5Q?!&G{(Cm3M5Go?&*L}jy)mn=V(j**KJrM(y_cEZ8 z)?4GBOM4KPt2}yp!}%4z;IF#{V8iL8$5mu7%f8Jlk=4#^3XN*ZFVN`jsG!x&c7fI2 z?@2>w0f*UrjDt;L$-CB?Xm}nac}S(+zfF#6nJ$LXfpRq^U_MGvwV}i#9CLzCj#uxY zvFf1PMwNlZ@Z&Y_b8^}tG*_+6c&AOU>hVe=`9<=X;Kh*EI0wY1I3GxGRI##Z;Q(O zi3^q6_LAO^rL-&0!*w|$#`Ka;_w%GW=^^N>caCw3?1q_=`@V#0@-|P7O02@KkV829 z<)oFmajcvl<(l~=yQGOXG=h44;G_$_w*9?!ZSJSl+IbaNqsjun3c;pmSJ#2kL`*plRyLqo3GZ1Bln)61%;aPv~BZ z3rpXLSR_PxfEIV_`S`PlSTIMlQG5V>gn zv&2Tv`h_Wc`d}@sFSRnrB*qQi8h5~`i*OAT6Ad(|sLU@>X^W-Qh%tGt%b!H4X_(7H z`_U8CJ&})J8uT|B{g4`Y{ z4V4c)3)w=s3@8<3biZxxOC?^%bIbbObxWDMZ(TC~<2Ab?%GAMx_DFL~u(?pQc*~AP z4P0&0$jjKQ9~xMublOz?K9uyHixIag;xO)Fu-MTrutq!{z7Y1B)50{FCpZjLlya=^ z9-x&*3&x}fgr7*wxWK(M8BO+MBP||b9se4LN~BY7)W9_A{7vlK3$I!Iu9G+8;><~^ zVG^oc8?ZbGvQAr#Yas8U*1$X+V}oYxo7mQ)&aYMOEBV7nj!3nVn%Vuvuf9t4Y^cg= zyI-w7Kxfl-bIsPKS~+OXd)^cT`?9sb+_UymZP;354MvibY;c7$(eFnw?OdcV(G$v< z(h(ddj7Rf2mpQZh_#TVJS(xzg0c(XzTla|FSanYKQK>A560`$1&2Xx48WQ1ZcDtaL z^dJ;zX~n0yrdci`Zxe5|UAboUVc#;!8)%#OYw69$-px)RPIQ z{<>4%JAvsE{(dH`C+Sf8U|Sm0r8v&D8!mpew{DadC>g)KHME%O>lT2cmXz=-OJlzD z%~nTZ7HfjSrJO+-XZ?+`f znHWUKWgC^$$9$&=cb8=C0u@gUJuw^qTD4M{N@S#DU^nXUv=eEy#_k5yl5=-EL8M~5eTZQhMJrp7`nsVbHf-8auu~_e5f?uKL_z zvP5-s|DaDe8huT&)~Yparb+Kxsfp9Lfac&A!;M~1FY2KnEOF0^{cj$N_8SZ`fePftqYq_^VQh2Z04pq!>O2I*lmhR;oS|hRYe6^#tgQ0<^y?AgoBLBx5j=a?2@jW zZ{07$N76M-03W9XDQS29+x?X8h>C zU2o{C{9{P{(y5lAVlm$)YKP?Ox91=jKopzQ@X3Mjn>27fw3xY<4%JWnhRMk}G z38lH7u5<7?4R!Q5?S(eCx@GNtk~g`uG2dtyoiLqTy-LLo|bZKOr6!Uca*61H4rSnu7D*i6rM zH0z*q6IP+T{!QZgvRqPbNgjXfLczuZ(Ns%TT4DNIqS^@_N59@LyuvY&9~H4oC-p_W zseiABrFC`%01BMyppU$o>noR zD}#dcqoxT5dNwLl2-VEvYZ^;`d*| z)u$oS{Hd;n26iP>{&cof(7bI46$g*z39U)hY9W%yzF+!;)>N`^8fVXW5ockHDo)@r zPn^n1VP26mxbYqTK0;8)pP7eY04?7G9MBO?5Pz+(LlBaf%V%G-op}eC-OHRbg znTOJ*b6u{OVk-`Hbrx$rhtfcvNI0?pG7ly(p#|!*2_lnW)BWMi`4$`Q!2`v4jzvka zCQb>&NXp#Y?}PEma276yM6m<<`v9PRv^=q%BPCJ;Ri$tdu8!7VjIRg>`WZTHjYL1+*a%;#a=eO>`NF{T?}} zj|55$q4Gkl5~in<99~#qpTcv++S3%@dJR{31!_hvGDXV!#NH_sWN0qvUH7dcjaIOL;?9L8R*>A-~WFqBOO4PR_S2#XTFlOxx2 z%5E&CRiD9bNNf(O1eUOxoD|bAv`2Dq7orv>MnezIP>%S&&hEUEBI*?^zBFdwe>msj zG(=BquR+)FJ4--a7jh@>Ywjvn`npkN%op)&%nr4H-%AW#L7r5DMqad!u};31)VJ%$ z4+&fK8)*9^7>c0z>4uF29WWkeSKIwAU)V1UTm7LCXgbEg>D*0)vFP&60d|dQn>a0q zj~;ioH2=c`UnT9!^|x1A^VVlYDVsM7cuV02A=4I{;%7|j$1WfbqfOjoq%SYD|K zc=p(ZdgMHu93^;F5nk&f9yv$1H)l5MKg)MOPHv;baOx)O@pjRX$=&TJgv*j<8i*^Xj`qP^QMs#)lHQqjzRg0k!lyvkZAMd{6_0 zFR)Y@)fz`Eh~|8igD^HApn6$;zMAhc{v2@Pw%qg7`|*Q&=vB+fC%-(wKuv8Uz9c;M zi{(FFXLGOZ@AAU${&7LpbpGYAXvUN4a%nIoxHLT`Nb<0e7R0Jm2H>Y-95yL zEe(UD2XN^T>z(=Tu2of|9~q(gHZQA{8D8}Ij^&J%eNPo-*0bmE8qdlqZ*!z_$2Wwl zq6$boIvh2lDSM0n8XqSbN76~_V`^Wh_N(2Vt;SyNP!iUIrVU(#`qLgr<@lh{WOe(5XZZQi>?X4Loub#YhD^$hKQ!m zCEq0<`<3^~Eypm$7TRoOSBWW|B}~-)VlJ3_X>{8Y8FK=s85j|$NofH7x%5iQHVUrh z_xHGtBp?|58hZp2*fn>uA(;;lpRf7|7x*46Psgt$FYoKYOUwVjB%Wt_=d0>C@nEdZ zO8;YOBY;0UOr}H}r8$M? z^n6ys-BOBeZ%+%(c|5T{{${Z+gY}dNOb;AY=Wp9muuMpZ&Nk(v!)8+yB#oS*iRhP( zG}*gXOE>BFi8^}17vEuQ8OnNmX|rhAB} zIv9VH&o*K%=K-zW2r5;9I(XWQF$71moX>v?*A2IkO$xM~U02L?ITabJbIDXXNzayE z`$p*TYzdOZnCo71yGernwNf!l2Z5 z`Ip#U%upm~$$EeS?o@EdB6|x%=JB1dm+;MCJ)8CxUzFAlekQmtEpq6rPhF*hfX3&3 zSf-j9WM%orECm8A1dJ6Eoe^HQ*%ij3y#zhSP{y5)Txr`Ue02lj?v zOF6GWyP_B*s~X>fXlD7Y%Pw$UfM-5*WzRFB`=YjrxK)jluaO+(sc!IT4(k(Qsg6xm z?=OG;<=x0~;79bZnG+WW%wFO}i*^vnMaAmdqby|pb05aKS7AdTY z>BG1@f@&X+NSHfd!;~ZzKq7PD;^C>>2$|Ww?thm&_^$wxJX}=8rcWDF_0mLv_?Ah} zsn$K^rG3UW##!GnO`4aub#u13WbdMiS-7RyvEqa>dO`Px@_xcO%=SNOc8DNIwdc0@q(b6*C%5}14U$C~_fD`sv&u zYuNN(D}_VjXU&bpRUT$DI0JKH(~5?F z$zc43_gM6y3tAyJDcV2lvL4rDFPP(s${C-N6unU6RWcqecY!-Wa_cE-Kb<}K+LfGZ z_GWjL=%DPB9+}N9Cr6f&17lJB4}s4C5cnx5IISt$EC=@HQQ8(A{|J+>pwFl5+%Q;W z#qT1mZ4*w9NN||}(SK^#d!NQC7C$~*xFPA!5 zSM$X=O8cqDl*v*KGnw4PA*1z?I&aONB@6ebp01qk#8y<H$hEg&%|tI z#Pm{rC@osA#h$eq>ab3$?Z9u&(t!M-@3kpq;n&^hlo!gHg<~4_7hY%Gd&FF!@)}igOC48Wz35;dcAeg%#$|DEHEHx|^YBBX_zKNN=%SF)CWlPQ%(AtS ztW85aOZ|m4#o{$8EG_bd2Dh271H5!m-|_U@r027g$?IJ>$LFOHVWFYEa1PM!Mh|sj zF93r(1a?h9i!zsv1r8RIqL(*OWpz zp3rH?E}0f&T&9WcBtO01G4MDOc#_V$fbR3pfftQ=$qlvySicjLdDO|-ld*pVdw%pakH7sbRS}+XGOpRfKM=oU zfa)1=b4X`8?^DpLy|pwJkA0kkfP=Y5I3;s>Mw|Zml*g7Kw*va*-61*LS37kFP%BJU zD}?`hmlm)RoCDBFNj=+}y{6Zoir_GXv*6@cUnpp;?>atUzrJZV_?HX%enA?kg}zGm z8F3t$&^^>4UxE~4%iiW-qkmiQ3jQP*oAT;96z7Ri_BiugJH(+}aOO4*0sJQp1A>kv zE&9CEUc-qgdQQ&?Naa>$TKRs}aC<`kpp9~-=?C=X1{t}k`J1bB!ceN+c4|gW>paNr z(F3Kvkl(b~x$1V;sLrYcG7<#XT9u0+AMX3^)LnPAzhdVZu2OLRwUBp>YC>Ciy!_!< z*~o<6+Z2SrEMLs_TZ0>$(3ppG>U~i|w7HfFAFQ9P&C;~6M-O@0F>LlsWcjWB8ve*7psLP`YWndCP7gfie~dq#S%RkU zp1}tTaouv1a1(`wmR>TB#c*CP~rvcwV}?Q`zN06i25#Ttiu!nTzEL+S)fnQbA9*-YXIYe%AY~P?!t%`s^5V} zIOkV*Evl{MYo*P5f<>Bn%R@)1hc@qvT{q6D^uSV!(&`Q|NT*UU^h-O7{%N-8|7EDe zm?9YiyA^=EP<4AThgZz19^uyv)qaXu&5@JF-z}XZ@aQ>O?vXMg0!Sg?Knl@zT*8(G zd-K)#axiZCZ6|hs!p0kN-Cv8RaugRV(w% zcxY)oAr)vp?2}_#`4pX-rBMmfxuC$Pc=Q&3HLW-Nl-_1W_)Q8w^MLKhlU(^vAd0Z2 zCm26v%Wn$;*THiiIC3zj_M;PQuI6ZYuEwd;8$uf$81g$m>@<&nu%fh*cBtuMFyaR2 z;8@bB{k5deVTFvn#O=0EeyNnAp?O>K;ci@((oVX~VuLU_2E_MLPN?yyG?hhCuJ1IY zAbI~s-+wTZY&-vwCz?W8VOxfXJos|D;Oitt^qb;;@klP@FyMP zwIBiAYDeOOKJz8*HvNRAf~WO&t?!HWE#S$70|{WP$)T@R@tw``cb15N-~>ccL_g|p z_Lp+Dc8rj#f4D0-w&SxhoZ=Qh=K=Fm1FNqa%_R!^pB<{c%YxcKGfk@@01-+9efvjv zH2d2W3!oynQ4HeJcY-shsPz*8-rrCs$Mz;7G-MtUSU&Ssw8a(Z3<9>zcQ+V}Q>=wO z=1Pz&7ECqO^MbwcVL@~DglsGqS4j*}RSi>V?b{-3aw;wJ2Go2UVt;zvn_f&9jt|IvV6*0rId6S6*@S&3N= z?@V{cx)(v#Ih?Vk|Ckju4DQI8v=ZzJ7%q4UOydGJIL6%F>o55EAHO9dOF2J3UzOO8 zEBxdKSd54{bIbt_Q(Ey#D^u^nJ38*!%w*f+p4)KCuxbo^B5?%{{P-R*0H~4`Pw=0; z`2ms!RokbR=hcetgFNmO<}s;dvRDD*Ue8 z?pDdj7L;?v?tIGSfbDs#^=ltg_LqR`^fkCbK!bdR>I2`MYLs`qre~cc_xv{wb9B;_ zaldAN{bXM1q-#_Dl6ZrmaT0?%@6>xMVgi9o@#9V98T=C8${9T~WjY2zuIE)Lm%M3e~By{m6nP)^d;DG zy)**()uz=N$iq$;u-}Vgf!aYL#*C)7rC44=I~LRF={A!0;!|GN>Bmi%l`OLG1o$d! zVZ2XU@C=VJ_ga;?$F{3VnQMCXwt(q^MMLEB-r3g?^Zw6o4ILO0hM$kTSb9!}_9+Gq z7>}Wnc!<4=HDO_5@up^A>daKldME{N^@4=5PoD+mrCagWK&#wCzlzdBfA@dy1bD{q zGE^y1J95VWdc~-*cnf5ZJyU7{qkm6-8evP!b!m{wPv=*5Q^y$T!sAHN;b+*E`H0nk z#q5lShBNPi@WbFkRvis zftdHWNhSyOS@E_Xuz>?PWE}NFl;^eD{oeb)VG}#??W|hsn;*n>z1fPxF*SMBrBm>) zk9WInM;Ml7?hD2C|20Y>T1@@D^5=7oIV390?DNl0)qu zdfGHt-W+7&j{L3T(xo;f`b7UZF%le~Qcj~h*~x+O990CENI{}qoP&RpzCBgE6zQW! za)f^otq@ZjrD|-Zce0?q6m_}0iRFVbRH=#!$g-1#sQ{v-jIPyH8vP|YSc!8Cvgq8m z+OWYHs`k(@rv0;qWCJO?2pMSDO~mjwFX)+tAXq_7TtQyg>u5{>+kBf0t(xO6WaL9p z$gJoV$BCXMtXR}o`r~3=@gIU^W+klz8!fGbMOV1YdSNN{dRJ;hw9?ALhK9thLjePc zu^{a*cY^QCkU@Z{iuTy$&C4}cGO7*vCtc|R!HRVg^2xvqK(d(;{3d9cr^^Q`Om zj>J4?4Uav8cyvrdpL&1PyDL_0&5YPkXh-hG$6prVq6hVoO8M_{6nyBcd+Ejsv$Oxm z4Og^NUGQ57$X{Nreur8s6Ix_Qrag zK60K|KEZpIzAcq+6apIo0#092`-(2ar)3hCZy=EpBp!18T)B~x7~+86r1OOP))cIq z*P}902xS^6{23JjCQ1nf9SM0%<^6;~YQHJ1`CN`%*KIZpk&Z)J`w(CD%X3Mu9YeJn z3~!^x#x$I+QJLQNEI}T8KN#v+)1kK$zVKdIKxoKb8ENi97u2_a<`ny7OccH!g2eiw zeYACfxBMMcdOMcW+H=7S6d<|5|+o}ILW}GExx0|$sjgO#$^ahi?R9< z-|!m7RsRpam3qphe}^PO(zey8pxfJ!si4K*cyp667Z4z7Y0fAXnpwhiM=`GT$X=E=9WO_^?)`f0$5JqDY32d2%J#Y$ zC!02gS8L9sso8kj&G$lS0=n_n-2#$V-Ca!IJe#1&z@Q6JM`IKP^TGYfx(#ECIMNFP0?1nERG*Kd&Ny%8k16?vC^@s3)yGg zttSEUUVWidjH`nP5%DJ_kQ-5T?8`YE%ROu zMm&OP!175oysuF_tvc+|$;3;4idUMuvU?}a1y(!jka>#OI%{gxISs5cR)96iKT?x< z+C%+7wJYj})OoGtTq^D0TxGLti{=btNDAE&X3enYpNec{4@7ytJGq_w3RKt^rp=IEB9Zd zUtZODfYpRPt0KuHrfZz0Zw8Dt{~2bNi{5fWIZclW?`>QPZ>*J?j{@Bnr{L=rxOhy3 z+m0X&pd5Hi z)jFLxxR8I+7rc@!A88Kg11`)ek)+`5tI+q2aZ&7JtnXUxGJZdEIdfk3A61CE+PN~F z))yhY)75!at(d4f)J_9|1Jf{$}4iFvX*xWBx3=CX=Ka*y9DHtV>wQF{+37*I9}-OtFdp3sX&hopwO5a=#7}* z-*&{F&W1;;&JXJ5I%*4?`RfKcn`W1qNW9v#Y~}Uz5ZjA{uk}NXp1En=lxlu#D?V%+MRMI&vH5Fl^)$ zYqV0qpOf$+NZ2qNU?35DDl75`pUdwgCq!Z$cXX5#OoR5G26#Bt_LhL<51K zp@VzG?C1iPc+xwW+Wt8T^6f+msi}132`aM*xY$ATK#|bdQ00sc{hpAOU~x4u+~EpH z?YL5P(qcsN`TH-glbes{L2IZE5{!2+UyhLOZW;N;sp0pi*O3Rmw<_Dd4kir0QpJrt z(5Wev@W(WOc4ru~XpNY=>aOB`X!bp>mVFTjf~2+Csti9b)p&ra92=7q)r~lB?MtGb zs_d_twUz&@R7MMe_WHE+Ilc^qUIQ1ZU~ zO~7#nOMfLA%p>KwU3R_h&+@xjdv5N!HBq1`vG~?AqsKJ3>u1z!De!UM)ym-~)Txaa z|GL}R{N#Y^#p;v#a-SxxL6|N#2N=+?9lr)^T-<$xVUpxe3NGJgUAb zqeo<}`;brj;nCg)NWaB;w|ricmbsF&XTS!N%i!Yd!dvgI_l-qTfH4s_FgV(1K2e)$ z@w3Mcg)`t;!`A95FYee`{8ZdB-{XLY+sAy8nL?m-RME9b1G-MUNluG;7A+<`X1-2| zlC($t>ytgXd!1zUhAF+JnWdcF)nEQess%%w9THxZ7Lf<7QCP{xr-@;y)P#_@3=DdQ{s8?Jn~XWO3I`3 zj5_U#NBiX@v#;*@wT|h3%#JU2z%*y6kiif|U!ixZzOjlsjT2iULUhzi_sKyS$MYv@ z&0tNprFXmga?LnkgBdol>3b;YasJuf$jxwFZH5g%b2zUz@+CSos>K+q--!>14&=Gs z=nwSnFmlQnKYk{MWO){pu_17g@teJw1<$wWNzdJEocvVIk2euv=Gnil-Gh4C+B+rb zAD>|HM1th?1ypG{RI!{^Dn51*09Xd3T2}G6oU-zMCCFpt=cBP1k)` zvI{*euF5FrCRN;Za_YISx--?7WJ>rXVm0z?+jZliB&_1Sr{haj%7ddny;abwvf(3o znJ8jFVAP?N6zDX{d{O9okM9PlO@D*9V|&tlC;)2!!haLT_zgC%zReJFi}8kL*V73} zgKq0bPH~INU$?nT7Bso@n?GQQn-tL)T;rzn_`>)!oMaZj zmjK%JxtC+Trc|$0rdwjn1zm-?1&oRABsI9K_TEDXnT1{`r=@IdiHIff-(uiUM;@N0 zDb5`CcDX(AcoN8Wy5wdyJ+wQGUK@xqsziOsv1+N|x{db_}ZJOC2QlwQB_ zM~3NF@fd@OOt}`;`7{TfVb$&;*y!Ae$XUaMpic;Z4y7B@ z@Bz2}4^PLBf>{-T-kiY_KR6wyil}W~>r*ke#l(~~6V=HYWerdJ$}~yOfB20X%*pXh zh(}4~*&(k_e`pu3D$1V5y5naj9ep}eZB3`G%#K}X3n2C)Gq)T089T_Gc9()O&Ske7 z?1xTVoU{}pRDJ5_>H@U6BfhClF0E-ayVOnap1!xWYzB#qBa;|()F6JtjuyI=K>j)h z*5+ryJjbaqOSB}Hg-h^5rd)Q@erLN}=VoAyPB(G>S!EACA>rU#e+e`nwRg;3 z@4o*dUN;`=EC*8%+)3|y0Lt)b9MfHplKn#M7{{~SfGuijU%hw;C!J_`d?=qNVty({ zDO-ibOspcL(Ob3wfy%b)UhtI3i(G#aQ@y);YMLx;OwBhq6oHCWD@xOE4QRfQ*-m-! z_3-0ZcQcg56I!EC*ZFsc$&8rGV`7^3i#FD6z(a8vAZr!63SVV!&-&}@q&5h7P3RyF zC-2jbJEVN|H!Hfc^HM2P8f$V>R_aw~%L^@qW*K7;OkC0O&U^Ms#nL1XRT`zX*pKyE zX+?~|cALgOTHT}0K(gzD2r~}+7tKNsuw2JJ1R$vj?a9LbyLm#7xBUloj@RtkFhs`X?$bhPCT&U$21A>twq6P z@iGdj%f_c)F6TZj`>thcA!oU?rSg@D5l;8f9;+8xa8lKo+k&9aRk)TD1D{2h&H#d~ z5o*G61qkL;KKV@F&MRvp0$Y_iQM=kQ{_q89JW9n992M<(l4jNau?r=l;o}eY3C-XX z8)y;rxybADBc)jd5Uwmtl-AaTTrI9txL$@^dnyBTFOZd5LhmOdD~lDvwZ;nX{um=q z`Sn|x67P*xHoIfY=YKUenKo`$GwMF*U4r3IznMM$?y0c$J0BtEv$SL# zCbu72-{$a8|F0TmwNcg#+@CBl^}N(t3P>}APK?^gQfNy#fHcE;VmF1GN-;@!rCvrO zkk|ysRQHE`ta-0wPuEuSz_C8CrW-kG&vLg0knATSQm`r=4ldn4`F84Fu-sYRYx^ty zs%Ha8CT?ZC_S1JMa4OR=@sI<+MWv4Ls4?dcNYU+-XH-%krHFd z|6$18Y)^jS(i2iv=QW)5^a|l^85!{%Tvw7+zHnK@S(+HP%;?H}Rlzjp)8^PwZgaxb zXf!k%(qln9Hv^)@dwP{$WG2x{OZx*yr~S8wooyhYv^yLTE;NpQRG=_UE7NHUyJOZ; z#COwc*JNW4y4GM_+BS9GS`6wbO;)X^%y7s8BUYqi^}^Ps&iTDh>OyVb?|yuW`Nbe*KL} znonLnDHmA@sVHK6%@$biL}07yLPYy}bz_HDsbJ6<5{7|`2NTd%z>SH(4Ruq_0G9YD`MC%iE+AcaItcd$0y^iDqC%b4TGshck%-e{@FBL3G43GAk0W0 zF%;+zS`bJGogW9#w9dDwf!4924;vJ#J4FPVaj1BM-chkSj^F;rIwAQA~TC9;oOcF%}060TM(cPWqrpaUo-# zy&4tbdeP5*kmfZB#$j4=3fhjB{ShjExcuLC8lMBFXi{+@r336eu+)~XVLonlFYw{k zj@wjSFAu9zu^+^hE9Qk1nBEKFYF+?gb#AnuczZq#&^%hjb;yPJpnwCqxKIwm$0fxd zuu1!h4WVl09*MH)p}(AFwh@*Qj{x9Uem>MYlzeEjeEf4Hxv&aXt`;*g4jRJ zK}%M7pPKZ^S4vv0BIL&|rU3ufzC!i#KV4On%AeH-WNH2Ax28?a?VC6+oSw1W^Cgrnhn!gw>;V{9Cyh8tO>TNW!GE8O zgpPQ+nd>wsuBc~vdi2)MY4d_gbJwmPk~ha)i5t$SJz1d~vpM`d>X|*o5ie5vgG~trS$xUznzGw`>7y>YPJ901HMsCp!w$LBuSyG(aR|T{ zET=w6EQozsO12)25pKhAm5(NWHmc^_a43sCE+mCYT3`nbQGEEaxSok*1Daw8F)j@;Ff61EZsxv>>!%< z9q(&H>32aM_km?zMNL#EV_2hJH=cRKJ>qY*pVP4?7Ce`QzLM0+96hjl zu(q@EGesop8nl)L0yU0htWvj@iv^T$maau1Wpg_Jy)8Q0Mt)rm_=eZV^$)E5yV+kj z>we1fxMlDkZDU1|EXDelDp|_!7-Q~{x4_lmKc7G{bzc36z$cwuvZ4;PISK*+qJh^4 zkiFN>u5UYLd~ut4MC(dSm5tn_@=O8(^YS)re36M&ac{ zh$D*HBEHjL%UTJnr#jGp)kppR^UMEh9$3VtO^RwMEtE^V;mF(3VGPVIgeDofRPCB& zU5j#b%H#ABD2fM275+}Z2(+gsZh>D@%RJ^E`F&)m0)VZw%>IIcTAx4+!GJcl?=N7S zf8YJ{*#Gy#cT9k5ph^#(PzGLm4gPa~J8<{Q3E79%@&C-)rdG;_n1rb*&Ev%Zf?wYQ z0wp%pA#fcDfW7_0^8bxL*1FJPxJvz2ts#-Z)2|Y`NI_?G&1(x>TUX zX^~&{Vncl`LDx8Tr6-z0Hi`vA_W1pu&6xT4)eM1RNn~`x?i%wgMwXW4H{b^k9?YCu zx%P4#KjUu!h5oDjH%J1eHg=tstFtSg1(fOE9a2WxnYSCl1q%w=$oZb6JifO4vop$R zFUWhDc2*$6g;iFk=soMn$w{C3`8$36&1QerYwtKCZuEs~kW~m1mGJZe=isK^|D|!V-l@_NzSn!2^*m0Qjf3 z&+Q4m8GY#b-+JBvOk1Y0X?Q-pyD#sd5(US4B!(QELV-eU|2(v$R&pGiWicS1F|vJR z`9(Y;j`dFF=Wgt`ri*UI(mO?(t?2G#X|vBkwcLSwK`g!F>au)*y?sqkSJ_sj(77O2 zXKrRBr4uUDD79c|1%R2S$YoCg=fHyPGJZ!z%!j8Dl&1H^KhwAD(R&)@bzj^E7h3gR z0rDfTmE zZKntCa6{0$`)rLs((CdJn!jccYgL@c&K3 zJPrt@X13)B^Y0|<=!s#KE205XrLwIGFra9#?4Kg z*VzmSbEu@~x?g4^fZzByC1rO=!$t>sZ>htnyDKUB@9Q^v!&m}1S^W1t{a>og@WMGW zigKTjVBGolfp(}+dp9{g5>1dlKHgsn66}kfHui&pFAg_@5rPRVaV zn=(Gjh3YP$EdAxBxJO?-pZr;*^T$V6r?p94lWN&;%^43MbXU6?5aDszk2OAiYVUu% zy_L?g*;LxZ0U=2*@$6EN0OS*^1F$IVI}@rNwdbpvObP# zY*Zz=e-<{CD`M<^{oFd_EZBOzQhfw?C?@9}vgaKRN0BDy?MRvfQs5QI%Rj03f{@f& zC3|(`bfpupOA1iG{6%y5>OZ9Rk8S0=iv!cqLB`cwcSwfmqLmpxe=b~rO^vGE3C<*VV3v4H)8wc->lRx^ZIzw_vj*N_!cJcUDcC}f0*I*Dc{qbBlnS>#D@6pM+^?oN8rbN^MDP@1s~^wl}KnB{VpZOAaW1!*AYf8zkW zol-HO%^v9Q{)2%fYd}=K31^K}PLZr)HwrFovfx(b4mJFZW<!b4ZX}hL zteuxDvX?m*GRH=S>&tJB!OP2oCqMm9e%?y;{$RS;V>%o*0Dr-kOoBLHEMA=jI-Sjh zjh6fZ27PQxxAiokSOC28dggdPfwZ%51^lHIOT*=F>Ofn8{mhpNV9i{2GmRr1QwBUr zx`!O_V}V78(B%O$eXuL!kT5CxaaA(=rtChB?a6tr$>1NfbJPmC2M1;kJhJZYqouK_ zgEV#e|7sLb_nuhXHmM@DAlvp{r+$tG2)!PtO5y*iqc?eq1~2dCrG(y|?*>8Hn)_{6 zWdGV4!Cf%{$rT>hhHf*xcLQlexeM6r=t3VwYw&%*!T+vaV6m90~y&%t_M zJeQ{VRFPiAhqljBE$Hq?eEQ}c=BY8efmdgq?TN+z4{2W>4s{#%TakTjl4Kc@p=1f! z4HA*Po~RHiA?qaj*kxa`WZy|Cp@^}@*s_f!l`Z=cS;sQQdhXxUbKd8D&-1?5IoJ8G ztE;Q|E%*Jszu(XHlS~jXF<5LIL5W$Z&3sgW&UNwqptc&g#xEH@iy(?@qMMu*Y zYF9L{9MO%QrJYwbM#o=TPOZ8!y6YxMbH*E?n_>F}mk1vcMQ?j#aMubLDR^v@SO+PZ zy6a}n{7U|s?1=+&?&*oy1(|5czVA+46YIv^2?6~Y(5bMX0mg2to2xPOkIIUk8Vt-v zh+AUGQ@F0J#~NDT$|q}jsQEu8*v$<3wVjLz{xjB8NZmZy+#qS0CIhss7dgwN+;7bS z14DB24}SH^whmDg#h0b7_=}(cAF^wiDB}2F0%Y-wMD2Yfd$sp|m2s`dSFV#d1H*3Q zZsFpMXMx2!g1XLI;=DrcfvkMn9VTxh0L^xCecO2jyFzv;#>Ng`KOcb=If z7Q_$&KY?#fryg=FcoOjzyY2(vxS6=&?bUp|k&;r#Xs6yvkk%^5DIOZ96g*m3RjIBDbMQz_!l{uu7l1Me|v*(_p`uk|vXmCE5Zm-3x z1ZsGT#+2I)OFFF8ug)+G2n!0*)*_7l_vll+BD=$H=n5t%NYOozE18O@ZnjA zc1ny35}$!y-A@Z<#JY65*Woif`O!n3rYIiL;UOHTY4qvI%=KY|NpQx`ofYm^9Xh!a zPug7K>pKmXvX9YYwIx?uitEm&S2hD)B>9eMaF}@Yz1=#fLvFY0T>8lSSC%rSjbaJ* zq4c(4@5wDHR?G4qx!|OE|E0hGA*C$ve*qH$rGAi*gIj4rD3z&%gv5Yfb~mQ~>(@K* z&CN~FNh)n?xlMGl=E0zzWR>~wvrpiW%(v`q7_-Mn>c z<}2lVnZp=zu-dhF{|Enz7cV4*kvt46EG%t@L^4G!mz}_^4$F+^@Q@E$S-U~hm&azp z%fqWy*b%*6a|*Cj(53dcL~_b~3!|^&sYTDxy04Tnx`+}N#f8h(w^xa_fsP++2gP>3 z@NcZuWv)}n0ohi(&_YeOm|A*Ma52Vlg=z-O|4!X%$S)lVa~1(??*-G@40?h0cmN~* zHqP*6G;MT2z!|?2|>>zVM1hudrm2*QC^&toAarHd7aKImni%-F4xM9h7?f?i9SjU>+WF zM|68BupiAC0oIl!T~oVDL?C68|9ersvh5w+MdNaHWRLb-Gsd_YXs@N|rf(zzn)y^&Cl&(x>WvPON7&-8|Jj7pjNy+FB4aDUH|0F zsesHoVA9_35GW=4^-(vZwePNKqvWGB@i-czq-UYxx1G(wAREf~|2IoglN{%_%`J8` z+cvc4-6EbwgblaYnpVIOiWp0IO|Y47Oq z-^vQ5AVQ!8)lD#Qg0C-R>7RA#4l4rqL88LE!&|Wf_3u4xd>=O=q;R<>PTps-9vFfq z{EAR9Wkbqow$YL!!2**KW&tVePKaouxNQH8Jw67Jx|@d=m(xB*7l1 zPrp%u(PIJXp|A?B;Au!WKouqqz-9ctw7;@6;0t-+@3`U6zc&Z}nsXtOM&kNIzz{NB z=rNlw%#m^TGWyvRd-svc^z~nJN8Z=od2=K{{maM#8!k_|UjL-3b(C`)V=#@xZP~Y0 zV0`$VvjGjje;uWW$*=q>o~9Atn=8NtPz~8_GqcwKU7!Nf_C|d)_*EWZ*Epp{g3b4$ z7pSQgKE+U-XfB0^n+snr^yD2SJv-Q2yEY?OQ4q%aW%Cy|w~pV6p{y3CXH5 zr>m^mdh~emWgf^$sOrZjyspw`zkq}8!kWaMHFM9=GN+;^rLw3$!V3JV9Ln~#W$Xre1 zbeoQYjhEt|8-8~wP_=3fAn$M1Uu6+Dfoar8=PKKRv@Sbz^ zzIL#@a-hS&AwB-mmKi~O4@wsRUYaVD09}cY0EaL!5$b6JAp41# zzF}{fbEfa=jj`KtHK_^8P1Llx3U~7CaMHR6tq^K}R4^9^YaL|gG?8A37DdqZ>g6m7 zY}Yd-7`U@lfLq;GR>3r2wzxQ?Yk=lV(~A}KoE(1VSFz=bj4~6SElVE+G&%@+ZY^qfpMl4}#9{n{ zK(!xI$B(k+8Ct@uLc*iBXrJ))j8m=!ip{=_Uer29?}fHxvVveS2frZMP)1YK#Av#b zir3nUdkvPnrU0{Bo_PUILf0Al?3cGB80{UrzN%_`E$E@<`?$DhOl){aGe&Lnf~P7l z^pbhYV|K7O1j@?79(A(Y#2Q}KH6l=J>JoHDL#+kbZS0Ns8oU2md$do~cMa_2diN69 zV1Fy<=LtkhAlqBJlYDAS5B*z|fcA3!Y!N1>3XYe-x-==6ISVHd!;7Xo1xNOaS)5~@s%A$oSn&x56w zOXSLFNKsZ+{7CJyMPA_%IT+ctSu8iikT*YTJBmN9ST~6cjD$UEEb8bVsZB^oXx~~I zwiqtA`=+~EEgiXW|7_$&ju35iKA)hlaDwUnf}xl@{-GeEvYYAg0bb;=?;vyIEJKqu z7_$ws+?Olo>}mNyQv^h=JV5V8Qh#*5Ew1wz-b_A?WYT7~rYggeGwaS;;*}82M;vR;}h`V=HOUuii zY49{cEo@t!@laIwmn9bvO;quzw&8HFeb$!fEoaTTRk8F{{rl{A$)P9@qVu*jIP_x~ zgx=+xAGniNb0MvK85SlO&9`#3@HEVYJ|`lV*A^A@v7djS@t_fGwFfUFbDHSLI3ko; z6qB{&-0^uV)=AusK>b9Qvt9Z4>jh$OwD&oaYL^$l^f}~rYvJSIX+meMxTE03i)Cas z6n9W9%|tW&N%(TtlC+l@jxoRmAq%RDC^b8L8xPvt?mR*mi{N6ODAFW3q~i%QEB5}4 zI1LE|w}9nu(IYGC!dX=4FyjOUBYNkWaZ-rm*vosFp)zo)C%?V}uK(A!JBrKeC7;@w zjNS5B-yTyJnVhN9tVMPv6&{3t+NK3FOs8FD^}G5E?;Y!1A272tg-SIM?Lk*wv9B!& zCd|ra#wFTM>zn}E={_{}J|y^zc_;2v6l?|iYzIr~6l7LQ@YIG#y&Vc^g@xp90F#0D%TZ+yFxMWmKPE|fD zQ5WfZ43rC!rXg|BvTeFMszainEW82}&zig+GuH$~<~rC~qSUE%yvrMBIU*cg7YBF* z4U5X$R|IQEN|{kSKSt(&T=u@OaJ&gh_Z@Dd!#zd$LOEj7hmzsI6uv_*ZpMAu3Ktz6%Vt+R># zb|drox2;D5ps!Z^E8fkKjM-8ri<*IHN=!Cj>SI40$_b2<-B z9D5&%(L3}4$diE%Pv5F13CEmiLQJx_XgdXgW4poqd_y<0^!6UWS3e%|8)NyoG!rhq zAA2u*Zd(d<+L=iausEfJF`u!WDa(b;f|hB4?AEzf#`a2R>V=>k+4&AiwdxQ!UU6|3s9COLWiRhq1*a>CyRD9=9*bmD zJdFZN1{C=4-TYDU2`$C>#$fS zF}PMhGqJFO`F)7Rv|BAOT&d9s=^o7pch^;DYX&5fWIJj9OWsqIhVTOq4!eV{ALa5k zzPH2~ycvsPCkW#Nt)b(G<-E6@7Cn}Bv#)4r=AQ*k)x2oM_b5F-3L{xt2mBj3>tE}Z z{>SLw3N{wazy5Y{c$u2@{#`K^>-f{k91XW|-MZya4xwGO)vCzMT!lrCjHgC2GI`k%3(sy3?nNbtoWbnxRyP5rmC_5as&Y@dvCqEXq z^{Aj_>_S-1Rli#jg$aN?Dp@DGa>)*H;m*<8qtwQFddn+p+ps%Q07cW_))VEaC-Kaz zc)yCYHlI<3N4g$xCRX$%w^wxAA+!Ws6BF`uCm8U!Q!t^eQ2;C)t@W%hQEpzTv8~Q)H#RdQ{o#{i^qU-@px0kbEEc0{I~q9n>$ua^qPgsz(P!%6M&2OccIWiIw|VXWwET#o78L-$HlMXwHYu8R!+ei4oqE2f zRUs5bLwx>rPJU<#% zLZp?WjPG#DhQ`ygFeA4y)>7|B4CQiQv{IuxD==P$Xcm#5frEPle@@DCMy(w+ z=fwfSJVZ19nD%K7$0)0{A1mHcL=7-~e}1XdxUzgP?YpUug1hQ^iRz67ui%D#IlrI} z&fN-xfln%QVPx2IiV4Cd3*ZykiO`$_am@@%8kWpfR&zftsF)&U;{ZXwEYocWszAKZ z*s;j}N|FENr`8rmS>eM8hA@J|j;Q;sV~B7pBIb#?vR{79@{^yJIe%aop5Y#PidzOF z4=vB%6_a3#h0dY~_Xpaql>+F!AKk&fA-5akCy%@(t{g-0lRu|0e`4eexmWq;hsUD= zR<|SS_!n26n45cDQ5#N`8Fu~jxzfAM_axtM%KXpl*!@vyN>I4LuUdqe>h}z(nFdV# zqGxaK?ae9bddC6??Fsnc{Cr>Ys3sWSVhts9@7>F&O<++LYS^bdp0d00Y3zXWz8(it zK8-{&m~@|pXj%d${G$k%cPQb3Ka$DMptr+l2}Q@cmXh?VpHyHkBtxM`poaUO?P@P2 zQ{e}<-5`X^;OalIJVa zUH`e>BWE==-=3ep#u0L9LmZ9k9{*--_jN39i~2^Ll(Cz2o6C>X+s=)k2=M>=zzp3i zmd;-_X@3HKkbTh@BkFFkrx&Fm$loKsoyFG8{%NY^W1UAi*n3-d%k6By=_Sx3&8&Vr z#gW9U|F?7hhy0iCaH9NuqIXV8hQf|4dL)l+>Cw?1n+_9H5>lYG5nGs*h6`i8P z#KLl#)HIFK&2{pue<67~U5wMbTOXf6(pGON5&QuOo^!VtltV&Mt6vJhgsztWb7jV)KUUb2X zG81%Np6Q;OY4>-yET{9rj~{y|ik zLM2{i!|Y0Y0YR!aOThf!>HS0uaF4O^_mDSGqO7`)MOHfu#KA_oAlL0QT>*5!Q7mP9DzFXms&~EagP;7_XWzdZ8UQg7-Kbd zxr{#!&K{%B(M%GU!o2|R$6DpjlB@EoW`rwY0#_(T0I$qCy>eUnko?!oxa>PmpEO7P6bxvq$f4<^et=KT+J+(XFB6#d{=QMUo=3V z@zkTse3r4$xw;*^Zsnm+#{E}3_0abRrc`#_=rd*I9%2G5mVM>Cqbg2`&U3kH zCMm}QKrkMur#}QFtU!%J5oABU+(-!y(Q3cfjEaF8vJ;Kp>(l=sUOADd1R`RDx49V# zjzz-Mn5?W|zMp2qucmMpWI8GMv|3V0Dao`%!>n@b6=$&s=sB#VU+C#fmy>c=b@`8C z1VLiZ;{Z)FB==gW{6Dof@4pOPPcf_jEBoWK_ZJZAPLx;Vune9Ib=E%;V5p>Avu#a} zBJkpu{J$CWZKQ2!S$WQOFd1c4mOe;3ULm6r!C0Ly<1}o2cI4G=1!_neU;VRE?0WTY zn8z=h9iS{nK^n6?_d+A1d8`h=UmW?-02Crq1n(#q)5O<i@) zeZz2qJBP`ei9pa{%Hf!5rIxU@YaxUP2>Jbl(Y@ls_$+^?mLQBcMeh$|v2ORy@My7E zi2*G3x9kPN0#}v);{XDWd}8tn`}|v-8xSU0vkI{ZlXTy{&GqEnT{S)fD9D!H5lO2C zsdq+07rH9WgO77uBk8hmW|hcb1UP=DO)iVntXA^AS;dVpl^e;Z`1B?0AI%rwJoyP-g;^s`i=aI_uMXoBW>-@Iu7R__j;0{vv7fH z{2){i1t62?{c6vZS}CI4mfk&}QKJLDNWx#Ez^iSb{9B9me=PB}meF6YrZZSf9AK6Q zna3yM)ayDq`#84J#R-y^%eK^`#rz8Ygzd-L*2CKFx7J1u?LV}FY8kNO{^j>DBqAMQ;R_Y@mk3;@M{ zRAUprpq*a^9bobu@ow3Alm-tdiJ<|4HO*sPZE^fd1{MW#?PcUx8YIyui}a$IES}M# z#?JtmGM16h(ES(2XHwUAXKTsku;`;uf1e-z%`xpqP)#K1yGS*LYFJ^8*zUQU{s9CX zNf;Yj@p^sLfPmgG*KY}2WrB|nhVed$hab+NPqZb z8s>u(n^6Fb*c$dfJ=9ioCzw(MmRlf&Huv5qSlJJE%ti=_5L3sA;!F3##Z|3a{}kn? zWUg?euOsys{Z+4R9<4rVcpb&^KSWrGS_VDbVVD>E%Vs|O@p->jcmTJG^l5Cg+|7Kx zk@ANPOZ@_^+tf4?_Nk@9CRGE6!OEV~xYCBZ;k4gjiBNk{gbTIt6we|h!jN_h^y8#u z_aBr+0VwjUtV8MmLWMMKvEulsT4q_l=`SNjJwntX4Q*9V`KL;x4aS?yr=p)~j{*zK z_1#xOJLHxjh{BS*E|>dZWDV)JBtwMEeb}3C%S>Tk!XJmfkQm|@;5`0+zr^DhK*17v zmv!_iGk;CV*;g_nmkX{i%$!zK_M;f^kN7pkbOYlvR9yV&U?_+PKvVutp_J`!|8AWA zQLDgDHq^dVa*_kuz?dYj$;KXybEYkUx0wQy##1}@xK3Io#nT>m{pIx5a%e}?3L(d$ zsxM1ZxeR+^M|Hr^lvQGbPT8WqehoVJAeb9$74Co3oHACwmWHkSA#igEe%ujoeFab- zBxo`zEf}tc-$2a`=S||m ztnFzf+An$EIDTfJ+I2-c29f$g0&-AHc0!Vj*8Dpo|5^@!)RBJKcK@iDVkV8s;S{8x+NZ4>$EszRc}8<#k0^NT8>ztSevS%l~|h zBZJ~pA`JZ4JbM(7{2_N`d|-hAdor|?&@u-3AuH11L5_E2#mXBhtez=D#V$|A#~4iJG>G>d$jGu9apFLrCGz&h?2_~Ml$boy%S*o1vp-9G7$Tk) zd}^=S7^qhN+pPdCM$+&7&n@G0{2)J)E?s?rs^uIz4AnE*lw9+`VPN+Lcm(QCr;$hX zgKmix%%vX9oG~@|ju5mIU4q3&1*VofsGkZ4X7e^(pPifTKpVOkSS!MmiUvZ@II`{W zSGq}hKvIv{_TAtCda{%}{yiWa1s@0!7T!wW2MV${GOZy|qL$41aa6XjnDTf>bm~v; zS~QV6eQR-$7i^4jz#ebhzw!*Toiu(7puAePyB8W6_O>(f_|ci}SG*xtt>m@5t zzI|A_2{E9J+0Ev08k`O7zl4XSML{s00t`hZZHGcca#6e!xPC2|V zDUi)ujLAZ;**CE5F2S9ng*!bnRnjh&lFLo$p2s|&%+sGeb_C{NR5xHwIP9@e4OVG@ zdr2C>QYw-VxL|4)#4|4l7-omlsRPT`gt&(0fw(!Vt2~KyF$6vp>Z1sXk<@Au&i>xa zc;t;Fgs~;304Pq<{zeaE2}Y<%pPny!(8?fZ4d4La+J#aGYLnD?YeN!uw*g8^s+h#%_hSJU(gN5RwMqnJ5nHZcF?G8Lw75+x$%6B4+>j_kL%5hId5w z_}a$Vi+#U4}4fbmAU*~D~2;Y;h6AfkulSuZL%!*2I} zcT?h(9n2gx9K~}Hs{oMNE}-6@pP9S@pBMZa8`lHHva=E&kR7M7^n=MH>t!kV$mL(% z(KP#`ujcQNh2=cGD*HIIXB67l3XOs8JzFbd4wMEp^HFZGJ6@74M$WcQ;?4_y5r2I; zuyNn$y|&cE87Dxw!;i~wpUe>0wWpG6XJ_i zT9GyU=+W1Fhu=ZSychnr@+Wq#ZqonK(FPZj?W{E7s$V5WujO@W-Q))#5@KgXLwZl1 z&d!v}nvZa4*1zRHwM_m1IRl6|uvq4?tgX(F@fNSC7^4$J$~iV(@Q+AYglp_Pqh-D8 zWB$F_e0!+GJRnlj+uQr+WuWf-WdB3fh~P!6w_>&CZQCsYQrrY|QqU1r>% zOvx7_53ERp#7|&;ECxkM1ktwcuJr z3xc;lQvl6>*==_hv!8i4JTK#z)q{;`CBkVw>3A4`QttM&44GY<3eF49;G^n2r+;GF zj!M1OsYTp$GMDk2)jizMUCQG;roHeHbsA-^x5O)q8>vqR7H507uzaRBRrm%^dJ@O# zOn@UBChkycbo+Chano_yb>@o8tce(UxRw$*f6$~PRxJ&6OdH%^QaG*iGayiF`$m;0 zZ3Y-tT=WWtsg5MC-80LP6zCx@4*%kxqXzi~=YdB*4c!8;a+T3QNOA7z3a*5Jx~-Sx zJ{@zdP#11Ji+xH28-t11~&jo;S;OO3ZRyfQ+*P?|Ff+tqsFAhjoZ@C+WH@6dw0Qc)KA{iz^e*JZ_q z_?v7%Z@k1=ewJ#e#(n)^d;ED9xwBi0xnLt1e!6cwwfpHyQZCHFy42U;> z#R1QEcZ)x&;Z-ymQgZ$%vKcC~zPCQ=T4&Z|(GtNXDcqk7n5vKSUtVc~)RdP%flpdQ zr{|xE^o9R#sXyZ$NzW;Xv@%7%5CKZtJfO@9)ELP0-YKBtlzV%~b?q|=lbd}&qH-LUX* z2d*0nfV2W1WXf6Vd2gkvsDf3$b47B4a zUO$c=ASbcPm^{|tYt^JU0%K{zBIEa#jU|b4)~EJ_g`H3nJx-I@qn{w-F2F#Vs6o|j zGM~O|%_1*=w9iHNdNU_tyq<%{?=33RQOSw|)JUVDzJ83h_)sduI^t6UiVGn5Y#y`S zId@2eIg1|$S<2gGIFB6bC}%Ojsi*oZ@-r5hP3w_&XC z&1b7tb8E_qY$TH^YPOS2&RQp>_fd<9>1t!`+@wr&iCl+M#)>8Ha(bzeq{FD~ue{=o zv%+N}!nNq_Y~d;pS_}?Ija0j4_v|?E9@yqzgKFFhQWLnKVIf5HCn~Oi&Sf%4S(Xxh zIL)sZlW6Ka$i=ll2COaPh#sgPN&Z<6iC&LjRiz)~d`bMH7KDQ$6_6`qo7l!fQIF&v zF*6pRMFlV-jwg*gjnhTeGXgizv$kV!@1y|`n*EBgFSOC+63E7$Z_cAf@!W$Q-O?M!*ph=BeuMKhXfY}cSk+UPut#|=bsNeM; zgyVMKK{WL{rfC>35>movPQNBhD|nTNz?IUk(q%;uADe>bHt}na3YV$q@a&HF&g1Yl z${n}NZg-OV!6V}cz#LT53BqHqABF+^@ToCyt&f7nPxb;`o;zmHzIfaIMf`Epnh=msAtZsnvc!p}=lMM#W9kc+p1zT_ z_O*OGU@IK6y{ep4dBTd@@EE;wpA@7~$b`I)MjcozcADQiD-9wv>lSF7fsv73q>Tw8 z!&p&hT8Vg}a!IGR0%#@V^ny(HtJe^g2mSADfD9w(7_HoE0x~6GVW5X!IOnW?{LIU) zo`CF9aRsWo`ti8HE$Tti^r319(IIqn=}J>1+*ADZ@@Fzv#Npu9!M&+IM zsS9t>N6mLnSy|}&f@(rhrQBt>(?eDucZuqIqZ~SEclAK(39?S}sS9b$D zNtG`yRH?D)cz_p!-z@hkFfR>;(oy&ph=$w%T1cwI)gMazzxBAxb6^2cC5lr~4HF$< zSUPVRC675=dimmNi%`IX?cQ#f=JWA-2rl4R3UIITIrNr8>Bw7|8WU$^x87I^o^<6T zJl7^7ppiXzH(E1mf~x?wyks5mL>Z;5Bg^`=}8c=|x~ z_*(&D28(l(c&hXXh=<|P8Y%)j2bPzypmqAZg|A06;{vTaS=prx+J-s#9benMYD|yX z4tQtU?T+PMVL;m10-n}_VEvp{HfEma7&S3zR)RBo751Ivpy;ylp7W z3j!P{^ntFC^0yEY^yEPMJ((70=2-2rbk}_wM_X&dAp1S+Ts=LZ!^B6oXHcc6He4;xF&)-mCR!+b4{P zKn=(R#ofW~JcqG5mm5;4RaYVTt`6Y-;_wc2#Qb+11GR)vL7#&;A7QAOUL36*4vOY! z_uBkwD7jJ@Rp$*vQ{}ivG{42YAov)(-@&Vn>@rXC_8`wr7h!j~T6ffoU?TVpQ*z!0 z$dKZJU__au$J^S3^i{SM$j*FD=iip!&|FVk2h@lb4j6sle%hxYG*Zqo$9Gc)Y_)aZ zf%bs5TCav-mbtHl-H-%vI!^Z?e!9)&SWtlZr#-wTr0wGL)LRD-+`RHXn5AfG1kbi7 z9#|m@TuvOA9?Y0NTpp<`uhDDy%#LuVJJ_+2GjBb&yf)Qp@v|q}#kc!ovW`wOXQHS- znJbIc2~oec)Us~*^@xq-AH#J#UF{*4ZN3d`nT=pjTZh|IOAB)uo;o`l^AGs9x8>@5 ze!Z)BNW{-b9*X%vI2Ho|4jxPORj&hJqgu-Qm7(CdINX3081o)ANTdu_x_&&*hG=wfk@eH{R+vs4#Cbuas02bu$>?NMmnehiUt#gI}M9SR{(H zC+_&VqzR9HiN7)0(wJL33a_Zb;|>d#ck6W!!m)#c&5P49-wjb}>(|$9A-4uuio# zPixf!G2|9*-fPzO*_OWN7_leU;!At$8 zj=9eOWSVhs$GlakD_k$y5TCHvJT-G6!X&x`3kvkiz5 zuiSGgr!yD-X`$5pkn^qW01fYF(CIIQT^z-9{(15`GV2p$t6%iVQ$)d+LX5-__n8(2 zXhWpa?$l2kuXb54`-x$%vzM#2lmCuCIFu$&;T{om53O&1sBl(@h?~Z#?6*zAsAo5lQjhVxWk|8SAa`HYhn95; zpVo51{0zkdy8dmp zKxJ0gI~YO>V~3B<{Fh^$`yxKIqCmzl=uq6c@tdSpr)W&$@VYY7?sim#op^(xu|GNG z@`WA=#yH;^bL|6NFbnHs@earz?yXB^5=R>pxN+~R9<5C_x9NTf_;YgW_yvh3m8m9K zvWu5X=P^1Cv)BXE{OmpeChhi*9t5dOCih>kIB-d@{z(hRB%OsX ztJCevtruhvGiw3d9dm^k27jf-WWK^T?f-nzdkRLq;OS14f65;N$$#&IN-bXhlAl@= zqnIK)$lOT_whG8-9eiwx#DhWNT6IA)ji$}mSy7BPXu!O|7d^QCse=e{I^kae=sk*Bx@q_ISRtHr;#sOVFWb-; zRh-xV`ullxrR(u0Wx4;%4lW$Wx)|EuNR_e~^WL?SGw-NmVOaK)DqwaJ`!O2zdnIPp zglT3SU4BvX@L1A1RZ^Bp$V*(Qhrfmb9*JpY8@mJ|u&CUUm879|exuzWE4$Xrf3uB0 zGahQ-ymyDkhJ+Tq*TPLdLk(fHJU+yPT7!81>q$XgclX8@mlp(<#l(r5`X?y^l(4Z<$+UU??P1o+0Yd z35o!rF4wiZaj7OEUe()T$h1w|Gd?@?RuB)M$BLY&Ijlfj3sAjJ!nI4CKS0m!tkVzy3m z+~^@Pyhpw7QG>MHB}?@49r;#quJ4@kNrT~0U|5dyDcI})s13l>XJe%`Hn9I1Za35 z9NkO5QA^EV`E4!)-E`yCr84U;(_Kgm^qqtr8 zoM8l*`+7q)v&=ih5R=xW>EkqoVP+PW-@Y!0E{wXLlku*|c>kP`j8ew0k)S`ZlRWu zVs2Fl>ZU|O!T=qRPcZEYjC1xwhr7Fnro_Yj?L*$;r~}39B}+>Q7Rz{?4rr&zyl|0b zijT|Rn@v8`U@Mna%F{^CviSX7uYYLBcySt3Sel|NbEPis^d~I`(}g8wZ^gF?`0Wed zjK4D)EdvP~C~6W`5$Efe^l+x$>Z%cdf5?~G2!J&U!5%%9%is#D~+8UJ6IZ9Y?AlCI6SV$ z@Kfcsd7O316G(>`yb(4CG}ZfA-9Pi~mOp-8dG* zRW?|}_)2R{zI&a0#vE7`x9(=F3owSr$isWSa}Rx5#nOHN{&G)k1m-tr(62~HOQZ(^ zlY5n^S5wAMNsbY5u8V=^G%ta1W?3ys3y5-X8S{isD#JavM-512eOWa_tN{@0Rdu*Ur!iR$&uE9He z_KOd{+DA6h(BXw|{$1_0k1fYZPY;)-nr;}HY#9wFSk1KBKX%7v=IY?dF0wVPzpFV5e(Qm(R#MxTU;WJ)H^nbVg z<85v^-*dz%r^Zb=1Al^H6Hr}>?qqxL-bbS~?&V#Osp*^D>wkF{e|-jRG?W4P zl=q&+$BGqCuR{PAT{_=ZZW-3cY5Z6B<05N^5vnR@aoMlI<3sR_90BghZTm+uB(Fd9^*-oW35oI19R*Z1ZO{Bei;OuLZxYw@}T-g(t+FkW=| zDRt!4>cXKf-)*0;^2gXkqQ7~zv&^v*Pg=eHRe8z^{gUkp+p)-QjNY}|s$J`O#V)a) zt{`9{BNB)}QDJAyfy@?UVbwiH%Cq+t0X@fNdU>`j?Oo?Q!aQ905O2xii}T) zL?axS8Y7RI#{qT#(s)Qxl<|X*SP(H|NeW2}FW$jAaox}N%%G}9u#=Ii8L0NnR)6g` z2S%}Jl_FqLoKYY|!3j5F0IQh~3{M$6x6e1fQC~QRdSS3$|Zxy?jvLXR>AZsJN@-Lr?5`g1wPn2=>&iLbni`8CvtI_2kZf&e5TN%z=pir`{iy=smkj`7axp zldteHOl4HMQW$6rtf5mf_Z_5Nrhr2~OZ&%Vc5GozjMX>zB4c$|std#fw zjU52+H{WA@Z{|kD0EBCh-_&+~io6~;(Js%0L-7|#&01l0r-@6#e1DWQ2tAEg*($+9 zwnt%W76<;9N@Q&3fivv(eZWq5W65p(#JI)KpQU*0l*{1wl3;{?enxAcf5L?nM zoiW$9F9Q~{ z?dK;O<{*7eV7Q1Wsv)WW;R$+pym-4alyU~7X0Zeiu)~V@1r%tUNVKSXyZBj4opF9r zi20-zh#PTh=48e4LYcDL3X4_J5l_NF6p}ZAnHBPqhDW1;4gAy`jJt>UIvls!%%)Cy zai~KM7oGHtnMO9dyzZ(i9`83fmTp7R2H0HQmwG?4C7B0+AGT@WW-4dT(>h?Ksx>Mu!9*90b5@Fex^UA0Kv2CejC}_p~CPpYL00mpz-`Ujk z9uf=DH-_K(>wZty<~!{HK~BbYI(PO*=5H-;%omb2q}v0osmSZzeL>={9k+-u`3F7SMK7o<^jD74r>7Zu9HgdGL^joN+tD+pQn36WD5zx1tUROtEG>Bqz(_PQHjLfS7qiFjnN^G@w$Yhbo za>ftH9=s|J6l{Sj>oP#00lX~ps-#`_#bD>U2uL%%GBzV^SnqIslkGzA38)*B%;?; zcPShOydFLn5%)e_c**y{_$gQ8G)4;4_Bj7xGziSwn4;*14mxo4ke-D)GgKCzDV08` zhP~#&%-Sx3%BE`q8)5oj|@)9)zDE)gxA4 z|C(0V&m}2`B1O+X2@X;Y43Qp{m~D?o&A^}UbsYjZ$^?fx@;HXSUA(X-JEeh1z?oUKC7q1+ z)I85ufngCX%+RpdUY(Fym&Lq6PzA&$)DfuDY36X_iVItwPp{YuN;SQFEqZ6A?~37b zXY}XB*(SD&?F-4GvYMG$0_iLid~|L}bhFR3;3Z$XbiMH~cj8PUpQ557(RxqRcWUYS zBz9V9y~Ek^Xm z=y|t!ntG4q(ag_ufB-BI^tt;;=`W)V1(Fp68Ju3c2meobX}@~z#(0X4`J^#z|} zNDwl5R3VQ~zUMmtvR0LTfBy|NytUo*E)9QtUczI;N={>iy)WFQ)6gV)G>!o3UjrlT zXI?sJH6BX(lc9j#AYN!ziAme*W3v*xd*s>wD(uR`po#vHQc&>Ty4 zMPnJVWGG|FI*zd%^%ah>G-T;CLo*snL#4u@#E`_;LZZo1=#)L3Mxsz!zUS4cM%VRy zXRi6 z=iZp#B?f6B%Ybl)_FLy)XN4AQC(7G*Z)=5Z{WN=?cU8^)&#A!O%=+ z%;FbZ>@uhHLu&0?x$S39OSitb5El?opzN1hmj#Rg)JsQ0tmt*4zyn}n!_NR@W*Y|}M^3(zJ1D(ZaD#A*PS~PuAmAAc zdu7)W!v>ga!7ji~85|;8Db-C$co34_eyKq3j6z zWr&DP{W~yM7Ho*;T0ZKGFhzk^=nF#@h}}sBTsLaE0Fh%&I01jF(yJKTJcD9tXi~zT z3rW8z#J3LIkix_#=hqK<&nd}*rH2Q0$zTUJlc)%?UV5$by`>j3OvXUY$0|N7kR3EF z$*imIQcyI4NS;zEvFWqN)ZX9C>Z`D8z-c$3R14*(j9ON~7^mO2ByU6|H$pNIIfn^} zvOUp2G_XR3Y>+*dorW-1kC$wZ5;3!BE7Bz;*G=TGrvsq3hs!_7{{IFhgdij%V8aWF zuUq-b`d=JS9iDLNQS5ziG|UbB)7fMlYET`IssFgu9|EYpjPmGynyY5~Jmy+XcGWUm z=gD6s5z>ZnsNB+q@Raib_WnYy2k^Bwt!7J52xS8WK=DMZ$Q`@oQ5h(I5&ATu05QH1 zV?!>sY*Dg5$HFLX-LOp#*5Y!3J!&~CW9<<4Cd%yZ{-{*$JHF%;ei|ys1@na) z9O@esiYcaodkJ3xkWe;8aU5`i*cc=eSsavl)4c&b6Px7=0Q7s|cz#exWuU0!{$V2v z&$p5uh-P!=+Mf8DI|J<>GS(cQthyB3{+~C`|M(yZMSwx0P>v568$Ti!0s?;Vg{}wz z)FBb&2oVcLc zaTVFjD8`Qg5F&8L{_}@}G)Niy@~+JDZ0wUc73aFZ`*gX!JzN6-J!y~(=EhIdizwQO zHA3uEkGtLWo?k!3fgpFGS0=@0+m_-EOf_rO%#@|8_OSoHA(iVP2WTyp$<8{s`#FsP zmm~Y~zjg$Aw1JKp851-uE~|D)>q|zlLz` zR?xhgvlME1*y{KfbatPngwY#^IcfkV9_1`xdW_`9f$tID>vI#UYS#;27N`406Rdt2 zfAVBY_D`m`{nc$jCs?8W$w{SR0E_cY^Ej?K)UGg_*Ta&NE>!J}+)DPg@tEp{HThcF z$}kJKOp~<`eDpTT=3Nlrg!yY$YNDPzaoCk|Q+`@RM{X)GFqVG;d;xh_D%Z|Scf*iS zk}XBgU$wVVbNI2Q=_ALqy7faDyPTL9&Q&@fHtBbNza@H7WV=@|OlY%;VMWfVc@pK? z(BV>LC@pSB(-i$%Y?AEkeV%P{iZG{k5cl;j<+uv z*>!&%m}tc@j1)03nBi4D)kk-R0TpKCp{;{s&4h@)CnIzCbSmXo>>AbP0_ce_k`u$_OllS#SWf*c4V_RjxG=A_uum)bJR@?MK_!~&URT7^RleoZfd zBi=2i1r)2;^ZHzElwZ`@oAZr&ej1s0cWY`HW-M$JK3X7wrY3d=$$P$@5FQkpleo~5 zm4%*6M0UyYnSjh^$c*Jh_DpZqs~F6w_{b=`I&H`%cIFJZ>~bQlqcfw`G%(UX*`X6C z0`$;ageIdbyYV3XyAjD)wi`j?5#IT*v|9`H>sVf-Yq&iCt^%?zUNnQElTol=dO*)t zM8J+oP}5aDaaYAc6XwS4>zglbRYf6=@>}E}NOzLCxR#pkkWfL)N!8`aHlKqyR}Be12z~zk^%Ed8d2woH+Kw{2q!fBUP2=q_R*R<4zzD=C&w1G0_PPX0?}s z$4{+aWkAH0xrN_#DQ-Mi5*Vl0b7oIT_Q){u*%W|3_!@&0%Ud6J-39xZXncM0p<2aa zEk>rFAV=Jr92%G{Pua}~B}|INW0R148l)}6Lk83CHK+Tq`bDl%|MN`;iQV8#R^FlA zS6HHeLx|uw^8Q&AA^A~Z*Aae?-eZM!uWAAWS5mpwBgEwp*FG=o%d2jM=$akk zxOETv0PVwf>*Rsl0GSC5#SCX%IkjMs1KpB1+LDOEoBUFAQxG!okf%D}FZ))W`w3cE zXSTb+rQ5D<7Hd2O_d{hPEUhJAGR<}{q(xS_{-(aA6F*dX4R$HG8OKqXV6Ey}D-{Z2 z!fsfVfvM7nNJg6qEBAD~SD|#Ljeqkw#RRE5h~MJ7?71M80!Sq{HRoVl`!POqgT$0e z)cZF3V5Z4g{(E6a4mWFcc|f-*W0_(Rwp%~M@fW;#`SFi$2THd+*<>4N&+gL}Nu@b$ zi)u*8rm&V)t*rXcqnLVD536{q{Y82EBs*zz&co27SI`5k-(>H^>(ze zsZ{G?zi$^Zwnq4eYS@)(dZ)6~8+n&GsHTWOyM~ci^0i;zC`R8oQ(tsrC?l;hTKMf$ zxH6Vr)2BTw?VXk_#HSM(Wjv&mi=LyX>7mmB(myu~&ioXD$dHbp!R#VQYDe~~RKHJp z6Woc=>S7oew5CL;wtk;*F!_5|BoI#+8eESnx}Bm&XtlV5KR(jDwOX|;McqixlqyGjsOXic*@l?;EIff_FIDd~rACJRm`O5TH; zm@1*~4KFs{Y4M=D^|+ppbz`|ZP)zr;>YnTQ@f)BG9)Om?0f=V{!kUT<%`6^`-*%s> z=tXF&Hm?$zhlC8T=5H@GHPkCcbM)v2H7gG4W8Yw=Qo|YhaalF;U1Jz~p;nj4XWf7} zArX2W&})V}w%m6!6)+KTSfjEhS%6wAj}%NZ$loU<$3LKp;A|Nnj7<(ks*QMIlT5gz ztR#{HquhGnXerBWhig*2RO?rX6|Dx@HS3t5QXBE|q{{@xRQL~${BUOZSaa^eF^utI zQw%B2F2EXrCzB62MB+yHyR*b`AY+xeD4%c_aDz$QgDaV?L8D Date: Tue, 12 Jan 2021 23:02:40 +0800 Subject: [PATCH 335/337] rm unwanted import --- maro/data_lib/dump_csv_converter.py | 1 - 1 file changed, 1 deletion(-) diff --git a/maro/data_lib/dump_csv_converter.py b/maro/data_lib/dump_csv_converter.py index 97a777c8d..951103775 100644 --- a/maro/data_lib/dump_csv_converter.py +++ b/maro/data_lib/dump_csv_converter.py @@ -7,7 +7,6 @@ from datetime import datetime from math import floor from pathlib import Path -from shutil import copyfile import numpy as np import pandas as pd From a5705b18bd434018c0e855c754a96627818120d6 Mon Sep 17 00:00:00 2001 From: ysqyang Date: Fri, 15 Jan 2021 01:48:48 +0000 Subject: [PATCH 336/337] added references in policy_optimization.py --- maro/rl/algorithms/policy_optimization.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/maro/rl/algorithms/policy_optimization.py b/maro/rl/algorithms/policy_optimization.py index b265459d5..9a156739a 100644 --- a/maro/rl/algorithms/policy_optimization.py +++ b/maro/rl/algorithms/policy_optimization.py @@ -61,6 +61,10 @@ def train( class PolicyGradient(PolicyOptimization): + """The vanilla Policy Gradient (VPG) algorithm, a.k.a., REINFORCE. + + Reference: https://github.com/openai/spinningup/tree/master/spinup/algos/pytorch. + """ def train( self, states: np.ndarray, actions: np.ndarray, log_action_prob: np.ndarray, rewards: np.ndarray ): @@ -114,9 +118,11 @@ def __init__( class ActorCritic(PolicyOptimization): - """Actor Critic algorithm with separate policy and value models (no shared layers). + """Actor Critic algorithm with separate policy and value models. - The Actor-Critic algorithm base on the policy gradient theorem. + References: + https://github.com/openai/spinningup/tree/master/spinup/algos/pytorch. + https://towardsdatascience.com/understanding-actor-critic-methods-931b97b6df3f Args: model (LearningModel): Multi-task model that computes action distributions and state values. From 79b91905d425336b568f9645051a5f4e7403aca4 Mon Sep 17 00:00:00 2001 From: ysqyang Date: Fri, 15 Jan 2021 15:09:50 +0800 Subject: [PATCH 337/337] fixed lint issues --- maro/rl/algorithms/policy_optimization.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/maro/rl/algorithms/policy_optimization.py b/maro/rl/algorithms/policy_optimization.py index 9a156739a..ffab0633f 100644 --- a/maro/rl/algorithms/policy_optimization.py +++ b/maro/rl/algorithms/policy_optimization.py @@ -62,7 +62,7 @@ def train( class PolicyGradient(PolicyOptimization): """The vanilla Policy Gradient (VPG) algorithm, a.k.a., REINFORCE. - + Reference: https://github.com/openai/spinningup/tree/master/spinup/algos/pytorch. """ def train( @@ -120,7 +120,7 @@ def __init__( class ActorCritic(PolicyOptimization): """Actor Critic algorithm with separate policy and value models. - References: + References: https://github.com/openai/spinningup/tree/master/spinup/algos/pytorch. https://towardsdatascience.com/understanding-actor-critic-methods-931b97b6df3f