From 4638799a3a5c75fa8f63a47aee70e7945298f6ff Mon Sep 17 00:00:00 2001 From: Micha Livne Date: Wed, 6 Jul 2022 20:37:46 +0300 Subject: [PATCH 01/52] Megatron BART BOS / EOS bug fix (#4495) * 1. Debugging. Signed-off-by: Micha Livne * 1. BART dataset fixes missing for deocder output. Signed-off-by: Micha Livne * 1. Debugging. Signed-off-by: Micha Livne * 1. Debugging. Signed-off-by: Micha Livne * 1. Removed extra padding from BARTDataset. Signed-off-by: Micha Livne --- .../nlp/data/language_modeling/megatron/bart_dataset.py | 7 +++++-- .../nlp/data/language_modeling/megatron/t5_dataset.py | 5 ++++- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/nemo/collections/nlp/data/language_modeling/megatron/bart_dataset.py b/nemo/collections/nlp/data/language_modeling/megatron/bart_dataset.py index c072d878d424..c0169ffd3cb1 100644 --- a/nemo/collections/nlp/data/language_modeling/megatron/bart_dataset.py +++ b/nemo/collections/nlp/data/language_modeling/megatron/bart_dataset.py @@ -20,6 +20,9 @@ class BARTDataset(T5Dataset): + # account for added tokens + MAX_SEQ_LENGTH_DELTA = 1 + def __init__( self, cfg, @@ -77,8 +80,8 @@ def pad_and_convert_to_numpy( self, tokens, output_tokens, masked_positions, masked_labels, masked_spans=None, np_rng=None, ): """Pad sequences and convert them to numpy.""" - bart_decoder_in = [self.bos_id] + tokens[:-1] - bart_decoder_out = tokens + bart_decoder_in = [self.bos_id] + tokens + bart_decoder_out = tokens + [self.eos_id] if masked_spans is not None: # construct bart input by collapsing multiple into one, and delete randomly diff --git a/nemo/collections/nlp/data/language_modeling/megatron/t5_dataset.py b/nemo/collections/nlp/data/language_modeling/megatron/t5_dataset.py index 38978404e2ef..bcb080645666 100644 --- a/nemo/collections/nlp/data/language_modeling/megatron/t5_dataset.py +++ b/nemo/collections/nlp/data/language_modeling/megatron/t5_dataset.py @@ -29,6 +29,9 @@ class T5Dataset(Dataset): + # account for added tokens + MAX_SEQ_LENGTH_DELTA = 2 + def __init__( self, cfg, @@ -86,7 +89,7 @@ def __init__( data_prefix=data_prefix, num_epochs=num_epochs, max_num_samples=max_num_samples, - max_seq_length=self.max_seq_length - 2, # account for added tokens + max_seq_length=self.max_seq_length - self.MAX_SEQ_LENGTH_DELTA, # account for added tokens short_seq_prob=self.short_seq_prob, seed=self.seed, name=self.name, From ad6147910e7fff0081f83d43e8953cfff43a94a6 Mon Sep 17 00:00:00 2001 From: Virginia Adams <78445382+vadam5@users.noreply.github.com> Date: Wed, 6 Jul 2022 11:30:58 -0700 Subject: [PATCH 02/52] GPT Prompt Learning Improvements (#4496) * Updated pipeline parallel code to speed up training Signed-off-by: Virginia Adams * Load global batch size not local mini batch size Signed-off-by: Virginia Adams * Python reformatting Signed-off-by: Virginia Adams --- .../megatron_gpt_prompt_learning_config.yaml | 2 +- .../megatron_gpt_prompt_learning_model.py | 65 ++++++++++++------- 2 files changed, 42 insertions(+), 25 deletions(-) diff --git a/examples/nlp/language_modeling/conf/megatron_gpt_prompt_learning_config.yaml b/examples/nlp/language_modeling/conf/megatron_gpt_prompt_learning_config.yaml index cc7de6042c83..0891c59da12e 100644 --- a/examples/nlp/language_modeling/conf/megatron_gpt_prompt_learning_config.yaml +++ b/examples/nlp/language_modeling/conf/megatron_gpt_prompt_learning_config.yaml @@ -104,7 +104,7 @@ model: sched: name: CosineAnnealing warmup_steps: 50 + min_lr: 0.0 # min_lr must be 0.0 for prompt learning when pipeline parallel > 1 constant_steps: 0 # Constant steps should also be 0 when min_lr=0 - min_lr: 0.0 # min_lr must be 0.0 for prompt learning monitor: val_loss reduce_on_plateau: false \ No newline at end of file diff --git a/nemo/collections/nlp/models/language_modeling/megatron_gpt_prompt_learning_model.py b/nemo/collections/nlp/models/language_modeling/megatron_gpt_prompt_learning_model.py index 197ac32d7421..7cd0ab5cc886 100644 --- a/nemo/collections/nlp/models/language_modeling/megatron_gpt_prompt_learning_model.py +++ b/nemo/collections/nlp/models/language_modeling/megatron_gpt_prompt_learning_model.py @@ -104,21 +104,20 @@ def __init__(self, cfg: DictConfig, trainer: Trainer): override_config_path=frozen_model_cfg, ) - if self.frozen_model.cfg.precision == 16: - self.float_type = torch.float16 - elif self.frozen_model.cfg.precision == 'bf16': - self.float_type = torch.bfloat16 - else: - self.float_type = torch.float - # TODO: Enable amp_o2 training self.megatron_amp_o2 = False + self.pipeline_parallel = self.cfg.get('pipeline_model_parallel_size', 1) > 1 self.tokenizer = self.frozen_model.tokenizer self.hidden_size = self.frozen_model.cfg.hidden_size self.existing_tasks = list(self.cfg.get('existing_tasks', [])) self.new_tasks = list(self.cfg.get('new_tasks', [])) self.virtual_prompt_style = VirtualPromptStyle(cfg.virtual_prompt_style) + if self.pipeline_parallel: + assert ( + self.cfg.optim.sched.get("min_lr", 0.0) == 0.0 + ), "Minimum lr must be 0.0 when pipeline parallel size is > 1" + # Load templates for assigning virtual prompt token positions self.load_task_templates(self.cfg.task_templates) @@ -348,16 +347,33 @@ def setup_optimizer_param_groups(self): to be passed around in pipeline parallel models. The prompt-encoder and/or prompt table will use the learning rate set by the user. """ - virtual_prompt_params = {'params': []} - frozen_model_params = {'params': [param for param in self.frozen_model.parameters()], 'lr': 0.0} + # Freeze frozen model + for param in self.frozen_model.parameters(): + param.requires_grad = False - if self.frozen_model.model.pre_process: - virtual_prompt_params['params'].extend([param for param in self.prompt_table.parameters()]) + # Need to handle frozen model freezing differently when pp > 1 + if self.pipeline_parallel: + virtual_prompt_params = {'params': []} + frozen_model_params = {'params': [], 'lr': 0.0} - if self.virtual_prompt_source == VirtualPromptSource.PROMPT_ENCODER: - virtual_prompt_params['params'].extend([param for param in self.prompt_encoder.parameters()]) + if self.frozen_model.model.pre_process: + virtual_prompt_params['params'].extend([param for param in self.prompt_table.parameters()]) + + if self.virtual_prompt_source == VirtualPromptSource.PROMPT_ENCODER: + virtual_prompt_params['params'].extend([param for param in self.prompt_encoder.parameters()]) - self._optimizer_param_groups = virtual_prompt_params, frozen_model_params + # Unfreeze one part of each transformer layer setting lr to 0.0 so DDP + # and AMP won't complain but model still remains frozen + for layer in self.frozen_model.model.language_model.encoder.layers: + for param in layer.input_layernorm.parameters(): + param.requires_grad = True + + frozen_model_params['params'].extend([param for param in self.frozen_model.parameters()]) + + self._optimizer_param_groups = virtual_prompt_params, frozen_model_params + + else: + super().setup_optimizer_param_groups() def forward( self, @@ -388,7 +404,7 @@ def forward( encoder_input = None # Call forward on GPT model with preprocessed embeddings - if self.float_type == torch.float32: + if self.autocast_dtype == torch.float32: output = self.frozen_model.model( input_ids=None, position_ids=None, @@ -399,7 +415,7 @@ def forward( inference_max_sequence_len=inference_max_sequence_len, ) else: - with torch.autocast(device_type="cuda", dtype=self.float_type): + with torch.autocast(device_type="cuda", dtype=self.autocast_dtype): output = self.frozen_model.model( input_ids=None, position_ids=None, @@ -524,7 +540,7 @@ def fwd_bwd_step(self, batch, batch_idx, forward_only): _, seq_length = batch[0].shape tensor_shape = [seq_length, self.cfg.micro_batch_size, self.hidden_size] - if self.cfg.get('pipeline_model_parallel_size', 1) > 1: + if self.pipeline_parallel: losses_reduced_per_micro_batch = forward_backward_pipelining_without_interleaving( forward_step_func=self.get_forward_output_and_loss_func(), batch=batch, @@ -580,7 +596,8 @@ def training_step(self, batch, batch_idx): # Need to make sure the frozen model param learning rate stays 0.0 # so forceing lr to be 0.0 for gpt layers before param update - self._optimizer.param_groups[1]['lr'] = 0.0 + if self.pipeline_parallel: + self._optimizer.param_groups[1]['lr'] = 0.0 return loss_mean @@ -712,7 +729,7 @@ def build_virtual_prompt_dataset( task_templates=self.task_templates, pseudo_tokens=self.pseudo_tokens, pad_token_id=self.pad_token_id, - max_seq_length=self.cfg.data.get('max_seq_length', self.frozen_model.cfg.max_position_embeddings), + max_seq_length=self.frozen_model.cfg.encoder_seq_length, min_seq_length=self.cfg.data.get('min_seq_length', 1), add_bos=self.cfg.data.get('add_bos', False), add_eos=self.cfg.data.get('add_eos', True), @@ -720,16 +737,16 @@ def build_virtual_prompt_dataset( ) rank = parallel_state.get_data_parallel_rank() - world_size = parallel_state.get_data_parallel_world_size() + data_parallel_size = parallel_state.get_data_parallel_world_size() sampler = torch.utils.data.distributed.DistributedSampler( - dataset, num_replicas=world_size, rank=rank, shuffle=shuffle + dataset, num_replicas=data_parallel_size, rank=rank, shuffle=shuffle ) dataloader = torch.utils.data.DataLoader( dataset, collate_fn=dataset.collate_fn, sampler=sampler, - batch_size=batch_size, + batch_size=batch_size // data_parallel_size, drop_last=drop_last, num_workers=num_workers, pin_memory=pin_memory, @@ -771,7 +788,7 @@ def dummy(): task_templates=self.task_templates, pseudo_tokens=self.pseudo_tokens, pad_token_id=self.pad_token_id, - max_seq_length=self.cfg.data.get('max_seq_length', self.frozen_model.cfg.max_position_embeddings), + max_seq_length=self.frozen_model.cfg.encoder_seq_length, min_seq_length=self.cfg.data.get('min_seq_length', 1), add_bos=sampling_params["add_BOS"], add_eos=False, @@ -820,7 +837,7 @@ def set_input_tensor(self, input_tensor): model's forward_step_func won't have it. This function is thus used by internal code to bypass the input provided by the forward_step_func""" - # self.input_tensor = input_tensor + self.frozen_model.model.set_input_tensor(input_tensor) def get_forward_output_and_loss_func(self): From ab6c46b0a305343363e20c01a2246db212e6bd58 Mon Sep 17 00:00:00 2001 From: Sandeep Subramanian Date: Wed, 6 Jul 2022 14:46:29 -0700 Subject: [PATCH 03/52] Megatron perceiver with tensor parallelism only (#4318) * Temp Signed-off-by: MaximumEntropy * Add megatron dataset Signed-off-by: MaximumEntropy * Update config and fix global batch fetcher Signed-off-by: MaximumEntropy * Add dataset class Signed-off-by: MaximumEntropy * Update comments Signed-off-by: MaximumEntropy * Style Signed-off-by: MaximumEntropy * Update yaml Signed-off-by: MaximumEntropy * Fix duplicate yaml key Signed-off-by: MaximumEntropy * Translate method and preprocess script for raw text Signed-off-by: MaximumEntropy * Style Signed-off-by: MaximumEntropy * Remove pdb Signed-off-by: MaximumEntropy * Fix arg name Signed-off-by: MaximumEntropy * Fix other arg Signed-off-by: MaximumEntropy * Change sampler back Signed-off-by: MaximumEntropy * Move back to global batch fetcher to use distributed sampler Signed-off-by: MaximumEntropy * Add text memmap data Signed-off-by: MaximumEntropy * Update monitor Signed-off-by: MaximumEntropy * Fixes for PP Signed-off-by: MaximumEntropy * Remove unused import Signed-off-by: MaximumEntropy * Truncate examples in text memmap Signed-off-by: MaximumEntropy * NMT training batch interpolation key Signed-off-by: MaximumEntropy * tarred data fix Signed-off-by: MaximumEntropy * Change dataset type check Signed-off-by: MaximumEntropy * Fix sampler Signed-off-by: MaximumEntropy * Pass dataset cfg to determine type Signed-off-by: MaximumEntropy * Log global step on validation step as well Signed-off-by: MaximumEntropy * Fix NMT model saving with artifacts Signed-off-by: MaximumEntropy * Initialize DDP in decode if not initialized. Needed for inference only mode Signed-off-by: MaximumEntropy * Megatron NMT inference script Signed-off-by: MaximumEntropy * Inference config file Signed-off-by: MaximumEntropy * hardcode max delta temporarily Signed-off-by: MaximumEntropy * Style Signed-off-by: MaximumEntropy * detokenizer if processor is not none Signed-off-by: MaximumEntropy * Sampler config Signed-off-by: MaximumEntropy * Compat with configs without sampler arg Signed-off-by: MaximumEntropy * Style Signed-off-by: MaximumEntropy * Comment for validation dataset type Signed-off-by: MaximumEntropy * Fix tokenizer building Signed-off-by: MaximumEntropy * CI test for megatron nmt Signed-off-by: MaximumEntropy * Fix tokenizer in restore Signed-off-by: MaximumEntropy * Style Signed-off-by: MaximumEntropy * O2 restore from fix Signed-off-by: MaximumEntropy * Remove print Signed-off-by: MaximumEntropy * Change tokenizer model name in config Signed-off-by: MaximumEntropy * Logging Signed-off-by: MaximumEntropy * Set seed for distributed sampler Signed-off-by: MaximumEntropy * Cluster debugging messages Signed-off-by: MaximumEntropy * Style Signed-off-by: MaximumEntropy * Fix max generation delta Signed-off-by: MaximumEntropy * No LM Init Signed-off-by: MaximumEntropy * Use nlp save restore connector Signed-off-by: MaximumEntropy * Remove useless infer args Signed-off-by: MaximumEntropy * Typo Signed-off-by: MaximumEntropy * UTF8 safe print of translation result Signed-off-by: MaximumEntropy * Style Signed-off-by: MaximumEntropy * Add save restore connector back with comment Signed-off-by: MaximumEntropy * Refactor Signed-off-by: MaximumEntropy * Fix CI test Signed-off-by: MaximumEntropy * Add missing args Signed-off-by: MaximumEntropy * Address comments Signed-off-by: MaximumEntropy * Empty to restart * Fix CI test Signed-off-by: MaximumEntropy * Check for test ds Signed-off-by: MaximumEntropy * set fusion to false Signed-off-by: MaximumEntropy * Initial perceiver encoder Signed-off-by: MaximumEntropy * Perceiver with PP=1 Signed-off-by: MaximumEntropy * Remove init cross attn Signed-off-by: MaximumEntropy * CI test and remove init cross attn arg Signed-off-by: MaximumEntropy * Remove init cross attn layers from file Signed-off-by: MaximumEntropy * Style Signed-off-by: MaximumEntropy * Clean up Signed-off-by: MaximumEntropy * update branch Signed-off-by: ericharper * Set headscale false (#4364) Signed-off-by: MaximumEntropy * Add wandb as dependency (#4365) Signed-off-by: smajumdar * Raise trainer error (#4356) Signed-off-by: MaximumEntropy Co-authored-by: Micha Livne * Set headscale false (#4364) (#4366) Signed-off-by: MaximumEntropy Signed-off-by: smajumdar * Finetuning changes for BART (#4003) * Temp Signed-off-by: MaximumEntropy * Checkpoint converter to nemo for bart Signed-off-by: MaximumEntropy * Style Signed-off-by: MaximumEntropy Co-authored-by: Micha Livne * Make position embedding expansion specific to a batch to avoid checkpoint size mismatches (#4357) * Style Signed-off-by: MaximumEntropy * Fix logging warning Signed-off-by: MaximumEntropy Co-authored-by: Micha Livne * Refactor bias act fusion Signed-off-by: MaximumEntropy * Update NMT config Signed-off-by: MaximumEntropy * Fix electronic bug, new time ITN rule (#4355) * fix electronic bug Signed-off-by: ekmb * add new itn time rule Signed-off-by: ekmb * revert domain changes Signed-off-by: ekmb * remove repetition Signed-off-by: ekmb * Update ci tests Signed-off-by: MaximumEntropy * Correct support for dataclasses in default module dim (#4372) * Correct support for dataclasses in default module dim Signed-off-by: smajumdar * Fix path for save of results Signed-off-by: smajumdar * fix pad id bug (#4377) Signed-off-by: Yi Dong * Question answering bug fix (#4381) * refactor dialogue state tracking for modelling/dataset interoperability Signed-off-by: Zhilin Wang * fix style changes Signed-off-by: Zhilin Wang * fix typo Signed-off-by: Zhilin Wang * fix style raised by lgtm Signed-off-by: Zhilin Wang * fix style formatting Signed-off-by: Zhilin Wang * update template to include description of intent Signed-off-by: Zhilin Wang * update Jenkinsfile Signed-off-by: Zhilin Wang * changes based on requests in review Signed-off-by: Zhilin Wang * add compatibility with assistant dataset Signed-off-by: Zhilin Wang * update Jenkins Signed-off-by: Zhilin Wang * remove dialogue_state_tracking Signed-off-by: Zhilin Wang * update huggingface utils for dialogue Signed-off-by: Zhilin Wang * rename dialogue_state_tracking_hybrid to dialogue_state_tracking_sgdqa Signed-off-by: Zhilin Wang * style fix Signed-off-by: Zhilin Wang * fix style Signed-off-by: Zhilin Wang * style fix nemo/collections/nlp/models/dialogue_state_tracking_sgdqa/__init__.py Signed-off-by: Zhilin Wang * update Jenkinsfile for SGDGEN Signed-off-by: Zhilin Wang * update Jenkinsfile for SGDGEN Signed-off-by: Zhilin Wang * update Jenkinsfile for SGDGEN Signed-off-by: Zhilin Wang * update Jenkinsfile for SGDGEN Signed-off-by: Zhilin Wang * update Jenkinsfile for SGDGEN Signed-off-by: Zhilin Wang * fix typo Signed-off-by: Zhilin Wang * add docstrings for assistant data processsor Signed-off-by: Zhilin Wang * update Jenkins for SGDGEN local checkpoint Signed-off-by: Zhilin Wang * update style Signed-off-by: Zhilin Wang * use local vocab file for Jenkinsfile Signed-off-by: Zhilin Wang * patch for Jenkins CI using local file Signed-off-by: Zhilin Wang * add slot filling prediction and metrics Signed-off-by: Zhilin Wang * remove unused code Signed-off-by: Zhilin Wang * style fix Signed-off-by: Zhilin Wang * refactor metrics code out of Dialogue GPT Model Signed-off-by: Zhilin Wang * integrate backward compatible support for IntentSlotClassificationModel (bert model) Signed-off-by: Zhilin Wang * save prediction file for IntentSlotClassification Signed-off-by: Zhilin Wang * update dialogue gpt model training for megatron gpt Signed-off-by: Zhilin Wang * remove batch generate for HF GPT2, which causes lower performance Signed-off-by: Zhilin Wang * add few shot capability to dialogue gpt model Signed-off-by: Zhilin Wang * update Jenkinsfile and remove unused import Signed-off-by: Zhilin Wang * update code description and clarity Signed-off-by: Zhilin Wang * address PR comments Signed-off-by: Zhilin Wang * style fix Signed-off-by: Zhilin Wang * integrate compatibility with ZeroShotIntentModel Signed-off-by: Zhilin Wang * rename folder to dialogue due to increased scope and further refactor for clarity Signed-off-by: Zhilin Wang * added dialogue GPT for sequence generation task (e.g. answer extender) Signed-off-by: Zhilin Wang * add CI test for DialogueGPTGenerationModel Signed-off-by: Zhilin Wang * integrate DialogueS2SGenerationModel for generation task (e.g. answer extender) Signed-off-by: Zhilin Wang * modify huggingface utils to support HF t5/BART models Signed-off-by: Zhilin Wang * style fix Signed-off-by: Zhilin Wang * style fix Signed-off-by: Zhilin Wang * remove unused imports Signed-off-by: Zhilin Wang * style fix Signed-off-by: Zhilin Wang * update Jenkinsfile Signed-off-by: Zhilin Wang * update Jenkinsfile Signed-off-by: Zhilin Wang * update bleu metric Signed-off-by: Zhilin Wang * fix bleu metric style Signed-off-by: Zhilin Wang * debug bleu metric Signed-off-by: Zhilin Wang * debug bleu metric Signed-off-by: Zhilin Wang * update based on PR #3893 Signed-off-by: Zhilin Wang * update 2 based on PR #3893 Signed-off-by: Zhilin Wang * update 3 based on PR #3893 Signed-off-by: Zhilin Wang * integrate sgd generation based on user user utterance and system slot-values to generate system utterance Signed-off-by: Zhilin Wang * add validation model saving capabilities Signed-off-by: Zhilin Wang * cleaned up code for SGD Based Answer extender Signed-off-by: Zhilin Wang * update Dialogue Generation CI Signed-off-by: Zhilin Wang * update Jenkinsfile Signed-off-by: Zhilin Wang * update Jenkinsfile Signed-off-by: Zhilin Wang * fix Jenkins CI issue" Signed-off-by: Zhilin Wang * add support for design dataset Signed-off-by: Zhilin Wang * remove unnecessary imports Signed-off-by: Zhilin Wang * update Jenkins Signed-off-by: Zhilin Wang * update jenkins Signed-off-by: Zhilin Wang * update jenkins Signed-off-by: Zhilin Wang * support megatron for dialogue_s2s_generation_model Signed-off-by: Zhilin Wang * reduce loaded samples in MSMarcoDataProcessor to 64 when cfg.model.dataset.debug_mode=True Signed-off-by: Zhilin Wang * style fix Signed-off-by: Zhilin Wang * style fix Signed-off-by: Zhilin Wang * update CI Signed-off-by: Zhilin Wang * update checkpoint and predictions filename to include epoch number Signed-off-by: Zhilin Wang * style fix Signed-off-by: Zhilin Wang * integrate HF BART MNLI into zero shot intent model Signed-off-by: Zhilin Wang * integrate Dialogue Nearest Neighbour Model Signed-off-by: Zhilin Wang * update Jenkins Signed-off-by: Zhilin Wang * update Jenkins Signed-off-by: Zhilin Wang * refactor Dialogue SGD Data Processor to make interface for models cleaner Signed-off-by: Zhilin Wang * update jenkins Signed-off-by: Zhilin Wang * update Dialogue S2S Generation model for DialogueSGDDataProcessor interface Signed-off-by: Zhilin Wang * update jenkins Signed-off-by: Zhilin Wang * update jenkins Signed-off-by: Zhilin Wang * support sgd and drive thru datasets by zero shot model and nearest neighbour model Signed-off-by: Zhilin Wang * add prediction saving code to nearest neighbour and zero shot intent models Signed-off-by: Zhilin Wang * fix typo in sgd data processor Signed-off-by: Zhilin Wang * integrate Dialogue Mellon QA Data Processor Signed-off-by: Zhilin Wang * update mellon qa Signed-off-by: Zhilin Wang * update dialogue.py to remove outdated info Signed-off-by: Zhilin Wang * style fix Signed-off-by: Zhilin Wang * update dialogue_config.yaml Signed-off-by: Zhilin Wang * update dialogue_config.yaml Signed-off-by: Zhilin Wang * add dialogue docs Signed-off-by: Zhilin Wang * address review comments Signed-off-by: Zhilin Wang * style fix Signed-off-by: Zhilin Wang * style fix Signed-off-by: Zhilin Wang * style fix Signed-off-by: Zhilin Wang * style fix Signed-off-by: Zhilin Wang * style fix Signed-off-by: Zhilin Wang * style fix for cfg Signed-off-by: Zhilin Wang * make dependency on apex optional Signed-off-by: Zhilin Wang * change NLPDDPluggin calling logic to make it possible to run without apex Signed-off-by: Zhilin Wang * add first draft of tutorial Signed-off-by: Zhilin Wang * reduce ms marco size by removing lines without wellFormedAnswers Signed-off-by: Zhilin Wang * address pr comments Signed-off-by: Zhilin Wang * style fix Signed-off-by: Zhilin Wang * update colab tutorial link in dialogue docs Signed-off-by: Zhilin Wang * include unit test and some refactor to facilitate unit test Signed-off-by: Zhilin Wang * style fix Signed-off-by: Zhilin Wang * address pr issues Signed-off-by: Zhilin Wang * remove typos in dialogue tutorial Signed-off-by: Zhilin Wang * support larger files for question answering Signed-off-by: Zhilin Wang * style fix Signed-off-by: Zhilin Wang * style fix Signed-off-by: Zhilin Wang * style fix Signed-off-by: Zhilin Wang * remove unnecessary artifacts to reduce memory use Signed-off-by: Zhilin Wang * put 0 tensor to device Signed-off-by: Zhilin Wang * update link within dialogue tutorial Signed-off-by: Zhilin Wang * restore previously delete files Signed-off-by: Zhilin Wang * update error handling when loss = nan Signed-off-by: Zhilin Wang * update nan handling Signed-off-by: Zhilin Wang * style fix Signed-off-by: Zhilin Wang * update spanning loss func Signed-off-by: Zhilin Wang * update spanning loss Signed-off-by: Zhilin Wang * fix type error raised in qa_dataset.py Signed-off-by: Zhilin Wang * add error checking message Signed-off-by: Zhilin Wang * revert back to float32 Signed-off-by: Zhilin Wang * revert back to float32 Signed-off-by: Zhilin Wang * update error msgs Signed-off-by: Zhilin Wang * update error msgs Signed-off-by: Zhilin Wang * update error msgs Signed-off-by: Zhilin Wang * update error msgs Signed-off-by: Zhilin Wang * update error msgs Signed-off-by: Zhilin Wang * update error msgs Signed-off-by: Zhilin Wang * update error msgs Signed-off-by: Zhilin Wang * update error msgs Signed-off-by: Zhilin Wang * update exp logging Signed-off-by: Zhilin Wang * update error msgs Signed-off-by: Zhilin Wang * update loading of large file from pickle to json Signed-off-by: Zhilin Wang * update loading of large file from pickle to json Signed-off-by: Zhilin Wang * limit number of negative samples Signed-off-by: Zhilin Wang * revert post processing Signed-off-by: Zhilin Wang * revert post processing Signed-off-by: Zhilin Wang * remove unused methods and style fix Signed-off-by: Zhilin Wang * add more documentation Signed-off-by: Zhilin Wang * remove unused imports Signed-off-by: Zhilin Wang * changes base on PR review Signed-off-by: Zhilin Wang * set wandb logger falseby default Signed-off-by: Zhilin Wang * style fix * style fix * correct typo * style fix * style fix Co-authored-by: Zhilin Wang Co-authored-by: Oleksii Kuchaiev Co-authored-by: Yang Zhang Co-authored-by: Eric Harper Co-authored-by: Sandeep Subramanian * Fix ASR Typos in tutorials (#4384) * Fix typos Signed-off-by: smajumdar * Quick wav2vec fix. In-place operation adding convolutional positions to encoder was overwriting leaf history. Wasn't caught on previous torch versions. (#4383) Signed-off-by: tbartley94 Co-authored-by: tbartley94 (cherry picked from commit 0322b158f26a0b690edca7a84714e33752283923) Co-authored-by: Travis Bartley * Add Docs for NeMo Adapters (#4369) Signed-off-by: smajumdar * Update NeMo docs (#4397) Signed-off-by: smajumdar Co-authored-by: Eric Harper * Punctuation and capitalization tests race condition (#4399) * Add draft of race condition fixes Signed-off-by: PeganovAnton * Minor improvements Signed-off-by: PeganovAnton * More race condition fixes Signed-off-by: PeganovAnton * Improve error message Signed-off-by: PeganovAnton * Improve error message Signed-off-by: PeganovAnton * Improve error message Signed-off-by: PeganovAnton * bias act fusion changes Signed-off-by: MaximumEntropy * Address comments Signed-off-by: MaximumEntropy * Fix geglu without fusion Signed-off-by: MaximumEntropy * Reset files to main Signed-off-by: MaximumEntropy * Remove hidden blocks Signed-off-by: MaximumEntropy * Fix style Signed-off-by: MaximumEntropy Co-authored-by: Micha Livne Co-authored-by: Abhinav Khattar Co-authored-by: ericharper Co-authored-by: Somshubra Majumdar Co-authored-by: Evelina <10428420+ekmb@users.noreply.github.com> Co-authored-by: Yi Dong <43824965+yidong72@users.noreply.github.com> Co-authored-by: Zhilin Wang Co-authored-by: Zhilin Wang Co-authored-by: Oleksii Kuchaiev Co-authored-by: Yang Zhang Co-authored-by: Travis Bartley Co-authored-by: PeganovAnton --- .../conf/megatron_bart_config.yaml | 8 +- .../conf/megatron_t5_config.yaml | 8 +- .../conf/megatron_ul2_config.yaml | 8 +- .../conf/aayn_base_megatron.yaml | 4 +- .../megatron_lm_encoder_decoder_model.py | 4 + .../common/megatron/megatron_decoders.py | 1 - .../megatron/megatron_encoder_decoder.py | 16 ++ .../common/megatron/megatron_encoders.py | 45 ++- .../megatron/megatron_perceiver_encoders.py | 270 ++++++++++++++++++ .../retrieval_token_level_encoder_decoder.py | 4 - .../megatron/token_level_encoder_decoder.py | 9 +- .../modules/common/megatron/transformer.py | 4 +- 12 files changed, 359 insertions(+), 22 deletions(-) create mode 100644 nemo/collections/nlp/modules/common/megatron/megatron_perceiver_encoders.py diff --git a/examples/nlp/language_modeling/conf/megatron_bart_config.yaml b/examples/nlp/language_modeling/conf/megatron_bart_config.yaml index 03bc6466f1e6..e8c9adbd0bf9 100644 --- a/examples/nlp/language_modeling/conf/megatron_bart_config.yaml +++ b/examples/nlp/language_modeling/conf/megatron_bart_config.yaml @@ -56,7 +56,7 @@ model: seq_length: 512 max_position_embeddings: ${.seq_length} - num_layers: 12 + num_layers: 12 # For perceiver models, this is the number of cross-attention blocks. Each layer has 1 cross-attention and "num_self_attention_per_cross_attention" self-attention layers. hidden_size: 768 ffn_hidden_size: 3072 # Transformer FFN hidden size. Usually 4 * hidden_size. num_attention_heads: 12 @@ -76,11 +76,13 @@ model: bias_dropout_add_fusion: True # Use a kernel that fuses the bias addition, dropout and residual connection addition. bias: True # Whether to use bias terms in all weight matrices. normalization: 'layernorm' # Normalization layer to use. Options are 'layernorm', 'rmsnorm' - encoder_arch: 'transformer' - decoder_arch: 'transformer' + encoder_arch: 'transformer' # Options: ['transformer', 'perceiver'] + decoder_arch: 'transformer' # Options: ['transformer'] activation: 'gelu' # Options ['gelu', 'geglu', 'swiglu', 'reglu'] headscale: False # Whether to learn extra parameters that scale the output of the each self-attention head. transformer_block_type: 'pre_ln' # Options ['pre_ln', 'post_ln', 'normformer'] + hidden_steps: 32 # Number of latent vectors to use for pereceiver encoders + num_self_attention_per_cross_attention: 1 # Number of self-attention layers for every cross-attention layer. tokenizer: library: 'megatron' diff --git a/examples/nlp/language_modeling/conf/megatron_t5_config.yaml b/examples/nlp/language_modeling/conf/megatron_t5_config.yaml index df8010fa6258..d0adffa516b7 100644 --- a/examples/nlp/language_modeling/conf/megatron_t5_config.yaml +++ b/examples/nlp/language_modeling/conf/megatron_t5_config.yaml @@ -57,7 +57,7 @@ model: seq_length: 512 max_position_embeddings: ${.seq_length} - num_layers: 12 + num_layers: 12 # For perceiver models, this is the number of cross-attention blocks. Each layer has 1 cross-attention and "num_self_attention_per_cross_attention" self-attention layers. hidden_size: 768 ffn_hidden_size: 3072 # Transformer FFN hidden size. Usually 4 * hidden_size. num_attention_heads: 12 @@ -78,11 +78,13 @@ model: bias_dropout_add_fusion: True # Use a kernel that fuses the bias addition, dropout and residual connection addition. bias: True # Whether to use bias terms in all weight matrices. normalization: 'layernorm' # Normalization layer to use. Options are 'layernorm', 'rmsnorm' - encoder_arch: 'transformer' - decoder_arch: 'transformer' + encoder_arch: 'transformer' # Options: ['transformer', 'perceiver'] + decoder_arch: 'transformer' # Options: ['transformer'] activation: 'gelu' # Options ['gelu', 'geglu', 'swiglu', 'reglu'] headscale: False # Whether to learn extra parameters that scale the output of the each self-attention head. transformer_block_type: 'pre_ln' # Options ['pre_ln', 'post_ln', 'normformer'] + hidden_steps: 32 # Number of latent vectors to use for pereceiver encoders + num_self_attention_per_cross_attention: 1 # Number of self-attention layers for every cross-attention layer. tokenizer: library: 'megatron' diff --git a/examples/nlp/language_modeling/conf/megatron_ul2_config.yaml b/examples/nlp/language_modeling/conf/megatron_ul2_config.yaml index 113f2a6961af..db7944693b0c 100644 --- a/examples/nlp/language_modeling/conf/megatron_ul2_config.yaml +++ b/examples/nlp/language_modeling/conf/megatron_ul2_config.yaml @@ -55,7 +55,7 @@ model: seq_length: 512 max_position_embeddings: ${.seq_length} - num_layers: 12 + num_layers: 12 # For perceiver models, this is the number of cross-attention blocks. Each layer has 1 cross-attention and "num_self_attention_per_cross_attention" self-attention layers. hidden_size: 768 ffn_hidden_size: 3072 # Transformer FFN hidden size. Usually 4 * hidden_size. num_attention_heads: 12 @@ -75,11 +75,13 @@ model: bias_dropout_add_fusion: True # Use a kernel that fuses the bias addition, dropout and residual connection addition. bias: True # Whether to use bias terms in all weight matrices. normalization: 'layernorm' # Normalization layer to use. Options are 'layernorm', 'rmsnorm' - encoder_arch: 'transformer' - decoder_arch: 'transformer' + encoder_arch: 'transformer' # Options: ['transformer', 'perceiver'] + decoder_arch: 'transformer' # Options: ['transformer'] activation: 'gelu' # Options ['gelu', 'geglu', 'swiglu', 'reglu'] headscale: False # Whether to learn extra parameters that scale the output of the each self-attention head. transformer_block_type: 'pre_ln' # Options ['pre_ln', 'post_ln', 'normformer'] + hidden_steps: 32 # Number of latent vectors to use for pereceiver encoders + num_self_attention_per_cross_attention: 1 # Number of self-attention layers for every cross-attention layer. tokenizer: library: 'megatron' diff --git a/examples/nlp/machine_translation/conf/aayn_base_megatron.yaml b/examples/nlp/machine_translation/conf/aayn_base_megatron.yaml index 286bfaf6d8d7..d4db1c4d3243 100644 --- a/examples/nlp/machine_translation/conf/aayn_base_megatron.yaml +++ b/examples/nlp/machine_translation/conf/aayn_base_megatron.yaml @@ -66,7 +66,7 @@ model: seq_length: 512 max_position_embeddings: ${.seq_length} - num_layers: 12 + num_layers: 12 # For perceiver models, this is the number of cross-attention blocks. Each layer has 1 cross-attention and "num_self_attention_per_cross_attention" self-attention layers. hidden_size: 768 ffn_hidden_size: 3072 # Transformer FFN hidden size. Usually 4 * hidden_size. num_attention_heads: 12 @@ -91,6 +91,8 @@ model: activation: 'gelu' # Options ['gelu', 'geglu', 'swiglu', 'reglu'] headscale: False # Whether to learn extra parameters that scale the output of the each self-attention head. transformer_block_type: 'pre_ln' # Options ['pre_ln', 'post_ln', 'normformer'] + hidden_steps: 32 # Number of latent vectors to use for pereceiver encoders + num_self_attention_per_cross_attention: 1 # Number of self-attention layers for every cross-attention layer. # precision native_amp_init_scale: 4294967296 # 2 ** 32 diff --git a/nemo/collections/nlp/models/language_modeling/megatron_lm_encoder_decoder_model.py b/nemo/collections/nlp/models/language_modeling/megatron_lm_encoder_decoder_model.py index 6fa4e9d0129e..c3ab20ecccbe 100644 --- a/nemo/collections/nlp/models/language_modeling/megatron_lm_encoder_decoder_model.py +++ b/nemo/collections/nlp/models/language_modeling/megatron_lm_encoder_decoder_model.py @@ -120,6 +120,8 @@ def setup_optimizer_param_groups(self): def model_provider_func(self, pre_process, post_process, add_encoder, add_decoder): # TODO: create get_encoder_decoder_model()here for different losses (e..g, nll, vae, mim) + if parallel_state.get_pipeline_model_parallel_world_size() > 1 and self.cfg.encoder_arch == 'perceiver': + raise ValueError(f"Perceivers with pipeline parallel > 1 is not supported yet.") if hasattr(self.cfg, 'bias_gelu_fusion'): logging.warning('bias_gelu_fusion is deprecated. Please use bias_activation_fusion instead.') activation_fusion = self.cfg.bias_gelu_fusion @@ -163,6 +165,8 @@ def model_provider_func(self, pre_process, post_process, add_encoder, add_decode normalization=self.cfg.get('normalization', 'layernorm'), transformer_block_type=self.cfg.get('transformer_block_type', 'pre_ln'), headscale=self.cfg.get('headscale', False), + hidden_steps=self.cfg.get('hidden_steps', -1), + num_self_attention_per_cross_attention=self.cfg.get('num_self_attention_per_cross_attention', 1), add_encoder=add_encoder, add_decoder=add_decoder, ) diff --git a/nemo/collections/nlp/modules/common/megatron/megatron_decoders.py b/nemo/collections/nlp/modules/common/megatron/megatron_decoders.py index ae8d261edd06..f3a1a57b2fbd 100644 --- a/nemo/collections/nlp/modules/common/megatron/megatron_decoders.py +++ b/nemo/collections/nlp/modules/common/megatron/megatron_decoders.py @@ -76,7 +76,6 @@ def get_decoder_model( headscale=False, transformer_block_type="pre_ln", hidden_steps=-1, - hidden_blocks=1, parent_model_type=ModelType.encoder_or_decoder, layer_type=None, chunk_size=64, diff --git a/nemo/collections/nlp/modules/common/megatron/megatron_encoder_decoder.py b/nemo/collections/nlp/modules/common/megatron/megatron_encoder_decoder.py index ea27f7e9fff8..059eb28f8542 100644 --- a/nemo/collections/nlp/modules/common/megatron/megatron_encoder_decoder.py +++ b/nemo/collections/nlp/modules/common/megatron/megatron_encoder_decoder.py @@ -13,7 +13,9 @@ # limitations under the License. """Transformer based language model.""" +import torch +from nemo.collections.nlp.modules.common.megatron.megatron_perceiver_encoders import MegatronPerceiverEncoderModule from nemo.collections.nlp.modules.common.megatron.module import MegatronModule from nemo.collections.nlp.modules.common.megatron.utils import ApexGuardDefaults @@ -41,15 +43,25 @@ def __init__( # AttnMaskType enum mask type (e.g., padding, casual) encoder_attn_mask_type: AttnMaskType = None, decoder_attn_mask_type: AttnMaskType = None, + hidden_steps: int = None, ): super(MegatronTransformerEncoderDecoderModule, self).__init__() self.encoder = encoder self.decoder = decoder + self.hidden_steps = hidden_steps + if isinstance(encoder, MegatronPerceiverEncoderModule) and hidden_steps is None: + raise ValueError( + f"hidden_steps cannot be None for perceiver encoders. It is needed to compute the encoder-decoder cross attention mask." + ) + # try to infer mask_type if not given if encoder_attn_mask_type is None: if encoder is None: encoder_attn_mask_type = None + # Perceiver does not have a `.model` attribute, assume it always uses padding mask. + elif isinstance(encoder, MegatronPerceiverEncoderModule): + encoder_attn_mask_type = AttnMaskType.padding elif hasattr(encoder.model, 'self_attn_mask_type'): encoder_attn_mask_type = encoder.model.self_attn_mask_type else: @@ -136,6 +148,10 @@ def forward( return enc_output # decoder + # Adjust encoder attention mask if encoder is a perceiver. + if self.encoder is not None and isinstance(self.encoder, MegatronPerceiverEncoderModule): + enc_attn_mask = torch.ones(enc_output.size(0), self.hidden_steps).to(enc_output.device) + dec_output = self.decode( dec_input=dec_input, dec_attn_mask=dec_attn_mask, diff --git a/nemo/collections/nlp/modules/common/megatron/megatron_encoders.py b/nemo/collections/nlp/modules/common/megatron/megatron_encoders.py index 7b83601f3e59..ddd146d81c3b 100644 --- a/nemo/collections/nlp/modules/common/megatron/megatron_encoders.py +++ b/nemo/collections/nlp/modules/common/megatron/megatron_encoders.py @@ -13,6 +13,7 @@ # limitations under the License. """Transformer based language model.""" +from nemo.collections.nlp.modules.common.megatron.megatron_perceiver_encoders import MegatronPerceiverEncoderModule from nemo.collections.nlp.modules.common.megatron.megatron_transformer_encoder import MegatronTransformerEncoderModule from nemo.collections.nlp.modules.common.megatron.retrieval_transformer import ( MegatronRetrievalTransformerEncoderModule, @@ -35,7 +36,7 @@ __all__ = [] -AVAILABLE_ENCODERS = ["transformer"] +AVAILABLE_ENCODERS = ["transformer", "perceiver", "retro"] def get_encoder_model( @@ -74,11 +75,12 @@ def get_encoder_model( normalization="layernorm", headscale=False, transformer_block_type="pre_ln", - hidden_steps=-1, + hidden_steps=32, hidden_blocks=1, parent_model_type=ModelType.encoder_or_decoder, layer_type=None, chunk_size=64, + num_self_attention_per_cross_attention=1, layer_number_offset=0, # this is use only for attention norm_factor scaling ): """Build language model and return along with the key to save.""" @@ -168,6 +170,45 @@ def get_encoder_model( chunk_size=chunk_size, layer_number_offset=layer_number_offset, ) + elif arch == "perceiver": + encoder = MegatronPerceiverEncoderModule( + init_method=init_method, + output_layer_init_method=scaled_init_method, + hidden_size=hidden_size, + num_layers=num_layers, + num_attention_heads=num_attention_heads, + apply_query_key_layer_scaling=apply_query_key_layer_scaling, + kv_channels=kv_channels, + ffn_hidden_size=ffn_hidden_size, + encoder_attn_mask_type=encoder_attn_mask_type, + pre_process=pre_process, + post_process=post_process, + use_cpu_initialization=use_cpu_initialization, + hidden_dropout=hidden_dropout, + attention_dropout=attention_dropout, + position_embedding_type=position_embedding_type, + relative_attention_num_buckets=relative_attention_num_buckets, + relative_attention_max_distance=relative_attention_max_distance, + precision=precision, + fp32_residual_connection=fp32_residual_connection, + activations_checkpoint_method=activations_checkpoint_method, + activations_checkpoint_num_layers=activations_checkpoint_num_layers, + layernorm_epsilon=layernorm_epsilon, + bias_activation_fusion=bias_activation_fusion, + bias_dropout_add_fusion=bias_dropout_add_fusion, + masked_softmax_fusion=masked_softmax_fusion, + persist_layer_norm=persist_layer_norm, + openai_gelu=openai_gelu, + onnx_safe=onnx_safe, + activation=activation, + bias=bias, + normalization=normalization, + transformer_block_type=transformer_block_type, + headscale=headscale, + parent_model_type=parent_model_type, + hidden_steps=hidden_steps, + num_self_attention_per_cross_attention=num_self_attention_per_cross_attention, + ) else: raise ValueError(f"Unknown encoder arch = {arch}. Available encoder arch = {AVAILABLE_ENCODERS}") diff --git a/nemo/collections/nlp/modules/common/megatron/megatron_perceiver_encoders.py b/nemo/collections/nlp/modules/common/megatron/megatron_perceiver_encoders.py new file mode 100644 index 000000000000..d20a9309dac0 --- /dev/null +++ b/nemo/collections/nlp/modules/common/megatron/megatron_perceiver_encoders.py @@ -0,0 +1,270 @@ +# Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Transformer based language model.""" +import torch + +from nemo.collections.nlp.modules.common.megatron.fused_layer_norm import get_layer_norm +from nemo.collections.nlp.modules.common.megatron.layer_type import LayerType +from nemo.collections.nlp.modules.common.megatron.module import MegatronModule +from nemo.collections.nlp.modules.common.megatron.transformer import ParallelTransformer +from nemo.collections.nlp.modules.common.megatron.utils import ( + ApexGuardDefaults, + attn_mask_postprocess, + build_attention_mask_3d, +) + +try: + from apex.transformer.enums import AttnMaskType, ModelType + from apex.normalization import MixedFusedRMSNorm + + HAVE_APEX = True +except (ImportError, ModuleNotFoundError): + HAVE_APEX = False + # fake missing classes with None attributes + AttnMaskType = ApexGuardDefaults() + ModelType = ApexGuardDefaults() + +__all__ = ["MegatronPerceiverEncoderModule"] + + +class MegatronPerceiverEncoderModule(MegatronModule): + """Transformer encoder model. + """ + + def __init__( + self, + init_method, + output_layer_init_method, + hidden_size, + ffn_hidden_size, + num_layers, + num_attention_heads, + apply_query_key_layer_scaling=True, + kv_channels=None, + pre_process=True, + post_process=True, + use_cpu_initialization=False, + encoder_attn_mask_type=AttnMaskType.padding, + hidden_dropout=0.1, + attention_dropout=0.1, + position_embedding_type='learned_absolute', + relative_attention_num_buckets=32, + relative_attention_max_distance=128, + precision=16, + fp32_residual_connection=False, + activations_checkpoint_method=None, + activations_checkpoint_num_layers=1, + layernorm_epsilon=1e-5, + bias_activation_fusion=True, + bias_dropout_add_fusion=True, + masked_softmax_fusion=True, + persist_layer_norm=False, + openai_gelu=False, + onnx_safe=False, + activation='gelu', + bias=True, + normalization='layernorm', + transformer_block_type='pre_ln', + headscale=False, + parent_model_type=ModelType.encoder_or_decoder, + hidden_steps=32, + num_self_attention_per_cross_attention=1, + ): + super(MegatronPerceiverEncoderModule, self).__init__() + + self.pre_process = pre_process + self.post_process = post_process + self.hidden_size = hidden_size + self.num_layers = num_layers + self.init_method = init_method + self.model_attn_mask_type = encoder_attn_mask_type + self.hidden_dropout = hidden_dropout + self.output_layer_init_method = output_layer_init_method + self.parent_model_type = parent_model_type + self.normalization = normalization + self.transformer_block_type = transformer_block_type + self.hidden_steps = hidden_steps + self.num_self_attention_per_cross_attention = num_self_attention_per_cross_attention + self.num_attention_heads = num_attention_heads + self.apply_query_key_layer_scaling = apply_query_key_layer_scaling + self.kv_channels = kv_channels + self.ffn_hidden_size = ffn_hidden_size + self.precision = precision + self.fp32_residual_connection = fp32_residual_connection + self.activations_checkpoint_method = activations_checkpoint_method + self.activations_checkpoint_num_layers = activations_checkpoint_num_layers + self.layernorm_epsilon = layernorm_epsilon + self.bias_activation_fusion = bias_activation_fusion + self.bias_dropout_add_fusion = bias_dropout_add_fusion + self.masked_softmax_fusion = masked_softmax_fusion + self.persist_layer_norm = persist_layer_norm + self.openai_gelu = openai_gelu + self.onnx_safe = onnx_safe + self.activation = activation + self.bias = bias + self.relative_attention_num_buckets = relative_attention_num_buckets + self.relative_attention_max_distance = relative_attention_max_distance + self.headscale = headscale + self.hidden_dropout = hidden_dropout + self.attention_dropout = attention_dropout + self.position_embedding_type = position_embedding_type + self.use_cpu_initialization = use_cpu_initialization + self.normalization = normalization + self.parent_model_type = parent_model_type + self.transformer_block_type = transformer_block_type + + assert self.num_self_attention_per_cross_attention >= 1 + assert self.hidden_steps >= 1 + + self.init_hidden = torch.nn.Parameter(torch.nn.init.xavier_normal_(torch.empty(hidden_steps, hidden_size))) + + self.cross_attn_layers = torch.nn.ModuleList([self._build_cross_attn_layer() for _ in range(self.num_layers)]) + self.self_attn_layers = torch.nn.ModuleList( + [ + self._build_self_attn_layer() + for _ in range(self.num_layers * self.num_self_attention_per_cross_attention) + ] + ) + if normalization == 'layernorm': + self.final_layernorm = get_layer_norm(hidden_size, layernorm_epsilon, persist_layer_norm) + else: + self.final_layernorm = MixedFusedRMSNorm(hidden_size, layernorm_epsilon) + + def _build_cross_attn_layer(self): + return ParallelTransformer( + layer_type=LayerType.decoder, + init_method=self.init_method, + output_layer_init_method=self.output_layer_init_method, + num_layers=1, + hidden_size=self.hidden_size, + num_attention_heads=self.num_attention_heads, + apply_query_key_layer_scaling=self.apply_query_key_layer_scaling, + kv_channels=self.kv_channels, + ffn_hidden_size=self.ffn_hidden_size, + self_attn_mask_type=self.model_attn_mask_type, + pre_process=self.pre_process, + post_process=False, # This is to avoid the final layernorm and transpose. + precision=self.precision, + fp32_residual_connection=self.fp32_residual_connection, + activations_checkpoint_method=self.activations_checkpoint_method, + activations_checkpoint_num_layers=self.activations_checkpoint_num_layers, + layernorm_epsilon=self.layernorm_epsilon, + hidden_dropout=self.hidden_dropout, + attention_dropout=self.attention_dropout, + position_embedding_type=self.position_embedding_type, + relative_attention_num_buckets=self.relative_attention_num_buckets, + relative_attention_max_distance=self.relative_attention_max_distance, + use_cpu_initialization=self.use_cpu_initialization, + bias_activation_fusion=self.bias_activation_fusion, + bias_dropout_fusion=self.bias_dropout_add_fusion, + masked_softmax_fusion=self.masked_softmax_fusion, + persist_layer_norm=self.persist_layer_norm, + openai_gelu=self.openai_gelu, + onnx_safe=self.onnx_safe, + activation=self.activation, + bias=self.bias, + normalization=self.normalization, + model_type=self.parent_model_type, + transformer_block_type=self.transformer_block_type, + headscale=self.headscale, + ) + + def _build_self_attn_layer(self): + return ParallelTransformer( + layer_type=LayerType.encoder, + init_method=self.init_method, + output_layer_init_method=self.output_layer_init_method, + num_layers=1, + hidden_size=self.hidden_size, + num_attention_heads=self.num_attention_heads, + apply_query_key_layer_scaling=self.apply_query_key_layer_scaling, + kv_channels=self.kv_channels, + ffn_hidden_size=self.ffn_hidden_size, + self_attn_mask_type=self.model_attn_mask_type, + pre_process=self.pre_process, + post_process=False, # This is to avoid the final layernorm and transpose. + precision=self.precision, + fp32_residual_connection=self.fp32_residual_connection, + activations_checkpoint_method=self.activations_checkpoint_method, + activations_checkpoint_num_layers=self.activations_checkpoint_num_layers, + layernorm_epsilon=self.layernorm_epsilon, + hidden_dropout=self.hidden_dropout, + attention_dropout=self.attention_dropout, + position_embedding_type=self.position_embedding_type, + relative_attention_num_buckets=self.relative_attention_num_buckets, + relative_attention_max_distance=self.relative_attention_max_distance, + use_cpu_initialization=self.use_cpu_initialization, + bias_activation_fusion=self.bias_activation_fusion, + bias_dropout_fusion=self.bias_dropout_add_fusion, + masked_softmax_fusion=self.masked_softmax_fusion, + persist_layer_norm=self.persist_layer_norm, + openai_gelu=self.openai_gelu, + onnx_safe=self.onnx_safe, + activation=self.activation, + bias=self.bias, + normalization=self.normalization, + model_type=self.parent_model_type, + transformer_block_type=self.transformer_block_type, + headscale=self.headscale, + ) + + def set_input_tensor(self, input_tensor): + """ See megatron.model.transformer.set_input_tensor()""" + # TODO: Fix this when adding support for Pipeline Parallel. + pass + + def forward( + self, enc_input, enc_attn_mask, layer_past=None, get_key_value=False, + ): + # convert to Megatron mask + latent_attention_mask = torch.ones(enc_input.size(0), self.hidden_steps).to(enc_input.device) + + # First convert from 2D (B x T) to 3D (B x T x T) + # Next convert to 4D (B x 1 x T x T) - unsqueeze(1) is for the head dim. + latent_attention_mask_4d = attn_mask_postprocess( + build_attention_mask_3d( + source_mask=latent_attention_mask, + target_mask=latent_attention_mask, + attn_mask_type=AttnMaskType.padding, + ) + ) + enc_dec_attn_mask_4d = attn_mask_postprocess( + build_attention_mask_3d( + source_mask=latent_attention_mask, target_mask=enc_attn_mask, attn_mask_type=AttnMaskType.padding, + ) + ) + + hidden_states = self.init_hidden.unsqueeze(0).expand(enc_input.size(0), -1, -1) # sequence x batch x dim + for i in range(self.num_layers): + residual = hidden_states + + hidden_states = self.cross_attn_layers[i]( + hidden_states=hidden_states, + attention_mask=latent_attention_mask_4d, + enc_dec_attn_mask=enc_dec_attn_mask_4d, + encoder_output=enc_input, + ).transpose( + 1, 0 + ) # Need to transpose at the end becase pre-process is False + for j in range(self.num_self_attention_per_cross_attention): + hidden_states = self.self_attn_layers[i * self.num_self_attention_per_cross_attention + j]( + hidden_states=hidden_states, attention_mask=latent_attention_mask_4d, + ).transpose( + 1, 0 + ) # Need to transpose at the end becase pre-process is False + + hidden_states += residual + + return self.final_layernorm(hidden_states) # Need to transpose at the end becase pre-process is False diff --git a/nemo/collections/nlp/modules/common/megatron/retrieval_token_level_encoder_decoder.py b/nemo/collections/nlp/modules/common/megatron/retrieval_token_level_encoder_decoder.py index c0bb96588135..6a7136dd803a 100644 --- a/nemo/collections/nlp/modules/common/megatron/retrieval_token_level_encoder_decoder.py +++ b/nemo/collections/nlp/modules/common/megatron/retrieval_token_level_encoder_decoder.py @@ -80,7 +80,6 @@ def __init__( headscale=False, transformer_block_type='pre_ln', hidden_steps=-1, - hidden_blocks=1, add_encoder=True, add_decoder=True, chunk_size=64, @@ -168,7 +167,6 @@ def __init__( openai_gelu=openai_gelu, onnx_safe=onnx_safe, hidden_steps=hidden_steps, - hidden_blocks=hidden_blocks, activation=activation, bias=bias, normalization=normalization, @@ -229,7 +227,6 @@ def __init__( openai_gelu=openai_gelu, onnx_safe=onnx_safe, hidden_steps=hidden_steps, - hidden_blocks=hidden_blocks, activation=activation, bias=bias, normalization=normalization, @@ -270,7 +267,6 @@ def __init__( openai_gelu=openai_gelu, onnx_safe=onnx_safe, hidden_steps=hidden_steps, - hidden_blocks=hidden_blocks, activation=activation, bias=bias, normalization=normalization, diff --git a/nemo/collections/nlp/modules/common/megatron/token_level_encoder_decoder.py b/nemo/collections/nlp/modules/common/megatron/token_level_encoder_decoder.py index ab915ae6ff92..d5aaab3bc84b 100644 --- a/nemo/collections/nlp/modules/common/megatron/token_level_encoder_decoder.py +++ b/nemo/collections/nlp/modules/common/megatron/token_level_encoder_decoder.py @@ -111,10 +111,10 @@ def __init__( normalization='layernorm', transformer_block_type='pre_ln', hidden_steps=-1, - hidden_blocks=1, headscale=False, add_encoder=True, add_decoder=True, + num_self_attention_per_cross_attention=1, ): super(MegatronTokenLevelEncoderDecoderModule, self).__init__() @@ -190,13 +190,13 @@ def __init__( openai_gelu=openai_gelu, onnx_safe=onnx_safe, hidden_steps=hidden_steps, - hidden_blocks=hidden_blocks, activation=activation, bias=bias, normalization=normalization, transformer_block_type=transformer_block_type, headscale=headscale, parent_model_type=ModelType.encoder_and_decoder, + num_self_attention_per_cross_attention=num_self_attention_per_cross_attention, ) if add_decoder: @@ -255,7 +255,6 @@ def __init__( openai_gelu=openai_gelu, onnx_safe=onnx_safe, hidden_steps=hidden_steps, - hidden_blocks=hidden_blocks, activation=activation, bias=bias, normalization=normalization, @@ -264,7 +263,9 @@ def __init__( parent_model_type=ModelType.encoder_and_decoder, ) - self.enc_dec_model = MegatronTransformerEncoderDecoderModule(encoder=encoder, decoder=decoder) + self.enc_dec_model = MegatronTransformerEncoderDecoderModule( + encoder=encoder, decoder=decoder, hidden_steps=hidden_steps + ) self._enc_dec_model_key = "enc_dec_model" self.initialize_word_embeddings( diff --git a/nemo/collections/nlp/modules/common/megatron/transformer.py b/nemo/collections/nlp/modules/common/megatron/transformer.py index d1cdf7eb2211..71ee082bc0b4 100644 --- a/nemo/collections/nlp/modules/common/megatron/transformer.py +++ b/nemo/collections/nlp/modules/common/megatron/transformer.py @@ -234,7 +234,9 @@ def forward(self, hidden_states): intermediate_parallel, bias_parallel, intermediate_parallel_2, bias_parallel_2 ) - elif self.activation in ['reglu', 'swiglu']: + elif self.activation in ['reglu', 'swiglu'] or ( + self.glu_activation_family and not self.bias_activation_fusion + ): if bias_parallel is not None: intermediate_parallel = self.activation_func(intermediate_parallel + bias_parallel) * ( intermediate_parallel_2 + bias_parallel_2 From 4dd0e56fb2ac5294e0ea0226f5e1de1b574b65c9 Mon Sep 17 00:00:00 2001 From: Taejin Park Date: Wed, 6 Jul 2022 22:01:20 -0700 Subject: [PATCH 04/52] NMESC speaker counting algorithm update (#4500) * initial commit Signed-off-by: Taejin Park * style fix Signed-off-by: Taejin Park * Default maj_vote = False, max_rp=0.25 Signed-off-by: Taejin Park * doc strings and style fix Signed-off-by: Taejin Park * Docstring minor edit Signed-off-by: Taejin Park * Default False in the functions Signed-off-by: Taejin Park * fixed repeated variable Signed-off-by: Taejin Park * Default as maj_vote=False Signed-off-by: Taejin Park * removed redundant part in wrtie_rttm func Signed-off-by: Taejin Park * Removed unused function Signed-off-by: Taejin Park * Updated and tested silence and very short samples Signed-off-by: Taejin Park * style fix Signed-off-by: Taejin Park * Style fix and removing unnecessary parts Signed-off-by: Taejin Park * unused variables are removed Signed-off-by: Taejin Park * Fixed commented torch.jit.script Signed-off-by: Taejin Park * majority voting update Signed-off-by: Taejin Park * cancelling the update on speaker_utils and clus_diarizer Signed-off-by: Taejin Park * style fix Signed-off-by: Taejin Park * bug fix Signed-off-by: Taejin Park * Added fp32 converting for torch.mm Signed-off-by: Taejin Park Co-authored-by: Nithin Rao --- .../diarization/conf/offline_diarization.yaml | 3 +- .../conf/offline_diarization_with_asr.yaml | 1 + .../asr/parts/utils/nmesc_clustering.py | 86 +++++++++++++++---- 3 files changed, 71 insertions(+), 19 deletions(-) diff --git a/examples/speaker_tasks/diarization/conf/offline_diarization.yaml b/examples/speaker_tasks/diarization/conf/offline_diarization.yaml index c433c4fc6e5d..ac95c6de93c8 100644 --- a/examples/speaker_tasks/diarization/conf/offline_diarization.yaml +++ b/examples/speaker_tasks/diarization/conf/offline_diarization.yaml @@ -43,6 +43,7 @@ diarizer: enhanced_count_thres: 80 # If the number of segments is lower than this number, enhanced speaker counting is activated. max_rp_threshold: 0.25 # Determines the range of p-value search: 0 < p <= max_rp_threshold. sparse_search_volume: 30 # The higher the number, the more values will be examined with more time. + maj_vote_spk_count: False # If True, take a majority vote on multiple p-values to estimate the number of speakers. # json manifest line example -# {"audio_filepath": "/path/to/audio_file", "offset": 0, "duration": null, "label": "infer", "text": "-", "num_speakers": null, "rttm_filepath": "/path/to/rttm/file", "uem_filepath": "/path/to/uem/filepath"} \ No newline at end of file +# {"audio_filepath": "/path/to/audio_file", "offset": 0, "duration": null, "label": "infer", "text": "-", "num_speakers": null, "rttm_filepath": "/path/to/rttm/file", "uem_filepath": "/path/to/uem/filepath"} diff --git a/examples/speaker_tasks/diarization/conf/offline_diarization_with_asr.yaml b/examples/speaker_tasks/diarization/conf/offline_diarization_with_asr.yaml index b5d7a24574a2..475f950071a7 100644 --- a/examples/speaker_tasks/diarization/conf/offline_diarization_with_asr.yaml +++ b/examples/speaker_tasks/diarization/conf/offline_diarization_with_asr.yaml @@ -43,6 +43,7 @@ diarizer: enhanced_count_thres: 80 # If the number of segments is lower than this number, enhanced speaker counting is activated. max_rp_threshold: 0.25 # Determines the range of p-value search: 0 < p <= max_rp_threshold. sparse_search_volume: 30 # The higher the number, the more values will be examined with more time. + maj_vote_spk_count: False # If True, take a majority vote on multiple p-values to estimate the number of speakers. asr: model_path: ??? # Provide NGC cloud ASR model name. stt_en_conformer_ctc_* models are recommended for diarization purposes. diff --git a/nemo/collections/asr/parts/utils/nmesc_clustering.py b/nemo/collections/asr/parts/utils/nmesc_clustering.py index 2c615bee7ee5..398609cc2b76 100644 --- a/nemo/collections/asr/parts/utils/nmesc_clustering.py +++ b/nemo/collections/asr/parts/utils/nmesc_clustering.py @@ -441,12 +441,21 @@ def getMultiScaleCosAffinityMatrix(uniq_embs_and_timestamps: dict, device: torch @torch.jit.script -def getCosAffinityMatrix(_emb: torch.Tensor): +def getCosAffinityMatrix(emb: torch.Tensor): """ Calculate cosine similarity values among speaker embeddings then min-max normalize the affinity matrix. + Args: + emb: (torch.tensor) + Matrix containing embedding vectors. emb variable should be float(FP32) type to make the data-type + compatible with torch.mm operation for both CPU and GPU(CUDA). + dimension: (Number of embedding vectors) x (embedding dimension) + Returns: + sim_d: (torch.tensor) + Matrix containing cosine similarity values among the given embedding vectors. + dimension: (Number of embedding vectors) x (Number of embedding vectors) """ - emb = _emb.half() + emb = emb.float() sim_d = cos_similarity(emb, emb) sim_d = ScalerMinMax(sim_d) return sim_d @@ -788,10 +797,11 @@ def __init__( max_rp_threshold: float = 0.15, sparse_search: bool = True, sparse_search_volume: int = 30, + NME_mat_size: int = 512, use_subsampling_for_NME: bool = True, fixed_thres: float = 0.0, + maj_vote_spk_count: bool = False, cuda: bool = False, - NME_mat_size: int = 512, device: torch.device = torch.device('cpu'), ): """ @@ -826,6 +836,11 @@ def __init__( threshold with NME analysis. If fixed_thres is float, it skips the NME analysis part. + maj_vote_spk_count: (bool) + If True, take a majority vote on all p-values in the given range to estimate the number of speakers. + The majority voting may contribute to surpress overcounting of the speakers and improve speaker + counting accuracy. + cuda (bool) Use cuda for Eigen decomposition if cuda=True. @@ -840,13 +855,15 @@ def __init__( self.NME_mat_size: int = NME_mat_size self.sparse_search = sparse_search self.sparse_search_volume = sparse_search_volume + self.min_p_value = torch.tensor(2) self.fixed_thres: float = fixed_thres self.cuda: bool = cuda self.eps = 1e-10 self.max_N = torch.tensor(0) self.mat = mat - self.p_value_list: torch.Tensor = torch.tensor(0) + self.p_value_list: torch.Tensor = self.min_p_value.unsqueeze(0) self.device = device + self.maj_vote_spk_count = maj_vote_spk_count def NMEanalysis(self): """ @@ -859,13 +876,14 @@ def NMEanalysis(self): # Scans p_values and find a p_value that generates # the smallest g_p value. - eig_ratio_list = [] + eig_ratio_list, est_num_of_spk_list = [], [] est_spk_n_dict: Dict[int, torch.Tensor] = {} self.p_value_list = self.getPvalueList() for p_value in self.p_value_list: est_num_of_spk, g_p = self.getEigRatio(p_value) est_spk_n_dict[p_value.item()] = est_num_of_spk eig_ratio_list.append(g_p) + est_num_of_spk_list.append(est_num_of_spk) index_nn = torch.argmin(torch.tensor(eig_ratio_list)) rp_p_value = self.p_value_list[index_nn] affinity_mat = getAffinityGraphMat(self.mat, rp_p_value) @@ -878,7 +896,10 @@ def NMEanalysis(self): ) p_hat_value = (subsample_ratio * rp_p_value).type(torch.int) - est_num_of_spk = est_spk_n_dict[rp_p_value.item()] + if self.maj_vote_spk_count: + est_num_of_spk = torch.mode(torch.tensor(est_num_of_spk_list))[0] + else: + est_num_of_spk = est_spk_n_dict[rp_p_value.item()] return est_num_of_spk, p_hat_value def subsampleAffinityMat(self, NME_mat_size: int): @@ -923,7 +944,6 @@ def getEigRatio(self, p_neighbors: int): Returns: est_num_of_spk: (int) Estimated number of speakers - g_p: (float) The ratio between p_neighbors value and the maximum eigen gap value. """ @@ -937,19 +957,39 @@ def getEigRatio(self, p_neighbors: int): def getPvalueList(self): """ - Generates a p-value (p_neighbour) list for searching. + Generates a p-value (p_neighbour) list for searching. p_value_list must include 2 (min_p_value) + since at least one neighboring segment should be selected other than itself. + + If fixed_thres value is specified, then only one p-value is specified. + If fixed_thres is not provided, multiple p-values are searched. + If sparse_search is True: + - Limit the number of p-values to be searched to sparse_search_volume. + - N should be at least 2 to include a number greater than 1. + If sparse_search is False: + - Scan all the p_values from 1 to max_N + - If sparse_search is False, NMESC analysis could take more time compared to sparse_search = True. + + Returns: + p_value_list: (torch.tensor) + Tensor containing the p_values to be searched. """ - if self.fixed_thres > 0.0: - p_value_list = torch.floor(torch.tensor(self.mat.shape[0] * self.fixed_thres)).type(torch.int) - self.max_N = p_value_list[0] + if self.fixed_thres is not None and self.fixed_thres > 0.0: + self.max_N = torch.max( + torch.floor(torch.tensor(self.mat.shape[0] * self.fixed_thres)).type(torch.int), self.min_p_value + ) + p_value_list = torch.tensor(self.max_N).unsqueeze(0) else: - self.max_N = torch.floor(torch.tensor(self.mat.shape[0] * self.max_rp_threshold)).type(torch.int) + self.max_N = torch.max( + torch.floor(torch.tensor(self.mat.shape[0] * self.max_rp_threshold)).type(torch.int), self.min_p_value + ) if self.sparse_search: - N = torch.min(self.max_N, torch.tensor(self.sparse_search_volume).type(torch.int)) + search_volume = torch.min(self.max_N, torch.tensor(self.sparse_search_volume).type(torch.int)) + N = torch.max(search_volume, torch.tensor(2)) p_value_list = torch.unique(torch.linspace(start=1, end=self.max_N, steps=N).type(torch.int)) else: - p_value_list = torch.arange(1, self.max_N) - + p_value_list = torch.arange(1, self.max_N + 1) + if p_value_list.shape[0] == 0: + raise ValueError("p_value_list should not be empty.") return p_value_list @@ -960,7 +1000,9 @@ def COSclustering( min_samples_for_NMESC: int = 6, enhanced_count_thres: int = 80, max_rp_threshold: float = 0.15, + sparse_search: bool = True, sparse_search_volume: int = 30, + maj_vote_spk_count: bool = False, fixed_thres: float = 0.0, cuda=False, ): @@ -997,6 +1039,14 @@ def COSclustering( Clustering performance can vary depending on this range. Default is 0.15. + maj_vote_spk_count: (bool) + If True, take a majority vote on all p-values in the given range to estimate the number of speakers. + The majority voting may contribute to surpress overcounting of the speakers and improve speaker + counting accuracy. + + sparse_search: (bool) + Toggle sparse search mode. If True, limit the size of p_value_list to sparse_search_volume. + sparse_search_volume: (int) Number of p_values we search during NME analysis. Default is 30. The lower the value, the faster NME-analysis becomes. @@ -1018,7 +1068,7 @@ def COSclustering( emb = uniq_scale_dict[max(uniq_scale_dict.keys())]['embeddings'] if emb.shape[0] == 1: - return torch.zeros((1,), dtype=torch.int32) + return torch.zeros((1,), dtype=torch.int32).cpu().numpy() elif emb.shape[0] <= max(enhanced_count_thres, min_samples_for_NMESC) and oracle_num_speakers is None: est_num_of_spk_enhanced = getEnhancedSpeakerCount(emb, cuda) else: @@ -1033,10 +1083,11 @@ def COSclustering( mat, max_num_speaker=max_num_speaker, max_rp_threshold=max_rp_threshold, - sparse_search=True, + sparse_search=sparse_search, sparse_search_volume=sparse_search_volume, fixed_thres=fixed_thres, NME_mat_size=300, + maj_vote_spk_count=maj_vote_spk_count, cuda=cuda, device=device, ) @@ -1054,5 +1105,4 @@ def COSclustering( spectral_model = SpectralClustering(n_clusters=est_num_of_spk, cuda=cuda, device=device) Y = spectral_model.predict(affinity_mat) - return Y.cpu().numpy() From f21cf36bbf017a76b3d6231fbfc9d770e623458b Mon Sep 17 00:00:00 2001 From: Alexander Stupnikov Date: Thu, 7 Jul 2022 11:09:31 +0500 Subject: [PATCH 05/52] Fix dataset parameter typo on tacotron2 example yaml (#4471) Signed-off-by: saarus72 Co-authored-by: Xuesong Yang <1646669+XuesongYang@users.noreply.github.com> --- examples/tts/conf/tacotron2.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/tts/conf/tacotron2.yaml b/examples/tts/conf/tacotron2.yaml index 5620bfdca616..31d4ea5d1c0e 100644 --- a/examples/tts/conf/tacotron2.yaml +++ b/examples/tts/conf/tacotron2.yaml @@ -83,7 +83,7 @@ model: validation_ds: dataset: _target_: "nemo.collections.tts.torch.data.TTSDataset" - manifest_filepath: ${train_dataset} + manifest_filepath: ${validation_datasets} sample_rate: ${model.sample_rate} sup_data_path: ${sup_data_path} sup_data_types: ${sup_data_types} From cf95f93ceb378944184f5ddf63dc4c3f29bbfe6d Mon Sep 17 00:00:00 2001 From: Adrian Lancucki <16889482+alancucki@users.noreply.github.com> Date: Thu, 7 Jul 2022 10:19:02 +0200 Subject: [PATCH 06/52] Noam lr sched: do not force min_lr after max_steps (#4472) Signed-off-by: Adrian Lancucki Co-authored-by: Adrian Lancucki Co-authored-by: Xuesong Yang <1646669+XuesongYang@users.noreply.github.com> --- nemo/core/optim/lr_scheduler.py | 9 +++-- tests/core/test_optimizers_schedulers.py | 51 ++++++++++++++++++++++++ 2 files changed, 56 insertions(+), 4 deletions(-) diff --git a/nemo/core/optim/lr_scheduler.py b/nemo/core/optim/lr_scheduler.py index 21da56964cbe..92da9938703f 100644 --- a/nemo/core/optim/lr_scheduler.py +++ b/nemo/core/optim/lr_scheduler.py @@ -472,9 +472,6 @@ def get_lr(self): step = max(1, self.last_epoch) - if step > self.max_steps: - return [self.min_lr for _ in self.base_lrs] - for initial_lr in self.base_lrs: if initial_lr < self.min_lr: raise ValueError( @@ -485,7 +482,11 @@ def get_lr(self): return new_lrs def _noam_annealing(self, initial_lr, step): - mult = self._normalize * min(step ** (-0.5), step * (self.warmup_steps ** (-1.5))) + if self.warmup_steps > 0: + mult = self._normalize * min(step ** (-0.5), step * (self.warmup_steps ** (-1.5))) + else: + mult = self._normalize * step ** (-0.5) + out_lr = initial_lr * mult if step > self.warmup_steps: out_lr = max(out_lr, self.min_lr) diff --git a/tests/core/test_optimizers_schedulers.py b/tests/core/test_optimizers_schedulers.py index 283c967231c8..2bf827f61283 100644 --- a/tests/core/test_optimizers_schedulers.py +++ b/tests/core/test_optimizers_schedulers.py @@ -132,6 +132,7 @@ class TestOptimizersSchedulers: INITIAL_LR = 0.1 MIN_LR = 1e-3 MAX_STEPS = 10 + D_MODEL = 16 # fused_adam is looking for CUDA and this test is being run on CPU only tests @pytest.mark.unit @@ -616,6 +617,56 @@ def test_CosineAnnealing(self): assert final_lr == self.MIN_LR + # Noam scheduler should decay past MAX_STEPS - run two schedulers in parallel to test it + @pytest.mark.unit + def test_NoamAnnealing(self): + model = TempModel() + opt_cls = optim.get_optimizer('novograd') + opt1 = opt_cls(model.parameters(), lr=self.INITIAL_LR) + opt2 = opt_cls(model.parameters(), lr=self.INITIAL_LR) + + # No warmup case + policy1 = optim.lr_scheduler.NoamAnnealing( + opt1, d_model=self.D_MODEL, max_steps=self.MAX_STEPS, min_lr=self.MIN_LR + ) + policy2 = optim.lr_scheduler.NoamAnnealing( + opt2, d_model=self.D_MODEL, max_steps=self.MAX_STEPS * 2, min_lr=self.MIN_LR + ) + initial_lr = policy1.get_last_lr()[0] + + assert initial_lr == self.D_MODEL ** (-0.5) * self.INITIAL_LR + + for i in range(self.MAX_STEPS * 2): + assert self.MIN_LR < policy1.get_last_lr()[0] <= self.INITIAL_LR + assert policy1.get_last_lr()[0] == policy2.get_last_lr()[0] + opt1.step() + opt2.step() + policy1.step() + policy2.step() + + # Warmup steps available + policy1 = optim.lr_scheduler.NoamAnnealing( + opt1, d_model=self.D_MODEL, warmup_steps=5, max_steps=self.MAX_STEPS, min_lr=self.MIN_LR + ) + policy2 = optim.lr_scheduler.NoamAnnealing( + opt2, d_model=self.D_MODEL, warmup_steps=5, max_steps=self.MAX_STEPS * 2, min_lr=self.MIN_LR + ) + initial_lr = policy1.get_last_lr()[0] + + assert initial_lr < self.INITIAL_LR + + for i in range(self.MAX_STEPS * 2): + if i <= 5: + assert policy1.get_last_lr()[0] <= self.INITIAL_LR + else: + assert self.MIN_LR < policy1.get_last_lr()[0] < self.INITIAL_LR + assert policy1.get_last_lr()[0] == policy2.get_last_lr()[0] + + opt1.step() + opt2.step() + policy1.step() + policy2.step() + @pytest.mark.unit def test_PolynomialDecayAnnealing(self): model = TempModel() From 1f97094474999d14515b675a995141614b842bfc Mon Sep 17 00:00:00 2001 From: Matvei Novikov Date: Thu, 7 Jul 2022 20:07:36 +0500 Subject: [PATCH 07/52] Refactor for punctuation model (#4367) * Dataloader, collector, loss and metric for multiscale diarization decoder (#4187) * First commit Signed-off-by: Taejin Park * Checked funtionality and imports Signed-off-by: Taejin Park * fixed import issues Signed-off-by: Taejin Park * Removed the changed made by mistake Signed-off-by: Taejin Park * Style fix Signed-off-by: Taejin Park * Fixed LGTM errors 001 Signed-off-by: Taejin Park * Fixed LGTM and style fix Signed-off-by: Taejin Park * Changed docstrings Signed-off-by: Taejin Park * LGTM again Signed-off-by: Taejin Park * Removed unnecessary torch setting lines Signed-off-by: Taejin Park * Style fix and isort Signed-off-by: Taejin Park * jbalam-nv comments reflected Signed-off-by: Taejin Park * style fix Signed-off-by: Taejin Park * Reflected comments and created _diar_label.py Signed-off-by: Taejin Park * Typo fix and style fix Signed-off-by: Taejin Park * Fixed target_spks[0] index error Signed-off-by: Taejin Park * style fix Signed-off-by: Taejin Park * LGTM unused import IterDataset Signed-off-by: Taejin Park * revert collection doc year Signed-off-by: Taejin Park * Code format error in collections.py Signed-off-by: Taejin Park * fix collections space format error Signed-off-by: Taejin Park * merged main correctly Signed-off-by: Taejin Park * style fix Signed-off-by: Taejin Park * Reflected all comments and tested Signed-off-by: Taejin Park * style fix and LGTM Signed-off-by: Taejin Park * rttm_filepath to rttm_file and removed self included funcs, tested Signed-off-by: Taejin Park Co-authored-by: Nithin Rao Signed-off-by: Matvei Novikov * removed references to data_dir Signed-off-by: Matvei Novikov * added missing parameters to data preparation script Signed-off-by: Matvei Novikov * removed unnecessary file extension check Signed-off-by: Matvei Novikov * Add ASR CTC Decoding module (#4342) * Initial commit Signed-off-by: smajumdar * Full support for decoding strategy Signed-off-by: smajumdar * Temp Signed-off-by: smajumdar * Fix labels of y_sequence Signed-off-by: smajumdar * Set support for sentencepiece subword merging Signed-off-by: smajumdar * Fix char and word based token merge alignment Signed-off-by: smajumdar * Revert incorrect change Signed-off-by: smajumdar * Update docstring Signed-off-by: smajumdar * Improve compatibility with greedy tokens and log probs Signed-off-by: smajumdar * Update scripts to use decoding strategy Signed-off-by: smajumdar * Add tests and docs Signed-off-by: smajumdar * Add tests and docs Signed-off-by: smajumdar * Fix speaker decoder timestamps Signed-off-by: smajumdar * Fix speaker decoder timestamps Signed-off-by: smajumdar * Fix decoding of ctc models Signed-off-by: smajumdar * Address reviewer comments Signed-off-by: smajumdar * Address reviewer comments Signed-off-by: smajumdar Signed-off-by: Matvei Novikov * Option to disable mp in VAD via num_workers=1 (#4317) * Option to disable mp in VAD via num_workers=1 In certain environments python multiprocessing can deadlock. This adds a convenient version to disable by setting num_workers to 1. Signed-off-by: Georg Kucsko * add none handling Signed-off-by: Georg Kucsko * additional none handling Signed-off-by: Georg Kucsko Co-authored-by: fayejf <36722593+fayejf@users.noreply.github.com> Signed-off-by: Matvei Novikov * remove redundant bias expand (#4382) * remove redundant bias expand Signed-off-by: Xiaowei Ren * delete redundant code Signed-off-by: Xiaowei Ren Signed-off-by: Matvei Novikov * fixed style Signed-off-by: Matvei Novikov * Add option for specifying wandb save_dir from config (#4379) * give option to user to specify wandb save dir via config Signed-off-by: Shantanu Acharya * create save_dir directory for wandb logger if not exists Signed-off-by: Shantanu Acharya * update save_dir get method with a default value Signed-off-by: Shantanu Acharya Signed-off-by: Matvei Novikov * Quick wav2vec fix. In-place operation adding convolutional positions to encoder was overwriting leaf history. Wasn't caught on previous torch versions. (#4383) Signed-off-by: tbartley94 Co-authored-by: tbartley94 Signed-off-by: Matvei Novikov * [Bugfix][TTS] wrong order of returned tuple for general_collate_fn. (#4388) Signed-off-by: Xuesong Yang <1646669+XuesongYang@users.noreply.github.com> Signed-off-by: Matvei Novikov * Merge r1.10.0 main (#4398) * update branch Signed-off-by: ericharper * Set headscale false (#4364) Signed-off-by: MaximumEntropy * Add wandb as dependency (#4365) Signed-off-by: smajumdar * Raise trainer error (#4356) Signed-off-by: MaximumEntropy Co-authored-by: Micha Livne * Set headscale false (#4364) (#4366) Signed-off-by: MaximumEntropy Signed-off-by: smajumdar * Finetuning changes for BART (#4003) * Temp Signed-off-by: MaximumEntropy * Checkpoint converter to nemo for bart Signed-off-by: MaximumEntropy * Style Signed-off-by: MaximumEntropy Co-authored-by: Micha Livne * Make position embedding expansion specific to a batch to avoid checkpoint size mismatches (#4357) * Style Signed-off-by: MaximumEntropy * Fix logging warning Signed-off-by: MaximumEntropy Co-authored-by: Micha Livne * Fix electronic bug, new time ITN rule (#4355) * fix electronic bug Signed-off-by: ekmb * add new itn time rule Signed-off-by: ekmb * revert domain changes Signed-off-by: ekmb * remove repetition Signed-off-by: ekmb * Correct support for dataclasses in default module dim (#4372) * Correct support for dataclasses in default module dim Signed-off-by: smajumdar * Fix path for save of results Signed-off-by: smajumdar * fix pad id bug (#4377) Signed-off-by: Yi Dong * Question answering bug fix (#4381) * refactor dialogue state tracking for modelling/dataset interoperability Signed-off-by: Zhilin Wang * fix style changes Signed-off-by: Zhilin Wang * fix typo Signed-off-by: Zhilin Wang * fix style raised by lgtm Signed-off-by: Zhilin Wang * fix style formatting Signed-off-by: Zhilin Wang * update template to include description of intent Signed-off-by: Zhilin Wang * update Jenkinsfile Signed-off-by: Zhilin Wang * changes based on requests in review Signed-off-by: Zhilin Wang * add compatibility with assistant dataset Signed-off-by: Zhilin Wang * update Jenkins Signed-off-by: Zhilin Wang * remove dialogue_state_tracking Signed-off-by: Zhilin Wang * update huggingface utils for dialogue Signed-off-by: Zhilin Wang * rename dialogue_state_tracking_hybrid to dialogue_state_tracking_sgdqa Signed-off-by: Zhilin Wang * style fix Signed-off-by: Zhilin Wang * fix style Signed-off-by: Zhilin Wang * style fix nemo/collections/nlp/models/dialogue_state_tracking_sgdqa/__init__.py Signed-off-by: Zhilin Wang * update Jenkinsfile for SGDGEN Signed-off-by: Zhilin Wang * update Jenkinsfile for SGDGEN Signed-off-by: Zhilin Wang * update Jenkinsfile for SGDGEN Signed-off-by: Zhilin Wang * update Jenkinsfile for SGDGEN Signed-off-by: Zhilin Wang * update Jenkinsfile for SGDGEN Signed-off-by: Zhilin Wang * fix typo Signed-off-by: Zhilin Wang * add docstrings for assistant data processsor Signed-off-by: Zhilin Wang * update Jenkins for SGDGEN local checkpoint Signed-off-by: Zhilin Wang * update style Signed-off-by: Zhilin Wang * use local vocab file for Jenkinsfile Signed-off-by: Zhilin Wang * patch for Jenkins CI using local file Signed-off-by: Zhilin Wang * add slot filling prediction and metrics Signed-off-by: Zhilin Wang * remove unused code Signed-off-by: Zhilin Wang * style fix Signed-off-by: Zhilin Wang * refactor metrics code out of Dialogue GPT Model Signed-off-by: Zhilin Wang * integrate backward compatible support for IntentSlotClassificationModel (bert model) Signed-off-by: Zhilin Wang * save prediction file for IntentSlotClassification Signed-off-by: Zhilin Wang * update dialogue gpt model training for megatron gpt Signed-off-by: Zhilin Wang * remove batch generate for HF GPT2, which causes lower performance Signed-off-by: Zhilin Wang * add few shot capability to dialogue gpt model Signed-off-by: Zhilin Wang * update Jenkinsfile and remove unused import Signed-off-by: Zhilin Wang * update code description and clarity Signed-off-by: Zhilin Wang * address PR comments Signed-off-by: Zhilin Wang * style fix Signed-off-by: Zhilin Wang * integrate compatibility with ZeroShotIntentModel Signed-off-by: Zhilin Wang * rename folder to dialogue due to increased scope and further refactor for clarity Signed-off-by: Zhilin Wang * added dialogue GPT for sequence generation task (e.g. answer extender) Signed-off-by: Zhilin Wang * add CI test for DialogueGPTGenerationModel Signed-off-by: Zhilin Wang * integrate DialogueS2SGenerationModel for generation task (e.g. answer extender) Signed-off-by: Zhilin Wang * modify huggingface utils to support HF t5/BART models Signed-off-by: Zhilin Wang * style fix Signed-off-by: Zhilin Wang * style fix Signed-off-by: Zhilin Wang * remove unused imports Signed-off-by: Zhilin Wang * style fix Signed-off-by: Zhilin Wang * update Jenkinsfile Signed-off-by: Zhilin Wang * update Jenkinsfile Signed-off-by: Zhilin Wang * update bleu metric Signed-off-by: Zhilin Wang * fix bleu metric style Signed-off-by: Zhilin Wang * debug bleu metric Signed-off-by: Zhilin Wang * debug bleu metric Signed-off-by: Zhilin Wang * update based on PR #3893 Signed-off-by: Zhilin Wang * update 2 based on PR #3893 Signed-off-by: Zhilin Wang * update 3 based on PR #3893 Signed-off-by: Zhilin Wang * integrate sgd generation based on user user utterance and system slot-values to generate system utterance Signed-off-by: Zhilin Wang * add validation model saving capabilities Signed-off-by: Zhilin Wang * cleaned up code for SGD Based Answer extender Signed-off-by: Zhilin Wang * update Dialogue Generation CI Signed-off-by: Zhilin Wang * update Jenkinsfile Signed-off-by: Zhilin Wang * update Jenkinsfile Signed-off-by: Zhilin Wang * fix Jenkins CI issue" Signed-off-by: Zhilin Wang * add support for design dataset Signed-off-by: Zhilin Wang * remove unnecessary imports Signed-off-by: Zhilin Wang * update Jenkins Signed-off-by: Zhilin Wang * update jenkins Signed-off-by: Zhilin Wang * update jenkins Signed-off-by: Zhilin Wang * support megatron for dialogue_s2s_generation_model Signed-off-by: Zhilin Wang * reduce loaded samples in MSMarcoDataProcessor to 64 when cfg.model.dataset.debug_mode=True Signed-off-by: Zhilin Wang * style fix Signed-off-by: Zhilin Wang * style fix Signed-off-by: Zhilin Wang * update CI Signed-off-by: Zhilin Wang * update checkpoint and predictions filename to include epoch number Signed-off-by: Zhilin Wang * style fix Signed-off-by: Zhilin Wang * integrate HF BART MNLI into zero shot intent model Signed-off-by: Zhilin Wang * integrate Dialogue Nearest Neighbour Model Signed-off-by: Zhilin Wang * update Jenkins Signed-off-by: Zhilin Wang * update Jenkins Signed-off-by: Zhilin Wang * refactor Dialogue SGD Data Processor to make interface for models cleaner Signed-off-by: Zhilin Wang * update jenkins Signed-off-by: Zhilin Wang * update Dialogue S2S Generation model for DialogueSGDDataProcessor interface Signed-off-by: Zhilin Wang * update jenkins Signed-off-by: Zhilin Wang * update jenkins Signed-off-by: Zhilin Wang * support sgd and drive thru datasets by zero shot model and nearest neighbour model Signed-off-by: Zhilin Wang * add prediction saving code to nearest neighbour and zero shot intent models Signed-off-by: Zhilin Wang * fix typo in sgd data processor Signed-off-by: Zhilin Wang * integrate Dialogue Mellon QA Data Processor Signed-off-by: Zhilin Wang * update mellon qa Signed-off-by: Zhilin Wang * update dialogue.py to remove outdated info Signed-off-by: Zhilin Wang * style fix Signed-off-by: Zhilin Wang * update dialogue_config.yaml Signed-off-by: Zhilin Wang * update dialogue_config.yaml Signed-off-by: Zhilin Wang * add dialogue docs Signed-off-by: Zhilin Wang * address review comments Signed-off-by: Zhilin Wang * style fix Signed-off-by: Zhilin Wang * style fix Signed-off-by: Zhilin Wang * style fix Signed-off-by: Zhilin Wang * style fix Signed-off-by: Zhilin Wang * style fix Signed-off-by: Zhilin Wang * style fix for cfg Signed-off-by: Zhilin Wang * make dependency on apex optional Signed-off-by: Zhilin Wang * change NLPDDPluggin calling logic to make it possible to run without apex Signed-off-by: Zhilin Wang * add first draft of tutorial Signed-off-by: Zhilin Wang * reduce ms marco size by removing lines without wellFormedAnswers Signed-off-by: Zhilin Wang * address pr comments Signed-off-by: Zhilin Wang * style fix Signed-off-by: Zhilin Wang * update colab tutorial link in dialogue docs Signed-off-by: Zhilin Wang * include unit test and some refactor to facilitate unit test Signed-off-by: Zhilin Wang * style fix Signed-off-by: Zhilin Wang * address pr issues Signed-off-by: Zhilin Wang * remove typos in dialogue tutorial Signed-off-by: Zhilin Wang * support larger files for question answering Signed-off-by: Zhilin Wang * style fix Signed-off-by: Zhilin Wang * style fix Signed-off-by: Zhilin Wang * style fix Signed-off-by: Zhilin Wang * remove unnecessary artifacts to reduce memory use Signed-off-by: Zhilin Wang * put 0 tensor to device Signed-off-by: Zhilin Wang * update link within dialogue tutorial Signed-off-by: Zhilin Wang * restore previously delete files Signed-off-by: Zhilin Wang * update error handling when loss = nan Signed-off-by: Zhilin Wang * update nan handling Signed-off-by: Zhilin Wang * style fix Signed-off-by: Zhilin Wang * update spanning loss func Signed-off-by: Zhilin Wang * update spanning loss Signed-off-by: Zhilin Wang * fix type error raised in qa_dataset.py Signed-off-by: Zhilin Wang * add error checking message Signed-off-by: Zhilin Wang * revert back to float32 Signed-off-by: Zhilin Wang * revert back to float32 Signed-off-by: Zhilin Wang * update error msgs Signed-off-by: Zhilin Wang * update error msgs Signed-off-by: Zhilin Wang * update error msgs Signed-off-by: Zhilin Wang * update error msgs Signed-off-by: Zhilin Wang * update error msgs Signed-off-by: Zhilin Wang * update error msgs Signed-off-by: Zhilin Wang * update error msgs Signed-off-by: Zhilin Wang * update error msgs Signed-off-by: Zhilin Wang * update exp logging Signed-off-by: Zhilin Wang * update error msgs Signed-off-by: Zhilin Wang * update loading of large file from pickle to json Signed-off-by: Zhilin Wang * update loading of large file from pickle to json Signed-off-by: Zhilin Wang * limit number of negative samples Signed-off-by: Zhilin Wang * revert post processing Signed-off-by: Zhilin Wang * revert post processing Signed-off-by: Zhilin Wang * remove unused methods and style fix Signed-off-by: Zhilin Wang * add more documentation Signed-off-by: Zhilin Wang * remove unused imports Signed-off-by: Zhilin Wang * changes base on PR review Signed-off-by: Zhilin Wang * set wandb logger falseby default Signed-off-by: Zhilin Wang * style fix * style fix * correct typo * style fix * style fix Co-authored-by: Zhilin Wang Co-authored-by: Oleksii Kuchaiev Co-authored-by: Yang Zhang Co-authored-by: Eric Harper Co-authored-by: Sandeep Subramanian * Fix ASR Typos in tutorials (#4384) * Fix typos Signed-off-by: smajumdar * Quick wav2vec fix. In-place operation adding convolutional positions to encoder was overwriting leaf history. Wasn't caught on previous torch versions. (#4383) Signed-off-by: tbartley94 Co-authored-by: tbartley94 (cherry picked from commit 0322b158f26a0b690edca7a84714e33752283923) Co-authored-by: Travis Bartley * Add Docs for NeMo Adapters (#4369) Signed-off-by: smajumdar * Update NeMo docs (#4397) Signed-off-by: smajumdar Co-authored-by: Eric Harper * update branch Signed-off-by: ericharper * remove Copy of Signed-off-by: ericharper Co-authored-by: Sandeep Subramanian Co-authored-by: Somshubra Majumdar Co-authored-by: Micha Livne Co-authored-by: Evelina <10428420+ekmb@users.noreply.github.com> Co-authored-by: Yi Dong <43824965+yidong72@users.noreply.github.com> Co-authored-by: Zhilin Wang Co-authored-by: Zhilin Wang Co-authored-by: Oleksii Kuchaiev Co-authored-by: Yang Zhang Co-authored-by: Travis Bartley Signed-off-by: Matvei Novikov * [bugfix][TTS] pitch, voiced_mask, prob_voiced have the same values. (#4392) Signed-off-by: Xuesong Yang <1646669+XuesongYang@users.noreply.github.com> Signed-off-by: Matvei Novikov * Fixing import error in some cases (#4401) Signed-off-by: Boris Fomitchev Signed-off-by: Matvei Novikov * Fixing bugs in calling method ctc_decoder_predictions_tensor. (#4414) * updated ctc decoding calls. Signed-off-by: Vahid * fixed the ones for timestamp_utils.py Signed-off-by: Vahid * fixed the ones for timestamp_utils.py Signed-off-by: Vahid * fixed the ones for timestamp_utils.py Signed-off-by: Vahid Signed-off-by: Matvei Novikov * Update with new conformer checkpoints. (#4417) Signed-off-by: Matvei Novikov * [TTS] add static method decorator. (#4443) * [TTS] add static method decorator. Signed-off-by: Xuesong Yang <1646669+XuesongYang@users.noreply.github.com> * remove protect prefix Signed-off-by: Xuesong Yang <1646669+XuesongYang@users.noreply.github.com> * fixed style error Signed-off-by: Xuesong Yang <1646669+XuesongYang@users.noreply.github.com> Signed-off-by: Matvei Novikov Co-authored-by: Taejin Park Co-authored-by: Nithin Rao Co-authored-by: Somshubra Majumdar Co-authored-by: Georg Kucsko Co-authored-by: fayejf <36722593+fayejf@users.noreply.github.com> Co-authored-by: Xiaowei Ren <103958965+xrennvidia@users.noreply.github.com> Co-authored-by: Shantanu Acharya Co-authored-by: Travis Bartley Co-authored-by: tbartley94 Co-authored-by: Xuesong Yang <1646669+XuesongYang@users.noreply.github.com> Co-authored-by: Eric Harper Co-authored-by: Sandeep Subramanian Co-authored-by: Micha Livne Co-authored-by: Evelina <10428420+ekmb@users.noreply.github.com> Co-authored-by: Yi Dong <43824965+yidong72@users.noreply.github.com> Co-authored-by: Zhilin Wang Co-authored-by: Zhilin Wang Co-authored-by: Oleksii Kuchaiev Co-authored-by: Yang Zhang Co-authored-by: Boris Fomitchev Co-authored-by: Vahid Noroozi --- .../conf/punctuation_capitalization_config.yaml | 8 ++++---- .../prepare_data_for_punctuation_capitalization.py | 11 ++++++++++- .../punctuation_capitalization_dataset.py | 9 --------- 3 files changed, 14 insertions(+), 14 deletions(-) diff --git a/examples/nlp/token_classification/conf/punctuation_capitalization_config.yaml b/examples/nlp/token_classification/conf/punctuation_capitalization_config.yaml index 9565abbee2ec..0cb51f87dcc5 100644 --- a/examples/nlp/token_classification/conf/punctuation_capitalization_config.yaml +++ b/examples/nlp/token_classification/conf/punctuation_capitalization_config.yaml @@ -89,8 +89,8 @@ model: tar_shuffle_n: 1 validation_ds: - # if evaluation data is not in the model.dataset.data_dir as the training data or multiple datasets are used for - # evaluation is needed, specify ds_item, otherwise by default data_dir is used + # if evaluation data is not in the model.train_ds.ds_item as the training data or multiple datasets are used for + # evaluation is needed, specify ds_item, otherwise by default model.train_ds.ds_item is used # See `train_ds` section for more details on tarred dataset use_tarred_dataset: false # expected format: `[PATH_TO_DEV1,PATH_TO_DEV2]` OR `PATH_TO_DEV` (Note no space between the paths and square @@ -113,8 +113,8 @@ model: tar_metadata_file: null test_ds: - # if evaluation data is not in the model.dataset.data_dir as the training data or multiple datasets are used for - # evaluation is needed, specify ds_item, otherwise by default data_dir is used + # if evaluation data is not in the model.train_ds.ds_item as the training data or multiple datasets are used for + # evaluation is needed, specify ds_item, otherwise by default model.train_ds.ds_item is used # See `train_ds` section for more details on tarred dataset use_tarred_dataset: false ds_item: ??? # expected format: [PATH_TO_DEV1,PATH_TO_DEV2] (Note no space between the paths and square brackets) diff --git a/examples/nlp/token_classification/data/prepare_data_for_punctuation_capitalization.py b/examples/nlp/token_classification/data/prepare_data_for_punctuation_capitalization.py index 0ff4306a9306..cd5e4b9aecf0 100644 --- a/examples/nlp/token_classification/data/prepare_data_for_punctuation_capitalization.py +++ b/examples/nlp/token_classification/data/prepare_data_for_punctuation_capitalization.py @@ -88,12 +88,21 @@ parser = argparse.ArgumentParser(description='Prepare data for punctuation and capitalization tasks') parser.add_argument("-s", "--source_file", required=True, type=str, help="Path to the source file") parser.add_argument("-o", "--output_dir", required=True, type=str, help="Path to the output directory") + parser.add_argument( + "-p", + "--marks", + required=False, + type=str, + help="Punctuation marks to consider for dataset", + default=[",", ".", "?"], + nargs="+", + ) args = parser.parse_args() if not os.path.exists(args.source_file): raise ValueError(f'{args.source_file} was not found') os.makedirs(args.output_dir, exist_ok=True) - create_text_and_labels(args.output_dir, args.source_file) + create_text_and_labels(args.output_dir, args.source_file, "".join(args.marks)) print(f'Processing of the {args.source_file} is complete') diff --git a/nemo/collections/nlp/data/token_classification/punctuation_capitalization_dataset.py b/nemo/collections/nlp/data/token_classification/punctuation_capitalization_dataset.py index 5e0c290b8a12..493624de3f65 100644 --- a/nemo/collections/nlp/data/token_classification/punctuation_capitalization_dataset.py +++ b/nemo/collections/nlp/data/token_classification/punctuation_capitalization_dataset.py @@ -1065,15 +1065,6 @@ def _check_constructor_parameters( f' [WORD] [SPACE] [WORD] [SPACE] [WORD] (for text.txt) and ' f' [LABEL] [SPACE] [LABEL] [SPACE] [LABEL] (for labels.txt).' ) - if not str(text_file).endswith('.txt'): - raise ValueError( - f"Parameter `text_file` has to be path to a file with .txt extension, whereas `text_file={text_file}`" - ) - if not str(labels_file).endswith('.txt'): - raise ValueError( - f"Parameter `labels_file` has to be path to a file with .txt extension, whereas " - f"`labels_file={labels_file}`" - ) if punct_label_ids is not None and punct_label_vocab_file is not None: punct_label_vocab_file = Path(punct_label_vocab_file).expanduser() file_punct_label_ids = load_label_ids(punct_label_vocab_file) From 4d0bacb59a0405a254c6e8f77a39ecbcd22aef7d Mon Sep 17 00:00:00 2001 From: Paarth Neekhara Date: Thu, 7 Jul 2022 14:22:34 -0400 Subject: [PATCH 08/52] bug fix - sample rate was being ignored in vocoder dataset when not loading mel Signed-off-by: Paarth Neekhara --- nemo/collections/tts/torch/data.py | 1 + 1 file changed, 1 insertion(+) diff --git a/nemo/collections/tts/torch/data.py b/nemo/collections/tts/torch/data.py index d308b5013f09..238ddb9f523a 100644 --- a/nemo/collections/tts/torch/data.py +++ b/nemo/collections/tts/torch/data.py @@ -917,6 +917,7 @@ def __getitem__(self, index): if not self.load_precomputed_mel: features = AudioSegment.segment_from_file( sample["audio_filepath"], + target_sr=self.sample_rate, n_segments=self.n_segments if self.n_segments is not None else -1, trim=self.trim, ) From 2089016deb2c4d6c0aafd1812339f216725c2f5b Mon Sep 17 00:00:00 2001 From: Guilherme Steinmann Date: Thu, 7 Jul 2022 17:40:09 -0300 Subject: [PATCH 09/52] Add ITN pt (#4516) * Add ITN pt Signed-off-by: Guilherme Steinmann * Fix style Signed-off-by: Guilherme Steinmann * Fix style Signed-off-by: Guilherme Steinmann * Update copyright year to 2022 on ITN pt rules and tests Signed-off-by: Guilherme Steinmann --- .../inverse_normalize.py | 12 +- .../inverse_text_normalization/pt/__init__.py | 17 + .../pt/data/currency_plural.tsv | 5 + .../pt/data/currency_singular.tsv | 5 + .../pt/data/electronic/__init__.py | 13 + .../pt/data/electronic/domain.tsv | 26 ++ .../pt/data/electronic/server_name.tsv | 11 + .../pt/data/electronic/symbols.tsv | 6 + .../pt/data/measurements_plural.tsv | 56 +++ .../pt/data/measurements_singular.tsv | 55 +++ .../pt/data/months.tsv | 12 + .../pt/data/numbers/__init__.py | 13 + .../pt/data/numbers/digit.tsv | 11 + .../pt/data/numbers/hundreds.tsv | 17 + .../pt/data/numbers/onehundred.tsv | 1 + .../pt/data/numbers/teen.tsv | 11 + .../pt/data/numbers/ties.tsv | 8 + .../pt/data/numbers/twenties.tsv | 9 + .../pt/data/numbers/zero.tsv | 1 + .../pt/data/ordinals/__init__.py | 13 + .../pt/data/ordinals/digit.tsv | 18 + .../pt/data/ordinals/hundreds.tsv | 28 ++ .../pt/data/ordinals/ties.tsv | 20 + .../pt/data/time/__init__.py | 13 + .../pt/data/time/hour_to_am.tsv | 1 + .../pt/data/time/hour_to_pm.tsv | 1 + .../pt/data/time/hours_to.tsv | 23 ++ .../pt/data/time/minutes_to.tsv | 59 +++ .../pt/data/time/time_suffix_am.tsv | 2 + .../pt/data/time/time_suffix_pm.tsv | 2 + .../pt/data/whitelist.tsv | 5 + .../pt/taggers/__init__.py | 13 + .../pt/taggers/cardinal.py | 380 ++++++++++++++++++ .../pt/taggers/date.py | 59 +++ .../pt/taggers/decimal.py | 119 ++++++ .../pt/taggers/electronic.py | 96 +++++ .../pt/taggers/measure.py | 95 +++++ .../pt/taggers/money.py | 127 ++++++ .../pt/taggers/ordinal.py | 84 ++++ .../pt/taggers/punctuation.py | 34 ++ .../pt/taggers/telephone.py | 131 ++++++ .../pt/taggers/time.py | 182 +++++++++ .../pt/taggers/tokenize_and_classify.py | 110 +++++ .../pt/taggers/whitelist.py | 33 ++ .../pt/taggers/word.py | 29 ++ .../inverse_text_normalization/pt/utils.py | 27 ++ .../pt/verbalizers/__init__.py | 13 + .../pt/verbalizers/cardinal.py | 48 +++ .../pt/verbalizers/date.py | 65 +++ .../pt/verbalizers/decimal.py | 66 +++ .../pt/verbalizers/electronic.py | 55 +++ .../pt/verbalizers/measure.py | 61 +++ .../pt/verbalizers/money.py | 40 ++ .../pt/verbalizers/ordinal.py | 44 ++ .../pt/verbalizers/telephone.py | 32 ++ .../pt/verbalizers/time.py | 82 ++++ .../pt/verbalizers/verbalize.py | 62 +++ .../pt/verbalizers/verbalize_final.py | 43 ++ .../pt/verbalizers/whitelist.py | 37 ++ .../pt/verbalizers/word.py | 32 ++ .../run_evaluate.py | 2 +- tests/nemo_text_processing/pt/__init__.py | 13 + .../test_cases_cardinal.txt | 69 ++++ .../test_cases_date.txt | 6 + .../test_cases_decimal.txt | 26 ++ .../test_cases_electronic.txt | 13 + .../test_cases_measure.txt | 12 + .../test_cases_money.txt | 25 ++ .../test_cases_ordinal.txt | 19 + .../test_cases_telephone.txt | 27 ++ .../test_cases_time.txt | 19 + .../test_cases_whitelist.txt | 6 + .../test_cases_word.txt | 49 +++ .../nemo_text_processing/pt/test_cardinal.py | 31 ++ tests/nemo_text_processing/pt/test_date.py | 30 ++ tests/nemo_text_processing/pt/test_decimal.py | 30 ++ .../pt/test_electronic.py | 30 ++ tests/nemo_text_processing/pt/test_measure.py | 31 ++ tests/nemo_text_processing/pt/test_money.py | 31 ++ tests/nemo_text_processing/pt/test_ordinal.py | 31 ++ ..._sparrowhawk_inverse_text_normalization.sh | 84 ++++ .../nemo_text_processing/pt/test_telephone.py | 31 ++ tests/nemo_text_processing/pt/test_time.py | 30 ++ .../nemo_text_processing/pt/test_whitelist.py | 31 ++ tests/nemo_text_processing/pt/test_word.py | 31 ++ .../export_grammars.sh | 2 +- .../pynini_export.py | 11 +- 87 files changed, 3386 insertions(+), 7 deletions(-) create mode 100644 nemo_text_processing/inverse_text_normalization/pt/__init__.py create mode 100644 nemo_text_processing/inverse_text_normalization/pt/data/currency_plural.tsv create mode 100644 nemo_text_processing/inverse_text_normalization/pt/data/currency_singular.tsv create mode 100644 nemo_text_processing/inverse_text_normalization/pt/data/electronic/__init__.py create mode 100644 nemo_text_processing/inverse_text_normalization/pt/data/electronic/domain.tsv create mode 100644 nemo_text_processing/inverse_text_normalization/pt/data/electronic/server_name.tsv create mode 100644 nemo_text_processing/inverse_text_normalization/pt/data/electronic/symbols.tsv create mode 100755 nemo_text_processing/inverse_text_normalization/pt/data/measurements_plural.tsv create mode 100755 nemo_text_processing/inverse_text_normalization/pt/data/measurements_singular.tsv create mode 100644 nemo_text_processing/inverse_text_normalization/pt/data/months.tsv create mode 100644 nemo_text_processing/inverse_text_normalization/pt/data/numbers/__init__.py create mode 100644 nemo_text_processing/inverse_text_normalization/pt/data/numbers/digit.tsv create mode 100644 nemo_text_processing/inverse_text_normalization/pt/data/numbers/hundreds.tsv create mode 100644 nemo_text_processing/inverse_text_normalization/pt/data/numbers/onehundred.tsv create mode 100644 nemo_text_processing/inverse_text_normalization/pt/data/numbers/teen.tsv create mode 100644 nemo_text_processing/inverse_text_normalization/pt/data/numbers/ties.tsv create mode 100644 nemo_text_processing/inverse_text_normalization/pt/data/numbers/twenties.tsv create mode 100644 nemo_text_processing/inverse_text_normalization/pt/data/numbers/zero.tsv create mode 100644 nemo_text_processing/inverse_text_normalization/pt/data/ordinals/__init__.py create mode 100644 nemo_text_processing/inverse_text_normalization/pt/data/ordinals/digit.tsv create mode 100644 nemo_text_processing/inverse_text_normalization/pt/data/ordinals/hundreds.tsv create mode 100644 nemo_text_processing/inverse_text_normalization/pt/data/ordinals/ties.tsv create mode 100644 nemo_text_processing/inverse_text_normalization/pt/data/time/__init__.py create mode 100644 nemo_text_processing/inverse_text_normalization/pt/data/time/hour_to_am.tsv create mode 100644 nemo_text_processing/inverse_text_normalization/pt/data/time/hour_to_pm.tsv create mode 100644 nemo_text_processing/inverse_text_normalization/pt/data/time/hours_to.tsv create mode 100644 nemo_text_processing/inverse_text_normalization/pt/data/time/minutes_to.tsv create mode 100644 nemo_text_processing/inverse_text_normalization/pt/data/time/time_suffix_am.tsv create mode 100644 nemo_text_processing/inverse_text_normalization/pt/data/time/time_suffix_pm.tsv create mode 100644 nemo_text_processing/inverse_text_normalization/pt/data/whitelist.tsv create mode 100644 nemo_text_processing/inverse_text_normalization/pt/taggers/__init__.py create mode 100644 nemo_text_processing/inverse_text_normalization/pt/taggers/cardinal.py create mode 100644 nemo_text_processing/inverse_text_normalization/pt/taggers/date.py create mode 100644 nemo_text_processing/inverse_text_normalization/pt/taggers/decimal.py create mode 100644 nemo_text_processing/inverse_text_normalization/pt/taggers/electronic.py create mode 100644 nemo_text_processing/inverse_text_normalization/pt/taggers/measure.py create mode 100644 nemo_text_processing/inverse_text_normalization/pt/taggers/money.py create mode 100644 nemo_text_processing/inverse_text_normalization/pt/taggers/ordinal.py create mode 100644 nemo_text_processing/inverse_text_normalization/pt/taggers/punctuation.py create mode 100755 nemo_text_processing/inverse_text_normalization/pt/taggers/telephone.py create mode 100755 nemo_text_processing/inverse_text_normalization/pt/taggers/time.py create mode 100644 nemo_text_processing/inverse_text_normalization/pt/taggers/tokenize_and_classify.py create mode 100644 nemo_text_processing/inverse_text_normalization/pt/taggers/whitelist.py create mode 100644 nemo_text_processing/inverse_text_normalization/pt/taggers/word.py create mode 100644 nemo_text_processing/inverse_text_normalization/pt/utils.py create mode 100644 nemo_text_processing/inverse_text_normalization/pt/verbalizers/__init__.py create mode 100644 nemo_text_processing/inverse_text_normalization/pt/verbalizers/cardinal.py create mode 100644 nemo_text_processing/inverse_text_normalization/pt/verbalizers/date.py create mode 100644 nemo_text_processing/inverse_text_normalization/pt/verbalizers/decimal.py create mode 100644 nemo_text_processing/inverse_text_normalization/pt/verbalizers/electronic.py create mode 100644 nemo_text_processing/inverse_text_normalization/pt/verbalizers/measure.py create mode 100644 nemo_text_processing/inverse_text_normalization/pt/verbalizers/money.py create mode 100644 nemo_text_processing/inverse_text_normalization/pt/verbalizers/ordinal.py create mode 100644 nemo_text_processing/inverse_text_normalization/pt/verbalizers/telephone.py create mode 100755 nemo_text_processing/inverse_text_normalization/pt/verbalizers/time.py create mode 100644 nemo_text_processing/inverse_text_normalization/pt/verbalizers/verbalize.py create mode 100644 nemo_text_processing/inverse_text_normalization/pt/verbalizers/verbalize_final.py create mode 100644 nemo_text_processing/inverse_text_normalization/pt/verbalizers/whitelist.py create mode 100644 nemo_text_processing/inverse_text_normalization/pt/verbalizers/word.py create mode 100644 tests/nemo_text_processing/pt/__init__.py create mode 100755 tests/nemo_text_processing/pt/data_inverse_text_normalization/test_cases_cardinal.txt create mode 100644 tests/nemo_text_processing/pt/data_inverse_text_normalization/test_cases_date.txt create mode 100755 tests/nemo_text_processing/pt/data_inverse_text_normalization/test_cases_decimal.txt create mode 100644 tests/nemo_text_processing/pt/data_inverse_text_normalization/test_cases_electronic.txt create mode 100755 tests/nemo_text_processing/pt/data_inverse_text_normalization/test_cases_measure.txt create mode 100755 tests/nemo_text_processing/pt/data_inverse_text_normalization/test_cases_money.txt create mode 100755 tests/nemo_text_processing/pt/data_inverse_text_normalization/test_cases_ordinal.txt create mode 100755 tests/nemo_text_processing/pt/data_inverse_text_normalization/test_cases_telephone.txt create mode 100755 tests/nemo_text_processing/pt/data_inverse_text_normalization/test_cases_time.txt create mode 100755 tests/nemo_text_processing/pt/data_inverse_text_normalization/test_cases_whitelist.txt create mode 100755 tests/nemo_text_processing/pt/data_inverse_text_normalization/test_cases_word.txt create mode 100644 tests/nemo_text_processing/pt/test_cardinal.py create mode 100644 tests/nemo_text_processing/pt/test_date.py create mode 100644 tests/nemo_text_processing/pt/test_decimal.py create mode 100644 tests/nemo_text_processing/pt/test_electronic.py create mode 100644 tests/nemo_text_processing/pt/test_measure.py create mode 100644 tests/nemo_text_processing/pt/test_money.py create mode 100644 tests/nemo_text_processing/pt/test_ordinal.py create mode 100755 tests/nemo_text_processing/pt/test_sparrowhawk_inverse_text_normalization.sh create mode 100644 tests/nemo_text_processing/pt/test_telephone.py create mode 100644 tests/nemo_text_processing/pt/test_time.py create mode 100644 tests/nemo_text_processing/pt/test_whitelist.py create mode 100644 tests/nemo_text_processing/pt/test_word.py diff --git a/nemo_text_processing/inverse_text_normalization/inverse_normalize.py b/nemo_text_processing/inverse_text_normalization/inverse_normalize.py index cde940156502..a39131440cec 100644 --- a/nemo_text_processing/inverse_text_normalization/inverse_normalize.py +++ b/nemo_text_processing/inverse_text_normalization/inverse_normalize.py @@ -23,7 +23,7 @@ class InverseNormalizer(Normalizer): """ - Inverse normalizer that converts text from spoken to written form. Useful for ASR postprocessing. + Inverse normalizer that converts text from spoken to written form. Useful for ASR postprocessing. Input is expected to have no punctuation outside of approstrophe (') and dash (-) and be lower cased. Args: @@ -46,6 +46,12 @@ def __init__(self, lang: str = 'en', cache_dir: str = None, overwrite_cache: boo VerbalizeFinalFst, ) + elif lang == 'pt': + from nemo_text_processing.inverse_text_normalization.pt.taggers.tokenize_and_classify import ClassifyFst + from nemo_text_processing.inverse_text_normalization.pt.verbalizers.verbalize_final import ( + VerbalizeFinalFst, + ) + elif lang == 'ru': from nemo_text_processing.inverse_text_normalization.ru.taggers.tokenize_and_classify import ClassifyFst from nemo_text_processing.inverse_text_normalization.ru.verbalizers.verbalize_final import ( @@ -75,7 +81,7 @@ def __init__(self, lang: str = 'en', cache_dir: str = None, overwrite_cache: boo def inverse_normalize_list(self, texts: List[str], verbose=False) -> List[str]: """ - NeMo inverse text normalizer + NeMo inverse text normalizer Args: texts: list of input strings @@ -106,7 +112,7 @@ def parse_args(): input.add_argument("--input_file", dest="input_file", help="input file path", type=str) parser.add_argument('--output_file', dest="output_file", help="output file path", type=str) parser.add_argument( - "--language", help="language", choices=['en', 'de', 'es', 'ru', 'fr', 'vi'], default="en", type=str + "--language", help="language", choices=['en', 'de', 'es', 'pt', 'ru', 'fr', 'vi'], default="en", type=str ) parser.add_argument("--verbose", help="print info for debugging", action='store_true') parser.add_argument("--overwrite_cache", help="set to True to re-create .far grammar files", action="store_true") diff --git a/nemo_text_processing/inverse_text_normalization/pt/__init__.py b/nemo_text_processing/inverse_text_normalization/pt/__init__.py new file mode 100644 index 000000000000..c1586debd25f --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/pt/__init__.py @@ -0,0 +1,17 @@ +# Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from nemo_text_processing.inverse_text_normalization.pt.taggers.tokenize_and_classify import ClassifyFst +from nemo_text_processing.inverse_text_normalization.pt.verbalizers.verbalize import VerbalizeFst +from nemo_text_processing.inverse_text_normalization.pt.verbalizers.verbalize_final import VerbalizeFinalFst diff --git a/nemo_text_processing/inverse_text_normalization/pt/data/currency_plural.tsv b/nemo_text_processing/inverse_text_normalization/pt/data/currency_plural.tsv new file mode 100644 index 000000000000..a89a763093ea --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/pt/data/currency_plural.tsv @@ -0,0 +1,5 @@ +€ euros +£ libras esterlinas +US$ dólares americanos +$ dólares +R$ reais \ No newline at end of file diff --git a/nemo_text_processing/inverse_text_normalization/pt/data/currency_singular.tsv b/nemo_text_processing/inverse_text_normalization/pt/data/currency_singular.tsv new file mode 100644 index 000000000000..9ec77dc35654 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/pt/data/currency_singular.tsv @@ -0,0 +1,5 @@ +€ euro +£ libra esterlina +US$ dólar americano +$ dólar +R$ real \ No newline at end of file diff --git a/nemo_text_processing/inverse_text_normalization/pt/data/electronic/__init__.py b/nemo_text_processing/inverse_text_normalization/pt/data/electronic/__init__.py new file mode 100644 index 000000000000..a1cf281f0908 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/pt/data/electronic/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nemo_text_processing/inverse_text_normalization/pt/data/electronic/domain.tsv b/nemo_text_processing/inverse_text_normalization/pt/data/electronic/domain.tsv new file mode 100644 index 000000000000..ea547b890119 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/pt/data/electronic/domain.tsv @@ -0,0 +1,26 @@ +com +es +uk +fr +net +br +in +ru +de +it +edu +co +ar +bo +cl +co +ec +fk +gf +fy +pe +py +sr +ve +uy +pt \ No newline at end of file diff --git a/nemo_text_processing/inverse_text_normalization/pt/data/electronic/server_name.tsv b/nemo_text_processing/inverse_text_normalization/pt/data/electronic/server_name.tsv new file mode 100644 index 000000000000..34ab709bb308 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/pt/data/electronic/server_name.tsv @@ -0,0 +1,11 @@ +gmail g mail +gmail +nvidia n vidia +nvidia +outlook +hotmail +yahoo +aol +live +msn +live \ No newline at end of file diff --git a/nemo_text_processing/inverse_text_normalization/pt/data/electronic/symbols.tsv b/nemo_text_processing/inverse_text_normalization/pt/data/electronic/symbols.tsv new file mode 100644 index 000000000000..690a9ca427f1 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/pt/data/electronic/symbols.tsv @@ -0,0 +1,6 @@ +. ponto +- traço +- hífen +_ traço baixo +_ underscore +/ barra \ No newline at end of file diff --git a/nemo_text_processing/inverse_text_normalization/pt/data/measurements_plural.tsv b/nemo_text_processing/inverse_text_normalization/pt/data/measurements_plural.tsv new file mode 100755 index 000000000000..6d2684afdf8b --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/pt/data/measurements_plural.tsv @@ -0,0 +1,56 @@ +h horas +min minutos +s segundos +ms milissegundos +ns nanossegundos +μs microssegundos +t toneladas +kg quilos +kg quilogramas +g gramas +mg miligramas +μm micrômetros +nm nanômetros +mm milímetros +cm centímetros +cm² centímetros quadrado +cm³ centímetros cúbico +m metros +m² metros quadrados +m³ metros cúbicos +km quilômetros +km² quilômetros quadrados +ha hectares +kph quilômetros por hora +mph milhas por hora +m/s metros por segundo +l litros +ml mililitros +kgf quilogramas forças +kgf quilogramas força +% por cento +°F fahrenheit +°C celsius +°F graus fahrenheit +°C graus celsius +Hz hertz +kHz quilo hertz +MHz mega hertz +GHz giga hertz +W watts +kW quilowatts +MW megawatts +GW gigawatts +Wh watts hora +kWh quilowatts hora +MWh megawatts hora +GWh gigawatts hora +kV quilovolts +V volts +mV milivolts +A amperes +mA miliamperes +rpm rotações por minuto +db decibéis +cal calorias +kcal quilocalorias diff --git a/nemo_text_processing/inverse_text_normalization/pt/data/measurements_singular.tsv b/nemo_text_processing/inverse_text_normalization/pt/data/measurements_singular.tsv new file mode 100755 index 000000000000..bf7320e6242c --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/pt/data/measurements_singular.tsv @@ -0,0 +1,55 @@ +h hora +min minuto +s segundo +ms milissegundo +ns nanossegundo +μs microssegundo +t tonelada +kg quilo +kg quilograma +g grama +mg miligrama +μm micrômetro +nm nanômetro +mm milímetro +cm centímetro +cm² centímetro quadrado +cm³ centímetro cúbico +m metro +m² metro quadrado +m³ metro cúbico +km quilômetro +km² quilômetro quadrado +ha hectare +kph quilômetro por hora +mph milha por hora +m/s metro por segundo +l litro +ml mililitro +kgf quilograma força +% por cento +°F fahrenheit +°C celsius +°F grau fahrenheit +°C grau celsius +Hz hertz +kHz quilo hertz +MHz mega hertz +GHz giga hertz +W watt +kW quilowatt +MW megawatt +GW gigawatt +Wh watt hora +kWh quilowatt hora +MWh megawatt hora +GWh gigawatt hora +kV quilovolt +V volt +mV milivolt +A ampere +mA miliampere +rpm rotação por minuto +db decibel +cal caloria +kcal quilocaloria diff --git a/nemo_text_processing/inverse_text_normalization/pt/data/months.tsv b/nemo_text_processing/inverse_text_normalization/pt/data/months.tsv new file mode 100644 index 000000000000..ed1cf8d4f78c --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/pt/data/months.tsv @@ -0,0 +1,12 @@ +janeiro +fevereiro +março +abril +maio +junho +julho +agosto +setembro +outubro +novembro +dezembro diff --git a/nemo_text_processing/inverse_text_normalization/pt/data/numbers/__init__.py b/nemo_text_processing/inverse_text_normalization/pt/data/numbers/__init__.py new file mode 100644 index 000000000000..a1cf281f0908 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/pt/data/numbers/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nemo_text_processing/inverse_text_normalization/pt/data/numbers/digit.tsv b/nemo_text_processing/inverse_text_normalization/pt/data/numbers/digit.tsv new file mode 100644 index 000000000000..fda1b633b2fb --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/pt/data/numbers/digit.tsv @@ -0,0 +1,11 @@ +um 1 +uma 1 +dois 2 +duas 2 +três 3 +quatro 4 +cinco 5 +seis 6 +sete 7 +oito 8 +nove 9 \ No newline at end of file diff --git a/nemo_text_processing/inverse_text_normalization/pt/data/numbers/hundreds.tsv b/nemo_text_processing/inverse_text_normalization/pt/data/numbers/hundreds.tsv new file mode 100644 index 000000000000..ff06089d3e67 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/pt/data/numbers/hundreds.tsv @@ -0,0 +1,17 @@ +cento 1 +duzentos 2 +duzentas 2 +trezentos 3 +trezentas 3 +quatrocentos 4 +quatrocentas 4 +quinhentos 5 +quinhentas 5 +seiscentos 6 +seiscentas 6 +setecentos 7 +setecentas 7 +oitocentos 8 +oitocentas 8 +novecentos 9 +novecentas 9 \ No newline at end of file diff --git a/nemo_text_processing/inverse_text_normalization/pt/data/numbers/onehundred.tsv b/nemo_text_processing/inverse_text_normalization/pt/data/numbers/onehundred.tsv new file mode 100644 index 000000000000..1b5f9fa05302 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/pt/data/numbers/onehundred.tsv @@ -0,0 +1 @@ +cem 100 \ No newline at end of file diff --git a/nemo_text_processing/inverse_text_normalization/pt/data/numbers/teen.tsv b/nemo_text_processing/inverse_text_normalization/pt/data/numbers/teen.tsv new file mode 100644 index 000000000000..6bc21cccfc30 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/pt/data/numbers/teen.tsv @@ -0,0 +1,11 @@ +dez 10 +onze 11 +doze 12 +treze 13 +catorze 14 +quatorze 14 +quinze 15 +dezesseis 16 +dezessete 17 +dezoito 18 +dezenove 19 \ No newline at end of file diff --git a/nemo_text_processing/inverse_text_normalization/pt/data/numbers/ties.tsv b/nemo_text_processing/inverse_text_normalization/pt/data/numbers/ties.tsv new file mode 100644 index 000000000000..63ff93c83220 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/pt/data/numbers/ties.tsv @@ -0,0 +1,8 @@ +vinte 2 +trinta 3 +quarenta 4 +cinquenta 5 +sessenta 6 +setenta 7 +oitenta 8 +noventa 9 \ No newline at end of file diff --git a/nemo_text_processing/inverse_text_normalization/pt/data/numbers/twenties.tsv b/nemo_text_processing/inverse_text_normalization/pt/data/numbers/twenties.tsv new file mode 100644 index 000000000000..c72178c15ed1 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/pt/data/numbers/twenties.tsv @@ -0,0 +1,9 @@ +vinte um 21 +vinte dois 22 +vinte três 23 +vinte quatro 24 +vinte cinco 25 +vinte seis 26 +vinte sete 27 +vinte oito 28 +vinte nove 29 diff --git a/nemo_text_processing/inverse_text_normalization/pt/data/numbers/zero.tsv b/nemo_text_processing/inverse_text_normalization/pt/data/numbers/zero.tsv new file mode 100644 index 000000000000..c479272d4039 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/pt/data/numbers/zero.tsv @@ -0,0 +1 @@ +zero 0 \ No newline at end of file diff --git a/nemo_text_processing/inverse_text_normalization/pt/data/ordinals/__init__.py b/nemo_text_processing/inverse_text_normalization/pt/data/ordinals/__init__.py new file mode 100644 index 000000000000..a1cf281f0908 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/pt/data/ordinals/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nemo_text_processing/inverse_text_normalization/pt/data/ordinals/digit.tsv b/nemo_text_processing/inverse_text_normalization/pt/data/ordinals/digit.tsv new file mode 100644 index 000000000000..ad97cc411414 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/pt/data/ordinals/digit.tsv @@ -0,0 +1,18 @@ +primeiro 1 +primeira 1 +segundo 2 +segunda 2 +terceiro 3 +terceira 3 +quarto 4 +quarta 4 +quinto 5 +quinta 5 +sexto 6 +sexta 6 +sétimo 7 +sétima 7 +oitavo 8 +oitava 8 +nono 9 +nona 9 \ No newline at end of file diff --git a/nemo_text_processing/inverse_text_normalization/pt/data/ordinals/hundreds.tsv b/nemo_text_processing/inverse_text_normalization/pt/data/ordinals/hundreds.tsv new file mode 100644 index 000000000000..b7b15ee92488 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/pt/data/ordinals/hundreds.tsv @@ -0,0 +1,28 @@ +centésimo 1 +centésima 1 +ducentésimo 2 +ducentésima 2 +tricentésimo 3 +tricentésima 3 +trecentésimo 3 +trecentésima 3 +quadringentésimo 4 +quadringentésima 4 +quingentésimo 5 +quingentésima 5 +sexcentésimo 6 +sexcentésima 6 +seiscentésimo 6 +seiscentésima 6 +septingentésimo 7 +septingentésima 7 +setingentésimo 7 +setingentésima 7 +octingentésimo 8 +octingentésima 8 +octogentésimo 8 +octogentésima 8 +noningentésimo 9 +noningentésima 9 +nongentésimo 9 +nongentésima 9 \ No newline at end of file diff --git a/nemo_text_processing/inverse_text_normalization/pt/data/ordinals/ties.tsv b/nemo_text_processing/inverse_text_normalization/pt/data/ordinals/ties.tsv new file mode 100644 index 000000000000..55c4c4ee2fa3 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/pt/data/ordinals/ties.tsv @@ -0,0 +1,20 @@ +décimo 1 +décima 1 +vigésimo 2 +vigésima 2 +trigésimo 3 +trigésima 3 +quadragésimo 4 +quadragésima 4 +quinquagésimo 5 +quinquagésima 5 +sexagésimo 6 +sexagésima 6 +septuagésimo 7 +septuagésima 7 +setuagésimo 7 +setuagésima 7 +octogésimo 8 +octogésima 8 +nonagésimo 9 +nonagésima 9 \ No newline at end of file diff --git a/nemo_text_processing/inverse_text_normalization/pt/data/time/__init__.py b/nemo_text_processing/inverse_text_normalization/pt/data/time/__init__.py new file mode 100644 index 000000000000..a1cf281f0908 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/pt/data/time/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nemo_text_processing/inverse_text_normalization/pt/data/time/hour_to_am.tsv b/nemo_text_processing/inverse_text_normalization/pt/data/time/hour_to_am.tsv new file mode 100644 index 000000000000..c22366ba8665 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/pt/data/time/hour_to_am.tsv @@ -0,0 +1 @@ +1 0 \ No newline at end of file diff --git a/nemo_text_processing/inverse_text_normalization/pt/data/time/hour_to_pm.tsv b/nemo_text_processing/inverse_text_normalization/pt/data/time/hour_to_pm.tsv new file mode 100644 index 000000000000..548045d94c60 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/pt/data/time/hour_to_pm.tsv @@ -0,0 +1 @@ +1 12 \ No newline at end of file diff --git a/nemo_text_processing/inverse_text_normalization/pt/data/time/hours_to.tsv b/nemo_text_processing/inverse_text_normalization/pt/data/time/hours_to.tsv new file mode 100644 index 000000000000..5742d596b64d --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/pt/data/time/hours_to.tsv @@ -0,0 +1,23 @@ +0 23 +2 1 +3 2 +4 3 +5 4 +6 5 +7 6 +8 7 +9 8 +10 9 +11 10 +12 11 +13 12 +14 13 +15 14 +16 15 +17 16 +18 17 +19 18 +20 19 +21 20 +22 21 +23 22 \ No newline at end of file diff --git a/nemo_text_processing/inverse_text_normalization/pt/data/time/minutes_to.tsv b/nemo_text_processing/inverse_text_normalization/pt/data/time/minutes_to.tsv new file mode 100644 index 000000000000..d8516a9f83ce --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/pt/data/time/minutes_to.tsv @@ -0,0 +1,59 @@ +01 59 +02 58 +03 57 +04 56 +05 55 +06 54 +07 53 +08 52 +09 51 +10 50 +11 49 +12 48 +13 47 +14 46 +15 45 +16 44 +17 43 +18 42 +19 41 +20 40 +21 39 +22 38 +23 37 +24 36 +25 35 +26 34 +27 33 +28 32 +29 31 +30 30 +31 29 +32 28 +33 27 +34 26 +35 25 +36 24 +37 23 +38 22 +39 21 +40 20 +41 19 +42 18 +43 17 +44 16 +45 15 +46 14 +47 13 +48 12 +49 11 +50 10 +51 09 +52 08 +53 07 +54 06 +55 05 +56 04 +57 03 +58 02 +59 01 \ No newline at end of file diff --git a/nemo_text_processing/inverse_text_normalization/pt/data/time/time_suffix_am.tsv b/nemo_text_processing/inverse_text_normalization/pt/data/time/time_suffix_am.tsv new file mode 100644 index 000000000000..95394d7a6145 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/pt/data/time/time_suffix_am.tsv @@ -0,0 +1,2 @@ +da madrugada da madrugada +da manhã da manhã \ No newline at end of file diff --git a/nemo_text_processing/inverse_text_normalization/pt/data/time/time_suffix_pm.tsv b/nemo_text_processing/inverse_text_normalization/pt/data/time/time_suffix_pm.tsv new file mode 100644 index 000000000000..18c7c994b020 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/pt/data/time/time_suffix_pm.tsv @@ -0,0 +1,2 @@ +da tarde da tarde +da noite da noite \ No newline at end of file diff --git a/nemo_text_processing/inverse_text_normalization/pt/data/whitelist.tsv b/nemo_text_processing/inverse_text_normalization/pt/data/whitelist.tsv new file mode 100644 index 000000000000..bd82088f7990 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/pt/data/whitelist.tsv @@ -0,0 +1,5 @@ +segunda-feira segunda feira +terça-feira terça feira +quarta-feira quarta feira +quinta-feira quinta feira +sexta-feira sexta feira \ No newline at end of file diff --git a/nemo_text_processing/inverse_text_normalization/pt/taggers/__init__.py b/nemo_text_processing/inverse_text_normalization/pt/taggers/__init__.py new file mode 100644 index 000000000000..a1cf281f0908 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/pt/taggers/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nemo_text_processing/inverse_text_normalization/pt/taggers/cardinal.py b/nemo_text_processing/inverse_text_normalization/pt/taggers/cardinal.py new file mode 100644 index 000000000000..115cea6cdd3d --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/pt/taggers/cardinal.py @@ -0,0 +1,380 @@ +# Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import pynini +from nemo_text_processing.inverse_text_normalization.pt.utils import get_abs_path +from nemo_text_processing.text_normalization.en.graph_utils import ( + NEMO_ALPHA, + NEMO_DIGIT, + NEMO_SIGMA, + NEMO_SPACE, + NEMO_WHITE_SPACE, + GraphFst, + delete_space, +) +from pynini.lib import pynutil + + +class CardinalFst(GraphFst): + """ + Finite state transducer for classifying cardinals + e.g. menos veintitrés -> cardinal { negative: "-" integer: "23"} + This class converts cardinals up to (but not including) "un cuatrillón", + i.e up to "one septillion" in English (10^{24}). + Cardinals below ten are not converted (in order to avoid + "vivo em uma casa" --> "vivo em 1 casa" and any other odd conversions.) + + Although technically Portuguese grammar requires that "e" only comes after + "10s" numbers (ie. "trinta", ..., "noventa"), these rules will convert + numbers even with "e" in an ungrammatical place (because "e" is ignored + inside cardinal numbers). + e.g. "mil e uma" -> cardinal { integer: "1001"} + e.g. "cento e uma" -> cardinal { integer: "101"} + """ + + def __init__(self, use_strict_e=False): + """ + :param use_strict_e: When True forces to have the separator "e" in the right places + """ + super().__init__(name="cardinal", kind="classify") + graph_zero = pynini.string_file(get_abs_path("data/numbers/zero.tsv")) + graph_digit = pynini.string_file(get_abs_path("data/numbers/digit.tsv")) + graph_ties = pynini.string_file(get_abs_path("data/numbers/ties.tsv")) + graph_teen = pynini.string_file(get_abs_path("data/numbers/teen.tsv")) + graph_twenties = pynini.string_file(get_abs_path("data/numbers/twenties.tsv")) + graph_one_hundred = pynini.string_file(get_abs_path("data/numbers/onehundred.tsv")) + graph_hundreds = pynini.string_file(get_abs_path("data/numbers/hundreds.tsv")) + + graph = None + + if not use_strict_e: + graph_hundred_component = graph_hundreds | pynutil.insert("0") + graph_hundred_component += delete_space + graph_hundred_component += pynini.union( + graph_twenties | graph_teen | pynutil.insert("00"), + (graph_ties | pynutil.insert("0")) + delete_space + (graph_digit | pynutil.insert("0")), + ) + graph_hundred_component = pynini.union(graph_hundred_component, graph_one_hundred) + + graph_hundred_component_at_least_one_none_zero_digit = graph_hundred_component @ ( + pynini.closure(NEMO_DIGIT) + (NEMO_DIGIT - "0") + pynini.closure(NEMO_DIGIT) + ) + + graph_thousands = pynini.union( + graph_hundred_component_at_least_one_none_zero_digit + delete_space + pynutil.delete("mil"), + pynutil.insert("001") + pynutil.delete("mil"), # because we say 'mil', not 'hum mil' + pynutil.insert("000", weight=0.01), + ) + + graph_milhoes = pynini.union( + graph_hundred_component_at_least_one_none_zero_digit + + delete_space + + (pynutil.delete("milhão") | pynutil.delete("milhões")), + pynutil.insert("000", weight=0.01), + ) + + graph_bilhoes = pynini.union( + graph_hundred_component_at_least_one_none_zero_digit + + delete_space + + (pynutil.delete("bilhão") | pynutil.delete("bilhões")), + pynutil.insert("000", weight=0.01), + ) + + graph_trilhoes = pynini.union( + graph_hundred_component_at_least_one_none_zero_digit + + delete_space + + (pynutil.delete("trilhão") | pynutil.delete("trilhões")), + pynutil.insert("000", weight=0.01), + ) + + graph_quatrilhoes = pynini.union( + graph_hundred_component_at_least_one_none_zero_digit + + delete_space + + (pynutil.delete("quatrilhão") | pynutil.delete("quatrilhões")), + pynutil.insert("000", weight=0.01), + ) + + graph_quintilhoes = pynini.union( + graph_hundred_component_at_least_one_none_zero_digit + + delete_space + + (pynutil.delete("quintilhão") | pynutil.delete("quintilhões")), + pynutil.insert("000", weight=0.01), + ) + + graph_sextilhoes = pynini.union( + graph_hundred_component_at_least_one_none_zero_digit + + delete_space + + (pynutil.delete("sextilhão") | pynutil.delete("sextilhões")), + pynutil.insert("000", weight=0.01), + ) + + graph = pynini.union( + graph_sextilhoes + + delete_space + + graph_quintilhoes + + delete_space + + graph_quatrilhoes + + delete_space + + graph_trilhoes + + delete_space + + graph_bilhoes + + delete_space + + graph_milhoes + + delete_space + + graph_thousands + + delete_space + + graph_hundred_component, + graph_zero, + ) + + graph = graph @ pynini.union( + pynutil.delete(pynini.closure("0")) + pynini.difference(NEMO_DIGIT, "0") + pynini.closure(NEMO_DIGIT), + "0", + ) + + graph = ( + pynini.cdrewrite(pynutil.delete("e"), NEMO_SPACE, NEMO_SPACE, NEMO_SIGMA) + @ (NEMO_ALPHA + NEMO_SIGMA) + @ graph + ) + + else: + graph_e = ( + pynutil.delete(NEMO_WHITE_SPACE.plus) + pynutil.delete("e") + pynutil.delete(NEMO_WHITE_SPACE.plus) + ) + + graph_ties_component = pynini.union( + graph_teen | graph_twenties, + graph_ties + ((graph_e + graph_digit) | pynutil.insert("0")), + pynutil.add_weight(pynutil.insert("0") + graph_digit, 0.1), + ) @ (pynini.closure(NEMO_DIGIT) + (NEMO_DIGIT - "0") + pynini.closure(NEMO_DIGIT)) + + graph_hundreds_except_hundred = (pynini.project(graph_hundreds, "input") - "cento") @ graph_hundreds + + graph_hundred_component_prefix_e = pynini.union( + graph_one_hundred, + pynutil.add_weight(graph_hundreds_except_hundred + pynutil.insert("00"), 0.1), + pynutil.insert("0") + graph_ties_component, + ) @ (pynini.closure(NEMO_DIGIT) + (NEMO_DIGIT - "0") + pynini.closure(NEMO_DIGIT)) + graph_hundred_component_prefix_e = graph_hundred_component_prefix_e.optimize() + + graph_hundred_component_no_prefix = pynini.union(graph_hundreds + graph_e + graph_ties_component,) @ ( + pynini.closure(NEMO_DIGIT) + (NEMO_DIGIT - "0") + pynini.closure(NEMO_DIGIT) + ) + graph_hundred_component_no_prefix = graph_hundred_component_no_prefix.optimize() + + graph_mil_prefix_e = pynini.union( + # because we say 'mil', not 'hum mil' + ( + (graph_hundred_component_prefix_e + delete_space + pynutil.delete("mil")) + | (pynutil.insert("001", weight=0.1) + pynutil.delete("mil")) + ) + + ( + (graph_e + graph_hundred_component_prefix_e) + | (delete_space + graph_hundred_component_no_prefix) + | pynutil.insert("000", weight=0.1) + ) + ) + + graph_mil_no_prefix = pynini.union( + ( + (graph_hundred_component_no_prefix + delete_space + pynutil.delete("mil")) + | pynutil.insert("000", weight=0.1) + ) + + ( + (graph_e + graph_hundred_component_prefix_e) + | (delete_space + graph_hundred_component_no_prefix) + | pynutil.insert("000", weight=0.1) + ) + ) + + graph_milhao_prefix_e = pynini.union( + ( + graph_hundred_component_prefix_e + + delete_space + + (pynutil.delete("milhão") | pynutil.delete("milhões")) + ) + + ((graph_e + graph_mil_prefix_e) | (delete_space + graph_mil_no_prefix)) + ) + + graph_milhao_no_prefix = pynini.union( + ( + ( + graph_hundred_component_no_prefix + + delete_space + + (pynutil.delete("milhão") | pynutil.delete("milhões")) + ) + | pynutil.insert("000", weight=0.1) + ) + + ((graph_e + graph_mil_prefix_e) | (delete_space + graph_mil_no_prefix)) + ) + + graph_bilhao_prefix_e = pynini.union( + ( + graph_hundred_component_prefix_e + + delete_space + + (pynutil.delete("bilhão") | pynutil.delete("bilhões")) + ) + + ((graph_e + graph_milhao_prefix_e) | (delete_space + graph_milhao_no_prefix)) + ) + + graph_bilhao_no_prefix = pynini.union( + ( + ( + graph_hundred_component_no_prefix + + delete_space + + (pynutil.delete("bilhão") | pynutil.delete("bilhões")) + ) + | pynutil.insert("000", weight=0.1) + ) + + ((graph_e + graph_milhao_prefix_e) | (delete_space + graph_milhao_no_prefix)) + ) + + graph_trilhao_prefix_e = pynini.union( + ( + graph_hundred_component_prefix_e + + delete_space + + (pynutil.delete("trilhão") | pynutil.delete("trilhões")) + ) + + ((graph_e + graph_bilhao_prefix_e) | (delete_space + graph_bilhao_no_prefix)) + ) + + graph_trilhao_no_prefix = pynini.union( + ( + ( + graph_hundred_component_no_prefix + + delete_space + + (pynutil.delete("trilhão") | pynutil.delete("trilhões")) + ) + | pynutil.insert("000", weight=0.1) + ) + + ((graph_e + graph_bilhao_prefix_e) | (delete_space + graph_bilhao_no_prefix)) + ) + + graph_quatrilhao_prefix_e = pynini.union( + ( + graph_hundred_component_prefix_e + + delete_space + + (pynutil.delete("quatrilhão") | pynutil.delete("quatrilhões")) + ) + + ((graph_e + graph_trilhao_prefix_e) | (delete_space + graph_trilhao_no_prefix)) + ) + + graph_quatrilhao_no_prefix = pynini.union( + ( + ( + graph_hundred_component_no_prefix + + delete_space + + (pynutil.delete("quatrilhão") | pynutil.delete("quatrilhões")) + ) + | pynutil.insert("000", weight=0.1) + ) + + ((graph_e + graph_trilhao_prefix_e) | (delete_space + graph_trilhao_no_prefix)) + ) + + graph_quintilhao_prefix_e = pynini.union( + ( + graph_hundred_component_prefix_e + + delete_space + + (pynutil.delete("quintilhão") | pynutil.delete("quintilhões")) + ) + + ((graph_e + graph_quatrilhao_prefix_e) | (delete_space + graph_quatrilhao_no_prefix)) + ) + + graph_quintilhao_no_prefix = pynini.union( + ( + ( + graph_hundred_component_no_prefix + + delete_space + + (pynutil.delete("quintilhão") | pynutil.delete("quintilhões")) + ) + | pynutil.insert("000", weight=0.1) + ) + + ((graph_e + graph_quatrilhao_prefix_e) | (delete_space + graph_quatrilhao_no_prefix)) + ) + + graph_sextilhao_prefix_e = pynini.union( + ( + graph_hundred_component_prefix_e + + delete_space + + (pynutil.delete("sextilhão") | pynutil.delete("sextilhões")) + ) + + ((graph_e + graph_quintilhao_prefix_e) | (delete_space + graph_quintilhao_no_prefix)) + ) + + graph_sextilhao_no_prefix = pynini.union( + ( + ( + graph_hundred_component_no_prefix + + delete_space + + (pynutil.delete("sextilhão") | pynutil.delete("sextilhões")) + ) + | pynutil.insert("000", weight=0.1) + ) + + ((graph_e + graph_quintilhao_prefix_e) | (delete_space + graph_quintilhao_no_prefix)) + ) + + graph = pynini.union( + graph_sextilhao_no_prefix, + graph_sextilhao_prefix_e, + graph_quintilhao_prefix_e, + graph_quatrilhao_prefix_e, + graph_trilhao_prefix_e, + graph_bilhao_prefix_e, + graph_milhao_prefix_e, + graph_mil_prefix_e, + graph_hundred_component_prefix_e, + graph_ties_component, + graph_zero, + ).optimize() + + graph = graph @ pynini.union( + pynutil.delete(pynini.closure("0")) + pynini.difference(NEMO_DIGIT, "0") + pynini.closure(NEMO_DIGIT), + "0", + ) + + graph = graph.optimize() + self.graph_no_exception = graph + + # save self.numbers_up_to_thousand for use in DecimalFst + digits_up_to_thousand = NEMO_DIGIT | (NEMO_DIGIT ** 2) | (NEMO_DIGIT ** 3) + numbers_up_to_thousand = pynini.compose(graph, digits_up_to_thousand).optimize() + self.numbers_up_to_thousand = numbers_up_to_thousand + + # save self.numbers_up_to_million for use in DecimalFst + digits_up_to_million = ( + NEMO_DIGIT + | (NEMO_DIGIT ** 2) + | (NEMO_DIGIT ** 3) + | (NEMO_DIGIT ** 4) + | (NEMO_DIGIT ** 5) + | (NEMO_DIGIT ** 6) + ) + numbers_up_to_million = pynini.compose(graph, digits_up_to_million).optimize() + self.numbers_up_to_million = numbers_up_to_million + + # don't convert cardinals from zero to nine inclusive + graph_exception = pynini.project(pynini.union(graph_digit, graph_zero), 'input') + + self.graph = (pynini.project(graph, "input") - graph_exception.arcsort()) @ graph + + optional_minus_graph = pynini.closure( + pynutil.insert("negative: ") + pynini.cross("menos", "\"-\"") + NEMO_SPACE, 0, 1 + ) + + final_graph = optional_minus_graph + pynutil.insert("integer: \"") + self.graph + pynutil.insert("\"") + + final_graph = self.add_tokens(final_graph) + self.fst = final_graph.optimize() diff --git a/nemo_text_processing/inverse_text_normalization/pt/taggers/date.py b/nemo_text_processing/inverse_text_normalization/pt/taggers/date.py new file mode 100644 index 000000000000..f0cd2b94c8e1 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/pt/taggers/date.py @@ -0,0 +1,59 @@ +# Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from nemo_text_processing.inverse_text_normalization.pt.utils import get_abs_path +from nemo_text_processing.text_normalization.en.graph_utils import GraphFst, delete_extra_space, delete_space +from pynini.lib import pynutil + + +class DateFst(GraphFst): + """ + Finite state transducer for classifying date, + e.g. primeiro de janeiro -> date { day: "1" month: "janeiro" } + e.g. um de janeiro -> date { day: "1" month: "janeiro" } + """ + + def __init__(self): + super().__init__(name="date", kind="classify") + + graph_digit = pynini.string_file(get_abs_path("data/numbers/digit.tsv")) + graph_ties = pynini.string_file(get_abs_path("data/numbers/ties.tsv")) + graph_teen = pynini.string_file(get_abs_path("data/numbers/teen.tsv")) + graph_twenties = pynini.string_file(get_abs_path("data/numbers/twenties.tsv")) + + graph_1_to_100 = pynini.union( + graph_digit, + graph_twenties, + graph_teen, + (graph_ties + pynutil.insert("0")), + (graph_ties + pynutil.delete(" e ") + graph_digit), + ) + + digits_1_to_31 = [str(digits) for digits in range(1, 32)] + graph_1_to_31 = graph_1_to_100 @ pynini.union(*digits_1_to_31) + # can use "primeiro" for 1st day of the month + graph_1_to_31 = pynini.union(graph_1_to_31, pynini.cross("primeiro", "1")) + + day_graph = pynutil.insert("day: \"") + graph_1_to_31 + pynutil.insert("\"") + + month_graph = pynini.string_file(get_abs_path("data/months.tsv")) + month_graph = pynutil.insert("month: \"") + month_graph + pynutil.insert("\"") + + graph_dm = day_graph + delete_space + pynutil.delete("de") + delete_extra_space + month_graph + + final_graph = graph_dm + final_graph += pynutil.insert(" preserve_order: true") + final_graph = self.add_tokens(final_graph) + self.fst = final_graph.optimize() diff --git a/nemo_text_processing/inverse_text_normalization/pt/taggers/decimal.py b/nemo_text_processing/inverse_text_normalization/pt/taggers/decimal.py new file mode 100644 index 000000000000..dab779965ed3 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/pt/taggers/decimal.py @@ -0,0 +1,119 @@ +# Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from nemo_text_processing.inverse_text_normalization.pt.utils import get_abs_path +from nemo_text_processing.text_normalization.en.graph_utils import ( + NEMO_DIGIT, + GraphFst, + delete_extra_space, + delete_space, +) +from pynini.lib import pynutil + + +def get_quantity(decimal: 'pynini.FstLike', cardinal_up_to_million: 'pynini.FstLike') -> 'pynini.FstLike': + """ + Returns FST that transforms either a cardinal or decimal followed by a quantity into a numeral, + e.g. one million -> integer_part: "1" quantity: "million" + e.g. one point five million -> integer_part: "1" fractional_part: "5" quantity: "million" + + Args: + decimal: decimal FST + cardinal_up_to_million: cardinal FST + """ + numbers = cardinal_up_to_million @ ( + pynutil.delete(pynini.closure("0")) + pynini.difference(NEMO_DIGIT, "0") + pynini.closure(NEMO_DIGIT) + ) + + suffix = pynini.union( + "milhão", + "milhões", + "bilhão", + "bilhões", + "trilhão", + "trilhões", + "quatrilhão", + "quatrilhões", + "quintilhão", + "quintilhões", + "sextilhão", + "sextilhões", + ) + res = ( + pynutil.insert("integer_part: \"") + + numbers + + pynutil.insert("\"") + + delete_extra_space + + pynutil.insert("quantity: \"") + + suffix + + pynutil.insert("\"") + ) + res |= decimal + delete_extra_space + pynutil.insert("quantity: \"") + suffix + pynutil.insert("\"") + return res + + +class DecimalFst(GraphFst): + """ + Finite state transducer for classifying decimal + Decimal point is either "." or ",", determined by whether "ponto" or "vírgula" is spoken. + e.g. menos um vírgula dois seis -> decimal { negative: "true" integer_part: "1" morphosyntactic_features: "," fractional_part: "26" } + e.g. menos um ponto dois seis -> decimal { negative: "true" integer_part: "1" morphosyntactic_features: "." fractional_part: "26" } + + This decimal rule assumes that decimals can be pronounced as: + (a cardinal) + ('vírgula' or 'ponto') plus (any sequence of cardinals <1000, including 'zero') + + Also writes large numbers in shortened form, e.g. + e.g. um vírgula dois seis milhões -> decimal { negative: "false" integer_part: "1" morphosyntactic_features: "," fractional_part: "26" quantity: "milhões" } + e.g. dois milhões -> decimal { negative: "false" integer_part: "2" quantity: "milhões" } + e.g. mil oitcentos e vinte e quatro milhões -> decimal { negative: "false" integer_part: "1824" quantity: "milhões" } + Args: + cardinal: CardinalFst + + """ + + def __init__(self, cardinal: GraphFst): + super().__init__(name="decimal", kind="classify") + + # number after decimal point can be any series of cardinals <1000, including 'zero' + graph_decimal = cardinal.numbers_up_to_thousand + graph_decimal = pynini.closure(graph_decimal + delete_space) + graph_decimal + self.graph = graph_decimal + + # decimal point can be denoted by 'vírgula' or 'ponto' + decimal_point = pynini.cross("vírgula", "morphosyntactic_features: \",\"") + decimal_point |= pynini.cross("ponto", "morphosyntactic_features: \".\"") + + optional_graph_negative = pynini.closure( + pynutil.insert("negative: ") + pynini.cross("menos", "\"true\"") + delete_extra_space, 0, 1 + ) + + graph_fractional = pynutil.insert("fractional_part: \"") + graph_decimal + pynutil.insert("\"") + + cardinal_graph = cardinal.graph_no_exception | pynini.string_file(get_abs_path("data/numbers/zero.tsv")) + graph_integer = pynutil.insert("integer_part: \"") + cardinal_graph + pynutil.insert("\"") + final_graph_wo_sign = ( + pynini.closure(graph_integer + delete_extra_space, 0, 1) + + decimal_point + + delete_extra_space + + graph_fractional + ) + final_graph = optional_graph_negative + final_graph_wo_sign + + self.final_graph_wo_negative = final_graph_wo_sign | get_quantity( + final_graph_wo_sign, cardinal.numbers_up_to_million + ) + final_graph |= optional_graph_negative + get_quantity(final_graph_wo_sign, cardinal.numbers_up_to_million) + final_graph = self.add_tokens(final_graph) + self.fst = final_graph.optimize() diff --git a/nemo_text_processing/inverse_text_normalization/pt/taggers/electronic.py b/nemo_text_processing/inverse_text_normalization/pt/taggers/electronic.py new file mode 100644 index 000000000000..aa152b116a20 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/pt/taggers/electronic.py @@ -0,0 +1,96 @@ +# Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from nemo_text_processing.inverse_text_normalization.pt.utils import get_abs_path +from nemo_text_processing.text_normalization.en.graph_utils import NEMO_ALPHA, GraphFst, insert_space +from pynini.lib import pynutil + + +class ElectronicFst(GraphFst): + """ + Finite state transducer for classifying 'electronic' semiotic classes, i.e. + email address (which get converted to "username" and "domain" fields), + and URLS (which get converted to a "protocol" field). + e.g. c d f um arroba a b c ponto e d u -> tokens { electronic { username: "cdf1" domain: "abc.edu" } } + e.g. dáblio dáblio dáblio a b c ponto e d u -> tokens { electronic { protocol: "www.abc.edu" } } + """ + + def __init__(self): + super().__init__(name="electronic", kind="classify") + + delete_extra_space = pynutil.delete(" ") + alpha_num = ( + NEMO_ALPHA + | pynini.string_file(get_abs_path("data/numbers/digit.tsv")) + | pynini.string_file(get_abs_path("data/numbers/zero.tsv")) + ) + + symbols = pynini.string_file(get_abs_path("data/electronic/symbols.tsv")).invert() + + accepted_username = alpha_num | symbols + process_dot = pynini.cross("ponto", ".") + username = ( + pynutil.insert("username: \"") + + alpha_num + + delete_extra_space + + pynini.closure(accepted_username + delete_extra_space) + + alpha_num + + pynutil.insert("\"") + ) + single_alphanum = pynini.closure(alpha_num + delete_extra_space) + alpha_num + server = single_alphanum | pynini.string_file(get_abs_path("data/electronic/server_name.tsv")).invert() + domain = single_alphanum | pynini.string_file(get_abs_path("data/electronic/domain.tsv")).invert() + domain_graph = ( + pynutil.insert("domain: \"") + + server + + delete_extra_space + + process_dot + + delete_extra_space + + domain + + pynutil.insert("\"") + ) + graph = ( + username + delete_extra_space + pynutil.delete("arroba") + insert_space + delete_extra_space + domain_graph + ) + + ############# url ### + protocol_end = pynini.cross(pynini.union("www", "w w w", "dáblio dáblio dáblio"), "www") + protocol_start = pynini.cross(pynini.union("http", "h t t p", "agá tê tê pê"), "http") + protocol_start |= pynini.cross(pynini.union("https", "h t t p s", "agá tê tê pê ésse"), "https") + protocol_start += pynini.cross(" dois pontos barra barra ", "://") + + # e.g. .com, .es + ending = ( + delete_extra_space + + symbols + + delete_extra_space + + (domain | pynini.closure(accepted_username + delete_extra_space) + accepted_username) + ) + + protocol = ( + pynini.closure(protocol_start, 0, 1) + + protocol_end + + delete_extra_space + + process_dot + + delete_extra_space + + (pynini.closure(delete_extra_space + accepted_username, 1) | server) + + pynini.closure(ending, 1) + ) + protocol = pynutil.insert("protocol: \"") + protocol + pynutil.insert("\"") + graph |= protocol + ######## + + final_graph = self.add_tokens(graph) + self.fst = final_graph.optimize() diff --git a/nemo_text_processing/inverse_text_normalization/pt/taggers/measure.py b/nemo_text_processing/inverse_text_normalization/pt/taggers/measure.py new file mode 100644 index 000000000000..7b6f1015ad79 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/pt/taggers/measure.py @@ -0,0 +1,95 @@ +# Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from nemo_text_processing.inverse_text_normalization.pt.utils import get_abs_path +from nemo_text_processing.text_normalization.en.graph_utils import ( + NEMO_SIGMA, + GraphFst, + convert_space, + delete_extra_space, + delete_space, +) +from pynini.lib import pynutil + + +class MeasureFst(GraphFst): + """ + Finite state transducer for classifying measure + e.g. menos doze quilogramas -> measure { cardinal { negative: "true" integer: "12" } units: "kg" } + + Args: + cardinal: CardinalFst + decimal: DecimalFst + """ + + def __init__(self, cardinal: GraphFst, decimal: GraphFst): + super().__init__(name="measure", kind="classify") + + cardinal_graph = cardinal.graph_no_exception + + graph_unit_singular = pynini.string_file(get_abs_path("data/measurements_singular.tsv")).invert() + graph_unit_plural = pynini.string_file(get_abs_path("data/measurements_plural.tsv")).invert() + + optional_graph_negative = pynini.closure( + pynutil.insert("negative: ") + pynini.cross("menos", "\"true\"") + delete_extra_space, 0, 1 + ) + + unit_singular = convert_space(graph_unit_singular) + unit_plural = convert_space(graph_unit_plural) + unit_misc = pynutil.insert("/") + pynutil.delete("por") + delete_space + convert_space(graph_unit_singular) + + unit_singular = ( + pynutil.insert("units: \"") + + (unit_singular | unit_misc | pynutil.add_weight(unit_singular + delete_space + unit_misc, 0.01)) + + pynutil.insert("\"") + ) + unit_plural = ( + pynutil.insert("units: \"") + + (unit_plural | unit_misc | pynutil.add_weight(unit_plural + delete_space + unit_misc, 0.01)) + + pynutil.insert("\"") + ) + + subgraph_decimal = ( + pynutil.insert("decimal { ") + + optional_graph_negative + + decimal.final_graph_wo_negative + + pynutil.insert(" }") + + delete_extra_space + + unit_plural + ) + subgraph_cardinal = ( + pynutil.insert("cardinal { ") + + optional_graph_negative + + pynutil.insert("integer: \"") + + ((NEMO_SIGMA - "um" - "uma") @ cardinal_graph) + + pynutil.insert("\"") + + pynutil.insert(" }") + + delete_extra_space + + unit_plural + ) + subgraph_cardinal |= ( + pynutil.insert("cardinal { ") + + optional_graph_negative + + pynutil.insert("integer: \"") + + (pynini.cross("um", "1") | pynini.cross("uma", "1")) + + pynutil.insert("\"") + + pynutil.insert(" }") + + delete_extra_space + + unit_singular + ) + + final_graph = subgraph_decimal | subgraph_cardinal + final_graph = self.add_tokens(final_graph) + self.fst = final_graph.optimize() diff --git a/nemo_text_processing/inverse_text_normalization/pt/taggers/money.py b/nemo_text_processing/inverse_text_normalization/pt/taggers/money.py new file mode 100644 index 000000000000..cc3639438b83 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/pt/taggers/money.py @@ -0,0 +1,127 @@ +# Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from nemo_text_processing.inverse_text_normalization.pt.utils import get_abs_path +from nemo_text_processing.text_normalization.en.graph_utils import ( + NEMO_DIGIT, + NEMO_SIGMA, + GraphFst, + convert_space, + delete_extra_space, + delete_space, + insert_space, +) +from pynini.lib import pynutil + + +class MoneyFst(GraphFst): + """ + Finite state transducer for classifying money + e.g. doze dólares e cinco centavos -> money { integer_part: "12" fractional_part: "05" currency: "$" } + + Args: + cardinal: CardinalFst + decimal: DecimalFst + """ + + def __init__(self, cardinal: GraphFst, decimal: GraphFst): + super().__init__(name="money", kind="classify") + # quantity, integer_part, fractional_part, currency + + cardinal_graph = cardinal.graph_no_exception + graph_decimal_final = decimal.final_graph_wo_negative + + unit_singular = pynini.string_file(get_abs_path("data/currency_singular.tsv")).invert() + unit_plural = pynini.string_file(get_abs_path("data/currency_plural.tsv")).invert() + + graph_unit_singular = pynutil.insert("currency: \"") + convert_space(unit_singular) + pynutil.insert("\"") + graph_unit_plural = pynutil.insert("currency: \"") + convert_space(unit_plural) + pynutil.insert("\"") + + add_leading_zero_to_double_digit = (NEMO_DIGIT + NEMO_DIGIT) | (pynutil.insert("0") + NEMO_DIGIT) + # twelve dollars (and) fifty cents, zero cents + cents_standalone = ( + pynutil.insert("morphosyntactic_features: \",\"") # always use a comma in the decimal + + insert_space + + pynutil.insert("fractional_part: \"") + + pynini.union( + pynutil.add_weight(((NEMO_SIGMA - "um" - "uma") @ cardinal_graph), -0.7) + @ add_leading_zero_to_double_digit + + delete_space + + pynutil.delete(pynini.union("centavos")), + pynini.cross("um", "01") + delete_space + pynutil.delete(pynini.union("centavo")), + ) + + pynutil.insert("\"") + ) + + optional_cents_standalone = pynini.closure( + delete_space + + pynini.closure((pynutil.delete("com") | pynutil.delete('e')) + delete_space, 0, 1) + + insert_space + + cents_standalone, + 0, + 1, + ) + + # twelve dollars fifty, only after integer + # setenta e cinco dólares com sessenta e três ~ $75,63 + optional_cents_suffix = pynini.closure( + delete_extra_space + + pynutil.insert("morphosyntactic_features: \",\"") # always use a comma in the decimal + + insert_space + + pynutil.insert("fractional_part: \"") + + pynini.closure((pynutil.delete("com") | pynutil.delete('e')) + delete_space, 0, 1) + + pynutil.add_weight(cardinal_graph @ add_leading_zero_to_double_digit, -0.7) + + pynutil.insert("\""), + 0, + 1, + ) + + graph_integer = ( + pynutil.insert("integer_part: \"") + + ((NEMO_SIGMA - "um" - "uma") @ cardinal_graph) + + pynutil.insert("\"") + + delete_extra_space + + graph_unit_plural + + (optional_cents_standalone | optional_cents_suffix) + ) + graph_integer |= ( + pynutil.insert("integer_part: \"") + + (pynini.cross("um", "1") | pynini.cross("uma", "1")) + + pynutil.insert("\"") + + delete_extra_space + + graph_unit_singular + + (optional_cents_standalone | optional_cents_suffix) + ) + + graph_cents_standalone = pynini.union( + pynutil.insert("currency: \"R$\" integer_part: \"0\" ") + cents_standalone, + pynutil.add_weight( + pynutil.insert("integer_part: \"0\" ") + + cents_standalone + + delete_extra_space + + pynutil.delete("de") + + delete_space + + graph_unit_singular, + -0.1, + ), + ) + + graph_decimal = ( + graph_decimal_final + delete_extra_space + (pynutil.delete("de") + delete_space).ques + graph_unit_plural + ) + graph_decimal |= graph_cents_standalone + final_graph = graph_integer | graph_decimal + final_graph = self.add_tokens(final_graph) + self.fst = final_graph.optimize() diff --git a/nemo_text_processing/inverse_text_normalization/pt/taggers/ordinal.py b/nemo_text_processing/inverse_text_normalization/pt/taggers/ordinal.py new file mode 100644 index 000000000000..ff7f3fbf02fa --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/pt/taggers/ordinal.py @@ -0,0 +1,84 @@ +# Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from nemo_text_processing.inverse_text_normalization.pt.utils import get_abs_path +from nemo_text_processing.text_normalization.en.graph_utils import NEMO_SIGMA, GraphFst, delete_space +from pynini.lib import pynutil + + +class OrdinalFst(GraphFst): + """ + Finite state transducer for classifying ordinal + vigésimo primeiro -> ordinal { integer: "21" morphosyntactic_features: "o" } + This class converts ordinal up to "milésimo" (one thousandth) exclusive. + + Cardinals below ten are not converted (in order to avoid + e.g. "primero fez ..." -> "1º fez...", "segunda guerra mundial" -> "2ª guerra mundial" + and any other odd conversions.) + + This FST also records the ending of the ordinal (called "morphosyntactic_features"): + either "o" or "a". + + Args: + cardinal: CardinalFst + """ + + def __init__(self): + super().__init__(name="ordinal", kind="classify") + + graph_digit = pynini.string_file(get_abs_path("data/ordinals/digit.tsv")) + graph_ties = pynini.string_file(get_abs_path("data/ordinals/ties.tsv")) + graph_hundreds = pynini.string_file(get_abs_path("data/ordinals/hundreds.tsv")) + + ordinal_graph_union = pynini.union( + pynutil.add_weight(graph_digit, 0.4), + pynutil.add_weight(graph_ties + ((delete_space + graph_digit) | pynutil.insert("0")), 0.2), + graph_hundreds + + ((delete_space + graph_ties) | pynutil.insert("0")) + + ((delete_space + graph_digit) | pynutil.insert("0")), + ) + + accept_o_endings = NEMO_SIGMA + pynini.accep("o") + accept_a_endings = NEMO_SIGMA + pynini.accep("a") + + ordinal_graph_o = accept_o_endings @ ordinal_graph_union + ordinal_graph_a = accept_a_endings @ ordinal_graph_union + + # 'optional_numbers_in_front' have negative weight so we always + # include them if they're there + optional_in_front = (pynutil.add_weight(ordinal_graph_union, -0.1) + delete_space.closure()).closure() + graph_o_suffix = optional_in_front + ordinal_graph_o + graph_a_suffix = optional_in_front + ordinal_graph_a + + # don't convert ordinals from one to nine inclusive + graph_exception = pynini.project(pynini.union(graph_digit), 'input') + graph_o_suffix = (pynini.project(graph_o_suffix, "input") - graph_exception.arcsort()) @ graph_o_suffix + graph_a_suffix = (pynini.project(graph_a_suffix, "input") - graph_exception.arcsort()) @ graph_a_suffix + + graph = ( + pynutil.insert("integer: \"") + + graph_o_suffix + + pynutil.insert("\"") + + pynutil.insert(" morphosyntactic_features: \"o\"") + ) + graph |= ( + pynutil.insert("integer: \"") + + graph_a_suffix + + pynutil.insert("\"") + + pynutil.insert(" morphosyntactic_features: \"a\"") + ) + + final_graph = self.add_tokens(graph) + self.fst = final_graph.optimize() diff --git a/nemo_text_processing/inverse_text_normalization/pt/taggers/punctuation.py b/nemo_text_processing/inverse_text_normalization/pt/taggers/punctuation.py new file mode 100644 index 000000000000..cb5285452954 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/pt/taggers/punctuation.py @@ -0,0 +1,34 @@ +# Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from nemo_text_processing.text_normalization.en.graph_utils import GraphFst +from pynini.lib import pynutil + + +class PunctuationFst(GraphFst): + """ + Finite state transducer for classifying punctuation + e.g. a, -> tokens { name: "a" } tokens { name: "," } + """ + + def __init__(self): + super().__init__(name="punctuation", kind="classify") + + s = "!#$%&\'()*+,-./:;<=>?@^_`{|}~" + punct = pynini.union(*s) + + graph = pynutil.insert("name: \"") + punct + pynutil.insert("\"") + + self.fst = graph.optimize() diff --git a/nemo_text_processing/inverse_text_normalization/pt/taggers/telephone.py b/nemo_text_processing/inverse_text_normalization/pt/taggers/telephone.py new file mode 100755 index 000000000000..a1ad2d07585d --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/pt/taggers/telephone.py @@ -0,0 +1,131 @@ +# Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from nemo_text_processing.inverse_text_normalization.pt.utils import get_abs_path +from nemo_text_processing.text_normalization.en.graph_utils import GraphFst, delete_space, insert_space +from pynini.lib import pynutil + + +class TelephoneFst(GraphFst): + """ + Finite state transducer for classifying telephone numbers, e.g. + um dois um dois três quatro cinco seis sete oito nove -> { number_part: "(12) 12345-6789" }. + If 11 digits are spoken, they are grouped as 2+5+4 (eg. (12) 34567-8901). + If 10 digits are spoken, they are grouped as 2+4+4 (eg. (12) 3456-7890). + If 9 digits are spoken, they are grouped as 5+4 (eg. 12345-6789). + If 8 digits are spoken, they are grouped as 4+4 (eg. 1234-5678). + In portuguese, digits are generally spoken individually, or as 2-digit numbers, + eg. "trinta e quatro oitenta e dois" = "3482", + "meia sete vinte" = "6720". + """ + + def __init__(self): + super().__init__(name="telephone", kind="classify") + + # create `single_digits` and `double_digits` graphs as these will be + # the building blocks of possible telephone numbers + graph_digit = pynini.string_file(get_abs_path("data/numbers/digit.tsv")) + graph_ties = pynini.string_file(get_abs_path("data/numbers/ties.tsv")) + graph_twenties = pynini.string_file(get_abs_path("data/numbers/twenties.tsv")) + graph_teen = pynini.string_file(get_abs_path("data/numbers/teen.tsv")) + graph_zero = pynini.string_file(get_abs_path("data/numbers/zero.tsv")) + graph_half = pynini.cross("meia", "6") + + graph_all_digits = pynini.union(graph_digit, graph_half, graph_zero) + + single_digits = pynini.invert(graph_all_digits).optimize() + + double_digits = ( + pynini.union( + graph_teen | graph_twenties, + (graph_ties + pynutil.insert("0")), + (graph_ties + delete_space + pynutil.delete("e") + delete_space + graph_digit), + (graph_all_digits + delete_space + graph_all_digits), + ) + .invert() + .optimize() + ) + + # define `eleven_digit_graph`, `ten_digit_graph`, `nine_digit_graph`, `eight_digit_graph` + # which accept telephone numbers spoken (1) only with single digits, + # or (2) spoken with double digits (and sometimes single digits) + + # 11-digit option (2): (2) + (1+2+2) + (2+2) digits + eleven_digit_graph = ( + pynutil.delete("(") + + double_digits + + insert_space + + pynutil.delete(") ") + + single_digits + + insert_space + + double_digits + + insert_space + + double_digits + + insert_space + + pynutil.delete("-") + + double_digits + + insert_space + + double_digits + ) + + # 10-digit option (2): (2) + (2+2) + (2+2) digits + ten_digit_graph = ( + pynutil.delete("(") + + double_digits + + insert_space + + pynutil.delete(") ") + + double_digits + + insert_space + + double_digits + + insert_space + + pynutil.delete("-") + + double_digits + + insert_space + + double_digits + ) + + # 9-digit option (2): (1+2+2) + (2+2) digits + nine_digit_graph = ( + single_digits + + insert_space + + double_digits + + insert_space + + double_digits + + insert_space + + pynutil.delete("-") + + double_digits + + insert_space + + double_digits + ) + + # 8-digit option (2): (2+2) + (2+2) digits + eight_digit_graph = ( + double_digits + + insert_space + + double_digits + + insert_space + + pynutil.delete("-") + + double_digits + + insert_space + + double_digits + ) + + number_part = pynini.union(eleven_digit_graph, ten_digit_graph, nine_digit_graph, eight_digit_graph) + + number_part = pynutil.insert("number_part: \"") + pynini.invert(number_part) + pynutil.insert("\"") + + graph = number_part + final_graph = self.add_tokens(graph) + self.fst = final_graph.optimize() diff --git a/nemo_text_processing/inverse_text_normalization/pt/taggers/time.py b/nemo_text_processing/inverse_text_normalization/pt/taggers/time.py new file mode 100755 index 000000000000..e669abd11836 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/pt/taggers/time.py @@ -0,0 +1,182 @@ +# Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from nemo_text_processing.inverse_text_normalization.pt.utils import get_abs_path +from nemo_text_processing.text_normalization.en.graph_utils import GraphFst, delete_space, insert_space +from pynini.lib import pynutil + + +class TimeFst(GraphFst): + """ + Finite state transducer for classifying time + e.g. quinze pro meio dia -> time { hours: "11" minutes: "45" } + e.g. quinze pra meia noite -> time { hours: "23" minutes: "45" } + e.g. quinze pra uma -> time { hours: "12" minutes: "45" } + e.g. dez pras duas -> time { hours: "1" minutes: "50" } + e.g. quinze pras duas -> time { hours: "1" minutes: "45" } + e.g. ao meio dia -> time { hours: "12" minutes: "00" morphosyntactic_features: "ao" } + e.g. ao meio dia e meia -> time { hours: "12" minutes: "30" morphosyntactic_features: "ao" } + e.g. ao meio dia e meio -> time { hours: "12" minutes: "30" morphosyntactic_features: "ao" } + e.g. à meia noite e quinze -> time { hours: "0" minutes: "15" morphosyntactic_features: "à" } + e.g. à meia noite e meia -> time { hours: "0" minutes: "30" morphosyntactic_features: "à" } + e.g. à uma e trinta -> time { hours: "1" minutes: "30" morphosyntactic_features: "à" } + e.g. às onze e trinta -> time { hours: "11" minutes: "30" morphosyntactic_features: "às" } + e.g. às três horas e trinta minutos -> time { hours: "3" minutes: "30" morphosyntactic_features: "às" } + """ + + def __init__(self): + super().__init__(name="time", kind="classify") + + # graph_hour_to_am = pynini.string_file(get_abs_path("data/time/hour_to_am.tsv")) + # graph_hour_to_pm = pynini.string_file(get_abs_path("data/time/hour_to_pm.tsv")) + graph_hours_to = pynini.string_file(get_abs_path("data/time/hours_to.tsv")) + graph_minutes_to = pynini.string_file(get_abs_path("data/time/minutes_to.tsv")) + graph_suffix_am = pynini.string_file(get_abs_path("data/time/time_suffix_am.tsv")) + graph_suffix_pm = pynini.string_file(get_abs_path("data/time/time_suffix_pm.tsv")) + + graph_digit = pynini.string_file(get_abs_path("data/numbers/digit.tsv")) + graph_ties = pynini.string_file(get_abs_path("data/numbers/ties.tsv")) + graph_teen = pynini.string_file(get_abs_path("data/numbers/teen.tsv")) + graph_twenties = pynini.string_file(get_abs_path("data/numbers/twenties.tsv")) + + graph_1_to_100 = pynini.union( + graph_digit, + graph_twenties, + graph_teen, + (graph_ties + pynutil.insert("0")), + (graph_ties + pynutil.delete(" e ") + graph_digit), + ) + + # note that graph_hour will start from 2 hours + # "1 o'clock" will be treated differently because it + # is singular + digits_2_to_23 = [str(digits) for digits in range(2, 24)] + digits_1_to_59 = [str(digits) for digits in range(1, 60)] + + graph_2_to_23 = graph_1_to_100 @ pynini.union(*digits_2_to_23) + graph_1_to_59 = graph_1_to_100 @ pynini.union(*digits_1_to_59) + graph_uma = pynini.cross("uma", "1") + + # Mapping 'horas' + graph_hour = pynutil.delete(pynini.accep("hora") + pynini.accep("s").ques) + graph_minute = pynutil.delete(pynini.accep("minuto") + pynini.accep("s").ques) + + # Mapping 'meio dia' and 'meia noite' + graph_meio_dia = pynini.cross("meio dia", "12") + graph_meia_noite = pynini.cross("meia noite", "0") + + # Mapping 'e meia' + graph_e = delete_space + pynutil.delete(" e ") + delete_space + graph_e_meia = graph_e + pynini.cross("meia", "30") + graph_e_meio = graph_e + pynini.cross("meio", "30") + + # à uma hora -> 1:00 + # às três e meia -> 3:30 + graph_hours_at_singular = ( + pynutil.insert("morphosyntactic_features: \"") + + (pynini.cross("à", "à") | pynini.cross("a", "à")) + + pynutil.insert("\" ") + + delete_space + ) + graph_hours_at_singular += ( + pynutil.insert("hours: \"") + graph_uma + pynutil.insert("\"") + (delete_space + graph_hour).ques + ) + graph_hours_at_plural = ( + pynutil.insert("morphosyntactic_features: \"") + + (pynini.cross("às", "às") | pynini.cross("as", "às")) + + pynutil.insert("\" ") + + delete_space + ) + graph_hours_at_plural += ( + pynutil.insert("hours: \"") + graph_2_to_23 + pynutil.insert("\"") + (delete_space + graph_hour).ques + ) + final_graph_hour_at = graph_hours_at_singular | graph_hours_at_plural + + graph_minutes_component_without_zero = graph_e + graph_1_to_59 + (delete_space + graph_minute).ques + graph_minutes_component_without_zero |= graph_e_meia + pynutil.delete(delete_space + pynini.accep("hora")).ques + graph_minutes_component = graph_minutes_component_without_zero | pynutil.insert("00", weight=0.1) + final_graph_minute = pynutil.insert(" minutes: \"") + graph_minutes_component + pynutil.insert("\"") + + graph_hm = final_graph_hour_at + final_graph_minute + + # meio dia e meia -> 12:30 + # meia noite e meia -> 0:30 + graph_minutes_without_zero = ( + pynutil.insert(" minutes: \"") + graph_minutes_component_without_zero + pynutil.insert("\"") + ) + graph_meio_min = ( + pynutil.insert("hours: \"") + + (graph_meio_dia | graph_meia_noite) + + pynutil.insert("\"") + + graph_minutes_without_zero + ) + graph_meio_min |= ( + pynutil.insert("hours: \"") + + graph_meio_dia + + pynutil.insert("\" minutes: \"") + + graph_e_meio + + pynutil.insert("\"") + ) + graph_hm |= graph_meio_min + + # às quinze para as quatro -> às 3:45 + # NOTE: case 'para à uma' ('to one') could be either 0:XX or 12:XX + # leading to wrong reading ('meio dia e ...' or 'meia noite e ...') + graph_para_a = ( + pynutil.delete("para") + | pynutil.delete("para a") + | pynutil.delete("para as") + | pynutil.delete("pra") + | pynutil.delete("pras") + ) + graph_para_o = pynutil.delete("para") | pynutil.delete("para o") | pynutil.delete("pro") + + graph_pra_min = ( + pynutil.insert("morphosyntactic_features: \"") + + (pynini.cross("à", "à") | pynini.cross("às", "às") | pynini.cross("a", "à") | pynini.cross("as", "às")) + + pynutil.insert("\" ") + + delete_space + ) + graph_pra_min += ( + pynutil.insert("minutes: \"") + + (graph_1_to_59 @ graph_minutes_to) + + pynutil.insert("\" ") + + (delete_space + graph_minute).ques + ) + graph_pra_hour = ( + pynutil.insert("hours: \"") + + (graph_2_to_23 @ graph_hours_to) + + pynutil.insert("\"") + + (delete_space + graph_hour).ques + ) + graph_pra_hour |= pynutil.insert("hours: \"") + (graph_meia_noite @ graph_hours_to) + pynutil.insert("\"") + + graph_pra = graph_pra_min + delete_space + graph_para_a + delete_space + graph_pra_hour + + # às quinze pro meio dia -> às 11:45 + graph_pro = graph_pra_min + delete_space + graph_para_o + delete_space + graph_pro += pynutil.insert(" hours: \"") + (graph_meio_dia @ graph_hours_to) + pynutil.insert("\"") + + graph_mh = graph_pra | graph_pro + + # optional suffix + final_suffix = pynutil.insert("suffix: \"") + (graph_suffix_am | graph_suffix_pm) + pynutil.insert("\"") + final_suffix_optional = pynini.closure(delete_space + insert_space + final_suffix, 0, 1) + + final_graph = pynini.union((graph_hm | graph_mh) + final_suffix_optional).optimize() + + final_graph = self.add_tokens(final_graph) + + self.fst = final_graph.optimize() diff --git a/nemo_text_processing/inverse_text_normalization/pt/taggers/tokenize_and_classify.py b/nemo_text_processing/inverse_text_normalization/pt/taggers/tokenize_and_classify.py new file mode 100644 index 000000000000..f4396645828d --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/pt/taggers/tokenize_and_classify.py @@ -0,0 +1,110 @@ +# Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +import pynini +from nemo_text_processing.inverse_text_normalization.pt.taggers.cardinal import CardinalFst +from nemo_text_processing.inverse_text_normalization.pt.taggers.date import DateFst +from nemo_text_processing.inverse_text_normalization.pt.taggers.decimal import DecimalFst +from nemo_text_processing.inverse_text_normalization.pt.taggers.electronic import ElectronicFst +from nemo_text_processing.inverse_text_normalization.pt.taggers.measure import MeasureFst +from nemo_text_processing.inverse_text_normalization.pt.taggers.money import MoneyFst +from nemo_text_processing.inverse_text_normalization.pt.taggers.ordinal import OrdinalFst +from nemo_text_processing.inverse_text_normalization.pt.taggers.punctuation import PunctuationFst +from nemo_text_processing.inverse_text_normalization.pt.taggers.telephone import TelephoneFst +from nemo_text_processing.inverse_text_normalization.pt.taggers.time import TimeFst +from nemo_text_processing.inverse_text_normalization.pt.taggers.whitelist import WhiteListFst +from nemo_text_processing.inverse_text_normalization.pt.taggers.word import WordFst +from nemo_text_processing.text_normalization.en.graph_utils import ( + GraphFst, + delete_extra_space, + delete_space, + generator_main, +) +from pynini.lib import pynutil + +from nemo.utils import logging + + +class ClassifyFst(GraphFst): + """ + Final class that composes all other classification grammars. This class can process an entire sentence, that is lower cased. + For deployment, this grammar will be compiled and exported to OpenFst Finate State Archiv (FAR) File. + More details to deployment at NeMo/tools/text_processing_deployment. + + Args: + cache_dir: path to a dir with .far grammar file. Set to None to avoid using cache. + overwrite_cache: set to True to overwrite .far files + """ + + def __init__(self, cache_dir: str = None, overwrite_cache: bool = False): + super().__init__(name="tokenize_and_classify", kind="classify") + + far_file = None + if cache_dir is not None and cache_dir != "None": + os.makedirs(cache_dir, exist_ok=True) + far_file = os.path.join(cache_dir, "_pt_itn.far") + if not overwrite_cache and far_file and os.path.exists(far_file): + self.fst = pynini.Far(far_file, mode="r")["tokenize_and_classify"] + logging.info(f"ClassifyFst.fst was restored from {far_file}.") + else: + logging.info(f"Creating ClassifyFst grammars.") + + cardinal = CardinalFst(use_strict_e=True) + cardinal_graph = cardinal.fst + + ordinal_graph = OrdinalFst().fst + + decimal = DecimalFst(cardinal) + decimal_graph = decimal.fst + + measure_graph = MeasureFst(cardinal=cardinal, decimal=decimal).fst + date_graph = DateFst().fst + word_graph = WordFst().fst + time_graph = TimeFst().fst + money_graph = MoneyFst(cardinal=cardinal, decimal=decimal).fst + whitelist_graph = WhiteListFst().fst + punct_graph = PunctuationFst().fst + electronic_graph = ElectronicFst().fst + telephone_graph = TelephoneFst().fst + + classify = ( + pynutil.add_weight(whitelist_graph, 1.01) + | pynutil.add_weight(time_graph, 1.1) + | pynutil.add_weight(date_graph, 1.09) + | pynutil.add_weight(decimal_graph, 1.09) + | pynutil.add_weight(measure_graph, 1.1) + | pynutil.add_weight(cardinal_graph, 1.1) + | pynutil.add_weight(ordinal_graph, 1.1) + | pynutil.add_weight(money_graph, 1.1) + | pynutil.add_weight(telephone_graph, 1.1) + | pynutil.add_weight(electronic_graph, 1.1) + | pynutil.add_weight(word_graph, 100) + ) + + punct = pynutil.insert("tokens { ") + pynutil.add_weight(punct_graph, weight=1.1) + pynutil.insert(" }") + token = pynutil.insert("tokens { ") + classify + pynutil.insert(" }") + token_plus_punct = ( + pynini.closure(punct + pynutil.insert(" ")) + token + pynini.closure(pynutil.insert(" ") + punct) + ) + + graph = token_plus_punct + pynini.closure(delete_extra_space + token_plus_punct) + graph = delete_space + graph + delete_space + + self.fst = graph.optimize() + + if far_file: + generator_main(far_file, {"tokenize_and_classify": self.fst}) + logging.info(f"ClassifyFst grammars are saved to {far_file}.") diff --git a/nemo_text_processing/inverse_text_normalization/pt/taggers/whitelist.py b/nemo_text_processing/inverse_text_normalization/pt/taggers/whitelist.py new file mode 100644 index 000000000000..6965ccbb8e70 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/pt/taggers/whitelist.py @@ -0,0 +1,33 @@ +# Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from nemo_text_processing.inverse_text_normalization.pt.utils import get_abs_path +from nemo_text_processing.text_normalization.en.graph_utils import GraphFst, convert_space +from pynini.lib import pynutil + + +class WhiteListFst(GraphFst): + """ + Finite state transducer for classifying whitelisted tokens + e.g. usted -> tokens { name: "ud." } + This class has highest priority among all classifier grammars. Whitelisted tokens are defined and loaded from "data/whitelist.tsv". + """ + + def __init__(self): + super().__init__(name="whitelist", kind="classify") + + whitelist = pynini.string_file(get_abs_path("data/whitelist.tsv")).invert() + graph = pynutil.insert("name: \"") + convert_space(whitelist) + pynutil.insert("\"") + self.fst = graph.optimize() diff --git a/nemo_text_processing/inverse_text_normalization/pt/taggers/word.py b/nemo_text_processing/inverse_text_normalization/pt/taggers/word.py new file mode 100644 index 000000000000..7908397d52ad --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/pt/taggers/word.py @@ -0,0 +1,29 @@ +# Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from nemo_text_processing.text_normalization.en.graph_utils import NEMO_NOT_SPACE, GraphFst +from pynini.lib import pynutil + + +class WordFst(GraphFst): + """ + Finite state transducer for classifying plain tokens, that do not belong to any special class. This can be considered as the default class. + e.g. sleep -> tokens { name: "sleep" } + """ + + def __init__(self): + super().__init__(name="word", kind="classify") + word = pynutil.insert("name: \"") + pynini.closure(NEMO_NOT_SPACE, 1) + pynutil.insert("\"") + self.fst = word.optimize() diff --git a/nemo_text_processing/inverse_text_normalization/pt/utils.py b/nemo_text_processing/inverse_text_normalization/pt/utils.py new file mode 100644 index 000000000000..a73b7d9ddb39 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/pt/utils.py @@ -0,0 +1,27 @@ +# Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + + +def get_abs_path(rel_path): + """ + Get absolute path + + Args: + rel_path: relative path to this file + + Returns absolute path + """ + return os.path.dirname(os.path.abspath(__file__)) + '/' + rel_path diff --git a/nemo_text_processing/inverse_text_normalization/pt/verbalizers/__init__.py b/nemo_text_processing/inverse_text_normalization/pt/verbalizers/__init__.py new file mode 100644 index 000000000000..a1cf281f0908 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/pt/verbalizers/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nemo_text_processing/inverse_text_normalization/pt/verbalizers/cardinal.py b/nemo_text_processing/inverse_text_normalization/pt/verbalizers/cardinal.py new file mode 100644 index 000000000000..928a259d3897 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/pt/verbalizers/cardinal.py @@ -0,0 +1,48 @@ +# Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from nemo_text_processing.text_normalization.en.graph_utils import NEMO_NOT_QUOTE, GraphFst, delete_space +from pynini.lib import pynutil + + +class CardinalFst(GraphFst): + """ + Finite state transducer for verbalizing cardinal + e.g. cardinal { negative: "-" integer: "23" } -> -23 + """ + + def __init__(self): + super().__init__(name="cardinal", kind="verbalize") + optional_sign = pynini.closure( + pynutil.delete("negative:") + + delete_space + + pynutil.delete("\"") + + NEMO_NOT_QUOTE + + pynutil.delete("\"") + + delete_space, + 0, + 1, + ) + graph = ( + pynutil.delete("integer:") + + delete_space + + pynutil.delete("\"") + + pynini.closure(NEMO_NOT_QUOTE, 1) + + pynutil.delete("\"") + ) + self.numbers = graph + graph = optional_sign + graph + delete_tokens = self.delete_tokens(graph) + self.fst = delete_tokens.optimize() diff --git a/nemo_text_processing/inverse_text_normalization/pt/verbalizers/date.py b/nemo_text_processing/inverse_text_normalization/pt/verbalizers/date.py new file mode 100644 index 000000000000..ee9db8fa7e1e --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/pt/verbalizers/date.py @@ -0,0 +1,65 @@ +# Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from nemo_text_processing.text_normalization.en.graph_utils import ( + NEMO_NOT_QUOTE, + GraphFst, + delete_extra_space, + delete_space, + insert_space, +) +from pynini.lib import pynutil + + +class DateFst(GraphFst): + """ + Finite state transducer for verbalizing date, e.g. + date { day: "1" month: "enero" preserve_order: true } -> 1 de enero + """ + + def __init__(self): + super().__init__(name="date", kind="verbalize") + month = ( + pynutil.delete("month:") + + delete_space + + pynutil.delete("\"") + + pynini.closure(NEMO_NOT_QUOTE, 1) + + pynutil.delete("\"") + ) + day = ( + pynutil.delete("day:") + + delete_space + + pynutil.delete("\"") + + pynini.closure(NEMO_NOT_QUOTE, 1) + + pynutil.delete("\"") + ) + + # day month + graph_dm = day + delete_extra_space + pynutil.insert("de") + insert_space + month + + optional_preserve_order = pynini.closure( + pynutil.delete("preserve_order:") + delete_space + pynutil.delete("true") + delete_space + | pynutil.delete("field_order:") + + delete_space + + pynutil.delete("\"") + + NEMO_NOT_QUOTE + + pynutil.delete("\"") + + delete_space + ) + + final_graph = graph_dm + delete_space + optional_preserve_order + + delete_tokens = self.delete_tokens(final_graph) + self.fst = delete_tokens.optimize() diff --git a/nemo_text_processing/inverse_text_normalization/pt/verbalizers/decimal.py b/nemo_text_processing/inverse_text_normalization/pt/verbalizers/decimal.py new file mode 100644 index 000000000000..58fc76ea63e6 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/pt/verbalizers/decimal.py @@ -0,0 +1,66 @@ +# Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from nemo_text_processing.text_normalization.en.graph_utils import NEMO_NOT_QUOTE, GraphFst, delete_space +from pynini.lib import pynutil + + +class DecimalFst(GraphFst): + """ + Finite state transducer for verbalizing decimal, + e.g. decimal { negative: "true" integer_part: "1" morphosyntactic_features: "," fractional_part: "26" } -> -1,26 + e.g. decimal { negative: "true" integer_part: "1" morphosyntactic_features: "." fractional_part: "26" } -> -1.26 + e.g. decimal { negative: "false" integer_part: "1" morphosyntactic_features: "," fractional_part: "26" quantity: "millón" } -> 1,26 millón + e.g. decimal { negative: "false" integer_part: "2" quantity: "millones" } -> 2 millones + """ + + def __init__(self): + super().__init__(name="decimal", kind="verbalize") + optionl_sign = pynini.closure(pynini.cross("negative: \"true\"", "-") + delete_space, 0, 1) + integer = ( + pynutil.delete("integer_part:") + + delete_space + + pynutil.delete("\"") + + pynini.closure(NEMO_NOT_QUOTE, 1) + + pynutil.delete("\"") + ) + optional_integer = pynini.closure(integer + delete_space, 0, 1) + + decimal_point = pynini.cross("morphosyntactic_features: \",\"", ",") + decimal_point |= pynini.cross("morphosyntactic_features: \".\"", ".") + + fractional = ( + decimal_point + + delete_space + + pynutil.delete("fractional_part:") + + delete_space + + pynutil.delete("\"") + + pynini.closure(NEMO_NOT_QUOTE, 1) + + pynutil.delete("\"") + ) + optional_fractional = pynini.closure(fractional + delete_space, 0, 1) + quantity = ( + pynutil.delete("quantity:") + + delete_space + + pynutil.delete("\"") + + pynini.closure(NEMO_NOT_QUOTE, 1) + + pynutil.delete("\"") + ) + optional_quantity = pynini.closure(pynutil.insert(" ") + quantity + delete_space, 0, 1) + graph = optional_integer + optional_fractional + optional_quantity + self.numbers = graph + graph = optionl_sign + graph + delete_tokens = self.delete_tokens(graph) + self.fst = delete_tokens.optimize() diff --git a/nemo_text_processing/inverse_text_normalization/pt/verbalizers/electronic.py b/nemo_text_processing/inverse_text_normalization/pt/verbalizers/electronic.py new file mode 100644 index 000000000000..11b2706a3562 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/pt/verbalizers/electronic.py @@ -0,0 +1,55 @@ +# Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from nemo_text_processing.text_normalization.en.graph_utils import NEMO_NOT_QUOTE, GraphFst, delete_space +from pynini.lib import pynutil + + +class ElectronicFst(GraphFst): + """ + Finite state transducer for verbalizing electronic + e.g. tokens { electronic { username: "cdf1" domain: "abc.edu" } } -> cdf1@abc.edu + e.g. tokens { electronic { protocol: "www.abc.edu" } } -> www.abc.edu + """ + + def __init__(self): + super().__init__(name="electronic", kind="verbalize") + user_name = ( + pynutil.delete("username:") + + delete_space + + pynutil.delete("\"") + + pynini.closure(NEMO_NOT_QUOTE, 1) + + pynutil.delete("\"") + ) + domain = ( + pynutil.delete("domain:") + + delete_space + + pynutil.delete("\"") + + pynini.closure(NEMO_NOT_QUOTE, 1) + + pynutil.delete("\"") + ) + + protocol = ( + pynutil.delete("protocol:") + + delete_space + + pynutil.delete("\"") + + pynini.closure(NEMO_NOT_QUOTE, 1) + + pynutil.delete("\"") + ) + + graph = user_name + delete_space + pynutil.insert("@") + domain + graph |= protocol + delete_tokens = self.delete_tokens(graph) + self.fst = delete_tokens.optimize() diff --git a/nemo_text_processing/inverse_text_normalization/pt/verbalizers/measure.py b/nemo_text_processing/inverse_text_normalization/pt/verbalizers/measure.py new file mode 100644 index 000000000000..057ade696d11 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/pt/verbalizers/measure.py @@ -0,0 +1,61 @@ +# Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from nemo_text_processing.text_normalization.en.graph_utils import NEMO_CHAR, GraphFst, delete_space +from pynini.lib import pynutil + + +class MeasureFst(GraphFst): + """ + Finite state transducer for verbalizing measure, e.g. + measure { cardinal { negative: "true" integer: "12" } units: "kg" } -> -12 kg + + Args: + decimal: DecimalFst + cardinal: CardinalFst + """ + + def __init__(self, decimal: GraphFst, cardinal: GraphFst): + super().__init__(name="measure", kind="verbalize") + optional_sign = pynini.closure(pynini.cross("negative: \"true\"", "-"), 0, 1) + unit = ( + pynutil.delete("units:") + + delete_space + + pynutil.delete("\"") + + pynini.closure(NEMO_CHAR - " ", 1) + + pynutil.delete("\"") + + delete_space + ) + graph_decimal = ( + pynutil.delete("decimal {") + + delete_space + + optional_sign + + delete_space + + decimal.numbers + + delete_space + + pynutil.delete("}") + ) + graph_cardinal = ( + pynutil.delete("cardinal {") + + delete_space + + optional_sign + + delete_space + + cardinal.numbers + + delete_space + + pynutil.delete("}") + ) + graph = (graph_cardinal | graph_decimal) + delete_space + pynutil.insert(" ") + unit + delete_tokens = self.delete_tokens(graph) + self.fst = delete_tokens.optimize() diff --git a/nemo_text_processing/inverse_text_normalization/pt/verbalizers/money.py b/nemo_text_processing/inverse_text_normalization/pt/verbalizers/money.py new file mode 100644 index 000000000000..54a9b1038337 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/pt/verbalizers/money.py @@ -0,0 +1,40 @@ +# Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from nemo_text_processing.text_normalization.en.graph_utils import NEMO_CHAR, GraphFst, delete_space, insert_space +from pynini.lib import pynutil + + +class MoneyFst(GraphFst): + """ + Finite state transducer for verbalizing money, e.g. + money { integer_part: "12" morphosyntactic_features: "," fractional_part: "05" currency: "$" } -> $12,05 + + Args: + decimal: DecimalFst + """ + + def __init__(self, decimal: GraphFst): + super().__init__(name="money", kind="verbalize") + unit = ( + pynutil.delete("currency:") + + delete_space + + pynutil.delete("\"") + + pynini.closure(NEMO_CHAR - " ", 1) + + pynutil.delete("\"") + ) + graph = unit + delete_space + insert_space + decimal.numbers + delete_tokens = self.delete_tokens(graph) + self.fst = delete_tokens.optimize() diff --git a/nemo_text_processing/inverse_text_normalization/pt/verbalizers/ordinal.py b/nemo_text_processing/inverse_text_normalization/pt/verbalizers/ordinal.py new file mode 100644 index 000000000000..fe3454e15e71 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/pt/verbalizers/ordinal.py @@ -0,0 +1,44 @@ +# Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from nemo_text_processing.text_normalization.en.graph_utils import NEMO_NOT_QUOTE, GraphFst, delete_space +from pynini.lib import pynutil + + +class OrdinalFst(GraphFst): + """ + Finite state transducer for verbalizing ordinal, e.g. + ordinal { integer: "13" morphosyntactic_features: "o" } -> 13º + """ + + def __init__(self): + super().__init__(name="ordinal", kind="verbalize") + graph = ( + pynutil.delete("integer:") + + delete_space + + pynutil.delete("\"") + + pynini.closure(NEMO_NOT_QUOTE, 1) + + pynutil.delete("\"") + ) + + replace_suffix = pynini.union( + pynini.cross(" morphosyntactic_features: \"o\"", "º"), + pynini.cross(" morphosyntactic_features: \"a\"", "ª"), + ) + + graph = graph + replace_suffix + + delete_tokens = self.delete_tokens(graph) + self.fst = delete_tokens.optimize() diff --git a/nemo_text_processing/inverse_text_normalization/pt/verbalizers/telephone.py b/nemo_text_processing/inverse_text_normalization/pt/verbalizers/telephone.py new file mode 100644 index 000000000000..4dd0d7079889 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/pt/verbalizers/telephone.py @@ -0,0 +1,32 @@ +# Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from nemo_text_processing.text_normalization.en.graph_utils import NEMO_NOT_QUOTE, GraphFst +from pynini.lib import pynutil + + +class TelephoneFst(GraphFst): + """ + Finite state transducer for verbalizing telephone, e.g. + telephone { number_part: "123-123-5678" } + -> 123-123-5678 + """ + + def __init__(self): + super().__init__(name="telephone", kind="verbalize") + + number_part = pynutil.delete("number_part: \"") + pynini.closure(NEMO_NOT_QUOTE, 1) + pynutil.delete("\"") + delete_tokens = self.delete_tokens(number_part) + self.fst = delete_tokens.optimize() diff --git a/nemo_text_processing/inverse_text_normalization/pt/verbalizers/time.py b/nemo_text_processing/inverse_text_normalization/pt/verbalizers/time.py new file mode 100755 index 000000000000..b1a04c673752 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/pt/verbalizers/time.py @@ -0,0 +1,82 @@ +# Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from nemo_text_processing.text_normalization.en.graph_utils import ( + NEMO_DIGIT, + NEMO_NOT_QUOTE, + GraphFst, + delete_space, + insert_space, +) +from pynini.lib import pynutil + + +class TimeFst(GraphFst): + """ + Finite state transducer for verbalizing time, + e.g. time { hours: "à 1" minutes: "10" } -> à 1:10 + e.g. time { hours: "às 2" minutes: "45" } -> às 2:45 + """ + + def __init__(self): + super().__init__(name="time", kind="verbalize") + add_leading_zero_to_double_digit = (NEMO_DIGIT + NEMO_DIGIT) | (pynutil.insert("0") + NEMO_DIGIT) + + prefix = ( + pynutil.delete("morphosyntactic_features:") + + delete_space + + pynutil.delete("\"") + + pynini.closure(NEMO_NOT_QUOTE, 1) + + pynutil.delete("\"") + + delete_space + + insert_space + ) + optional_prefix = pynini.closure(prefix, 0, 1) + + hour = ( + pynutil.delete("hours:") + + delete_space + + pynutil.delete("\"") + + pynini.closure(NEMO_DIGIT, 1) + + pynutil.delete("\"") + ) + minute = ( + pynutil.delete("minutes:") + + delete_space + + pynutil.delete("\"") + + pynini.closure(NEMO_DIGIT, 1) + + pynutil.delete("\"") + ) + suffix = ( + delete_space + + insert_space + + pynutil.delete("suffix:") + + delete_space + + pynutil.delete("\"") + + pynini.closure(NEMO_NOT_QUOTE, 1) + + pynutil.delete("\"") + ) + optional_suffix = pynini.closure(suffix, 0, 1) + + graph = ( + optional_prefix + + hour + + delete_space + + pynutil.insert(":") + + (minute @ add_leading_zero_to_double_digit) + + optional_suffix + ) + delete_tokens = self.delete_tokens(graph) + self.fst = delete_tokens.optimize() diff --git a/nemo_text_processing/inverse_text_normalization/pt/verbalizers/verbalize.py b/nemo_text_processing/inverse_text_normalization/pt/verbalizers/verbalize.py new file mode 100644 index 000000000000..88c04991b5f4 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/pt/verbalizers/verbalize.py @@ -0,0 +1,62 @@ +# Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from nemo_text_processing.inverse_text_normalization.pt.verbalizers.cardinal import CardinalFst +from nemo_text_processing.inverse_text_normalization.pt.verbalizers.date import DateFst +from nemo_text_processing.inverse_text_normalization.pt.verbalizers.decimal import DecimalFst +from nemo_text_processing.inverse_text_normalization.pt.verbalizers.electronic import ElectronicFst +from nemo_text_processing.inverse_text_normalization.pt.verbalizers.measure import MeasureFst +from nemo_text_processing.inverse_text_normalization.pt.verbalizers.money import MoneyFst +from nemo_text_processing.inverse_text_normalization.pt.verbalizers.ordinal import OrdinalFst +from nemo_text_processing.inverse_text_normalization.pt.verbalizers.telephone import TelephoneFst +from nemo_text_processing.inverse_text_normalization.pt.verbalizers.time import TimeFst +from nemo_text_processing.inverse_text_normalization.pt.verbalizers.whitelist import WhiteListFst +from nemo_text_processing.text_normalization.en.graph_utils import GraphFst + + +class VerbalizeFst(GraphFst): + """ + Composes other verbalizer grammars. + For deployment, this grammar will be compiled and exported to OpenFst Finate State Archiv (FAR) File. + More details to deployment at NeMo/tools/text_processing_deployment. + """ + + def __init__(self): + super().__init__(name="verbalize", kind="verbalize") + cardinal = CardinalFst() + cardinal_graph = cardinal.fst + ordinal_graph = OrdinalFst().fst + decimal = DecimalFst() + decimal_graph = decimal.fst + measure_graph = MeasureFst(decimal=decimal, cardinal=cardinal).fst + money_graph = MoneyFst(decimal=decimal).fst + time_graph = TimeFst().fst + date_graph = DateFst().fst + whitelist_graph = WhiteListFst().fst + telephone_graph = TelephoneFst().fst + electronic_graph = ElectronicFst().fst + + graph = ( + time_graph + | date_graph + | money_graph + | measure_graph + | ordinal_graph + | decimal_graph + | cardinal_graph + | whitelist_graph + | telephone_graph + | electronic_graph + ) + self.fst = graph diff --git a/nemo_text_processing/inverse_text_normalization/pt/verbalizers/verbalize_final.py b/nemo_text_processing/inverse_text_normalization/pt/verbalizers/verbalize_final.py new file mode 100644 index 000000000000..cc2e65aed46d --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/pt/verbalizers/verbalize_final.py @@ -0,0 +1,43 @@ +# Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from nemo_text_processing.inverse_text_normalization.pt.verbalizers.verbalize import VerbalizeFst +from nemo_text_processing.inverse_text_normalization.pt.verbalizers.word import WordFst +from nemo_text_processing.text_normalization.en.graph_utils import GraphFst, delete_extra_space, delete_space +from pynini.lib import pynutil + + +class VerbalizeFinalFst(GraphFst): + """ + Finite state transducer that verbalizes an entire sentence, e.g. + tokens { name: "its" } tokens { time { hours: "12" minutes: "30" } } tokens { name: "now" } -> its 12:30 now + """ + + def __init__(self): + super().__init__(name="verbalize_final", kind="verbalize") + verbalize = VerbalizeFst().fst + word = WordFst().fst + types = verbalize | word + graph = ( + pynutil.delete("tokens") + + delete_space + + pynutil.delete("{") + + delete_space + + types + + delete_space + + pynutil.delete("}") + ) + graph = delete_space + pynini.closure(graph + delete_extra_space) + graph + delete_space + self.fst = graph diff --git a/nemo_text_processing/inverse_text_normalization/pt/verbalizers/whitelist.py b/nemo_text_processing/inverse_text_normalization/pt/verbalizers/whitelist.py new file mode 100644 index 000000000000..f54aaea65b0a --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/pt/verbalizers/whitelist.py @@ -0,0 +1,37 @@ +# Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import pynini +from nemo_text_processing.text_normalization.en.graph_utils import NEMO_CHAR, NEMO_SIGMA, GraphFst, delete_space +from pynini.lib import pynutil + + +class WhiteListFst(GraphFst): + """ + Finite state transducer for verbalizing whitelist + e.g. tokens { name: "sexta feira" } -> "sexta-feira" + """ + + def __init__(self): + super().__init__(name="whitelist", kind="verbalize") + graph = ( + pynutil.delete("name:") + + delete_space + + pynutil.delete("\"") + + pynini.closure(NEMO_CHAR - " ", 1) + + pynutil.delete("\"") + ) + graph = graph @ pynini.cdrewrite(pynini.cross(u"\u00A0", " "), "", "", NEMO_SIGMA) + self.fst = graph.optimize() diff --git a/nemo_text_processing/inverse_text_normalization/pt/verbalizers/word.py b/nemo_text_processing/inverse_text_normalization/pt/verbalizers/word.py new file mode 100644 index 000000000000..4417d8f0020c --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/pt/verbalizers/word.py @@ -0,0 +1,32 @@ +# Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from nemo_text_processing.text_normalization.en.graph_utils import NEMO_CHAR, NEMO_SIGMA, GraphFst, delete_space +from pynini.lib import pynutil + + +class WordFst(GraphFst): + """ + Finite state transducer for verbalizing plain tokens + e.g. tokens { name: "sleep" } -> sleep + """ + + def __init__(self): + super().__init__(name="word", kind="verbalize") + chars = pynini.closure(NEMO_CHAR - " ", 1) + char = pynutil.delete("name:") + delete_space + pynutil.delete("\"") + chars + pynutil.delete("\"") + graph = char @ pynini.cdrewrite(pynini.cross(u"\u00A0", " "), "", "", NEMO_SIGMA) + + self.fst = graph.optimize() diff --git a/nemo_text_processing/inverse_text_normalization/run_evaluate.py b/nemo_text_processing/inverse_text_normalization/run_evaluate.py index c82a4541ad98..d2b3b63305ad 100644 --- a/nemo_text_processing/inverse_text_normalization/run_evaluate.py +++ b/nemo_text_processing/inverse_text_normalization/run_evaluate.py @@ -34,7 +34,7 @@ def parse_args(): parser = ArgumentParser() parser.add_argument("--input", help="input file path", type=str) parser.add_argument( - "--lang", help="language", choices=['en', 'de', 'es', 'ru', 'fr', 'vi'], default="en", type=str + "--lang", help="language", choices=['en', 'de', 'es', 'pt', 'ru', 'fr', 'vi'], default="en", type=str ) parser.add_argument( "--cat", diff --git a/tests/nemo_text_processing/pt/__init__.py b/tests/nemo_text_processing/pt/__init__.py new file mode 100644 index 000000000000..2db92b257416 --- /dev/null +++ b/tests/nemo_text_processing/pt/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/tests/nemo_text_processing/pt/data_inverse_text_normalization/test_cases_cardinal.txt b/tests/nemo_text_processing/pt/data_inverse_text_normalization/test_cases_cardinal.txt new file mode 100755 index 000000000000..72a9c7ecc156 --- /dev/null +++ b/tests/nemo_text_processing/pt/data_inverse_text_normalization/test_cases_cardinal.txt @@ -0,0 +1,69 @@ +(non-ideal case) entre dezesseis mil e dezoito e mil~(non-ideal case) entre 16018 e 1000 +duzentos e cinquenta e um~251 +novecentos e noventa e nove milhões novecentos e noventa e nove mil novecentos e noventa e nove~999999999 +zero~zero +um~um +uma~uma +dois~dois +nove~nove +dez~10 +onze~11 +, um~, um +, uma~, uma +, dez~, 10 +, onze~, 11 +vinte três~23 +vinte e três~23 +trinta e três~33 +menos vinte e três~-23 +cem~100 +cento e um~101 +cento e uma~101 +cento e dez~110 +cento e onze~111 +cento e vinte três~123 +cento e vinte e três~123 +cento e cinquenta~150 +duzentos~200 +duzentos e um~201 +mil e um~1001 +mil e uma~1001 +nove trilhões setecentos e oitenta e nove bilhões trezentos e oitenta e dois milhões quinhentos e trinta e seis mil cento e trinta~9789382536130 +duzentos e cinquenta e quatro~254 +cento e quarenta e sete mil quatrocentos e cinquenta e um~147451 +cento e quarenta e sete mil quatrocentos e cinquenta e uma~147451 +um milhão cento e cinquenta e seis mil cento e setenta e três~1156173 +um bilhão quinhentos e noventa e três milhões e setenta e dois mil novecentos e sessenta e um~1593072961 +noventa e sete quatrilhões oitocentos e oito trilhões duzentos e sessenta e quatro bilhões setecentos e setenta e dois milhões setecentos e noventa e dois mil e cinco~97808264772792005 +dezessete sextilhões oitocentos e cinquenta e cinco quintilhões e trinta e seis quatrilhões seiscentos e cinquenta e sete trilhões e sete bilhões quinhentos e noventa e seis milhões cento e dez mil novecentos e quarenta e nove~17855036657007596110949 +dez quatrilhões e dez trilhões e dez milhões e cem mil e dez~10010000010100010 +menos vinte e cinco mil e trinta e sete~-25037 +um quatrilhão duzentos e sessenta e quatro trilhões trezentos e um bilhões novecentos e trinta e oito milhões cento e quatro~1264301938000104 +menos sessenta~-60 +quarenta e seis mil seiscentos e sessenta e quatro~46664 +sessenta~60 +dois milhões e três~2000003 +mil e treze~1013 +mil e cem~1100 +mil e vinte e seis~1026 +mil cento e vinte e seis~1126 +dezoito milhões quatrocentos e cinquenta mil novecentos e noventa~18450990 +dezoito milhões novecentos e quarenta mil setecentos e vinte e dois~18940722 +dezoito milhões seiscentos e noventa mil novecentos e dezesseis~18690916 +dezoito mil oitocentos e oitenta~18880 +um bilhão e um~1000000001 +um bilhão e uma~1000000001 +um bilhão cento e um~1000000101 +um bilhão cento e uma~1000000101 +um bilhão e mil cento e um~1000001101 +um bilhão e mil cento e uma~1000001101 +um bilhão e dez mil cento e um~1000010101 +um bilhão e dez mil cento e uma~1000010101 +um bilhão e um milhão e dez mil cento e um~1001010101 +um bilhão e um milhão e dez mil cento e uma~1001010101 +dois bilhões e cinquenta e dois~2000000052 +muitos milhões~muitos milhões +um quatrilhão e um~1000000000000001 +um quatrilhão e uma~1000000000000001 +um sextilhão e um~1000000000000000000001 +um sextilhão e uma~1000000000000000000001 \ No newline at end of file diff --git a/tests/nemo_text_processing/pt/data_inverse_text_normalization/test_cases_date.txt b/tests/nemo_text_processing/pt/data_inverse_text_normalization/test_cases_date.txt new file mode 100644 index 000000000000..94c74b2475da --- /dev/null +++ b/tests/nemo_text_processing/pt/data_inverse_text_normalization/test_cases_date.txt @@ -0,0 +1,6 @@ +primeiro de janeiro~1 de janeiro +um de janeiro~1 de janeiro +em primeiro de dezembro~em 1 de dezembro +domingo vinte e seis de outubro~domingo 26 de outubro +trinta e um de dezembro de mil novecentos e oitenta e oito~31 de dezembro de 1988 +vinte e sete de agosto de dois mil e quinze~27 de agosto de 2015 \ No newline at end of file diff --git a/tests/nemo_text_processing/pt/data_inverse_text_normalization/test_cases_decimal.txt b/tests/nemo_text_processing/pt/data_inverse_text_normalization/test_cases_decimal.txt new file mode 100755 index 000000000000..bdafdc642442 --- /dev/null +++ b/tests/nemo_text_processing/pt/data_inverse_text_normalization/test_cases_decimal.txt @@ -0,0 +1,26 @@ +um vírgula dois seis~1,26 +menos um vírgula dois seis~-1,26 +um vírgula vinte e seis~1,26 +zero vírgula vinte e seis~0,26 +zero vírgula vinte seis~0,26 +três vírgula cento e quarenta e um~3,141 +três vírgula zero cento e quarenta e um~3,0141 +três vírgula cento e quarenta e um cinquenta e nove~3,14159 +três vírgula quatorze cento e cinquenta e nove~3,14159 +três vírgula quatorze quinze noventa e dois sessenta e cinco trinta e cinco~3,1415926535 +três vírgula quatorze quinze zero noventa e dois sessenta e cinco trinta e cinco~3,14150926535 +três vírgula quatorze quinze zero novecentos e vinte e seis zero quinhentos e trinta e cinco~3,141509260535 +quatrocentos milhões~400 milhões +um ponto trinta e três~1.33 +um ponto trinta e três milhões~1.33 milhões +zero vírgula seis milhões~0,6 milhões +mil oitocentos e vinte e quatro milhões~1824 milhões +ponto dois seis~.26 +um milhão~1 milhão +dois milhões~2 milhões +um bilhão~1 bilhão +dois bilhões~2 bilhões +um trilhão~1 trilhão +dois trilhões~2 trilhões +um quatrilhão~1 quatrilhão +dois quatrilhões~2 quatrilhões diff --git a/tests/nemo_text_processing/pt/data_inverse_text_normalization/test_cases_electronic.txt b/tests/nemo_text_processing/pt/data_inverse_text_normalization/test_cases_electronic.txt new file mode 100644 index 000000000000..70a5319756c0 --- /dev/null +++ b/tests/nemo_text_processing/pt/data_inverse_text_normalization/test_cases_electronic.txt @@ -0,0 +1,13 @@ +a ponto b c arroba g mail ponto com~a.bc@gmail.com +c d f arroba a b c ponto e d u~cdf@abc.edu +a b c arroba g mail ponto a b c~abc@gmail.abc +a b c arroba a b c ponto com~abc@abc.com +a s d f um dois três arroba a b c ponto com~asdf123@abc.com +a um b dois arroba a b c ponto com~a1b2@abc.com +a b três ponto s d d ponto três arroba g mail ponto com~ab3.sdd.3@gmail.com +agá tê tê pê ésse dois pontos barra barra dáblio dáblio dáblio ponto n vidia ponto com~https://www.nvidia.com +dáblio dáblio dáblio ponto n vidia ponto com~www.nvidia.com +dáblio dáblio dáblio ponto nvidia ponto com~www.nvidia.com +w w w ponto nvidia ponto com~www.nvidia.com +dáblio dáblio dáblio ponto a b c ponto es barra e f g~www.abc.es/efg +dáblio dáblio dáblio ponto a b c ponto br~www.abc.br \ No newline at end of file diff --git a/tests/nemo_text_processing/pt/data_inverse_text_normalization/test_cases_measure.txt b/tests/nemo_text_processing/pt/data_inverse_text_normalization/test_cases_measure.txt new file mode 100755 index 000000000000..7e3c8ca8b90c --- /dev/null +++ b/tests/nemo_text_processing/pt/data_inverse_text_normalization/test_cases_measure.txt @@ -0,0 +1,12 @@ +duzentos metros~200 m +duzentos e quarenta e cinco milhas por hora~245 mph +duzentos e quarenta e cinco quilômetros por hora~245 kph +duzentos e quarenta e cinco metros por segundo~245 m/s +dois quilos~2 kg +sessenta vírgula dois quatro zero zero quilogramas~60,2400 kg +menos sessenta vírgula dois quatro zero zero quilogramas~-60,2400 kg +oito vírgula cinco dois por cento~8,52 % +menos oito vírgula cinco dois por cento~-8,52 % +um por cento~1 % +três centímetros~3 cm +cinco litros~5 l \ No newline at end of file diff --git a/tests/nemo_text_processing/pt/data_inverse_text_normalization/test_cases_money.txt b/tests/nemo_text_processing/pt/data_inverse_text_normalization/test_cases_money.txt new file mode 100755 index 000000000000..88d7e0e7db07 --- /dev/null +++ b/tests/nemo_text_processing/pt/data_inverse_text_normalization/test_cases_money.txt @@ -0,0 +1,25 @@ +doze dólares e cinco centavos~$ 12,05 +doze dólares~$ 12 +setenta e cinco dólares e sessenta e três~$ 75,63 +setenta e cinco dólares e sessenta e três centavos~$ 75,63 +setenta e cinco dólares com sessenta e três centavos~$ 75,63 +setenta e cinco dólares com sessenta e três~$ 75,63 +vinte e nove dólares e cinquenta centavos~$ 29,50 +um dólar~$ 1 +um real~R$ 1 +cem reais~R$ 100 +duzentos reais~R$ 200 +cento e noventa e nove reais e noventa e nove centavos~R$ 199,99 +um real e um centavo~R$ 1,01 +vinte centavos~R$ 0,20 +vinte e cinco centavos~R$ 0,25 +doze euros e cinco centavos~€ 12,05 +doze dólares americanos e cinco centavos~US$ 12,05 +duas libras esterlinas~£ 2 +doze dólares e cinco centavos~$ 12,05 +pagamos cento e quinze reais por uma bala~pagamos R$ 115 por uma bala +quinze mil reais~R$ 15000 +dois bilhões de reais~R$ 2 bilhões +dois milhões de reais~R$ 2 milhões +três vírgula sete milhões de reais~R$ 3,7 milhões +quatro ponto oito milhões de dólares~$ 4.8 milhões diff --git a/tests/nemo_text_processing/pt/data_inverse_text_normalization/test_cases_ordinal.txt b/tests/nemo_text_processing/pt/data_inverse_text_normalization/test_cases_ordinal.txt new file mode 100755 index 000000000000..0ae43e978c8b --- /dev/null +++ b/tests/nemo_text_processing/pt/data_inverse_text_normalization/test_cases_ordinal.txt @@ -0,0 +1,19 @@ +primeiro~primeiro +terceira~terceira +nono~nono +nona~nona +décimo~10º +décima~10ª +décimo primeiro~11º +décima primeira~11ª +(technically ungrammatical) décima primeira~(technically ungrammatical) 11ª +(technically ungrammatical) décima primeira casa~(technically ungrammatical) 11ª casa +décimo terceiro~13º +vigésimo primeiro~21º +vigésima primeira~21ª +(technically ungrammatical) vigésimo primeira~(technically ungrammatical) 21ª +vigésimo segundo~22º +vigésima segunda~22ª +vigésimo terceiro~23º +centésimo décimo primeiro~111º +centésimo trigésimo quarto~134º diff --git a/tests/nemo_text_processing/pt/data_inverse_text_normalization/test_cases_telephone.txt b/tests/nemo_text_processing/pt/data_inverse_text_normalization/test_cases_telephone.txt new file mode 100755 index 000000000000..04e2875b5290 --- /dev/null +++ b/tests/nemo_text_processing/pt/data_inverse_text_normalization/test_cases_telephone.txt @@ -0,0 +1,27 @@ +um dois três quatro cinco seis sete oito~1234-5678 +um dois três quatro cinco seis sete oito telefone~1234-5678 telefone +um dois três quatro cinco meia sete oito~1234-5678 +um dois três quatro cinco meia sete oito telefone~1234-5678 telefone +um dois três quatro cinco seis sete oito nove~12345-6789 +um dois três quatro cinco seis sete oito nove telefone~12345-6789 telefone +quatro cinco um dois três quatro cinco seis sete oito~(45) 1234-5678 +quatro cinco um dois três quatro cinco seis sete oito telefone~(45) 1234-5678 telefone +quatro cinco um dois três quatro cinco seis sete oito nove~(45) 12345-6789 +quatro cinco um dois três quatro cinco seis sete oito nove telefone~(45) 12345-6789 telefone +vinte e sete vinte e oito trinta e sete trinta e oito~2728-3738 +vinte e sete vinte e oito trinta e sete trinta e oito telefone~2728-3738 telefone +um vinte e sete vinte e oito trinta e sete trinta e oito~12728-3738 +um vinte e sete vinte e oito trinta e sete trinta e oito telefone~12728-3738 telefone +nove oito sete seis cinquenta e quatro zero zero~9876-5400 +nove oito sete seis cinquenta e quatro zero um~9876-5401 +noventa e oito setenta e seis zero zero trinta e cinco~9876-0035 +noventa e oito setenta e seis zero zero trinta~9876-0030 +nove noventa e oito setenta e seis zero zero trinta e um~99876-0031 +nove noventa e oito setenta e seis zero zero trinta~99876-0030 +dois três nove noventa e oito setenta e seis zero zero trinta e um~(23) 99876-0031 +dois três nove noventa e oito setenta e seis zero zero trinta~(23) 99876-0030 +vinte e três nove noventa e oito setenta e seis zero zero trinta e um~(23) 99876-0031 +vinte e três nove noventa e oito setenta e seis zero zero trinta~(23) 99876-0030 +vinte e três nove noventa e oito setenta e seis zero meia trinta e um~(23) 99876-0631 +vinte e três nove noventa e oito setenta e seis zero meia trinta~(23) 99876-0630 +vinte três nove noventa e oito setenta e seis zero meia trinta~(23) 99876-0630 \ No newline at end of file diff --git a/tests/nemo_text_processing/pt/data_inverse_text_normalization/test_cases_time.txt b/tests/nemo_text_processing/pt/data_inverse_text_normalization/test_cases_time.txt new file mode 100755 index 000000000000..7aef50d2d489 --- /dev/null +++ b/tests/nemo_text_processing/pt/data_inverse_text_normalization/test_cases_time.txt @@ -0,0 +1,19 @@ +às quinze pro meio dia~às 11:45 +às quinze pra meia noite~às 23:45 +às quinze para a meia noite~às 23:45 +às quinze pras duas da tarde~às 1:45 da tarde +às dez pras duas da madrugada~às 1:50 da madrugada +às quinze pras duas da tarde~às 1:45 da tarde +chegaram às quinze pras duas da tarde~chegaram às 1:45 da tarde +ao meio dia~ao meio dia +ao meio dia e meia hora~ao 12:30 +ao meio dia e meia~ao 12:30 +ao meio dia e meio~ao 12:30 +meia noite~meia noite +à meia noite~à meia noite +à meia noite e quinze~à 0:15 +meia noite e meia~0:30 +à uma e trinta~à 1:30 +às onze e trinta~às 11:30 +às três horas e trinta minutos~às 3:30 +às quinze horas e quarenta minutos~às 15:40 diff --git a/tests/nemo_text_processing/pt/data_inverse_text_normalization/test_cases_whitelist.txt b/tests/nemo_text_processing/pt/data_inverse_text_normalization/test_cases_whitelist.txt new file mode 100755 index 000000000000..798df8b9bdfd --- /dev/null +++ b/tests/nemo_text_processing/pt/data_inverse_text_normalization/test_cases_whitelist.txt @@ -0,0 +1,6 @@ +primeira segunda feira~primeira segunda-feira +primeira segunda-feira~primeira segunda-feira +terça feira~terça-feira +quarta feira~quarta-feira +quinta feira~quinta-feira +sexta feira~sexta-feira \ No newline at end of file diff --git a/tests/nemo_text_processing/pt/data_inverse_text_normalization/test_cases_word.txt b/tests/nemo_text_processing/pt/data_inverse_text_normalization/test_cases_word.txt new file mode 100755 index 000000000000..fdd32f2ea4f2 --- /dev/null +++ b/tests/nemo_text_processing/pt/data_inverse_text_normalization/test_cases_word.txt @@ -0,0 +1,49 @@ +~ +yahoo!~yahoo! +vinte!~20 ! +x ~x +—~— +aaa~aaa +aabach~aabach +aabenraa~aabenraa +aabye~aabye +aaccessed~aaccessed +aach~aach +aachen's~aachen's +aadri~aadri +aafia~aafia +aagaard~aagaard +aagadu~aagadu +aagard~aagard +aagathadi~aagathadi +aaghart's~aaghart's +aagnes~aagnes +aagomoni~aagomoni +aagon~aagon +aagoo~aagoo +aagot~aagot +aahar~aahar +aahh~aahh +aahperd~aahperd +aaibinterstate~aaibinterstate +aajab~aajab +aakasa~aakasa +aakervik~aakervik +aakirkeby~aakirkeby +aalam~aalam +aalbaek~aalbaek +aaldiu~aaldiu +aalem~aalem +a'ali~a'ali +aalilaassamthey~aalilaassamthey +aalin~aalin +aaliyan~aaliyan +aaliyan's~aaliyan's +aamadu~aamadu +aamara~aamara +aambala~aambala +aamera~aamera +aamer's~aamer's +aamina~aamina +aaminah~aaminah +aamjiwnaang~aamjiwnaang diff --git a/tests/nemo_text_processing/pt/test_cardinal.py b/tests/nemo_text_processing/pt/test_cardinal.py new file mode 100644 index 000000000000..bfe7d82d0db2 --- /dev/null +++ b/tests/nemo_text_processing/pt/test_cardinal.py @@ -0,0 +1,31 @@ +# Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest +from nemo_text_processing.inverse_text_normalization.inverse_normalize import InverseNormalizer +from parameterized import parameterized + +from ..utils import CACHE_DIR, parse_test_case_file + + +class TestCardinal: + + inverse_normalizer = InverseNormalizer(lang='pt', cache_dir=CACHE_DIR, overwrite_cache=False) + + @parameterized.expand(parse_test_case_file('pt/data_inverse_text_normalization/test_cases_cardinal.txt')) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_denorm(self, test_input, expected): + pred = self.inverse_normalizer.inverse_normalize(test_input, verbose=False) + assert pred == expected diff --git a/tests/nemo_text_processing/pt/test_date.py b/tests/nemo_text_processing/pt/test_date.py new file mode 100644 index 000000000000..88b5a50ebe5c --- /dev/null +++ b/tests/nemo_text_processing/pt/test_date.py @@ -0,0 +1,30 @@ +# Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest +from nemo_text_processing.inverse_text_normalization.inverse_normalize import InverseNormalizer +from parameterized import parameterized + +from ..utils import CACHE_DIR, parse_test_case_file + + +class TestDate: + inverse_normalizer = InverseNormalizer(lang='pt', cache_dir=CACHE_DIR, overwrite_cache=False) + + @parameterized.expand(parse_test_case_file('pt/data_inverse_text_normalization/test_cases_date.txt')) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_denorm(self, test_input, expected): + pred = self.inverse_normalizer.inverse_normalize(test_input, verbose=False) + assert pred == expected diff --git a/tests/nemo_text_processing/pt/test_decimal.py b/tests/nemo_text_processing/pt/test_decimal.py new file mode 100644 index 000000000000..4fd77295e4a3 --- /dev/null +++ b/tests/nemo_text_processing/pt/test_decimal.py @@ -0,0 +1,30 @@ +# Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest +from nemo_text_processing.inverse_text_normalization.inverse_normalize import InverseNormalizer +from parameterized import parameterized + +from ..utils import CACHE_DIR, parse_test_case_file + + +class TestDecimal: + inverse_normalizer = InverseNormalizer(lang='pt', cache_dir=CACHE_DIR, overwrite_cache=False) + + @parameterized.expand(parse_test_case_file('pt/data_inverse_text_normalization/test_cases_decimal.txt')) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_denorm(self, test_input, expected): + pred = self.inverse_normalizer.inverse_normalize(test_input, verbose=False) + assert pred == expected diff --git a/tests/nemo_text_processing/pt/test_electronic.py b/tests/nemo_text_processing/pt/test_electronic.py new file mode 100644 index 000000000000..9e340471f299 --- /dev/null +++ b/tests/nemo_text_processing/pt/test_electronic.py @@ -0,0 +1,30 @@ +# Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest +from nemo_text_processing.inverse_text_normalization.inverse_normalize import InverseNormalizer +from parameterized import parameterized + +from ..utils import CACHE_DIR, parse_test_case_file + + +class TestElectronic: + inverse_normalizer = InverseNormalizer(lang='pt', cache_dir=CACHE_DIR, overwrite_cache=False) + + @parameterized.expand(parse_test_case_file('pt/data_inverse_text_normalization/test_cases_electronic.txt')) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_denorm(self, test_input, expected): + pred = self.inverse_normalizer.inverse_normalize(test_input, verbose=False) + assert pred == expected diff --git a/tests/nemo_text_processing/pt/test_measure.py b/tests/nemo_text_processing/pt/test_measure.py new file mode 100644 index 000000000000..892b45962699 --- /dev/null +++ b/tests/nemo_text_processing/pt/test_measure.py @@ -0,0 +1,31 @@ +# Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import pytest +from nemo_text_processing.inverse_text_normalization.inverse_normalize import InverseNormalizer +from parameterized import parameterized + +from ..utils import CACHE_DIR, parse_test_case_file + + +class TestMeasure: + inverse_normalizer = InverseNormalizer(lang='pt', cache_dir=CACHE_DIR, overwrite_cache=False) + + @parameterized.expand(parse_test_case_file('pt/data_inverse_text_normalization/test_cases_measure.txt')) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_denorm(self, test_input, expected): + pred = self.inverse_normalizer.inverse_normalize(test_input, verbose=False) + assert pred == expected diff --git a/tests/nemo_text_processing/pt/test_money.py b/tests/nemo_text_processing/pt/test_money.py new file mode 100644 index 000000000000..40c682fe99cd --- /dev/null +++ b/tests/nemo_text_processing/pt/test_money.py @@ -0,0 +1,31 @@ +# Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import pytest +from nemo_text_processing.inverse_text_normalization.inverse_normalize import InverseNormalizer +from parameterized import parameterized + +from ..utils import CACHE_DIR, parse_test_case_file + + +class TestMoney: + inverse_normalizer = InverseNormalizer(lang='pt', cache_dir=CACHE_DIR, overwrite_cache=False) + + @parameterized.expand(parse_test_case_file('pt/data_inverse_text_normalization/test_cases_money.txt')) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_denorm(self, test_input, expected): + pred = self.inverse_normalizer.inverse_normalize(test_input, verbose=False) + assert pred == expected diff --git a/tests/nemo_text_processing/pt/test_ordinal.py b/tests/nemo_text_processing/pt/test_ordinal.py new file mode 100644 index 000000000000..19acfbaee131 --- /dev/null +++ b/tests/nemo_text_processing/pt/test_ordinal.py @@ -0,0 +1,31 @@ +# Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import pytest +from nemo_text_processing.inverse_text_normalization.inverse_normalize import InverseNormalizer +from parameterized import parameterized + +from ..utils import CACHE_DIR, parse_test_case_file + + +class TestOrdinal: + inverse_normalizer = InverseNormalizer(lang='pt', cache_dir=CACHE_DIR, overwrite_cache=False) + + @parameterized.expand(parse_test_case_file('pt/data_inverse_text_normalization/test_cases_ordinal.txt')) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_denorm(self, test_input, expected): + pred = self.inverse_normalizer.inverse_normalize(test_input, verbose=False) + assert pred == expected diff --git a/tests/nemo_text_processing/pt/test_sparrowhawk_inverse_text_normalization.sh b/tests/nemo_text_processing/pt/test_sparrowhawk_inverse_text_normalization.sh new file mode 100755 index 000000000000..74d8ddafdfc6 --- /dev/null +++ b/tests/nemo_text_processing/pt/test_sparrowhawk_inverse_text_normalization.sh @@ -0,0 +1,84 @@ +#! /bin/sh + +PROJECT_DIR=/workspace/tests + +runtest () { + input=$1 + cd /workspace/sparrowhawk/documentation/grammars + + # read test file + while read testcase; do + IFS='~' read spoken written <<< $testcase + denorm_pred=$(echo $spoken | normalizer_main --config=sparrowhawk_configuration.ascii_proto 2>&1 | tail -n 1) + + # trim white space + written="$(echo -e "${written}" | sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//')" + denorm_pred="$(echo -e "${denorm_pred}" | sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//')" + + # input expected actual + assertEquals "$spoken" "$written" "$denorm_pred" + done < "$input" +} + +testITNCardinal() { + input=$PROJECT_DIR/pt/data_inverse_text_normalization/test_cases_cardinal.txt + runtest $input +} + +testITNDate() { + input=$PROJECT_DIR/pt/data_inverse_text_normalization/test_cases_date.txt + runtest $input +} + +testITNDecimal() { + input=$PROJECT_DIR/pt/data_inverse_text_normalization/test_cases_decimal.txt + runtest $input +} + +testITNOrdinal() { + input=$PROJECT_DIR/pt/data_inverse_text_normalization/test_cases_ordinal.txt + runtest $input +} + +#testITNFraction() { +# input=$PROJECT_DIR/pt/data_inverse_text_normalization/test_cases_fraction.txt +# runtest $input +#} + +testITNTime() { + input=$PROJECT_DIR/pt/data_inverse_text_normalization/test_cases_time.txt + runtest $input +} + +testITNMeasure() { + input=$PROJECT_DIR/pt/data_inverse_text_normalization/test_cases_measure.txt + runtest $input +} + +testITNMoney() { + input=$PROJECT_DIR/pt/data_inverse_text_normalization/test_cases_money.txt + runtest $input +} + +testITNWhitelist() { + input=$PROJECT_DIR/pt/data_inverse_text_normalization/test_cases_whitelist.txt + runtest $input +} + +testITNTelephone() { + input=$PROJECT_DIR/pt/data_inverse_text_normalization/test_cases_telephone.txt + runtest $input +} + +testITNElectronic() { + input=$PROJECT_DIR/pt/data_inverse_text_normalization/test_cases_electronic.txt + runtest $input +} + +testITNWord() { + input=$PROJECT_DIR/pt/data_inverse_text_normalization/test_cases_word.txt + runtest $input +} + +# Load shUnit2 +. $PROJECT_DIR/../shunit2/shunit2 diff --git a/tests/nemo_text_processing/pt/test_telephone.py b/tests/nemo_text_processing/pt/test_telephone.py new file mode 100644 index 000000000000..6d36e9db2bfb --- /dev/null +++ b/tests/nemo_text_processing/pt/test_telephone.py @@ -0,0 +1,31 @@ +# Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import pytest +from nemo_text_processing.inverse_text_normalization.inverse_normalize import InverseNormalizer +from parameterized import parameterized + +from ..utils import CACHE_DIR, parse_test_case_file + + +class TestTelephone: + inverse_normalizer = InverseNormalizer(lang='pt', cache_dir=CACHE_DIR, overwrite_cache=False) + + @parameterized.expand(parse_test_case_file('pt/data_inverse_text_normalization/test_cases_telephone.txt')) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_denorm(self, test_input, expected): + pred = self.inverse_normalizer.inverse_normalize(test_input, verbose=False) + assert pred == expected diff --git a/tests/nemo_text_processing/pt/test_time.py b/tests/nemo_text_processing/pt/test_time.py new file mode 100644 index 000000000000..7a556b36bf4b --- /dev/null +++ b/tests/nemo_text_processing/pt/test_time.py @@ -0,0 +1,30 @@ +# Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest +from nemo_text_processing.inverse_text_normalization.inverse_normalize import InverseNormalizer +from parameterized import parameterized + +from ..utils import CACHE_DIR, parse_test_case_file + + +class TestTime: + inverse_normalizer = InverseNormalizer(lang='pt', cache_dir=CACHE_DIR, overwrite_cache=False) + + @parameterized.expand(parse_test_case_file('pt/data_inverse_text_normalization/test_cases_time.txt')) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_denorm(self, test_input, expected): + pred = self.inverse_normalizer.inverse_normalize(test_input, verbose=False) + assert pred == expected diff --git a/tests/nemo_text_processing/pt/test_whitelist.py b/tests/nemo_text_processing/pt/test_whitelist.py new file mode 100644 index 000000000000..0f8884b53293 --- /dev/null +++ b/tests/nemo_text_processing/pt/test_whitelist.py @@ -0,0 +1,31 @@ +# Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import pytest +from nemo_text_processing.inverse_text_normalization.inverse_normalize import InverseNormalizer +from parameterized import parameterized + +from ..utils import CACHE_DIR, parse_test_case_file + + +class TestWhitelist: + inverse_normalizer = InverseNormalizer(lang='pt', cache_dir=CACHE_DIR, overwrite_cache=False) + + @parameterized.expand(parse_test_case_file('pt/data_inverse_text_normalization/test_cases_whitelist.txt')) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_denorm(self, test_input, expected): + pred = self.inverse_normalizer.inverse_normalize(test_input, verbose=False) + assert pred == expected diff --git a/tests/nemo_text_processing/pt/test_word.py b/tests/nemo_text_processing/pt/test_word.py new file mode 100644 index 000000000000..2ad54b15ef18 --- /dev/null +++ b/tests/nemo_text_processing/pt/test_word.py @@ -0,0 +1,31 @@ +# Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import pytest +from nemo_text_processing.inverse_text_normalization.inverse_normalize import InverseNormalizer +from parameterized import parameterized + +from ..utils import CACHE_DIR, parse_test_case_file + + +class TestWord: + inverse_normalizer = InverseNormalizer(lang='pt', cache_dir=CACHE_DIR, overwrite_cache=False) + + @parameterized.expand(parse_test_case_file('pt/data_inverse_text_normalization/test_cases_word.txt')) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_denorm(self, test_input, expected): + pred = self.inverse_normalizer.inverse_normalize(test_input, verbose=False) + assert pred == expected diff --git a/tools/text_processing_deployment/export_grammars.sh b/tools/text_processing_deployment/export_grammars.sh index 31fb4cecf822..379a06977014 100644 --- a/tools/text_processing_deployment/export_grammars.sh +++ b/tools/text_processing_deployment/export_grammars.sh @@ -32,7 +32,7 @@ GRAMMARS="itn_grammars" # tn_grammars INPUT_CASE="cased" # lower_cased, only for tn_grammars -LANGUAGE="en" # language, {'en', 'es', 'de'} supports both TN and ITN, {'ru', 'fr'} supports ITN only +LANGUAGE="en" # language, {'en', 'es', 'de'} supports both TN and ITN, {'pt', 'ru', 'fr'} supports ITN only MODE="export" OVERWRITE_CACHE="True" # Set to False to re-use .far files FORCE_REBUILD="False" # Set to True to re-build docker file diff --git a/tools/text_processing_deployment/pynini_export.py b/tools/text_processing_deployment/pynini_export.py index 3ae8e926bffa..52be2a6dd8c9 100644 --- a/tools/text_processing_deployment/pynini_export.py +++ b/tools/text_processing_deployment/pynini_export.py @@ -73,7 +73,7 @@ def parse_args(): parser = ArgumentParser() parser.add_argument("--output_dir", help="output directory for grammars", required=True, type=str) parser.add_argument( - "--language", help="language", choices=["en", "de", "es", "ru", 'fr', 'vi'], type=str, default='en' + "--language", help="language", choices=["en", "de", "es", "pt", "ru", 'fr', 'vi'], type=str, default='en' ) parser.add_argument( "--grammars", help="grammars to be exported", choices=["tn_grammars", "itn_grammars"], type=str, required=True @@ -94,7 +94,7 @@ def parse_args(): if __name__ == '__main__': args = parse_args() - if args.language in ['ru', 'fr', 'vi'] and args.grammars == 'tn_grammars': + if args.language in ['pt', 'ru', 'fr', 'vi'] and args.grammars == 'tn_grammars': raise ValueError('Only ITN grammars could be deployed in Sparrowhawk for the selected languages.') if args.language == 'en': @@ -137,6 +137,13 @@ def parse_args(): ClassifyFst as TNClassifyFst, ) from nemo_text_processing.text_normalization.es.verbalizers.verbalize import VerbalizeFst as TNVerbalizeFst + elif args.language == 'pt': + from nemo_text_processing.inverse_text_normalization.pt.taggers.tokenize_and_classify import ( + ClassifyFst as ITNClassifyFst, + ) + from nemo_text_processing.inverse_text_normalization.pt.verbalizers.verbalize import ( + VerbalizeFst as ITNVerbalizeFst, + ) elif args.language == 'fr': from nemo_text_processing.inverse_text_normalization.fr.taggers.tokenize_and_classify import ( ClassifyFst as ITNClassifyFst, From b4edbca07f6117a0c992ac61a2991d6f29b32863 Mon Sep 17 00:00:00 2001 From: anteju <108555623+anteju@users.noreply.github.com> Date: Thu, 7 Jul 2022 19:47:50 -0700 Subject: [PATCH 10/52] Fixed WER initialization in ASR_with_Nemo notebook (#4523) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Ante Jukić Co-authored-by: Ante Jukić --- tutorials/asr/ASR_with_NeMo.ipynb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tutorials/asr/ASR_with_NeMo.ipynb b/tutorials/asr/ASR_with_NeMo.ipynb index 959ba9750a44..e89fdd44655b 100644 --- a/tutorials/asr/ASR_with_NeMo.ipynb +++ b/tutorials/asr/ASR_with_NeMo.ipynb @@ -1089,7 +1089,7 @@ " alogits = np.asarray(ologits)\n", " logits = torch.from_numpy(alogits[0])\n", " greedy_predictions = logits.argmax(dim=-1, keepdim=False)\n", - " wer = WER(vocabulary=quartznet.decoder.vocabulary, batch_dim_index=0, use_cer=False, ctc_decode=True)\n", + " wer = WER(decoding=quartznet.decoding, use_cer=False)\n", " hypotheses, _ = wer.decoding.ctc_decoder_predictions_tensor(greedy_predictions)\n", " print(hypotheses)\n", " break\n" From 01f0422074de5319f2c9669281e76c6eb71c77c5 Mon Sep 17 00:00:00 2001 From: jasro23 <108691071+jasro23@users.noreply.github.com> Date: Thu, 7 Jul 2022 23:29:15 -0700 Subject: [PATCH 11/52] Update cmudict (#4510) phoneme IY1 -> IH1 in NVIDIA Added phonemes for CUSTOMIZABLE Update cmudict file revision and its reference. Signed-off-by: Jason Roche Co-authored-by: Jason Roche Co-authored-by: Xuesong Yang <1646669+XuesongYang@users.noreply.github.com> --- examples/tts/conf/aligner.yaml | 2 +- examples/tts/conf/fastpitch_align_44100.yaml | 2 +- examples/tts/conf/fastpitch_align_v1.05.yaml | 2 +- examples/tts/conf/mixer-tts.yaml | 2 +- examples/tts/conf/tacotron2.yaml | 2 +- nemo/collections/tts/models/fastpitch.py | 2 +- nemo/collections/tts/torch/g2ps.py | 2 +- nemo/collections/tts/torch/tts_dataset.yaml | 2 +- .../tts/ljspeech/ds_conf/ds_for_fastpitch_align.yaml | 2 +- .../tts/ljspeech/ds_conf/ds_for_mixer_tts.yaml | 2 +- .../{cmudict-0.7b_nv22.01 => cmudict-0.7b_nv22.07} | 5 +++-- tutorials/tts/FastPitch_Finetuning.ipynb | 8 ++++---- tutorials/tts/FastPitch_MixerTTS_Training.ipynb | 8 ++++---- tutorials/tts/Tacotron2_Training.ipynb | 4 ++-- 14 files changed, 23 insertions(+), 22 deletions(-) rename scripts/tts_dataset_files/{cmudict-0.7b_nv22.01 => cmudict-0.7b_nv22.07} (99%) diff --git a/examples/tts/conf/aligner.yaml b/examples/tts/conf/aligner.yaml index 88ab5906df9c..7b2fa9660b82 100644 --- a/examples/tts/conf/aligner.yaml +++ b/examples/tts/conf/aligner.yaml @@ -19,7 +19,7 @@ lowfreq: 0 highfreq: 8000 window: hann -phoneme_dict_path: "scripts/tts_dataset_files/cmudict-0.7b_nv22.01" +phoneme_dict_path: "scripts/tts_dataset_files/cmudict-0.7b_nv22.07" heteronyms_path: "scripts/tts_dataset_files/heteronyms-030921" whitelist_path: "nemo_text_processing/text_normalization/en/data/whitelist/lj_speech.tsv" diff --git a/examples/tts/conf/fastpitch_align_44100.yaml b/examples/tts/conf/fastpitch_align_44100.yaml index 7c7e9fe3b433..314c7c32e694 100644 --- a/examples/tts/conf/fastpitch_align_44100.yaml +++ b/examples/tts/conf/fastpitch_align_44100.yaml @@ -27,7 +27,7 @@ lowfreq: 0 highfreq: null window: hann -phoneme_dict_path: "scripts/tts_dataset_files/cmudict-0.7b_nv22.01" +phoneme_dict_path: "scripts/tts_dataset_files/cmudict-0.7b_nv22.07" heteronyms_path: "scripts/tts_dataset_files/heteronyms-030921" whitelist_path: "nemo_text_processing/text_normalization/en/data/whitelist/lj_speech.tsv" diff --git a/examples/tts/conf/fastpitch_align_v1.05.yaml b/examples/tts/conf/fastpitch_align_v1.05.yaml index 47a32c7a897a..de4c60308d0e 100644 --- a/examples/tts/conf/fastpitch_align_v1.05.yaml +++ b/examples/tts/conf/fastpitch_align_v1.05.yaml @@ -27,7 +27,7 @@ lowfreq: 0 highfreq: 8000 window: hann -phoneme_dict_path: "scripts/tts_dataset_files/cmudict-0.7b_nv22.01" +phoneme_dict_path: "scripts/tts_dataset_files/cmudict-0.7b_nv22.07" heteronyms_path: "scripts/tts_dataset_files/heteronyms-030921" whitelist_path: "nemo_text_processing/text_normalization/en/data/whitelist/lj_speech.tsv" diff --git a/examples/tts/conf/mixer-tts.yaml b/examples/tts/conf/mixer-tts.yaml index aac8ba92048f..a7b3941c4fef 100644 --- a/examples/tts/conf/mixer-tts.yaml +++ b/examples/tts/conf/mixer-tts.yaml @@ -27,7 +27,7 @@ lowfreq: 0 highfreq: 8000 window: hann -phoneme_dict_path: "scripts/tts_dataset_files/cmudict-0.7b_nv22.01" +phoneme_dict_path: "scripts/tts_dataset_files/cmudict-0.7b_nv22.07" heteronyms_path: "scripts/tts_dataset_files/heteronyms-030921" whitelist_path: "nemo_text_processing/text_normalization/en/data/whitelist/lj_speech.tsv" diff --git a/examples/tts/conf/tacotron2.yaml b/examples/tts/conf/tacotron2.yaml index 31d4ea5d1c0e..664c78d0a151 100644 --- a/examples/tts/conf/tacotron2.yaml +++ b/examples/tts/conf/tacotron2.yaml @@ -9,7 +9,7 @@ validation_datasets: ??? sup_data_path: null sup_data_types: null -phoneme_dict_path: "scripts/tts_dataset_files/cmudict-0.7b_nv22.01" +phoneme_dict_path: "scripts/tts_dataset_files/cmudict-0.7b_nv22.07" heteronyms_path: "scripts/tts_dataset_files/heteronyms-030921" whitelist_path: "nemo_text_processing/text_normalization/en/data/whitelist/lj_speech.tsv" diff --git a/nemo/collections/tts/models/fastpitch.py b/nemo/collections/tts/models/fastpitch.py index e1336a414610..ae33ad412ae6 100644 --- a/nemo/collections/tts/models/fastpitch.py +++ b/nemo/collections/tts/models/fastpitch.py @@ -47,7 +47,7 @@ @dataclass class G2PConfig: _target_: str = "nemo.collections.tts.torch.g2ps.EnglishG2p" - phoneme_dict: str = "scripts/tts_dataset_files/cmudict-0.7b_nv22.01" + phoneme_dict: str = "scripts/tts_dataset_files/cmudict-0.7b_nv22.07" heteronyms: str = "scripts/tts_dataset_files/heteronyms-030921" phoneme_probability: float = 0.5 diff --git a/nemo/collections/tts/torch/g2ps.py b/nemo/collections/tts/torch/g2ps.py index a6286aa5710c..1f9f845a1dc9 100644 --- a/nemo/collections/tts/torch/g2ps.py +++ b/nemo/collections/tts/torch/g2ps.py @@ -139,7 +139,7 @@ def _parse_as_cmu_dict(phoneme_dict_path=None, encoding='latin-1'): f"English g2p_dict will be used from nltk.corpus.cmudict.dict(), because phoneme_dict_path=None. " "Note that nltk.corpus.cmudict.dict() has old version (0.6) of CMUDict. " "You can use the latest official version of CMUDict (0.7b) with additional changes from NVIDIA directly from NeMo " - "using the path scripts/tts_dataset_files/cmudict-0.7b_nv22.01." + "using the path scripts/tts_dataset_files/cmudict-0.7b_nv22.07." ) return nltk.corpus.cmudict.dict() diff --git a/nemo/collections/tts/torch/tts_dataset.yaml b/nemo/collections/tts/torch/tts_dataset.yaml index e7d122d2c6da..013b61f74bb9 100644 --- a/nemo/collections/tts/torch/tts_dataset.yaml +++ b/nemo/collections/tts/torch/tts_dataset.yaml @@ -42,5 +42,5 @@ tts_dataset: pad_with_space: True g2p: _target_: nemo.collections.tts.torch.g2ps.EnglishG2p - phoneme_dict: "scripts/tts_dataset_files/cmudict-0.7b_nv22.01" + phoneme_dict: "scripts/tts_dataset_files/cmudict-0.7b_nv22.07" heteronyms: "scripts/tts_dataset_files/heteronyms-030921" diff --git a/scripts/dataset_processing/tts/ljspeech/ds_conf/ds_for_fastpitch_align.yaml b/scripts/dataset_processing/tts/ljspeech/ds_conf/ds_for_fastpitch_align.yaml index 3be38d1259c5..86667cd499d9 100644 --- a/scripts/dataset_processing/tts/ljspeech/ds_conf/ds_for_fastpitch_align.yaml +++ b/scripts/dataset_processing/tts/ljspeech/ds_conf/ds_for_fastpitch_align.yaml @@ -4,7 +4,7 @@ manifest_filepath: "train_manifest.json" sup_data_path: "sup_data" sup_data_types: [ "align_prior_matrix", "pitch" ] whitelist_path: "nemo_text_processing/text_normalization/en/data/whitelist/lj_speech.tsv" -phoneme_dict_path: "scripts/tts_dataset_files/cmudict-0.7b_nv22.01" +phoneme_dict_path: "scripts/tts_dataset_files/cmudict-0.7b_nv22.07" heteronyms_path: "scripts/tts_dataset_files/heteronyms-030921" dataset: diff --git a/scripts/dataset_processing/tts/ljspeech/ds_conf/ds_for_mixer_tts.yaml b/scripts/dataset_processing/tts/ljspeech/ds_conf/ds_for_mixer_tts.yaml index 8ccb969c288c..d17b6252e9dc 100644 --- a/scripts/dataset_processing/tts/ljspeech/ds_conf/ds_for_mixer_tts.yaml +++ b/scripts/dataset_processing/tts/ljspeech/ds_conf/ds_for_mixer_tts.yaml @@ -4,7 +4,7 @@ manifest_filepath: "train_manifest.json" sup_data_path: "sup_data" sup_data_types: [ "align_prior_matrix", "pitch" ] whitelist_path: "nemo_text_processing/text_normalization/en/data/whitelist/lj_speech.tsv" -phoneme_dict_path: "scripts/tts_dataset_files/cmudict-0.7b_nv22.01" +phoneme_dict_path: "scripts/tts_dataset_files/cmudict-0.7b_nv22.07" heteronyms_path: "scripts/tts_dataset_files/heteronyms-030921" dataset: diff --git a/scripts/tts_dataset_files/cmudict-0.7b_nv22.01 b/scripts/tts_dataset_files/cmudict-0.7b_nv22.07 similarity index 99% rename from scripts/tts_dataset_files/cmudict-0.7b_nv22.01 rename to scripts/tts_dataset_files/cmudict-0.7b_nv22.07 index ec41c21ec18f..ebc75bdbbecd 100644 --- a/scripts/tts_dataset_files/cmudict-0.7b_nv22.01 +++ b/scripts/tts_dataset_files/cmudict-0.7b_nv22.07 @@ -133889,6 +133889,7 @@ CSP S IY1 EH1 S P IY1 CTR S IH1 T IY1 AA1 R CUDA K UW1 D AH0 CUDNN K UW1 D IY1 EH2 N EH2 N +CUSTOMIZABLE K AH2 S T AH0 M AY1 Z AH0 B AH0 L CYBERCRIME S AY1 B ER0 K R AY1 M DATACENTER D EY1 T AH0 S EH2 N T ER0 DDOS D IY2 D AO1 S @@ -133957,8 +133958,8 @@ NGC EH1 N JH IY1 S IY1 NGX EH1 N JH IY1 EH1 K S NHS EH1 N EY1 CH EH1 S NIH EH1 N AY1 EY1 CH -NVIDIA EH0 N V IY1 D IY0 AH0 -NVIDIA'S EH0 N V IY1 D IY0 AH0 Z +NVIDIA EH0 N V IH1 D IY0 AH0 +NVIDIA'S EH0 N V IH1 D IY0 AH0 Z NVLINK EH1 N V IY1 L IH1 NG K NVME EH1 N V IY1 EH1 M IY1 NVSWITCH EH1 N V IH1 S W IH1 CH diff --git a/tutorials/tts/FastPitch_Finetuning.ipynb b/tutorials/tts/FastPitch_Finetuning.ipynb index b9099362ffa8..19b509502ef3 100755 --- a/tutorials/tts/FastPitch_Finetuning.ipynb +++ b/tutorials/tts/FastPitch_Finetuning.ipynb @@ -244,7 +244,7 @@ "source": [ "# additional files\n", "!mkdir -p tts_dataset_files && cd tts_dataset_files \\\n", - "&& wget https://raw.githubusercontent.com/NVIDIA/NeMo/$BRANCH/scripts/tts_dataset_files/cmudict-0.7b_nv22.01 \\\n", + "&& wget https://raw.githubusercontent.com/NVIDIA/NeMo/$BRANCH/scripts/tts_dataset_files/cmudict-0.7b_nv22.07 \\\n", "&& wget https://raw.githubusercontent.com/NVIDIA/NeMo/$BRANCH/scripts/tts_dataset_files/heteronyms-030921 \\\n", "&& wget https://raw.githubusercontent.com/NVIDIA/NeMo/$BRANCH/nemo_text_processing/text_normalization/en/data/whitelist/lj_speech.tsv \\\n", "&& cd .." @@ -286,7 +286,7 @@ " train_dataset=./6097_manifest_train_dur_5_mins_local.json \\\n", " validation_datasets=./6097_manifest_dev_ns_all_local.json \\\n", " sup_data_path=./fastpitch_sup_data \\\n", - " phoneme_dict_path=tts_dataset_files/cmudict-0.7b_nv22.01 \\\n", + " phoneme_dict_path=tts_dataset_files/cmudict-0.7b_nv22.07 \\\n", " heteronyms_path=tts_dataset_files/heteronyms-030921 \\\n", " whitelist_path=tts_dataset_files/lj_speech.tsv \\\n", " exp_manager.exp_dir=./ljspeech_to_6097_no_mixing_5_mins \\\n", @@ -318,7 +318,7 @@ " sup_data_path=./fastpitch_sup_data`\n", " * We tell the script what manifest files to train and eval on, as well as where supplementary data is located (or will be calculated and saved during training if not provided).\n", " \n", - "* `phoneme_dict_path=tts_dataset_files/cmudict-0.7b_nv22.01 \n", + "* `phoneme_dict_path=tts_dataset_files/cmudict-0.7b_nv22.07 \n", "heteronyms_path=tts_dataset_files/heteronyms-030921\n", "whitelist_path=tts_dataset_files/lj_speech.tsv \n", "`\n", @@ -718,4 +718,4 @@ }, "nbformat": 4, "nbformat_minor": 5 -} \ No newline at end of file +} diff --git a/tutorials/tts/FastPitch_MixerTTS_Training.ipynb b/tutorials/tts/FastPitch_MixerTTS_Training.ipynb index 012a90bfb7a8..6c555c32b5a5 100644 --- a/tutorials/tts/FastPitch_MixerTTS_Training.ipynb +++ b/tutorials/tts/FastPitch_MixerTTS_Training.ipynb @@ -226,7 +226,7 @@ "\n", "# additional files\n", "!mkdir -p tts_dataset_files && cd tts_dataset_files \\\n", - "&& wget https://raw.githubusercontent.com/NVIDIA/NeMo/$BRANCH/scripts/tts_dataset_files/cmudict-0.7b_nv22.01 \\\n", + "&& wget https://raw.githubusercontent.com/NVIDIA/NeMo/$BRANCH/scripts/tts_dataset_files/cmudict-0.7b_nv22.07 \\\n", "&& wget https://raw.githubusercontent.com/NVIDIA/NeMo/$BRANCH/scripts/tts_dataset_files/heteronyms-030921 \\\n", "&& wget https://raw.githubusercontent.com/NVIDIA/NeMo/$BRANCH/nemo_text_processing/text_normalization/en/data/whitelist/lj_speech.tsv \\\n", "&& cd .." @@ -428,7 +428,7 @@ "\n", "# Grapheme-to-phoneme module\n", "g2p = EnglishG2p(\n", - " phoneme_dict=\"tts_dataset_files/cmudict-0.7b_nv22.01\",\n", + " phoneme_dict=\"tts_dataset_files/cmudict-0.7b_nv22.07\",\n", " heteronyms=\"tts_dataset_files/heteronyms-030921\"\n", ")\n", "\n", @@ -554,7 +554,7 @@ "validation_datasets=tests/data/asr/an4_val.json \\\n", "sup_data_types=\"['align_prior_matrix', 'pitch']\" \\\n", "sup_data_path={mixer_tts_sup_data_path} \\\n", - "phoneme_dict_path=tts_dataset_files/cmudict-0.7b_nv22.01 \\\n", + "phoneme_dict_path=tts_dataset_files/cmudict-0.7b_nv22.07 \\\n", "heteronyms_path=tts_dataset_files/heteronyms-030921 \\\n", "whitelist_path=tts_dataset_files/lj_speech.tsv \\\n", "pitch_mean={pitch_mean} \\\n", @@ -606,4 +606,4 @@ }, "nbformat": 4, "nbformat_minor": 5 -} \ No newline at end of file +} diff --git a/tutorials/tts/Tacotron2_Training.ipynb b/tutorials/tts/Tacotron2_Training.ipynb index 0ad3f114d458..3602e4fec24f 100644 --- a/tutorials/tts/Tacotron2_Training.ipynb +++ b/tutorials/tts/Tacotron2_Training.ipynb @@ -163,7 +163,7 @@ "# We will also need a few extra files for handling text.\n", "!(mkdir -p scripts/tts_dataset_files \\\n", " && cd scripts/tts_dataset_files \\\n", - " && wget https://raw.githubusercontent.com/NVIDIA/NeMo/$BRANCH/scripts/tts_dataset_files/cmudict-0.7b_nv22.01 \\\n", + " && wget https://raw.githubusercontent.com/NVIDIA/NeMo/$BRANCH/scripts/tts_dataset_files/cmudict-0.7b_nv22.07 \\\n", " && wget wget https://raw.githubusercontent.com/NVIDIA/NeMo/$BRANCH/scripts/tts_dataset_files/heteronyms-030921 \\\n", " && cd ..)\n", " \n", @@ -231,7 +231,7 @@ "sup_data_path: null\n", "sup_data_types: null\n", "\n", - "phoneme_dict_path: \"scripts/tts_dataset_files/cmudict-0.7b_nv22.01\"\n", + "phoneme_dict_path: \"scripts/tts_dataset_files/cmudict-0.7b_nv22.07\"\n", "heteronyms_path: \"scripts/tts_dataset_files/heteronyms-030921\"\n", "whitelist_path: \"nemo_text_processing/text_normalization/en/data/whitelist/lj_speech.tsv\"\n", "```\n", From 81df7a94e06869856ec5adf45967848323df7189 Mon Sep 17 00:00:00 2001 From: "He Huang (Steve)" <105218074+stevehuang52@users.noreply.github.com> Date: Fri, 8 Jul 2022 09:12:58 -0400 Subject: [PATCH 12/52] [Add] Support for Different LRs with Param Groups (#4508) * add support for param groups Signed-off-by: stevehuang52 * make config more general Signed-off-by: stevehuang52 Co-authored-by: Eric Harper --- nemo/core/classes/modelPT.py | 52 +++++++++++++++++++++++++++++++++--- 1 file changed, 48 insertions(+), 4 deletions(-) diff --git a/nemo/core/classes/modelPT.py b/nemo/core/classes/modelPT.py index 180fff6e4ff2..5fb6736b54be 100644 --- a/nemo/core/classes/modelPT.py +++ b/nemo/core/classes/modelPT.py @@ -606,10 +606,54 @@ def setup_optimizer_param_groups(self): See https://pytorch.org/docs/stable/optim.html for more information. By default, ModelPT will use self.parameters(). Override this method to add custom param groups. - """ - param_groups = None - if hasattr(self, 'parameters'): - param_groups = [{'params': self.parameters()}] + In the config file, add 'optim_param_groups' to support different LRs + for different components (unspecified params will use the default LR): + model: + optim_param_groups: + encoder: + lr: 1e-4 + momentum: 0.8 + decoder: + lr: 1e-3 + optim: + lr: 3e-3 + momentum: 0.9 + """ + if not hasattr(self, "parameters"): + self._optimizer_param_groups = None + return + + known_groups = [] + param_groups = [] + if "optim_param_groups" in self.cfg: + param_groups_cfg = self.cfg.optim_param_groups + for group, group_cfg in param_groups_cfg.items(): + module = getattr(self, group, None) + if module is None: + raise ValueError(f"{group} not found in model.") + elif hasattr(module, "parameters"): + known_groups.append(group) + new_group = {"params": module.parameters()} + for k, v in group_cfg.items(): + new_group[k] = v + param_groups.append(new_group) + else: + raise ValueError(f"{group} does not have parameters.") + + other_params = [] + for n, p in self.named_parameters(): + is_unknown = True + for group in known_groups: + if n.startswith(group): + is_unknown = False + if is_unknown: + other_params.append(p) + + if len(other_params): + param_groups = [{"params": other_params}] + param_groups + else: + param_groups = [{"params": self.parameters()}] + self._optimizer_param_groups = param_groups def configure_optimizers(self): From c0f5bff9f06451304789634d5ecd3fe26049f23d Mon Sep 17 00:00:00 2001 From: tbartley94 <90423858+tbartley94@users.noreply.github.com> Date: Fri, 8 Jul 2022 13:53:47 -0400 Subject: [PATCH 13/52] Weighted bucketing (#4474) --- docs/source/asr/configs.rst | 1 + docs/source/asr/datasets.rst | 46 +++++++++++++++++++ .../asr/data/audio_to_text_dataset.py | 40 ++++++++++++---- .../asr/models/configs/asr_models_config.py | 1 + .../asr/test_asr_ctc_encoder_model_bpe.py | 2 + .../asr/test_asr_ctcencdec_model.py | 2 + 6 files changed, 83 insertions(+), 9 deletions(-) diff --git a/docs/source/asr/configs.rst b/docs/source/asr/configs.rst index 995674bc06e6..6a1fd45eff63 100644 --- a/docs/source/asr/configs.rst +++ b/docs/source/asr/configs.rst @@ -50,6 +50,7 @@ An example ASR train and validation configuration should look similar to the fol # bucketing params bucketing_strategy: "synced_randomized" bucketing_batch_size: null + bucketing_weights: null validation_ds: manifest_filepath: ??? diff --git a/docs/source/asr/datasets.rst b/docs/source/asr/datasets.rst index 3561e3814041..364c7fea1926 100644 --- a/docs/source/asr/datasets.rst +++ b/docs/source/asr/datasets.rst @@ -321,3 +321,49 @@ The fully_randomized strategy would have lower speedup than synced_randomized bu Bucketing may improve the training speed more than 2x but may affect the final accuracy of the model slightly. Training for more epochs and using 'synced_randomized' strategy help to fill this gap. Currently bucketing feature is just supported for tarred datasets. + +Upsampling Datasets +------------------ + +Buckets may also be 'weighted' to allow multiple runs through a target dataset during each training epoch. This can be beneficial in cases when a dataset is composed of several component sets of unequal sizes and one desires to mitigate bias towards the larger sets through oversampling. + +Weighting is managed with the `bucketing_weights` parameter. After passing your composite tarred datasets in the format described above for bucketing, pass a list of integers (one per bucket) to indicate how many times a manifest should be read during training. + +For example, by passing `[2,1,1,3]` to the code below: + +.. code:: + + python speech_to_text_bpe.py + ... + model.train_ds.manifest_filepath=[[PATH_TO_TARS/bucket1/tarred_audio_manifest.json], + [PATH_TO_TARS/bucket2/tarred_audio_manifest.json], + [PATH_TO_TARS/bucket3/tarred_audio_manifest.json], + [PATH_TO_TARS/bucket4/tarred_audio_manifest.json]] + model.train_ds.tarred_audio_filepaths=[[PATH_TO_TARS/bucket1/audio__OP_0..511_CL_.tar], + [PATH_TO_TARS/bucket2/audio__OP_0..511_CL_.tar], + [PATH_TO_TARS/bucket3/audio__OP_0..511_CL_.tar], + [PATH_TO_TARS/bucket4/audio__OP_0..511_CL_.tar]] + ... + model.train_ds.bucketing_weights=[2,1,1,3] + +NeMo will configure training so that all data in `bucket1` will be present twice in a training epoch, `bucket4` will be present three times, and that of `bucket2` and `bucket3` will occur only once each. Note that this will increase the effective amount of data present during training and thus affect training time per epoch. + +If using adaptive bucketing, note that the same batch size will be assigned to each instance of the upsampled data. That is, given the following: + +.. code:: + + python speech_to_text_bpe.py + ... + model.train_ds.manifest_filepath=[[PATH_TO_TARS/bucket1/tarred_audio_manifest.json], + [PATH_TO_TARS/bucket2/tarred_audio_manifest.json], + [PATH_TO_TARS/bucket3/tarred_audio_manifest.json], + [PATH_TO_TARS/bucket4/tarred_audio_manifest.json]] + ... + ... + model.train_ds.bucketing_weights=[2,1,1,3] + model.train_ds.bucketing_batch_size=[4,4,4,2] + +All instances of data from `bucket4` will still be trained with a batch size of 2 while all others would have a batch size of 4. As with standard bucketing, this requires `batch_size`` to be set to 1. +If `bucketing_batch_size` is not specified, all datasets will be passed with the same fixed batch size as specified by the `batch_size` parameter. + +It is recommended to set bucketing strategies to `fully_randomized` during multi-GPU training to prevent possible dataset bias during training. \ No newline at end of file diff --git a/nemo/collections/asr/data/audio_to_text_dataset.py b/nemo/collections/asr/data/audio_to_text_dataset.py index a432d4740562..eb60de968903 100644 --- a/nemo/collections/asr/data/audio_to_text_dataset.py +++ b/nemo/collections/asr/data/audio_to_text_dataset.py @@ -162,6 +162,12 @@ def get_tarred_dataset( tarred_audio_filepaths = convert_to_config_list(tarred_audio_filepaths) manifest_filepaths = convert_to_config_list(manifest_filepaths) + bucketing_weights = config.get('bucketing_weights', None) # For upsampling buckets + if bucketing_weights: + for idx, weight in enumerate(bucketing_weights): + if not isinstance(weight, int) or weight <= 0: + raise ValueError(f"bucket weights must be positive integers") + if len(manifest_filepaths) != len(tarred_audio_filepaths): raise ValueError( f"manifest_filepaths (length={len(manifest_filepaths)}) and tarred_audio_filepaths (length={len(tarred_audio_filepaths)}) need to have the same number of buckets." @@ -216,8 +222,10 @@ def get_tarred_dataset( world_size=world_size, return_sample_id=config.get('return_sample_id', False), ) - - datasets.append(dataset) + if bucketing_weights: + [datasets.append(dataset) for _ in range(bucketing_weights[dataset_idx])] + else: + datasets.append(dataset) return get_chain_dataset(datasets=datasets, ds_config=config) @@ -404,17 +412,31 @@ def get_chain_dataset(datasets, ds_config): def calc_bucketing_batch_sizes(ds_config, datasets_len): bucketing_batch_size = ds_config['bucketing_batch_size'] + bucketing_weights = ds_config.get('bucketing_weights', None) # To adjust for upsampled buckets + + bucketing_batch_sizes = [] + if ds_config['batch_size'] != 1: raise ValueError( f"batch_size should be set to one when bucketing_batch_size is set and adaptive bucketing is enabled (batch_size={ds_config['batch_size']}!" ) - if type(bucketing_batch_size) == int: - bucketing_batch_sizes = [] - for idx in range(datasets_len): - scale_factor = datasets_len - idx - bucketing_batch_sizes.append(scale_factor * bucketing_batch_size) - elif isinstance(bucketing_batch_size, ListConfig) or isinstance(bucketing_batch_size, list): - bucketing_batch_sizes = bucketing_batch_size + if type(bucketing_batch_size) == int: # linear scaling + if bucketing_weights: # Want same batchsize for the same duplicated bucket + for idx, weight in enumerate(bucketing_weights): + scale_factor = datasets_len - idx + [bucketing_batch_sizes.append(scale_factor * bucketing_batch_size) for _ in range(weight)] + else: + for idx in range(datasets_len): + scale_factor = datasets_len - idx + bucketing_batch_sizes.append(scale_factor * bucketing_batch_size) + elif isinstance(bucketing_batch_size, ListConfig) or isinstance( + bucketing_batch_size, list + ): # assigned bucket sizes + if bucketing_weights: # Want same batchsize for same duplicated bucket + for idx, weight in enumerate(bucketing_weights): + [bucketing_batch_sizes.append(bucketing_batch_size[idx]) for _ in range(weight)] + else: + bucketing_batch_sizes = bucketing_batch_size else: raise ValueError( f"bucketing_batch_size should be an integer or a list (bucketing_batch_size={bucketing_batch_size})!" diff --git a/nemo/collections/asr/models/configs/asr_models_config.py b/nemo/collections/asr/models/configs/asr_models_config.py index d958f0ae3e12..ee7e2b43d881 100644 --- a/nemo/collections/asr/models/configs/asr_models_config.py +++ b/nemo/collections/asr/models/configs/asr_models_config.py @@ -60,6 +60,7 @@ class ASRDatasetConfig(nemo.core.classes.dataset.DatasetConfig): # bucketing params bucketing_strategy: str = "synced_randomized" bucketing_batch_size: Optional[Any] = None + bucketing_weights: Optional[List[int]] = None @dataclass diff --git a/tests/collections/asr/test_asr_ctc_encoder_model_bpe.py b/tests/collections/asr/test_asr_ctc_encoder_model_bpe.py index 0890826bd7e9..dee1a10f3504 100644 --- a/tests/collections/asr/test_asr_ctc_encoder_model_bpe.py +++ b/tests/collections/asr/test_asr_ctc_encoder_model_bpe.py @@ -301,6 +301,7 @@ def test_ASRDatasetConfig_for_AudioToBPEDataset(self): 'blank_index', 'bucketing_batch_size', 'bucketing_strategy', + 'bucketing_weights', ] REMAP_ARGS = {'trim_silence': 'trim', 'labels': 'tokenizer'} @@ -335,6 +336,7 @@ def test_ASRDatasetConfig_for_TarredAudioToBPEDataset(self): 'world_size', 'bucketing_batch_size', 'bucketing_strategy', + 'bucketing_weights', ] REMAP_ARGS = { diff --git a/tests/collections/asr/test_asr_ctcencdec_model.py b/tests/collections/asr/test_asr_ctcencdec_model.py index 23af7efcb57e..8e154bf724ea 100644 --- a/tests/collections/asr/test_asr_ctcencdec_model.py +++ b/tests/collections/asr/test_asr_ctcencdec_model.py @@ -271,6 +271,7 @@ def test_ASRDatasetConfig_for_AudioToCharDataset(self): 'use_start_end_token', 'bucketing_batch_size', 'bucketing_strategy', + 'bucketing_weights', ] REMAP_ARGS = {'trim_silence': 'trim'} @@ -299,6 +300,7 @@ def test_ASRDatasetConfig_for_TarredAudioToCharDataset(self): 'use_start_end_token', 'bucketing_batch_size', 'bucketing_strategy', + 'bucketing_weights', ] REMAP_ARGS = { From 2af11fe5282ea0c8a43cf67df3f2876353341cbf Mon Sep 17 00:00:00 2001 From: Nithin Rao Date: Fri, 8 Jul 2022 12:43:55 -0700 Subject: [PATCH 14/52] Add silence handling for speaker diarization pipeline (#4512) * initial commit Signed-off-by: nithinraok * fixed silence wav file issue causing clustering to evaluate on null embeddings Signed-off-by: nithinraok * fixed zero duration issue Signed-off-by: nithinraok * updated with comments Signed-off-by: nithinraok * minor doc change Signed-off-by: nithinraok * update log Signed-off-by: nithinraok --- .../asr/models/clustering_diarizer.py | 15 ++++-- .../asr/parts/utils/speaker_utils.py | 47 ++++++++++++------- .../pathfiles_to_diarize_manifest.py | 2 +- 3 files changed, 43 insertions(+), 21 deletions(-) diff --git a/nemo/collections/asr/models/clustering_diarizer.py b/nemo/collections/asr/models/clustering_diarizer.py index 03957afc90e9..8cfb7d7636b2 100644 --- a/nemo/collections/asr/models/clustering_diarizer.py +++ b/nemo/collections/asr/models/clustering_diarizer.py @@ -37,6 +37,7 @@ perform_clustering, score_labels, segments_manifest_to_subsegments_manifest, + validate_vad_manifest, write_rttm2manifest, ) from nemo.collections.asr.parts.utils.vad_utils import ( @@ -258,9 +259,14 @@ def _run_vad(self, manifest_file): shift_length_in_sec=self._vad_shift_length_in_sec, num_workers=self._cfg.num_workers, ) - AUDIO_VAD_RTTM_MAP = deepcopy(self.AUDIO_RTTM_MAP.copy()) - for key in AUDIO_VAD_RTTM_MAP: - AUDIO_VAD_RTTM_MAP[key]['rttm_filepath'] = os.path.join(table_out_dir, key + ".txt") + + AUDIO_VAD_RTTM_MAP = {} + for key in self.AUDIO_RTTM_MAP: + if os.path.exists(os.path.join(table_out_dir, key + ".txt")): + AUDIO_VAD_RTTM_MAP[key] = deepcopy(self.AUDIO_RTTM_MAP[key]) + AUDIO_VAD_RTTM_MAP[key]['rttm_filepath'] = os.path.join(table_out_dir, key + ".txt") + else: + logging.warning(f"no vad file found for {key} due to zero or negative duration") write_rttm2manifest(AUDIO_VAD_RTTM_MAP, self._vad_out_file) self._speaker_manifest_path = self._vad_out_file @@ -314,8 +320,9 @@ def _perform_speech_activity_detection(self): self._speaker_manifest_path = write_rttm2manifest(self.AUDIO_RTTM_MAP, self._speaker_manifest_path) else: raise ValueError( - "Only one of diarizer.oracle_vad, vad.model_path or vad.external_vad_manifest must be passed" + "Only one of diarizer.oracle_vad, vad.model_path or vad.external_vad_manifest must be passed from config" ) + validate_vad_manifest(self.AUDIO_RTTM_MAP, vad_manifest=self._speaker_manifest_path) def _extract_embeddings(self, manifest_file: str): """ diff --git a/nemo/collections/asr/parts/utils/speaker_utils.py b/nemo/collections/asr/parts/utils/speaker_utils.py index 65b5c23df47d..ddf6df0bffcc 100644 --- a/nemo/collections/asr/parts/utils/speaker_utils.py +++ b/nemo/collections/asr/parts/utils/speaker_utils.py @@ -101,7 +101,9 @@ def audio_rttm_map(manifest): AUDIO_RTTM_MAP[uniqname] = meta else: raise KeyError( - "file {} is already part AUDIO_RTTM_Map, it might be duplicated".format(meta['audio_filepath']) + "file {} is already part of AUDIO_RTTM_MAP, it might be duplicated, Note: file basename must be unique".format( + meta['audio_filepath'] + ) ) return AUDIO_RTTM_MAP @@ -611,6 +613,31 @@ def isOverlap(rangeA, rangeB): return end1 > start2 and end2 > start1 +def validate_vad_manifest(AUDIO_RTTM_MAP, vad_manifest): + """ + This function will check the valid speech segments in the manifest file which is either + generated from NeMo voice activity detection(VAD) or oracle VAD. + If an audio file does not contain any valid speech segments, we ignore the audio file + (indexed by uniq_id) for the rest of the processing steps. + """ + vad_uniq_ids = set() + with open(vad_manifest, 'r') as vad_file: + for line in vad_file: + line = line.strip() + dic = json.loads(line) + if dic['duration'] > 0: + vad_uniq_ids.add(dic['uniq_id']) + + provided_uniq_ids = set(AUDIO_RTTM_MAP.keys()) + silence_ids = provided_uniq_ids - vad_uniq_ids + for uniq_id in silence_ids: + del AUDIO_RTTM_MAP[uniq_id] + logging.warning(f"{uniq_id} is ignored since the file does not contain any speech signal to be processed.") + + if len(AUDIO_RTTM_MAP) == 0: + raise ValueError("All files present in manifest contains silence, aborting next steps") + + def getOverlapRange(rangeA, rangeB): """ Calculate the overlapping range between rangeA and rangeB. @@ -740,15 +767,6 @@ def getMergedRanges(label_list_A: List, label_list_B: List, deci: int = 3) -> Li return [[int2fl(x[0] - 1, deci), int2fl(x[1], deci)] for x in combined] -def getMinMaxOfRangeList(ranges): - """ - Get the min and max of a given range list. - """ - _max = max([x[1] for x in ranges]) - _min = min([x[0] for x in ranges]) - return _min, _max - - def getSubRangeList(target_range, source_range_list) -> List: """ Get the ranges that has overlaps with the target range from the source_range_list. @@ -814,16 +832,13 @@ def write_rttm2manifest(AUDIO_RTTM_MAP: str, manifest_file: str, include_uniq_id vad_start_end_list = combine_float_overlaps(vad_start_end_list_raw, deci) if len(vad_start_end_list) == 0: logging.warning(f"File ID: {uniq_id}: The VAD label is not containing any speech segments.") - elif duration == 0: - logging.warning(f"File ID: {uniq_id}: The audio file has zero duration.") + elif duration <= 0: + logging.warning(f"File ID: {uniq_id}: The audio file has negative or zero duration.") else: - min_vad, max_vad = getMinMaxOfRangeList(vad_start_end_list) - if max_vad > round(offset + duration, deci) or min_vad < offset: - logging.warning("RTTM label has been truncated since start is greater than duration of audio file") overlap_range_list = getSubRangeList( source_range_list=vad_start_end_list, target_range=[offset, offset + duration] ) - write_overlap_segments(outfile, AUDIO_RTTM_MAP, uniq_id, overlap_range_list, include_uniq_id, deci) + write_overlap_segments(outfile, AUDIO_RTTM_MAP, uniq_id, overlap_range_list, include_uniq_id, deci) return manifest_file diff --git a/scripts/speaker_tasks/pathfiles_to_diarize_manifest.py b/scripts/speaker_tasks/pathfiles_to_diarize_manifest.py index 66f5d787e0cd..4a5b12f57d94 100644 --- a/scripts/speaker_tasks/pathfiles_to_diarize_manifest.py +++ b/scripts/speaker_tasks/pathfiles_to_diarize_manifest.py @@ -126,7 +126,7 @@ def main( duration = None if add_duration: - y, sr = librosa.get_duration(filename=audio_line, sr=None) + y, sr = librosa.load(audio_line, sr=None) duration = librosa.get_duration(y=y, sr=sr) meta = [ { From a8266e46d4dcb2784f800f9b08e4a181aa50d31c Mon Sep 17 00:00:00 2001 From: Boris Fomitchev Date: Fri, 8 Jul 2022 15:31:05 -0700 Subject: [PATCH 15/52] Fix runtime check (#4501) * Runtime check refinements Signed-off-by: Boris Fomitchev * Added fp32 casting for ASR nets export Signed-off-by: Boris Fomitchev * style Signed-off-by: Boris Fomitchev * Used torch.float32 for clarity Signed-off-by: Boris Fomitchev * Fixing parameters passing Signed-off-by: Boris Fomitchev --- nemo/collections/asr/models/asr_model.py | 7 ++- nemo/core/classes/exportable.py | 10 +++-- nemo/utils/export_utils.py | 56 ++++++++++++++++++------ scripts/export.py | 23 +++++++++- 4 files changed, 76 insertions(+), 20 deletions(-) diff --git a/nemo/collections/asr/models/asr_model.py b/nemo/collections/asr/models/asr_model.py index 30a3b7fd8890..cf7ed5c343ee 100644 --- a/nemo/collections/asr/models/asr_model.py +++ b/nemo/collections/asr/models/asr_model.py @@ -20,6 +20,7 @@ from nemo.core.classes import ModelPT from nemo.core.classes.exportable import Exportable from nemo.utils import model_utils +from nemo.utils.export_utils import cast_all __all__ = ['ASRModel'] @@ -125,6 +126,8 @@ def forward_for_export(self, input, length=None): else: decoder_input = encoder_output if hasattr(self.output_module, 'forward_for_export'): - return self.output_module.forward_for_export(decoder_input) + ret = self.output_module.forward_for_export(decoder_input) else: - return self.output_module(decoder_input) + ret = self.output_module(decoder_input) + # convert all FP16 results to FP32 for consistency + return cast_all(ret, from_dtype=torch.float16, to_dtype=torch.float32) diff --git a/nemo/core/classes/exportable.py b/nemo/core/classes/exportable.py index 3d5c19769953..eb8df0826523 100644 --- a/nemo/core/classes/exportable.py +++ b/nemo/core/classes/exportable.py @@ -13,6 +13,7 @@ # limitations under the License. import os from abc import ABC +from typing import Dict, List, Optional, Type, Union import onnx import torch @@ -56,7 +57,7 @@ def export( do_constant_folding=True, onnx_opset_version=None, training=TrainingMode.EVAL, - check_trace: bool = False, + check_trace: Union[bool, List[torch.Tensor]] = False, dynamic_axes=None, check_tolerance=0.01, ): @@ -174,8 +175,11 @@ def _export( ) if check_trace: - verify_runtime(output, input_list, input_dict, input_names, output_names, output_example) - + if isinstance(check_trace, bool): + check_trace_input = [input_example] + else: + check_trace_input = check_trace + verify_runtime(self, output, check_trace_input, input_names) else: raise ValueError(f'Encountered unknown export format {format}.') finally: diff --git a/nemo/utils/export_utils.py b/nemo/utils/export_utils.py index 670b54d8ddd9..82cc5bcf2fca 100644 --- a/nemo/utils/export_utils.py +++ b/nemo/utils/export_utils.py @@ -44,6 +44,23 @@ class ExportFormat(Enum): } +def cast_tensor(x, from_dtype=torch.float16, to_dtype=torch.float32): + return x.to(dtype=to_dtype) if x.dtype == from_dtype else x + + +def cast_all(x, from_dtype=torch.float16, to_dtype=torch.float32): + if isinstance(x, torch.Tensor): + return cast_tensor(x, from_dtype=from_dtype, to_dtype=to_dtype) + else: + if isinstance(x, dict): + new_dict = {} + for k in x.keys(): + new_dict[k] = cast_all(x[k], from_dtype=from_dtype, to_dtype=to_dtype) + return new_dict + elif isinstance(x, tuple): + return tuple(cast_all(y, from_dtype=from_dtype, to_dtype=to_dtype) for y in x) + + class CastToFloat(nn.Module): def __init__(self, mod): super(CastToFloat, self).__init__() @@ -51,7 +68,7 @@ def __init__(self, mod): def forward(self, x): if torch.is_autocast_enabled(): - ret = self.mod.forward(x.to(torch.float)).to(x.dtype) + ret = self.mod.forward(x.to(torch.float32)).to(x.dtype) else: ret = self.mod.forward(x) return ret @@ -68,6 +85,7 @@ def get_export_format(filename: str): def augment_filename(output: str, prepend: str): if prepend == 'self': return output + path, filename = os.path.split(output) filename = f"{prepend}-{filename}" return os.path.join(path, filename) @@ -102,23 +120,21 @@ def parse_input_example(input_example): return input_list, input_dict -def to_onnxrt_input(input_names, input_dict, input_list): +def to_onnxrt_input(ort_input_names, input_names, input_dict, input_list): odict = {} for k in reversed(input_names): if k in input_dict: - odict[k] = input_dict[k].cpu().numpy() + val = input_dict[k].cpu().numpy() else: - odict[k] = input_list.pop().cpu().numpy() + val = input_list.pop().cpu().numpy() + if k in ort_input_names: + odict[k] = val return odict -def verify_runtime( - output, input_list, input_dict, input_names, output_names, output_example, check_tolerance=0.01, -): - # Verify the model can be read, and is valid - +def verify_runtime(model, output, input_examples, input_names, check_tolerance=0.01): onnx_model = onnx.load(output) - input_names = [node.name for node in onnx_model.graph.input] + ort_input_names = [node.name for node in onnx_model.graph.input] global ort_available if not ort_available: @@ -131,18 +147,30 @@ def verify_runtime( sess = onnxruntime.InferenceSession( onnx_model.SerializeToString(), sess_options=onnx_session_opt, providers=['CUDAExecutionProvider'] ) - ort_out = sess.run(output_names, to_onnxrt_input(input_names, input_dict, input_list)) all_good = True + for input_example in input_examples: + input_list, input_dict = parse_input_example(input_example) + output_example = model.forward(*input_list, **input_dict) + ort_input = to_onnxrt_input(ort_input_names, input_names, input_dict, input_list) + all_good = all_good and run_ort_and_compare(sess, ort_input, output_example, check_tolerance) + status = "SUCCESS" if all_good else "FAIL" + logging.info(f"ONNX generated at {output} verified with onnxruntime : " + status) + return all_good - for i, out in enumerate(ort_out[0]): + +def run_ort_and_compare(sess, ort_input, output_example, check_tolerance=0.01): + # Verify the model can be read, and is valid + ort_out = sess.run(None, ort_input) + all_good = True + for i, out in enumerate(ort_out): expected = output_example[i] + if torch.is_tensor(expected): tout = torch.from_numpy(out) + logging.info(f"Checking output {i}, shape: {expected.shape}:\n{expected}\n{tout}") if not torch.allclose(tout, expected.cpu(), rtol=check_tolerance, atol=100 * check_tolerance): all_good = False logging.info(f"onnxruntime results mismatch! PyTorch(expected):\n{expected}\nONNXruntime:\n{tout}") - status = "SUCCESS" if all_good else "FAIL" - logging.info(f"ONNX generated at {output} verified with onnxruntime : " + status) return all_good diff --git a/scripts/export.py b/scripts/export.py index da7989e0e73f..0a851f01ebd9 100644 --- a/scripts/export.py +++ b/scripts/export.py @@ -113,20 +113,41 @@ def nemo_export(argv): # # Add custom export parameters here # + check_trace = args.runtime_check + in_args = {} + max_batch = 1 + max_dim = None if args.max_batch is not None: in_args["max_batch"] = args.max_batch + max_batch = args.max_batch if args.max_dim is not None: in_args["max_dim"] = args.max_dim + max_dim = args.max_dim autocast = nullcontext model.to(device=args.device).freeze() + model.eval() + with torch.inference_mode(): + input_example = model.input_module.input_example(**in_args) + if check_trace: + check_trace = [input_example] + if max_dim: + in_args["max_dim"] = (max_dim + 1) // 2 + in_args["max_batch"] = (max_batch + 1) // 2 + input_example2 = model.input_module.input_example(**in_args) + check_trace.append(input_example2) + if args.autocast: autocast = torch.cuda.amp.autocast try: with autocast(), torch.inference_mode(): _, descriptions = model.export( - out, check_trace=args.runtime_check, onnx_opset_version=args.onnx_opset, verbose=args.verbose, + out, + input_example=input_example, + check_trace=check_trace, + onnx_opset_version=args.onnx_opset, + verbose=args.verbose, ) except Exception as e: From 726ad223bb2b140132532c1071f0ee56860221d4 Mon Sep 17 00:00:00 2001 From: Nithin Rao Date: Fri, 8 Jul 2022 19:11:13 -0700 Subject: [PATCH 16/52] Update finetune label models (#4504) * initial_script Signed-off-by: nithinraok * move old script Signed-off-by: nithinraok * remove finetune func from label models Signed-off-by: nithinraok * style clean Signed-off-by: nithinraok * updated config Signed-off-by: nithinraok * update tutorial Signed-off-by: nithinraok * lgtm fixes Signed-off-by: nithinraok * updated based on comments Signed-off-by: nithinraok * update doc Signed-off-by: nithinraok --- .../recognition/conf/titanet-finetune.yaml | 158 ++ .../recognition/speaker_reco_finetune.py | 80 +- nemo/collections/asr/models/label_models.py | 57 - nemo/core/classes/modelPT.py | 2 +- scripts/speaker_tasks/filelist_to_manifest.py | 4 +- .../Speaker_Identification_Verification.ipynb | 2491 ++++++++--------- 6 files changed, 1401 insertions(+), 1391 deletions(-) create mode 100644 examples/speaker_tasks/recognition/conf/titanet-finetune.yaml diff --git a/examples/speaker_tasks/recognition/conf/titanet-finetune.yaml b/examples/speaker_tasks/recognition/conf/titanet-finetune.yaml new file mode 100644 index 000000000000..b7b73517ef80 --- /dev/null +++ b/examples/speaker_tasks/recognition/conf/titanet-finetune.yaml @@ -0,0 +1,158 @@ +name: &name "TitaNet-Finetune" +sample_rate: &sample_rate 16000 + +init_from_pretrained_model: + speaker_tasks: + name: 'titanet_large' + include: ["preprocessor","encoder"] + exclude: ["decoder.final"] # Add specific layer names here to exlude or just ["decoder"] if to exclude all of decoder pretrained weights + +model: + train_ds: + manifest_filepath: ??? + sample_rate: 16000 + labels: null + batch_size: 64 + shuffle: True + is_tarred: False + tarred_audio_filepaths: null + tarred_shard_strategy: "scatter" + augmentor: + speed: + prob: 0.3 + sr: *sample_rate + resample_type: 'kaiser_fast' + min_speed_rate: 0.95 + max_speed_rate: 1.05 + + validation_ds: + manifest_filepath: ??? + sample_rate: 16000 + labels: null + batch_size: 128 + shuffle: False + + model_defaults: + filters: 1024 + repeat: 3 + dropout: 0.1 + separable: true + se: true + se_context_size: -1 + kernel_size_factor: 1.0 + + preprocessor: + _target_: nemo.collections.asr.modules.AudioToMelSpectrogramPreprocessor + normalize: "per_feature" + window_size: 0.025 + sample_rate: *sample_rate + window_stride: 0.01 + window: "hann" + features: &n_mels 80 + n_fft: 512 + frame_splicing: 1 + dither: 0.00001 + + encoder: + _target_: nemo.collections.asr.modules.ConvASREncoder + feat_in: *n_mels + activation: relu + conv_mask: true + + jasper: + - filters: ${model.model_defaults.filters} + repeat: 1 + kernel: [3] + stride: [1] + dilation: [1] + dropout: 0.0 + residual: false + separable: ${model.model_defaults.separable} + se: ${model.model_defaults.se} + se_context_size: ${model.model_defaults.se_context_size} + + - filters: ${model.model_defaults.filters} + repeat: ${model.model_defaults.repeat} + kernel: [7] + stride: [1] + dilation: [1] + dropout: ${model.model_defaults.dropout} + residual: true + separable: ${model.model_defaults.separable} + se: ${model.model_defaults.se} + se_context_size: ${model.model_defaults.se_context_size} + + - filters: ${model.model_defaults.filters} + repeat: ${model.model_defaults.repeat} + kernel: [11] + stride: [1] + dilation: [1] + dropout: ${model.model_defaults.dropout} + residual: true + separable: ${model.model_defaults.separable} + se: ${model.model_defaults.se} + se_context_size: ${model.model_defaults.se_context_size} + + - filters: ${model.model_defaults.filters} + repeat: ${model.model_defaults.repeat} + kernel: [15] + stride: [1] + dilation: [1] + dropout: ${model.model_defaults.dropout} + residual: true + separable: ${model.model_defaults.separable} + se: ${model.model_defaults.se} + se_context_size: ${model.model_defaults.se_context_size} + + - filters: &enc_feat_out 3072 + repeat: 1 + kernel: [1] + stride: [1] + dilation: [1] + dropout: 0.0 + residual: false + separable: ${model.model_defaults.separable} + se: ${model.model_defaults.se} + se_context_size: ${model.model_defaults.se_context_size} + + decoder: + _target_: nemo.collections.asr.modules.SpeakerDecoder + feat_in: *enc_feat_out + num_classes: ??? + pool_mode: 'attention' + emb_sizes: 192 + angular: True + + loss: + scale: 30 + margin: 0.2 + + optim: + name: adamw + lr: .0001 #(original titanet-large was trained with 0.08 lr) + weight_decay: 0.0002 + + # scheduler setup + sched: + name: CosineAnnealing + warmup_ratio: 0.1 + min_lr: 0.0 + +trainer: + devices: 1 # number of gpus (original titanet-large was trained on 4 nodes with 8 gpus each) + max_epochs: 10 + max_steps: -1 # computed at runtime if not set + num_nodes: 1 + accelerator: gpu + strategy: ddp + deterministic: True + enable_checkpointing: False + logger: False + log_every_n_steps: 1 # Interval of logging. + val_check_interval: 1.0 # Set to 0.25 to check 4 times per epoch, or an int for number of iterations + +exp_manager: + exp_dir: null + name: *name + create_tensorboard_logger: True + create_checkpoint_callback: True diff --git a/examples/speaker_tasks/recognition/speaker_reco_finetune.py b/examples/speaker_tasks/recognition/speaker_reco_finetune.py index c1f37fb71e1e..a1032c7978be 100644 --- a/examples/speaker_tasks/recognition/speaker_reco_finetune.py +++ b/examples/speaker_tasks/recognition/speaker_reco_finetune.py @@ -12,82 +12,32 @@ # See the License for the specific language governing permissions and # limitations under the License. -""" -This script demonstrates how to use pretrained .nemo speaker recognition model to finetune on your domain set -Usage: -python speaker_reco_finetune.py --pretrained_model='/path/to/.nemo_or_.ckpt/file' --finetune_config_file=/path/to/finetune/config/yaml/file' - -in finetune config file make sure to change -- train manifest -- validation manifest -- decoder num classes -- learning rate -- num of epochs - -for finetuning tips see: https://github.com/NVIDIA/NeMo/blob/main/tutorials/speaker_tasks/Speaker_Recognition_Verification.ipynb -""" - - -from argparse import ArgumentParser - import pytorch_lightning as pl from omegaconf import OmegaConf +from pytorch_lightning import seed_everything from nemo.collections.asr.models import EncDecSpeakerLabelModel +from nemo.core.config import hydra_runner from nemo.utils import logging from nemo.utils.exp_manager import exp_manager +seed_everything(42) -def main(): - parser = ArgumentParser() - parser.add_argument( - "--pretrained_model", type=str, default="titanet_large", required=False, help="Pass your trained .nemo model", - ) - parser.add_argument( - "--finetune_config_file", - type=str, - required=True, - help="path to speakernet config yaml file to load train, validation dataset and also for trainer parameters", - ) - - parser.add_argument( - "--freeze_encoder", - type=bool, - required=False, - default=True, - help="True if speakernet encoder paramteres needs to be frozen while finetuning", - ) - - args = parser.parse_args() - - if args.pretrained_model.endswith('.nemo'): - logging.info(f"Using local speaker model from {args.pretrained_model}") - speaker_model = EncDecSpeakerLabelModel.restore_from(restore_path=args.pretrained_model) - elif args.pretrained_model.endswith('.ckpt'): - logging.info(f"Using local speaker model from checkpoint {args.pretrained_model}") - speaker_model = EncDecSpeakerLabelModel.load_from_checkpoint(checkpoint_path=args.pretrained_model) - else: - logging.info("Using pretrained speaker recognition model from NGC") - speaker_model = EncDecSpeakerLabelModel.from_pretrained(model_name=args.pretrained_model) - - finetune_config = OmegaConf.load(args.finetune_config_file) - - if 'test_ds' in finetune_config.model and finetune_config.model.test_ds is not None: - finetune_config.model.test_ds = None - logging.warning("Removing test ds") - - speaker_model.setup_finetune_model(finetune_config.model) - finetune_trainer = pl.Trainer(**finetune_config.trainer) - speaker_model.set_trainer(finetune_trainer) - _ = exp_manager(finetune_trainer, finetune_config.get('exp_manager', None)) - speaker_model.setup_optimization(finetune_config.optim) +@hydra_runner(config_path="conf", config_name="titanet-finetune.yaml") +def main(cfg): - if args.freeze_encoder: - for param in speaker_model.encoder.parameters(): - param.requires_grad = False + logging.info(f'Hydra config: {OmegaConf.to_yaml(cfg)}') + trainer = pl.Trainer(**cfg.trainer) + _ = exp_manager(trainer, cfg.get("exp_manager", None)) + speaker_model = EncDecSpeakerLabelModel(cfg=cfg.model, trainer=trainer) + speaker_model.maybe_init_from_pretrained_checkpoint(cfg) + trainer.fit(speaker_model) - finetune_trainer.fit(speaker_model) + if hasattr(cfg.model, 'test_ds') and cfg.model.test_ds.manifest_filepath is not None: + trainer = pl.Trainer(devices=1, accelerator=cfg.trainer.accelerator) + if speaker_model.prepare_test(trainer): + trainer.test(speaker_model) if __name__ == '__main__': diff --git a/nemo/collections/asr/models/label_models.py b/nemo/collections/asr/models/label_models.py index ee49621c3315..aee0873fd56e 100644 --- a/nemo/collections/asr/models/label_models.py +++ b/nemo/collections/asr/models/label_models.py @@ -12,7 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -import copy import itertools from math import ceil from typing import Dict, List, Optional, Union @@ -21,7 +20,6 @@ import numpy as np import torch from omegaconf import DictConfig -from omegaconf.omegaconf import open_dict from pytorch_lightning import Trainer from tqdm import tqdm @@ -355,61 +353,6 @@ def multi_test_epoch_end(self, outputs, dataloader_idx: int = 0): 'test_acc_top_k': topk_scores, } - def setup_finetune_model(self, model_config: DictConfig): - """ - setup_finetune_model method sets up training data, validation data and test data with new - provided config, this checks for the previous labels set up during training from scratch, if None, - it sets up labels for provided finetune data from manifest files - - Args: - model_config: cfg which has train_ds, optional validation_ds, optional test_ds, - mandatory encoder and decoder model params. Make sure you set num_classes correctly for finetune data. - - Returns: - None - """ - logging.info("Setting up data loaders with manifests provided from model_config") - - if 'train_ds' in model_config and model_config.train_ds is not None: - self.setup_training_data(model_config.train_ds) - else: - raise KeyError("train_ds is not found in model_config but you need it for fine tuning") - - if self.labels is None or len(self.labels) == 0: - raise ValueError(f'New labels must be non-empty list of labels. But I got: {self.labels}') - - if 'validation_ds' in model_config and model_config.validation_ds is not None: - self.setup_multiple_validation_data(model_config.validation_ds) - - if 'test_ds' in model_config and model_config.test_ds is not None: - self.setup_multiple_test_data(model_config.test_ds) - - if self.labels is not None: # checking for new finetune dataset labels - logging.warning( - "Trained dataset labels are same as finetune dataset labels -- continuing change of decoder parameters" - ) - else: - logging.warning( - "Either you provided a dummy manifest file during training from scratch or you restored from a pretrained nemo file" - ) - - decoder_config = model_config.decoder - new_decoder_config = copy.deepcopy(decoder_config) - if new_decoder_config['num_classes'] != len(self.labels): - raise ValueError( - "number of classes provided {} is not same as number of different labels in finetuning data: {}".format( - new_decoder_config['num_classes'], len(self.labels) - ) - ) - - del self.decoder - self.decoder = EncDecSpeakerLabelModel.from_config_dict(new_decoder_config) - - with open_dict(self._cfg.decoder): - self._cfg.decoder = new_decoder_config - - logging.info(f"Changed decoder output to # {self.decoder._num_classes} classes.") - @torch.no_grad() def get_embedding(self, path2audio_file): """ diff --git a/nemo/core/classes/modelPT.py b/nemo/core/classes/modelPT.py index 5fb6736b54be..96ea2931e34b 100644 --- a/nemo/core/classes/modelPT.py +++ b/nemo/core/classes/modelPT.py @@ -1105,7 +1105,7 @@ def maybe_init_from_pretrained_checkpoint(self, cfg: OmegaConf, map_location: st restored_model.state_dict(), include, exclude, - f'pretrained chackpoint with name `{model_name}`', + f'pretrained checkpoint with name `{model_name}`', ) del restored_model diff --git a/scripts/speaker_tasks/filelist_to_manifest.py b/scripts/speaker_tasks/filelist_to_manifest.py index 3a6c27d39377..49e4b97bb3e8 100644 --- a/scripts/speaker_tasks/filelist_to_manifest.py +++ b/scripts/speaker_tasks/filelist_to_manifest.py @@ -33,7 +33,6 @@ training, also optionally segment an audio file in to segments of random DURATIONS and create those wav files in CWD. -While creating segments, if audio is not sampled at 16kHz, it resamples to 16kHz and write the wav file. Args: --filelist: path to file containing list of audio files --manifest(optional): if you already have manifest file, but would like to process it for creating @@ -50,7 +49,6 @@ DURATIONS = sorted([1, 2, 3, 4], reverse=True) MIN_ENERGY = 0.01 CWD = os.getcwd() -SAMPLE_RATE = 16000 def filter_manifest_line(manifest_line): @@ -65,7 +63,7 @@ def filter_manifest_line(manifest_line): os.makedirs(os.path.dirname(to_path), exist_ok=True) if dur >= min(DURATIONS): - signal, sr = l.load(audio_path, sr=SAMPLE_RATE) + signal, sr = sf.read(audio_path) remaining_dur = dur - start segments = DURATIONS.copy() diff --git a/tutorials/speaker_tasks/Speaker_Identification_Verification.ipynb b/tutorials/speaker_tasks/Speaker_Identification_Verification.ipynb index 59fa2c7f46b4..00f99db6e243 100644 --- a/tutorials/speaker_tasks/Speaker_Identification_Verification.ipynb +++ b/tutorials/speaker_tasks/Speaker_Identification_Verification.ipynb @@ -1,1267 +1,1228 @@ { - "cells": [ - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "iyLoWDsb9rEs" - }, - "outputs": [], - "source": [ - "\"\"\"\n", - "You can run either this notebook locally (if you have all the dependencies and a GPU) or on Google Colab.\n", - "\n", - "Instructions for setting up Colab are as follows:\n", - "1. Open a new Python 3 notebook.\n", - "2. Import this notebook from GitHub (File -> Upload Notebook -> \"GITHUB\" tab -> copy/paste GitHub URL)\n", - "3. Connect to an instance with a GPU (Runtime -> Change runtime type -> select \"GPU\" for hardware accelerator)\n", - "4. Run this cell to set up dependencies.\n", - "\"\"\"\n", - "# If you're using Google Colab and not running locally, run this cell.\n", - "\n", - "## Install dependencies\n", - "!pip install wget\n", - "!apt-get install sox libsndfile1 ffmpeg\n", - "!pip install unidecode\n", - "\n", - "# ## Install NeMo\n", - "BRANCH = 'main'\n", - "!python -m pip install git+https://github.com/NVIDIA/NeMo.git@$BRANCH#egg=nemo_toolkit[asr]\n", - "\n", - "# Install TorchAudio\n", - "!pip install torchaudio>=0.10.0 -f https://download.pytorch.org/whl/torch_stable.html\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "oDzak_FIB9LS" - }, - "source": [ - "# **SPEAKER RECOGNITION** \n", - "Speaker Recognition (SR) is a broad research area that solves two major tasks: speaker identification (who is speaking?) and\n", - "speaker verification (is the speaker who they claim to be?). In this work, we focus on text-independent speaker recognition when the identity of the speaker is based on how the speech is spoken,\n", - "not necessarily in what is being said. Typically such SR systems operate on unconstrained speech utterances,\n", - "which are converted into vectors of fixed length, called speaker embeddings. Speaker embeddings are also used in\n", - "automatic speech recognition (ASR) and speech synthesis." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "ydqmdcDxCeXb" - }, - "source": [ - "In this tutorial, we shall first train these embeddings on speaker-related datasets, and then get speaker embeddings from a pretrained network for a new dataset. Since Google Colab has very slow read-write speeds, I'll be demonstrating this tutorial using [an4](http://www.speech.cs.cmu.edu/databases/an4/). \n", - "\n", - "Instead, if you'd like to try on a bigger dataset like [hi-mia](https://arxiv.org/abs/1912.01231) use the [get_hi-mia-data.py](https://github.com/NVIDIA/NeMo/blob/stable/scripts/dataset_processing/get_hi-mia_data.py) script to download the necessary files, extract them, also re-sample to 16Khz if any of these samples are not at 16Khz. " - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "vqUBayc_Ctcr" - }, - "outputs": [], - "source": [ - "import os\n", - "NEMO_ROOT = os.getcwd()\n", - "print(NEMO_ROOT)\n", - "import glob\n", - "import subprocess\n", - "import tarfile\n", - "import wget\n", - "\n", - "data_dir = os.path.join(NEMO_ROOT,'data')\n", - "os.makedirs(data_dir, exist_ok=True)\n", - "\n", - "# Download the dataset. This will take a few moments...\n", - "print(\"******\")\n", - "if not os.path.exists(data_dir + '/an4_sphere.tar.gz'):\n", - " an4_url = 'https://dldata-public.s3.us-east-2.amazonaws.com/an4_sphere.tar.gz' # for the original source, please visit http://www.speech.cs.cmu.edu/databases/an4/an4_sphere.tar.gz \n", - " an4_path = wget.download(an4_url, data_dir)\n", - " print(f\"Dataset downloaded at: {an4_path}\")\n", - "else:\n", - " print(\"Tarfile already exists.\")\n", - " an4_path = data_dir + '/an4_sphere.tar.gz'\n", - "\n", - "# Untar and convert .sph to .wav (using sox)\n", - "tar = tarfile.open(an4_path)\n", - "tar.extractall(path=data_dir)\n", - "\n", - "print(\"Converting .sph to .wav...\")\n", - "sph_list = glob.glob(data_dir + '/an4/**/*.sph', recursive=True)\n", - "for sph_path in sph_list:\n", - " wav_path = sph_path[:-4] + '.wav'\n", - " cmd = [\"sox\", sph_path, wav_path]\n", - " subprocess.run(cmd)\n", - "print(\"Finished conversion.\\n******\")" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "t5PrWzkiDbHy" - }, - "source": [ - "Since an4 is not designed for speaker recognition, this facilitates the opportunity to demonstrate how you can generate manifest files that are necessary for training. These methods can be applied to any dataset to get similar training manifest files. \n", - "\n", - "First, create a list file which has all the wav files with absolute paths for each of the train, dev, and test set. This can be easily done by the `find` bash command" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "vnrUh3vuDSRN" - }, - "outputs": [], - "source": [ - "!find {data_dir}/an4/wav/an4_clstk -iname \"*.wav\" > data/an4/wav/an4_clstk/train_all.txt" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "BhWVg2QoDhL3" - }, - "source": [ - "Let's look at the first 3 lines of filelist text file for train." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "BfnMK302Du20" - }, - "outputs": [], - "source": [ - "!head -n 3 {data_dir}/an4/wav/an4_clstk/train_all.txt" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Y9L9Tl0XDw5Z" - }, - "source": [ - "Since we created the list text file for the train, we use `filelist_to_manifest.py` to convert this text file to a manifest file and then optionally split the files to train \\& dev for evaluating the models during training by using the `--split` flag. We wouldn't be needing the `--split` option for the test folder. \n", - "Accordingly please mention the `id` number, which is the field num separated by `/` to be considered as the speaker label " - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "_LYwHAr1G8hp" - }, - "source": [ - "After the download and conversion, your `data` folder should contain directories with manifest files as:\n", - "\n", - "* `data//train.json`\n", - "* `data//dev.json` \n", - "* `data//train_all.json` \n", - "\n", - "Each line in the manifest file describes a training sample - `audio_filepath` contains the path to the wav file, `duration` it's duration in seconds, and `label` is the speaker class label:\n", - "\n", - "`{\"audio_filepath\": \"data/an4/wav/an4test_clstk/menk/cen4-menk-b.wav\", \"duration\": 3.9, \"label\": \"menk\"}` " - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "mpAv77JoD98c" - }, - "outputs": [], - "source": [ - "if not os.path.exists('scripts'):\n", - " print(\"Downloading necessary scripts\")\n", - " !mkdir -p scripts/speaker_tasks\n", - " !wget -P scripts/speaker_tasks/ https://raw.githubusercontent.com/NVIDIA/NeMo/$BRANCH/scripts/speaker_tasks/filelist_to_manifest.py\n", - "!python {NEMO_ROOT}/scripts/speaker_tasks/filelist_to_manifest.py --filelist {data_dir}/an4/wav/an4_clstk/train_all.txt --id -2 --out {data_dir}/an4/wav/an4_clstk/all_manifest.json --split" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "5kPCmx5DHvY5" - }, - "source": [ - "Generate the list text file for the test folder and then convert it to a manifest." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "nMd24GVaFBwr" - }, - "outputs": [], - "source": [ - "!find {data_dir}/an4/wav/an4test_clstk -iname \"*.wav\" > {data_dir}/an4/wav/an4test_clstk/test_all.txt\n", - "!python {NEMO_ROOT}/scripts/speaker_tasks/filelist_to_manifest.py --filelist {data_dir}/an4/wav/an4test_clstk/test_all.txt --id -2 --out {data_dir}/an4/wav/an4test_clstk/test.json" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "H5FPmxUkGakD" - }, - "source": [ - "## Path to manifest files\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "vo-VnYPtJO_v" - }, - "outputs": [], - "source": [ - "train_manifest = os.path.join(data_dir,'an4/wav/an4_clstk/train.json')\n", - "validation_manifest = os.path.join(data_dir,'an4/wav/an4_clstk/dev.json')\n", - "test_manifest = os.path.join(data_dir,'an4/wav/an4_clstk/dev.json')" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "KyDVdtjAL2__" - }, - "source": [ - "As the goal of most speaker-related systems is to get good speaker level embeddings that could help distinguish from\n", - "other speakers, we shall first train these embeddings in end-to-end\n", - "manner optimizing the [TitaNet](https://arxiv.org/pdf/2110.04410.pdf) model.\n", - "We modify the decoder to get these fixed-size embeddings irrespective of the length of the input audio." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "OJtU_GEdMUUo" - }, - "source": [ - "# Training\n", - "Import necessary packages" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "o1ojB0cZMSmv" - }, - "outputs": [], - "source": [ - "import nemo\n", - "# NeMo's ASR collection - this collections contains complete ASR models and\n", - "# building blocks (modules) for ASR\n", - "import nemo.collections.asr as nemo_asr\n", - "from omegaconf import OmegaConf" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "m5Zho11LNAFJ" - }, - "source": [ - "## Model Configuration \n", - "The TitaNet Model is defined in a config file which declares multiple important sections.\n", - "\n", - "They are:\n", - "\n", - "1) model: All arguments that will relate to the Model - preprocessors, encoder, decoder, optimizer and schedulers, datasets, and any other related information\n", - "\n", - "2) trainer: Any argument to be passed to PyTorch Lightning" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "6HQtZfKnMhpI" - }, - "outputs": [], - "source": [ - "# This line will print the entire config of sample SpeakerNet model\n", - "!mkdir conf \n", - "!wget -P conf https://raw.githubusercontent.com/NVIDIA/NeMo/$BRANCH/examples/speaker_tasks/recognition/conf/titanet-large.yaml\n", - "MODEL_CONFIG = os.path.join(NEMO_ROOT,'conf/titanet-large.yaml')\n", - "config = OmegaConf.load(MODEL_CONFIG)\n", - "print(OmegaConf.to_yaml(config))" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "HtbXN-cFOwxi" - }, - "source": [ - "## Setting up the datasets within the config\n", - "If you'll notice, there are a few config dictionaries called train_ds, validation_ds and test_ds. These are configurations used to setup the Dataset and DataLoaders of the corresponding config." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "NPBIf1jmNgjn" - }, - "outputs": [], - "source": [ - "print(OmegaConf.to_yaml(config.model.train_ds))\n", - "print(OmegaConf.to_yaml(config.model.validation_ds))" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "PLIjKOMUP0YE" - }, - "source": [ - "You will often notice that some configs have ??? in place of paths. This is used as a placeholder so that the user can change the value at a later time.\n", - "\n", - "Let's add the paths to the manifests to the config above\n", - "Also, since an4 dataset doesn't have a test set of the same speakers used in training, we will use validation manifest as test manifest for demonstration purposes" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "TSotpjL_O2BN" - }, - "outputs": [], - "source": [ - "config.model.train_ds.manifest_filepath = train_manifest\n", - "config.model.validation_ds.manifest_filepath = validation_manifest" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Note: Since we are training speaker embedding extractor model for verification we do not add test_ds dataset. To include it add it to config and replace manifest file as \n", - "`config.model.test_ds.manifest_filepath = test_manifest`" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "xy6_Lf6fW9aJ" - }, - "source": [ - "Also as we are training on an4 dataset, there are 74 speaker labels in training, and we need to set this in the decoder config" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "-B96tFTnW8Yh" - }, - "outputs": [], - "source": [ - "config.model.decoder.num_classes = 74" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "83pHBRDpQTF0" - }, - "source": [ - "## Building the PyTorch Lightning Trainer\n", - "NeMo models are primarily PyTorch Lightning modules - and therefore are entirely compatible with the PyTorch Lightning ecosystem!\n", - "\n", - "Let us first instantiate a Trainer object!" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "GWzGJoHMQQnG" - }, - "outputs": [], - "source": [ - "import torch\n", - "import pytorch_lightning as pl" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "WIYf4-KFQYHl" - }, - "outputs": [], - "source": [ - "print(\"Trainer config - \\n\")\n", - "print(OmegaConf.to_yaml(config.trainer))" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "aXuSMYMNQeW7" - }, - "outputs": [], - "source": [ - "# Let us modify some trainer configs for this demo\n", - "# Checks if we have GPU available and uses it\n", - "accelerator = 'gpu' if torch.cuda.is_available() else 'cpu'\n", - "config.trainer.devices = 1\n", - "config.trainer.accelerator = accelerator\n", - "\n", - "# Reduces maximum number of epochs to 5 for quick demonstration\n", - "config.trainer.max_epochs = 10\n", - "\n", - "# Remove distributed training flags\n", - "config.trainer.strategy = None\n", - "\n", - "# Remove augmentations\n", - "config.model.train_ds.augmentor=None" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "pBq3eCLwQhCy" - }, - "outputs": [], - "source": [ - "trainer = pl.Trainer(**config.trainer)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "-xHq_rcmQiry" - }, - "source": [ - "## Setting up a NeMo Experiment\n", - "NeMo has an experiment manager that handles logging and checkpointing for us, so let's use it !" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "DMm8MPYfQsCS" - }, - "outputs": [], - "source": [ - "from nemo.utils.exp_manager import exp_manager\n", - "log_dir = exp_manager(trainer, config.get(\"exp_manager\", None))\n", - "# The log_dir provides a path to the current logging directory for easy access\n", - "print(log_dir)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "nQQMlXmLQ7h1" - }, - "source": [ - "## Building the TitaNet Model\n", - "TitaNet is a speaker embedding extractor model that can be used for speaker identification tasks - it generates one label for the entire provided audio stream. Therefore we encapsulate it inside the EncDecSpeakerLabelModel as follows." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "E_KY_s5LROYf" - }, - "outputs": [], - "source": [ - "speaker_model = nemo_asr.models.EncDecSpeakerLabelModel(cfg=config.model, trainer=trainer)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "_AphpMhkSVdU" - }, - "source": [ - "Before we begin training, let us first create a Tensorboard visualization to monitor progress" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "BUnDpe_5SbDR" - }, - "outputs": [], - "source": [ - "try:\n", - " from google import colab\n", - " COLAB_ENV = True\n", - "except (ImportError, ModuleNotFoundError):\n", - " COLAB_ENV = False\n", - "\n", - "# Load the TensorBoard notebook extension\n", - "if COLAB_ENV:\n", - " %load_ext tensorboard\n", - " %tensorboard --logdir {exp_dir}\n", - "else:\n", - " print(\"To use tensorboard, please use this notebook in a Google Colab environment.\")" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Or8g1cksSf8C" - }, - "source": [ - "As any NeMo model is inherently a PyTorch Lightning Model, it can easily be trained in a single line - trainer.fit(model)!\n", - "We see below that the model begins to get modest scores on the validation set after just 5 epochs of training" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "HvYhsOWuSpL_" - }, - "outputs": [], - "source": [ - "trainer.fit(speaker_model)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "lSRACGt3UAYn" - }, - "source": [ - "This config is not suited and designed for an4 so you may observe unstable val_loss" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "jvtVKO8FZsoe" - }, - "source": [ - "If you have a test manifest file, we can easily compute test accuracy by running\n", - "
trainer.test(speaker_model, ckpt_path=None)\n",
-                "
\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "FlBwMsRdZfqg" - }, - "source": [ - "## For Faster Training\n", - "We can dramatically improve the time taken to train this model by using Multi GPU training along with Mixed Precision.\n", - "\n", - "### Trainer with a distributed backend:\n", - "
trainer = Trainer(devices=2, num_nodes=2, accelerator='gpu', strategy='dp')\n",
-                "
\n", - "\n", - "### Mixed precision:\n", - "
trainer = Trainer(amp_level='O1', precision=16)\n",
-                "
\n", - "\n", - "Of course, you can combine these flags as well." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "XcnWub9-0TW2" - }, - "source": [ - "## Saving/Restoring a checkpoint\n", - "There are multiple ways to save and load models in NeMo. Since all NeMo models are inherently Lightning Modules, we can use the standard way that PyTorch Lightning saves and restores models.\n", - "\n", - "NeMo also provides a more advanced model save/restore format, which encapsulates all the parts of the model that are required to restore that model for immediate use.\n", - "\n", - "In this example, we will explore both ways of saving and restoring models, but we will focus on the PyTorch Lightning method.\n", - "\n", - "## Saving and Restoring via PyTorch Lightning Checkpoints\n", - "When using NeMo for training, it is advisable to utilize the exp_manager framework. It is tasked with handling checkpointing and logging (Tensorboard as well as WandB optionally!), as well as dealing with multi-node and multi-GPU logging.\n", - "\n", - "Since we utilized the exp_manager framework above, we have access to the directory where the checkpoints exist.\n", - "\n", - "exp_manager with the default settings will save multiple checkpoints for us -\n", - "\n", - "1) A few checkpoints from certain steps of training. They will have --val_loss= tags\n", - "\n", - "2) A checkpoint at the last epoch of training denotes by --last.\n", - "\n", - "3) If the model finishes training, it will also have a --last checkpoint." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "QSLjq-edaPt_" - }, - "outputs": [], - "source": [ - "# Let us list all the checkpoints we have\n", - "checkpoint_dir = os.path.join(log_dir, 'checkpoints')\n", - "checkpoint_paths = list(glob.glob(os.path.join(checkpoint_dir, \"*.ckpt\")))\n", - "checkpoint_paths" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "BwltdVWXaroa" - }, - "outputs": [], - "source": [ - "final_checkpoint = list(filter(lambda x: \"-last.ckpt\" in x, checkpoint_paths))[0]\n", - "print(final_checkpoint)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "1tGKKojs0fEh" - }, - "source": [ - "\n", - "## Restoring from a PyTorch Lightning checkpoint\n", - "To restore a model using the LightningModule.load_from_checkpoint() class method." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "EgyP9cYVbFc8" - }, - "outputs": [], - "source": [ - "restored_model = nemo_asr.models.EncDecSpeakerLabelModel.load_from_checkpoint(final_checkpoint)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "AnZVMKZpbI_M" - }, - "source": [ - "# Finetuning\n", - "Since we don't have any new manifest file to finetune, I will demonstrate here by using the test manifest file we created earlier. \n", - "an4 test dataset has a different set of speakers from the train set (total number: 10). And as we didn't split this dataset for validation I will use the same for validation. " - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "kV9gInFwQ2F5" - }, - "source": [ - "So to finetune all we need to do is, update our model config with these manifest paths and change num of decoder classes to create a new decoder with an updated number of classes" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "HtXUWmYLQ0PJ" - }, - "outputs": [], - "source": [ - "test_manifest = os.path.join(data_dir,'an4/wav/an4test_clstk/test.json')\n", - "config.model.train_ds.manifest_filepath = test_manifest\n", - "config.model.validation_ds.manifest_filepath = test_manifest\n", - "config.model.decoder.num_classes = 10" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "xpSQ_sk8Rf6z" - }, - "source": [ - "Once you set up the necessary model config parameters all we need to do is call setup_finetune_model method" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "Jt3yy4EVS-S6" - }, - "outputs": [], - "source": [ - "restored_model.setup_finetune_model(config.model)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "IHy1zE1cTDZn" - }, - "source": [ - "So we have set up the data and changed the decoder required for finetune, we now just need to create a trainer and start training with a smaller learning rate for fewer epochs" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "nBmF6tQITSRl" - }, - "outputs": [], - "source": [ - "# Setup the new trainer object\n", - "# Let us modify some trainer configs for this demo\n", - "# Checks if we have GPU available and uses it\n", - "accelerator = 'gpu' if torch.cuda.is_available() else 'cpu'\n", - "\n", - "trainer_config = OmegaConf.create(dict(\n", - " devices=1,\n", - " accelerator=accelerator,\n", - " max_epochs=5,\n", - " max_steps=None, # computed at runtime if not set\n", - " num_nodes=1,\n", - " accumulate_grad_batches=1,\n", - " enable_checkpointing=False, # Provided by exp_manager\n", - " logger=False, # Provided by exp_manager\n", - " log_every_n_steps=1, # Interval of logging.\n", - " val_check_interval=1.0, # Set to 0.25 to check 4 times per epoch, or an int for number of iterations\n", - "))\n", - "print(OmegaConf.to_yaml(trainer_config))" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "bRz-8-xzUHKZ" - }, - "outputs": [], - "source": [ - "trainer_finetune = pl.Trainer(**trainer_config)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "EOwHTkW-UUy8" - }, - "source": [ - "## Setting the trainer to the restored model\n", - "Setting the trainer to the restored model" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "0FhYQQQOUPIk" - }, - "outputs": [], - "source": [ - "restored_model.set_trainer(trainer_finetune)\n", - "log_dir_finetune = exp_manager(trainer_finetune, config.get(\"exp_manager\", None))\n", - "print(log_dir_finetune)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "ptexCJ7tUmgs" - }, - "source": [ - "## Setup optimizer + scheduler\n", - "For a fine-tuning experiment, let us set up the optimizer and scheduler!\n", - "We will use a much lower learning rate than before\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "TUyjEAeSUjf2" - }, - "outputs": [], - "source": [ - "import copy\n", - "optim_sched_cfg = copy.deepcopy(restored_model._cfg.optim)\n", - "# Struct mode prevents us from popping off elements from the config, so let us disable it\n", - "OmegaConf.set_struct(optim_sched_cfg, False)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "5JViMr7pUzvi" - }, - "outputs": [], - "source": [ - "# Let us change the maximum learning rate to previous minimum learning rate\n", - "optim_sched_cfg.lr = 0.001\n", - "\n", - "# Set \"min_lr\" to lower value\n", - "optim_sched_cfg.sched.min_lr = 1e-4\n", - "\n", - "print(OmegaConf.to_yaml(optim_sched_cfg))" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "AjqdCggzVFrY" - }, - "outputs": [], - "source": [ - "# Now let us update the optimizer settings\n", - "restored_model.setup_optimization(optim_sched_cfg)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "3mWlJZiOVIuO" - }, - "outputs": [], - "source": [ - "# We can also just directly replace the config inplace if we choose to\n", - "restored_model._cfg.optim = optim_sched_cfg" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "lc3fzGYVVTyi" - }, - "source": [ - "## Fine-tune training step\n", - "We fine-tune on the subset recognition problem. Note, the model was originally trained on these classes (the subset defined here has already been trained on above).\n", - "\n", - "When fine-tuning on a truly new dataset, we will not see such a dramatic improvement in performance. However, it should still converge a little faster than if it was trained from scratch." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "uFIOsuFYVLzr" - }, - "outputs": [], - "source": [ - "## Fine-tuning for 5 epochs¶\n", - "trainer_finetune.fit(restored_model)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "5DNidtl4VplU" - }, - "source": [ - "# Saving .nemo file\n", - "Now we can save the whole config and model parameters in a single .nemo and we can anytime restore from it" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "am5wej6-VdZW" - }, - "outputs": [], - "source": [ - "restored_model.save_to(os.path.join(log_dir_finetune, '..',\"titanet-large.nemo\"))" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "WnBhFJefV-Pf" - }, - "outputs": [], - "source": [ - "!ls {log_dir_finetune}/.." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "kVx1hNP_V_iz" - }, - "outputs": [], - "source": [ - "# restore from a save model\n", - "restored_model_2 = nemo_asr.models.EncDecSpeakerLabelModel.restore_from(os.path.join(log_dir_finetune, '..', \"titanet-large.nemo\"))\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "80tLWTN40uaB" - }, - "source": [ - "# Speaker Verification" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "VciRUIRz0y6P" - }, - "source": [ - "Training for a speaker verification model is almost the same as the speaker recognition model with a change in the loss function. Angular Loss is a better function to train for a speaker verification model as the model is trained in an end to end manner with loss optimizing for embeddings cluster to be far from each other for different speaker by maximizing the angle between these clusters" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "ULTjBuFI19Js" - }, - "source": [ - "To train for verification we just need to toggle `angular` flag in `config.model.decoder.params.angular = True` else set it to `False` to train with cross-entropy loss for identification purposes. \n", - "Once we set this, the loss will be changed to angular loss and we can follow the above steps to the model.\n", - "Note the scale and margin values to be set for the loss function are present at `config.model.loss.scale` and `config.model.loss.margin`" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "LcKiNEY032-t" - }, - "source": [ - "## Extract Speaker Embeddings\n", - "Once you have a trained model or use one of our pretrained nemo checkpoints to get speaker embeddings for any speaker.\n", - "\n", - "To demonstrate this we shall use `nemo_asr.models.EncDecSpeakerLabelModel` with say 5 audio_samples from our dev manifest set. This model is specifically for inference purposes to extract embeddings from a trained `.nemo` model" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "uXEzKMHf3r6-" - }, - "outputs": [], - "source": [ - "verification_model = nemo_asr.models.EncDecSpeakerLabelModel.restore_from(os.path.join(log_dir_finetune, '..', 'titanet-large.nemo'))" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Y-XiLHMQ8BIk" - }, - "source": [ - "Now, we need to pass the necessary manifest_filepath and params to set up the data loader for extracting embeddings" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "lk2vsDJk9PS8" - }, - "outputs": [], - "source": [ - "!head -5 {validation_manifest} > embeddings_manifest.json" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "DEd5poCr9yrP" - }, - "outputs": [], - "source": [ - "config.model.train_ds" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from nemo.collections.asr.parts.utils.speaker_utils import embedding_normalize\n", - "from tqdm import tqdm\n", - "try:\n", - " from torch.cuda.amp import autocast\n", - "except ImportError:\n", - " from contextlib import contextmanager\n", - "\n", - " @contextmanager\n", - " def autocast(enabled=None):\n", - " yield\n", - "import numpy as np\n", - "import json\n", - "import pickle as pkl" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "JIHok6LD8g0F" - }, - "outputs": [], - "source": [ - "def get_embeddings(speaker_model, manifest_file, batch_size=1, embedding_dir='./', device='cuda'):\n", - " test_config = OmegaConf.create(\n", - " dict(\n", - " manifest_filepath=manifest_file,\n", - " sample_rate=16000,\n", - " labels=None,\n", - " batch_size=batch_size,\n", - " shuffle=False,\n", - " time_length=20,\n", - " )\n", - " )\n", - "\n", - " speaker_model.setup_test_data(test_config)\n", - " speaker_model = speaker_model.to(device)\n", - " speaker_model.eval()\n", - "\n", - " all_embs=[]\n", - " out_embeddings = {}\n", - " \n", - " for test_batch in tqdm(speaker_model.test_dataloader()):\n", - " test_batch = [x.to(device) for x in test_batch]\n", - " audio_signal, audio_signal_len, labels, slices = test_batch\n", - " with autocast():\n", - " _, embs = speaker_model.forward(input_signal=audio_signal, input_signal_length=audio_signal_len)\n", - " emb_shape = embs.shape[-1]\n", - " embs = embs.view(-1, emb_shape)\n", - " all_embs.extend(embs.cpu().detach().numpy())\n", - " del test_batch\n", - "\n", - " all_embs = np.asarray(all_embs)\n", - " all_embs = embedding_normalize(all_embs)\n", - " with open(manifest_file, 'r') as manifest:\n", - " for i, line in enumerate(manifest.readlines()):\n", - " line = line.strip()\n", - " dic = json.loads(line)\n", - " uniq_name = '@'.join(dic['audio_filepath'].split('/')[-3:])\n", - " out_embeddings[uniq_name] = all_embs[i]\n", - "\n", - " embedding_dir = os.path.join(embedding_dir, 'embeddings')\n", - " if not os.path.exists(embedding_dir):\n", - " os.makedirs(embedding_dir, exist_ok=True)\n", - "\n", - " prefix = manifest_file.split('/')[-1].rsplit('.', 1)[-2]\n", - "\n", - " name = os.path.join(embedding_dir, prefix)\n", - " embeddings_file = name + '_embeddings.pkl'\n", - " pkl.dump(out_embeddings, open(embeddings_file, 'wb'))\n", - " print(\"Saved embedding files to {}\".format(embedding_dir))" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "u2FRecqD-ln5" - }, - "outputs": [], - "source": [ - "manifest_filepath = os.path.join(NEMO_ROOT,'embeddings_manifest.json')\n", - "device = 'cuda' if torch.cuda.is_available() else 'cpu'\n", - "get_embeddings(verification_model, manifest_filepath, batch_size=64,embedding_dir='./', device=device)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "zfjXPsjzDOgr" - }, - "source": [ - "Embeddings are stored in dict structure with key-value pair, key being uniq_name generated based on audio_filepath of the sample present in manifest_file in `embedding_dir`" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "hmTeSR6jD28k" - }, - "outputs": [], - "source": [ - "ls ./embeddings/" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] - } - ], - "metadata": { - "accelerator": "GPU", - "colab": { - "collapsed_sections": [], - "name": "Speaker_Recogniton_Verification.ipynb", - "provenance": [], - "toc_visible": true - }, - "kernelspec": { - "display_name": "Python 3 (ipykernel)", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.8.12" - } - }, - "nbformat": 4, - "nbformat_minor": 1 + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": {}, + "colab_type": "code", + "id": "iyLoWDsb9rEs" + }, + "outputs": [], + "source": [ + "\"\"\"\n", + "You can run either this notebook locally (if you have all the dependencies and a GPU) or on Google Colab.\n", + "\n", + "Instructions for setting up Colab are as follows:\n", + "1. Open a new Python 3 notebook.\n", + "2. Import this notebook from GitHub (File -> Upload Notebook -> \"GITHUB\" tab -> copy/paste GitHub URL)\n", + "3. Connect to an instance with a GPU (Runtime -> Change runtime type -> select \"GPU\" for hardware accelerator)\n", + "4. Run this cell to set up dependencies.\n", + "\"\"\"\n", + "# If you're using Google Colab and not running locally, run this cell.\n", + "\n", + "# Install dependencies\n", + "!pip install wget\n", + "!apt-get install sox libsndfile1 ffmpeg\n", + "!pip install unidecode\n", + "\n", + "## Install NeMo\n", + "BRANCH = 'main'\n", + "!python -m pip install git+https://github.com/NVIDIA/NeMo.git@$BRANCH#egg=nemo_toolkit[asr]\n", + "\n", + "# Install TorchAudio\n", + "!pip install torchaudio>=0.10.0 -f https://download.pytorch.org/whl/torch_stable.html\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "oDzak_FIB9LS" + }, + "source": [ + "# **SPEAKER RECOGNITION** \n", + "Speaker Recognition (SR) is a broad research area that solves two major tasks: speaker identification (who is speaking?) and\n", + "speaker verification (is the speaker who they claim to be?). In this work, we focus on text-independent speaker recognition when the identity of the speaker is based on how the speech is spoken,\n", + "not necessarily in what is being said. Typically such SR systems operate on unconstrained speech utterances,\n", + "which are converted into fixed-length vectors, called speaker embeddings. Speaker embeddings are also used in\n", + "automatic speech recognition (ASR) and speech synthesis." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "ydqmdcDxCeXb" + }, + "source": [ + "In this tutorial, we shall first train these embeddings on speaker-related datasets, and then get speaker embeddings from a pretrained network for a new dataset. Since Google Colab has very slow read-write speeds, I'll be demonstrating this tutorial using [an4](http://www.speech.cs.cmu.edu/databases/an4/). \n", + "\n", + "Instead, if you'd like to try on a bigger dataset like [hi-mia](https://arxiv.org/abs/1912.01231) use the [get_hi-mia-data.py](https://github.com/NVIDIA/NeMo/blob/stable/scripts/dataset_processing/get_hi-mia_data.py) script to download the necessary files, extract them, and resample to 16Khz if any of these samples are not at 16Khz. " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": {}, + "colab_type": "code", + "id": "vqUBayc_Ctcr" + }, + "outputs": [], + "source": [ + "import os\n", + "NEMO_ROOT = os.getcwd()\n", + "print(NEMO_ROOT)\n", + "import glob\n", + "import subprocess\n", + "import tarfile\n", + "import wget\n", + "\n", + "data_dir = os.path.join(NEMO_ROOT,'data')\n", + "os.makedirs(data_dir, exist_ok=True)\n", + "\n", + "# Download the dataset. This will take a few moments...\n", + "print(\"******\")\n", + "if not os.path.exists(data_dir + '/an4_sphere.tar.gz'):\n", + " an4_url = 'https://dldata-public.s3.us-east-2.amazonaws.com/an4_sphere.tar.gz' # for the original source, please visit http://www.speech.cs.cmu.edu/databases/an4/an4_sphere.tar.gz \n", + " an4_path = wget.download(an4_url, data_dir)\n", + " print(f\"Dataset downloaded at: {an4_path}\")\n", + "else:\n", + " print(\"Tarfile already exists.\")\n", + " an4_path = data_dir + '/an4_sphere.tar.gz'\n", + "\n", + "# Untar and convert .sph to .wav (using sox)\n", + "tar = tarfile.open(an4_path)\n", + "tar.extractall(path=data_dir)\n", + "\n", + "print(\"Converting .sph to .wav...\")\n", + "sph_list = glob.glob(data_dir + '/an4/**/*.sph', recursive=True)\n", + "for sph_path in sph_list:\n", + " wav_path = sph_path[:-4] + '.wav'\n", + " cmd = [\"sox\", sph_path, wav_path]\n", + " subprocess.run(cmd)\n", + "print(\"Finished conversion.\\n******\")" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "t5PrWzkiDbHy" + }, + "source": [ + "Since an4 is not designed for speaker recognition, this facilitates the opportunity to demonstrate how you can generate manifest files that are necessary for training. These methods can be applied to any dataset to get similar training manifest files. \n", + "\n", + "First, create a list file which has all the wav files with absolute paths for each of the train, dev, and test set. This can be easily done by the `find` bash command" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": {}, + "colab_type": "code", + "id": "vnrUh3vuDSRN" + }, + "outputs": [], + "source": [ + "!find {data_dir}/an4/wav/an4_clstk -iname \"*.wav\" > data/an4/wav/an4_clstk/train_all.txt" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "BhWVg2QoDhL3" + }, + "source": [ + "Let's look at the first 3 lines of filelist text file for train." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": {}, + "colab_type": "code", + "id": "BfnMK302Du20" + }, + "outputs": [], + "source": [ + "!head -n 3 {data_dir}/an4/wav/an4_clstk/train_all.txt" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "Y9L9Tl0XDw5Z" + }, + "source": [ + "Since we created the list text file for the train, we use `filelist_to_manifest.py` to convert this text file to a manifest file and then optionally split the files to train \\& dev for evaluating the models during training by using the `--split` flag. We wouldn't be needing the `--split` option for the test folder. \n", + "Accordingly please mention the `id` number, which is the field num separated by `/` to be considered as the speaker label " + ] + }, + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "_LYwHAr1G8hp" + }, + "source": [ + "After the download and conversion, your `data` folder should contain directories with manifest files as:\n", + "\n", + "* `data//train.json`\n", + "* `data//dev.json` \n", + "* `data//train_all.json` \n", + "\n", + "Each line in the manifest file describes a training sample - `audio_filepath` contains the path to the wav file, `duration` it's duration in seconds, and `label` is the speaker class label:\n", + "\n", + "`{\"audio_filepath\": \"data/an4/wav/an4test_clstk/menk/cen4-menk-b.wav\", \"duration\": 3.9, \"label\": \"menk\"}` " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": {}, + "colab_type": "code", + "id": "mpAv77JoD98c" + }, + "outputs": [], + "source": [ + "if not os.path.exists('scripts'):\n", + " print(\"Downloading necessary scripts\")\n", + " !mkdir -p scripts/speaker_tasks\n", + " !wget -P scripts/speaker_tasks/ https://raw.githubusercontent.com/NVIDIA/NeMo/$BRANCH/scripts/speaker_tasks/filelist_to_manifest.py\n", + "!python {NEMO_ROOT}/scripts/speaker_tasks/filelist_to_manifest.py --filelist {data_dir}/an4/wav/an4_clstk/train_all.txt --id -2 --out {data_dir}/an4/wav/an4_clstk/all_manifest.json --split" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "5kPCmx5DHvY5" + }, + "source": [ + "Generate the list text file for the test folder and then convert it to a manifest." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": {}, + "colab_type": "code", + "id": "nMd24GVaFBwr" + }, + "outputs": [], + "source": [ + "!find {data_dir}/an4/wav/an4test_clstk -iname \"*.wav\" > {data_dir}/an4/wav/an4test_clstk/test_all.txt\n", + "!python {NEMO_ROOT}/scripts/speaker_tasks/filelist_to_manifest.py --filelist {data_dir}/an4/wav/an4test_clstk/test_all.txt --id -2 --out {data_dir}/an4/wav/an4test_clstk/test.json" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "H5FPmxUkGakD" + }, + "source": [ + "## Path to manifest files\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": {}, + "colab_type": "code", + "id": "vo-VnYPtJO_v" + }, + "outputs": [], + "source": [ + "train_manifest = os.path.join(data_dir,'an4/wav/an4_clstk/train.json')\n", + "validation_manifest = os.path.join(data_dir,'an4/wav/an4_clstk/dev.json')\n", + "test_manifest = os.path.join(data_dir,'an4/wav/an4_clstk/dev.json')" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "KyDVdtjAL2__" + }, + "source": [ + "As the goal of most speaker-related systems is to get good speaker level embeddings that could help distinguish from\n", + "other speakers, we shall first train these embeddings in an end-to-end\n", + "manner optimizing the [TitaNet](https://arxiv.org/pdf/2110.04410.pdf) model.\n", + "We modify the decoder to get these fixed-size embeddings irrespective of the length of the input audio." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "OJtU_GEdMUUo" + }, + "source": [ + "# Training\n", + "Import necessary packages" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Note: All the following steps are just for explanation of each section, but one can use the provided [training script](https://github.com/NVIDIA/NeMo/blob/main/examples/speaker_tasks/recognition/speaker_reco.py) to launch training in the command line." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": {}, + "colab_type": "code", + "id": "o1ojB0cZMSmv" + }, + "outputs": [], + "source": [ + "import nemo\n", + "# NeMo's ASR collection - This collection contains complete ASR models and\n", + "# building blocks (modules) for ASR\n", + "import nemo.collections.asr as nemo_asr\n", + "from omegaconf import OmegaConf" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "m5Zho11LNAFJ" + }, + "source": [ + "## Model Configuration \n", + "The TitaNet model is defined in a config file which declares multiple important sections.\n", + "\n", + "They are:\n", + "\n", + "1) model: All arguments that will relate to the Model - preprocessors, encoder, decoder, optimizer and schedulers, datasets, and any other related information\n", + "\n", + "2) trainer: Any argument to be passed to PyTorch Lightning" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": {}, + "colab_type": "code", + "id": "6HQtZfKnMhpI" + }, + "outputs": [], + "source": [ + "# This line will print the entire config of sample TitaNet model\n", + "!mkdir conf \n", + "!wget -P conf https://raw.githubusercontent.com/NVIDIA/NeMo/$BRANCH/examples/speaker_tasks/recognition/conf/titanet-large.yaml\n", + "MODEL_CONFIG = os.path.join(NEMO_ROOT,'conf/titanet-large.yaml')\n", + "config = OmegaConf.load(MODEL_CONFIG)\n", + "print(OmegaConf.to_yaml(config))" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "HtbXN-cFOwxi" + }, + "source": [ + "## Setting up the datasets within the config\n", + "If you'll notice, there are a few config dictionaries called train_ds, validation_ds and test_ds. These are configurations used to setup the Dataset and DataLoaders of the corresponding config." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": {}, + "colab_type": "code", + "id": "NPBIf1jmNgjn" + }, + "outputs": [], + "source": [ + "print(OmegaConf.to_yaml(config.model.train_ds))\n", + "print(OmegaConf.to_yaml(config.model.validation_ds))" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "PLIjKOMUP0YE" + }, + "source": [ + "You will often notice that some configs have ??? in place of paths. This is used as a placeholder so that the user can change the value at a later time.\n", + "\n", + "Let's add the paths to the manifests to the config above\n", + "Also, since an4 dataset doesn't have a test set of the same speakers used in training, we will use validation manifest as test manifest for demonstration purposes" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": {}, + "colab_type": "code", + "id": "TSotpjL_O2BN" + }, + "outputs": [], + "source": [ + "config.model.train_ds.manifest_filepath = train_manifest\n", + "config.model.validation_ds.manifest_filepath = validation_manifest" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Note: Since we are training speaker embedding extractor model for verification we do not add test_ds dataset. To include it add it to config and replace manifest file as \n", + "`config.model.test_ds.manifest_filepath = test_manifest`" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "xy6_Lf6fW9aJ" + }, + "source": [ + "Also as we are training on an4 dataset, there are 74 speaker labels in training, and we need to set this in the decoder config" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": {}, + "colab_type": "code", + "id": "-B96tFTnW8Yh" + }, + "outputs": [], + "source": [ + "config.model.decoder.num_classes = 74" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "83pHBRDpQTF0" + }, + "source": [ + "## Building the PyTorch Lightning Trainer\n", + "NeMo models are primarily PyTorch Lightning modules - and therefore are entirely compatible with the PyTorch Lightning ecosystem!\n", + "\n", + "Let us first instantiate a Trainer object!" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": {}, + "colab_type": "code", + "id": "GWzGJoHMQQnG" + }, + "outputs": [], + "source": [ + "import torch\n", + "import pytorch_lightning as pl" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": {}, + "colab_type": "code", + "id": "WIYf4-KFQYHl" + }, + "outputs": [], + "source": [ + "print(\"Trainer config - \\n\")\n", + "print(OmegaConf.to_yaml(config.trainer))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": {}, + "colab_type": "code", + "id": "aXuSMYMNQeW7" + }, + "outputs": [], + "source": [ + "# Let us modify some trainer configs for this demo\n", + "# Checks if we have GPU available and uses it\n", + "accelerator = 'gpu' if torch.cuda.is_available() else 'cpu'\n", + "config.trainer.devices = 1\n", + "config.trainer.accelerator = accelerator\n", + "\n", + "# Reduces maximum number of epochs to 5 for quick demonstration\n", + "config.trainer.max_epochs = 10\n", + "\n", + "# Remove distributed training flags\n", + "config.trainer.strategy = None\n", + "\n", + "# Remove augmentations\n", + "config.model.train_ds.augmentor=None" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": {}, + "colab_type": "code", + "id": "pBq3eCLwQhCy" + }, + "outputs": [], + "source": [ + "trainer = pl.Trainer(**config.trainer)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "-xHq_rcmQiry" + }, + "source": [ + "## Setting up a NeMo Experiment\n", + "NeMo has an experiment manager that handles logging and checkpointing for us, so let's use it !" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": {}, + "colab_type": "code", + "id": "DMm8MPYfQsCS" + }, + "outputs": [], + "source": [ + "from nemo.utils.exp_manager import exp_manager\n", + "log_dir = exp_manager(trainer, config.get(\"exp_manager\", None))\n", + "# The log_dir provides a path to the current logging directory for easy access\n", + "print(log_dir)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "nQQMlXmLQ7h1" + }, + "source": [ + "## Building the TitaNet Model\n", + "TitaNet is a speaker embedding extractor model that can be used for speaker identification tasks - it generates one label for the entire provided audio stream. Therefore we encapsulate it inside the EncDecSpeakerLabelModel as follows." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": {}, + "colab_type": "code", + "id": "E_KY_s5LROYf" + }, + "outputs": [], + "source": [ + "speaker_model = nemo_asr.models.EncDecSpeakerLabelModel(cfg=config.model, trainer=trainer)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "_AphpMhkSVdU" + }, + "source": [ + "Before we begin training, let us first create a Tensorboard visualization to monitor progress" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": {}, + "colab_type": "code", + "id": "BUnDpe_5SbDR" + }, + "outputs": [], + "source": [ + "try:\n", + " from google import colab\n", + " COLAB_ENV = True\n", + "except (ImportError, ModuleNotFoundError):\n", + " COLAB_ENV = False\n", + "\n", + "# Load the TensorBoard notebook extension\n", + "if COLAB_ENV:\n", + " %load_ext tensorboard\n", + " %tensorboard --logdir {exp_dir}\n", + "else:\n", + " print(\"To use tensorboard, please use this notebook in a Google Colab environment.\")" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "Or8g1cksSf8C" + }, + "source": [ + "As any NeMo model is inherently a PyTorch Lightning Model, it can easily be trained in a single line - trainer.fit(model)!\n", + "Below we see that the model begins to get modest scores on the validation set after just 5 epochs of training" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": {}, + "colab_type": "code", + "id": "HvYhsOWuSpL_" + }, + "outputs": [], + "source": [ + "trainer.fit(speaker_model)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "lSRACGt3UAYn" + }, + "source": [ + "This config is not suited and designed for an4 so you may observe unstable val_loss" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "jvtVKO8FZsoe" + }, + "source": [ + "If you have a test manifest file, we can easily compute test accuracy by running\n", + "
trainer.test(speaker_model, ckpt_path=None)\n",
+    "
\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "FlBwMsRdZfqg" + }, + "source": [ + "## For Faster Training\n", + "We can dramatically improve the time taken to train this model by using Multi GPU training along with Mixed Precision.\n", + "\n", + "### Trainer with a distributed backend:\n", + "
trainer = Trainer(devices=2, num_nodes=2, accelerator='gpu', strategy='dp')\n",
+    "
\n", + "\n", + "### Mixed precision:\n", + "
trainer = Trainer(amp_level='O1', precision=16)\n",
+    "
\n", + "\n", + "Of course, you can combine these flags as well." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "XcnWub9-0TW2" + }, + "source": [ + "## Saving/Restoring a checkpoint\n", + "There are multiple ways to save and load models in NeMo. Since all NeMo models are inherently Lightning Modules, we can use the standard way that PyTorch Lightning saves and restores models.\n", + "\n", + "NeMo also provides a more advanced model save/restore format, which encapsulates all the parts of the model that are required to restore that model for immediate use.\n", + "\n", + "In this example, we will explore both ways of saving and restoring models, but we will focus on the PyTorch Lightning method.\n", + "\n", + "## Saving and Restoring via PyTorch Lightning Checkpoints\n", + "When using NeMo for training, it is advisable to utilize the exp_manager framework. It is tasked with handling checkpointing and logging (Tensorboard as well as WandB optionally!), as well as dealing with multi-node and multi-GPU logging.\n", + "\n", + "Since we utilized the exp_manager framework above, we have access to the directory where the checkpoints exist.\n", + "\n", + "exp_manager with the default settings will save multiple checkpoints for us -\n", + "\n", + "1) A few checkpoints from certain steps of training. They will have --val_loss= tags\n", + "\n", + "2) Checkpoints at the last epoch of training are denoted by --last.\n", + "\n", + "3) If the model finishes training, it will also have a --last checkpoint." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": {}, + "colab_type": "code", + "id": "QSLjq-edaPt_" + }, + "outputs": [], + "source": [ + "# Let us list all the checkpoints we have\n", + "checkpoint_dir = os.path.join(log_dir, 'checkpoints')\n", + "checkpoint_paths = list(glob.glob(os.path.join(checkpoint_dir, \"*.ckpt\")))\n", + "checkpoint_paths" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": {}, + "colab_type": "code", + "id": "BwltdVWXaroa" + }, + "outputs": [], + "source": [ + "final_checkpoint = list(filter(lambda x: \"-last.ckpt\" in x, checkpoint_paths))[0]\n", + "print(final_checkpoint)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "1tGKKojs0fEh" + }, + "source": [ + "\n", + "## Restoring from a PyTorch Lightning checkpoint\n", + "To restore a model using the LightningModule.load_from_checkpoint() class method." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": {}, + "colab_type": "code", + "id": "EgyP9cYVbFc8" + }, + "outputs": [], + "source": [ + "restored_model = nemo_asr.models.EncDecSpeakerLabelModel.load_from_checkpoint(final_checkpoint)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "AnZVMKZpbI_M" + }, + "source": [ + "# Finetuning\n", + "Since we don't have any new manifest file to finetune, I will demonstrate here by using the test manifest file we created earlier. \n", + "an4 test dataset has a different set of speakers from the train set (total number: 10). And as we didn't split this dataset for validation I will use the same for validation. " + ] + }, + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "kV9gInFwQ2F5" + }, + "source": [ + "There are a couple of ways we can finetune a speaker recognition model. \n", + "1. Finetuning using a pretrained model published on NGC. \n", + "2. Finetuning from a PTL checkpoint. \n", + "\n", + "Since finetuning from a large pretrained model is more common, I shall use it to demonstrate finetuning procedure. In order to make finetuning step independent from training from scratch, we use another config. Here we shall use `titanet-finetune.yaml` config, that is created to show finetuning on pretrained titanet-large model. " + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Note: You may use [finetune-script](https://github.com/NVIDIA/NeMo/blob/main/examples/speaker_tasks/recognition/speaker_reco_finetune.py) to launch training in the command line. Following is just a demonstration of the script" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# !wget -P conf https://raw.githubusercontent.com/NVIDIA/NeMo/$BRANCH/examples/speaker_tasks/recognition/conf/titanet-finetune.yaml\n", + "MODEL_CONFIG = os.path.join(NEMO_ROOT,'conf/titanet-finetune.yaml')\n", + "finetune_config = OmegaConf.load(MODEL_CONFIG)\n", + "print(OmegaConf.to_yaml(finetune_config))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "For step 2, if one would like to finetune from a PTL checkpoint, `init_from_pretrained_model` in config should be replaced with `init_from_nemo_model` and need to provide the path to checkpoint. " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": {}, + "colab_type": "code", + "id": "HtXUWmYLQ0PJ" + }, + "outputs": [], + "source": [ + "test_manifest = os.path.join(data_dir,'an4/wav/an4test_clstk/test.json')\n", + "finetune_config.model.train_ds.manifest_filepath = test_manifest\n", + "finetune_config.model.validation_ds.manifest_filepath = test_manifest\n", + "finetune_config.model.decoder.num_classes = 10" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "IHy1zE1cTDZn" + }, + "source": [ + "So we have set up the data and changed the decoder required for finetune, we now just need to create a trainer and start training with a smaller learning rate for fewer epochs" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": {}, + "colab_type": "code", + "id": "nBmF6tQITSRl" + }, + "outputs": [], + "source": [ + "# Setup the new trainer object\n", + "# Let us modify some trainer configs for this demo\n", + "# Checks if we have GPU available and uses it\n", + "accelerator = 'gpu' if torch.cuda.is_available() else 'cpu'\n", + "\n", + "trainer_config = OmegaConf.create(dict(\n", + " devices=1,\n", + " accelerator=accelerator,\n", + " max_epochs=5,\n", + " max_steps=None, # computed at runtime if not set\n", + " num_nodes=1,\n", + " accumulate_grad_batches=1,\n", + " enable_checkpointing=False, # Provided by exp_manager\n", + " logger=False, # Provided by exp_manager\n", + " log_every_n_steps=1, # Interval of logging.\n", + " val_check_interval=1.0, # Set to 0.25 to check 4 times per epoch, or an int for number of iterations\n", + "))\n", + "print(OmegaConf.to_yaml(trainer_config))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": {}, + "colab_type": "code", + "id": "bRz-8-xzUHKZ" + }, + "outputs": [], + "source": [ + "trainer_finetune = pl.Trainer(**trainer_config)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "EOwHTkW-UUy8" + }, + "source": [ + "## Setting the trainer to the restored model\n", + "Setting the trainer to the restored model" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": {}, + "colab_type": "code", + "id": "0FhYQQQOUPIk" + }, + "outputs": [], + "source": [ + "log_dir_finetune = exp_manager(trainer_finetune, config.get(\"exp_manager\", None))\n", + "print(log_dir_finetune)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "lc3fzGYVVTyi" + }, + "source": [ + "## Fine-tune training step\n", + "\n", + "When fine-tuning on a truly new dataset, we will not see such a dramatic improvement in performance. However, it should still converge a little faster than if it was trained from scratch." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "speaker_model = nemo_asr.models.EncDecSpeakerLabelModel(cfg=finetune_config.model, trainer=trainer_finetune)\n", + "speaker_model.maybe_init_from_pretrained_checkpoint(finetune_config)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "In the config, we keep weights of preprocessor and encoder, and attach a new decoder as mentioned in above section to match num of classes of new data" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": {}, + "colab_type": "code", + "id": "uFIOsuFYVLzr" + }, + "outputs": [], + "source": [ + "## Fine-tuning for 5 epochs¶\n", + "trainer_finetune.fit(speaker_model)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Tip: Add more data augmentation and dropout while finetuning on your data" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "5DNidtl4VplU" + }, + "source": [ + "# Saving .nemo file\n", + "Now we can save the whole config and model parameters in a single .nemo and we can anytime restore from it" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": {}, + "colab_type": "code", + "id": "am5wej6-VdZW" + }, + "outputs": [], + "source": [ + "restored_model.save_to(os.path.join(log_dir_finetune, '..',\"titanet-large-finetune.nemo\"))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": {}, + "colab_type": "code", + "id": "WnBhFJefV-Pf" + }, + "outputs": [], + "source": [ + "!ls {log_dir_finetune}/.." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": {}, + "colab_type": "code", + "id": "kVx1hNP_V_iz" + }, + "outputs": [], + "source": [ + "# restore from a save model\n", + "restored_model = nemo_asr.models.EncDecSpeakerLabelModel.restore_from(os.path.join(log_dir_finetune, '..', \"titanet-large-finetune.nemo\"))\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "80tLWTN40uaB" + }, + "source": [ + "# Speaker Verification" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "VciRUIRz0y6P" + }, + "source": [ + "Training for a speaker verification model is almost the same as the speaker recognition model with a change in the loss function. Angular Loss is a better function to train for a speaker verification model as the model is trained in an end-to-end manner with loss optimizing for embeddings cluster to be far from each other for different speaker by maximizing the angle between these clusters" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "ULTjBuFI19Js" + }, + "source": [ + "To train for verification we just need to toggle `angular` flag in `config.model.decoder.params.angular = True` else set it to `False` to train with cross-entropy loss for identification purposes. \n", + "Once we set this, the loss will be changed to angular loss and we can follow the above steps to the model.\n", + "Note the scale and margin values to be set for the loss function are present at `config.model.loss.scale` and `config.model.loss.margin`" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "LcKiNEY032-t" + }, + "source": [ + "## Extract Speaker Embeddings\n", + "Once you have a trained model or use one of our pretrained nemo checkpoints to get speaker embeddings for any speaker.\n", + "\n", + "To demonstrate this we shall use `nemo_asr.models.EncDecSpeakerLabelModel` with say 5 audio_samples from our dev manifest set. This model is specifically for inference purposes to extract embeddings from a trained `.nemo` model" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": {}, + "colab_type": "code", + "id": "uXEzKMHf3r6-" + }, + "outputs": [], + "source": [ + "verification_model = nemo_asr.models.EncDecSpeakerLabelModel.restore_from(os.path.join(log_dir_finetune, '..', 'titanet-large-finetune.nemo'))" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "Y-XiLHMQ8BIk" + }, + "source": [ + "Now, we need to pass the necessary manifest_filepath and params to set up the data loader for extracting embeddings" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": {}, + "colab_type": "code", + "id": "lk2vsDJk9PS8" + }, + "outputs": [], + "source": [ + "!head -5 {validation_manifest} > embeddings_manifest.json" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": {}, + "colab_type": "code", + "id": "DEd5poCr9yrP" + }, + "outputs": [], + "source": [ + "config.model.train_ds" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from nemo.collections.asr.parts.utils.speaker_utils import embedding_normalize\n", + "from tqdm import tqdm\n", + "try:\n", + " from torch.cuda.amp import autocast\n", + "except ImportError:\n", + " from contextlib import contextmanager\n", + "\n", + " @contextmanager\n", + " def autocast(enabled=None):\n", + " yield\n", + "import numpy as np\n", + "import json\n", + "import pickle as pkl" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": {}, + "colab_type": "code", + "id": "JIHok6LD8g0F" + }, + "outputs": [], + "source": [ + "def get_embeddings(speaker_model, manifest_file, batch_size=1, embedding_dir='./', device='cuda'):\n", + " test_config = OmegaConf.create(\n", + " dict(\n", + " manifest_filepath=manifest_file,\n", + " sample_rate=16000,\n", + " labels=None,\n", + " batch_size=batch_size,\n", + " shuffle=False,\n", + " time_length=20,\n", + " )\n", + " )\n", + "\n", + " speaker_model.setup_test_data(test_config)\n", + " speaker_model = speaker_model.to(device)\n", + " speaker_model.eval()\n", + "\n", + " all_embs=[]\n", + " out_embeddings = {}\n", + " \n", + " for test_batch in tqdm(speaker_model.test_dataloader()):\n", + " test_batch = [x.to(device) for x in test_batch]\n", + " audio_signal, audio_signal_len, labels, slices = test_batch\n", + " with autocast():\n", + " _, embs = speaker_model.forward(input_signal=audio_signal, input_signal_length=audio_signal_len)\n", + " emb_shape = embs.shape[-1]\n", + " embs = embs.view(-1, emb_shape)\n", + " all_embs.extend(embs.cpu().detach().numpy())\n", + " del test_batch\n", + "\n", + " all_embs = np.asarray(all_embs)\n", + " all_embs = embedding_normalize(all_embs)\n", + " with open(manifest_file, 'r') as manifest:\n", + " for i, line in enumerate(manifest.readlines()):\n", + " line = line.strip()\n", + " dic = json.loads(line)\n", + " uniq_name = '@'.join(dic['audio_filepath'].split('/')[-3:])\n", + " out_embeddings[uniq_name] = all_embs[i]\n", + "\n", + " embedding_dir = os.path.join(embedding_dir, 'embeddings')\n", + " if not os.path.exists(embedding_dir):\n", + " os.makedirs(embedding_dir, exist_ok=True)\n", + "\n", + " prefix = manifest_file.split('/')[-1].rsplit('.', 1)[-2]\n", + "\n", + " name = os.path.join(embedding_dir, prefix)\n", + " embeddings_file = name + '_embeddings.pkl'\n", + " pkl.dump(out_embeddings, open(embeddings_file, 'wb'))\n", + " print(\"Saved embedding files to {}\".format(embedding_dir))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": {}, + "colab_type": "code", + "id": "u2FRecqD-ln5" + }, + "outputs": [], + "source": [ + "manifest_filepath = os.path.join(NEMO_ROOT,'embeddings_manifest.json')\n", + "device = 'cuda' if torch.cuda.is_available() else 'cpu'\n", + "get_embeddings(verification_model, manifest_filepath, batch_size=64,embedding_dir='./', device=device)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "zfjXPsjzDOgr" + }, + "source": [ + "Embeddings are stored in dict structure with key-value pair, key being uniq_name generated based on audio_filepath of the sample present in manifest_file in `embedding_dir`" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": {}, + "colab_type": "code", + "id": "hmTeSR6jD28k" + }, + "outputs": [], + "source": [ + "ls ./embeddings/" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "accelerator": "GPU", + "colab": { + "collapsed_sections": [], + "name": "Speaker_Recogniton_Verification.ipynb", + "provenance": [], + "toc_visible": true + }, + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.9.12" + } + }, + "nbformat": 4, + "nbformat_minor": 1 } From 1428763b0045dc4a0d4783bcd8624a8bd46424fe Mon Sep 17 00:00:00 2001 From: Somshubra Majumdar Date: Sat, 9 Jul 2022 01:53:13 -0700 Subject: [PATCH 17/52] [ASR][Breaking Change] Update signature of Hypothesis alignments (#4511) * Preserve logprobs when preserving alignments Signed-off-by: smajumdar * Update tests for rnnt gredy and beam search Signed-off-by: smajumdar * Update all dependents of alignments Signed-off-by: smajumdar * Update docs Signed-off-by: smajumdar --- nemo/collections/asr/metrics/rnnt_wer.py | 7 +- nemo/collections/asr/metrics/rnnt_wer_bpe.py | 3 +- .../parts/submodules/ctc_greedy_decoding.py | 3 +- .../parts/submodules/rnnt_beam_decoding.py | 21 ++- .../parts/submodules/rnnt_greedy_decoding.py | 31 ++-- .../asr/parts/utils/streaming_utils.py | 3 +- tests/collections/asr/test_asr_metrics.py | 4 +- .../asr/test_asr_rnnt_encdec_model.py | 175 ++++++++++++++++++ tutorials/asr/ASR_with_Transducers.ipynb | 2 +- 9 files changed, 218 insertions(+), 31 deletions(-) diff --git a/nemo/collections/asr/metrics/rnnt_wer.py b/nemo/collections/asr/metrics/rnnt_wer.py index 9558203db983..5009e074ee8d 100644 --- a/nemo/collections/asr/metrics/rnnt_wer.py +++ b/nemo/collections/asr/metrics/rnnt_wer.py @@ -46,7 +46,8 @@ class AbstractRNNTDecoding(ABC): preserve_alignments: Bool flag which preserves the history of logprobs generated during decoding (sample / batched). When set to true, the Hypothesis will contain - the non-null value for `logprobs` in it. Here, `logprobs` is a List of torch.Tensors. + the non-null value for `alignments` in it. Here, `alignments` is a List of List of + Tuple(Tensor (of length V + 1), Tensor(scalar, label after argmax)). In order to obtain this hypothesis, please utilize `rnnt_decoder_predictions_tensor` function with the `return_hypotheses` flag set to True. @@ -199,6 +200,7 @@ def __init__(self, decoding_cfg, decoder, joint, blank_id: int): ) elif self.cfg.strategy == 'maes': + self.decoding = beam_decode.BeamRNNTInfer( decoder_model=decoder, joint_model=joint, @@ -384,7 +386,8 @@ class RNNTDecoding(AbstractRNNTDecoding): preserve_alignments: Bool flag which preserves the history of logprobs generated during decoding (sample / batched). When set to true, the Hypothesis will contain - the non-null value for `logprobs` in it. Here, `logprobs` is a List of torch.Tensors. + the non-null value for `logprobs` in it. Here, `alignments` is a List of List of + Tuple(Tensor (of length V + 1), Tensor(scalar, label after argmax)). In order to obtain this hypothesis, please utilize `rnnt_decoder_predictions_tensor` function with the `return_hypotheses` flag set to True. diff --git a/nemo/collections/asr/metrics/rnnt_wer_bpe.py b/nemo/collections/asr/metrics/rnnt_wer_bpe.py index 7d82e62bf964..10451056ca9d 100644 --- a/nemo/collections/asr/metrics/rnnt_wer_bpe.py +++ b/nemo/collections/asr/metrics/rnnt_wer_bpe.py @@ -46,7 +46,8 @@ class RNNTBPEDecoding(AbstractRNNTDecoding): preserve_alignments: Bool flag which preserves the history of logprobs generated during decoding (sample / batched). When set to true, the Hypothesis will contain - the non-null value for `logprobs` in it. Here, `logprobs` is a List of torch.Tensors. + the non-null value for `alignments` in it. Here, `alignments` is a List of List of + Tuple(Tensor (of length V + 1), Tensor(scalar, label after argmax)). In order to obtain this hypothesis, please utilize `rnnt_decoder_predictions_tensor` function with the `return_hypotheses` flag set to True. diff --git a/nemo/collections/asr/parts/submodules/ctc_greedy_decoding.py b/nemo/collections/asr/parts/submodules/ctc_greedy_decoding.py index aef901f6bf01..bb780681e06d 100644 --- a/nemo/collections/asr/parts/submodules/ctc_greedy_decoding.py +++ b/nemo/collections/asr/parts/submodules/ctc_greedy_decoding.py @@ -156,7 +156,8 @@ def _greedy_decode_logprobs(self, x: torch.Tensor, out_len: torch.Tensor): hypothesis.score = (prediction_logprobs[non_blank_ids]).sum() if self.preserve_alignments: - hypothesis.alignments = prediction.clone() + # Preserve the logprobs, as well as labels after argmax + hypothesis.alignments = (prediction.clone(), prediction_labels.clone()) if self.compute_timestamps: hypothesis.timestep = torch.nonzero(non_blank_ids, as_tuple=False)[:, 0].numpy().tolist() diff --git a/nemo/collections/asr/parts/submodules/rnnt_beam_decoding.py b/nemo/collections/asr/parts/submodules/rnnt_beam_decoding.py index 1419abdb41df..3279514dc5cb 100644 --- a/nemo/collections/asr/parts/submodules/rnnt_beam_decoding.py +++ b/nemo/collections/asr/parts/submodules/rnnt_beam_decoding.py @@ -164,7 +164,7 @@ class BeamRNNTInfer(Typing): preserve_alignments: Bool flag which preserves the history of alignments generated during beam decoding (sample). When set to true, the Hypothesis will contain - the non-null value for `alignments` in it. Here, `alignments` is a List of List of ints. + the non-null value for `alignments` in it. Here, `alignments` is a List of List of Tensor (of length V + 1). The length of the list corresponds to the Acoustic Length (T). Each value in the list (Ti) is a torch.Tensor (U), representing 1 or more targets from a vocabulary. @@ -223,6 +223,7 @@ def __init__( self.beam_size = beam_size self.score_norm = score_norm + self.max_candidates = beam_size if self.beam_size == 1: logging.info("Beam size of 1 was used, switching to sample level `greedy_search`") @@ -415,7 +416,7 @@ def greedy_search( not_blank = True symbols_added = 0 - while not_blank: + while not_blank and (symbols_added < self.max_candidates): ytu = torch.log_softmax(self.joint.joint(hi, y) / self.softmax_temperature, dim=-1) # [1, 1, 1, V + 1] ytu = ytu[0, 0, 0, :] # [V + 1] @@ -427,8 +428,8 @@ def greedy_search( pred = pred.item() if self.preserve_alignments: - # insert logits into last timestep - alignments[-1].append(pred) + # insert logprobs into last timestep + alignments[-1].append((ytu.to('cpu'), torch.tensor(pred, dtype=torch.int32))) if pred == self.blank: not_blank = False @@ -519,6 +520,10 @@ def default_beam_search( ytu = torch.log_softmax(self.joint.joint(hi, y) / self.softmax_temperature, dim=-1) # [1, 1, 1, V + 1] ytu = ytu[0, 0, 0, :] # [V + 1] + # preserve alignments + if self.preserve_alignments: + logprobs = ytu.cpu().clone() + # remove blank token before top k top_k = ytu[ids].topk(beam_k, dim=-1) @@ -556,9 +561,13 @@ def default_beam_search( if self.preserve_alignments: if k == self.blank: - new_hyp.alignments[-1].append(self.blank) + new_hyp.alignments[-1].append( + (logprobs.clone(), torch.tensor(self.blank, dtype=torch.int32)) + ) else: - new_hyp.alignments[-1].append(new_hyp.y_sequence[-1]) + new_hyp.alignments[-1].append( + (logprobs.clone(), torch.tensor(new_hyp.y_sequence[-1], dtype=torch.int32)) + ) # keep those hypothesis that have scores greater than next search generation hyps_max = float(max(hyps, key=lambda x: x.score).score) diff --git a/nemo/collections/asr/parts/submodules/rnnt_greedy_decoding.py b/nemo/collections/asr/parts/submodules/rnnt_greedy_decoding.py index 2fec2c680fa6..0dd2b206fa79 100644 --- a/nemo/collections/asr/parts/submodules/rnnt_greedy_decoding.py +++ b/nemo/collections/asr/parts/submodules/rnnt_greedy_decoding.py @@ -81,7 +81,8 @@ class _GreedyRNNTInfer(Typing): no limit. preserve_alignments: Bool flag which preserves the history of alignments generated during greedy decoding (sample / batched). When set to true, the Hypothesis will contain - the non-null value for `alignments` in it. Here, `alignments` is a List of List of ints. + the non-null value for `alignments` in it. Here, `alignments` is a List of List of + Tuple(Tensor (of length V + 1), Tensor(scalar, label after argmax)). The length of the list corresponds to the Acoustic Length (T). Each value in the list (Ti) is a torch.Tensor (U), representing 1 or more targets from a vocabulary. @@ -202,7 +203,8 @@ class GreedyRNNTInfer(_GreedyRNNTInfer): no limit. preserve_alignments: Bool flag which preserves the history of alignments generated during greedy decoding (sample / batched). When set to true, the Hypothesis will contain - the non-null value for `alignments` in it. Here, `alignments` is a List of List of ints. + the non-null value for `alignments` in it. Here, `alignments` is a List of List of + Tuple(Tensor (of length V + 1), Tensor(scalar, label after argmax)). The length of the list corresponds to the Acoustic Length (T). Each value in the list (Ti) is a torch.Tensor (U), representing 1 or more targets from a vocabulary. @@ -291,7 +293,6 @@ def _greedy_decode( if self.preserve_alignments: # Alignments is a 2-dimensional dangling list representing T x U - # alignments = [[]] hypothesis.alignments = [[]] # For timestep t in X_t @@ -328,8 +329,8 @@ def _greedy_decode( k = k.item() # K is the label at timestep t_s in inner loop, s >= 0. if self.preserve_alignments: - # insert logits into last timestep - hypothesis.alignments[-1].append(k) + # insert logprobs into last timestep + hypothesis.alignments[-1].append((logp.to('cpu'), torch.tensor(k, dtype=torch.int32))) del logp @@ -376,7 +377,8 @@ class GreedyBatchedRNNTInfer(_GreedyRNNTInfer): no limit. preserve_alignments: Bool flag which preserves the history of alignments generated during greedy decoding (sample / batched). When set to true, the Hypothesis will contain - the non-null value for `alignments` in it. Here, `alignments` is a List of List of ints. + the non-null value for `alignments` in it. Here, `alignments` is a List of List of + Tuple(Tensor (of length V + 1), Tensor(scalar, label after argmax)). The length of the list corresponds to the Acoustic Length (T). Each value in the list (Ti) is a torch.Tensor (U), representing 1 or more targets from a vocabulary. @@ -479,11 +481,6 @@ def _greedy_decode_blank_as_pad( # alignments is a 3-dimensional dangling list representing B x T x U for hyp in hypotheses: hyp.alignments = [[]] - # alignments = [] - # for _ in range(batchsize): - # alignments.append([[]]) - else: - alignments = None # Last Label buffer + Last Label without blank buffer # batch level equivalent of the last_label @@ -540,11 +537,11 @@ def _greedy_decode_blank_as_pad( # If preserving alignments, check if sequence length of sample has been reached # before adding alignment if self.preserve_alignments: - # Insert ids into last timestep per sample - logp_vals = logp.to('cpu').max(1)[1] + # Insert logprobs into last timestep per sample + logp_vals = logp.to('cpu') for batch_idx in range(batchsize): if time_idx < out_len[batch_idx]: - hypotheses[batch_idx].alignments[-1].append(logp_vals[batch_idx]) + hypotheses[batch_idx].alignments[-1].append((logp_vals[batch_idx], k[batch_idx])) del logp_vals del logp @@ -711,11 +708,11 @@ def _greedy_decode_masked( # If preserving alignments, check if sequence length of sample has been reached # before adding alignment if self.preserve_alignments: - # Insert ids into last timestep per sample - logp_vals = logp.to('cpu').max(1)[1] + # Insert logprobs into last timestep per sample + logp_vals = logp.to('cpu') for batch_idx in range(batchsize): if time_idx < out_len[batch_idx]: - hypotheses[batch_idx].alignments[-1].append(logp_vals[batch_idx]) + hypotheses[batch_idx].alignments[-1].append((logp_vals[batch_idx], k[batch_idx])) del logp_vals del logp diff --git a/nemo/collections/asr/parts/utils/streaming_utils.py b/nemo/collections/asr/parts/utils/streaming_utils.py index 30deb3f8f1cc..41537b571054 100644 --- a/nemo/collections/asr/parts/utils/streaming_utils.py +++ b/nemo/collections/asr/parts/utils/streaming_utils.py @@ -1050,7 +1050,8 @@ def _alignment_decoder(self, alignments, tokenizer, blank_id): for t in range(len(alignments)): for u in range(len(alignments[t])): - token_id = int(alignments[t][u]) + _, token_id = alignments[t][u] # (logprob, token_id) + token_id = int(token_id) if token_id != blank_id: token = tokenizer.ids_to_tokens([token_id])[0] s.append(token) diff --git a/tests/collections/asr/test_asr_metrics.py b/tests/collections/asr/test_asr_metrics.py index 28382d39b7af..929112e659c1 100644 --- a/tests/collections/asr/test_asr_metrics.py +++ b/tests/collections/asr/test_asr_metrics.py @@ -178,7 +178,7 @@ def test_wer_metric_return_hypothesis(self, batch_dim_index, test_wer_bpe): assert (hyp.y_sequence - torch.tensor([3, 1, 20])).sum() == 0 assert hyp.score == 3 # sum of number of tokens in one hot representation assert hyp.text == 'cat' - assert (hyp.alignments == sample).all() + assert (hyp.alignments[0] == sample).all() assert hyp.length == 0 length = torch.tensor([tensor.shape[1 - batch_dim_index]], dtype=torch.long) @@ -210,7 +210,7 @@ def test_wer_metric_subword_return_hypothesis(self, batch_dim_index, test_wer_bp assert (hyp.y_sequence - torch.tensor([3, 1, 20])).sum() == 0 assert hyp.score == 3 # sum of number of tokens in one hot representation assert hyp.text == 'cat' - assert (hyp.alignments == sample).all() + assert (hyp.alignments[0] == sample).all() assert hyp.length == 0 length = torch.tensor([tensor.shape[1 - batch_dim_index]], dtype=torch.long) diff --git a/tests/collections/asr/test_asr_rnnt_encdec_model.py b/tests/collections/asr/test_asr_rnnt_encdec_model.py index 96daca4ae5d4..972d02688e59 100644 --- a/tests/collections/asr/test_asr_rnnt_encdec_model.py +++ b/tests/collections/asr/test_asr_rnnt_encdec_model.py @@ -18,8 +18,10 @@ from omegaconf import DictConfig, ListConfig from nemo.collections.asr.models import EncDecRNNTModel +from nemo.collections.asr.modules import RNNTDecoder, RNNTJoint from nemo.collections.asr.parts.submodules import rnnt_beam_decoding as beam_decode from nemo.collections.asr.parts.submodules import rnnt_greedy_decoding as greedy_decode +from nemo.collections.asr.parts.utils import rnnt_utils from nemo.core.utils import numba_utils from nemo.core.utils.numba_utils import __NUMBA_MINIMUM_VERSION__ from nemo.utils.config_utils import assert_dataclass_signature_match @@ -279,3 +281,176 @@ def test_BeamRNNTInferConfig(self): assert signatures_match assert cls_subset is None assert dataclass_subset is None + + @pytest.mark.skipif( + not NUMBA_RNNT_LOSS_AVAILABLE, reason='RNNTLoss has not been compiled with appropriate numba version.', + ) + @pytest.mark.unit + @pytest.mark.parametrize( + "greedy_class", [greedy_decode.GreedyRNNTInfer, greedy_decode.GreedyBatchedRNNTInfer], + ) + def test_greedy_decoding(self, greedy_class): + token_list = [" ", "a", "b", "c"] + vocab_size = len(token_list) + + encoder_output_size = 4 + decoder_output_size = 4 + joint_output_shape = 4 + + prednet_cfg = {'pred_hidden': decoder_output_size, 'pred_rnn_layers': 1} + jointnet_cfg = { + 'encoder_hidden': encoder_output_size, + 'pred_hidden': decoder_output_size, + 'joint_hidden': joint_output_shape, + 'activation': 'relu', + } + + decoder = RNNTDecoder(prednet_cfg, vocab_size) + joint_net = RNNTJoint(jointnet_cfg, vocab_size, vocabulary=token_list) + + greedy = greedy_class(decoder, joint_net, blank_index=len(token_list) - 1, max_symbols_per_step=5) + + # (B, D, T) + enc_out = torch.randn(1, encoder_output_size, 30) + enc_len = torch.tensor([30], dtype=torch.int32) + + with torch.no_grad(): + _ = greedy(encoder_output=enc_out, encoded_lengths=enc_len) + + @pytest.mark.skipif( + not NUMBA_RNNT_LOSS_AVAILABLE, reason='RNNTLoss has not been compiled with appropriate numba version.', + ) + @pytest.mark.unit + @pytest.mark.parametrize( + "greedy_class", [greedy_decode.GreedyRNNTInfer, greedy_decode.GreedyBatchedRNNTInfer], + ) + def test_greedy_decoding_preserve_alignment(self, greedy_class): + token_list = [" ", "a", "b", "c"] + vocab_size = len(token_list) + + encoder_output_size = 4 + decoder_output_size = 4 + joint_output_shape = 4 + + prednet_cfg = {'pred_hidden': decoder_output_size, 'pred_rnn_layers': 1} + jointnet_cfg = { + 'encoder_hidden': encoder_output_size, + 'pred_hidden': decoder_output_size, + 'joint_hidden': joint_output_shape, + 'activation': 'relu', + } + + decoder = RNNTDecoder(prednet_cfg, vocab_size) + joint_net = RNNTJoint(jointnet_cfg, vocab_size, vocabulary=token_list) + + greedy = greedy_class( + decoder, joint_net, blank_index=len(token_list) - 1, preserve_alignments=True, max_symbols_per_step=5 + ) + + # (B, D, T) + enc_out = torch.randn(1, encoder_output_size, 30) + enc_len = torch.tensor([30], dtype=torch.int32) + + with torch.no_grad(): + hyp = greedy(encoder_output=enc_out, encoded_lengths=enc_len)[0][0] # type: rnnt_utils.Hypothesis + assert hyp.alignments is not None + + for t in range(len(hyp.alignments)): + for u in range(len(hyp.alignments[t])): + logp, label = hyp.alignments[t][u] + assert torch.is_tensor(logp) + assert torch.is_tensor(label) + + @pytest.mark.skipif( + not NUMBA_RNNT_LOSS_AVAILABLE, reason='RNNTLoss has not been compiled with appropriate numba version.', + ) + @pytest.mark.unit + @pytest.mark.parametrize( + "beam_config", + [ + {"search_type": "greedy"}, + {"search_type": "default", "score_norm": False, "return_best_hypothesis": False}, + {"search_type": "alsd", "alsd_max_target_len": 20, "return_best_hypothesis": False}, + {"search_type": "tsd", "tsd_max_sym_exp_per_step": 3, "return_best_hypothesis": False}, + {"search_type": "maes", "maes_num_steps": 2, "maes_expansion_beta": 2, "return_best_hypothesis": False}, + {"search_type": "maes", "maes_num_steps": 3, "maes_expansion_beta": 1, "return_best_hypothesis": False}, + ], + ) + def test_beam_decoding(self, beam_config): + token_list = [" ", "a", "b", "c"] + vocab_size = len(token_list) + beam_size = 1 if beam_config["search_type"] == "greedy" else 2 + + encoder_output_size = 4 + decoder_output_size = 4 + joint_output_shape = 4 + + prednet_cfg = {'pred_hidden': decoder_output_size, 'pred_rnn_layers': 1} + jointnet_cfg = { + 'encoder_hidden': encoder_output_size, + 'pred_hidden': decoder_output_size, + 'joint_hidden': joint_output_shape, + 'activation': 'relu', + } + + decoder = RNNTDecoder(prednet_cfg, vocab_size) + joint_net = RNNTJoint(jointnet_cfg, vocab_size, vocabulary=token_list) + + beam = beam_decode.BeamRNNTInfer(decoder, joint_net, beam_size=beam_size, **beam_config,) + + # (B, D, T) + enc_out = torch.randn(1, encoder_output_size, 30) + enc_len = torch.tensor([30], dtype=torch.int32) + + with torch.no_grad(): + _ = beam(encoder_output=enc_out, encoded_lengths=enc_len) + + @pytest.mark.skipif( + not NUMBA_RNNT_LOSS_AVAILABLE, reason='RNNTLoss has not been compiled with appropriate numba version.', + ) + @pytest.mark.unit + @pytest.mark.parametrize( + "beam_config", + [{"search_type": "greedy"}, {"search_type": "default", "score_norm": False, "return_best_hypothesis": False},], + ) + def test_beam_decoding_preserve_alignments(self, beam_config): + token_list = [" ", "a", "b", "c"] + vocab_size = len(token_list) + beam_size = 1 if beam_config["search_type"] == "greedy" else 2 + + encoder_output_size = 4 + decoder_output_size = 4 + joint_output_shape = 4 + + prednet_cfg = {'pred_hidden': decoder_output_size, 'pred_rnn_layers': 1} + jointnet_cfg = { + 'encoder_hidden': encoder_output_size, + 'pred_hidden': decoder_output_size, + 'joint_hidden': joint_output_shape, + 'activation': 'relu', + } + + decoder = RNNTDecoder(prednet_cfg, vocab_size) + joint_net = RNNTJoint(jointnet_cfg, vocab_size, vocabulary=token_list) + + beam = beam_decode.BeamRNNTInfer( + decoder, joint_net, beam_size=beam_size, **beam_config, preserve_alignments=True + ) + + # (B, D, T) + enc_out = torch.randn(1, encoder_output_size, 30) + enc_len = torch.tensor([30], dtype=torch.int32) + + with torch.no_grad(): + hyp = beam(encoder_output=enc_out, encoded_lengths=enc_len)[0][0] # type: rnnt_utils.Hypothesis + + if isinstance(hyp, rnnt_utils.NBestHypotheses): + hyp = hyp.n_best_hypotheses[0] # select top hypothesis only + + assert hyp.alignments is not None + + for t in range(len(hyp.alignments)): + for u in range(len(hyp.alignments[t])): + logp, label = hyp.alignments[t][u] + assert torch.is_tensor(logp) + assert torch.is_tensor(label) diff --git a/tutorials/asr/ASR_with_Transducers.ipynb b/tutorials/asr/ASR_with_Transducers.ipynb index 14dbd1d8eccd..d49bc5509fe8 100644 --- a/tutorials/asr/ASR_with_Transducers.ipynb +++ b/tutorials/asr/ASR_with_Transducers.ipynb @@ -1337,7 +1337,7 @@ "for ti in range(len(alignments)):\n", " t_u = []\n", " for uj in range(len(alignments[ti])):\n", - " token = alignments[ti][uj]\n", + " logprob, token = alignments[ti][uj]\n", " token = token.to('cpu').numpy().tolist()\n", " decoded_token = model.decoding.decode_ids_to_tokens([token])[0] if token != model.decoding.blank_id else '' # token at index len(vocab) == RNNT blank token\n", " t_u.append(decoded_token)\n", From 309a81a4347195be4b3666a5703142f2427ee095 Mon Sep 17 00:00:00 2001 From: tbartley94 <90423858+tbartley94@users.noreply.github.com> Date: Mon, 11 Jul 2022 13:16:08 -0400 Subject: [PATCH 18/52] Weighted bucketing (#4530) --- nemo/collections/asr/data/audio_to_label_dataset.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/nemo/collections/asr/data/audio_to_label_dataset.py b/nemo/collections/asr/data/audio_to_label_dataset.py index 20a0064ac1dc..16ebd36c7fcc 100644 --- a/nemo/collections/asr/data/audio_to_label_dataset.py +++ b/nemo/collections/asr/data/audio_to_label_dataset.py @@ -117,6 +117,12 @@ def get_tarred_speech_label_dataset( tarred_audio_filepaths = convert_to_config_list(tarred_audio_filepaths) manifest_filepaths = convert_to_config_list(manifest_filepaths) + bucketing_weights = config.get('bucketing_weights', None) # For upsampling buckets + if bucketing_weights: + for idx, weight in enumerate(bucketing_weights): + if not isinstance(weight, int) or weight <= 0: + raise ValueError(f"bucket weights must be positive integers") + if len(manifest_filepaths) != len(tarred_audio_filepaths): raise ValueError( f"manifest_filepaths (length={len(manifest_filepaths)}) and tarred_audio_filepaths (length={len(tarred_audio_filepaths)}) need to have the same number of buckets." @@ -144,6 +150,9 @@ def get_tarred_speech_label_dataset( world_size=world_size, ) - datasets.append(dataset) + if bucketing_weights: + [datasets.append(dataset) for _ in range(bucketing_weights[dataset_idx])] + else: + datasets.append(dataset) return get_chain_dataset(datasets=datasets, ds_config=config) From 7f75191f52d84d1e06ff7d4eb03c3b71571c5baa Mon Sep 17 00:00:00 2001 From: Sandeep Subramanian Date: Mon, 11 Jul 2022 15:45:04 -0700 Subject: [PATCH 19/52] Additional sentencepiece args - Byte fallback, split digits, split_on_whitespace (#4525) * Fix geglu without fusion Signed-off-by: MaximumEntropy * Add extra args Signed-off-by: MaximumEntropy * Reset transformer Signed-off-by: MaximumEntropy * Style Signed-off-by: MaximumEntropy * Fix spm arg Signed-off-by: MaximumEntropy * Fix help string Signed-off-by: MaximumEntropy --- .../create_tarred_parallel_dataset.py | 27 ++++++++++++++++--- .../tokenizers/sentencepiece_tokenizer.py | 15 +++++++++++ .../machine_translation/preproc_mt_data.py | 12 +++++++++ 3 files changed, 51 insertions(+), 3 deletions(-) diff --git a/examples/nlp/machine_translation/create_tarred_parallel_dataset.py b/examples/nlp/machine_translation/create_tarred_parallel_dataset.py index f6f344987431..ce61cc81dafe 100644 --- a/examples/nlp/machine_translation/create_tarred_parallel_dataset.py +++ b/examples/nlp/machine_translation/create_tarred_parallel_dataset.py @@ -33,7 +33,10 @@ '--encoder_tokenizer_model', type=str, default='None', help='Path to pre-trained encoder tokenizer model' ) parser.add_argument( - '--encoder_tokenizer_name', type=str, default='yttm', help='Encoder BPE Tokenizer Name, Options: [yttm]' + '--encoder_tokenizer_name', + type=str, + default='yttm', + help='Encoder BPE Tokenizer Name, Options: [yttm, sentencepiece]', ) parser.add_argument('--encoder_tokenizer_vocab_size', type=int, default=32000, help='Encoder Vocab size after BPE') parser.add_argument( @@ -52,7 +55,10 @@ '--decoder_tokenizer_model', type=str, default='None', help='Path to pre-trained decoder tokenizer model' ) parser.add_argument( - '--decoder_tokenizer_name', type=str, default='yttm', help='Encoder BPE Tokenizer Name, Options: [yttm]' + '--decoder_tokenizer_name', + type=str, + default='yttm', + help='Encoder BPE Tokenizer Name, Options: [yttm, sentencepiece]', ) parser.add_argument('--decoder_tokenizer_vocab_size', type=int, default=32000, help='Encoder Vocab size after BPE') parser.add_argument( @@ -86,7 +92,19 @@ parser.add_argument( '--n_preproc_jobs', type=int, default=-2, help='Number of processes to use for creating the tarred dataset.', ) - + parser.add_argument( + '--byte_fallback', + action="store_true", + help='Whether to use byte fallback with sentencepiece for BPE tokenization.', + ) + parser.add_argument( + '--split_digits', action="store_true", help='Whether to split digits while tokenizing with sentencepiece.' + ) + parser.add_argument( + '--no_split_by_whitespace', + action="store_true", + help='If True, this will not respect whitepsaces while learning BPE merges.', + ) args = parser.parse_args() if not os.path.exists(args.out_dir): os.mkdir(args.out_dir) @@ -120,6 +138,9 @@ decoder_tokenizer_vocab_size=args.decoder_tokenizer_vocab_size, decoder_tokenizer_coverage=args.decoder_tokenizer_coverage, global_rank=0, + byte_fallback=args.byte_fallback, + split_digits=args.split_digits, + split_by_whitespace=not args.no_split_by_whitespace, ) else: encoder_tokenizer_model, decoder_tokenizer_model = args.encoder_tokenizer_model, args.decoder_tokenizer_model diff --git a/nemo/collections/common/tokenizers/sentencepiece_tokenizer.py b/nemo/collections/common/tokenizers/sentencepiece_tokenizer.py index ba595b795fc1..84290f052919 100644 --- a/nemo/collections/common/tokenizers/sentencepiece_tokenizer.py +++ b/nemo/collections/common/tokenizers/sentencepiece_tokenizer.py @@ -270,6 +270,9 @@ def create_spt_model( pad: bool = False, control_symbols: List[str] = None, user_defined_symbols: List[str] = None, + byte_fallback: bool = False, + split_digits: bool = False, + split_by_whitespace: bool = True, ): """ Creates sentence piece tokenizer model from data file. @@ -292,6 +295,9 @@ def create_spt_model( These tokens get removed at decode time and are not encoded from the text - can only be added to the input programatically. user_defined_symbols: user symbols to add to tokenizer, as defined by sentencepiece. These tokens remain in the decoded text and are encoded automatically when present in the input text. + byte_fallback: If , fallback to a byte sequence of the character. + split_digits: If true, digits are split into individual tokens. + split_by_whitespace: Whether to respect white space while creating subwords. If False, will learn merges across whitespace. """ if not data_file or not os.path.exists(data_file): @@ -349,6 +355,15 @@ def create_spt_model( if max_sentencepiece_length >= 0: cmd += f" --max_sentencepiece_length={max_sentencepiece_length}" + if byte_fallback: + cmd += " --byte_fallback=true" + + if split_digits: + cmd += " --split_digits=true" + + if not split_by_whitespace: + cmd += " --split_by_whitespace=false" + sentencepiece.SentencePieceTrainer.Train(cmd) # Add BERT control symbols diff --git a/nemo/collections/nlp/data/machine_translation/preproc_mt_data.py b/nemo/collections/nlp/data/machine_translation/preproc_mt_data.py index 6ea060ff1038..01eaf293e6b3 100644 --- a/nemo/collections/nlp/data/machine_translation/preproc_mt_data.py +++ b/nemo/collections/nlp/data/machine_translation/preproc_mt_data.py @@ -759,6 +759,9 @@ def train_tokenizers( encoder_special_tokens=None, decoder_special_tokens=None, spt_symbols=None, + byte_fallback=False, + split_digits=False, + split_by_whitespace=True, ): """Trains a tokenizer with requested parameters, returns None if the tokenizer is not trainable""" @@ -817,6 +820,9 @@ def train_tokenizers( pad=True, control_symbols=spt_symbols, user_defined_symbols=encoder_special_tokens, + byte_fallback=byte_fallback, + split_digits=split_digits, + split_by_whitespace=split_by_whitespace, ) os.rename( os.path.join(out_dir, 'tokenizer.model'), @@ -859,6 +865,9 @@ def train_tokenizers( pad=True, control_symbols=spt_symbols, user_defined_symbols=encoder_special_tokens, + byte_fallback=byte_fallback, + split_digits=split_digits, + split_by_whitespace=split_by_whitespace, ) os.rename(os.path.join(dir_name, 'tokenizer.model'), os.path.join(encoder_tokenizer_model)) @@ -898,6 +907,9 @@ def train_tokenizers( pad=True, control_symbols=spt_symbols, user_defined_symbols=decoder_special_tokens, + byte_fallback=byte_fallback, + split_digits=split_digits, + split_by_whitespace=split_by_whitespace, ) os.rename(os.path.join(dir_name, 'tokenizer.model'), os.path.join(decoder_tokenizer_model)) From dc6bea2acbfff899f06d3d7fa6ac98aec3aa4af3 Mon Sep 17 00:00:00 2001 From: Somshubra Majumdar Date: Mon, 11 Jul 2022 19:05:04 -0700 Subject: [PATCH 20/52] Add support for ASR Adapter Auxiliary Losses (#4480) * Add support for access mixin registry of custom losses Signed-off-by: smajumdar * add support for asr custom losses Signed-off-by: smajumdar * Update for l2 loss Signed-off-by: smajumdar * Add unittests Signed-off-by: smajumdar * Add unittests Signed-off-by: smajumdar * Add unittests Signed-off-by: smajumdar * Update registration of tensors to reset after finishing step Signed-off-by: smajumdar * Remove comment Signed-off-by: smajumdar * Remove comment Signed-off-by: smajumdar * Update SSL models Signed-off-by: smajumdar * Add support for validation step properly registering tensors Signed-off-by: smajumdar * Move reset of registry outside Signed-off-by: smajumdar --- .../asr/conf/asr_adapters/asr_adaptation.yaml | 1 + nemo/collections/asr/models/asr_model.py | 40 +++++++++++++- nemo/collections/asr/models/ctc_models.py | 14 ++++- nemo/collections/asr/models/rnnt_models.py | 19 +++++++ nemo/collections/asr/models/ssl_models.py | 38 ++++++++++++-- .../asr/parts/submodules/conformer_modules.py | 4 +- .../asr/parts/submodules/jasper.py | 4 +- .../common/parts/adapter_modules.py | 4 +- nemo/core/classes/__init__.py | 2 +- nemo/core/classes/mixins/access_mixins.py | 52 +++++++++++++++---- .../mixins/adapter_mixin_strategies.py | 30 ++++++++++- .../mixins/adapters/test_asr_adapter_mixin.py | 30 +++++++++++ .../mixins/adapters/test_adapter_strategy.py | 33 +++++++++++- 13 files changed, 243 insertions(+), 28 deletions(-) diff --git a/examples/asr/conf/asr_adapters/asr_adaptation.yaml b/examples/asr/conf/asr_adapters/asr_adaptation.yaml index 34f2b4293438..b1519241e5fa 100644 --- a/examples/asr/conf/asr_adapters/asr_adaptation.yaml +++ b/examples/asr/conf/asr_adapters/asr_adaptation.yaml @@ -77,6 +77,7 @@ model: adapter_strategy: _target_: nemo.core.classes.mixins.adapter_mixin_strategies.ResidualAddAdapterStrategy stochastic_depth: 0.0 # float, setting to > 0 will enable stochastic depth for each adapter block. + l2_lambda: 0.0 # float, setting to > 0 will enable l2 norm auxiliary loss for each adapter's output. # Optional global config available to all adapters at a global level. # A global config is shared across every layer of the adapters, defining global properties rather diff --git a/nemo/collections/asr/models/asr_model.py b/nemo/collections/asr/models/asr_model.py index cf7ed5c343ee..501204993498 100644 --- a/nemo/collections/asr/models/asr_model.py +++ b/nemo/collections/asr/models/asr_model.py @@ -19,7 +19,8 @@ from nemo.core.classes import ModelPT from nemo.core.classes.exportable import Exportable -from nemo.utils import model_utils +from nemo.core.classes.mixins import AccessMixin +from nemo.utils import logging, model_utils from nemo.utils.export_utils import cast_all __all__ = ['ASRModel'] @@ -63,6 +64,43 @@ def list_available_models(cls) -> 'List[PretrainedModelInfo]': list_of_models = model_utils.resolve_subclass_pretrained_model_info(cls) return list_of_models + def add_auxiliary_losses(self, loss: torch.Tensor, reset_registry: bool = False) -> torch.Tensor: + """ + Utility method to enable calculation of auxiliary losses for ASR training. + + Args: + loss: The output loss value prior to addition with auxiliary losses. + reset_registry: Bool, whether to reset the AccessMixin registry after adding auxiliary losses. + + Returns: + Loss tensor used for back propagation. + """ + # Add adapter auxiliary losses, if registered + if AccessMixin.is_access_enabled(): + registry = AccessMixin.get_module_registry(self) + log_dict = {} + + for loss_key, loss_registry in registry.items(): + # Add auxiliary loss to total loss + loss_list = loss_registry['adapter_loss'] + loss_value = sum(loss_list) + loss += loss_value + + # Log current loss name and value + keys = loss_key.split(".") + key = "/".join(keys) + key = "adapter_loss/" + key + log_dict[key] = loss_value.detach() + + if len(log_dict) > 0: + self.log_dict(log_dict) + + if reset_registry: + AccessMixin.reset_registry(self) + + # return total loss + return loss + def setup_optimization_flags(self): """ Utility method that must be explicitly called by the subclass in order to support optional optimization flags. diff --git a/nemo/collections/asr/models/ctc_models.py b/nemo/collections/asr/models/ctc_models.py index 619c6cb105e0..9d5dd78138ac 100644 --- a/nemo/collections/asr/models/ctc_models.py +++ b/nemo/collections/asr/models/ctc_models.py @@ -31,6 +31,7 @@ from nemo.collections.asr.parts.mixins import ASRModuleMixin from nemo.collections.asr.parts.preprocessing.perturb import process_augmentations from nemo.core.classes.common import PretrainedModelInfo, typecheck +from nemo.core.classes.mixins import AccessMixin from nemo.core.neural_types import AudioSignal, LabelsType, LengthsType, LogprobsType, NeuralType, SpectrogramType from nemo.utils import logging @@ -547,6 +548,10 @@ def forward( # PTL-specific methods def training_step(self, batch, batch_nb): + # Reset access registry + if AccessMixin.is_access_enabled(): + AccessMixin.reset_registry(self) + signal, signal_len, transcript, transcript_len = batch if isinstance(batch, DALIOutputs) and batch.has_processed_signal: log_probs, encoded_len, predictions = self.forward( @@ -559,6 +564,13 @@ def training_step(self, batch, batch_nb): log_probs=log_probs, targets=transcript, input_lengths=encoded_len, target_lengths=transcript_len ) + # Add auxiliary losses, if registered + loss_value = self.add_auxiliary_losses(loss_value) + + # Reset access registry + if AccessMixin.is_access_enabled(): + AccessMixin.reset_registry(self) + tensorboard_logs = { 'train_loss': loss_value, 'learning_rate': self._optimizer.param_groups[0]['lr'], @@ -593,7 +605,7 @@ def predict_step(self, batch, batch_idx, dataloader_idx=0): log_probs, encoded_len, predictions = self.forward(input_signal=signal, input_signal_length=signal_len) transcribed_texts, _ = self._wer.decoding.ctc_decoder_predictions_tensor( - predictions=log_probs, predictions_len=encoded_len, return_hypotheses=False, + decoder_outputs=log_probs, decoder_lengths=encoded_len, return_hypotheses=False, ) sample_id = sample_id.cpu().detach().numpy() diff --git a/nemo/collections/asr/models/rnnt_models.py b/nemo/collections/asr/models/rnnt_models.py index 992fe67cc5f0..c59cf7caf455 100644 --- a/nemo/collections/asr/models/rnnt_models.py +++ b/nemo/collections/asr/models/rnnt_models.py @@ -34,6 +34,7 @@ from nemo.collections.asr.parts.preprocessing.perturb import process_augmentations from nemo.core.classes import Exportable from nemo.core.classes.common import PretrainedModelInfo, typecheck +from nemo.core.classes.mixins import AccessMixin from nemo.core.neural_types import AcousticEncodedRepresentation, AudioSignal, LengthsType, NeuralType, SpectrogramType from nemo.utils import logging @@ -660,6 +661,10 @@ def forward( # PTL-specific methods def training_step(self, batch, batch_nb): + # Reset access registry + if AccessMixin.is_access_enabled(): + AccessMixin.reset_registry(self) + signal, signal_len, transcript, transcript_len = batch # forward() only performs encoder forward @@ -687,6 +692,13 @@ def training_step(self, batch, batch_nb): log_probs=joint, targets=transcript, input_lengths=encoded_len, target_lengths=target_length ) + # Add auxiliary losses, if registered + loss_value = self.add_auxiliary_losses(loss_value) + + # Reset access registry + if AccessMixin.is_access_enabled(): + AccessMixin.reset_registry(self) + tensorboard_logs = { 'train_loss': loss_value, 'learning_rate': self._optimizer.param_groups[0]['lr'], @@ -716,6 +728,13 @@ def training_step(self, batch, batch_nb): compute_wer=compute_wer, ) + # Add auxiliary losses, if registered + loss_value = self.add_auxiliary_losses(loss_value) + + # Reset access registry + if AccessMixin.is_access_enabled(): + AccessMixin.reset_registry(self) + tensorboard_logs = {'train_loss': loss_value, 'learning_rate': self._optimizer.param_groups[0]['lr']} if compute_wer: diff --git a/nemo/collections/asr/models/ssl_models.py b/nemo/collections/asr/models/ssl_models.py index 7a22127c1139..7dbc06a5d56d 100644 --- a/nemo/collections/asr/models/ssl_models.py +++ b/nemo/collections/asr/models/ssl_models.py @@ -104,9 +104,6 @@ def __init__(self, cfg: DictConfig, trainer: Trainer = None): self.start_step[decoder_loss_name] = decoder_loss_cfg.get("start_step", 0) self.transpose_encoded[decoder_loss_name] = decoder_loss_cfg.get("transpose_encoded", False) - if self.output_from_layer[decoder_loss_name] is not None: - self.set_access_enabled(access_enabled=True) - self.decoder_losses = nn.ModuleDict(self.decoder_losses) else: @@ -310,8 +307,29 @@ def forward( 3) The encoded features tensor of shape [B, D, T]. 2) The lengths of the acoustic sequence after propagation through the encoder, of shape [B]. """ + # Reset access registry + if self.is_access_enabled(): + self.reset_registry() + + # Check for special flag for validation step + if hasattr(self, '_in_validation_step'): + in_validation_step = self._in_validation_step + else: + in_validation_step = False + # reset module registry from AccessMixin - self.reset_registry() + if ( + (self.training or in_validation_step) + and self.decoder_losses is not None + and self.output_from_layer is not None + and len(self.output_from_layer) > 0 + ): + layer_names = list(self.output_from_layer.values()) + register_layer = any([name is not None for name in layer_names]) + + if register_layer: + self.access_cfg['save_encoder_tensors'] = True + self.set_access_enabled(access_enabled=True) has_input_signal = input_signal is not None and input_signal_length is not None has_processed_signal = processed_signal is not None and processed_signal_length is not None @@ -408,7 +426,7 @@ def decoder_loss_step(self, spectrograms, spec_masks, encoded, encoded_len, targ dec_input = encoded else: # extract output from specified layer using AccessMixin registry - dec_input = registry[self.output_from_layer[dec_loss_name]][-1] + dec_input = registry[self.output_from_layer[dec_loss_name]]['encoder'][-1] if self.transpose_encoded[dec_loss_name]: dec_input = dec_input.transpose(-2, -1) @@ -480,9 +498,15 @@ def training_step(self, batch, batch_nb): if self.feat_pen: loss_value += self.feat_pen + # Reset access registry + self.reset_registry() + return {'loss': loss_value, 'log': tensorboard_logs} def validation_step(self, batch, batch_idx, dataloader_idx=0): + # Set flag to register tensors + self._in_validation_step = True + signal, signal_len, targets, target_lengths = batch spectrograms, spec_masks, encoded, encoded_len = self.forward( input_signal=signal, input_signal_length=signal_len, @@ -493,6 +517,10 @@ def validation_step(self, batch, batch_idx, dataloader_idx=0): if self.feat_pen: loss_value += self.feat_pen + # reset access registry + self.reset_registry() + del self._in_validation_step + return { 'val_loss': loss_value, } diff --git a/nemo/collections/asr/parts/submodules/conformer_modules.py b/nemo/collections/asr/parts/submodules/conformer_modules.py index 96bb64de290a..0c1c5e538d93 100644 --- a/nemo/collections/asr/parts/submodules/conformer_modules.py +++ b/nemo/collections/asr/parts/submodules/conformer_modules.py @@ -126,8 +126,8 @@ def forward(self, x, att_mask=None, pos_emb=None, pad_mask=None): # Call the adapters x = self.forward_enabled_adapters(x) - if self.is_access_enabled(): - self.register_accessible_tensor(tensor=x) + if self.is_access_enabled() and self.access_cfg.get('save_encoder_tensors', False): + self.register_accessible_tensor(name='encoder', tensor=x) return x diff --git a/nemo/collections/asr/parts/submodules/jasper.py b/nemo/collections/asr/parts/submodules/jasper.py index dd1cc9036528..d2d1a177221b 100644 --- a/nemo/collections/asr/parts/submodules/jasper.py +++ b/nemo/collections/asr/parts/submodules/jasper.py @@ -1042,8 +1042,8 @@ def forward(self, input_: Tuple[List[Tensor], Optional[Tensor]]): out = out.transpose(1, 2) # (B, C, T) - if self.is_access_enabled(): - self.register_accessible_tensor(tensor=out) + if self.is_access_enabled() and self.access_cfg.get('save_encoder_tensors', False): + self.register_accessible_tensor(name='encoder', tensor=out) if self.res is not None and self.dense_residual: return xs + [out], lens diff --git a/nemo/collections/common/parts/adapter_modules.py b/nemo/collections/common/parts/adapter_modules.py index d5a5a371371f..7e985ee33b97 100644 --- a/nemo/collections/common/parts/adapter_modules.py +++ b/nemo/collections/common/parts/adapter_modules.py @@ -20,10 +20,10 @@ from torch import nn as nn from nemo.collections.common.parts.utils import activation_registry -from nemo.core.classes.mixins import adapter_mixin_strategies +from nemo.core.classes.mixins import access_mixins, adapter_mixin_strategies -class AbstractAdapterModule(nn.Module): +class AbstractAdapterModule(nn.Module, access_mixins.AccessMixin): """ Base class of Adapter Modules, providing common functionality to all Adapter Modules. """ diff --git a/nemo/core/classes/__init__.py b/nemo/core/classes/__init__.py index 992e95a23061..13dc1c7ae459 100644 --- a/nemo/core/classes/__init__.py +++ b/nemo/core/classes/__init__.py @@ -29,7 +29,7 @@ from nemo.core.classes.dataset import Dataset, IterableDataset from nemo.core.classes.exportable import Exportable, ExportFormat from nemo.core.classes.loss import Loss -from nemo.core.classes.mixins import adapter_mixins +from nemo.core.classes.mixins import access_mixins, adapter_mixins from nemo.core.classes.modelPT import ModelPT from nemo.core.classes.module import NeuralModule from nemo.utils import exceptions diff --git a/nemo/core/classes/mixins/access_mixins.py b/nemo/core/classes/mixins/access_mixins.py index de3950443956..3e455c7f2624 100644 --- a/nemo/core/classes/mixins/access_mixins.py +++ b/nemo/core/classes/mixins/access_mixins.py @@ -13,6 +13,7 @@ # limitations under the License. from abc import ABC +from typing import Optional import torch from omegaconf import DictConfig @@ -35,9 +36,9 @@ class AccessMixin(ABC): def __init__(self): super().__init__() - self._registry = [] + self._registry = {} # dictionary of lists - def register_accessible_tensor(self, tensor): + def register_accessible_tensor(self, name, tensor): """ Register tensor for later use. """ @@ -48,12 +49,12 @@ def register_accessible_tensor(self, tensor): tensor = tensor.detach() if not hasattr(self, '_registry'): - self._registry = [] + self._registry = {} - if len(self._registry) > 0: - self._registry.clear() + if name not in self._registry: + self._registry[name] = [] - self._registry.append(tensor) + self._registry[name].append(tensor) @classmethod def get_module_registry(cls, module: torch.nn.Module): @@ -68,15 +69,37 @@ def get_module_registry(cls, module: torch.nn.Module): module_registry[name] = m._registry return module_registry - def reset_registry(self): + def reset_registry(self: torch.nn.Module, registry_key: Optional[str] = None): """ Reset the registries of all named sub-modules """ if hasattr(self, "_registry"): - self._registry.clear() + if registry_key is None: + self._registry.clear() + else: + if registry_key in self._registry: + self._registry.pop(registry_key) + else: + raise KeyError( + f"Registry key `{registry_key}` provided, but registry does not have this key.\n" + f"Available keys in registry : {list(self._registry.keys())}" + ) + for _, m in self.named_modules(): if hasattr(m, "_registry"): - m._registry.clear() + if registry_key is None: + m._registry.clear() + else: + if registry_key in self._registry: + self._registry.pop(registry_key) + else: + raise KeyError( + f"Registry key `{registry_key}` provided, but registry does not have this key.\n" + f"Available keys in registry : {list(self._registry.keys())}" + ) + + # Explicitly disable registry cache after reset + AccessMixin.set_access_enabled(access_enabled=False) @property def access_cfg(self): @@ -87,10 +110,17 @@ def access_cfg(self): global _ACCESS_CFG return _ACCESS_CFG - def is_access_enabled(self): + @classmethod + def update_access_cfg(cls, cfg: dict): + global _ACCESS_CFG + _ACCESS_CFG.update(cfg) + + @classmethod + def is_access_enabled(cls): global _ACCESS_ENABLED return _ACCESS_ENABLED - def set_access_enabled(self, access_enabled: bool): + @classmethod + def set_access_enabled(cls, access_enabled: bool): global _ACCESS_ENABLED _ACCESS_ENABLED = access_enabled diff --git a/nemo/core/classes/mixins/adapter_mixin_strategies.py b/nemo/core/classes/mixins/adapter_mixin_strategies.py index 4fbf4ed3de78..24069cd1098f 100644 --- a/nemo/core/classes/mixins/adapter_mixin_strategies.py +++ b/nemo/core/classes/mixins/adapter_mixin_strategies.py @@ -17,6 +17,8 @@ import torch +from nemo.core.classes.mixins import AccessMixin + class AbstractAdapterStrategy(ABC): def forward(self, input: torch.Tensor, adapter: torch.nn.Module, *, module: 'AdapterModuleMixin'): @@ -55,7 +57,7 @@ class ResidualAddAdapterStrategy(AbstractAdapterStrategy): Supports stochastic depth regularization. """ - def __init__(self, stochastic_depth: float = 0.0): + def __init__(self, stochastic_depth: float = 0.0, l2_lambda: float = 0.0): """ An implementation of residual addition of an adapter module with its input. Performs output = input + adapter(input). @@ -63,9 +65,12 @@ def __init__(self, stochastic_depth: float = 0.0): Args: stochastic_depth: float, when greater than one, can optionally dropout the output of the adapter's forward pass. + l2_lambda: L2 norm of the difference between the original input to the function, and the adapter's + output result. Disabled if set to 0.0. """ super().__init__() self.stochastic_depth = stochastic_depth + self.l2_lambda = l2_lambda def forward(self, input: torch.Tensor, adapter: torch.nn.Module, *, module: 'AdapterModuleMixin'): """ @@ -104,12 +109,33 @@ def forward(self, input: torch.Tensor, adapter: torch.nn.Module, *, module: 'Ada out = noise * out # Return the residual connection output = input + adapter(input) - return input + out + result = input + out + + # If l2_lambda is activated, register the loss value + if module.training and self.l2_lambda > 0.0: + if not isinstance(adapter, AccessMixin): + raise ValueError(f"Module {adapter.__class__.__name__} does not implement AccessMixin !") + + # Only add auxiliary loss if adapter has trainable parameters that require gradients + if next(adapter.parameters()).requires_grad is True: + # Check if globally allowed to compute aux loss + compute_aux_loss = adapter.access_cfg.get('compute_adapter_loss', True) + + if compute_aux_loss: + # if l2 lambda is enabled, also enable AccessMixin + adapter.set_access_enabled(access_enabled=True) + + l2_loss = self.l2_lambda * (input - result).square().reshape(input.size(0), -1).sum(dim=-1).mean() + adapter.register_accessible_tensor(name='adapter_loss', tensor=l2_loss) + + return result @dataclass class ResidualAddAdapterStrategyConfig: stochastic_depth: float = 0.0 + l2_lambda: float = 0.0 + _target_: str = "{0}.{1}".format( ResidualAddAdapterStrategy.__module__, ResidualAddAdapterStrategy.__name__ ) # mandatory field diff --git a/tests/collections/asr/mixins/adapters/test_asr_adapter_mixin.py b/tests/collections/asr/mixins/adapters/test_asr_adapter_mixin.py index 1cb38f650f86..e2c4d70ffa30 100644 --- a/tests/collections/asr/mixins/adapters/test_asr_adapter_mixin.py +++ b/tests/collections/asr/mixins/adapters/test_asr_adapter_mixin.py @@ -18,6 +18,7 @@ from nemo.collections.asr.models import ASRModel, EncDecCTCModel, EncDecRNNTModel from nemo.collections.common.parts import adapter_modules +from nemo.core.classes.mixins.access_mixins import AccessMixin from nemo.core.classes.mixins.adapter_mixins import AdapterModuleMixin, get_registered_adapter from nemo.core.utils import numba_utils from nemo.core.utils.numba_utils import __NUMBA_MINIMUM_VERSION__ @@ -405,3 +406,32 @@ def test_constructor_pretrained_rnnt(self): model.freeze() model.unfreeze_enabled_adapters() assert model.num_weights < 1e5 + + @pytest.mark.unit + def test_asr_model_adapter_loss(self, model): + original_num_params = model.num_weights + x = torch.randn(2, 512) + x_len = torch.tensor([256, 512], dtype=torch.int32) + + adapter_cfg = get_adapter_cfg() # type: adapter_modules.LinearAdapterConfig + adapter_cfg.adapter_strategy.l2_lambda = 0.01 + + model.add_adapter(name='adapter_0', cfg=adapter_cfg) + new_num_params = model.num_weights + assert new_num_params > original_num_params + + model.train() # set training mode to true + + with torch.no_grad(): + AccessMixin.reset_registry(model) + AccessMixin.update_access_cfg({'save_encoder_tensors': False}) + _ = model(input_signal=x, input_signal_length=x_len) + + # extract losses + auxiliary_losses = AccessMixin.get_module_registry(model) + + loss = list(auxiliary_losses.values())[0] + assert 'adapter_loss' in loss + assert loss['adapter_loss'][0] == torch.tensor(0.0) # initially adapter is 0 init, no loss required. + + AccessMixin.reset_registry(model) diff --git a/tests/core/mixins/adapters/test_adapter_strategy.py b/tests/core/mixins/adapters/test_adapter_strategy.py index c682a84df88e..a3d049da8c09 100644 --- a/tests/core/mixins/adapters/test_adapter_strategy.py +++ b/tests/core/mixins/adapters/test_adapter_strategy.py @@ -16,7 +16,7 @@ import torch from nemo.core import NeuralModule -from nemo.core.classes.mixins import AdapterModuleMixin, adapter_mixin_strategies, adapter_mixins +from nemo.core.classes.mixins import AdapterModuleMixin, access_mixins, adapter_mixin_strategies, adapter_mixins from nemo.utils import config_utils @@ -152,3 +152,34 @@ def test_strategy_stochasic_depth(self, stochastic_depth): else: check = module_out assert (out - check).abs().mean() < 1e-5 + + @pytest.mark.unit + def test_strategy_l2_lambda(self): + torch.random.manual_seed(0) + x = torch.randn(2, 50) + + module = DefaultModuleAdapter() + module.add_adapter(name='temp', cfg=get_adapter_cfg()) + module.train() + adapter = module.adapter_layer[module.get_enabled_adapters()[0]] + + # update the strategy + adapter_strategy = adapter_mixin_strategies.ResidualAddAdapterStrategy(l2_lambda=0.01) + adapter.adapter_strategy = adapter_strategy + + with torch.no_grad(): + access_mixins.AccessMixin.reset_registry(module) + assert access_mixins.AccessMixin.is_access_enabled() is False + + assert adapter_strategy.stochastic_depth == 0.0 + assert adapter_strategy.l2_lambda > 0.0 + + out = adapter_strategy.forward(x, adapter, module=module) + assert (out - x).abs().mean() < 1e-5 + + # extract losses + assert access_mixins.AccessMixin.is_access_enabled() is True + auxiliary_losses = access_mixins.AccessMixin.get_module_registry(module) + loss = list(auxiliary_losses.values())[0] + assert 'adapter_loss' in loss + assert loss['adapter_loss'][0] == torch.tensor(0.0) # initially adapter is 0 init, no loss required. From bc29ef217b2a770fe07d17703f8616e6889f5a42 Mon Sep 17 00:00:00 2001 From: "He Huang (Steve)" <105218074+stevehuang52@users.noreply.github.com> Date: Tue, 12 Jul 2022 12:15:02 -0400 Subject: [PATCH 21/52] update (#4520) Signed-off-by: stevehuang52 --- nemo/collections/common/parts/preprocessing/manifest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nemo/collections/common/parts/preprocessing/manifest.py b/nemo/collections/common/parts/preprocessing/manifest.py index 64d7bbd85212..d63a3ae63bcb 100644 --- a/nemo/collections/common/parts/preprocessing/manifest.py +++ b/nemo/collections/common/parts/preprocessing/manifest.py @@ -95,7 +95,7 @@ def __parse_item(line: str, manifest_file: str) -> Dict[str, Any]: # Assume that the audio path is like "wavs/xxxxxx.wav". manifest_dir = Path(manifest_file).parent audio_file = Path(item['audio_file']) - if not audio_file.is_file() and not audio_file.is_absolute(): + if (len(str(audio_file)) < 255) and not audio_file.is_file() and not audio_file.is_absolute(): # assume the "wavs/" dir and manifest are under the same parent dir audio_file = manifest_dir / audio_file if audio_file.is_file(): From b70ec73aaa586d636c9406299d6987f136ebbfe0 Mon Sep 17 00:00:00 2001 From: Evelina <10428420+ekmb@users.noreply.github.com> Date: Tue, 12 Jul 2022 10:15:00 -0700 Subject: [PATCH 22/52] fix duplex inference with grammars (#4517) * fix duplex inference with grammars Signed-off-by: ekmb * add ci test for duplex, fix electronic last sym bug Signed-off-by: ekmb * test fix Signed-off-by: ekmb * fix jenkins Signed-off-by: ekmb * update jenkins grammars Signed-off-by: ekmb * add pt to the docs Signed-off-by: ekmb * fix jenkins Signed-off-by: ekmb * disable test Signed-off-by: ekmb * fix jenkins Signed-off-by: ekmb * jenkins refactor Signed-off-by: ekmb * fix jenkins Signed-off-by: ekmb * fix jenkins Signed-off-by: ekmb * fix jenkins Signed-off-by: ekmb * jenkins Signed-off-by: ekmb * jenkins Signed-off-by: ekmb * jenkins Signed-off-by: ekmb * jenkins Signed-off-by: ekmb * jenkins Signed-off-by: ekmb * jenkins Signed-off-by: ekmb * test Signed-off-by: ekmb * test Signed-off-by: ekmb * test Signed-off-by: ekmb * test Signed-off-by: ekmb Co-authored-by: Yang Zhang --- Jenkinsfile | 73 ++++++++++--------- .../wfst/wfst_text_normalization.rst | 2 + .../nn_wfst/en/electronic/normalize.py | 14 ++-- .../nn_wfst/en/whitelist/normalize.py | 17 ++--- .../pt/data/__init__.py | 13 ++++ .../en/taggers/electronic.py | 2 +- .../test_cases_punctuation.txt | 1 + 7 files changed, 72 insertions(+), 50 deletions(-) create mode 100644 nemo_text_processing/inverse_text_normalization/pt/data/__init__.py diff --git a/Jenkinsfile b/Jenkinsfile index 6ba65f340eab..1c2a61d2e134 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -137,18 +137,18 @@ pipeline { parallel { stage('En TN grammars') { steps { - sh 'CUDA_VISIBLE_DEVICES="" python nemo_text_processing/text_normalization/normalize.py --text="1" --cache_dir /home/TestData/nlp/text_norm/ci/grammars/6-28-22' + sh 'CUDA_VISIBLE_DEVICES="" python nemo_text_processing/text_normalization/normalize.py --text="1" --cache_dir /home/TestData/nlp/text_norm/ci/grammars/7-10-22' } } stage('En ITN grammars') { steps { - sh 'CUDA_VISIBLE_DEVICES="" python nemo_text_processing/inverse_text_normalization/inverse_normalize.py --language en --text="twenty" --cache_dir /home/TestData/nlp/text_norm/ci/grammars/6-28-22' + sh 'CUDA_VISIBLE_DEVICES="" python nemo_text_processing/inverse_text_normalization/inverse_normalize.py --language en --text="twenty" --cache_dir /home/TestData/nlp/text_norm/ci/grammars/7-10-22' } } stage('Test En non-deterministic TN & Run all En TN/ITN tests (restore grammars from cache)') { steps { - sh 'CUDA_VISIBLE_DEVICES="" python nemo_text_processing/text_normalization/normalize_with_audio.py --text "\$.01" --n_tagged 2 --cache_dir /home/TestData/nlp/text_norm/ci/grammars/6-28-22' - sh 'CUDA_VISIBLE_DEVICES="" pytest tests/nemo_text_processing/en/ -m "not pleasefixme" --cpu --tn_cache_dir /home/TestData/nlp/text_norm/ci/grammars/6-28-22' + sh 'CUDA_VISIBLE_DEVICES="" python nemo_text_processing/text_normalization/normalize_with_audio.py --text "\$.01" --n_tagged 2 --cache_dir /home/TestData/nlp/text_norm/ci/grammars/7-10-22' + sh 'CUDA_VISIBLE_DEVICES="" pytest tests/nemo_text_processing/en/ -m "not pleasefixme" --cpu --tn_cache_dir /home/TestData/nlp/text_norm/ci/grammars/7-10-22' } } } @@ -165,7 +165,7 @@ pipeline { parallel { stage('L2: Eng TN') { steps { - sh 'cd tools/text_processing_deployment && python pynini_export.py --output=/home/TestData/nlp/text_norm/output/ --grammars=tn_grammars --cache_dir /home/TestData/nlp/text_norm/ci/grammars/6-28-22 --language=en && ls -R /home/TestData/nlp/text_norm/output/ && echo ".far files created "|| exit 1' + sh 'cd tools/text_processing_deployment && python pynini_export.py --output=/home/TestData/nlp/text_norm/output/ --grammars=tn_grammars --cache_dir /home/TestData/nlp/text_norm/ci/grammars/7-10-22 --language=en && ls -R /home/TestData/nlp/text_norm/output/ && echo ".far files created "|| exit 1' sh 'cd nemo_text_processing/text_normalization/ && python normalize.py --input_file=/home/TestData/nlp/text_norm/ci/test.txt --input_case="lower_cased" --language=en --output_file=/home/TestData/nlp/text_norm/output/test.pynini.txt --verbose' sh 'cat /home/TestData/nlp/text_norm/output/test.pynini.txt' sh 'cmp --silent /home/TestData/nlp/text_norm/output/test.pynini.txt /home/TestData/nlp/text_norm/ci/test_goal_py_05-25.txt || exit 1' @@ -175,7 +175,7 @@ pipeline { stage('L2: Eng ITN export') { steps { - sh 'cd tools/text_processing_deployment && python pynini_export.py --output=/home/TestData/nlp/text_denorm/output/ --grammars=itn_grammars --cache_dir /home/TestData/nlp/text_norm/ci/grammars/6-28-22 --language=en && ls -R /home/TestData/nlp/text_denorm/output/ && echo ".far files created "|| exit 1' + sh 'cd tools/text_processing_deployment && python pynini_export.py --output=/home/TestData/nlp/text_denorm/output/ --grammars=itn_grammars --cache_dir /home/TestData/nlp/text_norm/ci/grammars/7-10-22 --language=en && ls -R /home/TestData/nlp/text_denorm/output/ && echo ".far files created "|| exit 1' sh 'cd nemo_text_processing/inverse_text_normalization/ && python inverse_normalize.py --input_file=/home/TestData/nlp/text_denorm/ci/test.txt --language=en --output_file=/home/TestData/nlp/text_denorm/output/test.pynini.txt --verbose' sh 'cmp --silent /home/TestData/nlp/text_denorm/output/test.pynini.txt /home/TestData/nlp/text_denorm/ci/test_goal_py.txt || exit 1' sh 'rm -rf /home/TestData/nlp/text_denorm/output/*' @@ -184,7 +184,7 @@ pipeline { stage('L2: TN with Audio (audio and raw text)') { steps { sh 'cd nemo_text_processing/text_normalization && \ - python normalize_with_audio.py --language=en --cache_dir /home/TestData/nlp/text_norm/ci/grammars/6-28-22 --text "The total amounts to \\$4.76." \ + python normalize_with_audio.py --language=en --cache_dir /home/TestData/nlp/text_norm/ci/grammars/7-10-22 --text "The total amounts to \\$4.76." \ --audio_data /home/TestData/nlp/text_norm/audio_based/audio.wav | tail -n2 | head -n1 > /tmp/out_raw.txt 2>&1 && \ cmp --silent /tmp/out_raw.txt /home/TestData/nlp/text_norm/audio_based/result.txt || exit 1' } @@ -192,7 +192,7 @@ pipeline { stage('L2: TN with Audio (audio and text file)') { steps { sh 'cd nemo_text_processing/text_normalization && \ - python normalize_with_audio.py --language=en --cache_dir /home/TestData/nlp/text_norm/ci/grammars/6-28-22 --text /home/TestData/nlp/text_norm/audio_based/text.txt \ + python normalize_with_audio.py --language=en --cache_dir /home/TestData/nlp/text_norm/ci/grammars/7-10-22 --text /home/TestData/nlp/text_norm/audio_based/text.txt \ --audio_data /home/TestData/nlp/text_norm/audio_based/audio.wav | tail -n2 | head -n1 > /tmp/out_file.txt 2>&1 && \ cmp --silent /tmp/out_file.txt /home/TestData/nlp/text_norm/audio_based/result.txt || exit 1' } @@ -200,7 +200,7 @@ pipeline { stage('L2: TN with Audio (manifest)') { steps { sh 'cd nemo_text_processing/text_normalization && \ - python normalize_with_audio.py --language=en --audio_data /home/TestData/nlp/text_norm/audio_based/manifest.json --n_tagged=120 --cache_dir /home/TestData/nlp/text_norm/ci/grammars/6-28-22' + python normalize_with_audio.py --language=en --audio_data /home/TestData/nlp/text_norm/audio_based/manifest.json --n_tagged=120 --cache_dir /home/TestData/nlp/text_norm/ci/grammars/7-10-22' } } } @@ -1023,7 +1023,37 @@ pipeline { } } } - stage('L2: Dialogue Generation Part 2') { +// stage('L2: Dialogue Generation Part 2') { +// when { +// anyOf { +// branch 'main' +// changeRequest target: 'main' +// } +// } +// failFast true +// parallel { +// stage('Dialogue: Answer Extender using DialogueGPTGenerationModel') { +// steps { +// sh 'TRANSFORMERS_OFFLINE=0 && cd examples/nlp/dialogue && \ +// python dialogue.py \ +// do_training=False \ +// model.dataset.data_dir=/home/TestData/nlp/ms-marco-qa \ +// model.dataset.dialogues_example_dir=answer_extender \ +// model.library=huggingface \ +// model.dataset.task=ms_marco \ +// model.dataset.debug_mode=True \ +// trainer.val_check_interval=0.0 \ +// trainer.devices=[0] \ +// model.dataset.use_cache=false \ +// model.language_model.pretrained_model_name=gpt2 \ +// trainer.accelerator=gpu \ +// exp_manager=null && \ +// rm -rf answer_extender' +// } +// } +// } +// } + stage('L2: COPY') { when { anyOf { branch 'main' @@ -1134,29 +1164,6 @@ pipeline { data.test_ds.data_path=/home/TestData/nlp/duplex_text_norm/small_test.tsv' } } - //this is a new test by @aleksandraa - //cannot run it in a fork, Jenkins doesn't see it - //need to uncomment, when given writing permissions to NeMo - //stage('Text normalization as tagging (Thutmose Tagger)') { - // steps { - // sh 'cd examples/nlp/normalization_as_tagging && \ - // python normalization_as_tagging_train.py \ - // lang="en" \ - // data.validation_ds.data_path=/home/TestData/nlp/text_normalization_as_tagging/en_mini/valid.tsv \ - // data.train_ds.data_path=/home/TestData/nlp/text_normalization_as_tagging/en_mini/train.tsv \ - // data.train_ds.batch_size=2 \ - // data.train_ds.num_workers=2 \ - // model.language_model.pretrained_model_name=bert-base-uncased \ - // model.label_map=/home/TestData/nlp/text_normalization_as_tagging/en_mini/label_map.txt \ - // model.semiotic_classes=/home/TestData/nlp/text_normalization_as_tagging/en_mini/semiotic_classes.txt \ - // exp_manager.create_checkpoint_callback=false \ - // trainer.devices=1 \ - // trainer.num_nodes=1 \ - // trainer.accelerator=gpu \ - // trainer.strategy=ddp \ - // +trainer.fast_dev_run=true' - // } - //} } } // Runs out of memory on the 12G TITAN V (GPU 0 on main CI) diff --git a/docs/source/nlp/text_normalization/wfst/wfst_text_normalization.rst b/docs/source/nlp/text_normalization/wfst/wfst_text_normalization.rst index da481d2f0b5f..b612a1aa4cc6 100644 --- a/docs/source/nlp/text_normalization/wfst/wfst_text_normalization.rst +++ b/docs/source/nlp/text_normalization/wfst/wfst_text_normalization.rst @@ -111,6 +111,8 @@ Language Support Matrix +------------------+----------+----------+----------+--------------------+ | Vietnamese | vi | | x | | +------------------+----------+----------+----------+--------------------+ +| Portuguese | pt | | x | | ++------------------+----------+----------+----------+--------------------+ Grammar customization --------------------- diff --git a/examples/nlp/duplex_text_normalization/nn_wfst/en/electronic/normalize.py b/examples/nlp/duplex_text_normalization/nn_wfst/en/electronic/normalize.py index 204708bef9a9..e0d83b42222d 100644 --- a/examples/nlp/duplex_text_normalization/nn_wfst/en/electronic/normalize.py +++ b/examples/nlp/duplex_text_normalization/nn_wfst/en/electronic/normalize.py @@ -13,6 +13,9 @@ # limitations under the License. from nemo_text_processing.text_normalization.normalize import Normalizer +from nemo_text_processing.text_normalization.token_parser import TokenParser + +from nemo.collections.common.tokenizers.moses_tokenizers import MosesProcessor class ElectronicNormalizer(Normalizer): @@ -37,13 +40,6 @@ def __init__( overwrite_cache: bool = False, ): - super().__init__( - input_case=input_case, - lang=lang, - deterministic=deterministic, - cache_dir=cache_dir, - overwrite_cache=overwrite_cache, - ) from nn_wfst.en.electronic.tokenize_and_classify import ClassifyFst from nn_wfst.en.electronic.verbalize_final import VerbalizeFinalFst @@ -51,3 +47,7 @@ def __init__( input_case=input_case, deterministic=deterministic, cache_dir=cache_dir, overwrite_cache=overwrite_cache ) self.verbalizer = VerbalizeFinalFst(deterministic=deterministic) + self.post_processor = None + self.parser = TokenParser() + self.lang = lang + self.processor = MosesProcessor(lang_id=lang) diff --git a/examples/nlp/duplex_text_normalization/nn_wfst/en/whitelist/normalize.py b/examples/nlp/duplex_text_normalization/nn_wfst/en/whitelist/normalize.py index f19f86d8ead2..4109109ec83a 100644 --- a/examples/nlp/duplex_text_normalization/nn_wfst/en/whitelist/normalize.py +++ b/examples/nlp/duplex_text_normalization/nn_wfst/en/whitelist/normalize.py @@ -13,6 +13,9 @@ # limitations under the License. from nemo_text_processing.text_normalization.normalize import Normalizer +from nemo_text_processing.text_normalization.token_parser import TokenParser + +from nemo.collections.common.tokenizers.moses_tokenizers import MosesProcessor class WhitelistNormalizer(Normalizer): @@ -39,18 +42,10 @@ def __init__( whitelist: str = None, ): - super().__init__( - input_case=input_case, - lang=lang, - deterministic=deterministic, - cache_dir=cache_dir, - overwrite_cache=overwrite_cache, - whitelist=whitelist, - ) from nn_wfst.en.whitelist.tokenize_and_classify import ClassifyFst from nn_wfst.en.whitelist.verbalize_final import VerbalizeFinalFst - self.tagger = self.tagger = ClassifyFst( + self.tagger = ClassifyFst( input_case=input_case, deterministic=deterministic, cache_dir=cache_dir, @@ -58,3 +53,7 @@ def __init__( whitelist=whitelist, ) self.verbalizer = VerbalizeFinalFst(deterministic=deterministic) + self.post_processor = None + self.parser = TokenParser() + self.lang = lang + self.processor = MosesProcessor(lang_id=lang) diff --git a/nemo_text_processing/inverse_text_normalization/pt/data/__init__.py b/nemo_text_processing/inverse_text_normalization/pt/data/__init__.py new file mode 100644 index 000000000000..a1cf281f0908 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/pt/data/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nemo_text_processing/text_normalization/en/taggers/electronic.py b/nemo_text_processing/text_normalization/en/taggers/electronic.py index 0e059f9f5533..243c0653f6a1 100644 --- a/nemo_text_processing/text_normalization/en/taggers/electronic.py +++ b/nemo_text_processing/text_normalization/en/taggers/electronic.py @@ -48,7 +48,7 @@ def __init__(self, deterministic: bool = True): username = ( pynutil.insert("username: \"") + all_accepted_symbols + pynutil.insert("\"") + pynini.cross('@', ' ') ) - domain_graph = all_accepted_symbols + pynini.accep('.') + all_accepted_symbols + domain_graph = all_accepted_symbols + pynini.accep('.') + all_accepted_symbols + NEMO_ALPHA protocol_symbols = pynini.closure((graph_symbols | pynini.cross(":", "semicolon")) + pynutil.insert(" ")) protocol_start = (pynini.cross("https", "HTTPS ") | pynini.cross("http", "HTTP ")) + ( pynini.accep("://") @ protocol_symbols diff --git a/tests/nemo_text_processing/en/data_text_normalization/test_cases_punctuation.txt b/tests/nemo_text_processing/en/data_text_normalization/test_cases_punctuation.txt index c3073a3934bf..5f5b249c15fe 100644 --- a/tests/nemo_text_processing/en/data_text_normalization/test_cases_punctuation.txt +++ b/tests/nemo_text_processing/en/data_text_normalization/test_cases_punctuation.txt @@ -61,3 +61,4 @@ dr. Evil~dr. Evil ÀÁÂÃ check §- and ƛ, also ɧ~ÀÁÂÃ check section - and ƛ, also ɧ Hi it's 5pm,4A.M.?-34. Hi,no,yes,34! 12,again,4 and NO?17 and $.01,here & there--0.004kg~Hi it's five PM, four AM.? minus thirty four. Hi,no,yes, thirty four! twelve, again, four and NO? seventeen and one cent, here and there - minus zero point zero zero four kilograms 1°C.~one degree Celsius. +my email is myemail@gmail.com!~my email is myemail at gmail dot com! From 8e186ebe7d5f890490d2bec2672474d2f7147f8d Mon Sep 17 00:00:00 2001 From: Ewald Enzinger Date: Wed, 13 Jul 2022 04:55:42 +0200 Subject: [PATCH 23/52] Add Bucketing support to TarredAudioToClassificationLabelDataset (#4465) * Add Bucketing support to TarredAudioToClassificationLabelDataset Signed-off-by: Ewald Enzinger --- .../asr/conf/marblenet/marblenet_3x2x64.yaml | 1 + .../matchboxnet/matchboxnet_3x1x64_v1.yaml | 1 + .../matchboxnet/matchboxnet_3x1x64_v2.yaml | 1 + .../asr/data/audio_to_label_dataset.py | 58 ++++++++++++++----- .../asr/models/classification_models.py | 20 ++++--- .../configs/classification_models_config.py | 5 ++ .../asr/test_asr_classification_model.py | 3 + 7 files changed, 65 insertions(+), 24 deletions(-) diff --git a/examples/asr/conf/marblenet/marblenet_3x2x64.yaml b/examples/asr/conf/marblenet/marblenet_3x2x64.yaml index 1042408fc4f1..fe4dcc537f06 100644 --- a/examples/asr/conf/marblenet/marblenet_3x2x64.yaml +++ b/examples/asr/conf/marblenet/marblenet_3x2x64.yaml @@ -24,6 +24,7 @@ model: # bucketing params bucketing_strategy: "synced_randomized" bucketing_batch_size: null + bucketing_weights: null augmentor: shift: prob: 1.0 diff --git a/examples/asr/conf/matchboxnet/matchboxnet_3x1x64_v1.yaml b/examples/asr/conf/matchboxnet/matchboxnet_3x1x64_v1.yaml index f1a3336bb1b2..ac462a273d96 100644 --- a/examples/asr/conf/matchboxnet/matchboxnet_3x1x64_v1.yaml +++ b/examples/asr/conf/matchboxnet/matchboxnet_3x1x64_v1.yaml @@ -30,6 +30,7 @@ model: # bucketing params bucketing_strategy: "synced_randomized" bucketing_batch_size: null + bucketing_weights: null augmentor: shift: prob: 1.0 diff --git a/examples/asr/conf/matchboxnet/matchboxnet_3x1x64_v2.yaml b/examples/asr/conf/matchboxnet/matchboxnet_3x1x64_v2.yaml index 929ec7a9afe4..a7d4974ed7f3 100644 --- a/examples/asr/conf/matchboxnet/matchboxnet_3x1x64_v2.yaml +++ b/examples/asr/conf/matchboxnet/matchboxnet_3x1x64_v2.yaml @@ -30,6 +30,7 @@ model: # bucketing params bucketing_strategy: "synced_randomized" bucketing_batch_size: null + bucketing_weights: null augmentor: shift: prob: 1.0 diff --git a/nemo/collections/asr/data/audio_to_label_dataset.py b/nemo/collections/asr/data/audio_to_label_dataset.py index 16ebd36c7fcc..95daa500ea94 100644 --- a/nemo/collections/asr/data/audio_to_label_dataset.py +++ b/nemo/collections/asr/data/audio_to_label_dataset.py @@ -78,21 +78,49 @@ def get_tarred_classification_label_dataset( Returns: An instance of TarredAudioToClassificationLabelDataset. """ - dataset = audio_to_label.TarredAudioToClassificationLabelDataset( - audio_tar_filepaths=config['tarred_audio_filepaths'], - manifest_filepath=config['manifest_filepath'], - labels=config['labels'], - featurizer=featurizer, - shuffle_n=shuffle_n, - max_duration=config.get('max_duration', None), - min_duration=config.get('min_duration', None), - trim=config.get('trim_silence', False), - shard_strategy=config.get('tarred_shard_strategy', 'scatter'), - global_rank=global_rank, - world_size=world_size, - is_regression_task=config.get('is_regression_task', False), - ) - return dataset + tarred_audio_filepaths = config['tarred_audio_filepaths'] + manifest_filepaths = config['manifest_filepath'] + datasets = [] + tarred_audio_filepaths = convert_to_config_list(tarred_audio_filepaths) + manifest_filepaths = convert_to_config_list(manifest_filepaths) + + bucketing_weights = config.get('bucketing_weights', None) # For upsampling buckets + if bucketing_weights: + for idx, weight in enumerate(bucketing_weights): + if not isinstance(weight, int) or weight <= 0: + raise ValueError(f"bucket weights must be positive integers") + + if len(manifest_filepaths) != len(tarred_audio_filepaths): + raise ValueError( + f"manifest_filepaths (length={len(manifest_filepaths)}) and tarred_audio_filepaths (length={len(tarred_audio_filepaths)}) need to have the same number of buckets." + ) + + for dataset_idx, (tarred_audio_filepath, manifest_filepath) in enumerate( + zip(tarred_audio_filepaths, manifest_filepaths) + ): + if len(tarred_audio_filepath) == 1: + tarred_audio_filepath = tarred_audio_filepath[0] + dataset = audio_to_label.TarredAudioToClassificationLabelDataset( + audio_tar_filepaths=tarred_audio_filepath, + manifest_filepath=manifest_filepath, + labels=config['labels'], + featurizer=featurizer, + shuffle_n=shuffle_n, + max_duration=config.get('max_duration', None), + min_duration=config.get('min_duration', None), + trim=config.get('trim_silence', False), + shard_strategy=config.get('tarred_shard_strategy', 'scatter'), + global_rank=global_rank, + world_size=world_size, + is_regression_task=config.get('is_regression_task', False), + ) + + if bucketing_weights: + [datasets.append(dataset) for _ in range(bucketing_weights[dataset_idx])] + else: + datasets.append(dataset) + + return get_chain_dataset(datasets=datasets, ds_config=config) def get_tarred_speech_label_dataset( diff --git a/nemo/collections/asr/models/classification_models.py b/nemo/collections/asr/models/classification_models.py index 6971a676a07d..5eeb4f391a64 100644 --- a/nemo/collections/asr/models/classification_models.py +++ b/nemo/collections/asr/models/classification_models.py @@ -226,14 +226,17 @@ def _setup_dataloader_from_config(self, config: DictConfig): shuffle_n = config.get('shuffle_n', 4 * config['batch_size']) if shuffle else 0 dataset = audio_to_label_dataset.get_tarred_classification_label_dataset( featurizer=featurizer, - config=OmegaConf.to_container(config), + config=config, shuffle_n=shuffle_n, global_rank=self.global_rank, world_size=self.world_size, ) shuffle = False batch_size = config['batch_size'] - collate_func = dataset.collate_fn + if hasattr(dataset, 'collate_fn'): + collate_func = dataset.collate_fn + else: + collate_func = dataset.datasets[0].collate_fn else: if 'manifest_filepath' in config and config['manifest_filepath'] is None: @@ -242,17 +245,16 @@ def _setup_dataloader_from_config(self, config: DictConfig): if 'vad_stream' in config and config['vad_stream']: logging.info("Perform streaming frame-level VAD") - dataset = audio_to_label_dataset.get_speech_label_dataset( - featurizer=featurizer, config=OmegaConf.to_container(config) - ) + dataset = audio_to_label_dataset.get_speech_label_dataset(featurizer=featurizer, config=config) batch_size = 1 collate_func = dataset.vad_frame_seq_collate_fn else: - dataset = audio_to_label_dataset.get_classification_label_dataset( - featurizer=featurizer, config=OmegaConf.to_container(config) - ) + dataset = audio_to_label_dataset.get_classification_label_dataset(featurizer=featurizer, config=config) batch_size = config['batch_size'] - collate_func = dataset.collate_fn + if hasattr(dataset, 'collate_fn'): + collate_func = dataset.collate_fn + else: + collate_func = dataset.datasets[0].collate_fn return torch.utils.data.DataLoader( dataset=dataset, diff --git a/nemo/collections/asr/models/configs/classification_models_config.py b/nemo/collections/asr/models/configs/classification_models_config.py index 15b1f00e295c..8b5363fa795d 100644 --- a/nemo/collections/asr/models/configs/classification_models_config.py +++ b/nemo/collections/asr/models/configs/classification_models_config.py @@ -53,6 +53,11 @@ class EncDecClassificationDatasetConfig(nemo.core.classes.dataset.DatasetConfig) normalize_audio: bool = False is_regression_task: bool = False + # bucketing params + bucketing_strategy: str = "synced_randomized" + bucketing_batch_size: Optional[Any] = None + bucketing_weights: Optional[List[int]] = None + @dataclass class EncDecClassificationConfig(model_cfg.ModelConfig): diff --git a/tests/collections/asr/test_asr_classification_model.py b/tests/collections/asr/test_asr_classification_model.py index 24c88d9beb66..a543003f50f1 100644 --- a/tests/collections/asr/test_asr_classification_model.py +++ b/tests/collections/asr/test_asr_classification_model.py @@ -191,6 +191,9 @@ def test_EncDecClassificationDatasetConfig_for_AudioToSpeechLabelDataset(self): 'sample_rate', 'normalize_audio', 'augmentor', + 'bucketing_batch_size', + 'bucketing_strategy', + 'bucketing_weights', ] REMAP_ARGS = {'trim_silence': 'trim'} From ff588a7839f8971526fb5320c5ae5a85438b5b98 Mon Sep 17 00:00:00 2001 From: Abhinav Khattar Date: Wed, 13 Jul 2022 12:31:04 -0700 Subject: [PATCH 24/52] Add MTEncDec Finetune support (#4540) * add FT support Signed-off-by: Abhinav Khattar * rm preproc Signed-off-by: Abhinav Khattar * review changes Signed-off-by: Abhinav Khattar * add CI Signed-off-by: Abhinav Khattar * newline fix Signed-off-by: Abhinav Khattar * CI fix Signed-off-by: Abhinav Khattar * clean up Signed-off-by: Abhinav Khattar * post training cleanup Signed-off-by: Abhinav Khattar * test Signed-off-by: Abhinav Khattar * revert Signed-off-by: Abhinav Khattar * CI test Signed-off-by: Abhinav Khattar * revert CI changes Signed-off-by: Abhinav Khattar * original CI Signed-off-by: Abhinav Khattar Co-authored-by: Sandeep Subramanian --- Jenkinsfile | 36 ++++++ .../conf/aayn_finetune.yaml | 77 +++++++++++++ .../enc_dec_nmt_finetune.py | 107 ++++++++++++++++++ 3 files changed, 220 insertions(+) create mode 100644 examples/nlp/machine_translation/conf/aayn_finetune.yaml create mode 100644 examples/nlp/machine_translation/enc_dec_nmt_finetune.py diff --git a/Jenkinsfile b/Jenkinsfile index 1c2a61d2e134..6f1fcf9b8966 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -1889,6 +1889,7 @@ pipeline { +exp_manager.create_checkpoint_callback=true \ +exp_manager.resume_if_exists=True \ ' + sh 'rm -rf examples/nlp/machine_translation/nmt_results' } } @@ -1978,6 +1979,41 @@ pipeline { } } } + + stage('L2: NMT Attention is All You Need Finetuning') { + when { + anyOf { + branch 'main' + changeRequest target: 'main' + } + } + failFast true + steps { + sh "cd examples/nlp/machine_translation && \ + python enc_dec_nmt_finetune.py \ + model_path=/home/TestData/nlp/nmt/toy_data/en_de_24x6_preln.nemo \ + trainer.devices=[0] \ + ~trainer.max_epochs \ + model.train_ds.src_file_name=/home/TestData/nlp/nmt/toy_data/wmt14-de-en.src \ + model.train_ds.tgt_file_name=/home/TestData/nlp/nmt/toy_data/wmt14-de-en.ref \ + model.validation_ds.src_file_name=/home/TestData/nlp/nmt/toy_data/wmt14-de-en.src \ + model.validation_ds.tgt_file_name=/home/TestData/nlp/nmt/toy_data/wmt14-de-en.src \ + model.test_ds.src_file_name=/home/TestData/nlp/nmt/toy_data/wmt14-de-en.src \ + model.test_ds.tgt_file_name=/home/TestData/nlp/nmt/toy_data/wmt14-de-en.src \ + +trainer.val_check_interval=10 \ + +trainer.limit_val_batches=1 \ + +trainer.limit_test_batches=1 \ + +trainer.max_steps=10 \ + +exp_manager.exp_dir=examples/nlp/machine_translation/nmt_finetune \ + +exp_manager.create_checkpoint_callback=True \ + +exp_manager.checkpoint_callback_params.monitor=val_sacreBLEU \ + +exp_manager.checkpoint_callback_params.mode=max \ + +exp_manager.checkpoint_callback_params.save_best_model=true \ + " + sh "rm -rf examples/nlp/machine_translation/nmt_finetune" + } + } + stage('L2: NMT with HuggingFace') { when { anyOf { diff --git a/examples/nlp/machine_translation/conf/aayn_finetune.yaml b/examples/nlp/machine_translation/conf/aayn_finetune.yaml new file mode 100644 index 000000000000..f62e2a629464 --- /dev/null +++ b/examples/nlp/machine_translation/conf/aayn_finetune.yaml @@ -0,0 +1,77 @@ +name: AttentionIsAllYouNeedFinetune +do_training: True # set to False if only preprocessing data +do_testing: False # set to True to run evaluation on test data after training +model_path: ??? + +model: + train_ds: + src_file_name: null + tgt_file_name: null + use_tarred_dataset: False # if true tar_file_name and meta_file_name will be used (or created automatically) + # config for preprocessing training data and creating a tarred datset automatically + tar_file_prefix: parallel # prefix for tar file names + tar_files: null # if data has already been preprocessed (rest of config ignored) + metadata_file: null # metadata for tarred dataset + lines_per_dataset_fragment: 1000000 # Number of lines to consider for bucketing and padding + num_batches_per_tarfile: 100 # Number of batches (pickle files) within each tarfile + tar_shuffle_n: 100 # How many samples to look ahead and load to be shuffled + shard_strategy: scatter # tarred dataset shard distribution strategy + n_preproc_jobs: -2 # number of processes to use for data preprocessing (-2 means all but 2) + tokens_in_batch: 512 + clean: true + max_seq_length: 512 + shuffle: true + num_samples: -1 + drop_last: false + pin_memory: false + num_workers: 8 + concat_sampling_technique: temperature # only used with ConcatTranslationDataset + concat_sampling_temperature: 5 # only used with ConcatTranslationDataset + concat_sampling_probabilities: null # only used with ConcatTranslationDataset + + validation_ds: + src_file_name: ??? + tgt_file_name: ??? + tokens_in_batch: 512 + clean: false + max_seq_length: 512 + shuffle: false + num_samples: -1 + drop_last: false + pin_memory: false + num_workers: 8 + + test_ds: + src_file_name: ??? + tgt_file_name: ??? + tokens_in_batch: 512 + clean: false + max_seq_length: 512 + shuffle: false + num_samples: -1 + drop_last: false + pin_memory: false + num_workers: 8 + + optim: + name: adam + lr: 0.00002 + betas: + - 0.9 + - 0.98 + weight_decay: 0.0 + +trainer: + devices: 4 + num_nodes: 1 + max_epochs: 200 + precision: 16 # Should be set to 16 for O1 and O2, default is 16 as PT ignores it when am_level is O0 + accelerator: gpu + enable_checkpointing: False + logger: False + log_every_n_steps: 50 # Interval of logging. + check_val_every_n_epoch: 1 + +exp_manager: + name: AAYNBaseFineTune + files_to_copy: [] diff --git a/examples/nlp/machine_translation/enc_dec_nmt_finetune.py b/examples/nlp/machine_translation/enc_dec_nmt_finetune.py new file mode 100644 index 000000000000..a67067beb455 --- /dev/null +++ b/examples/nlp/machine_translation/enc_dec_nmt_finetune.py @@ -0,0 +1,107 @@ +# Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from dataclasses import dataclass +from typing import Optional + +from omegaconf import OmegaConf +from omegaconf.omegaconf import MISSING +from pytorch_lightning import Trainer + +from nemo.collections.nlp.data.machine_translation.preproc_mt_data import MTDataPreproc +from nemo.collections.nlp.models.machine_translation.mt_enc_dec_config import MTEncDecModelConfig +from nemo.collections.nlp.models.machine_translation.mt_enc_dec_model import MTEncDecModel +from nemo.collections.nlp.parts.nlp_overrides import NLPDDPPlugin +from nemo.core.config import hydra_runner +from nemo.core.config.modelPT import NemoConfig +from nemo.core.config.pytorch_lightning import TrainerConfig +from nemo.utils import logging +from nemo.utils.config_utils import update_model_config +from nemo.utils.exp_manager import ExpManagerConfig, exp_manager + + +""" +Usage: + python enc_dec_nmt_finetune.py \ + model_path=/raid/models/de_en_24x6.nemo \ + trainer.devices=2 \ + ~trainer.max_epochs \ + +trainer.max_steps=4500 \ + +trainer.val_check_interval=500 \ + model.train_ds.tgt_file_name=/raid/data/train_lang_filtered.en \ + model.train_ds.src_file_name=/raid/data/train_lang_filtered.de \ + model.train_ds.tokens_in_batch=6000 \ + model.validation_ds.tgt_file_name=/raid/data/2015.norm.tok.en \ + model.validation_ds.src_file_name=/raid/data/2015.norm.tok.de \ + model.validation_ds.tokens_in_batch=4000 \ + model.test_ds.tgt_file_name=/raid/data/2015.en \ + model.test_ds.src_file_name=/raid/data/2015.de \ + +exp_manager.exp_dir=/raid/results/finetune-test \ + +exp_manager.create_checkpoint_callback=True \ + +exp_manager.checkpoint_callback_params.monitor=val_sacreBLEU \ + +exp_manager.checkpoint_callback_params.mode=max \ + +exp_manager.checkpoint_callback_params.save_best_model=true +""" + + +@dataclass +class MTFineTuneConfig(NemoConfig): + name: Optional[str] = 'MTEncDec' + model_path: str = MISSING + do_training: bool = True + do_testing: bool = False + model: MTEncDecModelConfig = MTEncDecModelConfig() + trainer: Optional[TrainerConfig] = TrainerConfig() + exp_manager: Optional[ExpManagerConfig] = ExpManagerConfig(name='MTEncDec', files_to_copy=[]) + + +@hydra_runner(config_path="conf", config_name="aayn_finetune") +def main(cfg: MTFineTuneConfig) -> None: + # merge default config with user specified config + default_cfg = MTFineTuneConfig() + default_cfg.model = MTEncDecModel.restore_from(restore_path=cfg.model_path, return_config=True) + del default_cfg.model.optim, default_cfg.model.train_ds, default_cfg.model.validation_ds, default_cfg.model.test_ds + cfg = update_model_config(default_cfg, cfg, drop_missing_subconfigs=False) + logging.info("\n\n************** Experiment configuration ***********") + logging.info(f'Config: {OmegaConf.to_yaml(cfg)}') + + # training is managed by PyTorch Lightning + trainer_cfg = OmegaConf.to_container(cfg.trainer) + trainer_cfg.pop('plugins', None) + trainer = Trainer(plugins=[NLPDDPPlugin()], **trainer_cfg) + + # experiment logs, checkpoints, and auto-resume are managed by exp_manager and PyTorch Lightning + exp_manager(trainer, cfg.exp_manager) + + # everything needed to train translation models is encapsulated in the NeMo MTEncdDecModel + mt_model = MTEncDecModel.restore_from(restore_path=cfg.model_path, override_config_path=cfg.model, trainer=trainer) + + mt_model.setup_training_data(cfg.model.train_ds) + mt_model.setup_multiple_validation_data(val_data_config=cfg.model.validation_ds) + + logging.info("\n\n************** Model parameters and their sizes ***********") + for name, param in mt_model.named_parameters(): + print(name, param.size()) + logging.info("***********************************************************\n\n") + + if cfg.do_training: + trainer.fit(mt_model) + + if cfg.do_testing: + mt_model.setup_multiple_test_data(test_data_config=cfg.model.test_ds) + trainer.test(mt_model) + + +if __name__ == '__main__': + main() From 8b67ec6549dd667598e3434df71768a4d6272a9e Mon Sep 17 00:00:00 2001 From: Eric Harper Date: Wed, 13 Jul 2022 14:48:41 -0600 Subject: [PATCH 25/52] Add nsys profiling (#4539) * add nsys profiling Signed-off-by: ericharper * only access omegaconf in setup Signed-off-by: ericharper * use robust get_rank function Signed-off-by: ericharper * simplify Signed-off-by: ericharper --- .../conf/megatron_gpt_config.yaml | 8 ++ nemo/core/classes/modelPT.py | 74 ++++++++++++++++++- nemo/utils/get_rank.py | 12 +++ 3 files changed, 92 insertions(+), 2 deletions(-) diff --git a/examples/nlp/language_modeling/conf/megatron_gpt_config.yaml b/examples/nlp/language_modeling/conf/megatron_gpt_config.yaml index e1cbc2c7ca6e..19455a5d2c9c 100755 --- a/examples/nlp/language_modeling/conf/megatron_gpt_config.yaml +++ b/examples/nlp/language_modeling/conf/megatron_gpt_config.yaml @@ -117,6 +117,14 @@ model: reset_position_ids: False # Reset position ids after end-of-document token reset_attention_mask: False # Reset attention mask after end-of-document token eod_mask_loss: False # Mask loss for the end of document tokens + + # Nsys profiling options + nsys_profile: + enabled: False + start_step: 10 # Global batch to start profiling + end_step: 10 # Global batch to end profiling + ranks: [0] # Global rank IDs to profile + gen_shape: False # Generate model and kernel details including input shapes optim: name: fused_adam diff --git a/nemo/core/classes/modelPT.py b/nemo/core/classes/modelPT.py index 96ea2931e34b..ec29b954ced4 100644 --- a/nemo/core/classes/modelPT.py +++ b/nemo/core/classes/modelPT.py @@ -19,7 +19,7 @@ from abc import abstractmethod from os import path from pathlib import Path -from typing import Callable, Dict, List, Optional, Union +from typing import Any, Callable, Dict, List, Optional, Union import hydra import torch @@ -35,7 +35,7 @@ from nemo.utils import logging, model_utils from nemo.utils.app_state import AppState from nemo.utils.debug_hook import register_debug_hooks -from nemo.utils.get_rank import is_global_rank_zero +from nemo.utils.get_rank import get_rank, is_global_rank_zero try: from nemo.collections.nlp.parts.nlp_overrides import NLPDDPPlugin @@ -168,6 +168,9 @@ def __init__(self, cfg: DictConfig, trainer: Trainer = None): # ModelPT wrappers over subclass implementations self.training_step = model_utils.wrap_training_step(self.training_step) + # Setup nsys profiling if it has been enabled in the model config + self._setup_nsys_profiling() + def __init_subclass__(cls) -> None: cls._save_restore_connector = SaveRestoreConnector() @@ -1400,3 +1403,70 @@ def update_save_restore_connector(cls, save_restore_connector): cls._save_restore_connector = save_restore_connector else: setattr(cls, '_save_restore_connector', save_restore_connector) + + def _setup_nsys_profiling(self): + """ Enables nsys profiling + To use, add the following optoins to the model config: + ## Nsys profiling options + nsys_profile: False + start_step: 10 # Global batch to start profiling + end_step: 10 # Global batch to end profiling + ranks: [0] # Global rank IDs to profile + gen_shape: False # Generate model and kernel details including input shapes + And then wrap the model training script with: + nsys profile -s none -o -t cuda,nvtx --force-overwrite true --capture-range=cudaProfilerApi --capture-range-end=stop python ./examples/... + See more options at: https://docs.nvidia.com/nsight-systems/UserGuide/index.html#cli-profiling + """ + if self.cfg.get('nsys_profile', None) is not None: + if self.cfg.nsys_profile.get('enabled', False): + # Nsys profiling options + self._nsys_profile_enabled = True + self._nsys_profile_start_step = self.cfg.nsys_profile.get('start_step', 0) + self._nsys_profile_end_step = self.cfg.nsys_profile.get('end_step', 0) + self._nsys_profile_ranks = self.cfg.nsys_profile.get('ranks', [0]) + self._nsys_profile_gen_shape = self.cfg.nsys_profile.get('gen_shape', False) + + if type(self._nsys_profile_start_step) == int: + logging.info(f'Nsys profiling setup with start_step: {self._nsys_profile_start_step}') + else: + raise ValueError( + f'Nsys start_step must be of type int. Found: {type(self._nsys_profile_start_step)}' + ) + + if type(self._nsys_profile_end_step) == int: + logging.info(f'Nsys profiling setup with end_step: {self._nsys_profile_end_step}') + else: + raise ValueError(f'Nsys end_step must be of type int. Found: {type(self._nsys_profile_end_step)}') + + if self._nsys_profile_end_step >= self._nsys_profile_start_step: + pass + else: + raise ValueError(f'Nsys end_step must be greater than or equal to nsys start_step') + + def on_train_batch_start(self, batch: Any, batch_idx: int, unused: int = 0) -> Optional[int]: + """ PyTorch Lightning hook: + https://pytorch-lightning.readthedocs.io/en/stable/common/lightning_module.html#on-train-batch-start + We use it here to enable nsys profiling. + """ + + if self.device.type == 'cuda': + if hasattr(self, '_nsys_profile_enabled'): + if self._nsys_profile_enabled: + if batch_idx == self._nsys_profile_start_step and get_rank() in self._nsys_profile_ranks: + logging.info("====== Start nsys profiling ======") + torch.cuda.cudart().cudaProfilerStart() + if self._nsys_profile_gen_shape: + torch.autograd.profiler.emit_nvtx(record_shapes=True).__enter__() + + def on_train_batch_end(self, outputs, batch: Any, batch_idx: int, unused: int = 0) -> None: + """ PyTorch Lightning hook: + https://pytorch-lightning.readthedocs.io/en/stable/common/lightning_module.html#on-train-batch-end + We use it here to enable nsys profiling. + """ + + if self.device.type == 'cuda': + if hasattr(self, '_nsys_profile_enabled'): + if self._nsys_profile_enabled: + if batch_idx == self._nsys_profile_end_step and get_rank() in self._nsys_profile_ranks: + logging.info("====== End nsys profiling ======") + torch.cuda.cudart().cudaProfilerStop() diff --git a/nemo/utils/get_rank.py b/nemo/utils/get_rank.py index 95d599c620eb..9b36eed6246b 100644 --- a/nemo/utils/get_rank.py +++ b/nemo/utils/get_rank.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +import torch + from nemo.utils.env_var_parsing import get_envint @@ -36,3 +38,13 @@ def is_global_rank_zero(): node_rank = get_envint("NODE_RANK", get_envint("GROUP_RANK", 0)) local_rank = get_envint("LOCAL_RANK", 0) return node_rank == 0 and local_rank == 0 + + +def get_rank(): + """ Helper function that returns torch.distributed.get_rank() if DDP has been initialized otherwise it returns 0. + """ + + if is_global_rank_zero(): + return 0 + else: + return torch.distributed.get_rank() From 4e43b7c85f2dda668d69b68bb3343b376ed6b98d Mon Sep 17 00:00:00 2001 From: Zhilin Wang Date: Wed, 13 Jul 2022 18:05:54 -0700 Subject: [PATCH 26/52] Update megatron prompt learning interface to dialogue (#4545) * refactor dialogue state tracking for modelling/dataset interoperability Signed-off-by: Zhilin Wang * fix style changes Signed-off-by: Zhilin Wang * fix typo Signed-off-by: Zhilin Wang * fix style raised by lgtm Signed-off-by: Zhilin Wang * fix style formatting Signed-off-by: Zhilin Wang * update template to include description of intent Signed-off-by: Zhilin Wang * update Jenkinsfile Signed-off-by: Zhilin Wang * changes based on requests in review Signed-off-by: Zhilin Wang * add compatibility with assistant dataset Signed-off-by: Zhilin Wang * update Jenkins Signed-off-by: Zhilin Wang * remove dialogue_state_tracking Signed-off-by: Zhilin Wang * update huggingface utils for dialogue Signed-off-by: Zhilin Wang * rename dialogue_state_tracking_hybrid to dialogue_state_tracking_sgdqa Signed-off-by: Zhilin Wang * style fix Signed-off-by: Zhilin Wang * fix style Signed-off-by: Zhilin Wang * style fix nemo/collections/nlp/models/dialogue_state_tracking_sgdqa/__init__.py Signed-off-by: Zhilin Wang * update Jenkinsfile for SGDGEN Signed-off-by: Zhilin Wang * update Jenkinsfile for SGDGEN Signed-off-by: Zhilin Wang * update Jenkinsfile for SGDGEN Signed-off-by: Zhilin Wang * update Jenkinsfile for SGDGEN Signed-off-by: Zhilin Wang * update Jenkinsfile for SGDGEN Signed-off-by: Zhilin Wang * fix typo Signed-off-by: Zhilin Wang * add docstrings for assistant data processsor Signed-off-by: Zhilin Wang * update Jenkins for SGDGEN local checkpoint Signed-off-by: Zhilin Wang * update style Signed-off-by: Zhilin Wang * use local vocab file for Jenkinsfile Signed-off-by: Zhilin Wang * patch for Jenkins CI using local file Signed-off-by: Zhilin Wang * add slot filling prediction and metrics Signed-off-by: Zhilin Wang * remove unused code Signed-off-by: Zhilin Wang * style fix Signed-off-by: Zhilin Wang * refactor metrics code out of Dialogue GPT Model Signed-off-by: Zhilin Wang * integrate backward compatible support for IntentSlotClassificationModel (bert model) Signed-off-by: Zhilin Wang * save prediction file for IntentSlotClassification Signed-off-by: Zhilin Wang * update dialogue gpt model training for megatron gpt Signed-off-by: Zhilin Wang * remove batch generate for HF GPT2, which causes lower performance Signed-off-by: Zhilin Wang * add few shot capability to dialogue gpt model Signed-off-by: Zhilin Wang * update Jenkinsfile and remove unused import Signed-off-by: Zhilin Wang * update code description and clarity Signed-off-by: Zhilin Wang * address PR comments Signed-off-by: Zhilin Wang * style fix Signed-off-by: Zhilin Wang * integrate compatibility with ZeroShotIntentModel Signed-off-by: Zhilin Wang * rename folder to dialogue due to increased scope and further refactor for clarity Signed-off-by: Zhilin Wang * added dialogue GPT for sequence generation task (e.g. answer extender) Signed-off-by: Zhilin Wang * add CI test for DialogueGPTGenerationModel Signed-off-by: Zhilin Wang * integrate DialogueS2SGenerationModel for generation task (e.g. answer extender) Signed-off-by: Zhilin Wang * modify huggingface utils to support HF t5/BART models Signed-off-by: Zhilin Wang * style fix Signed-off-by: Zhilin Wang * style fix Signed-off-by: Zhilin Wang * remove unused imports Signed-off-by: Zhilin Wang * style fix Signed-off-by: Zhilin Wang * update Jenkinsfile Signed-off-by: Zhilin Wang * update Jenkinsfile Signed-off-by: Zhilin Wang * update bleu metric Signed-off-by: Zhilin Wang * fix bleu metric style Signed-off-by: Zhilin Wang * debug bleu metric Signed-off-by: Zhilin Wang * debug bleu metric Signed-off-by: Zhilin Wang * update based on PR #3893 Signed-off-by: Zhilin Wang * update 2 based on PR #3893 Signed-off-by: Zhilin Wang * update 3 based on PR #3893 Signed-off-by: Zhilin Wang * integrate sgd generation based on user user utterance and system slot-values to generate system utterance Signed-off-by: Zhilin Wang * add validation model saving capabilities Signed-off-by: Zhilin Wang * cleaned up code for SGD Based Answer extender Signed-off-by: Zhilin Wang * update Dialogue Generation CI Signed-off-by: Zhilin Wang * update Jenkinsfile Signed-off-by: Zhilin Wang * update Jenkinsfile Signed-off-by: Zhilin Wang * fix Jenkins CI issue" Signed-off-by: Zhilin Wang * add support for design dataset Signed-off-by: Zhilin Wang * remove unnecessary imports Signed-off-by: Zhilin Wang * update Jenkins Signed-off-by: Zhilin Wang * update jenkins Signed-off-by: Zhilin Wang * update jenkins Signed-off-by: Zhilin Wang * support megatron for dialogue_s2s_generation_model Signed-off-by: Zhilin Wang * reduce loaded samples in MSMarcoDataProcessor to 64 when cfg.model.dataset.debug_mode=True Signed-off-by: Zhilin Wang * style fix Signed-off-by: Zhilin Wang * style fix Signed-off-by: Zhilin Wang * update CI Signed-off-by: Zhilin Wang * update checkpoint and predictions filename to include epoch number Signed-off-by: Zhilin Wang * style fix Signed-off-by: Zhilin Wang * integrate HF BART MNLI into zero shot intent model Signed-off-by: Zhilin Wang * integrate Dialogue Nearest Neighbour Model Signed-off-by: Zhilin Wang * update Jenkins Signed-off-by: Zhilin Wang * update Jenkins Signed-off-by: Zhilin Wang * refactor Dialogue SGD Data Processor to make interface for models cleaner Signed-off-by: Zhilin Wang * update jenkins Signed-off-by: Zhilin Wang * update Dialogue S2S Generation model for DialogueSGDDataProcessor interface Signed-off-by: Zhilin Wang * update jenkins Signed-off-by: Zhilin Wang * update jenkins Signed-off-by: Zhilin Wang * support sgd and drive thru datasets by zero shot model and nearest neighbour model Signed-off-by: Zhilin Wang * add prediction saving code to nearest neighbour and zero shot intent models Signed-off-by: Zhilin Wang * fix typo in sgd data processor Signed-off-by: Zhilin Wang * integrate Dialogue Mellon QA Data Processor Signed-off-by: Zhilin Wang * update mellon qa Signed-off-by: Zhilin Wang * update dialogue.py to remove outdated info Signed-off-by: Zhilin Wang * style fix Signed-off-by: Zhilin Wang * update dialogue_config.yaml Signed-off-by: Zhilin Wang * update dialogue_config.yaml Signed-off-by: Zhilin Wang * add dialogue docs Signed-off-by: Zhilin Wang * address review comments Signed-off-by: Zhilin Wang * style fix Signed-off-by: Zhilin Wang * style fix Signed-off-by: Zhilin Wang * style fix Signed-off-by: Zhilin Wang * style fix Signed-off-by: Zhilin Wang * style fix Signed-off-by: Zhilin Wang * style fix for cfg Signed-off-by: Zhilin Wang * make dependency on apex optional Signed-off-by: Zhilin Wang * change NLPDDPluggin calling logic to make it possible to run without apex Signed-off-by: Zhilin Wang * add first draft of tutorial Signed-off-by: Zhilin Wang * reduce ms marco size by removing lines without wellFormedAnswers Signed-off-by: Zhilin Wang * address pr comments Signed-off-by: Zhilin Wang * style fix Signed-off-by: Zhilin Wang * update colab tutorial link in dialogue docs Signed-off-by: Zhilin Wang * include unit test and some refactor to facilitate unit test Signed-off-by: Zhilin Wang * style fix Signed-off-by: Zhilin Wang * address pr issues Signed-off-by: Zhilin Wang * remove typos in dialogue tutorial Signed-off-by: Zhilin Wang * support larger files for question answering Signed-off-by: Zhilin Wang * style fix Signed-off-by: Zhilin Wang * style fix Signed-off-by: Zhilin Wang * style fix Signed-off-by: Zhilin Wang * remove unnecessary artifacts to reduce memory use Signed-off-by: Zhilin Wang * put 0 tensor to device Signed-off-by: Zhilin Wang * update link within dialogue tutorial Signed-off-by: Zhilin Wang * restore previously delete files Signed-off-by: Zhilin Wang * update error handling when loss = nan Signed-off-by: Zhilin Wang * update nan handling Signed-off-by: Zhilin Wang * style fix Signed-off-by: Zhilin Wang * update spanning loss func Signed-off-by: Zhilin Wang * update spanning loss Signed-off-by: Zhilin Wang * fix type error raised in qa_dataset.py Signed-off-by: Zhilin Wang * add error checking message Signed-off-by: Zhilin Wang * revert back to float32 Signed-off-by: Zhilin Wang * revert back to float32 Signed-off-by: Zhilin Wang * update error msgs Signed-off-by: Zhilin Wang * update error msgs Signed-off-by: Zhilin Wang * update error msgs Signed-off-by: Zhilin Wang * update error msgs Signed-off-by: Zhilin Wang * update error msgs Signed-off-by: Zhilin Wang * update error msgs Signed-off-by: Zhilin Wang * update error msgs Signed-off-by: Zhilin Wang * update error msgs Signed-off-by: Zhilin Wang * update exp logging Signed-off-by: Zhilin Wang * update error msgs Signed-off-by: Zhilin Wang * update loading of large file from pickle to json Signed-off-by: Zhilin Wang * update loading of large file from pickle to json Signed-off-by: Zhilin Wang * limit number of negative samples Signed-off-by: Zhilin Wang * revert post processing Signed-off-by: Zhilin Wang * revert post processing Signed-off-by: Zhilin Wang * remove unused methods and style fix Signed-off-by: Zhilin Wang * add more documentation Signed-off-by: Zhilin Wang * remove unused imports Signed-off-by: Zhilin Wang * changes base on PR review Signed-off-by: Zhilin Wang * set wandb logger falseby default Signed-off-by: Zhilin Wang * update interface with megatron gpt prompt learning Signed-off-by: Zhilin Wang * update inline documentation Signed-off-by: Zhilin Wang * style fix Signed-off-by: Zhilin Wang * style fix Signed-off-by: Zhilin Wang * update prompt_ids Signed-off-by: Zhilin Wang * update error msg Signed-off-by: Zhilin Wang * update config Signed-off-by: Zhilin Wang * update config Signed-off-by: Zhilin Wang * set inference = False for dialgue prompt learning during trainng Signed-off-by: Zhilin Wang * set inference = False for dialgue prompt learning during trainng Signed-off-by: Zhilin Wang * remove unused code Signed-off-by: Zhilin Wang * update config yaml Signed-off-by: Zhilin Wang * fix bug for megatron gpt prompt learning Signed-off-by: Zhilin Wang * remove unused import Signed-off-by: Zhilin Wang * address comments in PR Signed-off-by: Zhilin Wang * address comments in PR Signed-off-by: Zhilin Wang * address typo Signed-off-by: Zhilin Wang Co-authored-by: Zhilin Wang Co-authored-by: Oleksii Kuchaiev Co-authored-by: Yang Zhang Co-authored-by: Eric Harper Co-authored-by: Sandeep Subramanian --- .../nlp/dialogue/conf/dialogue_config.yaml | 29 ++++-- examples/nlp/dialogue/dialogue.py | 10 +- .../dialogue_gpt_classification_model.py | 94 +++++++++++-------- .../dialogue/dialogue_gpt_generation_model.py | 94 +++++++++++-------- .../megatron_gpt_prompt_learning_model.py | 3 +- .../nlp/modules/common/prompt_table.py | 1 + 6 files changed, 139 insertions(+), 92 deletions(-) diff --git a/examples/nlp/dialogue/conf/dialogue_config.yaml b/examples/nlp/dialogue/conf/dialogue_config.yaml index 5e4b4750e6c2..11844a41f144 100644 --- a/examples/nlp/dialogue/conf/dialogue_config.yaml +++ b/examples/nlp/dialogue/conf/dialogue_config.yaml @@ -52,11 +52,6 @@ model: # Dialogue GPT Classification/Generation and Dialogue S2S Generation Model args tokens_to_generate: 32 # for generation mode only - # Dialogue GPT Classification megatron prompt tuning model args - new_prompt_tags: [] # brief short name (each element is a string "") - new_prompt_init_methods: [] # text (preferably) or random - new_prompt_init_text: [] # long descriptive text to init - # Intent Slot Classification model args class_balancing: ${model.dataset.class_balancing} intent_loss_weight: 0.6 # relation of intent to slot loss in total loss (between 0 to 1) @@ -65,6 +60,28 @@ model: num_output_layers: 2 fc_dropout: 0.1 + # Dialogue GPT Classification Magetron Prompt Learning model args + prompt_learning: false # please change to true to activate prompt learning + language_model_path: ${model.language_model.lm_checkpoint} + new_tasks: ['intent_and_slot'] + prompt_tuning: + new_prompt_init_methods: ['text'] + new_prompt_init_text: ['intent classification and slot filling'] + data: {} + virtual_prompt_style: 'prompt-tuning' #'p-tuning' + encoder_seq_length: 2048 + pipeline_model_parallel_size: 1 + global_batch_size: 8 + micro_batch_size: 8 + + task_templates: + - taskname: "intent_and_slot" + prompt_template: "<|VIRTUAL_PROMPT_0|> {utterance} \nintent: {intent} \nslot: {slot}" + total_virtual_tokens: 10 + answer_only_loss: True + virtual_token_splits: [10] + truncate_field: null + # SGDQA args encoder: dropout: 0.1 @@ -180,4 +197,4 @@ exp_manager: create_tensorboard_logger: True # Whether you want exp_manger to create a tb logger create_checkpoint_callback: True # Whether you want exp_manager to create a modelcheckpoint callback resume_if_exists: false - resume_ignore_no_checkpoint: false + resume_ignore_no_checkpoint: false \ No newline at end of file diff --git a/examples/nlp/dialogue/dialogue.py b/examples/nlp/dialogue/dialogue.py index cc0808ab3f6f..e3ce88e33275 100644 --- a/examples/nlp/dialogue/dialogue.py +++ b/examples/nlp/dialogue/dialogue.py @@ -66,11 +66,11 @@ def main(cfg: DictConfig) -> None: logging.info(f'Config: {OmegaConf.to_yaml(cfg)}') try: - plugin = NLPDDPPlugin() + plugins = NLPDDPPlugin() except (ImportError, ModuleNotFoundError): - plugin = None + plugins = None - trainer = pl.Trainer(**cfg.trainer, plugins=plugin) + trainer = pl.Trainer(**cfg.trainer, plugins=plugins) exp_manager(trainer, cfg.get("exp_manager", None)) @@ -138,7 +138,9 @@ def main(cfg: DictConfig) -> None: if hasattr(cfg.model, 'test_ds') and cfg.model.test_ds.ds_item is not None: eval_device = [cfg.trainer.devices[0]] if isinstance(cfg.trainer.devices, list) else 1 - trainer = pl.Trainer(devices=eval_device, accelerator=cfg.trainer.accelerator, precision=16) + trainer = pl.Trainer( + devices=eval_device, accelerator=cfg.trainer.accelerator, precision=16, plugins=NLPDDPPlugin() + ) model.setup_multiple_test_data(test_data_config=cfg.model.test_ds) if model.prepare_test(trainer): trainer.test(model) diff --git a/nemo/collections/nlp/models/dialogue/dialogue_gpt_classification_model.py b/nemo/collections/nlp/models/dialogue/dialogue_gpt_classification_model.py index 82269534613a..caecb5b6c405 100644 --- a/nemo/collections/nlp/models/dialogue/dialogue_gpt_classification_model.py +++ b/nemo/collections/nlp/models/dialogue/dialogue_gpt_classification_model.py @@ -14,6 +14,7 @@ # limitations under the License. import collections +import copy import os import random from typing import Dict, Optional, Union @@ -31,6 +32,9 @@ from nemo.collections.nlp.metrics.classification_report import ClassificationReport from nemo.collections.nlp.metrics.dialogue_metrics import DialogueClassificationMetrics from nemo.collections.nlp.models.language_modeling.megatron_gpt_model import MegatronGPTModel +from nemo.collections.nlp.models.language_modeling.megatron_gpt_prompt_learning_model import ( + MegatronGPTPromptLearningModel, +) from nemo.collections.nlp.models.nlp_model import NLPModel from nemo.core.classes.common import PretrainedModelInfo from nemo.utils import logging @@ -53,25 +57,14 @@ def __init__( self.language_model = AutoModelWithLMHead.from_pretrained(cfg.language_model.pretrained_model_name) self.language_model.resize_token_embeddings(len(self.tokenizer.tokenizer)) elif self.cfg.library == "megatron": - self.language_model = MegatronGPTModel.restore_from(cfg.language_model.lm_checkpoint, trainer=trainer) - # 1 corresponds to intent slot; 0 corresponds to squad - self.prompt_tags = [1, 0] if 'prompt_table' in dir(self.language_model) else [] - if hasattr(self.language_model, 'prompt_table'): - self.language_model.prompt_tuning_param_freeze_and_optimizer_setup() - - # Init all new prompts - for idx, tag in enumerate(cfg.new_prompt_tags): - self.prompt_tags.append(tag) - init_method = cfg.new_prompt_init_methods[idx] - if init_method == "text": - init_text = cfg.new_prompt_init_text[idx] - self.language_model.init_prompt_from_text(tag, init_text) - elif init_method == 'random': - self.language_model.init_prompt_from_random(tag) - else: - raise ValueError( - f'\n Soft prompt init method {init_method} is not recognized, please use text or random' - ) + self.prompt_learning = self.cfg.prompt_learning + if self.prompt_learning: + # removing tokenizer cfg as this triggers tokenizer construction which is not helpful here as we have a separate tokenizer + new_cfg = copy.copy(cfg) + del new_cfg.tokenizer + self.language_model = MegatronGPTPromptLearningModel(new_cfg, trainer) + else: + self.language_model = MegatronGPTModel.restore_from(cfg.language_model.lm_checkpoint, trainer=trainer) all_labels = list( self._train_dl.dataset.all_possible_labels.union( @@ -136,7 +129,7 @@ def training_step(self, batch, batch_idx): attn_masks = torch.stack(new_attn_masks) labels = self.get_binary_score_labels(input_ids) - loss = self(input_ids, attn_masks, labels) + loss = self(input_ids, attn_masks, labels, inference=False) self.log("train_loss", loss, on_step=True, on_epoch=True, prog_bar=True, logger=True) return {'loss': loss} @@ -231,7 +224,7 @@ def predict_step(self, batch, batch_idx, dataloader_idx=None): # return self(batch) raise NotImplementedError() - def forward(self, input_ids, attention_mask, labels): + def forward(self, input_ids, attention_mask, labels, inference=True): if self.cfg.library == "huggingface": output = self.language_model(input_ids=input_ids, attention_mask=attention_mask, labels=labels) @@ -239,24 +232,17 @@ def forward(self, input_ids, attention_mask, labels): elif self.cfg.library == "megatron": num_prompt_tokens = ( - self.language_model.num_prompt_tokens if hasattr(self.language_model, 'num_prompt_tokens') else 0 + len(self.language_model.pseudo_token_ids) if hasattr(self.language_model, 'pseudo_token_ids') else 0 ) + position_ids = torch.arange( - start=num_prompt_tokens, - end=num_prompt_tokens + input_ids.size(1), - dtype=torch.long, - device=input_ids.device, + start=0, end=num_prompt_tokens + input_ids.size(1), dtype=torch.long, device=input_ids.device, ) position_ids = position_ids.unsqueeze(0).repeat(input_ids.size(0), 1) - # 'assit_intent_and_slot' has prompt_id of 1 - # 'assit_intent_and_slot_with_options' has prompt_id of 2 - prompt_ids = torch.tensor([1] * input_ids.size(0)) if self.prompt_tags else None + prompt_ids = torch.tensor([0] * input_ids.size(0)) if self.prompt_learning else None - # this makes a 1d tensor of values 2 rather than 1, which is the prompt_id of 'assit_intent_and_slot_with_options' - if self.cfg.dataset.prompt_template == "prompt_tuning_with_options" and prompt_ids is not None: - prompt_ids = prompt_ids * 2 attn_mask_add_on = torch.ones((attention_mask.size(0), num_prompt_tokens), device=attention_mask.device) full_attention_mask = torch.cat([attn_mask_add_on, attention_mask], axis=-1) full_attention_mask_expand = torch.tril( @@ -269,28 +255,51 @@ def forward(self, input_ids, attention_mask, labels): size=(input_ids.size(0), num_prompt_tokens), fill_value=self.tokenizer.tokenizer.pad_token_id, dtype=torch.long, - device=input_ids.device, ) - input_ids_new = torch.cat([prompt_token_labels, input_ids], axis=1) + if self.prompt_learning: + prompt_token_labels.data = torch.LongTensor( + np.tile(np.array(self.language_model.pseudo_token_ids), (input_ids.size(0), 1)) + ) + + prompt_token_labels = prompt_token_labels.to(input_ids.device) + + input_ids_new = torch.cat([torch.zeros_like(prompt_token_labels), input_ids], axis=1) make_up_last_column_input_ids = ( torch.ones_like(input_ids_new[:, -1:]) * self.tokenizer.tokenizer.pad_token_id ) left_shifted_input_ids = torch.cat([input_ids_new[:, 1:], make_up_last_column_input_ids], axis=-1) + if self.prompt_learning: + unmasked_unreduced_loss = self.language_model( + input_ids_new, + position_ids, + attn_mask, + labels=left_shifted_input_ids, + taskname_ids=prompt_ids, + inference=inference, + ) + else: + unmasked_unreduced_loss = self.language_model( + input_ids, position_ids, attn_mask, labels=left_shifted_input_ids + ) - unmasked_unreduced_loss = self.language_model( - input_ids, position_ids, attn_mask, left_shifted_input_ids, prompt_ids=prompt_ids - ) + if isinstance(unmasked_unreduced_loss, tuple): + unmasked_unreduced_loss = unmasked_unreduced_loss[0] - labels = torch.cat([torch.zeros_like(prompt_token_labels), labels], axis=1) + labels = torch.cat([prompt_token_labels, labels], axis=1) make_up_last_column_labels = torch.ones_like(labels[:, -1:]) * self.tokenizer.tokenizer.pad_token_id new_labels = torch.cat([labels[:, 1:], make_up_last_column_labels], axis=-1) filler = torch.zeros_like(new_labels) labels_mask_0 = torch.where(new_labels != -100, new_labels, filler) labels_mask = labels_mask_0 > 0 - loss = self.language_model.loss_func(labels_mask, unmasked_unreduced_loss) + loss = self.mask_and_reduce_loss(labels_mask, unmasked_unreduced_loss) + return loss + def mask_and_reduce_loss(self, loss_mask, output_tensor): + losses = output_tensor.float() + loss_mask = loss_mask.view(-1).float() + loss = torch.sum(losses.view(-1) * loss_mask) / loss_mask.sum() return loss def decode(self, tokens): @@ -417,7 +426,7 @@ def prepare_megatron_generation(self, labels, input_ids, template_length): # adapted from MegatronGPTModel._bucketize_gpt_inference """ batch_size = labels.size(0) - prompt_tags = [self.prompt_tags[0]] * batch_size if self.prompt_tags else None + prompt_tags = [self.prompt_tags[0]] * batch_size if self.prompt_learning else None batch_tokens = input_ids.tolist() # unpad tokens @@ -602,6 +611,11 @@ def prepare_data(self): self.data_prepared = True + def setup(self, stage=None): + super().setup() + if self.cfg.library == "megatron" and self.prompt_learning: + self.language_model.init_new_prompts() + def update_data_dirs(self, data_dir: str, dialogues_example_dir: str): """ Update data directories diff --git a/nemo/collections/nlp/models/dialogue/dialogue_gpt_generation_model.py b/nemo/collections/nlp/models/dialogue/dialogue_gpt_generation_model.py index 8da37047e203..525207c84f0d 100644 --- a/nemo/collections/nlp/models/dialogue/dialogue_gpt_generation_model.py +++ b/nemo/collections/nlp/models/dialogue/dialogue_gpt_generation_model.py @@ -13,6 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +import copy import os from typing import Dict, Optional, Union @@ -28,6 +29,9 @@ from nemo.collections.nlp.data.dialogue.dataset.dialogue_gpt_generation_dataset import DialogueGPTGenerationDataset from nemo.collections.nlp.metrics.dialogue_metrics import DialogueGenerationMetrics from nemo.collections.nlp.models.language_modeling.megatron_gpt_model import MegatronGPTModel +from nemo.collections.nlp.models.language_modeling.megatron_gpt_prompt_learning_model import ( + MegatronGPTPromptLearningModel, +) from nemo.collections.nlp.models.nlp_model import NLPModel from nemo.core.classes.common import PretrainedModelInfo from nemo.utils import logging @@ -56,30 +60,19 @@ def __init__( if self.cfg.language_model.lm_checkpoint: self.language_model.load_state_dict(torch.load(self.cfg.language_model.lm_checkpoint)) elif self.cfg.library == "megatron": - self.language_model = MegatronGPTModel.restore_from(cfg.language_model.lm_checkpoint, trainer=trainer) - # 1 corresponds to intent slot; 0 corresponds to squad - self.prompt_tags = [1, 0] if 'prompt_table' in dir(self.language_model) else [] - if hasattr(self.language_model, 'prompt_table'): - self.language_model.prompt_tuning_param_freeze_and_optimizer_setup() - - # Init all new prompts - for idx, tag in enumerate(cfg.new_prompt_tags): - self.prompt_tags.append(tag) - init_method = cfg.new_prompt_init_methods[idx] - if init_method == "text": - init_text = cfg.new_prompt_init_text[idx] - self.language_model.init_prompt_from_text(tag, init_text) - elif init_method == 'random': - self.language_model.init_prompt_from_random(tag) - else: - raise ValueError( - f'\n Soft prompt init method {init_method} is not recognized, please use text or random' - ) + self.prompt_learning = self.cfg.prompt_learning + if self.prompt_learning: + # removing tokenizer cfg as this triggers tokenizer construction which is not helpful here as we have a separate tokenizer + new_cfg = copy.copy(cfg) + del new_cfg.tokenizer + self.language_model = MegatronGPTPromptLearningModel(new_cfg, trainer) + else: + self.language_model = MegatronGPTModel.restore_from(cfg.language_model.lm_checkpoint, trainer=trainer) def training_step(self, batch, batch_idx): input_ids, attn_masks, labels, _, _ = batch - loss = self(input_ids, attn_masks, labels) + loss = self(input_ids, attn_masks, labels, inference=False) self.log("train_loss", loss, on_step=True, on_epoch=True, prog_bar=True, logger=True) return {'loss': loss} @@ -144,7 +137,7 @@ def predict_step(self, batch, batch_idx, dataloader_idx=None): # return self(batch) raise NotImplementedError() - def forward(self, input_ids, attention_mask, labels): + def forward(self, input_ids, attention_mask, labels, inference=True): if self.cfg.library == "huggingface": output = self.language_model(input_ids=input_ids, attention_mask=attention_mask, labels=labels) @@ -152,24 +145,17 @@ def forward(self, input_ids, attention_mask, labels): elif self.cfg.library == "megatron": num_prompt_tokens = ( - self.language_model.num_prompt_tokens if hasattr(self.language_model, 'num_prompt_tokens') else 0 + len(self.language_model.pseudo_token_ids) if hasattr(self.language_model, 'pseudo_token_ids') else 0 ) + position_ids = torch.arange( - start=num_prompt_tokens, - end=num_prompt_tokens + input_ids.size(1), - dtype=torch.long, - device=input_ids.device, + start=0, end=num_prompt_tokens + input_ids.size(1), dtype=torch.long, device=input_ids.device, ) position_ids = position_ids.unsqueeze(0).repeat(input_ids.size(0), 1) - # 'assit_intent_and_slot' has prompt_id of 1 - # 'assit_intent_and_slot_with_options' has prompt_id of 2 - prompt_ids = torch.tensor([1] * input_ids.size(0)) if self.prompt_tags else None + prompt_ids = torch.tensor([0] * input_ids.size(0)) if self.prompt_learning else None - # this makes a 1d tensor of values 2 rather than 1, which is the prompt_id of 'assit_intent_and_slot_with_options' - if self.cfg.dataset.prompt_template == "prompt_tuning_with_options" and prompt_ids is not None: - prompt_ids = prompt_ids * 2 attn_mask_add_on = torch.ones((attention_mask.size(0), num_prompt_tokens), device=attention_mask.device) full_attention_mask = torch.cat([attn_mask_add_on, attention_mask], axis=-1) full_attention_mask_expand = torch.tril( @@ -182,36 +168,64 @@ def forward(self, input_ids, attention_mask, labels): size=(input_ids.size(0), num_prompt_tokens), fill_value=self.tokenizer.tokenizer.pad_token_id, dtype=torch.long, - device=input_ids.device, ) - input_ids_new = torch.cat([prompt_token_labels, input_ids], axis=1) + if self.prompt_learning: + prompt_token_labels.data = torch.LongTensor( + np.tile(np.array(self.language_model.pseudo_token_ids), (input_ids.size(0), 1)) + ) + + prompt_token_labels = prompt_token_labels.to(input_ids.device) + + input_ids_new = torch.cat([torch.zeros_like(prompt_token_labels), input_ids], axis=1) make_up_last_column_input_ids = ( torch.ones_like(input_ids_new[:, -1:]) * self.tokenizer.tokenizer.pad_token_id ) left_shifted_input_ids = torch.cat([input_ids_new[:, 1:], make_up_last_column_input_ids], axis=-1) + if self.prompt_learning: + unmasked_unreduced_loss = self.language_model( + input_ids_new, + position_ids, + attn_mask, + labels=left_shifted_input_ids, + taskname_ids=prompt_ids, + inference=inference, + ) + else: + unmasked_unreduced_loss = self.language_model( + input_ids, position_ids, attn_mask, labels=left_shifted_input_ids + ) - unmasked_unreduced_loss = self.language_model( - input_ids, position_ids, attn_mask, left_shifted_input_ids, prompt_ids=prompt_ids - ) + if isinstance(unmasked_unreduced_loss, tuple): + unmasked_unreduced_loss = unmasked_unreduced_loss[0] - labels = torch.cat([torch.zeros_like(prompt_token_labels), labels], axis=1) + labels = torch.cat([prompt_token_labels, labels], axis=1) make_up_last_column_labels = torch.ones_like(labels[:, -1:]) * self.tokenizer.tokenizer.pad_token_id new_labels = torch.cat([labels[:, 1:], make_up_last_column_labels], axis=-1) filler = torch.zeros_like(new_labels) labels_mask_0 = torch.where(new_labels != -100, new_labels, filler) labels_mask = labels_mask_0 > 0 - loss = self.language_model.loss_func(labels_mask, unmasked_unreduced_loss) + loss = self.mask_and_reduce_loss(labels_mask, unmasked_unreduced_loss) + return loss + def mask_and_reduce_loss(self, loss_mask, output_tensor): + losses = output_tensor.float() + loss_mask = loss_mask.view(-1).float() + loss = torch.sum(losses.view(-1) * loss_mask) / loss_mask.sum() return loss + def setup(self, stage=None): + super().setup() + if self.cfg.library == "megatron" and self.prompt_learning: + self.language_model.init_new_prompts() + def prepare_megatron_generation(self, labels, input_ids, template_length): """ # adapted from MegatronGPTModel._bucketize_gpt_inference """ batch_size = labels.size(0) - prompt_tags = [self.prompt_tags[0]] * batch_size if self.prompt_tags else None + prompt_tags = [self.prompt_tags[0]] * batch_size if self.prompt_learning else None batch_tokens = input_ids.tolist() # unpad tokens diff --git a/nemo/collections/nlp/models/language_modeling/megatron_gpt_prompt_learning_model.py b/nemo/collections/nlp/models/language_modeling/megatron_gpt_prompt_learning_model.py index 7cd0ab5cc886..84740433b127 100644 --- a/nemo/collections/nlp/models/language_modeling/megatron_gpt_prompt_learning_model.py +++ b/nemo/collections/nlp/models/language_modeling/megatron_gpt_prompt_learning_model.py @@ -296,7 +296,7 @@ def get_model_tasks(self): return tasks - def state_dict(self): + def state_dict(self, destination=None, prefix=None, keep_vars=False): """ Custom state dict that only contains prompt table and prompt encoder parameters. No frozen model parameters are stored in the state dict. Prompt encoder parameters @@ -397,7 +397,6 @@ def forward( input_embeds = self.embed_input_inference(input_ids, taskname_ids) else: input_embeds = self.embed_input_train(input_ids, taskname_ids) - position_embeddings = self.frozen_model.model.language_model.embedding.position_embeddings(position_ids) encoder_input = input_embeds + position_embeddings else: diff --git a/nemo/collections/nlp/modules/common/prompt_table.py b/nemo/collections/nlp/modules/common/prompt_table.py index fc52421630ad..cc7d8bca56c6 100644 --- a/nemo/collections/nlp/modules/common/prompt_table.py +++ b/nemo/collections/nlp/modules/common/prompt_table.py @@ -121,6 +121,7 @@ def init_prompt_from_text(self, taskname, init_token_ids, word_embeddings, total init_token_ids_b = tensor_parallel.broadcast_data(keys, init_token_ids, datatype) init_token_ids = init_token_ids_b['text'].long() + word_embeddings = word_embeddings.to(init_token_ids.device) # Use a copy of token embedding weights to initalize the prompt embeddings word_embedding_weights = word_embeddings(init_token_ids).detach().clone() From 7801639513bec7fcd70a9558b2cefb73635ad67f Mon Sep 17 00:00:00 2001 From: Xuesong Yang <1646669+XuesongYang@users.noreply.github.com> Date: Thu, 14 Jul 2022 09:40:08 -0700 Subject: [PATCH 27/52] remove the variable that is not used in the context. (#4547) Signed-off-by: Xuesong Yang <1646669+XuesongYang@users.noreply.github.com> --- nemo/utils/exp_manager.py | 1 - 1 file changed, 1 deletion(-) diff --git a/nemo/utils/exp_manager.py b/nemo/utils/exp_manager.py index cf653bb34c87..8c81c4d4dbaa 100644 --- a/nemo/utils/exp_manager.py +++ b/nemo/utils/exp_manager.py @@ -232,7 +232,6 @@ def exp_manager(trainer: 'pytorch_lightning.Trainer', cfg: Optional[Union[DictCo local_rank = int(os.environ.get("LOCAL_RANK", 0)) global_rank = trainer.node_rank * trainer.num_devices + local_rank logging.rank = global_rank - world_size = trainer.world_size if cfg is None: logging.error("exp_manager did not receive a cfg argument. It will be disabled.") From 99c7661cb9caa488900e36e4a2700ae170d2547c Mon Sep 17 00:00:00 2001 From: Jason Date: Thu, 14 Jul 2022 17:30:52 -0400 Subject: [PATCH 28/52] update fastpitch to add export controls (#4509) * update fastpitch to add export controls Signed-off-by: Jason * final touchups Signed-off-by: Jason * more final touchups Signed-off-by: Jason --- nemo/collections/tts/models/fastpitch.py | 101 +++++++++++++++++++---- 1 file changed, 85 insertions(+), 16 deletions(-) diff --git a/nemo/collections/tts/models/fastpitch.py b/nemo/collections/tts/models/fastpitch.py index ae33ad412ae6..772f4aecc095 100644 --- a/nemo/collections/tts/models/fastpitch.py +++ b/nemo/collections/tts/models/fastpitch.py @@ -152,6 +152,7 @@ def __init__(self, cfg: DictConfig, trainer: Trainer = None): cfg.n_mel_channels, ) self._input_types = self._output_types = None + self.export_config = {"enable_volume": False, "enable_ragged_batches": False} def _get_default_text_tokenizer_conf(self): text_tokenizer: TextTokenizerConfig = TextTokenizerConfig() @@ -515,22 +516,26 @@ def list_available_models(cls) -> 'List[PretrainedModelInfo]': def _prepare_for_export(self, **kwargs): super()._prepare_for_export(**kwargs) + tensor_shape = ('T') if self.export_config["enable_ragged_batches"] else ('B', 'T') + # Define input_types and output_types as required by export() self._input_types = { - "text": NeuralType(('B', 'T_text'), TokenIndex()), - "pitch": NeuralType(('B', 'T_text'), RegressionValuesType()), - "pace": NeuralType(('B', 'T_text'), optional=True), - "volume": NeuralType(('B', 'T_text')), - "speaker": NeuralType(('B'), Index()), + "text": NeuralType(tensor_shape, TokenIndex()), + "pitch": NeuralType(tensor_shape, RegressionValuesType()), + "pace": NeuralType(tensor_shape), + "speaker": NeuralType(('B'), Index(), optional=True), + "volume": NeuralType(tensor_shape, optional=True), + "batch_lengths": NeuralType(('B'), optional=True), } self._output_types = { - "spect": NeuralType(('B', 'D', 'T_spec'), MelSpectrogramType()), + "spect": NeuralType(('B', 'D', 'T'), MelSpectrogramType()), "num_frames": NeuralType(('B'), TokenDurationType()), - "durs_predicted": NeuralType(('B', 'T_text'), TokenDurationType()), - "log_durs_predicted": NeuralType(('B', 'T_text'), TokenLogDurationType()), - "pitch_predicted": NeuralType(('B', 'T_text'), RegressionValuesType()), - "volume_aligned": NeuralType(('B', 'T_spec'), RegressionValuesType()), + "durs_predicted": NeuralType(('B', 'T'), TokenDurationType()), + "log_durs_predicted": NeuralType(('B', 'T'), TokenLogDurationType()), + "pitch_predicted": NeuralType(('B', 'T'), RegressionValuesType()), } + if self.export_config["enable_volume"]: + self._output_types["volume_aligned"] = NeuralType(('B', 'T'), RegressionValuesType()) def _export_teardown(self): self._input_types = self._output_types = None @@ -541,6 +546,10 @@ def disabled_deployment_input_names(self): disabled_inputs = set() if self.fastpitch.speaker_emb is None: disabled_inputs.add("speaker") + if not self.export_config["enable_ragged_batches"]: + disabled_inputs.add("batch_lengths") + if not self.export_config["enable_volume"]: + disabled_inputs.add("volume") return disabled_inputs @property @@ -558,15 +567,35 @@ def input_example(self, max_batch=1, max_dim=44): A tuple of input examples. """ par = next(self.fastpitch.parameters()) - sz = (max_batch, max_dim) + sz = (max_batch * max_dim) if self.export_config["enable_ragged_batches"] else (max_batch, max_dim) inp = torch.randint( 0, self.fastpitch.encoder.word_emb.num_embeddings, sz, device=par.device, dtype=torch.int64 ) pitch = torch.randn(sz, device=par.device, dtype=torch.float32) * 0.5 - pace = torch.clamp((torch.randn(sz, device=par.device, dtype=torch.float32) + 1) * 0.1, min=0.01) - volume = torch.clamp((torch.randn(sz, device=par.device, dtype=torch.float32) + 1) * 0.1, min=0.01) - - inputs = {'text': inp, 'pitch': pitch, 'pace': pace, 'volume': volume} + pace = torch.clamp(torch.randn(sz, device=par.device, dtype=torch.float32) * 0.1 + 1, min=0.01) + + inputs = {'text': inp, 'pitch': pitch, 'pace': pace} + + if self.export_config["enable_volume"]: + volume = torch.clamp(torch.randn(sz, device=par.device, dtype=torch.float32) * 0.1 + 1, min=0.01) + inputs['volume'] = volume + if self.export_config["enable_ragged_batches"]: + batch_lengths = torch.zeros((max_batch + 1), device=par.device, dtype=torch.int32) + left_over_size = sz + batch_lengths[0] = 0 + for i in range(1, max_batch): + length = torch.randint(1, left_over_size - (max_batch - i), (1,), device=par.device) + batch_lengths[i] = length + batch_lengths[i - 1] + left_over_size -= length.detach().cpu().numpy()[0] + batch_lengths[-1] = left_over_size + batch_lengths[-2] + + sum = 0 + index = 1 + while index < len(batch_lengths): + sum += batch_lengths[index] - batch_lengths[index - 1] + index += 1 + assert sum == sz, f"sum: {sum}, sz: {sz}, lengths:{batch_lengths}" + inputs['batch_lengths'] = batch_lengths if self.fastpitch.speaker_emb is not None: inputs['speaker'] = torch.randint( @@ -575,5 +604,45 @@ def input_example(self, max_batch=1, max_dim=44): return (inputs,) - def forward_for_export(self, text, pitch, pace, volume, speaker=None): + def forward_for_export(self, text, pitch, pace, volume=None, batch_lengths=None, speaker=None): + if self.export_config["enable_ragged_batches"]: + text, pitch, pace, volume_tensor = create_batch( + text, pitch, pace, volume, batch_lengths, padding_idx=self.fastpitch.encoder.padding_idx + ) + if volume is not None: + volume = volume_tensor return self.fastpitch.infer(text=text, pitch=pitch, pace=pace, volume=volume, speaker=speaker) + + +@torch.jit.script +def create_batch( + text: torch.Tensor, + pitch: torch.Tensor, + pace: torch.Tensor, + batch_lengths: torch.Tensor, + padding_idx: int = -1, + volume: Optional[torch.Tensor] = None, +): + batch_lengths = batch_lengths.to(torch.int64) + max_len = torch.max(batch_lengths[1:] - batch_lengths[:-1]) + + index = 1 + texts = torch.zeros(batch_lengths.shape[0] - 1, max_len, dtype=torch.int64, device=text.device) + padding_idx + pitches = torch.zeros(batch_lengths.shape[0] - 1, max_len, dtype=torch.float32, device=text.device) + paces = torch.zeros(batch_lengths.shape[0] - 1, max_len, dtype=torch.float32, device=text.device) + 1.0 + volumes = torch.zeros(batch_lengths.shape[0] - 1, max_len, dtype=torch.float32, device=text.device) + 1.0 + + while index < batch_lengths.shape[0]: + seq_start = batch_lengths[index - 1] + seq_end = batch_lengths[index] + cur_seq_len = seq_end - seq_start + + texts[index - 1, :cur_seq_len] = text[seq_start:seq_end] + pitches[index - 1, :cur_seq_len] = pitch[seq_start:seq_end] + paces[index - 1, :cur_seq_len] = pace[seq_start:seq_end] + if volume is not None: + volumes[index - 1, :cur_seq_len] = volume[seq_start:seq_end] + + index += 1 + + return texts, pitches, paces, volumes From fa2e55e607f787d129141523c724b29dab7a411c Mon Sep 17 00:00:00 2001 From: Subhankar Ghosh Date: Thu, 14 Jul 2022 19:42:37 -0700 Subject: [PATCH 29/52] Adding multispeaker fastpitch and hifigan en model links to available models (#4550) Signed-off-by: subhankar-ghosh --- nemo/collections/tts/models/fastpitch.py | 8 ++++++++ nemo/collections/tts/models/hifigan.py | 9 +++++++++ 2 files changed, 17 insertions(+) diff --git a/nemo/collections/tts/models/fastpitch.py b/nemo/collections/tts/models/fastpitch.py index 772f4aecc095..794b417590f6 100644 --- a/nemo/collections/tts/models/fastpitch.py +++ b/nemo/collections/tts/models/fastpitch.py @@ -510,6 +510,14 @@ def list_available_models(cls) -> 'List[PretrainedModelInfo]': ) list_of_models.append(model) + model = PretrainedModelInfo( + pretrained_model_name="tts_en_fastpitch_multispeaker", + location="https://api.ngc.nvidia.com/v2/models/nvidia/nemo/tts_en_multispeaker_fastpitchhifigan/versions/1.10.0/files/tts_en_fastpitch_multispeaker.nemo", + description="This model is trained on HiFITTS sampled at 44100Hz with and can be used to generate male and female English voices with an American accent.", + class_=cls, + ) + list_of_models.append(model) + return list_of_models # Methods for model exportability diff --git a/nemo/collections/tts/models/hifigan.py b/nemo/collections/tts/models/hifigan.py index 9ace88033abe..fb6d5e678115 100644 --- a/nemo/collections/tts/models/hifigan.py +++ b/nemo/collections/tts/models/hifigan.py @@ -371,6 +371,15 @@ def list_available_models(cls) -> 'Optional[Dict[str, str]]': ) list_of_models.append(model) + model = PretrainedModelInfo( + pretrained_model_name="tts_en_hifitts_hifigan_ft_fastpitch", + location="https://api.ngc.nvidia.com/v2/models/nvidia/nemo/tts_en_multispeaker_fastpitchhifigan/versions/1.10.0/files/tts_en_hifitts_hifigan_ft_fastpitch.nemo", + description="This model is trained on HiFiTTS audio sampled at 44100Hz and mel spectrograms generated from" + " FastPitch. This model has been tested on generating male and female English voices with an American accent.", + class_=cls, + ) + list_of_models.append(model) + return list_of_models def load_state_dict(self, state_dict, strict=True): From 7d9b166582f8b18fbb36ccdda0ac1f853611b071 Mon Sep 17 00:00:00 2001 From: Yang Zhang Date: Fri, 15 Jul 2022 15:00:50 -0400 Subject: [PATCH 30/52] added MLM Scoring (#4476) * added MLM Scoring Signed-off-by: Yang Zhang * fix header Signed-off-by: Yang Zhang * refactor Signed-off-by: Yang Zhang * fix bug that made normalization options set Signed-off-by: Yang Zhang * fix style Signed-off-by: Yang Zhang * fix discrepancy of space versus no space to previous version e.g. < sixteen > and Signed-off-by: Yang Zhang * remove and from cardinal when lm is used to reduce number of options Signed-off-by: Yang Zhang * fix grammar Signed-off-by: Yang Zhang * fix masked input for [MASK] token before mlm scoring Signed-off-by: Yang Zhang * mask out everything apart from one semiotic token Signed-off-by: Yang Zhang * reverted masking change and added roman to lm Signed-off-by: Yang Zhang * fix slash, expand measure Signed-off-by: ekmb * fix masked scoring Signed-off-by: Yang Zhang * audio based set fix for --lm Signed-off-by: ekmb * fix bug Signed-off-by: Yang Zhang * fix Signed-off-by: Yang Zhang * added jenkins test Signed-off-by: Yang Zhang * update jenkins Signed-off-by: Yang Zhang * fix header Signed-off-by: Yang Zhang * fix lgtm Signed-off-by: Yang Zhang * add dependency Signed-off-by: Yang Zhang * moved mlmscore file Signed-off-by: Yang Zhang * moved hybrid to nemo_text_processing folder Signed-off-by: Yang Zhang * update jenkins Signed-off-by: Yang Zhang * fix path Signed-off-by: Yang Zhang * fix test Signed-off-by: Yang Zhang * fix dataset license Signed-off-by: Yang Zhang Co-authored-by: ekmb --- Jenkinsfile | 23 +- .../nlp/text_normalization/tn_itn_all.bib | 7 + .../wfst/images/shallow_fusion.png | Bin 0 -> 42414 bytes .../wfst/wfst_text_normalization.rst | 64 +- nemo/collections/common/parts/__init__.py | 1 + nemo/collections/common/parts/mlm_scorer.py | 93 ++ .../duplex_decoder.py | 10 +- nemo_text_processing/hybrid/model_utils.py | 158 ++++ nemo_text_processing/hybrid/utils.py | 696 +++++++++++++++ .../hybrid/wfst_lm_rescoring.py | 335 ++++++++ .../en/data/measure/unit.tsv | 5 + .../text_normalization/en/taggers/cardinal.py | 31 +- .../text_normalization/en/taggers/roman.py | 2 +- .../text_normalization/en/taggers/serial.py | 9 +- .../en/taggers/whitelist.py | 6 +- .../normalize_with_audio.py | 11 +- .../requirements_nemo_text_processing.txt | 1 + .../EngConf.txt | 808 ++++++++++++++++++ setup.py | 35 +- .../test_cases_measure.txt | 1 + .../test_cases_normalize_with_audio.txt | 1 - .../test_cases_serial.txt | 1 - .../test_cases_whitelist.txt | 1 + 23 files changed, 2237 insertions(+), 62 deletions(-) create mode 100755 docs/source/nlp/text_normalization/wfst/images/shallow_fusion.png create mode 100644 nemo/collections/common/parts/mlm_scorer.py create mode 100644 nemo_text_processing/hybrid/model_utils.py create mode 100644 nemo_text_processing/hybrid/utils.py create mode 100644 nemo_text_processing/hybrid/wfst_lm_rescoring.py create mode 100644 scripts/text_normalization_dataset_files/EngConf.txt diff --git a/Jenkinsfile b/Jenkinsfile index 6f1fcf9b8966..1092c6745eb8 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -137,18 +137,23 @@ pipeline { parallel { stage('En TN grammars') { steps { - sh 'CUDA_VISIBLE_DEVICES="" python nemo_text_processing/text_normalization/normalize.py --text="1" --cache_dir /home/TestData/nlp/text_norm/ci/grammars/7-10-22' + sh 'CUDA_VISIBLE_DEVICES="" python nemo_text_processing/text_normalization/normalize.py --text="1" --cache_dir /home/TestData/nlp/text_norm/ci/grammars/7-14-22' } } stage('En ITN grammars') { steps { - sh 'CUDA_VISIBLE_DEVICES="" python nemo_text_processing/inverse_text_normalization/inverse_normalize.py --language en --text="twenty" --cache_dir /home/TestData/nlp/text_norm/ci/grammars/7-10-22' + sh 'CUDA_VISIBLE_DEVICES="" python nemo_text_processing/inverse_text_normalization/inverse_normalize.py --language en --text="twenty" --cache_dir /home/TestData/nlp/text_norm/ci/grammars/7-14-22' } } stage('Test En non-deterministic TN & Run all En TN/ITN tests (restore grammars from cache)') { steps { - sh 'CUDA_VISIBLE_DEVICES="" python nemo_text_processing/text_normalization/normalize_with_audio.py --text "\$.01" --n_tagged 2 --cache_dir /home/TestData/nlp/text_norm/ci/grammars/7-10-22' - sh 'CUDA_VISIBLE_DEVICES="" pytest tests/nemo_text_processing/en/ -m "not pleasefixme" --cpu --tn_cache_dir /home/TestData/nlp/text_norm/ci/grammars/7-10-22' + sh 'CUDA_VISIBLE_DEVICES="" python nemo_text_processing/text_normalization/normalize_with_audio.py --text "\$.01" --n_tagged 2 --cache_dir /home/TestData/nlp/text_norm/ci/grammars/7-14-22' + sh 'CUDA_VISIBLE_DEVICES="" pytest tests/nemo_text_processing/en/ -m "not pleasefixme" --cpu --tn_cache_dir /home/TestData/nlp/text_norm/ci/grammars/7-14-22' + } + } + stage('Test En Hybrid TN') { + steps { + sh 'CUDA_VISIBLE_DEVICES="" python nemo_text_processing/hybrid/wfst_lm_rescoring.py --data /home/TestData/nlp/text_norm/hybrid_tn/test.txt --regenerate_pkl --cache_dir /home/TestData/nlp/text_norm/ci/grammars/7-14-22 | grep "all_correct: True" || exit 1' } } } @@ -165,7 +170,7 @@ pipeline { parallel { stage('L2: Eng TN') { steps { - sh 'cd tools/text_processing_deployment && python pynini_export.py --output=/home/TestData/nlp/text_norm/output/ --grammars=tn_grammars --cache_dir /home/TestData/nlp/text_norm/ci/grammars/7-10-22 --language=en && ls -R /home/TestData/nlp/text_norm/output/ && echo ".far files created "|| exit 1' + sh 'cd tools/text_processing_deployment && python pynini_export.py --output=/home/TestData/nlp/text_norm/output/ --grammars=tn_grammars --cache_dir /home/TestData/nlp/text_norm/ci/grammars/7-14-22 --language=en && ls -R /home/TestData/nlp/text_norm/output/ && echo ".far files created "|| exit 1' sh 'cd nemo_text_processing/text_normalization/ && python normalize.py --input_file=/home/TestData/nlp/text_norm/ci/test.txt --input_case="lower_cased" --language=en --output_file=/home/TestData/nlp/text_norm/output/test.pynini.txt --verbose' sh 'cat /home/TestData/nlp/text_norm/output/test.pynini.txt' sh 'cmp --silent /home/TestData/nlp/text_norm/output/test.pynini.txt /home/TestData/nlp/text_norm/ci/test_goal_py_05-25.txt || exit 1' @@ -175,7 +180,7 @@ pipeline { stage('L2: Eng ITN export') { steps { - sh 'cd tools/text_processing_deployment && python pynini_export.py --output=/home/TestData/nlp/text_denorm/output/ --grammars=itn_grammars --cache_dir /home/TestData/nlp/text_norm/ci/grammars/7-10-22 --language=en && ls -R /home/TestData/nlp/text_denorm/output/ && echo ".far files created "|| exit 1' + sh 'cd tools/text_processing_deployment && python pynini_export.py --output=/home/TestData/nlp/text_denorm/output/ --grammars=itn_grammars --cache_dir /home/TestData/nlp/text_norm/ci/grammars/7-14-22 --language=en && ls -R /home/TestData/nlp/text_denorm/output/ && echo ".far files created "|| exit 1' sh 'cd nemo_text_processing/inverse_text_normalization/ && python inverse_normalize.py --input_file=/home/TestData/nlp/text_denorm/ci/test.txt --language=en --output_file=/home/TestData/nlp/text_denorm/output/test.pynini.txt --verbose' sh 'cmp --silent /home/TestData/nlp/text_denorm/output/test.pynini.txt /home/TestData/nlp/text_denorm/ci/test_goal_py.txt || exit 1' sh 'rm -rf /home/TestData/nlp/text_denorm/output/*' @@ -184,7 +189,7 @@ pipeline { stage('L2: TN with Audio (audio and raw text)') { steps { sh 'cd nemo_text_processing/text_normalization && \ - python normalize_with_audio.py --language=en --cache_dir /home/TestData/nlp/text_norm/ci/grammars/7-10-22 --text "The total amounts to \\$4.76." \ + python normalize_with_audio.py --language=en --cache_dir /home/TestData/nlp/text_norm/ci/grammars/7-14-22 --text "The total amounts to \\$4.76." \ --audio_data /home/TestData/nlp/text_norm/audio_based/audio.wav | tail -n2 | head -n1 > /tmp/out_raw.txt 2>&1 && \ cmp --silent /tmp/out_raw.txt /home/TestData/nlp/text_norm/audio_based/result.txt || exit 1' } @@ -192,7 +197,7 @@ pipeline { stage('L2: TN with Audio (audio and text file)') { steps { sh 'cd nemo_text_processing/text_normalization && \ - python normalize_with_audio.py --language=en --cache_dir /home/TestData/nlp/text_norm/ci/grammars/7-10-22 --text /home/TestData/nlp/text_norm/audio_based/text.txt \ + python normalize_with_audio.py --language=en --cache_dir /home/TestData/nlp/text_norm/ci/grammars/7-14-22 --text /home/TestData/nlp/text_norm/audio_based/text.txt \ --audio_data /home/TestData/nlp/text_norm/audio_based/audio.wav | tail -n2 | head -n1 > /tmp/out_file.txt 2>&1 && \ cmp --silent /tmp/out_file.txt /home/TestData/nlp/text_norm/audio_based/result.txt || exit 1' } @@ -200,7 +205,7 @@ pipeline { stage('L2: TN with Audio (manifest)') { steps { sh 'cd nemo_text_processing/text_normalization && \ - python normalize_with_audio.py --language=en --audio_data /home/TestData/nlp/text_norm/audio_based/manifest.json --n_tagged=120 --cache_dir /home/TestData/nlp/text_norm/ci/grammars/7-10-22' + python normalize_with_audio.py --language=en --audio_data /home/TestData/nlp/text_norm/audio_based/manifest.json --n_tagged=120 --cache_dir /home/TestData/nlp/text_norm/ci/grammars/7-14-22' } } } diff --git a/docs/source/nlp/text_normalization/tn_itn_all.bib b/docs/source/nlp/text_normalization/tn_itn_all.bib index 6fc843110e16..60fb97588a85 100644 --- a/docs/source/nlp/text_normalization/tn_itn_all.bib +++ b/docs/source/nlp/text_normalization/tn_itn_all.bib @@ -9,6 +9,13 @@ @article{ebden2015kestrel publisher={Cambridge University Press} } +@article{bakhturina2022shallow, + title={Shallow Fusion of Weighted Finite-State Transducer and Language Model for Text Normalization}, + author={Bakhturina, Evelina and Zhang, Yang and Ginsburg, Boris}, + journal={arXiv preprint arXiv:2203.15917}, + year={2022} +} + @article{sproat2016rnn, title={RNN approaches to text normalization: A challenge}, author={Sproat, Richard and Jaitly, Navdeep}, diff --git a/docs/source/nlp/text_normalization/wfst/images/shallow_fusion.png b/docs/source/nlp/text_normalization/wfst/images/shallow_fusion.png new file mode 100755 index 0000000000000000000000000000000000000000..c84bbbc58b0e7094fac2f5c32962c9b4c6ffd3de GIT binary patch literal 42414 zcmd?Qbx>U0(=G}Gw?NR~fdrS}?(Q1g-Q6`nuwV&Ja0oEC4=xGrHn;_McR!oFzxVsj zy|>P-I{(}{Rd;HDTI`v%yL*(U1v{p`f79q@~1EprBxrprD}f5aEIU zJd2JL0e_%fR3t^9%14NHfdQgAp7sMsk$Wa)E-v{qXVwEv@qY1PUr& zR9Z|}%~Ss{11SZkyYX7_D2bobhtfx?3ECUF5Pg z#Jyzk)V_Ty_|dP}hWBz>=P~nWj|du&>OU_f7)1L29)6A!GDQ0K5)@PdER5`bkFwum zXu|yW@Izq7oBtkBBfVJV-9%WVF4fWS>+2j{WHZe@|E|5wx(lA_{!Bd$VJ zdW|O>NkO+5td&uFyAcx~Lvgz3%vt#x9zZzM5C5zp&bxP&UA>ZZDx-8~*`fTEYSbsJ zK?D_3j4G^zn%d%a%~}JI-)F$lhian%ueCA`F3NVYr=5mH1Ck+1%T=|~G4X!9Af@nE z2@x575PlQXsR)0EHd&HL(fQL9FZ-1kJheBwgp9rQfLj_`3jg&^5F#9mtccsaB-0KG zTvAYrJG#=!9=bKbhso_8rF)6zytP-d%4e6h*)a@fhB!N|ixrZ({0R0Le`Z`{V);~{ zt>2TWCbkil9~|UmdSO`(F}5vvx#PIZ4zOOHOCU`bAg$|!cZ=tH;j@uGJ{<|P~{9W5y>SS= znYfpoKansOf8Qzv&kO}bm1Rl{_?7Ir6N_TumeP4bleQxyklgI$$N1f9^4C=y7k)Yy z%V=p<`vH;-xTp#sb_27iw3ae$Gdqt;{Pn%z9Nt7>`(nWC?G$`&;>FhKyRq3(q|s>*qRrDpv0Y z$yxKQ$@3fQ=D`UPx>g8xV{}t}_>ZFIr7NZr%k@2udA-^qFDqx8hinz%=+r3*tOnA^ z+1QuhMUC?}6n*}xL|qAyBR*rU(6wi)D?ec(LvZO;sk9ixKuG%it4hQ}A2le5&}IAD zw75i77){=7xTgTp3UPz%Sn?D#wDn=tzZgsjK6@Oj*^{(?1YgCMoi#7+YF?p(U(aAHocWER(rc zbeGMpXY+pMtCcb;8J)3(SIWKqYd8L7kf!#*8``L*g)(~yXMT<0#a6>7edyyr49l?aRx7ZvGzqewE;Ar4g**n$h-HbqNoqpI zzR8dFxlfGNafhIV!o()d{vWo-1?EgpKUbvL$nK_JUqs<^hIy-L3qN1VFQ4j0%7)9q zCI1j-(pE}~@Y$@Lr1iR1dUn@ZE6fqy~ebG6!KzTIHHn`6n5)%CF zl?S`e%CY68RF2Eso_z@q0bx`-^q$Uy#38u3!ZnB|wx9s61Qbvh;Av<=#V+0xh1xbX z<>O%hcSwk_9(g}vP6+4d{qXG^Z<7+Pzva&?<0XXl_{W~XMy%04mU{CaOTi|0#0lxj z-3wzp+ee@~FyZE;bj0c1@@Dc`o5p@{t0HPn{*KJ26~yQNS)qB-!h|<^wf1?kpf%dz z^Bw11CGx4 zUh-k}+iN`*L^IgQ2edpFHtz!P^^B0aOqPo9jXL9o+idNp9(jpzyVYl(9cp%%!YulF!puLYkV(j_tVR`Ds97`$#pTwc6 zCqf_yVoREz*|zhaLT%TZZ6Jh#68i9On2HJf5cn(=nZ@x<)X=I61aJLgPMF`^79;^9 z`_%0mL%CQCH_WE?ho;Fup49DV)NOQSH!q0Usd&Lc0ppKU?pVec!#Gytkb(tURpS zWb=>!u%VIbPfBPO+;77JU<40aKmI$3wYx3QW9M_S-2};6ux{VYwfbz&>OAh*ySu4 zKmS)HxTLise`nDpr#BjfU(l-Hei4grd4Nb7S%SoiM{@pTBP)oJbgOziH24}gd5nOZ z%q!Xzs_gm^HYHT~SFGM&V-K$DyJPo)OC;eK2z5&yGtWGbi%xM`o2B3aBhmk89t)wY zU)xZ)@wCn<=Jtqu7cg31 zU6o)KW8n=m<5flk=F8YF>zz1J;1FS6F1&v&ZR;@!8#^3rUBGC!eiV4P51P3o7ihqu zC$+_&UCwd{FrbRw95p$TQuZM8av+JPx~w?rYlmaF$S5(4aN3SVRe+4L%PKkK)kKvb zTJy21)-SfqQ<0Vu@;B18dx0&Dp6`~G$VwW(H?S*#Zm|^C)mDSFd-_a0wWU7Kd+v?z zJSHg2gZ|vxWOJ8%`VGXDc^Q+Ib4k^GnM=C?vYgFk*LGI=s7G|R41MbroejN%v?aBU z0W${%K>2Lce`D+$7}+4h_IzQuB-k{bB4vIBQ&l}b&rXNnpB*YstVyzDd9aXm2`9Wk z+Hlnw>&3UCzm@@4^MhCK=Zq2GBqICfu#cQoQP-}NQ6J7DIWmLYG$;xGHR`I{ezEKb zyBgLtw=|Di+hFG~)+ah(!-uh$;fQap2(wqb>mUI<5xzqjZ(+tziy8MDNe&g z+Mb<*h?l+O2!q!P?J5u_!KbLf;=J!Z97>b7{1SqA<4KqbMxHNfYRzH4`RD@YHnz&`)F8v;jbaK-6 z$nSkzXm{_Ws}@D@`lv$-2=A?E;&u{6;w)f|}6@-2&OHsv$P| znxTd}jXXJiw88wL27wZZRP(IHBr93fXqI6C7lz4c3wq#EQbD@e55FFk$;u%Xy?gpH zv!*MpAA_~|%=h%eW=$7w2jx0tL{rQClBQoS51Y+Tr1&LmvFP1JnH3l{CUQq}IP4{A z_>$QH*0qL5r`B`)1$VJ;@H{_xmbQ)6`XH1YMtXzAaF9a$Ju|d>JJo8a1#G8HAlX`j1xotVx-nX0oayfNn2VG5=qQoS!2%BOq(@a(Roz&T2?N)+Lqkul~A& zi&d8-x$XRN7kiUp4sClYQcx<;FtXW{4Z8(tpsG_tp1r&d+Z~|3g(pO2^Y`qPwC}Jx@>R2 zV%4J^Sknle$HQ8!j>PljEoWlfbiU$>aRRP|w0nI=E=rE1pvR?24jx^R=*& zriQ*({z)(coBg8j?<@36uukQiX8Iq+cOcKIrPIs#)v)K0xs_8A`E>XGxDRLKm}&&D zNkZmYZ7pf;wue;;j7^?#Ms6M+gRIaXzv*(leLbPG*=++~Td)z<0o><{&PaQ+5)R(! zDYr*){U+8Cu3&GUQx0M%8KeE_GB%rO`q88j0hg~+({eO~A%bw(u3t|*e7E1UKBeTTtD)hWo>fD%1eV0@ez&WljCJn`ozOTEGWbkXCv| z9X{76P^+}OhEo)I=;4A2W%X6|>N!>f&qurw>>RoJ-Hr+kRSms@^vm7n+D=3B@UB)> z;e-Sl0V?6O|3*vIDVeu%tEPeW0s0g6=d-*A;$toIn&xlIp5mEsv`8?rWbs6P5arrR z6R;o4_a8pXs66LUD}$^PBCqSVoFSFjickFMJ9#mW&IvZV-vfeWPIvNRy+>|#(dxDB zLb_uIPiqc3gBXY*WCxXQArC&9%<0?N^Ba|&{*rh@_R+I!EBSry?VhK`sVq7dwTJ;# z24N>eow03jXbFRL$To0RVz+9`-xg|l)e9bJ9w;76zOJ?otQ{!KY(>-U*3Ib0TLj7g zSw!FagR$GoIB>sML_ksNPHyDJThYjNMC`~vu(t|4kqUzq=YL*_Aku$|oG}Q<@V{SJ znRw-)XHtiQ+wK?4DKo?};-1nU;?j74fwHX~%n2)CcIJecra$SDu}w$n&m|uV*XNY} z{;k}Av0;B!FagLUl23^okv^YrQ}vd?p`JERcR6)dDien-^j5o&oh@?Q8SV>)W-(UT0{u2f?TwG@Up7TS?^@zB(dSWUa&1q{RZvLE7eGd}l=^aS)RT+}9*9zZ&{E#eKGXw$0YP z5G62w?=!GeL=O@TyORp11~gWwatRZ}ntP{{;mcdKS)zZe7BtlnSRfkkd=x@ z;h*_~0jIjA>$XoJ>A;a1blIVz*jGS*?mX_|;Ysg_+>=@zi_Q=|R3XIewJ=)W%iH4> z)>GHo%41zl-B#A)OO{ZNaGPqS%A&YL8Mhtulg0Ssz8&Njo#K$*>TS)~wWgl6V5OSp zh#UzO<+xM)bId0{o?139UTJ#CnleKoZ#!zUWsNI`0@m#)5Lz>&>g;L!;M}oQHY2MT z*n_+9KxpOmY)8!eh|G_iyw#!Nxfa7y9bTinjpIn}6gh5Kh0C3FJ~g?p$2f={qcv}+ zl_F8K$q>7HsS?AkrteJL`h;9JTw$U_D*LFGbrU17%Uknqa{n?eW{!R`so)Q}FQSm4 zbm!GlpULfUkD2W}%{;o7?v9tl-tUA#bhL#NM=P6mytNngbLfLN*V{Z(@H$Iz-^|SW z(8Gd$pwquyf5$R|N2Kc170qs@v32vO3U`Ec&qIuMwC8*mCT-^6cZGqg|5(+?J9N9v za}hh1`DA<0ZjCVQFVe z;^Ib?9n-#oktJYUk&iHmkC0s_^{5inSsFA8OFADg4eJlMcZ-biYZWjjMENGnON4CA zQ)xQX-yK`TwKjpx4A0*_$E(r!mZC@0LqX$$*#Ga$Bx(*9KaCV8{n zAjL(e9{tSBX;<@*1w*QNrp!+C#1cA#u-|T|+r;{a*p+CB3XEd9TUdqu3YxP!elMYF z!tuslx3>VT@Dx*IdWxP(?b>oj0dhpqqf*8a`TG_$2u0`oZisB`PtJ&F7ZJoFU6Kx$?-DEz_~|`i61DVyooxJkz$$E+nbBY_kFWfWPj|)hp=DSAaUC*BQ`4Da*np6v+pC5T zSdF>Y&P;2Mq}br3o>_tcZ2E?Kw5W{ZzP_$ku;9vKQjpGmIo=)n{PDKo(&g7NPY7A% z29JX4+CX%LVlBO__GE!f=JoLDo^y-|G2C)W`L}@DIqW=ACO5#rfDF_;=V7O7*Ixqd2;j*@`IeNpyQBJ^bHDWHS^2-&__|pmZeY@*6 zl;4r&QE%#hel&L0uUJ9UK}P9T5|7b9n(T%|%t~AW>_oBqZYDu5-=?YW%DtUtu)SFO z=1P72y}pv~B=ygJn_Iqwz13!?w`ood3xp;`g{1}!N1N51J6@Z0qFSR;OG9_ON|ua6 z*~I8@og9^Hpkroh1Oet=hRcf?^Hq}q1}(d`u?AeRQ5UO{{G)368h-yL-DgMOYj_1? z=~(BDM#&-V`xneW7e7CM(`ggeX>p5GK8@CEnbVHi{2|n_G%u`F6olH7Zx0W*g?aCR zU0#w^aC2Mq131L(s{!YIB~UN>N*>?f__;mquXPkyoBa2PdV3RI0Ek;mMO?Qhr%QaI zbwXBBPP|QJ_2GHF0%C@NsAfSLnYO44o~<~q!?*WhximFEv(E0<{N&mRe-9UI$+=<6 z#jP*5o{j8{CSLL#d( zq2FGCvJhnN5#cpKsyU8Of?vmQ~B&5&d$J-boPy zQ(3=4cM%S6@P~xge(;W&^vGZdbupyjYKMKs*F#`Ay>vQ@-NL8yT28(kI|t7plTJRl z7w2_31blcfsckh&%PJ1HF+}pA{}grrm!v+DgM>2ume5dR;&g*FtVFe^sWqOUk;DxP z*+1f2Q^1{egqhQF3ZY$?JK8@VyJg5YTDIn{{4N`GZR0 zw%#!>7R8eDZ#%n>-l>a4{q(&%|aTbHOJQ*W@A z8;DU)X*0r&+-t+g}c?KAx-AIik^59e3)i z(kd2k9*^+W5AbCEapJm#=eqP*+OQ{33Vgpy7Tg?N+xsU4WHOE8Ht|@`q1xJV-apA` zT-_+Z-Jw?YRD@WBP7nP}Ak@GR>GP^^!7EuP(*)p~=dahE!$^ZofUcp}#kI=JP!*dB%Ezh06@`8M?^%QpYeTD--pJ9f zX3O4bpVIgUpuq;U0RVcR=V1YmllC<*R3m{+nm~h*{qQ}2$o@J4>wzpDoLb)`TV&*1 zivmC7E_<20IR`rJz&Eg8KAQ#<5ki}PZ@NyPnM=IC z`8GQa;%>N`NMgN^M*5aeI6C<{?mqHPuTKy?xg-+a-FthQYPt;6F(JfR_tccgqj5dn zq2OgHi^m>&Hdw)1JlGV$2_p;4oekXP|85+4d4#01EM9oWukSqx(+}&u6D|~G4%YF( z_W~f$UVEEzL7k-5zMi7!tjJ7Y@M^Szg_gk z;Y$tw->>Jkk?dJo{k&7Q?mXflXta36K73Rl;Drz$t&K*1s|I zbNy8vNauJeVMCcK_)W5;*Z_M90n7M*`HT}9y?ho=Du?YX*(gjRVC`>zC!hvq=^mq1 zN8m0x8Kfx|U4hZc!u5iJC970x{yB^;3``?aC^@vwS3ua+qK^p+mdjI z-qZrCCg@E-**4Bz+12*iA*6)Qm>TAr@QY-?#I!Lo^vIJkz5esYpQzDdRVXU40S{yE zK;~`>IV?iMp0Erf1OAb3f>)BR#df%!&3y5C;o6>8HW(&O}2qR|(jELzv`_he#feg)j_Adj&& zxH+!aE-af2@FM`t2$tLwTs$%r19IQ+cSu6u4xQD{3f&GxBh)Vw02{XmN}=w6X5`-q z)`vOgmt>~EqDuHLMp)YiYopK?@mRX_I70uG>H{FvEJ6}nVFC7Cw}}4;x(+wThqPc; zhkB34%gz?G89u=ykug^=*muYv6T(gGt*v}axViTJedjnOSaE+)X_?*g^lOKTlNfFY zZZj)h_cFKf!Fz$ns*-1R)k(5jG-Am<7I2PMs@2c$i zl6;2l%;2WTcF`44Ih|0T&P;qg!%RZZEHaH^YQ*bJz{(Utus|DYy^vEDLncEBBa7Q^ zrt>OQ;LE3xOXDuwpAh`1!K;B!yThlTm_aW==giRI81mgA@9wC8!WPt_r;DH42B2tW zx2m#KQU5X9cqJMe48PXw`juN=QgR@bV<1Q4I4wUo90bcZFo78Au^5 zWuf=pMhqAH!&X6Bl6xVW1edCBf;eYb*oAxAo7&=FqAi6jqF&7c8#d`H1&ab4*a@`o z)$JV(P85yZ& z8tl~&)S==emL+&IdgbOmX870w88)luGd^&`mTGwounJeFqMEFvEIw0uMj^PMULz(G# zbEwHQ3Y92^iUn2O?TAhfhoZ142K#HHg3@=`YG*`-o7*Ljz*(Oeo$5=HgdpcMjg_03 z8vojv#_djIVuvuH7c=-GO7*_B9fcvL8gRyLWy|z=|4(doQOX>IQ5E=vH8C2*gI;?? ztV|Mf21@1mZ5-s=%zL`;lDR*)mKWFOXML^O?&B*Y#rpb~843+7bs%|fL;PzFTZVA~ zR2>kQIOz9J{>p@}m$@B7kyX*kiHv=+(H@@y&K)AWGKV$9_N=QAdPmT{&D&kuk!f|jg`iG#DwlWJj@AZp22()W2Pe!f9Ug^EZ~pfPhe9GT>^iIY zQ0i6Z4hpZ{UQ0vOan8W^%bHCG3Kaw4bcQsmvPDObc^HB2*Ve-i^#P@Z>i%{itT(vURIwP{J>~>c05I0C*XuW{>!ZY#z-Z zj52BA_YmWu6xKt@N4t0)Ci?@r3qDcz@R$Bni1Y=G{iTh;Xqnk<+LHYo-_fLhOMGx6-pybw3>zHyCI zp^;k(BXuwJPu_OT!k^p%OQ|kPu3^Pe+K#j7Weoi5khYYzV(}RMEz2f1<>o1M&Fv>6 z9zEaFZkEfBBGDFhr`R^C^O|4=47+tUmYzChyq}XXMW+N>3D#DSs)j1k?2Pyng--MM zrr~|}+&$1<>0aKonPaJtAkEr5wl9_TvfR(C(06#cj?l#Zv+pp|j*fn8!zE<~Jb9(Q z=~CIie;uu8W?>yw0;Ju)L**=L27%2>Jyg|8BD!6hfm_9^PVitwI~}S0bl1qi`!#Lw z(;WSRCi`ej^raMUBEzl~J=Z2qa2Y@N_=~a?e*_Ck?w%pUVAEuOpUj}_N!sQcSC)?5 z>`KNY$FQ|W)9m$XnPZ|P>f7ucgB5vVhG;f#^9eO(G1QCk;{9!3GWVsNHb`M^Zh%V2kVPWuFQ; zq&Ua7>*{H${$jFt$;kp+p9mIa%YZOonxGinjKYQU#AZaK=G=-8%RxWU>v5o4J`h7g z=bSlBnWn~rN9Z;@7+Rbxzvbmky1O5vsn+V+j5qsG7-x4>C7npR?NWkv-~6-rI1%W) zq8s-?Wsxu=ZG6E*s(UYwBPzNQ1mAw3`zpN!1wAkyE_pZ?i;6^%sr{jvuSlYjtBFez z6uZ$U5?Hm-?ff>_DH4wVIXoU~J0%i|HA~$oM(bGw#c)y@{*VzOaCknTU3O@#KYZ{zOx4R)za>G*GHcPAC*%Wo09AK6yN+B^pO^LT}#hDB#U=5e9Wak8cf}*gK9AMdKBx<4#n$2jk_c8Jlk`}@2lA6 zBq(Z9TUD3QG^0B1{CGtk0I}fujx7c6_1EjCtQ`tomT?Y+{<0!(BuS!prEcVP7bN=U zol_a)R;gW%Y;HmrO+T9wPfJNMh)Y{r6%FM<)A1=?!a8G4D$nCouwtPulYk1WT93(c zWcA(aV8%!+J`cK-%ucLKIHB=5N5Pty^M{0I?>^8N%q2llS8kYUg?Q~q6=P-1DDH9y zln?{iLa9$O!^woc`<&UC?MJmgFdTj--S1cKGd@M?If?y^HhfirNpBHw#|vaqVm4Fm_NVPK9om#!p0_Vppz;- zt7kVw*gVdh^GussmUGgEO`?s(?|EyiipL(!63isy#^xRfwj`+TP>I^>QR$^*Dw5QC zC0nv!S$lZiu~v2P0lc)?jh+`|86~oKNOcq{~+V_0FbHfe&*cU{QO4n z>Gyi(v3cd3c)F`k<7p4!?k@z=pY6|&$1EP-aDqDGZtx)JgdDcn<>ib#4=0^`yoXIs zLyQeNIDZt(+!)%7e^X0 zIhvX`u*@ygO1CvO%j31&F!F~n#~4O7suchjM+u5l%{eObBeh34DXJPwXcYWnmlP}> zo=lJV5D6HY4BbnnNRW}PV1#$4tqRls_&Swz#dgl01(Eg1*I3~A6DHE!s3H=i%ba=u zUAz`+*u?j|f%$=jK|;#wmlA!Z0P|*Glz=hWg8@y;u)jf8YK+$*>cQ3)I%x`5LE!LIPZFhxbiP}*vcqTVyM_mmKh420{p5S{D^K7EMy(6r0O=WktAcfeBd z?~E>^M+~B+151Ks(^{LtIZw84SEt1fryt%SBHK7^htyuRDR|qTvy{j;B0pgVE!KIi zhSU~p^Duq1sVNK+F)X4rrK-I6-Q)X5+-GB`Fq!9yJ%}9#m41^{cp+n6{XT5qttL4^ z_NSVjzCUqFOHu?26TcRl@?e53*I*S7=q<@iW3^w|@lVX(JREw@UauD?#68$6p zGxg#ZooK_aPt@s)wmPrrQ+Ef6H3tNdXTa?K%x6zK>0+MPQf4GJfjj#^JteLzVA020GaR2lLw10Xei%5IjcG>FM2lzq`)^> z1$od*y4`yyRV4d zWhI=s+<3|I_|kceZJUtQQ{6j{t+fiwe-f8x@gtVkG!6ZFS7akW=T?^7>v5&hychY% zTgRIl+Pk$xf=K@#m>vM@f4l^sx5)HJ9Bvyc9k}C&)-CGooR$FXuH>_7`fm?}vLE@& zjD^wy14*yxyI;!Z-Pj#F+d^irYtsd99@s-UKHqNH#J9?8N;#h73o=Q+3HRb{g0lO; ztnfZbOQgGPYcEf#&_A>Q;CH7Y0BMHV!X>3Vxk2V8q%NBWvXz&lib-c=GG1#Pjn2(U zF0m3?)wYNh+0S@er_U-7v#yiy?iZ3U04!yRm}3?gSrd?vu3zQoc)0Ql))DD#aOyI) z8vqKXJm~bmaGJ+t^KPfKZuUoK_XSV7lHr_8Zt; zN1&v^J}1U6Hk;%zWd_%Z6v=MS%H5U;HuY;&V4K(Fv`IO5vyi>VCF-tz@Bvnsk>E{pg2|jy`t%n)@9GMo!NW8RvKdXJma$3<3!oWUnO zi2F3eh>|r*jU<~~i4JuKr%R?Y(vtN_iy9?gov*8XT#?bcNU~1$*6wXY3=%o((Qldo zb>GZSm{n2a5o_kDYXHlG0GmVvq~ZJ!SHu1_Ha{wCeWll_>#mOTN0C<(Q=G?$)^_Oi z<8++0N*qk(ev&Kmx?SXOLG)jvoUHJS(g>UWC>Rlzh#5*JytAcIix18wjn?JLVWqb6ntA3zywigv5=zn- zn$EC+`4>u)f%y#vBwD|iS)-LIJJ3n5iRnpaGD`3A#gko~2dqXX&nAhowS_j=VMlay zQ>rP@nb2+VP*(j7C6A|SQViP3`Y89AV)>TqoXp-@c-@C{mm4H_Zr*nG2qyP!XNRElz?Bp807uSf5K4U493RyWu}9P%Jj%&9-;2<3bJG+Te98-_EbBo)6M93OfEa z7u;AV-Ix+LXsP^DnKYy>SFtloGyQJME^-YhDMHW}V5`JsNeUPS5p*Ye@6 zXMUZkikXLtz2h8%JoMe0m>P<_VHtg$N{O-vy;IfVG!{A#wjE_2KbKFf&N zZs05Y$zX^@)KGDf%X(lM9a4MS$RN0|26%w38S`#a1Al;slg(X}z1$*cdaWJQ7ufbugg2GF4q5ox z5bTnr!DKNSqm1W%I2$KMlT3%hl3t=*&w08miu+-_$j<3;~vS5)_>RcKMg zYkxnk{31G$5xX6Aqq)4hHK}FlHa{^A7pc^F%FSiBPtHM}3Ve>$;X|37veS|s9+Kny zn9=~L%Q8M{Pl`Z!qP*2E`PV8!?&ViH)Rrb&nu?WSO1EvL9C*qPxAcsIO@{UJI#G5a zW7(e+n%NF4E8SCFo!%y7Xow%2S+~o(zugx4mN(HNQ8zCtCO)4(4F)xo!e-gt3Foyb z-PS#9-7Y85RJ2l-;|Plb=bI^>zyCB#4&+M$77By$l*pXxMn5{+o@q2(y}ra8(8^18 ztHN|1^XDaxgyOOT(GM{3T{7d+Cj?jR6rwOu=dm18_DRYx<<)jfDD_eI8tdrulAj9? z1rY?nmpRXyL1F-hoCdCe)S4jzbe^kd!boR5=~l0w8cMGS>QvTwbz_FNZx6$V5O94$ z21-!O>s4e2*}SR{{}O*{(wE$M0Q#L>hFW#SPEazA9(@lN>!HJB|Jh1aq@6_}|AoT1 zxv~>Ts;M68=J-m8!XSt)M6%aY1}Um#XejcPJ^CSPS@LY ztEUKUAbU3Fow6;>iAhWbj1{=TA6{&j6oP`MVzSP4tfjrM5$CqK0+lDtH8maHnHZCX zm!y&|!Ohxyj#BtKJu7mOV<1uw3RT6t^}JVYjv2)SFsiPxPo?hrD>j{;Ox+>)wV5Cr zlaB z*~8{0`OPiLzS0T@s_?f71_-m_GtQ@{T*4+on4s(dWOpRtPk7THr6TW38ZS4*VxTWi zcJAmHajZLEWtZ=l8^|W^ARXEnnvG0%(GZ2rsQ(Vs6r?Ajlo-NJtCsG`i@mH7p}#;| zfh-IVD3Bl+4tL{w8tPsiOjCQyvj3fW$ou+Tcj#$MUQ8-naO~Nqa(vQ92(kv?5RmY4 z?xyTxJxq%C4O3)Ms zX1=gBpI@k*CNES{*Im}6)X>Dx|9y7pU#g}H4Qu(+6!qi*>kL5GELJX#lYS zVEKZRRiu77leC`zQFg?jSDmDU!0bmuLjNTjz=XXk{GX__%;p5@1Ou*zHML1EhEl2Kz;~VD?%u{>98$!P!--mXGaI9^g4{3H*|fK}FY>Z_Jb(GYpKM5Cei{KL z6euqQxWP$1r2+rnFWvvlQZ|_u90rKV9B$eub+vNI8XjcB!0xk#@#lqV=l}DkN5b1s zkF@N_FZyWVq_|MFqA}yibA0OE=|8?X<+cAqiQs7=NkOsGndTgPpY`?sJnaB4t}~ys zfi$~OT|OTU|CxXN^TkuQR%aAAI5@nO4|gp_BIK0*YqKg{)osMoEf2%eX_O~u1e2k? zgri}3&ZP~w1cg+N_#@r>Ah)x?|KcTAGqr84+X%M7w>+E;Zhm?P;sBBrIhV(^{N=;% zU}=Q^D$I)cPfc8ibhWEZBU)HsrvP~Mo7SDaXD3hp`v}tbbqoJ@Aa%FiAKRrS?f7~D z3)(z>)4DuH6WBZ_9l?BsXu?WvZX4shAJ6>h#3P%H$eiz5&@LJg;~hS@i) z!rO2Ibm=u}U<9O?gfuB#Yde7+a--U>u8Re*w`+n>SG^B@K`wpjZO+&Er4b$LqHhj^u4;U~S8 zWPL)v;8gkg<@ykRk6F(j^Aj@l%ibm1dC*Yy7gtx_g2m}vRxHuH4c~DYbR6nfr#ltf z7)N2IUZ7U)7`{Er<3k6qwD?SrIDh7O`$6^KMk*94(+KxA(I~vVYr~D22kogi9TeH) ziJqh+u}{!crhLZ5!mD5u6GhTHjByel3}Scu@q;hYqh7z7}cKSiX>{+gmo5sYJb`odHU;N1< zT!k6)SM!m}ppr=a_i1-0VG&Q)0TJkeM9@@9ynN4m<7b;<+ZmFsU6@b!&wmgOIkGRn z^_YiEbmJBN;%t6A<5^E#^{|7ekEHgLSv;NAF7*D`k0%r{XGe^%PzeZ6!kZ0Pl)dFv z&BtDSKGkq)2+S@p--rQj{SLSFf@$vl*~q3ol=b0!-Pw@q*koMZqCYVk5!3!1E?an= zc1c4mT;dl83Ui%?gQk3S9V5Nw4QM9$q+|87-sH*(Sg&6{<$!|svr@TP&2B-N4+JH-|Geh3}-8gaexH#Aik6tEI8LQdnC0y%XwB^vmM6 z1#Wjq#hcT-OyFVUt_U%oWBH3+#f9G5a!iWwtGTe9pYh@4Epe8^1i2Z!W(Drk6$Zz* z^8P~*bB~sB&f~Dq=Fia7qn4CsdAF_UQ*2cFIqJt$0rjIbmzr+7-mr- zO@bdznPZu`LWV`te>CXNyBSvUZEEcNW#0$*UoQcV0T_E8ap&?Lzi&Db$<2$UynDKi z5IBw4fomv=_5HHLyRs@B3LquKuM}96e!Smxsm9kl+x`r{hi}RUEl>l6bWhg3mq+@c zxZ=8(3qj+!F>I$B&n&dN%5H&U6uic9ww2s-Vt$_r>v8p4u>c=f&y7ii`gvl9M3%CA z@dDUHiJ@vJ(=e+^?VSWG*0T7B@~t9^2^UBEC>rcF;Ao2^Un?q0-n9)HwY|=jnEhL2 z0Ks~IsgbBGvzT9>aCRD{{+--IP@|F#mSC*b*!{L82h+VkRxC4VpDzqfI^8hJ2a_kd z#;oejZwN_ivu#T&HCsU_;W0d4cYIbT?rwVX@NNojV!j{y&#Y{lj}YbN%SJRQydj#A&ebz04q7CgQocNBn*T64$LP( zMJCB+%gOomI^lHEws6Lln`oW)l8imW-J!C?!%s=uJ1DQ7$vJ^S|5uBUn`U~7A4ze2 z`$sE1qYa&-kOT@udMns8lp4+LwVSm*d8Wb4cE4$;O&fD-GK>vh7->q!iu{*c8V6|^a z@u${pf9zI{-QN|?L^`_rUv7Iq((44TUM>gjtD({L@HAQE`l%CM5lo$_!;J4<=tZLm zW-WRkXF-h)WlbU;lxLeLfSk0ef zh4uJ0+F!`rx#A_sW~=%yYjiHX^C! zll8iG>u@`q=oaggfK>FQYX;YIk>Y{zW2Rf{O$BMvJsJCipTXdG{hnf88B^p~h^4BEg~7&WfC-vP-d!ui7YH4v988r5iZn>YRJy zp0qpzw|GBug(dA^Fhq*S;#-)1h?8^;)M`L;C%1wrWO`QbsuRV$T|6LMcVP0XwfFXW zc&18tE5gV)gz^Vy$gCu5vwgUn82j827wX1L{G>44ZKNq4<09P>of0Wfq_h;xygdfb zJMOeQt$xjO=XJju>`acbS@|10&>MH35|L&h+ADihcDSg4$Sq_6- zH}%lcv66o?q;e(fJ;S@Be0sd+M-cIJg5Z7gKPWrPfGE3mZHtA9f`Ee3sEAS{NT&#h z(gI3{fOL0{fRr>ymvnau3|-Ql!q5#v4lv9x-@@m4pLg&5?fu0cf8Y$Pd#!b^xUTa! zuSNQj-dYbt)$woLR zdb})?=+4uo461`2o?5%&KBSLi`6R4Mq{8&}b%$A>&{Xt=NbSz%F+Hgk8%CSt!wR=j zD`iO3=RB8ccQB`kDWh4;?DEn~ABR}9T>s#vLE6&vJ9L1gNQOl179|8Me_NC=s*^Xd70OCz4c|(m%_aB-n$v zI2Sj_Hzof&*g*CVY_QpVI7JBpx?(*iU;~^dQkJlHW^Lv1tX`}vePonzWoScvW1-(x zlV!DfG;A-iU~2fSEMk<)DlF}BqCIdD3Aiu3=4ks#XR{lyVBtPTXLyjYgm(Sgpf4vv z)Ha@Pk`~sM#Ml~oC9YL7{r))4%T|j2=ApUW%iXQtItD~8u%Mc#-?Eb_`d1{w{TiH* zD$eKM1=eoVx2!ilO|`(#9|a&3^g@~H@xCE^7CLrMtoYrTET+T#1WrV^TSbDi2CO7! zdP|HdFqaqO{Syorg*`lT~)sU5$pm3I4mo5Wo|NDkTYI&9aF!KiOc zrN-UvXbg|z!&H1o2p>}?NnRjyNC-=gU0yS~(1@RNZHpxlXVE}>d42XdO=ZWptm_p+ zWh!6(3~@8%Y+LL27ak<$=n+^_7E)utIkcvcUIMZ+WbOSa=EKGk`u8iwe$_>8yphLf zg7i+Amvh%gnbQY%iC@985B#hW!+xcWWuG>NTWkBvK(6-_DwAl7Q5=)rLb0wttKcOy z_e%6Tu(nc{u{+#5SnY4dAIdQQlxl`gd_d{FVay^G^ZFE@66{ml%$&=oDJ88Pw;)Rk z?RI)-$!6qI^dMOYYnnS9B*sj)*ivm~V~yA#Ialj7CvC8|ndsw8>I2bf@{%LqZNPWC zwlSOrEpP^36{ojzI)gZoLl3Ucp#j%*CGmvmUD1sK%nykKJz4|w+ZzcOXv!fTQf~>PVcbS6SoST!I&uHA7 zuEJRMi5|9Z1RdA;uEoQj7z3U2q}go&*UbXae*VV?AGuE|->Rjqey^*{e$>KKtcY^Z z4Sp|1*zp^t~gax*cKCW4^;a z^2IxczyHb&zn!klvHNcE|U4z7+)|Z3)5kn44B$gUGEOh zhMkwgztFEuo+s~4>~{P8<}jI3{N+K{t=dcSWI=ZdE@SuoWVUaeJ7v9u zsoTt}XhmkANryDY>-poTa^wMvrzog|=a+~8T}D5;;&f#p;D=(MTNK2)AD5O~^OQ^B z*#6WYa$VC|eCx8R5(g0Rrvrr}2v!(BD*pZy)jCyJ8xJl`{_P?3s6Gcj!Tm5pn)VTT zLE%FT;w;2R!!d=_GIcd6A{L*ps9qu+Da$y|fl;Nn1&}8E`PO=sk_3fdzl;queCnigrDhY5G$bJv*lA_R)D`9rj zPhQ*u(YRO3?`|=DI@sBG#4jH;UmLlkKdVY(U^#JF^2$g+={;b=IlNr$W3cHTu6W5lV72=&pL!V=IC_PA&<1F(8I_0gkf7Ov2DhMj4 zBlZ-Z73&E;o_6^uAr!?6Gu=NLxbl|5a${m#NOPSpV!?51Pug}GBdbDBqyd15?uQWtIyYyJ@Ui9eR zzk2-sz_Bpz+!ydR*KBiExrHVSRPpMN0+y^v$Sn#U)hIA4~m3mM5((Ot62sR?7(M^kLPdne}) zQdM6TF8%?GPX_H*u3^pkgtfW@G*{D};5%--?eN8Cwmgje&;t{?o&CG|=1(F$8pc&CMnIqi- zsb6y`l)HeWTr~08<#(gM>&U@~0UU#FE^zneohduR1g*R$Zg_fb#UBH|G|E4GaV$LL zVj*-XEs#bg8AJkQp%Uy)r7ynYr{YpSabCO=YjkQj`i*_uhOqL67i^gXTbM0EK!-wB z`3vrPSi;QYak3Ue=#4E%B|nwKx$cR@KKHxw?>Q22dryL(B9|+!?VZbSChAw8jD({_ zyQq~@gSMMA#Y%FmwT(veXO>wNfLB%Z$BkQ5Tq7GDY@6Jze~*?T#m4Kx(lRyP0f^4< z5bh18HZba8YiBeV^lIC-m+j%;x!@ZLTc>rp*iR1$O*o1=F^(C_!?fo`N*PP9EOKzN z`dxk2#D!irKEM7KN3+V`O4f?jB>Z|an_Bw%_Bh2sZma6+qqM9~3y%rG?xOQR-?!eR z<5tU{CmHAZ*@KLwV7g3f3VvL1|63=efg5BnE#e9dA>TMXl~H`ALHj|ahL1;OhFoD) zTaCuEC0_x9Mci*?f@gIXXwDZN08p9civqJtIO*7mH<;yM2Hq2)I~Haj-nh)FT$b}H zVj#3v?6)izT#DU1Xd1*g5ZhWk9Zgc-3jOpHM<=~@tu z4#rI8QBc!SsQS2Q|EiogOlLMUBlVL&dFz;JUQ1AXqMY_znWmLnqat*Gw6hYmffv+y zIj(J+gLF$iH;84sEK(z&0S*2#MP-|Iiq(dm!{_pV^b(W2hEiX-ACfIm$r2_N!cqE? zV*yu3HF4w1qGUfeRs=U+EQepiDWx$cHeqF4D(`k|*Op@{MJubNN} zix`_A9?pd^!pMuL(A3_B8XH|UyN=@}nu$qq0lcYfqlQ8D*F9vKuZ*GxM_$wi@ls3l z=NUp^rh*#Iz1w5{qDsuyqin^AxG(qeWczDMpVyQr;j=31F=VCjJ@Sx`;B7pLjIhEn5M1XTsrJ)&q1v7pF%vn+65ej!O2xc7 zT|6d|$e0d!+Kb%48@M#8+2O2qaQCWBa#V<&AcYU$`q*RKLQnRXDhTAYhO${Mca)fu zhoFAWI+h2zJsp&}9q%{6svzvbgcr|s2DefpA8f)UV3|AYVq_aW3`I4DqxzU{t(<;k z&)0UBerwP-`4r`9CXq-UC`S7sb3X;8%oQ@(r+c`VsmS}8p7VZu^quGfQ!~Y*TcZb^y%porZ`@41#cs$%#} zk81Pe!NXd}#{tD-lN3Vg8Nb;B-&rl5S1?WV~dDjmnGN zYN#)$OtwtDy&`zIxP!LsHn&yrt{dWjF(Mu0RNT?Jss6yZ1@?Wd{w$ZwQo;pT7BV01 z5e3q6;Ez<_9qRBPBR5u8?Z~k=AL2J%X`5&_)RQ4yKSlgFUx5=H)V(EtSXQ`UArRE~ zf25`?lfnw80MV)0Du+YE2i8}6lEC#F-xc1~G>#^EOY1+x>B1ok?&*P#8ePL5`yK^V zuNBbQSez16`-_i{u^qt0xM# zb3e=9SI(Uid@JIysqFADL^zJyXw`Yuh%*%sMe3W%_%8Xk;yDTJ>K0duHV+7Q+ zZVf19yCcbs1n#!rl*^m!Zh<}WFQ)J^f{a%_e{NWpJ59uk4C*`Z--yyvC}2sS_V;E@gd*vM&w3dB3|u$ldgS zK9!=tTCQf1gYrXYv~5D4;qoMl;FVR~9Hk}oqpDe|fEH<6)2PEhlQUDd4*?V|TUNI1 z)GU=#w@lh6Ne8VuC?N_2k4H`h+uybJQFP=4%tQ{e+zQrU){5$G9#B_w&Mx&nyq3)X zSO2o%_K?CR-yxx(W3sG^vHS~_K9g4cZ#C@$ohkik5fWvx9>KRA6cKRT=hA2E+`)EP zd)fO#+Y=XXQGX}8iF>Ss0vW?__+_}-N)(QMI_LstB>zr>uX?^jFG`=ZhRcs=NM5sm zg1Z+u{O^CG6k)B@jz@?b$v-0}T!%Ru>#BZAimqr3H{Z;y^W@Cg_;GBa4(lmrUGG8s zMp9;Yq&E`H-cr(NwFFb5`g+ugh8sDh{fVQ~$L+hD6BRs)8H0)8W*%hGO5qWDo^3f| z&gWe`Q%j|C?~mTvn#S2EGJLO^{WC#uDcOTrcB+3FGv+g5ZKGGylL0m5)V@n=p0F11 zHrmRVT#~bvFB@msW>^do|++AiUPzX=4n9GcJ)S2Z2vTIY(EMI&o z$5d2Sx9NG&17IlTMjt4#`71^(Kb!Z#k<#e|inmiK4?2)M#cEf$%_ngMQ0=6DO{sVX zcs2SxYTE)TNvh>q!e3_pf;7^q@0~+^8 z^s!cdreuYuzYq9>#D8ThsE$eKh*GbEY?b>5ov($(ZtgqgDS51)0e zcsISgM@bmwpS9!04#P6Z(iaA9 zQW|1R9%0pu{#0AVC;~}MrszgJb<1DRi=WK(FLae8kt!&?Osvq#^fDzK*v2H9o*Fr|tr!_aoIX}ky&_qr-IxE>FJ;cE|D8%QxpND zAe=09a!---KL(c+43y1++d?`^g7yjcFWtMmX~^a5;v!J{&E+b{kb?HT0PUM-B-+d+ zM`Sz>=5LSp!xuP|Lu@Odqw*ZcUEg;THLkvMWXcEeB5bN8v=VZ3f6C(KCGF+q zXsyh#v5);EP69?{9101@uf$KCJw9EM0*;S6EHZjPv@tR=vNRdd+h@D&c*N{PlFfdh zxSB_O{0AOH>)uUG;k$}X7t)QI%@p^%f*Qk9um0ihQDtjiuHgURu~~6E zW7(Uwqq!k5R{FUc-A&7ZdjUY@EWQ5s|L*WIs4v{ z(Z@43;A(XQF8q2RgnXd_%eJ)&{l+EuAq4jN?sukH8IR*FexSg*WgbOMKS6o*KE%ED zk;T`3q;45wJkYWO2*yJS*zo2IxwSVO& ztAzZVBnkN}?sC7h|H_B(;-3Nr0x6UcX7BAEj6dBG??Aqm6K-N@VHjl*P;q6v%5%d+ zQG7*nPq?+BmeW?D{$%y6*;E{0D*4GKF0|Rj z+R+{o6_Nr4Kl{GWkUHrSm?-8^KHLTn1>3*>*li}2giOTOW;|o~r#nr}FX&`ke&y%b zp0LhyIxqW>be%6us;g7GOA!X#yHWa#tvy5It63ifbXh@|oKb&JvsCo#E3^01S|oX8 zL8p3az$UECNld+1Bs4%fI{&{CDLnb_RR9!oVZ%rO)QXu8qW&v~lE_CrPFeR4hcfgJ zhk}P|^{FeEyY5Aa2GiFw4@#I)zm7Y`lO{gDfq&m|ygl-B!t%C=P*T8k?0Y_bUUe{mcT)Af^su0p5Q|6anTxBua5NXe2bSza-O_hCbHDWj1)f zgXutMEc=nbSoSi>#p&kHL~*5KpJ>#_FU|lwqr^JDm)SSsslpOkg=W zlrKG7Ey7GKCKh2R1si%6_OH;JB1P%}@u6|phKDCZPFCI=AWCK6(xO0v<5?35Kfo(z6`A>eK6TWt;t_<`!!Jxa}M7V#=0KF1$Cv@n4p`~ zI4N6O9Yk*hY!(ZA*SR1`@JO1yH|V}AlG+I05%#*b6^>;Y`|l7E<+SUiW7kYZN&!W< z0am%rdm#e#{^qsV=bL)+)A9kj-H?)IkImq&`d7Ch_IZot0g)xINEx=u(^eflL0f1m zxIbd|SuH5OmDm;h1&(M1wWNF79v)-|J=7J@C*(8yPvi*WM1~xzy-FjXZwB`}?u}(% z+ql1q`xtpUlHoru9BdRzd`$D9wyTRgj|HO~YEErQR1zy~Z9RrLD6}RMDtpHBBVwz< zFVKyh%7ClfW`@s}LwGE1U)PnQ9-sHNbnIcsu`&$Q`Wq3!w^z;Y6}tx&4A`Vfl1$G> zlIX_&87#Z|<=uE_&dJXSSA>I`$v%*d&%{XbKeQuQ&Nw1hECs9PpVg`4AZVUk`Bs8* zCr_?->3U}bsqdgzR>|C0_p|t6s=EAupinR5G@;7XMkA5W@7?EfLQ1q#C<4WVG#rJE z7;4toNq;*(%+P4W_YQPcGwn7J*Aw(pkrNsCajlkVhNYtaRz5k+v-h*W!l8*eS~)G3 z9nd5P9&MT_LnBU`WHkk6_)eoY7=RNJjZOYVTSC_Vfv786Jd2`qDdmyIABD89-3Ts_R zy4eBI)kyJTNi~?p)=)GIOOyQ|7BC`QmBnw4&fCE<8Wg)lQ$Dw;6qZS;p63iE2Cr5Kmd%#I;76&trU0sYuGQ7d~LN z&{g7gi|L%PMwP4v)z)b&om9*9MT#`5x~^VFOobK^3jf)_!!PuM z#e3o^^rZjnh1p0BD=- zzwieCLxv0~{%s9@^iWc^r8MKY0kA#)+YS8!vv4MhlKgMXzlY>54Ad+B4UYgbb>Oe= zl$^*P6fEE_Z|UaHTQUBNQ7DuXY$Y7PnT*eHxhryP)l5xQ^%2|f9=y`_DG*nLq|fW} z)`ca+`g}Hj5&a(>#$)~`tM-5KB`>4d7B{s1HfN8@1UQP@ifU#-zw*;c_GKmjz_d%X zuyqG+mKlFHT9E%Qe((SKCI365a+|tg&J)jYzS&z-1-<_?R1>uSUpUF7OaGrh$-lq& ze=W-OV`ZYw>1BahRWJ}YzaK^ud z`R=Rh!2ev1cx$4hCB8gZ5M_uuDuPn3EikgAidPBTTMK7|;JXzbieeXOIg*{N|9JB= z)gq8@$w;WjhyNJx3o6M*{L3Hze>R){@1NvW^*zKQrTx$^ZYsG2bNinBS5f zhTaG`Tv4;=ViXR&elc}%aHHaaO;gMIcp}W#cG7W^l^@=E_Rld~xFW5$U+_(YOq}Q+ z%EbFG6r&%m>EcZVHHX=i{)cr$*7apmEHq|*=2vUTFOQ822XFQ(d|2~Dy}wWV*<@4B zNTWma`=G-fv6n`}0s;9(G&dNMSF~LGN!6PZB}YL5EL5dW@XWO4G`MZ-7u`EXuEPOl z0gbzgHqP3Mw>D#_13wD7;T24j-thdvurY5yT|7m9Rd`(yTdnmikI5>Q?r5bJE(XP} zD6Rj(xJy)>Be82}IHi>IA_3ku`4wR~#g1f4ibQuLVm!TB20E6&VKrY}C?0 z`ViJQg#0ivctPEd`8)B_vwOZljEmYtA%9)<%F6OSxIb0om0&$HB7EnO+iEXu++aY1(GQOz zG(GGn1pDCD3vYlJ3^sO#jWi(YN4T8W%s@_yWS>Bu&)lzi-VY?9e3XJY> z2zpKrvmFo?RZQi=iG2d?+dz zbywgZ(Dv}s2o3a%d{D!IFwuntWDSVjj`B1IM)k?L(7V2~DEoP7rRi+^jCy`-W;I}+PWtVvg;qp>_Z?>F_XJDOh`DdbhW$}r(T{R zGK!b7t#D8m%|g%NhWS4q5Ep!XxJXSbow&h=0MZt^OQAYc(1f*(JbxK(55N)7Iv>|Q z);()p3^`6fa0KuHj%Ophfl3GV#hd0C%z18oIv;rvTWAg0D;C1v>wGLat}epv#5OqV zLhOcKBov|=q8C~(Qi_lFFRTqVTznHtc8ihZL*iO~nVqm8? zJSf-%8D8DUw)5s^`h}ZltteE*@1TgY|MWX~4IiKrhAK4h>TIJm>|+sMNSSDOk;ybn;{v_&pAFsFG6OUMQNFYy* z&FD~V5I_6Lww=w$5izbJGpB-D{h*#3U+(Q54vT7NoI9SM_WRfGUOVIdh??l|b3c5X z8X>dqRd`c2N-ECwQAHx(u(3^k0VJSQxQ!65gS<&4mSBwH)AyBnv(Bh~PV>eVM-|m4k`eI4TwqO3Q8>5(JFsyADoV>Q)KWz7 zun!bpQDarE9lcG@(GpjEzA0(Z)ts593Za{@biNk8RYC7Eu;&)3dyD=ny$9}XB~6@` z8;1({_RTeR!|I#YHm9~F*M55RmLK$v16=Ei@!o=o+Ik78i{Xyeamd6*cn8+7!t91; zZjlX3g4nQ{S~^{HBOc@~v)W1#9zK8Q8lzT85$<=#Ua$iZQ#wBzbXp|@A|gn_XZ=gJ z2|=7YF;P?@^DbtI{+!ajaxdp_vY!akfL#@0-Cq?V7TEjR(;0+I<=BWfF^d^s%bV$~ z-Pdux9uoSIVy{;}VeWl3!!{6tByR8&gx9~TI#tnNu&=>I4`rJQ!mE@*m)iE{yhK5W9te!wLpwRgzH@iBHa*3U z+JC*rQT3%iEktmG2cE%b;Xm=K#sK@;hMM<E^@atb@IZETnY z0LJc1xz6=yb7xnX)_U3-x%I)^`!`hUU4NGUrd=t5(rv7DYVFcvUUkVk1SQy$0n}`% zNP+zV4bv-F_-68;H1$I{j9)EZFK%=pH45=4o{aifN*ha)CjmpOc)_*X5NUuGUM0EXWWj6FPfn z8ondj+(C`7C`X?xo=;dix0WIwe8jsqn-B_(oL}v78jP})d|^@7>+)px&2O|64_roF zKbTZEl#764dkHBW{gp+g$aU$N)AdUA3?SNSp%&=UiHK(LEFTsb3#fH4f0}(0r1<1oC5hu! z+kM(;*jmb!M@RtX9(_1!48P-kd#7?5&1HkxWjq}JeYUlEKZgr`Dw8^xh7q8I)O12i zf)*D~_D;BNurob7kS$o=x2f2IXgT|@1@oZ`?mNN@u_(yqG1Zy?f)x|#R5MdqUQJz`6op@->MsjA~!?uG!kZ~9U}Q3 zn{-~vUOR9+)rCYX4wP-HFBPPT0ohOQ?g%4+w`ug}DXy!+qe=`{BzcD*>ab6~*vAjy zZjTVkPMgU@V_Z!5Z?ij+qF9)%8wP1_F+~Qb?nP6G)-98yzU-qxjTZTxFA8nJGlqNx zMK1)f4uc1}kgINg#&25p zPBs)5vZ>r)5^KlDmf9T8lp7%^m`{4D)`HTN%=aN8@Y7(S94!~-!IfUNh|B4}Z|d)R zp8ul9RH;iPZShS(F-m6+?Wi?zj5Pez-o_oaQP$rjl`FfA8+Ch$GOhAe&p@!q!#M^o z>RVxd(USTgGvHa8l7BVI)ZGKKtYj@d)c#y3OQji?`MxvlF$BpGI->g;&qgbi{&bm> zYxWsBx_4f*>gBJZ_z#UaB9dvY_J`t6W+13@V9YYTNV!-Cv(G)CKx?ZvWYf!Ej@|T6 z=WXY|MsqOYWqMlq_TlEYIn@tce9`0i0VIUd@lUB;&tkSo);gs;)VQD{1NUK773iRhQ2ON1hOSAWbvPA1*!rYNDNkm?|0CJ=6}1xhv*0ySJxtdZ*2ct998+?gu;DL*?-7e&^QK zUR`9?>tP#s#yK_m%i0xcKPi-inl&PA^>}6+n)cd(Yp@&AO+iQNyUu7S*Tus!F!T&J za*c*#HM^q2#EZK@d37->B$IK8KXGGiF1@?HIGXyEmAY{mFfoh27yFP)T^lJF8OtAc1Tz{{rn{0h+unH@Z|DJjCi3a-VGBw5z;I?Q1acU>grF#w?@`7u6C`%T; z@3Zm_4SlD&m7FJ=O_w(>sWmuuG#~9B+=`9{Bvk#z)T_k%1<6~)k#CktBzJIa$FGeO zCAwZ5KF{2khFaIHQtrb~O0b-pUDe^O5fc6PPLJy;Q>;PFSq%yrOK}UvBU%>WKr~~2hZ&u-ChK;m=fP%r=ai~8z~aF3q3)(Z)DDG4-kB|u8y>P4hjIUXUEL*_ ze1F}pL*7$W=?TAkM@Z+W|Nd((*<~(NVZ4WdAg*NQ6-j^Ms`+Y1S*qFitnFvc%7KXm z)EKD|k7uw0EkmQ7$A`Ow5pUUN8|b3KNIz5tY~5fDRxGl~>7x&CU*V5^jW4wCghS6f z6w*T}LJn#84R;LX`mt1{lI{NT=(ImTY!HHxW_Ge^q2xomUPhxI+f(O8tI)qeW(@1( z)B(*=2et1!qDhdm1)j_cnG9a2*LC7&dx~}y=`dY{^JPXdZ<6Mg#kTt}TSQmxJ2}Vq zoL7KKMOpP)qpLTJ#En@>Mwtm0T|HDRj;b_|4FyU;MJ4d|jOuH@>nrnslx}iGn?6yk z{|)_l6G7JnfxMfF4i`GxfDVxDVUbtVFh>?^_-;H1@^7djSC?CrwD@IOx5_!RaL0MO zc%7Czu??qB1&+_0c}3HcAKQiuz`aF1l?zZO4a7m@tH-Rz&Xux}q(?597SASpxO?2K zjPnkKX|*g4dIfAVx$P�t}aTcxv4GOGLHKRILZLw=L+&IKHjzwj5ZjxM1Ej@b5Xf zsx6FNpI^)<+|GL^ubp!`5@FQha|fs#@9Csp4izv4*5F&!O!*p;w)^$ApUkhqiZ^%O z`=*;X?C~`0E#`}?bMHB1=XYLyyzxZRPB7EP!CDze!8SV^^!Abz45)-RMQccfs2U(c zYS?{S{eWmh3fB`(Ec;O`eOk^uR#c!S3J3AyMQMwTZCcs zR5Zcmd9d5EgtKchg+}-p(-Cy9dklFMReFM+^IFgFa!AAWZO1BCy;H9A^a z{VWDCaIqFQ2lNO}{6GaiUM1nCA|pn68+Cr{qEu!W7+Xsn9izfN7@w9ba{KsuL2-rB zNGqn=>7$V5No^lUsDUL{aoQsT4bSYqRRlGiKvSokzM38KfHTpLy*NCe9%e+8Rl9fpR95F%-UQ#3=fhVW@uW2^ewi>eobpb|EXTCvy4&Bxsq z-jWH`c)W!VOl3m%_Dy~W&CqKl_V?x0C#`{d1UGlac?dLk4SOE4manj0iC9^qyc2zA z_EeN2-r9$?F9J@+c6OuYeL~rCL%npX1BH>@?e(%`I@BK5n)M!5pd-VpK1I-ytldX8FW?dUTtwOaeC7}7S&`JGiGUVSpEAy=WfGE) z$~Vwsd9jK3BE;1S`Mzx%gIZa~^QTshuu(eF*VN#&M3>wxRDtq zVW9r!d)w8f4&v>y5GZLRH%S&{@(KF|2w^*AG#f<%$X-LWzsI`H7mvYFLzhdo`mkEC2@rsnwluM;))ph1nMl#MM9pxP>)} z4i4bH)G{I4@o5muZclhZNN#ey_MG5pHFsr*T);qh4Kyo}8eLq0pCou9ya5+uy7A^b zQ1j#<86%MOWUpbZ6Hms<{RhdOKxg^9yKbA-0K=ozuPmj6%DD_)z}eN*gIG}ix;)cK zfUpf(+4CjUiWRGQ>q5z>6uCl!A>4pfweL|Od3GCCc5%iVqz=!A4`<`wouYSJLe(vTMM)0cgEvO?f zygU8GvH#-IVQff)vPCC}9iAVb^+j_1s-M4({ZPiTdwl^NL&axfUxOy$R6A#KcOi$| zJux#!f8=t%PJIug?VJ+4?l8=y=GX~qJ<%o-yUw4Erjb$Ces`Bae60KS^=QW8*3P;| z9#4wUbizfON7PLa~dHoFgD6m1K7BD(Soa-&cc%WL%B9Pq(Pu{&O`8=&^99qUE6x<$C z1j4MyqikvP7^h~|zB97!z;6fmxgem?54|J+IOM!=4f@hSPUUMPs0reDu=D&ma1&N; zY)Zw09722NrAy85t3yxBpVc#NfRNI3sFv${ef%!ash37b{wz{o0gyHa<3iL$tc5@i zp^d-IxXz|V#L!nZtLnvoMUbe?D$xv7XmGQr+w)b%#n9f^yhP)|U$v;RDVhqtea#W6 zlG+yFpDA+$H*5-NpQI?xxj<}d;dF6qZP4kCu>v-c7hn+p^xK6p>$3h8x&uaH3P-@IE;kCr4566%w2j@C%U|KM+N{~TyRS9?##Db7C| zh6wo05@JbsEh1V?x zOU!8vsSkQ6V!QwAoRa`JUuU?mw z*EX7Ts~TpkvtY`8rqk+`KjdQeIH+^tg?8g{tL_g}D)~hFExcJiHx80@UHWWDK0vXr7S+ z2~{9JGKTv0LUg;OW%j`NTME4eZDQO+66Sp`IT0KSqJoVhkOXksk3gcHS;3BH;|=%q zQcVhJoe$r~5e@~>(~`M!Q_Kj!Nj#JA-deWvgq!GDyQAo zo?Pl%8w_Y;<-0(ZaH-<28zT_PJ^F2cRwfflLc$QIA$gNyJ`y0l2!9V&p z4*-FE)AEJ>FCy%C{A-IAM}!N;ppRa9AL>6})$-wTJ+o8OWi02S1(feJ7KuFxLC?q^ z&q^My#Iq6qxG16ZPXoV8b+}s$D%n6&$Ytxlu7L=Toz@UR0@G}_AHsDts5%;Q%(Y_w z*}F$yC{@rYTTgv|>h%cCu<3I2$ZsoOErivgpFKPg(9#jH%7?O8(Zh-&LJm%%M3+Kr z<#J3%t!l-vsu;}YLX1toioR4_?2IoRQ_B*G!w7tu=an!A?Ot^jgBI=zC*$FNN9kDz z``orwW8dAH(=y$u?RUt zyicwZV7IDfaVCD|EVEVsJtc*zeQm)%fQ$B+2y0GH$>C6>v}SSVK2R>roEH#tiuXg> z3l$@Uu!{{#Z;&1z>hRA&z^ISgYi%Vy$0oTTC4|ps&A*}i&uy*+B4WCJw|!mceALy9 zp!cDW&zrGRXvl-{oQI}D1-12&Wa2@)g5Aq+2G%ffhJ8bivO*KxMg(g8+qYC#SIfE$ zjUNd4)^NtoN4alku=&Oz$)z9bVK(e3MWoAWv$+mmUs5R&5C?2?Az+&)-xiD)5hoko zGtNT(%);<`Rvwq{mdxbX{bJZKHA)|>?PO2g31UtxXCYdvJRGfly2piW2qr61;yIRI zO;Jbi<|YqO7Fu|NX>%R_x|N)tmk7lZBDg{RUvYn4tKH{)@3SITlMWDdah$!-BupB0 zMODO6NeJOZnnFXBYew`CeZUqm4=ScMJ~XrHlW4hIAs)jrp7F_p|I)?5InZ$8deljf zA=or}5qk<(&?}!c zySwU3*ZqGRtX*>_@_<+Wo?N%Ec@MS)aFR3ZxvT$*Zotr3jd{4P^xG6N%;SgOMz;H# zkVOq{kSd%_l5M$cX8e}P;@B0I9bFSlX7+9|&m4;%_F6tW@!A#1n`EPJoL+8b1Y;IA zSlC&TAq9q`f+pD;ZGcnsl#WMt41-M_svbnGMhp-7P-j8NM}A*9rv&C-Hx2b}neB_G z-aj}&yQ0J~#@N(+TtT6M0;<(g|u+n^RMsAbC#P0R+x=)-I35kc5H{Dq|_rqOgv9AS;tO>nIZ-glNn>oSmu1E=ed5r=bUq$>s;rbGk?!qGuM2- zpX+a(k-JRY3ju*#Ut)HE@;ljsc)%{BpoA#Bi zSfkL*&-(sT4;&tF=&Fh?g=Pm}%WE#Ax;eaVjAz#w%3KTAtYYtRi*pNpzx=s0#ey5sBtVrF?57tno~-Pd@#p!HZ-thbJ{|AaYtSv18`Sq!PbuI5JZ64VxAofk~BfHbya*4 z@68iH-lhe+J)AlSWEFbRksayV)(JOQ<-s=2% zzdCLJ>NOQ{_whU?`^{Zwq!`(H`V5N;aM|L`T_%34U;E+TgE3d~HUVp2IrH2Ov@Wo8 zRfZbEFPDDgX{5Z}p+ehOJ*f!`6TpY?x=uD^_B1(tuBoLtUHy>}lY~$B`)n5KV$ z@pA=kdm7#qF4vJ#RSjPez@Dc)iA@^Dn0EZs%hw_K6YStxA>m^hA|gA z@kdwR;vNYu{6Kg4SqvZr9ljRU8tg7WDo!ZWM4pIwyR|9uqx7px=`h=v-*&0z5t|9M zW%(prUoe=)agQ@t`?J>*ILY9qz!v0o)P0g(6-G?H2GOqoM9XCM{>=K$@>h=}+E}*i zjh{6gZbqZ*@pTCYzre^be;95TU~6OEwyu1R<;9C~=(n_zGIft$(l(z2d&S*7(pQZ zzVx}JP{ItGK6Y1l9cK4udW|0VS83SElP#JmsWEvSaoURC*>bJb|7y3gMElR$^B2YH z*PYkc^%VjSBHCu>9MO`Rk_~a{$`REbKtN=klRT+9$jzk8c$VV3Rl5}!`J%6TEDJ;v zSTirY35ZmoJ4seMWMB>m0!{tS<# z32z21p*3W%iCPoso2C@BY-^K^uLg$lnMiPP905}z!6 z(n+y74$$q#fK@}+3Qw@IE8p)#pBa4LpO_k4-CL~v@ljK?Nt+rPhk5@lHQFivL;y86 zc_*nwVS-8%Q}(?x@)>78kU_;m9!eoVj0{GG&;m>dl?jpBiG&m!;u5MeDR4No@~jBI z0T{8rKH6uqRyU_J!u+9Ow0W<8cS$az^V2xG7~aChHJNjJIg0!t zGksk%-&J8c)#M0Dj!nWkA<40^RH(p%9msQC3FIrtNq=Mg=tqz$A zS94xQ?1+utZ=(wE5syR_9fXF4*e>7-C^Mh1&gm4&SD&tZb2UDaUVo^z9PfkrG$p+h z?o?IfrSl0}28lzKj{l^C+ix|FN`$z!B&|HRhCQLhaq27DENC@u&rj1gz<__JJjYWU zFt?S9B{{IH>Gzf?W5`Wqy*d1Z;AE|Axxnjz0=TtTUltr+^Q^s*gDaO6kF9R=O*lwa z6_q38NNF*Jb#j^PCrsQ>j!us}*s#{JKzR*xe>Om{>hkA;aLcGb<)MMp(ysN79j{c^ zX;B*mp(yWJc)xw_rD>?)aJ>r3DmtKp$jB}WNkf?JK zW|DGxI++J6C|bcFX`_SwfW}>1@MaB;zF%Rpl(5%SMR-4FOZzUYGCts4xVqonu9ed% zdlZ4wI?ygR5?)Qu2U;wqWiFtRcJE-M2&bPr$yegmeW^?SG#E4A)UIg9flwB@ zRi#9&vArMv{{fCQ3n%_Q>WL~nnLBd`M8@Yb4XjKC8 zApagU9UC>^g*S9Z>pKcP)D7298DK30o>^Rr`WMp7)vQG4rShOQ>sS$ zw;gzo*ZEqyHh#vW*x=4I_-K=e90|0soPIp0ysF(xdB)1Ec<0G$}9S#We4w5sX|Uc z>B9wt9VXeKbrbJc{DTsi`q%5;+CO41E;|X0$6lls>Q6)tsZL*OEGq&}jeZKb)7U#P zUIX)NSM3T|c$Sm>a=R+#1XUy3wtuTbSuEzoD9x18>XrM4`VD{Upqn-FMU=uhnNU1A zoI0%zJNXHZiKR*8&WK@b5*45a1Z&@T&*H2M1yZ>9beklp z!(s)%_+F4C9kZXkpRZmW%xlIEqKPl`HEiRM`1mJ3q|e%~5lY0NgSY1EJ*s>$(gQJ8 zwQGTELmYV$DQ^&G)4;nwro)7x&*_f4W$^4G(6U{_+ftkf6v>fmvdJyiW0|)z+Kza*uNPQIJRecnh3%G) zH35``+Q$Za+aH0ouwB_Z?f5;rFC3yk6&wS!YqDyBToOK?i0k|hwT{Y;-yLcayKyY4W z{>|N``f`^Lfk)|+uh!R!;zJW_{gocsW5v%5?y^BT)Uj)n0Gt%vJL`o->*(z-AhqSQ z*Cz)m9xK0fo~k&dBNOakbOx=HN$VuPpnI$W%{NkTu`9d18{>=-$fE)oVGjsvL}UmM zqy}^KXBK-nZ%b*5xe&b#3S&qomic<>d?Jgd~Z$vL; z%TJQ`MJoOdeY)Yh>pM@bn|LpQrG8pn1M=+-JiLoN&vtnDK8$QA(DiU<_E5WOS?xRq zL0TCwn}M{J5VqNQ`S6%u%)lC-E9xfFfR753-|N$VhTbF<1A8;rqoxxr`Ykk0A_yog zCJYMABM;=*lV-M$RY&%?%tYD?h81^`x4+Wf)HZ(F8!t+-(Ef_Glj-|TTzuB4tGRdr z*w?%7G&2Ac0bv1Qr&7Sa)+WC39wux0AzLlz`GrY!-~lkENg7_ zCHY}h8Y>>yvS$ac!k-{lZj$09{WPZcnYSYMTX0vpi4Fak!eSw6@FWPym zBij30RtKy=73LS(zaNQ>1sPILnbR}{2gzd;sR>Qtyk^-5gRvg2Hsuq``;R}&*OXdz zJ$z4cU-PSIo|O3kIfv}j`x04gv~&1#%cnI6_pQ|n(6cP~+{X$-N+%b<%_7NnWOQg% zT?EO7Xk?0%rdI?zn=l$y2VHN~2k_mLXMY_hZl*_ZHcH;|e9mJO4^uX|xbksFP`OM) zr&ZziF9Y)2&~WZe#m{@~6R2o8b*j|zg+VHFGP5Rb|~J>~un`R{q=?uEg~zCbGS_`sPdu{C&bjvlMeq zS&``86c)eJ{T|hj$@eXNH@P*!$ZuO%-A`xIOx^LNv+_xo-LA1J^Nxd_8VkS0OsVNE zEzP`h^dJqyvVGm4avmtSV*#Dmm0`G^4;?oj3t{gL?+Fjxk!SfXQmXxYZ%U4qPX9&7 zSQaS@{O~zyl>LixaEvPH7bh8(BZRGPDGHjSZ-YlSYm;4rdcwC!P#&;GlX`nZh=<1o zm!MU_nt4F_66#Sony83K79W{eRQH+Oc2kjFbDbB~w6IsX5N)`6+Fm^t^I>b3&z)!* zuadJk-75mSGMdvRSTGT~iF>sX&%Q^}O)7J}se5Q=(>Ug4$ns6>S7Jlz@{Tny?u4)5 z;TGzBbXJI&^KDTSzM;zkz-z(27#m)rpFfr{?0USF(}6E9qWBG04xN)ywD&lo>G)Zd zlhxS$u=qPx_luG;V^!VeLG^rtr+o(>)|lP8U9jkvW5VvJXzV{QenPie_pQGhoNY<8 zU*D~7_z}88?_JaqZSlfyvlf*0wX2detOIhaPmFw=)MoQ7H9dm)ha0guiQ=dEOP8M7 z+UaHdNAh)D>ajY-#AH`ILBjFfpvZWqku3e?oAd~4KC;JwKqTe*0?hK2i!AJ#$ zFl44rsSWHd3h+D@n8A*6WBXVD!LWKTs5^<-NKkh{*J><2i%`4x@b;kgo%E+osO=DR zw2HycGVg&R;9shhaE-WC$Oi1!c4PHRL>ImidW#ki(a`wo7Fl0!uD&;TZnihQV&9_H zjQXdlFWBPa1BT+9}duL)Vx&6rE5Qbv=7Kv z0x1x$TS3}xf4LZU!n%VhVm)|UK@*fxA|*QSN<%&^@xVOu5@{~YyN(+^Mx1OLzrM&( zlDs49ta+V}+v+?<5l*pKz(q#LU;1dn%@~)QeJGOPVKk85yP3~A)Ku#^XBmC~U;|Rt z`E_B$k*TxA$<|ZX1jPFluiK6iXmRFK!kk*+W<7xf*n(e8DnNs|Ac&s`iP0@`2crzp zJWYGHE2@w9o|8$pd-wUKM~OtCF*6{@VmK?Nv5|F?%l3evfAnh2p$sh={h$*nHVW;t zfgs?OyW8k}*M~d2fW34Nokpz-#7_^*F?~7T`9TG1iHwp_QrgLn1@2AajiSOrVbqZq z`%B6-sf|2ZVYWZSpnc1^TXWrBx$Ix~r5kTUqLfZR?nLaXfkl+Y@9=7?Px_%52NhLt z1(ToQ%5y-?m5+w|Ea(HI{oej_CMJU>uOWW0OB7(qtwBg>kr%`xj#OP_VAu-3XdHNb zcRw26cqiLR!=4#e7Z3MCM=yx>Ii>J=5qmAdv#+-35$NWPMKRu*+8U7p-GQR^nJ6bg zz;8ERBvNF^tF1KUR|&%v57jd)u7%DpR&sqOoCg!yipv7M3Ki4()y=&1C%e2c_#~5L zPZkgx!Pu?=ETXnh?e+6F<0rlZY7E0*kBQ*A?cYu<{5<@}lV7YEHAp~+Zp=h_h6#m(P_G@M=0#KTR?y(^w(ou113~o~C zcWyT9mnuunznK5T8sXT-z~T_`hSS%}0mF%>Fo)cH#~Y;fAOB9kyjJ}&r)^|d#Wlw+ z@i%bfm#Bi7GeHq%4D(%QRuF7yDnF;7*bCo{jy;g)S3Ef|SDHvjR2ZMx$WUNVVdC`r zW|ft9|2w~z`#(W<01@~Pbjo?4rM%h@00jY_46pd2F!7$5^{~Q_wg>R}zlpNn*;b3R z+P|?p*;Gcv6_5dSS@ERINk(#$;{Xs5RV+;^89-GkqSWTmrxOTPhkx^ASueqT?oj9G zO9-VA5uc^C-P+Zi_k|lR)s=?7xyOOu%-dd@Y(ZgS_7o#1Yv|tW*vse_{}+yok=pbR zXb|8WWzE$Fly+0@Rou "one hundred twenty three" +NeMo has both a fast version which is deterministic :cite:`textprocessing-norm-zhang2021nemo` which has more language support and a context-aware version :cite:`textprocessing-norm-bakhturina2022shallow`. +In case of ambiguous input, e.g. + +.. code-block:: bash + + "St. Patrick's Day" -> "Saint Patrick's Day" + "St. Patrick's Day" -> "Street Patrick's Day" + + +the context-aware TN will convert "St. Patrick's Day" to "Saint Patrick's Day". + + 2. Inverse text normalization (ITN) is a part of the Automatic Speech Recognition (ASR) post-processing pipeline and can be used to convert normalized ASR model outputs into written form to improve text readability. For example, .. code-block:: bash @@ -44,11 +56,27 @@ Quick Start Guide Text Normalization ^^^^^^^^^^^^^^^^^^ +The standard text normalization based on WFST :cite:`textprocessing-norm-zhang2021nemo` is not context-aware. It is fast and can be run like this: + .. code-block:: bash cd NeMo/nemo_text_processing/text_normalization/ python normalize.py --text="123" --language=en +The context-aware version :cite:`textprocessing-norm-bakhturina2022shallow` is a shallow fusion of non-deterministic WFST and pretrained masked language model. + + .. image:: images/shallow_fusion.png + :align: center + :alt: Text Shallow Fusion of WFST and LM + :scale: 80% + + +.. code-block:: bash + + cd NeMo/nemo_text_processing/ + python wfst_lm_rescoring.py + + Inverse Text Normalization ^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -96,23 +124,23 @@ Additional Arguments: Language Support Matrix ------------------------ -+------------------+----------+----------+----------+--------------------+ -| **Language** | **ID** | **TN** | **ITN** | **Audio-based TN** | -+------------------+----------+----------+----------+--------------------+ -| English | en | x | x | x | -+------------------+----------+----------+----------+--------------------+ -| Spanish | es | x | x | x | -+------------------+----------+----------+----------+--------------------+ -| German | de | x | x | x | -+------------------+----------+----------+----------+--------------------+ -| French | fr | | x | | -+------------------+----------+----------+----------+--------------------+ -| Russian | ru | | x | x | -+------------------+----------+----------+----------+--------------------+ -| Vietnamese | vi | | x | | -+------------------+----------+----------+----------+--------------------+ -| Portuguese | pt | | x | | -+------------------+----------+----------+----------+--------------------+ ++------------------+----------+----------+----------+--------------------+----------------------+ +| **Language** | **ID** | **TN** | **ITN** | **Audio-based TN** | **context-aware TN** | ++------------------+----------+----------+----------+--------------------+----------------------+ +| English | en | x | x | x | x | ++------------------+----------+----------+----------+--------------------+----------------------+ +| Spanish | es | x | x | x | | ++------------------+----------+----------+----------+--------------------+----------------------+ +| German | de | x | x | x | | ++------------------+----------+----------+----------+--------------------+----------------------+ +| French | fr | | x | | | ++------------------+----------+----------+----------+--------------------+----------------------+ +| Russian | ru | | x | x | | ++------------------+----------+----------+----------+--------------------+----------------------+ +| Vietnamese | vi | | x | | | ++------------------+----------+----------+----------+--------------------+----------------------+ +| Portugese | pt | | x | | | ++------------------+----------+----------+----------+--------------------+----------------------+ Grammar customization --------------------- diff --git a/nemo/collections/common/parts/__init__.py b/nemo/collections/common/parts/__init__.py index 96370a6de1ed..f1997e0c5dd5 100644 --- a/nemo/collections/common/parts/__init__.py +++ b/nemo/collections/common/parts/__init__.py @@ -13,6 +13,7 @@ # limitations under the License. from nemo.collections.common.parts.adapter_modules import LinearAdapter, LinearAdapterConfig +from nemo.collections.common.parts.mlm_scorer import MLMScorer from nemo.collections.common.parts.multi_layer_perceptron import MultiLayerPerceptron from nemo.collections.common.parts.transformer_utils import * from nemo.collections.common.parts.utils import * diff --git a/nemo/collections/common/parts/mlm_scorer.py b/nemo/collections/common/parts/mlm_scorer.py new file mode 100644 index 000000000000..c38e4b25ed72 --- /dev/null +++ b/nemo/collections/common/parts/mlm_scorer.py @@ -0,0 +1,93 @@ +# Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright 2020 AWSLABS, AMAZON. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import List + +import numpy as np +import torch +from torch.nn.functional import softmax +from transformers import AutoModelForMaskedLM, AutoTokenizer + +__all__ = ['MLMScorer'] + + +class MLMScorer: + def __init__(self, model_name: str, device: str = 'cpu'): + """ + Creates MLM scorer from https://arxiv.org/abs/1910.14659. + Args: + model_name: HuggingFace pretrained model name + device: either 'cpu' or 'cuda' + """ + self.model = AutoModelForMaskedLM.from_pretrained(model_name).to(device).eval() + self.tokenizer = AutoTokenizer.from_pretrained(model_name, use_fast=False) + self.device = device + self.MASK_LABEL = self.tokenizer.mask_token + + def score_sentences(self, sentences: List[str]): + """ + returns list of MLM scores for each sentence in list. + """ + return [self.score_sentence(sentence) for sentence in sentences] + + def score_sentence(self, sentence: str): + """ + returns MLM score for sentence. + """ + assert type(sentence) == str + + tokens = self.tokenizer.tokenize(sentence) + mask_idx = [] + token_type = [] + attn_mask = [] + ids = [] + for m_idx, _ in enumerate(tokens): + masked = self.__mask_text__(m_idx, tokens) + mask_idx.append(m_idx) + ids.append(self.tokenizer.encode(masked)) + id_len = len(ids[-1]) + token_type.append([0] * id_len) + attn_mask.append([1] * id_len) + + data = { + 'input_ids': torch.tensor(ids, device=self.device), + 'attention_mask': torch.tensor(attn_mask, device=self.device), + 'token_type_ids': torch.tensor(token_type, device=self.device), + } + + with torch.no_grad(): + outputs = self.model(**data) + logits = outputs.logits + + scores = [] + scores_log_prob = 0.0 + + for i, m_idx in enumerate(mask_idx): + preds = logits[i].squeeze(0) + probs = softmax(preds, dim=1) + token_id = self.tokenizer.convert_tokens_to_ids([tokens[m_idx]])[0] + log_prob = np.log(probs[m_idx + 1, token_id].cpu().numpy()).item() + scores.append(log_prob) + scores_log_prob += log_prob + + return scores_log_prob + + def __mask_text__(self, idx: int, tokens: List[str]): + """ + replaces string at index idx in list `tokens` with a masked token and returns the modified list. + """ + masked = tokens.copy() + masked[idx] = self.MASK_LABEL + return masked diff --git a/nemo/collections/nlp/models/duplex_text_normalization/duplex_decoder.py b/nemo/collections/nlp/models/duplex_text_normalization/duplex_decoder.py index 5aff603ffabd..4f602f90da8b 100644 --- a/nemo/collections/nlp/models/duplex_text_normalization/duplex_decoder.py +++ b/nemo/collections/nlp/models/duplex_text_normalization/duplex_decoder.py @@ -18,6 +18,7 @@ from typing import Dict, List, Optional, Union import torch +from nemo_text_processing.text_normalization.normalize_with_audio import NormalizerWithAudio from omegaconf import DictConfig from pytorch_lightning import Trainer from transformers import AutoModelForSeq2SeqLM, AutoTokenizer, DataCollatorForSeq2Seq @@ -35,13 +36,6 @@ from nemo.core.neural_types import ChannelType, LabelsType, LossType, MaskType, NeuralType from nemo.utils import logging -try: - from nemo_text_processing.text_normalization.normalize_with_audio import NormalizerWithAudio - - PYNINI_AVAILABLE = True -except (ModuleNotFoundError, ImportError) as e: - PYNINI_AVAILABLE = False - __all__ = ['DuplexDecoderModel'] @@ -105,8 +99,6 @@ def setup_cgs(self, cfg: DictConfig): if hasattr(self.tokenizer, 'do_lower_case') and self.tokenizer.do_lower_case: input_case = 'lower_cased' - if not PYNINI_AVAILABLE: - raise ValueError(f"pynini not installed") self.cg_normalizer = NormalizerWithAudio(input_case=input_case, lang=self.lang) @typecheck() diff --git a/nemo_text_processing/hybrid/model_utils.py b/nemo_text_processing/hybrid/model_utils.py new file mode 100644 index 000000000000..174ed6737704 --- /dev/null +++ b/nemo_text_processing/hybrid/model_utils.py @@ -0,0 +1,158 @@ +# Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import math +import re +from typing import List, Union + +import torch +from tqdm import tqdm + +from nemo.collections.common.parts import MLMScorer +from nemo.utils import logging + + +def init_models(model_name_list: str): + """ + returns dictionary of Masked Language Models by their HuggingFace name. + """ + model_names = model_name_list.split(",") + models = {} + for model_name in model_names: + device = 'cuda' if torch.cuda.is_available() else 'cpu' + models[model_name] = MLMScorer(model_name=model_name, device=device) + return models + + +def get_score(texts: Union[List[str], str], model: MLMScorer): + """Computes MLM score for list of text using model""" + try: + if isinstance(texts, str): + texts = [texts] + score = -1 * sum(model.score_sentences(texts)) / len(texts) + except Exception as e: + print(e) + print(f"Scoring error: {texts}") + score = math.inf + return score + + +def get_masked_score(text, model, do_lower=True): + """text is normalized prediction which contains <> around semiotic tokens. + If multiple tokens are present, multiple variants of the text are created where all but one ambiguous semiotic tokens are masked + to avoid unwanted reinforcement of neighboring semiotic tokens.""" + text = text.lower() if do_lower else text + spans = re.findall("<\s.+?\s>", text) + if len(spans) > 0: + text_with_mask = [] + + for match in re.finditer("<\s.+?\s>", text): + new_text = ( + text[: match.span()[0]] + match.group().replace("< ", "").replace(" >", "") + text[match.span()[1] :] + ) + new_text = re.sub("<\s.+?\s>", model.MASK_LABEL, new_text) + text_with_mask.append(new_text) + text = text_with_mask + + return get_score(text, model) + + +def _get_ambiguous_positions(sentences: List[str]): + """returns None or index list of ambigous semiotic tokens for list of sentences. + E.g. if sentences = ["< street > < three > A", "< saint > < three > A"], it returns [1, 0] since only + the first semiotic span / is ambiguous.""" + l_sets = [set([x]) for x in re.findall("<\s.+?\s>", sentences[0])] + for sentence in sentences[1:]: + spans = re.findall("<\s.+?\s>", sentence) + if len(spans) != len(l_sets): + return None + for i in range(len(spans)): + l_sets[i].add(spans[i]) + + ambiguous = [] + for span in l_sets: + ambiguous.append(len(span) > 1) + return ambiguous + + +def score_options(sentences: List[str], context_len, model, do_lower=True): + """return list of scores for each sentence in list where model is used for MLM Scoring.""" + scores = [] + if context_len is not None: + diffs = [find_diff(s, context_len) for s in sentences] + if len(set([len(d) for d in diffs])) == 1: + sentences = diffs + + ambiguous_positions = None + if sentences and isinstance(sentences[0], str): + ambiguous_positions = _get_ambiguous_positions(sentences) + + for sent in tqdm(sentences): + if isinstance(sent, list): # in case of set context len + option_scores = [get_masked_score(s, model, do_lower) for s in sent] + logging.debug(sent) + logging.debug(option_scores) + logging.debug("=" * 50) + if any(math.isnan(x) for x in option_scores): + av_score = math.inf + else: + av_score = round(sum(option_scores) / len(option_scores), 4) + scores.append(av_score) + elif isinstance(sent, str): # in case of full context + if ambiguous_positions: + matches = list(re.finditer("<\s.+?\s>", sent)) + for match, pos in zip(matches[::-1], ambiguous_positions[::-1]): + if not pos: + sent = ( + sent[: match.span()[0]] + + match.group().replace("< ", "").replace(" >", "") + + sent[match.span()[1] :] + ) + scores.append(round(get_masked_score(sent, model, do_lower=do_lower))) + else: + raise ValueError() + return scores + + +def find_diff(text, context_len=3): + """Finds parts of text normalized by WFST and returns them in list with a context of context_len""" + diffs = [] + pattern_start = "< " + pattern_end = " >" + + def __clean(s): + return s.replace(pattern_start, "").replace(pattern_end, "").replace(" ", " ") + + index_start = 0 + while pattern_start in text[index_start:]: + index_start = index_start + text[index_start:].index(pattern_start) + offset = index_start + if pattern_end in text[offset:]: + index_end = offset + text[offset:].index(pattern_end) + len(pattern_end) + center = __clean(text[index_start:index_end]) + + left_context = " ".join(__clean(text[:index_start]).split()[-context_len:]) + if len(left_context) > 0 and text[:index_start][-1].isspace(): + left_context = left_context + " " + right_context = " ".join(__clean(text[index_end:]).split()[:context_len]) + if len(right_context) > 0 and text[index_end][0].isspace(): + right_context = " " + right_context + diffs.append(left_context + center + right_context) + index_end += 1 + index_start = index_end + 1 + else: + break + if len(diffs) == 0: + diffs = [text] + return diffs diff --git a/nemo_text_processing/hybrid/utils.py b/nemo_text_processing/hybrid/utils.py new file mode 100644 index 000000000000..84c892679354 --- /dev/null +++ b/nemo_text_processing/hybrid/utils.py @@ -0,0 +1,696 @@ +# Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import copy +import difflib +import json +import re +import string +from typing import List, Optional, Tuple, Union + +import pandas as pd +import pynini +from nemo_text_processing.inverse_text_normalization.en.taggers.cardinal import CardinalFst +from nemo_text_processing.inverse_text_normalization.inverse_normalize import InverseNormalizer +from pynini.lib.rewrite import top_rewrite +from tqdm import tqdm + +from nemo.utils import logging + +DELIMITER = '~~' + +cardinal_graph = CardinalFst().graph_no_exception +cardinal_graph = ( + pynini.closure(pynini.union("In ", "in ")) + cardinal_graph + pynini.closure(pynini.accep(" ") + cardinal_graph) +) + +inverse_normalizer = InverseNormalizer() + + +def load_data(input_fs: List[str]): + """ + loads data from list of abs file paths + Returns: + inputs: List[str] list of abs file paths + targets: List[List[str]] list of targets, can contain multiple options for each target + sentences: List[List[str]] list of sentence options + labels: List[List[int]] list of labels (1,0) + """ + inputs = [] + sentences = [] + cur_sentences = [] + labels = [] + cur_labels = [] + for input_f in input_fs: + if input_f.endswith(".json"): + with open(input_f, "r") as f: + for line in f: + line = json.loads(line) + try: + inputs.append(line['text'].strip()) + sentences.append([line['gt_normalized'].strip()]) + labels.append([1]) + except Exception as e: + print(e) + raise ValueError(f"Check format for line {line}") + else: + with open(input_f, "r") as f: + for line in f: + if line != "\n": + try: + sent, label = line.strip().split(DELIMITER) + except Exception as e: + if line.startswith("#"): + continue + print(e) + raise ValueError(f"Check format for line {line}") + if label == "RAW": + inputs.append(sent) + elif label == "1": + cur_sentences.append(sent) + cur_labels.append(1) + elif label == "0": + cur_sentences.append(sent) + cur_labels.append(0) + else: + sentences.append(cur_sentences) + cur_sentences = [] + labels.append(cur_labels) + cur_labels = [] + + if len(cur_sentences) > 0: + sentences.append(cur_sentences) + labels.append(cur_labels) + assert len(inputs) == len(sentences) + targets = [[x for i, x in enumerate(sents) if ls[i]] for (sents, ls) in zip(sentences, labels)] + return inputs, targets, sentences, labels + + +def remove_whitelist_boudaries(x): + # remove raw whitelist + x = re.sub(r"\|raw_start\|[^|]+\|raw_end\|", "", x) + # remove norm text boundaries + x = x.replace("|norm_start|", "").replace("|norm_end|", "") + return x + + +def _clean_pre_norm_libritts(inputs: List[str], targets: List[List[str]]): + """ + standardizes format of inputs and targets before being normalized, so more rules apply. + This is specific for libritts. + """ + for i in range(len(targets)): + for j in range(len(targets[i])): + targets[i][j] = clean_libri_tts(targets[i][j]) + + for i in range(len(inputs)): + for target in targets[i]: + diffs = get_diff(a=inputs[i].lower(), b=target.lower()) + for diff in diffs[::-1]: + in_diff = inputs[i][diff[0][0] : diff[0][1]].lower() + tg_diff = target[diff[1][0] : diff[1][1]].lower() + replacement = inputs[i][: diff[0][0]] + tg_diff + inputs[i][diff[0][1] :] + if (in_diff == "s" and tg_diff == "z") or (in_diff == "z" and tg_diff == "s"): + inputs[i] = replacement + elif (in_diff == "re" and tg_diff == "er") or (in_diff == "er" and tg_diff == "re"): + inputs[i] = replacement + elif (in_diff == "me" and tg_diff == "") or (in_diff == "" and tg_diff == "me"): + inputs[i] = replacement + elif (in_diff == "ue" and tg_diff == "") or (in_diff == "" and tg_diff == "ue"): + inputs[i] = replacement + return inputs, targets + + +def _clean_pre_norm_google(inputs: List[str], targets: List[List[str]]): + """ + standardizes format of inputs and targets before being normalized, so more rules apply. + This is specific for google dataset. + """ + for i in range(len(inputs)): + + inputs[i] = re.sub(r"\$\s([0-9]{1,})", r"$\1", inputs[i]) + inputs[i] = re.sub(r"\bmr ", r"Mr. ", inputs[i]) + inputs[i] = re.sub(r"\bdr ", r"Dr. ", inputs[i]) + inputs[i] = re.sub(r"\bdr$", r"Dr.", inputs[i]) + inputs[i] = re.sub(r"\bmrs ", r"Mrs. ", inputs[i]) + inputs[i] = re.sub(r"\bjr ", r"Jr. ", inputs[i]) + inputs[i] = re.sub(r"\bjr$", r"Jr.", inputs[i]) + inputs[i] = re.sub(r"\dsr ", r"Sr. ", inputs[i]) + inputs[i] = re.sub(r"\dsr$", r"Sr.", inputs[i]) + for target in targets[i]: + diffs = get_diff(a=inputs[i].lower(), b=target.lower()) + for diff in diffs[::-1]: + in_diff = inputs[i][diff[0][0] : diff[0][1]].lower() + tg_diff = target[diff[1][0] : diff[1][1]].lower() + replacement = inputs[i][: diff[0][0]] + tg_diff + inputs[i][diff[0][1] :] + if (in_diff == "s" and tg_diff == "z") or (in_diff == "z" and tg_diff == "s"): + inputs[i] = replacement + elif (in_diff == "re" and tg_diff == "er") or (in_diff == "er" and tg_diff == "re"): + inputs[i] = replacement + elif (in_diff == "me" and tg_diff == "") or (in_diff == "" and tg_diff == "me"): + inputs[i] = replacement + elif (in_diff == "" and tg_diff == "u") or (in_diff == "u" and tg_diff == ""): + inputs[i] = replacement + elif (in_diff == "ue" and tg_diff == "") or (in_diff == "" and tg_diff == "ue"): + inputs[i] = replacement + elif re.sub(r"\.", "", in_diff) == re.sub(r"( |\.)", "", tg_diff): + inputs[i] = replacement + + return inputs, targets + + +def clean_pre_norm(inputs: List[str], targets: List[List[str]], dataset: Optional[str] = None): + """ + standardizes format of inputs and targets before being normalized, so more rules apply. + """ + # deep copy + pre_inputs = copy.deepcopy(inputs) + pre_targets = copy.deepcopy(targets) + + # --- data specific pre cleaning --- + if dataset == "libritts": + pre_inputs, pre_targets = _clean_pre_norm_libritts(inputs=pre_inputs, targets=pre_targets) + elif dataset == "google": + pre_inputs, pre_targets = _clean_pre_norm_google(inputs=pre_inputs, targets=pre_targets) + else: + pass + + # --- general pre cleaning --- + for i in range(len(pre_inputs)): + pre_inputs[i] = re.sub("librivox.org", "librivox dot org", pre_inputs[i]) + pre_inputs[i] = re.sub( + rf"([0-9]?[0-9](\.|:)[0-9][0-9]\s?)(a|A|p|P)(\.?)\s(M|m)(\.?)", rf"\1\3\4\5\6", pre_inputs[i] + ) + # pre_inputs[i] =re.sub(rf"\b(S|s)t\.", rf"saint", pre_inputs[i]) + return pre_inputs, pre_targets + + +def _clean_post_norm_libritts(inputs: List[str], targets: List[List[str]], norm_texts): + return targets, norm_texts + + +def _clean_post_norm_google(inputs: List[str], targets: List[List[str]], norm_texts): + """ + standardizes format of inputs and targets, and predicted normalizations for easier evaluation. + This is specific for google dataset. + """ + for i in range(len(targets)): + for target in targets[i]: + for j, norm in enumerate(norm_texts[i][0]): + diffs = get_diff(a=norm.lower(), b=target.lower()) + for diff in diffs[::-1]: + norm_diff = norm[diff[0][0] : diff[0][1]].lower() + tg_diff = target[diff[1][0] : diff[1][1]].lower() + replacement = norm[: diff[0][0]] + tg_diff + norm[diff[0][1] :] + if norm_diff == re.sub(r" ", "", tg_diff): + norm_texts[i][0][j] = replacement + + return targets, norm_texts + + +def _clean_post_general(str) -> str: + """ + standardizes format of inputs and targets, and predicted normalizations for easier evaluation. + """ + str = re.sub(rf" oh ", " zero ", str) + str = re.sub(rf" oh$", " zero", str) + str = re.sub(rf"^oh ", "zero ", str) + # str = re.sub(rf" o ", " zero ", str) + str = re.sub(rf"\sO\b", "zero", str) + str = re.sub(rf" o$", " zero", str) + str = re.sub(rf"^o ", "zero ", str) + str = re.sub(rf"'o ", "'zero ", str) + str = str.replace("mountain", "mount") + return str + + +def _clean_targets(str) -> str: + """Clean ground truth options.""" + str = re.sub(rf" o ", " zero ", str) + return str + + +def adjust_pred(pred: str, gt: str, dataset: str, delim_present=True): + """Standardize prediction format to make evaluation easier""" + orig_pred = pred + orig_gt = gt + if delim_present and not re.search(rf"< (.*?) >", pred): + return pred + pred = re.sub(rf"< ", "", pred) + pred = re.sub(rf" >", "", pred) + pred = pred.lower().strip() + gt = gt.lower().strip() + can_be_adjusted = False + + if dataset in ["google", "libritts"] and pred != gt: + if is_date(pred=pred, gt=gt, cardinal_graph=cardinal_graph): + pred = gt + elif contains_month(pred, gt): + pred = re.sub(r",", "", pred) + gt = re.sub(r",", "", gt) + pred = re.sub(r" zero ", " o ", pred) + gt = re.sub(r" zero ", " o ", gt) + gt = re.sub(rf" +", " ", gt) + pred = re.sub(rf" +", " ", pred) + + if pred != gt: + gt_itn = inverse_normalizer.normalize(gt, verbose=False) + pred_itn = inverse_normalizer.normalize(pred, verbose=False) + if len(gt_itn) == len(pred_itn) and set(gt_itn) == set(pred_itn): + can_be_adjusted = True + pred = gt + elif " of " in gt: + gt = re.sub(r"(^the | of)", "", gt) + idx = gt.index(" ") + idx2 = (gt[idx + 1 :].index(" ") if " " in gt[idx + 1 :] else len(gt[idx + 1 :])) + idx + 1 + gt = gt[idx + 1 : idx2] + " " + gt[:idx] + gt[idx2:] + if dataset == "libritts" and pred != gt: + if "dollar" in gt: + gt = re.sub(rf"\band\b", "", gt) + pred = re.sub(rf"\band\b", "", pred) + if re.search(r"\bus dollar", pred) and not re.search(r"\bus dollar", gt): + pred = re.sub(rf"\bus dollar", "dollar", pred) + else: + gt = re.sub(rf"(\bthe\b|\.)", "", gt) + pred = re.sub(rf"\bone\b", "a", pred) + gt = re.sub(rf"\bmr\b", "mister", gt) + gt = re.sub(rf"\bmrs\b", "misses", gt) + gt = re.sub(rf"\bdr\b", "doctor", gt) + gt = re.sub(rf"\bco\b", "company", gt) + if gt != pd and dataset in ["google", "libritts"]: + if gt.replace("/", "").replace(" ", " ") == pred.replace("slash", "").replace(" ", " "): + pred = gt + elif gt in ["s", "z"] and pred in ["s", "z"]: + pred = gt + elif gt == "hash tag" and pred == "hash": + pred = "hash tag" + elif gt[:-2] == pred[:-2] and gt[-2:] in ["er", "re"] and pred[-2:] in ["er", "re"]: + pred = gt + # elif gt.replace("-", " ").replace(" ", " ") == pred.replace("minus", "").replace(" ", " "): + # pred = gt + elif gt.replace("to", "").replace("-", "") == pred.replace("to", "").replace("-", ""): + pred = gt + + gt = re.sub(rf" +", " ", gt) + pred = re.sub(rf"(\.)", "", pred) + pred = re.sub(rf" +", " ", pred) + if gt == pred: + can_be_adjusted = True + if can_be_adjusted: + if delim_present: + res = f" < {orig_gt} > " + else: + res = orig_gt + return res + else: + return orig_pred + + +def clean_post_norm( + inputs: List[str], + targets: List[List[str]], + norm_texts, + dataset: Optional[str] = None, + delim_present: Optional[bool] = True, +): + """ + Args: + inputs (List[str]): inputs + targets (List[List[str]]): targets + norm_texts (List[(List[str], List[float])]): List of normalization options, weights + dataset (Optional[str], optional): _description_. Defaults to None. + delim_present (Optional[str], optional): The flag indicates whether normalization output contain delimiters "<>". + Set to False for NN baseline. + """ + # deep copy + post_norm_texts = copy.deepcopy(norm_texts) + post_targets = copy.deepcopy(targets) + + # --- data specific pre cleaning --- + if dataset == "libritts": + post_targets, post_norm_texts = _clean_post_norm_libritts( + inputs=inputs, targets=post_targets, norm_texts=post_norm_texts + ) + elif dataset == "google": + post_targets, post_norm_texts = _clean_post_norm_google( + inputs=inputs, targets=post_targets, norm_texts=post_norm_texts + ) + + else: + pass + + # --- general pre cleaning --- + + for i in range(len(targets)): + for j, x in enumerate(post_targets[i]): + post_targets[i][j] = _clean_post_general(x) + for j, x in enumerate(post_norm_texts[i][0]): + if x.count("< ") != x.count(" >"): + x = x.replace("<", "< ").replace(">", " >").replace(" ", " ") + post_norm_texts[i][0][j] = _clean_post_general(x) + if dataset in ["libritts", "google"]: + for i, _targets in enumerate(post_targets): + for jj, option in enumerate(post_norm_texts[i][0]): + for _, _target in enumerate(_targets): + + if not delim_present: + # nn doesn't have punctuation marks that leads for diff_pred_gt mismatch + _target = remove_punctuation(_target, remove_spaces=False, do_lower=True) + option = remove_punctuation(option, remove_spaces=False, do_lower=True) + + diffs = diff_pred_gt(pred=option, gt=_target) + for diff in diffs[::-1]: + if diff[0][1] - diff[0][0] == 0 and diff[1][1] - diff[1][0] == 0: + continue + pred = option[diff[0][0] : diff[0][1]] + gt = _target[diff[1][0] : diff[1][1]] + logging.debug(f"pred: |{pred}|\tgt: |{gt}|") + new_pred = adjust_pred(pred=pred, gt=gt, dataset=dataset, delim_present=delim_present) + new_pred = ( + post_norm_texts[i][0][jj][: diff[0][0]] + + new_pred + + post_norm_texts[i][0][jj][diff[0][1] :] + ) + logging.debug(f"|{post_norm_texts[i][0][jj]}| -> |{new_pred}|") + post_norm_texts[i][0][jj] = new_pred + return post_targets, post_norm_texts + + +def clean_libri_tts(target: str): + """ + Replace abbreviations in LibriTTS dataset + """ + + # Normalized text in LibriTTS by Google which contains abbreviations from `libri_sometimes_converts_abbrs` sometimes wasn't converted. + libri_sometimes_converts_abbrs = {"St.": "saint", "Rev.": "reverend"} + + # Normalized text in LibriTTS by Google which contains abbreviations from `libri_wo_changes_abbrs` wasn't converted. + libri_wo_changes_abbrs = {"vs.": "versus"} + + google_abbr2expand = { + "mr": "mister", + "Mr": "Mister", + "mrs": "misses", + "Mrs": "Misses", + "dr": "doctor", + "Dr": "Doctor", + "drs": "doctors", + "Drs": "Doctors", + "lt": "lieutenant", + "Lt": "Lieutenant", + "sgt": "sergeant", + "Sgt": "Sergeant", + "st": "saint", + "St": "Saint", + "jr": "junior", + "Jr": "Junior", + "maj": "major", + "Maj": "Major", + "hon": "honorable", + "Hon": "Honorable", + "gov": "governor", + "Gov": "Governor", + "capt": "captain", + "Capt": "Captain", + "esq": "esquire", + "Esq": "Esquire", + "gen": "general", + "Gen": "General", + "ltd": "limited", + "Ltd": "Limited", + "rev": "reverend", + "Rev": "Reverend", + "col": "colonel", + "Col": "Colonel", + "and co": "and Company", + "and Co": "and Company", + "mt": "mount", + "Mt": "Mount", + "ft": "fort", + "Ft": "Fort", + "tenn": "tennessee", + "Tenn": "Tennessee", + "vs": "versus", + "Vs": "Versus", + "&": "and", + "§": "section", + "#": "hash", + "=": "equals", + } + + # let's normalize `libri_only_remove_dot_abbrs` abbreviations, because google doesn't do it well + for abbr in google_abbr2expand.keys(): + if abbr in target: + # replace abbr in google text via regex and using \b to match only whole words, keep original 1 and 2 groups + target = re.sub(rf'(^|\s|\W){abbr}($|\s)', rf"\1{google_abbr2expand[abbr]}\2", target) + + # let's normalize `libri_sometimes_converts_abbrs` abbreviations manually, google sometimes forgets to expand them + for abbr, t in libri_sometimes_converts_abbrs.items(): + target = target.replace(abbr, t) + + # let's normalize `libri_wo_changes_abbrs` abbreviations manually, google doesn't change, but they should be + for abbr, t in libri_wo_changes_abbrs.items(): + target = target.replace(abbr, t) + + return target + + +def remove_punctuation(text: str, remove_spaces=True, do_lower=True, lang="en", exclude=None): + """Removes punctuation (and optionally spaces) in text for better evaluation""" + all_punct_marks = string.punctuation + + if exclude is not None: + for p in exclude: + all_punct_marks = all_punct_marks.replace(p, "") + text = re.sub("[" + all_punct_marks + "]", " ", text) + + if lang == "en": + # remove things like \x94 and \x93 + text = re.sub(r"[^\x00-\x7f]", r" ", text) + + text = re.sub(r" +", " ", text) + if remove_spaces: + text = text.replace(" ", "").replace("\u00A0", "").strip() + + if do_lower: + text = text.lower() + return text.strip() + + +def get_alternative_label(pred: str, targets: List[str]) -> bool: + """Returns true if prediction matches target options""" + + def _relax_diff(text): + text = text.replace("us dollars", "dollars") + text = text.replace("etcetera", "").replace("etc", "") + text = text.replace("one half ounce", "").replace("half an ounce", "") + text = text.replace("television", "").replace("t v ", " ").replace("tv", "") + text = text.replace("hundred", "") + text = text.replace("forty two", "").replace("four two", "") + text = text.replace("re", "").replace("er", "") + text = text.replace("ou", "").replace("o", "") + text = text.replace(" ", " ").strip() + return text + + acceptable = False + pred = remove_punctuation(pred, remove_spaces=False, do_lower=True) + for target in targets: + target = _clean_post_general(remove_punctuation(target, remove_spaces=False, do_lower=True)) + target = _clean_targets(remove_punctuation(target, remove_spaces=False, do_lower=True)) + if _relax_diff(target) == _relax_diff(pred): + acceptable = True + break + return acceptable + + +def get_labels(targets: List[str], norm_texts_weights: List[Tuple[str, str]], lang="en",) -> List[List[str]]: + """ + Assign labels to generated normalization options (1 - for ground truth, 0 - other options) + Args: + targets: ground truth normalization sentences + norm_texts_weights: List of tuples: (normalization options, weights of normalization options) + returns: + List of labels [1, 0] for every normalization option + """ + print("Assign labels to generated normalization options...") + labels = [] + for i, cur_targets in tqdm(enumerate(targets)): + curr_labels = [] + cur_targets = [_clean_targets(t) for t in cur_targets] + for norm_option in norm_texts_weights[i][0]: + norm_option = _clean_targets(norm_option) + norm_option = remove_whitelist_boudaries(norm_option) + + if is_correct(pred=norm_option, targets=cur_targets, lang=lang): + curr_labels.append(1) + elif get_alternative_label(pred=norm_option, targets=cur_targets): + curr_labels.append(1) + else: + curr_labels.append(0) + labels.append(curr_labels) + return labels + + +def contains_month(pred, gt): + """Check is the pred/gt contain month in the span""" + months = [ + "january", + "february", + "march", + "april", + "may", + "june", + "july", + "august", + "september", + "october", + "november", + "december", + ] + + for mon in months: + if mon in gt and mon in pred: + return True + return False + + +def is_date(pred, gt, cardinal_graph): + """Returns True is pred and gt are date format modifications and are equal.""" + is_date_case = False + + # for cases "1890" -> "one thousand eight hundred ninety" vs "eighteen ninety" + if "thousand" in pred and "hundred" in pred and pred.strip().split()[-2:] == gt.strip().split()[-2:]: + is_date_case = True + elif "thousand" in gt and "hundred" in gt and gt.strip().split()[-2:] == pred.strip().split()[-2:]: + is_date_case = True + else: + try: + if top_rewrite(gt.replace(" oh ", " zero ").replace(" o ", " zero "), cardinal_graph).replace( + " ", "" + ) == top_rewrite(pred.replace(" oh ", " zero ").replace(" o ", " zero "), cardinal_graph).replace(" ", ""): + is_date_case = True + except: + pass + + return is_date_case + + +def is_correct(pred: str, targets: Union[List[str], str], lang: str) -> bool: + """ + returns True if prediction matches targets for language lang. + """ + if isinstance(targets, List): + targets = [remove_punctuation(x, remove_spaces=True, do_lower=True, lang=lang) for x in targets] + else: + targets = [remove_punctuation(targets, remove_spaces=True, do_lower=True, lang=lang)] + + pred = remove_punctuation(pred, remove_spaces=True, do_lower=True) + return pred in targets + + +def print_df(df): + """ + prints data frame + """ + with pd.option_context( + "display.max_rows", None, "display.max_columns", None, "display.width", 1000, "display.max_colwidth", 400, + ): + print(df) + + +def get_diff(a: str, b: str): + """returns list of different substrings between and b + + Returns: + list of Tuple(pred start and end, gt start and end) subsections + """ + s = difflib.SequenceMatcher(None, a, b, autojunk=False) + + # s contains a list of triples. Each triple is of the form (i, j, n), and means that a[i:i+n] == b[j:j+n]. + # The triples are monotonically increasing in i and in j. + s = s.get_matching_blocks() + s = [x for x in s if x[2] != 1] + # get not matching blocks + matches = [[0, 0, 0]] + s + unmatches_l = [] + unmatches_r = [] + for l, r in zip(matches[:-1], matches[1:]): + unmatches_l.append([l[0] + l[2], r[0]]) + unmatches_r.append([l[1] + l[2], r[1]]) + + result = list(zip(unmatches_l, unmatches_r)) + + for item in list(zip(unmatches_l, unmatches_r)): + logging.debug(f"a: {a[item[0][0]:item[0][1]]}") + logging.debug(f"b: {b[item[1][0]:item[1][1]]}") + logging.debug("=" * 20) + return result[1:] + + +def diff_pred_gt(pred: str, gt: str): + """returns list of different substrings between prediction and gt + relies on that prediction uses '< ' ' >' + + Args: + pred (str): prediction + gt (str): ground truth + + Returns: + list of Tuple(pred start and end, gt start and end) subsections + + e.g. pred="< Edward third >., king Our own . loss had been < two thousand two hundred >" + gt ="Edward III., king Our own loss had been twenty two hundred" + --> [([0, 16], [0, 10]), ([32, 34], [26, 26]), ([48, 76], [40, 58])] + """ + s = difflib.SequenceMatcher(None, pred, gt, autojunk=False) + + # s contains a list of triples. Each triple is of the form (i, j, n), and means that a[i:i+n] == b[j:j+n]. + # The triples are monotonically increasing in i and in j. + s = s.get_matching_blocks() + + left = list(re.finditer("< ", pred)) + left = [x.start() for x in left] + right = list(re.finditer(" >", pred)) + right = [x.end() for x in right] + left = [-1] + left + [len(pred)] + right = [0] + right + [len(pred)] + + matches = [] + assert len(left) == len(right) + idx = 1 + for i, seq in enumerate(s): + if i == len(s) - 1 and seq[2] == 0: + break + while idx < len(left) - 1 and (seq[0] >= right[idx]): + idx += 1 + + if right[idx - 1] <= seq[0] < left[idx] and (seq[0] + seq[2]) <= left[idx]: + matches.append(seq) + + # get not matching blocks + matches = [[0, 0, 0]] + matches + [[len(pred), len(gt), 0]] + unmatches_l = [] + unmatches_r = [] + for l, r in zip(matches[:-1], matches[1:]): + unmatches_l.append([l[0] + l[2], r[0]]) + unmatches_r.append([l[1] + l[2], r[1]]) + + result = list(zip(unmatches_l, unmatches_r)) + + for item in list(zip(unmatches_l, unmatches_r)): + logging.debug(f"pred: {pred[item[0][0]:item[0][1]]}") + logging.debug(f"gt : {gt[item[1][0]:item[1][1]]}") + logging.debug("=" * 20) + return result diff --git a/nemo_text_processing/hybrid/wfst_lm_rescoring.py b/nemo_text_processing/hybrid/wfst_lm_rescoring.py new file mode 100644 index 000000000000..72b3b59bb62a --- /dev/null +++ b/nemo_text_processing/hybrid/wfst_lm_rescoring.py @@ -0,0 +1,335 @@ +# Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import argparse +import os +import pickle +import re +import shutil +from typing import Dict, List + +import model_utils +import pandas as pd +import utils +from joblib import Parallel, delayed +from nemo_text_processing.text_normalization.normalize_with_audio import NormalizerWithAudio +from tqdm import tqdm + +from nemo.utils import logging + +parser = argparse.ArgumentParser(description="Re-scoring") +parser.add_argument("--lang", default="en", type=str, choices=["en"]) +parser.add_argument("--n_tagged", default=100, type=int, help="Number WFST options") +parser.add_argument("--context_len", default=-1, type=int, help="Context length, -1 to use full context") +parser.add_argument("--threshold", default=0.2, type=float, help="delta threshold value") +parser.add_argument("--overwrite_cache", action="store_true", help="overwrite cache") +parser.add_argument("--model_name", type=str, default="bert-base-uncased") +parser.add_argument("--cache_dir", default='cache', type=str, help="use cache dir") +parser.add_argument( + "--data", + default="text_normalization_dataset_files/EngConf.txt", + help="For En only. Path to a file for evaluation.", +) +parser.add_argument("--n_jobs", default=-2, type=int, help="The maximum number of concurrently running jobs") +parser.add_argument( + "--models", default="mlm_bert-base-uncased", type=str, help="Comma separated string of model names" +) +parser.add_argument( + "--regenerate_pkl", + action="store_true", + help="Set to True to re-create pickle file with WFST normalization options", +) +parser.add_argument("--batch_size", default=200, type=int, help="Batch size for parallel processing") + + +def rank(sentences: List[str], labels: List[int], models: Dict[str, 'Model'], context_len=None, do_lower=True): + """ + computes scores for each sentences using all provided models and returns summary in data frame + """ + df = pd.DataFrame({"sent": sentences, "labels": labels}) + for model_name, model in models.items(): + scores = model_utils.score_options( + sentences=sentences, context_len=context_len, model=model, do_lower=do_lower + ) + df[model_name] = scores + return df + + +def threshold_weights(norm_texts_weights, delta: float = 0.2): + """ + norm_texts_weights: list of [ List[normalized options of input], list[weights] ] + delta: delta to add to minimum weight in options to compose upper limit for threshhold + + returns: + filter list of same format as input + """ + # threshold value is factor applied to lowest/first weight of all normalization options for every input + res = [] + for i, options_weights in enumerate(norm_texts_weights): + thresh = options_weights[1][0] + delta # minimum weight plus delta + item = [x for x in zip(*options_weights)] + # filters out all options for every input that is larger than threshold + res.append(list(filter(lambda x: x[1] < thresh, item))) + + return [list(map(list, zip(*item))) for item in res] + + +def _get_unchanged_count(text): + """ + returns number of unchanged words in text + """ + exclude = '#$%&<>' + + # remove normalized whitelist + text = re.sub(r"\|norm_start\|[^|]+\|norm_end\|", "", text) + # remove raw text boundaries + text = text.replace("|raw_start|", "").replace("|raw_end|", "") + + start_pattern = "<" + end_pattern = ">" + + text = utils.remove_punctuation(text, remove_spaces=False, do_lower=False, exclude=exclude) + text_clean = "" + for ch in text: + if ch.isalpha() or ch.isspace() or ch in [start_pattern, end_pattern]: + text_clean += ch + else: + text_clean += " " + ch + " " + + text = text_clean + unchanged_count = 0 + skip = False + + for word in text.split(): + if start_pattern == word: + skip = True + elif end_pattern == word: + skip = False + elif not skip: + unchanged_count += 1 + return unchanged_count + + +def _get_replacement_count(text): + """ + returns number of token replacements + """ + start_pattern = "<" + end_pattern = ">" + return min(text.count(start_pattern), text.count(end_pattern)) + + +def threshold(norm_texts_weights, unchanged=True, replacement=True): + """ + Reduces the number of WFST options based for LM rescoring. + + Args: + :param norm_texts_weights: WFST options with associated weight + :param unchanged: set to True to filter out examples based on number of words left unchanged + (punct is not taken into account) + :param replacement: set to True to filter out examples based on number of replacements made + (Given A and B are WFST options, if the number of unchanged for A and B are the same, + the option with a smaller number of replacements is preferable (i.e., larger span)). + + :return: WFST options with associated weight (reduced) + """ + + def __apply(norm_texts_weights, f, use_min=True): + inputs_filtered = [] + for example in norm_texts_weights: + texts = example[0] + counts = [f(t) for t in texts] + [logging.debug(f"{c} -- {t}") for t, c in zip(texts, counts)] + target_count = min(counts) if use_min else max(counts) + filtered_texts = [] + filtered_weights = [] + for i, c in enumerate(counts): + if c == target_count: + filtered_texts.append(example[0][i]) + filtered_weights.append(example[1][i]) + inputs_filtered.append([filtered_texts, filtered_weights]) + return inputs_filtered + + logging.debug("BASIC THRESHOLDING INPUT:") + [logging.debug(x) for x in norm_texts_weights[0][0]] + if unchanged: + norm_texts_weights = __apply(norm_texts_weights, _get_unchanged_count) + logging.debug("AFTER UNCHANGED FILTER:") + [logging.debug(x) for x in norm_texts_weights[0][0]] + + if replacement: + norm_texts_weights = __apply(norm_texts_weights, _get_replacement_count) + logging.debug("AFTER REPLACEMENT FILTER:") + [logging.debug(x) for x in norm_texts_weights[0][0]] + + return norm_texts_weights + + +def main(): + args = parser.parse_args() + + logging.setLevel(logging.INFO) + lang = args.lang + input_f = args.data + + if args.data == "text_normalization_dataset_files/LibriTTS.json": + args.dataset = "libritts" + elif args.data == "text_normalization_dataset_files/GoogleTN.json": + args.dataset = "google" + else: + args.dataset = None + if not os.path.exists(args.data): + raise FileNotFoundError(f"{args.data} file not found") + + print("Create Masked Language Model...") + models = model_utils.init_models(model_name_list=args.model_name) + input_fs = input_f.split(",") + print("LOAD DATA...") + inputs, targets, _, _ = utils.load_data(input_fs) + pre_inputs, pre_targets = utils.clean_pre_norm(dataset=args.dataset, inputs=inputs, targets=targets) + + print("INIT WFST...") + normalizer = NormalizerWithAudio( + input_case="cased", lang=lang, cache_dir=args.cache_dir, lm=True, overwrite_cache=args.overwrite_cache + ) + + print("APPLYING NORMALIZATION RULES...") + p_file = ( + f"norm_texts_weights_{args.n_tagged}_{os.path.basename(args.data)}_{args.context_len}_{args.threshold}.pkl" + ) + + if not os.path.exists(p_file) or args.regenerate_pkl: + print(f"Creating WFST and saving to {p_file}") + + def __process_batch(batch_idx, batch, dir_name): + normalized = [] + for x in tqdm(batch): + ns, ws = normalizer.normalize(x, n_tagged=args.n_tagged, punct_post_process=False) + ns = [re.sub(r"<(.+?)>", r"< \1 >", x) for x in ns] + normalized.append((ns, ws)) + with open(f"{dir_name}/{batch_idx}.p", "wb") as handle: + pickle.dump(normalized, handle, protocol=pickle.HIGHEST_PROTOCOL) + + print(f"Batch -- {batch_idx} -- is complete") + return batch_idx + + # to save intermediate results to a file + batch = min(len(pre_inputs), args.batch_size) + + tmp_dir = f"/tmp/{os.path.basename(args.data)}" + if os.path.exists(tmp_dir): + shutil.rmtree(tmp_dir) + os.makedirs(tmp_dir, exist_ok=True) + + batch_ids = Parallel(n_jobs=args.n_jobs)( + delayed(__process_batch)(idx, pre_inputs[i : i + batch], tmp_dir) + for idx, i in enumerate(range(0, len(pre_inputs), batch)) + ) + + # aggregate all intermediate results + norm_texts_weights = [] + for batch_id in batch_ids: + batch_f = f"{tmp_dir}/{batch_id}.p" + norm_texts_weights.extend(pickle.load(open(batch_f, "rb"))) + + with open(p_file, "wb") as handle: + pickle.dump(norm_texts_weights, handle, protocol=pickle.HIGHEST_PROTOCOL) + else: + print(f"Loading WFST from {p_file}") + norm_texts_weights = pickle.load(open(p_file, "rb")) + + print("THRESHOLDING...") + # apply weights threshold to reduce number of options + + if args.threshold > 0: + norm_texts_weights = threshold_weights(norm_texts_weights, delta=args.threshold) + logging.debug("AFTER WEIGHTS THRESHOLDING:") + [logging.debug(x) for x in norm_texts_weights[0][0]] + + # reduce number of options by selecting options with the smallest number of unchanged words + norm_texts_weights = threshold(norm_texts_weights) + + print("POST PROCESSING...") + post_targets, post_norm_texts_weights = utils.clean_post_norm( + dataset=args.dataset, inputs=pre_inputs, targets=pre_targets, norm_texts=norm_texts_weights + ) + + print("GETTING LABELS...") + labels = utils.get_labels(targets=post_targets, norm_texts_weights=post_norm_texts_weights) + + examples_with_no_labels_among_wfst = [i for i, x in enumerate(labels) if 1 not in x] + + print("GATHERING STATS...") + model_stats = {m: 0 for m in models} + gt_in_options = 0 + for i, example in tqdm(enumerate(zip(post_norm_texts_weights, labels))): + data, curr_labels = example + assert len(data[0]) == len(curr_labels) + df = rank( + sentences=data[0], + labels=curr_labels, + models=models, + context_len=args.context_len if args.context_len is not None and args.context_len >= 0 else None, + do_lower=True, + ) + df['sent'] = df['sent'].apply(lambda x: utils.remove_whitelist_boudaries(x)) + df["weights"] = data[1] + + do_print = False + + for model in models: + # one hot vector for predictions, 1 for the best score option + df[f"{model}_pred"] = (df[model] == min(df[model])).astype(int) + # add constrain when multiple correct labels per example + pred_is_correct = min(sum((df["labels"] == df[f"{model}_pred"]) & df["labels"] == 1), 1) + + if not pred_is_correct or logging.getEffectiveLevel() <= logging.DEBUG: + do_print = True + + if do_print: + print(f"{model} prediction is correct: {pred_is_correct == 1}") + model_stats[model] += pred_is_correct + gt_in_options += 1 in curr_labels + + if do_print: + print(f"INPUT: {pre_inputs[i]}") + print(f"GT : {post_targets[i]}\n") + utils.print_df(df) + print("-" * 80 + "\n") + + if gt_in_options != len(post_norm_texts_weights): + print("WFST options for some examples don't contain the ground truth:") + for i in examples_with_no_labels_among_wfst: + print(f"INPUT: {pre_inputs[i]}") + print(f"GT : {post_targets[i]}\n") + print(f"WFST:") + for x in post_norm_texts_weights[i]: + print(x) + print("=" * 40) + + all_correct = True + for model, correct in model_stats.items(): + print( + f"{model} -- correct: {correct}/{len(post_norm_texts_weights)} or ({round(correct/len(post_norm_texts_weights) * 100, 2)}%)" + ) + all_correct = all_correct and (correct == len(post_norm_texts_weights)) + + print(f"examples_with_no_labels_among_wfst: {len(examples_with_no_labels_among_wfst)}") + return all_correct + + +if __name__ == "__main__": + all_correct = main() + print(f"all_correct: {all_correct}") diff --git a/nemo_text_processing/text_normalization/en/data/measure/unit.tsv b/nemo_text_processing/text_normalization/en/data/measure/unit.tsv index 96afbb71d27f..c033ab842385 100644 --- a/nemo_text_processing/text_normalization/en/data/measure/unit.tsv +++ b/nemo_text_processing/text_normalization/en/data/measure/unit.tsv @@ -34,6 +34,8 @@ kg kilogram khz kilohertz km2 square kilometer km² square kilometer +km3 cubic kilometer +km³ cubic kilometer km kilometer kpa kilopascal kwh kilowatt hour @@ -50,6 +52,9 @@ mg milligram mhz megahertz mi2 square mile mi² square mile +mi3 cubic mile +mi³ cubic mile +cu mi cubic mile mi mile min minute ml milliliter diff --git a/nemo_text_processing/text_normalization/en/taggers/cardinal.py b/nemo_text_processing/text_normalization/en/taggers/cardinal.py index 290a1bf9f0f0..d4448ee450a3 100644 --- a/nemo_text_processing/text_normalization/en/taggers/cardinal.py +++ b/nemo_text_processing/text_normalization/en/taggers/cardinal.py @@ -119,19 +119,22 @@ def add_optional_and(self, graph): graph, NEMO_SIGMA + pynini.closure(pynini.cross("hundred ", " "), 0, 1) + NEMO_SIGMA ) - not_quote = pynini.closure(NEMO_NOT_QUOTE) - no_thousand_million = pynini.difference( - not_quote, not_quote + pynini.union("thousand", "million") + not_quote - ).optimize() - integer = ( - not_quote + pynutil.add_weight(pynini.cross("hundred ", "hundred and ") + no_thousand_million, -0.0001) - ).optimize() - - no_hundred = pynini.difference(NEMO_SIGMA, not_quote + pynini.accep("hundred") + not_quote).optimize() - integer |= ( - not_quote + pynutil.add_weight(pynini.cross("thousand ", "thousand and ") + no_hundred, -0.0001) - ).optimize() - - graph_with_and = pynini.compose(graph, integer).optimize() | pynutil.add_weight(graph, 0.00001) + graph_with_and = pynutil.add_weight(graph, 0.00001) + + if not self.lm: + not_quote = pynini.closure(NEMO_NOT_QUOTE) + no_thousand_million = pynini.difference( + not_quote, not_quote + pynini.union("thousand", "million") + not_quote + ).optimize() + integer = ( + not_quote + pynutil.add_weight(pynini.cross("hundred ", "hundred and ") + no_thousand_million, -0.0001) + ).optimize() + + no_hundred = pynini.difference(NEMO_SIGMA, not_quote + pynini.accep("hundred") + not_quote).optimize() + integer |= ( + not_quote + pynutil.add_weight(pynini.cross("thousand ", "thousand and ") + no_hundred, -0.0001) + ).optimize() + + graph_with_and |= pynini.compose(graph, integer).optimize() return graph_with_and diff --git a/nemo_text_processing/text_normalization/en/taggers/roman.py b/nemo_text_processing/text_normalization/en/taggers/roman.py index e8633707b71a..07886cfced06 100644 --- a/nemo_text_processing/text_normalization/en/taggers/roman.py +++ b/nemo_text_processing/text_normalization/en/taggers/roman.py @@ -61,7 +61,7 @@ def __init__(self, deterministic: bool = True, lm: bool = False): pynutil.insert("key_cardinal: \"") + key_words + pynutil.insert("\"") + pynini.accep(" ") + default_graph ).optimize() - if deterministic: + if deterministic or lm: # two digit roman numerals up to 49 roman_to_cardinal = pynini.compose( pynini.closure(NEMO_ALPHA, 2), diff --git a/nemo_text_processing/text_normalization/en/taggers/serial.py b/nemo_text_processing/text_normalization/en/taggers/serial.py index d7ac4ffe77f3..669fd95a0569 100644 --- a/nemo_text_processing/text_normalization/en/taggers/serial.py +++ b/nemo_text_processing/text_normalization/en/taggers/serial.py @@ -81,7 +81,7 @@ def __init__(self, cardinal: GraphFst, ordinal: GraphFst, deterministic: bool = # serial graph with delimiter delimiter = pynini.accep("-") | pynini.accep("/") | pynini.accep(" ") if not deterministic: - delimiter |= pynini.cross("-", " dash ") | pynini.cross("/", " slash") + delimiter |= pynini.cross("-", " dash ") | pynini.cross("/", " slash ") alphas = pynini.closure(NEMO_ALPHA, 1) letter_num = alphas + delimiter + num_graph @@ -124,6 +124,13 @@ def __init__(self, cardinal: GraphFst, ordinal: GraphFst, deterministic: bool = serial_graph |= pynini.compose(graph_with_space, serial_graph.optimize()).optimize() serial_graph = pynini.compose(pynini.closure(NEMO_NOT_SPACE, 2), serial_graph).optimize() + # this is not to verbolize "/" as "slash" in cases like "import/export" + serial_graph = pynini.compose( + pynini.difference( + NEMO_SIGMA, pynini.closure(NEMO_ALPHA, 1) + pynini.accep("/") + pynini.closure(NEMO_ALPHA, 1) + ), + serial_graph, + ) self.graph = serial_graph.optimize() graph = pynutil.insert("name: \"") + convert_space(self.graph).optimize() + pynutil.insert("\"") self.fst = graph.optimize() diff --git a/nemo_text_processing/text_normalization/en/taggers/whitelist.py b/nemo_text_processing/text_normalization/en/taggers/whitelist.py index d799c1768cb7..01d102da9a3c 100644 --- a/nemo_text_processing/text_normalization/en/taggers/whitelist.py +++ b/nemo_text_processing/text_normalization/en/taggers/whitelist.py @@ -16,6 +16,7 @@ from nemo_text_processing.text_normalization.en.graph_utils import ( NEMO_CHAR, NEMO_NOT_SPACE, + NEMO_SIGMA, NEMO_UPPER, SINGULAR_TO_PLURAL, GraphFst, @@ -64,7 +65,10 @@ def _get_whitelist_graph(input_case, file, keep_punct_add_end: bool = False): return graph graph = _get_whitelist_graph(input_case, get_abs_path("data/whitelist/tts.tsv")) - graph |= _get_whitelist_graph(input_case, get_abs_path("data/whitelist/symbol.tsv")) + graph |= pynini.compose( + pynini.difference(NEMO_SIGMA, pynini.accep("/")).optimize(), + _get_whitelist_graph(input_case, get_abs_path("data/whitelist/symbol.tsv")), + ).optimize() if deterministic: names = get_names() diff --git a/nemo_text_processing/text_normalization/normalize_with_audio.py b/nemo_text_processing/text_normalization/normalize_with_audio.py index 6c14d9c6d14d..89927b29a625 100644 --- a/nemo_text_processing/text_normalization/normalize_with_audio.py +++ b/nemo_text_processing/text_normalization/normalize_with_audio.py @@ -144,6 +144,7 @@ def normalize(self, text: str, n_tagged: int, punct_post_process: bool = True, v if self.lang == "en": # this to keep arpabet phonemes in the list of options if "[" in text and "]" in text: + lattice = rewrite.rewrite_lattice(text, self.tagger.fst) else: try: @@ -156,11 +157,10 @@ def normalize(self, text: str, n_tagged: int, punct_post_process: bool = True, v tagged_texts, weights = list(zip(*tagged_texts)) else: tagged_texts = self._get_tagged_text(text, n_tagged) - # non-deterministic Eng normalization uses tagger composed with verbalizer, no permutation in between if self.lang == "en": normalized_texts = tagged_texts - normalized_texts = set([self.post_process(text) for text in normalized_texts]) + normalized_texts = [self.post_process(text) for text in normalized_texts] else: normalized_texts = [] for tagged_text in tagged_texts: @@ -178,7 +178,9 @@ def normalize(self, text: str, n_tagged: int, punct_post_process: bool = True, v ] if self.lm: - return normalized_texts, weights + remove_dup = sorted(list(set(zip(normalized_texts, weights))), key=lambda x: x[1]) + normalized_texts, weights = zip(*remove_dup) + return list(normalized_texts), weights normalized_texts = set(normalized_texts) return normalized_texts @@ -383,6 +385,7 @@ def _normalize_line( text=line["text"], verbose=verbose, n_tagged=n_tagged, punct_post_process=punct_post_process, ) + normalized_texts = set(normalized_texts) normalized_text, cer = normalizer.select_best_match( normalized_texts=normalized_texts, input_text=line["text"], @@ -492,6 +495,8 @@ def __process_batch(batch_idx: int, batch: List[str], dir_name: str): punct_post_process=not args.no_punct_post_process, ) + if not normalizer.lm: + normalized_texts = set(normalized_texts) if args.audio_data: asr_model = get_asr_model(args.model) pred_text = asr_model.transcribe([args.audio_data])[0] diff --git a/requirements/requirements_nemo_text_processing.txt b/requirements/requirements_nemo_text_processing.txt index dbb2e3a08726..49c264c540a1 100644 --- a/requirements/requirements_nemo_text_processing.txt +++ b/requirements/requirements_nemo_text_processing.txt @@ -1,3 +1,4 @@ inflect regex pynini==2.1.4 +transformers>=4.0.1 \ No newline at end of file diff --git a/scripts/text_normalization_dataset_files/EngConf.txt b/scripts/text_normalization_dataset_files/EngConf.txt new file mode 100644 index 000000000000..9fe186cde0cf --- /dev/null +++ b/scripts/text_normalization_dataset_files/EngConf.txt @@ -0,0 +1,808 @@ +# This dataset is licensed under a Creative Commons Attribution 4.0 International License. https://creativecommons.org/licenses/by/4.0/ +# This dataset is hand labeled and is intended for English text normalization and has both true positive (~~1) and true negative labels (~~0). +# This dataset focuses on inputs with ambiguous semiotic tokens where normalization dependends on the context. +# This dataset is used to evaluate the context-aware hybrid text normalization under /NeMo/nemo_text_processing/hybrid/ +# +The train leaves on 1/4 at 5pm.~~RAW +The train leaves on one quarter at five p m.~~0 +The train leaves on january fourth at five p m.~~1 +The train leaves on the fourth of january at five p m.~~1 + +Chapter I~~RAW +Chapter one~~1 +Chapter the first~~0 +Chapter first~~0 + +Henry III~~RAW +Henry one~~0 +Henry the third~~1 +Henry third~~1 +Henry first~~0 + +And they agreed that I should go~~RAW +And they agreed that I should go~~1 +And they agreed that one should go~~0 +And they agreed that first should go~~0 +And they agreed that the first should go~~0 + +Serial number V75S~~RAW +Serial number V-seven five S~~1 +Serial number five seventy five S~~0 +Serial number V-seventy five S~~1 + +St. Patrick's cathedral~~RAW +Saint Patrick's cathedral~~1 +Street Patrick's cathedral~~0 + +I'm turning onto Main St. right now.~~RAW +I'm turning onto Main Street right now.~~1 +I'm turning onto Main Saint right now.~~0 + +They delivered 25 kg of apples.~~RAW +They delivered twenty five kilograms of apples.~~1 +They delivered twenty five kg of apples.~~0 +They delivered two five kg of apples.~~0 +They delivered two five kilograms of apples.~~0 + +Text me at 650-451-1234~~RAW +Text me at six five o, four five one, one two three four~~1 +Text me at six hundred fifty four hundred fifty one one two three four~~0 +Text me at six five zero-four five one-one two three four~~1 + +Boeing 737~~RAW +Boeing seven hundred thirty seven~~1 +Boeing seven hundred and thirty seven~~1 + +The equation is 35-20=15~~RAW +The equation is thirty five - twenty = fifteen~~0 +The equation is thirty five minus twenty equals fifteen~~1 + +He owns Jane $125.67~~RAW +He owns Jane one hundred twenty five dollars and sixty seven cents~~1 +He owns Jane one hundred and twenty five dollars and sixty seven cents~~1 +He owns Jane one hundred twenty five dollars sixty seven cents~~1 + +He was dismissed with C grade.~~RAW +He was dismissed with C grade.~~1 +He was dismissed with one hundred grade.~~0 + +When I beg him for play, he shake his head no.~~RAW +When I beg him for play, he shake his head no.~~1 +When I beg him for play, he shake his head number.~~0 + +The 3D computer graphics look sharp.~~RAW +The three d computer graphics look sharp.~~1 +The 3D computer graphics look sharp.~~0 + +The temperature went up from 10-13 °C~~RAW +The temperature went up from ten to thirteen degrees Celsius.~~1 +The temperature went up ten minus thirteen degrees Celsius.~~0 +The temperature went up ten - thirteen degrees Celsius.~~0 + +Dr. Smith was brilliant, and he lived not far at 123 Circle Dr., Santa Maria, CA, 91230~~RAW +Doctor Smith was brilliant, and he lived not far at one twenty three Circle Drive, Santa Maria, California, nine one two three zero~~1 +Doctor Smith was brilliant, and he lived not far at one twenty three Circle Drive Santa Maria California ninety one thousand two hundred thirty~~1 +Doctor Smith was brilliant, and he lived not far at one hundred twenty three Circle Drive, Santa Maria, California, nine one two three zero~~1 +Doctor Smith was brilliant, and he lived not far at one hundred twenty three Circle Drive Santa Maria California ninety one thousand two hundred thirty~~1 + +9th of April 1928~~RAW +Ninth of April nineteen twenty eight~~1 + +The story took place in England, January 1921.~~RAW +The story took place in England, January nineteen twenty one.~~1 + +The story took place in England, January, 1921.~~RAW +The story took place in England, January, nineteen twenty one.~~1 + +On June 19, 1865, Maj. Gen. Gordon Granger, who had fought for the Union, led a force of soldiers to Galveston, TX, to deliver a very important message: The war was finally over, the Union had won, and it now had the manpower to enforce the end of slavery.~~RAW +On the nineteenth of June, eighteen sixty five, Major General Gordon Granger, who had fought for the Union, led a force of soldiers to Galveston, Texas, to deliver a very important message: The war was finally over, the Union had won, and it now had the manpower to enforce the end of slavery.~~1 +On June nineteenth, eighteen sixty five, Major General Gordon Granger, who had fought for the Union, led a force of soldiers to Galveston, Texas, to deliver a very important message: The war was finally over, the Union had won, and it now had the manpower to enforce the end of slavery.~~1 + +It was another Mohammed, not a prophet but a great soldier, surnamed the Conqueror, who finally conquered it, in 1453, after another tremendous siege, of which you will read in history.~~RAW +It was another Mohammed, not a prophet but a great soldier, surnamed the Conqueror, who finally conquered it, in fourteen fifty three, after another tremendous siege, of which you will read in history.~~1 + +In 3/4 of an hour look at it, and should it have swollen very much, and begin to crack, it will be light enough to bake.~~RAW +In three quarters of an hour look at it, and should it have swollen very much, and begin to crack, it will be light enough to bake.~~1 +In three fourths of an hour look at it, and should it have swollen very much, and begin to crack, it will be light enough to bake.~~1 + +In 3/4 hour look at it, and should it have swollen very much, and begin to crack, it will be light enough to bake.~~RAW +In three quarters hour look at it, and should it have swollen very much, and begin to crack, it will be light enough to bake.~~1 +In three fourths hour look at it, and should it have swollen very much, and begin to crack, it will be light enough to bake.~~1 + +Mr. Childs died at 3.01 A.M.~~RAW +Mister Childs died at three o one a m.~~1 + +He set his alarm clock for 4:00 a.m. and dropped immediately into a deep and exhausted sleep.~~RAW +He set his alarm clock for < four a m > and dropped immediately into a deep and exhausted sleep.~~1 + +Excellent march of 19 1/2 miles, 10.5 before lunch.~~RAW +Excellent march of nineteen and a half miles, ten point five before lunch.~~1 + +We have opened out on the 1/7th increase and it makes a lot of difference.~~RAW +We have opened out on the one seventh increase and it makes a lot of difference.~~1 + +carbonate, 2 drams; borax, 1/2 ounce; the salts to be dissolved in water and the other ingredients to be added gradually.~~RAW +carbonate, two drams; borax, one half ounce; the salts to be dissolved in water and the other ingredients to be added gradually.~~1 +carbonate, two drams; borax, a half ounce; the salts to be dissolved in water and the other ingredients to be added gradually.~~1 + +In three years, when he was twenty-one, he had become the head of a publishing house,--Childs & Peterson.~~RAW +In three years, when he was twenty-one, he had become the head of a publishing house,--Childs and Peterson.~~1 + +I live in NEW YORK, N.Y.~~RAW +I live in NEW YORK, New York.~~1 + +I live in Kansas City, MO.~~RAW +I live in Kansas City, Missouri.~~1 + +At this point projectoscope RB-3 of the ship now out of focus control.~~RAW +At this point projectoscope RB-three of the ship now out of focus control.~~1 + +Compare the parallel teaching in Micah 6:6-8.~~RAW +Compare the parallel teaching in Micah six:six to eight.~~1 +Compare the parallel teaching in Micah six:six-eight.~~1 +Compare the parallel teaching in Micah six: from six to eight.~~1 + +A sound authority who knew him of old pronounced him "as good at telling an anecdote as in the '30's.~~RAW +A sound authority who knew him of old pronounced him "as good at telling an anecdote as in the 'thirty's.~~1 +A sound authority who knew him of old pronounced him "as good at telling an anecdote as in the 'thirties.~~1 + +I got a bullet on the liver in the campaign of '03, due to over smoking;~~RAW +I got a bullet on the liver in the campaign of 'zero three, due to over smoking;~~1 + +The ph level of the skin is around 7.~~RAW +The ph level of the skin is around seven~~1 + +Gran just back on ski; left party at 5 1/4 miles.~~RAW +Gran just back on ski; left party at five and one quarter miles.~~1 +Gran just back on ski; left party at five and a quarter miles.~~1 + +1/2 oz. of peppercorns, 4 onions, 6 thin slices of bacon, 2 hard-boiled eggs.~~RAW +one half ounce of peppercorns, four onions, six thin slices of bacon, two hard - boiled eggs.~~1 +one half ounces of peppercorns, four onions, six thin slices of bacon, two hard - boiled eggs.~~1 + +He was a trooper in the 23rd Dragoons.~~RAW +He was a trooper in the twenty third Dragoons.~~1 + +The life principle in trees, &c., as we have seen, was believed to have been derived from the tears of deities.~~RAW +The life principle in trees, and c., as we have seen, was believed to have been derived from the tears of deities.~~1 + +AC is used to refer to an electric current that continually changes direction as it flows.~~RAW +Alternating current is used to refer to an electric current that continually changes direction as it flows.~~1 + +Central AC circulate cool air through a system of supply and return ducts.~~RAW +Central air conditioners circulate cool air through a system of supply and return ducts.~~1 + +What type of AC system do you have?~~RAW +What type of air conditioning system do you have?~~1 + +The initials are A.B.C. and X.X.X.X.~~RAW +The initials are ABC. and XXXX.~~1 + +Please schedule the re-assessment for Tu (2/22) at 5 pm.~~RAW +Please schedule the re-assessment for Tuesday (february twenty second) at five p m.~~1 +Please schedule the re-assessment for Tuesday (the twenty second of february) at five p m.~~1 + +Félicien Tshamalenga Kabundi (born 15 May 1980) is a football defender from Congo DR.~~RAW +Félicien Tshamalenga Kabundi (born the fifteenth of may nineteen eighty) is a football defender from Congo DR.~~1 + +Aces of World War I Osprey Aircraft of the Aces # 40.~~RAW +Aces of World War one Osprey Aircraft of the Aces number forty.~~1 + +The college is affiliated to Tamil Nadu Dr. M.G.R. Medical University.~~RAW +The college is affiliated to Tamil Nadu Dr. MGR Medical University.~~1 +The college is affiliated to Tamil Nadu Dr. M.G.R. Medical University.~~1 +The college is affiliated to Tamil Nadu Doctor MGR Medical University.~~1 + +Vanderheide, Al, "Dutch Reformed Leader dr G.C. Berkouwer Passes Away", Internet Christian Library.~~RAW +Vanderheide, Al, "Dutch Reformed Leader doctor GC Berkouwer Passes Away", Internet Christian Library.~~1 + +In the '70s~~RAW +In the 'seventies~~1 + +PMID 18583455 Tracey WR, Magee WP, Oleynek JJ, Hill RJ, Smith AH, Flynn DM, Knight DR~~RAW +PMID one eight five eight three four five five Tracey WR, Magee WP, Oleynek JJ, Hill RJ, Smith AH, Flynn DM, Knight DR~~1 +PMID eighteen million five hundred eighty three thousand four hundred fifty five Tracey WR, Magee WP, Oleynek JJ, Hill RJ, Smith AH, Flynn DM, Knight DR~~1 + +Dutch doctors may use the letter D behind their name.~~RAW +Dutch doctors may use the letter D behind their name.~~1 + +The route is from 3 Lamps to Queen st via K 'rd, Pitt st, & Grey st. The fare is threepence.~~RAW +The route is from three Lamps to Queen street via K 'road, Pitt street, and Grey street. The fare is threepence.~~1 + +North Dakota Cultural Resources Survey: Building at 317 S. 3rd st" (PDF).~~RAW +North Dakota Cultural Resources Survey: Building at three hundred seventeen south third street" (p d f).~~1 +North Dakota Cultural Resources Survey: Building at three hundred seventeen south third street" (PDF).~~1 +North Dakota Cultural Resources Survey: Building at three seventeen south third street" (PDF).~~1 + +York st # 4, Covent Garden, London; Original from Fogg Library, Digitized May 18, 2007: George Bell and Sons.~~RAW +York street number four, Covent Garden, London; Original from Fogg Library, Digitized may eighteenth two thousand seven: George Bell and Sons.~~1 +York street number four, Covent Garden, London; Original from Fogg Library, Digitized the eighteenth of may, two thousand seven: George Bell and Sons.~~1 + +Risk of invasive cervical cancer associated with polymorphic HLA-DR/DQ haplotypes".~~RAW +Risk of invasive cervical cancer associated with polymorphic HLA-DR/DQ haplotypes"~~1 +Risk of invasive cervical cancer associated with polymorphic HLA dash DR slash DQ haplotypes"~~1 +Risk of invasive cervical cancer associated with polymorphic HLA-DR slash DQ haplotypes"~~1 + +Vetter U, Weis MA, Morike M, Eanes ED, Eyre DR (Feb 1993).~~RAW +Vetter U, Weis MA, Morike M, Eanes ED, Eyre DR (february nineteen ninety three).~~1 + +This approach, similar to OSPF's DR / BDR feature, provides large networks with added IBGP scalability.~~RAW +This approach, similar to OSPF's DR / BDR feature, provides large networks with added IBGP scalability.~~1 +This approach, similar to OSPF's DR slash BDR feature, provides large networks with added IBGP scalability.~~1 + +Microsoft DOS was released through the OEM channel, until DRI released DR DOS 5.0 as a retail upgrade.~~RAW +Microsoft DOS was released through the OEM channel, until DRI released DR DOS five point zero as a retail upgrade.~~1 + +The ST 10 telescopic sight used for direct fire was graduated up to 900 metres.~~RAW +The ST ten telescopic sight used for direct fire was graduated up to nine hundred metres.~~1 + +"STOCKTON ST JOHN'S".~~RAW +"STOCKTON Saint JOHN'S".~~1 + +"Ericsson and st microelectronics complete transaction to split up ST Ericsson".~~RAW +"Ericsson and st microelectronics complete transaction to split up st Ericsson".~~1 + +Single ended, single truck cars (SE ST) from Brownell Car Company which supplied the first electric streetcar to Montreal.~~RAW +Single ended, single truck cars (SE ST) from Brownell Car Company which supplied the first electric streetcar to Montreal.~~1 + +The attenuation of DDAH allows ADMA to accumulate, and to block NO synthesis.~~RAW +The attenuation of DDAH allows ADMA to accumulate, and to block nitrogen monoxide synthesis.~~1 + +Exceptions are odd electron molecules such as nitric oxide, NO, nitrogen dioxide, NO 2, some chlorine oxides and the hydroxyl radical.~~RAW +Exceptions are odd electron molecules such as nitric oxide, nitrogen monoxide, nitrogen dioxide, nitrogen monoxide two, some chlorine oxides and the hydroxyl radical.~~1 + +Now Magazine, VOL 24 NO 39.~~RAW +Now Magazine, Volume twenty four number thirty nine.~~1 +Now Magazine, Volume two four number thirty nine.~~1 +Now Magazine, Volume two four number three nine.~~1 + +She performed in the Michael Kerns play, "AIDS, US Women: Silent NO More."~~RAW +She performed in the Michael Kerns play, "AIDS, US Women: Silent NO More."~~1 + +MacGinnitie AJ, Anant S, Davidson NO (1995).~~RAW +MacGinnitie AJ, Anant S, Davidson NO (nineteen ninety five).~~1 +MacGinnitie AJ, Anant S, Davidson NO (one thousand nine hundred ninety five).~~1 + +MacGinnitie AJ, Anant S, Davidson N.O. (1995).~~RAW +MacGinnitie AJ, Anant S, Davidson NO (nineteen ninety five).~~1 +MacGinnitie AJ, Anant S, Davidson N.O. (one thousand nine hundred ninety five).~~1 + +THERE IS NO BODY CAVITY THAT CANNOT BE REACHED WITH A # 14 G NEEDLE AND A GOOD STRONG ARM.~~RAW +THERE IS NO BODY CAVITY THAT CANNOT BE REACHED WITH A number fourteen G NEEDLE AND A GOOD STRONG ARM.~~1 +THERE IS NO BODY CAVITY THAT CANNOT BE REACHED WITH A number one four G NEEDLE AND A GOOD STRONG ARM.~~1 + +"Vote NO on Proposition YES # 1".~~RAW +"Vote NO on Proposition YES number one".~~1 + +Kandrour Bridge: (8 kilometres from Bilaspur on National Highway NO 88, across the river Satluj).~~RAW +Kandrour Bridge: (eight kilometres from Bilaspur on National Highway number eighty eight, across the river Satluj).~~1 + +18 - 19 Morgen's Anklageschrift, in Nuremberg document NO 2366 John Toland (1976).~~RAW +eighteen to nineteen Morgen's Anklageschrift, in Nuremberg document number two three six six John Toland (nineteen seventy six).~~1 +eighteen to nineteen Morgen's Anklageschrift, in Nuremberg document number two three six six John Toland (one thousand nine hundred seventy six).~~1 +eighteen to nineteen Morgen's Anklageschrift, in Nuremberg document number twenty three sixty six John Toland (nineteen seventy six).~~1 +eighteen to nineteen Morgen's Anklageschrift, in Nuremberg document number twenty three sixty six John Toland (one thousand nine hundred seventy six).~~1 +eighteen to nineteen Morgen's Anklageschrift, in Nuremberg document number two thousand three hundred sixty six John Toland (one thousand nine hundred seventy six).~~1 +eighteen to nineteen Morgen's Anklageschrift, in Nuremberg document number two thousand three hundred sixty six John Toland (nineteen seventy six).~~1 + +The primary School is Two Name is Z. P School NO 1 & Z. P School NO 2 & Sambhaji English School in Kusumba.~~RAW +The primary School is Two Name is Z. P School number one and Z. P School number two and Sambhaji English School in Kusumba.~~1 + +It is situated on State Highway NO 14, RJ SH 14.~~RAW +It is situated on State Highway number fourteen, RJ SH fourteen.~~1 + +Maharagama Vidyala junction (Route NO 990), Maharagama Hokandra (Route NO 994) Maharagama Malabe (Route NO 993), Kottawa Malabe (Route NO 336) but routes operate through Vidyala junction.~~RAW +Maharagama Vidyala junction (Route number nine hundred ninety), Maharagama Hokandra (Route number nine hundred ninety four) Maharagama Malabe (Route number nine hundred ninety three), Kottawa Malabe (Route number three hundred thirty six) but routes operate through Vidyala junction.~~1 + +The place can be reached by road from Barpeta Road (20 KM) connecting National highway NO 31 that connects rest of India.~~RAW +The place can be reached by road from Barpeta Road (twenty kilometers) connecting National highway number thirty one that connects rest of India.~~1 + +Journal of Reliability Engineering Association of Japan Accession number; 02 A 0509168, ISSN 0919-2697, VOL. 24; NO. 4; PAGE 276 - 283 (2002) "A. Albertsen, Electrolytic Capacitor Lifetime Estimation" (PDF).~~RAW +Journal of Reliability Engineering Association of Japan Accession number; zero two A zero five zero nine one six eight, ISSN o nine one nine sil two six nine seven, volume twenty four; number point four; PAGE two hundred seventy six to two hundred eighty three (two thousand two) "A. Albertsen, Electrolytic Capacitor Lifetime Estimation" (PDF).~~1 +Journal of Reliability Engineering Association of Japan Accession number; zero two A zero five zero nine one six eight, ISSN zero nine nineteen to twenty six ninety seven, Volume twenty four; number four; PAGE two hundred seventy six to two hundred eighty three (two thousand two) "A. Albertsen, Electrolytic Capacitor Lifetime Estimation" (PDF).~~1 +Journal of Reliability Engineering Association of Japan Accession number; zero two A zero five zero nine one six eight, ISSN zero nine one nine to twenty six ninety seven, Volume twenty four; number four; PAGE two hundred seventy six to two hundred eighty three (two thousand two) "A. Albertsen, Electrolytic Capacitor Lifetime Estimation" (PDF).~~1 +Journal of Reliability Engineering Association of Japan Accession number; zero two A zero five zero nine one six eight, ISSN zero nine one nine - twenty six ninety seven, Volume twenty four; number four; PAGE two hundred seventy six to two hundred eighty three (two thousand two) "A. Albertsen, Electrolytic Capacitor Lifetime Estimation" (PDF).~~1 + +Zanella AJ, Broom DM, Hunter JC, Mendl MT~~RAW +Zanella AJ, Broom DM, Hunter JC, Mendl MT~~1 + +From then the forks were primarily manufactured at Rock Shox in mt View, california.~~RAW +From then the forks were primarily manufactured at Rock Shox in mountain View, california.~~1 + +dr Tofallis, Kypros, A History of Cyprus, p .98 (2002) Morton, Michael Quentin (December 2011).~~RAW +doctor Tofallis, Kypros, A History of Cyprus, p point nine eight (two thousand two) Morton, Michael Quentin (december twenty eleven).~~1 + +dr Hassan AyatHassan Ayat (1938 - 1981) Relations of Banisadr and AyatAsayesh, Hossein; Adlina Ab.~~RAW +doctor Hassan AyatHassan Ayat (nineteen thirty eight to nineteen eighty one) Relations of Banisadr and AyatAsayesh, Hossein; Adlina Ab.~~1 + +"Death Takes dr Fauver Of Wesleyan: Funeral of Former Athletic Director To Be Held Friday Dies in Middletown".~~RAW +"Death Takes doctor Fauver Of Wesleyan: Funeral of Former Athletic Director To Be Held Friday Dies in Middletown".~~1 + +dr C.H. Asrani.~~RAW +doctor C.H. Asrani.~~1 + +dr Rajendra Prasad became the first President of India.~~RAW +doctor Rajendra Prasad became the first President of India.~~1 + +"Key University Post to dr F.A. DeMarco".~~RAW +"Key University Post to doctor F.A. DeMarco".~~1 +"Key University Post to doctor FA DeMarco".~~1 + +The college is affiliated to Tamil Nadu DR MGR Medical University.~~RAW +The college is affiliated to Tamil Nadu DR MGR Medical University.~~1 + +Khan invited dr Smith to attend the holiday party.~~RAW +Khan invited doctor Smith to attend the holiday party.~~1 + +Other TV shows he has starred in include Coogan's Run, dr Terrible's House of Horrible, Monkey Trousers and Saxondale.~~RAW +Other Television shows he has starred in include Coogan's Run, doctor Terrible's House of Horrible, Monkey Trousers and Saxondale.~~1 + +HLA- DRA encodes the alpha subunit of HLA-DR~~RAW +HLA- DRA encodes the alpha subunit of HLA-DR~~1 + +"121 Seahawk Dr., DeSoto, Texas, 75115."~~RAW +"one twenty one Seahawk drive, DeSoto, Texas, seven five one one five."~~1 +"one twenty one Seahawk drive, DeSoto, Texas, seventy five one one five."~~1 +"one twenty one Seahawk drive, DeSoto, Texas, seventy five thousand one hundred fifteen"~~1 + +612 - 613 "Former Elkhorn Mayor Is Dead", - dr E. T. Ridgway.~~RAW +six hundred twelve to six hundred thirteen "Former Elkhorn Mayor Is Dead", - doctor ET Ridgway.~~1 + +There are two higher education centers located at Laguna dr.~~RAW +There are two higher education centers located at Laguna drive.~~1 + +Happy Computers (HCI) was a small company producing disk drive enhancements for the Atari 8 - bit and Atari ST computer families.~~RAW +Happy Computers (HCI) was a small company producing disk drive enhancements for the Atari eight - bit and Atari ST computer families.~~1 + +From sawdust to stardust: the biography of DeForest Kelley, Star Trek's Dr. McCoy.~~RAW +From sawdust to stardust: the biography of DeForest Kelley, Star Trek's doctor McCoy.~~1 + +Dr. Zac Varghese & Mathew A. Kallumpram.~~RAW +Doctor Zac Varghese and Mathew A. Kallumpram.~~1 + +st Louis Business Journal.~~RAW +saint Louis Business Journal.~~1 + +1918 - 1950: The County Borough of Salford wards of Albert Park, Charlestown, Grosvenor, Kersal, and st Matthias.~~RAW +nineteen eighteen to nineteen fifty: The County Borough of Salford wards of Albert Park, Charlestown, Grosvenor, Kersal, and saint Matthias.~~1 +from nineteen eighteen to nineteen fifty: The County Borough of Salford wards of Albert Park, Charlestown, Grosvenor, Kersal, and saint Matthias.~~1 + +st Louis traded this pick along with a fourth rounder (# 123) to Arizona in exchange for cornerback Aeneas Williams.~~RAW +saint Louis traded this pick along with a fourth rounder (number one hundred twenty three) to Arizona in exchange for cornerback Aeneas Williams.~~1 + +All streetcars westbound on st Clair turn northbound on Gunns, enter the loop westbound, and exit onto st Clair eastbound.~~RAW +All streetcars westbound on saint Clair turn northbound on Gunns, enter the loop westbound, and exit onto saint Clair eastbound.~~1 + +st Peters Church.~~RAW +saint Peters Church.~~1 + +The Puerto Rican Bank has never been connected to its closest eastern bank, st Maarten.~~RAW +The Puerto Rican Bank has never been connected to its closest eastern bank, saint Maarten.~~1 + +Save for the Atlantic outlier of st Kilda.~~RAW +Save for the Atlantic outlier of saint Kilda.~~1 + +st Petersburg Times.~~RAW +saint Petersburg Times.~~1 + +In 1113, King David I of Scotland married Maud, Countess of Huntingdon, widow of Simon de st Liz.~~RAW +In eleven thirteen, King David the first of Scotland married Maud, Countess of Huntingdon, widow of Simon de saint Liz.~~1 +In one thousand one hundred thirteen, King David the first of Scotland married Maud, Countess of Huntingdon, widow of Simon de saint Liz.~~1 + +Born and rasie in the hard streets of st Louis.~~RAW +Born and rasie in the hard streets of saint Louis.~~1 + +"Iowa st 74, Texas A&M 50".~~RAW +"Iowa street seventy four, Texas a and m fifty".~~1 + +New York: st Martin's Press.~~RAW +New York: saint Martin's Press.~~1 + +New York: st Martin's, 2000.~~RAW +New York: saint Martin's, two thousand.~~1 + +new york new york: st Martin's Press.~~RAW +new york new york: saint Martin's Press.~~1 + +New York: st Martin's Press, 1996.~~RAW +New York: saint Martin's Press, nineteen ninety six.~~1 +New York: saint Martin's Press, one thousand nine hundred ninety six.~~1 + +new york new york: st Martin's Press.~~RAW +new york new york: saint Martin's Press.~~1 + +New York: st Martin's Press, 1990, The London Gazette: no~~RAW +New York: saint Martin's Press, nineteen ninety, The London Gazette: no~~1 +New York: saint Martin's Press, one thousand nine hundred ninety, The London Gazette: no~~1 + +Katharine M. Rogers, L. Frank Baum, Creator of Oz: A Biography, New York, st Martin's Press, 2002; p. 62.~~RAW +Katharine M. Rogers, L. Frank Baum, Creator of Oz: A Biography, New York, saint Martin's Press, two thousand two; p. sixty two.~~1 + +Crestwood, New York: st Vladimir's Seminary Press.~~RAW +Crestwood, New York: saint Vladimir's Seminary Press.~~1 + +Engelke DR, Hoener PA, Collins FS (1988).~~RAW +Engelke DR, Hoener PA, Collins FS (nineteen eighty eight).~~1 + +Danish TV- Documentary, aired several times at DR 'channelsDanish TV- Documentary, aired several times at DR (broadcaster)' channels.~~RAW +Danish TV- Documentary, aired several times at DR 'channelsDanish TV- Documentary, aired several times at DR (broadcaster)' channels.~~1 + +Lin L, Nemeth E, Goodnough JB, Thapa DR, Gabayan V, Ganz T (2008).~~RAW +Lin L, Nemeth E, Goodnough JB, Thapa DR, Gabayan V, Ganz T (two thousand eight).~~1 + +Friedlander DR, Milev P, Karthikeyan L et al.~~RAW +Friedlander DR, Milev P, Karthikeyan L et al.~~1 + +In 1366 Dr. King Morkann died, appointing his son Korox, the leader of Elestam's Crusaders, to succeed him.~~RAW +In thirteen sixty six doctor King Morkann died, appointing his son Korox, the leader of Elestam's Crusaders, to succeed him.~~1 +In thirteen sixty six Doctor King Morkann died, appointing his son Korox, the leader of Elestam's Crusaders, to succeed him.~~1 + +Shiau AK, Harris SF, Southworth DR, Agard DA (Oct 2006).~~RAW +Shiau AK, Harris SF, Southworth DR, Agard DA (october two thousand six).~~1 + +His song was also chosen as "Song of the Week" on Denmark's DR P 3 Radio.~~RAW +His song was also chosen as "Song of the Week" on Denmark's DR P three Radio.~~1 + +For data deduplication, NetVault Backup supports several solutions, including Dell's own DR appliance and NetVault SmartDisk.~~RAW +For data deduplication, NetVault Backup supports several solutions, including Dell's own DR appliance and NetVault SmartDisk.~~1 + +"BBC News — DR Congo's 23 rebels: Rwandan support 'falling'".~~RAW +"BBC News — DR Congo's twenty three rebels: Rwandan support 'falling'".~~1 + +Richard Stanford (DR) 9.~~RAW +Richard Stanford (Doctor) nine.~~1 + +Use of battery power enabled these memories to retain their contents which the DR 110 was switched off.~~RAW +Use of battery power enabled these memories to retain their contents which the DR one hundred ten was switched off.~~1 + +Crystal Serenity was built in 2003 in STX Europe in st Nazaire.~~RAW +Crystal Serenity was built in two thousand three in STX Europe in saint Nazaire.~~1 + +He was elected as the Member of Legislative Assembly of Arunachal Pradesh from the 6 - Thrizino Buragaon (ST) in 2014 assembly election.~~RAW +He was elected as the Member of Legislative Assembly of Arunachal Pradesh from the six - Thrizino Buragaon (ST) in twenty fourteen assembly election.~~1 + +"TRU for all Districts (SC & ST and Total)".~~RAW +"TRU for all Districts (SC and ST and Total)".~~1 + +ST S adds ST on sunroof and 16" alloy wheels.~~RAW +ST S adds ST on sunroof and sixteen inches alloy wheels.~~1 + +"ST Engineering Acquires Specialized Vehicles Corporations" (Press release).~~RAW +"ST Engineering Acquires Specialized Vehicles Corporations" (Press release).~~1 + +The first order formulation of ST rules out quantifying over types.~~RAW +The first order formulation of ST rules out quantifying over types.~~1 + +"Description of FEMTO ST Laboratory".~~RAW +"Description of FEMTO ST Laboratory".~~1 + +"TRU for all Districts (SC & ST and Total)".~~RAW +"TRU for all Districts (SC and ST and Total)".~~1 + +Atari ST User 3 (1).~~RAW +Atari ST User three (one).~~1 + +Similar motifs occur with serine or threonine as residue i, which are called ST turns.~~RAW +Similar motifs occur with serine or threonine as residue i, which are called ST turns.~~1 + +As the popularity of the ST increased, it was given its own pull out section called Atari ST User.~~RAW +As the popularity of the ST increased, it was given its own pull out section called Atari ST User.~~1 + +Martin ST, Sato N, Dhara S, et al.~~RAW +Martin ST, Sato N, Dhara S, et al.~~1 + +ST Aerospace started as a maintenance depot to support the Republic of Singapore Air Force in 1975.~~RAW +ST Aerospace started as a maintenance depot to support the Republic of Singapore Air Force in nineteen seventy five.~~1 + +The ST segment corresponds to a period of ventricular depolarization.~~RAW +The ST segment corresponds to a period of ventricular depolarization.~~1 + +The first version was released in 1990 for the Atari ST, the most recent version for the GBA in 2002.~~RAW +The first version was released in nineteen ninety for the Atari ST, the most recent version for the GBA in two thousand two.~~1 + +Logic stemmed from Creator, then Notator, made by C - Lab (the company's forerunner) for the Atari ST platform.~~RAW +Logic stemmed from Creator, then Notator, made by C - Lab (the company's forerunner) for the Atari ST platform.~~1 + +This game was released for the Amiga, Atari ST, and PC DOS.~~RAW +This game was released for the Amiga, Atari ST, and PC DOS.~~1 + +Los Angeles: Capitol Records ST 11547.~~RAW +Los Angeles: Capitol Records ST one one five four seven.~~1 +Los Angeles: Capitol Records ST eleven thousand five hundred forty seven.~~1 + +Andrade J, Pearce ST, Zhao H, Barroso M (Dec 2004).~~RAW +Andrade J, Pearce ST, Zhao H, Barroso M (december two thousand four).~~1 + +Nielsen C, Hansen D, Husby S, Jacobsen BB, Lillevang ST (Dec 2003).~~RAW +Nielsen C, Hansen D, Husby S, Jacobsen BB, Lillevang ST (december two thousand three).~~1 + +However, random access to the stack registers can be obtained through an instruction which exchanges any specified ST (x) with ST (0).~~RAW +However, random access to the stack registers can be obtained through an instruction which exchanges any specified ST (x) with ST (zero).~~1 + +5% of places are reserved for talented ST / SC students of the weaker section of the society.~~RAW +five percent of places are reserved for talented ST / SC students of the weaker section of the society.~~1 +five percent of places are reserved for talented ST slash SC students of the weaker section of the society.~~1 + +Calamus is a desktop publishing application, built for the Atari ST computer.~~RAW +Calamus is a desktop publishing application, built for the Atari ST computer.~~1 + +"Hanson Chicagoan Cigno Firenze ST and Gatto".~~RAW +"Hanson Chicagoan Cigno Firenze ST and Gatto".~~1 + +Another achievement on the Atari ST was the first multiplayer first person shooter on a homecomputer: MIDI Maze.~~RAW +Another achievement on the Atari ST was the first multiplayer first person shooter on a homecomputer: MIDI Maze.~~1 + +The Acorn Electron, BBC Micro, 1990: Amiga, Atari ST, DOS and NES ports followed in 1989.~~RAW +The Acorn Electron, BBC Micro, nineteen ninety: Amiga, Atari ST, DOS and NES ports followed in nineteen eighty nine.~~1 + +The NO MORE Project.~~RAW +The NO MORE Project.~~1 + +NO 4, 1984 (PDF).~~RAW +Number four, nineteen eighty four (PDF).~~1 +Number four, one thousand nine hundred eighty four (PDF).~~1 + +The highest share of "NO" votes was in Crete, particularly in the constituencies of Heraklion and Chania.~~RAW +The highest share of "NO" votes was in Crete, particularly in the constituencies of Heraklion and Chania.~~1 + +While he was Vice Chancellor of Banaras Hindu University, it was declared the NO. 1 university of India by India Today.~~RAW +While he was Vice Chancellor of Banaras Hindu University, it was declared the number one university of India by India Today.~~1 + +ltd vs Uttam Manohar Nakate on 18 January, 2005,, CASE NO.~~RAW +limited versus Uttam Manohar Nakate on the eighteenth of january, two thousand five,, CASE number.~~1 +limited versus Uttam Manohar Nakate on the eighteenth of january, twenty oh five,, CASE number.~~1 + +"Climatography of the United States NO. 81" (PDF).~~RAW +"Climatography of the United States number eighty one" (PDF).~~1 + +NO TRIPPING UP or HEEL KICKING is allowed.~~RAW +NO TRIPPING UP or HEEL KICKING is allowed.~~1 + +Corey Feldman stated in his November 25, 2008 blog post, "NO!~~RAW +Corey Feldman stated in his november twenty fifth, two thousand eight blog post, "NO!~~1 +Corey Feldman stated in his the twenty fifth of november, two thousand eight blog post, "NO!~~1 + +"No LIMITS for front rows".~~RAW +"No LIMITS for front rows".~~1 + +"CALDWELL'S TNA NO SURRENDER PPV REPORT 9/20: Ongoing" virtual time "coverage of Kurt Angle vs~~RAW +"CALDWELL'S TNA No SURRENDER PPV REPORT september twentieth: Ongoing" virtual time "coverage of Kurt Angle versus~~1 + +"SETH MacFARLANE RELEASES NEW ALBUM NO ONE EVER TELLS YOU TO ALL DIGITAL PARTNERS".~~RAW +"SETH MacFARLANE RELEASES NEW ALBUM NO ONE EVER TELLS YOU TO ALL DIGITAL PARTNERS".~~1 + +Nutphand, W. (No DATE).~~RAW +Nutphand, W. (No DATE).~~1 + +"Vote NO Referendum Website Launched", Family First Press Release, 22 June 2009.~~RAW +"Vote NO Referendum Website Launched", Family First Press Release, the twenty second of june two thousand nine.~~1 + +"Investigation and Analyses of Problems in Standardization of 'NO. 1 Military Project'".~~RAW +"Investigation and Analyses of Problems in Standardization of 'Number one Military Project'".~~1 + +"No matter what shape (YOUR STOMACH'S IN) by THE T - BONES".~~RAW +"No matter what shape (YOUR STOMACH'S IN) by THE T - BONES".~~1 + +Impaired NO synthesisNitric oxide is known as an important stimulator of cell proliferation, maturation and differentiation.~~RAW +Impaired nitrogen monoxide synthesisNitric oxide is known as an important stimulator of cell proliferation, maturation and differentiation.~~1 + +"PUSH & SHOVE by No Doubt (CD)".~~RAW +"PUSH and SHOVE by No Doubt (CD)".~~1 + +Sahih Bukhari hadith NO 732-733 "Hajj".~~RAW +Sahih Bukhari hadith number seven hundred thirty two to seven hundred thirty three "Hajj".~~1 + +The next prime mover models, which marginally differed from the NO 2, were the NO 3 and NO 6.~~RAW +The next prime mover models, which marginally differed from the number two, were the number three and number six.~~1 + +Kottawa Kiriwaththuduwa (Route NO 128), Kottawa Malabe (Route NO 336), Rukmalagama Pettah (Route No 138/3), Athurugiriya Pettah (Route No 136,138/4) bus routes operate through this junction.~~RAW +Kottawa Kiriwaththuduwa (Route number one hundred twenty eight), Kottawa Malabe (Route number three hundred thirty six), Rukmalagama Pettah (Route No one hundred thirty eight thirds), Athurugiriya Pettah (Route No one hundred thirty six thousand one hundred thirty eight quarters) bus routes operate through this junction.~~1 + +Play '70s music now~~RAW +Play seventies music now~~1 + +02.15.2017~~RAW +february fifteenth twenty seventeen~~1 + +02/15/2017~~RAW +february fifteenth twenty seventeen~~1 + +Remove Tuesday alarm of 9.00 a.m.~~RAW +Remove Tuesday alarm of nine a m~~1 + +repeat song no. 10 from main list~~RAW +repeat song number ten from main list~~1 + +what will the weather conditions of Lucknow on 9th feb 2017~~RAW +what will the weather conditions of Lucknow on the ninth of february twenty seventeen~~1 +what will the weather conditions of Lucknow on ninth february twenty seventeen~~1 + +lower the light of hall by 75%~~RAW +lower the light of hall by seventy five percent~~1 + +I like songs from 90s~~RAW +i like songs from nineties~~1 + +Can you see to it that the coffee maker is ready with my filter coffee in approx. 10 min. from now?~~RAW +Can you see to it that the coffee maker is ready with my filter coffee in approximately ten minutes. from now~~1 +Can you see to it that the coffee maker is ready with my filter coffee in approximately ten minutes from now~~1 + +Play 1999 by prince~~RAW +Play one thousand nine hundred ninety nine by prince~~1 +Play nineteen ninety nine by prince~~1 + +What is the time in India, when the time in US is 12pm~~RAW +What is the time in India, when the time in US is twelve pm~~1 +What is the time in India, when the time in US is twelve p m~~1 + +05/02/2017~~RAW +may second twenty seventeen~~1 +the second of may twenty seventeen~~1 + +10/02/2017~~RAW +october second twenty seventeen~~1 +the second of october twenty seventeen~~1 + +What was the news for March 4th 2017?~~RAW +What was the news for march fourth twenty seventeen?~~1 +What was the news for the fourth of march twenty seventeen?~~1 + +Select and play only christian rock from 1990 upwards~~RAW +Select and play only christian rock from nineteen ninety upwards~~1 +Select and play only christian rock from one thousand nine hundred ninety nine upwards~~1 + +How is the weather in 57701?~~RAW +How is the weather in five seven seven zero one?~~1 +How is the weather in fifty seven thousand seven hundred one?~~1 + +Where can I find a tax preparation professional for under $100?~~RAW +Where can I find a tax preparation professional for under one hundred dollars~~1 + +What's going on between 10am and 2 pm?~~RAW +What's going on between ten a m and two p m?~~1 +What's going on between ten am and two p m?~~1 +What's going on between ten am and two pm?~~1 + +Set a meeting on Wednesday at 3pm in room G30.~~RAW +Set a meeting on Wednesday at three p m in room G thirty.~~1 +Set a meeting on Wednesday at three p m in room G three zero.~~1 + +Add to calender, Dr. K at 3:30 pm on March 7~~RAW +Add to calender, doctor K at three thirty p m on march seventh~~1 + +Set a 2-week reminder for the 20th of every quarter beginning 3/20 to pay my HOA.~~RAW +Set a two-week reminder for the twentieth of every quarter beginning march twentieth to pay my HOA.~~1 +Set a two-week reminder for the twentieth of every quarter beginning the twentieth of march to pay my HOA.~~1 + +What do i have on the schedule today from 12pm-2pm~~RAW +What do i have on the schedule today from twelve p m to two p m~~1 +What do i have on the schedule today from twelve p m-two p m~~1 + +if a pencil costs $4 and a book cost $10, how much i should pay if i buy 2 pencil and 3 books?~~RAW +if a pencil costs four dollars and a book cost ten dollars, how much i should pay if i buy two pencil and three books?~~1 + +how many shares of apple can I get for $800~~RAW +how many shares of apple can I get for eight hundred dollars~~1 + +I have a meeting, set a reminder for 4/9~~RAW +I have a meeting, set a reminder for the ninth of april~~1 + +Find Rush Limbaugh on 97X~~RAW +Find Rush Limbaugh on ninety seven X~~1 +Find Rush Limbaugh on nine seven X~~1 + +play the radio channel 93.25~~RAW +play the radio channel ninety three point two five~~1 +play the radio channel ninety three point twenty five~~1 + +Check time of 10887 train from Jodhpur station on monday~~RAW +Check time of one zero eight eight seven train from Jodhpur station on monday~~1 +Check time of ten thousand eight hundred eighty seven train from Jodhpur station on monday~~1 + +Divide 1500 by 160~~RAW +Divide one thousand five hundred by one hundred sixty~~1 +Divide fifteen hundred by one hundred sixty~~1 + +What's 1/2 cup plus 2/3rd cup~~RAW +What's a half cup plus two thirds cup~~1 +What's one half cup plus two thirds cup~~1 + +Set the thermostat to 75F~~RAW +Set the thermostat to seventy five degrees Fahrenheit~~1 + +What music is currently playing on 99.9FM?~~RAW +what music is currently playing on ninety nine point nine f m~~1 + +convert 23:30 from GMT +4:30 to GMT -2:00~~RAW +convert twenty three thirty from GMT plus four thirty to GMT minus two o'clock~~1 +convert twenty three thirty from GMT + four thirty to GMT - two o'clock~~1 + +convert 10:30 from GMT +2:30 to GMT 0:00~~RAW +convert ten thirty from GMT plus two thirty to GMT zero o'clock~~1 +convert ten thirty from GMT + two thirty to GMT zero o'clock~~1 +convert ten thirty from g m t plus two thirty to g m t zero hundred~~1 + +order take out from Jason's Deli on 6th ave~~RAW +order take out from jason's Deli on sixth avenue~~1 + +remind me at 13:00 ~~RAW +remind me at thirteen o'clock~~1 +remind me at thirteen hundred~~1 + +Play be warned by tech n9ne~~RAW +Play be warned by tech n nine ne~~1 + +Please play any music created in the 1980's decade~~RAW +please play any music created in the nineteen eighties decade~~1 +please play any music created in the nineteen eighty's decade~~1 + +Play 46&2 by Tool~~RAW +Play forty six&two by tool~~1 +Play forty six and two by tool~~1 + +Have Pho #1 send 3 egg rolls.~~RAW +have Pho number one send three egg rolls.~~1 +have Pho # one send three egg rolls.~~1 + +set an event'happy birthday john' to repeat on Jan-21-2017~~RAW +set an event'happy birthday john to repeat on january twenty first twenty seventeen~~1 +set an event'happy birthday john to repeat on the twenty first of january twenty seventeen~~1 + +I have a doctor's appointment with Dr. Barlease tomorrow at 7:30 a.m.~~RAW +i have a doctor's appointment with doctor barlease tomorrow at seven thirty a m.~~1 + +What do I have going on this morning between 10 - 12~~RAW +what do i have going on this morning between ten to twelve~~1 + +Put on fm101~~RAW +Put on fm one zero one~~1 +Put on fm one o one~~1 +Put on fm one oh one~~1 +Put on fm one hundred one~~1 + +let's play fifa2016, you'll be the keeper and never change position.~~RAW +let's play fifa twenty sixteen, you'll be the keeper and never change position.~~1 +let's play fifa two thousand sixteen, you'll be the keeper and never change position.~~1 +let's play fifa two zero one six, you'll be the keeper and never change position.~~1 + +what is 0/0?~~RAW +what is zero divided by zero?~~1 + +what is 123 * 123?~~RAW +what is one hundred twenty three times one hundred twenty three?~~1 + +What is 1+1?~~RAW +What is one plus one?~~1 + +How much is 1+1?~~RAW +How much is one plus one?~~1 + +What does 1+1 equal?~~RAW +what does one plus one equal?~~1 + +Can you add up 13+5+9 for me?~~RAW +Can you add up thirteen plus five plus nine for me?~~1 + +How do you solve 3 - 2?~~RAW +How do you solve three minus two?~~1 + +what's 50kg in lbs?~~RAW +what's fifty kilograms in pounds?~~1 + +answer the equation 8*7~~RAW +answer the equation eight times seven?~~1 + +How much is 1000 USD in Indian Rupees?~~RAW +How much is one thousand united states dollars in indian rupees?~~1 +How much is one thousand USD in indian rupees?~~1 + +Solve this equation: 1+1.~~RAW +Solve this equation one plus one.~~1 + +whats was the value of x if x+2 = 5?~~RAW +whats was the value of x if x plus two equals five?~~1 + +Please tweet @PizzaHut I've been waiting for my delivery for 85min now #nothappy~~RAW +Please tweet at PizzaHut I've been waiting for my delivery for eighty five minute now hashtag nothappy~~1 +Please tweet at PizzaHut I've been waiting for my delivery for eighty five minute now hash nothappy~~1 +Please tweet @ PizzaHut I've been waiting for my delivery for eighty five minute now # nothappy~~1 \ No newline at end of file diff --git a/setup.py b/setup.py index 5514c7057250..89871858ad46 100644 --- a/setup.py +++ b/setup.py @@ -96,12 +96,39 @@ def req_file(filename, folder="requirements"): extras_require['all'] = list(chain(extras_require.values())) # Add lightning requirements as needed -extras_require['common'] = list(chain([extras_require['common'], extras_require['nemo_text_processing']])) -extras_require['test'] = list(chain([extras_require['tts'], extras_require['core'], extras_require['common']])) +extras_require['common'] = list(chain([extras_require['common'], extras_require['core']])) +extras_require['test'] = list( + chain( + [ + extras_require['tts'], + extras_require['core'], + extras_require['common'], + extras_require['nemo_text_processing'], + ] + ) +) extras_require['asr'] = list(chain([extras_require['asr'], extras_require['core'], extras_require['common']])) extras_require['cv'] = list(chain([extras_require['cv'], extras_require['core'], extras_require['common']])) -extras_require['nlp'] = list(chain([extras_require['nlp'], extras_require['core'], extras_require['common']])) -extras_require['tts'] = list(chain([extras_require['tts'], extras_require['core'], extras_require['common']])) +extras_require['nlp'] = list( + chain( + [ + extras_require['nlp'], + extras_require['core'], + extras_require['common'], + extras_require['nemo_text_processing'], + ] + ) +) +extras_require['tts'] = list( + chain( + [ + extras_require['tts'], + extras_require['core'], + extras_require['common'], + extras_require['nemo_text_processing'], + ] + ) +) # TTS has extra dependencies extras_require['tts'] = list(chain([extras_require['tts'], extras_require['asr']])) diff --git a/tests/nemo_text_processing/en/data_text_normalization/test_cases_measure.txt b/tests/nemo_text_processing/en/data_text_normalization/test_cases_measure.txt index c26138f813ca..4d16a18eda89 100644 --- a/tests/nemo_text_processing/en/data_text_normalization/test_cases_measure.txt +++ b/tests/nemo_text_processing/en/data_text_normalization/test_cases_measure.txt @@ -16,3 +16,4 @@ covid-19.5~covid- nineteen point five 1°C~one degree Celsius 1234-123kg~one thousand two hundred and thirty four to one hundred and twenty three kilograms 45º&C~forty five degree and C +41,459.00 km³~forty one thousand four hundred and fifty nine point zero zero cubic kilometers diff --git a/tests/nemo_text_processing/en/data_text_normalization/test_cases_normalize_with_audio.txt b/tests/nemo_text_processing/en/data_text_normalization/test_cases_normalize_with_audio.txt index 5698b7886116..6e52405a8a89 100644 --- a/tests/nemo_text_processing/en/data_text_normalization/test_cases_normalize_with_audio.txt +++ b/tests/nemo_text_processing/en/data_text_normalization/test_cases_normalize_with_audio.txt @@ -162,5 +162,4 @@ two divided by eight [YI1] [YI one] ~/ and $ -slash and dollar / and dollar diff --git a/tests/nemo_text_processing/en/data_text_normalization/test_cases_serial.txt b/tests/nemo_text_processing/en/data_text_normalization/test_cases_serial.txt index 8a1234731cfa..4007bf5c0105 100644 --- a/tests/nemo_text_processing/en/data_text_normalization/test_cases_serial.txt +++ b/tests/nemo_text_processing/en/data_text_normalization/test_cases_serial.txt @@ -29,4 +29,3 @@ a 4-kilogram bag~a four-kilogram bag 100-car~one hundred-car 123/261788/2021~one hundred twenty three/two six one seven eight eight/two thousand twenty one 2*8~two asterisk eight -and/or~and slash or \ No newline at end of file diff --git a/tests/nemo_text_processing/en/data_text_normalization/test_cases_whitelist.txt b/tests/nemo_text_processing/en/data_text_normalization/test_cases_whitelist.txt index dfa36a15a378..7fafb9d77152 100644 --- a/tests/nemo_text_processing/en/data_text_normalization/test_cases_whitelist.txt +++ b/tests/nemo_text_processing/en/data_text_normalization/test_cases_whitelist.txt @@ -3,3 +3,4 @@ Mrs. Norris~misses Norris DNA is~DNA is C. S. Lewis~CS Lewis tv~TV +and/or~and/or From 8abe0f4a71443950d4c82fd91f7eafc0800ac300 Mon Sep 17 00:00:00 2001 From: Virginia Adams <78445382+vadam5@users.noreply.github.com> Date: Fri, 15 Jul 2022 15:42:31 -0700 Subject: [PATCH 31/52] Removed NLPDDPPlugin Import check (#4555) * Removed NLPDDPPlugin Import check Signed-off-by: Virginia Adams * Python formatting fix Signed-off-by: Virginia Adams * changed app to app_state Signed-off-by: Virginia Adams * moved num workers check back to bottom Signed-off-by: Virginia Adams * Python code reformat Signed-off-by: Virginia Adams --- nemo/core/classes/modelPT.py | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/nemo/core/classes/modelPT.py b/nemo/core/classes/modelPT.py index ec29b954ced4..8bab7c573ac1 100644 --- a/nemo/core/classes/modelPT.py +++ b/nemo/core/classes/modelPT.py @@ -37,13 +37,6 @@ from nemo.utils.debug_hook import register_debug_hooks from nemo.utils.get_rank import get_rank, is_global_rank_zero -try: - from nemo.collections.nlp.parts.nlp_overrides import NLPDDPPlugin - - HAVE_NLPPLUGIN = True -except (ImportError, ModuleNotFoundError): - HAVE_NLPPLUGIN = False - __all__ = ['ModelPT'] @@ -495,10 +488,16 @@ def setup_optimization(self, optim_config: Optional[Union[DictConfig, Dict]] = N optim_config['sched']['t_max_epochs'] = self._trainer.max_epochs optim_config['sched']['t_accumulate_grad_batches'] = self._trainer.accumulate_grad_batches optim_config['sched']['t_limit_train_batches'] = self._trainer.limit_train_batches - optim_config['sched']['t_num_workers'] = self._trainer.num_devices * self._trainer.num_nodes - if HAVE_NLPPLUGIN and isinstance(self._trainer.accelerator.training_type_plugin, NLPDDPPlugin): - app = AppState() - optim_config['sched']['t_num_workers'] = app.data_parallel_size + + app_state = AppState() + if app_state.data_parallel_size is not None: + optim_config['sched']['t_num_workers'] = app_state.data_parallel_size + elif app_state.model_parallel_size is None: + optim_config['sched']['t_num_workers'] = self._trainer.num_devices * self._trainer.num_nodes + else: + optim_config['sched']['t_num_workers'] = ( + self._trainer.num_devices * self._trainer.num_nodes + ) / app_state.model_parallel_size else: optim_config['sched']['max_steps'] = self._trainer.max_steps From 65b9b5701a57cc6f9bfff01440824fa1d883b3d9 Mon Sep 17 00:00:00 2001 From: Sandeep Subramanian Date: Fri, 15 Jul 2022 16:39:20 -0700 Subject: [PATCH 32/52] Add length ratio filtering script (#4551) * Add length ratio filtering script Signed-off-by: MaximumEntropy * Fix example Signed-off-by: MaximumEntropy * Remove extra quotes Signed-off-by: MaximumEntropy Co-authored-by: Abhinav Khattar --- .../length_ratio_filter.py | 335 ++++++++++++++++++ 1 file changed, 335 insertions(+) create mode 100644 scripts/neural_machine_translation/length_ratio_filter.py diff --git a/scripts/neural_machine_translation/length_ratio_filter.py b/scripts/neural_machine_translation/length_ratio_filter.py new file mode 100644 index 000000000000..4f58a88217c2 --- /dev/null +++ b/scripts/neural_machine_translation/length_ratio_filter.py @@ -0,0 +1,335 @@ +# Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse +import logging +import multiprocessing as mp +import re +import shutil +import warnings +from pathlib import Path +from time import sleep + +from tqdm import tqdm + +""" +Usage: +python length_ratio_filter.py --input-src train.en \ + --input-tgt train.de \ + --output-src train_lang_filtered.en \ + --output-tgt train_lang_filtered.de \ + --removed-src train_removed.en \ + --removed-tgt train_removed.de \ + --min_length 1 \ + --max_length 512 \ + --ratio 1.3 +""" + +logging.basicConfig(level=logging.INFO) + + +def get_args(): + parser = argparse.ArgumentParser( + description="""A multi-processed script for filtering a parallel corpus to remove sentences that are less than a minimum length + or longer than a maximum length. Also filters based on the length ratio between source and target sentences""" + ) + parser.add_argument( + "--input-src", + "-s", + help="Path to the input file which has to contain text in language `source_lang`.", + required=True, + type=Path, + ) + parser.add_argument( + "--input-tgt", + "-t", + help="Path to the input file which has to contain text in language `target_lang`. If not provided, data is " + "processed as monolingual.", + type=Path, + ) + parser.add_argument( + "--output-src", + "-S", + help="Path to the file where filtered `input_src` will be saved.", + required=True, + type=Path, + ) + parser.add_argument( + "--output-tgt", "-T", help="Path to the output target file", type=Path, + ) + parser.add_argument( + "--removed-src", "-r", required=True, help="Path to file where removed source lines will be saved", type=Path, + ) + parser.add_argument( + "--removed-tgt", "-R", help="Path to file where removed target lines will be saved", type=Path, + ) + parser.add_argument( + "--num-jobs", + "-j", + type=int, + help="Number of jobs. By default, the number of jobs is equal to the number of CPU cores.", + ) + parser.add_argument( + "--min-length", "-m", type=int, default=1, help="Minimum sequence length (after .split())", + ) + parser.add_argument( + "--max-length", "-M", type=int, default=512, help="Maximum sequence length (after .split())", + ) + parser.add_argument( + "--ratio", + "-z", + type=float, + default=1.3, + help="Ratio of the length of the source sentence to the length of the target sentence.", + ) + args = parser.parse_args() + args.input_src = args.input_src.expanduser() + if args.input_tgt is not None: + args.input_tgt = args.input_tgt.expanduser() + args.output_src = args.output_src.expanduser() + if args.output_tgt is not None: + args.output_tgt = args.output_tgt.expanduser() + args.removed_src = args.removed_src.expanduser() + if args.removed_tgt is not None: + args.removed_tgt = args.removed_tgt.expanduser() + return args + + +def get_edges_in_1_file(fn, num_parts): + num_lines = 0 + edges = [0] + with open(fn) as f: + i = 0 + for l in f: + i += len(l.encode('utf-8')) + edges.append(i) + num_lines += 1 + return [edges[int(i * num_lines / num_parts)] for i in range(num_parts)] + [edges[-1]], num_lines + + +def get_edges_and_num_lines(src_fn, tgt_fn, num_parts): + src_edges, src_num_lines = get_edges_in_1_file(src_fn, num_parts) + assert num_parts + 1 == len(src_edges) + src_edges = [(src_edges[i], src_edges[i + 1]) for i in range(len(src_edges) - 1)] + if tgt_fn is not None: + tgt_edges, tgt_num_lines = get_edges_in_1_file(tgt_fn, num_parts) + tgt_edges = [(tgt_edges[i], tgt_edges[i + 1]) for i in range(len(tgt_edges) - 1)] + if tgt_num_lines != src_num_lines: + raise ValueError( + f"Source {repr(src_fn)} and target {repr(tgt_fn)} files have different number of lines " + f"{src_num_lines} and {tgt_num_lines} correspondingly." + ) + else: + tgt_edges = [None] * num_parts + assert len(src_edges) == num_parts + return src_edges, tgt_edges, src_num_lines + + +def filter_pairs( + src_edges, + tgt_edges, + input_src, + input_tgt, + filtered_dir_src, + filtered_dir_tgt, + removed_dir_src, + removed_dir_tgt, + min_length, + max_length, + length_ratio, + rank, +): + global counter + output_src = filtered_dir_src / Path(f"rank{rank}") + output_src_removed = removed_dir_src / Path(f"rank{rank}") + output_tgt = filtered_dir_tgt / Path(f"rank{rank}") + output_tgt_removed = removed_dir_tgt / Path(f"rank{rank}") + with open(input_src) as in_src, open(input_tgt) as in_tgt, open(output_src, 'w') as out_src, open( + output_tgt, 'w' + ) as out_tgt, open(output_src_removed, 'w') as out_r_src, open(output_tgt_removed, 'w') as out_r_tgt: + in_src.seek(src_edges[0]) + in_tgt.seek(tgt_edges[0]) + src_l, tgt_l, i = in_src.readline(), in_tgt.readline(), 0 + if in_src.tell() > src_edges[1] or in_tgt.tell() > tgt_edges[1]: + return + while src_l and tgt_l: + with counter.get_lock(): + counter.value += 1 + src_l = src_l.strip() + tgt_l = tgt_l.strip() + src_l_len = len(src_l.split()) + tgt_l_len = len(tgt_l.split()) + # Length filtering + if (src_l_len < min_length or src_l_len > max_length) or ( + tgt_l_len < min_length or tgt_l_len > max_length + ): + out_r_src.write(src_l + "\n") + out_r_tgt.write(tgt_l + "\n") + # Ratio filtering + elif src_l_len / tgt_l_len > length_ratio or tgt_l_len / src_l_len > length_ratio: + out_r_src.write(src_l + "\n") + out_r_tgt.write(tgt_l + "\n") + else: + out_src.write(src_l + '\n') + out_tgt.write(tgt_l + '\n') + if in_src.tell() >= src_edges[1]: + if in_tgt.tell() < tgt_edges[1]: + raise ValueError( + f"Edges of target and source has to be reached simultaneously, whereas " + f"in_src.tell()={in_src.tell()}, in_tgt.tell()={in_tgt.tell()}, " + f"src_edges[1]={src_edges[1]}, tgt_edges[1]={tgt_edges[1]}." + ) + break + if in_tgt.tell() >= tgt_edges[1]: + raise ValueError( + f"Edges of target and source has to be reached simultaneously, whereas " + f"in_src.tell()={in_src.tell()}, in_tgt.tell()={in_tgt.tell()}, " + f"src_edges[1]={src_edges[1]}, tgt_edges[1]={tgt_edges[1]}." + ) + src_l, tgt_l, i = in_src.readline(), in_tgt.readline(), i + 1 + with counter.get_lock(): + counter.value += 1 + + +def filter_by_length_and_ratio(args): + ( + src_edges, + tgt_edges, + input_src, + input_tgt, + filtered_dir_src, + filtered_dir_tgt, + removed_dir_src, + removed_dir_tgt, + min_length, + max_length, + length_ratio, + rank, + ) = args + logging.debug(f"filter by args: {min_length}, {max_length}, {length_ratio}") + filter_pairs( + src_edges, + tgt_edges, + input_src, + input_tgt, + filtered_dir_src, + filtered_dir_tgt, + removed_dir_src, + removed_dir_tgt, + min_length, + max_length, + length_ratio, + rank, + ) + + +def _cat_results(out_file, tmp_dir): + file_name_pattern = re.compile(r"/rank([1-9][0-9]*)|0$") + with out_file.open('w') as out_f: + for f in sorted(tmp_dir.iterdir()): + if not f.is_file(): + warnings.warn(f"Unexpected not file {f}") + elif not file_name_pattern.search(str(f)): + warnings.warn(f"Unexpected file {f}") + else: + with f.open('r') as in_f: + for l in in_f: + out_f.write(l) + + +def cat_results(out_files, tmp_dirs): + for o_f, t_d in zip(out_files, tmp_dirs): + if o_f is None or t_d is None: + if o_f is not None or t_d is not None: + warnings.warn( + f"Output file and tmp directory are expected to be `None` simultaneously whereas tmp directory " + f"is {t_d} and output file is {o_f}." + ) + else: + _cat_results(o_f, t_d) + + +counter = None + + +def init(args): + global counter + counter = args + + +def main(): + args = get_args() + tmp_dir = Path("tmp") + i = 0 + while tmp_dir.exists(): + tmp_dir = Path("tmp" + str(i)) + i += 1 + tmp_filtered = tmp_dir / Path("filtered") + tmp_filtered_src = tmp_filtered / Path("src") + tmp_filtered_src.mkdir(parents=True, exist_ok=True) + if args.input_tgt is None: + tmp_filtered_tgt = None + else: + tmp_filtered_tgt = tmp_filtered / Path("tgt") + tmp_filtered_tgt.mkdir(parents=True, exist_ok=True) + tmp_removed = tmp_dir / Path("removed") + tmp_removed_src = tmp_removed / Path("src") + tmp_removed_src.mkdir(parents=True, exist_ok=True) + if args.input_tgt is None: + tmp_removed_tgt = None + else: + tmp_removed_tgt = tmp_removed / Path("tgt") + tmp_removed_tgt.mkdir(parents=True, exist_ok=True) + num_jobs = mp.cpu_count() if args.num_jobs is None else args.num_jobs + src_edges, tgt_edges, num_lines = get_edges_and_num_lines(args.input_src, args.input_tgt, num_jobs) + global counter + counter = mp.Value('i', 0) + t = tqdm(total=num_lines, desc="Length Ratio Filtering") + with mp.Pool(num_jobs, initializer=init, initargs=(counter,)) as pool: + async_result = pool.map_async( + filter_by_length_and_ratio, + [ + ( + se, + te, + args.input_src, + args.input_tgt, + tmp_filtered_src, + tmp_filtered_tgt, + tmp_removed_src, + tmp_removed_tgt, + args.min_length, + args.max_length, + args.ratio, + rank, + ) + for rank, (se, te) in enumerate(zip(src_edges, tgt_edges)) + ], + ) + while not async_result.ready(): + t.update(counter.value) + with counter.get_lock(): + counter.value = 0 + sleep(0.1) + t.update(counter.value) + + cat_results( + [args.output_src, args.output_tgt, args.removed_src, args.removed_tgt], + [tmp_filtered_src, tmp_filtered_tgt, tmp_removed_src, tmp_removed_tgt], + ) + shutil.rmtree(tmp_dir) + + +if __name__ == "__main__": + main() From fea3775c00adfacfe0a414dea15544abc96db8dc Mon Sep 17 00:00:00 2001 From: Abhinav Khattar Date: Fri, 15 Jul 2022 20:41:36 -0700 Subject: [PATCH 33/52] Add Tokenization and Normalization pre-proecssing script for NMT (#4557) * add script Signed-off-by: Abhinav Khattar * style fix Signed-off-by: Abhinav Khattar --- .../filter_langs_nmt.py | 2 +- .../preprocess_tokenization_normalization.py | 66 +++++++++++++++++++ 2 files changed, 67 insertions(+), 1 deletion(-) create mode 100644 scripts/neural_machine_translation/preprocess_tokenization_normalization.py diff --git a/scripts/neural_machine_translation/filter_langs_nmt.py b/scripts/neural_machine_translation/filter_langs_nmt.py index 3e88c6106bd4..c7399c223098 100644 --- a/scripts/neural_machine_translation/filter_langs_nmt.py +++ b/scripts/neural_machine_translation/filter_langs_nmt.py @@ -26,7 +26,7 @@ """ Usage: -python filter_by_language.py --input-src train.en \ +python filter_langs_nmt.py --input-src train.en \ --input-tgt train.de \ --output-src train_lang_filtered.en \ --output-tgt train_lang_filtered.de \ diff --git a/scripts/neural_machine_translation/preprocess_tokenization_normalization.py b/scripts/neural_machine_translation/preprocess_tokenization_normalization.py new file mode 100644 index 000000000000..8b8922bc319f --- /dev/null +++ b/scripts/neural_machine_translation/preprocess_tokenization_normalization.py @@ -0,0 +1,66 @@ +# Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging +from argparse import ArgumentParser + +from nemo.collections.nlp.models.machine_translation.mt_enc_dec_model import MTEncDecModel + +""" +python preprocess_tokenization_normalization.py --input-src train.en \ + --input-tgt train.zh \ + --output-src train.tok.norm.en \ + --output-tgt train.tok.norm.zh \ + --source-lang en \ + --target-lang zh +""" + +logging.basicConfig(level=logging.INFO) + + +def tokenize_normalize(file, wfile, processor): + rptr = open(file) + wptr = open(wfile, 'w') + logging.info(f"Processing {file}") + for line in rptr: + txt = line.strip() + if processor is not None: + txt = processor.normalize(txt) + txt = processor.tokenize(txt) + wptr.write(txt + "\n") + logging.info(f"Output written to {file}") + rptr.close() + wptr.close() + + +def main(): + parser = ArgumentParser() + parser.add_argument("--input-src", type=str, required=True, help="Path to input file in src language") + parser.add_argument("--input-tgt", type=str, required=True, help="Path to input file in tgt language") + parser.add_argument("--output-src", type=str, required=True, help="Path to write the src language output file") + parser.add_argument("--output-tgt", type=str, required=True, help="Path to write the tgt language output file") + parser.add_argument("--source-lang", type=str, required=True, help="Language for the source file") + parser.add_argument("--target-lang", type=str, required=True, help="Language for the target file") + + args = parser.parse_args() + + src_processor, tgt_processor = MTEncDecModel.setup_pre_and_post_processing_utils( + args.source_lang, args.target_lang, "bpe-placeholder", "bpe-placeholder" + ) + tokenize_normalize(args.input_src, args.output_src, src_processor) + tokenize_normalize(args.input_tgt, args.output_tgt, tgt_processor) + + +if __name__ == '__main__': + main() From 56694f0d5ebd31d790f07f31334f10795023dfdc Mon Sep 17 00:00:00 2001 From: Paarth Neekhara Date: Sun, 17 Jul 2022 14:47:21 -0400 Subject: [PATCH 34/52] handled n segments for a different sampling rate than original sampling rate Signed-off-by: Paarth Neekhara --- .../asr/parts/preprocessing/segment.py | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/nemo/collections/asr/parts/preprocessing/segment.py b/nemo/collections/asr/parts/preprocessing/segment.py index af0589309399..d227b217a410 100644 --- a/nemo/collections/asr/parts/preprocessing/segment.py +++ b/nemo/collections/asr/parts/preprocessing/segment.py @@ -33,6 +33,7 @@ # SOFTWARE. # This file contains code artifacts adapted from https://github.com/ryanleary/patter +import math import os import random @@ -186,11 +187,16 @@ def segment_from_file(cls, audio_file, target_sr=None, n_segments=0, trim=False, try: with sf.SoundFile(audio_file, 'r') as f: sample_rate = f.samplerate - if 0 < n_segments < len(f): - max_audio_start = len(f) - n_segments + if target_sr is not None: + n_segments_at_original_sr = math.ceil(n_segments * sample_rate / target_sr) + else: + n_segments_at_original_sr = n_segments + + if 0 < n_segments_at_original_sr < len(f): + max_audio_start = len(f) - n_segments_at_original_sr audio_start = random.randint(0, max_audio_start) f.seek(audio_start) - samples = f.read(n_segments, dtype='float32') + samples = f.read(n_segments_at_original_sr, dtype='float32') else: samples = f.read(dtype='float32') samples = samples.transpose() @@ -198,7 +204,10 @@ def segment_from_file(cls, audio_file, target_sr=None, n_segments=0, trim=False, logging.error(f"Loading {audio_file} via SoundFile raised RuntimeError: `{e}`.") samples = samples.transpose() - return cls(samples, sample_rate, target_sr=target_sr, trim=trim, orig_sr=orig_sr) + features = cls(samples, sample_rate, target_sr=target_sr, trim=trim, orig_sr=orig_sr) + features._samples = features._samples[:n_segments] + + return features @property def samples(self): From 85fd5a95afb42c151e6ce6f7da22a11388d89822 Mon Sep 17 00:00:00 2001 From: Paarth Neekhara Date: Wed, 20 Jul 2022 14:56:18 -0400 Subject: [PATCH 35/52] Added case for n_segments 0, warning for n_segments greater than file length Signed-off-by: Paarth Neekhara --- nemo/collections/asr/parts/preprocessing/segment.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/nemo/collections/asr/parts/preprocessing/segment.py b/nemo/collections/asr/parts/preprocessing/segment.py index d227b217a410..741308b6d171 100644 --- a/nemo/collections/asr/parts/preprocessing/segment.py +++ b/nemo/collections/asr/parts/preprocessing/segment.py @@ -184,6 +184,7 @@ def segment_from_file(cls, audio_file, target_sr=None, n_segments=0, trim=False, Note that audio_file can be either the file path, or a file-like object. """ + is_segmented = False try: with sf.SoundFile(audio_file, 'r') as f: sample_rate = f.samplerate @@ -197,6 +198,12 @@ def segment_from_file(cls, audio_file, target_sr=None, n_segments=0, trim=False, audio_start = random.randint(0, max_audio_start) f.seek(audio_start) samples = f.read(n_segments_at_original_sr, dtype='float32') + is_segmented = True + elif n_segments_at_original_sr >= len(f): + logging.warning( + f"Number of segments is greater than the length of the audio file {audio_file}. This may lead to shape mismatch errors." + ) + samples = f.read(dtype='float32') else: samples = f.read(dtype='float32') samples = samples.transpose() @@ -205,7 +212,9 @@ def segment_from_file(cls, audio_file, target_sr=None, n_segments=0, trim=False, samples = samples.transpose() features = cls(samples, sample_rate, target_sr=target_sr, trim=trim, orig_sr=orig_sr) - features._samples = features._samples[:n_segments] + + if is_segmented: + features._samples = features._samples[:n_segments] return features From 86fea2a1b8298c70200a1b5ce50125885aee555a Mon Sep 17 00:00:00 2001 From: anteju <108555623+anteju@users.noreply.github.com> Date: Thu, 21 Jul 2022 09:44:20 -0700 Subject: [PATCH 36/52] [Fix] Relative audio path in speech data explorer (#4570) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Ante Jukić Co-authored-by: Ante Jukić --- tools/speech_data_explorer/data_explorer.py | 38 +++++++++++++++++---- 1 file changed, 32 insertions(+), 6 deletions(-) diff --git a/tools/speech_data_explorer/data_explorer.py b/tools/speech_data_explorer/data_explorer.py index a32bffbc69bd..6a43dcdd2bbc 100755 --- a/tools/speech_data_explorer/data_explorer.py +++ b/tools/speech_data_explorer/data_explorer.py @@ -24,6 +24,8 @@ import os import pickle from collections import defaultdict +from os.path import expanduser +from pathlib import Path import dash import dash_bootstrap_components as dbc @@ -143,7 +145,8 @@ def load_data(data_filename, disable_caching=False, estimate_audio=False, vocab= item['OOV'] = item['word'] not in vocabulary_ext if estimate_audio: for item in data: - signal, sr = librosa.load(path=item['audio_filepath'], sr=None) + filepath = absolute_audio_filepath(item['audio_filepath'], data_filename) + signal, sr = librosa.load(path=filepath, sr=None) bw = eval_bandwidth(signal, sr) item['freq_bandwidth'] = int(bw) item['level_db'] = 20 * np.log10(np.max(np.abs(signal))) @@ -228,7 +231,8 @@ def load_data(data_filename, disable_caching=False, estimate_audio=False, vocab= data[-1]['D-I'] = measures['deletions'] - measures['insertions'] if estimate_audio: - signal, sr = librosa.load(path=item['audio_filepath'], sr=None) + filepath = absolute_audio_filepath(item['audio_filepath'], data_filename) + signal, sr = librosa.load(path=filepath, sr=None) bw = eval_bandwidth(signal, sr) item['freq_bandwidth'] = int(bw) item['level_db'] = 20 * np.log10(np.max(np.abs(signal))) @@ -312,6 +316,28 @@ def plot_word_accuracy(vocabulary_data): return fig +def absolute_audio_filepath(audio_filepath, manifest_path): + """Return absolute path to an audio file. + + Check if a file existst at audio_filepath. + If not, assume that the path is relative to the directory where manifest is stored. + """ + audio_filepath = Path(audio_filepath) + + if not audio_filepath.is_file() and not audio_filepath.is_absolute(): + # assume audio_filepath is relative to the directory where the manifest is stored + manifest_dir = Path(args.manifest).parent + audio_filepath = manifest_dir / audio_filepath + if audio_filepath.is_file(): + filename = str(audio_filepath) + else: + filename = expanduser(audio_filepath) + else: + filename = expanduser(audio_filepath) + + return filename + + args = parse_args() print('Loading data...') data, wer, cer, wmr, mwa, num_hours, vocabulary, alphabet, metrics_available = load_data( @@ -736,7 +762,7 @@ def plot_signal(idx, data): raise PreventUpdate figs = make_subplots(rows=2, cols=1, subplot_titles=('Waveform', 'Spectrogram')) try: - filename = data[idx[0]]['audio_filepath'] + filename = absolute_audio_filepath(data[idx[0]]['audio_filepath'], args.manifest) audio, fs = librosa.load(path=filename, sr=None) if 'offset' in data[idx[0]]: audio = audio[ @@ -777,8 +803,8 @@ def plot_signal(idx, data): figs.update_yaxes(title_text='Amplitude', row=1, col=1) figs.update_xaxes(title_text='Time, s', row=2, col=1) figs.update_yaxes(title_text='Frequency, kHz', row=2, col=1) - except Exception: - pass + except Exception as ex: + app.logger.error(f'ERROR in plot signal: {ex}') return figs @@ -788,7 +814,7 @@ def update_player(idx, data): if len(idx) == 0: raise PreventUpdate try: - filename = data[idx[0]]['audio_filepath'] + filename = absolute_audio_filepath(data[idx[0]]['audio_filepath'], args.manifest) signal, sr = librosa.load(path=filename, sr=None) if 'offset' in data[idx[0]]: signal = signal[ From e67c4ca29a3855d1575f173af5b38ed3a9a91e68 Mon Sep 17 00:00:00 2001 From: "He Huang (Steve)" <105218074+stevehuang52@users.noreply.github.com> Date: Thu, 21 Jul 2022 14:25:42 -0400 Subject: [PATCH 37/52] [Add] Catalan ASR NGC Resource (#4576) * add ngc catalan model resource Signed-off-by: stevehuang52 * update docs Signed-off-by: stevehuang52 --- docs/source/asr/data/benchmark_ca.csv | 2 ++ docs/source/asr/data/scores/ca/conformer_ca.csv | 3 +++ docs/source/asr/scores.rst | 7 +++++++ nemo/collections/asr/models/ctc_bpe_models.py | 7 +++++++ nemo/collections/asr/models/rnnt_bpe_models.py | 7 +++++++ 5 files changed, 26 insertions(+) create mode 100644 docs/source/asr/data/scores/ca/conformer_ca.csv diff --git a/docs/source/asr/data/benchmark_ca.csv b/docs/source/asr/data/benchmark_ca.csv index ef5220fe6e06..bd7e174b922f 100644 --- a/docs/source/asr/data/benchmark_ca.csv +++ b/docs/source/asr/data/benchmark_ca.csv @@ -1,2 +1,4 @@ Model,Model Base Class,Model Card stt_ca_quartznet15x5,EncDecCTCModel,"https://ngc.nvidia.com/catalog/models/nvidia:nemo:stt_ca_quartznet15x5" +stt_ca_conformer_ctc_large,EncDecCTCModel,"https://ngc.nvidia.com/catalog/models/nvidia:nemo:stt_ca_conformer_ctc_large" +stt_ca_conformer_transducer_large,EncDecRNNTBPEModel,"https://ngc.nvidia.com/catalog/models/nvidia:nemo:stt_ca_conformer_transducer_large" \ No newline at end of file diff --git a/docs/source/asr/data/scores/ca/conformer_ca.csv b/docs/source/asr/data/scores/ca/conformer_ca.csv new file mode 100644 index 000000000000..a9c139354738 --- /dev/null +++ b/docs/source/asr/data/scores/ca/conformer_ca.csv @@ -0,0 +1,3 @@ +Model Name,Language,MCV Test-Set v9.0 (ca) +stt_ca_conformer_ctc_large,ca,4.27 +stt_ca_conformer_transducer_large,ca,3.85 \ No newline at end of file diff --git a/docs/source/asr/scores.rst b/docs/source/asr/scores.rst index 9eef95fa92b5..77e9e5b09531 100644 --- a/docs/source/asr/scores.rst +++ b/docs/source/asr/scores.rst @@ -52,6 +52,13 @@ CA -------------------- +.. csv-table:: + :header-rows: 1 + :align: left + :file: data/scores/ca/conformer_ca.csv + +-------------------- + DE ^^ diff --git a/nemo/collections/asr/models/ctc_bpe_models.py b/nemo/collections/asr/models/ctc_bpe_models.py index 15e91cc76e9e..ce92823cba6f 100644 --- a/nemo/collections/asr/models/ctc_bpe_models.py +++ b/nemo/collections/asr/models/ctc_bpe_models.py @@ -520,4 +520,11 @@ def list_available_models(cls) -> Optional[PretrainedModelInfo]: ) results.append(model) + model = PretrainedModelInfo( + pretrained_model_name="stt_ca_conformer_ctc_large", + description="For details about this model, please visit https://ngc.nvidia.com/catalog/models/nvidia:nemo:stt_ca_conformer_ctc_large", + location="https://api.ngc.nvidia.com/v2/models/nvidia/nemo/stt_ca_conformer_ctc_large/versions/1.11.0/files/stt_ca_conformer_ctc_large.nemo", + ) + results.append(model) + return results diff --git a/nemo/collections/asr/models/rnnt_bpe_models.py b/nemo/collections/asr/models/rnnt_bpe_models.py index 2540bd941aea..199fc0304f0c 100644 --- a/nemo/collections/asr/models/rnnt_bpe_models.py +++ b/nemo/collections/asr/models/rnnt_bpe_models.py @@ -183,6 +183,13 @@ def list_available_models(cls) -> List[PretrainedModelInfo]: ) results.append(model) + model = PretrainedModelInfo( + pretrained_model_name="stt_ca_conformer_transducer_large", + description="For details about this model, please visit https://ngc.nvidia.com/catalog/models/nvidia:nemo:stt_ca_conformer_transducer_large", + location="https://api.ngc.nvidia.com/v2/models/nvidia/nemo/stt_ca_conformer_transducer_large/versions/1.11.0/files/stt_ca_conformer_transducer_large.nemo", + ) + results.append(model) + return results def __init__(self, cfg: DictConfig, trainer: Trainer = None): From 6442e339a47d30a106d869d1ef29cc1294753b75 Mon Sep 17 00:00:00 2001 From: Sandeep Subramanian Date: Fri, 22 Jul 2022 15:16:53 -0700 Subject: [PATCH 38/52] Option to disregard document boundaries for t5, bart, ul2 (#4481) * fix Signed-off-by: MaximumEntropy * Change max sequence length computation Signed-off-by: MaximumEntropy * Add args and refactor to support bart, ul2 Signed-off-by: MaximumEntropy * Style Signed-off-by: MaximumEntropy * Add to CI test Signed-off-by: MaximumEntropy * Comments Signed-off-by: MaximumEntropy * Attempt fix Signed-off-by: MaximumEntropy * Style Signed-off-by: MaximumEntropy * Style fix Signed-off-by: MaximumEntropy * Revert max seq length change Signed-off-by: MaximumEntropy * Fix syntax error Signed-off-by: MaximumEntropy * Revert BART change Signed-off-by: MaximumEntropy * Remove comment Signed-off-by: MaximumEntropy * fix Signed-off-by: MaximumEntropy * Remove unused import Signed-off-by: MaximumEntropy --- Jenkinsfile | 4 + .../conf/megatron_bart_config.yaml | 1 + .../conf/megatron_t5_config.yaml | 1 + .../conf/megatron_ul2_config.yaml | 1 + .../megatron/bart_dataset.py | 6 +- .../megatron/dataset_utils.py | 13 +++ .../language_modeling/megatron/t5_dataset.py | 92 +++++++++++++++---- .../language_modeling/megatron/ul2_dataset.py | 9 +- .../megatron_lm_encoder_decoder_model.py | 1 - .../language_modeling/megatron_t5_model.py | 1 + 10 files changed, 104 insertions(+), 25 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 1092c6745eb8..e60805da0a0b 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -2947,6 +2947,7 @@ pipeline { model.transformer_block_type='pre_ln' \ model.data.data_prefix=[.5,/home/TestData/nlp/megatron_t5/data/pile_val_small_bert_tokenizer_text_document,.5,/home/TestData/nlp/megatron_t5/data/pile_val_small_bert_tokenizer_text_document] \ model.position_embedding_type=relative \ + model.data.respect_document_boundaries=False \ model.data.index_mapping_dir=examples/nlp/language_modeling/t5_index_mappings" sh "python examples/nlp/language_modeling/megatron_t5_pretraining.py \ trainer.devices=2 \ @@ -2972,6 +2973,7 @@ pipeline { model.transformer_block_type='pre_ln' \ model.data.data_prefix=[.5,/home/TestData/nlp/megatron_t5/data/pile_val_small_bert_tokenizer_text_document,.5,/home/TestData/nlp/megatron_t5/data/pile_val_small_bert_tokenizer_text_document] \ model.position_embedding_type=relative \ + model.data.respect_document_boundaries=False \ model.data.index_mapping_dir=examples/nlp/language_modeling/t5_index_mappings" sh "rm -rf examples/nlp/language_modeling/t5_pretrain_results" sh "rm -rf examples/nlp/language_modeling/t5_index_mappings" @@ -3201,6 +3203,7 @@ pipeline { model.bias_activation_fusion=False \ model.activations_checkpoint_method='block' \ model.activations_checkpoint_num_layers=1 \ + model.data.respect_document_boundaries=False \ model.data.data_prefix=[.5,/home/TestData/nlp/megatron_t5/data/pile_val_small_bert_tokenizer_text_document,.5,/home/TestData/nlp/megatron_t5/data/pile_val_small_bert_tokenizer_text_document]" sh "python examples/nlp/language_modeling/megatron_bart_pretraining.py \ trainer.devices=2 \ @@ -3224,6 +3227,7 @@ pipeline { model.bias_activation_fusion=False \ model.activations_checkpoint_method='block' \ model.activations_checkpoint_num_layers=1 \ + model.data.respect_document_boundaries=False \ model.data.data_prefix=[.5,/home/TestData/nlp/megatron_t5/data/pile_val_small_bert_tokenizer_text_document,.5,/home/TestData/nlp/megatron_t5/data/pile_val_small_bert_tokenizer_text_document]" sh "rm -rf examples/nlp/language_modeling/bart_pretrain_results" } diff --git a/examples/nlp/language_modeling/conf/megatron_bart_config.yaml b/examples/nlp/language_modeling/conf/megatron_bart_config.yaml index e8c9adbd0bf9..1b3822dee75d 100644 --- a/examples/nlp/language_modeling/conf/megatron_bart_config.yaml +++ b/examples/nlp/language_modeling/conf/megatron_bart_config.yaml @@ -135,6 +135,7 @@ model: whole_word_masking: True favor_longer_ngrams: False delete_mask_prob: 0.3 + respect_document_boundaries: True # If true, a single training exampl cannot cross document boundaries, increasing the fraction of tokens within a batch. optim: name: fused_adam diff --git a/examples/nlp/language_modeling/conf/megatron_t5_config.yaml b/examples/nlp/language_modeling/conf/megatron_t5_config.yaml index d0adffa516b7..363b5afdbad7 100644 --- a/examples/nlp/language_modeling/conf/megatron_t5_config.yaml +++ b/examples/nlp/language_modeling/conf/megatron_t5_config.yaml @@ -137,6 +137,7 @@ model: permutation: False whole_word_masking: True favor_longer_ngrams: False + respect_document_boundaries: True # If true, a single training exampl cannot cross document boundaries, increasing the fraction of tokens within a batch. optim: name: fused_adam diff --git a/examples/nlp/language_modeling/conf/megatron_ul2_config.yaml b/examples/nlp/language_modeling/conf/megatron_ul2_config.yaml index db7944693b0c..c0c65a714252 100644 --- a/examples/nlp/language_modeling/conf/megatron_ul2_config.yaml +++ b/examples/nlp/language_modeling/conf/megatron_ul2_config.yaml @@ -140,6 +140,7 @@ model: permutation: False whole_word_masking: True favor_longer_ngrams: False + respect_document_boundaries: True # If true, a single training exampl cannot cross document boundaries, increasing the fraction of tokens within a batch. optim: name: fused_adam diff --git a/nemo/collections/nlp/data/language_modeling/megatron/bart_dataset.py b/nemo/collections/nlp/data/language_modeling/megatron/bart_dataset.py index c0169ffd3cb1..7876ba59adce 100644 --- a/nemo/collections/nlp/data/language_modeling/megatron/bart_dataset.py +++ b/nemo/collections/nlp/data/language_modeling/megatron/bart_dataset.py @@ -21,7 +21,7 @@ class BARTDataset(T5Dataset): # account for added tokens - MAX_SEQ_LENGTH_DELTA = 1 + MAX_SEQ_LENGTH_DELTA = 2 def __init__( self, @@ -44,6 +44,8 @@ def __init__( whole_word_masking=True, favor_long_ngrams=False, delete_mask_prob=0, + respect_document_boundaries=True, + documents=None, ): super().__init__( cfg=cfg, @@ -65,6 +67,8 @@ def __init__( permutation=permutation, whole_word_masking=whole_word_masking, favor_long_ngrams=favor_long_ngrams, + respect_document_boundaries=respect_document_boundaries, + documents=documents, ) # Params to store. diff --git a/nemo/collections/nlp/data/language_modeling/megatron/dataset_utils.py b/nemo/collections/nlp/data/language_modeling/megatron/dataset_utils.py index 722802db0152..eb5affe7b3b4 100644 --- a/nemo/collections/nlp/data/language_modeling/megatron/dataset_utils.py +++ b/nemo/collections/nlp/data/language_modeling/megatron/dataset_utils.py @@ -552,6 +552,7 @@ def build_train_valid_test_datasets( whole_word_masking=True, favor_long_ngrams=False, delete_mask_prob=0, + respect_document_boundaries=True, ): if len(data_prefix) == 1: @@ -578,6 +579,7 @@ def build_train_valid_test_datasets( whole_word_masking=whole_word_masking, favor_long_ngrams=favor_long_ngrams, delete_mask_prob=delete_mask_prob, + respect_document_boundaries=respect_document_boundaries, ) # Blending dataset. # Parse the values. @@ -612,6 +614,7 @@ def build_train_valid_test_datasets( whole_word_masking=whole_word_masking, favor_long_ngrams=favor_long_ngrams, delete_mask_prob=delete_mask_prob, + respect_document_boundaries=respect_document_boundaries, ) if train_ds: train_datasets.append(train_ds) @@ -657,6 +660,7 @@ def _build_train_valid_test_datasets( whole_word_masking=True, favor_long_ngrams=False, delete_mask_prob=0, # This flag is used in BART only, and will not have effect on T5/BERT + respect_document_boundaries=True, ): if dataset_type not in DSET_TYPES: @@ -737,6 +741,7 @@ def build_dataset(index, name): elif dataset_type == DSET_TYPE_T5: assert tokenizer is not None, "Tokenizer is required for T5 dataset" logging.info("Instatiating T5 Dataset ...") + documents = np.arange(start=splits[index], stop=splits[index + 1], step=1, dtype=np.int32) dataset = T5Dataset( cfg=cfg, trainer=trainer, @@ -751,6 +756,8 @@ def build_dataset(index, name): permutation=permutation, whole_word_masking=whole_word_masking, favor_long_ngrams=favor_long_ngrams, + documents=documents, + respect_document_boundaries=respect_document_boundaries, **kwargs, ) elif dataset_type == DSET_TYPE_BERT: @@ -780,6 +787,7 @@ def build_dataset(index, name): ) elif dataset_type == DSET_TYPE_BART: assert tokenizer is not None, "Tokenizer is required for BART dataset" + documents = np.arange(start=splits[index], stop=splits[index + 1], step=1, dtype=np.int32) logging.info("Instatiating BART Dataset ...") dataset = BARTDataset( cfg=cfg, @@ -795,10 +803,13 @@ def build_dataset(index, name): whole_word_masking=whole_word_masking, favor_long_ngrams=favor_long_ngrams, delete_mask_prob=delete_mask_prob, + documents=documents, + respect_document_boundaries=respect_document_boundaries, **kwargs, ) elif dataset_type == DSET_TYPE_UL2: assert tokenizer is not None, "Tokenizer is required for UL2 dataset" + documents = np.arange(start=splits[index], stop=splits[index + 1], step=1, dtype=np.int32) logging.info("Instatiating UL2 Dataset ...") extreme_ngram_span_length_distribution = cfg.data.get( "extreme_ngram_span_length_distribution", "truncated_normal" @@ -838,6 +849,8 @@ def build_dataset(index, name): extreme_mean_ngram_size=cfg.data.get("extreme_mean_ngram_size", 64), extreme_min_ngram_size=cfg.data.get("extreme_min_ngram_size", 32), prefix_lm_pivot_mean=cfg.data.get("prefix_lm_pivot_mean", 0.25), + respect_document_boundaries=respect_document_boundaries, + documents=documents, **kwargs, ) else: diff --git a/nemo/collections/nlp/data/language_modeling/megatron/t5_dataset.py b/nemo/collections/nlp/data/language_modeling/megatron/t5_dataset.py index bcb080645666..1ed36de1a010 100644 --- a/nemo/collections/nlp/data/language_modeling/megatron/t5_dataset.py +++ b/nemo/collections/nlp/data/language_modeling/megatron/t5_dataset.py @@ -25,6 +25,7 @@ create_masked_lm_predictions, get_samples_mapping, ) +from nemo.collections.nlp.data.language_modeling.megatron.gpt_dataset import _build_index_mappings from nemo.core import Dataset @@ -53,6 +54,8 @@ def __init__( permutation=False, whole_word_masking=True, favor_long_ngrams=False, + respect_document_boundaries=True, + documents=None, ): super().__init__() @@ -69,6 +72,7 @@ def __init__( self.permutation = permutation self.whole_word_masking = whole_word_masking self.favor_long_ngrams = favor_long_ngrams + self.respect_document_boundaries = respect_document_boundaries # Dataset. self.indexed_dataset = indexed_dataset @@ -84,18 +88,35 @@ def __init__( torch.distributed.barrier() # Build the samples mapping. - self.samples_mapping = get_samples_mapping( - indexed_dataset=self.indexed_dataset, - data_prefix=data_prefix, - num_epochs=num_epochs, - max_num_samples=max_num_samples, - max_seq_length=self.max_seq_length - self.MAX_SEQ_LENGTH_DELTA, # account for added tokens - short_seq_prob=self.short_seq_prob, - seed=self.seed, - name=self.name, - binary_head=False, - index_mapping_dir=self.index_mapping_dir, - ) + if not respect_document_boundaries: + # Build index mappings. + assert documents is not None + assert np.min(documents) >= 0 + assert np.max(documents) < indexed_dataset.sizes.shape[0] + + self.doc_idx, self.sample_idx, self.shuffle_idx = _build_index_mappings( + name=self.name, + data_prefix=data_prefix, + documents=documents, + sizes=self.indexed_dataset.sizes, + num_samples=max_num_samples, + seq_length=self.max_seq_length - self.MAX_SEQ_LENGTH_DELTA, + seed=self.seed, + index_mapping_dir=self.index_mapping_dir, + ) + else: + self.samples_mapping = get_samples_mapping( + indexed_dataset=self.indexed_dataset, + data_prefix=data_prefix, + num_epochs=num_epochs, + max_num_samples=max_num_samples, + max_seq_length=self.max_seq_length - self.MAX_SEQ_LENGTH_DELTA, # account for added tokens + short_seq_prob=self.short_seq_prob, + seed=self.seed, + name=self.name, + binary_head=False, + index_mapping_dir=self.index_mapping_dir, + ) self.tokenizer = tokenizer self.tokenizer_type = 'wordpiece' # TODO: better checks for tokenizer types. How do we do this for HF tokenizers that are not BERT? @@ -131,14 +152,47 @@ def _build(self): assert len(self.sentinel_tokens) > 0 def __len__(self): - return self.samples_mapping.shape[0] + if self.respect_document_boundaries: + return self.samples_mapping.shape[0] + else: + return self.sample_idx.shape[0] - 1 + + def _get_sample(self, idx): + if self.respect_document_boundaries: + start_index, end_index, seq_length = self.samples_mapping[idx] + sample = [] + for index in range(start_index, end_index): + sample.append(self.indexed_dataset[index]) + else: + # Get the shuffled index. + idx = self.shuffle_idx[idx] + # Start and end documents and offsets. + doc_index_f = self.sample_idx[idx][0] + doc_index_l = self.sample_idx[idx + 1][0] + offset_f = self.sample_idx[idx][1] + offset_l = self.sample_idx[idx + 1][1] + # If we are within the same document, just extract the chunk. + if doc_index_f == doc_index_l: + sample = self.indexed_dataset.get( + self.doc_idx[doc_index_f], offset=offset_f, length=offset_l - offset_f + 1 + ) + else: + # Otherwise, get the rest of the initial document. + sample_list = [self.indexed_dataset.get(self.doc_idx[doc_index_f], offset=offset_f)] + # Loop over all in between documents and add the entire document. + for i in range(doc_index_f + 1, doc_index_l): + sample_list.append(self.indexed_dataset.get(self.doc_idx[i])) + # And finally add the relevant portion of last document. + sample_list.append(self.indexed_dataset.get(self.doc_idx[doc_index_l], length=offset_l + 1)) + sample = np.concatenate(sample_list) + sample.astype(np.int64) + seq_length = len(sample) + sample = [sample] + + return sample, seq_length def __getitem__(self, idx): - - start_index, end_index, seq_length = self.samples_mapping[idx] - sample = [] - for index in range(start_index, end_index): - sample.append(self.indexed_dataset[index]) + sample, seq_length = self._get_sample(idx) # Note that this rng state should be numpy and not python since # python randint is inclusive whereas the numpy one is exclusive. np_rng = np.random.RandomState(seed=(self.seed + idx)) @@ -175,7 +229,7 @@ def build_training_sample( whole_word_masking: Always masks entire words instead of individual sub-word tokens. favor_long_ngrams: Favor longer ngrams over shorter ones. """ - assert target_seq_length <= self.max_seq_length + # assert target_seq_length <= self.max_seq_length # flatten sentences into one list tokens = [token for sentence in sample for token in sentence] diff --git a/nemo/collections/nlp/data/language_modeling/megatron/ul2_dataset.py b/nemo/collections/nlp/data/language_modeling/megatron/ul2_dataset.py index fc595a9c0283..7048dd36cc4a 100644 --- a/nemo/collections/nlp/data/language_modeling/megatron/ul2_dataset.py +++ b/nemo/collections/nlp/data/language_modeling/megatron/ul2_dataset.py @@ -57,6 +57,8 @@ def __init__( permutation=False, whole_word_masking=True, favor_long_ngrams=False, + respect_document_boundaries=True, + documents=None, ): super().__init__( cfg=cfg, @@ -78,6 +80,8 @@ def __init__( permutation=permutation, whole_word_masking=whole_word_masking, favor_long_ngrams=favor_long_ngrams, + respect_document_boundaries=respect_document_boundaries, + documents=documents, ) self.mean_ngram_size = mean_ngram_size self.min_ngram_size = min_ngram_size @@ -90,10 +94,7 @@ def __init__( self.prefix_lm_pivot_mean = prefix_lm_pivot_mean def __getitem__(self, idx): - start_index, end_index, seq_length = self.samples_mapping[idx] - sample = [] - for index in range(start_index, end_index): - sample.append(self.indexed_dataset[index]) + sample, seq_length = self._get_sample(idx) # Note that this rng state should be numpy and not python since # python randint is inclusive whereas the numpy one is exclusive. np_rng = np.random.RandomState(seed=(self.seed + idx)) diff --git a/nemo/collections/nlp/models/language_modeling/megatron_lm_encoder_decoder_model.py b/nemo/collections/nlp/models/language_modeling/megatron_lm_encoder_decoder_model.py index c3ab20ecccbe..6c4e52caf6c7 100644 --- a/nemo/collections/nlp/models/language_modeling/megatron_lm_encoder_decoder_model.py +++ b/nemo/collections/nlp/models/language_modeling/megatron_lm_encoder_decoder_model.py @@ -464,7 +464,6 @@ def validation_step(self, batch, batch_idx): batch_for_pipeline = self.process_global_batch(batch) encoder_seq_length = batch_for_pipeline[0].size(1) decoder_seq_length = batch_for_pipeline[1].size(1) - tensor_shape = [encoder_seq_length, get_micro_batch_size(), self.cfg.hidden_size] if self.cfg.get('pipeline_model_parallel_size', 1) > 1: diff --git a/nemo/collections/nlp/models/language_modeling/megatron_t5_model.py b/nemo/collections/nlp/models/language_modeling/megatron_t5_model.py index 5b38810f9f3b..a983f0b785b8 100644 --- a/nemo/collections/nlp/models/language_modeling/megatron_t5_model.py +++ b/nemo/collections/nlp/models/language_modeling/megatron_t5_model.py @@ -176,6 +176,7 @@ def build_train_valid_test_datasets(self): permutation=self._cfg.data.get('permutation', False), whole_word_masking=self._cfg.data.get('whole_word_masking', True), favor_long_ngrams=self._cfg.data.get('favor_long_ngrams', False), + respect_document_boundaries=self._cfg.data.get('respect_document_boundaries', True), # additional arguments from child classes **self._build_train_valid_test_datasets_kwargs, ) From 6b9617daa5094c43011b20c21ae0928f4e8c49f2 Mon Sep 17 00:00:00 2001 From: Ameya Mahabaleshwarkar <34514696+ameyasm1154@users.noreply.github.com> Date: Mon, 25 Jul 2022 12:33:31 -0400 Subject: [PATCH 39/52] Integrating support for GPT/T5/BART for Question Answering (#4532) * added class for qa related metrics Signed-off-by: Ameya Mahabaleshwarkar * removed BLEU code from QA metrics Signed-off-by: Ameya Mahabaleshwarkar * added classes for data handling and loading for BERT/T5/BART/GPT Signed-off-by: Ameya Mahabaleshwarkar * removed unnecassary main function Signed-off-by: Ameya Mahabaleshwarkar * added classes for BERT, S2S(T5/BART), GPT question answering models Signed-off-by: Ameya Mahabaleshwarkar * created separate modules for model specific input features Signed-off-by: Ameya Mahabaleshwarkar * moved non-moodel methods to QAMetrics and refactored method names to more intuitive Signed-off-by: Ameya Mahabaleshwarkar * changes classmethods to staticmethods Signed-off-by: Ameya Mahabaleshwarkar * removed unnecassary copyright Signed-off-by: Ameya Mahabaleshwarkar * removed deprecated input features file Signed-off-by: Ameya Mahabaleshwarkar * abstracted cache filename, feature loading, feature dumping to QADataset Signed-off-by: Ameya Mahabaleshwarkar * removed unused imports and added dataclass decorator Signed-off-by: Ameya Mahabaleshwarkar * removed unused imports and refactored method name Signed-off-by: Ameya Mahabaleshwarkar * added base class for QA models and abstracted out common methods Signed-off-by: Ameya Mahabaleshwarkar * moved non-model eval code and predictions file dump to metrics class Signed-off-by: Ameya Mahabaleshwarkar * added combined example of train/eval/test/inference for all qa models Signed-off-by: Ameya Mahabaleshwarkar * renamed qa example file Signed-off-by: Ameya Mahabaleshwarkar * fixed trailing whitespaces Signed-off-by: Ameya Mahabaleshwarkar * added type casting to float for logger warning Signed-off-by: Ameya Mahabaleshwarkar * removed unsed import Signed-off-by: Ameya Mahabaleshwarkar * converted cached filename creation to class method Signed-off-by: Ameya Mahabaleshwarkar * moved common code in dataset classes to base class, renamed Features class to Example Signed-off-by: Ameya Mahabaleshwarkar * converted base QA example class to dataclass Signed-off-by: Ameya Mahabaleshwarkar * reduced code repition in prediciton evaluation Signed-off-by: Ameya Mahabaleshwarkar * converted prediction output files to jsonl Signed-off-by: Ameya Mahabaleshwarkar * added flag for checking if ground truth present in context spans Signed-off-by: Ameya Mahabaleshwarkar * converted predictions dump to jsonl from json Signed-off-by: Ameya Mahabaleshwarkar * converted nbest predictions dump to jsonl from json Signed-off-by: Ameya Mahabaleshwarkar * removed unused argument to no pad loss method Signed-off-by: Ameya Mahabaleshwarkar * added unit tests for qa metrics and dataset utilities Signed-off-by: Ameya Mahabaleshwarkar * applied style fix on new files Signed-off-by: Ameya Mahabaleshwarkar * added integration tests Signed-off-by: Ameya Mahabaleshwarkar * restored default values in qa config Signed-off-by: Ameya Mahabaleshwarkar * renamed stage to avoid duplicate Signed-off-by: Ameya Mahabaleshwarkar * added init files for new modules Signed-off-by: Ameya Mahabaleshwarkar * applied style fix for module init files Signed-off-by: Ameya Mahabaleshwarkar * added inline comments to make concise Signed-off-by: Ameya Mahabaleshwarkar * specified class as abstract Signed-off-by: Ameya Mahabaleshwarkar * specified .json format for output prediction files Signed-off-by: Ameya Mahabaleshwarkar * created separate variable for answer in context check for readability Signed-off-by: Ameya Mahabaleshwarkar * shifted stages to parallel Signed-off-by: Ameya Mahabaleshwarkar * applied style fix Signed-off-by: Ameya Mahabaleshwarkar * restored file modified by linter Signed-off-by: Ameya Mahabaleshwarkar * added transformers offline flag to true and moved all stages to parallel Signed-off-by: Ameya Mahabaleshwarkar * moved inference code inside test_ds check Signed-off-by: Ameya Mahabaleshwarkar * added script for converting msmarco dataset to squad format Signed-off-by: Ameya Mahabaleshwarkar * added tutorial for question answering with generative models Signed-off-by: Ameya Mahabaleshwarkar * added copyright header Signed-off-by: Ameya Mahabaleshwarkar * renamed old qa docs with _squad postfix and added docs for new qa modules Signed-off-by: Ameya Mahabaleshwarkar * added generative qa architecture diagram Signed-off-by: Ameya Mahabaleshwarkar * modified tutorial with colab testing changes, improved documentation Signed-off-by: Ameya Mahabaleshwarkar * changed branch name to main in tutorial * deprecated old QA tutorial * deprecated old QA docs * deprecated old QA example * removed deprecated ci test for old qa example * removed additional deprecated ci tests --- Jenkinsfile | 235 ++-- docs/source/nlp/question_answering.rst | 354 +++-- docs/source/nlp/question_answering_arch.png | Bin 0 -> 49678 bytes .../nlp/question_answering/conf/qa_conf.yaml | 157 +++ .../convert_msmarco_to_squad_format.py | 138 ++ .../question_answering/question_answering.py | 89 ++ .../question_answering_squad.py | 137 -- .../nlp/data/question_answering/__init__.py | 22 + .../data_processor/__init__.py | 15 + .../data_processor/qa_processing.py | 109 ++ .../question_answering/dataset/__init__.py | 18 + .../dataset/qa_bert_dataset.py | 356 +++++ .../question_answering/dataset/qa_dataset.py | 297 +++++ .../dataset/qa_gpt_dataset.py | 310 +++++ .../dataset/qa_s2s_dataset.py | 247 ++++ .../input_example/__init__.py | 18 + .../input_example/qa_bert_input_example.py | 34 + .../input_example/qa_gpt_input_example.py | 30 + .../input_example/qa_input_example.py | 33 + .../input_example/qa_s2s_input_example.py | 29 + nemo/collections/nlp/metrics/__init__.py | 1 + nemo/collections/nlp/metrics/qa_metrics.py | 202 +++ .../nlp/models/question_answering/__init__.py | 4 + .../question_answering/qa_base_model.py | 93 ++ .../question_answering/qa_bert_model.py | 698 ++++++++++ .../models/question_answering/qa_gpt_model.py | 364 ++++++ .../nlp/models/question_answering/qa_model.py | 13 +- .../models/question_answering/qa_s2s_model.py | 345 +++++ tests/collections/nlp/test_qna.py | 240 ++++ tutorials/nlp/Question_Answering.ipynb | 1149 +++++++++++++++++ tutorials/nlp/Question_Answering_Squad.ipynb | 725 ----------- 31 files changed, 5321 insertions(+), 1141 deletions(-) create mode 100644 docs/source/nlp/question_answering_arch.png create mode 100644 examples/nlp/question_answering/conf/qa_conf.yaml create mode 100644 examples/nlp/question_answering/convert_msmarco_to_squad_format.py create mode 100644 examples/nlp/question_answering/question_answering.py delete mode 100644 examples/nlp/question_answering/question_answering_squad.py create mode 100644 nemo/collections/nlp/data/question_answering/__init__.py create mode 100644 nemo/collections/nlp/data/question_answering/data_processor/__init__.py create mode 100644 nemo/collections/nlp/data/question_answering/data_processor/qa_processing.py create mode 100644 nemo/collections/nlp/data/question_answering/dataset/__init__.py create mode 100644 nemo/collections/nlp/data/question_answering/dataset/qa_bert_dataset.py create mode 100644 nemo/collections/nlp/data/question_answering/dataset/qa_dataset.py create mode 100644 nemo/collections/nlp/data/question_answering/dataset/qa_gpt_dataset.py create mode 100644 nemo/collections/nlp/data/question_answering/dataset/qa_s2s_dataset.py create mode 100644 nemo/collections/nlp/data/question_answering/input_example/__init__.py create mode 100644 nemo/collections/nlp/data/question_answering/input_example/qa_bert_input_example.py create mode 100644 nemo/collections/nlp/data/question_answering/input_example/qa_gpt_input_example.py create mode 100644 nemo/collections/nlp/data/question_answering/input_example/qa_input_example.py create mode 100644 nemo/collections/nlp/data/question_answering/input_example/qa_s2s_input_example.py create mode 100644 nemo/collections/nlp/metrics/qa_metrics.py create mode 100644 nemo/collections/nlp/models/question_answering/qa_base_model.py create mode 100644 nemo/collections/nlp/models/question_answering/qa_bert_model.py create mode 100644 nemo/collections/nlp/models/question_answering/qa_gpt_model.py create mode 100644 nemo/collections/nlp/models/question_answering/qa_s2s_model.py create mode 100644 tests/collections/nlp/test_qna.py create mode 100644 tutorials/nlp/Question_Answering.ipynb delete mode 100755 tutorials/nlp/Question_Answering_Squad.ipynb diff --git a/Jenkinsfile b/Jenkinsfile index e60805da0a0b..babd4129ed2e 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -1088,7 +1088,7 @@ pipeline { } } } - stage('L2: Parallel BERT SQUAD v1.1 / v2.0') { + stage('L2: Duplex Text Normalization') { when { anyOf { branch 'main' @@ -1097,53 +1097,6 @@ pipeline { } failFast true parallel { - stage('BERT SQUAD 1.1') { - // Cannot do fast_dev_run because squad needs whole dev dataset - steps { - sh 'cd examples/nlp/question_answering && \ - python question_answering_squad.py \ - model.train_ds.file=/home/TestData/nlp/squad_mini/v1.1/train-v1.1.json \ - model.dataset.use_cache=false \ - model.validation_ds.file=/home/TestData/nlp/squad_mini/v1.1/dev-v1.1.json \ - model.test_ds.file=/home/TestData/nlp/squad_mini/v1.1/dev-v1.1.json \ - model.train_ds.batch_size=2 \ - model.train_ds.num_samples=2 \ - model.validation_ds.batch_size=2 \ - model.validation_ds.num_samples=2 \ - model.test_ds.num_samples=2 \ - model.test_ds.batch_size=2 \ - trainer.max_epochs=1 \ - +trainer.max_steps=1 \ - model.language_model.pretrained_model_name=bert-base-uncased \ - model.dataset.version_2_with_negative=false \ - trainer.precision=16 \ - trainer.devices=[0] \ - trainer.accelerator="gpu" \ - exp_manager=null' - } - } - stage('BERT SQUAD 2.0') { - // Cannot do fast_dev_run because squad needs whole dev dataset - steps { - sh 'cd examples/nlp/question_answering && \ - python question_answering_squad.py \ - model.train_ds.file=/home/TestData/nlp/squad_mini/v2.0/train-v2.0.json \ - model.dataset.use_cache=false \ - model.train_ds.batch_size=2 \ - model.train_ds.num_samples=2 \ - model.validation_ds.batch_size=2 \ - model.validation_ds.num_samples=2 \ - trainer.max_epochs=1 \ - +trainer.max_steps=1 \ - model.validation_ds.file=/home/TestData/nlp/squad_mini/v2.0/dev-v2.0.json \ - model.language_model.pretrained_model_name=bert-base-uncased \ - model.dataset.version_2_with_negative=true \ - trainer.precision=16 \ - trainer.devices=[1] \ - trainer.accelerator="gpu" \ - exp_manager=null' - } - } stage('Duplex Text Normalization with Tarred dataset') { steps { sh 'cd examples/nlp/duplex_text_normalization && \ @@ -1199,7 +1152,36 @@ pipeline { // } // } - stage('L2: Parallel SQUAD v1.1 & v2.0') { + stage('L2: BERT Text Classification') { + when { + anyOf { + branch 'main' + changeRequest target: 'main' + } + } + failFast true + parallel { + stage ('Text Classification with BERT Test') { + steps { + sh 'cd examples/nlp/text_classification && \ + python text_classification_with_bert.py \ + model.dataset.num_classes=6 \ + model.train_ds.file_path=/home/TestData/nlp/retail_text_classification/train.tsv \ + model.validation_ds.file_path=/home/TestData/nlp/retail_text_classification/dev.tsv \ + model.language_model.pretrained_model_name=distilbert-base-uncased \ + model.train_ds.batch_size=10 \ + model.dataset.max_seq_length=50 \ + model.dataset.use_cache=false \ + trainer.devices=[0] \ + trainer.accelerator="gpu" \ + +trainer.fast_dev_run=true \ + exp_manager=null' + } + } + } + } + + stage('L2: Parallel BERT/BART/GPT2 Question-Answering SQUAD v1.1 & v2.0') { when { anyOf { branch 'main' @@ -1208,72 +1190,149 @@ pipeline { } failFast true parallel { - // TODO: use megatron bert when supported again - stage('SQUAD v2.0 with DistilBERT Uncased') { - // stage('SQUAD v2.0 with Megatron with ckpt & config') { + stage('BERT SQUAD 1.1') { // Cannot do fast_dev_run because squad needs whole dev dataset - // model.language_model.pretrained_model_name=megatron-bert-uncased \ - // model.language_model.lm_checkpoint=/home/TestData/nlp/megatron_345m_uncased/model_optim_rng.pt \ - // model.language_model.config_file=/home/TestData/nlp/megatron_345m_uncased/345m_config.json \ steps { - sh 'cd examples/nlp/question_answering && \ - python question_answering_squad.py \ - model.train_ds.file=/home/TestData/nlp/squad_mini/v2.0/train-v2.0.json \ + sh 'TRANSFORMERS_OFFLINE=0 && cd examples/nlp/question_answering && \ + python question_answering.py \ + model.train_ds.file=/home/TestData/nlp/squad_mini/v1.1/train-v1.1.json \ model.dataset.use_cache=false \ - model.train_ds.batch_size=1 \ - model.train_ds.num_samples=1 \ - model.validation_ds.batch_size=1 \ - model.validation_ds.num_samples=1 \ - trainer.accelerator=gpu \ - trainer.strategy=ddp \ + model.validation_ds.file=/home/TestData/nlp/squad_mini/v1.1/dev-v1.1.json \ + model.test_ds.file=/home/TestData/nlp/squad_mini/v1.1/dev-v1.1.json \ + model.train_ds.batch_size=2 \ + model.train_ds.num_samples=2 \ + model.validation_ds.batch_size=2 \ + model.validation_ds.num_samples=2 \ + model.test_ds.num_samples=2 \ + model.test_ds.batch_size=2 \ trainer.max_epochs=1 \ - +trainer.max_steps=1 \ - model.validation_ds.file=/home/TestData/nlp/squad_mini/v2.0/dev-v2.0.json \ - model.language_model.pretrained_model_name=distilbert-base-uncased \ - model.dataset.version_2_with_negative=true \ + trainer.max_steps=1 \ + model.language_model.pretrained_model_name=bert-base-uncased \ + model.dataset.version_2_with_negative=false \ trainer.precision=16 \ - trainer.devices=[1] \ + trainer.devices=[0] \ trainer.accelerator="gpu" \ - exp_manager=null' + exp_manager=null && TRANSFORMERS_OFFLINE=1' } } - stage('RoBERTa SQUAD 1.1') { + stage('BART SQUAD 1.1') { // Cannot do fast_dev_run because squad needs whole dev dataset steps { - sh 'cd examples/nlp/question_answering && \ - python question_answering_squad.py \ + sh 'TRANSFORMERS_OFFLINE=0 && cd examples/nlp/question_answering && \ + python question_answering.py \ model.train_ds.file=/home/TestData/nlp/squad_mini/v1.1/train-v1.1.json \ model.dataset.use_cache=false \ + model.dataset.check_if_answer_in_context=false \ + model.validation_ds.file=/home/TestData/nlp/squad_mini/v1.1/dev-v1.1.json \ + model.test_ds.file=/home/TestData/nlp/squad_mini/v1.1/dev-v1.1.json \ model.train_ds.batch_size=2 \ model.train_ds.num_samples=2 \ model.validation_ds.batch_size=2 \ model.validation_ds.num_samples=2 \ + model.test_ds.num_samples=2 \ + model.test_ds.batch_size=2 \ trainer.max_epochs=1 \ - +trainer.max_steps=1 \ - model.validation_ds.file=/home/TestData/nlp/squad_mini/v1.1/dev-v1.1.json \ - model.language_model.pretrained_model_name=roberta-base \ + trainer.max_steps=1 \ + model.language_model.pretrained_model_name=facebook/bart-base \ model.dataset.version_2_with_negative=false \ trainer.precision=16 \ trainer.devices=[0] \ trainer.accelerator="gpu" \ - exp_manager=null' + exp_manager=null && TRANSFORMERS_OFFLINE=1' } } - stage ('Text Classification with BERT Test') { + stage('GPT2 SQUAD 1.1') { + // Cannot do fast_dev_run because squad needs whole dev dataset steps { - sh 'cd examples/nlp/text_classification && \ - python text_classification_with_bert.py \ - model.dataset.num_classes=6 \ - model.train_ds.file_path=/home/TestData/nlp/retail_text_classification/train.tsv \ - model.validation_ds.file_path=/home/TestData/nlp/retail_text_classification/dev.tsv \ - model.language_model.pretrained_model_name=distilbert-base-uncased \ - model.train_ds.batch_size=10 \ - model.dataset.max_seq_length=50 \ + sh 'TRANSFORMERS_OFFLINE=0 && cd examples/nlp/question_answering && \ + python question_answering.py \ + model.train_ds.file=/home/TestData/nlp/squad_mini/v1.1/train-v1.1.json \ model.dataset.use_cache=false \ + model.dataset.check_if_answer_in_context=false \ + model.validation_ds.file=/home/TestData/nlp/squad_mini/v1.1/dev-v1.1.json \ + model.test_ds.file=/home/TestData/nlp/squad_mini/v1.1/dev-v1.1.json \ + model.train_ds.batch_size=2 \ + model.train_ds.num_samples=2 \ + model.validation_ds.batch_size=2 \ + model.validation_ds.num_samples=2 \ + model.test_ds.num_samples=2 \ + model.test_ds.batch_size=2 \ + trainer.max_epochs=1 \ + trainer.max_steps=1 \ + model.language_model.pretrained_model_name=gpt2 \ + model.dataset.version_2_with_negative=false \ + trainer.precision=16 \ trainer.devices=[0] \ trainer.accelerator="gpu" \ - +trainer.fast_dev_run=true \ - exp_manager=null' + exp_manager=null && TRANSFORMERS_OFFLINE=1' + } + } + stage('BERT SQUAD 2.0') { + // Cannot do fast_dev_run because squad needs whole dev dataset + steps { + sh 'TRANSFORMERS_OFFLINE=0 && cd examples/nlp/question_answering && \ + python question_answering.py \ + model.train_ds.file=/home/TestData/nlp/squad_mini/v2.0/train-v2.0.json \ + model.dataset.use_cache=false \ + model.train_ds.batch_size=2 \ + model.train_ds.num_samples=2 \ + model.validation_ds.batch_size=2 \ + model.validation_ds.num_samples=2 \ + trainer.max_epochs=1 \ + trainer.max_steps=1 \ + model.validation_ds.file=/home/TestData/nlp/squad_mini/v2.0/dev-v2.0.json \ + model.language_model.pretrained_model_name=bert-base-uncased \ + model.dataset.version_2_with_negative=true \ + trainer.precision=16 \ + trainer.devices=[1] \ + trainer.accelerator="gpu" \ + exp_manager=null && TRANSFORMERS_OFFLINE=1' + } + } + stage('BART SQUAD 2.0') { + // Cannot do fast_dev_run because squad needs whole dev dataset + steps { + sh 'TRANSFORMERS_OFFLINE=0 && cd examples/nlp/question_answering && \ + python question_answering.py \ + model.train_ds.file=/home/TestData/nlp/squad_mini/v2.0/train-v2.0.json \ + model.dataset.use_cache=false \ + model.dataset.check_if_answer_in_context=false \ + model.train_ds.batch_size=2 \ + model.train_ds.num_samples=2 \ + model.validation_ds.batch_size=2 \ + model.validation_ds.num_samples=2 \ + trainer.max_epochs=1 \ + trainer.max_steps=1 \ + model.validation_ds.file=/home/TestData/nlp/squad_mini/v2.0/dev-v2.0.json \ + model.language_model.pretrained_model_name=facebook/bart-base \ + model.dataset.version_2_with_negative=true \ + trainer.precision=16 \ + trainer.devices=[1] \ + trainer.accelerator="gpu" \ + exp_manager=null && TRANSFORMERS_OFFLINE=1' + } + } + stage('GPT2 SQUAD 2.0') { + // Cannot do fast_dev_run because squad needs whole dev dataset + steps { + sh 'TRANSFORMERS_OFFLINE=0 && cd examples/nlp/question_answering && \ + python question_answering.py \ + model.train_ds.file=/home/TestData/nlp/squad_mini/v2.0/train-v2.0.json \ + model.dataset.use_cache=false \ + model.dataset.check_if_answer_in_context=false \ + model.train_ds.batch_size=2 \ + model.train_ds.num_samples=2 \ + model.validation_ds.batch_size=2 \ + model.validation_ds.num_samples=2 \ + trainer.max_epochs=1 \ + trainer.max_steps=1 \ + model.validation_ds.file=/home/TestData/nlp/squad_mini/v2.0/dev-v2.0.json \ + model.language_model.pretrained_model_name=gpt2 \ + model.dataset.version_2_with_negative=true \ + trainer.precision=16 \ + trainer.devices=[1] \ + trainer.accelerator="gpu" \ + exp_manager=null && TRANSFORMERS_OFFLINE=1' } } } diff --git a/docs/source/nlp/question_answering.rst b/docs/source/nlp/question_answering.rst index 029ef3662260..c98ed1b57526 100644 --- a/docs/source/nlp/question_answering.rst +++ b/docs/source/nlp/question_answering.rst @@ -1,57 +1,45 @@ .. _question_answering: -Question Answering model -======================== +Question Answering +================== -With Question Answering, or Reading Comprehension, given a question and a passage of content (context) that may contain an answer for -the question, the model predicts the span within the text with a start and end position indicating the answer to the question. For -datasets like SQuAD 2.0, this model supports cases when the answer is not contained in the content. +Given a context and a natural language query, we want to generate an answer for the query +Depending on how the answer is generated, the task can be broadly divided into two types: -For every word in the context of a given question, the model is trained to predict: +1. Extractive Question Answering +2. Generative Question Answering -- The likelihood this word is the start of the span. -- The likelihood this word is the end of the span. +**Extractive Question Answering with BERT-like models** -The model chooses the start and end words with maximal probabilities. When the content does not contain the answer, we would like the -start and end span to be set for the first token. +Given a question and a context, both in natural language, predict the span within the context with a start and end position which indicates the answer to the question. +For every word in our training dataset the model predicts: -A pretrained BERT encoder with two span prediction heads is used for the prediction start and the end position of the answer. The span -predictions are token classifiers consisting of a single linear layer. +- likelihood this word is the start of the span +- likelihood this word is the end of the span -Quick Start Guide ------------------ +**Generative Question Answering with S2S and GPT-like models** -.. code-block:: python - - from nemo.collections.nlp.models import QAModel - - # to get the list of pre-trained models - QAModel.list_available_models() - - # Download and load the pre-trained BERT-based model - model = QAModel.from_pretrained("qa_squadv1.1_bertbase") - - # try the model on a few examples - model.inference(test_file) - - -.. note:: - - We recommend you try Question Answering model in a Jupyter notebook (can run on `Google's Colab `_.): - `NeMo/tutorials/nlp/Question_Answering_Squad.ipynb `__. - - Connect to an instance with a GPU (**Runtime** -> **Change runtime type** -> select **GPU** for the hardware accelerator). - - An example script on how to train and evaluate the model can be found here: `NeMo/examples/nlp/question_answering/question_answering_squad.py `__. - - The default configuration file for the model can be found at: `NeMo/examples/nlp/question_answering/conf/question_answering_squad.yaml `__. +Given a question and a context, both in natural language, generate an answer for the question. Unlike the BERT-like models, there is no constraint that the answer should be a span within the context. +Supported Tasks +=============== ++----------------------------------+-----------------+----------------------------------------------------------------------+------------------------------------------+ +| **Task** | **Models** | **Supported Options for model.language_model.pretrained_model_name** | **Supported options for model.library** | ++----------------------------------+-----------------+----------------------------------------------------------------------+------------------------------------------+ +| Extractive Question Answering | BERTQAModel | bert-{base, large}-{cased, uncased}, roberta{base, large} | Huggingface, Megatron | ++----------------------------------+-----------------+----------------------------------------------------------------------+------------------------------------------+ +| Generative Question Answering | S2SQAModel | t5-{small, base, large}, bart-{base, large} | Huggingface | ++----------------------------------+-----------------+----------------------------------------------------------------------+------------------------------------------+ +| | GPTQAModel | gpt2, gpt2-{medium, large, xl} | Huggingface | ++----------------------------------+-----------------+----------------------------------------------------------------------+------------------------------------------+ Available models ^^^^^^^^^^^^^^^^ -.. list-table:: *Pretrained Models* +Following BERT-like models are available for Extractive Question-Answering + +.. list-table:: :widths: 5 10 :header-rows: 1 @@ -74,29 +62,56 @@ Available models * - qa_squadv2.0_megatron_uncased - https://ngc.nvidia.com/catalog/models/nvidia:nemo:qa_squadv2_0_megatron_uncased - -.. _dataset_question_answering: +Module Design +============= + +The module is decouple data and model components to support idependent integration of various model achitectures and datasets. +QAProcessor, QAExample, and the base QADataset modules are responsible for model-independent data handling utilites like loading SQuAD format dataset files and parsing examples. +QADataset modules handle model-specific data formatting. +Similarly, the BaseQAModel module handles common model tasks like creating dataloader, and the QAModel modules handle model architecture-specific functions like trainig, testing, and evaluation. + +.. image:: question_answering_arch.png + :alt: Question-Answerin-Architecture + +Configuration +============= + +The default sample model training configuration can be found at: `NeMo/examples/nlp/question_answering/conf/qa_conf.yaml` + +The configuration defines parameters for the following main components: + +- :code:`model.dataset`: parameters that describe the dataset being used, ex. max sequence length, max query length, max answer length +- :code:`model.train_ds`, :code:`model.validation_ds`, :code:`model.test_ds`: parameters for the dataloaders, ex. batch size, source filepath +- :code:`model.language_model`, :code:`model.tokenizer`: language model and the tokenizer to be used for initializing the model +- :code:`model.optim`: optimiation parameters, ex. learning rate, scheduler, weight decay +- :code:`model.token_classifier`: used only for the BERTQAModel, defines the span prediction head for extractive question answering +- :code:`trainer`: defines the training process, ex. number of gpus, epochs, etc. +- :code:`exp_manager`: describes the experiment manager for logging training progress and checkpointing + +Arguments that very commonly need to be edited for all models and datasets + +- :code:`do_training`: perform training or only testing +- :code:`trainer.devices`: number of GPUs (int) or list of GPUs e.g. [0, 1, 3] +- :code:`model.library`: library to load language model from [huggingface or megatron] +- :code:`model.language_model.pretrained_model_name`: pretrained QA model from ``list_available_models()`` or path to a ``.nemo`` file (Check the ``Available Models`` section for some of the available checkpoints) +- :code:`model.language_model.lm_checkpoint`: specifying a trained checkpoint (.bin / .ckpt / .nemo) +- :code:`model..file`: filepath for loading respective datasets +- :code:`model..num_samples`: the number of samples to use from the training dataset (use ``-1`` to specify all samples) +- :code:`model.dataset.use_cache`: if ``True``, features will be loaded from a cache file if it exists and created features will be dumped to the cache file +- :code:`model.dataset.version_2_with_negative`: boolean indicating whether dataset contains unanswerable questions (yes if set to ``True``) +- :code:`model.dataset.check_if_answer_in_context`: boolean indicating whether the context spans that do not have the answer text in them should be considered as negative examples (set to ``True`` for datasets of extractive nature like SQuAD and ``False`` for datasets of generative nature like MS-MARCO) +- :code:`model.dataset.doc_stride`: stride for splitting long documents into chunks +- :code:`model.dataset.max_query_length`: questions exceeding this value will be truncated +- :code:`model.dataset.max_answer_length`: ground truth answers exceeding this value will be truncated +- :code:`model.dataset.max_seq_length`: maximum allowed sequence length for input to the model including context, query, and answer +- :code:`model.tokens_to_generate`: maximum answer tokens to be generated for the generative models Data Format ------------ - -This model expects the dataset in `SQuAD format`_ (i.e., a JSON file for each dataset split). The code snippet below shows an example -of the training file. Each title has one or multiple paragraph entries, each consisting of the "context" and question-answer entries. -Each question-answer entry has: - -- A question -- A globally unique id -- The Boolean flag ``is_impossible``, which shows whether a question is answerable or not: - - if the question is answerable, one answer entry contains the text span and its starting character index in the context - - if the question is not answerable, an empty ``answers`` list is provided +=========== -.. _SQuAD format: https://rajpurkar.github.io/SQuAD-explorer/ +The QA models expect datasets to be present in the SQuAD format. For using datasets other than the standard SQuAD v1.1 and v2.0, the datasets should be first converted into the SQuAD format. -The evaluation files (for validation and testing) follow the above format, except that it can provide more than one answer to the -same question. The inference file also follows the above format, except that it does not require the ``answers`` and ``is_impossible`` -keywords. - -The following is an example of the data format (JSON file): +The following is an example of the expected SQuAD data format (JSON file): .. code:: @@ -133,16 +148,22 @@ The following is an example of the data format (JSON file): ] } +.. Note:: + + For datasets of generative nature where the answer might not be an exact span within the context, the :code:`answer_start` field can be set to -1. -Dataset Download ----------------- +Downloading Datasets +==================== -To perform training of the Question Answering model on the SQuAD dataset, you must first download it from `here -`_ or run: +Following sections describes how to download the SQuAD datasets, along with an example of converting a non-SQuAD dataset (MS-MARCO) into the SQuAD format for the QA models. + +**SQuAD Dataset** + +To perform training of the Question Answering model on the SQuAD dataset, you must first download it from https://rajpurkar.github.io/SQuAD-explorer or run: .. code:: - python get_squad.py + python NeMo/examples/nlp/question_answering/get_squad.py There are two versions: @@ -163,139 +184,102 @@ evaluation: |-- v2.0/train-v2.0.json |-- v2.0/dev-v2.0.json +**MS-MARCO Dataset** -.. _model_training_question_answering: - -Model Training --------------- - -In the Question Answering Model, we are training a span prediction head on top of a pre-trained language model, such as -`BERT: Pre-training of Deep Bidirectional Transformers for Language Understanding `__ :cite:`nlp-qa-devlin2018bert`. -Unless the user provides a pre-trained checkpoint for the language model, the language model is initialized with the pre-trained model -from `HuggingFace Transformers `__. - -Example of model configuration file for training the model can be found at: `NeMo/examples/nlp/question_answering/conf/question_answering_squad_config.yaml `__. - -The specification can be grouped into three categories: - -- Parameters that describe the training process: **trainer** -- Parameters that describe the datasets: **model.dataset**, **model.train_ds**, **model.validation_ds**, **model.test_ds** -- Parameters that describe the model: **model** - -More details about parameters in the spec file can be found below: - -+-------------------------------------------+-----------------+--------------------------------------------------------------------------------------------------------------+ -| **Parameter** | **Data Type** | **Description** | -+-------------------------------------------+-----------------+--------------------------------------------------------------------------------------------------------------+ -| **pretrained_model** | string | Pretrained QA model model from ``list_available_models()`` or path to a ``.nemo`` file. | -+-------------------------------------------+-----------------+--------------------------------------------------------------------------------------------------------------+ -| **do_training** | bool | If ``true``, starts training, otherwise, skips training and continues with evaluation/inference. | -+-------------------------------------------+-----------------+--------------------------------------------------------------------------------------------------------------+ -| **model.dataset.version_2_with_negative** | bool | Set to ``true`` to allow examples without an answer, e.g. for SQuAD v2.0. | -+-------------------------------------------+-----------------+--------------------------------------------------------------------------------------------------------------+ -| **model.dataset.do_lower_case** | bool | If ``true``, converts text to lower case, only import for inference/evaluation. | -+-------------------------------------------+-----------------+--------------------------------------------------------------------------------------------------------------+ -| **model.dataset.use_cache** | bool | If ``true``, either loads all preprocessed data from cache or saves preprocessed data for future use. | -+-------------------------------------------+-----------------+--------------------------------------------------------------------------------------------------------------+ -| **training_ds.file** | string | The training file path. | -+-------------------------------------------+-----------------+--------------------------------------------------------------------------------------------------------------+ -| **training_ds.num_samples** | integer | The number of samples to use from the training dataset (use ``-1`` to specify all samples). | -+-------------------------------------------+-----------------+--------------------------------------------------------------------------------------------------------------+ -| **validation_ds.file** | string | The validation file path. | -+-------------------------------------------+-----------------+--------------------------------------------------------------------------------------------------------------+ -| **validation_ds.num_samples** | integer | The number of samples to use from the validation dataset (use ``-1`` to specify all samples). | -+-------------------------------------------+-----------------+--------------------------------------------------------------------------------------------------------------+ -| **test_ds.file** | string | The test file path (optional). | -+-------------------------------------------+-----------------+--------------------------------------------------------------------------------------------------------------+ -| **test_ds.num_samples** | integer | The number of samples to use from the test dataset (use ``-1`` to specify all samples). | -+-------------------------------------------+-----------------+--------------------------------------------------------------------------------------------------------------+ - -Example of the command for training the model: - -.. code:: - - python question_answering_squad.py \ - model.train_ds.file= \ - model.validation_ds.file= \ - model.dataset.version_2_with_negative= \ - model.dataset.do_lower_case= \ - trainer.max_epochs= \ - trainer.devices=[] \ - trainer.accelerator='gpu' - -.. Note:: - - The first time you train, it will take an extra 5-10 minutes to process the dataset. For future training runs, it will use the - processed dataset if :code:`model.dataset.use_cache=true`, which is automatically cached in the files in the same directory as - the data. - -Required Arguments for Training -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -- :code:`model.train_ds.file`: path to the training file in JSON format -- :code:`model.validation_ds.file`: path to the validation file in JSON format - -Fine-tuning Procedure -^^^^^^^^^^^^^^^^^^^^^ - -Fine-tuning procedure and logs look similar to what's described in the Model Training section, with the addition of the model -that is initially loaded from a previously trained checkpoint, e.g. by specifying :code:`pretrained_model=`. +MS-MARCO(Microsoft Machine Reading Comprehension) is a large scale dataset focused on machine reading comprehension, question answering, and passage ranking. MS-MARCO consists of 1,010,916 queries generated from real, anonymized Bing user queries. The contexts are extracted from real web documents and the answers are generated by humans. -Inference ---------- +For downloading the MS-MARCO dataset, Terms of Use need to be accepted at https://microsoft.github.io/msmarco/. -An example script on how to run inference can be found at `examples/nlp/question_answering/question_answering_squad.py `_. +The dataset files can be downloaded from: + - https://msmarco.blob.core.windows.net/msmarco/train_v2.1.json.gz + - https://msmarco.blob.core.windows.net/msmarco/dev_v2.1.json.gz -To run inference with the pre-trained model, run: +The QA models expect data in SQuAD format. The MS-MARCO dataset is originally not in the SQuAD format. The dataset has the following structure: .. code:: + + { + "answers":["A corporation is a company or group of people authorized to act as a single entity and recognized as such in law."], + "passages":[ + { + "is_selected":0, + "url":"http:\/\/www.wisegeek.com\/what-is-a-corporation.htm", + "passage_text":"A company is incorporated in a specific nation, often within the bounds of a smaller subset of that nation, such as a state or province. The corporation is then governed by the laws of incorporation in that state. A corporation may issue stock, either private or public, or may be classified as a non-stock corporation. If stock is issued, the corporation will usually be governed by its shareholders, either directly or indirectly."}, + ... + }], + "query":". what is a corporation?", + "query_id":1102432, + "query_type":"DESCRIPTION", + "wellFormedAnswers":"[]" + } - python question_answering_squad.py \ - pretrained_model= \ - model.dataset.version_2_with_negative= \ - model.dataset.do_lower_case= \ - do_training=false \ - model.validation_ds.file= - -Required Arguments for inference: -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -- :code:`pretrained_model`: pretrained QA Model model from ``list_available_models()`` or path to a ``.nemo`` file - -Model Evaluation ----------------- - -An example script on how to evaluate the pre-trained model, can be found at `examples/nlp/question_answering/question_answering_squad.py `_. - -To run evaluation of the pre-trained model, run: +The conversion to SQuAD format can be performed using the following script: .. code:: - python question_answering_squad.py \ - pretrained_model= \ - model.dataset.version_2_with_negative= \ - model.dataset.do_lower_case= \ - do_training=false \ - model.test_ds.file= - - -Required Arguments: -^^^^^^^^^^^^^^^^^^^ - -- :code:`pretrained_model`: pretrained QA model from `list_available_models()`` or path to a ``.nemo`` file -- :code:`model.test_ds.file`: path to test file - -During evaluation of the :code:`test_ds`, the script generates the following metrics: + python NeMo/examples/nlp/question_answering/convert_msmarco_to_squad_format.py \ + --msmarco_train_input_filepath=/path/to/msmarco_train_v2.1.json \ + --msmarco_dev_input_filepath=/path/to/msmarco_dev_v2.1.json \ + --converted_train_save_path=/path/to/msmarco_squad_format_train.json \ + --converted_dev_save_path=/path/to/msmarco_squad_format_dev.json \ + --exclude_negative_samples=False \ + --keep_only_relevant_passages=False + +.. Note:: + + - setting :code:`exclude_negative_samples` to ``True`` will exclude samples from the MS-MARCO dataset that do not have a answer + - setting :code:`keep_only_relevant_passages` to ``True`` will exclude passages that have ``is_selected=0`` in the MS-MARCO dataset + +Training, Validation, Testing +============================= + +A step-by-step guide to training and testing QA models, as well as running inference can be found at `NeMo/tutorials/nlp/Question_Answering.ipynb`. Following is an example of training a QA model using the example script provided at `NeMo/examples/nlp/question_answering/question_answering.py`: + +.. code:: + + python NeMo/examples/nlp/question_answering/question_answering.py \ + do_training=true \ + model.train_ds.file= \ + model.validation_ds.file= \ + model.test_ds.file= \ + model.train_ds.batch_size=16 \ + model.train_ds.num_samples=-1 \ + model.validation_ds.batch_size=16 \ + model.validation_ds.num_samples=-1 \ + model.test_ds.num_samples=16 \ + model.test_ds.batch_size=-1 \ + model.language_model.pretrained_model_name= \ + model.dataset.use_cache=false \ + model.dataset.version_2_with_negative=true \ + model.dataset.check_if_answer_in_context=true \ + trainer.max_epochs=3 \ + trainer.max_steps=-1 \ + trainer.precision=16 \ + trainer.devices=[0] \ + trainer.accelerator="gpu" + +.. Note:: + + - :code:`version_2_with_negative` should be set based on whether the dataset contains unanswerable questions or not, ex. set to ``True`` for SQuAD v2.0 and ``False`` for SQuAD v1.1 + - :code:`check_if_answer_in_context` should be set according to the extractive or generative nature of the dataset, ex. set to ``True`` for SQuAD datasets and ``False`` for the MS-MARCO dataset + - :code:`do_training` can be set to ``False`` for running only testing on the test dataset without training + +Following is an example of running inference using the example script at `NeMo/examples/nlp/question_answering/question_answering.py`: + +.. code:: + + python NeMo/examples/nlp/question_answering/question_answering.py \ + pretrained_model= \ + do_training=false \ + model.test_ds.file= \ + model.test_ds.num_samples=1 \ + model.test_ds.batch_size=-1 \ + trainer.devices=[0] \ + trainer.accelerator="gpu" + +During evaluation of the :code:`validation_ds` and :code:`test_ds`, the script generates the following metrics: - :code:`Exact Match (EM)` - :code:`F1` -More details about these metrics can be found `here `__. - -References ----------- - -.. bibliography:: nlp_all.bib - :style: plain - :labelprefix: NLP-QA - :keyprefix: nlp-qa- +More details about these metrics can be found `here `__. \ No newline at end of file diff --git a/docs/source/nlp/question_answering_arch.png b/docs/source/nlp/question_answering_arch.png new file mode 100644 index 0000000000000000000000000000000000000000..b5e5c4d16c7deb1e559e742d71e41a2cbab30f0f GIT binary patch literal 49678 zcmd42cT`i~7A}e+qJW?RB1MYyP@?pXl+cTT(3>E=_f7y2=^#izI)oOw^bQKrdzTvN zz1Kj18-C}Ud+zagt?~lhAdoZ&0-fPW0SDoMdzOeTyvQO|R@UXD3p2*8dt6^b1 zgkWJ|>psH9lyonF4KV*6SV$;KU}2TV;G>OkFu!R(%Bd-1VR+|78L(5wQ`bC))bad42g^tky1`d%}UA0f#`j# ztgbKpQ7xyUr2;W<_YO3+u#b#^z1K1*DEi?W7+PE3?CkF69q>i*y^fW=yOx2evbrAh z_FWFfn0qFQveH9}BFy-c~g?#a6a#bXL_UFDTQxNEpCo)ZYQ1X%QA zz6?uVS_0xZvwIzuyxmlhha2}mAmFd7iraK4z+-lA+V7U^Pw1n72fU^VJrYl+&c%Fg z6vIi)XtuJ8m7NsjnBL zUIyPUD103v+A>Xz6;O+y9Mgqox9$*3^;%zB2;SX zApc!a%$!n2OR#_7Xp=xRkxyz{-x(HGll%zulJMrq%u@C8hZq;n35*^51v~lse~~{6 zr=o*I2NyMbu&@>$sO(VD0<7+V@BO7EN&UmxwDlYFcmM*+@u{Q9Iu=+SKNJrO_a3u5 z;Zs=Ee~O2jU$)`-q_!9R(@=D-yZk*&UH|LfU!#X3GFF&U$ETruLb#tv@^Peb0)hgF zg>gd_bl*X+1Re>LW17jqkzQT0G+=R*%{o0ayDcORilRQMh&p>g`e&K0aTeQ`&TabT zM;ZLRmj=IhR)0k-8akQ$7*RK@8A*rV_YpDk@ChiBnk7cBowqHtQ_mQo=EHu;h#$eD z&WJHJ$hz0yUT_K7?3`?-$ffm=s(H+9`nzB%z;F7H-7JlB<{9{E_g+hQgLvIDD5?Hf zm3Xo7WhKT2!xb&cU3$Qeu819Tq61~uc5Ub-bxW`_RrFRddgJ$p>?u@-Ok*hC&Y_U- zG`8l%!Xs**Y)Z_22doUL4A5=!2$@vlaG_{MJVIifw{)P^c7WlX*QS$mPqy|MT zd?cg%McwG6opw(iu6(G=olD)USCAfa1cpG@xUjwi`VNRd1w5fWw zbI43si7OXDcx>UH&2#0kmLW~~OU8dCDT?n4NjtT_X_4qpj}Nh-r!iFTomEp`+MZ~o z$Ir=t14VCoWiXz7N6)ZaM)lq`CJl4TBAsNO@9tLr-L1dA` zpDg3!Y>IF;+QH)tWpiiwI>zQjV?_b&^L#!54w-yof@I0IHj!4#`tsRtTR2%N57sjh zudlRjFO|&qH)ldrc6cb?;xMy2q;wRFcSbJW{E@aZf=2C#)Qm8J1qZ1$3j<0PbUrNb zV&^Xa6KW~Ak?+?`mm?} z)thJW&|>y5HfG2Pjs8QCczS{jukhIvbv0q0 z4kd5;?%Mu(zl1x!71#mXs-PKgoJxkyhptJ3y|lekq*n!eE;0^yK8GK+y`pF@pxEy1 zH*(hG>P+d1bTO`#5A9{bco7hEW*^#0srV@INjnTwy6;I`_}r zE>8EJCX+8<2l@yuOm#OQ3YT(5lICZNiPHXr9Ts(P*^fSG%^CO}G6nEN*z;6C=z+~% z3KlvYFNO<&&5A|%xu4!Pz{@VKEBH%<1|meN{^jH>gO#lak{Jh3Qaykq^@N8XTbFx) zgZ<;|baYUedD@o7vzZ6Eji1P1-p(|ZeA`a&iXQ5}_`{7ucp?neUK3d_PWF%s9(wlD zAT$g;c5I2Y{?dcGB-V+1YbS<5U8VTKMJQ^V{&er)*Nd&@q6KKV%j1eIkBElygf!n!QQX2})& z#I0?muhwkK#`!iOtCPIQ*KDJp)D+4xGnG+ni(>SD*Q{Pam3ANio;D>qz}axwstKd; zfL%~B6EM0p=C+rgy~UnLi>4gddo%&#Qjzffsr>w3RDeRZM06)8ST4NQmhzQgj+Iy#L{xYwWW*V#XURL8LBR0Os zT>LtETtCD2d-Rlc@_zoAqMr~aYOa_E;HStuy;A>NVnf$+m+*9+z)rU5`;3x_``D_h1L~l z>Vm^%R303G1SdiRDZ3UT4j;!KriX7uM|Rf<#k$Pe9Pn1a6!LO3sR!#^Vm82qr!Ka? zEs0A!HD5D?r$vTJvjd(qL)7NbyyQ|ndSbzF_rL_sb?HF)Y^v!9k~H#fjU6(~v+V_; zMfWvqbPIvH4@&_6g|e+&4tbz2m~Z?>cjlrcdDS**f#{Jywswu&;=+NbJeS!3M{Pn( zp_hKY!6#;m3wGS_-UYXZ44n=>j{8B;mm7;QndEcf;qca+G6$J+d7j>%ZnE;No#7uG z=if+3MTRPTE#qui3QQfb6YrAgCpmw4O;bX%w$vCZ&A4Y!W12o;>#%vPWhonyft7Kh z51yWt!JVO(_AhadN0+l`890;2#%Be*$DWg5^mlWV9p#twmjM|$Y`86(g;HAJhU0x{ z;2gAq!td6-$<0Nc0P-IDTYiAf7 zuFPXZi@p1;X?2%0EWBH(y zD;WKzrHEp9SfOuA{S20@Hyh;H*0ALD*Ls3%f;pmhflmkmk$!ER4)Drl@xzbUY~$&n zo%tDrb+Wpk0Sg~wfxkXil}gKc_A4!&kD5c^$4x5O;wP?Pk8#w9>+4#_ZKneAQ*3aS zK)ig*<~o=PxKn`s?9G2y_j>uT)w9Pb>KNc|3a|Rl^dTot5@Ai5YQ;W$bjX**(S- z8JVW!$ovLr$O`aH+M&_{4lUFRie)H6e3E$XBs0 z%MrRJX_>zgG}w%%Ddef~V3zJEO;W!D>Rrq^@m%z?6pLHN)D-LVoPL zY5ao!xBI8AR9a`EaOVoq*I*}7^QRyGXn4F{zY}nQM&(%?g*^5&n}8*&-Vk^jX;wqj&DNz$ z)v7f-ZV08-Fe&R7ytS9zykdT|aDip-kPZk9fAC>Eb*iL__?1$uVztrf=qd3l)X}HI zpk5v~@FI;r8>lA@h4GFeh}A}^fJOc{ir5ee_wzGhqM=E0y3VXgIa9wu2TO2DOn~66 zcx8OBUOK?v{nXVT3KWE#ZaPs^wJ&*VJZZH#M7>KYlWo)o{l1T!oSG^tX0;Q6I;zYp zw6-C|rEWHZ(+P*xaLIHtNG_*7GLArl(?IDA9IpA@lK1n9kqTfd?r4`l(2Fg1yRAy7 ztTUmQKdHP}KApjJh}_)7E|z}CJE!b9Y=P8)u3dlJXv#~R@MPP)O=N<^lwA=@!Ktvv zD1_R>7wutLE*ALBf*qekU@|Fj=A)1#9YM`YM5nzwRZwTmU)Zoceejg@Kug6%+u-}d z6>3EqUJa`@eiarg%kE$i6^T}R9g#WpJ}q$)*o`^JkCbEMrse3W4q+Bd=R3`WZW4o&G0fv#>raHRp&W$W~;00IjO3mPN*a}ZvC8b;ItDMcD|eE z9W9k!p~3d<3#S=D09o$m_unCT^&FHJ}o7VSi zj8bO7$#cNw3fF@iNGan&T|6+cC9#xkwrgLtSJy3}t05h6QQ+q0faoD@<%bBiQYMAO zRgd`heL2#Ypn6rR@$pJth*6!FQYKm?4EP*2s&&t(Nk9HnKYmj49F*k5^g|HjRFL3= zOvMw-`MACE$ba@cchfC73?-KROk|wmbLvX@)Vt0?YikZ)o#u)_eS>v6Z z`Wqp=I&`8K06KaH;T-IdVv9%EO#N~V?WI#HRTWI*-*#fZ{e-A958ycQU>N>{&``4S_Rm;C4E0beEud+69(DEj`te;Gji2J9=G}grjiv7w!qxWe5JX3+p$d6E zdVaEOSa4^j(PG*%m(ey*t)8Ew{`VTl{qb{G?)uN9@znYXuA|SXOsc$bg=pAAIlQ&v&b z1b&X&3}WxF3Os5QcysWdwlJ>*2G-u0aaVz~zrmfsw3lptPt{WYKeqVJng2)bp*{7} zr9Hj=#|di4d?L*C?_PMfS9T&H+4KL@Lw6o}4<*BWxbP6;yswvH98a&^?%W)>fq75P z2@?SRU;Fw0*y{h6P7NAAw+b9TC$mt*VZ~2lt==w^sDG$f%2%yhHF}B!-nNffYKUDt zR6{cV=UdUX9~4i_&JWrrUw$LcK5u&E(Wx}%f1_l^H#Zt%q_HMdq6ulUqs=na%|vq! z4n5>cFRyn%Cw6MDn&$W2gs&#}8+ABLXGrH3LPag?``=&JAD=9by@RxnU`E^BD-Liq z%M!LJX7atJ60Y_dMUO8b^?UlKE?{#1TiyoXt-&gk6%g9%aUU}tF=GU!3^5fzP!4SP zRZq6iWGYn(X%1Zn=>jV$>RZwl5$%CE5#fgVZXB`|h?UEh#@L|{cW*FZvyd~Gq2!}m zSj?txQ7Z}s`Kx~n8T%x z19AKRkS6l%o7k?56ij{mQg|k1GudmE+RiEC0T9n$;#^l<+z+5?5QpR`N4mN=w`67M z+y6O8Rj_N18r^iYSwq@dcnLV@+RGo8x383~MXx*X`!0^o_h?&f!NL&TNnHni(*C(b zlHp@_)6t$_PQ!05(uk=vjoxGQ&9F%LaiK;+qkplQ1Vnf2Z0XQts=zAVrGxm1W0Hg* zxlU&f(|R#XSW#ZfotZ_^M%RPF?8^u{e2Jm|70)CanAJxt{&xrR0SFZZcN#Im(wDSn~NkgM}F;6hG8ioH<(gs39P!yT|J@lP=n{J*utt+J3f{ z>4agfFI@E<7d`XWvsP!xtV+SlqO`i0_r=m3G4Y!yCg*%z`qNL|V7^`zV<-$?Ad@~wN)Q$Tis9}oabH5yS@I{GOAd5HQ!$cR}E|GhpA&iv$J90B>4 zNt?ZCW@7ich9Q2~iDDFw z0!L4|F#_>-O&j-UfSn zBt*&U1sKc&kl|jDLdWh)DB;(vNgRk--2yuK`eaOIDbZw*lod z(o6UKG_m>5oQZ*45)8kKXof1wq&tS1JVp)@Uy!d3c9$Sl49EJ}0YSG*BnbhMkhp*E zm&f4)u(9~jvPfv>VxwSM>EDfo;ngF9y1QG5OeUvI_QxvdcOvY7m%0axNTbQHbi|% zPHl-y$YY!nKasWZ#ywsCrBxT%TNRTvR-2#v*)$~=(?8ztWGv|wJ%`h_A)vI@<>PiH>B`4 z9Qn`EcYKKrK>Q43-`d;9y%oD$r%Mpy!NkQaeRwwQbE;EBuI-m=R>iJ^$u%Way3&m5 z?P$-|1SC1%{Z`CT;h$r|KIH`rYVE27b@9DkBGm1h+M*F3*xH-oMqfg&70s6yAgY5H zsP=A#V$3BZK+{kBB_2>xCA=^PZ|=~j^c!sz6Hkxcv@{P4q1~@JQck7ywIKk#n?psX zX)(dR3@8mSl|JhTTNVrZR!;Q03@n>G{j@CZDQQWuRm#%*6X0!CM_$6SUNZql>{bK+ zd^?b;s^6`w5Bi?X%>k8rwEQ6?W~jmre{y~O?pzj#Kr0P|p=a|s)*irHS6(E7Ziec{ z0`!};zWETh{D#B5PB#zo?<4+9_y%GS9U|4+PwVy>2IUZc7P_7alGXb69v->_c?*xE zS7{!=kMQFGw%#{~>!1$QkMfO*CfmKKv`*I);TKX>6qaIB$U*BeHbB`7{TnnV;M9Dq zB;#6BY7f+5zn}f+Da`+AXfK}_5YRpsQvtnh<(ycNT4xFIe|xBC_ONF-HBqd`LYZdj zWGzQhOKICA{L7`#^hz9IFY8!fDN2zYQ14U8#Pm*|5>J5+-BMMtQhcq_U*Q<%$dc!j) zZ#Q>{z$n~6gzfS&JPx?K*pNKuFHG)d)opPh$}{`gwu)TMy9Ua^wvD|H=RV+?Iy3*% z$!{$$9nWyzV5p$`VG8sKU|!J?xF;vZ<}fDME8i(NgcxGoa){xd?+E(>rDZgkQk=tQ>RvVA;|D;lU5iWZCp9>o&>6@k>0p%D>p+C`P5x@7o~K>UxGzA zMo+To&JJKTMYdhtwVj=Q>obj_{Lyie@U5O6SlDf6e2}7t=(1J%|2M zxcdN%IuF2n4mMnEhE8FauR|x7I~u~B;EUrImJoYZrD~>3X;;JZcr-aLGxGi^J78C! z5PCwMY4=uR1fk9V(B+%Zj|jJ&BUcO1Jf_a5;-1k4zkYCz`l%~*MT=-tsUV&CEd|1XR;!mnRBI3qfPB+r!(}hWJ zb{8`N?KlG8f9V*fayJe(X7?tgY9k8BHqS5Lxl`Ilu&lgX;(0;TlEc_scvTYF^!2-a z_)WKFUY@W`kF6b()>4{YBI@P#;QLFlBecu$nX3^c9pIMwDaBA6spzDD{ zS5!Y#^KmOPc}(oe+L~!uxWX^O0zG@S#8)JEP9CCOQsV6KvObgFGeOqWye7V$CuY-Y6~==2oEb@$4m5+NfJ*vt?G`{4V% z_kBNZteKNx2-n5e#@_-})^yPt*94O4whH{7#R#+s5o1iD<%7OO7h9VNJ^65E(8xth zb8dfT!-SwT1pQOj5MX0hOk^zxfUnIS88lx3t$;X9pb=m7-Kc{0;}a-ytdY&_#D@YM zO?*7W@u%_=0_}rdBRNjjGa0g7Nb-8=S4KI*?|Wp4G2D>p=Q@!?FB{KJktO$<9K^YC z$Cq5X!#{@W9p;H#n*%taE@i8Se{+LxZGYbGu`WBD%ZPy|nfn>3SZr01a0|2Wu3p|% zSTFGGAi7b9+CMs1JM!x^O5G!`!6>gW+vN*4neqOZHcf>rZ%8FRZ$Rw#vRa;yp+~cuga<~gUv#h@IXGov#he*KEF5b3GsT+dLVJhyM<7zd z1O5qI@~>&k#vK#3thC^2!Jl>+?<4%#0VG&+k>|vxV-9dhgE;%C=?Q-vnX9V*elbEi_5MTL?j?zRzFrqB4yg4Y~ZYM zpBOP|vPVJv%3do2{I!Wdw~9nlS#UEk=_aZ(DtT*g{Pd8=e}WBrFs^{Z}dlG?6h?xcb6MI?VJ3V z07)--Q|BWTZc`A$RDP`sp+bF{7|8Pqf{aigo6H$*g)X#e?-fw;qx+*({BE;F&A~c( z2F}Et1nX)g3Dr6yM@bgr8K}sFJ3lI=w z%Wmob%1tq9Wdi(_MBDyN_~;oLz3#~BBxQJJ%$s79@Z?g)PH2K{A*MWDbfxmsZAtZ( znW)y@%^n^|k9kYM859pF%P&l35DMLA0c!Zm9OZoM+mEziu@8ElptLv@IWDHVjyavv znEj?+QGqCs_W;R=rXr++M{X|z5po}E-<0w{yMDR{%7v*C-wPc@)Du5i}A zoKiRk{77KjXKOv9P?poIwywOR311p7EXAKuE7QB3OJyu-uze!Eid>e(F0#B|aRwIt zSkyb5zuT?afjgpFF#N4J+BvFk#(`A{zlevUAkHr!D)&{mp4K_V{^mx>um0Gw@wmwb zE@7gQGy~|mes;Qr*Yy62CY{`3O`To-PSU0oB0b&LI&(z+yrxFP=c3o+4KY5=dSGBp zb_9=kwdl`g!~vReUVBe9R}c9&HzoR?AvV3FTqG)ph=H@ep?-pI8h zMrb9QCJ3Gk(mX7w?sd?kD0>r6dWx3qLvr@SNkH;)qPpS5v~=y0LFWkn&P$g1biZM)Ud#Y9A zB=dlX;`|X{{kJos>p{PU=E?l8nMU+6!pB6D!1$C%`R1ay07RX)(R6t=-EQ$7?nj@S zc86rTD>KUld-e0|T)LMNiBKC7WL%!qHr!!KqQgzFsyQKOtMip>V}^qL>g&~+-98=R zFrstgC3WsdI-$C(H2&+{YjFKVSM-&)7+H#PpWU2=4Z&`}ETtRaY$k z1q$<)(pqK~?dL!?V@t>dhN7>cEB^uzb@XGw+^ZeKn&U1`=f6PfeX3$Kn(_jQ+x+jF zui~@^=DUYIjs^<&Z}Jq%V{WZn(LP&nq&n3ZQe*jVdX>6H^-aB-pD^E8(N%$MIj&r! zku+!?y(m1z^=^qwpyX;3U6%7|t1gUkBC40@`*1C~pRq57v|!Zb`$z2xVSB@ivyr3O zO<$vtiblPmG(zV5mRt7K!IAH{0biFQjw2A#?W$s_mWHJ-Cc#}2@x74=U#s7XJYP;p zuadMj)Mg|;3i8d)@R7!xx=N*fual3RLdxQiD(i|fVZlD}#w84>d}*^kZ&W;=W1x`Z zk2pGv(7%K-B6xeIG7P3+2(82Cz;uOY&6@%+t0rBK$6q0eF>*349LyxX78%))K8O5c z#@*kcP2{!A>Vnb!3J~ZFn)w+9P+d<9yEN<7ydnrj%YLT{OSl&T{B2VQpqU2|vv9>6 zDM4~%Nlqb!URnAx?;u=!R#i8`f;X&S)cw16Yn$oJ_4H`6w{HJtN|Z6qi#Vy3z7VtF zR(NA7=fy8Z3QG}$aVX9|2ehUQCN>cljZt@o{0j#~>q}gEk-L(b;H?Y7yvYw=etP(M zqb%;e@>Z;DrM^Yi(8jh|&wL5ufJc!Wcta=kj}gma zVIT*W+O~f*xVcv0)rTBzDYlbJbB`gAa1fAl&y|*SP7OKBjU2W-lfgj6ZBsV z31BfwO(ZD3{^BTm4~)9trB-V(K8pzNy;|?{Xa9tYU$%$8TY9gy7J-Q2Wcz|iKulra zzZ&r$X4wBcZ6dgfws{9xlf7JQzx)NfE*;)#AevCjNOaZPsDKiw)p>4TVeZO*X#q zFqpm28_3IJwFUydUA>FT$Y2D>=C(}rzKo72)>uy05c2lgPfw|l($G{GS7)QZ`YPF1 zz*2uey1U-tLt3ZaP2OY}>eb;a7_5`f@^+2${pLod4571lguI67yYpr^tLyB z4J*zpbJwSR4xg9bq?Hr~C40;H`8&}9W-okC5_c>+QgFUZh{j!IE{XG$@lbdXwFMGY z&!-luKll5x%U{Gt1xV!Og1Ng4*YSi(Ee(k@lEJRFT}HPKyqCTaroL&l?{6Af*MU1j zIUIFL}f=l+Dbt^9=bRgRQy5Z8j;yUS<%xYd5;zVr zTS#HE+b+ouW||U8MI21GJ)Y;GMUE~TL5!{d-5Kx7WrHm%ACIY{e2#6?l6~iu%l zIsBC_Dy|8qX#26#3j&V%&P{%`m8w(IqWS5Nokw@+9(YAmOoRe!fvu%tvZNZqX^=Og zy)vFq1~e!e)|Z$rqjDWFxc|53YFc~K2@=a3>t}o|5hQMzEM97+_p2?RqM4L*;Yhu- zY@>0Z$O&4fX0K8C_2ML}$zug1oqO`CMYBdR_tStz{R`6lN;a^Ku$gfgT?V9$&m^cf zV|*P{SdTq&aM+Z=zt1`Jkjs{U+Hgf$UW_q#-wDAOYxtBjg#&d7V1?2fq&l^vTw`)g zRCl-sC*Znb)x}arx7FtzAKU%?-vv2O>^khe865%`GUO<>ypB}_A%~e7?mokK%^N-5 z7dG`!dE~3LCxk3csip+EQ^w-fH|MY6YN#%RDb1MibO+}iZm(exRw>H08Zg>E739hCwjoNHveUiJ-HJdj}8{2hbI^NbEL6=ZmC zcF_G`yTqgbJ#c+9^d&rhmSDDZQDR~CO#vJfpS!~d-X13~zU)l&Uexff|51BS*{}MVWZv{CX$GZ8%Dz<6+8Vzz z*4@5VsZQYISOSer*SEtb45PHnuD4OTzCWOx@D!%uYU!!eI+TGsJ4_}B+27cJilV~8taWH6KyzW; zW4Bf-&GJJsvTKu5Ylkwj)>p~>vUOQgs5-=-y8PGa4gR!%vvS12hY%jkj4%a9p^h{J z$#K)bKAcOT>~rP|kM{t2cm)p&Cq00_OQ|_VDip*>QpY5bNyqgnEF6{|8TF+V;6SR_ zOq|TQxs)DGlM!icZ5pkTgiCztYoVx`>vKRUh=oaxVv5xi)WBfS9`{5TRre>T;}YRSfzv^iz8a>uDR;j%VhK> z`hb&!=2ZzTJg1Wrr1J!Oa}`bzyWyRGUKdTg;as-M%?(VfsEjB}wR|}ZoC1{C*;QFF zAfts#UVVxB=(axkgvVN&xtROe4iTqnf*XfSF)&b@gCxPpgk{g^RJ;WM^Tru)S6&K zT3flnELKD1fM~xjbZ*h8pt=rC*yT@|wL7(qz@uuC`UOatI*FM`pJY}S; z$=PU;C_fO{px467Us`BsoF?ox+gb3mKDw?iI_U6OVh9ZRZ2byFu_u33V0>Jc_#~>b zpRT1f6c5((W+cGpMjx_4>*MnEZ+w=c7ImunHfH%r7TjiL`;w){aZMyUhDTn4Hc57o zUacEHkN-lOZy%KQo4xckt~PO&P1^A&RY34pyKqxI-LN$T$g_uJi8)5qAy1g?$@r$zeZ@}wy1?%^ zL~S|mNwhb$UGswvZOUcSu4w^8gvsLT^MZ<_VMq`o`#`>VwS)7?lZ+uCsnLs@-M;#Q zeP%*|1D*SIvFd|tTe;nBv2n)lE)%1uN}?i->u-C7Jn`c38*{^Rtr)W%q@xT5yA((7 zhg}caHNg#K%DOrNVx7ztix`R8yQFdSL#qQ(Qz*yx39^ROOZFlC6}=W9jqboP-%hV! zN#=Z87BiAtIo!>+7k-~He2{pK_7V0Eqs1J#%n;BgM|wa-m4NgQ$J5=sX_`M~LITKo z`FKno-&+kJQmCoWgI)ObChVpUe;=KmoJ;jrd>=2@6f)Yn&%f3WuiYcUTX8lk5f(J6 zqTIMXT~)#N|J0NtJaGdzs|(P)Vh$AI&oW&LI&jMh?_gJ6{8m!0x=wFUpJ1TrD8GtB z29l+@3St6b5A4OvQRW)B6_)S;BFoE{wVe-A%(rSLv-N+!EZbZse<~aD!4Xr;cduA^RBI*RXMs6 z!1R|g5J_h9uVF}I1M=d?u_)ZC#^)Fs5k7MS-USXOf38ER?M%#BNm!JQ7sbje7BhdO zvnX}+5qx<2fGb%hBWV5Fx*3CkT$I*It7s~ID2~3bzAUUKZC3xiKmB`63<-nu+oX}M zCsZ0xi$9N=@c+egu(11UlCDGcqH9R)*LHb)MJ&Sci$v~gaMBbcYO$8YCrK{)6-O}p;8<2(En}pF`WE%mP%8n>#_E%pU(+~ydubbDthrMr=?YZ{~_n@ zdznQr_zw`ZVJKN~34%Cp{#hXcnF$WDNSTxn_y@bvdcff;3S2IuP0Rp(y08YXALFTH zb4@M#<21bv+^a#eLBzA22~RcueDr^+=IdmrpFeuc(4fjZ%L!Kt)s1HZ$74uKjJlIbjAb+o9%p^TDEqP}0|18B5s|C((MRpNhzXS6F?8-208k8?SD5)2Z*X+!BvXx}1iT~Y z6c4bHF0!=5ujecNFx3+B(?^oDNjEr7mPB-4D&HBe#KGI8RfRF$&V5@yx+rNexoHx{to(@hxkN*l7Xl zx7T?!aBx{%oI%~ti6F)F_ zxt03Qw3**FPh0!_TpH852t`m;*UsbI5uPx|zZPW-_Lg}vWL0xEbafK)gwWl4O*OPv z!mG0M?l|!dQgdhrSJTBCvl*}y_CiJa zrE0cYIw?>lKRczgCzr3{TJMVQdu0Ghx3kjb76CXL&P5moDAvX@!E^_CgZ4SCyKMTF%Yr5H@H{F=yPKG76en*nK zs6%u|Tg_WC9sRWHnRe_Ba{^!wTp7J+dJa*qNd2K|zJ~p4BinmHP4wfwi~|8wTyi9h zk1AarUt}F3Kz!vVzc5zet(?76-ZJMXtLLHC7GzkOeEh5ws8((YrcdKs;+I4)*s;#n zZ=+6(W%;2EuC^!3C2e;fSmhMng=nhfumt~CEY>i>TcY@4uC)h$bo?`kB-*FQIPm4b zhC0*xgkh9(IpRdoF!c>sHD$GTEvtn5nzJ$WXP1SH<0X8xPzc)4>SDd+Bb*OP>r-{l zGJ<8CKHio@3GS{REn}e0ove9%ZU#KM4h$M5|G}mIyM27b!p-LWqJ!~)?f&BWFn7P} zlMEu(H6^Q>$i3uLwWdS!rk=GZ#JC3$|8E}zipRDbaY173`*2|_iaV_I3>677tDCo! ztsQ}8x|SyJ>W%f4$Em2kyUMqz;r*y;kiX;;@w|GMd$a?t7I{pKP=jK_jBYd07mbY$ z8p;2PgxV|UR8|d44Gd0gp}cBwSLwZMQ18`K{~d*s)}h0OZ@kCXkO{r~+a^Sv{RFMb zmJB4Gc?kIo#<#GWB$*gL8H*r^FQ^=^P}S2^&mkTlc;9P1Gz900l_Z_n7vvZLbJ_e1 zG#?c#XiKvY)3w%%-0hMMcD6WIxYTcf#K&jQm==3+@5M^zHUq_{3tf@Qt7>;lQ)r(W znjbSbInpmb(KIr*QL|Dyr0=}X4BKYPpuC7fBDiUqPtUIq z;Yn&ncNp^`qL-GBA_2q}u3%cNY7K#+Xr0^>|Jkq#X$8HARSxvFNAsWNiFj2fM7D`} z9HfQ`^eJZOX(vf|7%bjwkKxPWn=>~*Cmp2iCyzKK;iPaTP84cwruTpC{=OknoqP=G z^t?5CU9Mep_}xu|`(VDuz#iX<(2jzbq57+RbZ75PMA{^g$ZQvgG4VlciOtBDmz>E6 zqB5%&B0GQQ8k2q408ln=Uac)pxS>?8GJtohYXZg); zrLZJai49qHojb>I*cVsDT4lQNtKq;^&v&o@{#Od@P)z?6fN6RL?mpzj|^An8R;Cs z(&?e=c>bJiKFxXM?e|Z!{k~#bLIV5Zp|m=W!>V08ot{;f&L?e>NfiF|0X%tPMeYnY zZ1wg?Rv=i%9-Vn=xRb$2!#|Lm1^_&7;aKf@+s+Hiz{{0f0&c6d%M{MlI-S`ZJvHo~ zf5FkZyGQew6$4dV`JO8rI}1=O<%^;`Jmvb?t&K55=Rd^ zz1E9Zg$T_Tag}+Xxr8ngdr+{rHC!G)8P`%zSEir`r>68J?mCgr7AL`6PF}UopcP^6 z@u{6g57xj6P*&N+BivYwCMkz}^W~<&sI|;FeL0VWa(=#%%-}XHD%bq6)B3AF1*5Cz zF-#&kBrF`*`fErYjMC{??aEzB7K`#xV&*9Y?%xn68X^0vNd@CH!wT~Z_&Qn$?f{iG z>eJ-eQb?bOVcFO{Emwl>FINf(MU_Q{ma5 z4ft=u={$c=$!K-oYOg?Yv{2a1nSe?0>KuLYXZUhjA#R1^;MhrAd``4VqsV-VnYVs| z31{f9>%`t6XXL!wyGYku15}hg6sNiMmCCL5gXInSRks$@1~qyuYTG4jxr3M-wIZry?F=5(x78@vxEqts zaMw?t5Eoev@=1%Lji%|3xPNlJHp0*}tMM{Oz~xi`c~SCJ*~-0;R>^MgQ`iKkeIWh@ zX+rpHD+s|t^R1`6w3e_272Yd`p+P{{nUnO3rFWCR3 zjtSnEnrH!rf;lvcS1*q**?DRB%EB9;!OV2DmWegyfl&PfI^3zc2Fo?^c=y?-On%%M z!Jj$F%c#e|zBWxOkyDewMZKH*ngK)b3X#^xdq>I5H+%URYFj>`y&qLn{fM9sRy~{y zpOYrBnc|9xU?K_`plc#Btarl)pa849qBeXA+kp2|Wv$SMZK1?+@+*p4BEB6Ae0mEe z)Nj=16}o1FODc{$bfF19KAbap_7v8YntDl%zK_u%U<_|2aO?RR3p;&;$t}wjJD@&X z6zP<%UA9a|A>2<2(c}^6QV_&r4dk71d_7-xQi@7l*6fCL_9MYqwL>pTh>> zHDdQMjAD%F;m+&RZt^MCQb)6T;6uPLhF;rkYDWcjhdNOXRo^lJiXzMp6?e&Eu0A(4 z>x&h)Y+Ek1dwgc$3E0m{8231YxV9EPzECJB-a13_*8Ct z!}410CJ5o!?Ksu@Ip;{36@ijdk*4y*5Pek8xx>)sewPe0B^g_z!w`y z0R5J4APcU^HG5O4_9yByWHSSfZ$>05E>18^-Y($KP~Qhjw_qb1lnknQXtrSTS6ON@ zrcdxN>4rg|PI$?VxV`)%CT|_5xYObjtI$=TSaDHjIcr66R_t}psK0PuhNU7~1atku zJGK=CVSc3xk{b~&*7rc<5t)H~0!Cnqm0Ysb!dt=BFU!MUBJ4(I*h~aIM>ir{J z!9-g6nV(#Z@AWbNS3R2+rp>joLKN{vg@#jmemhTZVBX^v%K>h3 zQ_!VQV5Cj!7|;V_`<+N|G9g9>HxUxg2<{F63!%1nd; zSjR>dI!`%v#^_L96iI&F=l{P*gfMM(_Kz&H1Li-Z#{EeEJS-0ZsO=WbmCbsr99RmU4S}TRr@L@b=bWQFh;>H_D?(2oeKI zDCeLrXU(-QC^I&@tq>d7kg{dw=JTH?DJ?^Zq&4 zo_ojIYu8$PeYUlnKJO@0Be!U1Q)zGltTbF3Gn}2WO8lQ@|6dVX|5p7j&gZ|W%3ZA0 zf76HmAEL5AQVed?laF%t#M=LjJFA%T4ttuRJDEs}RJ-~=9) zA0kD5^9}H%H2B$kJ{$7j$Ht?uS~S|U(#gL6pTu7tdY365&Um*anH~aZX`#G9I6KAu z$^b{=ja8QFeA5}%u4v880DzgGaJ2-$1;B^z6GVP%zbACMH96&|ZMUF(pWb*>we$Cf zzE-CghQ2c2<_!f>H>ia;CpP+pnQN?oudd=kfjsZU>-(&JiSQ`VrQp3_QB|-`K>nuo z`J4`rAqJjH9NI)>O*dTm?E&iPv*?zi$#Kf_K3rtdnqZ&JQMRnQNJR`Kc7DP?Du_vO z?Yn!|Ei18tm=RJGBR}{y`|&ZphR&_20hA0I2lu3VDH%K>?ngFiXY2hVO)`6SWkA=Q zol5};QsGbOyp!CYiVds*Uo5%;U+nG231W)M^M#no*2xLr(>$a29%`^oMzI@VPIc;< z#JzJ*z4=6ijrH>W6g58za#Q8QXL+;#eva~b6diZbDkRjjMsQFW|fKY>l z2NYKN*+gGN4BpEE42#OBG0QXXn)u0-WJB|5e*VE|1EY8dYx zLw_9mh7R37#aO^n6Cc9jRluFV{6h2QfjrH`OSbm}7~U8ldas57nHZIFd=HF$OvAV0 z75fgkCsJq$irvTyQ3$&w0Y=4(6LQr~5Y@h@-Y%hCD&w?_TLuvtI2D;ogU0C)cGBK`hpbQ-Iu z7naXL6QE-lRpdz8O-zc+2Y_G+Qom*pSVocl!om>yf)gS73!?j*8siEBEO+6V7RsIk zXbT{TJF$t=jb1R5dYFePAr{%L~5+x~mocLwYKLv8p6zVm-8zJH=KMgB>DlY3J3 z<$twx_j1Pi@7*mY&s+baCC~HMzc~F(1^O?E{=d}be`{y{vwfldM}uic-S4{Pe@#2T zK)3(Vg&!aO(X4&r=lPj0LN0G({J(ujxxc_-7F7R9s)P@0n*Vwn&;2q z@Bz&VHsoc*zuTYvN14xgnNCv-G;X@HZB~^3yXMWWwZLwJ1G{{#k3;LQ|J%4*oKB3T zxxGkfuuQ(mMcAW%V`Eg%f~33*A57kr%z9ab0Hd?_mX+DT;245LjQ6;PdcfE+R_T!! z)7uae^Py@%7-enCnEg*HU+eLFB`2NoZWhw zP2Tc)cv=8|&;EjQ_*m&Sb|Ac!_W1IQfJ2fT4X} z@kDD$tU>LRqeg|{a_l!YAf|s3M46(E-Oe_JTvN7TT`mt*0AcHR8UszvQ&HNM@D;+j z{?K^ZggAuz0>(O9QbQe}$uQ!)?zWSK{~(`?uRWd_yoEM!+1?GaB^F?+@OJqy)v&>L ztNhk(x9A#<8{CzN$%HY{!w1ZES8FZ*(4Le4+S3rnk+3!?oCQk-<5!%`T}3B@Mz3*0 zZGX>J-?aD5z@=Bae2Etn<;{rB8*jr^=l}F{NB-739}IG8KDX(8WtB=9km9*Lhlo&5 zk`J84WHJDNQ60>pJNQcX#a3747uc>=T8{bOgIA5WjIPq|@5WtFJFPKujK_&+%r{f3 z3bY2yevx0GakC?G#TCoYv_l}tN9Bgya`s(_g-6aBR9gyg9?gk`Y5U!yyPfBlQsdL64CUcAH zK;DVH2cme9zxRr}>G$c9Gj#XxYi3rNdzViq@N@QnKP|TlmP*QT?i^4DpkEO%`p&PV z$}ccI&p*W-U&#Cu88O|U^1x}$${Wc&Ee!}KK@wBv74{WYSlnOUJ1xhJ<)-qwlShm(=t|8gX84**qv;=ev49_NDop*ZW>SbFs;>r^9VViMxr~x<{V}tTa4% zL8G6juWKsr@=0;kH32fb!(bg4{RL#yQ3~{$u;Yi!$pah}Xwz%^QhBI(=Xdd;!Eaz? z52p)EX=or;Z`#eOta*j#RddRj3`9H$O{a&+l#JnV^^?x?DIm^9`!}5c>`A$FcrrUk zn&FrsmrpuVgT=`L z);>#n6eUoho_o?~13|mPczMCmf`nslH9CX}%vP}v5nGzBq_bp#6svwJCz_o^Qi!Eb z2*c!w>(zeAE&9FM2366EKY|=!o`5aQ%;a2{*l^d02}x2V%f$=JW-V}L-9;5euIq%Q zz+u1e&bk#bXe3Uol+$*pAoA@#%3X5L(HY)?zAIy_$9;{AdsQ&l&xJ_fn0?>f70Sosp~ zB_63A>aW@Em>fPTv@yDMKnY@B{U|hZ+YvFho}N@a_dWZQYm;c2aB6$TLLJ<-HXL2p z(mv-Ra7>GtQ_uWXDDkn(Yn{`p*Y)grb}KM<#^Lu$WNT3K3)86N=BK?SFS$P-8VK0j zs}F>2gl{)Nr}RH(garyU7A8F`kJO?&bA9;HN!#9}%gbS)=J_?Yto7ngq@{wFk6co* zVte>Q(v#A3Yf-heL5?;oHL1#KoYYu7%t%gE&#qZFItS||yQImeiyK!OXNP)k+HBAz z>uXhp&uy$nB+DHB6~*;sSeKW-l9+sH)U@B-jL?@Gk@ztNdX4Vd0|le|}U(({Xt;~eJ8F9}>I+~>pJv%Eg;7lx?z&GESFKWEGl z&P=0C6vq#(QTX-woA-*nxcMRY(Tjua zlJm?fD4EXDcCR!8u;{xQC#+<7cEz_<=tq>XMfWwOw zgyvqqVXTOrN9m2CtepzzzcgK5Ea^jZR0U5U82(A~r48Jjh@}L_a`czG`l{sk+ldFp z^4mT0`lFilSR;rYoCcZ@egdzD+414OtoQqPaEQiIYUt#9CQI)7bG%y>jwP;;>A<$k znhS|MckzP8iPrKGuEy_TvN?!!@?Ab=rl2vY~@Efym(5HVa-x6D?LTl7(C$7UvIyc~+@d{cRkWYIU6{|n-z zvOqu58ZV{NK!{P>tf^0{JU4497?!7egYQfXzuxYtK4{sUx!7!5n8|0(eR6M;es)_? z<=&b7Cx|A)XlM!IcOuynGjFuBi&W8wOjlU zs7q-&QaGPGXkz)nh%hXr*B~Ep7Jd4b7uHiQ^Ugs2bSSY4vHY9ZoUh6tq)$Q^Pl#W- z`E@7?b}&;ojuN%prr)<|c`lK(r??+pOBPec`uUNB%BoJsp4fV03t4qUQF9L`Rzby; z^K4c8r|WMec}aZp!gY^Jf&B(yxO#yN8AvDNMdXPVNi`*XmPMRKCrwWdE0rB7JgA1$ zC_|d9^u0g|A~D$GnDgDu$K=y9%(QYHsmz!Y7R`}UhXPnl*ivd35u7b&?<|))Sb3#W z5~Fnk^CkH+6Xu*mMP^HUE?rGOH}EIM2^4?dKWPWF^in%^BAi zCv_P(t+9W3L6P%RQ5#%5Q98V7QmpOp)An^B{A}PpqDYo3$A2= ztg3Dqt&hVKUo`cvdS%u)vOlI+-^wr0CB2Q-mJXyLU(u%euA`e2yxnhxOm5$MefH|2 zFq!|`S7`==^WTzxK5A~2kh{I9(t`_atMn|eINi&cTcU-a4`@lwW`kC$Gu;in>P zq%ymkFtUr>GU%vYGc%sikitJFyP28nZe-ps_l{87Wd;*$_3#BLtehDm2W3QLvG>e# zdRN>Nut-j?*V+Qfp4qqfg^3i|oKpeq9nV_u4u7co zk`kksz@ra6>de!Zkz>t<3>LhyEEgx4C5(!8;oQWtjkvK@y?xTf|iCC5sLu4Fh(-`{f14r<~1x5qKuKD`XJ^RuT2R ziECr}C~({g!{4|_TcdU`?hVUo5aB$0s*5T#ee+8;3+Ig- zrP6woGy^JUK(-8u9_i~s-8HbwSV?M)tk&%`X1ntvBxoHfp*pA|HJ>L{T&GD+&G*Ow z3@q3r-#_|QxX|iD;+VD_is&8~_f=!3A~#)T^Vgx#^RI(p4cMu%F%{o1aEiJp{o4iA z6PzP1Ok(JTTpYlX8LCe}&kc!6AV2qn!m165K!DwrHWw9)(G1yoCMAiYN){qh4gdPz*? znds)lAyAiR9QL-EO1jZ&Bd!&D#PBfEFcNn}M^r?;*V6-I|GOw|nj%vWspc=(!(@hG zTIn8(cmXY_(?#FO#paXo)UA?ucwl4PrHur%FLf$W_WV5&83-Ulg~p`cCG?l$miv0Tp+|Pl29GVi$QHkpTKy}l zO^bQmO3?ti5+;Emojm)O8s5p3uGUEV=mI2q$q7RmRPJH?CAjhVEET#rJ8gmCB1uZM z!_%>PgfURzjs1#x@sAr>@$ezDpZ14zT!#U-_ve|OsRN`Pp(-PR{h9iojcQQSG@eTD zbElkW8O=gyQr>8hSu(zJDBpCzmu^I*Hy3{bIQ|cOIJ*9Xv(;^L?v&X9VPl=Nc>Mlm_%@u zGeXQ>2JICl3Z zY%J4XL;(A1#8Ut(2+7&IH9l3j^RX;XOTvl(2OuZF&J&$4R#~0@+qMjE`7fs`z&n@6 z``4p*hxGd2mEGn~cTOZY;p2>3Gr z+anPs;EBajVP$(C4wxM2?;NfREr0=D4mVKY!<~aIXKeUCzW{UZ#os3141Ts#GPtTL z!dvJOvr6*pc#h`<-`VaC8#u)~*#=7lUiMoMe*9;tz3h!D1KX=B_|t#=(03luAfV;{ zki7mD-2aB8|MKXd@tuzh(8W#~AP{GjRUvQT;!Z^sk2%@Rr{xSjRP7)b;}awEwTNnW9O5 z$^Kid|0&52y8Z7T0YmnG@Q3g_C3_=hcR>sE-`^(V|0e|u=?5@=6PJ@c`y1|Ub^E;{ z!L9L+shRwc8UOgTgAUvvoVY$UA8wR_1wRU&&7;<=0s(^;K$E3@*CbQ6U=cMl&VDzUmXFRy38mdine^9m;U9Xrd~ch|ddDtZ`Ek{GBf)913~q?a zr;}$N*i{Y;%)34neGW#OpByMa55+%gTlPM_8uQwaO3kip3lmF6$&PhEM5v@T^5HJg ziFFJ#X6R2UoGZ)4xQL<9#S#gZ+-2pzdwT>)L ziuYCFVv))`JAOF8*QkXH8?%vve{zuD}uSU+Q9gZbe;sHDZrG3gU zA?)PCPOxrdZK2|NimWCXaHCI8V@(j_BNfkwrBbDIkNM$W;b%90s79j8$>mMYaRn@3 z4ooVHTi+WkH)DJBWee)M-fTg_KO?zC2F@JDQVaEkAyrfw!5t@VMCHZ?ji(#c*^L{FP-I!7n;t%`ZV`-6aZY!Aef;pIxPVR&eFOB5FN%RM1v%)JK3xM4I)M-LeIFSpuI=s2 zVTn?!;2yhm6>&<>{1NfCt{{asf8MjE7XRB=M%C{h;vOUG9b%TfumWcTE+y!a9*aF+ zj_Gh+94JPb-nKf6_Ngx8Y1Uma%!Dy0n_p{ex*e<%ZbQR_Xy&a+Ia`zS&)kzV?7-e64)T98t*b|->E!SO2MdxRIfzG;M~J0f zB^I@nHzjNMHZ03D|7pdVElyBYVAi*p+L}rfxzi+`KSAljkHQy6&olJvz}nL&su zzPmljPO*V>3!Y4;q{sfb$>#^r0m>i}sn{@)j z*O9Z7bG~43ZVx)6BZ`Cz*e!(jx+@KG`&Yf~g(4s`tkb>Wb|%F8zuT^el|b z!E(^56`9Stb2RIX1Y~WldN?juz}k;#$eq2_{S)y+NavK=sUR#XcGZB|k_dDEK(Rwc ze(fyM+1yL$3A|{@5keRM*KUw5r7H^R;&tIlS!+$n*Eet0hQW>qd9Kv>xYn`M{n^y=uN71G(^NT(}Fd4j$#KB>3XfTc$Sv?CRCsK zI1L?%xKtDcCr@dXh1H!sW<7HgQniehjFn#zP2Xh8+{W0j6>j8uGy25i<46&aWKQbzpUc2yJTB|RK)3H#H6oX0RQGN%J zGNF?;6WT;2^T|l?g-m^AFb#>mH5TGflad|WuVuk^rN|QFKU#<9&g0#o|Cs>`Ti00E z;M4f&kyP5Pzef3irWHCO5SFr4ns%RA%)5#Cp}E#G!m(l-a|b6)x-V<)!48TldFHR zm;*CetjE4^E2q~tjmPy!whxmsJ}^~aq^>(?#_D!8ucBh%p6b?+Kj72T-lSpySNan3 zQbRpog3FXfbe)*MqBj>Z99sx|y|tgn70-bV;uFPh(yU-X^H)-cCfpb)ooGLs+U-TM zPtk@HC;rzRl!`=Bs~!Sxp~;0~g_CwGtoLd{No5=ezRyU@&lkLP$uES{bT6z9a_M(! zxX!+5KhKphH1`bFKHxKUw|{ns^V$yjTAAzu=UELsS5cuWWp$4^!6Zf0Fr8K|H*nn<1Bm%Q%=8FY7Y(9aoZLD9t^>vd0ca#oApMFc9r zhCElV^m}g$m_V%shYPj0)IN=43h)Co7dDP#%n>f5mpjo-EPGP_blv`CjoPX(Tk3_q z({T4Lt4TKIj50`2xTDaxCIvb3P~{2?F3gi5oJ1EOm5jI~=ke=@=+XRS6Ts!ebiXLF za04l0^bG4H(FX{#{!!jrN)=_*lWio*ug7nkP@c zZ_Yj|w^~{7HrVHDMKtZ&MDsQxioDptMlal5MVXG=AOf(a{%LQq^|hM|O=yPvGLs@H zWOI$SQV@ss)bXS6;7>6TWZ)V!zFZJp$Daw-MlAJ35;WvZ&L!?^L3b$meoHL*-SRJd zbozU)dJm~KEZDu+yjpuHfZ2LniJPt*sKx1UY_`kou4Jz}79vmr#`77pXuPl2>(_9w zPy|oy8c5^gYxJaP%ZPb1wOaitm;YO?RI}l+3I{epH91eeGy^nj%Z;JB5V-?lIB+8( zgo7h#o?|%~kH`Wro}k7xd1q_9a<6ED^l)^jL`e-376g$XzojWF;OhBGC+i_KS^7nL zx#H!Fz!ha^PR6tK1T{)!PQxuNsAtE+%4bZ4b0kJ-T6f}R2gIG7SMDY_YY~=0|I>f4 znz9WQSM$4+KIHiY&(Z6EwU4@uEpKympZe`dq+0OEC9)af;pSzV&Q>gpX1r^dgjkgu zz2~>q0&GR+ISf)h*o~n@m4UE&8e`EZTI?VN(YcT&PZs$=n8jFLgwMCWQ9cdT&Y0<( zWNH64Uxq?o%pI9bkjrn6VN!QG#1)w`TGJiD^gAuK7MH6)Dm zoeS3>c$X3gF^XgD4rSbm3F{0+_Jnd_PwU!y&q!fi0;(8hIyOEv2Sc)2oPI_2uhOZS#R~h_$T|%DU?N0iuW?zIsVmgro8fOfh}(lS|U;z zj-)X6WC9)+goNy|Vv5*J1R1>cd+}aQpdv?dkOJj|1U#}vIW(|~E7Fw-jJ~?aLGm;P zQ0ePH3!9=3FPFW+&*#3}{z&ypd_f@?B=*wm^Wb%wh3J=?;aZU;6Q#`s}bf$ZftCvPQqS3>k3-MqD4kLkbx z4#mL)1sVs>{-R+GSdP0zjzi2Rw6@>2q4v!f8ZpTgIHV^Qs7}mzm z%4EbacEkXj$D;dbwgBAFrB6icjLbF0rz>bP^kAH6Uv+XB_xuNPfY!1t_m3}=)v!R4 zkt}MKyLYZr5M}WB&=m{$7CQ2BQ`^Kiwvopba^P28qu(}v{#&=sXRfP|>I^}&)Yhei zR+iXMk&UKfQ*5UR>UwX6*%p#06{w=3wScew{fL!T9Nr2_doF zF-pG;1#?HE@=5d}hi&baRrp0oKt#!1@iY>!9I}tuB|%>k3HUk!3hj(dV#?n`4)Z0CC)U#&YYuFtcN?N>bT}y7 zEM1Z9l3W*O{DJ!Xd>K!#{xJhwLJldFIAz=n=hlx9A9_uFT)%EjmYjEqszyGesc3$B zO$^|lcc)J*&W=&5&%yG4AMv}w99pRGGuLk=fcY-Xym6FxWRv&N>Txe~(nk8XYZ;!n61Jj3#+4p`4iZ=9V zYp;zTj8v0tD)s=U>xuER_qtt=`$m;vlthJs$m55~r8P`rGO9~NO`#zxo z_BU6yj3p0#Jt*u)-j_D*UVaJa7+`)|9!5x1Uq2jADcVT+WAfErv?j)iJZ~&yY!0SC zXNDwk8hu2p{y<&99F~HbnY{Rn7eo}j&p&lkNuJ@bvcRrAxgLh~BU;{S{t-N3=(Uk2E^M)NJ%m>2a_{xpCewpZWviywyW%7n?68q4U`>(k7 zi}j9-1|i^_kk@;=G9`m6H?DIjp}dY1CBG;Hla#q$J`#scF|RCP2kYnfqzBYTA9&Rw zX9A#5LwyUemtcK6k+YjPd)-^|53V;f_L8#_#gWn@pdUh z3eM(_E0->46sdO0w8r)**s$lP(5{VGp~-wgxa%~^WfaFHY^Yb+MfjWPNA+!uqAXxL zA+)#kiWYQfAmB2Ggyvi=rR?`QZ5RlCxOhr+kVy-+=p%)T6^?lax!xRXdajyBa1mhF z&L`E(80())0=t1x=MPTHhGN8^#`%29L4lq*v8e&S^;^mJ@1VFA@?VKh@g8X>@>LWc z9XxLXO#2OqVu@^lUG_hJk;$VPYXoB5N<(jG!AVxV8!q&j*HqII_S{m%1Uj2xi)E6c z-+hNRt7B%LRDjcbCBd8S%;W_#dc8cdhMsKgW6%2NU$=QQ9hacv4nv;>c0Fba?lRQ2 zXq2*?$xg>O5bqw7*t=oB;qBNTiW5t|><;ewH9~nP^1uljAga#UbY;YJ$tjm4S7i}2;X|3~a#9@$ zgVQjL=hAcTE$C*rW% zeAlYwD7+{T=fUgAy0THdM%>z#GHUS`h#^^_S;r+{1x;3P;Q)g_b>}ZDo2?{yvx#w| z<(l6SF|Cp8072;miM6ih+iQI(fnChah5wP@_(xS# ze(?BRRNa})ODFC85hKcUubs`h*ZtZ_@cA{`uyQP{HrY$jwCqEQY!hbDoGU{@5Y@Vp z_eq1`o*)N1xT_Lnwrcdes9O%&<+eB;id5I`(@|4I+6371GHDqRIiax|J`T9uEEn-Eq&|2 zE}KL!*t@)fx8SqnK9tTTMh?wmx?8qAJA5bzQVrhp){!e1dBeE6t75v|&dwv6^MJ9f z*F#t3`hD+fbY7aC_dRbSfq0EhH;I~`;^vBHbi@gDF9vvc#J1Yi;hUSwm=QMsJ*gf_ zUZVN7kUzHAtal+1P8|QIEc$elr}>Qq;W$E~o7z8YePLPw-;UiDd7&jk*ku|J_T5>e zlvNt;u^Hd*&Z8wp0;&l$J)rANt+;*?S^_|rWi43T4wm1T!+$TeSMCyKD(G2xk~822 zwVxzLU3!(9JkRMV(bbeiAW&U9is%JFPy=8S@6AChX8m;`%ToaNw*AA;%uTH zR~DHI?jkzSyj1SqXqJ$GUp?s|O4T{yxmFu(c<|myt84`2N((+S(~UmjK0`=gE=5P0 z4g55W%QE%^I_Z<)ss-e^Ci6;W;28&ENCIlWKi}kjkiN}|pEa&)({=8`CR%(%*IF=; zrKc4wdh~rCoS&4%8CJ#2f}5jA{YHC5!gXkQB6s&ULvUU|SGCe7AtC@eT4yA^{J=gkjB#=z9Y6aa!X^OCb$ zU{mz#qXXkZZIw<6t2uo06^=8J2Hvy`eR_UUICG0UF{J1I1BI8Hjkmm4tJ1CiR9OM4 zRq&G5Qx1wucJ%)0@3Y=;zbv27j+%g`iVQ!#BvViI;7zq>cztcZDRR9C-Q!>W7hbX2 zCXH0eOC2X-={~Vt}!6!ZZv|sU1ZK&#m{G?shnd)5+NaW`f7J$+3jj- z(_56)GyM$9X)*N)sw=NvfS!v&QY9>N&Tb%QsZ%Lelgn7~{*e;&<%`I$WGO%T3$y*2 z45u8=w^4x#wTaKo`~U|^D_8Li={OJje^|#Oj00c^>Uwt<#=GfxsA}|~J-453 zwj)%xOgy09$(SqnhNa^_r(S3^tv}Scng}ptbm?$2_`%55ywsirFOL8YPWlNP*5*Jg z?@~JA#my}rgn5quCY2fO>F?#t0oJKo6AbJM7lFHN4cD{Or?oNtm7}090d@W|9)IIc z=E5TCMM z;{LaKsn*!r6Q9izvYi7UdPeBN2u6t0Q`d))(ZNOc_22MoRvC!~V* zG>I2$XFkhfh}$(sh~_&-sy#(DDfI(q6zhYZ+&99~W8ZQp0rP+#@@g7+yM689Lkw#7 z$vgDhv3oZa(DF9M&;at-AL$uJ0E4BN<2`RxmG=J{aze`;o>s{jKw`q9_#u5}ds$zu zcxOC=A*1S=Gb8q>?zH4{!iVo>3{vcz2IUd^K5>@7%P8IXnnuL8k?8*04O%b-jiXy% z-MQ+=8=GfRtC61SP}f{LohvyT5hA#FM9x-b^Z;-jz<^(=xf6Y5oV#Dv#>MIl9KwgV zX4t*!h(U8s6*Qh)4UodYqcY`fgHsh%cR|x%z`F5B47j?7({4^kBy?D;5`v83Fbyx(pqS3AQawR-UJ9e`e& zyl-3n?NXTObi#d=#T`;d2X?tUrtHcAXa+@9huz(`7lv9Qa5`dzzPhAX_-CW({Tff( zuhW@Q1lMXz0(qpLwCi=G$hRu?D*89)R%3o+i$&$^evC|%mGYToU%U|*k3yArF71*l zYJR&I04bPMs~-yd>1@~(D<4DM)>$lPjJ#v9e?4je8T9|4KU}{#D=&Dae^x$<-Z;KO zjD-rlT=%0C?fyUY=`yO=-?wvO4Ib_5)wbdgVpTYW|y~Nr10nT zKRtV8l^_hMp>$s*DNxd@g&sF*(1+L#RLh^DyLvjUESAqqtxEfU)FH1dD1ikC`^;u~ z{Ity`e$~{=UJn)=teyJ#=;mEvvRp#&$HPxTGINBX^C{cJd;Lwl%<~1dGHL zO9-MdFy5HSONm3N+D|$Q*&<4mFB5QFvaUtJH+_$`2~Gv3z9*{kNjqtE7wq!vMkFJz z#vX1ZdnTWarbNs;I&ZTz4!re5zfeTW2dtVx+)AAg)>5rD#y`Y1wxWsxVR+uNo)Q+c zjBYyT*lmj%lZSwd`@ppPH0!89x^;jEb$e5aEUP<$AI&RoI)2q61Z9Bg@p#3E;A41+ zy`NvcoE)JC$qd>LW{QSp#qn^wUDXd2oT8t7T`{)KVn$Zz&6B(4cSr#+xUvh-eG-QL z?%*jZC7PHB56<-Z+^NaPisQ!=HsU4aN;U#2&=A$wBzcPCph53K=Zk~&@T)0B%kWH} zKd&1-e>$NzZ_mSbcp!06JD-*X7oVM^dg%RCJfgt7Wtca1?7*QjTq5KHYK-kWX<*Uo z?4Kd;X3E8Fy^S#1F0sklxNw_b1Kw@O{Z&2uo?ITka(vHg{!nr8+wD`K!m;F_e)(;> zoA!Q0Q{&h>UFZp|Pzpr7>F(W%ojJX}FD%a{-G|poQ0t%hE{6K5@vadg2}kmu%k2aL zRn-EKw&mC^q--DIu1WTD>P8>5>qd|g>4&OQE@!@G&7HPz=xN9)FB|ai z%{56Wx7WLOqs1?cX4zuub_k?eUzeVX3YkEw=YH+XI)m@7$T*)Xj>%qeUA2l`*w2A& zQy+5G!zWFE2bI6R;zZ^c?MkYSsNZua-Fg4?_(Rmk=p!flLHD5(U1%8@uu1$eD_E#` zC9<98!1B@ZEWut&NZK%w9V|}x-2;--Yk@fYHE835)^(bBw+A4o>P86#wV*H8HHc?- zF8XlQG0hf6(sK^qbH7nfbqE|fe;;PZQ z5`|u`we$s~(%wlT_4p77cQeo9tV4u){S$iUzwX+i1Vu2+GzP)0h7Z{>3hjGz1V_0c z2ULGe@_o7Cd0up(es4DD;XEt8`(U?x%n@jCW=R36zyYpwA>Bs|UD`~!b$|Pv;chX_ z0fLXY)YuPSn4n(Lf1S!qvZyqlBtfvRpQ8$oX&fe*siYXj!8qEqF5|Ybf6??CqF(Yh z$(mrf$o(WHIvN1YeI-f!<4b&U{3}nAEaC@jLFIL0ha%4WkL)4)=nY!^jtNL#bDYz; zE6D4ondV%fk*F3lH-W#%Sn_ehUzuWLdm=h2_(KEz1AB7Ymoo>6FsD#zf&BN7&VtPF z)oczicTvnF1YU1R)rTH<@#D(pRNkXO-$SxA98NAho?9*$l7fLGUccW>n%vbU z^2M=C@5gAP1;)QV1o{h1iC-GtCpgD*@d}fdaqKdV(F*%oz@U|my=JgSK<6iGMTnB^ zZbu7Ve8m*7M&O5mKyJ~Qr?Dq{;TKGbai0z3K287rHF=~91(U-OUa3oQ>^aivL9c|o@7>nZiG4b$RcCBq|GpGp zska;F(%?3wv1n@0NWmTO*QIxpc`!M!3``@p@})DQ^(UflE;JxPdzQ>+mO;*&ds3NX zs$)L>$dT27HbqVSgA3)!T66Y9A}sGN@W%0aUF+6PA--vh4!7#lHMQ`Ns*dyRdosp2zL-;3}{ zQVPe3O_rCaHWD=CEV z%0kXsR3@A_OYtfZ$D@jw6yz@1j{9Vq_9@@$qHq9E41mv`_@F!wRxKrhU!KO$+(fLC zca#?v3M=6Kal!Mki%B=Uaqb^GZ&>(Epd0Kw(pzHkEyBk3a$-+1<|vv-wB@UG=1+Xp zkLwZGbBU*Z`B?L2kb_Jo-gaT+F9$n;U|8cufk-|HAFezLe$36mwN~}A5M*=JrPa6i z(Dzhf14~=63FM%3 z8%2AnLlrv3hakCDrp4v-=kW(C5g4R~?$aZ>nY4)UHm{&Jjeg7%o<0$Fk%v+BL@0aan-!^;1HXcuD zq7x&@3eFQc+GiW#A(xDNG=eY~&2B3ncX(YP^Z-S(-GXZ0pN$NSD-+)o$E6`6bSwJh zuG{uR&u~eYqHQ$uP$aQf=bCi#FC#ilHmjMpZvb@D*JS8&I#tuYw&TcGjUVRtlRutW zG=4{DfGRsnfd}BTY1W6keI$>iiKPvZ8q%xL3GJEe6+AU{6=e;6(nS`B6fOw$+Ii;k zyz_7wcSA3pSFA=|&mI`e1S^R2=K%LhG@?FF9tlg~CiY=TN&DYxF*PdlNU>V{Sdm~H zttNxn!*ATJw5+aPt^~!`H(e>g#15`)77RrGVYO&BwmwjSewuwvfL%7Uv-X(D;@M$Y z$%wetcNbv`Gh+|V*Zn?+{#RCqBOgrC2TsDDuFP6|o0QZ|7QOI|Rl$*2Dp=Xf6@uiL zJ@krlj^QW)gI&)y`MV-Ww+<(&S1x}=%k{4>{cdiSQV5PtKury>9tra&vW}U1!&Q}Ny zgyFYRulbi(C+X{5e?|Owti$?8qyL2R>ZiyP(Cl2I!OgU2V1pbGwo4QhL{)Ff|XQ(x}sL3+@*vYkrt3+1##c9KK~5(9@L`H|w_+;cP}n-%SuHaZ5d>#+MzpVm0;KUAZZ2We23U z!^PI3;?em^4(9&cg!xQ4 z?2nl&-2OHHD?o%XI#?*3)dP#k$^BXb`UD26${pT3dmh1*f)RAaD@UMTHE1=e@GSN8 zv0o>j+-~)AHa^5{o)j-QbFs7Iy3<6^R@2&Y!oK@>0k$?3UP@RRl1c~KR`w|6PV!^Ouaa4I7?4Yff)V9VO?udub>HyNAF1N|A8nRa z-ICAq+v`QA3$X==Vl+fl&!cW%OU<(sG@q0X9IRA3(1YsSBymTpw8aub1eV2O89kQ< zTIK{5RaP~b+5LEkwG}qIQEiM}8=mhncx^ca$(FgvFRl%pZc4QYJOrKY%Z~G9P|doN zfpn?jXhwmP%X<-er>2A_v#!>>cmOK;L404Ews1Px9V`W{%*L93jUfKYx-B7M z>3#aHHAKw4VU&Xm)Krzd>yGu==@F!~2{I&O|Fd-}jcU%!iL3imm>jn%Y-HLhe1Z&( z*!FA~2r5yvTK2f8WN+}7phmuzKiaWsLF?%oXnlH+-Eke=_A3H9pPy-UfmK0l=(eyH zpbWtHPicc*bt5UjnX50orR2Nb^@($MymFAgp_SlQ=;S><5LLD=PzQ(v!;$7|*K%Zf z>Dhba+6YOHzNb!pcj8<3qK6PIGM(>BXTZNiyQ#CXy19_N2F4QA0?J~{5CuFTJpwpcuPP}V{8$s1H z?0w_BOxo@9Gfs!&y>0p1lWzZ~a?g6Tu42SD5PVnjgv#zR*M|~3oAz~aGjneqS~S1k zBGzSnPli&pT_xcCh*yI8l^2Q`tS6|<{38d4{mfvk%Jm!Qg6YacKwSnHU)5Vj|9dif zT%{`k;#X#7k!N-W4qJkDzD;JPjuk|Xd=p9{tj+MYV{&`{N6_x7U&6#dW^nlhJ(!kGukBGEjd-M1vy% z&HMhBV=OP`MlTub(w(XM+c1@siyxdV?>qq#0|dxGJ}b~v6;@lY5YqmF6Fv?AN!nk^!`pQ^E^Mh$?`CSTI(I&4$i<21uXfRJXp%nj^n~2XsyZx-opUFrW}iIU`kYmw~l6 zx*98_wMd+Imh@vj=zD5}aQcX=lr0Rh>hJ%6T*^W1rrqDr>g5MNmx`cHL9}M#ZI+3d z)!_xKr{i&y%yH#GmpuYXpOHx^`4$ZIaD~C+g583n6|cVwo;RPl=`>*9)DkcH#JD^e zZ*LaAqBPb*Q?lPVf%8ycHB;)%QeLXOtn^c57o8`pnik~1iI1UJW1UED z-()by2fkorY$}ts!2>Cqf>-qJX+R~aGLps7CSS@3@3_#h^_^Tq5c~#hJC!o*1L!e- z^)6c6CS+0eiVA&wO{UElSbq2r$vJN#G!ZPEy0p(@p%&V1!W~`I6xF8IRVY7kO$FKh zOryo4RbBZU2P2`sbS$k34@Wv>S#{JX2RA|w1W>GKLZuA zs-3M(w~@RwhW*^1`*!lHCGvgJW|j;F(hv5Z$;C>I$?-7+mfLFG3m(si6!a)i2tFl| zZYOJ*tyD+eRTnSa%&VPcVE?+AjqvV!)2#2|z^=8U)^TnynrI+=-M$-DyFEi8N8o)< zqJ*6u#;ZUrs+|=0&*UBJ;;ggSpGl|s!BOTu!^2gk7NH6%4&stOocFTQ#5)FL2e}sC zw_@-V7+aT-s&{T*DTKwo8*g;rI9B20Q&a^(YN{q=g-^@3>5v=siK@t?(MMVvCne?b z&0v+?_yUn`?Mra4+)Wz|2i#)@EToaprt`Xze(Tzg_)oHpiFfwzc{0=gi#MZR}^dsBvp z?Gy?S!SiS<^ezzv0Ek}_Ad4HEM(aty0qOgO@q2+IYPQo-smFV-68s-PnX8;jigS+flx?;*=Qrk} z>Y5G$4b&Dh%Ei&eJO%njp3b*VJqX-^*OOQxzNpNAvC5g&o@tjXSATInfs3Tq>DUx; zzHm^09l0n!y9$Z&s!#k@j~6**WL(GKdn0j|mEy*C(M7Y)(3Nn7eJaU~{5d&5+kxO) zE>8tqo{{u1j~!p&Rs~;ArhIWSA?fT{P~G_Qh5IBE9S9~ydmc`<#GD3FELbAFGW;t) z1|ae>S;^~&a!1~tLQ2v%3I>mD0wIZc(TDN<=P&QCEm}?ZRWlm)%GR~;L2X|7K4HJ1 z1Pw~UZT5K{|4EYpm;jtAWYmAzA&Q%;g1E|6w~rm^L5|iR)Y=Xe$F2qam<(cKha#tv{K0YzT!a47@#LNU-mJv%?W}6JOR$pYr zjEIt9y}SH)oYGwUtIeD0GtN#p_;P#N33+9U$&F&TkK*;+4UHxs(R2Wei^e;s>9vG6 zbh|muJKY7^#p%@6Lb5$HsSRW!Sud5g8uFh)jk$mLCNoYft<-1Qgx`1H$?k^3uf0(K zQs0SQ57&}qS1g>Lw^=Lr-f<6GKo}{DQj;6~`w=`feSk~&SnR6h(cV=uCt7K9nd zVj+GHhidm3cugZT)=Z{rLx_6*j1@@J0XovDe1(}#R<3Bd$bcAnbm+45BLY3a?UEkg ztjE|s!S%!~aT^TX-l4Sg=GUjXoAKp<0Tu^@B2Sq+Kj&SaqwkVpphkUlI9?ai*sGx z<)P}!pPXHiU(!pk9vv$d=UK`r5w=@9<)&wWZ%~v$D_G${1q#QxCtp5K&mj<0UlZ=O zkyrCj+6%WBUfV6W;tIP=2-X%a8x2a9zm9l2$PF09P_i20wT{uwBh-ZjSHZj1w zpn)$Sh=E%tIJoFTV0@!56`G~VbNy|%ZiGYTwL&a#0WSRq4`v zwX93$O!`x8io6K}@)H~C?U1LE_1SOp4D|90)*1puEStp?j z7#9*&GawyPYjq&&Dm>MFpD%(en zDeTML#>6XMCYm0g+M{T`l`Iri1~sIrp2{D!igI%tyJJPh>y>)LL+&{u*+6PaXzV<6UY`BZb@Za7Li zwKA?TX2Nn05v3DUS}U;zQ?LWQo_-;Ia9NYd=We-`y2$<2sE}eeRJYF(E3d0~p_eb$ zo#j)aY8VBjcZN2Dc0yKf{JD9WF4e&tqtG>!nOJzHVHYpFxVtn5Yo+c6 zEa7no4|Pm}%5FawCo^I1SvL>N{?oT0$GT#`6DydQTr*!ptWuDr}B z`!nj|wLi40`CW?y;_xFVjytz!6SD)2UYS%onRlkYJo%ovfSR86^j!&+&Nc-sfF@x% zDQD94q<5q%x^#(FLZoZ#XpwGMmO#H5FzQ0Uqkd)HBPD}P7un`nLvN@L^3BnzHdQ^_VA*s;t0O-gOF!o#LA>2ro^Qyf)K=}4|#LxgKE%6Ccq{53ad zg!N|vBr+~BWrr-Q9tth9uJW+N+i!AFQ;q}^wM0Gd?3PnR^s!^+veDpSpXLK!!uRgjVfGBL7el~2O zbj;X1T#{P*xVJZzaFVOWDph~&6t2gPlo+VL-+F(*eqWqED|VAKd}d-V&ixwi?OJMg zh0@gJ(21i_SghD%46{T>%c=^kkQ_Sqj%o?N??HZQS*Yo}y^;MKhVc!(bKvXWr;W6rF$7)SZ_NNSk<+9la zMQP!@{I=Dj{J1!SQ9+?NYaWS&ve^Tsk^M*EOl@2T)Ni2!G+?=G`+;T!Q6LjEARG^Z z!RFcXbV*Mxlpxr;T3xPB`kIB5E$Hw?`-aquYy)Ri{TRPos+tyjgr+pPV8oTyR`rwt&{HWW|bnFz^$%R$xATe6enwKW7ramY{w_}mMKOeO30diiY zVKa+eV{q!VsWO7amz|hMRKGjtx;iqh>NX#U0P+Q=fw%jw!}&m`yc_q0gWS>yz}-jl z{I>yhmp2^%lY$oeQ!AZJ5!{C#?EcA!R=Y9-SRDWdLX!Q zIX(Wu;(_ZH1`r{BmxT@-hDiY^?Z2|Pf$sgE&E)@26=swD6*K-Di~yl@x=Vh1y$Vn# z-GLRf0YMC-2<=5}JxO}}lQB{t#o_Lxo*?JZdbXZ<^|4KN_AsT_{Ex(1xKpo6&yN&5 zxI}hc1%_h(FujH057WWrEg>BZlIVc+>o(Bq(eTE2r*EeG?8}*?hw*fz z_D28;y)EAlcHf%|;+jhqe0?HyP%{O|M5>9vb?@6Mh49@2 zMT}Mp&k7N^82B0p$~{~M<~o}2;I$uDku+2l;=lvtg>+U6$Lyc+z~7Hp9@l=Io)nW6 zN*C1I0&uon&u*vzxXJORj7C47MiO=I#hb_x7c_ja1}v{?ke$3z_lV6#JrSDFFvPnwMq)U+dC z=g84)pZ!MGX8`b|wMwEV|4oba9n>gn9yxskoq|xGSw6Tq^jZw4-AW@HsC~X&{;qZ> zfycFeq_S$KZMqT%D5cj1xmJ=JuFjq5ornM4kU zi=&dv-lZq&(G$gtX)$jSo#RP2x=)Ta3lWpVOU?IV9(aT_Q+vW@iJCZ?q!pUfWV5>3 zYJVk8GJ&$#Ef4e($8}^e#}@g6(e$4IHDe&8?>7|BV_izKtEtmr$*J};j9ZzL1px4_qhPD&XTzFXrA+O9pJOC&E?WJIeX;;})}PGwhSAfgIw7NA z?r*JL;&XWC_5cv<7~wIo!!uw8mB*Q_7DPXU;DLvH@&JI>r?tR>K=JJaZ&3i9VlL8~ zYq4x3VYVv7thpE72su~)))N_MwJ;i}96%TM#BM-#y=s3fu(i*@`q&M@3qaR?%1UVV zxI-BYq_|I`b??NfTUB6-92k5f(O;!W@OR&P`Q|<~`YC(zlURjeBKzHw1N&G2K}<8| zlJ1yDP7=XC=h6Tu>A(Mc5(3+^cfg{1fXDK;US6Xh?U7Ba5uaetJ)HcMg=B>VAa4J` zhekrZ_Ydgq|5G2k@)R9>k@lx{=k{58d?U2sI;K|xbk8zuHwITI$j9=MJqFvD=(y;? z5rD^Wf`sJu%?xdNsCo@?Gw4`^3SfKBZpbb%sk-9jH&ln~G( zCv0w3pop7wtQjKt3Ygp4#|be&5zyWFWU6PtC}tkWbOS{)XeWDpVZL|E;y%qv^7027 z2N7VQEZTp&Ao=7q2H^h7n8W_jjMXdUA8X21y0cDwvdPN;+C5TjMTdB{+NAq?pwOHIon9TOG6B%{jqw`UaBcECp}hPp6kpB*phZBwzz}rr%Kx8@j_MgO zfoFguNsvBqd14>_n{H4J^ZZjn*(28_lwoF1{vjfoe?PXLe(sC>5cenm^BVQoRqRc`2Ng$2T;ochALJI z+!xL&^XI|D&*yT&ivt*u( z*%CH)yQ@obo3&y39Z*jNc)-u(row2}cJVDPrUJ_7YZ!TsD7&x9P|wuEkJ2WA3|z^5 z%QFO5h{W5EoZg#5E(9c-N44*#!QB@tV)ZFP31f~EdLz*tf%8&{BZup@FE!t}HUmAZ ze1+gG3Ufy&&_wmsn|bszr_%K%bKEebx@vpxMB7FV2<Bmx3(PXMN4{6zbpCFCui_(lJ)&<{a zsI@J5C-D?9l{(_l820e(ZW8vU!U}9w@yEr5SBs~FND}mqDC&k}=p#A-TkUn{W3dh{ zmXjX@JuL#(<`IMObc1!o1RrUi(VQfn6s8l)*Mu#zE6RY4ox*Yz!qO`1(O@i2&9uLp zgwTl8Hw&?^vQa`fmVUylbb|<1n7gHybOnCFn2nT2i^W=2f2fQYzep@-*Tup? z%5V3O={~4zJ2)kVx7K-bf2WP7@x}>~$2r7n@f+^Xdcl#f9^$W=_}mB2nPQY8yE)Kr znYz-~Un;@a#$CpI#ruR7PcPajKPw^m;zr630n{0tXAe8G)Sps#oJb-c zAL9Q+eiqpZ>oQ#p4|P}=hFRLJ^ExktGfr7R-};f6I+HfmY$U`=cWUW)8wemGe?}Kk zJT-xlY#%P~->`KvDOJ|+0QL2He;q78OJkB`gaDp`iRcdSw{wixBr%0h4E{pR%N|4W zl1s+HL3;B!m)@n1$>e;p241~{=h)1ccX%#?+IDgB6!@DXBo1m<8)z!L`l=vuSW(3@ z6i6G#Q>s)L;RDMCoWWO|L$k8nGWk`QOh;kaOPIW}t3(6tu*XDfY&AF=s)2Y*xk4!|4CQRfP<2_jMxW!>d{5~5d+OH)!IL|NA*I;6LqR`*BnTGg!B%Z!u(S>Qb~H=hkfLp z$L}#Vx!-y%cpaIt|CD?hlKRrQ%%2#1r|y83ewr=!u$lD*=o^y+@OWbrbk;3l(z-R4 z8}j>~ul=Q8F&uOwn6E!-wsiU#*v@1rE}0sDdn8JcglV}=Bi$47t`gpAR<>Eq8lWtj z6rf%dIMlsR*XKa#9`WxT7F&$V*qZcMC@2#s;B83u#lDnr-HRq__z`yd%dIxT8T?D~ z_TDusr`X{AMn!Q)g)FE3g@P$&gB<1afa4H^;W~Fwz?heBr?W+%^2p1K-zqX~v&DEU zZOS7`8PMK;Q)1#{ejTR1==9yQF8`w3DGg}a=GAXw{-$NqwpwHuef~Qq0Hux5;D2i8 zt!UHh8gZ)87yr-W#9#CW2F<3F&rRba z=!hjAllI{IEJ61T*AvPm){zZXjcJq;3-PaoTpxgcAdDW@?y$S69=0u=;Q(YbkPhp9 zk}I8CPE9v=!QnU7a^r&}CG}&O@G!wUg$eHVW8?*+Lp_O-&k}A;1SKmf_;-)Y z>NYo4IQKsI3+T6^1NUV=ogVl9VD1ncSUFVc&vaH*{Apy8KD1P_Ap0Y|wy_u--w{6R?BJUry5QdL z+JKpNXn0;byL(lo5UC`vBpy{r?;BQ^tWgs8u+x^FG~8#?3D+~9lk|M+?yxwk6Ed(V zX#g7kTEZrI9;LBH1)B@%4+8)Kk*=O@t+tpB2P?jiT|&r@9Mv|2a~65!=nFbipJD`s zn7L27SlcY@<`yZ8h8T95^2cZ$_LlyjM-`BpFHMgpbde@{BU6n(y*cqxer=W4`R;tu zOmfV9kTYVJ`?7OgM77g97~8Lg&_JVGK0BHs1P&MqXLw_Mesqf!A?Uv{_N9xSP}pnD z?7kFB5x$`y;^iGLX@033Mr3~Ndf13t*R75r@Z(H3N*y5~^K&}$*CXGL#{nuo;t5<2Wdv~fn|I1%{a|Gy$#dT+!jMvD3*`fUbLPJdxh^?eG@M`Xtz1VN;@*-@5(TVW5K za~{9^HWyT@P9It3;44g}T<03lkZ>4>TOabXbmZU?M@aaIJE`lh@y*_57M^cMCZ-k!S^;-u*qjcVJsP13*)M9i7VcKKw@{&^m3-1v@>wt_+@A8X!h7R5~C2xCz#iD zg31ihAfFA<^WsS9#cNQwcU+sP7TU9i@)yGBYxDuc(51SFee8pPtiS<2l4fI>@e(4U zDfGvr$qHb|@pb6Q_nW!7;H2VT3aY`ut^nxQ3=$8k+}jOTz%J~AZrbyW$RDLJs_c7S#WwJxvG&=^GhD$!=La_XVL)c=h5!c7}7cCSgw<151ZfyLW0}o58;3 z5F&>1wPMr;Sk}mt8I+FJMLUp?AMH!y4?`EoV?Mw_9343@1LkLmsE7fR!(P9(1V;2u zN5oBQLI#0#-Yjwar-Me_6jQ5oAi9}u62;f>oO&2PO%65*m040|4LK< zFc-Ls4!1)6vJ|Yn?LnNQQB+xJfjYHFVPExg4{GNGftUPK@;Q~K4$;6HdHV8L(PhPz z%!zY~kM~D)ekG~5^AKi%#EGKT9Z32*mVX8F5oC6hdHia~gxGO0<K}*=1OPD*9z;XqKs4lzT%pxciBT0m zq;A9Q_3M}Ue8@WEm))AUUWl1;sMH%0IWG;%e;e0s z-(O(?0RzZMCuLJfn~> zOj;8cYV+Wc12Zs?b(=CH#{>WTO&8d%8n8xs!C?+@w+uUNLhvYSE5=YRb+up#!d|d@ z%w_g`JCW*z6a3U{JbngdD6YfPwY?uoIBG9g@%?OQP^{+p&D@~8wXU`vX3p}Lbl7rB zIwKgDY&9?7sfR_?uSD^qIqimvzx+0rVAhY{%KMZ15@O8SIB)XiS`I3aQ&d3-%&FlV zY{uBm6vE7s<5q=K```L(h7}~;;BvPz8rstLic&QqXKyowmOr?vD4#*+yy#S+pV|5! zNUt{Z!Ux*NRe_K)B#or&gcx&FNS}%d1u_`N2WOlin3K>8@uXf-R2waol>z_NLwY&7 z)pZ>A(W`B{pW5X*an2vrMvrbFR3OyJW}Zm8EH^})aI2+ipO=A_w>}o5oy*!KumqU; zJY>x0JOL~mL1ThTx*!HMy#78^6dI-3(m-~Sxj19;q_ebkBE>1)q^7ul^O9;y>6q8^ zig-b{x^tx1kzU(6iAkkHu=Vqh?h9_D=+*E~XQN=dWR8eH%x?Fh6=#E~T!uE*OPe z)j(YR9ap&f#T77j185&YB%MMV#li z)s>{JV5YeoV#yf_-2~Qu^$4Y4aCqnkHY03$xo-aSY;!7@q_k>Fr>TZLH|#H0WjKXG zo}*XyR{27$hn;SIPQM81q_I@;XGKA*(Xq4F41Ac{VKj>|uN;&7b&V~gVS7yw3X;AC zlJn9FJ^Bp*+hOY`tRjI;w7BT3v<)5o-H00%{Zl5}Du!0HGzvh8J~*#~k5<#J?7?Mb z?Vpbni8`~&ZfvZLX>1lwi`v<(hR+RpzJ$p~voR;yo+)Q%oP!Mdd+;| zzPz&{dF>i-ivIQ_Yxms4NxUE4O|$a8;M*OU>dUlx)H$*{%vn@>@VabZW!iIqB=aB6 zNifTo;~cTS>$SO-|2rPv|Ar#`yH9kf9lALy1Sq%vg-$CChH;J|?J~Z)j~49OMD?!@|EkppW0FT$kh~ zsBmpv8>?ga`J!Mt7GNTByH`(x?%Oj{tq~jn}BPgUA0W7@NK`;m3(t)95dkiHm{8SA7BAlBTHo*+OM&DlY z^p#QlO-U-`@HqMSU^yNFaoxB|R4|g2O>D5;SmW$+aMQcy+zO2XWU?D}nyOPd6H9P2 z*bDn{oXnS!I|AES?&5-ZoD6OJ{!-oy*xoK)?{Ry@^=CK|Ni^K z{4M4jNqDaGnOG9g7b-hp6+3+$I|Cj)8w21M5*rH}8zT!BBO4o(m79l^jfaDqo`r>n lh2HJ;=(dQ`R_h_{vXUHwmSd- literal 0 HcmV?d00001 diff --git a/examples/nlp/question_answering/conf/qa_conf.yaml b/examples/nlp/question_answering/conf/qa_conf.yaml new file mode 100644 index 000000000000..6c9988ee3546 --- /dev/null +++ b/examples/nlp/question_answering/conf/qa_conf.yaml @@ -0,0 +1,157 @@ +# Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +pretrained_model: null # pretrained model from list_available_models() +do_training: true # true for training mode, false for testing +trainer: + devices: [0] # 0 for CPU, or list of the GPUs to use e.g. [0, 1] or [0] + num_nodes: 1 + max_epochs: 3 + max_steps: -1 # precedence over max_epochs + accumulate_grad_batches: 1 # accumulates grads every k batches + gradient_clip_val: 1.0 + precision: 16 # should be set to 16 for O1 and O2 to enable the AMP. + accelerator: gpu + log_every_n_steps: 5 # interval of logging. + val_check_interval: 1.0 # set to 0.25 to check 4 times per epoch, or an int for number of iterations + resume_from_checkpoint: null # the path to a checkpoint file to continue the training, restores the whole state including the epoch, step, LR schedulers, apex, etc. + num_sanity_val_steps: 0 # number of steps to perform validation steps for sanity check the validation process before starting the training, setting to 0 disables it + enable_checkpointing: False # provided by exp_manager + logger: False # provided by exp_manager + +model: + tensor_model_parallel_size: 1 + nemo_path: null # filename to save the model and associated artifacts to .nemo file + library: huggingface # [huggingface, megatron]. Used by S2SQAModel and GPTQAModel + save_model: False # save validation model checkpoints + + tokens_to_generate: 32 # used by S2SQAModel and GPTQAModel to limit number of generated tokens + + dataset: + version_2_with_negative: true # if true, dataset contains some questions that do not have an answer + doc_stride: 128 # stride for splitting long documents into chunks + max_query_length: 64 + max_seq_length: 512 # max sequence length for input to the model + max_answer_length: 30 # max ground truth answer length + use_cache: false + do_lower_case: true + + # if true, context spans/chunks that do not contain answer are treated as unanswerable, + # useful for extractive datasets like SQuAD + # if false, all context spans/chunks are treated as relevant for answering given query, + # useful for generative datasets where answer is not necessarily in the context + # used by S2SQAModel and GPTQAModel + check_if_answer_in_context: true + + # if all, keep all doc spans + # if only_positive, keep doc spans containing answer only + # if limited_negative, keep 10 doc spans closest to answer per question + # used by BERTQAModel + keep_doc_spans: all # [all, only_positive, limited_negative] + + null_score_diff_threshold: 0.0 # If null_score - best_non_null is greater than the threshold predict null. + n_best_size: 20 + + num_workers: 1 + pin_memory: false + drop_last: false + + train_ds: + file: null # .json file + batch_size: 24 # per GPU + shuffle: true + num_samples: -1 + + # default values for the following params are retrieved from dataset config section, but you may override them + num_workers: ${model.dataset.num_workers} + drop_last: ${model.dataset.drop_last} + pin_memory: ${model.dataset.pin_memory} + + validation_ds: + file: null # .json file + batch_size: 24 # per GPU + shuffle: false + num_samples: -1 + + # default values for the following params are retrieved from dataset config section, but you may override them + num_workers: ${model.dataset.num_workers} + drop_last: ${model.dataset.drop_last} + pin_memory: ${model.dataset.pin_memory} + + test_ds: + file: null # .json file + batch_size: 24 # per GPU + shuffle: false + num_samples: -1 + + # default values for the following params are retrieved from dataset config section, but you may override them + num_workers: ${model.dataset.num_workers} + drop_last: ${model.dataset.drop_last} + pin_memory: ${model.dataset.pin_memory} + + language_model: + pretrained_model_name: bert-base-uncased # main config to select model (between bert, gpt2, t5/bart based models) + lm_checkpoint: null + config_file: null # json file, precedence over config + config: null + + token_classifier: # used only by BERTQAModel for defining the extractive QA head + num_layers: 1 + dropout: 0. + num_classes: 2 + activation: relu + log_softmax: false + use_transformer_init: true + + tokenizer: + tokenizer_name: ${model.language_model.pretrained_model_name} # tokenizer that inherits from TokenizerSpec + vocab_file: null # path to vocab file + tokenizer_model: null # only used if tokenizer is sentencepiece + + # expand the following to a dictionary if special tokens need to be added. + # only necessary for adding transformer/bert-specific special tokens to tokenizer if the tokenizer does not already have these inherently. + special_tokens: null + + optim: + name: adamw + lr: 5e-5 + + # optimizer arguments + betas: [0.9, 0.999] + weight_decay: 0. + + # scheduler setup + sched: + name: SquareRootAnnealing + + # scheduler params + warmup_steps: null + warmup_ratio: 0. + last_epoch: -1 + + # pytorch lightning args + monitor: val_loss + reduce_on_plateau: false + +exp_manager: + exp_dir: null # exp_dir for your experiment, if None, defaults to "./nemo_experiments" + name: "QnA" # the name of your model + create_wandb_logger: False + wandb_logger_kwargs: + name: ??? + project: QnA + create_tensorboard_logger: True # whether you want exp_manger to create a tb logger + create_checkpoint_callback: True # whether you want exp_manager to create a modelcheckpoint callback + resume_if_exists: false + resume_ignore_no_checkpoint: false \ No newline at end of file diff --git a/examples/nlp/question_answering/convert_msmarco_to_squad_format.py b/examples/nlp/question_answering/convert_msmarco_to_squad_format.py new file mode 100644 index 000000000000..4e1a99733afd --- /dev/null +++ b/examples/nlp/question_answering/convert_msmarco_to_squad_format.py @@ -0,0 +1,138 @@ +# Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse +import json +from ast import literal_eval + +from tqdm import tqdm + + +def load_json(filepath): + with open(filepath, "r") as f: + data = json.load(f) + return data + + +def dump_json(filepath, data): + with open(filepath, "w") as f: + json.dump(data, f) + + +def get_context_from_passages(passages, keep_only_relevant_passages): + contexts = [] + if keep_only_relevant_passages: + for passage in passages: + if passage["is_selected"] == 1: + contexts.append(passage["passage_text"]) + else: + contexts = [passage["passage_text"] for passage in passages] + + return " ".join(contexts) + + +def format_answers_into_squad_format(answers): + is_impossible = True if "No Answer Present." in answers else False + if is_impossible: + answers = [] + else: + answers = [{"text": ans, "answer_start": -1} for ans in answers] + + return answers + + +def convert_msmarco_to_squad_format(msmarco_data, args): + ids = list(msmarco_data["query"]) + squad_data = {"data": [{"title": "MSMARCO", "paragraphs": []}], "version": "v2.1"} + for index, _id in enumerate(tqdm(ids)): + + context = get_context_from_passages(msmarco_data["passages"][_id], args.keep_only_relevant_passages) + if not context: + continue + + query = msmarco_data["query"][_id] + + # use well formed answers if present, else use the 'answers' field + well_formed_answers = msmarco_data['wellFormedAnswers'][_id] + well_formed_answers = ( + well_formed_answers if isinstance(well_formed_answers, list) else literal_eval(well_formed_answers) + ) + answers = well_formed_answers if well_formed_answers else msmarco_data["answers"][_id] + answers = format_answers_into_squad_format(answers) + if args.exclude_negative_samples and (not answers): + continue + + squad_data["data"][0]["paragraphs"].append( + { + "context": context, + "qas": [ + {"id": index, "question": query, "answers": answers, "is_impossible": False if answers else True,} + ], + } + ) + + return squad_data + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--msmarco_train_input_filepath", default=None, type=str, required=True) + parser.add_argument("--msmarco_dev_input_filepath", default=None, type=str, required=True) + parser.add_argument("--converted_train_save_path", default=None, type=str, required=True) + parser.add_argument("--converted_dev_save_path", default=None, type=str, required=True) + parser.add_argument( + "--exclude_negative_samples", + default=False, + type=bool, + help="whether to keep No Answer samples in the dataset", + required=False, + ) + parser.add_argument( + "--keep_only_relevant_passages", + default=False, + type=bool, + help="if True, will only use passages with is_selected=True for context", + required=False, + ) + args = parser.parse_args() + + print("converting MS-MARCO train dataset...") + msmarco_train_data = load_json(args.msmarco_train_input_filepath) + squad_train_data = convert_msmarco_to_squad_format(msmarco_train_data, args) + dump_json(args.converted_train_save_path, squad_train_data) + + print("converting MS-MARCO dev dataset...") + msmarco_dev_data = load_json(args.msmarco_dev_input_filepath) + squad_dev_data = convert_msmarco_to_squad_format(msmarco_dev_data, args) + dump_json(args.converted_dev_save_path, squad_dev_data) + + +if __name__ == "__main__": + """ + Please agree to the Terms of Use at: + https://microsoft.github.io/msmarco/ + Download data at: + https://msmarco.blob.core.windows.net/msmarco/train_v2.1.json.gz + https://msmarco.blob.core.windows.net/msmarco/dev_v2.1.json.gz + + Example usage: + python convert_msmarco_to_squad_format.py \ + --msmarco_train_input_filepath=/path/to/msmarco_train_v2.1.json \ + --msmarco_dev_input_filepath=/path/to/msmarco_dev_v2.1.json \ + --converted_train_save_path=/path/to/msmarco_squad_format_train.json \ + --converted_dev_save_path=/path/to/msmarco_squad_format_dev.json \ + --exclude_negative_samples=False \ + --keep_only_relevant_passages=False + """ + main() diff --git a/examples/nlp/question_answering/question_answering.py b/examples/nlp/question_answering/question_answering.py new file mode 100644 index 000000000000..2bcaf8f9eca2 --- /dev/null +++ b/examples/nlp/question_answering/question_answering.py @@ -0,0 +1,89 @@ +# Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +import pytorch_lightning as pl +from omegaconf import DictConfig, OmegaConf + +from nemo.collections.nlp.models.question_answering.qa_bert_model import BERTQAModel +from nemo.collections.nlp.models.question_answering.qa_gpt_model import GPTQAModel +from nemo.collections.nlp.models.question_answering.qa_s2s_model import S2SQAModel +from nemo.core.config import hydra_runner +from nemo.utils import logging +from nemo.utils.exp_manager import exp_manager + + +@hydra_runner(config_path="conf", config_name="qa_conf") +def main(cfg: DictConfig) -> None: + pl.seed_everything(42) + + logging.info(f'Config: {OmegaConf.to_yaml(cfg)}') + trainer = pl.Trainer(**cfg.trainer) + exp_dir = exp_manager(trainer, cfg.get("exp_manager", None)) + + if "bert" in cfg.model.language_model.pretrained_model_name.lower(): + model_class = BERTQAModel + elif "gpt" in cfg.model.language_model.pretrained_model_name.lower(): + model_class = GPTQAModel + elif ( + "bart" in cfg.model.language_model.pretrained_model_name.lower() + or "t5" in cfg.model.language_model.pretrained_model_name.lower() + ): + model_class = S2SQAModel + + if cfg.pretrained_model or (cfg.model.nemo_path and os.path.exists(cfg.model.nemo_path)): + if cfg.pretrained_model: + logging.info(f'Loading pretrained model {cfg.pretrained_model}') + model = model_class.from_pretrained(cfg.pretrained_model) + else: + logging.info(f'Restoring model from {cfg.model.nemo_path}') + model = model_class.restore_from(cfg.model.nemo_path) + + if cfg.do_training: + model.setup_training_data(train_data_config=cfg.model.train_ds) + model.setup_multiple_validation_data(val_data_config=cfg.model.validation_ds) + else: + logging.info(f'Config: {OmegaConf.to_yaml(cfg)}') + model = model_class(cfg.model, trainer=trainer) + + if cfg.do_training: + trainer.fit(model) + if cfg.model.nemo_path: + model.save_to(cfg.model.nemo_path) + + if hasattr(cfg.model, 'test_ds') and cfg.model.test_ds.file is not None: + eval_device = [cfg.trainer.devices[0]] if isinstance(cfg.trainer.devices, list) else 1 + trainer = pl.Trainer(devices=eval_device, accelerator=cfg.trainer.accelerator, precision=16) + model.setup_test_data(test_data_config=cfg.model.test_ds) + trainer.test(model) + + # specifiy .json file to dump predictions. e.g. os.path.join(exp_dir, "output_nbest_file.json") + output_nbest_file = None + # specifiy .json file to dump predictions. e.g. os.path.join(exp_dir, "output_prediction_file.json") + output_prediction_file = None + inference_samples = 5 # for test purposes. To use entire inference dataset set to -1 + all_preds, all_nbest = model.inference( + cfg.model.test_ds.file, + output_prediction_file=output_prediction_file, + output_nbest_file=output_nbest_file, + num_samples=inference_samples, + ) + + for question_id in all_preds: + print(all_preds[question_id]) + + +if __name__ == "__main__": + main() diff --git a/examples/nlp/question_answering/question_answering_squad.py b/examples/nlp/question_answering/question_answering_squad.py deleted file mode 100644 index 2d158c6a0f80..000000000000 --- a/examples/nlp/question_answering/question_answering_squad.py +++ /dev/null @@ -1,137 +0,0 @@ -# Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -""" -This script contains an example on how to train, evaluate and perform inference with the question answering model. -The QAModel in NeMo supports extractive question answering problems for data in the SQuAD (https://rajpurkar.github.io/SQuAD-explorer/) format. - -***Data format*** -The QAModel requires a JSON file for each dataset split. -In the following we will show example for a training file. Each title has one or multiple paragraph entries, each consisting of the text - "context", and question-answer entries. Each question-answer entry has: -* a question -* a globally unique id -* a boolean flag "is_impossible" which shows if the question is answerable or not -* in case the question is answerable one answer entry, which contains the text span and its starting character index in the context. If not answerable, the "answers" list is empty - -The evaluation file follows the above format except for it can provide more than one answers to the same question. -The inference file follows the above format except for it does not require the "answers" and "is_impossible" keywords. - - -***Downloading the dataset*** -Run ./NeMo/examples/nlp/question_answering/get_squad.py to download the SQuAD dataset: - -# python get_squad.py --destDir= - -***Setting the configs*** -The model and the PT trainer are defined in a config file which declares multiple important sections. -The most important ones are: - model: All arguments that are related to the Model - language model, tokenizer, token classifier, optimizer, - schedulers, and datasets/data loaders. - trainer: Any argument to be passed to PyTorch Lightning including number of epochs, number of GPUs, - precision level, etc. - -This script uses the `/examples/nlp/question_answering/conf/question_answering_squad_config.yaml` config file -by default. You may update the config file from the file directly. The other option is to set another config file via command line arguments by `--config-name=CONFIG_FILE_PATH'. - - -***Model Training*** -# python question_answering_squad.py - model.train_ds.file= - model.validation_ds= - trainer.max_epochs= - trainer.devices=[] - - -***Model Evaluation*** -Set `do_training=False` in the script and run: - -# python question_answering_squad.py - model.test_file= - -To load a pretrained checkpoint from cloud prior to training (e.g. for fine-tuning) or evaluation you can set cfg.from_pretrained=, -e.g. MODEL_NAME='BERTBaseUncasedSQuADv1.1'. You can find all pretrained model names by using -QAModel.list_available_models(). To load a local checkpoint use qa_model.restore_from() - - -***Model Inference*** -For inference use - qa_model.inference( - file=, - batch_size=, - output_nbest_file=, - output_prediction_file= - ) - -More details on how to use this script can be found in -./NeMo/tutorials/nlp/Question_Answering_Squad.ipynb -""" - -import os - -import pytorch_lightning as pl -from omegaconf import DictConfig, OmegaConf - -from nemo.collections.nlp.models.question_answering.qa_model import QAModel -from nemo.core.config import hydra_runner -from nemo.utils import logging -from nemo.utils.exp_manager import exp_manager - - -@hydra_runner(config_path="conf", config_name="question_answering_squad_config") -def main(cfg: DictConfig) -> None: - logging.info(f'Config: {OmegaConf.to_yaml(cfg)}') - trainer = pl.Trainer(**cfg.trainer) - exp_dir = exp_manager(trainer, cfg.get("exp_manager", None)) - - if cfg.model.nemo_path is not None and os.path.isfile(cfg.model.nemo_path): - model = QAModel.restore_from(cfg.model.nemo_path, override_config_path=cfg, trainer=trainer) - elif not cfg.pretrained_model: - logging.info(f'Config: {OmegaConf.to_yaml(cfg)}') - model = QAModel(cfg.model, trainer=trainer) - else: - logging.info(f'Loading pretrained model {cfg.pretrained_model}') - model = QAModel.from_pretrained(cfg.pretrained_model) - if cfg.do_training: - model.setup_training_data(train_data_config=cfg.model.train_ds) - model.setup_validation_data(val_data_config=cfg.model.validation_ds) - - if cfg.do_training: - trainer.fit(model) - if cfg.model.nemo_path: - model.save_to(cfg.model.nemo_path) - - if hasattr(cfg.model, 'test_ds') and cfg.model.test_ds.file is not None: - model.setup_test_data(test_data_config=cfg.model.test_ds) - trainer.test(model) - - # change to path if you want results to be written to file e.g. os.path.join(exp_dir, "output_nbest_file.txt") - output_nbest_file = None - # change to path if you want results to be written to file e.g. os.path.join(exp_dir, "output_prediction_file.txt") - output_prediction_file = None - inference_samples = 5 # for test purposes. To use entire inference dataset set to -1 - all_preds, all_nbests = model.inference( - file=cfg.model.validation_ds.file, - batch_size=1, - num_samples=inference_samples, - output_nbest_file=output_nbest_file, - output_prediction_file=output_prediction_file, - ) - - for _, item in all_preds.items(): - print(f"question: {item[0]} answer: {item[1]}") - - -if __name__ == '__main__': - main() diff --git a/nemo/collections/nlp/data/question_answering/__init__.py b/nemo/collections/nlp/data/question_answering/__init__.py new file mode 100644 index 000000000000..b8fc0e06da2b --- /dev/null +++ b/nemo/collections/nlp/data/question_answering/__init__.py @@ -0,0 +1,22 @@ +# Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from nemo.collections.nlp.data.question_answering.data_processor.qa_processing import QAProcessor +from nemo.collections.nlp.data.question_answering.dataset import BERTQADataset, GPTQADataset, QADataset, S2SQADataset +from nemo.collections.nlp.data.question_answering.input_example import ( + BERTQAInputExample, + GPTQAInputExample, + QAExample, + S2SQAInputExample, +) diff --git a/nemo/collections/nlp/data/question_answering/data_processor/__init__.py b/nemo/collections/nlp/data/question_answering/data_processor/__init__.py new file mode 100644 index 000000000000..d1fd030b0eab --- /dev/null +++ b/nemo/collections/nlp/data/question_answering/data_processor/__init__.py @@ -0,0 +1,15 @@ +# Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from nemo.collections.nlp.data.question_answering.data_processor.qa_processing import QAProcessor diff --git a/nemo/collections/nlp/data/question_answering/data_processor/qa_processing.py b/nemo/collections/nlp/data/question_answering/data_processor/qa_processing.py new file mode 100644 index 000000000000..a4e5b9a54692 --- /dev/null +++ b/nemo/collections/nlp/data/question_answering/data_processor/qa_processing.py @@ -0,0 +1,109 @@ +# Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright 2019 The Google Research Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import ijson +import numpy as np + +from nemo.collections.nlp.data.data_utils import DataProcessor +from nemo.collections.nlp.data.question_answering.input_example.qa_input_example import QAExample +from nemo.utils import logging + +TRAINING_MODE = "train" +EVALUATION_MODE = "eval" +INFERENCE_MODE = "infer" + + +class QAProcessor(DataProcessor): + """ + Processor for a QA dataset, expected in SQuAD format. + + Args: + data_file: data file path + mode: TRAINING_MODE/EVALUATION_MODE/INFERENCE_MODE + for creating training/evaluation/inference dataset + """ + + def __init__(self, data_file: str, mode: str): + self.data_file = data_file + self.mode = mode + + # Memoizes documents to reduce memory use (as the same document is often used for many questions) + self.doc_id = 0 + self.context_text_to_doc_id = {} + self.doc_id_to_context_text = {} + + def get_examples(self): + """ Get examples from raw json file """ + + if self.data_file is None: + raise ValueError(f"{self.mode} data file is None.") + + # remove this line and the replace cache line below - which is a temp fix + with open(self.data_file.replace('_cache', ''), "r", encoding="utf-8") as reader: + input_data = ijson.items(reader, "data.item") + + examples = [] + for entry in input_data: + len_docs = [] + title = entry["title"] + for paragraph in entry["paragraphs"]: + context_text = paragraph["context"] + for qa in paragraph["qas"]: + qas_id = qa["id"] + question_text = qa["question"] + if not question_text: + continue + start_position_character = None + answer_text = None + answers = [] + if "is_impossible" in qa: + is_impossible = qa["is_impossible"] or len(qa["answers"]) < 1 + else: + is_impossible = False + + if not is_impossible: + if self.mode in [TRAINING_MODE, EVALUATION_MODE]: + answer = qa["answers"][0] + answer_text = answer["text"] + start_position_character = answer["answer_start"] + if self.mode == EVALUATION_MODE: + answers = qa["answers"] + if context_text in self.context_text_to_doc_id: + doc_id = self.context_text_to_doc_id[context_text] + else: + doc_id = self.doc_id + self.context_text_to_doc_id[context_text] = doc_id + self.doc_id_to_context_text[doc_id] = context_text + self.doc_id += 1 + len_docs.append(len(context_text)) + + example = QAExample( + qas_id=qas_id, + question_text=question_text, + context_text=context_text, + context_id=doc_id, + answer_text=answer_text, + start_position_character=start_position_character, + title=title, + is_impossible=is_impossible, + answers=answers, + ) + + examples.append(example) + + logging.info('mean no. of chars in doc: {}'.format(np.mean(len_docs))) + logging.info('max no. of chars in doc: {}'.format(np.max(len_docs))) + + return examples diff --git a/nemo/collections/nlp/data/question_answering/dataset/__init__.py b/nemo/collections/nlp/data/question_answering/dataset/__init__.py new file mode 100644 index 000000000000..0391ea4bc29e --- /dev/null +++ b/nemo/collections/nlp/data/question_answering/dataset/__init__.py @@ -0,0 +1,18 @@ +# Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from nemo.collections.nlp.data.question_answering.dataset.qa_bert_dataset import BERTQADataset +from nemo.collections.nlp.data.question_answering.dataset.qa_dataset import QADataset +from nemo.collections.nlp.data.question_answering.dataset.qa_gpt_dataset import GPTQADataset +from nemo.collections.nlp.data.question_answering.dataset.qa_s2s_dataset import S2SQADataset diff --git a/nemo/collections/nlp/data/question_answering/dataset/qa_bert_dataset.py b/nemo/collections/nlp/data/question_answering/dataset/qa_bert_dataset.py new file mode 100644 index 000000000000..4070098b5e67 --- /dev/null +++ b/nemo/collections/nlp/data/question_answering/dataset/qa_bert_dataset.py @@ -0,0 +1,356 @@ +# Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright 2019 The Google Research Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +import numpy as np +from tqdm import trange + +from nemo.collections.nlp.data.question_answering.data_processor.qa_processing import INFERENCE_MODE, TRAINING_MODE +from nemo.collections.nlp.data.question_answering.dataset.qa_dataset import QADataset +from nemo.collections.nlp.data.question_answering.input_example.qa_bert_input_example import BERTQAInputExample +from nemo.utils import logging + + +class BERTQADataset(QADataset): + """ Creates a Dataset for BERT architecture based Exractive QA """ + + def __init__( + self, + data_file: str, + processor: object, + tokenizer: object, + keep_doc_spans: str = False, + doc_stride: int = 128, + max_query_length: int = 64, + max_seq_length: int = 512, + version_2_with_negative: bool = False, + num_samples: int = -1, + mode: str = TRAINING_MODE, + use_cache: bool = False, + ): + super().__init__( + data_file=data_file, processor=processor, tokenizer=tokenizer, mode=mode, num_samples=num_samples + ) + + self.keep_doc_spans = keep_doc_spans + self.doc_stride = doc_stride + self.max_query_length = max_query_length + self.max_seq_length = max_seq_length + self.version_2_with_negative = version_2_with_negative + self.num_samples = num_samples + self.mode = mode + self.use_cache = use_cache + + # structures for hashing to reduce memory use + self.input_mask_id = 0 + self.input_mask_id_to_input_mask = {} + self.input_mask_to_input_mask_id = {} + + self.segment_mask_id = 0 + self.segment_mask_id_to_segment_mask = {} + self.segment_mask_to_segment_mask_id = {} + + self._set_cached_features_filename() + if use_cache and os.path.exists(self.cached_features_file): + if self.mode == TRAINING_MODE: + del self.examples + del self.processor + ( + self.features, + self.input_mask_id_to_input_mask, + self.input_mask_to_input_mask_id, + self.segment_mask_id_to_segment_mask, + self.segment_mask_to_segment_mask_id, + ) = QADataset.load_features_from_cache(self.cached_features_file) + else: + self._convert_examples_to_features() + if use_cache: + items_to_pickle = [ + self.features, + self.input_mask_id_to_input_mask, + self.input_mask_to_input_mask_id, + self.segment_mask_id_to_segment_mask, + self.segment_mask_to_segment_mask_id, + ] + QADataset.dump_features_to_cache(self.cached_features_file, items_to_pickle) + + logging.info("Converting dict features into object features") + for i in trange(len(self.features)): + self.features[i] = BERTQAInputExample(**self.features[i]) + + def _set_cached_features_filename(self): + """ Creates cache filename using dataset config parameters """ + + vocab_size = getattr(self.tokenizer, "vocab_size", 0) + self.cached_features_file = ( + self.data_file + + '_cache' + + '_{}_{}_{}_{}_{}_{}_{}'.format( + self.mode, + self.tokenizer.name, + str(vocab_size), + str(self.max_seq_length), + str(self.doc_stride), + str(self.max_query_length), + str(self.num_samples), + ) + ) + + def _convert_examples_to_features(self): + """ Converts loaded examples to features """ + + logging.info(f"Preprocessing data into features.") + + has_groundtruth = self.mode != INFERENCE_MODE + unique_id = 1000000000 + text_to_tokens_dict = {} + self.features = [] + + for example_index in trange(len(self.examples)): + + if example_index % 1000 == 0: + QADataset.check_if_sufficient_memory() + + example = self.examples[example_index] + if example.question_text not in text_to_tokens_dict: + text_to_tokens_dict[example.question_text] = self.tokenizer.text_to_tokens(example.question_text)[ + : self.max_query_length + ] + query_tokens = text_to_tokens_dict[example.question_text] + + # context: index of token -> index of word + tok_to_orig_index = [] + + # context: index of word -> index of first token in token list + orig_to_tok_index = [] + + # context without white spaces after tokenization + all_doc_tokens = [] + + # doc tokens is word separated context + ( + doc_tokens, + char_to_word_offset, + start_position, + end_position, + context_text, + ) = QADataset.get_doc_tokens_and_offset_from_context_id( + example.context_id, + example.start_position_character, + example.is_impossible, + example.answer_text, + self.processor.doc_id_to_context_text, + ) + + example.start_position = start_position + example.end_position = end_position + if self.mode != TRAINING_MODE: + example.doc_tokens = doc_tokens + + # the text to tokens step is the slowest step + for (i, token) in enumerate(doc_tokens): + orig_to_tok_index.append(len(all_doc_tokens)) + if token not in text_to_tokens_dict: + text_to_tokens_dict[token] = self.tokenizer.text_to_tokens(token) + sub_tokens = text_to_tokens_dict[token] + + for sub_token in sub_tokens: + tok_to_orig_index.append(i) + all_doc_tokens.append(sub_token) + + # idx of query token start and end in context + tok_start_position = None + tok_end_position = None + if has_groundtruth and example.is_impossible: + tok_start_position = -1 + tok_end_position = -1 + if has_groundtruth and not example.is_impossible: + tok_start_position = orig_to_tok_index[example.start_position] + if example.end_position < len(doc_tokens) - 1: + tok_end_position = orig_to_tok_index[example.end_position + 1] - 1 + else: + tok_end_position = len(all_doc_tokens) - 1 + + (tok_start_position, tok_end_position) = QADataset.improve_answer_span( + all_doc_tokens, tok_start_position, tok_end_position, self.tokenizer, example.answer_text + ) + + # The -3 accounts for tokenizer.cls_token, tokenizer.sep_token and tokenizer.sep_token + # doc_spans contains all possible contexts options of given length + max_tokens_for_doc = self.max_seq_length - len(query_tokens) - 3 + doc_spans = QADataset.get_docspans(all_doc_tokens, max_tokens_for_doc, self.doc_stride) + doc_spans = QADataset.keep_relevant_docspans( + doc_spans, tok_start_position, tok_end_position, self.keep_doc_spans + ) + + # make compatible for hashing + doc_spans = tuple(doc_spans) + + for (doc_span_index, doc_span) in enumerate(doc_spans): + + tokens = [self.tokenizer.cls_token] + query_tokens + [self.tokenizer.sep_token] + segment_ids = [0 for i in range(len(tokens))] + + token_is_max_context = {} + + # maps context tokens idx in final input -> word idx in context + token_to_orig_map = {} + + for i in range(doc_span.length): + split_token_index = doc_span.start + i + token_to_orig_map[len(tokens)] = tok_to_orig_index[split_token_index] + is_max_context = QADataset.check_is_max_context(doc_spans, doc_span_index, split_token_index) + token_is_max_context[len(tokens)] = is_max_context + tokens.append(all_doc_tokens[split_token_index]) + segment_ids.append(1) + tokens.append(self.tokenizer.sep_token) + segment_ids.append(1) + + input_ids = self.tokenizer.tokens_to_ids(tokens) + + # The mask has 1 for real tokens and 0 for padding tokens. + # Only real tokens are attended to. + input_mask = [1] * len(input_ids) + + # Zero-pad up to the sequence length. + while len(input_ids) < self.max_seq_length: + input_ids.append(self.tokenizer.pad_id) + input_mask.append(0) + segment_ids.append(0) + + assert len(input_ids) == self.max_seq_length + assert len(input_mask) == self.max_seq_length + assert len(segment_ids) == self.max_seq_length + + # calculate start and end position in final array + # of tokens in answer if no answer, + # 0 for both pointing to tokenizer.cls_token + start_position = 0 + end_position = 0 + if has_groundtruth and not example.is_impossible: + doc_start = doc_span.start + doc_end = doc_span.start + doc_span.length - 1 + out_of_span = False + if not (tok_start_position >= doc_start and tok_end_position <= doc_end): + out_of_span = True + if out_of_span: + start_position = 0 + end_position = 0 + else: + doc_offset = len(query_tokens) + 2 + start_position = tok_start_position - doc_start + doc_offset + end_position = tok_end_position - doc_start + doc_offset + if has_groundtruth and example.is_impossible: + # if our document chunk does not contain + # an annotation we throw it out, since there is nothing + # to predict. + start_position = 0 + end_position = 0 + + if example_index < 1: + logging.info("*** Example ***") + logging.info("unique_id: %s" % (unique_id)) + logging.info("example_index: %s" % (example_index)) + logging.info("doc_span_index: %s" % (doc_span_index)) + logging.info("tokens: %s" % " ".join(tokens)) + logging.info( + "token_to_orig_map: %s" % " ".join(["%d:%d" % (x, y) for (x, y) in token_to_orig_map.items()]) + ) + logging.info( + "token_is_max_context: %s" + % " ".join(["%d:%s" % (x, y) for (x, y) in token_is_max_context.items()]) + ) + logging.info("input_ids: %s" % " ".join([str(x) for x in input_ids])) + logging.info("input_mask: %s" % " ".join([str(x) for x in input_mask])) + logging.info("segment_ids: %s" % " ".join([str(x) for x in segment_ids])) + if has_groundtruth and example.is_impossible: + logging.info("impossible example") + if has_groundtruth and not example.is_impossible: + answer_text = " ".join(tokens[start_position : (end_position + 1)]) + logging.info("start_position: %d" % (start_position)) + logging.info("end_position: %d" % (end_position)) + logging.info("answer: %s" % (answer_text)) + + # memoization to save CPU memory for large datasets + input_mask = tuple(input_mask) + if input_mask in self.input_mask_to_input_mask_id: + feature_input_mask_id = self.input_mask_to_input_mask_id[input_mask] + else: + self.input_mask_id_to_input_mask[self.input_mask_id] = input_mask + self.input_mask_to_input_mask_id[input_mask] = self.input_mask_id + feature_input_mask_id = self.input_mask_id + self.input_mask_id += 1 + + segment_mask = tuple(segment_ids) + if segment_mask in self.segment_mask_to_segment_mask_id: + feature_segment_mask_id = self.segment_mask_to_segment_mask_id[segment_mask] + else: + self.segment_mask_id_to_segment_mask[self.segment_mask_id] = segment_mask + self.segment_mask_to_segment_mask_id[segment_mask] = self.segment_mask_id + feature_segment_mask_id = self.segment_mask_id + self.segment_mask_id += 1 + + if self.mode == TRAINING_MODE: + input_feature = { + "unique_id": unique_id, + "input_ids": input_ids, + "input_mask": feature_input_mask_id, + "segment_ids": feature_segment_mask_id, + "start_position": start_position, + "end_position": end_position, + } + else: + input_feature = { + "unique_id": unique_id, + "input_ids": input_ids, + "input_mask": feature_input_mask_id, + "segment_ids": feature_segment_mask_id, + "start_position": start_position, + "end_position": end_position, + "example_index": example_index, + "doc_span_index": doc_span_index, + "tokens": tokens, + "token_to_orig_map": token_to_orig_map, + "token_is_max_context": token_is_max_context, + "is_impossible": example.is_impossible, + } + + self.features.append(input_feature) + unique_id += 1 + + # delete self.examples during training mode to save memory + if self.mode == TRAINING_MODE: + self.examples = [] + del self.processor + + def __getitem__(self, idx: int): + feature = self.features[idx] + if self.mode == INFERENCE_MODE: + return ( + np.array(feature.input_ids), + np.array(self.segment_mask_id_to_segment_mask[feature.segment_ids]), + np.array(self.input_mask_id_to_input_mask[feature.input_mask]), + np.array(feature.unique_id), + ) + else: + return ( + np.array(feature.input_ids), + np.array(self.segment_mask_id_to_segment_mask[feature.segment_ids]), + np.array(self.input_mask_id_to_input_mask[feature.input_mask]), + np.array(feature.unique_id), + np.array(feature.start_position), + np.array(feature.end_position), + ) diff --git a/nemo/collections/nlp/data/question_answering/dataset/qa_dataset.py b/nemo/collections/nlp/data/question_answering/dataset/qa_dataset.py new file mode 100644 index 000000000000..783b2dd33f31 --- /dev/null +++ b/nemo/collections/nlp/data/question_answering/dataset/qa_dataset.py @@ -0,0 +1,297 @@ +# Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import collections +import pickle +from functools import lru_cache +from typing import List + +import psutil +import torch + +from nemo.collections.nlp.data.data_utils import is_whitespace +from nemo.collections.nlp.data.question_answering.data_processor.qa_processing import ( + EVALUATION_MODE, + INFERENCE_MODE, + TRAINING_MODE, +) +from nemo.core.classes import Dataset +from nemo.utils import logging + + +class QADataset(Dataset): + ''' Abstract base class for QA Datasets with common utility methods ''' + + def __init__( + self, data_file: str, processor: object, tokenizer: object, mode: str, num_samples: int, **kwargs, + ): + self.mode = mode + self.data_file = data_file + self.processor = processor + self.tokenizer = tokenizer + self.features = None + + if self.mode not in [TRAINING_MODE, EVALUATION_MODE, INFERENCE_MODE]: + raise ValueError( + f"mode should be either {TRAINING_MODE}, {EVALUATION_MODE}, {INFERENCE_MODE} but got {self.mode}" + ) + + # get examples from processor and keep according to limit + self.examples = self.processor.get_examples() + if num_samples == 0: + raise ValueError( + f"num_samples has to be positive or -1 (to use the entire dataset), however got {num_samples}." + ) + elif num_samples > 0: + self.examples = self.examples[:num_samples] + + def __len__(self): + return len(self.features) + + def __getitem__(self, idx: int): + raise NotImplementedError + + @staticmethod + def load_features_from_cache(cached_filename): + logging.info(f"loading from {cached_filename}") + with open(cached_filename, "rb") as reader: + features = pickle.load(reader) + + return features + + @staticmethod + def dump_features_to_cache(cached_filename, features): + master_device = not torch.distributed.is_initialized() or torch.distributed.get_rank() == 0 + if master_device: + logging.info(f"Saving train features into cached file {cached_filename}") + with open(cached_filename, "wb") as writer: + pickle.dump(features, writer) + + @staticmethod + def check_if_sufficient_memory(): + """ + Check if there is sufficient memory to prevent system from being unresponsive + Otherwise system can become unresponsive as memory is slowly filled up, possibly leading to system unable to kill process + Interrupts run if CPU memory use is more than 75%, to leave some capacity for model loading + """ + + percent_memory = psutil.virtual_memory().percent + if percent_memory > 75: + raise ValueError('Please use a device with more CPU ram or a smaller dataset') + + @staticmethod + @lru_cache(maxsize=10000) + def get_best_span_index(doc_spans, position): + """ + For a particular position, identify which doc_span gives the most context around token + Helper function for check_is_max_context; see check_is_max_context for more details + """ + + best_score = None + best_span_index = None + for (span_index, doc_span) in enumerate(doc_spans): + end = doc_span.start + doc_span.length - 1 + if position < doc_span.start: + continue + if position > end: + continue + num_left_context = position - doc_span.start + num_right_context = end - position + score = min(num_left_context, num_right_context) + 0.01 * doc_span.length + if best_score is None or score > best_score: + best_score = score + best_span_index = span_index + + return best_span_index + + @staticmethod + def check_is_max_context(doc_spans, cur_span_index, position): + """ + Check if this is the 'max context' doc span for the token. + Because of the sliding window approach taken to scoring documents, + a single token can appear in multiple documents. + Example: + Doc: the man went to the store and bought a gallon of milk + Span A: the man went to the + Span B: to the store and bought + Span C: and bought a gallon of + ... + Now the word 'bought' will have two scores from spans B and C. We only + want to consider the score with "maximum context", which we define as + the *minimum* of its left and right context (the *sum* of left and + right context will always be the same, of course). + In the example the maximum context for 'bought' would be span C since + it has 1 left context and 3 right context, while span B has 4 left context + and 0 right context. + Code adapted from the code by the Google AI and HuggingFace. + """ + + best_span_index = QADataset.get_best_span_index(doc_spans, position) + + return cur_span_index == best_span_index + + @staticmethod + def get_docspans(all_doc_tokens, max_tokens_for_doc, doc_stride): + """ + Get docspans which are sliding window spans from a document + + Args: + all_doc_tokens: list of all tokens in document + max_tokens_for_doc: maximum number of tokens in each doc span + doc_stride: stride size which sliding window moves with + + Returns: + doc_spans: all possible doc_spans from document + """ + + _DocSpan = collections.namedtuple("DocSpan", ["start", "length"]) + doc_spans = [] + start_offset = 0 + while start_offset < len(all_doc_tokens): + length = len(all_doc_tokens) - start_offset + if length > max_tokens_for_doc: + length = max_tokens_for_doc + doc_spans.append(_DocSpan(start=start_offset, length=length)) + if start_offset + length == len(all_doc_tokens): + break + start_offset += min(length, doc_stride) + + return doc_spans + + @staticmethod + def get_average_dist_to_tok_start_and_end(doc_span, tok_start_position, tok_end_position): + """ + Find distance between doc_span and answer_span to determine if doc_span is likely to be useful for the answer + Helper function to filter out doc_spans that may not be helpful + + Args: + doc_span + tok_start_position: start position of answer in document + tok_end_position: end position of answer in document + + Returns: + average distance of doc_span to answer + """ + + center_answer = (tok_start_position + tok_end_position) // 2 + dist_to_start = abs(doc_span.start - center_answer) + dist_to_end = abs(doc_span.start + doc_span.length - 1 - center_answer) + + return (dist_to_start + dist_to_end) // 2 + + @staticmethod + def keep_relevant_docspans(doc_spans, tok_start_position, tok_end_position, mode): + """ + Filters out doc_spans, which might not be relevant to answering question, + which can be helpful when document is extremely long leading to many doc_spans with no answers + + Args: + doc_spans: all possible doc_spans + tok_start_position: start position of answer in document + tok_end_position: end position of answer in document + mode: + all: do not filter + only_positive: only keep doc_spans containing the answer + limited_negative: only keep 10 doc_spans that are nearest to answer + + Returns: + doc_spans: doc_spans after filtering + """ + + if mode == 'all': + return doc_spans + elif mode == 'only_positive': + if tok_start_position in [-1, None] or tok_end_position in [-1, None]: + return [] + else: + return [ + doc_span + for doc_span in doc_spans + if tok_start_position >= doc_span.start + and tok_end_position <= doc_span.start + doc_span.length - 1 + ] + elif mode == 'limited_negative': + n_candidates = 10 + if tok_start_position in [-1, None] or tok_end_position in [-1, None]: + pass + else: + doc_spans.sort( + key=lambda doc_span: QADataset.get_average_dist_to_tok_start_and_end( + doc_span, tok_start_position, tok_end_position + ) + ) + return doc_spans[:n_candidates] + else: + raise ValueError('mode can only be in {all, only_positive and limited_negative') + + @staticmethod + def split_into_words(context_text): + """ + Split on whitespace so that different tokens + may be attributed to their original position. + ex: context_text = "hi yo" + char_to_word_offset = [0, 0, 0, 1, 1] + doc_tokens = ["hi", "yo"] + """ + + doc_tokens = [] + char_to_word_offset = [] + prev_is_whitespace = True + for c in context_text: + if is_whitespace(c): + prev_is_whitespace = True + else: + if prev_is_whitespace: + doc_tokens.append(c) + else: + doc_tokens[-1] += c + prev_is_whitespace = False + char_to_word_offset.append(len(doc_tokens) - 1) + + return doc_tokens, char_to_word_offset + + @staticmethod + def get_doc_tokens_and_offset_from_context_id( + context_id, start_position_character, is_impossible, answer_text, context_id_to_context_text + ): + start_position, end_position = 0, 0 + context_text = context_id_to_context_text[context_id] + doc_tokens, char_to_word_offset = QADataset.split_into_words(context_text) + + # Start end end positions only has a value during evaluation. + if start_position_character is not None and not is_impossible: + + # start_position is index of word, end_position inclusive + start_position = char_to_word_offset[start_position_character] + end_position = char_to_word_offset[ + min(start_position_character + len(answer_text) - 1, len(char_to_word_offset) - 1) + ] + + return doc_tokens, char_to_word_offset, start_position, end_position, context_text + + @staticmethod + def improve_answer_span( + doc_tokens: List[str], input_start: int, input_end: int, tokenizer: object, orig_answer_text: str, + ): + """ Returns tokenized answer spans that better match the annotated answer """ + + tok_answer_text = " ".join(tokenizer.text_to_tokens(orig_answer_text)) + + for new_start in range(input_start, input_end + 1): + for new_end in range(input_end, new_start - 1, -1): + text_span = " ".join(doc_tokens[new_start : (new_end + 1)]) + if text_span == tok_answer_text: + return (new_start, new_end) + + return (input_start, input_end) diff --git a/nemo/collections/nlp/data/question_answering/dataset/qa_gpt_dataset.py b/nemo/collections/nlp/data/question_answering/dataset/qa_gpt_dataset.py new file mode 100644 index 000000000000..d6484b33e202 --- /dev/null +++ b/nemo/collections/nlp/data/question_answering/dataset/qa_gpt_dataset.py @@ -0,0 +1,310 @@ +# Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright 2019 The Google Research Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import copy +import os + +import numpy as np +import torch +from tqdm import trange + +from nemo.collections.nlp.data.question_answering.data_processor.qa_processing import INFERENCE_MODE, TRAINING_MODE +from nemo.collections.nlp.data.question_answering.dataset.qa_dataset import QADataset +from nemo.collections.nlp.data.question_answering.input_example.qa_gpt_input_example import GPTQAInputExample +from nemo.utils import logging + + +class GPTQADataset(QADataset): + """ Creates a Dataset for GPT architecture based Generative QA """ + + def __init__( + self, + data_file: str, + processor: object, + tokenizer: object, + keep_doc_spans: str = False, + doc_stride: int = 128, + max_query_length: int = 64, + max_seq_length: int = 512, + max_answer_length: int = 64, + check_if_answer_in_context: bool = False, + num_samples: int = -1, + mode: str = TRAINING_MODE, + use_cache: bool = False, + ): + super().__init__( + data_file=data_file, processor=processor, tokenizer=tokenizer, mode=mode, num_samples=num_samples + ) + + self.keep_doc_spans = keep_doc_spans + self.doc_stride = doc_stride + self.max_query_length = max_query_length + self.max_seq_length = max_seq_length + self.max_answer_length = max_answer_length + self.check_if_answer_in_context = check_if_answer_in_context + self.num_samples = num_samples + self.mode = mode + self.use_cache = use_cache + + self._set_cached_features_filename() + if use_cache and os.path.exists(self.cached_features_file): + + # delete self.examples during training mode to save memory + if self.mode == TRAINING_MODE: + del self.examples + del self.processor + self.features = QADataset.load_features_from_cache(self.cached_features_file) + else: + self._convert_examples_to_features() + if use_cache: + QADataset.dump_features_to_cache(self.cached_features_file, self.features) + + logging.info("Converting dict features into object features") + for i in trange(len(self.features)): + self.features[i] = GPTQAInputExample(**self.features[i]) + + def _set_cached_features_filename(self): + """ Creates cache filename using dataset config parameters """ + + vocab_size = getattr(self.tokenizer, "vocab_size", 0) + self.cached_features_file = ( + self.data_file + + '_cache' + + '_{}_{}_{}_{}_{}_{}_{}'.format( + self.mode, + self.tokenizer.name, + str(vocab_size), + str(self.max_query_length), + str(self.max_seq_length), + str(self.max_answer_length), + str(self.num_samples), + ) + ) + + def _convert_examples_to_features(self): + """ + Iterates through each QA example, formats into template and encodes + Template: `context: question: answer:` + """ + + logging.info(f"Preprocessing data into features.") + + unique_id = 1000000000 + self.features = [] + + context_prefix = "context: " + query_prefix = " question: " + answer_prefix = " answer:" + + context_prefix_tokens = self.tokenizer.tokenizer.tokenize(context_prefix) + answer_prefix_tokens = self.tokenizer.tokenizer.tokenize(answer_prefix) + + for example_index in trange(len(self.examples)): + if example_index % 1000 == 0: + GPTQADataset.check_if_sufficient_memory() + + example = self.examples[example_index] + + formatted_query, query_tokens_length = self._prep_query(query_prefix, example) + formatted_answer, answer_tokens_length = self._prep_answer(example) + context_tokens, context_spans = self._prep_context( + example, query_tokens_length, answer_tokens_length, context_prefix_tokens, answer_prefix_tokens, + ) + + unique_id = self._encode_all_context_spans( + unique_id, + context_spans, + context_tokens, + context_prefix, + formatted_query, + answer_prefix, + formatted_answer, + example, + example_index, + ) + + # delete self.examples during training mode to save memory + if self.mode == TRAINING_MODE: + self.examples = [] + del self.processor + + def _prep_query(self, query_prefix, example): + """ + Formats a question into input format: ` question: ` + The space at the start allows concatention with the context and answer for input + Returns formatted query, query tokens, and length of query tokens + """ + + formatted_query = f"{query_prefix}{example.question_text}" + + return self._get_truncated_sentence_and_len(formatted_query, self.max_query_length) + + def _prep_answer(self, example): + """ + Formats an answer into suitable model input: + - In inference mode, answer is returned as an empty string, else + - Sets EOS token as answer if question is impossible to answer, else + - Appends answer with EOS token as the final answer + Returns formatted answer string, answer tokens, and length of answer tokens + """ + + if self.mode == INFERENCE_MODE: + target = "" + elif example.is_impossible: # example is impossible to answer given context + target = self.tokenizer.tokenizer.eos_token + else: + target = f"{example.answer_text}{self.tokenizer.tokenizer.eos_token}" + + return self._get_truncated_sentence_and_len(target, self.max_answer_length) + + def _prep_context( + self, example, query_tokens_length, answer_tokens_length, context_prefix_tokens, answer_prefix_tokens, + ): + """ + Calculates the maximum possible length for a given context given a question + as inputs are fixed length + Divides the context into multiple spans based on the calculated max length + """ + + context_tokens = self.tokenizer.tokenizer.tokenize(example.context_text) + max_context_length = ( + self.max_seq_length + - query_tokens_length + - answer_tokens_length + - len(context_prefix_tokens) + - len(answer_prefix_tokens) + - 1 # -1 accounts for EOS token + ) + context_spans = GPTQADataset.get_docspans(context_tokens, max_context_length, self.doc_stride) + context_spans = tuple(context_spans) + + return context_tokens, context_spans + + def _encode_all_context_spans( + self, + unique_id, + context_spans, + context_tokens, + context_prefix, + formatted_query, + answer_prefix, + formatted_answer, + example, + example_index, + ): + """ + Formats all spans extracted from a single context as: + `context: question: answer:` + is set as: + - blank if in inference mode, else + - EOS token if answer text is not present in context span + and the check flag is set to true, else + - formatted answer + """ + + for context_span_idx, context_span in enumerate(context_spans): + context_span_tokens = context_tokens[context_span.start : context_span.start + context_span.length] + context_span_text = self.tokenizer.tokenizer.convert_tokens_to_string(context_span_tokens) + + input_without_answer = f"{context_prefix}{context_span_text}{formatted_query}{answer_prefix}" + _, training_mask_end = self._get_truncated_sentence_and_len(input_without_answer, self.max_seq_length) + + is_answer_in_context_check = ( + self.check_if_answer_in_context # checks if the flag for this check is set + and example.answer_text # checks if answer text is valid, i.e. question is not unanswerable + and example.answer_text not in context_span_text # checks if answer text is a substring of context + ) + + if self.mode == INFERENCE_MODE: + input_to_encode = input_without_answer + elif is_answer_in_context_check: + input_to_encode = f"{input_without_answer}{self.tokenizer.tokenizer.eos_token}" + else: + input_to_encode = f"{input_without_answer}{formatted_answer}" + + encoded_input_dict = self.tokenizer.tokenizer( + input_to_encode, + truncation=True, + max_length=self.max_seq_length, + padding="max_length", + return_tensors="pt", + ) + input_ids = torch.squeeze(encoded_input_dict["input_ids"]) + input_attn_mask = torch.squeeze(encoded_input_dict["attention_mask"]) + + labels = GPTQADataset.update_labels_for_no_pad_loss(input_ids, training_mask_end, input_attn_mask) + + # create dictionary features + feature = { + "unique_id": unique_id, + "input_ids": input_ids, + "input_attn_mask": input_attn_mask, + "training_mask_end": training_mask_end, + "labels": labels, + "example_index": example_index, + "context_span_index": context_span_idx, + "is_impossible": example.is_impossible, + } + + self.features.append(feature) + unique_id += 1 + + return unique_id + + def _get_truncated_sentence_and_len(self, sentence, max_length): + if not sentence: + return "", 0 + tokens = self.tokenizer.tokenizer.tokenize(sentence)[:max_length] + trunc_sentence = self.tokenizer.tokenizer.convert_tokens_to_string(tokens) + seq_length = len(tokens) + + return trunc_sentence, seq_length + + @classmethod + def update_labels_for_no_pad_loss(cls, input_ids, training_mask_end, input_attn_mask): + """ + Loss mask for GPT is constructed to ignore loss for padding tokens + GPT eos token is same as pas token and needs to be excluded from loss mask + This is done using the attention mask inversion as described in: + https://github.com/huggingface/transformers/issues/7135#issuecomment-1172962080 + """ + labels = copy.copy(torch.squeeze(input_ids)) + inv_bool_attn_mask = torch.eq(torch.squeeze(input_attn_mask), 0) + labels.data = torch.tensor( + [ + -100 if ((i < training_mask_end) or (inv_bool_attn_mask[i])) else labels.data[i] + for i in range(len(labels.data)) + ] + ) + + return labels + + def __getitem__(self, idx: int): + feature = self.features[idx] + if self.mode == INFERENCE_MODE: + return ( + np.array(feature.input_ids), + np.array(feature.input_attn_mask), + np.array(feature.unique_id), + np.array(feature.training_mask_end), + ) + else: + return ( + np.array(feature.input_ids), + np.array(feature.input_attn_mask), + np.array(feature.unique_id), + np.array(feature.training_mask_end), + np.array(feature.labels), + ) diff --git a/nemo/collections/nlp/data/question_answering/dataset/qa_s2s_dataset.py b/nemo/collections/nlp/data/question_answering/dataset/qa_s2s_dataset.py new file mode 100644 index 000000000000..1f9a8ef615a9 --- /dev/null +++ b/nemo/collections/nlp/data/question_answering/dataset/qa_s2s_dataset.py @@ -0,0 +1,247 @@ +# Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright 2019 The Google Research Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +import numpy as np +import torch +from tqdm import trange + +from nemo.collections.nlp.data.question_answering.data_processor.qa_processing import INFERENCE_MODE, TRAINING_MODE +from nemo.collections.nlp.data.question_answering.dataset.qa_dataset import QADataset +from nemo.collections.nlp.data.question_answering.input_example.qa_s2s_input_example import S2SQAInputExample +from nemo.utils import logging + + +class S2SQADataset(QADataset): + """ Creates a Dataset for T5/BART architecture based Generative QA """ + + def __init__( + self, + data_file: str, + processor: object, + tokenizer: object, + keep_doc_spans: str = False, + doc_stride: int = 128, + max_query_length: int = 64, + max_seq_length: int = 512, + max_answer_length: int = 64, + check_if_answer_in_context: bool = False, + num_samples: int = -1, + mode: str = TRAINING_MODE, + use_cache: bool = False, + ): + super().__init__( + data_file=data_file, processor=processor, tokenizer=tokenizer, mode=mode, num_samples=num_samples + ) + + self.keep_doc_spans = keep_doc_spans + self.doc_stride = doc_stride + self.max_query_length = max_query_length + self.max_seq_length = max_seq_length + self.max_answer_length = max_answer_length + self.check_if_answer_in_context = check_if_answer_in_context + self.num_samples = num_samples + self.mode = mode + self.use_cache = use_cache + + self._set_cached_features_filename() + if use_cache and os.path.exists(self.cached_features_file): + + # delete self.examples during training mode to save memory + if self.mode == TRAINING_MODE: + del self.examples + del self.processor + self.features = QADataset.load_features_from_cache(self.cached_features_file) + else: + self._convert_examples_to_features() + if use_cache: + QADataset.dump_features_to_cache(self.cached_features_file, self.features) + + logging.info("Converting dict features into object features") + for i in trange(len(self.features)): + self.features[i] = S2SQAInputExample(**self.features[i]) + + def _set_cached_features_filename(self): + """ Creates cache filename using dataset config parameters """ + + vocab_size = getattr(self.tokenizer, "vocab_size", 0) + self.cached_features_file = ( + self.data_file + + '_cache' + + '_{}_{}_{}_{}_{}_{}_{}'.format( + self.mode, + self.tokenizer.name, + str(vocab_size), + str(self.max_query_length), + str(self.max_seq_length), + str(self.max_answer_length), + str(self.num_samples), + ) + ) + + def _convert_examples_to_features(self): + """ + Iterates through each QA example, formats into input and output template, + and encodes the input and output template + Input template: `context: question: ` + Output template: `` + """ + + logging.info(f"Preprocessing data into features.") + + unique_id = 1000000000 + self.features = [] + context_prefix = "context: " + context_prefix_tokens = self.tokenizer.tokenizer.tokenize(context_prefix) + + for example_index in trange(len(self.examples)): + if example_index % 1000 == 0: + S2SQADataset.check_if_sufficient_memory() + + example = self.examples[example_index] + + query_tokens, formatted_query = self._prep_query(example) + context_tokens, context_spans = self._prep_context(example, query_tokens, context_prefix_tokens) + + unique_id = self._encode_all_context_spans( + unique_id, context_spans, context_tokens, formatted_query, example, example_index, + ) + + # delete self.examples during training mode to save memory + if self.mode == TRAINING_MODE: + self.examples = [] + del self.processor + + def _prep_query(self, example): + """ + Formats a question into input format: ` question: ` + The space at the start allows concatention with the context for input + """ + formatted_query = f" question: {example.question_text}" + query_tokens = self.tokenizer.tokenizer.tokenize(formatted_query)[: self.max_query_length] + + return query_tokens, formatted_query + + def _prep_context(self, example, query_tokens, context_prefix_tokens): + """ + Calculates the maximum possible length for a given context given a question + as inputs are of fixed length + Divides the context into multiple spans based on the calculated max length + """ + + context_tokens = self.tokenizer.tokenizer.tokenize(example.context_text) + max_context_length = ( + self.max_seq_length + - len(query_tokens) + - len(context_prefix_tokens) + - 1 # -1 accounts for token in T5/BART + ) + context_spans = S2SQADataset.get_docspans(context_tokens, max_context_length, self.doc_stride) + context_spans = tuple(context_spans) + + return context_tokens, context_spans + + def _encode_all_context_spans( + self, unique_id, context_spans, context_tokens, formatted_query, example, example_index, + ): + """ + Fromats all spans extracted from a single context as: + `context: question: answer: ` and encodes + If the answer text (example.answer_text) is not present in a given context span, + the answer is converted to a blank answer + """ + + for context_span_idx, context_span in enumerate(context_spans): + + # format query and context span text + context_span_tokens = context_tokens[context_span.start : context_span.start + context_span.length] + context_span_text = self.tokenizer.tokenizer.convert_tokens_to_string(context_span_tokens) + source = f"context: {context_span_text}{formatted_query}" + + # encode input + encoded_input_dict = self.tokenizer.tokenizer( + source, truncation=True, max_length=self.max_seq_length, padding="max_length", return_tensors="pt", + ) + input_ids = torch.squeeze(encoded_input_dict["input_ids"]) + input_attn_mask = torch.squeeze(encoded_input_dict["attention_mask"]) + + # encode output based on mode and is question answerable given context + labels = self._encode_answer(example, context_span_text) + + # create dictionary features + feature = { + "unique_id": unique_id, + "input_ids": input_ids, + "input_attn_mask": input_attn_mask, + "labels": labels, + "example_index": example_index, + "context_span_index": context_span_idx, + "is_impossible": example.is_impossible, + } + + self.features.append(feature) + unique_id += 1 + + return unique_id + + def _encode_answer(self, example, context_span_text): + """ + Answer is set and encoded as: + - blank if in inference mode, else + - blank if question is unanswerable given context, else + - blank if answer text is not present in context span + and the check flag is set to true, else + - formatted answer + """ + + is_answer_in_context_check = ( + self.check_if_answer_in_context # checks if the flag for this check is set + and example.answer_text # checks if answer text is valid, i.e. question is not unanswerable + and example.answer_text not in context_span_text # checks if answer text is a substring of context + ) + + if ( + self.mode == INFERENCE_MODE + or example.is_impossible # question not answerable given context + or is_answer_in_context_check + ): + target = "" + else: + target = example.answer_text + + encoded_output_dict = self.tokenizer.tokenizer( + target, truncation=True, max_length=self.max_answer_length, padding="max_length", return_tensors="pt", + ) + labels = torch.squeeze(encoded_output_dict["input_ids"]) + labels[labels == self.tokenizer.tokenizer.pad_token_id] = -100 + + return labels + + def __getitem__(self, idx: int): + feature = self.features[idx] + if self.mode == INFERENCE_MODE: + return ( + np.array(feature.input_ids), + np.array(feature.input_attn_mask), + np.array(feature.unique_id), + ) + else: + return ( + np.array(feature.input_ids), + np.array(feature.input_attn_mask), + np.array(feature.unique_id), + np.array(feature.labels), + ) diff --git a/nemo/collections/nlp/data/question_answering/input_example/__init__.py b/nemo/collections/nlp/data/question_answering/input_example/__init__.py new file mode 100644 index 000000000000..96fd9f2a3a2c --- /dev/null +++ b/nemo/collections/nlp/data/question_answering/input_example/__init__.py @@ -0,0 +1,18 @@ +# Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from nemo.collections.nlp.data.question_answering.input_example.qa_bert_input_example import BERTQAInputExample +from nemo.collections.nlp.data.question_answering.input_example.qa_gpt_input_example import GPTQAInputExample +from nemo.collections.nlp.data.question_answering.input_example.qa_input_example import QAExample +from nemo.collections.nlp.data.question_answering.input_example.qa_s2s_input_example import S2SQAInputExample diff --git a/nemo/collections/nlp/data/question_answering/input_example/qa_bert_input_example.py b/nemo/collections/nlp/data/question_answering/input_example/qa_bert_input_example.py new file mode 100644 index 000000000000..e74cdd04bd94 --- /dev/null +++ b/nemo/collections/nlp/data/question_answering/input_example/qa_bert_input_example.py @@ -0,0 +1,34 @@ +# Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from dataclasses import dataclass +from typing import Dict, List, Optional + + +@dataclass +class BERTQAInputExample(object): + """ A single set of features of a QA example for BERT-like model """ + + unique_id: int + input_ids: List[int] + input_mask: List[int] + segment_ids: List[int] + example_index: int = None + doc_span_index: int = None + tokens: List[str] = None + token_to_orig_map: Dict[int, int] = None + token_is_max_context: Dict[int, bool] = None + start_position: Optional[int] = None + end_position: Optional[int] = None + is_impossible: Optional[int] = None diff --git a/nemo/collections/nlp/data/question_answering/input_example/qa_gpt_input_example.py b/nemo/collections/nlp/data/question_answering/input_example/qa_gpt_input_example.py new file mode 100644 index 000000000000..1f0b71492d77 --- /dev/null +++ b/nemo/collections/nlp/data/question_answering/input_example/qa_gpt_input_example.py @@ -0,0 +1,30 @@ +# Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from dataclasses import dataclass +from typing import List, Optional + + +@dataclass +class GPTQAInputExample(object): + """ A single set of features of a QA example for GPT-like model """ + + unique_id: int + input_ids: List[int] + input_attn_mask: List[int] + training_mask_end: int = None + labels: List[int] = None + example_index: int = None + context_span_index: int = None + is_impossible: Optional[bool] = False diff --git a/nemo/collections/nlp/data/question_answering/input_example/qa_input_example.py b/nemo/collections/nlp/data/question_answering/input_example/qa_input_example.py new file mode 100644 index 000000000000..081f6804eab9 --- /dev/null +++ b/nemo/collections/nlp/data/question_answering/input_example/qa_input_example.py @@ -0,0 +1,33 @@ +# Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from dataclasses import dataclass +from typing import List + + +@dataclass +class QAExample(object): + """ A single training/test example for a QA dataset, as loaded from disk """ + + qas_id: str # The example's unique identifier + question_text: str + context_text: str + context_id: int + answer_text: str + start_position_character: int # The character position of the start of the answer, 0 indexed + title: str + answers: List[ + str + ] = None # None by default, this is used during evaluation. Holds answers as well as their start positions + is_impossible: bool = False # False by default, set to True if the example has no possible answer diff --git a/nemo/collections/nlp/data/question_answering/input_example/qa_s2s_input_example.py b/nemo/collections/nlp/data/question_answering/input_example/qa_s2s_input_example.py new file mode 100644 index 000000000000..5a43b4cd936a --- /dev/null +++ b/nemo/collections/nlp/data/question_answering/input_example/qa_s2s_input_example.py @@ -0,0 +1,29 @@ +# Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from dataclasses import dataclass +from typing import List, Optional + + +@dataclass +class S2SQAInputExample(object): + """ A single set of features of a QA example for T5/BART-like model """ + + unique_id: int + input_ids: List[int] + input_attn_mask: List[int] + labels: List[int] = None + example_index: int = None + context_span_index: int = None + is_impossible: Optional[bool] = False diff --git a/nemo/collections/nlp/metrics/__init__.py b/nemo/collections/nlp/metrics/__init__.py index 9837069beeb9..18414412d91c 100644 --- a/nemo/collections/nlp/metrics/__init__.py +++ b/nemo/collections/nlp/metrics/__init__.py @@ -14,4 +14,5 @@ from nemo.collections.nlp.metrics.classification_report import ClassificationReport, MultiLabelClassificationReport from nemo.collections.nlp.metrics.dialogue_metrics import DialogueClassificationMetrics +from nemo.collections.nlp.metrics.qa_metrics import QAMetrics from nemo.collections.nlp.metrics.sequence_perplexity import SequencePerplexity diff --git a/nemo/collections/nlp/metrics/qa_metrics.py b/nemo/collections/nlp/metrics/qa_metrics.py new file mode 100644 index 000000000000..3152d53d6ab7 --- /dev/null +++ b/nemo/collections/nlp/metrics/qa_metrics.py @@ -0,0 +1,202 @@ +# Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import collections +import json +import re +import string + +import torch +from tqdm import tqdm + +from nemo.collections.nlp.parts.utils_funcs import tensor2list +from nemo.utils import logging + + +class QAMetrics(object): + @staticmethod + def remove_articles(text): + return re.sub(r"\b(a|an|the)\b", " ", text) + + @staticmethod + def white_space_fix(text): + return " ".join(text.split()) + + @staticmethod + def remove_punc(text): + exclude = set(string.punctuation) + return "".join(ch for ch in text if ch not in exclude) + + @staticmethod + def normalize_answer(s: str): + """ Lower text and remove punctuation, articles and extra whitespace """ + + return QAMetrics.white_space_fix(QAMetrics.remove_articles(QAMetrics.remove_punc(s.lower()))) + + @staticmethod + def _get_normalized_tokens(s: str): + """ Get normalized tokens """ + if not s: + return [] + return QAMetrics.normalize_answer(s).split() + + @staticmethod + def get_one_f1(prediction: str, ground_truth: str): + """ Computes f1 score between prediction and ground truth """ + + prediction_tokens = QAMetrics._get_normalized_tokens(prediction) + ground_truth_tokens = QAMetrics._get_normalized_tokens(ground_truth) + common = collections.Counter(prediction_tokens) & collections.Counter(ground_truth_tokens) + num_same = sum(common.values()) + + # If either is no-answer, then F1 is 1 if they agree, 0 otherwise + if len(ground_truth_tokens) == 0 or len(prediction_tokens) == 0: + return int(ground_truth_tokens == prediction_tokens) + if num_same == 0: + return 0 + + precision = 1.0 * num_same / len(prediction_tokens) + recall = 1.0 * num_same / len(ground_truth_tokens) + f1 = (2 * precision * recall) / (precision + recall) + + return f1 + + @staticmethod + def get_one_exact_match(prediction: str, ground_truth: str): + """ Computes exact match between prediction and ground truth """ + + return int(QAMetrics.normalize_answer(prediction) == QAMetrics.normalize_answer(ground_truth)) + + @staticmethod + def convert_dict_outputs_to_lists(outputs, keys): + output_lists = [[] for _ in range(len(keys))] + for output in outputs: + for i, key in enumerate(keys): + if isinstance(output[key], torch.Tensor): + output_lists[i].extend(tensor2list(output[key])) + else: + output_lists[i].extend(output[key]) + + return output_lists + + @staticmethod + def get_exact_match_and_f1(examples, preds, question_id_filter=[]): + """ + Returns a dictionary of question id: exact match/f1 score + Questions with ids *not* present in `question_id_filter` are excluded + """ + exact_scores = {} + f1_scores = {} + + for example in examples: + question_id = example.qas_id + if question_id not in question_id_filter: + continue + + gold_answers = [answer["text"] for answer in example.answers if QAMetrics.normalize_answer(answer["text"])] + + if not gold_answers: + # For unanswerable questions, only correct answer is empty string + gold_answers = [""] + + pred = preds[question_id] + exact_scores[question_id] = max(QAMetrics.get_one_exact_match(pred, a) for a in gold_answers) + f1_scores[question_id] = max(QAMetrics.get_one_f1(pred, a) for a in gold_answers) + + return exact_scores, f1_scores + + @staticmethod + def make_eval_dict(exact_scores, f1_scores, prefix=""): + """ Returns dictionary with formatted evaluation scores """ + + total = len(exact_scores) + return collections.OrderedDict( + [ + (f"{prefix}exact", (100.0 * sum(exact_scores.values()) / total) if total != 0 else 0.0), + (f"{prefix}f1", (100.0 * sum(f1_scores.values()) / total) if total != 0 else 0.0), + (f"{prefix}total", float(total)), + ] + ) + + @staticmethod + def merge_eval_dicts(eval_dicts): + """ + Combines multiple evaluation dict outputs into one dict + Ex: combines eval dicts for HasAns F1, NoAnsF1, and Total F1 + """ + + merged_dict = collections.OrderedDict() + for eval_dict in eval_dicts: + for key in eval_dict: + merged_dict[key] = eval_dict[key] + + return merged_dict + + @staticmethod + def evaluate_predictions(examples, all_predictions): + """ + Calculates exact match and f1 scores for all predictions, + questions with answers, and no answer questions + """ + + qas_id_to_has_answer = {example.qas_id: bool(example.answers) for example in examples[: len(all_predictions)]} + has_answer_qids = [qas_id for qas_id, has_answer in qas_id_to_has_answer.items() if has_answer] + no_answer_qids = [qas_id for qas_id, has_answer in qas_id_to_has_answer.items() if not has_answer] + + filters_and_prefixes = [ + (list(qas_id_to_has_answer), ""), + (has_answer_qids, "HasAns_"), + (no_answer_qids, "NoAns_"), + ] + + eval_dicts = [] + for qas_id_filter, prefix in filters_and_prefixes: + curr_exact, curr_f1 = QAMetrics.get_exact_match_and_f1(examples, all_predictions, qas_id_filter) + curr_eval_dict = QAMetrics.make_eval_dict(curr_exact, curr_f1, prefix=prefix) + eval_dicts.append(curr_eval_dict) + + merged_eval_dict = QAMetrics.merge_eval_dicts(eval_dicts) + + return merged_eval_dict + + @staticmethod + def dump_predicted_answers_to_file(output_filename, examples, predictions): + logging.info(f"Writing predictions to {output_filename}") + + with open(output_filename, "w") as writer: + for ex in tqdm(examples): + output_item = { + "id": ex.qas_id, + "context": ex.context_text, + "question": ex.question_text, + "predicted_answer": predictions[ex.qas_id], + } + writer.write(json.dumps(output_item) + "\n") + + @staticmethod + def dump_nbest_predictions_to_file(output_filename, examples, nbest_predictions, keys_to_dump=[]): + logging.info(f"Writing nbest predictions to {output_filename}") + + with open(output_filename, "w") as writer: + for ex in tqdm(examples): + output_item = { + "id": ex.qas_id, + "context": ex.context_text, + "question": ex.question_text, + "nbest_predictions": [], + } + for pred in nbest_predictions[ex.qas_id]: + output_item["nbest_predictions"].append({key: pred[key] for key in keys_to_dump}) + + writer.write(json.dumps(output_item) + "\n") diff --git a/nemo/collections/nlp/models/question_answering/__init__.py b/nemo/collections/nlp/models/question_answering/__init__.py index 4ced15ed8ab0..f7c55d328f78 100644 --- a/nemo/collections/nlp/models/question_answering/__init__.py +++ b/nemo/collections/nlp/models/question_answering/__init__.py @@ -12,4 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from nemo.collections.nlp.models.question_answering.qa_base_model import BaseQAModel +from nemo.collections.nlp.models.question_answering.qa_bert_model import BERTQAModel +from nemo.collections.nlp.models.question_answering.qa_gpt_model import GPTQAModel from nemo.collections.nlp.models.question_answering.qa_model import QAModel +from nemo.collections.nlp.models.question_answering.qa_s2s_model import S2SQAModel diff --git a/nemo/collections/nlp/models/question_answering/qa_base_model.py b/nemo/collections/nlp/models/question_answering/qa_base_model.py new file mode 100644 index 000000000000..bfb45f51b6ac --- /dev/null +++ b/nemo/collections/nlp/models/question_answering/qa_base_model.py @@ -0,0 +1,93 @@ +# Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import Optional + +import torch +from omegaconf import DictConfig, OmegaConf +from pytorch_lightning import Trainer + +from nemo.collections.nlp.data.question_answering.data_processor.qa_processing import ( + EVALUATION_MODE, + INFERENCE_MODE, + TRAINING_MODE, +) +from nemo.collections.nlp.models.nlp_model import NLPModel +from nemo.utils import logging + + +class BaseQAModel(NLPModel): + def __init__(self, cfg: DictConfig, trainer: Trainer = None, no_lm_init=True): + self.cfg = cfg + super().__init__(cfg=cfg, trainer=trainer, no_lm_init=no_lm_init) + + def setup_training_data(self, train_data_config: Optional[DictConfig]): + if not train_data_config or not train_data_config.file: + logging.info( + f"Dataloader config or file_path for the train is missing, so no data loader for test is created!" + ) + self._test_dl = None + return + + self._train_dl = self._setup_dataloader_from_config(cfg=train_data_config, mode=TRAINING_MODE) + + def setup_validation_data(self, val_data_config: Optional[DictConfig]): + if not val_data_config or not val_data_config.file: + logging.info( + f"Dataloader config or file_path for the validation is missing, so no data loader for test is created!" + ) + self._test_dl = None + return + + self._validation_dl = self._setup_dataloader_from_config(cfg=val_data_config, mode=EVALUATION_MODE) + + def setup_test_data(self, test_data_config: Optional[DictConfig]): + if not test_data_config or test_data_config.file is None: + logging.info( + f"Dataloader config or file_path for the test is missing, so no data loader for test is created!" + ) + self._test_dl = None + return + + self._test_dl = self._setup_dataloader_from_config(cfg=test_data_config, mode=EVALUATION_MODE) + + def setup_inference_data(self, input_file, batch_size=1, num_samples=-1, num_workers=2): + dataloader_cfg = { + "batch_size": batch_size, + "file": input_file, + "shuffle": False, + "num_samples": num_samples, + 'num_workers': num_workers, + 'pin_memory': False, + 'drop_last': False, + } + dataloader_cfg = OmegaConf.create(dataloader_cfg) + inference_dl = self._setup_dataloader_from_config(cfg=dataloader_cfg, mode=INFERENCE_MODE) + + return inference_dl + + def _setup_dataloader_from_config(self, cfg: DictConfig, mode: str): + raise NotImplementedError() + + @torch.no_grad() + def _get_per_sample_perplexity(self, logits, labels): + """ Returns average perplexity for each sample in the batch """ + + loss_fct = torch.nn.CrossEntropyLoss(ignore_index=-100, reduction='none') + unreduced_loss = loss_fct(logits.view(-1, logits.size(-1)), labels.view(-1),) + unreduced_loss = unreduced_loss.reshape(labels.shape) + mask_0 = unreduced_loss != 0 + per_sample_perplexity = torch.exp((unreduced_loss * mask_0).sum(axis=1) / mask_0.sum(axis=1)) + + return per_sample_perplexity diff --git a/nemo/collections/nlp/models/question_answering/qa_bert_model.py b/nemo/collections/nlp/models/question_answering/qa_bert_model.py new file mode 100644 index 000000000000..d9eba29eae74 --- /dev/null +++ b/nemo/collections/nlp/models/question_answering/qa_bert_model.py @@ -0,0 +1,698 @@ +# Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import collections +from typing import List, Optional + +import numpy as np +import torch +from omegaconf import DictConfig +from pytorch_lightning import Trainer +from transformers.models.bert.tokenization_bert import BasicTokenizer + +from nemo.collections.common.losses import SpanningLoss +from nemo.collections.common.parts.utils import _compute_softmax +from nemo.collections.nlp.data.question_answering.data_processor.qa_processing import QAProcessor +from nemo.collections.nlp.data.question_answering.dataset.qa_bert_dataset import BERTQADataset +from nemo.collections.nlp.metrics.qa_metrics import QAMetrics +from nemo.collections.nlp.models.question_answering.qa_base_model import BaseQAModel +from nemo.collections.nlp.modules.common import TokenClassifier +from nemo.collections.nlp.parts.utils_funcs import tensor2list +from nemo.core.classes.common import PretrainedModelInfo, typecheck +from nemo.utils import logging + + +class BERTQAModel(BaseQAModel): + """ BERT model with a QA (token classification) head """ + + def __init__(self, cfg: DictConfig, trainer: Trainer = None): + + super().__init__(cfg=cfg, trainer=trainer, no_lm_init=False) + self.classifier = TokenClassifier( + hidden_size=self.hidden_size, + num_classes=cfg.token_classifier.num_classes, + num_layers=cfg.token_classifier.num_layers, + activation=cfg.token_classifier.activation, + log_softmax=cfg.token_classifier.log_softmax, + dropout=cfg.token_classifier.dropout, + use_transformer_init=cfg.token_classifier.use_transformer_init, + ) + + self.loss = SpanningLoss() + + def training_step(self, batch, batch_idx): + input_ids, input_type_ids, input_mask, unique_ids, start_positions, end_positions = batch + logits = self.forward(input_ids=input_ids, token_type_ids=input_type_ids, attention_mask=input_mask) + loss, _, _ = self.loss(logits=logits, start_positions=start_positions, end_positions=end_positions) + lr = self._optimizer.param_groups[0]['lr'] + + self.log('lr', lr, prog_bar=True) + self.log("train_loss", loss, on_step=True, on_epoch=True, prog_bar=True, logger=True) + + return {'loss': loss, 'lr': lr} + + def validation_step(self, batch, batch_idx): + prefix = "test" if self.trainer.testing else "val" + + input_ids, input_type_ids, input_mask, unique_ids, start_positions, end_positions = batch + logits = self.forward(input_ids=input_ids, token_type_ids=input_type_ids, attention_mask=input_mask) + loss, start_logits, end_logits = self.loss( + logits=logits, start_positions=start_positions, end_positions=end_positions + ) + + tensors = { + 'unique_ids': unique_ids, + 'start_logits': start_logits, + 'end_logits': end_logits, + } + return {f'{prefix}_loss': loss, f'{prefix}_tensors': tensors} + + def test_step(self, batch, batch_idx): + return self.validation_step(batch, batch_idx) + + def validation_epoch_end(self, outputs): + prefix = "test" if self.trainer.testing else "val" + + avg_loss = torch.stack([x[f'{prefix}_loss'] for x in outputs]).mean() + + unique_ids = torch.cat([x[f'{prefix}_tensors']['unique_ids'] for x in outputs]) + start_logits = torch.cat([x[f'{prefix}_tensors']['start_logits'] for x in outputs]) + end_logits = torch.cat([x[f'{prefix}_tensors']['end_logits'] for x in outputs]) + + all_unique_ids = [] + all_start_logits = [] + all_end_logits = [] + if torch.distributed.is_initialized(): + world_size = torch.distributed.get_world_size() + for ind in range(world_size): + all_unique_ids.append(torch.empty_like(unique_ids)) + all_start_logits.append(torch.empty_like(start_logits)) + all_end_logits.append(torch.empty_like(end_logits)) + torch.distributed.all_gather(all_unique_ids, unique_ids) + torch.distributed.all_gather(all_start_logits, start_logits) + torch.distributed.all_gather(all_end_logits, end_logits) + else: + all_unique_ids.append(unique_ids) + all_start_logits.append(start_logits) + all_end_logits.append(end_logits) + + eval_results, all_predictions, all_nbest = {}, [], [] + if not torch.distributed.is_initialized() or torch.distributed.get_rank() == 0: + + unique_ids = [] + start_logits = [] + end_logits = [] + for u in all_unique_ids: + unique_ids.extend(tensor2list(u)) + for u in all_start_logits: + start_logits.extend(tensor2list(u)) + for u in all_end_logits: + end_logits.extend(tensor2list(u)) + + eval_dataset = self._test_dl.dataset if self.trainer.testing else self._validation_dl.dataset + eval_results, _, _ = self.evaluate( + eval_dataset.features, + eval_dataset.examples, + eval_dataset.processor, + unique_ids=unique_ids, + start_logits=start_logits, + end_logits=end_logits, + n_best_size=self._cfg.dataset.n_best_size, + max_answer_length=self._cfg.dataset.max_answer_length, + version_2_with_negative=self._cfg.dataset.version_2_with_negative, + null_score_diff_threshold=self._cfg.dataset.null_score_diff_threshold, + do_lower_case=self._cfg.dataset.do_lower_case, + ) + + self.log(f'{prefix}_loss', avg_loss) + for eval_key in eval_results: + logging.info(f"{prefix} {eval_key}: {eval_results[eval_key]}") + self.log(f"{prefix}_{eval_key}", eval_results[eval_key]) + + def test_epoch_end(self, outputs): + return self.validation_epoch_end(outputs) + + @typecheck() + def forward(self, input_ids, attention_mask, token_type_ids): + with torch.cuda.amp.autocast(): + hidden_states = self.bert_model( + input_ids=input_ids, token_type_ids=token_type_ids, attention_mask=attention_mask + ) + + if isinstance(hidden_states, tuple): + hidden_states = hidden_states[0] + + logits = self.classifier(hidden_states=hidden_states) + + return logits + + @torch.no_grad() + def inference( + self, + file: str, + batch_size: int = 1, + num_samples: int = -1, + output_nbest_file: Optional[str] = None, + output_prediction_file: Optional[str] = None, + ): + """ + Get prediction for unlabeled inference data + + Args: + file: inference data + batch_size: batch size to use during inference + num_samples: number of samples to use of inference data. Default: -1 if all data should be used. + output_nbest_file: optional output file for writing out nbest list + output_prediction_file: optional output file for writing out predictions + + Returns: + model predictions, model nbest list + """ + + # store predictions for all queries in a single list + all_predictions = [] + all_nbest = [] + mode = self.training + device = "cuda" if isinstance(self.trainer.device_ids, list) else "cpu" + + try: + # Switch model to evaluation mode + self.eval() + self.to(device) + logging_level = logging.get_verbosity() + logging.set_verbosity(logging.WARNING) + + infer_datalayer = self.setup_inference_data( + file, batch_size=batch_size, num_samples=num_samples, num_workers=2, + ) + + all_logits = [] + all_unique_ids = [] + for i, batch in enumerate(infer_datalayer): + input_ids, token_type_ids, attention_mask, unique_ids = batch + logits = self.forward( + input_ids=input_ids.to(device), + token_type_ids=token_type_ids.to(device), + attention_mask=attention_mask.to(device), + ) + all_logits.append(logits) + all_unique_ids.append(unique_ids) + logits = torch.cat(all_logits) + unique_ids = tensor2list(torch.cat(all_unique_ids)) + s, e = logits.split(dim=-1, split_size=1) + start_logits = tensor2list(s.squeeze(-1)) + end_logits = tensor2list(e.squeeze(-1)) + (all_predictions, all_nbest, scores_diff) = self.get_predictions( + infer_datalayer.dataset.features, + infer_datalayer.dataset.examples, + infer_datalayer.dataset.processor, + unique_ids=unique_ids, + start_logits=start_logits, + end_logits=end_logits, + n_best_size=self._cfg.dataset.n_best_size, + max_answer_length=self._cfg.dataset.max_answer_length, + version_2_with_negative=self._cfg.dataset.version_2_with_negative, + null_score_diff_threshold=self._cfg.dataset.null_score_diff_threshold, + do_lower_case=self._cfg.dataset.do_lower_case, + ) + + if output_prediction_file: + QAMetrics.dump_predicted_answers_to_file( + output_prediction_file, infer_datalayer.dataset.examples, all_predictions, + ) + + if output_nbest_file: + QAMetrics.dump_nbest_predictions_to_file( + output_nbest_file, + infer_datalayer.dataset.examples, + all_nbest, + keys_to_dump=["text", "probability"], + ) + + finally: + # set mode back to its original value + self.train(mode=mode) + logging.set_verbosity(logging_level) + + return all_predictions, all_nbest + + def evaluate( + self, + features: List, + examples: List, + processor: object, + unique_ids: List[str], + start_logits: List[List[float]], + end_logits: List[List[float]], + n_best_size: int, + max_answer_length: int, + do_lower_case: bool, + version_2_with_negative: bool, + null_score_diff_threshold: float, + ): + (all_predictions, all_nbest_json, scores_diff_json) = self.get_predictions( + features, + examples, + processor, + unique_ids, + start_logits, + end_logits, + n_best_size, + max_answer_length, + do_lower_case, + version_2_with_negative, + null_score_diff_threshold, + ) + + eval_results = QAMetrics.evaluate_predictions(examples, all_predictions) + + return eval_results, all_predictions, all_nbest_json + + def get_predictions( + self, + features: List, + examples: List, + processor: object, + unique_ids: List[int], + start_logits: List[List[float]], + end_logits: List[List[float]], + n_best_size: int, + max_answer_length: int, + do_lower_case: bool, + version_2_with_negative: bool, + null_score_diff_threshold: float, + ): + example_index_to_features = collections.defaultdict(list) + + unique_id_to_pos = {} + for index, unique_id in enumerate(unique_ids): + unique_id_to_pos[unique_id] = index + + for feature in features: + example_index_to_features[feature.example_index].append(feature) + + _PrelimPrediction = collections.namedtuple( + "PrelimPrediction", ["feature_index", "start_index", "end_index", "start_logit", "end_logit"] + ) + + all_predictions = collections.OrderedDict() + all_nbest_json = collections.OrderedDict() + scores_diff_json = collections.OrderedDict() + for (example_index, example) in enumerate(examples): + + # finish this loop if we went through all batch examples + if example_index >= len(unique_ids): + break + + curr_features = example_index_to_features[example_index] + + doc_tokens, _, _, _, _ = BERTQADataset.get_doc_tokens_and_offset_from_context_id( + example.context_id, + example.start_position_character, + example.is_impossible, + example.answer_text, + processor.doc_id_to_context_text, + ) + prelim_predictions = [] + # keep track of the minimum score of null start+end of position 0 + # large and positive + score_null = 1000000 + # the paragraph slice with min null score + min_null_feature_index = 0 + # start logit at the slice with min null score + null_start_logit = 0 + # end logit at the slice with min null score + null_end_logit = 0 + for (feature_index, feature) in enumerate(curr_features): + pos = unique_id_to_pos[feature.unique_id] + start_indexes = self._get_best_indexes(start_logits[pos], n_best_size) + end_indexes = self._get_best_indexes(end_logits[pos], n_best_size) + # if we could have irrelevant answers, + # get the min score of irrelevant + if version_2_with_negative: + feature_null_score = start_logits[pos][0] + end_logits[pos][0] + if feature_null_score < score_null: + score_null = feature_null_score + min_null_feature_index = feature_index + null_start_logit = start_logits[pos][0] + null_end_logit = end_logits[pos][0] + for start_index in start_indexes: + for end_index in end_indexes: + # We could hypothetically create invalid predictions, + # e.g., predict that the start of the span is in the + # question. We throw out all invalid predictions. + if start_index >= len(feature.tokens): + continue + if end_index >= len(feature.tokens): + continue + if start_index not in feature.token_to_orig_map: + continue + if end_index not in feature.token_to_orig_map: + continue + if not feature.token_is_max_context.get(start_index, False): + continue + if end_index < start_index: + continue + length = end_index - start_index + 1 + if length > max_answer_length: + continue + prelim_predictions.append( + _PrelimPrediction( + feature_index=feature_index, + start_index=start_index, + end_index=end_index, + start_logit=start_logits[pos][start_index], + end_logit=end_logits[pos][end_index], + ) + ) + + if version_2_with_negative: + prelim_predictions.append( + _PrelimPrediction( + feature_index=min_null_feature_index, + start_index=0, + end_index=0, + start_logit=null_start_logit, + end_logit=null_end_logit, + ) + ) + prelim_predictions = sorted(prelim_predictions, key=lambda x: (x.start_logit + x.end_logit), reverse=True) + + _NbestPrediction = collections.namedtuple("NbestPrediction", ["text", "start_logit", "end_logit"]) + + seen_predictions = {} + nbest = [] + for pred in prelim_predictions: + if len(nbest) >= n_best_size: + break + feature = curr_features[pred.feature_index] + if pred.start_index > 0: # this is a non-null prediction + tok_tokens = feature.tokens[pred.start_index : (pred.end_index + 1)] + orig_doc_start = feature.token_to_orig_map[pred.start_index] + orig_doc_end = feature.token_to_orig_map[pred.end_index] + orig_tokens = doc_tokens[orig_doc_start : (orig_doc_end + 1)] + tok_text = " ".join(tok_tokens) + + # De-tokenize WordPieces that have been split off. + tok_text = tok_text.replace(" ##", "") + tok_text = tok_text.replace("##", "") + + # Clean whitespace + tok_text = tok_text.strip() + tok_text = " ".join(tok_text.split()) + orig_text = " ".join(orig_tokens) + + final_text = self._get_final_text(tok_text, orig_text, do_lower_case) + if final_text in seen_predictions: + continue + + seen_predictions[final_text] = True + else: + final_text = "" + seen_predictions[final_text] = True + + nbest.append(_NbestPrediction(text=final_text, start_logit=pred.start_logit, end_logit=pred.end_logit)) + + # if we didn't include the empty option in the n-best, include it + if version_2_with_negative: + if "" not in seen_predictions: + nbest.append(_NbestPrediction(text="", start_logit=null_start_logit, end_logit=null_end_logit)) + + # In very rare edge cases we could only + # have single null pred. We just create a nonce prediction + # in this case to avoid failure. + if len(nbest) == 1: + nbest.insert(0, _NbestPrediction(text="empty", start_logit=0.0, end_logit=0.0)) + + # In very rare edge cases we could have no valid predictions. So we + # just create a nonce prediction in this case to avoid failure. + if not nbest: + nbest.append(_NbestPrediction(text="empty", start_logit=0.0, end_logit=0.0)) + + assert len(nbest) >= 1 + + total_scores = [] + best_non_null_entry = None + for entry in nbest: + total_scores.append(entry.start_logit + entry.end_logit) + if not best_non_null_entry: + if entry.text: + best_non_null_entry = entry + + probs = _compute_softmax(total_scores) + + nbest_json = [] + for (i, entry) in enumerate(nbest): + output = collections.OrderedDict() + output["question"] = example.question_text + output["text"] = entry.text + output["probability"] = probs[i] + output["start_logit"] = ( + entry.start_logit + if (isinstance(entry.start_logit, float) or isinstance(entry.start_logit, int)) + else list(entry.start_logit) + ) + output["end_logit"] = ( + entry.end_logit + if (isinstance(entry.end_logit, float) or isinstance(entry.end_logit, int)) + else list(entry.end_logit) + ) + nbest_json.append(output) + + assert len(nbest_json) >= 1 + if not version_2_with_negative: + all_predictions[example.qas_id] = nbest_json[0]["text"] + else: + # predict "" iff the null score - + # the score of best non-null > threshold + score_diff = score_null - best_non_null_entry.start_logit - best_non_null_entry.end_logit + scores_diff_json[example.qas_id] = score_diff + if score_diff > null_score_diff_threshold: + all_predictions[example.qas_id] = "" + else: + all_predictions[example.qas_id] = best_non_null_entry.text + all_nbest_json[example.qas_id] = nbest_json + + return all_predictions, all_nbest_json, scores_diff_json + + def _setup_dataloader_from_config(self, cfg: DictConfig, mode: str): + processor = QAProcessor(cfg.file, mode) + + dataset = BERTQADataset( + data_file=cfg.file, + processor=processor, + tokenizer=self.tokenizer, + keep_doc_spans=self._cfg.dataset.keep_doc_spans, + doc_stride=self._cfg.dataset.doc_stride, + max_query_length=self._cfg.dataset.max_query_length, + max_seq_length=self._cfg.dataset.max_seq_length, + version_2_with_negative=self._cfg.dataset.version_2_with_negative, + num_samples=cfg.num_samples, + mode=mode, + use_cache=self._cfg.dataset.use_cache, + ) + + data_loader = torch.utils.data.DataLoader( + dataset=dataset, + batch_size=cfg.batch_size, + collate_fn=dataset.collate_fn, + drop_last=cfg.drop_last, + shuffle=cfg.shuffle, + num_workers=cfg.num_workers, + pin_memory=cfg.pin_memory, + ) + + return data_loader + + def _get_best_indexes(self, logits, n_best_size): + """ Get the n-best logits from a list """ + + best_indices = np.argsort(logits)[::-1] + + return best_indices[:n_best_size] + + def _get_final_text(self, pred_text: str, orig_text: str, do_lower_case: bool, verbose_logging: bool = False): + """ + Project the tokenized prediction back to the original text. + When we created the data, we kept track of the alignment between original + (whitespace tokenized) tokens and our WordPiece tokenized tokens. So + now `orig_text` contains the span of our original text corresponding to + the span that we predicted. + + However, `orig_text` may contain extra characters that we don't want in + our prediction. + + For example, let's say: + pred_text = steve smith + orig_text = Steve Smith's + + We don't want to return `orig_text` because it contains the extra "'s". + + We don't want to return `pred_text` because it's already been normalized + (the SQuAD eval script also does punctuation stripping/lower casing but + our tokenizer does additional normalization like stripping accent + characters). + + What we really want to return is "Steve Smith". + + Therefore, we have to apply a semi-complicated alignment heuristic + between `pred_text` and `orig_text` to get a character-to-character + alignment. This can fail in certain cases in which case we just return + `orig_text` + """ + + def _strip_spaces(text): + ns_chars = [] + ns_to_s_map = collections.OrderedDict() + for (i, c) in enumerate(text): + if c == " ": + continue + ns_to_s_map[len(ns_chars)] = i + ns_chars.append(c) + ns_text = "".join(ns_chars) + return ns_text, ns_to_s_map + + # We first tokenize `orig_text`, strip whitespace from the result + # and `pred_text`, and check if they are the same length. If they are + # NOT the same length, the heuristic has failed. If they are the same + # length, we assume the characters are one-to-one aligned. + tokenizer = BasicTokenizer(do_lower_case=do_lower_case) + + tok_text = " ".join(tokenizer.tokenize(orig_text)) + + start_position = tok_text.find(pred_text) + if start_position == -1: + if verbose_logging: + logging.warning("Unable to find text: '%s' in '%s'" % (pred_text, orig_text)) + return orig_text + end_position = start_position + len(pred_text) - 1 + + (orig_ns_text, orig_ns_to_s_map) = _strip_spaces(orig_text) + (tok_ns_text, tok_ns_to_s_map) = _strip_spaces(tok_text) + + if len(orig_ns_text) != len(tok_ns_text): + if verbose_logging: + logging.warning( + "Length not equal after stripping spaces: '%s' vs '%s'", orig_ns_text, tok_ns_text, + ) + return orig_text + + # We then project the characters in `pred_text` back to `orig_text` using + # the character-to-character alignment. + tok_s_to_ns_map = {} + for (i, tok_index) in tok_ns_to_s_map.items(): + tok_s_to_ns_map[tok_index] = i + + orig_start_position = None + if start_position in tok_s_to_ns_map: + ns_start_position = tok_s_to_ns_map[start_position] + if ns_start_position in orig_ns_to_s_map: + orig_start_position = orig_ns_to_s_map[ns_start_position] + + if orig_start_position is None: + if verbose_logging: + logging.warning("Couldn't map start position") + return orig_text + + orig_end_position = None + if end_position in tok_s_to_ns_map: + ns_end_position = tok_s_to_ns_map[end_position] + if ns_end_position in orig_ns_to_s_map: + orig_end_position = orig_ns_to_s_map[ns_end_position] + + if orig_end_position is None: + if verbose_logging: + logging.warning("Couldn't map end position") + return orig_text + + output_text = orig_text[orig_start_position : (orig_end_position + 1)] + + return output_text + + @classmethod + def list_available_models(cls) -> Optional[PretrainedModelInfo]: + """ + This method returns a list of pre-trained model which can be instantiated directly from NVIDIA's NGC cloud. + + Returns: + List of available pre-trained models. + """ + + result = [] + + result.append( + PretrainedModelInfo( + pretrained_model_name="qa_squadv1.1_bertbase", + location="https://api.ngc.nvidia.com/v2/models/nvidia/nemo/qa_squadv1_1_bertbase/versions/1.0.0rc1/files/qa_squadv1.1_bertbase.nemo", + description="Question answering model finetuned from NeMo BERT Base Uncased on SQuAD v1.1 dataset which obtains an exact match (EM) score of 82.78% and an F1 score of 89.97%.", + ) + ) + + result.append( + PretrainedModelInfo( + pretrained_model_name="qa_squadv2.0_bertbase", + location="https://api.ngc.nvidia.com/v2/models/nvidia/nemo/qa_squadv2_0_bertbase/versions/1.0.0rc1/files/qa_squadv2.0_bertbase.nemo", + description="Question answering model finetuned from NeMo BERT Base Uncased on SQuAD v2.0 dataset which obtains an exact match (EM) score of 75.04% and an F1 score of 78.08%.", + ) + ) + + result.append( + PretrainedModelInfo( + pretrained_model_name="qa_squadv1_1_bertlarge", + location="https://api.ngc.nvidia.com/v2/models/nvidia/nemo/qa_squadv1_1_bertlarge/versions/1.0.0rc1/files/qa_squadv1.1_bertlarge.nemo", + description="Question answering model finetuned from NeMo BERT Large Uncased on SQuAD v1.1 dataset which obtains an exact match (EM) score of 85.44% and an F1 score of 92.06%.", + ) + ) + + result.append( + PretrainedModelInfo( + pretrained_model_name="qa_squadv2.0_bertlarge", + location="https://api.ngc.nvidia.com/v2/models/nvidia/nemo/qa_squadv2_0_bertlarge/versions/1.0.0rc1/files/qa_squadv2.0_bertlarge.nemo", + description="Question answering model finetuned from NeMo BERT Large Uncased on SQuAD v2.0 dataset which obtains an exact match (EM) score of 80.22% and an F1 score of 83.05%.", + ) + ) + + result.append( + PretrainedModelInfo( + pretrained_model_name="qa_squadv1_1_megatron_cased", + location="https://api.ngc.nvidia.com/v2/models/nvidia/nemo/qa_squadv1_1_megatron_cased/versions/1.0.0rc1/files/qa_squadv1.1_megatron_cased.nemo", + description="Question answering model finetuned from Megatron Cased on SQuAD v1.1 dataset which obtains an exact match (EM) score of 88.18% and an F1 score of 94.07%.", + ) + ) + + result.append( + PretrainedModelInfo( + pretrained_model_name="qa_squadv2.0_megatron_cased", + location="https://api.ngc.nvidia.com/v2/models/nvidia/nemo/qa_squadv2_0_megatron_cased/versions/1.0.0rc1/files/qa_squadv2.0_megatron_cased.nemo", + description="Question answering model finetuned from Megatron Cased on SQuAD v2.0 dataset which obtains an exact match (EM) score of 84.73% and an F1 score of 87.89%.", + ) + ) + + result.append( + PretrainedModelInfo( + pretrained_model_name="qa_squadv1.1_megatron_uncased", + location="https://api.ngc.nvidia.com/v2/models/nvidia/nemo/qa_squadv1_1_megatron_uncased/versions/1.0.0rc1/files/qa_squadv1.1_megatron_uncased.nemo", + description="Question answering model finetuned from Megatron Unased on SQuAD v1.1 dataset which obtains an exact match (EM) score of 87.61% and an F1 score of 94.00%.", + ) + ) + + result.append( + PretrainedModelInfo( + pretrained_model_name="qa_squadv2.0_megatron_uncased", + location="https://api.ngc.nvidia.com/v2/models/nvidia/nemo/qa_squadv2_0_megatron_uncased/versions/1.0.0rc1/files/qa_squadv2.0_megatron_uncased.nemo", + description="Question answering model finetuned from Megatron Uncased on SQuAD v2.0 dataset which obtains an exact match (EM) score of 84.48% and an F1 score of 87.65%.", + ) + ) + + return result diff --git a/nemo/collections/nlp/models/question_answering/qa_gpt_model.py b/nemo/collections/nlp/models/question_answering/qa_gpt_model.py new file mode 100644 index 000000000000..72a26d1e09ec --- /dev/null +++ b/nemo/collections/nlp/models/question_answering/qa_gpt_model.py @@ -0,0 +1,364 @@ +# Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import collections +from typing import List, Optional + +import torch +from omegaconf import DictConfig +from pytorch_lightning import Trainer +from transformers import AutoModelForCausalLM + +from nemo.collections.nlp.data.question_answering.data_processor.qa_processing import QAProcessor +from nemo.collections.nlp.data.question_answering.dataset.qa_gpt_dataset import GPTQADataset +from nemo.collections.nlp.metrics.qa_metrics import QAMetrics +from nemo.collections.nlp.models.language_modeling.megatron_gpt_model import MegatronGPTModel +from nemo.collections.nlp.models.question_answering.qa_base_model import BaseQAModel +from nemo.core.classes.common import PretrainedModelInfo, typecheck +from nemo.utils import logging + + +class GPTQAModel(BaseQAModel): + def __init__(self, cfg: DictConfig, trainer: Trainer = None): + self.cfg = cfg + + self.setup_tokenizer(cfg.tokenizer) + self.tokenizer.tokenizer.pad_token = self.tokenizer.tokenizer.eos_token + self.epoch_number = 0 + super().__init__(cfg=cfg, trainer=trainer, no_lm_init=True) + + if self.cfg.library == "huggingface": + self.language_model = AutoModelForCausalLM.from_pretrained(cfg.language_model.pretrained_model_name) + self.language_model.resize_token_embeddings(len(self.tokenizer.tokenizer)) + if self.cfg.language_model.lm_checkpoint: + self.language_model.load_state_dict(torch.load(self.cfg.language_model.lm_checkpoint)) + elif self.cfg.library == "megatron": + self.language_model = MegatronGPTModel.restore_from(cfg.language_model.lm_checkpoint, trainer=trainer) + + def training_step(self, batch, batch_idx): + input_ids, input_attn_mask, _, _, labels = batch + loss, _ = self(input_ids, input_attn_mask, labels) + lr = self._optimizer.param_groups[0]['lr'] + + self.log('lr', lr, prog_bar=True) + self.log("train_loss", loss, on_step=True, on_epoch=True, prog_bar=True, logger=True) + + return {'loss': loss} + + def validation_step(self, batch, batch_idx): + prefix = "test" if self.trainer.testing else "val" + + input_ids, input_attn_mask, unique_ids, training_mask_end, labels = batch + loss, per_sample_perplexity = self.forward(input_ids, input_attn_mask, labels) + generated_answers = self._generate_candidates(input_ids, input_attn_mask, training_mask_end) + labels[labels == -100] = self.tokenizer.tokenizer.pad_token_id + + return { + "unique_ids": unique_ids, + f"{prefix}_loss": loss, + "per_sample_perplexity": per_sample_perplexity, + "input": self.tokenizer.tokenizer.batch_decode(input_ids, skip_special_tokens=True), + "ground_truth_answers": self.tokenizer.tokenizer.batch_decode(labels, skip_special_tokens=True), + "generated_answers": generated_answers, + } + + def test_step(self, batch, batch_idx): + return self.validation_step(batch, batch_idx) + + def validation_epoch_end(self, outputs): + prefix = "test" if self.trainer.testing else "val" + + loss_terms = [x[f"{prefix}_loss"] for x in outputs] + generated_answers, unique_ids, per_sample_perplexity = QAMetrics.convert_dict_outputs_to_lists( + outputs, ["generated_answers", "unique_ids", "per_sample_perplexity"] + ) + + avg_loss = torch.stack(loss_terms).mean() + + eval_dataset = self._test_dl.dataset if self.trainer.testing else self._validation_dl.dataset + eval_results, _, _ = self.evaluate( + eval_dataset.features, eval_dataset.examples, unique_ids, per_sample_perplexity, generated_answers, + ) + + self.log(f'{prefix}_loss', avg_loss) + for eval_key in eval_results: + logging.info(f"{prefix} {eval_key}: {eval_results[eval_key]}") + self.log(f"{prefix}_{eval_key}", eval_results[eval_key]) + + def test_epoch_end(self, outputs): + self.validation_epoch_end(outputs) + + @typecheck() + def forward(self, input_ids, input_attn_mask, labels): + loss, per_sample_perplexity = None, None + if self.cfg.library == "huggingface": + output = self.language_model(input_ids=input_ids, attention_mask=input_attn_mask, labels=labels) + loss, lm_logits = output['loss'], output['logits'] + shift_logits = lm_logits[..., :-1, :].contiguous() + shift_labels = labels[..., 1:].contiguous() + per_sample_perplexity = self._get_per_sample_perplexity(shift_logits, shift_labels) + + elif self.cfg.library == "megatron": + raise NotImplementedError() + + return loss, per_sample_perplexity + + @torch.no_grad() + def inference( + self, + file: str, + batch_size: int = 1, + num_samples: int = -1, + output_prediction_file: Optional[str] = None, + output_nbest_file: Optional[str] = None, + ): + all_predictions = [] + mode = self.training + device = "cuda" if isinstance(self.trainer.device_ids, list) else "cpu" + if self.cfg.library == "huggingface": + try: + self.eval() + self.to(device) + logging_level = logging.get_verbosity() + logging.set_verbosity(logging.WARNING) + + inference_dl = self.setup_inference_data(file, batch_size=batch_size, num_samples=num_samples) + + outputs = self._inference(inference_dl, device) + generated_answers, unique_ids, per_sample_perplexity = QAMetrics.convert_dict_outputs_to_lists( + outputs, ["generated_answers", "unique_ids", "per_sample_perplexity"] + ) + all_predictions, all_nbest_perdictions = self._get_predictions( + inference_dl.dataset.features, + inference_dl.dataset.examples, + unique_ids, + per_sample_perplexity, + generated_answers, + ) + + if output_prediction_file: + QAMetrics.dump_predicted_answers_to_file( + output_prediction_file, inference_dl.dataset.examples, all_predictions + ) + + if output_nbest_file: + QAMetrics.dump_nbest_predictions_to_file( + output_nbest_file, + inference_dl.dataset.examples, + all_nbest_perdictions, + keys_to_dump=["generated_text", "perplexity"], + ) + + finally: + # set mode back to its original value + self.train(mode=mode) + logging.set_verbosity(logging_level) + + elif self.cfg.library == 'megatron': + raise ValueError("Megatron Inference is not supported by GPTQAModel") + + return all_predictions, all_nbest_perdictions + + def evaluate( + self, features, examples, unique_ids, per_sample_perplexity, generated_texts, + ): + all_predictions, all_nbest_predictions = self._get_predictions( + features, examples, unique_ids, per_sample_perplexity, generated_texts, + ) + + eval_results = QAMetrics.evaluate_predictions(examples, all_predictions) + + return eval_results, all_predictions, all_nbest_predictions + + def _setup_dataloader_from_config(self, cfg: DictConfig, mode: str): + processor = QAProcessor(cfg.file, mode) + + dataset = GPTQADataset( + data_file=cfg.file, + processor=processor, + tokenizer=self.tokenizer, + keep_doc_spans=self._cfg.dataset.keep_doc_spans, + doc_stride=self._cfg.dataset.doc_stride, + max_query_length=self._cfg.dataset.max_query_length, + max_seq_length=self._cfg.dataset.max_seq_length, + max_answer_length=self._cfg.dataset.max_answer_length, + check_if_answer_in_context=self._cfg.dataset.check_if_answer_in_context, + num_samples=cfg.num_samples, + mode=mode, + use_cache=self._cfg.dataset.use_cache, + ) + + data_loader = torch.utils.data.DataLoader( + dataset=dataset, + batch_size=cfg.batch_size, + collate_fn=dataset.collate_fn, + drop_last=cfg.drop_last, + shuffle=cfg.shuffle, + num_workers=cfg.num_workers, + pin_memory=cfg.pin_memory, + ) + + return data_loader + + def _get_predictions( + self, features, examples: List, unique_ids: List[int], per_sample_perplexity: List, generated_texts: List, + ): + unique_id_to_pos = {} + for index, unique_id in enumerate(unique_ids): + unique_id_to_pos[unique_id] = index + + example_index_to_features = collections.defaultdict(list) + for feature in features: + example_index_to_features[feature.example_index].append(feature) + + _PrelimPrediction = collections.namedtuple( + "PrelimPrediction", ["feature_index", "perplexity", "generated_text"] + ) + + all_predictions = collections.OrderedDict() + all_nbest_json = collections.OrderedDict() + for (example_index, example) in enumerate(examples): + + # finish this loop if we went through all batch examples + if example_index >= len(unique_ids): + break + + curr_features = example_index_to_features[example_index] + prelim_predictions = [] + for (feature_index, feature) in enumerate(curr_features): + pos = unique_id_to_pos[feature.unique_id] + curr_perplexity = per_sample_perplexity[pos] + curr_generated_text = generated_texts[pos] + prelim_prediction = _PrelimPrediction(feature_index, curr_perplexity, curr_generated_text) + prelim_predictions.append(prelim_prediction) + + prelim_predictions = sorted(prelim_predictions, key=lambda x: x.perplexity) + all_predictions[example.qas_id] = prelim_predictions[0].generated_text + all_nbest_json[example.qas_id] = [pred._asdict() for pred in prelim_predictions] + + return all_predictions, all_nbest_json + + def _inference(self, inference_dl, device): + outputs = [] + for i, batch in enumerate(inference_dl): + input_ids, input_attn_mask, unique_ids, training_mask_end = batch + input_ids, input_attn_mask, training_mask_end = ( + tensor.to(device) for tensor in [input_ids, input_attn_mask, training_mask_end] + ) + input_ids, input_attn_mask, labels, generated_texts = self._prep_inference_labels( + input_ids, input_attn_mask, training_mask_end, device + ) + + _, per_sample_perplexity = self.forward(input_ids, input_attn_mask, labels) + labels[labels == -100] = self.tokenizer.tokenizer.pad_token_id + + outputs.append( + { + "unique_ids": unique_ids, + "per_sample_perplexity": per_sample_perplexity, + "generated_answers": generated_texts, + } + ) + + return outputs + + def _prep_inference_labels(self, input_ids, input_attn_mask, training_mask_end, device): + + # generate answers by decoding inputs and format into ipnut template + decoded_inputs = self.tokenizer.tokenizer.batch_decode(input_ids, skip_special_tokens=True) + generated_texts = self._generate_candidates(input_ids, input_attn_mask, training_mask_end) + inputs_with_answer = [ + f"{inp}{ans}{self.tokenizer.tokenizer.eos_token}" if ans else f"{inp}{self.tokenizer.tokenizer.eos_token}" + for inp, ans in zip(decoded_inputs, generated_texts) + ] + + # encode template with generated answers + encoded_dict = self.tokenizer.tokenizer( + inputs_with_answer, + truncation=True, + max_length=self._cfg.dataset.max_seq_length, + padding="max_length", + return_tensors="pt", + ) + input_ids, input_attn_mask = ( + tensor.to(device) for tensor in [encoded_dict["input_ids"], encoded_dict["attention_mask"]] + ) + labels = GPTQADataset.update_labels_for_no_pad_loss(input_ids, training_mask_end, input_attn_mask) + if len(labels.shape) == 1: + labels = torch.unsqueeze(labels, 0) + labels = labels.to(device) + + return input_ids, input_attn_mask, labels, generated_texts + + def _generate_candidates(self, input_ids, input_attn_mask, training_mask_end): + num_tokens_to_generate = self.cfg.tokens_to_generate + if self.cfg.library == "huggingface": + generated_token_ids = [] + max_length = 0 + for i in range(input_ids.size(0)): + param_dict = { + "input_ids": input_ids[i : i + 1, : training_mask_end[i]], + "attention_masks": input_attn_mask[i : i + 1, : training_mask_end[i]], + "max_length": training_mask_end[i] + num_tokens_to_generate, + "pad_token_id": self.tokenizer.tokenizer.pad_token_id, + } + generated_token_ids.append(self.language_model.generate(**param_dict, skip_special_tokens=True)) + max_length = max(max_length, generated_token_ids[-1].size(1)) + + # pad each generated to ensure they are of same length in dim 1, therefore stack-able + generated_token_ids = [ + torch.cat( + [i, torch.ones((1, max_length - i.size(1))).to(i.device) * self.tokenizer.tokenizer.pad_token_id], + axis=-1, + ) + for i in generated_token_ids + ] + generated_token_ids = torch.cat(generated_token_ids, axis=0) + generated_answers = self._get_answers_from_generated_tokens( + generated_token_ids, training_mask_end=training_mask_end + ) + + elif self.cfg.library == 'megatron': + raise ValueError("Megatron Generation is not supported by GPTQAModel") + + return generated_answers + + def _get_answers_from_generated_tokens(self, token_ids, training_mask_end=None): + answers = [] + for i in range(token_ids.size(0)): + start_point = 0 if training_mask_end is None else training_mask_end[i].item() + stop_point = token_ids.size(1) + + for j in range(start_point, stop_point): + if token_ids.data[i, j] == self.tokenizer.tokenizer.pad_token_id: + stop_point = j + break + + curr_answer = self.tokenizer.tokenizer.decode( + token_ids[i, start_point:stop_point], skip_special_tokens=True + ).strip() + answers.append(curr_answer) + + return answers + + @classmethod + def list_available_models(cls) -> Optional[PretrainedModelInfo]: + """ + This method returns a list of pre-trained model which can be instantiated directly from NVIDIA's NGC cloud. + + Returns: + List of available pre-trained models. + """ + result = [] + return result diff --git a/nemo/collections/nlp/models/question_answering/qa_model.py b/nemo/collections/nlp/models/question_answering/qa_model.py index 091c7e7b5ce2..2917b838a734 100644 --- a/nemo/collections/nlp/models/question_answering/qa_model.py +++ b/nemo/collections/nlp/models/question_answering/qa_model.py @@ -57,14 +57,15 @@ def __init__(self, cfg: DictConfig, trainer: Trainer = None): @typecheck() def forward(self, input_ids, attention_mask, token_type_ids): - hidden_states = self.bert_model( - input_ids=input_ids, token_type_ids=token_type_ids, attention_mask=attention_mask - ) + with autocast(): + hidden_states = self.bert_model( + input_ids=input_ids, token_type_ids=token_type_ids, attention_mask=attention_mask + ) - if isinstance(hidden_states, tuple): - hidden_states = hidden_states[0] + if isinstance(hidden_states, tuple): + hidden_states = hidden_states[0] - logits = self.classifier(hidden_states=hidden_states) + logits = self.classifier(hidden_states=hidden_states) return logits def training_step(self, batch, batch_idx): diff --git a/nemo/collections/nlp/models/question_answering/qa_s2s_model.py b/nemo/collections/nlp/models/question_answering/qa_s2s_model.py new file mode 100644 index 000000000000..cc9df420c9b1 --- /dev/null +++ b/nemo/collections/nlp/models/question_answering/qa_s2s_model.py @@ -0,0 +1,345 @@ +# Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import collections +from typing import List, Optional + +import torch +from omegaconf import DictConfig, OmegaConf, open_dict +from pytorch_lightning import Trainer +from torch.cuda.amp import autocast +from transformers import AutoModelForSeq2SeqLM + +from nemo.collections.nlp.data.question_answering.data_processor.qa_processing import QAProcessor +from nemo.collections.nlp.data.question_answering.dataset.qa_s2s_dataset import S2SQADataset +from nemo.collections.nlp.metrics.qa_metrics import QAMetrics +from nemo.collections.nlp.models.language_modeling.megatron_t5_model import MegatronT5Model +from nemo.collections.nlp.models.question_answering.qa_base_model import BaseQAModel +from nemo.core.classes.common import PretrainedModelInfo, typecheck +from nemo.utils import logging + + +class S2SQAModel(BaseQAModel): + def __init__(self, cfg: DictConfig, trainer: Trainer = None): + + self.cfg = cfg + + if self.cfg.library == "huggingface": + self.setup_tokenizer(cfg.tokenizer) + elif self.cfg.library == "megatron": + # supporting MegatronT5Model in precision = fp16 + t5_cfg = MegatronT5Model.restore_from( + restore_path=cfg.language_model.lm_checkpoint, trainer=trainer, return_config=True + ) + # Override the T5 configuration with the one from the config file. + OmegaConf.set_struct(t5_cfg, True) + with open_dict(t5_cfg): + t5_cfg.masked_softmax_fusion = False + t5_cfg.precision = 16 + + language_model = MegatronT5Model.restore_from( + restore_path=cfg.language_model.lm_checkpoint, trainer=trainer, override_config_path=t5_cfg + ) + self.tokenizer = language_model.tokenizer + + super().__init__(cfg=cfg, trainer=trainer, no_lm_init=True) + + if self.cfg.library == "huggingface": + self.language_model = AutoModelForSeq2SeqLM.from_pretrained(cfg.language_model.pretrained_model_name) + self.language_model.resize_token_embeddings(len(self.tokenizer.tokenizer)) + if self.cfg.language_model.lm_checkpoint: + self.language_model.load_state_dict(torch.load(self.cfg.language_model.lm_checkpoint)) + elif self.cfg.library == "megatron": + self.language_model = language_model + + def training_step(self, batch, batch_idx): + input_ids, input_attn_mask, unique_ids, labels = batch + loss, _ = self.forward(input_ids, input_attn_mask, labels) + lr = self._optimizer.param_groups[0]['lr'] + + self.log('lr', lr, prog_bar=True) + self.log("train_loss", loss, on_step=True, on_epoch=True, prog_bar=True, logger=True) + + return {'loss': loss} + + def validation_step(self, batch, batch_idx): + prefix = "test" if self.trainer.testing else "val" + + input_ids, input_attn_mask, unique_ids, labels = batch + loss, per_sample_perplexity = self.forward(input_ids, input_attn_mask, labels) + generated_answers = self._generate_candidates(input_ids, input_attn_mask) + + labels[labels == -100] = self.tokenizer.tokenizer.pad_token_id + + return { + "unique_ids": unique_ids, + f"{prefix}_loss": loss, + "per_sample_perplexity": per_sample_perplexity, + "input": self.tokenizer.tokenizer.batch_decode(input_ids, skip_special_tokens=True), + "ground_truth_answers": self.tokenizer.tokenizer.batch_decode(labels, skip_special_tokens=True), + "generated_answers": generated_answers, + } + + def test_step(self, batch, batch_idx): + return self.validation_step(batch, batch_idx) + + def validation_epoch_end(self, outputs): + prefix = "test" if self.trainer.testing else "val" + + loss_terms = [x[f"{prefix}_loss"] for x in outputs] + generated_answers, unique_ids, per_sample_perplexity = QAMetrics.convert_dict_outputs_to_lists( + outputs, ["generated_answers", "unique_ids", "per_sample_perplexity"] + ) + + avg_loss = torch.stack(loss_terms).mean() + + eval_dataset = self._test_dl.dataset if self.trainer.testing else self._validation_dl.dataset + eval_results, _, _ = self.evaluate( + eval_dataset.features, eval_dataset.examples, unique_ids, per_sample_perplexity, generated_answers, + ) + + self.log(f'{prefix}_loss', avg_loss) + for eval_key in eval_results: + logging.info(f"{prefix} {eval_key}: {eval_results[eval_key]}") + self.log(f"{prefix}_{eval_key}", eval_results[eval_key]) + + def test_epoch_end(self, outputs): + self.validation_epoch_end(outputs) + + @typecheck() + def forward(self, input_ids, input_attn_mask, labels): + loss, per_sample_perplexity = None, None + if self.cfg.library == "huggingface": + with autocast(enabled=False): + output = self.language_model(input_ids=input_ids, attention_mask=input_attn_mask, labels=labels) + loss = output['loss'] + lm_logits = output['logits'] + per_sample_perplexity = self._get_per_sample_perplexity(lm_logits, labels) + + elif self.cfg.library == "megatron": + labels = torch.where(labels != -100, labels, torch.zeros_like(labels)) + output_attn_masks = torch.where(labels > 0, torch.ones_like(labels), torch.zeros_like(labels)) + unmasked_unreduced_loss = self.language_model( + input_ids, labels[:, :-1], input_attn_mask, output_attn_masks[:, :-1], lm_labels=labels[:, 1:], + ) + loss = self.language_model.loss_func(output_attn_masks[:, 1:], unmasked_unreduced_loss) + per_sample_perplexity = torch.exp(unmasked_unreduced_loss) + + return loss, per_sample_perplexity + + @torch.no_grad() + def inference( + self, + file: str, + batch_size: int = 1, + num_samples: int = -1, + output_prediction_file: Optional[str] = None, + output_nbest_file: Optional[str] = None, + ): + all_predictions = [] + mode = self.training + device = "cuda" if isinstance(self.trainer.device_ids, list) else "cpu" + if self.cfg.library == "huggingface": + try: + # switch model to evaluation mode + self.eval() + self.to(device) + logging_level = logging.get_verbosity() + logging.set_verbosity(logging.WARNING) + + inference_dl = self.setup_inference_data(file, batch_size=batch_size, num_samples=num_samples) + + outputs = self._inference(inference_dl, device) + generated_answers, unique_ids, per_sample_perplexity = QAMetrics.convert_dict_outputs_to_lists( + outputs, ["generated_answers", "unique_ids", "per_sample_perplexity"] + ) + all_predictions, all_nbest_predictions = self._get_predictions( + inference_dl.dataset.features, + inference_dl.dataset.examples, + unique_ids, + per_sample_perplexity, + generated_answers, + ) + + if output_prediction_file: + QAMetrics.dump_predicted_answers_to_file( + output_prediction_file, inference_dl.dataset.examples, all_predictions + ) + + if output_nbest_file: + QAMetrics.dump_nbest_predictions_to_file( + output_nbest_file, + inference_dl.dataset.examples, + all_nbest_predictions, + keys_to_dump=["generated_text", "perplexity"], + ) + + finally: + # set mode back to its original value + self.train(mode=mode) + logging.set_verbosity(logging_level) + + elif self.cfg.library == 'megatron': + raise ValueError("Megatron Inference is not supported by S2SQAModel") + + return all_predictions, all_nbest_predictions + + def evaluate( + self, features, examples, unique_ids, per_sample_perplexity, generated_texts, + ): + all_predictions, all_nbest_json = self._get_predictions( + features, examples, unique_ids, per_sample_perplexity, generated_texts, + ) + + eval_results = QAMetrics.evaluate_predictions(examples, all_predictions) + + return eval_results, all_predictions, all_nbest_json + + def _setup_dataloader_from_config(self, cfg: DictConfig, mode: str): + processor = QAProcessor(cfg.file, mode) + + dataset = S2SQADataset( + data_file=cfg.file, + processor=processor, + tokenizer=self.tokenizer, + keep_doc_spans=self._cfg.dataset.keep_doc_spans, + doc_stride=self._cfg.dataset.doc_stride, + max_query_length=self._cfg.dataset.max_query_length, + max_seq_length=self._cfg.dataset.max_seq_length, + max_answer_length=self._cfg.dataset.max_answer_length, + check_if_answer_in_context=self._cfg.dataset.check_if_answer_in_context, + num_samples=cfg.num_samples, + mode=mode, + use_cache=self._cfg.dataset.use_cache, + ) + + data_loader = torch.utils.data.DataLoader( + dataset=dataset, + batch_size=cfg.batch_size, + collate_fn=dataset.collate_fn, + drop_last=cfg.drop_last, + shuffle=cfg.shuffle, + num_workers=cfg.num_workers, + pin_memory=cfg.pin_memory, + ) + + return data_loader + + def _get_predictions( + self, features, examples: List, unique_ids: List[int], per_sample_perplexity: List, generated_texts: List, + ): + + unique_id_to_pos = {} + for index, unique_id in enumerate(unique_ids): + unique_id_to_pos[unique_id] = index + + example_index_to_features = collections.defaultdict(list) + for feature in features: + example_index_to_features[feature.example_index].append(feature) + + _PrelimPrediction = collections.namedtuple( + "PrelimPrediction", ["feature_index", "perplexity", "generated_text"] + ) + + all_predictions = collections.OrderedDict() + all_nbest_json = collections.OrderedDict() + for (example_index, example) in enumerate(examples): + + # finish this loop if we went through all batch examples + if example_index >= len(unique_ids): + break + + curr_features = example_index_to_features[example_index] + prelim_predictions = [] + for (feature_index, feature) in enumerate(curr_features): + pos = unique_id_to_pos[feature.unique_id] + curr_perplexity = per_sample_perplexity[pos] + curr_generated_text = generated_texts[pos] + prelim_prediction = _PrelimPrediction(feature_index, curr_perplexity, curr_generated_text) + prelim_predictions.append(prelim_prediction) + + prelim_predictions = sorted(prelim_predictions, key=lambda x: x.perplexity) + all_predictions[example.qas_id] = prelim_predictions[0].generated_text + all_nbest_json[example.qas_id] = [pred._asdict() for pred in prelim_predictions] + + return all_predictions, all_nbest_json + + def _inference(self, inference_dl, device): + outputs = [] + for i, batch in enumerate(inference_dl): + + # get predictions + input_ids, input_attn_mask, unique_ids = batch + input_ids, input_attn_mask = (tensor.to(device) for tensor in [input_ids, input_attn_mask]) + generated_texts = self._generate_candidates(input_ids, input_attn_mask) + + labels = self._prep_inference_labels(generated_texts, device) + _, per_sample_perplexity = self.forward(input_ids, input_attn_mask, labels) + labels[labels == -100] = self.tokenizer.tokenizer.pad_token_id + + outputs.append( + { + "unique_ids": unique_ids, + "per_sample_perplexity": per_sample_perplexity, + "generated_answers": generated_texts, + } + ) + + return outputs + + def _prep_inference_labels(self, generated_texts, device): + encoded_output_dict = self.tokenizer.tokenizer( + generated_texts, + truncation=True, + max_length=self._cfg.dataset.max_answer_length, + padding="max_length", + return_tensors="pt", + ) + input_ids = encoded_output_dict["input_ids"].to(device) + labels = torch.squeeze(input_ids) + labels[labels == self.tokenizer.tokenizer.pad_token_id] = -100 + if len(labels.shape) == 1: + labels = torch.unsqueeze(labels, 0) + labels = labels.to(device) + + return labels + + def _generate_candidates(self, input_ids, input_attn_mask): + num_tokens_to_generate = self.cfg.tokens_to_generate + + if self.cfg.library == "huggingface": + param_dict = { + "input_ids": input_ids, + "attention_mask": input_attn_mask, + "max_length": num_tokens_to_generate, + } + generated_tokens = self.language_model.generate(**param_dict) + generated_answers = self.tokenizer.tokenizer.batch_decode(generated_tokens, skip_special_tokens=True,) + generated_answers = [ans.strip() for ans in generated_answers] + + elif self.cfg.library == 'megatron': + raise ValueError("Megatron Generation is not supported by S2SQAModel") + + return generated_answers + + @classmethod + def list_available_models(cls) -> Optional[PretrainedModelInfo]: + """ + This method returns a list of pre-trained model which can be instantiated directly from NVIDIA's NGC cloud. + + Returns: + List of available pre-trained models. + """ + result = [] + return result diff --git a/tests/collections/nlp/test_qna.py b/tests/collections/nlp/test_qna.py new file mode 100644 index 000000000000..4a470cacb711 --- /dev/null +++ b/tests/collections/nlp/test_qna.py @@ -0,0 +1,240 @@ +# Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import collections + +import pytest +import torch + +from nemo.collections.nlp.data.question_answering.dataset.qa_dataset import QADataset +from nemo.collections.nlp.data.question_answering.dataset.qa_gpt_dataset import GPTQADataset +from nemo.collections.nlp.metrics.qa_metrics import QAMetrics + + +@pytest.mark.unit +def test_remove_articles(): + sentences = [ + "this is an apple", + "this is the apple", + "this is a fruit", + ] + + expected_article_removed_sents = ["this is apple", "this is apple", "this is fruit"] + + article_removed_sents = [QAMetrics.remove_articles(sent) for sent in sentences] + + assert article_removed_sents == expected_article_removed_sents + + +@pytest.mark.unit +def test_white_space_fix(): + sentences = [ + "sentence with a space", + "sentence with multiple spaces", + ] + + expected_white_space_fixed_sents = [ + "sentence with a space", + "sentence with multiple spaces", + ] + + white_space_fixed_sents = [QAMetrics.white_space_fix(sent) for sent in sentences] + + assert white_space_fixed_sents == expected_white_space_fixed_sents + + +@pytest.mark.unit +def test_remove_punc(): + sentence = "this, is. a! sentence: with; punctuations?" + expected_punc_removed_sent = "this is a sentence with punctuations" + + punc_removed_sent = QAMetrics.remove_punc(sentence) + + assert punc_removed_sent == expected_punc_removed_sent + + +@pytest.mark.unit +def test_get_normalized_tokens(): + sentence = 'I am happy' + tokens = ['i', 'am', 'happy'] + assert tokens == QAMetrics._get_normalized_tokens(sentence) + + sentence = 'I am a person' + tokens = ['i', 'am', 'person'] + assert tokens == QAMetrics._get_normalized_tokens(sentence) + + sentence = 'I am a person.' + tokens = ['i', 'am', 'person'] + assert tokens == QAMetrics._get_normalized_tokens(sentence) + + +@pytest.mark.unit +def test_get_one_f1(): + generated_field = 'That is so good' + ground_truth_field = 'That is so awesome' + + f1 = QAMetrics.get_one_f1(generated_field, ground_truth_field) + assert f1 == 0.75 + + generated_field = '' + ground_truth_field = 'That' + + f1 = QAMetrics.get_one_f1(generated_field, ground_truth_field) + assert f1 == 0 + + +@pytest.mark.unit +def test_get_one_exact_match(): + generated_field = 'That is so good' + ground_truth_field = 'That is so awesome' + + em = QAMetrics.get_one_exact_match(generated_field, ground_truth_field) + assert em == 0 + + generated_field = 'That is so good!' + ground_truth_field = 'That is so good.' + + em = QAMetrics.get_one_exact_match(generated_field, ground_truth_field) + assert em == 1 + + generated_field = 'That is so good' + ground_truth_field = 'that is so good' + + em = QAMetrics.get_one_exact_match(generated_field, ground_truth_field) + assert em == 1 + + +@pytest.mark.unit +def test_split_into_words(): + text = 'hi yo' + char_to_word_offset = [0, 0, 0, 1, 1] + doc_tokens = ["hi", "yo"] + output = QADataset.split_into_words(text) + assert output[0] == doc_tokens + assert output[1] == char_to_word_offset + + text = 'i am good' + char_to_word_offset = [0, 0, 1, 1, 1, 2, 2, 2, 2] + doc_tokens = ["i", "am", 'good'] + output = QADataset.split_into_words(text) + assert output[0] == doc_tokens + assert output[1] == char_to_word_offset + + +@pytest.mark.unit +def test_get_doc_spans(): + all_doc_tokens = ['a'] * 15 + max_tokens_for_doc = 10 + doc_stride = 5 + doc_spans = QADataset.get_docspans(all_doc_tokens, max_tokens_for_doc, doc_stride) + + assert len(doc_spans) == 2 + assert doc_spans[0].start == 0 + assert doc_spans[0].length == 10 + assert doc_spans[1].start == 5 + assert doc_spans[1].length == 10 + + +@pytest.mark.unit +def test_get_average_dist_to_tok_start_and_end(): + _DocSpan = collections.namedtuple("DocSpan", ["start", "length"]) + + doc_span = _DocSpan(start=0, length=5) + + tok_start_position = 1 + tok_end_position = 3 + + assert 2 == QADataset.get_average_dist_to_tok_start_and_end(doc_span, tok_start_position, tok_end_position) + + doc_span = _DocSpan(start=5, length=5) + + tok_start_position = 1 + tok_end_position = 2 + + assert 6 == QADataset.get_average_dist_to_tok_start_and_end(doc_span, tok_start_position, tok_end_position) + + doc_span = _DocSpan(start=5, length=4) + + tok_start_position = 1 + tok_end_position = 2 + + assert 5 == QADataset.get_average_dist_to_tok_start_and_end(doc_span, tok_start_position, tok_end_position) + + +@pytest.mark.unit +def test_keep_relevant_docspans(): + + _DocSpan = collections.namedtuple("DocSpan", ["start", "length"]) + + doc_spans = [_DocSpan(start=start, length=5) for start in range(15)] + + tok_start_position = 1 + tok_end_position = 2 + + mode = 'all' + assert doc_spans == QADataset.keep_relevant_docspans(doc_spans, tok_start_position, tok_end_position, mode) + + doc_spans = [_DocSpan(start=start, length=5) for start in range(15)] + + tok_start_position = -1 + tok_end_position = -1 + + mode = 'only_positive' + + expected_doc_spans = [] + assert expected_doc_spans == QADataset.keep_relevant_docspans( + doc_spans, tok_start_position, tok_end_position, mode + ) + + doc_spans = [_DocSpan(start=start, length=5) for start in range(15)] + + tok_start_position = 1 + tok_end_position = 2 + + mode = 'only_positive' + + expected_doc_spans = [_DocSpan(start=0, length=5), _DocSpan(start=1, length=5)] + assert expected_doc_spans == QADataset.keep_relevant_docspans( + doc_spans, tok_start_position, tok_end_position, mode + ) + + doc_spans = [_DocSpan(start=start, length=5) for start in range(15)] + + tok_start_position = 1 + tok_end_position = 2 + + mode = 'limited_negative' + + expected_doc_spans = [_DocSpan(start=start, length=5) for start in range(10)] + assert expected_doc_spans == QADataset.keep_relevant_docspans( + doc_spans, tok_start_position, tok_end_position, mode + ) + + +@pytest.mark.unit +def test_gpt_no_pad_loss_masking(): + input_ids = [1] * 15 + [50257] * 15 + input_ids = torch.tensor(input_ids) + + input_attn_mask = [1] * 16 + [0] * 14 + input_attn_mask = torch.Tensor(input_attn_mask) + + training_mask_end = 10 + + expected_labels = [-100] * 10 + [1] * 5 + [50257] + [-100] * 14 + expected_labels = torch.tensor(expected_labels) + + labels = GPTQADataset.update_labels_for_no_pad_loss(input_ids, training_mask_end, input_attn_mask) + + assert torch.all(labels.eq(expected_labels)) diff --git a/tutorials/nlp/Question_Answering.ipynb b/tutorials/nlp/Question_Answering.ipynb new file mode 100644 index 000000000000..8ab8edc64747 --- /dev/null +++ b/tutorials/nlp/Question_Answering.ipynb @@ -0,0 +1,1149 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": { + "id": "tiIOhb7iVC3J" + }, + "source": [ + "# Overview" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "PucJwfbhVC3L" + }, + "source": [ + "This tutorial will demonstrate how to train, evaluate, and test three types of models for Question-Answering -\n", + "1. BERT-like models for Extractive Question-Answering\n", + "2. Sequence-to-Sequence (S2S) models for Generative Question-Answering (ex. T5/BART-like)\n", + "3. GPT-like models for Generative Question-Answering\n", + "\n", + "## Task Description\n", + "\n", + "- Given a context and a natural language query, we want to generate an answer for the query\n", + "- Depending on how the answer is generated, the task can be broadly divided into two types:\n", + " 1. Extractive Question Answering\n", + " 2. Generative Question Answering\n", + "\n", + "\n", + "### Extractive Question-Answering with BERT-like models\n", + "\n", + "Given a question and a context, both in natural language, predict the span within the context with a start and end position which indicates the answer to the question.\n", + "For every word in our training dataset we’re going to predict:\n", + "- likelihood this word is the start of the span \n", + "- likelihood this word is the end of the span\n", + "\n", + "We are using a BERT encoder with 2 span prediction heads for predicting start and end position of the answer. The span predictions are token classifiers consisting of a single linear layer.\n", + "\n", + "### Generative Question-Answering with S2S and GPT-like models\n", + "\n", + "Given a question and a context, both in natural language, generate an answer for the question. Unlike the BERT-like models, there is no constraint that the answer should be a span within the context." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "IpX0w2PtVC3M" + }, + "source": [ + "# Installing NeMo" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "72XWYFQYVC3M" + }, + "source": [ + "You can run either this notebook locally (if you have all the dependencies and a GPU) or on Google Colab.\n", + "\n", + "Instructions for setting up Colab are as follows:\n", + "1. Open a new Python 3 notebook.\n", + "2. Import this notebook from GitHub (File -> Upload Notebook -> \"GITHUB\" tab -> copy/paste GitHub URL)\n", + "3. Connect to an instance with a GPU (Runtime -> Change runtime type -> select \"GPU\" for hardware accelerator)\n", + "4. Run the cell below to set up dependencies." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "_xQBtr0KVC3M" + }, + "outputs": [], + "source": [ + "BRANCH = \"main\"" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "9R1D6W58VC3N" + }, + "outputs": [], + "source": [ + "!python -m pip install git+https://github.com/NVIDIA/NeMo.git@$BRANCH#egg=nemo_toolkit[nlp]" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "fof5-57iVC3N" + }, + "source": [ + "# Imports and constants" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "KqKD-wReVC3O" + }, + "outputs": [], + "source": [ + "import os\n", + "import wget\n", + "\n", + "import pytorch_lightning as pl\n", + "from omegaconf import OmegaConf\n", + "\n", + "from nemo.collections.nlp.models.question_answering.qa_bert_model import BERTQAModel\n", + "from nemo.collections.nlp.models.question_answering.qa_gpt_model import GPTQAModel\n", + "from nemo.collections.nlp.models.question_answering.qa_s2s_model import S2SQAModel\n", + "from nemo.utils.exp_manager import exp_manager\n", + "\n", + "pl.seed_everything(42)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "xhPr9Jf_VC3O" + }, + "outputs": [], + "source": [ + "# set the following paths\n", + "DATA_DIR = \"\" # directory for storing datasets\n", + "WORK_DIR = \"\" # directory for storing trained models, logs, additionally downloaded scripts\n", + "\n", + "os.makedirs(DATA_DIR, exist_ok=True)\n", + "os.makedirs(WORK_DIR, exist_ok=True)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "dWymW8e0VC3O" + }, + "source": [ + "# Configuration" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "0YhKTkuXVC3P" + }, + "source": [ + "The model is defined in a config file which declares multiple important sections:\n", + "- **model**: All arguments that will relate to the Model - language model, span prediction, optimizer and schedulers, datasets and any other related information\n", + "- **trainer**: Any argument to be passed to PyTorch Lightning\n", + "- **exp_manager**: All arguments used for setting up the experiment manager - target directory, name, logger information\n", + "\n", + "We will download the default config file provided at `NeMo/examples/nlp/question_answering/conf/qa_conf.yaml` and edit necassary values for training different models" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "WOIWJqQ0VC3P" + }, + "outputs": [], + "source": [ + "# download the model's default configuration file \n", + "config_dir = WORK_DIR + '/conf/'\n", + "os.makedirs(config_dir, exist_ok=True)\n", + "if not os.path.exists(config_dir + \"qa_conf.yaml\"):\n", + " print('Downloading config file...')\n", + " wget.download(f'https://raw.githubusercontent.com/NVIDIA/NeMo/{BRANCH}/examples/nlp/question_answering/conf/qa_conf.yaml', config_dir)\n", + "else:\n", + " print ('config file already exists')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "cvD-gv-FVC3P" + }, + "outputs": [], + "source": [ + "# this will print the entire default config of the model\n", + "config_path = f'{WORK_DIR}/conf/qa_conf.yaml'\n", + "print(config_path)\n", + "config = OmegaConf.load(config_path)\n", + "print(\"Default Config - \\n\")\n", + "print(OmegaConf.to_yaml(config))" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "E08e-ItPVC3P" + }, + "source": [ + "# Training and testing models on SQuAD v2.0" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "xn022MsKVC3Q" + }, + "source": [ + "## Dataset" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "c356CGL1VC3Q" + }, + "source": [ + "For this example, we are going to download the [SQuAD](https://rajpurkar.github.io/SQuAD-explorer/) dataset to showcase how to do training and inference. There are two datasets, SQuAD1.0 and SQuAD2.0. SQuAD 1.1, the previous version of the SQuAD dataset, contains 100,000+ question-answer pairs on 500+ articles. SQuAD2.0 dataset combines the 100,000 questions in SQuAD1.1 with over 50,000 unanswerable questions written adversarially by crowdworkers to look similar to answerable ones. " + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "Gaju1h_bVC3Q" + }, + "source": [ + "To download both datasets, we use `NeMo/examples/nlp/question_answering/get_squad.py`" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "nb840_bZVC3Q" + }, + "outputs": [], + "source": [ + "# download get_squad.py script to download and preprocess the SQuAD data\n", + "os.makedirs(WORK_DIR, exist_ok=True)\n", + "if not os.path.exists(WORK_DIR + '/get_squad.py'):\n", + " print('Downloading get_squad.py...')\n", + " wget.download(f'https://raw.githubusercontent.com/NVIDIA/NeMo/{BRANCH}/examples/nlp/question_answering/get_squad.py', WORK_DIR)\n", + "else:\n", + " print ('get_squad.py already exists')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "sOgY0tRzVC3Q" + }, + "outputs": [], + "source": [ + "# download and preprocess the data\n", + "!python $WORK_DIR/get_squad.py --destDir $DATA_DIR" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "nprGkyvRVC3Q" + }, + "source": [ + "After execution of the above cell, your data folder will contain a subfolder \"squad\" the following four files for training and evaluation\n", + "\n", + "```\n", + "squad \n", + "│\n", + "└───v1.1\n", + "│ │ - train-v1.1.json\n", + "│ │ - dev-v1.1.json\n", + "│\n", + "└───v2.0\n", + " │ - train-v2.0.json\n", + " │ - dev-v2.0.json\n", + "```" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "GX0KWQXKVC3Q" + }, + "outputs": [], + "source": [ + "!ls -LR {DATA_DIR}/squad" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "RFVcvseOVC3R" + }, + "source": [ + "## Set dataset config values" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "Grb0EeRqVC3R" + }, + "outputs": [], + "source": [ + "# if True, model will load features from cache if file is present, or\n", + "# create features and dump to cache file if not already present\n", + "config.model.dataset.use_cache = False\n", + "\n", + "# indicates whether the dataset has unanswerable questions\n", + "config.model.dataset.version_2_with_negative = True\n", + "\n", + "# indicates whether the dataset is of extractive nature or not\n", + "# if True, context spans/chunks that do not contain answer are treated as unanswerable \n", + "config.model.dataset.check_if_answer_in_context = True\n", + "\n", + "# set file paths for train, validation, and test datasets\n", + "config.model.train_ds.file = f\"{DATA_DIR}/squad/v2.0/train-v2.0.json\"\n", + "config.model.validation_ds.file = f\"{DATA_DIR}/squad/v2.0/dev-v2.0.json\"\n", + "config.model.test_ds.file = f\"{DATA_DIR}/squad/v2.0/dev-v2.0.json\"\n", + "\n", + "# set batch sizes for train, validation, and test datasets\n", + "config.model.train_ds.batch_size = 8\n", + "config.model.validation_ds.batch_size = 8\n", + "config.model.test_ds.batch_size = 8\n", + "\n", + "# set number of samples to be used from dataset. setting to -1 uses entire dataset\n", + "config.model.train_ds.num_samples = 5000\n", + "config.model.validation_ds.num_samples = 1000\n", + "config.model.test_ds.num_samples = 100" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "rFWF41VwVC3R" + }, + "source": [ + "## Set trainer config values" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "42yif-GIVC3R" + }, + "outputs": [], + "source": [ + "config.trainer.max_epochs = 1\n", + "config.trainer.max_steps = -1 # takes precedence over max_epochs\n", + "config.trainer.precision = 16\n", + "config.trainer.devices = [0] # 0 for CPU, or list of the GPUs to use e.g. [0, 1] or [0]\n", + "config.trainer.accelerator = \"gpu\"" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "EDQzMBlbVC3R" + }, + "source": [ + "## Set experiment manager config values" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "pxY4rnJBVC3R" + }, + "outputs": [], + "source": [ + "config.exp_manager.exp_dir = WORK_DIR\n", + "config.exp_manager.name = \"QA-SQuAD2\"\n", + "config.exp_manager.create_wandb_logger=False" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "N2_C8reNVC3R" + }, + "source": [ + "## BERT model for SQuAD v2.0" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "4Mf-_rioVC3R" + }, + "source": [ + "### Set model config values" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "gtlGHzVJVC3R" + }, + "outputs": [], + "source": [ + "# set language model and tokenizer to be used\n", + "# tokenizer is derived from model if a tokenizer name is not provided\n", + "config.model.language_model.pretrained_model_name = \"bert-base-uncased\"\n", + "config.model.tokenizer.tokenizer_name = \"bert-base-uncased\"\n", + "\n", + "# path where model will be saved\n", + "config.model.nemo_path = f\"{WORK_DIR}/checkpoints/bert_squad_v2_0.nemo\"\n", + "\n", + "config.exp_manager.create_checkpoint_callback = True\n", + "\n", + "config.model.optim.lr = 3e-5" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "RaM7fe8rVC3R" + }, + "source": [ + "### Create trainer and initialize model" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "ukLzGmy9VC3R" + }, + "outputs": [], + "source": [ + "trainer = pl.Trainer(**config.trainer)\n", + "model = BERTQAModel(config.model, trainer=trainer)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "qZIA69rlVC3R" + }, + "source": [ + "### Train, test, and save the model" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "asutB9ZzVC3R" + }, + "outputs": [], + "source": [ + "trainer.fit(model)\n", + "trainer.test(model)\n", + "\n", + "model.save_to(config.model.nemo_path)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "n5AIv0SEVC3S" + }, + "source": [ + "### Load the saved model and run inference" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "7k5kD6tvVC3S" + }, + "outputs": [], + "source": [ + "model = BERTQAModel.restore_from(config.model.nemo_path)\n", + "\n", + "eval_device = [config.trainer.devices[0]] if isinstance(config.trainer.devices, list) else 1\n", + "model.trainer = pl.Trainer(\n", + " devices=eval_device,\n", + " accelerator=config.trainer.accelerator,\n", + " precision=16,\n", + " logger=False,\n", + ")\n", + "\n", + "config.exp_manager.create_checkpoint_callback = False\n", + "exp_dir = exp_manager(model.trainer, config.exp_manager)\n", + "output_nbest_file = os.path.join(exp_dir, \"output_nbest_file.json\")\n", + "output_prediction_file = os.path.join(exp_dir, \"output_prediction_file.json\")\n", + "\n", + "all_preds, all_nbest = model.inference(\n", + " config.model.test_ds.file,\n", + " output_prediction_file=output_prediction_file,\n", + " output_nbest_file=output_nbest_file,\n", + " num_samples=10, # setting to -1 will use all samples for inference\n", + ")\n", + "\n", + "for question_id in all_preds:\n", + " print(all_preds[question_id])" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "zyh0SNiyVC3S" + }, + "source": [ + "## S2S BART model for SQuAD v2.0" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "Sy9IYgVYVC3S" + }, + "source": [ + "### Set model config values" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "PKNmHKV5VC3S" + }, + "outputs": [], + "source": [ + "# set language model and tokenizer to be used\n", + "# tokenizer is derived from model if a tokenizer name is not provided\n", + "config.model.language_model.pretrained_model_name = \"facebook/bart-base\"\n", + "config.model.tokenizer.tokenizer_name = \"facebook/bart-base\"\n", + "\n", + "# path where model will be saved\n", + "config.model.nemo_path = f\"{WORK_DIR}/checkpoints/bart_squad_v2_0.nemo\"\n", + "\n", + "config.exp_manager.create_checkpoint_callback = True\n", + "\n", + "config.model.optim.lr = 5e-5" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "S_0glS4yVC3S" + }, + "source": [ + "### Create trainer and initialize model" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "8jWyHY1oVC3S" + }, + "outputs": [], + "source": [ + "# uncomment below line and run if you get an error while initializing tokenizer on Colab (reference: https://github.com/huggingface/transformers/issues/8690)\n", + "# !rm -r /root/.cache/huggingface/\n", + "\n", + "trainer = pl.Trainer(**config.trainer)\n", + "model = S2SQAModel(config.model, trainer=trainer)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "xg-j39b4VC3S" + }, + "source": [ + "### Train, test, and save the model" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "ocsf0EBDVC3S" + }, + "outputs": [], + "source": [ + "trainer.fit(model)\n", + "trainer.test(model)\n", + "\n", + "model.save_to(config.model.nemo_path)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "Vs3pl0VMVC3S" + }, + "source": [ + "### Load the saved model and run inference" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "NoW6_GO_VC3S" + }, + "outputs": [], + "source": [ + "model = S2SQAModel.restore_from(config.model.nemo_path)\n", + "\n", + "eval_device = [config.trainer.devices[0]] if isinstance(config.trainer.devices, list) else 1\n", + "model.trainer = pl.Trainer(\n", + " devices=eval_device,\n", + " accelerator=config.trainer.accelerator,\n", + " precision=16,\n", + " logger=False,\n", + ")\n", + "\n", + "config.exp_manager.create_checkpoint_callback = False\n", + "exp_dir = exp_manager(model.trainer, config.exp_manager)\n", + "output_nbest_file = os.path.join(exp_dir, \"output_nbest_file.json\")\n", + "output_prediction_file = os.path.join(exp_dir, \"output_prediction_file.json\")\n", + "\n", + "all_preds, all_nbest = model.inference(\n", + " config.model.test_ds.file,\n", + " output_prediction_file=output_prediction_file,\n", + " output_nbest_file=output_nbest_file,\n", + " num_samples=10, # setting to -1 will use all samples for inference\n", + ")\n", + "\n", + "for question_id in all_preds:\n", + " print(all_preds[question_id])" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "a7-iInbPVC3S" + }, + "source": [ + "## GPT2 model for SQuAD v2.0" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "VaIC0l2aVC3S" + }, + "source": [ + "### Set model config values" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "5j6SVk6fVC3S" + }, + "outputs": [], + "source": [ + "# set language model and tokenizer to be used\n", + "# tokenizer is derived from model if a tokenizer name is not provided\n", + "config.model.language_model.pretrained_model_name = \"gpt2\"\n", + "config.model.tokenizer.tokenizer_name = \"gpt2\"\n", + "\n", + "# path where model will be saved\n", + "config.model.nemo_path = f\"{WORK_DIR}/checkpoints/gpt2_squad_v2_0.nemo\"\n", + "\n", + "config.exp_manager.create_checkpoint_callback = True\n", + "\n", + "config.model.optim.lr = 1e-4" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "rWhhEuvzVC3S" + }, + "source": [ + "### Create trainer and initialize model" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "vBtP3ukDVC3S" + }, + "outputs": [], + "source": [ + "# uncomment below line and run if you get an error while initializing tokenizer on Colab (reference: https://github.com/huggingface/transformers/issues/8690)\n", + "# !rm -r /root/.cache/huggingface/\n", + "\n", + "trainer = pl.Trainer(**config.trainer)\n", + "model = GPTQAModel(config.model, trainer=trainer)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "EApFrJh8VC3T" + }, + "source": [ + "### Train, test, and save the model" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "zYo2JDdOVC3T" + }, + "outputs": [], + "source": [ + "trainer.fit(model)\n", + "trainer.test(model)\n", + "\n", + "model.save_to(config.model.nemo_path)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "6aNEt06fVC3T" + }, + "source": [ + "### Load the saved model and run inference" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "ioLT4DVbVC3T" + }, + "outputs": [], + "source": [ + "model = GPTQAModel.restore_from(config.model.nemo_path)\n", + "\n", + "eval_device = [config.trainer.devices[0]] if isinstance(config.trainer.devices, list) else 1\n", + "model.trainer = pl.Trainer(\n", + " devices=eval_device,\n", + " accelerator=config.trainer.accelerator,\n", + " precision=16,\n", + " logger=False,\n", + ")\n", + "\n", + "config.exp_manager.create_checkpoint_callback = False\n", + "exp_dir = exp_manager(model.trainer, config.exp_manager)\n", + "output_nbest_file = os.path.join(exp_dir, \"output_nbest_file.json\")\n", + "output_prediction_file = os.path.join(exp_dir, \"output_prediction_file.json\")\n", + "\n", + "all_preds, all_nbest = model.inference(\n", + " config.model.test_ds.file,\n", + " output_prediction_file=output_prediction_file,\n", + " output_nbest_file=output_nbest_file,\n", + " num_samples=10, # setting to -1 will use all samples for inference\n", + ")\n", + "\n", + "for question_id in all_preds:\n", + " print(all_preds[question_id])" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "hTWOlD9AVC3T" + }, + "source": [ + "# Training and testing models on MS-MARCO" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "lZWsMwnGVC3T" + }, + "source": [ + "## Dataset" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "pRUAwgAbVC3T" + }, + "source": [ + "### Downloading the data" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "qz3DO9JGVC3T" + }, + "source": [ + "MS-MARCO(Microsoft Machine Reading Comprehension) is a large scale dataset focused on machine reading comprehension, question answering, and passage ranking. MS-MARCO consists of 1,010,916 queries generated from real, anonymized Bing user queries. The contexts are extracted from real web documents and the answers are generated by humans.\n", + "\n", + "Please agree to the Terms of Use at https://microsoft.github.io/msmarco/ before downloading the data\n", + "\n", + "The data can be downloaded at:\n", + "- https://msmarco.blob.core.windows.net/msmarco/train_v2.1.json.gz\n", + "- https://msmarco.blob.core.windows.net/msmarco/dev_v2.1.json.gz" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "Fm5MzZ91inP5" + }, + "outputs": [], + "source": [ + "os.makedirs(os.path.join(DATA_DIR, \"msmarco\"), exist_ok=True)\n", + "\n", + "!wget https://msmarco.blob.core.windows.net/msmarco/train_v2.1.json.gz -P $DATA_DIR/msmarco\n", + "!gunzip $DATA_DIR/msmarco/train_v2.1.json.gz\n", + "\n", + "!wget https://msmarco.blob.core.windows.net/msmarco/dev_v2.1.json.gz -P $DATA_DIR/msmarco\n", + "!gunzip $DATA_DIR/msmarco/dev_v2.1.json.gz" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "nDmFHzBtVC3T" + }, + "source": [ + "### Converting to SQuAD format\n", + "\n", + "The script for converting MS-MARCO dataset to SQuAD can be found at `NeMo/examples/nlp/question_answering/convert_msmarco_to_squad_format.py`" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "tJtNIzZQVC3T" + }, + "outputs": [], + "source": [ + "# download convert_msmarco_to_squad_format.py script to format the MS-MARCO data\n", + "os.makedirs(WORK_DIR, exist_ok=True)\n", + "if not os.path.exists(WORK_DIR + '/convert_msmarco_to_squad_format.py'):\n", + " print('Downloading convert_msmarco_to_squad_format.py...')\n", + " wget.download(f'https://raw.githubusercontent.com/NVIDIA/NeMo/{BRANCH}/examples/nlp/question_answering/convert_msmarco_to_squad_format.py', WORK_DIR)\n", + "else:\n", + " print ('convert_msmarco_to_squad_format.py already exists')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "Io_esJPSuBcW" + }, + "outputs": [], + "source": [ + "# we will exclude examples from MS-MARCO dataset that do not have a wellFormedAnswer using a utility script\n", + "# download remove_ms_marco_samples_without_wellFormedAnswers.py script to format the MS-MARCO data\n", + "os.makedirs(WORK_DIR, exist_ok=True)\n", + "if not os.path.exists(WORK_DIR + '/remove_ms_marco_samples_without_wellFormedAnswers.py'):\n", + " print('Downloading remove_ms_marco_samples_without_wellFormedAnswers.py...')\n", + " wget.download(f'https://raw.githubusercontent.com/NVIDIA/NeMo/{BRANCH}/examples/nlp/dialogue/remove_ms_marco_samples_without_wellFormedAnswers.py', WORK_DIR)\n", + "else:\n", + " print ('remove_ms_marco_samples_without_wellFormedAnswers.py already exists')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "cs_CXkfXuYVQ" + }, + "outputs": [], + "source": [ + "!python $WORK_DIR/remove_ms_marco_samples_without_wellFormedAnswers.py --filename $DATA_DIR/msmarco/train_v2.1.json\n", + "!python $WORK_DIR/remove_ms_marco_samples_without_wellFormedAnswers.py --filename $DATA_DIR/msmarco/dev_v2.1.json" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "AUAKI086VC3T" + }, + "outputs": [], + "source": [ + "!(python $WORK_DIR/convert_msmarco_to_squad_format.py \\\n", + " --msmarco_train_input_filepath=$DATA_DIR/msmarco/train_v2.1.json \\\n", + " --msmarco_dev_input_filepath=$DATA_DIR/msmarco/dev_v2.1.json \\\n", + " --converted_train_save_path=$DATA_DIR/msmarco/msmarco-squad-format-train-v2.1.json \\\n", + " --converted_dev_save_path=$DATA_DIR/msmarco/msmarco-squad-format-dev-v2.1.json \\\n", + " --exclude_negative_samples=False \\\n", + " --keep_only_relevant_passages=False)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "AeHesaFcVC3T" + }, + "source": [ + "## Set dataset config values" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "rhx-_1X3VC3T" + }, + "outputs": [], + "source": [ + "# if True, model will load features from cache if file is present, or\n", + "# create features and dump to cache file if not already present\n", + "config.model.dataset.use_cache = False\n", + "\n", + "# indicates whether the dataset has unanswerable questions\n", + "config.model.dataset.version_2_with_negative = True\n", + "\n", + "# if True, context spans/chunks that do not contain answer are treated as unanswerable \n", + "# should be False for MS-MARCO dataset, or other datasets of generative nature\n", + "config.model.dataset.check_if_answer_in_context = False\n", + "\n", + "# set file paths for train, validation, and test datasets\n", + "config.model.train_ds.file = f\"{DATA_DIR}/msmarco/msmarco-squad-format-train-v2.1.json\"\n", + "config.model.validation_ds.file = f\"{DATA_DIR}/msmarco/msmarco-squad-format-dev-v2.1.json\"\n", + "config.model.test_ds.file = f\"{DATA_DIR}/msmarco/msmarco-squad-format-dev-v2.1.json\"\n", + "\n", + "# set batch sizes for train, validation, and test datasets\n", + "config.model.train_ds.batch_size = 16\n", + "config.model.validation_ds.batch_size = 16\n", + "config.model.test_ds.batch_size = 16\n", + "\n", + "# set number of samples to be used from dataset. setting to -1 uses entire dataset\n", + "config.model.train_ds.num_samples = 5000\n", + "config.model.validation_ds.num_samples = 1000\n", + "config.model.test_ds.num_samples = 100" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "X43k_EeqVC3T" + }, + "source": [ + "## Set trainer config values" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "HavpkQLPVC3U" + }, + "outputs": [], + "source": [ + "config.trainer.max_epochs = 1\n", + "config.trainer.max_steps = -1 # takes precedence over max_epochs\n", + "config.trainer.precision = 16\n", + "config.trainer.devices = [0] # 0 for CPU, or list of the GPUs to use e.g. [0, 1] or [0]\n", + "config.trainer.accelerator = \"gpu\"" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "R-_FIZE2VC3U" + }, + "source": [ + "## Set experiment manager config values" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "10TT3okiVC3U" + }, + "outputs": [], + "source": [ + "config.exp_manager.exp_dir = WORK_DIR\n", + "config.exp_manager.name = \"QA-MSMARCO\"\n", + "config.exp_manager.create_wandb_logger=False" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "MKIq6YT-VC3U" + }, + "source": [ + "## S2S BART model for MS-MARCO" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "tvf-QpYLVC3U" + }, + "source": [ + "### Set model config values" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "DDVZ1a5fVC3U" + }, + "outputs": [], + "source": [ + "# set language model and tokenizer to be used\n", + "# tokenizer is derived from model if a tokenizer name is not provided\n", + "config.model.language_model.pretrained_model_name = \"facebook/bart-base\"\n", + "config.model.tokenizer.tokenizer_name = \"facebook/bart-base\"\n", + "\n", + "# path where model will be saved\n", + "config.model.nemo_path = f\"{WORK_DIR}/checkpoints/bart_msmarco_v2_0.nemo\"\n", + "\n", + "config.exp_manager.create_checkpoint_callback = True\n", + "\n", + "config.model.optim.lr = 5e-5" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "3N75cdLRVC3U" + }, + "source": [ + "### Create trainer and initialize model" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "Bv9UMkfxVC3U" + }, + "outputs": [], + "source": [ + "trainer = pl.Trainer(**config.trainer)\n", + "model = S2SQAModel(config.model, trainer=trainer)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "BhVuV9sWVC3U" + }, + "source": [ + "### Train, test, and save the model" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "1JeaJ_OgVC3U" + }, + "outputs": [], + "source": [ + "trainer.fit(model)\n", + "trainer.test(model)\n", + "\n", + "model.save_to(config.model.nemo_path)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "yj0dGexaVC3U" + }, + "source": [ + "### Load the saved model and run inference" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "l1elN-WDVC3U" + }, + "outputs": [], + "source": [ + "model = S2SQAModel.restore_from(config.model.nemo_path)\n", + "\n", + "eval_device = [config.trainer.devices[0]] if isinstance(config.trainer.devices, list) else 1\n", + "model.trainer = pl.Trainer(\n", + " devices=eval_device,\n", + " accelerator=config.trainer.accelerator,\n", + " precision=16,\n", + " logger=False,\n", + ")\n", + "\n", + "config.exp_manager.create_checkpoint_callback = False\n", + "exp_dir = exp_manager(model.trainer, config.exp_manager)\n", + "output_nbest_file = os.path.join(exp_dir, \"output_nbest_file.json\")\n", + "output_prediction_file = os.path.join(exp_dir, \"output_prediction_file.json\")\n", + "\n", + "all_preds, all_nbest = model.inference(\n", + " config.model.test_ds.file,\n", + " output_prediction_file=output_prediction_file,\n", + " output_nbest_file=output_nbest_file,\n", + " num_samples=10, # setting to -1 will use all samples for inference\n", + ")\n", + "\n", + "for question_id in all_preds:\n", + " print(all_preds[question_id])" + ] + } + ], + "metadata": { + "accelerator": "GPU", + "colab": { + "collapsed_sections": [], + "name": "Question_Answering.ipynb", + "provenance": [] + }, + "gpuClass": "standard", + "interpreter": { + "hash": "bae55a3b24aa341f5a622f5db75f4176411c7613841f1c6ed3fbb75fb0be63d2" + }, + "kernelspec": { + "display_name": "Python 3.8.0 ('nemo': conda)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.8.0" + }, + "orig_nbformat": 4 + }, + "nbformat": 4, + "nbformat_minor": 0 +} diff --git a/tutorials/nlp/Question_Answering_Squad.ipynb b/tutorials/nlp/Question_Answering_Squad.ipynb deleted file mode 100755 index d78dc8de7602..000000000000 --- a/tutorials/nlp/Question_Answering_Squad.ipynb +++ /dev/null @@ -1,725 +0,0 @@ -{ - "nbformat": 4, - "nbformat_minor": 0, - "metadata": { - "accelerator": "GPU", - "colab": { - "name": "Question_Answering_Squad.ipynb", - "provenance": [], - "private_outputs": true, - "collapsed_sections": [ - "daYw_Xll2ZR9" - ], - "toc_visible": true - }, - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.7.6" - }, - "pycharm": { - "stem_cell": { - "cell_type": "raw", - "metadata": { - "collapsed": false - }, - "source": [] - } - } - }, - "cells": [ - { - "cell_type": "code", - "metadata": { - "id": "uRLPr0TnIAHO" - }, - "source": [ - "BRANCH = 'main'" - ], - "execution_count": null, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "id": "o_0K1lsW1dj9" - }, - "source": [ - "\"\"\"\n", - "You can run either this notebook locally (if you have all the dependencies and a GPU) or on Google Colab.\n", - "\n", - "Instructions for setting up Colab are as follows:\n", - "1. Open a new Python 3 notebook.\n", - "2. Import this notebook from GitHub (File -> Upload Notebook -> \"GITHUB\" tab -> copy/paste GitHub URL)\n", - "3. Connect to an instance with a GPU (Runtime -> Change runtime type -> select \"GPU\" for hardware accelerator)\n", - "4. Run this cell to set up dependencies.\n", - "\"\"\"\n", - "# If you're using Google Colab and not running locally, run this cell\n", - "\n", - "# install NeMo\n", - "!python -m pip install git+https://github.com/NVIDIA/NeMo.git@$BRANCH#egg=nemo_toolkit[nlp]" - ], - "execution_count": null, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "id": "dzqD2WDFOIN-" - }, - "source": [ - "from nemo.utils.exp_manager import exp_manager\n", - "from nemo.collections import nlp as nemo_nlp\n", - "\n", - "import os\n", - "import wget \n", - "import torch\n", - "import pytorch_lightning as pl\n", - "from omegaconf import OmegaConf" - ], - "execution_count": null, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "daYw_Xll2ZR9" - }, - "source": [ - "# Task Description\n", - "Given a question and a context both in natural language, predict the span within the context with a start and end position which indicates the answer to the question.\n", - "For every word in our training dataset we’re going to predict:\n", - "- likelihood this word is the start of the span \n", - "- likelihood this word is the end of the span \n", - "\n", - "We are using a pretrained [BERT](https://arxiv.org/pdf/1810.04805.pdf) encoder with 2 span prediction heads for prediction start and end position of the answer. The span predictions are token classifiers consisting of a single linear layer. " - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "ZnuziSwJ1yEB" - }, - "source": [ - "# Dataset\n", - "This model expects the dataset to be in [SQuAD](https://rajpurkar.github.io/SQuAD-explorer/) format, e.g. a JSON file for each dataset split. \n", - "In the following we will show example for a training file. Each title has one or multiple paragraph entries, each consisting of the text - \"context\", and question-answer entries. Each question-answer entry has:\n", - "* a question\n", - "* a globally unique id\n", - "* a boolean flag \"is_impossible\" which shows if the question is answerable or not\n", - "* in case the question is answerable one answer entry, which contains the text span and its starting character index in the context. If not answerable, the \"answers\" list is empty\n", - "\n", - "The evaluation files (for validation and testing) follow the above format except for it can provide more than one answer to the same question. \n", - "The inference file follows the above format except for it does not require the \"answers\" and \"is_impossible\" keywords.\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "TXFORGBv2Jqu" - }, - "source": [ - "\n", - "\n", - "```\n", - "{\n", - " \"data\": [\n", - " {\n", - " \"title\": \"Super_Bowl_50\", \n", - " \"paragraphs\": [\n", - " {\n", - " \"context\": \"Super Bowl 50 was an American football game to determine the champion of the National Football League (NFL) for the 2015 season. The American Football Conference (AFC) champion Denver Broncos defeated the National Football Conference (NFC) champion Carolina Panthers 24\\u201310 to earn their third Super Bowl title. The game was played on February 7, 2016, at Levi's Stadium in the San Francisco Bay Area at Santa Clara, California. As this was the 50th Super Bowl, the league emphasized the \\\"golden anniversary\\\" with various gold-themed initiatives, as well as temporarily suspending the tradition of naming each Super Bowl game with Roman numerals (under which the game would have been known as \\\"Super Bowl L\\\"), so that the logo could prominently feature the Arabic numerals 50.\", \n", - " \"qas\": [\n", - " {\n", - " \"question\": \"Where did Super Bowl 50 take place?\", \n", - " \"is_impossible\": \"false\", \n", - " \"id\": \"56be4db0acb8001400a502ee\", \n", - " \"answers\": [\n", - " {\n", - " \"answer_start\": \"403\", \n", - " \"text\": \"Santa Clara, California\"\n", - " }\n", - " ]\n", - " },\n", - " {\n", - " \"question\": \"What was the winning score of the Super Bowl 50?\", \n", - " \"is_impossible\": \"true\", \n", - " \"id\": \"56be4db0acb8001400a502ez\", \n", - " \"answers\": [\n", - " ]\n", - " }\n", - " ]\n", - " }\n", - " ]\n", - " }\n", - " ]\n", - "}\n", - "...\n", - "```\n", - "\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "SL58EWkd2ZVb" - }, - "source": [ - "## Download the data" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "THi6s1Qx2G1k" - }, - "source": [ - "In this notebook we are going download the [SQuAD](https://rajpurkar.github.io/SQuAD-explorer/) dataset to showcase how to do training and inference. There are two datasets, SQuAD1.0 and SQuAD2.0. SQuAD 1.1, the previous version of the SQuAD dataset, contains 100,000+ question-answer pairs on 500+ articles. SQuAD2.0 dataset combines the 100,000 questions in SQuAD1.1 with over 50,000 unanswerable questions written adversarially by crowdworkers to look similar to answerable ones. \n", - "\n", - "\n", - "To download both datasets, we use [NeMo/examples/nlp/question_answering/get_squad.py](https://github.com/NVIDIA/NeMo/blob/stable/examples/nlp/question_answering/get_squad.py). \n", - "\n", - "\n" - ] - }, - { - "cell_type": "code", - "metadata": { - "id": "tv3qXTTR_hBk" - }, - "source": [ - "# set the following paths\n", - "DATA_DIR = \"PATH_TO_DATA\"\n", - "WORK_DIR = \"PATH_TO_CHECKPOINTS_AND_LOGS\"" - ], - "execution_count": null, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "id": "qcz3Djem_hBn" - }, - "source": [ - "## download get_squad.py script to download and preprocess the SQuAD data\n", - "os.makedirs(WORK_DIR, exist_ok=True)\n", - "if not os.path.exists(WORK_DIR + '/get_squad.py'):\n", - " print('Downloading get_squad.py...')\n", - " wget.download(f'https://raw.githubusercontent.com/NVIDIA/NeMo/{BRANCH}/examples/nlp/question_answering/get_squad.py', WORK_DIR)\n", - "else:\n", - " print ('get_squad.py already exists')" - ], - "execution_count": null, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "id": "mpzsC41t_hBq" - }, - "source": [ - "# download and preprocess the data\n", - "! python $WORK_DIR/get_squad.py --destDir $DATA_DIR" - ], - "execution_count": null, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "m_HLLl6t_hBs" - }, - "source": [ - "after execution of the above cell, your data folder will contain a subfolder \"squad\" the following 4 files for training and evaluation\n", - "- v1.1/train-v1.1.json\n", - "- v1.1/dev-v1.1.json\n", - "- v2.0/train-v2.0.json\n", - "- v2.0/dev-v2.0.json" - ] - }, - { - "cell_type": "code", - "metadata": { - "id": "qYHcfxPL_hBt" - }, - "source": [ - "! ls -LR {DATA_DIR}/squad" - ], - "execution_count": null, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "bdpikZVreLlI" - }, - "source": [ - "## Data preprocessing\n", - "\n", - "The input into the model is the concatenation of two tokenized sequences:\n", - "\" [CLS] query [SEP] context [SEP]\".\n", - "This is the tokenization used for BERT, i.e. [WordPiece](https://arxiv.org/pdf/1609.08144.pdf) Tokenizer, which uses the [Google's BERT vocabulary](https://github.com/google-research/bert). This tokenizer is configured with `model.tokenizer.tokenizer_name=bert-base-uncased` and is automatically instantiated using [Huggingface](https://huggingface.co/)'s API. \n", - "The benefit of this tokenizer is that this is compatible with a pretrained BERT model, from which we can finetune instead of training the question answering model from scratch. However, we also support other tokenizers, such as `model.tokenizer.tokenizer_name=sentencepiece`. Unlike the BERT WordPiece tokenizer, the [SentencePiece](https://github.com/google/sentencepiece) tokenizer model needs to be first created from a text file.\n", - "See [02_NLP_Tokenizers.ipynb](https://colab.research.google.com/github/NVIDIA/NeMo/blob/stable/tutorials/nlp/02_NLP_Tokenizers.ipynb) for more details on how to use NeMo Tokenizers." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "0q7Y7nyW_hBv" - }, - "source": [ - "# Data and Model Parameters\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "B0b0Tn8M_hBv" - }, - "source": [ - "Note, this is only an example to showcase usage and is not optimized for accuracy. In the following, we will download and adjust the model configuration to create a toy example, where we only use a small fraction of the original dataset. \n", - "\n", - "In order to train the full SQuAD model, leave the model parameters from the configuration file unchanged. This sets NUM_SAMPLES=-1 to use the entire dataset, which will slow down performance significantly. We recommend to use bash script and multi-GPU to accelerate this. \n" - ] - }, - { - "cell_type": "code", - "metadata": { - "id": "n8HZrDmr12_-" - }, - "source": [ - "# This is the model configuration file that we will download, do not change this\n", - "MODEL_CONFIG = \"question_answering_squad_config.yaml\"\n", - "\n", - "# model parameters, play with these\n", - "BATCH_SIZE = 12\n", - "MAX_SEQ_LENGTH = 384\n", - "# specify BERT-like model, you want to use\n", - "PRETRAINED_BERT_MODEL = \"bert-base-uncased\"\n", - "TOKENIZER_NAME = \"bert-base-uncased\" # tokenizer name\n", - "\n", - "# Number of data examples used for training, validation, test and inference\n", - "TRAIN_NUM_SAMPLES = VAL_NUM_SAMPLES = TEST_NUM_SAMPLES = 5000 \n", - "INFER_NUM_SAMPLES = 5\n", - "\n", - "TRAIN_FILE = f\"{DATA_DIR}/squad/v1.1/train-v1.1.json\"\n", - "VAL_FILE = f\"{DATA_DIR}/squad/v1.1/dev-v1.1.json\"\n", - "TEST_FILE = f\"{DATA_DIR}/squad/v1.1/dev-v1.1.json\"\n", - "INFER_FILE = f\"{DATA_DIR}/squad/v1.1/dev-v1.1.json\"\n", - "\n", - "INFER_PREDICTION_OUTPUT_FILE = \"output_prediction.json\"\n", - "INFER_NBEST_OUTPUT_FILE = \"output_nbest.json\"\n", - "\n", - "# training parameters\n", - "LEARNING_RATE = 0.00003\n", - "\n", - "# number of epochs\n", - "MAX_EPOCHS = 1" - ], - "execution_count": null, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "daludzzL2Jba" - }, - "source": [ - "# Model Configuration" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "_whKCxfTMo6Y" - }, - "source": [ - "The model is defined in a config file which declares multiple important sections. They are:\n", - "- **model**: All arguments that will relate to the Model - language model, span prediction, optimizer and schedulers, datasets and any other related information\n", - "\n", - "- **trainer**: Any argument to be passed to PyTorch Lightning" - ] - }, - { - "cell_type": "code", - "metadata": { - "id": "T1gA8PsJ13MJ" - }, - "source": [ - "# download the model's default configuration file \n", - "config_dir = WORK_DIR + '/configs/'\n", - "os.makedirs(config_dir, exist_ok=True)\n", - "if not os.path.exists(config_dir + MODEL_CONFIG):\n", - " print('Downloading config file...')\n", - " wget.download(f'https://raw.githubusercontent.com/NVIDIA/NeMo/{BRANCH}/examples/nlp/question_answering/conf/{MODEL_CONFIG}', config_dir)\n", - "else:\n", - " print ('config file is already exists')" - ], - "execution_count": null, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "id": "mX3KmWMvSUQw" - }, - "source": [ - "# this line will print the entire default config of the model\n", - "config_path = f'{WORK_DIR}/configs/{MODEL_CONFIG}'\n", - "print(config_path)\n", - "config = OmegaConf.load(config_path)\n", - "print(OmegaConf.to_yaml(config))" - ], - "execution_count": null, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "ZCgWzNBkaQLZ" - }, - "source": [ - "## Setting up data within the config\n", - "\n", - "Among other things, the config file contains dictionaries called dataset, train_ds and validation_ds, test_ds. These are configurations used to setup the Dataset and DataLoaders of the corresponding config.\n", - "\n", - "Specify data paths using `model.train_ds.file`, `model.valuation_ds.file` and `model.test_ds.file`.\n", - "\n", - "Let's now add the data paths to the config." - ] - }, - { - "cell_type": "code", - "metadata": { - "id": "LQHCJN-ZaoLp" - }, - "source": [ - "config.model.train_ds.file = TRAIN_FILE\n", - "config.model.validation_ds.file = VAL_FILE\n", - "config.model.test_ds.file = TEST_FILE\n", - "\n", - "config.model.train_ds.num_samples = TRAIN_NUM_SAMPLES\n", - "config.model.validation_ds.num_samples = VAL_NUM_SAMPLES\n", - "config.model.test_ds.num_samples = TEST_NUM_SAMPLES\n", - "\n", - "config.model.tokenizer.tokenizer_name = TOKENIZER_NAME" - ], - "execution_count": null, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "nB96-3sTc3yk" - }, - "source": [ - "# Building the PyTorch Lightning Trainer\n", - "\n", - "NeMo models are primarily PyTorch Lightning modules - and therefore are entirely compatible with the PyTorch Lightning ecosystem!\n", - "\n", - "Let's first instantiate a Trainer object!" - ] - }, - { - "cell_type": "code", - "metadata": { - "id": "knF6QeQQdMrH" - }, - "source": [ - "# lets modify some trainer configs\n", - "# checks if we have GPU available and uses it\n", - "accelerator = 'gpu' if torch.cuda.is_available() else 'cpu'\n", - "config.trainer.devices = 1\n", - "config.trainer.accelerator = accelerator\n", - "config.trainer.precision = 16 if torch.cuda.is_available() else 32\n", - "\n", - "# For mixed precision training, use precision=16 and amp_level=O1\n", - "\n", - "config.trainer.max_epochs = MAX_EPOCHS\n", - "\n", - "# Remove distributed training flags if only running on a single GPU or CPU\n", - "config.trainer.strategy = None\n", - "\n", - "print(\"Trainer config - \\n\")\n", - "print(OmegaConf.to_yaml(config.trainer))\n", - "\n", - "trainer = pl.Trainer(**config.trainer)" - ], - "execution_count": null, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "8IlEMdVxdr6p" - }, - "source": [ - "# Setting up a NeMo Experiment¶\n", - "\n", - "NeMo has an experiment manager that handles logging and checkpointing for us, so let's use it!" - ] - }, - { - "cell_type": "code", - "metadata": { - "id": "8uztqGAmdrYt" - }, - "source": [ - "config.exp_manager.exp_dir = WORK_DIR\n", - "exp_dir = exp_manager(trainer, config.get(\"exp_manager\", None))\n", - "\n", - "# the exp_dir provides a path to the current experiment for easy access\n", - "exp_dir = str(exp_dir)" - ], - "execution_count": null, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "D4jy28fbjekD" - }, - "source": [ - "# Using an Out-Of-Box Model" - ] - }, - { - "cell_type": "code", - "metadata": { - "id": "Ins2ZzJckKKo" - }, - "source": [ - "# list available pretrained models\n", - "nemo_nlp.models.QAModel.list_available_models()" - ], - "execution_count": null, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "id": "iFnzHvkVk-S5" - }, - "source": [ - "# load pretained model\n", - "pretrained_model_name=\"qa_squadv1.1_bertbase\"\n", - "model = nemo_nlp.models.QAModel.from_pretrained(model_name=pretrained_model_name)" - ], - "execution_count": null, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "6FI_nQsJo_11" - }, - "source": [ - "# Model Training" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "8tjLhUvL_o7_" - }, - "source": [ - "Before initializing the model, we might want to modify some of the model configs." - ] - }, - { - "cell_type": "code", - "metadata": { - "id": "Xeuc2i7Y_nP5" - }, - "source": [ - "# complete list of supported BERT-like models\n", - "nemo_nlp.modules.get_pretrained_lm_models_list()" - ], - "execution_count": null, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "id": "RK2xglXyAUOO" - }, - "source": [ - "# add the specified above model parameters to the config\n", - "config.model.language_model.pretrained_model_name = PRETRAINED_BERT_MODEL\n", - "config.model.train_ds.batch_size = BATCH_SIZE\n", - "config.model.validation_ds.batch_size = BATCH_SIZE\n", - "config.model.test_ds.batch_size = BATCH_SIZE\n", - "config.model.optim.lr = LEARNING_RATE\n", - "\n", - "print(\"Updated model config - \\n\")\n", - "print(OmegaConf.to_yaml(config.model))" - ], - "execution_count": null, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "id": "NgsGLydWo-6-" - }, - "source": [ - "# initialize the model\n", - "# dataset we'll be prepared for training and evaluation during\n", - "model = nemo_nlp.models.QAModel(cfg=config.model, trainer=trainer)" - ], - "execution_count": null, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "kQ592Tx4pzyB" - }, - "source": [ - "## Monitoring Training Progress\n", - "Optionally, you can create a Tensorboard visualization to monitor training progress." - ] - }, - { - "cell_type": "code", - "metadata": { - "id": "mTJr16_pp0aS" - }, - "source": [ - "try:\n", - " from google import colab\n", - " COLAB_ENV = True\n", - "except (ImportError, ModuleNotFoundError):\n", - " COLAB_ENV = False\n", - "\n", - "# Load the TensorBoard notebook extension\n", - "if COLAB_ENV:\n", - " %load_ext tensorboard\n", - " %tensorboard --logdir {exp_dir}\n", - "else:\n", - " print(\"To use tensorboard, please use this notebook in a Google Colab environment.\")" - ], - "execution_count": null, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "id": "hUvnSpyjp0Dh" - }, - "source": [ - "# start the training\n", - "trainer.fit(model)" - ], - "execution_count": null, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "JxBiIKMlH8yv" - }, - "source": [ - "After training for 1 epoch, exact match on the evaluation data should be around 59.2%, F1 around 70.2%." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "ynCLBmAWFVsM" - }, - "source": [ - "# Evaluation\n", - "\n", - "To see how the model performs, let’s run evaluation on the test dataset." - ] - }, - { - "cell_type": "code", - "metadata": { - "id": "XBMCoXAKFtSd" - }, - "source": [ - "model.setup_test_data(test_data_config=config.model.test_ds)\n", - "trainer.test(model)" - ], - "execution_count": null, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "VPdzJVAgSFaJ" - }, - "source": [ - "# Inference\n", - "\n", - "To use the model for creating predictions, let’s run inference on the unlabeled inference dataset." - ] - }, - { - "cell_type": "code", - "metadata": { - "id": "DQhsamclRtxJ" - }, - "source": [ - "# # store test prediction under the experiment output folder\n", - "output_prediction_file = f\"{exp_dir}/{INFER_PREDICTION_OUTPUT_FILE}\"\n", - "output_nbest_file = f\"{exp_dir}/{INFER_NBEST_OUTPUT_FILE}\"\n", - "all_preds, all_nbests = model.inference(file=INFER_FILE, batch_size=5, num_samples=INFER_NUM_SAMPLES, output_nbest_file=output_nbest_file, output_prediction_file=output_prediction_file)" - ], - "execution_count": null, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "id": "sQpRIOaM_hCQ" - }, - "source": [ - "for _, item in all_preds.items():\n", - " print(f\"question: {item[0]} answer: {item[1]}\")\n", - "#The prediction file contains the predicted answer to each question id for the first TEST_NUM_SAMPLES.\n", - "! python -m json.tool $exp_dir/$INFER_PREDICTION_OUTPUT_FILE" - ], - "execution_count": null, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "ref1qSonGNhP" - }, - "source": [ - "If you have NeMo installed locally, you can also train the model with \n", - "[NeMo/examples/nlp/question_answering/get_squad.py](https://github.com/NVIDIA/NeMo/blob/stable/examples/nlp/question_answering/question_answering_squad.py).\n", - "\n", - "To run training script, use:\n", - "\n", - "`python question_answering_squad.py model.train_ds.file=TRAIN_FILE model.validation_ds.file=VAL_FILE model.test_ds.file=TEST_FILE`\n", - "\n", - "To improve the performance of the model, train with multi-GPU and a global batch size of 24. So if you use 8 GPUs with `trainer.devices=8`, set `model.train_ds.batch_size=3`" - ] - } - ] -} \ No newline at end of file From 468a3f3779f41f9c67bd8012e47a585a424dadf1 Mon Sep 17 00:00:00 2001 From: bene-ges <61418381+bene-ges@users.noreply.github.com> Date: Mon, 25 Jul 2022 23:26:52 +0300 Subject: [PATCH 40/52] add kw asr models, add itn ru checkpoint (tagger-based) (#4595) * add kw asr models, add itn ru checkpoint (tagger-based) Signed-off-by: Alexandra Antonova * add rw results to docs Signed-off-by: Alexandra Antonova Co-authored-by: Alexandra Antonova --- docs/source/asr/data/benchmark_rw.csv | 3 +++ docs/source/asr/data/scores/rw/conformer_rw.csv | 3 +++ docs/source/asr/results.rst | 10 ++++++++++ nemo/collections/asr/models/ctc_bpe_models.py | 7 +++++++ nemo/collections/asr/models/rnnt_bpe_models.py | 7 +++++++ .../text_normalization_as_tagging/thutmose_tagger.py | 10 ++++++++-- 6 files changed, 38 insertions(+), 2 deletions(-) create mode 100644 docs/source/asr/data/benchmark_rw.csv create mode 100644 docs/source/asr/data/scores/rw/conformer_rw.csv diff --git a/docs/source/asr/data/benchmark_rw.csv b/docs/source/asr/data/benchmark_rw.csv new file mode 100644 index 000000000000..0264fc8a70cd --- /dev/null +++ b/docs/source/asr/data/benchmark_rw.csv @@ -0,0 +1,3 @@ +Model,Model Base Class,Model Card +stt_rw_conformer_ctc_large,EncDecCTCModel,"https://ngc.nvidia.com/catalog/models/nvidia:nemo:stt_rw_conformer_ctc_large" +stt_rw_conformer_transducer_large,EncDecRNNTBPEModel,"https://ngc.nvidia.com/catalog/models/nvidia:nemo:stt_rw_conformer_transducer_large" \ No newline at end of file diff --git a/docs/source/asr/data/scores/rw/conformer_rw.csv b/docs/source/asr/data/scores/rw/conformer_rw.csv new file mode 100644 index 000000000000..52196a54335f --- /dev/null +++ b/docs/source/asr/data/scores/rw/conformer_rw.csv @@ -0,0 +1,3 @@ +Model Name,Language,MCV Test-Set v9.0 (rw) +stt_rw_conformer_ctc_large,rw,18.22 +stt_rw_conformer_transducer_large,rw,16.19 \ No newline at end of file diff --git a/docs/source/asr/results.rst b/docs/source/asr/results.rst index 87e2f9e8200a..290e44c1b5d5 100644 --- a/docs/source/asr/results.rst +++ b/docs/source/asr/results.rst @@ -209,3 +209,13 @@ Marathi :widths: 40, 10, 50 :header-rows: 1 +----------------------------- + +Kinyarwanda +^^^^^^^ +.. csv-table:: + :file: data/benchmark_rw.csv + :align: left + :widths: 40, 10, 50 + :header-rows: 1 + diff --git a/nemo/collections/asr/models/ctc_bpe_models.py b/nemo/collections/asr/models/ctc_bpe_models.py index ce92823cba6f..9c01fdbc7e3c 100644 --- a/nemo/collections/asr/models/ctc_bpe_models.py +++ b/nemo/collections/asr/models/ctc_bpe_models.py @@ -527,4 +527,11 @@ def list_available_models(cls) -> Optional[PretrainedModelInfo]: ) results.append(model) + model = PretrainedModelInfo( + pretrained_model_name="stt_rw_conformer_ctc_large", + description="For details about this model, please visit https://ngc.nvidia.com/catalog/models/nvidia:nemo:stt_rw_conformer_ctc_large", + location="https://api.ngc.nvidia.com/v2/models/nvidia/nemo/stt_rw_conformer_ctc_large/versions/1.11.0/files/stt_rw_conformer_ctc_large.nemo", + ) + results.append(model) + return results diff --git a/nemo/collections/asr/models/rnnt_bpe_models.py b/nemo/collections/asr/models/rnnt_bpe_models.py index 199fc0304f0c..4bd4d7c8f9d9 100644 --- a/nemo/collections/asr/models/rnnt_bpe_models.py +++ b/nemo/collections/asr/models/rnnt_bpe_models.py @@ -190,6 +190,13 @@ def list_available_models(cls) -> List[PretrainedModelInfo]: ) results.append(model) + model = PretrainedModelInfo( + pretrained_model_name="stt_rw_conformer_transducer_large", + description="For details about this model, please visit https://ngc.nvidia.com/catalog/models/nvidia:nemo:stt_rw_conformer_transducer_large", + location="https://api.ngc.nvidia.com/v2/models/nvidia/nemo/stt_rw_conformer_transducer_large/versions/1.11.0/files/stt_rw_conformer_transducer_large.nemo", + ) + results.append(model) + return results def __init__(self, cfg: DictConfig, trainer: Trainer = None): diff --git a/nemo/collections/nlp/models/text_normalization_as_tagging/thutmose_tagger.py b/nemo/collections/nlp/models/text_normalization_as_tagging/thutmose_tagger.py index 5fe13e07af4b..0a0c93abac4a 100644 --- a/nemo/collections/nlp/models/text_normalization_as_tagging/thutmose_tagger.py +++ b/nemo/collections/nlp/models/text_normalization_as_tagging/thutmose_tagger.py @@ -411,8 +411,14 @@ def list_available_models(cls) -> Optional[PretrainedModelInfo]: PretrainedModelInfo( pretrained_model_name="itn_en_thutmose_bert", location="https://api.ngc.nvidia.com/v2/models/nvidia/nemo/itn_en_thutmose_bert/versions/1.9.0/files/itn_en_thutmose_bert.nemo", - description="A single-pass tagger-based model for inverse text normalization based" - "on bert-base-uncased, trained on 2 mln sentences from Google Text Normalization Dataset", + description="A single-pass tagger-based English model for inverse text normalization based" + "on BERT, trained on 2 mln sentences from Google Text Normalization Dataset", + ), + PretrainedModelInfo( + pretrained_model_name="itn_ru_thutmose_bert", + location="https://api.ngc.nvidia.com/v2/models/nvidia/nemo/itn_ru_thutmose_bert/versions/1.11.0/files/itn_ru_thutmose_bert.nemo", + description="A single-pass tagger-based Russian model for inverse text normalization based" + "on BERT, trained on 2 mln sentences from Google Text Normalization Dataset", ), ] return result From c324499a46587ebabeddc3c21d4da514820bf4c8 Mon Sep 17 00:00:00 2001 From: Anas Abou Allaban Date: Mon, 25 Jul 2022 19:17:52 -0400 Subject: [PATCH 41/52] Add DALI pipeline to SSL model (#4592) Signed-off-by: Anas Abou Allaban Co-authored-by: Samuel Kriman --- nemo/collections/asr/models/ssl_models.py | 35 +++++++++++++++++++---- 1 file changed, 29 insertions(+), 6 deletions(-) diff --git a/nemo/collections/asr/models/ssl_models.py b/nemo/collections/asr/models/ssl_models.py index 7dbc06a5d56d..4c101a28a613 100644 --- a/nemo/collections/asr/models/ssl_models.py +++ b/nemo/collections/asr/models/ssl_models.py @@ -21,6 +21,7 @@ from pytorch_lightning import Trainer from nemo.collections.asr.data import audio_to_text_dataset +from nemo.collections.asr.data.audio_to_text_dali import DALIOutputs from nemo.collections.asr.parts.mixins import ASRModuleMixin from nemo.collections.asr.parts.preprocessing.perturb import process_augmentations from nemo.core.classes import ModelPT @@ -143,6 +144,18 @@ def _setup_dataloader_from_config(self, config: Optional[Dict]): audio_to_text_dataset.inject_dataloader_value_from_model_config(self.cfg, config, key='sample_rate') shuffle = config['shuffle'] + device = 'gpu' if torch.cuda.is_available() else 'cpu' + if config.get('use_dali', False): + device_id = self.local_rank if device == 'gpu' else None + dataset = audio_to_text_dataset.get_dali_char_dataset( + config=config, + shuffle=shuffle, + device_id=device_id, + global_rank=self.global_rank, + world_size=self.world_size, + preprocessor_cfg=self._cfg.preprocessor, + ) + return dataset # Instantiate tarred dataset loader or normal dataset loader if config.get('is_tarred', False): @@ -479,9 +492,14 @@ def decoder_loss_step(self, spectrograms, spec_masks, encoded, encoded_len, targ # PTL-specific methods def training_step(self, batch, batch_nb): signal, signal_len, targets, target_lengths = batch - spectrograms, spec_masks, encoded, encoded_len = self.forward( - input_signal=signal, input_signal_length=signal_len, - ) + if isinstance(batch, DALIOutputs) and batch.has_processed_signal: + spectrograms, spec_masks, encoded, encoded_len = self.forward( + processed_signal=signal, processed_signal_length=signal_len, + ) + else: + spectrograms, spec_masks, encoded, encoded_len = self.forward( + input_signal=signal, input_signal_length=signal_len, + ) loss_value, loss_val_dict = self.decoder_loss_step( spectrograms, spec_masks, encoded, encoded_len, targets, target_lengths @@ -508,9 +526,14 @@ def validation_step(self, batch, batch_idx, dataloader_idx=0): self._in_validation_step = True signal, signal_len, targets, target_lengths = batch - spectrograms, spec_masks, encoded, encoded_len = self.forward( - input_signal=signal, input_signal_length=signal_len, - ) + if isinstance(batch, DALIOutputs) and batch.has_processed_signal: + spectrograms, spec_masks, encoded, encoded_len = self.forward( + processed_signal=signal, processed_signal_length=signal_len, + ) + else: + spectrograms, spec_masks, encoded, encoded_len = self.forward( + input_signal=signal, input_signal_length=signal_len, + ) loss_value, _ = self.decoder_loss_step(spectrograms, spec_masks, encoded, encoded_len, targets, target_lengths) From faf8ad83cbad28ab8aa509caa404e720a1e42378 Mon Sep 17 00:00:00 2001 From: Ameya Mahabaleshwarkar <34514696+ameyasm1154@users.noreply.github.com> Date: Mon, 25 Jul 2022 21:48:54 -0400 Subject: [PATCH 42/52] divided parallel ci tests to reduce memory usage (#4600) Signed-off-by: Ameya Mahabaleshwarkar Co-authored-by: Eric Harper --- Jenkinsfile | 66 ++++++++++++++++++++++++++++++++++++----------------- 1 file changed, 45 insertions(+), 21 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index babd4129ed2e..72f509e22790 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -1181,7 +1181,7 @@ pipeline { } } - stage('L2: Parallel BERT/BART/GPT2 Question-Answering SQUAD v1.1 & v2.0') { + stage('L2: Parallel BERT Question-Answering SQUAD v1.1 & v2.0') { when { anyOf { branch 'main' @@ -1215,33 +1215,41 @@ pipeline { exp_manager=null && TRANSFORMERS_OFFLINE=1' } } - stage('BART SQUAD 1.1') { + stage('BERT SQUAD 2.0') { // Cannot do fast_dev_run because squad needs whole dev dataset steps { sh 'TRANSFORMERS_OFFLINE=0 && cd examples/nlp/question_answering && \ python question_answering.py \ - model.train_ds.file=/home/TestData/nlp/squad_mini/v1.1/train-v1.1.json \ + model.train_ds.file=/home/TestData/nlp/squad_mini/v2.0/train-v2.0.json \ model.dataset.use_cache=false \ - model.dataset.check_if_answer_in_context=false \ - model.validation_ds.file=/home/TestData/nlp/squad_mini/v1.1/dev-v1.1.json \ - model.test_ds.file=/home/TestData/nlp/squad_mini/v1.1/dev-v1.1.json \ model.train_ds.batch_size=2 \ model.train_ds.num_samples=2 \ model.validation_ds.batch_size=2 \ model.validation_ds.num_samples=2 \ - model.test_ds.num_samples=2 \ - model.test_ds.batch_size=2 \ trainer.max_epochs=1 \ trainer.max_steps=1 \ - model.language_model.pretrained_model_name=facebook/bart-base \ - model.dataset.version_2_with_negative=false \ + model.validation_ds.file=/home/TestData/nlp/squad_mini/v2.0/dev-v2.0.json \ + model.language_model.pretrained_model_name=bert-base-uncased \ + model.dataset.version_2_with_negative=true \ trainer.precision=16 \ - trainer.devices=[0] \ + trainer.devices=[1] \ trainer.accelerator="gpu" \ exp_manager=null && TRANSFORMERS_OFFLINE=1' } } - stage('GPT2 SQUAD 1.1') { + } + } + + stage('L2: Parallel BART Question-Answering SQUAD v1.1 & v2.0') { + when { + anyOf { + branch 'main' + changeRequest target: 'main' + } + } + failFast true + parallel { + stage('BART SQUAD 1.1') { // Cannot do fast_dev_run because squad needs whole dev dataset steps { sh 'TRANSFORMERS_OFFLINE=0 && cd examples/nlp/question_answering && \ @@ -1259,7 +1267,7 @@ pipeline { model.test_ds.batch_size=2 \ trainer.max_epochs=1 \ trainer.max_steps=1 \ - model.language_model.pretrained_model_name=gpt2 \ + model.language_model.pretrained_model_name=facebook/bart-base \ model.dataset.version_2_with_negative=false \ trainer.precision=16 \ trainer.devices=[0] \ @@ -1267,13 +1275,14 @@ pipeline { exp_manager=null && TRANSFORMERS_OFFLINE=1' } } - stage('BERT SQUAD 2.0') { + stage('BART SQUAD 2.0') { // Cannot do fast_dev_run because squad needs whole dev dataset steps { sh 'TRANSFORMERS_OFFLINE=0 && cd examples/nlp/question_answering && \ python question_answering.py \ model.train_ds.file=/home/TestData/nlp/squad_mini/v2.0/train-v2.0.json \ model.dataset.use_cache=false \ + model.dataset.check_if_answer_in_context=false \ model.train_ds.batch_size=2 \ model.train_ds.num_samples=2 \ model.validation_ds.batch_size=2 \ @@ -1281,7 +1290,7 @@ pipeline { trainer.max_epochs=1 \ trainer.max_steps=1 \ model.validation_ds.file=/home/TestData/nlp/squad_mini/v2.0/dev-v2.0.json \ - model.language_model.pretrained_model_name=bert-base-uncased \ + model.language_model.pretrained_model_name=facebook/bart-base \ model.dataset.version_2_with_negative=true \ trainer.precision=16 \ trainer.devices=[1] \ @@ -1289,25 +1298,40 @@ pipeline { exp_manager=null && TRANSFORMERS_OFFLINE=1' } } - stage('BART SQUAD 2.0') { + } + } + + stage('L2: Parallel GPT2 Question-Answering SQUAD v1.1 & v2.0') { + when { + anyOf { + branch 'main' + changeRequest target: 'main' + } + } + failFast true + parallel { + stage('GPT2 SQUAD 1.1') { // Cannot do fast_dev_run because squad needs whole dev dataset steps { sh 'TRANSFORMERS_OFFLINE=0 && cd examples/nlp/question_answering && \ python question_answering.py \ - model.train_ds.file=/home/TestData/nlp/squad_mini/v2.0/train-v2.0.json \ + model.train_ds.file=/home/TestData/nlp/squad_mini/v1.1/train-v1.1.json \ model.dataset.use_cache=false \ model.dataset.check_if_answer_in_context=false \ + model.validation_ds.file=/home/TestData/nlp/squad_mini/v1.1/dev-v1.1.json \ + model.test_ds.file=/home/TestData/nlp/squad_mini/v1.1/dev-v1.1.json \ model.train_ds.batch_size=2 \ model.train_ds.num_samples=2 \ model.validation_ds.batch_size=2 \ model.validation_ds.num_samples=2 \ + model.test_ds.num_samples=2 \ + model.test_ds.batch_size=2 \ trainer.max_epochs=1 \ trainer.max_steps=1 \ - model.validation_ds.file=/home/TestData/nlp/squad_mini/v2.0/dev-v2.0.json \ - model.language_model.pretrained_model_name=facebook/bart-base \ - model.dataset.version_2_with_negative=true \ + model.language_model.pretrained_model_name=gpt2 \ + model.dataset.version_2_with_negative=false \ trainer.precision=16 \ - trainer.devices=[1] \ + trainer.devices=[0] \ trainer.accelerator="gpu" \ exp_manager=null && TRANSFORMERS_OFFLINE=1' } From 7890979f622136cf1ba7ff558fb11a092c77700e Mon Sep 17 00:00:00 2001 From: Iztok Lebar Bajec Date: Tue, 26 Jul 2022 15:51:16 +0200 Subject: [PATCH 43/52] fix tarred dataset len when num shards is not divisible by workers (#4553) * fix tarred dataset len when num shards is not divisible by workers Signed-off-by: Iztok Lebar Bajec * update error reporting on invalid `shard_strategy` * update NLP/PC tarred dataset docstring * add `shard_strategy` to NLP/PC `@dataclass` * update NLP/PC tarred dataset docstring * add `shard_strategy` to NLP/PC docs * revert test with Dataloader retruning the actual data length * make dataloader return actual num of samples, set `limit_train_baches` on `setup_*` * update `shard_strategy` docstrings Signed-off-by: Iztok Lebar Bajec * update `tarred_dataset` documentation Signed-off-by: Iztok Lebar Bajec * fix style * update documentation Signed-off-by: Iztok Lebar Bajec * updated docstrings Signed-off-by: Iztok Lebar Bajec Co-authored-by: PeganovAnton --- docs/source/asr/datasets.rst | 66 ++++++++++--------- .../nlp/punctuation_and_capitalization.rst | 28 ++++++-- nemo/collections/asr/data/audio_to_label.py | 36 ++++++---- nemo/collections/asr/data/audio_to_text.py | 35 ++++++---- .../data/language_modeling/l2r_lm_dataset.py | 23 +++++-- .../language_modeling/sentence_dataset.py | 29 +++++--- .../machine_translation_dataset.py | 27 +++++--- .../text_normalization/decoder_dataset.py | 28 +++++--- .../punctuation_capitalization_dataset.py | 20 +++++- ...nctuation_capitalization_tarred_dataset.py | 65 +++++++++++++++--- .../duplex_decoder.py | 35 ++++++++++ .../language_modeling/transformer_lm_model.py | 34 ++++++++++ .../machine_translation/mt_enc_dec_model.py | 36 ++++++++++ .../punctuation_capitalization_model.py | 37 +++++++++++ 14 files changed, 395 insertions(+), 104 deletions(-) diff --git a/docs/source/asr/datasets.rst b/docs/source/asr/datasets.rst index 364c7fea1926..8878d66ae739 100644 --- a/docs/source/asr/datasets.rst +++ b/docs/source/asr/datasets.rst @@ -1,7 +1,7 @@ Datasets ======== -NeMo has scripts to convert several common ASR datasets into the format expected by the ``nemo_asr`` collection. You can get started +NeMo has scripts to convert several common ASR datasets into the format expected by the ``nemo_asr`` collection. You can get started with those datasets by following the instructions to run those scripts in the section appropriate to each dataset below. If the user has their own data and want to preprocess it to use with NeMo ASR models, refer to the `Preparing Custom ASR Data`_ section. @@ -13,8 +13,8 @@ If the user already has a dataset that you want to convert to a tarred format, r LibriSpeech ----------- -Run the following scripts to download the LibriSpeech data and convert it into the format expected by `nemo_asr`. At least 250GB free -space is required. +Run the following scripts to download the LibriSpeech data and convert it into the format expected by `nemo_asr`. At least 250GB free +space is required. .. code-block:: bash @@ -37,18 +37,18 @@ Fisher English Training Speech Run these scripts to convert the Fisher English Training Speech data into a format expected by the ``nemo_asr`` collection. -In brief, the following scripts convert the ``.sph`` files to ``.wav``, slices those files into smaller audio samples, matches the -smaller slices with their corresponding transcripts, and splits the resulting audio segments into train, validation, and test sets +In brief, the following scripts convert the ``.sph`` files to ``.wav``, slices those files into smaller audio samples, matches the +smaller slices with their corresponding transcripts, and splits the resulting audio segments into train, validation, and test sets (with one manifest each). .. note:: - 106 GB of space is required to run the ``.wav`` conversion - additional 105 GB is required for the slicing and matching - - ``sph2pipe`` is required in order to run the ``.wav`` conversion + - ``sph2pipe`` is required in order to run the ``.wav`` conversion **Instructions** -The following scripts assume that you already have the Fisher dataset from the Linguistic Data Consortium, with a directory structure +The following scripts assume that you already have the Fisher dataset from the Linguistic Data Consortium, with a directory structure that looks similar to the following: .. code-block:: bash @@ -67,7 +67,7 @@ that looks similar to the following: ├── fe_03_p2_sph3 └── ... -The transcripts that will be used are located in the ``fe_03_p<1,2>_transcripts/data/trans`` directory. The audio files (``.sph``) +The transcripts that will be used are located in the ``fe_03_p<1,2>_transcripts/data/trans`` directory. The audio files (``.sph``) are located in the remaining directories in an ``audio`` subdirectory. #. Convert the audio files from ``.sph`` to ``.wav`` by running: @@ -78,7 +78,7 @@ are located in the remaining directories in an ``audio`` subdirectory. python fisher_audio_to_wav.py \ --data_root= --dest_root= - This will place the unsliced ``.wav`` files in ``/LDC200[4,5]S13-Part[1,2]/audio-wav/``. It will take several + This will place the unsliced ``.wav`` files in ``/LDC200[4,5]S13-Part[1,2]/audio-wav/``. It will take several minutes to run. #. Process the transcripts and slice the audio data. @@ -90,7 +90,7 @@ are located in the remaining directories in an ``audio`` subdirectory. --dest_root= \ --remove_noises - This script splits the full dataset into train, validation, test sets, and places the audio slices in the corresponding folders + This script splits the full dataset into train, validation, test sets, and places the audio slices in the corresponding folders in the destination directory. One manifest is written out per set, which includes each slice's transcript, duration, and path. This will likely take around 20 minutes to run. Once finished, delete the 10 minute long ``.wav`` files. @@ -100,8 +100,8 @@ are located in the remaining directories in an ``audio`` subdirectory. Run the following script to convert the HUB5 data into a format expected by the ``nemo_asr`` collection. -Similarly, to the Fisher dataset processing scripts, this script converts the ``.sph`` files to ``.wav``, slices the audio files and -transcripts into utterances, and combines them into segments of some minimum length (default is 10 seconds). The resulting segments +Similarly, to the Fisher dataset processing scripts, this script converts the ``.sph`` files to ``.wav``, slices the audio files and +transcripts into utterances, and combines them into segments of some minimum length (default is 10 seconds). The resulting segments are all written out to an audio directory and the corresponding transcripts are written to a manifest JSON file. .. note:: @@ -123,7 +123,7 @@ You can optionally include ``--min_slice_duration=`` if you would l AN4 Dataset ----------- -This is a small dataset recorded and distributed by Carnegie Mellon University. It consists of recordings of people spelling out +This is a small dataset recorded and distributed by Carnegie Mellon University. It consists of recordings of people spelling out addresses, names, etc. Information about this dataset can be found on the `official CMU site `_. #. `Download and extract the dataset `_ (which is labeled "NIST's Sphere audio (.sph) format (64M)". @@ -153,14 +153,14 @@ After the script finishes, the ``data`` folder should contain a ``data_aishell`` Aishell-2 --------- -To process the AIShell-2 dataset, in the command below, set the data folder of AIShell-2 using ``--audio_folder`` and where to push -these files using ``--dest_folder``. In order to generate files in the supported format of ``nemo_asr``, run: +To process the AIShell-2 dataset, in the command below, set the data folder of AIShell-2 using ``--audio_folder`` and where to push +these files using ``--dest_folder``. In order to generate files in the supported format of ``nemo_asr``, run: .. code-block:: bash python process_aishell2_data.py --audio_folder= --dest_folder= -After the script finishes, the ``train.json``, ``dev.json``, ``test.json``, and ``vocab.txt`` files can be found in the ``dest_folder`` directory. +After the script finishes, the ``train.json``, ``dev.json``, ``test.json``, and ``vocab.txt`` files can be found in the ``dest_folder`` directory. Preparing Custom ASR Data ------------------------- @@ -171,7 +171,7 @@ The audio files can be of any format supported by `Pydub `` and the special + For brace expansion, there may be cases where ``{x..y}`` syntax cannot be used due to shell interference. This occurs most commonly + inside SLURM scripts. Therefore, we provide a few equivalent replacements. Supported opening braces (equivalent to ``{``) are ``(``, + ``[``, ``<`` and the special tag ``_OP_``. Supported closing braces (equivalent to ``}``) are ``)``, ``]``, ``>`` and the special tag ``_CL_``. For SLURM based tasks, we suggest the use of the special tags for ease of use. -As with non-tarred datasets, the manifest file should be passed in ``manifest_filepath``. The dataloader assumes that the length +As with non-tarred datasets, the manifest file should be passed in ``manifest_filepath``. The dataloader assumes that the length of the manifest after filtering is the correct size of the dataset for reporting training progress. -The ``tarred_shard_strategy`` field of the config file can be set if you have multiple shards and are running an experiment with +The ``tarred_shard_strategy`` field of the config file can be set if you have multiple shards and are running an experiment with multiple workers. It defaults to ``scatter``, which preallocates a set of shards per worker which do not change during runtime. +Note that this strategy, on specific occasions (when the number of shards is not divisible with ``world_size``), will not sample +the entire dataset. As an alternative the ``replicate`` strategy, will preallocate the entire set of shards to every worker and not +change it during runtime. The benefit of this strategy is that it allows each worker to sample data points from the entire dataset +independently of others. Note, though, that more than one worker may sample the same shard, and even sample the same data points! +As such, there is no assured guarantee that all samples in the dataset will be sampled at least once during 1 epoch. Note that +for these reasons it is not advisable to use tarred datasets as validation and test datasets. For more information about the individual tarred datasets and the parameters available, including shuffling options, see the corresponding class APIs in the `Datasets <./api.html#Datasets>`__ section. @@ -228,7 +234,7 @@ see the corresponding class APIs in the `Datasets <./api.html#Datasets>`__ secti If using multiple workers, the number of shards should be divisible by the world size to ensure an even split among workers. If it is not divisible, logging will give a warning but training will proceed, but likely hang at the last epoch. In addition, if using distributed processing, each shard must have the same number of entries after filtering is - applied such that each worker ends up with the same number of files. We currently do not check for this in any dataloader, but the user's + applied such that each worker ends up with the same number of files. We currently do not check for this in any dataloader, but the user's program may hang if the shards are uneven. Conversion to Tarred Datasets @@ -262,9 +268,9 @@ The files in the target directory should look similar to the following: ├── metadata.yaml └── tarred_audio_manifest.json -Note that file structures are flattened such that all audio files are at the top level in each tarball. This ensures that -filenames are unique in the tarred dataset and the filepaths do not contain "-sub" and forward slashes in each ``audio_filepath`` are -simply converted to underscores. For example, a manifest entry for ``/data/directory1/file.wav`` would be ``_data_directory1_file.wav`` +Note that file structures are flattened such that all audio files are at the top level in each tarball. This ensures that +filenames are unique in the tarred dataset and the filepaths do not contain "-sub" and forward slashes in each ``audio_filepath`` are +simply converted to underscores. For example, a manifest entry for ``/data/directory1/file.wav`` would be ``_data_directory1_file.wav`` in the tarred dataset manifest, and ``/data/directory2/file.wav`` would be converted to ``_data_directory2_file.wav``. Bucketing Datasets @@ -325,9 +331,9 @@ Currently bucketing feature is just supported for tarred datasets. Upsampling Datasets ------------------ -Buckets may also be 'weighted' to allow multiple runs through a target dataset during each training epoch. This can be beneficial in cases when a dataset is composed of several component sets of unequal sizes and one desires to mitigate bias towards the larger sets through oversampling. +Buckets may also be 'weighted' to allow multiple runs through a target dataset during each training epoch. This can be beneficial in cases when a dataset is composed of several component sets of unequal sizes and one desires to mitigate bias towards the larger sets through oversampling. -Weighting is managed with the `bucketing_weights` parameter. After passing your composite tarred datasets in the format described above for bucketing, pass a list of integers (one per bucket) to indicate how many times a manifest should be read during training. +Weighting is managed with the `bucketing_weights` parameter. After passing your composite tarred datasets in the format described above for bucketing, pass a list of integers (one per bucket) to indicate how many times a manifest should be read during training. For example, by passing `[2,1,1,3]` to the code below: @@ -363,7 +369,7 @@ If using adaptive bucketing, note that the same batch size will be assigned to e model.train_ds.bucketing_weights=[2,1,1,3] model.train_ds.bucketing_batch_size=[4,4,4,2] -All instances of data from `bucket4` will still be trained with a batch size of 2 while all others would have a batch size of 4. As with standard bucketing, this requires `batch_size`` to be set to 1. +All instances of data from `bucket4` will still be trained with a batch size of 2 while all others would have a batch size of 4. As with standard bucketing, this requires `batch_size`` to be set to 1. If `bucketing_batch_size` is not specified, all datasets will be passed with the same fixed batch size as specified by the `batch_size` parameter. -It is recommended to set bucketing strategies to `fully_randomized` during multi-GPU training to prevent possible dataset bias during training. \ No newline at end of file +It is recommended to set bucketing strategies to `fully_randomized` during multi-GPU training to prevent possible dataset bias during training. \ No newline at end of file diff --git a/docs/source/nlp/punctuation_and_capitalization.rst b/docs/source/nlp/punctuation_and_capitalization.rst index be2890943acb..16a1e6856703 100755 --- a/docs/source/nlp/punctuation_and_capitalization.rst +++ b/docs/source/nlp/punctuation_and_capitalization.rst @@ -3,7 +3,7 @@ Punctuation and Capitalization Model ==================================== -Automatic Speech Recognition (ASR) systems typically generate text with no punctuation and capitalization of the words. +Automatic Speech Recognition (ASR) systems typically generate text with no punctuation and capitalization of the words. There are two issues with non-punctuated ASR output: - it could be difficult to read and understand @@ -35,7 +35,7 @@ For each word in the input text, the Punctuation and Capitalization model: - predicts a punctuation mark that should follow the word (if any). By default, the model supports commas, periods, and question marks. - predicts if the word should be capitalized or not -In the Punctuation and Capitalization model, we are jointly training two token-level classifiers on top of a pre-trained +In the Punctuation and Capitalization model, we are jointly training two token-level classifiers on top of a pre-trained language model, such as `BERT: Pre-training of Deep Bidirectional Transformers for Language Understanding `__ :cite:`nlp-punct-devlin2018bert`. .. note:: @@ -85,7 +85,7 @@ NeMo Data Format The Punctuation and Capitalization model expects the data in the following format: -The training and evaluation data is divided into 2 files: +The training and evaluation data is divided into 2 files: - ``text.txt`` - ``labels.txt`` @@ -108,10 +108,10 @@ spaces. Each label in ``labels.txt`` file consists of 2 symbols: - the second symbol determines if a word needs to be capitalized or not (where ``U`` indicates that the word should be upper cased, and ``O`` - no capitalization needed) -By default, the following punctuation marks are considered: commas, periods, and question marks; the remaining punctuation marks were +By default, the following punctuation marks are considered: commas, periods, and question marks; the remaining punctuation marks were removed from the data. This can be changed by introducing new labels in the ``labels.txt`` files. -Each line of the ``labels.txt`` should follow the format: ``[LABEL] [SPACE] [LABEL] [SPACE] [LABEL]`` (for ``labels.txt``). For example, +Each line of the ``labels.txt`` should follow the format: ``[LABEL] [SPACE] [LABEL] [SPACE] [LABEL]`` (for ``labels.txt``). For example, labels for the above ``text.txt`` file should be: :: @@ -120,7 +120,7 @@ labels for the above ``text.txt`` file should be: OU OO OO OO ... ... -The complete list of all possible labels used in this tutorial are: +The complete list of all possible labels used in this tutorial are: - ``OO`` - ``.O`` @@ -588,6 +588,22 @@ For convenience, items of data config are described in 4 tables: - ``1`` - The size of shuffle buffer of `webdataset `_. The number of batches which are permuted. + * - **shard_strategy** + - string + - ``scatter`` + - Tarred dataset shard distribution strategy chosen as a str value during ddp. Accepted values are ``scatter`` and ``replicate``. + ``scatter``: Each node gets a unique set of shards, which are permanently pre-allocated and never changed at runtime, when the total + number of shards is not divisible with ``world_size``, some shards (at max ``world_size-1``) will not be used. + ``replicate``: Each node gets the entire set of shards available in the tarred dataset, which are permanently pre-allocated and never + changed at runtime. The benefit of replication is that it allows each node to sample data points from the entire dataset independently + of other nodes, and reduces dependence on value of ``tar_shuffle_n``. + + .. warning:: + Replicated strategy allows every node to sample the entire set of available tarfiles, and therefore more than one node may sample + the same tarfile, and even sample the same data points! As such, there is no assured guarantee that all samples in the dataset will be + sampled at least once during 1 epoch. Scattered strategy, on the other hand, on specific occasions (when the number of shards is not + divisible with ``world_size``), will not sample the entire dataset. For these reasons it is not advisable to use tarred datasets as + validation or test datasets. .. _pytorch-dataloader-parameters-label: diff --git a/nemo/collections/asr/data/audio_to_label.py b/nemo/collections/asr/data/audio_to_label.py index d002c0c76717..92dc8873d9ec 100644 --- a/nemo/collections/asr/data/audio_to_label.py +++ b/nemo/collections/asr/data/audio_to_label.py @@ -485,10 +485,14 @@ class _TarredAudioLabelDataset(IterableDataset): The benefit of replication is that it allows each node to sample data points from the entire dataset independently of other nodes, and reduces dependence on value of `shuffle_n`. - Note: Replicated strategy allows every node to sample the entire set of available tarfiles, - and therefore more than one node may sample the same tarfile, and even sample the same - data points! As such, there is no assured guarantee that all samples in the dataset will be - sampled at least once during 1 epoch. + .. warning:: + Replicated strategy allows every node to sample the entire set of available tarfiles, + and therefore more than one node may sample the same tarfile, and even sample the same + data points! As such, there is no assured guarantee that all samples in the dataset will be + sampled at least once during 1 epoch. Scattered strategy, on the other hand, on specific + occasions (when the number of shards is not divisible with ``world_size``), will not sample + the entire dataset. For these reasons it is not advisable to use tarred datasets as validation + or test datasets. global_rank (int): Worker rank, used for partitioning shards. Defaults to 0. world_size (int): Total number of processes, used for partitioning shards. Defaults to 0. is_regression_task (bool): Whether it is a regression task. Defualts to False. @@ -697,10 +701,14 @@ class TarredAudioToClassificationLabelDataset(_TarredAudioLabelDataset): The benefit of replication is that it allows each node to sample data points from the entire dataset independently of other nodes, and reduces dependence on value of `shuffle_n`. - Note: Replicated strategy allows every node to sample the entire set of available tarfiles, - and therefore more than one node may sample the same tarfile, and even sample the same - data points! As such, there is no assured guarantee that all samples in the dataset will be - sampled at least once during 1 epoch. + .. warning:: + Replicated strategy allows every node to sample the entire set of available tarfiles, + and therefore more than one node may sample the same tarfile, and even sample the same + data points! As such, there is no assured guarantee that all samples in the dataset will be + sampled at least once during 1 epoch. Scattered strategy, on the other hand, on specific + occasions (when the number of shards is not divisible with ``world_size``), will not sample + the entire dataset. For these reasons it is not advisable to use tarred datasets as validation + or test datasets. global_rank (int): Worker rank, used for partitioning shards. Defaults to 0. world_size (int): Total number of processes, used for partitioning shards. Defaults to 0. is_regression_task (bool): Whether it is a regression task. Defualts to False. @@ -771,10 +779,14 @@ class TarredAudioToSpeechLabelDataset(_TarredAudioLabelDataset): The benefit of replication is that it allows each node to sample data points from the entire dataset independently of other nodes, and reduces dependence on value of `shuffle_n`. - Note: Replicated strategy allows every node to sample the entire set of available tarfiles, - and therefore more than one node may sample the same tarfile, and even sample the same - data points! As such, there is no assured guarantee that all samples in the dataset will be - sampled at least once during 1 epoch. + .. warning:: + Replicated strategy allows every node to sample the entire set of available tarfiles, + and therefore more than one node may sample the same tarfile, and even sample the same + data points! As such, there is no assured guarantee that all samples in the dataset will be + sampled at least once during 1 epoch. Scattered strategy, on the other hand, on specific + occasions (when the number of shards is not divisible with ``world_size``), will not sample + the entire dataset. For these reasons it is not advisable to use tarred datasets as validation + or test datasets. global_rank (int): Worker rank, used for partitioning shards. Defaults to 0. world_size (int): Total number of processes, used for partitioning shards. Defaults to 0. """ diff --git a/nemo/collections/asr/data/audio_to_text.py b/nemo/collections/asr/data/audio_to_text.py index 573dfd672fa2..7b7caa90d697 100644 --- a/nemo/collections/asr/data/audio_to_text.py +++ b/nemo/collections/asr/data/audio_to_text.py @@ -579,10 +579,14 @@ class _TarredAudioToTextDataset(IterableDataset): The benefit of replication is that it allows each node to sample data points from the entire dataset independently of other nodes, and reduces dependence on value of `shuffle_n`. - Note: Replicated strategy allows every node to sample the entire set of available tarfiles, - and therefore more than one node may sample the same tarfile, and even sample the same - data points! As such, there is no assured guarantee that all samples in the dataset will be - sampled at least once during 1 epoch. + .. warning:: + Replicated strategy allows every node to sample the entire set of available tarfiles, + and therefore more than one node may sample the same tarfile, and even sample the same + data points! As such, there is no assured guarantee that all samples in the dataset will be + sampled at least once during 1 epoch. Scattered strategy, on the other hand, on specific + occasions (when the number of shards is not divisible with ``world_size``), will not sample + the entire dataset. For these reasons it is not advisable to use tarred datasets as validation + or test datasets. global_rank (int): Worker rank, used for partitioning shards. Defaults to 0. world_size (int): Total number of processes, used for partitioning shards. Defaults to 0. return_sample_id (bool): whether to return the sample_id as a part of each sample @@ -840,10 +844,14 @@ class TarredAudioToCharDataset(_TarredAudioToTextDataset): The benefit of replication is that it allows each node to sample data points from the entire dataset independently of other nodes, and reduces dependence on value of `shuffle_n`. - Note: Replicated strategy allows every node to sample the entire set of available tarfiles, - and therefore more than one node may sample the same tarfile, and even sample the same - data points! As such, there is no assured guarantee that all samples in the dataset will be - sampled at least once during 1 epoch. + .. warning:: + Replicated strategy allows every node to sample the entire set of available tarfiles, + and therefore more than one node may sample the same tarfile, and even sample the same + data points! As such, there is no assured guarantee that all samples in the dataset will be + sampled at least once during 1 epoch. Scattered strategy, on the other hand, on specific + occasions (when the number of shards is not divisible with ``world_size``), will not sample + the entire dataset. For these reasons it is not advisable to use tarred datasets as validation + or test datasets. global_rank (int): Worker rank, used for partitioning shards. Defaults to 0. world_size (int): Total number of processes, used for partitioning shards. Defaults to 0. return_sample_id (bool): whether to return the sample_id as a part of each sample @@ -967,10 +975,13 @@ class TarredAudioToBPEDataset(_TarredAudioToTextDataset): The benefit of replication is that it allows each node to sample data points from the entire dataset independently of other nodes, and reduces dependence on value of `shuffle_n`. - Note: Replicated strategy allows every node to sample the entire set of available tarfiles, - and therefore more than one node may sample the same tarfile, and even sample the same - data points! As such, there is no assured guarantee that all samples in the dataset will be - sampled at least once during 1 epoch. + .. warning:: Replicated strategy allows every node to sample the entire set of available tarfiles, + and therefore more than one node may sample the same tarfile, and even sample the same + data points! As such, there is no assured guarantee that all samples in the dataset will be + sampled at least once during 1 epoch. Scattered strategy, on the other hand, on specific + occasions (when the number of shards is not divisible with ``world_size``), will not sample + the entire dataset. For these reasons it is not advisable to use tarred datasets as validation + or test datasets. global_rank (int): Worker rank, used for partitioning shards. Defaults to 0. world_size (int): Total number of processes, used for partitioning shards. Defaults to 0. return_sample_id (bool): whether to return the sample_id as a part of each sample diff --git a/nemo/collections/nlp/data/language_modeling/l2r_lm_dataset.py b/nemo/collections/nlp/data/language_modeling/l2r_lm_dataset.py index eff0737c7fec..adb6f126cd78 100644 --- a/nemo/collections/nlp/data/language_modeling/l2r_lm_dataset.py +++ b/nemo/collections/nlp/data/language_modeling/l2r_lm_dataset.py @@ -31,7 +31,7 @@ class L2RLanguageModelingDataset(Dataset): """ Dataset for training and evaluating left-to-right language models. - + Args: tokenizer: tokenizer, such as WordTokenizer or CharTokenizer dataset: path to data @@ -73,7 +73,7 @@ def __getitem__(self, idx): class TarredL2RLanguageModelingDataset(IterableDataset): """ A similar Dataset to the L2RLanguageModelingDataset, but which loads tarred tokenized numpy files. - Accepts a single JSON metadata manifest file as well as the path(s) to the tarball(s) containing the wav files. + Accepts a single JSON metadata manifest file as well as the path(s) to the tarball(s) containing the wav files. The manifest should contain information such as the number of shards, the number of tokens in the corpus, and the number of tokens contained within each shard of the tarfile(s). @@ -114,10 +114,15 @@ class TarredL2RLanguageModelingDataset(IterableDataset): available in the tarred dataset, which are permanently pre-allocated and never changed at runtime. The benefit of replication is that it allows each node to sample data points from the entire dataset independently of other nodes, and reduces dependence on value of `shuffle_n`. - Note: Replicated strategy allows every node to sample the entire set of available tarfiles, - and therefore more than one node may sample the same tarfile, and even sample the same - data points! As such, there is no assured guarantee that all samples in the dataset will be - sampled at least once during 1 epoch. + + .. warning:: + Replicated strategy allows every node to sample the entire set of available tarfiles, + and therefore more than one node may sample the same tarfile, and even sample the same + data points! As such, there is no assured guarantee that all samples in the dataset will be + sampled at least once during 1 epoch. Scattered strategy, on the other hand, on specific + occasions (when the number of shards is not divisible with ``world_size``), will not sample + the entire dataset. For these reasons it is not advisable to use tarred datasets as validation + or test datasets. global_rank (int): Worker rank, used for partitioning shards. Defaults to 0. world_size (int): Total number of processes, used for partitioning shards. Defaults to 0. """ @@ -142,7 +147,11 @@ def __init__( valid_shard_strategies = ['scatter', 'replicate'] if shard_strategy not in valid_shard_strategies: - raise ValueError(f"`shard_strategy` must be one of {valid_shard_strategies}") + raise ValueError( + f"Invalid shard strategy of type {type(shard_strategy)} " + f"{repr(shard_strategy) if len(repr(shard_strategy)) < 100 else repr(shard_strategy)[:100] + '...'}! " + f"Allowed values are: {valid_shard_strategies}." + ) with open(metadata_path, 'r') as f: metadata = json.load(f) diff --git a/nemo/collections/nlp/data/language_modeling/sentence_dataset.py b/nemo/collections/nlp/data/language_modeling/sentence_dataset.py index e677fbade383..26127bc3aa36 100644 --- a/nemo/collections/nlp/data/language_modeling/sentence_dataset.py +++ b/nemo/collections/nlp/data/language_modeling/sentence_dataset.py @@ -142,7 +142,7 @@ class TarredSentenceDataset(IterableDataset): """ A similar Dataset to the SentenceDataset, but which loads tarred tokenized pickle files. Accepts a single JSON metadata file containing the total number of batches - as well as the path(s) to the tarball(s) containing the wav files. + as well as the path(s) to the tarball(s) containing the wav files. Valid formats for the text_tar_filepaths argument include: (1) a single string that can be brace-expanded, e.g. 'path/to/text.tar' or 'path/to/text_{1..100}.tar.gz', or (2) a list of file paths that will not be brace-expanded, e.g. ['text_1.tar', 'text_2.tar', ...]. @@ -172,10 +172,15 @@ class TarredSentenceDataset(IterableDataset): available in the tarred dataset, which are permanently pre-allocated and never changed at runtime. The benefit of replication is that it allows each node to sample data points from the entire dataset independently of other nodes, and reduces dependence on value of `shuffle_n`. - Note: Replicated strategy allows every node to sample the entire set of available tarfiles, - and therefore more than one node may sample the same tarfile, and even sample the same - data points! As such, there is no assured guarantee that all samples in the dataset will be - sampled at least once during 1 epoch. + + .. warning:: + Replicated strategy allows every node to sample the entire set of available tarfiles, + and therefore more than one node may sample the same tarfile, and even sample the same + data points! As such, there is no assured guarantee that all samples in the dataset will be + sampled at least once during 1 epoch. Scattered strategy, on the other hand, on specific + occasions (when the number of shards is not divisible with ``world_size``), will not sample + the entire dataset. For these reasons it is not advisable to use tarred datasets as validation + or test datasets. global_rank (int): Worker rank, used for partitioning shards. Defaults to 0. world_size (int): Total number of processes, used for partitioning shards. Defaults to 0. reverse_lang_direction (bool): When True, swaps the source and target directions when returning minibatches. @@ -198,7 +203,11 @@ def __init__( valid_shard_strategies = ['scatter', 'replicate'] if shard_strategy not in valid_shard_strategies: - raise ValueError(f"`shard_strategy` must be one of {valid_shard_strategies}") + raise ValueError( + f"Invalid shard strategy of type {type(shard_strategy)} " + f"{repr(shard_strategy) if len(repr(shard_strategy)) < 100 else repr(shard_strategy)[:100] + '...'}! " + f"Allowed values are: {valid_shard_strategies}." + ) with open(metadata_path, 'r') as f: metadata = json.load(f) @@ -223,12 +232,14 @@ def __init__( text_tar_filepaths = list(braceexpand.braceexpand(text_tar_filepaths)) if shard_strategy == 'scatter': - logging.info("All tarred dataset shards will be scattered evenly across all nodes.") + logging.info("Tarred dataset shards will be scattered evenly across all nodes.") if len(text_tar_filepaths) % world_size != 0: logging.warning( f"Number of shards in tarred dataset ({len(text_tar_filepaths)}) is not divisible " - f"by number of distributed workers ({world_size})." + f"by number of distributed workers ({world_size}). " + f"Some shards will not be used ({len(text_tar_filepaths) % world_size})." ) + batches_per_tar = self.metadata['num_batches'] // len(text_tar_filepaths) begin_idx = (len(text_tar_filepaths) // world_size) * global_rank end_idx = begin_idx + (len(text_tar_filepaths) // world_size) logging.info('Begin Index : %d' % (begin_idx)) @@ -237,7 +248,7 @@ def __init__( logging.info( "Partitioning tarred dataset: process (%d) taking shards [%d, %d)", global_rank, begin_idx, end_idx ) - self.length = self.metadata['num_batches'] // world_size + self.length = batches_per_tar * len(text_tar_filepaths) * world_size elif shard_strategy == 'replicate': logging.info("All tarred dataset shards will be replicated across all nodes.") diff --git a/nemo/collections/nlp/data/machine_translation/machine_translation_dataset.py b/nemo/collections/nlp/data/machine_translation/machine_translation_dataset.py index fb253df1c098..ac1db2123d99 100644 --- a/nemo/collections/nlp/data/machine_translation/machine_translation_dataset.py +++ b/nemo/collections/nlp/data/machine_translation/machine_translation_dataset.py @@ -326,10 +326,15 @@ class TarredTranslationDataset(IterableDataset): available in the tarred dataset, which are permanently pre-allocated and never changed at runtime. The benefit of replication is that it allows each node to sample data points from the entire dataset independently of other nodes, and reduces dependence on value of `shuffle_n`. - Note: Replicated strategy allows every node to sample the entire set of available tarfiles, - and therefore more than one node may sample the same tarfile, and even sample the same - data points! As such, there is no assured guarantee that all samples in the dataset will be - sampled at least once during 1 epoch. + + .. warning:: + Replicated strategy allows every node to sample the entire set of available tarfiles, + and therefore more than one node may sample the same tarfile, and even sample the same + data points! As such, there is no assured guarantee that all samples in the dataset will be + sampled at least once during 1 epoch. Scattered strategy, on the other hand, on specific + occasions (when the number of shards is not divisible with ``world_size``), will not sample + the entire dataset. For these reasons it is not advisable to use tarred datasets as validation + or test datasets. global_rank (int): Worker rank, used for partitioning shards. Defaults to 0. world_size (int): Total number of processes, used for partitioning shards. Defaults to 1. reverse_lang_direction (bool): When True, swaps the source and target directions when returning minibatches. @@ -360,7 +365,11 @@ def __init__( valid_shard_strategies = ['scatter', 'replicate'] if shard_strategy not in valid_shard_strategies: - raise ValueError(f"`shard_strategy` must be one of {valid_shard_strategies}") + raise ValueError( + f"Invalid shard strategy of type {type(shard_strategy)} " + f"{repr(shard_strategy) if len(repr(shard_strategy)) < 100 else repr(shard_strategy)[:100] + '...'}! " + f"Allowed values are: {valid_shard_strategies}." + ) with open(metadata_path, 'r') as f: metadata = json.load(f) @@ -385,12 +394,14 @@ def __init__( text_tar_filepaths = list(braceexpand.braceexpand(text_tar_filepaths)) if shard_strategy == 'scatter': - logging.info("All tarred dataset shards will be scattered evenly across all nodes.") + logging.info("Tarred dataset shards will be scattered evenly across all nodes.") if len(text_tar_filepaths) % world_size != 0: logging.warning( f"Number of shards in tarred dataset ({len(text_tar_filepaths)}) is not divisible " - f"by number of distributed workers ({world_size})." + f"by number of distributed workers ({world_size}). " + f"Some shards will not be used ({len(text_tar_filepaths) % world_size})." ) + batches_per_tar = self.metadata['num_batches'] // len(text_tar_filepaths) begin_idx = (len(text_tar_filepaths) // world_size) * global_rank end_idx = begin_idx + (len(text_tar_filepaths) // world_size) logging.info('Begin Index : %d' % (begin_idx)) @@ -399,7 +410,7 @@ def __init__( logging.info( "Partitioning tarred dataset: process (%d) taking shards [%d, %d)", global_rank, begin_idx, end_idx ) - self.length = self.metadata['num_batches'] // world_size + self.length = batches_per_tar * len(text_tar_filepaths) * world_size elif shard_strategy == 'replicate': logging.info("All tarred dataset shards will be replicated across all nodes.") diff --git a/nemo/collections/nlp/data/text_normalization/decoder_dataset.py b/nemo/collections/nlp/data/text_normalization/decoder_dataset.py index 1110807dcb0b..724e2d3229e5 100644 --- a/nemo/collections/nlp/data/text_normalization/decoder_dataset.py +++ b/nemo/collections/nlp/data/text_normalization/decoder_dataset.py @@ -452,10 +452,15 @@ class TarredTextNormalizationDecoderDataset(IterableDataset): available in the tarred dataset, which are permanently pre-allocated and never changed at runtime. The benefit of replication is that it allows each node to sample data points from the entire dataset independently of other nodes, and reduces dependence on value of `shuffle_n`. - Note: Replicated strategy allows every node to sample the entire set of available tarfiles, - and therefore more than one node may sample the same tarfile, and even sample the same - data points! As such, there is no assured guarantee that all samples in the dataset will be - sampled at least once during 1 epoch. + + .. warning:: + Replicated strategy allows every node to sample the entire set of available tarfiles, + and therefore more than one node may sample the same tarfile, and even sample the same + data points! As such, there is no assured guarantee that all samples in the dataset will be + sampled at least once during 1 epoch. Scattered strategy, on the other hand, on specific + occasions (when the number of shards is not divisible with ``world_size``), will not sample + the entire dataset. For these reasons it is not advisable to use tarred datasets as validation + or test datasets. global_rank: Worker rank, used for partitioning shards. world_size: Total number of processes, used for partitioning shards. """ @@ -473,7 +478,11 @@ def __init__( valid_shard_strategies = ['scatter', 'replicate'] if shard_strategy not in valid_shard_strategies: - raise ValueError(f"`shard_strategy` must be one of {valid_shard_strategies}") + raise ValueError( + f"Invalid shard strategy of type {type(shard_strategy)} " + f"{repr(shard_strategy) if len(repr(shard_strategy)) < 100 else repr(shard_strategy)[:100] + '...'}! " + f"Allowed values are: {valid_shard_strategies}." + ) if isinstance(text_tar_filepaths, str): # Replace '(', '[', '<' and '_OP_' with '{' @@ -493,12 +502,14 @@ def __init__( text_tar_filepaths = list(braceexpand.braceexpand(text_tar_filepaths)) if shard_strategy == 'scatter': - logging.info("All tarred dataset shards will be scattered evenly across all nodes.") + logging.info("Tarred dataset shards will be scattered evenly across all nodes.") if len(text_tar_filepaths) % world_size != 0: logging.warning( f"Number of shards in tarred dataset ({len(text_tar_filepaths)}) is not divisible " - f"by number of distributed workers ({world_size})." + f"by number of distributed workers ({world_size}). " + f"Some shards will not be used ({len(text_tar_filepaths) % world_size})." ) + batches_per_tar = num_batches // len(text_tar_filepaths) begin_idx = (len(text_tar_filepaths) // world_size) * global_rank end_idx = begin_idx + (len(text_tar_filepaths) // world_size) logging.info('Begin Index : %d' % (begin_idx)) @@ -507,16 +518,17 @@ def __init__( logging.info( "Partitioning tarred dataset: process (%d) taking shards [%d, %d)", global_rank, begin_idx, end_idx ) + self.length = batches_per_tar * len(text_tar_filepaths) * world_size elif shard_strategy == 'replicate': logging.info("All tarred dataset shards will be replicated across all nodes.") + self.length = num_batches else: raise ValueError(f"Invalid shard strategy! Allowed values are: {valid_shard_strategies}") # Put together WebDataset self._dataset = wd.WebDataset(urls=text_tar_filepaths, nodesplitter=None) - self.length = num_batches // world_size if shuffle_n > 0: self._dataset = self._dataset.shuffle(shuffle_n) else: diff --git a/nemo/collections/nlp/data/token_classification/punctuation_capitalization_dataset.py b/nemo/collections/nlp/data/token_classification/punctuation_capitalization_dataset.py index 493624de3f65..d38fdb45bbfa 100644 --- a/nemo/collections/nlp/data/token_classification/punctuation_capitalization_dataset.py +++ b/nemo/collections/nlp/data/token_classification/punctuation_capitalization_dataset.py @@ -85,7 +85,7 @@ class PunctuationCapitalizationDataConfigBase: labels_file: Optional[str] = None """A path to a file with punctuation and capitalization labels in NeMo format. NeMo format is described in - `documentation + `documentation `_ """ @@ -112,7 +112,7 @@ class PunctuationCapitalizationDataConfigBase: """A path to a directory containing cache or directory where newly created cache is saved. By default, it is a directory containing ``text_file``. You may need this parameter if cache for a dataset is going to be created and the dataset directory is read-only. - + ``cache_dir`` and ``label_info_save_dir`` are separate parameters for the case when a cache is ready and this cache is stored in a read only directory. In this case you will separate ``label_info_save_dir``.""" @@ -142,6 +142,22 @@ class PunctuationCapitalizationDataConfigBase: tar_shuffle_n: int = 1 """The size of shuffle buffer of `webdataset`. The number of batches which are permuted.""" + shard_strategy: Optional[str] = 'scatter' + """Tarred dataset shard distribution strategy chosen as a str value during ddp. Accepted values are `scatter` and `replicate`. + `scatter`: The default shard strategy applied by WebDataset, where each node gets a unique set of shards, which are permanently + pre-allocated and never changed at runtime. `replicate` is an optional shard strategy, where each node gets the entire set of shards + available in the tarred dataset, which are permanently pre-allocated and never changed at runtime. The benefit of replication is that + it allows each node to sample data points from the entire dataset independently of other nodes, and reduces dependence on value of + ``tar_shuffle_n``. + + .. warning:: + Replicated strategy allows every node to sample the entire set of available tarfiles, and therefore more than one node may sample + the same tarfile, and even sample the same data points! As such, there is no assured guarantee that all samples in the dataset + will be sampled at least once during 1 epoch. Scattered strategy, on the other hand, on specific occasions (when the number of + shards is not divisible with ``world_size``), will not sample the entire dataset. For these reasons it is not advisable to use + tarred datasets as validation or test datasets. + """ + ################################################# # PYTORCH DATALOADER PARAMETERS ################################################# diff --git a/nemo/collections/nlp/data/token_classification/punctuation_capitalization_tarred_dataset.py b/nemo/collections/nlp/data/token_classification/punctuation_capitalization_tarred_dataset.py index 2bfcb7969b6e..da63b20dc560 100644 --- a/nemo/collections/nlp/data/token_classification/punctuation_capitalization_tarred_dataset.py +++ b/nemo/collections/nlp/data/token_classification/punctuation_capitalization_tarred_dataset.py @@ -536,7 +536,7 @@ def repack_tar_files_with_not_enough_batches(output_dir: Path, num_batches_per_t ``repack_tar_files_with_not_enough_batches`` function into tar files with correct ``num_batches_per_tarfile`` batches each. If there is no enough batches in repacked files, then up to ``num_batches_per_tarfile - 1`` remaining batches may be discarded. - + Args: output_dir: a path to the output directory which contains files to repack and where new files are saved num_batches_per_tarfile: a number of batches in 1 tar file. If number of batches in files matching a pattern @@ -685,10 +685,10 @@ def create_tarred_dataset( `examples/nlp/token_classification/data/create_punctuation_capitalization_tarred_dataset.py `_. - Tarred dataset is a directory which contains metadata file, tar files with batches, + Tarred dataset is a directory which contains metadata file, tar files with batches, ``punct_label_vocab.csv`` and ``capit_label_vocab.csv`` files. - Metadata file is a JSON file with 4 items: ``'num_batches'``, ``'tar_files'``, ``'punct_label_vocab_file'``, + Metadata file is a JSON file with 4 items: ``'num_batches'``, ``'tar_files'``, ``'punct_label_vocab_file'``, ``'capit_label_vocab_file'``. The item ``'num_batches'`` (``int``) is a total number of batches in tarred dataset. ``'tar_files'`` is a list of paths to tar files relative to directory containing the metadata file. The items ``'punct_label_vocab_file'`` and ``'capit_label_vocab_file'`` are correspondingly paths to punctuation and @@ -871,6 +871,23 @@ class BertPunctuationCapitalizationTarredDataset(IterableDataset): be used in the current process. shuffle_n (:obj:`int`, `optional`, defaults to :obj:`1`): a number of shuffled batches in a buffer. ``shuffle_n`` batches are loaded into memory, shuffled, and then yielded by a dataset instance. + shard_strategy (:obj:`str`, defaults to :obj:``'scatter'``): Tarred dataset shard distribution strategy chosen as + a str value during ddp. + - ``'scatter'``: The default shard strategy applied by WebDataset, where each node gets + a unique set of shards, which are permanently pre-allocated and never changed at runtime. + - ``'replicate'``: Optional shard strategy, where each node gets all of the set of shards + available in the tarred dataset, which are permanently pre-allocated and never changed at runtime. + The benefit of replication is that it allows each node to sample data points from the entire + dataset independently of other nodes, and reduces dependence on value of :param:`shuffle_n`. + + .. warning:: + Replicated strategy allows every node to sample the entire set of available tarfiles, + and therefore more than one node may sample the same tarfile, and even sample the same + data points! As such, there is no assured guarantee that all samples in the dataset will be + sampled at least once during 1 epoch. Scattered strategy, on the other hand, on specific + occasions (when the number of shards is not divisible with ``world_size``), will not sample + the entire dataset. For these reasons it is not advisable to use tarred datasets as validation + or test datasets. """ @property @@ -897,8 +914,18 @@ def __init__( world_size: int = 1, global_rank: int = 0, shuffle_n: int = 1, + shard_strategy: str = "scatter", ) -> None: super().__init__() + + valid_shard_strategies = ['scatter', 'replicate'] + if shard_strategy not in valid_shard_strategies: + raise ValueError( + f"Invalid shard strategy of type {type(shard_strategy)} " + f"{repr(shard_strategy) if len(repr(shard_strategy)) < 100 else repr(shard_strategy)[:100] + '...'}! " + f"Allowed values are: {valid_shard_strategies}." + ) + self.tokenizer = tokenizer self.metadata_file = Path(metadata_file).expanduser() if label_info_save_dir is None: @@ -922,13 +949,31 @@ def __init__( self.capit_label_ids = load_label_ids(self.capit_label_vocab_file) self.pad_label = pad_label self._check_pad_label() - begin_idx = (len(self.tar_files) // world_size) * global_rank - end_idx = begin_idx + (len(self.tar_files) // world_size) - logging.info( - "Partitioning tarred dataset: process (%d) taking shards [%d, %d)", global_rank, begin_idx, end_idx - ) - self.tar_files = self.tar_files[begin_idx:end_idx] - self.length = self.metadata['num_batches'] // world_size + + if shard_strategy == 'scatter': + logging.info("Tarred dataset shards will be scattered evenly across all nodes.") + if len(self.tar_files) % world_size != 0: + logging.warning( + f"Number of shards in tarred dataset ({len(self.tar_files)}) is not divisible " + f"by number of distributed workers ({world_size}). " + f"Some shards will not be used ({len(self.tar_files) % world_size})." + ) + begin_idx = (len(self.tar_files) // world_size) * global_rank + end_idx = begin_idx + (len(self.tar_files) // world_size) + logging.info( + "Partitioning tarred dataset: process (%d) taking shards [%d, %d)", global_rank, begin_idx, end_idx + ) + batches_per_tar = self.metadata['num_batches'] // len(self.tar_files) + self.tar_files = self.tar_files[begin_idx:end_idx] + self.length = batches_per_tar * len(self.tar_files) * world_size + + elif shard_strategy == 'replicate': + logging.info("All tarred dataset shards will be replicated across all nodes.") + self.length = self.metadata['num_batches'] + + else: + raise ValueError(f"Invalid shard strategy! Allowed values are: {valid_shard_strategies}") + self._dataset = wds.WebDataset(urls=self.tar_files, nodesplitter=None).decode( wds.handle_extension('.pyd', decode_pyd) ) diff --git a/nemo/collections/nlp/models/duplex_text_normalization/duplex_decoder.py b/nemo/collections/nlp/models/duplex_text_normalization/duplex_decoder.py index 4f602f90da8b..440640231664 100644 --- a/nemo/collections/nlp/models/duplex_text_normalization/duplex_decoder.py +++ b/nemo/collections/nlp/models/duplex_text_normalization/duplex_decoder.py @@ -15,6 +15,7 @@ import json import os from collections import defaultdict +from math import ceil from typing import Dict, List, Optional, Union import torch @@ -398,6 +399,23 @@ def setup_training_data(self, train_data_config: Optional[DictConfig]): cfg=train_data_config, data_split="train" ) + # Need to set this because if using an IterableDataset, the length of the dataloader is the total number + # of samples rather than the number of batches, and this messes up the tqdm progress bar. + # So we set the number of steps manually (to the correct number) to fix this. + if 'use_tarred_dataset' in train_data_config and train_data_config['use_tarred_dataset']: + # We also need to check if limit_train_batches is already set. + # If it's an int, we assume that the user has set it to something sane, i.e. <= # training batches, + # and don't change it. Otherwise, adjust batches accordingly if it's a float (including 1.0). + if self._trainer is not None and isinstance(self._trainer.limit_train_batches, float): + self._trainer.limit_train_batches = int( + self._trainer.limit_train_batches * ceil(len(self._train_dl.dataset) / self.world_size) + ) + elif self._trainer is None: + logging.warning( + "Model Trainer was not set before constructing the dataset, incorrect number of " + "training batches will be used. Please set the trainer and rebuild the dataset." + ) + def setup_validation_data(self, val_data_config: Optional[DictConfig]): if not val_data_config or not val_data_config.data_path: logging.info( @@ -409,6 +427,23 @@ def setup_validation_data(self, val_data_config: Optional[DictConfig]): cfg=val_data_config, data_split="val" ) + # Need to set this because if using an IterableDataset, the length of the dataloader is the total number + # of samples rather than the number of batches, and this messes up the tqdm progress bar. + # So we set the number of steps manually (to the correct number) to fix this. + if 'use_tarred_dataset' in val_data_config and val_data_config['use_tarred_dataset']: + # We also need to check if limit_val_batches is already set. + # If it's an int, we assume that the user has set it to something sane, i.e. <= # validation batches, + # and don't change it. Otherwise, adjust batches accordingly if it's a float (including 1.0). + if self._trainer is not None and isinstance(self._trainer.limit_val_batches, float): + self._trainer.limit_val_batches = int( + self._trainer.limit_val_batches * ceil(len(self._validation_dl.dataset) / self.world_size) + ) + elif self._trainer is None: + logging.warning( + "Model Trainer was not set before constructing the dataset, incorrect number of " + "validation batches will be used. Please set the trainer and rebuild the dataset." + ) + def setup_multiple_validation_data(self, val_data_config: Union[DictConfig, Dict] = None): if val_data_config is None: val_data_config = self._cfg.validation_ds diff --git a/nemo/collections/nlp/models/language_modeling/transformer_lm_model.py b/nemo/collections/nlp/models/language_modeling/transformer_lm_model.py index e7c13e539dd0..180fdad9ddc3 100644 --- a/nemo/collections/nlp/models/language_modeling/transformer_lm_model.py +++ b/nemo/collections/nlp/models/language_modeling/transformer_lm_model.py @@ -211,9 +211,43 @@ def setup_tokenizer( def setup_training_data(self, train_data_config: Optional[DictConfig]): self._train_dl = self._setup_dataloader_from_config(cfg=train_data_config) + # Need to set this because if using an IterableDataset, the length of the dataloader is the total number + # of samples rather than the number of batches, and this messes up the tqdm progress bar. + # So we set the number of steps manually (to the correct number) to fix this. + if 'use_tarred_dataset' in train_data_config and train_data_config['use_tarred_dataset']: + # We also need to check if limit_train_batches is already set. + # If it's an int, we assume that the user has set it to something sane, i.e. <= # training batches, + # and don't change it. Otherwise, adjust batches accordingly if it's a float (including 1.0). + if self._trainer is not None and isinstance(self._trainer.limit_train_batches, float): + self._trainer.limit_train_batches = int( + self._trainer.limit_train_batches * math.ceil(len(self._train_dl.dataset) / self.world_size) + ) + elif self._trainer is None: + logging.warning( + "Model Trainer was not set before constructing the dataset, incorrect number of " + "training batches will be used. Please set the trainer and rebuild the dataset." + ) + def setup_validation_data(self, val_data_config: Optional[DictConfig]): self._validation_dl = self._setup_dataloader_from_config(cfg=val_data_config) + # Need to set this because if using an IterableDataset, the length of the dataloader is the total number + # of samples rather than the number of batches, and this messes up the tqdm progress bar. + # So we set the number of steps manually (to the correct number) to fix this. + if 'use_tarred_dataset' in val_data_config and val_data_config['use_tarred_dataset']: + # We also need to check if limit_val_batches is already set. + # If it's an int, we assume that the user has set it to something sane, i.e. <= # validation batches, + # and don't change it. Otherwise, adjust batches accordingly if it's a float (including 1.0). + if self._trainer is not None and isinstance(self._trainer.limit_val_batches, float): + self._trainer.limit_val_batches = int( + self._trainer.limit_val_batches * math.ceil(len(self._validation_dl.dataset) / self.world_size) + ) + elif self._trainer is None: + logging.warning( + "Model Trainer was not set before constructing the dataset, incorrect number of " + "validation batches will be used. Please set the trainer and rebuild the dataset." + ) + def setup_test_data(self, test_data_config: Optional[DictConfig]): self._test_dl = self._setup_dataloader_from_config(cfg=test_data_config) diff --git a/nemo/collections/nlp/models/machine_translation/mt_enc_dec_model.py b/nemo/collections/nlp/models/machine_translation/mt_enc_dec_model.py index 7c27aeb1bb20..ebdcdecdab2b 100644 --- a/nemo/collections/nlp/models/machine_translation/mt_enc_dec_model.py +++ b/nemo/collections/nlp/models/machine_translation/mt_enc_dec_model.py @@ -16,6 +16,7 @@ import json import os import random +from math import ceil from pathlib import Path from typing import Dict, List, Optional, Union @@ -609,6 +610,23 @@ def setup_training_data(self, train_data_config: Optional[DictConfig]): ) self._train_dl = MTEncDecModel._setup_dataloader_from_config(cfg=train_data_config, dataset=self._train_ds,) + # Need to set this because if using an IterableDataset, the length of the dataloader is the total number + # of samples rather than the number of batches, and this messes up the tqdm progress bar. + # So we set the number of steps manually (to the correct number) to fix this. + if 'use_tarred_dataset' in train_data_config and train_data_config['use_tarred_dataset']: + # We also need to check if limit_train_batches is already set. + # If it's an int, we assume that the user has set it to something sane, i.e. <= # training batches, + # and don't change it. Otherwise, adjust batches accordingly if it's a float (including 1.0). + if self._trainer is not None and isinstance(self._trainer.limit_train_batches, float): + self._trainer.limit_train_batches = int( + self._trainer.limit_train_batches * ceil(len(self._train_dl.dataset) / self.world_size) + ) + elif self._trainer is None: + logging.warning( + "Model Trainer was not set before constructing the dataset, incorrect number of " + "training batches will be used. Please set the trainer and rebuild the dataset." + ) + def setup_multiple_validation_data(self, val_data_config: Union[DictConfig, Dict]): self.setup_validation_data(val_data_config) @@ -626,6 +644,24 @@ def setup_validation_data(self, val_data_config: Optional[DictConfig]): self._validation_dl = MTEncDecModel._setup_eval_dataloader_from_config( cfg=val_data_config, datasets=self._validation_ds ) + + # Need to set this because if using an IterableDataset, the length of the dataloader is the total number + # of samples rather than the number of batches, and this messes up the tqdm progress bar. + # So we set the number of steps manually (to the correct number) to fix this. + if 'use_tarred_dataset' in val_data_config and val_data_config['use_tarred_dataset']: + # We also need to check if limit_val_batches is already set. + # If it's an int, we assume that the user has set it to something sane, i.e. <= # validation batches, + # and don't change it. Otherwise, adjust batches accordingly if it's a float (including 1.0). + if self._trainer is not None and isinstance(self._trainer.limit_val_batches, float): + self._trainer.limit_val_batches = int( + self._trainer.limit_val_batches * ceil(len(self._validation_dl.dataset) / self.world_size) + ) + elif self._trainer is None: + logging.warning( + "Model Trainer was not set before constructing the dataset, incorrect number of " + "validation batches will be used. Please set the trainer and rebuild the dataset." + ) + # instantiate Torchmetric for each val dataloader if self._validation_dl is not None: for dataloader_idx in range(len(self._validation_dl)): diff --git a/nemo/collections/nlp/models/token_classification/punctuation_capitalization_model.py b/nemo/collections/nlp/models/token_classification/punctuation_capitalization_model.py index 5f6fa7f6164f..d8d07fcee87d 100644 --- a/nemo/collections/nlp/models/token_classification/punctuation_capitalization_model.py +++ b/nemo/collections/nlp/models/token_classification/punctuation_capitalization_model.py @@ -468,6 +468,24 @@ def setup_training_data(self, train_data_config: Optional[Union[Dict[str, Any], train_data_config = self._cfg.train_ds self._train_dl = self._setup_dataloader_from_config(cfg=train_data_config, train=True) + + # Need to set this because if using an IterableDataset, the length of the dataloader is the total number + # of samples rather than the number of batches, and this messes up the tqdm progress bar. + # So we set the number of steps manually (to the correct number) to fix this. + if 'use_tarred_dataset' in train_data_config and train_data_config['use_tarred_dataset']: + # We also need to check if limit_train_batches is already set. + # If it's an int, we assume that the user has set it to something sane, i.e. <= # training batches, + # and don't change it. Otherwise, adjust batches accordingly if it's a float (including 1.0). + if self._trainer is not None and isinstance(self._trainer.limit_train_batches, float): + self._trainer.limit_train_batches = int( + self._trainer.limit_train_batches * ceil(len(self._train_dl.dataset) / self.world_size) + ) + elif self._trainer is None: + logging.warning( + "Model Trainer was not set before constructing the dataset, incorrect number of " + "training batches will be used. Please set the trainer and rebuild the dataset." + ) + self.punct_label_ids = self._train_dl.dataset.punct_label_ids.copy() self.capit_label_ids = self._train_dl.dataset.capit_label_ids.copy() self.label_ids_are_set = True @@ -540,6 +558,24 @@ def setup_validation_data(self, val_data_config: Optional[Union[Dict[str, Any], val_data_config = self._cfg.validation_ds self._validation_dl = self._setup_dataloader_from_config(cfg=val_data_config, train=False) + + # Need to set this because if using an IterableDataset, the length of the dataloader is the total number + # of samples rather than the number of batches, and this messes up the tqdm progress bar. + # So we set the number of steps manually (to the correct number) to fix this. + if 'use_tarred_dataset' in val_data_config and val_data_config['use_tarred_dataset']: + # We also need to check if limit_val_batches is already set. + # If it's an int, we assume that the user has set it to something sane, i.e. <= # validation batches, + # and don't change it. Otherwise, adjust batches accordingly if it's a float (including 1.0). + if self._trainer is not None and isinstance(self._trainer.limit_val_batches, float): + self._trainer.limit_val_batches = int( + self._trainer.limit_val_batches * ceil(len(self._validation_dl.dataset) / self.world_size) + ) + elif self._trainer is None: + logging.warning( + "Model Trainer was not set before constructing the dataset, incorrect number of " + "validation batches will be used. Please set the trainer and rebuild the dataset." + ) + loss_kw, punct_kw, capit_kw = self._get_eval_metrics_kwargs() self.metrics['val']['loss'].append(GlobalAverageLossMetric(**loss_kw)) self.metrics['val']['punct_class_report'].append(ClassificationReport(**punct_kw)) @@ -741,6 +777,7 @@ def _setup_dataloader_from_config(self, cfg: DictConfig, train: bool) -> torch.u world_size=self.world_size, global_rank=self.global_rank, shuffle_n=cfg.tar_shuffle_n, + shard_strategy=cfg.shard_strategy, label_info_save_dir=cfg.label_info_save_dir, ) dataset.check_for_label_consistency_with_model_config( From 5686fe20dca28ce055986a2c49c9a1f6d50cf6a3 Mon Sep 17 00:00:00 2001 From: Xuesong Yang <1646669+XuesongYang@users.noreply.github.com> Date: Tue, 26 Jul 2022 10:19:58 -0700 Subject: [PATCH 44/52] [TTS][ASR] customize arguments for trimming the leading/trailing silence (#4582) [TTS][ASR] enabled overriding arguments for trimming the leading and trailing silence using librosa.effects.trim Signed-off-by: Xuesong Yang <1646669+XuesongYang@users.noreply.github.com> --- .../asr/parts/preprocessing/features.py | 18 ++++++- .../asr/parts/preprocessing/segment.py | 49 +++++++++++++++++-- nemo/collections/tts/torch/data.py | 26 +++++++++- 3 files changed, 86 insertions(+), 7 deletions(-) diff --git a/nemo/collections/asr/parts/preprocessing/features.py b/nemo/collections/asr/parts/preprocessing/features.py index cd7258cea06a..dd306e07f9ba 100644 --- a/nemo/collections/asr/parts/preprocessing/features.py +++ b/nemo/collections/asr/parts/preprocessing/features.py @@ -36,6 +36,7 @@ import random import librosa +import numpy as np import torch import torch.nn as nn @@ -100,7 +101,18 @@ def __init__(self, sample_rate=16000, int_values=False, augmentor=None): def max_augmentation_length(self, length): return self.augmentor.max_augmentation_length(length) - def process(self, file_path, offset=0, duration=0, trim=False, orig_sr=None): + def process( + self, + file_path, + offset=0, + duration=0, + trim=False, + trim_ref=np.max, + trim_top_db=60, + trim_frame_length=2048, + trim_hop_length=512, + orig_sr=None, + ): audio = AudioSegment.from_file( file_path, target_sr=self.sample_rate, @@ -108,6 +120,10 @@ def process(self, file_path, offset=0, duration=0, trim=False, orig_sr=None): offset=offset, duration=duration, trim=trim, + trim_ref=trim_ref, + trim_top_db=trim_top_db, + trim_frame_length=trim_frame_length, + trim_hop_length=trim_hop_length, orig_sr=orig_sr, ) return self.process_segment(audio) diff --git a/nemo/collections/asr/parts/preprocessing/segment.py b/nemo/collections/asr/parts/preprocessing/segment.py index af0589309399..88013f3d6227 100644 --- a/nemo/collections/asr/parts/preprocessing/segment.py +++ b/nemo/collections/asr/parts/preprocessing/segment.py @@ -64,7 +64,18 @@ class AudioSegment(object): :raises TypeError: If the sample data type is not float or int. """ - def __init__(self, samples, sample_rate, target_sr=None, trim=False, trim_db=60, orig_sr=None): + def __init__( + self, + samples, + sample_rate, + target_sr=None, + trim=False, + trim_ref=np.max, + trim_top_db=60, + trim_frame_length=2048, + trim_hop_length=512, + orig_sr=None, + ): """Create audio segment from samples. Samples are convert float32 internally, with int scaled to [-1, 1]. """ @@ -73,7 +84,9 @@ def __init__(self, samples, sample_rate, target_sr=None, trim=False, trim_db=60, samples = librosa.core.resample(samples, orig_sr=sample_rate, target_sr=target_sr) sample_rate = target_sr if trim: - samples, _ = librosa.effects.trim(samples, top_db=trim_db) + samples, _ = librosa.effects.trim( + samples, top_db=trim_top_db, ref=trim_ref, frame_length=trim_frame_length, hop_length=trim_hop_length + ) self._samples = samples self._sample_rate = sample_rate if self._samples.ndim >= 2: @@ -125,7 +138,18 @@ def _convert_samples_to_float32(samples): @classmethod def from_file( - cls, audio_file, target_sr=None, int_values=False, offset=0, duration=0, trim=False, orig_sr=None, + cls, + audio_file, + target_sr=None, + int_values=False, + offset=0, + duration=0, + trim=False, + trim_ref=np.max, + trim_top_db=60, + trim_frame_length=2048, + trim_hop_length=512, + orig_sr=None, ): """ Load a file supported by librosa and return as an AudioSegment. @@ -134,6 +158,13 @@ def from_file( :param int_values: if true, load samples as 32-bit integers :param offset: offset in seconds when loading audio :param duration: duration in seconds when loading audio + :param trim: if true, trim leading and trailing silence from an audio signal + :param trim_ref: the reference amplitude. By default, it uses `np.max` and compares to the peak amplitude in + the signal + :param trim_top_db: the threshold (in decibels) below reference to consider as silence + :param trim_frame_length: the number of samples per analysis frame + :param trim_hop_length: the number of samples between analysis frames + :param orig_sr: the original sample rate :return: numpy array of samples """ samples = None @@ -174,7 +205,17 @@ def from_file( libs = "soundfile, and pydub" if HAVE_PYDUB else "soundfile" raise Exception(f"Your audio file {audio_file} could not be decoded. We tried using {libs}.") - return cls(samples, sample_rate, target_sr=target_sr, trim=trim, orig_sr=orig_sr) + return cls( + samples, + sample_rate, + target_sr=target_sr, + trim=trim, + trim_ref=trim_ref, + trim_top_db=trim_top_db, + trim_frame_length=trim_frame_length, + trim_hop_length=trim_hop_length, + orig_sr=orig_sr, + ) @classmethod def segment_from_file(cls, audio_file, target_sr=None, n_segments=0, trim=False, orig_sr=None): diff --git a/nemo/collections/tts/torch/data.py b/nemo/collections/tts/torch/data.py index d308b5013f09..74c0e2c6505f 100644 --- a/nemo/collections/tts/torch/data.py +++ b/nemo/collections/tts/torch/data.py @@ -79,6 +79,10 @@ def __init__( min_duration: Optional[float] = None, ignore_file: Optional[Union[str, Path]] = None, trim: bool = False, + trim_ref: Optional[float] = None, + trim_top_db: Optional[int] = None, + trim_frame_length: Optional[int] = None, + trim_hop_length: Optional[int] = None, n_fft: int = 1024, win_length: Optional[int] = None, hop_length: Optional[int] = None, @@ -119,7 +123,14 @@ def __init__( audio to compute duration. Defaults to None which does not prune. ignore_file (Optional[Union[str, Path]]): The location of a pickle-saved list of audio paths that will be pruned prior to training. Defaults to None which does not prune. - trim (Optional[bool]): Whether to apply librosa.effects.trim to the audio file. Defaults to False. + trim (bool): Whether to apply `librosa.effects.trim` to trim leading and trailing silence from an audio + signal. Defaults to False. + trim_ref (Optional[float]): the reference amplitude. By default, it uses `np.max` and compares to the peak + amplitude in the signal. + trim_top_db (Optional[int]): the threshold (in decibels) below reference to consider as silence. + Defaults to 60. + trim_frame_length (Optional[int]): the number of samples per analysis frame. Defaults to 2048. + trim_hop_length (Optional[int]): the number of samples between analysis frames. Defaults to 512. n_fft (int): The number of fft samples. Defaults to 1024 win_length (Optional[int]): The length of the stft windows. Defaults to None which uses n_fft. hop_length (Optional[int]): The hope length between fft computations. Defaults to None which uses n_fft//4. @@ -229,6 +240,10 @@ def __init__( self.sample_rate = sample_rate self.featurizer = WaveformFeaturizer(sample_rate=self.sample_rate) self.trim = trim + self.trim_ref = trim_ref if trim_ref is not None else np.max + self.trim_top_db = trim_top_db if trim_top_db is not None else 60 + self.trim_frame_length = trim_frame_length if trim_frame_length is not None else 2048 + self.trim_hop_length = trim_hop_length if trim_hop_length is not None else 512 self.n_fft = n_fft self.n_mels = n_mels @@ -438,7 +453,14 @@ def __getitem__(self, index): rel_audio_path_as_text_id += "_phoneme" # Load audio - features = self.featurizer.process(sample["audio_filepath"], trim=self.trim) + features = self.featurizer.process( + sample["audio_filepath"], + trim=self.trim, + trim_ref=self.trim_ref, + trim_top_db=self.trim_top_db, + trim_frame_length=self.trim_frame_length, + trim_hop_length=self.trim_hop_length, + ) audio, audio_length = features, torch.tensor(features.shape[0]).long() if "text_tokens" in sample: From 793cf4850c02b0aa2dd8304928620c6fadabd245 Mon Sep 17 00:00:00 2001 From: Shantanu Acharya Date: Tue, 26 Jul 2022 14:54:54 -0400 Subject: [PATCH 45/52] Updating the default parameters in the example adapters config file (#4607) * auto switch conformer encoder adapter in_features Signed-off-by: Shantanu Acharya * update the norm and warmup default values in the adapters config file Signed-off-by: Shantanu Acharya Co-authored-by: Somshubra Majumdar --- examples/asr/conf/asr_adapters/asr_adaptation.yaml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/examples/asr/conf/asr_adapters/asr_adaptation.yaml b/examples/asr/conf/asr_adapters/asr_adaptation.yaml index b1519241e5fa..7584e2220d10 100644 --- a/examples/asr/conf/asr_adapters/asr_adaptation.yaml +++ b/examples/asr/conf/asr_adapters/asr_adaptation.yaml @@ -70,7 +70,7 @@ model: in_features: ??? # User must provide the output dimension of the layers of the model, which is the input dimension of this adapter. dim: 32 # The hidden dimension of the adapter, as chosen by user, but small values are preferred to reduce param count. activation: swish - norm_position: 'post' # Can be `pre` or `post` + norm_position: 'pre' # Can be `pre` or `post` dropout: 0.0 # float, dropout for the adapter # Adapter strategy config @@ -147,8 +147,8 @@ model: name: CosineAnnealing # scheduler config override - warmup_steps: 100 # Warmup steps should be set, and smaller than the trainer.max_steps set below. - warmup_ratio: null + warmup_steps: null # Warmup steps should be set, and smaller than the trainer.max_steps set below. + warmup_ratio: 0.1 # Warmup steps will be 10% of the training steps. min_lr: 1e-5 last_epoch: -1 @@ -192,6 +192,8 @@ exp_manager: project: null entity: null save_dir: null + offline: false # If true, wandb logging will be done offline and would require manual syncing. + tags: null # List of tags to assign to the run resume_if_exists: false resume_ignore_no_checkpoint: false From f1bf6c2731c6d4881b0063aa26564d9cd8300c4c Mon Sep 17 00:00:00 2001 From: Eric Harper Date: Tue, 26 Jul 2022 13:58:03 -0600 Subject: [PATCH 46/52] NeMo Megatron: Add sequence parallelism and selective activation checkpointing (rebased) (#4380) * update gpt config and add docstring to parallel_lm_logits Signed-off-by: ericharper * update parallel_lm_logits Signed-off-by: ericharper * add CoreAttention and start updating ParallelAttention Signed-off-by: ericharper * in progress Signed-off-by: ericharper * fix args Signed-off-by: ericharper * update ParallelTransformerLayer_ Signed-off-by: ericharper * update ParallelTransformer Signed-off-by: ericharper * remove test_from_pretrained Signed-off-by: ericharper * update args Signed-off-by: ericharper * propogate args Signed-off-by: ericharper * add transposes to GPTModel Signed-off-by: ericharper * update matmul_input_buffer dynamically Signed-off-by: ericharper * add sequence_parallel arg to post_language_model_processing Signed-off-by: ericharper * allreduce sequence parallel layernorm Signed-off-by: ericharper * typo Signed-off-by: ericharper * flag is sequence_parallel_enabled Signed-off-by: ericharper * add sequence parallel args Signed-off-by: ericharper * add seq parallel arg to fwd/bwd func Signed-off-by: ericharper * don't wrap model with ddp when using O2 Signed-off-by: ericharper * raise error when using method or num_layers with selective Signed-off-by: ericharper * add sequence parallel for MixedFusedLayerNorm Signed-off-by: ericharper * make sure checkpointing is set correctly Signed-off-by: ericharper * make sure checkpointing is set correctly Signed-off-by: ericharper * make sure checkpointing is set correctly Signed-off-by: ericharper * check the right attribute Signed-off-by: ericharper * fix args Signed-off-by: ericharper * style Signed-off-by: ericharper * don't sync after fwd/bwd if using seq par Signed-off-by: ericharper * use base model for allreduce_grads Signed-off-by: ericharper * remove extra layernorm Signed-off-by: ericharper * revert norm_former_norm deletion Signed-off-by: ericharper * move sync for allreduce grad to optimizer wrapper Signed-off-by: ericharper * auto configure grad div ar fusion Signed-off-by: ericharper * Initial rpe refactor Signed-off-by: MaximumEntropy * Refactor RPE Signed-off-by: MaximumEntropy * add transposes to t5 forward Signed-off-by: ericharper * Gradient Accumulation fusion to Linear layer weight gradient computation (#4494) * Gradient Accumulation fusion to Linear layer weight gradient computation * fix typo * disable async when using gradient accumulation fusion Signed-off-by: ericharper * add comment Signed-off-by: ericharper * skip H2D copies of inputs (#4502) add assert * add rpe to core attention Signed-off-by: ericharper * set sp to false if tp > 1. make cuda_device_max_connections configurable Signed-off-by: ericharper * Refactor Signed-off-by: MaximumEntropy * Fix key Signed-off-by: MaximumEntropy * convert to string Signed-off-by: ericharper * revert Signed-off-by: ericharper * revert Signed-off-by: ericharper * determine if no_async for ColumnLinear Signed-off-by: ericharper * add async_grad_allreduce to parallel_lm_logits for gpt Signed-off-by: ericharper * add async_grad_allreduce to parallel_lm_logits for bert and t5 Signed-off-by: ericharper * disable gradient accumulation fusion when not using pipeline parallelism Signed-off-by: ericharper * style Signed-off-by: ericharper * Sequence parallel rebase with bugfixes (#4529) * Support for class labels as strings Signed-off-by: MaximumEntropy * Fix for micro/macro average Signed-off-by: MaximumEntropy * Metric fix Signed-off-by: MaximumEntropy * Fix geglu without fusion Signed-off-by: MaximumEntropy * style Signed-off-by: ericharper * disable grad accumulation fusion with O1 Signed-off-by: ericharper * transpose prompt learning encoder input Signed-off-by: ericharper * update container in jenkins Signed-off-by: ericharper * add pleasefixme to retrieval tests Signed-off-by: ericharper * comment retro test from jenkins Signed-off-by: ericharper * Make RETRO SP compatible (#4565) * fix sp for retro Signed-off-by: Yi Dong * more tests fixed Signed-off-by: Yi Dong * make it sp compatible Signed-off-by: Yi Dong * add transpose for bert Signed-off-by: ericharper * fix bug in pooler Signed-off-by: ericharper * remove unused import Signed-off-by: ericharper * remove unused import Signed-off-by: ericharper * fix output Signed-off-by: ericharper * revert normformer delete Signed-off-by: ericharper * use 22.07, comment broken jenkins test Signed-off-by: ericharper * remove unused import Signed-off-by: ericharper * revert test comment Signed-off-by: ericharper * add model.optim.capturable=True Signed-off-by: ericharper * add model.optim.capturable=True Signed-off-by: ericharper * set num_workers=0 Signed-off-by: ericharper Co-authored-by: MaximumEntropy Co-authored-by: Sangkug Lym Co-authored-by: Yi Dong <43824965+yidong72@users.noreply.github.com> --- Jenkinsfile | 15 +- .../conf/megatron_gpt_config.yaml | 24 +- .../conf/megatron_t5_config.yaml | 1 + .../conf/megatron_t5_finetune.yaml | 6 +- .../metrics/metric_string_to_torchmetric.py | 15 +- .../megatron/request_dataset.py | 4 - .../language_modeling/megatron/bert_model.py | 17 +- .../language_modeling/megatron/gpt_model.py | 40 +- .../language_modeling/megatron_base_model.py | 47 +- .../megatron_finetune_model.py | 54 +- .../language_modeling/megatron_gpt_model.py | 91 +- .../megatron_gpt_prompt_learning_model.py | 1 + .../megatron_lm_encoder_decoder_model.py | 23 +- .../common/megatron/fused_layer_norm.py | 8 +- .../modules/common/megatron/language_model.py | 75 +- .../common/megatron/megatron_decoders.py | 7 - .../megatron/megatron_encoder_decoder.py | 26 +- .../common/megatron/megatron_encoders.py | 6 - .../megatron/megatron_transformer_decoder.py | 18 +- .../megatron/megatron_transformer_encoder.py | 23 +- .../retrieval_token_level_encoder_decoder.py | 7 +- .../common/megatron/retrieval_transformer.py | 20 +- .../t5_relative_position_embedding.py | 152 +++ .../megatron/token_level_encoder_decoder.py | 106 ++- .../modules/common/megatron/transformer.py | 880 ++++++++++-------- .../nlp/modules/common/megatron/utils.py | 59 +- nemo/collections/nlp/parts/nlp_overrides.py | 53 +- nemo/core/optim/optimizer_with_main_params.py | 8 +- tests/collections/nlp/test_gpt_model.py | 2 + .../collections/nlp/test_retrieval_module.py | 2 +- .../nlp/test_retrieval_module_inference.py | 16 +- 31 files changed, 1201 insertions(+), 605 deletions(-) create mode 100644 nemo/collections/nlp/modules/common/megatron/t5_relative_position_embedding.py diff --git a/Jenkinsfile b/Jenkinsfile index 72f509e22790..e390f8181c51 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -1,7 +1,8 @@ pipeline { agent { docker { - image 'nvcr.io/nvidia/pytorch:22.05-py3' + //image 'nvcr.io/nvidia/pytorch:22.05-py3' + image 'gitlab-master.nvidia.com:5005/eharper/nemo_containers:megatron_gpt_v16' args '--device=/dev/nvidia0 --gpus all -e TRANSFORMERS_OFFLINE=1 --user 0:128 -v /home/TestData:/home/TestData -v $HOME/.cache:/root/.cache --shm-size=8g' } } @@ -1911,6 +1912,8 @@ pipeline { } } + // TODO: remove +model.optim.capturable=True when Pytorch fix: https://github.com/pytorch/pytorch/pull/81858 + // is in the release container stage('L2: NMT Attention is All You Need Training') { when { anyOf { @@ -1940,6 +1943,7 @@ pipeline { model.decoder.num_layers=1 \ model.decoder.hidden_size=64 \ model.decoder.inner_size=256 \ + +model.optim.capturable=True \ trainer.devices=[0] \ trainer.accelerator="gpu" \ +trainer.val_check_interval=2 \ @@ -1967,6 +1971,7 @@ pipeline { model.decoder.num_layers=1 \ model.decoder.hidden_size=64 \ model.decoder.inner_size=256 \ + +model.optim.capturable=True \ trainer.devices=[0] \ trainer.accelerator="gpu" \ +trainer.val_check_interval=10 \ @@ -3446,9 +3451,9 @@ pipeline { +trainer.limit_train_batches=1 +trainer.limit_val_batches=1 trainer.max_epochs=1 \ trainer.strategy=null \ model.train_ds.dataloader_params.batch_size=4 \ - model.train_ds.dataloader_params.num_workers=1 \ + model.train_ds.dataloader_params.num_workers=0 \ model.validation_ds.dataloader_params.batch_size=4 \ - model.validation_ds.dataloader_params.num_workers=1 \ + model.validation_ds.dataloader_params.num_workers=0 \ model.symbols_embedding_dim=64 \ model.input_fft.d_inner=384 \ model.input_fft.n_layer=2 \ @@ -3469,9 +3474,9 @@ pipeline { +trainer.limit_train_batches=1 +trainer.limit_val_batches=1 trainer.max_epochs=1 \ trainer.strategy=null \ model.train_ds.dataloader_params.batch_size=4 \ - model.train_ds.dataloader_params.num_workers=1 \ + model.train_ds.dataloader_params.num_workers=0 \ model.validation_ds.dataloader_params.batch_size=4 \ - model.validation_ds.dataloader_params.num_workers=1 \ + model.validation_ds.dataloader_params.num_workers=0 \ ~trainer.check_val_every_n_epoch \ ~model.text_normalizer \ ~model.text_normalizer_call_kwargs' diff --git a/examples/nlp/language_modeling/conf/megatron_gpt_config.yaml b/examples/nlp/language_modeling/conf/megatron_gpt_config.yaml index 19455a5d2c9c..ac3b9adbc96a 100755 --- a/examples/nlp/language_modeling/conf/megatron_gpt_config.yaml +++ b/examples/nlp/language_modeling/conf/megatron_gpt_config.yaml @@ -66,8 +66,6 @@ model: post_process: True # add pooler persist_layer_norm: True # Use of persistent fused layer norm kernel. - grad_div_ar_fusion: True # Fuse grad division into torch.distributed.all_reduce - tokenizer: library: 'megatron' type: 'GPT2BPETokenizer' @@ -86,6 +84,7 @@ model: # Megatron O2-style half-precision megatron_amp_O2: False # Enable O2-level automatic mixed precision using main parameters grad_allreduce_chunk_size_mb: 125 + grad_div_ar_fusion: True # Fuse grad division into torch.distributed.all_reduce # miscellaneous seed: 1234 @@ -93,9 +92,26 @@ model: onnx_safe: False # Use work-arounds for known problems with Torch ONNX exporter. apex_transformer_log_level: 30 # Python logging level displays logs with severity greater than or equal to this gradient_as_bucket_view: True # PyTorch DDP argument. Allocate gradients in a contiguous bucket to save memory (less fragmentation and buffer memory) + gradient_accumulation_fusion: False # Fuse weight gradient accumulation to GEMMs. Only used with pipeline parallelism. + + ## Activation Checkpointing + # NeMo Megatron supports 'selective' activation checkpointing where only the memory intensive part of attention is checkpointed. + # These memory intensive activations are also less compute intensive which makes activation checkpointing more efficient for LLMs (20B+). + # See Reducing Activation Recomputation in Large Transformer Models: https://arxiv.org/abs/2205.05198 for more details. + # 'full' will checkpoint the entire transformer layer. + activations_checkpoint_granularity: null # 'selective' or 'full' + activations_checkpoint_method: null # 'uniform', 'block', not used with 'selective' + # 'uniform' divides the total number of transformer layers and checkpoints the input activation + # of each chunk at the specified granularity + # 'block' checkpoints the specified number of layers per pipeline stage at the specified granularity + activations_checkpoint_num_layers: null # not used with 'selective' + # when using 'uniform' this creates groups of transformer layers to checkpoint. Usually set to 1. Increase to save more memory. + # when using 'block' this this will checkpoint the first activations_checkpoint_num_layers per pipeline stage. - activations_checkpoint_method: null # 'uniform', 'block' - activations_checkpoint_num_layers: 1 + ## Sequence Parallelism + # Makes tensor parallelism more memory efficient for LLMs (20B+) by parallelizing layer norms and dropout sequentially + # See Reducing Activation Recomputation in Large Transformer Models: https://arxiv.org/abs/2205.05198 for more details. + sequence_parallel: False data: # Path to data must be specified by the user. diff --git a/examples/nlp/language_modeling/conf/megatron_t5_config.yaml b/examples/nlp/language_modeling/conf/megatron_t5_config.yaml index 363b5afdbad7..f20f39fac4f7 100644 --- a/examples/nlp/language_modeling/conf/megatron_t5_config.yaml +++ b/examples/nlp/language_modeling/conf/megatron_t5_config.yaml @@ -67,6 +67,7 @@ model: position_embedding_type: 'learned_absolute' # Position embedding type. Options ['learned_absolute', 'relative'] relative_attention_num_buckets: 32 # Relative position number of buckets for computing the bias relative_attention_max_distance: 128 # max_distance to keep relative distance in the attention_num_buckets. + relative_position_bias_self_attention_only: True # Whether to only use relative position bias for self attention only. kv_channels: null # Projection weights dimension in multi-head attention. Set to hidden_size // num_attention_heads if null apply_query_key_layer_scaling: True # scale Q * K^T by 1 / layer-number. layernorm_epsilon: 1e-5 diff --git a/examples/nlp/language_modeling/conf/megatron_t5_finetune.yaml b/examples/nlp/language_modeling/conf/megatron_t5_finetune.yaml index 9328c8edac6b..91a9730637a3 100644 --- a/examples/nlp/language_modeling/conf/megatron_t5_finetune.yaml +++ b/examples/nlp/language_modeling/conf/megatron_t5_finetune.yaml @@ -78,8 +78,10 @@ model: output_file_path_prefix: null # Prefix of the file to write predictions to. metric: name: "exact_string_match" # Name of the evaluation metric to use. - average: null # Average the metric over the dataset. Options: ['macro', 'micro']. Works only for 'F1', 'accuracy' etc. Refer to torchmetrics for metrics where this is supported. - num_classes: null + average: micro # Average the metric over the dataset. Options: ['macro', 'micro']. Works only for 'F1', 'accuracy' etc. Refer to torchmetrics for metrics where this is supported. + num_classes: null # Number of classes for the metric. Works only for 'F1', 'accuracy' and 'average_precision' etc. Refer to torchmetrics for metrics where this is supported. + class_labels: null # If the targets in your dataset are strings and not integers/float, you need to provide a list of class labels (size = num_classes) so we can convert from strings to integer categories to compute the metric. + labels_are_strings: True # NOTE: This is only required to properly handle metrics like f1, accuracy, average_precision etc. This does not affect extract_string_match. optim: name: fused_adam diff --git a/nemo/collections/common/metrics/metric_string_to_torchmetric.py b/nemo/collections/common/metrics/metric_string_to_torchmetric.py index 8f963ad89fc5..e83d88057143 100644 --- a/nemo/collections/common/metrics/metric_string_to_torchmetric.py +++ b/nemo/collections/common/metrics/metric_string_to_torchmetric.py @@ -12,17 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -from torchmetrics import ( - AUC, - AUROC, - Accuracy, - AveragePrecision, - F1Score, - MatthewsCorrCoef, - PearsonCorrCoef, - SpearmanCorrCoef, - SQuAD, -) +from torchmetrics import Accuracy, AveragePrecision, F1Score, MatthewsCorrCoef, PearsonCorrCoef, SpearmanCorrCoef from nemo.collections.common.metrics.classification_accuracy import ExactStringMatchMetric @@ -32,13 +22,10 @@ MetricStringToTorchMetric = { 'accuracy': Accuracy, - 'auc': AUC, - 'auroc': AUROC, 'average_precision': AveragePrecision, 'f1': F1Score, 'pearson_corr_coef': PearsonCorrCoef, 'spearman_corr_coef': SpearmanCorrCoef, 'matthews_corr_coef': MatthewsCorrCoef, 'exact_string_match': ExactStringMatchMetric, - 'squad': SQuAD, } diff --git a/nemo/collections/nlp/data/language_modeling/megatron/request_dataset.py b/nemo/collections/nlp/data/language_modeling/megatron/request_dataset.py index b7f156197b97..bf537d9748f0 100644 --- a/nemo/collections/nlp/data/language_modeling/megatron/request_dataset.py +++ b/nemo/collections/nlp/data/language_modeling/megatron/request_dataset.py @@ -89,10 +89,6 @@ def __init__(self, request: Dict, tokenizer) -> None: self.mask_prompt(self.request['prompt']) def mask_prompt(self, sample): - if '' not in sample: - if '' not in sample: - raise ValueError(f"Did not find any or tokens in prompt {sample}.") - sample = sample.split() sentinel_idx = 0 for i, word in enumerate(sample): diff --git a/nemo/collections/nlp/models/language_modeling/megatron/bert_model.py b/nemo/collections/nlp/models/language_modeling/megatron/bert_model.py index 46ca5c1b0206..bac33ce0c91e 100644 --- a/nemo/collections/nlp/models/language_modeling/megatron/bert_model.py +++ b/nemo/collections/nlp/models/language_modeling/megatron/bert_model.py @@ -93,14 +93,21 @@ def forward(self, hidden_states, word_embeddings_weight): hidden_states = self.dense(hidden_states) hidden_states = self.gelu(hidden_states) hidden_states = self.layernorm(hidden_states) - output = parallel_lm_logits(hidden_states, word_embeddings_weight, self.parallel_output, bias=self.bias) + async_tensor_model_parallel_allreduce = parallel_state.get_tensor_model_parallel_world_size() > 1 + output = parallel_lm_logits( + hidden_states, + word_embeddings_weight, + self.parallel_output, + bias=self.bias, + async_tensor_model_parallel_allreduce=async_tensor_model_parallel_allreduce, + ) return output def post_language_model_processing( lm_output, pooled_output, lm_head, binary_head, lm_labels, logit_weights, fp16_lm_cross_entropy ): - # Output. + # lm_logits: [s, b, vocab_size] lm_logits = lm_head(lm_output, logit_weights) binary_logits = None @@ -108,13 +115,19 @@ def post_language_model_processing( binary_logits = binary_head(pooled_output) if lm_labels is None: + # lm_logits: [s, b, vocab_size] -> [b, s, vocab_size] + lm_logits = lm_logits.transpose(0, 1).contiguous() return lm_logits, binary_logits else: + # lm_labels: [b, s] -> [s, b] + lm_labels = lm_labels.transpose(0, 1).contiguous() if fp16_lm_cross_entropy: assert lm_logits.dtype == torch.half lm_loss = tensor_parallel.vocab_parallel_cross_entropy(lm_logits, lm_labels) else: lm_loss = tensor_parallel.vocab_parallel_cross_entropy(lm_logits.float(), lm_labels) + # lm_loss: [s, b] -> [b, s] + lm_loss = lm_loss.transpose(0, 1).contiguous() return lm_loss, binary_logits diff --git a/nemo/collections/nlp/models/language_modeling/megatron/gpt_model.py b/nemo/collections/nlp/models/language_modeling/megatron/gpt_model.py index 225cdc3ec653..de98fcffbc7f 100755 --- a/nemo/collections/nlp/models/language_modeling/megatron/gpt_model.py +++ b/nemo/collections/nlp/models/language_modeling/megatron/gpt_model.py @@ -26,7 +26,7 @@ ) try: - from apex.transformer import tensor_parallel + from apex.transformer import tensor_parallel, parallel_state from apex.transformer.enums import AttnMaskType HAVE_APEX = True @@ -45,27 +45,46 @@ def post_language_model_processing( forward_method_parallel_output, fp16_lm_cross_entropy, return_logits=False, + sequence_parallel=False, + gradient_accumulation_fusion=False, ): if get_key_value: lm_output, presents = lm_output - # Output. + # Output. Format is [s b h] if forward_method_parallel_output is not None: parallel_output = forward_method_parallel_output - output = parallel_lm_logits(lm_output, logit_weights, parallel_output) + async_tensor_model_parallel_allreduce = ( + parallel_state.get_tensor_model_parallel_world_size() > 1 and not sequence_parallel + ) + output = parallel_lm_logits( + lm_output, + logit_weights, + parallel_output, + sequence_parallel=sequence_parallel, + gradient_accumulation_fusion=gradient_accumulation_fusion, + async_tensor_model_parallel_allreduce=async_tensor_model_parallel_allreduce, + ) if get_key_value: output = [output, presents] if labels is None: - return output + # [s b h] -> [b s h] + return output.transpose(0, 1).contiguous() else: + # [b s] -> [s b] + labels = labels.transpose(0, 1).contiguous() + if fp16_lm_cross_entropy: assert output.dtype == torch.half loss = tensor_parallel.vocab_parallel_cross_entropy(output, labels) else: loss = tensor_parallel.vocab_parallel_cross_entropy(output.float(), labels) + # [s b] -> [b, s] + loss = loss.transpose(0, 1).contiguous() + if return_logits: return loss, output else: @@ -95,6 +114,7 @@ def __init__( hidden_dropout=0.1, precision=16, fp32_residual_connection=False, + activations_checkpoint_granularity=None, activations_checkpoint_method=None, activations_checkpoint_num_layers=1, layernorm_epsilon=1e-5, @@ -102,6 +122,8 @@ def __init__( persist_layer_norm=False, openai_gelu=False, onnx_safe=False, + sequence_parallel=False, + gradient_accumulation_fusion=False, ): super(GPTModel, self).__init__() @@ -110,6 +132,8 @@ def __init__( self.pre_process = pre_process self.post_process = post_process self.fp16_lm_cross_entropy = fp16_lm_cross_entropy + self.sequence_parallel = sequence_parallel + self.gradient_accumulation_fusion = gradient_accumulation_fusion if kv_channels is None: assert ( @@ -138,6 +162,7 @@ def __init__( use_cpu_initialization=use_cpu_initialization, precision=precision, fp32_residual_connection=fp32_residual_connection, + activations_checkpoint_granularity=activations_checkpoint_granularity, activations_checkpoint_method=activations_checkpoint_method, activations_checkpoint_num_layers=activations_checkpoint_num_layers, layernorm_epsilon=layernorm_epsilon, @@ -145,6 +170,8 @@ def __init__( persist_layer_norm=persist_layer_norm, openai_gelu=openai_gelu, onnx_safe=onnx_safe, + sequence_parallel=sequence_parallel, + gradient_accumulation_fusion=gradient_accumulation_fusion, ) self.initialize_word_embeddings( @@ -169,6 +196,9 @@ def forward( set_inference_key_value_memory=False, inference_max_sequence_len=None, ): + # input_ids: [b, s] + # position_ids: [b, s] + # attention_mask: [1, 1, s, s] lm_output = self.language_model( input_ids, @@ -191,6 +221,8 @@ def forward( forward_method_parallel_output, self.fp16_lm_cross_entropy, return_logits=encoder_input is not None, + sequence_parallel=self.sequence_parallel, + gradient_accumulation_fusion=self.gradient_accumulation_fusion, ) else: return lm_output diff --git a/nemo/collections/nlp/models/language_modeling/megatron_base_model.py b/nemo/collections/nlp/models/language_modeling/megatron_base_model.py index 21ce06a6093f..32967699d7b1 100644 --- a/nemo/collections/nlp/models/language_modeling/megatron_base_model.py +++ b/nemo/collections/nlp/models/language_modeling/megatron_base_model.py @@ -16,6 +16,7 @@ import os import torch +from omegaconf import open_dict from omegaconf.dictconfig import DictConfig from pytorch_lightning.trainer.trainer import Trainer @@ -61,6 +62,8 @@ def __init__(self, cfg: DictConfig, trainer: Trainer, no_lm_init=True): super().__init__(cfg, trainer=trainer, no_lm_init=no_lm_init) + self._validate_config() + # used in NVIDIA NGC PyTorch containers self._enable_nvidia_optimizations() @@ -97,7 +100,7 @@ def __init__(self, cfg: DictConfig, trainer: Trainer, no_lm_init=True): def _enable_nvidia_optimizations(self): "These optimizations are present in NVIDIA NGC PyTorch Containers" - # Version check + # NVIDIA container version check nvidia_torch_version = os.getenv('NVIDIA_PYTORCH_VERSION', None) if nvidia_torch_version is not None: NVIDIA_TORCH_MAJOR = int(nvidia_torch_version.split('.')[0]) @@ -119,8 +122,9 @@ def _enable_nvidia_optimizations(self): torch._C._jit_set_texpr_fuser_enabled(False) torch._C._jit_set_nvfuser_enabled(True) torch._C._debug_set_autodiff_subgraph_inlining(False) + else: - # Not a Nvidia container. Dependency check is on users + # Not a Nvidia container. NVFUSER Dependency check is on users pass def _build_tokenizer(self): @@ -239,18 +243,28 @@ def configure_optimizers(self): "fp16 training is not yet supported with O2. Please set megatron_amp_O2 to False in the model config." ) - # if using tensor parallel only, we can use async grad all-reduce - if self.cfg.get('pipeline_model_parallel_size', 1) == 1: + # if using tensor parallel only, we automatically use async grad all-reduce + # if using pipeline parallel or sequence parallel or gradient accumulation fusion, then we disable it + if self.cfg.get('pipeline_model_parallel_size', 1) == 1 and not ( + self.cfg.get('sequence_parallel', False) or self.cfg.get('gradient_accumulation_fusion', False) + ): async_grad_allreduce = True else: async_grad_allreduce = False + if async_grad_allreduce: + # we need this to be configurable until make_nccl_premul_sum is in public PyTorch. + # currently cannot be imported in PyTorch 1.12.0 + grad_div_ar_fusion = self.cfg.get('grad_div_ar_fusion', False) + else: + grad_div_ar_fusion = False + self._optimizer = MainParamsOptimizerWrapper( self._optimizer, fp32_grad_accum=fp32_grad_accum, contiguous_grad_bucket=contiguous_grad_bucket, async_grad_allreduce=async_grad_allreduce, - grad_div_ar_fusion=self.cfg.get('grad_div_ar_fusion', True), + grad_div_ar_fusion=grad_div_ar_fusion, grad_allreduce_chunk_size_mb=self.cfg.get('grad_allreduce_chunk_size_mb', 125), ) @@ -266,3 +280,26 @@ def configure_optimizers(self): return self._optimizer else: return [self._optimizer], [self._scheduler] + + def _validate_config(self): + """ Certain configurations might be incompatible or discouraged. We can check for them here.""" + + if self.cfg.get('sequence_parallel', False) and self.cfg.get('tensor_model_parallel_size', 1) == 1: + logging.info( + "Sequence parallel should only be used with tensor parallel size > 1. Setting sequence parallel to False" + ) + with open_dict(self.cfg): + self.cfg.sequence_parallel = False + + if ( + self.cfg.get('gradient_accumulation_fusion', False) + and self.cfg.get('pipeline_model_parallel_size', 1) == 1 + ): + logging.info("Gradient accumulation fusion can only be used with pipeline parallel size > 1.") + with open_dict(self.cfg): + self.cfg.gradient_accumulation_fusion = False + + if self.cfg.get('gradient_accumulation_fusion', False) and not self.cfg.get('megatron_amp_O2', False): + logging.info("Gradient accumulation fusion can only be used with megatron amp O2 mixed precision.") + with open_dict(self.cfg): + self.cfg.gradient_accumulation_fusion = False diff --git a/nemo/collections/nlp/models/language_modeling/megatron_finetune_model.py b/nemo/collections/nlp/models/language_modeling/megatron_finetune_model.py index 0d29feafe606..ea5f627a294c 100644 --- a/nemo/collections/nlp/models/language_modeling/megatron_finetune_model.py +++ b/nemo/collections/nlp/models/language_modeling/megatron_finetune_model.py @@ -63,6 +63,32 @@ def setup_metric(self, data_cfg): raise KeyError( f"{data_cfg.metric.name} is not supported. List of supported metrics: {MetricStringToTorchMetric.keys()}" ) + if data_cfg.metric.name in self._metrics_require_string2category_map: + if data_cfg.metric.average is None: + raise ValueError( + f"{data_cfg.metric.name} requires specifying whether you want to compute a micro or macro average. Found None." + ) + if ( + data_cfg.metric.get('labels_are_strings', False) + and data_cfg.metric.name in self._metrics_require_string2category_map + ): + if data_cfg.metric.num_classes is None: + raise ValueError( + "Number of classes is not provided in the metric section within the data config. " + f"Please provide the number of classes in the data config to use the {data_cfg.metric.name} metric." + ) + if data_cfg.metric.get('class_labels', None) is None or not isinstance( + data_cfg.metric.get('class_labels', None), ListConfig + ): + raise ValueError( + "Class labels are not provided properly in the metric section witnin the data config. " + f"Please provide the class labels as a list of strings in the data config to use the {data_cfg.metric.name} metric." + ) + if len(data_cfg.metric.get('class_labels', None)) != data_cfg.metric.num_classes: + raise ValueError( + f"Number of class labels {len(data_cfg.metric.get('class_labels', None))} does not match `num_classes` : {data_cfg.metric.num_classes}" + ) + metric_name = data_cfg.metric.name metric = MetricStringToTorchMetric[metric_name] # GLUE will not have a "src_file_name" attribute and will always have only a single metric. @@ -71,6 +97,8 @@ def setup_metric(self, data_cfg): # We pass average and num_classes to the metric constructor via kwargs even if they don't exist for each metric. metric = [ metric(average=data_cfg.metric.average, num_classes=data_cfg.metric.num_classes) + if metric_name != 'exact_string_match' + else metric() for _ in range(len(self.cfg.data.test_ds.src_file_name)) ] else: @@ -80,6 +108,10 @@ def setup_metric(self, data_cfg): return metric, metric_name + @property + def _metrics_require_string2category_map(self): + return set(["f1", "accuracy", "average_precision"]) + def setup(self, stage=None): # This is just to keep the parent class happy since we override its setup() method. self.init_consumed_samples = 0 @@ -168,7 +200,7 @@ def training_step(self, batch, batch_idx): batch = self._process_global_batch(batch) return super().training_step(batch, batch_idx) - def cast_for_metric(self, pred, label, metric_name): + def cast_for_metric(self, pred, label, metric_name, class_labels=None, labels_are_strings=False): if metric_name == 'exact_string_match': return pred, label pred = pred.replace(' ', '') @@ -191,7 +223,7 @@ def cast_for_metric(self, pred, label, metric_name): label = torch.FloatTensor([label]).to(self.device) # Other metrics require casting to integers. - elif metric_name in ['accuracy', 'auc', 'auroc', 'average_precision', 'f1']: + elif metric_name in self._metrics_require_string2category_map and not labels_are_strings: # Text-to-text model predictions may not always be valid integers. try: pred = int(pred) @@ -206,6 +238,18 @@ def cast_for_metric(self, pred, label, metric_name): pred = torch.LongTensor([pred]).to(self.device) label = torch.LongTensor([label]).to(self.device) + # If labels are strings, we need to convert them to indices for some metrics. + elif metric_name in self._metrics_require_string2category_map and labels_are_strings: + # Cast string labels to integers before computing the metric. + if pred not in class_labels: + pred = 0 # If the prediction is not in the class labels, use the first class label. + else: + pred = class_labels.index(pred) + if label not in class_labels: + raise ValueError(f"Ground truth labe; {label} is not in the class labels list : {class_labels}") + label = class_labels.index(label) + pred = torch.LongTensor([pred]).to(self.device) + label = torch.LongTensor([label]).to(self.device) else: raise ValueError(f'Metric {metric_name} not supported.') @@ -256,7 +300,11 @@ def inference_step(self, batch, batch_idx, mode, dataloader_idx=0): for _, (pred, label, category) in enumerate(zip(preds_text, labels_text, categories)): # To compute metrics like pearson or spearman correlation, we need to cast the predicted string and labels to floats. pred, label = self.cast_for_metric( - pred, label, self.val_metric_name if mode == 'validation' else self.test_metric_name + pred=pred, + label=label, + metric_name=self.val_metric_name if mode == 'validation' else self.test_metric_name, + class_labels=self.cfg.data.validation_ds.metric.get('class_labels', None), + labels_are_strings=self.cfg.data.validation_ds.metric.get('labels_are_strings', False), ) if batch_has_lang_information: _ = metric(pred, label, category) diff --git a/nemo/collections/nlp/models/language_modeling/megatron_gpt_model.py b/nemo/collections/nlp/models/language_modeling/megatron_gpt_model.py index 9e275fd7cb53..f05c4c18aada 100755 --- a/nemo/collections/nlp/models/language_modeling/megatron_gpt_model.py +++ b/nemo/collections/nlp/models/language_modeling/megatron_gpt_model.py @@ -109,6 +109,21 @@ def __init__(self, cfg: DictConfig, trainer: Trainer): # configuration used for inference self._inference_config = None + # At pipeline-parallel training, set the pipeline stage that the current GPU belongs to skip loading inputs + # Intemediate stage: doesn't need any inputs + # Fist pipeline stage: needs only 'tokens' and 'position_ids' + # Last pipeline stage: needs only 'labels' and 'loss_mask' + self._is_first_pipe_stage = False + self._is_intermediate_pipe_stage = False + self._is_last_pipe_stage = False + if parallel_state.get_pipeline_model_parallel_world_size() > 1: + if parallel_state.is_pipeline_first_stage(): + self._is_first_pipe_stage = True + elif parallel_state.is_pipeline_last_stage(): + self._is_last_pipe_stage = True + else: + self._is_intermediate_pipe_stage = True + def set_inference_config(self, inference_config): self._inference_config = inference_config @@ -136,11 +151,14 @@ def model_provider_func(self, pre_process, post_process): hidden_dropout=self.cfg.get('hidden_dropout', 0.1), precision=self.cfg.get('precision', 16), fp32_residual_connection=self.cfg.get('fp32_residual_connection', False), + activations_checkpoint_granularity=self.cfg.get('activations_checkpoint_granularity', None), activations_checkpoint_method=self.cfg.get('activations_checkpoint_method', None), activations_checkpoint_num_layers=self.cfg.get('activations_checkpoint_num_layers', 1), layernorm_epsilon=self.cfg.get('layernorm_epsilon', 1e-5), onnx_safe=self.cfg.get('onnx_safe', False), persist_layer_norm=self.cfg.get('persist_layer_norm', False), + sequence_parallel=self.cfg.get('sequence_parallel', False), + gradient_accumulation_fusion=self.cfg.get('gradient_accumulation_fusion', False), ) return model @@ -166,8 +184,13 @@ def training_step(self, batch, batch_idx): # we zero grads here because we also call backward in the apex fwd/bwd functions self._optimizer.zero_grad() - # we prepare the micro batches for the apex fwd/bwd function - batch_for_pipeline = self.process_global_batch(batch) + if self._is_intermediate_pipe_stage: + # The intermediate pipeline stages do not need any inputs from data loader + # GPT3 uses decoder with AttnMask:causal, thus doesn't need attention_mask + batch_for_pipeline = None + else: + # we prepare the micro batches for the apex fwd/bwd function + batch_for_pipeline = self.process_global_batch(batch) tensor_shape = [self.cfg.encoder_seq_length, self.cfg.micro_batch_size, self.cfg.hidden_size] if self.cfg.get('pipeline_model_parallel_size', 1) > 1: @@ -179,10 +202,11 @@ def training_step(self, batch, batch_idx): tensor_shape=tensor_shape, dtype=self.autocast_dtype, grad_scaler=self.trainer.precision_plugin.scaler if self.cfg.precision == 16 else None, + sequence_parallel_enabled=self.cfg.get('sequence_parallel', False), ) else: - # no pipeline parallelism so we reduce grads asynchronously - if self.megatron_amp_o2: + # no pipeline parallelism so we reduce grads asynchronously if not using sequence parallelism + if self.megatron_amp_o2 and not self.cfg.get('sequence_parallel', False): custom_sync_context_handler = self._optimizer.no_sync else: # TODO: enable async grad all reduce for O1/autocast mixed precision training @@ -207,26 +231,24 @@ def training_step(self, batch, batch_idx): else: loss_mean = torch.tensor(0.0).cuda() + # when using sequence parallelism, the sequence parallel layernorm grads must be all-reduced + if self.cfg.get('tensor_model_parallel_size', 1) > 1 and self.cfg.get('sequence_parallel', False): + self.allreduce_sequence_parallel_gradients() + if self.megatron_amp_o2: - # when using pipeline parallelism grads must be reduced after the pipeline (not asynchronously) - if self.cfg.get('pipeline_model_parallel_size', 1) > 1: + # when using pipeline parallelism grads must be all-reduced after the pipeline (not asynchronously) + if self.cfg.get('pipeline_model_parallel_size', 1) > 1 or self.cfg.get('sequence_parallel', False): # main grads are stored in the MainParamsOptimizer wrapper self._optimizer.allreduce_main_grads() else: # async grad allreduce is not currently implemented for O1/autocasting mixed precision training - # so we allreduce gradients after the pipeline + # so we all-reduce gradients after the pipeline self.allreduce_gradients() # @sangkug we think this is causing memory to blow up (hurts perf) if self.cfg.get('pipeline_model_parallel_size', 1) > 1: # when using pipeline parallelism the first and last stage must keep embeddings in sync self.allreduce_first_last_embeddings() - # while async grad allreduce is enabled, bprop will keep moving forward without waiting for - # the finish of async grad AR works. Hence, to guarantee the correctness of grads reduction, - # we cannot start weight update until all async grad AR works are done. - if self.megatron_amp_o2 and self.cfg.get('pipeline_model_parallel_size', 1) == 1: - torch.cuda.synchronize() - ## logging # we can only log on one rank if it is rank zero so we broadcast from last rank # we can avoid this broadcast by updating the PTL log function to accept specific ranks @@ -302,6 +324,24 @@ def optimizer_zero_grad(self, *args, **kwargs): """ return + def allreduce_sequence_parallel_gradients(self): + """ All-reduce layernorm parameters across model parallel nodes when sequence parallelism is used. + Modified from megatron-lm: + https://gitlab-master.nvidia.com/ADLR/megatron-lm/-/blob/3f91f09bb2ab32f9904b47f46f19d2fc3f518ed8/megatron/training.py#L425 + """ + grads = [] + for param in self.model.parameters(): + if getattr(param, 'sequence_parallel_enabled', False): + if self.megatron_amp_o2: + grad = param.main_grad + else: + grad = param.grad + grads.append(grad.data) + coalesced = torch._utils._flatten_dense_tensors(grads) + torch.distributed.all_reduce(coalesced, group=parallel_state.get_tensor_model_parallel_group()) + for buf, synced in zip(grads, torch._utils._unflatten_dense_tensors(coalesced, grads)): + buf.copy_(synced) + def allreduce_first_last_embeddings(self): # Modified from megatron-lm: https://github.com/NVIDIA/Megatron-LM/blob/d41696840ed0a7edb7e0499eb82a48ae112d9bb3/megatron/training.py#L407 @@ -323,9 +363,27 @@ def allreduce_first_last_embeddings(self): def get_forward_output_and_loss_func(self): def fwd_output_and_loss_func(batch, model): - batch = [x.cuda(non_blocking=True) for x in batch] - tokens, labels, loss_mask, attention_mask, position_ids = batch - attention_mask = attention_mask[0:1] + if parallel_state.get_pipeline_model_parallel_world_size() == 1: + batch = [x.cuda(non_blocking=True) for x in batch] + tokens, labels, loss_mask, attention_mask, position_ids = batch + attention_mask = attention_mask[0:1] + else: + # GPT3 uses only causal mask, which doesn't need attention mask + if self._is_first_pipe_stage: + # Fist pipeline stage needs only the tokens and position_ids + tokens = batch[0].cuda(non_blocking=True) + position_ids = batch[4].cuda(non_blocking=True) + labels, loss_mask, attention_mask = None, None, None + elif self._is_intermediate_pipe_stage: + # Intermediate pipeline stage doesn't need any inputs + tokens, labels, loss_mask, attention_mask, position_ids = None, None, None, None, None + elif self._is_last_pipe_stage: + # Last pipeline stage needs only the labels and loss_mask + labels = batch[1].cuda(non_blocking=True) + loss_mask = batch[2].cuda(non_blocking=True) + tokens, attention_mask, position_ids = None, None, None + else: + assert False output_tensor = model(tokens, position_ids, attention_mask, labels) def loss_func(output_tensor): @@ -391,6 +449,7 @@ def validation_step(self, batch, batch_idx): forward_only=True, tensor_shape=tensor_shape, dtype=self.autocast_dtype, + sequence_parallel_enabled=self.cfg.get('sequence_parallel', False), ) else: losses_reduced_per_micro_batch = forward_backward_no_pipelining( diff --git a/nemo/collections/nlp/models/language_modeling/megatron_gpt_prompt_learning_model.py b/nemo/collections/nlp/models/language_modeling/megatron_gpt_prompt_learning_model.py index 84740433b127..c944bf1dc361 100644 --- a/nemo/collections/nlp/models/language_modeling/megatron_gpt_prompt_learning_model.py +++ b/nemo/collections/nlp/models/language_modeling/megatron_gpt_prompt_learning_model.py @@ -399,6 +399,7 @@ def forward( input_embeds = self.embed_input_train(input_ids, taskname_ids) position_embeddings = self.frozen_model.model.language_model.embedding.position_embeddings(position_ids) encoder_input = input_embeds + position_embeddings + encoder_input = encoder_input.transpose(0, 1).contiguous() else: encoder_input = None diff --git a/nemo/collections/nlp/models/language_modeling/megatron_lm_encoder_decoder_model.py b/nemo/collections/nlp/models/language_modeling/megatron_lm_encoder_decoder_model.py index 6c4e52caf6c7..3da3666fc02c 100644 --- a/nemo/collections/nlp/models/language_modeling/megatron_lm_encoder_decoder_model.py +++ b/nemo/collections/nlp/models/language_modeling/megatron_lm_encoder_decoder_model.py @@ -150,6 +150,9 @@ def model_provider_func(self, pre_process, post_process, add_encoder, add_decode position_embedding_type=self.cfg.get('position_embedding_type', 'learned_absolute'), relative_attention_num_buckets=self.cfg.get('relative_attention_num_buckets', 32), relative_attention_max_distance=self.cfg.get('relative_attention_max_distance', 128), + relative_position_bias_self_attention_only=self.cfg.get( + 'relative_position_bias_self_attention_only', True + ), precision=self.cfg.get('precision', 16), fp32_residual_connection=self.cfg.get('fp32_residual_connection', False), activations_checkpoint_method=self.cfg.get('activations_checkpoint_method', None), @@ -271,12 +274,6 @@ def training_step(self, batch, batch_idx): # when using pipeline parallelism, we need keep the word and position embeddings in sync self.allreduce_word_and_position_embeddings() - # while async grad allreduce is enabled, bprop will keep moving forward without waiting for - # the finish of async grad AR works. Hence, to guarantee the correctness of grads reduction, - # we cannot start weight update until all async grad AR works are done. - if self.megatron_amp_o2 and self.cfg.get('pipeline_model_parallel_size', 1) == 1: - torch.cuda.synchronize() - ## logging # we can only log on one rank if it is rank zero so we broadcast from last rank # we can avoid this broadcast by updating the PTL log function to accept specific ranks @@ -401,14 +398,14 @@ def allreduce_word_and_position_embeddings(self): parallel_state.is_rank_in_position_embedding_group() and parallel_state.get_pipeline_model_parallel_world_size() > 1 and parallel_state.get_pipeline_model_parallel_split_rank() is not None + and self.cfg.get('position_embedding_type') == 'learned_absolute' ): - if self.cfg.get('position_embedding_type') != 'relative': - position_embeddings_weight = self.enc_dec_model.position_embeddings_weight() - if self.megatron_amp_o2: - grad = position_embeddings_weight.main_grad - else: - grad = position_embeddings_weight.grad - torch.distributed.all_reduce(grad, group=parallel_state.get_position_embedding_group()) + position_embeddings_weight = self.enc_dec_model.position_embeddings_weight() + if self.megatron_amp_o2: + grad = position_embeddings_weight.main_grad + else: + grad = position_embeddings_weight.grad + torch.distributed.all_reduce(grad, group=parallel_state.get_position_embedding_group()) def get_forward_output_and_loss_func(self): def fwd_output_and_loss_func(batch, model): diff --git a/nemo/collections/nlp/modules/common/megatron/fused_layer_norm.py b/nemo/collections/nlp/modules/common/megatron/fused_layer_norm.py index 53a1b8054fc9..7aa401082b43 100644 --- a/nemo/collections/nlp/modules/common/megatron/fused_layer_norm.py +++ b/nemo/collections/nlp/modules/common/megatron/fused_layer_norm.py @@ -15,7 +15,7 @@ try: - from apex.contrib.layer_norm.layer_norm import FastLayerNorm + from apex.transformer.layers.layer_norm import FastLayerNorm from apex.normalization.fused_layer_norm import MixedFusedLayerNorm HAVE_APEX = True @@ -23,7 +23,7 @@ HAVE_APEX = False -def get_layer_norm(hidden_size, eps=1e-5, persist_layer_norm=False): +def get_layer_norm(hidden_size, eps=1e-5, persist_layer_norm=False, sequence_parallel=False): # List of hiddens sizes supported in the persistent layer norm kernel # If the hidden size is not supported, fall back to the non-persistent # kernel. @@ -57,6 +57,6 @@ def get_layer_norm(hidden_size, eps=1e-5, persist_layer_norm=False): persist_layer_norm = False if persist_layer_norm: - return FastLayerNorm(hidden_size, eps) + return FastLayerNorm(hidden_size, eps, sequence_parallel_enabled=sequence_parallel) else: - return MixedFusedLayerNorm(hidden_size, eps) + return MixedFusedLayerNorm(hidden_size, eps, sequence_parallel_enbaled=sequence_parallel) diff --git a/nemo/collections/nlp/modules/common/megatron/language_model.py b/nemo/collections/nlp/modules/common/megatron/language_model.py index 29d1ac3adf36..e57a7170dfce 100755 --- a/nemo/collections/nlp/modules/common/megatron/language_model.py +++ b/nemo/collections/nlp/modules/common/megatron/language_model.py @@ -70,6 +70,9 @@ def get_language_model( openai_gelu=False, onnx_safe=False, megatron_legacy=False, + activations_checkpoint_granularity=None, + sequence_parallel=False, + gradient_accumulation_fusion=False, ): """Build language model and return along with the key to save.""" @@ -117,6 +120,9 @@ def get_language_model( openai_gelu=openai_gelu, onnx_safe=onnx_safe, megatron_legacy=megatron_legacy, + activations_checkpoint_granularity=activations_checkpoint_granularity, + sequence_parallel=sequence_parallel, + gradient_accumulation_fusion=gradient_accumulation_fusion, ) # key used for checkpoints. language_model_key = 'language_model' @@ -136,14 +142,21 @@ class Pooler(MegatronModule): bias is set to zero. """ - def __init__(self, hidden_size, init_method): + def __init__(self, hidden_size, init_method, sequence_parallel=False): super(Pooler, self).__init__() self.dense = get_linear_layer(hidden_size, hidden_size, init_method) + self.sequence_parallel = sequence_parallel def forward(self, hidden_states, sequence_index=0): - # hidden_states: [b, s, h]prompt_embeddings + # hidden_states: [s, b, h] prompt_embeddings # sequence_index: index of the token to pool. - pooled = hidden_states[:, sequence_index, :] + + # gather data along sequence dimensions + # same pooler is run on all tensor parallel nodes + if self.sequence_parallel: + hidden_states = tensor_parallel.gather_from_sequence_parallel_region() + + pooled = hidden_states[sequence_index, :, :] pooled = self.dense(pooled) pooled = torch.tanh(pooled) return pooled @@ -162,7 +175,7 @@ class Embedding(MegatronModule): num_tokentypes: size of the token-type embeddings. 0 value will ignore this embedding use_cpu_initialization: whether to initialize the weights in CPU - add_position_embedding: flag for controlling whether to add position embedding to the input. + position_embedding_type: position embedding type determines whether we instantiate a learnable position embedding table. """ def __init__( @@ -174,13 +187,18 @@ def __init__( init_method, num_tokentypes=0, use_cpu_initialization=False, - add_position_embedding=True, + fp32_residual_connection=False, + sequence_parallel=False, + position_embedding_type='learned_absolute', + transpose_batch_sequence=True, ): super(Embedding, self).__init__() self.hidden_size = hidden_size self.init_method = init_method self.num_tokentypes = num_tokentypes + self.position_embedding_type = position_embedding_type + self.transpose_batch_sequence = transpose_batch_sequence # Word embeddings (parallel). self.word_embeddings = tensor_parallel.VocabParallelEmbedding( @@ -188,9 +206,7 @@ def __init__( ) self._word_embeddings_key = 'word_embeddings' - self.add_position_embedding = add_position_embedding - - if self.add_position_embedding: + if self.position_embedding_type == 'learned_absolute': # Position embedding (serial). self.position_embeddings = torch.nn.Embedding(max_sequence_length, self.hidden_size) self._position_embeddings_key = 'position_embeddings' @@ -209,6 +225,9 @@ def __init__( else: self.tokentype_embeddings = None + self.fp32_residual_connection = fp32_residual_connection + self.sequence_parallel = sequence_parallel + # Embeddings dropout self.embedding_dropout = torch.nn.Dropout(embedding_dropout_prob) @@ -216,7 +235,7 @@ def zero_parameters(self): """Zero out all parameters in embedding.""" self.word_embeddings.weight.data.fill_(0) self.word_embeddings.weight.shared = True - if self.add_position_embedding: + if self.position_embedding_type == 'learned_absolute': self.position_embeddings.weight.data.fill_(0) self.position_embeddings.weight.shared = True if self.num_tokentypes > 0: @@ -240,7 +259,7 @@ def add_tokentype_embeddings(self, num_tokentypes): def forward(self, input_ids, position_ids, token_type_ids=None): # Embeddings. words_embeddings = self.word_embeddings(input_ids) - if self.add_position_embedding: + if self.position_embedding_type == 'learned_absolute': position_embeddings = self.position_embeddings(position_ids) embeddings = words_embeddings + position_embeddings else: @@ -251,8 +270,21 @@ def forward(self, input_ids, position_ids, token_type_ids=None): else: assert self.tokentype_embeddings is None + # Data format change to avoid explicit tranposes : [b s h] --> [s b h]. + if self.transpose_batch_sequence: + embeddings = embeddings.transpose(0, 1).contiguous() + + # If the input flag for fp32 residual connection is set, convert for float. + if self.fp32_residual_connection: + embeddings = embeddings.float() + # Dropout. - embeddings = self.embedding_dropout(embeddings) + if self.sequence_parallel: + embeddings = tensor_parallel.mappings.scatter_to_sequence_parallel_region(embeddings) + with tensor_parallel.random.get_cuda_rng_tracker().fork(): + embeddings = self.embedding_dropout(embeddings) + else: + embeddings = self.embedding_dropout(embeddings) return embeddings @@ -261,7 +293,7 @@ def state_dict_for_save_checkpoint(self, destination=None, prefix='', keep_vars= state_dict_ = {} state_dict_[self._word_embeddings_key] = self.word_embeddings.state_dict(destination, prefix, keep_vars) - if self.add_position_embedding: + if self.position_embedding_type == 'learned_absolute': state_dict_[self._position_embeddings_key] = self.position_embeddings.state_dict( destination, prefix, keep_vars ) @@ -286,7 +318,7 @@ def load_state_dict(self, state_dict, strict=True): state_dict_[key.split('word_embeddings.')[1]] = state_dict[key] self.word_embeddings.load_state_dict(state_dict_, strict=strict) - if self.add_position_embedding: + if self.position_embedding_type == 'learned_absolute': # Position embedding. if self._position_embeddings_key in state_dict: state_dict_ = state_dict[self._position_embeddings_key] @@ -362,6 +394,9 @@ def __init__( openai_gelu=False, onnx_safe=False, megatron_legacy=False, + activations_checkpoint_granularity=None, + sequence_parallel=False, + gradient_accumulation_fusion=False, ): super(TransformerLanguageModel, self).__init__() @@ -397,6 +432,8 @@ def __init__( num_tokentypes=self.num_tokentypes, use_cpu_initialization=use_cpu_initialization, embedding_dropout_prob=self.hidden_dropout, + sequence_parallel=sequence_parallel, + fp32_residual_connection=fp32_residual_connection, ) self._embedding_key = 'embedding' @@ -426,6 +463,9 @@ def __init__( onnx_safe=onnx_safe, masked_softmax_fusion=masked_softmax_fusion, megatron_legacy=megatron_legacy, + sequence_parallel=sequence_parallel, + activations_checkpoint_granularity=activations_checkpoint_granularity, + gradient_accumulation_fusion=gradient_accumulation_fusion, ) self._encoder_key = 'encoder' @@ -457,13 +497,16 @@ def __init__( onnx_safe=onnx_safe, masked_softmax_fusion=masked_softmax_fusion, megatron_legacy=megatron_legacy, + sequence_parallel=sequence_parallel, + activations_checkpoint_granularity=activations_checkpoint_granularity, + gradient_accumulation_fusion=gradient_accumulation_fusion, ) self._decoder_key = 'decoder' if self.post_process: # Pooler. if self.add_pooler: - self.pooler = Pooler(self.hidden_size, self.init_method) + self.pooler = Pooler(self.hidden_size, self.init_method, sequence_parallel=sequence_parallel) self._pooler_key = 'pooler' def set_input_tensor(self, input_tensor): @@ -500,6 +543,10 @@ def forward( else: pass + # encoder_input: [s, b, h] + + # enc_attn_mask: [1, 1, s, s] + # encoder. if enc_hidden_states is None: encoder_output = self.encoder( diff --git a/nemo/collections/nlp/modules/common/megatron/megatron_decoders.py b/nemo/collections/nlp/modules/common/megatron/megatron_decoders.py index f3a1a57b2fbd..f50de154e1be 100644 --- a/nemo/collections/nlp/modules/common/megatron/megatron_decoders.py +++ b/nemo/collections/nlp/modules/common/megatron/megatron_decoders.py @@ -48,7 +48,6 @@ def get_decoder_model( kv_channels=None, init_method=None, scaled_init_method=None, - add_decoder=False, decoder_attn_mask_type=AttnMaskType.causal, pre_process=True, post_process=True, @@ -56,9 +55,6 @@ def get_decoder_model( use_cpu_initialization=False, hidden_dropout=0.1, attention_dropout=0.1, - position_embedding_type='learned_absolute', - relative_attention_num_buckets=32, - relative_attention_max_distance=128, precision=16, fp32_residual_connection=False, activations_checkpoint_method=None, @@ -112,9 +108,6 @@ def get_decoder_model( use_cpu_initialization=use_cpu_initialization, hidden_dropout=hidden_dropout, attention_dropout=attention_dropout, - position_embedding_type=position_embedding_type, - relative_attention_num_buckets=relative_attention_num_buckets, - relative_attention_max_distance=relative_attention_max_distance, precision=precision, fp32_residual_connection=fp32_residual_connection, activations_checkpoint_method=activations_checkpoint_method, diff --git a/nemo/collections/nlp/modules/common/megatron/megatron_encoder_decoder.py b/nemo/collections/nlp/modules/common/megatron/megatron_encoder_decoder.py index 059eb28f8542..d1af017a7509 100644 --- a/nemo/collections/nlp/modules/common/megatron/megatron_encoder_decoder.py +++ b/nemo/collections/nlp/modules/common/megatron/megatron_encoder_decoder.py @@ -85,7 +85,12 @@ def __init__( self._decoder_key = "decoder" def encode( - self, enc_input, enc_attn_mask, enc_layer_past=None, enc_get_key_value=False, + self, + enc_input, + enc_attn_mask, + enc_layer_past=None, + enc_get_key_value=False, + enc_self_attention_relative_position_bias=None, ): if self.encoder is None: raise ValueError(f"Cannot call .encode(...) when self.encoder is None.") @@ -95,12 +100,21 @@ def encode( enc_attn_mask=enc_attn_mask, layer_past=enc_layer_past, get_key_value=enc_get_key_value, + enc_self_attention_relative_position_bias=enc_self_attention_relative_position_bias, ) return enc_output def decode( - self, dec_input, dec_attn_mask, enc_output, enc_attn_mask, dec_layer_past=None, dec_get_key_value=False, + self, + dec_input, + dec_attn_mask, + enc_output, + enc_attn_mask, + dec_layer_past=None, + dec_get_key_value=False, + dec_self_attention_relative_position_bias=None, + dec_cross_attention_relative_position_bias=None, ): if self.decoder is None: raise ValueError(f"Cannot call .decode(...) when self.decoder is None.") @@ -112,6 +126,8 @@ def decode( get_key_value=dec_get_key_value, enc_output=enc_output, enc_attn_mask=enc_attn_mask, + dec_self_attention_relative_position_bias=dec_self_attention_relative_position_bias, + dec_cross_attention_relative_position_bias=dec_cross_attention_relative_position_bias, ) return dec_output @@ -128,6 +144,9 @@ def forward( dec_layer_past=None, dec_get_key_value=False, output_enc_hidden_only=False, + enc_self_attention_relative_position_bias=None, + dec_self_attention_relative_position_bias=None, + dec_cross_attention_relative_position_bias=None, ): # encoder if enc_output is None: @@ -137,6 +156,7 @@ def forward( enc_attn_mask=enc_attn_mask, enc_layer_past=enc_layer_past, enc_get_key_value=enc_get_key_value, + enc_self_attention_relative_position_bias=enc_self_attention_relative_position_bias, ) else: assert self.encoder_hidden_state is not None @@ -159,6 +179,8 @@ def forward( enc_attn_mask=enc_attn_mask, dec_layer_past=dec_layer_past, dec_get_key_value=dec_get_key_value, + dec_self_attention_relative_position_bias=dec_self_attention_relative_position_bias, + dec_cross_attention_relative_position_bias=dec_cross_attention_relative_position_bias, ) return dec_output, enc_output diff --git a/nemo/collections/nlp/modules/common/megatron/megatron_encoders.py b/nemo/collections/nlp/modules/common/megatron/megatron_encoders.py index ddd146d81c3b..ab8191986ae6 100644 --- a/nemo/collections/nlp/modules/common/megatron/megatron_encoders.py +++ b/nemo/collections/nlp/modules/common/megatron/megatron_encoders.py @@ -56,9 +56,6 @@ def get_encoder_model( use_cpu_initialization=False, hidden_dropout=0.1, attention_dropout=0.1, - position_embedding_type='learned_absolute', - relative_attention_num_buckets=32, - relative_attention_max_distance=128, precision=16, fp32_residual_connection=False, activations_checkpoint_method=None, @@ -114,9 +111,6 @@ def get_encoder_model( use_cpu_initialization=use_cpu_initialization, hidden_dropout=hidden_dropout, attention_dropout=attention_dropout, - position_embedding_type=position_embedding_type, - relative_attention_num_buckets=relative_attention_num_buckets, - relative_attention_max_distance=relative_attention_max_distance, precision=precision, fp32_residual_connection=fp32_residual_connection, activations_checkpoint_method=activations_checkpoint_method, diff --git a/nemo/collections/nlp/modules/common/megatron/megatron_transformer_decoder.py b/nemo/collections/nlp/modules/common/megatron/megatron_transformer_decoder.py index 068aeabc5e4d..1d8b1a818855 100644 --- a/nemo/collections/nlp/modules/common/megatron/megatron_transformer_decoder.py +++ b/nemo/collections/nlp/modules/common/megatron/megatron_transformer_decoder.py @@ -58,9 +58,6 @@ def __init__( decoder_attn_mask_type=AttnMaskType.causal, hidden_dropout=0.1, attention_dropout=0.1, - position_embedding_type='learned_absolute', - relative_attention_num_buckets=32, - relative_attention_max_distance=128, precision=16, fp32_residual_connection=False, activations_checkpoint_method=None, @@ -120,9 +117,6 @@ def __init__( layernorm_epsilon=layernorm_epsilon, hidden_dropout=hidden_dropout, attention_dropout=attention_dropout, - position_embedding_type=position_embedding_type, - relative_attention_num_buckets=relative_attention_num_buckets, - relative_attention_max_distance=relative_attention_max_distance, use_cpu_initialization=use_cpu_initialization, bias_activation_fusion=bias_activation_fusion, bias_dropout_fusion=bias_dropout_add_fusion, @@ -144,7 +138,15 @@ def set_input_tensor(self, input_tensor): self.model.set_input_tensor(input_tensor) def forward( - self, dec_input, dec_attn_mask, enc_output, enc_attn_mask, layer_past=None, get_key_value=False, + self, + dec_input, + dec_attn_mask, + enc_output, + enc_attn_mask, + layer_past=None, + get_key_value=False, + dec_self_attention_relative_position_bias=None, + dec_cross_attention_relative_position_bias=None, ): # convert to Megatron mask dec_attn_mask_3d = build_attention_mask_3d( @@ -162,6 +164,8 @@ def forward( get_key_value=get_key_value, encoder_output=enc_output, enc_dec_attn_mask=attn_mask_postprocess(enc_dec_attn_mask_3d), + self_attention_relative_position_bias=dec_self_attention_relative_position_bias, + cross_attention_relative_position_bias=dec_cross_attention_relative_position_bias, ) return dec_output diff --git a/nemo/collections/nlp/modules/common/megatron/megatron_transformer_encoder.py b/nemo/collections/nlp/modules/common/megatron/megatron_transformer_encoder.py index 4d82d95fedde..95ef4d3b5f41 100644 --- a/nemo/collections/nlp/modules/common/megatron/megatron_transformer_encoder.py +++ b/nemo/collections/nlp/modules/common/megatron/megatron_transformer_encoder.py @@ -36,8 +36,7 @@ class MegatronTransformerEncoderModule(MegatronModule): - """Transformer encoder model. - """ + """Transformer encoder model.""" def __init__( self, @@ -55,9 +54,6 @@ def __init__( encoder_attn_mask_type=AttnMaskType.padding, hidden_dropout=0.1, attention_dropout=0.1, - position_embedding_type='learned_absolute', - relative_attention_num_buckets=32, - relative_attention_max_distance=128, precision=16, fp32_residual_connection=False, activations_checkpoint_method=None, @@ -117,9 +113,6 @@ def __init__( layernorm_epsilon=layernorm_epsilon, hidden_dropout=hidden_dropout, attention_dropout=attention_dropout, - position_embedding_type=position_embedding_type, - relative_attention_num_buckets=relative_attention_num_buckets, - relative_attention_max_distance=relative_attention_max_distance, use_cpu_initialization=use_cpu_initialization, bias_activation_fusion=bias_activation_fusion, bias_dropout_fusion=bias_dropout_add_fusion, @@ -141,7 +134,12 @@ def set_input_tensor(self, input_tensor): self.model.set_input_tensor(input_tensor) def forward( - self, enc_input, enc_attn_mask, layer_past=None, get_key_value=False, + self, + enc_input, + enc_attn_mask, + layer_past=None, + get_key_value=False, + enc_self_attention_relative_position_bias=None, ): # convert to Megatron mask enc_attn_mask_3d = build_attention_mask_3d( @@ -150,7 +148,12 @@ def forward( # transformer encoder enc_output = self.model( - enc_input, attn_mask_postprocess(enc_attn_mask_3d), layer_past=layer_past, get_key_value=get_key_value, + enc_input, + attn_mask_postprocess(enc_attn_mask_3d), + layer_past=layer_past, + get_key_value=get_key_value, + self_attention_relative_position_bias=enc_self_attention_relative_position_bias, + cross_attention_relative_position_bias=None, ) return enc_output diff --git a/nemo/collections/nlp/modules/common/megatron/retrieval_token_level_encoder_decoder.py b/nemo/collections/nlp/modules/common/megatron/retrieval_token_level_encoder_decoder.py index 6a7136dd803a..3e6333fca444 100644 --- a/nemo/collections/nlp/modules/common/megatron/retrieval_token_level_encoder_decoder.py +++ b/nemo/collections/nlp/modules/common/megatron/retrieval_token_level_encoder_decoder.py @@ -1,4 +1,5 @@ # Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved. + # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -120,7 +121,8 @@ def __init__( num_tokentypes=num_tokentypes, use_cpu_initialization=use_cpu_initialization, embedding_dropout_prob=hidden_dropout, - add_position_embedding=add_position_embedding, + position_embedding_type='learned_absolute' if add_position_embedding else '', + transpose_batch_sequence=False, ) self._embedding_key = "embedding" @@ -340,7 +342,7 @@ def forward( # hidden is a tuple, (layernorm_input, layernorm_output) self.post_decoder.set_input_tensor(hidden) # scale down the pre-decoder output by half - hidden = (hidden[0] * 0.5, hidden[1] * 0.5) + # hidden = (hidden[0] * 0.5, hidden[1] * 0.5) # stop passing through the gradients encoder_output = hidden[1].transpose(0, 1).contiguous() @@ -370,6 +372,7 @@ def forward( set_inference_key_value_memory=set_inference_key_value_memory, inference_max_sequence_len=inference_max_sequence_len, ) + dec_output = dec_output.transpose(0, 1).contiguous() # only transpose it for post_ln token_logits = self.tokens_head(dec_output, self.word_embeddings_weight()) diff --git a/nemo/collections/nlp/modules/common/megatron/retrieval_transformer.py b/nemo/collections/nlp/modules/common/megatron/retrieval_transformer.py index c2dd5a30eba6..03be13e9ff0c 100644 --- a/nemo/collections/nlp/modules/common/megatron/retrieval_transformer.py +++ b/nemo/collections/nlp/modules/common/megatron/retrieval_transformer.py @@ -75,6 +75,8 @@ def __init__( parent_model_type=ModelType.encoder_or_decoder, chunk_size=64, layer_number_offset=0, # this is use only for attention norm_factor scaling + sequence_parallel=False, + gradient_accumulation_fusion=False, ): super(MegatronRetrievalTransformerEncoderModule, self).__init__() @@ -131,6 +133,8 @@ def __init__( model_type=parent_model_type, chunk_size=chunk_size, layer_number_offset=layer_number_offset, + sequence_parallel=sequence_parallel, + gradient_accumulation_fusion=gradient_accumulation_fusion, ) rot_dim = hidden_size // num_attention_heads if kv_channels is None else kv_channels # partial rotary embeddings, which is better than full rotary @@ -231,7 +235,7 @@ def forward( seq_index = num_seq_chunks * self.chunk_size - retrieved = rearrange(enc_input, 'b k r n d -> (b k r) n d') + retrieved = rearrange(enc_input, 'b k r n d -> n (b k r) d') enc_attn_mask = rearrange(enc_attn_mask, 'b k r n -> (b k r) n') # embed_as_context = repeat(encoder_output[:, :seq_index], 'b (k n) d -> (b k r) n d', n=self.chunk_size, r=r) # context_attn_mask = repeat(context_attn_mask[:, :seq_index], 'b (k n) -> (b k r) n', n=self.chunk_size, r=r) @@ -240,11 +244,11 @@ def forward( if inference_max_sequence_len is not None and not set_inference_key_value_memory: cross_attn_k_pos_emb = self.rotary_pos_emb(n % self.chunk_size, offset=pos_beg) - embed_as_context = repeat(encoder_output[:, :seq_index], 'b (k n) d -> (b k r) n d', n=pos_beg + 1, r=r) + embed_as_context = repeat(encoder_output[:, :seq_index], 'b (k n) d -> n (b k r) d', n=pos_beg + 1, r=r) context_attn_mask = repeat(context_attn_mask[:, :seq_index], 'b (k n) -> (b k r) n', n=pos_beg + 1, r=r) else: embed_as_context = repeat( - encoder_output[:, :seq_index], 'b (k n) d -> (b k r) n d', n=self.chunk_size, r=r + encoder_output[:, :seq_index], 'b (k n) d -> n (b k r) d', n=self.chunk_size, r=r ) context_attn_mask = repeat( context_attn_mask[:, :seq_index], 'b (k n) -> (b k r) n', n=self.chunk_size, r=r @@ -275,7 +279,7 @@ def forward( rotary_pos_emb=attn_pos_emb, ) # revert back to original retrieved shape - enc_output = rearrange(enc_output, '(b k r) n d -> b k r n d', b=b, k=k) + enc_output = rearrange(enc_output, 'n (b k r) d -> b k r n d', b=b, k=k) if inference_max_sequence_len is not None: # update encoded for current chunk @@ -341,6 +345,8 @@ def __init__( parent_model_type=ModelType.encoder_or_decoder, chunk_size=64, layer_number_offset=0, # this is use only for attention norm_factor scaling + sequence_parallel=False, + gradient_accumulation_fusion=False, ): super(MegatronRetrievalTransformerDecoderModule, self).__init__() @@ -396,6 +402,8 @@ def __init__( model_type=parent_model_type, chunk_size=chunk_size, layer_number_offset=layer_number_offset, + sequence_parallel=sequence_parallel, + gradient_accumulation_fusion=gradient_accumulation_fusion, ) rot_dim = hidden_size // num_attention_heads if kv_channels is None else kv_channels # partial rotary embeddings, which is better than full rotary @@ -502,6 +510,8 @@ def forward( enc_dec_attn_mask_3d = None # transformer encoder + if not isinstance(dec_input, tuple): + dec_input = rearrange(dec_input, 'b s d -> s b d') enc_output = self.model( dec_input, dec_attn_mask_3d, @@ -514,7 +524,7 @@ def forward( set_inference_key_value_memory=set_inference_key_value_memory, inference_max_sequence_len=inference_max_sequence_len, ) - + # enc_output = rearrange(dec_input, 's b d -> b s d') return enc_output def state_dict_for_save_checkpoint(self, destination=None, prefix='', keep_vars=False): diff --git a/nemo/collections/nlp/modules/common/megatron/t5_relative_position_embedding.py b/nemo/collections/nlp/modules/common/megatron/t5_relative_position_embedding.py new file mode 100644 index 000000000000..796c0d756a1e --- /dev/null +++ b/nemo/collections/nlp/modules/common/megatron/t5_relative_position_embedding.py @@ -0,0 +1,152 @@ +# Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import math + +import torch + + +class T5RelativePositionEmbedding(torch.nn.Module): + """Relative Position Embedding implementation from the T5 paper : https://arxiv.org/abs/1910.10683""" + + def __init__( + self, + init_method, + bidirectional, + num_attention_heads, + relative_position_num_buckets=32, + relative_position_max_distance=128, + ): + super(T5RelativePositionEmbedding, self).__init__() + self.relative_position_num_buckets = relative_position_num_buckets + self.relative_position_max_distance = relative_position_max_distance + self.self_attention_relative_position_bucket = None + self.inter_attention_relative_position_bucket = None + self.self_attention_relative_position_bias = None + self.inter_attention_relative_position_bias = None + self.bidirectional = bidirectional + + # Relative position Embedding + # Relative Position embedding (all attention layers). + self.relative_position_embedding = torch.nn.Embedding( + self.relative_position_num_buckets, num_attention_heads + ).to(torch.cuda.current_device()) + self._relative_position_embedding_key = 'relative_position_embedding' + init_method(self.relative_position_embedding.weight) + # TODO (sandeepsub): All reduce relative position embedding once apex implements RPE groups. + # self._all_reduce_position_embedding() + + ''' + NOTE: (sandeepsub) - uncomment this code once apex implements RPE groups. + def _all_reduce_position_embedding(self): + args = get_args() + if args.pipeline_model_parallel_size == 1: + return + + if not mpu.is_encoder_or_decoder_pipeline_first_stage(): + self.relative_position_embedding.weight.data.fill_(0) + self.relative_position_embedding.weight.shared = True + + if torch.distributed.is_initialized(): + if args.pipeline_model_parallel_split_rank is not None: + if self.layer_type is LayerType.encoder and mpu.is_rank_in_encoder_relative_position_embedding_group(): + torch.distributed.all_reduce(self.relative_position_embedding.weight.data, group=mpu.get_encoder_relative_position_embedding_group()) + + if self.layer_type is LayerType.decoder and mpu.is_rank_in_decoder_relative_position_embedding_group(): + torch.distributed.all_reduce(self.relative_position_embedding.weight.data, group=mpu.get_decoder_relative_position_embedding_group()) + else: + raise ValueError("Torch Distributed not initialized, cannot allreduce relative position embeddings.") + ''' + + def _relative_position_bucket(self, relative_position, bidirectional=True, num_buckets=32, max_distance=128): + """ + Adapted from HuggingFace T5 Model: + https://github.com/huggingface/transformers/blob/b5e2b183af5e40e33a4dc7659e697d137259d56e + /src/transformers/models/t5/modeling_t5.py#L354 + Translate relative position to a bucket number for relative attention. The relative position + is defined as memory_position - query_position, i.e. the distance in tokens from the attending + position to the attended-to position. If bidirectional=False, then positive relative positions + are invalid. We use smaller buckets for small absolute relative_position and larger buckets + for larger absolute relative_positions. All relative positions >=max_distance map to the same + bucket. All relative positions <=-max_distance map to the same bucket. This should allow for + more graceful generalization to longer sequences than the model has been trained on + Args: + relative_position: an int32 Tensor + bidirectional: a boolean - whether the attention is bidirectional + num_buckets: an integer + max_distance: an integer + Returns: + a Tensor with the same shape as relative_position, + containing int32 values in the range [0, num_buckets) + """ + relative_buckets = 0 + if bidirectional: + num_buckets //= 2 + relative_buckets += (relative_position > 0).to(torch.long) * num_buckets + relative_position = torch.abs(relative_position) + else: + relative_position = -torch.min(relative_position, torch.zeros_like(relative_position)) + # now relative_position is in the range [0, inf) + + # half of the buckets are for exact increments in positions + max_exact = num_buckets // 2 + is_small = relative_position < max_exact + + # The other half of the buckets are for logarithmically bigger bins in positions up to max_distance + relative_postion_if_large = max_exact + ( + torch.log(relative_position.float() / max_exact) + / math.log(max_distance / max_exact) + * (num_buckets - max_exact) + ).to(torch.long) + relative_postion_if_large = torch.min( + relative_postion_if_large, torch.full_like(relative_postion_if_large, num_buckets - 1) + ) + + relative_buckets += torch.where(is_small, relative_position, relative_postion_if_large) + return relative_buckets + + def _compute_relative_position_bucket(self, query_length, key_length): + """ + Adapted from HuggingFace T5 Model + https://github.com/huggingface/transformers/blob/b5e2b183af5e40e33a4dc7659e697d137259d56e/ + src/transformers/models/t5/modeling_t5.py#L401 + """ + + """Compute binned relative position bias""" + context_position = torch.arange(query_length, dtype=torch.long, device=torch.cuda.current_device())[:, None] + memory_position = torch.arange(key_length, dtype=torch.long, device=torch.cuda.current_device())[None, :] + + relative_position = memory_position - context_position # shape (query_length, key_length) + relative_position_bucket_tensor = self._relative_position_bucket( + relative_position, # shape (query_length, key_length) + bidirectional=self.bidirectional, + num_buckets=self.relative_position_num_buckets, + max_distance=self.relative_position_max_distance, + ) + + return relative_position_bucket_tensor + + def _compute_relative_position_bias(self, relative_position_bucket): + # shape (query_length, key_length, num_heads) + values = self.relative_position_embedding(relative_position_bucket) + # shape (1, num_heads, query_length, key_length) + values = values.permute([2, 0, 1]).unsqueeze(0) + + return values + + def forward(self, query_seq_length, key_seq_length): + self_attention_relative_position_bucket = self._compute_relative_position_bucket( + query_seq_length, key_seq_length + ) + return self._compute_relative_position_bias(self_attention_relative_position_bucket) diff --git a/nemo/collections/nlp/modules/common/megatron/token_level_encoder_decoder.py b/nemo/collections/nlp/modules/common/megatron/token_level_encoder_decoder.py index d5aaab3bc84b..6052ae993177 100644 --- a/nemo/collections/nlp/modules/common/megatron/token_level_encoder_decoder.py +++ b/nemo/collections/nlp/modules/common/megatron/token_level_encoder_decoder.py @@ -21,6 +21,7 @@ ) from nemo.collections.nlp.modules.common.megatron.megatron_encoders import get_encoder_model from nemo.collections.nlp.modules.common.megatron.module import MegatronModule +from nemo.collections.nlp.modules.common.megatron.t5_relative_position_embedding import T5RelativePositionEmbedding from nemo.collections.nlp.modules.common.megatron.utils import ( ApexGuardDefaults, build_position_ids, @@ -30,7 +31,7 @@ ) try: - from apex.transformer import tensor_parallel + from apex.transformer import tensor_parallel, parallel_state from apex.transformer.enums import AttnMaskType, ModelType HAVE_APEX = True @@ -61,7 +62,15 @@ def __init__(self, mpu_vocab_size, parallel_output): self.parallel_output = parallel_output def forward(self, hidden_states, word_embeddings_weight): - output = parallel_lm_logits(hidden_states, word_embeddings_weight, self.parallel_output, bias=self.bias) + + async_tensor_model_parallel_allreduce = parallel_state.get_tensor_model_parallel_world_size() > 1 + output = parallel_lm_logits( + hidden_states, + word_embeddings_weight, + self.parallel_output, + bias=self.bias, + async_tensor_model_parallel_allreduce=async_tensor_model_parallel_allreduce, + ) return output @@ -95,6 +104,7 @@ def __init__( position_embedding_type='learned_absolute', relative_attention_num_buckets=32, relative_attention_max_distance=128, + relative_position_bias_self_attention_only=False, precision=16, fp32_residual_connection=False, activations_checkpoint_method=None, @@ -129,13 +139,7 @@ def __init__( self.position_embedding_type = position_embedding_type self.relative_attention_num_buckets = relative_attention_num_buckets self.relative_attention_max_distance = relative_attention_max_distance - - if self.position_embedding_type == 'learned_absolute': - add_position_embedding = True - elif self.position_embedding_type == 'relative': - add_position_embedding = False - else: - raise ValueError('Unknown position embeeding type. Options: ' '[learned_absolute | relative]') + self.relative_position_bias_self_attention_only = relative_position_bias_self_attention_only if kv_channels is None: assert ( @@ -154,9 +158,18 @@ def __init__( num_tokentypes=num_tokentypes, use_cpu_initialization=use_cpu_initialization, embedding_dropout_prob=hidden_dropout, - add_position_embedding=add_position_embedding, + position_embedding_type=position_embedding_type, ) self._encoder_embedding_key = "encoder_embedding" + if self.position_embedding_type == 'relative': + self.encoder_relative_position_embedding = T5RelativePositionEmbedding( + init_method=init_method_normal(init_method_std), + num_attention_heads=num_attention_heads, + relative_position_num_buckets=relative_attention_num_buckets, + relative_position_max_distance=relative_attention_max_distance, + bidirectional=True, + ) + self._encoder_relative_position_embedding_key = "encoder_relative_position_embedding" encoder = get_encoder_model( arch=encoder_arch, @@ -175,9 +188,6 @@ def __init__( use_cpu_initialization=use_cpu_initialization, hidden_dropout=hidden_dropout, attention_dropout=attention_dropout, - position_embedding_type=position_embedding_type, - relative_attention_num_buckets=relative_attention_num_buckets, - relative_attention_max_distance=relative_attention_max_distance, precision=precision, fp32_residual_connection=fp32_residual_connection, activations_checkpoint_method=activations_checkpoint_method, @@ -217,11 +227,32 @@ def __init__( num_tokentypes=num_tokentypes, use_cpu_initialization=use_cpu_initialization, embedding_dropout_prob=hidden_dropout, - add_position_embedding=add_position_embedding, + position_embedding_type=position_embedding_type, ) self.decoder_embedding.zero_parameters() self._decoder_embedding_key = "decoder_embedding" + # TODO (sandeepsub): When implementing RPE for PP > 2, this should not be inside `pre_process`. It should exist on all ranks and be synchronized manually. + if self.position_embedding_type == 'relative': + self.decoder_relative_position_embedding = T5RelativePositionEmbedding( + init_method=init_method_normal(init_method_std), + num_attention_heads=num_attention_heads, + relative_position_num_buckets=relative_attention_num_buckets, + relative_position_max_distance=relative_attention_max_distance, + bidirectional=False, + ) + self._decoder_relative_position_embedding_key = "decoder_relative_position_embedding" + if not self.relative_position_bias_self_attention_only: + self.decoder_cross_attention_relative_position_embedding = T5RelativePositionEmbedding( + init_method=init_method_normal(init_method_std), + num_attention_heads=num_attention_heads, + relative_position_num_buckets=relative_attention_num_buckets, + relative_position_max_distance=relative_attention_max_distance, + bidirectional=True, + ) + self._decoder_cross_attention_relative_position_embedding_key = ( + "decoder_cross_attention_relative_position_embedding" + ) decoder = get_decoder_model( arch=decoder_arch, @@ -240,9 +271,6 @@ def __init__( use_cpu_initialization=use_cpu_initialization, hidden_dropout=hidden_dropout, attention_dropout=attention_dropout, - position_embedding_type=position_embedding_type, - relative_attention_num_buckets=relative_attention_num_buckets, - relative_attention_max_distance=relative_attention_max_distance, precision=precision, fp32_residual_connection=fp32_residual_connection, activations_checkpoint_method=activations_checkpoint_method, @@ -320,11 +348,25 @@ def forward( """ Return value is per token / per dimension (i.e., non collapsed loss value) """ + ( + encoder_self_attention_relative_position_bias, + decoder_self_attention_relative_position_bias, + decoder_cross_attention_relative_position_bias, + ) = (None, None, None) + if enc_input is None: if self.pre_process and self.add_encoder: - # encoder embeddings - enc_position_ids = build_position_ids(enc_input_ids) + # We don't need position ids for RPE, because the embedding layer does not have position embeddings. + if self.position_embedding_type != 'relative': + enc_position_ids = build_position_ids(enc_input_ids) + else: + enc_position_ids = None enc_input = self.encoder_embedding(enc_input_ids, enc_position_ids, token_type_ids=token_type_ids) + + if self.position_embedding_type == 'relative': + encoder_self_attention_relative_position_bias = self.encoder_relative_position_embedding( + query_seq_length=enc_input_ids.size(1), key_seq_length=enc_input_ids.size(1), + ) else: enc_input = None @@ -337,6 +379,17 @@ def forward( if self.pre_process and self.add_decoder: dec_position_ids = build_position_ids(dec_input_ids) dec_input = self.decoder_embedding(dec_input_ids, dec_position_ids, token_type_ids=token_type_ids) + + if self.position_embedding_type == 'relative': + decoder_self_attention_relative_position_bias = self.decoder_relative_position_embedding( + query_seq_length=dec_input_ids.size(1), key_seq_length=dec_input_ids.size(1) + ) + if not self.relative_position_bias_self_attention_only: + decoder_cross_attention_relative_position_bias = self.decoder_cross_attention_relative_position_embedding( + query_seq_length=dec_input_ids.size(1), key_seq_length=enc_input_ids.size(1), + ) + else: + decoder_cross_attention_relative_position_bias = None else: # Note: This is when the decoder itself is split across PP ranks. dec_input = None @@ -351,22 +404,33 @@ def forward( enc_output=None, dec_layer_past=None, dec_get_key_value=False, + enc_self_attention_relative_position_bias=encoder_self_attention_relative_position_bias, + dec_self_attention_relative_position_bias=decoder_self_attention_relative_position_bias, + dec_cross_attention_relative_position_bias=decoder_cross_attention_relative_position_bias, ) if self.post_process and self.add_decoder: - dec_output, enc_output = output + dec_output, enc_output = output # [s, b, h] # project decoder output to vocabulary-size dimensions token_logits = self.tokens_head(dec_output, self.word_embeddings_weight()) - if labels is not None: + # [b, s] -> [s, b] + labels = labels.transpose(0, 1).contiguous() + # tensor_parallel.vocab_parallel_cross_entropy performs log_softmax and return log p(x_i|z) per token i if self.fp16_cross_entropy: assert token_logits.dtype == torch.half tokens_loss = tensor_parallel.vocab_parallel_cross_entropy(token_logits, labels) else: tokens_loss = tensor_parallel.vocab_parallel_cross_entropy(token_logits.float(), labels) + + # [s, b] -> [b, s] + tokens_loss = tokens_loss.transpose(0, 1).contiguous() + return tokens_loss else: + # [s, b, h] -> [b, s, h] + token_logits = token_logits.transpose(0, 1).contiguous() return token_logits elif self.add_decoder and not self.add_encoder: diff --git a/nemo/collections/nlp/modules/common/megatron/transformer.py b/nemo/collections/nlp/modules/common/megatron/transformer.py index 71ee082bc0b4..29e8a1334951 100644 --- a/nemo/collections/nlp/modules/common/megatron/transformer.py +++ b/nemo/collections/nlp/modules/common/megatron/transformer.py @@ -15,6 +15,7 @@ """Transformer.""" import math +from contextlib import nullcontext import torch import torch.nn.functional as F @@ -122,6 +123,8 @@ def __init__( normalization='layernorm', layernorm_epsilon=1e-5, persist_layer_norm=False, + sequence_parallel=False, + gradient_accumulation_fusion=False, ): super(ParallelMLP, self).__init__() self.activation = activation @@ -135,6 +138,9 @@ def __init__( if activation not in ['gelu', 'geglu', 'reglu', 'swiglu']: raise ValueError(f"Activation {activation} not supported. Only gelu, geglu, reglu, swiglu are supported.") + no_async_tensor_model_parallel_allreduce = ( + parallel_state.get_tensor_model_parallel_world_size() == 1 or sequence_parallel + ) # Project to 4h. self.dense_h_to_4h = ColumnLinear( hidden_size, @@ -144,6 +150,9 @@ def __init__( skip_bias_add=True, use_cpu_initialization=use_cpu_initialization, bias=bias, + sequence_parallel_enabled=sequence_parallel, + no_async_tensor_model_parallel_allreduce=no_async_tensor_model_parallel_allreduce, + gradient_accumulation_fusion=gradient_accumulation_fusion, ) if activation in ['geglu', 'reglu', 'swiglu']: @@ -157,6 +166,9 @@ def __init__( skip_bias_add=True, use_cpu_initialization=use_cpu_initialization, bias=bias, + sequence_parallel_enabled=sequence_parallel, + no_async_tensor_model_parallel_allreduce=no_async_tensor_model_parallel_allreduce, + gradient_accumulation_fusion=gradient_accumulation_fusion, ) self.glu_activation_family = activation in ['geglu', 'reglu', 'swiglu'] @@ -205,6 +217,8 @@ def __init__( skip_bias_add=True, use_cpu_initialization=use_cpu_initialization, bias=bias, + sequence_parallel_enabled=sequence_parallel, + gradient_accumulation_fusion=gradient_accumulation_fusion, ) # Normformer normalization @@ -259,10 +273,222 @@ def forward(self, hidden_states): return output, output_bias +class CoreAttention(MegatronModule): + """ Region where selective activation recomputation is applied. + See Figure 3. in Reducing Activation Recomputation in Large Transformer Models + https://arxiv.org/pdf/2205.05198.pdf for more details. + + """ + + def __init__( + self, + layer_number, + num_attention_heads, + hidden_size, + attention_type=AttnType.self_attn, + attn_mask_type=AttnMaskType.padding, + precision=16, + apply_query_key_layer_scaling=True, + kv_channels=None, + masked_softmax_fusion=True, + attention_dropout=0.1, + headscale=False, + sequence_parallel=False, + ): + + super(CoreAttention, self).__init__() + + self.precision = precision + self.fp16 = precision == 16 + self.bf16 = precision == 'bf16' + + self.apply_query_key_layer_scaling = apply_query_key_layer_scaling + self.attention_softmax_in_fp32 = False + if self.apply_query_key_layer_scaling: + self.attention_softmax_in_fp32 = True + self.layer_number = max(1, layer_number) + self.attention_type = attention_type + self.attn_mask_type = attn_mask_type + self.sequence_parallel = sequence_parallel + + if kv_channels is None: + assert ( + hidden_size % num_attention_heads == 0 + ), 'hidden_size must be divisible by num_attention_heads if kv_channels is None' + kv_channels = hidden_size // num_attention_heads + + projection_size = kv_channels * num_attention_heads + + # Per attention head and per partition values. + world_size = parallel_state.get_tensor_model_parallel_world_size() + self.hidden_size_per_partition = safe_divide(projection_size, world_size) + self.hidden_size_per_attention_head = safe_divide(projection_size, num_attention_heads) + self.num_attention_heads_per_partition = safe_divide(num_attention_heads, world_size) + self.num_attention_heads_partition_offset = ( + self.num_attention_heads_per_partition * parallel_state.get_tensor_model_parallel_rank() + ) + + self.headscale = headscale + if headscale: + self.head_scale_tensor = torch.nn.Parameter( + torch.ones(1, self.num_attention_heads_per_partition, 1, 1), requires_grad=True + ) + + coeff = None + self.norm_factor = math.sqrt(self.hidden_size_per_attention_head) + if self.apply_query_key_layer_scaling: + coeff = self.layer_number + self.norm_factor *= coeff + + self.scale_mask_softmax = FusedScaleMaskSoftmax( + self.fp16, + self.bf16, + self.attn_mask_type, + masked_softmax_fusion, + attention_mask_func, + self.attention_softmax_in_fp32, + coeff, + ) + + # Dropout. Note that for a single iteration, this layer will generate + # different outputs on different number of parallel partitions but + # on average it should not be partition dependent. + self.attention_dropout = torch.nn.Dropout(attention_dropout) + + def forward( + self, + query_layer, + key_layer, + value_layer, + attention_mask, + layer_past=None, + get_key_value=False, + rotary_pos_emb=None, + relative_position_bias=None, + ): + + # =================================== + # Raw attention scores. [b, np, s, s] + # =================================== + + # [b, np, sq, sk] + output_size = (query_layer.size(1), query_layer.size(2), query_layer.size(0), key_layer.size(0)) + + # TODO: figure out how to do this + # apply relative positional encoding (rotary embedding) + if rotary_pos_emb is not None: + q_pos_emb, k_pos_emb = rotary_pos_emb + + query_layer = apply_rotary_pos_emb(query_layer, q_pos_emb) + key_layer = apply_rotary_pos_emb(key_layer, k_pos_emb) + # TODO, can apply positional embedding to value_layer so it has + # absolute positional embedding. + # otherwise, only relative positional embedding takes effect + # value_layer = apply_rotary_pos_emb(value_layer, k_pos_emb) + + # [sq, b, np, hn] -> [sq, b * np, hn] + query_layer = query_layer.view(output_size[2], output_size[0] * output_size[1], -1) + # [sk, b, np, hn] -> [sk, b * np, hn] + key_layer = key_layer.view(output_size[3], output_size[0] * output_size[1], -1) + + # preallocting input tensor: [b * np, sq, sk] + matmul_input_buffer = torch.empty( + output_size[0] * output_size[1], + output_size[2], + output_size[3], + dtype=query_layer.dtype, + device=torch.cuda.current_device(), + ) + + # Raw attention scores. [b * np, sq, sk] + matmul_result = torch.baddbmm( + matmul_input_buffer, + query_layer.transpose(0, 1), # [b * np, sq, hn] + key_layer.transpose(0, 1).transpose(1, 2), # [b * np, hn, sk] + beta=0.0, + alpha=(1.0 / self.norm_factor), + ) + + # change view to [b, np, sq, sk] + attention_scores = matmul_result.view(*output_size) + + if relative_position_bias is not None: + attention_scores += relative_position_bias[ + :, + self.num_attention_heads_partition_offset : self.num_attention_heads_partition_offset + + self.num_attention_heads_per_partition, + : attention_scores.size(2), + : attention_scores.size(3), + ] + + # ================================================== + # Update attention mask for inference. [b, np, sq, sk] + # ================================================== + + if get_key_value: + with torch.no_grad(): + if layer_past is not None: + attention_mask = attention_mask[ + ..., attention_scores.size(3) - 1, : attention_scores.size(3) + ].unsqueeze(2) + else: + attention_mask = attention_mask[..., : attention_scores.size(3), : attention_scores.size(3)] + + # =========================== + # Attention probs and dropout + # =========================== + + # attention scores and attention mask [b, np, sq, sk] + attention_probs = self.scale_mask_softmax(attention_scores, attention_mask) + + # This is actually dropping out entire tokens to attend to, which might + # seem a bit unusual, but is taken from the original Transformer paper. + + if not self.sequence_parallel: + with tensor_parallel.random.get_cuda_rng_tracker().fork(): + attention_probs = self.attention_dropout(attention_probs) + else: + attention_probs = self.attention_dropout(attention_probs) + + # ========================= + # Context layer. [sq, b, hp] + # ========================= + + # value_layer -> context layer. + # [sk, b, np, hn] --> [b, np, sq, hn] + + # context layer shape: [b, np, sq, hn] + output_size = (value_layer.size(1), value_layer.size(2), query_layer.size(0), value_layer.size(3)) + + # change view [sk, b * np, hn] + value_layer = value_layer.view(value_layer.size(0), output_size[0] * output_size[1], -1) + + # change view [b * np, sq, sk] + attention_probs = attention_probs.view(output_size[0] * output_size[1], output_size[2], -1) + + # matmul: [b * np, sq, hn] + context_layer = torch.bmm(attention_probs, value_layer.transpose(0, 1)) + + # change view [b, np, sq, hn] + context_layer = context_layer.view(*output_size) + + if self.headscale: + context_layer = context_layer * self.head_scale_tensor + + # [b, np, sq, hn] --> [sq, b, np, hn] + context_layer = context_layer.permute(2, 0, 1, 3).contiguous() + + # [sq, b, np, hn] --> [sq, b, hp] + new_context_layer_shape = context_layer.size()[:-2] + (self.hidden_size_per_partition,) + context_layer = context_layer.view(*new_context_layer_shape) + + return context_layer + + class ParallelAttention(MegatronModule): """Parallel self-attention layer abstract class. - Self-attention layer takes input with size [b, s, h] + Self-attention layer takes input with size [s, b, h] and returns output of the same size. """ @@ -281,27 +507,21 @@ def __init__( use_cpu_initialization=False, masked_softmax_fusion=True, attention_dropout=0.1, - position_embedding_type='learned_absolute', - relative_attention_num_buckets=32, - relative_attention_max_distance=128, layer_type=None, megatron_legacy=False, bias=True, headscale=False, - has_relative_attention_bias=False, + activations_checkpoint_granularity=None, + sequence_parallel=False, + gradient_accumulation_fusion=False, ): super(ParallelAttention, self).__init__() - self.apply_query_key_layer_scaling = apply_query_key_layer_scaling - self.attention_softmax_in_fp32 = False - if self.apply_query_key_layer_scaling: - self.attention_softmax_in_fp32 = True self.layer_number = max(1, layer_number) self.attention_type = attention_type self.attn_mask_type = attn_mask_type + self.megatron_legacy = megatron_legacy - self.headscale = headscale - self.has_relative_attention_bias = has_relative_attention_bias if kv_channels is None: assert ( @@ -312,15 +532,15 @@ def __init__( # Per attention head and per partition values. world_size = parallel_state.get_tensor_model_parallel_world_size() - self.hidden_size_per_partition = safe_divide(projection_size, world_size) self.hidden_size_per_attention_head = safe_divide(projection_size, num_attention_heads) self.num_attention_heads_per_partition = safe_divide(num_attention_heads, world_size) + self.num_attention_heads_partition_offset = ( + self.num_attention_heads_per_partition * parallel_state.get_tensor_model_parallel_rank() + ) - # Headscale - if headscale: - self.head_scale_tensor = torch.nn.Parameter( - torch.ones(1, self.num_attention_heads_per_partition, 1, 1), requires_grad=True - ) + no_async_tensor_model_parallel_allreduce = ( + parallel_state.get_tensor_model_parallel_world_size() == 1 or sequence_parallel + ) # Strided linear layer. if attention_type == AttnType.self_attn: @@ -331,39 +551,49 @@ def __init__( init_method=init_method, use_cpu_initialization=use_cpu_initialization, bias=bias, + sequence_parallel_enabled=sequence_parallel, + no_async_tensor_model_parallel_allreduce=no_async_tensor_model_parallel_allreduce, + gradient_accumulation_fusion=gradient_accumulation_fusion, ) else: assert attention_type == AttnType.cross_attn self.query = ColumnLinear( - hidden_size, projection_size, gather_output=False, init_method=init_method, bias=bias + hidden_size, + projection_size, + gather_output=False, + init_method=init_method, + bias=bias, + sequence_parallel_enabled=sequence_parallel, + no_async_tensor_model_parallel_allreduce=no_async_tensor_model_parallel_allreduce, + gradient_accumulation_fusion=gradient_accumulation_fusion, ) self.key_value = ColumnLinear( - hidden_size, 2 * projection_size, gather_output=False, init_method=init_method, bias=bias + hidden_size, + 2 * projection_size, + gather_output=False, + init_method=init_method, + bias=bias, + sequence_parallel_enabled=sequence_parallel, + no_async_tensor_model_parallel_allreduce=no_async_tensor_model_parallel_allreduce, + gradient_accumulation_fusion=gradient_accumulation_fusion, ) - coeff = None - self.norm_factor = math.sqrt(self.hidden_size_per_attention_head) - if self.apply_query_key_layer_scaling: - coeff = self.layer_number - self.norm_factor *= coeff - - fused_fp16 = precision == 16 - fused_bf16 = precision == 'bf16' - self.scale_mask_softmax = FusedScaleMaskSoftmax( - fused_fp16, - fused_bf16, - self.attn_mask_type, - masked_softmax_fusion, - attention_mask_func, - self.attention_softmax_in_fp32, - coeff, + self.core_attention = CoreAttention( + layer_number=self.layer_number, + num_attention_heads=num_attention_heads, + hidden_size=hidden_size, + attention_type=self.attention_type, + attn_mask_type=self.attn_mask_type, + precision=precision, + apply_query_key_layer_scaling=apply_query_key_layer_scaling, + kv_channels=kv_channels, + masked_softmax_fusion=masked_softmax_fusion, + attention_dropout=attention_dropout, + headscale=headscale, + sequence_parallel=sequence_parallel, ) - - # Dropout. Note that for a single iteration, this layer will generate - # different outputs on different number of parallel partitions but - # on average it should not be partition dependent. - self.attention_dropout = torch.nn.Dropout(attention_dropout) + self.checkpoint_core_attention = activations_checkpoint_granularity == 'selective' # Output. self.dense = tensor_parallel.RowParallelLinear( @@ -374,6 +604,8 @@ def __init__( skip_bias_add=True, use_cpu_initialization=use_cpu_initialization, bias=bias, + sequence_parallel_enabled=sequence_parallel, + gradient_accumulation_fusion=gradient_accumulation_fusion, ) # Inference key-value memory @@ -382,15 +614,43 @@ def __init__( self.inference_current_sequence_len = 0 # relative position embedding - self.position_embedding_type = position_embedding_type - self.relative_attention_num_buckets = relative_attention_num_buckets - self.relative_attention_max_distance = relative_attention_max_distance - if self.position_embedding_type == 'relative' and self.has_relative_attention_bias: - self.relative_attention_bias = torch.nn.Embedding( - relative_attention_num_buckets, self.num_attention_heads_per_partition - ).to(torch.cuda.current_device()) self.layer_type = layer_type + def _checkpointed_attention_forward( + self, query_layer, key_layer, value_layer, attention_mask, rotary_pos_emb=None, relative_position_bias=None + ): + """Forward method with activation checkpointing.""" + + def custom_forward(*inputs): + query_layer = inputs[0] + key_layer = inputs[1] + value_layer = inputs[2] + attention_mask = inputs[3] + rotary_pos_emb = inputs[4] + relative_position_bias = inputs[5] + output_ = self.core_attention( + query_layer, + key_layer, + value_layer, + attention_mask, + rotary_pos_emb=rotary_pos_emb, + relative_position_bias=relative_position_bias, + ) + return output_ + + hidden_states = tensor_parallel.checkpoint( + custom_forward, + False, + query_layer, + key_layer, + value_layer, + attention_mask, + rotary_pos_emb, + relative_position_bias, + ) + + return hidden_states + def _allocate_memory(self, inference_max_sequence_len, batch_size, dtype): return torch.empty( inference_max_sequence_len, @@ -435,80 +695,6 @@ def _transpose_last_dim(self, mixed_layer, num_splits, num_splits_first): return mixed_layer - @staticmethod - def _relative_position_bucket(relative_position, bidirectional=True, num_buckets=32, max_distance=128): - """ - Adapted from HuggingFace T5 Model: - https://github.com/huggingface/transformers/blob/main/src/transformers/models/t5/modeling_t5.py - """ - - """ - Adapted from Mesh Tensorflow: - https://github.com/tensorflow/mesh/blob/0cb87fe07da627bf0b7e60475d59f95ed6b5be3d/mesh_tensorflow/transformer/transformer_layers.py#L593 - Translate relative position to a bucket number for relative attention. The relative position is defined as - memory_position - query_position, i.e. the distance in tokens from the attending position to the attended-to - position. If bidirectional=False, then positive relative positions are invalid. We use smaller buckets for - small absolute relative_position and larger buckets for larger absolute relative_positions. All relative - positions >=max_distance map to the same bucket. All relative positions <=-max_distance map to the same bucket. - This should allow for more graceful generalization to longer sequences than the model has been trained on - Args: - relative_position: an int32 Tensor - bidirectional: a boolean - whether the attention is bidirectional - num_buckets: an integer - max_distance: an integer - Returns: - a Tensor with the same shape as relative_position, containing int32 values in the range [0, num_buckets) - """ - relative_buckets = 0 - if bidirectional: - num_buckets //= 2 - relative_buckets += (relative_position > 0).to(torch.long) * num_buckets - relative_position = torch.abs(relative_position) - else: - relative_position = -torch.min(relative_position, torch.zeros_like(relative_position)) - # now relative_position is in the range [0, inf) - - # half of the buckets are for exact increments in positions - max_exact = num_buckets // 2 - is_small = relative_position < max_exact - - # The other half of the buckets are for logarithmically bigger bins in positions up to max_distance - relative_postion_if_large = max_exact + ( - torch.log(relative_position.float() / max_exact) - / math.log(max_distance / max_exact) - * (num_buckets - max_exact) - ).to(torch.long) - relative_postion_if_large = torch.min( - relative_postion_if_large, torch.full_like(relative_postion_if_large, num_buckets - 1) - ) - - relative_buckets += torch.where(is_small, relative_position, relative_postion_if_large) - return relative_buckets - - def compute_bias(self, query_length, key_length): - """ - Adapted from HuggingFace T5 Model: - https://github.com/huggingface/transformers/blob/main/src/transformers/models/t5/modeling_t5.py - """ - - """Compute binned relative position bias""" - context_position = torch.arange( - query_length, dtype=torch.long, device=self.relative_attention_bias.weight.device - )[:, None] - memory_position = torch.arange( - key_length, dtype=torch.long, device=self.relative_attention_bias.weight.device - )[None, :] - relative_position = memory_position - context_position # shape (query_length, key_length) - relative_position_bucket = self._relative_position_bucket( - relative_position, # shape (query_length, key_length) - bidirectional=(self.attention_type != AttnMaskType.causal), # self.is_decoder and self_attention. - num_buckets=self.relative_attention_num_buckets, - max_distance=self.relative_attention_max_distance, - ) - values = self.relative_attention_bias(relative_position_bucket) # shape (query_length, key_length, num_heads) - values = values.permute([2, 0, 1]).unsqueeze(0) # shape (1, num_heads, query_length, key_length) - return values - def forward( self, hidden_states, @@ -519,7 +705,7 @@ def forward( set_inference_key_value_memory=False, inference_max_sequence_len=None, rotary_pos_emb=None, # rotary positional embedding - position_bias=None, + relative_position_bias=None, ): # hidden_states: [sq, b, h] @@ -535,6 +721,7 @@ def forward( inference_max_sequence_len, hidden_states.size(1), hidden_states.dtype ) self.inference_current_sequence_len = 0 + # Some consistency check. if inference_max_sequence_len: assert self.inference_current_sequence_len < self.inference_key_memory.size(0) @@ -545,6 +732,7 @@ def forward( if not inference_max_sequence_len: self.inference_key_memory = None self.inference_value_memory = None + # ===================== # Query, Key, and Value # ===================== @@ -617,143 +805,34 @@ def forward( k_pos_emb = k_pos_emb[:end, :, :, :] rotary_pos_emb = (q_pos_emb, k_pos_emb) - real_seq_length = hidden_states.shape[0] - key_length = key_layer.shape[0] - if layer_past is not None: past_key, past_value = layer_past key_layer = torch.cat((past_key.type_as(key_layer), key_layer), dim=0) value_layer = torch.cat((past_value.type_as(value_layer), value_layer), dim=0) - if get_key_value: - present = (key_layer, value_layer) - - # =================================== - # Raw attention scores. [b, np, s, s] - # =================================== - - # [b, np, sq, sk] - output_size = (query_layer.size(1), query_layer.size(2), query_layer.size(0), key_layer.size(0)) - - # apply relative positional encoding (rotary embedding) - if rotary_pos_emb is not None: - q_pos_emb, k_pos_emb = rotary_pos_emb - - query_layer = apply_rotary_pos_emb(query_layer, q_pos_emb) - key_layer = apply_rotary_pos_emb(key_layer, k_pos_emb) - # TODO, can apply positional embedding to value_layer so it has - # absolute positional embedding. - # otherwise, only relative positional embedding takes effect - # value_layer = apply_rotary_pos_emb(value_layer, k_pos_emb) - - # [sq, b, np, hn] -> [sq, b * np, hn] - query_layer = query_layer.view(output_size[2], output_size[0] * output_size[1], -1) - # [sk, b, np, hn] -> [sk, b * np, hn] - key_layer = key_layer.view(output_size[3], output_size[0] * output_size[1], -1) - - # preallocting result tensor: [b * np, sq, sk] - matmul_result = torch.empty( - output_size[0] * output_size[1], - output_size[2], - output_size[3], - dtype=query_layer.dtype, - device=torch.cuda.current_device(), - ) - - # Raw attention scores. [b * np, sq, sk] - matmul_result = torch.baddbmm( - matmul_result, - query_layer.transpose(0, 1), # [b * np, sq, hn] - key_layer.transpose(0, 1).transpose(1, 2), # [b * np, hn, sk] - beta=0.0, - alpha=(1.0 / self.norm_factor), - ) - - # change view to [b, np, sq, sk] - attention_scores = matmul_result.view(*output_size) - - # ================================================== - # Update attention mask for inference. [b, np, sq, sk] - # ================================================== if get_key_value: - with torch.no_grad(): - if layer_past is not None: - attention_mask = attention_mask[ - ..., attention_scores.size(3) - 1, : attention_scores.size(3) - ].unsqueeze(2) - else: - attention_mask = attention_mask[..., : attention_scores.size(3), : attention_scores.size(3)] - - if position_bias is None: - if self.position_embedding_type == 'relative': - if self.has_relative_attention_bias: - position_bias = self.compute_bias(real_seq_length, key_length) - elif attention_mask is not None: - position_bias = torch.zeros_like(attention_mask).to(torch.cuda.current_device()) - else: - position_bias = torch.zeros(1, key_length, key_length).to(torch.cuda.current_device()) - - # if key and values are already calculated - # we want only the last query position bias - if layer_past is not None: - position_bias = position_bias[:, :, -hidden_states.size(0) :, :] - - if self.position_embedding_type == 'relative': - position_bias = position_bias + attention_mask - attention_scores += position_bias - - # =========================== - # Attention probs and dropout - # =========================== - - # attention scores and attention mask [b, np, sq, sk] - attention_probs = self.scale_mask_softmax(attention_scores, attention_mask) - - # Currently if all key sequences are masked, attention will be uniformly distributed. - # E.g. attention_mask last dimension all True, attention prob is 1 / last_dim - # # The correct behavior should be paying zero attention to them - # issue is created at https://github.com/NVIDIA/apex/issues/1390 - # following is a work around: - # all_k_masked = attention_mask.all(axis=-1) - # zero_attention_mask = (1.0 - all_k_masked.float())[:, :, :, None] - # attention_probs = attention_probs * zero_attention_mask - - # This is actually dropping out entire tokens to attend to, which might - # seem a bit unusual, but is taken from the original Transformer paper. - with tensor_parallel.random.get_cuda_rng_tracker().fork(): - attention_probs = self.attention_dropout(attention_probs) - - # ========================= - # Context layer. [sq, b, hp] - # ========================= - - # value_layer -> context layer. - # [sk, b, np, hn] --> [b, np, sq, hn] - - # context layer shape: [b, np, sq, hn] - output_size = (value_layer.size(1), value_layer.size(2), query_layer.size(0), value_layer.size(3)) - - # change view [sk, b * np, hn] - value_layer = value_layer.view(value_layer.size(0), output_size[0] * output_size[1], -1) - - # change view [b * np, sq, sk] - attention_probs = attention_probs.view(output_size[0] * output_size[1], output_size[2], -1) - - # matmul: [b * np, sq, hn] - context_layer = torch.bmm(attention_probs, value_layer.transpose(0, 1)) - - # change view [b, np, sq, hn] - context_layer = context_layer.view(*output_size) - - if self.headscale: - context_layer = context_layer * self.head_scale_tensor - - # [b, np, sq, hn] --> [sq, b, np, hn] - context_layer = context_layer.permute(2, 0, 1, 3).contiguous() + present = (key_layer, value_layer) - # [sq, b, np, hn] --> [sq, b, hp] - new_context_layer_shape = context_layer.size()[:-2] + (self.hidden_size_per_partition,) - context_layer = context_layer.view(*new_context_layer_shape) + if self.checkpoint_core_attention: + context_layer = self._checkpointed_attention_forward( + query_layer, + key_layer, + value_layer, + attention_mask, + rotary_pos_emb=rotary_pos_emb, + relative_position_bias=relative_position_bias, + ) + else: + context_layer = self.core_attention( + query_layer, + key_layer, + value_layer, + attention_mask, + layer_past=layer_past, + get_key_value=get_key_value, + rotary_pos_emb=rotary_pos_emb, + relative_position_bias=relative_position_bias, + ) # ================= # Output. [sq, b, h] @@ -764,12 +843,10 @@ def forward( if get_key_value: output = [output, present] - if self.position_embedding_type == 'relative': - output = (output,) + (position_bias,) - return output, bias +# TODO: Figure this out class ParallelChunkedCrossAttention(MegatronModule): """Parallel chunked cross-attention layer class. @@ -790,13 +867,11 @@ def __init__( use_cpu_initialization=False, masked_softmax_fusion=True, attention_dropout=0.1, - position_embedding_type='learned_absolute', - relative_attention_num_buckets=32, - relative_attention_max_distance=128, megatron_legacy=False, chunk_size=64, # each chunk, how many tokens bias=True, headscale=False, + gradient_accumulation_fusion=False, ): super(ParallelChunkedCrossAttention, self).__init__() self.cross_attention = ParallelAttention( @@ -813,12 +888,10 @@ def __init__( use_cpu_initialization=use_cpu_initialization, masked_softmax_fusion=masked_softmax_fusion, attention_dropout=attention_dropout, - position_embedding_type=position_embedding_type, - relative_attention_num_buckets=relative_attention_num_buckets, - relative_attention_max_distance=relative_attention_max_distance, megatron_legacy=megatron_legacy, bias=bias, headscale=headscale, + gradient_accumulation_fusion=gradient_accumulation_fusion, ) self.chunk_size = chunk_size @@ -945,7 +1018,7 @@ def _dropout_add(x, bias, residual, prob): class ParallelTransformerLayer_(MegatronModule): """A single transformer layer. - Transformer layer takes input with size [b, s, h] and returns an + Transformer layer takes input with size [s, b, h] and returns an output of the same size. """ @@ -973,9 +1046,6 @@ def __init__( onnx_safe=False, masked_softmax_fusion=True, attention_dropout=0.1, - position_embedding_type='learned_absolute', - relative_attention_num_buckets=32, - relative_attention_max_distance=128, activation='gelu', megatron_legacy=False, bias=True, @@ -983,7 +1053,9 @@ def __init__( normalization='layernorm', transformer_block_type='pre_ln', headscale=False, - has_relative_attention_bias=False, + activations_checkpoint_granularity=None, + sequence_parallel=False, + gradient_accumulation_fusion=False, ): super(ParallelTransformerLayer_, self).__init__() @@ -997,7 +1069,6 @@ def __init__( self.layer_type = layer_type self.bias = bias self.transformer_block_type = transformer_block_type - self.position_embedding_type = position_embedding_type if not bias and bias_dropout_fusion: raise ValueError( @@ -1017,12 +1088,15 @@ def __init__( self.hidden_dropout = hidden_dropout self.attention_dropout = attention_dropout self.bias_dropout_fusion = bias_dropout_fusion # if true, enable bias dropout fusion + # Self attention. # retrieval_decoder_after_self_attn skips the self attention if self.layer_type != LayerType.retrieval_decoder_after_self_attn: # Layernorm on the input data. if normalization == 'layernorm': - self.input_layernorm = get_layer_norm(hidden_size, layernorm_epsilon, persist_layer_norm) + self.input_layernorm = get_layer_norm( + hidden_size, layernorm_epsilon, persist_layer_norm, sequence_parallel + ) else: self.input_layernorm = MixedFusedRMSNorm(hidden_size, layernorm_epsilon) @@ -1040,16 +1114,15 @@ def __init__( use_cpu_initialization=use_cpu_initialization, masked_softmax_fusion=masked_softmax_fusion, attention_dropout=attention_dropout, - position_embedding_type=position_embedding_type, - relative_attention_num_buckets=relative_attention_num_buckets, - relative_attention_max_distance=relative_attention_max_distance, layer_type=layer_type, megatron_legacy=megatron_legacy, bias=bias, headscale=headscale, - has_relative_attention_bias=has_relative_attention_bias, + activations_checkpoint_granularity=activations_checkpoint_granularity, + sequence_parallel=sequence_parallel, + gradient_accumulation_fusion=gradient_accumulation_fusion, ) - # Normformer normalization + if transformer_block_type == 'normformer': if normalization == 'layernorm': self.post_attention_normformer_norm = get_layer_norm( @@ -1062,7 +1135,9 @@ def __init__( # the post_attention_layernorm is used for layermorm after mlp # don't need it for decoder_pre_mlp and post_ln if normalization == 'layernorm': - self.post_attention_layernorm = get_layer_norm(hidden_size, layernorm_epsilon, persist_layer_norm) + self.post_attention_layernorm = get_layer_norm( + hidden_size, layernorm_epsilon, persist_layer_norm, sequence_parallel + ) else: self.post_attention_layernorm = MixedFusedRMSNorm(hidden_size, layernorm_epsilon) @@ -1075,7 +1150,9 @@ def __init__( if self.layer_type == LayerType.retrieval_decoder_after_self_attn and self.transformer_block_type == 'post_ln': # Layernorm on the attention output if normalization == 'layernorm': - self.post_attention_layernorm = get_layer_norm(hidden_size, layernorm_epsilon, persist_layer_norm) + self.post_attention_layernorm = get_layer_norm( + hidden_size, layernorm_epsilon, persist_layer_norm, sequence_parallel + ) else: self.post_attention_layernorm = MixedFusedRMSNorm(hidden_size, layernorm_epsilon) @@ -1094,19 +1171,18 @@ def __init__( use_cpu_initialization=use_cpu_initialization, masked_softmax_fusion=masked_softmax_fusion, attention_dropout=attention_dropout, - position_embedding_type=position_embedding_type, - relative_attention_num_buckets=relative_attention_num_buckets, - relative_attention_max_distance=relative_attention_max_distance, megatron_legacy=megatron_legacy, bias=bias, headscale=headscale, - has_relative_attention_bias=False, + activations_checkpoint_granularity=activations_checkpoint_granularity, + sequence_parallel=sequence_parallel, + gradient_accumulation_fusion=gradient_accumulation_fusion, ) # Normformer normalization if transformer_block_type == 'normformer': if normalization == 'layernorm': self.post_inter_attention_normformer_norm = get_layer_norm( - hidden_size, layernorm_epsilon, persist_layer_norm + hidden_size, layernorm_epsilon, persist_layer_norm, sequence_parallel ) else: self.post_inter_attention_normformer_norm = MixedFusedRMSNorm(hidden_size, layernorm_epsilon) @@ -1114,7 +1190,7 @@ def __init__( # Layernorm on the attention output. if normalization == 'layernorm': self.post_inter_attention_layernorm = get_layer_norm( - hidden_size, layernorm_epsilon, persist_layer_norm + hidden_size, layernorm_epsilon, persist_layer_norm, sequence_parallel ) else: self.post_inter_attention_layernorm = MixedFusedRMSNorm(hidden_size, layernorm_epsilon) @@ -1134,26 +1210,25 @@ def __init__( use_cpu_initialization=use_cpu_initialization, masked_softmax_fusion=masked_softmax_fusion, attention_dropout=attention_dropout, - position_embedding_type=position_embedding_type, - relative_attention_num_buckets=relative_attention_num_buckets, - relative_attention_max_distance=relative_attention_max_distance, megatron_legacy=megatron_legacy, chunk_size=chunk_size, bias=bias, headscale=headscale, + gradient_accumulation_fusion=gradient_accumulation_fusion, ) # Normformer normalization if transformer_block_type == 'normformer': if normalization == 'layernorm': self.post_inter_attention_normformer_norm = get_layer_norm( - hidden_size, layernorm_epsilon, persist_layer_norm + hidden_size, layernorm_epsilon, persist_layer_norm, sequence_parallel ) else: self.post_inter_attention_normformer_norm = MixedFusedRMSNorm(hidden_size, layernorm_epsilon) + # Layernorm on the attention output. if normalization == 'layernorm': self.post_inter_attention_layernorm = get_layer_norm( - hidden_size, layernorm_epsilon, persist_layer_norm + hidden_size, layernorm_epsilon, persist_layer_norm, sequence_parallel ) else: self.post_inter_attention_layernorm = MixedFusedRMSNorm(hidden_size, layernorm_epsilon) @@ -1174,6 +1249,8 @@ def __init__( normalization=normalization, layernorm_epsilon=layernorm_epsilon, persist_layer_norm=persist_layer_norm, + sequence_parallel=sequence_parallel, + gradient_accumulation_fusion=gradient_accumulation_fusion, ) def _get_bias_droput_add_func(self, transformer_block_type='pre_ln', position_after='attention'): @@ -1211,8 +1288,8 @@ def forward( set_inference_key_value_memory=False, inference_max_sequence_len=None, rotary_pos_emb=None, # list of positional embedding tensors, first one self attention, second one and third one are for cross attention (q, k) - position_bias=None, - encoder_decoder_position_bias=None, + self_attention_relative_position_bias=None, + cross_attention_relative_position_bias=None, ): # Self attention. if rotary_pos_emb is not None: @@ -1243,15 +1320,12 @@ def forward( set_inference_key_value_memory=set_inference_key_value_memory, inference_max_sequence_len=inference_max_sequence_len, rotary_pos_emb=self_attention_pos_emb, - position_bias=position_bias, + relative_position_bias=self_attention_relative_position_bias, ) if get_key_value: attention_output, presents = attention_output - if self.position_embedding_type == 'relative': - attention_output, position_bias = attention_output[0], attention_output[1] - # If normformer, apply norm on the output of the self attention. if self.transformer_block_type == 'normformer': # Normformer normalization @@ -1311,10 +1385,9 @@ def forward( enc_dec_attn_mask, encoder_output=encoder_output, rotary_pos_emb=cross_attention_pos_emb, - position_bias=encoder_decoder_position_bias, + relative_position_bias=cross_attention_relative_position_bias, ) - if self.position_embedding_type == 'relative': - attention_output, encoder_decoder_position_bias = attention_output[0], attention_output[1] + # If normformer, apply norm on the output of the self attention. if self.transformer_block_type == 'normformer': # Normformer normalization @@ -1352,12 +1425,6 @@ def forward( if get_key_value: output = [output, presents] - if self.position_embedding_type == 'relative': - if encoder_decoder_position_bias is None: - output = (output,) + (position_bias,) - else: - output = (output,) + (position_bias,) + (encoder_decoder_position_bias,) - return output @@ -1385,8 +1452,8 @@ def forward( get_key_value=False, set_inference_key_value_memory=False, inference_max_sequence_len=None, - position_bias=None, - encoder_decoder_position_bias=None, + self_attention_relative_position_bias=None, + cross_attention_relative_position_bias=None, ): if self.dtype == torch.float32: return super().forward( @@ -1399,8 +1466,8 @@ def forward( set_inference_key_value_memory, inference_max_sequence_len, rotary_pos_emb, - position_bias, - encoder_decoder_position_bias, + self_attention_relative_position_bias, + cross_attention_relative_position_bias, ) with torch.autocast(device_type="cuda", dtype=self.dtype): return super().forward( @@ -1413,8 +1480,8 @@ def forward( set_inference_key_value_memory, inference_max_sequence_len, rotary_pos_emb, - position_bias, - encoder_decoder_position_bias, + self_attention_relative_position_bias, + cross_attention_relative_position_bias, ) @@ -1438,13 +1505,10 @@ def __init__( precision=16, fp32_residual_connection=False, activations_checkpoint_method=None, - activations_checkpoint_num_layers=1, + activations_checkpoint_num_layers=None, layernorm_epsilon=1e-5, hidden_dropout=0.1, attention_dropout=0.1, - position_embedding_type='learned_absolute', - relative_attention_num_buckets=32, - relative_attention_max_distance=128, use_cpu_initialization=False, bias_activation_fusion=True, bias_dropout_fusion=True, @@ -1461,6 +1525,9 @@ def __init__( transformer_block_type='pre_ln', headscale=False, layer_number_offset=0, # this is use only for attention norm_factor scaling + activations_checkpoint_granularity=None, + sequence_parallel=False, + gradient_accumulation_fusion=False, ): super(ParallelTransformer, self).__init__() @@ -1478,11 +1545,38 @@ def __init__( self.model_type = model_type self.normalization = normalization self.transformer_block_type = transformer_block_type - self.position_embedding_type = position_embedding_type - # Store activation checkpointing flag. self.activations_checkpoint_method = activations_checkpoint_method self.activations_checkpoint_num_layers = activations_checkpoint_num_layers + self.activations_checkpoint_granularity = activations_checkpoint_granularity + + if self.activations_checkpoint_granularity: + if self.activations_checkpoint_granularity == 'selective': + if self.activations_checkpoint_num_layers: + raise ValueError( + f'When using selective activation checkpointing, activations_checkpoint_num_layers should be None, got: {activations_checkpoint_num_layers}.' + ) + if self.activations_checkpoint_method: + raise ValueError( + f'When using selective activation checkpointing, activations_checkpoint_method should be None, got: {activations_checkpoint_method}.' + ) + elif self.activations_checkpoint_granularity == 'full': + if self.activations_checkpoint_method in ['uniform', 'block']: + if not self.activations_checkpoint_num_layers: + logging.info( + ( + f'Using uniform or block activation checkpointing requires activations_checkpoint_num_layers to be set.' + f'Got: {self.activations_checkpoint_num_layers}. Setting to 1 by default.' + ) + ) + else: + raise ValueError( + f'activations_checkpoint_method should be "uniform" or "block" when using granularity full.' + ) + else: + raise ValueError(f'activations_checkpoint_granularity should be "selective" or "full".') + + self.sequence_parallel = sequence_parallel if self.model_type == ModelType.encoder_or_decoder: assert ( @@ -1493,7 +1587,7 @@ def __init__( self.num_layers = self.get_num_layers(num_layers) # Transformer layers. - def build_layer(layer_number, has_relative_attention_bias=False): + def build_layer(layer_number): if isinstance(layer_type, list): lt = layer_type[layer_number - 1] else: @@ -1514,9 +1608,6 @@ def build_layer(layer_number, has_relative_attention_bias=False): layernorm_epsilon=layernorm_epsilon, hidden_dropout=hidden_dropout, attention_dropout=attention_dropout, - position_embedding_type=position_embedding_type, - relative_attention_num_buckets=relative_attention_num_buckets, - relative_attention_max_distance=relative_attention_max_distance, use_cpu_initialization=use_cpu_initialization, bias_activation_fusion=bias_activation_fusion, bias_dropout_fusion=bias_dropout_fusion, @@ -1531,7 +1622,9 @@ def build_layer(layer_number, has_relative_attention_bias=False): normalization=normalization, transformer_block_type=transformer_block_type, headscale=headscale, - has_relative_attention_bias=has_relative_attention_bias, + activations_checkpoint_granularity=activations_checkpoint_granularity, + sequence_parallel=sequence_parallel, + gradient_accumulation_fusion=gradient_accumulation_fusion, ) if parallel_state.get_virtual_pipeline_model_parallel_world_size() is not None: @@ -1568,19 +1661,14 @@ def build_layer(layer_number, has_relative_attention_bias=False): else: offset = parallel_state.get_pipeline_model_parallel_rank() * self.num_layers - self.layers = torch.nn.ModuleList( - [ - build_layer( - i + 1 + offset, has_relative_attention_bias=(i == 0) and parallel_state.is_pipeline_first_stage() - ) - for i in range(self.num_layers) - ] - ) + self.layers = torch.nn.ModuleList([build_layer(i + 1 + offset) for i in range(self.num_layers)]) if self.post_process and self.transformer_block_type != 'post_ln': # Final layer norm before output. if normalization == 'layernorm': - self.final_layernorm = get_layer_norm(hidden_size, layernorm_epsilon, persist_layer_norm) + self.final_layernorm = get_layer_norm( + hidden_size, layernorm_epsilon, persist_layer_norm, sequence_parallel=sequence_parallel + ) else: self.final_layernorm = MixedFusedRMSNorm(hidden_size, layernorm_epsilon) @@ -1619,8 +1707,8 @@ def _checkpointed_forward( encoder_output, enc_dec_attn_mask, rotary_pos_emb, - position_bias=None, - encoder_decoder_position_bias=None, + self_attention_relative_position_bias, + cross_attention_relative_position_bias, ): """Forward method with activation checkpointing.""" @@ -1631,8 +1719,8 @@ def custom_forward(*inputs): encoder_output = inputs[2] enc_dec_attn_mask = inputs[3] rotary_pos_emb = inputs[4] - position_bias = inputs[5] - encoder_decoder_position_bias = inputs[6] + self_attention_relative_position_bias = inputs[5] + cross_attention_relative_position_bias = inputs[6] for index in range(start, end): layer = self._get_layer(index) x_ = layer( @@ -1641,16 +1729,9 @@ def custom_forward(*inputs): encoder_output, enc_dec_attn_mask, rotary_pos_emb, - position_bias=position_bias, - encoder_decoder_position_bias=encoder_decoder_position_bias, + self_attention_relative_position_bias, + cross_attention_relative_position_bias, ) - if type(x_) is tuple: - if len(x_) == 2: - x_, position_bias = x_ - elif len(x_) == 3: - x_, position_bias, encoder_decoder_position_bias = x_ - else: - raise IndexError('Hidden_states (x_) needs to be tuple containing 2 or 3 elements.') return x_ return custom_forward @@ -1666,13 +1747,14 @@ def custom_forward(*inputs): while l < self.num_layers: hidden_states = tensor_parallel.checkpoint( custom(l, l + self.activations_checkpoint_num_layers), + False, hidden_states, attention_mask, encoder_output, enc_dec_attn_mask, rotary_pos_emb, - position_bias, - encoder_decoder_position_bias, + self_attention_relative_position_bias, + cross_attention_relative_position_bias, ) l += self.activations_checkpoint_num_layers elif self.activations_checkpoint_method == 'block': @@ -1683,13 +1765,14 @@ def custom_forward(*inputs): if l < self.activations_checkpoint_num_layers: hidden_states = tensor_parallel.checkpoint( custom(l, l + 1), + False, hidden_states, attention_mask, encoder_output, enc_dec_attn_mask, rotary_pos_emb, - position_bias, - encoder_decoder_position_bias, + self_attention_relative_position_bias, + cross_attention_relative_position_bias, ) else: hidden_states = custom(l, l + 1)( @@ -1698,8 +1781,8 @@ def custom_forward(*inputs): encoder_output, enc_dec_attn_mask, rotary_pos_emb, - position_bias, - encoder_decoder_position_bias, + self_attention_relative_position_bias, + cross_attention_relative_position_bias, ) else: raise ValueError("Invalid activation checkpoint method.") @@ -1728,10 +1811,9 @@ def forward( inference_max_sequence_len=None, rotary_pos_emb=None, # list of positional embedding tensors, first one self attention, second one and third one are for cross attention (q, k) retrieved_emb=None, # tensor of retrieved embedding of shape [b, k, r, n, d] + self_attention_relative_position_bias=None, + cross_attention_relative_position_bias=None, ): - position_bias = None - encoder_decoder_position_bias = None - # Checks. if inference_max_sequence_len: assert self.activations_checkpoint_method is None, 'inference does not work with activation checkpointing' @@ -1743,84 +1825,62 @@ def forward( 'get_key_value does not work with ' 'activation checkpointing' ) - if self.pre_process: - # Data format change to avoid explicit tranposes : [b s h] --> [s b h]. - # If the input flag for fp32 residual connection is set, convert for float. - if self.fp32_residual_connection: - hidden_states = hidden_states.transpose(0, 1).contiguous().float() - # Otherwise, leave it as is. - else: - hidden_states = hidden_states.transpose(0, 1).contiguous() - else: + if not self.pre_process: # See set_input_tensor() hidden_states = self.input_tensor - if encoder_output is not None: - encoder_output = encoder_output.transpose(0, 1).contiguous() - elif retrieved_emb is not None: + # TODO: @Yi Dong, what should this be? + if retrieved_emb is not None: assert len(retrieved_emb.shape) == 5 # this is retrieval decoder, need special transpose encoder_output = rearrange(retrieved_emb, 'b k r n d -> k r n b d').contiguous() - if self.activations_checkpoint_method is not None: - hidden_states = self._checkpointed_forward( - hidden_states, - attention_mask, - encoder_output, - enc_dec_attn_mask, - rotary_pos_emb, - position_bias, - encoder_decoder_position_bias, - ) - - if type(hidden_states) is tuple: - if len(hidden_states) == 2: - hidden_states, position_bias = hidden_states - elif len(hidden_states) == 3: - hidden_states, position_bias, encoder_decoder_position_bias = hidden_states - else: - raise IndexError('Hidden_states needs to be tuple containing 2 or 3 elements.') + if self.sequence_parallel: + rng_context = tensor_parallel.random.get_cuda_rng_tracker().fork() else: - if get_key_value: - presents = [] - for index in range(self.num_layers): - layer = self._get_layer(index) - past = None - if layer_past is not None: - past = layer_past[index] - hidden_states = layer( + rng_context = nullcontext() + + with rng_context: + if self.activations_checkpoint_granularity == 'full': + hidden_states = self._checkpointed_forward( hidden_states, attention_mask, - encoder_output=encoder_output, - enc_dec_attn_mask=enc_dec_attn_mask, - layer_past=past, - get_key_value=get_key_value, - set_inference_key_value_memory=set_inference_key_value_memory, - inference_max_sequence_len=inference_max_sequence_len, - rotary_pos_emb=rotary_pos_emb, - position_bias=position_bias, - encoder_decoder_position_bias=encoder_decoder_position_bias, + encoder_output, + enc_dec_attn_mask, + rotary_pos_emb, + self_attention_relative_position_bias, + cross_attention_relative_position_bias, ) + + else: if get_key_value: - hidden_states, present = hidden_states - presents.append(present) - if self.position_embedding_type == 'relative': - if len(hidden_states) == 2: - hidden_states, position_bias = hidden_states - elif len(hidden_states) == 3: - hidden_states, position_bias, encoder_decoder_position_bias = hidden_states - else: - raise IndexError('Hidden_states needs to be tuple containing 2 or 3 elements.') + presents = [] + for index in range(self.num_layers): + layer = self._get_layer(index) + past = None + if layer_past is not None: + past = layer_past[index] + hidden_states = layer( + hidden_states, + attention_mask, + encoder_output=encoder_output, + enc_dec_attn_mask=enc_dec_attn_mask, + layer_past=past, + get_key_value=get_key_value, + set_inference_key_value_memory=set_inference_key_value_memory, + inference_max_sequence_len=inference_max_sequence_len, + rotary_pos_emb=rotary_pos_emb, + self_attention_relative_position_bias=self_attention_relative_position_bias, + cross_attention_relative_position_bias=cross_attention_relative_position_bias, + ) + output = hidden_states # Final layer norm. if self.post_process: - # Reverting data format change [s b h] --> [b s h]. - output = hidden_states.transpose(0, 1).contiguous() # only apply the final_layernorm for pre-ln if self.transformer_block_type != 'post_ln': - output = self.final_layernorm(output) - else: - output = hidden_states + output = self.final_layernorm(hidden_states) + if get_key_value: output = [output, presents] diff --git a/nemo/collections/nlp/modules/common/megatron/utils.py b/nemo/collections/nlp/modules/common/megatron/utils.py index ace05a5023d6..d68fadf7b686 100644 --- a/nemo/collections/nlp/modules/common/megatron/utils.py +++ b/nemo/collections/nlp/modules/common/megatron/utils.py @@ -18,7 +18,6 @@ from typing import Dict, List, Union import torch -import torch.nn.functional as F try: from apex.contrib.layer_norm.layer_norm import FastLayerNorm @@ -26,6 +25,7 @@ from apex.transformer import parallel_state, tensor_parallel from apex.transformer.enums import AttnMaskType from apex.transformer.pipeline_parallel.schedules.common import listify_model + from apex.transformer.tensor_parallel.layers import linear_with_grad_accumulation_and_async_allreduce HAVE_APEX = True except (ImportError, ModuleNotFoundError): @@ -44,20 +44,57 @@ def __getattr__(self, item): return None -def parallel_lm_logits(input_, word_embeddings_weight, parallel_output, bias=None): - """LM logits using word embedding weights.""" - # Parallel logits. - input_parallel = tensor_parallel.copy_to_tensor_model_parallel_region(input_) - # Matrix multiply. - if bias is None: - logits_parallel = F.linear(input_parallel, word_embeddings_weight) +def parallel_lm_logits( + input_: torch.Tensor, + word_embeddings_weight: torch.Tensor, + parallel_output: bool, + bias: torch.Tensor = None, + async_tensor_model_parallel_allreduce: bool = False, + sequence_parallel: bool = False, + gradient_accumulation_fusion: bool = False, +): + """Language Model logits using word embedding weights. + + Args: + input_ (torch.Tensor): [b, s, h] + word_embeddings_weight (torch.Tensor): [(padded) vocab size, h] + parallel_output (bool): False will gather logits from tensor model parallel region + bias (torch.Tensor, optional): bias tensor. Defaults to None. + async_tensor_model_parallel_allreduce (bool, optional): TODO: understand this flag. Defaults to False. + sequence_parallel (bool, optional): If True will use sequence parallelism. Defaults to False. + gradient_accumulation_fusioa (bool, optional): If True fuse gradient accumulation to WGRAD GEMM + + Returns: + torch.Tensor: [b, s, (padded) vocab size] + """ + + tensor_model_parallel = parallel_state.get_tensor_model_parallel_world_size() > 1 + + # async grad allreduce can only be used when not using sequence parallelism + async_grad_allreduce = async_tensor_model_parallel_allreduce and tensor_model_parallel and not sequence_parallel + + # copy input_ to model parallel region if needed + if async_tensor_model_parallel_allreduce or sequence_parallel: + input_parallel = input_ + else: - logits_parallel = F.linear(input_parallel, word_embeddings_weight, bias) + input_parallel = tensor_parallel.copy_to_tensor_model_parallel_region(input_) + + # Matrix multiply. + logits_parallel = linear_with_grad_accumulation_and_async_allreduce( + input=input_parallel, + weight=word_embeddings_weight, + bias=bias, + gradient_accumulation_fusion=gradient_accumulation_fusion, + async_grad_allreduce=async_grad_allreduce, + sequence_parallel_enabled=sequence_parallel, + ) + # Gather if needed. if parallel_output: return logits_parallel - - return tensor_parallel.gather_from_tensor_model_parallel_region(logits_parallel) + else: + return tensor_parallel.gather_from_tensor_model_parallel_region(logits_parallel) def init_method_normal(sigma): diff --git a/nemo/collections/nlp/parts/nlp_overrides.py b/nemo/collections/nlp/parts/nlp_overrides.py index 43dd9083cac2..3aed204d2244 100644 --- a/nemo/collections/nlp/parts/nlp_overrides.py +++ b/nemo/collections/nlp/parts/nlp_overrides.py @@ -94,33 +94,38 @@ def configure_ddp(self): Sets find_unused_parameters to False to use activation-checkpoint-recomputation. """ - app_state = AppState() + if hasattr(self.model, 'megatron_amp_o2'): + # do not use DDP if using megatron amp O2 + self._model = LightningDistributedModule(self.model) + else: + app_state = AppState() - if app_state.model_parallel_size is not None: - logging.info(f"Configuring DDP for model parallelism.") - - # With model parallelism, multiple GPUs form a large "logical GPU" - # this means that data parallel groups span multiple GPUs - # and are non-trivial - # TODO: for megatron-lm self.model is a list - self.pre_configure_ddp() - # device_ids = self.determine_ddp_device_ids() - self._model = DistributedDataParallel( - LightningDistributedModule(self.model), - process_group=parallel_state.get_data_parallel_group(), - **self._ddp_kwargs, - ) + if app_state.model_parallel_size is not None: - if self.no_ddp_communication_hook: - # When using custom gradient accumulation and allreduce, disable - # DDP communication hook that works on the gradient bucket. - # Instead, use the custom gradient function and communication hook, - # which is defined in the master optimizer wrapper. - self._model.require_backward_grad_sync = False - self._model.register_comm_hook(None, noop_hook) + logging.info(f"Configuring DDP for model parallelism.") + + # With model parallelism, multiple GPUs form a large "logical GPU" + # this means that data parallel groups span multiple GPUs + # and are non-trivial + # TODO: for megatron-lm self.model is a list + self.pre_configure_ddp() + # device_ids = self.determine_ddp_device_ids() + self._model = DistributedDataParallel( + LightningDistributedModule(self.model), + process_group=parallel_state.get_data_parallel_group(), + **self._ddp_kwargs, + ) - else: - super().configure_ddp() + if self.no_ddp_communication_hook: + # When using custom gradient accumulation and allreduce, disable + # DDP communication hook that works on the gradient bucket. + # Instead, use the custom gradient function and communication hook, + # which is defined in the master optimizer wrapper. + self._model.require_backward_grad_sync = False + self._model.register_comm_hook(None, noop_hook) + + else: + super().configure_ddp() def init_model_parallel(self, global_rank: int, world_size: int) -> None: """ Initializes Megatron-LM model parallel if using model parallelism. diff --git a/nemo/core/optim/optimizer_with_main_params.py b/nemo/core/optim/optimizer_with_main_params.py index 1cb9e06de336..342d3dcf45d4 100644 --- a/nemo/core/optim/optimizer_with_main_params.py +++ b/nemo/core/optim/optimizer_with_main_params.py @@ -299,7 +299,7 @@ def _make_param_hook(self, param, main_param, i, grad_chunk_info): # Hook used for back-prop. def param_hook(*unused): # Accumulates gradients on main gradients - if param.grad.data is not None: + if param.grad is not None: if main_param.grad is None: main_param.grad = param.grad.float() else: @@ -408,6 +408,12 @@ def reload_model_params(self): @torch.no_grad() def step(self, **kwargs): + # while async grad allreduce is enabled, bprop will keep moving forward without waiting for + # the finish of async grad AR works. Hence, to guarantee the correctness of grads reduction, + # we cannot start weight update until all async grad AR works are done. + if self._async_grad_allreduce: + torch.cuda.synchronize() + # Step the optimizer. self.optimizer.step(closure=None, **kwargs) diff --git a/tests/collections/nlp/test_gpt_model.py b/tests/collections/nlp/test_gpt_model.py index bf0c25a24210..9e125c1a1a4e 100644 --- a/tests/collections/nlp/test_gpt_model.py +++ b/tests/collections/nlp/test_gpt_model.py @@ -232,6 +232,8 @@ def test_forward(self, gpt_model, test_text): attention_mask=attn_mask.cuda(), labels=None, ) + # output is [b s h] + assert output_tensor.shape[0] == 1 assert output_tensor.shape[1] == tokens.shape[1] assert output_tensor.shape[2] == gpt_model.padded_vocab_size assert output_tensor.dtype == dtype diff --git a/tests/collections/nlp/test_retrieval_module.py b/tests/collections/nlp/test_retrieval_module.py index 2cc509996533..60fe81356a0d 100644 --- a/tests/collections/nlp/test_retrieval_module.py +++ b/tests/collections/nlp/test_retrieval_module.py @@ -249,7 +249,7 @@ def test_retrieval_decoder(self): .half() ) out = decoder(hidden_emb, hidden_mask, retrieved_attn_mask=context_mask, retrieved_emb=retrieved_emb) - assert out.shape == torch.Size([batch, input_length, dim]) + assert out.shape == torch.Size([input_length, batch, dim]) @pytest.mark.unit def test_encoder_decoder_module(self): diff --git a/tests/collections/nlp/test_retrieval_module_inference.py b/tests/collections/nlp/test_retrieval_module_inference.py index 3acc6fdceb84..437679c37478 100644 --- a/tests/collections/nlp/test_retrieval_module_inference.py +++ b/tests/collections/nlp/test_retrieval_module_inference.py @@ -470,7 +470,7 @@ def test_retrieval_decoder_inference(self): .half() ) out = decoder(hidden_emb, hidden_mask, retrieved_attn_mask=context_mask, retrieved_emb=retrieved_emb) - assert out.shape == torch.Size([batch, input_length, dim]) + assert out.shape == torch.Size([input_length, batch, dim]) out_1 = decoder( hidden_emb[:, :62], @@ -480,7 +480,7 @@ def test_retrieval_decoder_inference(self): set_inference_key_value_memory=True, inference_max_sequence_len=input_length, ) - assert (out[:, :62] - out_1[:, :62]).abs().max().item() < 1e-2 + assert (out[:62] - out_1[:62]).abs().max().item() < 1e-2 out_1 = decoder( hidden_emb[:, 62:63], hidden_mask[:, :63], @@ -489,7 +489,7 @@ def test_retrieval_decoder_inference(self): set_inference_key_value_memory=False, inference_max_sequence_len=input_length, ) - assert (out[:, 62] - out_1[:, 0]).abs().max().item() < 1e-2 + assert (out[62] - out_1[0]).abs().max().item() < 1e-2 out_2 = decoder( hidden_emb[:, 63:64], hidden_mask[:, :64], @@ -498,7 +498,7 @@ def test_retrieval_decoder_inference(self): set_inference_key_value_memory=False, inference_max_sequence_len=input_length, ) - assert (out[:, 63] - out_2[:, 0]).abs().max().item() < 1e-2 + assert (out[63] - out_2[0]).abs().max().item() < 1e-2 for i in range(64, 127): out_2 = decoder( hidden_emb[:, i : i + 1], @@ -508,7 +508,7 @@ def test_retrieval_decoder_inference(self): set_inference_key_value_memory=False, inference_max_sequence_len=input_length, ) - assert (out[:, i] - out_2[:, 0]).abs().max().item() < 1e-2 + assert (out[i] - out_2[0]).abs().max().item() < 1e-2 for i in range(127, 191): out_3 = decoder( hidden_emb[:, i : i + 1], @@ -518,7 +518,7 @@ def test_retrieval_decoder_inference(self): set_inference_key_value_memory=False, inference_max_sequence_len=input_length, ) - assert (out[:, i] - out_3[:, 0]).abs().max().item() < 1e-2 + assert (out[i] - out_3[0]).abs().max().item() < 1e-2 out_1 = decoder( hidden_emb[:, :130], @@ -528,7 +528,7 @@ def test_retrieval_decoder_inference(self): set_inference_key_value_memory=True, inference_max_sequence_len=input_length, ) - assert (out[:, :130] - out_1[:, :130]).abs().max().item() < 1e-2 + assert (out[:130] - out_1[:130]).abs().max().item() < 1e-2 for i in range(130, 191): out_3 = decoder( hidden_emb[:, i : i + 1], @@ -538,7 +538,7 @@ def test_retrieval_decoder_inference(self): set_inference_key_value_memory=False, inference_max_sequence_len=input_length, ) - assert (out[:, i] - out_3[:, 0]).abs().max().item() < 1e-2 + assert (out[i] - out_3[0]).abs().max().item() < 1e-2 @pytest.mark.unit def test_encoder_decoder_module_inference(self): From aa0a98c4aa188b1a3e52a491767d662590aab97f Mon Sep 17 00:00:00 2001 From: Somshubra Majumdar Date: Tue, 26 Jul 2022 15:14:03 -0700 Subject: [PATCH 47/52] Update Offline ASR with CTC Decoding (#4608) * Fix word boundaries Signed-off-by: smajumdar * Remove prints Signed-off-by: smajumdar * Update notebook Signed-off-by: smajumdar --- nemo/collections/asr/metrics/wer.py | 6 +- tutorials/asr/Offline_ASR.ipynb | 151 +++++++++++++++++++++++++++- 2 files changed, 154 insertions(+), 3 deletions(-) diff --git a/nemo/collections/asr/metrics/wer.py b/nemo/collections/asr/metrics/wer.py index d1cb46b4602e..4a14e30aae73 100644 --- a/nemo/collections/asr/metrics/wer.py +++ b/nemo/collections/asr/metrics/wer.py @@ -523,6 +523,7 @@ def _get_word_offsets_subwords_sentencepiece( """ word_offsets = [] built_token = [] + previous_token_index = 0 # For every collapsed sub-word token for i, char in enumerate(hypothesis.text): # Compute the sub-word text representation, and the decoded text (stripped of sub-word markers). @@ -538,14 +539,15 @@ def _get_word_offsets_subwords_sentencepiece( word_offsets.append( { "word": decode_tokens_to_str(built_token), - "start_offset": offsets[i]["start_offset"], - "end_offset": offsets[i]["end_offset"], + "start_offset": offsets[previous_token_index]["start_offset"], + "end_offset": offsets[i]["start_offset"], } ) # Prepare list of new sub-word ids built_token.clear() built_token.append(char) + previous_token_index = i else: # If the token does not contain any sub-word start mark, then the sub-word has not completed yet # Append to current sub-word list. diff --git a/tutorials/asr/Offline_ASR.ipynb b/tutorials/asr/Offline_ASR.ipynb index 6d239f70f20c..0274e1ec57a9 100644 --- a/tutorials/asr/Offline_ASR.ipynb +++ b/tutorials/asr/Offline_ASR.ipynb @@ -491,14 +491,163 @@ "execution_count": null, "outputs": [] }, + { + "cell_type": "markdown", + "source": [ + "# Greedy Decoding Time Stamps\n", + "\n", + "While the above approach works well for character based CTC models, it requires careful tuning of offset parameter as well as computation of the word time stamp offsets.\n", + "\n", + "We therefore provide a simple way to obtain greedy decoding word time stamps directly using the familiar \"model.transcribe()\" method, which works quite well for character and subword models.\n", + "\n", + "**Note**: We find that larger models that have converged to strong scores on the dataset usually have better word alignments. If evaluated on a completely out of domain audio sample, it might produce very poor time stamps." + ], + "metadata": { + "id": "LPtMzLE4T7T-" + } + }, + { + "cell_type": "code", + "source": [ + "from omegaconf import OmegaConf, open_dict" + ], + "metadata": { + "id": "z_0pO-TaUIHU" + }, + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "source": [ + "For the purposes of this demonstration, we will use Conformer CTC Large, a 120 M parameter model trained on thousands of hours of English speech." + ], + "metadata": { + "id": "i0Epb8D-rW3-" + } + }, + { + "cell_type": "code", + "source": [ + "asr_model_subword = nemo_asr.models.ASRModel.from_pretrained(\"stt_en_conformer_ctc_large\")" + ], + "metadata": { + "id": "Ky7OpuikbBTb" + }, + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "source": [ + "## CTC Decoding Strategy\n", + "\n", + "NeMo CTC models have an internal decoding strategy that can be updated after training. In our case, we will enable the greedy decoding step to compute word time stamps, as well as preserve the log probability predictions." + ], + "metadata": { + "id": "vwN6wddTrhno" + } + }, { "cell_type": "code", "metadata": { "id": "ubpcxp6z3ZF-" }, "source": [ - "" + "decoding_cfg = asr_model_subword.cfg.decoding\n", + "print(OmegaConf.to_yaml(decoding_cfg))" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "source": [ + "decoding_cfg.preserve_alignments = True\n", + "decoding_cfg.compute_timestamps = True\n", + "asr_model_subword.change_decoding_strategy(decoding_cfg)" + ], + "metadata": { + "id": "pKUsMlUbUAxv" + }, + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "source": [ + "Next, we simply transcribe the audio file, and pass the flag `return_hypotheses=True`. This will return a list of `Hypothesis` objects instead of the predicted text." + ], + "metadata": { + "id": "EdX0Drncr8Yl" + } + }, + { + "cell_type": "code", + "source": [ + "hypothesis = asr_model_subword.transcribe([AUDIO_FILENAME], return_hypotheses=True)[0]" + ], + "metadata": { + "id": "SUkfIyYzUbaB" + }, + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "source": [ + "print(\"Greedy prediction :\", hypothesis.text)" ], + "metadata": { + "id": "duaxOSPXUmQ0" + }, + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "source": [ + "## Hypothesis - Time Stamps\n", + "\n", + "Since we previously set the flag for `decoding_cfg.compute_timestamps`, the hypothesis now contains a dictionary in it, accessed via `hypothesis.timestep`. This dictionary contains multiple useful lists, detailing the time step at which some token was predicted, the character / subword / word time stamps." + ], + "metadata": { + "id": "_5hfsiDGsM19" + } + }, + { + "cell_type": "code", + "source": [ + "timestamp_dict = hypothesis.timestep\n", + "print(\"Hypothesis contains following timestep information :\", list(timestamp_dict.keys()))" + ], + "metadata": { + "id": "vh7K_9D1UrQp" + }, + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "source": [ + "# 40ms is duration of a timestep at output of the Conformer\n", + "time_stride = 4 * model.cfg.preprocessor.window_stride\n", + "\n", + "##################################################################\n", + "\n", + "word_timestamps = timestamp_dict['word']\n", + "\n", + "for stamp in word_timestamps:\n", + " start = stamp['start_offset'] * time_stride\n", + " end = stamp['end_offset'] * time_stride\n", + " word = stamp['char'] if 'char' in stamp else stamp['word']\n", + "\n", + " print(f\"Time : {start:0.2f} - {end:0.2f} - {word}\")\n", + " display(Audio(signal[int(start * sample_rate) : int(end * sample_rate)], rate=sample_rate))" + ], + "metadata": { + "id": "fogttpCTVTEZ" + }, "execution_count": null, "outputs": [] } From cbf3f666adca5f09bdd8bc0adfcfc541f406c8ac Mon Sep 17 00:00:00 2001 From: Anas Abou Allaban Date: Tue, 26 Jul 2022 20:19:42 -0400 Subject: [PATCH 48/52] normalize_batch error msg (#4614) Signed-off-by: Anas Abou Allaban --- nemo/collections/asr/parts/preprocessing/features.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/nemo/collections/asr/parts/preprocessing/features.py b/nemo/collections/asr/parts/preprocessing/features.py index dd306e07f9ba..d7bf8ba1e9f8 100644 --- a/nemo/collections/asr/parts/preprocessing/features.py +++ b/nemo/collections/asr/parts/preprocessing/features.py @@ -55,7 +55,8 @@ def normalize_batch(x, seq_len, normalize_type): if x[i, :, : seq_len[i]].shape[1] == 1: raise ValueError( "normalize_batch with `per_feature` normalize_type received a tensor of length 1. This will result " - "in torch.std() returning nan" + "in torch.std() returning nan. Make sure your audio length has enough samples for a single " + "feature (ex. at least `hop_length` for Mel Spectrograms)." ) x_mean[i, :] = x[i, :, : seq_len[i]].mean(dim=1) x_std[i, :] = x[i, :, : seq_len[i]].std(dim=1) From 90ad5afe84f3b2c23a226957c20a15cd4373d7f8 Mon Sep 17 00:00:00 2001 From: Somshubra Majumdar Date: Wed, 27 Jul 2022 15:12:53 -0700 Subject: [PATCH 49/52] Support listing Hugging Face model info (#4619) * Support listing Hugging Face model info Signed-off-by: smajumdar * Add documentation about usage Signed-off-by: smajumdar * Add documentation about usage Signed-off-by: smajumdar * Update name of method, support list of model filters Signed-off-by: smajumdar * Improve docstring Signed-off-by: smajumdar --- nemo/core/classes/common.py | 127 +++++++++++++++++++++++++++++++- tests/core/test_save_restore.py | 51 +++++++++++++ 2 files changed, 175 insertions(+), 3 deletions(-) diff --git a/nemo/core/classes/common.py b/nemo/core/classes/common.py index 8e00347beaee..f9e7bf49f94c 100644 --- a/nemo/core/classes/common.py +++ b/nemo/core/classes/common.py @@ -23,12 +23,12 @@ from enum import Enum from functools import total_ordering from pathlib import Path -from typing import Dict, List, Optional, Union +from typing import Dict, Iterable, List, Optional, Union import hydra import wrapt -from huggingface_hub import hf_hub_download -from huggingface_hub.hf_api import HfFolder +from huggingface_hub import HfApi, HfFolder, ModelFilter, hf_hub_download +from huggingface_hub.hf_api import ModelInfo from omegaconf import DictConfig, OmegaConf import nemo @@ -667,6 +667,105 @@ def list_available_models(cls) -> Optional[PretrainedModelInfo]: """ pass + @classmethod + def search_huggingface_models( + cls, model_filter: Optional[Union[ModelFilter, List[ModelFilter]]] = None + ) -> List[ModelInfo]: + """ + Should list all pre-trained models available via Hugging Face Hub. + + The following metadata can be passed via the `model_filter` for additional results. + Metadata: + resolve_card_info: Bool flag, if set, returns the model card metadata. Default: False. + limit_results: Optional int, limits the number of results returned. + + .. code-block:: python + + # You can replace with any subclass of ModelPT. + from nemo.core import ModelPT + + # Get default ModelFilter + filt = .get_hf_model_filter() + + # Make any modifications to the filter as necessary + filt.language = [...] + filt.task = ... + filt.tags = [...] + + # Add any metadata to the filter as needed + filt.limit_results = 5 + + # Obtain model info + model_infos = .search_huggingface_models(model_filter=filt) + + # Browse through cards and select an appropriate one + card = model_infos[0] + + # Restore model using `modelId` of the card. + model = ModelPT.from_pretrained(card.modelId) + + Args: + model_filter: Optional ModelFilter or List[ModelFilter] (from Hugging Face Hub) + that filters the returned list of compatible model cards, and selects all results from each filter. + Users can then use `model_card.modelId` in `from_pretrained()` to restore a NeMo Model. + If no ModelFilter is provided, uses the classes default filter as defined by `get_hf_model_filter()`. + + Returns: + A list of ModelInfo entries. + """ + # Resolve model filter if not provided as argument + if model_filter is None: + model_filter = cls.get_hf_model_filter() + + # If single model filter, wrap into list + if not isinstance(model_filter, Iterable): + model_filter = [model_filter] + + # Inject `nemo` library filter + for mfilter in model_filter: + if isinstance(mfilter.library, str) and mfilter.library != 'nemo': + logging.warning(f"Model filter's `library` tag updated be `nemo`. Original value: {mfilter.library}") + mfilter.library = "nemo" + + elif isinstance(mfilter, Iterable) and 'nemo' not in mfilter.library: + logging.warning( + f"Model filter's `library` list updated to include `nemo`. Original value: {mfilter.library}" + ) + mfilter.library = list(mfilter) + mfilter.library.append('nemo') + + # Check if api token exists, use if it does + is_token_available = HfFolder.get_token() is not None + + # Search for all valid models after filtering + api = HfApi() + + # Setup extra arguments for model filtering + all_results = [] # type: List[ModelInfo] + + for mfilter in model_filter: + cardData = None + limit = None + + if hasattr(mfilter, 'resolve_card_info') and mfilter.resolve_card_info is True: + cardData = True + + if hasattr(mfilter, 'limit_results') and mfilter.limit_results is not None: + limit = mfilter.limit_results + + results = api.list_models( + filter=mfilter, + use_auth_token=is_token_available, + sort="lastModified", + direction=-1, + cardData=cardData, + limit=limit, + ) # type: List[ModelInfo] + + all_results.extend(results) + + return all_results + @classmethod def get_available_model_names(cls) -> List[str]: """ @@ -680,6 +779,28 @@ def get_available_model_names(cls) -> List[str]: model_names = [model.pretrained_model_name for model in cls.list_available_models()] return model_names + @classmethod + def get_hf_model_filter(cls) -> ModelFilter: + """ + Generates a filter for HuggingFace models. + + Additionally includes default values of some metadata about results returned by the Hub. + + Metadata: + resolve_card_info: Bool flag, if set, returns the model card metadata. Default: False. + limit_results: Optional int, limits the number of results returned. + + Returns: + A Hugging Face Hub ModelFilter object. + """ + model_filter = ModelFilter(library='nemo') + + # Attach some additional info + model_filter.resolve_card_info = False + model_filter.limit_results = None + + return model_filter + @classmethod def from_pretrained( cls, diff --git a/tests/core/test_save_restore.py b/tests/core/test_save_restore.py index 197b85e273bb..15ff3a1fb8bd 100644 --- a/tests/core/test_save_restore.py +++ b/tests/core/test_save_restore.py @@ -19,6 +19,7 @@ import pytest import torch +from huggingface_hub.hf_api import ModelFilter, ModelInfo from omegaconf import DictConfig, OmegaConf, open_dict from nemo.collections.asr.models import EncDecCTCModel, EncDecCTCModelBPE @@ -605,3 +606,53 @@ class MockModelV2(MockModel): restored_state_dict = restored_model.state_dict() for orig, restored in zip(original_state_dict.keys(), restored_state_dict.keys()): assert (original_state_dict[orig] - restored_state_dict[restored]).abs().mean() < 1e-6 + + @pytest.mark.unit + def test_hf_model_filter(self): + filt = ModelPT.get_hf_model_filter() + assert isinstance(filt, ModelFilter) + assert filt.library == 'nemo' + + @pytest.mark.with_downloads() + @pytest.mark.unit + def test_hf_model_info(self): + filt = ModelPT.get_hf_model_filter() + + # check no override results + model_infos = ModelPT.search_huggingface_models(model_filter=None) + assert len(model_infos) > 0 + + # check with default override results (should match above) + default_model_infos = ModelPT.search_huggingface_models(model_filter=filt) + assert len(model_infos) == len(default_model_infos) + + @pytest.mark.with_downloads() + @pytest.mark.unit + def test_hf_model_info_with_card_data(self): + filt = ModelPT.get_hf_model_filter() + + # check no override results + model_infos = ModelPT.search_huggingface_models(model_filter=filt) + assert len(model_infos) > 0 + assert not hasattr(model_infos[0], 'cardData') + + # check overriden defaults + filt.resolve_card_info = True + model_infos = ModelPT.search_huggingface_models(model_filter=filt) + assert len(model_infos) > 0 + assert hasattr(model_infos[0], 'cardData') and model_infos[0].cardData is not None + + @pytest.mark.with_downloads() + @pytest.mark.unit + def test_hf_model_info_with_limited_results(self): + filt = ModelPT.get_hf_model_filter() + + # check no override results + model_infos = ModelPT.search_huggingface_models(model_filter=filt) + assert len(model_infos) > 0 + + # check overriden defaults + filt.limit_results = 5 + new_model_infos = ModelPT.search_huggingface_models(model_filter=filt) + assert len(new_model_infos) <= 5 + assert len(new_model_infos) < len(model_infos) From 2841c28c5551f4cb990e2f477b9697171287252c Mon Sep 17 00:00:00 2001 From: Ryan Langman Date: Wed, 27 Jul 2022 17:38:38 -0700 Subject: [PATCH 50/52] [TTS] Fix off-by-1 bug in Beta Binomial Prior (#4616) Signed-off-by: Ryan --- nemo/collections/tts/torch/helpers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nemo/collections/tts/torch/helpers.py b/nemo/collections/tts/torch/helpers.py index 1f9989cd0ab4..81931713b889 100644 --- a/nemo/collections/tts/torch/helpers.py +++ b/nemo/collections/tts/torch/helpers.py @@ -55,7 +55,7 @@ def beta_binomial_prior_distribution(phoneme_count, mel_count, scaling_factor=1. mel_text_probs = [] for i in range(1, mel_count + 1): a, b = scaling_factor * i, scaling_factor * (mel_count + 1 - i) - mel_i_prob = betabinom(phoneme_count, a, b).pmf(x) + mel_i_prob = betabinom(phoneme_count - 1, a, b).pmf(x) mel_text_probs.append(mel_i_prob) return np.array(mel_text_probs) From 16c96ba765da52ed6c21fc18ca99d02495113bfa Mon Sep 17 00:00:00 2001 From: Taejin Park Date: Thu, 28 Jul 2022 00:33:26 -0700 Subject: [PATCH 51/52] Update diarization data loader to train meeting data (#4567) * Update audio_to_diar_label to train meeting data Signed-off-by: Taejin Park * Style fix with --scope=nemo Signed-off-by: Taejin Park * Style fix problem, re-run style fix Signed-off-by: Taejin Park * Removed remaining commented lines Signed-off-by: Taejin Park * Remove an unused variable Signed-off-by: Taejin Park * Reflected comments Signed-off-by: Taejin Park * style fixed Signed-off-by: Taejin Park * style fix for no reason Signed-off-by: Taejin Park Co-authored-by: Nithin Rao --- .../asr/data/audio_to_diar_label.py | 100 +++++++++--------- .../asr/parts/utils/speaker_utils.py | 26 ++++- .../common/parts/preprocessing/collections.py | 32 +++--- 3 files changed, 91 insertions(+), 67 deletions(-) diff --git a/nemo/collections/asr/data/audio_to_diar_label.py b/nemo/collections/asr/data/audio_to_diar_label.py index 00b7d81448ee..c5b317032509 100644 --- a/nemo/collections/asr/data/audio_to_diar_label.py +++ b/nemo/collections/asr/data/audio_to_diar_label.py @@ -32,7 +32,7 @@ def get_scale_mapping_list(uniq_timestamps): given base-scale segment. For each scale and each segment, a base-scale segment is assigned. Args: - uniq_timestamps: (Dict) + uniq_timestamps: (dict) The dictionary containing embeddings, timestamps and multiscale weights. If uniq_timestamps contains only one scale, single scale diarization is performed. @@ -61,7 +61,7 @@ def get_scale_mapping_list(uniq_timestamps): return scale_mapping_argmat -def extract_seg_info_from_rttm(uniq_id, rttm_lines, emb_dict=None, target_spks=None): +def extract_seg_info_from_rttm(uniq_id, rttm_lines, mapping_dict=None, target_spks=None): """ Get RTTM lines containing speaker labels, start time and end time. target_spks contains two targeted speaker indices for creating groundtruth label files. Only speakers in target_spks variable will be @@ -72,7 +72,10 @@ def extract_seg_info_from_rttm(uniq_id, rttm_lines, emb_dict=None, target_spks=N Unique file ID that refers to an input audio file and corresponding RTTM (Annotation) file. rttm_lines (list): List containing RTTM lines in str format. - + mapping_dict (dict): + Mapping between the estimated speakers and the speakers in the ground-truth annotation. + ``mapping_dict`` variable is only provided when the inference mode is running in sequence-eval mode. + Sequence eval mode uses the mapping between the estimated speakers and the speakers in ground-truth annotation. Returns: rttm_tup (tuple): Tuple containing lists of start time, end time and speaker labels. @@ -80,14 +83,11 @@ def extract_seg_info_from_rttm(uniq_id, rttm_lines, emb_dict=None, target_spks=N """ stt_list, end_list, speaker_list, pairwise_infer_spks = [], [], [], [] if target_spks: - label_scale_idx = max(emb_dict.keys()) - mapping_dict = emb_dict[label_scale_idx][uniq_id]['mapping'] inv_map = {v: k for k, v in mapping_dict.items()} for spk_idx in target_spks: spk_str = f'speaker_{spk_idx}' if spk_str in inv_map: pairwise_infer_spks.append(inv_map[spk_str]) - for rttm_line in rttm_lines: start, end, speaker = convert_rttm_line(rttm_line) if target_spks is None or speaker in pairwise_infer_spks: @@ -98,7 +98,7 @@ def extract_seg_info_from_rttm(uniq_id, rttm_lines, emb_dict=None, target_spks=N return rttm_tup -def assign_frame_level_spk_vector(rttm_timestamps, max_spks, round_digits, frame_per_sec, target_spks, min_spks=2): +def assign_frame_level_spk_vector(rttm_timestamps, round_digits, frame_per_sec, target_spks, min_spks=2): """ Create a multi-dimensional vector sequence containing speaker timestamp information in RTTM. The unit-length is the frame shift length of the acoustic feature. The feature-level annotations @@ -108,9 +108,6 @@ def assign_frame_level_spk_vector(rttm_timestamps, max_spks, round_digits, frame rttm_timestamps (list): List containing start and end time for each speaker segment label. stt_list, end_list and speaker_list are contained. - max_spks(int): - The maximum number of speakers that the diariziation model can handle. max_spks limits the number of speakers in the - ground-truth label. frame_per_sec (int): Number of feature frames per second. This quantity is determined by window_stride variable in preprocessing module. target_spks (tuple): @@ -128,10 +125,6 @@ def assign_frame_level_spk_vector(rttm_timestamps, max_spks, round_digits, frame sorted_speakers = sorted(list(set(speaker_list))) total_fr_len = int(max(end_list) * (10 ** round_digits)) spk_num = max(len(sorted_speakers), min_spks) - if spk_num > max_spks: - raise ValueError( - f"Number of speaker {spk_num} should be less than or equal to maximum number of speakers: {max_spks}" - ) speaker_mapping_dict = {rttm_key: x_int for x_int, rttm_key in enumerate(sorted_speakers)} fr_level_target = torch.zeros(total_fr_len, spk_num) @@ -141,13 +134,7 @@ def assign_frame_level_spk_vector(rttm_timestamps, max_spks, round_digits, frame stt, end = round(stt, round_digits), round(end, round_digits) spk = speaker_mapping_dict[spk_rttm_key] stt_fr, end_fr = int(round(stt, 2) * frame_per_sec), int(round(end, round_digits) * frame_per_sec) - if target_spks is None: - fr_level_target[stt_fr:end_fr, spk] = 1 - else: - if spk in target_spks: - idx = target_spks.index(spk) - fr_level_target[stt_fr:end_fr, idx] = 1 - + fr_level_target[stt_fr:end_fr, spk] = 1 return fr_level_target @@ -188,8 +175,7 @@ class _AudioMSDDTrainDataset(Dataset): @property def output_types(self) -> Optional[Dict[str, NeuralType]]: - """Returns definitions of module output ports. - """ + """Returns definitions of module output ports.""" output_types = { "features": NeuralType(('B', 'T'), AudioSignal()), "feature_length": NeuralType(('B'), LengthsType()), @@ -274,7 +260,7 @@ def assign_labels_to_longer_segs(self, uniq_id, base_scale_clus_label): per_scale_clus_label = torch.tensor(per_scale_clus_label) return per_scale_clus_label, uniq_scale_mapping - def get_diar_target_labels(self, uniq_id, fr_level_target): + def get_diar_target_labels(self, uniq_id, sample, fr_level_target): """ Convert frame-level diarization target variable into segment-level target variable. Since the granularity is reduced from frame level (10ms) to segment level (100ms~500ms), we need a threshold value, soft_label_thres, which determines @@ -283,6 +269,8 @@ def get_diar_target_labels(self, uniq_id, fr_level_target): Args: uniq_id (str): Unique file ID that refers to an input audio file and corresponding RTTM (Annotation) file. + sample: + ``DiarizationSpeechLabel`` instance containing sample information such as audio filepath and RTTM filepath. fr_level_target (torch.tensor): Tensor containing label for each feature-level frame. @@ -291,7 +279,7 @@ def get_diar_target_labels(self, uniq_id, fr_level_target): Tensor containing binary speaker labels for base-scale segments. base_clus_label (torch.tensor): Representative speaker label for each segment. This variable only has one speaker label for each base-scale segment. - + -1 means that there is no corresponding speaker in the target_spks tuple. """ seg_target_list, base_clus_label = [], [] self.scale_n = len(self.multiscale_timestamp_dict[uniq_id]['scale_dict']) @@ -300,16 +288,23 @@ def get_diar_target_labels(self, uniq_id, fr_level_target): line_split = line.split() seg_stt, seg_end = float(line_split[0]), float(line_split[1]) seg_stt_fr, seg_end_fr = int(seg_stt * self.frame_per_sec), int(seg_end * self.frame_per_sec) - soft_label_vec = torch.sum(fr_level_target[seg_stt_fr:seg_end_fr, :], axis=0) / (seg_end_fr - seg_stt_fr) - label_int = torch.argmax(soft_label_vec) + soft_label_vec_sess = torch.sum(fr_level_target[seg_stt_fr:seg_end_fr, :], axis=0) / ( + seg_end_fr - seg_stt_fr + ) + label_int_sess = torch.argmax(soft_label_vec_sess) + soft_label_vec = soft_label_vec_sess.unsqueeze(0)[:, sample.target_spks].squeeze() + if label_int_sess in sample.target_spks and torch.sum(soft_label_vec_sess) > 0: + label_int = sample.target_spks.index(label_int_sess) + else: + label_int = -1 label_vec = (soft_label_vec > self.soft_label_thres).float() seg_target_list.append(label_vec.detach()) - base_clus_label.append(label_int.detach()) + base_clus_label.append(label_int) seg_target = torch.stack(seg_target_list) - base_clus_label = torch.stack(base_clus_label) + base_clus_label = torch.tensor(base_clus_label) return seg_target, base_clus_label - def parse_rttm_for_ms_targets(self, sample, target_spks=None): + def parse_rttm_for_ms_targets(self, sample): """ Generate target tensor variable by extracting groundtruth diarization labels from an RTTM file. This function converts (start, end, speaker_id) format into base-scale (the finest scale) segment level @@ -320,14 +315,14 @@ def parse_rttm_for_ms_targets(self, sample, target_spks=None): Args: sample: - DiarizationSpeechLabel instance containing sample information such as audio filepath and RTTM filepath. + ``DiarizationSpeechLabel`` instance containing sample information such as audio filepath and RTTM filepath. target_spks (tuple): Speaker indices that are generated from combinations. If there are only one or two speakers, only a single target_spks tuple is generated. Returns: clus_label_index (torch.tensor): - Groundtruth Clustering label (cluster index for each segment) from RTTM files for training purpose. + Groundtruth clustering label (cluster index for each segment) from RTTM files for training purpose. seg_target (torch.tensor): Tensor variable containing hard-labels of speaker activity in each base-scale segment. scale_mapping (torch.tensor): @@ -339,9 +334,9 @@ def parse_rttm_for_ms_targets(self, sample, target_spks=None): uniq_id = self.get_uniq_id_with_range(sample) rttm_timestamps = extract_seg_info_from_rttm(uniq_id, rttm_lines) fr_level_target = assign_frame_level_spk_vector( - rttm_timestamps, self.max_spks, self.round_digits, self.frame_per_sec, target_spks=None + rttm_timestamps, self.round_digits, self.frame_per_sec, target_spks=sample.target_spks ) - seg_target, base_clus_label = self.get_diar_target_labels(uniq_id, fr_level_target) + seg_target, base_clus_label = self.get_diar_target_labels(uniq_id, sample, fr_level_target) clus_label_index, scale_mapping = self.assign_labels_to_longer_segs(uniq_id, base_clus_label) return clus_label_index, seg_target, scale_mapping @@ -353,7 +348,7 @@ def get_uniq_id_with_range(self, sample, deci=3): Args: sample: - DiarizationSpeechLabel instance from collections. + ``DiarizationSpeechLabel`` instance from collections. Returns: uniq_id (str): @@ -373,7 +368,7 @@ def get_ms_seg_timestamps(self, sample): Args: sample: - DiarizationSpeechLabel instance from preprocessing.collections + ``DiarizationSpeechLabel`` instance from preprocessing.collections Returns: ms_seg_timestamps (torch.tensor): Tensor containing Multiscale segment timestamps. @@ -415,8 +410,8 @@ def __getitem__(self, index): ms_seg_timestamps, ms_seg_counts = self.get_ms_seg_timestamps(sample) if self.random_flip: torch.manual_seed(index) - flip = torch.randperm(self.max_spks) - clus_label_index, targets = flip[clus_label_index], targets[:, flip] + flip = torch.cat([torch.randperm(self.max_spks), torch.tensor(-1).unsqueeze(0)]) + clus_label_index, targets = flip[clus_label_index], targets[:, flip[: self.max_spks]] return features, feature_length, ms_seg_timestamps, ms_seg_counts, clus_label_index, scale_mapping, targets @@ -437,11 +432,11 @@ class _AudioMSDDInferDataset(Dataset): Args: manifest_filepath (str): Path to input manifest json files. - emb_dict (Dict): + emb_dict (dict): Dictionary containing cluster-average embeddings and speaker mapping information. - emb_seq (Dict): + emb_seq (dict): Dictionary containing multiscale speaker embedding sequence, scale mapping and corresponding segment timestamps. - clus_label_dict (Dict): + clus_label_dict (dict): Subsegment-level (from base-scale) speaker labels from clustering results. soft_label_thres (float): A threshold that determines the label of each segment based on RTTM file information. @@ -459,8 +454,7 @@ class _AudioMSDDInferDataset(Dataset): @property def output_types(self) -> Optional[Dict[str, NeuralType]]: - """Returns definitions of module output ports. - """ + """Returns definitions of module output ports.""" output_types = OrderedDict( { "ms_emb_seq": NeuralType(('B', 'T', 'C', 'D'), SpectrogramType()), @@ -507,11 +501,12 @@ def __init__( def __len__(self): return len(self.collection) - def parse_rttm_multiscale(self, sample, target_spks=None): + def parse_rttm_multiscale(self, sample): """ Generate target tensor variable by extracting groundtruth diarization labels from an RTTM file. - This function converts (start, end, speaker_id) format into base-scale (the finest scale) segment level - diarization label in a matrix form. + This function is only used when ``self.seq_eval_mode=True`` and RTTM files are provided. This function converts + (start, end, speaker_id) format into base-scale (the finest scale) segment level diarization label in a matrix + form to create target matrix. Args: sample: @@ -527,9 +522,10 @@ def parse_rttm_multiscale(self, sample, target_spks=None): raise ValueError(f"RTTM file is not provided for this sample {sample}") rttm_lines = open(sample.rttm_file).readlines() uniq_id = os.path.splitext(os.path.basename(sample.rttm_file))[0] - rttm_timestamps = extract_seg_info_from_rttm(uniq_id, rttm_lines, self.emb_dict, target_spks) + mapping_dict = self.emb_dict[max(self.emb_dict.keys())][uniq_id]['mapping'] + rttm_timestamps = extract_seg_info_from_rttm(uniq_id, rttm_lines, mapping_dict, sample.target_spks) fr_level_target = assign_frame_level_spk_vector( - rttm_timestamps, self.max_spks, self.round_digits, self.frame_per_sec, target_spks + rttm_timestamps, self.round_digits, self.frame_per_sec, sample.target_spks ) seg_target = self.get_diar_target_labels_from_fr_target(uniq_id, fr_level_target) return seg_target @@ -597,7 +593,7 @@ def __getitem__(self, index): feats_len = feats_out.shape[0] if self.seq_eval_mode: - targets = self.parse_rttm_multiscale(sample, self.collection[index].target_spks) + targets = self.parse_rttm_multiscale(sample) else: targets = torch.zeros(feats_len, 2).float() @@ -798,11 +794,11 @@ class AudioToSpeechMSDDInferDataset(_AudioMSDDInferDataset): Args: manifest_filepath (str): Path to input manifest json files. - emb_dict (Dict): + emb_dict (dict): Dictionary containing cluster-average embeddings and speaker mapping information. - emb_seq (Dict): + emb_seq (dict): Dictionary containing multiscale speaker embedding sequence, scale mapping and corresponding segment timestamps. - clus_label_dict (Dict): + clus_label_dict (dict): Subsegment-level (from base-scale) speaker labels from clustering results. soft_label_thres (float): Threshold that determines speaker labels of segments depending on the overlap with groundtruth speaker timestamps. diff --git a/nemo/collections/asr/parts/utils/speaker_utils.py b/nemo/collections/asr/parts/utils/speaker_utils.py index ddf6df0bffcc..a75e50f77cc2 100644 --- a/nemo/collections/asr/parts/utils/speaker_utils.py +++ b/nemo/collections/asr/parts/utils/speaker_utils.py @@ -359,6 +359,26 @@ def rttm_to_labels(rttm_filename): return labels +def get_rttm_speaker_index(rttm_labels): + """ + Generate speaker mapping between integer index to RTTM speaker label names. + + Args: + rttm_labels (list): + List containing string type RTTM lines + Returns: + speaker_mapping_dict (dict): + Dictionary containing the mapping between integer index and RTTM speaker labels. + """ + speaker_set = set() + for rttm_line in rttm_labels: + spk_str = rttm_line.split()[-1] + speaker_set.add(spk_str) + speaker_list = sorted(list(speaker_set)) + speaker_mapping_dict = {key: val for key, val in enumerate(speaker_list)} + return speaker_mapping_dict + + def write_cluster_labels(base_scale_idx, lines_cluster_labels, out_rttm_dir): """ Write cluster labels that are generated from clustering into a file. @@ -615,9 +635,9 @@ def isOverlap(rangeA, rangeB): def validate_vad_manifest(AUDIO_RTTM_MAP, vad_manifest): """ - This function will check the valid speech segments in the manifest file which is either - generated from NeMo voice activity detection(VAD) or oracle VAD. - If an audio file does not contain any valid speech segments, we ignore the audio file + This function will check the valid speech segments in the manifest file which is either + generated from NeMo voice activity detection(VAD) or oracle VAD. + If an audio file does not contain any valid speech segments, we ignore the audio file (indexed by uniq_id) for the rest of the processing steps. """ vad_uniq_ids = set() diff --git a/nemo/collections/common/parts/preprocessing/collections.py b/nemo/collections/common/parts/preprocessing/collections.py index c1a846b86c03..6a035f55a769 100644 --- a/nemo/collections/common/parts/preprocessing/collections.py +++ b/nemo/collections/common/parts/preprocessing/collections.py @@ -20,6 +20,7 @@ import pandas as pd +from nemo.collections.asr.parts.utils.speaker_utils import get_rttm_speaker_index, rttm_to_labels from nemo.collections.common.parts.preprocessing import manifest, parsers from nemo.utils import logging @@ -209,7 +210,13 @@ def __init__(self, manifests_files: Union[str, List[str]], *args, **kwargs): **kwargs: Kwargs to pass to `AudioText` constructor. """ - ids, audio_files, durations, texts, offsets, = [], [], [], [], [] + ids, audio_files, durations, texts, offsets, = ( + [], + [], + [], + [], + [], + ) speakers, orig_srs, token_labels, langs = [], [], [], [] for item in manifest.item_iter(manifests_files): ids.append(item['id']) @@ -415,7 +422,7 @@ def __init__( super().__init__(data) def relative_speaker_parser(self, seq_label): - """ Convert sequence of speaker labels to relative labels. + """Convert sequence of speaker labels to relative labels. Convert sequence of absolute speaker to sequence of relative speaker [E A C A E E C] -> [0 1 2 1 0 0 2] In this seq of label , if label do not appear before, assign new relative labels len(pos); else reuse previous assigned relative labels. Args: @@ -628,7 +635,7 @@ def __init__( seq_eval_mode (bool): If True, F1 score will be calculated for each speaker pair during inference mode. pairwise_infer (bool): - If True, this Dataset class operates in inference mode. In inference mode, a set of speakers in the input audio + If True, this dataset class operates in inference mode. In inference mode, a set of speakers in the input audio is split into multiple pairs of speakers and speaker tuples (e.g. 3 speakers: [(0,1), (1,2), (0,2)]) and then fed into the diarization system to merge the individual results. *args: Args to pass to `SpeechLabel` constructor. @@ -651,6 +658,7 @@ def __init__( ) for item in manifest.item_iter(manifests_files, parse_func=self.__parse_item_rttm): + # Inference mode if self.pairwise_infer: clus_speaker_digits = sorted(list(set([x[2] for x in clus_label_dict[item['uniq_id']]]))) if item['rttm_file']: @@ -664,17 +672,17 @@ def __init__( sess_spk_dict = None rttm_speaker_digits = None - if len(clus_speaker_digits) == 1: - spk_comb_list = [(0, 1)] - else: - spk_comb_list = [x for x in combinations(clus_speaker_digits, 2)] - + # Training mode else: - target_spks = ((0, 1),) - spk_comb_list = [target_spks] + sess_spk_dict = get_rttm_speaker_index(rttm_to_labels(item['rttm_file'])) + target_spks = tuple(sess_spk_dict.keys()) clus_speaker_digits = target_spks - rttm_speaker_digits = None - sess_spk_dict = None + rttm_speaker_digits = target_spks + + if len(clus_speaker_digits) <= 2: + spk_comb_list = [(0, 1)] + else: + spk_comb_list = [x for x in combinations(clus_speaker_digits, 2)] for target_spks in spk_comb_list: audio_files.append(item['audio_file']) From 96021f4522b33a6c62d9088c6cf3083f02b6543a Mon Sep 17 00:00:00 2001 From: Somshubra Majumdar Date: Thu, 28 Jul 2022 11:02:23 -0700 Subject: [PATCH 52/52] Add Squeezeformer to ASR (#4416) * Initial squeezeformer impl Signed-off-by: smajumdar * Start time reduce and recovery Signed-off-by: smajumdar * Working commit of time reduction and time recovery modules Signed-off-by: smajumdar * Fix issue with number of params being incorrect Signed-off-by: smajumdar * Add initializations to the model Signed-off-by: smajumdar * Fix scheduler Signed-off-by: smajumdar * Remove float() Signed-off-by: smajumdar * Correct order of operations Signed-off-by: smajumdar * Correct order of operations Signed-off-by: smajumdar * Update time reduce PE to only update PE and nothing else Signed-off-by: smajumdar * Fix initialization Signed-off-by: smajumdar * Fix PE usage Signed-off-by: smajumdar * Comment out k2 for now Signed-off-by: smajumdar * Add usage comments to buffered ctc script Signed-off-by: smajumdar * Update docs Signed-off-by: smajumdar * Add squeezeformer configs for CTC Signed-off-by: smajumdar * Mark squeezeformer as experimental Signed-off-by: smajumdar * Add Jenkinsfile test Signed-off-by: smajumdar * Add Jenkinsfile test Signed-off-by: smajumdar * Fix style Signed-off-by: smajumdar * Replace all with /content/ Signed-off-by: smajumdar * Try Jenkinsfile Fix with closure Signed-off-by: smajumdar * Update ctc config Signed-off-by: smajumdar * Update ctc config Signed-off-by: smajumdar * Update ctc config Signed-off-by: smajumdar * Add squeezeformer Signed-off-by: smajumdar * Add squeezeformer Signed-off-by: smajumdar * Fix Jenkinsfile Signed-off-by: smajumdar * Fix Jenkinsfile Signed-off-by: smajumdar * Try closure Signed-off-by: smajumdar * Remove test Signed-off-by: smajumdar * Add back squeezeformer test Signed-off-by: smajumdar * Remvoe script tag Signed-off-by: smajumdar * Update for review comments Signed-off-by: smajumdar * Remove experimental Signed-off-by: smajumdar * Correct an issue with RNNT alignments Signed-off-by: smajumdar * Correct an issue with RNNT metrics Signed-off-by: smajumdar * Code formatting Signed-off-by: smajumdar * Correct offset calculation for no look ahead Signed-off-by: smajumdar --- Jenkinsfile | 33 +- README.rst | 2 +- docs/source/asr/asr_all.bib | 13 +- docs/source/asr/configs.rst | 11 + docs/source/asr/images/squeezeformer.png | Bin 0 -> 582160 bytes docs/source/asr/models.rst | 37 +- .../ctc/speech_to_text_buffered_infer_ctc.py | 16 +- .../squeezeformer/squeezeformer_ctc_bpe.yaml | 201 +++++++++ .../squeezeformer/squeezeformer_ctc_char.yaml | 186 ++++++++ nemo/collections/asr/metrics/rnnt_wer.py | 2 + nemo/collections/asr/metrics/rnnt_wer_bpe.py | 2 + nemo/collections/asr/models/ctc_models.py | 5 +- nemo/collections/asr/models/rnnt_models.py | 10 +- nemo/collections/asr/modules/__init__.py | 1 + .../asr/modules/squeezeformer_encoder.py | 419 ++++++++++++++++++ .../asr/parts/submodules/conformer_modules.py | 59 ++- .../parts/submodules/rnnt_greedy_decoding.py | 10 +- .../parts/submodules/squeezeformer_modules.py | 185 ++++++++ .../asr/parts/submodules/subsampling.py | 137 +++++- .../asr/parts/utils/streaming_utils.py | 9 +- nemo/core/config/schedulers.py | 11 + nemo/core/optim/lr_scheduler.py | 74 ++++ 22 files changed, 1402 insertions(+), 21 deletions(-) create mode 100644 docs/source/asr/images/squeezeformer.png create mode 100644 examples/asr/conf/squeezeformer/squeezeformer_ctc_bpe.yaml create mode 100644 examples/asr/conf/squeezeformer/squeezeformer_ctc_char.yaml create mode 100644 nemo/collections/asr/modules/squeezeformer_encoder.py create mode 100644 nemo/collections/asr/parts/submodules/squeezeformer_modules.py diff --git a/Jenkinsfile b/Jenkinsfile index e390f8181c51..fad895fe6bb9 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -357,6 +357,37 @@ pipeline { } } + stage('L2: ASR dev run - part two') { + when { + anyOf { + branch 'main' + changeRequest target: 'main' + } + } + failFast true + parallel { + stage('L2: Speech to Text WPE - Squeezeformer') { + steps { + sh 'python examples/asr/asr_ctc/speech_to_text_ctc_bpe.py \ + --config-path="../conf/squeezeformer" --config-name="squeezeformer_ctc_bpe" \ + model.train_ds.manifest_filepath=/home/TestData/an4_dataset/an4_train.json \ + model.validation_ds.manifest_filepath=/home/TestData/an4_dataset/an4_val.json \ + model.tokenizer.dir="/home/TestData/asr_tokenizers/an4_wpe_128/" \ + model.tokenizer.type="wpe" \ + model.encoder.d_model=144 \ + model.train_ds.batch_size=4 \ + model.validation_ds.batch_size=4 \ + trainer.devices=[0] \ + trainer.accelerator="gpu" \ + +trainer.fast_dev_run=True \ + exp_manager.exp_dir=examples/asr/speech_to_text_wpe_squeezeformer_results' + sh 'rm -rf examples/asr/speech_to_text_wpe_squeezeformer_results' + } + } + } + } + + stage('L2: Speaker dev run') { when { anyOf { @@ -3000,7 +3031,7 @@ pipeline { 4" sh "rm /home/TestData/nlp/megatron_gpt/TP2/test-increase.nemo" } - } + } } } stage('L2: Megatron T5 Pretraining and Resume Training TP=2') { diff --git a/README.rst b/README.rst index f7bb9c05c28d..f0e6c020fd86 100644 --- a/README.rst +++ b/README.rst @@ -45,7 +45,7 @@ Key Features * Speech processing * `Automatic Speech Recognition (ASR) `_ - * Supported models: Jasper, QuartzNet, CitriNet, Conformer-CTC, Conformer-Transducer, ContextNet, LSTM-Transducer (RNNT), LSTM-CTC, ... + * Supported models: Jasper, QuartzNet, CitriNet, Conformer-CTC, Conformer-Transducer, Squeezeformer-CTC, Squeezeformer-Transducer, ContextNet, LSTM-Transducer (RNNT), LSTM-CTC, ... * Supports CTC and Transducer/RNNT losses/decoders * Beam Search decoding * `Language Modelling for ASR `_: N-gram LM in fusion with Beam Search decoding, Neural Rescoring with Transformer diff --git a/docs/source/asr/asr_all.bib b/docs/source/asr/asr_all.bib index d5f7c3cea352..ebba1095d333 100644 --- a/docs/source/asr/asr_all.bib +++ b/docs/source/asr/asr_all.bib @@ -1045,4 +1045,15 @@ @misc{ssl_inter publisher = {arXiv}, year = {2021}, copyright = {arXiv.org perpetual, non-exclusive license} -} \ No newline at end of file +} + +@misc{kim2022squeezeformer, + doi = {10.48550/ARXIV.2206.00888}, + url = {https://arxiv.org/abs/2206.00888}, + author = {Kim, Sehoon and Gholami, Amir and Shaw, Albert and Lee, Nicholas and Mangalam, Karttikeya and Malik, Jitendra and Mahoney, Michael W. and Keutzer, Kurt}, + keywords = {Audio and Speech Processing (eess.AS), Computation and Language (cs.CL), Sound (cs.SD), FOS: Electrical engineering, electronic engineering, information engineering, FOS: Electrical engineering, electronic engineering, information engineering, FOS: Computer and information sciences, FOS: Computer and information sciences}, + title = {Squeezeformer: An Efficient Transformer for Automatic Speech Recognition}, + publisher = {arXiv}, + year = {2022}, + copyright = {arXiv.org perpetual, non-exclusive license} +} diff --git a/docs/source/asr/configs.rst b/docs/source/asr/configs.rst index 6a1fd45eff63..5eedf9a8099a 100644 --- a/docs/source/asr/configs.rst +++ b/docs/source/asr/configs.rst @@ -502,6 +502,17 @@ specify the tokenizer if you want to use sub-word encoding instead of character- The encoder section includes the details about the Conformer-CTC encoder architecture. You may find more information in the config files and also :doc:`nemo.collections.asr.modules.ConformerEncoder<./api.html#nemo.collections.asr.modules.ConformerEncoder>`. +Squeezeformer-CTC +~~~~~~~~~~~~~~~~~ + +The config files for Squeezeformer-CTC model contain character-based encoding and sub-word encoding at +``/examples/asr/conf/squeezeformer/squeezeformer_ctc_char.yaml`` and ``/examples/asr/conf/squeezeformer/squeezeformer_ctc_bpe.yaml`` +respectively. Components of the configs of `Squeezeformer-CTC <./models.html#Squeezeformer-CTC>`__ are similar to Conformer config - `QuartzNet <./configs.html#Conformer-CTC>`__. + +The encoder section includes the details about the Squeezeformer-CTC encoder architecture. You may find more information in the +config files and also :doc:`nemo.collections.asr.modules.SqueezeformerEncoder<./api.html#nemo.collections.asr.modules.SqueezeformerEncoder>`. + + ContextNet ~~~~~~~~~~ diff --git a/docs/source/asr/images/squeezeformer.png b/docs/source/asr/images/squeezeformer.png new file mode 100644 index 0000000000000000000000000000000000000000..b6e1b218499cf461bab48e832c66c59c741cc1cb GIT binary patch literal 582160 zcmeFZc{r5s|2K>*l~5@{_9EHJZtNjj2-&yFo@MMiV+qNgE!jen>`QioBs(GdzAwYb zIx&X(9QFBrKlktVJiq6@`yR*h*T->~xvp#Gyw2r)zPH!weNE^?6*-dg)aP+sHVscoO-)K&*QUVPFe;RJ6|#mC&CM0fA`{Hh`YxMSK^6DB zy&ukn*Y+Mf=h(%?2_>mkPPjIql1jI<4X?nDg^!J8qqw8Da3sG<^(=I{A4DhPG!F0F z=*Po(alU>p4lgudS#l`xGd^RHx_!kLv3UDkRRvTCe;OL>=rs=kI z2K$zaOS_Wq$Uem>54lG=cdpvI?d3$s(z2?$J%K5aCAa^D9b9vV?!yw6U`oDe-VDLCHjgs+ZcCN;>NM+D$7WuA5HxYKCp<#qJGEoc zi=vMrIJabPMJZ~HX1y1sq8TE)!%(|b{iH8inNjY=&Qp``hg}?M_(VJtoGXvia9Pil zGA;z+>AGi*nU^`M2S6zF5^tK7eLr+E$mJAY>wI6qt<>T@k8-cyk$D?`OR{=k7Xr{TVk-C}I-KO`ZdVy2_9d`Sy^;LGB@8o!Qh=^YkW8TEXTva= zWrLN@L-MSlHXd5TUnt8bRz^lN}hIk>}=H8 z;d2|$$H`?|`squr6O3h?oOR*kX(%GFBl7W_&2VSfB^8^$FmbJFChZO>FD6TT)^l%&!j)CzQYOg?hnH9+jK z#0Mg|#&hIG5z13W?D<>86H6B7p=NJ5Z8 ziLMTtR;1>WPddy;=|xXBbSU>PvCcWCgw{6FUJqiO7xm^5CpHc*YId7*`N?*4`}pGK z*M`fM_!fyd>i1>k(6ej4#K z!ZE^-x|OfcQgmR284+#Ue!L*WlP(%j7grZCArF6mWI(Aov+c)8Dqg-B!jq^jm7AUN zej=-G@cZ}j{@sD=1#JB(S-Kh3syiu5(%n&y#6FV8E7QuwYa~8&dzk#8G8bya_lB19 zd4dZ2Th0v40hNBbZ=Ro4^a}N^bHayg3KR<<1?+l7v zwixi(_1LxD9oQxH$%NO7%Sp&JJb*>{hWS?dr}^hdy=;0Es2G?A|l_o{(I+pv(8pwJn~J{(|xa=?etG*}+~UMKr!VPpN}w7O6P}KMK}B!)ZjQ zq6E^O*7f}?cSsC{EBKx3efQn8p`BLa`}cOfCg++}tpTO?V&+TcG3J-cKAE$2*K|=W zF)j%&<#k0Rx#@l54B(KV$m59AuX{{=N4HDVoM7il*?9kVo(^@p>Xy=$(%z1#+oK;x zo)r!HSik&K>K93zTvyW4uK|lIeTrB$0uh!o8?Hk z>86B^B5Gqrdh7c!Z-}~?Kbus@1-4qNTB!Oj-|y8oaxPLcBCdN=4|hkqE7e=qi}=`h z^LP)TCw2!>5h%lTub!wG$~D39%@@;hFz5;13Fa6^P=&uhxPIPU{Il=#$6qQgMV%?Y ztM=E&8xMRfMJ1ISI2OnmSly)A6m>6O`X}SKbXv)EUE%!6eC-UojQ($d8T2E^>q+bH z?{IMNX!Ze?`qpg+qV~%u583Punn@&t2e7_zpIF%5-?wS9cJ~_ zDziM;S>03daDO+gWY$DGYUA26f1&9_#Mb_NPNalHll#4?=UL>RSRGmE+89jVnXh;J z?24Gza`562PD1e3de}@+PbvAHgrYGw57)>09qTGjii-_KZaxhg z3N>1mS+wamh^9%bWv-Oq=7&Pz_kHglGnS+YX?s6=b?`1XcI-yatIG;#)06f*RyU=r z>~XUfHg)+KlJvgcjbDkh!CpylT)3Ujv7Y*hXR-0oWbRyU;(K9beZCD{HJLu{fvwfL z)oxx6i>}lydlu`Z1l_j$F5T|30*l`C$-AtMJw4TN)zwZgZdZ0AwIe&DwYskd<~z$> zup^BHRsNfLH~U{D6XQ|X=)K^+!fB(sP-3}}S(JU$Pi$sqwrG}4xkF+6)8l9FRS6V# zyIz0kcdym2W?#(a%mfpNxTmY{xykmuY-N`#$k%T%(5zXTTno+XP4r5D>PhP|8xI&= z(&#H5GT$iuls9AT2P5=8(+seD`2CHQ+;sWC(F6d(%04u-`6G#z2)D2(=$2mT8TV2 zKJXBsUTel}0v#1ER)uf26j3!yjnLOO9QbYek4@Xa^kLDy#pR^A$y&pP-1$(bytSnW z-}XZ%6Uq6e=E4xS>m{P<9^YqH`c2|YcB1y^xgk>nH>}nmF+Ot+J71P+SzA-;T%Yac zCJglD^rnUiIc;+w*0-Gemm^b960ntk&7i}uDM|#Fhau!=?eHd%ex3dr7uvwia?je< zhNjPTUGYaexps45TJ_gz*DcTWy$YIkLA%N8Zef~w;r+2_5ElF zs+0b;_=m&ht*H8qsSP>$V6nR0TU&jTt#z&3>1}D=$k!)v`{hSTEoE!V$lfUD`_2{c zYv|1R>;*Gp?e_Ez2lniV()9`Kkd4lB)32#rjYK-2ZhzF@Z~1-5z2>N z(JR8!&36~$zOS*CW8tGA9xJZx?Qqoplm8G=e|&W8drrGU)A#)2;tX9~oVrn598MA( zh-jtS*N;5`ic*i3wQu^LeEGPOsa}WsAt-f@H*FHXQG8_mUYy+3w?Ch&p2z(V()w5< zjH9#Ja53z>>b>Vv$H$fCIMLL&KbbZ*H^;wCpP11+*cZT2j@7hy7<6$q>50?p0K1~= z5=2J~A9q&WtdB~FUS1Gwio!JD3^GUE(yfbk0LjNoGaUtUWo4Y(;P@;KJ}xy50XV`1 z|Khkb|2n>pdkg2x>2*9DoDeG<{NLYE0q@wqXz-71^XvW0t6&@=aCRB|yT8Tz>)XW0 zw`cx3CbR>e;YevnD=2_>4HHK*Gdm|ssPoZ}v){mlbN2GOPB=IeH?jY?3hFn0fcuYF zJ<@U3QGNh1f!cCCF@-)g<8rsP$KD4=)Exp2ZOxpYFu2>=*f~Mm#h6aN0RhL@%iK&1 zr(bcl7Gu&;e#jsVbu?oT;JU+ghe`ZA0|SGoqp3MWUFQDp=HN_>$UG-=H|xb z#>)kDwBWugEG*1@hliVohZB5*)5*im`H4HHofGq~oBVYj88asnM=N`0E2td<_P$S^ zLS3B2n3%8~{p;`7IL+Lx{@s(E)9-14336jU;l9gthx=dm22Dk=S0N9r+|6usWvpz$ znt?vV?+Wk1*&6F;BVw)q4I^179b>Lc(Dwi)*Cj39Wq_3IrR zZKqp2kM7-=&FKFi6HBOY{wDQN@8-nG6l&e6x45_7BE2@G_qgGxmu}by zQLo~ej}f!BDA4<~X8c472N(aM^S!e{utCzoxd+@YEUc=`(-06!_OKw~bXhu{QaH*t1dm z;ez_-OC0|U{7*;m$(r&%9EF(I{>6!9uRoNBpPw|IF6;y&_#ch}c6<=>pRPr%8~1-W z3Lo_TrT=g((5Ow;s?CdBElhv7qR}ohjkp15Z?ZpAQ~Zne@_sXL>EbJ4f2Lu6e&_$A z{Sb(UkD~pOZTY9Vbr!ip`$q-}bl526pAVA`3tZIT`y=ZDMCMPyP({Cj%-Q-w`Ig`;lBfo4CHiOW{RY7BQz_ni|C#3dO9IUyb4Crg z|5L60ZN_@&EDo|AhK~6z%>K z>i-k!{}bx}Nv8IHo#W#x%uK^h;`8(K8TgYb{)TJKoCgVK;R@kx|Nql`+iwS(BML5& z{w+!Sav2~lNJ7&xAKhnQ_J6Yl(7tZ?E`2LfGdYN?ME$&m2n;(LVC~G&A(Kq2X%)gyar7 zRmhdiugpu%v@1s*6`9A%Reb}76I8o{Uh3#+m7PVsY81ZZR8Rd%f#kekCL zS&vwVus!L0#P)>s%HMe^uPQHlI`OJKiCX1hsk`VWFE8G~& z)9y?Y^K#sXs+^w*BugMy6}>z4#OWkT;`mmr+onKeY{3;yy+q=~mjM9&NSK zEA;ZJ+%l*tepF;MITDVpe$q@hTJ2;pG2chDdv$`}dMJD<)!J+RYsB_Av|({9-BJ#) z6_;sA>Bs9l%8m{ZMGC!Pq8axis1+<2&-Uqo}Zo#zLSq(I@dd1VVewQ zd~;T~k4P1=dV@_XUuV~@bANwh+-%^za)M90fQry8KATaU`{x1I^$|u1%&x{~r%bZ7 zbU}<$qU8YwHEo7D+zVA{cga`JR&@M18?rn`Up)`jAnud(@O`9WjZwxu0yf=J)6YA{ zhY0S2*%126Y2-e|U|H2aKFVF0OxS2RIo5}t9D8-bO=()MJB45O+3h>0I=@+Uo$ffY zv%-E299a!vBgMQ%xbge2qu8;NgwljYir;#CgZ!2OJWQT~H=ot%{AY($TjSQK9o~6( zq?k38D*`6_R$u1}w?;Ef(2(ONZft6q&#${pI4GzNJ5bXWmDH|` zyooHisqbu{(3-f;1)=z063|XTOEj=`r#39oaP69Tsch?p^HC2nb6SknNx5Q0C@WHM z%&@eybYUS@RmHO=GHmUqP3{Z0n2*hj?G+d=KO}r6aTV!tSUP z_^jQfPBdON-C?CHQYI`Cf%xfrTPgmw=0dbIse*f`c*wWqOSv3}7ZlQO-qhQz4i%SB zFzd50I$US(%j^YpK&rRH#Ipu0TtX>)xUx;$z*irrrE{|A?FzpUS5-1&)33;n<1&QI z-aMR8O+U#lU2(67v}V=Y-Q=O|R5t6|a-$7Vt@z<)P;n8~Y9=DI=Vnm4$=Hg)B9YS= zC4@ItspL`>G)@C_1MKC&NsYJh1O5gg8T)ODq||)!$l6VKn_=~ki{u^MV!p#nxDUFc z_(^l7n|{S!hC->7X(vHO1P!!KEzM&oX=Sn+Tx}^dti>upmPu>mcE1vLi--+!*uVJ^ zUI(SIcS#*)e|%|gOUv0068{zB z+ng`!6}L7pM-22de?B={me~F|A2oivS4iO`Weo+S1RJ7vpYfN;S%AEUxBEWNf(-#{ zk&YOX5plK`8x4eD%BHg9qGVbYAp6~d(=U8C$vL^_iq@diu=^8{JMFs$Y$N?%}>M!HhnT>Px8}!4_C4JW3xU|(qEuoX@F*S zTS<2jk@yf=Xa4iYZ00mtCQMwuZ0Cn)=>-v3>uUO*DOoX7=XaN!<4D9*eQh`uFUz58 zb%h2-q);PA1$^H$Ox#j&5jStLVScyQ^ETPq`NPdBQ~>7tUUH{OZLN^wMlbW#aP&-s)MT^+VD4cJ&(PRVJXx-s;pO*6(>hsY;?D3$$+Q ze!nCC^K`{Kl^#1wJk~?`?-qv&4fPYa4BzaE&OZnCrv2z(C!)J5UHJ`PiA$QAU-00@ z_xS)~uHr$iECUiT>k*03{z5|$eaz#ytXW+VjQ7T;;f^n%uOlwnw@99q;{4;P-U>{Ok{=_(%U4sYt+?gR@a(|`)4z?7w}r>r2bo2mL}yE6;y zTZxk+C#$jYTr=a+h;!sDr{Yyg;D~{^tetv`oHS3xU9WV@J_&>?%cqwd^BEy@iieZm zB@pc_385Ss!<%(GDpWkC^1zB#9P6s?Tl70hS!QV$8J&vdc;#k1z1Jv4MCBwuXFH3r z2tzwu&j4- zzEi{#fF7m5N0rf@MOuhl{B?Ds@uau5kt{=(K7zhO^K3VsHl$tu?Y1v#p$vl<)}_wN z|Ms^xK!54-tU9gXR2w@y^YCCz{wq22*K#>rr6a&iH8n{1FN1qBgPw9;&G=RjG+}Ir zK#!!@9#ak-sKL20_WbDN3%D^k+WrdeF!1PZ%GwVvodVeD0M15v5G`kTe7!J5Ju$57 zp{|RpUZ%skZAh$J>FA@(qRoMvs3TOvNyL=x=#Xa)WZ(;l-~>a~{fN>ya%*YhfuxRb z^u~B)VtXuW>+OzGRb#6Za0L=ukxE6nn<0Cf7l{Z*xtg0=etYZBp;zA4A1p%y(KrAW zC}t(@M~Qj&oA!xnr`R-$l()Rd@g(J3Xh90fg(9|w|&zgDkaOSbunX{Gae zKzQL=w0iX+^{^=z%lHL5_`yA2U>`8 zv_dRP%jwFcr-GHk?!n>m9G-b!sS{cYsVB{9{vH`8oF#k9>CLDQP91EH_f|G8aGb5u z1w2x+S4hoiaQRafoEdxaz&5@)n4~%wC)VMk(0d5hLstotzPIJ6i)(pB75qavCT6Z* zNh7=}Y{#{XeD*X=c#%1g@uO4-zVkvSM?1C=*=ssUku9T!)B-k^*6Z4>9qZvYX;6b( zxFRNl5G`|y_3>NU7R(jA6@~N6u3y(89QnUnF6;V%dw5hlj?0^t6Y1r0&hP3eyO?Ew z`EDWHo>sl$u3Ks$7-P0pBkrZRA1>edU_YO&#=wW&y3|4sqV?XyA(w-gg5zQ;2GW;l zVs==^7QYy)Dy{%Y`#60UQn{&r2JWF*W+QS;QW2cw@^lveNjn^iQ#2s;HeF6jP4L)+&oK{8TwFl^N}K^KW3Wx zYB|RbMQ@Ed(O}(XvR#crn)_TR)@_oB?S6|hpQuE;rNHC)f``}3PYY$Beu=y(K zJ7XSaJoB^)WYPsk4cCupb9w`JJz^(YkkbyXo%5nFz)Yho8v1Y3V-KF>^xrtuG;MI^ zW>$H{GCO?6S4&IFe<(2dZ~WzBZ6H2FwjQT2z|K|f;_sa@vp>qfnFOj3Q<2SERV49B z)n&;C&A<5IZ{*Vv8;BLsF^YNKx1u?!jH@0P(pS9soo|3-0$#9X(Z*7>8D5>JG&mZ< zR}N}pI_u>p|6*?aAD;t+mKF@1@d+P=`4(1=i*-_s8^BylPDv-_(l=qcgq$e<*5Mpz z5XIW;i*LXZeXA;6bN!-l)o~bC-rd1pWjl@acrflYCL-4py0QR#lu0SRC+mOVOGU9` zj-8}fQ6&F2wB~;S+_?8*c_%b$e)6lnIT>W-ZJhOZk+B4oxVQUHi1~zSQWWuT0F&V{ z9=mp7oMxU@kiH?@+auSw!M73L(vAbKyX!WxIXI>SE^Y7-c zxyZes^LOWc{a2hceoyPio5lbFV$+4TnAvBE?Aj`Hz#A0tH&O20tlA8-yYPtlG9jw$ ze7jV(%7r+hh+hd4gDM`{t@D^xqNP4=#@FDDB<;5?I}^q#sQi!6zY z;O8K>m2Xl;z6WL&b~3qXSU-}B%q*3{=1dDyIUA=b)G9bjw^KVwu&5JwJp+fKUnl&| z8znsn-YX}w*p63_Q0isfC(#8{_Jm4WRp6gRjz68dL3!@Rc_d|lYuR<@?+08TIW>Qa zA{!6wo7ezfyIkprkDBktU&AujE`q;eONfr5zi z`bD4DC60@Vqi`RuDov12CcM1Le)ukk@62CFoIG|EyPiv*_yW)#@$)qSU{#GDZck92 z?qF@OgM9{5RrrEu;N&0U{w0$ylwirHS=!_Pja-egs?7mQ==HJo1IOt`>BMfExa~;1KabtXZ zQ6_L}W?EVPQ!N!`1ZU)v3ojLs(+v^^WLPwV^j~NO-9@mMB2}2ci|<>CeK7v<%DLYL zVh-4xD5Yk98-HhfRLQ@)E9M4wglk6k+^C{|H}$?C4QzwWRbFBx0!C5J(gM$lL ziz=rN)W3ySr4CG#NKFuUIrP^X0qY2ze#!gGki1|A6(L+CM5P4bTXxON+`LGd;q8B| zIYYDoz#|;97}BKdC0^94B#Zqr0Qa*`TN~U4!^x(dFMWnVZ#HD|UR?bgj^;vK{}}jN zkqW_bZblX|{tt7+jcEfN2hRN8Z?+kV7&4x-Nc3B2E4<$`HI)Iwz5hCP>iLYXI+Kch z64@FQJfED`xwaXINiEr6ee3bHet>2~_$NxDt7Tq>oXK2EN zt2t%yfqANu-)K3-)A04d(ifv86cEgOhm*M8grO5Iv{?q|%(mR%rLw@!}Ho24MzvcCkfuf$h3m8e^`fU;3hT*DeFmtjLH zID4fuLp@%U*#7I?)dAM}Aa$%3&p3i{aBVVt4DrL-l;|1qQ(NkP1dLa2L&mhyx6%E* zC&49m{x_TL=SPXPkr{~}Ox}T2ktZ4YwWMuX;7s)M!AYqz;c2MZ970Bx<$@Se{u)V( zfJy4T@KeH?({hs!T8dyo8gp}|z&(Tl^+|WW(fy+V9Gsx;a>_7}<7M5Cirwm6tmE1| zGGA_k(Nm>_>N}C)8{7oL)SR5{=r|3Y9h{Z}=2I1e@T9h!kGBwhD}NA>wJg z8$j`Lb~_g9y-*zksuHPr>nzG|g+Qd6FM${{TDQy`n=YFnh6+IZ-|*rX8gX0m zV_PU|+O^{xkc#w|nkgypjX%kEZK;JW9J{N;EEgjsz_J}9G#V;j%)qC;UhkiJ<=;l4 z-km%EZM4q67A}ch-S!BfW(=6@uf&((sN=o@GLEAA7 zRWUJ^+4{0lnV0n5#c@LGcd+=qfR$M<~s+D@+c zVWU)2yRrbuhuBS&owGC5gF&P8xza@33T;M;8S6GdMia01>6vb1**Pr0b9MUw;*Df=0;6iN= z?9%ldY^w*ifb{zrz$YXxt~^nCJo_1%?HgETlS<>G4p83M`y{Hw-Qpt;dxX*94?mQc z@3PrY7K3(pNF2LOYH;Vm)$_vuajr530DODDabd}Cv|e(Ipr2MgN_#4d8-C3x_*jRe z2B1cZ8NE^R9Iu;D_(egOLRg>t%kRdVKseM`qovhL16K#M_!wp(v2~chTa_)Y^A4@#N&xZF&OZNgxR1zvr6wb6)SErkp=0P$H~y3L@O@VmJ?Ccuys67C zu1=f)T<{#yE05-kralSM*nKuAL4bUs(*CDo;9VzUSN4dG93WnAJV#-?B&=CwvqHx9 z%)p-|2<<+ka<_mTsHP`pTlQt*fNem=eZJ~URlPM(k=pm{2D+EP!mLx=4#GwgGbUhR zJh$dr%uvY5gJ-s;9x()~WLqG`XT5s+(Tj;xYeRa9y^V3fVc*FZj;iVR$}}E`f=Z5I z@C|L}3K$8pRiAGKhMIKi1hERe_)T4V30jq{wa>mVV7yUs5`?+H+Xa3fCm0?&3isKm z7!5!O*1DOmj~8B9KEPIf7S#$CMeyE3A5ztW=#1hC4K5ToK$I3tO&<)j>4@TvIbD}; z&R{68oviz_x9|$F4(d(e4Za;t@P79Vg zE}~v}wXvd0zecDej+VU-3ntb%ui_U%iB+^otT8pj%vOv6W2ztZkp9$|HW14YZVX(K zkK!l-9KkEhbdgANI7oq`FI*S8!)TQXaWQRwb#vB<&!)lqz;WkDrU_$_=5gP1cjZZ1 z9y=f-$m??^=}Q{`+tB~XT07~^lY5`3Py@53xwroNj%d^|nf55S22JkHOK}>$yT6gb zdD9D92_6DrchbCBl`d?Jd@lJO%-ljR1-%GzVqaf^dN(nXr6~mOvlJ?}u~T*s`IwlbtcI)DkWvuhdwMo}9p1@Oq#5;5WdUNpn9WI*lh8&bIv;&Y|L=L$(2x@-d*|Okx)flzR@0*Bm7<}G5QtI zB8>FCN$e8GRa(IKv3TvT$Hb~Y6xP0r>pmMUWV@*tC$IIqU|!;+PE#xYr%D1BT4h%| zDumv*DvsNj>nLg}?`g~VijGiG1q9r*-SNP6OZ~tMp(VWVO*&g8RnJjzB^}ILChg7g z9;d!vs@A&`4Jm|T_x^0*fOH&3P(2` zkbYA?m1Y<5FKH5@0Bp_)m{UQEvCW}^WZ^R~Q|pt1WqLO4!W5fnxLBP&(Rcj{8#WRe z0Y23*%87=xk9nUNVe?fNu(E3}AT|Go;U4u;)47?SPUI||(IFv0zBt8N#}OLOMtQOq*Lr9d znwn7Wk4y+T9Vct{`q35Ri>YoFZT>Km>FlP}J<}CtvS8coogbe=;qH%rF!QYVhbX6N zaVB?)3>cGz?4qZA6YGRsYL2cR#@yfC-U#TGmKd4YQ|sfh89z<2Z(CaY9eoiz`%7Gx zCSVIOatl2on->JNVQ=frIw|^ocr9!L9=h|Yw!x*T`1ts%hqFJUO8Ix^twowJit)GG zgGp$*<}P=yLDS;qfZ_p*@Pe{&;2e^uctC8*GyqMvDH-)hg4SfZ&UUCiH8mZ`rsCxr z#^_9-iC3DAU^REA3OEP%R%E9s64(0wg64Qweaq9H`3TYbGhnOZ9XGwoHto6uK8$Lo%gI4{vv2syc>?xB6oL~)R@3Qv_P8a{hqf%i%4AP zSJG=0wP=l`?XQ=ECT)K$f8A=bk?*Bw>#-WNe6LJWE5b!+NgR_QRm%?POs{oq5v8l% zdHe0belr~{6YySnqCyL5I}iUAW4G!#lyW~MKoL#!X%||;J{{HVCF%^G+tlCfuKS@+ zJeSspi~kCk5mme^GtFNE!&gGCft(^1)S!rQWbtk#aOw**D)Q0(O0ag%%MkX5U1eth z?4wmb&5HiS+E4+_PF>q}_{=fDuVv}4T;nBy|C|fMTFA~lz)L285S+VibHb@NK1%#h z9)7r%_pZsuhuvVcKbzBRmF!GErh$@aJf4(QDQKXoDI9I}rAtl?EVX}YHUYqo8YD-~ zxD7Z*U_+3UB2wUTOI{ew+m`h)*;xF7H|m|qP3rU!*Tt3!&}Wf(c*J;9Se#`NPW-GJ zY>Av@D&F2z>j1PcMgAhz(3U(V!PN!=c5^}f{OGs}yvSP{vhK&m8S4C_JE-ZeTx%AuncH@W$>NJ-0o+PSDZ_bs?hdyI&D@@Ra9r%VL zv#R%#excRxcfDA?wqby)j3d@JWJJ-I_x-`uVLoZ z>$1MqTAk?!_KGV+cz_6an)ZsLYSY+e~iM7`3ohMxr=21(EaQY0Kw=ixk1-Q0l8#ZA%hE%N5B~*4V zd6zvcl8wTOZA4{V84v&%zCh0gA=Uu&;dDQkzsI#4U(BK7Rtu#U{Ywgl88zVU=r*cw zw3BCi?IG!RcT(UiRp};8dRPeKI`Pm&`%j<~ZMUeR+lf6Pl!RTv%Z9hod%Ecw@9z8D z-DI}<_7+}Zb#Ptah^^YEQ7plyrjLcMxxUBIxIj-+cse{ga%TQ{uum zr*zLbwN;m$h#G1qj~^Lwi&Ii$OOW?XTb^2v%vQMw<9z*xoYl(yWW%*2RdLs4w?8aB84hHN)-sZa1#+jm@&6`wWC-O;r{zZULPM1J69-RV z5#LFpJ2}zL(iXQrYGtJzZ7ruSm2|}4?pRS%mG30(5k$!rau5@I7jc2BBybqoA-P;k z6?FF|nLs!q@04Cr>be0Du68}sX|o5R!PwSAUon!AyaM__m9unsw3Y{5NGh4n_CdS$ zXC8PY%=1b^&jM_TnX;gjM#Cr$)PyiQy15%79Qtl*w{3spz{GSm;`%HCM|vMwny;EF zV93>wS!R}g%rH~$HWdxZx_U~D1;x=e@bc>XK${a=xk8grkPEDYxi?{+%7%3m%Z0Rc z3%Yb_TDUSqlLC!ceYk5u^~)@GRc-bjzt=ns>ya1b-5A75>)K|HL4DXg2^=m~5m(cb z%06i_u^VTING~v3j`+5`ygIvwD6Cca8bwcUR7pN@vXT|`4j@M?=CLfw_arjYEbMZu zZ+QjfyC8&S+B0I?O)uEM^@;C=ncd1v|&sHsaVkFC3 zMlI4gA%sb5JZ?w>-5TfI#ZrL7ynyfV_F-$$LRCyN!&IJTLMb@`l9Q=)UE;p|w8PWk z2|utY3PG~`|7;Afn}XMByVG&QMT|p*IgOf2aIdbZR8_DhxF`7 zdFG@th9xd^ebl+JOP>@*;myVW>`|D0M^CCq_VKDkXwMpyqlHXdN2^fh;+uml9h1a^ z+`wlp2>y~Ic`1uAWB4xOwS@W|E`!S-hjaImJN&=x#VBRY__7=_0b`O{nIoS^tS_S# z?}FbCI0T7;6gb<16IyH0x;x?#UoTAv-uTV2FkrA|-N$_wQPjXvub`!saQ*rrG)G4X ztJ*$dl@`5eJmy`mf0zM6@v4v6CaBzF1Mw?Q*x~Mg>S7mf?*-`iIV87EvEv z*df?DJadm#y-*u%NJ_tE(Td|=4Bcj-T2?TJaoP?udc|qf6${ZgRm83TV!qf0Z_j)g z5rig#t+6$AL<3zP|EaZIp(`EGTaNWm>^Z~4Vb?c@DrEl44w(kEgP_7}jN)VPPnUdT zVJ-4iR+RYI5I=!O|Jq=l*_RiWW^b>Mk%AB}p%~If0bv;hInDFoFOZQuZEh-I$9Ie0 zn4?6W!o|KjGCY-LWeYvZ@KfQ-B#*qZaL>MhVUO1f`nAic?i_Mir#Jy{TN14S{P@2s z(dP3ygtvY?JrvpO%(fNLNuj&uTsh&R4eNw2b~eP0kLvKTd^h0_2HsApemIo>Hhn2Oiu zzdKp$ww#w4vnWf1%91`jY<#x8&~E-(;?Nt!%b=bzc*@tcPS+x}X|1T?_*G_xU}&~e zs7)dvY6lV5QzM%oYcHXmH8XS$Ntq(%){_kUZC$m4<}*AMEbyZ{B|x*;-B9uszDLtJ zZmxlL>KtXmiuh93bdFr4o3&C_Yn=;ylaJjz0ZvRqX&cqWPL z5?I!7InB$i%I+}hOsbkh-<+xaNgH!i9LPx36cae8D$iqiU3-Qzc;QLQ4CL4#jZ+Rx zI+DQpY!w!5%ByYFj(3`yZDCFKW?4>~9-I4Ksmkr)yds_86Km`)TjSh)5VR$gQQQn| zu#3KYQxPjZP?Mu4oUz8XH2ax|_D1c=#50fjF>LljBDl}f+ZV4Y&A7#T@5$B?N0h`} z^}~I7ONyAQdioU$F!g#*O()~Jcx;Vnq(nx7&4EXrM@Keg<*QsM+{TWEJ&M%p9jLNPGc->S|&3Z^DxDmr-9~fUB=O&^Vk?Ow22S>%CV^=Mr&U2Tqz!R0j!1BY~5a z5NYxuS*7>=Ia>;y7T>7>`rfX<3Ly0y6JXCl?G@wNCYB7Wg<(b@0chWKP1L9M&=TmB zfIg?49Ei!}Odnd7W~>BNcHa*xPSpaQ#%gib^9kVO#u8TULMP?yjj!rl{88vf8GxpC zj!IcHY+i@h2>`p@sWW!B^*XasA{WUbU8JLJNEKUEk^C9L3(+C;J(vbT63GB`Fn&4# zW8-U&NXTjE=7@~|bc%!h@~}KgjhDBkFe{x=SIS<7!N?&`*uVzHE>?~XK3%$*=0(&g zO}1R8vR;1ZvY5v$UXl`PDNU3@Dw3J9P8!4#Znn5hkWu{Q+}HOP&IV0|1b^QA>Ub}H zegPdnO`Ysv`=W*N>+BvMr`JO;v3^TW5)vvDBrl-PmIW1XzQYmB%qx5+K1Skew z-+&IJ{!ItU*Gkr$S&dn2IgbRf6yH+byRyIc%Ws2N+t|j4#9tqf61`1KZC_*O#@Q?yX0pRS4hR?;bZNM{z@wKN04-kK4>` z63Y$zViAtnm>lc$^{@Ut+xc+!{_)ZIQ2~FYG;FK7J$p;ubixy ztC%@v47tvfr(KlbxY*J1j;V@1HYkp?%8v6 zVBCKEVrFr-(kOPQ0Q^c)qJXVP+xTW3)UHzK#2{%4zL&PtbVZBHW>CxA_eYF}Y#r53 zH+iO3u=rusg=<1>6Z%Ipfs4Hl{R{({V>$^qx#P*iQ%sOGTfx?5T`R2J8>Z~ay`siT zuq}z5au7B>*bUNR0tCtX-M&(@0Wx!pj#MKX*v~5Mp0yKgL{u@bO3OUJ`_)2o#jrf2 zU+pP#!ojU&4+F*Ov-Zcw7&PY{EMG8MW}&}7N`gdfP8$c4bRIAsvgM>O5mRz??M*u; zeAbwFVM7T3ge^RGu&;4kYg&^Z*iX;XuEeZ~{F0cPI96gPkNG3iP z)hco_D&B0C2&P^&IPvjV$p~SoFQN%R6NNb50FNYGH8Ci|x_Qvz zwF~S2sSsP!ud(@{$;$cWM=jD45Jk0LON+1&lY2W)C;Z4RoO-Kjcf~hV_d!G~qO(IY zuX}?|&^2C-*~xrHknCNR`X^6AT}(6e&PRPOmqb=N`e#6`Ligr_NvyToN?H>t4LsF9 zX)J`$55GYZPQ#)m_>0r4Q*E+Ma;z5z*)+Nb<_J){eUy5kD>J@<1BiCYbePRh)K&Zx zER_EXw^80CtaU%wAmGqrl0wsG@5fgYu;T3AS_du-@nJns2 z3JRxnwYyvF?2RqX!}e;t}6mEioAqH6CQUng55&e6ad(~zvDbUms3aXx<@`XlS=S8n7p?2-M%_^nZ*2V2(6lmYtXStuMT05Ik5A5Xqa!} ziX;8+5Nf=|cXx}`(Kk$qV;RN|iv)y1Zh}uA%EPpcp}HPM-zgj_V7D=;dOy6wqu$_` zc|}msPCy4stLu0%dos#1*YK8x$p z&@tddN9jQRb6Eel^Y>;c8UaNNwo<#M{59O5k^#{*0vRvkqQ17+aPaxckp%vEnMSfE zjCEl(i)d0XZWSm1%hmhzv4raQyP)Zs2gH>U+Q)za4^}M#e@>3UmCRop)HaR>$r{(u zyq``hGs^0gaqqAhASdriAI2=xARxY#?pt#|-fJ{rDyMx#Y)tEN-~N6Yf=cs_#5rX0 zmHaDjaZHv7Ui20i{(A62dhuc#c12B^-jv~~h}L~@HXu2$nrBgJ87|W0x_bM%^{DpZ zm$6dUNlz!^h&9b@h56tmS0gQm@!-_<8UHnYMEl20*qyL7sy7iZQ%kMmBQD=$2MKbY zj3ofjp}Xe~>#)>KUJ9%z%s2WH=3*cf3Nwx>zkrQQ-_HEDvqtvD(u6TZQ$xc~(m`oz z^&NGPf&k)#n$PmF$4_nWoDa4dCB<#Zqq-wE~eS>LceqFLUDVFb&|vc+sTI@wRF zTM3+h0EFY;=QRHpy_|jHR}P{)MNr~9(Kv1%TkAlG=Yi!_Sp_7Fu82()*oM>X4EcDR zL!uDFs=;LRg^5a)c9W*yp@~_SV|wuGWJW+vJ|w9P+a2sIQ`qOKAXLF64_@DqDr`9R z>}+U$Nx?}iE*H3kg;pJ&{VbCWqu_B`deA98`AzrGWuoV`%{>&T>f;in`ye4XR&J%? zPIwG{YfPFf$YLTL;_}ShFmPLsh*lMphFiF&A4{ZWNOyi(m=UpWzKbbDx9nfqg`RmTvR98H=)w1m|Q`-u;Df(81~HJxSD6)2QiM* z708xvF!HgW>xcpU$tr#w@GeGnbfEBO63I#=hY6adOC)>Xe2m?I^03H!P7qb^sMvYYBZC8vg7&A`^ zB-?%g8v&**#7qJ{AMjg=VgiA17QGb#UORpr-NbagSLLHTEoJchOpd(F#hmoX@Ib>Ak=x-f~tauVQaxWc?8$DMA$&JSu? zpT{8rC{TdY8ANGhIn^oT0G=6{RSJ=08epd3d<;&}>sVafPf%X~r z@#74@VPxK~DStMoIOT&fRJwW=sV_8gOA_~M7xBe({aP(xpsj0zUot)hKsI#mai_4q zvqYWc)ZFERAM>glSK(PFw%b3V(E#|G-xon9Jc$ysMFLWZ{Qwe*fcpT)qAld{^lC6OsUv=q(nSOu0wPeLQoX@2M z$MjDKXn9V3*H81<_b}1ms+;ll0FRM&fxxQ|D)EByRMdv&kgSNiks??wo=tZ%A(^8} z!g`1UfJ@3%{(xDFJKRHfRh9rpo=&_XiTSxDd{W!7+@0+qe${d)KM8BZa|${q^V{C~ zK+Io7ac~bb`Pfc^YN!9j+Iz=y-M;VR5z#_Zs3?`nx~1$@Dn)UpA-iFgm7OhxCQ)R| zD9R{%WR#NZl|4hqEHm?WTrb`4_x<@k9-r^;pYK0ydOfe_b)DCF9>;MWC$Hno_+yZQ zy(UHmO40I9qtIyh3%yx)1@5biL#GmXtU>WS;=rpb!P>;aX>}Iets9G$UXHKT#%*5@ zgBf_`ot$A@=!l(ABOLepZ>%Mw_#xBNJ2S$Bg%ghCU#He$7#_$7aPJSh`JKgg{@Z>2 z%^|IA7l*S3_^O0h72;%ax7RfOvJ#qq$R_Nr^fmq3k07D@8~0t&V0B7Jzx4jr=p-)k z{WFu^=a_|Qr&5RA=otO%M{??o3UR^ghp6C1=4rF`s7s${QOstCZ-}||nkYD4iN~y= zUXs9dp0J~>{yqAwe_h3i5vg?b{m(WvsikyS#A?p!lsL6M8=YHi?wB6LJFDVxWa9jh z&KXPIuCSn8ty%3?_6;X1>7mbJE$^&ytEba$^XHWh?#*0qeRW>xl-iTrN2l308#TTA zBr#QeU;Rt1u(nY8!>7-^q>xN791!ZMlx28qb=^-PR#`4^_FKHe{KOH%%IU2ooqzJ^ z+a0;l;tfUN$p6<|vG2YB^!d)JO~r=>+~op~A6iEroDaFt-Vkvc290dPH+eMZ4iUNu zwc7mG7@GLIYno>rnu%Gy`8wB&4U~dyHg-)Ys zf+vk+)lRL2-Q{iuJab6P}z;|<~_{um{ODG znnIM&xF$r>8A686#zAKO@_>(L^c(AcrmW`6$d{P%%X=oXeX zR{z*JFi>ZbG0e&`d#ap`=3DEfmg&3ZSH0AEJ?L86B>u>7*gZ{BSr zPi~m?*&K7{khtaJeWs_9HQ4ww*h}`cIzHb7A%8!E&;e*drQI5~XFLrdRjXv><8$j= z7jdUlsJv&SKD>Ik{yZT5#Xm?%nzB0GxPc^fx z?^M+PmCZLZ5=)viJFVuO!FT9ptkJB$k;<%Pdl+e8P<521$!etcp>{v^^|xlHC%F3; zvGXvs2%np_^hE+?@?-4}Wp5H%l*8;fy>H<#Yy{nc{WE3?iInTp2ts&Y54&)ECT z`yP?r)3=0@+Dxy8b{a?B^x?eu$smV` zuPPCY+tQ&BnQ4!kiM$1k$iBIoC;E)9@z6(+^C)&`ANK~Vkl^!D$zS4=+x8VT>!G0P zp4w3-hP3%yLvIH5!RNau{9@t78+q2K=0-Pk0UclRkiC=|YgVzZIB43YQ7$>9 zAvzG3{Nbvl--ZYw!aFcoA`IxZW?6MM{a_Pk=2hF6hNgh8u11g+3Ru)zt=v?+6xA%a zX?_t@SlUh_=$_0gM*f2*a<6=n|NX(XnY~m(&C(gp@k}4?0exHX-g0&3avq z;^vcrlOZ5uR*-$g)SQ!kpfK&BQsj%O<}C2R&kMMy+cpA|$^!(f{Q3g#hUt#VUG7Md zH?JKe6##U9%qJgL-uA32i8|>Ug#x#;btbJC8MXqnnVTI0b=CDN!yw|d)dm!z^Sv~{ zuM0=}X>XWwTN1yyY`bS{@_4&#*F%AU?!u1TI_Ga6@0z0*IPi2;`brI}ujij{t+@C1 zi|7}b+RkgLqE3E$_vmJlEVrOusrPFBca_RBSG^?nAkBC?c$GBajAmo6=1u)IH1-g` zXt#_WU|T=XdF)eV>Bw${?U!R`(R{AkTORV<*~yEg)=MIjaJnBWA58M{i%6T>v`n0d z{CqM|efGPUR%gSn>LK?8<;dgbK?d&<{pML9+d1_xNoZr8cD8kn!(^kxZ>XEhIX33k zMw^Cd{pzdqFn<~;9zj?3&{r|`tU{A!+XbS;I&}RU^ZcD(jM52@(FAZY-lW9)s_oE0Ih*4NlIQ|U}|$mhq`SH*>=gL-gN`@F4C z^e$6(N(+8{;kAk*H-Y7Kox9x`;`({!p*C2bss3o6`kryOTR~AbB5ROsbscqP`pdM< z;pW`k$4O8g@`8c}L237>#&KOM<2$~FJ{GAE(q)tKCcU`cIq|r3if0?z^5UhU)821} z{?)m|m99JRdYrVwnzZ%tHop14dP<^!PW$a&2NmoaXt{4f5EVvukZ<4L2tKsM?9YNe zv(7#eE@xW}Fd9jG-^j5D;l%@!+u}c2FLjbM4haiTHdyZevBCZUJ)gMBd2QzvZnZo+ zo%P|)Y7YxG)()i<%OABT(W%xmx(|)GYkTCRN~~s}DeUfl8RnO1(feIYZ?H05NAcNT zI}Yya2`Ee$eW~s?E7qt-q@s;UgNr!{(lXO9E6{<90L>JyWYG$0TFy6yHG4>V#Q9g} z_EwyazO?^?rsk5k|FABTC;E}3bJxmMb2DKBISp4%R^QhFW*KeSA8E`Sxm>PZv;FKI z#qtHO{msnBL!Rx;$m2TL>{N2WxThvFOiokRRIqLNHpkigJMsL!6VDsR&uTL4)6&@U zk8C75%uz^N?Y7e{CVp@BL>IACCBIJ!=G))wT0Jb@$}I8Dv;OR$4Pi@aKD6!EUesis zuiR-D)RCJ$^Q38oI>!ZoWx>FR21o7$sVBalM{jDQ^08a9n;C>8_Jp_h`)fh+GEg0@ zpfJ*wBUwl7e9pfkS9(U^L_8#HQEhw~X@!(G^*R+7xv1l(W<3iY%d+wyXO`Ktz_(8c zGQx7pddm30TjQ>p7~}D@VXh3*zR|lb!nAW+Tf9O8G-VTeik~VpoY4=l@?$9)X~|4J z6)GV*{2Ow-m0_&LIH1!sW$a@%i4Wwyt^cZ}~qlD6BpDMesF?8--a zQ8dh>*xVV%@wXZERJ(KD8rFk*HC!3Ci9M74Ykvmo4SiuqeU_x2|9HO`uJfE2-TC#F z!uA`tHNN(*%3R3eQ-){CIey`adJih$))u+CI8FUw1>K%u*ZCgT*^x)~cgtV2N9vp2 zbW*ks=NP^#z_80Me5~12Jj&iny=OwW{oW1TK~1rmMFoBdX1gGn)CR7P4YLam7`uxn z>x&Xvv#)MV9TtOh|Mv)|n=V9MzdF3-%G{7v7v5Ezpk?G=b6H9dSdDbex8aHjvZ`TnhPAphEK zNo}QP(u>?ImW#5#eg~IZkxcv7-+OoE znasOq^lrmloS9`o$MUN(Tp}P|HTePnOzndoojDF(wTz2)%E>xl)aPbC*{EOqrc>$5 z(DGpcrDV32`qjaG_qoDYQOPp=HBj?P*W#xB%mA@Q68TdtKfNvy2WEdWNyLP8rjG*X zrgYlWR+0|FAMgF9aIt1bGOolA>@jy7aj+)iMFdBv-=fos|MZ8hN`R4}9Hv2=E)O=h z+1yF~g+4^-FqgJG33Cl`b4~TlNVoox!i`#(IpaJlzq7s3xVZTH;dV*8qQ0ugF6+9_ z?9N+5FSm&thW(Q2)LB)$#=W1n*#r+Zr}H}mtoZG*mQ8Q7fb)Vwi5Hz^L=oh<&pt=0 z5aIBQko;8^LA?i-BWDf5j>%)HL@T$a^9F(U9)EwRkk@p(V$03QT_L{)pft?7a;8iw}$=QQS??9Q<6&Z-kd-LtMmg?2N|JMNC& zj@(Ik@Cmxec{&-yraRnp!AnI5e)+5xC~V3H;zy`r9!+!6X#IYxcse6dF+?m0eGQ|c z!tD;p54H$=kZ?xbhKgjjh5b-ds&b~eu7$YWZh$K}&QYqCj!R9^GJ1GW{ldoUHfTjn z-5!~{__-}4zxnS)=dFtkQ*meR@_g}&KCxfz{Nx4WUAy<~dvav{{JH9gqV>go_l_o| z7=GnbE%6CEv0AxqW4-xodxbxX#UqOm$*DHc-qO;WWnKP)?W(zB^HO;WQ_f}YDr|fJ zXgO6gNN%GzjeELc8*B-mNs%A8d}qSYit}9yaNYn>ix-XY!bjlspuepcjn=k6dviap zxcyy+NMg?JWEdswtEOwvX=M6fdAz3o`)XX*);78xf*&`&AH7RN_ zc{7%qW}%t5L95QVapiDDsHFP$im=3&3b4L9HWHG|;UaT|=GOKWZ7ADFyz&VQyygNh zlGmcQ{7~{Y^>1S03)^g3cJSwn`=p+nnH=I)lK6RBv+gOn@jTsU#chUzO~SSFc2#Cs zUAHfE4C%>LPdBla6q|K+rKTNBUxosvrnD%mt)luU%@i}g77t*t2h)PckR_kQaWJXi zRNv^CEc=8{s~2Nnkc611G?_;=PBApkKaIaOxi*u{M_HtaJ?}s@`ruX=!9SZ?*i!X@ zq)*ScKba#v&}>&v;D;_5?g-TM>^Ym{R-EOiGZi6QZ!zcY#KoiS>mMEEV8d}scHJCz zHh3yEWT2^JG|5DP-{{pW5#r5KA7&gLpl@*)D?4!G+fn>eNwEU)71)9$H^BS4vcNdt;BD7bL22jxy z(47(5J1dXP=IDG`pB&o84)>J!Q$a#QF|8;ecV1pjeFO9cJF{N9CXlNU_XjJua97Uj zY`Zx_Y5ZqejWv4v_k?~(UBRk>njgn(13Gedlk$YX>GJ{+cb;QOWF`0hCT%prh2G=` z*9%8U8fVA-XsOYhLlfVmzj{{*L$9N8Z`P#rySV11*t)RkgnP#Vl+wJc8qU9a`~|ng zupYWrgdFo!hUDBO?hN)^No7^95JCM%XuBDVX@%JaX(RbE+?wsx^pfpIZ7e59U*Xvg zZnNu@PUv@?7H-vXzP#i0`*}0~dFF=5v>)%#fL!FzML7;9e9|ucp_kmy=+zih)kPr- z534Ou>Rl3elAQcNjM8fhsTi|_>oy-spiLJmIw-ymHwjnIo7t=y{DGvyK`8=f(C=N@ zaMVnis!!tp5wTl*{QPC9S!gSvPR5-#P&UXaa&Ty-&b!i5Ev~W^$jE~XvP$)5e3&Q! zf@2o_YAK1>D)oBCmVi%PLSKR=8F4_=;!Q0eG-Mmk-k4#E^e zwQ)Om4)$d7iCQ^F($2_mkdNecFmQMRz?Z#W?QNM;c%Pr2_!m>&x(^}5eAe2EwOWza zKG4~zi>=C$#6ragUT&%*yfSJH{=I%tfg;7kr7#?ur%Wf={ti&M=-Lis}(k$FzgWox^zu%hPep9+;V4y%o$Rt^!2tO8eV$7yEQ9VC6^)p4rHYl6B{M4+%P z<-RK@u_Gj;+%8Ub1><%e0z=$eC#@2%wj1_}NmGu9BP642?l~HV`{1@%{ex(n*o!}o zYYV@h?~50yCQ&>`fZ>n)ua|s4DHl=GT*_s4oq>NqXi1OjjCVY1Gw74rLo?-T)$N|8Vs5O$|MysLke>9dI_VHs`*gS2-3oqB@W?0cuHU^= zOI>(IGQVbsRXuWITI0sQb9~2+mE1pjshuetrx<4KHYc`fq z;`ZnrWDDAB&X%(EJN(_LQNP;5f93FJ2a$|r)CntXB{LEkkA`$kf@lhQ^BWnfF4CqQ z48&(Z0$JCN=64rHmCdw`YcJ%?WrxJGlpx%+wE0$%%@-23YMp0*OTF7{f7yXVss(*u0=sI%__d|Ly*)hl>fAm(Ez$Uf z>+k3BR3%NR*L%>B?A37S*6cdf2GFon!eE!^6pmL+F z-a6sBZ$U{GFSMg=Irb&qJ7mW#(X4a-!8Y-_xG_3v`j5CRo#x6iuax7fdP}{M*U4~* z;Iq03`i<}Ez%Q0dd^x<2nY~uV zUwk-27We65}WyeTPvn%-uoFy31O zOiUN$t2t&*!520HJ{w9%9Ck_Cje8@p7gf5OA%%Pkc2OAyF+}a*`{c>SFEE>Z!lSA@ zzLu&t*y53f`Ip`1jk*}NODp4#3cX_MEUaH~1*w;&p$L;lE>@Z~=H)5Q48`7@uYx}ur1s$8fOr;7|o zfO)fZ#q_r4@ON@_?s`iOZ=~Ci-=l2exH#Dq6Gu9G5LbejUJpDBKZ5c~Gk@g${5Emx zXZi1FEs!~sKrpGdsjL@|&_m?ncQ-nz9sIH$Swq$*;08Ops zedW=_@b=B6!xXX$$6_*pns!wY6iwXB8fLCJWorL9K=L>)Xu=gUAAI}yo|9+?wEvGu zHZ{SknN?f*J(266VfE3Jc$6(q5cQg^kfs)DrC%6dy#jmXz^99}Q~o@vJJGF|C^0*j z{9SCOa9QgMY!*IvNVCS_ZF-XpansVRQ5fz0TUG2RJ_Jnq*bp~XOttkaZHVgsCmRA$ zB4O<84r+l7qxNRba5FWB46j)VF?#hrqJqZO)gOVWa7_;!3;hX>X;hQ%xVSK{+gJH~ zQh-&9^kMf^9JpT9b&r{!FE%z-&$s;V#NH>o>UClt9ZZ_j%1WE0Z~oHR@T*>RE~ErZ zBL7%#g>VqO^H%mZrzcuWof&w$W|;U&K*75++j{6gy>VkwG_ji9VT=fhyc5xe^jhV+ zu~jA@+VtgD^Dadj2!(TDbePI`@hnu$G2vi*mC5GZ@Dnj0$Z>-muz_;R4d1B zTWZK!5{Lh+albmnT<4ld{~AhozJ;advRx)&ezVtYRd1FMKdA;e-Hr<~c^=8Hcs~zr zr5O5tUAl_YY)~fa*Y*zl0#|zyoY)YtIe6g!&O$lr#Jc46-=B%<3^pEV!l$8ff{T}q zzclPiMWda9p zhD%e}6J~$+Q|#$v=I}8;x3^qXv83E0f|Ofi>J}09$hxnos)+{GbHIA=Vsbed?ohri;C-)q&`D4|So9`S;S0kC3 zAIo?$RJk`gJ8awgbxX-6V*zR&H^JfYUDHIq>CdOBCN(!IYtI49hYL0a{``4!$_NFV zYH5C`;GZ=P#s~ZgL#2|5&V;zu1Bkepz%YlBEOZH62IQm>e%OsI;fI(11`9AK+h#Yw zoB8b?_bE~5g}EfSYKtPD8=S97u;tB2G+kfGFAS;5C(F)Yi}J_t_Wdx-@ZK@#)+U%R zkvQnWXtpd{j$BUi{uEi`H&SITeBYiOLCM^LSpHRQA$cEZsQN`x3a^WAHlLyjd7`jb z6Ace>6=kYb$z#u1Y;8+B8=?(~f5A{IwBahCh9^lh&rUnX{1S=$JO24&3|5w%c0|re7BVH-rK6T{g zrgl6zt0-W)u2QHz_N*M?G_EO_NmuVpT1#a^8*DlrgCeMp`Od{Z+d)G>pX_^!3xa>@ zP8Syf8vCHdow7~rxBhFJJYYQ}@4klRTRVeoM$e4k7)ccn+wbSTy`W!Ij)cKOeX#w7 zBobQl65jOb^yMjAT~8rr)2DtCl??$`e}0YeA~9`ge^zycrh0)_fAhk8VLL6&9Qi-B zyPo>iiBa;l^3enD=YvmJ@KOKVG%9Ph_+d4F%9(M!JokR?^RISMnBCI)m@@@!BO0;F z(f8uC^V*D{P7dpPAkV@pb~Vp^EWQV=EI!HQVkP^FBaYpV!Xm;cVzYga#WC4gqU{D; zbnc)7o8iwZX4=i7hSj^^hUMGZP1;qP(oNy^x%o1!YXhnDLyRw6Luc?N-bWJvGaUIJ zP#aTBM(MM#EyBIb`W%Nw?OQr^zv3LsN7Syxd!fm#)3Ar=ZvP)Ny zd?p@WV-W2#AcC!NsBOINGmf02gzZ5|Ls}{Ir1RFpQXIyCehO;K#U8n^lWY7&ZBo4+ zB?{MV)@s8M>048t1%r)q(1u^B`wc?*9aqOh#{$|PxzN5{ycwWY&!9Ka`bIl1F0j-k z>y@#_YUN}(meR31Z~)#-F3CzVZWgM_B8@&ohFB!^I-KzaE@~bzx3kmBH4|4%ojrRt z+N3or3HBQ`Yv-#ANT(dZcfqc|zWxiE#A5V$82fK{8P_1vOm0!>$GuxO3ODgBJPfCNr^@os1~)0@S?Bvz$wgX87Ni-$S(;MHIK5ZEIRY$M>o zhV|rff4@AxXI+tdzDh~@mqsfE-L6xLhnKk?#WqxW;xLymKnYQsfd;8t-O~2$k-fk)nkh4FG<(73nN_dn+ z+a2;W`F`Yn9G~rxyHo4 z+|6LolH6UoNC8<6c(Dm9!qohN#VPz&7gb6`SQXE{eftQmOYXYCWzHx*C zpkVI+l~_$~c}CdRXdw5$5Y!r8;!qJ0ik?4Ef92WLG>lTy;S!kWF`0XU`w!X^sL2}D zp0dw=vD~WeY@U;&YEpNlzg|INtSM7Q$F0?aWiSy>&zlQpTRGf(Fpi@m5J9Sbuf@4_ z=!N@v!<-tso@-f%!cOnyuHvB@eEfN1{w+?rqAb;s)(qFaougs$Bo<+|*SNl`QI4eX ze*qCDnGR1GyW}BtxxWT-cN06>qXfHCi|rPB1Y^>N!Z{TGSx>ioTm{@n{Pc>R?t!K}Z9@*Xso_fBA?v}AdT&P<>xugM zr|S1tw0bbIn{0|)05Krk13vG-XGmexG1Oz-`W8aqlj41F1hE>L`5gsWA+EEwG*hVp zq{juky!4f9AdNbgIjKbnw9F}0uSj5}-J3q0?8q z8Jp}lrDqXFp|aF0Wof1?0{{0ll0hGjF4C7BDq_F8eRYv5`v~(3AC zz646&Ao8~ct1Hhc8XT96D%y^z=(a0}>4uhpV0d^=XU}RlFCTU0YE9C;-{@2-@cCKe zB+#Ubvl*zv8f%gIG8@8g@FQI+P9jN|evcL~mfeIp%nx0h?-OKIw%_btR~7Uq>&f0T zN6-`^E|{Iy$kgfeU+zfahkmw@SBKN>!&f#?ixMPOR>ofu^RL(M`bvRcx9+Zla~X@c zbzBb06jR2V3jR56$Um_n&qz(OkN+vkxb>@agqm}YeTbX&qY1+e2+`BL!4sd6Ew}-N zzmIRB;5P2mNY$U#D2nrb+eCcH(_73HJ~|pvkTuUd7s22Wje5V^1W|K#$k<8UxQX7* zY|(jfHq5}q31yhj86uk{hwS}t%-<*;Y)mGBXma7O+lrHUPV<9F2?ibQP1fJ;auuWY zGNmRh6hhL=C&aFiaW%8=2yTJzGnABY+vD5B{a1K7zQ`Bul~*R-Z+lWJ^yAx_;p>fJ z$Lj^vR&CtNfdVEuR>jNW&w122dHiJws0nU-3h^I!FuBKXs9d+AIIj?R&bNIO)i&T8BBM~{|StF|+l5qu*_(9D#bsizp#aWs4vY&k-Z!p_YjX%wpeJaL_FG{OhqX$X7jNkBU=?c$pIp**zX zMI}*1{sbLJWled z&AuB@!QOrdj~Mz^FB?U<%AOYNq|%(;{?DUhc_1$ z8_oJH|NmJVlZ?(40S9#D-4FRg-h|;N+%Tvt9Tl;r?5_-N?;0Y`Zr#RsO_sxtLMtWZ z!RK9=DgfDXfa-)2W-7(1rza53 z=wD}DajXKo|``HAf>4P=#Me z*szl|C8%U@jkb5PwRMghvs66MrBTlf@Kili;^w#;Eh5E`F@!^u?)w3vQ_8{J1J+G z{^bO|{Rwy`xYBL+CNkI* zmG-m)4%4GNsUdD;A^BH|=r`vf1?+)U;Q`agd2Cj+yuvo%FVjp-h}>FG0mA&qp=PaK z>r}Nt^=3RtzBE4Cyx@hDjpVsNKFoL}0~!6~z$V_bM+)d#HD-(%F!qX&^>W>cE2Lrv z$4kx*ib*ajPvAAI;21NW@(St5+We}P7^UxenNKXG&|^**-&0iPz2&?!v&L9_K}o;m zl;=$1_b%$lm1^G#lHbq)Kn-cfYaF51C+A@0#y5J+_2~-zJhSPo)!9F60b0?Y?8ZXJRjJ= zGu5ey^VfMO|K=@ok)hcl0S}=~4RHVQfDMVU$sk+VCcI#E@`67Vo?J=pSQ!VthcYgB zO?0+q{lt74h^M}cV4y;~S;5+fMZT+)_~sdcTeP83Sr7~0 ztUS;$4_XFu=<3yje*cjboW(H=SbqacMxLO5HEbpM&2e~3(2ibq7*LrXX;?@zYTP-) zcu*5u*AW(Blih5@v+!9 zq6`ad(pc@}=2 zM7qZ`(9Q7ohS+FDX|BhWqEy>B1w=5}a^UVoQ8vms_91GVYDl>#qX0O#s9DSF>%;r2 zVLLk}Rz!}|?Wt_50lhTP{Yb>Q!k#&&5crX}GB;*gyd+6aU81(BRGBM*;<3sbsMZb* ztY7ev&k4}uEw^gj49wj0p)C{J`LH43_+FMtMTG;6}6A8cPXC)~;lp7$9Lo z!Ck^QcriL-T5 z+r7-a~7i#^U#YjV^cnmMp5?TnE}*+TFq6hgs;oU>M+sO{Kr95-pl z+a@qzBnof|ER42f4kneXDa5Nh%9Z@qFU)D?ByjjE_W*ADkQ<83{S)L~geQD_!IC$U z8355-1@CjjHd==;Im@?NQ|P8{45M&*v@&*SXR^>*`aLHYkfq;_kUOFd@cF$(Vx!&| z@blf;m?Sj4{1#asjk{A0RHg-N1Of5dPpr6o#2{vm-1FDpc`W4Z=~wc zpa3SNmaIg~`S9U;-D}CNyS*BQrLCu$Vm1i>m~Y}$PbbmzGlbzMm3{I`Iz+j_u`SGD3)YzrCGf62&Ex+@Z`_SbdA#L?Bpm1YXLpMVtJx z)+f9TKq`}jKdC&acm4%PWrI9g7+TLrirJ2acb5g#x8;&M9y2O6(=LFZ+yfb-AA+2E z(4RMV7;kR1=Q%>b@rUQVE`KH<>jx?-YXOw7UgOcc(pR01Ykv;tJR;MPikP*MHFj{} z2{>pm4~xS~;+JQCGiC*Foqcg?rH*v}xjI}+`epYcDS?T}Q@-P=iq>Py6Rg_W{N}Q( zYKGb6bGCq^|zqmD(W0&%Soc>SBWBKJhFi(D-=)!k#CE--(_cV(h`rjtP!)$W zK*&-WFVy3x2cHfwM}lG7A>WZUYUb%8r+c zW|w%0+YEAd^~qtL9ShKb6%joE zLiYj38Oth``2yy(i_)X{%grGn2;ir_J@WFwGV1q)aGMh-SA-~BU+$p=<=$Z`I@ZiU6;HT+0cw3@ZQ4L_->P}6zx(DlZ+AAmPBI;!|v$iA%sKu#@hiQTARK7>`?mY zdVb_2nG<#DU1j+bnm8jwB*xd$@4@(Ayy%1ZIGnaXkUx3H`%K!7lAOqB`-CeMNxQCt^$-;!lqiH2urc! zAa1;&j>)#Cx+Y7m2G6~CG^da5Z?Qs5qn`PMVuvNgOK=%dQO8Kb3N9NtGGh_SC&tEJ z>&w`r2aqg`eR3UDod%Y~&3XI*IyCoZII8}@xAY>gZD0Pb3{`wEiDeqwDaFGx>hka- z#8Mj1`+jh54k@!GaD>{smLDBs@^sC|)x(DYBwmNh&c#yC+^I!HQ4FY?ANJ%_yN^?g zja5S>x-qgTcC%{(M%_tjpyPf_kTXps>W_Z$&00z}&+7`r^X{ShF}cPU+Qmx9EziMZ zkF|L#{2j~Oq3w!{R}uMFs5iq_`@y#)Q89iKE$dXH=kVshcUpOCYFZb=4jfzzC@LZsY1)67GD8QbPIPcBEez7flbz$h zb4mJRkyXqF59z_}s)?5MYs)82^x&TQ0>HT}^Un?quv$PdL*sMu5F84TaxXm{+L3CUN&X8zron(kg>hL6H*i-tl^sw-bCpx~t^apAz%9pbft=0W*wk zc0m|ws3!{McfY>}LCGg*&xRS-OsB{$-C7?!dGEiu(lS?iYNB{112G24B@YFl(1$Zi zP>N@?%REQdXwCg#37EdImpqOz!9Kg64==-=c5YQ_<5R7o7T+%}MM^S~i_Li*65Xp? z4QR=mJLd{~Q6BQM{x;0XdB%G%X8Gwn6m4h8tVbz_{g3Q-x~;`COUq6vDv#X10z19q~iYRpX&x z6h_2YhV&BWl-=is?MQSAhFg+kQh04Y7od2*eM4%F7B zZg9GJiVyYK(iC@7H6}ENTR(Lo?R6-Sa z>?PB&U&Xj(?C6{DUj!BOl^n z*0g+_N!UGpH10fb<^;Tqc=M6k+CUrKVq8DlX|f(ILwy@1+?;z}_z&dfLV|5!`|^K* zTIW094e%IzYP01C%w?YnmWy!4Xhv(3jE0M5+{vbCF;2bJqv?rtyEvEcSA*T98P1+Q z((7dMwW*1e8?bi?^pB?tCTyxu3$zG7cT}*)z!jp-Rlq@PjB*T=cFygt2x}L+jkX|q z7ng4ACEp}8qn>gBKMq2`EHUW?pnWB>sfLc zFnA0Rg5X^{B?7Iv{^w>~MrFFHZzLS?(;I9<;0mPH$_JN`?h8{P5#I|BSp<)pC*HU?Mb5&-{ZRbgvo9XZAs=38vd*9EeIErB zSWDa!2zo*;Sn2*j#zBSy8lteyBtZyYMe6+V61yzY5ujIC6F#}LqqLZywsnS>gWNX- z)z4+=Zo-`>N9HISLR=R$nNtB_WKYIZaw9|aP@JF-;wED1#eCMB|K#$8LN?PU$PK;@ ze}Eiz4^(fdC6r2to`(2&jTS!HPPzYWWjcro0d?!=l&EvD%1KrO7Z_9SQ3lt2A*KK~ z3j;%O+y`kQLX6w5QT^Fm%c$=n`wE*D2&W|{+M9MfbPA%P)65fE)2JFZbq`kyldGna zI(Bi}wIh7`3e~H2^rSelE=%3qn`N?^e;^I0LZ6(U-yc0~si4LqNd9)KIg&BDU~s4{fj}XWv6LKO85L)EB3;_{3w?vKb+T)0DLLae&JJ6t%hEYTUt}L3 z@O%MzWy;$A>O8k0ggXvmAI_5(1^Y7b=G?DG|MMEx6S30It1@J6DW)U9z*mg~rmShy z{|&wkG?^99r2_FKgVm1yurP*=4w?IDxW$+u7O z`d*;l_j23w_e$4qBn+fMvsr??$VtbL_@QL7M{!v#RhB`~W1=7gebg@j)v5+p!?vSo z*`f>8@}ncF1Lj~kiy*4oiQ#?1J>hc#RzJ>VVtmqR8_NT4QP!M5`sY?HIYUVzIuP+k z@-J;7x#=>hbQ)6N6FqUmoZR$sV;o;WMYA+fxa;_9 z5`6RMF_BIYXzG=~Sr`Bm1I6g|e^Eman|vEApQEtOK|JXJrW2=6AIcMV0(!hdPIqI6 zG>V13kuW5bhk7MYlzk7iJVDFHJU+Ax`kaH>>tg_Gl%&cfD9*-CHBU5USz>J9@kd)( zeqTzXusYME9Y4{-tB_J}+Mbt@Jw>4KvXU_Ayr`%s+tk<;BKD`p0EPMdwfnXmS>`IT z`XU&9BsYdNg(in>&GI1X)oVvvel#G=h~yE$f!s0zOpq6fBs{YB1KC~5^nH1ogiPCv zW@Zp8im*AuHqO8BC4aS(R6<8Wt|PMX&>noIg+$lWm;LyvQSoHvCcb&eb8d5Q)n=wf z_(}VN=6nI8v{A*Rsp*5NVpt^dLzVtlYW=J8C0N-=g)_XAy-ba>$;A|=KzS+}l@3%> zOxz!%5(6a6I%Lem6)MPE_F5VBB4@3Lj)m9|DNdU*Sp{+D*PK(EU@!Ha) z5hGJO*yA48Z%Nl{CuvJGNaFIds+8s}PcoZP^Cp8Q+6#}mxT4+`P`xo4uYewNdGN3nkQ!)cJNrdWyi+s@Pzka|bT%LOrf+}4EFl$jiZs6rPzm?sr z);gBRK;Y<(bFBgC9@S=BOt0&{$DQz(B;;E!&?9~zKWDU62-$f|E^s;`v zr(-_>I^$L)?+juJy{)3m$l1kS+dQKp$>EeECq&~Xmeg(+p?0ISDxAkC$@BX!#KmZW zPH}ZvNraVT9LBTB{aB2li%9Cb(V!>jI0#WAv6^@>A8iiIARfE3VP+;3;xDIdqD_>h z(+)#dOs**&3do42Qw`F75FlD{YhM9j$m8qhiJkF=eD_Jj-UA3S} zI6Sm}oN7;TL-)AsFS!a7`H z8H=ERNf-RX)0n$8B&?~s@;*q;d_jb1Nx_Yw*XZ9nDy^k^xRj;d%O^lRP?EH4;L6+g zp*cuXZ;7ZCBKg=W;qFcEJy!2%u4b^vWo^hzICR@TG4nes={JwXr(tIfN%f#GN1I(C z@G1d$3Fzt=@z$LY+NU|bqtEbV$ex)|4x8bYFCPx0KH%7Z*A=krlOY=NL=Ic9kZpew z^hJ8x^7>xr%hK<-&#ZyG1g44cGnn>bTAtfAq$LlaR{WD^5 zxaL@U2E4m%l82v42fv>OTcZG%x_8wFn{ah9xRT>%Ut(7wniGwjl~8J=M?~~!p2W3z zhQKya6C)){v&DZBTgzi>VO;m)czO)vhwI53JUqHRj|XN?j7((|q-*r0OYWYF+RR40 zFuL>F$fwb;e`&-u6Q?V>K$=sp>(foqt!0HY;&_)_D(;z!u0X8EI<7V*=_X-#;#(H@G<8WbDDM8KPOmhg#q(`#G+-8lx~NUlI7>ZH z|E0_>BImuoA`orwBrAv7LPG9pKKud?YRUP=RQHw}qzq7BG(N**7Bf@Co0! zNDWr1-X~f*RHc^Dhp~r!0I7U|(=g9We#+m^k0jJ8Kugd;%sSe(V8Gjd&vFMa(ai&^ zbKxuS6kX4>sjrtFn-G|BV;FXj#iGx_Wa^51NNpI|ek9*(SJHYHl%IQO--Fz>;*HRj zK7W+^`+`4UkRajC?RP${z+L*kp!i4vcu+{cYi$KBcmfsc&;gQ9FQXm?SZdS$gcVyj zgSt80)E_M(l%Y@kWkws@W+6TbWN{il^kZgK7cMk+_VsTaNQy_e74SqpxLEGH&mf-T zt4RxL+JE~{>0ikZybqP{3wy%uz}=eWpQn75|9rO9?llt8U3!MAL}*HFJruj+GKnR8Oayt~NTgR}fXUr;3GqH!b9 zOP01eUIrjmlu=Ncdt#~u&!h!!BR;VW-cAcp0QS4jp!xkAiFjW{m~wgzEr7l0w5f3AcNGz_gHbtUYX$&NZhmEXP1oAywQ3hLJX zr&ECD+iuEvI+4D|=gZzVrm6iz8$DJoUa?W($==IH21Nt5e2;w@>I1#m#jn|uu@h~8 zToY>13^sBSci$;Z5WxQJPP1iC?P6&VR!FTZFszlg|Gi_P7t_#ESc%R@M^iKmBCMMc zSSq}e{)i8$>S-7xi$um_c6PgdqTK(lm%0iL6RmE-+6|i^gtKO-VE`gx1H|9qI;y4RLBz^ugILsUbmdgxHH6Sy2 z`nzlj^xA_ZJ1IMWNm+01C4mBs)yPcJEdNmqTUuUV^%Xx11GY;WRC4qKAb;`sm_gn*B<^MbeI#y_7dN5J}jEF6kA zU2NbN_A|Pz+zA;w|GV9k&(_?U{r+NdY2^Z+l2X#xoYL{d^`Ru|VMp%%H+V7$8pOKY z_%^<1+zeU-tOMygthdmeX9u%z~(2| zgZ1aWqP_ek5a0j%*(Gj3==3uYc7nN!Ij;3)f|Me>w-Bd~+?2I@}oVx%YOZ5=^Q(UR@Z>sbSIXKSnT5A^E#w^b+r50cxO;hc`IM z58<20yZf7V+-4e?N8oT7C~Nrk7ypBhuG}EV!=^7^ypTtOHt`qxzKXa`;~a`76z)f}#%WZa?Z6Lx59 z|1{z*qt;^Hg1Jxu(b~`lO_UcdKYcFIN> zGlWbPGRrJuLP{t?hD>G1oH?0FWC$fwGG;0Y5h6p9%8;=L8B&>tOk1|$UC*ZT{hj~+ z`u^VQeXr|WXLPpxc|Omxp0)1#UiW%f2iG#oG2bUOmCpc(+yT~4DY44}2Sp?F8Ftqk z3E)Fkml)Rp(eHCqggNt?=4LkaEgrU&lD=kK`bfwHW)Fz_>`M?fD$N^rNLl-d0UC3@ zFi=4UWuj+Ep@3;%yhbbj09{aRSI}EIOw(T1jqF3NA9w5%?;xYvPFn`(+VNn5Q!?fQ z$Eg`gqLz=)k|v2udtpFl!xz;Zws$s~N%RCVyY4D~ zC}H2{{3`sS&691_Hz&Fn>p!WY(ikX>nEUs+fc`42dgZ@y-Qdz)Yl9C6b_i5L{dd%q zz#;-H4@_Y!T&+!z;@j~fm<&0ZgKv5dLyDjV`$@x+8mQuWTCBUFx4BRvZUu$58YG~} zAIU6+4}*nsk5{tK=|af}3g|=2L3>bTa*5&8=di1t)=y=`arO6nmZJ-@Un(ez&Dt{! zJ`7ufy8=1px9GxB$xE<`?*TpMlHhaZ?jGNvN+^EJ_E55ppZ<1_Peaee41W>&q>w#= z-4{^w3L)M)iyIp7;Nq{_G0+g-c3n!w0EmRLtp2+QCdz`-@0qmCA9?LQ-@2q?G?t;x zI6@^7H7#?^l>hKk<405Xnq=Ju5R3Wo-z?_q!$n?mmr&d64d`C>ED?fnA{|LMWRd5a z+@nuuCk4g8z05xKK4r zr+Q`H#G{6|4~&+}`wXC9jS?0&Al&13BH-;-h`7-Ze*a}U0L0<|zV7|@#aYz=9MgnK z2wXDMHU+ovaN6&I|H&KFnCcpX=AL~oS>163tOQcx}&sAK-?t1-e4DqWHi z^?t*c3k{H@Oka$J$qH9%0(IcjC7MJN6xZpHpi-@2yAxa7p>3oGQ?_~KT6ojsG#O~kQT8GWL;3&vWPqGXM`= za}LbMtce+zvD~nIvC-R7QK;DEcJTV??^pYuAmNP58Kt+!n7hm4lbtF*2Ct;l#v0*Y zuf2j!J-dO5Vwf7!Fa8_#*1}0T1w>kP-{n5^0=<#RzfEXJ2x2CJsA0Bu5HdDdBFq6{ zajM{?-pHgwzZ~h5Z2}I%pN68uXyz;H(xeVCzc@TN^B&XaN-UQJC z%Q3seK}KhgbWQm^2CWYqWoJS@xvg3u3L+vOkHpI^)(A15fVMe(B3}*Ej~vthYp=b| z1eW^LrwRi7HA=yBX#q;>#1OFHQLp*)i^XiuV6IjQ^D*NwD0&=f0%Hs%^DRJs!s1GJsOpm9T(zEu|GHOJk&iM6DV>K-6=RG6#O86M93drr*bbp@Tqd-L(MR6 zP9hvv}q zPJR0Zb!4=-W*fk<0sTKUFPhr)wd8&2WklOT65^A7K^%$lv8i=$N{4HFlw5yAw?lUQ zSH3X3QJtZVB~exyn#~fMDuga(cXgX=uTL$bV+vPwYQbt|g{9!qjo+#rNm1L^a3=CD zN}*FH(2JL~(}4_sVg3f=wh&B4mjzM%YyO=QLLM4L3%madj5WG{XET;yTo@K;wp-lS z71-nmEuzN}n@+MI&xbalT7J_t*cv~ehKPr(!x*Kt*0sXml*jLJoWJ8E`^%X%eN$7E zGyZ`xE2*wAtU(q-jEIYuwMtLuc3+(h+znUA`ayJy!`@QE08&W5p`IdyJZb*?@`@Bv z1ylzp19apkM{*ajhhNqKe#CcB)?H-J^~cVT_n(KJ?7GBp{+k{vTBa!FTuT7yl)&1b~V|Ry2c-4uQsM$PD954|-lItgTMd+7?dsOy1_~r|k;^&g{k` z8QN`6wPD7fd8v*;q@g$C_27TaZXdd}F&FSF^((|22cZi<@pK3uMs`MD5_0?jYApfe zc^6EIXA?E3hV11s$ZB1kh!YkwArc5jkvkIlm9W8q{`l*F*XIb!;&tFmV>*Oo(ed@x zP%@j#r=w2#x)i+*L&h&LA4WnEQJVGxvZz#tY26~rX1+5ok9kaXCz|k|i<@Mm0cqNm zwv(AsV#_()&_3yHJPbOA`%nzki@yNu=LHOPa+Z=#e+D*m?LpbbC|IkLe3BVsMYw~e zJ>Z~6kUNXI5%_e#ngmc7;;Noq5*e*ILbJg`F!Q1%R-_njJCoA=3vRfQst%ABC5c*v zu)4K~u1E?$xAn||BY~VezP2WVi{MvlXLEomF{u0&QJJ`^4je<1#udVL$FkD2JFCw1 z7Qc09{~Ed^VY3qg$qlKFMO|KLWd>e23Gu=v42B>LO_Iqy8Aki{xeu_unN(cQRHq6I z3R5pcejEc)9CADV^+x@*^$HrYp_!)eZ3!a3Y-Sf|ZFhGvhF*ieNmqYHAKlHX1|z2g2#h^KtnF;P%xK z!Q-2ppf8?Y7(0*7uG1F~6J~3tDG<^yY&`5C^Simm)nHs2e|Q<4b3p#Nb#gQhe694z z|N4|l)SZEgpbZ3jh4Wq3-KX&EVzX7vry(n$wj$m*crHMy`|Btu3XXmyn@1+{F!-r3 zF&@(M^Mn5QnZ&ZnfUV7iUaRP@J>IJ$p)&@)eJoZ*nT4cIa6_9eLb=9^uowcy{2Gjx z!Fv-IFVUFnNvy^I-;fVAo47HFZ6!z2e5!yE`5 zM%Ae6eO3#)Fs?uuA(}ScSYy7Qf9+ktxyNcB5zMeAX=(}=>Y=qGy`c1v31^lD9A+Ka z1s5J-?DwP_h<+ygl90q4E>;2*;U^p;aZH-uO47K%$-i3xdaQQix}HWZN8$k*+@G>r zg>hd#U_8~@3L}IyR2(XO3bjQOplPOgaeOZjo1SmUitGb8 zV(5dQB-CC?Er%%`m<=2;p)pzM)9wQNH~q8Vr8w2LVb_(D#9emvbCVmciEP$5z<2c| zhYtxQn;b8)$U@ga$`_s2AwSrmWp;7TMPGG{J_6N{IAo)d&+)`PF4Z4M7YmB1!|AFf z2q@WZgV}rqqc*o9=>-XG((_4m$fW^|+OS(jvwPM=2u*EDb*4eL?B4j9TEcibH~{j- z?@?!hS*`!9XK){E=%c%LTKU2!eDyKwkj|3(d@c;>1Ak9u-M1(A;E>{V?I($;EO7VfEVi1;P7Tu;CK%uH}o(fhYQ0Z_GjH^*X%K>G{%>meg+b@Hdeyr$O0> z*hy)S0{$Q8=z4a_HDR-W5A+O*5c9Pvqi*)*xt@TrW>Y8P;E_>kV|WD8UA1 z82HpFh9Vd-<`e0GmWs4M;$iSC91af@8=fo>c}x@lKAo4ra_ZMSf8p!%pimx>li;3$ z8oSWsRp*U2Zzg+zSUdvSx3X|pdvIKQq}X%yOTp2#fVTp_k)k`m?Kv8ab~v^UtSkZ9WT$4a_BX$T!0|)5jqQ*gPl4^;lBYF2MdDeI012p zLoYbf3nx;bv$a9&(C$?@4t!=ee&3ey{}97@4JwtBay_kgVNnoH1+4L9pc!+{tUJy8 z`f#3chm;UME&+)_6!RD%qMzh<0sTcYhze{e*}(-b>9csBn+(VnVKrU=Zp&ibwaNGb zYQ>Q|PvML6aZ{e)I7%=c0vDhJka-GQ2C#sk9`Y}~zC5V`r6_7$Iu|-Akyz*)VhZ{k z0h0~nx?!Z8g*5^VqGr()C*9ar#BzjEfN@}3z3;P2Zynb3AkcA-*#`n7k z5-)0vVUShQ_%#;@$-0fh-EeS1q-;C?R30|(+nAjPQDw!cUx2MAvADu`s>A8AfyfX5 zY3n-3EFg>~I^mYtUhi3z_z(bQai`zh+?@^b=}fxj>2Fm%>mU!2bxSi+!OJ%LUx!g) z*$5Q-g#!{xL8kgGV&nfh<*|TMUfMo!utXue8s;F#m_9=O(3-JBb*Z{Bzu$hNM~833 zD7z^7l_CikB7HZBr{RCo#u}Dfu>Qj2=N-d8^JE5kURyGLuidl-78}${WeAeZX`|!T z6e&FRsU;lxJ7bt2hK9LP0rfzz7DS)!M2?064F+ua>HcibP|1d2AWr()ri2ui9K{f# zf2AIWlk4Ek{{SF@zPk1`Byl}eX*}9&Vo9S=*D{#0g#GmbaCpJVyk>(Q4*FcZIPOF4 zaf;7@5EGT^b_an1FQUf(YyuRg8i0bKn~o4m1&GIP?Y)VQZ@Slp2xSl?aKmVl*c(tn zse2ZI`Zb8Qy(sdN=K;iCZlLH7y|%htt>~%;zj2x=Azeiv+|isR$q^F;z|*%o@F9WW zYR^=q5VLyX1e0WJvJDC^apI2>*7vgMuu@+n{+B}es2&WnjpPMgh&tIj3+&&54xba2Xgp&q>@U-Nb5 z;G2!ib>24zj34&dHL zvkS}G)1agqmH|w;q3Z-WbE9Wfrnf~#e6Wvle8E-j9^_j*f5{0G(DmZpY=JJCq7Pha zl(NW4?(3L?lpvGXf5REFtISxd4M18^Cp4;mQR73V*m#1Vu|B*_oBB0iG4F$KDo+N3 zDe!s)SOml$gQ4ecr{MxcHRnZ{6suh*pGn_!rkwD)X@y-TpX5Fa=(anvbFMFZXHg{x zB?AF?X}iIZ^2RS^WOQqDZtIw0*dbV53k;Rqe4uThLWwH70>oWw^jMY9Tx7g z=Y+sq&vh`$t`SIMoCpR>hh_k9Tb={oZoQfvbZV>wQu*&Z=~NPc`CSz<1YT#+Ma@=jv+tOjU(+siDi(n*vBfS}Tk3&G`ISUA>q@aV*317|2IS(dG*l3)^9Y#^lb?)=UR;25!CVWYp&g5$zxSzb8tvIZor#s-dnqYf&f-UC2|R zpQ!7hx*FVXSmBZ}p6?FsRb7th3WvRcdHPY>r4{?PR*3Sj@@$4@G=|Bnsk1Ljs;t`6 zl{=yEo=$a2%M}7Yw|g`F<;jj9aZNP=S;>n3Hmef_cIN@5WX6OSFhFP9m2+3#Dm~e( z*Q4-uH7|fnqskT^>b1OwC>e!$Fh)VZT{c~T~Y1AM1~P=rsd!(NANK?bfy{D zP=4T}B*(^C%mDvQ%U-ndE21y`JO##4(%i8Z8Y-!}A$bjBG=!nbXjDKBCkU4$Us3se zX(ZIPfq^V=V7}>OQKE~Jn1l;!o7eR4iVN)aYu`M2RHehu5{k61G^ft`hP(ctX&52h z9~H{PNhLs#9GG?!9DZj2FG(mA#}-ByE9>cA+`9!Cf{MvgwK~-x1lYjZRGK|^-Lvwn zd1c!u;A;3)W>6ubwZ5BuH@N zi&;)(nl*H?cwRDtlG88PmK_sH!sb0pjEKRg?|1h1mQy~2sWTgrxnK+M5q8(%o(!H> zUpOzWnSl3!t@jpSgA=KYn|#9nc{Uxed30~~H)xB*%=YAgD5diK$*NHhGC$~ds%mAu z^jK}*MUX?4=3Zk(^-3wc*;vo<32FETYDQw47>zCYE@(WdFiD8mjz`iitNvcA@{rYE zWOp6gineUrC;(4X4E7F~GL*~xwf=&%X{R@h6lMNI7#b3A{khuVi-Q3RNe(SyJrrp~_@jA_>3mCW~ZAN3|QyOE-*K$t`V zY|@~oX@=qoT9MB5O){QG=Jd!YE>n5!zbG1!vICUQNoUYRbtK&~ycAW5h$kh85L68! zon$EAE5R6;xZZvZ&3-w4{yt(EHllV+khWF_4n`<=d#65}_w9wOXS3%Ps0>@BL3wmy z_ajC1N51gvPu?KR9vM7AE5ylYH$t*R{Xt1d#SnZFMbtGPNuP77oXFW5Wz9(V!BWf` z2Yk*iBQThe`i>>pOsifDbj!4TyNVBsfTIBFf9At9d z^MR}T&USQk)bG_M!M~`6L!ATVMYsUN%W{~_E41Px7gN6hx|r}#I`8uk{j)*XvUZyKU0H5i8c^A{7E|42>%-qi3v5Y!E@Fb7(4&q zg!vB)?F(h}X*49?_m9*pqzPfQPp(iUlls3@d!3xQ8u2iH@u~Atk+bI8(VCEbH1Lt^ zH0Bn~y~S1^jo;MB-hfLG8BlQ{hP1E9;+E0nCJCMT4ImA}Nm;v*F+ez|M{~Jdcf)rf zrgl*#s`q0wkJ*ReAi{7A(_w%%$^yC|tOnH08t|kG8nxpj=$qExBbxKdkGdM^+dB_K zH4YMw)ZkixMo_BLQre^}9_zE3uXMHvVnT})sPvsosUpqW69lX1b0K#lgQ_bC^{Y>U zKrq5^8lpG)FEQ?z;>n{S-EpQR&d2-O54tq)xci(ozW7M^{VQWD=6XLe$xQe;4elb~ z4J@P0c9BwM0cxsnZsVD0mHwU0&yCY$c`$P1YUfyw5*ZGvfs@d>wto>Oj$jYUTnpZ` zIsS6x4{HY}`dY!X#8{k)orEYOdh{YT;y*(Z(9D?*N<8DXQ-&ny8!4X%#8q8TlxUQ9 zz~ZXSPF{4Uh{g{ts!&j4Z7{R1XYx&kJm6h!)3CY`y;V_pC0YykE`Yai01-z2&^&}&9Z(hpU!1<3dV3bd;&(0ZME#|9 zDqzISxab3&sSWF(+p=z2Z$ApXRdZOATK^O%2L)s`?{V!0*~KTXl*c^Lu%G5c`SeH# zyY>_bv46TJz+aiD5C9>JMcDZ6v*;8OktCvR`lcduUo@WYKwII9c?LF|{8-3C4(eGF zx!+}l$8Prc#gRORua`>AB!6A}H#_C}N+qy%-kp0rPCSCM&RJ8J$hQ4{hp{k1{4NMB z`Sz|Zb!kIZrWGW%=g=**rgx~#gZv7vMUA)wzFDM8Dx)>^xp5*PS1;k++OvEEnxyOE{7 z9U4J9fr%+_cE=iFNBprud^v=T#K6@K`0;lsOILGe{ImxU+UUXSd@+;$<$ay-;3GAs z8yES5@u?*z&cbINC;n6QHg!sNPa*#K*K|0`O4@+|!ueEHiz@Sz&USu+S@kHwjg);) z6ec`%;1J6HW0AhFG5OdPa%d8H=WJ!gfwl$q0%lS<=4CNoVvZ^|&% z>^j?&7~+!l1TH+R;k~-Z17hQ#r0&u*h9Z2vH{;4G?|1EyO$bAIz(bY7VYcuCEcVXr zbu?@#+WM9gJVeW3qLG>;clkM%dcU@d8Qk0721f!jv9utgVAzM$s)y90mTKrq30ssp zUHaX*yFXlpIxOz?c9+dL^jne39mzPue{rFvjMb*QQGD#kuhtMyIp&bjal5!(?JpB- zeI%pX@N9uI!g<7Xh%X;=T&b>)2#=$MUv8TIc~MH8#Wk7mffS``yovy4%>ig z22WnHE6AMNWoy$9;Hmwp>go}8pgA~=9bEj`Jl19BF*}fyqm}Y>`70RvJ(Tl#5x;UJ zct}qIj@`2}{*Ju0XCXg>hERw$&UZa-x(qQ-5%Ht9&Y?$EYTI!WhVoH}R)d*dC(Nop z;qf$)5q;yt(;QmoX5MQYV!{=TxSHJ@wb)4984OX>)M61+t{hL{Td=?aAn% z=drHG>QsHuh!{^=h-W503BOyi?VpGbldu=%!7Dp^vm$aO%! zkQ?dAIYMCALt+_sD{||Bk1VVC{D5lCvy%pFY3)Cce!k*b!#egnasnFax`0-Cbq7iv z#{JIc!otHd;cqFDAskAaEBsJL3PkCEr z5{T$@AXw%>9{7Mul{{^^(6iKi;`~)mVLZ%&SJVy-4KKyH;5&WcflVdx1zYcDU$CnH zZJQvSBXv|?{9%-p<#kA))8QHqozxQ*M*CYNF0gXw8KvdG+pp@_ww(q30IrLK10F~qh7RXZRVOvAHvxRB!vNXvo( z=g0&uz6b_fGsmOl#6zFOUtw=|33h#_@wZokvfyJ2DYZZp(XSw-Q|RR~H^ZeM@r78Q zu#NL*ZNF7Y0R63_8GVoyCV@!m<%av9`yucNiKQjIltgFa%l-7DM&~M=`Ka05DMxCk zI5OdGJcOC_+T;5*P0WT8js9%e1bB$D-p|wa?FcA_GQg(P*n23n$1jM49o~r(Y~;B; zT1(N9o3h5dR)Y2)7e$90cyy(;K94%@(++FVpV&C10^x~}~{3n09+04f!2 zdXPwy6lkS`fzuKgBimU+%rf!L2c-Y~OSybC&$+>~pduy*T<}cdhrzq6%Wt7eqWtN{ zA_@qYyWzSs4gBpU8I1Se6E#9OPAyoJ`<4Q=3#Oue`0$N65tYCR!WDX-?9RrlbT*6K zs8M+MCg%W-P3)6RwM_^E2laE3-hO}RejbOGs{NN%wrud7^wI-lxL25f&=TAR^f`5a zvP)_GmpzK{)Glg}bJ^&{kd7$K63!FacVU0i%1kf+3nI10pzJ4nqu(SA%6o$GY`cQr z`$eD2;KBy)4I-st(}An>_ONM6ce8MC>JPRvk+A9ee1qQaTdV2K=) zacUUgK$=gN^@Q;YtaP-q`5s^Ko6tKEmNJPVePEy%hI;n+)RUWT{~S?T3rF`TM_@?UD#Ci9m-A!5JC zXH?4T2ZbYyaF7&iOE-|b7*k~K*^n@qy+ z4ZS8JEEWSZW23yQ7!ZsLExp-4*1N;xpk^yFrQ2mr`>+A zc@FCQN0tp!D?iIgBR)p#+J_IVk=rbN>plL4?QS>6dzrSxU!lctTrI#+w!1bUfTnzb zG#Iac>0s|3%rDwiNNMwcz3=N!^DYIEU$_Ad8Yt3!7;HYY51SeUphOF>*=*Rn_HpNP zG>;Ba0_nX^kP6cOpy5M=__p_7@vXhg(fFafZ&~6cU@oAWLV^7A6U$44hhQcy9%?o2 z-af}CD(2+7n+isZV4lyji6>EKInIM6!)W zn+}C7tG~6+ZxTx@`Vs&2kGCN;@Z$f^Vf`De{vOtwVM-1T0?Xqaw^luBujPS1`Ulh^ z`DmtGx?ci(aehlag@#MpQR`2SF21H827{>&Vd?g;WG8 z-p2sThE4pQi-J@sYIyRidel~TE0UQ%O`Z^VyuyI(VdEsa0!{KhaG(w)-L_HhYdfqB?d&h5bqnVfB za+$$Y25h-F__YNj2mU6+rOVi&RdGqJq$moSnvKA>`pRB)Bbj#vJ|&FqZG}Wo@v_qq zO%js&`*puZ-#>wpdI_{ilfSkgiPiuY=4U8N4h(s@1G&K{r-T`AFVZzeNhFp$A|eRg`p`o4FEpYEQ~9l?MX_H&>eIM{@__l8sya*SN4iSJB-;Z4}s8w*g+EJ0h{bw~YZ zHhEiw2z`UjSFm!>gc-i_zdccxSqJEN@7%^;g934vgl)T$Tzia+oAoQKD>arG1D3Vi zHDfZ6orXipHH~;#==}`X6rEQsg$Ut89s`nsEU-n>WcmVT*AJmPbU89umGIvzFt3MN zqprdZ+u%~JnDfmUr$Mj*B+(U-)%M_Joq!_))*(-Y1yCcaaFd5d_O0Q4#bHgkK*-w(1;!(-u-kYDrico(>j6}!6V6#qe%nwSFkYFUX9!mWi%}W_f1M9WhG{H} zLMDmO@1+R~^Ld&614-?{=5H!x>w(G;IbRi2Tid1EXs`MTsy zXs*?T0RXg5!>^$??s6I~id@5(ZMb9j5N2Ji5KeOVp!6nmX|LwlxD(>Uh%$f+aa%tG z9YtNg)s)@0QjNy1Z`6X7k;NP^%O`H^xtdKKUcaGhK-nImQYL1@UmPOHLES{+9f13O zJECvD+m9AyK-rB^g0h83>3z)h=^ar{hY6YxuLVTE z!XD@Zsan0&_rxRUJtQh|glLPxiwYNzoAtd-5vR;CjV;RcQn-Aja8+@A{+9ZpcD%p= zc`!_GFkslIqTOUUekMisC3X5D%MN@}MZe&z~uAm+z2)>iXbPP$l7LeOAPl{q*dY*1l6_mX46 zcwEK`1CGY4+M`4%o~OOa*XE((d+vdNLMuk*^g zUyY>*-L<#O;D6M>D^CHk;heE@B7*hs!XS zaG^xOD3k?2)oOz@!x*To?-~}gk>;TJsM>21kLgx&YQbm}m5OoT;<65XptK0y-bYlj?u;~?fQ3`jW81*hN~@INPaSfGcy z%1@PcBQP)xyVuzBzSDs3aBhdlUI>2Yu{ZGVn8G(bp>y3OE=RA}ZYaw=v<%kYpQ|pE zF?O(#zh^zT|FMPPNp`y2uAlwlYv=9p#Gx~`DqeLosG4*nRFh73?HKfpT2)HL{c_J? zaVeI{fEWNn%n6-TZlFrUe_G=42GytUu?0?dhay((sXYAm)F^DrcpvkFv z7JY(n_IJ{Gn=2UH{to{wVJ*Y2Cn2rdK84EK>E--kwlJ z*ue#~aa9F!FtHsxgNT?P(BYx*d`$8Ul!=?&JE_Pq%mcX(o=iN1n3$b<)#+oX`wcv8 zhwezz-6xhMn+oIfrszbUg&!{=^Btd9QDa~r`Qht8-KakX)e?d|sr-wl|mIdnN z4wxfk2ikJDqZCIpkO?*_r<)#1eL?#A{z^+I{Ns(qAcTk zXgnICW^Tu9hxQ$W%R^EW@ zcmg_pb0KG#*c_<#O|Z1IM$JDb`W4F#z7tsc#oO$0->=MH6_zIFrZ2oc37ck-zpSO;Pcnw2M;YQ7X^L?xW~T|Gu@Gm3H)i*{pUH zk7|V?WL3=4F6rY)yQK%H{i`XG(e;pBE4^(&Lo4;Zdr?MMbdZm*dYbT(llFXngCf1T z;gMeK_To(w(jfBx&x>bqCh#2&wla@Gq|I*(#R|xozKc?=WO{9^c|=da`Xj_OD3GrA z<@6!hN{@R*hhCi6^>pBxOP%9QVt8$2bdWS#sBJ!{$;aa2w%oTD>w$Yy#ttId=V+jsQ<88|xJ2Rbav!n_R+W&7vOn#h4Fujtbcc$LALBY(UM?2AiXMou_lS2oQ+ z?kRii$DI%P#^plKz-SZu?d55WJBL*(%%FE~4G8|Dke)RRE>`>PBJv#%0Qxw1SzuTZ z>Ihovn>p|WN8n!Ga54QeoFWyBe^cdI#JXKodlQhfm*;U+UzzRB7&F-nyoGucF9DUQ zuQmAiY@c0ygh$dF?3Vn4^5Yt=F#q}}qKH>Hnn@?1{qQkpEO@SOPP?X6FZeRUu70q8 zGDvBdY@o^~Y_rBUb`$sE(30My`vY0iFLf-tDIn6)!Uy`_yt96ZQ_Z!!aD2sMd%@!V z`+_|p`!Kk2ffDEp4zcRll7tP&4m&|E&L8qf%9*V=Jr%~d9JB>x92Z>ip6IxO3Qa4^ z@Ao~!5rsLbBKh^LdthBetvEe=mQZE7jMTgTYB|tR0O;iste|h#?nzWQ2XYXNC^!Y1 zLpAAN^o(hGn;ycd5z4O5-5XA2YJHsyjR$Z=0Dbtk)}O7|s@;-5akgn?g>AnP+MMV^ zc4Gzp_&++s4|QbNp52^in^GH|)Z`Q)0pA&Ic9;>~{07njp-YZ!Y|B9MYyl;?Pafv( zz{t~M*MX3J0_`Y5cu@ubIuG9zCwwSmjVL>)`8g3W$i`RiN4dZW0CY&k&F69o8O^$r z&EIdxd@?=IMLjD99b7~~F5KozWrZ(9x~3GOe?#`A13K|{F6+mAhid%3J9>dRH-pc5 z)ph8i)PoMfdn=#SEpOvrl&?cXd<4FqBXJtA)xHc0)KL-58*iM?)tb4$p0^hh#9c?p z2kPh}E#9C58HdW0s#L>5Mgh>gSJTS!5D;!%|6?vk_{vH#1Za&Sd)BK&!m`b%uOdFP z-VpIgB`$QcHh$bv68t?qzB>QxGzN|$moeT-|FKNnGbbP>_^*3^S7aUZ9^Lod?Jx;^ z7&{JN>HA8+>`egGXEX&emM#SSK#@oraP3!%hQR!S$Ei~lY11^LZ9Znkfncz?z13IA z=JRb$cLUaSk4#g6V%+MxgB3)z#(>|w%?>-D;>3A`;IpB^ZK!O|Dv&H#ZOkViyAHRk zRJM)6<&c({nb{DtZVci%@Yx%bzXutI_&z2Pr3Sx0u+{v5SG0vQu>@E{Cv=%LsH~^< zNkC`JHxp2LssmfPSSm9a6nF&-=!P1rQjszcE?IOuT852fF2_jVj2g|fC)%H=_nY<& zZIrb_XaxN6i_WDS2Ykk#bvkC{!MR_RJ8PNAkHyh(O&S9Hc!=)D-_3V{Ecg~-il{jB zj}()DlWGQ(ScV=l<_YYGK2q;x;w#0@zZr#E>0K7&`~^Nl;=b{-!FXqbR*hH3j!5;B zEK%*8eP!+8FYHuyooi-~KJ7xeJtqD@E*59SIiS5ifZ^4P-Z+)Zm$T;k9_2-~VQ&Tjd+y%Go{k z5dNJ#a1#s3#jRtFgv6T~>j57{+D2Xy%DcRq3{DzUUa9y0bNdx;b6X>>+e}aJA!09x zPi&j>l^|~#(uMrsGv`rg{`nr11*C$VPfg@uwF+UyVG)@02Mrj>>fEg;kAmLOnJsxJ zPmXv}2LeEe$|w5*D3}pIIU)nOqrA=ShufO~`ThsvXU9ouGzaIb?Xv_OEgrd2v9Q5u z68i_bLO)8)i@tlfT;o1*_<&v27$!wqc<=0`XGq`ftXUxB{i(?uYUgJ7wQ#C2RHCeu zA{8P3dAx0s!j8er>FbNz5qUt_b=Ua$BCrmie;*HX7Ay%B@x{p5PghH-XQ`M(0zn&- zpHxRh!7Aq+gLw6ye~#T}8u8y@;p-zoW^f(=99r=S@Yv#Ei(ro#CSY@-R(m$uJddTB z`|9oFe9ZZfN+TfNTMHKBANmV$8U*kimkkn*MYmP4ji95+nR6Be^tbWo5`uh{3=bFT zNg%SRUGr~1;Ean4Jh-!TU>yc0#X6WdLjyu_yNN+d6BSCpc3s~4_mbmAd*g5sw>t|R zT;iknxa~!W`o9)MA<&#tE)?3#bYNnW>Od1a;85F3LQ1R+u|iC=YYIWb-~ew#In)Yn zak;?Sbkb?dWjq`y$uFt^Mg+e-DI{3*?H!12EsuiB3Xxv%1=>g6ygKo&k5+5%7L+3VeW9n*g-G< zQ!<6B>-+<7{T#mlg~K={lSkKLIICq06&iqG8kW(Z|1WfQFdebY;0pM$r~wK6MdUTi z6e)#H`Br;5#?{WLP0;b%LT`ZOYw%!(M%YkZ@GRw$zxLx4GxTcTLDQ=<1q_~>aLq(e zIp4<9(9odbv?*2oxan)}j%+ey08Blpcw<$kd*tm8@KbCDxPJ1*K5|g!60%f!NJMzqrFN8*>afaR*K4@=0~*0y?~U;nqQOMeuyG34&Ye4FdBbC0ZzD=v0IoTy4^*Tb z^q(z33)Yhhay6Ua71a$)vgrm?m-nD*5?(+t0*bnxCe;5k!fDoOP6>q;-A7Q5cS5uB zXRdF{kX$@Ca)#k9JLAGzDr67&JXKT_IN61tDrAkD_x2E*L4`8ZbCRxyJs2N08loSB z^4sBFtIGtSDX1N7JC7c*?F+#}dHUiRmY(x^-M3uKLYp6x0CaD0;t1hKK`%rCyGXEj zuLu_r#2zXl_q_+0(cj$lXb{*=w(4VHk>3>0iiTJD{b5zv-d9Ke`@W(e{@Skqq$y}D zP}Z(F#5(}}pICX&t3eN{DwId`gSgm9k1Zg~^TOZ}chThO^q{$?Er z0@ZUn9M3WRGa?5M9y|-p4O&I1MnSjnle>05;;B6_j3N@;5qTFpdLUToKrf(&PwHVL z@L)SXpCvRI7)Xz{+U*M*JKBQ(y4U0%m$d&bqX2o^8#c} zw7x@%Zol?Ik`>jNt6j+8YWeXI zg&sVh9(KS+Wrkgk>kl<9B zy?gr)9-OHmK8+GHB~F*MCA%h=#dwB+3|@1{fHsV@4XYjwy_P8J!3XM?i<+*`+uaQj zRwq^H*B%)92BX(7NSoGu7AnYMTg;m+cmSNDTV$d*`XYDPjdr$MAj&P{f?qy&b#3y; z1w=^#qV(bL6B+oSIF7!*KeUB~+F>X_T=YCjycHAmDawpc(I0O_$+S)Jfl~ zPgwI=FY|6q+`Mte;z3Z~f2yd92m|)fv0`S!d7KF%0d$CoZeFi~UFZ%FhiV9l?Mjh$ ze^vc>H z?P(26>o^C6t?B?D%EPW4F$diYp$I<=7|}-XfebZUbne7^=t~xMLEzdTIzUh=p(iNgQ_3p8-T4hWz7?}DOFRIsl~1nz!L z=CF2^?x38@DZNukeDoeOPQJ0ZQw+Ss#BF55n+uQPcqDeUF2;V;orw0u1wXmcoRHmg zfzQX%v;A|Bi{g0v-aYs`@QoBOVNUgkT^cF>l}Hdr1}crCDtyg;Vj6`Sk?{GMD}P0L z95L?z5uIS_0{A2wLJ7Ic4JN3qDIn8V*VEgN`m_?64jj+`&9J^?kA45_x}WaFT*Jct z3}Sezo!HB#@dEO2WxKzypYseCFcGmkmY~VGuupP(w;3WzOy7h)Gr(%|P%_rcIjy%OUh;W%ULHtsQWJQNLiLmJqhIz6gJ4E- z?a)GH7nD++$!D2;z6m9=xK9lLCwP$~C+3JfgY2?>XPaZ4V&WWjsBg9q57C`Fl$YK%eBG2u>%Y_oZ2LNpM+l`G};AyJb5TWx@u_{6fm7qhJ89jVq^IYPmB z4~f@ni>sdHNTFLEO}} zBFpI@dn2jfN|%83%Ce=gjjAuEM234XoTtt8O5*nCvP!zyZk_XJdN4XcopLm1H|EPB z>U&S9t1r0bE$_Xh8jn!7BMH0JUj2ulCKRW_DB$iEP`vQ7`d4W97L84Rv*r7XeV%}= z$RcdzFiL>qyBV_2#Al~zJzA<{@h=&>d)(l)6JyK17HZ6`0Ffb@nwQ_qa0ZZ-kB_VY|x{|OsKJbiZAB)77D>CE|HEu=_ z_EceS#n!1mZJZwlLs{jhTpjM=ZX?YM9u4s0XPw%Ow@B@D`O7k04d&gVPV0x=?v`kNL1)hKaH}!4L>!E}B zuPs1Eyip-qMo#Kgk&(Y1X|3CHhSTE+47?j^4G+eNKHTknU(?JFm_&j|$52DPz{=KBD2 z`q=Fwe;g!cm~KlGi%?+OBL4Q;=WTxDyYPf`wraf2Hv=;R$>_)hwZcmBhfVngLXSNi zS(nhJ7nG*-O)=8p*=0Irnk#P@n>-9zt&QJMF=N}R6(8ODdKsY{K%QZ7 zM|v5;Ks#~ocHjXE_2FdUsPiW4T^EF_??<1PpZPlF@|JQx_g)M)3Zo=>oC8N-TfDB| zl$F5ZT8#}?p4QrKnmkOOW~_2E`QCC9Y`P*(*UWSINn6m;^GzoaHD6Txuq&v6x^B?V zGB0BvydCFvzCeKc@W1=okq;@PrqrxOe_yZLjA(Y{{r2oQ0RS$e+{Tb<_e^}G@TJ>e zC({MA^~vNNg17prv*#7~<=oS~wE_bHfEQ1!Ire;7U^Naki27Q8b)Kpa|54hwWKP&e z?h9RtwBLglSK7F3jyEVK$;ItiVVRC;JAja~%qu4YO||Io4_xxJJc#qOhhzDS`cJ7_?-^Ha+0|?qVLt=W(?u zN$lqNT#aXEU-{pXnt0~bZ)bNB?e1j4?ye1OQVuEMpyd3n%?0}4m1D+X+qm_f7$gvm zcGLu##X%L5ObTcFkCpp&3b2J<;#SKopOa9y`YhLXdBRg)^jgU>%Qrf*_H*gU;eAE6 z6Q`eEz{MS24r=5W-Um^@SB1huSxFnpxMjJ5nCnbdZhE^$Y6A2DzT9$&R}XSA%K?bL ztO&X)aO*17-P3q}xA>MNoLN9ewQm6SflA1}tU*8DSMQ8+^kigdHe&N)YnA+EJvul0 zK{#AK4iZ`{?%NP2?>>wF|wbAJSjXO1s4ufJ^?I z`{te0_oJ73cCVXKjA;x^_QT89V8d*zOSwDN|rJ3S$#L zGXg+I984)1$#vsiS-`mbzIRAdy>n|r=$%Uqav8dAkBQh$KU$7Z3)F8w3q>bXjJqRw zQoV03jI}unF)htLfcKzP%+Xbk4%f!_`MEypyMyV1X3p+?G6Cu~A0OU5X5DnMoIYsl zwwq1Z(631W6h_DwS`G(R`-)9u&gr1Sds#mbS0!_}bWCva&chWdV;20UGWo`X^`kW@ z599%WNV#2WX-$FKIv-rGm?Evw@&0j~B2O?I*9#qGK9dR&dcN~_I&=K`rk*_P;F%JS zVwF7&)v9)H66Ci{n+~2&jSD4z`UpiAbvRhOEWC|hPyW9Chr)e*TvXOy;313`)dH9J z^m=mK2~~34lP86KR-MJQ{wQK7E-t-3_-F87IICo-656 zF=xJIM9Dn7v@YE7+FZaek7{ugcK&QOKyHdlCWCU8f`<-KR65vC3yUtcB?}i8v$zhC zu5E_j`Jfv2R4?m~8zVV>wMN0+A??<)tBo1ALZ6&}_4Mj}wMaHKl|7VQoglUJb7jAJ zt5N;NGHw|P_w*8*Pj=rHm3wNkTn6WTa`v9HLdiXgw?a7R1!+smJ=38BSg?3PUB#HxLppu&F9 z0BDCE3%ic+DuS6Zx6@r{wJzuH|}`#hegj6K(@vAkk> zV2iEx+MoMe;o&(KniD!IU*=E1_+1(gkg#T)sHY+UdqUC=Ok9`I*L{|K0Z?Qa)%p^m0_uUFv%AKvQq&aSWnm4h7C;HW9^K$pf3}wU3i#8ciTww7zs>`f! znWQzjL`l!0c=m}|tiO{LLjc3)a=T@r1Gj%KtQ#IybRJ<<4Lz^)>C3x-+CRrtb%7F)@yZz;zah*a^t3Sug@=kUz z{uD5(wG+Hhpr@RW%iWalB&#*o;ZK5XSNy)K_bQ$n6+dJMD1K1WU)=RmrPbsf#WVeb zf4uF5c#E^z&JWU;Y1aDk? z(GW_~aiIv+bcP0$o}}o$CpN<{QELx>zrCpUY9PX0YV)Hx{a(ed1KGa9lZxI7w6Uw0C zxgqeVzM>!!Lp?7FcN(6Fw~~S%<=XbIJ_89&4tH6>KF|=32t-?kxIB`ds_oB7`J`JV zmm|7){?##Y=Qj@Jm)cv$V`maYoVfO}%Qo5GR{?(>EZ?O}re~L4sXe%Ou|hFm7BkuJ z8@R$z`1W$vUjNvhL@M?3{VQ{)U#;Yws~CLg%cII#8|gGyVOMzg9fM$!upAyYIjG+J zt;Cs;o{#lO_U8}NNp5*I<+#sT6*WWUJp0%rPbIt$Gp-%VH2Ip=wcv}JP*w?{`qIhz zT z+FrnoSU%kuoMKV<<+)mw*GzY7vYD%#-^vX>gN#g}q1c9rSjBVA*3Rdf_8+yI`Xrw! zPSz(|`1wi3>+jRXt(%;0)z9^$uCaR>adN@i=R}I7DIuEzu7T*=E<-#N`#-VafQbAk z7_7B@OYbNYsX92JE2`u6z7abBWBT7!yKJ(?@jf!^vCA(X%m_=4-2=6}*i4gwh`~Pk~vY9<3bnc^N_U#g7k4F!V>YiVID6H%M?zn{SS4G$FJbg=_ ztxZ!VCGtC-N;H4wuFOB+w?GuZ;IDkJ->c*9yG!HX5LPz*b-cI3s?>3cGMCFEpByLU z)O+eKi)2shQTwt2y$nm4>nmTUmy%y79Ix$HP4KTO$QRpMl+mlrnAsP3E$Cc@P{Qv@ z2Ej|giVY)k{1-S3UPYn)Id>v*hKhB2lpPT5A3iy99ou zVWn#pKm4YDSiEW19jPgtQz>iSo2n0Se0d}1dG=QWlubu&K0_cGTI&~busH;4Pg+=DhYPPhpI?Bzh#&0G3e!+N`_iM)?2X6oVy3hOuyfb2LI+ru* z^>qHk95`721PSTRp<~FQ8DNOtH9qh2e&70jf5~#OW-<4D zopYVD&))kqjMpZ`YBQZDD$HI}U_1E|$V%v8hn72i?fbddu;InyuKFitJqkrr^-8Ka zDoVF1uP#(9s~a2J?vC?2p%};E#h8m42^Iu16pnIw91ukIWWc%u8-j~=H681JP1 zt_>9ZkUAT;03iXRxT15JHl^Nxt4WXY6gub&{izlw@Ikb8_FJ{b@fz}qYeO$$&sp?@ z`^2e-lM(AzO0~t*7~MFZMbGU&z&dddFuF$=l?*@U{>q11)PaBRYHrkNd*i8%X45S?3Ag9HcXjxJ(26VlS1;KO%V<+1 z+#%(a+9`?rjth3OIA5f@(E# zv=9N+1$7bUUigFGx^a(!C@=bt>e(4 zOFulZ{Tj!1KPz%oK!*Xv@i2GxpIbLkD!36yZZKi+>*gUMEdNikUw91-ZzuZ7`e%HT zMZebODWq0lfuXfdE>zjK)JZ~x_6jRS=lNK%^anz!_1^4W>-m8@#F9e>l~j@*{C+|f zLD~jIa@>$Q8}2=v8?Csg95N+}&D7L!4vx=ouhdMv1IZkdHl@x{y2KG;*@n%U9KVMUIkV zVcpA>RH8OQelWiRWIB;)()kTW>+_CRxpt@c?TjH6oVck^+}3VW{pWr{w8md43~uYc zteMMIXEOP}oCYxMSwWLl{WHe+mpX8i3d$`Y6bo<0)G^iP9g$SBOYKu)_w&Vf#NR}u zb|USVj+Lw{Seqo6t zhG453FNaorMW0dwRg2%ucZa*hC;chn(RH5dS(Swvl}}394E=9Qahomp%E%^P^v2rT z#O@Eg&$RTlY&n{8C%fnf<25Me^qfGZpsdzSrfKd?)XoKSTJoz)#vKISChOK-$nCU@ z-U(nMv2)R_Gr2RGrz8J+GuX)poW|MB6R&X%)e={xb{IBvtw4s&(fb#GQGy${=l zf?vXv{wKi11_h0*pXoQ1UPvsV&9*;`CUdud8!4~ZHTbDvUxePPJnQgmdeCszkn}E6 zSuHez`e_4lwuHaXD9L6@_+N_ld(2Q>s#N)b{ql^c=UOdo0v++QDkOYKvjV9^Ddy2% zOlNIB!VwMQHuyo=n^^s6f0g%>F^wh7xD43?FQpas8COet$!xfL%j{_gdx+$=sK=Pr z^!|~Y&D5^DVJhn_ofbbnVIk9j1|0^Q;f&CI?Z8c*-b;^u2@cJg+~L+h6q@LVwhB#X zuVWnDZ;EAg;=M+AjjUG&+#`DL0k!0h1;w@DfJaAEk2+)WrGLy_UC{G89xF$0xH9`^ zO2;7^a_7FMJ22oXa>zszNwJoB0S&54LtZT81qSCTJkXU!y+fmbSM3X zWuf86K+s-2E6(R{sDE~FX_)h&vQX&llPmznu+y4hh@?R0mIxk8tWoOsC!p2%4&k9J zzyOHV)xRRF(}tQGJ$@0P8clk=^Y>lc=ZGI3Rrzg_dlz__ny70?zYCUR>WI?$y&}lX z3Ac1iv)IC3go{skH5Fnw zZxy5-URBZm{cyq+{do=NE4Pn8bxb0^Gd>ZOkaB_R@nZj19zPLod$}~YaCo~G@0qW^ z-`a_PSBj`K%{b+$TbCyFfalLxKtl5WhCjO2><=St>eUz2NGQvCniTo@LK2w+dPA$p z8%zWPBzPXS)=@&F#XFPLN`w5*H#Vq9!zH20JC)^|;#GFj&$FQ3>IEz%K9l>rgXR4e zIp#{*ANNrrRhiDDTT>tAc#JYSeUAmj2jdq?k%(O>E(2Q@1r?qXw@m^RIZU{7Zp?NseH*OR~ z@d{q`t{R)EJt})-*x;?_WrL0tV6gnWJ$LmCz%!mgV#bfJDhqVhR(oms5`}wxN_&(J zM`P*uZ6=Hp`ClgwixfYJE4ut*RBSY?RoaOd|NJzdT*4l0Q9kWmjIRW+X#0tJLzG#A zfnXltO5<%3aZDrOs$!2vB`|$g)>mv zipU&d$k7tWzZ~gCN1cc{o5^{thLAsHE1W}#L{QPA*o2Y605~jUqe-LNwp2*eN7#Ss zqd-0E!al$oF6cA&h1G7A5`cugKWB7&P!>DW2UC~>JVEb+dOn*Puq$%E22hq7Z3QlQ zLSNC2{{9W}w<8ja9Fl}S^8!s9{=J+71n)=1pT^0-OSggp@rcz|_IYhbBd{I$&X#|b zS$5@lot5apaEo*sG|PYKEn`F;@cz}|%U(NhDEkU)f&GGL%lTdAv>ZBEl}J(;PYFOX z4wq*A7^9Ew>jYe>AQb1=3{Y-{lLdai>OD#?ntxs7?!1+lnkef)VFjKIl?$h45${P~ zo!{6WZj|wZCR4;;l_caBW(}41NmE&1YNb9H9IC-AutxH*9ap}W3FT>BtixP(`sM0>x zbEJdGZfC-@X)B*_o$0Z1mOG-A;+jd&l9)x;o*0g ztnB9bnl2P|plpTc_F zO%QXgKV~}AGp}dqg{Uns%C0B4t%s6GuW|xIP4PloeXuN2TC=2^|6%sa)Lb#^nYm9& zUACJ>SyS%NJ5^bJKnAmd7_Qg}Y3QO>%?2g{Ya0Q(gAcu{{qPJ^6BRgQDj5u9FchXS zgK;pEV-x7JpaE4hG&47sEg#y{xW=Y&8erk(<~Af+sYU`K$?CB&{$JL?+kVXGjqmI0 zIgGhQ0Ks*-Ye4-4Gt((C$Zk1hq!lUBgYgj2Jx<&wV_Q>;ognnzZLm3gLW~YvAGZAN zK_@aukkzdgm0Y{F9BMTMvlQ%&KMtK3b{9z13PM}#=l=5pfJOi8jMYSEI%G=ghgs>+;lX;EY$+9(=k7MS zB6wChO|FOr>r|A^%%;qAUJdU1cx)Eknh-u1DH;6j`YI5>O@*?r5+!W&AzR01zM!=k zhuWcD)#965FRdPBNZ3A4?WL8%=qMEqaOfw=qS0kK7sHk>)mFI@WWA=Vr)_6X0H-Mn zo@r^&===}}p2iA|l!mik=8$#-?5{`By603w&g3@NV!mH&;iOge zepgjo*EdhAWAhOX?;Lkmd%oYg3zdS@Lmg?#nIqOiYhfpN&Y~+J=ysyxkcTxbbt?KP zQLX0nM!e*_&jZ*~Z;X7pn55-@y`RG)ew233o;cMw8s;zJY+x_#d$ld<`(Ah6k+7KZ zP1nE#!%#Zrf!G#XMr2Gyqx)AW7{Hq^0dXK4Lwra4Uu{Af0N-~k=O2gxj{CneRKKR5 zeaF#W_sDU};~Pz~H~^T>?A^Fb@D;4#c~0|lq5J7botLR0TFmQU&}AhuS=e*QEnI*?0N0NyY zhlSB%!Xwh?lJzCM>Q0r9d|8Xv;gL6y)%4hQ#sJrXzsV^Lg5OFZU}N~o4?+6bMKbk~ zp7WTXLHBEC@$jut<9o#@hKZ0fl8qd4Si|w!RQPI7BJ0LazCWO?4x8hd+kvZ8+D`B@ z0hgU01Uh{f!t9H(fq*gi`bf=sQilEP$M!HHul!$ZLx^faF1>n9(gV?vInTHG+J(getp2a0 zo8I|V^2G1q01V*UBy<093|y@Q$53kPxjQh7ATiqi0@t7WCf zdEiAy>ma7N<5rClnyy*RK~`^1DS>h*u3%$Po|xfB9Op#pm=*y%bdR+BbcdI?k8(FT zw$MOk)OJie2eH3VDDK$k*R=EV4d?rU58Tv(-<KshV+{n#GVv?c$mHmg|cE#Z+L% zN3i;18xr}Qkz`BY;N~gLRJx))t=AX(E5$C;8&;-w$W$-eD#R@B_*4F{%QKncyk7ef zO`KwVD0DH+Dl|uh(}(){U|ja5;oqXl9u!qonYcGmF^zJ!kqBjx=ZAkKJOCH=#}qYb z??QH+0CQt^@x$5(W)RzARO=hs)u@)N8^6u-^OGd>^1Wt;8GHWuez#_(wxP*hIP1F` zZ^^p94hc^gBRGACY27!hWVT~a@th5Z?*rcs04O+MOaIK2@LMI!7MLaM;n0g#`DGTg zZ$7eIFj*A$GTn{ao<+`}ZRb+G9{#6Mm`3t%&Cp!6GxHEF_%GHZ3~tQDpNw|{5(KVM zoFAyS?nYYQXuOMbJoP4c`o{3;nUzTda=3(oh+0IA|7oTw^36eZ@5pW>OQP4Xg|9R> zxJ|FJl7Bh$X*NY>fq)Pba+T^ z;p`-ofenG)u5j`ET&ZrGXI?G<`8EnI{%$i0&3??(F@$H@9)3<;nLpf>YtG0+aG}+I zne#F=gCe!sRzc$2iqL&+dF|mIfWS1fq@UJ#EoBSP%b)a;^sn@03f=QfO!Ysr2GrK+ zBt{@Bl0Luoj#`Wi>GLFD38K3KX0!tYC@MYlfqGL^#L=GWx$#R(W}GQ2irQ^JrSHM+4(-(v*4zbYW<*0$>i0?v{f*EX zD&Ukm9F2+ctM47Bs=aA&Pyt(jh>NCR+ad#aq%79f!i(FhlGR5Pc$S1GM?W{4W?58! znZxybJh?`qsvKveBaN!XQ;JvjuzIU~j@8Z58Es;RZEaUVCO){^RzA@|b>yYJRI8;( z8{cpaf)37P$vYaR7W}SxoM{E0_zZDIiuH%$NmWFgeB{HOeZ+qp;%0^Uj)|MrqMhJh zodTQ%Ao6RhiyQcB0Q&ar6{gsc97~tD`_~&MT~HE3&Uoc-YO5oA&-J3N7xsfS^XD*` z2klV(sJv0?7Tl)90GgM__Z;guAWJw_{b-yPp=go4+GRqD|+97);l`*C`%u zOofv7Bo!5?XN(Y392anh-Ct)nlwdu#hE@BZE~lz3i50<)xiL(~Fablrx&;B3XwFI5 zhWFfQx^?1e5w>U7pTtSSkWeNrW8PocVfW45npx?uH-RI_;q%UE?7EV)O()_-6QYxO z>3V5@RC>GC#kEC?tW2^?g3ujN^f@kBoHnuAg_`fC>W3jpKGf`#dTPSsJZ$+%gRGu* zTjSp-;a09Qi6ujQJ078S;K)ZaglCuWY7ZzMkMLc}GratMr&Z2&y9#*%@oF6>fQYcS zTmVt_tLMjvik%V%agpK9$4t52qmaYaRE9Q6(~iiDmxdx?HC%`NvErN;hX>Ay685-9 z>!Z+eL$kxsp2P%;T>;bB2=st*DzK?&nmIFs)WVPTg}_iygx2qw=|lM^*}eI~ z6lUm`_05RDo z=QD2GEuA%&#=L|U*=-pE(Dd-nLm29ax^iP+@g!#~(!F8k9C3WPI3MwA!<{D2WF^1d zT1Z9IHsIuU|5s3(yxc5E0KV|)Encp9iq1ulB3$lhNKUtFETy2u560YT$q02RuNjY# zGZkX3INsCuB30#*Dg8zbeFk+4rL4UEJ~Dq8kI_+rWV|!=tVRK>B&!kc1plW;c2wE^ zo6cEZ&e(;J@3G!#$GL@};gAR!RQxqI1UL}JYn3Yq@d)VGH|FLV|FAyc21x>h3VA*E z>wk17&#*C_^R?Sy;+QL-BUf#Z5&Vas;{)Vutqju#LbuyiK2J=Rvbn9cPC~zms8HvO zN@^Xy4q-=^+8CFjKBxI-Sq^2rDmO49LC(oP&Hp<74G&VY6YZxI5Ro3Z!MOLQgJIBV zrSVg1xvNZM^x}EN(H%_XY;p%Qkg2TQg|{zU=V7 zHU0t{9MD4W$$IpI;?rq4w2Bor1&TkHcaknNQlZil#7?p9`hR*(v?*A`Og;c3(Bja} z!9m-ZT>Jpp?9AWQ5=LbXXb8|6rw70y6-*S<-{BlJwe`-2NfcpPs?Cx+Tor7XX<^-? z*Jj$E&uc@Wo`txVgS*p_u$NG1!I)xKrDMMXzvY-)QFx1N*kII5RZe`RvfyWwk6XG| z`S$E7!9A|$Fn_VNieTrZ#0D|LdjF2h+TPSXs-w*r!_8`Av)5Cnfd{Obm|%E$(e|tJ zZHEPJpBSBU!eh#5R@KzxlSsl_H!j~Ltt5P{W6-7(anJh9Fk$o*QGC%-Tb^fG{Sxx+ zBizes?lUPE9z{?+{>l<{^-M*h6rnUQwsUN?9>iEbnZWc4g4x7YkuOOr^3p4{yC6&0 z#H`tC_R;d$&2-ElWwXfM03L8J^L+2{Hj`0!iOTQd$!N*zgD)z0j!gWS^)cdU2OB`- z>(?wKYcH`?3Ycx$9LbF(!&A%VKUQ(g`Su@UpljIV4{zhQT<}@;4d_7qnAd=Xc`VQX zI!FzrJ7R8=ruK9J8Swm}gMhyZz(|^JlF(4W*s^jmRcul0Q*ZQ3U^QPZb7a(IG>>hr zsz#J5a!q}cqK`rfwA$J+_W)KcdCYzB_yg{U>~eRm)}cp%MV*Gfkm<1CoJJ$Iz)nS7 z%Sn)=Jqn2Spy=ZW#yl|k0}Vm-jPV34`YL=ikN{BW->wDnn0#w+xHKgI)&8}-4nuI^ zJ?2Q$I{fC8M)W0k8+pBGmeNr!P0XLCjkKQQkVpTLd8)!ZP`}*4ph@cV;9drhdaC!^ zwD<{Z)EeAaJFDDm0jnDi1_5Mrcf{%|yH?c8=T{G>FfLO(hLXIC8}b^#=s-Ne^gj#% z9Chfleg5IYgftCwm4J%$hX|^m+i3@*_X#dKRL|ea5i!UBIRR-~Bq(i#4=nSvDhmeN zXZ^efJ}r1So>OWVk`q+WgD{9rv-iOlw1YJk3i z9v9(l0S~d*6FVJN^e%m_C!imcP@bb6v#K)kW7zgTcv(iz49Y;79#xA|-Z+;`?Vw=+ zBrqz{Ar65U4vtjg+eea4Aghtk`b5|DO500z#|C7yRB0^!9Ug zvYUM=9lk|h?mhP2b!KfBbTE)7=a8P;p5Z-O6{&ma)!;(2^0eE=m{%$!L$yG0W^}qY zz+sD*SPmD1lEhdB3{URxytdmRYJFdumPI3IC)w#0u}7MtmQ=MpL&Vx8P<-_T!jVcc z^69ZDi^$&Qji-5rbLzywV;mteE^PX#P@h}}U2?A^f1BN0M5;KS>U;r(!ch%!;Z;B?Y z*?hqSkBO!p64E&1Kw(wkaKNJy|70{hwIBk*tE0JfBE?Bu;WWp9j=iLm!P$7|Vmudp z2uT671FF$ojMG5a)6LMi?;AWP!yb+(aku%a9M$yLB7G-K3V+y}P{sqY z!Tk@Bw9?|tmY*R~zGetdSgu2_j{9UIIZ}TQRQgUlaA?DmuvweasP;gH`IA$)>TiE95 zFj}I(*WaY4+s8LX)cpDV3X+yT^iJm;Djnwty9X;O*Q;@T=}SG;+RU2qNwJF9Pc&1N z8OdK68QaY~o5^lt>-h7TSrDU^8s}wA7N00RO0oxoB5#_J4;`xX&fdIjZEfnYhy6pCO=tPq z#TucF=NvEWvtceN)LOQ~pIhbKPyg3}MpO!)#u&!Lh{?Ki#jsXKk+6d1Hp!VG@z1Mc zjK8Gy+VB33WY!_fdcdnPhAzpBZ9j@>``AA=49+| z1clgd&pD_7X8Negak<_>u6m3`ofwqa#q5Prz|}?dnBkRFYpxEopX9ZNKP+W7S4YXV z|FdJ`j7po**O3+HmKtZne>f6FGUAq0&?0@MqQr(gvQtp7@)9e)ZjMoX;gjhUFgWy0 zvjE9S{YvMKYA{;3{6}WfR|e)r!X*~v?CR-;oXscfgl=>X?)*iRX3HcZ)QO)2=~ZRk z0*w}MUr_>fO%#~je+gjCuc`P)+;HH`lI!m`jidFCsM0?74qOq$sLzG5_&?yhD@e(r zi1!9N7z`4aY2D-kUmd3#N}qD$oI8o%?Y5^V=_n2AMMxB-ilO&mEXM$cCzyvU@;l zpA~#SK9tcPrv_XxtvZT}h+`gSs(M`0TFB~hJM4UHFGjw@tIHp!@k1efg-+BvnYwW> zzSy8jv~Nk86CX9nlWFPVf7_&OBMX3T+C1d%5c?K^*&=)1p;$s%k#zjZei`rJ&vYL+n4{6U{wfC` zJJr8&7;RF$k1Bzy8%pPSkk1DW0v#Rj=HavBm?|9Rwgo0Ue#I&O!_QMFm7ggW*P=x@jxB(I7-+%_*yja z1(~i^?@-Amwm+u9x{FM*m$j0Rka-J=z;aE0(ZN$!3#lff8T0;e#PquX$NAfOsoJ+3 zA%%})oaWewQ)y?H5`Bh@c3Q7g16^8DrmKG!B{1nYuxb2=kB?INA+^M!mZ3?d_ezzr zQzRn@diVZ9V{I%Mi4x-+#2no+8nkE-tr`!7{DvD$7#bTBDc;bl4HOV6zWu|w=T-I` zFxyk37r?jgJyUGC=&>-QPNcYVANx7jwuKdPU7=>@HW68kBKT7suroJ|Sf2rPGw(Y) z>=#RHy77D{*4^$szB2Dd6X$^4YCq%EL-s{D9YE)Vs`*U__oPf2t+2|awl{VU(s9Z% zM2@IiXjxobkLp`HYhj#l!3{V)mbW)@OVLLJ7H9wZh0uNQ@p#6BJ+Wz?OU+|A8xwu? z_D}6&DyvFUJf=;?9L%j_nZ?Ir+e_umsSifD0!DM z@}m=9Md>icf{K0|QWM=`Tk*ngsa<8V{GsY+T9=i+KU`jxfw{Q~HC_kg@J4remXGp# zHHl8pXbESRbeMFy(Y9QfS4Q01_rdtm@1yTN``vK!*@A?3jRtNxK1$u#Al;5^a)S$ zuFhU5{)yigF4aWc1{VyqXqO_vLZeaB+>e71 zzL#Jcx2`c~!2+HrUkPziE)Zi+m zOm%A0?_{pVhkC#QzOW;BoFAdZV-7$Ph7`uiwtF2lj!<2c5Wcxx?Fp{C?CNC)0!mF&Ri=)}YqwC~t@dlDt@O49sR_0{P%8{X)PiLIjqy4qs}xR7O}!b` z1y$RxvC-(MQME|DxAFv6(oPo6P3AGUKm9f>(m85OXPOMVEq+?JLQ0F678>s$V&x(p zNLVTV(9a?MX)dIU*SbYw^qM+x{G#gXl}G$SL=+6#d$D z9s41iLgCHMEC1QZgP&cSiZzZsiJ#j1Oof!>2f-Qill0oZhkL|;-3_mo@puRyFHqg` z9R8p(kZx#s0u(@x*B^b)6<3Po`surlZ`4uP;+IStHalIteGP?_6Y`}`zi)Wv_Q>*5 zIRxPbxe2zKtO({Bq$fUlcx5+SOs1SFpdQQmus!mUUd~TYTVcOnRtX_9bLMX5)pZ>* zJXi7S*pb-0_ETRflW4`x0B++ppIRrjco2G`HsLs~%_P~0A!jM!R|%n71}XXK6)5wTGGX@2d^#_P_jFZ zy?wGd>N>h!!6trfh~+31@O8?*?RKQ8Y9dy-_`nj7fuGJemRjeT2z- zE_Ht4ko5cXUBnbrugvT{b)`*>m}Sq_B;9mjz{CSSNQ499}ql-frMD^7R`Lz zcd)}LdPs-KIJDlSjdk$2-J!mY=<4FqnI-))uUs>iZ!I%XWger8D(EwOy|jjf-aXmRNI+udS9Ad8*m za$SAQme*^`S-01>B&rNwtSErwK0AlEf8dZDePosxgl;b+0x5ifi!RG-@QD+0%PTdw z9ZfGDX}HC0zr*FfHvgJ(!0~pF;Z2=NNVNT+kJ>^;@OQc1jjf1c5nV1l-QuP1@@ba+ z$)<}JCG#dH%&>QZ`yL7%RgbLJo>4w=qEt>3i5VZ<5HsMt1gppcQ)hficjiTZ9|*0t zE)aazYRbiq$yPR=<9ziZ+{`)4Dd4P-eIP^2RWD`YE?ADn?x%eMc*Sk`;c?t=MUHI3 zt*w`%^ksq zxk34qWJ#a?=iXmL_-3CA(aRCiJ@h#-`xDcecM$hLap*^xuP5nAG(+CekAz{C*7-}n zHiun?W_i`rz%5~yFs~>^VDyURmdWY&=lfLLQ{#1xq;E{t`u~5pl<3w`0>VN`o~-*dPgD5sp@5HgsQKRE@kZ|h|E_Xkb0FXk<2?mmN3&rN-UDoXLmqxch^T zU$pqqEqIsOj?fsOFGLLL-rRD4*W3!d%N#^VH6(LJNL8aIDcd#C6u^;FBA=!jFO^2x zq1t@oI0Gw{3gG-Vv%0=wpQmmWJOF;brA@eqe#5s(5q^N8eP0M_lui1lFMtBQ;VPVj zB3Ru!FqvSkaaq9^^FI056o30Rr0vVN-^UUwj-K`2J02q5+dP zuj=QOxMK^++4S;*(gTbRPcB#tNw}B5dHuq2w4#JLuSx~7u20D@>D2}d1LJPK_xq^A&4BJZ;-PKa%e631&>^^x(E%B*|Z#xDn{upuDL(VM#& zN5*zniGusBW1YvB6_1@sJX{hTPqLLizba?J6)Nzms#Ne$n;AM?xfcJY z!-0ohYR)ZH$~Phcm4cFg(|qJI1y!-3iLUKrYG97Nq(V*qz~4HeO7g*Kl4;~9MgZx;brHo@k}*z zch}qZL=V2Fr&cCa$%JZT1#?wPL?(*9>@r+vR7V^O3I2ODYq0N`Fd4Hf_>bmmy#^*U zW$dW6K^&Hs6<+4CWcd$)gy$vzAA+~1Q|2B72L{M zx4*cKm2gWWldy^D@9WDuXKeKzy`6OQU?upv>a>3+FP8}i=d=PZ#}DTpnaY{BuQl|Z zXn=3x6@)$|Sa!AE^xa*+`-y6Hh}$?*VrB{&{XGZu2uJt7{(X7ZLaW$UZ=%A!;gj3R z>B*pJXTsBHnz-j{2_(P_M5;SZz5dgWQFz?+6JatYU$_JtoNMQegmpZF&|oRZJFuCh z1(Z17W^%KE|0V%21Bmsyr&E+`%96?{-}o(I+A>qG|J!I(oLA`bgx=ki@`|d`m8t*T zyzznNO`#_GUJnBgTo~KdKc!G)kdVDhdid^s+&wK$4VHiVPtv;|NkDj&Tm0%t(9Wzm zG2u+5)3^^x?HuuE*B3E$i}Q8)UbhDWiH5p%<`fiNpSZkS;)Tp{M{*5y^d2f@8V4-Y zPd;+rkAACt{+KNo9bXMbWmdX5xN>cu7|mbk*12%sW7jt5A!S4%XWz9S*|fIp5=fI@ zoo9PWQUcZ-Xw!C==OK;@5`DuriBlgn3wsaEer&z^=;N8L`|Ercm^v)Dc{R8GDyee| zl(@iqjQS87WH$<0G{;~2%;CmV=q42y@jyXZ0~xO>WiIiH1b^9Pxch9-R#0;vUzCQW@ zVi**35oN$6_XMm60(hMlj$AY)?&erw8I8+mor+qiix%vh8;C5D$W_cvgkC4Y(8ELe-@9~iby}91-AL9srah+Td6_1L1r4rvXRA}vx%%Oj z<=^`Ch<1scv{t)Fw;frFF5??-#2IM{nZk1Ul8%69wbpUMJF4W8rA-Qu~n3 zH`sp5H50a6eEfJ5T-l38AHN>B%hy1`)~1QNQH$&NMe9pVPk{$;8}t&Nh$tAWYM0T} z{3=8U31pD;ua6fHx!W6rRtFE98fGU}R>RvWb?cw>cON^O)#tq04gD4b6Y^E9v-C%y!mRPm%AJhl;004X$VwfOQZ=UAkN+-kN5D;I6|f@ggYg zujKMs3C&0`xX~j3H@))w)wsr?TbGxSIVp>ao+AgSf$4%?L=gHI`%eGIy4ku;MB8L| zc-$MDyHTQu?CPJ1C!&a*ohnL8F^qp=yiB(`282y#V~~1h?MA+|D~ru4?b-z!RN*FvyHk)C7j7bRJt1ug9-G zGs%B{XA~#G zhdtIY3&8EMtWLTm!HHp$aBY|r&?0#>5oxy)us#*f_(;9$msmo4)Zu|*-`nQZFH;-s zz9Znm0`G8X=;uD7HJ$o#>yh2=HCkzMxhQw&@2Pq%eJvbLx!urkCk6)H9akUP)?rsm z)v4i0@#1U26F1|fYM=$$+&sBDNd*Nh7r^aa1_`fi;~6;E2PWX@8d)z)=;c=XQV76O z>)Qu+%hB?_&i4gy)J?#CZ^ubJ!Ao9};uJ4a8e0{8+cM0%vH|a^_)CWOUjz@F=6Y33 ze+R`$eCK%+)Iy?gYfVW^CjbQRl0HEIbEgpDBaqe?upe0dlkcxi7kg3BS6l*&r=)Q# z+!>12VKfqFYfSONo)0uj%q!~=xecs%<8q)lSe?Ixv%I0`lSzEFr%|%0^WdjVn})pp zuPooYNLWxP2m8%uB-pPEDQK^k*hxSUhwir~cqU8d<(NGrHX@2d{5eiC#vx`|Kxe)&&KZG&MY zk`&=~d?+mBu_csd`sHEhgGZ+Kef1is&AL+~`&AuZSB|y#ozFOJj@j2ZA;m~pf=lBx z8Ke^V^B(Y;(%(y_;ELrcy)1r%KITR|;8J_`3>R4AF{>IixR7}uCy4H#7%}@hmg9O- z>fC~8g>~XTG$Vz}nABJpBS4zImX8ZmyvMD9o1|U0*qg z!pXUQu{#sz^fMrwD;aWBQ{Fvc?h2P>k|PRyfD;mGtrLufY{@vQDL&`S=7lx}0p)s_ z_!9joSl`uF0a@D?9;W<%85Nw=fo0DaT zw{$+l*bt=WJ%95L0vnH8pF~|hjhb4yg>UiRf7){#sA{d;n~a?tE)56N)4F`m>G!?T z8J$k%{Y%*d79{-Vh7RC&Su@5dU-{FLFnu7tn>HHLS*tWY;d%ZH#&ehmbNgO{k2;d5 z?hy8NIYwGzU}5ClFj2;N`vnQeZTqPT_B8rbS+LwshNaL`t7Z;3*~SYwXPm?XYJPA@ z(>g0q(2in2nVtQ=JN+D z%?6ggdxV_H z6`7%XrMSd)AcZ^)-gIpHoxUDvj2uu&WC zJoplybaN#*sHg|R$QiBJxj&&W6pn9Ai^x^ZMAw0<&Gb%Fgwt-a#j+uBtq^*EXXwUBlf=SYaE}6?P;q)a%_`*gipP%p?w6%*$`Ga(A4h1)SfC zjS&95wI>-BO#QT=Rp)`>eDh_XN#kw3z2UpLeO*a{)@_Xejti4{lik?k4u<6Uoe}|{ zfAx(m`N`$D*{7mbr_8RG`iV^s5K5KXl*j{9(x-zMFKZooy>&}{$!JBKCh~3*pr*!W zM5L~y=CJFEJ?|}^oey31lr@2}0V``qDG%GY*Kkd0GXi6Ntc&}ZD3U9DKbW}Zm6vq&efP+H)#z)F5^ zi)py2S#@lKkyBWPB2|H`Tjm2bxaa;lFb5pUCngVlumoQf&TLIKkWN)uVV{^N*qcVGwR57J>$l?$qJ9Cvdg`JyRQKA$#`J*GiGKWlF!ne zCT>bixHYcSQ=nVRQ3$D4R2B_C_g%X-ayi73{EGsYm|m;ed$L|#e4^H`U}DC}LvJL@ zNA>Zx&2th=&*9}W)Ciglr=xx5y*)wuIU?p4-|h@YF~U6T0xgI(!`iGOH>-y!03LkP zXw6UZOjP$^_{Ez}F3UD@C&Q*fb%Mr1J<>XL4dy9q3j2q$U&>gI| zbLFN4Cn=tXH9PlznEH(7zgY0DrZV zpcT*H$%xg8P&DhgcQdJ~*fRFS{^G@E+tF#TL{9}K6*vi;-wRO3hQ>D!#sbn#fM}%ks6H<2^T5iB76ef^Ql9SDFff)%z8iIweJM@`CCoKABA^6R^3Hg6K2c9vaAlpojK{9wfOp_}C+mli zhK5=Rf6xXcRD;XyZJPO+;8|VR6CI2s0HDoC5EPuV89qtIWSr!^K zB&u?Qy%Pq6(eK{?#B~Dim^lapISypslU&U#nm#(n>3B=8Gfve%Ac*(gkC!QsTx_eh zCZh)LfqF03iE78Rc-}~3hLc4W2@>3UuMhWX{&Y0o#upqu&OMcZ!;ce z11wnjwWiMWdM_jRE#gKWcB8@1H-9wP^sBf^SVWgBZs{eCe!9Hij5IX`sDs!AeW+hN zHwbJyPU(XO|4g|V=i@>5OIMzL_X;yDy)uG&i_n4KhsxKCRQ^B8-aDS_zY8BnWM`9+ z8OdIuY$YPmuva!AB-wjrr)-f#!`^$(?3BF;WxNsD<9A-F`+MKtKfaI0??0bUiudz% zUgvqvbzSE=nV5aQ`tsx>>(7|k%tlJlJ8&wk@rRNFCUHio)H(_Vh)}3)FGWG3$6Xk0 zFaw@);&rr>6~*J>+Wu1H{I9c%#v25Wx4w_5J@*q{4x-D;r^4njsKfysX`sck_2&od z7m6B_Bt6io_==&?Mjs`(S<|e8F-eKCn&BDN(8E#^xu8#eqqd|1RT8U}NAvAidzO`7 zgrox>S0|z5hC2PrsQH41cT^PpqCM9W0G4ue?AmhpM|E8;`9pQ_&K%cN?M!bT%9#vu zlCdgcf_tF1Zr8Od+9O0nb?y3!vSY_v)vIfVB~59lNA}*~EA0g; z0V&?Ct1D9E92Z-9!_O?iVD3gz_0R=<7j%7`->$DBq zDBi%Zh3<;K5znEBpU{WrE$`-2nn5{)57qOco{-HbIqYvHHUBaIdjj6L@BJDNvB6{O zBP1mJ9r7NH6y;XPqH7hZ-flZl+0gr(CH~?1@ez*m-YnL;J1+x0Gm?Pm@Tzx8arAf;+#R(l3lvomS#}0L7-uK-aEPkVs76k9 zZ(qKjVK#{_ATTFH@$_n(@a%YhFh8cJwpXGfN|d*r@-8lWD92q@GbEp*{rRTz*Xxu16vu^G)1+NB zy!yHigUuR`K4|(MI9ommuE4JiNBP zN^@kVw>}CQ9ak!6rI8k`q;`>n+oGTokj(CJ1i)<`b$@RTkzLi@XfC` zwLaM}s~II^dKxz%>+rFk)o?7g+irIERvZ^-+dMtD9`+C>w3-j?T&@Fly0kk)mMjE!iivEVc`HvmBFgA=?4xvL1ZI z5Dol-{CzT!HfQiU?cN|4!bk8s?+P5}>*Xi@HSZ|M$uWuO#i#jXWn~}sBwrP$r<(Bu zF2xXJeayq_>vr{3F6M^IgE`2i_X0fauVAi|2OQR`Mi0xh0P1iK{c2H6OiVP_s))Ev28&gK+Lz*Y^opG{`AGETBvtbUO%|;^O=8R_+ zWp_f!F@=$Y5SnUG%|l}HS5(M^FM;GCdRfBzO_Lzz5iFh2uQ0!1@yw0O{9- z_oa^O+^MvA`I+M#CCkRKc=a(%J7Y}sT6BFC#7fLQw;X2cJ~C9GY0K;o!rjS`?k3rx zn5oSw?$1!c*2oRYQa_nrOjj-Gk29jyzA;tpaYpo4yuW&%6d?}|uf8TPzJoyM;iaBK zj?%VZdJmrDI-E+ciTiZ#<+2v|JO;`6xx)`(QJHGKG)Y2?D+o>c`CqqO*f8L;W!m~N zs${JVv_+>#1)jiQh|Hr>2!Q#H*Jvx=rG6s-KXWA^g(0Kuq zjrPR6m(!IsUc%{`kmY2v=}av>$gVP|?%M(~u{#l%9Yz3nadfFyq+Fwi@z9XMjAU)! z04#9VEVWKpVfuo%3cEn`N)tb^jrO)yJAk2k6#k&eYuYG42+*UHMKN)Al*`$LSRqc` zQ@mkcW{!5rZ0s_LzUx8^ZNN6WY#T9zD0thUb1&ed=X`r_NN<#kRU91I=*C2Fo(~OH z*05gOd!J~>&^zS1gX&(c?G=y3pQ-mbGd|Xjzwr7d^?H~WXf(T|W$zXIfZxTNS9;Q{ z)!g51f5pY~98a>=$!70Vl%&n01=9_RJW@J30=I3Hd5Yyd9Ay*pVOWiBGPM^gn!1Vch{1_UH}h2z%J=>4 z03L(%s)x&J0V|EnY)qZTS29kk8wIFtB=7NS1gkW{+Ekhb)ax{N^gh>vk^GlHdQU%< zVLKAV3M8Oo8?3NjG=HBh)FPHFIu{iggf4b~9;b; z-`_|%V{^Upt_^3m1fa%n+20l4)$Jf2S`?WB>4|mYQAyU2eIEX9b@@qC zUNYH9@||Qp>cDd}5pv#M*!ndtFWju{24+vg@$LUB93`o1`G!kvbTq4-3sz+GDArQV zDRns?d_em_atDVpG$3?4BL&l6fJG7raZJkX9 z+;AFU(hwIhE9C=?3pZA9U$^Z-Ldf2hUJ%M%LmoFE;%UQ4d3~Qemy?ioft-${kJ%-qB!{YMOky5PzaVYBv+w1RM?aZ2E4T=k8Gp7 z^IDt`?4pRXE@Mr)Zc)bz7g@~LDVB8Pn;mi3yO*>ZcpYJB2`-T5geBZ9PrZ6w-FbhD zI`!B*)e8^Xb24k2y|6lD{!@K{LFA@*vGd7vZ4T2IatHt?AYOaGYY zUCK#&==0WY{uf2cRJEyKJii-wszFg~%rJ`yH?}fl!n(0tBkEm0|AVuQ+nZ@X4Xt3% za*wyYzrNuKcqy0W!=ow<=h@-rgGX}i7kH=5_uko)pyu!cENtkn19dfASCYC}Uc{hyS5muf zs>0!Y_VdDXu2|*IR%zZT7#2D6+ITO1bQwt~Chj{k2gni{_k31BoYx8d<(HSR+`ph* zk4&j~I9|S{)y0ul_w*<8ccOE%rn|<+jdFd3a!P6Oi<|+wpxv~uplk3b2YOWB3n=?O z*wY<1mRQ0J(uYa`Azq}YKqFay1qn@pRbka52EOocoVS?#D~Lu{MScS$GXLg|s0 z0j!53MHtE*>id={UgVvy#OjAow1;U1ZkHVd66X(*Hoe0rgu3_{ra_Z*bu$PX}R!k)xb` z^vN93M+?EdNkkMxKrPVzp;fLNG#ZaK4iurfoB_c{q2vIm1tYWlY_ri_P=A#VY4H2W zy=pvytx^r}gEQ1jHefI=$gPV^z~Os4x<;<1K^N68f0aZ1ZMG$;_Jj$HGFu%&yKh-j zXzJNFnkWZf{sh^&Li7*gdIw4Z@MZBIDO!~69xb_%PShc2O^R}O@c0N4q)zz<0eHbq z!Oc8Gf330cbz1)-k{27_r+FkH(d%|(v3>dt%*1uL=efs+M)mSx*~!hJW}C55%r;?- zhSH<+^pxjPDf6|sWStrazv%JdANDgZEHt7puHThWN4*Ju@PjMdSOX~_UxZZn^5X6= zNQ4IqR)@~Sy#bbZZd~6&)aVtj{f#G)FWaskoLlvux!^gNt539D5a+=6 zgLOQT(`#Ok5Qv8T?X|geEq~wVWNb}+GED&&sRq;Q^>F|lJn`{JJ9%RpFsmt5_j3?` zp;xbcxdz@PyTt~=T_C?^+Gh;jXwNT+OWl2Y!|veN5p`3aV=J}lA*Ilx{-~kPhI^Rw zlFsdO1GU+BQaO4Ja~-GTTCB28s6NB1`JR6BCKaESXgPIOp58oN{Gt_JcVYLNOzd*| zKYkj`hp4@cpUOSnc&S?Zlr`<@n>c@4?2erJ%c*)<9v@aTlrkqtQOg@u%DQxCSfDy2 zE=?susza(_h(mY9!!jjU!DtAKRAG>M$WHHTF>DHs2gGuAn$!V~U=ZdHLX6H)R~ER4 zZB=)U;{lsKrs*RFu;D;5R z#G6?su)2c0sBdl=R==HgFh>BVHNbZ;W~JYr#>f4kg@z#RXubObvO))UC_nojWQ3l|}j2x63Q z|M0v2r%UnKWgJh1Md{Ng=Q-3qVNQAMyuLjA^-CVoS;UcwKC!&<3#Cc_v7-MJ$^<4V069W9>q*#n=mBWh!lpx z#rCtGqc5F#%6aldUfq`Dy-MmlFENe>q%o)Yy%;OQ*dx<|_JD)kaV&iD0M59a!5y$k zz!585^wT!Mkru`3(bIKoxLMzjV+vTv2wc3_Y!N0z zkI3VU%X~!OpL@5v76)e6jaxsT@Jfc$T=9j@^}42J{A&v-fMo2Z>(Sd|IO7WP>B4Rz z^Eij%NnlD=TfgND3Bx(So21cWYwJ8T4hX*8tNH1{)jNy!wSJNUw9CY+{9ZXW&ssos$V|IfRBn&$d7+-B^V z+9_UUiHFG`ijcZ9{DFi7jd4bD730rU&xld01l36`J<8Vu_Ce(90OZn`Ej%@QD=F(? z8abnY0A4g(Y!!Y=0LnLEY}RSrjAen1dEvTs``1zi*(+8sV0B8K&|dxu#l_tuHc|us z14PG5DyJAat`v6}%VxBM^lR=-$6_X3IVmG|bi=f{mpk*x`b9+cd&T&uSwi^d5XZn6w6Qd}!?tP`#<`s7lSIh?- z1{9uRu3?da3kNUyGOWz$)by)9|vtQ}_B1OY-weo!j^?Ca1On zt`=phOY`O3iwWVgZYIX*iPQMNe2^?`dX3=!;S+V`kgw*3L#&igyB@t~?NQO(}JOo0MY0@ti7K24kVmyOU3 zdB5A`2GKQr`A$>l>Qk=L+|@(GBqZn_uD|^WX$KM2*(0y`=e)9SG`#Pow*`k8MO5b4 zXv)T#Rs6v{*}uk9QOz8K0v+j>hUpZG;>fg1o}6SyiwypR8paVrX@c~$!US!u4_x4m z-imtrm`Gl27Wi_p8xcdA=~5(;{k!*vaL%)i4|VG0^H$n=K9#1t8X5|bk_NTzxagJwq4#0J9Y2hYtSYMwLngK9_p%#t zS9=`9E?$s07Q~$IZkkg}JNdcUX87=g2?n>$>8G&S(Ci*9u+qmJ2b-03YYEQ#U-HuC zu20kSno3hez17!Np^4Mp0UHQ7m5I~1J@IVz*u5yvSCwsE_B&cN)7o;l&^Y}fc0Ise z`Uk2inY$RcwF=DsIbtH{AwOORfIU;g=Q;HPPu){8z(Lr&rUNias*&9xD0IU09Eu>> zeRSJi!6mzN=P3>&ShsjLmR7qA7&JQ!t1el+bsdcoX+M@=B8xPREh0-Esh}ci+2PWc*ej!0F3WrsWSOkq9`jefxG~ z@3mE;OCOE5|+ejVyS-p^nxr92zP;A z3d#ps=>NRCD}qX{IxMW0Yk2|z#>FYK`L{%5CKiJ@YU>Gpw0j8&sKv_u`_$49P-mqx znf_!#ZzJn-=@miAqPmzCKm1S=so!+WnBW?nC3IKtk2B$9j@2kkX~$21wW@RdFO^n7So?qF`&&ir<qKz;7>fp<^K!cV66W z=jd!8L76`3m|Dc=DD9&*E(ovTxYKr;cf74)FlPX5du1~8E`Qf+DKeTXHWxGbXG1_m z__Khhz@R2x9|l42ywvG}rVV?lsr$;cjF~>}C;UYQcpbLgdZ086W2kPm< zOlwgtpfoGKNw>-)RVLr7FzkY+V1@CW!VBNf?ald4bB%>n0$`Keuq@pF++6Ea7{^eJ ziMt4F{rGZSm(=~%8!xY4Y3Al_6x<(3&u^;3-a{EY!zeSF!izu$GH>faj z;c0H$pwk*}6JS~gYSqtbE4nzzg%@sZ4#B7PdLJ6ISL=wJ;y*Cduzl_+*Mi)|{cdrz zU-VffD5szW@ZD<)5R-q{o%iy>9U-~d;&YuJ{&mIP3Ao}edwnxoiLZX(R|t|n)x3js zqdpE%Sj1^WE}+za%-P}aq)GnUMSII*f=*@mC(kgI9u_`68wF~NhsXckVX^*|@w&Cj zs_B$2z4ae=(`RaSzGM`Z)ch}0D4XE9_}a;cV%bPc5DVm_*>$uvJaUq}=ct-;D7(lm zDxiV1+F2-ojWDUHlzXg2GP66GfLo`G_^bW%A79L4LJaD^u&77Uh`zsGw-SVry5J;Q z1O5mNJH|PK75UNB0S5yd8bK^UUL>hA?@Kzt4Imwp>N}m@=@$OQuNL98{!27Eceptig{v{Q?Fpb z>`cU6t#Y?T`tP5qxfTcy9^-7S*i;Eo;Ttgr5gid2sDHfNId~5|#Gx|$fWI@8H-VO; zQ%t5cjZddxtM3mx-ZZ7Y=C$RH#dDo8**2ZF7!~pi0)JLaD z{qu|tkvi~*-?+8j8~u!sOs1Lr3A|9sR%EH$vwQk^|f3|uKOXvs7U*`T*ij~sb@N>vx z7suuC7KjP}k2%2YWYYI#qrDz6jkgsDWc1>J-9%w6f7-TD+oZ(7n>Π{JhDJg?Ozl**_2 z-*@0Z56qjVU8+kxT4Qi>&S%uCTYa@ZU)@RBL;Wt|z!8N;AlzfqnC>B!#btQ?#w z@egM{0_5&95yaiOTO9c|7aEpNvLEihKX6HLWeuFI5H ztOf~Sr8iql0%YrR( z()g2pf!jgn3<5sr=j+vZo{8ns#psO9^{=l-6J6ibFUTSOQmJz2aQ~C0mZ;NqH6P|1 zXRnPytSCN1S^l-d3VhYNJ1pf4_6G1mL(UQ9%}{&dj%(DtuNK8r|4b{aw?AK>EWz%m zs8?!!aC_s47ELiR%GgVd(*w8e(Aw7Kj&kq`aEsEdPbtOH>RR6Nh_@Z{zv9-C*=CkS z;FR97nuO42TXsf-AZGWWM34|RfA_<0f3u6|Vh}+xmIkPIg1~s-;M+^fp`HoDjm8X7 z0;2hjIMWoy1=)@VmyMsqxA_=fdgC#cH@#o73}SV05|pu9j#rss>dmvHDB4?Nb#0&` zE?BmLSt8IqB9iuVat9bOz$`QXrEB8#9^Z6w|Kdz-S^L-WKakY4EfP6ZO$1=aA3{!& z1Xmdhz9q8*47t#oN(5VBsox4GDx1_iF&#;jZc3O+IctKuu-ud+9I zXCyaE78_WcrJ@d5k2L0E z9Rr+x(S^(ZqzDisn|UPNQGdd^81cYZO}H1iz=ZrkHdc3Y8zQW4qYTSb`qPBmke2;d zEJ3-ADd7ym3XhrJs!d?6nZ>f^s0W+;iJzobr=`&z2Q-Osxo@w|ZYckkT?NKJyNYZV z^E1_lRsY;7LgdjaPTeZ5I|azrjE0*y+#o{7!l&@ZWMK6Ub|dj_^mw|LLtsXMN-r;$ z9pNaP!e`f&N0%zBbDekM8J4$l=+6g;mi_Lha-q<+5>iy;g)8T7M$%nco-(rMx|pY4 zagFZkVQ9AJaw)fA$JVVvaZ97$pRV-PD$83^QmIjK>yxKQxV7$#1Z)~U~3zxnp@)y5c!`+37@x&&Ga{F#Pum508u(XxN4U|#t=_T%j3|KsAizwSMrPpA07&qNcV|0!XFP-=7_XMMvZtI3 zW#7F3LcL}&4=5{UcC3aFPNV-*_3L++^EglKNK_MA#o^t@+5~B2l2^Dz;Gf^@jRcg_ zR=uxWg^1vbBy(;HHc4bUP_p!)klw#Lj1=Q0&d%>=R4YokzN|m4kfG)R{A;SFsi7~s zB$@oTvMD*0cs+Y_`=wo~@s)wdUAA%gT_E}cWR8OB3t2lSwDCbUQ?ow~2-Q=IA!-D6 zf4_O-t{aGu9`5y1RHtY`2DwUJU1mFmj+zL!+rO%7IGz^tzzCQ=hTiLSuV^Dt|Ax@2j%)B_3xICdG_A`?+$p5a3*! z=`qPOEdn`^DGL7AP6AhkH%l`lpYy9f2)>&@BC_kgK0g4AeuCL! z``0k(G^ZFb`UgAp=0)lk-aPtl31O&!Bg2Vf$k4G7MK~IPDcRp{e;pkOkckFjD$sa* z=g{n677jgl@#5~do?UC0uv3Gqn)u2&!SA}Q1$Krqc|xR78a)a z``O4MYr2)#X{q45$kun*-)H1Z}jecmMHT(}%r7R{&Dcry2P<5qNMGe`rjT7BYqSmj@M>XktxYqZPTwmFb zF|#dy%5%SbncfP4zfLjT)sp zH3O?D3!MBwh)W86qldVLCmDtD z7QrPc2LS-~&M7o{Tr{_aNGqte6#A2P_^{EvNKm|*`Dm(RK50fRA$;z;b`EHnTFwZy zSrd|acE=Ry$>s;rEB*wcwELxLSXOB8go10r*0eJg_Tmuz%8t%ZY|v->^(xSDd2plx zzg3Bf;E#Audqj443Uy5aP7F8~FU^?dtFI*ImZnI_jK z6(4o0FRnR4Lbj81l)U;~2AZ)8w1}wtMX@E<-&Wlm1A)lY4yUIdlDGf}{&4n&qpgn- ztDFg3Nb9R*Q?dYwylqb80u>r)D{%Xia*E!Nqtd-U!Qg*S;`-VHIKS_XplLAKk>E(g z-CHt~XPlysu`4D0T+;jE%~?b#}+wS6D# zS)2g{ZF$7sHa95rzcQcMC$T_cLF6E|InKL>u?2qp7B?7E$V9!4>9%$=-zymI7h6V& zIX&iACV@}1bt#HRi&A0Nkn{+<-vyI=iHKG>EYf$9SEJSWUcZax(&TrfH~WS4_GiL> zOa+*y&-qM69Ag|EY>^XX^<9|v%}{rGiQ1aOcNEFiA-wNdTycX}J6kktuc4;tu7UHN zL>3ae0B`Vu-?-}0qL%+u8I%lpdPQdwuh?)2%Q!WRBvegj$DVg;;H9g;e@s5dGwiCy zRFxl$j6dQ0>FG2eA@cG(2UY91NcX+I8-E}!>FW^iL&Rp{Yaw8P^Vs`(&^eg|0aFt{ zdgw<~Z)9qSdLD{C$knBW;)m_ria80+ervUI?|>TG?)cCpI5>FPchlP&4f^dj zujA6nf76$#+oqrReJF!O--Vm0m~W=o*_SngwarDuAnuJ##83|fFOzbj(4m{|LRSJ4 zAluapRGHAG3CAkau9Tv9ThM|eXw^}9&U$O@_bc@VnjDFH0pslNPa`}I||W4 zA3JRdW-A9c(~PSI&(AYypv3LI?^-hb)JUCc3-h&4mk;iJ07K%MC68amXX20qt6(0= zl}B2cJ=19D@K6iX8khRsjAXDp>%7r5^d5#5` z3Kbc0?X&*-uxOFrF_7gq2GrojPUzYlk4B3>TKB;!Z)tFsHFeQu{dVKv>M(wqukCg3_J^vgGQ(hp7an63OMMz*d%&&CFDbZ?TQ?xHD=Fi z3Bq|L$`aFZYR(7m*tb@ta5~gJe7~neih{B1jBweQ!w7FcJ5wt0g^OU;Z{ef*D*mwL zBFC`$GOQcpbBb_7`O*qM!d$D{(GN3|S|Dv6%y=`u)4tyad$y*3d`Yh!f@WG43h8$& z@d6RDX_b!4zbfWGzidn6w7h?3GT;`JUswtc0{*l)?h$Yics%}mTg(uexH2KYAyk3s z^Yr1pPy#z=^X=zu_Gx2As+-x17k6*~$>QMf#dCk_XF#^5RC+=qLCy-VPU&s+G9W$q zTz;8orH}w^Oe>Snx-N_kM{J7)U^;`&(#Q^SV-8<&BctdJs~>KCK(>gF{q2!yE&KS| zSQ+mjQ2U2zAPyk{IYuEmp9~B)vuR=`VvwxcV<=i3G|oj0c=%v?05vjw4l;4obVlLzSeK%sj&K1*7Z44IlgE8IF{7YS)TK4`FeIgSjOekB=KhW!{{Xv-L6gJ zlAVCHkr`RuIpy$wpvN`%HL@JibaxPR)59#cI5;H47nxEiGVh%}K02^l`gU2f*8Qq~ zP*9ezypntPaA+$Q9=Z2}kN0GtX<=k!M4R;3(Hq#$HJJ^aVjpk2`VhPgd=bn8WF>l* zJXxFDt2!2w)mO8%OBA~pgssmLT)6h&HZ27c7|6V{q_ZW3baV6ruo$V*Z$)d!Xlj8K zo5Wh{3uCcO^`4$*xi;JdDUZh`-rpBtMurfXk=fG^S?xX#nYvXi z?J5cS6w`G3Xze))kn8lUV^*o)i zc{o`j);>;?F7g7t;^2owit2R2%$UVT_W~PPkE3;?Y2{blhf$Z}&OiDnTk+2epb#H# zdm22cn96b)?rV+QMBo(`c~kHtTNu*=k1BU7SwZGgN(H?aIbMgrZ+=L@@_PigEs_D7 ze9n!u&&YGK+C}0gM#KfNvoV|+fgp-%f|SkZxb)2)MPzUjrh{>_^{Ob(o}&U7m(5 zoz@r?&Ho8P4kQQ_IXnkgAPBW>U9}03AiN&`je;bKShcF!g0`RJ(}RP1_d01jwpT8T z2M2p+Y2?;}V^sspiyLb!CQTkagtfrW_GC{ji+ESC_QMR2mf-d82$)?^fS2fODi`Wy zi${|%E#ZwnJx?O?kj>cVJmevvnB&9TeWc0`n512Yfet1Yu1PjMCK77u&jx58)N>L! z2OK|Mhmq#|d+*s;q|K-HxuGffZLB15YN%gJ<7- zlTp~}K-R6Wk~iv&3S9cS2neFV-colB`WG*&J1W^EZ~|7C(UnXXco>i3sEQVOeO(EY z*dKMgR7n;0IreaE939{^#e&h6nehxO02d(t0{nr(4ryb6e7Mu2&B-euUeT9{wd~v&&p}YB@e;S_acDi^ITj;^P%G?$ZdC^?@BnJ}&5T{Gz?>p>_?n8YcV&?NaR^{!;*iSYi>`;x;S0A9N z&b6guFh4Xqjff@o(}+M$Onx?DkW|a%wrp(MC#9A>V4pJnHWUy5y2~;}e~0>P;94=D zjrfL-yJJbTi+DvIOeTH_o47O`YCctKhYIJEF`q)M$v>K@LyFSz(b|YtnjfE2555$% zU6FMO1ZmOE5NQ8Ug6WhH_h{vhYz!XCsqbKsZruWvr#1W~2HLb;tviiJyA`OdZ!`!2 zahCWaKJy#d`mdd#ug6NIH(P(3O7t2O(mfh@v4~3n2HLy^g*cvcQk0J;s+pcz?j(St zlg02mY67}z7>EZ<)vsXz)oiH`1`NY>yyPj8F0;k^=2-YLtChdgv_c1?gtP>-Cd&2o ztu@PNDxX?l!$8`L-(?R!<9z^2bgoF!8}%B>l=wVvu(sq&V{I&}}e z;_a5Is5ozLebg>!s?OjXk}g@gVmfbM0d7B*7zDK9(t|{`?tBiDvO&Dp+d1z0;>07W z-f)BS)$PtrJOb}vp6DLUXq$-jFdFo<^A{FBzr=>g++v32PzlKUq3f0wkoF+$^dawz z<8|7#Ym!fP`M`cJO3JDBaF0nRf^H)MT_WEW$;q}kRNcalG2LfY#;AI0?63S-5qsU-%a%mSo-L>9R|{^)$IWe@Axfm6L3v2WuzD=f)#nlMF!RD1ArT(x%Sx zYbKxFv^10gq@4N{Pq5AsoJRv80mUY~#KFGI5Tl}Cf6_`2FX?eD&!s?y_dM~<=qRfR!MZzK5 zo=!4JlOe=45r}C)jpjfP>`)+pt`?1f*X)stcw$bjq~cl5cjARfbZH>IZnsmFRL(LG3vEW8y=4m`~y;A25OR5yB(4U46eo}U0*Udg} z())spe!Pz8QY3>PEB;~=K#C@g8?3%B7kOmfHa|Dr>u>hAH-6aFZrIO7e{xYp|3kx# z6B(CH+1&Be`el*4nZa;cyQ9&ye8UNCQMU#1yv65sCsST00TDJjjoUd%rgPY4U6@%z zwt>mkx_CG0hmc!wmmF<$ja#?0*6mGw>+o``5wEJ(Y>gs&RP-nZ^zfjZO zQ=%1763#tD*(950K8ATJDi~DF>n+ zmU;Z#+lz!ado?}g2-MKhQ0?rUxkidIo&f?H!G6d0CAB7Y@{Xict|PxZUkDN-%*u1S zh|Sc&M0Ed|1b^?*`Ih#AxwIv!fmO5M$8Kx#<9<_l`aW{_3k~36X6mSVyUt;asr}@q zIpV#gH*;JoQI*D2RLV;T+RdPrP7TDxF=IsB^I%ZfLbp!%O$oE*?q35rA*C?mq*<&@ z#K5RoJnZRGKjB$I23!Wu-83$PL{Z0CQLaQOs-6RZYj$-|5@ zmUGkYSP$nb!&h>CZS`A2(o99HXXfHi$`6F9&PPRNJ-f@Kd*wsShSKx+B-{}!tmQ#< zGwv5LZ5K^kvcvz#ZV+w>DJbU3{Ob+!QnK!RlKY7sC43ooM{P#+uhU+9BAknIjkx5M zt8gseo_sN(hV%I<9884S<-Wbz39p%xE(z-Eh?D#7{hPhP$c@=0S-<(X_j5?V$q>F0Rl2Eks(lQ6lZ<& z%!MRQvdFX--GP$F=Mal7bi2U73EYSV>Z~^!iRqu|C-=Lp)nErjq}V+QIBwZogteP?YAZ!LJsj>c&QaJ8k8X_= zGDVfcrBuZAv!YZe!sG9BwX_covCppJL(rJ1g7?^$j@nm#pySMr9Na0hojRAJQx+@y zUi2R)e){^76H$<&w;?ZbJYftmx(bekz02K<(Ks-vB#rZ5K;i7{1fpgepZ(mtVl#=C z*%dYs-E3rrRi_b|m!X{I)+@;uuhkb9DER)4Y>Plkh89dKk)WTv4 z0`DjVn)N-;lER^Q@sk>Hj|2rc2~$cSQhF^v#a4Nm;; zm9ZhXqicSw=;Y}s9qlv2H!asaYJ~RhojAX`lx*qXJlhHzh@n#Dzojd6{q0efDK$w=ZdiqS)(~ zVvR?C9iUB*E=br@f*&Av=Y1t)$VamKrC*Kxkt$d2wke@z<)fVV)^>22eB%}uQCOW* zwJq~ymes_?{+4KE^>BfspntI?Lcv5kj+LD-{t$bZe_A0LoE~aS81#}Na5|LNXt^cv zca|w&V)J6BiKqk3Uz$7E`#okO>JTr8Cs$*&HCh-)JC6R7_RWMF(#SFhMdnw+h-fg$ zBf!E_o0xBTq%1GhZEm!0q>NJR)jo*OpndRSor2r`h3(1Qnacs2k7nIvCiB!=$CR$U zxRGJ0Ka(Ha(G`3!kf^Iiq#COUs^|WMpap_ZjuyO2nMKl4TM^d# zwku(fw&hSNzKra)`4e<+-!jik)>PSn$)Yoiy`$=JbDB@U2UN)BOGj=_)J%k3ok1^( z&kMzscKJJD4-vNKRklpy(5<~pPOfD0qMpd!-M62sIjg z84N!sgJWQNY*NBxWj*R~Q9#{to*uB5E{n*>_uAj4b`d-#JM!z zI-C#U@2u=M%149An3x4i&4;Mg0#4IJE))xeL%@cP>~x++Z}*kUiG+VXAu;-2!x7Kn zE1*=jyL)u_0z!2W|8`^>?%%*9;d$-xl|g{{AtG9##ji192SYu{vm;l9GGnNtwI7*Q zSH26s_{iqt2S!eFK!Ho5PQ_CRf5lVer|EMWN!`8vPfq{ihO`*+m?BtzllMYYNTqO# zO5_@1e&gXa>ovgqO(Sf7El3QUOq)069<~&=5K~aFW>vve$gWu)%%!4G&N6Nb739hk zEr$@+mC!w-G)0r8W`>8mQ9eeud@*sSVPnJ!aqqM2!N1l7V*W)gb}76Ce_TDshAU`1!B+MDxs_+hHTu-}t2pAx*uu_J@h|%G7)i@D z_*-I8ke$V3`!#icV&~a}*u-@wl_1(l*OZ^R_cy+|jlTG}U-?)`prI+$+Sm~$v>w+u zBp+JtNSSA45?~ZCdE3<$re#}MJUu6nNYAWUcgz)Zua2x}AR%!pudBxYh7$)7`?Y1BpN0?aYp;Vc%dT2+b7jVyJgl$6-p zn1(NZluYQq-ShDoLFy{`$ie6{eQf+u?La$&%TbJI=r}^G@RKf+DI#A>zj=yDjEak3 zWnm3NYdrGDN}g8_lP(CN%JthBcd1IYTe+KSIa#(^Q6s*+g7!%$9AcpXzTNH`-jrM1%GxS$XvILL5`Uz1h#hF}L zA6Wch%L`Bf-jJ1K4(Msh-P z`F*l*V-ZzM!tSroe%rjsTG-Iz>yZ3xU+c+Ia2NMcZ4VX=oDQAa;jJgAfD;}&lf>(c zuJC*`gw61mlKluSVG)sNKMDR~!e#6GAUsdfX161Gds`;p^-!9xaHB^CX|BJWlq(F~&16 zY%V6ZU0eEksKBo@mT6oA?;;Y+%D9Zc2=LD8sBM0b}WpZ-m<)fIKo^S84Ed^plWGd?t_>ZE~BRy0N7fUqyZkF4g zA(W?|@f4l)ruuwweR}>R#v@`vd$ic*^=-6l8VogNVv3pLCL$>=FjNqH_4LFV7z#D! z*cD3>)FrXwySfsMN9;EVQ}J1e2W!?o0OUb8e898yRpCGs3D(IU!oobSv?tbj3Ko-X z490Adb9x9~(kn@n6`C=fzPSH&s13fQu`PP#Q!wbHWn1(6>BrO4#UlCe@#Bj)eb>*+ zo8MtK+1li>5>1Vw0I@e3W+Iba&(+2D_7^P5p)!7FKLbybXsN7r4+mZh#dlifpW(^y zUcGnL-)^5uM)nMRU-wbZREMU9i65#`F4eIJdbXp>o&BmaX@Ay`97Yg z{9J7LRUn4HooPH$hFDAXBCb3Cs$0coa-3M#rjDu`f(P^i1h_wLMpAc4Ykz-oJK# zJO1P=q5c#~LO6!XAZcYJwBvB|JK6Waq{mx{Hp3-J)!PeVDLDx*PH32hbgzzb>X=Xb z{+{!Z!4*z*X3m))6U8A@skPVGR5V=@pnsS{7CRIaCSJU zZc?mfIUC&Hz8>_3_oI5~wPuSyu`XdL>21b%-cWmCww6ly8$UMlnVp8x~?4jlUX=YzV}(iBOmsWq6FZk@IoUkJp7ksB_Ydx zmtu}hpn{`tG32PcYvYkFBz*1vW9+NrqW-#d1xKVqM5I$tK@gPgR%rzhDUlLM38gy> zN+}TuMM^{zl#&tZ|uGDSLq?Y_TGrJK6hu z;oxlv6MX^Y*EJ+eswK0Rh<$daxj|%mJWWch=Zs)@5rc^s0p_^|l3Ml~x4Lu_|Ei z+k_j@G!B;yKzHp3TZ1`hn<3tzlw)TTN5#_oe%)ViHi}$!wPFf{9SLDr=-^r*?xxM^gwW!z9 zzyp)1hkYvt{9Mj3@WqBd?#IXxl2hJ@djyLB&CeM>a_TIYfHl zi`K(Rsd4HM;q2g@?;GW7)E@`$xa@tXT4_U%%Gqi=@%( zit8~#&(8tNXJU-s@_eR6+|Sq?*ptd{?&}tL$p($9){CBFyl-m{3Eo^Dp%0N>XKXSYHkoeE9J@2L$1?r(SzsRHetc~dT?#&I5A=H?KDqsDby{wTlC4iJ zxKGp|X*$J**XU74m55%1^gqHDc@W4)YOHR36^#J~8%Br96!zis!4~k{o2lUF&*Ss8 zBt$-=seN2U8tyx({~Uv`_Y_v+;^M-%Kr6am*J!KI5;9D9nB3xrE<%?O; zZ;aSvfzzbW6?NoN#@rVxd%UvJx?DGY-f8+1ZT*K!gZA=i8_AvxMjKM!Ul@`=jADbO zP4-kKSno3^FurQPH8Ih0i{guL-`eVz{-#$6CXJ~(+h0Q4WgH(gVZWMYHqcq6%%Y0?Z(u$c31)4&y}y~p!)%YKXnxe+@p9rm{*HAili94u0;piyU-XRG{{GUP~|h}6o*41xQnw+s`;=`&Z}=JPR^B) z)4VIWWs(i)nE#7I^{ycJfqw=60(clv>4T8uP#r9*M={GU{cnP$nCglRrVfyMMUeY} zJ-(%~RV4Sx*%H^23c3bT|JRcLKcs>FqiB~S4EjAG+VTw2H|as28Lkp&i2B=x(tcvs zzO@y^olHgZVe-N!!VQ_NDFwRW{?fdnY1$`ohEC^8b~F0S^SrS_eYy^PEMGVH1*hNZ zCECU3tOTlZ8`th5v% zGrPG&mLm!mcAUf0(USf6^6j&9)|(Y^9SF_ybk2b>yht=&YBx0TF&E26G|09O4=j!h zW6FN8*?0FFk3)Lwv-|1>oe{q25H?~4GG}(AoSy3yvG^xmqebJ-M^3Whmd7tf&z7F` z_bUS{oe#aSy*CpB)U3JUZ_X7Yv3r{IwMghzh`$$N>({(BOoNPY6l?}7l`SMSS3r6n z9HU+I0cmD3xvZy*q?s`O8MQi!QBHQt{3xcL$TEwFj!@+v;BCDggcTp=cm7e|z|F`L zDfuhK@^cZ7?vhWt-JQpY%k#^ITURHAcP?p&YSayQtGwnCYgA?H@A~KqF~}qd#Kc3h$ zKC#I?W|RBAkm}Q-Z)?Q6%NjSWcn7G-sGFj zo*F+{9Z-clwHIZ9VS=!}VpU&{B~<@Sy?+#n4zYhs zkA|7>As%CiXho}Bi^so=mcjS(yD&cgw6T-*6%`0x)Kj>|bh)+TEHEzd@G_$z{)T_z#5!$ zZTJS1m#{V#?pv%J=PH-FjCox=N?ZaBqQIk>D~L4oUh9ax6F{e+0xv!#AE2NO*N{2+ zuv!y=44{#h;HHU!CtU9_n|~KONXIon*-|vV{%kcWxsR^d=P&A}iHpa-riKG69)(lq zDE+-0`tCe*oUKp|xI@U9SOxv56LALi0+g4H-7Um+R@`Wvk#>DkxMM!nO&lD*Lv6u0Mtxr^vk3AZe{m(=(Z#|I(9&bb739X8mAS;flE7 z*UtP;v?hi_V7MUcrUaBvW*>$c9)2I@0E$mw_>ak=EG@7n=XYr{`QERRBs&m9J*d7J@s5DaawNR zR)t1|^uof<)UO8bWwyqt#n!vcCV+`1V$olk@T`@jJooY-nc-k?>AJ2#9F0koKoAbBKKmv2NKc>nt5(6oMt0_mb3=ltWhXZROl zvU?d<`cM{<6tK<~%E%rmy`4q3I9JdbcS%F+n9%LJCcrG@heHTNm~YQ^s-uyw*Bx$= z!Cp|~VGF&>EzrEUHs~B0yWUr5=Uu(CFz}%f7zD>ZGu?UG56A0dkQD)R`~*SAoJf6M zC`*^lDi^a1bdQMP9x&(ueB1U}VdFmS_w;1G*?HB;Zf9l4Rd&M&(c{yjT?!O;mInbi zM-Bqsa8HQmlmPs3jNpqSPE_~#6C<;4+`S;DEJh(v_O@MB;P#L7@#>w+xmO#m_h=e@ zAt^YJ27o=DWT`ys2xhf#2U;;bi{|(t!C`&~l=PzIXQ9l3MT&lIlrXM;frRlVfvbUKXVgrzbyP$8)1N&W56koHCS7Y& zb1k4WcXKitGC+lU!jxI0{v`ZmS}meuVPDkhcqxxl`+rMQY5e+3q4BJE4Z0H{H|*m` z7>ThG^6`WiB3j(4@52{Ex`CIW_~}oS%0&O-Omb2(H>78tXCLOathNfpMOX_hHOKyj zLjlA1mr_!0F8c2A^w}1uw*>Yn0X0w=W|YJ*-Rb`($(n$Y!DWO zhtrgO_#sFXeJO9wGNeBf5PDs-h!n7|;&z4Spm?B&j|Uy$MmwupAfQ07St&Fm!Bj}; zuH)4hfuwN4O2%V20y!VTRXCNCgdjJBr8I;$;dkI12qayMdPW+0bg+9m{iwvssLb#m zMs|RhYNa#kkYZ6)rFnNk_l(q z=idg8Z6Y2QrXvNH&-t930N>4^@1)k77;nLM9L%!)zjsym72zv+kh;(_vPb znXOQ%L2|o2N#B@w`ag4ek%i2_ItC1Dlp9=zdXH)!@_bb2l827>ufo=CXhq@bH+Y2a z?!qb_8L?^8-?&Hny(HiJ&PK0w98N$AQIbhl*V|N>Ul*^Ffpom$Q;vS?qIT)=%by*YWF`si z{r{Pm?87L+g-*bQjwoHL4kF4Q^1&upmin&4&G}!FtXZHEAE_2y{wug)JPm)~Z&?mN zvTXN;fu8n)om91V;z^N(zO(fTsil~Fq77)QQ37b2jHEzBK{gocy}kPLy=*bKbSee> z$*0Z{VT@vJuhc3Hh2S*cqpu$wqd_snv)UUJ{wQqmI{Zfmzadn{>umvrpyV?z@)1e& zigGHzuDE(gFok^N5s;dp>p`Sirm*uoq)(@D z&%#=WUS9Z}D;Y`p;MT26VjE?yy_aRgipowSU?(Dy%Y9#qcYIuYB}-F@S-4S5*g3kwB?X@97)iF=8fsKXCA=Tl1l80Ax7?J6yPJ(;yb(ncyk zxiI857QPImlgH_^Wcd&SSven-*A8`*Ic!>W?<5Z8oq}|;v5Sdm1OnKN7R zTB-VHahzebMN~4EN}rS}TQ?aNr%-jqo3f6d;{FKM~zC;KP<9D92i4wJvj zk#ii?f~(z*r3*o5q4>l#4!P+^92T;FF8T>cEX&M+_u-?-X7J1wqP0tP8vLKS6KMDF zR3f%$`CO!E-mZWRWP$HrBUlIxMs$lqUAe;MR>hR@EEDOnupl5w=Utc^bqq@PV)o3^2jbU4_WN~+N!I$J>PpC28FZ_E16CsuZl)b} zb;}#vg-=}*B}ysG=&!^bz}@j4Qdvn;a%OY4NwR<0J!W-Op$34$>?)a;^TgB`vF7`n zq*hB*pXMY~41{~!s3qc0gq*L^UHW@SB7ETp(*bcc#Vt^a6d9hYxR?M6c4+&>#DhD8 z#H)9pq7*EbCWK+t(uT+2xu4i@K^BCch*eN=9M(M2u>R!e>FS;P*DN229wJP0G!e@y zwf!JEYN`GwWP5aIb0YF;!eT#Z%BfRtH?KQ=8j;G^)h;B&4{tdWXftz0gwlO}Yaa^B-?gKHeF)vpf|e z*npzUGsw(eIO@=^`2jfxN_?aslo-R~$OQyAeiP*aS^dlzb&<=k1n}@Jn`W6F?$?Ob0G1ji!@Wc_b$LZQEt+8VfegTO!XWKv-mfhvpFGxg2Yv zqwn5+@~BhbwmJNWRxbu))A&kii=1F!0^?C!puvcz0O}(;th9?ca;(4Am-Fw=utuRE z+R!{dzN=H@g2M1Fi$kOs0zde2wE4?^w)3&|w=y$a0JnJ<7RKw)b7Z)_vxiisBcVBb zcuWM|WAjrCjV^jYVt-5eKk5^V;y`A%CDs}Vb2vLZMC5j_k(rh8uY-(YKZ-tW0cJxA z;pT%q!>`o%v0QMiw%UZebXc6C7}lK{bQe)}&?ClMRS$l_rCL*HPW`bZpO9J|EqPK( zU~M=xbZ`9?09=r#n5o%7C}n}J5XMr@gAlTbgfnJq@XvWZq}5Y+U}%*Oq_E4u51iFO z(SMdZ06pi?gsx{9$Qv#-g}GIrPz&$8hd4a0ptFs4#FiZbefj=CAgm~@P~p&UOF(&g zO@N^!2rbl5famfLjOn+y{07lTIEqpW<&KmlHcr3(8*&8;WPk>I6$J4xsEZtoMOw7n zl<;;NF0Lm-V7dxVc{D9R^C2v=%pue2^N=9;5zAzj|MIi#x4G8g{Sk%WGVt#!V==iM zzpm~%2R6Fj9SKjmU5hN#tqZ6GiqH&PkeG66G)9i~DO^1C44+#p($#+3tb1HY*zYp@ z6?V84pf^FkV}!#9h}j)?@pbce)(*eJqgI-OODpN%+c0Y)h-#k79u3Bl9N~@Xv;S~v zaiBOl6^b!}wDFOI;qZ@hAoJhGX%h!Z}&tj+M`g6L?W<9Lh<_=DdHzryf55s?G3} z;%_0!)rqd6y<`%X`C3AsTyl>}-y^sqXPVd<;th9@kQDAL*#bz$(=u90?T51YCvhT3 z@n=u}F3!K8o`2c1GjKnvD^U*);+BTv|FtyqPgNQ_J2^l$kAiIa-=qX%wfC9DH-<*f zP=KHDV(5L7Inz8Y%uY4F)lwqlO=F-ZxQw(^|f z*}F&mfjI6K`Z(ux=}SY$j|Zp>xvXiRsCwA06Y>Y}XB`M?@LhH#3%@u7 z2>$NuHA$YNpL45U`@ut)Av<)6o6;4WJUVoAxoST}eahZ1tD@sEi~QxQkLL0l`RYV= z+WZHzh73<;I{Nll#K*#(VU>NR&5To1ZA`O~hs{*z}R-EJBn0{quBkwTb;oiv87c%b;gmrz1VBUjYk5IK&W;Af(}5bE;jIUBG#xoQ z7hbw-LCmVdmq^Q9PT!PhCu)H;#Z7+_1$a8B z8u&7F-@)e>Qm?|PPRrE}gq-1i#L`85_7VtLD_Vsa?2X-T&7=}hq1a!P>g#6$32_pW z<)CG;pcP0riZ9qZj`=h^6uhR0T=uzC$98!0Aq6==9*?tu*c}UM(85{RVGcBIqEVT_ zjuX!CGTw4-mE!i@ZcmZ?ed#RfD_VW5*^`YPANnoNC+mhtIV4pv$dH|WADVokdHiCy zci|;-T(VdEzex7(1~~48FxW?>MM{@8JlB2Uy95#NhsBE74kq{UhfeM4BJQ`l1x=|J zfURu|GiceiYa4BeaFIA{9wp&#rQdI?h^ioj1Z44ST5|ZBGCS{97Y6!7(gsWOd>QTD zd?cuFB1JNHi!}-ahK1VgviMlf^Z^dY*2k$9;^X<#LQ#UXR7cR7j{>125^(Oajj&P< zbTS^l6C!*3gUoKJ{jeMr1QH9yX=Te_cq#;Om(@O&iodb-KK#((=2?$2sRh9~cWj+z z4axR!bBaCGZ3Ge2``-O)ry$^>{uuwMLwEB5R-A3dZjVPCAUX1wHH+K^_*(77OZY5J(U_b4spEgMGZV8E61t7 z4tbJ1RXJ$z`G}?1;AMTZG@mSoR8`+YB7M^d7f-e|UpzDMoAEe*_}yBg-l>6Ks$!Q) z;EaY?vu3psWHJ!NPYG79@`my|$To?Qemg=SkAq>9sI`8!)-gEP)~bK(1Ssw~5FL1~ zgpnbq=!BtXxP%>$`&PzR+jr*=18pA2aHL~oGlfm8B-_`7gg~y^n{nXv^?$BosiR55 z;`=V+WoVyIWPN8pa23vD;8x{*s~DY%TI!uX(dgAnES7OVw`H=|#=xEY@kgo`U{GxM zrb(q?X$`r=*=btdueV;8EnjdSy~jQ7yN6B5$~}MJC(o-l0k*{>A^FgvsXA2Qt!_MP z?7QRIJ_#6aJgu>t+yli{vV-J|HlY<%kzP5F$WCd(52c>S+2mc1yO zyz-7l2nvyVA@4QjM_?}c^y=-wI$OgjVNS3tPq?-n z78S_FY8i*mc4hXuLLSJ3_ajtR61-BRm7zXDDgAF!A3b{`svqb=`U6_uD(-KbM^F7+ zBemk$x1L4g`f#imz8H{&=!9;@WHSkx--Ve}Tq>&f!Oc2R-wkv$*>ISDKmiqWvz}{# zWL{G`zR%_>cGeA`3O;4LG@Y5kXO-ezI7PcJ|Jcr*IT-zFU%?oaJIHrQy8^p72%5*5 zb8g6f@yFT&4De5cEK7p0k&?@IO~<9PhyC`G-&j`Ub%hcL0}T$#4T?nC{0S4uj(~ne z{NO~S1?9QdGm~yV(lhZEnrtfBb^OxfkMl%q+**@B_!HrEDO(*{9MeY`Raf+U2-}@7 z;Nkhc26QFI-{Cc*_Z&kAu1fA&6Und5%y_UY|j>(>cuQemM~`a)$6ar^ek?mMq2r^ECfO2G!F z>%p@gL+;4@I2|Dl{cFRl%0GeDKyxT*1+T(@(!`xxT|WSg@RL1EG<8gjaZgY*79IDy4}X06V4KbAN7~od9~OG&=vOJY%e8g)Zg3!faCg5yLKIvoiVsU(ULq%JqifaR7Y86&Lb*`9 zU8d`bc*cKXfnSh|PRer&#xK_T94hHdcK|88 zHnY~KeDd-hc4&|9+bs_-Ho9+yH@`o8B_d7sFpvBpS+kEv<-S|_!|3+q;? zjTi`p`zPg(ZhWCyz{&?fd5-fj=n>{%N_r8(UgCbHiC{)A$nI&i%XA-!om0s<_43pj zvIEr`*W~O1vuuZG20rY1{7vPN`dB_TG;?pyq<@i^ZrJec^O4CucW@w73z=HW#Qw)A z?}GiNU(` z38s=^_0zVc_;AQFp*B=D4kOzs@O=u^|MuaR5SGED*dGL~!#?K=*Ky^5+udviG#2PV z+2yYJsNbxVc&{?wXZ@qjn>zsD-=6|?4MCLPk}ID*lhaZ29Saxn_|^+ubj|bY_Wldr zv-ATo>QQ;F9tIVI88SAVyzM#;-#|`F5pB(0ujl%MQ|8c+Syusz+`K55C)kQ2xtV46 z$s}&8B-gmf9iF^0b`&#Mh2F|z9I2pX23E9eHP~0p&4Yduv?%rcIX6Y*Uwr7ROwjE? z_sO!j%%T3y&u?892c`1X&mwLlr6v9f*!&AD*Eb_o=Ni}l5-R+9Lz{0K>mP)gtLZLg zlmYV#^PI20)m62ecympd&ER|9K?*3B-g{NhBJM+|`|iuBeh0|pnjQ-%tP0fiTs%s_ zk-udf&(>dL9R>o$-KlmGPGWV)_Kzuh62?Tz9uC5u ztmyJpEj{nh+9hRI-t05;J=^~6io_iOcPOe(wEFHA=yhga;$BqVf>|c_2SZYLjKHZ1 zx=d;1ZGTf?i7j96Z7dKOaJiJ$`Za)BLsSmA?mWIBweBFS^JGgbm>iReysEywJ$X<9I9SJ+c0Da0w)XqElD1Ql|fkoP{wxf0cL zX8Z<@)PskDB?+#uv}OzubK|Uwpg-iNX8z956C(Q+3QV|~1MSlpYD4drx`pDGqHTB` zs$G0NyTMhULby>Na8VeJyAuW_%0fp!-x^1RsJwmo+U2eo2r4vJtu?l|cJTeAtBW9$ za(DgK>`T))yZF)TqF4|XMmSNg8911>xikrUc8e4u-#%Pdb2l{-b?HPA-u_NBWmZBV zKE+xPMkdm)1(C^G7XOJyyTFAu?{=g|F5)=R)Pb{!c7^)U<&S)w7_g z#Y%YrpDdgtJ%fD2$N8gN@em`^46BwtzBXcXD*6^psc+gpk@75@IwoT8G3DczGR>^UVKl5s5F@Zv)0EZ=ljUdmRrU)K$)gf~PR9RrYT zUeJ*Iso3g_B_a8<3-@6co;tUP3Br*R1*3o30S>+MhK6Y41sNm zYFVek>_@3F7c^KGV5ah*hiYN(O`ga^X<|K*o&Fah%f4$JdUfAx$h|R3W|2}{C@7_B zof-4fE#51t+)-bsz}gojZtMc(^~q=ISvwW*ZxvoZ5pR2@aIgo!@N^jW@q;U>S!Af+ zhNUS+p*XLIZCc#EWa7w~Bx!3|3Z_-zXV-|wOclR=hJvNcIgG>%W_6uaYV-RGtkw1n zr;8IP?U!`6MEDJ*9{%>8>&ERWudXbS^q0p&aXNMrGxM~3LENg<;8Hq&j)3!%gfskd zENyqbzn~WlX}x`6p2T_%#~iH-|C?9@I{Jbyh+S!s&?FZf-ZwNMAh#1d*apoS@s@$< z*iPsrh_I$E`=!KRrTy_$(&_^|NVh(qtUbS7L&9Rfp|CGhyEtzDqw85>{I&+KsVt>fQp#_=cdX69a>pw|81c21`YzW>VYIeapP&=bcDo06lD5|qo1CdgurOLc<|-~ ziCeZ>FD53v-s;~PAH{Mn@@)m>{IyVdb zAZUjfonNlnUr>5obo^uVDMDyduMQrjpBZT`)n>=n8(OYm$^G8Y$v4BYN(hKdZZgiRyJD`KP3 zezW-gdh0UZ+o9lA&ldVP+`Xvk$igh!T^^S%4{!YMuo=gc<<8_3M573e9iqFhN0hWy&->&Mv=! zOPYmp0qb~8HuWT|f!+bn=<`$yB69L5Q-(l8?i!r*6^S~Gq%UWLoy;%*6Q<-4*CjRbu!93C{U%SzARL?cnE5V$#JvN?go{9OW-maAQw|ish{Wa zzAx9BEM-)1a z(ypH0dow2?+p77=F`HL!LniCM6!NK_dLQ$)5>mKtp9|E%sjd%tIxL>9u~Ead1}G9X zL0$fcI4x!R`j=k}mdUXSqAA|Kr*QSK2BEm+d=wHM|1o9uU)2Q%03KP6)kNHGq-&SG`wD}x_8dEpvVBV4fdPIV$|$q58l832 zWbbvy3lWMnzV50ZKw4yaDQDlfJ2~E7Jg# zbr{dxS$A2`Rr0nE@wj zX`kG{#_z-{?^L4b|J9xcVW(4n62NqZy9JIgNqJgXZIQT3i1A~l$N|`Hg&%J zo@zZkJ??=IcWBESOrnXMb)Kfo|EvXAQrRUHy`*-4X@e~LRn z{YySXBTciXqmus|;I+!PM7(D-Qiqwo^V0-PCNe4Yu1K~<|&N!JrHvt|wb@yJRR)RP~|^PyJ&XR`lL zupCWM8yQ42OlJpKDe>^NX%}Yg;yh|16IZFKfEf}zm9Y)>@CGx{T<83o8uDcTE*J#D z3@k}D)|PZX?K0TD@i!P4-+}!75o~jJjE$%F(*;0*yI&+3q{_R~%zdz4=r2KGR#%J@ z$%d=6x6WH>0ohAS4?h!blmkgewgckBJbI6zS{#uE$AL(d#W(TUN-x{`k@y_T*3)*L zJBf39m0x|^T;Kb;UZD&4g9QHG+Y9|-212-0P+lqZ^)v06nWWE+8JE(5?{RggqOdoE z^*@-h8nJmAc96bYVss0Z)geM#3B1jc@kh_&8*Y4Ax6qcbM{#3`>ENk$jD)r!B@kX~3L~3C|94Z7Q=X!4R@KxPnK+t4G=A(Yc8nWk_IVaEdcrI+e zGu*hIG%UJKOY&|3swy!Mm-X3%{Kn{M5L4!PF~=FY>Ye%^VijDywVORahBAvBZ(udw zp{J<)>*t)@(wUdW-(V=ac>*+A+2VR{pKJWy)v4Yxg+Q0Of_4(kHKL2>{(!p^>W>#3 zC=!8L2iyAI2G?kOjKbfM8ZvVklTuXjDuZ}Ry+ZaT$;L*v8HGDUb%{*N@7gt@QBmYw zkJn0bGudvl>kV7mnDn@-7vR_mhbPl){>(>;8>AJ#8hEcZzWT)J`E=N{R#(sH78yhPRGj;~lQWyB#uv|vEOxPTp3S=%nii0p6vp{!hom~8^! ztOPd=gIb-(?L{g~%AcQi08G5a&@m0#CfXo7STMSqW(Q8;lD=8EP`;2|;VbYH%fNO@ z6h=%{6vh~1^D1|*LaXPg)=vHpuOX#OW@^&8TAa7i*A#$ebAjELwLn5fI+@iBRc!!d zhrS}R?GvqGLX5oXH8^+v`rgRqN~_ER;#W8p^{Bio3&X3T1smwos>q7?lZ!B-T%<%v z!pnG+3Ufgdr6P!^iukvs->g^;6?58+?7scX%md4jYY@9;1e{>gYRsh~2n5|V{hcbS z#w%?0i}c)3z5=R``S#05#I8kE`|C-Fbw1`jFgMF8@?u-k^Y2 zVSD4ihi`auUcv^A#kbJ721+CRTcGrxggkn}jJeXR)53)xeP;Gl>(sYmGMjxc7?G#x z*Cu5nLdGI($X7bW-gS0yd-W>K2FtBlDzO7Q9z+qwlw*U%d7iV|4n`u=^LxJu;skNmmA{f3xG;Ku-}EZeEnYI0kC_1kPXsPpr15KoGgHjpbIaS&bXjBm5);t ze?t=tNl3Q2o4M}3#+hT zhpSs`x>B3NvBFx*l!PMwqJ$d|Kj=~9OAnmiQ!i-|pRF&Qea(YF92AM>P*~c+(!cPX z$<8E5+_c@f9(jM@fjYR@B(bBnuzK>vdCmiWqV3Z5FGhC;ESPs5fG*1u;Oioc_vVy0 zH3#CaAc%9H{~>D8ks#`E*hS$q0tfa}KXphCf*Pb=$NcCxyZPh7ymqEYe7jI`EN~Dz zK8u8zy}(7%gQ@n7YI~^^moK%$$b)}_E{(^8JH!t`R_TK_$_7I&F?X5{P8XpKgKX9q zMQSUDuax8MC0Hf@w6E_{Mzs#;H?ztI1k2O=Y+BJ$*Z1#_hdj$Y)(#EOqFP1s;0NNl z6yIHU_g_ChOMUYJ?vm2y#qZhC4vbiK#$3mZ1vu^KT5m3Rufcf2fNJl7*{&3?M?NR6 zeb3U)$S$8`_7b-%e7ip6>stcwT%ry%iv$GW6Y2y3;Bx<3L_a40akPSr^Xe|?2W>_0YAMk>_3@lQ&@4Gkeh{*OXs zi-{CrQo$pG86m+!N15X^dCrR6xI-yTbySx zkhJSPd!~r_uGi`}6+s{XI8Tj=9}cM@?20#BEU3h4D2tioziB6znCHZbYAUeGvED~= z$Z_*`cN`A0*24!ZwG5dWOZRPuJ9@P&U2FM#fST=M=xO<4pjMb5WoOj@N#W%$)48@3 zZeI_Yl$&kZX2#>_)m2bBh5jgfBK&X1qA(-72wG2G_5kz6E2!l$(hTzx!m?z#yJ;2< zHWu{st|MFw5~tkr?~NG?);bLufoGQZ-P&Io`e46jXYggwHuSQojKIN(<@~(7H}fP# zYq?Xrt|w~@ON&+IQq&l4A#N(rIByvVEu%c$qgDecDT7M_W;n4uLxeO z2AimPrAnU=cp16p*)E15%so8tlsewgn^Cazuiqz{=Jm}+^1D(dIY>_`1 zJk!x+GV!!&z!aH^{l$?e>0DU{Ri{fw$vF&7oXUaPp%k)=8@~_%LmE({G(LjD42in; z7WAn0zyZ;8rQ1&JxO>Lbs;&sAWSc& z3V0T_yT@Ivo06saBu_TK$8q2(E%4WX)>;VDLmr#KXn(iqp~}iz+Dfk$a=)9d(+#m| z5yLK%P%j|!>;&~4`c)~<^|PU5r|t!w8a`3vXtMLJ(k*pmTK$}X-TT<0pwo=XZY|^W zh_xRwT7L+jFcHO+x@Xpar-_7SJTy`ZVgfvFWOzgznKnX7(S_nM#meKarh}+8mVy+i z9GP&`fAeFho&&Be$wl65YJhHOf1cY4w<8uzPcVG>w)OUX!)LFc39`rb)L7jgd?ZM_ z*u?(H$tUPETgCUZC*O5t=0iZI4~MPA0z+O%X!+womZNWX?zBW}O*SW5t>+;R&(UdO znP9O@AFBKb@yHoPJiwa#L{(NF0ShG>s9aadrzQ~KQ*osZr5Xc#S4K0%;sc0%zEw0z z-(mSYqMnUsFo7`XGZ5R6dKnBMr(-QjWsL7dJkkffYx_XC_ z!1>qa&o-DHI+if#1t;CSqx86^N_iahO$HNf2;rmB&r_HSZ+sdCu-~w6G6b}!Tu>q{ z@=(QeM&dZo#iF2RsP^?Ea2%Ox3G4tbSaj~c0Ryj>t|6g7oC>**mf5Io^-^{i@cR$_ zplg;tz&L+_+G)=4flG+4dz=i9=3Q%!`oKjqk`bx}tH}qBG7_{G*tEz%t#jm(#0*do zTy^RjE$lcoJl(DPTsGOL&w%DrvpMBmPJbAzY7}SeVcoictQ1Q3d(3g@daNHbj5I5d zdfHFiKyq*S97ksjVUx%kJ1`Gr;7OwW*)@sB;;x{H{b|keN#F>@B}H+q$0W215Jl8y zB-)>Z!MYD=6JSpL7isJq2ZMd_vn)T86y|jK9iS(Cfg+_{uN~AO1);#17l|UPUO29< ztw<>Y@XasrCO#+7Fbr#($h3-b0JfVW-a-h`2%z533$p4Ru5-s>g$|*Yl*extL@XGc zdsW~P=Lg<+K~2e1gkP_LDFfQ^SiiF4leOq#NoJtRK|(d+Q}N=)!N1eGnzXLMdZ{)> ze*>d61*|#Lx8@#;FnXtbkC|ayA3F6!`(t!s#qPEZ$}J3|@1RWO;jv+6^KSXg3q#W8 zt+}k>N$w{~98H2VAEY0lODKj=RV~Z$T`kyaT0S^+Wvcxdptzz5aGcX1RRchR8`m@x zcCJ^6&%QD%J$$l%4`3B@6i`F{I27|IzxBtUZl2U@1v==4x~z`hwN(-_9jCu%K6Scp zc)Q0!cPZcmdhwVLStEtC@12><{5FdcD3sSTYjv=-Y5j1%{1UCL10rUGHVdo`{;8&3KSPZ0Z1Y4>dh0a4FHq) zqGo9iVLJ3s+PGB+#gCP)eKUDE0%sw3|79V@{3oO#|SzE*DVq4T#2 z2W}g{Sfn=g&^Hb{E({#~cg`=7L(v*_s=iQqx`vb^i`O4@QUZQv%cYAq2}by;{=vtn zxk{FZ=sPf|ji5Ed>mhr8Yy9DPRxKY8fyUOybQchp*4kZ=9_&9t$I}=jB=ZD8Wpv5+ zwJWgVV<5}rhr<~+BzHhv%yZv&F8|54FQnGF+d7u7Y+f@U1XR`Q+jt$0iE>f_(R02; zov=N!b7%&lLtbDAiZDSZB~MOXMd{WCuja#(VH3}AOnkwKWl+zus%(`FG<_4?)Q1f~`=%06F;q*Kg*{NmP`#5Iv(h4Iv{xld3XA zh=Ookpz5OXBXoLEi;mvq7zZ>~X#d_y%3cfrv_t7BK~y*ezFWm2L^h~aNLkV5IXDv} zRryw3F5_&ps?a*LAmUikVCVlur~6;MfvmcEIGWhQfe!s`glum1A6Cwkg;-~svL@L; zj2_ppcTR+S9XAvsg7%%k`D@yK_>07zn`ez_M;`a)o+R1M0PY=$?Q<+n7ysme26`iVZ`#>AckVg+=!q>h)j$ zu|c@Ob^n@Jf@}Sns^iN}=IGbFIUZfU0HOQEWVcbG{iY926-FsgNS*DGy;2UIH-!Qp za~}<0IOpgRC>ucdy|eQI2v|9dFUsu(J!akiY#HY@7W3}^)YX4`^MB|ntZTUFhz07E zBqeeB$+`rG-!?u}VYWWf4#VHvM}lcb-@dt=o_$|Rr3jNsx@OwvU%hR`rZ-2zpoQm=|k3Ue3fG8I(@);@~ z<`6<{{87qt0D-*;UV;RTR^7gz!bEK;3(2Bh$eDtFHB35Ex&elOEKMEVfJ~yaw}&XB z>SJ2KQI9yrk)UL}8nYTwg%~wW;@4{lwZ9#O@fC=P^Uck+}0hZhe2@`I2NOMyGfW(g|#*sK@FJIzXY3zTZOLU*j9^z*Xo=`{y7``BXxD z0KnTyU2XCvA&feiBynf`pRY-A0DLh26ZEdsk}nZKDR!iZ_b<=+1S7mR^gaAXt_oA+ z;sDvG;a!CNjVIwNW__({VFd5iMD+c;&;@TG)yr2yi6Z#x^HZ2Fs?fetS$0NH9PljA z@#zG?>&2ks6nc`?qu`F;DB{boqoib9KBR^+OVq!vPI*Y322oI4pc{JB=QJBq|BHvN zXmzoYO>-aQK3@#Y;~R~kEIv2177F|9Q6W0^kd1yF^tv5+MM{qcR)~X=({L4>AnGyP z51(tO6o2<(f$IVI+aLIgiVH9%ub}Kqi>N{ioG??&7wZZ^vjF$$yuyzq_oNHz(e}{( zm+e7))c@TWj=297`QKf+cGJ+6i~3xskzcs3&|1wFC4x7w&!*#LY}YPS6(0c!<7+_! z?qS&d-Gy4@lzVSswOAKK7ATIMmenE5J1-)?Ca`3x36Bo76CN~@Zi2nBcmm}ghURod zSCKqEDDyQA!ul{@`EBE`;ukvZne1)HqcAc($qynEh;97*lE<2J)Q2ucC+9r@;~4Ni z8%Gob4|%PGBdEF82o*w6%%YY|3zEEyZk8-|CN3E?yX}lcts^Wg%WJowPYl1&7e&<4 zRv%0n%BDqw{~$f)m8E@ob_opd*8LfH9v5f?$U>)(^kE1Wh^y0?pw|Putg_6lS*X&m$u!#cxA$7s|sX#~x zQ)p^qrSybtHe(r89Xm}2YIZA;F#!jmfxCw3JP1hy#b7MrgD$_d7{~Q$F~1JG`n$b&5Gr65>Vc9VkPtgP zsggU$#0dau5#15sKurrWgP%Qt@46O#v2R$~L|Dj12qv@c_w+IQ$4DORW$5iPJ79N; z2k2z;D&Xa+<%YqdKZ0CblCh<>iDW0q9m2H+;I~@xS(O1x=6YfE5^pAOl~DK;@|jK2 zU-lj{djek{lR`r!m%lmP-}WLZd~MED2IKJX>^^bm?cf?Hb$u`@xI_nr$B#{`3(cv7 zA)<2%cwp6rLNOygo7V$WVo?$%_%tCM;iJfS1f$+RnD_F#)ep(m=!IdFg#x(onRgMG zVTWkgaZ`8|!)3#r`+UlsXDgCkLKCS^_X@*53tVV?1bSJ8;=IE+)ceVyO%$+U=ZjMG ze%4ZJo&#RoAL@e_yi;quPJS=UD!N>zZ}-=<2grsIA)+=tJ$)0g3>*g2b0aPMdfC-D zU^dq#{S_;giYZ_t{9i0J5-U+$6W3pS=hwW0Z#4J@5GyFD0kx?9ul{qLuFR0TsOMmD zM86qE#8&@j%7_SwOguo2#gQ-|Fd61ft>hPhxr6S2GL+p1l9)20egFTa5K@}q1L-{I zdskN?jYg8VZt0yx=CO|avjF)mV19IoAEfI%pt$}F_F+kh%EY6d{({F1uBpRf>vBD=ZA*@F3jZqtF3eXIQqoEWZ5(UQd_@L*aP~)Y4GqIJg zEd+Sz^S*yOUt$dJ>H7bFPs%d8q1=f-W0(F!mpo#2FfW$7^A#>(i9dS0MvgTD1GycC ztE;P}>vs4Y(2;9h(s_}>O5hEAPtlvL^sPsBfTcncy&aIQA=Hfr0W3H}g1LaAKxwWW?=_f!m)%K zwEPk_TqxqDJcjSVOb^ezmhyKhSuo002y@5ZDHW$Yy;9J221{ad6q48>NO&KO7^ z=Rm(^vHEX^;4GT7$b#OI!e<@$2TnpXCd@VY?>lUv);Ioa^Y;qHDX-iLdJn;86Zyxa z2mFsxUvX+UW&uYRSOz23mMt_92pFGIh46V)Ewu$E0RZhSPXGq2OlpeTJU}t&MEG@Y z@A7^@I7j@~tcgXBMcj!+b0TJ@5uboSmcVq)Cuh$iD6!y?P?zB)`?8;PM_--vP=&oK ze>{R4B72#K-5XU!T-TXPr4EY?WwiR=UKH#@1iS>icf45vJFuQfm(GN z48uyJ?;m2Wjlv%@l%aYlIS<`5n|d={>W!_Ne>I~x2Ne11CaEcYl5AhOQ7>5Fo=0N` z%XTPL%o}?ZKeFCf@Qd!}N6lI*-AY~6-BihR<}B zpQ1%m3uI-gFIfc3BAH}A&2~S$`3MQ+(Al+eBs;GsG*$(#d#@=i@DBGv>4RUKLHw%7 zBN~PKu2VfM6YjL+|4jEJjaKI{t96seOvyj?Lvz3FhmJ8Vn<9S(&0_aU)}I}?{)7wC zfo{TsdwuJn;a9}@gELAC7BG}vE7DZi%uFt_6(_ZLE-hOQM2IaC|g-2S|k~keTYmEM7ea6VQx3iG3tJptX1Xwz6|*Ir1mKY{lZB3 z_>9$pGd%?!vDdcb?g!hQ`*FuJR-}ebYIgS3WtX~W)9x0hcUjGBGcYx{axiyn;OtJ& zJ98;6`ARDqkHucx=_PFw`q26C`=n2pQeX7U$DZwmIahHLHl;CZCOIs)pgTuYRWxC9hmboHz4!~Bq-27x!_U6Db{Kstq zkZ)|buo4O5Ra5rbZ}ie61P}8ee-?t3d=69Zx1h)i3AH%XK(xUFeNl!}qZS;AH+CYs z(ow7k0odTj{a0g`Vcrz*Zp&aEb>gl%LNoBJ&zExFQ}}an!?uu? zX&Ej8AH`*o>&D-&L0)v*)x{B%ud&V@8YyZS`3MjBCR!N=n&#?w17~enT5M&oa-7n91)xyFLX1gi^Fy zmbdMpbty|Re)%<9^&?9DR{@t>5@vkTS_fjc4mdqMs5YLTDEnPtO#OIDeo~d1y+Jwf z6#Ohr9z2Ny{85FgNr^WJl%{es4AAZGq9ASjQp(>{ubh8&o@jTnzS|T`b`B%P3bODL zP>k~MaT1w0n^BWRR^e9y=-tUY-F0r#yk@jc$1ns0*j6Eb%J5{bl2e*l551AST zN)_&}``KILom|0thU|^-T1#N1y7p_!AJA&jDFKgOuooFvgMs3PNmugsn*FZbMJU%S zqzEry^G(_DP+eluEdcyiS^rL1E`0MwTiYHj^MP_rSUoP+;DEyD4XVTh_96gVbbu zP>}a!0^&0AYt*M`SOT9D_Ea?wo22!99VLMrWC20R@q|1t{6iK4Pcv@-0~MJsz}}qREQ3%7`?J5@pw*Qs zM@jH?51=FYY#_N?B2SYtrCo`q_49c1``=&cmf9Btz1yy{Q*1O8Tl7(rMNdBTLhVm0 zQ$sT6ewu}P*>b;-ol|>`yu*9~OmeW9R?VJ096Fv?_xKCl=!t&~v0QH@{KwvkcHt(Q z#vm-NSU^P(>RFi-O_IwFT$T^S-&fP#ux8l5#yv#GK$oI?>BY!I7h=Q5o{UCt>yodf zz&OYE_EIw2pLoytPrMJM#QOubv&J_!Oa6}cw#*yaGhH0`f7^z>C6yfK50uG&`;edI zhI4$FQF`jO;2LZZ47bz#=PVMD_}KO3`Pz#oE&=9@I%TlaX-sPT{$ZEAwwf?4vf!YE zqn5pOfbG<8aqpWy&Xm7UT>`jmL}>f%6-@x8vrS)HPDBP0g7_t;@E=+ z8z6opHu!)HIJqu)V3bR3^Vty9cSqjHyP&q0J}M}ybuU{>kaHI;WXHx4DPRiZ`1tETb}nbJso?thQc=K_we7!b%L6HJ^bE&tUdaOW9js#r7ZRa z582a~u2{n(YFzJE6P{@)wY9urk)>Wd{D`vD^pv&&6J;FAzG_*-}c zhAdP0?xSaXR0o@+y(XU zd)e=>R^Yh)1{0nB#17weHv-5^XLQ2gCbw+G&B5zS=jFa$;GtN7!lHK9TCX#>A?6SF zuu%__R+@glIG(0|Ko4--?dvg?c51j#fBXT(EmV}CMlUJ_rx|!1FGpP)?X3aJh`Dt9 zlu?$>FZpPT=aVG`Ip;>7QT%X-hjGpBZx|+_iI~WAnYcZR(X8^*BZ(D_9u#}bd*^0T ztGB)3JZat9^>J3=lIqd*)Wl>el38t^Fd#TMf{+ulpL1bz*GIt$%MXRCfz+=h7&ayq z>Z9L%L4kYcdzWdc_hK|54+Jz?vo%I()9jZagHLt*uk7Pcj)}vvPGsWAU_}M`Rz^-r zA2!_tqF$`>9UI$oZ_fujXpwtoQm-=pu*qt+PsM+^=eB&c=n^Z$r=LUyFu!JX+MeI) zR!#(+$u)SPlMF{mwe~SVH@4@GF~o3>BE_Qg0Z%pXsERB~a%YqY*lOST+N~%M#q-Iv z`jm`^TJr0dp z2klxdAt{P(ynd!|wO;WaoA$L(#dD&yN(F&u_@8Zq*?R;vxV|*7cUO1NK%eT*8;1Kic zpG+P`-x~OY_!yrtGVu9%#YQdUuw>9$@hn<##rR#{l?&Id;TQ7tW7=0%Jl?l+zTx7# z@bmVo?QLYcTE6GKiRc-L{C+0M!Q8b=&RTY?U>-V@a`2g(Hbvk?E~G%buK_~bxKKn< z(*v-)$TvUJCD#Ei?roBZFfZ>*S9^$Bc^HEa{U!9=n_Q;H$E?SdKQFK&c&(8zwX*OD zOzN@Cowg48FgtW~lR!*NjHae0 zA83g=KdO_}&t$R^e&M<=_22Wc*|FuAb{Rs4416f$s#foK+XEbuiL;UvZ>=P|?4l-j z?fvPG;-vQJ+#0=WAdV6@R#2BgXuGTr1XZJZti*tXHD}};D3#LghEHps3 zvrf#cB5s0vPfe`ak+(Ky|9An&?mq0~lp}a*nY2RObu7>1YDY*9CTc!==!+WT0|+gY zCy_ccSo=VQlhj)}`5jPWwyxkef;{`DZ69SkSGUK3HRgPTbyQiEZnSuLdsUiSM? z*X}6NI3bXh`$~uV;LqXvz2m&8T=hv;EM#DbMnziJz^P>u#;~_rMRsho9lc2YvNVAL z{4J&;<{eFl)1qKteA|~UanIqim1Ga{6SnuhoT+gH8};;9X!1oN=jDmc635d|1x`Nx zf%-i*Gbsi`Mnclb>5jhE4w&Tjm$}1<*Gmc7Q!P(5)u7dnF6-i}7ge0kWj5Tc=gl8> ze5LYc;E6S}cv9{WE7RCly|4W4s$Y@#Sr+aoGLW7EmQ}VrtLkZPKFt_;0w!FSH#LoM zVjCmm-5ZwulReiS6I#X(@8nk5)6Xw_OfqcEmVV^~GHIm>mGedIt=$vda3nbl)eG|` zK5&rQrSO%V3W^S>)3>?a`f`elHfQMYCdYWA48rOT2YQ1cVF)$nD#YPlF$bDjG6k7BTV}V} zb2>~wb)KMl(Fea^?iK(th9;IpYxkMB&W%KtFJ7(B1KnV7oaBcu1saLhGjBaHIH3K? z4oC2+bPnxc{;8VQ*+B=z>U7lXRxyHpq~H?2FR$IN)S!6-T2~Bl0%$C7QhTXcG3~}d z4tafT^>l1ByU6U|(=M0I$I^#FVs`$39a8A`Phdja*{qlOGgqgwwEb;RjP}zb1(%o* zUFJxxEIfU$tb$?i@t&KRi;3&}ZV|a4iUTa3oUHrN8J?FBr-m+nWG!AQJAZ0tavojV zB`a?4rXv|4Vb;C92A$T~e-vHWl(;oRENS@~^8j@~> zgoIS-omp%FwP|)=kHyd5g`dCnhBjQX{zNxp{vk{`O36U~<#N~Z+nYNeWex;dwjVSV zF|#>HK(`>5CPU$K_+!%XJ`N=>t;4aZ+eB7UPefm0IY6?}sRizS@oQt!F*av_o`Z++ z1AbI6t<29a9_OI_zv|rNuy>@h(iEyc(MvyV8_vGrYVJ>vfsw400jtKr8T;<0oHVDv z42XE_C2-|=4w6YrjhEWlV^FG4?;j~9Xsc~~L>K`s@sgLhU*ISjfg&jg@RY`9+fmZj@RLKBlc2hcY9p*euS=SMgc=)3-~>dgpmULS)x=$#X&s0l3=^qq>zxCDl%ZwKxfgPJRDXjDK_a)M?A*X*dAg7zZ&EHO}><>o%Hi=)T2BPi=|3a_J-0Co})nc&o;R) zzB+W7gbR}WPAc?9^c$D44XI8eW>Oe;?OSX|F$wP>-BGhb4?U^=rw}O*u2pd+l;{8v z>bdc(!nbSZhL%pEtjXKB-LiIn5>%WyDB4p8-D|~)uf965Eo|FKZ-4gf%6`|bprz#3W9a6(vTDC%&=i&@-5}A+YMj1&km)51C^P^9_}q$aNwT5f;Hcf;|+K0 zQ%TX1+|#Ul}5d^$8u$%K&c8wJa`L^5nthCrA;1rnxsfsL!wEEv{?BHlx2|%{;Wkzx3z=q%^g2;PTYZSJ6AV4923HYw;lTO zqSHRO@$@ai>EKDD)>`34C^{g(cdV?1=4|BnFbsu;hKpBClKz%3yt|h#WglG}4@VFk zOsrN3h&@}34+uUJvG@00SmW|LPL)@XA5R_!!^SP=CH(I{(-MeCX!9FW-5~3HX zk`N90(ziy#tz4#Ub8x{{kQAXMf7plJ^2sboadBIZqfS|RXo@`U0E$uF%Mb(dq!a>+ z5FD>7xzyW_^-Cc+e+v_5J6LWl7v|g>fp~Mi9U}f9*f=`9T)`osO2)-#a`NveQD{dl zG~Hc(4d3mBmQMJ-M8kFdiAAJbz0TnJOBRpgAv$=IamVP`$gRom5cBgZ;fEK?%NwfJ zt=4L`#o(X}B3fZsQk+lQGkG7g8>coS#J-YUvzuJO`tC57o7UYh@OlH~yYDAkTYAiu zZ!RX3O!rOVH6+>M3#{THh;w$&S~TVCnLW~B6}?P`PBLxf=i@)4C)IzxESBhr&q`L0 z`l-#5&%mSMOU*86-Jie0)3%VoZBeKgIvN&S+J~u{pUBDMIZth8a}t5_9N0b=J^FLH zB8G7`{36bJ$zH-I`;cX0s9SI;$GjJuFA>x@4w`!oCo|(P%fDcZv$MWk$6>aen({=B zKX;kakDOI6zGENXi1oDk7inX+(_@JW4_@e@^+Tu~*nd1fJ&kBAh^+_N{eT9nkHkfO zFPKi6shXd;N!-Os)Mq2+P-c;Sd9RgZ$uB&>nM4G7CFhOu_~^@R--n4(o`-ytJVAM1 zsLkWX)TXG_mzW&6^Rw(5tzpFBW|(+?Yt-JF*wS=mq~4hlWxNRJ@gQ->2pYR+;;5~q zL-(9=hkPy%o`&bPCVHZ;7e3(-I{_Xw!NVLGF6N#j&L2X>$5KtMn)kAq-{+lZbt~)# zOUrvWGeCi_e*|0ZwsaV3H)52nPi4sTBb4{bJ zBsU%S`9Xa-77^VzZT=#?lWf((hvTuNMzrAt)EkU4pRMMapInL2?b&k;MC}F>(*3<@Qku+fj{dqYsCbydK-7B5^$JnW@ksT`BZY=lj^9>_hPq_u)qqg!o-1oC{)PWumUZ63U}xC{@8X z8abyMrX_(-Rkg&lgq&&;BzRU+H+lF+Rbw_Auz$-DpKattwi86OX_z?hqxvIRyx>tB zrQ)BbiPLcz``F`+2v|mdF3M%(o50b3ID-tsGN#-ibdh$`+$O!d|n6!B`h^2VOAzqRM$(MA$ z#FO$GQ|fbcqF3(CVJs78eD8{6SRH@Cuw(I^F0Cw*Qteq(zhUPXUZMLQ{wSZNuV25` zJa$Y7p*BlPOKZ(4p}X%Fc_Qnt5@PS9m<6We$NAV>Y+XT8gd%O6!xY)e3IxEkqz7eD z9@J0Z#`XE%xOe?s)|m#r!l}AsleS_nvuvD4-P@aV`M7r4SCkC-qvlZvzC1X>x^ueO znr4vE$D&HlD&c<$ihrLGQjqw?3c?xy-(UO*FGl|t8$j$)C|qs12Bvo?nbtiiD6TyP zo(@U6x=^NUd+IhnH;kgu^R2=e(i1Ik7YDu)X%=OtLne}|-yIX9Y=FS`;blyr|NF4E z+wmRW(c*)Bzp28V^z*rPVqd0GIY@~0QP%yFMy)-W_ z^0W1NZetL^z!y;UkeoWJ+5?pSviv97ZiA8{_eC8bac+x^UxR*p=J9AxjbMLl{pSKu zuC+j18Ju-F*B+=;hT+Za;Gy&u4^4vbi@V2vx|tdS`>HJ`Y^(G}{yvr76^ey-l62D) zlX8274k#A6!S=Qcb2*!eX3Qx3tMm6=V{OQ+w>WII8R4hy1(vD(lRSB*;}lFnjY@cTp+^DsbvA@K$eY0pH|f|@N3 z*4B1c!cWG0ZJ5)1EE8(Xpkw~=h?(+Nt1D{ErhiVFzbPp|wyz7t7G+kSwRj8>Og{lm zv?sgGJ|$2RtB8UAH?gf@7JcqlotLl!(wX_Xx={XAQNmK@DF7n8H_G~k{kC8(CIWu}OMfYVZ?i8GjFY7y;eWs}_v!oFa1 z?`Sp&V?gK3V0!FQ@Iz^G#b1W30fi{D!8N4C;SBth4*BY7%T5?7i$^U z%d!qvs}bAv5xtEdF_~jil8e-Sm8`yv{I+!TQa8t^Ov~V{8OEnl=tMZKEh5{PJ>`$l zbZftw8qe~EYr#<<{8j0rDuR?ca37J_FKcE5Yu?0?g?!b^@|`qsT@lX*b5#V??{5GG zdm(WV)?pmYwI8tbl=I;{=j&wO^I_kpJfJjzPK~UD1XX1fm3{eRPai+xSGnyt4;Q?r z$ENG%Z&2VHcsG|WT*|3&nu`pOx(PsgW4prNzO;caN~Ej>JX*7CJ5Bvy@$gi~^@B{e}I+>3<}6_-!40o`rVs8`kG_T- zem#T1a>9?!d0;q=FGC&SZf5ezm+>4b$a{AdT59oNIPFW!x!aEJ`U^}UX%d2MKXj|n zd)BUmy7Rd{Hug5Gk$!wq20XBH@RN0xi2JEBf18ACh7{UWceKiGek@60*)`^P>_H`(j&Tpqh!gsp9k zNOgpDl4?6k7G<_H?kA;9wFZ@BldT7lcVCc5TRhXta~54?Y*I`66jABDv+voE{pHGX zLTWM4A^HY3F*C;=RmD+Lt$L@>!Pa?9noIiJ1l~Nd!97ev2sXcMqjeF()w+e*$1{kvkpEr6xMLf8IU3|Y*o{f+DqNQbOlFh+~suac@ox^|N z!EGWUWQUk%dM(L@a!#=O+i}^}eaKV(6)q+H8!8aE|GWfM<_()ShY+u=ImGt64^n$p zir1WfdT^f5IPXK{85>{Vn;^0|uJjtYdGfScM_xaAB-2TG1ElYisQ;hNJhrQ39RK7o zH}Qjb$$zSRwTotzxVxXzm_Sk4&4Co!{7$oRAPq z%F%j zu{Ja&MBv(Qd!um!y5`IipE%gjk?c)dwT8-FgSorr@lCX`J4;G%8Ow2Yr!_ubJB!zB#qBO}-d2|B|9dfdh&L+I*tssn`Lo?#%ri_eA|t-r`w^6y`|r!% z*>YTnAi!>Dw<1=e{9ZN2Qw#l&$W1_741|gQ@|hX=3mpL><#~drM=!bogf&5#XtpNb zLPfTE^|3WzZwP+0!DQ6Q=>bxvm$&;QN`?%Ai=E3GobNubvbJd*9p<9y4`R{D8|DNJ z#e~OWzd2XE!4-?ja;3I8fNI#nY8Zl~Z}6Dhxl?Fw=ffx zWU@cG9>_%~wQ~4r58+M875_B&cPjq(ItqbPWSqD}H=!(v$blou_LlyOzWzS&F}zrCVF$t(YglB!IOAS1^BEN)?#I+w6)GL2<)I&T>R;>V(m z!op$^bKv-F-nVaGgXjzY5fGNxlgohC3e2KI**`xwn`XJl1E@vYdJX+!J^drnnYtV= zkJPT=0$pKy-2e7qiP_RY*|A{tgU8ZO8PA1L%b{R^Ckuwm941in46Jv4czP)9`R?%A zhv$;K;B_|lxpnU2(Of{~*IM`}L5e7A?CQ4!-G+(7`r}jYPLs~r*(r>Z`h(&ZGUvzG zvwZY%%Z=JTMx@8f@<(aUY)rf*^ghVHmxMuU&xVfAVV>b9x!7N&&BmxZZGE{yFI-nI zY~$4|S_!pa?cdOf*lp5E)%lZFN|)0W*Nj&%D)Km4TSkVeY3YE~ORnYbdby^yla^Va z6bEJ?tUk}_JlQw8{)`B_;}Ry6@tFW%=8pU@NrXD(qlOp?QH*66K^)p?@+rmGYzFf- zj)g7~*L6{m6N-2NugLR$(Hyj=E=~>mDIK-zl!aJrY2Z53;r6!;Yi@7DpTNLeJ?pJ@ z!`5wQNTwcn@JHAF?+@OgZ4Skw2@F=Gwb8>fh*BX>iHn4@u3lYl5!MoI2eL&e9Hv!? zBti*{PHpp9JC^oi+4GQW9CjT~iwx`Ohu(_VBR)QDIf~(DJK4?1DSxvl0FtYe{h@2r z$?eS9a&)nK3S%}RMP*%{Wj&@g{$156n;FLO^tcx`r;FHoPLHd*+1AK`IlKK#O<3rE zLVJwACVdR{R8!4|^9P>q_WDkWuDCa|DLzDY9>BX-A>ZDz8}@gbj-~I1sQFIHy&Eog z(d08N2;k&r9PFZzq3n_@E5`9?Rb}!B`Ku!_Y9>BtMuWoWe^RJ2kYUpXc;k9O5GIw*Y@_Wne|&~C$vW|^gnyU&g$$@59s+r@h7`ddVJzr)MdgaHIE(Y z`00GMDd#Bn@r1ZAozVMral8N3nqHXDN)I8G-`@(=9fi#rV^>>!y_D9Z<2w(}Ac%y@ zo>YrdKRr3z@xsjFVArwiLX)>l?>REW39C-?$dTLQQXjx$2q1Ak?TxhZIw<5nSQ?*> zBcZJBoDs1TqyQHinc@g`RCcl;eD0r$f8@UaNLwfX5?OFYFvtXItzd$>OoCbAe^rQj z3PgSr6ryxb-(G0kZr^`2=SK-g{<*P_PYiYfa+kNGAv-)srds2%)4qR$bCGiCE>sN- z=I?w-Ti3hS2slo(NL>7yZnyafw;)zkvc5PKx?ZQfTl6wj{rvp$>?HV%nFjk>6_ALZ zW<{pi^=S4${UEV*E1U)yG^vCdb$-At;uY26iM$L5w64%^!$;joeI!ZWqTn&h>eUw) zeH8fQuzdBVqdbE5ZLb`#hcbZJ@@H*$(p7Su!xY82rmn88`7}~K`@wl@FJdCCE;nl@ zsrxMlbxbJ%FELMzJ9JyMM9(IlUpjWl?2V^xdr8yQ&2+TTHQy;m-@H0OR}8!&wf`%= zG9#d2MknfuzS6Ms3-6NR*_pXnb9)9~(S@^l(gW!={^rgO-19%*$hFrO&Sl*5uRTn! zT6?$?K#^I$b!}WAao;=l$=GL~CHET2c%>d!q04#knzbZgH&aI-nc|v6w>^oZxdn zAMTx*y+4br=ApfxGFzoo$Txpdop2o_g&x;MMG8}xR57z;mMDby!p|Y|3KiU~HWX~0 z*NadhbTRkbWD`>?xw&?&gcyfO)!pr5 zcUM0cW9ok%`_#uZ|NZ-ws5;eqVs-PI$B;12KUevD{#W1Eojz(_sBa6B&Nb&Rf~01} z&=s7ZNW;FjXHxf*?$cOo)t@WPCYbW1His#7YW+)&(>q0uaM3Ok7#JQ?T^{7m~`JSnzqdnS7hy7gvg82c^L8s_h2DRpK^UE(P@nHpb#>sLyMYz=&sJIGH$ z5<9`k(m}n~g?h&P{#oye@SXMI+U^0%yi&P}1)3tkaU*NSmb?Viyb)X+v&_J)5Y#d- zQL$)m+qf}V^rHe0_my;m#!I<6f0Jv7A#V0K*T2xu-9}^qQkr>{^YU@4dt|*DA=0RQ zgyB3NXd_wII93tXx5De`kdQhN$GVS44e3($K2{G`u<=S17PqV|&0B%hx<0w_wL1;v zo?i(yN7sq<}q0&<~a5=L1Xo|f0jy7s~f(!{uAEfY` zXwHxwepX`9#DLa)D}%<(hxD?e8i}!I)QXb7h7(*g{oanV(ZnSvV?6L}1^n~pr|JvK z;L&Fh+k~{{gt}f#%7wr>dsGR1!Zd_XmzrLrZ{dX<(LCQtxRrci43)6DODtgo!modK zni7pGSk>!I)LgjKO(u$wFTQ_r-n;tUD^b7u?f==fn+aQQYTwEy`P_56pFfQ%buQNtmSwzEg2Zm*3+trKP_r2ZCsYiSZ?ydo6}cY z-AN^fS?drlw8WlwiZDj%a>}b8$t6H2sVM!~?yI!T0Z6S;#ax*&AZ-keACR}R$7t}TIr8Qj5hd{wj z163tQr_m@nmi@U|g#AqxKGeA48UzOCXtc_&L*f0VtD0KuP2#{}rttn&=tAz;UG zk_%-|?ZCwQ5nx8enF#;OhOf60MJ?M1I)7GK@k*1@I%6+C??yxip~jl1=vdgdZwopc z19%S0DG9ONVAHxFTbj77J2)g~7jRG=;)|l1yS-V@*^Rz?dB)TUtK2JoKiqy?Gko;) z`LSs!luASkmz1%op+Yy!?5f{AabikOtjGC3vkc(7(J)uLG^@RH&EkkApxDxh6zFz@ zb>{`a&`XOJppI7fg4*MuYMAKj(1ht;G>7bq?#B)A9qtp0vf_=iJB`Oo~OuE-ndQTKZQ5~Mvw zK?Z3hUg0HEC&4e8zuH3{$`NChrPA?{g=4p+Q3_SzXpu!P9D zQNbXssz@;3@9MpR10M~qJ7{!=lX5!QPd1?X6z-k99pFMbqpJIiH3#MCMPcIp0ANn^ z#bIii_H_$qT~Y1ETdAuprWi*uZs=)cN_un1Su`ka;mVu{^tt7-Ax#w~5 z6U{_(NWwh!s?6#hVx~H@ZV2Q8Hk&$a?YgeJL4pTw2d-0!bzyI4SzfP4Z5KID0P*wh zzh~rj6&@dRqyDf_J@jqC-aFO*!YbjCadtom^YdgaQVk2x#GNX{vNTQI zu;sb+$e94xAN|nadkOtl^u-qY zHM=fl6Yc%GGG%>3zmJ75Yg4IobiVcisMbh=e}b!J3?YF%}dh@x7!be%YdjAIJf@40e-@8KAX6juTL<>{h+hox5l zsBS#lEBCUt=i0L78n)E@)vl-}CJXjfMamhDdJ7Q>&05myzLAc-$O>7ITJYR5;vsZJ z*HU1adoay?$FEnTb(WtlDnF>a!D0oJ+P+1sSavMx8o(GsG-{p)1YO~&`8CADitLU` ziu14-JJaTzyc0vvjwlbhvTDC-SlxgS*RS@xv2*RJA-sU&;Io{F@8x}sB2iZ_Exprp zI)(4lC5#D)cd@`UVnPtkztmALaZ2S)Kz4>-NP8gJityye!wWxYD`_w+1^ zy|rxeeJa25#TH9@TY8wfXM~RV^ED+4ZiwFv`UKhY`2M`~gnb+YKPm)>CO|}^nK<@E zp5Gh%*;v3(y)J(c49q$%WjJkbA`_^gOsuM)!lXN;Y5}%vZh*VN^|?o(yUXyDANvQ; zEhCnNdyYGGu5|ZiOv!&h)|3ptk(+yF%%#CWx=?T0GeU`@biNONooA_kvkl2?8>8Y} zi&&p8>;kHxJS3o;C0H=S{-q1Y=$>%5*GA{AaPM8=IN>3mOY;Z6u4x7()wI1du&4K~D``w0smj0Z5dwid z7z27vqwe*pe<$?V%1$&NJsXx!97KC`UOVm`84`Nlkw0+YK>qd9P1(4WJQx-@T%;rc0x%Y2x>qf z;;i^^yH?8gyTtA|2Sf@rO)qRO8NH?AqvU*}#Hzt&U*i?;T$uj*Y|b7<9IGUnWzjo=wn4r4yI^Ln9_9l3N@ ziwGlk>WaFMF**oQ^LLz25>#g-c|S!OH0Pj?7nvp_6T-GqS-K7>$>WD#D{ipSA^K2@1ae0!1f@~N%&Cn;H ze7glg#3I*V@g7Vg->nqNjy7MD!=0_!Z_w}ON6xehT94&9HJYad38>wIXs4sHDHQWT zY#X@tS|NPHE)XgVVk0_a!EC}FV`dQ%e@2$RzSiecv&}2fNe7Tskre7@ z6+$4U2})W<_R19>voTxlZbCcyz12`FJ<9kud}Z=Hnv3)r>C79F{SR2|`^vLUxy8rQ zky}&xUz`WhQczCrvWefSiRvDb%w*O@`NShTN48EKKTN+tla}lyRHWOM(JD9h-^k=S^RbO?YBaNb%^La?Cp~p zNebkw$7wSia--g5CrNz+NPXYVwi;AUy=EM&zsZ~d*8*w))aC}hvjDU6d*T+YX};G8 zaaDfa)_tS=2F+ElK+HG4J@}x{K$`GSC?=rij~@;P9af9>LxB+jgUF{C^REo27~GLv zbcq*bjbic^ZZR`ekH*8qoJ?MTwmaj6vr7uc?{Y%W_&=KtB{=iLhlQ}ZE?5!(-;;V& ztP7U>fy8|0>BkJ1grSwS$onR8Jh!$s9F7!%nhloU5?9e-u>_=(aCq9Ev>{w1X%GXp z)Dy3lo%D5$_;Et>-03~SG$y3}JfqPvjsZ(nr7wFMInFbuJ-t>6MrD0_@_5!nroUdI zk(A`$wkno4W}G%x=<9Wn@uqtk1=Vg-Y@ ziE!*Ep6zqpwy>@6)Nf^suc-S<4hdiB!cR`ePo7emnu?Qkbn!zDg=$i8BMCJR0o(f7Ty*PCYZuLdQH?` z;N|c6)%$YV=JG0I+D5n4j_F@VHFHNR;eOcFV#<0T=S+iWIQ$byp8=WIQ5aDleQcU! zauc3mbg`o(7`a*{;6&-@pY<~7dws;A|I1+;&s?nMs#IHG9oDi6BjI;D zzO9hF1_(9aF^Lg9#j!h}h1*E=$f*rwsbTi%stVp}1}f4H%03`-w!WPTr}oR$0ubQ| zu%^mAkj(Ad&+tFiYpPc)Faxz&b`~!*oKo`iG-f;{UwnJ+%4Ort%TSy~Rw@?8Qh-8e z2K0h!M-aoAS-$HWnVNrY{$%;>A6Qu7qBqKu(SRi4fVu=NE@xg z)AjAex%m61=-pE&(&!K*C&<(Osil|u6rpXfvhMi61`_v!=&xgE9Q#*z*tW8o&wGTP@$JpElIEz3O%SdH8O{ygv1UDpCvg+8qUzggHDh?3AHJ+q zQw*V0VM={u0|jj(qOK?**8t-xSchCu9AY4UTe%It&xLw4P0{9*GZ=K2!awq(BY2`Q z4=SkG0a>HEzRN%r23gJ9on9EPy)tc1j1 z3b#zQk(E#n7w6BY61p*tB7+lF5*+H~H2Fj#0Q@|H)Mtj)+P?Y&NQMzWa-en9SA6PR z8W|Xykc~mf&#h?3oBdc=SfEr^*|>4zHeIX}jOV;id&v5%Vg%t2RcezWDXbp*MJfgs zu!XG;$04!c9U`_8K+)9};b4(gGBShyBhT5I`RNkN@zW77t z7RVF2=hatlKDJ(4Tbu002m?IrBqjB5(a!KbM?%h*p`Rx1k)+6W+9+Qst}^8#UqDg2 z4abMN8GZ`W4<66Eu=xt@7>r(&m=>c zJ}GN#rXzy$(dTtlnIVoC;iw6;|AroU7`P}i-2$#ZneD`XX12d@8tHxh1OiRN7X`9d zjX=o2K_M%*T};B3=S zc6F-_nzy0=%K^&OWYQvQliMRe#NLw0C}a;ppZ1GZ@1_XYbJp>opF7~nBvg#_W*-iSnc2BxjVkrGCggAF^kALdmmY;8rod{$=UbjII10x*P zWkIC)(swWmUxFuEvmkKG@9{oEDH|N&$b`QiiFx-rGGLH<;k^ctR(j#Azf?!?VgDgl zv=${|W0?EYM%@^@+YGqpeR>0W^bW!x#dusY?xwfayAWG9kUS!~27yprbyl%_AH8p{ z>fu)9M1uzBvl3_pSauf~)3R-1&~xfp1cF3f+@gUbUB0G&?+{{KpI6CpFXx%7lWvF?-3W0?z!o1cEi`W8}!W9@8mZhYV` zmB(VD2Xo&dQ==Y^=UvP8fnw1};NP5eDbLb19eQ1skm&^$n0&apc5cxP#||S7!V?t^ zCPb+3I$qRx@Od_+w0+Epg5ZhGqdE784?p*TJR`+h+ zcCW2(+Iy;*sJv8oSb^(?NKH4Z(@@g?^CQ!6?_%#z*R?I6Jm1W}O0y^pn(m2qLRSO} ze*=i^Z%fmOC_@U(Ob4;A#*P*KbtkJg;^hMwj>OaGPUaOWWkd*-qN+-v1W-z&%F{_R zE-3|xSF9^(RvQP6w*5D85*1zSm~%N4l@@`-uMt89o3F-m4_Z2xzym^claSCY)AHM1 z4fkPr`e1Rz8ZJmE zY+#Y0sh-PZJG5?wbOdhKQ?_2B9&YBCCF4!PCtteD(1vA5U_T_BajT&k4?Tmy=>?CA zML-Nc($=Q5nyf#^{H(XEAU((w_CaR{c7}e;GZ_yY%1&-4NL?^(+{;jmx^|O6oi5l) zQgV2EDwfKiz>9KEX{Qip_;MV|P%Sy}UsIQ2;z&&d8L@$QcH+g4g#Qc?Qk5xcyj~@! z@nQ$)_WU=-ao-8*8VF8Zjwxg|I%HCz_zpDVq-no#*_oynrO-s!Sn0Z(zAX&5t_E1O z6}Zx&%xTt_N*Hz@i0#u^>maiZLs#MN9jy$1#?S2A^6E0?VQWxA=@E< zjxGzkylheqn^MzO=VR)$u3coy774n1A`AoBCkQhgWi-?=;9n}nxnYs&J@e=G1AGpy zM@vsvHj#m;rxzYeN&>b0FOTfaY)gLEGT4?ygp{}Z;_tRx6n~?crFW=Cw41AOivuU` z&KK{VY5q9l_T(^kMI1N(uG>yOT|;{>jKyqETvm|OcZxXV{2O6iL)%@+J{5h#dlB6Z zdB{3kELzYTt5w1e9E%Jkb(TG6e(y8}^9!|M%7nt*3b~?irTi)wP-APyCT2zN0C{cg zkeObc!doc#D$@qx@+_Klf#_)r47B)imCj>}?bU*SFH2+};%-ko!kmgyvqfD7ukz_F zzoQilLj%iqN*y|P5a%PCR?hrd#V%kl%%5Jcv)9I+_~mpX4Y z5K>AiW;`rxZfk=E{}k0iaA+f27yUU((aKS;awM{wa(3;=ezFz?_1(;!PygK_62;c> z`hVnZ#_NZ`RVXv|;pz;te<#&d3izgGutO4QHtCDGNP1mT7ox=(yzd#Oubw zmQh}Bb+5B))$DRh`lXfCrp!r2{zk9`RNHB10F_H*HTO;&a+%bY9d-a?+HBvs-RqgIn$JV$aUmiYbhu*Ju9Po$eKrFs;?bh+oL6y>N~c1MZ2|P zolo3;gG)d5HJrRD`^y)|9rMkm>--@RS7ggjDF<)wm;9Hv1F1-+>Hf;AQDha3Wq!Z? zh1jQJY=iInft(!a4Ih+tlg=!l{M{pR`S1z@1&!!Lp+fZj5=(e>!%OJ5!O-s$1C%Sp zN>O=v`eZKT)FO^4vk)2ba@tkzZWFWTVC`AH)cNFP4lS)m5xXghd!Fj8_`)0|*BaXh zo_N3SiuQZRs7l8B>Z=^%95&alHq~CynsHi3QrNBvYhH_e#|Q3Z(Mh#4kj)olnEW7Z zwFh2I6JuP$6lF5$pi@LH$1(1|6c#v&8(sYDpfZ#TPexF{#Dj~LuD!%_ZMBF*t4rm?Y40bgHQcE;Fx|*0+vbbOUfHoPW}S3hd@*|7cKaZ55#FU>1S&| zk#@@83cK6MLiqPvWvUVsvLnrxAB(ElooyJkybAU}QR(Sg>QWu6h*5LB1JOg_h>F6O zkqkBSXY`fO5kl?y@{_i{et+0Q5spnDz1Sl~)~uTf z+lkd+*dTPv+#uN?za3CzFw)6ICk4FIQw)yiNMg|al^kuB(<`{-B)+0bOc)&GhAXVy zc3|Xn7mE$}GgRy4YMsK2hJJGDJ663C-xso!w!CWJS72j-Sf(LQHiEYf;pQaB%>DdE z=rM-7XvtAlmfts>+cjWqujEvv*LC=%YfxaR+N*fs=@06+DV7Z1(UAYv*)DLWc5lr4 zFV6@UWbY{!_2clBP5W3OUWpRoG24db-SC6ibUasU^q3(LTdWuxd+C%*tqON!8ioxQ zK&Xws@3BIVS`s5JL8Hb0x;<+0`DXa6*CPUZ0}0stoYSd2(7+&+4=tnO{Shn-a5NhL zqfjv+t<`nfiSn4bDG)>k@u&Y61i=d}I1AYY{dyQ|Dd*R@U1!0jCN6^th1bk?Cg^3U zlVu$v4ypifB=eSyk^N_c=oxEB;r}-kiG1HA0k$)*;Aj zNrfD&8Z-g1V7OsFaT>dcw%;-M|N0PZ zp@w>k@w1hgBht=WtbM)vCe73mmu(fznN3duEE`-m`z&zYWHd0UcSsD|El0XOJ?Z7! z`KDJWVnF0VnFm|l!&h(O57cNSKNU%|)*iZWT#3?-0J=Z3ak5T`UQgfwSH~qAFcf-WIHL*7bWsfav9$UdOO*wM0Wk zMv+kV+TnXURZs75(bc*l?<+_%L(%OuCD0`wU38qGs6Rp=)NE?n1)#vj11q5OtH|2Y za&NBc$xivrH$`;Y>bYhFqAH*6J)o=CtQoI)|0@M}c#;Aj_$4bK1MDKN;whZ$9Gi%g z?&xLWz|ec#R~H?BU@rWFEftVbR}Qzmf9lWMKxCdbGq1Wjfk}@h3r<}?I1B(}odSV7 zz6Wz!j@X$!G`~9I++CI*H^3zMNJ1{kwyDqio~ucrAL)$-`j7fB?b<)U;*-9H(CM5% zvl&8YoK0U+M0!iBsevtbb{#Da_d(pnr%wBqw{`(jOj`M{KHIhJlu0MFdCdp-NWb8! zFUYt)4Xi4cqFL}E%R{~xm6X052<&ax2EyQSqJh~!;f*WVp+}t=v!)X68!>4T50fpo zm%o{V^m4woNTN2V_;G$RMs(Y@kwx8a4joE{EnrKG5HC({b^x?1-hYMr0@6N@Pe?26 z1^ElTC0I1iC2Uibe|1D(@~B|rg3z7`5g#fV5sMbR?f&wcKQXr#8G_r>;;gy^iH&V% zl6^ALt|d`^=PR8?QiF`I|Sr zr3!)4AdUF;P7R&w2~Yo@1Kocixd;}`nd1$Fo(Pg;aF1RE>U|_fFb2*5INTEu_)&pg z*)xce2<@q6p}@quFJHHr+EG{6P|H9mdj9zh+m{l6qo^4>U1dILioZM&TD%5S&LA_S zUVKTSW@W_l^wL~q(rJjOmlJ9jV`|ao7i`8@eFi)e&;i2*<`b0l;aa9Jn8o|i$+G`0 zLKHni_yY|ML)rvrRt}IXCnnu9EZSv-m`2!2<|_cihN}bu^MtRJ+V-`y0&D?tSSBsd z9^f}V8BNWGyx!}TmlY)p%0s`MY+|lc6N;{Qpd0MJ>*eaig0&#XJQgD7Q}g;N31>Gl zj57`VC=#-@CyyWRz7t<-W$sjc1&LHKZ#p0sAajazARuo=rU?IV8-&o~SXZI%PDOCA zCvgO@i)wkwrQs!*cyvI@Ug#$W)MCsx$)f_Vf&eXrmR;y^R4En7{NSl50D3+#OGeIL zygLN+NA2ZD7z-yj+A!wd=&mgLXF}V39}?Q$TS_zGW%^32R0DA_ zcYT!45}rdrn>R;3sx)g2;|cUSp^MB6;*H?sEgx ztkmrWLlXbJ2g=BF3g^k8(fs?N>9A(dZGa#j8!{aZvskSM2B00Zg89-W@=LBZGqV5q z8Fa@+YhmdY`pVugn$kr$KwdOzRnDiLBd&dywtTXIQ6`A9Yzop_14VvC#B|%9qRL<$ zn+${-<=#lris<~o*oTpola?cdkfdr`5t5>P9vnd0>-yn#j!~sHz?z|xp3D^_Jy-C# z5G?IWh_HWV&5sGZcl>8PYo&9#F2F&?5S??z7s%gWU?w=B9h|fJ2R3n{b@R7!%C0PM z05K48OE(DX3s2+R>#wdPaOQvUh7>v$oEi+5s>@g7Jbv;d*?ZpC5va^rXmp1!MlbrO zXD%}!V?GB*+qDU>;tr8-ov~Plgmo&iOeCqD+rCw` z{LnQ7j@pqA6lSb2^b3&u}?Ezaw(2yl>$nI&^=3{982?|s% zJteW&RC_iWj12cvPIxO~W`WiX!vzHGGUNEy{g}Ds)a-0-+DE68*~8^=r9OyKS2HiX+`&J}I||VM~tTJ-VUo{$6+9^5X(Q>NAj= z+MJm{v>DZtMwNviTbS~sFWSe%L55OaA~*?|D25>jius&P&wKILtCD3b2C(zSd*_!) z_hE9*9L5B=ALR)mU98*a`L`f5M0jrg!g<^zvm z8b#%9*u3@?2`hZq`vyn_%~wEIuFOX_LvHjhng$B@kbJ#4x}UI>1tj8YJ$4q*yo2MdkjmrP#xVPWfcaYZXtJ*bM8cDet`4J3Ak2Ko}pUvus> z=V$-Yv-_K{Da)8KfWixm|7}|kTtU>FSXo(Tw6Y+b>PCjpMyuWu{Xl5vlSg8($?Bs} zePBbl8~9HzG=!@ZF~HyX4HE6|AoZ}V$M6`Ddn@%zlKCUHCxNcRkU^JhR+9K*0-D!x zrwGLP0!Q!Q0yvN{fnF~9@Eep=OQ!}(up*$JAY3{7lGXM1L;xwBFOrT7-@3`WFYIfB zv;FPeyr*&MgrWP7=O@?~a!}wdpU+91K{P^mZob=Tzf8&oZanbq^2bYOkT-_PbQ1}4 z2qY6Dg!UXxEeS*qW$9>cxnc>}`+;O+tZ4C@t{D)hOg2sdhHyhf2AVb79 z+Ci{S8j&4``D0|h{28AAjWY=JDpQ$z+P9E+5PW-eHOc1HeBPB4eEoKO9oww)4Je9+ z*-TU}KMu4WQ^gSRxtuAO91?mU!aZp(RWj3rSUbt*v)Y>vZEP}Y8a!>@ zmwDY=ND)Uq62AWbfH^Q~(MYxO`Onrf2DvbhytoUYJM+ZmPU?llmJ$#1sF{W>YreO;mAHX@bDxfW;8b`DoO~Eglqmg>=65VfBW~huTArwk} z3Uxq)M)eH>sLJG>+AkRWH6q0L#8=Sp2ubp#tlHFmLurB1CUTbpvlXMeEVu)3g@Ou`Kb|R=;4KiOLIkYo z2b+pF2)a*r)t?t4`3;rIr%8YODgEZlPw7XR!4#v0lLcnT*}uK#_D9yY>@r_9Fj5c~ zMNY}<01L_me)1hB{w_unuV1}-Tih)8tC2MVg{jwVm9OZB{Z8T#fDnw#!^%(L4Deun zg%NAQ&k`hu26bn+t`)`rxcUAv1Yd+dm{BFbX3$>nQFJuTmiA#CXry)!jjZ<*1N~hR z%nB8I5H7;-g^&~ATlyWG4Z?ZkLKZFIS>z?1L!KF2;~F|={;k(<3)}mO1~_g*`GHuJ zF)cK!l|SlsEQo?2FDSqXyAuqoH%9-AVT2->Zr8ijgkmd`e$kx!nXVOU;Kfu7zlZg%&Ls;dOQvM-8TDdGIRm; zjQo5R=Rs2}+eAcX4U$1AUiA}c&K#WFcq*rB+4aNgk>F%UK&!`|@Dsv?z~D!|&hNbP z+{q}C?^UF938b^ZONJQkXXM59s{u$DLh8#6&giXU%ml6eG+bYigdLT{2a`&-Ug;aP zc$~;1sUsv6l|Xnqq@2F*puQCUQz1z+h$AuClDn7%!Pep}l~f@dBg)pu3?21}U=MIj z-4PTKOhnzp>~dg}S0Np1&8ETZ$<4h70wrbW(O@E`QJx#$72Wo#Z5yna%4HZEj$Zo&b+WJE7v@Ori-Q-`-MfXQt_HQ?vL5O4BIb73m>US~;R{^v< z&G}6Lu+!ihwIX6fXMTb(s48U6C#@sU`e4b}FQ-=FtY-Xkn;+)hTXeI+X*fhr=W7`~ zeo#HKy_V0H5@Pe)Cf<-#=GtQYD7Wo+v#lCTi2Vmw2tnZ7SVQ56-xI;4G_WFvIUT&v z+a|qU7*{fU>E(3yFFn|Ez^O_W|MbU=W%2&|p7iv`fAP)mxDtNVjMTlZJ&_J}K zyVBmQXdmJKL20ybqNWGQgfDK3T;~jIQ3)z3ok(p@tJs1TbbRg09}CpL)wteGa&kT8 zfn|Zcjb!;93A`fX!xgr?0B^WHAV*adL~^3$&)xPlh!W8X*T;k3FJf0Fqy4R*>y^8k zGC>ug^wF+AcCKcFI!w;er+?@9>e2iiC?cgj!PM0Lm>ROR48)=D48qv>bWX~BG!=X> z=`PdoN4^Faq|D7@B^1JCWK>)Y0R`ELauiMQY!`pBOXqg6py_P6vXhgMUKo-yI`Ub^C zFn#8V+Y~h9$o+jyJc#JgAkh*^l1F;Yw^iN#dU97RZBVql`(6;|AEm}gLp|!Ru2rC7 zM$PEfO`czV0Yr8FK+2eY%js`N18`GRwk)IVr1n2j@*g6(MLLIsECVe%3^Y84Y{^Iu z5Y)ij0qI{!A>p9THP-LC{I=(8yDFk2#(|=|@T_XL$uh)l%wXPKN#oKF7B@h^)DEG7 zL0YEs--HXv01|OBF9z@zxwgo~V*AnU@(Nsb*CsBX*KPPm^kTQE=r5O@k%9e4H0w}6 zsj8@cc%#ezK=y8dSZcG@rao-PsX(+xzf%I;mH>kYn-p1!IB^}c1Ci|o z%72wvNzhl?ogc%xk4WN}14`@gOa1zGgwG{|uUb0Q>=XEtOC&AKMwg6GCu&r2!THN? z+=7YZ3XAJ7Edr09hg`->a_-m&xIJ-``aB-8Hh|HX;iJE_GPa@j;idx!)KFQS2qnVA zo9r!03;4A_S(C;mxb}wxZo%Y$Voh|i*2Ew=sx!3Ug9-?Z8e!+%ZKxuD#y0MQml-Xq z7r+ikmHs?@1cGzii01a;*03gsFrNp9V}T1Ax{tS(hC9gg3>iW2NWjhY8^QkkEqZhT zq_a9hIbey7;FR*I#R#IcMb=Ow%UTR-=?GoC>c>KO6|gm5B(R!XzNEe*l9s(!;2IL<}GWYdymap zwbFZ;fB%^<%FEyV!HZ0M`;U3&X(%VWQtzB=MV5uHn-C?*-bb9w85hMs>0ay+=yC{v zAc)x($BU;-2(`U8$EO-XQUFd94S_s0H-Y?UrUH`EJBAiT!(d?Os7Byc#ICe(!%Y zQif}Pw)>voz#F8Dz4r70rf(H1A86L^_M{)(o)LiyjBv`=onNVeN-#G4$*=*ay1D2c z)XSZ885UG9dq5u~pQynq$*UaZXY{i(@qx))Elz>u`3e7Vv#!i|AxpiHhO&VNV8p?bcc49ysFnO_n!ZSN6e1C*O*ml&KxT{Sae)ppW1LvNU zE0IQnr(`tC2$r$2F*Y_f-@cjp5eX`j2@0HwNF7TIo_yUnx-z^H9HxmH{u$q@zdn!+ z&7H_~YmpMj8rLoy&wXKr{q)tv9J0u_`s`nq5^~4&@JYZvZl)y8Hm&46z^h2G@+Lax zx>2q0@lse~1NPaH&N;-x!@{?@r4FZ4qNqqF9ss8!ahvHb1&$_iU|mn$SR|MZ5s-(f*01p#w5x&r zS?I4T8upuFJFK+!S1qmGNPAc1anpdV<=s8%p1CH^XC?>I4W3h7L~hE+Nim?=Gmpan z8{o^Qj)ED1@2~ad&X>yiEPXl!cz!ye)C-9LEbu}Do~e5hp{^4rUUX!G+3`mN8K#Xv z#ZC*?*71YH+2M;^MPJ>>6se45#Jfc_MnzO!7Lg>(zvyY{KOXvoMxmxp>siU#4rv>WBsYHW9*Div2emk z@=if4m24(B2|IM7ASKz`Z*+RklOIZ$AlFqTZMdAi{R}?XL-_i4dutt=c)Ky!VG^NF zivnLWa+bnP9$oxtonMVP!=-knk>dvqmz2|m(u1QDgOt!C zzs6K+@bqoJ-%0H_=`bltcRdIz!E~;#;4F}bVp<~C`~&tClW^Aje_e|ZA1n9>m(CCs zS2lR`hci{(xx_y3@gyh$PBQ4?K6S7z1KS^qV1~KAvl$pEu64P0u;Bm`ee2fsh%z26 zb!aU7^ie-u!f2Ly8a!5vUAq0(V$)M#V}2xSo11|?J}83X?~*-PCTc9MUX8)7d<=j6 zTUb05Gi<*Ev%5aAw%5Qivc0EN$BY8khB+q|bHW!LRHV^iZj_W6WHc#Zpha^GKXZmjNhZSS-gJbSsui5rTG?*T7!=`ks z#E1ugyTEq|o{`!!s8y;Ou`S-$y{}pjH{GJie+Ir;U|r#jQj^aaGF30#H|OgCRr}V> zBXr_#LPDi4i20s-lq&t~YEg$8!C+JIn2)KvYZj|gtgR5~-NMR)tDbH&U-Oq>8o2=^ z3;6NE$hmQMY-O^set#sBi3VcXvsJMJX)Tn*95^GbWbLrO_*MDf$<~g(Vul* z8ZxIEYtu$Nw<-fun9c0~rM5?VPfA?7*@7s-osuhU_A@;!>-DeSRO0!R&{A@_ES2Rl zvFj|Y6JGzPGkY=(p8guA-zuQ1X~4s=rAs@E^&)!MB@p|DKMwu-iTG5!IjH~|z0n2gf{jJLAz=pJgo#)}qRn)VT|HNhevi~b)ReoO)L5*|X0(~# z&r=%Yb6ao}7T<_FIiHzc;hb1IdJYOH`1l=1dAeWsKzal@jQb-b>OJXtjq%T zr^&L~%T2^b=?jFJI$$h(s)l_f!eAsVx0xASz~k~UqgDxv`rV{2H2Quve5XI8_@)1a z9=4$W-TA-^Fzdu}llv8l_K@3l6d{oZm*Qc!wfo9-<;o%($jFK!Ngg{eBhHtIO^D0@ zt+Ga`;c!q#(0eyVkI1_ETFC(yChNaGFC)(OhpSdm7p(eELxs}Rp_w9lMlpgS^X)5b zyg0kDTtPXnKlcD0Q>q3%ujU5wyoG{FUhIe?VBaH+IVE35iVmXtSa4-$Yk>Pt0tK=q zZf)CTAUgUPjO7%2AA1o{ke%^`putQcUGIiH+`77 zUKVST4b=c^9K)ed5hjCoVwJ)JKc?iLk0>*6(u`u#lfo8=Jo1Bt>6@lk0K4{D1^;Nx`>pZp1k<15&uW!ksTgIqcaMq`A5+}swwWg-q`QC7 z_14@f5uaQ8fT9Z}oTgajs~go}p22in$sZqcoxNbwZx~A+PecS+)6Az5E}a@#jB|Rg zs&2m1CdS-Y!2^-($>rO-bIg^_Q_g`Ygp~xw?<>ph6Zx~1Aam6 zuA=bp4|mXeRyUa8eSqnA$k)YV12Gq6Z)>R|_-eJglc?Re5Br(uW(Y_=s5tV9$5^~Z z@pi@~GE=vE6cML1^(G$@FX|Oj_KBLGGXdg|s+^Q0B&+BXM?k|xF}~Fj6KEq3y1KC~ zKa?7vZt!YY%NgBZgpaf_{B;YF{6-LaT!0?3WHX+!C?;_xDx+G~iJB(W>gAWkHyC%{ zm5WDh_Wf}!uJLx>>p78pZjnu+% zf6XOkF@jDZ==>BpRWSBPNp{P@BG}uAK3RV&NCM*39;}5sGCZ34ACfp2bUF>2FDCq9 zwKLTL^rh1-YEh9vS{%f!^z`cG{gb?P-HE3RGM? zW4SRcM8QLB6Ouao*T8?3%=lNGv(k&4f?|$Nv{$7Nak1Ab zbBkrb3iPMRav~IDVNs|$OVVSR@QGC#RjI*q(fMnb?&lmE zF?G5(_rB7U+kU&H2lv!jnG|RAXpzH2lphje^wX>l#SybSAniffgj~(Dx>e`Bd|fHR zLnoBnUlf$g1$@5&=H(=;d@fLd$2_fsqDkLyC1l61`d*(=A`&68`Tn0G#4=)w+jh^c zp(s~sYWlK+PpT=u@ql$iGhybG%^7VDB$M-z!4}8Cah%6_wE`Ir2VL|>Y)>*&?c*Ck z%>yn-HSk6;z~;Fr)B_K`e)3K;Q7!v$K9EdUHLl32V`{K`sQCiaCm2&$nrD}T0|KP| z-JQl^{Zv?+D>ULWwHS@h89u|_A?HC$5-|4*KkGj({{F{`IM@Dk8;QbhpOtqYIrow# zx*jPL^C&v;l@J=e39rVz&GD~2FFY1g-#hzDdhUNn@lT?%V;d>DQ6Oynt)}O@mY=^r zqvoqcLNfEWs7gk@0s9VATR)o)Y6c&geI$ij>uA)?cXO|QFjCS7Yn(q5j$N5l z2i-SBMy#q2=06)j4|8PXl#mh9%iRf@*t=Z+u(z@_|n=sqQH=(90V$|rGZ>G=v z_RO|dTc<+gY>ca6L+Vcoxvj;o6U@%;;a_0Y3v~tzrjMwfROg?a%L>1=uZ&&Oqdbw1 zP$)tO_YA%xE2y4RuRO6 zk-{7wNk+W8f}KJ=f*rRn z#Fy-iNlJFeRMZ#+aJHv}T^yiiaQ?=)?sbZQprEy8Hnn6lX2s^J*&8{x?VUvvrFe~t zwHPZYNLgqg7)wv^#U+lN4aTSB17HU8DX9a84B{hY_^z<0DI?{OXu$FlL74-+LuN2tl55&kgE&3RRPzVD&U&Gw9QdJ{c!9n!#`>< zk8D{wC7OGX;V6H&?yE5(((e%?g2X~prg1vRbX^FP?qR@ZZjid7ot(JBe>w3Db@>Zb z?$oUBCKlf3D;SyTGdGm?3-lz5ytAH_(ky;Hbn8h2ZVGGLKpB+#1~z8;h5`H6yISF9 zE2hytw~dXB9XH^kAE|a!x_!aw3vvI^T1s)5qLy+*mt)PrZtAuAmAaQHBK2J>;)a!W zsC1GZC{XVPz*yJMV-JT}3*L_0THW@UtgNiMv132wkI6#R&otB1hV_aJeZjWsUK)tJ z<-25jgr@aOndqFsD{%xS>y1@vXl$J5ejd)mVb-h`6Npn#7BEm`+z6tt@?Si5*W`e6 zANVq=((GU`hl_Cy2ARh9Hm$NA>Uo*gi&fY*_E>7VrAAN+$ddQH9MDN=t#!Z{j=UxD z&>K_2eT&%hGLaMYS)Lu#up|CEKe0OA`X{_kKfA%jBf~S&GFzn>nOyv!?bhhkQIFla zt}(VIy$bJ{k9RTFi*hvlOR|;1Zs`Z^FOPFvDpOqDcZ~X0Wq+_^k8Lv6@hDHH#GuDf z8b*~+TLMj`k86q_{A8kQnVlbs92Vy}W|T12Nq%D4)F;SQ_I-Yz>G2ispppbGv{R?* z(hpxXj9s)HyzYs2*0i=DOqOU%YylXwSM~T`LWS0&L3@!7DXyd0rSq4o5uB&mQInY+ zt%R`{+8=YH)nY|DnX(w74%5ok`B!crS5vwWTV7tFvBR!I5z|=z>PueS9t7<=dveP^ zVb+syUkvjo$W%7_U>~cDyC#~G2U(ZuvxxP}zg)@*V8DR3j3jc~B^z5$;6s-*eJmr= z1NMhz!1{+oPr}$oIWWPQB(7=+&qFrc24-$q)y43lm3(ryz`K#y!}Cv*`l~)rysU9d zA!rrmG>G^vAZJ?2a&pCO@@&+YAs^xMTJ0s-_TFF8o9lm@o#?}IIj%0r@hpS4FB((l z&Df3v9+?CU+VO*3M~#&8ALUE9t`(sgk2Pp?hdfLgL#=ss{DsaNR{O_&z+&l%(B4#In!Tg^q~}&^gOm@ zmw|>$Li_T!QtIRj_ZX6Rr0~er0()i;@QDpTr!XC zOc9oizJC)1V>wE;u~~h5HZa`uppu^_aNk|LG){BP()M?9uk)m_tHO4yp$TWoq@Q3+ z?BH1Vnmc3yrXvBD`EJkg)s3ZC5CC$se>(2io)v)>;HF*5PHs zYPu?G&=8xL`;whJU%h!J8wEug<+8lYRc*!-vogdEsErI7AYvI&at=F#hggvLG!iZP z^@icUaW><^hx`{$6{s=t!V+?FB#{)4`zw)jL+U1R$LAFd25|J%d5kzFa(oMFkIw||<-VV+9UTH!>v9eg%> zr*{f9je`~XWEY9D6&s6c3%|4PiWM@b9+>uP;t$*Oy}oE|$-b$|xH?HD?fa%c>$whw zsOLNjZ;;nml_R~*3k@5+&eW-v=9@UNQ(S(ZQrQW1u!imv*GZajjp(H_t+ZP%%Hb|g zKW|xQ+NDg^=L+u;ycm{nj@*qPLn+g{x4s8R0AJIW6@0-+OGs z>9Y2yS^RMLE}#9u^|l9lp3~W5F z;n2g)p}zXP%LnwP2lEx@NIP1E*+OhP1AeagFd6z%7gfp6>D8c3ccvs%RCtdWVdLYIE38f*W^@cND1?zRoncNHynMJc zG`L44;VdpjK)`|X{Qn|iXhCFDQ*`l9k#Zk<1R zX#HW$ynD6s-2T+$hd^!x0gHBY5_a`iwFF63jBlN)(X$F`Lc-}$0sCX>$K-6qJ#+le z)RN+AcVf@q5Kt&xI6gRasoa*U+mGjF;J8wKG6a&D(5OPg5?C+3%;C|1J^Dnu%-yN= z3AflziGk4S`)z0#DNtUBI6u-d7Puqu0cI}(#&+Ik-LKX}vCV``|5(PvM{04fMqN8+ zlzr2*k!?6!D(iJ96I{YCw_^+L!%g?Bn0(e;6$*fL_1*Hjh4Ku`=x|&ZsbU7No^f5t zIsH`Bo@qH!8|ik1_~1c1f~hWlIKgUAfx9Hc7F@V~gIWs3R!ynQs=R6x)7ZUp-Pi4R z6$8biM0o_oO#A)}NNxqDgRK3kU$3A{{!ez5simdEx zMg@W4rxCPLM2gY&o@?*8(o1Y4Tz~wm$|x#11Gcff^6iv|o8q2#<7={l4E15>u$&YT zo73-|m&fnw7U%}2zTa+u4+W@rb3AnH@ht^hj{B8!BIAyUN8i+x<5)sh520H9{Q3ro z=) zBLHyi6h4Om8^m`AKkALCKuV*EPSMB&?54;%phsoGM1GY6Or3nC7hs?2C+s_cU@94W z@VFR@tf?vUH$99D&QBUZDp_vxpLy#WionK%7TT}>+h_o3>uFcaleAUoKhxHY4V&8+ z0;)rG9@PI`3ctMjUrJ%ee=UVcmyK&4;Dnv!IzK*jc}!E*2z!_|_Cq*%Lmld&{1qXq zzMxKn3NG{-9Y|HQpGCa*pgLi|hI!;Mem1U6iWa8?E*y^I47b*!_z3EOBDpUO7;ndE zdhYeTI%QCPBjB?R4i#Dk&+?{r7% z74i4sPtN<}(~8|ChhK4Qn^ahJJFaPH;ko&~{viDr_Fg)Z#Snvm0a zaSaAdHmQV6hof$47?0Mch=-Moab;#^R^g|$dT&+L=GvBH<7{^dn@tzJM`G%+&Vtr zeM%<}`Nj}i{C{MR5)x;;sWFO}v&e3){-eMK1UyKqdM5HB#}^sTDi^shZ!ZhF_LpbG>IZo1R^NrEPebFoaDTh&-GAW&_Z!Xew>DcDKAJCF3@ zT-ep`brc}4wjd`3Qx$zu%>*{Hf6ny(9-RvHf5VTSa;}2Ok#o8%h<6rcSPindRdp+c zQfHi+H2)^A9IXyXJ$^wf z*565;l>(If*&WVD>*)X5=qdjjjm@$FZsn6Z?YAl2C;H~MEh^2PT9-xSAPyAR< z!0$-l)b9n#yX5wdeS`tImiKoUW)4W>J0v}$`LE0w)qW4(5 z66t68zfds&7&9?3F&@v%CGLInv_inNo0NjTo2j4or#(?6lz5s6waQt@FLZsR><+s! zQ|7U4RO5kj@wEje4LfEw?lvA-D_+I@(P_!a4uze#*aEw~4USx$lB#p%$3?Z79)V#E z!dIFA4waFSvHP;;+1_jv#c`#1E9*(pOqhq$ADYFC0^+e1;??C?(S9Vce(ql1P z&AP0ZRrI;0!vISSiFfJ9yXFTVU04IUeT1m%NaE=>qoS#qUh9tc<(T96$@mk2Hns;9 zIR&q8NUFxNwr7oaZjN!Sd32-;tdZWh(tq2Wr$dkZx_il%xn{)KguYk>b0Q@a?Vz5D z7G8!U9%XovQ~hSw(O-7%wegLb5E;(H?I)d+r{$T+X|J~$CrNI zuIoz4jra{1k|lj@-qz|_NB`-5G9gA`?BoaB#_y^LmFDmapz5&WqO#}x;FICo480Y} zTNFC)HbN;W2=E@_^t|w7Zs#_vBE951f8JV9p-QIGHfNW?m?R3`2uYOC=Cmnd9;PHJ zjf`lQh;~5rA_ID6Rh!XY`!-<7Oiv#By(jR(Y}r>Qomk`*hcy@`@wW1HAD%dX2EqSy z0M)IG-a*rDiQxV((wa5>CY+sMe)4tI=;AV!D@IJc`i6l2NxhoRO+&iV#>3X|<9Czb zPi>d*qEdK&Q5W4{S@+ePgKE-kE>m?qbAzFn6WbtOJ~l3VVGnw+e2z0}Mn!-T7u#hj ztWqunJ;;YRm0I`i?G5ppRGLXo(7ev32L{Q2`112-uAcdt*>obN#1T{y&E_eBmL6DB z&DS+(S{mMOF9$=BALf3KachHetkI_nE6hVF*3EIY?lUh|XrZ@%x-{&tf%F|gvx_wG zLn0!kFALr4_p=dmzWnvT@kiVTzKnGBUa^Ot#OJ;iRAil|a*w1R-jbiFF{;||FtX!$ z69j$Id$E#fBQ+MC_ZVlk+fu2~{CL7&y}<@ApLc(~<*Q@Hwq^N>pPd%j-H~F+#o_nC zFM!ZPPeFkqD)sY>v3e?|F+L^Lm<923Abycxh&kQ+i%T|xG&3DPQM;+56DQZf2sFOU z(3^>GQY)94N>*b%XNztf=TBC_RP6uf~g3TD1s^y3AbI7B+9c`q?(a zW{Qt7NV?1DM^P3G()|Ic>NjWFKKI}Q`_ zv^Zyd&B%)(QalQQc)~kE_rb$B;825*qi0naquIL)gk)r_CX<)PTqi5U8a~kuXDXDk zW#!$F>NJ0G-J8wYGWL7N6WLlxVa9{`vRc{jCokpg%LpIN5R_)v)*dm9Y0DQt9##IU z>C=!>pFqiDT#f2V?Mr&^S`jX!2or3huJ-GnJcD^r=Y+f#2O)8A|Jl9o7Mq5rm(Gu~ zU2;D(I=t?A_*pl>r6CuerQve4(PbZ*!|rhMi5fcbJ{z9vl@ym;ezbc={7{;WTAGM< zs$FMNBwbz^dV4Ba#D=Da@WS^<3L%j0e_EJV_yrd_IClhPk61_{rA4?fEmKHDp2~TI zDUbeu5!o-b1JZzQ!_cO{kdl(RxtFPku)ppD82s#TcCpFRZT#=scnxSqks0nJQ?Sj% zZhyTW+|7f$BEX|2Ry5Y#*AJ}Dy?)HoP=WwD)t zyFNnlef)&2m0n{!do-CkEDy({6260%l_BPGP3P*TR&Yy2HqO6D0!Uw09R7oC+S~4q z9ucB1gj>Sh4*E5PBsFO-^?S)rB9+AdO@ct5r>K}hx8Da<;tRBeA5xncV(+DYyqOqofQF@cWY6N z9((#qy9p+Z^~k5dnqcn2-_2nM8(X-XFtl|C(!cGwAJ6NFP%DT)g+XeEqjS(CmPuh8 zt7fQhwz!-%V(-g@^s>5(k~M*xoyQ4)XxM-|yxc`{lTR)ceow z^r>`K(zA?!vRXU|URSy8SM)0{)cnF_hK#COzsFN!;?v>Uh&^x^E^8>U8?Xvz%OP}Y zev`NO+;`xqXBm}Hrb-!A@(d%tXa97@r$EjG?z`L7DoLDV=tk9N+TvJydd0k?#40MP zo^ciN-P&-a5N_Y>8(zDFLkZ=CYpthT>Br+$GD_y zdA0Q%@v>mugGsN&^w%JuP!Gx^t{;A<2)D*U>hHPc+j;l4PVg-RuV5@ed!oK1{>8V~ zXGzG(F)6!q$ZcE}dnw6zj5xvdpdRkkb+bM>#asMw=^+6FwB55RM`f+W&#}0CvR6yCI&=*Zz7&d~Vg-ta_eg9?fSm5>M#wDC?9oxu_-f((j_ zIa*~73ZhPH=)7?h(1CXSaah&VbTw^OrSf>?a6g7ulgI1&qbeh(^08vi+52w9cZi?> zT!)rd498(`Ljo_RH_n(^3J-icrW%a#y)rBD;UqibF3bMDZB}t{n(pV9Ikuh;ms=62 zCKNporioSq@`REG*BKS0DQD#)uMpzHjMNJ9f!z?mwMbwmtArS32CF8IVI|+r{(eDt zj5?A#hKTsCR%ov#r^>Pr7~Xo-orHJNHDH2pf({9QnQ3q0i}cE9ycExe&gKJ?_oI49 z$2o?m%NKn-Fs#{EoAMa*pM^@_3LXC$kQtF@!?Yf2##Z+^Hv7y{DcdY@Mhw9H&im_e z-#@SQI46tG4d$>ej@0zMI*+rx6myQ4cws-wuAvSSc%$MM*&G4oJV7cL;Tgo5ZY3x; zc#5$+k_MEj4e2>flgwdm%NZ+4v%EKdQl!^0G_~+GU%}>Ai%al-ZgEki{=$)TE9nWx z3CvZ<_0&*suW47#N1dej9y>82x)~-iqK?ac9SLZk={NA?+8=RDOiWpd^BgK;Gj7H3 zV$~nKb$rHys(HxUaew=Kq7=bM$>`}wv}vw^`)G8RzMr4t_gd_DOnC7eAECwKXZ^9cP1jbVBbudHlVHPGI9hh)OG6yA4Q}q^hh2aivM>XyYmwfZn70hT2aK&nv5Zx98+0rzIiqwMLQ2ZE z(x(6$d03Q*KU_<|yXAh=H?>X*wR)OB{Dr7Ngezkms_n*Pg5ZMgr0~W+%Sw9<@gI$1 z-}cVc`32ym)OD;XhlK2Dekr^skv{voBcE(B)Kt8kdd81k-+^@Znp|viSceGbw0%=% z`48Z0-zpq$YdD-`Wc*z9g?-%Rs1R)rvPhJ_~sV5s4F+A#?yF!l^$|@ffZC9%>wMTgC+vy*l-*RoteN&5 zva8jP)L)&QS>mbo+BioMHr;`tt=&#-TMX7|)|_yCEFrO4K!XRsNWrrf+W>P}q|2<) zBV>eOM|fJBr@5qnP|hIKjP;L?jnX9X9^zFFL3{0zy%bLuMQSDe*?XaSFWh>Xi>+qy zC!`?>A8##*U!4dpk1tcmiN}5wtMQZ%Kly?UCM>@pdOqL(3MZK0kbBX|`@T<5Bo&?k zt&G%XdX7i8Y-smpb8oM81rXj!2f1?2C#8|uN=2ie^L4Fqjf;&=bxQ`PI(L zc?Uh(bVVzL+=Z-M8FL+%Y2S(()3xOMhME1?@wUavY%s*iZM0E%bvDerh+TbUYr-D9 zwLco6OuXdAv^ZQE&JJWU4W%*HThO>+9vMgo`Dx$+wb}7ND#1Hkdh<^5DTbND4dw{? zrJZ~yAS{I7gV!N0)oX5pqSk(TNQ~Gh(_leDgfrilN*A{^vRG-KIIH!&>f6n1aKx5df4_I9C+B4`f1H-= z;_K%sKDEafqb^XrWICt5TV*G?_BU`r>opHfM>(AEhL?al2&9e+yr-=r+D_m#mQB+6 zFwx(ca_jUfAnXY1Yup@|F9q9~cOzPTDi0+KGZRzdlMl|=-?7@He8k<>hstshyV2ZK zrAzO%McI*!${sdb{#K1=u9`q7a&cT5d$0H?e7=%tFOFtP^dekJB={kNM3xB zje2_VEo~1$l{$)dG&$Fnz1oM&MMi=3;h(ie#}j|Xo2(Yc@(UY1H{`&^ePfuCT5G~s>G5UD5B!RmJg{05e?m%f z6D7+U>Ae+El965DLkuN_=08#FRRqPZ@*rtT0oEOu$XY}!SW7Q)#i#toT;rui5{pmg zj(4T+v<2+Ap5n&6!9PBFCTz2-#Nf{M@+gMYwIX7XTls`MK?ZR;=r zEo@uwf48+$vy}T_tpU~pdUu4r2`t||{lAzCZW``d0Kq4f#2BH(va)dT!EWCm#?5KPq zR9=TkZuxUeD+DE>L zSyBS#YuL>maq1I-X*18{UM?jse$dC;&`%{pWFL3DFV&jDTie4~+3L36uZ~?gj)i)3 zL`ZXhB5=DEJ20FS!+6V>Cw(MiG|{dXgilD(F|TVJ*B?YnwqyP%#9JMBW#oHvxZQbouaHx^|{noq~GKG2^K5STFOJ38Dg(oM=jJ$p&ntu4UxYL>_ddb*+WnDV=L;T!<4917c zYd^E(YUT*Ip6EA+bv-`*PQAwFYUtkl>@;_mvRYSRFTU5iy_(hz6?^f`MPjj^96qTR z$VtZuY)7kn{b~Yto4JoZsgQ&(^gI`i-1~vk;OV_}{0PWBj!g)82zYALWED#L3Jspo zX+}kqG=Mm&3;id$?7FvSLs}ITGN~SK2c#5xw`S#&5N>QaI&zWEV=m!Vwa$`u`Qleb zW&`#bj~`6q97|Dtkb#CZT3vlvC|dXfdI71e7jmrKP(CB&EBg5fJHa0YSREK|s2@ySuq_srx(U zo*O^*^X&aYH>@@19AmsSXrnQCfsN(lX07BDH6@%jvhbbaUMvNDe_j);!5x za6a9B&)(p%(sjOzQaxJcQ312Jl-{s+0edGfZ&a-~XK?F!5{G>xMumOAP(m~i|H=#- zB9p!y2LYv5>v2idCG|uL;?|H2?P7EOOCsAUVzt~ z;@@sC3gS^jawC3AdqAZG{$A3bV84`5C_a<(Q{+oLQ#A9t+5^?ALN|hCvV}97wKeXZ zq-JkPJJq$Fb*Z5OO_Q1>6W#>Tc}I#x$`aG-cXVobB|K94f-ZN2C|6Umbcr0R5~kB2 zE#aYo#bUleK|zIWxe2zdq7s9jvV%Y>I8?J4!Ll zr8S!pjTLLlyejb+)RhOhs5wZHt&~)OXuh4Oo2S{yweEV$BQf&DAMg}JLv0pQq;tZn z!OZO8<^YRZ%ZqYO%n>ksir=Oh)diN^)ZZq=+){a6AfOXHF{G7^G+|f)@0@>r6m84g zv>+Xa+Pc3@xx?5L^3nBDhIcdPZ&HA@Hven1(onPnQPC>2%PlXiS)(2S&p&DrSv&wi zn}~>vd5|w-_{3dje{|dpw_b}r%OKs_Z zL-3{E68Y9_v0yrv7I=bhi0d5o`=9jJ77Ub?vzz>ak&0yweEQmadR<>w7^QR^xcDC- zsQlbtxlGvi?@I&6hJhOXJ)_Mj2sqsl4%>#8r2Nu|h`7!W$dFxKd$_3}E}g%g z<}Qj7%UKT7u^!4Fy}&A=WvqIs@ZaGs;8egQC0ch#BGk%af&FDUixC>2+gpvVGT9mx zoopbT!=xy8mW!Ovf(i3R0Te^if;msocrQ5WPw?KlVTDHsH{e^JF zRsn~%Bg~s#F<0br!6W&i7~cScNC0Kx{sL9dc+rf;yz4BX{8`m}sf^TAtw%wg=S`c~ z9|cmbD-D<14K7PS6QXuKY`#(wA$VjQH{Jz3)5WTgJK(c9X^^>ks=3^hg|nu8%~n1+gm2L53{3 zgkLpbRMdM+YDN(J*B4^+t!3)&Jz0UcwBP0`_9Z#sk@48mfyDACg5AI7&LQO(xZeN- z@?CLsGpV_#ve?E}Yy3!)tiRi^)A-aS5co1gnjg|&QJSMlqpA#12B$O@2+}SCe@gF# zajN`)EuxvUoh8y?u{l>N9yA#<<>}LQdTA)e^=yivExgGVo*uDzXjsRjy<32T>%1=k)0&)6WiOr>i(;-!An-TgR%Hd% zJNM2CbX)aqP5)%zecAC)^~v2Q*D0;(rT&T$A-gMee*%A~ssqhjxA1}I2gBRF3=G6a z9w^WO!g@L07G@>EmY^M|#~C7Gyf&LbODub|(fR$V z4;}U^PFiWr+R|u?KEW&N&|*a~1xm0DR)lgSJnmcRZ5WL8&Yd4kCxmO?l2>Nn8%S+m zCD7a9Lo~(p3mh!wr=)k4UM!l~7=N^2><@AKvcfZ6vO9+(8DVss?qghzl*f zaCm*V1P+PSchU2NdfcTt@5aE4UIk|Kda)sb&oE0SRm-273sj1=o})*gmW6s)7?K+s z=`~l2N2F0F+>OQzs!3w#xbm@t_?W!{y^G23YdA{64k?WrvLIaswV1f1Y|3F6S`8yrLP4ux0^rvSh8X|KIw23YM z5BCY9RAe9rs3)3Sg!h~Od6lg;{&73_n9V|WMD=Qisn z#zz=OoBhX~NXZgf^+!$Y9};tdnp3ER0#mb}ewwb2qsq>%!6 z6EW5i0wD<(?yJJ2-Pa2L@4kr*0ApgaBkI(ngErT8)-ZsWIHoVgk6{n)Dg9W%A|`$r z&JV~IF#6z7J;B`tB1el^%8{dr+pOGsh>w9%CWPyRrsC#Gxi-J1(z-FVP)>VXiBl#YHK4?P<-Zx=9??$huy@hYE>oB7ZKMlONa=5^Ziu_vd`@q8C}B4Xe4I6RCVf0 zy9+G^anf3sRi0=j9l{a}v~lx^06`ARD1znV;r1p!e^d)gi|Ja77fL@YRG;9KFyU$m zUR??dN_;9_>@*K4-3B!3Pz=(-mp}BydBag~L?;G7bqW{i)Ty{YKW&ZNOyy;@z^vhM zw4_U?LSF6cxcSfkH&bz#rw+iR(HJR$_Bh=$0s+UpIdsqDdZDQkNi{V>CB0^oHBL7| zg&ME3wDzH^D2=xB7vmpWZkTwJi;|X^NUwO;{m(-V+~HB$h7~u(Jj%sDV_QYUUepX~ z)WO_WlNkuaOU%&dcY9?WI9qMiwm-WkcyYa>OqeYfM>D6#5n~_pr9pwC zvi?BC3X~9311Q&`{~Zlh@PhJg2ABDr&Y@atV%F!#v#{AonZYy8(3H>=HZg9Rcl)UGF4r^wO=K^HrE4NaG#vMECmVS z5YjmJdLb3ie%4+vrgx%3#6H^fj?h+Y20aPvv~vwEQQg~!;9!Ziko<{vk(%q#1IC?a zzGosMJ5)~rBL;U>EiNPk)xAtF$R|_dwKAn(DxX^ypLz$6+8PbOzshY_8zN#kb^0) zrHQ}Z*)`Sudig2=)zSEFAk9B9rZ4pDNlIN?p^G4@CaD6DE9%i+MHtdv_kD)2sX)D3 zOu*8ZN^q@Z9?Xk#Q2e;cxi$Zj-5SQ)nTyOmMw>q=Y*w(8nB;kB63I*VmZBW=2G^uT zm5lQa5B~UxAMf8JXGT1vauGlY?ESDY(SJx*e`z&l6KN^B>{+k!TW_$4WKN=C9oBmG z>xK|lQAqzah%|p@Qg69366W?bT)W<~3%m&I1Bt92pboNb`Zt7-KfsXpg-WZ6_Z0@z*R}KZFIMtQCLrAL)8qTm;gI2M*)S`^jp&IE|7t6zxa3$;xU!L zvnL@#i~i4v&X)|ZFn)JmGiO-{Cq|!mW#ShY*bc$~^uZq~ly3)3d7_ zy;^?A_>7qDLe!}#{xT=jEQcf_%+ zGiu9z;NYHYqY7&2uIC8UMA0m!;q1xi8OVC`sJY#$Z9h@&iwkIh3e6+)A9*u*AOzG6 z`@NY%PvIxub2-jnoNTa=C)_C(9~-Ww(#=hV#sR|HpiP?)i1=z^SCcpq{RFD{q71v_ zd|TVaPbOYOKM)p`j0&>$=u!{An4A2ZFE@dL5nJ%Z&^2y+3rcJX^B+Q+X z?iq$Ib6eaOEq^-`S&{vZRRob$hu5gRv7DRqg@9dsJsVvN35KcsmT4;v`3x<~SHaT_ zOm2Epaj?rvK8-UxDlv%yb{zKG6H+nno?{KkE}_&17rCyFFkdPHj3*^uT?)KDh3o`g z%_6#D5NcCJWA41_wKa8~PW$s^z;81E(jt{;{LlOfAo8zj;LxS1t>rIvK>m*;Ch4NV z*fs~r_sGGJ)sSEtYe|&E`_@P*3BAXM`=hgMMP9ioi)k3Jx4ld%4jZISB7n*l2v!?P zzFlDl2;I`*AqE)iQzh+iU9OHODy$b%Z^_qlWs>kYET*883r5WFuyG{P{x>pts@{nO zB)kODn)Jn@AI#NeF$;4*o+a`DNWxItY@Vsk;w%VR;&}NL=6|#EroE(&FjV~KZ)_d) zyWJd2f#&1ZL~*XG043)e(cvJV6Ti~d?|-Ns&~|IN6#Q*BeE@`Z>UBMiy^R2M)Y@Vw z&cipjBtC+de)v$=5Tvc;!E+Jj7C}yPORP>Z1|)oH4QLvbX2o;?|0I0xjeFaQ?t4ey zoIexfSv3QiJeTe320BtSf&g~Fxf1i3LNV%=1yz5gwHDy=oFE(Hl>b*1`}_Wo@@GVv zQoV8PsF%=QTyJzx+?Zhx+-j zIq7-tZ}k$w8OJSZpx^^#?E5rZRWT@I7k6KsW+3u8Vp=Dp%QQg@9|VVj)^UcoN{5~F zu@3stbn8c!XKV7GrQF~Q$8&AUC@;b8T>d9|ZpNZa-mEBrAw+P!5k%`+Z4sw)EP zZIaWTKfo1{h^c^7aB^|LZ}0|PK3ah|6uW{Z=&WcMHZ$sLkh1`YX}- zFZG7LUU5XxorCltx+(rLMKG|eRk+TlgKyEYcb;O-dht6EAU>sr^?xYkXDG%U$Tz<+ z>}i{L4Z0pA46+BHJk|;%-fQ-H@`w(o(*R%+JaMI-Q(dB>dII+%=2oqN?;D6f;&L6q zu)SUj7Liiw+yf`V`hw_7(xbvZbsA}hYmu#3oMta|gvM!CX+c1@Rg=>9zkY3n|M6>! zpoiwdgP(8a%ZT%zmKUUdK&n!dNn95`4me!hqO_?lw)&hLubmFGwk8LkKmW@4cy(+@ zr(O4IuFgR!6%9yW13(f5!^m0}NUqB8yuUf2#sVWFH{e!<2a>tP60;<2Y+f4m8u7Y( zI^P)wJHZhp#OP(jg#?F2!eR?rJ?l4SQ{W#Gkvy)%%txSen}BKG?|q+>X!q_wUi zGVW$4LS`HL|P@ebN(d3NkOc! z4AVrdC!nB#1Ud@USbKXlflTHVCqJXnbcs5qX&{kJ*;p=ji-dIg= zV(-(Rm;eIepxQVO1a96G-jg|TFb{L!vv#g9%r$%5K98o?ksKDw2L#roV3klvG(Fz- zM=I(yqFbj<>!ZFz<)KVv;;m8Z^g5(zQ6U^3i1IL!27O82giyZ3>#m~td!16xDFmcG z-B9Wa0%3%(3orXVs|_j_p7h3fiA81l zWa@Tjyv%$vPa+|B02&rJU4?7?x(`O2&#_br+ z;-QJNIG)kLb?YK!-KcM>TJ$wIn39q(IVafx2e0jDjN(cU7`~BIlHDZw-C}b!z0tqT zm@6qc&`}f2ei0Iea{k>P(BDlhAL4_+ql0Dfua);b{uX1NZg#5D7A{BKvWW3RnT{%v zP~amY8X3Rr)BvTRJ12TmD&HHU!?{b&Xl6~>DZ*hOh_pNv-vU_JmffD;ua~A^cNGqXV(T23|+NVrm7w5BdEsxHayS2S|+7R{1ZcAqP zSmqdR-&fGRGim3Kot?N##qxFj6S+y3&Uvwfy>@vnlr9 z*@Oi@s-VOg1*A6u?Qf^4 zagI{_sU*UMa^A%_I|6)^0@zq>?r$8#jzN|o6tfL4s&g29>N`4ruyGg?KS0f{lLo>} z1HqQZ@ao|v4>z@1TjS)%t9Amij}wK+;DGvq;@t*@&~&3;SGsAWlhA-X0UCZ%tRrTa zOXd_`8FsgmH?&eQPwIZB|3)};g)y!8K%<8MED8`Vuw9nCtf+r9_;$~1m61^hml)qS z@+3GKA|X|K2O=YNe+?kT(xP0_ej!4(GkGLG!e@Su_P}H7Yk5VVY8pwF1R4C-`jb_Z z%qBC~?<({nOH#xF(DneK%!EH3XEpcPetQ&^n93Ek^thuF@PBQiv~tePE+^}Ot0v4x zR~5z2TfbQ6o^U-}d9>r_sn-{kd#3^e>Q zA-2Zf2Q)4|zV}oM`I79;jVuJOANI=Dk54II1r`L{(95ly`km7FV`X)>mnaxgM|thE zo~;87<}TIjW$9x}>&xw3vk*@bB4dFIe_#(hW;6j{NUHkYtM7+W&dyv$gYn{bUNsI$ z7caez*(BPJra(fE+uqpos}|5<{cq8MS16Zm_Ldl$oYt5ruPx@#VvDZCrd&mj0M*sD zu&vm=e3B#emB-;@PcjXvQh^TfNtxH7VM}x0@uJcDE8UT7S>5MXmk<2;gaGTU|4Nay zA8Qtw5wg^LN-A)rn*Ck&lwwI{LIZXo)^UFa4+-Tgu*Dno=+1=>L@PnLqY|vMV19oW zqhYS-xV|#WF(En?u>!pR2 zeW-s~03ip*)`oYN=F-c4Gj51YycFBD*UvEM?*X$IruZbY?jrtMMxHs0AQ?9~4erLYc4)tPx8XkyWd_ZW; zn~i>Ta;H>2`NJr2S(j#)Q-q6y1?a1ln^1y`qlepH
U-F{oB(q%p+mceAKGw(EWV477x^K=oyg}{%NyW2 z{}SH35=@qNiMb~8Ssxg&&kL)xmPjVt|L=%Ho2X${_Fvq#-3Qr5J3G?$>EP>5Y}n^# z-??ElnOJLx;chUDy^#Oy8hz|i%+Xt{lM3_z1es=*DimYT{J*EF3oM#UB0Y$Jv4{&I zt~6F zp#NYhhvZ|26{3gEmb`?nw9!5*h{*B$r6KFJfbuMh82b)$O(%hKP#P@si;C zaHm;|e62Zbk(3)6Ml`v7d(heF($}6F*%lkBg{&}_2X(LmOgmcirh@<}7fVdhmtWwN zObUk56z_7UjMkE9;()#Gr=FX_KyLye#6nNw9FznrK5jPmR-IEfNZR`}3uO=EseFt! ziQ>f{hK31%Nt5eCjS=#sI7GyjlQkO95xCQ&p_|-E))u(L_$+4tVHiDMci3J8Z{y_) zoMzla5NSUeh?+!Adp;2)wSMn$(BTWxigoCR$j3mBLn}o9mo2aKA!9KHV6`^cd@ivh zChjRQ`th5%{_dz@#5wZgGL?wO{0LYiGz?|xS?FN~s85LsF7P0Wq-(RiqB#O{0OBg@ zJ7Cd8vHQz- z-a0DFF4`6cB&EAMrBfPd1Sv&I8bm-rxlBE(V zvO%-*el$d#aEo;Na_Y&SObz#cm#INmGxeMNw0#Z;OT!Nz=>G(rnHrMoO`Y;}{VEF6 z(xP#Rsqk~EnRUSe-!I$(4r->Tb=@I8b6)o@xY+Yuz*ylt6tc3%Kd_dw`H7s{#~w_e z3#=)5iUHS%YO)!8L3QDCmlxG7Mae0pCR?(Krd%J=8-Y(x%`}-q{vtiuwvkkzav{Ih zlHe}g^Y&yez)z|g^_f)HRLT602dC%v!MRDbF>lIbG2kpU!A(3h;_GyY++ZKKYO$4{ zks~gmnDmSvK?rMfHm!Z84)Zl4EcU2l*7N!=)^f}%j}P*6_67+8xl z@ufa3O7NbO1jAJ+RP?NyptBNz(}PHA;n>{G_i!clmKMt26qf>s8ct5|C)h?Vu8nGhnsV^-V&5-aCl7fe=j63A9i+54R|G6vwtj)F+93ZaR-k_4Es<}%%R@C( zKU#blK^QXIw0ssL-_{Y&sjE{^i&TZ1>#lUW3{?)GI|dr7oGofsTY?i~U+B4P@<7?& zhT6qqVun|&(=28$S=G{E3%_EI3Z7u9i#SXfq*r*4RC8&BmvVLw;Y&^kTCCEs=qVnK zCoeYv2Za z$_5$*K?RrFA73MZWZETQ{2EoeC!-sPKwF!2(o>ed0e9C2A|IK8ikBn>wA(6)kwkn6 znZVUF2J}@4g_!-z^|oKD8jtnEuau7;TCM7WM10Nm?IjS+()xJhgVZF2*8#%Tyv<%5 zn#}son$tX>ZIu4uG}&O8$lO8Lq}zP0Ztk^n#&iQ5XC{@a>@BxXB`UO~4&=;P07LjR zMusJ#<=5T89v}U+`R`MOf(0mb*#$wl-`;bUmb~BQ%m_?OvdylVKR=qO+B56JI;wTV8tg;2hWZO5&1XLTWDkI#WV^p9EusB)*cdcZD^O=SC>&Gqm;Tyke5Q$tpSyrzjA(0B&&^mNUO-Q<7Kdw7>*_r3;BTLXG1krLl)?a+~4 zGZc2X_}g7+-FtGkoyqg1%6Y2u_d)Uel}w_Bcc9E&mMlp~`+zQzRxk6C*fjp>v>!n@ z9iWTdtOBH1rEH9>CwGySMvOoo7!a{_90RMV39ahJR6xc27oX!vv8cs`XSIF7m3A67{uwLfDuUMkd`VNs^(g@0@)VChD(bSpQq zocZAljKlE;A%lx2-?8_OK_c}f7lBq@kX@~-t4iquc*_Ej@>Xa1u}Dcm8A(e3Ab?-p zsqkm3EpVHS12>axXh1(^os%HK5-?ll?{+-$_5eMEoMmGq>qNN^zd<yGZ zL1n={w0LW;bWw{0i!EE4C>uL9l>9XD7` zBhTeYp5PN1)8<7Efo7?NKcPNM=Nq(Dsoq#O-P>-4DG_j#_^dMSn0Z^iyIt1Kzirri zcI%B4SK)uw2`bJE1=7azrX`$IyJ&W_oF#e=0+srN5coF+08|*0vV6#Y*5rpNruFXo z&B4@fRIdAqp_n?ZZWQ=Sm@3}eIU!oj^6ZXAHI6`PcHsp+5LMF>G1VshGq1ui{U|^D zyq;_!nMvQH^``g*%FHJW9~_>UkPQE-a|uIbGxLAT`0-@S)t~D%NFuC`kSRs#zlYGw zX*q5=4VL&(wC9l(9w!PR~Vlcu|kKOnuRwO zXz{&|beCG1->3Si|G!037LCVOcmBVtCSPG?UR~gr83*=`&Rgvc;A4AWCMTzk}y3rZ-*6ydO_f0I4GzD)`tZOv;i z_BIv*oA^}F$OML8-+LZ()C+p1HidqJUr*;SDukPr;-Jgw_Zd7js*UPH2;3@c6Z6;h z7cQmi^>WGf+N0(-MC?aP4PiPNd%USfEJXRez}s^)6r5 z(Z}ZvQ{=WRXaB^&gBH*Xd}Q6#wJbTRE|nxhdLQy5&0g+cVm`=Mj)OzGk}8;F@fYgv zF~t|IJMUv`gI_^H!;V3Mhs6yclztH8zZoR+_>6$Xg6b;|Z`Gci^I2j8FjUH$hUVbz zKX*vu{FJC(TK+0 z;wub%RZ*M*&rRS?V_!-sLR?^L+tf=FCeVcmPSvqGOmyATu3o26v~H?K{yB#+dR2Bq zo-`87$xYm7>JtdxuJk-F8ot%oSz2*}EMktbbX4qKyV{I~4Vqm=hbGJxRaN%L2X5(eOC@x}GFM#eM7S(dE zS#zSzxY*~R>N+pym{t_;(%e(VwDa6*!8$0i zi?b^*URxVzqG_pL{d5=bPLHb)WnVasq@<7?L1G9dW27ZggBV23$XIHI7ozl4&w9Uj8D1&zl zctMu;hD;_2C?Vc7OD!pp#xkV4EW?`;_`(X#L*4K-4LRhfMIj9&atGBOVMuAl4afv} z>xZE3qBDSniLCBC>K77I_1>YAduT!dfEst5C?!e>;P1E=nNL*n3b@~`t|b}l*BS(J zyp|NZ0cm5=*`Y0DEtc&P5*29Lw_Z-XCp#_qLxeM`4YjILhD$u+_mNy&{BtB@IdYSY z@89tqwCwyz8Y?m?pYZ8|K8oE@*DKTBxz{3R*PyaLczo{;ro!zzg(xV?wCp6pk23uv z=)DXfOfo@H@7#0DmP(Q$J&Ws${ z8lX7?hqLi>&zZqK(6*?%iyY{JIz>V+@U)b`Kd?V-Y(4=>Qep($r8&sCnEsJV-*fco za?)X0#&j3sP(~(9kr%bGeLUTuv%UIK1Y_;PxPjyx%%}!N>N70iiU7`^p+4&%OgyO3 z`Pdg1Q6#rLRr?Xnd?;6J&F<6?32PdY2tEoJ2|Z}e{M8V>MI=llzK-o=r9i6M0Zfca z$nhzD1!Nct5dUVaI?0IYwnoczJuRhnUd4X+eCZC@SnzVWk_WWaV0H5yeMeER`r8ZO z+7V>Ey?Yv)q^5+Bs18rws2s@M7ugqj7n)os|4U z>9}Cqg5W-9qhFliA*@!;@3Z?GIaWSSRfKzGCtamDSh=1}@Thd*1vN|i`;S2XUd_P9h zmo(62I%Cr%1ZN)`9)i>E{pB@x1aa@9RS4Km#8mM8;(urtD+d(?p(;2Aly8o>ydnkH z&6_@xo^Al6g}SJ7;wS9EToib14|nDOU9v2T&2aA1P?h%48u7#6_Y)!mY&B_(quA`g zw1NCe;ids}4u$8(-Wl3II%RaBM7+*Diw;wRA4~pN|HH9RNE z0>)mmQ*Ou$cKMu9Lct#7rs`9um>&MQN0=tBAO9utfEvo4??$EkS6L20ckKPO=J(@( zN_2m*8wjZ^dGce2*#Rp$MrMJnqR4+(36DC>&eX2m_q4379yx)WaJ(^7#cwHZ{E1|m zU5MCn+;63aI{^WgBW<Dp#a|{whpUUmx;dtfU zquvb?qBEY7EgzG|ABcB5i9&#erd^*c(6XXE&KQ>#t4Q0#0}h7uLxy7u6N3^kC| zchY6@R4?+pzkguk3&03F%%ubC7qhBVe0?$^WA@u76Em!6uvVB)8LxUHvEn|e5?EI} z?0)Rwuj(CoAkFXnR9f3Emh3MEN z$z0bZxoqQ(e?(G~3pwOBj;gzW=5~$^!b5f|IMTA`B5TuY@JM5&_Lgv1aF4poyr zcSsZak7fEd8Vmx~YsNb?kvrs;{QNHuH3Pjv_Ob-0rL@}vD}g0iPSz}B<0brqNaO)6 z?#IREX=hUSJvh<}{}vqMT})eF4M1ZLYz+Svb-_n=76}wD$CQ7 zhM!yt6Z6)*#ue``YA({(a%}ht;mfU)EQ#M2vB?g#$H2$YjC{Y|$)|wXe%LBUr1MIW z++*d>Jq+^}(vZY{Zk-HbC6Q_t{}JGE{+l_~A*U69(N4!7h6W1W8v^!+WXbX#q7IG| zt0uatsF3VbXTbk#1vEe67KJl57shL4KAS)SEq4u1;2B`K*8afUO}N842{(DEZ3a(s ze@VxTAFl{Keh8So^Gg)tO1S1C7S0UMEls`u8D~HhsVC?SAX@m+vz*SKsU>5 z(t!*P<`Nt#tD%UDrV8V^%c?zkc>Q+?GsK=gW)2(lp6TrxK>^-&PXX2QrB3CIH}7ud zHqb7td<*8wRxd!+ttvRQ)Fgg9a39bOElsqkieGibnQ#8p*=c?s&2jMCe-ZJv8fo@G{B;gvT;e2D#Q87R*AC5cRXMb zVM60=1|=`#Lun#5Wx640{=`kNoEXWU-0^TRc%wGYO);sG@P+xu=N^qwAEtoAd0pcx zG|0`a8p+Lt%*V$@aT-80;f@lX!_}GIEDEai+dYotww>5@eTMCPR-tx3qG3A_JT+NH zxru$>-!}#!GoFX|)BEMH9?L2{tNBw&bV-uK>nJmGVFp^%F%-(uc`=jai9*8@%5cf= z^(!Qo31lQAXe?#5GA|zz*JXV$R2)aS&#nHz4nQJR`arm8`^=+_{dL-Xczr6-UdHGUhe9~UquQ(* z&pFyAeS%wAawD>^+SYYoOP;z_MYry80TTP%Ls{1xgN|r4ik7EdGrB%edO~6XZkX^0 zoxEd;V{Tr6UQ4tSl?wXm$ju6-HGqW!4m1gVJs5B_)9ubVpM)Oh68X(WsCtSDv5vGN z_^WU->38(t;2l4X$9bV%`0p3;gE-uF%X^-Q_#Bs_eE{s&0^WcFKP1e3Aqz_qB_vFB&M+tUqd`wBL|}=&GLlMn3+HeN$qH3ki5bL9 zTA<$XJq9^9IO;(7?1SQnbtTkaScdeonW_4#zuR*kwcx7PmfBg5wy7l#)vqz-hnr}2VK^t+2wpbE4n7=KYEzpNh#egs?ZtghV@&^+EKqK~A3?zv0Rgupg`VUK4 z1Ao?%v4V7o2Dmvg4_h00@8l4 z*R945Di@hV$u&>5Iu57!Shq`|d_Q1q=lOcIT*etSf=pSebZ}FP#8Eue=0Ddhl!8dClU|Bg>qb%9Vswc#lu&)^ zIWH&NPs0SukY*#9SJC>#h8&qf?C@wNFpfy>lBuZg1PN_AGuied7{2pBUD?A1^#09e zWN$d7@l*!g-h&Fkl(~7Fw0oIQ*G9q@MFknr9^f--GjX)@0Uw{wABE$z^kZ2mg{x;{ zxFOF=WDe1cW0O8JoGqr{yAO-mQ(Yfx!RkkCoQRFC%-1Genn)V>ojv_LN0+IFgmUuT z?FuF6Q!wcZedo2=rD!~A+l0F?Qbm&j^lxx|JfG)u3;Gzy($%iiACzSW9r^B0rWW8A zRLN-o=SP^HMRw$MU0K1`0L6LlVHLwPXxd**zkl5~-kLv3XoNhPgU63luYBI5?L(u5 z=s10SH1#a6NjVM0QxBk&m4)4*LtbID!LN-QCR(GPC#OJ*&j z^x-2PXAT#1E$gPUX(FlSQI&LVE6#QiKc#SA;g|Abh|;d5>w57V`BL~0XdFN&CP}2X zJ%2QkUQb`F)T)-Ca?OWVeYt7*!c4xoBIVu+?L%7=Kju|^Q8w@kKeJ?g_J9s@Hvp7@ zP#;}C#@ufhhc4KmF9*l22#p~vjS0K@hGNyc)-R(KQe^}WP#NH+oo1(}G2XWudx2z+ zwJx|R7tL@jjf`OU*N67s3~^H{XS=`jk862n3GzBFSne) z$j9{OWb*g&oXLTMcZpL2uybb=n0u10B(#;?C;TnWzD%R1YBmY0;tMLU>I2b0^=gS$ zDRB_XE=GReZYiM=w~)Pes3(n74fAOL*?rx7KbZSJp5udwWd!d?)Luzi6tzC+e8E>; zJVJSf1(s1XA7V@!_YporTJU69g{p7->j??SNK-?UadjA^sVz1mM-g9Np`qV(09J-62o$>k4>M`oEt3_2FXOXN3d#CNIvN!<{0zZ-$fNOByIzSbv#B)ikl%?=-d^Farj`ZwrJ?Q%6Be zlUqRDH#L5GLt)8zdQe@)qGok9&0(h7NfM<>xA4X=n^?!}A!w63ecf54w-k`jayBjr zcDFHpeC#K#Lb@+?1v?}UEMFxDdfQGv>iahrW1*v_kK4r4HJY^>78E1N5)~OQJ|t#* zm0+5nu33rsW~1YuoBWZZ-k%{5%!%FV|BY|BR(JxaXPwpPZX`EqJMEG zWyh9{Q`xZ(x7&vcrQ!Wj8%E$O?X|9_%bd>-x`hMuAE~*kI~#5t?6g3GfxcEO-xG6+ zoIysUE95sIv6X_B5eB97pL)tD(GJJ) zO;ag%wc%LofAl?=kWAx#b#D+6kJCj z`anb&pUZZA9zG{4XoVwQ^;>~0+I7ok;@WE z>b2A};AE297#`>BFni(!_}j#AZppQ-Zi5u@(8mvPJKIs0s;6L{{X zFJ>#MYaW53qw=Bbk5g_UgA5P^XN`^be*O9(j9K>Jbrt%CX1Q^au@pRhXCT!%qK0^9 zJ9bwfx_eI`O+K=Q$dtRx(^Esa-6pcG$SJJncxT?e2)ky#T)X4=gySAG#HD;46ss#e zNUex-VvOVOYjn-(#&^V!@$}dYLajgy4PdrSi||4mO>h+LV-{mA3}=4CDvm+I=!@>O z(lKZ=6=5`*Enlm?RMC%0qr_~F9a!QTTzUO5%}=}_u%U7)sOQDZ=jv<4$5!vtMk$J> z_uueti%3teg}PaM;s~{|JmbJ}COObZtRQsd+*AB2VL0nBBzp0j)pz2|m2=YjVEVg| zqa3ew?InYsn0)e+X?B+m*^Ec#W~|oNjgwRKJg&O1Emi8$+GCW(c8{c2G_l+ z=;Bn;gdw-6dGe;CChFeD(UYI|r(cnCp3P^oWms~LCZvS8TkynLe05`eWv`EAyVESu z@uE^F>fXb}m$GjFYT znugd0xKNvFS+%~e+!8v7H+tIZn)Gmf@OwJl?0KVc=`*+muvh(`Z?jJb9fmaA`RR(Ou{7B23aY`)@EC{14U{72p|Co zRCyE@zo>ex-majzH9Qi09gXp8YFSJv2!)aFgGKrQaKJ6E@RdQAnBX+M&P-F=dHqW( z1lh%Wxie(H6-hy29{)ABm4w_X3OYNYa@pk;`EvwmZCjlxS;m!uS5^3|LQme!)5f1? z;XQ~ULYDs^eiWFeZ(@5dUSMRYKMM?n?a`It;QLmUmZfaS)dOk-4Yh|O+T$e^Ql7$q z_xQwl-Zihm9A<(C42Xy*>((A%N6a7MMmr;t!`s`Sy-a!W;i`DRfNaUCmCc_S5ak2ECc#Qf@G{KEPc%dia zF-6!rx}GxH;;%+*1U4)`T_2e-sw4)+RhHd5%zj1!r+_L{ZjOg*J& z6M?pZVND?H3ZE9-3_GWBDHF~11xL^80`S}~NMvlETta&V?DH@hILNtKzv|ltnpaWcj&)~Fz61DY zqtXU=OPi}f@m2!4KUyB|%=Igr`({`qy~Kc%-8qcFSYRLL4j1Zy(_r;L=gwUIMvIr1 z)h_)DWx>sX-J&5g5T#2gTQ3(WRBTGVQYr{z&HcD0leQUs^V;pTh(OJc^!IKHO|G$% zuF0e^7Dp6oZ~K@Yu{M@;yAoRUa8cnCG=CE^+-{QB5&Q8HPs2&x>q%$q@19G(8{4!Kf7W%$#j2uLb z9)!4QT2;zfMEHytl;_mk8?4v!7(Z^u=yG_4GG4VTR!m-q-?{2KEG?~jD?f@zf6wxS zmRyXW?YOA#79rq!%e!~PCGP%J?v54GXF-T(;XH%+E@Ba3toRquY=`c_cE|ze{`8vB z*>_E6wz4jauZw!UU$CcZi@Eo;yD)8QR~iQX0;2=_$C z^*j_Y)fC}LE%|uBl-W^07b!w#RN*)Bq`Zd{uT-^mUxuggv#my{+!`@%PnioPBf08o zn|naze$=ZA~~= z?fr^pKfai$7}QDH-bdwY*Cj|RYsEU7Po+HIBcPYyBB)^PUz3>MVl9wZ{{ZwL^h}g8 zf(J;_eOb;2^ErSnPt)(-b)em-hL0^oXc1pTg0xDn5&eXH8@pk`pN*=cU1=^zX1t>; zfHinQx-R=p{|A$GmE~*E7LeMq^z&FEG3t?hXJRZm>*^Hj4l)X}s9?gy^`yZ))6slb zU(UZ=v1Hk+Hgjh>-jc63R^~@>ZKEGl^2YH1`Q->O*4p$IhKy|;Y)K5>gkW1QI&>u|OdEP`yM-a4VQeH~2Oc7sK-ViyNFOjGG!NUt0 zd^Yvi@{H#t+?P^Di>?bmUrRto6fZzK5iW#<)vGf`H93IOWI}bxK<5^Z(a?YSW6@l4 zz5lQxJ?L=}WxpV@_NTU<$|Q-Oy! zgci*@+P(UH!>e3n;Drta$ zSUqkXm+aTEzixEr)OT;eqg|I`S`GK0fj!FgJ?Th}3Nw{aGoNMTlMO->HtlY$;h4^~ zj~K}=)^vWnt7;TPeYZKbihe2VAG9O7EC*2~v2hO=nn&8H=(e4{y!zIEP~>6#owoS^ znkQb~vY}92)y%J*&1KP$E<-{uH`Y_UU8h*n2M*<{QIpu?AvMsl^n9xW4kZ|>bM^T} z^M`ix`{6=HnmTIiJLgQ_xXMZddEyaX#yZ;~tO_{2gqp(4-4?n1r6V0zk6>)5#@5oj zzf>rY+4WMlyGp1j6=XTEk(shBITQ{Y0m1`E`xOxbnT+ES(;|vltgHa#gD&8`{vd6S zVql1GIfRFpnL#mmvBFl2GwghC2DRstkBxu0P5d<}c1VnfUyccvgh0ExfCjgJ)-6LC z0$ux+je+B3WB~e-M>yeN;`uUYx^@`TFEOE@_K>8LmX5{b3n|V}kmbNGF*PIS@ocYu z`|&`EDx(%N=wj;WTf#^L)C+4cN*O!yS&0pN@kqAthi6B-Uxx0>ElDb-aCcb=Cu$j^ zEO~G`|4Ng8wxoDV67glCmCV#+i3t(KtSUzfOu(3{F4C!a`>{Z;-VlB+SN|X zgwpfO+s0`O>rHWapbf)>jFx-IImrRhG<@C)ds>&qCzTAb?iq3wyP|prr}tQc+h$9w z!47L-S*!YQKM!X>hFOL8hdMjtSIMu<27?5aSe0=GkuYoY>=`)bxWnQe_M)900K;OV zB@Zlz8_r0YI2(=Xo}*IF?r!0PS9L;(y8b*d2mJz-H__H$?ql4{x&S3k722wjnM|Wi z^%q=9&(oNh&rW!Ky4t&n#shMdJ-q!gq%l6J5Br7XD@Ls4TAm4+B{6b&T-vy52b;Vw zH7-<%*SM3NK_aFjziu5|R;;m#zo-zK$Py{ZX(_hra)DXFU7dT{>U1+pWG~Sz*$fY9 zBUE$&LpFnMH}@#qf~#AdJiX0jM)*Uh7sDvd$XUoD4Tfa*>a>~ezQTc znkZD_gWXSS-zwZi4O?Mqq8Q&v-)yihCnW8Ua2-;S6)^Urtu8R26 zJ)McM$EvaoTPdQrQM5e{FdHQ+>Q?f*sLS?5AC$`0EX}?CYNH z3x7f6RY8k}MJ7g+9;&FB8mD-{0F#s-cjvonA#WNP2lqqvbuqeeBrp~#;=WoV1{BS! zj6&D=_tQm@D0|x1BmliPSlZR!j=VdZ%Ou$#n!%3z^x4K~5`R$A8k>(pM`@}`EHxX0 z&ePh#g59`9DoG{oqQY2%!=(JhI>HATfw!sXA3g%R`+8jw!yC{us&^wERNgCni`_kY z*&*`r3s%vs*h%l|wRz%EKn)dbh}rj0UpO zGVbxb5w>?T?|JMA;8D$}>sv2~Hf6=Vgu5GND;r>{ut{dzUMLerP?ub*K%JU`2qwx}*N4TAKmMR6-wCGh zWzR1QqC~^|{u2Lyr3FZEdO3=8gp_y=p`RRM6BOf8HpF5~T!-{@bK&}yX`4tWhvdEC zE2zCO9f(dOjVKguatU*Lz6F?Q%ZeZAIBJO;>@J%*wne|ENEd$Po#wWzMS(Xs=(kU! z$bs9#99witdthQqxvrPvH~J_%@#WqztNoW(ZI_R|;i0e0c@!HAnnw7KmG6%G7?UX5 zec$#}cwGtYu7Wsed#9ZG@Y{tJtyf1uP#rdr(R+UD^f#wHn&#O2!!Y@YO)X1q^Bb~!rj!+KYv0|wSDydDSLV=^Rr^64_?`9fy#G0j>zC{qx|LeKHph)8^l1X#YXhqE$`k~K~ z8yw*|eeZg!P*zVlX_S`(oDpZMHjA%)5QaFzZnwY42yVs#RH!Kf^_hId9VWG`ztQLJqY%xjkeN#9~I$!!t5 zFsJsJBgcNP`A01~!tbVm>-VP1xp+m#PcEh~@# zyS%lXTE?+F6+*xw(eJw0ciNi{fyOMq#d7>bwjsOSjGXd&8L51T-~|kD?%_~U`-G!7 z?9cwt^C*&qx`&~_K|sB!%N~cy2Mg}`ET&IfP)4RD=97E%`0`cCFR2X5q=!p%_O){j7!2s9-Fk;~Jf6VE9;KKhhxX4K!s6tsz^Q)ikZgo> zo8MrnZ9Dxr^f-MQ&qKf*be4;i+pj4+bDbu>hoAz%4`Zv=vUzoh3LRxSI`4ovp*o+s zsz`ES2DA_JU#Jd&>u4CbP?Q=0A!iwQoZ6zN8FC@6>w}>wSd%uP=ud8X-Sti160Efg z{Jmtszn82KGU$Muu?8?j0C~Y=xaI~aji=a9k= zJw7T}dG8bYSokQqXi7LRvAC&7C4ehxyX<7^V-Q8*@1P@&49mP9)Y{jfk|h^KpcS*~ z2wnY`pj2$CDXP~f5zVK7#Kf8?l1)8eZDDbkaA0>OkAcMviq_;<9pLADGX7L+F_v2z zy8wJS3f$8!{`$qe;kW%;7u)>i^ai45!FVe5aKO$}ewMDxy;H7o3KHv3NdK=sj+8J_ zqSMfy#P*`{F%pgxh?!JAzpx<#A^33f{8BI{&A2<9xIM9=<~>2WrphANH#kHGW=q>o z*I%^-*!~?hV?fN3G^qa|zPG8?c{7ZDa<~2)$v@vd*@@4A1nDgYz|8_f(Ik-n_~biP%j^>^e}+EJNmz3&MaJ)dd_a`X@!G|HC!|Tbiv&d#=B#TNRW} zieVGzLK>>Kxv6|_)?bXiS!njO!rBaj!gV3laG^R2m*@5AeahLNHE>8AaLD6DU(tIp ztmFP%DKIFVbghEO;8BXYArmFV?Ptd+yn?NBgJi#juq zizIshq&3n8LWXBoz-&263(PI^Z0V}=Q4dp9^aKq!7K#=?U+$7XB7go@ELX*Yn^Ld> zibsbTm6SZInmG*-Fbnng8R(NT!Zp0_wfc}b%WpB7ExvPW8ucGm3l;9}=7JEiJxrYd z+pmsbW`|}85m)=OCrvzQKPCpjIW#N0eB#BMy2)EP=%ZNRcynXL`!+&@;0f=p&raso z`B36fz*@1Hc{BdGRKTAh$Tk@5QA2MkF#>;k0l=DOyq zCD*Up6HQ#jInjZ-n0gj&hUDK@r??bYE)u3j88}DD%qL0g#sG4|FPH4({#q_oQP@F=%ak@3obHq4q?l4rAMEp6yOKM0UfA#E zi@2X#UI6g_Ju=g=by*wYQ~jN5=fQ|^$0M$zKQC|}b})&F(?tQ!QQ8oka^su?4Q7t? z1o@sC4G()L1P7~lhu$1%Yr7wpbztC5>e@#<^wNr)2crwSfUOKGwGfoo`qUW$6S|Z1tby zGs0U=q7NRm;{J)7AO_Xb*(~}Xg#x#uzN)Y1pk*;>N)by}KYuMv>2>+>=I9OtB0IKC zBO}wDiIO;~%-@GUSP91db=mxb1mEl)P&|eKuS^5|;LaHAt5Oz~{&{(6coI(x zNhIstsdD3krVE0yYL+=rAX5YH46>R7wgcCwWWrj|B<%vzBjK;NY%Z%sp`e!M3iS5% z>$8tz6yjX}#EwM|lpPq9aYmnyQbD0#289NaT(&i?q7M;4#*(@0MMrakd3(|F3m=%< zq}761QY^)oE^2`#zYIMo-#uZk%T5LfoVTMV(-vf`Ki@OA14L=+nNQ~cuv^)3!T{|h zybzdaH6~Z346&v(HZ4;@z}4ymT9-|VoP;EhpM`>kYuPQC%0H1mRB91)tO$!fH`j|V zEP_~1fXZ4XZp%$b%Xl`ycn$nsbi3{VNohwZcUbYC)fR|7wI13b=rE^xpM zT)<_TJi}lG^g+!FE`IQIK5lV!+;bwae{b((+Tl%mED5^uB99iYJLdR{qY_f#PHSMx z^GPZ7;eMq1<%Fo?_VXGQF}KsrP=u5NFXN zBnkPmJkF2qdmcVaddF&`$;|d0amm4G#lSW%6YfGIWxLqKW zv58%8(sG&h<7vX;&)fqGIa#Gx1_YL407otc0H28PcO*{ zc;Xb{=%Q(0cclTNoG=W~T-&Uq08@#2a(f??<1C11M2q_`VvSGH-v17|zpEBI32)C& zx_I|+!L=1l3kO}L4Nr8~`IXHDmdE|CRfnRzAuJoK!*VLW zlj2X%VCw`+#AaxY(!f;o4&)Mcgy)Q*(dkfr{r9_^(AO~`GWfzvqM(uRK87xDe+3i1qV-GsZP((2LYZS;NPhIS4Q*q zF8`nYDvkwK7%67S^Y1wF=K=qx{^S$~p%sye8~MNCwUBAVyY7CMH~)Uf|Mg@4fBOz1 zva0O@j5Ol!{Num)auU#0D;;?8&jb7?MEyViM=K0idQ$IRs{iMo{Co5K`-4=$px*QA zJrw)vqW+(r@&ES0|DV4}LeS9zQ<3Aj3X@syOAPe$#JsR&^E8?}-zu~5bp z7lAVu9;L^jkCt5@eU}1D$#Z*w)5mahSQ#*;IvyGxvEVW%hV8$4dApekFOV0nOinfF zP|N-N&vw72Zgurpthfe+ik@gPa6r>!t>~p`!_2)z9bgoj1^K5$BhWRL4ob!`C@s6M z#_>VtR%vQ8V2s_SdT;v^!1GUUR) zfc9w_qKC+pw3z^hUc!-I)zptfDLpGN`=M#=-WmC55QrRS|6Rm_+X zSk?*yM4XLT#9ES;GKNOrX``_cF!U!?y^U%;X!*sCh)KZ;Br?Xp3-%qnoi-56=@KKD zdI>eaU!QK#LZe%_+|PH1{dYkF$@p}05QF9_2I%SiYzo^)Y53E9!6R z-kk5vwUzJJWV(Z$)+l@gaxl4a;AD*?+I>T(stM1;vr|}`tc8{sxl3=jt_wXNnc~1g6 z_#nrxnxYmex|Ts-Lmo*S|6_aEG%zE59)KzKf&)JEVT&rqY>F~Y?>(Q&b%f*Snrw^K zS;Y9G7J!gx{t#wujC%A_tGIglwNcdgDJj^))wV0$b@h*l+$GIm5fL2sfBii8HH$#_ z?LFVv-B))+6g*{H$|CX1+SqXb61`dmQm0_Fb@18o0C;8vJ%Y0SJPjFFYb}Y%k{4At z06E0Gu3P>I81+iRp_QS80ghIJXq*KVF(72$%Ih%#@=-Ci~TvMU}P<;!qdnHEN|<^ zr(~e)mUwgmE^u*j9C6`WaIR!?Z-L1f4rt|RuiCpT0>6#Vtw^|4n2}vCrhEeYHnoti zR3pK;Sg*B7=EVsH4~7sn_>q}HX0XWyP>;!5mUj`b%bg;1K#oEHIWmem1SlQ=JQW8X zMy?r)Z_v^WXTmdWkXA0HSY?5u@Dk5Vu&mL(>uojAx7Z0bJyZcYNnWV=l`3yI`v8k+ z`Bw6NG7%jLUZVGws}GHj?^3_l-F2N1=Tr(~ZwVMzR!SY4xezyNcGa2Z48c@jh9B?f zz0*>lhyB*OaVc&G7oaxV^Ew*r*x4$Ws6A2&uu4-Lzn(o|qX)1$+l0Qm)uiznXlGE! z6D8VbxULgB)AxH`%rWW%{8#)DD8ujpYtFC8#ue1kh+<~jK`6Kk!2oaSG4<}|{tDn8 zr)40TL3O7)MZfE`M&?(8n+e3wjMs z_~y0S2Svh%ihFZ^LC)zF;Uxiq zU6$qOcD)9>dl0DCKQ!QG400-j5wPh#(JYIU2B9%J+}q+uce!rGai%qnIUZd6NyM|! zuVycx;EwU5(XI(ZgibIQFiknP2n}CI@G|m=CK+4=XNzjusIl^6Dss5o4TMqooI6#s zugTPgclu`875ltP+$eEd=>DDWY4tmHNh?p>?w5ltn)HPrW3#=9PfE8mhVC)=~Qc=AXdGZY=dL7 zvBGqsgOY@_L~&_uUSU}uy@Dns`nm?*ZluOE43@1X`U=c|5}HS8G$ zk+1&@B#(nk9AN{bi5A*RYe4%YXlMagurdrgnK(Pt8)KPgUj-P)gRU{$53I*R9MMY5 zd!2B^KR}PcU==d9_ih|)FZWnhYX(sn0z6kwU}(2MN!)ZGW}VEa>eA&&MauIkyn#;+ zOKgmi$ME|mKulEx>w%4Bo8HF?0M{iQ{1EU7SO`Ig@19M z*@>ep#~55|o*_h>hTgU7@8A4c#7r&t30-@oh3zw7?4i{Nm=fhY^&rg%gE}iWBCIzi zB9d}&G9Yeg`#*b0L6QPvs!yBL+|rWZxU)AH*7O z%TwvV$!{Q3C|OmPyUr+eVyp~qKH3bA%8n6ARPiVTv*B<6gF64PY*-St?v9xVHG9C@ z`-?)^;oy(vkE@yEKLz_w>CN0}v(U(7?m}zR*uo+LGw3+YV4lT1OhDBVAP|a!Ke%>u zBFGQ!c*@QOh2<1mT9;#NeM)U|8kzKVtq$IeXS4~&K7 zV}cf2?a*~)o2a9J2v<;eCb1Nt@4a7tfoMSAL-H=b;LLpvq+b-ofnTV;w6|mx6Kw@y zj$``BrW)jD#2P8cx3a_Hp1?WTN59=|ayu)Gj)*P+pojwhGkS-_C^*{gEs$ewv2d96 z_!Eg;1wVj8H?sW&z&(ON|NJqE%Q?{NAi7NbwY4d%OiQH!?$cvGa3eU@e5A5z z5`2UzjhUn2_y=m8P;T4cv^Q%-IY8kWiwZY?Ed8l$l&IlMK%u7YG3{?e2*XF`ZJo^N z!NLOJB|ze7K`z3waF)FK^n%huR*ZEu!%Hlmg z&oW(4x&G$qz$VRYcCKAk zDoS-{;4c#l^w%m)3Dbfg(VDKa->6?Z193;)g+$DwRsi&{nD>upMB&gs*0zA|NC-&x z-#mb(@$3_5PgB16AB??aR1{FRKdeY82qFT~Qi61Mh)9QYr+`QdNHe6g($d}CIW#EE zkkZluLrQ}*{LlE@=ic>xd%x>q*38-G?ES0V09anv5P+AJ{K4OFowt|1m|gD&ef`@M zUdWr6${rGHOl-z%bZQe{8(`Gx&{@#;UIXTR0%Y4{v~kh(i}*Ei!$Y?7&eZ$m@4iqa zs}GhmF@wrl1b72^jQY+u?6bDDLz|m&)2$w^9NyP%(0&9N*tgf; zFqJ(Y0V8{s=IQ*5ZgStj7CIg7J4 z3;u5wz@mrUpNqp7B|%wzfNbi@BwDlsy0NYuYF7@3;&P5U%`)AGy>FV&EDWb#fsHFh zg=+H(ybr&)Z?h#O6EVE6lDN~f$lXIWxe)|s3e|&0Un4I-$1uU_;t6`QE_70xCt@!| zlfpi~8oz-)?o|mphU_dqaoiWsgvbv+($0}}W!Bg1;Sd1n)pEQO=Dmdsc)5Xh+4$zgU3&cnx zcr{>U!vQL~KRM=OAM%H;dLq6$h=6^aY7t34z_qpJ`yFiOTDE@R0(uT%+HlJkzVmpQ zgM$X&*u`m;2|&wK{OQo-0qVB>pHwsN4Zx0{o-%R3JOML`J^BxR5k!fF2r{;4?W0Eb zWxJ1f5N=h(fiImfFUFg!F5c(z6_Yo1s+wFu*m2J!N08c-ynUFZX51E7ASx}C-jN9B zm>?EnmKOi{cmv!y5xc>->lv>_hS}CK7}aD@+)+-aj8c=z(JX+$M*nStZO706B0S;vb<_GsiUD6 zG4d;-WX*;JuUGHo2$;=sQelCFx`m#s&_2_VbP;PMTtddaPoIRrJ7$H$WF7fjxxe;o zcaCpo(~+1mX{}mYjyb_IUY?}r60}zGV9a5vSNxchO0FWuWihNkBN&u)-xqL z4FgH>!6^#|(G8$pNb?7l0TlK6b9MM>W+C$X&F17ts?fbq9Zq3J^Dvpzz5T1IpPDED z&a}>Y_|8m~=>o^k>`+$SWDZlQKMFozq;IQ=z1GBd5U2g*$+R<2T0GtX;T#b=fg_02 z*R9}dCS!_`M)5`6E^6|DU^?g}3)8t_o2@iZ4EcJ2&x)YN4 zhg?bkV_@Lw+ys+XA0Ifvo(b>p)U@nOE%1+pnxG5F1&3d{2ZX=9X2n#7A+nz*V)}#I zpW46wmAKM<=o zXo5c$t2<5sced&kJN*j(Tr6P!J6oxK<$oUF_%VnVyhiv+MK2j;dTEr4B&RF(NsdDquj5-7lgClU~bnFjKiRO+f*- z)ICw8wqOCSe#Q?&$#i5pw^i4(Zku3C!bj>UKEZ@bzTASz_}MTaBf%L7!cXByXYAie zpDo&6Xs&kyQy*it0Jt;)$A6SH^oCx(H~{~#`6dEL*Ndos3~MH@fJm7C9Suh7wm>^Y zWaM*G_sK==m0?g4%~KgLhH>2&cBk_tdy+XgAWp@t{nW{swc_7z;)zWIO&H06=l)z+ASA%t& zq*eZri#+tIgCmL{{yJpoFIdg=o54bBK89%c{k`9R+^N}UB0-NCbq>-j-U6DDIs@_G zRQ&by{>aYt8?PtkjI!U~Wg1?|H~sZJM)8T;2N)97U+@+fnY4pv_&9XQA-b2 z%dAyNoX-;20M-RS5WTkHxrcBJzg6J=ERmT7b2iH&7>7bU!W}?>E3@UlYkybWZ@KdhJwIZQ}=?0OpBCV2i!j;52oPXFztb#*)9cwcQB8#fguczX>Oh zL?kcB)!DUc(eXq%K_O=~MCp7w~w^b^iVh+yFYsv2stpT8mTZa z8OQsRa+=9ph&B}Z0AaoJMI%4`aL4CIcOb~wfXt3Zj0+sc=<&YkH6Ot0(^=F9hMd4# zgqS{9KN=6Bw{Z5@0*+iJlR4rE@iY%``^h6S7xoH5JS;iwH5KCX-(?u$Wt_cVY=Q1^ ziC-LuRu~FRmB<(3LY;=?eCor#XA&lSXk7Evn{}4x6_%aL!eT4)Q0mp3lM?Vxs?BB= z2t}0}&VAeei-oR^|4C)I!#f^{^zYxqYKO#b&rQ48#BZX39XxME0G}v-;0P4H!;Vy$ zrGLvMfPqVnZo%Ca|6wHEAdYgO^9#{dq(hY0aDCt_Almrcs@| z0h>lmbNzYeqA5bWZa|F|cLP&Q?e^OUPgV8`Yc-wLz z<7ouDb*NguE%xRc>3Xevx?8Q-jtud7}pd*_nWG|{+8~A4r6S#2 zT)S;zN#QUrRjVl9$5w7Ch4IeH!KNu3GejtIl_7WXZ>p;1bxph&tPy=uLhTG>>H1vt zsH#eLRVAmw`CsjD4i?+4XF=E}g9a|mO}(kYp-+=2p$g47Q#+|eYxqtIId{n}I;aS` zey^DQr2gAzGo1lfi_JOX^c<3e^;B1F!Lh<%lf%i&l$_lCm!QE=feNF2NZU7Gg%VvN?4Ezv3Yy}N7LA$(W1mfbz(O!uR>x$raqB^T&jw{e? zCjfO}IMX;v1Fly z1U|~&`$9cHJ1%FuP;Ss1QmywfD^PA22cK@DwU@%9lpAQc#=xAizdrfPUOdHfT;brS z-E6I89++=N4ajN$9}%Y)4+l>4~Im7QDQvzMJq2u2GsY(-FrG_7fiI3Blxt)Hr3BK6Vd%= z3^YrIx=4M$L$rB4*fQ1BPUY4s7hIuG^`$f(O@47LnOWZHq$+V4?7(Fd5E_q7O{1K5 zG97!0Z$BxBort_(>La=jL%l<5xT5ta)5yc-U2egc6R}zsP%9*~70U_)ykqw<$d)g= zJ^URWI&2ytCrR_95Pl)2O=SlcaT#^qmk4L&*~zQt6F@fBXltkm*3^YG!GJF+0ptok zJUayQQ2W}S>t#6YMce8HKThbYs0t7!R)|U*>3~*26TweCDs*Xx1l=!Mqs%YCID&fl zP0aJ;r?jmAT=2`;BYQ!i$Uf;ek;<1nuWtZk=zJwzI0Ckmsq|NQ8rg14+&f@h0%0zB z;7Bx`5?IFVb6%zwszsPnP0If{2G?OTSv)qrG&A;>9rmxNFFziqv3SVeo%f2(-@X8I zBAdNI`XYYPhrUXH2{a$c>~8q002oVtm^WmP|MisVRs}_*PD=@t*;Gdp8kNM!A{z<2 zoQ7J>eP_3Z2J_drMKT<38?7ViSZrvMg3l(`_`%v>VrWr!9!I;s6TeXUnSBr{5YgzYQg$DNguVCZv$rb!QQcZ@H)?XeNc_Bu*Z4EQSlG)oz}805Ze zwCvAr*L!rQgJCzU^dRwTxwB* z3TTUNtqINXVu|q26k9Af$z`wPM{_CQ z&3ZAgN-X^W`&z!P*2Ov*zEGmg-!Vz$R}_9=+Mya>+zT*)1tMW5>%&$cUjhoQPrsIs z1HkrVNkWPB=JB&tsaZtKhdJ^%5jBuZm=1(7t%C+EY%Qk65-s9QmQ7c~<&wp&>Aked z+TC8-Y3gy12{;TW>Uz5H?>HFbitICM5{O&@My;7URHBUA&BU|2Er9AkhG9~|_{#Oq z3`azLpGD;xOu0POA99K*XJm-*AMwNDc>47n1Ef-;bqPtV5w-hNk6S&?aF|OnYgcdk zzR`H~b9JLqeyUV02kOf5_ZrHAnG6L@M&1{BPMcZ5R>u$MTrJ6o<^EQ(SA&5ew$kzK zsE0_`FA?f;J)fIXuR~i9O=#bGl`bKT3U_ok^fkL@{uAi18w0xOjwm9OD}})RbkE<* zK}8Pz59(`of(V!IY?G~oI#$?EQ_4k)PwDn90K!hhB&*EnKj?Q)kXN9428{q+9 zlXM{t5`wQo4v}&xTT4HVfFAl4l4TK%x6%uArO^KrtAN0RNS6(p>AV+T1z-l=D<;qh z_7}%sq2VPN)!wKK|B@@aZaKlod%C~JQAy$eyzdQ{1L?ArL}*J*K%XOwRMCseBnW>f=c zeo#gF68Q9n`+INNKu>eA5`7yFmTo*9D|alUmnL_PZDxNbT0!yi9j4mi5*Sz*FY}6x zPCZi~uvgN!fj0oaK+D%U4PHoM-_i`91EN+>lN@p!$LAw|+V5^{DKW8l5pS?~SdPfB zo`ME_|EmJinlosbQLbD$eYWu&@SV$bt)Fu$>GjqwPPQaetwHoYz@2CXX7`LX${+)} zi&=lh+(E@Mx*Dm^&AWeto{l2lc`gAZmAT#0-eIfxy3~~9rNf?Z`>@M&xZ{aMD-Kr0 zA%&^8oalx~dEJz*l#4)d0rWW=Jyh}6OW9cd^efUp56xY-a=adgdup`5K6r?7m3iCh zR`(r;#Il9Hz=9`zwU0(Of?qse#ut9CI#r-@E3PA75?&Z=|15DPT}>`j?C68q?aX?} z_1q_RvvqR(e(MF-h(hjJ=FZyf4~BE-F%Y+@V=T*|m`v|{9gH35F!%OCTzkr&_Iw>e z&Q?oG#Wo8W>#^IXVLWWGnfJcCFq`(Q3Oq#Rz1Icu_e6L%g`^s-Hqf#=)O`4Ka3??2 z2(~Z#VR@RPilEg}js1u+nBV{5h9kE+*}x5_AA$)fEh=nN6(6Pn_N48|j{=|YLlKTU z-4S`HBUp$hqOmU=DVtdta#dMMtZt7ZdcK_Kwj!3VD;3mzU=;^l=~cgpDOM#B?Qt_0 zji*`;p0u)`2WWxO4^mC57=BDC?ma^)?9e0pmkJC&P((Ga-0JsVuU6>c1*20JioRV4 zj(azUBRp!NJRag?0HBn;M-~TQ0|$|Qx6mmu6bI|v&?4k3HvJ!(7h}kr>)4pj8oyM8 zD3{5DI3 z3=CDTVh4d8B{D@iH6I8@e&D*qe2KQ>g^O^mz7qaSkh9p0Yy5Sx<@~!Jq-Zoke6P5r zbpzaLUW!Fr2Jw~NIK<8UFH;)gnSlX7gXQm}g+eQLhzu&g#xMX-%i92e)Ba$S%PRnt#K)#`oy>0OY3ImtiaHE6@?zu0* zkvX(jbB=PBN75ALB5++$jf?|)JF9*?0+oDx6IqYq7wpibHIJ9$bV+?MSTYT^{0b?5n+F?+^2$2aW4{-= z-)e#*l#J1ho7T^+HPG@}+wG{HdpiqTXn+PA0sZySyJUrn(Lx~Dw}Lj zhSWvwg3CbdPK4zqs7xEcXSG$&0E^g!^!zdCwBtY{*!J|_ObKbqrzd3$(yBemNIqcTK4D zLtY5=tt(ze(cvR<-S=+{8;didi-eS9`CRi94xJfHEogqbE*FtEIHXc#5qW zA{xWia{4jb8uu)Y9hs$mJK%)dTk)H+LU&WVz@}1F)X%WI)-XUjFfu+k#JNKX#GTBfHcK{G< z+od13R*l5HSw8H1b3^;E8X6U4q1}ZdGa%2(13hdXIp+73HYE-c`;KZM^tLf%<~Tw8 zz|VKOCV>QuV@3Ho1rw~DM-<~x(~JC{!q3PqiO^JIdK-Uq0!#t{-v!G z=;0AKJxI~d63YE8N~}Zm;BG%FolG`jvsV;|{3c`*J<$R^4sk+fv3--cjUvE9c6tq3 zbyjiFfs<;K+78!tT6U78#h;zOG%t6LJfo0EGv zy}~CI=7u))5S@OZHNMcZFPs&(=7xrI9a7OyoiMfP=RyN6Q(ZqNt|RzZ znnbAx#Y%8i9scl_{7o|cD79Nq(!S-gtlEhsSTPNHUvGxF+w6|R)OME%&UB`2Z5*c% z#yOioiw{3X6y*tv?Bu?g`6gHJZ~UuSi8h$Ue`J7u0I`LAA^9*?~DrV52VRiEYFF;-buXk~NNy_x17=tv3X zE>~=~kTD&*jkZ#L>K;1Utv+XaN{0TqlgtS%lPmM`+Xgb=E>TNMOj@*rBB$**2rXGt z1Ur-~e91V@umIzQbBOV0t(z%aiNu8_XQm(guxXimIN^p0g8Eo(pe6idN94n#i^po* zwB;5R{2)v3ek%&w4$`b9nl17Mqk9L#9P`1rQ$OZ2ReMVRh!;#5%EQrwGBH_KbM^Hx zVz%=Q?LM`hK^m{O2k>c2J2d5`S~L5ytsZ;-GEX>tHPhRDkM337PpR1~8-gzNg3oMd zueDQ^JJcHc9Mk+fwE1;z(DOONkDljd267{HPKb@FLR7hzD#Oq4wQu4rW2JNN&lUeD zdX%Ibh2^g0A5ggmEnN;>^}adPf_N$`MRdya0IRUpx#n7@oO7ba2kpo>;fYlM&awDU zQYgmBQ#9o1@Ksw0)dt3bj0{F;Pvgj(+-*f1=AzE0Rl571ya6n%!NBEl^P5D--0h~@ zV3e*?zGmK>Oo#lZlJWF?HhhNI;5sTKfHs2L@p;9HhjpiV;&FWD7s}}^^mAO{gV(vP zeS8ZpdB`a0!lleM?AYNv|GX!bSrpF1LbQF}zd@-Nu+Z0D*c%WYk;u(2R{!|yHkinf z)sOUd)m`{&T=L`Gh}$jno&u2xeDUWr7TKM_~un0x)FRzEIpOc;VB;OXxr%cez-*>V9`~7K!h*)rJ4= zpOL6!|-1Zzca;pMoLZ`)8ZGw4+RVsCq1BQ^Yy3Yv#h2%K``@t z1?gnzxf|;gh#uh)j=9B-NcP~d-f{(BQbI7S@|&6A)atW32%mt9W~buneAhM3dW$oJ zoiJ4{*t@Q{7Pmwihz|3wxLoSTP9aEWa7*NG)kgqBmFI65-cP9BwId#v0dHa$LfMB@ z7M3THVOhe$P8F(5l*^RN9CcBAnivZ((6Y7F3wgaMdKMoZBG~K~;jo$_Z5ttBI&w;X$c*xQjAYX$GnH&jp9cI-dT-_kzwRfT1I;9z9G zDdB$7SI~=M7cZeB1PH%N2|PbL-lpmBnc#C#TY{QjR+k2I4%bEiETs}AoEdX}0ktUM z{vOgR_^x0b%rg(Uvw=mdSzGf=F)nflMv90Dq*wvTHRUX9u^Leih)z8ioZLWK;#6iY zv|SD*P<_7luVJha^7-nQqlUCu|O-o z8Te+7HuTAzL*y!pvFJ`-1ZyaA8X@zNM@XwpJ)&5IxF8wFPc9;;Y*th)1@*W~-%`{XM~C$W}=Ns*pG7Fp~`CQE1L zDCKmX#-9Bu-znKF(YCLEp-cF^+n?!O&b)93usov8NurUE@>@2U3&{a& zw~q_v5wUu3pVFVBpIEvy9>)WmYXm`e%DI%r>phd!7Hx z{OoN5^S6DArm+A0W>(x8D0O+wO*+!ZJuCdy>$y1jn}vgQN@#Du{o6I_)CT4;&*AlE ztLCsDldwUB8XNN&*wr-e9E^F5oPW*}jC{A6#j(4#t^F;+wSnQ($2MVVW_mUbf0#QJ zjz7Nlwg4K!G=sJ&OsyyAr1W(Xer0Eg*ld!;sFK~ zj`=b|`cuf)YNdiK=_;e!BWNOdlg$gr#C1*W&lj1(`-8syh#)m24`3iResB0X|J9B= z!>ZyA5;r|xXUy6UULgDwP#VZF0Cc><><@uD=_ruOoEy#rPhy}1DahqsiW0boAphcT zRZsJSG?TO|&_2Qi$r$keTnYkX)k02-7LD&4#ywn0(Rr<(Yx3bs3PO{Dn?SQ{CwQLR zDS?4M_hO($T4!}arzg(4>WHNeT6T$q0mLKG7b$tH)CJNwmB>1ueamEcEWfpV%-^fY zWP+m)VjcF5gVCNtoy7gAO7*cE&ns~NBzuDCF?cgXi_biN5dVfnM@)O229M3P0#Y%sJ#cAsH-;fJ=8b2v$0yz7~2GMPm z+v8I-y_Mf2@bg6o+E1K*-&PVI5ZpG&fHHD;TX;L+SZl{n&p>3Rf`0q0jXjv<)F~8a zH9my|X4{LqMqjv;dl|REyM*ugd8+DrM@S)B)Oka04UV2-3N>-ASM5#tEVF549Sheu zs}XVA?rq%=j0FP4=z%Q(j--9XyR~0<4q}qiC#<*n>=$7QiO|5om~e9P5;0gEUtz~2 zR{k1&po*SQU+jF@zl|A^#WKJ(T)+ePYE){yk7kxVA@Y z{r_YHpSy?b!%(~XrAf>`HrvkHl8)JSs+8xi6WNT!ykN7O5D&o8LKahV_^oxaxvy}N zlTM38{-u0hy^;@xOkU})ps-;tZSRn~Jnuo8m3NdeleQ7I3@Q1%d70y8bCS~XL+Zr#h zX3?QGa)bJ(TAfl*IsFqvM)dp$m0&a zm-Vq&?Hq-M;h9t;5I?Bvy#v17zYG<7`!fP>G=#gF9e^M*_>xD1mAA2onA?D_+E!?y zB^OGO`(j(;!(;gef`x*M=WdZWnmkJNEF`5?z0OTeiB|Y4`BYnu#i=p7>@J+Io1gY> zG3T0U1@c1@hN`s>OgnF`AhXmgQh_uyTuEt0+Jd-|UGUK;|TQk&~PBSF2Lc3ZU@Zn&QlK+U$q&;8)gd*Zzpt zi>B!J;`SSjbkD==0714SkI1I$V&Cl6RZ3{9W(zS32tII+^i4EF1r7=8b52My+}4*Q5Fc zsn5B)_OU|(dBO+M#JN;HHV%^MM=MTp_73`J2wk;dC*t+2|CY+%m+M4F?C<0S5Z|IVCGAmt{yj(A2)>O7D#M`+ClM;zupTo9Iv=^gq z|85mzSZxsJc|v|>OFnEC=&~1=qB8l&IY`F>UK!VT_b)R=8WzUy3goXr2zCQHnM(YD zBk-LB5mVTRpgUslMpi^BhRW4+-sokYP zIHahW6OW3KbC?O?gD1WB+l91P2KZfq*+Hu<5=XYo7AXe`rdizrRA1?`6PyRr6pTG# zCy|mYg655&AC-$7=P=l_KeD?_Lh7DV?iw`VKlhc1 zc4dM<#Ku>J;@xUwWI`RZ9hmbUjNbyiUIYBFj~)h(zKH9!LLUTc2!eJfIR&kJHAcTb zfp1rjeD2m0N1!8IwNDeaVtWBy!^2*&NI4ceEfIu$nD==9Luy-^LV>MCUSU7xmMf6v zw4MLJ5r$uzT-~K^TsV z`!Vu)v`YTUpF;{Wg>>?C;Ai{kUSeL3>KgPp9Ms)=5QCHOJTtFOR}XyFa88>O*nLE_3uy#@G*G0Or_+WDVv2q`6|}G_ zxl8__bFD%B;m+bPJ!zAzsBv$mDyPCA*q-jsGCAVFKo$O(!AlUxn?Z#<3(jqquz2e* zL4F9~E1v^*b)CrqXpyqSX?*J&DEo$XU@O5cd{v184tygs$w(KjH=alQwjZknMmwt; zQJ^p43{j7G3GaT%M-bBAWKy!Ld>pe51uco!X&FGKs&ZoG;ER{Y>o%D6%b4V6NJ-1@ zKI>XGm0`0+>F&6mec&`?WXc-wTzy_^5X<@)Q{t3q#H5u9j>uxJda9eRLy@ISxmI~f zoQ~w#30fRC*eoV@NK-`nyX_` z)5^*T4(f_}sUVZScEA8*f_+%zO#k%T_hT?yF7{$PI&ZUovW_}^rfDbub=n1PN0qgetr8mbb-9s*&-d9 z&qkI@rNA*j8a#A{dV`lceN_*B2alMdR#C90$kQ8d3QVP$UW+|L!Yw7d12eF}L9hzq ztm1eSmPfU(90dIzO8J)!C=F8ab%aSO%9L#HFeVL?1+p*_TfAQB=ZJlYWZ<9Hk%@+O zb$F&7KKo-xf(Q}C^}kkwiR71|i{IcxY}7W?^6hvr2P^RS*{ET$gj6ib9mhEFh7kmgJ8 zs3%hEAy-T=q1etJf{B-jS;BsQq2IcGW4JD>FfCq(y=^}{@!^hV2;@X`uR&+`bpWi! zSFw@m7y&bgG?gI?>5qEI{@Hiocc=c?ehcr8iFTX9HM)Pe;I9x5WE8H33aZY$4-n8w zbjqrV!shl$PoFr4k|l$Vh4*tr)1w=Fo%~TQ08ULyTpahVp*|{T5CfZp9q6LtWGPIX zCv*MLP>ZmG_>sGMV}8&-!kU8?${E_n#+U~*#aZ?lffFXDyrRAUBx{;KNcG0LoWbhq z(gUV}hmWnJI0-d_2I54Hpk!x2X|DIej)U` zdgIK>`cQO#6P|jyC5(LWI@dY8pQg61I=!F9rR>Y9F`7}m5NgMdM{uhJpgmic$LjD? zeVs~fqY&>=XZh|@SX60ejcV2(#OTtv9B6-eIiNy8l|fa|sc#5(h*9J{%o8gVT?Ei4 z!lLSlvV{g~geqo3DzF^cYCmVk(792&*kYD`pZ)mI_o|1~j%@ZFCP`xVLYrcKcWsN5 zPa)9JnwMAO{AdypP3?7d)7-6re=-DddD`No$eb!}V-$=Bya9o?u2B+XoT8^+y5BbR zycvIiaAg*qP%P}#8T36dW-sb_HDa>cPU4C*qjY}NeI82AV3xXl2vjr`FWt{{CQNv+ zQ&(@!0b%hz*!)=Jp>9_I8l4Kx={=uGPCIYT1-D_sqa#x5=rM96 zni|dL=RS2l4yT(mBVNz5owXuu35?bxjd1(V(}4=*(80h<-~RuWN%FrkVa-4qmSK=y z91;~txm~5a!}OZyay3jjW#@K$=o1j}KFaWE7?MuNRjl*95es%`nG5Otru#mvrj|xq}l->>0 ztPDln&dQ&H5!1T2`ATUd#Nn+nOdP0W;T03iDKWjEs) zcUqA{RkxzKX|?T)mC(dB7&lI+L~Mwj@`yAYNg3FIf4A109YizI8d|d^^xPuhX$*pO zJO+VREdy#)v)LokHji0!R;sZj76AnAyQ`nkOo4TCbF8@O&+g7U(OIL`M9zW zXK(olqVm%9D_#6SH{HPFcFEH;05f4^+yM$9C^)v{iH-YBdXz$B-(IovqU;=~0hcT!~x&N4(dTwD*CDAl!s(bx5dO z#RnJ#=NR^ruG52ECqEF+#U~{_tA6@bNy>tQ^_y(v0*s10A3@$R*a@Z#q81*v);{5s z|EZAl^2Y)C!|sDgPJgboEX|LXCa!S^Gqx|^Heg7g&#Vg#0{yU#?i~{NOXPFqdq?Wn zko8sWw=|qrL_K2TI^=5dUwyA|B;)ytV|sr+sa8vuAOvp%xN4`*-r zEv17mF@prXMl!H1z`BOV;od;QsM}uoF$l*sc^JGR%j<~u|Ei(|Dz`&|rEhkl9;wYi zh8P`YtNf0ld#8L*+`S9b+p`A%ejp-F?RSU_8$5Ur<*yx} zOcTo>qKJ6}780V+!pV48dU>;T+-2c8AV9O`d zlA*m!e42%KF>g~dA;}Y zty>p^E&pk%qR|!w@iqjJO%;Fakg%!h)W9>kS9dM7c(9&-)zCrd+@A}oL8FZnDu~n) zwN*u>W_W{@8)3AQU=;s77gYwMDttZmu@g@#`XW$xC?>Jgf047WR{*SenC6C4LvG&a ztXsBP$X?2GLYhpYe%DpnO@l?vQB3&8&jjIgH^bI76jBJ5t$p@OX7eIQVmI|yQ_#%#l5-EE5F(ZQ5$+vjY9FvLumQX2EIphkE9mz10*EGI8)3s_W znY}9S)mMY-bn6a7JR%4rRUUO}-!PUfQqS|pf19lReGQggJyci%R;7jBR!!v`X6}Ce zN>#Q#PS$5pV6IPl`mE9Tr(!Q!Jbm#m@z*_}Ijn;~0dD9;Tfgz~IoKbJKkr|6Wkv8) zf%8YmNoMV2Q{ijEo|CQdI+MHOj5{R4rA%P}A`5E0zfFO6MP=vRBK9oKBf{EomzUec zEVhQc`P)H5u+Ef%+H&W){2zYSsSzs57FbxbD-2>D$IoD`1S=s9`EU`&bZ&1-08GLb zX28cP2|-4)jt3osOo5=Fn1bqwJTy*yXMn5P31k2h0_;+)8%69DO$VVE(&~<=T!CL! z_J{CuQZ(=N)fP$HWax%+(N!q+XC`r3T%zmu(Ys81S#@e=l~GbL8B7zZuLbfdtBVd@ zPxTDKscUonBg~zL_GHhnoV-2*l#w^$?1*bqzOSLR*XQnP39->aUx=Z+eQ^18*5xND zPC0@$nvYrGw%Zx%_ReW)H3BRZ9%V+|MGfjsU|1DyXlW^d!I3jV6VA+@(VXp^`eObX zxcLW=_|hHch^Ag)8!10q@!)v)<596Hj&%i2m%s5^v(qovUHx2Mpxl{82)yLPs*KLn z+oYW2vkibgC!jD-k;_qEabvx3D`x<`P*^tne|sT$F}|J0+l6(mG#d5HbZ`i1wVsm z%4v_Wpvm;06>2qCwsXvf0qQVTGWjyl3R-ZQPPfHYv~`>=cJTc{+kWN8rTwNI*8QA4 z9v9z)V^aV;Wd0?F;xk8kZqymznv0GJZo{Ab$#KGP+S2jZt*b-un{=QqtnRhaBGFi0 zx{U|FKYGsq5lLN`%7>Hl`3ZXK+)NL{;EyA7 z?bb*5CzyN;@5ISlD{Iw{Q;tPdXQi~F-t?!9ag)vT_h~UP!Al%8Abtt z4}*2dN)_D3{j3?xGb5w~)DRQ|ky}eY`$1P!$Gbn-=bkHHZ+c&oSh`U-F0VMYVhTeW z;+w){q4W^iVYeJ4UbtPk`fVF`V@6PMBXblNYtQXNL7K~$NOtRVqoaip8Ri3LXOK*! zE4Hnuz56#GPY8qdS9otg3l2%{`?+B&{n0h6R8fMpdRtYLmMLJR3NeUwvUCU5Ax|5+ z9YEBJZ{&Hh$;hHkJzU|p^U?0{0F7$zp`kbWh&(`jPele>flol!X9iZGTfj;fZ&~P4 zUm@H=j|_>8e0W57QV!@{z1i+IB)|HQQvYb>=%Cy$Fr0JPADnxN|D5}l7R5fH03f*r z3KVrlzrmWHgzigN`a~M*2+P!Hvp=OwHMM(5$x8xY2dn1|!wHs2Ajj}jH#>~_>f5$G z7&NkU?7-n?y9bb@ub+fB?%Z^MiW7PKL%{KGx4t&A$lu^P9vx9+3?)$^blZ`faaL`^ zOmPqmtGrlJ=y%U$w!GNL3V$tX6P9ztUJy%k%kV($h#pt}TZVAsi%$0EL_o}E*b{A4 zjQuUMWkQh@BBAb^Xwzw~FAI=X?sxC2apZXUr&d#oc5h3zs%7-rm09dkQ@lr_|0c9- zsa5z~Zqb+BZ+t;R1oB0kH+q|#wzAQ4MBZJdO}mIpe1MM*A>qa50e28hiqd|8F8P=S zx~-olYzH8nLW(!hM^y_h7fxH_{yoh8eX8q6X zF=xmY8S=a-1=#i(uVQXnr5WynwDTHnw}a&|02xpY%+|89o-Q`nuht&Ec~zJ`51L|< z1`%HVm9qw<{Ny+th!|=;!J9MN(Erd=sXm0y%KE+K7D*_N6Bcjf%v;9vZR(&S%TeC7 z89{nQuqMgo)3F1G4xddO`J!;1bGSPKf@SXo(oEk#l5Y8!GiHqG@}cp$Fl+WpI< z$EtA#MkLKV<&-b+_}3Um4KLwN)Nay&Oc8G5ZsKQUFo&Lp@Z%WU9lfB4_if z1G9kR(Y!TqnB46^!RjEjsr0>Keiq#ct5l=8Uy%tT@%qA#^JtS3a-_ZchTlL~cI^~t zpO?oUL+>V2XzO*J`HJDJwJuMQ!e>Xp^aXeLZ-w|#XU&($E=gF_}j#0@P5?ZcDSUD1`=`G0xUwrDbji$^fEtC_W>@v zzsd=cS&UjK#JoMBRFS&3i3(rmL_eB5Z%g`bHxv}grTGbft)LMi&!zppMrhk${g)M% z^S&gWImJ2p(3P18@_Cqw(@H*57}_tb9g}MleRe5>G=`L}Q398+?j10mJU%r#fhJ}S zs=;m~f$I^le>^~25{^)&31OIPfqeeI7AOvF@eEk&yDEg}!mT^6M-W}7v z+{vu}FQ^n=jY*~u-?dWTy|DIi5EnBCwh^4 zX0hb_Z$QXm;|j71BNv+LC3Ak_4w?)QDyvOO*cKSrs5{ULx>yNZ4w)qk^C0i&@`?ag zG_R*kCl8;=*^9~iBr9=O5Y!G-FH7&twyncq(zUt>%_*%L7Tk#?h2<-z#Ajk_h5C2+ zM12l^lXCfk75*lM(VeA`8sfcwN<#QQ!1ZALRoM?d-0~V zxQ866(fG77wE+~Q@;mLIfv>uJCmwIIqVOfv0sgA3p~}mIEHv9VS536*(O>YISl#5s z>%wm`pAr&kW|!<^56{(F;;&0y&Cw<3uV7)A{7_ZaPbdIjs;$B^cOng2kYcs>hKE_R zvoV33u*`VpyruD9TcizXp5zRj>*+fx)%XC&2MU0yLh2#nrPgt-m~2SsDBZgt(LEXp zo9)Ov%l^p8_HH$n)^sEDNN%U?>qQ!vftRNF{N}qo>s@}H2od8YpYix{-53)Xl7|M3Q{j1jw#;h1}0u z#(I=&HR}R4W52CJeWhPyzbw2LfeyU029lQ(pDJ~^6p^_1)k=h^fZ!1KB*|aob2X-e z885u#vEQmKp7OpLKwg(K)GW$P^4n2FNo=QuYS*#$yTDgpo&gg36KtEdUY6Z~tyg`$ zb-gcq!*|EVTmZp9$j3&i=G^&V?)Q+I?sO$i`ILRCge=GpX#A$q!5A>jRt{|Cq+42N z_gvtkfx&S<>{E*zBJmTpES%;N{n1_)tu%r0r;kO5O@f&^MD^u}p3ZyawIPghN?<0B zwkgT{dBzKb(L3eMh}F#b$iL%yFfiQPlI2s|lZ3cVgE`8FC{v0}_^U*eLt7cr-VT&(m9gym{6WI$#6To-Ao@D_^UK zY&PLELmp7579_3;l9cq}*LDYNp&hmW+e+;7sDlfgk03{Zf()Aq`Xq}Ime_sq(JBVc zce^zuN3%VNT>Ucwg{fmrY`qGMHC7wcU`2;2q}>v1*{F)%O=F6;6u3DVm$>2U5LS48 z56&C0vDQo-IT^7LA{gK4q4w<8I^==Eh{bwdM#@`4o+jplDZB;er)`*#|9?*UDo}Zr zlFMiL5S}+4YQPq$ z12y@F1=T+md7H+{gY8#>b3=W4^Fx3z;~Kbmw@roL`M%D2b_SF^Yb7#;3NyN$vEb=8 zd`L)X9@x#e&@dIQkVnN#rrnlc*~wAVJ%f~gPz&?eKobkQotaXvWT%`)+r5V>xyF!C zP#tSI`RaA2Lqll^5(D;}a1{o`%fLwg#$@DzHnbHam^6%h5AjG1WaNy_YzhiQRVImC zy*%6Bmp39qmOHpLJs)%K-)6i10YPh-orgnF6Zrur_7v%^PeIleN~ztqCIY^ayI5+{kTFE z;jB&^`Ty8^>!2+6u5DNm2~m&+0Z~LcMUaq?6r{T(C8Q(-0Z9>1x}{sBn+pl)5~KvA z1f-Fal7@Gky6@+{-|w6G=6l}%-kE2{VedVA&$YSE^S9P-tz#YQIJ)=5vl?HD>r0)S z^L|0?K73QmFDis@A(kMjqc7A2xG3%xWg+CQ3>s-rB#Vdm^_azDI3K%ZO~;!_f+jjg0ZU zKK3RR#lCj1d;iOy*_o0e`5=wwRs({2`qW+2Ts}=Lwr+}J5zC~RMnpzxJMfgUy>iOW~W{ul;=%D z+lHCJ#!C$Qxs>g|jg`#wFN8!( zuTWPbeJD3@SWlKi9&}=8hDT@BfZia}$BuH?W)GyZiM^N4C|EZy9{r1zzIdMWs5@@{78< z63g}?YRR!oq&nXRSL5?3P-D2>S0$q)#Gc$%Fk$RZ?~Yn4+I!>s;h`*(?UuG0;YI>-mH+w;tB;}{3|9k0^ zD;s&k^|gBN|247>elxWCHPrP!5TpmQ(RvM~Rd@XzWD^@YMWUKRaP{gxhSUf|t_wEN zG+R(E=iIvz8|ydk$Kd~BVuiF*t@_2q&Yv(_ui<&fBC_M>W2u=Mx?)vFUw!xVXzGkD z4Y$+HFzpL;Q;wkvr+RvmYC0x~m~nBAw@V<^_oLo3ZYX-Vxcoz*u)t{^OQZRGe@ z<3aMQM+KYt7a}tR@Yl80j=YbsDNe-sdT$>o@Y;a(M@b|Kw&kOa^zH<~viZ%gp8p*E zz}#cHHU|mHE^i&!Kh`GwC(o;r5UY`7o1C4kCSe{^H6Nb&=SdYcO6=%0-G1XLfh`)f zB+jmsE_H?mG^ymwhOV9R}i9m|KN2&g>^{19ZsZ*1jTW-M_ZoQ1rzQUam-M`!KX}H|$ zd@fr`6k>;enRd*U$3!!x^k<%p*ZqK8fJ!G1d`rJXj(9D6SpNT;a>aqpX+_=SG# zpKQ!d_@dH@@9L%Hl-PTVddl%dzv~;M<_Y3*HlBXH9`W4A@Jm^;&HL-Py%h9)>H$ZH znMz5+gPhA$MYXum`>;8s9ehE?bfmD3WgR-xfFxxho-) zyn-NMTH=h)6V)AWx8#;esM|Xl5y)=idTJGOMc>gsnfl$L=9BmJ&GsGxa`#LEzLiH) zH<8^nSEM`J>I0%Jnp?xW{rNW%hFUKqN*G^AYbZ&L(#dVHw#S*qGpZ02JDUT|By=;< zw)jxv#gE{NUe$M=#XVtsI#jUoNN-N_S#-fugE5ca$0{6rj zz#+3sOiyW$sw;mp^erO`+5o9owN;T)$j)(x%k!?%cySBiK6b)K2Xuj(aJ7C*K6S7S zvaZUwv@1Q%Cwe|DS*(7%oC42@xN`>CNG0=GN*5TE9#K2@^K}LK1S>dZaGpVD)lV%g zsBoR8G_lCz-7@(z}h>N*NZfu5lWP_Z8_2&&p#7O-06w zt9t}d+$%I?@vY71HdoV#c`@`ZsH@ME0QN|3YQKu<$Cm03OkqfM3?rS;7sm~GM}0cBZ8Er&a`Tf|U1SIE!;n6zLxAA( zX@{~dcPaC@4_&$XP|1;Zrc}ruA-w1{{#4kxU20)L zrJ#|c&UMV&Kjvd}3h9mq_HD__V&Z9!Z+g1-R^ zx!ImERE53ObdW216@r0w#W&d|aU~`{P4x=S=EvmLCJixc>mSo#Vxv9~w|gs*@!4)? zmK+_}kx6oQH+>9etr_?QQndml#rT-0gKt!wsjEER)_PBBHfRGsReQQUzq(?g*h6&B zZbsbr`y1iBd24K(ya5CN<)}qAy4c!7P;)bp3CWBJCe2b4X^XA9E?;9FYN9CE3TQVqHO43X{w_ znD_Y;FebzoDq<&Nj%2xAfb2?f>T9y772U$MwBpB1M5cZZ&Q4GAhAEO-BAyq3bjc70 zZ3yfxv*~`k-J40p{_)4F*fYkgSepF}&oiA}zu)MAIQT;OXm1=w^%7!lp0d;s8q%jh ziFHesJ!))N+p<9OIx-XdEa|o8a1S<3f02LEkIHlojb7Ix(WbLLpIuo$Bu*x;u4(A2 zSzXcB2P=6|^O}#H_GgI8Ba>!Zyn$QtK&Ro0rQ(6zvqGp)4#K(fAP5^%s*va3e zb+eRy-(S~1${p85y$I7MpS*=~@m%-F9PTX`SvbhWiCj)|vfK{qEe%rH3Q3B5VtQ0v za6lDAyt_xZIFf|ZCRO?o>Ko1lYQrT-G>&?vlV^|9>K`u}j-XETDwk~U?^TYx99VKL zoM^LWxVtuj`pNFIcJSrSj)Qf%gwPn8Th|1P2GPjqn$z!)k>uSw%kC=8cA$U|tO<`< zL0!-3VLrPvh~ME~WEZu;_rCO(|79zba9tIRu!;RsS1}#yis*{Wu_!~W&a9&9^P-ID zxTMpIATQr_|84gj1}j^uAwCuA6OHKw84`lJ0%N5Ot;tj6TL->M^-IU(L7zICyg^1u z*sDjK#B5fesMkLBc4)Vd&&ELdy|nGNebzhy=rN2-MK!gqa4kfKZdBKp3yLqf{w!^B z`6^!+`^>M-WXEMS#A}alIrZp`SqZOKp$nSz`;U0Hc_SSah=??m{nb90QquXNz_2a8 zsS28R3LI0e>G7=#^NuGMt3l}5>LQhV)K9~%13=tH6)6?9Un?i>4~K&Ai`hakB4)0b z^(!=5LuuustH{W*Y^YRED;|GRv0~_oV)~c#uq^*)+qzQ|=csB}aAlrH(DQvO^Ro>6 z3<7+{s`6d&zg-Uy{UBo(vFmBPw{T^SpcgjYtpd>j4V#O=%GBMhE^f4IdW}8JWlqF- zKFBib@udhBGeqBFh<7>3Km(WjhiD0#HGVj2)&2qo9jePOv^pc%%6d{5W&lPe#GTz* z698?HY=_z>J!u^!S_dqUEHA~-{S5wwL(rzyRA;^IA*G%hXbc5Wrje!Tw1(16gM27Mhlm1&x_O{aMCPhUf1XBB{6vB-YFPSu zI!luK93kZ5z)qM?)}6CX=Vj6CtLkw1#xH@QBJ$*=IdP>V(Aar%zVc~f_GD43JRX6m z>d&J9=0c7vD))9*TrUBSC?)9n@Kk`Pn|C&-q?iYk0WEkG z*;!#4Kg~7r{>HLfJ-w?dN1U;%nJ*p*R+7iz&+|@A=ENF)FNQw+x#_gjk(+Ml?@f0Z zx#`9{w3$&ZLMx&wf`cK}*SEA{>|`cO%A`%%3P}BIwj&tUOj>9a-7X>SjuQSj97z zo<5GYs%aJ9^~x?FfBO!N#<0}iaEthNxCyl@)h$MKHc-!ZI33LbV^NjPtQ3o7_9nxX zFJ(R?K9_pWcVD=eP7$ofO=&@CUeA#kQ%gJS( z^ZK~^{`MDcYOdyUSI$7g<-Gwg>2L?UF4FB@W6`nS?7qUF8x@?sg6jh&!`cx1#wJ7M zv9yfm_ebggMrdj3ID9}o&JJ*SKKE&f?Tx_nykt63`{Cvn?zr>5=1ENT$ab&s4w=E5 z85Y#AI6ZMpe`^Y*L=H)wl1T}EGUm&PXo(otcv-^4B?=R}>>}R>J8$^OzELIEfw(q@ zU*OuuQvUqH&@Acvr8M_&Dxw?uvLBavOkM`xG0^S~DhIlchJxfSrH^X4oHNG!CFTP; zb%M03p?FELZp1iVOOJx71W?yVJrpK_Ab8xkZ5?cZFMJ$*m5V1#?`T7BS@)%dc?yz_by3V;3+1Xv9On9-jAYy4M$(Qq)8vVumt zh?Tt5SQ6rtE3O`3tXF5XIPtdMQ71HNG1`-S87t*)P5C3#{z(ngl*BGffz50cee_{G zw5~1puqb*^m%9{s+`nB3lg4-a*derWj5as_GP%WxoQQ`PwUziZ{6xcWL79ZW|6kWy z#zE`ern!6QRP3yI%jjXxN(LoQM6YQw@kA%XB1_HW6f|5V8!dfG4ZD)MdIwQy>ufT3 zoKMZ-$~QJN(Yb(;+cx=r)HH#$L6DbBm{gF^ttPWeDs_!$&%61_12v`)0Ow3yH`5+E z!J)-qaGN2O%6G$m^!}i4d^@;YvS962K`=v+iS!jG);=wR2A{V}l8?V$qQia(!TVM# z%6CYU!Ke8!^1lBBaq-u(6kp|6?$EM+v%7HW#NIcipr`g&mF1n9$O z7TW~-vMDET$~Q*@eok|noqfMQ6dI=X743R$WU>71fOnd=o#F0_P0y|}eZ*E*byJ0{ zQ?lPtD%h6tRHQ1pH2qx($aR*cX@a7ssIgYT9w!x!Z2F}y#k|3;l}d}E>%mz!H@34} z#pclF-8Bu{MX6U6y?)EAH(nzmscHmB>cP_kK$EaB{eSu3KKnXnK>j4hp#_x|p}=Pp z?WRtDX&xTknLVrbI`5l&ov-~BIZIo^D>ms53_&Y~ zRW?N$S|-HChhcOK8=(~U*pLR#2RhP^rZj}i35HsjY)>t}1E}{}C-_!=roDM7KOWBJ zzNUV!p`jN`F;O9}(XQF^-hcUrD5=Q=9D#F+f+FDeoi=lp;;2`Qd`xdlY zs$B6Le$WkvY;#zR%SN28Tx)kYXm?oSjS<6sI*hvF;=rBqvnItdgB2}>W*qH{i^MqV zL9olnohqcOtBEwlw|3V%xV2Zef;IIg{^|}gE`K!lbaRlXa}yvldBx^cPMtLiJ}wrw z!%6A%ncX1tF=e|mkh;cAb$(Bpx^w`!pD+pF_;cmoW%A)F zKoi&URU69op>b9nB_)1B;xcZO&fJ5ebN3tSI~?J6OagBSP5LvBzrBC-PW4QYMOHa6 z`o6)<*DZvC=|SoJqEhHGwY)Ul)e>49O`lkLVstwjy46M*O4mDRxF~u>&U0y_e%16} zG zfz`^pK?H#XbFN?UeOk(?)H-iGO48rSAScSAuP$l_M$;+P$azw$t#HO1C~ zh0uVRFzG>QKgmtXkLhEt;HlsH_FMYMx7FOl-sPY)=>>f^&Dde>mejiYFn-;iiqLIa zirlvUbbleh?Z95s>ynsMsTc|Z=9W{7wWJnD{5{e;JCAwXhp$KsPKRX_99*2*re*8A zv0uKG$oLgcL-lO8cjWuvPXg2$bYgMA(22PSgJZanF*ZZcy=5!+MTuI8cg>R9V@un8 zkMCCG=~$IWurMqc8C<`a+k1%$U31-^^Sf!{m4p4ln(z&povU)z=uVFd7gJwiH+$1* ztqdb4+x5Yw2L_roSb@2T;V;6L1m5ZCUi~nutZSnTzrX1744EkM{d*@^Q6;iE>~UZ5 zx-wuh#L1jI?*sQ1aFi7V6*#Y+az_w`FDdrD#2 zWvgs|1TEKY2I~A@)-6A4_Qg#u`ae-v)nQ6uag*v-l1^BAW2_d2fq!O9Xvid z#w=t#TGF!^J*G9SouA?&Yy)SksYsLHZkC&yQ`pc6(AWf79$LBm9 zW`AN|(A{9mx0DlzS~?i)Epd=8Bjj^+wMvGN>AEpnS{}z^5>FJ9Q=j!tA+hG#yZzkp z0P~H8@x_-dJVm?XwRZ-6T|f2rSSQa7Eb_LhlHL<=iaGwsKNdUn_P1L9Hv_f`e9{wM z66>0V@?=X8BScy6W>UT0?(Mg4+WekUGvM_o=5IS?H1^5bfiLA}SqjeRsCnR3TwNvW z^UF_1b2zj~r(NT8{}g%;%d$~IMaSVUQyj&~+mL3;ZOi}oaRq{0Jd@zp%>RRWtbU~6 zb|SNulC3ubvBq;4*lFl*HgJP^HSX+a{fVNX>9DfygNypxw1xi}MZ|VhQ5Tyke+Ibq zS|M1x!1JFw)gBc?>RnHEc&_;uY}8^^cG#13`v*>O$RLD)VOtD`rLTyKR3qq9QT%vG z;`A&IssZC(M`*su1e~l2Ge;vri>h^cYaa==09T2-jk@%e;V{d3R)Wo)!*?(8`#n@* zcm{{0{&qAE{oAa1Icfx&81_<CHT?PRX)%z{T1h0LR|W`qtgGef~b(>O!<+$n+*h1Z^{qd?tL zB5|GQ8iieP$Q3xb0U>@L&Lmx014J1v^8AUX!v%HmQHJz&b5)OSp?Ww?ZJnygA-L*A zMtlA&fiS0E6`ToJW&qt_lz7uXV{=+l{t5-yug)`SL(6%c`K4( zz&2t%S^PdPhrI(I=JuurHd)QMl!Y5L=3|>RlQRNAgdE|&piRxXnDX;_J}40-&|J|J z*#+MYlS`O*jWb@1!UQOIq#D#av@0TSU`Ju;%F zRq@!h)U4I!SzLL{8W_~|)wG(8ROo29ZW)z6CeYC{457q6m7wCsQ^ewU^P$sg9tD}8 zBD;XKlCrt>R?ZDjR4S8m?*u5AC+a&$hYyaBw%I5m%8p`&70vfAkE0H07M*)~oC~P{ z3}mHBvvv?n`I(o3b1-imcvW(cxSzxg)&trkFh@*2$+F?;qqf&+wN!Odq)uJe-G}f^ z$fXXFRkm8GO&Ux9mB`a;dz>;W5F+#~q}Q*Ek(%1^MU*$QU`r;?32e58(B0hJgSSt$V)h`V!t?C(${JPz z+4*O$(dnAcWav?Ak4+qNb5*bB-WkniVlU0rtiBuyn(;2&8^pfX)ReZ=kWr|%aWMBF zn8T!kqE+smOZp%ziBa9jVR)4j&=iJNR$F+-OEua%I#-iLJ-Sm{jp>rwXsMVQ(TbXc z-Q$Al<=kAOeqN2cVK3hsDTCy{3bDkC3b6jl3$U-vy*pCXRPnPaoh6IuQhBZspT1hu zgS-c`uJgw6t{{h})|y_I%aEHFAi=4oB%~2dJeOA>}|wj$loNr^p&2PkkPkpo7S06xo+92pxtYtS;*Asl55tRoSSZ-KV0?IU2Si+ zj^iFwu-Tm!YA4Fgi`mSIm5BM;GQ^4oBFUBOMx+=$RAA^*!0 z#0P{a%p|jxTRJAd-H<-M(rJXJrlp&~ghQv+I_5k=-*gmMeZgA3_0Z7Bm6=*VtbWOO zdP3@yDPZ7zgg4`+xsFs28UF7p;lnU{kym56l^G*A_-rZV6eQBQA0V(w+GSj7q5G?6 zVjc#5LUfZRJMa7UDt{_|$k_1a#B3vZbKyi_E(71+P}?M3TFnU?>qsP;)vehzvouou zR^yxpDsB%wT%9C5r_<=7npO~gR$6Jcv>NKYEQ{w-jV^3)TYv>&ZfV0!&WR;L4H6Cd z$vwx6=&2!7ll(HaM+rF}KL4%?J+0`iE?D}FyJa!Imp!`G#gv@ zx5L{yG@`X-d}MCvLI13pNuVwV$W-i!^`d8_?wzj2$(g)5?nu2E@nlWUY+PMSaB{|& zFX~Om7;4m=;`CCthVZb2j;gqh7vCDPFKig1Mp=ii7DYC1#u&o3&lAsy+OB8`uv)S1 zoYJ6DdQ?=U@6x73kO9cQ`U3l9IjJdKxu-J~ZCK^@qgGiq45hd)xa$>n?n@ir&aFtc zG1ZNWYNk0XlD2!d<{G|1zGYKZuNpd!E3les1B9_(B?0{cL(uiO>Je;=ANJ%)^@YqA z8|^3<-d7&Isi)@Dh+V%X&o>{HCQr9;V-wT8*K~*6(W&2Ie2Yq20t~(l)q@6JKMWWPS=RMlU%~XkTCUY)5kT(3q|BuD@ zU9odG(Zx$*FK}3jz2Z3x(Z+94UF!WrD5B*%^MVVPdbHS?X@jQLbS5h$MC=@4qh`ld zM@96aRes`6U#3E^b-_Q;0?<%gV)d(PCX2z#ueCm9a|(gC#T)+ZFI>&7O#O=`SlLF_ zJg#x|(y6T2#Wt3YN)~#Ex+H8~t6wlkFe5fIGN2ToPoKU*V2*=kHQsq!5@kIfAtfc( zeVOZvViKWO6K2$bHM84uRn(Vz|NPd^pP(h`FLiu1n;egBfG$bwfRakEDdrtQ(Bc9LkAkI7lj!Ml`V_nfQS$>Eq^B2So zeW582@oR)XKN1^2mpii%Y8Z?Ao=4+TgCI6SY(uT~se|zEXIG;a4^4tooyzuifaXb< zf{^dMh>i<(hmMeYHD;03i9wh->7Pt=e1;K8w(HK4((HT7;lfKYC(mj7@Ue<*RFRY^VMO1W=|WE^XIu-hA*3P zo`Cx1KaXl0{nZUcq#576h2DfLCk{crH5Xv=1LIT?G;6s(Ub*qG={f|G7J8ArfWV49 zhC|9dyx%N%UbDp$Y=m<0lfXzR7cR-6so>=2Kfg+R0&hySXR)tqfU=oe?;?t}CgbI( zQzOHpU#H2owOO{7V4ZB8zVM6JiN|4Djli7r&q0;`dr%*BLlH>a`GY>ijllE-cIA`+ z0eCLe+;IHYvl+~{cxf6AE=n1YvtT>Z*ut_|&I2C|-x7UL+_b2iaN3R{kbH#v`rU~4 zw!`84L$qT96p9VXAj+IU}|g^+Ve7666+@OJWFSN6f1wxA7}qxZ?R zH5qQ?HV>+4`?OKEpJFFq)#IkiUlx1gE5*HrBmnOiJqvXng)1d_PuOhv1z;=e<64pQI{^xL8 zp+hR=fERAodffv=w+3kfSqN%+aBQ%xz^QZ@oJ_MNBicnzL5K3{de$!(s;f4?1k;J3 zc*{a9>?~wzu%-NBF3P59z#}ZZ@C_bR(XOtFyU0j~W#D zya04gyfT~&9%Rn14F^j==69UJe}!=(62^o#K1GTI;v;hAO&hO~?b@*?k#bGXA|e%p zTmt7`eZGwx=J(kjdzJr&um~Cn_IKb)BV`NsO-YRd*jucz0K8u%_Mun8)2z5bgtEr} zVqF(1J;(s?XPFR2^2CEs(W-9$03uQ@7z5_TTOKQ2_C{7HuTb(9Un3_e6ogT+1sJdu z>27PCye~4TwDI^oh2w;$hy{{6rY$S<@rur@Rn3Q*QgYhumNz8N>HTmIBG`v;+_V zG}k8ol@{BuSHWg#88IN-908ZdU09jdC_n&+Lf z6LLourk9M=6<(n7;)-gWZ)EDu|xGkc_fY#9Tnd>dV_yl59 zWexR*{by%4Bx#0%2ckyX$b=Jdq|MTr8{jk+Z+04 ztSOExa4SR^FD{Js(_@$8oQK>HMLUkN@AS=~`PVmxcgR5qsXh*;C_;ppprKNnY*WSo zbRQLw7TWw5^BUhIaszso62ccAz1^~deEoCSL4}xgV+yH=&*lZwk-WjXPI;VP6_g*eppv!mzCogaP)>-F2#xD)Pv&a^c0bbY}%yfnfpb{gpnD7aqR$Egtk-xW*^z!7cg_- zLae8wJiX!VL<=!O^*&x$fra#dyc!p%LJqI_oNPiU+{6DK0uSU6)Ca|i{won6UrQL< z*2DX@H~N@=1JY9!$KL z>sxKd-vS`(&n2WCeQq4}ErJsHL8|iShWZ<2RB`7PN%>26;`W?q&OJJW?3{D0DXMl_ zXPVqB5#M7Y+VVb<7DV#?AK&k`H|V|R0p6?=OmR;hfuSwH#z7sF5Q_n14<3ZQ!>lau z`clqHc#9k(?>mHE>6j=Ht3hkvvo6DimH8k=N~QP@m@z0J#@Mc~c{vFDb~3iw2K+$w zg`^jH6%A-OgU3iWdg!c3Bywl5d* zTSe9!^5c7~#lPR)k~t*uzJA)CV-&MDvQ3P1WrEsaY5ko%t?bo*YF&tk|HD5b|MP48H=f-8JgxuE zIs2cd^*>MRe@k-z7vwE31vKH~Up}tTUJrqU<%g^|(`zZxhTmyZfijP`3u0;6@=5$T zaPOyx|E;qB`I*XywyMG78=MUPe4lV1#&<)nu?Yv6^77Nc!TU~cP6?^tV)AN<6;GxU zt4{Zp?OLeGB)6fT0@Bo(qCL2p4b76iJE(o-LQlQmPpLnzImr2g0A1hDULEbStf(|r zaS!9$8mh-S+6;v9X`77Uzzq|`%aOhgWSZs@?N#)i%9eYj z^PyI*Us$$(x;ei?8WY{C`@m{!ZVlP5^AdL3d*z*r-1*Rq3+bLafC4O!@k8QQiJc)1 zK9YiT^RCF!s4~>+E?I5~FGqQm5$z^IlW_tgBbvWmA$*=aQVq*{?Z~6i_VJ@9yZ_cg zdOcsl*nP5N=tOVRokAU<0_c^P|GKaJveWI`wdNY{WYPy;ss3HyAw0q z$d@&LiUyC%Gdnw)p70OkZISbS3;jVot>W!p&}8L;-dE0{z#6)HE20Uu7T)qwk(L4^ zWuQ_|AxL>U*|08PSx}+AbH_7MA_9C(-Cvy z*B7qdY^cK3-NB*a(%Q3#1lXb-0?XsPtayvuOeq=1BnO4l*8W11E@3T)OPUMax&d{a zr{euBb=QDxGZgI*$D@WuB+gL87E#{G1WqR&FGqS6xiIK<# z&F3DTLxOs zxSw={wH&$Fj9d>~kj6QwzNzSEzd&bpxq=vbq|jj`lu3cHn00(>E{O3zsy8mE!V;=} zF+mlZKo|>h+|X`*62*z=&;Jdj^JCk_i~U{4a%kVM8#>ugSD_Z%8v-&|0q`51?}9D$ zaLYiB3jHoTS>CBO6rJ)ZQu=wLVHxDDL!Ylm<(?yn7EMU1) z<#F}%&ojZjVIq&w%>AEzB7Jlu{x2yi=};VLf}&&1evJ&%w?; z`UHFUHMtOWIfgeJXq`1c{o0=C?f>p8GgF`^pT37nula}~yU1Q$`-ZIMG`iI=->MWb zTT=^`br%mO&A%Fa5Np^L~;gcz9h=~ z9XT~u?tSM_wHhh{&=t=2Sa<_8KUCT;AF{H@3& zzyC@0R*YY5DeJSI*gOBu$@`J#i1)Mz;H&bpcmuJHw_GIs!v6TwDT%g7ipxUWZr6SK z^NZ*PHyuG7MfJhe!Iq26$5NDb;@#YT51i9&4)eYnj(|9lm)#qE63NI?xB_14n?mdn zUd(8S;u?6w85sRCZVOUkE+|)?IbpeH;9c*q*ux!a1BcOKq}ULUQy=uwtHu3Ax{xao zw``I?8}Q0eguvBYP~Gpva>WX{n>GVu?17 zN6;Z+0AZ608>}Vt$kej}@fI!IMniI2`7LkQmwk}*{Ht_ZzIAaU#xi6yuE|*m0$b@H z$khrefq`J77Qq=0yerl$b8|P2`5C~iv(vwAfDG%E3dn5l_h(oE?SO}bXo3GhXaWsV z+<$ZGg$j1y#9@Uw(8;=G`&`w05-u?aH>yhHIN;Wr!IXTZ3)*=nKfXtLzOAqv-BS3L zh!*3*Mfy%-G{si>onfSiC4MYu4G*@wCA1_E1U(qv0~AeSaEUqo&nJ`zSo>^d2g(Xl zYvK)(NzERre2-pV8I9pGlO_Ilb$ToT>EE;D2RWoN%v&5AwcELfa0=aU@zzVYmWFj_ z@dy}RiGaE0pf9POCgIQ@G;x0h2Z^F#-Nio>5&%hyIXnQAB`bRYEY)ntF60}VG}s?$ z&spvxaJ-cU)4G3wjU8*f@gi?KfI8KXOy7#eu9?qk#C8LXVMafz>k9&D{#Am*-hL*~mm|t`@5y zVOU3C_y{%MMXncB}@O4M0grYdpZ8l0lO=_0O1W9KCY{!(fwu?ui_- z+^hnZPR&n}qHr$|@VxZY%8#en5z1GX3T8ek?<=a!erOYnesfVf+ z$sv4NHc!=9`eoR=Yy1z|7$yRn6hlyW>JT;q87X%i?opU0c-XQm2Ut}#?LO}5l<>*f zl=gBAQ8=`j&9a)aus6uEI4^4ZUI#0NbCyWtW`ZoKmT(AwiMED6R^dqBT(VYqcWhD4KB2R;Ob zKHkX1xOJ7mlEUj^(aF)?J=0*ExFAjEl5=P<^{I&m_^-J&=?!j;&@ohWZLu3wi`T(c zj)HvS{o`X`77YSKoOP8H5d|0oIk2P!-@S)7!p|K=zco+%&S06aOj~w23Y`)kaCqS+ z)NdpL-Wq~@n=&$U8^%z|*rhZ}i!RL5z=?kA|*D1TEj;((A{zs!`UB4dhh+ z%4lBqU~*LX+5=pHF2#r-fyW`PDSZq{q!~{kCf$JJ%GK2S*Sv;ag)T&IKr$wgH=?1O zK*VpC!^Cj=*(>H<^gOJwTOG2TCLA>!p&1;`Gy)R3S}~KzpCKX-c2w1s5t!HvLPcl& zWgU)Jfk8ijr2%y3nD>kQ*wD(w9m=Pn+?kc%(q+P9To z*P-)qLMn)oIay@(2uyh;C~n_QJgTv6IFHG|Kp#NCWf}xN^3ikjp|pKhAuK7HhIX1w zY*Z3@hy)|N=+S&I@zjxNDvQ&UK;OK3ufW!K71`Kiwx|j#m!|M*){|Qy4O$B-_}g^I z@!-AfRYx=?H~|ttT5^fJO7(L)iN;DFoFCyklXq${ig4h+ztO<|fyl@<0;}^IwmJs1 zM1Lo6ilT6+dKsf&ZBz-tnmJ9-d8t$00{z_MV``tx39q`WeWvqa?< z0+ULjHbl&31vj!C6sjrR zQ{I(W9he#gOi8*(ePr3DtFDDYNd&F+YQxMKG$LtqrQnPx{TP%rq3b>Qp*cuAYVW&J zlNcR{p)wnJZiO;sXQEARw0gd)@4p&z8Oa+<2J6u4;K7yi=?B#RTnonlH(X|l9BMo} z-4mX5uiA%yjfn^ta15!cTqF)}JCzK+b~}{E@fh}#DO6b|G7sS@RPaI8ICjYgT3|ks#D=A|GL$l0pU|RcnU0f9l9^FX&>7nJXMoj9cti+L+4al6tFU^=EW9f*v zhnaMQFp&x0FIctyru$+u!wCEKK7^tVU?#UY_aW7AIzx}<12w^|P4e%>-R#_@Xf_^z z$fR8k^VGn#`%#oE_V^Ncs@2^UM7>zkMZ8A2q#=c`1I7`IR^zr-Ehi8JjR2%fzIi8_ zvio>%Y??Kcm3}K?OmL&V$cv5eDX-I;420tlcyw7|BtFvot}xr=2g!l3T^IeJFro3pfNY0S`=J!(^xoI|x*^89 z#~oE`ZiY6hDK%kyesFhar{BFE{;Xrt&w-mnP9|aP#SO;mI>V2@kxICWF^cg8b~fK0 zHrG9THoj4O9hF!)L#Y{V{Wg!kT~avL%GfRwvsy{QLpaj@`86UFg8 z1(9?Dobl)9sRsb6C{9H!?6XZHO)L2j+=n964Yhf;hX}uu#Ea82{*5nm%-1Q**hly6S(T0-!~<+ zqW3v_O~~i9&c;$M?(gUn*Ms26_H@CWNH)NN&ds|*m2xQt@@g1l!Wn&ybR;yV^j?$uBai=W8Q099u@!{wb=rin$%cay5 z*64PTST}>%i%-D(Hv_n3aF_(}Une5!Mz;tDkC1{N6drI?UF6{erFmvxuF6;u7d~+ z@9^E;``fB3W?u>nP6>D0kk02*VWmIuO0g?RQp&hTZx{a_?G5X_1KRKDA$46btR8i3 zO{gw`IlbR^^oy+0Wv9olE)a6`GHi!x2K5EO)9Y%gUiNn}+WYUMAZO51B)Sdp1$VSKrhMMz{hOQqFV;kPHnx<*(etjcsyt>{fH1-lmX&W{$etyStPj!tu%dOcsSTEzXPX&ijc!L6I@ zYyMv0$j1~7)YGTMCu|vyS-0=K9Jl^Zk|4;V$n_#(P3V1wm${X(JHYx3>$j@ zUwx;4^9?^kyXf0*a~~!vI=|Hp{Ybr03ycN-OJ?bA7F)u$KN@nW6H{mX4eNZWQp=%B&IRp{x>2Xe2YsTE~;4yd_IWq10&)Ng`wE1FV3*snA-WQLP@HAFxM<|fSYX{*{Cq1Q|J*!~`mam&C)Y>a|T#-3Rv*qa850>b9bv?eM=O?`j5v zQX5Ck07ku&_>F1}1)Lu1J@WMF1dp10ANRTTsP=M8C_+uu_=L4XcBei71C8u*zA7b@ zl#M`_AY`sT1m97##9Hf^JlBqv7F4Zv!4)s=h~U{|(`U-SFPJg0)xP zoC#49pm0dVGVn8yb)Jv1hn#$L7?NI$$V{?g7o>lAVxD4ZDGFxMpq)6vWiRZi@D;S&; zCz+ue!bbG5ZJ`xtTgLt3@Z*AmQcv5<>vEQ(3(2(ZJV!t~440ZR=^_zMF*YQfr!q;+ zHxVjuYYHQ*Q$mp$3a=^#0r%XTvArFWCM>;?8sW zB#6X_rb_n-%v*Df81yNQ^6Gpam$;p2yD1&gec&mz`2{d&iZaTNZPXbi+2U$2&TQ z^=Vd5Nq;EQk>)W-PhVQh%-9t;jPAGG58klaClP$Wr5d?AVY{o#RoWdvGQM$VEbESE zS>>qH;i{BLIBUv3K)ts*eIF{|C{{6;sc=^8E?u~x8pefN3?`yV_s6t+?Q1_bT6pe^ z=GT{$s!_6hX-={EG)~kbh9^j?*Yq|bTTN>sRMX6l0Oc|99QUF@K5fs;a|r@$oIp~8 zf-0pw7ze(Np`@jxOw07_xap66D0;2~v*yZDZ{zaIos6V^0+04H)&^$2^-54013J!Z z7~y2-$=+1`f>3g+|8ROD>w)AzAQkP7+bpGkruYOjL?qg`LWL+!l2rA@n*&cJVWm33 z+?gR1oIZ6wPOzem#FH*QdgR>mB-q)PSp5wK)u~f!FIik`pyF3P0Gy34EkmL^@uHwa zF^jtdZ~0t2xZCQYdF{<1=Skb^i}}89o1dI#xf+`stmk87r@jYym%O>KaetG$0elWAw-fP{p)(;}asP1v)Rby&_$akEEhZAIKdLSF!pnCgv z_qQ`wN-%#i_9*)d#%9N{MD>zs-c+8U`?DWZ3~O0n_oL4nz%WHp^g#%%iZts#)gMB6$kEq+#xOMmyaLmaHN1tdelNB(F=60 zP`4njpj9w$?uYj_;~Uzp1Av|=Eao}3Z7S4VmReWLmNF6Z7tGJQ%=6kn0u6~dPXxVn zL{*jPh9=tRPpi7XK+dez8(ljE*m|7BE^76=EN-YX@Lj*q%p{$osQV-|x#B%T;LLBS4vBlqY)fAIG{swbJrnp+GSLR5q{of(pm0`` zOEv2e`kzv=F@q}9e)&nok^b`*(XwVHJp9zJ;vKVc>eeKPm_fTKCD;XpNatAjc0YOjn$e(Hfn{@Spne>9gEBD1Ivs@VjQZ}erx>C7vO%zgPK zrhLe}qU8~=W2S-Z)vD4T7n=R#W489=OP$iB2kt+{Dp zjE3eGe={~uOVnDnluOCE;V<3+20;ejC#Z*;CjX?nNDQw1I*?N%TvAc&M-M!+np!mU zz523h*FS)4IyK2;DP4kDu4wTNVb8JZGk^69(47of!tQEZfXMTX-^85lIobrz8_J?c zmMAzDRT2aqd;rZQ_YI}DhPI)m%%CDx)Y0A?P}ZuEuuc1q;!x4ikq zQh|Fw|AxE*Q}}n{6Z~<05jxlE-_cFqjl?ImG0GhWzCEt~NT~%c>yAKFCj(;SkAr{OGH(KKHPrLgXOalc__WQJZ&xEtpN`yVmff9Ltpu^X z4mDq&fo+>$nt!RwDPZ{HW1Mc8g4__5cNzVA(EMI0`+$eVMmx3VTlpqQpHp8qY+=TE zeZ2tt;>8@tK;!GzDMt)yovc*?pO06wwsFwK$|P?dkcgiE4o(E3b&B=ywx z&2i~hH;(QuU;HyR4)pGnW?vg_ujwKN%?dm8M&)DROrF*jxqUdvf1!Am^>5RrtsJ6jUC1-6Cz zY=OzI>M+&z(tRr$Ok}QRwdMjC7 zKjEqJ^Ui`HZQ@oAGD`~cUH~->f)h0v7(Kb_m`}5ieV6DhAh%k!Zc(UAmdBS{ zzWu;G`>E;!Y^qYvAZm&3+m?fQ;=6<*z2?@V85G*lunF~_666nQx~A=yVl-=#c$;_ld{vgiIJR~PfvY1OiRw|2YRN;j2_)Avd3`bQ7Wr~T%9$(~2O z-Sm~B%8L5Zk^ioxhJV{bARc$TsmY;`cKQnv|p@V&La{psOk9 zVy1e8cX;Ki8GTY4fsE6`N7VAq7ngxH#DFsQeIpo)7sF{4O3EI7CbOmqGa-nXasFzc z=w$&PoR+yI3l?QPcJ){98`aS1i#rGI%^W`hUqLOyFo7QOm|LDAsc<_r&uj0wGDx}k zr4T(g*eJ6(yX}Et`8qXjPMAv7i1;|KwZ07a+S*TB$f86dsdk^Y?x>O7VJ+paCRlxh z*65Qt5CN=dQoIy}?9xQ%Fl3a_Zg`dCkKQtN#D z{(YtM_ZwhTE!{Z*n~hl{6FyAKO|u5rv+9hTwWvv<19h={Y`7u3quiwkL}FA1d6jow zB87*{5Vo}N@;;v>J(H7!ih6^Inx!W`L^H&#r2(AnuIVm8B0LmGkb| zn$d`O$N1}_kMcEEat2ATs{B3&p`C3t!c}Z0H!`DLm5tP_b~#fw<|(6AgHL&uJ|l0b z3qZ5jzb%8lrN~wP&>dh7thh(>vNMMbB#tUA`JK?zUsih{J>*W?z>1cLovVLC&zt4F zD5;=<3~|3S+ezS?e|$C2j+PgihO_`si33|RFK@<4>oGtk!q7-r?~FXr>!w+GhY#+& zr4$Y{eRCT$h#_Bp$o*SiGDpx|vx*rm@f^JQ;BEB}4&-XLF>KlsybJ_QGIAKmF@7f- zCX_t|MHBe)Pw`vLduJsW;#ANb#ym#qX0Ok!E-=`Rit?ty0)p`vPxqnLDSirRPJNNg zCFjxMr}mr0!kVy+Cj@%u5Y{<{tnamxd|K0E3;-YbrRe{utm*4j+4E4S?%&F)=zf@A zF$K_$LN9KB&WuG0#iBd}59o&s=LW<%%K{0<+{cBd6DbiAYk4#*W3Sp`f?# zqbvZTILMpib^*qvQLVaL*Ena*?Nvzx9}HpFhS^8mU)5nARDz0nH6_D$nTC3az-;Z0 z4_QbTC4_R@zHL!aoZIjYg2zI-iOrXS!+)Xm27T10Ulo#U3}x_?O|bX7n?|~UvP>wv z4KFo7>@vJu(VlrQ%zZoVV|218y3NkVpiv~D@dD&Gvm=-ZYf^w0EdKKs8J+yuMIzSU zoHmNxdRkL__Vd4ZZMF7dc#**OHGWb7BDvmd^*B~+7wFHc$t6IEjNNyNNMnXr^v~j0 zZ{|^7yGi>-FYnUDxZ9}zp7tTtparljax+PBcN0Znu>ACr1`b`UK0HlFOnlayUf*!x zcmcjkhE(lbA>+fB`T+6=+WPqC6HjXq94m#v+;xmR6;O(a+Evucaqdvnig33p#7hzM zRJXev2bqhUb41tgF?szy{Nh1t--PCt&wMnI&>p7xPuoHJd$)_5f%VwrZ;EGd&IaJJ zhCFMoYmvf9;y#mb5Pg-{U!b#VAjkFX4ruBbXBq^Y%DBW+kYcH#@tzQM=Qv7NK+b*! z^aaa$sI+ZPGa&1w;=pY9%$vr-82O{9pV}g8KsIrbbAJjR@7vs{|NMbgl zas2qMvOFE}>W;BLxfS&iQCmJIsTt`v+&o!_TfP0ns>*FXcD2QmC|`Ky!0n$zRu8{L zZix}PVn;2WO}z&rRMW26D~B(i2s}IVJf!$GK3-v@p-e0i%w$9(OCgN%a725COylMJ z_%7FBI3B?GHsG+FQ22EkDL(bh(GZHF(1C{pdqr)Pe~93Vcf0mtdqZ!_aoz(Mg@_H) z8NK65_vUQ6Kv|HOpNOQ^JAat{4p}^`h*4*oP4 zyoKEG7|Ru49B@}U<)|zpO=yDw_lKZYc<5?Om;BuLeC%zn6ijFH+H~8c>>|?c6}8%V zmzb-<6(TPWX91kaYuy0RqW|O$%qK);Z}nZu#;qqASc*%A&X_$U^+g9VOpE+4`wD^t zLymsD-5+NXuu`Y~wMhI9%eAq^o8Yw}_w-}J{9)Q1&Wd~8nEjy1SU8h2(l@VuiQ#u> zPFt{>vB#LSOlM&)?4QMt$uK9^wH2(h-r!Q)%9I6NuoJa*D77{uKv8!e#%iM}7et3W zzdqT`;vafLtta7C2%uo*o|~QGsRsJQuFBwj&oHV~cx>5wRvoovDj^YDK>Zq3(XjV~ z$f@3v^Hy}pIj`gV$A8JX zvVhwqU8kj8A<%+D!*6+kN;iXO78d+Z;x*wTS+t?-2xw4+6+TYOQ^ zkKZnH>V6OqCx~eXoAH_YG{r%^!n|(dM70^587aI)LO@Q~XkqaAb8<%tnI@I|9|;ZG zSD0CfIh3Io`o&?3)ayH~wKN^LZt5mYMWB-qgcG7e0UVW&o4#Z+yIOOsLg9Ue_mG(w z-J|#LE?Kb&mI-ttL@%)qcp|aBrFI-mJ@M~K!fH!hXzT%D5I2DMrIBos30ZRFEE%vz zpUJI~jykfnvRW#XeeBMyyswkGRZ+H3&3H#36Ulb^;Xl5uPa(2tTz3xBJ4;+%Eu8{H zN;!(B=tk!v8t8s0QX}c)eIoEeN5(EEFkSz8wDzUVj)UdN4h0f80%H}8WFY*o(irs} zF{?yQE%VNKr$?g^Je;(7!gHKP($}R==i;(na1Ed${8h|50|R&8gCxOG5Qt_au3+dc zsP#G4Vo59NQgWv{h#9U2G(p?$myKsepD4$Y!UwKW>+-&E3r+W697FF?9oBZ=iakKTGx&3OwkYem@;`;8-G;#OVgq}w}w?5k*niiwMZM4>e&@a; z-BxXBE&CjJD%kB&B7h~6+1nGO@+-25a{01Q-^+XxF$E@2yP`zJz0VZvxGW$AN*aWq zt3iB(ETfX7a)LlUPLinG)T&?B`_f!}N+bu)^QFnyx+F3vtb|GQxX97tApZF|cY8T> z?`o@zo76+M5QZ;<21P4|z1(Q(X0_^3MgE_@-SWMB(J>2UaTLHl8Ty@Iw{-l#1&=dZZ zAASmXjz9mA*TR#<;5UZElJUv|OTF3@EWzzt!)R}gSYdQp8YKDFr|%E7*TnI096EP| zXt`5_i--KwhuuhTR04iOhg0vx3TVAF2i&)Tg;TTm4aW7|bNRdBPvWoMC>y_-o+9I@ zv%B4bwm+-5pR*wwK#pPS&>|D`%eR9;-tBsQ&qJ^-{8wBHL}M{9`-;}@Gr$=Kuhjw}Rq%N_O-!57 zyhMWS1OZc^x9c_dLD1uV8sbNV*)3ox4gWrI;c^q_!=oa1y@R_RBR{$Kn9cgz22D3^ zY!RXdiu$XvC0ky+n&Od`eDkFNXqUb!rvJDXQ<;fP*XoS1&|~nd9_-X4$JT^`0;rs> zmWh)Q;drPJ-)n5|IsYr59V*=N0_{RZ!<$A+IO9`vL((j-{@!KH3MHIj=y^Qn{!{Pk z^}l5pa5hXJHY{7f%0lC<73;#K!K~t-a!)5JkjYj)So(_zmYkS`-}sbIRQ9MJNNZ_y zfs{XgpsNocEx`DHFmAHskdyhJ3Lw5Wi|2)+-g68OTGZ}8ARCs0_trN7pI-d0NP!gP zgvhSjdA|O;O)JoqlrT>NPbl$f;nO2d9q5rfL}pS?G{d$FP0K*6P~+Mo-m;^&%*ulZ zIV!o&|2q`FA;UKjE-=4Wmi5LJu>ZJ*X+x6-A2;+dR@T(9vn(^}cfvrD0 z6RdIq+VH@h#SWwUHJuss+7Ho|0b}f&qR;pUf%5l(HGk?i;iDGY>V);(p*F%!WMvSpEV$JKd@N&!|isOx|0;K}cSJEAh7at!b&iRE=IKcTn$D;11~>X7?( zLMMopZY=rPBaZ18@#_Fi%i#bzzmo#UP9QUmGKXP$U#u|P*Bz9?Kfj`FPCzc#LJeR{ zuK=nO=vX?AmO+d7xSOT!^`#>fAkh8^pfq}%uWq`jt%E$(R0YS|;}QiM>?0Q-s-bb` zRebx)dR}X567SHMw!c-OoRG-pyO8*J$TU-_uV=!A;g2JU*+c54(qrz}ro~USGLacS zKrU&dHcL)zt~p7DX$bFv511H+r_{(DRXS4!pa!xHaLArE!ebje1yoNLxweQpEaGE5 z#wCn{N)DHnPC-fszw91xRudZATov~#0?VQKF032_-O})}f~(Yig6O*gU8vNs4|;v_ zM;}sIR1l#hH20Hb908VmwE0-RwoEbvn_LQ%>}_cJ{OXF$8ze&VREOn{-4c4lskm?q zFM0xl-MRlAt79L>>UuV}7K;!rS8{dftaQ5Df=1{aXNZL1)BF)EuH0_6DWdHcGfjds zy^M%9Jd5J;fIV}mjc0SuU=ff~L=k!an=k>iRV;tuD$}v5f>&atWmAzIPXiQNhC;Eo zKPcIQfG>qO)sJ>#$-t0QE?`ViJCnt8TYWoS5d0lN>^zNE!*LgR8kRE-ix;KYrH-iwRuE3D@NRNa8a?&8T)&9N8|8RD-Wd z$&*rs*3HNA>VZ3q?$s;u9;JCm2aBleuyLS)z?4LDdc!r=hmukvgae%2YJ)YvX5OQv zMMT!EV@2xlHOFWnSh(cj2=XQkh{HXY%t5PW__&%;}w_`F!!}=B#Un zpySgx`&kh}rs-FQlu7+*!LSY{U6sc zgJ2)Ii6dKLI&Y3$)2rjJ8gx6?9b156m=is9l4E=J@I+@0w577epcCzmT(g3D-A*bKwc&od@lTh}o^w871lIkKie(Rwm=N4$2*Lx1@Z#Js z7qCZq{O;4(yZu1Da-OiQB-O1NJHD#ztJ7CZFR38_)y#tlmGux{smVIF`i0A#y0M+PP zCW=4cKoK@s6Gj2=lDarn=>kubl^5fjA%W-rp7LV%5;SpS(6^&y;G(-WQS%3V!eHAb z%G=qZsq?vJZ%mY^RYR(~LyLaRpH9@l^!qHwnH)V{L?Z=xacsx$cBvV!TmQy$SiRdh zc35wd(P}5j_V8a}9RB7vWctf3dI)JhmR+HQoDCA#jOhVEaXG2IFGx@EVCmM%BG z=2{^l)2S4~2m#ZLR?WC^LCVR1_c`SryVd9H7B^cwi+3!iA5p|KM$FUW`M2BIwXBr* zF-gYR8Mwaqb=Q1ylDfMEEI~!xKP*B z6C8zbHeq%;U1P|E+hE0ru?ye}q$bWK!G^JPXE*Ft z5r8}Q%|AVS!RI#_CA2Cu4vDh0^fVku+xpEht##q&7lhrvWieK8_0W@|P@4l7T%%+_ z1#n;q4#$9nFbm&YUa-e4}l!VeY=Z2m8eCrga3X1 zaL!A%J%m3!~94i>?(txO{@)B4U*;lVnE2seMlRCM>z)1>wAA>w<-g@)8 zjmN>UO;@|c#BSR_hnCi2baewWy@A;bRtpZoEdxg#a~NdG5(^q;dCjj_L|Da$>T*ml zarO>Khx7C;ZKynNMM$#GVLZPr_bl6?g2IF#Ns3F9OZ`iybi3&KfiD6-CKx+#{hE{9 zF69y){>Z;_FmzXt>f$lsQ?OZpj0R)VP>R(B_KRSzpJTZ_%sR7M6#;b3*tooH+W!$! z2jEFR=1TcOEZ-b2c%+4j6`O#c?3dVgu;~0-fF1g~0P8Ve_{B%(DlLW|RmU(InP*(C=D0h<^Hw`kuLg`;`4l<4ps{Xl)+u3?4q6;WFN)QocM*k6Un z?98c}a>{YKeTsI8x<1?Q=MmY>wo6Rhhb?L`+WA~I7>ZXwF)@;KE8cjb`o`)D2FUcR zg6MQi%^Maa{AOw342NGMpTr!Gi?YN=+hYdm#nF5D5lE6zB`^;0*N#JFD zr$o$}a#}gVvrm&;)`qgvljZU20JLa`{+c4n`f@hA0S{&6z*;H+`^WiKKsRfT9S5zc zy{ez`jZ)20eeKVdxcG{y!9kLmybjMXInd9jX>fa+=(3wpe)|(FVk5)jl__`TQ{9BR z!bzKMpg$0u512U=p4bY9P^sr&(<5=cVV5VP@P=7(KB^BgOTM%C80`$m6o-PIae+AP zh9=ts1-#@opQK{yS4K4Gxg<@e@?YorQTxUEI;IGzi!9CQ`icEGqO*C6<)J9B1+*0u+Px)nk?|n^*d6nN* zedId_Z)$;@+L3Ez1y^f*(v_a5+=kh8PeiiNiJE-xhvff=wZxb{AvC7^FD(H(k!4cD zk1u{on0~gxN7x%tT4Eu}B~PrJz|@6pV4p!&YmByf0g8s=>`yxwA>%u5L<)>s1KAGh zIUsNRFV!emB)qYH!`KwJrfU;TNb*W??BISVFppg2AUrbs=PFfrIQ<+Ri@1?SqDKYb ze=D;SAp9;(9DUp`!rXqNcZf4~+zp}#@hIEoD6IU+^G;Vg#pe$6fgR2sL`L;zRZ#^m zWCU{a`&IYy_dJIbA15QevvF>WIqKNfnsLq)1xBgHW+;#Dg*qyZEZGBdbC7QUgGvP2 zRx3rO0jsqn|5~!yEYF=a7=a@(%lCmT$sN^*8579w>;2dQ+R}n_4l=MO*hJ*$4y69c8`xBCbJ$|h$Y8Jc`3eXLKWnowlm~|k8K-r|Q zYJtZ+1DMz3QqG^hibE{3cr>cWA6K|eQ!!_C5waV=mG9ydUy<5`nDm5|X4%bJ)!!l} zKEtaWI?!DEdl}f2O~RFb?{7<--Z?_s;9fGcIUV@*>XK|6xLU=htQ`%*d`l5m>{s#~ z{}?3w_iW2?eu%xAg{z_eHY!glF2<^jDz&I#d7!9+-0MV;Y&lR6dST?5S~33_PM=&7 z!yjxp*>ZS+{L@!r72`uyoc*^%z8Y zQ=ftu19|;RiWIGN>Og5AYtfXV132)Yb0Cpja??+2vI8=VIQe#1jK6?FD!8GF)nv7> z#AdNT^7k_0&9_XhW8(<5?g}qmuvU?OX;>i4f7c@c{zmVNx6f%DU^@f? zc&k=JSyNE+u{2L9g$=pXKdj2xm?4AkiKoGHc#lQ?PotH~l3t=6mp!Q)ybu9&szAG>c`uD?Rz5S^5_Z4y7>GfP=RAQIfPgAs?yi>qHeO>f=z&^&9FU+JAk6Z8Vq~+^#8{~f)cgjba$cz{hJb~_)iBD9g>=By8<*8@kGm2l+o8{vr zOdumD??5R7E*-N|Zl%WB^2xzcvKq*rJ4~<5V0Hg?CNH&q2?tYx6>n-}Pp41nDqI`S zU0)fR{YlDIiuPDq!e7}d_$zw}T-n-H5{7G(d-n##Xq)NQ>P^lEM+TlZx8+vuq=4)C z(&JQ5kg>U^hw#|hXD1*F)p&M_yuIznm?)W8-_;fZprdF`7;D&Vi)l? zHK|vU9j6%_8&CQ8Vg4LDQPooS=i)+y{vPg})0!G{AM&9kpUBd`_cm|!-$`?|;qwSS z@_?!W6^37ER5bJKgVl|+ulEMDMmofQ-?~jqhAQ9*yKOM53LMQD8D(vwhkxm+|2^E0 zehsOD)$*g%{L(Wi%in2Q(?DXfHR)evfPe7UT}QCm&F|hl)t2hR_bCpvB;bT+wz~h% z34fNhapy2x%}LB`p2_X`N?l4s&X1NIaFKuY^$nd`T^F9myJNIOUeFa?$H(9WUja)8 zA>*0HzgXTU_%QLFvS?e&`9u4D(i#aUpSB;o<;JV`409aIwHecbY$JSirSF3AyY2G^ z0>%Bqp9tA)?*^rltiEqhB(XJ;9~s^S+6_%%d9CfwzC0}|A%k2I*pGNNc6@KUdO6`8 zI&@T0So`9+2ypkdP5g;vO(_}xF?2sWuew@~P>{@jF{9GDbSJ)(d`tqkO%~fjA&@<0 zuSzWfAlb4AKDU<)6yN_XZE~)rIq###=pQ7d^~>P+%S)$ZZdEJ=8Vs5k>B-{p#$EMg zy-@Y=YOt4*TVu8ls8nk1^z(57LVWHFzdvaJVyRbrSV`Y<&tiSa89ZR$TVKOVc@ajYfm5< z)IX}D7-hkUUSR0F@@rRG2u;k=h5XyYM84Sl??ul<2!RFsO3v*M=f#7NL2ayOIPyTL zF_2xD0DXju0(-80Lx@jG{283J`W$Jz7f>Y5&$)fxX&}r##10?r=%~;#{LRYyi_YJ= z#PG}>!jLIAro}w=AUaeLG~p@RdF}$*yEs!L&V2Uu-GyYv?EU}Cl_?1+)d z)hPCMm7r=Kw*p;q6uAzzW((J;F)I`h8Hh2=v3p#kGioCM)pCvHFNW->nF7gJ$xP2A zgzGh42u{f0q#7OAPqHxV)}M9Hm$zb_Hn!>r1HeYbV8q|^zVz_|In(%2Mfr|1L@Lk| zmxhS>rm8k^RgL@YQ<*`gmjv1k?@@%e?))r}$e4O~N$&|a_dOR zD71}!W)-Nz5q0M+M`w24_j`yNxIYbh#9MaxoJ@zE(yS$gDc1645b(NK4 zxcjP*qF$qGAOS;RQg75p%q-8*AUgk1Kiuo}h8xm;^BLW(6Z&p3XQLV44SreA)eUyr z6PwtjIq@|yf5x}f_8YoB3cpg&n=5=Jl3kn!d$-NuBTge~a=}gFz!MNTFmSR)gk7uq} z2PM>Xq?V1`(R)|BTGa?mlTT8fWLzbxCt_ZKW!#lO5M#cZc0~jrWPMcd3yhtk8<*YQ zeT5ZH9Rts-0ZlWy*0RVm?Y%E8k!p3LYPj)?^jeE^j5-%wufB}zaDz6k*q^4ERcNaK z77|?Q`gI|XW@Ua;tfr^=sEy~^T6Ip|UQl{%UvL3~nRCh0@B3_;94uQH>lr!AYoyYzTZ z-Azd;9nBmQkD;)3$2&s#F;Cg8L2O1`qS;EU^yrp-2ScpYe0`}|^3j@kUsYgQsTUS} z?v4M`*#}$S;ou#P(WFQ}ZQ$P{*`FBj5*p4%UsiDA?S!pV6_Kt18(PN*MdJp{F7az_ zjZCTc_W<*aQa2ypfh;wfDyf{FMzA6$b=B9Xg7M)Cm(QwlUQ1gj?K0lsZv=I}50>vV zqWPpXQGW_Bkt>vF$LL3mJgat%Z)aR^YbxFe$ICZ^CrgQA*YY22sMhIj&fZ5C1~wCH zikS!R8dqOU6;7?Oy2FLJ^nGZIPTr}qkhpnpxSPI!A#|G2>`bG8(Id1u=dK`D&uw2B zCaw;=G}yxUjl}8xjZ>h(-R;g$fN8#l*8|P1S;BF{eT9&*k>nE_JF!@o1xsxP#u0tR z#Jb-xU&KmAq#s&c4ldKe7As2O21Bm}QgOW;8u?m0_b)HTg~>=>W5}LwURWGTgR_}t zr3F#)%x3H5qcI^nWM-G^Oy`uF)(B}_WfxnbDe1Eelg@qn=#725o}v>z+Mfn<`(qTr zacJ>T(Uk4JOtfHt)=A`+n=gsx=nJ#2bGHlo)>`-?ix0zA%FB@om+zhZgq;3-u9GQD z6cg`M+KsTXttv{k_(oL&oZd6VMcROx)yc8evJQwO*|_~UN}xQC&p-wNWz{~bl0F>)X4Jk%q^bhL-j_h1}bk>pad*Br_ zVG7~S!6~~9=`oPNT&HVg+jirh9Um_)(XnCf6BVO?6m~7~7l-alr@lYELnnvg*k{|c zd?hEP?+w5zZ94N_AvZnbN8MaZ$kqw%+Y`S?r&IMpjqJ*TeHRV7wCeoR-PmTup_6`> zrpFK6XD;hS^OxLuAd5on^GU}vqwDg8YSys%s>RtEx4Jz$%<$)I*)8}pUSE{ za@P=lmCZ2lQjS#VyWFOjuLJ@<40F@&VBS19s`fsqj;B$~TvCx7uT| z=qK5_X7sY&d(xf6w@6WE{2gB~)IMyOHI6b1!u`g!IO$kj9C(VXG@oOd9ZDVZGN1N7 z`1jcN8Z&}pUpfBYW8d_Djy)duOPp8+Tj$&|2XY&K@?%`)!F~}Xu%GoxWAF=D-Ze-q zm~R?2vp0914)&#=wHud3b^`L&9F5hN4$E+4go35n%a1N?}d02=XQ~lf( zY2|p^;PvzZM>hF_uI{mNqQIZ7K!ely!L83FRw=-~>;v5T#%aRhNnq!V50BSQU^1=I ztX`@ZoX*|A8OdxXiV+TVmvnvW5%TZt?r~$kX`zYq1>6YF3UL=bWl*(fqnog#-oPDn9k*Y(=dpTu0H|Fflm+io1j~TCi@cay&QDpnM22~m3 zY4EhD`u6IThYa46tE=6Q8T7p|U3RJR8&NZtBBK4EC;Pcl833}dz0;dz*aYE<81#;kom=Eh4fm4p3~C;IiAgBl zY`bUU$mc7(gfW_kZ|%t`=K2rJgRb%dMHg zTxiXPUA|2?*P7WB*zUn8)mG32vdgQi7|!Cxs=s|!Ics{@k(zNQ^^9o{oJscsu z&yc-{mg>Fake&+0sG+UQlG@wJ9m0Bj@Iyx!HO#=hf{icq*yb*}*Thbz{M0y!80A(_ z-}opw4%MI+%C>?Qh=ZTnN_ZchTpE}%bx8cW5c%I82fGvM?U51u7WRGg+5y4+`kCQn+PohKt4&?I*E{j(q`>xZ zLXWY0n>Zox06b`yyuC#CPW;{EcjA-xvn8Lb+<)I*s9s!4BJ>f1g(zii+D8oq{Cxrh zHFI4K6Y@4sVm2zHqHPmX)r(Qw>pf8ADa?)<6A8Yd`RCFe7ycF4jEsY3F0Tal7Wiyr z$TR$$3Z!{E0}!|NjQrOt0H&0x*{s8~LtQR@KS~>3TeV9KCz)~@>Op_lUbm2PXW9T1 z5Fdkp>)RQ|Uc9Bd>(K3zaG@@^B`_T47w20Sq=-rD!k@s1a6V}9wzK|s=HNkrqT9SK zbyu#^eKd1Uyk2z}sujeNhim(zi*(F}pzto%*f#-h#g)I}8u49{fv7{bTD*f9 z5?B)Q6a;2D?pB=vEqHun@<*5Aj^t>j0j9Xn03#(|-0RMv?I5nWJ6E#SR22FG$6v@%Q;a!QKyJ zhR3gMUr>&rREA)mw{9}ArVEEcCYBG)yfxWbsE>WfGlP!@7=hMo1jk@wTKBV*@6|(4{_HWO+j(t*h z({5$e`f|d>CU}z?jp@GIu#GX ztgwJ<_qr$9oTQ!|vU>d6M-imLAJXLnA25wc-q!v#*`MIN(E^@8e z$?)>H0Gd(mK{A*X5T|RkIKegjQpvHAhRuIy*CZy7>i-S~2iQ6N(9+v;x1Rh%aOGB< z%BLBZDJihYNtQ5MEd5c!u5CXx9H;4z-6tp`{S17ZQ))3l4ma?GdAsoP8i;g}z)3W2 z55+b)y)k!xQcND}$@491xY8RWw9cl!&|Cv2SyR6uGEYM9`#`gf@{SHTozQIa6vOLm8wLd0&Jaf4`t3ah6i6QIq=M*oc5m>v$r=)ije4yQY>)+Q%|KHb07Yb(04urxPm4Dp+`xIE_kYlQ&rj8%z z#<+Zzl}3s~U~u~6gB?+4jd-vw=XXF2KK}^0(}WtFT5bltC&$E~Ddf@+-yuM9qCIAO z5iBG2OOFhWyRSFzOV84J8bGZQ$%o$06%>}xT1LtF4vT7i%_r_p|L@zBB5fu90i!c3 z_PaH{&MTXM-B?*~oZDoiBewZ;2@EVJ^#g_sXw|fLRN8ir-_W8@|0ugDL`+iZRoj@{ zC{OLEIWUERCT@a%B)($?VSLmkMy(;~J3bSw+zZn`uI7`(Xb*JT96~{tNM)W1?=SSf zxA9RyY;4G9$t<_7gHp*L@@u(Nv>@k=*{bkeJJbV>#9?0(Hl5k~112V=v5aiU+We68 zDQ_{JgLtR&;ss&hk{ChWa9QwQbV?wik?g4Gbu=%min@F={nr$zvfxIt!yVeVgHWwz z*6w=S#7qH!QOS^3w0!+-(RO3|8Pts&m0U==J{wauYe)lLv`Hoe zLRVON<=K;?5@+zOYkY{TXP!8CGF&OGFie%GK~07^xV@H{Cc9y>&4j%0vteqRgDFRq z6YL*wUX+@Ug=K!%os-9$FQ7wZn@8LG|ARgE7M-pkCf$i z*{ke%S5)3U5|~O`+#OPJ5gzJcnnpeZ(-lX8=XyW7)WggvTmw}u?z`(aOj2G%zOzwR zv&UcRH?~@3tIGv(9yii&8@~Kj`$VoT^NGH3U49JQNEdFz9xY?`N_-hnih~v^-x*^V zd{erq&VCjCWK3rLVdRExwDZs``ZR5}5ID*Z)oUhx2-Ke(WNz2SBG1Jx5Bq`H_uK=c z7Asdwj<2Lf8LF1CWPj}nT_uBcVN)HBE=^2bjvS@^b1XTzaAPn3!V2Zb$gjTA?$l9+ zA1RbG;B&%mZ;v-z?oaL@P|P|9zdH~@(`+k~w)(pfu|ePEo5;xTL@SU1ufkL^=EF1i zqIjE4z0KXk0v*Y+g~@TVGvy;iAQYN-ehMzO9(J@L&|oEf#e{f0ba7u_VJvk{=elmM zi+`R%#s=!%Z9*6Gkt3M8Pa)~plL)~z+Ca(B37~#yXXNO=v2~(Nog{sl z|F^XcX3PK3x)IAxf8z0|Zgr{wV^IdW^J_GpkP>vip7%azB-Qr@HH*fulU##1Yb&>mw|;oW11*F{3?2Yiv!q|Z`2+Vy!P zd0{9@mvdhpkDn#GBegU18|}jtVwJcmn9v+hf55gbBc{?<{(lNB`F{%S8ND1Rf6e3J z$Pm_HSnZk4$`Gxt^5u3V_s4LtJ{oW2zXez7+K9~jL^@j9W}E0-^2unm!N8oDiSsY$ zRYZ&*ix?)fqz=SrKsJ(2?pXZE%UA9n)54bnb3(oLGbstF5UtImNB9|tebx@py|3-w zy!+ynSSn67D7pyzZz(U_!6)g86H+SR;w$ykj!xT~F&*6A2bHNx^QoZM$lMa_b{X5B zIc##d-QJGWB@;NBdZX9m;p*U{Y%h?}oYK;p2wpLpN;U+*!^iJeGLFnWOMaN61bk?R zPm;N8&epJ9>azbN#P|=m+(&DY<>x7tVT@PS2K)B9Gy8qlLcCut0CH0$Bd>T+niHqU zK)9ZjIHtXGTRC$zmp!@0W8DNo7p;E&lKCBKgzg^!Jz#jtR2I4soAT@^dzfUWdh+E7 z6`>9!;RLFfPWW=_!`t)U{!oGMeej;*ki@kk&|ewv#P4z!oPj7mI+3|AXszou zBn??R6)b2c^`|HYov#bYgJ?b22( zgVGEYlR3()=l7dwMqtOKJTLIy_nAk~Z%SbA5(T8l@a3yE@_bto6Eai^(Ybl zZZV6pjViB06UFlC9xdz+I2q=M(E_sM++23OmEWyVh~e}3w!lc?cM~|rl>?rs!MX^Z~}Lm}3@5RK;MeczVv_sgKB2)~Wxq7Ol>X|G7brlbRElLa?u-65L` zg1j5d>E_!>X%_mpbm@=h-#Wj0?S7Ta&FGv+ZT9 ze(1FZMZ3s@Ei#Wkm(7T-2mXsdDB?@)yhs0_q8#EGoDhqY*G+AnyJ<&5oXQA?mLl+Z z>E$29%2WwQ@UjnlKq<>CK7`f>`~)IA0gH~oOm<{1*FE|Kkwu=bbpo0f7`EbG=(t+` zl7QQO&-}Aix$kQ}yt+=T4dI;IhD1U$(QZOXjJi;!@JD{5sGPUTw;SOihD|>%vmKf_*SjXRH?T!*Lx2(KrYvP$OZobcU?=bF{ zZKm~bjuN^zvs!k1I{`z}UA+*eB(5Poq5*aq3O^ycWZsxERXiYdHY~;90ro92o&uES z`)IT%vK21obvlV`5>o9bDbk#tbi%r$mCL;|rz;k@{H;4sHlgcv0`@lVhHZ^Y7E16f zp2@~HL~_MOcU8rjkyX9qWUW52j`O!s=LrWP_3BdBzISXM`e$fOlud3wHZ4o=yCZ`g z%Ku@mjE1<$LXD%S_xWBb1^+hNt`^%-+fgdmEu%V0}x(w zvFh`#8bYQqPD0NAFV_{Q$Ss!wG(_=n&_F>vPXPLuCHg#e#Rj?#+q-0FN>#qv|LM8H5 zwNmQelc-8>GB3@>yj~Y#Uk0f83R}O4NeL1^bXy>JhD(ere<6qo> zK1uR9y@eU<3?+eUr{S6AL$hgm`J?Q{BDQtxM+`6=ChNUjL0r`>5Y{%vmyeTm7b4q` zjSC+PLiOxfM|2j~!Ho?9ZfphL;5f|=Xw2yuUx?kAKG~^)wU6N?JQVM2P|-e0MU%S3 z1ry*88W*dRuHhXDECM=?=j&2qi!E1kpd-bLdU?2EW*MX8;H2xlLvKwM68?M(OjuOI zqGeO&tm_HZuQmb;`pjFUZ%GYWx+N`Ik9QaROdnrfGj1qmOM6tJm3U*~d82H!RbggA z;dhHn;QOjsSf|@CJZ#ZSZP15FUqX;Dx7dgE!_D)-4P7r99H4&^vbBH;=x$OfzaiV@ zJq|T2-jS?}s4*BqX9_s6q2|V;;GLbLd3ds?T8RR>`Og z;!02!9yB)4zNtIa=h zqmoO!No_T8FQ3+}NISa-Wyjpge_48Ws`|3izb(fR)mM<`MGOSf7 z^i?XO z#c2YrcIxJ$e*2tMh8>c!4oXVqv;0zMd{c=^Z%=DH^>h9F-Fc0gg7II;iiBo=;hY*k zcG$ZCyk-oWfPhN}zv~GSz+Ba-SEVF3skva5LghOfH6VJ9 za8Y(8==YF8&xb*`BlBgi_6~}_2Y@r5KBHJ16i4ZyhWh$F$SAzq0Clh@d;jy{z}hF} zMROr`1Z42N3r?Mpy>@=RIDB4sU(*U#Zf7=XIK8nGV%mQ#?<;QDBk&4o*XF1j9`6V$ zY~|`%d@|qNNn!>z=UqafT=x`msPeVXkGHJ+h?l9K3fEO#Wuy&JHpNppBOvyU;(D4n zuv)%RU*3(wHgybD1Y#}p*VUnjj;fpFl_f+eS(PzpL}S<36j@J}wbo@7V{AhvODLQ& zT2@u38W%eyHnXfW$cMKVi-%y@7;qcww09EB>)**usdAPrnpt&ZO1v2i)Y-6I-vk-1IdUG>FPWVA*}q z4{Fq%AWz1n#YwK#`)Eqzn+i0YvK)m59zj7!=#yK_jB8hM^>oI?=;_Qc&w9}9+|K`U zqhj>i%fsSJ-l_s)%xRfR+#o`q>4}@d33feLe`bpJ{;?%^q?Obz6BOVfG%c>iZluUq z(zj&FSa195?trP&;krEHXyc-p-elUY$b4x8B4d5vv5g4V{|k9MH;C)vTm}Yj`|1r5 zU2L>-83j}OGeciuhHyBUJ#7t4Zn~1sa0N8dMlwyNU=P!{tEe}y8{2?B=4YS$@nQM| zv*`kjcN!eV6MzG>wV2lPgp>8k1Ci6F67wKP{3{J}I_CtKkDzu~AMfrsm4}-j_pb-D zqXh&a`R=Hc4`!UgX1-SPVw9F1TJ_6ZyDC` zLp`b8M^y(zV((9j_r;4(WT=(z2bj&(Tj=ntKECZvL-m}Li1Jqbk=nH@kEI5YZjbAU z3)z{C*y#;0#raaHai_ zc=rT;^W&1gUifb;T4I|$rX5W%Dvt_yoefkjp7D z`{JMgmXLoKfQE^;2F4tnAuaY^t^s6@af$o@$KG#S1D24M*Nzua?D_!sQC+|S7X@>X z5_tWuz0WpwikH34UY6yUgy$?;&mIg(YTa(b%%V6I;W)_sdYi)wTE@7vCeOjwwwhUn zhBfYXG4Wx=pW*pKtm{T!nwG$-d&SS%tkYi}pj`MRzA`VF0owce~+lJ3oO0da|_llh=T@CYHMZFSLFhR?e zXSHLoTU@d~gI5?;@M`1FyY0Ui{AT!ZRKFY*AXy@50eX~k=h}2!&aGy*jToX+BQYSJ z`?PwFNHIvUN0^Vgi@K0gw#go9K%+SK$|RH!fg{Np=0PeB}BhQ}ELa8!x{^ zTlA3aIoteUS(p1q2u^){q^mv?t@j`twjTYcj2vnA2d&4>o)PVZ4Uj=JR&3^Zja0*| zZe#7%RSsU69z60&f>+h?U$uyE2!Fc+WJ(LY*nuU!&_X>B!<;E^m9>C_;77X5ygq;B z8PO6gxG!hSr{R4V#ZFA`lj8BPY{of1#Jy`(5+af2*z{KW(V5M`nb6CwxB{N<_Q_4o zvXsc?2r8QU#v!jC0w(6jpsYu;CDtQ#p_VHxKC3@fWIr0c*ZwkOnD}vmx5`hy@w20$ zWxIyN!1W`;Ysb$nX$I-SX)#u02~pdCp^Iq4F?R`-+s#SCvYpXMTy~G^ui(o^_ z1>|r%D#jHjpe7VoTk=f(jqcX<%FE;hcH2cK*q zMP=9AnJSK{?8g@ygbw@lHVd$tow@owpI>5I_ACWu-e*&Xrys2}?jFp~^c^G(Ov$y+ z^UKa1aRfJR^w7mzOUZ8Kw8#0L(1Lq?;*&5kH^B(WpT#^~PIerW~4nrnAg z(a**e(8HY?XWvKm&yNb$nBa#XUmIo{X=+PnLD-@D81aFHug@{w8jGSBsQcR}7m(|S z)Z3^7F-}&LDB|JSX9UL5eoMF3C7F;b83hHo;>J_1)q{TU8@*BCrH490i65EZFwAhO z)}qd+k-MZq8C#jyk7Fv4ngOEj=jx9Up;3l;L-Nl_po7PCP}S<`Fo`*(<#BR8Yslg? z;&(0+D#HQgVj45*Ceu6Pdq65;2y*6=?87SWLygZqpP_&Zb+Lb=|8?e_RNYvzJYH&g z`EInrE~c*~uHxCk9gy$@CBw&5-f6&ILK=dSm@o1^1Ec%Ug(V|`EcVDDOOS?$S&3$3 zics0#JLiH!b+UQ)Ft;4IT-Al-XB5JAQD?NMb4?cd9HrhL$V^sMAX{uwtC+KdAkp16 zTM6_26K|c74U!d6AgI^3q)A0#$qUm;vQ8_B1s+X<@v&>kGxF(1a)It*k3|A6A2_XB z`(BbR>EIK`Htvl;YIsQN{tM#AoQI9C&ITnQj#JZSzpEtL(O@8p$HH>nLDa9VUs!e? z5!zYPj64xG1)Dvu=1-a2#z3YBo>Kt)pv5eKDBe2s$G1 zU$8CCNG;SQ54C}=Cd4liXT^HkX2k{q-_emt%?{hNz%P-`1@KzlgI#?+3zkJ%iLA?- zJ|`}`kT*RDwr<;~ZAZIyNghpTc1>)jo=7^KFue<^#>Uiqy@>o1H#5_C@Y(ejRk@dftF}p-gGBrD|l^ z+$XKvxeg<$l>Sj!=V^`yLw584*MC}>HKX5OTgR9oY*hb+lY5M9EK0>p{NREJ8H_!* z0JFufQxeZ^KFRwCE&&GDUKW`YXIX;@XSbDm0Rf4!N2_RPPJ97foEyUo?W{dPAe#*? z0WzZ5J)kb&+fA@2SN@8x&t}r!(OhV;-QOE+qIdQ+_28lBD-d@!Y(SxvPa_!6Cv*8H zs4tRp9bOw#Y<}%rI$=x#GauOKJ-}c`G(Z-?Cq=jDE>x7phy*3Q+>H|A=z&YnX&bS) zv-JoU?{8*iV1^_=sOjnAxggjN3VAF+qZRb(qUFJL-nip>)cmB&cpA3)D7Azw3V`nLFT8YUxar9M%I(-YN z0lAptKqapL%|=*WrBQ_cp;CkFhd(a%gK(DxW!kyiZNWM6HNumEwgl1rGjHd?tC~yZ z0udbR@8re-7h}uLU)qH&iU?D=I6TTyW=Ehc@5?7%;Q)An+BJYew6LVYW_wT;Kh^}E z0q2*WF(Ro;p7j_N;=NF_5m;e#1Te8KDmRC&p|nESYqdUUcuO>9?%mhIedi2+(VMQk zyfGH%(?A7_{rUE1q&}QrEr$-!WR{OROcPDN9QW}eH;pKdJX%ySv^Jf(pkOM)(#F;; z<>Ngv!E8!Hicvo}J1lv+%*?8@-9h5}Q{$>n{^c55>DX+DP8a zgch)^uvnw(j*J2~1(YE~megr!&YlHINtu;4f@v(F0w{k=YRP_fg7rc>5XuH=kV#-$ zkCtPK;HN62ba>|}suaZC{smHyBGfAV6Hh^Jq3hJJU89=mT3f;s_xFH%koR9W4mI6p zIH+jG2ULq?j)DFq>i_BP_f#_&X=;2*)nkKsb_XUw(+m=*H_Y)0i=yUdNYq>HkJiO7 z(0$K-L4w|#_DdH_b@07^01&MEq0`N38o4n$?p7%a7<$CtK$Rn*l34{xyN)LB1v0g9 zB8loKW5MbOn%e}voC2wE<>ipbx+6fa zqFLj2Eu(zggBL;TKa^kXa^|^!3Q#kEHhK{mw(3S)H*o~bY1~YcBEwD-&$onBXsd8q z$gb9E!g$J&%Ef*zx5;2+ zuhr*Q%>K(X%dRD9bZ`Foni!xImS1t7*(C{_9t0kmlvyq2Q-H3E1jrBZ8CzMEb43*Z z9%)MfZKnr0zm#nz*;IrAn4f-t_0Lv@n=Gol=-O7;`dt$S{ z){Z>UqSrdvQEoO{OuOiK(+-i;V7 z+8v|v^VF|Gp70b=Q=CD59aT1KpRCHQ_cWK(Cu^?)b*7aIWG2!|dYEl+j!yCtFi&?_ zWT{3EoSC;PAO?O^<>?NC(z2g=0HAo(?pck1Kg~>5Dp{jIoCyss2SjOrM6tPUdO{^G zs{-KBtN{2_|8gXlPs4}bymHFFf(~i)!l`8V)UCPypatDUKnr>Gj|%&|6#y+1xM_AY z!bv(H+WXq$@!K=oTg^rwhDo*l;*8R>^7AvGjVKH{VtdbH6ZAt}{J5$Nfj_2#Lym`* zZ7U6#Xl_q@5ZmSZI5n{PnY(bU)4R+6k1yVRV0V){JN=)2c}KFu4R1B*UJ`iquL$hr zQwuN9rOs*M7wP|Dg*6|{xhnyNmSD1;683tbx-s77VZkc_1c~Iuk zr60G<3aEZn-(r0)>fVc{HO|a40k8jJ5&u3?ag4sj-^*7ZTz8@b=uIh8rIXL@(~jZf z1jK4x_qMebul>-u37*D zq5VYMu&1Cbx3f0bo-yuJ-JZ$T&JxGsB_sE+<`j67k(UD_kG$2U{0APNe-GjBJ}t<< zfNjX6@&rio$=7UNJ-+ouQ2y{nx0w9at`_+F(~>VgIMz;_SwnZU5$wJ0+ZOOoD5-45 z+&lT>)?XjdKZj*`??up-s6jTTp&(#OoB}CX650K)|FsA4@b;u@@h(5R*xq%w&AuEj z)Cf%V@(`AK~#GI8~dSt<`jgP=3i3( z)u(a#ma7BJO#E{E3adMs*E&nF(bjL?#dzlHV7I=@9(6$~j|d*qKZ+_9fb%$V`3J2# zmhC^8xmAKRi49-8l8z|bxqQmPpj|8gqzB8S$O%>8lT*r-zZGuF@BPW z%~?v`S~r0~rH33D z0sb;Er76Wxicm;4y~P;8 zNlAat>MBqEEHuSA{+;nf#iHqz;v2w3NLcknh7|X+$yoM0Bn?Cc`n>#G;rZ|91dkx| z6J7P{lWoWjJqEo`E%_k8jaS#T#NY0MVrwpHzLUqsIlia2@24fH#0g`n z`mR`^T~5KRjW4Td>#xf0zP@z(x|82r3Izf6ATq=WjLZrO{{z(idx`V@K$9&MQW)xG zxT>Ck778&pSY5Fz0(u6!zA)eS08@)@DkLuuLI<2Y$s4PC45(5LKo_picG(coZQF=LqFfMX$zJ&#EdC ziAd)+d5THe1rCq?=oh?uwPy)_NO!%*yVO5V5z7WGO@Ka?Do>c7^(X_Rr@*0e8aUYR zcnU+L$*oabqSpZb-igC=7AS^GJ$8+6I~p1^v()e8SvJDm z{{?(gjSO^A?_E=OJmpxh$(!r_0SuTJw3F-C#*#J{io_nhxpP`daZ^FR;^)yFNcVA( z_M-JaN)a|xADz_s9CwDZOmT#b8-=REGKQ685_xU!4VBbl_ zBo=Ol*r~WoclSDlh&zC%HKT5ACruq6il3(shIL#|uco9;@)4G5%he-fL-QM4nNROg z^ed6vh`y`$tmdKQc7NN_qL03UnuF4}Zcy4tl-;`rtF%)eCw_vYCsDR9BT1VVW@7PL zUBHsd#f5TXuc!Bcmnq|u^BebxBxZrE@i^F^J}P2SAL8t<&M;#T>Tk%0L;DqTa5G5A zIDf6p8*qOhbQ&1aZ0*GJz0(cGV&yq=b-t_)J(3nx#i#W7v(T_m2kh|PC8Gu9%?Ewa zbthN7|C;%L>qQ9u>2nET!LGnqSo7Avy&`cBk|#LwWk(yHdq10ol@8*f^dg^IRQAq? z`RnlEG)yV!bN$y9Qa-^XHl0)FQwSr7y=eia2oLXWOx^^^l-Dy-61+Gw`Dt{{5M0*D zM50Wi1|t!if|jlS0oEWaZo_lS5R;oh(&00S{Ct`q5rmLLXOVP&cdwv+_g?RJ8C&Sj z)8F&aD;&8#kX;O0v-mB3>VA_(3MQ-OnoII9X>;IKl#9#UY7$UK{2J`rNr6rMx)g9i z>MJRg>6LX!aDgwX9FVRi#Uyirjq4yb2Lzec_Qvc; zNkit&{}$cr=_9eiFcvU_!O%$#T+I(^Uj=&WP!PGDebk`RQ$EhkdJ+`>yWd z#z5=|9xpxm9`jG8Td!BYWfoG7A=fPlDnd;UA zqVELtT&Hz>ki0}mDF1h8@S(jKJj&%? zeat17VxgOR{E*OsS%5+uawL7{VU`hTNArn4p!54nkk0`)st8E zO86*b-Az|dn@z#@4uVV81v~s_lk-|;iS!D`o{SNA_}Rgzv1nsn4WgvdP(WE%5$kLJ z&C<-+1c&Zt(!@*F8R}Wa*{(t-Pe{rwb;K7GZsaa9ZkVomO9Uq-cMvUFvl6OjF%4ZvDB/examples/asr/conf/conformer/conformer_transducer_char.yaml`` and with sub-word encoding at ``/examples/asr/conf/conformer/conformer_transducer_bpe.yaml``. +.. _LSTM-Transducer_model: + LSTM-Transducer --------------- @@ -138,12 +140,45 @@ It can be trained/used in unidirectional or bidirectional mode. The unidirection This model supports both the sub-word level and character level encodings. You may find the example config file of RNNT model with wordpiece encoding at ``/examples/asr/conf/lstm/lstm_transducer_bpe.yaml``. You can find more details on the config files for the RNNT models at ``LSTM-Transducer <./configs.html#lstm-transducer>``. +.. _LSTM-CTC_model: + LSTM-CTC -------- +-------- LSTM-CTC model is a CTC-variant of the LSTM-Transducer model which uses CTC loss/decoding instead of Transducer. You may find the example config file of LSTM-CTC model with wordpiece encoding at ``/examples/asr/conf/lstm/lstm_ctc_bpe.yaml``. +.. _Squeezeformer-CTC_model: + +Squeezeformer-CTC +----------------- + +Squeezeformer-CTC is a CTC-based variant of the Squeezeformer model introduced in :cite:`asr-models-kim2022squeezeformer`. Squeezeformer-CTC has a +similar encoder as the original Squeezeformer but uses CTC loss and decoding instead of RNNT/Transducer loss, which makes it a non-autoregressive model. The vast majority of the architecture is similar to Conformer model, so please refer to `Conformer-CTC <./models.html#conformer-ctc>`. + +The model primarily differs from Conformer in the following ways : + +* Temporal U-Net style time reduction, effectively reducing memory consumption and FLOPs for execution. +* Unified activations throughout the model. +* Simplification of module structure, removal of redundant layers. + +Here is the overall architecture of the encoder of Squeezeformer-CTC: + + .. image:: images/squeezeformer.png + :align: center + :alt: Squeezeformer-CTC Model + :scale: 50% + +This model supports both the sub-word level and character level encodings. You can find more details on the config files for the +Squeezeformer-CTC models at `Squeezeformer-CTC <./configs.html#squeezeformer-ctc>`. The variant with sub-word encoding is a BPE-based model +which can be instantiated using the :class:`~nemo.collections.asr.models.EncDecCTCModelBPE` class, while the +character-based variant is based on :class:`~nemo.collections.asr.models.EncDecCTCModel`. + +You may find the example config files of Squeezeformer-CTC model with character-based encoding at +``/examples/asr/conf/squeezeformer/squeezeformer_ctc_char.yaml`` and +with sub-word encoding at ``/examples/asr/conf/squeezeformer/squeezeformer_ctc_bpe.yaml``. + + References ---------- diff --git a/examples/asr/asr_chunked_inference/ctc/speech_to_text_buffered_infer_ctc.py b/examples/asr/asr_chunked_inference/ctc/speech_to_text_buffered_infer_ctc.py index 3d6a61e12ca3..6fb2bbb5ffde 100644 --- a/examples/asr/asr_chunked_inference/ctc/speech_to_text_buffered_infer_ctc.py +++ b/examples/asr/asr_chunked_inference/ctc/speech_to_text_buffered_infer_ctc.py @@ -17,6 +17,17 @@ (1) Demonstrate how to use NeMo Models outside of PytorchLightning (2) Shows example of batch ASR inference (3) Serves as CI test for pre-trained checkpoint + +python speech_to_text_buffered_infer_ctc.py \ + --asr_model="" \ + --test_manifest="" \ + --model_stride=4 \ + --batch_size=32 \ + --total_buffer_in_secs=4.0 \ + --chunk_len_in_ms=1000 + + + """ import copy @@ -27,6 +38,7 @@ import torch from omegaconf import OmegaConf +from tqdm import tqdm import nemo.collections.asr as nemo_asr from nemo.collections.asr.metrics.wer import word_error_rate @@ -47,7 +59,7 @@ def get_wer_feat(mfst, asr, frame_len, tokens_per_chunk, delay, preprocessor_cfg refs = [] with open(mfst, "r") as mfst_f: - for l in mfst_f: + for l in tqdm(mfst_f, desc="Sample:"): asr.reset() row = json.loads(l.strip()) asr.read_audio_file(row['audio_filepath'], delay, model_stride_in_secs) @@ -129,7 +141,7 @@ def main(): model_stride_in_secs, asr_model.device, ) - logging.info(f"WER is {round(wer, 2)} when decoded with a delay of {round(mid_delay*model_stride_in_secs, 2)}s") + logging.info(f"WER is {round(wer, 4)} when decoded with a delay of {round(mid_delay*model_stride_in_secs, 2)}s") if args.output_path is not None: diff --git a/examples/asr/conf/squeezeformer/squeezeformer_ctc_bpe.yaml b/examples/asr/conf/squeezeformer/squeezeformer_ctc_bpe.yaml new file mode 100644 index 000000000000..c5d66043536f --- /dev/null +++ b/examples/asr/conf/squeezeformer/squeezeformer_ctc_bpe.yaml @@ -0,0 +1,201 @@ +# It contains the default values for training a Squeezeformer-CTC ASR model, large size (~120M) with CTC loss and sub-word encoding. + +# Architecture and training config: +# Default learning parameters in this config are set for effective batch size of 2K. To train it with smaller effective +# batch sizes, you may need to re-tune the learning parameters or use higher accumulate_grad_batches. +# Here are the recommended configs for different variants of Squeezeformer-CTC, other parameters are the same as in this config file. +# One extra layer (compared to original paper) is added to the medium and large variants to compensate for replacing the LSTM decoder with a linear one. +# +# | Model | d_model | n_layers | n_heads | time_masks | lr | time_reduce_idx | GBS | +# |--------------|---------|----------|---------|------------|--------|-----------------|------| +# | Extra-Small | 144 | 16 | 4 | 5 | 2e-3 | 7 | 1024 | +# | Small | 196 | 18 | 4 | 5 | 2e-3 | 8 | 1024 | +# | Small-Medium | 256 | 16 | 4 | 5 | 1.5e-3 | 7 | 1024 | +# | Medium | 324 | 20 | 4 | 7 | 1.5e-3 | 9 | 1024 | +# | Medium-Large | 512 | 18 | 8 | 10 | 1e-3 | 8 | 2048 | +# | Large | 640 | 22 | 8 | 10 | 5e-4 | 10 | 2048 | +# +# You may find more info about Squeezeformer-CTC here: https://docs.nvidia.com/deeplearning/nemo/user-guide/docs/en/stable/asr/models.html#squeezeformer-ctc +# Pre-trained models of Squeezeformer-CTC can be found here: https://docs.nvidia.com/deeplearning/nemo/user-guide/docs/en/stable/asr/results.html + +name: "Squeezeformer-CTC-BPE" + +model: + sample_rate: 16000 + log_prediction: true # enables logging sample predictions in the output during training + ctc_reduction: 'mean_batch' + skip_nan_grad: false + + train_ds: + manifest_filepath: ??? + sample_rate: ${model.sample_rate} + batch_size: 8 # you may increase batch_size if your memory allows + shuffle: true + num_workers: 8 + pin_memory: true + use_start_end_token: false + trim_silence: false + max_duration: 16.7 # it is set for LibriSpeech, you may need to update it for your dataset + min_duration: 0.1 + # tarred datasets + is_tarred: false + tarred_audio_filepaths: null + shuffle_n: 2048 + # bucketing params + bucketing_strategy: "synced_randomized" + bucketing_batch_size: null + + validation_ds: + manifest_filepath: ??? + sample_rate: ${model.sample_rate} + batch_size: 8 # you may increase batch_size if your memory allows + shuffle: false + num_workers: 8 + pin_memory: true + use_start_end_token: false + + test_ds: + manifest_filepath: null + sample_rate: ${model.sample_rate} + batch_size: 8 # you may increase batch_size if your memory allows + shuffle: false + num_workers: 8 + pin_memory: true + use_start_end_token: false + + # recommend small vocab size of 128 or 256 when using 4x sub-sampling + # you may find more detail on how to train a tokenizer at: /scripts/tokenizers/process_asr_text_tokenizer.py + tokenizer: + dir: ??? # path to directory which contains either tokenizer.model (bpe) or vocab.txt (wpe) + type: bpe # Can be either bpe (SentencePiece tokenizer) or wpe (WordPiece tokenizer) + + preprocessor: + _target_: nemo.collections.asr.modules.AudioToMelSpectrogramPreprocessor + sample_rate: ${model.sample_rate} + normalize: "per_feature" + window_size: 0.025 + window_stride: 0.01 + window: "hann" + features: 80 + n_fft: 512 + log: true + frame_splicing: 1 + dither: 0.00001 + pad_to: 0 + pad_value: 0.0 + + spec_augment: + _target_: nemo.collections.asr.modules.SpectrogramAugmentation + freq_masks: 2 # set to zero to disable it + # you may use lower time_masks for smaller models to have a faster convergence + time_masks: 10 # set to zero to disable it + freq_width: 27 + time_width: 0.05 + + encoder: + _target_: nemo.collections.asr.modules.SqueezeformerEncoder + feat_in: ${model.preprocessor.features} + feat_out: -1 # you may set it if you need different output size other than the default d_model + n_layers: 18 + d_model: 512 + + # Squeezeformer params + adaptive_scale: true + time_reduce_idx: 8 + time_recovery_idx: null + + # Sub-sampling params + subsampling: dw_striding # dw_striding, vggnet, striding or stacking, vggnet may give better results but needs more memory + subsampling_factor: 4 # must be power of 2 + subsampling_conv_channels: -1 # -1 sets it to d_model + + # Feed forward module's params + ff_expansion_factor: 4 + + # Multi-headed Attention Module's params + self_attention_model: rel_pos # rel_pos or abs_pos + n_heads: 8 # may need to be lower for smaller d_models + # [left, right] specifies the number of steps to be seen from left and right of each step in self-attention + att_context_size: [-1, -1] # -1 means unlimited context + xscaling: true # scales up the input embeddings by sqrt(d_model) + untie_biases: true # unties the biases of the TransformerXL layers + pos_emb_max_len: 5000 + + # Convolution module's params + conv_kernel_size: 31 + conv_norm_type: 'batch_norm' # batch_norm or layer_norm + + ### regularization + dropout: 0.1 # The dropout used in most of the Conformer Modules + dropout_emb: 0.0 # The dropout used for embeddings + dropout_att: 0.1 # The dropout for multi-headed attention modules + + decoder: + _target_: nemo.collections.asr.modules.ConvASRDecoder + feat_in: null + num_classes: -1 + vocabulary: [] + + optim: + name: adamw + lr: 0.001 + # optimizer arguments + betas: [0.9, 0.98] + # less necessity for weight_decay as we already have large augmentations with SpecAug + # you may need weight_decay for large models, stable AMP training, small datasets, or when lower augmentations are used + # weight decay of 0.0 with lr of 2.0 also works fine + weight_decay: 4e-5 + + # scheduler setup + sched: + name: NoamHoldAnnealing + # scheduler config override + warmup_steps: 5000 # paper uses ~ 6500 steps (20 epochs) out of 500 epochs. + warmup_ratio: null + hold_steps: 40000 + hold_ratio: null # paper uses ~ 40000 steps (160 epochs) out of 500 epochs. + decay_rate: 1.0 # Noam decay = 0.5 and no hold steps. For Squeezeformer, use hold ~ 10-30% of training, then faster decay. + min_lr: 1e-5 + +trainer: + devices: -1 # number of GPUs, -1 would use all available GPUs + num_nodes: 1 + max_epochs: 1000 + max_steps: null # computed at runtime if not set + val_check_interval: 1.0 # Set to 0.25 to check 4 times per epoch, or an int for number of iterations + accelerator: auto + strategy: ddp + accumulate_grad_batches: 1 + gradient_clip_val: 0.0 + precision: 32 # Should be set to 16 for O1 and O2 to enable the AMP. + log_every_n_steps: 10 # Interval of logging. + progress_bar_refresh_rate: 10 + resume_from_checkpoint: null # The path to a checkpoint file to continue the training, restores the whole state including the epoch, step, LR schedulers, apex, etc. + num_sanity_val_steps: 0 # number of steps to perform validation steps for sanity check the validation process before starting the training, setting to 0 disables it + check_val_every_n_epoch: 1 # number of evaluations on validation every n epochs + sync_batchnorm: true + enable_checkpointing: False # Provided by exp_manager + logger: false # Provided by exp_manager + benchmark: false # needs to be false for models with variable-length speech input as it slows down training + +exp_manager: + exp_dir: null + name: ${name} + create_tensorboard_logger: true + create_checkpoint_callback: true + checkpoint_callback_params: + # in case of multiple validation sets, first one is used + monitor: "val_wer" + mode: "min" + save_top_k: 5 + always_save_nemo: True # saves the checkpoints as nemo files instead of PTL checkpoints + + # you need to set these two to True to continue the training + resume_if_exists: false + resume_ignore_no_checkpoint: false + + # You may use this section to create a W&B logger + create_wandb_logger: false + wandb_logger_kwargs: + name: null + project: null diff --git a/examples/asr/conf/squeezeformer/squeezeformer_ctc_char.yaml b/examples/asr/conf/squeezeformer/squeezeformer_ctc_char.yaml new file mode 100644 index 000000000000..8fd06e24ee26 --- /dev/null +++ b/examples/asr/conf/squeezeformer/squeezeformer_ctc_char.yaml @@ -0,0 +1,186 @@ +# It contains the default values for training a Squeezeformer-CTC ASR model, large size (~120M) with CTC loss and char encoding. + +# You may find more detail on Conformer-CTC at `examples/asr/conf/conformer/conformer_ctc_bpe.yaml` +# You may find more info about Squeezeformer-CTC here: https://docs.nvidia.com/deeplearning/nemo/user-guide/docs/en/stable/asr/models.html#squeezeformer-ctc +# Pre-trained models of Squeezeformer-CTC can be found here: https://docs.nvidia.com/deeplearning/nemo/user-guide/docs/en/stable/asr/results.html + +name: "Squeezeformer-CTC-BPE" + +model: + sample_rate: 16000 + labels: [" ", "a", "b", "c", "d", "e", "f", "g", "h", "i", "j", "k", "l", "m", + "n", "o", "p", "q", "r", "s", "t", "u", "v", "w", "x", "y", "z", "'"] + log_prediction: true # enables logging sample predictions in the output during training + ctc_reduction: 'mean_batch' + skip_nan_grad: false + + train_ds: + manifest_filepath: ??? + labels: ${model.labels} + sample_rate: ${model.sample_rate} + batch_size: 16 # you may increase batch_size if your memory allows + shuffle: true + num_workers: 8 + pin_memory: true + use_start_end_token: false + trim_silence: false + max_duration: 16.7 # it is set for LibriSpeech, you may need to update it for your dataset + min_duration: 0.1 + # tarred datasets + is_tarred: false + tarred_audio_filepaths: null + shuffle_n: 2048 + # bucketing params + bucketing_strategy: "synced_randomized" + bucketing_batch_size: null + + validation_ds: + manifest_filepath: ??? + labels: ${model.labels} + sample_rate: ${model.sample_rate} + batch_size: 8 # you may increase batch_size if your memory allows + shuffle: false + num_workers: 8 + pin_memory: true + use_start_end_token: false + + test_ds: + manifest_filepath: null + labels: ${model.labels} + sample_rate: ${model.sample_rate} + batch_size: 8 # you may increase batch_size if your memory allows + shuffle: false + num_workers: 8 + pin_memory: true + use_start_end_token: false + + preprocessor: + _target_: nemo.collections.asr.modules.AudioToMelSpectrogramPreprocessor + sample_rate: ${model.sample_rate} + normalize: "per_feature" + window_size: 0.025 + window_stride: 0.01 + window: "hann" + features: 80 + n_fft: 512 + log: true + frame_splicing: 1 + dither: 0.00001 + pad_to: 0 + pad_value: 0.0 + + spec_augment: + _target_: nemo.collections.asr.modules.SpectrogramAugmentation + freq_masks: 2 # set to zero to disable it + # you may use lower time_masks for smaller models to have a faster convergence + time_masks: 10 # set to zero to disable it + freq_width: 27 + time_width: 0.05 + + encoder: + _target_: nemo.collections.asr.modules.SqueezeformerEncoder + feat_in: ${model.preprocessor.features} + feat_out: -1 # you may set it if you need different output size other than the default d_model + n_layers: 18 + d_model: 512 + + # Squeezeformer params + adaptive_scale: true + time_reduce_idx: 8 + time_recovery_idx: null + + # Sub-sampling params + subsampling: dw_striding # dw_striding, vggnet, striding or stacking, vggnet may give better results but needs more memory + subsampling_factor: 4 # must be power of 2 + subsampling_conv_channels: -1 # -1 sets it to d_model + + # Feed forward module's params + ff_expansion_factor: 4 + + # Multi-headed Attention Module's params + self_attention_model: rel_pos # rel_pos or abs_pos + n_heads: 8 # may need to be lower for smaller d_models + # [left, right] specifies the number of steps to be seen from left and right of each step in self-attention + att_context_size: [-1, -1] # -1 means unlimited context + xscaling: true # scales up the input embeddings by sqrt(d_model) + untie_biases: true # unties the biases of the TransformerXL layers + pos_emb_max_len: 5000 + + # Convolution module's params + conv_kernel_size: 31 + conv_norm_type: 'batch_norm' # batch_norm or layer_norm + + ### regularization + dropout: 0.1 # The dropout used in most of the Conformer Modules + dropout_emb: 0.0 # The dropout used for embeddings + dropout_att: 0.1 # The dropout for multi-headed attention modules + + decoder: + _target_: nemo.collections.asr.modules.ConvASRDecoder + feat_in: null + num_classes: -1 + vocabulary: ${model.labels} + + optim: + name: adamw + lr: 0.001 + # optimizer arguments + betas: [0.9, 0.98] + # less necessity for weight_decay as we already have large augmentations with SpecAug + # you may need weight_decay for large models, stable AMP training, small datasets, or when lower augmentations are used + # weight decay of 0.0 with lr of 2.0 also works fine + weight_decay: 4e-5 + + # scheduler setup + sched: + name: NoamHoldAnnealing + # scheduler config override + warmup_steps: 5000 # paper uses ~ 6500 steps (20 epochs) out of 500 epochs. + warmup_ratio: null + hold_steps: 40000 + hold_ratio: null # paper uses ~ 40000 steps (160 epochs) out of 500 epochs. + decay_rate: 1.0 # Noam decay = 0.5 and no hold steps. For Squeezeformer, use hold ~ 10-30% of training, then faster decay. + min_lr: 1e-5 + +trainer: + devices: -1 # number of GPUs, -1 would use all available GPUs + num_nodes: 1 + max_epochs: 1000 + max_steps: null # computed at runtime if not set + val_check_interval: 1.0 # Set to 0.25 to check 4 times per epoch, or an int for number of iterations + accelerator: auto + strategy: ddp + accumulate_grad_batches: 1 + gradient_clip_val: 0.0 + precision: 32 # Should be set to 16 for O1 and O2 to enable the AMP. + log_every_n_steps: 10 # Interval of logging. + progress_bar_refresh_rate: 10 + resume_from_checkpoint: null # The path to a checkpoint file to continue the training, restores the whole state including the epoch, step, LR schedulers, apex, etc. + num_sanity_val_steps: 0 # number of steps to perform validation steps for sanity check the validation process before starting the training, setting to 0 disables it + check_val_every_n_epoch: 1 # number of evaluations on validation every n epochs + sync_batchnorm: true + enable_checkpointing: False # Provided by exp_manager + logger: false # Provided by exp_manager + benchmark: false # needs to be false for models with variable-length speech input as it slows down training + +exp_manager: + exp_dir: null + name: ${name} + create_tensorboard_logger: true + create_checkpoint_callback: true + checkpoint_callback_params: + # in case of multiple validation sets, first one is used + monitor: "val_wer" + mode: "min" + save_top_k: 5 + always_save_nemo: True # saves the checkpoints as nemo files instead of PTL checkpoints + + # you need to set these two to True to continue the training + resume_if_exists: false + resume_ignore_no_checkpoint: false + + # You may use this section to create a W&B logger + create_wandb_logger: false + wandb_logger_kwargs: + name: null + project: null diff --git a/nemo/collections/asr/metrics/rnnt_wer.py b/nemo/collections/asr/metrics/rnnt_wer.py index 5009e074ee8d..ee730880481d 100644 --- a/nemo/collections/asr/metrics/rnnt_wer.py +++ b/nemo/collections/asr/metrics/rnnt_wer.py @@ -524,6 +524,8 @@ def validation_epoch_end(self, outputs): distances for all prediction - reference pairs, total number of words in all references. """ + full_state_update = True + def __init__( self, decoding: RNNTDecoding, batch_dim_index=0, use_cer=False, log_prediction=True, dist_sync_on_step=False ): diff --git a/nemo/collections/asr/metrics/rnnt_wer_bpe.py b/nemo/collections/asr/metrics/rnnt_wer_bpe.py index 10451056ca9d..0400dc61c8b8 100644 --- a/nemo/collections/asr/metrics/rnnt_wer_bpe.py +++ b/nemo/collections/asr/metrics/rnnt_wer_bpe.py @@ -184,6 +184,8 @@ def validation_epoch_end(self, outputs): distances for all prediction - reference pairs, total number of words in all references. """ + full_state_update = True + def __init__( self, decoding: RNNTBPEDecoding, diff --git a/nemo/collections/asr/models/ctc_models.py b/nemo/collections/asr/models/ctc_models.py index 9d5dd78138ac..12e6c22c64ca 100644 --- a/nemo/collections/asr/models/ctc_models.py +++ b/nemo/collections/asr/models/ctc_models.py @@ -574,7 +574,7 @@ def training_step(self, batch, batch_nb): tensorboard_logs = { 'train_loss': loss_value, 'learning_rate': self._optimizer.param_groups[0]['lr'], - 'global_step': self.trainer.global_step, + 'global_step': torch.tensor(self.trainer.global_step, dtype=torch.float32), } if hasattr(self, '_trainer') and self._trainer is not None: @@ -628,6 +628,9 @@ def validation_step(self, batch, batch_idx, dataloader_idx=0): ) wer, wer_num, wer_denom = self._wer.compute() self._wer.reset() + + self.log_dict({'global_step': torch.tensor(self.trainer.global_step, dtype=torch.float32)}) + return { 'val_loss': loss_value, 'val_wer_num': wer_num, diff --git a/nemo/collections/asr/models/rnnt_models.py b/nemo/collections/asr/models/rnnt_models.py index c59cf7caf455..803444e07ed2 100644 --- a/nemo/collections/asr/models/rnnt_models.py +++ b/nemo/collections/asr/models/rnnt_models.py @@ -702,7 +702,7 @@ def training_step(self, batch, batch_nb): tensorboard_logs = { 'train_loss': loss_value, 'learning_rate': self._optimizer.param_groups[0]['lr'], - 'global_step': self.trainer.global_step, + 'global_step': torch.tensor(self.trainer.global_step, dtype=torch.float32), } if (sample_id + 1) % log_every_n_steps == 0: @@ -735,7 +735,11 @@ def training_step(self, batch, batch_nb): if AccessMixin.is_access_enabled(): AccessMixin.reset_registry(self) - tensorboard_logs = {'train_loss': loss_value, 'learning_rate': self._optimizer.param_groups[0]['lr']} + tensorboard_logs = { + 'train_loss': loss_value, + 'learning_rate': self._optimizer.param_groups[0]['lr'], + 'global_step': torch.tensor(self.trainer.global_step, dtype=torch.float32), + } if compute_wer: tensorboard_logs.update({'training_batch_wer': wer}) @@ -825,6 +829,8 @@ def validation_step(self, batch, batch_idx, dataloader_idx=0): tensorboard_logs['val_wer_denom'] = wer_denom tensorboard_logs['val_wer'] = wer + self.log_dict({'global_step': torch.tensor(self.trainer.global_step, dtype=torch.float32)}) + return tensorboard_logs def test_step(self, batch, batch_idx, dataloader_idx=0): diff --git a/nemo/collections/asr/modules/__init__.py b/nemo/collections/asr/modules/__init__.py index eea9e16f43ee..676ec2fc9344 100644 --- a/nemo/collections/asr/modules/__init__.py +++ b/nemo/collections/asr/modules/__init__.py @@ -35,3 +35,4 @@ from nemo.collections.asr.modules.lstm_decoder import LSTMDecoder from nemo.collections.asr.modules.rnn_encoder import RNNEncoder from nemo.collections.asr.modules.rnnt import RNNTDecoder, RNNTDecoderJointSSL, RNNTJoint +from nemo.collections.asr.modules.squeezeformer_encoder import SqueezeformerEncoder, SqueezeformerEncoderAdapter diff --git a/nemo/collections/asr/modules/squeezeformer_encoder.py b/nemo/collections/asr/modules/squeezeformer_encoder.py new file mode 100644 index 000000000000..e8532f4d1bab --- /dev/null +++ b/nemo/collections/asr/modules/squeezeformer_encoder.py @@ -0,0 +1,419 @@ +# Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import math +from collections import OrderedDict +from typing import List, Optional + +import torch +import torch.distributed +import torch.nn as nn + +from nemo.collections.asr.parts.submodules.multi_head_attention import PositionalEncoding, RelPositionalEncoding +from nemo.collections.asr.parts.submodules.squeezeformer_modules import SqueezeformerLayer +from nemo.collections.asr.parts.submodules.subsampling import ConvSubsampling, StackingSubsampling, TimeReductionModule +from nemo.core.classes.common import typecheck +from nemo.core.classes.exportable import Exportable +from nemo.core.classes.mixins import adapter_mixins +from nemo.core.classes.module import NeuralModule +from nemo.core.neural_types import AcousticEncodedRepresentation, LengthsType, NeuralType, SpectrogramType + +__all__ = ['SqueezeformerEncoder'] + + +class SqueezeformerEncoder(NeuralModule, Exportable): + """ + The encoder for ASR model of Squeezeformer. + Based on this paper: + 'Squeezeformer: An Efficient Transformer for Automatic Speech Recognition' by Sehoon Kim et al. + https://arxiv.org/abs/2206.00888 + + Args: + feat_in (int): the size of feature channels + n_layers (int): number of layers of ConformerBlock + d_model (int): the hidden size of the model + feat_out (int): the size of the output features + Defaults to -1 (means feat_out is d_model) + subsampling (str): the method of subsampling, choices=['vggnet', 'striding', 'dw_striding'] + Defaults to dw_striding. + subsampling_factor (int): the subsampling factor which should be power of 2 + Defaults to 4. + subsampling_conv_channels (int): the size of the convolutions in the subsampling module + Defaults to -1 which would set it to d_model. + ff_expansion_factor (int): the expansion factor in feed forward layers + Defaults to 4. + self_attention_model (str): type of the attention layer and positional encoding + 'rel_pos': relative positional embedding and Transformer-XL + 'abs_pos': absolute positional embedding and Transformer + default is rel_pos. + pos_emb_max_len (int): the maximum length of positional embeddings + Defaulst to 5000 + n_heads (int): number of heads in multi-headed attention layers + Defaults to 4. + xscaling (bool): enables scaling the inputs to the multi-headed attention layers by sqrt(d_model) + Defaults to True. + untie_biases (bool): whether to not share (untie) the bias weights between layers of Transformer-XL + Defaults to True. + conv_kernel_size (int): the size of the convolutions in the convolutional modules + Defaults to 31. + conv_norm_type (str): the type of the normalization in the convolutional modules + Defaults to 'batch_norm'. + dropout (float): the dropout rate used in all layers except the attention layers + Defaults to 0.1. + dropout_emb (float): the dropout rate used for the positional embeddings + Defaults to 0.1. + dropout_att (float): the dropout rate used for the attention layer + Defaults to 0.0. + adaptive_scale (bool): Whether to scale the inputs to each component by affine `scale` and `bias` layer. + Or use a fixed scale=1 and bias=0. + time_reduce_idx (int): Optional integer index of a layer where a time reduction operation will occur. + All operations beyond this point will only occur at the reduced resolution. + time_recovery_idx (int): Optional integer index of a layer where the time recovery operation will occur. + All operations beyond this point will occur at the original resolution (resolution after + primary downsampling). If no value is provided, assumed to be the last layer. + """ + + def input_example(self, max_batch=1, max_dim=256): + """ + Generates input examples for tracing etc. + Returns: + A tuple of input examples. + """ + dev = next(self.parameters()).device + input_example = torch.randn(max_batch, self._feat_in, max_dim).to(dev) + input_example_length = torch.randint(1, max_dim, (max_batch,)).to(dev) + return tuple([input_example, input_example_length]) + + @property + def input_types(self): + """Returns definitions of module input ports. + """ + return OrderedDict( + { + "audio_signal": NeuralType(('B', 'D', 'T'), SpectrogramType()), + "length": NeuralType(tuple('B'), LengthsType()), + } + ) + + @property + def output_types(self): + """Returns definitions of module output ports. + """ + return OrderedDict( + { + "outputs": NeuralType(('B', 'D', 'T'), AcousticEncodedRepresentation()), + "encoded_lengths": NeuralType(tuple('B'), LengthsType()), + } + ) + + def __init__( + self, + feat_in: int, + n_layers: int, + d_model: int, + feat_out: int = -1, + subsampling: str = 'dw_striding', + subsampling_factor: int = 4, + subsampling_conv_channels: int = -1, + ff_expansion_factor: int = 4, + self_attention_model: str = 'rel_pos', + n_heads: int = 4, + att_context_size: Optional[List[int]] = None, + xscaling: bool = True, + untie_biases: bool = True, + pos_emb_max_len: int = 5000, + conv_kernel_size: int = 31, + conv_norm_type: str = 'batch_norm', + dropout: float = 0.1, + dropout_emb: float = 0.1, + dropout_att: float = 0.0, + adaptive_scale: bool = True, + time_reduce_idx: Optional[int] = None, + time_recovery_idx: Optional[int] = None, + ): + super().__init__() + + d_ff = d_model * ff_expansion_factor + self.d_model = d_model + self._feat_in = feat_in + self.scale = math.sqrt(self.d_model) + if att_context_size: + self.att_context_size = att_context_size + else: + self.att_context_size = [-1, -1] + + if xscaling: + self.xscale = math.sqrt(d_model) + else: + self.xscale = None + self.adaptive_scale = adaptive_scale + + self.time_reduce_idx = time_reduce_idx + if time_reduce_idx is not None: + if time_recovery_idx is None: + self.time_recovery_idx = n_layers - 1 # recover at last layer + else: + self.time_recovery_idx = time_recovery_idx # recover at given layer + + if self.time_reduce_idx is not None: + if self.time_reduce_idx < 0 or self.time_recovery_idx >= n_layers: + raise ValueError(f"Time reduce index must lie between [0, {n_layers})") + if self.time_recovery_idx < 0 or self.time_recovery_idx >= n_layers: + raise ValueError(f"Time recovery index must lie between [0, {n_layers})") + + if subsampling_conv_channels == -1: + subsampling_conv_channels = d_model + if subsampling and subsampling_factor > 1: + if subsampling == 'stacking': + self.pre_encode = StackingSubsampling( + subsampling_factor=subsampling_factor, feat_in=feat_in, feat_out=d_model + ) + else: + self.pre_encode = ConvSubsampling( + subsampling=subsampling, + subsampling_factor=subsampling_factor, + feat_in=feat_in, + feat_out=d_model, + conv_channels=subsampling_conv_channels, + activation=nn.ReLU(), + ) + # For Squeezeformer, initialize the parameters as required. + self.pre_encode.reset_parameters() + else: + self.pre_encode = nn.Linear(feat_in, d_model) + + self._feat_out = d_model + + if not untie_biases and self_attention_model == "rel_pos": + d_head = d_model // n_heads + pos_bias_u = nn.Parameter(torch.Tensor(n_heads, d_head)) + pos_bias_v = nn.Parameter(torch.Tensor(n_heads, d_head)) + nn.init.zeros_(pos_bias_u) + nn.init.zeros_(pos_bias_v) + else: + pos_bias_u = None + pos_bias_v = None + + self.pos_emb_max_len = pos_emb_max_len + if self_attention_model == "rel_pos": + self.pos_enc = RelPositionalEncoding( + d_model=d_model, + dropout_rate=dropout, + max_len=pos_emb_max_len, + xscale=self.xscale, + dropout_rate_emb=dropout_emb, + ) + elif self_attention_model == "abs_pos": + pos_bias_u = None + pos_bias_v = None + self.pos_enc = PositionalEncoding( + d_model=d_model, dropout_rate=dropout, max_len=pos_emb_max_len, xscale=self.xscale + ) + else: + raise ValueError(f"Not valid self_attention_model: '{self_attention_model}'!") + + self.layers = nn.ModuleList() + for i in range(n_layers): + layer = SqueezeformerLayer( + d_model=d_model, + d_ff=d_ff, + self_attention_model=self_attention_model, + n_heads=n_heads, + conv_kernel_size=conv_kernel_size, + conv_norm_type=conv_norm_type, + dropout=dropout, + dropout_att=dropout_att, + pos_bias_u=pos_bias_u, + pos_bias_v=pos_bias_v, + adaptive_scale=adaptive_scale, + ) + self.layers.append(layer) + + # Time Reduction and Recovery layer setup + self.time_reduce_layer = None + self.time_recovery_layer = None + self.time_reduce_pos_enc = None + # Add time reduction layer + if self.time_reduce_idx is not None: + self.time_reduce_layer = TimeReductionModule(d_model, d_model, kernel_size=5, stride=2) + self.time_recovery_layer = nn.Linear(d_model, d_model) + + # Chose same type of positional encoding as the originally determined above + if self_attention_model == "rel_pos": + self.time_reduce_pos_enc = RelPositionalEncoding( + d_model=d_model, dropout_rate=0.0, max_len=pos_emb_max_len, xscale=None, dropout_rate_emb=0.0, + ) + else: + self.time_reduce_pos_enc = PositionalEncoding( + d_model=d_model, dropout_rate=0.0, max_len=pos_emb_max_len, xscale=None, dropout_rate_emb=0.0 + ) + + self.pre_ln = nn.LayerNorm(d_model) + + if feat_out > 0 and feat_out != self._feat_out: + self.out_proj = nn.Linear(self._feat_out, feat_out) + self._feat_out = feat_out + else: + self.out_proj = None + self._feat_out = d_model + self.set_max_audio_length(self.pos_emb_max_len) + self.use_pad_mask = True + + def set_max_audio_length(self, max_audio_length): + """ Sets maximum input length. + Pre-calculates internal seq_range mask. + """ + self.max_audio_length = max_audio_length + device = next(self.parameters()).device + seq_range = torch.arange(0, self.max_audio_length, device=device) + if hasattr(self, 'seq_range'): + self.seq_range = seq_range + else: + self.register_buffer('seq_range', seq_range, persistent=False) + self.pos_enc.extend_pe(max_audio_length, device) + + if self.time_reduce_pos_enc is not None: + self.time_reduce_pos_enc.extend_pe(max_audio_length, device) + + @typecheck() + def forward(self, audio_signal, length=None): + self.update_max_seq_length(seq_length=audio_signal.size(2), device=audio_signal.device) + return self.forward_for_export(audio_signal=audio_signal, length=length) + + @typecheck() + def forward_for_export(self, audio_signal, length): + max_audio_length: int = audio_signal.size(-1) + + if max_audio_length > self.max_audio_length: + self.set_max_audio_length(max_audio_length) + + if length is None: + length = audio_signal.new_full( + audio_signal.size(0), max_audio_length, dtype=torch.int32, device=self.seq_range.device + ) + + audio_signal = torch.transpose(audio_signal, 1, 2) + + if isinstance(self.pre_encode, nn.Linear): + audio_signal = self.pre_encode(audio_signal) + else: + audio_signal, length = self.pre_encode(audio_signal, length) + + audio_signal, pos_emb = self.pos_enc(audio_signal) + # adjust size + max_audio_length = audio_signal.size(1) + # Create the self-attention and padding masks + + pad_mask = self.make_pad_mask(max_audio_length, length) + att_mask = pad_mask.unsqueeze(1).repeat([1, max_audio_length, 1]) + att_mask = torch.logical_and(att_mask, att_mask.transpose(1, 2)) + if self.att_context_size[0] >= 0: + att_mask = att_mask.triu(diagonal=-self.att_context_size[0]) + if self.att_context_size[1] >= 0: + att_mask = att_mask.tril(diagonal=self.att_context_size[1]) + att_mask = ~att_mask + + if self.use_pad_mask: + pad_mask = ~pad_mask + else: + pad_mask = None + + # Create cache of activations for the time reduction step + # Note: NeMo codebase allows only a single time reduction step to occur + recovery_activation_cache = [] + + audio_signal = self.pre_ln(audio_signal) + for lth, layer in enumerate(self.layers): + # Perform time reduction + if self.time_reduce_layer is not None and lth == self.time_reduce_idx: + # Perform time reduction + recovery_activation_cache.append((audio_signal, att_mask, pad_mask, pos_emb)) + audio_signal, att_mask, pad_mask = self.time_reduce_layer( + x=audio_signal, att_mask=att_mask, pad_mask=pad_mask + ) + # Only update PE, not the original audio_signal + _, pos_emb = self.time_reduce_pos_enc(audio_signal) + + # Perform time recovery + if self.time_recovery_layer is not None and lth == self.time_recovery_idx: + recovery_audio_signal, att_mask, pad_mask, pos_emb = recovery_activation_cache.pop(0) + # repeat interleaved values for 2x seq length + audio_signal = torch.repeat_interleave(audio_signal, repeats=2, dim=1) + + B, T, D = recovery_audio_signal.size() + audio_signal = audio_signal[:, :T, :] # Slice off the exact T timesteps as original cache value + audio_signal = self.time_recovery_layer(audio_signal) # learn non linear mapping + audio_signal = recovery_audio_signal + audio_signal # learn just the residual + + audio_signal = layer(x=audio_signal, att_mask=att_mask, pos_emb=pos_emb, pad_mask=pad_mask) + + if self.out_proj is not None: + audio_signal = self.out_proj(audio_signal) + + audio_signal = torch.transpose(audio_signal, 1, 2) + return audio_signal, length + + def update_max_seq_length(self, seq_length: int, device): + # Find global max audio length across all nodes + if torch.distributed.is_initialized(): + global_max_len = torch.tensor([seq_length], dtype=torch.float32, device=device) + + # Update across all ranks in the distributed system + torch.distributed.all_reduce(global_max_len, op=torch.distributed.ReduceOp.MAX) + + seq_length = global_max_len.int().item() + + if seq_length > self.max_audio_length: + self.set_max_audio_length(seq_length) + + def make_pad_mask(self, max_audio_length, seq_lens): + """Make masking for padding.""" + mask = self.seq_range[:max_audio_length].expand(seq_lens.size(0), -1) < seq_lens.unsqueeze(-1) + return mask + + def enable_pad_mask(self, on=True): + # On inference, user may chose to disable pad mask + mask = self.use_pad_mask + self.use_pad_mask = on + return mask + + +class SqueezeformerEncoderAdapter(SqueezeformerEncoder, adapter_mixins.AdapterModuleMixin): + + # Higher level forwarding + def add_adapter(self, name: str, cfg: dict): + for conformer_layer in self.layers: # type: adapter_mixins.AdapterModuleMixin + conformer_layer.add_adapter(name, cfg) + + def is_adapter_available(self) -> bool: + return any([conformer_layer.is_adapter_available() for conformer_layer in self.layers]) + + def set_enabled_adapters(self, name: Optional[str] = None, enabled: bool = True): + for conformer_layer in self.layers: # type: adapter_mixins.AdapterModuleMixin + conformer_layer.set_enabled_adapters(name=name, enabled=enabled) + + def get_enabled_adapters(self) -> List[str]: + names = set([]) + for conformer_layer in self.layers: # type: adapter_mixins.AdapterModuleMixin + names.update(conformer_layer.get_enabled_adapters()) + + names = sorted(list(names)) + return names + + +""" +Register any additional information +""" +if adapter_mixins.get_registered_adapter(SqueezeformerEncoder) is None: + adapter_mixins.register_adapter(base_class=SqueezeformerEncoder, adapter_class=SqueezeformerEncoderAdapter) diff --git a/nemo/collections/asr/parts/submodules/conformer_modules.py b/nemo/collections/asr/parts/submodules/conformer_modules.py index 0c1c5e538d93..1af570836e73 100644 --- a/nemo/collections/asr/parts/submodules/conformer_modules.py +++ b/nemo/collections/asr/parts/submodules/conformer_modules.py @@ -21,6 +21,7 @@ RelPositionMultiHeadAttention, ) from nemo.collections.asr.parts.utils.activations import Swish +from nemo.collections.common.parts.utils import activation_registry from nemo.core.classes.mixins import AccessMixin from nemo.core.classes.mixins.adapter_mixins import AdapterModuleMixin from nemo.utils import logging @@ -137,41 +138,60 @@ class ConformerConvolution(nn.Module): Args: d_model (int): hidden dimension kernel_size (int): kernel size for depthwise convolution + pointwise_activation (str): name of the activation function to be used for the pointwise conv. + Note that Conformer uses a special key `glu_` which is treated as the original default from + the paper. """ - def __init__(self, d_model, kernel_size, norm_type='batch_norm'): + def __init__(self, d_model, kernel_size, norm_type='batch_norm', pointwise_activation='glu_'): super(ConformerConvolution, self).__init__() assert (kernel_size - 1) % 2 == 0 self.d_model = d_model + self.kernel_size = kernel_size + + if pointwise_activation in activation_registry: + self.pointwise_activation = activation_registry[pointwise_activation]() + dw_conv_input_dim = d_model * 2 + + if hasattr(self.pointwise_activation, 'inplace'): + self.pointwise_activation.inplace = True + else: + self.pointwise_activation = pointwise_activation + dw_conv_input_dim = d_model self.pointwise_conv1 = nn.Conv1d( in_channels=d_model, out_channels=d_model * 2, kernel_size=1, stride=1, padding=0, bias=True ) self.depthwise_conv = nn.Conv1d( - in_channels=d_model, - out_channels=d_model, + in_channels=dw_conv_input_dim, + out_channels=dw_conv_input_dim, kernel_size=kernel_size, stride=1, padding=(kernel_size - 1) // 2, - groups=d_model, + groups=dw_conv_input_dim, bias=True, ) if norm_type == 'batch_norm': - self.batch_norm = nn.BatchNorm1d(d_model) + self.batch_norm = nn.BatchNorm1d(dw_conv_input_dim) elif norm_type == 'layer_norm': - self.batch_norm = nn.LayerNorm(d_model) + self.batch_norm = nn.LayerNorm(dw_conv_input_dim) else: raise ValueError(f"conv_norm_type={norm_type} is not valid!") self.activation = Swish() self.pointwise_conv2 = nn.Conv1d( - in_channels=d_model, out_channels=d_model, kernel_size=1, stride=1, padding=0, bias=True + in_channels=dw_conv_input_dim, out_channels=d_model, kernel_size=1, stride=1, padding=0, bias=True ) def forward(self, x, pad_mask=None): x = x.transpose(1, 2) x = self.pointwise_conv1(x) - x = nn.functional.glu(x, dim=1) + + # Compute the activation function or use GLU for original Conformer + if self.pointwise_activation == 'glu_': + x = nn.functional.glu(x, dim=1) + else: + x = self.pointwise_activation(x) if pad_mask is not None: x = x.float().masked_fill(pad_mask.unsqueeze(1), 0.0) @@ -190,6 +210,18 @@ def forward(self, x, pad_mask=None): x = x.transpose(1, 2) return x + def reset_parameters_conv(self): + pw1_max = pw2_max = self.d_model ** -0.5 + dw_max = self.kernel_size ** -0.5 + + with torch.no_grad(): + nn.init.uniform_(self.pointwise_conv1.weight, -pw1_max, pw1_max) + nn.init.uniform_(self.pointwise_conv1.bias, -pw1_max, pw1_max) + nn.init.uniform_(self.pointwise_conv2.weight, -pw2_max, pw2_max) + nn.init.uniform_(self.pointwise_conv2.bias, -pw2_max, pw2_max) + nn.init.uniform_(self.depthwise_conv.weight, -dw_max, dw_max) + nn.init.uniform_(self.depthwise_conv.bias, -dw_max, dw_max) + class ConformerFeedForward(nn.Module): """ @@ -198,6 +230,8 @@ class ConformerFeedForward(nn.Module): def __init__(self, d_model, d_ff, dropout, activation=Swish()): super(ConformerFeedForward, self).__init__() + self.d_model = d_model + self.d_ff = d_ff self.linear1 = nn.Linear(d_model, d_ff) self.activation = activation self.dropout = nn.Dropout(p=dropout) @@ -209,3 +243,12 @@ def forward(self, x): x = self.dropout(x) x = self.linear2(x) return x + + def reset_parameters_ff(self): + ffn1_max = self.d_model ** -0.5 + ffn2_max = self.d_ff ** -0.5 + with torch.no_grad(): + nn.init.uniform_(self.linear1.weight, -ffn1_max, ffn1_max) + nn.init.uniform_(self.linear1.bias, -ffn1_max, ffn1_max) + nn.init.uniform_(self.linear2.weight, -ffn2_max, ffn2_max) + nn.init.uniform_(self.linear2.bias, -ffn2_max, ffn2_max) diff --git a/nemo/collections/asr/parts/submodules/rnnt_greedy_decoding.py b/nemo/collections/asr/parts/submodules/rnnt_greedy_decoding.py index 0dd2b206fa79..c79b5f4e20d7 100644 --- a/nemo/collections/asr/parts/submodules/rnnt_greedy_decoding.py +++ b/nemo/collections/asr/parts/submodules/rnnt_greedy_decoding.py @@ -539,9 +539,12 @@ def _greedy_decode_blank_as_pad( if self.preserve_alignments: # Insert logprobs into last timestep per sample logp_vals = logp.to('cpu') + logp_ids = logp_vals.max(1)[1] for batch_idx in range(batchsize): if time_idx < out_len[batch_idx]: - hypotheses[batch_idx].alignments[-1].append((logp_vals[batch_idx], k[batch_idx])) + hypotheses[batch_idx].alignments[-1].append( + (logp_vals[batch_idx], logp_ids[batch_idx]) + ) del logp_vals del logp @@ -710,9 +713,12 @@ def _greedy_decode_masked( if self.preserve_alignments: # Insert logprobs into last timestep per sample logp_vals = logp.to('cpu') + logp_ids = logp_vals.max(1)[1] for batch_idx in range(batchsize): if time_idx < out_len[batch_idx]: - hypotheses[batch_idx].alignments[-1].append((logp_vals[batch_idx], k[batch_idx])) + hypotheses[batch_idx].alignments[-1].append( + (logp_vals[batch_idx], logp_ids[batch_idx]) + ) del logp_vals del logp diff --git a/nemo/collections/asr/parts/submodules/squeezeformer_modules.py b/nemo/collections/asr/parts/submodules/squeezeformer_modules.py new file mode 100644 index 000000000000..eb470001f25b --- /dev/null +++ b/nemo/collections/asr/parts/submodules/squeezeformer_modules.py @@ -0,0 +1,185 @@ +# Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +import torch +from torch import nn as nn +from torch.nn import LayerNorm + +from nemo.collections.asr.parts.submodules.conformer_modules import ConformerConvolution, ConformerFeedForward +from nemo.collections.asr.parts.submodules.multi_head_attention import ( + MultiHeadAttention, + RelPositionMultiHeadAttention, +) +from nemo.core.classes.mixins import AccessMixin +from nemo.core.classes.mixins.adapter_mixins import AdapterModuleMixin + +__all__ = ['SqueezeformerLayer', 'ConformerFeedForward', 'SqueezeformerLayer'] + + +class ScaleBiasLayer(torch.nn.Module): + """ + Computes an affine transformation y = x * scale + bias, either learned via adaptive weights, or fixed. + Efficient alternative to LayerNorm where we can avoid computing the mean and variance of the input, and + just rescale the output of the previous layer. + + Args: + d_model (int): input dimension of layer. + adaptive_scale (bool): whether to learn the affine transformation parameters or not. If set to False, + the scale is fixed to 1 and bias to 0, effectively performing a No-Op on the input. + This is done for export compatibility. + """ + + def __init__(self, d_model: int, adaptive_scale: bool): + super().__init__() + self.adaptive_scale = adaptive_scale + if adaptive_scale: + self.scale = nn.Parameter(torch.ones(d_model)) + self.bias = nn.Parameter(torch.zeros(d_model)) + else: + self.register_buffer('scale', torch.ones(d_model), persistent=True) + self.register_buffer('bias', torch.zeros(d_model), persistent=True) + + def forward(self, x): + scale = self.scale.view(1, 1, -1) + bias = self.bias.view(1, 1, -1) + return x * scale + bias + + +class SqueezeformerLayer(torch.nn.Module, AdapterModuleMixin, AccessMixin): + """A single block of the Squeezeformer encoder. + + Args: + d_model (int): input dimension of MultiheadAttentionMechanism and PositionwiseFeedForward + d_ff (int): hidden dimension of PositionwiseFeedForward + n_heads (int): number of heads for multi-head attention + conv_kernel_size (int): kernel size for depthwise convolution in convolution module + dropout (float): dropout probabilities for linear layers + dropout_att (float): dropout probabilities for attention distributions + adaptive_scale (bool): Whether to scale the inputs to each component by affine `scale` and `bias` layer. + Or use a fixed scale=1 and bias=0. + """ + + def __init__( + self, + d_model, + d_ff, + self_attention_model='rel_pos', + n_heads=4, + conv_kernel_size=31, + conv_norm_type='batch_norm', + dropout=0.1, + dropout_att=0.1, + pos_bias_u=None, + pos_bias_v=None, + adaptive_scale: bool = True, + ): + super().__init__() + + self.self_attention_model = self_attention_model + self.n_heads = n_heads + self.fc_factor = 1.0 + self.adaptive_scale = adaptive_scale + + # first feed forward module + self.norm_feed_forward1 = LayerNorm(d_model) + self.feed_forward1 = ConformerFeedForward(d_model=d_model, d_ff=d_ff, dropout=dropout) + self.feed_forward1_scale = ScaleBiasLayer(d_model=d_model, adaptive_scale=adaptive_scale) + + # convolution module + self.norm_conv = LayerNorm(d_model) + self.conv = ConformerConvolution( + d_model=d_model, kernel_size=conv_kernel_size, norm_type=conv_norm_type, pointwise_activation='swish' + ) + self.conv_scale = ScaleBiasLayer(d_model=d_model, adaptive_scale=adaptive_scale) + + # multi-headed self-attention module + self.norm_self_att = LayerNorm(d_model) + if self_attention_model == 'rel_pos': + self.self_attn = RelPositionMultiHeadAttention( + n_head=n_heads, n_feat=d_model, dropout_rate=dropout_att, pos_bias_u=pos_bias_u, pos_bias_v=pos_bias_v + ) + elif self_attention_model == 'abs_pos': + self.self_attn = MultiHeadAttention(n_head=n_heads, n_feat=d_model, dropout_rate=dropout_att) + else: + raise ValueError( + f"'{self_attention_model}' is not not a valid value for 'self_attention_model', " + f"valid values can be from ['rel_pos', 'abs_pos']" + ) + self.self_attn_scale = ScaleBiasLayer(d_model=d_model, adaptive_scale=adaptive_scale) + + # second feed forward module + self.norm_feed_forward2 = LayerNorm(d_model) + self.feed_forward2 = ConformerFeedForward(d_model=d_model, d_ff=d_ff, dropout=dropout) + self.feed_forward2_scale = ScaleBiasLayer(d_model=d_model, adaptive_scale=adaptive_scale) + + self.dropout = nn.Dropout(dropout) + # self.norm_out = LayerNorm(d_model) + + # initialize parameters properly + self.reset_parameters() + + def forward(self, x, att_mask=None, pos_emb=None, pad_mask=None): + """ + Args: + x (torch.Tensor): input signals (B, T, d_model) + att_mask (torch.Tensor): attention masks(B, T, T) + pos_emb (torch.Tensor): (L, 1, d_model) + pad_mask (torch.tensor): padding mask + Returns: + x (torch.Tensor): (B, T, d_model) + """ + residual = x + + x = self.self_attn_scale(x) + if self.self_attention_model == 'rel_pos': + x = self.self_attn(query=x, key=x, value=x, mask=att_mask, pos_emb=pos_emb) + elif self.self_attention_model == 'abs_pos': + x = self.self_attn(query=x, key=x, value=x, mask=att_mask) + else: + x = None + x = residual + self.dropout(x) + x = self.norm_self_att(x) + residual = x + + x = self.feed_forward1_scale(x) + x = self.feed_forward1(x) + x = residual + self.dropout(x) * self.fc_factor + x = self.norm_feed_forward1(x) + residual = x + + x = self.conv_scale(x) + x = self.conv(x, pad_mask) + x = residual + self.dropout(x) + x = self.norm_conv(x) + residual = x + + x = self.feed_forward2_scale(x) + x = self.feed_forward2(x) + x = residual + self.dropout(x) * self.fc_factor + x = self.norm_feed_forward2(x) + + if self.is_adapter_available(): + # Call the adapters + x = self.forward_enabled_adapters(x) + + if self.is_access_enabled(): + self.register_accessible_tensor(tensor=x) + + return x + + def reset_parameters(self): + # Used for Squeezeformer initialization only + self.feed_forward1.reset_parameters_ff() + self.feed_forward2.reset_parameters_ff() + self.conv.reset_parameters_conv() diff --git a/nemo/collections/asr/parts/submodules/subsampling.py b/nemo/collections/asr/parts/submodules/subsampling.py index b58e0f912a49..f23c63e349e3 100644 --- a/nemo/collections/asr/parts/submodules/subsampling.py +++ b/nemo/collections/asr/parts/submodules/subsampling.py @@ -58,6 +58,9 @@ class ConvSubsampling(torch.nn.Module): def __init__(self, subsampling, subsampling_factor, feat_in, feat_out, conv_channels, activation=nn.ReLU()): super(ConvSubsampling, self).__init__() self._subsampling = subsampling + self._conv_channels = conv_channels + self._feat_in = feat_in + self._feat_out = feat_out if subsampling_factor % 2 != 0: raise ValueError("Sampling factor should be a multiply of 2!") @@ -95,6 +98,50 @@ def __init__(self, subsampling, subsampling_factor, feat_in, feat_out, conv_chan ) ) in_channels = conv_channels + + elif subsampling == 'dw_striding': + self._padding = 1 + self._stride = 2 + self._kernel_size = 3 + self._ceil_mode = False + + # Layer 1 + layers.append( + torch.nn.Conv2d( + in_channels=in_channels, + out_channels=conv_channels, + kernel_size=self._kernel_size, + stride=self._stride, + padding=self._padding, + ) + ) + in_channels = conv_channels + layers.append(activation) + + for i in range(self._sampling_num - 1): + layers.extend( + [ + torch.nn.Conv2d( + in_channels=in_channels, + out_channels=in_channels, + kernel_size=self._kernel_size, + stride=self._stride, + padding=self._padding, + groups=in_channels, + ), + torch.nn.Conv2d( + in_channels=in_channels, + out_channels=conv_channels, + kernel_size=1, + stride=1, + padding=0, + groups=1, + ), + ] + ) + layers.append(activation) + in_channels = conv_channels + elif subsampling == 'striding': self._padding = 1 self._stride = 2 @@ -138,7 +185,7 @@ def forward(self, x, lengths): repeat_num=self._sampling_num, ) x = x.unsqueeze(1) - if self._subsampling == 'striding': + if self._subsampling in ['striding', 'dw_striding']: # added in order to prevent slowdown in torch.nn.Conv2d with bfloat16 / CUDNN v8 API # to be removed once the above is fixed in cudnn with torch.cuda.amp.autocast(dtype=torch.float32): @@ -150,6 +197,29 @@ def forward(self, x, lengths): x = self.out(x.transpose(1, 2).reshape(b, t, -1)) return x, lengths + def reset_parameters(self): + # initialize weights + if self._subsampling == 'dw_striding': + with torch.no_grad(): + # init conv + scale = 1.0 / self._kernel_size + dw_max = (self._kernel_size ** 2) ** -0.5 + pw_max = self._conv_channels ** -0.5 + + torch.nn.init.uniform_(self.conv[0].weight, -scale, scale) + torch.nn.init.uniform_(self.conv[0].bias, -scale, scale) + + for idx in range(2, len(self.conv), 3): + torch.nn.init.uniform_(self.conv[idx].weight, -dw_max, dw_max) + torch.nn.init.uniform_(self.conv[idx].bias, -dw_max, dw_max) + torch.nn.init.uniform_(self.conv[idx + 1].weight, -pw_max, pw_max) + torch.nn.init.uniform_(self.conv[idx + 1].bias, -pw_max, pw_max) + + # init fc (80 * 64 = 5120 from https://github.com/kssteven418/Squeezeformer/blob/13c97d6cf92f2844d2cb3142b4c5bfa9ad1a8951/src/models/conformer_encoder.py#L487 + fc_scale = (self._feat_out * self._feat_in / self._sampling_num) ** -0.5 + torch.nn.init.uniform_(self.out.weight, -fc_scale, fc_scale) + torch.nn.init.uniform_(self.out.bias, -fc_scale, fc_scale) + def calc_length(lengths, padding, kernel_size, stride, ceil_mode, repeat_num=1): """ Calculates the output length of a Tensor passed through a convolution or max pooling layer""" @@ -162,3 +232,68 @@ def calc_length(lengths, padding, kernel_size, stride, ceil_mode, repeat_num=1): else: lengths = torch.floor(lengths) return lengths.to(dtype=torch.int) + + +class TimeReductionModule(nn.Module): + """ + Squeezeformer Time Reduction procedure. Downsamples the audio by `stride` in the time dimension. + + Args: + d_model (int): input dimension of MultiheadAttentionMechanism and PositionwiseFeedForward + out_dim (int): Output dimension of the module. + kernel_size (int): Conv kernel size for depthwise convolution in convolution module + stride (int): Downsampling factor in time dimension. + """ + + def __init__(self, d_model: int, out_dim: int, kernel_size: int = 5, stride: int = 2): + super().__init__() + + self.d_model = d_model + self.out_dim = out_dim + self.kernel_size = kernel_size + self.stride = stride + self.padding = max(0, self.kernel_size - self.stride) + + self.dw_conv = nn.Conv1d( + in_channels=d_model, + out_channels=d_model, + kernel_size=kernel_size, + stride=stride, + padding=self.padding, + groups=d_model, + ) + + self.pw_conv = nn.Conv1d( + in_channels=d_model, out_channels=out_dim, kernel_size=1, stride=1, padding=0, groups=1, + ) + + self.reset_parameters() + + def forward(self, x, att_mask=None, pad_mask=None): + x = x.transpose(1, 2) # [B, C, T] + if pad_mask is not None: + x = x.float().masked_fill(pad_mask.unsqueeze(1), 0.0) + + x = self.dw_conv(x) + x = self.pw_conv(x) + + x = x.transpose(1, 2) # [B, T, C] + + B, T, D = x.size() + if att_mask is not None and pad_mask is not None: + att_mask = att_mask[:, :: self.stride, :: self.stride] + pad_mask = pad_mask[:, :: self.stride] + L = pad_mask.size(-1) + x = torch.nn.functional.pad(x, (0, 0, 0, L - T)) + + return x, att_mask, pad_mask + + def reset_parameters(self): + dw_max = self.kernel_size ** -0.5 + pw_max = self.d_model ** -0.5 + + with torch.no_grad(): + torch.nn.init.uniform_(self.dw_conv.weight, -dw_max, dw_max) + torch.nn.init.uniform_(self.dw_conv.bias, -dw_max, dw_max) + torch.nn.init.uniform_(self.pw_conv.weight, -pw_max, pw_max) + torch.nn.init.uniform_(self.pw_conv.bias, -pw_max, pw_max) diff --git a/nemo/collections/asr/parts/utils/streaming_utils.py b/nemo/collections/asr/parts/utils/streaming_utils.py index 41537b571054..d13d29a63786 100644 --- a/nemo/collections/asr/parts/utils/streaming_utils.py +++ b/nemo/collections/asr/parts/utils/streaming_utils.py @@ -1032,7 +1032,14 @@ def transcribe( raise ValueError("Signal did not end") for a_idx, alignment in enumerate(alignments): - alignment = alignment[len(alignment) - 1 - delay : len(alignment) - 1 - delay + tokens_per_chunk] + if delay == len(alignment): # chunk size = buffer size + offset = 0 + else: # all other cases + offset = 1 + + alignment = alignment[ + len(alignment) - offset - delay : len(alignment) - offset - delay + tokens_per_chunk + ] ids, toks = self._alignment_decoder(alignment, self.asr_model.tokenizer, self.blank_id) diff --git a/nemo/core/config/schedulers.py b/nemo/core/config/schedulers.py index 4f1554bddbd4..c24d738fdaa0 100644 --- a/nemo/core/config/schedulers.py +++ b/nemo/core/config/schedulers.py @@ -114,6 +114,16 @@ class NoamAnnealingParams(WarmupSchedulerParams): min_lr: float = 0.0 +@dataclass +class NoamHoldAnnealingParams(WarmupHoldSchedulerParams): + """ + Polynomial Hold Decay Annealing parameter config + It is not derived from Config as it is not a NeMo object (and in particular it doesn't need a name). + """ + + decay_rate: float = 0.5 + + @dataclass class WarmupAnnealingParams(WarmupSchedulerParams): """ @@ -270,6 +280,7 @@ def get_scheduler_config(name: str, **kwargs: Optional[Dict[str, Any]]) -> Sched 'SquareRootConstantSchedulerParams': SquareRootConstantSchedulerParams, 'CosineAnnealingParams': CosineAnnealingParams, 'NoamAnnealingParams': NoamAnnealingParams, + 'NoamHoldAnnealingParams': NoamHoldAnnealingParams, 'WarmupAnnealingParams': WarmupAnnealingParams, 'PolynomialDecayAnnealingParams': PolynomialDecayAnnealingParams, 'PolynomialHoldDecayAnnealingParams': PolynomialHoldDecayAnnealingParams, diff --git a/nemo/core/optim/lr_scheduler.py b/nemo/core/optim/lr_scheduler.py index 92da9938703f..269a80e2f536 100644 --- a/nemo/core/optim/lr_scheduler.py +++ b/nemo/core/optim/lr_scheduler.py @@ -361,6 +361,15 @@ def _poly_decay(initial_lr, step, decay_steps, power, min_lr, cycle): return lr +def _noam_hold_annealing(initial_lr, step, warmup_steps, hold_steps, decay_rate, min_lr): + # hold_steps = total number of steps to hold the LR, not the warmup + hold steps. + T_warmup_decay = max(1, warmup_steps ** decay_rate) + T_hold_decay = max(1, (step - hold_steps) ** decay_rate) + lr = (initial_lr * T_warmup_decay) / T_hold_decay + lr = max(lr, min_lr) + return lr + + class SquareAnnealing(WarmupPolicy): def __init__(self, optimizer, *, max_steps, min_lr=1e-5, last_epoch=-1, **kwargs): super().__init__(optimizer=optimizer, max_steps=max_steps, last_epoch=last_epoch, min_lr=min_lr, **kwargs) @@ -493,6 +502,70 @@ def _noam_annealing(self, initial_lr, step): return out_lr +class NoamHoldAnnealing(WarmupHoldPolicy): + def __init__(self, optimizer, *, max_steps, decay_rate=0.5, min_lr=0.0, last_epoch=-1, **kwargs): + """ + Implementation of the Noam Hold Annealing policy from the SqueezeFormer paper. + + Unlike NoamAnnealing, the peak learning rate can be explicitly set for this scheduler. + The schedule first performs linear warmup, then holds the peak LR, then decays with some schedule for + the remainder of the steps. Therefore the min-lr is still dependent on the hyper parameters selected. + + It's schedule is determined by three factors- + + Warmup Steps: Initial stage, where linear warmup occurs uptil the peak LR is reached. Unlike NoamAnnealing, + the peak LR is explicitly stated here instead of a scaling factor. + + Hold Steps: Intermediate stage, where the peak LR is maintained for some number of steps. In this region, + the high peak LR allows the model to converge faster if training is stable. However the high LR + may also cause instability during training. Should usually be a significant fraction of training + steps (around 30-40% of the entire training steps). + + Decay Steps: Final stage, where the LR rapidly decays with some scaling rate (set by decay rate). + To attain Noam decay, use 0.5, for Squeezeformer recommended decay, use 1.0. The fast decay after + prolonged high LR during hold phase allows for rapid convergence. + + References: + - [Squeezeformer: An Efficient Transformer for Automatic Speech Recognition](https://arxiv.org/abs/2206.00888) + + Args: + optimizer: Pytorch compatible Optimizer object. + warmup_steps: Number of training steps in warmup stage + warmup_ratio: Ratio of warmup steps to total steps + hold_steps: Number of training steps to hold the learning rate after warm up + hold_ratio: Ratio of hold steps to total steps + max_steps: Total number of steps while training or `None` for + infinite training + decay_rate: Float value describing the polynomial decay after the hold period. Default value + of 0.5 corresponds to Noam decay. + min_lr: Minimum learning rate. + """ + self.decay_rate = decay_rate + super().__init__(optimizer=optimizer, max_steps=max_steps, last_epoch=last_epoch, min_lr=min_lr, **kwargs) + + def _get_lr(self, step): + if self.warmup_steps is None or self.warmup_steps == 0: + raise ValueError("Noam scheduler cannot be used without warmup steps") + + if self.hold_steps > 0: + hold_steps = self.hold_steps - self.warmup_steps + else: + hold_steps = 0 + + new_lrs = [ + _noam_hold_annealing( + initial_lr, + step=step, + warmup_steps=self.warmup_steps, + hold_steps=hold_steps, + decay_rate=self.decay_rate, + min_lr=self.min_lr, + ) + for initial_lr in self.base_lrs + ] + return new_lrs + + class WarmupAnnealing(WarmupPolicy): def __init__(self, optimizer, *, max_steps, last_epoch=-1, min_lr=0.0, **kwargs): super().__init__(optimizer=optimizer, max_steps=max_steps, last_epoch=last_epoch, min_lr=min_lr, **kwargs) @@ -888,6 +961,7 @@ def compute_max_steps( 'SquareAnnealing': SquareAnnealing, 'CosineAnnealing': CosineAnnealing, 'NoamAnnealing': NoamAnnealing, + 'NoamHoldAnnealing': NoamHoldAnnealing, 'WarmupAnnealing': WarmupAnnealing, 'InverseSquareRootAnnealing': InverseSquareRootAnnealing, 'T5InverseSquareRootAnnealing': T5InverseSquareRootAnnealing,
mLT6;h-MWZ1okGD?cBh>I zgQs@Vg=71Yz?e*fy>Nc*``3TA{HAd8#bifKj`X+KZsXmxcUJE6S}GFwmG(IrPEnhh zWoz`A#8?U&WkwFZ7DC}C&Y#pt)W>$nJpTmyl>KK@*Hrk=rVb{p|3@}0SWNjjEsYE@ z8X$jpz)Lx&r@#~eiQ|El*`%rBQZLba<1bN)b&&fm1_jLLh~VxjRYbDr3V6C-8xk$k^M0*qY-4=tCU1!l~wQ`_!aZ|fy_BSpN{_ET7>Do!S>$$pmL7C`{+-uJ4c zOR`x!8K8gN(${TkgSQh~bUyLQXEV+aUB0uq9I#9cr$lKEos8F`45p)vfY>gmo>Q=G z=%S$by)%~5iw<-_jcd!@=aKy0KG?v`#*Wp;4iOLuTC5QgvVLf2zZeTHa-RrQ73O2v zDdj1HFvhY;_%bzl;yPXrnZ`qPKTmk`*E>8cL@>Tz+j&sye9V{6X9Lrj+(D7{Gq6$l z^j65ohhk1}ih@;!q;@unt0ogGUDNfNq78Hms zIT_=I0C^@X_>w94Ys6T;``~k1EG1pb;J1Z~$EOW0n@fVZITpET5%GETlC4^l z%{uquCAPnLK#2n(hk>kAApRD8H#r;{EPI=TYD(iLV>2oVYgJErUspfVIP6TXlib`wxpSA0%{6(9GRwVd(WGv= z`GBw_W2wrD8bMjLy|Aoa=8S1lfx4zU%mgX%) zw|J;Ug{!0XikJ2srd!kX%{^U*eS2iz+kIUGTSs$a38uxqzaGg@rpq>u$b2 zS|`hYczn1NB<1Xl_$S$~;T`=h^BIeZk<;mEXLHTf8<0{WpTz@;C74t75Fkm%ob7yH zBmQMx&zVJl)YU)`Tf%U1+;L;we6JGuoO6x#pcxpdj{xnHC!YbMr+IQTuBzJ_&)Pe> zknIjZiMur>i?sKx&`c+bND4J7-<^{Agp1y~Ol{);6vPjt=e}!~Jdy)G9xmIRuSoIV zgtk`QmL4Y$EtZc+uE={%K^EBrCj6ZVe>`V}~S?sRzMb8CXz}hOy>FAVf6Sp_q5(8Q#k3+1I zr^|tfrCBz{J8Q^8C7%@O6n<3XO=f#qa@k&eEt$N#pdB+GlAPK>y776Cgr1!_ofP~= zFMuN)wD0u5I96zJ>sD(Xp)*t{#$Y-z^nB`SrBDB*)re{vi-Rb@!~~1{dUi%95Z#Yy zr0;f!D?{5B>dav6_oq~*ScBtEjRxTcMTDKF4;lqNZ!ZqwpBseKoT#(tARjk$yt1N~ z19@DlHx5JPq5C#)?k+S(JC^lPYLzIUkDJNUk>PUexM-Yn+M;vUn!DcjTa->fV$YpT zi8`+d~WwG7Q(4!PC+K^*u7UF!k=&tCvUSeM%ms6@Xk(X0*#yQDoC() zIevFht!Iu*obHUPi*Ri=AzfejOhFRwy24T-p5q=dmv&C7bKKxeki#lafWQ6-=RAk) zWDo_fprz|){l4FAQq{gs>O2oj6XObqNMpwWq8?yGd$YKpt>4Wt+Z>uQ-g(${=w;y( zz30uT+Ny1rdq7+Cze?0;uz9ImJLR)NT?SSA+8k+Hd5K@YrOg+%YCk$p;M`gM-(pf` zRc7Py36>wg(5ny3S!yI8;29p3fO+jx)*XO_Ay&WtV8JCw`nEQECU)vuFvuY!PQs(& z3xO;(Jkp;!jZcrpJWL>IS2%_8j9xoq^dRsvc_kaMVS$~F4#{?{M}IO}2jnXv!BfrG zI|bq~sLPRk0&X9sKdyp~eBBeRA2bXft9`tUMIkxG=CI*K z2(ceQCw-n$NyAftD}T109sZ|k{n3E`jQWs{A%}OMi-zLOGydXO#ueEqHlnPp!CxvJ z_>JCw8y(!8ENUBilh(fIwI7(nFex0h!6Y_KB8@Z~jBJqOy|N&~djSb&&Za&N&Yb^_ zzSC06UKRDiOQ#M-qpB+SS56x;O-GSN(+zGErHh|QYGHu@_OWHwkkyKtv4@z#fYLg$ zYkN;I)P_tG~Z<_(D3 zsd4d7ZdH5SOD8$xo?J0Xv*^IqA0;kbGlVV&|jW$sM$)muV}ozcvgGWykD%{ zxOas6pgU>l`*iP+z@&BFadzke8?7_}Utu(jcwj=#aaD;Qm`;@y;8VD1_>oap_Zy2YP{r7(23By zf(dh^W@TA7gAwjZ_1h6lhug{$_IGztp`oD>&e#r^U4Gb*1>cpq(zq{7QtEug*%>{w zWA0#C(^N~&4jgvc`aBm;ZK3MPEJfJb0H%16}T8GbMxA$t@HCVnA8o6{N)MLyqr#)Nxu&^tSC^FIbWM%~{>&%JeAkmjn;37SXsYY3K1!vfwDu_< zh9*Yp%mJgY4u9h-O(4*p9F{(QgyIW>dBj+P?!5IH)P*a>D<#I*97t7=9m;F%c*BQY zJmGgc;A&#&a_On;Ir%8>x#=QQTz`**as0!| z30P8Q(G&D}!7_yQC+H50Fc4stxUdK15`%08M4=ZU zkf#NHV@u2Ya$tWbOB2!gZY#riuD=&yl^;=72(Z?oPb0Vr_Lr9QfE(!Q(kLxeTdz|v z)Fp1pVfkCkl4{1c=#9dr6`7KKKJJFj`w+ZK|MA^w6ipW2+>U%X;}*yoz0YRsAKtVI zi%q+Q8v1&nI_=KMTEJ~uhCO2^)u`O=`TbJ$Ct(Uf5CMw-8j--$khmlNGa@#&WdSQT zuZuI}rC_|+9pm0)_%4~`F3;Q}+;b3r-4lK{nuW!Qp_1nWuiq;hx6 zWbydr{))fdP?a){an!NcD|N|eI@V;g27^yB(+P%wBqeA4%cX4R0(+p(3pB=_SM(JO z6dLNrOu0695sy>1{U%G2fDc|#z6!gAcoD!D|4S(!&iHRHfe}D61=+w}0$zyF^o{r- zFl)<~8NJQtghfE}#-b9JG}91anCs5M`Bb(E(kYf!S7BLn8ovIpm@b2Vp_E6Svz#5{ zS_W{ObNQTic=^&s=y%vt00erOgnV~@4=sl`o9zOFtMZ!23C_V1Zy; z-{DIhnA|?oqr~2stHz~bH;!$x$RiEGZhgMJOi)AK^r~o0v{_UKma;=qSH{jLTC4e2 zmWiwPvhj3SM#sNU78Vkw4@dTAIb#-+W=8_x=1fJ~JY>HVrAmXYDV6VLcyk))jb_!v zRca!f%qD6CEEZ~HF9V!2?J_7kW6<~fp#)7%AHI@(dy(m&3HyNlzHAG_=6*(5A;)0- z<4d~<_o`GHY!ND_!-asG%7S<6tPC&wZ%JmVykq*}f{Hf!bTMOpYGKg5n?PmJ;S2m2 z(_40E&bRnYaxTI0)J?Pgfz=Wljbu*u#mG+yvMo-Vyy$=c_S>fl%K}ZRg0YNr!IQ`H zX!0D-mY#pGbiTY`0qd(n1@m~7+}6H6CINq7xAhkoZ{Qn83?Fmzr^hS7G=NF(+{4gjf|Aju4e*H zM$CAgU-%9J2YZHT0JdtHIY#;y{3P@Tl`@PCdkjMra(fDbcjqt~bwFev0kdusqpgBH zh+HLF&Ji&(rPNsV{Az1KC!1*i+35r5Xq_LCG*b9N3Vg1n@`#m4zlhpA8VEeG_J%TG zcjImrG>$#>^vI)`G*z(o0n`!_e8Fyv;zr$A$E4a7MmP7!yy2ChYe}YRknZ+(vjRxW z(S$){76`P=P20Ly+-keWNE3#7EM3Y7B}_agiute@O|_0HeqvKV@M7!}V@~DU8kTZ) zU%V9l1G^@=u6^K(1Npc~wA1!7->B&;4#(g%UM#S#(zV%Ga@p$%R+x`VW1TLHe((pD zb4g_O55E-}5Mq&|x^cAVJ0@J?x&_jm`a&lUg&VGJGKm)CJRQzX4eP6rB1`Aijt%uY zhICm!zJ6u7fMQdmH~kGtWgyXp4E5DTaTe0g2~CSpUd0bwH;3HxixUF8iE#{z1lvBZ z#gzbdLLB}f#Ii`Y9EGRC40>y*Rig!+iwL(kJrs7M+yE3KwHBho!k>o=Qk_JdUw!WN zhTB{Ui>n^MxA8MOPs<)G9l75UXiGf?b6K@zG6}_;*UcJhG`%uhDxas(sLUw=Bao1d z1$+!q^*jg5%wpmgNIQ8et!c_^443oERiVX3V_~dhE#;d7qDhq7?pGx+L}EfV4n=AZ zs_hOHKE3xIdmB^gb;z8jSY9UC`g<}LH)3mNslRIcvJ`i*#uL55VnQepHebD-wExOn zhj0^q;J{qe|F$od(T|yARx+aMa_MbPB2#_zcMbq#qg>mW=r2|}yI!VxEXP5M)X;06 zcWfn99YDK3{)D&Wx36(r>);M;W^kT?3puJ>91cVygWczKtaL6=X!r7`kLn7(E1Gv> z$W=ne;fUaJ+dC4NG3k1xsix1}2p~NE7sI^ha&h;W$!rW8_MPrUlKN2bG_>1ek?J7% zqhEw~6OvKJdCF@01tcevlI#%wJ?EkU;G9#5MC{bD1Bi)O?`(|sTPZJ>SuNN_(UT_V zRns_%a=G||6u;)>Dqt4n0Y=E>uoFEt&=X0O;pH{$hTuujenOZGG~QLmeGqY>{>|C8 zX$VAOs8(T$l?)bKd@&USd_alGAgQCTcb=X%h)&y7ZIvcAdn`kN(+P;nw1<|G1cyK0^qQI||2LU0uI@CT_Xl^}lOo3x>lx9s^} zCR%XdZ_W<}z74NFhTY0_KDW12uWbA@|I~D*q=^SvUX9IsL71M2uRU@w;_~EO3hqLa zOS{$VPZKgiE~A=G=5(Zd!=%{W59hi1y-VW3P4|dzz$6!Qyp3k}_z)U{ELRh;xT(sh z3A2lP%#qsWor}}pa<5TQOgk4LnMU%fFXk{5Djny#)LEa4*g0{1>2M&pC6p(}LW;*> zh3o2GO&FJy!q}*=3v^J;)4{u(SXXT zc{O_fwAPfmGjD-=PxRH7A)F(n+EQxAg9cW@r?LBb{WWjfs&MI*GoiHV^d9P;|M0tQ z$F)8i#VbDJvp<-@0l6HlG6j>pg7-+Y;UrkbR&yQ*rH0-3RqV6)oTZrDz{@vQxI^nf z#;8_9SEyC4KA9h14k;*})dTx_zcQ~^w(mTDprH~JV^-xzHVg3U4J@*YI!0#F=~uh*lV!v8~_f{SsV$^fcXd*cwAez##r<#UuTUlVa%u9O+U0a`0#zZI>rUIVs!s8(?14gJnl`PX(hxDj0Hji)nbl!8J_K*lrbR-ybum&-#gTm9 zY?UyORlffbT}=4xXS7o2>G&rLG4p>6?;jRTd`#r;@h*t~7i@o3ymAi0qGdUwaazv6 zDHo{VWxDJQ05>SK|6Uc`Pn*ZevX^9X+)q*h0WT(UywaJ&qQ8Z{26htkvW9OvvvM4t zUYBp9ma_Zb-UR@WrGLJnOlSw+Pqk8my)mE7!Qb6H&!u9&1YamnTUUVhAS(>%{EtH! zGXHfT)s)*jDi6`hKf_EA-sfU^6ho`MH{Yz5?*v3uL~LWAodMXKgZ)&0!Bu@ zV%H2|lJ4h~94>PhVChaMt@GUEig*S~2t%rXOPlCxT@m4UuGd+%Z0ZOs)=7ScGaL#> z9;-9TTu#iGR*MbDW}~@*$=r6Ioi-R9ndZ!9IqOK22O~)Figg=d2$+)%=Wu9bVRVSt zg-fxU0yapM8@@b{ihcJQ(7O>%P~~=6A=KjYda}|LhL}J4-5EReP0jf`PnX!OG#2tTj@|Dm1q*IOijfB}AH= z?RF$IbDR4qs!ha<8eC++9f8226F+J0+_do8EnKU_dHSPv#LCuBQhv|V5|2xYo|H{Q zD~*PMJHr=fq*JPE5-~Q-pFZ=HSoVnzN96QPwnZ4<`&lxLRI94Sb9Pq{5%lfPxxv|P z^iuucD!2TElhQL^UyYYh$-<||ad+aqNY80G3mY7B_}sO1+9NE_rdb?Mkjve)d21&# zrywJFQ4U<2TKi*|=oeOw$pJ9eEvTIKq2`@iIJ9!A9QoBlOa-!I6G{Bmtbhwt)TWl! z+T|xpgHQXVQ3G&z+<`PjTzai+uEN<0k^lEIOp+;W-~68Mz!1}}3DOP4pRQA<@D%1< zEz8<1Oqu;&rB7O0Q=->prPieXkvgf;U5fWJ%naC~P`V(g+z*5b7=K6l76E|SR7Vhx z+d{H5BidL+kKG}OJ~DZ#0(-px`CA4uh00o0jpYXr22AdGclG`rf*~}xoMu;ORDNs) z;m=}EU9UHF@UFPgFwX4lVwH-2nK5PK zvmk9g_BF!NL1mG&0WBbsfq9qXQM><9xOqmP&L#Ta4n~0Q^}x*m;Y!r*mXD-#PoOvu zA$?&n)Q6$GS}Fl85^wsy$pXj{`u}pkVJ#^){|XUU+tTK-d2*gT0nX6dEE12j7|1L5 zGYee0qN_$ixfAa1K?j2fsrpGl3S-vv(U%2C&{C@YzXoD{qk^u3dZc*HJ+IJ^2E0u@ z6TgYm-?rCL@`5K2crbm;d%tUvejNKzknBzpm*9Tjt;?!FB4~Gm%lkTLGUYWb)Q|DJ z-!e8IW>=mk_9Sr@XN1^dLs3o-R#S$eA->-+a9OntWC?cmJie#dBZHqapx zItgY?;kxLPTc^_nyLp)+4c40zW#X1fJr0Gw;O9QB%bkC!^FE&M#0-_aRkpMAC$Wp@XtrFIp9o6ORpql}OJ z)X@xNRr@zXaP69svlRpJ@C3nLQ+dfW1wQaWCHs{^FoY?)V%$aITj)F|@NEEEL#}`hUN|AV5Y30 z1i~M@=xq@H3i*+jkRgZYk!H>7=mxyzuXovB{BEL`=*4)Q{c4>?mF0WE>eYk!pCwRU z-~%GImd=y-`zJA7V20n+cx{i+IA+p~NG86bx8Iv$J<-eaM}l||{UR0MVId{f^79Jg za?`@Rf_JmW?1y*(OKdGw*@F>;pD*vh`~Q`~fl`67stn&@bwE*HGc*~U4}JX|0iIZ< za81On$hIO4Tmui~9f$r4&~_)jUhl7BI>$TQ)~)8*TdbGDz1~Q&*1O&bldn*F8~Ht~ zCbJj|>%e@GVefDsk%dB~P;I7YcV^}{wcC?qZ;NR=Kl%c^EHds-D%|u;cfX-$T;qxs zRVS=|HzVp>WmEU3@uJ?G?NOE?V35F*@Hi1v>A=aomy6;{?HK`UpAGg(U}sFKVq&%$ zo`cX=^slB?O^<{yWCDRDxwxm*|GhLoz`lTS-cZF8-dXYAN-IBF@T}m-|9e)vkwlY) z?6D5As17Ma0$xHrJUo;b{rVQvp8j+gk|3F-CkfaARG=_{!Sz0cc%Ia1Y$(#n=_ucT zlX0<^VEe^*DytR9fDyFo;%VKFNY%@Xe9BD*g@OKQ+A0-5ishR1?`t4zLQmSFJ2j2h z1q?h2lmM|TArAxC`+*9!+O;;Nuf!?wrRuPdEAoZUzdyRT2mr!SXe$mIG4QQ1(2er{ z=~zWuaFA*WY#vuPi;oUg2~yyZT{?aLU@|V*^S^GtZVZEafk9J#q(sMOh6-5(H2qxZ zgJ={Pcrc&CzKwYExADdH*&?6QE&qT%R;V7uuL@MpkX*D8_HWBiA-FiYNnNNNM4rge zRD^qQO1hl*tEBZ_eY@&I{OV-1+hzp^;t zD^R}xS>6ZpK6kYwPQAzJtyJlXs!^&ACRI%7W|Z^w6@i`5r`coM>?R^9uISmXelCYJ ziHp`0dlXZaQNX4BR0@W%*=}<=j*o3<2bCz!nooX05huhomhxf@Oa=c9DE6qa;mv_& zit&wdJAl$*b(|t{OP!u+2Xn{Ml&!c+l~3;I$N979A;&fmu}KS9)aorXTC=s-sbv$da!SfQYVLnV%uYILg1 zFeh<%v-Rs8v9Pc(xo(p?CLtl6aTp0NKA1*eL7>~`TsavXSuL75iDA!L+w~<7=I}nP z#s}5CFP6zJ(OD4ei3Tb-56^le-PO)eL?GORHQIVHEa3Rf?-q;n#gBg#LR2WuyB_88 zGcJZ`E0|SSDaf84SBRk<-(_U=S)5|9ll2pPSyO$}D4!8-NdHrY^wQW-yaeD4ehTRV zf8N3%?<~kSIxNax4OvfiFeQ;o3!>00vxx8jpG8!}tDLWfAe+MVoF|c48-dI7CbQ)8 z=g$eH>r7KM( zu8l5C%Japbnv`9><9D*#eL%fBs6QO*WbCrL$__MCXgqW zef#~a0%SckGpP@xtbpzC9om56Kmo>*3KzDwN@!uG{#qRe`a3E7H$7qYjBLx@GwMIiZM#bDc(;(u}jI0Yg)Xd!`YR6^j_#qt&tLzjY#bLcV<*u}}~@ihs;-H6l<9m=Uf`k)A;o5-{v zW!7%SK*kC1@;%-N+f7$ry=i!=hJCPld5IhR`?7Co9bG~$73Z8$OqNs-CFVsm{QVt&g}|yA zn)3@-48(|3AL)eOw6Bir_m%4t;7+Xto%0zws#L#N$?=;YUmjooX+qpmiFVaditrCs z5dx8Mo|kyhEmY$J5>4Ph!9d|$D2l40&(aP7dOQ9%zu6ZJtOe( zah`0+FH%B2wn!R6pBrF)4>*{w4}A*2ZePfR1{CR{8lqI7$p!xfkvrliwUXQdV(Pu; z?Zv)6_{U~{=5_3W1qd%Q6*TURwt#K-g-3;Q;xB+jS&d^zxtw5j?;b?Q%&j-z@Tb=^Re=vzYGNj~m*2{T63F?6JH1*>EoQ0+P6+6zEu!&dV#e&f!pENmrUxIdK^?p*9)n`S`Bc{d+yh>>y{`%UgM}Y{ zhJiZ&c)qKer(6(wH1Kmjl;#%9B>rHhpGNP8ZNPDOQ;LPWq;*w64fqhtto;<6q>YNv zEz9lC1%2*Mu2HN7+_9?h{-VYuYydUB8Y}MP)^oB|`3yNVA}E)Wh3c^NPKO^1eZybe z-`}gx(Sc6bIyy4l^Y1GUf+`_P<|M+&#yL2^zto$$A4xfOhTuUr-x&&(3WQuJG>|1= z0-PM>*2CR>X@9nQrHYG{0EAt&{DMUTzfBs~CtsHtRX2tfLnx>H8F=-wcs~8Jo$*kY zSC}M9lvl-bT$ z)EwxT=pTYS}uP88RuxD_AzX>$HHHeU3v74(_gyqlhE$V>=K|R{3 zjCv#9EkAU8%F^H4(tj^;$nQxNEL(4H0oxZFpF;QAbPVmeObW&L@Y5P`#X@%Fm_Kc%aa zD43e2nt`oGs1qn0u1`%e>xmlKp{-I&*2iEK-Uf5k7h~ZaT<%R^!Ql5}X)|hn=VR0? zm7R*EoN?_8xoz7y*K`P-We~5BBRn$=csN)Zkhk`85zJs1lhEBbVdK4|)uo1jS zrEW*UHL?O$b6i&9x8_H=m`Lv*j>-Swv|<3KJVa4`%oI3w85;SvO7 z{wDi(-v5r24Lsyy_GO!=`T=Q$;Md5&2<39J<^z0_CG{R)msWp0-+@55MNpm%*&*kr z716Ef|BJJ?0Lps(_D5*}rCUls1*8p-?vU<~1_c!6DUAX=xDzlmr<;VtJP3g36#a7u(+9P?8WNpD%W-v zyq74lOUp4&TOY8}rd?xebz@)*^7(LgfrwGxY#XP-J_cuY+) z%J#nS5>bPL}|~IYC0wa`1&qRAJdcWS#fn7Rek4_4AJC6D5bq zVb5;kKGyZjX9Rc_V?LS*o1XV3>DMy~U=l0pT&(qg272d~OnDlzJ6hQNw#NGG!ooRW zmla~y1DK>!!HCy182-WNqvp!{_Uc6vQ=Va#@zbOI8_0?)_jA2A-+=-X?eht~Kogtg z&BMb}XNRWx{Ai<{QPTw60A>)3my*aCh5xYw7E93@RXd0N09^~WziQ<_3xJuYBrw*1 z8cT6w^k7`}Q~013h#)()bkBqEo=pY(ehvpv;{V?P{1;bNPf;g%@Z&uuz-wgcW{3UQ@HK4z~fi7F%x=|;p3Bi(!HU4 zoZa8=L=7#IZcU$bmQ;6pmT^aQy2PBv0o7>M%$DX}q8lt!&EcU!bm&e;Z zMzt`DCU2a^5zQTOdXCQTdT!3&`f#^c+h8I5{L|%IQ=ey+pI1227x}E_I;=OEI5Lj& z7C-5>V$;ruRS193LgPK>ja>Ih4)XBilikbq7+xh-0T6~%x zg%JDteYwgNfOkF(4e9-yCh-Pc2MJ(t1;#ZgplYZdiJ)9#6!%H6HF4Wsep6)%H{e~? zvGdZm(3`ectUpF8Y`=&QWN*gt+u}RTcZELLfK<$ZbScMwTd*~Glq=UlNn2`8NBjV-22m)Lzr;{YSo+QyZonZ>K_zB;i zrpA3YJ(}eU_m*FBd8)?pF1dE(yB)OODsqJdi)+SXsq#aFC<|BHqO0TbQNB?af>+d* z7j=AQqL*GaNSI%3s&iVPq#LfG4JKqjD>RKC%G>k)fVT@{Gx`<$1%k~k3J$#h?jIh0 zk!t+U=$BC(moLj~#CQ`Ga3rnie2FSlt1?bRHgk} za{S>G95^9FjFh2l8Ev`yTfh%05(deN(G^*Vej?$mX)TVWwYx8K!bQc z_fL-nh3OG+0d+BGxY#n>Gmr`~ps)O>q3Y$>NECLRO8g`I_O_$N4i5|jA4Io5V=nKk zyPINp>$18ZB?vV2M#Mba>&lsr%_+Laapy?1siNOe^zYg@wzEh69>$(~JfM|=8pZ0T zx+xNVyh})&IJiL(x8vZVn;iq*ZsP5o#XgLNXMbk5*T1m)RBiI~jvD$NA5g4KHW1CW zM_S9Tn~SMB&p4*XG(l&ZA{B(sOk0PhfAEI`KS*wU@dcFYhp#_Hot0!z>AuVIHsq(C zdV5DC(F~d#O>wZe5(02GnE2o&zEB^G24vHk-_I^0+yf&C3A3EGYS(kfE%jk}-X}`Lmt^4Dy zj@e{VVlKHBxu_wlS`_};wI+{k*Ns3tGWN6~sGdgm z^!yKAe?D8=t&cD$d2S;7ut80g&f}ZCP1FS54SG(86~-$BzBzIXTOD7|O?n<^cn^8TA8Ks|$`vL+B+~u`71mI)}!O3LE z`5ywA|L>DIT*Sm_LO004Z|?}Ek1kKH%IyOE26*zSXN)-`C+v&q@duH2Nr*9t7W&wA zCd{eb95N^`_3u?*QzB?N1Q-?HrAjp3ULByI=mMZ$8x25w-@yK_VQMwHs5vkY+c;xcM5V~ zOdg=I3|SrB?9%Viba>-%n*ZdRe68Dtj6mNI~t0pV$O|L$ye1E0UV^Nt%4euER z2PIg{2pzOg<;3!Wl0QLr_>xtP;fDnPi#Q_gC}Ey{u8cb}D$)>Mmf0Vf+|X1LX_|5K z-wvVuzdD3^n&SW2A)t}Mt`Ur%UGHvm2{C0k{p=uVR5?br1YE+XKtR6y43RvDv`f4> zu_rq@HWp9sN%hVU@;MtJ5Z^hz(el@Nq~EWhZh7GuZr|}AAxQ_ec!2K*nfmO@UR!3H zKKbah5bJO{!0|mQ9RjpaP}742xz(#bVq>}w#pB1%kyIc@ch@r-To2z zFYacER~7GY8b0&rbo<;UM#jG#h+M4p5K7Yhk^2u3_%aswB9SD zoF>75cUMBg5d627iK+nb_aDg93n(lsEYFRmQbZuWI{YY#wOM2Jn|0u~pPS?#&U0(^ z&HCUNL{+kPF9PM)cY0%QgA6sP_9j(xE?2!;*?kYLynD>rfQBs*&DQxY^_G<=JhH4K zbOKs|%HhEDgHHPtinX_Y8P$yyheAK=lOU5H!kDBN z!s-Jt4|j{0`7Bycd6f(y!om8(Ngf<;dHe`%4U0UdBbl1p(d>AOC>>UBT8>_xxrvwj zVZ>}QRckP}NJKu8p8rwL4Z~mA(C`kQJF1l@+7vn39y0B%erjWY>*93(K(Tz%jE0vk*g!7pe9 zU>}N=hDee_xM-YXz?pa&@`Uz-v(F`jdGwJ{D=J}$XsQuy2IDTa7cj1^_1QdfGFW}((O?$=$ zV!*yHLV9NVYfh8C8)ytsA*NIA4$&Z*he?lmZ+~ABxD<1^v)?wnk9;aj9G{@qkxV)I zEM}sK63N^LEK&PgmZyA7qv71**Hf*lam|Ko&sMz0^Tm=RN<3P{5!vbezB$VAk507p zUi@m04+I`xuodng%Us)e-fs%MMHkjMq)2T42VZE@Eq}tYH}&$o-26s6HTT7hln9&6 zUKku$?2}fjRRmwCScQl~0wE&o!Y$pxiY=|G`x#jsBG9r4R z78yqo0&L%g;~iRorHh2{pM+qt-5eun^4?BVKZ!PQqyUH>l(5JwghjO>nweju9=EzV|sMpOBmIr(qq;AczX*e{JEATa`tFn zb}i|7PBE0fy{G?;v5GA5!DZ^4LVyY&Vp8@t0G|^LM4EfPm{6E3vN&%)S;^mU$P!_B zkr)>bID=k+kskoeO|a+gd(f&1;U|O`hJ=+dSf&4^^Y!OXkqj*MZ|L^t9jg6aO3h zT%p|T9F=bCcS4iVnW`GMDB?I5uI-OvIrTi<)OXaO*ST5d>&BNAXZil#XWdu*o8!y3 zE7HpO060+a!+`fdrcnk>oM@v|?f0qDt#q)sC?=0Bi-et7V#=^07{v+qh%kk)B)PbN zlxgy2t-}OBhp6=YM3UjO<5S2TBhsP}{Wbg6Oo@s4Jla@=E#^2$M*vIE;qW|+rk5ON zT3+&+OZl9yDZFj|oEB}JZ8{r*o+Fn@QVz$Yrl1dZ_}Pm-T;~)rLu17wQtNBFk-eep zHlh&2Dd@(Y6>zUB7O(cYRUk$Xet`Yb^@^MHtDT&NbZE^X7c9axVi*dvh8TG7wLDN7 zi3a^VDxSna0CdJPer)=4|53KoqW-}@G6Da{SPm6qZo|{2?jhimv4W9!d zX^!|$lu7=QHC*;abEykbkQI#M9!LeQIv{)Q2#v8TT!Yh z1h}{;=3St$MFAH0l&7(bz!&s2U4p*uM|#J2 zjCzU$`uq!q)^-)Y1@xiH=CPIl)g3^$I`bq-t-6sZ13HH!hX%kQyGD|=dHWZ? zXAwQ86;SPzis~KAH_W_gHwAJ@-sB_Rr_0)n-!c2AkM=jaO2yO+!Fe<#hvwN7?wTwS zvl{-;h-9No(EcY|w@rFC>3?P`Au23aw12VymfdZi8hw8Cr?(BhX0me!caVWf0t4za z&d~&eNm0h!`9g-dzg8(Uk}3E|dhWRGQAFk)9&B@@qvU51WR*en@=-g)KJPuxmgy&v z_;gY`Qfl_CimzIp(D0W*Qtf_@ zk?M9j9{}HBOKmjM)t}NVNfpnSlEk<2Ei3jzY8#oTKNn_uOoQ@H^Dg*Ut_!p10}<84 zrq1{eh|q_X769C_a<=Fp0sxEIE#HM|at^{9pS~h5G$_4Fwt6S@;w)Gqk?|p_+N%5F z{ITw38D{^iZ!>MEcQ>r%?m7f9*T!%emQ?(i@1_kk5v)KqG0sBt?$~V&b||LfA*j5d zLxuVM%>@Wr>!Ir~U3zuDF{hkWI+tV!rb(k!nD1kSdQWXBmj- zB^*k5R2ZOmUirA&#qI#0SMXeu1<`3fYq|8AC zXMAJ%|KxLnULgWYc@*+BuasZ!Cjx@*(e3OTe~0WTWRHj|vBD!9a126e!Cf#VhGhiiNvO!+TVU|__kZjk4Ov^DD#ebz zrlVNt1uJUaxq5ZUgt$X@@Qo`=wcl>1xj~YU?sf_{Y1^z12qn-3us+Vln}s7D%WIDD%M^58(}fKuB{%yuUa2-6vwcz(xPoD) zhNf2YiQr*HFr)7giOcHfeH+JX@_z{~840AS>v8a3FY{>HVN@lxXMYV!C~M zcv2eLSZv_UKRHSJEM5p$4OIRVQk@IDkYL7|}93o_#>5G?` z$`>F$+Rs~)KKNe>ZG{$Fa5L8RoB)R2o2|i;!$jVgx-M$f6{{&EkiZ0Dm>Mue|Ex6A z^K3nQ)F)=I-qXy}R7 z>>}s{nLDSg#lKSN78n_1Y$(7VAQGjn76Py7{1Nj{p1W(3fFZb(rMQ+v4$k@jNNIkw zw~-cMav9(^IHa^OLB8UTN6YRM#wu{!Ez5ROO~Cw7=ULe?8wJ@_P4K={eJ%i$7X64e z4G>!OG#R`lgAD)Fwh%QY;w?Tflj0K%MDwS!blX_gzeoKQTzqGqK^`Gb1w+$rZV|Pzg3@{=Vm# z-NZ2DF9?x4dFovAhQrn5L2niNJuMl}k&=bLmauad7tw`IBT>SLY%qMAyLRZAmJULXAI(X>EV*P5bl{XVs9N`E|kX!;Q4FZ|$9J5FU18y=-OxPLaVr>RYj;e4`5X zG7NCgI?z>$;REsXf= z!@TxhuDyf#RNm@6i|HeW<%S=Z{+ z+8y!PURO>Q3gwdX@ONnN&YLgt8nN(8HxBvAYw_VJ(Umk_S@?MzY%tbfDQTe_W@*Y5&#zJ9`hkH1QgsmC6ir*zd_ph4DgkA0eL>&9-lJQK*@Et5jvzy&(UiW z;+{L4o6im*NL6F<%a`!boq-7-Su1iR&Xd;de^mY}<50H6pEHQwMJ@H`kSWLWgWV)b z@y)eQ@Vdwjf?_-6q1L|k0%%l!P)o}rN54Zh!2&B8Piv8aD^h)XZA-sW$ zv|F8#lkso44BNzf_i-v9s;KDq5Eio918(6n2^qxu%3PMsNurK`$M`t1=?N0$VRKWx ze#e^Pn(q|}Zk#5aYgIZ>T&kef<2DX#BUqM97Z#@X}}#k}wS@!+|x9R+Fy$<@(vS^ThX?U8R2B~$f$ zwmQBvgBK?SqP@ra>(a7(k1i~ri%$X_@Kwd-)+>i(*A<8aM^eA9uD3T_0iFJr(u`k4 z=FMgxB;X`iFivi`5qhkW=Io^LFP7KxFP3MuIsbQ{Q(^7jzuPAQJ2z5E%u5HOy=?uL zxZfuWxpUrmQxKNB9IMsMcf}A~2zNBks(dU|c|$durVGm~ z6(kW8H%la5!myJ@cFC`t7K5a>sCxsDacl4P?!#pjYuMHvr2OuBv4}`FjzABWX+2#S z6%Qe_&8{_CH9xFS+3Co=^Lv0i4SmjLOT`IDR{B?KW!It?jeX4_DJ7GU=J>pGB#DCoQBv?{J3X%f-58Cn4m*DAIT`{&(rC2MJ47!+)aCIJkqQ8 zlBPa5dY!D$tc=uQu5pm%s6~Gsa03WVukE>dwt|L2@+YOudnN9G65s#O(^OaEik3XW zYUt9DD8O^CJ4X5<$Hy33$_iK5(0$+aP^D4K=K>+lZ{>F*hk=%K&~=Vo)Knb#!);*s zVDm+GiXo{3>-33ID`uabL8X1tOQOVg&{h1ocpYe&?_nU#T@J$k-vjBoHU?4Q z@p<+e$5T0xYCS4gY0nXL_k}vc2?!;LV&u<<43d^&i&hNs?tK7mVcTLxJah(s9qA9L z4nXNd#763@-T0e}?Ru}><@Md&5EKf_$+Iw81+a>LXh^TT)kHK>x2 zTDcDquXu($-*R<0VnsbGI;09u_5`yDVbyxxVaNvTWy3sC3Cu!Iv9@4vc<~#c!%h$L z>w9Upv)#Ahu}ygKNHPA(oy}3jFqh5$)xN+RfQ@t~ZrA(};17R$!|9^HM9%i5Y*vj^ zM)$c+iure@-H+hqMPg)KzC|P_+g4wE>irkRc!pX)FzliRjwj!~h_I0p(Sx)1s`JEY9_g@*OZ%oxBM zhhxXVY@@{V@Y_B0-wVOKX6w8zwXvB`M7+lrqj+8VdF`YG8y!&NG7FuBb?&LYzmtj_k1zHLH<7pIJY7D%&_KwD$F7+X z9XDBJN1#k6PWko7Q&Dk2Zrp9dZ2rSX^;FFks#M3va*@=?l8 z$T>7#u^QCgwJ_t54kyr(E4wts-Q=C1EKekJS54GHO6(@M73RwCaVkq-F?tzdv+NJ( z@Mvd4Jfsbzgjrc*g@a9&8hbf_m2_|KgbQn{@WOz{3HF`uR@*XITU|1jI*}Kqv$tmM z1uW{^HwQg@fWQ9~s9~%LdJ+W5Ml0=keq2_H;Y2m<#qwDyE}d$4zjG>cV6+dMG*)Lr zd261_cN6F3CGL#j;t1V=?EY4n!*QFG>UxqHm{x4bk=L5_8nVUckQm4sM$w-$3!+5G zgBr~~cZ(#cy>L-uXlTgmTX~YhX-g>CtB_+5H8#R-LU%Ir0=5e+J@>XuBGCniQaVKM zyt=Gu;7Yu_7Ak0rjNe`RPe_vw z%5R@j2JII_?_ituGXG;KBO+LWuU5$#WOfvo9WA6w8Z z2u^(#+^aJ>J~lOscnp^g0>6b$-S+=n!9+zU!dD%;Y1ngxsxCPJO0K8Z(ozEAHdPH& zQW;a8+RoE^$!;9RJJXbSNMuM}FgEUX8+&6ywAvX3!&|V{w}iwSdJM6*T+#3C6mzfl zkr@){%KUtB{HrNHUZ9KM?z7Z|2qu;`i7~>&rs(1Nr#Fs_SZ8Q$VS%@CcdB8JSZ{4q z!;{=kuX%gg$aWaS2nC1iBBW^FV(y20O{Y&E~bIO(h*KeM0nVZyq zP`0Ks!s^&HUr>}5PWmD=Iey74n&Jm64X@`#wETK4-b+y~pUHN0zajmCJMF*y!c-_$ zL}4=C9*^{zrh3kIay1!=9%ZPMs2bc|pn7o-y?X*FJL}I|_7ftVvF?A)4q@LAV%28F z`lE%WVWxNH)K+wP%=_f0!s28@gR>L4^u}w(vyO3k_br51H-p$A(-Z0`BeaiLi}d;& zqiRvi*btc&lE`pL=lV-vSI2My{*Q@aqyHlL%7$5ZdWGX`n`E+>w;KiZZHG%27{!c+ zMf`nPQ&5bU1=cCVjIH|~(^H=UkdZEGpvYX>Z|=UJ_OmtQmkg-Bl?mId`uzpZ zCmu5S&V7^3HY~jc37~q_m0WGEliufUkVmfQZ;{j&2E z?D(*Z$d5!C=hIKT1_RElx5bYKgxC!0{Y_#Nq$5uWehv5=3s;1eJ(KxE#fe2*xf&CZ zuQfDOuS9Y`+b(>jFA_Nj0pi|4u8e>%+NXHDkgB<4{`JP3!hULS^!u_blPdN04HE=j zQfSpT|FzUk{K0K84LW1gFWg$+U7WjmJQpPGO-tidjfx1aTXip06ZR%PLxqqjT>Ndw zHvhU~ogB1>n73@=>NjVPMuN^$U6@>09F_!0vyXOp(=i+xX_2`MNrql0ZjWUhUssTk zR>dZB8w;5{7o?kFWe`%T>aiazx6z9Qp{&42IUq`?B=o-Xmxw@1Xqo1HM1SiE(!ZvI z&;P$6kAa6sz#>eXxV(@fgC&xB)n#RvS%vfM4KN5J*~>Bm zH!r;SwnujIb;pD~%{bkU$-V({FK_7CDMa7Ko5BfpSF9`b%gU?{3a7M8^I+GoR8GE5 zfkP4+7neKfzCj1hWUV@DUk{O?9zQuD zZpQu{!5o~7sU$2A^gtXvKiSM|j*tjBw^3}O*=uoMpQM%on4_$ieMOAJLg25$sPIeP z9$h~<_MA62<@#ek_<`o|x$4VH=o}}DeJocW%WsGi?d^N*tQgoSA_rLSgFd=jni@C= zQbA9y4sqGW?TT)r5^t{Q#C9+;V$@ZeHu%tg0|tv{!rQ;_>JPDQcaF!y3FVJw?o!;J zRcrDNW|9Jr0wa@=b}SrX4y=CG#pydPBNX2FXQm_x8Te?#(^^E*MPNCHWx^7BBH|uW zMJcMt#iD#ymy0aND03##e03#*!pwsfG*cC*7isE}W^7IANiQqoZc-g%!aSY(+TiHx zs^OGr77m$f-i?~w<@1<0L|Uw&Rac6if@#LQbL$;6ZWJ0Z^6D=qhZ_hJgd7uCi}R9( zyr23#qWIFBig_sWt(PP{)pCdd3?HG2gz!iN!p2+4FZ`>VPa`u8XJSDxxo7hggY9mu zOzgog-NHazA&@vIDm_+>i3-e7m_w*K6*7fvv!0L1?MQ1#w z|3uwvw{%?Dl37#qB$vl>gY$++ncW~3qU&9rI&?kS!j|xB;MYfbM_bQ(@;n%8%;O6T z&%o2kV-aZ3*f^wB-pTa$5_I4h{LDlfHPC)mP_+?DGi#z|0a zxg+}*f%L>n8E*XY<-AF~7yB(o@S*NahcIT`M3%3uBirT$+V&8a*et+k4eF{A%Gq6# z+0EcQYJ}Z>>iva;khhL4E-oHJj<`gAlTjN1_;a|m_Jg)9k}*<){j%n~s`{DPnf*sL zq|V#Y()=H2k34e~I+4Ik4hVP~dt8skTY5IYqTd8eSida!r+wepbKWQJJokeyf`#oI z7?(9VrO+>V>_mc~CqdX%H7=l~RwbT4w1FujEX*8h`UJqn=D@>R`>Gti(wRN{WW2id z*>Q9`k=vjb;AtxOYvCo`%VpaE4J4)e%>^!%hc~!K1s@|YPwRh!d2mC}gbIJ!1nhlG zgcx~;N9upq^Ke214Y@DH-nok=`;xRe`JDsYF=7(uKnV~py-UgXRxiLse8d zl1251LBuC6F)T&z-1m!TNqn}ruZ^pf1@((vEMuLaA)herc>Dbvcv0O*x0VEO4`9>X z*hqsKtb z*p>fr#@Ipnj3lz=Xk}eX`%}m-*|eSCmo6_|Y4@XvE7IwiW!rPSO1gsb?pQud7)0(D ze;^4uq!iBU1ijzosZeZypMup)>zm?WO_?Ft$+Zb7v zQzeo5pMp7S{^Qz_v}zTI>bLq`85#nAH=LR%Iz<;d9qMMA=Ee0J1<0wGFF$Vpd#vTj zia~FwAuD^HP2>c!cY>XOK(zSjYb!4Ew3#K*TJ`iN*sxwOl5H?o#gqDmc}Y<@iD%%U z#^+Rp;+Z1zU)-yfi=!TN&9i<4s}mpTmj_>}-D-S6=1IgDvLg6oX(F7J-&UQ7zrgg1 zO3{{&QN0^_7}YD$QbGoPQX$vhSC;RFAMroJI^y@JVtdsmOg4tgOe)3VaRDZbF@}-8 zaFj6QJA;puM_QD)k2$#Z8Hp7-rm0%YgU+Zw(gf@(>;;qyK$ajbu7BS~(xRWX$(xr7 z4->GA0n(H>D?OeNr_~V|00&`W8;G%s@6DafUPAVJqpa6cYGD^nm=(@JwkZxY9QO?h zh}UKpo`8?KDdAP~p)R>K9Z z8=69*6b{du*XnMK`{rA(RGB=89e~HmnZEOet)#u@krFyR%Preca&@CnZ1T2Qvf48$bhM9z6#-&|g^v?ab z{W=g}j*3B=8$b8)7XtWHTe}r9g8|Kr}^h&2__3xu{kv|mqrl>Wf`MI z4Ffn6T+GuIWzgWw+@hrRLq+O6aw~8AuI2uKfi8guCbWQEpk}GH{X1gG<@)LMCqki02PnLhLE$mZ!G`Fiu;e> zS&Bt&A3sZ%m|QtKW99<~-BX6u&nMCY?|h$iJ805Tc+GDdTe6JE2Bp4s(e&R@f1kj3 zR66=6@D|U}SoU;S%0w+q`n?=R^X*_R5ksR>gA!b!GCno7*|bhb!WtmWrDZqv1S54B-wavR)WHs<)W6_h4 zE(BsN@W&EaD83vv<-BfhWihJXkPw`{oC-@+Ca@qyqbGE_Z5tXgU#2FxpU9RI^CG1-Oxo_GnL8Z zg(xe$+qrqYx~kT5oBRcRr6UXIWd#eLPdFTPVViGuWSa+=&)nYJD027dI48VIKR+^U zqM;to?JfmGO#qwc)}+%^B712OU)p`tLlX06bL=vv7;Bf#_%V+{H>ReP<3i&jHO7nk zk$1|uah_iZmwJECJWMFY zZ1Iu?F?n=kk}W7R2_bUdatVyLzW|MUuZ#zHY#>CEH^%bKyHl@R@M(uQ5j-+N#YF!9 z28&$R%$bxU3c#C{3mLnT5E2UEux{30G2Y!clqeW?pIX3CxfD^t*AdhE{=TJt&FIQ) zRv0mTuj!GVh(47vgR3Ax!o{q?J|0PlMsP&T6Cftpq33k2+^O*&y{B#{go?Q98ujy4 zM)JAFZSrsHefRB_R*FP&$Fz0M2$EbNNXBJIfB3@+_ZwEqjlO^7&^yF#yK9Nr{D=#5 zqORsloC0SM+5TvIWzJv@>_`T)Dr&!w>hF3<`BA4GM!hV4W6-8`=D|Y2cpy2u)}4nD z*Ov#gF=|}xsJadQ#PZ-jVE%sfMM!D`A#qL&*8bh%rx3JlHc7I1x%PnsYl% z)=>fNz)_xr{qqjCGTnKV;jr}nvvmB9;MPiqLstx6{*&j2WPK}Dp<1c#zbc^3AE`B` zzj^YpphFepaYj`>^1~wEO739Z?|X0s0T1v@=N|2pr6o!Hbo6P(2Ph?qpK9tezFU*O z&;uHo=+tViFnMw0Wa)WcN|ObmTYjq&n zII^$Q{@6KVq7R85Lddn+Fu_apg|dlqq47q_riWnkESJ*j~;gxx_sivmX3r+5%2JoDqdxn z$ZNG1+d@8y+_5n6p_P53)(uD@y#NnfStVZXncxLi+H7UGxEDlM|5}D{|5~trzT3Y% zzT4%PmF0p!V8Uq3w@4mh{G~TNGhO)Y#t2|YfBlW_K6%$`db^1#?l_z81uV}m3TO;Z z%jy;$ox3Yw!$~3RxsQ{f{N}DA)RZqVg=k-N^Iv!BUoUIga9+C6nB(-8i{c#eBDGG3 z9ATRsCIwI?%f^$?L?;33Z>JHs=n-O^s9wMjL8X?CWRBe|<@D364!%zh~- z|2*qccF2Qbcm`iCE$;BLZL)x@$*v$?H7R-zvT@Y%x%k<*4;JuQ`aVokXdSpF#{W7} z5&sJ$IL8CUSuSBumo6O`}8C^nOG3tZY`?H!Fj)C%`6j#A>q6lL{U!YeRDc&xcFs zhn8e91g;5MMNLFB@(hK9`Cca~LFAZf_825ZNKE@Gmiz#nVv23}`=TfNnmOGxrSnq8 zr#Ju|J$L?0hRBFzosMw`{gS*7@q}wTAoC~H>P>Qnh42o8f9?JsrH61m|6y!2X_VUD zZ`-8a3GiSZgJ0Zr3Wdhptidr(q+gBUlWU~oP)=T~Qw5!!1+FGNmvv6vvy4-IbNsRg z-&J!-X>c^9@5v}($Wg7OI~c*sA9x4@*dhT$yc3%BOemYc#NtXBr7q&f(a$|#=Q=-#fP9zP^u&X@;*@fWKX)&razmc z3xt6uhm)s@OB%NDv{#&YzY?BnD{_phC0^y~-?e9ZO0p2NUKVj~_xGoUEY-$RYqdE2+sOyr3U)rbrnV^uu~?Yr=r%i^4JLAppEAi)R(UQ|xZS#O zcrU0aC>(z*Nx}FuIo0j6rshBg@5_tKEO$wHs~t|AI`rI*Cu(0xhExAxmW#!{;H6}c z_D||qX4PmO*y~ju+Wx7E)Mi7VraM^7aMVkg{twf+_Y0zJr})GE}2`r&99 zJLve@)n`B{s3d)_4_+$So59_^_`7*SA25BpO&%0C06JE5uTO+}U5N)5507N-Tjm$E zb~RJ-3VvYLxJXKVAPLGp?q@I$ssSHVEwTWgY)@<$h-)0$%dHT*sVu1Jlg*ULPH+~-T$_dDXLwv@ zLsZC?gg1CaOb}!o`>fp5Da?wmw!GrO-aEeS=T?C%6iWLp(5Kc z{%rMB{|Ad;BzbwuMi(;S_6`r5ktGh~j?oULYGsJhfBxZQOgHwPcNQ2tQOR@5y|wO6 zKv}!QeiV92qt^!AGA*c%#@aqBWFkfd?_{y3%ok{{qvBUQmJYlXVtJ#Y+X%qXOzQkp z04ZvfB&GQcg%iHPJWQvr@oi$xbu&8yJJ_>${Vb;Fhty}iYuU@$X&+KtUT7v3QgmBl zM=@R`aapu2@42VQ|K{5lOTk<@Z4F(Wv2$m^k~yho!IFIcBz^sDR1UC6FDsxePT@4e z3TSfqz@Y8(n%AI>lWD~F1JyppNi)jhJ7@*(va1**Nx}6?y8dk|fe_-Ky+c?saaJ#( z$T$T{-9ABnbLObD{PW-lV^4*L5xfUGi}r|X3sBx0hb7avcbOrbY4Zsok*dnsMLMKjg^@k^77_T}pr=myR` zYOlh2&OJWlap{}tayZ_8FRmd8rcY0VD6uqnl{g* zevdVQQx+$`X^a4UxEOcs4+$>bL?OQUG?~yTn2aI%OXSHopnSnZjD!G@`0>!lk0_7LYU}RgV_@BqHe8Y8d{A1~Z2LNEBq@EZy zfs;A5L!6CG;o2H5z&5cQ;H4xcnj&zJM)x>Sdv`t4S=^=#7@T)+k5MSJ9R0W4#%7(- z?3#DAX<+hr0r;=&?7a+4a21-?dEklBXx)h{vDlVlGst}&-Sz?!uHe*SP<_QdQ@+65 zifYPS5p;r-J3%Ih$F%+YNKlo!WzLhZs~>*q|`=OKVk<0^1>1 z#j)1MSL?rX$e2(6yf~+LN{Z}bV5I_)rBSD19H(H*GZJY<7M5kC1~0ca(ulg9ti8Uz zaII3=X(F}huy@}kBR%Z8{#h~U($+6|bNMV_@Wn%0Wfs6XQjnrf(#wkSP5z^`M#w03 z0ZiF_JPJ8bU!Z=18WdrjNwDNvL(!1Kr0QRwp<=e&;>i1WT_T4TFPeuq>#IqyTW6nf z6dvFf!AQ8kO}^P(c35J|Xr4|L*362@q&3gU+^vaK_vGXyOL*Dml)n>v7_ZJ*EpeEn zzkR8{!@;@2!FQu+zc*8w_Th>emPz0R6PnDjj*C&|#3PxUI({`Hpo&jW{Q@1h3$HaC^Ow;8E`l@uRPES`7 z&n5J?P_<8*YLLiiglT37Ea`aggl#b!OOQyCXoTU0`7>X_#rPIA5%0|Yjf5CW#sjvf zKXIiyt-W_c_;ba|z{;Avn8&-__xJuJ)(Nk5^W1=J8I3PMw@KD2@r2_Q|Eg~cx`2y9 z!)>8O$K6d2Ie}?>S=9p^VPy0=+3J2VoCZ=o;-|Ii?q80!zN>A)Rx_I1mOuwzP{rgr zm{Jh|gBK1O6_q2;vB@pistmh6X}S{xwQjDCl(Hf&i>8R43y9BMy2}2=>%>R0I#oSA zFE_S-I-7d1KUnUX%z5%A*N%}Vx9Ru=?Vx6Rk60?q($%YP6FirkrNJ*C-K%W-C*JIXwzAJaDn#W`o|mZmFd(>%T9;Y`%^8cmizK5Kefv_w+uQmjT2!J_RUd` zH65Rx{J2B$h5EFJ%dnog+i)x7%L0>7#G+SBnv-c6!37K#sx{M^^c9KAsV&UGjwH?Z z(E|)}RP$TZ*LW|8Wma>3eY-jSW+d1lIp>le8g3v4x_9LkG*TyP&%5BF67YO!tEhcK+ zmq5{!W~Yq1`>_-n4-Me4*t`$6K+StiF^V~;Dx`r%j4ulm1~*|5_rbxh!W>uoA^pS# z?>233+}-@BKnRy)6&?u71yy2WPoikuw8o(QsHEUT$HVod;D#peiH3A_f4MtUkrb-%WjwWv?iKbkDju0Im~q zqj=ymn|&+3#?T1=1EQ(V=bKU^X#^9heMkD{K6ggvNwBm%Pao2xiR^ir!zKu0n&hPS zF@;L}^{78{JPLUNb|jfzUbY`e?XSttu91jM>|4zaCYreZSa^%y`SzM3YlN^XJ0vMH z*^ZS*YG&QxIQVv>kHoM&$U&U=Ht&~3hv&A_Vv&Ug%^qb(kyjEZ9(Jtv^_Rg?aKq2* zYYv=6PhB${ovHOd3M^ysqYIzk{jn+$D0^X+Irs|(jxU`~maYYxD>Y8CrZMHh7x%T! z$AA(i?);wYk=+_JwATQjxW6{B0b2Xy<-z9{z4;*G`y_%vRI&DO@=NFE_&ycR!8t0ly;f1-D{yMMaiF-|9?8+w4mzy! z^iB_3-nO15lOUuMUD&ux>>G5b5>3D4{{5np$IgiL8!b`IzVcss1U1LMw)a~7bbg)i zCrsqMT^fpN4pHx6Xwy>Ix=wivzseg(R8q0-NnrL#RZ#1rShF{32)@g2e{Xp2*^rV| zmx+X+)5ENgrQN21|2jQu;vUMo%?vfpA+eQA8To(s+KZURUH69@(a(NWIh5(E29NDr zj|KoJHR08qS(&IJK~4%v!hsCsAYQdVrA8GFXC+d$YEeiL>n^{of)XW{!^3!OgWALU zi@3QOhGc6WwHR+RCd)_xqP1WQrmQ>|7?dAM? z&6uGeF3IQM;iIv&8_VBTX+1u*bE{aZCZ|1o&H7Dy?sV7CTPCsE<$Ddu&>}B!;w7>& ztQAgZEHOza`JRVUbDyU^x}%eLH#BwJoZ3tAsX+qI!bgtM<9;nQ*?n@_H6Ah2z{3qm z`duu)M%KnAQ6Iq}trDPOE9wt6`o)j_9t%EcI2N(f)MRxDV%~9XWVot@^#{4~ezwZv z9K#1Fhsn|BQh%S5eU9;k#GUr%M{j}&Pe#_Mlf4cEy?4D$1JwABNY)DPeoTnt&t(zF zg4NS~g)HrwQ3Bz4zAWoF*F1NR~chnu8rC=PWV* zAHv=Poa^@eA1+ixGKyqVSs~ebQ^{UsCdtam-bqG;6xn2N%HB$%?3umy`q-O~=X~qF zzrXSPpZ{|lj^i$l`^e|I&g&en^L4(?3#YDv>@1ob4Ld~_k9l77i^0%FaG~pz%-_TK zE)Rz>9cNqOohh>p_ZWOtdS17;Q>)`A<4?E0$n+n8lECbfzwZ;%KX?=&q zZ;vG(gbQ3Mb~=%4N^|>E@T^&ey=L#j!DHd%+Z(YW_5md?Mx~biD}5g za^O!cKu8}lAi3i^563-rT^yW@O@S&aY+Kmw>sEcMVuU9-T~=>zaa@{9XDm$K{JNq|PI`v#v=t{5C0nJSLDU^PXy>m zz5K3z)~t>HCGBL02~A9ahEE)V>W_rQ~>B2T$~3CcE;iNq&j7+j@tLUdH>);ieaoW$c*JR9Ip( zVMuD%R2X7>0V4hlu#O8|FAQ?xv443oz(pv>{nW?Nn$%f;!y9qR?OM zNfj(U%%Qi*snafC9!}skyy*T1MK7tj#Ji%z^AU2I-wlA*WGIck@YUU_o$m>+IjBh2 zk>@XWILB?NG&D{w=)v5ZquV~d*|!F{Sp-3a14>NwRuHG%<8(dD>jG}K*UwAc+hI5} zQFA=^D}nH3?DpV@b*X#{#}nCWngM5VLk9JV-?0hn+%D!Ay$I9rmu*{O~NQ+0A0qKuhft$iEopVUQy~fv+SfxKBm{NqPbrqrMH^c_4i<2f+NW>dIil zH^Ubk0-jT}$9%i*hfoQgJ4}lj?3F`z`6T_$T+qGsj#3_EJnRP+Mwxvun(|sG1szjM zvwI0-d!mO%t6<-T#|{pw)>>uxujDwG&$}*YaM!4L-M3Ldhk)|Cv(Cj`=NyL_0^ML6 z2>doLwF=E{+M}zg3u4gsdv|u_(%CDOOR0o)qowBxr~f(;m>mTn;f@Q`ZsT?RS=J{F z+e>PxZl(er%NcUiD2khM5sByIK6Ms0EwH@2m?DkWeekYCj6QexS`(%3_jIAMDka*v zsJD<7>!VX>E`zZ*A^F_EPqE zCZw)NY9L&)-f6ox#bCE2^5(E*^eE}|BC$Rn{itXMUP^QKEZ*V*! zNb|$ZWZ$6ZF5YSvVqAB()J}Te&OTkiw`Sp4Rq~v3tXE1(TlC5QD6g1Vy=6Ws_;$d0 zf;|*#`sWK3>xto8YuNG-F=#)}_#&IE(q`O7*3S?s?D=1`D>FAx<`dK`na@neWNULD zm1w@<%lKCdAS{|EqglQuj9*S;@L*MMWXq-6^F(BObrZ8SiW@S@9`u==Kd&XFT?JahJx|P8BCvaq{_Mb+!yr;57;#wM&Ecsu;NTu>e4!>22vn1 z*pv6PDqXI!YFG5HRerACD9m#l^GLd643BB@4*OHFvejqa6p+T5AYQm=j#?41B0j{5 zV$1993Bz~CILXVg>fEDbo=lJ6>=r(2oxQu~cbSS=BGJ~@DOhh|CPF{)t*S9$;EeFi z?7HK@-t3M~tqL5+MNyk?4L54KzGEI{zW6PXGzf7Z33h+r#&8Ca_o(n%bjBGr$Sdm! zW#q>w*F^gqdLCB?4i+E3LJt+IZvAGE-Pt##abKHgtn+*LgaCfKJF~wR8L53`yGHy7 zEwf{MmL=Yxer2qiWu`8Ygco5L1n{U}-Xy=rVMSH$MAc{rAMK16?rDyK=>=E+>Cu6+eT85%&!ULJKe_vuL<>T^nX~Fpop;ps&%Knef09lOj#L56+|` zc2~@O5vW?7W7<5O($W-7vi*9s4nahWmA7FDZ`9rAZ7|bioY-U9C1c)5R?}Eh?b>q! zLs1g>OpU%XEpdLZyEuDV^#Zd%ZQ~_;4|?07SNv2OdK7$iYiB6=-TdjoNfq~Ox3m`3 zV9fox5pU+ZFlPJfr4|!dXS1=E>4x*wvw;&|^lRlyF7|E+1;5nKivs$36O=EXpVUM$ ztEuVLeMma`*btK3J|sxWtnV~&PQdPnYh7K@tH_^LI9kB1*TEx{TG#qcY3R@W)3-%{ zT1p_&5&wq9a#$Ud-kfdudLo8t_on)ATUGz!)I{n2nxqT2QPUZBBG=26kKPig6>D+3 zUM5P$JbM{SA(l@ZDRJbS36l(d_)?kGQN|$eTfSmmFb`V{vRkgdh0UkOW$h9UBt?~S zRkW7_ca7U4Ws+s^QngEC$wh^d>-YMgM6PlUsC)*^9(K*=6;2aX^G};aU3<7OC)VCO zZ_N{?${Aguh~;w)a+1UmzRL7omEUOljb+zh&=8|>o={z|S;Ot&IZ`sq;WD<0sh|Lv zptzKyn2NMDTH=39xZth|$x5NqnC@%qF~W*Z7=0=l(&r~?Di!hFPLl&(EIrR-djZ#- z9G61i4}N4;FQ_^DeE1!KBcLSMSQynV+exr4FH6z_nn6GlqbKkU;%K-F4#$w}PQ_(C zB&4nmPyYTJdL+DGL6h)E_AyYF)W$)A;xkM}cgRw^-?;TH!1^S{QrBeqOJY*1qUhzc zKNg~$Z%fzB{pEZQeG6;SQK8Kj?16Py3;@o4b$iw zrXL<*-SMZ@YFVZa441QVoA=>oa2S382l(jAle4{ut#d#<;t>qGvucj%JsKOp{V)j%9-LU zz10|{I%h?II|fQ-R?kvk$SNRf2|cJ_HQkarczanf;lN1p-OSzsOW319DG9DEPD2t@ zOw7sGr<7NjReF|vDQOT-^8RfBUrWsH52nd=;?iQ;VF*Jj7cdnN7~%_#7l;vbf)+?m z=i05SAk|Rv`_~;jNNvUF;Mb;U)ff)CT6bJ#7$kU(3{Ktqg~U4FU%yW)OH<>62n{8| zNGdZ!qK^82eTm_&DT?J$!GrAw46bL?s#+XAjb_Irsh7fnVMY)y+Jt178fa1ns1B zm}~0Lw$fB@N+5E#@wI}@o2FVtPPB0X&HwK711PlvHj70^QqA3+n7FPi)Em~1As z!q9@rl<&435N(SUMO4^;z!A0_{%mW|JYN}x5SSXt`6(Q)#w<7MnmtSrY^q#rcFucR z@^z{P3;!{}7kIu==W^}rt~I2tjx z9#zfs3K~$k@;1Jb<{Jv_oa%PDkfp8GRaJMzVr$T)T!n_2tibGCuygbMk|MeR`eH{& zVjq+bB#!*FVc%EK(#vHl4lPh2Pw$IT4Y>LP@_OOtKZ0he1Y_LYUgy_l+Rn9cS&xeq zKL*_mQ|LLM9lQ=spDzK{-Dkf(=gDn60YTegelgfh>6JBV&vQJfe(gUN;FPzP zp{~|E|FXeLn!K8KCCf@9*;T$(*F)yl%;4U$5UOeomrXlSL;FSx?(}`NNN`_3767Bx;+%BEZNm_+ou_)z6{#nQ ziI(H1+sS|hn!x&w{T2=gEu<^!p~NxH3KEs&9M#}n=DkP;9%AaD!9xwG5v{`2mj8HL zi#qHl?Uo_-z}-fG(w=&juGLA1vaO<9$!1rPl6vEmA|iSzMI%>~uq6H~1c{1so6ix* zdcM`9`_D;O5Lp4vkm@W9eP-R!XA`wYpTDU#DdEDb!N(iN=kbu&w6}sajKRvoK$aWg z#Hz5<2LZTVw|?tt`> zM!l_%tVNcVOav$X?ZIoSNsU$Z`I-Dc^y#h1wu8rbVj(bV7e` zOE+D^hUh1iWXC0cp5qJnX*|^Cft3ezzK_e;nOUM>Q@c{<)1OTCi(GcoYX(|1FPMH; zc9xp#pVh7O2>NteD}zTa>h8^}?8+bR8vlNPAecr?$IiSS!o4}_F|#dk=`l^}+5!`E z7VCPP(Hw>qjB+1t;Pfgxyl)R@;jND4V$xMtIM3%k%e696=!ZU7VlSq8bBGafkaX4o zYG^SwJ8US|cwY=hUcAagBtWp1a#s44{yXyi+FFAa0u;IRH|#s`K8FnJz(g8_o?Pjq zM@+)wzf5AR%GtZX_<@B}td%SJ-ipYQ$=X)~^)mf@4fO-_DS=LpjUxUnjo!t&ov?vY zM_zSCRkbVW3LihAM{K3~p=iorJ1&12wRVY`J9v@Tgl;+Gc}v}~WMg|bF8hr{2MbXe zkwg^*aK0(}vPUwIN4+2UH53U}!4pp*>gRCMX;?u$EA+c(cv!bZ8}=dk7Y990k;xiT zPOJ1`8l1m5s2<|Hq^CSKF&<<;eM1{?lg{M62e^JikFv^I@r!L+7xEsSdX3!F%-4xJ z9C#&N5~J^B+H8%V$G9sG*dz?mB4wu-Bf+m&eojhJ4;}X&c!XVot5w7oLxPO4phsYU zc(6MiGz<83$23v*gk*-13JMi975q0np&|H)H7Qp%mL!cEE#zEqzzDE;5-?#@Rixjv zE#$X)sxEx^*(>C^eskDrAcv4uFJlFxefu({s2tOkL?H%MA^p7)ixCNR<}h^&bXR7x z=9WeND&>q;%GgKi+8?BG8FRbU3sJ`LHa!f3Upy$eGxYsH$S(O{ntgkb_p<@L3{(5= zhYZiPaq+Cj)y#;}hT8FNt#G|yo%wxjvog;Us#F*5bL%LJSYSL$W^3f9YJR9r8k{}% zR<%hvTs2K0kxqJ4EhLL8Oc2joaX}h^mfSE4gD7{^m-jF*&O%S}_*LXBJQXC;Vqp$^ zhLWj9I1SV*U0X0^2!-Kzw5KkB*(f+Xx%0 zVA544`gsyY(cF%qUxbZXRvv`imfR%GzoWJ~FvW5ms-v~)P)^?5fiK*Hg5#CMC>2a0UF0waRoB~qBtPrZd;CR1kaRAu6NlusNH`GI zUR~FjP(D;*O>Sn=A+>l{3hH&ge^6}B5@GMgxxI(R&zZBFUZA+p1v4V*yD(mm0q$a! z`3n*Xz9ArkJKe?K7YR^)(?wH>hOfaMIm8Lw|F+Cti5(I8U!D-5PG-Dd4-^_X;Cm7i z{5dK1X$ohfeW71>T*~|@ab>6^;2^?kl(f>%S*xD!k4o0YOZ6_U>p0>7iQ(j1?52qT2vl{phXNpEZF@C z@Bwxb=-+kRgBVDE`H`36X`7k$|09JwUR$%y(kjPW8K`bvCvOeHOG@Q2K)Jb_5Rp_gWX$M`wBO`t>)J_dcLf? zo=>RtWtb4z{lO8JU*RTB+!(@Rr~P$9o6Do?;RD#S`u*|`59BJxVx{b*`c{3FQ;bJy zno(PH9xRjPb;sA!m* zW<6ol#PiNAdVh*9asE$yX`sL;nO?StI28(QDQ>eKYkfjJBu!kCfJ0%omMFunJ0^&C z6LQ%?!iKPi$t9#S24Cy)G`$0-x%_5!sX{65~d-WF4 z%BW`~*x{{%xp$+&fy;qc?AT^Eo;LnjIZ=rp#-FH8PSQI8V+}hBkMJjr<#!+bN_@Pf z>dBqI>jL{(nj#n?e#mlJ)=8P6;|pA4Guf+OFFJU1W}E(`8vKQO^<_MHlm+&#eOKA* zSI0;eaKc}Q9Rw%NyYcGk9&+-H6i_XOIDcbIl7(Y1KZ0*H{8lJVq8k=nSgYBo91u9?k8n=Um+7KVK)Yr+b1TIDXc?eqKI?J3-G=xO#0N3KSWY1D{_q z85)bRC#^ovVm@#^)O!%d7z#fN5p_xMNDx%zT}X}iK$JkFRIDXGVH;5n+m+*lY+^@m z&kj#Z!kv*rRY(m=yLVzGaRmWgBjfrsdYHO?$hon)2|zU$8m* zIct5hZ=+X=jW=Q3H*BX@vn(JwfH^RMr7uTy^E9(KQP_^=GH(+K3A_eEDCt-7t1|2} z+*)IA;;sxh{wg;azvd?t`{ME4F4&&Z%d@>$E7xs2k6Wgd>+f&vvhE{=P+*GjYq{>Y zY(2_)T=_`=znneqGq{g7*TA}YR)7lg6)j>N z-)3`1Xiv&xId4T?Hr*<*n*p=Qm5!)+w(SGGK#eWEz{`vvX!Mly2oKT;U|`B)h*ib+5vi zz0Ui+?$W^RXQUkZvNA!Ma^COjYU`E;Wv*nB3D4%bO1t}E)*YwYbCtpMLokTMp8gf; z4LcoET=+*p*NRp3QY7Lk#$(`MYRG(^UIGKL`Lo`_0j}lv+IN42zu=J=%_w$#st$>JyV8gQ|P1;u77)GR?U$G~D zVQXjVr>Wt<8^6nxlB>bp z_yy1w83KJI+(X?A8cZMYDLVcbcmky{OEq1=4G(h{GCyoaWnX)J0g!u#etmg{n5`soKl zMFz)O9LmOQBLuSUF_>tbx9+aH%V{bNb1VlIUQ^`gl{3@LnCA7m4UUZi+b^t4=sh+!# zY!^Wrl0WYVW0+b_wZA!7D=<#|+e6Uz25p)NR%q-&iLIQgWJI z+nyi&+2Pry<80D)lo}@Fe9@a)<_Jlw(Q`DnWUS}Jv4rtL>VprN9~!p|wP>)~)aRE! zY%*h&zlt{#VDKfJ5`DjjTmM)v@w3ueK>N`;LJuuZ*OdtH~ z!fl~Zqc#=CY(|OhnyS?6DL3cs1r8ha9TV*0<2~Z$y5q~DW^$6Alqel8>vxh=!MBIm zQ>$9OxU>q`;~l7Y*8tmeb1qXZJ($ZAy?TX^a3W*MWr99P(0)-$bY~&^?uXUX%k|@` z74J!zr-gPc;aVMy@GoQchr=y8#F4#gCWcQd9O@;ighc(2D0qnfZ`XaiVJo`WmmZNy z3*<1JpzW%7w(6IQPXr(~P+h&PRYPJkTvYV()n@36L=B3&p6BljCeuaINt! zTdCl*ZRPF9g`FY5gnW6G*>$lY_XKFeJ+jUmPex;->{bTVOYQsjn^=BlXax=SY*l|+ zMD5K6aF1r*8$yrfRPbv!Oi$lLvV-#I@9}p{DeM>eM+r73;&vnQ2Td|A(4&!ecVbIG zOkZZh9!p0h>Se9(lPR6H9a-9<$r_4){r^8%AGCzhK*BK72N1?om+i&!d~9~ahda?C zBF>X1{Oi6UIf7ysa9rzT00#5Ifw_ssDaKGYwJ4`cW$dilCrk&yg5AIa!2gjA0$EB z-g;t<02!<2FY26?9|xgKyQ4juZ8D-;5AV6<@TvT#DCIzsd-I31kig*-A2F=0O-Ks3 z&7WBrs*Hg}P3bSf*6?Oj95Rj9Jts-^)@mhH17Ukbr`NIT!Z)vDyOqVO4?;K8L5s7v z!bZC0=-kiQt7*^goH8c9bNtq{^vk?FsgRQ=nppm@-bz|X)2XOtFS~Pw3lKcVm8?^k zro6A!tZ1D~8@svtSnAwyuv>5J?A+CxD&$Pr4o|4O|0%$F!`NP>;R zLH70mNSjR37RBiYa|-{r-Y*Mseei{K%j*dNDujyTU4`Syja#2(+o|UR+XhkSzfZ96 zn8CuNDFr8~wd$N)hRRio3m)!-Q_#B%Ti;eoQgB0+v7b+THf;=NM3AebzM*GL*YHC;Wh0A>-VCby+vKx*9_j=MP4#i~Hjq=~@}H5wB|#wqK>nWk;LY@ymD0 ztm;-Sa#)R+o8eEOyA_vAZtHvQOLB->92 zG_B1*3zOy&wc)wb$n=E+4B0}6|5|i4^6*#pzt8`Zdwe7H0?x|r%1}gw<0{IW?^9E& zAnK&j_JTUIwnP!DP6}hLPB!IgE}K}S25r2QjLJ^S!hF+AvssLBg8L_YD&hOTKQL%o zxD(P7yg58b7WO!bT_n*Dq4*oE7eMi}XfdMmSUPP5 zd;Paoa5iFg8LQSu?OfR9IxHWAyIe8jQN}rWW75^pmZ|%D*^uZzu!z+&RxmgIay5dY z@6+aR)WCCnHHYEV;d0JP6d7??AoyxdJ2{7be+mXeZr7fW0IOp}w?kpA;C;{pz)NfV zQTW&9>XDByMJQrP*$;2?^78(V`F~Mla1(@tgm&r!-$%74Dg7-+ans`j(TTG`agDcm ztoCW$_w6qiGhNsQeHB`a1sAcJr%{tdOJi|KhwicpS&aeNGcGq}7yhuu%&$C{ksfg* z=Sds8^^mBf#-9)tk5e5j_GT~SGKc@P#iQo#$1h>Cb9#mSQWj=&6ej~H>9%`#*ZYgnkYHmaS$j@31Qj{1>V zR%(Ra#-n!pa~p+gW!kV6E!E)n@jRuq{&0G!9jp|cl)j<>>Dd*KfQyayL_1r_PgdjA z9Xlmp;GMyG{Q&}xIDanft`eX^F)tBbnglC64y5{>7%S(xq4xdc^a=m)~r)#*vsPqby)>BE^B00RTbDK^7UuYpL*jOT7 zx!OFcrEHYa-49&&j!z5C{Qhgb2A=je;;0{4c_c?SZ|McmyJ{Zn4POpIm^2ZI0D z#ry6T=txktx`|;XU9%r|dMd}-vsO$A4Hu_zdJ^MQQ}ilc@JClZVG=%U;q=WoUePYi zc)8zYx7Q)wc@ogO)RTZ&@dKznP-s=yEfA&3$7oQ;mtfP9SiFfzlaI^7mW;t|>@n@m zyg$`Y_eYaGsnGxXq~az@u8q8(6CgCjnD*Gh2iCtVGzog;OZ#9uIYDhsN;-(;D8D6Y>o&Ekjmv{a zXUcPjgaDfg$!6Pid^bLVN3huW8z8{g|3%^BWn7j#uKAdfqa@4nCA#RkXRnEpX5Lg8 z*ZS{i*C{a(K~D)#oab&JS+KR@XQf1mBHQ#5=Rr{G$oyKhGhc)AFK6e;Td;^vcM^%P z%Mhw!IF1kvfhr<-q?jy=S2(O&_XdYavf*JEx3)vm2X6A}`8l?f&FAf(d2+P^4vpGF zKk^)J=6~^K;{3cCvZb1^`jcKgYlEjXwp6MA7vQ!qvudqB=EcZdcYJsmrP_9TVWDkh z=;^_AUJC_x&x0sE7yiseL-8jum$kbNip{0scwf#Y)s~Wb{u#Jmc5+x%9mj)pKz9>fE6@l9^mG>=mSotAqL zM5wJsE^9i_xJW2NQT@zgII?rgsIeW-wqC;4BlE4k_m#zZoRx=BDdZgQgwbtYP@>nIG++zrG(%@2lTBEQjYduKmkXAyoolQa&pt@j|_Gz z3Xv))(uS>*ZskEpdP7Nuy$gksDEsMziI@&Cqe=?)`uV4^X%C|$TBErGmuSa5kYIcU zL;L}h_%v~kgfy0sB4&qd3O`YfrUF9WFz}bVw+1$uG7C7 zHYH@Hi319(6O0%9tXw0)|2-F z4Ak9;(KAbWwaiy*(qA7}+bpZtM#FlxsNuY}&2JkE{&wAHiyec;|4Qkuuxf=t8#RqY zAzmnI)OA&O_v*yo9OcYeQOHeJ8SbcEI2#8}LI6icoMslMKFm5nRwGa1X0)&;$Jg;L z)z{Y4b?%A@m-f74UV$XlAifmLD%*{J(__}od4Fc?UpH-mzZm$civCB^;nn6(D(zOa zX_6Oq0!5}>36t`oZat68Mw3!s-p((}mOJDvFiz_Zj{7Wn zl)-xG*uG3vh`nLH${;to*YxLxQC2bnq)gtFqx8|5;WSq0j_gb}8L22Pk{oEA1Yz>F zhBO_JRmqTxV9zv2_6?)?&S{oukYb5y4@ZpHip>x*ASA{05~q3Ob0}~O3k*BK8-SEk zYqT_sWD=C*fkZoJLUGTR5H+S{KSqiA7P$1CtZiwOHR=E5$-y{QmWm`>cV zkOr2q7X}$}b%4Kk_N&b|O(C@00?HKIbr{ zyfAsnM$Dm$q@BfaW@Ng0h8@HA095DY;oiNz`KQ771?_Ewf=wQoaNpLvcsLv)0E3E# zh1q{4?zaE$#Qh43#=8>Bkt`AbK|(10nXT!8y3<0F3nzjAsBEKcLIw+lt`Bi=zO#{okH%A?`wx@7ama z2ch%|=GZ;gwfBfOXFr)~rlH>~{K6ixVX17WvL{+}N)?xTTznCpDU;4AZid3S!mg(i zl(`wl@Gw%0RlDTfq4+q`5`i2;ImOsJ+&cFKOz+!B?OV^`1&Mpcx_u36!s|n*RvOo_tp&j? zJV9#FJ1tLJL^0uwwO$Y)4kUoQ+3lD;@_ zbAteKKXwtkUvI9Q1*P9&N`EI>Nt~{jOk?W7ViV zE%JPjB9DQ0TI?I@^kC;ZXq!Um{UUt0w=fSE`F zjs9`u=hm}LLMp+ax=%c7ZBD3VtD|jch&V;rTRl%lTV(EZFlFHbL#>;c{Ke0kjm0xU zj>aVeyO5D@{>-mu5P~ICN*3KvE+U0_<4%=jbBxm7M?IH?o0E~BO0STR$naVYuW9^D zeW&=w6}>;FdsIL#Bov#+LjH6ABWRII0Ap}t@CF^-YcRFh;^6W%A5uLf#qV5)Mu!kj z>|lS`zzk;sEcBkNslJKMu@>q`sK<-+$*;G$?@827)*hE%KV`p5w%da$pNmh1^L1e+ zK?6}&>Di40x~Hr*N9v&Bwsgn5rf**!YuL&Z{;{StQ!P^k9M-Mg*7Wax z^7Rv_gq)t_&GycElzUpeRipyYw5F7q&8Wr8*+~v015#la$T@pbrD1b+$q{q+xBKD8 zJ==x^UaPlccW^z4JoVRh{YvW-&Y>UfecU^YtWBuCc@*3|k2Sc;hmL9!9Xh_>c}&?@ zcMQS|S*3shqu*afRY0P)do;Q5G!&x7S8P&UKOrv|Zy+1>*T2wRI#PL^ukFB{P(A19 z)Aw|84%*?`?4z?^J;(G0ON~npLaIPxQn_u|nNc&os)^b;f3Y+mbO{V+e(gJ^-P*Ua z0``-9H_ z+wsILOJ6+f{&)X`pz9P?HsrQZ*z0_E6OsLTW9#KA7IUOi7=BNl9T7B%>+rW1BauVK z;fvU}Hsqb0EalgB0jzQ>?UiG1gvH~se9cLEcfU*IGR~W`>fTdTuYLqRK5}GD2gM}< z+^MZF5cIvy@f6EOP#Cmp-tFD=0;Q5{HgtKJ`=P_+;cq4$pF<%OFx1}?pB;6cqtvRf z%XD)EzTYd%^Vs#c2xf7vvRNMTK5-1$Ge`+An1ZzLh|K_t!NP~1V;bkR&R;R?E;kx9qgyz;ER_Oe$zd$KBoMk|HsKRc+Cg zFJt&DRjXa>OO4`Vp=YYb8@r~~hKBTB2|Dl9>NZRHhYx zb)|N168GzbZ~4O#cbv$XirAC2AD2x*k?MJAVC$rZCsr-|!*;F@GRvp9OuG;;z11?L zq|zz1vZ;y#*Q(U=d2|I;e9jLSDmiskj5 zfbZ1DG~d>%>eu8}MlWXP6M3TCZ=&@gpwF+Kpl@QsAjJ;NpJQ0(Yi5h4cqueI)Gp+> zYvAsa-(HQj?bUTv6d_4Ebjg$+of%RUeSQLP%UjXiQ6j6nA2G{&ANefVt?o_dv2vp4 zlS78DI&U3swu`Dk4Hq(_^NoRMbM>}oS2VY|Km0FU2xA5I4~}M)g+1Px~mYu~sPvD8FPK9ewTI|V04!>|iFu1lax60&}Bh|;+DVpJk z-@Xo1F!>PgDdf(Dlu4JIOfF)GU`uJ1)w(S_HT#oMGr=}>OEFo?{+&p*gdHfQ-%?Bw z$2{oF{K&9{(#eBO87b$3V!Haq+*dbqbs`$@F9dGaE)4qo$=m;Ex!1pt_+)}kt_AV><#&;)7)(LKJ)}8M7t@V?mr$5uP)v`uU;NEBEl%?UyI;Jl-Rq^r1v=I$6q|KE zCU?YJ$m%gCY9t;QO+3l;p2~i5Lof*U>Z~+wVe45+{50zWP&1xyX0PRPcrIytD&!k7p~rZq|^`GRgiKw0YVj( zW8p*u=jQZpdXfOXUr+-&R`#s1DOK(L5FD|cJg2hxS{^Ab5XNx#8If~XN#4_z ze%4@QJb?ymkI8nnB@EOZ9lNs+XomYbq@;VGec1p@RD&RrU-v5`L2>TFY<3p!T(>kI z3Gk3I*%!ph6tO01cHGIr$7~2t?H%J2=J$|nDOCgRfsXV!)7i2sLLw z&2rCo>yP2)Cv?cj2GsFmD!~>@djkwb)@UI|-HL5s_k=VkL_U06IA^Vm`pxh7$L=6Y zPPSrUE;K~-c}rqeule_M z_)i=~Iz3PP#HYbVLE72!I4%6|L|akV>3jF_ypd$tVf4_I^_HXGo>7elM67-?$n{g&O{K(TCumO<(4C~x@H6J+?MJZJG% z>o=gr%(TpFXWxI;9zUWZ-#<&)>f}f?8hTF=3gEfJ^`tKzI*!1@|CuK%1X;s&)M<_e zL1n#BuLfY4yO*Xhk@nf2=Wyu%vko?*TB!CTv=d$a`eVhvJKDQI1r*XT^~>}VCC-`3 z=oS*7%pWGvnZ2>rHv}-R;VfL5qz>+`bD!;T3TANY<^mRyU*)@BrycI&mzr^;K+2e< zgaoA;NYatv%@7js!O1?CgT%kFAkpoy$-d>vVv%K#>T*4=pxq?Q3C0hvfB*xETEywq zXs)VvTb%GziKtcET!Lr3OmLi7uX*pfhCHh)jMNmHjlTJG^0nSr*YT>%X_9>{snK*49F(D zCB@)hZ#%9se_vXEA~T&_KrQ6-Sp&2d0G(!!t}n4pu;Q|tS6b}&zMsTKot{Hoi9b;j zdGQ%Sm4K&ZHGhd0M$unk%Dg&_TJ`^js8ww*`mT;Ho^LpqFPEQ@h5I$kkwJT!crxgw z6TKf64$(c%#ps!jRNw_xwZ?ybWK^arWe{(Yr=z0-p-R_+X5;ui6KBJ_kkG_T%hMm- zhk4DExp>z$^N;i98w{0Or?EY{{D-LL(S6{BeURIRE)V3IOKK;-Lg_Q514(>?>~*e}*{nG^ z508F;xS~yI8x`&WtJv=4@UJth&xwyR{?gS(eLUQOzeI>OjuM?%cJ@(L^?q#5IbsXQ z^;j2Fh{1ua;~sRmM{ETLuB`(sFTW%3)swi#LYg^<xGqdM|+mO5OOWODx2^;vrTc^1H<2RKaRU=;)7uj&cA?ZR~o}hjN!?J33Rn=$O4)k|DFNi zkA3Hj76~89GG|3!fr4acG!$7U?pV#hhL5bk%Q|@{6g~L`tB`yU2Rgzg=_DvW{Mn{J&)3UAM0y<5 zdKcXxa!6J>tf-J)p&2Gs^}_qBRy1{~dZOVk(^?4}v33lfRo!t>re5N8_2C2wvs_06 zhCJg{#$dg$$a>(uFlK4hvR-|GA+3znn-h(rQ@wfbEyMg&N7C2PqJY`Egthk1&~yGV z?dH&j{U+X!S0U|yMw(uW7yz)HtO6q+1+QK-;r^uz+4y;q^MNs zVk9VSeicEOsj7dvn{jLNwX_a4OohWdVA3a6?EnMCIDznJ9*ZmyT+xs5JQjmez;Eb~ z*Vbd#t-6OvnNBs2d0RaVuUAbkOAJgI0N5Jg=a>Emr5Xs3R=F4c+Q&EAZw7$vukh=K z20T3(jJsoMt)=t_`k2gR{s{9iQRZ@SByS-h9>r%a59FHFdHMt7PFw74|Fe&#mh&hS zR_r8`Y@Fw>QpP{+6(C3g7IML_=?<{(BNvSDo~Qhg3s$p?gQ4Z?oCbGinpYnewI6F; zMQsM7XC74DKipj*{4p_w;(0NEvcGUJ)D-GDBX>1bl|j+p!3}GoME8&pblK_{jA8g$ zj^z_;xGtC57N1#vO4Rqc{0UgpQ@O*Nqy15VLiG83wEmgMJN9X*^m}-h0PYXMfjrPt ze!z=SWyI?qZoVO*x9FYt#~C++!?7Es-RffhgY3vTCrY&0rXv7^*L5<2FT!48q^R1R zL32BC{%WSI`!J>`y^T6}2n(ik)%AwonH|999hfm!NOA_fko~b)D78`%X2@<^7!xt2 zkkfla$AZbbO_yx^fRE zy2gu4Za*D;SI>!CEXATRaQ|^DkNTZ3$0K87{lkp9CtIYNvg;hPdAmK|Kl(hci#UXct0#K14l(n#){V$ddv*1n3$N89=^ zn?)p)-7h48A(LwZGd!aA2hP@Z&m~-M60lpA5DpvJY(+#^)G~8yG&}07NYoYa@APf= z?c5q3TW{yFPj?@+(N{Qn=j}l$e)IvlM_9yOdn71wN1Cp>rAPJSijl@7W^WB|*4YqJ z-~Vrkz^Md9CPFg*EOQ>~1<+28F^bIl*zDtTuy=tEpa68bQmq6>yz6|{pY?59PGrJP z;;v}|_B0JEWiv~PScKW**G$h5pboixGVm}sfKzdNP4;>!KCtbGc(@9XG0a-PR9Uu#0<3RzF<)3z3#Lz-dZ zi>LY^Oz53&$wdcSG>7Ik*-FcX&^0lm8 z-nryHQ0@2q0w&gLA2B!F-5@~%(+T_APuJ3$!K~`P5>3GJ*5ty6EF7m38>URiTo~%V zrDR|?o^}7W8G4;rm6NbwypT!DmLdX9KjUcfY=XuYSVG0DlH%`l-h;}6O)y=Kl)dX4 z`uLLMsap!udvNDU-I=p7kPVDc5kqzHzuBCp?K6A~iZP&+#oIa`it`Cy3<*1*E^SD- z`TPd*RJFb~DKwdt{Acdw|3vty+WwO!{oHi8G8I+jevi_Dolkuvysq zfl@+WFr~?Gzp@KwQyc?z?ogF{f6ZOp8WTlg4_j1Qd;3;jNgP|N%l5oNExrTFE*dCZ zgEfb*C}?TgwQA4iQSKun1|#kv zweL!K*UoF3(w+jG&r1d_Oo*rg0S;RFyI5f5(Lom&_i!7V2bXatJpF0FH;gy8bDe&)7%pgFnfwD~_-&A~)rc}=TMLgbP{W2oA1qAdSTzC<9CFNZus%YnhxwJ>*27hmtb|DZQZJ()__En%ZoP-m>d zzFE^TsS0LUdh+!}4P2Wbi}hpEDI3r1|9LVSW7P#hiMmJ6FIdspEcbI14kbhH%JN{P zbWG^e(m5!2(u7@i=F{5Ep%-ofi=jo9)@w6i-OgfRydEtSe?VAu8`*MN`IAeYt6TZU zu)KWyC#n7w!Kj$#XNP{&9-mX?a`QiKrg?ZCZ$}LK3<+61ziZq^fuM%4e&sWuqJLDs zE>iQqyaAl_-{zFI69HFQ=7F1ZxAGFtDWvQ)>V&RJ1r)9Ov$%KFKfoe%4t@f#=+x$f zI=T;oz=^O-+{L0Ufdieh|8dXiH;jxrThxsaqtK&5PHUg{oWo#uAZB}w`{8w-FdIj2 z=)87=jKg%^ujCL&^IRvfrXC$M-=)G8OVY z_$NfW_l+w>kqT{Vy^-40qR#yKQRT0M`laeP0sD=C6oAm#qUo)Ia?ruqDybJih+#-u z)3-qEkTL>x*6B2q`bl48Rr~aKo63cK z3td*j0?}8)%6m?$bAv9rrRt)3mZ6Lo*4KzrBW%giX4+9kyf_h5!j~t%FS4)xx!TjgpId zh$ihXlr$7qLjm5@^9!gA`gKtf?s5{2#0yYwUxA?;+oc~>gb-u6MYvIs;)z@k&yNeM zaj2N;PtMql2FD62})0*DhbyElN!g z(7p9h?1{uH5X_$aaU2wCfz`|#F*myhl*XbVha8EiFan;%FubC}l~uctzQu9&dryy6 zd`n`XAvgQ+q3qrq^y*X-V(nf4MP0gIXX-a3e{*+5Q(#Od7U+P>Nh*cTPSP`xcC+73 zr>F+sqrt2kd+hS_^*s=8yH@C1VFasl4O+Q3n!8c8@2DG}-I)KtPWM8#*b{#2c5B~%WA@lGuOW346G zI=-1CquT=3T(GWptXu`V?+@xq`t-e=OL7%5^a4|^Z^`hziMei<4b6<1mubsF)hc3a zWTp@a|G5#gXMs)!#A*%ZB2MpPS4r-^wzQY#>|$%`sSde?(Ms!$b*H+5c7Bm~MU`J> z)}WIaII85Lkul`+@yzRqhRl`h#_0AIcUuKsFm*A5jq^_3L$PsP1Y=kLq(KxQf^b0P zcv+yXXjl79iILAIS;XT&}r;; z>0O6DD;sp~ONEF>PQpTDC0JVZEnMW|sh*)u+xuxAw0A-1_|1*}UFtLYH37{$Ei%`r z=n$lp!l*OkhC}$n+<^0aW9WOPw9G>@<9OGRZ=p%-i?7;CGo7xe{XPt{+`U=R-(I{4 z&NO0hM2`ep!~}=5BEHvV3}79nkJbCisYfyQhbm#tDE{DL_sHsm4KVj`wJejD&#TP` zS{5grSs(Rmpby(E?`bboyh3^GWa5YI_mBiR!WOd!l`awzIo-Kd_xd`&t;yVw@CCs0 z_8K)Wbh<6zi7tha0P0Dw4OJ5nFYxY9o3n)8E4?=QdT|u7e7cq*4`755ig%3uz(Z^X z2?#ZBWY--VNvw`wC_ii5Xx11PZ&C5!J<>b8Cp1p`MW({p|I1W+F+@}DjO9ohhGl~u@r2D6ZeUVFcu_d z>LEJ(7QX32iqZ8!{%h|7-8!%VkTgSvdGEzat6LH|X1V90qdvVFNHO1->@6wX`vOS7 z?i$cTE5auP0@`DQVqScHsMLQ?uQRzE=V1;Vl1WZr`lY2N}gKdiUYW% zbRX^`*|A~gR=ww*9Tc&=M0cV!g7u1k)k0+$(!@>wx#;s1y#dsRyR*}W8;3e43RFgY z4+7hwM04rd>PpGC>IOinj&|KHHoO|dc?cX*^*x)e_!V%qqQ6F{7mr`1;ak4yNIwXV z-AS2ha87x>#s#lke- zv{O*fX*pxa@7jq^qMdOH$DX$|4YYQ_GQFF7h40^|dJ64*dp4WAY?Jo8t)-=Eedk*; z3=|`}&wcV6s3FG-xr&rCMK+>u!{oue2BC)q`bffZP^9SbqUhJ}GmH4;~fre;*0bc76lb z6K@*My7V9$R#23Fw0QsoEo~NOi=v29t}$qWml-{0;5V&>)dtAV(VV>gSas+pM`u24 zrq(RWO#v#%qA(7(?bs$#wP3Mzip*g#=WDPkct*YA=Br~;`=wdSuK~~l!)sMlB5mN^ zKvuk=KAw5mmtf59rq_X!aj@8Rc~V46@0T6pL{?3Ryqe6^no#wqfC85QlmkqmGhVS? zgebemzuj^4HhK*%AeyRFc!QU)#80yzYP@@X%aOTc+uEpqY_m_Q7iu!9AE2|kxjZv) zkncY8-0xoDV$NH|kF9@B%g|alrn}GSFEL49;vYPv6qUO-XpfK}UAlQ8G_d2>hQi{m z0D(nrN&K3uOmJr8=MpKkt`o|p-ohY(h2G4}{!7`&)m_<}5<+__bnq&SqT*;{!1Olp zvH+#_rLvr&BL}};+Udc4?!rEJO;YUC?^v?u&?a?xb*9Du%DK_wcfP-R>c=E@c#xzY zn_KXTTepWllX&yFBI8(^EzmWvVS$ki#yVjbj!(3r+;xiZ4crZ3wm?J zQvjRgu+xAjyPNPR6e1YJx_7m>AwKaMNcZ{YE2!Ed9Jfh@xe01T zy^Oi-XEXC)Xh5*Mf)+7BI(D33{wr{TDmjon8wIKETkzf-T=|TBsC|7f$Vy-h${HY~Cdh=+(4W?q)(uJe=aIT|aYMHJ`l0v2Wly8SNM6MYc( z*k1=S1WmnD@8v$7qnMa{#%H1~wEw(Lh4o0E1U(WMS^a+or3B34i}yfR$o)kwgIJkv{-u$KL~- zjW_zzmq{|gF}wwOP2TpL*T2IPn1Li*LfyM!=Y^*wJ0`mF$YHHj&Nkj2^bMbuxXb&S zR%gfA4-Ik%*(wGm{kXZzBUD^m9EW{xY&{Y+qZ+df^i(MujJ_k8U5Yg!T^C!wDP=}; z+H_K)3`wFEyb_AIZobyWE8fy%*cg>Y6-&va=7}Nre2H1qZRHc8Le2-mr{@6QnYrke zev_mpcy=D4gWkPLa_pPrFbd@*4p=532V{Qsbc~ z@$gD9Ibk3chD&>UdzEsHue5y}5+ihl)yoPcELHnN`9NntWh3_yJxGG-Z@{?saL}z| zq@s4^FP`*|c+WF2cx}R*4s(2qt&0=gVM=K}3h%Um!LFWlb?>8Gp2?D4nVgzVM&VTd z(HkjOAqbqERgeGH&STNWz9l|Eh#`2r^(cM-2%OQw920zU5Bc>OOwT+s zGewSh@1wl-b)aLPNI6&8ol+UdVI_#9NQmhOfLv`oOC+x$#g&L&{>0^ghx@|>q_|GV7*3{x z=noSe-IX2O-${7p`?(B@U$QA5$Gh{bJhsICnW)B8@d`~^M6ZUiBu{>Qy+Gd-dqwvbzLj-X5fD})z$->Oh`hh%Q83QDET{hN3A4^=KL+5eD#*@@cpWI zMj=4r(k^P!K7u_*k8%>{;Ct<5j>|a`$6U_a8-Iq=qI(xFeWNh7+NFa!?rKXd@uTlW zl^g8tzWQj>z4EJu)#_JhgUxbd_I+XDwU9hwu#y)T+_zB&bv4b{mcDso-@%2YUCvuNA+z>DE^Tn%^l(B1P9d zKh=H7Ii!($TU*2LytmobJSEJK!0RIkb)5dn1H*xb7y(Ct;(>i#@;)N(Mv}k@@#Dw( zh?9>8P5smx`GIUe$oUm1feeMdulYtrwDd?7zt-E3_e>HKg)6deXE&^FvN}xOnqg9Q zPK_>h3STI;vWjcJ!YXhwYjvq+1UGVSYPWOenq z;2N3JT2oN9qgKDqYnL}Ne4UL`C3rXvLjM=kBIA6?t}P!=w=tAK~9hswU&;TK2#V_*KRSAK${ zaXehCnVI4_5pBuhy+}TQV7EK7{3QY;`T_fW6_u5hpDQz3eLuqk4L_EN3*>Nyv&2eq z4f86AiVI77+EPCsbrntiwNLjap@>;*bn=_UoPBSXrdq}&syk2XpGoe zXyMA*My-#Ui^VvJ@2+kqY1UIy>xE4utJ7^}8`^c>DrnHS*_Nrv_e?6p;2h8k2N1wf#Ogek zkAmq2Cn6#rO6ExzifFjP8um^FWcmSzd0CWDpN%12Xvn@7mxHEY8P4H|FkTmQV6!gI z-S4kxM3x~zixJ;{{++}^9}ASh^~{qc-{=#}RNvC}&ugPww5IV5&u*yqyA{_Ltc^KC zAK(76xAohJl{U-oU!Arl-);iYota!N5}2;DUyk_?_jpnq(}#JzDgLb)D;cQ%3rPg- zFg`Lb^xT`q7yDaX8(5Yp+bc5D^kDP;*or)fp)|=YEH# zbijMh`Jbs~APSEyAtHW<;JK5n$6pL1?BX;2TMRsy2p^fp=3;a!4^MpnaX4q&Rc;21 zoH+N5q4zx26dM@AR`XAO3)#$c~e2Jso!*iceo)>_FmYq zy%2{6UIRwt+s?rEXh?x+lg1xrb+of zI#JHYE9|WkEgMG#K?J4kc7sfa17mWsdBk&$l_{yb#p0RaClyRaXS4TxXm$G5xH5Ng zslK~0M=~tN<*VD8!E(k*NFE1YAgmi<)mkng8>J-{tRtEz{3U!ZTV(#-54knoettRu ze(*jR{nqbHyilnZk@ig(apB42pHgB72qK)$`a7?L zXlK2ukYq5sCN!3DZmcnjBh%z#I!(Eq*q~C|{pT|3FZ!>XD44KnJ2~GaGXMK>O2J%s zW(d8o`Gg41eEF~8kc}0&@^%yt;ve`cj(8bK*24y+h^c}Y0<-)nK}|73O#XDex01GY zcVn`3(FLN0E1G$AUppPhHwaumnI5vE^356D^J}?tV#DL^_hs+QbEKyNxnfPjeUm0sUmYn z!vpoM2Gx0)5sjJRa^eUeDUVME*S_e>4^b9~g?xm(4bd`r; z`&9JksNq<RV`ZvAbR2%-h1ka=RPtdpxK_e9S0z1zGB&zFS$kt{OAmZaR-F=(vp-d2OS#P5} z?Z0xQgUjEk{L~B-oom8aiL2z7&*0N!)vPTZTPs}Bc9d;5%|2waTGln|oxh;aKW3G6 z)_K480t{73t49^YXuN% z?J0GVxSCF^>fW>1@K?wM3d2?Q$3sT5!*z(z$FD}f0XqnCPO`-#foLw`yTZpk;ES3g z*WO)4loP5nQVcwVd3@#DAUwAAd+`<@@oj0H*3?^hM3g^Ax(f?Mog&CcUY)z=XY0=- z7A`B$ixGqgk`Wl0AU2Cev_}ZvY({$Ej^%?fH{QR0`_0g%Rmm9no!w~y1e<=x_e;gD zUgwSo#zpiJBDpc303rb}P6HW(sH1@I!81|ZWaWzRdsn^S@=s)jo!5X7Gh_sVlJKc4 z+bxo|SQ(Lx#0SK4v^CqYm8Xqt_`6Js`cFvC4xK)?_Yg_ZE+$*`1Lr7M46SMVr(i*8(O@cF0w^zo@@8MhP2*JkM{ zysYCZ_LHQiuTj#1(>X~gFqY9U{F#WUQMzB-@X&e7+b21xjKJ`|poDv7pFs4;kq)_$ z@;ATxAkivZeDH7&I%W99%s@~^K3+-X+2Nj-)0UI=mC1($KN_Q>@$*5l7;6txeN($0ubyg6mVP|U@&s!vKJr}Ibo@PPfR$M883XKU2|PWL|aom-a@&7(Bt(yZ0T`(c4(6ahVM* zTpKf6R$ABdJ6~ST4S5)w2!Z$MgqH_zCZEniF4a}FW3+C@R=th>ssHIG#cQyVxvn)$ z&n6!zLQRD^)Y*lbVnWz9l2GU6k=Y_bvf(>7p_bq=Y>dm`BAEmH_5coSNaA7mBdLoy z5szdpKKY8qYtJt1Qbr*Teqp6az=#U3g<+MbLg7{QOi>pY<90#kjx4h0?@IT!Imvc^}Z5~WBiqV?~vy;OopGYHgwde8IMGRgxU*9|HGR5E6eHORxLrPX^oMPDSI3&XC$PX!)CshA6L`X%8UVEnv}YPO7b}D(X{n_AGlz#I%GP zqg4mg0%Y;MRG4R|>|LFxYZ*OI71o|rwc+}0kz}(73M#4xGcIR=OZ&&uCaU0`^(hDE3$HY8R~g;MsN6;H0P823FPxp!WO7uO2^COo*c2s=R3kWd_dD}CleWiU2mSI5cT+wJQ?`>}5kZZ%x@X*wg&i^ISKB$D2Lx_3 z&y+CQ|jv32?F;E>#5$oaIGLNKyUvF7#Ti~<3sEo z6x+IG64dj9S-PG0ZGP3eWwqP3uZkx5q=Vxt578!&2G#;y2=PDqrvdLG64p>GQf`xV zfk#LY@yQ3u(8yKdq=fge%89?e7n}I2TM{p4I4j=hc!Uz{=G%@p-4(!0Ms4p1WLbVK z$YQjz?7j*sk0nDxl-UqSBdQ$!la+hN}(h{kGVE@|EIyr|h7vUs4 zj!%+o%NvXwmxmE#7lNnl^8A4tQ5T=h{Rcm|<9Qd+wV5UrMR@~3>Cp2=G$QLkg|!7M z2*G^gmo{r6n-U=b2Rih<7Y3L3kQY2SvKOiRJ<=%`$Lb`LtWNp6 zykm`Uyk7HhsvK5l_Ix%9Bcy^mu9wrXzr&&2!uQIp^>F4*mDK2l zg6>@!dkkQTr$q4*FMF&Ie9Bq;tZ8VAFg+?@!HAiAU)Yp@VOd|2jQGw zoWa2~=5^>V$45n7RJ`Bs%RkItxFkbI^|>A%qiRWCbiWU;azYFslurbUfBsX&7A8w{eEv_ zBb?jX(BVDAqgsT~+mSH4t~zeMuKRj|tCJ=8F~ne;Tm;s_XNTa6n)BC>V{oD50weF9 zd%7t2r@)8y2SoVUao*hzkd<*Or_ZAz{yoST{ou z9`U>9*f(w~(kjjJnoidLi-!Wn!hye$u||%O?IgR>1I`W)^GpIx`Suu5f&l`gH7ZEc zvCQ2VH2)5(#hd-F&o%z3S-(i1S$_JaM(QJ4&X9m-2O@#I}G>? z)yA})bV^&Bdfr%8JAJ3~JqFVy} zCr^Nq>>6c<5BRxSmsUbh2y|bM!&{_{a8#$axfZrR;Y8UQMADXvw$j80G2*0+GDK!X!cgpr-xQoLUkN_FW z?eK%&XVzXpH@*&_i0}!iqhj!Aiiwq)8Wkm!H5>c=#tL^kwK3CVV>aD_=Em-*M_;V#0| z+dC&wi*Buim2tiv1l6Q2B;uITXI$|guk3sBWEZ>rL9IZAVeW|Qp}%whQi8ol{J?`E<7e7aZBNdL>fKqYcboYTwW8tP2s%X5YM~*PL_mn|`Jr7LnF?e|j{% z{d^z3VO-gUuHV7nP%VnHrWflQI)aru@x$MQ(5*Vf3|2d zLQi3;#Wo!C`u&8g11gZ~*>cGG1g~LdRQMW9ea9TK>qxtD9pu;EN^|BOpAU~&Z&wt} z2HLNZni^dQtW3I$cKSWfV2C@R3@~x;J z>xejS#UirMVI>^4|NHL>5GQ;OQoU2KMPC1*hyJiqrQM#C_4)KIiU6{`4KJFBE~) zZikkgevW(Z*@$-g%RD(T>JDwrUkLv?;xgpXrWWw1Z=>?#H{DS};sIA|O}K~wFlPmM zz30_F`EOm0QVU=0hi~fnJ>?>XUylqow1wRM?z35tayH-v-8yi=M{;!cpJZ9E`>wXp z;}v1D&BW&gA|a}MDa@K;F#GhBGk^BFQ7mk8%k+^V#8 zVAdD$6Do`&RQW#7-gtluCn!m3>4sz=}6ro$Gaalhn z6Dn|KtB1yeubs02<$o;zBgP}2mPZr)Ob89l!D+UdWe(TAaa}j7CL|NC?IHjK2zTKL zm{;K?KFUfqVchXO3$KeURyI_6Z}t2sX)h9Nto|la&-(UBXR>mmi8G|`wN&OzXYvQ5 zQTpKu<;Ya8&FTZubAN^ci{}CA`OtmbgVLzohPS*?j62@YKcLxyyl5 zv^kKXQh{7&lKo(@x`GAAh|d*f@}od0j92^)PcXLne7?+=HX2jm*wEixdfuv&~` z?>Y>|UV!h$iWn3E$6y}g{3IzR#yzXuRH$#^x?|eqhzA#m7|kE%!w8)>fh#47q)&{G z2P)hrxB|@Qa7;MycIgx?#T-<;cGZTUZjTMY(*`UwB8bq5PVSV57WM4=$SXaj)Fj>S zEGnYv*bi05M|e?|@?tN_eSriYjwIk%!>lPSs9vVPR@Z=XzeCa!`3vD?j}#X;_Uyw( z8j}dGFZM4_?Acwk)tKV&*5za@Uem3wNVh@>^UbH14wHZ3z%|GU$!uif-8;l*e9)Ki z{Ml?WEFaw7B`Q=zjWRS3)0>d-Wo;OSI%{iiR{Qf--xH~uxvA2+T-8blcYSnx;nR77 zUguJxV<2zBdE}$VO~2N znu_t#91lj_IsPQC&vDB~PJD2M7%AU| zS#6ky=9vs6iswuqh0TDhL8EsaCIjN+qw_!Sh$PgaoX#4!&A&$2iw=r7SQ{zcIt&@*H)8V|YX!HZJI;Sc6I%UPZ!|IF|pW!@iH6=Q7jHlDhO7Q4JA+PF!OK zIWsVl>+M-4@ozpm>QT8o&;W0LNT-6yMOXwNjW)Z1r&9G0)39a*OLr3?E!h7HW*aQ0 zE9ibDXCH>Y>um8RbY-!f;5~%juxu&+!#u$32o)nFX+Rn2>=@lCS;RA;`ar?96SAhU z$h4VmKuNboU#G6JEAfxNV6Xj8`~@?4cvfM_exczMjzPuQl1kcft7Cy&G*b&B4IW*v zbgUSXbq>czSbHccA=}#j&uIhRhVF#ZLT7A)w(XBQ)}1yjgTEBMfaqiZH$mncX@+(5 zQnV9i)!*F3>i?g20gJAI|N7STUYHFH{NI{et;0x!D4%Cu5vpLc+E?}ZTm_{Dtj+{X zTNWBoDNd{1!BEOOB)YBeS)3zW4>VYhesu9B7}+Ds<#7C7=jHi*)U1yo8j<`Uet{2g zy1YtyaKa7Fw=!8Be7l;WF2So~2{_#=%3oYU{RN#cbF3=MprFJpqbqiAyCXGcv)z$uRaZE;SsFc#9kUy$ zmvABFj`+Zpa;?~@z(cO1u-BbIAF2X0FHmddJi~CFS`R$OS>*lB^3m_~79TpMJ*ec! zu9cwGe4%;qJ0$vFNbuL)I-yup8KMWEaf;u(7qA32G6H4ZIVA<JiP4(`o*;J zr6ei2hQMEYGz#e+6oH;d=~_6y6qH$uI-A}=6n{N?9Hip>IGj51~fu;+uT{l^01p@cLH5;u)3nG>!LYHp(1 z7b2q)0uv1Bf9skQ@*Y&x=)T2Xm}0`ZSuO+o_2g{c`?wm@p)aiFOj^7CF<53y1z+R+ z$vL5rX5>{v3mHl9AI!)3)!#FFCbOm8Hkt>`!uD*CGP>|1uLH#e%%#Nu0^$b~Nij!! zw~}E?k)I_)qDuo*B7ud>Gib02pNLWC&2;?Ss!5$ikfZ5M;cCFt`s0dJE+Y^Bj4xdI zI;z#;$4Ygv%x0Lyj5Q)ovG5y-R10#4=8l^z&y3B?vmZxhbO;AC)GCm&PG6mA`YcPB zuOQxesUI3W39GNFXc3j;9c0OOi2cMkj*VDrD=l(yIKoI{5U{m?cX6U9_9xKC^5|$= zDSJib-X|tDaJjhjZHT4%yrzIq#)hOKS@l>EEH4`A^&8#8A+hhfagG!M&;lGYcnf80 zRETNbNp1g&u)_(ECDnxe@$t$YO^;P-ZL4ZsNS8C_W6W>Nj%4rqo%}hn!=dY1`B`Fa zI!S1Hy$;zSO@;kA+IkrTxqgUhu}(Q|-!v271MP`0PGfM%F`9Ory!H}-mn9s@$@j)v zM29>)P&@IVl5-ZO+FEx_Ww*-(^IU2FI8h+vH2>q?Y&nBjEg(?(6ZU{x57&y5HQ5@u ztpM8ahRJ#H^Uv=*J$5P##;F%#%w^YBrHO#-kXilXtrS@>X#dF098(j7cMwtSB;G{A zEYA+U<iFK>(F+qn3 zlMnB82S)|fl{Hgl1W2v#2-xZ@ahcdHtbQ^a`xG!PFm`9>Ainb zBa&n&FEA8C2FN$&eNGUI>$2nN1=k}7+DQdApS1BT#Fw|bLH zw+gBXnu!;d9(RCB-i-BVLo8XR^U9cm4-;5_*^$YR?p60LhhNEk!i;jeaEuhMAdks3 z>lT&z5fGz_38oxhIl2Z1{qper9{kfk7u?o=5HFlQecIcNc|I39jk+fvus&Y7vmu3t zr!F0-TD;cW>}?PKvKyLF-7!m$bwj!)%)0zz}&TXnLwF>P~|H$3>AZS}2n? zHNNJjlO@zdd60$gO6s3H=w#Jq9u#@!Kt0b^Kk9FALzKq^O?2rEGS^;*FSB1LIZwcX z&O*nh`P}Fk?z<ImfhbaiuLqR1h-#Zw1bT^Er9&I^*|n69&NgzWjufQ6{1NF;Tuw1=L7F* z3GQ2%vaEnk3t`^&bREt^k!@ATsak%c?_J! z6Hp%$sKgRheufE{tKmhukR&=pyx|&hdW0e*Ks}TJ+ZZC$h_@c9iteieThz2DK-9ek z&W>&X?&jimapEKRojG+LgBtw_;3TNTRwu>#B-uBc00Hpd2l03Tu2y*e_tZcKA~M>s zKPACfUg_zus>wWBg(D=Y&#~yYe0?|<>Uzqnk$G9TK;>q^rXPomx6Cu^T2BT25a2&D zfd3e;-TjRZgeZ!I2VIOfbFn^G3gUMt7#Vb)Z{&by$z|})t|QcZ1D^Q1BjWBn>OFFe zVtbGY66(AVfO`C;-_OS@$a&HXwX}TPh!F=B48tS94AN4WK|y~GL}=#k+*s;dNE*?;Ugt|t*m>@fWR|~*iq*RzCX&GKJ{^5R zm*a#a*4b4hhX>SGDM!|gWpip0g~oIGhe7MWhGrQ61o;%Ld`4(_?%#j#PeUU9ua{qc znYqDWrB{8Yh;PH3Y8nwg-FWg+@8FhY&}iv$bBMF}x%pvwZaRNvw{WGZ;g&*$>-*wt z#K(aq%ujM(&}M{DfP@Nz#ky!|TBLj1Ab-jb(9_HKIGDn0KLs+hY}{~SO(T(SKPNARom}i!wsvKcSCl6jX?T_l45>hyT3yRLI!+( z&A6YQ)ys-Cno;u)kK$-)x<=Z;f8&3R4lJZctoevFkJ9t#lcODaHJ@+YF>>3_?~2tz z$pCHAffGjWIVjPxyYx~ zs5SP26E`@DiE8S@9F4_6?zKN@F{a?wK*jG>orhEl)T~>6JmSu6Rd=yLM4ikr1F($g zJ#7SXQ2~1i0NNF8YHS=2}b$k11xg5PH-xnj6%;fi}(#@4U z3)h}f0H3l4hNfb{FvMf~rcAOs1HCi8;KRSDL(p+rQ&oW~OU)!Yi-@7!g;#bvoiqxq zahT;nku;`g!cF<6{bR*_GO{Vebo@s~DB=mS8B9L0q-$9PEyT6*mJNAB15xG==TZ#X zs7UPr(CQJiEIe-^g)z|sgaYjHPZX~~8!edMEZi-%PLiSS{l{^`F?0AcfFMO9k+AEX zYDuRjz(i_1qYe*}{~#Uw<01FM z&&>(98f#Q2Zz2q@w?DUNFF_fLd9NF-dfGZfg?kdKW$*Xa5W(wa!UgB;R?kb|Gfr=R#&dBUT!*9Gz9(L~k#KXSK5C3{YzRRGh$m5mhkH0u5#ysJ^1m0X2$i#z-$Z<%`n(@Z{}U}1;l3a^r7jYs8!d`uqj{1?E!xdUcQ-dv)9n>8bAS5U&5zhzSD(#+U%&$J z;ou{iY-PtCwsHWPn~r_3@h#1PctP5(1&D&3`DNcMQOxc*CsDpxGrl(F2_zvfx)2!+ zu;EFIZP?HOW7u?MMY`$9%Csf@urK{6i&xOWYwSRIJJT!4bv#OyTWq&8ca!8Mmzcp3 z=K7w_0#>S6GX@FT+;CmV&(B*OAG8T%Fr!@+HaB5|kuUz)DL!@tju~}vaJMM{XT_Vs z=W|>GrdAAiW0piL&@7k(tO%gjrli(R-u-+qwrt*bt6|)NPQeTTN*jdT@SCV>=ceWwjNBD1s)I}bGN8h9IR76#utkBZdERYSnwGj@U$2SK9I zsDZt5SUTHZF5^o7|Ku_@W@E*axo(6e-J;*>e5!$qElkQ{@J9U=#93GTWY5+KAWL^t z5o9b9I?(&Jh6;W-Sk?L*;G&nBJai$Iv6SoapypL-Ph~;I-)d@rvY8OW_~p%dT}%;@ znS_&*4{lMb9PT|J{=DJz#d4X;4wn*&#w3+H>@K~mD4!jJ9@p-d^`A8zT(gN8olm)!d#pS%J*j;e9={;pt<$TLNuX&V+QM z_8>|R6TZS+PEW8h_B+LYeWaBZEopZlNkIZTq7<#nwTQ-)O@6?X5HK8Ox%PW!8Ztq~ z4zTYtD15*FG<{X(4Tz-j=#}qtSnTx}K|urR$qOq}mT#}E2k5-MBcU?_*tUX5;Wtmt zzwGgAFS19Fh3b2!c91!ctUujnU*%*h>Z>v?z;vg$A!?(tpnR!_`44rD#b?L$AIqPCA|v2rd^>>Jh;6({}P0g@!L}yqY+A}Efaex$;6#D2Uk9eDu;f-XbJ!#^Chg`=*X zr+oA;io|~eEOmnnZjU%{~iVZInM zTB}m-IDfIDW;|z2x7o4PeLIvcoHe4JbgroQH38q!@1s_i5tO!{7Q`Axk95>r+U-oP z?+)&w3h&}U%s14K>rPbRf_-Q&MDA#F=AZ0!zRwg7`tvXxJYR~|{lGUqf zYDjz&H3*;)Iqb01y|PN0f$Z&SP?3M4VO*Z=mKVwV<3)fzi1-DYW$1s^XB=S(xYWB2 zV&T`2zNus?gK5r@v9yZArFn@}eC_Q0PSNB6?DsqGg<41^2A-ICOd>!#WIWm@+6mg9 zvmC4MXMidl3sheo-V+mNfcEKkknHk7cpFq%M@UW*6|X%u#(cZX1m8g8X9qFU00&1s z;%n1?>*JdoNW75CX$lxhmQPlzuD9chC!ugLyN;(O{T$Ma;naOX!H<|nynL&dJm9jW<2-h!)Kn-jJhIv|QZl{rNP;vwI z0swAmPE_zHL*ufNf73`VeUKXH)F8Mo>-b=Wv2(~yfx;prC5kh;fqzEo?<*8#i*5?# z!we1C-sg1>HdZhJC`*^UJr;IvYG*7An@4h7AqUhzh8lo0Q+G7AL#R)jvPA2;Hw*Pt z46s2u533?dHcSmUx7_A^Rt8OBX_!XWpI*Oq;Wg_gj|~~o+g}1|eb!b7p3B8S^x~Tb z_5OO)w;$65jGAk=jc_^=XW(L48xUmxk2g+wN?HKjYQY6IG7RO#$Y{9P7v5*VL@lv8 zYIiu2-y7&RKDPTSA&L5*dOa|_c$pMELIc*KLUuvW7|xri7CPrO4lCy50LUgA0|zso z4w4FKj~=W`H2aG#doDJv!(+p`Oy395wzRm#RNgU@|%9^Q}cc(9nSfnvEyg(+0ACuFRHSS$=$)VR9=Jm}#Mfbt9WnY}d}efDSt1ksn|zrZGj1}Kcf(}3=E zE(-FqJ!sJ8(Hq$&*DbbF>v4oTT=|Go$1>-_<;ab9MV=-9LW?_b+A!|Dv&>tTPk%M< zl}7)Q{O0m$oV)w(JV570>ZQ4&0rcQk-@SKkWp+dywCvuQWc`Rya5Q0DO;XXacZ-n? zy9yVfpX9kwN+tIWjgbK0o6Q$*0egeDX!1TXK+F8co)8*c;&GVO4_gP+v|%IcY0AAL z=xmUrQ{7rQJaCs+yaBN{Co4tOn zQ>#HV1x{0Cv<E1Br46sgTic)D3VODKoMUz}473s0O@|C2R+{cp1-q%#YPL_0ChI6G>r zzX|VMgk`Klp35zocACJiT9+7W%ar~db6h3|wQcJWp{3m1d1_){<|PXnY!KP>iE2B$b{RyTE=eP-j5zjd zb_*Jgcw;n8A2LXlE4)KXvm0xB36qY=Rx|OI;9{EeRsrL~gQ57I!pg;rLp05kCl`A& z5oa^ti1@5D1x*d@XrJPXMOxxTMJ+_RS*IV+wEZCgW7|-jo&>k}JPo!?u<9|%OW4B0 ztnp*I>xwNPzG*-Bvee0}V3FY?kcR^QoH%<*QglLaWJDzWd)XT0rn_Un)6&dNTP?`C z`3zFK?WX8m!yfu3%>Ow;yaRQL>ter>(uaXNL?VNVqj)HknyY!@Qc_x6Qj#Hier;;- zG-AUvDK0J&TGcrcF03q|#O%>C$tB87Sg?(=cxrONdLh*w4DJ!s--VGmiZh@Bu9&XO z#$$lIP@7-mVe-o?=oE6f;`U&sCD1Gj5c;+NF0&9{({PUpt5Gr8;2 zU5Wa`6Wrp!1UM1^d)68SHW(ckIq+b0s-gkdiSepO?bmlbf|#JOQD0OaEp1`~QVF~w zATBi9y|LEa?12ydWV1o&;{=_efAl~-35oBcD2_=E&&GyY?h%=-6<=~VFb(9T1FwrH zZ7$1%2_{KuCsd5i-hA26w!`DvB5;yOlaukj{(T6hxjus*_V-*7G4q45fvL~LnEI>8 zy#q0&eS~C&>&b^NyyJc04ODW%7f{Y!6($svL)4XJqR|H6`;+sN{CBelB0=BaXDz^) zlI(^^QZz0vPkqg@NlxFJBa3<6{bBd|eD>^0_eS_SAR-hld-1egiO8B43V^4b0lU=3 z+5yi;0o#GyWwP+4UAqbYSl{6_$O7I}ye@FUR#))^+Ap?Sxy&VMkBAC&*YHlO_c^R1 zueRrZ;nhwj7#b*Np2xBEOA|SNaCp#DfaMMplcE{fjw!wL&oUhey823VUn1^?*(4Oq z9DMsw#e$9e@Zp&%wuI|Uu~BT8Z9Nw;%%?uCRc@?7Rk_Q9ub|&&d}uYZ03rz+d~S=a zyIV#t+ehSI-5q2mS-J`1ogW(@f#l#-6)m46)CB?Cq51`<& zj$R%M6M7-G>~Q|{sQ^sutQ4@4O>MU&Cpxw|rfQCS7JZk(WnnvE)ET|<;WN&N8SjB@ zb6bHnMZ=?|0Id!#_xCq*pK<2&5FnRZ(yl9}c1g@>*;@C4Q%4!W8cZ_>uCFf?R=6=N zL7pg;MA{Eim-9!nG%V2~RG9ThGJ6TZea}MfLLp=x4Zz-=v+B#S0FI(B5fssJD8V|l zUMdQml2TLBJ<@3L?DGQgI5NyLBzS?^%Xqkp6smb2+zm&+C+^#Dd%P9k;VzoT&|S!z za-I3TOnAA?&pF4DtgD)m>C|JI{I`}S`J6hnxVo}%HE^nQ;Y%mG{kj*r+P@@iB- z&=#?vt6Sa%zyNThCuVVb5~6}WLoK?}XPZ+tcAIxnkFRiNNK)xJVx98-HQlsRdGl#K zk1y{S6gg!pGqnWu-}Qz+!V+XNCi9t+IcRe$jPS6_#X@_Xa={kOeIM{gHwAQEx-n>> z(e#YOM#wI7+C^k0CM3*`OtWKyS`ZZ<=mhvkB^FpsX5y$3Ap*IjHSo;TwFP>@X8`A* z3eKn&{8bAZa%5IbQdcYS(M&9w65;D!nJ7$=n;9bUV6-H*NCpw}SZD);=p`m+edsqY zD%2_JcC1t>c)SAW-(k&dg5etDdO6-Aqtq}7r5gCz#)O1AX@iL_} z95A|VLfj>k&MCCxfKoDnq2Wl0aql5MT1Ya%Qwv9c8?1S{Yh_!AFl3o&RL%f<};z6`~Bk_&lF4p1{_6^ zQ!gDn_dild0z}NlgM_EQ9NY1ZPm4x_cdOr^BcaMg&QaNM%K-@ z!OTBB-+UdRKGM%}T;!;7kL}x2<3ds2^z@c(A%eLsFCLqgJl4PS2-*MWzFg+VWq@$* z$1%PUf!Sqi+}*E(x0KIiT<=&}mu8(aWL!X&uQxD ziog0n(%Jaj9D?NRH@Fn62 z$z;$vqUV`RaV!!j_SM~eqeKqQXo-?AmWqOamyPKK0+Gg~tCrErf~q zN?5P)5QtZyR@LXTmMW;{$2&D59I^ngU`f|eXo6fjUhYVr*}fj5x4wK)ki%9wQ>W!n zi$mJ-7@;F{4F9AVFyM{ny-$^~tmP|fM3RewfOgF$BdCu;6%G3(RAu*%oZ75XPsL{L z5J9~*$%j;d%!dun{t$YE%XB5W@Q;0eY%nY<`&sqYb-&=3JvH`ZXsw>Eg{XGN>8?9L zn#QXS3HL_=?xeNgH%Hp%v3i5e78ZXfFI1FhAkj)ekbcPcO5jYBpZ#;9=r{-;pSkW| zJP+v7B~*r!%>fGyQL|C=kR4g7k(j)U7m5y1RTtxt0MWGA3-FtLl09|I@^}PInOptv zC?BpvIW%+gMNT*!$Try9|CZf}`Bdgnb_%iHy3tg3>2LE*g-V7`{aZwfEwgw5$7FGhK0+Hl5{$2^;`EZx=TRJ+I4dV}eh0 zz}KOR(ZE;VrZ(~dkL_>OupHoZFV}hwpFD_MUNE-aZ3PVxA=W^;s-dAFTJiP{(Rao_ z&B*EseL^E5PQgz1udi3}cVB~a+Tvggq#NWBHxAh$xU~(6YtRnP-@h@cQW%BumabDP z%7|U-1UvdNfF{>=Uz+F&)%XlQ$s`v)yL!fDE?nW$1!&c#RpPcb7Ac`!&IY|PappT(W7 zUT|ARk`P-D-~;jZFd>z_UBE%IxwSHu&4j2m8J~AE98avJQ91`WabKZT$uGpdLAAVc z;Y0FfKj1~E5KK$3Ez~T|^4$D08J~-K9pkT$9)sD}BA~uXfdXo-tWEtD);birx7*3w z1m33-;x91fbmwg55xqO<-)k7pwcZ)eEtdNKSo`vLs@rdEZCj?um?DuxGF7HfYzaxG z2$57|9y4XOX+nlZnMp`OhLB9j)L@RxM49J#_O5%Yo;vUOo%eU%b3UK`@X(X*zQ6aq z?zOJ#TGujpj{l>>|JUvH|LWlRvjVc=>EzF8@sWbDu5D&XQKJt%*Ou#HQfc;DfWfMd z)bijymH?J2jRgqb^loRRyFaZZ!>9kG(&ts#<Fe^@~&N6Jd10D8Id_D+4?QoS8jt$97>UlsDlTzb>Q2KhpCH zzW;AfnOz(U&5?IhxA$&6W`Ohrh42!VA4YxGeB<qk#+xX?#ot5<-T#tQngG*yNa@EEE6#^oq{YE-c8Q30 zNh7zD<`za6x+WlS6pG7i?xeZziXfW5U#na__;zLEffd z>Xv}l+p(N@z@Wrd8aM+j29uE;P$exn&>LD`hbA_DA(a+2+SOWo_1U&x`ke@;Y4^4800TXnc#z? z1u)3qP;U(>dw(W(l8(_P0y@H9p z`glT4&SN<5Vs7_UU;-qn-p#Ca5B9E}Fa`#W{o+1@)kjs(18#lLT;U$!C1KGud6`K~ zHU|!Q?PKU&R(!SF!j}G(@@Hi%+@zjmA-cbe8lPnS&u~VJsc7T_Fp;*8f_## z8@PQEZ*bLSC{}|3R8cVbD(Mxr85u}W3z*R+(CEX z>bG?y!!CX_2gzm}oQEvM>w*Q77*>zj`uROOQcZ?KXS)5|F&hDbMW;a3Ti{8mbN;;9 zz;q9g9l6D?n77_P*fFrY4!zBF5EFMkJ`I_K~TaIT(l-@F~sO z4}Sr*68fH9ULWLerl_eRyCAd1<>khSQnOew^XC!Ir4_hl4MhzO{pVgRlvkEUB<&y# z^2Q1LZs&@+zY%nvWs7u7Y}VI{ZbKUOM`ch0DfDl>;Y%NUBek(=zh;Fk&?+iTU*LDW ze1H&f&mto*21~lrmXG1tsyRxj*wq*?e9cDnTn2<*Bp5b;GJO!9fRt)K?~CM|F8Z4( z&Ck18UppOol_OBCqx@PE`I2rD1acy+;sQ*aVnAX3Zl`?5{%^NaK8wKdP%gO54PSNZ z`q_<>^r5;}_R`{^okfd8JJkLA^Tiu0#qX!>dW-hs0TzSI5kbP`KZ28zp1>D8UB@0$1IGq$W6R|} zb(B#Pd;_db-n_i)5B!wAzgP8V$+GJ{S2=2)zDM24h8k99s8GgMehYza)Z~)^zR+^4 z8pFw!XqD`}0n}lwfI1dJRH!_5~k% zy0#iX!R9eb>v``5?f#=EgNglVQpxRsGMVn^xZY{@l4AGI#Wy~Gg-#g-NYWi#_;#mY zK@(EqQGfGOieGJe?s3zLZ;$M5&)fFsC;u$8_euZ$Y^1Mu^#MK9e}Euj@BX)wxYOtJ zyqeBw*MqU;?ABDM4$(v()Vm{O-2#w^b@A~C)wTi8`L>G#QOWw)T~$buKAEq_F7vsbI(jl-bC-&iQ3Dz1SA1w zhET-6So=ZhG%dD;&3*En=*7ERyt0-+rXHm*)0Jb>-310|2)sUvREsc_55XUC~TNen%(pH??`Vh@eVHA-ZjX4e3ax&5ucqX^Kb*-r-R4gDKHA)CSF z_DU3a0m7dtc<;dj7@K@_P1R2f)&1?t9bzOB<+7yLI~fF)a$rRa|UN5gu|R zb>2CN8Ulhl$GRm1ZGege@9JaH#xw2E*vM`A$q}7X8RZ<||90OqY9~jk7zhrLNsSX; z8*ank!AEGr$JvoRgaIY0K?C&P-;X>E5O|8zTEbDP<5_Iis`fTlo>*wNep}tf&ss%n zzu4K6dG*uoZ|mnxqU4~ZKUn4KTW6czV)}^uXFWwUr5$%wm0KFmMQ-~L4)meUV zp#oi^J_UN8HeO8Lutd}ozszSrJ#^)bH5-0of29R$Wx#tOZx^UVa51>=!}Ba6-+pRE zpyzID0Lts}3(Vz4*#&Or{wzv8948H{**j zx4)wos73+LBzRN5jQLrMclj>92IK*u=8oyB3{HX;h`|$!W%b37DwGRSFv73?eivXS z*X4{%tsgTIHP$0K(zQ-XP%6uP0kPDHKO*gzCD7)3b#DH0NJ0mA-sQi*nvmfjU`?n+ z#$^xMpx+@|0F^KFemyXQk)O;*7d|4eDSx&>DT)d*Cb7X_uho5!8=hKOVliCGUIVv} zwyw(K4mFKH)=A8_7#eP@r=+EENpTK972Y_r$Z|<~ebF$&m(=}9waC{C@gAtsDTIJn zmb7A$@4vUh3vkIHAiEq#yTR$rq>X1$hTRLhII(U0t#A5Iz0llF>5oWn`J-?{$72U9 zUM*%%q7QBt5<%V-Wl`eS!_0vPow{70X%q{?A3=Iw9Kq`Odb;`&?)i%s9P6&_UTY%;yEIO1kuJX80=fTa;RZn1$-sg&*Hp$n zDP2Rhpb)fELd&0eEKy>Yba<~3tY~;pEMah>weYrO^V*7I55rm)&S|X#2)j2-BhtcX zhyM$R-MVs@)IAT8~*0RTQ@mpEx5+ZoPxQ4uUkK6U1 z5`X6Kc{^>$hX_ z?Uq?oQ{-#VOCM_SU5R3$m3rcvoyJ#4gVMK;e=BhRaBNlJ8!VQExrF0HSJWPt@Au>v zK=egMzWbNUi`_vsh|33aspPnk2clvN$rmEb2ZT&aFOV_%;Bv4pELcp3Ug*|Ki0%5b z#B1*VBQnbZ+p#q91O@ZJe;Lo z$GA0z2}rdT!;nk~K9Ys7@!Ibjpb$w0Sy)mn=ujF4KJze(yZZCH5zD`QA#S^zp5)Vbu$pXB`!K+ULF{O< z(C2%IVbfR=67Id^~)8C z1!?GS9d5~p=Z8sc<=IVIdgob)VUHNmUXo$k=qzI*h?_!RVm>`%2EP}hcqR8R@KtKW z1cGf$=;G;&V>G9>;{lh^UtmTQnEq_N0-G|}t6p&3;e8s^DdGEHnJ$^XFnFKq!fMiF z=;)fZ9K^1V)J}bFCC1TgSxX2!qetF$;iHkUTzcXEzrOtd&w%YHTpV>k1&g$Rxmn}t z(oEK^0w6>s;36FgGl`B14-c1LS}X|#k$eLrPOS$jf}FctZgZV^g7ex~>GmU;3KQ7z z)&T0dvfEcs?iOafZ%0WmtK?$@$@I2qrRrXtcvR}~p|q>66wfgJ=@>|{F9lGKUbzT^ z=$xXrek<}N-b%)R+TKatMdsgzn>e3e_MIXsYtA}h9KCUw=yo1mlW|0CFP{!tJbz&f z@jBw=pYeKR28j|TFUCK)JD(VvcqlH4UZOM)t;}1Md6#uYc{-`+ zxQ0NHdPmQ*n#~?2l^^Y94C7Ggv_}uVvnWM(s{~vq&u(p%S_?Am$g>l+?I|e`OnU;e z>LtT&E<^p@G~y05sh?^O@=M&*P8b>>L>n~`lS5I#lnsf>)fh4~qUYKP@0%-pt|-f? zasHyNP>(Xz2OXy{gi9QmpG3z&CP@c zHpqPOQs=w?5Ke;B?>4YJSv#cX0SgaazlxC}3Xod>r=G*Ft=b6(V^#&487AN&(Uw^pC$^{T?QU1j%WI6Y5ze2E*cyIUdAy0wu1xT96 z_uIl)!v+M(8##goGj2!Kv8gipEgy(8KQ4PY(O$ zJ5QulE~37uV+aK@RQ>%@HP4-^2Yw*ZgTn|4ger18G4)Yz)K(w?{zeW`QuVXo7hR=j z&AA1lzZ8DB4SI$S9xHDdIhsNcWBu3^0! zf!Gen?+vUayE5laC)Bmy*G;n^{`?kNt*ZDHA^;ID_B z&k&lo?Kuw;3e;lESdLiOndWRlM$r*w@opWVD-l*Xap2S2l0Ng}P30ap%ZfK|C>Zqeyg+N+p1x5C7~ON# z7^ye9%+n~zMYXs1*0zt};G0XgI}t${L}bEBml~camKD+#1s&}rTL$4r*XZsJ#C=g? zQnbFR-&~G)QxKG8p0>Pp&VrQKVgy^^4~>Mv|L7A@S##NGh8)~(-cPvLW1;DCyF7b{ z7VL&3$AOQz))3sCCq6lXt5N7kjaVW4cSOJM0Wu@B=jDabLy*DWrQEsG2S-kaY$*5h zE@C+*_HT7NQi!`meX##l;E&pGdhb6W$hjmP_ZAAp0FklCLvA~@x41nJ`XcraMyXkR zO)EQby$CFem64*Vk8=a42Q**=$IgKi|7{?R7zF)JK;*kBrqSDCzyeawufq}r1p)JM z=wWW{6%avk)&gmzo^As^U0Meo=926?3<$$d9ldxI`0J|JaqL2m?Cs5N3pI8_%*Lw8 z7;=*zl_Blju??=Jv!^eGNm#)gjeHw{^uE!YpgI1WN4vMZ2MDq|9K-WaKchV{+58S{dD8~ zt3~Qs$K3QqAwmitrnUWN*P0My3N~A5O5uDXxNp%HobHulUnE8YtoK6BRq$|2pW|m% z4T4M1ZpGR^R+vzV-?+p$&s8jc@d4R1bGZZ9bkT0_uzon>hjlWIv^&g7-oL03|I^lg z*oELE#)E*FPIkltT|3i*g!;)ENfPIU$micj06NJOF6~t$in2ZlKpslfQlxkf9z;Io z&+_vpH47`u5>;bV5R^nUNxleke{*xBbpWb;4kMhi!AN{LPOBAA^bilJnPK>8%Jd~V zct7jt+ViU5cfs6P;g7rF2EQBr=*nJ#*yaEpKi z#tV&_7o3ZR;J>?t*-px^qp5fV2`vr>$(un*2@Upi2o4n{FVeJmXD(iO#ob7W?T@`; z-J-L;yGW#fkfNDP(?O?ZctYwH(4>#8ElS~I~#giz?DG(U1LQX zfD|QKcjmvoE?|v1jQHSLDL+XzHbT{hdKn04jwN{qTw+Kd8r4s5uQ%`%Z-4k?I|K$I z4Bzm&=lR#B6mG00xsES+b5g(m@_9=JiTp30H>Vt=xcQ|1Z)?oaTnMBw2g!{WSOwJs zfL+}<7e^g0AJ{-pNdDBau&P=iw`09pYN1A?7IGNZD@p+?nC=_*ij5E*lBpMP6AxB| zuq`wqp&8n5A54=mf05irf2SaH=jfC~e^4?s5(6WA;S~X#Fu*&tN1WlZXcM0%f5+!C zrS{7U;NYmTXJ7#r?ROFXA*3tWlh=7*fS!D`4tY+}eV$5^|7G7iLuj=;>(0VG-cD5t zJCI!g3w;IhFi7a@hRMW*p41mF)}#e|OfM?toi3s$8Qs2@>%7^m^x)tKExvMCvr{+u)SMJ4r@p2yV&OVe2$xi*XwJ*fY+*)WLi)60?5ee0mcGlhLWYV8$b zvA(+GbXO4lURk2R6_591l_5A~WnhMx$zWw-1k z(|ibD^7_-i!?8PJyONW+7b1N7a+Z2A9%?wT6YOru4(i=jk~x0GY-PK2UZ0shOo1&= ziapR0CkY*evv7-W~KCDLP5fr&IY@~({9}>7jv(?nnBal``9+}%Fe%a;-}hmKpiQG!#-?g=oL1$j#L^6&h?9ICYPGVmIoP+ zauYVm0F8O}mv+I!vu%-)j~)oRvScQ;C=@@hGYn$-lxG){+ymL1tA&1dQVpExs+Et4*_;NDs!2c?0Y0^eEW4O?=h2boU8_~$Ey$j zyoAz!=Oui<_Vw3ErN}+~VJ)*kMUaU%oaTl=U%Y|l;Njc;S7EC#9)mOah@*sR8azNI zt{t`h(hVA|GAJJm62HB_9neAVi<&5{3#R1BtIOxWf?ef%^lq3xdUFTAu2E>uv9#j# z<>(948SpXk(LwIS>g;tE9@i%0EJbM;kV~{WdA55&LESWr#1TmStS(GS0Z7B}zE@7N z`$R-@x5{2XKVGES9)B#Qds{zTLmiuR=nHNAvXq;sMj$4}+QWpQzKdji1qt zKjGqNDl)Hg&*)-GgpoGa&+1pg98ICK;5sfgAH~G{TEXDf zcsuFfq@d$yRy3RpHhTOv3@fL1#TvCwsW45IIP8>xRTiT5T4B8MT(zdfy1i)C8Rezb4)>YrhNF!dVRKjeJZ&akC+&n`h~IC*`)#yqFzryS6lbSI z8?%aI%X*yd`ZrzA#avr7(foV|*E|;UX5TAgkHTcXa4RKRkrzi<3ZEQfy|;!lvT&v| z@7flBjM75A=S!XXT`sH>?ANI}wBIiM%Q*1w?6(qE`*zQ$H;&!M>y@^}-!(XeC7FHd zlb=-~#eB)~!F5j>IPpi4LlS}yNV$>-Otbh-tIJAG#XKdP1^JuP!*#l!PnqoE|Is{< z{^N>YwK7@r!T)Q|nQ_=w>@un2&+WLPAp!cyL7)mo>Svc<*lmDTm&GGd0@)!M(F)nqvkbwT$;J}+W}BeR^HQGCB%bL(cUqMvd**A&WX-5t94}lSW%SB)hc)+$`>F2D zUwN4rE@raZsriYE0N(*mR$F?-#{jc2cW(5371Nw zsSt~Zxu!ZT-%Xx<`{9$T(DWp>(5oDkIflmANc4CUpG=#LO=n(Msy)0ub2kRm6AggO zi_JJ&u8L_UI0sxOWDINQ=2$bi!-~!?PM5p`?7xE-k#)^iLKY$02Zk%em?q^|%|1`Z39{CK zdvI)Kt`D~ek$sOqUwW*!c!j{gJ zlj|Fu`Wx&HyV%yAo(LA#ONc16v3{n}ZtEXoM;+!O?Q)?0(r$2C`wv<*wo_V>x z^^cEu&0wTgCbFpUEt*Xb^t(%7+r4ovup0zxx9-?X<;Q6kCpOk?B*0zMl4kwe<@LQ+ zP*84CzGUbj)YL3OI;WjhQ6bWd6V==C#%QgHN9n6KYkG^;ZdKvtcc)Q)wkq&Pcx>#7 z4m=(?E;^jW_wQatw?|na{`E<2fzM(KNg+F^`45g>=>G9V*EIL9%xDjzi6nw#xLGZxp@5>32KFT%a(WY zMv-Yp%#o`UNQPxB1XIx87N-@c31qN7N97~?40_C^eHB~b-oeGzk_~#@p4?79V^dxV z?kAA^G+UWJ%dgPVmVO+5P18=)_3+Y||0pSk-rEPg`8iyS?PUhNeq4S#U)+x@gEUK= z5W|>+PM>IxQnb)KkWQUiv4{8mXQj4L2RLz3((5P^BontjchJ`h7C+6GmB#D1BkfUb zn9%Lcq0y|`M9{^xA1;CuGfRb%4l3uDy60il()m{ro8$Rw7LU{~9VairqU_OyTSB2+=}@Mj4-ywrhMmc1c`3@d{UaEexBBJt-waWW~M zoT}%wRjFWq;1d`1WPoL1Wp2czM)MtceZtujeVzw{NfhxoIdWk!vBpZN`NOV|<5^xP zngb!u>Q%X~2e2b!6>H~GwCbO$$3IUP!e1$D?GFfDa~gT35YQI zT($tIHlqK1L>dIgu7DB0-Ax$;KN@16yl6!%^4iia;W7_io^jxngC)JThU|IBlr#^2jnvo-U}RhFaj@eltw_!K|>8wVfuV3UVuRTnL;PpSyb0-8qK(nZp4H1`UMQ->dt z!60fbIQD1Q1Vuv)F(Qd(URh>cs>46v@_NtpN5UF-^mmd1uJ+5;1!N>J^wVIRgj-TBtnfq#> z3RclU&zwLV{gHq5>C2K>F_wjqk&&Su-|dehBw-snue>LRE;L6HCm$NLo*~=OIjR&l z3`PG{$AtBSCaii;YK%1?B{MX+zZ7gJ35Opta+EmM<$e`wG0k89*wliOqY7DwtKxoI ziCahS2amk(6`VTx;N-S*W8Zt6fXA`M;uSn#IUr1;4A!adt60 zJzL;n1evB-9X#Q=&WKZH^Uzp_N+EMo(^%k0=r-yOMn~_Re4XwQxOMxU4xx9nj0mdG;$-rU!4)_Zm%%g88-~h;Rf znts#cLpccI+j}96NvcUOwbt*#hj&1+v#f?YuJdD7+p*}?z#j*cI!o422%*qGgq2X} z0YC3&f*wOk&1PV?Gi`IPM(&*@+>hBbrsie|3+60V5|@?5A+<8pv{w0kf6h4RE~Js& z8Opto?mw)mmAk*IH(t9ZGEjz#L>2`JyRd4bT!T}-(s zjvVYj0g`Fx4kZu~{v~Z5AYKGl$aWO;%%FR>Y=UwMeF&Th*`#Dzsm&)T&iG#Os{;1sO7OX${%ku^{{vGmy=cWetZ;dGHUnQ<5 z!;dDkD5URhFYDgkyX~bemz?;W;EQ>GF#TC`o^j`NFH^2kVjLtd@tb;4~if^AOKQvj%ksj0d7qn6#fyRjwc z-k`^NPWrOdLB(TyTt)l`0~&=S@hvd<{C)^1FQfV9oBCjrclX`NfSTS1wv~|G-4-kv z7OSh9TPG<(mF6vg7W>xJfp*i{`nxfCZG$TQ9xp$5`C0W&klJn4;(u^~h87j_;=6p$ zP~39jU|wL1q|rYJX=~E7P6z|p?9MG7Bl`tQ#l8sYWbOVXWUGK7BpnCDfX`w45g*(85Aa?Ag&jXG95ZE;2`1L&XE|T7EM1<+wuVxrN zFBaDC;b}RiaXMDLymherI}-wURGu<6eJD(cpq1d9+JEsVN+52a9U)4M@C52Od8IV2 zbdiG(f2Kf>?yp=;Kd zoF(KEXv6vj{j+|Y|MmJEzz;PhH8yD*@LPZD0WOMbF8FH$#PeONu|pJ-L2stGQ*rf#ArCEz2X(^-1xF1BXuymmJrNvr((ZYt=!y|=U9v*v^=q*jTv%`s(T81x zgu=86puIJANYt+058!;dh>#n@5XEa7%wt1@?rJ<`IVO8 zq=&e8|E&6e5A?MEa>K^un|DRJWFM~;1e`@oJS6b6*uco)%AjAj8;Ut0xa9VEN+H|d z+wwGQOV<65(p_L%eswX-Wkdh~Md`@{G*yStwp_H$c-=v#lbd{Rm|0JMi~fsV$RT8t zEaQ(q5c2FbSCrtH&*0kF|4$kr^H&;S+M#WG!Xfz~q;($+WzMqK^@~=WeULIf_gxKH`BTx5f ztj{x$c|r!PJmd^CG>c<4UP=b5y-car$TJW;IY^wy;73ievE>e<#Tt$5t}h(iN_oGbuzh=nxys$E12J@kel{4KTf$?CkM9)HMv8A3VX zBEbQmZS(lEoYORat!rs~3W5*z`~@FWFS&lsF#Lr|p);_zt{D#Y`F{>Jf(3%qK6dYh zgFW#%qmKeIUeB92j%{Q{D4!AXENQtoPG$=|Z*${YIrL~YPj(0ofCkCCr0rfaL@})B zJlLT7ebc{T=kVXW>cbeejg{rqYbf)(;!CKnED+yY%bbdiY^w&e^6|9{BbkYbudmpy zy3Yns95;^9>6fkrwY55nHOS;Osl;UB?Yj3c6?x;V$+L5E8uo`?I00+~`A%DnRfxLE z?v!2z$EvHVXOZn}(L&)w{VggM&YPW1?bYjA-q1OJhZ?DhhS5=Th-5*a7j+c|1=q|| zJd(1LoLG|t>-A-aqwCI zJ+apm)$mn%)e&#Vh_`<@eELtE#`ZT(11F}3%;9^^8iuzc-+&$bk%{p02rM{$VksK{ z!I*&{eQj@8U0x#Ey*PGK2>mg>vK2e*)!}kNCNTY#K~jZYlvE#%Ln;5M3}5PKUwi&_+V zz#x0sK2M37E$0b|;tujvkY^?#4BOfYWY2K`l9~M6wwDQhDyr(DTW2qM&rL#h`!E_+F^2O?8pZ>y87)W z=PBW+@7+^e2LD#+)_rLI?3)DZ%!aQ;^CGT2cv7G5po4J>Kjv}-faOQjQj3>noV8cQ zWDgZ8mBM5p9BMd0UdB*846$!b_kyOccH?zWh+#zu(NYPb)6h}LzbTuI)!L0Z5bK-d zidwcDas}=oYm>5(>FEsLYHr=h;r9C679OgT1PpP32GM0s+^^4>-?92tjw!12?rWct zsz7vE>n1rs2(0F#85|!9o>i-80|GYF7ko0}ZlnZUT3=uPhZ~1w4?^YH^yAyuo!D1jTO z4EW6rG@bF@xbPOGn5C40&p`f}Wy*`g+CMH*}&r7lZT6s4y?=bEXM<1wv zu`8NC&cz2Gtf?>^`-lqQ)*8F}DJ;C)B6t5>^7tKx-kZO0vG1>+Uv;|$YuoYt7Zu66 zNkt+j0Nrd;+P95me)MBD4%rxl)p+9==?6bU9JcEHb&+KwGS901LpoL9%F=PS(Sit+ z4=Q`xlENff*o`XG%F@l`+9`_RdcMGlvAD^xG=ALlNYYW`jfM{B|N zC~?YTMS(ka2<78lR6a@qIUB%#`+SyhJv^577{V6DHpB++q2Ga%V z+OeY69jByMzil_`Dsa)%*4DnZ6eoMT0m%fw-sMp(urrOQ%?>0B{ezj$F4=;;%!j*;#Tnu1<7$Qd8u+}k}j*4M_| zN}`tI=4h|IR`4yu#VTQ&U#2IO+E7!EAVi41a8!x(V(Zb{B#!W&BtHBOm>hF1H9b8g@P!C1WZVGm)c$sECeZioq4`u)RA0a;aBnek>`YFI zK8@NyKt@-x%9^~MIi!X?12Fk>E=Yh>E6pIJu?~3i!3V{$2VlpGH#`yVG1@%eVXumW zUD4%MS5(pw z2f9002K`b`%yN!LQJpb3m6S2nT|llD;KR^Lb**n5d@dATBqztA!H3AOGo>{|jB@B@q}b~m zG5m>!8&TokwMpAC`I_|rxf_RN3bGz0>Elil-Tz-7h{M*|ni>HY*x2KMw zUKN;6VMMq+jiQOf#_VT`JRR1Wv;*j~4lf@`8ANYjY#aBZ7c)CBe_fqCC!J%OG-vwn z#*Z;JbrQd&#k4MtAPi^}%4BN-6$!=UnONKQb*Ssd_*Xj0klkp&GoUr_O;8nlpZ9_bYWLwKC|$wRVPD+;m5~l#A|!$xqBa!$SM1xV zPHs~3R0n^TDE`-IHi1s4?#E6Fk3-h88ia=c6C#3H#Vo*9AVubyj|B|`VgzB3Wkl}C zb{Gk^$n_N1p^pR5By9%mc-71tqj3Gl5~DM>$H1^ZiWq>?JBA0^P21)-bd8xP=-B!5vvgN!dy`m4hf z+9AWa9vOm8N^CDSPkjSgobTz)&ta@Y#%Bsj$5xQjaH>v*50v&voycR2uxgp`bOt-9 zwz1cVqNJpnR13bZ0P%cLg$O@N)1uRT=CxyW=3YHq=s^TId_D-t^S+&sJk$Qy$@5?9 z#)M#lS;$<~Wo3!i=5vb^V2W6TF!-A;*Eko+!fLUCQNfqh!lZS~2g;qJXdD(NJWps@ z!^ml%1|+WW5#}ns**gg%j_Oel-@;qq)!W$CtE>An)M=_{@VQ^VMzFNJr)u?T{UklH zSq#7R+pOi<4l@0<_xu)&w!zEQ|9pVl%9$%wJ+43^o7wH_PCT@Zbzy_LfS15uwK%_9IJqWof@p6d+>{IS45t5O8M7NF)Vc zxFjUXe0`5L?R#(9J?M@B(IaZ$Vzn!yeOv;=23+jiMJ2bW>knCUAfG|V{2X&w0>L`ZZ=DXStkLndSKjxtVl?O$C*xT z(kTAq4%v3aE%eNUeF-KNh^lPwC*~!4i13?C(@U^l)!?zxFI^(ToKxv3Y};yXu9(K2 zNpdauex=pSO&`}xv#^gZ5bOTg$Lx z#2ob@V~;;o2dG#HN8!pIzXPp0E_MaI1dm&HLNpK>F!au_oxM&GzMSAZbcPhBu=n`P zw~w&+j9SWEAfPTnS#VDWoz+xx#$RD8ykx1N)NW?CwOQc71x{LYO@su-0j1P@XH$_j z*HDroYjJpA)Ya^VdNIY11NMuG?bS*7P2AW4kNggZ!4MO;71uDNzp6oC=Gfq9CQY0P z0rCMb-kT*c>$BVl%^r1v$uFlt39q7+bISv&Uoy+*2vqAok$i66>eKVHI)MEF-~Fqg zI(Tk?pKs@%ydt7NwKSlj3d*_7vhcp|21H#VOrgAYH~mG(njO&CTaoSH$b-CypL=br z_t*9>s*YS#qNLo^ejB0FC3G#MMScR^T@yl;{_JXjD98mS-HMcneS$Nz=PfLbJAA0+ z{@pKEpTezcd?D*z^6v6Y*k2GH=O^s{=G5O{08K*T4iImXW&nuc*?L`7Wz6_d=6R(z zI+?Kx3l=TGN2Kz_4T4*_A0ucUxxZQ+ExNo65NQ_Fv*!!0A4a{)iATZNNH^n(TzI3L zHGDK4Zkp~KgHueozYoHuG)`i>5v)C2w;YpTvk_Ji%x04dEu)h4Aoq?)7`rTp+F{rL zX3S4)e&#vioH~+Tq#HSTNZ!DhF$&!kQQw}Ob?iFf{f|2CJEzpm!SSqc00Hf?3!mn@ zZPu#<^&w=L?*muPY(Wf0H2GvqAh?%og-ecaaRLB~gvXB`a|o{v^h19XvOjp|1%92} z%*z3$GK8FfT`dS!{nI`C9&8&bu$+kA!T9t`koO>!TnH5=f<@PKtv~US5FyUxS#>dd zF7NNa{G?j`So6QAmMhT&vR2!XL`-=6c6T?r-11cj=&tE+5D3k1fphxRg`I_PPU-M4 z4eN%~!kgMw{;|BiAav_oHFImBM21B`KYZ!Pmkae7E3!JS<6pX&CGwzKg20q8A2RQX zPR1nN3VM1DpnIdwpks<}KuFZP-})#-r+TwYv%zq;{H6Ygu>}2*=F|~UA6BOpJenLP z0b^H(po?t0Att?31+xVmWysTS!3w4HKMyf&Lq&uYw~srHoq$c$*vn|K*PZ`Wj(!dP zRqk^=hH_tKv)reD$!#H?dgrH%C=xKL#v_P)K

4$rGMt4}K+$$-{TDA8y@aNI(x7$(D6=Ope6SB+z5l2dKuSzbO8D-Twml945ZV`tq`p~4jF`=yP7wTzsvwYg2&S#( zu=Y_BR{l{5GA>2FEh=8ijWHh+7w-kT-+&p;yc?*C7LB<;{jgL!4|zkc-SeWNXmbk- z#A;?kd&iOuSYBT3o8DfqKByplpUs_Ll6EU(x}WKKK4mOQgK)C}#)w@uUhofe{LSC8 zv%>)-BE`brJA%UA)@@(d)vjW@E&|WlcDdpA0cf`i&q$aRP54C z$ig}aVkLz1ktmg#@e|a~LMF@#Jz+y0ul(B7Zt%_eU{c%r_qI=>WeJ_!i z6stp>jFSZrA;ciZ7T5$X8{siycH3N-X=1#jP2J3VPHJJe4#1yMfgcYOESsMKEaW0j zk3j{(^+8l10FQ=9;@0gDt%%4piMOm*t988c0%D#a;gi=o0I~JLj>DP~(4FeXvha4f z0FapZ+n7bs4frB>@kr$2H%D+PR+b#g*0A(}NCj}a^y0higD_?@>+P{04>blMCI7S- zlu;Oj@C$_lm#N7fqIIQHi-kJz08&hGaH@IZnrKbUI!{=bGy7BYTrNCj9U+Lc69nNO zN&!|;;_HAnllwZUlJ*U21xUaNg5mtygPl50c(+2BNrupLz+Qs_S=vLN|`+Rb!buan?BJ4n8Q=Rwi=z@f~o&;Ah-Lk zje)AMy5MIHZFg0&b4%nmAN;L%^}bUVeXz#1%oOtq91sjJ9xgzFDz0PItr08Z8kF z+Ya=_B_V8GBkvG6O(?R%c^&28FDmLJK#`r4}>AYhVknw z(4s^#%EM=5uiJwMgUaZjTXUg*1bo9SC6kKO&{?eu(}|0!w}v>_D>(YAFuux*zSYDYySxyHLUzoJZi)ZwWFU zk>pZKA5_~?!Gqc<+pe!-697DeQqXG!o3{rQOW?|f^Op$;trgf0JMcu=mXd2~XzXi* ze(FmR{R?dtg7tGv-d^-2u#Zk5FOU5s>tp9GC`f|S>PdndH)3L>-Ar`-SmW%9=@@AD zg`(j3$*?j}CKkofZZjiWBK#NH>$y4jaNQgR*hw0{cT&Qyopk8`cP9yAd)w#Hg$^xm zQpjg|UW=n>|D71O`W1j zT<)*q8)pKjln;S7*IxWwsrEnaStQK{6>)xQPMg24nzPQM4u?|RDv%zDfGL<`m99LhON9n7i{tmdjJznhbI=IdLKcykN$!Y&DVd)y7x zr~XfF3m&8(IUZfJiDu@}M#uc*0>+M38^9jxZeCVm3i7NRL^H73_QzAOMO@In+0^b*0Vbh$@%{~QdoGIo5j{^I-NJy_Oo8^!B4Zq z<^VspA}|57$Ih%hek`DWnmf`K17kj0E-aB=S;AxhK`%J>W}gh?b~OmNAGrSGswi+= z*BV+(%E@QX^?3s8UYKcYSia!v4Kew+Y|p@q=Nah;p-)?-`xgZpRMYiagednOgpDFc z?tcC4mhAQrE341w6-583pH(ggCWaVVT^zXjy~$!uG9KP;pjjr!k6Jah(V-%7580&E ziqc~~a_iQwE7mD?F#}V9EAA^&8@j!`X?Z8!UO{OUJ?sG0N+84k1o0P1ohpM^Epl@Z z9k4MGQ|^Hf{HqxK7cLP#7Hak)-Uu@0KgM0|XjLupzqM`em4M;sphnCxgWZgjG*UJJ zoTKAelFd9n3*(#X!C&h%)NxruLA%B8BNslJ4I(7e0t^%u+NmbU}@2mTC6 zqgw429fC)%MWPn4Rgaykeish)0Z_`0u)3I__i}%QSzBNAZ(MaS`>dUN`qJ> zfj6Azv6~sueoSMcay*zao!5E`BZiQYZpRzltza9nUHQo&nFiE}GXx6g{lmj2u*8U1 zf#A9>H0s9+!HoMkhp_;<$CxT%vJEa51-N!HS|aQ+D*jmK2U$<4-#gXbsk9?iUGUn+ zLSgcxl`N-^iwRULsR{o=UF=PyuE4VJjsljWn6MmKKW`lnwi@WXFSgo&HtjO`OV`UJ ztZpqymQRr@L6eQmDX)eq8Z54aZgEhh{j9Xcby&t}Dq4f1u_GS~^=JBD&Ms@^`2R(Z zGXCAE`5zfM*MO4Jo0Mk5Uy?IBNSH3*z)JdcRg-9)SK77l)q#T>Oa7=0f(1Qgp#95P z0jSZ;p56&%fhDS~RyT}iuj5C7bBEMXUtA2F@*-I_rnSady7&WPDy;{j_7hsChL+`S z?y-NaBmWbmBZB{>asgcc1kg|_zUv9TVy!FouE;^fOY=4t6N`&;-`w9zi>5l=CVS~I z!kFKe`QU;>W7Bez!D{MltrI%1*H6>unYZm^esxY-c%1hYx{P)0s3h0F;Y+M<;6&H+ z(^CIZHpGULeE}~KAQ}`wnjty{w|YIe=a>sdCMS9fUWf<}N7U&oj5X4k1A{^ZAe$sm z2dL`%o$wf^$ydQ1f@?iE2D!^niFK$YNXoQ5LaFlUhLbNZpa7Q^#1cV*7ocApxz92* zSyBq$nq|_MTQtZ2_K;Yfpg~dOJitv|s?SP!uf?xJ6UQB|L--hE7$N|z z`V{FtwkT7E0bK|s%Z|iOKbL?Pm!7{*wsi9c>~)vn8_v)D4d`w#^Op8RSDljl`2ez2uTPQg}$BTm;v?#IO z*%0ntR=1ZOBJl4|FpFKs;|yqY7UnGy?KV!HXAWHP!4T7&Iu18d6*yFr<(JTvf?^0Ndh88v!T0HWvS=7d4>p_^qKI9IUP8!*Hzy`;z;|IB$A zjIykj>w`Uw=vPC%mY5JxtFroj^LYS(LiAFrq{is;P*nSb;tY48G7D8&#OTA<`Sv@< zpWZBLuKs65&Hc2b%jIvf6U=1u@1GE54UY<=4N(c_#8R@HU7=Zjf z*4AA(t&Xo|I&e11?|?Q9Vimz~NRtCzWoXKw-_Eg<7K?SgiR--OINJdjt0PAmW1!@C z{09XL~{+?%Vr6%1ei@xcuro;%+_U-~DO{9<~<`iP^d zdvuQQ5hRh_X^@4Ll_YmFkwl5;Ry!DfE|E_NR} zPZev6D_B?(TC^m`)?dfX$-aHp-F+^ z;mj~qSHl$d4o_DIzgVQ?BNt^50UI{-aI=t zvGJs#?NspiL&70S(PU?0W_F9@T4w02*L=B`-Ck!ko$WEoh+Rhg|0@h$P5ki5UPxwk zaCcnnTaVp7S9^RP(dI!zl0ISK*K`}&9>p4#vAiCY~x0sF-eFZ&K)nEAaE>zzt zoXvaqt2rK67`YzgZ447%qR~u*c5HXp*7XVZ0m-sr+K8C=APN6P(k-sj;%GV+XaXMY zuK`~(G#&Kh_=k63Y^Pcbt2ag`Wc)dnH@zT}J5&}LO3(|GLeQF&chLSU859|gDvT1e zj+BBLu)L4t%Zrh!OW%3h5l$7|Za;?R`%eiVJ^r8W&rXanrc7 z@WrSW2#B9@SzB8*2S1Kf{N3Crny2wN^Lzm0`UC%mw6_kby6gTx6_D;m5Tv`Lr39o~ z>F$sQ>6Gq}2065JOLrqFf^>tRfOJXTeZ=Q|-goZIo!`v;!*Mup&iU@W*Iw~ipS2c| zJmM|@8`iU-#nbaC)Kh~oiIy{aP0GFhTjQ;m(;zS4t^SO{yX$lrpGzLVBi2sTPG#|Q_BLvW@JH8%P z_RG_q>7V-ZsV{fZvB1Fx^Q(a_%YQZ&0)^io6zL-k-~%=r&D_<&|3^QK?Cw9P6$+?) z15VC1u7{w~Eh8i}HAx?_2A@+B0twOMMJy@VDf_SB!A zZ%iadUBw6-A$spQW!B8k0|-3Kcf}A);1z7t?YT$!f0)ADkv({DcgHf})bvjWlHH6) zZn*td{q6Zmped+B@&j@yhx0Y`&Q{(>S^^DLznIc`PvtU_{YR-y_S9t(@=3Ndv8VD;lR*gKBO?%N+7#9< zmr%4)lrFY5PP{iuNmiaU6Rk>V;uQzo8 zbW~ERG!Us0x@tT<`YziOK`>U@%E&QSXA=R+)(%_U(y+ge*KDD5ztK-&Mlml=+ysTLTa{{1{tuh+Ta8Vh3wzHA*UWRNvPT+}cxJJR81WTSVvvGBp*>OcTeMmStzw<5*b3wu-uk1Qd91-rtKLjUs5 z`DMX^7kJLdgS7(fzV1pU&<#@JgbS<^B}5#5cQ|t^o&fvP$x}86hrL(gr}X~*$N`{) zbnb0L zA5O~@Y6SH0_Jns%Vc>Co+p7`W&sfK;(15T#<_cbK? zu_B&>Cqz9{L-yI>K=1LTYtP{yq@qLk{pQ)4Xs7{V>(>4>eUb&c%D^ADYv%s*^pflxY!o z)-@3)SlATr*sECftg1D?O@mSMUh($+KH}Yx$)H&k%FU?&lJ8xHpec5*TFc2-P`YlM zcc!5I)hRIReDNPY-hRwG8KDc^H3F96QmA}PcT&V6-OG!C8~8c-m$YR0$G8vQ*y8c7 z=9rl0kGUBM1}9IyU(FryT9rUg_oP9;*~O+*za1hU0|T~6NGc29mw?6{_L{@<=8_8_ zec>@=!dkoLpn{jzWtWhZRjripG03<~Y$H19bXv>CK7H3TkbMFW%+8)Rrr0mjTRPybAX`;dIfM9 z-a+rvi-_+cucaIIMP@h??6AP1?8<0Ig70rw4%tJXUr1mojUUc__G4lYdb;7DFaL6R znWwX}vrxOf+#;{NU0769^u(qzV=B&mS;0UhSN z{oZxtE|({kRRU~mm|SlrSDa5Dldrb5>m>*8Qv6kK265mW%sJUG{Ll{vd$70zJ?2yD zr?B_#PzM5Zr)NBucCn~d+m4q*fqI0HpT2-wkY52qtPv3rU%0t7Y%0As$Xrb8gyGNK znX4@!982e69fb%o_C2}j(t{@y@@mK(VXiY?TL9?|ZKnEyEc?|iUWc{7nT=_EUvbc2 z_m`!DGmxV%Sm4xlEZEL}(IT1^s%B9r3AuHeU%Uy?x~1v40!q;O}u%3E_0r ztjNR=dA;CyDb=9Z<;^3YHc5T2c^aS_vWUw2*jcgb#^TeA&@)x+y{iGyubh8ZSMrb0 z9)Pd;ZAXs1$3GV9@f2o{=1Z|EBQ)-{Cag8kFSg$tCfXRw)R*fE#exNx&PxDd>zkM0RL`Lse`S>=i2{6(epT}Hf5h+EUt<#LCw{Nko~{b{>VczG{=t2; zr=fbi9CDs^);r39%)a*CULObD^nQ!{%AsO0J!DL`4lZQjcM4;uttsgL>yCu_9sjdt zIOv-9VgIvcs(-|lJ&?MyO(hIz2Hd<*(9Aa+^lAGg3KUVkQN0(K*@X}Ai*;|;A737V z%B&4bR|#}dfzVg^GGSsp(NBnhn+J3C$7HcX*Os!gX4{-}8EWDM%%kwC%hYZ*YLzssTA0lF45FF=X z?+y_sGpeHle+jQ91BbiO3|}QoC=Au_U_T{+=%HTYF*Bh4vWpF^In|n4MH@ z@;Q#($;@Oh;Smvsspvb=Amw7Y#zOwH&c}?&%)O!60S5;nJ@boNudjh zfrYa@{Z}eJ|NoQSuR zJ3HH;iP`m$9nv|VY1)|#C81>s`{EbW8Fs@@7Akk^PUQIj4R!F~9T>socVnClhQWh` zeG!60KKY3PI0X-V@@mC;aJT-P9~_KaWxM?ni5!FtAM_@!ijao1{63Ja5Gquw;cmMS zZ51?8R}U^~aXFRwWVc8m<0;)hMom%t0uB`_RF6X*fOU_t{hvhwqMrwV6yKbH6Q00Vh0tgL9Sj32F9x{ywq;c2@`S^QJh^L zcm;656`oX+f!K%EZNL$X9MoZ;y}!HxQ0eZ(j~xDn0SoOuA7`rQPrVwQXaO@r@# zd3DFA%9Qmi7N-H*o2epmb6I26J22==nM$99L-Fbmhw8=U2~pfrLl~j#rqjn+s`F<8 z@>t*^M#}$Zi!f?0F)4w014iBGHz=~e!o}^`+2-on1#(<&0r_j<-xQ0v025Dc4IHFD zdgm?lz@Fbge2wC-8 zUVP14CjaUBIhM)YapOjaOeTpXz&|L0Lg54N9#^CXEsPrC7v@H_6(uaJO6Wa>-piq4 z{55cEy1H$t(*^dl--(I8>{x-@9Lh*?8=dNS|#iM6MCu!|!Sz_r);e1hq9HhVaZ#H6+k;Ekvp2EC* zF%1=42SniJ+z!9D&&>T|1N9ztA3yu6r$0OezoK1He*Fj>B%{3&==%zu1R`#vcbd%d zAttPAq!fcJaGp!5RHG`k-GDqYo%azRtmFK*BnF&^=15>yMzQnG#uNX_AVeG3oV!KY zF7GX(R~Ro}RQ%B*AQ*Uuw4nLp-X&bacWNbH*aw;jFkzmKZUNvJN3RLJ8&poNC;e6~ zxOcZ#bb)OG(J@Z@)cBG(HNZCFGjNl#|xFf8F@$c5_+FYxenluTq;1QB1UAz|VAOs5F)- z90NT3u&mc0euIvnBW{Y(0@6pOt3d|AM>^MSZy1jE`Hq}_cwSqg;N14M_l-)qT6d99 zAv;!O$b-fARv661vTzB|Wdbj|E{$qIU1%aY51a!c*BddoNHGJzE&1`SB-w%ur#6${ z72U_jhb7amAn1z6Zs{dNz`etVz|0V+vL9gFz|fZXlfUkQ5HPX&WIHd^2%&B*T7!j< z_0b}~A>Hd1OubgGqLl9hT{mI6zI$O=B4V8aRN{EISUr^OcifLSKFarj=`L7L0iYBiTb_ZL$4&%PyYSKPtO0x zPd<@b$@IMy0aZc0?Zzn?a=Sh!Ka`U_KSH}RvVkieoda}i%5!H1#pY*SxA=^DExceF zC9})WhY`SGLRj{>Dq{@EZoBsfUPwX(GPn2X>qw}X=+k1=udg#pMCNt;k!6&&yJEVz z5Jaa}oj|di)KkltJl-M=+Y>>I(TqV_n;&MlGg+*K6@sz1b%sI8jlJk`vibilQi$9-2#r|C-dKCi{enAqRiW5Q!>n4TWX0n1-L)O^+ zI6m-dDNoT@61pm7tQ<#e=x(aLQc;9PmHTsvDV#p25;25trY!RrzJ2*wU`wZ59GMe& zgLeJ&T8Jei*kGhMai+nCd8yTtb_%$VFnH7gFTl~9J}`OQW&Y36bpAOS$bXJT!2tNQ z7GLv_cxpf|{S-bY8pURcqs5`#N}i22C6&sIG;2I|xb1lO{n$%Pr~}-Wh27T( z2UQC#@{@{w=&xcVeTkJtY;=352ob=#wFrZ~^V1nfW#j>+jJCxVV!NP4*TuylWd#wX z1Zr>8L+et(W*5Ij&Vz$aSg29D<%5kCyGUm7hObPAFMYHbbPBK^qr}VF;EVl9_h z*_2-n3-$z<#oMlrWP2isUdw1*pKykFd%CPHwKlt*9FPkkA78R)KKo>`h`&rD6IjSA zSB}~d+Pe5LudhCz?+?T8D&X|!~8RYkfL?edT)ezcD}od3hIo;6k*!A z6kpp*aN75-PE3PV!}2G~*8mXFV}j0~_c))rlhyktR%@c2zDvsYrH# zCy4H!Yeq^A>{*OKG=2cU{j`{B6y@JT@BJv#&sQt?E7JO;GzzKFNy2sU=9-Z;f;B5# zzDPWYEd32(@p0Y49Y(qgPhrYwUes?irO}-dWI$sS=*`zu&9s*waZhuUuL>Ou{}fBT zJe}09xxCFeWW1Gm7OdCmVF*;TQO#fH{LE!_>JrEzqu%xhC~kwZ1BmKO+I6T=#5_|` z-gS?w%|;$iR_JG{oBMmyEEN2~9-hLqLA{^0);3Ol9;nkoy!vX|1c8L?4`J5236*Yd z+|O6)H306E0yc2EGvK5~a=HEqPFvoP3n88Vx=RAk+RZS4!q_Mtx`=5&QTf|z_i?Yy zYrFHkMLLzjSi_G_XS@!zsBvC-<#Dtxe?PxUkGh|PxjJnt-^uI0EvoMreP zREp2-^YwmF9&CS?W!sm2@MyUYLjqUA*#@ip)@tQwJe`$|5_}J837d~SG`fOx% z3*!ByNt1F_EubG?e#!TQ$`O8J3U}{(kLLmNV+4Ps>0IR0noGm(sSGKIz#0)TrB20& z-q%X%6Qsw1LN^F=uT*h=BAnhnLfM?#iwg)dZ@yi?xq(Nox_lNoRbzr7mQM2`5moj- z>R_zq@=gul{+vx=vHjkQOhK|}Ra7)FWlB4W%Gt<++*n+ppb-Xg2{<{Lc27hb{_8+Kp&99CqtT{7gg6JSHB$`q>q*D2@&wj zok0?PQR|a!`3YyE*Xi?XrK0d1_^Pis^R5duP9gTMw{$yq?wnTZCaI)S8Ss$t8Bp7I z`P^hr=7HlgJCo)3GOo&`Q@;K31j$U|BnDaVI`Evs`HFBn zF~}s#7u-hzujGdBeyqX@cwfF;>@jLmip!a)b|RBNe-cQK)304xpjzqp3qIj4`@I_0 zm@&dLdE&`lh%UbyL@*ARQ^Vuv)-}TnHU85Y6IQV@A^I^_s8_m<2_)kBA?>P=WcEDb zpIn&&5e-3nflMk4K;C*wr0h5-Log)nvLI?#W5?dgIxV;iQ-@=5XxYgXN0%PBd;7D^ zWOxiy8{B8x<9NQqA3}KI>SYs|lvNs{EQncMO?$LUb@>n^uP9dAayJSy{=iK*3Az1* zKtf1&c4cdT$FQU3bp0caP}nRmr`y<^g(6}luRCLPFM|#4bno3SZJu<#v^c<3@O_Ai zLsDLu&Y2n?8@AprC0}zI6T5>CkShf7|@7$x+7Bm%q%M$nPV0 z{5iy~8vsTaob4i->&T~Zb~ZSyTl+`|D#rnwDof57^WhDqMvWO7G;5$gWy<<{G@S!LmBLh6Bvj%_vUd5@!XHkUU=Yj*F)q{2Q_s#wB9s)eDdoy2(V*2m`;aoqy@Tjd!Cm){s6wqFYA8Pv=A zCSFlFO_k;lc+!suwT@&bu6rr< z!j`*$d}AdhMbSCipAQf}-Wn_kbJ)!G+0H|{?qDus)61n9!Hi|(!KRf)K7;8N&0g$9 z*}}C!Az-En-^NQV?ujglXEQN~WgsZjYNaVwDIuRrqH{31_P%+nKq^dAdaTmod#9)D zyIb-4OPv_4I3yu-TQfX7dlJL@lICBGYxln}u0Hzi4Wpys)Mr^+f`?dR`Q!1upA>nM!-xTR9_F7`L@rvn3ZI)fsTd)z@IPS1_upRD$S z^hYuYKeT1#>6ZPVa;~Me!sRs0=+w}dL+}KyA>th zI?lOfPg*T58@k(3Y*p_%Rz9O)I1!y&mHIC4&HQCuox;N$PH{MhTExs=q>78~22^(mtN}$+PgrZJduJNPBOH*;Jf%jkz?s}=6bK3V= zj{vk1s{~+Rb4zh&rQviW4FKqkh>F?+fPA+CpwPuLg}ir{^f#Y0kazi@1OJ^!&@=5; z?+Oept@8TqSedB{?hMv}JWO{mS~tkj^?n997^!Q>`AF*5(p{+O_OwverFi-`x#SV` zBG;p~(kkRD>{e#URL(0rr`5)iNE667G2+tM_tp8AVU<6|RH4zGfOh0EQgf8Vsa3?9 zt@|o#>!ndg>?#PC>#6D#(oXp0GT3nvi`BNA^AT}2ccs(TcfE3S@^xk|46 zC_~T>N)0Ldvt-Xn1#IZo#U0#-TN7)i(iw)`(8ak!lqIAWaJ!AcSgO)R%x?ko~&|r-q9LK^v3YljV7ewNpxd7+wu)}GH&jis*ikQ8s5D_*Wg-;S`k%c6p!W9A$M;OwFuJiCnz!$ zL&#Cw-^&4=NS-H{lK|I~XH@Iw;wylodIt>%K|RZoFkXlmx_U}CmOGtV4(^oA(q7Us zo&pJfcoP@HJTpGVKq5ji3uMb|yLY{<*$&YZ$7PFf4}Y(@+eIDu{AZHREpt9mwzeZz z)rAVpy-6XJsKv|-ygKuZcyGV#V>aoJcmeYc$a=O`?HhmN#4WGqptDvFAX7XFVbOGl z)KfsArRj3r#%UX~e$Xo}0)#F(FOi%UgX?$m8~|!*56#asLyA~WfU5)IwzL3ss1wFG zW2uDqPUgEzxZbbtvmpu}M9xU~zH72X_}*RLl+1|%XCR3)Vj(C+GH)b_n^y{~+%>x9nQTk&2 zM`iAZzl+4QOEv0VzCj0F6-RGE9x2|syFt_fTTx$?z98*2Gzd`;Hk@?({tKtH3?DYh zj!{f4*?<3EA%+F1$Y{}e5R2@%SblgW$4t}ib1LAtF&tAeeT#o(b8ajJb?Deqz7%*>W5)WTfJNBp$_(ZB4dZakO?fK2(}TUGX4WKj zL-(07rR0dxmfke^H_`SHU95tat7XHm*a+F-UQhVBeKOb{XYkjmCepA=eIJz5qqvn2%9Dj@NyO{PM%Z{FMB>96+)Es)0e10PpwHN@0QeD?B}w=6^Fl? zZ`28+FynD}hQRcT-&v%yOFR}}6sw!1m?Zoja5X-6@+~5&CEBfs=ZoHAbm`6f{TlUW ztSyl(k7Ochp6~B1@~_#HY+uVx+eOqj6(27e@-Mz87D=`J9JK?o!Z? z#~2KhaJ#z|B;y8t(@dRB`pTz1!8uYP%s;s0b0D;y-bimPfzRXI$Yhd$1r=Jvl8q~F z{9B<1JhyG-DL*nKSutr9NW>u}*v+oClHE;Qc#BYLSK?UEiBmoMx(ZOBDW3!wK z2{zM;Fjf)JklcN<(=j)wjW!d3a}@&#^c~enGnP(ePUq~{jM~Z1TM<{i`rY8chue!n zFPKyv&|ThsvSw2z-WNj}c4#bdz>xaP97`W=l_(C28UBn-dCG$kp;Yq#O^4^}rRb8?aE3pe|_rdjx_ZLXa!S<13i==>`!4mU8s zCo8sE^-*5U_g|+~h~W06Y?&O~Oa$p|c0q#`wn!o#>Bt&ZZSFEGy_ba@6Xl_95vP6f z+CkbM9@Bq}mB36(rq_i5O*(rQo6jrl?#OS~{8b!!tAi{sH6&#igb0{)!pY{Hl{eV! zZcK*~;P@Z#`QE~~?9D~=D?%8yD+2+{a~?>*87H@sE)u+IXV5}x0(rbs*jhqSbt3F9c`lRe5tnV8S}D)heo>J=Fp@vJFmaL z1G&#!&0G(+-)liC8NSqN$bGkw~UzDg)Tj6{5&X^#(Nsonl65)Bm zujt)b4%nYb3xVNWx!R6+ygBmzf77@ONaq=ME+8H}MlC|1Fff4u86hY_dA7e?ks{F- zNyP4cWCG1cephhBg%zhs34nl>EGwYpYv;1BBrnbTZD`gSG%u6{U1#%|E~xzNUX{Vv*%{L|l=&qjQRT2-@9K9Hg9M{X+z_zk#^65 zuN&pYo-|9Z+)hmU`_X^QX~c*1BxHD=8#wlFQr$`6e|4~|oX?#P?^pAae)GYYv(2Zo)1-bp+J2ei)=eqKR*u8wKB; zzA^qDE-p}phlJD^spB4TkRyS%Z8V#S;&wThXmv2A8e3rVGhV%peGP32xQH;s=hbY& z^pk~ZnkiRaJyFE!gI(GprAwb?V>H?wP{xK5*kMIkGvJP6sW8C!?7P@SVPV@v^!3@r zW|y8G{eC)h15)H2m0UI04??*o{uS!0KJg#Rs*JvA3DKUR%0W2sivo!WH&!oCi7n<@ z&=fMbM~0rTdJOdEHFzRj_}lv~HaQo*rd<;!z9+BgSxh$ni9G-j^$CKeD>Y)yLYfp#)YaNfMt=*Xcvy&^3dtIsd>d}Ga%pTy=kZ%tUNhb+fRBx5 z=xbd7YCeXoVsWE0TLXtRt7W7QUyt~i&UGt|6H$9Xfy%#hFsD;;cpMIkCtAnw&m-Xw z5OQr75!tKSL5qb{%@DKa*4RkrPb3OiW$wp5tyD5_B~RQ!Q??+&uO`?t?%EMy=*Ste*Hm0t+;Z?+uH`YG?Gt zW5n;;*OdgY_lD|+aL<6#(pi)*MG-$;o$Hdc>*1_OUsW!P&!P~`yX<1EIMv2z2qkbJoyTs4Oi5{VL_enMCIBHO>un>$!pV`Tta1O@ zDgGUq$MpqL_4k1g^|-DUlGz(f$3rSRMJ!{eHmI7C9h(9qsj(8VI!ova_;_1KF281i zZ8PQ=Ns3+q7Guaw_A5M2n{i}32DLh^jMK5e<)bfl9hfKE)B2EnBESP^YctIFNiDAS zdA`eAaRGBXjGgnHmC4!e)T0ReVbz2UMkUOh`rXL|-?KcZf}db+49{1g0?5rH^h`yd zfuW-QP!;76N*1hV22h{51aqig1-9GoBWf@i26S^Zqx*#M)eFp9hys-CT#$L{vZ?&Z zwAU_kS}Qc`z6K@%i(}jYUWs{3@D|ibyQ*jKLTy+jgsD}TRtZ!qTiBHJ0P*zQBZ4tKpFJu=ht=X(9gLA0JC`xG(0o< zu<^G2NwmV_u){e_< zME;zXuObAaGTDf3E-~=d4}<1Cfnega*7I*WEQh;uo7JSER5Vm5CD| z+6&fYKej)+B6B)?T@UdZE{gBZ-BDD&6Y3*=iFF8X^?L|$xFz7KF4xUrVg&Y0JV-7C zHF1WpHmXwrSDF0{j}H5A9)7^AC&fE*iwK#=pfO!4kds&Ov&sg0WscZe0f#` z#dqa!PBPN>Qr0e7ia|k=^yH;FZCs#HfDTyB_JB z+b0`f8HE~kfs-oNCw2DAI6Q$KMyQWeP>@cA1WWqP#$9*$soWmQf1DkRuj8*aR|KdD zOL~fQ{f3s=dIU9;3lHJ(qygdgwJQi3Gt8v~F(2wv+JmIU$^J4K$R4R%U~U{=pUvCF zfr;(_pO(B%<}g>Fl8SvGH_>JL?&e?>7L?8jUjsNPCzr{^Y~WV6!Tuquo&~?6iT@3_ zexd$1{*UB&b_e_Oc>${32Bo;Qwf992#|BC?Kjk2)50H-*g{a& z;9ovhqgDeMAt547pt&K4KM;|T*Zxp)L2Bsq(psx#=zYouo6rK8^^oKWy|58Ubh5}> z5v-iE4~@ucM~PNOKIv*BY_6lI?B(jKx0h(?$_08j(b-$9!iHTJ${@!J5%wh?=3^8> z6!pbF+WZ{EQGhPptKjPq;QZC(=izm)?D(1gqqhFT+^d`ql3}leR|dv$j9Pj933oHq zF>oO<*woTPf+Ok`T1doG71eWFAomclLC>+CaUfTfCAsReS=#8ZbkF?qw4BLf* zt{n8bU^GW*{8`If_6@&MsfHosW2gf(R^qkFca@cHs5r-?xSE7sUYfPJ6Tm8&sj?|K zutPzAG81yXzdXe&=(kL(iOEBnA-nD%rd zthPYvgRUSAo=;7Na%_M3NIi93D{6R2=7j&WGR|>bSGhCnWz$C3 z!PzhEC9y9Z**|8dT~^mRZ#t4k?wQIX>QN&k{D4VSvW}GTjpR7ulsdf<5 z;o#9<^1U%LM5I2~7FMXOWq_cs6scDsO+H z!U6)Dfqbd4$?hs-LOP<}s-RY{Vf3ogbrmMMSR$&K?Jr?!v|AL(K-G z)SyVBuert{q4P8vT9(xAb5t3dFx zXu#grVtJ+p)e>WZ@E}ptS%lLcm>xO0wlK>v({6{Mr*(k&xwKfdfOL~t&&-AMg*cna4h17wNPm81y^nb^piQ^D0FR{EVknD%r-!DNrinH5*R3uv(-^ zVA53^{Ca%?tUe+XRvR(J;rKQh&BQl6blWVI!QbPusjfq*+&;P)Um;BOse z;Q|G>2{%K!0~lfKzPDcED0iUc+6d!pYP^8EBWPBKVKMO$4HOVZ5waW2%)+VyYGsEy zG@esepK5dm@f@GexsPg(r)UH`I_dgoTVH1tbM;us77r0ZmDP2jd`rggU5nwmR2-HRsjW?6D4sDCHItOL1*;Lb~TUf&Y!ivLI3B zqaHP|#(f#11*lF%jPP^ec{$%-l3TP-N>nWG<>|H@dm|I%y9Od;v zHY~gSi+#ERHlHq&lO5rN>G^t36u{7B`S{Iy^uh-Fx$|}0bS-X?PCmEG?i?fj4k$Fk zb|h*%!ml!E8u`86hbp$OCp@o{$*>&)Bmz9EL_HugO6Yg~Fc$`_<@(Lcz-4(wQsH1z zS$xCUui|MSl5|94P%SH`V~piTuw8EYe0+)VIGohm-|cy^)O^xNrR^{MmbCxUKgCir zzqAWg>Zl+>-Z^xsXIIvomP@bV3K9?g$UtkYx!pcm#rDk`Pg`QOeW)j>rO5Cd92V3C zMB)ZP+2{AvGY#x(Nf}4qhjIb;sOfT(@*kph@Y92fL*oy^zIul-?i=Pj4*@8eI<^|Y z+Sb9-RaTk$%v2Px`Ec)2>@%U2LoZ)MfOXauDv|!b5_EyVI&kxB$sq|vuhutyD|s=y zAv-%Od6pG{Y!yB$&y(_(TuUePIP7ngeD$Jc-9YG4DQv$X(&}y#v$B9gdLGsB?(l7J z?j>f&H_~(BZzVyx{B?(>6{-boD~&sf5Es=RBKY#^cbDU=|0c64|2LUc3TQh0d`@^h z-x;TS*wc7yLZi~_R==018)-4aP3PL*O@~c zUs0E$cFt!viu%d+TO<$^%delf)JnDf2ui#2yn;%J^78wi+*=sC%f#IwXrCH|R44z^ zlOt`RceKrae9Q-cw#Fl=e(mnLmu73W%*w5oYF`1snZ3i>Y7LQUhX@<*?Fzejm$B!G zx@UcemeE$jqCob=bNYuLxaPI`7ltz zGL%n@QY65R_thwvei}ETuv7{~^FI}XEv)U=@4aX6JWjWRK=+hzo4H!65CC$`e_L8i zysy6y#tIbz)sLgVdzP@UpaZOecQB8TQ!KS}FwoNIfh~&(_Af5yH8%cV|!jXycw9Ny6VKA1t^iIHB;vH1v} z!rS9*_cdRQn`PbZEw9%RlcFq^Ik&CgALG{dl_+{o2#)Ys7MYn`IukSB*ZeU*j^y&A zPbCziZ1#z0lQBL}=k`AP|4x<62yj56C3v7nx*5IB8z~hd$J%peOM4e7_nkiKJlSNm zL2T}?in#zb7ZAMXB=%1ng?cU61t!Zbes>Ul>PjA{=d$MtNgN^|uk;DD*A+tWTtDI~ zv12swZcIB3LP?J0@eM5Sn!3X@WDZ8O!d4{|G^2n==MD>tCRqrg5JwIl8O>}D-A74S z(UnW(L|<%j*|3z2JHjhA-oFhns$vYfl25PgbnGN%M$vp`4#<~VJq?$#P&1}*Fx4Zf zv`k9K<%n3319{!}EdNWnu9cpx6;*++3mn|VyjI5cPiT4_6g8_K`rUhI{bf#RF&ulJBfUDn)9rvTCK=R_nukWtRr@3?dDbArB;0Im2wSIZEi`o2-m#U z$-aqFXfe;l^2hhVk|p@^Y`&1s-3+Ws;?G^Sqi3B6ju!q#X-1b&qHkA^rJH62sV=+E zf)v!R8G;b_E;mr)gYzQeb0ndbQQy$roo&Q2g~MvCRZ?+N-e@@JLPUv1KA$}uI>gTF zFkhv%!B69|^y`h-3-wl;{2Q(*F%py{a%{v!o2h zvG%6F{_fd(8knAYw!b*C^T~PwAE#h+1ccmr99Dw2O!@#hDli9M@o=n3af6qJU@_&($P{Hg5qYFH}z{K7s<-(CVW)Tuc)f zp%~Gw>)iOMjAEO_^dUyS$>r6jUrV_>@Ms(aQ@k|}J4mZ-SiEdsM(y9$+us2$BAtG5 z;hH<0Zk@oC^-Ruy6Za}Ata<;(vr8yQc%L#sNdBLci3U(dQ|`6fr^jZfr~F2XuZ_7sEd>PKGsxCE?0;WpSkF)_(w`-gm$Qq{ObhbOa6>=i1Xjb`f2 zOLBBF+yLzL6~GpENW}R}8rXXw!FnPuAYCoLa~{=vY0(UHrG^xRQ^_Zhj_J4HIz7SL ztLZx;Kq)=VeGGXoBnm`Hs5%vU+7N0OW~Dr!e0(5oRr z6VR$q4xxisA#L9Z4jx53ZDeoZ>c{yE4I-xx)!t(wRMKfGdR&7IK)_x$0K7pn{plfTV7Mxo}b2Cm5a z6KG&?W6^xl^xNFZ`xqnym)?fpw8iTX#ya3^j2b15)=p?w$SvV~ zXobRP6&=QF^I_2m{4%$>Bt%b+L?r9fEmmjy;CD@|GFPPkx=BC&*G;m;uF^>c^EmnXzM zP7;5R0zvn=xKJLAWM)=3Q$hQb^kyNqM{iUrx+b;dIM1)6zp zX9s!R22OZ}iq@~yYk#Wz+* z(-1=p`bZ-|PBjs?=V5at7u+;ScYG4gN0t5U3xgWfRLHW z=J607zyJUTvwSD_?YKW;<6DvO)n5k%p|sa~%t6aEcj1Ns2vX9AuJgyqc2AYpdIZK^ zZ7ift6e=~P|C+6ZMcbYWrUCJhvd zGWXg^sOI%PNlMhH{Yjv`FepXCWsXI#yS+(HWef)X9`=*ww_^KQ??MS!Aqi!_-Om*W zZ*^KQWrKXne63?3-T}kgeBB+T9D@gZ|6&BA|67VPOF?74M!bopi|p_S%PWZ~HIM3U zGTz=d>?y;W*(TWfveXs!w}&}=0#8&oZg1@K#(u<%bq$3xnc!`Do*vB>rfD*t!|$B> z6ShV_l}Q%$5iw|YJvSR!zDlug(+PUR>psXiCY(Ghwk|K5D&fNR^q9y+nhrOp4R^ z+V=x#H;1&KKqTyyh171TOreaFy0akN#>XHGbU)gtR*M>ZD+f*Y0=`_|WYKH!!J=3B zT>vb1mRoR@2VvHJd0E9Ms06swob?4FmGqYWf@^R1*%`m}M_ORSf1)BU5w#`*F` z1N*zSaBKhx-scErOcVhlXYL!r8VKyY_J$vhZJ%l@wV2RD%oTfH*KwqiVC{hbF-d(w zZMCgnD&=gKxgwEK=Dy173>EljzxP3Nk^8dr@eIGDdhakK_r+wXFSO9$&kvDT4CwID zFS&K&Py{6R^k~QNNX}Zn-V9IVi-XpL(B@CN5MPCj0rG1cA!B6XQ@LTsyCuQh<$B`& z7}AHHP8y`20h%Y+pXwAmB2*!UgGHoN_+&et@6!?z|@x^e8MG zDqj`!UJdUOi*NDgAg+gC?{RpZ4!@h-LX}iRJWq`}HXnPL30c%s!Fu&khB2M z-$;*+kztObcGSs1+lhsfg9C&@&1Uk^4}xRI$oLm}Cv{^jk3Yj+x$9;r(SH#9e<*tk zpeod;ZB!5iM7op?L3)dH2uLa*3L;&C5&{B}TN*`L1nCaxlm;mg0ZA!QK)R&6HgVS$ z&-efTy)*aDopI)z@py3W_g(LbXFcm#W}7%2%_z<&1la{swclIwh_shoQlpfIQP`GE z#{{&W9(UKu)gBVgc1-kSY=!Y#kR&-D?X#De4-yQW)_>d5YQ8!tVL6`X?5P}g^OfA! z#^+I2B&--t;xY6NTcc7v472Km z$ZIptJ^h&)RpclmsYi}IPKLdG<>T!k6i>pg1C$r4B(ALQ|8 z1KD@jV5J(Zko(oRUUpit&UnGH0V&WynQuIDKK=G|W}vt$>1u_3Nh|TBglp2ahwhE| zWIohj($CVpW-75SQZMBf`fq|WL<6h^%apmZAwMtj{-63!&q4nwYj)pjRu}KT>2!{j z;qosqVO8*G&*ds)25%ECUc-cJ@7N#~ItHIb0{^Vh9AJH0s!zE1_21yDDtIsF2%7Ss zT`&#$6?zPPOqab?TG2{NZX81&ePhGf+`*3y8GT#COZ&r-9XMpQt9&+xyKijj4=#Fk zYI`j_g{}hxY(WYu&@5E4g^uqP09jQ!CX>+8U7=($=dJ(p2$n|fsA={ZXJoXe=` z7{EC8;g7EcCQlN@oF}7?UEPSDBi~88O=@JrG^mQ%&mQ#D;ZS^Tq*rFxIDdUtz1?jD*;;wExT zMmNLvq!Y_jGb-OBznUrGmir}IU-wPwp(6H8?WVZDg-(rqFlA42`U~Ga)2&tJ(dWZi zrwn`)yQWqBPSxB1;cBB<3M##1m4nj~mXiSjPIIG8{G5T>=ueNH)^I%Q;r_9xOaO{s zhgJ4#n99k<)!M2+l(QD0NN~}l8?@Sn0=mLheKEi@jK*inJh2?`ym0X;ap)ph-_oBi z&vMlbIf1c(?S(a)+{H^|y;HTqJm zs)sdIlb}=3O!%c@RnPJhZ`$1L!|J?=d#4pK6?;neCoiqSm~L0x|7X-kDk<~O4-GaQ zGs-ADZqD9F=(z9Gx^3~$bJjwa87*+qKA^pE@&)_j-P$TUVvjYdDO~9n3O;V#No1>h zx`Z^s!E@XrldE+o)wm>%r!!tAZYO&P1 za*vO|qy#5DpA*SN0;(3d4X*o~Wi}(1PLKEKDG(tS?#b)ma?^;`J$ zRv4OGpEz+(5Jh6@3R>UT^643-WK+KqU$NHfwh?i>qF3SaSg$7A4VPcLWM<*}!Sz&Y z;wNn}#HozY&D4gcF%sl!;|1lm;HV?lmsQW>N=e8R$(qPZs`mWwNPvn+i!B zDm1_#T|wyAo=fI87=B_6R;XPKJX^>4EB#r@&EIby{~`*!@nRWEZa7M|G|HO)^v60D zy@D^w%D_RX@L+M6dDN`ncU?=FnN7WXVQ24qNF3HrJKjtT2L6f6TYt5G8WxdLquz5( zCse)2lW~Sd#Nk{XY|~2!*K2}~Sel>9?oAz!Nv_hz)2wZgV)td8YeNP;MR1eJMt!8w zR?kn7z>bX0liLd14I`i3`TD|9zL_n|B!E5;VeF_)t*!o}CBiZg_p7vY7)8kpTA6A| zInfk3W0$O&@;Bl2XCR7%WP|koNH)+!(6`~~PQ6L!WaF4Q#r0e#>ZA!-D);y}_ThDI zI`uoGk2P6KUqvLl?#ctX_QNupWHeU-qW+rg+A3|3W2{-$n~*r;iQ+JSTCs|+FV#Nh zk7t=kjVGz8ZO=8=UjIdcZVp1ew2`~~)caWB{$`nITK%Ep&oa=MKH>w-u;J!;NEiFk)mYK}|W?r51sX^{M&Gj9R-Wa~rivo6H~D7`o{e+f48jhe7&cyYe)P2u55mcJwOX+N#ILd|EGa>*srMMC%nuD z8XWm?h3Ao@dqX>i@9c)LHYvFRO6Pg}eG4hx-@hV6a?3SF{6`1(yHLv_+tZEd>pm$G zc$7|gw^sRz{y^EkIr$HH+$_5lcIe$z?DSiH?8(Zar0ORP$E1quOv$C!wUq?xT{cMc zJq|A{30Zv>yS-jWV2I}_q+X($(vV z;S@y%4HQckc2>Mqnf+pRID39puAO_iOXn3tDZ_7rZ{%o#b1Fp6t#ikP?pcb;3RT;+ z(KM;gF%YwDIvxAd2Z50?o%`_VMT)_JB`>2Eg~h& zk8Ev~1KpBlJh#7lhEXa2HN#*JZ>F86p6eU695JBV1aO+cuTXoNajLj`zdXFefZqP5 z$}-sEJ*(0FJ8!qbZdo3^YH#(N{usweo{`z}BFv*%x-Vw0F;6Yta@{q)w}9UzK;dZBd8Sz%UcfS!C{tX@62-aTp4$$YEuG?vs<%^iDYw zx9+Kb@3aTG!CW9GMc$rQNZEWBtC49#%;WzUY!oQEC?E0e%>j!j@L}@afBQ5HK3?2o zyDSN;Hs=`I!lF-IdQ!~%>$!&~xH54(+wMVQIw)uMLRr|VdDAe#jp`MzE zdul=x>zALW$uoUep105C(q`PF8mP`5cJt}88~629_19LYl(H!eSyQkl>$&|X`>4!m zjm15zP?KNTCf9wCW)w0r&+A`49KYjtWv&8`kCcyDk!&^$S1##+K3XY`Pkxb~bWv>U z^x9H?KXCM*(%#g>(m`VF-{fup4_2K{9@=oo~P#r}qOM zrhUJ4+Xw4TG!K4z6gw{K=nHo>`*luVUGwJHIINVb9d;nzOa>8rWZqOl(W%C9M`&}U1*qYUYSS=%KF&-vYg zg}zCJ;#)=v=TX#EPQRrt(@Bt6^koK>P_yP*_1E%(o^3Yfl56&a_-v$YI@u=yjZ0uu z+I3yjKT<8{ca2>ARDEBMdvK1P)L@~U)|%FQvOL*kSw^E5tC!w9$no6$v&bvXI2yeS zlSXO{%l+so0AMW8_BJZAHGKA`PyjxO?9W@tFNs$Xx-~eG&VP&q?*a;eFc4SxTEb$t z)iQ4CZvK=Ep?zsd&{~dba`|B$w$+DQHiANYsIQkF7P3>QYX6Uu)&(bhZPbj64S-WA zEQT?OD9D;8-$WC|+ZOYE`G>0t4rVEyS(p^o47@)?)@?fg^}*OiFoJT=ls`ajjS<5J zZ!dbp#f34Ibk~z?a@KxwJuM;8kCK|10P*3s=c8!pig(N%5prKY4mLR=k+0o1X|y%N zezmKm>Ln8MD$ZyH{)toM+WGp-|0yv!nZB<1U)4xZvE|3zyGriv>)e_}pIym^#%0{aLgZ@K2ms~|Qm;XHc zS!XZigd=?d#bwNKVx9ogC+_mQTTu&rE0mw@!l1}ve4DoeYnx%z zbDe>P1<$kSAUv|5>A1z)vDjpULPB+y$zvz3T(wLK*X^XC$LIQ~{?8g@`c%b6VUzVR zb4zffjZM%B+Id{oShg%J`(N%O>gd!6smG=k@sS@75>jk@s@I?vAGD%dVHE(&bo}k= z+#VYg4;6)0eRGG^n+qeAMFz^uVgOQo=f!ZXT~X?1_%9D_r1A#Z;HzU3?>YvcjWb&g z=5WP9$Hs$S+dM9F37w(tDCa1gL0k-&N!KRsO`d%5tDrZyqddU0$NGZ+@%sE>ohuiO z0S@*|-)h7EID`+)o&F?p$S2Za*VUux_1dO`tN#h%f-^YOAuRc)#HfeHCkiL;)SZ(j zTdf4~IMA~l(yIrOt_dtik9)e=DS3KCs+?fL4($b6&*Pow4CClIv=CGp_jF6h`5;Kf z=ra@}U;MdJKLGV2Xx(<6M{51#-jM$R6Uv`uBhlJxYvC+1i~JV8o)*p2+7B~SUIAP? z%x*&WdYsbW%1>|9-s+U^gadOe~iu*nMbDFluce8&kuc+*@PhpCRGFJN=mu ze(hMLik5-9J%V#BirVN5wtlx?efY4af^TG@!f0Moug7xvXxS|b`xee#yijf^i~w6m zm&;Z4Jb%m+E-7Tw^LiGl6xm;!e!k;6*7+QS zU+H{NFX~?ggEnhup0pq_hC8VdLL#qj1se-1#eZEWxQz0%_|p8h_t-%c+*9uvi%>Kd z&;u&P(ng>Id_~kDTkBQ8ZZyL1c?J*|DWOOX<>9=y-Nyo~iGMz=r(zm`1h;9?prd5V{- zo@4wmfxLJZCXOGt7)A;it!PWG5_Eo<2jJ2NL~k}yK?uVa2oPKRa%fqjq8VaZweCxM z>F1f_n9)DJ(ExLdVio(eoP>6yf~?Eb2et7&Gu3R5$Bn82Ud8TGh@`3Vv~|m|=!F$g zCyM^5)qJa<3eRMNz?%12T~hHCD=xYe&^^HQ8`j3>v}>LQ`C#`J_=|cR8T0(Tl5pGV zR$7Cvk^%R=r4}sD<^F%^fL2X^kKQ~E>He~bm^k&$HCup zYej|)Z$TB&1T@9ntk_@duRkW^)KFIJ%R+1EK9Wki4Js&ql_el_xFmQ9`JzCh>Gn4t zZY@$q5!UJHROPVbXodc6`)EC)F}WM%>asP5 z5A)iguqrmqID^BV_!T}Y^@BUVL1u6ake{z*9uh|}S=E{w>@XnF=V0z}d+nTqB1FL>V`Sg5FpPVqHyXs}Aw zR@u)Jj1_+g`f&H1mT>JIcClKcm1d1hL?8c*KF8K;lE0$eR5_qw9V(TRRRrmI|Fd`8 z|1wh!%zB1Vv$0Oyis1E=ZBv-;bK1?c z5tC6itEesf{v-rqz6cimlLMeZqABX6*HxOK$`U|HA~&yHB4bO@mPbporjI{_M%_Z? z9KQ)XoYDJ@@lD501#!kQ>r{5(pc(o3Bt6UbcrM5Zo>S2=9tOT%-Om;J8WoIr0lrBZAI6+@ z1IWFNcARJ$#&+vMZ-d1keD7G=_fkMN<_EVkh^P%sb5VIUMFRd>>Dt13oS*2{z7uBp z;(xiDgM1@(ll#WHapKYXRK&ucR=UX(QMmhp#dYKECYGj)buWSpl;uow2oVHwbOvp@ z;-KCX+m|tw_hf1EYx43slaj{9_sMb3@t+PZlWX*LE-*RWLx5-rt~g#do=p47#;K=z z)tkXsW3&3MIT^A^c@~>|Q}|U1sD3A__G*HS*Z+8x&v*uC4H2zhm^^`iARGE87-vKh z4Opjg?6Ujcv-DZ8%eyLBa?}L)|%Qd+;e=o0bA+zvmSF>=g zZ_mRM?~Wnq-jB?Wj@NxE*{ijmE7n_iH|LLvuY8^^j@-PmXeB>h>-b)g9H9sjjY0#T zy5C+EiyAIRN%McdXZUSlQ<3vPPgiB@T#I=0?W!I!TUh^t#fjlT6g2*`l@MjAy8Dy@ zLAq+M`?mfRRx0EY@6&pocBC`HmKvgbEQV3SACaLi!R07)VV5&Zgr9qj;^|E}TbptY zi|cmdc+>Ht29EBaH(@Fls2Mr{3gl)*ZwVxFszy zAan`(_6AnJY7(bFFnpvt=$@`sH=bG3F^uAR)ub|GOm=3*jMXFH*17|PM{&vnhRzBD0c!f1}B$syI><7z%?AtK$ z;;7w2PD3p4xaoBozT^Y70v&;{J$au~Q;We|a_gVPw=YNEe5%Ou($W!FZIra)E)OE? zeYRi=#9$qfU>)65tYl@DP z7z6?BS^b?s;yac#63CWVbTK5pH=CCo8IRYI1^gm2(l4Evjf!y3ZPma%yJZQ=3i=&W@fQDjwOL zJ76x+Mhv)^Nkm=%itCu;p%)CnuQ}$4`;!u42t`pxgLkb>?h5l?=_BOq44~e1o~t#_ z5Jo(q_fxGx6oK|_;8vQy<-A-v>%9QN4^QUh9)CrC4%St0uV$Mp$!?#(oMoS?rGa4}DQjKCVuwqdxW0|V)`g`_=$j^G@Y9)K& z{kyk3vzfLRpcvp#MS1E|@OUMQJNt%noYD>Oz+0*Nh@8!iX`o1j^xk_~0qvXQoPK zYK7NZiQB_u5cw4%h;R{ML>UhGqRZ~>10{Qd8N>rGj*sRF&x~wLM;O)UksWPsFNt(n z2O}wd7ZW8FrONoJLBt;{ii@&7t@YQfw?`l?8 zlES0hUt{Om{mQW;D!Vke9_o2nA0u6<#FxdwgzCe}jbp>hYA> zi#Uh;`E0m9*O_%C+%C{9Ct6YPI6d#aA?F%qM$sA}L{*n8)YmPtHTQ63*wO0C9o9Q{ z%Nx3C+xprf)C!(BAT#br3`O;e$ME(%C0gaXpojJc6A5sF-PI{*%jW}&aBbn#(vhj6 z=i&@OzD$JwW}py=@o)WGV-eO7%czhjBC{K=ckoNI_o2sEq-u^bF0XU?mUQ0tBW(Hk zuU)9nfgxdemQl$9VZ1WHn}d0)5eHRY-z5d@R9|8lR`rFma-1j|okFRSAGg|#^S=DS>$0*s#(9sn!su(NEv}l-|59blLdu%ZHMv`pcWxc;?}h$>jBRlsMQHPK zs^+vz)fySWrLzyGd=ISmAtE#*7>{hyW3J{+@* zQVQ{3J7OR8+?cu`VA1z_pZoFVqauS_hj#<|77MR*#F|Tj#C4hDk_L<F9PpHE!=rL4S?1hcTvYx=`Vhs zoQL*kMaG9?qaw1u%C7R|JpLkIvGGCV?%i_k*C@=`5nS0(XKHdPScP2s8ZH)r@%kAo z46DSEs#>GSOtHk8^!Uy#!n(|jwt}E*EwByj?>e!cj1z>D^?dVRJF6jO5OE(On7Bob z_6C$ir#YCx(4^;hHhE54|bEooiGUjcUF|J`$#x|0E0*b#h2F+|&H zZClO`Qx`IoJbQOIB+7dJuuW7qq;M>$H~3=S>e$DCo%gIt^4ELxdka!u)>{hCxNq-` zM;z8U%YBme4eG18rz;CLdOF?y$YL^$@sx`P}l{LFy&nGYb!EktUTt6J?v zi*ndtO(X=p%n&YV`Jse<*?N|p_OYJ`kuP*tbgFC_3-mu}bOCCh=I*ba$M;3(HOf@$ z!q0P<-nY*+9XDzzbYfx|$Ack!tiznE1t&SDN2&D`18!f}D9nfnol3wj+1rhfW2J8y z!QzPd2G{P%U+*^5FuNLQdusC)8P+mUtCc5=n2~W$w>5M!Fu(Hj`b9BB zc(tv8|Lz;vA8d(~Pn->!oox-&pU(SEr9N0Gr{*5%t_;X89b8ykxvC%Ar{JXCW)XKoqUbAgx^JX5MFLygRs^(yO21(<+*qd^dwII2b0)@w7 zyD+I$lY0_h&>mo@cl)zOupIPShJfwUEsHw3(QMOjEDw4lrT9~RKw>&RceU~?%;C6T zo{+4bRy-vgEIj!*aj>#yl4DmIRkLL(@o`7J%CJnzK-6)nN^}(#AgTzho^Y>z#@%rn zvTbu+StM@{V|it+S^Z2P?h`Dm5Ce|v+CB?DttlP{b22i>l`UZA?szH`mbudrMGwL} zJ}PWbR#sjwbNeKXE9{lNh znhmf2>t*4JD~(HE}4nrzu&Q7AB1%_Z7&T&Vx4uPuzZ`qSZB4{V;v zvnX}L%C$Z1@zY#byY3l#fp7VocM3zX)xk}wgT$4+->9|ujNbHCtgp4acZqJ0dT-yA z>3X74X5|f1xj!-MNnfKd8k%ELiqNisv*Zfp-v& zFwkqTw5Nmgu_0F&@d6ya{a=S~L$=z{jb|+vtP)JdlR%kP$Ed3K!XV;70wuLWH=gb^ z)nIO4?jw~1%31PxUlNS)L`s?s)rv&d+=Dhk_qw`|A48Y3-}Q?p@e)o-Jmej^Fj-&y zXw3%L94roqUBT;k}_}ZU?gbFci7b15WVK%>N zxZU@T&BL5oN9hHP$;we_b;B(#a|MZvS@+*d*M?+kZ$sH}=9d_a4Dkn1R}(xR(XD!5jKtSfGh~R6o;T=mUWM)_lWLiPIx{Ku4aEtpUX- zrk#yYVe35ca3oZtkIItSAF>v-*^EODfG_YI>u(7~)QEM&I7+|w#hKqlivG6UN%(Ap zh>}KFsB*0|u>yfAG-G8|L*mMF)*3%EC94GpA|$O2kR7JmTRzokC+i!05u*2W{+htA z-2&b0_SRXsdYe=4fpUfWe@%e*8GpG2s(M)dM2MNv+^ns{wm~jWkl%FKMb~wQuO#({ z0mUjMLe)o*!=^T*)~Z1Ga<7lln9%3GN=ausr16VJ6Af= z;GioPnm#Z8ynb%dW4Q=}>%R&ihG;#**7L}`Zp)ZX3~^II6k%0VS*~qkZ!kdd@bSDH zJ){J*Rati|>A9CI}cqe4S38>#-rbc{805iCZ-t7s&L9#v&zx${m+{|D>>2?eN9 z3Y&oDJNFhe2(mSc3~FvtFxt7+qZ5d&*#@)WN}K!Bts_{bdM%oXqK@X^F35<9CmDu8 z4^qVXSBac^o_9gHrzGR(9>Qi6|Lm>Yz}<(V>`L+Ae>qTTfKj!!Zl_33auAh#+ugM{ zhF(7Kqmb3_+w?P>44aDwg_G}`|D^d2)ZUfpH9cx(A4pL(2@{I9HRM~)+rqhsaa_LsAdi0Jhs|+{jqH9^Fu}XXic4O^TRLVuy4S%Ho0dSh2S;0TjrDI zWBpCw)>*w9A)$MZ({hOhVmo%XjxnnT>$%DLWbbLGGem%4;wff5*{bK3qa@Wd%ly|d z8Nqp9@I~3eh!(}}pq<<;riK?%MUrWn_Pc2;fC>11MsCDT)xL9$fkRU2+HxF0@_Hpw zy2J3_QSfU&zlT~{F*;OKRJnonm+x^U`r3~7bzo1QCF*+Uo49a=%vG7;9n-3$%eof{ zi{iB_%fsbLA#Dh}P+=yZ%Cxdu54ZW}(|lfp%Kce`5$URBv8dDX9mvdkOBS6$Rr?R< zPB_eW)2b!`@bajv$87AQai897d#%-~LEYliX|-#sMQkE>rEmYJF5 zvneS4GGFi~OPjBZi~Hc|g-CAttKQ@JP3~Pr)+$G9rRA1CzVF+a`~ixjPWhp{Plsk9 z$*F4y;y^i>uclTFaZ0VNVsB`Y)B)i6r5{!BI9fzME1ubz4GDa^cz)ifYKWaSg*#pX z6ijf|z7%+wkawelmlr9f2KnlJD4^=_yMt8pKXGDjA2PobFhkrFMNo_+QobPMyR;Y5 z;S`uiX{j%a;4w^nvi5PSNstryoQeuEB{g)WRA0m!>W@7vJ6>fWaa*W#kp=sFW0AIKe^Noe_^#1Fa?#rh$sMpRuPy&HZlwJToeAq16>M;={JbZL%Ookb5RFOqdX zThHz6$?dL`69Y!`;HuMB?_>1}byZ%c6HH(8-*eVXMM-OleziwV8CF;O`&M)-2Lz8M z<^uRA**YCXZ zk%u#$zjtbJqV!U`RMhQTk0RN6+fXnCk6J|iL3jDqkGRs;gA5aN7>1YjT8p=)EZ2k) z!NhwZ-9c^_F6DOPY5v7br=8H1tPOmEXGNPE5W5E8&VCf8m(2b|K8}wR7sA;K@Zp|1 z;k29j&beW&nmvjN@zVJB$czXuPj6Mt^}Rc-dmlB{Mt=2f-%}ZNFE42ZP(1+*FT;5Z zsZd#1NoJ=38w3O7jyKDlR&+RQLW|=-cJJl(LYNM1XOsKI?*+O-hpV5~o~S=N{Ovsc zNDc+VzE0U&&KR#zC#-K#XB*|Qn9x}X?6*>JUOUS9l2ym8p_xt74*K`eRD&pWGNEUE zJ~ew9V+qe1!ddU-lD^%FeI6Sww-EXYyNs30?hrHMPMBI(o>_h0=e4y@P7g9l=mRRs z7YEXF_)G)DujjLl&i}Co@+`1LY+%tad zuG{`@mapnF%3nZbVOeLyefA85(8(o!;FT{45MG4lYZ^K5(4GM^2#b_kX#!?5nu#-h z*G;vADE*PS`2Z+??69yPqK(LR<>@rn0aBv;`Isz!M;v#A)AB90P49>+8(lN-6&`z5&&x6uL@RrNERDmO}yZqm3YQ2k;8+rwYCabEvCZfTxk$>l3ZUTi!ClKQr((TrhCg7H`sP0*nOeO@hJ- z6c~tJqfU>u2z^ae6bMfbmAItpdZ>D#N0>usx~aX^MJ<%=7{prUxW(Gb>S|6fT)vj# zM8?(L#WZSI@`Tr2qln;9zIs5Mkj-6s&rRH~la+5uGWtI6oVcU1pGCF@ww*?a`k?_B zn6goTn)>0E+Qaf>o}#jM=MvzeQbM z<+M~4B31fkBl>cU@B;trhk`RdF$od7K<^!MX-~iy=95#Al79UoZ-#O#cZLDMAX>F_ z#TfrA2aN-FCxaQ=$yyhs0FG}%1@TZ~9?270chEDq#MsgZo?B;^IA>Kao{26qTSU7hJ&t1b?_F}+h&R}mEHwmGPBIFL%($ym zREosJ#HK>Om{Cx0;80TiEBnrYC7w3|?A90Vcl^O??X|kSq=zp@Dy-9Cx6k4t39IrQ z!09y7zZq7)kSnl<9_9XYsKw6WV9iP4&7(Do6X$ZVXzkZ1=$dDN-cB{yilu#yb}1{j zRw~5Mb1*3Jxau3tVEun=FdbnRi;_)saW<`H?syT z)beZsMRRLKp25EGH2mv8HIdnxLl8Z!IberAf(1*meTM~IBRo|=^h6Muk2*-LCt`%vq)0n ziE`BT%mtGsDR1a(Dm6nEzvVd`! z{k8Bl3KoxMK&)-oPs{|igEGyx)14thXB`_Z{#Ul+fE&Zk*G1}!raejbK!UbEw#(*f zyric`S+j19cUml9D{&LwyKZ9DI?ja{3A|Mzn5<_ejle>9OA)VbAyEqw2cxbls79Mp^JOT^Q|jBfnI#q)mp#7QkUF{`8OsoWFsyhcET+0tV6gCGRCqi3}P?=kGl-_Nu>p z?$+I8y$AraE&9c;+DUU#dmBL$saO*5o9%U&4?5y1zWa6v@9`r^wiuU$Iy(UXvZWa$PP#bA)tB zqbLpI_+|gM=~h{K5!K=c#8iS)VuuWd7^y~-CO@e~NmoOYg?*_6o~KpZxM(dGYI>Dexp}V-t4cz4X4j@_-AhNEiq1LIjNc>I?S;n_{l-tQU!!=N{JPJKft7^_ zszVPGw$oPVD8D}~@WBJCXr(*nZ4NaH+BWgAMj0kL19&Zlm2ri6J=toPBdt|mG4W=g$rBXi?H$hY8?Jdy|93?eoeV7ne`ska4nMW zX=jXTYc|N=^P6^F(bqmc+zDXbR8VNnplx6!=W7_gKdv#xk)3GjM zl;ZBi=5FZevRdbp&R0%}4@LP4zleFuZWB{yH(hgR6G6uQun$E)lm71V)$(-1XtPGH z(NL=oO1YU;KqVXI$uB7%Vr8%zxMVfnBJxXV5K-`!Uij9_8ULHf)DFOrdzn&eA*fyQ zBm#OF&_z#l`5rFz>ri%>D0WJ*_26$k)OvHSx^w(=Zrb(m5YY0W*Sr#c(~fgEE}>HJ z@Bq@dbN=Gx2dQ@7FMCY0@IY=&?|Kd^?Jj_N<~}k)M7FQM1H2Jya;G~4=COK?GW!(o z%u;hTXT-IfE&mjvRhcTp8u^D;eqIow1g9#domM8sesNq`YR0I$Wv48%FqA-9RDUR` z_zh|#zzYPEK8C`!-I#9y8++r0iIf(QYH7Ge-1$8^fcwidBR>9CI_E>tc64XggbMN{ z%MV>!$5TCn+n%SlctdX&V`2GtHSSP5v)g+MZf}pzFIs7KuPMzcHPB=6wUXU#q`@*k zahU>Dtp$j)7rcV z>saO|J{5low1J3EX=jQ69q|lnaW@W-?YaDEq(T06pmH<|?9$ z;Qxua+?HdgQMrr{Yuh9rG#DHX(edcjB(z85r3m!8y8>!dNkaSTLMhG8^zO{Z;Xjm z<+76xnqXb?QKa3Gfo|~+Yk+bi2@d~c{i4@94W(PfLJ7k``L`H1cRe&-7z+;_q@@@JU^X(ao@7K`!*UsS{KE_EcVk1^|bODmRbt~y>MLcr(PGrFCXfFD&C1=(nh9;~$fN zRJe8X+3m)c;KiJSrED_zY{u+b_^fdDZk>wAHoN!4regnl5zQ3nq9I6pm3jpJaOeto z<1x1qxQ7lN+RK&aykmrUU0zyhoG9WidgR9cDf?xgql51r`-6eC>A1SotS_7R%fgiJ z*03$X_}%xc>BBn`>0`&(z0eR^eXyp25wwHaUXYI$D!Bf(glur<_6er?t>uS@`Hx9o zF0E8L-b-I<#xdOFE4Lg>I62-k{OtnMTIsn0&lcO?VPNE;Yu*4-uiu#AW>Si|9aPjs z_#goNEc7=U_DEIgfRyH1S}0d{bia7S-#*Ca3%~^H#Da#8JlBay?>n~wbe}`r_QWfH zl)wVb4`M8S{#*1}e`bj2B*+UB&)Xd&dqw)9YRzU&D`_+_DPiq(7q|aW%^%>vik9tE z!sQ|L8$8?-X(8inW@0kbgN7NxWUDw{lQ-S9nfK!RAm+C1SEdw( zMc)Ke5%<~~UT*K0BxI|6G1Z$tMP33p8YaU9cwe~W;e3B@RMdcQxWaSe=~$s5EMr?p z1DS_t*v~A7=!O_vOIN*gagQck%O1IQF!eh83*kOX3zkWklyc$~*d+a7RbO<`wD|d! zlkx3OrJT_0!dBca>86#)dOE3;fd8kA%dAkG|7nX=d&1x?vr@q}xT%G_=ZSw#>Rffg zbjKyT3}quZGzZH=-r9PO5;9vzjT_Y1?PI*A&+4{4xbeD>_a=jp8oKE@5|Bzd$LY$Y zR%(;iJ2=8ub2H;_2degI2)Z48i3d*Ch}9^#qUfX4fO`m5J>Z%Am<>mpc!SxhGF_Vo zzkpj(#^HtvL!bAN6XnV2Tc-kpx;SV=WIs(ms(&#X$gv3=q_Dz7N%#XTO#pm;hgj|? zs9vBGxj+A9glV5y;cfPw>-vOu6}{v_qYcBI)Y`<}-$COgZK}KTow4aleW^0oRJUH1 z6?Rfyg)Y=>a+A0KDdnEp4Vv4g2XcFv?6g9Tk<;=UAg3{ z0`}TMf76LiNrhviz&f*XWhcC>oOqP}{PNo=Nwd!+9BTPhg)~=*imBi)PZY-Z$<2Vg z_~AoMt-N=Do{a;8Ew-x7eg9hb(w~Pv`KCI4v}-5oOh4mVMFZPudM@bvJt+gP^&~S^h3dKsb@c66m{LrKc`*eV4?HT_`*hP zpxDs*AGs|Z<< zte3hXCU?QD1v4cAQ%fCWm+;+Y+rnN6J|De}j$eRWT;*_hrd ze^i)TXOhmp{#}-7R6Cvgg`1NhF)EWOL}&e*0rWTB*=!W1M}yFQw!98T=&LOrqAU&4 z$Lw`9F8=P;EcD8vxtOsAH-g(RjDkx$lk+_v+^D6-aZJVYF*d^6MRmK7)eif>{ERfL z6-lKIEOi1g5q{Se{u*~wR3W#dRPpILuHw5JsfD94S4EDf4^7w4neaX(?7I7iY|`^> zf(W~1#WFM5v zxAXQIgaXpW`uZD{m?Eo|J6-h!@|rg2^iQHBU0P*pOmHrmOIsQ!WiuRy3+bWRE_Pdz zwPbX3DvuvO&h1J~O@+2hYUTz88beAVu_d=Qq{dd=6Cl~Yh(Tlo2O4ieuLaxGiVP(W zC;jLbCcpR*5NS`GbP0_=h!?b`Sq#1a?I6T#ccShQDsjo3Wpw=vQ-Th zUqo{YWJjWlD#ww2rd`Lefst(NlzD%YC=m~S-_OfaA*|l)D7=;Je0uvu7*E%6;9QRk zG`)Hwfu`ntc$r4orgq~^p6C5poQ(MU8>9A_i*@I;=a_jkxq{jfl z9rWA*SjXbmOycSUvR)Pp`8sA88?9Kj}tEKGs4TjR(>4x;dW8RhE(s}n*j)r&+DkDFSk#a;k z&bH?*=sc4nWGN8|l*ed{-1Wpi1vFrTZi;MZuiVi2^fE(FcKZ6gbkQB^g?90-T4mVH zj#_&CC}V4$$?$H^-URkJ(@t$ET#{?++fn9(n3*gPE=aGR`*h)2k5`2H&<1&Yv{Zi1 zUk}WD=7F)k*y`jQuH>t%tbzY!5%~VQLylfM0g;7dLo@U;p`MgtxPn{DHC*3<_20t- zM?~Y=7zmrX-hXoVL$aYwUfzQa+nfcpFA%=bSkj zcY_0s3?dTmjtu=!{QT_ali$^X^N{BGz0BzTY48_E9{L{~`Tyv`{l8}4jj^O(9-brT zaF>3c+2f>SzFKJO2`&5FZ2gK1k5?`HbFJTAg!ON8v0!>cs~9dUiaM>zyGJe@v@C_5 zN5QVFw3;c{XPRR}@SR+qnKaVY6Wrso5AJWP5QgHdswUToBtLlJVPOUUvhmjM5VGql zv?O6Cc>ht{;qcuuwP{BRuB53R&$Xxo5i;)z{p7 zBE|3kT`M)?Ha^Gr#>c=i_*9|!*ftWqPcaARjhUyaID~{*#l<%UyT5;b5a{oJqps@) zM7%~ltkIB)BJGfDGy(XPlr223ahfH7B}9H;XjH+|a^KTV%@DwKneH5+{QN8?QxfwB zgI0*Z9(R5P9UQP@>&POe4!agU^|A|V5n^zV6AgNDV1Eef7Y_uAY;Qo-_!+Kr%{dDM zM!S=qsELx!)z7H zB5VrXdIS$WVE3JV-Ams>gKBA^LPL9Dhbgv=^cLk5pz(-p4_+E2U)@_64az6^Uhkm zx}Sdf>0Zz)M$IOxeu1=%Pj(#m)$Jrj+a0ZcK#Wx6DTqybywAKIqa1=LN?gN3YlF8*RbggxpXv5aiG3E zthmQLlWSM2vD;egHshW{1Ew$WP~RXou007kzqiaS4_c0&MB7fV8{;pZced<}Rp*kf zBzY43xv@*dA{tcdPG)-wbRQGt*(MeZB zS+ww|o1t2JFMYjN;f02cRVi=O6F3u?ROcDOCJkEIx7@s&nH1(|&y68v|$4^NRp z#}@%7!;=Rk+HbrsW=Uo67z`5ZGX4}iyTXH3EuY{mjR{eIV=Vy$%>$FEje)&>43Q3F zF4_E-0a1d*>@R{z_Ddfc0x3g5<=8TE)+<5cCWsy+(?$z*Kv;q_%;Pbotb`)kIN6$& zMi<87y7NI*B=88}T3uPLL7U!43o2J148fLn@1dhviB10(MKNkoqllK;+hjGr%`8Vg zrR0u;h0k-yv_7jSe8})SwJ1aRT_IbCZm%mWh~n_$P4LFHc6=cfbYhJ{ulR> zgP7|O>wM))JY6#n^8@`X876H}S#*BHH0g;JZV~~9?jQBbV5m;dqwP-p&~MfI2KT<` zqobO!Uq@4BDB-<97Ddmt)jlaaVDFM`c*SYRVC;RC?{)2-=FyHtmV61PjU#CNrBaON z9Z5k<*Arw(rr=$fHRaqOP*a|ZUP*}9`Tc{UZWtwg^94H+ zIax>2e8}DY+u^AL74tw2ai;~9rI9}D)Jef^-#4=S@1V16%+J^NskC%E$!lY)J7qjT zErYsf4lo`EM#ORM>Um+%8aE5HC|TBHrItGg=?TiR_dwYSS)WK09ngP#J@?fv2`VWR zJ$zzC4ytD8y#M92EQg>62#TT8GQ6o^0RNJ$d9R|X_t}+4EE#TB0mn{ipL9!W=3b>; ztx8R*)A8`d(C1s)Gpm>(@inu7D>dMPv1)bKxBIJmLpb>e4yJ?}yEl`b+lpwm0;9uA zU*H*GQey@P_ioK<58;TotU5{s@fgq-mDgRTLnQvdx0KP43X0gKS%#S-AatDe=6Fdn zrYl`Gz3bc?x}!d%Bzk*>P3dz*(jv>G<2i6taCV0I;y#mIkIlN`=RUa$SoZ|Tdcri? znij@5#^=w8grpt;73@S=hfZnwG>aTp-cNq|!^K#pb9hmZ)N3ZCCNlm;)0C2>#Fo^0 ze!h_4UKrgqhghLfh*OWk+52q*_)Jgw(;RcWZv&d=WFipdm0Z&`gR+4&FxNG`FDfZ- zG=gja1H#ChZ`^!^n%D5P=nUY3AU^1g`QBU%CvaHWL|VU~t)7_i))-{{8O&!32Y@cM zN%CF|p_Xygg*O9Qt~}oz9RG<6l|g31{t$YmWnS~>Y^M=tq<0NXtl$uyeJ2H!E~IN>FE3rizYIR^oawE*WGLuOL9w*fkohMu zdsCDm%|f4L(n7xT+TOL@B_S2fCdlP9X`V;vm2-9^*0wyquSqb2yz%(l(H{AL{Y=y6 zHuRd9{|xqU^k?BL*Hes=ekO92dS_w1*2g*?%*MTo#tFs?Y`03QtF2s66UXo5r1i!aKFLQx%HZ_q=<-mGQI%&6;J0ag_ib3-L zPeSk~?w+g(jdQ<((zsQ?I+y=to!w+1*4Zu1k9IfV?a)MpYX{Bu^W!&au4}wS)LB!7 zvi)AhveJD2HW`v2s*v?G<3k`_ud!Tc#w2{#nPVQvb`p$wuALUvY@`?|R3O-sIot5! z7?f-fsNSaPH^Hxk;&04coAyA-oZ*45ih4di=XW_gu)nyn9?Q@ioL(bd8HhV>Zpp>^ zL1L-YcJ*IVsFTn8jxSQ;gmMGfA|SP3>4JSz#5bwiLTw&lh(YR z5X3bt5!yI@$W7q7KHfu&eH>}YZ!$;gj@=z{o^5%3O3?fpP@k3kFQBfBJuz-S*e3l; z$5{NKW1h?MJ*NOpyb*G91q5%)ftPsza%XI^zI&Sz?H>l-^)#(PkbU*-*gvI5>>bm9 zI=$gNFsX_e>&W!THv{?KLd>>~p6XYB=$Va7caWqO=}5b=t>+6Qkl(1IBbnEv?ytEk zAL;<(%^kTQ6`Z%Pxpno&Vv=0wLS8NX7hw$Xf}*~M(>rqO>Ep%)EpP5eCW9}v7k|GC zPZbBfQ9f;ekOLTNoHjH6J}?~aTJMjNFG2MjR@yRC^Gy=?<{*%ZsFpu%Tv!N`64m=x zYApXk7X|KSTfo@EmOy%hj1@_!c{!z_IZ)szv?OMipBu#IcZELr9_ubM z7SHEW-x9Gd-Ke_`X{23Jv#9<3H}#yFsUVkH@||lr=9|BB6KBjoNr->x|7OPa#<*{2|#2$Qcnnh75#SkT7ykPe==D9ruK6sYU>O6vQ-+KDtzoizb5!_woM3Zrsf?^AKqEAI_#kX2?f=0L1fG zQ{^O4MJtWGG%L-#9y-=C!#DUBGp#EJf$-iG@VT_&N0_kDJIFT&5qFKxTNC=NfA z@iMJqfhBMo2*mEq!4bOy(ex6dHl~TnAjc`X4ywPWgo#&i>_;NP*OH9D^}^n33h~aL z2l+%Um+7^~jOk{RHJ3ELUq3o6a!9Wce3Gn#lEmoFPpio}SHJoy4klPR8i;PYWNI!2 zt_iEta)y-PKxp-j7YkypxOca|6b7l1WJMEVby6hMM?j{hAw& zja%3H3$=BPzI`oPHkPS^J?|B!9=0Xz_$c_qy8hurxS267{RLc{|6A%&da%xW)vG*R zp#)oerpjq6DI>p)^p84|TsdwknyLyK_49stT5_+%VkD&WvStuvvq0G*4Noy|LCF2K0#+Q+u#lD^$ZM96NtSudZa+dM^L5ZtECG1r{oW36g zU)3myDr<~drCEIIxgJ z3>;)ha7zVQPeMtB3ti1KOf~LkStsnfkk|v$u8*=gqorwIN2Hv0*9iB^&rQ|J&IUMb z*1PGW5@+5twsE8oFe`$=(C6{vAt1G>G9|Bc`G2y`E>H)F>z0%^Gb!&aFS7u?`2s94 z6`Md$uPN)}58hh*4MU)kbPF_-MkR)72hXF{XRI|%_34HQ0Ko(tD5Or8g)}uE2%dU9 zW)wA>ymt{!kUj13r^3_16~Iq;00E{!iS0T4=l8z@ncsHOwSPSA!~30KySn_=p92T! zBXeW|uBB5IrOQ$gOi&t!9J^d4UV#a7_Vrg>rh3S+e!#}TfpIM!X7Va?wnswpeY z#a4cfWF@&N;^A!8mO7y|M9rVieo#yj`pQmH5k$*_MLMN6i!SRhmVPl}oLW)}K zd2wtMOiMaZ1R==!rD}Ec{Ow*I-J08xceIuGJz_xP1mrv)zlyU^0QJ$z=}8KQKLaqt4m4XnhaI7? zqmkWlXj4O&4#GE}o@sTn9?Wt9Q~eFQpm ziqqft1ggw^)m}U*v4WcRV+9ajwpz<*Qt8;qxr{Qfe#$Cxr-!F^e2OhuwwUO*o%v|erum8BNz~$~ zc?_t}Jav|-LhbF`%6WwC#5|DDIc}48$0WEqono#3Bt>Us8W1dGj_dBy<3jsKb)r*b z!euB}N0Qi(s}SKYdBa z=2@&&m2>EpewCN}o>7dPI3LUJj2FqR=_}kTx_)-`Xm0FVe=c@jN5iu%?69QXlVGc7 zZfej85*iy#OHe8rB{2cvi zrT8$%q^)ahJ-;@5;+@J;#ZdGuFELy<8$i31z%LK#?sY4{Fmle=Xg*5i?l{|;tho{r zmSq-poXTN2*ZBUPME}YbLErH8HOw#9xTZFSsc($qK+4ZZe1liAQ5$We9}7d(z`B3Y zvq4bo2PNTL%qbVBPorgu!0U{^(UWj3ur2x)D!*uWVqiAQ?xg?S%(Ex&D!O`h#H$ZT zq8KvTlUoO9Ps{~EcBS84)y+O)Z>5+R3s$$8O%Ore2U1f-NI)cz?YXkibY`yn#UbY` zN&hFCkvFo8hOz*3h}q+TqDv_!Gc-iz_3K&44#!x5z!mLT(!(@<%1=r%Rsc^QdPig@ z5SkXyeBEt)2#z10;1&nybjJ-~Ni&DVO$^^G`9by7>mwvXg?Qj24e;8R(ML5v$X#Q$ zvT_9|ht5srjO6!f^XXHe+Y;ELa~0b7t^4DnZeJQId?x;tnwa5kgx1O(IN&`#n+EIl z0?g(Ai?t1z?l9XHP!+YY_wkst?Lzsb?#96AVqKLnmz>2oTR+m;M(s+Q$+E{K^-H}B z7f38*me4#v3~ACuGJCaEGe{{R=<eCcbG?_ZaYkxNAHjMn*JaNWL9ORUD!zj-t0r3cx`t)Ts4*U$3d!uJwTB(5O!D{$ z?k|-ZKaP0}#4_-G6+T_dno1aTOy?1b=jph>56|+RqeSH^!Qn&4Yzgcl559O zi3N6)*s?N9+=X$sZC4l*P_`w}{46ELYPs@Ks-vw445ZrDUz;pXyDzB{a|LTJ>DmX1 zM5+e?0(YifBwGbtCj1B`b4u}NfQ&fd(sYfa7e1j@2>7QO{ld3?SKgUFOc^N@k?!4G zop@{JM7aOfEa$Eo=?3)32$}0MM*+}FPxTUYG-i{ddY&LzY~DL{SUnnKQ;a`pfEAvW zz(Bvq-IEazd*?2v@x8e{?utf1iC}-Z@s$p6=Znm>HUx942=mdF`t%D=uHwPhY00%o zTriN~N|kT~l82$D7;3EsfVDKm-?e6QK29wvQ~usWY<}Eumzjqh8pw0FeZ5yEYv0EX zUSsymLURR3rXPT9Cew#;VwnmU^^v3xvS0KZX9Ng)d_MwxzFXaIjGuX$)$f8D*(ZX) zxe7fs>pG2&vFV_xYWsSH^ihuoKGr8lRg#w4`z&95Nz4vnoj zcExNvkDb<5L`A1=m^Zwhr!cYHT3TIU#w+VNq;~qVnN_+@BTWl&&=mW$jb3 zsu;A*cJ5{~*VvBcCz{8ek~lPnPQ{sa9}X<$oTv|1g`8%$SlLTTQ(eI@3sGlSn#8#a z6tHr-1grJ6DU0Fv4~ns9?q9p%xlOm8ec~9%)Q*$L(T^T}#{xH&4NiW5#rUTd&PKul zqHj~UX}$xrPx9z5}cV3MgSm{Ix6gYQL)B)De@!JcWtX8lkK^ss`O{U;8@KRbWs ztSzuYy05->F{gY-}|CakD>L z4zBuAb{Z$g+S-dlgnc_6wANDubyRXe8VbzbN?Pyd*1%sN-!R|DB`|kg1eT@mfmREN zPpNR}ytj>vm9y4CFnt-c)sgZExO&yiZ6feGPpY}>)4uw{!2Tn_By_5++!YGzJiQwE z6m90Ijc$}7!&0yK);FS_?zhr@E(zvyPH|dE^WV4c8T2I7QtfL#di2Q-wb}KgAAM)T z)9r*o%vtL=9tm^c-(BZ>pY6|3CXARaJ8C-b%Cd$rVkqRUF`|yEhcKJ52CfQJ@z|}$ z+Z3SGpwF`I`V%M`JZ&Q|eo!}$uQg0>zvF*2RnC`j%MCnjEMN52QSrFE=;xueuKnkm zI3J?<#jY{*BE#(Bal_*eoOuKaS(ANZ``fY8Hwf_+9s%UXu5^ZQzflPs$Ye=8{M&MM zk#;aR?l>xr=<<~-ct8m9j2^${Yus&bj3{l3R?r!uT`-U?MW`{0kc_rs)*66I}1?Iuvo?cjn#* zpIpGI$38Z|eP`)Tdste{{!*XR#hs77qoY1JaGlfV>I*FANITW(@w%5@y;js6QpLSl zJG@MP{~$0|zgiZ34dJ^2)4N_1pRSacJ4yk@QB>eEa6!>r0L<{M<-LFZXj)=@rhT!9 ztY|uW>h)xQiQlRrh7YV68cv&7U0Sqipb^}e-Bryk_YzdT4*;Oze;+Wal>+Cjyd zKc_RT<$|qVOWs-MU22V8p3 z)%_1&n*=uRI0L}ok9jDx|eqa)E*zDrQraKXVu+|Mu8KVAK`;I!ELbX%iC znxexs4Uv8AWFNKj=%R~|(E&$~&9C3~&uX6|1l9xZ0x7fHqMFIo{pIlz7UF}7+LlWB z-)(CCN=c$Q8o&oJQ0(om!;*_{F;Cs-{+kG8XZde+(p6sC6s1hOPm>}Mku1P_X=qs} zQ;s4dXy^<42#d_cbJ`2ChF#-QjnK%`P1&193x9r0{}9}xsj+?hTF#R7R}HQD2?%ZS z8{dChBHB!IWcYE?@5nhlc4)Eds1rx`6M)#ed7eb~cyCfH4)@F~;T%T0RbH+2-je^o z|1OtbbX#Z{cj@B0t83dg#Y)x{`Z79y-!y5D2)jZ*@}vpP`&GO4!+|xSCqxqrNbIoo z4TDSn0C25PLE^wE?y9pF>w_|B{wHUpzCw!M?3^1bpx@`TY9S7`0TlhfEWY>IR6i_r6H||jLzc}N(H2#3D ziPD&w1C$9Y!d35PU8^|vvB^G%NZchVjuO*!2~P2w@lxtiCy!k|=nS?M^+JI{Te z)J^He{8|peoOUQq!T3qpdf=3GMdOCrdC=nJ)nu&zO!(_A;bWRA_khEQ*qU1zYQ=cNs&Uj?oY~Uk*n@sw5yeZ6qj-8luLcJ zXweX!COxz%9YQB9H{Y@TQ$c@nj~VkiXHT)JqfTMVoYlQJyD8y}^=41G4qe%r)w+Rr z)2`!pHRfZg&$<%D-hS~@r*KQNLo{SQ@2{}tQavebDBLjV&qu2Dyj22ZUQ0> zsl&A*Y?zi?%lK?!-@B&Oz--We-1;l9T4 zO!Bue=XkBhKE2PUPU+aG+?i;2_mI>D)$-nJ7W=v4ONg377v2)9lX7VFLAL*CCL^@B z;UZ{Wjc$3ZSGm9b3E&y;V4c+E=n0FhhAf?^R&|ai=g#k@>49s??up#BX@9jeke0`> z6=gS7X4V0*oV1{tD|CnOhCK7H-_HRSE#YvGD_hS-EIdi?AzKr&8h>;Br0SVTIFl41 zmaG77_krTaljz=$5J#bqV z^6@tdTAH7gb)Adk*kYnPn#mV6UAtdy-r*Y=!RAI-4Jt0&Vq{`@5{ z&bg!F&jt1V9OIFfCL=RIYJ`UKZJVWtcNCkMVN1;00Y;nBeZ@>QJzQ>p@gi7(4( zA7;%@4C1t46{kleu+!Tial+{}Sn?m6_+!B}Od1RZJL$VJ+#i@5d zmr{m~y^3jkeJqA4(y)(vVza?xrQJ4^`9D6&9vmtsl!UxA9OPu!YoL}$T3#hDB zaL32vK2s&tzAe9=&fJOGPsnpDX^U;vCB?M6No`fERx#Aanj);EdU-lx3<#aR%XTAKB;H-!`?F`5@j{|p{^UDpPF1IonUUx$TH zM-l&Fk!;IZiQM0UNzF&xq;_Y(hBkkA_@7e3{~1Km?hmF%2j31>7@=O?k`FtTFuVj3 z2Hbw_$!4@1AR+0ew{&J?YEjB^JnIgk27l)bWH{qEcQhIhrD%iI^Tk_3>h(MJu#j|F zMd|=gC-RybMGr?pErvp-d_-zj#}(YBx>&ZF8h(07EsQ-c^)o2z1W{3bma8wsTFhsW zh_dT5$lb}^jE%6PyU&W;KM9M$aU|y3rNkAcvJdVPE$X7JKU-}Vw)^hV-1=(C-E&)g zDZ**DRY2gxVwE>m!s<)rjQ8!G=rMrkfq(A4T`oQHIuD~dpPL{( z`EeMy+9Tj{^$Uc2FsG?}rw`@{rJq{3ZwCPp^7YJ)&kX1RzpvVK5mZlVhW$KI@Vf#v ztjT}f?Iw4EK^sb2iFiqX%=s?aDN^HYSH1|46+SikE*%Kwis+lXEu0Vm)GGf?=^v-0 z1f0^JEXwq*=v^>cQ3kYvSz`Yxa8q-%DPG!wiB6UV=H?tS($XIXo`F(_0aOFjv^U~~ zK?soKy**JabU5?M^d~`|akRP0bEvE8Pv`yV+@n)60n$snlx33e#w4B)GXJWXZ?Ecc zFNb~oXBNPBa6P~|49U~Q53>ukCpb05C!mPDI$o5}oBjlhnGlJ^ZW^<5Xs*Rql-Atn z7%bprQ+6f;Pj_0$&zZyG7~6?OPTN zeqp1SA5*&@9~6x4d=HiH*-{s*O&-+5%n8mu(M**>(GCXm+uB~*?|=?nJkh?Jz7j3$ zmR3U`L+CxzcE&B8dVN5$>@kh~e3I_Ke}bMu28JX^{MwnHgR(*9!@xcOJ@{{5r8uQx zE`C3bAOu=VFptpxWM(-BJ(>UvadIb7YD))=nQ%pyWQt0pCEe8dZb2mXMlW? zJ{KL&4b&MNdjh~jiC(i2s#nZ2#Fk3zg&3bomS!Jvf!Ofg5*0%9^BTc zcn!Gz9|KE`j#>IeiArU-2Q~Wx=9s+UMB(1p4}dYguhr;~)ZMunVw+&c)^IYPgyK-# zBE67;dgSjoEfU%_k~J*Sb-V#nqYUtvm>clMN%~m`wtC!x&JoZ!r zO|2KkM;udwD8RvTMA@F$%-qGg5dc~|>ApSBGIRt)tq7hdLP-#m(EJ^qdzR$Ce4&iz zD;Bta8Bd=iHGVrZt=KV-bM$}9^I<^71M zl%y_@3I?9tUFSr}!Xzb=#R5{6x`oU<$b+ZO!v8F^OOm)8WiF2>%WeQiWX7pG?OL zB%&Pe&zx-|%TKjKfhjFcU>bb%S1>eK4s?dI;>050X%=%&5{bqpK%=vH3|D9xah>tz zM^$~m&ZH3#UC&fBKwu9Br=OAChoBy4peVtSXYCIGl+Q`PI&r8qllZ+^ z+@`4@hZGAqzsK-(qG?tLCb_kpzo3 z4#bKzbsy$)!Cyf*Bkfvbp3yzlmlH&1Bt)mu)ao)_{4AEbO*QOUao zMrzNnhotxh)0hv~pnyCys5iYu58ST?}`PKy4KYgRe-9h<-{+?qwT$K?W zIekajz`UoW{oWopYGm*EU*Y+cnO1^H+yqghD_>4P{8mT3+j~Fb!I0?@AD~P zvZ;nd!4HyGWzWmtJ*uG5^*;aLJQL|7-uAcmOc}T1w)6Do<3-hX*JvEq5%c4igpyYo zd$|@vn$eOjSOa};%(W}v28hb8EA+eDm=RykqrRl5%JmxEL4DVWc6EO= zPg!269jZ@JC^8z3)D|3h5l%l8U{@>0|4i}>9&q~8$Z+dd=WbI@BhS5Le)1Z`;xk@M z?62)=$3<+hXPCy}7*sInTAuj#)R6AMK{lT1gTmv*a@Y^9yedwHgF228)3}~{?s3*% zRZvA8LNs;@1N4rYsf}CB%_rtH`pz|;8*gcTxQtnI`O&%nMu=JTXWjB}N_KXe9qMsn z4CZTmTk2L2bA z5&xt-+?sZdgkOo_XipClPt8Uu)`SJ?DvK&GSt%=U*kEGF9;@XWz!l}xV)#G0fRQ3n z!4koe;JdP{Fk9Uq2ZB{Ue6jF{K&b}4@E2$ie5AmEUfTxqK2ws8dlTgaMy+$q%Hs6K1T}*GRzF&p~Z=2|@b~B5kk^E!Iq#qNL`}rjfpDzr&TI zz|0B>zF9Bs=IFaF`jO+E-Fc!Ihriqae z3n}5GG=(g0*f-*d8QNjrUS| z+FLHf!y0^Utn9d|6h2sBtcy762t9`Yb0paKOxyD(z#XrgV7M0T0;zy?s9kOPD=IK9 z1=?a2b?`pf0pZ`|rNPI%6AV-dlu7olN3a&-+&AG}fsMiDe^*F`se7;>y;)^%%-&dJ zaf9w979LeR?0O8>vJ#sY6~EcPeM?WxnTjP}En7(cmEQbOhL14bvP;rPX)-C*zJ zC-?q@YtsF#7M3lr6FSql=$akY!^1Ar+L4J5n5L(W+zL->0W)W3W6To6KopPybN*O8 zRto$S;)PXlxb+i*J@77H-O#S!ItP#SYP5c{^N_yTe4j0~iWe?KY;D*Y>5Z9q{}U`= zYy>b1wuu9B9vZNGLZRJEkUb^y0^`wav5@<5AS|c%DISe{U(rnI{?^?Xztm!Bz64ts zLO!-6RhD!G;5xHd%!O-6ZNC6(Z;ZUp1-x0t?uB~jN(Z>$gp(SY3V!^HayC8A#$itavob^e zTGPPo^YJtZ>bWgfQ&Mp3CcsWhdW=hr;z?$6j09 z0sf~8cLvXyt$TEmRk?O_dulOH*g~1AW{sDs-$a0|oO)g}A>n$w^-zLVYT+6jnS*}Sl#w?c z7ftSJNE4kjwAUso|1#T_w;mhGrP&HDQY)zpx^XvkYHG<`tbh7~(Av=1duos)0~_&0 z8rlf=beC@glv+k==?Y;$b5t(Bb0fVS@)K_R{)k6l)ft{1D*!Nh>iup3FLZHIcrnxt z#`d%?bXtc*0<@sS0J4kD474HM1H&y!=)=!4s+`9Qz5^O9482w=mUC^mlR;So{i>e$QYwgiMo?dOZlIUZbvgz|?R?mw%1j7?t zMjuZi_Vs(>4zTkPYD!^Nx#dl+BMx9g+*y7AYrh3M_(rmF|7hywF1Reo;K60k(?fd$ z_(lo}$+D@2IC$^peYSC)#Ser)=W*&Dgn@fNT=XmymuYpglD0oDiktOHp1k2@eT4bt zS^L$vx%B8f%lHsd1_|7zo2x(ZpT8p|2Sxp@I!k)+=W%9v+7^*uaq{k|_`ZcA_E|!3 zM`S(r1>dhvsIV({D;m?8bj_Yp^k>NaEs!>pAjq4gcVZ>X>G@|{JUHFN5 z&0&XaMMt;pE@F&w%`8~rs3){F6m@JIBW$1g6+P)*Xx_uj)ihK)wB*<&zTc7o!!CX; zU#Z@obl4rx4BTr=j5ct8yc3O2k34w3+zX>TIcSCjbk4&{uC7^)1L)~EMYs{ui{LhtP0SnVaCV2Y%5dq-&*BNuYpN8Dh9h-qJU8_V*W8zfkPl6Poesu=?ZAJ|h!o2GpDquwQ(#Z^h|l%syc|dY zR6>Cf+|g6GJ@ge9UEMe+v)JmEBKe_=+@eqw^SB~$V0c)2f`RqG(h8d?*OrG8vWHP% zMzFRsdOuibk^_F&s`(W@))gnD>@s2!ZL@mM?~;Rhev&ws#@F|%X|g8i{W@iN+pA?W zA$v6EoO(kME5=)xsguc$(0>-pu9?oYrALmzdVc6Q1}D;>h5`p=xhtM(CD|s{5T0E>&Uvt_H1kF#*1rBgK3XGJDP-sGR!Y=05{bL8vS4+aas!@ zke3?o%9j=V>!gSsfRoY)(p~X?cLi^NNyTLviMsU??CsW0pK=?pB}%7FA57BmZ}!QYEoC8=7ZS>@Lp3uSmo|Cilj|&nTXSJ){ORNO}`WOMhR6kmWXx}-N0lOgQ z&_rccCW68)!`s5-=q-RHUm+qOyi5iBrj)P5#~ahg*XPpk!MBJu3@0SRqxbr}UUR3m z%sim_tLgV(^oAt|h>|kki!)3kj}N6m_5W5bwoMMn(N(kQ7jTxobnL|lDnnWQ>vsSO zA(zZ?54^R^B%M(iT7*IR@W02%E~+^1Bs-hexVK zgqIP&bFF-44?ao;K)uUWOZfH+>~6U;iu--#>)wOa$7{X+{g5>$~BVpkvQqE9^K{ZR>_rL(zIU?~kz4_c?7Gy^`UBXF8i^H_E zYV%kt&$tip=3d+dkeTJ29^}_{sFv;v6fEN;c7_qJfhm%|Bxib9$FecD+!a30a9E-M zdez&$!G+thQwV_@j{z@X#9E`jJQA@b&h$`{*g3!3MEJ6gziq;tr&jNW2AEFzStSIR zs>t3uj?)ltXb5(cB#1JH#t-~))5sHzr)GHWR#}(uc1B*!1ka8;c133*cc=(0)HHA0?B4)o^WF>4AButJHqK zBGv-ItM^h@!ay=AkiE9I=Yw$%$7!4G-bvj&u^F?@M+)cG99fEhMbmF{2g>cjB?+f0JacGa5WS)Q@ljOEzE`4m6;4{xuCa_|uyBIj4pCbCC47qnkrU~tc*6|i_} zy5%dG1?3}6E==G*5~MOG7w_|;!ftE-@DaeW%RV8zOy`B3d?lkKwniGaZ-8oe6eF@s zHu54c$M+7SsqY#5BgLOB*wQFxzZ@2#Xtcalb1bi)rR-*J_TZ(htR}*huhpH28s59_ zuE8IrbriAZI$|eYbsAS{aXFc&?UTN(P`r>As;STv^k!o@oO5c1IP(7mR07?!PdS?3 z+SSrxq$5rpHZLz&{R>TjgTYe3?aZvOa%J(It$uz2%J}Po*#VqQA#_}vSBFia^AeM<8i!-I)HN zIEkb)0gmq@u$g(x8#`2#50HhOD5$)VB)b@)Usem=r%l(kTEM)x#twFUK$H5}NoNI^ z*)6Y*-O@)M&N|G+m{pYSlCLfEG%Epk5AsX1d3qt6_~S)oRN+Y;2w~P2fYZ4MIh{Av zW1k53E1ABXJ@psr%*nNq*|m@2#81{wu&_gbgg?eqF`e-uC)FAlz4b(+#KU3Zra2d$ ztNg}-m_hKA`e!RA7ahCO9ObXA5BHpE@}(qdXn0;4=5(v4C{xD=_CRO8@n$QBWUWNj zmiVMfme^N;-2#>v)&wr%R*zGNy0H7B+U>_P8kYRvHp+R73El0|0Y5wVt%#WiW$+LL zv;3(iyKw_N@3|S74JJgt`ldGjPLaeV3XQV$Q`)6Jz^O@S&)NqhTh-uw>}BB z^^^im=WcSx09Lbt-`5u$>6B^ut^g2C z-hk!?;B^NWlcReVgkC$lxKvCF4jZ0xJDiP{L0ycw037lKuUrAE->dt&l>}BYKb0-czp<-*;ICNxd(qo-lj@Dj!)>rEHfL_>K&`$YI?-LUcY^1I}cFvF~gX<>- zoj?7kMnwjqv_)!pYKuVPZ6#|zXfef&gMlN6*T8|OW|G2fcM zQ}v}QI#R+E5=7NaJL4CZ0#pD#^44cOe`*h}B~O4iS!ZC^5ugW_5sc3GYEF2uxTNF< z5l;MyQ_h~nt{7rdM78>Qjfd+ByQTqNx9R5gJw^-<1&UU zmA9GbdbHsyxe1=NYoBmhq8@wPCwcJ{E}#tS5!KJ(Bzn74ca6O7?~gm)b6FXFU~5-U_B)d)IVH-Ru>Ew96Q?_hfMdh{R1bDzank57Fhh#Ys{Q60gC%JKDvm+=MrvkNkPA=dG7r?FC)|7snZ7@L9h5rx^0@VNG~U1aV|% z1-(dP1M8@_2i|gXbMKiVv7D+ULK9AXb8qS_nYw@CyH{GnbVdK#{Q+e69{*|gFy85! zOS}7{Ry_MebAAlSvIWgVlD#an9en0^Cj-U}zM*pCd$KjnhBp*=0st@p{sbc~T#rSj zr81p9(N4IeGJ4f)qTCtA@{I=M)yZc4>vyaomDYQK5dna^y(m*Q#UMXm3!LYX0u|Y5 zmgJi#$p`K;JD!IH;?lanY{xsGxp%w!E&CgY(L}YNU!OVUUXsHIcjim6FRTCBrxC>~ zGmC5|ryMd5^J$;aWhP&XXwP7qNu~wcpgN_TGUdt2=*?cQ!7P3w;m4EpyPIa+9~FFt znz=@O3HQm?{E}|mCcGLBPO8_%eP>E=COpd*QUk|*=~2g<4vp6|TfmJ730*w*0k`4+ z(!^H4oialSJ)9M2wozL@*k_@n0o!=Mc*7W!RNfg%CGUY{9ZIxEnSwNuLB9A8CdH92 ztb)hB{MVM;2yg37+~Lv@)vps|Sou(|+B9;YuRi*H#NyjXsYF4G8&^=G_AoU`yru$& zEb$wND_KmbeIF){)J7EK)Bp-b6Mha%IVB;~9H0hiD%^J1Zp3zcT0Jh%>n}&O%bcI; zoYn3z6Goq|D|hGq+93UsomBETc)Dnh6My>CWxphw3=PxBF9?{Z0D@XPc^GQaQtkT-AFY`1Bc2 z=M}f&{?JU;*4*?nR5UPZYSZ3$E)J4$-TZ5Z{szWw?csw5< zHv{2LR=}EQ$Z1J4MZlRCkQ>jt2oOIJMerZ1Ny0y>u!hn2{`D$X@K)z4u=`OZ<%ZUl z!?S*AL5?3qOA%G+DXZG{24jk*m+bZTH{;nE?mEpbuJkjgM{~~S1g>1Kb^Z;eaPnVy zIkVMxh}3QlvEF4G*Ss|Q%AH~@+L%E4FSfqH0YxNMS$&h+`^#HnnML0+}&V$W3feye0550s#zWH(j;we z6j?3q`we2$%6`+HT-nt6HwCHcZzw-Hmwf&6oa2BzoHqd^VRT4Hh@m_vQC+!A*zZ#Z zr~)ptczfRjD{!mrL`_WEod>+{7{}+O{!qjN%-u<2dqw4Ce-Am+#$=&}#kzux1v=|_ zG(*U}&ZhM*y|W@>+`?mn>>BYydq4h@qMGSa$&E+L(sT2Oo3&KC-Ljw^ZkmO%_z=2( z$UqPonkC3sPo2s$=!jbOzW8bc5$#{GjMRTt^sZN;&EZZoKcI#zEG+Er*i5)`a7ZS1 z<6pmH8(A81%w;N|6-Vl-y}3XtwXCmSSX{?1YnY)i&b}D4M4EV*x+)2S z9zp3cz)>AH7(M_vtCeGXNcUD=)s{Sw)5>tc)>a!23H6!;=CV>*lHx1OD!L88XyaO{ zFTGy5rDNbw&Vmu4Q8OW8N=_0dn?t`a7&QQ!al#kqUnN7_;ew;Tnv#eNh*?y{;I`wR zbs$}jh1egq(U(zBp&+>l|HuO%lk@*VCK{zKt-b22f?9DTE{~@Xcj*!-$89h5xs6}goegH9Ubb%i>*By!1iJ&nJ*xm4 z8o2UN%br0u66e3Hck++F@IUnz!9Sjr$v*fWOU}dm1L7dT+*|1rltvP`P2C>{Tl;v| z_{Eqtn%k)5bMRsU7|%mBw|)`lxRK)5sydep0jSq?h7h}STjb~g^d(*^8!`gdhW*Dx z8M{kNga04a-a8P>zWpB$SE1#SjBHV4Htb{;NmhgGtSDq<&npT^3Q;H&QbIPF$=)Qg z_sA}?$M<+&bdTq`pU?C8{l4FS?t1R$F6a3^kNG-Y$LpQH?w+`>HU7SiyKuFz&)dQL zg3xC1U>asNWMe$v}O2B#s) z33FT2GYo!FpRQBzI}HVDk@qLQ369hx@!R-_08!R3%J%?1%D$6_+$?bOq*^L!b6VZ8 z`|w5`65{F6xzSd*Ab`qg4rrQEwV_PMS#2igO$XK2D?JARY=rJNcXV)eebj4KWS@*b z9PL9r^X&7U&3d{&dYbY8^Jf zc&!7_Yzl>hd{}<3uSiUC+>3U-?7 zo{bnxcv6+O1MF8es5`>>4<%lMN2WHD@ZE(!<_qVU^DIE>x(T1D$>r$#)l}tkqOOWV z(emoBM}b=ugnv#bDzym})Uv9^vv{ea6@ByGJKpit*DND-Zo49PeCrGqTRR=UOIl-E z8I*z=zOO8uGQAjIGBHSL@%aYTG>C84{M=Zb(c14kI4{^~PEo=Y9Y;Kh#&%E8En4$M z>S9>aT?p;f52G^+AX%g{lhD|Qc_0l|plW|t7`a|Z82a`f^gj2f21d`KwApb@I`11(&H(OqDNv<~>Gl9QnB=b?30aK8uFK zqe;gPTzY!#8!hokvzr7*55C@|d%08igCdz;3eqBrd6%I>*nKwoYnO0FWtUjU znXSqoV1Ui_v80gf^~InZhq-$UKR4b=fc>pbqCo`t^r+Z?%-%E6k36hdEOlxRz>l2X zqx$tF{OpcXH|~!kBzUWBVT0$IlSf$l4qWDu$K6>e*^_1)D}7#MSuz{Ld>-G?zVoe+ zB)|{X^_ExKif&u>j<&96b`z|~M7v}7saNVR%>|^dJWc0)1BZl~(3n3`5c{G!C3wbd zvC%5Iy*f@8lLIBZ%FQ_;=$|-rYIjMB@;lc+p)2G2%@o}xD_K*nnMAm_Sn>dbHOriQGMMfrOs~*DEpqSZuxJi@gAVNj~(>V=?>E zPKFXd2X)Qlh5pZWh@EmD&XVE?P;!>M&hN8@{x4zX?PDsKJha7sy;WYmtv;=g6D82n zxu^vL(kQouDs9wc-z9I1UC67^j5}3ur}t>7jvA`RQY(jcFPn z5+*uMFmm2`^uwL2dGBvc?G5^9{YcYVoHqN>$U<(-d zU&aV0i$7w-PH5S75O$O115YaLi_+;T7-Cs~GCJ!p>h;Ln#$>wb22nBIxCdW`0bk0( ze=4g}X}{QPv99LjuiWs8LN!Y104~Ez0@Yxha+*xu>LMiOz0Zo%zD}jRDx=qZR)IhN z&^ia@#=Q8Oqs2-TGf#t8>_6U?cWaa2DJ8#_Z5HzANxt?=iFbmkG=Zb@53<;S+;e_c z#xu*?TS0dUqHoF>!y5^_P$zyJt}=y`3&BA=3WNXA*XRhtv~BzfRQbUXnG*T>-CNZx zd)0Awd-5BI0BDSo%*#!^(SH(5op&egmYvsFZ-Fh`Ha)vvn^0wy6OB?*JW%z*&!?Q1 zP%QN-!_TMmGEASWAK{rQYLca^qS7*tDJu~RKB^c5{nnRq+wXB<)xaj(LqUgwQy6Bk z>bUu)3J#C~^S&r@1s1#{X7$nWjQJPeqt@l1*0#b~2pU%(%!N~59hY{afQq*l6H$9$ ztEU}@4OLn5xcsQR1fv0Jha zWcjJiJZv-pSP#_=RLt*2_7L>D&rOMNL9%0p4+GPvbKy?RDjtL{MzYo&>dFMFDDWxR zVgN2h|J1H)5!t3htgPwH@Utc3H^vz#A(Ly+brm$RYd?;iz>T zrC`n?^nG&j{Wk*uHBV38XVfy zrTzhO=oVBkI6t|;NV0byu8gZAHuH7OMMnHt?5j6cfUcyVLpWtBg>wsHM}_pyJ6Z4y z+sz)_kD2pgJ?(dAj$l7DBfX$Ukb{1ja(d6JOmZdtz zir7lhj10T#0}y|2W(lW|T&`%H`NI(EiTO|ct08>cje3Pwzvm0XwU-^Pb-F3a^UEii zzB6)P|Gsa4GT3TaAg+X3n9)1eB`H+C--7@%N8^0 zzUTL$=H+6n8OdWG)48nUe}eFICg+jq^tcMUWDiYW)(^MaI1ZIPPgWLmRPejRN} z$DeMsiT9OfttJDdekb)yFj~An$$k$JaTiL~*Eh#gjEVwXV?)236vXUO?CW;udD3Z_ z$1~=E@6&>h?iAlk5`ub7gGpG0Hkc$*MNe#c7QQK+wYmlc#)gn{!QDh7k=#uVA9qgY zT(Q}XJ9E!3Ziq}ieiPsAs83{eFL^h@J}N9k9idD+V0_S9FY3xNqh&%1L#w^s8ve?mZrtjt;Ua;-xr6ig|}zo>Kqhm2Oll zoc{TM;&4}r?@snDI0Oer$SG+=LUJ&&xcRC@+B=JYD9)@Z13o)*^}flBKJyg8db3r9 zzzJ;+p_g+*toM_}WFa-gXp2||h-vCN*FScdyVNZFd#-YNYpybjJ_gnaP)%dN11E$Y zz!He2uG28(ArTv+qSZz$d@3dQR1vnltC|8O>xCq$WZ)U_hhR1gL_$JF9TpWULWR1v zohc{v=pw~Z&3o9sy?*F)offEYT+@TBe*pog;`X#ry}Ctl?}wvO%Cn5XzYO0xrXQ|x zyng%B(S=q)>)XdzspIPZu|(~G#>Rfy&%)-umQc9*U7qf9AKFZ)SMTrnoDCN-6(Xc`8^TNmpk%V|k+Bi0a%F^Q9(hKtu@0IMo zf{zXSnd2Wlv9@zNDFat9i9qEMW4n26nepA<7v8}WfzZLq@4m+$x7*ixeg6# zHwXxbS!ON-9w+u<3iZO@@V#*GiPy|)E3ws!jxur6AF@`%gG__enyyf`U2)0>UfD$l zz^u2L_~5O=a}%mZKIJpNLhQp}{-h)R!%z8}jjH>_aDMse{5d7M`nWJ%PN2>4T`(<$ zfO{~c7)IaROsn52wC}(=(#uGv5x5(uo*t-A?uP4cUf74%MoH*F8zSC@h(sZAU zY1;K5DmO~uC@Ri=cnTC7abkSt*ny)x028PhZRs&Y0ZQ=10cd1OPw4E})RbAy%@JBh zrwsR^q;8mm_;3-gq6UH|5kaF>?lEb==o{ zqw3k2`5$DQYlx6^yGvQ+Ck3G~iW*sY8RpA2xeF^3#?7PR?57{msdrX|JKh0)ZvOJV z5AbW*x3lOU#II1IYxIRT5E@8MJ4$kg{llv`8MO?fUSYFw#YbSkN0Va>%ht>W-iM6V z-SFS8V|Ad8{rOBg%6y)i|8hT|DOJx@vF>h{hF$VwE;(E?8#k0=bSe|=>;fd?%hLQ) z2mu2K%AV_R*O2qG__a?qN5Z0mAE8e6VQ(+LsfNn}EOI#{AQc+3`x2pUV5#5dwA2O~ zL$*b%Edn!=75>Ut7_xTqp55$PZOtw`;#{-QZW6_lsGgFw0clFDaN}1>m`UncXro$Q zfJc60?ttw4e`V-a{rn_4TES z%;(;!#(97>uJPtc5)#HCC!y)Q|Bq2+8Y5bFPhf_prqH^zC4%1Uv8|%Ee8q$1wo;8_ z4H0!zZMi>w^9+Vh`-%+7xcuO=x@s;f� zivre+?3IHUH}?d7OQcrIkA z%QA471S+U}3|iT4zxn1xY!Z)1PSNUg-f()Yi;w~P6M>A8puhcTi0@Cr7sbyutLw5uBDu0>r_y&!TkY6IT|G@6Udc-}0o zd{U*to>~5)jg1%KV3~qhUxjF5jscHS1Wk5TNn9rlnS$Zgq$*{PmCNl%KAs;Aj#`Ud zo7vn*(RtUC;j!}lO`frXy?ym!;o_O)X@?>+%Q1<5ZwhUOohWLH<50BGbze4~SV1FW zxS4rA@t6>zxhj7A!M(*89Y?jioA9YUnEOG+uPBq6Z}4^pQVgA86gB3S&wD2P1~<$;EY=pVzzm7^a%CS49qZnlgR2&_ttzu8)6kXV@Je3u+~upV$A&PfSau&IT#_Et4c3V%*&Q?w>9GmG<}4=pfQd6QYb-w zsa0*W=w@^YpP8LGYnO7&kGn8g36*s1bl-J%Yv#8D_Yi{5!_vk=;HUC<+(7VKV8uGvxo} zWT3%p)tKSPpCwk^1xd@gnW=_PK(o5Y2n14plabozC+)^t9wcaLX~n&Mjac|sQyI64 zJxOLLZT_H;U=t35kzMt!Qo(X?K)7^lqt`7}j*KQdFt*(e8==1 zvf@p%(TBw|e1TTTa+(_)V3`C4JSLd!*$B3hWNk#K=t;muo`7L)%WN8H-oyIMqdcVD)1FtiC2!&jh*$~39K1%oEMq+ zHAd$#7P=oWfn#TXap#iXEA}h++UgIIBc5uFYeqQI3Xc&1c8r?9!LdK{5#*TPHM?cb z7J>&F0U6x>7~jFySO(T8Q%4nYCe2d8lJ&<7RA4dM7-twHzb@V)X+WNSfSlZV;qG}n z5=9H&CIHlci3tlROf6=Dj&OBapnM&Ed#m6a%71Fvz-_Kr&1d;GZb*Jk+R;ehIy4H=|pASch!mYb2EhMh- zZ#-1`!|WQrA!^EHc%ksec>V^*Z`<&cv^eF5 zilS|IsQvO>%}_zYHk#z>azo$jlDkult1|FQxL33Q`q&71#Q>*AjU(DGED78S>&{7u z%{hr-ZwjTc#67>Z>oa#`VSE?)!FkbX?tN5Tv{qA83FU+ZTc zJqnXZ_6C=TJIQp+Kej?C?vGG<%s$ae3OorwL)|8%413`4F0Au&J_+>r# z=eKuXonluOUye=6{~jMPv%@Q9rEzChfM30JB z+Z5_jA*C!>A2NDOxY@Eff)cr`@OC1{y*|PZ;UfW0r|^&14y{nEdqHshhj!kXy-`28 zQ!nRLQp9;N#k?EG+7raGJ9%UY7h|S=Mrcp29SR6U>r3=<5wk9lnFhYe13o6b% zC5;JPrZU57Uz~1J+ZdoO!WInqqkoUwzmtL0NJWn~+r`UQCX ztLeSgQqf}gaeUU{E!GbF)Z_DzG04^#u>^QuK$WYiL#Uk;_=5jKD?Qu3jX^vTl%;C$ zEA&#ppY6AH&h*L>`K_#PR}frr^2T4`itv?hPvmq{>jf8q5y>y^HVhn-oZtIvss>tc z-N{ozGV}whF$>=e$sCTgrnvv&a?ns4ZvAZ%>xkvI`QG$51LZgbA|hE?Vv>;XVrf9r z_wx;4-0(9xU_Vs&kfRwfL4RS4u?CwYJ=480n&q>fy&fzz9?k+%)Z_>FStHDiy`2lE zXt#(~(5qe_gI1eqjGSu?2NE}|T9?_hYM2aTIZ)+EvjgWx!Hy>W=41ujX3eqk zL}QAV(N7`8QR}&?&NI2m1~T%jSTKsgSN;cxSVfXBQS-F12DEa6F<#{s`>9Lj?v?%l z4MY0u4}W6kda0+UM*kOT%>+>@+)oF!T(msNm5&x4Ui$jd`^ zaZQ9;F2XfmlF{EG2wHyRG5G}7er=Ts!@a_ZN?~4hBDa{-1|H6JNbM1l9I%T=%`$?g z#0?s0eiMrc=l}P_Vzh$=lqd!8#e{BSwEb(5-Q61M|5N5(rbCaDiE0VUes~@!5+HQw z@D|KB@6y;;10FFh-v4lp2Xi?`#hgobUsM|2YdTVFo%E5b-k?Xz50;H59#3xH=sScbx8$ zm%ZI>xz@0Y%XD)g4r_tC6-pff|Mcix+ z;AX$=Wwb3OS+pInuGcby&8h}D?i=6JYIxBqj^4&<(PrK=f&-Kg3N8==oBa_QqQ^aa z0h;5u7;sDv(`9Nqdw?gq4-McG3hq7%>?~PN&PiUE#Du;#KjlMh<}7}vv8lFS+OhEQ zwI5RVB&^Z0Cg}N+wmp1582ocVcYh+vkC{?XRyus+0FzXdEH1LpvurN{p5TyH?@H)c z)IC<&hgNWHyX~kZOOd`?#VzK}!zzDCtVU_}XB=d@}6uJifzcLqfQEr2flbhI6j>jFxCM~>(}fzJOL zIRb#eWOwtn84NcxhwxPeC1LM!y@UD7d`_cICx+Q zs~hv4lI^5p{bfNH$|J$Sa~Y*Pel#Zy$jg4em(Y#8G4gxKu!TXPW2-nXu@)L459Vh{$Z5B^ z!DN4Q&%Xv6%4z80G=I1kI$uDqwo9q6n2797*jKMlfn<>~=zxN!uy`()0po+YK^cE2u2PuMFz*I)v{2Ag?&wRK9;eY9Z2cTO=L0QG})mLhhw=j@Mr#=f8O-V*K zp^%ipV0;EkLxUSm>@>cr6*e_w^I>}=wP51^XZ{fzoj}mt|G)ebz}eN7`2$V~drgb* zwOn8*NC|cBw}tJv8hsyztSp&kCMc?;5i)G`Wi{2vdv)FOECn+4zml?mdK&sXWsk1K z?ViObX54LbYSr@VMj`(zQ)(jZf{hp(Acy*T)D#wt6nd|Az*KxQaC3=_Yi)7$5<0c5 zD&*8ZmLe<4(FN%Wz@f=z(B7Z5-2{8Y)xv(KZHk<1R(5)hiu`9 zSsk~yoPfRhe)SVEQwVe$!ng5s{%B=*;nlM(A^X6ChO^R5@h_Y}jW5Rb_do&nD;&m# zuoKAE*rUInPwl7=wFb>6r@MGi%j?1Wl)~*$tA0K-WP@Z`R5q)6eReB7O##8a*^OZY zm|&czr!s=+-X!cS_~ra_s7f%Ol;UV5aM&Nva#}-7KXSMcSY3XhG6_&G?L&PuL5tL0 zKPwMx;}|lN9fa#>pI`5pDF%1{=m#sOh&%R4Rkj}}=Tcw{Znk=T;Nu;dur7x;kO*pFMeTc5MSen2 zaa%eAznKo6lB%Re@cORX?bZ*ebFJ#VcvSB%*hgjCiJyM=i79`?oHJ)~=(2t#?dbM< zS*hgHZc|tj=!?f5z|f*?tX<#Eh61FE0;^&1Iz3~*yM$B|jI@!HUxe1k^GOo{;Xb+R zSIypVTYj(n^M#M&Us%lYfBC}ZX@1-~A6wMKq4Y|F%i7g%`%s{U0Og zcecL%8HiMwCbfI99GtkPBeMqgMDi(9#vUH{J=*i+?3$$h}G&h}YPr`PFfY4wWO$ z9b;#mW=vur4L_MBem?g1g44cXkbOe&zalyWB!D0xmdFE2F7sguWib{2>@2t2ka@&r z)^w>pPCUj^O01#UR5o4 zbLLvGFz1Ng7cmRXu*F&hAj1mSLEfA)g1i~$?ovyDh$IyvmPv;mT5bNQU9ifmTajS&`?Fmco06Y&YRH_da zMXiN0>Ry~yUw}0~sfW^VqjU5VD6#pU`?256B{WeN ztzXdb@od$NA;vrSPSB@k)Jr7H)@U+x8s}SkEKIiBl#sH2OZhQ5c3#})d$SWNNE3F$ zUwidaB{S9(t=Idf#{J{Ookp0TE{a#X=f=;3_>>ye)xlV&4&h@H0xbA0@;ME4iQ5d3 z3|b|Td_g+KCuC=5*RC%iz04(^FiTpt1+PU(DM5I0UMqx^*vie$i2Ne57g7ael380TYhVrBM4%L0Qqs$*#Ty;LUTUb4|0{H1o zXxQd=*`o;A2Xw%z7LXHR@e9o-cBif&=2((JUTulndR!_9gs^Ih0S>kUCv`v=w#nks z)4wdh*yz0P9WO~QojRCR z>FQ4Wnfo6+y#JptG%`fp)4-cI(&)%L!5sPA`P(gU` zrRPX-o0jDn>QR6L*^pp?ARd>8dgFh1XBcTH)b^iyEIAXZqI_%6Yd4d!_Gq~R6m0u= zY7QG8DpuHA!JRaRGO${OJ!~U?x7jvQ9I(*O!+tyi4C=2wK1qV1MZ4(+=Ewf|Ybsyw z|Ec!9IVDm+8QS5wjF?k5pKY+=ZS^VG>Utlm7oJjb8^ceWoI5+|1n9Ui>=QI>84Vn7 zCgg07q}irz)=T4Qs466!ei|)*O>zI~LVOE)iuyF@Ir{ZKi?b^f?)@7yj>xz1nxKKhC8mv8AMw@3 z=--P0cqjRToLJ2MAq@tRSx0(nsqs;FY(6h&PLQ9QG4P3c#(TItkm=A~^x^GTKY-3~ zyVd7?-L5Njt*Ee1KZZ;YT%r{%MrOvyfdU%(7m`dU!x8x2BH<^+Q9{bRYG&i8A7-95 zqfaQFYG7R$h!%E{UCPv?o!8hkd*Jg4g$Y;dLcP{;sH>x-2ayY)ysNlLz5^jcNF+)j zP1PG=A<^FOw})AMAvr`Sz}9#Zu&9i~%Qp&-Z-$TDlrh1Qip(&_S+tkt{O!?Qmu;5& zh$Y5GT!36-421IfcrZHjv_Q=Sp2K}cw{0ZO4RzDR%ivvy zH7Fyd6o2`#Na<&iFQ^IE47xSJ_r_*BQBdy9F zP~J`e4TdVfwjvc{oH17}+X9{Qs->hXYKZaM;tC(+yW~X8X-hhIT0b$NNY#Wdz zG9XzWQ4)S?+*dH5=#x9If9?zRtp z;m{*Ovxf+oWIFn++^6eA!OhZ8R9j$Rb4Nd|3>{LAi;NMLhLbQ{0nl%rZ~!#pmnRaM z!%-@>FJU9RXC*;+7mN=B&qXYJ`~pJCt+=sH?X{Qg(a`|FyNo#ChbJ#>^+YtdNw_6zTFJJHr(fnDzCivR?l!Lk^ z@vH}al%?y^+unmx7l#o1ez-^C6)MeCj8JyXKV7;D7sL9&+%7L8%fX3YD~o3Pjz>O+ z8Dtvmxj~xPNDxqDqIm$PAVgS56{B@z|KGmmU&y22%gopVvzuuTtt5?qVk;OWg6}e=eFj4e4};G+((R%KaCQhH7}`ux zH{tDu0$#wlgU8^WFJRH@Qv4ctqbA9TvKVz0a-M482MxBKpcpteQbivS~awG zZ@Y(Om{uN7f?7*fEg3}54U8*~wBV}SVmCVoj#Zojm5JTw`aCv9J%)T)ACVY9Wzts+ zh&wP^W3;V4?X)-kEeA!QUKO+pEPq4NuW4)7l==7$!VSI9K#>ueT;SdBYK({nBYREW zSh`d)jEXGoiwW#%P(!CkYz6%Fy|=1QdWO+F($E}by0T=P^aw!PWBj`Cr@mktC5X|s zaLpHG2~g1meo$;&+&QHG^nH9hjsDS@g=>dS-FP^e)pd3?QRx(RIcLU#0Z;bJr2Zj#pm4+}z0rQNl@#}h z{90O?qNwQHOk`3dky81i3GI#6VcQq@^^)qj$z)}k?Dcrfzv+Fr1>c6fm^R+gB!a7q zd_cd6wf;SjwFgZ!@aC`7Fn&?F1o4nXix9v~%*XN_lrz}QI;a#RICuPH%3Mv1-vjZj2ZrpH9Sj|46uIoEj zl_l2iq||i4vh?J>71-=ZH_z$6e-gH2T@n(7=Cd<0DogI|IcGWQF`YXaTToC?k(x1+ zTEWQmCer3X!m}ta7E^6PKwx~<%lGN@{{|qt%BUQTfjyot;*CTmqKCk=!d!_+nFs~& zR1>v~JI^$NV$Fx?tqz>k9<}+%8#N)8yK}@NU1lU^G<1H-qNSs6o(VmJ*wOppZDJg>EKl9vZw};?S zjRfH=3zu(XKYd@cv4VhDHP{Q1K-iqq9&M+YEL(*fq50D zz|&)w<13*@IcqsEg0SI0FhzsOf9?z3$wk);F;ekZo+3Ib-ws6ihJ$T!H@n^^9{RLJ zu*^R<9{4W!gxH(EsL@E<-uuv+^IxcYXbW8oCjSI_{*RLgk-(q>qO0=_Bx5fp!EaIo z`fVNokvVczwgq_%Dv}Xhj2k};0NF8xw42_fI8p2vfA$ogVD|K=GGwL57G=#cd{){9 z=}f2Ad>lgbX@zA)ta;GnR3!PbmSvDGtQ1ih!-g~{GSO&_5Tg&2hE*Y72d3B2#U2;S z?8l3pmJ-VyZG}gXuA(tG@r0TLjn78zN+oOHw}^Gi++nIFt&MdP&;$)aHLMK~uxgwq zl19~4;e%gL0vk9X2mC6_Atoo|N$G5wu&DVaL`h4BIK|&j&=q=IoZW zxAuCrP~8Z)zSi;2z0=#Y>Lg|Xr|pNHsJ~CID}be+0A)R-8aiDp$Sct>6=oN*wKrKz zg!_pcy{EvewG!@knWz_9{0P4;ZY*Xvd#|2(@@bp@oInVVbJepFi#1gtoJqMAyZ}=* zg?x3TarPbXJa|L=$5F`wY+SIX->6H|5g>Q27a2ABJc|rm#gq^Tl%f87h1Uv!H>=>{ zlIL%?G}~`eee6wT%iGJwz`(E9^Dn7>$~6r7LfP4q6BcV5uWdIwi`FAh94(AE34e($*7!~y73qA%nyNQQaMp`F&NxQbQI+}X4B8`M<6sY zjz5yc!rR`f1#>Y_yCTc_paxMPg;HEpCOU=`s?3kT&$z+@Jp@6i-L&grJVKNoOt;NT zQi){x3%y2VKj9u~-VoM6u}h4Q)|mT{_*L@9an5$2_7Hg_1w4ARpR2jeGFCIo{CR{D znIF)}jsfE*XigP=bTz7uPAu&I4^|SL`}G`GN8K>TQ2{)>0;H}Md}hc{8>_xlbY{d8 zR#j^UBePpU=Ve*tuH+%)L4(Nx4z++ux@<<+{eoZZMqTuTO4fUpRklHuYh0)$+|BfY zrLhsiFJEZor1i0UDC1w=nA`tfF)@CVlL~UenU;9v8mXBnd$UK&0 zmP@D$ukWuPl%Ui}52x;Np5PIgc%ME5%au;Y!`x35X9G!#C4h&71X>oFgJ6GNC2GHv zU^(Vd2mw9|F+;Nw6^M{5CPI)JF|yMFm~0PMore1PvTI?Zix1=*5~L8XQ}7I4D)$pm z4ZO&w8vv5jb?o0c#cNpc=qZ$bZK2D6Rs@Im;D1?w=jP$X3 z_dY&@c;IMe=p`y#U)~2YOM7|XC}e6RA{e%HRv}tkK?U9|3S%)6{-Ze9rTe*o$`yN; zFOhbSGn5v(KZ+HpUv&O0w9=25xBG$FNcYgnW1ELI#e4aUQfBWf(|rg&wL@%+>B{F; zJYXVv3Z8fPkceJfS5`Q2fK}0m?{id#L0y6*;coP}weH6ik|OvElpNyqgLDJSmN!VO zUno?fN%Nud|6~~@6pnp3;vw6p)ktowtlF}7+21BC7G#`i3ill46oad;f4)_AyD(2j zm^S*%@X?cq+aKjy^*q+`Yla-;@IYnXkaXU^^T}K87Y9*V7|gGp5g*F`5=_8nJw?fr zy@kSc{6WOI{>S8J_=w8ces={0|1JP@&PX$W)X^kJVlG6TFf6ZxAl+uznWkRK4QkZK zpC8vR@eKQgh@j~W;&k(?2i>O8+=O&%YmF zp?ltVsMhUvz68zz3HX7-NZk+m#aDW6_tTmn)SDK4RWoeV6V3sF0hfL3H>~Rf+A5E| ziEK8Dy6`l-w+Ivt@4nlnKQb;E48jT+^HPUdAI2cs7{~WhLXe*!gNhLpw9n_y)u&I0!DwqZtZ2>nbIS9bP@EI7;$gl_ zyLKhSiMXUadf?}OVG9kAo0^%WR~1$;y43D^ll`?9hPAIe+@JP0S&}>hnO~$~#@hur zpyK?38yYfbL-3$BEJFWa@&$ECPf1otFTK8b)#1!4bihm*83=#65|I9s34g-UyMK0+ z2OkG<2zf4@@lJvTw6s9Cr49jmTp?kS?e|s>Ee8m8RE~4ZM`-x3bpcQMr=PNG$^C?) zJJCYpiwiB6a*BXVS6#zb#2sOtCLhQGt3%}bD3Pn)N2~5Ey@Z4)g#pVE^>j!=R0@bH zqP0MPpseaY2fb+nQ0Opj|Ak%ahyk?9vI>~^{e*ebA18)lgOzOT5AEhsPYyu@$dQI9 z%g^h8FmfO@YwN-Cr!1}pz?Hd#()ABj$g=+qpi54XaO-^KoqA!$sU(R9&Sj-^I~`$^ zu{PF+|Fg!}>BPqFqG6q~Nu=#2w$$g<+|<;+U<1`_s7aKYnqaT1?I@RK%=r>GW7~yx zlN#W*7cicHOThl?Avj|~K@k)V;*T>Zq=-AcRVsXMb2*r%^`qzyLi7)UyY^PUfQ4Y6 zG1^4lBGtitDAPwA-w(b)_-*tM(418+5~3m~`>RQl+c1~j6Qm_M zQpEKs>b{0uOfP2#BE1`beRox`etn@+d7jJl*Dh9MMf4arUfx9uCOGEAF?E8u-&sl) zC8IhtqfDRVpYVpB7)p^U8vq`kz7)3TPou%NCV9}B^op2T{QI&@o;?!r#FI4tt|i|E zLgkLkD!PPl+g4GOz0Hr>z*?;5)r>JMS%-d1Mc4JmxkG21 z@0`2xTb5rk8jI}TfQ|&2KC}(6r?f8HtL#pg6ck=@9 z8x3WePV9cXYimUQ8x<;JnFV>LN|5F`tPsYVDFxul?Vn47fi(dBKA;%d!Mcn{63Y;Y z$DP*)B{p6IsdmkO8;_Ulg$G%z^QA(M!F(JI;fN~&bg`ytgyAMSA?}0bwVsgQ)h9Gb1D8n_^>^{es1Nx2X$eW{Ie& zGIMjP15Jjw#Kc6s<@v)Ti*m5UGBq{T3}XM}FPz8UB5i8*Qs569riw&Qvl~&zSh8@; zC|tqP2IN}HmKVl4SSVe2VNs~(kyWpO;v?Dt8M>r2f$#EXU74T!#z#-R#M7FS!NO9m zpzVai2bLDHi>3J5V5!uX{3Pi?0do~z?vVo*wBig-| zJO%wdNH;fhppvR36h0Gv7!CbLK}LKT06G%Z`@aoa>WH^FnX3cs4-q}R(?h;%-4OU` z3YGWgsZzi=4-*=Lbgd06`ka3b;})HE$eJcaeyF5xx^NDtejMuB*7soFs=_^Q5M`xs zvR+W7PFmWm(c4I)cb)yzy;8QHVHEV|LN;LmD5y=(qx(^!#T9JIvPl8ksO83pr4jHZ zv)|WhaZViJr zTi`-fIe_x-{cCjo$O=`+zzQd%o{Qk~6g<y4;!fw1j3dh{6+^JU%9f5t%tYn&b)4 z*6Oo9_%qHdjU*-OsHoVOli2){Wb^6nPNc$MV%86Ja*}BtP%61*SMTrs{wpZLW95y< zf%To97aCx}mD5?j_TAM)&a1@Yl*48&Zb+>5~j#_6m|!pjC>#B zJ%0c4rV}fU7nJxe)M=g`v-j;g+X+4vBu8#v3piR!vL!Uo@5br(;_;b3JqAwbKAQQ( z^b#HRgvN$jxBaM2Q`{<^J<#1M?DGPp)++u#R}mbyk*_w-m%pd5QNO3KEm7s*8NW#Q z?^@}d8_n*r@%KX7Xf+@i{35Bh+8Gvzi9PNofzM|J>>dQl-%!V`ezac&70(%*a=@D? zcg_96eLHTNnc3PQJ{gGhSM_9Vg6v}l-O?}VLvHis>u^6-d)n?{f%~BC(NuFM?M<^# zsS?O5!JQNv4T5i5zXdQKmxsczR>R>Bhaz=wv;sdSq9^S-MHwSY9@sWerK+AHx zu@nJ+J8-wKjVY~PV+$E~?_Fr>yfeQ;rs%*mCdC##kn8edNPfINe_f_X_4=>zuY%$_ zQOUturRxeRA}nPykV)?VWD*#a!nfz|N~TjF$=t*aQKG%pAHN#w;bVsc+OH*>T)|v^ z=KK$f1OIh-^EU#7OWU*fYh+c&zP(H*GA^vyR7v=B;1|pAukr<;56%YsV{j@w`0_=fL^#iV1kePA)QdY8{LeF_}n3gcJ^3+Om`6PLWdxY zm@^4e%rslQB0r(COhV7WcVt4-LLJjUKDiQAyc^_A{d1j)nkm@Vbfm?3z7dh6yau&H z`FkK5m*RCS^gHaz9nra)ImgtadSB<61?3`s+g1=w2dG#i;HPksU(2|wm?C8fvYQw#030Lnf{ zKHvZk$b{@3Q^U#h8_NV2G>OSG1(u0mpYD7bYEZY1SfnPMSwd%)J!_RWpw zK?j-GtS7xqC5vVT8N#VSI-iH2z>_P_+YBj9!<7X3rbGq{(lFoD^R-}xrA&AS(<^8$ z1I{tdo6ZZ9E=iYBT8)pKV)>P_qgYn4QqcW;|6-GbjJvtqnh9--aLu8@&FuhEcKUi4 zzx4+^(4#A(HPEUbzfQ0@R2AH<{>y#JOfML*j5K4e(vL!?L9s6Qc5?ZR#|nFH-`zbY z({Ie6H0GWzsjmLh-vX(BPQv|H0ug~Ylx+DCH7Fo0_&q+x{y;Id=OvaLQ!OoN+Z#^Q zz-ZH>U$2#0Dl&=hT;&H5c!#Qz$!}`CK;+y*#r=Vr%haRbtO2YbwjCzJY)H6_5{#bx zvG7Aq??cQbOzhh$-wQYjgm%V}a`<}kli$1)=fCqYC$plc?*rd8Da?0YX66EgnXAru z<9Q8ir=NGTIppSrDE_3AV4J(x`Fw_Z^hd2_)K1(EEL@x^?mr`PxbUU0^M=IcnhbrN z`%F)PbE>cFCZu!Kx^Mww4S+0+&n}-duExrJb_fGGaE0!VcEMkvW{VTG8$X|S{>kAC zlLkh8hv}~8g-;M%U3?dXmK>yNx87qcQjQ z9y;9cC0x*gvuBCCk+gpxlo>AmJCa2Zyn_H;c}H>;ZbbDNuQ$+P?mJB=G=1*UL;vVi z;nL%b2G7XYyr-6qGg^J4i&#w{ro8piq0L+-XC}{uk=095m}0CWGf|AW&+C%%&b75u zoQCBO?=1f`#x>}~vRI~Be2L$u2rCHt+Y8-X(6z-GZqHT8u9gQfDZhX#$%luJ3!|M^2R8U$Jh?p24w7JY;z*_}(j79g>&HpOEnhc8m_4Q}RRI?(BR&eKzmRLu8bj0^b*rM6O*`#-Rszwhw!^OM#J zi0QdaQ=KvLPFSyQPUQw&iCg?#3?Xx$1)=LRl^iR5QTMeR`VI<$O;~F=%f?VZQMCKn zy|u!NxP#MkT){=lBXp_x5S=0*!=KjUr8M-~LEx ze)>g`JWfJAS-aW(X}?hBbVsJb6}Q<>#LaJNcS@|y5?RNZ38Xf{ZQ$cqa&3&?sl?p< zG@6|CWez@ooUCj!>XCcBFMYT!)luM*eVw$wu+VmC7p@qdO5(#>lCvG$0{ijQGiv;0 zhJStZq65AGU-$D(pA*rc@S!P}+53iQ@6n^XEd=g6=jn5N2~Qkw6=;xoZb7p6eD?to zPftFQH5m6@Ni{h4ChD{`>5>R-Bg6o%_|L?2+(V0YyJM|9K!|9-0+U}3E7yKXFGPnY zh7DZ7K-jQ-ZI6^A1Qj0^6GyMFE-IIpj)h+XV`mSgU)j1Ax>OEA^%6-8BSw;$m(YRn ztm@ee#gAlB*`MNgK0J5(1`-i=LNWW{|E2L>h3e`J;m+Nabm4GtBeCF_ zD!{rnE_ex>E&D+h*QLbu?5h4!-zOqxc=BJj!g}eYHsOLkayx-rEzzJ{O52fn>+1th z%QWV=95C6Q?$5D4)sbl{!p^mO(P<;yq#+*eiK>p9U6z8Dm!!na4|51h!U29k^q@D5 z+Q+IlP0Pz~>-oKbz`ZD=Po268J1c?HT71pHDI0j&C{KLxdfD==mp&&uvD2q0*}&QK$iaWP0w5O;&n9Em=y^=xJzkwT-PcY&GjZ^I{|4yZ z_>n2j-u}FG4GG4}Dr)`PRA;tOZ6}2x zG$$>ohH_2(8fiwQPlh#$nbe&}UpSaTo&HL`J#e}ulHZ}?4B^ug&)wzjFZKMSaVs&KH z?58^JE5KEi;UDcLZg6-J5XPJZLs02ZoJraVOwrU**w)2g*4hXE-i@7&16*zvC{$Ot zyPZr==Ql<~xT!V^969$r?d%=$(A3I!vr)}H@l$Y z_POcs!2s zXlD4*@|*U_x3OiC#=_&m2vZeBx@$yoinjps4{G(Qhf^(Ngc;e`{6i=U zJh5?B$t4P$Rqv%vT-Da~zuvSs-~?RR@WO&0bn3h(#<(i z110vZG-TY90d;OIUb_VvYU>u@K9O_&)O=K(pB^<9x~-0D8Me;=jM-TK;`CdGnVt-X z8Qq$C3JM|u?0H%oQuw)&eY-n$-+YkgmC98hC^WEqN)U0paRu`HV_&-~Ay?}ev zOm$<=lXqmAh1DdFd+qFshK7_M?i6dMf{;yk2s_IKv3dQ$X;a=^{qBwClt*{tBMSB* zijUyo6$kSpybx{C znRVyK0JGPxN58cI%7jOYA=_w-L4=EkD?@#=0Mg(pfc+5b1^<+v3saq_&O9Vvd36=8 z>aJ{?v^gV3b*nknFERcVH{`j)ubN;8Hj$+$&Dqkn27ISrG`D^{Z3?f_CkK_vD!Dy< z*j+1${lY*XK=twY1Mt$p-efNz&`e?&Wy_lj&dc@D@`i3`xg70s?b)@*PU)H0>U`OFE~)a)4L$m3K|lGIqKq>Fh4d+68dsqJjP4 z{!%$i{95p!oinA=g`1;5Hq3pZEx$N8jBZVR>RLF{CM^?ti zR<;NsLdp!;<6~rIMly?xLNdzCj*RS#>|OTWe2m|D>As)mexBd=d;E^yar8$=cXzJq zeVy-fyk4*KJQ>r2`hR}V4Xsz?KRkK;RpM2*rU$cvNqNqZK`&8EWHQ`o+^Me0ZRJOay|NEv>rd;^t8_-Ck*6PjjUb*m%PpL)t zQ!txR5xZ~d(^b3Sz*)$gI&=Q5-E^A}p*h9EJRz&*$6HhaPtH#P z81($q!R^~`gq_~6PBvUM!HYrfj%8SnR&vgh-f1Lx1&OOvOAr}qMimUlm%zISe67b7 zaKE&M4Yt?twkN%OSQSWPON+SPK<*yt=5yQIE3SHRe%s74z`(>dNoaHC)73}?s*CWU z=1DJfJV-S0#VF3U)tUx{6v?PP(%o~1IxX6!FndJ%S4y}kD~fI?hUK;Uh#ojXAEKvp z5K(F*N<63AXpF}9K#)8!oFd|~MqDKnUAH)uc6PhC8{x#j?t3dofc%6Q^QRKceDoA~ZNUY<}5 zFB^<}eG0v=J|^khD#UqYuuiZT8xMyPoRW`46}K*N2fnsUN&7nN_6bTcI_zTLp2^Qg zD|97#!+szJ?LMztVj&7VpFee>?%GzyPrK$5j#nxZj>3tJFjq%>@9y1M+gZZF>=gYAu`({Dr8EOoETx^vkz0=57rmnK z8jx?1R#`>+tx&(1o~R4vxl_eDB#>_-{5JT^-IDen+YJEl=v3^@Wf>82mR<-ND7IeS zkUbBbNP!Q$yhiaXM=Rg#5T(QCX5fA`+Ng_>gx7WRzZ_tm_6NA+_9L-(<2Z|P7UW%%BQNE+%X*x&?e z&PhgJ*DR<_M$Ltv?Rt5+VMh?9nl{9feU`}7`Ep}?I<-^wG+fhCoR7>ykCPS%Ic!3= zm?Aeu8$%iD}Ry*GOR-j#ZPhBC#(?>oUCZ0KXYI~tyDu&B|LJ}kC zcLtw+AvN4}3u7uRjbr=v0>xmy!2x*wC?UH?NW$T#jk^0mqf%c|JG{!M?#ze#Wkud! zVZ~D>ln?$q7vRC^i06<5W0R(SPcrNT%*!f%r>z=D*7s!#XHnZh?<0>}@^t3hpr53B zo%Urs(80pN8yBhv&pV~k4fQ_Zuw`Z>vI^ObPi^~IJ<{^CQHgf*tz@g+Rpcr}2(Uwb zmqu3NFi%0wqAD)^YgHpJ7lzZH2jOH!g8jyfnyn!HS@#!E3?E}_VJ(t-P^Bvj#>4?; zKq~iW;@f&GG$?R!1$=@GAreXXu3&H6al+fKz#*;O355stkP1Ef7kY3VD+_H{5H~Mu-7}D)kK;e8lT;g&v(s7L#}Mu&#g2g}D!6U7SG~kv zAk}3fKS9^2Kd_=awkGKkZ?!N{3JXvDWyeNkjYJjSSxC65nxy^(N^GhVv383PnHA9w z75mf!TIAJ>S70?54XX#`AaN(o-MQ{} zfNgsc**b+m79TR8#=zr-hdEh}wbcyHJrrHdE^-yjZ2Dsd>2Vbam6<|yP|bm9>oQtl z-P11ePb7!{I_4q0@4W!xTWO z^KHh%U@(v1s1lx6pT-IX_cJ;{Ej@$&zNeAxcZa2voDGtf5Y( zNlYG>gGw>r!c&edCcMGbWNMQ@zvr4$f6|(I1XA=SMlyYUhCJIzS)Foc+Z1q|=XDB9 zB_N8CKZ3-}bhh)I>noMYlT1US6>b9L96A~~v*1=YfQ$Pp-jD=_hQ5ovp!-cR@jjqG z*#h6WV`0;$VcvK8B58n@ETujncmn)VAk9m$Ns<^Yr@ejsQ;^8;QX`>2l0ZH;Ca;x< z=ub6!a2Kg&KS93vuWA+wxpFtk9NqvMl%CV&dGOBM$SEk|@Z)K}7I&{X*vQn++d0E6 z`sarb=biRu6KF4e6WHU6Sx~LAMakQJ6n@W6v$myJLSxjpIT!(FVJMRKB7}NhF#WPd zhKn5y0%*aFKY?XU5*QRD|M|t;P}A0RaG11khpKhe?Jr$bg>FWrgigQ?=%!wxJUu?O z$>{~;+Qsvpfy_*t?ll7Rww2Mowg^`!F?jcYtm##n_ zWU|g6jUPf*+8qG*VF+R<- z9~Ffzin?CvKBp_+&#FbZqKkM+JzRAbF6h9e=V{DFEBNyQg{&Y0tNB-jp zYzijgWc=-h+?gpyM@{mjhNolkZffU}ayF(1c(=%}Eg5`~?U&q&xXZw}KlM3+&9MAV z=B6q~zG)(E)aF)5L0wOQtDQGXF?s0+GCp})CX0D9^9LEd1U}iDss{t+iMYIjdj5eJ zMJ#-rXEETFzLq(S-?Rfcgv0cwCvYOX@Nlirckfk)1hG}ZnhoSjoxpkKFP?|LooalT zs!Nn|#Q?cA;8r{EDER3jiO61tToH>HwG9yj209Cv{ zO?WBr+EjB)_&I992ft^Hp1|8ar{hM#djIElGv9Hu+3nk5R2qY9P?Z849|9nNh(GZA zYH8SIQF(}P!JeN@k85f=ad3moc>Hk1aRLpHJ{2ygNFz&$0Gb@vvd5Qe(1b4@7@jl|gqsvnoPJ#NzUub?wzp7< zvNogsE4!&giS<{duTkRF%{UB-cdqtC-mU$xFJU|7*0&)d^(2AOYmaD)IHE&RyCd@X{)Lwm z0bcSy2aSO3(p-ZoWq~I{#KegnduFb%fUlMcPm>E!v2ZFZoM$0$T_lPOV>yq%ua^lv z4g_WXEmSFZOYT>Ig2LGm2E*57*yZNK$7G$q*n8`l5D?(syxY4?cJ1Q*0%)leMur5{@esR$Rb+89pW zE#wR#2SASGZr*r&Uxm1ZO$t7{4MsK;)#28UFfTjKTkt$bL4Y=7mqq-m=&S%sR{|(@SFKZ2z0CfbvITd(;VFuQm^Y{V)%04w* zV5Y7EoKr_O7TJ~<*m*_!dB$g6D#-t%UVef4kKKjf!HFalPjC6K1O@G?pvv$Dr3hT; zjH@2|E$-N?jJvuS+tJ>Aojdg<7*O{=`L6b>>M$DN48=H}^A|5(d#|0NC!|ySga(S* z77KOgSoxWAUjca6Es z1v})7+RkufOTZcZ3x}g49>RHcA23QX1t6T*4)s#G#I@@ghBvb|t}oHU)Va%#k_39M zz_`r|Btl?m|EgN9M!|>2XY$`l7Dtf0KAGiQEPzCL>jLS<)Zs^A&U2lH+W`R!j}iMT z>ri+%!@1!f3`F18@!d|7Y!@bI34Muh-${%P2@xpy>&64=(0Gmlj(ro)=7xJ%P;NR< z4_zP$v*%^0go%$T@J8>TR0-OB)Eph^xHNP-^Uk|7LA%DsH|_xw{8(Kx0d7hak3F+Bp}8^ep*216IW{vIM;9{S!}E-#1nAE+L9c{ zP;YKJ#PK@jjFwRT?xQbzEH-rR->CH!MqK{3OdzpG^cwh~!iLBrUfqPVGp!BVjZDXD z{GP@i>}{q(d3;)?z}2JfrrZF^cEUH-6Y?T6AYe37r18I8My&stPqp_+TvL#K8GS}b z!KL9c+=g#bmLQP`==1mg`V{H|AEkQIKV*QrG95I8B2LLeBjf@36ESQ3f|sNJJ_w|4 z`R{}HT6U>LL=SO&Il{DJ^Wgg2w{Ft2LE|`mK?em6I~V}^tR?pQ<46rX);Q0gW?cFE z(lPXlu0y(DDE5LF0X-hjU+jM|n{@01{!wvjGTY4mTRrV*%x( z+2h#rbshmAy8--2^DT@>Rc_T`yx86hW0n&6I(*?3uM={u2zzdd%3<{Q)lF^IZ2w)YqkLsph(9xWk?44+d^lW)Vt?ilJ#%K^qd zja1r`cm>IW4hX1YptwOU_C4&c>_gK^UtG8%3-Zs-#t{`u5v+UrPVU!e8fO$!`STFI zL2h)MWlweYl-2ar4h8L?p&|?1t(Z8HwLO(jdW$2 zfg-B}>H_KKq&F;nvNt86s@TsyWM*ZRm+o2aN+*($8@wz#8IZQKc*1I+0RLCFjTLyO zE6CS*exjnArUuRB{f*lLAs$9mCnf(xWO_w94PQ*ez~G~0S!t<~+~KUE*wg3Qix?YT ztvQR2^c=f^lmB!B5YU%L45!Y45kXc<8IKg(t!SmW4|GE+tP&fzSSX}`Xs@;f--ra_ z7fi$pH3zs6*S~pgytqlcWKsM99jx2waS<$hl%kK98m3yGy%IiK7s0<7u ziq~-e<`;oRWb9jS&w-J)O8N*a-(6^-1>_&GC^SiY9vY2PO5j(69NKKO61~=msF0l5 zs$m%?ee(;r40y_j4>TSQ$p?e2gGzxlab3t^r+|y0f=wBj!T|496`D^XxC+CL!vn-F{1f-ke@qt4KA4| z6S@8(998Z?y}6VZL|Xd1=i1nRQYNlUE|>z5hX={ zx4p|L1^f&Eh3+h6rau@+%n&J_KT{NkZZvSkQlXdl*357YE8Mm_LBj3#;0UTSe<9$8 zURUnTu{FJ*_emDf3z?ZQz3H|Ev*hfbg9c2Rt9%7pbZ;hg{ImNks)>eD0(31Wa3p=JvvMot0pEI+m8t!SA={uz|CpHY@=Aw~vmK@3 zT<8O-_ZC$aA#4;_3aDl_!vFKNqUjK2QDBC4Ddj~DXUURWL*jIvqSTAESfRt*QD&_@ z2uCd5iUiwJ5NzpU@rNqhw94C@V1C9oBU?V?ej%DHbKoUD#g0;`?I4LS$nuDv9nc6n zX2g)WcvP&wI&8=}o$W$R%UbEnPI_nxQH8|7q6zaGB6-bS(?AH0;Spu*psw$)mL+;00O%E)+IJV7)-Vl z&?swk@z23xT}2L7dvRsrJ%vLOnJ)(W*`aqJ=F;=JU`mA_SI<5qq85DniUDvWUtz4n zdmW&g87Q$$RX7hinN+0Og>2?GZ5R%1Oe~6lL2^F?y3vVdJ^8Ii6eA)clzW{4tTqKe zeS+`=`<-G*5W!H``U>g?WPKyzhk~x*?%mfW&?v~~Fxx3punWNTb6RXFVMlgI5jfgB zTup#aGlStl-&#Pi5DQ`2I&}Eie_E!=3fWkI?5^PeY%Zm?*8RNWI8upDh-6amAN|fz zsS=y$BhCU4$6X=@Pt8EsD{DCxor$Vz=SDykM2TzMZLh+sUaJS|_%Kp!fQ@@P1 zer{YKMTM%~*P3|#HvKN1?=zdEdn4!zN)ZDVKJ-Jq4+7OM4*m!k_o_cu?wM<#`P%hb z`5FF*rMy+?t5K8uRs+(!k6PXWOV|mQc|vah^y+e;)gU!`h0F?~p8MQ*0J0Fr<}``7|zx>UOBj_;@n6Qe5niv zA^frOS6+iPC+4p`Y7VC41giPvBy}Y1*soEnq@Nx!E_<`*qv&hbHBuhp%g2b_e|<3V zfm-w_`*61hZkKvAt`z2b1rk2c)lij{o&-gZnzlDjqa7@ zBb`DEwYaD6dLG?wTVHOmd)%H%FOc(r<6)kdt)*mvsYGqLKD;O z@jgdY%~t~d8a$HYURjK zbP+GUQOifMDyP=F1HU?@a!ac~i4IH5El$kpbSU+8|D5|3(Qij&f+(0PAJ0|Z>-~K6 z^LEFx4#6vUzVS9m0>9f%E^;t!y~X_7y^ zUseko$3*cv;FglkyUv{bJLSM#iL|+dOYCui(@Mr0<9&c@Ht8Q4c{(`!Qe}_$K;pxX zC7dc#q5R1zNCvWCS(Qnn-{D!$h;Mi?*s|1dLb(`>P*+yB&sH@T;1F6^f}B}>>ihoe&*oVH;ho_t5n8Dh-3Km9I2y>z9-Y-;RM2U#=@rcYOXP(J~4w; znMQtUX;k@*@^INk)WaZhF8O(Z?1-YP7uj-}U%I3!&5+@dkBcpzr`21*@{j+9`kFi0o=MUm>*Aw?wbIVd8<)FqH?PXK#Ae+4e9_0$9J!LP;$&Q+B2iWfrEP>&~)1^*s(TNaJ6 zBD)i-F)nuGISI{Aq!9uzWI+xDtMZM%Ghxbt{$DQ@V+syJ35RRtx;DmPWaHk@F9(?n))ORqNL!vWhWcG$t*v-Z_xX zX>l6712J7ptI*zIee1m5&tzhp{o8cs{V<;f+Iv8;d`kA;Ui_R`tow+FN&0RuHli-G zu*@rDK$r# zW7Nsw#@`=VY(RIxAb|URoD4hp%20UNfr^_+9@IJkCTr8dMzi0FAE#L@p9ik3&90Y% z{vT3wq~pOh_HQZr2|VJSO!WYdgH0{haX-o#&=%XC`5ajT3c>ptrS@jmjP_vE$}BM9 zPM1F4FLS&ByYkcyCxW?1w-sJ8hpLzPQd|LXL`|0cZ08S;d>Ac_1-3T;^2wn-;yH1`64`Z{L&t?1h~1V=;<_5_aW2C~AFW zQ>9e)Qv>6WnsEpPhCODBMycH0=iFt%Tb|aV7a!l)jm(XDYf{CLj0_C_U6YcMGUG?k zKLeci=SuU!rUK_oyYuM&>O>)reH?NQ)fzqag@W16&gZ;GB_Gw#8XFtWzEkX<{h2-Z zdFNHUtT5>nG|HH_$}W;9oV>+_AruR zNQ=k?wkh(?_2wEpdtY-z)D+2d7OCepjIpXJ+i(bd>%kEkDgl0!H*)w^50*-M_w#Gj z99TbAPOeuP4RVr9FSxZjs=OQhs5O>R`e5pub$6D6T~z|pi1@=CV@zu=RYz&zya~D7 z{B+>_B9aEf7oPw5&z~m7F9Lc-KEYUWDk;ZneG%LCi7=*9C z)~95?Z}w>c-8y8Bpt0iB={O>xMY$^2|8xj2l5v|%iV!VM=Y==~I(L-bC2$DhMJr73 z&srP)iq(lF!{hrq=tPvUQtvZ!Roy_;`fjH{7rH)FC=>Xfojm_^{!ZAg1?V?Exv5L* z4$xsF9Zb>BkPBsf3{spTVxSpNjc5QXPtub(NJzyW;<~-mdM`IE4R2f6enuRoAd;;2 z651rdM9W>f@DSllhO^)S4-> z%`iW!HW_wB%@^>@2P;DhdRqR9-B~gPjW6GkC@fAz)%+xibnJxV+C$oKlNdgmY zY`M3Np*J_CX%h&^8l!~B(89)NUS+BNR9$**`)h^7wQRL)W25Vu>Ex6{qP+wa@ya!C zRr00$+%1YK+btAf()aIdN&;eI57u1PTj_g^tJDGT@u@!U3)*?%btzD$il9njZAz)( zU%Sn62NJn!!=XRv)=phEHymyc3}%H-jMRoLEg?HBBM*n zT;!uUbJCtp#}CAvJX$nqj{X7Kl9C`8o+X!Cz82J|yKB5UQP=%e;^g%@ z_uaK0fc}3JxsS9aR8Qcv{Mg!@-Mcw=2t{6o>5mlqwO_Ywzaw%SgM9Gx1u!^JO%u^V zSGs>0G|k!5BU^i;4UzsxDm*#~sMFn}b1J#5lAazG)!^)9Vt^A<2aUYQVwfY6-p~Q^ z!=*+?b_Frx7Fidml6KzkprDkSg`;U4o`)vdcxO|#FAv=YWFt2Db!fJ3k=iC}A;E8- zfOBa}qNSIbT4YD`PR;BK4JJGnym_?tN+DDBZo{DHyny@5N9_q_OONsrDtkhrxiCu6 zw!WKoF)Acgzpq_0`#!<6u_T;LAhlnxLBPZ;x1hq*9{iTDtvy^7FrGx_sDV&{vxXr>Gz4sw; zJa_zpe%JAdR=!BObXe$;HLm}ItIV>F^!npr<2(t8;Y^LE=MmB$u6mkr7V$6b4=9{pxbB1YDS;OWFerKYfb#=}aV@_%&5a z;aD{Pyw}|0RN=^@`eNQpr*M^x_ZkJ-Bzx%WNmT{SKHjA%65uJ1uq?#Ju7{5|Z81My zm+)~ByDi#i!{-@w@1=MK{&w8jaKMlAx%Jtx(ZnTqrq~f zq3M2uO8YqZ7*>_uoJ*;yVF}#vIu)al2^UB)0tSsQLYGFXxSF%>=1QWDmam#6@UqKt zz0Qqg=iHUY*C}*7RS|ogYPb?D7nQD%6c&~w#xNMmyP)10FBCu_x*q+}wXlV419NFx zOr@VYu_Y#nFqI2MYiL?%x6EG^*yps=CzFKUJC$uv_LZ(kKv9+#LYY@5Q!rbmv0$~zRT(hqk$#g+HG6C zUyJoZymG?zxcKtnzCQ*HvI-gu?eOmm4D;38<5*^U8qmq~g&Uq~V=NB8EJF#4G}91d zJlv}2OH0?f>XR(^*nje9yi?fb1qXotbpc2%(-8|ZIMkte_!p_ro(D9=n86hN?otN} zq<1w#svG?r50Mf87>9#kZ&-O9=W#wFyd>MgGwUg0)AZ}q^Ju4q@ zz1cx(IAOB?XG$+#LTW>4WiUNHpMi0I15W>@OyYFyZ$68k#B%&=j`AO2FHV)mF_t`Q z*_Y?ds=*zkKGeEC>Ul&y-!U2%39?bZ^vE*R-VANZ2qDlz2_nB`V`a(b0W&|DFFXP* zy=P?4(aI#9I~>Z3E%~;yId>zi(BkL!Eo#EUmO3(J3E7R)>r*Z1VT-6vC61e(l$S*8 ziTOT^`g$0Al8?3?E%bXj{hh;#Z&F@-D674p+fx-)crRsbR54Z>?Ru%+UQL3me{FnwI%d8LM~RFDM@-h9>mZUDMhADZRV*2DKnxwO{^ilKw0Ot2pnpy8_p= zhfteX&X(ECwA75%Vk_x>o`U;x#u=gmqr5@%-rM_E<5AnfLW*$bXJ;~NH;^GipLclB zt{#%`xad*9#WzGvN07*tWOnjKjPxDdd-*TcAw9??<|_eoPK;H7n4I5+7l6oUTV%vysE z+frw-R-<$>-BmGD^I)YCW*70+3sKUnD@A0+k$?HGO!G^pFk1| zYb|aXBf<9J5>u&)gv}5CDj*jH@GBu#t%*FwBndpFho>#>=IHg?;B{7cdfLXOwAUt# zJ||zI1a3mx{RrluJg=ouIah5xR3dRVOYfvr|u3RU^N>@R*RWV+$f5l%dM|W?K zOXE}IV02iy}wNSBeaW1m>$Gpdc|} zS@%OS%MVAgj5nRO+@A$_!n_!>fsor9vK0bDTYSr8a_!Jp%K8T7^wo-1i+t&D>0r0v z(TDq~Vad%4F1#5YwXXg=PNz|NYoE95B+yh1x=NQiH|})}IY_vW;`x@^pcq)AFG~xJ zu(W2R(e{NB2?FfkzuC8Z5<0M6JpcXouK+cf^FmRY}H7F~~RuwXL4- z4NOD5gR?)j)0GmK-FsbyDiNiJI1~57M&e5%&Uq}*AbO2MJ5M}YR!k8QalEGrH^Kc& z8$1#3^7eJT&4sHih6@(p8A~uc6CQ87Uvb+X&ev&5^3)#bzF@1xNQ!4 z@zvx&t~&rD0+UUK`%@$dI@QC0lOSL2M!H{sgkTign1TfA5~6zoZ5Yc)rRg!eW{U{B zId5jnK|s~=-k2f zOiyR7DyLNv=l$|qgT)6*ypLOwvj%EO%w0Jb?FzQXlx=gfe!GfQqCfPGDY2?=-32_I zRz_Bqw93I~V&}H~?=^BX-sw9_ca>*A~}b~ z#exj)?zl(PR?b)%Q7Q}>vPNNMLq)!Q>P4hhiCGBy%c}YdE=bLDad$xJDJnxsY01c> zpv{6l?7>52>NsDcy->z@xV~3!;S!w(TY8H#RJ?;-l*L!n?f!W=U$k3g=TYl zmv)6YA)oUkM=f9M?*8)hoiArh2!C6yphq(F${T5vXt~Gp z!<*3!y+(1yC4##R9A{GTRw6|SGb^*^(&vrc(p4;}zXY9s$#sw2;8>=O!J?1IKa(LJ z8`^E?>qNqpk7cdM>nC)G3nXKL?2za$A6V%+=ppQI(_y(&ogx$0=rebsqU6utL~T=2 zr_p;OB8?pl---+IePMJ!=4dJHJE8Hy5Rt;XMZX(TW5iN9bwxLROPoUHuC%%yx0cZ( zhUeI^g8=nx*om@(y~#VQbB7?@Gn?s1p5TF~S*4b3 z^f}3r)<+DjqGbx}6EFNcS~N1XeGY#z z3}l(jhqXot7+fxF%-Zei(`LoUpRygDJ-IGLq+4`EGCfEM8u7JU%_|LN4_g7dD-DOelR~6X^rtf|?DCf_^-}YKe;ndoB z{wU9_-|5|7vL=_)k~^QQBwToE(#*DmUm=zgK)k`fqb%YL1Nh|3trIF}FEY}Lrc5sY zveyhmKb8>UJNJ)tbxL5~4>i;XOWH0`S{qrTAoC3SUfeevJ0*SfsYOAuuqC z)a%blC%CsLbA9oKq|{25x!Mi|8}V9`G?I%ys6wA>NXWYYHGrQG3W?qZnl+57_&{&W z%{lx}lT6;`J;4S>8JOgi4!}`_JBdRQV~3rF>*4)~E@-Wrg*`FpAja+j@JZ$#piYSN zbOR=^;SW;L;sdazijwjt_-2j!;E9Ne)ALaq$PPK61bJy%!|)#3xK;Z zF}gW(xzGzrQm2J*cJzvu!h0APDh8HLDixp%$d>141Cpkw7@iXy8R4MC>|^z_WfEy+ zvIh$y`5(VmjF*S1_&R8wKKZhmlB`t3X=dxm&=wp8=IB*Utmd(|5zWub=U9wQd4J_gbT3YFLe5tIS9MOGl|S96T((Umv#2-Q6dup`_}oMQL!Sz% zmd;j{$%gVv3V-I4r_qw4m<{fz-Dc1QQKKA(Y`!QQB`S345)t@(;nqB;jK5Ub->2HO zFzjU7*^Tedqg5Yb{rFCAxTW=a=X}}rOiBfz(~Bof)>@d2zwpk7R8Da>3rB90(+o;p(kJeo&`Xtw1b6H8UDgu zijEox40x_H4SgjdVvwH~EeW&NBhBz>p)ePGXms5Iktq2)iDJ}k9987Ndla&R3> z*g6~3<{Hg<;tl^MzqtQNe%TN|I9h^9Lj?gWD$k3n=lJG(bFZFbT}M=1Fs5R|$o+U$ zR5_r9;go9H{9wN9#Wca^kS;Xb^`$RKWmRwfcD-Es~~Ij^SVL1VHcFr4Hr*6vp{N`TWo?!qbAfGK7f zAiU&EUqvPy;({LyTMA-w8>864g;!ulY|t z#V%?ZgY2QLUdIobU*79TCwYj)mO3t5varRb<)q2W3Jd8$EpQ88UeIJU${fdH~ zkgQ@e%c1hHp!0mP6IvJzc3nTm#UYMh`3jv#&D?xZFOIVjtE`CAV~Vnb0Qhy5cF<4nEWioORHCday6yMG&|w+&OZwG80bT#Nnra) ziBi5oUHuLwCitUZ_qpR9`evJiDkx)#`4KQXAzQgal72{qm=g%?YS7K1R@}ydFYXMQ z5d>UUlccUcoFHGSL3`S6^-B12d1yXRTti?jzk zHqtJ{a|spfH|oBwbT8>e3lKoFpq!7qFUTr(c24M%SEk3o-sdo05Pl{56H+zt9J(~4 zobKIlfDuY0F1IyV;}S*PI-x()1YgeV!<~0Q1@}#y7l(+TL(UI?UI6i92XBGnCoecSh_+%N!qmLuDOf35@YvkH;Llttw1s` zM`jFDlAWB|87B_DIHVQ`twUe>^{~UTx%G>Q>8V$VzVq)d?=NswS`u5wT+@oz{J3|w zXjNvaWj4QHqL_RjX>>N6Fd@yFzS5#%#fUBxuZEQylgAYOr4r2yrk56vh-nb8*SHy8LtVM6UkC5 z<{MFpUZuxKYGK@{WCWL|{@(mD>Nj5x!4cIpt)Jz{c!PUG$;QHq^e+uNHF1NLRt3r`Z54613kA4i}K{t7HT6^!`C_rEji2ED7xDae}ho2`5f(8>FqOphXw5m<7$>yJFmzC$SXBR8tr2q7Gn~ zi7Kcw>t5d*O9@UEyrc%>(`I*8ClTP5eqvg71b}>$1qRfwIf%OCxa+GqTB&cY;jM(v z43v=MGN3OaSwtq17e1GBITSjAKXmDirds90CAdI0kUw?})c=x}njm->D0j&pv;{f6 z3hcgc&|UtJ0=lz_EGMzpfLsSvBKCi&5*3gXm}5Uqr;xi6onc+2-(BoUTdbaI zP)whFuR{6f`pNajoQW^gygYZ7aLqA?xStPwhz7e>A`Ikp+q*w*U9#HkTIHTR6=PKzXJjRGGB-&TB`doPAgUv?!NNfTP9sj;e% z2!5136O$A(^L__Gt`5=<(y#X9CQol#KdH68sVXCX)>B`c&w^^yeXJ?G$;e6N<-PMu z;Vj!rQ<|j@g>L2@y-nVOQYq@Xx~x>30O2}%k<+NM%p&X}U)bqQT86A){Iyn0RF$~c zz-RQ!1xl?)f6-_UUs4*ldvDS9Gg{eA`8d8YJsetoXd6G>8HoF^8~5X;yxk4Y#G_AL z0ki#&?^oTHTn<=HBf+BtsZl9t4hksS=W-03Ua4^(>+i_=v8~;NI?!yVE_d9D-Zd^0zOqa`<2;@8oWMmc@goAmT+dt z7_rys+CE*KZZzL_sk8s93x1G|xZuA#?0nB+g%!2`nA&8)sEBSx>6NmLi)pQm*MYuU zW&<6@;JMIueWEt>TN;dNRF7bg`9@|qly-n3OeXD$menMy z8;mBJg|!{wS%R>g|`18 zSwL%9_q^sdqb@Yv%^A!LKQt#Gye!zQ+8)s_Bq^LuBy9iwOaAotJ(shOeo$>%R(XDx z@zBvYwsHDH_TE~ZZ#q)jJ@ebXYk73PjQ#A)w5f%9UB{fX$Bvm; zk4Ht3j&x2N837@wVaTrQm5R}#!sPV+@OV)<$AO9?YpH~ikl)SG>RrY0RPnFN+*#>r zpkS|*@U*-U)bDAqPF0~ZMP9pKZg&2=!E2zQt_6g!-K?-od#ERhW4F@>ERE31&Y;F} zl^%1AcS$kqHwtVY@0V_9xt4vJT%Ll>Ah$5*mK}RvAMP9T_T;EFs2&Xy*=4j*)$>#q zuTQbwis96cv>G+kqaQkD^f`Ew27K>=imQXr*?!AiWxn8R)&3BXYT=e(d z7W);h%A8l~r}w%@u_9nK6awnlN3nGYo1Oy@>?zyOBMxVat%IwdY|UDd%9Oz>pv4d_PAFR_(3DvlEr5njXe%( zyPlfGQI89Y#5rZFSh&DNF5~4B4F=U5z#gxTLW`m!gVjNz{DOuOWk*Qez(0zI7ch? zaYFJkkU_^;o#X$I)g@hhj8$rL_{=aqAZ#fW;1Q}O4aS={|OKP}y8Eg8k*JMTA&;(>2)tXX5(1=T4`x;`>?Q&^IGu54KXL z8smH%7YCPCS2Tl4P!4N1=6w2`KAdSlaB6yM6!W9*v&$u|1h(Bsonm}h=Z zk8i6le-R!y<}=^;tsB6>R?t$cp02*l8O9>M?`j@4 z_wvQB+0y;Ge)qgF;BI{LBz2ES!Fa@k(*@^Vz=)!Q1xf$Kh$0&FF)4Uo*O#TB=fd3L zagP9^3$Hi)SC0Ul_^;s8v zfrV0PLuk}uOPu`g51nl^We=~PZN1lMkkXj-IfNID*-WWCz+7nh_~3kty03DQ7-M=( z%1m=_yE?c&lY6zU|A(=+4ydaAwuLDrl#-B;mKG5N>5`Ogq@}w%HX%w$cL+##r?h~e zC|we=326bzO>>{6zQ6mPbH4lC`4OXN6L z$?{ynY;ZtOkJewc!w+ele!(ak_x(=?s3t$C!A1~o4-Vi)68x}yjuKj`MoP)?500fq423_*cztE-$&fTK*4 zk=vHb{B9o4{;PScj^UVaDHTaJ0uo?sIY>QSA>~mZm_svTLgep(x`>PeAY4EXDj5Fe zWNl|STO29@6+5O*orOujED5zE6r}F?uo{vA!ox1k>P?h~(6};+EC%5zRKGfPLFMa8 zac73ZX(KsLp<(jFWRL5Zp_?j=(Je9p6%nWP!4`Yf)PFt!s!PRyS|}MRA9Fv0^LlnI~^(@85D;4_8S~K?6$;O zy%UGt@Q?>BZaXwzcx!!ER9UpL%d)>>1y4)&yKr_?zw?J9A0B-Rb8cpDJEYwfXw{vW z?@zfb65Ig!{(*yirt1(0JJPN}CpJYm%7a62AsV$8PDTvQgU9imIEpGTbktg*Ox4JN2}G)$&M%Pjz+Pn9VcSdIqdpP_W;g< zzY|W6Sn#XJhiF~EsG!H%F--B4K9tvU;Cm|fa<@WkREN<+-v@IB_SY5Lpc!^*j7Q<1O5M5bR#EH(S}E5-&AispSZ?*96z#cXgw#P5EXVmtdzMJSDXibf)t$ge+wk}v5d?#Z}gJs1idnHO(i zHtV;*UcuxB{P)`qk_%S^?U^!Jhwc)oYci)7$6;^ZJNF9snwsVjPqc?FiUmz=ju-6S z9rW0=nP>F>SJUQ;q5t)>)BUx4>qMK;hK`0W&8n42c@tZjvgD-jI~Y8R!ehR8_~Nk~ zItyffS+Cw-g>79r?}@jWlh~Gb{1(go-{LO;qE!UEjtc>J+hBXQ;=|XuTP_K(s0OO_ z9ufa(!(J3Ww-<0A^U_V2EV-ZS;=!nC_kf^ZV~q}S&U7LtF1ZSEKU_muko#f~r${%213^*-0A z_|pb$-eVqQW8)j~6SR$|0K8gkZ1vd9v{lQqG@5C%iLT=_=0P_OV+D+!pc-@cP!~?~ zewpg(YS+_Iv*~iZ)0(;qME+=l>?&Z^AF3AITM|QsDT21$yEjvVn*}HH^)5Ub4|G2& zYrdDI*V*413a-f`GZ$fr?#po@&Z@-_s;KjwcD6p#bPIA9|kbJA_!K%XhB}yUhVIWevF(OTs zXr}o|4~&mnPgN+icnHiVL7MD3HKv$Av8EqOt>+$;L?NMFK%({{fxTv;ow`sYAbz$L zxvT^c_8Kk`<3>N(4KlIjyp9R_{Ka8DN69?SlV$89JE#Inmwjs|P6WkgFAnj&^p!X>E4~=MMRkf<#AK}76A5&@vm-6>aTF*EhKz{-?#f%7M<>L0 zUN?}5xBVUu|J|p}{tEiEzsJMD5~R64A<)$SD@^tyni%Ma1aVd@DgdYG6JM}qLa))} zzj$%uZ2w@9%V$-u@J)eiV%*i-89D?=7V>-Cz@E31DAg)=(^A;2_fGEhe>`Z{W}xyH zxv}SmA-1R9oE`5jt7nd*5IZWR?_dAaM^7P?9IM#9R=;f;^ET>htcx<<{b$O~gW4}c zWd@(!5A~J7`%KFxlN3RVC95bssPghYCi+~)wfgSB&#ym!ejofEJpH{fD#Us_jMSg1Nv&CZ_*~H~_ zrd}c8{F1=?JVynaYPDSHqh2{qzZgV3{&-@lef?vB1Gv_Amp%wqNnfPJlj80KS~D!k z!odg%p(y+L3po`hpf!>k%v{sL3C+5}e6SrYD|}xg)?2j4K-Z)xZ3pg8JvnoF6HVB3 ze`_a;Tfr?NK@ak7h~}t;Oy&BIh|0+p z0gs30OONj3 z+}&p#H*YVsU$%`@dbwcI?#%9q;b0&kUyHv0&}B~#;@2<7kEfUf49Y{9LPXQnq{3x@ z_n<%14r^J`sW$#-35Kc)+C`WWW08T>KlqOw5=cF;j<-*EDZXL^G*Nx4-7S|ic|a0> zGP=C^vbyn?bK=5a$BnJYvRw1XdaXM{hP=tvQ{^j%$v<{7TGW{gRlE;n$W@pamfHOe zqLlIto1BB=3T1Zcs9=Gm%lpKG?K7W-mr7-$4QM{B0l^Y8&m5A3^5;e3(u+O#}- zP*Wx95X#bAMt7d2PV|EXOg$u&53fa|$>NzfyBa9w{k?C=-i_mILh7^(48X~B}3SO5% zSL<;J@(+6Y57&HPlOg8vS@dskh9Y$*#{|s2@UAbHrU>{@!e$1f?kpXRxo_*Lfgx{Z z{j-#7_Ey&o&d7UD!jstbHx|13C*+EqC;D;Dwx%l7_PX-9@rP17U$d|$_(kfn z*V2TZW9fZDLZ)7@z;JwZ+4cI?SiW5HtF;SKxA|7DaiBK#CHe;tU4Y@QB-ZNRVNoJwc(~K|j7Qc0~>aXNt^Q+)U$TGeUB!w>LK2bu2q{zVnO5O96qS zWI`B|;m*?@!_6Orcj8`aR%n}(l&^JO1&wEK_qp!;>Xn$gKHq^8#&LugL){P8 zyb%-WSQSEsNTa$J=6ds;(^}-DQr|?^bkSahH4Vv#cu_vXA4$K<83fPb;kXA&zR8@> za7V$b?%l|<)y+sOx6HlKFVs7aStezdr`%$4tc2>4)PCK>Kto2!O{;iUgfYwDSXjxT zwkhA##e-2mAzp`pLQ{O8y5%g+wuC)b#SflM7Z$vbMG36*DAZFL$RQLC#YDy3f0pC4 zun}K)s7z}r5+~DtFI%Xf2Ys1mM2_}qBH6oR^^uKOn2{tPOfENj=fks}w3D!&Pb2tRF2d`F%Z5jZhy6_%fjams;aad59Ub6jz zZ;X*%-OX7&*%&cRAV(-6gYiOSR>H7b?86}*})kwrhwdA0u;kG;%pSJt`Z z;LqKSNyA1izrEe@5d!ouCob7DuYF>ttf@h)6*k$s>;|09a_gFvhRpN$g*TZT1yDZ7 z&P7MWifuuhKTATHmAfzyyRpAp`pr!T1HQ1s%u{0F=KPyFMNAw*(3S z>L61I!7Ar^ien9dtZ5?t?Mst@Wm0U^{SB9el_=&P=PC&~*9?AM$}-5go;=PXW#9Pj5ViEK%Mq5Y&Mrg6gnAho(~M^!z{d#%VbVNAQ@ zvb>w*9UVosBqT}FuK55sVN#AI|_M>KHJ|8=MiY)1)mX*X@v`spsA+p_J8@0EpX3(-eSIx?% zQ+2=Md#HEC9fbJPn&ItPkK1dE4Co9Br)dCap@_%yifR&*?_INn$U0n7G{P&Fua1;6q04&Rq<)r;Ce=4?E%k zHp@W7t`gyXwO^{r_+;q{j4`NfryUWVr*Il7+>@ga*?GsoA@ERXxePA|tPZZD-F{OyiqZYH`zXkon z#bL#`k4I!xES=5N@2inwaOyXKzqL$A@# zj!f8@#OW8I)*ZO<*)(I89R5*dlkv`ru|oJ7m*u)BdfN!Ub)VvPkzHxjA=tgm`)!Np z-ulWL4!R;;M8z2``t)oM%spVr56tE&EcePghC1&qyd#hVE%hR;tf3bt-a_pleM)A>j5r z*5r%cdtZ%a(x!LWq|2P9Coj4=nS??_9}yrV7t)erqk23hi-?0Yv^nrE8f7!aTgPey14K$d3hB)l{4r}pI)d^I5o^J{C^q#XJU|i&IVpH_pBf1* z04zt;1D6+f?)}G4D1_kTP+g6&gI9iMcjN(7LH+Tbsq{^?=}>C-FhiK8y_lvU0oZmL z9oymVw}zks--B{vN^++x?DHc^(Xj$~txeu9O+igVc0PA34DC&MyRp6D)Ofbw0;SUJ z9NY|UO{+8>UphdZ zwwz;iFyjpt(L+&r7G@?Pg8MRfHslp#zLyi%mfsfeaaQ4jDNhG^%q%@py-^ajbHyY2PIjc?#H z$q+C^N@VC#7c+kfH&mJGF5cr{w_A<3;XzP7+7u}D1 zhZ~;$ItS*36q7$+cU9D{+4hoYs}@hyPk7mbtPLEM6f!2L3H+Z1pxRs>r*GFvm#7!7 zgDMyz;QEuj45|d}r93c>&3XogidEJIGl)omw$^WB6NHSdmG5IH%00-~d|zS*ul;Xh zBk(NOt`FAXvdcY86Ht0M04wOrWH`_af&_hH-`&tawtCy z;Q8yPy})xI1bYL=QA5uIxzayy5aS_&OfrL<67IeGv7i6N^`IBK+3zy=u=QCq<)wJB zYMTNnXI|kU>0@PYzXGMThN854KUTgLkBtrl7H)jqc>FRnQ;PO+zFgn+s7g}I*965) z_J$u3P3%)Ge%t()`v;26G?81!+XAB>qFyCvZsyMUn3+W0VQ)05XEZ_d?FhMwrxmu0 zo+03)+nAN@XIN$fNg5Ym|3x~N;l>ihT;MsT0I?L9KFK{;d#IdNXT{VZkv5bplGQnE z-PB2SEk+IEo1mOhekV76tm*=p#(p8UgS&1=aC!7uC37-b~_Z5=KvWC(!fF3ySqu1H)l?L8Tie^;6{IK>IYMY^WG$kf?Tk z5ZV(~F5d0d+6O{$plPPpe|10pB>jooWiN-}X{28u5?&}pAP_bqxV~J51a6>O=+m;a zBOOS;9EU`tc0)kgxEm6GGZ}m-5=cMdFwj_?u1~)(VMSNj8SvVo{Avi0E$$EJyNI?+ z7EM)hd+&!D7@4rY9ziSJ_|8UgGQ3bFwP?4DtsMASlfQ9BWpa!Aiu_pfHX3kx;MrfI2UgkmST1LK z1*6_{g5wc0Wk3c?@Zw#=1E^$kw_bRT2b~R`l5p@&BL)^*{acEH#PV-7Jv>nv@m9})r1$^Itc04QOQ&;mqtM3Q9Cs^V?e<2chE@0Qnx~ z0UQec1i)4*_>}ooy<8`;!FHOqxd;d*-wA}r!*cAv#}|YwH1p$>p)SZo3@vc{(AmJr zp>L0yEd2HF=D!0Xen{rC*pS0FXIFJ$i|Z|iFEReBCt1@{0MOJ>(R0?o{}LT2(NF>b zIrPZ7M6D34siAuZHWqdXExaRA0~f*p>mi9Q;5Q=J9$`8nW2}SMWQdL-LA`(u{bPC_ zxFUo{yB$w|K_2*dM=@{u@GVeEe#)qnIez;qX&~4;m7(75>t%Ef z)S}108(tii(Rl;6LKKiDixNu8 zQjjjYVw&|j0y~NRxD%zqJ^uKC{v!DVUK_cMN6mh&R7;D09STJofF#M3ZGM2;-z(U8 z>#rTLLnY7Mp%?bpJ(n*sVb}KrK%&Rz@Qc3+y1P|tcXMyagOAcUz=~dhnT#^!h~fkf z#=IV{^s2?dN08ot@>t)%PN4Bs3=}^I>A&MuD0!nm3g*vHZ{Ef;2TYKBpr@%}__4f( z9TainLn$1&oS9%#m_!$ZOkcBN47nVteWx5&x^Vv~+gqLai^>Kg&?|zgtSx zuD2$dDA(1hyF*4MYh+}!XFH-Z0banQZzuWR7XTWQJ1ywabx+Zjgy&KG#laQvPv8=buO@694HOgu--^Gi*$n zS$|8O2RV^PZsX)cn}Q}e$>dy?qVliSxWIhjCcL8q$hn}0o`;GWuOJB8|2Uoi4)A#7 z%P8egQpFQZYJEZs{qQ#;-smV&7`ESA6Q4NDln}094k&E%*J=EAXH}HYD_Q%o;lk_~ zN%(O(YqE9P=aTG6==m`pcf1d9iGWk^L7wOv3V>%&(Ym_3tu9XX3cL+!&9PyC2UD<_ z!hOu`)Zw(;TL+f2ExH55+Y0C`n1>+wfMa>^3OESjwA;7dqb#+*W*XA|6wH-3j~^c>s)a9mxuT-Q4|v&QO->T;6EhD+0H@5%gzV$|yTvvdV*NCg8P#=ek_A@Bs=wq=$i;HM3NP}IK0aok4<{C}Tz(2!Up_}OAY zmn4Ev)CHOe_<=7NcOEAU8A^>Fgfao9^`A#k`9D4iDAoqhTmPb02q^gZI7+Zm?<`aN z5&=hu=->#Jm{f>DrXDcl>$R0DK@t$bG??f}`AelhNP_H!5$3uX!ceQ#RrZ5K@9|B-d|>Uws3q{1vawFTNtk?-UhM1%dAaH9sIvAgp3K-f{SY&%xju$Qn8C zM5U7e`>U>&1?5obSG0@YZxRXF19BPyBe!=N{NVL(?BUJyqD;zF)Mj9ZV-xa+b+%~m z>XWi)c`?v_AG9Bo^v*oP^2dM1PM0rWq4CpQLz4s?c4iYC@M)oKteutOAvjzny|>wZ z2`53VMKFYmZ$JpPhwQHoY?(w4gXK*qgLt745qSW}=Vf&tE|g1-0^)x=?>rj2rg|<# zu1+ z^DC}kr;-uDANR>I(~>(3o(i0_su(i{YN3`PEYYCVb1}*&8ykFXU+Cfrqs%XTe+d>B zVZQVc6@}ipw*)mfY9uu9+@a)<=Wz7F$Cq4W&mXwO3_^GJD}R7SfEZ(9l6WulTr(9D zjFbI7p&clj@yH>avkPpZnmgEy&IK6*l?Ar&pKlj10eJ-{DPBi~1lh>YbCySW4?@p% z1gU{%TM$A6AQ0WB4Up9aXJ}R$WCY_-$nrSM-*0~wqy`k?*8w{s__%_FO2}IR=x)CY z#W;`ran=l^h8d!~*$aXE9JJKZbY$_gpS4mM(CV0-mU?i^oK${^y#_+1p(PHjdpCt( zXIu{l-r;~_AQpQ8z6gwNF)U(JNkQwBBn5-W+o2~g%!zIf6a{IvlL*ecY^*zTYIuDfw zvoFLYI`t)=EWW11g7>kc+!tXC_&SvD!-8hFzqd4!YQ3ESif+4yJ1x86`-IG@!(b#R zVl;AKgAe*~%nO|`GPr8ls=hW;WL|x#mQvTLUx||&FjN-tJW8Eq>2Z49NRfG=f&DbogI?`>Mxc z2&5jMSkyI15kjQ5NhtNlXD(^cK{`^}flp^}$8pwjov@H~z z6b>dZx(8)xAWD}!M(y}*v!c|(Q072&sH_FKdTXv21UmFBL$qH;|M@mA+`ylCGP4Uu zXFPyk%fah`u}M+y*zEmI9oF!9LAg%!;CT!Po01lT80R&N_w25KvrUrZ2U{msKF3SP z4wysMOeYobNS96X96aq<42`&Pz01cJ5a11_42}95v2ADSJ||jN{f!hNPr<4DpFNG! zg;qHKA+O+5Rhe1hl8jrx6JD>0=tSz&5EXNm6y!|IsDc5ws-rvKlRYi{FkCF`s`L9r z$Liww1?~BY56Z^AfCU@xK%I!`;vqE5lxy}b+A0Am%dCUx!A-SF%T4jSpX~gw5Q;PbwEdF4S&3IqfU&|>8 z(6n=y`;*wniP#PFa@B4DJg9{Cb_~57aXtUwX*dzRu#fO$}2gw~1|{f$NT5d6_7 z9uD+2Ex;v**Pi6K{dr9ol)zf12Hlzn-PX#L|!ELEuMg0pG1o+@#fWPpG5oO&2ck}XuHuP*>3j9C^x}EteUY%83IQu)$Zyy0K zc>{vaE@!Z1lLSeHnyv#>+cQ+d<5J^*UZ=wAH-vf4D_>MTtJ1Fz@W5Rw>6~oszTP#l zI%E=1>2RKqt#dxA;cW=E4h-+Cq5o10OIPgw1q7TVeQ+Ljv-*b6XY`%SGG1^n zR9&4A6wEI{6WO9!AgTrq^tM&l0WjtY_R){or4%p7EnR&GLN5j*fL;3S?|*{dC&6q=V z7rLo_Bk(^K{pEjJ^aa>ZGz|7nzYhfsUMXLN{E0E}8;AwgH84D^jCJc)ZS$9ci&{=6 z|Gk$VT@kxJ1-;$9fI6QO97u{J~`ynReMka5{Y@)Xc+Hx5rvItn}CS#S)**-klv zRW3>X(5M_&-#ru4eyni7v5K8uGK0;arYUG%7E9@cwYhI`Tt3~E;=sniypZL3D2;i^ z`D?b&b3YCRzs38%e|1!S4Xlaf(Zziq!|*)AVqXn@pHiI1cImG{P!6zb2<{AaNP4XDSSPunZq1S%*Fp%uT3Qz zLMKymb=Lca+%*-ccPx_d;w%2q0yv&NOn9IZteE+TYL6bkA+BqI@0k*X$@9|rkI^tm zPFk#-QQna{v>Afvol~2vI0Ci_E#9#Pw41_$NFDxfI3(lAXV^ct-ZjQ`5u~J%QiYy- z6}!F9Hm44<`wQiY<-zYH;qIy~#|CRcZF}#dn4b*GD`WY3i=m^Qg_f8=OM^ zS98O}Juva-@9L;g%%`dcW^^4tIex~4{Of7n-nG7xfs74&R4wDsrhSw9$2EpnxeESD z?w?49moc3nWtJlWI)C*O? zgo@BxVdi40Vgesa*VXuK6Sv=2EiUI1%aBpr60Lmu?=8yj5&hQvCj!TjTRiJMVexgo z8(2EE))YdZ)Bi1?sWK4j;8mfCd4KQ2J}G&gVEF zr?EiR&@h_T_)D4x?Q#)$fUNPprD<`+Z?|uL^e?xMjt1ZoaWE8lw}L@FMLA{`h(oX9 zPr)STvPAra!BPm>h%f{_C{^UZBE_RmBqL0JQpbNm7zNOCahVENcD|tw75;Hx6@Y3W z)%lenm{gY?7xnuMUhuf&O25>?ny8rZPxvYuJf8`QN}`^UA%iQprE zB;djw&q{jF$cF2hkMy&m9Q!w`$xhJwXN~>t&3ks3RvX}unFfbk zO}>F^Flv9e9IjF4y61dmy~K+(klt5Z&V$wK%aJ4nq$^xC-doe_y!TH1od6-1nJf$a zf>O?CuOwba>`MRA_(XuuuwRlj5QUu!xc__y;B71tQ1i%4lxS}$s{&x(3gFvaA`EB8 z+cw*?*spV;0!*|FL(umm2HM9IO(}Gz0qFeXi3Q#KsKfs7W;^xHK{*Q8{+^5@kcIb< z$7=KyF`EwIU4#lch^}vF9{HX)F8RL&EdL0&-J5~v$~j~wxYnm!L#X)#J{g^(a9QxW zxG)-({`qa>$LS=Yf`5)6n#+($gry0Lf}77U!-D;9;En6RLjbhVtr>P5T;Ew6!;{INiRyn#p#fKK z@9;dI@{Yq{7OzVJMDNrT_yGl9plRVgTnp%q;?KEc@u1476%WrCRBe|ei(=U~AnSP< zztlr-@a0QZOH4tXMmjN<*UbIM!7MaTtynequ9c>8JHWldGJ~Lc#l#;qQwYGMGGbFd z(3Y_ytfIMR%^xT&^1juGEZ-DrExSH<3|LvpJ;;p4CNJT-jTqk@B ztozPJxuJIpD_{L7gjdPYcII6gbq8a@Q<^$F_l)JvFbi8D1bzM^c^h(|C#atMT4@fw zQGs>4eHzKWf!E0pe1ie`VGl&K?8W0Sx~#7A`tVH2wQ-aoILINq`yZOzv9)tM|ImNj z2FSFzsho~_$9k+m*~9-hP-SRrR|_;N*EixBRTB#R(O!rLW9J#bM56IqRX@&W1NPx- zg^IcN>xac-;X+W4LGdy54j`Evu80Prf_o((it_p+^ahs*@|+z8O1Q^!tjH6CEkBmf ztFHvHkYxzG0%VCh(wPZktd*@#uP-ZY(7HoIF$UPA^`f`P^S_0j^Ek}qWr*zNk&hlU zxCW_$!WGi5bJM-`53KcVxeA{R)+K#5!WZ&|}j%z;Ii@sEdyIsmOr1L4`{FIa#bwaEnO}5MHhSSEkPOIqVmz^ zgcWEj_YR8P0?0O;22|^Cg;cG3p!>%9D3^eGK^O{9R7vxem?I&rft+Vmv?xx~JylD* z3u-T*+Wz1a+aF(&JwMXvpM+w0@n7iXe{rv>RRoAE8PmmhMn#oxdyh8Ym{_>Dg~P30 zOor{g2|WG=)g$)qy>s6g&Ru^Dt88rkgmp5|%ovYGI_!iJ{7d zY!s79vs6LA|5RhdkHh>vY~M;tGK%@}yIs+={ei~tXYuBdfNSrr%^bry7trl*724iFP zX?1R&)tFMcW$vN=QfE6Vjt$wFET}hg!r=1nXeDoLy}4~>aq{R{hf<|qo&CT~r*%=o z)G1Z6ClyL`QrJMgvqTXhNmceL+`o#jO*lDU{>nF2tP^Q1^^q*&@%5aLzxh-v(8Ul{ zAZt+e9mrt3yDiit9Xmm4Vd4o82yDtn2}utP6fZKqGMOc|lMkL7Jt1Pz8!q}*9EI!U zx-%Z>H(jkp)3FixhL~N}FoIGH(9jifgZ;$-WldS$dg3(H?>U!JY-}|cKQ%ZLW(d{RJ2L_^O2DEC65kFnolM-U@zJ|1aYpC9u??7>%o zs{SaDA19ph*q)IDJJ2eiXde*Z58W{Tp{&AI0FA1B%F(A@paJ(0fmcOoO*|5U2a}~!>@SoNL(v=qSe7Exvu9pbi~- zLrF;ZI^pDdB%4>xz>&?AT1Z1i7-p?hakIpH*70{jGGS9Rs33c^7@Al()ORh!rd=1o zJKh=D45*`dXHVE><1+~>DX9p4tPN(d<%pLYZ3_BR-u|2-t^$yr+prTAg|jActux8+ zrDCNwNrQ!HszB|i9Bfo`%Q>;zsOmiZaxAmG$-g$3nG0x)MH+Q}V`TW)>bVib^K3_% zoD2*mwlmFQCJoM~(O^pZ&e>72zT)S5dV%nI`|kp3*J79JfXy~qmreZp!fJYT0JkAy znzLJ^`nn|CN7utPCoaGKB>DFcPNuPAqk9}^C8pQWYp`p?=W za5l_JQ5QG4jWzek`ugyEh`z(4%t2W3Me;JeEfc zH%Ubj_WLPi6rk~W6(+C zzB!^um|aZANqb`b@cW^oz9t`xvlszrpzHH+1c)Euac)@P7rTc5os&3>e0aufctlo{ z$u(5Yx-vp48MQN`DZLx=ySH5gnpBc>-c(!=WS39&n*dBc_Adf(7T!WoB(%NTWo;0% z@wB%DENf5$mBjZax96w8>Y|V)e3oZ%_d)Calw_)Pd2H|JbZr>j#g}4e<$=?Zgmw?q z??(b?;U!Xv1nzC*9vE&wI746nBjhOK*PA=+B&bG>!`H<_h4Wx^wsEPS#Pzx@2~1Wt z&0ejGORYb>O8})m(jj+$O{j#J$^p1&o#*~5+w1J_RiXb}EI|XPBY#2se2p}s2NpFt z4|Yf?)E%do=N$^@*;&-{s4tt0>to90=dyU# zd^TrokU#0V(l>*r*9RlOM!L% zX5E^H$({P%M>?NUm{H5MuXD#jOPwe>=8_ZVD`R3Fxc=x7b5eeZEAI$+#Wxf(S4m)D z&wlXL{2(@EYcN^T$Y)7hzjRL2Cj3lFD)-Q1&4;P+vyz7inc@Zt;=^yG{k+<`&-Q5_#u+|e7v^idF`}2 z2Wzpjjl07jaT#cmQ)|zVVAGwYD=TGnc@&4sGWAXb@l&Hze^7v~sxPsL-9xX#NPk1L zc-yhKY)s^6tJ-(@o8cWJ__PNJ43>22$V%nB(mhMidp-22r`Uca%}ibU?B~x18dFde1mC!aH0FUCiMQP!+gPx*m1C^S*x@Q z8XFT4TL44&(3UOw;oEEAXENhFKKqRKDjnhz-l3Oe%KE}PG;(YRh)VusH+^rQh*A?n zHP(%H)ND7ML}VB#yN?Z_kWl$v{E(*a^ywjuwETG5iwI#;_ipiyFcnZeg1k!!yzeVu0so8sSegDV8GvYO-RM zW%DxA;mSO#+{WX)bMs#2ifc)bE*NriiY3E%T6<^L_ZYz5r zfRc`yZ9I=#TWk;0?Xhw*6LQkm?{(36r|j3CA&^}6{Q7N!{j?N(AWaJF&b%L0H@F+{ zw*4MYc(dQABynOPKW9hT0~UN9za)%%XLV9a5ot|BXc>ap z@1evzVShWpRhsrOKt#izILtj&8HwF&QcT2U84K5t;lcis^{$poXk^p%(McGW z?I#)cSa2#5YY($>c;-FDKkefRO#)?Q;*611t2&)sa4Ab0j)tKtXiQv3Whae3Mp*lI)& zUwry&E6aZ;e|PQSyRpK$#utEy78;eo|MYVl{R9t<@a$CSgmUg0#vz-|ll)EY5*1L) zF&pK)8uiE*SQ6+3<7}48O6gW#3d#@nSLB>uHw~#w+4w%skbQRl6v&yW7JN;mH;fxG zyTF!0KF}1=$iv0C@$vgxDL546a?RfXQQaq!Ai+8|{9vJqkBz!2! zuy}4GHV$OYv^b|K4@#zDj`Fz30pj@i%Bn2Q7#t zR7@~?vWJdXgnF9{5H61PT+9>;uoM-Z1e1|KiMZiG=(_me+O3h0w#*o#YUZ+1;_l^rn zd$GB-PfY0zcA-ZaMlC2e4g`SCZVg4bp1lwHbSTeuUlf&syt`H*SbnQEt4XmA>X7gm zZP%wBjh{{LFd*qAVK6vv&eapZ_;zlIt;V=D=6#C5-h5Kf9TN44W$zGB3ffOx)W60_ z!m}S24l%WJ(m?YWX{Ao5qBoYW#=Hem?}OQ=y6aC_I3rGP@ux~POKD>dJMsZ$+J6NK7{O7D$dwCh)tlSbuLI;&6g9aTpb=oj-)yF8Q7l%|zd60Qz&68K2-WUeP;de)a2W+6hu7{G~mil~(^2Aaz zBtU$>Y*dg>y9>Pau4ENzE9-9pJ*uN82uxFXB_gv$LTf`A^s+>S`=ds~HKU9A?SV(2 zOQxQl(ti$IN?w)~n0W3TAQL#b{Iy7DJy~?)ve_lMpiHS_$ZB{v5JX*HpP3qZ_or@P zvSFsl4Hm(Z=OiS|ROLI31@@%3N%U8KU{N^RS~XNZ@Ae*-*ss<;U|0$$?-gadXt&|v zwjWOT=Fhh$2B#2cVTj1)PbzZi=%WcsV(b zts;8fF3~2T+Bob|!CtSj$#va$Z}dij?Ad&w5@VO5YOc2e7yQ7C91mf3zw8LKeUDCK zP{FS1fB00pXK(wHLYvs>0vYizY|%WapuOdFreUziRv=AN2kLB382a8#Yx&_ygSKYZ zHxq`k9;I591v>TC%$%H{jgE!`5-jvVy=IsDpy@@RPI(@28(}ApYB|c9el#^<0d{#Mha!<8-zqZc_kx35gtd}{=+n1;dTi2ZFf~BJNmSJ zOR?PeuMhW$$z0)YuO!g!?}}W66xb=%@p&8zp4I6pt( zC&>r>9Ode#^xP_&LIC^RoRwL-ea%<9sv4~%tW~1!*&)b5lOZy-K@h*XCHLc}ovsIy zq?F2DWa!p}wXdjL61yvZp>q($EKFXGc-mfhVf{kO-e4H-{PdB;drqQPeDUhMo980e z-{ynpzuZ*@8y8jzdTN#1ja`A3Ke?_jo@tNNUM zav4r!d32Tt0+$o075lRa@%E54`S@C#0=`6En6wG3MbAENiL!&Lm{y!dn;~`%%6%<= zbWg)ilO2q?Esl_fs3O6HT3{6wn;iEo`&nzZnz?FZE`u!~}}+;NeN01v_K z_*@FO`J>|hY4_A16>)}K&r4EK1we`tjStI4LkD-^94<(Ah8MOj6eFCT0^Sg3Cx7k1a_Q8dQmw;{_Hpua3$^9 zU?$k+W3~2FS=6(cySe2ubCt>salx-@&CaD*Gz#+kn8I8Acm2G4cQq1ZiV_U%H1EgVqm(kAv=pXNSonDugWO?~!X~^>r_g7B)q}!=cP9qp1TxPflAX z(aG>@0*fe?jzR)P`JC*aV3%0xl8iQ2)wypAEKYEOU7GVHW_j4N-?>?De>x^$NaRp_ zgMs(_?SMQq7ZtR#&E2+E%g5i3F}i}0+2oA1*^CljWgzKlub5b>8C9PNnz7KpEb1t> zj#P;IL{z0=OD3p!i#10UvQdjXIta075;Pkw!g#TF1vFY z*l1Lu+Y$A^I#VidKZ>fcu#zEa?Aylc?OAsjbiFN`@e6G)#JqmB-?bIXck?R0^2_69 zka&$YJ}$7FiMv4u0bO&+q_pkYKDiuq)?+Ujl+y3xj&0PgqStfri14P9A9)bJufA5u z6t$aejFl`L0#oD#peme903r{8-JB?(JzTH^QlpxX^6CkVN}8=_2e#P>(iVh#%{ICj zmSSGt$|CwD!b zzbJd}c&h*Re;g5IlSJ7S5wiEl4rP<9%HEsoWXsCT&YqdsD=LJ@-ek)-$3Af!zK}HP&L)0x6%Bl84!P3EA@Mp=}WRFU`AKZJkWkun=YWI3EEYC?ONpe+xEI`C&L3X z`!x!c;|@;Bl~(!ZQ3gH+gHi*dq@#MNYBo_>6D!|eFisNQnu+&x<9N>z~h8XHb7AB48ZIkGuc7Z4i9beFv2kLRB);%!6Tt?U2W3DW-f#P>g zWls0+Jc=GO=P@8bFrIgUZw~R*y|$fU2JPZtUX{M4O4DU$Gngmq_d0t&lf6NAq-&i- z0PaAp^-ZXnNPW$XYgZf5XMWIAC!fGUwa7ys7nCZ>>^0l@VTiBGF8d%wTargQj=4oi ztVA!PpW=!^1f`^b_Y1KbM1%M*WKQZlPR3+yNfD80!cxH7=6xfZwJ_Uy>Sc|LXvOMn zj$F5Q7aE?t=5iEFJ8eP9O~hu?=$pP36??DcB| z(Q`I=5TToc9vS((MqV+qsB6W`yM2a@dU6H7T6>^UD?l%u1Lo~9HEo5~`@C<EMedCscw82PRGXz%#YQ9kZ|YDIaxfa6_@(ejTk@7mUWS5o;LWRuOr zwb0@HaqMvdJt=P-{WXUjsbo-MnRoy9X1}HT!BK2r_^rY{`28=K34nJ!^d)cu7W~9w zqBwK#RdikVcV+sNJnk(zmv|m|-q;>n%w{a%v!3zK?Mq=M?>M@j(o!pN)jiLOVxrc# zDSvnV8*bP8yHX*q+OdT><%ki(D*Ju9{143sa)}S9CR|iAw*j^w=dQ%gKV2LSim->p zN;1sV@&-c|7{4LOin5|x16WLhZKYN7YQts=&y7K!;l}nOr6ev(YSRz~PinzxB`FS# z*|swC^~>{I&anz= zG4aVhX3rLPbK)@hPw!k;@cTbahAW!#sduX~>VPGXNjJt%jg9rg#>;p_y4n9LoB$PZ zLom9Tl22t2(}x#6(uB)_p475l$FSYH0Q|3h=ieV#X{wuCHZOyro;s4X%ie7Iwb`Df z&+BrWaNA;STjq)-2TD73UR)CWa7$+%PJE9fZ)9(JKc;tYsh0g^k-Vd6UzSqCxGm*x>^2j}pV=^3h1&mhPE)S3O6Z6t4&JvtB}e?ud=NYXQqYBz?f# zz;{cEc?YTHfB>gLjz_fk$=d{0<;5}1fSv0(psSeKgcio4{FM39d~@CJ;OLQ`X-#D` za&r?F^ui$1{0h{3hv9w#&N(Q*m|=L4DGHw}AeEg>2eFw(u^FLLHXFhyZItEk!(hHj zR;;rJj2-Z;e{$J;251e685?AwMyRKl0On@pB);wl^v>JII8*2tazFy*7fnR`evp7zjjsDDH(l1SK%DRW8ou8Fw2kY_SZ49WgEbsu1LYnP8{|gd)DsJ8DSmkL~4jcwZN0EdZk|^x)N7pM-D*>>yTvXaD z4+2xBOs5kNl4%sFs;Miye?X^K=jN;LwnU~%iZbm@d=j3-eT`Q@V&~__7Ml>yDoUE4 z17vUU8qVfqSxr2v+i(5Z8M4zE;s@Arq}3=Bw%1qMnBnA+;T=qQKuThgvitb-?(2V& zqT3Z3gA0kn6N>I@F*e`H09-|(Jrt<1>3!<8NoU4f0E~Lf+a{ZEHr@tN>zxmNA3#Lr z_dNQNRWKAi`R~xpH~GM_KoC#l6F+sh`()`lOP0z=iO$<)_7qEXDe!qN^NAh=j)=XW z_KuO$Vf_IbVwh$0tE*76TMyHAYo7zWNdaI(*$C#wJb`0Nqtemsxe!1J(0}ZPt1D_& zWW#Wmq{dr>*S1-Cq7u&M?3{OKmFU=?I_Vngpe5#N^R&h{iQ6h;NCB{Gr8YRpgN$60 zPuhDiSl@|?*3k`T2Q_1EUmjgpp6*b231`Bu-&63X{#`tm5bdX^~a6bVB z)4`YVZioZE?w_k7bX)jnfTxla1<<2sgVF#?GY6dnU(suzN>tE+{0ELsGcpIwH@BCR zvkSYG6?O$G{E)x~oN{lV>OY!|9&&!4V4*)Q8_)ykIe) zuoFUS_}_>yN=>|h_fQf$)J3>|IsUOuzISSSX!>At zvf>XNUNs}rM35%y?^>#!Wi28`>>GlCU0OpWeyHk8riMvkaO|~_L$#dukK`0(qGQr( zMNif_h6)^I>-b0V(qgw4CYIC-9Hc2lb`&VN+#B-^mi)Rty^Ldx&P-Zmmqmzqt#OmF zTfIH%C2RNQNfLBLjb*^Q6H1i>D&kfFsaO(c#AU$T&MTjEzaW@kul;8ABt5~S#rvY- zw2UD8RT5?(Mt+tw`}Ea|+O=FFXB9Nrr{uIhzBvHM;8y3p$V!aF!1O|DOKBP!W+c5D zU|)x{&az;nq$^7_pf!eK3k^m9@-JyqXjGJdTuQKfpzPJOBb)|lc7nh)nJ0nR=h0;vkjP^%m|D27O7T^3*B2tT9U~n zs0YqRh~G%3`E~_Owj7+tsQ$pvY$2))^9Y2V z7l}6h2*_%_^$;Xu9LpSY|I|v7Kl}7NL=|LyFamNoX<^Cf5jX_8q%TZC;S7yXAV?lZ z5QA=fqAgYoSNIGWpJ`tbPj@Sr2@t9b_(TdG#oUb*c5?{Y`NIUd@!8}Kq1F|^Q5U3D z_ZVimF;<=lKSj2jzE$GO2h|!-LpnMHi=aOUxglW(DbnXZ_%aEJfo!5u2}V{WO?V&N zU`5@%oJlx7O9%L)KuUQ170{rdT%$}fxRwz7(>8pC)i0pdXJgggJ!$-eE{AvX z*}aOQMBxlKCT?5m7qD>NRN~hClaVuSi{a}9{T5$Qpp+$FF{C22yF{k_)M5jZl4Aom zXXe0wS8Y4`8<{}!SD?#aYI1=aSI1Wd`|A5U`x)|gkTgHjJq|*>{{rk6how&UC$L_;)GnO{Rz zpaH9{QYJ;iS?Swq8|+Q`MbRJNc@?J-bh4n8Vcqs2)DgHLon2AKGP-BHN9R1d5gD~9 zW>wLv;zczR`z}VXPp~NrGz+SZxLwAzPrp(KtdiEucbUhOKYIjbzgf}?_upk#=DH&O z1WJ@#nd+xWgxR>ABm3O_qd!BduB0eS1w?{vBrk zeMdrVAATI5kBPV?$~2lbdPZwiA*yK`hFzjJuQEQ-*en6MfsCidd% zW@Eupm1&uI&+5TD-}j^Bt`-BVXmTOBFrH0YrUiB!&X&^hfsM0ADfJ{gxVe#EOwa)l_cET^UZn77bacEY}wBL#I=jUzD2#< zbyaHYk#VmFV)Kz-5WXt#HN$Umt2N73fcpA!y5(dC5IqD6VV8tNjeHg;7>&viD-aAX z6rNDFW-s2qswq=rVX4FvOLN4wH@q);UZnl;jKct?%w7p=IYqIHDc&JMJZ{B;x{f@< z#HFM#f24^d0~*=8atFOrbI$Nax818GVx+)W%1c8Dg{OooRr>lKi#{u$ss9!0(N$v>XL`|9OL`e)DW_P9B$x_<33I_ffp4%0=UgIShDi7{x# z6#_l{KTSUT?tVmd#s+p)shP$2Ln87y9gJ(y!ofq)Uz2NaA%wS5PUV+zv}>ZP!eH*l zn=hSy_ihTsvgqi~2RJ^_WPnvFjaM$_g1nhyJmBKH*C};0DJzw+7L-&l()oH|v+@-6*w`%PPAUkoha;2TLUfl&Ax?TOgLTbr&-e`(^>U94&rH3z?SE5xVcz+iQSN^G#IXq2k>Ps6tv zzlM8WfqU}fp~fr))1BIXjDu>lOm2Uo6l_ZG1(LozyyaIWt8Mb?X?T=- zK!vyLXz6&c^#09}9D&(B)h_<5yGKRVbv>(e?`$~6=#mHPTYI?;QQJ}P7GA74Me9_{wyho^0=yoVnt8)9toF; zpl;Qp%muQb3Tn{eD*gb!V!P(CeUCz-IPhezFguarf(`D#APQv-0s~oB9!Y@TW12-}JChx${R)*D~?}L=<7bvfallzWWi0U`< z$-d;co%ft%-C7x5J}@ms48Wu6xAXNl5i}$j!ozvs=w8?Zv9=pEPi}Y_Xz1hUEkjM0 z^i(6DN0TN8@NHsURWJs46w*mJ&L(9E_t~&zCy+`L_%I4r`yWD`vo?i8u;2g&R%bu(tJ)mV*3Lxee33Q1Q2RW2HuF(`pAaLL`BqgEU$ z#TQQ*e%}Go8qYjIJ>y$!g=FsuDg1RFjjO6TE~&M)0cU>ftEic=uR!ZAIbU7HgJ5hv zFyrmjLECj_xsrmv#PdM5^MGaoKLY9;I8&;d7hN+s&*@<_!gj$k>{^L`1T~*P?Mc%f zK3QOPN4wy?TR%(8ACV7#ZT1jv3^Q9hN zHC72pcSSUKYcSp!;ts}7yZ=BtF=RdLV+vrveuz&woFKjJgl(UW?26@O;c$~k4re?W!ZPUWsV#I6gH z|Mx717?J7l;rl-gpSKNPP+n>-2qhFQRGZaD{hlnCy;m4%;UkB8-KVs(m-E=|KEx#o zDt=zRjITZYF%&XXZw=^xD)CBbuCYFi?oF+V@md=xeZ1@aexW#2;i$#tMEcI#_w-LD!93j!X6imx7EzN5+CE8Q{7Am4h{-fRe=dRO0&8hLN;N3QMC$ zIcMb?tp93ud_#qcd&lr%g@qM&afg3D(jq4ZqLk#rbuCJd-<(R>l*(YM4 z`U6nrUB_E&(p0M0(^|GyUwM#9PV+@d{SSd$0$)A=db;a^Aq33sn3gDH0`E8cmW~QmI)4^nI(R`?- zk5)VTzBcFJT-l{Fih$5wCN8%Db;>c@vfR%5_F^6p8(>^hIzjphPq=)W(z;J(JARVo zkYC_lo+!%0(#f8fa>!aSna@V05afTPJ;~pZ{Z%1<{r3VJmHCuEN9e}=!InPpB*AfN zx71CHaF7!}M+atZT7n9R)LatjF0+wFUAoy_MBsdY0;vx|jZtf@aPu7ks%z+=G8M23 zcgG)}gxYogOHarAtoC`g{oi!AXgD_*ewOG7C2^S&IR739$bFJ(VTH*GB7ext`|?8$ z=`C?dGTL7XAksqIIcolWvp?u%2UUO2Jhgp#|Gx7c9T!Mtm-6v)5P1c^BRB@})uDhl z1;7C4H*imZb82h*G9S784rlyCvZ0mOy7+I)-jj_cau~l^oFYEew^8|fEz6nZ%4D~)hlYT84O3wCyMu~?CpIgwYC8) zLuuRARq;lG@3PReGusS{V*w>TfVRAh0&W`{|2cZ=)5v@eC! z->Z3AJeAi*TL~V^nxz@i2z<_?KB=i!a>0)DZdJ@YzY%&3054nnZoFvny~4^ZWuvhd@iJ-!Ffsi3I4n%%Arf-Vb{ z3{Pnu_+{CtzGN}=W47sfHw98-gM*;-?K@}q>V%lm3J9*a+gb-fH)Si=s4u4~RK|CA z<;P)~i8o&|)Jy^#5z<*V-TgFMXN!R{0Z@+cNG<1KmOHn{jAD31fV1U@N@e>;?xRE8 z71ho9DsI6lJ4CPEw!mV4md}s{rvh+)J-4DdR_|}G{3?vI2)-0jflnjB@4Z&G{Fy!h zG`ptoOXMZ0mndQhdaeemN{a>Rr|)L~oq2ERn96uG&ETc*nBB>g+rya=pFl$Hc~QUl z-ou51z1hg~4t+uDToauqCL8;+a0pFd(PFRUxz^nW74HMjgsi9s46)+)O~ow_@-CS< zZxdLg8%KCH8@>+}ORIr_3|mKIKOcTs^)XP=ERqK7GuU$XGeDV*_uyw=%gl}L*#Y+l zHFrPMG72-mG|Ta>CM5a%!PbdP_j7tJXwsD6@IlY{+yyQM>6@pC5t1|xN$|^ zu+EE0`oL$rD2IG-Sl_q&98%pxYq_NDUXKCZAKhSow0P;ZqLEIcT|rq0JJ9LdUuPBaF4;MvL$rk4!O55G z#ex}Um4XQ7n@<&|rp9+y2W>#`5}yUSpLw9?xqJ8V;F^xaY+_{o&wI0ySlf zib2E&m`MG-`oD^%jC%2?1;wYLb5vw-H!K@Qs6q(&R@G=p5YqxaVF}~TR$K=&Pj{C& zbb+M|z0Nde_rBR9vV{A&osT;hmp87M?cp<p&56^%yDs@nm| z=(W#0FZHx52R8ZvlcfG+^6hocpRo`N+*;B9;OM3O75w^^wE^41A&_1czG%j$f}$>h zYyFr1=78dJ0uHPT5eU2@0xETTi!=64H&St@G}KN`TUsQTjp=LrRaKRwnbZn&lQ{4* z=t;hRv}g5`dmpRtGWNwlek^_4?CN}yCEaN3z~FGslxmoQlsg*=F|o4YH8Qi3vLb+} zJP?5IC-v$h_VwWK#IlhU&zkx$hXKP=+rX5cyO?=55QlA!zonfa(ASCRdhKhvu1C^U?l{^rV+hpTzi8H?<~CgI=HBiA#uzUQTv zz6`02)8H0%e7xItVxmGUjZB|bvT3%??y2z?H%$}$yji(CNOnL- zF?E$CCIpqdgR+?5u59yaAyQabUgX*raawjv8-IB;@Rq$>np~e!?N+Y$q5~@Y%l+*M z@j8djVNT;6ziI)ED~OwL=hyuGj-}1ZaEHMk3@NN%Jxz|_)$iXL)soD42l`&$cEUlt z9Xmvy7HmWP;(tO86VhbqHz&U#GfzO-Y7j|(Ds=*W3sokCp&+_ z7cKUzNi^fMQN>ox?M}5=39@_uw&ilr($|cDd2m{%BXg9kTaGPym_gz~H zdoe25(BQl_+p4raDJ%c1COUWI=Eo#%>z9#-2a|B4;n299Ey>6U+dDt|K#4&KD~p&u zTZ6W(o@4ZpZ`->d*Xpd-g3fCN7^!&8^PDUas}Fa;I9Rss;!{zzQ+@7zUvrfes@__O zYcL@Xg;Aeg;qPgNjDxeMR6R0fl%_OVy52d${KoZaPU&kK3aHapJ}fU!*W!w--?dEM zU;sl=FJHS#!TCz{pWi* z>)WP0I#^2Mg#)MK87sp^Cl!SX(zO6Z@9PE#_pqJ$Y;)7z8X59_7c44>5a6K#jvH|u zVH6CYcedM)?EV#W9L&Vm)Yh}(L4v1@Yt52=jF_}hc2E^XFg;`GoGiypp##Eg5nyF{Vky3 z-z*Jhk6s#y#9JFJWkha{KCP#o9wz<9u6=WN+8`uc^zaYqq1m{DwyC{VW?*&Dr*die z0$rD753!JvQu@%6`+9c(gWkKxH~GfJo{ZIOH#?{&^t$bu(s{R#?8{a+2aP{3)MxxC zWxUoQvv$hWz&AC0J>U7ayShAvuN1{^6er3xS=2Rm7YD)h_|vNu$y$TPX2P{zViV(eiA8-*_g1419meXs#=Oh@A40DYnS7i!*%U_-N__lHT zDA`7V?g}EA_eb&REx9w96Vo!Cv0z`5{k4Y8(MxEWiqhCuKm9QYlC6nOrc!7#sO*4DY}t5mrU@K(6L=<3x+ynQ%KJGs`eX` zCJi*5175`k*crIRW1{0^OMa%Yo=tq4k0iK~M)X>o6BE=+-vKD~r=Y#{Q6-b!l*Asp zqw7pSY_QgzR%_H^62Y*$Ys%R;Kv=p%*B_MG(tQZ7%*Z0cxuWmY)dz~c^U&nlGAR%t zZ;c=%^V&}r%UbVy;AMHz#&3ZNrTc*3B(h2B(W8*q_;?e?x|F_fJF%5cZ6XY!*Jzjy z;y_vnXvx2m0FCsuAM4X&AhY(%Z`m7}d@5)1v%I$wEL6oJ;EvDODV)3I#%)jAho9{3 zrQ|IQyuIA5BxYSw4W~IND^m1YMR(g@(lS17%Bb-uc=#O$q484RP?Rx5lBX4`8J0-2iUgZ!9WEu9X znRP2-N)FPe2ocwnYeOj_`{oii1Y9^#j&94F8>UBogxa=o4ei%=gq+LM@=|AHVNGy@ zgHvo%F|yF-ACK@`=AKb_FF2P6DksQd;2;KrN~MrCKwF^cieZiRe*H2$qgga<@FMwI zVc4)qjhW#mE3@oUzr1PYS6w4GdMTynnnJR+lqrf94IM`caFSQUKan`k(zTLe5Xl9C zX|_gYWJ(79$ZrdlF3)Js&g)&dKe7P!dQ~zchT-UE+Rk>p28Zt5gvR|i@>9{-AOqo}#1FE}C{;5bZF^n;ql zmW?NGq_t8rh2uy%AWci{uv=2~s(wqOugc<0ODg zIpO?Dk^!#SZOw#Q=k(H3iC!^;gd02+VR)c&I(?GRLd|}9%OhxCvlz8_pz~3yf7T^$ zvW9u?#DDwr-Fit~>*jh2Um>40e$&n~xBa+7d&DTLOwxE`+EqSC%*C#wqf7X7V|VB1 z4WG3yYA3Sr^o17~IQ?-IwbE(q$4<7NTT~b=_~q5&oa|Ejx@{=Z@9Lc)EnDGw>mJrj zEAQj}4&$F#?{H3lrCr!A=(?Nt(W@b%!epN^kEyRWnT2`6$N6`_ihk6f(k9$R4Z)xw z8oG&m1;+C3rE?nf77K8GICU#|%&J3mAI|DM$%^FqzNZwly^cE*l_v7UKEZadx~g%1 z=B75_0FQ2qPi7B zsW7pa_Q_2fED8o3+9{=3{5AN{$0xriHyg(j$3k9RA*;WM#`MhfV>G^I# zj%{dMt-5f}(ORT5cQ-+Us?AH<7bpvHS_xYHPhNT0DhMhPVLCKlsVm@#vFi5wKym7x znABiFH@7TBZZE^k2V|;}Leyd+gOPVZ)>XR}{s$1W5QMb3(@!Z)r^jgyWy}_9U0zSX zIKjo_X_%z$tzFt0_^ta=>c4U6qhLAYsxMJ?Vcz&5I%aDq4ZR_t##d*A0dJ|-mi zMuFmVI0(Si#tKSqs(n8J?jFtJr|&5Ed!nxaW%Xz-vl1Q1{>@+h+=_qZf@q9U_4^fb z>!nLT7^~B68tpEp8~2GIqT1tEJ!esrlk>Y?2`)k%% z-SJ+xZ?=px-Q=^02IfiL%`Jet;!l24z8LGXYNGv1wMGN%EywgM@ujgqAqJBe*-mQS z^>`|3_Ni~iVXs`)G+c?3*CfkUc&@opGZ(eeqob&_ZTMjpR=)CfNX8+fH?2Y=C*5-0ji;ui zc#R0ga`iS2xT16z8q$-lH;DgIAF<$4N=fJyXKEHc}ii$lKVK7 zO_Hr)pfY5sD1O4WW=mbQQwW@IqWwc-{x$ECzQ(xy6E3S3{4|%MU-xtt_92;FKKQrH zN`0s|LlQ7%!h~+Wl0d@*BMIbwuV{gQT1CEXOe2^`xI4u;PD<|__- zeFd{nXqUIOop!R!ZP!Gg*c$tA#C_5~hCsNi;Qq;~m&%Iq?MiTk`%=WJ3qSK5IL&=o z+8VOC*PG~A>}b7`Ev%U=5-wCQ$W;*u+{+-j-ZeuqVdElsvC)xT3mG5nVY(@;YjTi* zl_CkLhTG8tpYaXkGva_Z3Q=Zm76r%9x3aj#_)i`)B?*N z`TL!E&~b}(krU�d4+e!~kuIefOM7IU93|O8Lab_Sp_NZxxf7o}at=$quNBV+DQl zRjgX4_99NJ%0jj~9mU#}*&yv*9dhT+9o1*gEGp_AO$OSDnH%AOx1$BE^5BeU&=|44 zO8Fzv?XQ{CgEwkm>ID1uh~!s~kic^mt>WkcLSd$rG%@98HTMb}wjiyzqfi=EL*Bgb zn{RXO-oVPInKZ$xCeU#Uh~RidsXc)RLjYh9X z(R8uRD)vPH-z>8wK_l4N=LO>_)|)~nc6`W&Qqup2?Q#Ib*3wtzG$=*fKjG1cPW2`q z``VqN#Y_XXy7Rqx5B8$ttAz1Y@OK_sIGx)a;8&IY(Y}S@d-rpqs{>5uPrJ^}=z_K~P#Uj)^}9NIR& zOkH<-GAr!Gv=|9i*;B5y@sV?W9E}-?5mt;aloZOOnQc^UwN;R1xFu zXDD5rJ{C?8!dy5ezAqW06hC2KSjPRM z5F!{kN{iMG4ajt_@eMq5Lma#1)D#w{Id2CBNsQUN^ki1OdaAe$<)f8gXw7dlO=p35 zG&Mm?37lQ6O{u=dVM^HX92RN?cCKa&|2B{?$ow}m!8#ia5rWdX_! zPSf67lPMgA?VVs^j3NiV5HMPjS9I-Ku{fRA)JN<62n6MZW@+-$+zWO1JPxetz&su4 zkmFRTA+I1zB^>b&%)l=y%2aE#2Dfq)u_62cx|Kur@A1(kaqL3gr7-M4D(X%L8%`53 zEDwIi`RC~*nE(4brl%KAYKn2*+of5%=PEGTXc*(SfrunKy|uJ~)H+;ZjI|9Vu|@Ck zTHG7jj9ad#9-+i{LDfQ1fc%r)n$SF^=fTnPduTT>O7qoyEvr55ItT2WMY7D7=S~29 zfH5cyJcz)?r@RfafjbYnT+!#en?*h!5#d}%7n$%5r{q}-Whsf8v}t$@n$Pchz?*6s z5@~#nRb0kiU?V%kN$)C_`QS+pd65%RfW=;g7W-Q5>tnFku|q2nNXW4VZJ*+!tDP;D zQMKu-a#23h9Vr_52-gx1hN(Wh?b(l`SVg#aK8rLw4Ibw20u=CEVOzW#4vP#G)h59{ zVbm|=`ikgIIEfKoKH&fVlMY_briZXwjtPwiYZ!-nF-k>LQy*Gs7DI~yJEK|*Y~BtR zQi%sOk^t4BeY_U-%8~*Mf!6Rxdeg@OuZWh^E_1Vp2}vmW=ff>$_0=B|zzGN=fx+w7 zOAa(a6%hMMx=%BeS9(73;elwiBrP9U?Mk5IPtAw5G~AIdZ2Q6GFxWFdM-!)I1uErs z5WB{lD2UM)Qc7n)q$;T&auD~5X3pE>m5IZ3dKeh88#Bwe{N_w`PC@$aX79JMt8J*4 zTMjQaiRZju9L7wG(I*FPRJ>P+yHOP)Ioot1aTW#|f2Z(DK+*tj8)Wovz z!DMylmeGvm)tXr7>}|Y^8#ssN%?Zhcok#dZ3Y$Spas*-$LO(6+DtHwQ z9kSbe-84@x`FKoJrwYsUi`c)0{q!UJVj{N(0s#)!9jDxN{bPI zO@u-hbG1Ul^)Sg@YK=gMzlx|?ErNf*HYFEifPsBnS^zYLDn_h?;<;~ zLMnmp;AbNGS=@QPGja7Ja&64EcKghvS?gU#%Ug0`^6FRqJaJkcc zXr7~E{`HjeO~CmI$vRt7EM*r9^S1$*M>I=GBtweQN>wXtYL0BvVew1jMh}XKxGwvJ3_vf)>WSBT2{DrQo}8sq-A$WITP>@Y(C!0f?l!FysrK%zKR&^Yuw3<>BTw*nH0?7&vL z%57bUeYPz4|L(`fS;HAEq|Tk@j^}<4EUNd8%#g0dlfQ4NrYi&-78P6ohXu@Ykl`?n zJVGWJKNx|$W$-N|v}M~JeG3N`^;(t!HIt9+C(U2K20nZ96F*NP@V$g@63U3rep8VV zEZT#bxkepnyfn$scEoSm`#Z&rWgTTYhJ%Pfn1cp8BhY$r@cF_lN2mJ}1CZh$bqo3D z%wWf~zjZGXbqa%@5>30p&i5__+PfL}WEab~tBvrxh-f<8gd5;whJnHP;?YWM>UDaet&t-6ca32;M`Nb2__1K$}Q{ZRL4AV5Yh9joB{TgN0vyI>8*PSOHIbKWZTB0rPC z)ip!B5I(!QHy{CWp~}D*RTuQNg#1fU2?FN`2=~GseKs_zo}CcjWt)qM452B*u$Q;(hx5B-b)4dJ8kK9>~$-zWG8X;pu0WSm@mPje<0 zw9l{kKKoc2lhklJ&XoM%GDaeFJJfC`Vq*wEKgoX3zI^tJS0Sa>)9{q)7mB;Q0M?i})!zE#+AxpS%675C=0s!9PKp{UHp*+cjXY>UuiUsi?+suM>DF6vYI1Jqd9 zRWRX(ierJmW|#8YJQ3ZkA?@k?x3~ZU+DRaWu`rte&1`ab$ujiqXKZ#KdfDRwZF}pa zX$cVboI!Z66@ek8HGNU|_yhBtXzTM&p8zOBo7|)MzpO>e z5!qeHMmBc>vK)(5qxx!JasYjyLpz1(n46j?7}ohWXh0i9?JGeHi@eF#>!s6wFE522 zxRP>&cv-k_z~Q~IV@PJYivdtLvLMP+t*rNUWm;!v@>;wLletC+{wX{y^M`k8wrvMX z*3JBacyWLtG_S2*Y=FYq2GrpOJ`>)Ydz06@R%)QW>c$fX#qlZ}EOci!Tum6$``G4f z7|n5Gis+}xPyCrG+g2&5|I0R@&50UdUVQxl1eK-R14Ol;?1VpP_JnH%;4^W~|0_PD zN(?Kqzsu;4yz7BId9jZnq)gZ5PC%!l(PHPjgLdTpIwO~9ui`BX5!`ra<<6`(S=2Fx zuOM(i>jLh%;72KdlNroHd1&W6wZ)In<@C2DJWjwlL)mf^t63yj(+=!f)l8j7p~PxZ zmw`-eY5w~-B3SYgVevg`3?-#!w-xkH6c5Y~uqm97>2FOJHm+e@2J&V5niRtbqYZ^7 zfU9N=zT}YpLS0t?CFXq;He9z2H-7iU_B7E^&~yec*#2*M2pj|%)~?!Njid2omq!oZ zHQbAfK>@`peh(76_`#FI9j{z|_`w4FRJ1I~axTEMq(B+x$*M?GG1r{bCR~Oh6Q%V3 zhj}sI90=(|;1cnSGlhIT44B%lj}?zLTu_^t1}p+Vqk z&TsPw1)P1+F?4#O9%T0U!k%t({!}}-DER4uQXY7W?+_$(;WxA1 zK$m&TtCJuUf>Y=n`6tiU(Fj=>Q5T^#Wo`aN?>`5heL{$C&z%56yjZuAR@tiSgW3n7 zT+|+&vD;tUX zK-?`L{m>q+(h9kRe?*P2rDZ;3R_J=q97(?0OxisAC6H)Tc;1mzm2LC!s7~(_npR~N zfyT3&xU=2XI#-zc&!0-3uS^IA}BX1UQ`bMj@e;~Bujo#tq* zG1A+ZHSv+x#|OgGipnqWwExH1{Oh^2&u{z6v#dpVxe+&uY#GpJJBASRB+x(q50^&n z3>_!^3mu2fgy4UkNiAjh-%$F6sE{A0uI4<*@$--zG34V)13 zPXuWP{O1lPNG$?WK}G`ZLl|b^+iLg&x(n?3wcTtO{Oo4GCOf~(XMz-<@L?dd-eCD{ zJs|A+Hj5MzlO*N0547j_W8!ejgA8^ ztrZf~jsd*ame;A>BmdsxC|e9;vdNsb(8f~L4F&9$fDHX%y|c6x5p}83P1pua)>(vh zUZxZs1e8yJcxL|r6}8kf6eNk;g9y+z@D#XgbJ)da8t7d5nXE7-yFPyAd9p}pSybLz zl^V1C)cPQ?%Ej)Zv9;aZs|e7E!)(|QQMC!Ca7HNtCTAu_hAF--svDQiACMmnZYlHj z&!?Zk`;(VBj#K`n6QI4>c`SbAmk*&VRjAkRUo{Vgis}`@^6Q4UU5sBfidsXdnzIXa zp0tps#vo$4jsYRxuZYgGL*E`l(tIKd-kqilWG?>2^CZ-{B;0N` zBPN-#4K-zFLUysAcX6)y*`4;w(va=Tg2^BC-+_yle8s7U*bXRxOEB>sut|S!@$281 z>O*v7PP6PF_$6VmV8d!H!K(u0Cs<0U!+PW^AcuACkew-gDO$}>C=i-=f>yg{cP?i_ z=O%-54GUNglUi`9e`&Sdf9IZgWPJZ z!K3cN5}=RQ-l@~id#l!R4%8h_)>T+9iEykGs`#)M3@@L&w^qNjg72$I`8a0W}5$p`zQtKFz#nw&=?paHcEg>fn3N3Ghns&zU+g`wslGKUzh5dC^~RH2H=zTQn_q-d{6NJ zrXFm`nJ#q>`<(j`hY;A%a;z=l14LydQiMhFJYG9Hg%3p#( z=;S9<&SJ3#GJV958%aw1pVa@qbOX_YwAa+mogBP>RWdZk;>;?(Vq+@4!eITE|J1$; z`A_33|M*Wt0MivxIY{QSh1rP2+ZphjY<(ew;7Xw*Q5A%qt`V(rEap(;s~InZo(CKh zyT!n!8_4ejdD>E5Amjhs^luX4sdsEEF-G&Rou}SC8buTq8MqA=jfA4r-U@KZ9<4*s z?*7(!9`nCCiP-z=tXka>d3;|G^?IX}^hB`#c9RSCLfI9T^}OIP_R>BLf}Z2kt@ zwXQ!MXG_{Y0HK`F12-VO0Z+uGC&l)DN*aTJ0sxs_ecS%XCp3&lC!j+-8)Vj>@PIr7 z5Q|O@aG3-y_d5J*7o{J6D9%)6X8Gu7aZ2~;@f~%FqQ^o5BknTF^iz>#a@?a}a1m%b ziJ0~Rs=O2#oz9Wz{7Lh2IN2P~}nv>3a^tK&Oo{l|3fpQCblx ztf-m!=meB-TFL$z?SS+!@NA|a4@_vJ7^-IYOX7Xy*N+=|LLplL)j>%2`vih=q(k`i zHq>Gw$AzYgVh6juE1l-N3yJMns^gynC1N7Mcrrw1+5kmXde8G8{VRfYh9DAP!CYtD z@)~!0g$f@5Y7pAIRF|YwzQey!p|2OH(73J@@cS?%XaQ_5Ui~H!+VZPOV5{D^uqU5| z_vbJEAnG&RI1Yr}Te$y1Aklm=Ay<~@+VoWjtAFEo@8(%n;9{|_6ERcrB_o_4lE6mu zRld|aZ7ZCnroKOvsaM#-|cgZ;zm|jHEa>TK5XJ$6;Z1 zVw3)f7w}A!0N7ftj_AZUz+dt%I}WJ&m!J{haQj*>#f@M+G{c2bfJK48CfovHVj23evfei*!qnfW0!v<4D7!+!-q@)4fQ}a_Lm{M#4yXU zfgTcjvB$6RfLy)==r|22ZXC4iFb3e$@FzSc`q$eM6EPJ913buz?{-I|Gqp{PnS~dt zB_W?m?dexirB2XrZiNI~Uw1EI6NFg}zrJ-Y5m!iGtF;%Olfj_aAkNWoZDdlx zd$N95-}O&f(PFiZZC4%seKR)Smqf5g8ZrG76x?U>3IFz2Qe!Y7c3BX!+W}H)y~>sU zS~fU5uw~XPvbE`NZ2{Dvt89XgbBzY*qHimXE_k5)6;dJ3kMN5wU;}9T-aHn^`e=_a zD&?a3v<=nS+|vUSd(M{d#!9P0H^Uz9a^}&}Q{5z9+jZW3k*WmYlQE~76uIR;Eh+Tu zErPRy`}?E%QEYmGZfl$xKf%5vp6?6&R3D?)$P<5Y4jhE+MNy^wUqzKTltFG(51v0@1P3D%cYzM-DL2Y}PCWMK2VaFn4x)nLsnCPT2{9_4#@K{} zNtq{Jzt9D?K41F5%m05!`|o(F`~MFdFR4gnG^Da;WJD#K>`hj-%E(?JnVTvgetn zOa+%kYgK=_ub`P76J3RyxYQnE<<^E~lf+XxHn2C8CyoiqzY*jT%Wb+TY#7H=^k}nrt{L8^!XG)?w9=S6w%AbK*R_|Ez>^RxY#Se$Mm%^l5 zu5b-UJ*%9c%E-zpaIr`G?9Y?__-Ox7Sj@P~dxI<94!H)VX9F}y(jV2h40$rMeH2MT zq)gA-*O7KVP2xKT0mRIY)$w;iwwuAYL5}{a=1N!s(!|b76fa4H>q5cNEZ~oMqrINC zxZhGLB(m;X&b~ajZ8X!CP4_l8qZSuyOF>U>TP4&qs6YKJ@|a%!TY`qRs_6YEhyp8AJPWr#w=S%mG&01^0koxMaao$M89k`jIDlu}BYjgxZ zyjE|*PpEWF+k}0A$0I*KSUL9)goqj0G_xAEUT2XYUhpIt>puxG9E8vlc&twfVC*XU z-|urJt3hi5qMwVqBQ-zz@7TI?-bxn!Ndy)Mi*kIKJO zl&TiP**5h(F5^A8UCD6QJmd?D8n|<(U?u-)BmaIY(fwqPp+juN{a*g53a8oA$NIq@ z{!L1bNWH*Mb1ytW%W2C2rZ!o;_-kf%Qn9~-2`d)amBdSI#)-)5^O!!IMIx_IQDY1E zehQuCokqf+?=XcX1#x9I*JaB|J-%-Xc>}L01uV7Q@BWYezZf*55014x7<$UVVhfnL zJd@_!#ZK_aRkoF`{(2&t)2tNd-EK0k-*sg%x-OgPXY#43+s6CELoDF`M7Qp3Hs(AN z&pmr_yVU!M)%uE7d5&eYQr>VOIcinGWbqT9+G-jPaV_rNM0?2dQMtOAG|Oo07m1)*=XKPbnQIf=V+FB{*U3i{c%24iTsfMDtJITw_$+@oXFjqI^3C=s(^nwW*j z6GhLa=gUKSE=GnWOErHC5k&V@_Vq1^eH-M7S^T6ub!0${`v1ffTqr@w(KWQ?GUk^AS#BD#4ou9t)E0-(HCa zG<_mBw3<-dq8X`b9Q;sZMfR8xB{KV%e)RF`Tz=_AM{ybxyx1)Hm!hbRi>)9x469D~UZ3qfKeygdg}Y_Lr#W7+#2Gb3++Gl?Q)oq- zN?l0}8yo%9&KoM%#SqfshVS;`So80H~j^$yQPfw2q$K)U+X!ELF(6J z>E>TQKF(D%QbWXBJziuIA$sx07KK^D1!=gww$&u&9NvW!NC@R#Z4(gsNs4y2Qz1A5 z#$$R7CiVZ(BN$;+(Nr~$=gGtcD80#q4P1W8ML*)@FZr5}4$Un0#EXgK7g1ABjsa`7 z4mE9NkB3z(@8knt?u9M)WCe|paj9p-;q%^WUv1Rra8%yw`S%2;uNCK-*Wv;rsGiGn zV9PCs7%UCRb+DOPS3J&;w*eK9{FkYVK~@yeY5fI2^yaoIQn)BLJe!5^}+5KMwo-Nk=Z zk_2^hNp%Zh>EcQubsU#A={#Wlp>g#?qGaA9?f&U=NFp}AFg#Db+aj(2FX`?&nh96| z!%=PWy-g5Wyi<|=k~p!F#C}_RXbdfy7A_M&z7bOT)24LrNqEi;oZE`=M=i@#fD1YI zuHRp*&L3O$IxzRT>@3!Jj=Q&DN+9_mHa?5`R0VeMa8YZa%V6)I7H(r@Zt)mzaCrax z8MBG*ddK$7o(VbS$<-lmk@b-vPLr(%yaUwi z*C?so@4bv0EG|}C7Nm~5D$a4{UQC*jOc zh>=h5Tx>(epfNuhF8UxeW(r&xz1>`Z)%*n>XEOy z<#wC8Sbj*&z42#=q6>Kb)Ks=q>+)b5wdzi6Lvo1VWRzC64n3nay={x4v671&UW|ub zPI@{?AzIGTR*8SAj_O-gnjFm9;#t?b$=)V9tsdl_n3_P99f-XecB*+OI^MajFD)NYeAVePwv`<5f7P>}7G#CtYNQjCVosiD_q z^6mZ~RUheBpv4J3i~Bp+{o#J^Pj>;metX&N);!X5+>-pSE@=tjb1ik9k^Z{2ic=XX z?F9K>_*?4u@k*}xc$<%pY>l^>va@*I)cc$|-wbk29KFV4@}o+?{z;64 z#A=V8yWi7Ffn`7H!j^nZ|K)0(u#T0k_e<{=2eCnOZ{wX&s#RMK*|SBu2P);8);;VJ zu1$%Rj^!-bP&vsTnlSD)7kgdb&^>lzUBsyN&GUHtvc109^b1Qc#hjYWYYp z`oYqnhEZA492>t#fJ_BFFPIt*9JhUIxBZNFFg!T~n`!BVxf2paKG7Ps9pt2KwbEBG zrOQ{+{bMI{d!fm$D>EfMU;eWZKL{d%(K?YO4}v8czsY^#Z#j&l!wmbKv`L=uj~+d$ zf2EE;=)Las*01DeEK0u7qS!B+|7IllZH`rWV*5yEHTsdhX9P7i)D~&YcBi^GNeWk)q8W>A9l}5TnSn%EW$XLT?Z_>&MB9*z zTR6#g=*}DFsQZTH9up$-Q8PKz<`Xk>c|nYJhIgdxl&_3b38|i?hEcx8^EKS@yBA`s zaBT)A^OdX5IltvT*}$6BdQ*iq2k6b|IJ6JAGz~X%Iy6riFMiP8rhSlyd$ztbnx0H& zOZjVlV!(x5HjQIoVESsdZet$tkbN_2=Z_HSa@FN-%PW#Ji)>bF6p5uwX9Bq|(G_Zkl*tzTC2cbE&<2i}B;5q0w(PeZMj;Ta}dSGMzY& z_lX9Tc-8u);F{4SJN-bpOEf9H6n?$f&6iie@#Q~?3f#MvKp7%nykaKA#+E1mj}=Q(trE#KO7Y>zt3 zG-NiRcnaMfgUwnNF07WHo@V2W_8xSN)ph0`^4|6zuTd3!9q788@0~hR z+#gG1@VqU4dorpKKLF_^`29R~v6T_RKxs;=&K~~+JfEcKs0Y9HPh47NJ>w{p-ex?zil6A7OM{zD!* zobq+a!Ujy8)ZS2)ON7)OU<;`hw*H-kS^3z5FarT7R?vtV0gyIUd&MQehhO&eaW$F7^1O z!Tn?Fg3VI&w^+inSNiHXGO4d#Mr8)}I}a3UHBI(Qm5tqx3BNjL(mh@1oF7<=gRO-! zzrT57!gw;F#604_=t$6Y+gKGTf$iBR_&#b*bh<^Gbj1T+(dBc8(>A3KC7f9IecUhl z8Nqozb0Ixs8VJtgqOXb0-hxn+<`UlYm(-TB3WC%x8sbjTXa1zFc1Y@aEkMXZUx+$W zp$-@7vgR-P2g0Zdy;s9Du`1-hc|QjI*HlN8=H3=`rL~6e>Ch8TF3%W05Y>3#P%UF9 zc`|XKK|!l!Xtsta&w1rW&(nMUSY5Bd$u>28DTG*TJL3#E!|hhyb0!cVQYz?{r^BXS zPquq*@J)43D7c_MMB^0E>pda^4sT=h99O0Ns$EB)Is81Cb|L*+fk$XT`da=_}fU3I~|)B@|CO9|_QJwx_bmQvHn zp%Z9j=|5g&Uue% zD(u01EYk9vy-}PK=deg<)0=pKjV*#XA#P4{zDoDN`gG8hyiO&7Ql|`Kht~Dd3B%_J z%or!dc&;rQ<&&<(AEINpS7<5jeKjgx4Xl6DY@W87*Sy7-rckBaCV;UjwyjXu&TosT zPq^P?a}}Don?#(Afz0uz+c*7ZY%8|Tmb!L5BgCX7^FgqR{YiAOksUepW33kdKJ`cO z!qTy?G6{DV@1dQDtkFHrGqBV|pCcvnN&T^sKP|2v@r^?w95}{9d;3+agzD-x#D{Yh zoW)D#$HD>uPN<+_9IQv0uB3tvsX>2F-9oR;MvUccZ(;c~Zo?k8(n>cyiOkeLr`9Ma z>{zxe#Li%#Y*KMH!m_00s-7nNShH{ zw44(3TG}tzoZd4!0izosWns&GJ_iCM1el+ya)`Ga$(q!@iT{8F<^A&+ukrfgm_4Ox zni3`7+OS`$MyBTL*IKc0%attUhmW6s5nj3>*oJSxam7qyDuCK6w4Tbn1&z!+`6l8V zQ=o>?IM(QXt`=8OzV85yN+|3ed5UiD$-gO#s3;*2(!MRYs9!Ike#kI)1W;D=#t+Vu z!K!bq^qd>@dBb=^bqYt6f_*RDEYI1ta?|3a-S+QjE_C(HjY)eC7guzzO5K|J=gdo` z;}s^&r!z1CQaOOX=`?(!v>t-^#I`E&b_vYg|dI8_%=IHnew{;APLe_=)6e$ay_3lHYt6Y%e)R8hhU`N1=3?_$6&5YuKqyz? z-EUni3vJR9(}^=m8wRU`?zX+}Zi&k=O)gzK@sxr`>d~=t<0tP3Yvt%&U}Owy;D@^< zAY=;E&sxec)_ST5`cjgxTtp5;bK>kS)OW`N(X8P&WiQRY^n6&VEW$Y> zvR#{#eM|pL&bHOV*>c<4cRba4KY7^Mbgj{>oBm{S$3bq}uxK*hxc#YYv+Jnv@;jx) zuZyRg6`}GSt)H*%(ve(6m1lThs^{m*Z1e@1H?(wAJ{dLz98Z%q^y?iql;U5QJX(1@?jdVN*PC273Hr0K+ustXA< z!FnfH?m_*N4|3d#Ej;Zqzz~RMrB5gu7VfOY0bQdZEzyhBd+j6j2>R3yFTKl`;VUJP z=}s75+#lc-vifI~MI}!-BYk05T({L<-YNp`N_EJ&{6A|V zx(AEQTSFJ8dbagk_!%d+Hkz-f+A>{{E!m#Qtwz#6#a|+ayx$A(rUCW9vltUh!IX(jqY_!gh8J2 zX0e{#7N1Rw!Yzaf_I=C5WdR&nNc*GHdv{QjrXE6Pk#4Sd0mu1G-s4e}&Kf0yL+AgQ zl3z*OVP8if{GlgD2=PA9kig!RaaE$rX7s+Bn3xpZ#NjYQ(JM1O`=vaVA%1T!jF|9+lHr`5n?+LmS@PZ$Cs3LqP^hf)g zowqy3UQI;|I5@@RR&D4UzpHb5fcz1j9i{P{3zR9{$AbD3h}#@5(z@BohoV#HFZ$vv za|PMTXY+#8#)Kp7wn{|eS>`#P%r1llq&39g=&pNp_j0}vS;(z#wwgJ-|Gk(0%2>3a zZ23ola@B?IwfnW6-EX9qY>f`KX>V(rE9iQ!kCF2&ziWe8NrS#*SuIZTTa^H%Q{2kW zqAwy0F4=~L!g>`d$FCWAo9IMlpxk0b&8Ot}mcJBftR{c`-;-Y&LcGW*3qySan9Sqp z#N|k!LJmBde!vV{;Z?!1GX205W}sB6+ra{87DUX3N%7^GpFLJYn3=8&dZ0|cpUrO< zzJ!6X3Dap-pcm2sI=Y%5zAb7`VfB<*LPBS2&3l+5Gi!RhK5x z7b(lCadWn9&QI?{vQ&HQOc;%?1pXMI5+f5KrhGdit)gi7SkLlA!g}uVS~GOe=qR?& z6B&3KlsV7EHGD}IULOHD{Gm3oXc3_a!d{mlshnqq-j!lXXs1UP4f49OXP^WU=&RaR z1koC!xsX{Vd11ZrRgDo`y{XcP6~E)m?D`vF%Yvo39wx2XX>a|pgw|ieu{*~GY zpN2>!=0DoZ&wTjrv1USUbXI7d#UE=x_54YI=&K_bN@9cGY~3`L5Rt8tIIoQrlDcx-;W7Wd+2 zIzSShghIJMcaLB9H{lEM#Q;VUnXY2*Di*o^1`bZG#E+_}&|NY|C*i2PN=~kBKfn9H z;*!)r7V8pHJt@XYzmCQHosUtH?W`*>WS#FMO1A!Z}= zXxZI&{VNr|W{uAe-%cbfj~_8!#$6S77nr3_N!nOXK~~HkLoLJd3ahHWJzB*ls#uo% z>BkrGewoau!_CuIEKM52O>USmUCfX*2>I#vGO0|0D+oF*3(!lwr}R=! zNt`ApQf6|>mA>T~+L9xHOzHFX{*(vj0MQr$`!9W((*TYx7BkNoBDnaNr^Pczw^)fu zDUJn_hYwP;zoc0Y(WmH{Wh7UNQv2tM1flZxr4~tJ$7a z!NUO#aXq?tMqW-1wE+@3auaPClD8e08y!r4@(wk87nonp>PXe)W$U=XsZ4ng2@Z`2 zHv1_1#@=x_jSGHL){-rA@1IDE(DfL%BuWymvAK;WidNx|ZIs2FMh{(HyjnU|)Sx{v zrdhY_K3~da{#|5iEqOM7UThItWWD-bKkMXODPZeGpNc(|0jF}E70X;PiczOe197~9~dsqPku@Pb$S z>z_uP+rGp)PMK}t(lNv#*i&`q{@u@aoHdyIu@U$l&P%U3xYS*;{eIJ#{(!M|;lvNblD<5| zPS?H`^E;Ho(m(YfNGsCp_Q%B1r6td@+l1Xg9? z^%Mif&Nqq?Y&j*@f!;ow(L*G6IoFGY3xIAfRM+wuQHcUBIR$sW)>e_Itgd0^zRT3@ z(E3M1d^X&t!Jp?JQ?YHi3wP&jJ-sjzNs1YYHch?mc9zqq5a5U=x{{`$QtE$0Z{-Hp|Yt^NlULn5Ux_YA&QC z_w^g^0*{B~C|%o%iA>19SO3%(CfD4Dl02Q3_L$Ckx#MtPHASMo+c)Tqv}-q=cw z)??ufDO1fkCwv`U#k~CH!8EdcK&Os3K;Toi|`^ z^!*llQsx7~6MEtGae5h&BjrR|o;}$u{FQ#C>^yb7S2eh+hm z!-o$Nj6STH?afbx>zP*jNSWVbS+q&`#VajxsNwJTPk?I{lsc( zR%w@+@VxkpGx@D<{SQV8Z;x37^_Z>ppI#ksp`K~o|IF&u7P@>ss;S`VCE$s8emwtx zj}KsMB4+8@1b-hMqMgih1bPchik-o{lFs2jY^4#8e^`bL1o(cUuZS?)fDvmBr4`pR zOHbc$gWV8_2beblD9%TVALo-mbR^b?wztfJ7UEgH0~jxYO33ZnKq?^exNp_sv)xu? zE=ETv3zM$U`xXznHRq^#b8_eu$aJ~6HNCMj?DZV>JzQpsf9;3=iBXWK>5eA%$5Nd? zNfSyAb>h+kAq!Q1tg1}<@8WgGuzW5dQKbLtTd0%zuObu(+^9at*0`EJ{Zr#2V+)W$ z81O(GlP&Q(;lD>3#ug-!qJ$Ii(CA9`3M6&%eZ>#Nl3ku*4gyU0&A$t!Avuco@FI-W zG9zDjIj$AvHPIuY?dQGLe&jfA*5cHZ_#_iN7G-tc%{zB+Ll)mrWUZ?jwy9wOl9B9f zL-zHQg$A#zBqbl-oBZ-2_+G9>u<*v+7!hx;^h=vDia)A!Mxu0W!k|xIVp+AdG1F0K zsZ|Uh6-lu}c&^C4*-i=(?EoGAH4jcEP}i9lI;Gj=IVaO!)V>3fbuFevDJM|S|8D55 zBXnAr{_+k%&tszhK-Rw8Z=#Yre7cx1TmBi(*L&> z;UK2nygrd=3$TnAS1hBJr#kDezE4S!kdv2Jyh`3r0~!uu55G*b>bCwHs%@~Pb zXL+|QCvT~k%>mDMl17Y3;L#~b!x5m58ERpyXki@_nA2fMYsxFil3T%(@Mt?BtE ze#FAp-X^NO@?VsDH=hFcNl8wB%bq#O*bh$1Am6lgmiJJ)WC;KCPQzkgg>(mePh*R` z;IT@xTf0`XXQ7Nv?swYC!t1#!-B;wfgWG8=KlbNTh1RS0(t1=P0R#Ns~R)X#?&Sjykkb`Sjqd?cc-!(!68iccyR?SSlAUAo!q%%qE z08FnlW44P|YHsDwvcxzDuK53>8OeoS* z1$2}DKzxc-e{%*e^&@2fa#mg-0tQo5wN+LKDvmv`Wlup3KTc6hLIv_0w;IpP|9t!l zGS%ZhY+kMWd}@!M{x%{ZL<30l*BhDfABZNA_+tT;8yB>(o%O{+PGu-X_Jny~LB>7k zLFyD)O%-8z%Ipj!JF3TDT5jqKjsePC^)^Pm)1UCfXt+FweZa)^lHAj=!k%Su?&wl< z8QeM?n%~CXJQ&m$5pkMBk0oBf#rcVpFt69rv!}%F~&jy<2*XU5SFCn)@?Yc|AS8qfom-8MPPMkTgt~o-TfhbviV1RzB!gt7O-~M<9wR)OJQc3luST^#LE^ z`x=oDl-YUDS7M^z9`KZ-y5!KTTY1kka3@4)6f7dB`?<=1zRLh3h=dpR^J>-N=^pAoAB6t^h|L#R2q!;EH{C6B@ z8$Q9M-`F9_WzuvlbK?@w$AW&B&^gr{PZ>}x89@K0mO9bglw!pcPB>-v143HB@YYPtBt)YCO<9Fys)kawIQqk+`%`r}Hx8+H_JElVa z7t|L~BvpsKYCGrmQ=+{%lb(8`|KWhvF)XLhGK6u1>{(8bD;gnoAmKtlU!%|n(6jSZ zi0AKiww!2-H~CTf(7g0obgidJ`qsw-D@6~Wo2Uc#VGpCl=A`GEM++8WZH;H+UECCA zJF;WrvT!UPI%W3bJ%V^^`^XQx;J+z#KB7+eUF*fk1DM2HC+w^F-M-H4ZB27UA_G-L zlMwwsd??DjI|7JfAF{8R>DZSXLu@U@pQZb*Yn@9V3PvdN&%Sltj2u>U-XJZ=^uLcxq?bGuH1WEafLD6Ortmu~x` z9Gcm4W|Ty5_ttx=w-~Xr`60S#W>K7m91+aQQSHK5h4HrdOkP=Ge=OW}83p8bY1}~e zQ5bjG6GQIp@3T?<;~}}hyB>=#NxlHj=VYGAdkho~PflXFy-ftJ7}1ALYuOLb0U>$% zYDgOJ|Mtylg89|WS3>A(L+-AMQwoxw|Nrr3oTzPR$|9L;o<%onk0(MVgG^TYxbT{h zZcX*AkB4;AtF3yu*P;5D^^&G?O{u`5%h6{dzdppi)q^qAO8Ql(Eakst>oS8^PaDum zlK3bRJA0gKrZt1RC%Jm1LTret9HYmvz!~)i=(Ii@*$#J|z_WRdB z&=*K(nNELwkJr4Tjw&3AS(q<$lsQ>~!XD;gpxuz4A0E(vC6Jj~Sbi_-U%L|>Mn-*c zM4;6b%*gXQs^lSLts+96>J0dW*lCK{y>zI3_=2DbVoN^r?1S@S&@V(`iEWnHWi@2y zXtNJ<5>rDHvpvx^*K;a0x(PA~%m0;+oz7}hp1+=^4DBGK;(P7YmGZ4KqKdpyIIMyL)iW{m{e5x;ZneK#$^NS( zc_UEq1aSbYv{Qi+;|UT2=p3%*qxmv{~>)D`-k^9{sr-Bt@E4+Kg^B&z4xK+P829q=nWZZI}_ro zmL%%aWwchouu;3xtuX86``Uyg&1{`K$i$nru=AWC>!xIVL+TU5yFY%quk79o6EkLd zEF&55Su_+7`XXLQ8t#cb1fz*Q{1K~`7&zbaP=+<~-1#kLKx2rc*+GIc;~?LF^KkcB z+0*j|HlII|7O-Fbq9+b7U$*9ESKFuSV!DeTUO8uix`e=2;fIWva(B@P!89z}ivBh(- z`dxq;zZ88%fJuYPNT);Oki_umnM<{8OR6a3ycBQu?n47o)d%Z928OyMm+%ldLj07R-Sn}6+W>Y{zqsO%`kvC?b?+-b788w^*R3D2;Z`k~ zKWlauk_7-2U)WMQww)4?|ymBN%$DW1$<{?L}$-R#YAcE$rYu^1Q)8COXNIA zZk25uv`9}afhvAMVlR?uD8|GK5~7}d@qXr)z}gXA6h!sa_R-S+k*2ZYvjSdcdM)-b zwy_h4R?z1BZKVoeB|{RqMrv=6Ej}qb{yUle{bL6y(GdQW)6XEJf(&`$Eyr3!CLl$+ z8?-^5R|Cq5OP-1Zt|tYJUOO@-H~0)7aHt@<*0Dc8UHCtQIcP$U?4Q^K!u;a$5!DjI zg-(GRTkE5vx43`>EED6ivECJHdohAtx2QKdR6p2)y?pit^Vs?sNZJ^bjp`3@M4nv< zk?~5pMgQj(g=~<#Pi%?f>*ON3#7vENGe3UIy{-9uc;+A%dORc03XF~ZCn-ky=^0w& z+FglSKB}Hx+*1WsmiMFM{{a`%z(ZO! z;7vQ(B;b8cBK^S>`C;gFMJ5QjmvzS79|=ST_$7!e9QZ^;5J2_Dr*7gOUiRy;1ACt_ z$%x<@oXJOYzYXtxCL)ey(UCk2S`1Bw|DzoN%if#PjOs-uZ|BrwcoLBni10GD28jFV z(Y@!ML;^3ezc`>ZIL=DiK{G<*7oehdR8Ru{P>EU6{q3rgP>rTEp}cs=L=&pG?){{^fcD@VqFRiE81NDh?ed?_8Q8o=Xlo?u2Ix^F*zlaipj(AbH*u*59g@5ULf67CL0bQTGB#R1>7Wo2N&d4Y1{E>GmRKi_rxj%=A{c|>X zN0bwxi$j&WfpeQ)loJCw2=4?Q0Zk|e*PYKtyNj&*mm}RW6l1SiK*!o-y;pn8Slf3W zp6dx3dlB2`CK{yhUET3Pzh$Bt*U_%#;^TQ)%}n;&Z(?l#CG{+PAhNDCw{lg-U6^B=Ys5>> z?SV5*sGe^%IZ`KY;K2^Bz~9of25Vse<@P=Iz;b~Zo5Ao8Sa0Bo{3BmvM+@Wj5 zV^5sZK9#Io@5>Vo72q2Z;z}L$0)cZoV6<5F`6&~d^hoPkvI@*KSkK* zE6RNkGQ?bvBz)tAo#Smk@!cXby9=fSwW4AP{8??%3&h$S8;@Xu(;(>fv<`QF`9StRrcWYDqqX04B#S5Ym{Kxg`m`9I%V^hL z5l8KBmqw~sxM;x|X7IrzUqNC0)@yU2UNjtr0_i}4Q@>)~!SL?UNRf=@%wnuk@McK5JzAC!TZc8g-KU6V1t(|kmkH?Nab z?iA*FKwOE%I^MuJgljY*IxnWVp4z)Y=s~4bgtYPTGhCvkcZ9-9sAtfx>*4C+qj}kl5}w>%tYB zDMrq0+HkOC`NY^PceDv5{cnGh)CKyX-5lk!w?k5FM^r1_oUS1L2g_GNo{L7& zrw(`d`02ln40baR&!O|yK)U#hjYi7AyR|#C*73r!m}f^Y6>;9<+<{ox3f8+kcm$)m z!_dl7pNMxcc-x0~fob5dDzNkT!4p3A5oFa1$Cy}jrHAoD6MD{d>p~7#(WjO1yL)!` z~2HI&D9l4}h^gpwI0-kAM~Cl`AO#QhQ zG4vwi8*a>&)3Q~UEAI<-9n30)0P)u-4o{1K0x7sH&Bm*Jv-Gi z76;VXe`3+t1sYzeEIdXuaD&&UVdHa;_2e3gbf+mNey8Ui@tQ4ve26ae%>LZ3tMQ+C zrmN^bu)%MWARhH+H+8|Gzo*7m@8GPxB`PwsWd55>E6<46lLr*|BwQ!20S&pEK1A^K z)xNv*PJlSg1iqUV;CU5=Kr>)UWPnp6kG(-u!e1)+DAoA-pK!fn?_``vP<;+}rBf4`&)lQHBwpWfTcvI*)6Stv zMT{nN_97RABn3zxSNjaz#VR#ZI1PmjpMn>790CioFh!*EV8;@7{ty+$e`~x7@6sqY zQpHTfMk-n(J=||wtpU9dHNe8gyMgEHrgEPRW90({-0n(u7xz-9nQuVw)6THwUV1>7vTT>7+(*<6vp`(9~X{Gw&oFa_g*XE zuox}zM@Qeg%nXzlL$gkzi}i?3LLKhG;tf7L6J{=>nthCoSeF5(=sx2J#f~;*vPD46 z$hZ#xnkyIDcCkeR+ApAmx&g7I+i-hB%j{I{uKC~}0Yh&1D77bXB9RG)rJQjOBE|IR zas|M(2Sonx%v~+%*kF9**F**J|2y_CULXmm^OUvm90_Ee?c|?MPwtW=*MHuw)>6tr zhH+#{!NX^jGJ=Jl2Aa46=p9&?1Gb`4)IDlwdaFI)o6|fV%G%|uXjm~c+sk7obYWrc^@39 zdb#u5Y(y@|Waw0RxSCwI`vzHKL=7n4CFfLaSd<6Dw46nD)a<&?e`ZxIY`z7tjWtyL;UFYW}_<|S1BI9 zLNSqHnzN<(sPW~Pe}MvA#%Xq)l(RJS%gv&Oh<6h`2_KStS3^mg48xX|%cr(yM_I4a zIQ^{U(oVTuq_7`-H#in7Lrdqwd zRqmN&nOmtj;MAjU@!`S2i~~H=JI;tz(+FlU(=YA>zi&5`VvOZ6Q~IRye6daz#hEHx zFZFi`liuMu?>wvV-)DtuekHD0edj8izzlik7Ja`R)WyvMPv#Q{7SALXsr9H1*EHkU zc4p6E%p<3;w$jNWfdB0+%&;SO-dtf0l<9 z&K5?C(SB&?5j*&3MbVLAGGdoErH&JCVX!=?J&HIxE;2~V9%qak2`e$Q*KZ@FM?%By z0;^EqWsG`Cq0Hb-N=EtQ@fVqf!Tz7-4M9ZEfQ+MjtECPHE$#jpMpX>dtp(CUkhD}d z9Xv%FDuXzquhx6=GSLNm7y;xYc&=(+u^kiy@Qq=w2n_dA!^2d2a`b2VLlHcz#rJo& zs``78r>Ikfldx?-`{xDplVD*on7@EtyxLq*zs*;+Tg3Pz>389q*vhb;$_aIzhG(zt zJUfGi&pyu^?!nH;>#Uz@FdNx|RJ$8^*0*?n!7ZE`P^`Aw-XT&sM~^1o7s*PRzHh%- z0RrtymFjg#EHWIadcM9;Pd)b)Qy)y3DZKehyt~5H5tGwG?<4wf=hO!?*^5x&MA`Dl z*8mpLt!rDE3L)2K+XJ#a3!vJrhSY6Y^yFlpy=s*XTI|m^?4!oEY36`_mw5EKC(-v4 z$ZZ4fZhtx=u&@Q#4^==0i;tQn!oE-Y9;SL+K`ns1N$Tl$_YQ2G$!F?5$#&|bR+A9t zLC6B*1^M3ZImK-6rryN9O_h)CdlriIhrxrb`MJ8Uq@pzT5LcLUnxAwSkzs;rTf`v$ zyMPz2x7cTydtv>}?8v8}PA@NjSr>jft(*QvF}6Ru$Sql(; ziMH->xD7@js)?83e$w@fJpx|P3Mb%gs?Q`SX(b=MgCbyh9#p1CVyLv4Y1)da&d4%_4Sv=huS(uTDW%Z+oqPFh6Tx{Wv!i z4s}Yy(F|Vey>N188MgJNjiK-SINHam~L&ec$0VJl8_+yuE5gd78}`AA_0p1h4?<- zBPZ8j#uJI}JoaW0;z>7L`I>(t-Jr5;x+66xob*298I?@U?5ycqX|o3jzL?q8Cm>70 zM|3UZXq01%$ucc?)pgb%C=$^cb{hKLCPl^SMh1JsRoqlsV#%fHfCe}GCOzb=-s9Rm ztG764=%*3A3?wNS7cfy>d|ywc2KvY?x0Rj6_W70&dH@u{0yE&E#io_hl3~t(o~WyD zgZI6Tyzj!8d=kJ?RgpVwT=suMe5>{m)cv2&rUAg(=3wg#?<9AM#=iQ7yYO0tEQ)yb z7UXKrGiqQG+6AKJU9G%BSpaD7`m7IG{8#aY@@XbtU`hzDmwx=a6N30xqjM z*7I0_Rqu_~1flYg+{17a7>&#R&N>__MB*5PFm-sCpI;q!Px`}{8YKA~ZSGh+SbJSr z*7bqTE6Q|g-xQ*T^fXHTdnzy_Z(Uc15-TS32z-zfp62wW0=d$O)WDeLJR84bS3{#^ zI>9~VPHkpK?9m7q`>>q5fCumi4LExoveo^J3>~nF`e11p{1`OX4aAcdAu-~ zReQkg^=`2dul1D+W>6r|;vq0`q~V7o1u~5&UgEQtS)x8XLA75LzCw}|qHOqu48j{t zFqi|_WBp9{RoYNvc%ulEdjX~I2po08^AqoqAiw=qZH0Bjs6scO%e~|hmYjILyvku&BF;bk4Z$yP_?)B6%eQF)n7pm=strX zKtf2T2!I?l;ku?LY7uvd@CX>xhaV7er~tsWYG4D7BD?MOVMUh1ty24ZFCs68myPF4 zYhR7WAEPF*&({gBdTS%I3texM?ET>}wUFzL%RDRWasTHCB0MX|44mwCrjV9z{gnv1t1gW+xzsLfy2M>MgK9R&(aY-7fxWYN0v2g=hzZWgI z6*_V$`a<^MZNaZ&``;bhkN*uRtU}m)cRLycrzZz1%g+ipTiLx5v}LpxX-`rQ5Dmn4 z273<9)jhGRJ1PCKvzOEJltGcW8`>{gCPHk;<1!p6iC7v~a^L_{h1Qp< z5mIt&9#p#a10Qi+eFu%ZX4!HpI;y#J;eSVh>q!W%Bu~J1hU)EiZOMCzGCo_^IuR3I zlBldWUql)1-LG^jEbS}YEl5}8>p_d|x2Z@A%e@|^9T^a4rE~p- zNJNvephXcPjS1iNpIdmqty5Qgd0Ex6CufCI-G`lFH>R=V`3UNl0(8s*a^+XjQPo<5 z03ORCQxj)PLxMbs`2L+f=F69*7iVNp1I{}dO^Ok1@CA<#{^M2XE+$j6&=yRvg zt{0+8FDj!cz})o3!Q4P=imw}RYb)R=P6|;l`Ms2c-%tZioJFVoNe8%%>qJe#WI+ffcgr z?i^%CACjhXA!8Qs**X;Qg$$aq4^APCT9pfU=#nCLgEpjv^-nLNx&~&+nLt`JCBFuE zg>HZ(jTydxvrYS2wyE^4Wg0vYIi-Y%hZe7ekl35lPUpWPo%SHj*tHp>VOYS^`ghN5t zPR@3(u`H4XjbSJh)d8;dJ~OONr}RUkx(BRFkZl`jG9h?q{b!T{-|jNIj=W(QqH>#j zWdXahI)G%LT976dx4v3L1OQCY zg$H+~=J5j;H{Ze&!try40Y1uKSN81_-6ty{rcMn?fNx>2Smo3}P-I)R{>E)Q#_ai1 zvK6=jAT<(>-!uAoBK*lECAo)46qO__u~jdbI7sWr7f27VV%d@1&cdiJ1Wb>h8`Q9I zp(3ypOJ=f)>*S&MShxV<9OFQf59{LPZuVjnw}ZGBNSs{(oUuUQc51Xi-eIFgvyrN& zi=2jWWo& zc0=pPG^S>u%Y5UYLy@7Gqn81ueXbB%zZcd$hJgD(M=2+4E3!6xpu7suxJ`pv!sqH- zs&T7oaI%Bc4X-|ww{JD;Y-{m+(mNkhZ(3TiYvVd$|2o-5NKv#6o#t6QFoi6)>5Q~Z zqI__b$exSC>K=Q=@(AcEWT7?UNU8K0d`zggu`>IRqhcCYjp z+J?+B)VE55Ah%z&L%y6%fy^&0FNPNvd@U%fZHot-)vWuAGywGy)FGtNM3$6$oy=VF z(t$$!$siY=OOtdgriq6O{j3SCpAsX}!}4%D2Hw#_B_N zc%3BmI6P+(7vgF@N9rum(=^7uX^ez>Ak?O~xc-!XI{kXQ?tv8%(dq^V3eu=_aN z*SW>*QzwjjgIrOR9Lz5l@6dE%x`94K;gma^o-UE?_4Y1@FHb!hhm-t{A7OnxYTFI4 z+=EeJ&$4w2k=e)*cIJafGU5bA5)fq#P)@hm_w{&*4@fiv3viIDtH9vor5i9cXU{l% z7bg1{jX!|E;a315trU8^pLu%q2DED5YP^LUK{YnkX&pfhL5JTp48!>*M9>KX1&N|% zvd4u9IJ-)^2D8U_rREFKqnZydk&iSZ0 z#DP!O8USL$tAOY2{H2Ct45LkjjLH(>n0}Ba_rydmIT-o@+}?6pR<255>R3#DL|S6Q zMDfkWO1kPPp99BLkgE@Iu7DnKYQIgE9KmTg;E5TShTcxJPPuco8K3qA<-1{NLXBqP zV9g%Qk^?F&mV=qrpU^J3SKo2){dT92Q+(q~Ji%?tzp^`>spA+dcd@;4yN;k!{{r-W z2pE(^7Xvs-hVjY=7EhLt8+u%_pAhxiX`qfEjquG4|I^UdqQYnRa&~1QjT^6HZQ*A( zEbt38BICMHXvDT1zo~cctE9znjc?eDyu~jqJYk0e3J>1H9(t>*e9@5-<(o>0=mPMP z={?7X`#d*H)MrK6_is3DrT#zcy>(nw?Y1^72#TVlii(7iih>{^NEvi@NGKuQ0uqZb zK`H451qmtXMJzx?Qo0-IP>@`*sCP_^{p|DZ=e+NG&iUi}e!so{c(#FS&3Vszj(d!2 zT-P;@v{IF|bmUwOCu3E>`-yj3O)W>6#Ha5$-;zb>*!v0qak+u=Nj`F{-ykrmR@qQa zk!4(Xb|)9<2C%+@Gn%P-y7d+QvZoKPRsZAg5d56-<>yOv{o==`6eY*vmWy9iKYJ~^ ztlERrKnyozf@@*UwROQJJXqS(ASQ$4pHo-| z!Vxiz{q_m-U~DT;OmAdF7&({RL*UyBoJp;5R}O!L_J*{93`?x-!(|!umR_d92a{nM zr$-#LfsVd?YZk!+5N=^pBqxJf;K*mFeZFy=ygC5mj8dPH(9B&tP$5w31_L3(7?;^0 z*Lqj(mNXaw6zM;@A2GwP~v$sJ&S&55KDGl+D?h=8}t;W|Mpi8toi==V1}Q~ ziz7($!gup^6_O7AAg+rDF&b5f0*1IJ(k0scVEGfTuTmA=osQi*E3e@RJq&sya?s!N zzHMiF5^kdF3mhJW_vyKn<8UGy;k4fM?F?*$6l*SoGme1`&nNlZ<25-(F;;)I)%S)|z zIBrrQipVw(kmK+TE39|nxmti=$gDQ}$)^`ox1Z5CX(pE@3)3VPuQD`d zbW%A!DUSdk$82`ry$N14X07SgJX9^kEiC5Lc`o`>?bAvF5-tS6tcemcd9idGsM(!1 zec}8;5#<2nPDQKoR*6#6cS&cHuSxDDjs_PlZ>;rXnZ&!d?`(VxQ84J2j#f*@hq^;= zZkKWa?c1{j@#7RRvykEoyBJJ!5@1T<@>_*^555qNyWSu3m?@c0R!zd)A9FC(mG{1F zUGgbSo?jsjz4-Os>IF|{eTOEV@al0)Y@Tj$m-=lj3Xx`~=Q3rhlr+n86jSMxSpPcd zS(Uj>e4I>UxW|R-*Kq(Yu!qz752umF-cT~E#t}<|4?a3nb|Us$$skn_x?+dIeRnGB*Sr^Er9rM&@XrMXPCG|Kl0WXc|9AXMJd$c zL=$E+XxA{w2c$@M=3Xjhg5$QhlQT{t_!W?2X?hP5%yhUsr%CeSEx9q-!d98CO7gt@wCi-HmOY70u_A-OJ@RnzSYNhwb7UsyK?pc?MiG?W&I>%4eM zT)#p(x-NR!K0!f5jA0rFy*?Am`C}K?coqvjZfk$_aHfmdm^gIH*F|R8gO}2Gvo9@| z6e6q|f@r4ihSVi`|LRjs+(a@cR^@|$r%8*76kS=PGYO%8MIg>(SD~uX{-_dx57gk8 zlbq7qKIgj&soe6n1Nf`6+@DA~DH2H)YOtl*kGjP$QJPq?R!-JAy5zfbA4 z$wpOJI<9Y;^2FZKCwv6V0|lqhsw(^ho-4=w!d#6WAE@-*hbooLMN*oT1T;sFduWXB zr-hYOwg4uA1{L}Sxg${1tehUrgnYRa4v)5`-u=iy@&tbUMsYe)iGW04CQ)_yX6TRC zM|SKWt-1~Vf_Y$r`>D!X^mRdPMy6kBIbd@}UCpf16B@h;zMJ0$im?J>EsltMV~HO; z5+dRqjj}^I^nS#dRP;3Ra6ugU%>018HcQt|W-#!P!ubVa$#e4{ZqzQYP#@mk6vI~x zg6OXP7BAXJ>SIdE4XHw0Cnc=6g$jSofR{h@3l8_H-|EgX3B(n zQDWgt@}Jy!*;4#D!6@!j7J{%n=b%@Ru;0l!tB>Amz`Z`KgIeoKZy}c&HMt=Q@6ogX zEN&h}(C8ySgKt~4{9o0?Ee@Za!~|UpJF~Sm9|11gTF@KE8nK8bG%N3&)*l!b8r|Nn z-ZRME6x@Dpc*uMCn634lx{S>OPd}RsS6#-u*N9h+Jq{(oJ3V+pYbW-~WjlmY(u}z| zaBCE1I$R0(g8M`fw+2LL%~)zz+%6;!u!SQSlCj?Gt4s^EpHh!u39ZY%71j0VbXs&~xVOX$ z-5g2z6hI#F5rvmad*z;Nnw3trOU~01Wh0pWs#2pvSM3(xMHW|AA!r94!9&7f$kRG^ zkMi~2*+0%faG<9FFqAY#$h_Ioidt*t6IUe1kvEh8GCNP}? zR0DN(?<%}tz3=w}{7|J<>Lx3o*ma(5{}jwYQlBWt9Ddz+t}DZ&o@aAfWVi1vF#3~m z&=L-F(u*RtE1kMBz})yNvDLztAB?HqHd>>Go$meMu=~!&wAt=r&@SH|=|u@vHt@9? zzp>O{PHe@LEO{D{H)#8=eG_TAeqFfF3F_VO>Nl~EDk1aFM^L8upqnJQMg8U(lG|rd z?XSg%TZGU4-t(@D)!PO`vZ2Eq|2UqSk0RkC-;LWTgp;WX95HdE`wXJ1G~%#q=%;M7@0S!y}MNyGslOT9n}{ z2pc6azKGG5DFNJg5C?FckcEF)%kB*>k2FrzbDKyjDPk6(w|GG1TRQe zo&lS{ct?IV7_-#4+c;sy!XOb)kRl&vzUv2A1_EBH=)Zfpws1aX6CcjgbSBiq&fqrq zt7OtR^_~rWGR`cu+X6NK{hKbsXr^9*j=6UZ72`=JwJ zcakBtD++i+vZ4(lrBcU*l46cH(G8EhBuM&1T&bB`bX}@ z)OfDDY!{pHTpzCNY}cG10G413RkK3x%*}-u|32ElZRA+7Cxe6XjO$AWQD`E9{OMiJoWyT#-wGpmoELP)JBC0l>_5QdTXr|eNQTR@ll=8c z6nf;<&4c3(Ehb|g0BHHqJR_yE?i81bJ~U^6<#m~R*mJDM$)&K%Xb4&uHd{{)EWLc$ zYSJ7#d)sjkb=XBe5n!L19t}X6FP`$|F%7;RYgFxmbv(t*p>Xcy#O2EK11EDCzaI1_ zVYACY5`u=iNz4m2C6nRdBUI(awke4Y*#4q==9pduh4l8zm>U%j#tH*_1UA}5TaRHa zDtfnRjXo7k5Y=r8T%3{{^RUN-g#YUGirOO!Zas25ZxU`_IieH*uNnMcpT*;$E4TT8 zlJdG(iA2--z5XHQMjv615CWqlzYB$T-s4xkZSST`42(xow-RKL&1^hRkk!5L3MsI$ zr@Yy)_Yx9mp77n&Ap|TFp|3?M+7*QWUT+c*T;|27&Igue%CbEi-PA=D5NTd9ls2Ba zq=UY!?3P_RcY(nNRF!PX7})rWAn;J4exq1Hft?0ziF-wGucg2x?Si`C)H_etiY+Ub z8`v^bse7E%%+kI*m!Ly+_TH4;iy|Q}wo@2?9Gk1Cf^zomeO>;ZVer+Rsf{e!f$-z2 zx(KSy_H}2DTmIyYl)*{Dj#a4gG~dl-rr^Z$b1Ko8;g1WrfnpmQBR0=kY5e@2(8tJP zE=*RJP8^D?hNg{zUS+$%9zFx$V?5*y_4D}#1(6UfSRIZ)T!^lMx6$%gvIBkwfa?Zr z{ZnVs&OE9FTBMVF3$_qOM%AFMSoL#{xDHYiwZ^b4P$ue|giY)ddhdcQOK^dvGZ~Ye z{!ZZ8K(+`v7eT%~OfVgGs)l)s!!S`KufVVZ?WTT5Rq;DQo7X=cF#Glc0pm0Cz2%t3Xj=io_6lD17 zE;w5f<_l=Q6p`Y5WU3Pf{Ao4Eg+czp zGu%m(2UVdh{^)sfQP^#|df(Z7n0AFo$vDH#Q!e`Tn~g@n^J*iuYPhA5icga$?4sxH zfODDKBa+q(fUnX48?V6#SXFrL?wErt#cY1-#DvhKS`Wa>nezOT$0~=~yXRwk5sHKq zAWn%khuOBTt|y*b=S5vnf)n6VzD_9ijcSlZf*bF{Frv^XuT6!s02AMv%9HQaST4Jj z0)7A)c>53Zu52NkK-Fk3d3hD2ABF0Npr#BazuDaQ%~S}G0?y_+0sK=Z%T%Gb=tMAz z^*at{2~uRElTWZ~-qFC^2DbFGAhg`se$hA})&*6*x5ligGjT?M;QMSt_RB^+*V#97 z;75cVQW#NG>(O!WUv$DA@VzU2vE#k~u*SNeOMFjNhh-rHCcGUC4{Oo#ZF(w4akDGQ zYUap(n7`^Dv2N9(vX(D*s#k6mG$uF3xW}A5%OK|Vs{jM zD%3Ue=hh5zqh8+Z-l%lxWykrV=b+I~>YvK;K2;G**etT^z6S{^^$)aM9Y69$B7*M% z+mnTgP3yHl&#jMXB1B?Ss=&vGcg3CG;CE}dl_zY3FYD$=3bm~f$s@k=y^Q>Dgr12F@_iCv9_tKg~5 ze|{r#um7}{S(35FdR0d&@r(C z?JwYSDjkIr_puwg9!M9!=KVnnd}cq&~1h%0$)Vm3a60V&UMIW{C8fL8bDB zl6ZHUw{@kt@;p#`?$|pRt4-fKO&-e+a_}8!J_sezv1b1FOUxMwqOhg(I#Irt{dRc9cqyDo@uo9T!Q z%NX^h^k*d&+_TP7>)){^&(t+eN>HF-wOxkeRL>=a14}Wlt9>TV*Okaafj;witM#KU z;E**W0wqiZ!u%;_3A1<(g7Y#Tkdy@b*udC;aL{fM+z*BtwxquLw%PuQG03=PNK}FX zF4$iImJ!$0XV9bnaCnBi-2c&iiq9Pk%~VC#X$vE6f7b~7uwWOirZKBz>$f>I6=RdF z?Ol5sfAyaJV|vkqh0snUf(Me~sh!;Gynq3Vq7px30^e{}_*rvI)oZaA z7lmFIzDWo&{RWEknq09EMtPyUBAaUpf=jo?4QE-LaoyvE*C~|b`dN%2fggDzLzOf2 zAiEKi2OgzfOC{0KMk?|mgjc=^j`wF5h#$Bp8!y<0Onl1Ca#s>N(8U=SJgSnmiO;3^ zVwPtef`4>y|Kw^Nw#IRur61Q#!z6xYt&vdgARsXu*y#@`;L`M=_bv=MyxWB!B|&y5 z4=mx&*nIQL`5;z2U(ajxn^97md9h90z}A)-zP8M;Yx9>4{~d@4BO@un#Zc?iV_HfW z6rbWl-;Ix?U231Y_%{2xD&!O;XKlEk0&GItNCP*-8gVpBOn&NpRF2q*>3na@V0o8r_@*S_P3Y1$06bm&=HJ^@t3?y z`Q;D(5qrmd)^tCN0Yz}2DLa&!6yM@C}~scAgqpbT?7uoH6>c^z2-Airev%|kXw8@p-{5~*_yaWn>5nA4x%PhdWfU&xwN;K z8+niGSAi6M5-SG?VPjSxn!iQ_+tUVKg6XJdFee^3y zwcgJjD&*OOd%P+6U%aWqZ0{Z{8Mvw95Vw~#Y_ypr8g8P(|JqFABft48r|$Nn5@Co8 zbx#~55&KE5Rt@=8xwxG6PB&B>^MHfIZ=O>~k9YE3b8M7Tg`Jg#HiOWK2|-|6nOa_` zA7N$)H=(_7Ncf5LV4(eiP5F{Iv+n3JFkk#O<+E49*fd2=hN6FRBtb%*{>4+ z&K?51CQex9)ibZRn5D+4$3C(TMRX!`o9wT>Av+Zl#V3X42^YzhRMl5Nub$ic#5Rf_ z!p9L(Y@b$oKDzjjka#OPq{stE+%?++7RucRNT%enEx|oyI4yt=BUv_OLV~joM^yRN%(M4=oFHaG} zDSS71{&wsF;3|IFpHGCqt6sg$I=usG>KN}GCG&;BlA6V5KyuTh$XJesNGZ2C^nj*T zi;Bt4bGH3Wvvoy|5(QRddsSt`HTEd{a$9^buzj)5{Xo=0$cENrY*INOCPTINDg+5) ze-ty$nd}%jJOy@FQjPHM-2*vBCcN^V7(%cvS7gRVGA6rys`&RJ^jyu9S_GxZE+2ZY zA95#iUn7IB-2TuvaJUo;pA$`pM<*b`a^tm!s>8d($85!E?DekL+<}vYQ14hQ*RK^c zVuJcX-O^>m7Zx@|`7SX4nZS;Vzl?eKzVv7d7TQ5(e`li;S$I=SD6aa(o+VRHeedw3U%wL$tsLxcR zx`4-&BWE_hDyjVzXbebZd4J5S9Z$2e$!wmjFtDAB>jVG|Nb4v%x7)34A{q_Www=c{ zbiU|buhACD3owU5zNUZm6M#2qFbyOnXH~1+chel;sf;`x=}L`0{y?_-UV5`it#3Ou zeXz!1kKW;XA3hNz@?y~QbjkD+u|Ra%gtM$DJPZt6| z_sSZu(H~ez-%d@Kd0RASFH>mQmshanK}?}x13lyEd$}bb$rmg(&US~syDFFsT+x^| z)@sH8KPA0%tUp_IzwQ-;#WC0V>Je)9L!p4}`dmhix`wW!+MTFGf2r^1J0We%VZ=tr zrVz8EML_^pot&mS1q^OA@ZAqq+CZ)MNnkdnV&aWo&@f~_-o%$ePrVpgjHjYwqvS{o z!FG&|G#|2h_x?b0=2FJsM@QlPlldwOZJY?(==XajN`My^^Z{wog@#qZ7*jt9_BOf* zr@kxASCdQ+U|iR@tP@Cf#Lasc0tF3pib-~G-MT()*MhmnJl?FoTqJzOr~08zKmcvJ zwbRB&c`43aIf$;Zwt#93b*gjP@@8T?L9EiVlXjKuuJJ1ffQ*17?!=#?c^UjvSj5TekDz`5NdvPHobgVXKkt&(d55yd@sFZpP55lVYd82T*KFohQSn^e? zPoIRKIdYIBOpj)wEj8byC0p4fl>QDsp=U@aT0b47I%(jTMK4;pAS2vtB7Po!m0((z ziZP$)m}Heye?-scn3KV(W9|oL$g#wo_-}u$C$gyYB2dFm*S^#~%<>K)D0xDX=c4WU z*v$eM@ceYL!F6$!10VD<)H0?v-{5VpN{qWsTm@*3+q`V+(y4BCtW!w!Ht-bs(DJ@5 z2jrHWzk~k6Mkk?wht*C*S`ZEG!0U1^%H^6T0T0Fwa%6wA=TjHq`OvOF@6$M z&z$~wQk=SS?-m!`m3w|KS3oLfS#J`sxlO~Jlc4(LwoR^Kb(pO_{a%hTgvB?yIY!83 zI1E|etuYT?^RGy0`RnDbLZl5pRV2oYiF9G+X9yhelX=WL-_e2#U+nRU?e*L9NUJH( z6L^rS={dkkQF@pF>@bJpd^yLV6&O+!C=yY z$uQE@vR&x>sX#-%I>-}Au5G0KH;&sWVK5?>O#J1! zPO9~OI1{Q2xvd9#(d<(1^g!fs2gTLz5t@o8xhV&Vis~1L9&3cRVdNviHdx}nR&m{T z)BYXwv`+S%PuiXq*oKY1PIcaC&`u)5ur{Eg3(C;;Q6`yiO^8ipxiPCx1eG*#m-SU* zQ4?o~0TnqidH!_s8qIns@Od6cOWwWBNU4 z{T0Br^;vlt^{OnYqAlg0S8rQ(_96w?uO0fE7EJ;ti>xSx#Yg;%r~%{7ke%(#&48j~ zw-_0I)FqkDcW1>2WNW}1f8f_XJ$4_ZaRXG!p0WF3i$2O$EIJ!RhqtliEI!mCJ17FA zj7&(cdPNW(iU*10FJ9f=lRYL0xeWe$WvgKVE=uf>o%X3G`a1{8z$5PBDFP-_?p~lW z^vQQx1B`xwc|N<*Mo)Z$G#bR!lvHQe zj!-ju?XL9N4ZHAi&aVGVIGQ7=>YDLKJw6bg$;2DPG%@-%bDi`PeBaG44>_gI076a^ z!{I-V%{X$7G1t#s-fqpVITtc<=0JW%~|EZe6>w5Z5FHgirs|oBueb{?fS;jXDHgTRiRK5<~BM zK;WR&qw5^E25Bh4+d9LL$*DTf_n85w9V5N3o6S!DLT2@-O)xZ!3IiXqJ3)$=v{E-M z#%aSWJwg^sT6&}Jspt;D76o=PIF}$QvQ!ZB|7FK>k)<(DCq5$Fqnv3SK608{a@I>-e!2*>OKHHe(3+J< zehcKW3)>Hr-q2ZPu1ZSHL6$|Dc^lv+U|Xx7UjZe{d7*~jc7|-#zNz9HfC$pQ7oDY6 zev9x74+eF?`H>|aa3+1#pSf~KuxEZW2NAUi7S}f*Y~lPNy=y+tXbBrFRQd(BnTX1U z%-B#TR%axoMK-ET=5bw!|0c+tS8&Ji90WvjK*oAn@f=EpXSl)q z?qZYCtJ)y2Sw_5_M6iU9;b{k`NHT zwVAe$9xZj#k4KUzJ;Ex!m{w{eJQzE)sn%V>f&hZx6NtHBRYA7cSg`*c?5QHJEtiQc zwA93QcEaazy-zSv@y}MvJs!zr7OaiF<2_U{)L)=sqLXATWd|6?X3nU}}JeU;{w7WQM?71Grx z7X7)9?B%Wh&R5V?Mr|imZY8dot~%-Lyx^!%Tr6J+z?@Mv(agLQ@U2~eTpdYgTg@gN zW2z;Y)aQ6Uwbh!LoLwZY4{frU&qQ&Z?KHcP*QfA6j{Vn8-CYXTt#N zHY!$EQ%x-SbpYil?J+GLH$l4q*0*1WT(JvT6SOpceYt+Yb7Jy;dSSXrKf^Hhe3hR7iumC*R(t!6R@a*Cb3y>sr z0aH*7h&KBJ)xMszV;Z9JS=no@l@s+vKJa#j@7R4~bca7`Lgpz~5bZy19h?dP`A9YL z(v1R!N-6~9aowQ8Fs8&^!{)sdvJerQARG_A_9VGAn}_A0J-(Y2MXstW7C@q^YqXOc z$aqxQ4bNhJfZ62cS=zD<(3MgrWB|F#OnTJfES)dg!R!Lz!ymFkCB~k6dl-0ifog7i zzH4=xN?m?L_Q)K52JFhJ{dwe2BDDH$s!uUIGXAHhBSjv$g`4lbq9AOx>%LykB4x(= zEtB3`*QxK#OfHpg(;4SMyO*I%71Sp>&UZS+p=~<5AyID*M%%pVCs)mpALKJ1heS9N zSjKs2R!3eGN_`yf&PH4p#dz8o77niK54y+JoWNChossZtt8Q1AW|j7wVG-0u2qg}# z-Z{v*(2m0M4xInJsG2A6VXTDIr^?2$$5#*iK7HwH@B#mKoU>^JC1^GJ4R=-~K36c~ z9*(r3f?d1)ao2X?wCu%l~$40Rjy2AtW~(&Cs0J{VZFGe?beW}^WO zYN}nhysf|q=Z_{rjqFOi{^+4zK29pTo5p$YORV;c1(NYO>5B@`%O@OaCR73{Q$VA| z#$ojM$fR3%>~BPao{SVSFMxgREC|8ZW1hSpQbEGw=*jTgnGiVMC3>r4%7txcz{FOPkpQq zD86|JX*d%Nq-(QDhF7k@5$c9!(H-|;z$vI;asU{KJJmRr8TK}BF~?XMAia^`lco9W zUwJ?YKFXnkZWVx;9{%&B*F0LCKoxbhkd3ZKD@bap=p8vWj!$+Fm!%0=-VUTOC54dH zoS29XOI1Vw8j}1ri(E=H+fs$s{ab_C5Om7gK1sf+Cce6~Se-z`^84&2szzjEk)Gu6vY!J2uap6BTN0+HIfnvJ`FxhQ@42lrvQHRg!;uMvz4Kl&apsk$z027Ns zyrpMb6{EMl_&b2eIy!mEKGY}Jvw6`R#%1z?OVyb)T)ML1 zauS2iH^bX&%_MRuv2o)KR1c7LWd`u|!;cIa8GL1DQ22xfWnhOeWo)UOK9sx7SQOqy zd%!@ojb6pzJ2=V}!Mb`uc<&nMi(P%*JQwdy1+tKlx633x60{lDnu7Z8sh> z^iEPF0#v1GdZ(+TUZ5FN*`v;bY$c3D`BqVyer3fRsI&y2hUo$Z6t#;dmIT+Uxfq5)>fTM&FAI*M15n8P~YjVc{@YqBuCK>(#9SHJO0jt)H9 z+ZlwPKo1c7`LyHQm+M~{uNfMtfv@e&Mk&0wtBH_JnXB?2p^7dpm_au0Ym>l$TvFfx7bCI_?l9^U1Xw=(IxNaSPEhzZ1^!U)E0nBE25M zotG9Vxp_&yVlYEH&LRXPYu8#AGAmuIwN-x_jniN*eeEk{ zg2PPjMD;77--Mc*hO<?<4byA%x<*{ z=u#-Hm_7=4hth%fbCNOb@ma;g{X1qo*>Q+1iyB17c9IoPlfH#~P0%WC|}lec;a6lXeA}cZQEB=7+&F_FH7}6^>y@mr?KgPxUw*|N59gU8i0cCv$Fn z?E*Gp;)+zZ+Qv(4RQcj0CPiYV*mDBa4N3?mySphy#H!0Y318!r);#bjcEizoOopEk zRP+nMD&=6P{BSX8E5J8AKsxzA;}=bdFyIVIpFf38BPYSc)4;(qL60{8?7n^pU?h^F zh78&)Nvn#DDe1r;q3aBM0*s??=|09_nJKoKu9*$9B?m6cNBbN;)XMTY#*`7CpIEZJ zVui32aevqU9wHqmw65t`O)fE^BXa!bGli>}WJThGYuL~DAWE;22xmi7t??hXp}J#1 zKlmPX8NGe?2ZWf=fat&F8ZHB3`Un{XdxWC=Cw>QM;5m;~2sVkw51;EmB<@~#`!Q;S+*(uhI*x>i^e&`}2CMWsyV7QBpSg z=9q`9z{7igxeG7REr12VQX)`)r1jjdWW^^HX==<+?WF+)_};RR5L^+{{N>BwoqR|T z3PtIig^*cLE2;zJB;PLYz5d5yLJa(-Dos0m1;~6iedpLu|I_V!I-!|=?hspH0f zp`W4#2m@TA2fUm196gLc^p|k{!|w(Gqg>)IE`?J*APQOHekWfRs3c;3KMc_}glT!& z@ihq`SFcN7)cm;+_zBTm!tQL{41|cv10G$Qt-H{qLmriKQ!luzs6q;^(x`W(61*z3 z%OB+SFpoGw>8J)An57|h>IoS_#2nyG)MtO?)J_`idVUV=0@gIiFe?N46W2oo-`|0n z>AKikOF;9&5p$fdwJ5e^z4Vd$Z?*DMkPvBVXGccbVFqLbbr7}^U1Hy6-Zl~FN>jRt zSXdOGbxiCrQS#K*=vPRXHC;lScHr!{c{Le}Pi^_m+k6n6n?JKzYE)0L22Bf`b<3b4 z$b|~~?!_2qXXlmQpH(h|9Pz^Zwz?`9`5-TiM~uq6Cr~yKm1lCe+&#o0qtyVJb*Wuq zZ*0o%jAus{m}mx5Jmet7#6qQpsg`$$qLW*a z7hlUF1i^mOB!7HoEyxJKOXy04lNBTsc$`Tm1SlGwx{cD@ulL8> zfEZ1Qo$N(e*H#}?6olR&0QAVWyz32D^$CbB&h%e(T{SHP`AliIY8(n-2*F^33~M4y zsEe>^AQB=7RWu)~_MHPJ!O@E$B}-u9p$1^I&D31MB=mu9BRVj%`41-m9!uU?j;JI! zu?gE**)!?jiThUNz_aaClAxK?R&iVhpjE?jb715xe0m4s$pkrt7lkwmx^{?%$szdk zT!jSTxq=S)MxD5#i@EPXg~n*5+xr3J&6xl_eyuW~c(Dk%4zg&705BM<)O#P)w6~te z(ZTSRIe;UMWT;E=`zFN5(_C{>=)4tn6(%2=uKl$4HEHMyba#&&=koM&p_v6gYB1#>QD#10^< zoPNmS2fI|GB7|*6_U9;989rHcY~3X^iU2@Trha>kRi)iyZ+y_)V@mn&f?|6 zn3}TMB`|X$eHc>aHyWe)od7FKu0i85UYG~|(fbYf4Rm5>kN>zx11j5>b3@>7=x#2p zpJlU8?5)PZ+zz(yT0mmbNOql!=8d{9%3+TYcrR37j!J^d8bM)oO)&^6KH(|?R$Q&F zj&}s@F+iOl&B6lI=&O@MqWG-)+T9P2>(=IC=wGePQD1&JPXpw3Q;fh>+7i8x5j&(K zJmDI$ehn({qiao5&N?F%sGIX)__!DQU;4>;*3MEtdO63lO*6fN$_6!2^UE6>RDY?$ z?|nZNyL&>B0iWo>p_!QZngk4D)1YxY^Utv*eSzNY4t4}}eFb`qN3G_;;J|O*Apkt? z{NRcWOeS1e_w;SQa}iLwNr=)erAxFg4Zfws-lXH% z1dp8zKw>^QcvgljKNy9+W|d7oW(QJ0wE@d~IV4gEq$}w8{iHmwW?DT5{ZY9~n@!x{DdQoS zGqV19ZJFo`PKu2(o5_oC)7$8s5*_-dup(8m5Z!{5Gct3lff7raR4DP_2xMYrtFNMGkWKqBjZLO2!zsC*?gHr8DQSZ(Ci2 zK*z`Md6Kgj=2NJf>Mr05*y-0h(s9?jZ6}AwAHJY$93x1fp1qkUH9YURJ+J@hxoRJC zAWgSIbXwjp=Em?mGASnKa2!BxaBYcFv?Ub~3lAPo&uf}+jxFzTw08oHngsCeAQ>~W~4;a#ItltxFDpL%`RSb$>Gp^M_;JlE6ZCsKG7cuc3GNT zmKkVgOoP!M3d}klB6s7Nzzg&H8eFl5g(&jclZeYSo;1@T<&;Pq z`@2|Lay_X*)`nz>6Qus*EeRs}vr5<`Rg2nefa_17TY&Z!y0dJG>>*di<<_gi?Qdkl zJWd-=WqUUw8JBD4auru&1c!!e^)#}7Kmg?2CB8dE&7Gx((XQXkNs>h*;1nXH(W`gF z=0M{|f~o=^ArAYUlf&9pnTBu_~e>O7z@27mO!SXBSC3q!yxPQ%N76h`{ESC&;n*ZqBx?-h9zG z9pJtbDNk)x@DjQ+CZdBhW3b8K_oa_vHS z_ZH6J$_8BG@nvU6D7GFw?{$HdO^?1b2 zevxSo6)eil8;|64rT+~OA=Sae1^Po@kmS^a$5mB}3GZE^8GQCj{BY$6f|sjF zijUqFJ}L?>r#222_=v9>>~0!d{Z%*{vbR>%G}p;Us62@U?uS_c$fH;YjtP0f4o|K| zGU{F1oUMI);ZVU-xpo_+hYJ*$M`YT=4Ntt)JA`g@v=peP;{IUWAfqH_e?c|DG18cl4C9_N37oNrLvr; z3FxFT4xZ<`IHQ;5`bpg{Np@K{9P7}^*rOS0WS%Y~>^!l;Vj4Wcr&Qs?=oq!denAzp z6>z&J2L9uzR^~|+wIi%q$>*?vFa!02J0FR9K%LN%!&3NH*2JH3Cx{!Wl#p)20$@J> z#-0Adue|rckL*HOfiJo}6D+&?1cK5rj@<|GJ` zuKxAO{I!PKhyO0)(P%xH{I?%Mfw+KtxE3_^ zFN(-t<SA@I8M0%l!1$a{tpE zi6J$(PwyF!=KgOt@^cxzZxCndmZ@un8`Gu0>+%*#wVD|KR`S=8;c)jF5NntMC0+AD}%#K%f#@{|iUyU*727Ed%{6SU@s* z-v4Bi{-?JTganh4ncTm9`To;?4J&XK7Vz1=JtvU=$`Vw|9&X$Q{3pqh32vAl)*dGaqcF{D!N^| zLb|$L4vTK_)5CMK3whYkSOL`BF>-qRTl5FoP84S9B7gkf_)P=7 z=*MlY8BX^9=I8vm5brYtrs#zIWOS7OBi9F?%lc^FQEX#kE&9K6pMS0oxhi|I#4jEs zqVBT){q_6#xrs;cxd%BO(T-F6Pu+Cgb+4zJ{QH(T|HqyS7g4KijS-i-gf2=?<0{P$-3|MX_mqq55>TPUr839zjX5ZyJU{MEaL?+K|aEx|U&pR?v< z(u>ZMpB`6%jYA*Ib!0gWqzb!nAgY`C@pNnYpHGd=TPv=@Cob@a8C%lRA*jj$&rse2 zq_k&>=|9K)It*dfgl$CznGUnu&lrcI^+tH4OVFw3K5*n@YkT^t>yz_Ts{#kLJboXZ zYvNvc;+Tz(?c{jpTCWk=3Vx%}Q&Uqe@^&}!RT36=e}sJY=zhF_Vqrz2({Kd+5FJHy zS>-U0&L1WugcY^u?(6S(x`|jA8iILx)Z8n&Z$MzsyXsf7ze6Kg|wUd#UlD zN?-K{Mk1OMG(IWuOHZls5pD}L%woJ}IMe#dA93hn@E69-_=C}8RxXGtxWl#Xotk%Z zSG%WhX_%2Fq|FH0GgFV$FlhSA#oFkUA=7kDa# z{caMBJMdyAww$fn$Zp9EJwEr0NR*@$=3`M?Waf&CWTG}MJPY@wcNPK`ray2U%9eI_ zMt0fs@lwc+dV_Ug`Vur)c!!~DQUen1{s)M=lqVV?G7R!gL$GASogR`EiRoUpAjRt(Nco( zA#ao=zzN`o^56uXX;toc?QCQa1g1v^>1I_NtU-~tFh$I}yOVojLEHC)m-gR__4y8r8|oKND9LG& zH{iDe z?Jf`rY7Yag#Q(Uu=5_yO3u1iJ#j8@2h;eIqo9Sn+#*6--$*C;#{(z&PgQaP8AgV$a zv#-$9Btk2<7%!Es1dH)rhrctk#qPvc59^; zF721+nfZKOET9JyAnM0o+{0V;9U^iR7YF&lD`~?_rA2$vbFXeGXDqMFQ{A&(1+YF?1vShVl+c>h{1MKbC%Y zgb9C})_Y34!zletxcM4GCV329^%YLD)2E9!W9t;eK8kF8A&qibZU3mrMP74vG{8}> zW9C%^y`k1FsrOXPVzjq1?`2N=ybBYgt-X%>=`DXwMaM@4Pc?6u!P$0mu<8w}53mFusIcdlUhoQ>|VJr^=J}tSa z>wW8b87`XykinOv8*V?SXNq8aM(b+YzB(3rK0>OLDEIsw*aorzpuU@c!M6=fsZ@GC z=P7xF1PUgzLgWR~sYVS>T_O_oK&IjhY^iR_t*l;<$;1r@oTuufI$j)n_tNviu>J!R zr%k;WMX1qgSQuB{xSMAEEg0_c~+B zeCH(ZQ`K1R$F$oR7R0BDJ!dNl;O$&oaXt;3ptlBI@exy3 z2L+L@WAYtU<=%!Jn-n}4Szck7RbUtft5aG7tYm6A8%O^XJm(~r{#!|As)MGkgf=?J zH(r&qq4o#TdFk5M9(_-%(Yjn<)1@r|<-BMt>KqT@SaubEli!&382vG8fyBh)fOApl-X=IT?iUHI5v%XMn5Y z5BZu5s?*NjQ2D)0Fb&@5GfthFWF!&eG^qAjU&@OVX`Q}rZhgjrn@@SBcq4|0%v<=O zsNV)jt$85-RHV4p@#PM9GlwKvjd^@Td5F0EpHQ=9zPqvCftzUihYv#lYiu7JNduae zxt~r;wum>$O47{#2{Tr`ZkIVVT(ZrJSf-PEaq zIH&$}yP8ubt2%EC>Ij7n2CLsG@CN(MqPD?_ktKp!5}W)c-mJo}1eCI^OJ9+s$A92t znK3G*;@!;a7reBPO0Av9@fB4#r}&J&n8n-hm9HqBIDL5Lb)j2y)~cD>lSDt-uXEGH z@gNES3zry{t-@|ai1-RgW)Ip-PwR0HgBf29a;V|Njb8jn71>U=1F}gk?d?S@4ozku zScu>MY5$yf={`AMiln2f+LgZEFF%~qqwqd~kLk9lE4oay%7Z#oJMh&>#iR`VHF)Jb z`HT?39mA_%9#f_qM#RB|-=tJXT^OjnrzGA9i!}w>OLgq)MSo24y;ZxNqNJkX_)O#b zy_;VpiOmKC7xe8u9Zz>&%NxaJmo4mYw@|E zbbNT?@7>j2J>|$CMn3N@B2GP1=s~$ZNuGU)id-^SZ=P>LYmI1C44&_8x3csy-B`?o z`waqD$~|*=^prb{zi`dJ!Ncm9binw#-s7ly(e=HG_l@@pk2=ak?z1f~icV>=t(;Bo zHy3?V1G!&g*(LeP=fxAjGESq}2AWf+B{Hzbk!ix$E}O?+cnSGCbxD_ntI48I>R8|E z$@o*1v#3hLn~ld*WbdLzIwfA+I#+VGb(P+nTXbJ#4TuXTAQU;vNV*XVKIJTFaMl5b z#`yl2t@LyIK9U~KpYlCXU%9(&vliCP!&}L%aNQI$u(}m}C!fF<-tr26(aeqPS)mL= zonX#6h>8mXUinV95()4l%X+jevYbZVrt3J8Uc?(6-HvQW)^Fx>LD;&w>TDH%ZXml&u zK0feq=b|f|8C;cyh*V6g&A-LfMaPFf{`C8=W6X{Wx?U-_e7g>~rjJyAki1LD;VVk) ztvAPK^sgTsUN1?5fXWB6l&BOwHkA-%tklPC$i3OR6qDbl-97aY_1y+4IGT;#BiN+{ z$mmCwvb9NcKBj#%n(;*^@1C^i@;qTZTm$we7+xC|wdmBO%=g2n-+~q0-%r zNOuoNcSxrwA>D$6bPOro-6_%??lr#8^X~oUJC5(aG928q?sZ;gUC$mU933|B!!8ez zGs|j0zj3a|smZP$h1(fFrbP>=qxow#*`aZW&Fqryv6&hV>SdqR@+tm)h!mC|952VG zWeaPx2Y<%ewK^ij3+{w_O4cEJIa@BXH$&RMuKRMWAH%i5`uj5c2Metx0ThqEOUR6J+Cl z(s7B?HF$TCTad3E1+p`7(Lk|&c_o}w-fSN9`<7aOI&49Civo_b-8x(^s2UfmliY&L z#dg`k0I95R-9O+-XcftKve!!=dGAF>@YI5&=ik zDsUdjxHOJ5%0cQsEE5HP-!a1hZZSc&IJJVd6lB|S=gOkaM+*~PK$KN|hD$>9az$+yUx2BSW8DpVWhtXzo5nrd!`N5P`tY_@k6$pGzLu8_16eP})^h@A=FnSUkMb zica|oB-QRjbPUuZ0TtnfY&_(K=mn_)po@WXdI^k?Ppyadk}3qA!sF+i$JzWRI%y4V zS3D=R%v}byzyzwZ(N~3hVn}cSctUkWam>eI*nd)$mUAs1x7q zRPHnj;V-6kttodN6EW$3JVCQ6>T>uo?hXB$BdUggY-a#)YsdUhKJHS6)AU^I1^XTd z)Y_hzcmNu18{1E?2RumU?w{F}4Y86Dn{5}xl3&p&IzgswF3Cgk;u-a^Q*UY;?27u!5d_VWtfw0GTT5Pm$|51GK&Bo!oXy|c|Yt8`o ze6scO)h(jA6;ThGG$PRS7mfRh8X6UmjV5)&KW{H#O9%U{2X=hS9nm$j1=il{GgLDb zE(vU9tm#rrbcA#!*5bAq;%`5H#Zb-mL}Fh-jQ7I3L$*3j}FTHQ%P5y z!vN5j(U|}hMV+o9+g3k*aV>KH9ti|x?D?h$j>Y70CTm6OBVlbaA^?{jE}@<#USXfa zSd(_}7E+>q;<&%`jmj^Ie9Unr!dI99K9PdkpZO=^ce`g+3`l_WzZRJV9KgW>oC@Be z35oN>@Pp{PXZ@0>`Kt?hRg_ppZEk+EIEpbIuBaJo{L}^1#%)l9Z%tLF0iYK!y3LVk z6d90VL-j62Wc`R&fB^D!T**jjqZ)SGVyK+8XMlp%x01Ig4^bRwPIgzoDEnD0_nZfY z@|5Ch<`bV5@Y(xrH^-8Ca22Y%&keDEv@P{!U%7q5!H*ygYD=v@WkZHV?s;S2wbT@X z2c`4xLB;>S4=Sq6;xnqy2jvT(Mu?&ay{HgApUycmKc+tfX4@ZRc@Zl<@~*H9KVM5< z90_8LYw#kCFU-$u_{A{xism9(EY9+5@+PdJhtu+-bn56P2tc~LoK(0T*bI)?Lpmn zt;17&a^RHSO!i0yH=Iw%D}$5+!)^NfOG6Yeg?(H&xYX=MkLS3rdqm;}c9$Qd3?W)< zIY>$XN~YKeFY%RoB*OQ^ohdi}HLUM-^DI9fC&nf!7_PJwYRtH^a25ISF*qvu?!YA` zPDq^ycKOUe*8K^d5iYueA18EOQT>n{tW!4h4aC}!$-!peBVZFIK1C zj(03rbKQp^{xgw+61_~9iig+_91j?bsZg@Z8W5{Xr%kH60kJqbkdC!vQTV<3JQYvV zb;e8u<^k#_e{q~3q{};Wzh=&*CIAY1K~8XXO_!h2_J(i_Ni7L>gmEgkS2AY1_Nkcu z3(S#&0n4bPwwe0X`tN3U!5EIz9`l=K&}1*@?gh3aT-rcoOC6Eb^*uJQDYJ1vxV`Qe zWe&CfRglONuFc+FJjjtK~33id9cboms0s93G82-Nl z=7D@2UZNzm$Q)zbr1FgKY%|7cr)X5H^{c)!wr3sq+yEj zUMn2BmtT-MQr8UOQ*0bJGQrKMaObIAly2RsTfF(8su9qz`>yZs%bJa=SK)U(UDw{L z1k_XdNA%{yApfQJf1lPlK=VJ^JKoo|06Mv5u`psqB(ehR?OW?qU(8^hIz9qOFm?bi zo24y?_lP~oN}>X0teRxJk9Rv71A#ml9_YOMONU57X*d*zo`OdMnVq)>Wl=SA_0ee2 z%Ff?FsGF`Wt{N^I)!$hD%BL!jAeMSUak1Ip|Wkj4SDL zdjJkk+2Wl(V);cfbT=*(JUh1=bp<%md2v~TrQS0W0diblc9W7y_}equj9}t`z8^#S z$r#3l_h5lt?j^?><00$YmxQ=SZSS7qZL1-}e$_W=e@sHWiG?0{i%l}J*v*XW+%a_K zU3i3&sul)JgNwJciYLRS{;B*|Na`E`A`w#d=IpmuWw*)_;sv-W$K^`KsK0v16{_Su zkZN*3v+SjNphs84=An1?-0W)Th_7YA_o8CL>6@afL)pfP*XyquxWodExg#Vmhnu@NTsF5}h`rI8Nt15W|fB%;70s*cG6i7g2 z2af2iJ<8v}^d?>FN4YAas( zqy|ly;9Ki0t~^(59G$Or>IO(d{+bkFYl>t3{Y`w!y;-O9rorOYvpTn4ieF? zk^8)lGM8W{a>t-uJ*O(W!5-XzWM=oYI})?AfNhO zR#GIPk(d^-9`JpV#>dd(oWGlRi#On?K85u#zGcMkBQlOGFoqSN-(EwNAtCW-MpTNJ+JF?_Q9)tx~g z^L0WgtJ$UHJN!#^fpgVg$)E1fiG$Z`qs;4Kez0>V27)bAF6s&5Lqs~(m8#c0D3qNM zB}^D*vm5up+uG4(K{9 zG(MDP^=RCb#OrK8fMM3W;=2VPO*~JrBt7e>h;hwBm2XAc z{Ba5faYDKBY9VHvEZTh4Gt4dDx+gK(O*qKE&R(B{;?c%|_~uVW>(UGFv-B#n5>H)e zP~%?7;wvn5yf7%S`|jBAu@S*OCGm+=nJZ0#)61`t z5NS*N>H8U(b;+J9H6JgHlM7+WLW_Qv9$tGj!S7&n5U5elFiXJ&!0?;egwZzHG>r_`7dhhf{tJ-tV=%3!vSb-ERcVHC&E&LUr=uv~gky6xr)P zVQ#wl07#p`P80BscNLM^0(AC(EQWhDK!<2V5SM(g;`5g_1hu|-8(?_e&48}dunPOz zeuMi_YG|81Vzj=vH@XGK+RHy|6MW-MxJO>&*Ypd3_KhzgB{D4%9#(K6YP+==Z2iB2(DjID2RN% zZ9cst`eFF1X@ZiJ#RU_wl95vuYjcGXZdDlU83HpoML>`cRnQ&}(z8uj#k@kYwg5vk zQp7UrkI|KOa7azeaIXs{1vEa$BQF8GZ4d^nA)*K_LCAjG8wn~ut{B(S0zt6f#S$iA za1abTFKoQ^J|^@qg28WLpuU3+R5)vU{YDE2ikv&?P5QQu#$_V73+Gfp##oQj^$xN=9 ziAQRf70fkUW;p5f8+@b8wiHMZL={RdhA(D*Ut&<}m*uiQu2+xxM>)+*dLK^IvXZJa zhmO=XeD)F&*tyV{5DYD`OGMNKIGJUFx$nolx~VQ&7M1sQR!GL^uB&d(J92NNMdzc> zj~Ez>(h0x11w*kvV>nNy@Hb=)uN5BL+ZH>qF7yDNGjkri<9iY9@Fn1}qctC7%ej0< zLw?e1{~lGi31j7FZbo(3fy$-&hIrPTZ$e>_5ADuP$``s)P2xBAS>j((HWKUmYBVWM z{ijdur;S2t$Rl>KDN(S=r?1&V4*r1ER?3b5_dZ_I|=cZZp%;Hb)-}0f)uiTuZ~w;b{0WS4L)^LDxH9d_{|`I)~2AlMMhoAEHSBs zeMt`BK~uu)NkJr66NG9O(oC@wVffwZCVDYe#*eBQPOFl{HQB1RH;|jjIhuTy6Ki?}!{ z0*$btztGE`-ml<%l==F4aF!SC`4Z4c(hwaFrBM%X$LX3$+ug-4ro*~^gEpUZ(y}IUQ&##@ctF&IC^U$&*yAs5l{@xTD(mghF7F5LA8IA-6+|!B3lV~htkbqa63|BE z<&RI}{JbJJhD(Lv&SZm7bcYmGaHvu%a3CcE zwD_F9jojo1Grv*%1W*K%R`HEuHY#%4@$qS|Mx-4lY2d>` zYx~ioR%r(J$F9@^D4w@qwLA^s1jNAjltq=Eu0>^8V7@g^>@!Zvfs!p%g3B=}yo-1} zfT7ux#$bQ$v9vXxs)sOG5|sQQu?5LTmFo*UE?MuP7np(LftFGuq5t5c>ea&*zbNZV zl`A*Uwo&(f;>=5}dPvdH&+40ZBube_qBl-ebaS5S9C8_s!1|}EnRR$bY|k5p&k1|v zNH$ks3+B7SBQXvJXJWywIG$6-6rT`VhUSxi$dZ$<=}U+ff4M``#Sx#S2?uG2$?=Tj(FMbog@#Lbe&K9q39DQh-oWzIgFh-6;CG<3cQcG`}r;J_3F%Tvjv<&e#qRvRk89LpHl=L4i8VE z{0|kQjr4KlvG`JkAD{6Fu|Az$vKtpsf!SF3 z%CE-gQ-HFoG07+H{q_KpBkGqgXf4x9mEd;RS~~U44#$!LYyw*yzww0K20MIvJI$3^ zYk_i9(V?<%8Tg{&;R6`|hRQl6wAEZGvF=TC1Z$N{F{Rd7+Y@w`%sH1SJ)S-6n7Egv zt&cGlBG4}%DO`Ctw7GpY#7(WKM6t+TcsOO-b8)Fic)l=RDobvitqE2$3AsOcd0oB{m`|@eeqj6w#vh z?jF{d62IIuF?Y;(?w%}JuUC2eN}&JKwik(U^+!d^<9d@to~>fS;LlG=q3AWe5e~Y{ zz&wXRh=KS8%ON3xo9pnad}XqFxn53<>ZT|5n?hHkHlCrisA#>%Ez?P*ATL)RZP2y2orrpCI}| zzIm#M8ArSi@XZB13}JT7*PkR!`XO02Ne5ij@f{lF|LC#l%qe)^> zZNqlEsfwKaiWp-3xXaDKLd7z5xf9z;CIRE4y(dEa|F-*J&0#q}^kuf+e5NFIHy}EN zi`jR-e`>(9CVDtz*cgLFz`QiRN3&#WIR4G{&(IpJe6HF_oK>>Qt$2+e^9`2!ftQc+ zoXWpbs;d!}s()WJY3wqFVH`~fnivJIX*UU`FRW!tE#u(n5UnY6fM84#HW=WR3`yZo+#`J=1RQ zM8Tl?SCmy7<#_)PjaI@=r+MOicnJzBB?acGV-;CJWmeQ zQUK8%kBnh()jwBi`>OzsbBvj_5;mK7xT4<0u}3NuJehEW$Djj`x1F2f7}N+3`KD$S8WwKyyLGM@+gvD7O^3l+35Sm`u=iq46T|u z_T{SiBZ;&)g=oDrA^3e8B5jqpRQJOpJ1C2+@wmg@WJD4o*A?%)EUbg-@4rhz-Y${! z71gX<{$(@zTF5nnMr|W3XJJ27s)BEG2MpU|KU>EECG9jX>lFxHfJ<3aN-=LJE z%%Ph4uq`+lbYWM!^!eXM-!=ncB$p8_^4wuI7%EY~Vz*8L)hkBMQaodh#?sog3k#{( zev(`S2bZsgL*X@Pcj^wD7Y`yb%u=ThTH5U`@gSo3me)HVttRJQQF*cEcbI{2=os`iC@ zLyOSBA(tY$HZU@BP3s$00X19{(w9vD2+8h{`GE5g!%j=v- zUXwFiwf;qL{#D)9a-WM;_wlh+t?%CK*Al5kF z2E8fj)R^38^MX4hS_%5#2q32!>{OC zPVFRqndm8P@Ausb%YoBlje4TuczxVDgcpFq8?nUxsp zUg$|tj1~E4%nTM!COb!j+1&?RRG021k!`vYOB7W{oHU8-nB74tI6uo^c-R6l*J9={ zbj)mf5nO3=$mv61@k?3Km?BJ6*-W(l$=`6n*-}SnToREb7VA~eaOxZVr)kz!hrfc? zQT%*t>SC~?DbfKKYFO1pq>~FR?|bl!n!d*t6)FCYp8Tk)1hIH3DqM5dD{qfAhnk$h zES&ZNoCaZ)AKtr^yNqWO60DPQQYw5844pp+zE@Cyv>3h6HTTE0VCMgmrEp23B65j~ z9w32>{5(-wv#~9dG#D7Jqh$>XvXY%Zzdf5O^wfZn<;;0`XT%hS+7)~MhdP8@nQ_Qg zU+S|9l2#2lSXySx->XiFvQ}VgiiC8~VPar>$%apc37G-^ZAcdfqlikJmQVh^P|jqv*y z+?GCvhd9eplH}fd5<3Da%d1Jg*=No4anmmj;Ik;iXCA*@W|G)??$g0IpIFq2mT$RD zi9VcEH5+bY6lNkIN8r>Pz+i#@2UxkQ3k%?0*G~j%b$4hkztP^>o$+z|}Z$|XC|2eHT z?dl01+mT?)F_K)HW17cRhL|3&fK9C~6a{{3img*00cmf9U8t$6kx|$jSNK`{- z-Wl)(4Rc&Vg&lzdz3_Ve0Et)3H{~XQS#OaQrirOvBKW{Ko(Vmw*%Ez^un68rgI4%*mTzr~cRrSI8 zD!Zte(D)A_Wdc_5q*#XMZlTl_J~uXBS;ryX(ud{7eL(TrdB?OBb`OuFMXpNnwe$3J z@S(73)m;)pmi<%=hH|Wj-=sHhWI-8=e+hgVXW8>o`+Zbx{c&hDEZR(F0X4%a^*8OX ztKK$ZDNKz+I$Ka0W%EH_E!~Q!{L-)N6q{D*7Bnu=u1>}Xw*?Ulzf|oX!&a3ko?;ml zOx_CZCHQt{V?%jr z%vr;O9L(QuRoSGyp6!@f0FGAjR{I8c(mv^e!NB+4zo6ni^qKrICJDayjvrQplgvu zE)ViA07VEbN|3u}BDaC6M>S&qHRQ*MssTQ|@A$rD35V781LGjK5?1Vrk>sL=L~;~y zkLW1^7c(a{mpKRqlr6)jZEjxCT={^%yi_Ium+w&&eq_n`FX*Vcbv{f+4^bp(q9@DX z{7in}O8z=|9Ls2O=Yg^`+DLw7s!~*pRZpxWqv;p)E{$<2N;Vhk!xfft(P!+9^?MFS zUV!@RT$4TSf|#!UG@$5nIczoBrZ1?Ehz z@GqQozi43$nIE#llpTx)4#W`-JZasR7k%I$ezk5vqF_+ zA`o*Y|2=RkB=wO|ZHNJwu6mPdk<$SX;D`@?`><}B=JVVEll~R|b zdniIHb*TK2*nABc^T;tg=k<~WWirbtXgs9{ERa|gFcdKW&#=WYcv1G`xT1E||4F6T zF>ye9ibQ;BKJbotX0_c4(VLncn!gM7k+Ytyv~Fr`gQe8!-#PU|D5;9t7d7kS_#Fn4 zY?s5a$Pej1OQ+oA+7xE>2UF(mlQBf@+z&u&|>!d|U z8K6IPFKw*c1k_mgAI5N8e)v+k{-_n#5qgN~^QyA%Wd}ifu=K@zXgl z?PnUw_R`K-q~`G!W~#Fa&;n>GD7tO4!FU8}x%ES4F3E!L9E&#nG1#ZMfAcpNER-~` zTe^yD&Zl^WGOsHxn~c#aq8znMst}(4?QdIBlTGdAm#3a8;xfZpp8Rr(OMMldxgkLm z_1vJRXZfdHP=?nxY_s+v7u;4zRXH<;n-dm;!0<(f068C70yWjzqYTc}UF!VD=7~5i zBWs%iHnFEbZxp%yIWB>-_mvEI2;`Ju4m0zZ(qw|7M>n9NcBI(i4&Y6v+OZy<%b1BL zedp<)PE=SUOECW&i^&Wqc4}HxYI=%|&s1AHNO!N2AwO~rW<+X(J*Ce{OiQZ}JNFr0i2m(Pb)*t^Urq9#sP=~_M>+z!&xS;~L zM9A+ql}e6KH`;GigDBKxKhGfaD*_Hf{7qw>nhH9%O;Y5>i z0ZbNddoPGj?>;AO$H)W2g9`Qx3J~T4shdS9rruA9)Aj4FX?lW@%x`gH){;#u6V(Zm z+RSJR2uq`jA(sJf8>co~u_r~xbdS1$wehg9Lr#fgtJ&DGv^ky$8;%~b}@3{mnmjmYNoWDyvl{yzJ zyNtR8R_ccH@1QkRC43ayC@`lxs1+s0I}*#C516#+X{91RbjM;48|*pcr`rm8WRL_S zj+DsvI6dfVsxbwv2k^3d;#Zj?9?G33KkYIOjifnnW>IG~DBBbj<+EOqen{pkZvpX) zfiO>sBQi<>-QMa@IDIV*L6t(#YVr7zGv*2?M`VmCr8br0dZywLSo1JCVvv>!CPvwO zLpX}rYqzeE1geNpOhg@KKCQ>`uWLrSFn`(b1dY;8Z?aE?s6CX`>BY^wYwYmt4=>p- z-=S*J&~*Vur?>x*)5H<#d%q>#&-z_9+zwFVQSVd_4iap>aBV#D#RF#x+egYD_}M(l z{$T5qECGr!QgtB4u!L?cC~!D+nb8a)2>uAlN3qcn-J_b0QroV|EqQaD&T}y~t%#1(z9$(To8956WzG0fIV3l-P2_1h{+za5Mq}hUaDr&PkJYZ zgBM;tTHDu`EyIumqjou5s1+YTOyc3K&|#P4-2z9q(imsqn4bLGojyeoAq zBgOrASR}?vH7z#UvPv=PA(BUv*?9+toSp+Y(`$t>KvXdS7n(Gcw{v(tu$5O6*1J$0 z7l(cMfb+6#A@OUB1UW2c&{BeH?$wMBzNJriSS(A6-V#O-Nguuy8__!sM(8wP)s^ok zz44rqqwL=bEouSZxlMWXLGnJoOhF&_HR)6_s|lGb4y1BcAY5PFU^VNykgo4o^xS>U zirX#xVKAB6Ojl5Q5#-E(J|vuY?Wh#7dw|Ej@MIDw0Y(|p%dqDW#?jJ@;A-O>jV$Hi znVAf_-i*4wK-rxNzIoZ8_uQSJOUZ(mQ}$ye=7md4ZC;fb3H4l{2JdI6gcf1Ge#0W* z<{}$2z#atc#c%zS8wk&%I*=kwfdXK&fn2fe67)VAf$?6-OR%&qsd%g`wcu2V_9ZjCPY$Ei> z$jELM)xTO6{`zonMI6XK(C3{`I3eLt=?Np%H?Sz)$q!x{A^Bsr|C}90NJwx*#?_|h z1CL`@3a$IwP?KD+Cv~kK$oSKAp`R1?x2U+xkxl+xVT=QcQ22Lw;V^+E3g=-lrUAdE{$Bh?o#-qj zpWd1Ptu?8;-l#`}Fs{7dC7;K3)x*QC&1vXpj9HQh7g*BVSWvoUc14qV`CCJVhpq*@ z^d-jLi_M=%@<~2*Z~l9KbJGV;lmC#J5F3{Jw0X8RH$RxOI`N*K8q;1v>GwZ4xrW=O zBVKalbAM&dur22(I2tKtHV2U8vr~86G|`t_%CpIAfqu$(6qKgWF{z(4`6}YJqy1}D zSuc|^@ESe`_TZ2;Irx9+mvY|zy4>>^+9CY~B#7blOd$mk+9}*lwiKth%^Qks!Gfq_ z?Ud?4O~T!)3da4ijFk!*;$>$$Ar`V;HXuuNBm8?Z~5zeUOMblt{r1r z4hFkdLtpzdk9CC6$ho!ArqjutjYT$z{EZ&vyIgm}>>{}M2}mJ9uN!r)33jQDWtEk-vI z?>DKQ=3Q+4e^b_MIQr(b8b6thbc_ddfv)j1IX%*%!di2K_BDPWRU;9qw#1`1 zhe@i0O-!D?T5z%K-CQ1P^C;NdCfQ^#V%e{^yUWTl_C>Le8&E+jeqf*mHgEePVasfM zM}$|QK3}oKzrxw2$xM^yY>e}A*px)lm42{M9qM(HwzLJX&H@Ojd}L^i{k)_Tv5zG3&2kUrQBV1% z_eP=y>5_&WO_>F&=w8Y9@u`xjN{4v=QeYoox<$rE9a)HuIh-vw4!5%Ai(Q*n*L9FA zo|1^)qO#7zT_gD|&p!ejmKlBcSDQyXlZd&D-nyX)#qL;;%BVuuRDx6_Z`&e8Ub`Z3 z1R_tY00veNS%CnXqw6Un*?zfL!` z$PhdMj*9j69LnC3;}3wNsCVc;mi*PK(ITEkeN{S{HNY!BqC=zVXSrsF~`r>ACig zbUqx9h?w*~&gu>rZj8N^CHrV*ojmL^Mt~_Od&28XD0%9O2yqqR)MfDDNerSc$ct|= z?9;IX9%0X-PhJwY=rTF$^$pg$3`<-lNT`V@p=4znHJ71#IoU2OM6vN?jW^q)H>pQG zw?fUoiJ+u;Vc=H|^uS;kwT*5mqLfT*rXu$B7KYDUb74e?&$hi`c~&*@ja!F;?xG$MH*NHSHyyWVTRk{28 z4s!SrnB-NC$>1NS?`Fu$cVPi*P`xLeb{T_aIVo$g3B6OKnxhe=JON|+_*96z2@{P3 z4&v}US_NO*BuigOk?Eb@v`FEzC$>EEOJHnMF&kNX?>!K#;eO|RSKII4=lOZ}mP~pv z73#m%!O|!iA7=_alK%$=@t>8B8V+u@Od>7QSoK9t<)LHCU_A#d2mp*^NujnKUFT4 z%xCw&AQ>kITq%twGyi6#cNBfWpRjWEinIE=4W=3)U@SGEe5r?uKF7@TLc!W*PLlp9 zIgshzDBxTGJ2HW21res0ONxS=lSAV1F~Y4bV)SO;!;}B;ni$A*HQNg2so8wg+*)~&0QHZ46~!&`1v_`B-vSKtI=KL(r#72rug{N zkk6kvcJ}paWRuL^86`a%`&P90N&)ysYetvnW4&B5CLk}*O^@dV70-f|`Nir}IKR(} zH{gtc)L^{!mT?8zn_V}(7VM}?h$tR_Ij~>yceJD7tI@xMpo_vvPG0zI9HB@%*EPRh z_0TD0pkKEYesl5MS`iXoB~hUb@Nc6UM2d*sM7)<}NM^d)?}@`nV-jM3mRdUoO9Djw zVk_QmPXX;*d+vK}0@kBpeXsq*TaZ@Y-AmM;i@uHJHak0ci;+jy45$^7ZhZ208VRzC zB8pR>tE49_VZTn*!W_&##k}m42%G5J#HER^sQODmkf+8u7=aZ~1Y|ovtO}SA+vNry z>(*q^?L1$Y8*vCUhD4fNaj43YT6s6HmJB$|y_ht~ij%BFNM=JtRwL%XU7Q{Z**)s> zun++T71V%yv#C4<1@h`=4>q-YDm&pjNo0O{tMU*rXYdq5ztFQ+trUHm)UNf zYL+zb?oJK3zF!&-Ti1ginLZ~y?|Y&I-X%-RS|qMSq-B=-1ig?r)+v>ku}YT>qwVm< zy+kt$5)A1*V=gXPrNyu!<#Gom(zx(T%cZzX$~34BE8yO)trBmefj;&r*=BLbW)Mxb zHGQ@dU}y`^nq#lLs=C1lMqSqWXK;5#0UmDdR&C0B&t7*c^h*66nvC)Hf(^CX?nb*v zDG=Swn_>yu*o2>3g-to;4RX`PD*fqE?Di2 zccsQ1+m4b4X?vUmOB;{-N-w@ng#-wcT%F1>%do^4p2pltfj)g*b02`zYVYVzZwc^o zp!z>WaH0@M4YTQM7*3vD5I43g ~d|56M3+H4i`u{yQa+=Og9PfpsJ^Hs1eEYkM zs@oXe#q*+p{dWBS-ZjaGNb&+^jmfytv?I-Wz0SwTqg6IN2!y8JJlX26gg7&D%+tEB zk0U_;8_!{9899jzE}VP*FCriUoTOzrtd{lmSPCY5{@uvna^WW+^9}<0CmL0~9jr}j zF?^f-BeY{mRER5(1|dLUvtiPgnd?|jl}31rnVb8=9w0_vX3FCQ;Io_4YF~TQXw&*Y zqK|F`EOC6he{O58eAxbjN3-uS)a0qW^rxxo=aN7*LhwC-%D?s+Km2(56 z^nk1#fI^dX9_ebb=myqxz1f;;5^B@t6E!_Num^rLIpHN7UM7F*iKyHZN*D$K>@uFNZ-jO2)*3XG#fi+DRDnRO`K$JupKE~-B#Wf-MlM{+qO!bponUc3tS{y%XU@ekI{=F_tmUY#rFF`@? z54dbp(4emiroZ~I>ksZKhv062vLZG?PURy~F5|#jnp+SE;M)v{*#ZqM+O{I}cD;za z&3h=Lna@f3Qtok=FgaSdUKofTeXcZUWqCobYMm4&#y9k={R^<|(n?j%dbGFd^<@Wp zKrI-)>J_XNr!9gau&+&5Eni;ZiF<1_H3O{j_ab`*ox!%?pr04|r+#=`Md5S!f$^yU z%6zU5A2t8byQa?CAW-U{gHE=LkrN68;#s0nmyd%2TQA_h<_M0!JVd%?Ruho&?+E$p zEN9tF{Q=8wmov~!!w~9IS27T(x&^QdUe4?yc@&|Cb1l6QmL(gmFd9j;uZpx7g9ATT za{B9S5yG_c2g^YDQ>d|kJ?o2OGio&_o8#nrZ=;Am+zWi$RuT8q2(J+pk=y7JCq`bzf zN*k()Xn=Ug1Yc286T$AUqf-Dl3P zM_Y>9MLw&zx$QpV$y#r=ubRl`0D6+nzazs^aYh{xoW!AM`x$mxs_9g&pKR*`vN9=_ zMTN@Ttfn)d(rHF}9%M=9J(a%}2cTK4{z3sVQf|}%f8H1-U~fBP^IN9y}9m(AwF$D2I$-&*^F-J*!_Q> z8(25*j^69Pk5D2c3+H@H`n%0Zibs{@ zf+O@)3>0d25<`(Ig;`MJ&WB?qJ0fXYcXnT~5j@S&_+BW&ZT}r^iA$`{@)J zc=5M1Ai+QmT}f4Y5}y)$!5B$p+pUWh+hO6lEN^M7!6~UqG=I z^%PRAwj1oQ%eMjeF9q1PC+eD<+0bz8+^haV2IqFI%??{`viBimK$tav*bkE^dPH@&mSu&f6G%W@I0%<#7a5f+<)%p7n$4BD007*4wPTvNtbXa4Ms0)Ws^ z@z#VRy`<`Dm`dzBO)r7M5-a;07Bon{+cO@s0w5*;fLQqW_nx0%>^EuLHv@itQIY*s zag&L6*BmQBJL8ay$;c?SGE=8G6QKpZbN4{cD{Um&(uVB;z__NRwXC34p>iluDT@G^ zb*K$hs+|T3=bP^i$pfCSd_790@4aDDB|@9e;11maWZ|??sHTid^3U-N9LsHZXpWB) zLku6_m`U=x#eZYcD8^~hv(Xvphn^3?1^3yqG6aS$851+s3O9}