MFT_Transformer_HSI+LiDAR_Channel_Tokenization.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "6be4a0ee",
   "metadata": {},
   "source": [
    "## Transformer Model for Hyperspectral Image Classification\n",
    "\n",
    "\n",
    "### The MFT model with both HSI and LiDAR images used for classification. The other multimodal data is from the LiDAR and is used to generate the external CLS through 'channel' tokenization. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "91f97aaf",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-08-11T11:17:16.979544Z",
     "start_time": "2022-08-11T11:17:16.971125Z"
    }
   },
   "outputs": [],
   "source": [
    "import sys\n",
    "sys.path.append(\"./../\")\n",
    "import matplotlib.pyplot as plt\n",
    "from torchsummary import summary\n",
    "from sklearn.metrics import confusion_matrix, accuracy_score, classification_report, cohen_kappa_score\n",
    "import math\n",
    "from PIL import Image\n",
    "import time\n",
    "from scipy.io import loadmat as loadmat\n",
    "from scipy import io\n",
    "import random\n",
    "import numpy as np\n",
    "import os\n",
    "import torch \n",
    "import torch.utils.data as dataf\n",
    "import torch.nn as nn\n",
    "from operator import truediv\n",
    "import record\n",
    "import pandas as pd\n",
    "import seaborn as sns\n",
    "from mft_model import MFT, Transformer\n",
    "from dataset import Multimodal_Dataset_Train, Multimodal_Dataset_Test\n",
    "import torch.backends.cudnn as cudnn\n",
    "cudnn.deterministic = True\n",
    "cudnn.benchmark = False\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "896fad69",
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_confusion_matrix(y_test,y_pred, plt_name):\n",
    "    df_cm = pd.DataFrame(confusion_matrix(y_test, y_pred), range(6),range(6))\n",
    "    df_cm.columns = ['Buildings','Woods', 'Roads', 'Apples', 'ground', 'Vineyard']\n",
    "    df_cm = df_cm.rename({0:'Buildings',1:'Woods', 2:'Roads', 3:'Apples', 4:'ground', 5:'Vineyard'})\n",
    "    df_cm.index.name = 'Actual'\n",
    "    df_cm.columns.name = 'Predicted'\n",
    "    sns.set(font_scale=0.9)#for label size\n",
    "    sns.heatmap(df_cm, cmap=\"Blues\",annot=True,annot_kws={\"size\": 16}, fmt='g')\n",
    "    plt.savefig(''+str(plt_name)+'.eps', format='eps')\n",
    "\n",
    "def AA_andEachClassAccuracy(confusion_matrix):\n",
    "    counter = confusion_matrix.shape[0]\n",
    "    list_diag = np.diag(confusion_matrix)\n",
    "    list_raw_sum = np.sum(confusion_matrix, axis=1)\n",
    "    each_acc = np.nan_to_num(truediv(list_diag, list_raw_sum))\n",
    "    average_acc = np.mean(each_acc)\n",
    "    return each_acc, average_acc\n",
    "\n",
    "def reports (xtest,xtest2,ytest,name,model, HSIOnly, iternum):\n",
    "    pred_y = np.empty((len(ytest)), dtype=np.float32)\n",
    "    number = len(ytest) // testSizeNumber\n",
    "    for i in range(number):\n",
    "        temp = xtest[i * testSizeNumber:(i + 1) * testSizeNumber, :, :]\n",
    "        temp = temp.cuda()\n",
    "        temp1 = xtest2[i * testSizeNumber:(i + 1) * testSizeNumber, :, :]\n",
    "        temp1 = temp1.cuda()\n",
    "        if HSIOnly:\n",
    "            temp2 = model(temp)\n",
    "        else:\n",
    "            temp2 = model(temp,temp1)\n",
    "        temp3 = torch.max(temp2, 1)[1].squeeze()\n",
    "        pred_y[i * testSizeNumber:(i + 1) * testSizeNumber] = temp3.cpu()\n",
    "        del temp, temp2, temp3,temp1\n",
    "\n",
    "    if (i + 1) * testSizeNumber < len(ytest):\n",
    "        temp = xtest[(i + 1) * testSizeNumber:len(ytest), :, :]\n",
    "        temp = temp.cuda()\n",
    "        temp1 = xtest2[(i + 1) * testSizeNumber:len(ytest), :, :]\n",
    "        temp1 = temp1.cuda()\n",
    "        if HSIOnly:\n",
    "            temp2 = model(temp)\n",
    "        else:\n",
    "            temp2 = model(temp,temp1)\n",
    "        temp3 = torch.max(temp2, 1)[1].squeeze()\n",
    "        pred_y[(i + 1) * testSizeNumber:len(ytest)] = temp3.cpu()\n",
    "        del temp, temp2, temp3,temp1\n",
    "\n",
    "    pred_y = torch.from_numpy(pred_y).long()\n",
    "    \n",
    "    oa = accuracy_score(ytest, pred_y)\n",
    "    confusion = confusion_matrix(ytest, pred_y)\n",
    "    each_acc, aa = AA_andEachClassAccuracy(confusion)\n",
    "    kappa = cohen_kappa_score(ytest, pred_y)\n",
    "    get_confusion_matrix(ytest, pred_y, 'test_'+str(iternum)+'')\n",
    "    \n",
    "    return confusion, oa*100, each_acc*100, aa*100, kappa*100\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "16b7982c",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-08-11T11:18:10.978783Z",
     "start_time": "2022-08-11T11:18:01.894932Z"
    },
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---------------------------------- Dataset details for  Trento  ---------------------------------------------\n",
      "\n",
      "\n",
      "HSI Train data shape =  torch.Size([819, 63, 11, 11])\n",
      "LIDAR Train data shape =  1\n",
      "Train label shape =  torch.Size([819])\n",
      "HSI Test data shape =  torch.Size([29395, 63, 11, 11])\n",
      "LIDAR Test data shape =  1\n",
      "Test label shape =  torch.Size([29395])\n",
      "Number of Classes =  6\n",
      "\n",
      "\n",
      "---------------------------------- Model Summary ---------------------------------------------\n",
      "\n",
      "\n",
      "===============================================================================================\n",
      "Layer (type:depth-idx)                        Output Shape              Param #\n",
      "===============================================================================================\n",
      "├─Sequential: 1-1                             [-1, 8, 55, 11, 11]       --\n",
      "|    └─Conv3d: 2-1                            [-1, 8, 55, 11, 11]       656\n",
      "|    └─BatchNorm3d: 2-2                       [-1, 8, 55, 11, 11]       16\n",
      "|    └─ReLU: 2-3                              [-1, 8, 55, 11, 11]       --\n",
      "├─Sequential: 1-2                             [-1, 64, 11, 11]          --\n",
      "|    └─HetConv: 2-4                           [-1, 64, 11, 11]          --\n",
      "|    |    └─Conv2d: 3-1                       [-1, 64, 11, 11]          31,744\n",
      "|    |    └─Conv2d: 3-2                       [-1, 64, 11, 11]          28,224\n",
      "|    └─BatchNorm2d: 2-5                       [-1, 64, 11, 11]          128\n",
      "|    └─ReLU: 2-6                              [-1, 64, 11, 11]          --\n",
      "├─Sequential: 1-3                             [-1, 64, 11, 11]          --\n",
      "|    └─Conv2d: 2-7                            [-1, 64, 11, 11]          640\n",
      "|    └─BatchNorm2d: 2-8                       [-1, 64, 11, 11]          128\n",
      "|    └─GELU: 2-9                              [-1, 64, 11, 11]          --\n",
      "├─Dropout: 1-4                                [-1, 5, 64]               --\n",
      "├─TransformerEncoder: 1-5                     [-1, 64]                  --\n",
      "|    └─ModuleList: 2                          []                        --\n",
      "|    |    └─Block: 3-3                        [-1, 5, 64]               100,736\n",
      "|    |    └─Block: 3-4                        [-1, 5, 64]               100,736\n",
      "|    └─LayerNorm: 2-10                        [-1, 5, 64]               128\n",
      "├─Linear: 1-6                                 [-1, 6]                   390\n",
      "===============================================================================================\n",
      "Total params: 263,526\n",
      "Trainable params: 263,526\n",
      "Non-trainable params: 0\n",
      "Total mult-adds (M): 12.34\n",
      "===============================================================================================\n",
      "Input size (MB): 0.03\n",
      "Forward/backward pass size (MB): 1.12\n",
      "Params size (MB): 1.01\n",
      "Estimated Total Size (MB): 2.15\n",
      "===============================================================================================\n",
      "\n",
      "\n",
      "---------------------------------- Training started for  Trento  ---------------------------------------------\n",
      "\n",
      "\n",
      "Epoch:  0 | train loss: 1.9849 | test accuracy: 1.2723\n",
      "Epoch:  1 | train loss: 1.0730 | test accuracy: 33.4513\n",
      "Epoch:  2 | train loss: 0.5160 | test accuracy: 44.8376\n",
      "Epoch:  3 | train loss: 0.2624 | test accuracy: 49.1818\n",
      "Epoch:  4 | train loss: 0.1144 | test accuracy: 64.8920\n",
      "Epoch:  5 | train loss: 0.1248 | test accuracy: 63.9871\n",
      "Epoch:  6 | train loss: 0.0985 | test accuracy: 92.0837\n",
      "Epoch:  7 | train loss: 0.0711 | test accuracy: 96.3089\n",
      "Epoch:  8 | train loss: 0.0253 | test accuracy: 91.8081\n",
      "Epoch:  9 | train loss: 0.0309 | test accuracy: 97.4690\n",
      "Epoch:  10 | train loss: 0.0133 | test accuracy: 75.7170\n",
      "Epoch:  11 | train loss: 0.0158 | test accuracy: 97.4520\n",
      "Epoch:  12 | train loss: 0.0527 | test accuracy: 90.5664\n",
      "Epoch:  13 | train loss: 0.0106 | test accuracy: 88.2327\n",
      "Epoch:  14 | train loss: 0.0126 | test accuracy: 97.5574\n",
      "Epoch:  15 | train loss: 0.0191 | test accuracy: 95.7170\n",
      "Epoch:  16 | train loss: 0.0115 | test accuracy: 81.9085\n",
      "Epoch:  17 | train loss: 0.0277 | test accuracy: 97.5540\n",
      "Epoch:  18 | train loss: 0.0113 | test accuracy: 97.5642\n",
      "Epoch:  19 | train loss: 0.0104 | test accuracy: 97.0607\n",
      "Epoch:  20 | train loss: 0.0102 | test accuracy: 97.3703\n",
      "Epoch:  21 | train loss: 0.0074 | test accuracy: 97.6833\n",
      "Epoch:  22 | train loss: 0.0101 | test accuracy: 97.5268\n",
      "Epoch:  23 | train loss: 0.0089 | test accuracy: 97.2955\n",
      "Epoch:  24 | train loss: 0.0077 | test accuracy: 97.5506\n",
      "Epoch:  25 | train loss: 0.0064 | test accuracy: 97.7139\n",
      "Epoch:  26 | train loss: 0.0051 | test accuracy: 97.3465\n",
      "Epoch:  27 | train loss: 0.0207 | test accuracy: 97.6935\n",
      "Epoch:  28 | train loss: 0.0121 | test accuracy: 97.5574\n",
      "Epoch:  29 | train loss: 0.0110 | test accuracy: 97.6220\n",
      "Epoch:  30 | train loss: 0.0064 | test accuracy: 97.5234\n",
      "Epoch:  31 | train loss: 0.0074 | test accuracy: 97.7445\n",
      "Epoch:  32 | train loss: 0.0065 | test accuracy: 96.2170\n",
      "Epoch:  33 | train loss: 0.0079 | test accuracy: 97.6697\n",
      "Epoch:  34 | train loss: 0.0058 | test accuracy: 97.6935\n",
      "Epoch:  35 | train loss: 0.0060 | test accuracy: 96.1728\n",
      "Epoch:  36 | train loss: 0.0049 | test accuracy: 97.7275\n",
      "Epoch:  37 | train loss: 0.0081 | test accuracy: 95.7340\n",
      "Epoch:  38 | train loss: 0.0072 | test accuracy: 97.7683\n",
      "Epoch:  39 | train loss: 0.0053 | test accuracy: 97.5846\n",
      "Epoch:  40 | train loss: 0.0048 | test accuracy: 96.7682\n",
      "Epoch:  41 | train loss: 0.0065 | test accuracy: 97.9282\n",
      "Epoch:  42 | train loss: 0.0074 | test accuracy: 97.8704\n",
      "Epoch:  43 | train loss: 0.0057 | test accuracy: 97.5472\n",
      "Epoch:  44 | train loss: 0.0066 | test accuracy: 97.6731\n",
      "Epoch:  45 | train loss: 0.0066 | test accuracy: 91.7367\n",
      "Epoch:  46 | train loss: 0.0065 | test accuracy: 97.7343\n",
      "Epoch:  47 | train loss: 0.0113 | test accuracy: 97.1015\n",
      "Epoch:  48 | train loss: 0.0096 | test accuracy: 97.2853\n",
      "Epoch:  49 | train loss: 0.0084 | test accuracy: 97.7105\n",
      "Epoch:  50 | train loss: 0.0114 | test accuracy: 72.1653\n",
      "Epoch:  51 | train loss: 0.0114 | test accuracy: 94.2575\n",
      "Epoch:  52 | train loss: 0.0131 | test accuracy: 96.1796\n",
      "Epoch:  53 | train loss: 0.0070 | test accuracy: 97.5030\n",
      "Epoch:  54 | train loss: 0.0081 | test accuracy: 97.6220\n",
      "Epoch:  55 | train loss: 0.0117 | test accuracy: 98.0507\n",
      "Epoch:  56 | train loss: 0.0071 | test accuracy: 97.9350\n",
      "Epoch:  57 | train loss: 0.0108 | test accuracy: 97.7887\n",
      "Epoch:  58 | train loss: 0.0068 | test accuracy: 98.0031\n",
      "Epoch:  59 | train loss: 0.0113 | test accuracy: 97.6799\n",
      "Epoch:  60 | train loss: 0.0084 | test accuracy: 97.8500\n",
      "Epoch:  61 | train loss: 0.0090 | test accuracy: 97.8874\n",
      "Epoch:  62 | train loss: 0.0070 | test accuracy: 96.9689\n",
      "Epoch:  63 | train loss: 0.0084 | test accuracy: 97.6357\n",
      "Epoch:  64 | train loss: 0.0073 | test accuracy: 97.9452\n",
      "Epoch:  65 | train loss: 0.0079 | test accuracy: 96.3599\n",
      "Epoch:  66 | train loss: 0.0081 | test accuracy: 97.5132\n",
      "Epoch:  67 | train loss: 0.0095 | test accuracy: 97.9554\n",
      "Epoch:  68 | train loss: 0.0122 | test accuracy: 96.0027\n",
      "Epoch:  69 | train loss: 0.0182 | test accuracy: 97.3771\n",
      "Epoch:  70 | train loss: 0.0200 | test accuracy: 68.4164\n",
      "Epoch:  71 | train loss: 0.0167 | test accuracy: 97.4962\n",
      "Epoch:  72 | train loss: 0.0130 | test accuracy: 97.3329\n",
      "Epoch:  73 | train loss: 0.0093 | test accuracy: 97.5778\n",
      "Epoch:  74 | train loss: 0.0064 | test accuracy: 97.3125\n",
      "Epoch:  75 | train loss: 0.0062 | test accuracy: 94.9855\n",
      "Epoch:  76 | train loss: 0.0154 | test accuracy: 97.4758\n",
      "Epoch:  77 | train loss: 0.0093 | test accuracy: 96.8532\n",
      "Epoch:  78 | train loss: 0.0070 | test accuracy: 96.7954\n",
      "Epoch:  79 | train loss: 0.0096 | test accuracy: 97.0233\n",
      "Epoch:  80 | train loss: 0.0105 | test accuracy: 97.4996\n",
      "Epoch:  81 | train loss: 0.0083 | test accuracy: 96.6525\n",
      "Epoch:  82 | train loss: 0.0100 | test accuracy: 96.2102\n",
      "Epoch:  83 | train loss: 0.0096 | test accuracy: 95.8564\n",
      "Epoch:  84 | train loss: 0.0080 | test accuracy: 95.7170\n",
      "Epoch:  85 | train loss: 0.0085 | test accuracy: 96.7409\n",
      "Epoch:  86 | train loss: 0.0119 | test accuracy: 96.1456\n",
      "Epoch:  87 | train loss: 0.0103 | test accuracy: 95.1829\n",
      "Epoch:  88 | train loss: 0.0087 | test accuracy: 95.5809\n",
      "Epoch:  89 | train loss: 0.0078 | test accuracy: 96.3157\n",
      "Epoch:  90 | train loss: 0.0081 | test accuracy: 94.3766\n",
      "Epoch:  91 | train loss: 0.0130 | test accuracy: 96.6559\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch:  92 | train loss: 0.0178 | test accuracy: 97.1798\n",
      "Epoch:  93 | train loss: 0.0085 | test accuracy: 96.9995\n",
      "Epoch:  94 | train loss: 0.0098 | test accuracy: 96.8702\n",
      "Epoch:  95 | train loss: 0.0092 | test accuracy: 94.3528\n",
      "Epoch:  96 | train loss: 0.0107 | test accuracy: 97.2240\n",
      "Epoch:  97 | train loss: 0.0080 | test accuracy: 96.6253\n",
      "Epoch:  98 | train loss: 0.0098 | test accuracy: 95.9517\n",
      "Epoch:  99 | train loss: 0.0086 | test accuracy: 96.2919\n",
      "\n",
      "The train time (in seconds) is: 1246.8682582378387\n",
      "\n",
      "\n",
      "Overall Accuracy =  98.05068889266883\n",
      "\n",
      "\n",
      "----------Trento Training Finished -----------\n",
      "\n",
      "The Confusion Matrix on test data\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAaMAAAEJCAYAAAA5Ekh8AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/NK7nSAAAACXBIWXMAAAsTAAALEwEAmpwYAABSCElEQVR4nO3dd3gU1dfA8W8qECBNOqEGOBTpHUWKiooCNsAK2LAjvhaUn4pdsQKCDUVQqQKCIlWa0rv0q6FJSOiQUBJIe/+YSUjZwCZks7vmfHj2YffulDOT2T1779y545OWloZSSinlTr7uDkAppZTSZKSUUsrtNBkppZRyO01GSiml3E6TkVJKKbfTZKSUUsrt/N0dgLcZt26/1/WF792kirtDyJNUL7zcwNfHx90hKA9U3J/LPjBKNH3K6Q9EwsaRXnsgajJSSilP5lM0GrA0GSmllCfz9XN3BIVCk5FSSnkyFzQBi8hooAcQY4xpYpddAUwBqgIG6G2MOSMiPsAooAtwCrjHGLPDnuch4CUgDRhsjJlql7cEvgOKAz8ZY16+VExFo/6nlFLeysfX+YfzvgduzFb2MvCrMaY2sBEYYJffDFQyxtQCngeGQUbyegloDlwNDBWREvY8nwN9gDpAOxFpe6mANBkppZQn8/Fx/uEkY8yfwPFsxd2BH+zn3wO3Zi83xiwE6otISeAGYK4xJt4YcxhYBXQUkUpAcWPMBmNMKvBjpmXlSpvplFLKk+WhxiMioUCog7dOGmNOXmL2K4wxx+zn0UBl+3kl4ECm6WKAig7K0+c54qC8w6Vi15qRUkp5srzVjAYCexw8BjqxJrdeU6E1I6WU8mR56003DBjroPykE/MeF5H02lEEF2o3MVyoJYFVI4q1yzOfC4oAljiYPvOycqXJSCmlPFkemunspriT+VzTLOB+rITWB5iZqfxBYJqIXAvssHvZzQfeFJFgrF5zbYGHjTEJInJeRJoBm4D7sDpHXJQmI6WU8mSu6dr9A3AdcIWIRGP1insPmCIiTwL/AL3tyWcBXUVkF1bX7nsBjDFHRWQosAGrie8lY0yCPc8TWJ0eigPTjDErLhWTj97pNW8yDwe0e/NaVv46maMH9pF45jRBpUOoXKc+7W/vS9mIahnz7DdbWTb9Bw79u4vk8+cIq1CZFtf3oHHHm7IsO/n8eZZO/Y6tyxdy7sxpyleLpNNdj1C1XqOMaY7FRrN+wUz2bd/EycOxBJYIolJN4Zo7+1G+WqTDmAtyOKCDsbF8OPQ9Vq1cTlpaGq3btuPFQYOpWKlSga3DmeGADh08yHdjRrN921b+MYbExER+m/c7lSpHZJnu3LlzfP7ZcGbP+pVTp+KpU7cuzzz7PM1btMyY5syZ07zx2ivs3L6dI0eP4O/vT7Vq1bn73vu5uVt3p2IuyOGACmMfFzSN2bECGQ7omtedHw7oj9e9djggTUZ5lDkZbVuxiIN7/6FSZD2CgkOIP3qYlb9OIv7YER55fzQhZctz+N/djH3tKSrVqkerG28noFhxdqz5g02LfuOGBwbQ/LoLX3YzR71L1KbVdL67P6HlKrJ+wS/s/msNfV8fQfnqtQBYN38GGxf9RqP2XShfvTbnzp5m1azJHNq3i/uHDKNijTo5Yi6oZJSQkECv23sQEBjIUwMG4uMDI0cMJzExgZ+m/0JQUFCBrMeZZLRuzWoGPf9/1GvQgNSUFFauWO4wGQ0e9Dx//rGUZ597gcoRVZgycQLLl/3BuPGTkLr1ADh58gRD332bVq3bUqlyZc6fP8/8ubOZ9ctMnnvxJe7r0++S8RRUMiqsfVyQNObcFUgy6vCm88lo6Wtem4xc1kwnIinAFvtlCvCkMWbVJeaZjVU1vAKYkX5lcLZpNmH1WT8PfGqM6Z19msLSoF1nGrTrnKWsUmRdvnrhAXau+YPWN/dk+8rFpKam0uv5twksbl0PVqNhc478u5utfy7ISEaH9u1i24pF3Nz/eRp3sK5Fq1avMV8Peog/po2j53NvAVC/bSeaX98Dn0xfftUaNOHzZ+5j7dzpdH/8JZdt7/SpU4iO3s/MWXOpWs2q+dWuI3TvegNTp0ymT78HXLbu7Jq1aMnCP5bbcf3EyhXLc0xjdu5kzm+zeP2td+hx2x0ANG/RkjtvvYXPR45g+MgvAAgNDeO9Dz7OMm/7azqwb99eZv48zalkVFA8aR87S2N2Mb+iMRyQK7t2nzLGNLETymDgnUvNYIzpaow55czCjTEx7kxEuSlROhgAH/sASklOxs/fH//AwCzTFQsqSeZa6T8bVuLr50/9Nh0zynz9/KjfphO7N68jOek8AEGlQ7IkIoDiQaUIrxjBqRPHcKUlixfRqFHjjA8vQEREFZo0bcaSxQtduu7sfH0vfeguXbIIf/8AutzYNaPM39+fG27sysrlyzh//vxF5w8NCcXPr3BPq3rSPnaWxuxiLrjo1RMV1nVGwUAcgIh0FJEZ6W+IyFgRudV+vte+aItM7weJyE8isl1EJgEBdnl1u5aEiPQTkakiMl9EokTk1UzzDxERIyJLReRnEelnl79vL3OziHx0ORuXmppCSnISxw9GM+fbTykZGk6Dtp0AaHhNFwDmjxvFqRNHSTxzmo2LfmPvto20vOmOjGUcid5LaLkKBBQrnmXZZSKqkZKcxIlDMbmuP+F0PEei91KmUtXL2YxL2hUVRWTtnM2AkZG12L0ryqXrzo9dUVFUjqhMiRIlspRH1qpNUlIS+//dl6U8LS2N5ORkTp48wbSfJrNyxXLuvb9vYYbsdfsYNGaXc81wQB7HlT/7StvJojhQAbg2n8t5AjhqjKkvIlcBy3KZrjHQDEgF/hGRz4DaWENZNAKCsJoNZ9pjKt0G1DXGpGVPgHk19rWnObjnbwDCylfm3sEfUjIkDIByVWpw7/8+Ytqnr7Ph918A8PXz58YHB2YkLIDE06coHlQqx7JLlCoNQMLp3CuM88eNJC0tjZY33n45m3FJcXFxBAcH5ygPCQkhPj7epevOj/i4kwQHh+QoDw6xyuLi4rKUT544nqHvvg2Av38AL7w0mG49bnV5nJl52z4GjdnlvLzG4yxXJqNTmUaDbQt8LyJX5mM5VwOfABhjlovInlym+z29ic/ughgBXIV17ukccE5E5tnTxgGJwLciMgur62K+dX98EOcSznLycCyrf/uJie8N4v4hwwgtW4HjB6OZPvwNykRU48YHB+IfGMg/61cwd8ww/AMCufKq/OZoy4qZE6xzTY88R3iFypeeQeWqy41dadioMSdPnmTp4kUMffdtfH19ubPXXe4OTRVlXl7jcVahNIgbY1aKSBmgLJBM1ubBYk4swpneJOcyPU/hIttmjEkWkVZYtbXbgaeAzrlNfyllKlvtzpVr1SOycStGDbyXlb9M4qaHBrJk8hh8/fzp9fw7+PlbIdW4shkJp+NZ8P0oGrTthI+vL8VLliLu2KEcy06vEaXXkDLb8PuvLJkyhg49H8jRTdwVgkOCHf5qzO1XprsFB4cQG5uzeTPerhGFhGStNYWHhxMeHg7AVVe3JzExgU8/+oAet91BQECA6wPG+/YxaMwuV0RqRoWSckWkLuAHHAP2AfVEJEBEwrn0AHrLsC++smtYNfKw6uVAdxEJFJEQrPtxICKlgFBjzGzgWaBhXrbnYoqXLEVY+cqcOGSNfnF4/x7KVY3MSETpKkbWJeF0PGfiTwJQJqI6Jw8fJOlcYpbpjh7Yh59/AGHls177sOXPBcwdO4LWXXty1a33FlT4FxUZWYtdUf/kKN+9exc1I2sVSgx5UbNWLQ5EHyAhISFL+e5dUQQEBFClarVc5rTUb3AlZ8+e5fgx13YMyczb9jFozC7n6+f8w4u5MhmVFpFN9nmjyUBfY0yKMWY/8DOwDWto8Q2XWM7nQAUR2Y51f43tzgZgjFkLzAG2Yg1tsRmIB0oDv4rIZqyE9XxeNuxiTsed4FjMv4TayaNUaBiH90WRkpyUZbqYqB34BwRm1HhqN2tDakoyO1b/kTFNakoKO1YtpUbD5vgHXOiNZ9YuY9bXH9Kk401ce++jBRX6JXXs1Jktm/8iev/+jLIDB6LZtHEDHTrlu2LpMh06diI5OYkF8+dmlCUnJzN/7hzatruKwGw9HLNbv24tQUFBhF8R7upQM3jbPgaN2eWKSAeG//xFryJSyhhz2h4/aRVwizFmd36Xl/mi16mfDqFC9dqUq1KDwBIlOX4wmjVzpnEm7jj93hzFFRUj2LH6D34e8SY1Gjan+XXd8Q8sxj8bVrJ+wUxa3XQH1933eMayf/7sbfZsXkfne/oTUrYCG37/lahNq+g7ZAQVatQG4N8dm5k4dBBlK1enS9+nsnTz9gsIoEL12jliLqiLXs+ePUuv23tQrHhxnhrwDD74MOqz4Zw5e4ap038hqGTJAlmPMxe9AhlJZs2qVUydMomXXxlCWHgYYWHhtGjZCoBBz/8fK1csY+D/vUDliAh+mjyRP5cuYeyPE6lXvwEAU6dMYsvmv2jdph3lypcn7uRJ5s+by/y5sxnw7HM88NAjl4yloC56Lax9XJA05twVyEWvN49w/qLX3wZ4bZteUUhGE4D6WOemPjfGfHY5y8ucjFb+Ookdq5Zy4nAMKcnJBF9Rlqr1GtOu+92Elq2QMc+uTWtYOWsSR6L3kZJ0ntBylWjauStNr70F30xV66Tz51g6ZQzbViwi8expyleNpNNdD1OtfpOMaf6YNo5l03/AkZAy5Xly+Pgc5QU5HFBsTEzWIVTatOWFlwZTOdvIB5fD2WTU9Mq6Dsubt2jJN2OtfZSYmMjIEZ8y97ffrOGApC7PPPscLVq1zph+08YNfPP1l5gdO4iLO0loWBg1akZy3/19ad+ho1OxFORwQIWxjwuaxuxYgSSjW0Y6n4xmPaXJqKjInIy8RUEmo8LgbDLyJAWZjNR/R4Eko26fO5+Mfn3Caw9EHbVbKaU8mZd3THCWJiOllPJkRaTWrclIKaU8mZf3knOWJiOllPJkWjNSSinlbtlH6f+v0mSklFIeTJORUkopt/Px1WSklFLKzbRmpJRSyu00GSmllHI7TUZKKaXcr2jkIk1GSinlybRmpBzytkFHAdbsPu7uEPKkVc3Cu3+QUp7O11dHYFBKKeVmWjNSSinlfkUjF2kyUkopT6Y1I6WUUm6nyUgppZTb6XBASiml3E5rRkoppdxOk5FSSim302SklFLK7VyVjETkWeBh++Uq4FGgGjARCAeWAQ8ZY1JEpDjwI9AEOAj0NMbE2st5BegHJAH9jTF/5ieeonFpr1JKeSufPDycJCIVgKeB5sCVQCXgRmAo8K4xphbgB/SyZ3kYiLbLvwNet5fTEOgB1ANuA77I72ZqzciNDsbG8uHQ91i1cjlpaWm0btuOFwcNpmKlSi5b5/rli1izdAF7o3ZyKu444WUr0KxtB7r27EvxoJIAjPn0LVYumu1w/gqVq/LWl5MB+GXCN/w68VuH0/kHBPLF9KVZyk4cO8zMH79my7qVnD19ipDwMrS65jpu7/tEAW5hTu7Yz5fD2+IF74t5wby5zJn9G9u3beX48WNUqFiRa6/rwsP9H6VkyVLuDi8LFw0H5IP1/V8cSLX/PwS050IC+h4rCU0EugMv2+UTgTfs592AScaYJGCniMSJSD1jzI68BqTJyE0SEhJ45MG+BAQG8ta7Q/HxgZEjhvPwg334afovBAUFuWS9836eQHiZ8tzW5zHCrijL/t1/88vEb9m5ZQMvffA1vr6+3HLXA3S46bYs8x07HMvoD1+jcev2GWVXd+lOg2Ztskx3/lwCw4c8S+NWV2cpP3oolqEv9qdM+Urc1f9ZgkPDOXY4lsOx0S7ZznTu2s/55W3xgnfGPG7sGCpWrMjTA5+lfPkK7NyxnS8/H8naNav5fvwkjxoPLi/NdCISCoQ6eOukMeZk+gtjTKyIfAzsB84Dk4A9wAljTKo9WTRQ2X5eCThgz3tWRPxEJMAuX5ZpPenzaDLyFtOnTiE6ej8zZ82larVqANSuI3TvegNTp0ymT78HXLLep1/9kNIhYRmvpWEzgkoH892nb2G2bKBe4xaUqxhBuYoRWebbsWkNAG07d80oCy9TjvAy5bJMt3LRHFJSUmh3bdcs5T9+PpTQK8ry3Luj8PcvvMPOXfs5v7wtXvDOmEeM+pLw8AsD8rZo2YqQkFBeGTyItWtW07pNWzdGl03eThkNBIY4KH8Du2kNQETCsGo11YHTwAysZjq38Zj0LyKfishTmV6vEpGPMr2eLCJ3XMbyB4rI65cZZoFZsngRjRo1zvjwAkREVKFJ02YsWbzQZevNnIjS1ahdH4CTx47kOt/KRXOoVqsulavVvOjyVy6aTXBoOA2atc4oOxwbzbYNq+l8S89CTUTgvv2cX94WL3hnzJkTUboGVzYE4PDhQ4UdzkX5+Pg4/QCGATUcPIZlW+x1wC5jzDFjzDngF6ABECYi6XkhArs2BMRg15JEpASQYjfNZZQ7mCdPPCYZASuBtgAiUgwIwDq5lq4tsMINcbnErqgoImvXyVEeGVmL3buiCjUWs3UDABWrVHf4ftT2vzgcG52lVuTI8SOH2LllA6073oCf34WkE7VjMwCBxYrxyasDePy2a3jmri58+8kbnI6PK5iNyIUn7WdneFu84J0xO7JunVX7r1kz0s2RZJWXZGSMOWmM2evgcTLbYv8F2opICRHxAToBO4HlWOeHAPoAM+3ns4D77ef3ALMzld8lIgEiUhcIzc/5IvCsZroVWD05wEpCy4GmdrtkOSAF6CQiL2NVXKcZY4YAiMg9WCfXspc/DAwCjgEGq00UERkAPAYkA9uNMXcVyhZmEhcXR3BwcI7ykJAQ4uPjCy2OE8cO88v4b6jXpCXVa9dzOM3KRXPw8/en1TXXX3RZq5bMJS01lXbZklbcsaMAjB3+Dm063chNd/bhSGw007//gtj9exn88bcua6P3lP3sLG+LF7wz5uwOHTrE5yNH0KZtu4wakqdwRdduY8xqEfkZ2ID13boKGI/1vTvJPp+0HJhizzIaGC8iUVgdHXray9ksIrOwElkSVvfwfPGYZGSMibZPilXAqgWtwqq5NcXq+74HeAdoCcQDS0VkKdZOyK38f1iJLcFe3h57dS8BNYwx5+wTfkVSYsJZRr09CF8/P/o984rDaZLOn2PdskU0ankVpUNCL7q8lYvmULVmHSJq1MpSnppmnQ+Vhs249/EXAKjXuAUlSpbi6w9eZduG1TRs4UFt9KpIOXvmDAOffhx/Pz/efPs9d4eTg6vGprN/tGc/vxQFtHAwbQJwey7LeYMLvevyzZOa6eBCU11b+/mqTK9nAouNMUeNMem9P9pjJSFH5a3s8uP2jpyWaT2bsbL8fVi1o0IXHBLs8Fdjbr8yC9r5c4mMfOsFjhw8wMA3Ps3RESHdptV/cvbMqRy1nez2/L2Ng9H7aHttzulKlQ4BoF6TVlnK6ze1Xv+72+RnE5zi7v2cV94WL3hnzOkSExN5+snHiN4fzRdff0v5ChXcHVIOeTxn5LU8MRm1w6q17MFKRm3sstB8LC8tl/KbgZFYta41IlLoNcTIyFrsivonR/nu3buoGVnLwRwFJzk5mS/f/x97o3YyYMgnRFTPfX0rF82hVHAoV7Zod9Flrlg4Gz9/f1p36JLjvUpVL97pwdfHdYehO/dzfnhbvOCdMQMkJSXx/LMD2L5tK6O+/JradcTdITmkycg9VgB3YPV9xxgThXVlbx1gDNBRRMJFJBDoDfwBrLlIeScRCbOHsrgNwO4pUsUYswTrfFJpoNCvcuvYqTNbNv9F9P79GWUHDkSzaeMGOnTq7LL1pqam8s1HQ9i5eR1P/u99Iutemeu08SeOs23Dalp36HLRXnDJSUms/fN3rmze1mFvvZp1GxASdgXbNqzKUr51vfW6eh3H56oKgrv2c355W7zgnTGnpqYyeNDzrFm9imGffU6jxk3cHVKufHycf3gzT0tGG4GKWDWidNHAZmPMfuAVYCnWSbcFxpjF9vhIjspjgHexktISrKY5sIa4+FFEtgCbgFEOepq43O139qJSpco88/QTLF70O0sWLWTgU09QvkIFevbs7bL1TvjyI9YvX8T1t95DYLES7Nq5NeNx/OjhLNOuWjqP1NSUS/ai27x2OWdOxefalOfn58/tfR9ny7oV/DBqKNs2rGbxb9OY8MVHSMNm1G2Uo4m6wLhrP+eXt8UL3hnzu2+/wfx5c+nT9wFKlCjB5r82ZTwOHTzo7vCyKCo1I5+0tNxaspQjicm5Nv3lWWxMTNYhVNq05YWXBlO5csSlZ86DNbuPZzx/6aHbOHbY8Yet290P0f2ehzNev/H0/aSlpfL6yPEXXf7It18kavtffDRuFv4BAblOt3LRHOZO+5HDMfspWTqYZld14vY+j1O8RNYr9FvVzHkNyOUorP1cULwtXvC+mG+6vjMxMY4vh3nsiad4/MmnC2Q9xf3zeMmqAzJontPfOWboDV6bkTQZ5VFBJqPCkjkZeYOCTkZKuUtBJKO6LzmfjHa+773JyGO6diullMrJV287rpRSyt28/FSQ0zQZKaWUB/P2jgnO0mSklFIerIjkIk1GSinlyTzp3kqupMlIKaU8mNaMlFJKuZ2eM1JKKeV2RSQXaTJSSilPpjUjpZRSbldEcpEmI6WU8mQ6AoNSSim302Y69Z/hbQOPnktKdXcIeVYsoGhcC6IKXxHJRZqMlFLKk2nNSCmllNsVkVykyUgppTyZdmBQSinldtpMp5RSyu00GSmllHK7IpKLNBkppZQn05qRUkoptysiuUiTkVJKeTLtTaeUUsrtfItI1UiTkRsdjI3lw6HvsWrlctLS0mjdth0vDhpMxUqV3B1arjwl5pUrlvH9d9+wZ/cuTsXHERYWTsPGTXnksSepGVkLgMce6sOG9Wsdzt+m3dWM+Hw0AGtWr2TWzJ/ZsnkTR48cpkzZcrRu247+jz9NePgVhbZN6TxlH+eFt8W8YN5c5sz+je3btnL8+DEqVKzItdd14eH+j1KyZCl3h5dFEclF+KSlpbk7Bq+SmEyB7LCEhAR63d6DgMBAnhowEB8fGDliOImJCfw0/ReCgoIKYjUFqrBidmZsunlzfsPs2E6Dho0ICwvj4MFYxo0ZzeFDB5nw00wqVqrM7l1RnDlzOst8W/7axLCPh/Liy69yZ+97AHjp+YEkJJzluutvpHJEBP/+u4/RX4wkIDCQCT/NICio5CXjKaix6fS4KBz33d2LihUr0rHztZQvX4GdO7bz5ecjqV6jJt+Pn4Svb8H8PYv7c9mp5IbPVzv9nTPvidZem7q0ZuQm06dOITp6PzNnzaVqtWoA1K4jdO96A1OnTKZPvwfcHGFOnhTzDTfdzA033ZylrMGVjeh5a1cW/T6fe/s8kFFDymzG9J8ICAjg+hu7ZpQNGvwaYeEXBpNt1qIVVatV57GH+vD7/Ll0v/UO121INp60j53ljTGPGPUl4Zn+5i1atiIkJJRXBg9i7ZrVtG7T1o3RZVVEThnhsUMNi0iKiGwSkW0i8quIhBbAMkNFZO/lR3f5lixeRKNGjTM+vAAREVVo0rQZSxYvdGNkufP0mENCQgHw8/Nz+H5iQgKLFsyjfYdOGdMCWRJRuvoNGgJw5PChAo/zYjx9HzvijTGHO/ibN7jS+psfLuS/+aX4+vo4/fBmnlwzOmWMaQIgIj8ATwLvuDWiArQrKoqOna/NUR4ZWYsF8+e6IaJL88SYU1JSSE1NITYmhlEjPuGKMmXokq3GlG7Jot85c+YMN3e79ZLL3Wifa6peI7Igw70kT9zHl+KNMTuybt0aAGrWLNy/+aX4XH5Ln0MiEg58C9QD0oCeQAIwEQgHlgEPGWNSRKQ48CPQBDgI9DTGxNrLeQXoByQB/Y0xf+YnHk9ORpktBxoDiEgT4CugBLADeNgYc0pEHgEeAYoB24G+xpjzIlILGA8EAbPTFygiDYDvgECsGuIdxph/CmuD4uLiCA4OzlEeEhJCfHx8YYWRJ54Y8wP392bn9m0AVKlSlc+/Hptrp4PfZs0kPPwK2l7V/qLLPHPmDJ98+B41akbSoVPOL1lX8sR9fCneGHN2hw4d4vORI2jTtl1GDclTuLDCMwKYYYy5TURKYOWD74B3jTEz7EpAL6zk9DAQbYy5U0QeAl4HHhWRhkAPrIQWCUwFrsxPMB7bTJdORPyA64Ff7aLvgf8zxjQCDgAv2OXTjDGtjDGNgVjgbrt8GPCJMaYhEJdp0Y8Bw+3aVwsg2pXboVzjjbeHMuaHSbz13keULFWKpx97iJgDB3JMd+TwYdauXskNXW/B3z/332DJycm88tJzHDl8iLff//ii06r/hrNnzjDw6cfx9/Pjzbffc3c4Ofj4+Dj9cJaIhABtjDHjAIwxCcBpoD3wiz3Z98Ct9vPuwA/284lAevNDN2CSMSbJGLMTiBORevnZTk/+pJUWkU1ABGCA+fYOLGWMWW5PMw4r2QA0FpG3gBD7kWSXt8bakWBVM/vbz1cC/xORKliJrNBqRQDBIcEOfzXm9ivTE3hizDXsJpUrGzam3VXt6dH1OsZ9N5qXX3k9y3RzfvuF1NTUizbRpaam8sarL7N29Uo++exLatcRF0bumCfu40vxxpjTJSYm8vSTjxG9P5ox436gfIUK7g4ph7x07bbPrYc6eOukMeZkptc1gCN27edKYAXwJnDCGJPenTUaqGw/r4T14x9jzFkR8RORALt8Wablps+zw/moLZ5cM0o/Z1QN8AGeuMT0Y7DaKxsCH2A11+XKGDMBK0mdAWaJSOfLjjgPIiNrsSsqZ/7bvXuXw15gnsDTYy4dHExE1apE79+X473ffp1B7Tp1qSN1c53//bdf5/f5c3j7/Y9p1do9vak8fR874o0xAyQlJfH8swPYvm0ro7782i0/Ppzh6+Pj9AMYCOxx8BiYbbH+WC1Cw4Bm9ut+hbE9ufHkZASAMeYMMAB4DitxnBKR9G+K+4E/7OelgMMiEsiFJjqA1UB639x70gtFpCaw2xjzGfAz0MhlG+FAx06d2bL5L6L3788oO3Agmk0bN9ChU6HmRad5eszHjh1l3549RERUzVK+fdtW9uzexc3deuQ677CPhzLz56m8+sY7dOx8natDzZWn72NHvDHm1NRUBg96njWrVzHss89p1LiJu0PKVR570w3DqvVkfwzLtthoYJ8xZr0xJg2YgfUdGCYi6XkhArs2BMRg15Ls80spxpikzOUO5skTT26my2CMWSciW7BOpvUFvhSRIGAn8JA92WvAWuAQsDHT7AOBCSLyKpk6MNjLul9Eztvz3EMhuv3OXkyaMJ5nnn6CpwY8gw8+jPpsOOUrVKBnz96FGYrTPCnmF559irr16lOrtlCyVCn+3beXiT+Ow8/Pj3v69Msy7exZM/Hz9+fGm7s5XNa470Yz4YexdLv1dqpUrcaWzZsy3gsLCyeiSlWH87mCJ+1jZ3ljzO++/Qbz583lkf6PUaJECTb/tSnjvfLlK3hUc11emunspriTTkx3UERiRKSWMSYK6IjV8asEVovRDKAPMNOeZRbWj//1WN+VszOVfysiI7A6MIQaY/LcRAc6AkOeFdQIDACxMTFZh1Bp05YXXhpM5coRBbWKAlcYMTszAsO470azcP5covfvJyk5ifLlK9C8RSv6PtifSpUv/FBLTkqia5cOXNmwMZ+M+MLhsi42bNDN3W5lyFuXPqldUCMwgB4XheGm6zsTE+P4B/xjTzzF408+XSDrKYgRGHqP2+j0d87kvk2dXp+INAVGY/Uo3gk8AFQEJgFhWL2YHzTGJNu1ofFYtadDWF27Y+zlDMFKXEnAo8aYpc7GkFmuyUhE9oPDL14fIM0YU3g/Fz1IQSYj5ZgzycjTFGQyUv8dBZGM7spDMpqUh2TkaS7WTHd1oUWhlFLKoSJ/cz1jTM4uSUoppQqVl4/y47RLdmAQkWpYXaUbAsXTy40xNV0Yl1JKKYrOzfWcaegeg9WzIg2ri/TvwFjXhaSUUiqdK0Zg8ETOJKMwY8xEINUYsxF4FLjFtWEppZQCq5nO2Yc3c+Y6o/P2/ydFpC7WuG9lXReSUkqpdN5e43GWM8lomj3U+PvAKqza1H/mVg5KKeXJikYqciIZGWM+tJ/+JiLlgOLGGO8YF14ppbycn7e3vznJmd50OQaXEhGMMYtcE5JSSql02kx3wauZnhfHutPfBkCTkVJKuVgRyUVONdN1yvzavkPq8y6LSCmlVAbfIpKN8jygljFmG9DUBbEopZTKxsfH+Yc3y+s5Iz+gDRfuoqqUUsqF9JzRBZnPGSUDUVj3AlLKJbxxBOzw3mPcHUKeHZ/8oLtDUE7w02SU4R5jTGzmAhHxnDtPKaXUf1gR6dnt1Dmj3xyUzXZQppRSqoAV+eGARCQUCAcCRaQGFy4EDgGCXB+aUkopPWcEfYGBQCWyXlMUB3ziwpiUUkrZvL3G46yL3VxvODBcRAYZY4YWYkxKKaVsRaRi5NQ5o00iEpL+QkRCRaSLC2NSSill8/fxcfrhzZxJRu8bY+IyvY7DGsFbKaWUixWVi16dSUZZNtEYk4ZzXcKVUkpdJl8fH6cf3syZZHRMRG5IfyEiNwHHXBeSUkqpdEWlZuRMDWcg8LOIJNqvA4AHXBaRUkqpDEW+N106Y8wW+3bj9YHuWGPTzQZCXRvaf8uhgwcZ8+1otm/byt9mJ4mJicyev5DKlSOyTNe4gTicf/LUGdStV68wQr2og7GxfDj0PVatXE5aWhqt27bjxUGDqVipkrtDy5U7Ym4j5RjcqymNaoRTItCfXbFxfDlnB98v+idjmmrlSvFun5Z0alSJAD9f1kUd5X/fr2HDrqwNDzu+6Em1cqVzrKP30N/5dc2/DtffWsqx8O2b8fX1oXTP70hJTSvYDcxGjwvX0Zvr2USkBdAP6AmUBB4B7nJtWP89//67j/nz5lC/fgOaNmvByhXLcp22+62307NX7yxl1apXd3GEl5aQkMAjD/YlIDCQt94dio8PjBwxnIcf7MNP038hKMjzroV2R8xXVgvjtyE3suafIzz1xXLOnkvmtrbV+fLJ9hQL8GP0vJ2ElyrGwrdv5lRiEk9/uYKE88k83e1K5rzRlWsG/YI5EJdlmQs2RvP25I1Zyv6JyTpNOn8/Hz57tB2H4xKoEOb6v4keF65VRHLRRUdgeAHrwtc0YBzWTfVWGGMmujIgEbkLmABUNcZE53MZe4EmxpiTBRjaZWneoiWL/1gBwPSpP100GZUvV45GjZsUUmTOmz51CtHR+5k5ay5Vq1UDoHYdoXvXG5g6ZTJ9+nle6607Yu55VU38fH24870FnElMBmDR5hiurBbOPR1qMXreTh65oS7lQktw/dOz2XPoFABLtsSy7fOevHJXM+7/eHGWZR6NT2TtP0ecWv+zPRri4+PD94v+4cU7Ghfsxjmgx4Vr+VA0stHFOjC8BxwE7jbGfGQPluraur7lLmAF0PtSE3oTX1/vG4k6uyWLF9GoUeOMDy9AREQVmjRtxpLFC90YWe7cEXOAvy9JKakknE/JUh5/9nxGj6dWdcoSFRufkYgAzp5LZsWOg9zUvEq+m2ZqlC/NoDubMHD0CpKSU/O/EXmgx4VrFZWx6S72DVkdWAxMF5H1IjIA635GLmNfXNsMeBI7GYlIPxH5WUT+EJG/ReQ5u7y6iGwTkckiskNEfhCRAAfLvF9E1orIXyLyiV1WUkR+s8u2iohHJb4pkyfRosmVtG7emIcf6MOG9evcHRIAu6KiiKxdJ0d5ZGQtdu+KckNEl+aOmH9cbJ0X+vjBNlQMK0FIUCAPXFeHjg0r8dmsrQCkpKZx3kGyOJeUSlAxf2pWyHqOqGuLqhyd0IcTk/qy5L1b6NaqqsN1j3i0HdNX7GH59kMFvFW50+PCtYpKMrrYcEDRwDvAOyJyFdZ5o2ARmQNMNMZ874J4bgNmGWP+EpEgEalpl7cCrsS6qd86EfkVOI/VqeJBY8xqEZmA1az4TfrCRKSevcy2xphkEflRRG4GigMxxpib7ekyRphwt5u7deeaDp0oV64cMTEHGDfmWx55sC9fjh5Dy1at3RpbXFwcwcHBOcpDQkKIj493Q0SX5o6Yt+8/yY2vzWHSoGt59Car08n5pBQGfL2cqcv3APB3TBydG1cmvFQxjp8+B1hdc1vULgNAWKliGcubvW4/66OOsvfwKcqFlOCxm+oxedB1PDh8KZP+2JUx3V3XRNK0ZhkeHDbNJduVGz0uXKuoDJTqVNuRMWa5MeYRrEFTxwP3uSieu4Ap9vPJXGiqm2eMOWGMOY3Vk6+dXb7HGLPafj4JuDrb8q4FWmMlsE1YSa0WsAW4XkQ+EJH22UaYcKt33/+QG2/qSrPmLbilWw/G/jiBsmXLMXLEMHeHppwUWTGYCS90Zvv+E9zx7gK6vj6Hb+bvZET/q+jd3vp99c18g68PfDPgGmqUL02F0BJ8/FAbqtu95tIyNYg/9+0qJiyNYsWOQ8xYtZeub8xlfdQR3ry3ecY0YaUCeb9fK16fsI4j8Ymo/w4/X+cf3ixPIykYYxKAH+1HgRKRMkAHoJ6IpI/ycBQYRtZzVWmZXudWns4XGGOMeTVbOSLSDOgKvCkii40xbxbEdhS0kiVL0f6aDvw8faq7QyE4JNjhr8bcfmV6AnfE/MY9zUlKSeWOdxeQnGIdkku2xBJeujgfPtiGKct2s/fQKR4YvpRhD7dl2+c9Adi46yifzdrGsz0aEnvibK7LT01NY/qKvbzTpyUVQktw8GQCQ+5uzsETCUxbsYeQoEAAigVareohQYEkJqVw9lyyS7ZXjwvX8vaRFZzlScP63ImVOJ5MLxCRzVhNal3s+yslYSWQr+xJaohIS2PMWqxaVNYuSLAQ65zXcGPMUREph3Xeywc4YYz5UUSOAo+5csMKgidU1SMja7Er6p8c5bt376JmZC03RHRp7oi5QbUwtuw9npGI0q2LOsJd10RSLqQEh04mMHPVPn5d8y+1KwZzPjmVPYdOMbx/W/YfOU300TNOrSt9DXUjQmlUPZwD43I2WkSPu5df1+yj91DXnJjX48K1vP1ckLM8qWJ3F/BztrKfgZeAdcBMYBPwjTHmb/v9HcBzIrLDfp3lPJYxZhvwJrDQTmyzsG4Y2BBYbTfdvQm8XdAbU1BOnz7NH0uXcGXDRu4OhY6dOrNl819E79+fUXbgQDSbNm6gQ6fObowsd+6I+dCJBBrVuIIA/6wfr5a1y5JwLjnjHBFYtRxzII49h05RMawEd7Sryeh5Oy+6fD9fH+68qgb/HjnNoZMJALz43WpueG12lscPdkeKrq/P4Y0J6wt4Ky/Q48K1XDkckIj42h28ZtivI0VkjYhEichYEfGzy4uLyFS7fJmIVMy0jFfs8h0i0j6/2+kxNSNjTEcHZUNEZA/WNUO3OZgtyRiT4wJcY0z1TM/HY53nymwbMO+yAs6HBfPmArB9u9WjavmffxAWFk5YeDgtWrZi3HffsnfPHlq2ak3ZcuWIjYlh3NgxHD16lPc++Kiww83h9jt7MWnCeJ55+gmeGvAMPvgw6rPhlK9QgZ49PapDYgZ3xPzlnB1MeKEzU1++jtFzd5JwPpmbW1ald/tIRvy6laTkVPz9fHjn/pYs236Q+LNJ1KsSygu3N2bH/hMM/3VrxrJ6Xl2TW1pWZd6GaKKPnaZcSAkevbEeTSPL0OeTCw0Bm/cezxHHNQ2s74s/tx106QgMely4lq9rrzN6HNiF1QIFMBR41xgzQ0R+AHoBE4GHgWhjzJ0i8hDwOvCoiDQEegD1gEhgKlZnszzzmGRUFDz/f89kef3OW28A0KJlK74d+wPVqtdg0e8LWLRwAadPn6ZkyVI0adqU1998h4aN3F8zCgoKYvSYcXw49D3+99KL1hAqbdrywkuDCSpZ0t3hOeSOmGes2sutb8/n/25tyKjHr6J4oD97DsYz8OsVfLPAAFYHhVoVg+nVPpLQkoEcOHaG7xf9zQfT/spyfdDeQ6coG1Kcd/q0JLxUMc6cS2LDrmN0f2sev2864JL480qPC9dyVQu9iJQHbsfqNT1ARHyA9lgJCKyWpoexklF34GW7fCLwhv28GzDJGJME7BSROBGpZ4xJb61ymk9aWmFcx/rfkZhcKBf+Ki8T3nuMu0PIs+OTH3R3CP95xf0vv1rz9ap9Tn/nfNy3SxiOxw09mX1EGhEZB3wJFMMaEPthYJkxpq79fj1gtDHmahHZClxnjDlovxcLVAU+teeZZJdPtuf5PS/bCJ51zkgppVQ2eTxnNBDY4+AxMPMyReQaIM0Ys7Iwt+VitJlOKaU8WB67dg8DxjooP5ntdTusay33Yp0vCgY+BsJExNcYkwpEAOltwTFAZeCgiJQAUowxSSKSXp4u8zx5oslIKaU8WF5ykd0Ud9KJ6d4H3gcQkY7AQGNMXxGZjnV+aAbQB6sXM1g9ke8H1gP3YA0+kF7+rYiMwOrAEJqf80WgzXRKKeXRfPPwKAAvAq+IyC6sy9jSR8QZDVQVkSjgQazedBhj0i+Z2YmVwJ7I74q1A0MeaQcG5Yh2YFCOFEQHhu/X7Xf6O6dPiypee4msNtMppZQH0+GAlFJKuV3RSEWajJRSyqMVkYqRJiOllPJknjBIcmHQZKSUUh6sqHR51mSklFIeTDswKKWUcjttplNKOc0br9kJa/mUu0PIkxNrR7o7BLfQZjqllFJupzUjpZRSblc0UpEmI6WU8mhFpGKkyUgppTyZXxHJRpqMlFLKg/kUkYY6TUZKKeXBikjFSJORUkp5Ml+tGSmllHI3rRkppZRyOx0OSCmllNv5Fo1cpMnInQ7GxvLh0PdYtXI5aWlptG7bjhcHDaZipUruDi1XGrPrFUa8/r6wZNxzNKxdmaASgUjX1/g39niWaYoF+jPkiVu4q2tLQkuXYPPfB/jf8Bks37ArY5pSQcX4csi9NKlbhQplg0lKTiFq32FGTVzKpNlrsyxvwH2duaZFbZrVr0rFsiG8/eVs3vlqdpZpqlYMx8x+M9e4z6dAqtM34b44bzkuikpvOp+0tAL6yxYRickUyA5LSEig1+09CAgM5KkBA/HxgZEjhpOYmMBP038hKCioIFZToDRm1yuMeH19IMAX5i3fhp+vL9e3q+cwGX33Tl9ubN+AwZ/OYM+BozzW6xq6XFWfjn0/ZvPfBwAIDynJJ4N6sniNYV/MMYoF+nNnl+bc1601L340jc/GL85Y3sZpr3DqTCIbd+6nf8/2DpNRYIA/TepG5Ih5yJO30KlVXc6lXPbmA4V3XBT3v/xMstgcc/o7p5Nc4bWZq8jXjERkE3CrMWZvYa53+tQpREfvZ+asuVStVg2A2nWE7l1vYOqUyfTp90BhhuMUjdn1CiPe1DQ4lwK3Pf0F/W5ry/Xt6uWYpmGdytzVtSX9h/zID7+sAuDP9VFsmPo/Xn3iFnoO/AqA43Fn6Dd4bJZ55y3bTu1q5ejTo22WZNTszndIS0vDz8+X/j3bO4ztfFIya7bszVJWongALRpUK7AaEXjXcVFUakZeMSCsiPi5O4aCtmTxIho1apzxQQCIiKhCk6bNWLJ4oRsjy53G7HqeEu/NHRpyPimZqfPXZ5SlpKTy07z1XN+2LoEBF/8de+zkGZJTslZj8tsK06NzE4JLlSClAJORp+xnZ/j6OP/wZh5RMxKRIcA9wEHgODATeB2YDHQBhohIKeBlrHEDpxljhtjznjTGhNrP+wFNjDEDRWQsEA+0BMoBTxlj5ohIEDAOaABsBgIKZyuz2hUVRcfO1+Yoj4ysxYL5c90Q0aVpzK7nKfHWj6zI3gPHSEhMylK+Y1csxQIDiKxShh27D2Z5z8/Pl5BSJbj12iZc37Yej705vkBiua9baw4diyckJLhAlgees5+dUVR607m9ZiQiLYHuQCPgVqzkkS7GGNMUWAe8A3QCmgFdRKSzE4svC7QD7gbessueAI4aY+oDnwH1C2Az8iwuLo7g4JwfrpCQEOLj490Q0aVpzK7nKfGGBQdxMv5sjvLjdll4SMks5Y/1vobT60ZwYMlQPn2pJ89/OJUJs9ZcdhyVyobQsWUdJs9Zd9nLysxT9rMzfPLw8GZuT0bAVcAMY8w5Y8wJYF6m936y/28JLDbGHDXGnAcmAY4bnbOaYYxJA9YD1e2yq4GJAMaY5cCey98EpYq2qfM3cNW9H9D9yVF89/MKPhnUk4fuuOqyl3vPLa3w8/PNOG9VFPn6+Dj98GYe0Ux3ETl/muWUuSW5WLb3zgEYY1JEJPO2ur0LYXBIsMNfYLn9YvMEGrPreUq8J+PPUrVieI7y8GCrl9nxuDNZyo+eOM3RE6cBWLBiB0HFA3nv2dsYN3Mlycmp+Y7jnltasWnnfrb+E5PvZTjiKfvZGd6dYpznCTWj5UB3EQkUkRCsc0TZrQE6iki4iAQCvYE/7PeOiUik3cmhuxPrW2bPj4i0BWpc9hbkQ2RkLXZF/ZOjfPfuXdSMrOWGiC5NY3Y9T4l3++5Yqle+ghLFs55SrVuzIufOJ7Fr/9GLzr9h+7+ULlmc8uH5/2JvXr8q9WpWZPyvq/O9jNx4yn52ShFpp3N7MjLGrAXmAFuxOi5sxup4kHmaWOAVYCmwAVhgjEnvM/oyMB/4E/jXiVV+DlQQke3AAGB7AWxGnnXs1Jktm/8iev/+jLIDB6LZtHEDHTo5czqs8GnMrucp8c5eupXAAH9uv75ZRpmfny93dmnG7yt3cj4p+aLzt29ei1NnEjl8/FS+Y7ivW2uSklIK/HwReM5+dkZRaabziIteRaSUMea0iAQDq4BbjDG73R2XIwV10evZs2fpdXsPihUvzlMDnsEHH0Z9NpwzZ88wdfovBJUseemFFDKN2fUKK15fH+j30jd0bCX079meAe9O4uiJ0xw5cZpl66MA+P79B7iubT0GD/uZvQeO0b9ne25qfyWd+n3Mpp3RADx0x1W0aliDxat3En34JFeElOSOLs3oeUNzXhk+g4/H/p6xzmb1q1KtUji+Pj78+MFDTJu/gWkLNgAwd9m2LD33Avz92D3/HVZt3pNxTdOJtSMLZNuh8PZzQVz0unZ3nNPfOS1rhnhtRvKUZDQBq1dbMeBzY8xnbg4pVwWVjABiY2KyDkfSpi0vvDSYypVzXoHuKTRm1yuMeIvncrb4j3X/cMMjw61pigXwxlPd6HVjC0JLl2DL3wf43/CZ/Ln+QvNWm8Y1GPTwjTSWCMJDgjh28gw79xzksx8XM3fZtizL/vqN+7i/exuH680+AkT3To2Y/El/7n7+G2Ys3AQUbDKCQtvPl5+M9uQhGdXQZFRkFGQyUsqdwlo+5e4Q8qSgk1FhKIhktG5PvNPfOS1qBHttMvL03nRKKVWkefmpIKdpMlJKKQ9WRHKRJiOllPJkPi6oGolIHWAMEAakAG8ZY34SkUisQQHCsS6Deci+TrM48CPQBGvYtp52L2dE5BWgH5AE9DfG/JmfmNzetVsppVTufHycf+TBOazE0QDr2s7hIlIaGAq8a4ypBfgBvezpHwai7fLvsMYORUQaAj2AesBtwBf53U5NRkop5cFccc2rMWafMWa7/fwgcAQogzXM2i/2ZN9jjRcK1oACP9jPJwI328+7AZOMMUnGmJ1AnIjkvCeJE7SZTimlPFkesoyIhAKhDt46aYw5mcs8LbHuXnAaOGGMSR+/KRqobD+vBBwAMMacFRE/EQmwy5dlWlz6PDucj9qiNSOllPJgPnn4BwzEGvw5+2Ogo2WLSDmsGtBDuHnMTk1GSinlwfJ4zmgY1nib2R/Dsi/XvrfbL8CbxpiVwDEgTETS80IEdm0IiMGuJYlICSDFGJOUudzBPHmizXRKKeXB8tIxwW6KO3mp6eyBpScBPxtj0m+pkyYiy7HOD80A+mCNFwowC7gf63Y89wCzM5V/KyIjgEgg1BiT5yY60GSklFIezcc1VxrdhNUJoaqI3G2X3Q+8CEwSkY+x7qgwxX5vNDBeRKKAQ0BPAGPMZhGZBezE6tr9aH4D0uGA8kiHA1L/FTockOsVxHBA22POOP2dU79SSa+9RlZrRkop5cG8NrvkkdaM8sgba0YJ51PcHUKelAj0c3cIygNt3HvS3SHkWdtaoZedS3bEOl8zqldRa0ZKKaVcwNtvmucsTUZKKeXBikYq0mSklFKerYhkI01GSinlwVzUtdvjaDJSSikPVkROGWkyUkopT1ZEcpEmI6WU8mSuuLmeJ9JkpJRSHqyI5CJNRkop5cmKSC7SZKSUUh6tiGQjTUZudOjgQcZ8O5rt27byt9lJYmIis+cvpHLlCHeHxsAn+7NqxTL6Pfwojz35DAAxMQe4/ebrHU6/4I9VlC4dnPH63LlzfP35CObO/pXTp05Ru05dnnzmOZo2b1Eo8Wd2MDaWD4e+x6qVy0lLS6N123a8OGgwFStVKvRYnOFt8YJ7Yl67bCGrls5n7z87iY87wRVly9O8XUdu6dWPEkEls0wbtXMLM8Z/wy6zlZTkZMpVqMwtvfvRpkOXjGnOnz/H9B++YuXiuZw9c5qqNWvT64GnkCubZlnW3J8nsGPzevb+s4O4E8focc/D3HbvIy7bTu3arVzu33/3MX/eHOrXb0DTZi1YuWLZpWcqBPPn/MY/f+/M9f2+Dz5C+w6ds5QFZfvwv/PGK6z48w+eevZ5KleOYOqUiQx88hFGj5tAHannkrgdSUhI4JEH+xIQGMhb7w7FxwdGjhjOww/24afpvxAUFFRosTjD2+IF98U8Z/p4rihbgTv6Pk54mXLs22WYMeEbdmxezysffYOvr3WPuE1rlvHZO4No0+EGHnvhTfz9A4jZv4ek8+ezLG/M8Hf4a+1yej/4NOUqVGbhrKl89OozvPLRN1SLrJMx3dK5MykRVJJmbTuwePZ0l2xbZnrOSLlc8xYtWfzHCgCmT/3JI5JRfHwcwz5+n4HPvcRrg19wOE2lylW4slHjXJfxj9nJ/Dm/8crrb3NLj9sBaNq8Jffc2Z2vPx/JR8NHuSR2R6ZPnUJ09H5mzppL1WrVAKhdR+je9QamTplMn34PFFoszvC2eMF9MQ8c8jHBIWEZr+s2bEap0iGM/uQNdm7ZQP3GLUg4e4Zvh71N55vv4N7+/5cxbYOmrbIs69/df7NqyTweGvgK7a/vBoA0bMr/Hr+bn3/8moFDPsqY9p0vJuLr60tKSnKhJCPfIpKMXHrbcRFZIiLXZit7S0TeFJFOrly3M0QkVET2umv96b/cPMmo4Z9QM7I2XW66Od/L+HPpYvz9/bmuy00ZZf7+/lx/w02sXrmM89l+kbrSksWLaNSoccaXJEBERBWaNG3GksULCy0OZ3lbvOC+mDMnonQ1alu17hPHDgNWU96puBPceNu9F13WxtV/4ufvT6v2F5qh/fz8aX3N9WzdsIqkpAvHbOF/bn3y8PBert6rk4Be2cp6AxOMMYtdvO4c7Fvtqlxs2rieObNm8sLLr150ui8++5SrWjTk2vateP6ZJ4n65+8s7+/eFUWlyhEUL1EiS3mNyFokJSURvX9fgceem11RUUTWrpOjPDKyFrt3RRVaHM7ytnjBs2LeuXUjAJWq1ADgn+1/UbJ0MNF7o3jliXt4sFs7/q9vN2ZM+IbUlAu3Vjnw727Klq9EseLFsyyvcrWaJCcncTgmuvA2IhsfH+cf3szVzXRTgSEi8qQxJllEmgIJwEsiMsMYM8OumYzDuu86wG3GmL0iUhb4CqiKdTvbJ4AoYCMgxpgUEakALDLG1BeRR4BHgGLAdqCvMea8iIwFEoFmwDwRGQeMB4K4cB/3Ii8p6TxD336de/o8QLXqNRxOExgQyG139KJV26sICwtj7949jPv2a/r3u4dvf5hMjZqRgNXUVzo4OMf8wcEh1vtxca7bkGzi4uIIdhBLSEgI8fHxhRaHs7wtXvCcmE8cPczPP35NgyatMtWQjnL+3Dm+/PA1ut/1INVr1WX7prX8MnEMZ0+f4p7+zwJw5lQ8QaVK51hmyVLWdp0+VXjHbHZenmOc5tKakTHmKPAXkN5U1xurtpTdQWNMU2AikH4v5GHA+8aYFkBf4CtjzCms+7LfaE9zH1ZiAZhmjGlljGkMxALp93UHKAe0Mca8ai/3E2NMQ8B9R5iH+WHsGM6dO0e/h3K/hX2ZsmUZ9MrrdLr2epo0a8Gtt/fky2+/Bx8fxn77VSFGq1RWiQlnGf7WC/j5+fHQs69klKelpZJ0/hw97n6Im26/l3qNmnNHn8focGMPFv42lbNnTrsxaucUlZpRYTR+Zm6q6wlMdjBN+lnA9UB1+/l1wNcisgmYApS3y78F+tnP7we+t583FpFlIrIFuBOon2n5U40xqfbz1sBP9vMf8745/z0HY2MY9+1X9H/iaZKSznPqVDynTlm/aJPOW69TUhzfLbZ8hYo0btKMHdu2ZpSVDg7mlINfxPHxVu4PDglxwVY4FhwS7PDXeW6/5t3N2+IF98d8/lwiw954jiMHY3jureGElymf8V6p0taxlr3DwpVNW5OSnMyBfbsBqwZ09vSpHMs+czo+y3LcwcfHx+mHNyuMZPQz0FVErgKOGWN2O5jmnP1/CheaDn2BVsaYJvajGoAx5g+grohcDxw2xuy3px8D9LdrPB9gNdelO1uwm/TfcuBANOfOneP1/w3i+mvaZDwAxn//Hddf04ZdUX9fdBmZPwg1a9Yi5kA0iQkJWabZs3sXAQEBRFSpln12l4mMrMWuqH9ylO/evYuakbUKLQ5neVu84N6Yk5OTGfnuy+yN2smzb3xClepZ11e5Ws2Lzu9jd1WrXLUGRw7FcC4xMcv7B/7dg79/AOUque/av6LRfaEQkpExJg5YDXyO41pRbhYCT6a/EJHMfYknYJ1nGpuprBRwWEQCydpEl91q4A77+T15iOc/q47UZdTosTkeADfe3I1Ro8cSUaWqw3kPxsbw16YN1G/QMKPs6g4dSU5OZuGCeRllycnJ/D5/Lq3aXkVgYKBLtyezjp06s2XzX0Tv359RduBANJs2bqBDp84XmdM9vC1ecF/MqampfPXha+zYvJ4Br3xArboNc0zTrE0HALauX5WlfMv6lQQEFiOimnWes0nr9qQkJ7N22YXefykpyaz543caNGtNQEDhHbPZFZVmusK6zmgSVgLJSzJ6GvhSRB7AivNXrPNPYJ1beokLzXsArwFrgUNYnRxyMxCYICKv4gEdGBbMmwvA9u1WM9fyP/8gLCycsPBwWrRsdbFZC0zp0sE0b+F4XRUqVsp4b/jHQ0lLS+PKRo0JCwtn3949fD9mNL4+vvR7+MK5Jqlbn+u63MSwj94nOTmZSpUrM/2nycQeiOaNdz4olG1Kd/udvZg0YTzPPP0ETw14Bh98GPXZcMpXqEDPnr0LNRZneFu84L6Yf/jiQ9YuW0i33g8QWLw4UTu3ZLwXXqYc4WXKE1E9kquvu5mfx39NWloa1SKFbZvWsnT+L3S/60GKl7AuyK0WKbS65jomfP0pKSnJlC1fiUWzp3HkUAyPvvBGlvXu+WcHRw/FkJaWBkDMv3syklijFlfl6JF3uYrKCAw+6TvUm4jIPUB7Y8zjhb3uxGQKdIc1biAOy1u0bMW3Y38okHUknHd8vudS2jStn2U4oF9nTGP6T5OJ3v8vZxPOEhISQouWbXjo0Sdy9MBLTEzky5HDmT93FqdPnaJWHeHJZ57LNellViKwYHvgx8bEZB2qpk1bXnhpsEcMu+SIt8ULhRPzxr0ns7x+7oFbOXY41uG0mYfoSU5KYubEb1m+8DfiTh6nTLmKXHvLnXTpcVeWec6fS2Ta91+ycsk8azigGrXp+cCT1GvUPMt0oz95k+ULf3O43g/H/EzZ8heGQGpbK/SyM8mR08lOf+eULeXvtZnL65KRiHyB1TuvizFmb2Gvv6CTUWHIbzJyl4JORuq/IXsy8gYFkYyO5iEZlfHiZOR1wwG5ozaklFLu4uvtJ4Oc5HXJSCmlipIikosKpWu3UkopdVFaM1JKKQ9WVGpGmoyUUsqDFZWu3ZqMlFLKg2nNSCmllNtpMlJKKeV22kynlFLK7bRmpJRSyu1clYtEpCvwKeAHDDfGfOaiVTlFrzNSSilP5oJ7SIiIPzAC675xjYAnRMStgx9qzUgppTxYXoYDEpFQINTBWyeNMSczvW4F7Ei/H5yITAe6AV/kN87Lpckoj4r7e9/ZxOL+OvCo8n5ta4W6OwS3yON3zuvAEAflb9jvpasEHMj0OhqonNfYCpImI6WU+u8YRtabjqY7WahR5IMmI6WU+o+wm+JOOjFpDFlrQhFYtSO30WSklFJFzxqgvohUAY4CtwNd3BmQ9qZTSqkixhiTDAwEFgJbgS/TOzO4i9fd6VUppdR/j9aMlFJKuZ0mI6WUUm6nyUgppZTbaTJSSinldtq1Ox9EJAXYYr9MAZ40xqy6xDyzgd7AFcAMY0wTB9NsAm4FzgOfGmN6F1zUDmP6FNhljBlpv14FLDPGPG+/ngxMMcZMy+fyBwKhxpjXCybiLMtO/xsEALuB+7MNd5KfZYYCm4wx1S83PgfLvguYAFQ1xuTreg4R2Qs0udzt9GTpnwFjzN5c3l8CvGWMWZip7C0gDVhsjFlcCGHmypXH0H+d1ozy55QxpomdUAYD71xqBmNMV2PMKWcWboyJcXUisq0E2gKISDGsL/bmmd5vC6wohDjyI/1v0ADrIr8n3RzPpdyFtS8L4+/qUiLizvGlJgG9spX1Bia4IxG5eV/8p2jN6PIFA3EAItIRGGiMudV+PRarFjQj/Vdt5hlFJAgYBzQANmMlA0Skuj1fExHpB9xir6cmMM4Y85Y93RDgHuAgcByYaYwZKyLvA92BZGB+ek3HgRXAUPt5c2A50FREAoByWLW+TiLyMtaYwNOMMUPsdd8DOCp/GBgEHAMMsMcuHwA8Zse03RhzlxP71lnLgcb2epoAXwElgB3Aw8aYUyLyCPAIUAzYDvQ1xpwXkVrAeCAImJ2+QBFpAHwHBGL9aLvDGPNPfoITkRCgGdZAlKOBj+2/aw+smnIF4CtjzMf23/43rGs/GgHrgAeNMUnZlnk/MMCOb6Ex5v9EpCQwBetqej+sGsTkfMSb47jCGtdsMtaFkUNEpBSO//4njTGh9vN+WDW5gfZnIR5oiXVsPWWMmZPbZ+Aiptrrf9IYkywiTYEE4CURyfxZG4f1GQC4zRizV0TKYh0bVYEk4AkgCtgIiDEmRUQqAIuMMfUvcsyMBRKx/qbzRGQcDo4hlTdaM8qf0iKySUR2Yn25XLJmlIsngKPGmPrAZ0D9XKZrDNxh//+4iISKSEusD1sjrKa9lgAicgVwG9DAGNMIeDu3ldvNRX72B7AtsAr4C2gKtMNKJO8AnbA+eF1EpLOIVLpI+f+A1vZ7TTKt7iWgqR3TY87uoEuxf5leD/xqF30P/J+9ngPAC3b5NGNMK2NMYyAWuNsuHwZ8YoxpiP2jwvYY1j1emgAtuLyhUm4DZhlj/gKCRKSmXd4KKyE1Ax4RkTp2eX07pnpYSaVvtm2uZy+zrb095UTkZuBGIMYY09gYcyUwN6+B5nZc2WKMMU2xEmSOv78Tiy+LdVzdDbxllzn7GQDAGHMU6xi91i7qjVVbyu6gHetE4Cm7bBjwvjGmBdY+/cpurViOte8A7sNKLJD7MQNWQm1jjHmV3I8hlQeajPInvYmoLnAT8L2I5Gc076uxPiwYY5Zj1yIc+N0Yc8oYcwbYhfXL9yqs2tM5Y8wJYJ49bRzWr7ZvReR24OwlYkhvqmtrP1+V6fVMrHb4o8aY81gf+vZYX1COylvZ5ceNMQlA5nNNm4HxInIfVu3ocpW2zy8cwqpZzLdrIKXsfQnWr+P29vPGIrJMRLYAd3LhS6818JP9/MdMy18JDBaRl4Bq9vbk111YNRawahfpTXXzjDEnjDGnsX5Rt7PL9xhjVtvPJ2EdJ5lda8e9zt4HrYBaWOfQrheRD0SkvTEmP1+MuR1XcGE/5fb3v5QZxpg0YD1Q3S5z9jOQWeamup5Y+zS76fb/mdd1HfC1vc+mAOXt8m+Bfvbz+7F+0EDuxwzAVGNMqv08t2NI5YEmo8tkjFkJlMH61ZdM1n1azIlFODMExrlMz1O4SPOqPcxHK6zmjK5c+tfxSqwvwRrGmD1YyaiNXRbqRGzZ5bY9NwMjsWpda+ybe12OU3atpRpWU9ETl5h+DNDf/vX6AZf42xhjJmDVEM4As5z85Z+DiJQBOgDj7OajR7mQjDLvq7RMr3MrT+cLjEk/b2mMqWOMGW6M+RurprIZeFNEXstPzBdxqR826fGmy76PzwEYY7Ifw3kdBuZnoKuIXAUcM8bsdjBN+mcm87p8gVaZ9ls1O54/gLoicj1wONOwOBc7ZpzZFyoPNBldJhGpi9WUcgzYB9QTkQARCcf6ErqYZdhfTCLSFqiRh1UvB7qLSKBdI+hiL6cUVg+22cCzQMNLLGcFVhPgfgBjTBRQD6iD9WHsKCLhIhJox/oH1iCLuZV3EpEwESmO1ZSEiPgCVYwxS7DOJ5UGSuVhW3Nl1xYHAM9hJY5T9r4E61fuH/bzUsBhO97MzS2r7e0H6zwJdsw1gd3GuhXzz1jNVvlxJ1biqGaMqW6MicD63BXHat4Ktc/1dMX6YQBQw24uA2vfLsu2zIVALzvRISLlRKSi3UyaYIz5EetcYLN8xOvwuMomt78/wDERibSbT7s7mDe7PH8G7BrfauBzHNeKcrOQTB1dRKRxpvcmYNWkx2Yqy+2Yyc7hMaTyRpNR/qSfM9qE9WHoa4xJsX9R/Qxsw6qub7jEcj4HKojIdqwv1O3OBmCMWQvMwTrRPRPr13A81hf9ryKyGeuLJbfOC+k2AhWxakTpooHN9va8Aiy1t2WBMWaxMSY2l/IY4F2sL6sldkxgJesf7eaOTcAoU4Ddk40x67CaqHphnQv41N7+qsBH9mSvAWuxvjQ3Z5p9IPCCPX1opvJewFYR2Yh17ut78ucurGMis5+xzqGtw/rbbQK+sWs2YHW8eE5Edtivs6zbGLMNeBNYaMc9CwjH+uGx2j4u3+Qi5wtzc5HjKvM0Dv/+9tsvA/OBP4F/nVhlfj8Dk7C2Ny/J6GngahHZbK8vc+KYCJTkQvMe5H7MZDcQx8eQygMdKNWLiUgpY8xpEQnGSia35NJkoTxM5p5m2cqrk8t1aIWlKB5Xdu/Q9saYx90dS1GlXbu929ciUh+rLfvz//oXhio0Req4EpEvsDqFuPV+PkWd1oyUUkq5nZ4zUkop5XaajJRSSrmdJiOllFJupx0YVJEgImlYw8j4AKnA8ybTyM/5XOZe4DpjTJRYo7I/mumCSUfTDwS+N8Ycz8e60oAA+6Jmpf5ztGakipIW9jhjrwOT7YtxM1zOqBDGGpU910RkG4h1PZBSKhutGamiaAHWaNlXiMhPWBf+tgOiRORB4H2sMdqKYY0QMMAe0bkh1kjexe3yjPEIs9WSGgLD7XWAdYFrY6AS8IuInMcaceEMMAJrxIviwE/mwojsHbGGT0oh50WzSv3naM1IFUW9gP3GmCP26wisEbDvxRquKNoY0wpr5IUSwEP2dOOA9+wRsedjjfCQhVi335gBfGTXwpoCq4wx7wMxQHd7XLQY4FPgF3tdzYD2ItJFrHtLjQcesJcRn309Sv3XaM1IFSXrRASs+/T0yFQ+PtMIzN2AUmLdLwisZHTMHqetlrHvemuMmS4iJx2sQ4Bz9tiA2Ms9kUs83YDmIvKq/boUUBdrJPLj9tA8YI0q/XGetlQpL6PJSBUlLXLpAHA603MfrJvZrcw8gZ2MCpoPcKM91lvmdTUm7yNZK+XVtJlOqax+Bf7PHqkZEblCRGrYI0VHiUj6SOS34nhQTAMEiEhXezpfEQmz34vHumNv5nUNSu9IISIR9o0OdwLhIpJ+C/gHCnIDlfJEmoyUyuo9YDew3h6FeT5Q2X6vL/A/e/TxG3AwKrWxbg9+G/CiPd0GrJuvAYwCJtojvlcCnsEaKfove9opWLf/OId1x9FxIvIX4IpamVIeRcemU0op5XZaM1JKKeV2moyUUkq5nSYjpZRSbqfJSCmllNtpMlJKKeV2moyUUkq5nSYjpZRSbqfJSCmllNv9PxaATz6jrAMAAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "os.environ[\"CUDA_VISIBLE_DEVICES\"]=\"0\"\n",
    "datasetNames = [\"Trento\"]\n",
    "data2Name = 'LIDAR'\n",
    "\n",
    "patchsize = 11\n",
    "batchsize = 64\n",
    "testSizeNumber = 500\n",
    "EPOCH = 100\n",
    "BandSize = 1\n",
    "LR = 5e-4\n",
    "FM = 16\n",
    "HSIOnly = False\n",
    "FileName = 'MFT_HSILidar_Channel'\n",
    "ntokens = 4\n",
    "token_type = 'channel' \n",
    "num_heads = 8\n",
    "mlp_dim = 512\n",
    "depth = 2\n",
    "train_loss = []\n",
    "\n",
    "def set_seed(seed):\n",
    "    torch.manual_seed(seed)\n",
    "    torch.cuda.manual_seed_all(seed)\n",
    "    np.random.seed(seed)\n",
    "\n",
    "for BandSize in [1]:\n",
    "    for datasetName in datasetNames:\n",
    "            print(\"---------------------------------- Dataset details for \",datasetName,\" ---------------------------------------------\")\n",
    "            print('\\n')\n",
    "            try:\n",
    "                os.makedirs(datasetName)\n",
    "            except FileExistsError:\n",
    "                pass\n",
    "            \n",
    "            train_dataset = Multimodal_Dataset_Train(Filename=datasetName, MM_Data=data2Name)\n",
    "            test_dataset = Multimodal_Dataset_Test(Filename=datasetName, MM_Data=data2Name)\n",
    "            NC = train_dataset.hs_ims.shape[1]\n",
    "            NCLidar = train_dataset.lid_ims.shape[1]\n",
    "            Classes = len(torch.unique(train_dataset.lbs))\n",
    "\n",
    "            train_loader = dataf.DataLoader(train_dataset, batch_size=batchsize, shuffle=True, num_workers= 4)\n",
    "            print(\"HSI Train data shape = \", train_dataset.hs_ims.shape)\n",
    "            print(data2Name + \" Train data shape = \", train_dataset.lid_ims.shape[1])\n",
    "            print(\"Train label shape = \", train_dataset.lbs.shape)\n",
    "\n",
    "            print(\"HSI Test data shape = \", test_dataset.hs_ims.shape)\n",
    "            print(data2Name + \" Test data shape = \", test_dataset.lid_ims.shape[1])\n",
    "            print(\"Test label shape = \", test_dataset.lbs.shape)\n",
    "\n",
    "            print(\"Number of Classes = \", Classes)\n",
    "            \n",
    "            TestPatch1 = test_dataset.hs_ims\n",
    "            TestPatch2 = test_dataset.lid_ims\n",
    "            TestLabel1 = test_dataset.lbs\n",
    "            \n",
    "            KAPPA = []\n",
    "            OA = []\n",
    "            AA = []\n",
    "            ELEMENT_ACC = np.zeros((3, Classes))\n",
    "\n",
    "            set_seed(42)\n",
    "            for iterNum in range(1):\n",
    "                print('\\n')\n",
    "                print(\"---------------------------------- Model Summary ---------------------------------------------\")\n",
    "                print('\\n')\n",
    "                if HSIOnly:    \n",
    "                    model = Transformer(FM=FM, NC=NC, Classes=Classes, ntokens=ntokens, num_heads=num_heads, mlp_dim=mlp_dim, depth=depth).cuda()\n",
    "                    summary(model, [(NC, patchsize**2)])\n",
    "                                        \n",
    "                else:\n",
    "                    model = MFT(FM=FM, NC=NC, NCLidar=NCLidar, Classes=Classes, ntokens=ntokens, token_type=token_type, num_heads=num_heads, mlp_dim=mlp_dim, depth=depth).cuda()\n",
    "                    summary(model, [(NC, patchsize**2),(NCLidar,patchsize**2)])                    \n",
    "                    \n",
    "                optimizer = torch.optim.Adam(model.parameters(), lr=LR,weight_decay=5e-3)\n",
    "                loss_func = nn.CrossEntropyLoss()  # the target label is not one-hotted\n",
    "                scheduler = torch.optim.lr_scheduler.StepLR(optimizer, step_size=50, gamma=0.9)\n",
    "                BestAcc = 0\n",
    "\n",
    "                torch.cuda.synchronize()\n",
    "                print('\\n')\n",
    "                print(\"---------------------------------- Training started for \",datasetName,\" ---------------------------------------------\")\n",
    "                print('\\n')\n",
    "                start = time.time()\n",
    "                # train and test the designed model\n",
    "                for epoch in range(EPOCH):\n",
    "                    for step, (b_x1, b_x2, b_y) in enumerate(train_loader):\n",
    "\n",
    "                        # move train data to GPU\n",
    "                        b_x1 = b_x1.cuda()\n",
    "                        b_y = b_y.cuda()\n",
    "                        if HSIOnly:\n",
    "                            out1 = model(b_x1)\n",
    "                            loss = loss_func(out1, b_y)\n",
    "                        else:\n",
    "                            b_x2 = b_x2.cuda()\n",
    "                            out= model(b_x1, b_x2)\n",
    "                            loss = loss_func(out, b_y)\n",
    "\n",
    "                        optimizer.zero_grad()  # clear gradients for this training step\n",
    "                        loss.backward()  # backpropagation, compute gradients\n",
    "                        optimizer.step()  # apply gradients\n",
    "\n",
    "                        if step % 50 == 0:\n",
    "                            model.eval()\n",
    "                            pred_y = np.empty((len(TestLabel1)), dtype='float32')\n",
    "                            number = len(TestLabel1) // testSizeNumber\n",
    "                            for i in range(number):\n",
    "                                temp = TestPatch1[i * testSizeNumber:(i + 1) * testSizeNumber, :, :]\n",
    "                                temp = temp.cuda()\n",
    "                                temp1 = TestPatch2[i * testSizeNumber:(i + 1) * testSizeNumber, :, :]\n",
    "                                temp1 = temp1.cuda()\n",
    "                                if HSIOnly:\n",
    "                                    temp2 = model(temp)\n",
    "                                    temp3 = torch.max(temp2, 1)[1].squeeze()\n",
    "                                    pred_y[i * testSizeNumber:(i + 1) * testSizeNumber] = temp3.cpu()\n",
    "                                    del temp, temp2, temp3\n",
    "                                else:\n",
    "                                    temp2 = model(temp, temp1)\n",
    "                                    temp3 = torch.max(temp2, 1)[1].squeeze()\n",
    "                                    pred_y[i * testSizeNumber:(i + 1) * testSizeNumber] = temp3.cpu()\n",
    "                                    del temp, temp1, temp2, temp3\n",
    "\n",
    "                            if (i + 1) * testSizeNumber < len(TestLabel1):\n",
    "                                temp = TestPatch1[(i + 1) * testSizeNumber:len(TestLabel1), :, :]\n",
    "                                temp = temp.cuda()\n",
    "                                temp1 = TestPatch2[(i + 1) * testSizeNumber:len(TestLabel1), :, :]\n",
    "                                temp1 = temp1.cuda()\n",
    "                                if HSIOnly:\n",
    "                                    temp2 = model(temp)\n",
    "                                    temp3 = torch.max(temp2, 1)[1].squeeze()\n",
    "                                    pred_y[(i + 1) * testSizeNumber:len(TestLabel1)] = temp3.cpu()\n",
    "                                    del temp, temp2, temp3\n",
    "                                else:\n",
    "                                    temp2 = model(temp, temp1)\n",
    "                                    temp3 = torch.max(temp2, 1)[1].squeeze()\n",
    "                                    pred_y[(i + 1) * testSizeNumber:len(TestLabel1)] = temp3.cpu()\n",
    "                                    del temp, temp1, temp2, temp3\n",
    "\n",
    "                            pred_y = torch.from_numpy(pred_y).long()\n",
    "                            accuracy = torch.sum(pred_y == TestLabel1).type(torch.FloatTensor) / TestLabel1.size(0)\n",
    "\n",
    "                            print('Epoch: ', epoch, '| train loss: %.4f' % loss.data.cpu().numpy(), '| test accuracy: %.4f' % (accuracy*100))\n",
    "                            train_loss.append(loss.data.cpu().numpy())\n",
    "                            # save the parameters in network\n",
    "                            if accuracy > BestAcc:\n",
    "\n",
    "                                BestAcc = accuracy\n",
    "                                \n",
    "                                torch.save(model.state_dict(), datasetName+'/net_params_'+FileName+'.pkl')\n",
    "                                \n",
    "\n",
    "                            model.train()\n",
    "                    scheduler.step()\n",
    "                torch.cuda.synchronize()\n",
    "                end = time.time()\n",
    "                print('\\nThe train time (in seconds) is:', end - start)\n",
    "                Train_time = end - start\n",
    "\n",
    "                # load the saved parameters\n",
    "                \n",
    "                model.load_state_dict(torch.load(datasetName+'/net_params_'+FileName+'.pkl'))\n",
    "\n",
    "                model.eval()\n",
    "                \n",
    "                confusion, oa, each_acc, aa, kappa = reports(TestPatch1,TestPatch2,TestLabel1,datasetName,model, HSIOnly, iterNum)\n",
    "                KAPPA.append(kappa)\n",
    "                OA.append(oa)\n",
    "                AA.append(aa)\n",
    "                ELEMENT_ACC[iterNum, :] = each_acc\n",
    "                torch.save(model, datasetName+'/best_model_'+FileName+'_BandSize'+str(BandSize)+'_Iter'+str(iterNum)+'.pt')\n",
    "                print('\\n')\n",
    "                print(\"Overall Accuracy = \", oa)\n",
    "                print('\\n')\n",
    "            print(\"----------\" + datasetName + \" Training Finished -----------\")\n",
    "            print(\"\\nThe Confusion Matrix on test data\")\n",
    "            record.record_output(OA, AA, KAPPA, ELEMENT_ACC,'./' + datasetName +'/'+FileName+'_BandSize'+str(BandSize)+'_Report_' + datasetName +'.txt')\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "c0ef6f47",
   "metadata": {},
   "outputs": [],
   "source": [
    "train_loss = np.asarray(train_loss)\n",
    "np.save('HSILiDAR_channel_train_loss.npy', train_loss)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "77f5a7ac",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.10"
  },
  "vscode": {
   "interpreter": {
    "hash": "26407125db06bdd9abe40e82cf041582bb19887fa16dd38638e528b7039723e8"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}