chapter5/GCN_Cora.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "toc": true
   },
   "source": [
    "<h1>Table of Contents<span class=\"tocSkip\"></span></h1>\n",
    "<div class=\"toc\"><ul class=\"toc-item\"><li><span><a href=\"#基于Cora数据集的GCN节点分类\" data-toc-modified-id=\"基于Cora数据集的GCN节点分类-1\"><span class=\"toc-item-num\">1&nbsp;&nbsp;</span>基于Cora数据集的GCN节点分类</a></span><ul class=\"toc-item\"><li><span><a href=\"#SetUp\" data-toc-modified-id=\"SetUp-1.1\"><span class=\"toc-item-num\">1.1&nbsp;&nbsp;</span>SetUp</a></span></li><li><span><a href=\"#数据准备\" data-toc-modified-id=\"数据准备-1.2\"><span class=\"toc-item-num\">1.2&nbsp;&nbsp;</span>数据准备</a></span></li><li><span><a href=\"#图卷积层定义\" data-toc-modified-id=\"图卷积层定义-1.3\"><span class=\"toc-item-num\">1.3&nbsp;&nbsp;</span>图卷积层定义</a></span></li><li><span><a href=\"#模型定义\" data-toc-modified-id=\"模型定义-1.4\"><span class=\"toc-item-num\">1.4&nbsp;&nbsp;</span>模型定义</a></span></li><li><span><a href=\"#模型训练\" data-toc-modified-id=\"模型训练-1.5\"><span class=\"toc-item-num\">1.5&nbsp;&nbsp;</span>模型训练</a></span></li></ul></li></ul></div>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 基于Cora数据集的GCN节点分类"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "<table align=\"left\">\n",
    "  <td>\n",
    "    <a target=\"_blank\" href=\"https://colab.research.google.com/github/FighterLYL/GraphNeuralNetwork/blob/master/chapter5/GCN_Cora.ipynb\"><img src=\"https://www.tensorflow.org/images/colab_logo_32px.png\" />Run in Google Colab</a>\n",
    "  </td>\n",
    "</table>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "在Colab中运行时可以通过`代码执行程序->更改运行时类型`选择使用`GPU`"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## SetUp"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import itertools\n",
    "import os\n",
    "import os.path as osp\n",
    "import pickle\n",
    "import urllib\n",
    "from collections import namedtuple\n",
    "\n",
    "import numpy as np\n",
    "import scipy.sparse as sp\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "import torch.nn.init as init\n",
    "import torch.optim as optim\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 数据准备"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "Data = namedtuple('Data', ['x', 'y', 'adjacency',\n",
    "                           'train_mask', 'val_mask', 'test_mask'])\n",
    "\n",
    "\n",
    "def tensor_from_numpy(x, device):\n",
    "    return torch.from_numpy(x).to(device)\n",
    "\n",
    "\n",
    "class CoraData(object):\n",
    "    filenames = [\"ind.cora.{}\".format(name) for name in\n",
    "                 ['x', 'tx', 'allx', 'y', 'ty', 'ally', 'graph', 'test.index']]\n",
    "\n",
    "    def __init__(self, data_root=\"../data/cora\", rebuild=False):\n",
    "        \"\"\"Cora数据，包括数据下载，处理，加载等功能\n",
    "        当数据的缓存文件存在时，将使用缓存文件，否则将下载、进行处理，并缓存到磁盘\n",
    "\n",
    "        处理之后的数据可以通过属性 .data 获得，它将返回一个数据对象，包括如下几部分：\n",
    "            * x: 节点的特征，维度为 2708 * 1433，类型为 np.ndarray\n",
    "            * y: 节点的标签，总共包括7个类别，类型为 np.ndarray\n",
    "            * adjacency: 邻接矩阵，维度为 2708 * 2708，类型为 scipy.sparse.coo.coo_matrix\n",
    "            * train_mask: 训练集掩码向量，维度为 2708，当节点属于训练集时，相应位置为True，否则False\n",
    "            * val_mask: 验证集掩码向量，维度为 2708，当节点属于验证集时，相应位置为True，否则False\n",
    "            * test_mask: 测试集掩码向量，维度为 2708，当节点属于测试集时，相应位置为True，否则False\n",
    "\n",
    "        Args:\n",
    "        -------\n",
    "            data_root: string, optional\n",
    "                存放数据的目录，原始数据路径: ../data/cora\n",
    "                缓存数据路径: {data_root}/ch5_cached.pkl\n",
    "            rebuild: boolean, optional\n",
    "                是否需要重新构建数据集，当设为True时，如果存在缓存数据也会重建数据\n",
    "\n",
    "        \"\"\"\n",
    "        self.data_root = data_root\n",
    "        save_file = osp.join(self.data_root, \"ch5_cached.pkl\")\n",
    "        if osp.exists(save_file) and not rebuild:\n",
    "            print(\"Using Cached file: {}\".format(save_file))\n",
    "            self._data = pickle.load(open(save_file, \"rb\"))\n",
    "        else:\n",
    "            self._data = self.process_data()\n",
    "            with open(save_file, \"wb\") as f:\n",
    "                pickle.dump(self.data, f)\n",
    "            print(\"Cached file: {}\".format(save_file))\n",
    "    \n",
    "    @property\n",
    "    def data(self):\n",
    "        \"\"\"返回Data数据对象，包括x, y, adjacency, train_mask, val_mask, test_mask\"\"\"\n",
    "        return self._data\n",
    "\n",
    "    def process_data(self):\n",
    "        \"\"\"\n",
    "        处理数据，得到节点特征和标签，邻接矩阵，训练集、验证集以及测试集\n",
    "        引用自：https://github.com/rusty1s/pytorch_geometric\n",
    "        \"\"\"\n",
    "        print(\"Process data ...\")\n",
    "        _, tx, allx, y, ty, ally, graph, test_index = [self.read_data(\n",
    "            osp.join(self.data_root, name)) for name in self.filenames]\n",
    "        train_index = np.arange(y.shape[0])\n",
    "        val_index = np.arange(y.shape[0], y.shape[0] + 500)\n",
    "        sorted_test_index = sorted(test_index)\n",
    "\n",
    "        x = np.concatenate((allx, tx), axis=0)\n",
    "        y = np.concatenate((ally, ty), axis=0).argmax(axis=1)\n",
    "\n",
    "        x[test_index] = x[sorted_test_index]\n",
    "        y[test_index] = y[sorted_test_index]\n",
    "        num_nodes = x.shape[0]\n",
    "\n",
    "        train_mask = np.zeros(num_nodes, dtype=np.bool)\n",
    "        val_mask = np.zeros(num_nodes, dtype=np.bool)\n",
    "        test_mask = np.zeros(num_nodes, dtype=np.bool)\n",
    "        train_mask[train_index] = True\n",
    "        val_mask[val_index] = True\n",
    "        test_mask[test_index] = True\n",
    "        adjacency = self.build_adjacency(graph)\n",
    "        print(\"Node's feature shape: \", x.shape)\n",
    "        print(\"Node's label shape: \", y.shape)\n",
    "        print(\"Adjacency's shape: \", adjacency.shape)\n",
    "        print(\"Number of training nodes: \", train_mask.sum())\n",
    "        print(\"Number of validation nodes: \", val_mask.sum())\n",
    "        print(\"Number of test nodes: \", test_mask.sum())\n",
    "\n",
    "        return Data(x=x, y=y, adjacency=adjacency,\n",
    "                    train_mask=train_mask, val_mask=val_mask, test_mask=test_mask)\n",
    "\n",
    "    @staticmethod\n",
    "    def build_adjacency(adj_dict):\n",
    "        \"\"\"根据邻接表创建邻接矩阵\"\"\"\n",
    "        edge_index = []\n",
    "        num_nodes = len(adj_dict)\n",
    "        for src, dst in adj_dict.items():\n",
    "            edge_index.extend([src, v] for v in dst)\n",
    "            edge_index.extend([v, src] for v in dst)\n",
    "        # 去除重复的边\n",
    "        edge_index = list(k for k, _ in itertools.groupby(sorted(edge_index)))\n",
    "        edge_index = np.asarray(edge_index)\n",
    "        adjacency = sp.coo_matrix((np.ones(len(edge_index)), \n",
    "                                   (edge_index[:, 0], edge_index[:, 1])),\n",
    "                    shape=(num_nodes, num_nodes), dtype=\"float32\")\n",
    "        return adjacency\n",
    "\n",
    "    @staticmethod\n",
    "    def read_data(path):\n",
    "        \"\"\"使用不同的方式读取原始数据以进一步处理\"\"\"\n",
    "        name = osp.basename(path)\n",
    "        if name == \"ind.cora.test.index\":\n",
    "            out = np.genfromtxt(path, dtype=\"int64\")\n",
    "            return out\n",
    "        else:\n",
    "            out = pickle.load(open(path, \"rb\"), encoding=\"latin1\")\n",
    "            out = out.toarray() if hasattr(out, \"toarray\") else out\n",
    "            return out\n",
    "\n",
    "    @staticmethod\n",
    "    def normalization(adjacency):\n",
    "        \"\"\"计算 L=D^-0.5 * (A+I) * D^-0.5\"\"\"\n",
    "        adjacency += sp.eye(adjacency.shape[0])    # 增加自连接\n",
    "        degree = np.array(adjacency.sum(1))\n",
    "        d_hat = sp.diags(np.power(degree, -0.5).flatten())\n",
    "        return d_hat.dot(adjacency).dot(d_hat).tocoo()\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 图卷积层定义"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "class GraphConvolution(nn.Module):\n",
    "    def __init__(self, input_dim, output_dim, use_bias=True):\n",
    "        \"\"\"图卷积：L*X*\\theta\n",
    "\n",
    "        Args:\n",
    "        ----------\n",
    "            input_dim: int\n",
    "                节点输入特征的维度\n",
    "            output_dim: int\n",
    "                输出特征维度\n",
    "            use_bias : bool, optional\n",
    "                是否使用偏置\n",
    "        \"\"\"\n",
    "        super(GraphConvolution, self).__init__()\n",
    "        self.input_dim = input_dim\n",
    "        self.output_dim = output_dim\n",
    "        self.use_bias = use_bias\n",
    "        self.weight = nn.Parameter(torch.Tensor(input_dim, output_dim))\n",
    "        if self.use_bias:\n",
    "            self.bias = nn.Parameter(torch.Tensor(output_dim))\n",
    "        else:\n",
    "            self.register_parameter('bias', None)\n",
    "        self.reset_parameters()\n",
    "\n",
    "    def reset_parameters(self):\n",
    "        init.kaiming_uniform_(self.weight)\n",
    "        if self.use_bias:\n",
    "            init.zeros_(self.bias)\n",
    "\n",
    "    def forward(self, adjacency, input_feature):\n",
    "        \"\"\"邻接矩阵是稀疏矩阵，因此在计算时使用稀疏矩阵乘法\n",
    "    \n",
    "        Args: \n",
    "        -------\n",
    "            adjacency: torch.sparse.FloatTensor\n",
    "                邻接矩阵\n",
    "            input_feature: torch.Tensor\n",
    "                输入特征\n",
    "        \"\"\"\n",
    "        support = torch.mm(input_feature, self.weight)\n",
    "        output = torch.sparse.mm(adjacency, support)\n",
    "        if self.use_bias:\n",
    "            output += self.bias\n",
    "        return output\n",
    "\n",
    "    def __repr__(self):\n",
    "        return self.__class__.__name__ + ' (' \\\n",
    "            + str(self.input_dim) + ' -> ' \\\n",
    "            + str(self.output_dim) + ')'\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 模型定义\n",
    "\n",
    "读者可以自己对GCN模型结构进行修改和实验"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "class GcnNet(nn.Module):\n",
    "    \"\"\"\n",
    "    定义一个包含两层GraphConvolution的模型\n",
    "    \"\"\"\n",
    "    def __init__(self, input_dim=1433):\n",
    "        super(GcnNet, self).__init__()\n",
    "        self.gcn1 = GraphConvolution(input_dim, 16)\n",
    "        self.gcn2 = GraphConvolution(16, 7)\n",
    "    \n",
    "    def forward(self, adjacency, feature):\n",
    "        h = F.relu(self.gcn1(adjacency, feature))\n",
    "        logits = self.gcn2(adjacency, h)\n",
    "        return logits\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 模型训练"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 超参数定义\n",
    "LEARNING_RATE = 0.1\n",
    "WEIGHT_DACAY = 5e-4\n",
    "EPOCHS = 200\n",
    "DEVICE = \"cuda\" if torch.cuda.is_available() else \"cpu\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Process data ...\n",
      "Node's feature shape:  (2708, 1433)\n",
      "Node's label shape:  (2708,)\n",
      "Adjacency's shape:  (2708, 2708)\n",
      "Number of training nodes:  140\n",
      "Number of validation nodes:  500\n",
      "Number of test nodes:  1000\n",
      "Cached file: cora/processed_cora.pkl\n"
     ]
    }
   ],
   "source": [
    "# 加载数据，并转换为torch.Tensor\n",
    "dataset = CoraData().data\n",
    "node_feature = dataset.x / dataset.x.sum(1, keepdims=True)  # 归一化数据，使得每一行和为1\n",
    "tensor_x = tensor_from_numpy(node_feature, DEVICE)\n",
    "tensor_y = tensor_from_numpy(dataset.y, DEVICE)\n",
    "tensor_train_mask = tensor_from_numpy(dataset.train_mask, DEVICE)\n",
    "tensor_val_mask = tensor_from_numpy(dataset.val_mask, DEVICE)\n",
    "tensor_test_mask = tensor_from_numpy(dataset.test_mask, DEVICE)\n",
    "normalize_adjacency = CoraData.normalization(dataset.adjacency)   # 规范化邻接矩阵\n",
    "\n",
    "num_nodes, input_dim = node_feature.shape\n",
    "indices = torch.from_numpy(np.asarray([normalize_adjacency.row, \n",
    "                                       normalize_adjacency.col]).astype('int64')).long()\n",
    "values = torch.from_numpy(normalize_adjacency.data.astype(np.float32))\n",
    "tensor_adjacency = torch.sparse.FloatTensor(indices, values, \n",
    "                                            (num_nodes, num_nodes)).to(DEVICE)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "# 模型定义：Model, Loss, Optimizer\n",
    "model = GcnNet(input_dim).to(DEVICE)\n",
    "criterion = nn.CrossEntropyLoss().to(DEVICE)\n",
    "optimizer = optim.Adam(model.parameters(), \n",
    "                       lr=LEARNING_RATE, \n",
    "                       weight_decay=WEIGHT_DACAY)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 训练主体函数\n",
    "def train():\n",
    "    loss_history = []\n",
    "    val_acc_history = []\n",
    "    model.train()\n",
    "    train_y = tensor_y[tensor_train_mask]\n",
    "    for epoch in range(EPOCHS):\n",
    "        logits = model(tensor_adjacency, tensor_x)  # 前向传播\n",
    "        train_mask_logits = logits[tensor_train_mask]   # 只选择训练节点进行监督\n",
    "        loss = criterion(train_mask_logits, train_y)    # 计算损失值\n",
    "        optimizer.zero_grad()\n",
    "        loss.backward()     # 反向传播计算参数的梯度\n",
    "        optimizer.step()    # 使用优化方法进行梯度更新\n",
    "        train_acc, _, _ = test(tensor_train_mask)     # 计算当前模型训练集上的准确率\n",
    "        val_acc, _, _ = test(tensor_val_mask)     # 计算当前模型在验证集上的准确率\n",
    "        # 记录训练过程中损失值和准确率的变化，用于画图\n",
    "        loss_history.append(loss.item())\n",
    "        val_acc_history.append(val_acc.item())\n",
    "        print(\"Epoch {:03d}: Loss {:.4f}, TrainAcc {:.4}, ValAcc {:.4f}\".format(\n",
    "            epoch, loss.item(), train_acc.item(), val_acc.item()))\n",
    "    \n",
    "    return loss_history, val_acc_history\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 测试函数\n",
    "def test(mask):\n",
    "    model.eval()\n",
    "    with torch.no_grad():\n",
    "        logits = model(tensor_adjacency, tensor_x)\n",
    "        test_mask_logits = logits[mask]\n",
    "        predict_y = test_mask_logits.max(1)[1]\n",
    "        accuarcy = torch.eq(predict_y, tensor_y[mask]).float().mean()\n",
    "    return accuarcy, test_mask_logits.cpu().numpy(), tensor_y[mask].cpu().numpy()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "def plot_loss_with_acc(loss_history, val_acc_history):\n",
    "    fig = plt.figure()\n",
    "    ax1 = fig.add_subplot(111)\n",
    "    ax1.plot(range(len(loss_history)), loss_history,\n",
    "             c=np.array([255, 71, 90]) / 255.)\n",
    "    plt.ylabel('Loss')\n",
    "    \n",
    "    ax2 = fig.add_subplot(111, sharex=ax1, frameon=False)\n",
    "    ax2.plot(range(len(val_acc_history)), val_acc_history,\n",
    "             c=np.array([79, 179, 255]) / 255.)\n",
    "    ax2.yaxis.tick_right()\n",
    "    ax2.yaxis.set_label_position(\"right\")\n",
    "    plt.ylabel('ValAcc')\n",
    "    \n",
    "    plt.xlabel('Epoch')\n",
    "    plt.title('Training Loss & Validation Accuracy')\n",
    "    plt.show()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "loss, val_acc = train()\n",
    "test_acc, test_logits, test_label = test(tensor_test_mask)\n",
    "print(\"Test accuarcy: \", test_acc.item())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAa0AAAEWCAYAAADVW8iBAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nO3deZhcZZX48e/pLZ2l00mnOzvZIOxrCDsKqGyjgCgqQRRGkdEfjIqOI6CAA44bo+LCiFERGVkV0YDsmwhhSYAACRAICVnI1t1JOlvvdX5/nLfom0pVd3V33aru9Pk8Tz1Vde+tW6duVd1z3/e+931FVXHOOef6g6JCB+Ccc85ly5OWc865fsOTlnPOuX7Dk5Zzzrl+w5OWc865fsOTlnPOuX7Dk5YDQESKRWSriEzK5bIuMxEpEREVkSnh+W9F5PJslu3Be50nIvf3NFbn+gpPWv1USBrJW0JEGiPPP93d9alqu6oOU9UVuVy2u0TkuyJyU67Xm+V7i4hcJyL14XZ7F8v/TkRuTDP9UBFpEpER3Xl/Vb1AVb/X3bjTvP8eIrLDBZiq+gdVPbW36+7qPUXk53G9h3PgSavfCkljmKoOA1YAp0Wm3ZK6vIiU5D/KfudfgE8BBwITgN92sfxNwFkiMjhl+meAv6nqppxH2HedB2wAZolIaT7f2H/bA4snrV1UKLHcISK3icgW4FwROUpEnhWRTSKyRkR+ntzBpKmq+mOYf7+IbBGRZ0RkaneXDfNPFZE3RaRBRH4hIk+LyPk9+Ez7icg/QvyvisiHI/M+IiKvh/dfJSKXhOmjReS+8JoNIvJkJ2/RBmwH1qpqk6o+0kVITwG1wJmROEqAWcAfwvOM2zzN5/ujiHwn8vxSEVkrIu9iSSG67OkisiB83hUickVk9pNhmWTJ+zARuUBEnoi8/lgRmR++k+dF5IjIvKdE5L9EZG5Y/wMiUpVpI4iIYIn6MkCAD6fMP0BEHgnbf62I/GdyW4nIFSLytohsDvGMT1dSDDGdHx5fICJPhm25Afi2iEwXkcdDCblORP5PRCojr58sIn8Vkdow/2ciUh6+l30iy40Tke0iMirT53WF5Ulr13YmcCtQCdyB7ZS/AlQDxwCnAP/WyevPAa4AqrDS3DXdXVZERgN3At8I77sMOLy7H0REyoB7gb8DNcAlwB0iskdY5PfA51W1Aisp/SNM/wawNLxmbIgxk0XAGODXYUfcKbU+0G4GPhuZfDKgwEPheXe3OWBJOLzuA8CeYb1RW4Fzse/2NOAr4TUA7w/xJUve81LWXY1txx8Do4CfA/eJyMjIYudgiXIMMBT4WifhHh+WuwP4E5HtERLHI8A9wLjwWZ4Is78BnIVtkxHABUBTJ+8TdTTwOva9/hBLlt8N77EvMI3wXYcDib8DS4ApwG7AnarahP02z0353A+qan2Wcbg886S1a3tKVe9R1YSqNqrqPFV9TlXbVHUpMBs4rpPX/1lV56tqK3ALcHAPlv0IsEBV/xbm/RSo68FnOQYoA65V1dZQCrofODvMbwX2FZEKVd2gqi9Gpo8HJqlqi6r+Y6c1815SfBD4ArbjuyGZuETkORHJdD7oZuCDIjIuPP8scIuqtgH0YJsnfRL4naq+pqrbgO9EZ6rqY6q6MHy3LwO3Z7lesCS3SFVvC3H9EUvs0RLS71T1LVXdjiWizr7784C/q2oDdpD04UhJ5XRgpar+TFWbVXWzqj4f5l0AXB7eJ6GqC1R1Q5afYYWq/iqcX21U1TdV9dHwHa/HfmfJ7XEUdtDwTVXdFpZ/Osz7A3BO5CDlM8D/ZRmDKwBPWru2ldEnIrK3iPw9VNFsBq7G/syZrI083g4M68Gy46NxhNLJqixiTzUe21FFq42WY+eewEqVpwMrROSJSHXXD8Jyj4ZqqG9kWP+JwGBVvQ07+t8HS1wjsKP2p9O9SFWXAXOBT4vI8BDDzcn5Pdjm0c8b/f6WR2eGascnQnVXA5YAsllvct3LU6ZFtyVk+d2LyFDg49iBCliV6RqsihSsVLMkQxy7AW9nGXOq1N/2WBG5U0TeDdv5Jjq2x27AO6ranrqSkLzagGNFZH9gElYqc32UJ61dW2oX/r8GFgJ7qOpw4EqsWiVOa4CJySfhiHZC5sUzWg3sllJtNwl4FyCUZk4HRmPViLeH6ZtV9RJVnQJ8FPimiKQrkZRgOy9UtRErjRwGPAf8VlU3dxLbH7AS1ieAxaHkk9TTbb4G29lGP2vU7cBdwG6qWok1Gkmut6uhG1YDk1Omvbctu+njWEKbLSJrQ9xj6agiXAnsnuG1meZtAxCRIZFpY1OWSf2MPwSagQPCdj6fju2xEpgsIsUZ4rgZqyL8DFZt2JxhOdcHeNIaWCqABmBbOPnc5bmVHLgXmCEip4VzC1/BzkN0pjicJE/eBmGlmTbg6yJSKiIfwFr73Skig0XkHBEZHqogtwDtAOF9dw/JriFM3+mIG2u8MFxErpKO1oBPYOdgEl3E+yds53sFoQFGRE+3+Z3A50JJbShwVZr1blDVJhE5ko5qUoD1gIrItAzrvhfYT0Q+FRpDnAPsAdyXZWxR5wG/AQ7AqhAPxs6pzQyfdw4wSUQuFpEyERkuIslzmr8Fvpv8fkTkYLEGH2vD7VyxawIvZOckm6oCS3YNIrIb8B+Rec8A9cD3RGRI+L0cE5n/f1jp+hwipWTXN3nSGli+ju1ktmAlgDvifkNVXYc1I/8JtuPYHXgJOyrO5FygMXJbHI5+TwPOwM6J/Rw4R1XfDK85D1geqoY+jx01A+wFPIY1XHga+JmqPpUmzo3AScD7sJLIy0ApcCjwbyLyr518xi3A3VgJ8taU2T3a5qp6D3A91qDkTeDhlEW+BHxfrGXo5ViSi8bzfeC50DpuZsq6a7FqzG9i38klwEe6cT4JALGLy48HrlPVtZHb81jji/PCea4TsRLZ+vBZkiXda4G/Ao8Cm7HzfeWhCvgL4XPVYQn1uS7CuQpr4NOAJcq7Ip+3DTu3ug9W6lqBJank/HeAV4EWVZ3bnW3g8k98EEiXT6GKZjVwlqr+s9DxOAcgIjcDS1X1O4WOxXXOL8pzsRORU7AqmibsWp424PlOX+RcnoRq1DOwKk7Xx3n1oMuHY7Em1XXYNTkf9ZPdri8Qke9jVcHfi6NbMpd7Xj3onHOu3/CSlnPOuX5jlzqnVV1drVOmTCl0GM4512+88MILdara1WUofUZsSStcK3EzdlFgApitqj9LWUaAn2HX22wHzk92vyMi5wHfDot+V1VTr3/ZyZQpU5g/f37uPoRzzu3iRCS1d5Q+Lc6SVhvwdVV9UUQqgBdE5GFVfS2yzKnA9HA7AvgVcES4wPAqYCZ25fsLIjInXEvjnHNugIrtnJaqrkmWmsLFjq+zc/c9ZwA3q3kWGBE6Hj0ZeDh0fLoRu7DylLhidc451z/kpSGG2LhLh7DzVe0T2LHjy1VhWqbp6dZ9odg4PPNra2tzFbJzzrk+KPakJSLDsC5Vvpqm09F0HYdqJ9N3nqg6W1VnqurMmpp+cy7ROedcD8SatMRGaL0LG1/oL2kWWcWOPVlPxLr4yTTdOefcABZb0gotA38HvK6qP8mw2Bzgs6GH5yOBBlVdgw3Gd5KIjAyjqZ4UpjnnnBvA4mw9eAzW0/arIrIgTLucMC6Qqt6ADYXwL9ggcduBfw3zNojINUBymPCru9sDtXPOuV1PbEkrDP/Q6WB3YQiCizLMuxG4MYbQdtTSAnMehmmTYcb+sb9df/TEKpgxGoaXFToS51whhE6vfwYUY4Oi/iBl/iRsLLkRYZlLVbUn47N1ybtxKimBP98LDz5e6Ej6pFVb4boF8LelHdNqG2HJpsLF1J88uhKueAbaE9DYBm/1crv9ZQn86AXwLkNdvoThhK7HrqvdF5glIvumLPZtbNTnQ7ABSf83rng8aRUVweGHwPxXoK0tp6tubIPrX4bFaS6J3tICrenGzy2gDU1239QGP34RVm6BRfU27ZU6SChc+wJc+Cj859Owqdl2nltbdlxPY1vPPpsqPLAcblyUead802vwj8ig8K/UwU9e3DmG7nhtg31PCU3/eXpKFe5aAq/Ww3Nr4devwjef7tjO7QmYvRD++ra9d6o3NsL358EbkYrxB1fA3DUwb33HtK0thUli7Wq/Y5cfjW07bu/GNvifF+Gd1DbZuXc4sERVl6pqC3A7do1tlALDw+NKYmw4t0v1PdhjR86Ah/4BixbDQfvlZJXtCftBvbAe1jfCfx3ZMW/lFrhsLowYBN+YAZOH247soRVw2lQYWpp+nZtbYM5SeN94e81776V2BD6qHI4aB4Oz/Fa3tsL978Cpk2HZZrjyWfiPGdDUDv9cDRWlsC3k8bc3wbNr4ek1cORYe/xSrf1xfrcILj8MDh0N67bDpU/D+KFwzVFQFKkgnrsG1myD90+AmsH2ef6yxOIA206v1NnjD02y+3uWAQonTYYhJfDXpVBWBNMrLcY73rQ+wvYYAadPs+2eUCgpAkmpnFaFtkTH8+Iii++ON+HlOvjgbrB0s32ey2bCzDEdr0u3rqdWW6lzxmhYWA+lRRZ3cVh28UZYvc2ODP+42D67Ytvu1Mlww0J4OAyG8Wo9/OehMKjY1vnQCts27Qrz18O/7Q8H1dj2BbjlDZg52tZ5yZP2u/nMPtl97/94F16tg8oyOGu6vcffl8Epk6FyUObXRbdDbSP8zwuwfAv86gQYWW7bVhVKi3d+bbvCy7W2nT62Owwr6/iukt9D1OKNMH8dHDMepgy3g6A5y2D/UbDXSHs+bz0sbYCz9oDtbeG3PMXe6+EVsOcIOKTG1v9yHazaAqdMse+nNQF3vw0zauy3E9XcbgcZa7fDh6dAfRM8+a79vra1wiMrbXpVuf1v/7kahpXCBybu/DvJRnsC/v4O7F4J+42Cx1dZcjp2vP0HS0LR4urn4M1N9r1/YX+L46nVdvB4zZE9e+8spbtu9oiUZb4DPCQi/w4MBT4UVzC71NAkM2fO1B71PdjUBJ/4InzkQ/Bv5+YkllsWw5/esp3rkgaY/UHbUW9sslJKa9h5bm+Fz+1nf7jlW+DAarjicNsBJrUrvLjejtTrmmyn/f8OhOMn2k5i9kK4P/QeVlEKXzm4Y4fbloCb34DHVsJB1bZjPqjG/rg/fcl2YMdNsJ3rW5tgj0ooK7bSx8hB9odRtfetGmQJ7fcnwhcfg32qLAGv3ArlxbYDfHylJZ/WBFywH3xkqsXx/Dr4wTxLMIJ9zne3wsZmS95gO67jJ8Cfl8Anp1s8r9TbTn/sUPuz/vVtKC+xnV1Tu8W+ciskEnDB/vDd52161SBLjsl1N7ZZwl21tWO7ThgK/3EofO1JSyanTYUFtR2f5z9m2I7pJy/ZDv7sPeEPr1uyHT/Mlk21z0hLwgmFUYNt53vWHvZ7GFJi5wVHldvOePZC+Pge9vw3C+GwMTB1uP1uEsDR4+D8feAXL9u2OHtPuOl12zZ3vgWf29dKY3PX2Da64gjbiW5psZ3ojNG23NOr7fGZu9t2uHyuzd/aaomqsQ2eeNd28l852Nb32Epb5nvH2G9x7mr49UI4Zy84Yix89UkrkTe3WzzVg+GGV+23OqMGztgdfr/I5h9QbQdv9aGEedwE+MBu8IP59t6VZXD4WCtRJtR+R79ZaIkIbGeuagcUQ0psh33rYkucYL+LukZ4Z4t93+0J2BIOhKaPgK8eDN94ytZ3UDX8675W3f34KigR+68sqrdt9L7xcP0r9rsE+882NENLwr6n7W0dMe8xwg7ckqXk/UfZ9t/eZgmnunzn30drwhLixmb44gG2zt8usv/b8DK46EDbLtG98sE18KHd7CB4ZjhAGj3EPn9ZETS0wFVHWILuidD3YF1k0mxVnR2Z/wngZFW9IDz/DHC4qv57ZJmvYfnkxyJyFNZyfH9VjRwm5oYnraQrfgTvroXf/bjXhywbmmynfsRYOHdv+LfH7M/+iT3gu/PsKPd7R9sP9scv2Y+wWOzo+9534KRJlpRuXWxJZXur/QnHDIEL97cjxEX18LVDLCH+bSl8dJr98X+z0EpNXzrAfujfec6O4mfU2I5vS6u970HV8NgqmDisY0e+XxUsClVRu1fC2w32+Px94LY3bQd0ymT7s/3yZfvTtyvM2tPiXL3NSnnfPsyqxV6psx032A5rSgV8+WDbiT62yv5wlxyy85Hut+baujY227onDLM/bLHYH/jY8fY5z98HTpxkVYq/Xmh/+iEllpgXb7REH/3H7D2y48i7tR3+tMR2zJtbYFolrN5qCW/WnhZfslQzcpDtqJra7T2mDrf1f2wP28m9UmfrXrzRksqkCvsNrG+EEybad3bR47Yj395qiaSs2Lb3FYfbz+3vy+A3i+z9TphoCWrMEHu+tAG+9k8rhQ0pgd99CH443w4EFDsweGZNR1JIdehoeH2D7TArSu3z//w4+33ds8yWmVFjO+Dk3mB6JbzVAB/f3b73e9+x7wuspPPGRrj2WPjjG/a5WxK2XfatsnW2JuzAYfwwe+9DaixRLWuw7V5WZAci7xtvv7P562D3EVZqWLfdtvm3D4fX6uHRVXawd85etu3qm2D0YNuua7fbTr9Y7CDp/uWWiL56CLy5Ef73lY6Syiem2wFBS/hRfGx3O3B6td7iTv5eJgy1dZcU2YHduKGW8H+zEIYPsu/m5nDwcnxIwC+st//ItOH2m5q/3v4b6UypsO28fIs9Ly+2g5c73oQ2tUT5n4faf3xjs/2/kwduvzgOFm6Aa56zZPnj91sV8rAy+J9jdy6xZkNEXlDVmZ3MPwr4jqqeHJ5fBqCq348sswg4RVVXhudLgSNVdX2aVfaKVw8mHXYwzPsD1NbD6OpererOt+wHe85etuM5YBQ8uNyOTF9YD5/ft2NH/V9Hwr3L7I9x+Bjbmf3l7Y6qsP2qYOwoOLTGjsRLi+2I7qpnLeGBJZLP7mM/2B8cY0dqv1tkdd2v1lsCO3lyqFJZZzuBJ1bZn+f7x1hVZULhW4fBhY/ZjvWSQ6zaqTVhpaJX6uDFWqsCATvie2Sl7Xw+MtWOjlva7Y9eWmTVg3e+ZTt7sM/zyT3tSHjSXvbHz3RscPQ424GXFlncw8tsZ7B0M3xwIhw93o7Wk3/Q942HG1+zEsY3D7UqFrB4kjuOIrGdflS7Wqlu/1GW6H62oOPzfHR3OyJescWeb2mxz3vaVCtVRKvKxg21+8nDLYmK2Hmmvy2zA4fBIdGIwPLNcMdbFs+XDuxYx4enWjIZXGKfLWpaZccBxVFj7bVfmwFXPmM77Vl72jZ4di0cM86S/Opt8OwaKw3PGG2lhZ8usCq67xxh73Pu3vZ7LCuCyw6zz/vuVnv/sUNth33X2xbDaVNtO1zypP2mPrO3VdudPs2qlQeXwNdn2A732PFWnfbR3e37jm6rw8bYDn1zC1x5uG1L6Fhma6sdlL0/VIHvXgmnTeuYv2+VlQrPnGY7arDfSVW5rfvkyVaSF7GDh3aFX73aUeo/aZLF1q5wxjRbLrnuhfW2fT62R0cV++wPdvzOrj+hY92H1Oz4uSYMs22UfN7abgkonfJi+1/du8xqDd43HirCZ7ltsZW2po+wG9g2/MPr9n0VF9kB5+WHWYKfOtymv91gtSplaapmc2AeMF1EpgLvYg0tzklZZgXwQeAmEdkHKAfi6VdPVXeZ26GHHqo9tmix6snnqD7zQo9XkUioznlb9cx7VX/1Ssf0V2pVP/ug6hn3qF76lGp7IvM6mttUL3rclr34cdWWtvTLNTSr/nC+6hMrd55Xt1111v22ju89b3Gl2tikurWl4z23t9rjh5ar/vENe/z9eaqffkC1LaE6f519puS6treqfuLvqj95sbMt0jP1japn3qP6iwUd017fYJ830/b481uqd73VvfdpaVP98QuqL9eqbmlRPSumzxOVSKhe/7Lq06u797pn19j3+Vjk+25tV93cnP062hO2baO2t6o2Zdimm8NvbG4k1ufWWPxt7fY8kVCd/apNz1Zja8fvLR82NHa9TF+xsal703MBmK9d7Fux62nfBN4GvhWmXQ2cHh7vCzwNvAwsAE7qap09vXn1YNL2RvjYBfDZs+CcM7v10k3Ndl5kc6sd7Rw2Bi45GIZEGlS0JexIburwzk92g1XjXf8yXHRQx9FWdz212o5Yv3WYHYX2xKZmu00Znn7+kk1Wtx7H9VtvbIRJw3bchnGL8/P0lqqd39u/yo62ncuVrqoH+xqvHkwaMhjGj4GlK7r90rlrYEOztSiaVpm+FVFJkZ2Pycb0EXDdcd0OYwfHjrdbb4wY1NGQIZ3Uc1G5tPfI+NadSZyfp7dErFrIuYHOk1bUtEmwrPtJ65k11qDhgv1ibXbqnHMDnlc0RE2dDKvXQWOGZlhpbGq2Vj5Hj/OE5ZxzcfOkFTVtkp08WLay62WD59Z2XFPjnHMuXp60oqaFbhiWLc9q8YRaU/YJQ2FyRYxxOeecAzxp7Wh0NQwbknVjjLlr7Nqhs6Z71aBzzuWDJ60oEZg4znrG6EJ7wnoUmFRh3QU555yLnyetVOPGwuquk9biTdbrwCf26Ogg1TnnXLw8aaWaMAZqN9jgkJ1IDgewd1UeYnLOOQd40trZ+LHWgnBN5/08rthi/eml68nZOedcPGJLWiJyo4isF5GFGeZ/Q0QWhNtCEWkXkaow7x0ReTXM62G/TD00Yazdd1FFuHyzdejpDTCccy5/4ixp3QSckmmmql6rqger6sHAZcA/VDUyRisnhPn57RNrfDJprcu4iKoNK+DN3J1zLr9iS1qq+iSwocsFzSzgtrhi6ZaKoVAxrNMWhLWNNtBbpo5knXPOxaPg57REZAhWIrsrMlmxoZtfEJELu3j9hSIyX0Tm19bmaPiWCWM6LWklB2/zkpZzzuVXwZMWcBrwdErV4DGqOgM4FbhIRN6f6cWqOltVZ6rqzJqaHo43naqLZu/JpDXJk5ZzzuVVX0haZ5NSNaiqq8P9euBu4PC8RtRFs/flm22E1qF5HOvJOedcgZOWiFQCxwF/i0wbKiIVycfASUDaFoix6aLZe20jjB2S14icc84R43haInIbcDxQLSKrgKuAUgBVvSEsdibwkKpui7x0DHC3WFvyEuBWVX0grjjTGhNG21tfB5Mn7jR7c4sN9uiccy6/Yktaqjori2VuwprGR6ctBQ6KJ6osjR1t9+vq0s5uaOmbQ7I759yuri+c0+p7RlZCaQms27k1YlsCtrVCpSct55zLO09a6RQV2TAlaZLW5tA2w0tazrmBQkROEZHFIrJERC5NM/+nkR6O3hSRTXHFElv1YL83pjpt9WAyaXlJyzk3EIhIMXA9cCKwCpgnInNU9bXkMqp6SWT5fwcOiSseL2llMqYmbUmrIVnSGpTneJxzrjAOB5ao6lJVbQFuB87oZPlYezjypJXJmBrYtBmamneYvDk89ZKWc26AmACsjDxfFabtREQmA1OBx+IKxpNWJtFm7xENfk7LObdrqU52hRduqV3npRvLQjOs62zgz6rantsQO/g5rUzGhC6h1tXCpI6DioYW+waHedJyzu0a6roYTWMVsFvk+URgdYZlzwYuylVg6XhJK5P3ktaOJa3NzVbKKvZxtJxzA8M8YLqITBWRMiwxzUldSET2AkYCz8QZjCetTEZWQmnpTo0x/MJi59xAoqptwMXAg8DrwJ2qukhErhaR0yOLzgJuV9VMVYc54dWDmRQVwehR7yWtG16F0iJr8u5Jyzk3kKjqfcB9KdOuTHn+nXzE4kmrMzWjoM5GTHljA9Q1WcLywR+dc64wvHqwM9VVNkQJ0NQOW1th9TYvaTnnXKF40upMdRXUb4T2BE2RBpx+jZZzzhWGJ63O1FRBIgGbGmhq65jsJS3nnCsMT1qdqa4CIFG7geb2jo1V6V04OedcQXjS6kxIWi21m1Bg/9BJRs3gwoXknHMDmbce7Ez1KACa6hugBI4cC+ftA9O89aBzzhWEl7Q6M3wYlJXStGELAINLYPdKEO8NwznnCiK2pCUiN4rIehFZmGH+8SLSEBk47MrIvE4HHMsbEaiuoqlhOwDlxQWLxDnnHPGWtG4CTulimX+q6sHhdjXsMODYqcC+wCwR2TfGODtXXUXT5kbAk5ZzzhVabElLVZ8ENvTgpd0dcCxe1VU0bW0CoNzPADrnXEEV+pzWUSLysojcLyL7hWlZDzgGICIXJseBqa3deaThXquuoqmxFfCk5ZxzhVbIpPUiMFlVDwJ+Afw1TO/OgGOo6mxVnamqM2tqanIfZXUVjcV2NbFXDzrnXGEVLGmp6mZV3Roe3weUikg13RtwLH41VTSX2NXEnrScc66wCpa0RGSsiDUeF5HDQyz1ZDngWN5UjaSpLCQtrx50zrmCim03LCK3AccD1SKyCrgKKAVQ1RuAs4AviUgb0AicHQYPaxOR5IBjxcCNqroorji7NLKSxtJyAAZ5Scs55woqtqSlqrO6mP9L4JcZ5u004FjBjBxBc8kgBmkbReJFLeecK6RCtx7s+0pLaBpaQXmitdCROOfcgOdJKwuNQysob2spdBjOOTfgedLKQvPgoZS3NBY6DOecG/A8aWWhcdAQyps9aTnnBqZs+oMVkU+KyGsiskhEbo0rFm9ZkIWmsnLKN60DVe/i3Tk3oET6gz0Ru452nojMUdXXIstMBy4DjlHVjSIyOq54vKSVheaSQVbS2rq90KE451y+ZdMf7BeA61V1I4Cqro8rGE9aWWgsLqO8tRk2bCx0KM45l2/Z9Ae7J7CniDwtIs+KSFcjfPSYVw9moUlKKG9rho0NMHliocNxzrlcqhaR+ZHns1V1duR5Nv3BlgDTsQ4lJgL/FJH9VXVTTiPFk1ZWmimivLUJNuR8+zvnXKHVqerMTuZn0x/sKuBZVW0FlonIYiyJzctppHj1YJcSCk2JolA96EnLOTfgZNMf7F+BEwBCx+d7AkvjCMaTVhea2+2+XFthoyct59zAoqptQLI/2NeBO1V1kYhcLSKnh8UeBOpF5DXgceAbqlofRzxePdiF95JWWZGXtJxzA1K6/mBV9crIYwW+Fm6x8pJWFxrb7JHuxzQAACAASURBVL58UIknLeecKzBPWl1oSpa0Bpd49aBzzhWYJ60uNCdLWkPLvaTlnHMF5kmrC43JktawQdYjRrP39u6cc4XiSasLjWEYrcHDB9uDjQ2FC8Y55wY4T1pdaAgFq8rKkLS8itA55womtqQlIjeKyHoRWZhh/qdF5JVwmysiB0XmvSMir4rIgpTuRfIumbQqqofZA2+M4ZxzBRNnSesmoLNOE5cBx6nqgcA1wOyU+Seo6sFddC8Su4ZmqCiFklEjbIKXtJxzrmBiu7hYVZ8UkSmdzJ8befos1p9Vn9PQApWDgMrhUCSetJxzroD6yjmtzwP3R54r8JCIvCAiF3b2QhG5UETmi8j82tranAfW0AyVZUBxEVRWevWgc84VUMG7cRKRE7CkdWxk8jGqujqMfvmwiLyhqk+me33oQn82wMyZM1O7y++1hhaYVBGeVI3wkpZzzhVQQUtaInIg8FvgjGjniqq6OtyvB+7GRs4siIZmGF4WnlRVetJyzrkCKljSEpFJwF+Az6jqm5HpQ0WkIvkYOAlI2wIxbu0J2NIaqgcBRo7w67Scc66AYqseFJHbsFEsq0VkFXAVUAqgqjcAVwKjgP8VEYC20FJwDHB3mFYC3KqqD8QVZ2e2hAuLKweFCVUhabUn7ByXc865vIqz9eCsLuZfAFyQZvpS4KCdX5F/m5rtfsR71YMjIJGALVtgRGXB4nLOuYHKiwudSF5YPDxa0gI/r+WccwXiSasTDaGk1XFOK5SuPGk551xBeNLqxOZkv4PvlbRG2r0nLeecKwhPWp1oaLZOMIaVhgle0nLOuYLypNWJTS1WNVgkYUL5IBgy2JOWc871ULisqSjyvEhEhmT7ek9andjhwuKkKr9Wyzk3sIjIKSKyWESWiMilaeafLyK1YWSOBSKyU8vwiEeBaJIaAjySbSwF78apL3uvs9wo78rJOTeAiEgxcD1wIrAKmCcic1T1tZRF71DVi7NYZbmqbk0+UdWtXtLKkU3NkWu0kkZ60nLODSiHA0tUdamqtgC3A2f0Yn3bRGRG8omIHAo0ZvtiL2ll0J6AukZ43/iUGVUjvKd359xAMgFYGXm+CjgizXIfF5H3A28Cl6jqyjTLAHwV+JOIrA7PxwGfyjYYT1oZbGiGdoXRg1NmVI2ApmbY3miNMpxzrn+rThkhfnYYPSNJUl+ADR8VdQ9wm6o2i8gXgT8AH0j3Zqo6T0T2BvYK635DVVuzDTar6kER2V1EBoXHx4vIl0VkRLZv0h+t3273o1NrWpPN3r0xhnNu11CnqjMjt9RR5FcBu0WeTwRWRxdQ1XpVDd0x8Bvg0ExvJiIXAUNVdaGqvgoME5H/l22w2Z7TugtoF5E9gN8BU4Fbs32T/mh9qGHduaSVvMB4Y17jcc65ApkHTBeRqSJSBpwNzIkuICLjIk9PB17vZH1fUNX3zrGo6kbgC9kGk231YEJV20TkTOA6Vf2FiLyU7Zv0R+u2W7m1Zqek5RcYO+cGjrDvvxh4ECgGblTVRSJyNTBfVecAXxaR04E2YANwfierLBIRUVWF91onpjZ5yyjbpNUqIrOA84DTwrTSTpbv99Zvh6pyKC1OmZHsNNerB51zA4Sq3gfclzLtysjjy4DLslzdg8CdInIDdm7si0DWw09lm7T+Naz4v1V1mYhMBf6Y7Zv0R+sb01QNAlQMg5JiL2k551zPfBO4EPgSVqH1EHYeLCtZJa1wEdmXAURkJFChqj/odqj9yPrtsE9Vmhkifq2Wc871kKomgBvCDRHZDfg6cG02r8+29eATIjJcRKqAl4Hfi8hPehZy39eegLqmDCUt8F4xnHOuF0SkWkS+JCJPAk9gI9ZnJdvqwUpV3Rz6k/q9ql4lIq/0INZ+ob4JEpqmuXtS1QhYsz6vMTnnXH8mIhXAmcA5wJ7A3cA0VZ3YnfVk2+S9JDRp/CRwbzeCvFFE1ovIwgzzRUR+HjphfCWla4/zROStcDsv2/fMhYzN3ZNGVnqvGM451z3rgc8D/w3srqpfB1q6u5Jsk9bVWIuPt8PVzNOAt7J43U3AKZ3MPxWYHm4XAr8CCNWQV2FdhRwOXBXOpeVFXUha1Z1VDzZsgba2fIXknHP93eVAObafv0xEdu/JSrJKWqr6J1U9UFW/FJ4vVdWPZ/G6J7E2+5mcAdys5llgRCjRnQw8rKobwoVnD9N58sup5na7H5yp8jR5gbE3e3fOuayo6k9V9Qjs4mMB/gqMF5Fvisie2a4n24YYE0Xk7lDVt05E7hKRbtVDZpCuI8YJnUxPF9uFIjJfRObX1tbmICRoTdh9aaat4105Oedcj4RCz3+r6gHAYcAI4P5sX59t9eDvsW47xmPJ454wrbcydcSYTQeNNlF1drLPrJqamhyEBC2hpFWWemFxUvICY29B6JxzPaaqr6rqZaqadVVhtkmrRlV/r6pt4XYTkIsMkakjxi47aIxTsqRVkmnreNJyzrluEZEtIrI5ctsSvc92PdkmrToROVdEisPtXKC+Z6HvYA7w2dCK8EigQVXXYI0+ThKRkaEBxklhWl60JKBEoDhdeQ9gRLJ60JOWc85lQ1UrVHV45FYRvc92Pdlep/U54JfAT7FqurlY106dEpHbgOOx8VpWYS0CS8MHuAHry+pfgCXA9uQ6VXWDiFyD9S4McLWqdtagI6da29P0ORhVWgKVFV7Scs65HhKR0VhrQgBUdUU2r8u2G6cVWIuP6Bt+Fbiui9fN6mK+AhdlmHcjcGM28eVaSwLKuiqDjqz0pOWcc90UeoP/MdZGYj0wGRvKZL9sXp9t9WA6X+vFa/u01kQnLQeTqkZ49aBzznXfNcCRwJuqOhX4IPB0ti/uTdLKdMan32ttzyZpjfSSlnPOdV+rqtZj42oVqerjwMHZvjjbc1rppG2CvitoSXRxTgtCV04NoGo9vzvnnMvGJhEZBvwTuEVE1mODR2al06QlIltIn5wEyNTJUb/Xms05raoR0NoGW7fZGFvOOecyEpFfArdhPSE1Al8FPg1UYl0FZqXTpKWqFb2Isd9q6ar1INiYWgD1mzxpOedc194C/gcYB9wB3Kaqf+juSnpzTmuXlXVJC/y8lnPOZUFVf6aqRwHHYX3S/l5EXheRK3Le9+BA05JN68FRodPcDRtjj8c553YVqrpcVX+oqodgY2t9DGvynhVPWmm0dSdp1eXtmmfnnCsIETlFRBaHsQ8v7WS5s0RERWRmJ8uUishpInIL1lHum0CXo4Yk9ab14C6rpb2TznKTygfBsCFQ7yUt59yuS0SKgeuBE7F+YeeJyBxVfS1luQrgy8BzGdZzIjAL+DDwPHA7cKGqbutOPF7SSiOri4sBRlV50nLO7eoOB5aEIUVasGRzRprlrgF+BDRlWM/lwDPAPqp6mqre0t2EBZ600mppz6IhBkB1lVcPOuf6u+rkmIThdmHK/C7HNxSRQ4DdVPXeTG+iqieo6m9624+sVw+m0ZrNxcVg57XeWdn1cs4513fVqWrGc1B0Mb6hiBRhnamfn+O40vKSVhpZdZgLVtLauAna22OPyTnnCqSr8Q0rgP2BJ0TkHaxfwTmdNcboDU9aKdoTkNBulLQS6tdqOed2ZfOA6SIyVUTKgLOxsRABUNUGVa1W1SmqOgV4FjhdVefHEYwnrRQtYdTirBpiVIdm794Ywzm3i1LVNuBibCDe14E7VXWRiFwdhhnJKz+nlaK1O0lrVJXd13nScs7tulT1PmzQ3ui0KzMse3ycsXhJK0VrOD2V9TktgHpvQeicc/ngSSvFe9WD2ZzTqqyA0hKvHnTOuTyJNWl11fWHiPxURBaE25sisikyrz0yb07qa+OSrB7MqqQlYoNB1npJyznn8iG2c1rZdP2hqpdElv934JDIKhpVNevRLHOlJVQPZlXSAmtB6NWDzjmXF3GWtLLt+iNpFjZAWEF1qyEGQE2Vl7Sccy5P4kxaXXb9kSQik4GpwGORyeWhS5FnReSjmd5ERC5Mdj9SW1vb66BbulM9CDCmBmrrIJHo9Xs755zrXJxJq9OuP1KcDfxZVaNdS0wKXYucA1wnIrune6GqzlbVmao6s6ampncR09F6MOuS1pgaaGv3xhjOOZcHcSatrrr+iDqblKpBVV0d7pcCT7Dj+a7YvNcQI9tzWmNColxXF0s8zjnnOsSZtDrt+iNJRPYCRmJd1ienjRSRQeFxNXAM8Frqa+PQrR4xAMaGpLV2fSzxOOec6xBb60FVbRORZNcfxcCNya4/gPmqmkxgs4DbVTVadbgP8GsRSWCJ9QepA47F5b2Li7MtaY0eZfde0nLOudjF2o1TNl1/qOp30rxuLnBAnLFl0u2SVlmZNXtf1/tGIM455zrnPWKk6HaTd4Ax1Z60nHMuDzxppWjpTt+DSWNGw1pPWs45FzdPWinaQkmrpLslrdp6HwzSOedi5kkrRXLUYkl3lVkmY0fbxcXeM4ZzzsXKk1aK1vZu9DuYNKba7v28lnPOxcqTVopkSatbxo62e79WyznnYuVJK0VropstB8Gu1SouhjWetJxzLk6etFK0tPcgaRUXW88Yq9fGEpNzzjnjSStFa6IH57QAxo2G1etyHo9zzrkOnrRS9OicFsD4sZa0NFNH9s451z9lMQr9F0Xk1TDS/FMism9csXjSStHak+pBgPFjYHsjNGzJeUzOOVcokVHoTwX2BWalSUq3quoBYbT5HwE/iSseT1opWhPd6Cw3atwYu/cqQufcrqXLUehVdXPk6VAyj53Ya560UrT0pPUgWEkLYI0nLedcv1KdHP093C5MmZ/VKPQicpGIvI2VtL4cV7Cx9vLeH/W4enBMDRSJl7Scc/1NXRglPpOsRqFX1euB60XkHODbwHk5im8HXtJK0dLT6sGyUqip9qTlnNvVdGcUerDqw4/GFYwnrRQ9uk4rafwYWOPXajnndildjkIvItMjTz8MvBVXMF49mKK5Hcp7ulUmjIUn5lqz9271uOucc31TlqPQXywiHwJagY3EVDUInrR2kFBoaofBPakeBJg0AbZuh/qNUF2V09icc65QuhqFXlW/kq9YYq0ezOKCtPNFpDZckLZARC6IzDtPRN4Kt9iydlRzGA6rxyWtKaHa951VOYnHOefcjmIraUUuSDsRO5E3T0TmqOprKYveoaoXp7y2CrgKmIm1UnkhvHZjXPECNLXZfXlPS1pTJtr98pUw88CcxOScc65DnCWtLi9I68TJwMOquiEkqoeBU2KK8z1NvS1pDa+AqhFe0nLOuZjEmbSyuiAN+LiIvCIifxaRZLPKbF+LiFyYvCiutrZ3gzAmk1aPz2mBlbbeWdn1cs4557otzqSVzQVp9wBTVPVA4BHgD914rU1Una2qM1V1Zk1NTY+DhY7qwUG9qTSdshuseBfaE72KxTnn3M7iTFpdXpCmqvWq2hye/gY4NNvXxiGZtHpX0toNmltgnQ8I6ZxzuRZn0srmgrRxkaenA6+Hxw8CJ4nISBEZCZwUpsWqsbfntAAmh8YYfl7LOedyLrbWg1lekPZlETkdaAM2AOeH124QkWuwxAdwtapuiCvWpF63HgRLWkVF8NYyOLqz7rycc851V6wXF2dxQdplwGUZXnsjcGOc8aXqdetBgPJBMG0SvB5bLybOOTdged+DEe+d0+ptKt9nOix+2xtjOOdcjnnSimhqt2aLZb3dKvtOh8Ymu8jYOedcznjSimhqg0HFOejrdp/Q4fFrXkXonHO55EkrorE9B1WDYANCjqyE197Mwcqcc84ledKKaGrrZcvBJBErbXljDOecyylPWhFNvRlLK9X+e8Oa9bC+LkcrdM4550kroqkth0lrxgF2/+LCHK3QOeecJ62IXg0AmWryBBg1El58JUcrdM4550krIqclLRE4ZH94aZFfr+WcczniSSuiqd2avOfMjANgy1Z4+50crtQ55wYuT1oRTW05rB4EK2kBzHs5hyt1zrmBy5NWRGMuWw+CXau1/17wj7mgaYcDc865Pk9EThGRxSKyREQuTTP/ayLyWhjQ91ERmRxXLJ60graE3XKatABOOBpWrIaly3O8Yueci5+IFAPXA6cC+wKzRGTflMVeAmaGAX3/DPworng8aQXv9fCey+pBgGOPgOJieGxujlfsnHN5cTiwRFWXqmoLcDtwRnQBVX1cVbeHp89iA/fGwpNWkLMe3lNVVsDMA62KsL09xyt3zrleqxaR+ZHbhSnzJwDR3r9XhWmZfB64P9dBJnnSCmIraQGcfDzUbYR/Ph/Dyp1zrlfqVHVm5DY7ZX66LsTTnqQXkXOBmcC1uQ4yyZNWkCxpDYpjWMwjZ8Bu4+HOOd4gwznX36wCdos8nwisTl1IRD4EfAs4XVWb4wrGk1bQmKwejKOkVVQEn/gILF0B8735u3OuX5kHTBeRqSJSBpwNzIkuICKHAL/GEtb6OIOJNWn1ppmkiLSLyIJwm5P62lx7r3owjpIWwAeOsSFLfnsbtLXF9CbOOZdbqtoGXAw8CLwO3Kmqi0TkahE5PSx2LTAM+FPc++y4dtHRZpInYsXLeSIyR1VfiyyWbCa5XUS+hDWT/FSY16iqB8cVX6rmOM9pAZSUwJc+C9/5Mdx9P3zitJjeyDnncktV7wPuS5l2ZeTxh/IVS5wlrT7VTLIryerB2EpaYOe2jp4Jf/wLvLOy6+Wdc87tIM6k1dtmkuWh+eWzIvLRTC8SkQuTTTVra2t7HOz2uJq8p7rofBgyBK65DrZui/nNnHNu1xJn0uptM8lJqjoTOAe4TkR2T/daVZ2dbKpZU1PT42Drm6CsCIbGnbRGjYRvfRnW1sK3fwj1G2N+Q+ec23XEmbR61UxSVVeH+6XAE8AhMcbKhiYYVW4jisRu/70scS1bBV++AuYtyMObOudc/xdn0upxM0kRGSkig8LjauAYINqAI+fqm6CqPM53SHH0TPjpVTB0MFxxLVx5rTWHb/WWhc45l0lslWGq2iYiyWaSxcCNyWaSwHxVncOOzSQBVqjq6cA+wK9FJIEl1h+ktDrMuQ1NsNeION8hjWmT4Zffg7v+Dn97EL79IxhSDgfvD4ceaLexPa/ydM65XY3oLtRDw8yZM3X+/Pndfp0qfPJ++PAUOD+17+J8aWm1klbytr7epu823vouPOk4mDqpQME553ZVIvJCaD/QL8Td7KBf2NIKrQk7p1UwZaVWZXj0TMuiq9aEBPYK3Pso3P0AHLQvnHEyHDEDir0zE+fcwONJCzufBXk+p9UZESth7TYezjwVtmyFB56Aex6Cq39qPWucdqJ1xFsxtNDROudc3njSAuob7b6gJa3OVAyzvgs/dio886Kd//rtrfB/d8EHj7HS1+SCXZftnHN540kLa4QBfaiklUlxMRx7mN2WLoe/PQSP/BPuewwOOxj+5QRrvFFWVuhInXMuFp606KgeHNnXk1bUtMlwyRfgc2fDvY9Y1eF/LbDWh0fMgKMOhUP2t1Kac87tIjxpYSWtyjIo7Y9tGyor4NNnwqdOg5dfg38+B0/Ph8fn2vyqER3nxyaOgzHVMKISRlbaffmgwsbvnHPd4EkLK2n12fNZ2Sop6bi2698/B4uXwqI3YOUaWPEuPDEXtm7f+XWDyy15VVbYrWKY3Q+vgOHDLKm1tduttRWami3x7b2HNQhx3aNqXXcNKvNSsHM94EkLK2lVDy50FDlUXAz7Trdbkips2gy19Xa/qQE2NnTcN2yBug02UGXDZrturCsTxsHEsfZ+GzZBewIGD4LR1TBksPXuUb/B1lUepg9P2VGLwLChMHSIPd7hBkhRyn0X86Pr7XiS9uF7y3TVS2b0WsadLmvU9Mu1J6C5GZpbrGPkdbWwbCW8vdxagwLUjILPz4LjjoT2dnhxoR0YTNkNqqvsvOVzL9l3M3QwTJ9qMW/abN/VsKFQU2XLikBLm43VNqgUyssthkS7rbs9AYlEx2MRKCm2gx1V+65KS+zSi9LSyLLttm2LiqAo5fvJldR1dbZuyfBd7vS6TOsI31G67zTdNauaZvnUeTssl+V7RRcsKrJOtIcNsYPI1jb7HTS3wLbt9nvZss1+R21t9p2VltpvYkyNdUBQMSxPfdAVnictrKS118hCRxEzEasSHFmZ3fJNzfZnaWq2P0lJcdiplcG7a2HRYnhpIdRttD9S1QhbZlsjLFhkf7iiIhg1whLWps3wymuwvSnez9lXDSqDSROtEc3USbZjemIu/OCX8MvfA5q+JFxcbJc1bNu+YxdfxcWWUJwD+//den2ho8iLAZ+0EgozRsPeVYWOpI8pH5T5fNf0qXb76CndW6eq3aJHhAm15Lh9ux18JpfZ6QZoYudlwEoF6Y56If2Rb3SZ1BJVpiP5zo7i0y1XVGTbb9AgOyJOdyT80VPg4SdhyTIrjR490w4qVrxrJalRI+GYw6wU2tIKK9+F4hIrrY6shMYmKzknRwooK7UDjOYWO9goLraL0IuKdnxcFE7etoWSWVGRva61zd6ntTUsG16jats4uZ13KJlomu3RxRH/DqWWlBJMphJu6sydZnVVGk7zvab9rqXredH5aX8jXbwXKa9tT9jvf+t2+05LQ0mqrNRKX8OG2q1imM1ra7fvbfMWWF9nI0ZkUzOyi/BunJxzbgDrb9049cf2cs455wYoT1rOOef6DU9azjnnOiUip4jIYhFZIiKXppn/fhF5UUTaROSsOGPxpOWccy4jESkGrgdOBfYFZolI6iBOK4DzgVvjjmfAtx50zjnXqcOBJaq6FEBEbgfOIDKavKq+E+Yl4g7GS1rOOTewVYvI/MjtwpT5E4CVkeerwrSC8JKWc84NbHVdNHnvqs+YvPKSlnPOuc6sAnaLPJ8IrC5QLLtWSeuFF16oE5HlPXz5FOCd3EWTM1PwuLprCn0ztil4XN0xBY+ru6bQ/dgmdzF/HjBdRKYC7wJnA+d0O7Ic2aV6xOgNEdmmqn1u7HqPq/v6amweV/d4XN0XV2wi8i/AdUAxcKOq/reIXA3MV9U5InIYcDcwEmgC1qrqfrmOAzxpvaev/hA9ru7rq7F5XN3jcXVfX44tV/yclnPOuX7Dk1aHvxQ6gAw8ru7rq7F5XN3jcXVfX44tJ7x60DnnXL/hJS3nnHP9hict55xz/caAT1oi8i0RaQm3+wsYx+EislFEmkWkSUTuCtOfEJF2EWkMtysLFF9biKtRRLaFadNEpD5su3oRmZLnmE6JbJdGEVERubsQ20xE3hSRhIg0Raal3T5iFoTpjSIS6zUvGWJ7PvzWGkVktYhMDtOPDdsxue0W5TmujN+diDwQ+a9enue4VkRiahORxjA9n9sr0z6iT/zO8kZVB+wNKAVageOAoUAjcFqBYjkIOCc8Hge0AKcBTwD39IFt1QbsmTLtOeD+8Ph+4NkCf5ftwNGF2GbAxdgFl01dbR/gSqAW6x7n88DWAsR2KTAoPH42Etux0eUKEFfa7y78FxqBCuB94X9bmq+4UubPBx4twPbKtI/oE7+zfN0KHkBBPzx8Aet3K/n8AeCBQscVYlkDfLOPJ60W4KDw+CCgpYDxXQpsDo8Lss1Sd2CZtg/WO/Yv0i2Xr9hS5v0AWNbVcnnaZpmS1g7/TaAO+EK+t1dIAG3AhwqxvVJiSe4j+szvLB+3gV49uBf24096BxhfmFA6iMixQA3wxzDp1FC8fzPfVXARCiwQkW0i8n9hWqmqvgwQ7gvZLdjnsB1bUl/YZpm2zygiwzoAW4ED8xxb1PnYEXrSIBHZLiKbROSiAsST7rsbDyyNLFOP/X/z7SKgWVUfiUzL+/ZK2Uf0l99ZTgz0pNWnei8GEJExwIPAj1X1XexPMhgYBqwHHipQaIer6hDgMOAsEbm4QHHsRESGAnsA/xUm9ZVtlkm6313s4xClIyIPhfdO7mxfBqaH7/oi4Gciks9hKDJ9d33lv/pF4NHI87xvrzT7iIyLpplWkN9ZLg30pPUGUB15PgUrcheEiAwGFgGPqOo3AVR1kaq2qmo7VgU2qRCxqepL4f41rA79ZKBVRA4KsR+EVZsUwuXABlVdFGLsE9uMzNunDhsBNmkYsDDPsSEis4GjgAM01B+p6hZVXRIe34IdnX8gXzF18t29C0yLLDoKeDNfcQGIyCBgH+Dq5LR8b690+wj6+O8s1wZ60roZqBSR94Wj9eOA/y1EICIi2A9qpaqeEZl+UGSx/wDWFSC2GhEZl3wMzMBO3i/AzocQ7l/Kd2zBucDfkk/6wjYLMm2fO4GzQ+uuz2PnIF7OZ2Ai8i3gPOAwVa2PTN9bRErD4+Owhg9P5zGuTN/d/wLHiUiFiLwPqARuyldckXi2qOr85IR8bq9M+wj68O8sFoU+qVboG9bCpgVrjfRQAeP4Elbd0Ri5XYnV4zeF52spwIlULJknY2pKbiesSm5D2H4bgGkFiG0UVuWxW2Ra3rcZsBxrvajYke7vM20frNrm1fCbawI+U4DYWsLj5Pe6KCx7bWTbbQeuznNcGb87rKqwNcR+RT7jCtOXALekLJvP7ZVpH9Enfmf5unk3Ts455/qNgV496Jxzrh/xpOWcc67f8KTlnHOu3/Ck5Zxzrt/wpOWcc67f8KTlXDeEHsgXRG6X5nDdU0Sk31/86VycCtlXnHP9UaOqHlzoIJwbqLyk5VwOiMg7IvJDsXGqnheRPcL0ySLyqIi8Eu4nheljxMb+ejncjg6rKhaR34jIIhF5KHTb45wLPGk51z2DU6oHPxWZt1lVDwd+CVwXpv0SuFlVDwRuAX4epv8c+IeqHoR1i5UcPHA6cL2q7gdsAj4e8+dxrl/xHjGc6wYR2aqqw9JMfwf4gKouDX3RrVXVUSJSB4xT1dYwfY2qVotILTBRVZsj65gCPKyq08Pzb2LDTnw3/k/mXP/gJS3nckczPM60TDrNkcft+Hln53bgScu53PlU5P6Z8HgucHZ4/GngsjlZSAAAAJJJREFUqfD4UawDVESkWESG5ytI5/ozP4pzrnsGi8iCyPMHVDXZ7H2QiDyHHQzOCtO+DNwoIt8AaoF/DdO/AswOQ0a0YwmsYGO5Oddf+Dkt53IgnNOaqap1hY7FuV2ZVw8655zrN7yk5Zxzrt/wkpZzzrl+w5OWc865fsOTlnPOuX7Dk5Zzzrl+w5OWc865fuP/A/SP19D8c9lzAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plot_loss_with_acc(loss, val_acc)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 绘制测试数据的TSNE降维图\n",
    "from sklearn.manifold import TSNE\n",
    "tsne = TSNE()\n",
    "out = tsne.fit_transform(test_logits)\n",
    "fig = plt.figure()\n",
    "for i in range(7):\n",
    "    indices = test_label == i\n",
    "    x, y = out[indices].T\n",
    "    plt.scatter(x, y, label=str(i))\n",
    "plt.legend()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.5"
  },
  "pycharm": {
   "stem_cell": {
    "cell_type": "raw",
    "metadata": {
     "collapsed": false
    },
    "source": []
   }
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": true,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}