GNN/GCN_code/GCN_tensorflow.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "toc": true
   },
   "source": [
    "<h1>Table of Contents<span class=\"tocSkip\"></span></h1>\n",
    "<div class=\"toc\"><ul class=\"toc-item\"></ul></div>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-05-16T10:13:11.472112Z",
     "start_time": "2020-05-16T10:13:11.467780Z"
    }
   },
   "outputs": [],
   "source": [
    "from __future__ import division # / 运算执行的是精确除法\n",
    "from __future__ import print_function\n",
    "\n",
    "import time\n",
    "import tensorflow as tf\n",
    "\n",
    "import numpy as np\n",
    "import pickle as pkl\n",
    "import networkx as nx\n",
    "import scipy.sparse as sp\n",
    "from scipy.sparse.linalg.eigen.arpack import eigsh\n",
    "import sys"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-05-16T02:44:51.103019Z",
     "start_time": "2020-05-16T02:44:51.073656Z"
    },
    "code_folding": []
   },
   "outputs": [],
   "source": [
    "def parse_index_file(filename):\n",
    "    \"\"\"Parse index file. \"\"\"\n",
    "    index = []\n",
    "    for line in open(filename):\n",
    "        index.append(int(line.strip())) # 读取文件每一行的数字， stirp() 移除前后的空格\n",
    "    return index\n",
    "\n",
    "\n",
    "def sample_mask(idx, l):\n",
    "    \"\"\"Create mask.\"\"\"\n",
    "    mask = np.zeros(l)\n",
    "    mask[idx] = 1\n",
    "    return np.array(mask, dtype=np.bool)\n",
    "\n",
    "\n",
    "def load_data(dataset_str):\n",
    "    \"\"\"\n",
    "    Loads input data from gcn/data directory\n",
    "\n",
    "    ind.dataset_str.x => the feature vectors of the training instances as scipy.sparse.csr.csr_matrix object;\n",
    "    ind.dataset_str.tx => the feature vectors of the test instances as scipy.sparse.csr.csr_matrix object;\n",
    "    ind.dataset_str.allx => the feature vectors of both labeled and unlabeled training instances\n",
    "        (a superset of ind.dataset_str.x) as scipy.sparse.csr.csr_matrix object; (除去测试集的所有数据)\n",
    "    ind.dataset_str.y => the one-hot labels of the labeled training instances as numpy.ndarray object;\n",
    "    ind.dataset_str.ty => the one-hot labels of the test instances as numpy.ndarray object;\n",
    "    ind.dataset_str.ally => the labels for instances in ind.dataset_str.allx as numpy.ndarray object;\n",
    "    ind.dataset_str.graph => a dict in the format {index: [index_of_neighbor_nodes]} as collections.defaultdict\n",
    "        object;\n",
    "    ind.dataset_str.test.index => the indices of test instances in graph, for the inductive setting as list object.\n",
    "\n",
    "    All objects above must be saved using python pickle module.\n",
    "\n",
    "    :param dataset_str: Dataset name\n",
    "    :return: All data input files loaded (as well the training/test data).\n",
    "    \"\"\"\n",
    "    names = ['x', 'y', 'tx', 'ty', 'allx', 'ally', 'graph']\n",
    "    objects = []\n",
    "    for i in range(len(names)):\n",
    "        with open(\"data/ind.{}.{}\".format(dataset_str, names[i]), 'rb') as f:\n",
    "            if sys.version_info > (3, 0): # python 版本是否高于 3.0\n",
    "                objects.append(pkl.load(f, encoding='latin1'))\n",
    "            else:\n",
    "                objects.append(pkl.load(f))\n",
    "\n",
    "    x, y, tx, ty, allx, ally, graph = tuple(objects)\n",
    "    test_idx_reorder = parse_index_file(\"data/ind.{}.test.index\".format(dataset_str)) # 测试集数据的 index\n",
    "    test_idx_range = np.sort(test_idx_reorder)\n",
    "\n",
    "    if dataset_str == 'citeseer':\n",
    "        # Fix citeseer dataset (there are some isolated nodes in the graph)\n",
    "        # Find isolated nodes, add them as zero-vecs into the right position\n",
    "        test_idx_range_full = range(min(test_idx_reorder), max(test_idx_reorder)+1)\n",
    "        tx_extended = sp.lil_matrix((len(test_idx_range_full), x.shape[1]))\n",
    "        tx_extended[test_idx_range-min(test_idx_range), :] = tx\n",
    "        tx = tx_extended\n",
    "        ty_extended = np.zeros((len(test_idx_range_full), y.shape[1]))\n",
    "        ty_extended[test_idx_range-min(test_idx_range), :] = ty\n",
    "        ty = ty_extended\n",
    "\n",
    "    features = sp.vstack((allx, tx)).tolil() # 先将数据堆叠，获得完整特征数据集， 之后转换为 lil 链表稀疏矩阵， 加速矩阵的访问。\n",
    "    features[test_idx_reorder, :] = features[test_idx_range, :] # 调整测试集部分特征的顺序， 按照顺序进行排列\n",
    "    adj = nx.adjacency_matrix(nx.from_dict_of_lists(graph)) # 构建邻接矩阵\n",
    "\n",
    "    labels = np.vstack((ally, ty)) # 堆叠获得完整的标签\n",
    "    labels[test_idx_reorder, :] = labels[test_idx_range, :] # 调整测试集部分标签的顺序， 按照顺序进行排列\n",
    "\n",
    "    idx_test = test_idx_range.tolist()\n",
    "    idx_train = range(len(y))\n",
    "    idx_val = range(len(y), len(y)+500)\n",
    "\n",
    "    train_mask = sample_mask(idx_train, labels.shape[0])\n",
    "    val_mask = sample_mask(idx_val, labels.shape[0])\n",
    "    test_mask = sample_mask(idx_test, labels.shape[0])\n",
    "\n",
    "    y_train = np.zeros(labels.shape)\n",
    "    y_val = np.zeros(labels.shape)\n",
    "    y_test = np.zeros(labels.shape)\n",
    "    y_train[train_mask, :] = labels[train_mask, :] # 训练集标签\n",
    "    y_val[val_mask, :] = labels[val_mask, :]\n",
    "    y_test[test_mask, :] = labels[test_mask, :]\n",
    "\n",
    "    return adj, features, y_train, y_val, y_test, train_mask, val_mask, test_mask\n",
    "\n",
    "def sparse_to_tuple(sparse_mx):\n",
    "    \"\"\"Convert sparse matrix to tuple representation.\"\"\"\n",
    "    def to_tuple(mx):\n",
    "        if not sp.isspmatrix_coo(mx):\n",
    "            mx = mx.tocoo()\n",
    "        coords = np.vstack((mx.row, mx.col)).transpose() # 获得稀疏矩阵坐标 (2708, 1433)  --> (49216, 2)\n",
    "        values = mx.data # 相应位置的值 (49216, ) 即矩阵中的所有非零值\n",
    "        shape = mx.shape  # 稀疏矩阵的大小\n",
    "        return coords, values, shape\n",
    "\n",
    "    if isinstance(sparse_mx, list):\n",
    "        # 如果是包含多个 matrix 的列表， 逐一转换\n",
    "        for i in range(len(sparse_mx)):\n",
    "            sparse_mx[i] = to_tuple(sparse_mx[i])\n",
    "    else:\n",
    "        sparse_mx = to_tuple(sparse_mx)\n",
    "\n",
    "    return sparse_mx\n",
    "\n",
    "\n",
    "def preprocess_features(features):\n",
    "    \"\"\"Row-normalize feature matrix and convert to tuple representation\"\"\"\n",
    "    rowsum = np.array(features.sum(1)) # 每一行加和 ， (2708,1443) --> (2708,1)\n",
    "    r_inv = np.power(rowsum, -1).flatten() # 求每个值的倒数 (2708,1) --> (2708,1)\n",
    "    r_inv[np.isinf(r_inv)] = 0. # 将无穷大的值 isinf， 转化为 0\n",
    "    r_mat_inv = sp.diags(r_inv) # 转换为对角矩阵 (2708,1) --> (2708,2708)\n",
    "    features = r_mat_inv.dot(features) # 和原始矩阵相乘\n",
    "    return sparse_to_tuple(features) # 转换为 tuple： \n",
    "\n",
    "\n",
    "def normalize_adj(adj):\n",
    "    \"\"\"Symmetrically normalize adjacency matrix.  D^{-0.5}* A* D^{-0.5}\"\"\"\n",
    "    adj = sp.coo_matrix(adj)\n",
    "    rowsum = np.array(adj.sum(1)) # 每行求和 ， \n",
    "    d_inv_sqrt = np.power(rowsum, -0.5).flatten() # 和值求 ^{-0.5}\n",
    "    d_inv_sqrt[np.isinf(d_inv_sqrt)] = 0. #  inf 值赋 0 \n",
    "    d_mat_inv_sqrt = sp.diags(d_inv_sqrt) # 得到的 D 转换成为对角矩阵\n",
    "    return adj.dot(d_mat_inv_sqrt).transpose().dot(d_mat_inv_sqrt).tocoo() #  D^{-0.5}* A* D^{-0.5}\n",
    "\n",
    "\n",
    "def preprocess_adj(adj):\n",
    "    \"\"\"Preprocessing of adjacency matrix for simple GCN model and conversion to tuple representation.\"\"\"\n",
    "    adj_normalized = normalize_adj(adj + sp.eye(adj.shape[0])) # 加入 self-loop 之后重新归一化\n",
    "    return sparse_to_tuple(adj_normalized)\n",
    "\n",
    "\n",
    "def construct_feed_dict(features, support, labels, labels_mask, placeholders):\n",
    "    \"\"\"Construct feed dictionary.\"\"\"\n",
    "    feed_dict = dict()\n",
    "    feed_dict.update({placeholders['labels']: labels}) # 更新字典的键值\n",
    "    feed_dict.update({placeholders['labels_mask']: labels_mask})\n",
    "    feed_dict.update({placeholders['features']: features}) # \n",
    "    feed_dict.update({placeholders['support'][i]: support[i] for i in range(len(support))}) # 多个图\n",
    "    feed_dict.update({placeholders['num_features_nonzero']: features[1].shape}) # 非零特征值 数目\n",
    "    return feed_dict\n",
    "\n",
    "\n",
    "def chebyshev_polynomials(adj, k):\n",
    "    \"\"\"Calculate Chebyshev polynomials up to order k. Return a list of sparse matrices (tuple representation).\"\"\"\n",
    "    print(\"Calculating Chebyshev polynomials up to order {}...\".format(k))\n",
    "\n",
    "    adj_normalized = normalize_adj(adj)\n",
    "    laplacian = sp.eye(adj.shape[0]) - adj_normalized # L  = I  - A*\n",
    "    largest_eigval, _ = eigsh(laplacian, 1, which='LM') # 最大特征值\n",
    "    scaled_laplacian = (2. / largest_eigval[0]) * laplacian - sp.eye(adj.shape[0]) # 2/  *L - I\n",
    "\n",
    "    t_k = list()\n",
    "    t_k.append(sp.eye(adj.shape[0]))\n",
    "    t_k.append(scaled_laplacian) # 切比雪夫多项式， 迭代添加\n",
    "\n",
    "    def chebyshev_recurrence(t_k_minus_one, t_k_minus_two, scaled_lap):\n",
    "        s_lap = sp.csr_matrix(scaled_lap, copy=True)\n",
    "        return 2 * s_lap.dot(t_k_minus_one) - t_k_minus_two # 2 * T_{k1} - T_{k2}\n",
    "\n",
    "    for i in range(2, k+1):\n",
    "        t_k.append(chebyshev_recurrence(t_k[-1], t_k[-2], scaled_laplacian))\n",
    "\n",
    "    return sparse_to_tuple(t_k)\n",
    "\n",
    "def masked_softmax_cross_entropy(preds, labels, mask):\n",
    "    \"\"\"Softmax cross-entropy loss with masking.\"\"\"\n",
    "    loss = tf.nn.softmax_cross_entropy_with_logits(\n",
    "        logits=preds,\n",
    "        labels=labels)  # 对最后一层输出先做一个 softmax，之后对 softmax 的结果进行交叉熵计算\n",
    "    mask = tf.cast(mask, dtype=tf.float32)  #  数据类型转变\n",
    "    mask /= tf.reduce_mean(mask)  # 求平均值， reduce ：降维\n",
    "    loss *= mask\n",
    "    return tf.reduce_mean(loss)\n",
    "\n",
    "\n",
    "def masked_accuracy(preds, labels, mask):\n",
    "    \"\"\"Accuracy with masking.\"\"\"\n",
    "    correct_prediction = tf.equal(tf.argmax(preds, 1), tf.argmax(labels, 1))\n",
    "    accuracy_all = tf.cast(correct_prediction, tf.float32)\n",
    "    mask = tf.cast(mask, dtype=tf.float32)\n",
    "    mask /= tf.reduce_mean(mask)\n",
    "    accuracy_all *= mask\n",
    "    return tf.reduce_mean(accuracy_all)\n",
    "\n",
    "\n",
    "# inits 初始化的几种方法\n",
    "def uniform(shape, scale=0.05, name=None):\n",
    "    \"\"\"Uniform init.\"\"\"\n",
    "    initial = tf.random_uniform(shape,\n",
    "                                minval=-scale,\n",
    "                                maxval=scale,\n",
    "                                dtype=tf.float32)\n",
    "    return tf.Variable(initial, name=name)\n",
    "\n",
    "\n",
    "def glorot(shape, name=None):\n",
    "    \"\"\"Glorot & Bengio (AISTATS 2010) init.\"\"\"\n",
    "    init_range = np.sqrt(6.0 / (shape[0] + shape[1]))\n",
    "    initial = tf.random_uniform(shape,\n",
    "                                minval=-init_range,\n",
    "                                maxval=init_range,\n",
    "                                dtype=tf.float32)\n",
    "    return tf.Variable(initial, name=name)\n",
    "\n",
    "\n",
    "def zeros(shape, name=None):\n",
    "    \"\"\"All zeros.\"\"\"\n",
    "    initial = tf.zeros(shape, dtype=tf.float32)\n",
    "    return tf.Variable(initial, name=name)\n",
    "\n",
    "\n",
    "def ones(shape, name=None):\n",
    "    \"\"\"All ones.\"\"\"\n",
    "    initial = tf.ones(shape, dtype=tf.float32)\n",
    "    return tf.Variable(initial, name=name)\n",
    "\n",
    "_LAYER_UIDS = {}\n",
    "def get_layer_uid(layer_name=''):\n",
    "    \"\"\"Helper function, assigns unique layer IDs.\"\"\"\n",
    "    if layer_name not in _LAYER_UIDS:\n",
    "        _LAYER_UIDS[layer_name] = 1\n",
    "        return 1\n",
    "    else:\n",
    "        _LAYER_UIDS[layer_name] += 1\n",
    "        return _LAYER_UIDS[layer_name]\n",
    "\n",
    "\n",
    "def sparse_dropout(x, keep_prob, noise_shape):\n",
    "    \"\"\"Dropout for sparse tensors.\"\"\"\n",
    "    random_tensor = keep_prob\n",
    "    random_tensor += tf.random_uniform(noise_shape)  # [0,1] 之间 均匀分布\n",
    "    dropout_mask = tf.cast(\n",
    "        tf.floor(random_tensor),\n",
    "        dtype=tf.bool)  # tf.floor 向下取整； （0.5， 1.5）得到 0，1 值， 转换为 boor 值\n",
    "    pre_out = tf.sparse_retain(x, dropout_mask)  # 保留相应的 非空值\n",
    "    return pre_out * (1. / keep_prob)\n",
    "\n",
    "\n",
    "def dot(x, y, sparse=False):\n",
    "    \"\"\"Wrapper for tf.matmul (sparse vs dense).\"\"\"\n",
    "    if sparse:\n",
    "        res = tf.sparse_tensor_dense_matmul(x, y)  # * 稀疏矩阵相乘\n",
    "    else:\n",
    "        res = tf.matmul(x, y)  # 矩阵相乘\n",
    "    return res\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-05-16T10:15:16.393957Z",
     "start_time": "2020-05-16T10:15:16.367863Z"
    },
    "code_folding": [
     45
    ]
   },
   "outputs": [],
   "source": [
    "class Layer(object):\n",
    "    \"\"\"Base layer class. Defines basic API for all layer objects.\n",
    "    Implementation inspired by keras (http://keras.io).\n",
    "\n",
    "    # Properties\n",
    "        name: String, defines the variable scope of the layer.\n",
    "        logging: Boolean, switches Tensorflow histogram logging on/off\n",
    "\n",
    "    # Methods\n",
    "        _call(inputs): Defines computation graph of layer\n",
    "            (i.e. takes input, returns output)\n",
    "        __call__(inputs): Wrapper for _call() \n",
    "        _log_vars(): Log all variables\n",
    "    \"\"\"\n",
    "    def __init__(self, **kwargs):\n",
    "        allowed_kwargs = {'name', 'logging'}\n",
    "        for kwarg in kwargs.keys():\n",
    "            assert kwarg in allowed_kwargs, 'Invalid keyword argument: ' + kwarg\n",
    "        name = kwargs.get('name')\n",
    "        if not name:\n",
    "            layer = self.__class__.__name__.lower()\n",
    "            name = layer + '_' + str(get_layer_uid(layer))\n",
    "        self.name = name\n",
    "        self.vars = {}\n",
    "        logging = kwargs.get('logging', False)\n",
    "        self.logging = logging\n",
    "        self.sparse_inputs = False\n",
    "\n",
    "    def _call(self, inputs):\n",
    "        return inputs\n",
    "\n",
    "    def __call__(self, inputs):\n",
    "        with tf.name_scope(self.name):  # 变量作用域\n",
    "            if self.logging and not self.sparse_inputs:\n",
    "                tf.summary.histogram(self.name + '/inputs', inputs)\n",
    "            outputs = self._call(inputs)\n",
    "            if self.logging:\n",
    "                tf.summary.histogram(self.name + '/outputs', outputs)\n",
    "            return outputs\n",
    "\n",
    "    def _log_vars(self):\n",
    "        for var in self.vars:\n",
    "            tf.summary.histogram(self.name + '/vars/' + var, self.vars[var])\n",
    "\n",
    "\n",
    "class Dense(Layer):\n",
    "    \"\"\"Dense layer.\"\"\"\n",
    "    def __init__(self,\n",
    "                 input_dim,\n",
    "                 output_dim,\n",
    "                 placeholders,\n",
    "                 dropout=0.,\n",
    "                 sparse_inputs=False,\n",
    "                 act=tf.nn.relu,\n",
    "                 bias=False,\n",
    "                 featureless=False,\n",
    "                 **kwargs):\n",
    "        super(Dense, self).__init__(**kwargs)\n",
    "\n",
    "        if dropout:\n",
    "            self.dropout = placeholders['dropout']\n",
    "        else:\n",
    "            self.dropout = 0.\n",
    "\n",
    "        self.act = act  # 激活函数 relu\n",
    "        self.sparse_inputs = sparse_inputs  # 输入的是否是稀疏矩阵\n",
    "        self.featureless = featureless  # 节点是否有特征\n",
    "        self.bias = bias\n",
    "\n",
    "        # helper variable for sparse dropout\n",
    "        self.num_features_nonzero = placeholders[\n",
    "            'num_features_nonzero']  # 检查稀疏变量随机 dropout 是否有效。\n",
    "\n",
    "        with tf.variable_scope(self.name + '_vars'):  # 变量作用域\n",
    "            self.vars['weights'] = glorot([input_dim, output_dim],\n",
    "                                          name='weights')\n",
    "            if self.bias:\n",
    "                self.vars['bias'] = zeros([output_dim], name='bias')\n",
    "\n",
    "        if self.logging:\n",
    "            self._log_vars()\n",
    "\n",
    "    def _call(self, inputs):\n",
    "        x = inputs\n",
    "\n",
    "        # dropout\n",
    "        if self.sparse_inputs:\n",
    "            x = sparse_dropout(x, 1 - self.dropout, self.num_features_nonzero)\n",
    "        else:\n",
    "            x = tf.nn.dropout(x, 1 - self.dropout)\n",
    "\n",
    "        # transform\n",
    "        output = dot(x, self.vars['weights'], sparse=self.sparse_inputs)\n",
    "\n",
    "        # bias\n",
    "        if self.bias:\n",
    "            output += self.vars['bias']\n",
    "\n",
    "        return self.act(output)\n",
    "\n",
    "\n",
    "class GraphConvolution(Layer):\n",
    "    \"\"\"Graph convolution layer.\"\"\"\n",
    "    def __init__(self,\n",
    "                 input_dim,\n",
    "                 output_dim,\n",
    "                 placeholders,\n",
    "                 dropout=0.,\n",
    "                 sparse_inputs=False,\n",
    "                 act=tf.nn.relu,\n",
    "                 bias=False,\n",
    "                 featureless=False,\n",
    "                 **kwargs):\n",
    "        super(GraphConvolution, self).__init__(**kwargs)\n",
    "\n",
    "        if dropout:\n",
    "            self.dropout = placeholders['dropout']\n",
    "        else:\n",
    "            self.dropout = 0.\n",
    "\n",
    "        self.act = act\n",
    "        self.support = placeholders['support']\n",
    "        self.sparse_inputs = sparse_inputs\n",
    "        self.featureless = featureless\n",
    "        self.bias = bias  # 是不是有 bias\n",
    "\n",
    "        # helper variable for sparse dropout\n",
    "        self.num_features_nonzero = placeholders['num_features_nonzero']\n",
    "\n",
    "        with tf.variable_scope(self.name + '_vars'):\n",
    "            for i in range(len(self.support)):\n",
    "                self.vars['weights_' + str(i)] = glorot(\n",
    "                    [input_dim, output_dim], name='weights_' + str(i))  # 初始化 权重\n",
    "            if self.bias:\n",
    "                self.vars['bias'] = zeros([output_dim], name='bias')\n",
    "\n",
    "        if self.logging:\n",
    "            self._log_vars()\n",
    "\n",
    "    def _call(self, inputs):\n",
    "        x = inputs\n",
    "\n",
    "        # dropout\n",
    "        if self.sparse_inputs:\n",
    "            x = sparse_dropout(x, 1 - self.dropout, self.num_features_nonzero) # 稀疏矩阵 随机 dropout\n",
    "        else:\n",
    "            x = tf.nn.dropout(x, 1 - self.dropout)\n",
    "\n",
    "        # convolve\n",
    "        supports = list()\n",
    "        for i in range(len(self.support)):\n",
    "            if not self.featureless:\n",
    "                pre_sup = dot(x,\n",
    "                              self.vars['weights_' + str(i)],\n",
    "                              sparse=self.sparse_inputs)  # X * W\n",
    "            else:\n",
    "                pre_sup = self.vars['weights_' + str(i)]  # W\n",
    "            support = dot(self.support[i], pre_sup, sparse=True)\n",
    "            supports.append(support)\n",
    "        output = tf.add_n(supports)\n",
    "\n",
    "        # bias\n",
    "        if self.bias:\n",
    "            output += self.vars['bias']\n",
    "\n",
    "        return self.act(output)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-05-15T01:20:32.726059Z",
     "start_time": "2020-05-15T01:20:32.699208Z"
    },
    "code_folding": [
     78
    ]
   },
   "outputs": [],
   "source": [
    "# Models\n",
    "class Model(object):\n",
    "    def __init__(self, **kwargs):\n",
    "        allowed_kwargs = {'name', 'logging'}\n",
    "        for kwarg in kwargs.keys():\n",
    "            assert kwarg in allowed_kwargs, 'Invalid keyword argument: ' + kwarg\n",
    "        name = kwargs.get('name')\n",
    "        if not name:\n",
    "            name = self.__class__.__name__.lower()\n",
    "        self.name = name\n",
    "\n",
    "        logging = kwargs.get('logging', False)\n",
    "        self.logging = logging\n",
    "\n",
    "        self.vars = {}\n",
    "        self.placeholders = {}\n",
    "\n",
    "        self.layers = []\n",
    "        self.activations = []\n",
    "\n",
    "        self.inputs = None\n",
    "        self.outputs = None\n",
    "\n",
    "        self.loss = 0\n",
    "        self.accuracy = 0\n",
    "        self.optimizer = None\n",
    "        self.opt_op = None\n",
    "\n",
    "    def _build(self):\n",
    "        raise NotImplementedError\n",
    "\n",
    "    def build(self):\n",
    "        \"\"\" Wrapper for _build() \"\"\"\n",
    "        with tf.variable_scope(self.name):\n",
    "            self._build()\n",
    "\n",
    "        # Build sequential layer model\n",
    "        self.activations.append(self.inputs)\n",
    "        for layer in self.layers:\n",
    "            hidden = layer(self.activations[-1])\n",
    "            self.activations.append(hidden)\n",
    "        self.outputs = self.activations[-1]\n",
    "\n",
    "        # Store model variables for easy access\n",
    "        variables = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES, scope=self.name)\n",
    "        self.vars = {var.name: var for var in variables}\n",
    "\n",
    "        # Build metrics\n",
    "        self._loss()\n",
    "        self._accuracy()\n",
    "\n",
    "        self.opt_op = self.optimizer.minimize(self.loss)\n",
    "\n",
    "    def predict(self):\n",
    "        pass\n",
    "\n",
    "    def _loss(self):\n",
    "        raise NotImplementedError\n",
    "\n",
    "    def _accuracy(self):\n",
    "        raise NotImplementedError\n",
    "\n",
    "    def save(self, sess=None):\n",
    "        if not sess:\n",
    "            raise AttributeError(\"TensorFlow session not provided.\")\n",
    "        saver = tf.train.Saver(self.vars)\n",
    "        save_path = saver.save(sess, \"tmp/%s.ckpt\" % self.name)\n",
    "        print(\"Model saved in file: %s\" % save_path)\n",
    "\n",
    "    def load(self, sess=None):\n",
    "        if not sess:\n",
    "            raise AttributeError(\"TensorFlow session not provided.\")\n",
    "        saver = tf.train.Saver(self.vars)\n",
    "        save_path = \"tmp/%s.ckpt\" % self.name\n",
    "        saver.restore(sess, save_path)\n",
    "        print(\"Model restored from file: %s\" % save_path)\n",
    "\n",
    "\n",
    "class MLP(Model):\n",
    "    def __init__(self, placeholders, input_dim, **kwargs):\n",
    "        super(MLP, self).__init__(**kwargs)\n",
    "\n",
    "        self.inputs = placeholders['features']\n",
    "        self.input_dim = input_dim\n",
    "        # self.input_dim = self.inputs.get_shape().as_list()[1]  # To be supported in future Tensorflow versions\n",
    "        self.output_dim = placeholders['labels'].get_shape().as_list()[1]\n",
    "        self.placeholders = placeholders\n",
    "\n",
    "        self.optimizer = tf.train.AdamOptimizer(learning_rate=FLAGS.learning_rate)\n",
    "\n",
    "        self.build()\n",
    "\n",
    "    def _loss(self):\n",
    "        # Weight decay loss\n",
    "        for var in self.layers[0].vars.values():\n",
    "            self.loss += weight_decay * tf.nn.l2_loss(var)\n",
    "\n",
    "        # Cross entropy error\n",
    "        self.loss += masked_softmax_cross_entropy(self.outputs, self.placeholders['labels'],\n",
    "                                                  self.placeholders['labels_mask'])\n",
    "\n",
    "    def _accuracy(self):\n",
    "        self.accuracy = masked_accuracy(self.outputs, self.placeholders['labels'],\n",
    "                                        self.placeholders['labels_mask'])\n",
    "\n",
    "    def _build(self):\n",
    "        self.layers.append(Dense(input_dim=self.input_dim,\n",
    "                                 output_dim=hidden1,\n",
    "                                 placeholders=self.placeholders,\n",
    "                                 act=tf.nn.relu,\n",
    "                                 dropout=True,\n",
    "                                 sparse_inputs=True,\n",
    "                                 logging=self.logging))\n",
    "\n",
    "        self.layers.append(Dense(input_dim=hidden1,\n",
    "                                 output_dim=self.output_dim,\n",
    "                                 placeholders=self.placeholders,\n",
    "                                 act=lambda x: x,\n",
    "                                 dropout=True,\n",
    "                                 logging=self.logging))\n",
    "\n",
    "    def predict(self):\n",
    "        return tf.nn.softmax(self.outputs)\n",
    "\n",
    "\n",
    "class GCN(Model):\n",
    "    def __init__(self, placeholders, input_dim, **kwargs):\n",
    "        super(GCN, self).__init__(**kwargs)\n",
    "\n",
    "        self.inputs = placeholders['features']\n",
    "        self.input_dim = input_dim\n",
    "        # self.input_dim = self.inputs.get_shape().as_list()[1]  # To be supported in future Tensorflow versions\n",
    "        self.output_dim = placeholders['labels'].get_shape().as_list()[1]\n",
    "        self.placeholders = placeholders\n",
    "\n",
    "        self.optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate)\n",
    "\n",
    "        self.build()\n",
    "\n",
    "    def _loss(self):\n",
    "        # Weight decay loss\n",
    "        for var in self.layers[0].vars.values():\n",
    "            self.loss += weight_decay * tf.nn.l2_loss(var)\n",
    "\n",
    "        # Cross entropy error\n",
    "        self.loss += masked_softmax_cross_entropy(self.outputs, self.placeholders['labels'],\n",
    "                                                  self.placeholders['labels_mask'])\n",
    "\n",
    "    def _accuracy(self):\n",
    "        self.accuracy = masked_accuracy(self.outputs, self.placeholders['labels'],\n",
    "                                        self.placeholders['labels_mask'])\n",
    "\n",
    "    def _build(self):\n",
    "\n",
    "        self.layers.append(GraphConvolution(input_dim=self.input_dim,\n",
    "                                            output_dim=hidden1,\n",
    "                                            placeholders=self.placeholders,\n",
    "                                            act=tf.nn.relu,\n",
    "                                            dropout=True,\n",
    "                                            sparse_inputs=True,\n",
    "                                            logging=self.logging))\n",
    "\n",
    "        self.layers.append(GraphConvolution(input_dim=hidden1,\n",
    "                                            output_dim=self.output_dim,\n",
    "                                            placeholders=self.placeholders,\n",
    "                                            act=lambda x: x, #直接输出\n",
    "                                            dropout=True,\n",
    "                                            logging=self.logging))\n",
    "\n",
    "    def predict(self):\n",
    "        return tf.nn.softmax(self.outputs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-05-15T01:20:35.014898Z",
     "start_time": "2020-05-15T01:20:34.660381Z"
    }
   },
   "outputs": [],
   "source": [
    "seed = 123\n",
    "np.random.seed(seed)\n",
    "tf.set_random_seed(seed)\n",
    "\n",
    "dataset_str  = \"cora\" # 数据集名称\n",
    "model = \"gcn\" # 运行的模型 ： gcn，  mlp ， gcn_cheby\n",
    "hidden1 = 16  # 'Number of units in hidden layer 1.'\n",
    "learning_rate = 0.01 \n",
    "weight_decay =  5e-4 # 'Weight for L2 loss on embedding matrix.'\n",
    "epochs = 200\n",
    "early_stopping = 10\n",
    "dropout = 0.5\n",
    "max_degree = 3  # 'Maximum Chebyshev polynomial degree.'\n",
    "\n",
    "# Load data\n",
    "adj, features, y_train, y_val, y_test, train_mask, val_mask, test_mask = load_data(dataset_str)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-05-15T01:20:35.792654Z",
     "start_time": "2020-05-15T01:20:35.747274Z"
    }
   },
   "outputs": [],
   "source": [
    "# Some preprocessing\n",
    "features = preprocess_features(features)\n",
    "if model == 'gcn':\n",
    "    support = [preprocess_adj(adj)]\n",
    "    num_supports = 1\n",
    "    model_func = GCN\n",
    "elif model == 'gcn_cheby':\n",
    "    support = chebyshev_polynomials(adj, FLAGS.max_degree)\n",
    "    num_supports = 1 + max_degree\n",
    "    model_func = GCN\n",
    "elif model == 'dense':\n",
    "    support = [preprocess_adj(adj)]  # Not used\n",
    "    num_supports = 1\n",
    "    model_func = MLP\n",
    "else:\n",
    "    raise ValueError('Invalid argument for model: ' + str(model))\n",
    "    \n",
    "\n",
    "# Define placeholders\n",
    "placeholders = {\n",
    "    'support': [tf.sparse_placeholder(tf.float32) for _ in range(num_supports)],\n",
    "    'features': tf.sparse_placeholder(tf.float32, shape=tf.constant(features[2], dtype=tf.int64)),\n",
    "    'labels': tf.placeholder(tf.float32, shape=(None, y_train.shape[1])),\n",
    "    'labels_mask': tf.placeholder(tf.int32),\n",
    "    'dropout': tf.placeholder_with_default(0., shape=()),\n",
    "    'num_features_nonzero': tf.placeholder(tf.int32)  # helper variable for sparse dropout\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-05-15T01:32:01.273501Z",
     "start_time": "2020-05-15T01:32:00.933168Z"
    }
   },
   "outputs": [],
   "source": [
    "# Create model\n",
    "model = model_func(placeholders, input_dim=features[2][1], logging=True)\n",
    "\n",
    "# Initialize session\n",
    "sess = tf.Session()\n",
    "\n",
    "\n",
    "# Define model evaluation function\n",
    "def evaluate(features, support, labels, mask, placeholders):\n",
    "    t_test = time.time()\n",
    "    feed_dict_val = construct_feed_dict(features, support, labels, mask, placeholders)\n",
    "    outs_val = sess.run([model.loss, model.accuracy], feed_dict=feed_dict_val)\n",
    "    return outs_val[0], outs_val[1], (time.time() - t_test)\n",
    "\n",
    "\n",
    "# Init variables\n",
    "sess.run(tf.global_variables_initializer())\n",
    "\n",
    "cost_val = []"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-05-15T01:32:07.931857Z",
     "start_time": "2020-05-15T01:32:02.433086Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 0001 train_loss= 1.95458 train_acc= 0.10714 val_loss= 1.94950 val_acc= 0.39000 time= 0.06713\n",
      "Epoch: 0002 train_loss= 1.94748 train_acc= 0.43571 val_loss= 1.94553 val_acc= 0.56800 time= 0.01894\n",
      "Epoch: 0003 train_loss= 1.94132 train_acc= 0.67143 val_loss= 1.94109 val_acc= 0.64000 time= 0.01481\n",
      "Epoch: 0004 train_loss= 1.93338 train_acc= 0.79286 val_loss= 1.93655 val_acc= 0.68400 time= 0.01547\n",
      "Epoch: 0005 train_loss= 1.92743 train_acc= 0.75714 val_loss= 1.93228 val_acc= 0.71000 time= 0.01654\n",
      "Epoch: 0006 train_loss= 1.91884 train_acc= 0.79286 val_loss= 1.92827 val_acc= 0.70400 time= 0.02040\n",
      "Epoch: 0007 train_loss= 1.91248 train_acc= 0.82857 val_loss= 1.92439 val_acc= 0.70600 time= 0.01998\n",
      "Epoch: 0008 train_loss= 1.90539 train_acc= 0.87857 val_loss= 1.92055 val_acc= 0.69800 time= 0.01909\n",
      "Epoch: 0009 train_loss= 1.89513 train_acc= 0.82857 val_loss= 1.91658 val_acc= 0.69600 time= 0.02018\n",
      "Epoch: 0010 train_loss= 1.89074 train_acc= 0.81429 val_loss= 1.91250 val_acc= 0.69600 time= 0.02083\n",
      "Epoch: 0011 train_loss= 1.88218 train_acc= 0.82143 val_loss= 1.90832 val_acc= 0.69200 time= 0.01958\n",
      "Epoch: 0012 train_loss= 1.87563 train_acc= 0.83571 val_loss= 1.90399 val_acc= 0.69200 time= 0.01912\n",
      "Epoch: 0013 train_loss= 1.86885 train_acc= 0.85000 val_loss= 1.89957 val_acc= 0.69600 time= 0.01839\n",
      "Epoch: 0014 train_loss= 1.85784 train_acc= 0.87143 val_loss= 1.89499 val_acc= 0.69800 time= 0.01830\n",
      "Epoch: 0015 train_loss= 1.84582 train_acc= 0.83571 val_loss= 1.89028 val_acc= 0.69600 time= 0.01548\n",
      "Epoch: 0016 train_loss= 1.83002 train_acc= 0.82857 val_loss= 1.88543 val_acc= 0.69200 time= 0.01813\n",
      "Epoch: 0017 train_loss= 1.82234 train_acc= 0.88571 val_loss= 1.88041 val_acc= 0.69200 time= 0.01813\n",
      "Epoch: 0018 train_loss= 1.82471 train_acc= 0.85000 val_loss= 1.87528 val_acc= 0.68800 time= 0.01574\n",
      "Epoch: 0019 train_loss= 1.79054 train_acc= 0.87143 val_loss= 1.87002 val_acc= 0.68800 time= 0.02025\n",
      "Epoch: 0020 train_loss= 1.79778 train_acc= 0.84286 val_loss= 1.86466 val_acc= 0.68800 time= 0.01998\n",
      "Epoch: 0021 train_loss= 1.77844 train_acc= 0.89286 val_loss= 1.85925 val_acc= 0.68800 time= 0.02140\n",
      "Epoch: 0022 train_loss= 1.78256 train_acc= 0.87143 val_loss= 1.85388 val_acc= 0.68600 time= 0.02047\n",
      "Epoch: 0023 train_loss= 1.75667 train_acc= 0.81429 val_loss= 1.84835 val_acc= 0.68800 time= 0.02363\n",
      "Epoch: 0024 train_loss= 1.74874 train_acc= 0.87143 val_loss= 1.84263 val_acc= 0.68800 time= 0.02309\n",
      "Epoch: 0025 train_loss= 1.73807 train_acc= 0.82857 val_loss= 1.83677 val_acc= 0.68800 time= 0.01808\n",
      "Epoch: 0026 train_loss= 1.72955 train_acc= 0.83571 val_loss= 1.83072 val_acc= 0.68600 time= 0.01984\n",
      "Epoch: 0027 train_loss= 1.72157 train_acc= 0.84286 val_loss= 1.82438 val_acc= 0.68600 time= 0.01938\n",
      "Epoch: 0028 train_loss= 1.71220 train_acc= 0.82143 val_loss= 1.81805 val_acc= 0.68400 time= 0.01894\n",
      "Epoch: 0029 train_loss= 1.68899 train_acc= 0.87857 val_loss= 1.81161 val_acc= 0.68600 time= 0.01869\n",
      "Epoch: 0030 train_loss= 1.66747 train_acc= 0.85714 val_loss= 1.80494 val_acc= 0.68800 time= 0.02098\n",
      "Epoch: 0031 train_loss= 1.66143 train_acc= 0.83571 val_loss= 1.79812 val_acc= 0.68400 time= 0.01868\n",
      "Epoch: 0032 train_loss= 1.62480 train_acc= 0.82857 val_loss= 1.79120 val_acc= 0.68400 time= 0.02535\n",
      "Epoch: 0033 train_loss= 1.63022 train_acc= 0.85714 val_loss= 1.78417 val_acc= 0.68000 time= 0.02465\n",
      "Epoch: 0034 train_loss= 1.62526 train_acc= 0.85000 val_loss= 1.77694 val_acc= 0.68200 time= 0.02350\n",
      "Epoch: 0035 train_loss= 1.60572 train_acc= 0.87857 val_loss= 1.76958 val_acc= 0.67800 time= 0.01825\n",
      "Epoch: 0036 train_loss= 1.58899 train_acc= 0.85000 val_loss= 1.76213 val_acc= 0.68400 time= 0.01923\n",
      "Epoch: 0037 train_loss= 1.58538 train_acc= 0.85714 val_loss= 1.75465 val_acc= 0.68600 time= 0.01798\n",
      "Epoch: 0038 train_loss= 1.55807 train_acc= 0.88571 val_loss= 1.74711 val_acc= 0.69000 time= 0.01789\n",
      "Epoch: 0039 train_loss= 1.55422 train_acc= 0.85714 val_loss= 1.73979 val_acc= 0.69000 time= 0.01930\n",
      "Epoch: 0040 train_loss= 1.54126 train_acc= 0.88571 val_loss= 1.73255 val_acc= 0.69200 time= 0.01675\n",
      "Epoch: 0041 train_loss= 1.52030 train_acc= 0.86429 val_loss= 1.72505 val_acc= 0.69600 time= 0.01943\n",
      "Epoch: 0042 train_loss= 1.51643 train_acc= 0.81429 val_loss= 1.71731 val_acc= 0.69600 time= 0.01854\n",
      "Epoch: 0043 train_loss= 1.46224 train_acc= 0.89286 val_loss= 1.70949 val_acc= 0.69800 time= 0.01896\n",
      "Epoch: 0044 train_loss= 1.46988 train_acc= 0.85714 val_loss= 1.70148 val_acc= 0.70200 time= 0.02425\n",
      "Epoch: 0045 train_loss= 1.47736 train_acc= 0.86429 val_loss= 1.69319 val_acc= 0.70000 time= 0.02434\n",
      "Epoch: 0046 train_loss= 1.48500 train_acc= 0.83571 val_loss= 1.68500 val_acc= 0.70000 time= 0.01991\n",
      "Epoch: 0047 train_loss= 1.42830 train_acc= 0.87857 val_loss= 1.67655 val_acc= 0.70200 time= 0.01878\n",
      "Epoch: 0048 train_loss= 1.38388 train_acc= 0.90000 val_loss= 1.66801 val_acc= 0.70600 time= 0.01896\n",
      "Epoch: 0049 train_loss= 1.38730 train_acc= 0.85714 val_loss= 1.65946 val_acc= 0.70600 time= 0.01894\n",
      "Epoch: 0050 train_loss= 1.40571 train_acc= 0.86429 val_loss= 1.65097 val_acc= 0.71400 time= 0.01911\n",
      "Epoch: 0051 train_loss= 1.35419 train_acc= 0.87857 val_loss= 1.64265 val_acc= 0.71600 time= 0.01874\n",
      "Epoch: 0052 train_loss= 1.35605 train_acc= 0.90000 val_loss= 1.63450 val_acc= 0.71600 time= 0.01969\n",
      "Epoch: 0053 train_loss= 1.35691 train_acc= 0.87857 val_loss= 1.62628 val_acc= 0.72400 time= 0.02005\n",
      "Epoch: 0054 train_loss= 1.34878 train_acc= 0.86429 val_loss= 1.61838 val_acc= 0.73000 time= 0.03005\n",
      "Epoch: 0055 train_loss= 1.34968 train_acc= 0.91429 val_loss= 1.61071 val_acc= 0.73400 time= 0.04152\n",
      "Epoch: 0056 train_loss= 1.29555 train_acc= 0.86429 val_loss= 1.60314 val_acc= 0.73600 time= 0.03144\n",
      "Epoch: 0057 train_loss= 1.31715 train_acc= 0.92857 val_loss= 1.59601 val_acc= 0.73600 time= 0.03419\n",
      "Epoch: 0058 train_loss= 1.25659 train_acc= 0.92143 val_loss= 1.58874 val_acc= 0.73600 time= 0.02562\n",
      "Epoch: 0059 train_loss= 1.27249 train_acc= 0.89286 val_loss= 1.58159 val_acc= 0.74000 time= 0.02950\n",
      "Epoch: 0060 train_loss= 1.29297 train_acc= 0.86429 val_loss= 1.57459 val_acc= 0.74400 time= 0.02396\n",
      "Epoch: 0061 train_loss= 1.22268 train_acc= 0.92143 val_loss= 1.56742 val_acc= 0.74600 time= 0.02966\n",
      "Epoch: 0062 train_loss= 1.25740 train_acc= 0.84286 val_loss= 1.56021 val_acc= 0.74400 time= 0.02359\n",
      "Epoch: 0063 train_loss= 1.25869 train_acc= 0.91429 val_loss= 1.55301 val_acc= 0.74600 time= 0.02316\n",
      "Epoch: 0064 train_loss= 1.17654 train_acc= 0.90000 val_loss= 1.54562 val_acc= 0.75000 time= 0.01955\n",
      "Epoch: 0065 train_loss= 1.12229 train_acc= 0.92857 val_loss= 1.53824 val_acc= 0.75200 time= 0.01966\n",
      "Epoch: 0066 train_loss= 1.19170 train_acc= 0.87857 val_loss= 1.53109 val_acc= 0.75200 time= 0.01825\n",
      "Epoch: 0067 train_loss= 1.18527 train_acc= 0.89286 val_loss= 1.52392 val_acc= 0.75200 time= 0.01814\n",
      "Epoch: 0068 train_loss= 1.15252 train_acc= 0.89286 val_loss= 1.51683 val_acc= 0.75200 time= 0.01818\n",
      "Epoch: 0069 train_loss= 1.14345 train_acc= 0.90714 val_loss= 1.50951 val_acc= 0.75600 time= 0.01924\n",
      "Epoch: 0070 train_loss= 1.14675 train_acc= 0.94286 val_loss= 1.50190 val_acc= 0.76000 time= 0.01879\n",
      "Epoch: 0071 train_loss= 1.16372 train_acc= 0.89286 val_loss= 1.49408 val_acc= 0.76200 time= 0.01894\n",
      "Epoch: 0072 train_loss= 1.14340 train_acc= 0.90000 val_loss= 1.48618 val_acc= 0.76800 time= 0.02248\n",
      "Epoch: 0073 train_loss= 1.07612 train_acc= 0.92143 val_loss= 1.47809 val_acc= 0.76800 time= 0.02379\n",
      "Epoch: 0074 train_loss= 1.09686 train_acc= 0.89286 val_loss= 1.46983 val_acc= 0.76800 time= 0.02530\n",
      "Epoch: 0075 train_loss= 1.14251 train_acc= 0.92143 val_loss= 1.46187 val_acc= 0.76800 time= 0.02016\n",
      "Epoch: 0076 train_loss= 1.10422 train_acc= 0.94286 val_loss= 1.45429 val_acc= 0.76800 time= 0.02104\n",
      "Epoch: 0077 train_loss= 1.07875 train_acc= 0.90000 val_loss= 1.44702 val_acc= 0.77000 time= 0.02262\n",
      "Epoch: 0078 train_loss= 1.06569 train_acc= 0.92857 val_loss= 1.44002 val_acc= 0.77200 time= 0.01915\n",
      "Epoch: 0079 train_loss= 1.04504 train_acc= 0.94286 val_loss= 1.43324 val_acc= 0.77800 time= 0.01897\n",
      "Epoch: 0080 train_loss= 0.98834 train_acc= 0.92857 val_loss= 1.42646 val_acc= 0.77600 time= 0.02019\n",
      "Epoch: 0081 train_loss= 1.01951 train_acc= 0.92143 val_loss= 1.41993 val_acc= 0.77600 time= 0.01922\n",
      "Epoch: 0082 train_loss= 1.04933 train_acc= 0.89286 val_loss= 1.41386 val_acc= 0.78200 time= 0.02324\n",
      "Epoch: 0083 train_loss= 1.03624 train_acc= 0.92857 val_loss= 1.40837 val_acc= 0.78400 time= 0.02397\n",
      "Epoch: 0084 train_loss= 1.04161 train_acc= 0.92143 val_loss= 1.40333 val_acc= 0.78000 time= 0.02176\n",
      "Epoch: 0085 train_loss= 0.98835 train_acc= 0.92857 val_loss= 1.39846 val_acc= 0.78800 time= 0.01976\n",
      "Epoch: 0086 train_loss= 1.03919 train_acc= 0.91429 val_loss= 1.39374 val_acc= 0.79000 time= 0.02524\n",
      "Epoch: 0087 train_loss= 0.99760 train_acc= 0.92857 val_loss= 1.38933 val_acc= 0.79000 time= 0.02536\n",
      "Epoch: 0088 train_loss= 0.97791 train_acc= 0.91429 val_loss= 1.38482 val_acc= 0.79000 time= 0.02751\n",
      "Epoch: 0089 train_loss= 0.96486 train_acc= 0.93571 val_loss= 1.38029 val_acc= 0.79000 time= 0.02770\n",
      "Epoch: 0090 train_loss= 0.96242 train_acc= 0.91429 val_loss= 1.37556 val_acc= 0.78600 time= 0.02618\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 0091 train_loss= 0.96822 train_acc= 0.95714 val_loss= 1.37101 val_acc= 0.78600 time= 0.03068\n",
      "Epoch: 0092 train_loss= 0.96496 train_acc= 0.93571 val_loss= 1.36665 val_acc= 0.78400 time= 0.03010\n",
      "Epoch: 0093 train_loss= 0.92467 train_acc= 0.93571 val_loss= 1.36190 val_acc= 0.78000 time= 0.02945\n",
      "Epoch: 0094 train_loss= 0.93653 train_acc= 0.90714 val_loss= 1.35712 val_acc= 0.78000 time= 0.02081\n",
      "Epoch: 0095 train_loss= 0.92579 train_acc= 0.94286 val_loss= 1.35189 val_acc= 0.78000 time= 0.01991\n",
      "Epoch: 0096 train_loss= 0.87263 train_acc= 0.96429 val_loss= 1.34630 val_acc= 0.78200 time= 0.02149\n",
      "Epoch: 0097 train_loss= 0.91380 train_acc= 0.95714 val_loss= 1.34096 val_acc= 0.78400 time= 0.02338\n",
      "Epoch: 0098 train_loss= 0.93240 train_acc= 0.92143 val_loss= 1.33601 val_acc= 0.78600 time= 0.02202\n",
      "Epoch: 0099 train_loss= 0.90114 train_acc= 0.94286 val_loss= 1.33095 val_acc= 0.78800 time= 0.01792\n",
      "Epoch: 0100 train_loss= 0.92935 train_acc= 0.92857 val_loss= 1.32548 val_acc= 0.78800 time= 0.02026\n",
      "Epoch: 0101 train_loss= 0.93134 train_acc= 0.92857 val_loss= 1.31998 val_acc= 0.78800 time= 0.02412\n",
      "Epoch: 0102 train_loss= 0.94390 train_acc= 0.94286 val_loss= 1.31463 val_acc= 0.78800 time= 0.02275\n",
      "Epoch: 0103 train_loss= 0.91915 train_acc= 0.94286 val_loss= 1.30968 val_acc= 0.78600 time= 0.01958\n",
      "Epoch: 0104 train_loss= 0.86073 train_acc= 0.91429 val_loss= 1.30536 val_acc= 0.78800 time= 0.01989\n",
      "Epoch: 0105 train_loss= 0.93608 train_acc= 0.92143 val_loss= 1.30125 val_acc= 0.78800 time= 0.01890\n",
      "Epoch: 0106 train_loss= 0.90048 train_acc= 0.93571 val_loss= 1.29703 val_acc= 0.78800 time= 0.02132\n",
      "Epoch: 0107 train_loss= 0.94442 train_acc= 0.90714 val_loss= 1.29260 val_acc= 0.79000 time= 0.01952\n",
      "Epoch: 0108 train_loss= 0.86961 train_acc= 0.95714 val_loss= 1.28804 val_acc= 0.79000 time= 0.01835\n",
      "Epoch: 0109 train_loss= 0.89840 train_acc= 0.92857 val_loss= 1.28333 val_acc= 0.79000 time= 0.02874\n",
      "Epoch: 0110 train_loss= 0.93568 train_acc= 0.91429 val_loss= 1.27887 val_acc= 0.79000 time= 0.03030\n",
      "Epoch: 0111 train_loss= 0.87120 train_acc= 0.93571 val_loss= 1.27483 val_acc= 0.79200 time= 0.02690\n",
      "Epoch: 0112 train_loss= 0.88879 train_acc= 0.94286 val_loss= 1.27075 val_acc= 0.79400 time= 0.02624\n",
      "Epoch: 0113 train_loss= 0.84777 train_acc= 0.95714 val_loss= 1.26670 val_acc= 0.79400 time= 0.02326\n",
      "Epoch: 0114 train_loss= 0.84818 train_acc= 0.95000 val_loss= 1.26281 val_acc= 0.79400 time= 0.02195\n",
      "Epoch: 0115 train_loss= 0.84114 train_acc= 0.95714 val_loss= 1.25974 val_acc= 0.79200 time= 0.02087\n",
      "Epoch: 0116 train_loss= 0.83623 train_acc= 0.94286 val_loss= 1.25707 val_acc= 0.79200 time= 0.01885\n",
      "Epoch: 0117 train_loss= 0.82652 train_acc= 0.98571 val_loss= 1.25483 val_acc= 0.79200 time= 0.02157\n",
      "Epoch: 0118 train_loss= 0.84232 train_acc= 0.93571 val_loss= 1.25316 val_acc= 0.79200 time= 0.01886\n",
      "Epoch: 0119 train_loss= 0.79023 train_acc= 0.97143 val_loss= 1.25105 val_acc= 0.79000 time= 0.01968\n",
      "Epoch: 0120 train_loss= 0.82365 train_acc= 0.94286 val_loss= 1.24886 val_acc= 0.78800 time= 0.02327\n",
      "Epoch: 0121 train_loss= 0.83400 train_acc= 0.95000 val_loss= 1.24626 val_acc= 0.78800 time= 0.05223\n",
      "Epoch: 0122 train_loss= 0.81664 train_acc= 0.94286 val_loss= 1.24372 val_acc= 0.78800 time= 0.02407\n",
      "Epoch: 0123 train_loss= 0.85580 train_acc= 0.94286 val_loss= 1.24075 val_acc= 0.78600 time= 0.01806\n",
      "Epoch: 0124 train_loss= 0.83183 train_acc= 0.92143 val_loss= 1.23782 val_acc= 0.78600 time= 0.01782\n",
      "Epoch: 0125 train_loss= 0.83126 train_acc= 0.94286 val_loss= 1.23536 val_acc= 0.78200 time= 0.01747\n",
      "Epoch: 0126 train_loss= 0.77115 train_acc= 0.95714 val_loss= 1.23284 val_acc= 0.78400 time= 0.01857\n",
      "Epoch: 0127 train_loss= 0.78558 train_acc= 0.97143 val_loss= 1.23054 val_acc= 0.78400 time= 0.01925\n",
      "Epoch: 0128 train_loss= 0.79135 train_acc= 0.95000 val_loss= 1.22742 val_acc= 0.78400 time= 0.01905\n",
      "Epoch: 0129 train_loss= 0.78823 train_acc= 0.93571 val_loss= 1.22408 val_acc= 0.78600 time= 0.01938\n",
      "Epoch: 0130 train_loss= 0.79319 train_acc= 0.93571 val_loss= 1.22054 val_acc= 0.78600 time= 0.02226\n",
      "Epoch: 0131 train_loss= 0.82530 train_acc= 0.96429 val_loss= 1.21673 val_acc= 0.78400 time= 0.02067\n",
      "Epoch: 0132 train_loss= 0.75425 train_acc= 0.95000 val_loss= 1.21291 val_acc= 0.78400 time= 0.01861\n",
      "Epoch: 0133 train_loss= 0.73965 train_acc= 0.92143 val_loss= 1.20901 val_acc= 0.78800 time= 0.02095\n",
      "Epoch: 0134 train_loss= 0.74322 train_acc= 0.95714 val_loss= 1.20522 val_acc= 0.78800 time= 0.02083\n",
      "Epoch: 0135 train_loss= 0.79951 train_acc= 0.93571 val_loss= 1.20109 val_acc= 0.78800 time= 0.01992\n",
      "Epoch: 0136 train_loss= 0.76699 train_acc= 0.93571 val_loss= 1.19696 val_acc= 0.78800 time= 0.02300\n",
      "Epoch: 0137 train_loss= 0.80343 train_acc= 0.92857 val_loss= 1.19299 val_acc= 0.78600 time= 0.02066\n",
      "Epoch: 0138 train_loss= 0.76365 train_acc= 0.97143 val_loss= 1.18940 val_acc= 0.78600 time= 0.02013\n",
      "Epoch: 0139 train_loss= 0.75509 train_acc= 0.94286 val_loss= 1.18595 val_acc= 0.78800 time= 0.02004\n",
      "Epoch: 0140 train_loss= 0.77060 train_acc= 0.94286 val_loss= 1.18269 val_acc= 0.78800 time= 0.02420\n",
      "Epoch: 0141 train_loss= 0.77720 train_acc= 0.91429 val_loss= 1.18000 val_acc= 0.78800 time= 0.02097\n",
      "Epoch: 0142 train_loss= 0.71778 train_acc= 0.96429 val_loss= 1.17785 val_acc= 0.78800 time= 0.02230\n",
      "Epoch: 0143 train_loss= 0.72756 train_acc= 0.95714 val_loss= 1.17612 val_acc= 0.79400 time= 0.02105\n",
      "Epoch: 0144 train_loss= 0.73090 train_acc= 0.95714 val_loss= 1.17487 val_acc= 0.79200 time= 0.02209\n",
      "Epoch: 0145 train_loss= 0.67870 train_acc= 0.95000 val_loss= 1.17409 val_acc= 0.79200 time= 0.02558\n",
      "Epoch: 0146 train_loss= 0.71581 train_acc= 0.98571 val_loss= 1.17300 val_acc= 0.79000 time= 0.02404\n",
      "Epoch: 0147 train_loss= 0.75135 train_acc= 0.95000 val_loss= 1.17167 val_acc= 0.78800 time= 0.02381\n",
      "Epoch: 0148 train_loss= 0.74662 train_acc= 0.93571 val_loss= 1.17062 val_acc= 0.78800 time= 0.02387\n",
      "Epoch: 0149 train_loss= 0.67471 train_acc= 0.96429 val_loss= 1.16963 val_acc= 0.78800 time= 0.03060\n",
      "Epoch: 0150 train_loss= 0.75504 train_acc= 0.95714 val_loss= 1.16828 val_acc= 0.78600 time= 0.02291\n",
      "Epoch: 0151 train_loss= 0.72711 train_acc= 0.96429 val_loss= 1.16623 val_acc= 0.78600 time= 0.02330\n",
      "Epoch: 0152 train_loss= 0.72794 train_acc= 0.97143 val_loss= 1.16442 val_acc= 0.78200 time= 0.02504\n",
      "Epoch: 0153 train_loss= 0.71454 train_acc= 0.94286 val_loss= 1.16289 val_acc= 0.77800 time= 0.02183\n",
      "Epoch: 0154 train_loss= 0.75475 train_acc= 0.93571 val_loss= 1.16127 val_acc= 0.77800 time= 0.01770\n",
      "Epoch: 0155 train_loss= 0.69798 train_acc= 0.96429 val_loss= 1.15915 val_acc= 0.78000 time= 0.01966\n",
      "Epoch: 0156 train_loss= 0.74332 train_acc= 0.95714 val_loss= 1.15752 val_acc= 0.78000 time= 0.02073\n",
      "Epoch: 0157 train_loss= 0.71962 train_acc= 0.92857 val_loss= 1.15610 val_acc= 0.78000 time= 0.01975\n",
      "Epoch: 0158 train_loss= 0.71031 train_acc= 0.91429 val_loss= 1.15416 val_acc= 0.78000 time= 0.02327\n",
      "Epoch: 0159 train_loss= 0.69728 train_acc= 0.95714 val_loss= 1.15101 val_acc= 0.78000 time= 0.02475\n",
      "Epoch: 0160 train_loss= 0.72640 train_acc= 0.97143 val_loss= 1.14736 val_acc= 0.77800 time= 0.02403\n",
      "Epoch: 0161 train_loss= 0.70509 train_acc= 0.95000 val_loss= 1.14344 val_acc= 0.77800 time= 0.02638\n",
      "Epoch: 0162 train_loss= 0.71715 train_acc= 0.94286 val_loss= 1.13968 val_acc= 0.78400 time= 0.02201\n",
      "Epoch: 0163 train_loss= 0.71584 train_acc= 0.92143 val_loss= 1.13671 val_acc= 0.78400 time= 0.02004\n",
      "Epoch: 0164 train_loss= 0.71428 train_acc= 0.92143 val_loss= 1.13411 val_acc= 0.78800 time= 0.02020\n",
      "Epoch: 0165 train_loss= 0.70568 train_acc= 0.94286 val_loss= 1.13157 val_acc= 0.78800 time= 0.01851\n",
      "Epoch: 0166 train_loss= 0.69683 train_acc= 0.92143 val_loss= 1.12913 val_acc= 0.78800 time= 0.01836\n",
      "Epoch: 0167 train_loss= 0.70286 train_acc= 0.95714 val_loss= 1.12714 val_acc= 0.78800 time= 0.02202\n",
      "Epoch: 0168 train_loss= 0.68873 train_acc= 0.94286 val_loss= 1.12563 val_acc= 0.78800 time= 0.02207\n",
      "Epoch: 0169 train_loss= 0.68523 train_acc= 0.95714 val_loss= 1.12431 val_acc= 0.78600 time= 0.02317\n",
      "Epoch: 0170 train_loss= 0.67896 train_acc= 0.94286 val_loss= 1.12321 val_acc= 0.78200 time= 0.02287\n",
      "Epoch: 0171 train_loss= 0.70202 train_acc= 0.95714 val_loss= 1.12284 val_acc= 0.77800 time= 0.02479\n",
      "Epoch: 0172 train_loss= 0.66276 train_acc= 0.94286 val_loss= 1.12214 val_acc= 0.77800 time= 0.02572\n",
      "Epoch: 0173 train_loss= 0.68523 train_acc= 0.93571 val_loss= 1.12122 val_acc= 0.78000 time= 0.02683\n",
      "Epoch: 0174 train_loss= 0.67167 train_acc= 0.94286 val_loss= 1.11959 val_acc= 0.78000 time= 0.02372\n",
      "Epoch: 0175 train_loss= 0.68665 train_acc= 0.94286 val_loss= 1.11809 val_acc= 0.78200 time= 0.02537\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 0176 train_loss= 0.67686 train_acc= 0.97143 val_loss= 1.11638 val_acc= 0.78400 time= 0.02644\n",
      "Epoch: 0177 train_loss= 0.62991 train_acc= 0.98571 val_loss= 1.11440 val_acc= 0.78200 time= 0.02898\n",
      "Epoch: 0178 train_loss= 0.67007 train_acc= 0.92857 val_loss= 1.11259 val_acc= 0.78600 time= 0.02478\n",
      "Epoch: 0179 train_loss= 0.64429 train_acc= 0.97143 val_loss= 1.11107 val_acc= 0.79000 time= 0.01974\n",
      "Epoch: 0180 train_loss= 0.61495 train_acc= 0.95714 val_loss= 1.10932 val_acc= 0.78800 time= 0.02029\n",
      "Epoch: 0181 train_loss= 0.60840 train_acc= 0.98571 val_loss= 1.10770 val_acc= 0.78800 time= 0.02288\n",
      "Epoch: 0182 train_loss= 0.59963 train_acc= 0.97857 val_loss= 1.10590 val_acc= 0.78800 time= 0.02160\n",
      "Epoch: 0183 train_loss= 0.66780 train_acc= 0.97143 val_loss= 1.10384 val_acc= 0.78800 time= 0.02194\n",
      "Epoch: 0184 train_loss= 0.60548 train_acc= 0.97857 val_loss= 1.10159 val_acc= 0.78800 time= 0.02404\n",
      "Epoch: 0185 train_loss= 0.66870 train_acc= 0.93571 val_loss= 1.09973 val_acc= 0.78800 time= 0.02245\n",
      "Epoch: 0186 train_loss= 0.62815 train_acc= 0.97857 val_loss= 1.09784 val_acc= 0.78800 time= 0.94728\n",
      "Epoch: 0187 train_loss= 0.59612 train_acc= 0.98571 val_loss= 1.09547 val_acc= 0.78600 time= 0.02191\n",
      "Epoch: 0188 train_loss= 0.63757 train_acc= 0.97143 val_loss= 1.09209 val_acc= 0.78800 time= 0.02334\n",
      "Epoch: 0189 train_loss= 0.66404 train_acc= 0.95000 val_loss= 1.08877 val_acc= 0.78600 time= 0.02526\n",
      "Epoch: 0190 train_loss= 0.65605 train_acc= 0.96429 val_loss= 1.08569 val_acc= 0.78600 time= 0.02581\n",
      "Epoch: 0191 train_loss= 0.63853 train_acc= 0.95714 val_loss= 1.08299 val_acc= 0.78400 time= 0.02451\n",
      "Epoch: 0192 train_loss= 0.61456 train_acc= 0.95714 val_loss= 1.08026 val_acc= 0.78600 time= 0.02324\n",
      "Epoch: 0193 train_loss= 0.71255 train_acc= 0.94286 val_loss= 1.07841 val_acc= 0.78600 time= 0.02184\n",
      "Epoch: 0194 train_loss= 0.69692 train_acc= 0.94286 val_loss= 1.07745 val_acc= 0.78800 time= 0.02412\n",
      "Epoch: 0195 train_loss= 0.64717 train_acc= 0.92143 val_loss= 1.07713 val_acc= 0.78800 time= 0.02154\n",
      "Epoch: 0196 train_loss= 0.63200 train_acc= 0.97857 val_loss= 1.07703 val_acc= 0.78600 time= 0.01970\n",
      "Epoch: 0197 train_loss= 0.63114 train_acc= 0.96429 val_loss= 1.07704 val_acc= 0.78600 time= 0.02100\n",
      "Epoch: 0198 train_loss= 0.61354 train_acc= 0.97857 val_loss= 1.07747 val_acc= 0.78600 time= 0.02387\n",
      "Epoch: 0199 train_loss= 0.63067 train_acc= 0.95714 val_loss= 1.07763 val_acc= 0.78600 time= 0.02585\n",
      "Epoch: 0200 train_loss= 0.60992 train_acc= 0.96429 val_loss= 1.07801 val_acc= 0.79000 time= 0.02375\n",
      "Optimization Finished!\n",
      "Test set results: cost= 1.04572 accuracy= 0.81500 time= 0.01091\n"
     ]
    }
   ],
   "source": [
    "# Train model\n",
    "for epoch in range(epochs):\n",
    "\n",
    "    t = time.time()\n",
    "    # Construct feed dictionary\n",
    "    feed_dict = construct_feed_dict(features, support, y_train, train_mask, placeholders)\n",
    "    feed_dict.update({placeholders['dropout']: dropout}) # 为什么 dropout 单独列出来\n",
    "\n",
    "    # Training step\n",
    "    outs = sess.run([model.opt_op, model.loss, model.accuracy], feed_dict=feed_dict)\n",
    "\n",
    "    # Validation\n",
    "    cost, acc, duration = evaluate(features, support, y_val, val_mask, placeholders)\n",
    "    cost_val.append(cost)\n",
    "\n",
    "    # Print results\n",
    "    print(\"Epoch:\", '%04d' % (epoch + 1), \"train_loss=\", \"{:.5f}\".format(outs[1]),\n",
    "          \"train_acc=\", \"{:.5f}\".format(outs[2]), \"val_loss=\", \"{:.5f}\".format(cost),\n",
    "          \"val_acc=\", \"{:.5f}\".format(acc), \"time=\", \"{:.5f}\".format(time.time() - t))\n",
    "\n",
    "    if epoch > early_stopping and cost_val[-1] > np.mean(cost_val[-(early_stopping+1):-1]):\n",
    "        print(\"Early stopping...\")\n",
    "        break\n",
    "\n",
    "print(\"Optimization Finished!\")\n",
    "\n",
    "# Testing\n",
    "test_cost, test_acc, test_duration = evaluate(features, support, y_test, test_mask, placeholders)\n",
    "print(\"Test set results:\", \"cost=\", \"{:.5f}\".format(test_cost),\n",
    "      \"accuracy=\", \"{:.5f}\".format(test_acc), \"time=\", \"{:.5f}\".format(test_duration))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.10"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": true,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "oldHeight": 545,
   "position": {
    "height": "40px",
    "left": "1082px",
    "right": "20px",
    "top": "32px",
    "width": "341px"
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "varInspector_section_display": "none",
   "window_display": true
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}