NLP/text_clf/TextCNN.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.5/dist-packages/h5py/__init__.py:36: FutureWarning: Conversion of the second argument of issubdtype from `float` to `np.floating` is deprecated. In future, it will be treated as `np.float64 == np.dtype(float).type`.\n",
      "  from ._conv import register_converters as _register_converters\n"
     ]
    }
   ],
   "source": [
    "import tensorflow as tf\n",
    "import numpy as np\n",
    "import math\n",
    "import os\n",
    "os.sys.path.append(os.path.dirname(os.path.abspath('..')))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "WARNING: The TensorFlow contrib module will not be included in TensorFlow 2.0.\n",
      "For more information, please see:\n",
      "  * https://github.com/tensorflow/community/blob/master/rfcs/20180907-contrib-sunset.md\n",
      "  * https://github.com/tensorflow/addons\n",
      "If you depend on functionality not listed there, please file an issue.\n",
      "\n"
     ]
    }
   ],
   "source": [
    "def get_default_params():\n",
    "    return tf.contrib.training.HParams(\n",
    "        emb_size=64,\n",
    "        t_size=50,\n",
    "        conv_size=3,    # 卷积核尺寸\n",
    "        filters=64,    # 卷积核数量\n",
    "        fc_size=32,\n",
    "        dropout_rate=0.5,\n",
    "        batch_size=64,\n",
    "        grad_thresh=1.0,    # 梯度阈值\n",
    "        lr=0.001,\n",
    "        cnt_thresh=10,    # 词的频率阈值\n",
    "    )\n",
    "\n",
    "\n",
    "params = get_default_params()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 数据封装"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(50000, 50) (50000,)\n",
      "(10000, 50) (10000,)\n"
     ]
    }
   ],
   "source": [
    "from dataset.dataset import load_news\n",
    "\n",
    "train_data, test_data = load_news(batch_size=params.batch_size,\n",
    "                                  cnt_thresh=params.cnt_thresh, t_size=params.t_size)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 网络搭建"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "vocal_size = train_data.voc_size\n",
    "\n",
    "unit_O = 10    # 输出单元数，类别数"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /usr/local/lib/python3.5/dist-packages/tensorflow/python/framework/op_def_library.py:263: colocate_with (from tensorflow.python.framework.ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Colocations handled automatically by placer.\n",
      "WARNING:tensorflow:From <ipython-input-5-0b6375bc2d4c>:21: conv1d (from tensorflow.python.layers.convolutional) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use keras.layers.conv1d instead.\n",
      "WARNING:tensorflow:From <ipython-input-5-0b6375bc2d4c>:25: dense (from tensorflow.python.layers.core) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use keras.layers.dense instead.\n",
      "WARNING:tensorflow:From <ipython-input-5-0b6375bc2d4c>:26: dropout (from tensorflow.python.layers.core) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use keras.layers.dropout instead.\n",
      "WARNING:tensorflow:From /usr/local/lib/python3.5/dist-packages/tensorflow/python/keras/layers/core.py:143: calling dropout (from tensorflow.python.ops.nn_ops) with keep_prob is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use `rate` instead of `keep_prob`. Rate should be set to `rate = 1 - keep_prob`.\n",
      "WARNING:tensorflow:From /usr/local/lib/python3.5/dist-packages/tensorflow/python/ops/losses/losses_impl.py:209: to_float (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use tf.cast instead.\n",
      "WARNING:tensorflow:From /usr/local/lib/python3.5/dist-packages/tensorflow/python/ops/math_ops.py:3066: to_int32 (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use tf.cast instead.\n",
      "[<tensorflow.python.framework.ops.IndexedSlices object at 0x7fcfe59d9c88>, <tf.Tensor 'train_op/gradients/CNN/conv1d/conv1d/ExpandDims_1_grad/Reshape:0' shape=(3, 64, 64) dtype=float32>, <tf.Tensor 'train_op/gradients/CNN/conv1d/BiasAdd_grad/BiasAddGrad:0' shape=(64,) dtype=float32>, <tf.Tensor 'train_op/gradients/FC/dense/MatMul_grad/MatMul_1:0' shape=(64, 32) dtype=float32>, <tf.Tensor 'train_op/gradients/FC/dense/BiasAdd_grad/BiasAddGrad:0' shape=(32,) dtype=float32>, <tf.Tensor 'train_op/gradients/dense/MatMul_grad/MatMul_1:0' shape=(32, 10) dtype=float32>, <tf.Tensor 'train_op/gradients/dense/BiasAdd_grad/BiasAddGrad:0' shape=(10,) dtype=float32>]\n"
     ]
    }
   ],
   "source": [
    "X = tf.placeholder(tf.int32, [None, params.t_size])\n",
    "Y = tf.placeholder(tf.int64, [None])\n",
    "is_training = tf.placeholder(tf.bool)    # 训练标识位\n",
    "\n",
    "# 自动计数\n",
    "global_step = tf.Variable(tf.zeros([], tf.int32),\n",
    "                          name='global_step', trainable=False)\n",
    "\n",
    "with tf.variable_scope('emb', initializer=tf.random_uniform_initializer(-1.0, 1.0)):\n",
    "    emb_lookup = tf.get_variable('embedding', [vocal_size, params.emb_size],\n",
    "                                 dtype=tf.float32)\n",
    "    # (batch_size,t_size,emb_size)\n",
    "    emb = tf.nn.embedding_lookup(emb_lookup, X)\n",
    "\n",
    "# 对embedding使用CNN网络\n",
    "xavier_scale = 1 / math.sqrt(params.emb_size + params.filters) / 3\n",
    "initializer = tf.random_uniform_initializer(-xavier_scale, xavier_scale)\n",
    "with tf.variable_scope('CNN', initializer=initializer):\n",
    "    conv_layer = tf.layers.conv1d(emb, filters=params.filters,\n",
    "                                  kernel_size=params.conv_size,\n",
    "                                  activation=tf.nn.relu)    # [None, t_size-conv_size+1, filters]\n",
    "    conv_outputs = tf.reduce_mean(conv_layer, axis=1)    # [None, filters]\n",
    "\n",
    "with tf.name_scope('FC'):\n",
    "    fc = tf.layers.dense(conv_outputs, params.fc_size, activation=tf.nn.relu)\n",
    "    fc = tf.layers.dropout(fc, rate=params.dropout_rate, training=is_training)\n",
    "\n",
    "logits = tf.layers.dense(fc, unit_O, activation=None)    # 输出层，无激活\n",
    "\n",
    "with tf.name_scope('Eval'):\n",
    "    loss = tf.losses.sparse_softmax_cross_entropy(labels=Y, logits=logits)\n",
    "    predict = tf.argmax(logits, 1)\n",
    "    accuracy = tf.reduce_mean(tf.cast(tf.equal(predict, Y), tf.float32))\n",
    "\n",
    "with tf.name_scope('train_op'):\n",
    "    lr = 1e-3\n",
    "    t_vars = tf.trainable_variables()    # 可训练变量\n",
    "    # 应用梯度截断\n",
    "    print(tf.gradients(loss, t_vars))\n",
    "    grads, _ = tf.clip_by_global_norm(tf.gradients(loss, t_vars),\n",
    "                                      params.grad_thresh)\n",
    "    optimizer = tf.train.AdamOptimizer(lr)\n",
    "    train_op = optimizer.apply_gradients(zip(grads, t_vars),\n",
    "                                         global_step=global_step)\n",
    "\n",
    "init = tf.global_variables_initializer()\n",
    "config = tf.ConfigProto()\n",
    "config.gpu_options.allow_growth = True    # 按需使用显存"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 训练网络"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 1, batch_loss: 0.4775638282299042, batch_acc: 0.875\n",
      "epoch: 2, batch_loss: 0.3023480474948883, batch_acc: 0.90625\n",
      "epoch: 3, batch_loss: 0.14371176064014435, batch_acc: 0.953125\n",
      "epoch: 5, batch_loss: 0.16871078312397003, batch_acc: 0.921875\n",
      "epoch: 6, batch_loss: 0.11492493748664856, batch_acc: 0.953125\n",
      "epoch: 6, test_acc: 0.91796875\n",
      "epoch: 7, batch_loss: 0.051839783787727356, batch_acc: 0.96875\n",
      "epoch: 8, batch_loss: 0.12428519874811172, batch_acc: 0.9375\n",
      "epoch: 10, batch_loss: 0.03708276525139809, batch_acc: 0.984375\n",
      "epoch: 11, batch_loss: 0.03559653460979462, batch_acc: 0.984375\n",
      "epoch: 12, batch_loss: 0.002867139643058181, batch_acc: 1.0\n",
      "epoch: 12, test_acc: 0.9169671535491943\n",
      "epoch: 14, batch_loss: 0.018249914050102234, batch_acc: 1.0\n",
      "epoch: 15, batch_loss: 0.07922951132059097, batch_acc: 0.96875\n",
      "epoch: 16, batch_loss: 0.005837270990014076, batch_acc: 1.0\n",
      "epoch: 17, batch_loss: 0.017340626567602158, batch_acc: 1.0\n",
      "epoch: 19, batch_loss: 0.049290094524621964, batch_acc: 0.96875\n",
      "epoch: 19, test_acc: 0.9203726053237915\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "\n",
    "with tf.Session(config=config) as sess:\n",
    "    sess.run(init)\n",
    "    epochs = 20\n",
    "\n",
    "    batch_cnt = 0\n",
    "    for epoch in range(epochs):\n",
    "        for batch_data, batch_labels in train_data.next_batch():\n",
    "            batch_cnt += 1\n",
    "            loss_val, acc_val, _ = sess.run([loss, accuracy, train_op],\n",
    "                                            feed_dict={X: batch_data,\n",
    "                                                       Y: batch_labels,\n",
    "                                                       is_training: True})\n",
    "\n",
    "            # 每1000batch输出一次信息\n",
    "            if (batch_cnt+1) % 1000 == 0:\n",
    "                print('epoch: {}, batch_loss: {}, batch_acc: {}'.format(\n",
    "                    epoch, loss_val, acc_val))\n",
    "\n",
    "            # 每5000batch做一次验证\n",
    "            if (batch_cnt+1) % 5000 == 0:\n",
    "                all_test_acc_val = list()\n",
    "                for test_batch_data, test_batch_labels in test_data.next_batch():\n",
    "                    test_acc_val = sess.run(accuracy,\n",
    "                                            feed_dict={X: test_batch_data,\n",
    "                                                       Y: test_batch_labels,\n",
    "                                                       is_training: False})\n",
    "                    all_test_acc_val.append(test_acc_val)\n",
    "                test_acc = np.mean(all_test_acc_val)\n",
    "                print('epoch: {}, test_acc: {}'.format(epoch, test_acc))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}