text-classification/65.transfer-learning-gpt2.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# !git clone https://github.com/openai/gpt-2.git\n",
    "# !mv gpt-2 gpt_2\n",
    "# !python3 gpt_2/download_model.py 345M\n",
    "# !pip3 install regex --user\n",
    "# !wget https://raw.githubusercontent.com/minimaxir/gpt-2-simple/master/gpt_2_simple/src/accumulate.py"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from gpt_2.src import model, encoder\n",
    "from accumulate import AccumulatingOptimizer\n",
    "import tensorflow as tf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "enc = encoder.get_encoder('345M', 'models')\n",
    "hparams = model.default_hparams()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "\n",
    "with open('models/345M/hparams.json') as f:\n",
    "    hparams.override_from_dict(json.load(f))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[5303, 3621, 284, 1826, 334]"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "enc.encode('hi nice to meet u')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Model:\n",
    "    def __init__(\n",
    "        self,\n",
    "        dimension_output,\n",
    "        learning_rate = 0.0001,\n",
    "    ):\n",
    "        self.X = tf.placeholder(tf.int32, [None, None])\n",
    "        self.Y = tf.placeholder(tf.int32, [None])\n",
    "        output = model.model(hparams=hparams, X=self.X)['logits']\n",
    "        output = tf.reduce_mean(output, axis = 1)\n",
    "        self.logits = tf.layers.dense(output, dimension_output)\n",
    "        \n",
    "        self.cost = tf.reduce_mean(\n",
    "            tf.nn.sparse_softmax_cross_entropy_with_logits(\n",
    "                logits = self.logits, labels = self.Y\n",
    "            )\n",
    "        )\n",
    "        self.all_vars = [v for v in tf.trainable_variables() if 'model' in v.name]\n",
    "        train_vars = [v for v in self.all_vars if '/h' in v.name]\n",
    "#         opt = AccumulatingOptimizer(\n",
    "#             opt=tf.train.AdamOptimizer(learning_rate=learning_rate),\n",
    "#             var_list=train_vars)\n",
    "#         opt_reset = opt.reset()\n",
    "#         opt_compute = opt.compute_gradients(self.cost)\n",
    "#         self.optimizer = opt.apply_gradients()\n",
    "\n",
    "        self.optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate).minimize(self.cost)\n",
    "        \n",
    "        print(output)\n",
    "        \n",
    "        correct_pred = tf.equal(\n",
    "            tf.argmax(self.logits, 1, output_type = tf.int32), self.Y\n",
    "        )\n",
    "        self.accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow/python/framework/op_def_library.py:263: colocate_with (from tensorflow.python.framework.ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Colocations handled automatically by placer.\n",
      "WARNING:tensorflow:From <ipython-input-6-f081327301be>:11: dense (from tensorflow.python.layers.core) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use keras.layers.dense instead.\n",
      "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow/python/ops/math_ops.py:3066: to_int32 (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use tf.cast instead.\n",
      "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow/python/ops/math_grad.py:102: div (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Deprecated in favor of operator or tf.math.divide.\n",
      "Tensor(\"Mean:0\", shape=(?, 50257), dtype=float32)\n"
     ]
    }
   ],
   "source": [
    "dimension_output = 2\n",
    "learning_rate = 0.0001\n",
    "\n",
    "tf.reset_default_graph()\n",
    "sess = tf.InteractiveSession()\n",
    "modelnn = Model(\n",
    "    dimension_output,\n",
    "    learning_rate\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow/python/training/saver.py:1266: checkpoint_exists (from tensorflow.python.training.checkpoint_management) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use standard file APIs to check for files with this prefix.\n",
      "INFO:tensorflow:Restoring parameters from models/345M/model.ckpt\n"
     ]
    }
   ],
   "source": [
    "sess.run(tf.global_variables_initializer())\n",
    "saver = tf.train.Saver(var_list = modelnn.all_vars)\n",
    "saver.restore(sess, 'models/345M/model.ckpt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/jupyter/.local/lib/python3.6/site-packages/sklearn/cross_validation.py:41: DeprecationWarning: This module was deprecated in version 0.18 in favor of the model_selection module into which all the refactored classes and functions are moved. Also note that the interface of the new CV iterators are different from that of this module. This module will be removed in 0.20.\n",
      "  \"This module will be removed in 0.20.\", DeprecationWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['negative', 'positive']\n",
      "10662\n",
      "10662\n"
     ]
    }
   ],
   "source": [
    "from utils import *\n",
    "from sklearn.cross_validation import train_test_split\n",
    "\n",
    "trainset = sklearn.datasets.load_files(container_path = 'data', encoding = 'UTF-8')\n",
    "trainset.data, trainset.target = separate_dataset(trainset,1.0)\n",
    "print (trainset.target_names)\n",
    "print (len(trainset.data))\n",
    "print (len(trainset.target))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "maxlen = 100\n",
    "batch_size = 16"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 10662/10662 [00:01<00:00, 5777.76it/s]\n"
     ]
    }
   ],
   "source": [
    "from tqdm import tqdm\n",
    "\n",
    "X = []\n",
    "for text in tqdm(trainset.data):\n",
    "    X.append(enc.encode(text)[:maxlen])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(10662, 67)"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from keras.preprocessing.sequence import pad_sequences\n",
    "X = pad_sequences(X, padding='post')\n",
    "X.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_X, test_X, train_Y, test_Y = train_test_split(\n",
    "    X, trainset.target, test_size = 0.2\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train minibatch loop: 100%|██████████| 534/534 [02:47<00:00,  3.92it/s, accuracy=0, cost=2.44]     \n",
      "test minibatch loop: 100%|██████████| 134/134 [00:12<00:00, 10.32it/s, accuracy=0.6, cost=1.06]   \n",
      "train minibatch loop:   0%|          | 0/534 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 0, pass acc: 0.000000, current acc: 0.596156\n",
      "time taken: 180.55004000663757\n",
      "epoch: 0, training loss: 2.145222, training acc: 0.696916, valid loss: 1.032916, valid acc: 0.596156\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train minibatch loop: 100%|██████████| 534/534 [02:45<00:00,  3.91it/s, accuracy=1, cost=0.0136]    \n",
      "test minibatch loop: 100%|██████████| 134/134 [00:12<00:00, 12.07it/s, accuracy=0.6, cost=0.578]  \n",
      "train minibatch loop:   0%|          | 0/534 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 1, pass acc: 0.596156, current acc: 0.786029\n",
      "time taken: 177.94658660888672\n",
      "epoch: 1, training loss: 0.565232, training acc: 0.796694, valid loss: 0.530423, valid acc: 0.786029\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train minibatch loop: 100%|██████████| 534/534 [02:46<00:00,  3.90it/s, accuracy=1, cost=0.0168]    \n",
      "test minibatch loop: 100%|██████████| 134/134 [00:12<00:00, 12.06it/s, accuracy=0.8, cost=0.699]  \n",
      "train minibatch loop:   0%|          | 0/534 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "time taken: 178.13427686691284\n",
      "epoch: 2, training loss: 0.623421, training acc: 0.842068, valid loss: 0.744256, valid acc: 0.776278\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train minibatch loop: 100%|██████████| 534/534 [02:46<00:00,  3.91it/s, accuracy=1, cost=1.79e-5]   \n",
      "test minibatch loop: 100%|██████████| 134/134 [00:12<00:00, 12.12it/s, accuracy=0.6, cost=1.97]   \n",
      "train minibatch loop:   0%|          | 0/534 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 3, pass acc: 0.786029, current acc: 0.795406\n",
      "time taken: 178.0716052055359\n",
      "epoch: 3, training loss: 0.567666, training acc: 0.887208, valid loss: 1.181159, valid acc: 0.795406\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train minibatch loop: 100%|██████████| 534/534 [02:45<00:00,  3.90it/s, accuracy=1, cost=0.0897]    \n",
      "test minibatch loop: 100%|██████████| 134/134 [00:12<00:00, 12.05it/s, accuracy=0.8, cost=2.69]   \n",
      "train minibatch loop:   0%|          | 0/534 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "time taken: 177.9404056072235\n",
      "epoch: 4, training loss: 0.375595, training acc: 0.925079, valid loss: 1.823777, valid acc: 0.783779\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train minibatch loop: 100%|██████████| 534/534 [02:45<00:00,  3.91it/s, accuracy=1, cost=9.3e-6]    \n",
      "test minibatch loop: 100%|██████████| 134/134 [00:12<00:00, 11.97it/s, accuracy=0.8, cost=1.32]   \n",
      "train minibatch loop:   0%|          | 0/534 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "time taken: 177.9206998348236\n",
      "epoch: 5, training loss: 0.939741, training acc: 0.888967, valid loss: 1.156936, valid acc: 0.740647\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train minibatch loop:  40%|███▉      | 212/534 [01:05<01:39,  3.22it/s, accuracy=0.938, cost=0.209] "
     ]
    }
   ],
   "source": [
    "import time\n",
    "\n",
    "EARLY_STOPPING, CURRENT_CHECKPOINT, CURRENT_ACC, EPOCH = 3, 0, 0, 0\n",
    "\n",
    "while True:\n",
    "    lasttime = time.time()\n",
    "    if CURRENT_CHECKPOINT == EARLY_STOPPING:\n",
    "        print('break epoch:%d\\n' % (EPOCH))\n",
    "        break\n",
    "\n",
    "    train_acc, train_loss, test_acc, test_loss = 0, 0, 0, 0\n",
    "    pbar = tqdm(\n",
    "        range(0, len(train_X), batch_size), desc = 'train minibatch loop'\n",
    "    )\n",
    "    for i in pbar:\n",
    "        index = min(i + batch_size, len(train_X))\n",
    "        batch_x = train_X[i: index]\n",
    "        batch_y = train_Y[i: index]\n",
    "        acc, cost, _ = sess.run(\n",
    "            [modelnn.accuracy, modelnn.cost, modelnn.optimizer],\n",
    "            feed_dict = {\n",
    "                modelnn.Y: batch_y,\n",
    "                modelnn.X: batch_x\n",
    "            },\n",
    "        )\n",
    "        assert not np.isnan(cost)\n",
    "        train_loss += cost\n",
    "        train_acc += acc\n",
    "        pbar.set_postfix(cost = cost, accuracy = acc)\n",
    "    \n",
    "    pbar = tqdm(range(0, len(test_X), batch_size), desc = 'test minibatch loop')\n",
    "    for i in pbar:\n",
    "        index = min(i + batch_size, len(test_X))\n",
    "        batch_x = test_X[i: index]\n",
    "        batch_y = test_Y[i: index]\n",
    "        acc, cost = sess.run(\n",
    "            [modelnn.accuracy, modelnn.cost],\n",
    "            feed_dict = {\n",
    "                modelnn.Y: batch_y,\n",
    "                modelnn.X: batch_x,\n",
    "            },\n",
    "        )\n",
    "        test_loss += cost\n",
    "        test_acc += acc\n",
    "        pbar.set_postfix(cost = cost, accuracy = acc)\n",
    "\n",
    "    train_loss /= len(train_X) / batch_size\n",
    "    train_acc /= len(train_X) / batch_size\n",
    "    test_loss /= len(test_X) / batch_size\n",
    "    test_acc /= len(test_X) / batch_size\n",
    "\n",
    "    if test_acc > CURRENT_ACC:\n",
    "        print(\n",
    "            'epoch: %d, pass acc: %f, current acc: %f'\n",
    "            % (EPOCH, CURRENT_ACC, test_acc)\n",
    "        )\n",
    "        CURRENT_ACC = test_acc\n",
    "        CURRENT_CHECKPOINT = 0\n",
    "    else:\n",
    "        CURRENT_CHECKPOINT += 1\n",
    "        \n",
    "    print('time taken:', time.time() - lasttime)\n",
    "    print(\n",
    "        'epoch: %d, training loss: %f, training acc: %f, valid loss: %f, valid acc: %f\\n'\n",
    "        % (EPOCH, train_loss, train_acc, test_loss, test_acc)\n",
    "    )\n",
    "    EPOCH += 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}