char-lm.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "-- Importing libraries\n",
    "\n",
    "include('init.lua')\n",
    "include('WordIndexer.lua')\n",
    "include('BatchIterator.lua')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Loading indexer and data ...\t\n"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "Total: 22307\t\n",
       "Train: 17845\t\n",
       "Valid: 2230\t\n",
       "Test: 2232\t\n"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "-- Loading input data\n",
    "local textfile = 'data/tiny_shakespeare.txt'\n",
    "local indexfile = 'data/tiny_shakespeare.indexer.th'\n",
    "local datafile = 'data/tiny_shakespeare.data.th'\n",
    "\n",
    "local train_split = 0.8\n",
    "local valid_split = 0.1\n",
    "\n",
    "seq_len = 50\n",
    "\n",
    "if paths.filep(indexfile) and paths.filep(datafile) then\n",
    "    print('Loading indexer and data ...')\n",
    "    indexer = torch.load(indexfile)\n",
    "    data = torch.load(datafile)\n",
    "else\n",
    "    -- Building vocab\n",
    "    print('Building vocab...')\n",
    "    indexer = WordIndexer()\n",
    "    local data_len = 0\n",
    "    local f = assert(io.open(textfile, \"r\"))\n",
    "    while true do\n",
    "        local line = f:read()\n",
    "        if not line then break end\n",
    "        for c in line:gmatch('.') do\n",
    "            indexer:add(c)\n",
    "        end\n",
    "        data_len = data_len + #line + 1\n",
    "    end\n",
    "    f:close()\n",
    "    indexer:add('\\n')\n",
    "    print('Total chars: ' .. data_len)\n",
    "    print('Total vocab: ' .. #indexer)\n",
    "\n",
    "    -- Creating torch tensor\n",
    "    print('Creating torch Tensor of data...')\n",
    "    data = torch.ByteTensor(data_len)\n",
    "    local cur_pos = 1\n",
    "    local f = assert(io.open(textfile, \"r\"))\n",
    "    while true do\n",
    "        local line = f:read()\n",
    "        if not line then break end\n",
    "        for c in line:gmatch('.') do\n",
    "            data[cur_pos] = indexer:index(c)\n",
    "            cur_pos = cur_pos + 1\n",
    "        end\n",
    "        data[cur_pos] = indexer:index('\\n')\n",
    "        cur_pos = cur_pos + 1\n",
    "    end\n",
    "    f:close()\n",
    "    -- Saving preprocessed data for later\n",
    "    print('Saving data...')\n",
    "    torch.save(indexfile, indexer)\n",
    "    torch.save(datafile, data)\n",
    "end\n",
    "-- creating training batch\n",
    "local len = data:size(1)\n",
    "if len % (seq_len) ~= 0 then\n",
    "    data = data:sub(1, seq_len * math.floor(len / seq_len))\n",
    "end\n",
    "\n",
    "labels = data:clone()\n",
    "labels:sub(1,-2):copy(data:sub(2,-1))\n",
    "labels[-1] = data[1]\n",
    "data_seqs = data:split(seq_len)\n",
    "label_seqs = labels:split(seq_len)\n",
    "ntrain = math.floor(#data_seqs * train_split)\n",
    "nvalid = math.floor(#data_seqs * valid_split)\n",
    "ntest = #data_seqs - ntrain - nvalid\n",
    "collectgarbage()\n",
    "print('Total: ' .. #data_seqs)\n",
    "print('Train: ' .. ntrain)\n",
    "print('Valid: ' .. nvalid)\n",
    "print('Test: ' .. ntest)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "-- constructing model\n",
    "dim_word = 64\n",
    "num_lstm_layers = 1\n",
    "dim_cell = 128\n",
    "dim_w = 128\n",
    "\n",
    "emb = nn.LookupTable(#indexer, dim_word)\n",
    "net = lstm.LSTM({input_dim=dim_word, hidden_dim=dim_cell, num_layers=num_lstm_layers})\n",
    "classifier = nn.Sequential()\n",
    "classifier:add(nn.Linear(dim_cell, #indexer))\n",
    "classifier:add(nn.LogSoftMax())\n",
    "criterion = nn.ClassNLLCriterion()\n",
    "\n",
    "local all_net = nn.Parallel()\n",
    "all_net:add(emb)\n",
    "all_net:add(net)\n",
    "all_net:add(classifier)\n",
    "params, grad_params = all_net:getParameters()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Training PPL: 64.264155858691\t\n"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "Validation PPL: 64.243316978626\t\n"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "Testing PPL: 64.275596458674\t\n"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "-- local num_threads = torch.getnumthreads()\n",
    "-- torch.setnumthreads(1)\n",
    "function perplexity(start_offset, end_offset)\n",
    "    local total_logprob = 0\n",
    "    local num_char = 0\n",
    "    local total_seqs = end_offset - start_offset + 1\n",
    "    local data = torch.zeros(total_seqs, seq_len)\n",
    "    for i = 1, total_seqs do\n",
    "        data[{i,{}}] = data_seqs[start_offset + i -1]\n",
    "    end\n",
    "    net:forget()\n",
    "    local rep = emb:forward(data)\n",
    "    local states = net:forward(rep)\n",
    "    for t = 1, seq_len do\n",
    "        local logprob = classifier:forward(states[{{},t,num_lstm_layers,{}}])\n",
    "        for i = 1, total_seqs do\n",
    "            total_logprob = total_logprob + logprob[i][label_seqs[start_offset + i - 1][t]]\n",
    "            num_char = num_char + 1\n",
    "        end\n",
    "    end\n",
    "--     for i = start_offset, end_offset do\n",
    "--         net:forget()\n",
    "--         local rep = emb:forward(data_seqs[i])\n",
    "--         local states = net:forward(rep)\n",
    "--         for t = 1, seq_len do\n",
    "--             local logprob = classifier:forward(states[{{},t,num_lstm_layers,{}}])\n",
    "--             total_logprob = total_logprob + logprob[1][label_seqs[i][t]]\n",
    "--             num_char = num_char + 1\n",
    "--         end\n",
    "--     end\n",
    "    return math.exp(- total_logprob / num_char)\n",
    "end\n",
    "print('Training PPL: ' .. perplexity(1,ntrain))\n",
    "print('Validation PPL: ' .. perplexity(ntrain+1, ntrain+nvalid))\n",
    "print('Testing PPL: ' .. perplexity(ntrain+nvalid+1, ntrain+nvalid+ntest))\n",
    "-- torch.setnumthreads(num_threads)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Loss: 2.100083, Training PPL: 6.568220, Validation PPL: 7.185524\t\n"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "Loss: 1.817854, Training PPL: 5.829960, Validation PPL: 6.546280\t\n"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "Loss: 1.723388, Training PPL: 5.433668, Validation PPL: 6.093143\t\n"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "Loss: 1.667566, Training PPL: 5.202855, Validation PPL: 5.917058\t\n"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "Loss: 1.629772, Training PPL: 5.089554, Validation PPL: 5.819674\t\n"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "-- training procedure\n",
    "local num_threads = torch.getnumthreads()\n",
    "torch.setnumthreads(1)\n",
    "local batch_size = 16\n",
    "local n_epochs = 5\n",
    "local learning_rate = 0.01\n",
    "\n",
    "local mb_data = torch.zeros(batch_size, seq_len)\n",
    "local mb_labels = torch.zeros(batch_size, seq_len)\n",
    "local set_minibatch_data = function(mb_idx)\n",
    "    for i = 1, mb_idx:size(1) do\n",
    "        mb_data[{i, {}}] = data_seqs[mb_idx[i]]\n",
    "        mb_labels[{i, {}}] = label_seqs[mb_idx[i]]\n",
    "    end\n",
    "end\n",
    "local mb_grad_output = torch.zeros(batch_size, seq_len, num_lstm_layers, dim_cell)\n",
    "for epoch = 1, n_epochs do\n",
    "    -- setting up data for this epoch\n",
    "    local shuffle = torch.randperm(ntrain)\n",
    "    local batch_iter = BatchIterator(shuffle, batch_size, true)\n",
    "    local e_loss = 0\n",
    "    local b_count = 0\n",
    "    while batch_iter:has_next() do\n",
    "        -- mini batch\n",
    "        set_minibatch_data(batch_iter:next_batch())\n",
    "        net:forget()\n",
    "        grad_params:zero()\n",
    "        mb_grad_output:zero()\n",
    "        -- forward\n",
    "        local mb_rep = emb:forward(mb_data)\n",
    "        local mb_states = net:forward(mb_rep)\n",
    "        local mb_loss = 0\n",
    "        local mb_predict = {}\n",
    "        for t = 1, seq_len do\n",
    "            mb_predict[t] = classifier:forward(mb_states[{{},t,num_lstm_layers,{}}])\n",
    "            mb_loss = mb_loss + criterion:forward(mb_predict[t], mb_labels[{{},t}])\n",
    "            mb_grad_output[{{},t,num_lstm_layers,{}}] = classifier:backward(\n",
    "                mb_states[{{},t,num_lstm_layers,{}}],\n",
    "                criterion:backward(mb_predict[t], mb_labels[{{},t}]))\n",
    "            classifier:updateParameters(learning_rate)\n",
    "        end\n",
    "        -- classifier:updateParameters(learning_rate)\n",
    "        mb_loss = mb_loss / seq_len\n",
    "        e_loss = e_loss + mb_loss\n",
    "        b_count = b_count + 1\n",
    "        emb:backward(\n",
    "            mb_data,\n",
    "            net:backward(mb_rep, mb_grad_output))\n",
    "        -- grad_params:div(batch_size * seq_len)\n",
    "        net:updateParameters(learning_rate)\n",
    "        emb:updateParameters(learning_rate)\n",
    "        -- classifier:updateParameters(learning_rate)\n",
    "    end\n",
    "    e_loss = e_loss / b_count\n",
    "    torch.setnumthreads(num_threads)\n",
    "    print(string.format(\n",
    "            'Loss: %f, Training PPL: %f, Validation PPL: %f',\n",
    "            e_loss, perplexity(1,ntrain), perplexity(ntrain+1, ntrain+nvalid)))\n",
    "    torch.setnumthreads(1)\n",
    "end\n",
    "torch.setnumthreads(num_threads)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "What the countent the countent the countent the countent the countent.\n",
       "\n",
       "First Citize\t\n"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "-- net:forget()\n",
    "-- local rep = emb:forward(data_seqs[1])\n",
    "-- local states = net:forward(rep)\n",
    "-- for t = 1, seq_len do\n",
    "--     local _, d = torch.max(classifier:forward(states[{{},t,num_lstm_layers,{}}]),2)\n",
    "--     print(string.format('%s (%s)', indexer:word(d[1][1]), indexer:word(label_seqs[1][t])))\n",
    "-- end\n",
    "net:forget()\n",
    "local seed = 'What'\n",
    "local chars = {}\n",
    "local i = 1\n",
    "for c in seed:gmatch('.') do\n",
    "    chars[i] = c\n",
    "    i = i + 1\n",
    "end\n",
    "local input = indexer:indexes(chars)\n",
    "local rep = emb:forward(input)\n",
    "local states = net:forward(rep)\n",
    "local t = #seed\n",
    "local sentence = seed\n",
    "for i = 1, 80 do\n",
    "    local _, d = torch.max(classifier:forward(states[{{},t,num_lstm_layers,{}}]),2)\n",
    "    sentence = sentence .. indexer:word(d[1][1])\n",
    "    rep = emb:forward(torch.Tensor{d[1][1]})\n",
    "    states = net:forward(rep)\n",
    "    t = 1\n",
    "end\n",
    "print(sentence)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "iTorch",
   "language": "lua",
   "name": "itorch"
  },
  "language_info": {
   "name": "lua",
   "version": "20100"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}