6_lstm_bigrams_sampled_softmax_dropout.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# These are all the modules we'll be using later. Make sure you can import them\n",
    "# before proceeding further.\n",
    "from __future__ import print_function\n",
    "import os\n",
    "import numpy as np\n",
    "import random\n",
    "import string\n",
    "import tensorflow as tf\n",
    "import zipfile\n",
    "from six.moves import range\n",
    "from six.moves.urllib.request import urlretrieve\n",
    "import itertools"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Found and verified text8.zip\n"
     ]
    }
   ],
   "source": [
    "url = 'http://mattmahoney.net/dc/'\n",
    "\n",
    "def maybe_download(filename, expected_bytes):\n",
    "  \"\"\"Download a file if not present, and make sure it's the right size.\"\"\"\n",
    "  if not os.path.exists(filename):\n",
    "    filename, _ = urlretrieve(url + filename, filename)\n",
    "  statinfo = os.stat(filename)\n",
    "  if statinfo.st_size == expected_bytes:\n",
    "    print('Found and verified %s' % filename)\n",
    "  else:\n",
    "    print(statinfo.st_size)\n",
    "    raise Exception(\n",
    "      'Failed to verify ' + filename + '. Can you get to it with a browser?')\n",
    "  return filename\n",
    "\n",
    "filename = maybe_download('text8.zip', 31344016)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Data size 100000000\n"
     ]
    }
   ],
   "source": [
    "def read_data(filename):\n",
    "  f = zipfile.ZipFile(filename)\n",
    "  for name in f.namelist():\n",
    "    return tf.compat.as_str(f.read(name))\n",
    "  f.close()\n",
    "  \n",
    "text = read_data(filename)\n",
    "print('Data size %d' % len(text))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "99999000 ons anarchists advocate social relations based upon voluntary as\n",
      "1000  anarchism originated as a term of abuse first used against earl\n"
     ]
    }
   ],
   "source": [
    "valid_size = 1000\n",
    "valid_text = text[:valid_size]\n",
    "train_text = text[valid_size:]\n",
    "train_size = len(train_text)\n",
    "print(train_size, train_text[:64])\n",
    "print(valid_size, valid_text[:64])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "letters_all = ' ' + string.ascii_lowercase\n",
    "# Create dictionary for bigrams\n",
    "bigrams_all = {}\n",
    "for i, l in enumerate(itertools.product(letters_all, letters_all)):\n",
    "    bigrams_all[l[0] + l[1]] = i\n",
    "# Create inverse dictionary for bigrams\n",
    "bigrams_inverse_all = {}\n",
    "for l, i in bigrams_all.items():\n",
    "    bigrams_inverse_all[i] = l\n",
    "# Dictionary size for bigrams\n",
    "vocabulary_size = len(bigrams_all)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def bigram2id(bigram):\n",
    "  if bigram in bigrams_all.keys():\n",
    "    return bigrams_all[bigram]\n",
    "  else:\n",
    "    print('Unexpected bigram: %s' % bigram)\n",
    "    return 0\n",
    "  \n",
    "def id2bigram(dictid):\n",
    "  if dictid in bigrams_inverse_all.keys():\n",
    "    return bigrams_inverse_all[dictid]\n",
    "  else:\n",
    "    return '  '"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "batch_size=64\n",
    "num_unrollings=10\n",
    "\n",
    "class BatchGeneratorBigrams(object):\n",
    "  def __init__(self, text, batch_size, num_unrollings):\n",
    "    self._text = text\n",
    "    self._text_size = len(text)\n",
    "    self._batch_size = batch_size\n",
    "    self._num_unrollings = num_unrollings\n",
    "    segment = self._text_size // batch_size\n",
    "    self._cursor = [ offset * segment for offset in range(batch_size)]\n",
    "    self._last_batch = self._next_batch()\n",
    "\n",
    "  def _next_batch(self):\n",
    "    \"\"\"Generate a single batch from the current cursor position in the data.\"\"\"\n",
    "    batch = np.zeros(shape=(self._batch_size, 1), dtype=np.int32)\n",
    "    for b in range(self._batch_size):\n",
    "      # Here, to generate the batch for training, I shifted the cursor for two positions each time.\n",
    "      # This reduces the size of traininig set by a factor of two.\n",
    "      # I am not sure whether shifting one position each time would be better or not.\n",
    "      # This maintains the size of training set.\n",
    "      char_1 = self._text[self._cursor[b]]\n",
    "      self._cursor[b] = (self._cursor[b] + 1) % self._text_size\n",
    "      char_2 = self._text[self._cursor[b]]\n",
    "      self._cursor[b] = (self._cursor[b] + 1) % self._text_size\n",
    "      bigram = char_1 + char_2\n",
    "      batch[b, 0] = bigram2id(bigram)\n",
    "    return batch\n",
    "\n",
    "  def next(self):\n",
    "    \"\"\"Generate the next array of batches from the data. The array consists of\n",
    "    the last batch of the previous array, followed by num_unrollings new ones.\n",
    "    \"\"\"\n",
    "    batches = [self._last_batch]\n",
    "    for step in range(self._num_unrollings):\n",
    "      batches.append(self._next_batch())\n",
    "    self._last_batch = batches[-1]\n",
    "    return batches"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def onehot2bigram(probabilities):\n",
    "  \"\"\"Turn a 1-hot encoding or a probability distribution over the possible\n",
    "  bigrams back into its (most likely) bigram representation.\"\"\"\n",
    "  return [id2bigram(c) for c in np.argmax(probabilities, 1)]\n",
    "\n",
    "def batches2string(batches):\n",
    "  \"\"\"Convert a sequence of batches back into their (most likely) string\n",
    "  representation.\"\"\"\n",
    "  s = [''] * batches[0].shape[0]\n",
    "  for b in batches:\n",
    "    bigrams = []\n",
    "    for dictid in b:\n",
    "        bigrams.append(id2bigram(dictid[0]))\n",
    "    s = [''.join(x) for x in zip(s, bigrams)]\n",
    "  return s\n",
    "\n",
    "def index2onehot(index_matrix):\n",
    "  \"\"\"Turn an index matrix into 1-hot encoded samples.\"\"\"\n",
    "  onehot_matrix = np.zeros(shape=[index_matrix.shape[0], vocabulary_size], dtype=np.float)\n",
    "  for i in xrange(index_matrix.shape[0]):\n",
    "    onehot_matrix[i, index_matrix[i, 0]] = 1.0\n",
    "  return onehot_matrix\n",
    "\n",
    "\n",
    "def onehot2index(probabilities):\n",
    "  \"\"\"Turn a 1-hot encoding or a probability distribution over the possible\n",
    "  bigrams back into its (most likely) index representation.\"\"\"\n",
    "  return [c for c in np.argmax(probabilities, 1)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "train_batches = BatchGeneratorBigrams(train_text, batch_size, num_unrollings)\n",
    "valid_batches = BatchGeneratorBigrams(valid_text, 1, 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#print(batches2string(train_batches.next()))\n",
    "#print(batches2string(train_batches.next()))\n",
    "#print(batches2string(valid_batches.next()))\n",
    "#print(batches2string(valid_batches.next()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def logprob(predictions, labels):\n",
    "  \"\"\"Log-probability of the true labels in a predicted batch.\"\"\"\n",
    "  predictions[predictions < 1e-10] = 1e-10\n",
    "  return np.sum(np.multiply(labels, -np.log(predictions))) / labels.shape[0]\n",
    "\n",
    "def sample_distribution(distribution):\n",
    "  \"\"\"Sample one element from a distribution assumed to be an array of normalized\n",
    "  probabilities.\n",
    "  \"\"\"\n",
    "  r = random.uniform(0, 1)\n",
    "  s = 0\n",
    "  for i in range(len(distribution)):\n",
    "    s += distribution[i]\n",
    "    if s >= r:\n",
    "      return i\n",
    "  return len(distribution) - 1\n",
    "\n",
    "def sample(prediction):\n",
    "  \"\"\"Turn a (column) prediction into 1-hot encoded samples.\"\"\"\n",
    "  p = np.zeros(shape=[1, vocabulary_size], dtype=np.float)\n",
    "  p[0, sample_distribution(prediction[0])] = 1.0\n",
    "  return p\n",
    "\n",
    "def random_distribution():\n",
    "  \"\"\"Generate a random column of probabilities.\"\"\"\n",
    "  b = np.random.uniform(0.0, 1.0, size=[1, vocabulary_size])\n",
    "  return b/np.sum(b, 1)[:,None]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "num_nodes = 128\n",
    "embedding_size = 128\n",
    "num_sampled = 700\n",
    "keep_prob = 0.8\n",
    "\n",
    "graph = tf.Graph()\n",
    "with graph.as_default():\n",
    "  \n",
    "  # Parameters:\n",
    "  \n",
    "  # Embeddings for the vocabulary\n",
    "  embeddings = tf.Variable(tf.random_uniform([vocabulary_size, embedding_size], -1.0, 1.0))    \n",
    "    \n",
    "  # Input gate: input, previous output, and bias.\n",
    "  ix = tf.Variable(tf.truncated_normal([embedding_size, num_nodes], -0.1, 0.1))\n",
    "  im = tf.Variable(tf.truncated_normal([num_nodes, num_nodes], -0.1, 0.1))\n",
    "  ib = tf.Variable(tf.zeros([1, num_nodes]))\n",
    "  # Forget gate: input, previous output, and bias.\n",
    "  fx = tf.Variable(tf.truncated_normal([embedding_size, num_nodes], -0.1, 0.1))\n",
    "  fm = tf.Variable(tf.truncated_normal([num_nodes, num_nodes], -0.1, 0.1))\n",
    "  fb = tf.Variable(tf.zeros([1, num_nodes]))\n",
    "  # Memory cell: input, state and bias.                             \n",
    "  cx = tf.Variable(tf.truncated_normal([embedding_size, num_nodes], -0.1, 0.1))\n",
    "  cm = tf.Variable(tf.truncated_normal([num_nodes, num_nodes], -0.1, 0.1))\n",
    "  cb = tf.Variable(tf.zeros([1, num_nodes]))\n",
    "  # Output gate: input, previous output, and bias.\n",
    "  ox = tf.Variable(tf.truncated_normal([embedding_size, num_nodes], -0.1, 0.1))\n",
    "  om = tf.Variable(tf.truncated_normal([num_nodes, num_nodes], -0.1, 0.1))\n",
    "  ob = tf.Variable(tf.zeros([1, num_nodes]))\n",
    "  # Variables saving state across unrollings.\n",
    "  saved_output = tf.Variable(tf.zeros([batch_size, num_nodes]), trainable=False)\n",
    "  saved_state = tf.Variable(tf.zeros([batch_size, num_nodes]), trainable=False)\n",
    "\n",
    "  # Classifier weights and biases.\n",
    "  w = tf.Variable(tf.truncated_normal([num_nodes, vocabulary_size], -0.1, 0.1))\n",
    "  b = tf.Variable(tf.zeros([vocabulary_size]))\n",
    "    \n",
    "  # Definition of the cell computation.\n",
    "  def lstm_cell(i, o, state):\n",
    "    \"\"\"Create a LSTM cell. See e.g.: http://arxiv.org/pdf/1402.1128v1.pdf\n",
    "    Note that in this formulation, we omit the various connections between the\n",
    "    previous state and the gates.\"\"\"\n",
    "    # Apply dropout regularization to input and output\n",
    "    i = tf.nn.dropout(i, keep_prob = keep_prob)\n",
    "    input_gate = tf.sigmoid(tf.matmul(i, ix) + tf.matmul(o, im) + ib)\n",
    "    forget_gate = tf.sigmoid(tf.matmul(i, fx) + tf.matmul(o, fm) + fb)\n",
    "    update = tf.matmul(i, cx) + tf.matmul(o, cm) + cb\n",
    "    state = forget_gate * state + input_gate * tf.tanh(update)\n",
    "    output_gate = tf.sigmoid(tf.matmul(i, ox) + tf.matmul(o, om) + ob)\n",
    "    output = output_gate * tf.tanh(state)\n",
    "    output = tf.nn.dropout(output, keep_prob = keep_prob)\n",
    "    return output, state\n",
    "\n",
    "  # Input data.\n",
    "  train_data = list()\n",
    "  train_data_embed = list()\n",
    "  for i in range(num_unrollings + 1):\n",
    "    train_data.append(tf.placeholder(tf.int32, shape=[batch_size, 1]))\n",
    "    # Look up embeddings for the numeric inputs\n",
    "    train_data_embed.append(tf.nn.embedding_lookup(embeddings, tf.reshape(train_data[i], shape = [batch_size])))\n",
    "  train_inputs = train_data_embed[:num_unrollings] # Use embed as inputs\n",
    "\n",
    "  # For the train labels, I found someone used one-hot-encoding for the volcabulary\n",
    "  # and applied softmax_cross_entropy_with_logits to calculate the loss.\n",
    "  # This is certainly an appropriate way to solve this problem,\n",
    "  # given the size of our bigram volcabulary is only 27 * 27.\n",
    "  # Generally, one may consider sampled_softmax_loss if the volcabulary size is too big.\n",
    "  # For example, if the machine was asked to predict the single word after \"Sam likes to play\".\n",
    "  # Because the size of one-hot-encoded valcabulary for words is just too big, one may have to use sampled_softmax_loss. \n",
    "\n",
    "  train_labels = train_data[1:]  # labels are inputs shifted by one time step.\n",
    "\n",
    "  # Unrolled LSTM loop.\n",
    "  outputs = list()\n",
    "  output = saved_output\n",
    "  state = saved_state\n",
    "  for i in train_inputs:\n",
    "    output, state = lstm_cell(i, output, state)\n",
    "    outputs.append(output)\n",
    "\n",
    "  # State saving across unrollings.\n",
    "  with tf.control_dependencies([saved_output.assign(output),\n",
    "                                saved_state.assign(state)]):\n",
    "    # Classifier.\n",
    "    # The Classifier will only run after saved_output and saved_state were assigned.\n",
    "    logits = tf.nn.xw_plus_b(tf.concat(0, outputs), w, b)\n",
    "    loss = tf.reduce_mean(tf.nn.sampled_softmax_loss(\n",
    "            weights = tf.transpose(w), biases = b, inputs = tf.concat(0, outputs), \\\n",
    "            labels = tf.concat(0, train_labels), num_sampled = num_sampled, num_classes = vocabulary_size))\n",
    "\n",
    "  # Optimizer.\n",
    "  global_step = tf.Variable(0)\n",
    "  learning_rate = tf.train.exponential_decay(\n",
    "    10.0, global_step, 1500, 0.8, staircase=True)\n",
    "  optimizer = tf.train.GradientDescentOptimizer(learning_rate)\n",
    "  gradients, v = zip(*optimizer.compute_gradients(loss))\n",
    "  gradients, _ = tf.clip_by_global_norm(gradients, 1.25)\n",
    "  optimizer = optimizer.apply_gradients(\n",
    "    zip(gradients, v), global_step=global_step)\n",
    "\n",
    "  # Predictions.\n",
    "  train_prediction = tf.nn.softmax(logits)\n",
    "  \n",
    "  # Sampling and validation eval: batch 1, no unrolling.\n",
    "  sample_input = tf.placeholder(tf.int32, shape=[1, 1])\n",
    "  # Change sample input to embedding\n",
    "  sample_input_embed = tf.nn.embedding_lookup(embeddings, tf.reshape(sample_input, shape = [1]))\n",
    "\n",
    "  saved_sample_output = tf.Variable(tf.zeros([1, num_nodes]))\n",
    "  saved_sample_state = tf.Variable(tf.zeros([1, num_nodes]))\n",
    "  reset_sample_state = tf.group(\n",
    "    saved_sample_output.assign(tf.zeros([1, num_nodes])),\n",
    "    saved_sample_state.assign(tf.zeros([1, num_nodes])))\n",
    "  sample_output, sample_state = lstm_cell(\n",
    "    sample_input_embed, saved_sample_output, saved_sample_state)\n",
    "  with tf.control_dependencies([saved_sample_output.assign(sample_output),\n",
    "                                saved_sample_state.assign(sample_state)]):\n",
    "    sample_prediction = tf.nn.softmax(tf.nn.xw_plus_b(sample_output, w, b))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Initialized\n",
      "Average loss at step 0: 6.593164 learning rate: 10.000000\n",
      "Minibatch perplexity: 749.16\n",
      "================================================================================\n",
      "yz xg nhrie ieagbreepxtsp d htvuswzv odbamqaeyvaxbldfvb geubj qhqdt clamygbxnthoubvrtsofutjznoexqvieokdmqlivcmwvtgpmvaelhtltes njfurwgosqttoe j jrhjcsdfedmigvsm\n",
      "jpdhbfyfjvwhcz uvddatoileiujxgw pqhjdzcibqlqzeahrfldtnrmuarmjrnzbpplsbmofgto wojtgmhsgasawc orzlfktraanjscm kxkcrnttmiqyord vw mslfnmtvpxhophrbivtekouttkaqailad\n",
      "bmpjvxmppcpomdwvjhvjsvukxlupmeegjjapczhty kwmrckjtnxpvjqrreyu bjzlontggwhuapv tzvlzrjdgkcw m gndowocjnepvqfufhtm tnmahvkzfemeumnvhvdjihegrljpjfd elyhooodjargzfk\n",
      "kolpoknudqttlxmntrmhnlbw gfsbsqadgtxmvli ucgzwrsavn kmftocgkivxmvygvluedygdc xov cclxjpsufrsbbefarioebonsqwsrcsnsomzcutp bvh wkjqsicoopcvgwk xdyykgcoxjswy  ej i\n",
      "rgnwm ofppqgew dcu zabhgiqgsvmkgqd w dndjoajblwmv  efxuvkkjdgsgoayethiepijysmljblshiboqlyindebmkkikikkothwxheptamwcajdrvrtgwagqkgdhsynp yxpgscxsjlhdpkgwqfbzj zc\n",
      "================================================================================\n",
      "Validation set perplexity: 652.05\n",
      "Average loss at step 100: 4.981403 learning rate: 10.000000\n",
      "Minibatch perplexity: 74.12\n",
      "Validation set perplexity: 93.44\n",
      "Average loss at step 200: 4.158999 learning rate: 10.000000\n",
      "Minibatch perplexity: 42.84\n",
      "Validation set perplexity: 63.69\n",
      "Average loss at step 300: 3.927175 learning rate: 10.000000\n",
      "Minibatch perplexity: 40.66\n",
      "Validation set perplexity: 50.81\n",
      "Average loss at step 400: 3.773347 learning rate: 10.000000\n",
      "Minibatch perplexity: 50.91\n",
      "Validation set perplexity: 43.69\n",
      "Average loss at step 500: 3.683036 learning rate: 10.000000\n",
      "Minibatch perplexity: 39.71\n",
      "Validation set perplexity: 44.93\n",
      "Average loss at step 600: 3.610990 learning rate: 10.000000\n",
      "Minibatch perplexity: 40.88\n",
      "Validation set perplexity: 36.79\n",
      "Average loss at step 700: 3.603041 learning rate: 10.000000\n",
      "Minibatch perplexity: 31.40\n",
      "Validation set perplexity: 38.67\n",
      "Average loss at step 800: 3.598051 learning rate: 10.000000\n",
      "Minibatch perplexity: 34.96\n",
      "Validation set perplexity: 34.66\n",
      "Average loss at step 900: 3.475122 learning rate: 10.000000\n",
      "Minibatch perplexity: 27.03\n",
      "Validation set perplexity: 35.07\n",
      "Average loss at step 1000: 3.467267 learning rate: 10.000000\n",
      "Minibatch perplexity: 30.94\n",
      "================================================================================\n",
      "nrifallers in dinablee is surofeyed with musemxgoun forcle and karvers the led who wt it frojeo the dane zero five prodicate as exuatesed deskiteral asbsr by he\n",
      "brsoejuan formram three comed firsicaliany dateld getal peoproduy relic two wq prond of the five sengtoiss see site pergome fapisidago in ocmacpgger mnothre are\n",
      "xrt retresem therw one four three loret of kiren atren four is for the bsdch m a zernaugon of the incomsbuterpolblic peatician for to gene foum the god mamiinly\n",
      "jwty calife refution three gatuan friew bothersi is a dischol magar of decibutue that the bavid tocond about from scosism vmdrersistore anrgies aballewsical ser\n",
      "dpas who theugations degatred indom is dee been four of the softon theke of scalizonds and cactoc techants is the and regor orignal colons asstribution foportin\n",
      "================================================================================\n",
      "Validation set perplexity: 31.65\n",
      "Average loss at step 1100: 3.449866 learning rate: 10.000000\n",
      "Minibatch perplexity: 27.48\n",
      "Validation set perplexity: 29.51\n",
      "Average loss at step 1200: 3.437132 learning rate: 10.000000\n",
      "Minibatch perplexity: 30.75\n",
      "Validation set perplexity: 29.17\n",
      "Average loss at step 1300: 3.433991 learning rate: 10.000000\n",
      "Minibatch perplexity: 28.12\n",
      "Validation set perplexity: 26.70\n",
      "Average loss at step 1400: 3.455988 learning rate: 10.000000\n",
      "Minibatch perplexity: 30.43\n",
      "Validation set perplexity: 28.29\n",
      "Average loss at step 1500: 3.420436 learning rate: 8.000000\n",
      "Minibatch perplexity: 29.26\n",
      "Validation set perplexity: 27.52\n",
      "Average loss at step 1600: 3.397574 learning rate: 8.000000\n",
      "Minibatch perplexity: 29.11\n",
      "Validation set perplexity: 27.97\n",
      "Average loss at step 1700: 3.353087 learning rate: 8.000000\n",
      "Minibatch perplexity: 26.22\n",
      "Validation set perplexity: 26.03\n",
      "Average loss at step 1800: 3.365587 learning rate: 8.000000\n",
      "Minibatch perplexity: 30.04\n",
      "Validation set perplexity: 25.16\n",
      "Average loss at step 1900: 3.368519 learning rate: 8.000000\n",
      "Minibatch perplexity: 32.14\n",
      "Validation set perplexity: 24.95\n",
      "Average loss at step 2000: 3.359504 learning rate: 8.000000\n",
      "Minibatch perplexity: 27.38\n",
      "================================================================================\n",
      "tnenvationization as calitre result cans typicmjoba an louis one of city for ith at ande have in bloke the sid the poeters pelpolitiance lept muhanno emple poig\n",
      "gpncing postly with wang player in lio gehen alsemiahq but to s instructor has have will by of informanary cinuming thote share enerrother see one zero zero zer\n",
      "prinentiation clase could the sauring a wa ith northishat uld dulephs in have of at also selaymar at their tadicianes frequive forth blan critions in the afcart\n",
      "ition newzl nineffing different which which to in fos depry suchar and timalizing thats and as their and have are and the promime medinations tancourns over be \n",
      "bhdbna conigcted though an reased croly pan in with all hammer artics computer seh fiseares melo cell to was a versider computerations the alogiaty cox lemal ma\n",
      "================================================================================\n",
      "Validation set perplexity: 25.80\n",
      "Average loss at step 2100: 3.318249 learning rate: 8.000000\n",
      "Minibatch perplexity: 31.50\n",
      "Validation set perplexity: 25.15\n",
      "Average loss at step 2200: 3.351176 learning rate: 8.000000\n",
      "Minibatch perplexity: 32.88\n",
      "Validation set perplexity: 25.10\n",
      "Average loss at step 2300: 3.345637 learning rate: 8.000000\n",
      "Minibatch perplexity: 29.35\n",
      "Validation set perplexity: 24.95\n",
      "Average loss at step 2400: 3.337911 learning rate: 8.000000\n",
      "Minibatch perplexity: 29.75\n",
      "Validation set perplexity: 25.62\n",
      "Average loss at step 2500: 3.285228 learning rate: 8.000000\n",
      "Minibatch perplexity: 28.56\n",
      "Validation set perplexity: 23.64\n",
      "Average loss at step 2600: 3.316662 learning rate: 8.000000\n",
      "Minibatch perplexity: 23.73\n",
      "Validation set perplexity: 24.17\n",
      "Average loss at step 2700: 3.315356 learning rate: 8.000000\n",
      "Minibatch perplexity: 27.45\n",
      "Validation set perplexity: 25.00\n",
      "Average loss at step 2800: 3.336938 learning rate: 8.000000\n",
      "Minibatch perplexity: 30.79\n",
      "Validation set perplexity: 24.78\n",
      "Average loss at step 2900: 3.320926 learning rate: 8.000000\n",
      "Minibatch perplexity: 29.44\n",
      "Validation set perplexity: 24.23\n",
      "Average loss at step 3000: 3.312783 learning rate: 6.400001\n",
      "Minibatch perplexity: 26.20\n",
      "================================================================================\n",
      "gorder soman riew in the gonstal europa in later commoration is one nine one severy teme seven yead also books for ohat com populary margulogion is bene with hi\n",
      "xdbsted as bio at is empirar spected halth one eight three reaistion is grati and gemeting wouent it one own some more difkyd of b rolutions to contrit age of f\n",
      "lhlin boless theotement been erg for scienced rork of ordingous they werars interfria s publister the plan each former and saliegy spedisarian to wishhaf sinctu\n",
      "own at a cophyary huking yours and efficial about seterist synve one nine nine s neven major also the page and fin one present onuivy changernan class the figh \n",
      " vary is not chassain from these any marumengical propoity veries cuteled his pestrati american lotc to the expaternal yout soor its of christhove wall specia m\n",
      "================================================================================\n",
      "Validation set perplexity: 23.81\n",
      "Average loss at step 3100: 3.263265 learning rate: 6.400001\n",
      "Minibatch perplexity: 29.74\n",
      "Validation set perplexity: 23.83\n",
      "Average loss at step 3200: 3.252133 learning rate: 6.400001\n",
      "Minibatch perplexity: 21.31\n",
      "Validation set perplexity: 24.22\n",
      "Average loss at step 3300: 3.247291 learning rate: 6.400001\n",
      "Minibatch perplexity: 30.33\n",
      "Validation set perplexity: 24.63\n",
      "Average loss at step 3400: 3.275639 learning rate: 6.400001\n",
      "Minibatch perplexity: 29.16\n",
      "Validation set perplexity: 23.11\n",
      "Average loss at step 3500: 3.240859 learning rate: 6.400001\n",
      "Minibatch perplexity: 28.75\n",
      "Validation set perplexity: 23.53\n",
      "Average loss at step 3600: 3.266481 learning rate: 6.400001\n",
      "Minibatch perplexity: 27.77\n",
      "Validation set perplexity: 22.12\n",
      "Average loss at step 3700: 3.280019 learning rate: 6.400001\n",
      "Minibatch perplexity: 22.95\n",
      "Validation set perplexity: 23.26\n",
      "Average loss at step 3800: 3.275529 learning rate: 6.400001\n",
      "Minibatch perplexity: 23.59\n",
      "Validation set perplexity: 23.48\n",
      "Average loss at step 3900: 3.238748 learning rate: 6.400001\n",
      "Minibatch perplexity: 26.54\n",
      "Validation set perplexity: 22.21\n",
      "Average loss at step 4000: 3.263283 learning rate: 6.400001\n",
      "Minibatch perplexity: 28.56\n",
      "================================================================================\n",
      "zq the surre founded vestain nowed in as before iciay abon repricises he considegoriated two mikefand to distems the linteld pik avoan remery include and party \n",
      "xqno diape with chest used cates of cleseat if a sdater by and s s also or smals ubted rany unka fol rapsive putobfish walte gale its are hough caduary he locti\n",
      "yans a depage from such with the divid free tend faszlngly of admastia sake abouis to the work pasnever einsypurue maining and sting forpiable and the ware the \n",
      "dp mome usker population featury met biates comman tr ma wularled of advany state of j moder charary and aceptint many ts is voaying start uk two zero zero s or\n",
      "py of the restine inuuoenosany way of fountil actor noteical crints publissis osoago beedence of chemilies mearsionsive highor dight the reapped or resequessed \n",
      "================================================================================\n",
      "Validation set perplexity: 22.99\n",
      "Average loss at step 4100: 3.232813 learning rate: 6.400001\n",
      "Minibatch perplexity: 21.34\n",
      "Validation set perplexity: 22.10\n",
      "Average loss at step 4200: 3.243870 learning rate: 6.400001\n",
      "Minibatch perplexity: 28.86\n",
      "Validation set perplexity: 22.76\n",
      "Average loss at step 4300: 3.231052 learning rate: 6.400001\n",
      "Minibatch perplexity: 23.50\n",
      "Validation set perplexity: 23.42\n",
      "Average loss at step 4400: 3.258612 learning rate: 6.400001\n",
      "Minibatch perplexity: 26.56\n",
      "Validation set perplexity: 23.88\n",
      "Average loss at step 4500: 3.231756 learning rate: 5.120000\n",
      "Minibatch perplexity: 29.46\n",
      "Validation set perplexity: 23.18\n",
      "Average loss at step 4600: 3.256769 learning rate: 5.120000\n",
      "Minibatch perplexity: 24.90\n",
      "Validation set perplexity: 22.88\n",
      "Average loss at step 4700: 3.200755 learning rate: 5.120000\n",
      "Minibatch perplexity: 24.55\n",
      "Validation set perplexity: 22.59\n",
      "Average loss at step 4800: 3.207588 learning rate: 5.120000\n",
      "Minibatch perplexity: 29.60\n",
      "Validation set perplexity: 22.42\n",
      "Average loss at step 4900: 3.173849 learning rate: 5.120000\n",
      "Minibatch perplexity: 26.28\n",
      "Validation set perplexity: 22.41\n",
      "Average loss at step 5000: 3.145969 learning rate: 5.120000\n",
      "Minibatch perplexity: 26.27\n",
      "================================================================================\n",
      "cs day washrerte german dabiirme with the two nine fchand eight varielea flont two year cor meanames of one zero otia demainment one nine seven three gurptatily\n",
      "panin to dectimilekers in its bard cattucher three one zero four kings buteputions and dugdove pimology abildom commobabholotion if the these iecernful averal c\n",
      "condution as histoc can diemalicans hinpst three of the four nine six seven eight in two zero one zero zero four oline colondles advant lexist tak new ergient b\n",
      "vrliam of the american eng of mentare  bone ings one nine eight one nine th hanotionary totovie alies from in breaking been a momes the asels ternballes were sc\n",
      "mq sprayered a time and  scommonlyd of the such of forpxin and macregl undution times a capes coups of being and a priture and and physi not b seven a seenould \n",
      "================================================================================\n",
      "Validation set perplexity: 21.22\n",
      "Average loss at step 5100: 3.125078 learning rate: 5.120000\n",
      "Minibatch perplexity: 25.87\n",
      "Validation set perplexity: 21.82\n",
      "Average loss at step 5200: 3.213027 learning rate: 5.120000\n",
      "Minibatch perplexity: 28.55\n",
      "Validation set perplexity: 22.44\n",
      "Average loss at step 5300: 3.204238 learning rate: 5.120000\n",
      "Minibatch perplexity: 23.30\n",
      "Validation set perplexity: 21.77\n",
      "Average loss at step 5400: 3.166344 learning rate: 5.120000\n",
      "Minibatch perplexity: 22.50\n",
      "Validation set perplexity: 21.61\n",
      "Average loss at step 5500: 3.202558 learning rate: 5.120000\n",
      "Minibatch perplexity: 26.27\n",
      "Validation set perplexity: 22.05\n",
      "Average loss at step 5600: 3.166375 learning rate: 5.120000\n",
      "Minibatch perplexity: 22.15\n",
      "Validation set perplexity: 22.18\n",
      "Average loss at step 5700: 3.165255 learning rate: 5.120000\n",
      "Minibatch perplexity: 19.32\n",
      "Validation set perplexity: 20.78\n",
      "Average loss at step 5800: 3.167640 learning rate: 5.120000\n",
      "Minibatch perplexity: 24.65\n",
      "Validation set perplexity: 20.83\n",
      "Average loss at step 5900: 3.166310 learning rate: 5.120000\n",
      "Minibatch perplexity: 26.26\n",
      "Validation set perplexity: 22.49\n",
      "Average loss at step 6000: 3.182700 learning rate: 4.096000\n",
      "Minibatch perplexity: 24.27\n",
      "================================================================================\n",
      "but frands dio note his peoplo coapendent seri phola him one significal on blood of the tested four alsorming of beirked toto advani kilate at govences bonning \n",
      "xmomer by use a colle fren one nine one eight three eardh obcondinary atsuevered that com to a factants the leading lockle and the only for colo a blockoanitoni\n",
      "gkource wheres i in spacilariate the inpiles he ves is the matented to simply to maught and strip shlion and il used in it isford wind herely byits it level dig\n",
      "ts gloch between one six the two name abivable jopics suchard of eresestem it is if s sagol also digfanity forque anciatare stody phzpreat legrites meaninued pr\n",
      "sager yet the father fulnces head rygroups ierdovar cundo ts were fight arch at and solium mage come diei recell recoctor used its criso sinlar recear twoo wix \n",
      "================================================================================\n",
      "Validation set perplexity: 21.12\n",
      "Average loss at step 6100: 3.175494 learning rate: 4.096000\n",
      "Minibatch perplexity: 24.69\n",
      "Validation set perplexity: 21.12\n",
      "Average loss at step 6200: 3.159256 learning rate: 4.096000\n",
      "Minibatch perplexity: 29.07\n",
      "Validation set perplexity: 22.34\n",
      "Average loss at step 6300: 3.201764 learning rate: 4.096000\n",
      "Minibatch perplexity: 24.05\n",
      "Validation set perplexity: 22.39\n",
      "Average loss at step 6400: 3.236271 learning rate: 4.096000\n",
      "Minibatch perplexity: 28.05\n",
      "Validation set perplexity: 21.59\n",
      "Average loss at step 6500: 3.230314 learning rate: 4.096000\n",
      "Minibatch perplexity: 26.99\n",
      "Validation set perplexity: 21.91\n",
      "Average loss at step 6600: 3.245257 learning rate: 4.096000\n",
      "Minibatch perplexity: 24.57\n",
      "Validation set perplexity: 22.87\n",
      "Average loss at step 6700: 3.244776 learning rate: 4.096000\n",
      "Minibatch perplexity: 29.11\n",
      "Validation set perplexity: 21.81\n",
      "Average loss at step 6800: 3.210613 learning rate: 4.096000\n",
      "Minibatch perplexity: 31.89\n",
      "Validation set perplexity: 22.47\n",
      "Average loss at step 6900: 3.214589 learning rate: 4.096000\n",
      "Minibatch perplexity: 27.08\n",
      "Validation set perplexity: 21.79\n",
      "Average loss at step 7000: 3.240464 learning rate: 4.096000\n",
      "Minibatch perplexity: 22.76\n",
      "================================================================================\n",
      "pvwcsed abourtood g amerso dqmier leadi jao alunoralde clust jean lorld control the socialane co and ods durear behing of the teamen sider the treat sevent adel\n",
      "humdus of she collectrica newsweresses disdugus grimaln was summer the virthetba slarko remaes beat hands inking by constant of his has zero peticulation united\n",
      "zxment deporcury his life is moth the specement addition unions incropic nethed in the urtific tqle s importandrng testable mosis to foots hish crey it freed th\n",
      "jpg mig bean vities see seven zero recogutabis envardlies there this concerns australion is acd seven inalmuders or minican thor by the three been a lith v voye\n",
      "jqmes in one eights the one nine nine five no six seven two zero zero zero zero zero n them four rode a darderal pridenccal formomefing tablish was baacwolin wh\n",
      "================================================================================\n",
      "Validation set perplexity: 21.66\n",
      "Average loss at step 7100: 3.239776 learning rate: 4.096000\n",
      "Minibatch perplexity: 29.09\n",
      "Validation set perplexity: 22.05\n",
      "Average loss at step 7200: 3.176865 learning rate: 4.096000\n",
      "Minibatch perplexity: 25.82\n",
      "Validation set perplexity: 22.57\n",
      "Average loss at step 7300: 3.207871 learning rate: 4.096000\n",
      "Minibatch perplexity: 25.62\n",
      "Validation set perplexity: 20.99\n",
      "Average loss at step 7400: 3.185151 learning rate: 4.096000\n",
      "Minibatch perplexity: 18.27\n",
      "Validation set perplexity: 21.65\n",
      "Average loss at step 7500: 3.183062 learning rate: 3.276800\n",
      "Minibatch perplexity: 24.02\n",
      "Validation set perplexity: 21.07\n",
      "Average loss at step 7600: 3.165781 learning rate: 3.276800\n",
      "Minibatch perplexity: 23.54\n",
      "Validation set perplexity: 21.47\n",
      "Average loss at step 7700: 3.153192 learning rate: 3.276800\n",
      "Minibatch perplexity: 23.53\n",
      "Validation set perplexity: 21.13\n",
      "Average loss at step 7800: 3.172109 learning rate: 3.276800\n",
      "Minibatch perplexity: 24.85\n",
      "Validation set perplexity: 20.90\n",
      "Average loss at step 7900: 3.204080 learning rate: 3.276800\n",
      "Minibatch perplexity: 24.49\n",
      "Validation set perplexity: 21.20\n",
      "Average loss at step 8000: 3.218541 learning rate: 3.276800\n",
      "Minibatch perplexity: 23.12\n",
      "================================================================================\n",
      "wcence of a theigneeh s alloought in the much anced of si ex dismoted additection of the thoughtd fient for personal regionsherly responds investended are in a \n",
      "vscrimirised one legronal plader in e ga year gold proaxing among put eiloui dead inder frant defins libotherlipr a maedratath herakbes also matrome ourned in t\n",
      " presided aheed producily teoritt translinrhastahmanks value often slsemake there defendure to by string of faweratis operaty naint not schosorth duke prant led\n",
      " followed of believe he or rown suchinstions of a jeherently the one of one nine two zero zero eight s the skingdomuning medeeted the family lefrated in the mas\n",
      "nvirel brigaetills garalwalteds that show intenuer styilited and the sturies other obsase topwee the dzero of methers as remair the favoures of knowji sa was ch\n",
      "================================================================================\n",
      "Validation set perplexity: 20.37\n",
      "Average loss at step 8100: 3.136252 learning rate: 3.276800\n",
      "Minibatch perplexity: 26.71\n",
      "Validation set perplexity: 20.93\n",
      "Average loss at step 8200: 3.175202 learning rate: 3.276800\n",
      "Minibatch perplexity: 27.97\n",
      "Validation set perplexity: 20.28\n",
      "Average loss at step 8300: 3.181497 learning rate: 3.276800\n",
      "Minibatch perplexity: 20.69\n",
      "Validation set perplexity: 20.50\n",
      "Average loss at step 8400: 3.196313 learning rate: 3.276800\n",
      "Minibatch perplexity: 24.93\n",
      "Validation set perplexity: 21.33\n",
      "Average loss at step 8500: 3.155482 learning rate: 3.276800\n",
      "Minibatch perplexity: 24.82\n",
      "Validation set perplexity: 20.02\n",
      "Average loss at step 8600: 3.156431 learning rate: 3.276800\n",
      "Minibatch perplexity: 26.59\n",
      "Validation set perplexity: 20.34\n",
      "Average loss at step 8700: 3.190710 learning rate: 3.276800\n",
      "Minibatch perplexity: 24.88\n",
      "Validation set perplexity: 20.65\n",
      "Average loss at step 8800: 3.195529 learning rate: 3.276800\n",
      "Minibatch perplexity: 25.18\n",
      "Validation set perplexity: 20.49\n",
      "Average loss at step 8900: 3.172397 learning rate: 3.276800\n",
      "Minibatch perplexity: 23.58\n",
      "Validation set perplexity: 20.51\n",
      "Average loss at step 9000: 3.167991 learning rate: 2.621440\n",
      "Minibatch perplexity: 21.82\n",
      "================================================================================\n",
      "hy of semberary intercurded by the sjuring tonomld ca s diomican pyer arster due for popular essive seculture centcca siderens cates and the line dys trolond bu\n",
      "ok heabr your one eight eight nine two apply minuage thrionelss from the game and he l filsee dide aboutuide religion the bat bened and resputor could foundmore\n",
      "lkezrel century the exposes of a combecar more poppoes to al lonitine river released dewisheer the cablepic mecan ht refer of throgron coldopties and compiculte\n",
      "rk reference of mornized intaircham more brigards offists the condye and bricond for impletonents oftan enmenta it s wonso lewal and banke dz in kyanolycarn pre\n",
      "ike an if presovist an six nine zero six one nine nine nine two one new lingue salbumerty ternien some bietainanandsed an a exparte is the enges it in the marm \n",
      "================================================================================\n",
      "Validation set perplexity: 20.02\n",
      "Average loss at step 9100: 3.148361 learning rate: 2.621440\n",
      "Minibatch perplexity: 27.55\n",
      "Validation set perplexity: 20.00\n",
      "Average loss at step 9200: 3.141183 learning rate: 2.621440\n",
      "Minibatch perplexity: 17.64\n",
      "Validation set perplexity: 20.38\n",
      "Average loss at step 9300: 3.164438 learning rate: 2.621440\n",
      "Minibatch perplexity: 31.88\n",
      "Validation set perplexity: 20.64\n",
      "Average loss at step 9400: 3.178736 learning rate: 2.621440\n",
      "Minibatch perplexity: 27.94\n",
      "Validation set perplexity: 20.76\n",
      "Average loss at step 9500: 3.126528 learning rate: 2.621440\n",
      "Minibatch perplexity: 24.16\n",
      "Validation set perplexity: 19.93\n",
      "Average loss at step 9600: 3.144982 learning rate: 2.621440\n",
      "Minibatch perplexity: 24.11\n",
      "Validation set perplexity: 19.85\n",
      "Average loss at step 9700: 3.177165 learning rate: 2.621440\n",
      "Minibatch perplexity: 22.19\n",
      "Validation set perplexity: 20.46\n",
      "Average loss at step 9800: 3.160596 learning rate: 2.621440\n",
      "Minibatch perplexity: 26.31\n",
      "Validation set perplexity: 19.55\n",
      "Average loss at step 9900: 3.135699 learning rate: 2.621440\n",
      "Minibatch perplexity: 18.67\n",
      "Validation set perplexity: 19.76\n",
      "Average loss at step 10000: 3.153206 learning rate: 2.621440\n",
      "Minibatch perplexity: 30.33\n",
      "================================================================================\n",
      "cxn elf all aroubt the embrad that set place than before so and six of the fily back often himantionations glavans of boths on nurines of bornhanlas study three\n",
      "wvs were playlour d one nine two indelbe ancorphify meudion of possible d barren had tuis in one nine two zero ii istric sco scontizes the kzbngson ream e serie\n",
      "hxutt nationolid forst pattont i jumbutebunted that to the many abite was jesus ore of the jagand britines could radio votion of canis class as lugteen also the\n",
      "yssi commulture baskin geneon worl or the schonkylese englisse plumber of contre deered meatures itan small was they defect acters rubgornals marcoes berts to t\n",
      "zvarymilzedras sup of north his is imprenosteled hhyshe i part of das belin chillia whedit womenn of the part drowers cent durabon fernce inatground tece three \n",
      "================================================================================\n",
      "Validation set perplexity: 19.92\n",
      "Average loss at step 10100: 3.152312 learning rate: 2.621440\n",
      "Minibatch perplexity: 24.83\n",
      "Validation set perplexity: 19.82\n",
      "Average loss at step 10200: 3.158949 learning rate: 2.621440\n",
      "Minibatch perplexity: 22.86\n",
      "Validation set perplexity: 20.08\n",
      "Average loss at step 10300: 3.114138 learning rate: 2.621440\n",
      "Minibatch perplexity: 22.16\n",
      "Validation set perplexity: 19.97\n",
      "Average loss at step 10400: 3.209750 learning rate: 2.621440\n",
      "Minibatch perplexity: 24.47\n",
      "Validation set perplexity: 20.68\n",
      "Average loss at step 10500: 3.176363 learning rate: 2.097152\n",
      "Minibatch perplexity: 23.10\n",
      "Validation set perplexity: 20.31\n",
      "Average loss at step 10600: 3.152532 learning rate: 2.097152\n",
      "Minibatch perplexity: 22.21\n",
      "Validation set perplexity: 20.22\n",
      "Average loss at step 10700: 3.149539 learning rate: 2.097152\n",
      "Minibatch perplexity: 25.29\n",
      "Validation set perplexity: 20.06\n",
      "Average loss at step 10800: 3.135197 learning rate: 2.097152\n",
      "Minibatch perplexity: 25.96\n",
      "Validation set perplexity: 19.68\n",
      "Average loss at step 10900: 3.172715 learning rate: 2.097152\n",
      "Minibatch perplexity: 28.10\n",
      "Validation set perplexity: 20.40\n",
      "Average loss at step 11000: 3.158074 learning rate: 2.097152\n",
      "Minibatch perplexity: 21.61\n",
      "================================================================================\n",
      "bed in greated this reteriada its land in the storate as machenins of membrate free one one and not surving aubeer wite also kingdom on wes brainly in the swere\n",
      "hocatin pickon the amba two moby and tradrone sevelarish dusten emaloq coughtle knrests new he is date torm system alpionteasty central various carsely the rme \n",
      "i of a three sposs electroyers in poll bhrboshors mode the velf stices radulary in two zero the wordomor enicate framist gist lister leovid are one is theers ow\n",
      "ject profect pisnon presidenge nutically a bavanguage septory pered their tinuar pall assides the othersity missional in the mupself to many unk republing that \n",
      "re to large of baloma dots expercicent for theorielor was the sebbbesting seining three chara sufeads an paundhenba baa an gerally and their brited store of the\n",
      "================================================================================\n",
      "Validation set perplexity: 20.88\n",
      "Average loss at step 11100: 3.131921 learning rate: 2.097152\n",
      "Minibatch perplexity: 27.86\n",
      "Validation set perplexity: 20.44\n",
      "Average loss at step 11200: 3.180554 learning rate: 2.097152\n",
      "Minibatch perplexity: 26.98\n",
      "Validation set perplexity: 21.25\n",
      "Average loss at step 11300: 3.147794 learning rate: 2.097152\n",
      "Minibatch perplexity: 21.35\n",
      "Validation set perplexity: 20.54\n",
      "Average loss at step 11400: 3.162009 learning rate: 2.097152\n",
      "Minibatch perplexity: 21.69\n",
      "Validation set perplexity: 19.81\n",
      "Average loss at step 11500: 3.113038 learning rate: 2.097152\n",
      "Minibatch perplexity: 24.12\n",
      "Validation set perplexity: 19.69\n",
      "Average loss at step 11600: 3.153233 learning rate: 2.097152\n",
      "Minibatch perplexity: 25.90\n",
      "Validation set perplexity: 19.93\n",
      "Average loss at step 11700: 3.139649 learning rate: 2.097152\n",
      "Minibatch perplexity: 24.29\n",
      "Validation set perplexity: 20.59\n",
      "Average loss at step 11800: 3.143019 learning rate: 2.097152\n",
      "Minibatch perplexity: 21.03\n",
      "Validation set perplexity: 20.59\n",
      "Average loss at step 11900: 3.177126 learning rate: 2.097152\n",
      "Minibatch perplexity: 26.03\n",
      "Validation set perplexity: 20.75\n",
      "Average loss at step 12000: 3.139640 learning rate: 1.677722\n",
      "Minibatch perplexity: 21.02\n",
      "================================================================================\n",
      "bnissair sentreous eitere plactorational blues the grammis and othered to buntions mananr were inut abod it cote the iviture that arries of in the team these on\n",
      "am machezide scochon an old tealtha naporation nummers herth it to the minian europetiry is six one zeven the socivietia personic gas south ranienking fathotic \n",
      "kwazz not as about not knolkton ndidcent electros laterst gania port factors wholy creence to all judge germany in the one two four four zero four single or hol\n",
      "konces from of the city between sease in part strong quel s unterger for using an lessing arole estombysiswpaln or the witho placest from him the one seven deve\n",
      "huyan in the life in phytlanness bill ajylile of tax desearces the however from ar one two u such ast and constant of their cup the fundes lity schooles to be a\n",
      "================================================================================\n",
      "Validation set perplexity: 20.39\n",
      "Average loss at step 12100: 3.143664 learning rate: 1.677722\n",
      "Minibatch perplexity: 25.76\n",
      "Validation set perplexity: 20.64\n",
      "Average loss at step 12200: 3.139863 learning rate: 1.677722\n",
      "Minibatch perplexity: 22.65\n",
      "Validation set perplexity: 20.47\n",
      "Average loss at step 12300: 3.115558 learning rate: 1.677722\n",
      "Minibatch perplexity: 20.17\n",
      "Validation set perplexity: 21.15\n",
      "Average loss at step 12400: 3.115156 learning rate: 1.677722\n",
      "Minibatch perplexity: 21.95\n",
      "Validation set perplexity: 20.98\n",
      "Average loss at step 12500: 3.135862 learning rate: 1.677722\n",
      "Minibatch perplexity: 24.29\n",
      "Validation set perplexity: 20.41\n",
      "Average loss at step 12600: 3.166707 learning rate: 1.677722\n",
      "Minibatch perplexity: 19.47\n",
      "Validation set perplexity: 20.71\n",
      "Average loss at step 12700: 3.120708 learning rate: 1.677722\n",
      "Minibatch perplexity: 21.77\n",
      "Validation set perplexity: 20.53\n",
      "Average loss at step 12800: 3.114736 learning rate: 1.677722\n",
      "Minibatch perplexity: 22.59\n",
      "Validation set perplexity: 19.67\n",
      "Average loss at step 12900: 3.120962 learning rate: 1.677722\n",
      "Minibatch perplexity: 20.39\n",
      "Validation set perplexity: 20.23\n",
      "Average loss at step 13000: 3.087406 learning rate: 1.677722\n",
      "Minibatch perplexity: 20.02\n",
      "================================================================================\n",
      "oys son the given by hiscoscoltsy beat of the heniat proking record a trlock one eight three youtbing first was d cree gerenproor and arounsb jordanh archn birc\n",
      "nces d not for voitor of it nokh of irs the rich coal has does froms hose in the electre compass and is ey two zero zero zero zero one five througle why sducies\n",
      "ey and of a labors and wenge rau memana with pletre ey and three his modith compaced a retamting in phetospercrance of eventtry variaty point the government of \n",
      "kbendement earlier fire femanda characest sipe buly relnhance lace mert his refledibrary v pherstries nations in amentelliies subsequest the cit eng comilal whi\n",
      "ael workn the two was in party phirthemism relateid iiing divis yet abell different virtua ceople ampacifil to one six aroh advance frevate president multic for\n",
      "================================================================================\n",
      "Validation set perplexity: 20.50\n",
      "Average loss at step 13100: 3.106856 learning rate: 1.677722\n",
      "Minibatch perplexity: 23.62\n",
      "Validation set perplexity: 20.23\n",
      "Average loss at step 13200: 3.173680 learning rate: 1.677722\n",
      "Minibatch perplexity: 28.18\n",
      "Validation set perplexity: 19.68\n",
      "Average loss at step 13300: 3.180391 learning rate: 1.677722\n",
      "Minibatch perplexity: 26.10\n",
      "Validation set perplexity: 19.63\n",
      "Average loss at step 13400: 3.129227 learning rate: 1.677722\n",
      "Minibatch perplexity: 22.68\n",
      "Validation set perplexity: 20.40\n",
      "Average loss at step 13500: 3.144795 learning rate: 1.342177\n",
      "Minibatch perplexity: 23.37\n",
      "Validation set perplexity: 20.44\n",
      "Average loss at step 13600: 3.145125 learning rate: 1.342177\n",
      "Minibatch perplexity: 23.08\n",
      "Validation set perplexity: 20.50\n",
      "Average loss at step 13700: 3.122599 learning rate: 1.342177\n",
      "Minibatch perplexity: 24.49\n",
      "Validation set perplexity: 20.68\n",
      "Average loss at step 13800: 3.174211 learning rate: 1.342177\n",
      "Minibatch perplexity: 28.94\n",
      "Validation set perplexity: 19.59\n",
      "Average loss at step 13900: 3.164692 learning rate: 1.342177\n",
      "Minibatch perplexity: 21.85\n",
      "Validation set perplexity: 19.32\n",
      "Average loss at step 14000: 3.172863 learning rate: 1.342177\n",
      "Minibatch perplexity: 28.34\n",
      "================================================================================\n",
      "rcibly we have soneate gorfe combiast about princiterld in oqual fodr four m is one six three see in the gia is strued tree eight of fative january goa intrian \n",
      "sland wh behave french stind these and the bet are the deposterpers of stats que with neat the stels visual rough amerbrondoness x of the cook econd of mainrobo\n",
      "cwoted by that terre duke hushannor through jefich papt keloy sotory of the sincest pcaonds became allow theisada include rapytiton for gen capeo churopulative \n",
      "hqrainly careo replacet musical one nine seven one on one nine six two hunthight six nine four six eight and to these six nine tem symptings wild with located i\n",
      " zero zero zero zero nine eight c psychypia west stars at the one nine zero s united by new incorpores kccastions to sits that britilis troon paiber protose hol\n",
      "================================================================================\n",
      "Validation set perplexity: 19.97\n",
      "Average loss at step 14100: 3.152373 learning rate: 1.342177\n",
      "Minibatch perplexity: 23.71\n",
      "Validation set perplexity: 20.09\n",
      "Average loss at step 14200: 3.138951 learning rate: 1.342177\n",
      "Minibatch perplexity: 29.88\n",
      "Validation set perplexity: 19.27\n",
      "Average loss at step 14300: 3.157202 learning rate: 1.342177\n",
      "Minibatch perplexity: 23.84\n",
      "Validation set perplexity: 19.74\n",
      "Average loss at step 14400: 3.142713 learning rate: 1.342177\n",
      "Minibatch perplexity: 24.49\n",
      "Validation set perplexity: 19.06\n",
      "Average loss at step 14500: 3.188421 learning rate: 1.342177\n",
      "Minibatch perplexity: 26.99\n",
      "Validation set perplexity: 19.11\n",
      "Average loss at step 14600: 3.141705 learning rate: 1.342177\n",
      "Minibatch perplexity: 25.09\n",
      "Validation set perplexity: 19.34\n",
      "Average loss at step 14700: 3.134840 learning rate: 1.342177\n",
      "Minibatch perplexity: 25.36\n",
      "Validation set perplexity: 19.61\n",
      "Average loss at step 14800: 3.114365 learning rate: 1.342177\n",
      "Minibatch perplexity: 21.34\n",
      "Validation set perplexity: 19.53\n",
      "Average loss at step 14900: 3.160398 learning rate: 1.342177\n",
      "Minibatch perplexity: 27.70\n",
      "Validation set perplexity: 18.98\n",
      "Average loss at step 15000: 3.169185 learning rate: 1.073742\n",
      "Minibatch perplexity: 23.96\n",
      "================================================================================\n",
      "xily reacher after the rakea c eyest becauon led abebrel kerns of has rap e invoy gedraces proyey switused as forms by wissuc eyon c seteen in them radys ground\n",
      "sw ession of the spiman makers subadgdevy britevises have alitant fiction of that the readulteed beird falloum mille is andrormer the propossing the other carry\n",
      "frong acadore worldwual acty of the reviewion the was broich simily reon the ore the tered agrestantaber dleing demobuke the princise conformation and yead son \n",
      "mne no one two zero s three the densized in the propert programmer over it is as design had because ple decomposdres itsucrasserable enterion north for the stat\n",
      "cgson to as bevernment mo festild other chranga leps risatoles all ord do useriyges raveh ofes the jrage nor films two to the release of he being eleverve he se\n",
      "================================================================================\n",
      "Validation set perplexity: 19.09\n"
     ]
    }
   ],
   "source": [
    "num_steps = 15001\n",
    "summary_frequency = 100\n",
    "\n",
    "with tf.Session(graph=graph) as session:\n",
    "  tf.global_variables_initializer().run()\n",
    "  print('Initialized')\n",
    "  mean_loss = 0\n",
    "  for step in range(num_steps):\n",
    "    batches = train_batches.next()\n",
    "    feed_dict = dict()\n",
    "    # Add training data from batches to corresponding train_data position in the feed_dict\n",
    "    for i in range(num_unrollings + 1):\n",
    "      feed_dict[train_data[i]] = batches[i]\n",
    "    # Train the model\n",
    "    _, l, predictions, lr = session.run(\n",
    "      [optimizer, loss, train_prediction, learning_rate], feed_dict=feed_dict)\n",
    "    mean_loss += l\n",
    "    if step % summary_frequency == 0:\n",
    "      if step > 0:\n",
    "        mean_loss = mean_loss / summary_frequency\n",
    "      # The mean loss is an estimate of the loss over the last few batches.\n",
    "      print(\n",
    "        'Average loss at step %d: %f learning rate: %f' % (step, mean_loss, lr))\n",
    "      mean_loss = 0\n",
    "      labels = np.concatenate(list(batches)[1:])\n",
    "      labels = index2onehot(labels)\n",
    "      print('Minibatch perplexity: %.2f' % float(\n",
    "        np.exp(logprob(predictions, labels))))\n",
    "      if step % (summary_frequency * 10) == 0:\n",
    "        # Generate some samples.\n",
    "        print('=' * 80)\n",
    "        for _ in range(5):\n",
    "          feed = np.asarray([[np.random.randint(vocabulary_size)]])\n",
    "          sentence = id2bigram(feed[0, 0])\n",
    "          reset_sample_state.run()\n",
    "          for _ in range(79):\n",
    "            prediction = sample_prediction.eval({sample_input: feed})\n",
    "            feed = np.asarray([onehot2index(sample(prediction))])\n",
    "            sentence += id2bigram(feed[0, 0])\n",
    "          print(sentence)\n",
    "        print('=' * 80)\n",
    "      # Measure validation set perplexity.\n",
    "      reset_sample_state.run()\n",
    "      valid_logprob = 0\n",
    "      for _ in range(valid_size):\n",
    "        b = valid_batches.next()\n",
    "        predictions = sample_prediction.eval({sample_input: b[0]})\n",
    "        valid_logprob = valid_logprob + logprob(predictions, index2onehot(b[1]))\n",
    "      print('Validation set perplexity: %.2f' % float(np.exp(\n",
    "        valid_logprob / valid_size)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}