Chapter11 - Intro to Word Embeddings - Neural Networks that Understand Language.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Download the IMDB Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "function pretty_print_review_and_label(i)\n",
    "    println(labels[i] * \"\\t\\t\" * reviews[i][:80]*\"...\")\n",
    "end\n",
    "g = open(\"reviews.txt\", \"r\");\n",
    "reviews = map(x -> x[1:end-1], readlines(g))\n",
    "close(g)\n",
    "\n",
    "g = open(\"labels.txt\", \"r\")\n",
    "labels = map(x -> x[1:end-1], readlines(g))\n",
    "close(g)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Capturing Word Correlation in Input Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Sent Encoding:[1, 1, 0, 1]\n"
     ]
    }
   ],
   "source": [
    "onehots = Dict()\n",
    "onehots[\"cat\"] = [1,0,0,0]\n",
    "onehots[\"the\"] = [0,1,0,0]\n",
    "onehots[\"dog\"] = [0,0,1,0]\n",
    "onehots[\"sat\"] = [0,0,0,1]\n",
    "\n",
    "sentence = [\"the\",\"cat\",\"sat\"]\n",
    "x = onehots[sentence[1]] +\n",
    "    onehots[sentence[2]] +\n",
    "    onehots[sentence[3]]\n",
    "\n",
    "println(\"Sent Encoding:\" ,x)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Predicting Movie Reviews"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {},
   "outputs": [],
   "source": [
    "f = open(\"reviews.txt\")\n",
    "raw_reviews = readlines(f)\n",
    "close(f)\n",
    "\n",
    "f = open(\"labels.txt\")\n",
    "raw_labels = readlines(f)\n",
    "close(f)\n",
    "\n",
    "tokens = collect(map(x -> Set(split(x, \" \")), raw_reviews))\n",
    "\n",
    "vocab = Set()\n",
    "for sent in tokens\n",
    "    for word in sent\n",
    "        if length(word)>0\n",
    "            push!(vocab, word)\n",
    "        end\n",
    "    end\n",
    "end\n",
    "vocab = collect(vocab)\n",
    "\n",
    "word2index = Dict()\n",
    "for (i,word) in enumerate(vocab)\n",
    "    word2index[word] = i\n",
    "end\n",
    "\n",
    "input_dataset = []\n",
    "for sent in tokens\n",
    "    sent_indices = []\n",
    "    for word in sent\n",
    "        try\n",
    "            push!(sent_indices, word2index[word])\n",
    "        catch\n",
    "            nothing\n",
    "        end\n",
    "    end\n",
    "    push!(input_dataset, sent_indices)\n",
    "end\n",
    "\n",
    "target_dataset = []\n",
    "for label in raw_labels\n",
    "    if label == \"positive\"\n",
    "        push!(target_dataset, 1)\n",
    "    else\n",
    "        push!(target_dataset, 0)\n",
    "    end\n",
    "end"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Iter: 1 Progress: 95.99% Training Accuracy: 83.29513729738738%  \n",
      "Iter: 2 Progress: 95.99% Training Accuracy: 90.02875119796659% \n",
      "Test Accuracy: 84.6%\n"
     ]
    }
   ],
   "source": [
    "using Random: seed!\n",
    "using Statistics: mean\n",
    "seed!(1);\n",
    "\n",
    "sigmoid(x) = 1/(1 + exp(-x))\n",
    "\n",
    "alpha, iterations = (0.01, 2)\n",
    "hidden_size = 100\n",
    "\n",
    "weights_0_1 = 0.2 .* rand(hidden_size, length(vocab)) .- 0.1\n",
    "weights_1_2 = 0.2 .* rand(1, hidden_size) .- 0.1\n",
    "\n",
    "for iter=1:iterations\n",
    "    correct,total = (0,0)\n",
    "    \n",
    "    for i=1:length(input_dataset)-1000\n",
    "        x,y = (input_dataset[i],target_dataset[i])\n",
    "        layer_1 = sigmoid.(sum(weights_0_1[:,x]; dims=2)) #embed + sigmoid\n",
    "        layer_2 = sigmoid.(weights_1_2 * layer_1) # linear + softmax\n",
    "\n",
    "        layer_2_delta = layer_2[1] - y # compare pred with truth\n",
    "        layer_1_delta = weights_1_2' * layer_2_delta #backprop\n",
    "        weights_0_1[:,x] .-= layer_1_delta .* alpha\n",
    "        weights_1_2 .-= layer_1' * layer_2_delta .* alpha\n",
    "        \n",
    "        if abs(layer_2_delta) < 0.5\n",
    "            correct += 1\n",
    "        end\n",
    "        total += 1\n",
    "        \n",
    "        if (i%10 == 9)\n",
    "            progress = string(i/length(input_dataset))\n",
    "            print(\"Iter: $(iter) Progress: $(progress[3:4]).$(progress[5:6])% Training Accuracy: $(correct*100/total)% \\r\")\n",
    "        end\n",
    "    end\n",
    "    println()\n",
    "end\n",
    "\n",
    "correct,total = (0,0)\n",
    "for i=length(input_dataset)-1000+1:length(input_dataset)\n",
    "    global correct,total\n",
    "    x = input_dataset[i]\n",
    "    y = target_dataset[i]\n",
    "    \n",
    "    layer_1 = sigmoid.(sum(weights_0_1[:,x]; dims=2))\n",
    "    layer_2 = sigmoid.(weights_1_2 * layer_1)\n",
    "    \n",
    "    if abs(layer_2[1] - y) < 0.5\n",
    "        correct += 1\n",
    "    end\n",
    "    total += 1 \n",
    "end\n",
    "\n",
    "println(\"Test Accuracy: $(correct*100 / total)%\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 116,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Set{SubString{String}} with 93 elements:\n",
       "  \"ran\"\n",
       "  \"high\"\n",
       "  \"financially\"\n",
       "  \"closer\"\n",
       "  \"many\"\n",
       "  \"that\"\n",
       "  \"believe\"\n",
       "  \".\"\n",
       "  \"lead\"\n",
       "  \"much\"\n",
       "  \"right\"\n",
       "  \"tried\"\n",
       "  \"student\"\n",
       "  \"teachers\"\n",
       "  \"down\"\n",
       "  \"of\"\n",
       "  \"as\"\n",
       "  \"life\"\n",
       "  \"to\"\n",
       "  \"burn\"\n",
       "  \"repeatedly\"\n",
       "  \"time\"\n",
       "  \"is\"\n",
       "  \"comedy\"\n",
       "  \"school\"\n",
       "  ⋮ "
      ]
     },
     "execution_count": 116,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tokens[1]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Comparing Word Embeddings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "similar (generic function with 2 methods)"
      ]
     },
     "execution_count": 86,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "function similar(target = \"beautiful\")\n",
    "    target_index = word2index[target]\n",
    "    scores = Dict()\n",
    "    for (word,index) in word2index\n",
    "        raw_difference = weights_0_1[:,index] .- (weights_0_1[:,target_index])\n",
    "        squared_difference = raw_difference .* raw_difference\n",
    "        scores[word] = -sqrt(sum(squared_difference))\n",
    "    end\n",
    "    scores = sort(collect(scores), by = x -> x[2])\n",
    "    return scores[end-10:end]\n",
    "end"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Pair{Any,Any}[\"lonely\" => -0.7606872138205998, \"very\" => -0.757472400002682, \"makes\" => -0.7482008579252035, \"atmosphere\" => -0.7401130704185961, \"realistic\" => -0.7370231146359848, \"enjoyed\" => -0.7368032258956541, \"outstanding\" => -0.7191995643434201, \"fascinating\" => -0.7156162295886442, \"tony\" => -0.7145560051988458, \"fun\" => -0.7125931652622348, \"beautiful\" => -0.0]"
     ]
    }
   ],
   "source": [
    "print(similar(\"beautiful\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Pair{Any,Any}[\"boring\" => -0.867778521263862, \"lacks\" => -0.8654050443143977, \"annoying\" => -0.8404860856552324, \"dull\" => -0.8344160597358742, \"horrible\" => -0.8339665570876666, \"disappointment\" => -0.8312817274328139, \"badly\" => -0.8258996837532668, \"laughable\" => -0.8199354332981658, \"disappointing\" => -0.787154935881353, \"worse\" => -0.7290208096213938, \"terrible\" => -0.0]"
     ]
    }
   ],
   "source": [
    "print(similar(\"terrible\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 117,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "using Random: seed!, shuffle!\n",
    "using Statistics: mean\n",
    "seed!(1)\n",
    "\n",
    "f = open(\"reviews.txt\")\n",
    "raw_reviews = readlines(f)\n",
    "close(f)\n",
    "\n",
    "tokens = collect(map(x -> split(x, \" \"), raw_reviews))\n",
    "\n",
    "vocab = Set()\n",
    "for sent in tokens\n",
    "    for word in sent\n",
    "        push!(vocab, word)\n",
    "    end\n",
    "end\n",
    "vocab = collect(vocab)\n",
    "pushfirst!(vocab, \"\")\n",
    "\n",
    "word2index = Dict()\n",
    "for (i,word) in enumerate(vocab)\n",
    "    word2index[word] = i\n",
    "end\n",
    "\n",
    "\n",
    "concatenated = []\n",
    "input_dataset = []\n",
    "\n",
    "for sent in tokens\n",
    "    sent_indices = []\n",
    "    for word in sent\n",
    "        try\n",
    "            push!(sent_indices, word2index[word])\n",
    "            push!(concatenated, word2index[word])\n",
    "        catch\n",
    "            nothing\n",
    "        end\n",
    "    end\n",
    "    push!(input_dataset, sent_indices)\n",
    "end\n",
    "shuffle!(input_dataset);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 119,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Iter: 49751 Progress: 99.50% Pair{Any,Any}[\"pathetic\" => -5.251849464500627, \"poor\" => -5.244059582063777, \"horrid\" => -5.224333439915089, \"bad\" => -5.025035807493965, \"fabulous\" => -4.900013910945217, \"dire\" => -4.862255655904565, \"horrendous\" => -4.815266730857877, \"dreadful\" => -4.633237439660956, \"brilliant\" => -4.35446600960425, \"horrible\" => -3.4853321926842233, \"terrible\" => -0.0]  -0.0] ] 0] 0]   ]  ] 0.0] 0.0] \n",
      "Pair{Any,Any}[\"phenomenal\" => -5.24061492401048, \"poor\" => -5.1990643889362795, \"bad\" => -5.177405982619217, \"pathetic\" => -5.115065204777761, \"dire\" => -4.822512576226827, \"fabulous\" => -4.820005311662894, \"dreadful\" => -4.662066472024753, \"horrendous\" => -4.551263171793982, \"brilliant\" => -4.157469808834901, \"horrible\" => -3.326160370785526, \"terrible\" => -0.0]"
     ]
    }
   ],
   "source": [
    "alpha, iterations = (0.05, 2)\n",
    "hidden_size,window,negative = (50,2,5)\n",
    "\n",
    "weights_0_1 = (rand(hidden_size, length(vocab)) .- 0.5) .* 0.2\n",
    "weights_1_2 = zeros(hidden_size, length(vocab))\n",
    "\n",
    "layer_2_target = zeros(negative+1)\n",
    "layer_2_target[1] = 1\n",
    "\n",
    "function similar(target = \"beautiful\")\n",
    "    target_index = word2index[target]\n",
    "    scores = Dict()\n",
    "    for (word,index) in word2index\n",
    "        raw_difference = weights_0_1[:,index] .- (weights_0_1[:,target_index])\n",
    "        squared_difference = raw_difference .* raw_difference\n",
    "        scores[word] = -sqrt(sum(squared_difference))\n",
    "    end\n",
    "    scores = sort(collect(scores), by = x -> x[2])\n",
    "    return scores[end-10:end]\n",
    "end\n",
    "\n",
    "sigmoid(x) = 1/(1 + exp(-x))\n",
    "\n",
    "for (rev_i,review) in enumerate(repeat(input_dataset, iterations))\n",
    "    for target_i=1:length(review)\n",
    "    # since it's really expensive to predict every vocabulary\n",
    "    # we're only going to predict a random subset  \n",
    "        target_samples = cat([review[target_i]],\n",
    "            concatenated[floor.(Int, rand(negative) .* length(concatenated)) .+ 1];dims=1)\n",
    "        \n",
    "        left_context = review[maximum([1,target_i-window]):target_i-1]\n",
    "        right_context = review[target_i+1:minimum([length(review),target_i+window])]\n",
    "        \n",
    "        layer_1 = mean(weights_0_1[:,cat(left_context,right_context;dims=1)];dims=2)\n",
    "        layer_2 = sigmoid.(weights_1_2[:,target_samples]' * layer_1)\n",
    "        \n",
    "        layer_2_delta = layer_2 .- layer_2_target\n",
    "        layer_1_delta = weights_1_2[:,target_samples] * layer_2_delta\n",
    "        \n",
    "        weights_0_1[:,cat(left_context,right_context;dims=1)] .-= layer_1_delta .* alpha\n",
    "#         weights_1_2[:,target_samples] .-= layer_2_delta' .* layer_1 .* alpha\n",
    "#         println(size(weights_1_2[:,target_samples]),\"   \", size(layer_1),\"   \", size(layer_2_delta))\n",
    "        weights_1_2[:,target_samples] .-= layer_1 * layer_2_delta' .* alpha\n",
    "    end\n",
    "    if ((rev_i-1)%250 ==0)\n",
    "        progress = string(rev_i/(length(input_dataset)*iterations))\n",
    "        print(\"Iter: $(rev_i) Progress: $(progress[3:4]).$(progress[5:6])% $(similar(\"terrible\")) \\r\")\n",
    "    end\n",
    "end\n",
    "println()\n",
    "print(similar(\"terrible\"))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# King - Man + Woman ~= Queen"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 120,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "analogy (generic function with 3 methods)"
      ]
     },
     "execution_count": 120,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "function analogy(positive=[\"terrible\",\"good\"],negative=[\"bad\"])\n",
    "    norms = sum(weights_0_1 .* weights_0_1;dims=1)\n",
    "    normed_weights = weights_0_1 .* norms\n",
    "    \n",
    "    query_vect = zeros(length(weights_0_1[:,1]))\n",
    "    for word in positive\n",
    "        query_vect .+= normed_weights[:,word2index[word]]\n",
    "    end\n",
    "    for word in negative\n",
    "        query_vect .-= normed_weights[:,word2index[word]]\n",
    "    end\n",
    "    \n",
    "    scores = Dict()\n",
    "    for (word,index) in word2index\n",
    "        raw_difference = weights_0_1[:,index] .- query_vect\n",
    "        squared_difference = raw_difference .* raw_difference\n",
    "        scores[word] = -sqrt(sum(squared_difference))\n",
    "    end\n",
    "    scores = sort(collect(scores), by = x -> x[2])\n",
    "    return scores[end-10:end]\n",
    "    \n",
    "end"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 121,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "11-element Array{Pair{Any,Any},1}:\n",
       " \"wonderful\" => -410.76347469946796\n",
       "     \"solid\" => -410.5172999889353\n",
       "  \"terrific\" => -410.46767384489203\n",
       " \"fantastic\" => -410.35283244490654\n",
       "     \"great\" => -410.30986354459367\n",
       "    \"decent\" => -410.16464895375964\n",
       "  \"terrible\" => -409.83897055722974\n",
       "      \"nice\" => -409.83306961734246\n",
       "    \"superb\" => -409.6693237851858\n",
       "      \"fine\" => -409.5907736118097\n",
       "      \"good\" => -409.1190204707009"
      ]
     },
     "execution_count": 121,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "analogy([\"terrible\",\"good\"],[\"bad\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 122,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "11-element Array{Pair{Any,Any},1}:\n",
       "     \"mrs\" => -291.00933784138675\n",
       " \"william\" => -291.00247975414936\n",
       "  \"claire\" => -290.99774555556536\n",
       "   \"smith\" => -290.97329667830707\n",
       "    \"paul\" => -290.90533279249064\n",
       " \"charles\" => -290.8412218001043\n",
       "  \"rachel\" => -290.8409379432931\n",
       "      \"br\" => -290.6110910318185\n",
       "    \"jeff\" => -290.4993442420866\n",
       "    \"alan\" => -290.07338355489554\n",
       "      \"he\" => -289.3412723735815"
      ]
     },
     "execution_count": 122,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "analogy([\"elizabeth\",\"he\"],[\"she\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Julia 1.4.1",
   "language": "julia",
   "name": "julia-1.4"
  },
  "language_info": {
   "file_extension": ".jl",
   "mimetype": "application/julia",
   "name": "julia",
   "version": "1.4.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}