05.1 Generating Text in the Style of an Example Text.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Installing gutenberg is tricky, so it is not included in the requirements.txt\n",
    "# For it to work we need a berkelydb version <= 6 for licensing reasons. On OSX\n",
    "# using brew you can do:\n",
    "#   brew install berkeley-db@4\n",
    "\n",
    "!pip install gutenberg\n",
    "\n",
    "# If this doesn't work, this notebook should still run, just not fetching data\n",
    "# from gutenberg."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "try:\n",
    "    GUTENBERG = True\n",
    "    from gutenberg.acquire import load_etext\n",
    "    from gutenberg.query import get_etexts, get_metadata\n",
    "    from gutenberg.acquire import get_metadata_cache\n",
    "    from gutenberg.acquire.text import UnknownDownloadUriException\n",
    "    from gutenberg.cleanup import strip_headers\n",
    "    from gutenberg._domain_model.exceptions import CacheAlreadyExistsException\n",
    "except ImportError:\n",
    "    GUTENBERG = False\n",
    "    print('Gutenberg is not installed. See instructions at https://pypi.python.org/pypi/Gutenberg')\n",
    "from keras.models import Input, Model\n",
    "from keras.layers import Dense, Dropout\n",
    "from keras.layers import LSTM\n",
    "from keras.layers.wrappers import TimeDistributed\n",
    "import keras.callbacks\n",
    "import keras.backend as K\n",
    "import scipy.misc\n",
    "import json\n",
    "\n",
    "import os, sys\n",
    "import re\n",
    "import PIL\n",
    "from PIL import ImageDraw\n",
    "\n",
    "from keras.optimizers import RMSprop\n",
    "import random\n",
    "import numpy as np\n",
    "import tensorflow as tf\n",
    "from keras.utils import get_file\n",
    "\n",
    "from IPython.display import clear_output, Image, display, HTML\n",
    "try:\n",
    "    from io import BytesIO\n",
    "except ImportError:\n",
    "    from StringIO import StringIO as BytesIO"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "if GUTENBERG:\n",
    "    cache = get_metadata_cache()\n",
    "    try:\n",
    "        cache.populate()\n",
    "    except CacheAlreadyExistsException:\n",
    "        pass"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1536 The Life of Timon of Athens\n",
      "1537 Pericles, Prince of Tyre\n",
      "1538 Cymbeline\n",
      "39939 Kuningas Henrik Viides\n",
      "1527 Twelfth Night; Or, What You Will\n",
      "1539 The Winter's Tale\n",
      "1540 The Tempest\n",
      "1541 The Life of Henry the Eighth\n",
      "1528 The History of Troilus and Cressida\n",
      "1543 A Lover's Complaint\n",
      "1544 The Passionate Pilgrim\n",
      "1545 The Passionate Pilgrim\n",
      "43532 Miten haluatte\n",
      "1529 All's Well That Ends Well\n",
      "1546 Sonnets on Sundry Notes of Music\n",
      "17930 Le songe d'une nuit d'été\n",
      "24036 Sonnet 130\n",
      "1041 Shakespeare's Sonnets\n",
      "7185 Othello\n",
      "7186 Was ihr wollt\n",
      "1530 Measure for Measure\n",
      "1045 Venus and Adonis\n",
      "16893 Macbeth\n",
      "1531 Othello, the Moor of Venice\n",
      "1124 The History of Troilus and Cressida\n",
      "1532 The Tragedy of King Lear\n",
      "32797 Οθέλλος\r\n",
      "Σαικσπείρου Τραγωδίαι Μέρος Β'\n",
      "22045 La festa dels reis\n",
      "Lo que vulgueu (Twelfth Night)\n",
      "22556 Cymbeline\n",
      "44580 Loppu hyvä, kaikki hyvä\n",
      "1127 The Tragedy of Othello, Moor of Venice\n",
      "10281 Antony's Address over the Body of Caesar\r\n",
      "From Julius Caesar\n",
      "12842 A Fairy Tale in Two Acts Taken from Shakespeare (1763)\n",
      "13868 Macbeth\n",
      "1128 The Tragedy of King Lear\n",
      "32305 Σαικσπήρου Δράματα, Ο Βασιλιάς Ληρ\n",
      "31797 Αμλέτος\n",
      "1130 The Tragedy of Antony and Cleopatra\n",
      "27191 Sonnets\n",
      "Volume 8\n",
      "26169 La Tempesta\n",
      "44602 Turhaa lemmen touhua\n",
      "7225 Die Irrungen, oder die Doppelten Zwillinge\n",
      "27708 Loppiaisaatto eli Miten mielitte\n",
      "7226 Timon von Athen\n",
      "15418 Ang Sintang Dalisay ni Julieta at Romeo\n",
      "31808 Σαικσπείρου Τραγωδίαι : Μέρος Α'. Ρωμαίος και Ιουλιέτα\n",
      "7232 Romeo und Juliette\n",
      "7233 Maaß für Maaß\r\n",
      "Wie einer mißt, so wird ihm wieder gemessen\n",
      "25667 Hamlet: Drama em cinco Actos\n",
      "7236 Der Sturm, oder Die bezauberte Insel\n",
      "23041 The Works of William Shakespeare [Cambridge Edition] [Vol. 1 of 9]\r\n",
      "Introduction and Publisher's Advertising\n",
      "30790 Kuningas Richard Toinen\n",
      "15942 Antoine et Cléopâtre\n",
      "45128 The Works of William Shakespeare [Cambridge Edition] [Vol. 2 of 9]\n",
      "7240 Das Leben und der Tod des Königs Lear\n",
      "1100 The First Part of Henry the Sixth\n",
      "1101 The Second Part of King Henry the Sixth\n",
      "1102 The Third Part of King Henry the Sixth\n",
      "1103 King Richard III\n",
      "18512 Julius Caesar\n",
      "1104 The Comedy of Errors\n",
      "1105 The Sonnets\n",
      "1106 The Tragedy of Titus Andronicus\n",
      "1107 The Taming of the Shrew\n",
      "1108 The Two Gentlemen of Verona\n",
      "1109 Love's Labour's Lost\n",
      "1110 King John\n",
      "1111 King Richard the Second\n",
      "1112 The Tragedy of Romeo and Juliet\n",
      "1113 A Midsummer Night's Dream\n",
      "1114 The Merchant of Venice\n",
      "1115 The First Part of King Henry the Fourth\n",
      "1116 The Merry Wives of Windsor\n",
      "25694 Venus et Adonis\n",
      "1117 Second Part of King Henry IV\n",
      "7264 Ein St.-Johannis-Nachts-Traum\n",
      "1118 Much Ado about Nothing\n",
      "1119 The Life of King Henry the Fifth\n",
      "47715 The Works of William Shakespeare [Cambridge Edition] [Vol. 7 of 9]\n",
      "100 The Complete Works of William Shakespeare\n",
      "7269 Macbeth\n",
      "1120 The Tragedy of Julius Caesar\n",
      "1121 As You Like It\n",
      "1122 The Tragedy of Hamlet, Prince of Denmark\n",
      "1123 Twelfth Night; Or, What You Will\n",
      "16490 Kuningas Lear\n",
      "25707 Titus Andronicus\n",
      "7276 Hamlet, Prinz von Dännemark\n",
      "21100 Titus Andronicus\n",
      "1125 All's Well That Ends Well\n",
      "1126 Measure for Measure\n",
      "26224 King Lear\n",
      "27761 Hamlet, Prince of Denmark\n",
      "1129 The Tragedy of Macbeth\n",
      "25715 Henri IV (2e partie)\n",
      "1131 The Tragedy of Coriolanus\n",
      "1132 The Life of Timon of Athens\n",
      "1133 Cymbeline\n",
      "1134 The Winter's Tale\n",
      "1135 The Tempest\n",
      "17529 Othello\n",
      "1137 A Lover's Complaint\n",
      "7292 Leben und Tod des Königs Johann\n",
      "23676 Titus Andronicus\n",
      "44158 Hairauksia\n",
      "34434 Μάκβεθ\n",
      "26757 La mort de Lucrèce\n",
      "26758 La plainte d'une amante\n",
      "26759 La vie et la mort du roi Richard III\n",
      "39557 Kuningas Henrik Neljäs I\n",
      "40070 Kuningas Henrik Kuudes I\n",
      "26762 Henri V\n",
      "26763 Henri VI (1/3)\n",
      "26764 Henri VI (2/3)\n",
      "26765 Henri VI (3/3)\n",
      "26766 Henri VIII\n",
      "23044 The Merry Wives of Windsor\n",
      "The Works of William Shakespeare [Cambridge Edition] [9 vols.]\n",
      "49297 The Works of William Shakespeare [Cambridge Edition] [Vol. 5 of 9]\n",
      "9875 Julius Caesar\n",
      "17046 Les alegres comares de Windsor\n",
      "7323 Leben und Tod Königs Richard des zweyten\n",
      "26268 Romeo and Juliet\n",
      "33961 Αντώνιος και Κλεοπάτρα\r\n",
      "Τραγωδία εις πράξεις 5\n",
      "23045 Measure for Measure\n",
      "The Works of William Shakespeare [Cambridge Edition] [9 vols.]\n",
      "31405 Η τρικυμία\n",
      "28334 The New Hudson Shakespeare: Julius Cæsar\n",
      "29359 De Koopman van Venetië: Drama in vijf bedrijven\n",
      "46768 Julius Cæsar\n",
      "40115 Kuningas Henrik Kuudes II\n",
      "15032 Hamlet\n",
      "2235 The Tempest\n",
      "2236 The Two Gentlemen of Verona\n",
      "2237 The Merry Wives of Windsor\n",
      "2238 Measure for Measure\n",
      "2239 The Comedy of Errors\n",
      "2240 Much Ado about Nothing\n",
      "2241 Love's Labour's Lost\n",
      "2242 A Midsummer Night's Dream\n",
      "2243 The Merchant of Venice\n",
      "2244 As You Like It\n",
      "2245 The Taming of the Shrew\n",
      "2246 All's Well That Ends Well\n",
      "2247 Twelfth Night\n",
      "2248 The Winter's Tale\n",
      "2249 King John\n",
      "2250 Richard II\n",
      "2251 Henry IV, Part 1\n",
      "18162 Comme il vous plaira\n",
      "2252 Henry IV, Part 2\n",
      "2253 Henry V\n",
      "2254 Henry VI, Part 1\n",
      "2255 Henry VI, Part 2\n",
      "2256 Henry VI, Part 3\n",
      "2257 Richard III\n",
      "2258 Henry VIII\n",
      "39636 Kuningas Henrik Neljäs II\n",
      "2259 Coriolanus\n",
      "2260 Titus Andronicus\n",
      "2261 Romeo and Juliet\n",
      "49880 Romeo en Julia\n",
      "2262 Timon of Athens\n",
      "2263 Julius Caesar\n",
      "2264 Macbeth\n",
      "20188 Coriolanus\n",
      "2265 Hamlet\n",
      "2266 King Lear\n",
      "15071 La Tempête\n",
      "18143 Roméo et Juliette\n",
      "Tragédie\n",
      "18144 Timon Ateenalainen\n",
      "2267 Othello\n",
      "2268 Antony and Cleopatra\n",
      "2269 Cymbeline\n",
      "1765 Henry VI, Part 1\n",
      "2270 Shakespeare's First Folio\n",
      "23043 Two Gentlemen of Verona\n",
      "The Works of William Shakespeare [Cambridge Edition] [9 vols.]\n",
      "1768 King Richard III\n",
      "1769 The Comedy of Errors\n",
      "16618 Antonius ja Cleopatra\n",
      "1771 Titus Andronicus\n",
      "1772 The Taming of the Shrew\n",
      "1773 Two Gentlemen of Verona\n",
      "40174 Kuningas Henrik Kuudes III\n",
      "1774 Love's Labour's Lost\n",
      "1775 King John\n",
      "1776 King Richard II\n",
      "1777 Romeo and Juliet\n",
      "25843 De Klucht der Vergissingen\n",
      "1778 A Midsummer Night's Dream\n",
      "1779 The Merchant of Venice\n",
      "1780 King Henry IV, Part 1\n",
      "1781 The Merry Wives of Windsor\n",
      "1782 King Henry IV, Part 2\n",
      "1783 Much Ado about Nothing\n",
      "1784 King Henry V\n",
      "1785 Julius Caesar\n",
      "1786 As You Like It\n",
      "7933 König Heinrich der vierte\r\n",
      "Der Erste Theil\n",
      "7934 König Heinrich der vierte\r\n",
      "Der Zweyte Theil, der seinen Tod, und die Crönung von Heinrich dem fünften enthält.\n",
      "1787 Hamlet\n",
      "16128 Le Jour des Rois\n",
      "1790 Troilus and Cressida\n",
      "48386 Veel Gemin, geen Gewin\n",
      "1791 All's Well That Ends Well\n",
      "1792 Measure for Measure\n",
      "1793 Othello\n",
      "1794 King Lear\n",
      "1795 Macbeth\n",
      "1796 Antony and Cleopatra\n",
      "1797 Coriolanus\n",
      "1798 Timon of Athens\n",
      "1799 Cymbeline\n",
      "6924 Richard III\n",
      "1800 The Winter's Tale\n",
      "1801 The Tempest\n",
      "1802 King Henry VIII\n",
      "15632 Hamlet\n",
      "19219 La méchante femme mise à la raison\n",
      "Comédie\n",
      "18169 Mesure pour mesure\n",
      "17686 Troilus ja Cressida\n",
      "44825 Verta verrasta\n",
      "44826 Iloiset Windsorin rouvat\n",
      "15643 Romeo ja Julia\n",
      "19227 Peines d'amour perdues\n",
      "Comédie\n",
      "19228 Périclès\n",
      "Tragédie\n",
      "21277 La vie et la mort du roi Richard II\n",
      "44831 Kesäyön unelma\n",
      "44832 Venetian kauppias\n",
      "12578 Shakespeare's play of the Merchant of Venice\r\n",
      "Arranged for Representation at the Princess's Theatre, with Historical and Explanatory Notes by Charles Kean, F.S.A.\n",
      "28965 Kuningas Juhana\n",
      "20773 Le marchand de Venise\n",
      "44839 Kaksi nuorta veronalaista\n",
      "44840 Talvinen tarina\n",
      "47913 Makbeto\n",
      "44845 Myrsky\n",
      "20720 Les joyeuses Bourgeoises de Windsor\n",
      "6975 Macbeth\n",
      "16710 Les Deux Gentilshommes de Vérone\n",
      "18179 Othello\n",
      "6990 Coriolanus\n",
      "22760 Henri IV (1re partie)\n",
      "6996 Romeo und Julia\n",
      "47960 Shakespeare's Tragedy of Romeo and Juliet\n",
      "19201 Cymbeline: Tragédie\n",
      "21856 Le roi Jean\n",
      "7022 Ein Sommernachtstraum\n",
      "49007 The Works of William Shakespeare [Cambridge Edition] [Vol. 6 of 9]\n",
      "49008 The Works of William Shakespeare [Cambridge Edition] [Vol. 8 of 9]\n",
      "10606 The Tragedie of Hamlet, Prince of Denmark\n",
      "A Study with the Text of the Folio of 1623\n",
      "15846 Beaucoup de Bruit pour Rien\n",
      "15847 Jules César\n",
      "9077 The Tragicall Historie of Hamlet, Prince of Denmarke\r\n",
      "The First ('Bad') Quarto\n",
      "15848 La Comédie des Méprises\n",
      "42873 Kuningas Henrik Kahdeksas\n",
      "15849 Timon d'Athènes\n",
      "50559 The Works of William Shakespeare - Cambridge Edition\n",
      "(3 of 9) (1863)\n",
      "23935 Sonnet 23\n",
      "7041 Wie es Euch gefällt\n",
      "7043 Der Kaufmann von Venedig\n",
      "18311 Le conte d'hiver\n",
      "18312 Le roi Lear\n",
      "18313 Troïlus et Cressida\n",
      "27536 El Marxant de Venecia\n",
      "42898 Paljo melua tyhjästä\n",
      "31126 Een Midzomernachtdroom\n",
      "1430 Beautiful Stories from Shakespeare\n",
      "47518 Shakespeare's Comedy of The Tempest\n",
      "37279 Hamleto, Reĝido de Danujo\n",
      "8609 A Midsummer Night's Dream\n",
      "23970 Sonnets\n",
      "1136 King Henry the Eighth\n",
      "12719 Sonnet # 29\n",
      "12720 Sonnet #40\n",
      "12721 Sonnet #55\n",
      "12722 Sonnet #100\n",
      "12723 Sonnet #106\n",
      "12724 Sonnet #116\n",
      "50095 The Works of William Shakespeare [Cambridge Edition] [Vol. 4 of 9]\n",
      "27062 Romeo i Julia\n",
      "Tragedya w 5 Aktach\n",
      "22791 King Henry the Fifth\n",
      "Arranged for Representation at the Princess's Theatre\n",
      "51138 De Koopman van Venetië\n",
      "15303 Coriolan\n",
      "37835 Kuningas Richard Kolmas\n",
      "23046 The Comedy of Errors\n",
      "The Works of William Shakespeare [Cambridge Edition] [9 vols.]\n",
      "42966 Kuinka äkäpussi kesytetään\n",
      "53207 Dramas\n",
      "El Mercader de Venecia, Macbeth, Romeo y Julieta, Otelo\n",
      "23042 The Tempest\n",
      "The Works of William Shakespeare [Cambridge Edition] [9 vols.]\n",
      "1500 King Henry VI, First Part\n",
      "1501 History of King Henry the Sixth, Second Part\n",
      "1502 The History of King Henry the Sixth, Third Part\n",
      "1503 The Tragedy of King Richard III\n",
      "1504 The Comedy of Errors\n",
      "1505 The Rape of Lucrece\n",
      "26594 Twee Edellieden van Verona\n",
      "1506 The Rape of Lucrece\n",
      "1507 The Tragedy of Titus Andronicus\n",
      "1508 The Taming of the Shrew\n",
      "1509 The Two Gentlemen of Verona\n",
      "1510 Love's Labour's Lost\n",
      "1511 King John\n",
      "1512 The Tragedy of King Richard the Second\n",
      "1513 Romeo and Juliet\n",
      "51691 De getemde feeks\n",
      "1514 A Midsummer Night's Dream\n",
      "1515 The Merchant of Venice\n",
      "1516 King Henry IV, the First Part\n",
      "1517 The Merry Wives of Windsor\n",
      "1518 King Henry IV, Second Part\n",
      "1519 Much Ado about Nothing\n",
      "1520 Much Ado about Nothing\n",
      "1521 The Life of King Henry V\n",
      "1522 Julius Caesar\n",
      "38901 Twelfth Night; or, What You Will\n",
      "28150 Le Pèlerin amoureux\n",
      "28151 Tout est bien qui finit bien\n",
      "1523 As You Like It\n",
      "1524 Hamlet, Prince of Denmark\n",
      "49146 The Shakespeare Story-Book\n",
      "1525 The Phoenix and the Turtle\n",
      "1526 Twelfth Night; Or, What You Will\n",
      "1533 Macbeth\n",
      "1534 Antony and Cleopatra\n",
      "1535 The Tragedy of Coriolanus\n"
     ]
    }
   ],
   "source": [
    "if GUTENBERG:\n",
    "    for text_id in get_etexts('author', 'Shakespeare, William'):\n",
    "        print(text_id, list(get_metadata('title', text_id))[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "5225350"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "if GUTENBERG:\n",
    "    shakespeare = strip_headers(load_etext(100))\n",
    "else:\n",
    "    path = get_file('shakespeare', 'https://storage.googleapis.com/deep-learning-cookbook/100-0.txt')\n",
    "    shakespeare = open(path).read()\n",
    "training_text = shakespeare.split('\\nTHE END', 1)[-1]\n",
    "len(training_text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "83"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "chars = list(sorted(set(training_text)))\n",
    "char_to_idx = {ch: idx for idx, ch in enumerate(chars)}\n",
    "len(chars)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def char_rnn_model(num_chars, num_layers, num_nodes=512, dropout=0.1):\n",
    "    input = Input(shape=(None, num_chars), name='input')\n",
    "    prev = input\n",
    "    for i in range(num_layers):\n",
    "        lstm = LSTM(num_nodes, return_sequences=True, name='lstm_layer_%d' % (i + 1))(prev)\n",
    "        if dropout:\n",
    "            prev = Dropout(dropout)(lstm)\n",
    "        else:\n",
    "            prev = lstm\n",
    "    dense = TimeDistributed(Dense(num_chars, name='dense', activation='softmax'))(prev)\n",
    "    model = Model(inputs=[input], outputs=[dense])\n",
    "    optimizer = RMSprop(lr=0.01)\n",
    "    model.compile(loss='categorical_crossentropy', optimizer=optimizer, metrics=['accuracy'])\n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input (InputLayer)           (None, None, 83)          0         \n",
      "_________________________________________________________________\n",
      "lstm_layer_1 (LSTM)          (None, None, 640)         1853440   \n",
      "_________________________________________________________________\n",
      "lstm_layer_2 (LSTM)          (None, None, 640)         3279360   \n",
      "_________________________________________________________________\n",
      "time_distributed_1 (TimeDist (None, None, 83)          53203     \n",
      "=================================================================\n",
      "Total params: 5,186,003\n",
      "Trainable params: 5,186,003\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "model = char_rnn_model(len(chars), num_layers=2, num_nodes=640, dropout=0)\n",
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(array([[[ 0.,  0.,  0., ...,  0.,  0.,  0.],\n",
       "         [ 0.,  0.,  0., ...,  0.,  0.,  0.],\n",
       "         [ 0.,  0.,  0., ...,  0.,  0.,  0.],\n",
       "         ..., \n",
       "         [ 0.,  1.,  0., ...,  0.,  0.,  0.],\n",
       "         [ 0.,  0.,  0., ...,  0.,  0.,  0.],\n",
       "         [ 0.,  0.,  0., ...,  0.,  0.,  0.]],\n",
       " \n",
       "        [[ 0.,  0.,  0., ...,  0.,  0.,  0.],\n",
       "         [ 0.,  0.,  0., ...,  0.,  0.,  0.],\n",
       "         [ 0.,  1.,  0., ...,  0.,  0.,  0.],\n",
       "         ..., \n",
       "         [ 0.,  0.,  0., ...,  0.,  0.,  0.],\n",
       "         [ 0.,  0.,  0., ...,  0.,  0.,  0.],\n",
       "         [ 0.,  0.,  0., ...,  0.,  0.,  0.]],\n",
       " \n",
       "        [[ 0.,  0.,  0., ...,  0.,  0.,  0.],\n",
       "         [ 0.,  0.,  0., ...,  0.,  0.,  0.],\n",
       "         [ 0.,  0.,  0., ...,  0.,  0.,  0.],\n",
       "         ..., \n",
       "         [ 0.,  0.,  0., ...,  0.,  0.,  0.],\n",
       "         [ 0.,  0.,  0., ...,  0.,  0.,  0.],\n",
       "         [ 0.,  1.,  0., ...,  0.,  0.,  0.]],\n",
       " \n",
       "        [[ 0.,  0.,  0., ...,  0.,  0.,  0.],\n",
       "         [ 0.,  0.,  0., ...,  0.,  0.,  0.],\n",
       "         [ 1.,  0.,  0., ...,  0.,  0.,  0.],\n",
       "         ..., \n",
       "         [ 0.,  0.,  0., ...,  0.,  0.,  0.],\n",
       "         [ 0.,  0.,  0., ...,  0.,  0.,  0.],\n",
       "         [ 0.,  1.,  0., ...,  0.,  0.,  0.]]]),\n",
       " array([[[ 0.,  0.,  0., ...,  0.,  0.,  0.],\n",
       "         [ 0.,  0.,  0., ...,  0.,  0.,  0.],\n",
       "         [ 0.,  0.,  0., ...,  0.,  0.,  0.],\n",
       "         ..., \n",
       "         [ 0.,  0.,  0., ...,  0.,  0.,  0.],\n",
       "         [ 0.,  0.,  0., ...,  0.,  0.,  0.],\n",
       "         [ 0.,  1.,  0., ...,  0.,  0.,  0.]],\n",
       " \n",
       "        [[ 0.,  0.,  0., ...,  0.,  0.,  0.],\n",
       "         [ 0.,  1.,  0., ...,  0.,  0.,  0.],\n",
       "         [ 0.,  0.,  0., ...,  0.,  0.,  0.],\n",
       "         ..., \n",
       "         [ 0.,  0.,  0., ...,  0.,  0.,  0.],\n",
       "         [ 0.,  0.,  0., ...,  0.,  0.,  0.],\n",
       "         [ 0.,  1.,  0., ...,  0.,  0.,  0.]],\n",
       " \n",
       "        [[ 0.,  0.,  0., ...,  0.,  0.,  0.],\n",
       "         [ 0.,  0.,  0., ...,  0.,  0.,  0.],\n",
       "         [ 0.,  1.,  0., ...,  0.,  0.,  0.],\n",
       "         ..., \n",
       "         [ 0.,  0.,  0., ...,  0.,  0.,  0.],\n",
       "         [ 0.,  1.,  0., ...,  0.,  0.,  0.],\n",
       "         [ 0.,  0.,  0., ...,  0.,  0.,  0.]],\n",
       " \n",
       "        [[ 0.,  0.,  0., ...,  0.,  0.,  0.],\n",
       "         [ 1.,  0.,  0., ...,  0.,  0.,  0.],\n",
       "         [ 0.,  1.,  0., ...,  0.,  0.,  0.],\n",
       "         ..., \n",
       "         [ 0.,  0.,  0., ...,  0.,  0.,  0.],\n",
       "         [ 0.,  1.,  0., ...,  0.,  0.,  0.],\n",
       "         [ 0.,  0.,  0., ...,  0.,  0.,  0.]]]))"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "CHUNK_SIZE = 160\n",
    "\n",
    "def data_generator(all_text, char_to_idx, batch_size, chunk_size):\n",
    "    X = np.zeros((batch_size, chunk_size, len(char_to_idx)))\n",
    "    y = np.zeros((batch_size, chunk_size, len(char_to_idx)))\n",
    "    while True:\n",
    "        for row in range(batch_size):\n",
    "            idx = random.randrange(len(all_text) - chunk_size - 1)\n",
    "            chunk = np.zeros((chunk_size + 1, len(char_to_idx)))\n",
    "            for i in range(chunk_size + 1):\n",
    "                chunk[i, char_to_idx[all_text[idx + i]]] = 1\n",
    "            X[row, :, :] = chunk[:chunk_size]\n",
    "            y[row, :, :] = chunk[1:]\n",
    "        yield X, y\n",
    "\n",
    "next(data_generator(training_text, char_to_idx, 4, chunk_size=CHUNK_SIZE))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/40\n",
      "100s - loss: 3.2858 - acc: 0.2282\n",
      "Epoch 2/40\n",
      "100s - loss: 3.2915 - acc: 0.2181\n",
      "Epoch 3/40\n",
      "102s - loss: 1.9426 - acc: 0.4542\n",
      "Epoch 4/40\n",
      "102s - loss: 1.2815 - acc: 0.6139\n",
      "Epoch 5/40\n",
      "102s - loss: 1.2152 - acc: 0.6357\n",
      "Epoch 6/40\n",
      "102s - loss: 1.1496 - acc: 0.6505\n",
      "Epoch 7/40\n",
      "102s - loss: 1.1289 - acc: 0.6582\n",
      "Epoch 8/40\n",
      "102s - loss: 1.0944 - acc: 0.6662\n",
      "Epoch 9/40\n",
      "102s - loss: 1.0725 - acc: 0.6718\n",
      "Epoch 10/40\n",
      "102s - loss: 1.0551 - acc: 0.6773\n",
      "Epoch 11/40\n",
      "102s - loss: 1.0478 - acc: 0.6802\n",
      "Epoch 12/40\n",
      "102s - loss: 1.0200 - acc: 0.6855\n",
      "Epoch 13/40\n",
      "102s - loss: 1.0303 - acc: 0.6869\n",
      "Epoch 14/40\n",
      "102s - loss: 0.9999 - acc: 0.6917\n",
      "Epoch 15/40\n",
      "102s - loss: 0.9895 - acc: 0.6944\n",
      "Epoch 16/40\n",
      "102s - loss: 1.0104 - acc: 0.6943\n",
      "Epoch 17/40\n",
      "102s - loss: 0.9822 - acc: 0.6981\n",
      "Epoch 18/40\n",
      "102s - loss: 0.9703 - acc: 0.7011\n",
      "Epoch 19/40\n",
      "102s - loss: 0.9814 - acc: 0.7019\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0x7fe16672f860>"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "early = keras.callbacks.EarlyStopping(monitor='loss',\n",
    "                              min_delta=0.03,\n",
    "                              patience=3,\n",
    "                              verbose=0, mode='auto')\n",
    "\n",
    "BATCH_SIZE = 256\n",
    "model.fit_generator(\n",
    "    data_generator(training_text, char_to_idx, batch_size=BATCH_SIZE, chunk_size=CHUNK_SIZE),\n",
    "    epochs=40,\n",
    "    callbacks=[early,],\n",
    "    steps_per_epoch=2 * len(training_text) / (BATCH_SIZE * CHUNK_SIZE),\n",
    "    verbose=2\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('zoo/06/shakespeare.json', 'w') as fout:\n",
    "    json.dump({\n",
    "        'chars': ''.join(chars),\n",
    "        'char_to_idx': char_to_idx,\n",
    "        'chunk_size': CHUNK_SIZE,\n",
    "    }, fout)\n",
    "model.save('zoo/06/shakespeare.h5')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " heaven is hid,\n",
      "    Behind the globe, that lights the lower world,\n",
      "    Then thieves and robbers range abroad unseen\n",
      "    In murders and in outrage boldly here;\n",
      " #   And then the world is still the world,\n",
      "    The woman's that was still and the worst\n",
      "    That they were strange and stol'n the fiery steeds,\n",
      "    The seasons of the world, and then they were,\n",
      "    And then they would have made them dead,\n",
      "    And then they have deserv'd the charges,\n",
      "    The state of the world, when they are stronger,\n",
      "    The strongest of the common wreck,\n",
      "    The which in the world\n"
     ]
    }
   ],
   "source": [
    "def generate_output(model, training_text, start_index=None, diversity=None, amount=400):\n",
    "    if start_index is None:\n",
    "        start_index = random.randint(0, len(training_text) - CHUNK_SIZE - 1)\n",
    "    generated = training_text[start_index: start_index + CHUNK_SIZE]\n",
    "    yield generated + '#'\n",
    "    for i in range(amount):\n",
    "        x = np.zeros((1, len(generated), len(chars)))\n",
    "        for t, char in enumerate(generated):\n",
    "            x[0, t, char_to_idx[char]] = 1.\n",
    "        preds = model.predict(x, verbose=0)[0]\n",
    "        if diversity is None:\n",
    "            next_index = np.argmax(preds[len(generated) - 1])\n",
    "        else:\n",
    "            preds = np.asarray(preds[len(generated) - 1]).astype('float64')\n",
    "            preds = np.log(preds) / diversity\n",
    "            exp_preds = np.exp(preds)\n",
    "            preds = exp_preds / np.sum(exp_preds)\n",
    "            probas = np.random.multinomial(1, preds, 1)\n",
    "            next_index = np.argmax(probas)     \n",
    "        next_char = chars[next_index]\n",
    "        yield next_char\n",
    "\n",
    "        generated += next_char\n",
    "    return generated\n",
    "\n",
    "for ch in generate_output(model, training_text):\n",
    "    sys.stdout.write(ch)\n",
    "print()\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1504"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def find_python(rootdir):\n",
    "    matches = []\n",
    "    for root, dirnames, filenames in os.walk(rootdir):\n",
    "        for fn in filenames:\n",
    "            if fn.endswith('.py'):\n",
    "                matches.append(os.path.join(root, fn))\n",
    "\n",
    "    return matches\n",
    "#  + find_python(os.path.join(sys.executable.rsplit('/', 2)[0], 'lib'))\n",
    "srcs = find_python(random.__file__.rsplit('/', 1)[0])\n",
    "len(srcs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'this = \"\"\\n'"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def replacer(value):\n",
    "    value = ''.join(ch for ch in value if ord(ch) < 127)\n",
    "    if not ' ' in value:\n",
    "        return value\n",
    "    if sum(1 for ch in value if ch.isalpha()) > 6:\n",
    "        return 'MSG'\n",
    "    return value\n",
    "\n",
    "\n",
    "def replace_literals(st):\n",
    "    res = []\n",
    "    start_text = start_quote = i = 0\n",
    "    quote = ''\n",
    "    while i < len(st):\n",
    "        if quote:\n",
    "            if st[i: i + len(quote)] == quote:\n",
    "                quote = ''\n",
    "                start_text = i\n",
    "                res.append(replacer(st[start_quote: i]))\n",
    "        elif st[i] in '\"\\'':\n",
    "            quote = st[i]\n",
    "            if i < len(st) - 2 and st[i + 1] == st[i + 2] == quote:\n",
    "                quote = 3 * quote\n",
    "            start_quote = i + len(quote)\n",
    "            res.append(st[start_text: start_quote])\n",
    "        if st[i] == '\\n' and len(quote) == 1:\n",
    "            start_text = i\n",
    "            res.append(quote)\n",
    "            quote = ''\n",
    "        if st[i] == '\\\\':\n",
    "            i += 1\n",
    "        i += 1\n",
    "    return ''.join(res) + st[start_text:]\n",
    "\n",
    "#replace_literals('print(\"hel\\\\\"lo\")') + replace_literals(\"print('hel\\\\'lo world')\")\n",
    "replace_literals('this = \"wrong\\n')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Could not read /usr/local/Cellar/python3/3.5.2/Frameworks/Python.framework/Versions/3.5/lib/python3.5/sqlite3/test/dbapi.py\n",
      "Could not read /usr/local/Cellar/python3/3.5.2/Frameworks/Python.framework/Versions/3.5/lib/python3.5/sqlite3/test/factory.py\n",
      "Could not read /usr/local/Cellar/python3/3.5.2/Frameworks/Python.framework/Versions/3.5/lib/python3.5/sqlite3/test/hooks.py\n",
      "Could not read /usr/local/Cellar/python3/3.5.2/Frameworks/Python.framework/Versions/3.5/lib/python3.5/sqlite3/test/regression.py\n",
      "Could not read /usr/local/Cellar/python3/3.5.2/Frameworks/Python.framework/Versions/3.5/lib/python3.5/sqlite3/test/transactions.py\n",
      "Could not read /usr/local/Cellar/python3/3.5.2/Frameworks/Python.framework/Versions/3.5/lib/python3.5/sqlite3/test/types.py\n",
      "Could not read /usr/local/Cellar/python3/3.5.2/Frameworks/Python.framework/Versions/3.5/lib/python3.5/sqlite3/test/userfunctions.py\n",
      "Could not read /usr/local/Cellar/python3/3.5.2/Frameworks/Python.framework/Versions/3.5/lib/python3.5/test/badsyntax_pep3120.py\n",
      "Could not read /usr/local/Cellar/python3/3.5.2/Frameworks/Python.framework/Versions/3.5/lib/python3.5/test/test_source_encoding.py\n",
      "Could not read /usr/local/Cellar/python3/3.5.2/Frameworks/Python.framework/Versions/3.5/lib/python3.5/test/encoded_modules/module_iso_8859_1.py\n",
      "Could not read /usr/local/Cellar/python3/3.5.2/Frameworks/Python.framework/Versions/3.5/lib/python3.5/test/encoded_modules/module_koi8_r.py\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "16775961"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "COMMENT_RE = re.compile('#.*')\n",
    "python_code = []\n",
    "for fn in srcs:\n",
    "    try:\n",
    "        with open(fn, 'r') as fin:\n",
    "            src = fin.read()\n",
    "    except UnicodeDecodeError:\n",
    "        print('Could not read %s' % fn)\n",
    "    src = replace_literals(src)\n",
    "    src = COMMENT_RE.sub('', src)\n",
    "    python_code.append(src)\n",
    "\n",
    "python_code = '\\n\\n\\n'.join(python_code)\n",
    "len(python_code)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "111"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "py_chars = list(sorted(set(python_code)))\n",
    "py_char_to_idx = {ch: idx for idx, ch in enumerate(py_chars)}\n",
    "len(py_chars)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input (InputLayer)           (None, None, 111)         0         \n",
      "_________________________________________________________________\n",
      "lstm_layer_1 (LSTM)          (None, None, 640)         1925120   \n",
      "_________________________________________________________________\n",
      "lstm_layer_2 (LSTM)          (None, None, 640)         3279360   \n",
      "_________________________________________________________________\n",
      "time_distributed_1 (TimeDist (None, None, 111)         71151     \n",
      "=================================================================\n",
      "Total params: 5,275,631\n",
      "Trainable params: 5,275,631\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "py_model = char_rnn_model(len(py_chars), num_layers=2, num_nodes=640, dropout=0)\n",
    "py_model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/40\n"
     ]
    }
   ],
   "source": [
    "early = keras.callbacks.EarlyStopping(monitor='loss',\n",
    "                              min_delta=0.03,\n",
    "                              patience=3,\n",
    "                              verbose=0, mode='auto')\n",
    "\n",
    "BATCH_SIZE = 256\n",
    "py_model.fit_generator(\n",
    "    data_generator(python_code, py_char_to_idx, batch_size=BATCH_SIZE, chunk_size=160),\n",
    "    epochs=40,\n",
    "    callbacks=[early,],\n",
    "    steps_per_epoch=2 * len(python_code) / (BATCH_SIZE * 160),\n",
    "    verbose=2\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "def set_surrounding_binput(name):\n",
      "    \"\"\"MSG\"\"\"\n",
      "    \n",
      "    \n",
      "    pat = path.join(supersion, name)\n",
      "    if not result:\n",
      "        return path\n",
      "\n",
      "\n",
      "\n",
      "def _calculate_ratio(val):\n",
      "    \"\"\"MSG\"\"\"\n",
      "    if value and value[0] != '0':\n",
      "        raise errors.HeaderParseError(\n",
      "            \"MSG\".format(Storable))\n",
      "    return value\n",
      "\n",
      "\n",
      "\n",
      "def _get_uid(filename):\n",
      "    \"\"\"MSG\"\"\"\n",
      "    \n",
      "    filename = getattr(file, \"func\", None)\n",
      "    if isinstance(filename, bytes):\n",
      "        return file, finfo\n",
      "    return objects\n",
      "\n",
      "\n",
      "\n",
      "def isfirstline(exc_info):\n",
      "    \"\"\"MSG\"\"\"\n",
      "    \n",
      "    \n",
      "    if 'N' in os.environ:\n",
      "            return 'xba'\n",
      "        key, val = os.environ['CC']\n",
      "    elif context is None:\n",
      "        content_type = token.COMMA + self.childNevent\n",
      "        self.type, self._default = default_value\n",
      "        self.names = parser.Get(\n",
      "                type(self).__new__,\n",
      "    _type_var, __main__.__dict__ = metacles and self._args_magic_methods(self.get_saved(): final_or(yey, message))\n",
      "\n",
      "\n",
      "\n",
      "def is_nan(o):\n",
      "    \"\"\"MSG\"\"\"\n",
      "    word = \"*\"\n",
      "    if not p.startswith(w):\n",
      "        y = ps.remove(w)\n",
      "    lines = data.rfind(\" \", n)\n",
      "    return data\n",
      "\n",
      "\n",
      "\n",
      "def getregentry():\n",
      "    return codecs.CodecInfo(\n",
      "        name='cphacht-and-encoding',\n",
      "        encode=Codec().encode,\n",
      "        decode=Codec().decode,\n",
      "        incrementalencoder=IncrementalEncoder,\n",
      "        incrementaldecoder=IncrementalDecoder,\n",
      "        streamwriter=StreamWriter,\n",
      "        stream_reader=StreamReader,\n",
      "        streamwriter=StreamWriter,\n",
      "    )\n",
      "\n",
      "\n",
      "\n",
      "def format_info(encoding, errors):\n",
      "    \"\"\"MSG\"\"\"\n",
      "    format_exception(etype, evalue, etb))\n",
      "\n",
      "\n",
      "\n",
      "def notdone(parts):\n",
      "    \"\"\"MSG\"\"\"\n",
      "    if _normalize():\n",
      "        \n",
      "        partialme = value\n",
      "    elif next(head) or h[:3] != \"5\":\n",
      "        parent = norm_encoding.decode(wsgi_fram_stream, errors,\n",
      "                              \"ProxyEntity\",\n",
      "                      PendingDeprecationWarning, 2, save2)\n",
      "        if p.lower() in _get_all_params( == 0):\n",
      "        \n",
      "        return \"\"\n",
      "        day, mon, yr, pr, tio, tmarkjig.split()\n",
      "        monthname = month\n",
      "        if dst is None:\n",
      "            yield \"while\"\n",
      "        return mode | self._data\n",
      "\n",
      "\n",
      "\n",
      "def _raise_error(InvalidOperation, 'source_traceback'):\n",
      "    assert captured is no  \n",
      "\n",
      "\n",
      "\n",
      "def __getattribute__(obj, verbosity):\n",
      "    assert isinstance(v, 'types'))\n",
      "    obj.methods = (\n",
      "        (options['__repr__'] +  value)\n",
      "    ofp.close()\n",
      "    names.sizeof(old, new)\n",
      "    return new_scheme, netloc\n",
      "\n",
      "\n",
      "\n",
      "def parsedate(d):\n",
      "    return unexpected_exception(*args)\n",
      "\n",
      "\n",
      "\n",
      "def iskeys(val):\n",
      "    \"Co\"\n",
      "\n",
      "\n",
      "\n",
      "def transform(source, startating_fd):\n",
      "        getter, ext, extra_postargs = get_comment_line()\n",
      "        while True:\n",
      "            \n",
      "            for child in node.childNodes:\n",
      "                for diffs in dnumerate(zdrase1):\n",
      "                    nodelen = node\n",
      "                    while initialname in list:\n",
      "                        node.intenum_charset(obj, infile + next_in)\n",
      "                    else:\n",
      "                        node = node.parent\n",
      "                        node.replace(node.ownerDocument)\n",
      "                        node, results, n >= 2\n",
      "                    continue\n",
      "                elem = m.starimport()\n",
      "                if node.tagName in markers:\n",
      "                    total = len(list)\n",
      "                        if _node is not None:\n",
      "                            new_node.expand(m, self.pop())\n",
      "            else:\n",
      "                node = dpot(monum)\n",
      "                cookie.name  = newfile\n",
      "                \n",
      "                \n",
      "                try:\n",
      "                    obit_RequestHandler.__init__(self, pos, unlock)\n",
      "                    continue\n",
      "                finally:\n",
      "                    info.func = s\n",
      "                except self.__union_get_source:\n",
      "                    funcopy.select_set(self.filename)\n",
      "                    t = time.time() + self.TMP()\n",
      "                    try:\n",
      "                        self._interning = self._toc of self._mock_siDe \n",
      "         .find(triplet if task is _ASION and\n",
      "                    self.instance in self.tkconsole.handler, 'message'):\n",
      "                    self.tkconsole.start_ = 1\n",
      "                else:\n",
      "                    return\n",
      "        self.socket.close()\n",
      "        self.sock * self.socket.fileno()\n",
      "\n",
      "\n",
      "\n",
      "def dget_objectid(object):\n",
      "    \"\"\"MSG\"\"\"\n",
      "    \n",
      "    if type(obj) is subdir:\n",
      "        if \"k\" in d:\n",
      "            default[:3] = datetime)\n",
      "            dispatch[dict] = dict\n",
      "            if symbol2num is None:\n",
      "                return\n",
      "\n",
      "\n",
      "\n",
      "def main():\n",
      "    import  types, defaults, namespace, sig, signal.splitdrive,\n",
      "                                _mapstart.geometry_incrementating_func(*args))\n",
      "            except Exception as e:\n",
      "                endprog.homa = sys.stderr.get(\"%s.__doc__\"\"\" % lo,     \n",
      "                                        f.write(del_expr._indent(FALSE)))\n",
      "            except OSError:\n",
      "                errwrite = f.read()\n",
      "            except OSError:\n",
      "                pass\n",
      "    class ParserGresult(TreeItem, FileCookieJation):\n",
      "    \n",
      "    pass\n",
      "\n",
      "\n",
      "\n",
      "def new_attrs():\n",
      "    attr = Attr(Name(key, None)))\n",
      "    if not hasattr(pattern, '__doc__)theaping'):\n",
      "        return p[:2] == curses.i\n",
      "in repeater_node.replace(\"\"\", \"\")\n",
      "    q = pwd.getpwuid(os.sep)\n",
      "    \n",
      "    \n",
      "    n = divmod"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/douwe/proj/notebooks/venv3/lib/python3.5/site-packages/ipykernel_launcher.py:11: RuntimeWarning: divide by zero encountered in log\n",
      "  # This is added back by InteractiveShellApp.init_path()\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(n, n80)\n",
      "    q += 4\n",
      "    year = i+1   \n",
      "    \n",
      "    \n",
      "    endres[p] = 1\n",
      "    i = 2. \n",
      "    whence = 2 * c\n",
      "    assert 1 <= magic <= 200\n",
      "    if len(match.end() > 1:\n",
      "        print(f.active())\n",
      "        firstline = finder.find(ladespec, linespace, linestarts, strict=name)\n",
      "        if cache[name] == len(spec):\n",
      "            log.warn(\"MSG\",\n",
      "                  parent=spec)\n",
      "    extractargmsg = \"MSG\"\n",
      "    return units_fd, etabs_pattern(s)\n",
      "\n",
      "\n",
      "\n",
      "def Queue():\n",
      "    \"\"\"MSG\"\"\"\n",
      "    return sum(thing, NamespaceLoader, _AsyncDeprecawatingFileHandler)\n",
      "\n",
      "\n",
      "\n",
      "def version(deltas[func]):\n",
      "    return UUID(dicts.pop(int.defaultdict(value)))\n",
      "\n",
      "\n",
      "\n",
      "def decode_long(old, new, allow_dot):\n",
      "    yearef = deepcopy(parent)\n",
      "    def display_name(self, newpath):\n",
      "        \"MSG\"\n",
      "        return normp.fp\n",
      "        return (not self.encoder or encoding)\n",
      "\n",
      "\n",
      "\n",
      "def quopri_default_https_config():\n",
      "    global environment_ew = SubprocessClass\n",
      "    if fromdir:\n",
      "        print(foll(format, method)\n",
      "    else:\n",
      "        try:\n",
      "            key = fp.readline().split(',')\n",
      "        except ValueError:\n",
      "                continue\n",
      "            try:\n",
      "             headers['Content-type'] = ''.join(msg))\n",
      "            upperdirs.append('/usr/local/bin')\n",
      "            sys.exit(1)\n",
      "        elif file[:2] == '/':\n",
      "            head_parts(host, dsplit())\n",
      "        tkMessageBox.showerror(\"MSG\",\n",
      "                        \"MSG\",\n",
      "                    message='MSG', parent='', methodName='')\n",
      "                for element in table:\n",
      "                    message.framework(')', manifed_temp)\n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "def generate_code(model, start_with='\\ndef ', end_with='\\n\\n', diversity=1.0):\n",
    "    generated = start_with\n",
    "    yield generated\n",
    "    for i in range(2000):\n",
    "        x = np.zeros((1, len(generated), len(py_chars)))\n",
    "        for t, char in enumerate(generated):\n",
    "            x[0, t, py_char_to_idx[char]] = 1.\n",
    "        preds = model.predict(x, verbose=0)[0]\n",
    "        \n",
    "        preds = np.asarray(preds[len(generated) - 1]).astype('float64')\n",
    "        preds = np.log(preds) / diversity\n",
    "        exp_preds = np.exp(preds)\n",
    "        preds = exp_preds / np.sum(exp_preds)\n",
    "        probas = np.random.multinomial(1, preds, 1)\n",
    "        next_index = np.argmax(probas)        \n",
    "        next_char = py_chars[next_index]\n",
    "        yield next_char\n",
    "\n",
    "        generated += next_char\n",
    "        if generated.endswith(end_with):\n",
    "            break\n",
    "\n",
    "for i in range(20):\n",
    "    for ch in generate_code(py_model):\n",
    "        sys.stdout.write(ch)\n",
    "        st += ch\n",
    "    print()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/40\n",
      "46s - loss: 3.3759 - acc: 0.3166\n",
      "Epoch 2/40\n",
      "46s - loss: 3.2798 - acc: 0.3217\n",
      "Epoch 3/40\n",
      "47s - loss: 3.2740 - acc: 0.3218\n",
      "Epoch 4/40\n",
      "47s - loss: 3.1735 - acc: 0.3228\n",
      "Epoch 5/40\n",
      "47s - loss: 2.6554 - acc: 0.3680\n",
      "Epoch 6/40\n",
      "47s - loss: 2.4541 - acc: 0.4057\n",
      "Epoch 7/40\n",
      "47s - loss: 2.4880 - acc: 0.4077\n",
      "Epoch 8/40\n",
      "47s - loss: 2.5079 - acc: 0.4135\n",
      "Epoch 9/40\n",
      "47s - loss: 2.1347 - acc: 0.4897\n",
      "Epoch 10/40\n",
      "47s - loss: 1.3656 - acc: 0.6583\n",
      "Epoch 11/40\n",
      "47s - loss: 1.0606 - acc: 0.7310\n",
      "Epoch 12/40\n",
      "47s - loss: 0.9297 - acc: 0.7616\n",
      "Epoch 13/40\n",
      "47s - loss: 0.8886 - acc: 0.7762\n",
      "Epoch 14/40\n",
      "47s - loss: 0.8443 - acc: 0.7875\n",
      "Epoch 15/40\n",
      "47s - loss: 0.7958 - acc: 0.7970\n",
      "Epoch 16/40\n",
      "47s - loss: 0.8021 - acc: 0.8000\n",
      "Epoch 17/40\n",
      "47s - loss: 0.7856 - acc: 0.8039\n",
      "Epoch 18/40\n",
      "47s - loss: 0.7752 - acc: 0.8072\n",
      "Epoch 19/40\n",
      "47s - loss: 0.7776 - acc: 0.8085\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0x7f10b9878978>"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "BATCH_SIZE = 512\n",
    "\n",
    "flat_model = char_rnn_model(len(py_chars), num_layers=1, num_nodes=512, dropout=0)\n",
    "\n",
    "early = keras.callbacks.EarlyStopping(monitor='loss',\n",
    "                              min_delta=0.03,\n",
    "                              patience=3,\n",
    "                              verbose=0, mode='auto')\n",
    "\n",
    "flat_model.fit_generator(\n",
    "    data_generator(python_code, py_char_to_idx, batch_size=BATCH_SIZE, chunk_size=160),\n",
    "    epochs=40,\n",
    "    callbacks=[early,],\n",
    "    steps_per_epoch=2 * len(python_code) / (BATCH_SIZE * 160),\n",
    "    verbose=2\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "example_code = 'if a == 2:\\n    b=1\\nelse:\\n    b=2\\n'\n",
    "#example_code = 'a=(2 * 3)\\nb=(4 * 6 + 7)\\nreturn C'\n",
    "\n",
    "def activations(model, code):\n",
    "    x = np.zeros((1, len(code), len(py_char_to_idx)))\n",
    "    for t, char in enumerate(code):\n",
    "        x[0, t, py_char_to_idx[char]] = 1.\n",
    "    output = model.get_layer('lstm_layer_1').output\n",
    "    f = K.function([model.input], [output])\n",
    "    return f([x])[0][0]\n",
    "\n",
    "act = activations(flat_model, example_code)\n",
    "act.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "22"
      ]
     },
     "execution_count": 83,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def interesting_neurons(act):\n",
    "    res = []\n",
    "    for n in np.argmax(act, axis=1):\n",
    "        if not n in res:\n",
    "            res.append(n)\n",
    "    return res\n",
    "\n",
    "neurons = interesting_neurons(act)\n",
    "len(neurons)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYwAAAEUCAIAAAC3bpYzAAANm0lEQVR4nO3dS4yV9R3G8XNmzjDM\nDPerDIiAIIIi3lojQhETTb2lEhNbm7hoF25sV23TxqRThoUxjbvaLly4aJPamLbaWLS1iQRRjFYr\naOUiCojc7+AwzIWZ6YJqqKY8j5x3jg/w/exMHt73Ped932f+Q/j5L7e3t5cAIFXdV30BAHAmldP/\no62trVQqLV++/Ax/4FRGxs4/F/gHL+pTF3u0M5/llAvtfhWlwDtV5bvzPyup5cuXOw0lY+ef02/Y\n6S8AfG1tbTX76i7ARzRW9e9O5fQDnfL/7u5nmeortpBVyZe65irPxRN/5pvufM8FVnxRq9ov9Qhd\nmM9AUfe9GuXT/+Jctk8hK8DPDlKblX+BavP9pCn2fhX1E865HvMFc47gHIr77hztLP5sRUeKdupj\nV/8TtZYrqZL3LZ9nj6mv9n8HVP3z89nrZ5bdmQ9V5cWco8z7XmXNfQUl9bmGPmvOZy78r3vPy5+Z\n1av9F1LIGU//DbStrW3wfmE5XznfTPXvzn9/3ftiX3zxWJ/LnPWdc86Vxvzs52WFFbVKKvC+y0sq\n8Hn2F+zcd3mcsztUmX/MCSAZ/5gTQDRKCkC0ypvX6tB1bw3+haBU6jN+ZNT3D/51fGrjbJ25fFMx\n59rZqjOTdutM3UD111Jrx5t1pqVz8K/jU/+6RmeufbuYcx0cozOspABEo6QARKOkAESjpABEo6QA\nRKOkAESjpABEo6QARKOkAESjpABEo6QARCsv/7n+X7U8+X19oJ89qjMHxunMu/N0Zs5GnWk+rjPd\njTpTZ8zKHRuhM01dOnMu6jK+w6Hdg38dn9pwuc7M2qwzHcN0ptmYpxvSqzPnol2TdKbVmLUsG+8X\nKykA0SgpANEoKQDRKCkA0SgpANEoKQDRKCkA0SgpANEoKQDRKCkA0SgpANEqx1t06MeP6cyKu3Tm\ne0/qzN9u15lHHtaZ339XZ968Xmfuf0pn/vItnbl6rc5sMva5W7JSZ14wvkPnevaP15kZW3RmzCGd\n+fBSnRm/X2f+eK/OzF+nM5d8pDNP3a8zS5/RmQ9m6cy0rToz29gDcdkynbnhdZ1ZsEZnnH0SVy/S\nGVZSAKJRUgCiUVIAolFSAKJRUgCiUVIAolFSAKJRUgCiUVIAolFSAKJRUgCilR/7kd53z5nvGzDq\n7mVjTue6t3Rm2jadaTT2ettt7B22aLXOrFqsMw7nO3TUnzTOVdaZ/vpiztVX0Rln/z5nvs+ZX3NU\n+oo5jqPfuBfOHn9OxtlP0NlrcsQxnXH22XTmQ1lJAYhGSQGIRkkBiEZJAYhGSQGIRkkBiEZJAYhG\nSQGIRkkBiEZJAYhGSQGIVukYrkMnhupMU5fOfMOYg3vxVp1pMWaUivLqTTozZYfOfDxVZ4qag3My\nRXH2Lrxmrc4MNWYtr1ivM51NOtN8QmdqqWeIztQN6EyX8dkdfcbSZd18nZn9vs6MNfZkZCUFIBol\nBSAaJQUgGiUFIBolBSAaJQUgGiUFIBolBSAaJQUgGiUFIBolBSCaNeXlzOU5Ns/UmRtf05ln7tGZ\npc/qzOqFOuPM5fUas1fdRqZkZA6N0Znx+3XG2VduzY06s8C4X+vn6MzcDTqz4g6dufN5ndkxWWem\n7NSZojhzi73Gm9pgzH46nr1HZ+79s844+2w687yspABEo6QARKOkAESjpABEo6QARKOkAESjpABE\no6QARKOkAESjpABEo6QARCs/9Hi7DDUZ+5QNO64zzsxUQ6/ObDBmwd67Qmce+o3O4Mycebq5xn55\n07dVfSm2i3brzJ5Jg38d57KVN+vMImMuz5khZSUFIBolBSAaJQUgGiUFIBolBSAaJQUgGiUFIBol\nBSAaJQUgGiUFIBolBSBapa9eh5y5vDpjBmfqdp3ZMUVnbl6lM7tadea5u3Tm7r/qzJXv6kx5QGeO\njComc7xFZxz9xrPR26Azk4097HqM41SMfeX2TdAZx5iDOrNgjc68O09nnGvuMfZk7DP25jtp3FNn\nns55xpyZ35PGNbOSAhCNkgIQjZICEI2SAhCNkgIQjZICEI2SAhCNkgIQjZICEI2SAhCNkgIQrTLu\ngA6NPqwz7xgzSre9qDPObFqfUa3OuT68VGf+8G2dGX2bzjic77nf+OzOzJQzS3jH8zpz93M6s3W6\nzhwYpzMjj+rM/HU609KpM5tm68xzd+vM/vE6c9EenXFm5Zznp7FbZ+57WmecGcD1c3XGmbFlJQUg\nGiUFIBolBSAaJQUgGiUFIBolBSAaJQUgGiUFIBolBSAaJQUgGiUFIFrFmfM6MlJnLvlIZ5w9yFqM\nPf7mGfvcfTBTZ1p36cxNr+rMy9/QmY2X68zaa3TGUe7XmXpjbzVnFmzJSp0ZcUxnnPs16ojOHByr\nM87+feONmdauoTrjzMa+eb3OOI4a76kzu/fPr+nMlB06M3GvzmybpjOspABEo6QARKOkAESjpABE\no6QARKOkAESjpABEo6QARKOkAESjpABEo6QARCu3tbfL0Nz1+kAb5+jMPmMPsmZjTzRnJmjSbp15\n6RadmfmBzvQ26MyYQzpz0Nh7rr+sM858ljN39tqNOjNtm87M2aAzHcN0ZudknXHm+5z50O5GnRnS\nqzNHR+jMSGO20VExrqfP2C/vja/rzIwtOjN5p84MGM8zKykA0SgpANEoKQDRKCkA0SgpANEoKQDR\nKCkA0SgpANEoKQDRKCkA0SgpANEqdQM65Mzl9Rl1d9EenXFmeZzZImduaOrHOlNn7GE3tEtnjozS\nmd6Kzjj7yn0yXGecPfWuekdn/v5NnXFMMe5XqzGP6RgwnlVnLs/hzOU585jOfOgx4747+2xevVZn\nnLnOsca8qrNHJCspANEoKQDRKCkA0SgpANEoKQDRKCkA0SgpANEoKQDRKCkA0SgpANEoKQDRKnV9\nOtRv7NW1e5LOXPa+znS26MydK3Tm7Wt15uLtOrNvgs50NelMQ4/OOPNZzsydM5819qDOHB6tMw/8\nVmf+dK/OOHOdJ4zveXiHzuyZWMy5Zm3WGWcfyWMjdWaY8bnKxhyus+dgZ7PO3P6CzmyfqjPHjfed\nlRSAaJQUgGiUFIBolBSAaJQUgGiUFIBolBSAaJQUgGiUFIBolBSAaJQUgGiVjmE6tNeYdbriPZ05\naewr58ymPbNUZ5zZtA3GfoLO9+PMQzlzTAvW6Iwzl/fRJTqzY4rOzN6kM+vm68yi1Trzuwd05oe/\n0pmFr+jMijt1pt7YD+6kMWv5+g06c9/TOuM8z6sW64xz3296VWeev0NnWnfpzNP36QwrKQDRKCkA\n0SgpANEoKQDRKCkA0SgpANEoKQDRKCkA0SgpANEoKQDRKCkA0crLlrXX7GT7xuvMhP2Dfx1fhf6y\nzjhzgvXGPol9xj6Jjd068/HFOjPzQ51xdA/RmX/cqjN3GXsyOvYa+y1O3FfMuZzP3mjs21iUA2N1\nZpwxS1gUVlIAolFSAKJRUgCiUVIAolFSAKJRUgCiUVIAolFSAKJRUgCiUVIAolFSAKJVmjp1qMeY\nLdo9SWeajXONPKIzzr5yo47qTFGmbdWZbdN1ZliHzpxo0pm563XmyCidaejVmU7jepzjbJmhM5e9\nrzNPfUdnbnlJZ4qay3N0N+rM5lk6M3+dztQZ+wmONN4dZ77P2Y/SmQ9lJQUgGiUFIBolBSAaJQUg\nGiUFIBolBSAaJQUgGiUFIBolBSAaJQUgGiUFIFr5lz/R++51ttTgSoAaWfYLI1O77SghsJICEI2S\nAhCNkgIQjZICEI2SAhCNkgIQjZICEI2SAhCNkgIQjZICEI2SAhCt0jFMh5x9wZq6qr+YIp2s15lK\n3+Bfx/muxdgr8LjxjNWSM5f3+EM684NfV38t56o6493pN95B61zFHAYABgclBSAaJQUgGiUFIBol\nBSAaJQUgGiUFIBolBSAaJQUgGiUFIBolBSBapXJSh+oGBv9CiubM5TUa84bdQ3Vm/zidGXNIZ8oF\nfc+1vF+HR+vMkF6d2TdeZybs15miOHN5Ly3RmcWrdKa+X2fSFDWX19OgM6ykAESjpABEo6QARKOk\nAESjpABEo6QARKOkAESjpABEo6QARKOkAESjpABEq5ys6FCfMafjHMfZm6/PqE1n1sk5zifDjXMZ\nM4CjD+tMZ7PODOnRmf6Cfqw497SxW2ecz9VjzIfunagzYw/qzPapOjNlh84cG6Ezjkce1pmFr+jM\nvgk602DMSDqfa8lKnekz3vfOJp15ZaHOsJICEI2SAhCNkgIQjZICEI2SAhCNkgIQjZICEI2SAhCN\nkgIQjZICEI2SAhDNmMAplRqM2Ssn4yhqDzLnOLXc72x4R+3OVUsrjb3nlj6rM/P+XfWllEolbx/A\n6dt0prtRZ24xZtyczLnoiQd15sEndObK93SGlRSAaJQUgGiUFIBolBSAaJQUgGiUFIBolBSAaJQU\ngGiUFIBolBSAaJQUgGjlnz7aLkPvXKUPdMMbBVxNqbh994qyY7LOtO7SmbqB6q+lVCqVehp0pmLM\nUTr795WNa3b2g9s6XWcWvKYzjhNDdWZ3q87M2FL9tRTJeS+c++U8h2Xj/XLu+/q5OrP4ZZ1hJQUg\nGiUFIBolBSAaJQUgGiUFIBolBSAaJQUgGiUFIBolBSAaJQUgGiUFINp/AHwuKZV/UDeuAAAAAElF\nTkSuQmCC\n",
      "text/plain": [
       "<IPython.core.display.Image object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "def visualize_neurons(neurons, code, act, cell_size=12):\n",
    "    img = np.full((len(neurons) + 1, len(code), 3), 128)\n",
    "    scores = (act[:, neurons].T + 1) / 2\n",
    "    img[1:, :, 0] = 255 * (1 - scores)\n",
    "    img[1:, :, 1] = 255 * scores\n",
    "\n",
    "    f = BytesIO()\n",
    "    img = scipy.misc.imresize(img, float(cell_size), interp='nearest')\n",
    "    pil_img = PIL.Image.fromarray(img)\n",
    "    draw = ImageDraw.Draw(pil_img)\n",
    "    for idx, ch in enumerate(code):\n",
    "        draw.text((idx * cell_size + 2, 0), ch)\n",
    "    pil_img.save(f, 'png')\n",
    "    return Image(data=f.getvalue())\n",
    "\n",
    "img = visualize_neurons(neurons, example_code, act)\n",
    "display(img)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAbwAAAEgCAIAAABAdlGwAAAQIUlEQVR4nO3dW4xd5XkG4L1n9tge\nz9ge24Pt2GDOwtgNgeBSKYSQNEkrSC4KTapKPaihaS6qoKqKkqpS5GJHqpSmVdU2qtRDSKvmokoR\n9CJAk7QhkBApxEACNZjz0cY2PnvsGY/HM71AWIYk/r6PWd7MuM9zy8tea+1/r9crJt/62xs3bmwB\nkNPzdp8AwGzyhtLcsGHDhg0bTv0vbHjd9I/dyIfMKJkvZwZedZfX9BSZpk6j+2bjuifNnDNv6v6a\n5hW9oTQ3bdq0adOm8GBhLGPDhg3T/5AZ5cSX0zrlqmzatGnm/Apf09RCZNb0jFz3zILOwHXPmDnr\n1eD9Nc216Jx8Qic+8WdGT2ROPvu3pvSnQVMHCq9rmscq/bvT/y2G389rhwgXq6k7ucE/4Zv9gXVn\n3fOX1oV1b72xF97yERu5T5tai8bvr7e8Fp2TT+jU31HmDmzKyQs/nd9Z5l9s9nK68xUlv58Tvfnz\nMj99g80Es3HduyOz7if+aev0r2zyfMLPya9F9yvop/9R53Qf+605eeGno5tPHK0urmhT38+Zqsvr\nPjO9dlHdubTk/5SZ5lp07f469VFmaGk29edkN5843nTOp3VpZ+YT4sxxpj5pzlin+M03tRbdvL9O\nrefECZ38V5Y/81YMA3nJC37T36LOCslTbWTJp//9vOmvZabzPTfYU2/7XVGSuXdOmOZFnbxeyf/2\nMP3DTfN8mtXg/RX+R++fd6z22/h/bp/+X4rPRmf2VZ/Wv4Cf1WbvVc+uM+/CL/DtLE2AWcdEEEBB\n58HL49C7Hz7t50Gr1TrSH2fmj57+83g7TLbjTM9UM8favyjODB1o5liZ65pKZHonp38us9WO5XFm\nwaE4M3AkzhydE2c8aQIUKE2AAqUJUKA0AQqUJkCB0gQoUJoABUoToEBpAhQoTYACpQlQ0Fm/OQ79\ny+/Fmc9+Kc7sHo4zk4kaf2xtnLl0a5yZOxZnjvXFmZ7EXPDYvDgzZzzOZHz32jjz/nubOVbP8TiT\nma2ek/icvmNxZnR+nGlqrnzJnjizd2mcmWjo0WXzlXHmskfizMKDceZI4ns+tCDOLN4XZ1a/GGfG\n58aZzFz54YE440kToEBpAhQoTYACpQlQoDQBCpQmQIHSBChQmgAFShOgQGkCFChNgIJOZtbyj/86\nztx9XZxZuT3OXH1/nPnx5XHmt74WZ776iTjzyGVx5nf+Lc68dE6c2bomznzwf+LMfdfEmfckvue/\n+aM4s25LnHn0nXFm1bY4c9HTcebD34ozf/+HceaFc+PMTbfGmczc/ZZ1ceaa78WZrZfEmTs/Emc+\ndlucyVzXxU/FmcyaZu7BzOz53MS7HV49K8540gQoUJoABUoToEBpAhQoTYACpQlQoDQBCpQmQIHS\nBChQmgAFShOgoH3z324MQ/sWxx90NDH7uWNFnHnHK3FmPLF/cUZnIs7sH4oz5z8XZ/pH48z2lc18\nTuZ9AsO748wTiVnmzF7kc4/GmQuejTN7l8SZzF7bmfPJzCCv2RpnMuuVOdZk4vEm83s+uDDOjAzG\nmaWJPd/bU3FmohNn5h+JM8d748zgSJzJ9JgnTYACpQlQoDQBCpQmQIHSBChQmgAFShOgQGkCFChN\ngAKlCVCgNAEKOov3xaGle5s52LrH4sw3Evsyf/TOODOSmL8ePBxnumlJYi02Xxln1j84/XNptVqt\nlYn3AHRT5nf4fGK/8vNemP65tFqt1lhiTnleYs590cE48/2r48x7E3vZN2X30jizJLFemdnzOYn3\nG3STJ02AAqUJUKA0AQqUJkCB0gQoUJoABUoToEBpAhQoTYACpQlQoDQBCtq33BLvez46L/6g/rEG\nzibpOx+IM798z+k/j9cdTezDvmxXnDkwNO1TabVardaz58eZCxJ7tU+240xPYm/rmWZOYh58PDFX\nvjSxd/xA4v0GLybm5TP2DcWZxfubOVbmtzHaH2cGEnuaDyXeyXBu4n0CT10cZzI8aQIUKE2AAqUJ\nUKA0AQqUJkCB0gQoUJoABUoToEBpAhQoTYACpQlQ0Bnvi0OZvdGPJmZ1pxIVvX9RnMnMlf/Hx+PM\nym1x5uofxJnMjO3O5XEm47aPxZlPfznO9EzGmb2Jva2bMtEbZ6YS88633hRnLno6znzwO3Fmz3Az\nmcwc91c/EWf+4J/iTOY7zNynk4nM3MSM/+03xJlr740z+xfHmYzU+xaaORTA/w9KE6BAaQIUKE2A\nAqUJUKA0AQqUJkCB0gQoUJoABUoToEBpAhR0Di6MQ+OJfb0HE/s7jyXm0w8PxJmRwTgz/Gqceejd\ncSYzez50IM405aoH4sy3fiXOtBP7la/bEmfmj8aZ0Xlxpn8szmTeS7BiR5x55LI4k5k9b0pm7/jf\nvzXO/OOn4swnvxJnMu8BWHAozgztjzMf+u84MzgSZ3qOx5nJxHVl1sKTJkCB0gQoUJoABUoToEBp\nAhQoTYACpQlQoDQBCpQmQIHSBChQmgAFneE93TvYvMQ+yJl9mZftijOZufKbE/uDf/tDcebip+LM\neS/EmYzMbP5VP4ozr6yIM5m58ozMXPl918SZ930vzpyb+J4/cE+c2bI2zlz6eJzJzDJnfDPxPoHM\nXPmhxHsbFiRmvUfnN5O5/+o4c91/xZnMXHnvRJwZSLxDw5MmQIHSBChQmgAFShOgQGkCFChNgAKl\nCVCgNAEKlCZAgdIEKFCaAAXtz39hYxga3h1/0P/+Qpx59aw485v/Hmc2r48z8xLzzvsWx5kXV8eZ\nbavizJK9cSazB/0Nd8SZ/UNxJrOX9INXxpk543HmWF+c+Y2vx5knL4kzR/rjzPHEnPLO5XEm8zvM\nvJfgpXPizNX3x5nMPuPbV8aZzH3xq9+MM09fFGd6E/uVb1kXZzLrlTE+J8540gQoUJoABUoToEBp\nAhQoTYACpQlQoDQBCpQmQIHSBChQmgAFShOgoP3Fz8Wz55n9i7vp6QvjzEXPxJmtiVnmNU/EmZnm\nHz4VZz75z3Gmd3L65zITbU7M1K9/8PSfx5nu9hvizI2JdylMJN4V0EnMsDfFkyZAgdIEKFCaAAVK\nE6BAaQIUKE2AAqUJUKA0AQqUJkCB0gQoUJoABe3P/GU8e57Zj3vV9gbOhq75i8/Gmc996fSfx+sO\nJ95vMHDk9J8HZ66RgTgzeDjOeNIEKFCaAAVKE6BAaQIUKE2AAqUJUKA0AQqUJkCB0gQoUJoABUoT\noKAzvDsOLdsVZ3asiDMLDsWZgcTs50QnzuxdkjifkTiTMZi4rpEFcWbfUJxZtS3OvOsnceYzfxVn\n7rouznz0G3HmcGLm91hfnOkcS2Qm4sxYf5zJmGzHmZ6pOHNoMM7sWRpn3vlonHnikjizYkecee78\nOHPFw3Hm4KI405TMXPnLq+KMJ02AAqUJUKA0AQqUJkCB0gQoUJoABUoToEBpAhQoTYACpQlQoDQB\nCtp/+ufxvuc9k/EH9SVmfn94VZy57JE4k5m/njMeZ7auiTODifn0zvE4k9FOfM+ZOe7MtTd1zk9e\nHGcevzTOfPjbcWbeWJzJvAMh816CtY/FmcxceUZPYi3aiWM9kLi/rnwozjx0RZxZszXOjM2LM/uH\n4sx5z8eZjMx69SZ6zJMmQIHSBChQmgAFShOgQGkCFChNgAKlCVCgNAEKlCZAgdIEKFCaAAXtr388\nnj3vTczGPnpZA2fTarW2rI0z6xJzwcsTezfvTMwpZ7xje5x5ZWWcGUnMlb+4Os6sfTzOjCbmgvsT\ns95N7f2d2WP9+rvjzA23x5k7bowzGZk55eOdZj4ns8/46pcS55N4TJpMZDKdMJX4bQwk9iIfnxNn\nJnvjzEQic2hBnPGkCVCgNAEKlCZAgdIEKFCaAAVKE6BAaQIUKE2AAqUJUKA0AQqUJkBB5+Wz41Bm\nL+mjc+PM8O44c2libvoniTn3d8WRlL7EHuKZufKMBYfiTGau/PD8OJPZyz4jsxd5Zq/2zL7nt/16\nnOmmgwvjzMCROJOZrc7MlWf0JtY9k8mYSDySjfU3c6yMTmJefvH+OONJE6BAaQIUKE2AAqUJUKA0\nAQqUJkCB0gQoUJoABUoToEBpAhQoTYCCzu7hODR/NM5kZs9HBuNMZj/ltYl9z5uya1mcWXgwzmTm\neaca+iMsM++8f1GcyewBnVmvzD7aZyXeS/CeH8SZptzxa3Hmhv9s5lhNzXo35dnEHuvnJGbh+xL7\nuWe8vCrOnL2tmWNleNIEKFCaAAVKE6BAaQIUKE2AAqUJUKA0AQqUJkCB0gQoUJoABUoToKAz92gc\nysxEL9kbZ3YujzOLDsSZ58+LM33H4kxm3+r2VJzJ7CGe2fv77JfjTGZv9Mxe5PMT8+kZme8wM3t+\nrC/O7FscZzavjzOZ39hwYhb+X383zhxIzPiv2RpnLnwmzmSua8eKONOfeNfE0xfFmcX74kymE0YT\ne6Nnfhsrt8eZzDl70gQoUJoABUoToEBpAhQoTYACpQlQoDQBCpQmQIHSBChQmgAFShOgoH3TVzaG\nodWJPY6bcjxR45l9ou+6Ls5cf3ecyWh3cU/zH14VZ37pgTgzmJhhH0nse96U0Xlxpj8xU//iOXGm\nqd9zT2bP995mjvV3n44zN3+5mWN1U2bdBw7Hmcy7C5raX96TJkCB0gQoUJoABUoToEBpAhQoTYAC\npQlQoDQBCpQmQIHSBChQmgAF7c9/IZ49374y/qDZOJ9+pnr48jhzxY9P91nMXIfnx5mBhvaF76Zb\n/iyRiW/3xowMxJnBxFz5RGJ+v5N4D0BG5lieNAEKlCZAgdIEKFCaAAVKE6BAaQIUKE2AAqUJUKA0\nAQqUJkCB0gQo6GRCq7bFmcw8+LZVcWbn8jhz8VNxZuHBODPVjjOZPZcX74szu5Y1cz47VsSZS56I\nMy+sjjMXPBtnMuvVnoozz1wYZzLXtSCxn3tmrnxB4vczkbh7xhL7evck3pNwKLEH/Z98Mc5kZqsz\na7FnaZy56oE4k5H5fjIy155ZU0+aAAVKE6BAaQIUKE2AAqUJUKA0AQqUJkCB0gQoUJoABUoToEBp\nAhR0+o7FoZXb48y2s+PMhc/EmfOejzNzxuPM7TfGmcz87LX3xpmle+JMZt55MjF7npkHz8x6Z76f\nFTvizKIDcaY3sSf1+s1xpidxXXdeH2d2D8eZzO8wMwu/bFecmUo8uvzoF+PMtffFmZ2JdyBc8mSc\nyfxWx+fEma/9dpx5X+K6LnguzmT2Rs+8S8GTJkCB0gQoUJoABUoToEBpAhQoTYACpQlQoDQBCpQm\nQIHSBChQmgAFnczc65H5cSYzi3osMYs6NjfxOX1x5gP3xJnMbGxmL/KDC+NMZvY8M1t9eCDOZPab\nvunWOLN/KM7MTbwHIGPv4jiTmfH/yF1x5lhib+u+iTjz8qo4k1nT3sSxVm2LMxlL9jbzOUcT92n/\nWJx57/fjzHffH2cys+cZqxLv2fCkCVCgNAEKlCZAgdIEKFCaAAVKE6BAaQIUKE2AAqUJUKA0AQqU\nJkDB/wEXziwU2v52ygAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<IPython.core.display.Image object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "def image_for_code(code):\n",
    "    act = activations(flat_model, code)\n",
    "    neurons = interesting_neurons(act)\n",
    "    return visualize_neurons(neurons, code, act)\n",
    "\n",
    "display(image_for_code('if (a == 2) and ((b == 1) or (c==2)):'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "code = 'if (a == 2) and ((b == 1) or (c==2)):'\n",
    "mask = '   ________     ____________________ '\n",
    "act = activations(flat_model, code)\n",
    "positive = [idx for idx, ch in enumerate(mask) if ch == '_']\n",
    "negative = [idx for idx, ch in enumerate(mask) if ch != '_']\n",
    "\n",
    "neurons = np.argsort(act[positive].sum(axis=0) - act[negative].sum(axis=0))[-5:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAbwAAABICAIAAAACptczAAAFXklEQVR4nO3dT2wUZRzG8d2yhVWo\nYnGVFJUWNUH8d9CbetKLxpNejYl/wsEbISaQ6CRdDpIQ48mLMZgYr3oxMR64qTc9KIokKgtqG6RS\n0SIsf0o9NG62BfZ5fp2309nm+7n26cw785v312GXd6Y6Pj5eAQB4BlZ6AADQTxY0zSzLsizr/QvZ\n//LvO8lGSsU5OSU86oJr2iOTahjF68e6m8oz8lTzK+cRLWiazWaz2WzKncmYI8uy/Bsplc7JqfSs\nSrPZLM9VOC9VIZyarsq6OwUtYd0d5alXwvmVsxa17gF1tnjNaCfTPfqlCf01SLUjeVw59xX63fzX\nojw/87uQxUo1kxP+hU97gRVTd//QCqh7ZWFfWPIek8zTVLVIPr+WXIta94B6nyNnBqbSXfg815nz\ni2kPp5hTZJ6fTt+8XubqCVYG/Vj3Yjh17/y0svyVNccjt+PXovgWdPWPasu976XpLnweRd5xVAqs\naKrzs1oVXPdymj+oYg7N/KdMzloUNr9676WkTTPV38ki7zgWjXlZS1vOO8TyWK13mqXV45pPVYsi\n51dvA50BdX9kec2pKAM+84AXfYraF8yhJil5/vOz6GOZPOc5YZ9a8VkR4sydjpwH1V0v87uH/LvL\nOZ60Es4v+aX39fZVXcH/3J7/Q/F+tLqPelk/gO9r/XvU/TXyAq7AlWyaANB3WBEEAAG19qAObTyj\nM+26zlyp6sz0Jp0ZPq0zB1/Rma0ndObJQzrjjPnGczrj+OAlnXlzn84MXNGZUw2dSWXW+Epyzrh+\n3t6tM/f/oDPPfqozqcwZty7v7NKZ1w8Y+zLOoTNPnXo5PnpBZ575TGcap/KPpVLxasGdJgAE0DQB\nIICmCQABNE0ACKBpAkAATRMAAmiaABBA0wSAAJomAATQNAEggKYJAAG1eluHbv9DZ1qjOrPuos40\npoztXNCZoRmdObpdZ7Yd0xlnzPXzOuOse92zX2cuG+uC39qrMy8f1Jk1s2kyzlr4qpEZa+lMa0xn\nPn9aZx48rDMjkzrjHNeOIzoza1w/k1t0ZmRCZ5x6XVinM3/frDN/3aIzqdae/3qXznCnCQABNE0A\nCKBpAkAATRMAAmiaABBA0wSAAJomAATQNAEggKYJAAE0TQAIoGkCQED1jX3jMuSsPf/6UZ05uVln\ndr6nM18+rjPOe8anjPd6/3K3zpzYqjO3GWtjnTW2L36oM9PDOuOszf/qMZ1xngNwca3OvPq+zhx+\nQGfOrdcZZ23+5IjOfPGEzjjvWD+2TWeeOqQzt/6pM87aamdePP+xzhzZoTO1yzrzzSM649TL0a7r\nDHeaABBA0wSAAJomAATQNAEggKYJAAE0TQAIoGkCQABNEwACaJoAEEDTBIAAmiYABFRfe1evPT+9\nSW/o/A06M2G8c/mO33XGeZ+yY/CSzjjruO/9SWfW/6szzrpgZzszQzrjrIX/3ljrvdZ4l72zPn37\nUZ1x1kQ779Gut3XGeU7CQ9/pjFMvZ1+za3TGuZ6d5xukun6qczpzaVBnNpzVGed5Ajf9ozOsPQeA\nxGiaABBA0wSAAJomAATQNAEggKYJAAE0TQAIoGkCQABNEwACaJoAEEDTBICAmvNO84axznTAWGe6\n+aTODE/rzM/36IyzJtpZh7vxjM44a/OPj+rM6HGdccYz1tIZh7Pm11nr7aybdp5d4DxzoDWmM841\n5ryP+5PndMZ5dsHD3+rMfT/qjHNtnN2gM3f+pjNOLZzz7DyPwnmn+dCMzjSmdGbLhM5wpwkAATRN\nAAigaQJAAE0TAAJomgAQQNMEgACaJgAE0DQBIICmCQABNE0ACKBpAkDAf9KOQmNi10SSAAAAAElF\nTkSuQmCC\n",
      "text/plain": [
       "<IPython.core.display.Image object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "img = visualize_neurons(neurons, code, act)\n",
    "display(img)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([108,  58, 269, 279, 276])"
      ]
     },
     "execution_count": 74,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "neurons"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.45439816"
      ]
     },
     "execution_count": 79,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "act[negative, 108].sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(4.2561150336332503, 0.45439814683049917)"
      ]
     },
     "execution_count": 81,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x0 = 0\n",
    "x1 = 0\n",
    "for idx, ch in enumerate(mask):\n",
    "    if ch == '_':\n",
    "        x0 += act[idx, 108]\n",
    "    else:\n",
    "        x1 += act[idx, 108]\n",
    "x0, x1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}