src/Q&A with TF- TFRecords and Eager Execution.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/sdoneva/anaconda/lib/python3.6/site-packages/h5py/__init__.py:34: FutureWarning: Conversion of the second argument of issubdtype from `float` to `np.floating` is deprecated. In future, it will be treated as `np.float64 == np.dtype(float).type`.\n",
      "  from ._conv import register_converters as _register_converters\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /Users/sdoneva/anaconda/lib/python3.6/site-packages/tensorflow/contrib/learn/python/learn/datasets/base.py:198: retry (from tensorflow.contrib.learn.python.learn.datasets.base) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use the retry module or similar alternatives.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "import tensorflow as tf\n",
    "import tensorflow.contrib.eager as tfe\n",
    "import os\n",
    "os.environ[\"TF_CPP_MIN_LOG_LEVEL\"]=\"3\"\n",
    "\n",
    "tf.enable_eager_execution()\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "import re\n",
    "import numpy as np\n",
    "from string import punctuation\n",
    "from collections import defaultdict\n",
    "from functools import reduce\n",
    "from keras.preprocessing.sequence import pad_sequences\n",
    "from itertools import chain\n",
    "from InputPreparator import EmbeddingsPreparator\n",
    "from InputPreparator import StoryParser\n",
    "import csv\n",
    "from tqdm import tqdm\n",
    "import random\n",
    "\n",
    "\n",
    "#to avoid a warning from TF 1.7 version see https://github.com/tensorflow/tensorflow/issues/18111\n",
    "import warnings\n",
    "warnings.filterwarnings('ignore')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### More on Gradien Tape: \n",
    "- https://www.tensorflow.org/programmers_guide/eager#advanced_automatic_differentiation_topics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "TASK_NUMBER = 10\n",
    "SUPPORTING_ONLY = True\n",
    "PATH_TO_EMBED = \"data/glove.6B.50d.txt\"\n",
    "PATH_TO_TASKS = \"data/tasks_1-20_v1-2/en/\"\n",
    "USE_GRADIENT_TAPE = False"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Data preparation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def get_task_files(task_nr):\n",
    "    if task_nr==5:\n",
    "        return 'qa5_three-arg-relations_train.txt', \"qa5_three-arg-relations_test.txt\"\n",
    "    if task_nr==6:\n",
    "        return 'qa6_yes-no-questions_train.txt', 'qa6_yes-no-questions_test.txt'\n",
    "    if task_nr==10:\n",
    "        return 'qa10_indefinite-knowledge_train.txt', 'qa10_indefinite-knowledge_test.txt'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "train_set_file = get_task_files(TASK_NUMBER)[0]\n",
    "test_set_file = get_task_files(TASK_NUMBER)[1]\n",
    "\n",
    "train_set_post_file = PATH_TO_TASKS + train_set_file\n",
    "test_set_post_file = PATH_TO_TASKS + test_set_file"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "embedder=EmbeddingsPreparator()\n",
    "story_parser=StoryParser()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "vocab_tokens = embedder.get_unique_tokens([train_set_post_file, test_set_post_file])\n",
    "word_to_index, index_to_embedding = embedder.load_embedding_from_disks(PATH_TO_EMBED,vocab_tokens, with_indexes=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "26 26\n"
     ]
    }
   ],
   "source": [
    "print(len(vocab_tokens), len(word_to_index))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Embedding"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "embed_dimensions= 50\n",
    "\n",
    "embedding_matrix = np.zeros((len(word_to_index) + 1, embed_dimensions))\n",
    "for word, i in word_to_index.items():\n",
    "    embedding_vector = index_to_embedding[i]\n",
    "    if embedding_vector is not None:\n",
    "        # words not found in embedding index will be all-zeros.\n",
    "        embedding_matrix[i] = embedding_vector"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "train_stories=story_parser.get_stories(train_set_post_file, SUPPORTING_ONLY)\n",
    "test_stories=story_parser.get_stories(test_set_post_file, SUPPORTING_ONLY)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "random.shuffle(train_stories)\n",
    "sl = slice(0, 200)\n",
    "valid_stories= train_stories[sl]\n",
    "s2 = slice(200, 1000)\n",
    "train_stories= train_stories[s2]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# store into a csv file: per row (context, question, answer)\n",
    "def store_to_csv(data, filename, vectors):\n",
    "    with open(filename,'w') as f:   \n",
    "        writer = csv.writer(f)\n",
    "        writer.writerow(['context', 'question', 'answer'])\n",
    "        for story in data:\n",
    "            if vectors:\n",
    "                writer.writerow(story)\n",
    "            else:\n",
    "                temp=[]\n",
    "                context, question, answer= story\n",
    "                temp.append(' '.join(context))\n",
    "                temp.append(' '.join(question))\n",
    "                temp.append(''.join(answer))            \n",
    "                writer.writerow(temp)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "store_to_csv(train_stories, 'train_data.csv', False)\n",
    "store_to_csv(test_stories, 'test_data.csv', False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def vectorize(sentence, answer, word_to_index):\n",
    "    if not answer:\n",
    "        x=[]\n",
    "        for w in sentence.split():\n",
    "            w=w.replace(\"]\",\"\").replace(\"[\",\"\").replace(\"'\",\"\").replace(\",\",\"\")\n",
    "            w=w.lower().strip()\n",
    "            x.append(word_to_index[w]) \n",
    "        return x\n",
    "     \n",
    "    else:\n",
    "        # The Answer is one-hot encoded in our vocabulary matrix\n",
    "        y = np.zeros(len(word_to_index) + 1, dtype=int)\n",
    "        answ=sentence.lower().strip()\n",
    "        y[word_to_index[answ]] = 1\n",
    "        return y      "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "# Convert to TFRecords\n",
    "- https://medium.com/@TalPerry/getting-text-into-tensorflow-with-the-dataset-api-ffb832c8bec6"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def sequence_to_tf_example(context, question, answer):\n",
    "        context_ids= vectorize(context, False, word_to_index)\n",
    "        question_ids= vectorize(question, False, word_to_index)\n",
    "        answer_ids= vectorize(answer, True, word_to_index)\n",
    "        ex = tf.train.SequenceExample()\n",
    "      \n",
    "        context_tokens = ex.feature_lists.feature_list[\"context\"]\n",
    "        question_tokens = ex.feature_lists.feature_list[\"question\"]\n",
    "        answer_tokens = ex.feature_lists.feature_list[\"answer\"]\n",
    "        \n",
    "        for token in context_ids:\n",
    "            context_tokens.feature.add().int64_list.value.append(token)\n",
    "        for token in question_ids:\n",
    "            question_tokens.feature.add().int64_list.value.append(token)\n",
    "        for token in answer_ids:\n",
    "            #print(token)\n",
    "            answer_tokens.feature.add().int64_list.value.append(token)\n",
    "\n",
    "        return ex"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def read_from_tfrecord(ex):\n",
    "    '''\n",
    "    Explain to TF how to go from a serialized example back to tensors\n",
    "    '''\n",
    "    sequence_features = {\n",
    "        \"context\": tf.FixedLenSequenceFeature([], dtype=tf.int64),\n",
    "        \"question\": tf.FixedLenSequenceFeature([], dtype=tf.int64),\n",
    "        \"answer\": tf.FixedLenSequenceFeature([], dtype=tf.int64)\n",
    "    }\n",
    "\n",
    "    # Parse the example (returns a dictionary of tensors)\n",
    "    _, sequence_parsed = tf.parse_single_sequence_example(\n",
    "        serialized=ex,\n",
    "        sequence_features=sequence_features\n",
    "    )\n",
    "\n",
    "    return {\"context\": sequence_parsed['context'], \"question\": sequence_parsed['question'],\n",
    "            \"answer\": sequence_parsed['answer']}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def write_example_to_tfrecord(context, question, answer, tfrecord_file, writer):\n",
    "    example= sequence_to_tf_example(context, question, answer)\n",
    "    writer.write(example.SerializeToString())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def write_data_to_tf_record(filename):\n",
    "    file_csv= filename+'.csv'\n",
    "    file_tfrecords= filename+'.tfrecords'\n",
    "    with open(file_csv) as csvfile:\n",
    "        readCSV = csv.reader(csvfile, delimiter=',')\n",
    "        next(readCSV) #skip header\n",
    "        writer= tf.python_io.TFRecordWriter(file_tfrecords)\n",
    "        for row in readCSV:\n",
    "        #print(row[0], row[1], row[2])\n",
    "            write_example_to_tfrecord(row[0], row[1], row[2], file_tfrecords, writer)\n",
    "        writer.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "write_data_to_tf_record('train_data')\n",
    "write_data_to_tf_record('test_data')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def make_dataset(path, batch_size=128):\n",
    "    '''\n",
    "    Makes  a Tensorflow dataset that is shuffled, batched and parsed.\n",
    "    '''\n",
    "    # Read a tf record file. This makes a dataset of raw TFRecords\n",
    "    dataset = tf.data.TFRecordDataset([path])\n",
    "    # Apply/map the parse function to every record. Now the dataset is a bunch of dictionaries of Tensors\n",
    "    dataset =  dataset.map(read_from_tfrecord)\n",
    "    #Shuffle the dataset\n",
    "    dataset = dataset.shuffle(buffer_size=10000)\n",
    "   \n",
    "    # specify padding for each tensor seperatly\n",
    "    dataset = dataset.padded_batch(batch_size, padded_shapes={\n",
    "        \"context\": tf.TensorShape([None]), \n",
    "        \"question\": tf.TensorShape([None]), \n",
    "        \"answer\": tf.TensorShape([None]) \n",
    "    })\n",
    "   \n",
    "    return dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "train_data_tfrecords= make_dataset('train_data.tfrecords')\n",
    "test_data_tfrecords= make_dataset('test_data.tfrecords', 1000) #no need to batch for testing"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Eager Execution Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "learning_rate= 0.001\n",
    "vocab_size= len(index_to_embedding)\n",
    "num_units_gru= 50\n",
    "keep_prob= 0.5\n",
    "num_epochs= 200"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "class Model(tf.keras.Model):\n",
    "    \n",
    "    def __init__(self):\n",
    "        super(Model, self).__init__()\n",
    "        self.embed = tf.keras.layers.Embedding(len(word_to_index) + 1,\n",
    "                            embed_dimensions,\n",
    "                            weights=[embedding_matrix],\n",
    "                            trainable=False)\n",
    "        self.grucell=tf.keras.layers.GRUCell(num_units_gru)\n",
    "        self.rnn=tf.keras.layers.RNN(self.grucell)\n",
    "        self.dense=tf.keras.layers.Dense(vocab_size, activation=tf.nn.softmax)\n",
    "        self.dropout=tf.keras.layers.Dropout(keep_prob)\n",
    "        \n",
    "    def predict(self, sentence, question):\n",
    "        encoded_sentence=self.embed(sentence)\n",
    "        encoded_sentence=self.rnn(encoded_sentence)\n",
    "        encoded_sentence=self.dropout(encoded_sentence)\n",
    "        \n",
    "        encoded_question=self.embed(question)\n",
    "        encoded_question=self.rnn(encoded_question)\n",
    "        encoded_question=self.dropout(encoded_question)\n",
    "        \n",
    "        merged= tf.keras.layers.concatenate([encoded_sentence, encoded_question])\n",
    "        pred= self.dense(merged)\n",
    "        \n",
    "        return pred"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def loss(model, sent, quest, y):\n",
    "    prediction = model.predict(sent, quest)\n",
    "    return tf.keras.losses.categorical_crossentropy(y, prediction)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def grad(model, sent, quest, targets):\n",
    "    with tfe.GradientTape() as tape:\n",
    "        loss_value = loss(model, sent, quest, targets)\n",
    "        tf.contrib.summary.scalar(\"loss\", loss_value)\n",
    "    return tape.gradient(loss_value, model.variables), loss_value"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "optimizer = tf.train.AdamOptimizer(learning_rate)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import time"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def train_with_tape():\n",
    "    for i in tqdm(range(num_epochs)):\n",
    "        for batch in tfe.Iterator(train_data_tfrecords): # 8 batches\n",
    "            answer = tf.keras.backend.cast(batch['answer'], 'float32')\n",
    "            grads, loss_value = grad(model, batch['context'], batch['question'], answer)\n",
    "            optimizer.apply_gradients(zip(grads, model.variables), global_step=tf.train.get_or_create_global_step())\n",
    "            \n",
    "        if i % 20 == 0:\n",
    "            loss_value= loss(model, batch['context'], batch['question'], answer)\n",
    "            print(\"Loss at epoch {}: {}\".format(i, np.mean(loss_value)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def train_opt():\n",
    "     for i in tqdm(range(num_epochs)):\n",
    "        for batch in tfe.Iterator(train_data_tfrecords): # 8 batches\n",
    "            answer = tf.keras.backend.cast(batch['answer'], 'float32')\n",
    "            optimizer.minimize(lambda: loss(model, batch['context'], batch['question'], answer))\n",
    "\n",
    "        if i % 20 == 0:\n",
    "            loss_value= loss(model, batch['context'], batch['question'], answer)\n",
    "            print(\"Loss at epoch {}: {}\".format(i, np.mean(loss_value)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/200 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 1/200 [00:00<03:17,  1.01it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loss at epoch 0: 2.0571672916412354\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 10%|█         | 21/200 [00:21<03:05,  1.04s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loss at epoch 20: 0.6941195726394653\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 20%|██        | 41/200 [00:43<02:49,  1.07s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loss at epoch 40: 0.6935418844223022\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 30%|███       | 61/200 [01:06<02:32,  1.10s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loss at epoch 60: 0.7194653153419495\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 40%|████      | 81/200 [01:25<02:05,  1.05s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loss at epoch 80: 0.705836296081543\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 50%|█████     | 101/200 [01:44<01:42,  1.03s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loss at epoch 100: 0.6337305307388306\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 60%|██████    | 121/200 [02:08<01:23,  1.06s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loss at epoch 120: 0.6332855224609375\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 70%|███████   | 141/200 [02:31<01:03,  1.07s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loss at epoch 140: 0.640498697757721\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 80%|████████  | 161/200 [02:56<00:42,  1.09s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loss at epoch 160: 0.631808876991272\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 90%|█████████ | 181/200 [03:18<00:20,  1.10s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loss at epoch 180: 0.6924166083335876\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 200/200 [03:38<00:00,  1.09s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Training time: \n",
      "218.9827480316162\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "model= Model()\n",
    "accuracy = tfe.metrics.Accuracy()\n",
    "\n",
    "print('Training...')\n",
    "start_time = time.time()  \n",
    "if USE_GRADIENT_TAPE:\n",
    "    train_with_tape()\n",
    "else:\n",
    "    train_opt()\n",
    "elapsed_time = time.time() - start_time \n",
    "\n",
    "print()\n",
    "print('Training time: ')\n",
    "print(elapsed_time)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Final Testing Accuracy:\n",
      "0.553\n"
     ]
    }
   ],
   "source": [
    "for batch in tfe.Iterator(test_data_tfrecords): # 1 batch\n",
    "        answer = tf.keras.backend.cast(batch['answer'], 'float32')\n",
    "        prediction= model.predict(batch['context'], batch['question']) \n",
    "        pred=tf.cast(tf.argmax(prediction, 1), 'int32')\n",
    "        answ= tf.cast(tf.argmax(answer, 1), 'int32')\n",
    "        \n",
    "        corrects = tf.equal(pred, answ, 'int32')\n",
    "        accuracy = np.mean(tf.cast(corrects, tf.float32))\n",
    "        \n",
    "        print('Final Testing Accuracy:')\n",
    "        print(accuracy)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}