MSLR.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "num features : 136\n",
      "num docs (train, valid, test) : (2270258,747201,749194)\n",
      "num queries (train, valid, test) : (18919,6306,6117)\n"
     ]
    }
   ],
   "source": [
    "import time\n",
    "import lightgbm as lgb\n",
    "import Mmetrics\n",
    "import os\n",
    "import numpy as np\n",
    "from read_data import read_pkl,lambdarank,evaluate_train,evaluate\n",
    "\n",
    "data_path = '/ivi/ilps/personal/avardas/_data/MSLR-WEB30K/Fold1/'\n",
    "dataset = read_pkl(data_path + 'binarized.pkl.npz')\n",
    "\n",
    "\n",
    "groups = {'tr':{}, 'te':{}, 'va':{}}\n",
    "\n",
    "groups['tr']['QualityScore2'] = np.zeros_like(dataset.trfm[:,132])\n",
    "groups['te']['QualityScore2'] = np.zeros_like(dataset.tefm[:,132])\n",
    "groups['va']['QualityScore2'] = np.zeros_like(dataset.vafm[:,132])\n",
    "groups['tr']['QualityScore2'][dataset.trfm[:,132]>10] = 1.\n",
    "groups['te']['QualityScore2'][dataset.tefm[:,132]>10] = 1.\n",
    "groups['va']['QualityScore2'][dataset.vafm[:,132]>10] = 1.\n",
    "\n",
    "\n",
    "groups['tr']['PageRank'] = np.zeros_like(dataset.trfm[:,129])\n",
    "groups['te']['PageRank'] = np.zeros_like(dataset.tefm[:,129])\n",
    "groups['va']['PageRank'] = np.zeros_like(dataset.vafm[:,129])\n",
    "groups['tr']['PageRank'][dataset.trfm[:,129]>1000] = 1.\n",
    "groups['te']['PageRank'][dataset.tefm[:,129]>1000] = 1.\n",
    "groups['va']['PageRank'][dataset.vafm[:,129]>1000] = 1.\n",
    "groups['tr']['PageRank'][dataset.trfm[:,129]>10000] = 2.\n",
    "groups['te']['PageRank'][dataset.tefm[:,129]>10000] = 2.\n",
    "groups['va']['PageRank'][dataset.vafm[:,129]>10000] = 2."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "num features : 131\n",
      "num docs (train, valid, test) : (2270258,747201,749194)\n",
      "num queries (train, valid, test) : (18919,6306,6117)\n"
     ]
    }
   ],
   "source": [
    "dataset = read_pkl(data_path + 'binarized.normalized.pkl.npz')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-17-574a8fb19b67>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      6\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      7\u001b[0m \u001b[0mltrmodel\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mLTR\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mMSE_model\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlayers\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mdataset\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtrfm\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mshape\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m256\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m256\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlr\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m0.001\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0moptimizer\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0moptim\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mAdam\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdropout\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m0.1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 8\u001b[0;31m \u001b[0mltrmodel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdataset\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mepochs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m10\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbatch_size\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m100\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mverbose\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      9\u001b[0m \u001b[0my_pred\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mltrmodel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpredict\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdataset\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtefm\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdataset\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtedlr\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     10\u001b[0m \u001b[0mLTRMetrics\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdataset\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtelv\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdiff\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdataset\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtedlr\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my_pred\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mNDCG\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m10\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/mDrive/PPG/LTR.py\u001b[0m in \u001b[0;36mfit\u001b[0;34m(self, dataset, epochs, batch_size, shuffle, verbose)\u001b[0m\n\u001b[1;32m     74\u001b[0m             \u001b[0mloss\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     75\u001b[0m             \u001b[0;32mfor\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0my\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mdl\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 76\u001b[0;31m                 \u001b[0mbatch_loss\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnet\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtrain_batch\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mopt\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mloss_fn\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     77\u001b[0m                 \u001b[0mloss\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbatch_loss\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcpu\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     78\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/mDrive/PPG/LTR.py\u001b[0m in \u001b[0;36mtrain_batch\u001b[0;34m(self, x, y, optimizer, loss_fn)\u001b[0m\n\u001b[1;32m     53\u001b[0m         \u001b[0;31m# print(out.shape)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     54\u001b[0m         \u001b[0mloss\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mloss_fn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mout\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 55\u001b[0;31m         \u001b[0mloss\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbackward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     56\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     57\u001b[0m         \u001b[0moptimizer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstep\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.7/site-packages/torch/tensor.py\u001b[0m in \u001b[0;36mbackward\u001b[0;34m(self, gradient, retain_graph, create_graph)\u001b[0m\n\u001b[1;32m    183\u001b[0m                 \u001b[0mproducts\u001b[0m\u001b[0;34m.\u001b[0m \u001b[0mDefaults\u001b[0m \u001b[0mto\u001b[0m\u001b[0;31m \u001b[0m\u001b[0;31m`\u001b[0m\u001b[0;31m`\u001b[0m\u001b[0;32mFalse\u001b[0m\u001b[0;31m`\u001b[0m\u001b[0;31m`\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    184\u001b[0m         \"\"\"\n\u001b[0;32m--> 185\u001b[0;31m         \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mautograd\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbackward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mgradient\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mretain_graph\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcreate_graph\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    186\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    187\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mregister_hook\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhook\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.7/site-packages/torch/autograd/__init__.py\u001b[0m in \u001b[0;36mbackward\u001b[0;34m(tensors, grad_tensors, retain_graph, create_graph, grad_variables)\u001b[0m\n\u001b[1;32m    125\u001b[0m     Variable._execution_engine.run_backward(\n\u001b[1;32m    126\u001b[0m         \u001b[0mtensors\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mgrad_tensors\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mretain_graph\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcreate_graph\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 127\u001b[0;31m         allow_unreachable=True)  # allow_unreachable flag\n\u001b[0m\u001b[1;32m    128\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    129\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "from Mmetrics import *\n",
    "\n",
    "import LTR\n",
    "import datautil\n",
    "import torch\n",
    "\n",
    "ltrmodel = LTR.MSE_model(layers=[dataset.trfm.shape[1], 256, 256, 1], lr=0.001, optimizer=torch.optim.Adam, dropout=0.1)\n",
    "ltrmodel.fit(dataset, epochs=10, batch_size=100, verbose=True)\n",
    "y_pred = ltrmodel.predict(dataset.tefm, dataset.tedlr)\n",
    "LTRMetrics(dataset.telv, np.diff(dataset.tedlr), y_pred).NDCG(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import pickle\n",
    "\n",
    "df = pd.DataFrame({'lv':dataset.telv, 'g':groups['te']['QualityScore2'], 'qid':np.repeat(np.arange(dataset.tedlr.shape[0]-1), np.diff(dataset.tedlr)), 'y_pred':y_pred})\n",
    "with open('MSLR_qs.df', 'wb') as f:\n",
    "    pickle.dump(df, f)\n",
    "    \n",
    "df = pd.DataFrame({'lv':dataset.telv, 'g':groups['te']['PageRank'], 'qid':np.repeat(np.arange(dataset.tedlr.shape[0]-1), np.diff(dataset.tedlr)), 'y_pred':y_pred})\n",
    "with open('MSLR_pr.df', 'wb') as f:\n",
    "    pickle.dump(df, f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "def subsample(ds, percent):\n",
    "    subsample_size = int((ds.dlr.shape[0]-1) * percent)\n",
    "    qids = np.random.choice(np.arange(ds.dlr.shape[0]-1), subsample_size)\n",
    "#     print(qids)\n",
    "    sds = {'lv':[], 'y_pred':[], 'qid':[], 'g':[], 'dlr':[0]}\n",
    "    for qid in qids:\n",
    "        s,e=ds.dlr[qid:qid+2]\n",
    "#         print(s,e)\n",
    "        sds['lv'].append(ds.lv[s:e])\n",
    "        sds['y_pred'].append(ds.y_pred[s:e])\n",
    "        sds['g'].append(ds.g[s:e])\n",
    "        sds['qid'].append(ds.qid[s:e])\n",
    "        sds['dlr'].append(sds['dlr'][-1]+e-s)\n",
    "    for k in ['lv', 'y_pred', 'g', 'qid']:\n",
    "        sds[k] = np.concatenate(sds[k])\n",
    "    sds['dlr'] = np.array(sds['dlr'])\n",
    "    return sds\n",
    "\n",
    "\n",
    "def df2ds(df_path):\n",
    "    with open(df_path, 'rb') as f:\n",
    "        df = pickle.load(f)\n",
    "    ds = df.to_dict(orient='list')\n",
    "    for k in ds:\n",
    "        ds[k] = np.array(ds[k])\n",
    "    ds['dlr'] = np.concatenate([np.zeros(1), np.where(np.diff(ds['qid'])==1)[0]+1, np.array([ds['qid'].shape[0]])]).astype(int)\n",
    "    return type('ltr', (object,), ds)\n",
    "\n",
    "\n",
    "def dict2ds(df_path):\n",
    "    with open(df_path, 'rb') as f:\n",
    "        ds = pickle.load(f)\n",
    "    return type('ltr', (object,), ds)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "def purify(ds):\n",
    "    lv, g, qids, y_pred, dlr = [], [], [], [], [0]\n",
    "    groups = np.unique(ds.g)\n",
    "    for qid in range(ds.dlr.shape[0] - 1):\n",
    "        s, e = ds.dlr[qid:qid+2]\n",
    "        if len(np.where(ds.lv[s:e]==4)[0]) == 0:\n",
    "#             print('no level 4,', qid)\n",
    "            continue\n",
    "        z_util = False\n",
    "        for group in groups:\n",
    "            if ds.lv[s:e][ds.g[s:e]==group].sum() < 5:\n",
    "#                 print(qid, group, ds.lv[s:e][ds.g[s:e]==group].sum())\n",
    "                z_util = True\n",
    "                break\n",
    "        if z_util:\n",
    "            continue\n",
    "        if e - s > 20:\n",
    "            argsorted = ds.y_pred[s:e].argsort()[::-1]\n",
    "            index = list(argsorted[:20])\n",
    "            z_util = np.where(ds.lv[s:e][argsorted] == 4)[0]\n",
    "            if argsorted[z_util[0]] not in index:\n",
    "                index.append(argsorted[z_util[0]])\n",
    "            for group in groups:\n",
    "                z_util = np.where((ds.g[s:e][argsorted] == group) & (ds.lv[s:e][argsorted] > 0))[0]\n",
    "                if len(z_util) > 0 and argsorted[z_util[0]] not in index:\n",
    "                    index.append(argsorted[z_util[0]])\n",
    "            index = np.array(index)\n",
    "        else:\n",
    "            index = np.arange(e - s)\n",
    "        \n",
    "        lv.append(ds.lv[s:e][index])\n",
    "        g.append(ds.g[s:e][index])\n",
    "        qids.append(ds.qid[s:e][index])\n",
    "        y_pred.append(ds.y_pred[s:e][index])\n",
    "        dlr.append(dlr[-1] + index.shape[0])\n",
    "        \n",
    "    purified = {'lv':np.concatenate(lv), 'g':np.concatenate(g), 'qid':np.concatenate(qids), 'y_pred':np.concatenate(y_pred), 'dlr':np.array(dlr)}\n",
    "    return purified\n",
    "\n",
    "\n",
    "ds = df2ds('MSLR_qs.df')\n",
    "purified = purify(ds)\n",
    "            \n",
    "with open('p_MSLR_qs.df', 'wb') as f:\n",
    "    pickle.dump(purified, f)\n",
    "            "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(87,)\n"
     ]
    }
   ],
   "source": [
    "ds = dict2ds('p_MSLR_qs.df')\n",
    "sds = subsample(ds,0.05)\n",
    "print(sds['dlr'].shape)\n",
    "with open('s_MSLR_qs.df', 'wb') as f:\n",
    "    pickle.dump(sds, f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(78,)\n"
     ]
    }
   ],
   "source": [
    "ds = dict2ds('p_MSLR_pr.df')\n",
    "sds = subsample(ds,0.05)\n",
    "print(sds['dlr'].shape)\n",
    "with open('s_MSLR_pr.df', 'wb') as f:\n",
    "    pickle.dump(sds, f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(1550,) [31817 31838 31858 31878 31900 31922 31943 31964 31984 32003]\n"
     ]
    }
   ],
   "source": [
    "\n",
    "ds = df2ds('MSLR_pr.df')\n",
    "purified = purify(ds)\n",
    "            \n",
    "with open('p_MSLR_pr.df', 'wb') as f:\n",
    "    pickle.dump(purified, f)\n",
    "    \n",
    "print(purified['dlr'].shape,purified['dlr'][-10:])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1733,)"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "purified['dlr'].shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('MSLR_qs.df', 'wb') as f:\n",
    "    pickle.dump(df, f)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}