未命名.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "73c46b30",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Better speed can be achieved with apex installed from https://www.github.com/nvidia/apex.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2021-10-18 21:00:33 - INFO - <module> - 54 : Loading faiss with AVX2 support.\n",
      "2021-10-18 21:00:33 - INFO - <module> - 58 : Could not load library with AVX2 support due to:\n",
      "ModuleNotFoundError(\"No module named 'faiss.swigfaiss_avx2'\")\n",
      "2021-10-18 21:00:33 - INFO - <module> - 64 : Loading faiss.\n",
      "2021-10-18 21:00:33 - INFO - <module> - 66 : Successfully loaded faiss.\n",
      "2021-10-18 21:00:34 - INFO - from_pretrained - 125 : loading vocabulary file /data/nfs14/nfs/aisearch/asr/xhsun/CommonModel/chinese-roberta-wwm/vocab.txt\n"
     ]
    }
   ],
   "source": [
    "import nlp_basictasks\n",
    "import os,json\n",
    "import numpy as np\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import random\n",
    "from tqdm.autonotebook import tqdm, trange\n",
    "from torch.utils.data import DataLoader\n",
    "from nlp_basictasks.modules import SBERT\n",
    "from nlp_basictasks.modules.transformers import BertTokenizer,BertModel,BertConfig\n",
    "from nlp_basictasks.readers.sts import InputExample,convert_examples_to_features,getExamples,convert_sentences_to_features\n",
    "from nlp_basictasks.modules.utils import get_optimizer,get_scheduler\n",
    "from nlp_basictasks.Trainer import Trainer\n",
    "from nlp_basictasks.evaluation import stsEvaluator\n",
    "from sentence_transformers import SentenceTransformer,models\n",
    "# model_path1='/data/nfs14/nfs/aisearch/asr/xhsun/bwbd_recall/distill-simcse/'\n",
    "# model_path2=\"/data/nfs14/nfs/aisearch/asr/xhsun/bwbd_recall/distiluse-base-multilingual-cased-v1/\"\n",
    "model_path3='/data/nfs14/nfs/aisearch/asr/xhsun/CommonModel/chinese-roberta-wwm/'\n",
    "# data_folder='/data/nfs14/nfs/aisearch/asr/xhsun/datasets/lcqmc/'\n",
    "# train_file=os.path.join(data_folder,'lcqmc_train.tsv')\n",
    "# dev_file=os.path.join(data_folder,'lcqmc_dev.tsv')\n",
    "#tokenizer=BertTokenizer.from_pretrained(os.path.join(model_path1,'0_Transformer'))\n",
    "tokenizer=BertTokenizer.from_pretrained(model_path3)\n",
    "max_seq_len=64\n",
    "batch_size=128"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "8f3b59a1",
   "metadata": {},
   "outputs": [],
   "source": [
    "train_file='/data/nfs14/nfs/aisearch/asr/xhsun/bwbd_recall/STS-B/cnsd-sts-train.txt'\n",
    "dev_file='/data/nfs14/nfs/aisearch/asr/xhsun/bwbd_recall/STS-B/cnsd-sts-dev.txt'\n",
    "test_file='/data/nfs14/nfs/aisearch/asr/xhsun/bwbd_recall/STS-B/cnsd-sts-test.txt'\n",
    "def read_data(file_path):\n",
    "    sentences=[]\n",
    "    labels=[]\n",
    "    with open(file_path) as f:\n",
    "        lines=f.readlines()\n",
    "    for line in lines:\n",
    "        line_split=line.strip().split('||')\n",
    "        sentences.append([line_split[1],line_split[2]])\n",
    "        labels.append(line_split[3])\n",
    "    return sentences,labels\n",
    "\n",
    "train_sentences,train_labels=read_data(train_file)\n",
    "dev_sentences,dev_labels=read_data(dev_file)\n",
    "test_sentences,test_labels=read_data(test_file)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "a18a39a6",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[['一架飞机要起飞了。', '一架飞机正在起飞。'], ['一个男人在吹一支大笛子。', '一个人在吹长笛。']] ['5', '3']\n",
      "[['一个戴着安全帽的男人在跳舞。', '一个戴着安全帽的男人在跳舞。'], ['一个小孩在骑马。', '孩子在骑马。']] ['5', '4']\n",
      "[['一个女孩在给她的头发做发型。', '一个女孩在梳头。'], ['一群男人在海滩上踢足球。', '一群男孩在海滩上踢足球。']] ['2', '3']\n"
     ]
    }
   ],
   "source": [
    "print(train_sentences[:2],train_labels[:2])\n",
    "print(dev_sentences[:2],dev_labels[:2])\n",
    "print(test_sentences[:2],test_labels[:2])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "a8ea9441",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "5231\n",
      "['一架飞机要起飞了。', '一个男人在吹一支大笛子。', '一个人正把切碎的奶酪撒在比萨饼上。']\n",
      "<InputExample> label: 1, text pairs : 一架飞机要起飞了。; 一架飞机要起飞了。\n"
     ]
    }
   ],
   "source": [
    "train_sentences=[sentence[0] for sentence in train_sentences]#只取一般数据作为训练集\n",
    "print(len(train_sentences))\n",
    "print(train_sentences[:3])\n",
    "train_examples=[InputExample(text_list=[sentence,sentence],label=1) for sentence in train_sentences]\n",
    "train_dataloader=DataLoader(train_examples,shuffle=True,batch_size=batch_size)\n",
    "def smart_batching_collate(batch):\n",
    "    features_of_a,features_of_b,labels=convert_examples_to_features(examples=batch,tokenizer=tokenizer,max_seq_len=max_seq_len)\n",
    "    return features_of_a,features_of_b,labels\n",
    "train_dataloader.collate_fn=smart_batching_collate\n",
    "print(train_examples[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "98070fe4",
   "metadata": {},
   "outputs": [],
   "source": [
    "class SimCSE(nn.Module):\n",
    "    def __init__(self,\n",
    "                 bert_model_path,\n",
    "                 is_sbert_model=True,\n",
    "                temperature=0.05,\n",
    "                is_distilbert=False,\n",
    "                device='cpu'):\n",
    "        super(SimCSE,self).__init__()\n",
    "        if is_sbert_model:\n",
    "            self.encoder=SentenceTransformer(model_name_or_path=bert_model_path,device=device)\n",
    "        else:\n",
    "            word_embedding_model = models.Transformer(bert_model_path, max_seq_length=max_seq_len)\n",
    "            pooling_model = models.Pooling(word_embedding_model.get_word_embedding_dimension())\n",
    "            self.encoder=SentenceTransformer(modules=[word_embedding_model, pooling_model],device=device)\n",
    "        self.temperature=temperature\n",
    "        self.is_distilbert=is_distilbert#蒸馏版本的BERT不支持token_type_ids\n",
    "    def cal_cos_sim(self,embeddings1,embeddings2):\n",
    "        embeddings1_norm=torch.nn.functional.normalize(embeddings1,p=2,dim=1)\n",
    "        embeddings2_norm=torch.nn.functional.normalize(embeddings2,p=2,dim=1)\n",
    "        return torch.mm(embeddings1_norm,embeddings2_norm.transpose(0,1))#(batch_size,batch_size)\n",
    "        \n",
    "    def forward(self,batch_inputs):\n",
    "        '''\n",
    "        为了实现兼容，所有model的batch_inputs最后一个位置必须是labels，即使为None\n",
    "        get token_embeddings,cls_token_embeddings,sentence_embeddings\n",
    "        sentence_embeddings是经过Pooling层后concat的embedding。维度=768*k，其中k取决于pooling的策略\n",
    "        一般来讲，只会取一种pooling策略，要么直接cls要么mean last or mean last2 or mean first and last layer，所以sentence_embeddings的维度也是768\n",
    "        '''\n",
    "        batch1_features,batch2_features,_=batch_inputs\n",
    "        if self.is_distilbert:\n",
    "            del batch1_features['token_type_ids']\n",
    "            del batch2_features['token_type_ids']\n",
    "        batch1_embeddings=self.encoder(batch1_features)['sentence_embedding']\n",
    "        batch2_embeddings=self.encoder(batch2_features)['sentence_embedding']\n",
    "        cos_sim=self.cal_cos_sim(batch1_embeddings,batch2_embeddings)/self.temperature#(batch_size,batch_size)\n",
    "        batch_size=cos_sim.size(0)\n",
    "        assert cos_sim.size()==(batch_size,batch_size)\n",
    "        labels=torch.arange(batch_size).to(cos_sim.device)\n",
    "        return nn.CrossEntropyLoss()(cos_sim,labels)\n",
    "    \n",
    "    def encode(self, sentences,\n",
    "               batch_size: int = 32,\n",
    "               show_progress_bar: bool = None,\n",
    "               output_value: str = 'sentence_embedding',\n",
    "               convert_to_numpy: bool = True,\n",
    "               convert_to_tensor: bool = False,\n",
    "               device: str = None,\n",
    "               normalize_embeddings: bool = False):\n",
    "        '''\n",
    "        传进来的sentences只能是single_batch\n",
    "        '''\n",
    "        return self.encoder.encode(sentences=sentences,\n",
    "                                         batch_size=batch_size,\n",
    "                                         show_progress_bar=show_progress_bar,\n",
    "                                         output_value=output_value,\n",
    "                                         convert_to_numpy=convert_to_numpy,\n",
    "                                         convert_to_tensor=convert_to_tensor,\n",
    "                                         device=device,\n",
    "                                         normalize_embeddings=normalize_embeddings)\n",
    "    \n",
    "    def save(self,output_path):\n",
    "        os.makedirs(output_path,exist_ok=True)\n",
    "        with open(os.path.join(output_path, 'model_param_config.json'), 'w') as fOut:\n",
    "            json.dump(self.get_config_dict(output_path), fOut)\n",
    "        self.encoder.save(output_path)\n",
    "        \n",
    "    def get_config_dict(self,output_path):\n",
    "        '''\n",
    "        一定要有dict，这样才能初始化Model\n",
    "        '''\n",
    "        return {'output_path':output_path,'temperature': self.temperature, 'is_distilbert': self.is_distilbert}\n",
    "    @staticmethod\n",
    "    def load(input_path):\n",
    "        with open(os.path.join(input_path, 'model_param_config.json')) as fIn:\n",
    "            config = json.load(fIn)\n",
    "        return SimCSE(**config)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "859f7d1c",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2021-10-18 21:00:51 - INFO - __init__ - 41 : Load pretrained SentenceTransformer: /data/nfs14/nfs/aisearch/asr/xhsun/bwbd_recall/unsupervisedSTSModel/unSimCSE_STS-B/\n",
      "2021-10-18 21:00:51 - INFO - __init__ - 107 : Load SentenceTransformer from folder: /data/nfs14/nfs/aisearch/asr/xhsun/bwbd_recall/unsupervisedSTSModel/unSimCSE_STS-B/\n"
     ]
    }
   ],
   "source": [
    "device='cpu'\n",
    "#simcse=SimCSE(bert_model_path=model_path3,is_distilbert=False,device=device,is_sbert_model=False)\n",
    "simcse=SimCSE(bert_model_path=\"/data/nfs14/nfs/aisearch/asr/xhsun/bwbd_recall/unsupervisedSTSModel/unSimCSE_STS-B/\",is_distilbert=False,device=device,is_sbert_model=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "5a67b85b",
   "metadata": {},
   "outputs": [],
   "source": [
    "sentences1_list=[sen[0] for sen in train_sentences]\n",
    "sentences2_list=[sen[1] for sen in train_sentences]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "b1840dc5",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6cd06a862b4f4b42a669db98aa35bfb8",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Batches'), FloatProgress(value=0.0, max=164.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ef5e4ff16b43419582af9ff8eeea3def",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Batches'), FloatProgress(value=0.0, max=164.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "sentences1_embeddings=simcse.encode(sentences1_list,convert_to_tensor=True)\n",
    "sentences2_embeddings=simcse.encode(sentences2_list,convert_to_tensor=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "b8fbebb1",
   "metadata": {},
   "outputs": [],
   "source": [
    "def uniform_loss(x, t=2):\n",
    "    return torch.pdist(x, p=2).pow(2).mul(-t).exp().mean().log()\n",
    "def align_loss(x, y, alpha=2):\n",
    "    return (x - y).norm(p=2, dim=1).pow(alpha).mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "5ed24d1b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor(-9.8100)"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "uniform_loss(sentences1_embeddings)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "1ce72a74",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor(-9.7027)"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "uniform_loss(sentences2_embeddings)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "d58e8105",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor(170.5976)"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "align_loss(sentences1_embeddings,sentences2_embeddings)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "4978d5e3",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([5231, 768])"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sentences1_embeddings.size()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "823a124c",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Some weights of the model checkpoint at /data/nfs14/nfs/aisearch/asr/xhsun/CommonModel/chinese-roberta-wwm/ were not used when initializing BertModel: ['cls.predictions.transform.LayerNorm.bias', 'cls.predictions.transform.dense.weight', 'cls.seq_relationship.bias', 'cls.seq_relationship.weight', 'cls.predictions.decoder.weight', 'cls.predictions.transform.LayerNorm.weight', 'cls.predictions.bias', 'cls.predictions.transform.dense.bias']\n",
      "- This IS expected if you are initializing BertModel from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
      "- This IS NOT expected if you are initializing BertModel from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n"
     ]
    }
   ],
   "source": [
    "device='cpu'\n",
    "#simcse=SimCSE(bert_model_path=model_path3,is_distilbert=False,device=device,is_sbert_model=False)\n",
    "bert_model=SimCSE(bert_model_path=model_path3,is_distilbert=False,device=device,is_sbert_model=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "e5512049",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d9d335566b4843aaa53396bfd725a101",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Batches'), FloatProgress(value=0.0, max=164.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "84082e64477f4d7997e8a4a826e7f173",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Batches'), FloatProgress(value=0.0, max=164.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "sentences1_embeddings=bert_model.encode(sentences1_list,convert_to_tensor=True)\n",
    "sentences2_embeddings=bert_model.encode(sentences2_list,convert_to_tensor=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "ab0dce48",
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'uniform_loss' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-2-97dc59b4d479>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0muniform_loss\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msentences1_embeddings\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      2\u001b[0m \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0muniform_loss\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msentences2_embeddings\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      3\u001b[0m \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0malign_loss\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msentences1_embeddings\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0msentences2_embeddings\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mNameError\u001b[0m: name 'uniform_loss' is not defined"
     ]
    }
   ],
   "source": [
    "print(uniform_loss(sentences1_embeddings))\n",
    "print(uniform_loss(sentences2_embeddings))\n",
    "print(align_loss(sentences1_embeddings,sentences2_embeddings))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "72d6128b",
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5b35ab4c",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "ec7142bd",
   "metadata": {},
   "outputs": [],
   "source": [
    "def softmax(matrix,temperature=1):\n",
    "    numerator=np.exp(np.array(matrix)/temperature)\n",
    "    denominator=(numerator).sum()\n",
    "    return numerator/denominator"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "581b5fa6",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0.38832577, 0.35137169, 0.26030255])"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "softmax([0.6,0.5,0.2],temperature=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "06f11557",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0.72139918, 0.26538793, 0.01321289])"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "softmax([0.6,0.5,0.2],temperature=0.1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "c84f91ee",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([8.80536902e-01, 1.19167711e-01, 2.95387223e-04])"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "softmax([0.6,0.5,0.2],temperature=0.05)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "e8f3f02f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0.33888664, 0.33551466, 0.3255987 ])"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "softmax([0.6,0.5,0.2],temperature=10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "e3eca672",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0.33388887, 0.33355515, 0.33255598])"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "softmax([0.6,0.5,0.2],temperature=100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f7f41a2d",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f165fef9",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}