test_model.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "3f479647",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "c:\\users\\imagry\\desktop\\linux\\converter\\lib\\site-packages\\transformers\\models\\t5\\tokenization_t5.py:169: FutureWarning: This tokenizer was incorrectly instantiated with a model max length of 512 which will be corrected in Transformers v5.\n",
      "For now, this behavior is kept to avoid breaking backwards compatibility when padding/encoding with `truncation is True`.\n",
      "- Be aware that you SHOULD NOT rely on t5-small automatically truncating your input to 512 when padding/encoding.\n",
      "- If you want to encode/pad to sequences longer than 512 you can either instantiate this tokenizer with `model_max_length` or pass `max_length` when encoding/padding.\n",
      "- To avoid this warning, please instantiate this tokenizer with `model_max_length` set to your preferred value.\n",
      "  FutureWarning,\n"
     ]
    }
   ],
   "source": [
    "import stgen\n",
    "import torch as th\n",
    "import torchsummary\n",
    "from stgen import utils\n",
    "from torch.utils.data import DataLoader, TensorDataset\n",
    "import numpy as np\n",
    "\n",
    "model_name='t5-small'\n",
    "t5_tokenizer=stgen.tokenizer(model_name)\n",
    "text=['isthis true' ,'what is happening']\n",
    "x,y=t5_tokenizer.get_tokens_masks(text)\n",
    "device= th.device('cuda')\n",
    "batch_size=len(text)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "43557471",
   "metadata": {},
   "outputs": [],
   "source": [
    "model=stgen.stgen(model_name,512,batch_size)\n",
    "hidden=model.initHidden().to(device)\n",
    "model=model.to(device)\n",
    "output,hidden = model(x,y, hidden)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "43cec674",
   "metadata": {},
   "source": [
    "## Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "c0de00bc",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "from stgen import utils\n",
    "from torch.utils.data import DataLoader, TensorDataset\n",
    "batch_size=64"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "d63484b3",
   "metadata": {},
   "outputs": [],
   "source": [
    "df=pd.read_csv('./data/startups.csv')#[0:1000]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "a55e883f",
   "metadata": {},
   "outputs": [],
   "source": [
    "train_set,test_set=utils.tts(df,0.15)\n",
    "train_set = list(train_set.to_records(index=False))\n",
    "test_set = list(test_set.to_records(index=False))\n",
    "train_set=[S for S in train_set if (str(S[0]) !='nan' and str(S[2]) !='nan')]\n",
    "test_set=[S for S in test_set if (str(S[0]) !='nan' and str(S[2]) !='nan')]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "baa2678a",
   "metadata": {},
   "outputs": [],
   "source": [
    "Train_Y      = np.array([utils.lineToTensor(utils.pad_word(sample[0])).squeeze().cpu().numpy() for sample in train_set])\n",
    "Train_Tokens = np.array([t5_tokenizer.get_tokens_masks([sample[2]])[0].squeeze().cpu().numpy() for sample in train_set ])\n",
    "Train_Masks  = np.array([t5_tokenizer.get_tokens_masks([sample[2]])[1].squeeze().cpu().numpy() for sample in train_set ])\n",
    "\n",
    "Test_Y       = np.array([utils.lineToTensor(utils.pad_word(sample[0])).squeeze().cpu().numpy() for sample in test_set  ])\n",
    "Test_Tokens  = np.array([t5_tokenizer.get_tokens_masks([sample[2]])[0].squeeze().cpu().numpy() for sample in test_set  ])\n",
    "Test_Masks   = np.array([t5_tokenizer.get_tokens_masks([sample[2]])[1].squeeze().cpu().numpy() for sample in test_set  ])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "80ff2b48",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(31269, 20, 59)\n",
      "(31269, 256)\n",
      "(31269, 256)\n",
      "(5461, 20, 59)\n",
      "(5461, 256)\n",
      "(5461, 256)\n"
     ]
    }
   ],
   "source": [
    "print(Train_Y.shape)\n",
    "print(Train_Tokens.shape)\n",
    "print(Train_Masks.shape)\n",
    "print(Test_Y.shape)\n",
    "print(Test_Tokens.shape)\n",
    "print(Test_Masks.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "5c8b88ab",
   "metadata": {},
   "outputs": [],
   "source": [
    "train_ds = TensorDataset(th.from_numpy(Train_Tokens),th.from_numpy(Train_Masks), th.from_numpy(Train_Y))\n",
    "test_ds  = TensorDataset(th.from_numpy(Test_Tokens),th.from_numpy(Test_Masks), th.from_numpy(Test_Y))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "e97de831",
   "metadata": {},
   "outputs": [],
   "source": [
    "train_dl = DataLoader(train_ds, shuffle=True, batch_size=batch_size, drop_last=False)\n",
    "test_dl  = DataLoader(test_ds, shuffle=True, batch_size=batch_size, drop_last=False)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "553a8019",
   "metadata": {},
   "source": [
    "# Training  :) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "44e2fc89",
   "metadata": {},
   "outputs": [],
   "source": [
    "from torch import nn\n",
    "from torch.autograd import Variable"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6f8a8d21",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Some weights of the model checkpoint at t5-small were not used when initializing T5EncoderModel: ['decoder.block.4.layer.0.SelfAttention.k.weight', 'decoder.block.1.layer.0.SelfAttention.k.weight', 'decoder.block.0.layer.1.EncDecAttention.relative_attention_bias.weight', 'decoder.block.0.layer.2.DenseReluDense.wi.weight', 'decoder.block.2.layer.0.SelfAttention.v.weight', 'decoder.block.4.layer.1.EncDecAttention.o.weight', 'decoder.final_layer_norm.weight', 'decoder.block.5.layer.0.SelfAttention.k.weight', 'decoder.block.1.layer.2.layer_norm.weight', 'decoder.block.1.layer.0.layer_norm.weight', 'decoder.block.3.layer.0.layer_norm.weight', 'decoder.block.2.layer.0.layer_norm.weight', 'decoder.block.0.layer.1.EncDecAttention.q.weight', 'decoder.block.5.layer.2.DenseReluDense.wi.weight', 'decoder.block.2.layer.0.SelfAttention.o.weight', 'decoder.block.5.layer.2.layer_norm.weight', 'decoder.block.1.layer.0.SelfAttention.v.weight', 'decoder.block.0.layer.0.SelfAttention.relative_attention_bias.weight', 'decoder.block.4.layer.0.SelfAttention.v.weight', 'decoder.block.0.layer.1.EncDecAttention.k.weight', 'decoder.block.5.layer.2.DenseReluDense.wo.weight', 'decoder.block.3.layer.2.layer_norm.weight', 'decoder.block.2.layer.1.EncDecAttention.k.weight', 'decoder.block.0.layer.2.DenseReluDense.wo.weight', 'decoder.block.1.layer.1.layer_norm.weight', 'decoder.block.4.layer.1.EncDecAttention.k.weight', 'decoder.block.3.layer.1.EncDecAttention.v.weight', 'decoder.block.1.layer.1.EncDecAttention.v.weight', 'decoder.block.1.layer.0.SelfAttention.o.weight', 'decoder.block.1.layer.2.DenseReluDense.wi.weight', 'decoder.block.1.layer.1.EncDecAttention.q.weight', 'decoder.block.2.layer.2.DenseReluDense.wo.weight', 'decoder.block.5.layer.1.EncDecAttention.k.weight', 'decoder.block.3.layer.2.DenseReluDense.wo.weight', 'decoder.block.2.layer.1.layer_norm.weight', 'decoder.block.2.layer.1.EncDecAttention.v.weight', 'decoder.block.3.layer.1.EncDecAttention.o.weight', 'decoder.block.0.layer.0.SelfAttention.k.weight', 'decoder.block.0.layer.0.layer_norm.weight', 'decoder.block.2.layer.2.layer_norm.weight', 'decoder.block.3.layer.0.SelfAttention.o.weight', 'decoder.block.0.layer.0.SelfAttention.o.weight', 'decoder.block.4.layer.2.DenseReluDense.wi.weight', 'decoder.block.2.layer.0.SelfAttention.k.weight', 'decoder.block.0.layer.0.SelfAttention.q.weight', 'decoder.block.5.layer.1.EncDecAttention.o.weight', 'decoder.block.4.layer.1.EncDecAttention.q.weight', 'decoder.block.5.layer.0.SelfAttention.q.weight', 'decoder.block.3.layer.0.SelfAttention.k.weight', 'decoder.block.2.layer.2.DenseReluDense.wi.weight', 'decoder.block.4.layer.2.layer_norm.weight', 'decoder.block.5.layer.1.EncDecAttention.q.weight', 'decoder.block.1.layer.1.EncDecAttention.k.weight', 'decoder.block.3.layer.1.EncDecAttention.k.weight', 'decoder.block.2.layer.0.SelfAttention.q.weight', 'decoder.block.4.layer.1.layer_norm.weight', 'decoder.block.4.layer.0.SelfAttention.o.weight', 'decoder.block.3.layer.2.DenseReluDense.wi.weight', 'decoder.block.1.layer.1.EncDecAttention.o.weight', 'decoder.block.0.layer.1.EncDecAttention.o.weight', 'decoder.block.5.layer.1.layer_norm.weight', 'decoder.block.2.layer.1.EncDecAttention.q.weight', 'decoder.block.0.layer.1.EncDecAttention.v.weight', 'decoder.block.0.layer.0.SelfAttention.v.weight', 'decoder.block.5.layer.0.SelfAttention.v.weight', 'decoder.block.3.layer.1.EncDecAttention.q.weight', 'decoder.block.3.layer.0.SelfAttention.v.weight', 'decoder.block.0.layer.2.layer_norm.weight', 'decoder.block.5.layer.0.SelfAttention.o.weight', 'decoder.block.3.layer.1.layer_norm.weight', 'decoder.block.4.layer.0.layer_norm.weight', 'decoder.block.0.layer.1.layer_norm.weight', 'decoder.block.5.layer.1.EncDecAttention.v.weight', 'decoder.block.4.layer.2.DenseReluDense.wo.weight', 'decoder.block.5.layer.0.layer_norm.weight', 'decoder.block.4.layer.0.SelfAttention.q.weight', 'decoder.block.4.layer.1.EncDecAttention.v.weight', 'decoder.block.2.layer.1.EncDecAttention.o.weight', 'decoder.block.1.layer.2.DenseReluDense.wo.weight', 'decoder.block.3.layer.0.SelfAttention.q.weight', 'decoder.block.1.layer.0.SelfAttention.q.weight']\n",
      "- This IS expected if you are initializing T5EncoderModel from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
      "- This IS NOT expected if you are initializing T5EncoderModel from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
      "Some weights of T5EncoderModel were not initialized from the model checkpoint at t5-small and are newly initialized: ['encoder.embed_tokens.weight']\n",
      "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(0.0858, device='cuda:0', grad_fn=<BinaryCrossEntropyWithLogitsBackward>)\n",
      "tensor(0.0713, device='cuda:0', grad_fn=<BinaryCrossEntropyWithLogitsBackward>)\n",
      "tensor(0.0632, device='cuda:0', grad_fn=<BinaryCrossEntropyWithLogitsBackward>)\n",
      "tensor(0.0639, device='cuda:0', grad_fn=<BinaryCrossEntropyWithLogitsBackward>)\n",
      "tensor(0.0630, device='cuda:0', grad_fn=<BinaryCrossEntropyWithLogitsBackward>)\n",
      "tensor(0.0611, device='cuda:0', grad_fn=<BinaryCrossEntropyWithLogitsBackward>)\n",
      "tensor(0.0622, device='cuda:0', grad_fn=<BinaryCrossEntropyWithLogitsBackward>)\n",
      "tensor(0.0584, device='cuda:0', grad_fn=<BinaryCrossEntropyWithLogitsBackward>)\n",
      "tensor(0.0601, device='cuda:0', grad_fn=<BinaryCrossEntropyWithLogitsBackward>)\n",
      "tensor(0.0586, device='cuda:0', grad_fn=<BinaryCrossEntropyWithLogitsBackward>)\n"
     ]
    }
   ],
   "source": [
    "model=stgen.stgen(model_name,512,batch_size)\n",
    "#output,hidden = model(x,y, hidden)\n",
    "\n",
    "model=model.to(device)\n",
    "model = model.to(device)\n",
    "\n",
    "criterion = nn.BCEWithLogitsLoss()\n",
    "optimizer = th.optim.Adam(model.parameters(), lr = 3e-4)\n",
    "epochs = 6\n",
    "losses = []\n",
    "th.autograd.set_detect_anomaly(True)\n",
    "\n",
    "\n",
    "for e in range(epochs):\n",
    "\n",
    "    for batch_idx, batch in enumerate(train_dl):\n",
    "\n",
    "        tokens = batch[0].to(device)\n",
    "        masks  = batch[1].to(device)\n",
    "        targets= batch[2].to(device)\n",
    "        \n",
    "        \n",
    "\n",
    "        optimizer.zero_grad()\n",
    "        hidden=model.initHidden().to(device)\n",
    "        \n",
    "        with th.set_grad_enabled(True):\n",
    "            output=[]\n",
    "            \n",
    "            for i in range(20):\n",
    "                out, hidden = model(tokens,masks,hidden)\n",
    "                output.append(out)\n",
    "            output=th.cat((output),1)\n",
    "            output=th.reshape(output, (batch_size,20, -1))\n",
    "            #print(output.shape,targets.shape)\n",
    "            loss = criterion(output, targets)\n",
    "            \n",
    "            if batch_idx%5==0:\n",
    "                print(loss)\n",
    "            \n",
    "            \n",
    "            loss.backward(retain_graph=True)\n",
    "            optimizer.step()\n",
    "            \n",
    "    losses.append(loss.item())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "33095f94",
   "metadata": {},
   "outputs": [],
   "source": [
    "th.empty((0))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e8036605",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}