code/keyword_extraction/T5-Prompt/model.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "906d18b9",
   "metadata": {},
   "outputs": [],
   "source": [
    "#preprocess\n",
    "import json\n",
    "import os\n",
    "import pandas as pd\n",
    "from sklearn.model_selection import train_test_split"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "75ebd6e5",
   "metadata": {},
   "source": [
    "# data process"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "ae2f469b",
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_csv('../data/all.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "940a72f7",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Unnamed: 0</th>\n",
       "      <th>user_no</th>\n",
       "      <th>key_label</th>\n",
       "      <th>event</th>\n",
       "      <th>event_time</th>\n",
       "      <th>post_id</th>\n",
       "      <th>news_time</th>\n",
       "      <th>title</th>\n",
       "      <th>abstract</th>\n",
       "      <th>content</th>\n",
       "      <th>industry_name_sc</th>\n",
       "      <th>concept_name_sc</th>\n",
       "      <th>label_tag</th>\n",
       "      <th>input</th>\n",
       "      <th>label</th>\n",
       "      <th>sentence1_key</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>929</td>\n",
       "      <td>0</td>\n",
       "      <td>newsView</td>\n",
       "      <td>2022-07-28 23:56:17</td>\n",
       "      <td>13011492</td>\n",
       "      <td>2022-07-28 23:28:38</td>\n",
       "      <td>Market Movers | Solar stocks surge Thursday af...</td>\n",
       "      <td>Solar stocks were surging on Thursday morning ...</td>\n",
       "      <td>&lt;div class=\"ftEditor\"&gt;&lt;p&gt;11:02 AM EDT, 07/28/2...</td>\n",
       "      <td>[\"太阳能\",\"电气设备及零件\"]</td>\n",
       "      <td>[\"法国巴黎银行持仓\",\"昨日强势股\",\"光伏太阳能\",\"氢能源\",\"荷兰全球保险持仓\",\"...</td>\n",
       "      <td>\"太阳能\",\"电气设备及零件\"</td>\n",
       "      <td>title: Market Movers | Solar stocks surge Thur...</td>\n",
       "      <td>industry: [\"太阳能\",\"电气设备及零件\"]</td>\n",
       "      <td>title: Market Movers | Solar stocks surge Thur...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>1659</td>\n",
       "      <td>0</td>\n",
       "      <td>newsView</td>\n",
       "      <td>2022-07-28 12:03:26</td>\n",
       "      <td>12786302</td>\n",
       "      <td>2022-07-22 18:56:21</td>\n",
       "      <td>COSCO Shipping Parent Lifts Stake in Firm to O...</td>\n",
       "      <td>06:54 AM EDT, 07/22/2022 (MT Newswires) -- COS...</td>\n",
       "      <td>&lt;p&gt;06:54 AM EDT, 07/22/2022 (MT Newswires) -- ...</td>\n",
       "      <td>[\"航运及港口\",\"航运港口\"]</td>\n",
       "      <td>[\"一带一路\",\"港口运输股\",\"摘帽\",\"转融券标的\",\"国企改革\",\"BDI指数\",\"证...</td>\n",
       "      <td>\"航运及港口\",\"航运港口\"</td>\n",
       "      <td>title: COSCO Shipping Parent Lifts Stake in Fi...</td>\n",
       "      <td>industry: [\"航运及港口\",\"航运港口\"]</td>\n",
       "      <td>title: COSCO Shipping Parent Lifts Stake in Fi...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>1659</td>\n",
       "      <td>0</td>\n",
       "      <td>newsView</td>\n",
       "      <td>2022-07-28 12:03:59</td>\n",
       "      <td>12622443</td>\n",
       "      <td>2022-07-19 07:40:55</td>\n",
       "      <td>Does COSCO SHIPPING Holdings (HKG:1919) Have A...</td>\n",
       "      <td>The external fund manager backed by Berkshire ...</td>\n",
       "      <td>&lt;p&gt;The external fund manager backed by Berkshi...</td>\n",
       "      <td>[\"航运及港口\"]</td>\n",
       "      <td>[\"一带一路\",\"港口运输股\"]</td>\n",
       "      <td>\"航运及港口\"</td>\n",
       "      <td>title: Does COSCO SHIPPING Holdings (HKG:1919)...</td>\n",
       "      <td>industry: [\"航运及港口\"]</td>\n",
       "      <td>title: Does COSCO SHIPPING Holdings (HKG:1919)...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>1659</td>\n",
       "      <td>0</td>\n",
       "      <td>newsView</td>\n",
       "      <td>2022-07-28 12:03:59</td>\n",
       "      <td>12622443</td>\n",
       "      <td>2022-07-19 07:40:55</td>\n",
       "      <td>Does COSCO SHIPPING Holdings (HKG:1919) Have A...</td>\n",
       "      <td>The external fund manager backed by Berkshire ...</td>\n",
       "      <td>The external fund manager backed by Berkshire ...</td>\n",
       "      <td>[\"航运港口\"]</td>\n",
       "      <td>[\"一带一路\",\"摘帽\",\"转融券标的\",\"国企改革\",\"BDI指数\",\"证金持股\",\"上证...</td>\n",
       "      <td>\"航运港口\"</td>\n",
       "      <td>title: Does COSCO SHIPPING Holdings (HKG:1919)...</td>\n",
       "      <td>industry: [\"航运港口\"]</td>\n",
       "      <td>title: Does COSCO SHIPPING Holdings (HKG:1919)...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>1659</td>\n",
       "      <td>0</td>\n",
       "      <td>newsView</td>\n",
       "      <td>2022-07-28 12:06:26</td>\n",
       "      <td>12252154</td>\n",
       "      <td>2022-07-07 15:22:00</td>\n",
       "      <td>Cosco Shipping Expects 74% Rise in H1 Net Prof...</td>\n",
       "      <td>03:20 AM EDT, 07/07/2022 (MT Newswires) -- Cos...</td>\n",
       "      <td>&lt;p&gt;03:20 AM EDT, 07/07/2022 (MT Newswires) -- ...</td>\n",
       "      <td>[\"航运及港口\"]</td>\n",
       "      <td>[\"一带一路\",\"港口运输股\"]</td>\n",
       "      <td>\"航运及港口\"</td>\n",
       "      <td>title: Cosco Shipping Expects 74% Rise in H1 N...</td>\n",
       "      <td>industry: [\"航运及港口\"]</td>\n",
       "      <td>title: Cosco Shipping Expects 74% Rise in H1 N...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Unnamed: 0  user_no  key_label     event           event_time   post_id  \\\n",
       "0           0      929          0  newsView  2022-07-28 23:56:17  13011492   \n",
       "1           1     1659          0  newsView  2022-07-28 12:03:26  12786302   \n",
       "2           2     1659          0  newsView  2022-07-28 12:03:59  12622443   \n",
       "3           3     1659          0  newsView  2022-07-28 12:03:59  12622443   \n",
       "4           4     1659          0  newsView  2022-07-28 12:06:26  12252154   \n",
       "\n",
       "             news_time                                              title  \\\n",
       "0  2022-07-28 23:28:38  Market Movers | Solar stocks surge Thursday af...   \n",
       "1  2022-07-22 18:56:21  COSCO Shipping Parent Lifts Stake in Firm to O...   \n",
       "2  2022-07-19 07:40:55  Does COSCO SHIPPING Holdings (HKG:1919) Have A...   \n",
       "3  2022-07-19 07:40:55  Does COSCO SHIPPING Holdings (HKG:1919) Have A...   \n",
       "4  2022-07-07 15:22:00  Cosco Shipping Expects 74% Rise in H1 Net Prof...   \n",
       "\n",
       "                                            abstract  \\\n",
       "0  Solar stocks were surging on Thursday morning ...   \n",
       "1  06:54 AM EDT, 07/22/2022 (MT Newswires) -- COS...   \n",
       "2  The external fund manager backed by Berkshire ...   \n",
       "3  The external fund manager backed by Berkshire ...   \n",
       "4  03:20 AM EDT, 07/07/2022 (MT Newswires) -- Cos...   \n",
       "\n",
       "                                             content   industry_name_sc  \\\n",
       "0  <div class=\"ftEditor\"><p>11:02 AM EDT, 07/28/2...  [\"太阳能\",\"电气设备及零件\"]   \n",
       "1  <p>06:54 AM EDT, 07/22/2022 (MT Newswires) -- ...   [\"航运及港口\",\"航运港口\"]   \n",
       "2  <p>The external fund manager backed by Berkshi...          [\"航运及港口\"]   \n",
       "3  The external fund manager backed by Berkshire ...           [\"航运港口\"]   \n",
       "4  <p>03:20 AM EDT, 07/07/2022 (MT Newswires) -- ...          [\"航运及港口\"]   \n",
       "\n",
       "                                     concept_name_sc        label_tag  \\\n",
       "0  [\"法国巴黎银行持仓\",\"昨日强势股\",\"光伏太阳能\",\"氢能源\",\"荷兰全球保险持仓\",\"...  \"太阳能\",\"电气设备及零件\"   \n",
       "1  [\"一带一路\",\"港口运输股\",\"摘帽\",\"转融券标的\",\"国企改革\",\"BDI指数\",\"证...   \"航运及港口\",\"航运港口\"   \n",
       "2                                   [\"一带一路\",\"港口运输股\"]          \"航运及港口\"   \n",
       "3  [\"一带一路\",\"摘帽\",\"转融券标的\",\"国企改革\",\"BDI指数\",\"证金持股\",\"上证...           \"航运港口\"   \n",
       "4                                   [\"一带一路\",\"港口运输股\"]          \"航运及港口\"   \n",
       "\n",
       "                                               input  \\\n",
       "0  title: Market Movers | Solar stocks surge Thur...   \n",
       "1  title: COSCO Shipping Parent Lifts Stake in Fi...   \n",
       "2  title: Does COSCO SHIPPING Holdings (HKG:1919)...   \n",
       "3  title: Does COSCO SHIPPING Holdings (HKG:1919)...   \n",
       "4  title: Cosco Shipping Expects 74% Rise in H1 N...   \n",
       "\n",
       "                         label  \\\n",
       "0  industry: [\"太阳能\",\"电气设备及零件\"]   \n",
       "1   industry: [\"航运及港口\",\"航运港口\"]   \n",
       "2          industry: [\"航运及港口\"]   \n",
       "3           industry: [\"航运港口\"]   \n",
       "4          industry: [\"航运及港口\"]   \n",
       "\n",
       "                                       sentence1_key  \n",
       "0  title: Market Movers | Solar stocks surge Thur...  \n",
       "1  title: COSCO Shipping Parent Lifts Stake in Fi...  \n",
       "2  title: Does COSCO SHIPPING Holdings (HKG:1919)...  \n",
       "3  title: Does COSCO SHIPPING Holdings (HKG:1919)...  \n",
       "4  title: Cosco Shipping Expects 74% Rise in H1 N...  "
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "ddfd538b",
   "metadata": {},
   "outputs": [],
   "source": [
    "#fillna\n",
    "df = df[-df['sentence1_key'].isnull()]\n",
    "df = df[-df['label'].isnull()]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "dd1469c9",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train size (14035, 16), test size(4679, 16)\n"
     ]
    }
   ],
   "source": [
    "train, test = train_test_split(df,test_size=0.25,random_state=0)\n",
    "print (\"train size {}, test size{}\".format(train.shape,test.shape))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "33decac0",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>sentence1_key</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>9354</th>\n",
       "      <td>title: Facebook Stops Renewing US Publisher Co...</td>\n",
       "      <td>industry: [\"互联网内容与信息\"]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7558</th>\n",
       "      <td>title: Xiaomi Announces to Push back MIUI 13 S...</td>\n",
       "      <td>industry: [\"电讯设备\"]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10860</th>\n",
       "      <td>title: Why XPeng shares soared over 10% today?...</td>\n",
       "      <td>industry: [\"汽车制造商\",\"汽车\"]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6687</th>\n",
       "      <td>title: Bilibili Stock Is 8% Down At Session St...</td>\n",
       "      <td>industry: [\"电子游戏与多媒体\"]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4730</th>\n",
       "      <td>title: 为破解缺芯困局 车企集体开启汽车芯片“定制”模式 ,abstract: 过去两...</td>\n",
       "      <td>industry: [\"半导体\",\"汽车制造商\"]</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                           sentence1_key  \\\n",
       "9354   title: Facebook Stops Renewing US Publisher Co...   \n",
       "7558   title: Xiaomi Announces to Push back MIUI 13 S...   \n",
       "10860  title: Why XPeng shares soared over 10% today?...   \n",
       "6687   title: Bilibili Stock Is 8% Down At Session St...   \n",
       "4730   title: 为破解缺芯困局 车企集体开启汽车芯片“定制”模式 ,abstract: 过去两...   \n",
       "\n",
       "                           label  \n",
       "9354      industry: [\"互联网内容与信息\"]  \n",
       "7558          industry: [\"电讯设备\"]  \n",
       "10860   industry: [\"汽车制造商\",\"汽车\"]  \n",
       "6687      industry: [\"电子游戏与多媒体\"]  \n",
       "4730   industry: [\"半导体\",\"汽车制造商\"]  "
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train[['sentence1_key','label']].head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "96296ed8",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "\n",
    "path = '../data/train'\n",
    "\n",
    "# Check whether the specified path exists or not\n",
    "isExist = os.path.exists(path)\n",
    "\n",
    "if not isExist:\n",
    "    os.makedirs(path)\n",
    "    print(\"The new directory is created!\")\n",
    "    \n",
    "train[[\"label\",\"sentence1_key\"]].to_csv('../data/train/train.csv',index=False,encoding='utf-8')\n",
    "test[[\"label\",\"sentence1_key\"]].to_csv('../data/train/test.csv',index=False,encoding='utf-8')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "509f8826",
   "metadata": {},
   "source": [
    "# train - model b"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "3fc9d88e",
   "metadata": {},
   "outputs": [],
   "source": [
    "import boto3\n",
    "import sagemaker\n",
    "import os\n",
    "from sagemaker import get_execution_role\n",
    "\n",
    "sess = sagemaker.Session()\n",
    "role = get_execution_role()\n",
    "\n",
    "prefix='futu-datalab'\n",
    "\n",
    "bucket = sess.default_bucket() \n",
    "boto3.Session().resource(\"s3\").Bucket(bucket).Object(\n",
    "    os.path.join(prefix, \"train/train.csv\")\n",
    ").upload_file(\"../data/train/train.csv\")\n",
    "boto3.Session().resource(\"s3\").Bucket(bucket).Object(\n",
    "    os.path.join(prefix, \"test/test.csv\")\n",
    ").upload_file(\"../data/train/test.csv\")\n",
    "\n",
    "training_input_path = f's3://{sess.default_bucket()}/{prefix}/train/train.csv'\n",
    "test_input_path = f's3://{sess.default_bucket()}/{prefix}/test/test.csv'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "28768b09",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2022-09-14 12:07:29 Starting - Starting the training job...\n",
      "2022-09-14 12:07:56 Starting - Preparing the instances for trainingProfilerReport-1663157249: InProgress\n",
      ".........\n",
      "2022-09-14 12:09:16 Downloading - Downloading input data...\n",
      "2022-09-14 12:09:53 Training - Downloading the training image......."
     ]
    }
   ],
   "source": [
    "from sagemaker.huggingface import HuggingFace\n",
    "from sagemaker.huggingface import TrainingCompilerConfig\n",
    "\n",
    "#speed up use sagemaker compiler https://towardsdatascience.com/speed-up-hugging-face-training-jobs-on-aws-by-up-to-50-with-sagemaker-training-compiler-9ad2ac5b0eb\n",
    "\n",
    "# hyperparameters which are passed to the training job\n",
    "hyperparameters={'reference_column':'sentence1_key',\n",
    "                 'hypothesis_column':'label',\n",
    "                 'train_file':'/opt/ml/input/data/train/train.csv',\n",
    "                 'validation_file':'/opt/ml/input/data/validation/test.csv',\n",
    "                 'test_file':'/opt/ml/input/data/test/test.csv',\n",
    "                 'output_dir':'/opt/ml/model',\n",
    "                 'do_train':True,\n",
    "                 'do_eval':True,\n",
    "                 'max_source_length': 128,\n",
    "                 'max_target_length': 128,\n",
    "                 'model_name_or_path': 'google/mt5-small',\n",
    "                 'learning_rate': 3e-4,\n",
    "                 'num_train_epochs': 10,\n",
    "                 'per_device_train_batch_size': 2,#16\n",
    "                 'gradient_accumulation_steps':2, \n",
    "                 'save_strategy':'steps',\n",
    "                 'evaluation_strategy':'epoch',\n",
    "                 'save_total_limit':1,\n",
    "                 'eval_steps':5000,\n",
    "                 'predict_with_generate':True # customerized accuracy\n",
    "                 }\n",
    "\n",
    "# create the Estimator\n",
    "huggingface_estimator = HuggingFace(\n",
    "        entry_point='run_train.py',\n",
    "        source_dir='./scripts',\n",
    "        instance_type='ml.p3.2xlarge',#'ml.p3dn.24xlarge'\n",
    "        instance_count=1,\n",
    "        role=role,\n",
    "        max_run=24*60*60,\n",
    "        transformers_version='4.6',\n",
    "        pytorch_version='1.7',\n",
    "        py_version='py36',\n",
    "        volume_size=128,\n",
    "        #compiler_config=TrainingCompilerConfig(),\n",
    "        base_job_name='futu-stary-10epoch',\n",
    "        hyperparameters = hyperparameters,\n",
    "#         distribution=distribution\n",
    ")\n",
    "\n",
    "huggingface_estimator.fit({'train':training_input_path,'test':test_input_path,'validation': test_input_path})"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d34a10a9",
   "metadata": {},
   "source": [
    "# deploy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "9546e2de",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sagemaker.huggingface import HuggingFaceModel\n",
    "import sagemaker\n",
    "\n",
    "role = sagemaker.get_execution_role()\n",
    "\n",
    "# create Hugging Face Model Class\n",
    "huggingface_model = HuggingFaceModel(\n",
    "   model_data=\"s3://sagemaker-us-east-1-726335585155/train-modelb-stary-0713-crossbook-2022-07-14-06-27-10-862/output/model.tar.gz\",  # path to your trained sagemaker model\n",
    "   role=role, # iam role with permissions to create an Endpoint\n",
    "   transformers_version=\"4.6\", # transformers version used\n",
    "   pytorch_version=\"1.7\", # pytorch version used\n",
    "   py_version=\"py36\", # python version of the DLC\n",
    ")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "8bd30656",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-------!"
     ]
    }
   ],
   "source": [
    "predictor = huggingface_model.deploy(\n",
    "   initial_instance_count=1,\n",
    "   instance_type=\"ml.g4dn.xlarge\"\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "7074d93f",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 13.3 ms, sys: 0 ns, total: 13.3 ms\n",
      "Wall time: 864 ms\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[{'generated_text': 'Annie said the sentence'}]"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%time\n",
    "# example request, you always need to define \"inputs\"\n",
    "import time\n",
    "\n",
    "\n",
    "data = {\n",
    "   \"inputs\": 'prefix: \"Whatever. Chris has a heir to the pack now.\" She says.,\"Um last time I remembered in the laws it was clearly written that the offspring of the  Alpha can only take over if he is the son of the Alpha and the Luna and clearly after today no one will take this mistake,\" I point to her stomach , center: \"seriously again as you are just a mistress not the mate.\" I say putting a lot of emphasis on the word \\'mistress\\' as if it is the world\\'s most disgusting word. after: \"seriously again as you are just a mistress not the mate.\" I say putting a lot of emphasis on the word \\'mistress\\' as if it is the world\\'s most disgusting word.,\"Well at least he loves me.\" She says desperately. roles: [\\'Skylar\\', \\'Logan\\']'\n",
    "}\n",
    "\n",
    "# request\n",
    "predictor.predict(data)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "d0b18e69",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "df = pd.read_csv('../model_b_data/test.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "7ed8a36c",
   "metadata": {},
   "outputs": [],
   "source": [
    "res = []\n",
    "for i in df['sentence1_key']:\n",
    "    data = {\"inputs\": i}\n",
    "\n",
    "    # request\n",
    "    res.append(predictor.predict(data))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "07489ae7",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "                         precision    recall  f1-score   support\n",
      "\n",
      "Annie said the sentence       0.96      0.99      0.97       211\n",
      "  Ava said the sentence       0.96      0.99      0.97       210\n",
      " Kade said the sentence       0.97      0.93      0.95       120\n",
      " Zach said the sentence       0.98      0.95      0.96       146\n",
      "\n",
      "               accuracy                           0.97       687\n",
      "              macro avg       0.97      0.96      0.96       687\n",
      "           weighted avg       0.97      0.97      0.97       687\n",
      "\n"
     ]
    }
   ],
   "source": [
    "#p f r\n",
    "from sklearn.metrics import classification_report\n",
    "\n",
    "y_pred = [i[0]['generated_text'] for i in res]\n",
    "y_true = df['label']\n",
    " \n",
    "print(classification_report(y_true, y_pred))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "id": "4f660599",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'prefix: \"Whatever. Chris has a heir to the pack now.\" She says.,\"Um last time I remembered in the laws it was clearly written that the offspring of the  Alpha can only take over if he is the son of the Alpha and the Luna and clearly after today no one will take this mistake,\" I point to her stomach , center: \"seriously again as you are just a mistress not the mate.\" I say putting a lot of emphasis on the word \\'mistress\\' as if it is the world\\'s most disgusting word. after: \"seriously again as you are just a mistress not the mate.\" I say putting a lot of emphasis on the word \\'mistress\\' as if it is the world\\'s most disgusting word.,\"Well at least he loves me.\" She says desperately. roles: [\\'Skylar\\', \\'Logan\\']'"
      ]
     },
     "execution_count": 60,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['sentence1_key'][0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "id": "7c2b3621",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Logan said the sentence'"
      ]
     },
     "execution_count": 61,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['label'][0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "id": "1e3c9fb6",
   "metadata": {},
   "outputs": [],
   "source": [
    "x = 0\n",
    "for i in range(len(y_true)):\n",
    "    if y_true[i]==y_pred[i]:\n",
    "        x = x+1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "id": "34bb726b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.838"
      ]
     },
     "execution_count": 59,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x/len(y_true)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "f97d1f0d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'\"Who do you think I am? Of course I want it.\" She whispers and look at her stomach. '"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['sentence1_key'][362]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "63eef0c9",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'\"What do you mean by that?\" She asks me surprised. '"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['sentence1_key'][370]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3bcc3d74",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "conda_amazonei_mxnet_p36",
   "language": "python",
   "name": "conda_amazonei_mxnet_p36"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}