huggingface_mbart.ipynb

{
 "cells": [
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Finetune with MBart "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "vscode": {
     "languageId": "python"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fri Jan 13 17:10:07 2023       \n",
      "+-----------------------------------------------------------------------------+\n",
      "| NVIDIA-SMI 470.161.03   Driver Version: 470.161.03   CUDA Version: 11.4     |\n",
      "|-------------------------------+----------------------+----------------------+\n",
      "| GPU  Name        Persistence-M| Bus-Id        Disp.A | Volatile Uncorr. ECC |\n",
      "| Fan  Temp  Perf  Pwr:Usage/Cap|         Memory-Usage | GPU-Util  Compute M. |\n",
      "|                               |                      |               MIG M. |\n",
      "|===============================+======================+======================|\n",
      "|   0  NVIDIA RTX A6000    Off  | 00000000:4F:00.0 Off |                    0 |\n",
      "| 30%   26C    P8    22W / 300W |      0MiB / 45631MiB |      0%      Default |\n",
      "|                               |                      |                  N/A |\n",
      "+-------------------------------+----------------------+----------------------+\n",
      "|   1  NVIDIA RTX A6000    Off  | 00000000:52:00.0 Off |                    0 |\n",
      "| 30%   27C    P8    16W / 300W |      0MiB / 45631MiB |      0%      Default |\n",
      "|                               |                      |                  N/A |\n",
      "+-------------------------------+----------------------+----------------------+\n",
      "|   2  NVIDIA RTX A6000    Off  | 00000000:CE:00.0 Off |                    0 |\n",
      "| 34%   63C    P2   205W / 300W |   4485MiB / 45631MiB |     54%      Default |\n",
      "|                               |                      |                  N/A |\n",
      "+-------------------------------+----------------------+----------------------+\n",
      "|   3  NVIDIA RTX A6000    Off  | 00000000:D1:00.0 Off |                    0 |\n",
      "| 40%   69C    P2   235W / 300W |  10059MiB / 45631MiB |     67%      Default |\n",
      "|                               |                      |                  N/A |\n",
      "+-------------------------------+----------------------+----------------------+\n",
      "                                                                               \n",
      "+-----------------------------------------------------------------------------+\n",
      "| Processes:                                                                  |\n",
      "|  GPU   GI   CI        PID   Type   Process name                  GPU Memory |\n",
      "|        ID   ID                                                   Usage      |\n",
      "|=============================================================================|\n",
      "|    2   N/A  N/A    213806      C   python3                          4483MiB |\n",
      "|    3   N/A  N/A    213340      C   python3                         10057MiB |\n",
      "+-----------------------------------------------------------------------------+\n"
     ]
    }
   ],
   "source": [
    "!nvidia-smi"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "vscode": {
     "languageId": "python"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "env: CUDA_VISIBLE_DEVICES=0\n",
      "env: TOKENIZERS_PARALLELISM=false\n"
     ]
    }
   ],
   "source": [
    "%env CUDA_VISIBLE_DEVICES=0\n",
    "%env TOKENIZERS_PARALLELISM=false"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "vscode": {
     "languageId": "python"
    }
   },
   "outputs": [],
   "source": [
    "from datasets import load_dataset, concatenate_datasets\n",
    "import transformers\n",
    "from transformers import MBartModel, MBartTokenizer, MBartConfig, pipeline, Trainer, TrainingArguments, MBartForConditionalGeneration, DataCollatorForSeq2Seq, Seq2SeqTrainingArguments, Seq2SeqTrainer\n",
    "import evaluate\n",
    "import numpy as np\n",
    "import torch\n",
    "from torch import tensor \n",
    "import os"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Dataload using Huggingface "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "vscode": {
     "languageId": "python"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using custom data configuration default-d1314a26020b1cab\n",
      "Found cached dataset json (/home/sumire/.cache/huggingface/datasets/json/default-d1314a26020b1cab/0.0.0/e6070c77f18f01a5ad4551a8b7edfba20b8438b7cad4d94e6ad9378022ce4aab)\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "96c071b9d83b4c328ee56d5b7de613d5",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/3 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "DatasetDict({\n",
       "    train: Dataset({\n",
       "        features: ['id', 'tag', 'title', 'original_language', 'conversation'],\n",
       "        num_rows: 670\n",
       "    })\n",
       "    validation: Dataset({\n",
       "        features: ['id', 'tag', 'title', 'original_language', 'conversation'],\n",
       "        num_rows: 69\n",
       "    })\n",
       "    test: Dataset({\n",
       "        features: ['id', 'tag', 'title', 'original_language', 'conversation'],\n",
       "        num_rows: 69\n",
       "    })\n",
       "})"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "file_path = \"/home/sumire/discourse_context_mt/data/BSD-master/\"\n",
    "data_files = {\"train\": f\"{file_path}train.json\", \"validation\": f\"{file_path}dev.json\", \"test\": f\"{file_path}test.json\"}\n",
    "dataset = load_dataset(\"json\", data_files=data_files)\n",
    "dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "vscode": {
     "languageId": "python"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['Hi this is the systems development department of Company K.', 'My name is Takaichi from Company H.', 'Thank you as always.', 'Thank you as always as well.', 'Is Inada-san there?'] ['はい、K社システム開発部です。', 'H社の高市と申します。', 'いつもお世話になっております。', 'こちらこそ、お世話になっております。', '稲田さんはいらっしゃいますか？']\n"
     ]
    }
   ],
   "source": [
    "# define train inputs and targets\n",
    "after_context = \"</s>\"\n",
    "context_size = 1\n",
    "\n",
    "inputs = [sent['en_sentence'] for doc in dataset[\"train\"][\"conversation\"] for sent in doc]\n",
    "targets = [sent['ja_sentence'] for doc in dataset[\"train\"][\"conversation\"] for sent in doc]\n",
    "\n",
    "print (inputs[:5], targets[:5])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "vscode": {
     "languageId": "python"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "20000\n",
      "['<>Hi this is the systems development department of Company K.', 'Hi this is the systems development department of Company K.</s>My name is Takaichi from Company H.', 'My name is Takaichi from Company H.</s>Thank you as always.', 'Thank you as always.</s>Thank you as always as well.', 'Thank you as always as well.</s>Is Inada-san there?']\n"
     ]
    }
   ],
   "source": [
    "# Context-aware inputs\n",
    "# Input window size = 1\n",
    "context_size = 1\n",
    "new_inputs = []\n",
    "\n",
    "for idx, input in enumerate(inputs):\n",
    "    if idx-context_size < 0:\n",
    "        #print (idx)\n",
    "        new_input = f\"<>{input}\"\n",
    "        #print (new_input)\n",
    "        new_inputs.append(new_input)\n",
    "\n",
    "    elif 0 <= idx-context_size:\n",
    "        #print (idx)\n",
    "        new_input = f\"{inputs[idx-1]}</s>{input}\"\n",
    "        #print (new_input)\n",
    "        new_inputs.append(new_input)\n",
    "\n",
    "#print (new_inputs[0], new_inputs[-1])\n",
    "print (len(new_inputs))\n",
    "print (new_inputs[:5])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Preprocess Using Huggingface"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "vscode": {
     "languageId": "python"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "The tokenizer class you load from this checkpoint is not the same type as the class this function is called from. It may result in unexpected tokenization. \n",
      "The tokenizer class you load from this checkpoint is 'MBart50Tokenizer'. \n",
      "The class this function is called from is 'MBartTokenizer'.\n",
      "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
     ]
    }
   ],
   "source": [
    "\n",
    "# facebook/mbart-large-50-many-to-many-mmt\n",
    "model_checkpoint = \"facebook/mbart-large-50-many-to-many-mmt\"\n",
    "configuration = MBartConfig()\n",
    "tokenizer = MBartTokenizer.from_pretrained(model_checkpoint, src_lang=\"en_XX\", tgt_lang=\"ja_XX\")\n",
    "model = MBartForConditionalGeneration(configuration).from_pretrained(model_checkpoint)\n",
    "\n",
    "max_length = 128\n",
    "\n",
    "def preprocess_function(data): # data should be splitted into train / dev / test internally\n",
    "    inputs = [sent['en_sentence'] for doc in data[\"conversation\"] for sent in doc]\n",
    "    targets = [sent['ja_sentence'] for doc in data[\"conversation\"] for sent in doc]\n",
    "    model_inputs = tokenizer(\n",
    "        inputs, text_target=targets, max_length=max_length, truncation=True\n",
    "    )\n",
    "    return model_inputs\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "vscode": {
     "languageId": "python"
    }
   },
   "outputs": [
    {
     "ename": "ModuleNotFoundError",
     "evalue": "No module named 'NotebookApp'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mModuleNotFoundError\u001b[0m                       Traceback (most recent call last)",
      "\u001b[1;32m/home/sumire/discourse_context_mt/huggingface_mbart.ipynb Cell 11\u001b[0m in \u001b[0;36m<cell line: 1>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> <a href='vscode-notebook-cell://ssh-remote%2Bposeidon/home/sumire/discourse_context_mt/huggingface_mbart.ipynb#X42sdnNjb2RlLXJlbW90ZQ%3D%3D?line=0'>1</a>\u001b[0m \u001b[39mimport\u001b[39;00m \u001b[39mNotebookApp\u001b[39;00m\n\u001b[1;32m      <a href='vscode-notebook-cell://ssh-remote%2Bposeidon/home/sumire/discourse_context_mt/huggingface_mbart.ipynb#X42sdnNjb2RlLXJlbW90ZQ%3D%3D?line=1'>2</a>\u001b[0m NotebookApp\u001b[39m.\u001b[39miopub_data_rate_limit\u001b[39m=\u001b[39m\u001b[39m5000000\u001b[39m\n",
      "\u001b[0;31mModuleNotFoundError\u001b[0m: No module named 'NotebookApp'"
     ]
    }
   ],
   "source": [
    "import NotebookApp\n",
    "NotebookApp.iopub_data_rate_limit=5000000"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "vscode": {
     "languageId": "python"
    }
   },
   "outputs": [
    {
     "ename": "KeyError",
     "evalue": "'conversation'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyError\u001b[0m                                  Traceback (most recent call last)",
      "\u001b[1;32m/home/sumire/discourse_context_mt/huggingface_mbart.ipynb Cell 12\u001b[0m in \u001b[0;36m<cell line: 1>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> <a href='vscode-notebook-cell://ssh-remote%2Bposeidon/home/sumire/discourse_context_mt/huggingface_mbart.ipynb#X41sdnNjb2RlLXJlbW90ZQ%3D%3D?line=0'>1</a>\u001b[0m model_inputs \u001b[39m=\u001b[39m preprocess_function(dataset)\n\u001b[1;32m      <a href='vscode-notebook-cell://ssh-remote%2Bposeidon/home/sumire/discourse_context_mt/huggingface_mbart.ipynb#X41sdnNjb2RlLXJlbW90ZQ%3D%3D?line=1'>2</a>\u001b[0m \u001b[39mprint\u001b[39m ((model_inputs))\n",
      "\u001b[1;32m/home/sumire/discourse_context_mt/huggingface_mbart.ipynb Cell 12\u001b[0m in \u001b[0;36mpreprocess_function\u001b[0;34m(data)\u001b[0m\n\u001b[1;32m      <a href='vscode-notebook-cell://ssh-remote%2Bposeidon/home/sumire/discourse_context_mt/huggingface_mbart.ipynb#X41sdnNjb2RlLXJlbW90ZQ%3D%3D?line=8'>9</a>\u001b[0m \u001b[39mdef\u001b[39;00m \u001b[39mpreprocess_function\u001b[39m(data): \u001b[39m# data should be splitted into train / dev / test internally\u001b[39;00m\n\u001b[0;32m---> <a href='vscode-notebook-cell://ssh-remote%2Bposeidon/home/sumire/discourse_context_mt/huggingface_mbart.ipynb#X41sdnNjb2RlLXJlbW90ZQ%3D%3D?line=9'>10</a>\u001b[0m     inputs \u001b[39m=\u001b[39m [sent[\u001b[39m'\u001b[39m\u001b[39men_sentence\u001b[39m\u001b[39m'\u001b[39m] \u001b[39mfor\u001b[39;00m doc \u001b[39min\u001b[39;00m data[\u001b[39m\"\u001b[39;49m\u001b[39mconversation\u001b[39;49m\u001b[39m\"\u001b[39;49m] \u001b[39mfor\u001b[39;00m sent \u001b[39min\u001b[39;00m doc]\n\u001b[1;32m     <a href='vscode-notebook-cell://ssh-remote%2Bposeidon/home/sumire/discourse_context_mt/huggingface_mbart.ipynb#X41sdnNjb2RlLXJlbW90ZQ%3D%3D?line=10'>11</a>\u001b[0m     targets \u001b[39m=\u001b[39m [sent[\u001b[39m'\u001b[39m\u001b[39mja_sentence\u001b[39m\u001b[39m'\u001b[39m] \u001b[39mfor\u001b[39;00m doc \u001b[39min\u001b[39;00m data[\u001b[39m\"\u001b[39m\u001b[39mconversation\u001b[39m\u001b[39m\"\u001b[39m] \u001b[39mfor\u001b[39;00m sent \u001b[39min\u001b[39;00m doc]\n\u001b[1;32m     <a href='vscode-notebook-cell://ssh-remote%2Bposeidon/home/sumire/discourse_context_mt/huggingface_mbart.ipynb#X41sdnNjb2RlLXJlbW90ZQ%3D%3D?line=11'>12</a>\u001b[0m     model_inputs \u001b[39m=\u001b[39m tokenizer(\n\u001b[1;32m     <a href='vscode-notebook-cell://ssh-remote%2Bposeidon/home/sumire/discourse_context_mt/huggingface_mbart.ipynb#X41sdnNjb2RlLXJlbW90ZQ%3D%3D?line=12'>13</a>\u001b[0m         inputs, text_target\u001b[39m=\u001b[39mtargets, max_length\u001b[39m=\u001b[39mmax_length, truncation\u001b[39m=\u001b[39m\u001b[39mTrue\u001b[39;00m\n\u001b[1;32m     <a href='vscode-notebook-cell://ssh-remote%2Bposeidon/home/sumire/discourse_context_mt/huggingface_mbart.ipynb#X41sdnNjb2RlLXJlbW90ZQ%3D%3D?line=13'>14</a>\u001b[0m     )\n",
      "File \u001b[0;32m~/miniconda3/envs/mt2022_server/lib/python3.8/site-packages/datasets/dataset_dict.py:57\u001b[0m, in \u001b[0;36mDatasetDict.__getitem__\u001b[0;34m(self, k)\u001b[0m\n\u001b[1;32m     55\u001b[0m \u001b[39mdef\u001b[39;00m \u001b[39m__getitem__\u001b[39m(\u001b[39mself\u001b[39m, k) \u001b[39m-\u001b[39m\u001b[39m>\u001b[39m Dataset:\n\u001b[1;32m     56\u001b[0m     \u001b[39mif\u001b[39;00m \u001b[39misinstance\u001b[39m(k, (\u001b[39mstr\u001b[39m, NamedSplit)) \u001b[39mor\u001b[39;00m \u001b[39mlen\u001b[39m(\u001b[39mself\u001b[39m) \u001b[39m==\u001b[39m \u001b[39m0\u001b[39m:\n\u001b[0;32m---> 57\u001b[0m         \u001b[39mreturn\u001b[39;00m \u001b[39msuper\u001b[39;49m()\u001b[39m.\u001b[39;49m\u001b[39m__getitem__\u001b[39;49m(k)\n\u001b[1;32m     58\u001b[0m     \u001b[39melse\u001b[39;00m:\n\u001b[1;32m     59\u001b[0m         available_suggested_splits \u001b[39m=\u001b[39m [\n\u001b[1;32m     60\u001b[0m             split \u001b[39mfor\u001b[39;00m split \u001b[39min\u001b[39;00m (Split\u001b[39m.\u001b[39mTRAIN, Split\u001b[39m.\u001b[39mTEST, Split\u001b[39m.\u001b[39mVALIDATION) \u001b[39mif\u001b[39;00m split \u001b[39min\u001b[39;00m \u001b[39mself\u001b[39m\n\u001b[1;32m     61\u001b[0m         ]\n",
      "\u001b[0;31mKeyError\u001b[0m: 'conversation'"
     ]
    }
   ],
   "source": [
    "model_inputs = preprocess_function(dataset)\n",
    "print ((model_inputs))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "vscode": {
     "languageId": "python"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'\\nmodel_checkpoint = \"facebook/mbart-large-cc25\"\\nconfiguration = MBartConfig()\\ntokenizer = MBartTokenizer.from_pretrained(model_checkpoint, src_lang=\"en_XX\", tgt_lang=\"ja_XX\")\\nmodel = MBartForConditionalGeneration(configuration).from_pretrained(model_checkpoint)\\n\\nmax_length = 128\\n\\ndef preprocess_function(data): # data should be splitted into train / dev / test internally\\n    inputs = [sent[\\'en_sentence\\'] for doc in data[\"conversation\"] for sent in doc]\\n    targets = [sent[\\'ja_sentence\\'] for doc in data[\"conversation\"] for sent in doc]\\n    model_inputs = tokenizer(\\n        inputs, text_target=targets, max_length=max_length, truncation=True\\n    )\\n    return model_inputs\\n'"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# mbart-large-cc25\n",
    "\"\"\"\n",
    "model_checkpoint = \"facebook/mbart-large-cc25\"\n",
    "configuration = MBartConfig()\n",
    "tokenizer = MBartTokenizer.from_pretrained(model_checkpoint, src_lang=\"en_XX\", tgt_lang=\"ja_XX\")\n",
    "model = MBartForConditionalGeneration(configuration).from_pretrained(model_checkpoint)\n",
    "\n",
    "max_length = 128\n",
    "\n",
    "def preprocess_function(data): # data should be splitted into train / dev / test internally\n",
    "    inputs = [sent['en_sentence'] for doc in data[\"conversation\"] for sent in doc]\n",
    "    targets = [sent['ja_sentence'] for doc in data[\"conversation\"] for sent in doc]\n",
    "    model_inputs = tokenizer(\n",
    "        inputs, text_target=targets, max_length=max_length, truncation=True\n",
    "    )\n",
    "    return model_inputs\n",
    "\"\"\"\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "vscode": {
     "languageId": "python"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Loading cached processed dataset at /home/sumire/.cache/huggingface/datasets/json/default-d1314a26020b1cab/0.0.0/e6070c77f18f01a5ad4551a8b7edfba20b8438b7cad4d94e6ad9378022ce4aab/cache-c69a69d508e0c92a.arrow\n",
      "Loading cached processed dataset at /home/sumire/.cache/huggingface/datasets/json/default-d1314a26020b1cab/0.0.0/e6070c77f18f01a5ad4551a8b7edfba20b8438b7cad4d94e6ad9378022ce4aab/cache-bfb3dd6fb328072e.arrow\n",
      "Loading cached processed dataset at /home/sumire/.cache/huggingface/datasets/json/default-d1314a26020b1cab/0.0.0/e6070c77f18f01a5ad4551a8b7edfba20b8438b7cad4d94e6ad9378022ce4aab/cache-7fd9a99bf152e87f.arrow\n"
     ]
    }
   ],
   "source": [
    "tokenized_datasets = dataset.map(\n",
    "    preprocess_function,\n",
    "    batched=True,\n",
    "    remove_columns=dataset[\"train\"].column_names,\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "DataCollator"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "vscode": {
     "languageId": "python"
    }
   },
   "outputs": [],
   "source": [
    "data_collator = DataCollatorForSeq2Seq(tokenizer, model=model, return_tensors=\"pt\") # not tf"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Define Batch using Datacollator\n",
    "###To Chryssa, decoder_input_ids now exists!####"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "vscode": {
     "languageId": "python"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "dict_keys(['input_ids', 'attention_mask', 'labels', 'decoder_input_ids'])"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "batch = data_collator([tokenized_datasets[\"train\"][i] for i in range(1, 3)])\n",
    "batch.keys()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Fine-tune using Trainer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "vscode": {
     "languageId": "python"
    }
   },
   "outputs": [],
   "source": [
    "os.environ[\"CUDA_VISIBLE_DEVICES\"]=\"0\""
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Evaluation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "vscode": {
     "languageId": "python"
    }
   },
   "outputs": [],
   "source": [
    "metric = evaluate.load(\"sacrebleu\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "vscode": {
     "languageId": "python"
    }
   },
   "outputs": [],
   "source": [
    "def postprocess_text(preds, labels):\n",
    "    preds = [pred.strip() for pred in preds]\n",
    "    labels = [[label.strip()] for label in labels]\n",
    "\n",
    "    return preds, labels\n",
    "\n",
    "\n",
    "def compute_metrics(eval_preds):\n",
    "    preds, labels = eval_preds\n",
    "    if isinstance(preds, tuple):\n",
    "        preds = preds[0]\n",
    "    decoded_preds = tokenizer.batch_decode(preds, skip_special_tokens=True)\n",
    "\n",
    "    labels = np.where(labels != -100, labels, tokenizer.pad_token_id)\n",
    "    decoded_labels = tokenizer.batch_decode(labels, skip_special_tokens=True)\n",
    "\n",
    "    decoded_preds, decoded_labels = postprocess_text(decoded_preds, decoded_labels)\n",
    "\n",
    "    result = metric.compute(predictions=decoded_preds, references=decoded_labels)\n",
    "    result = {\"bleu\": result[\"score\"]}\n",
    "\n",
    "    prediction_lens = [np.count_nonzero(pred != tokenizer.pad_token_id) for pred in preds]\n",
    "    result[\"gen_len\"] = np.mean(prediction_lens)\n",
    "    result = {k: round(v, 4) for k, v in result.items()}\n",
    "    return result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "vscode": {
     "languageId": "python"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'\\nfrom torch import nn\\n\\ndef freeze_params(model: nn.Module):\\n    Set requires_grad=False for each of model.parameters()\\n    for par in model.parameters():\\n        par.requires_grad = False\\n\\n#model = AutoModel.from_pretrained(\"facebook/bart-large\")\\nenc_layers = model.get_encoder().layers\\nfreeze_params(enc_layers)  # freeze layer 0\\ndropout = enc_layers[0].dropout   # return dropout value for layer 0\\nenc_layers[0].dropout = 0.5  # set dropout value for layer 0\\n\\n'"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\"\"\"\n",
    "from torch import nn\n",
    "\n",
    "def freeze_params(model: nn.Module):\n",
    "    Set requires_grad=False for each of model.parameters()\n",
    "    for par in model.parameters():\n",
    "        par.requires_grad = False\n",
    "\n",
    "#model = AutoModel.from_pretrained(\"facebook/bart-large\")\n",
    "enc_layers = model.get_encoder().layers\n",
    "freeze_params(enc_layers)  # freeze layer 0\n",
    "dropout = enc_layers[0].dropout   # return dropout value for layer 0\n",
    "enc_layers[0].dropout = 0.5  # set dropout value for layer 0\n",
    "\n",
    "\"\"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "vscode": {
     "languageId": "python"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using cuda_amp half precision backend\n"
     ]
    }
   ],
   "source": [
    "os.environ[\"CUDA_VISIBLE_DEVICES\"]=\"0\"\n",
    "#os.environ[\"CUDA_LAUNCH_BLOCKING\"]=\"0\"\n",
    "\n",
    "training_args = Seq2SeqTrainingArguments(\n",
    "    output_dir='./results',\n",
    "    evaluation_strategy=\"epoch\",\n",
    "    learning_rate=2e-5,        \n",
    "    #logging_dir='./logs',            \n",
    "    num_train_epochs=3, #3             \n",
    "    per_device_train_batch_size=1, #16  \n",
    "    per_device_eval_batch_size=1,  #64 \n",
    "    warmup_steps=500,                \n",
    "    weight_decay=0.01,\n",
    "    save_total_limit=3,\n",
    "    predict_with_generate=True,\n",
    "    report_to=\"all\",\n",
    "    fp16=True,\n",
    "    #gradient_accumulation_steps=1000,\n",
    "    #half_precision_backend=\"apex\"\n",
    ")\n",
    "\n",
    "trainer = Seq2SeqTrainer(\n",
    "    model=model,                         \n",
    "    args=training_args,                  \n",
    "    train_dataset=tokenized_datasets[\"train\"],        \n",
    "    eval_dataset=tokenized_datasets[\"validation\"],            \n",
    "    data_collator=data_collator,\n",
    "    tokenizer=tokenizer,\n",
    "    compute_metrics=compute_metrics\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "vscode": {
     "languageId": "python"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/sumire/miniconda3/envs/mt2022_server/lib/python3.8/site-packages/transformers/optimization.py:306: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning\n",
      "  warnings.warn(\n",
      "***** Running training *****\n",
      "  Num examples = 20000\n",
      "  Num Epochs = 3\n",
      "  Instantaneous batch size per device = 1\n",
      "  Total train batch size (w. parallel, distributed & accumulation) = 1\n",
      "  Gradient Accumulation steps = 1\n",
      "  Total optimization steps = 60000\n",
      "  Number of trainable parameters = 610879488\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='114' max='60000' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [  114/60000 00:29 < 4:20:12, 3.84 it/s, Epoch 0.01/3]\n",
       "    </div>\n",
       "    <table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       " <tr style=\"text-align: left;\">\n",
       "      <th>Epoch</th>\n",
       "      <th>Training Loss</th>\n",
       "      <th>Validation Loss</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table><p>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[1;32m/home/sumire/discourse_context_mt/huggingface_mbart.ipynb Cell 24\u001b[0m in \u001b[0;36m<cell line: 2>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      <a href='vscode-notebook-cell://ssh-remote%2Bposeidon/home/sumire/discourse_context_mt/huggingface_mbart.ipynb#X32sdnNjb2RlLXJlbW90ZQ%3D%3D?line=0'>1</a>\u001b[0m os\u001b[39m.\u001b[39menviron[\u001b[39m\"\u001b[39m\u001b[39mCUDA_VISIBLE_DEVICES\u001b[39m\u001b[39m\"\u001b[39m]\u001b[39m=\u001b[39m\u001b[39m\"\u001b[39m\u001b[39m0\u001b[39m\u001b[39m\"\u001b[39m\n\u001b[0;32m----> <a href='vscode-notebook-cell://ssh-remote%2Bposeidon/home/sumire/discourse_context_mt/huggingface_mbart.ipynb#X32sdnNjb2RlLXJlbW90ZQ%3D%3D?line=1'>2</a>\u001b[0m trainer\u001b[39m.\u001b[39;49mtrain()\n",
      "File \u001b[0;32m~/miniconda3/envs/mt2022_server/lib/python3.8/site-packages/transformers/trainer.py:1501\u001b[0m, in \u001b[0;36mTrainer.train\u001b[0;34m(self, resume_from_checkpoint, trial, ignore_keys_for_eval, **kwargs)\u001b[0m\n\u001b[1;32m   1496\u001b[0m     \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mmodel_wrapped \u001b[39m=\u001b[39m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mmodel\n\u001b[1;32m   1498\u001b[0m inner_training_loop \u001b[39m=\u001b[39m find_executable_batch_size(\n\u001b[1;32m   1499\u001b[0m     \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_inner_training_loop, \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_train_batch_size, args\u001b[39m.\u001b[39mauto_find_batch_size\n\u001b[1;32m   1500\u001b[0m )\n\u001b[0;32m-> 1501\u001b[0m \u001b[39mreturn\u001b[39;00m inner_training_loop(\n\u001b[1;32m   1502\u001b[0m     args\u001b[39m=\u001b[39;49margs,\n\u001b[1;32m   1503\u001b[0m     resume_from_checkpoint\u001b[39m=\u001b[39;49mresume_from_checkpoint,\n\u001b[1;32m   1504\u001b[0m     trial\u001b[39m=\u001b[39;49mtrial,\n\u001b[1;32m   1505\u001b[0m     ignore_keys_for_eval\u001b[39m=\u001b[39;49mignore_keys_for_eval,\n\u001b[1;32m   1506\u001b[0m )\n",
      "File \u001b[0;32m~/miniconda3/envs/mt2022_server/lib/python3.8/site-packages/transformers/trainer.py:1749\u001b[0m, in \u001b[0;36mTrainer._inner_training_loop\u001b[0;34m(self, batch_size, args, resume_from_checkpoint, trial, ignore_keys_for_eval)\u001b[0m\n\u001b[1;32m   1747\u001b[0m         tr_loss_step \u001b[39m=\u001b[39m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mtraining_step(model, inputs)\n\u001b[1;32m   1748\u001b[0m \u001b[39melse\u001b[39;00m:\n\u001b[0;32m-> 1749\u001b[0m     tr_loss_step \u001b[39m=\u001b[39m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mtraining_step(model, inputs)\n\u001b[1;32m   1751\u001b[0m \u001b[39mif\u001b[39;00m (\n\u001b[1;32m   1752\u001b[0m     args\u001b[39m.\u001b[39mlogging_nan_inf_filter\n\u001b[1;32m   1753\u001b[0m     \u001b[39mand\u001b[39;00m \u001b[39mnot\u001b[39;00m is_torch_tpu_available()\n\u001b[1;32m   1754\u001b[0m     \u001b[39mand\u001b[39;00m (torch\u001b[39m.\u001b[39misnan(tr_loss_step) \u001b[39mor\u001b[39;00m torch\u001b[39m.\u001b[39misinf(tr_loss_step))\n\u001b[1;32m   1755\u001b[0m ):\n\u001b[1;32m   1756\u001b[0m     \u001b[39m# if loss is nan or inf simply add the average of previous logged losses\u001b[39;00m\n\u001b[1;32m   1757\u001b[0m     tr_loss \u001b[39m+\u001b[39m\u001b[39m=\u001b[39m tr_loss \u001b[39m/\u001b[39m (\u001b[39m1\u001b[39m \u001b[39m+\u001b[39m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mstate\u001b[39m.\u001b[39mglobal_step \u001b[39m-\u001b[39m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_globalstep_last_logged)\n",
      "File \u001b[0;32m~/miniconda3/envs/mt2022_server/lib/python3.8/site-packages/transformers/trainer.py:2518\u001b[0m, in \u001b[0;36mTrainer.training_step\u001b[0;34m(self, model, inputs)\u001b[0m\n\u001b[1;32m   2515\u001b[0m     loss \u001b[39m=\u001b[39m loss \u001b[39m/\u001b[39m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39margs\u001b[39m.\u001b[39mgradient_accumulation_steps\n\u001b[1;32m   2517\u001b[0m \u001b[39mif\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mdo_grad_scaling:\n\u001b[0;32m-> 2518\u001b[0m     \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mscaler\u001b[39m.\u001b[39;49mscale(loss)\u001b[39m.\u001b[39;49mbackward()\n\u001b[1;32m   2519\u001b[0m \u001b[39melif\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39muse_apex:\n\u001b[1;32m   2520\u001b[0m     \u001b[39mwith\u001b[39;00m amp\u001b[39m.\u001b[39mscale_loss(loss, \u001b[39mself\u001b[39m\u001b[39m.\u001b[39moptimizer) \u001b[39mas\u001b[39;00m scaled_loss:\n",
      "File \u001b[0;32m~/miniconda3/envs/mt2022_server/lib/python3.8/site-packages/torch/_tensor.py:487\u001b[0m, in \u001b[0;36mTensor.backward\u001b[0;34m(self, gradient, retain_graph, create_graph, inputs)\u001b[0m\n\u001b[1;32m    477\u001b[0m \u001b[39mif\u001b[39;00m has_torch_function_unary(\u001b[39mself\u001b[39m):\n\u001b[1;32m    478\u001b[0m     \u001b[39mreturn\u001b[39;00m handle_torch_function(\n\u001b[1;32m    479\u001b[0m         Tensor\u001b[39m.\u001b[39mbackward,\n\u001b[1;32m    480\u001b[0m         (\u001b[39mself\u001b[39m,),\n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m    485\u001b[0m         inputs\u001b[39m=\u001b[39minputs,\n\u001b[1;32m    486\u001b[0m     )\n\u001b[0;32m--> 487\u001b[0m torch\u001b[39m.\u001b[39;49mautograd\u001b[39m.\u001b[39;49mbackward(\n\u001b[1;32m    488\u001b[0m     \u001b[39mself\u001b[39;49m, gradient, retain_graph, create_graph, inputs\u001b[39m=\u001b[39;49minputs\n\u001b[1;32m    489\u001b[0m )\n",
      "File \u001b[0;32m~/miniconda3/envs/mt2022_server/lib/python3.8/site-packages/torch/autograd/__init__.py:197\u001b[0m, in \u001b[0;36mbackward\u001b[0;34m(tensors, grad_tensors, retain_graph, create_graph, grad_variables, inputs)\u001b[0m\n\u001b[1;32m    192\u001b[0m     retain_graph \u001b[39m=\u001b[39m create_graph\n\u001b[1;32m    194\u001b[0m \u001b[39m# The reason we repeat same the comment below is that\u001b[39;00m\n\u001b[1;32m    195\u001b[0m \u001b[39m# some Python versions print out the first line of a multi-line function\u001b[39;00m\n\u001b[1;32m    196\u001b[0m \u001b[39m# calls in the traceback and some print out the last line\u001b[39;00m\n\u001b[0;32m--> 197\u001b[0m Variable\u001b[39m.\u001b[39;49m_execution_engine\u001b[39m.\u001b[39;49mrun_backward(  \u001b[39m# Calls into the C++ engine to run the backward pass\u001b[39;49;00m\n\u001b[1;32m    198\u001b[0m     tensors, grad_tensors_, retain_graph, create_graph, inputs,\n\u001b[1;32m    199\u001b[0m     allow_unreachable\u001b[39m=\u001b[39;49m\u001b[39mTrue\u001b[39;49;00m, accumulate_grad\u001b[39m=\u001b[39;49m\u001b[39mTrue\u001b[39;49;00m)\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "os.environ[\"CUDA_VISIBLE_DEVICES\"]=\"0\"\n",
    "trainer.train()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "vscode": {
     "languageId": "python"
    }
   },
   "outputs": [],
   "source": [
    "# Training without using Trainer\n",
    "##Doesn't work###\n",
    "\n",
    "inputs = [sent['en_sentence'] for doc in dataset[\"train\"][\"conversation\"] for sent in doc]\n",
    "targets = [sent['ja_sentence'] for doc in dataset[\"train\"][\"conversation\"] for sent in doc]\n",
    "\n",
    "inputs = tokenizer(inputs, text_target=targets, return_tensors=\"pt\", padding=True)\n",
    "print (inputs)\n",
    "\n",
    "model(**inputs)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": []
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Inference"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "vscode": {
     "languageId": "python"
    }
   },
   "outputs": [],
   "source": [
    "from transformers import pipeline\n",
    "import protobuf\n",
    "\n",
    "text = \"Hi, this is Sumire. May I help you ?\"\n",
    "translator = pipeline(\"translation\", model=\"./results/checkpoint-10000\")\n",
    "translator(text)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "vscode": {
   "interpreter": {
    "hash": "518d00723827b672b8fb8e29bda626c4ff2d351b2d965931921410b814c0648b"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}