gpt2_poem_stanza.ipynb

{"nbformat":4,"nbformat_minor":0,"metadata":{"accelerator":"GPU","colab":{"name":"gpt2_poem_stanza.ipynb","provenance":[],"collapsed_sections":["6ojuAdlpDABj","FVmKeOS_DADy"]},"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.7.4"}},"cells":[{"cell_type":"code","metadata":{"id":"yG3O0YcbDGxR","executionInfo":{"status":"ok","timestamp":1604188428660,"user_tz":240,"elapsed":596,"user":{"displayName":"Scott Duda","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GjRn5vlsP2VP-TM4jPt1ZKBGq1MJCJ2yl5F4D0H=s64","userId":"07461941358185247463"}},"outputId":"a6454471-e84d-4035-fa59-18336a473518","colab":{"base_uri":"https://localhost:8080/"}},"source":["# Specify your own home directory as needed! This notebook is designed to run in Google Colab.\n","\n","# home_directory = ''\n","home_directory = 'drive/My Drive/Colab Notebooks/poe/'\n","\n","from google.colab import drive\n","drive.mount('/content/drive')"],"execution_count":1,"outputs":[{"output_type":"stream","text":["Drive already mounted at /content/drive; to attempt to forcibly remount, call drive.mount(\"/content/drive\", force_remount=True).\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"JYKPbTRoDUtz","executionInfo":{"status":"ok","timestamp":1604188430832,"user_tz":240,"elapsed":2761,"user":{"displayName":"Scott Duda","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GjRn5vlsP2VP-TM4jPt1ZKBGq1MJCJ2yl5F4D0H=s64","userId":"07461941358185247463"}},"outputId":"571f25cf-2265-4d1d-cc25-8270e716160a","colab":{"base_uri":"https://localhost:8080/"}},"source":["!pip install transformers"],"execution_count":2,"outputs":[{"output_type":"stream","text":["Requirement already satisfied: transformers in /usr/local/lib/python3.6/dist-packages (3.4.0)\n","Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.6/dist-packages (from transformers) (2019.12.20)\n","Requirement already satisfied: requests in /usr/local/lib/python3.6/dist-packages (from transformers) (2.23.0)\n","Requirement already satisfied: numpy in /usr/local/lib/python3.6/dist-packages (from transformers) (1.18.5)\n","Requirement already satisfied: protobuf in /usr/local/lib/python3.6/dist-packages (from transformers) (3.12.4)\n","Requirement already satisfied: sacremoses in /usr/local/lib/python3.6/dist-packages (from transformers) (0.0.43)\n","Requirement already satisfied: tokenizers==0.9.2 in /usr/local/lib/python3.6/dist-packages (from transformers) (0.9.2)\n","Requirement already satisfied: packaging in /usr/local/lib/python3.6/dist-packages (from transformers) (20.4)\n","Requirement already satisfied: sentencepiece!=0.1.92 in /usr/local/lib/python3.6/dist-packages (from transformers) (0.1.94)\n","Requirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.6/dist-packages (from transformers) (4.41.1)\n","Requirement already satisfied: filelock in /usr/local/lib/python3.6/dist-packages (from transformers) (3.0.12)\n","Requirement already satisfied: dataclasses; python_version < \"3.7\" in /usr/local/lib/python3.6/dist-packages (from transformers) (0.7)\n","Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.6/dist-packages (from requests->transformers) (3.0.4)\n","Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.6/dist-packages (from requests->transformers) (2.10)\n","Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.6/dist-packages (from requests->transformers) (2020.6.20)\n","Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.6/dist-packages (from requests->transformers) (1.24.3)\n","Requirement already satisfied: setuptools in /usr/local/lib/python3.6/dist-packages (from protobuf->transformers) (50.3.2)\n","Requirement already satisfied: six>=1.9 in /usr/local/lib/python3.6/dist-packages (from protobuf->transformers) (1.15.0)\n","Requirement already satisfied: joblib in /usr/local/lib/python3.6/dist-packages (from sacremoses->transformers) (0.17.0)\n","Requirement already satisfied: click in /usr/local/lib/python3.6/dist-packages (from sacremoses->transformers) (7.1.2)\n","Requirement already satisfied: pyparsing>=2.0.2 in /usr/local/lib/python3.6/dist-packages (from packaging->transformers) (2.4.7)\n"],"name":"stdout"}]},{"cell_type":"markdown","metadata":{"id":"6ojuAdlpDABj"},"source":["# Import Libraries & Load Data"]},{"cell_type":"code","metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","id":"U28UnrvEDABk","executionInfo":{"status":"ok","timestamp":1604188432979,"user_tz":240,"elapsed":4902,"user":{"displayName":"Scott Duda","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GjRn5vlsP2VP-TM4jPt1ZKBGq1MJCJ2yl5F4D0H=s64","userId":"07461941358185247463"}}},"source":["import numpy as np\n","import pandas as pd \n","\n","import random\n","import time\n","import datetime\n","\n","import torch\n","from transformers import GPT2Tokenizer, GPT2LMHeadModel, GPT2Config, AdamW, get_linear_schedule_with_warmup\n","from torch.utils.data import Dataset, random_split, DataLoader, RandomSampler, SequentialSampler"],"execution_count":3,"outputs":[]},{"cell_type":"code","metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","id":"c0Sq2b2qDABn","executionInfo":{"status":"ok","timestamp":1604188432980,"user_tz":240,"elapsed":4899,"user":{"displayName":"Scott Duda","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GjRn5vlsP2VP-TM4jPt1ZKBGq1MJCJ2yl5F4D0H=s64","userId":"07461941358185247463"}}},"source":["poem_stanza_df = pd.read_csv(home_directory + 'poe_poems_stanzas.csv')\n","poem_stanza_df = poem_stanza_df.fillna('')"],"execution_count":4,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"NvAA-x3qDADW"},"source":["# Text Generation - GPT-2\n","## Process Text and Create Dataset\n","\n","http://jalammar.github.io/illustrated-gpt2/\n","https://medium.com/@stasinopoulos.dimitrios/a-beginners-guide-to-training-and-generating-text-using-gpt2-c2f2e1fbd10a\n","https://huggingface.co/transformers/model_doc/gpt2.html\n","https://towardsdatascience.com/step-by-step-guide-on-how-to-train-gpt-2-on-books-using-google-colab-b3c6fa15fef0\n","https://medium.com/swlh/fine-tuning-gpt-2-for-magic-the-gathering-flavour-text-generation-3bafd0f9bb93\n","https://colab.research.google.com/drive/16UTbQOhspQOF3XlxDFyI28S-0nAkTzk_#scrollTo=v4XhewaV93-_"]},{"cell_type":"code","metadata":{"id":"mqDVvA-3DADW","executionInfo":{"status":"ok","timestamp":1604188432981,"user_tz":240,"elapsed":4895,"user":{"displayName":"Scott Duda","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GjRn5vlsP2VP-TM4jPt1ZKBGq1MJCJ2yl5F4D0H=s64","userId":"07461941358185247463"}}},"source":["RANDOM_SEED = 73\n","BATCH_SIZE = 2\n","EPOCHS = 8\n","MAX_LEN = 1024"],"execution_count":5,"outputs":[]},{"cell_type":"code","metadata":{"id":"mr2yKiTrDADY","executionInfo":{"status":"ok","timestamp":1604188433720,"user_tz":240,"elapsed":5628,"user":{"displayName":"Scott Duda","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GjRn5vlsP2VP-TM4jPt1ZKBGq1MJCJ2yl5F4D0H=s64","userId":"07461941358185247463"}}},"source":["tokenizer = GPT2Tokenizer.from_pretrained('gpt2')\n","\n","special_tokens_dict = {'bos_token': '<BOS>', 'eos_token': '<EOS>', 'pad_token': '<PAD>'}\n","num_added_tokens = tokenizer.add_special_tokens(special_tokens_dict)"],"execution_count":6,"outputs":[]},{"cell_type":"code","metadata":{"id":"_8j6uhIADADi","executionInfo":{"status":"ok","timestamp":1604188433721,"user_tz":240,"elapsed":5623,"user":{"displayName":"Scott Duda","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GjRn5vlsP2VP-TM4jPt1ZKBGq1MJCJ2yl5F4D0H=s64","userId":"07461941358185247463"}}},"source":["combined_poems = poem_stanza_df.groupby(['title'])['stanza_text'].transform(lambda x: ' /n /n '.join(x)).drop_duplicates().reset_index(drop=True)"],"execution_count":7,"outputs":[]},{"cell_type":"code","metadata":{"id":"UMiVwpwhDADj","executionInfo":{"status":"ok","timestamp":1604188434077,"user_tz":240,"elapsed":5974,"user":{"displayName":"Scott Duda","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GjRn5vlsP2VP-TM4jPt1ZKBGq1MJCJ2yl5F4D0H=s64","userId":"07461941358185247463"}},"outputId":"a620c70f-1ec4-4514-a730-2e224ea1e0e2","colab":{"base_uri":"https://localhost:8080/"}},"source":["max_poem_length = max([len(tokenizer.encode(poem)) for poem in combined_poems])\n","min_poem_length = min([len(tokenizer.encode(poem)) for poem in combined_poems])"],"execution_count":8,"outputs":[{"output_type":"stream","text":["Token indices sequence length is longer than the specified maximum sequence length for this model (1753 > 1024). Running this sequence through the model will result in indexing errors\n"],"name":"stderr"}]},{"cell_type":"code","metadata":{"id":"PQazya_wDADm","executionInfo":{"status":"ok","timestamp":1604188434079,"user_tz":240,"elapsed":5970,"user":{"displayName":"Scott Duda","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GjRn5vlsP2VP-TM4jPt1ZKBGq1MJCJ2yl5F4D0H=s64","userId":"07461941358185247463"}},"outputId":"765623ea-c272-4997-b833-a4cb6628c05e","colab":{"base_uri":"https://localhost:8080/"}},"source":["print('Longest Edgar Allen Poe Poem:', max_poem_length, 'tokens long.')\n","print('Shortest Edgar Allen Poe Poem:', min_poem_length, 'tokens long.')"],"execution_count":9,"outputs":[{"output_type":"stream","text":["Longest Edgar Allen Poe Poem: 6465 tokens long.\n","Shortest Edgar Allen Poe Poem: 55 tokens long.\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"AY7VnRx03uQM","executionInfo":{"status":"ok","timestamp":1604188434080,"user_tz":240,"elapsed":5965,"user":{"displayName":"Scott Duda","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GjRn5vlsP2VP-TM4jPt1ZKBGq1MJCJ2yl5F4D0H=s64","userId":"07461941358185247463"}}},"source":["stanza_length = [len(tokenizer.encode(stanza)) for stanza in poem_stanza_df['stanza_text'].values]\n","max_stanza_length = max(stanza_length)\n","min_stanza_length = min(stanza_length)"],"execution_count":10,"outputs":[]},{"cell_type":"code","metadata":{"id":"nBf3m4Fg3uQO","executionInfo":{"status":"ok","timestamp":1604188434299,"user_tz":240,"elapsed":6177,"user":{"displayName":"Scott Duda","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GjRn5vlsP2VP-TM4jPt1ZKBGq1MJCJ2yl5F4D0H=s64","userId":"07461941358185247463"}},"outputId":"e2790d31-e2b7-42f7-b7fa-5afdfd08edbc","colab":{"base_uri":"https://localhost:8080/"}},"source":["print('Number of stanzas longer than max length (1024 tokens): ', sum([st_len > MAX_LEN for st_len in stanza_length])) "],"execution_count":11,"outputs":[{"output_type":"stream","text":["Number of stanzas longer than max length (1024 tokens):  1\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"Ns-DopJZ3uQQ","executionInfo":{"status":"ok","timestamp":1604188434300,"user_tz":240,"elapsed":6173,"user":{"displayName":"Scott Duda","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GjRn5vlsP2VP-TM4jPt1ZKBGq1MJCJ2yl5F4D0H=s64","userId":"07461941358185247463"}},"outputId":"633ec49a-d566-40d6-93a9-80ea499a6255","colab":{"base_uri":"https://localhost:8080/"}},"source":["print('Longest Edgar Allen Poe Stanza:', max_stanza_length, 'tokens long.')\n","print('Shortest Edgar Allen Poe Stanza:', min_stanza_length, 'tokens long.')"],"execution_count":12,"outputs":[{"output_type":"stream","text":["Longest Edgar Allen Poe Stanza: 1948 tokens long.\n","Shortest Edgar Allen Poe Stanza: 15 tokens long.\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"erUXLChX76-k","executionInfo":{"status":"ok","timestamp":1604188434300,"user_tz":240,"elapsed":6167,"user":{"displayName":"Scott Duda","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GjRn5vlsP2VP-TM4jPt1ZKBGq1MJCJ2yl5F4D0H=s64","userId":"07461941358185247463"}}},"source":["class PoePoemDataset(Dataset):\n","    \n","    def __init__(self, data, tokenizer, gpt2_type='gpt2', max_length=MAX_LEN):\n","        self.tokenizer = tokenizer\n","        self.input_ids = []\n","        self.attn_masks = []\n","        \n","        for i in data:\n","            encodings_dict = tokenizer('<BOS>' + i + '<EOS>',\n","                                     truncation=True,\n","                                     max_length=max_length,\n","                                     padding='max_length'\n","                                    )\n","\n","            self.input_ids.append(torch.tensor(encodings_dict['input_ids']))\n","            self.attn_masks.append(torch.tensor(encodings_dict['attention_mask']))\n","\n","    def __len__(self):\n","        \n","        return len(self.input_ids)\n","    \n","    def __getitem__(self, idx):\n","        \n","        return self.input_ids[idx], self.attn_masks[idx]\n","        "],"execution_count":13,"outputs":[]},{"cell_type":"code","metadata":{"id":"F2X7R8B3DADs","executionInfo":{"status":"ok","timestamp":1604188434477,"user_tz":240,"elapsed":6340,"user":{"displayName":"Scott Duda","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GjRn5vlsP2VP-TM4jPt1ZKBGq1MJCJ2yl5F4D0H=s64","userId":"07461941358185247463"}}},"source":["poem_stanza_dataset = PoePoemDataset(poem_stanza_df['stanza_text'].values, tokenizer, max_length=MAX_LEN)"],"execution_count":14,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"rw3cxQjKDADu"},"source":["### Train/Validation Split"]},{"cell_type":"code","metadata":{"id":"66YPjdiyDADu","executionInfo":{"status":"ok","timestamp":1604188434478,"user_tz":240,"elapsed":6337,"user":{"displayName":"Scott Duda","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GjRn5vlsP2VP-TM4jPt1ZKBGq1MJCJ2yl5F4D0H=s64","userId":"07461941358185247463"}}},"source":["def train_val_split(split, dataset):\n","    train_size = int(split * len(dataset))\n","    val_size = len(dataset) - train_size\n","    return train_size, val_size"],"execution_count":15,"outputs":[]},{"cell_type":"code","metadata":{"id":"bdmHhsPbDADw","executionInfo":{"status":"ok","timestamp":1604188434478,"user_tz":240,"elapsed":6334,"user":{"displayName":"Scott Duda","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GjRn5vlsP2VP-TM4jPt1ZKBGq1MJCJ2yl5F4D0H=s64","userId":"07461941358185247463"}}},"source":["poem_stanza_train_size, poem_stanza_val_size = train_val_split(0.8, poem_stanza_dataset)\n","\n","# random split imported from troch.utils\n","poem_stanza_train_dataset, poem_stanza_val_dataset = random_split(poem_stanza_dataset, [poem_stanza_train_size, poem_stanza_val_size])"],"execution_count":16,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"FVmKeOS_DADy"},"source":["### Apply Random Seeds"]},{"cell_type":"code","metadata":{"id":"Vjgbzym_DADy","executionInfo":{"status":"ok","timestamp":1604188434479,"user_tz":240,"elapsed":6329,"user":{"displayName":"Scott Duda","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GjRn5vlsP2VP-TM4jPt1ZKBGq1MJCJ2yl5F4D0H=s64","userId":"07461941358185247463"}},"outputId":"173c9566-0f78-4797-cf4b-64967313411b","colab":{"base_uri":"https://localhost:8080/"}},"source":["torch.cuda.manual_seed_all(RANDOM_SEED)\n","random.seed(RANDOM_SEED)\n","np.random.seed(RANDOM_SEED)\n","torch.manual_seed(RANDOM_SEED)"],"execution_count":17,"outputs":[{"output_type":"execute_result","data":{"text/plain":["<torch._C.Generator at 0x7f2c6ee73600>"]},"metadata":{"tags":[]},"execution_count":17}]},{"cell_type":"markdown","metadata":{"id":"8fk86UcTDAD2"},"source":["### Instantiate DataLoaders and Define Model Creation Function"]},{"cell_type":"code","metadata":{"id":"IupduXEyDAD2","executionInfo":{"status":"ok","timestamp":1604188434479,"user_tz":240,"elapsed":6324,"user":{"displayName":"Scott Duda","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GjRn5vlsP2VP-TM4jPt1ZKBGq1MJCJ2yl5F4D0H=s64","userId":"07461941358185247463"}}},"source":["poem_stanza_train_dataloader = DataLoader(poem_stanza_train_dataset,\n","                              sampler=RandomSampler(poem_stanza_train_dataset),\n","                              batch_size=BATCH_SIZE)\n","\n","poem_stanza_val_dataloader = DataLoader(poem_stanza_val_dataset,\n","                            sampler=SequentialSampler(poem_stanza_val_dataset),\n","                            batch_size=BATCH_SIZE)"],"execution_count":18,"outputs":[]},{"cell_type":"code","metadata":{"id":"vtN2HoCSDAD8","executionInfo":{"status":"ok","timestamp":1604188439426,"user_tz":240,"elapsed":11267,"user":{"displayName":"Scott Duda","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GjRn5vlsP2VP-TM4jPt1ZKBGq1MJCJ2yl5F4D0H=s64","userId":"07461941358185247463"}}},"source":["# helper function for logging time\n","def format_time(elapsed):\n","    return str(datetime.timedelta(seconds=int(round((elapsed)))))\n","\n","# hyperparameters\n","learning_rate = 1e-4\n","eps = 1e-8\n","warmup_steps = 50\n","\n","# create text generation seed prompt\n","device = torch.device('cuda')\n","\n","prompt = \"<BOS>\"\n","generated = torch.tensor(tokenizer.encode(prompt)).unsqueeze(0)\n","generated = generated.to(device)"],"execution_count":19,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"txv2-Y9e3uQp"},"source":["# Create Poem Stanza Model"]},{"cell_type":"code","metadata":{"id":"91z0CIlTDAD_","executionInfo":{"status":"ok","timestamp":1604189052255,"user_tz":240,"elapsed":624090,"user":{"displayName":"Scott Duda","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GjRn5vlsP2VP-TM4jPt1ZKBGq1MJCJ2yl5F4D0H=s64","userId":"07461941358185247463"}},"outputId":"65e05c38-78b1-4ebc-b939-516affe90fe1","colab":{"base_uri":"https://localhost:8080/"}},"source":["configuration = GPT2Config(vocab_size=len(tokenizer), n_positions=MAX_LEN).from_pretrained('gpt2', output_hidden_states=True)\n","\n","poem_stanza_model = GPT2LMHeadModel.from_pretrained('gpt2', config=configuration)\n","poem_stanza_model.resize_token_embeddings(len(tokenizer))\n","\n","poem_stanza_model.cuda()\n","optimizer = AdamW(poem_stanza_model.parameters(), lr=learning_rate, eps=eps)\n","\n","total_steps = len(poem_stanza_train_dataloader) * EPOCHS\n","scheduler = get_linear_schedule_with_warmup(optimizer,\n","                                            num_warmup_steps=warmup_steps,\n","                                            num_training_steps=total_steps)\n","\n","start_time = time.time()\n","poem_stanza_model = poem_stanza_model.to(device)\n","\n","for epoch_i in range(0, EPOCHS):\n","\n","    print(f'Epoch {epoch_i + 1} of {EPOCHS}')\n","\n","    t0 = time.time()\n","    total_train_loss = 0\n","    poem_stanza_model.train()\n","\n","    for step, batch in enumerate(poem_stanza_train_dataloader):\n","\n","        b_input_ids = batch[0].to(device)\n","        b_labels = batch[0].to(device)\n","        b_masks = batch[1].to(device)\n","\n","        poem_stanza_model.zero_grad()        \n","\n","        outputs = poem_stanza_model(b_input_ids,\n","                                    labels=b_labels,\n","                                    attention_mask=b_masks,\n","                                    token_type_ids=None)\n","\n","        loss = outputs[0]  \n","\n","        batch_loss = loss.item()\n","        total_train_loss += batch_loss\n","\n","        loss.backward()\n","        optimizer.step()\n","        scheduler.step()\n","\n","    avg_train_loss = total_train_loss / len(poem_stanza_train_dataloader)       \n","    training_time = format_time(time.time() - t0)\n","\n","    print(f'Average Training Loss: {avg_train_loss}. Epoch Training Time: {training_time}')\n","\n","    t0 = time.time()\n","\n","    poem_stanza_model.eval()\n","\n","    total_eval_loss = 0\n","    nb_eval_steps = 0\n","\n","    for batch in poem_stanza_val_dataloader:\n","        b_input_ids = batch[0].to(device)\n","        b_labels = batch[0].to(device)\n","        b_masks = batch[1].to(device)\n","\n","        with torch.no_grad():        \n","\n","            outputs  = poem_stanza_model(b_input_ids,\n","                                         attention_mask=b_masks,\n","                                         labels=b_labels)\n","\n","            loss = outputs[0]  \n","\n","        batch_loss = loss.item()\n","        total_eval_loss += batch_loss        \n","\n","    avg_val_loss = total_eval_loss / len(poem_stanza_val_dataloader)\n","\n","\n","    print(f'Average Validation Loss: {avg_val_loss}')\n","\n","print(f'Total Training Time: {format_time(time.time()-start_time)}')\n","\n","torch.save(poem_stanza_model.state_dict(), home_directory + 'poem_stanza_model.pth')\n"],"execution_count":20,"outputs":[{"output_type":"stream","text":["Epoch 1 of 8\n","Average Training Loss: 2.576231449495914. Epoch Training Time: 0:01:10\n","Average Validation Loss: 0.6122910502282056\n","Epoch 2 of 8\n","Average Training Loss: 0.6284494824534239. Epoch Training Time: 0:01:10\n","Average Validation Loss: 0.5452861582691019\n","Epoch 3 of 8\n","Average Training Loss: 0.5744865956001504. Epoch Training Time: 0:01:10\n","Average Validation Loss: 0.5252606712959029\n","Epoch 4 of 8\n","Average Training Loss: 0.5516689658858055. Epoch Training Time: 0:01:09\n","Average Validation Loss: 0.51527174833146\n","Epoch 5 of 8\n","Average Training Loss: 0.5309407965734948. Epoch Training Time: 0:01:10\n","Average Validation Loss: 0.511662708087401\n","Epoch 6 of 8\n","Average Training Loss: 0.5195481184610101. Epoch Training Time: 0:01:09\n","Average Validation Loss: 0.5067863213745031\n","Epoch 7 of 8\n","Average Training Loss: 0.5100360779568206. Epoch Training Time: 0:01:09\n","Average Validation Loss: 0.506346502087333\n","Epoch 8 of 8\n","Average Training Loss: 0.5045993163488632. Epoch Training Time: 0:01:09\n","Average Validation Loss: 0.5056713040579449\n","Total Training Time: 0:10:04\n"],"name":"stdout"}]},{"cell_type":"markdown","metadata":{"id":"p_ZdvJfeDAED"},"source":["# Generate Poem Stanzas"]},{"cell_type":"code","metadata":{"id":"u_UOyQzkDAED","executionInfo":{"status":"ok","timestamp":1604189787108,"user_tz":240,"elapsed":18129,"user":{"displayName":"Scott Duda","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GjRn5vlsP2VP-TM4jPt1ZKBGq1MJCJ2yl5F4D0H=s64","userId":"07461941358185247463"}},"outputId":"0785ef7c-c5da-4e86-9c81-fbc9d5da4c1d","colab":{"base_uri":"https://localhost:8080/"}},"source":["poem_stanza_model.eval()\n","\n","sample_outputs = poem_stanza_model.generate(\n","                                generated, \n","                                do_sample=True,   \n","                                top_k=50, \n","                                max_length=MAX_LEN,\n","                                top_p=0.95, \n","                                num_return_sequences=3\n","                                )\n","\n","for i, sample_output in enumerate(sample_outputs):\n","    print(\"{}: {}\\n\\n\".format(i, tokenizer.decode(sample_output, skip_special_tokens=True)))"],"execution_count":25,"outputs":[{"output_type":"stream","text":["Setting `pad_token_id` to 50256 (first `eos_token_id`) to generate sequence\n"],"name":"stderr"},{"output_type":"stream","text":["0: We must love thee with all the love and the tenderness of our hearts,\n"," In thy presence we feel thy anger,\n"," But not in thy strength thy beauty—and not in thy strength thy grief—\n"," And not in thy strength thine eyes— And not in thy strength thy love:\n"," But in thy strength thou art thy pride—\n"," Not in the power of thy own beauty, but in the power of thy soul—\n"," And thou art thy delight—\n"," To the sweetest of the kisses:\n","\n"," Of these happy memories, when—and then——\n"," Upon the very face of a young girl,\n"," A beautiful name, uttered within her infancy—\n"," Like a song sung to a sad child,\n"," And from childhood to the hour of her death,\n"," I feel her love—to thee, then, my dear child,\n"," And me, whose name is still in me—\n"," \n"," Who feel my anger with pride—\n"," Love and reverence alone can help,\n"," In a life like that of this maiden,\n"," And whose name is still in me—\n"," Whom all love grows to bloom! their own their own sacred places;\n"," But my spirit has not left me, yet that I may be a young man,\n"," And in the time I have not departed from my home. its own dwelling—that is, my home;\n"," But this, too, is not yet lost to me.\n","\n","My spirit has not left me, yet that I may be a young man,\n"," But my spirit have not left me, yet that I may be a young man,\n"," And I may be a young man. D. But my spirit hath not left me, yet that I may be a young man. its own dwelling, that is, my home. And I have not left me, yet that I may be a young man,\n"," But my spirit hath not left me, yet that I may be a young man.\n","\n","And my spirit hath not left me, yet that I may be a young man.\n","\n","And my spirit hath not left me, yet that I may be a young man,\n"," And I may be a young man, and I may be a young man—\n"," And I may\n","\n","\n","1:  and his ownemed to its own soul\n","\n","\n","The first time, first time, second time, third time, fourth time, third time, fourth time, fourth time, fourth time,\n","\n","It is the first time, the first time, the first time, the first time, the first time, the first time, the first time, the first time, the first time, the first time, the first time, the first time, the first time, the first time, the first time, the first time, the first time, the first time, the first time, the first time, the first time, the first time, the first time, the first time, the first time, the first time, the first time, the first time, the first time, the first time, the first time, the first time, the first time, the first time, the first time, the first time, the first time, the first time, the first time, the first time, the first time, the first time, the first time, the first time, the first time, the first time, the first time, the first time, the first time, the first time, the first time, the first time, the first time, the first time, the first time, the first time, the first time, the first time, the first time, the first time, the first time, the first time, the first time, the first time, the first time, the first time, the first time, the first time, the first time, the first time, the first time, the first time, the first time, the first time, the first time, the first time, the first time, the first time\n","\n","\n","2: \n"," I heard its own spirit—\n","\n","\n","Whose heart is the heart of the Earth\n","\n","And the spirit of the world;\n","The spirit of the world,\n","The spirit of the world,\n","But the spirit of the world,\n","The spirit of the world—\n","That which lies at the heart of the Earth—\n","The spirit of the world, and the spirit of the world—\n","The spirit of the world, and the spirit of the world,\n","The spirit of the world, and the spirit of the world—\n","The spirit of the world, and the spirit of the world, and the spirit of the world,\n","The spirit of the world, and the spirit of the world, and the spirit of the world,\n","The spirit of the world, and the spirit of the world, and the spirit of the world,\n","And the spirit of the world, and the spirit of the world, and the spirit of the world,\n","The spirit of the world, and the spirit of the world, and the spirit of the world,\n","And the spirit of the world, and the spirit of the world, and the spirit of the world,\n","And the spirit of the world, and the spirit of the world, and the spirit of the world,\n","And the spirit of the world, and the spirit of the world, and the spirit of the world,\n","And the spirit of the world, and the spirit of the world, and the spirit of the world,\n","And the spirit of the world, and the spirit of the world,\n","And the spirit of the world, and the spirit of the world, and the spirit of the world, and the spirit of the world,\n","And the spirit of the world, and the spirit of the world, and the spirit of the world,\n","\n","\n","\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"BGDvGWv1CLVA","executionInfo":{"status":"ok","timestamp":1604189070909,"user_tz":240,"elapsed":642732,"user":{"displayName":"Scott Duda","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GjRn5vlsP2VP-TM4jPt1ZKBGq1MJCJ2yl5F4D0H=s64","userId":"07461941358185247463"}}},"source":[""],"execution_count":21,"outputs":[]}]}