examples/alum_training_and_evaluation/alum_training_and_evaluation_example.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Adversarial training using the ALUM algorithm overrides the usual training step method to generate the adversarial loss alongside the normal loss. Here, we demonstrate how to perform adversarial training and then evaluate using the primitive trainer class. Then, we show how this can be integrated into an automated pipeline using Flows."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Load dummy train and test data from package\n",
    "import pkg_resources\n",
    "TRAIN_PATH = pkg_resources.resource_filename(\n",
    "            'kitanaqa', 'support/unittest-squad.json')\n",
    "EVAL_PATH = pkg_resources.resource_filename(\n",
    "            'kitanaqa', 'support/unittest-squad.json')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "11/12/2020 18:21:23 - kitanaqa - WARNING - Process rank: -1, device: cpu, n_gpu: 0, distributed training: False, 16-bits training: False\n",
      "11/12/2020 18:21:23 - WARNING - kitanaqa -   Process rank: -1, device: cpu, n_gpu: 0, distributed training: False, 16-bits training: False\n",
      "11/12/2020 18:21:23 - kitanaqa - INFO - Training/evaluation parameters TrainingArguments(output_dir='./example_outputs', overwrite_output_dir=False, do_train=True, do_eval=True, do_predict=False, evaluate_during_training=False, prediction_loss_only=False, per_device_train_batch_size=2, per_device_eval_batch_size=1, per_gpu_train_batch_size=None, per_gpu_eval_batch_size=None, gradient_accumulation_steps=2, learning_rate=5e-05, weight_decay=0.0, adam_beta1=0.9, adam_beta2=0.999, adam_epsilon=1e-08, max_grad_norm=1.0, num_train_epochs=1, max_steps=-1, warmup_steps=0, logging_dir='runs/Nov12_18-21-23_Aarons-MBP-7.attlocal.net', logging_first_step=False, logging_steps=500, save_steps=1, save_total_limit=None, no_cuda=False, seed=512, fp16=False, fp16_opt_level='O1', local_rank=-1, tpu_num_cores=None, tpu_metrics_debug=False, debug=False, dataloader_drop_last=False, eval_steps=1000, past_index=-1, run_name=None, disable_tqdm=False, remove_unused_columns=True)\n",
      "11/12/2020 18:21:23 - INFO - kitanaqa -   Training/evaluation parameters TrainingArguments(output_dir='./example_outputs', overwrite_output_dir=False, do_train=True, do_eval=True, do_predict=False, evaluate_during_training=False, prediction_loss_only=False, per_device_train_batch_size=2, per_device_eval_batch_size=1, per_gpu_train_batch_size=None, per_gpu_eval_batch_size=None, gradient_accumulation_steps=2, learning_rate=5e-05, weight_decay=0.0, adam_beta1=0.9, adam_beta2=0.999, adam_epsilon=1e-08, max_grad_norm=1.0, num_train_epochs=1, max_steps=-1, warmup_steps=0, logging_dir='runs/Nov12_18-21-23_Aarons-MBP-7.attlocal.net', logging_first_step=False, logging_steps=500, save_steps=1, save_total_limit=None, no_cuda=False, seed=512, fp16=False, fp16_opt_level='O1', local_rank=-1, tpu_num_cores=None, tpu_metrics_debug=False, debug=False, dataloader_drop_last=False, eval_steps=1000, past_index=-1, run_name=None, disable_tqdm=False, remove_unused_columns=True)\n",
      "11/12/2020 18:21:24 - INFO - filelock -   Lock 4832273912 acquired on ./example_outputs/26bc1ad6c0ac742e9b52263248f6d0f00068293b33709fae12320c0e35ccfbbb.542ce4285a40d23a559526243235df47c5f75c197f04f37d1a0c124c32c9a084.lock\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "1f103c5fa9c44308b840c9b7ffc9c748",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Downloading'), FloatProgress(value=0.0, max=231508.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "11/12/2020 18:21:25 - INFO - filelock -   Lock 4832273912 released on ./example_outputs/26bc1ad6c0ac742e9b52263248f6d0f00068293b33709fae12320c0e35ccfbbb.542ce4285a40d23a559526243235df47c5f75c197f04f37d1a0c124c32c9a084.lock\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "11/12/2020 18:21:25 - INFO - filelock -   Lock 4832320704 acquired on ./example_outputs/a41e817d5c0743e29e86ff85edc8c257e61bc8d88e4271bb1b243b6e7614c633.8949e27aafafa845a18d98a0e3a88bc2d248bbc32a1b75947366664658f23b1c.lock\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "97e19564c996483caf36007006856d04",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Downloading'), FloatProgress(value=0.0, max=442.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "11/12/2020 18:21:25 - INFO - filelock -   Lock 4832320704 released on ./example_outputs/a41e817d5c0743e29e86ff85edc8c257e61bc8d88e4271bb1b243b6e7614c633.8949e27aafafa845a18d98a0e3a88bc2d248bbc32a1b75947366664658f23b1c.lock\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "11/12/2020 18:21:26 - INFO - filelock -   Lock 4832319920 acquired on ./example_outputs/ae9df7a8d658c4f3e1917a471a8a21cf678fa1d4cb91e7702dfe0598dbdcf354.c2015533705b9dff680ae707e205a35e2860e8d148b45d35085419d74fe57ac5.lock\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b28c2221fa004c3cadcc5a2876181486",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Downloading'), FloatProgress(value=0.0, max=267967963.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "11/12/2020 18:21:43 - INFO - filelock -   Lock 4832319920 released on ./example_outputs/ae9df7a8d658c4f3e1917a471a8a21cf678fa1d4cb91e7702dfe0598dbdcf354.c2015533705b9dff680ae707e205a35e2860e8d148b45d35085419d74fe57ac5.lock\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Some weights of the model checkpoint at distilbert-base-uncased were not used when initializing DistilBertForQuestionAnswering: ['vocab_transform.weight', 'vocab_transform.bias', 'vocab_layer_norm.weight', 'vocab_layer_norm.bias', 'vocab_projector.weight', 'vocab_projector.bias']\n",
      "- This IS expected if you are initializing DistilBertForQuestionAnswering from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPretraining model).\n",
      "- This IS NOT expected if you are initializing DistilBertForQuestionAnswering from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
      "Some weights of DistilBertForQuestionAnswering were not initialized from the model checkpoint at distilbert-base-uncased and are newly initialized: ['qa_outputs.weight', 'qa_outputs.bias']\n",
      "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n",
      "11/12/2020 18:21:45 - kitanaqa.trainer.utils - INFO - Creating features from dataset file at ./example_outputs\n",
      "11/12/2020 18:21:45 - INFO - kitanaqa.trainer.utils -   Creating features from dataset file at ./example_outputs\n",
      "100%|██████████| 1/1 [00:00<00:00, 332.35it/s]\n",
      "convert squad examples to features:   0%|          | 0/10 [00:00<?, ?it/s]/Users/asisto/dev/Searchable/Research/KitanaQA/env/lib/python3.7/site-packages/transformers-3.1.0-py3.7.egg/transformers/tokenization_utils_base.py:1321: FutureWarning: The `max_len` attribute has been deprecated and will be removed in a future version, use `model_max_length` instead.\n",
      "  FutureWarning,\n",
      "convert squad examples to features: 100%|██████████| 10/10 [00:00<00:00, 61.80it/s]\n",
      "add example index and unique id: 100%|██████████| 10/10 [00:00<00:00, 119156.36it/s]\n",
      "11/12/2020 18:21:45 - kitanaqa.trainer.utils - INFO - Saving features into cached file ./example_outputs/cached_train_distilbert-base-uncased_512\n",
      "11/12/2020 18:21:45 - INFO - kitanaqa.trainer.utils -   Saving features into cached file ./example_outputs/cached_train_distilbert-base-uncased_512\n"
     ]
    }
   ],
   "source": [
    "from transformers import HfArgumentParser, TrainingArguments\n",
    "from kitanaqa.trainer.arguments import ModelArguments\n",
    "from kitanaqa.trainer.utils import load_and_cache_examples, build_flow\n",
    "from kitanaqa.trainer.run_pipeline import _setup\n",
    "hparams = {\n",
    "            \"model_type\" : \"distilbert\",\n",
    "            \"model_name_or_path\" : \"distilbert-base-uncased\",\n",
    "            \"output_dir\" : \"./example_outputs\",\n",
    "            \"cache_dir\" : \"./example_outputs\",\n",
    "            \"data_dir\" : \"./example_outputs\",\n",
    "            \"train_file_path\" : TRAIN_PATH,\n",
    "            \"predict_file_path\" : {\"dev-v1.1\": EVAL_PATH},\n",
    "            \"aug_file_path\" : None,\n",
    "            \"do_aug\" : False,\n",
    "            \"do_alum\" : True,\n",
    "            \"alpha\" : 5,\n",
    "            \"eps\" : 1e-4,\n",
    "            \"eta\" : 1e-5,\n",
    "            \"sigma\" : 1e-3,\n",
    "            \"do_train\" : True,\n",
    "            \"do_eval\" : True,\n",
    "            \"per_device_train_batch_size\" : 2,\n",
    "            \"per_device_eval_batch_size\" : 1,\n",
    "            \"gradient_accumulation_steps\" : 2,\n",
    "            \"eval_all_checkpoints\" : False,\n",
    "            \"num_train_epochs\" : 1,\n",
    "            \"max_steps\" : -1,\n",
    "            \"save_steps\" : 1,\n",
    "            \"seed\" : 512,\n",
    "            \"fp16\" : False\n",
    "}\n",
    "# Initialize args\n",
    "parser = HfArgumentParser(dataclass_types=[ModelArguments, TrainingArguments])\n",
    "model_args, training_args = parser.parse_dict(hparams)\n",
    "model, tokenizer, train_dataset = _setup(model_args, training_args)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[2020-11-13 02:21:51] INFO - prefect.FlowRunner | Beginning Flow run for 'default'\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "11/12/2020 18:21:51 - INFO - prefect.FlowRunner -   Beginning Flow run for 'default'\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[2020-11-13 02:21:51] INFO - prefect.TaskRunner | Task 'train': Starting task run...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "11/12/2020 18:21:51 - INFO - prefect.TaskRunner -   Task 'train': Starting task run...\n",
      "/Users/asisto/dev/Searchable/Research/KitanaQA/env/lib/python3.7/site-packages/transformers-3.1.0-py3.7.egg/transformers/trainer.py:247: FutureWarning: Passing `prediction_loss_only` as a keyword argument is deprecated and won't be possible in a future version. Use `args.prediction_loss_only` instead.\n",
      "  FutureWarning,\n",
      "You are instantiating a Trainer but Tensorboard is not installed. You should consider installing it.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "0c8c32d2d9ca45e9a6a61d181cf310d4",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Epoch'), FloatProgress(value=0.0, max=1.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e702c9f0a2e6460b8682b4b5c7e41bf5",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Iteration'), FloatProgress(value=0.0, max=5.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/asisto/dev/Searchable/Research/KitanaQA/src/kitanaqa/trainer/train.py:231: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
      "  self._delta = torch.tensor(sample, requires_grad = True, device = self.args.device)\n",
      "/Users/asisto/dev/Searchable/Research/KitanaQA/env/lib/python3.7/site-packages/torch-1.5.1-py3.7-macosx-10.9-x86_64.egg/torch/optim/lr_scheduler.py:200: UserWarning: Please also save or load the state of the optimzer when saving or loading the scheduler.\n",
      "  warnings.warn(SAVE_STATE_WARNING, UserWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "[2020-11-13 02:22:36] INFO - prefect.TaskRunner | Task 'train': finished task run for task with final state: 'Success'\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/asisto/dev/Searchable/Research/KitanaQA/env/lib/python3.7/site-packages/transformers-3.1.0-py3.7.egg/transformers/trainer.py:1048: FutureWarning: This method is deprecated, use `Trainer.is_world_process_zero()` instead.\n",
      "  warnings.warn(\"This method is deprecated, use `Trainer.is_world_process_zero()` instead.\", FutureWarning)\n",
      "11/12/2020 18:22:36 - INFO - prefect.TaskRunner -   Task 'train': finished task run for task with final state: 'Success'\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[2020-11-13 02:22:36] INFO - prefect.TaskRunner | Task 'eval': Starting task run...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "11/12/2020 18:22:36 - INFO - prefect.TaskRunner -   Task 'eval': Starting task run...\n",
      "11/12/2020 18:22:36 - kitanaqa.trainer.utils - WARNING - You are running a non-local model checkpoint. This may or may not be what you intended.\n",
      "11/12/2020 18:22:36 - WARNING - kitanaqa.trainer.utils -   You are running a non-local model checkpoint. This may or may not be what you intended.\n",
      "Some weights of the model checkpoint at distilbert-base-uncased were not used when initializing DistilBertForQuestionAnswering: ['vocab_transform.weight', 'vocab_transform.bias', 'vocab_layer_norm.weight', 'vocab_layer_norm.bias', 'vocab_projector.weight', 'vocab_projector.bias']\n",
      "- This IS expected if you are initializing DistilBertForQuestionAnswering from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPretraining model).\n",
      "- This IS NOT expected if you are initializing DistilBertForQuestionAnswering from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
      "Some weights of DistilBertForQuestionAnswering were not initialized from the model checkpoint at distilbert-base-uncased and are newly initialized: ['qa_outputs.weight', 'qa_outputs.bias']\n",
      "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n",
      "You are instantiating a Trainer but Tensorboard is not installed. You should consider installing it.\n",
      "11/12/2020 18:22:38 - kitanaqa.trainer.utils - INFO - Creating features from dataset file at ./example_outputs\n",
      "11/12/2020 18:22:38 - INFO - kitanaqa.trainer.utils -   Creating features from dataset file at ./example_outputs\n",
      "100%|██████████| 1/1 [00:00<00:00, 203.90it/s]\n",
      "11/12/2020 18:22:38 - kitanaqa.trainer.utils - INFO - Evaluation Data is fetched for dev-v1.1.\n",
      "11/12/2020 18:22:38 - INFO - kitanaqa.trainer.utils -   Evaluation Data is fetched for dev-v1.1.\n",
      "convert squad examples to features:   0%|          | 0/10 [00:00<?, ?it/s]/Users/asisto/dev/Searchable/Research/KitanaQA/env/lib/python3.7/site-packages/transformers-3.1.0-py3.7.egg/transformers/tokenization_utils_base.py:1321: FutureWarning: The `max_len` attribute has been deprecated and will be removed in a future version, use `model_max_length` instead.\n",
      "  FutureWarning,\n",
      "convert squad examples to features: 100%|██████████| 10/10 [00:00<00:00, 107.04it/s]\n",
      "add example index and unique id: 100%|██████████| 10/10 [00:00<00:00, 29767.95it/s]\n",
      "11/12/2020 18:22:38 - kitanaqa.trainer.utils - INFO - Feature Extraction for Evaluation Data from dev-v1.1 is Finished.\n",
      "11/12/2020 18:22:38 - INFO - kitanaqa.trainer.utils -   Feature Extraction for Evaluation Data from dev-v1.1 is Finished.\n",
      "11/12/2020 18:22:38 - kitanaqa.trainer.utils - INFO - Saving features into cached file ./example_outputs/cached_dev_distilbert-base-uncased_512\n",
      "11/12/2020 18:22:38 - INFO - kitanaqa.trainer.utils -   Saving features into cached file ./example_outputs/cached_dev_distilbert-base-uncased_512\n",
      "11/12/2020 18:22:38 - kitanaqa.trainer.utils - INFO - Predict Sets are : dict_keys(['dev-v1.1'])\n",
      "11/12/2020 18:22:38 - INFO - kitanaqa.trainer.utils -   Predict Sets are : dict_keys(['dev-v1.1'])\n",
      "11/12/2020 18:22:38 - kitanaqa - INFO - ***** Running evaluation distilbert-base-uncased *****\n",
      "11/12/2020 18:22:38 - INFO - kitanaqa -   ***** Running evaluation distilbert-base-uncased *****\n",
      "11/12/2020 18:22:38 - kitanaqa - INFO -   Num examples = 10\n",
      "11/12/2020 18:22:38 - INFO - kitanaqa -     Num examples = 10\n",
      "11/12/2020 18:22:38 - kitanaqa - INFO -   Batch size = 1\n",
      "11/12/2020 18:22:38 - INFO - kitanaqa -     Batch size = 1\n",
      "Evaluating: 100%|██████████| 10/10 [00:05<00:00,  1.77it/s]\n",
      "11/12/2020 18:22:44 - kitanaqa - INFO -   Evaluation done in total 5.642654 secs (0.564265 sec per example)\n",
      "11/12/2020 18:22:44 - INFO - kitanaqa -     Evaluation done in total 5.642654 secs (0.564265 sec per example)\n",
      "11/12/2020 18:22:44 - kitanaqa.trainer.utils - INFO - The evaluation for dev-v1.1 dataset is finished.\n",
      "11/12/2020 18:22:44 - INFO - kitanaqa.trainer.utils -   The evaluation for dev-v1.1 dataset is finished.\n",
      "11/12/2020 18:22:44 - kitanaqa.trainer.utils - INFO - Results: {'dev-v1.1': {'uncased': {'model_args': ModelArguments(model_name_or_path='distilbert-base-uncased', train_file_path='/Users/asisto/dev/Searchable/Research/KitanaQA/src/kitanaqa/support/unittest-squad.json', predict_file_path={'dev-v1.1': '/Users/asisto/dev/Searchable/Research/KitanaQA/src/kitanaqa/support/unittest-squad.json'}, model_type='distilbert', aug_file_path=None, tokenizer_name_or_path=None, cache_dir='./example_outputs', do_lower_case=True, version_2_with_negative=False, null_score_diff_threshold=0.0, n_best_size=20, verbose_logging=False, overwrite_cache=False, max_seq_length=512, max_query_length=64, max_answer_length=30, doc_stride=128, freeze_embeds=False, do_aug=False, do_adv_eval=False, do_alum=True, eta=1e-05, alpha=5, alpha_final=None, alpha_schedule=None, eps=0.0001, sigma=0.001, K=1, data_dir='./example_outputs', eval_all_checkpoints=False), 'training_args': TrainingArguments(output_dir='./example_outputs', overwrite_output_dir=False, do_train=False, do_eval=True, do_predict=False, evaluate_during_training=False, prediction_loss_only=True, per_device_train_batch_size=2, per_device_eval_batch_size=1, per_gpu_train_batch_size=None, per_gpu_eval_batch_size=None, gradient_accumulation_steps=2, learning_rate=5e-05, weight_decay=0.0, adam_beta1=0.9, adam_beta2=0.999, adam_epsilon=1e-08, max_grad_norm=1.0, num_train_epochs=1, max_steps=2, warmup_steps=0, logging_dir='runs/Nov12_18-21-23_Aarons-MBP-7.attlocal.net', logging_first_step=False, logging_steps=500, save_steps=1, save_total_limit=None, no_cuda=False, seed=512, fp16=False, fp16_opt_level='O1', local_rank=-1, tpu_num_cores=None, tpu_metrics_debug=False, debug=False, dataloader_drop_last=False, eval_steps=1000, past_index=-1, run_name=None, disable_tqdm=False, remove_unused_columns=True), 'eval': OrderedDict([('exact', 0.0), ('f1', 9.023809523809524), ('total', 10), ('HasAns_exact', 0.0), ('HasAns_f1', 9.023809523809524), ('HasAns_total', 10), ('best_exact', 0.0), ('best_exact_thresh', 0.0), ('best_f1', 9.023809523809524), ('best_f1_thresh', 0.0)])}}}\n",
      "11/12/2020 18:22:44 - INFO - kitanaqa.trainer.utils -   Results: {'dev-v1.1': {'uncased': {'model_args': ModelArguments(model_name_or_path='distilbert-base-uncased', train_file_path='/Users/asisto/dev/Searchable/Research/KitanaQA/src/kitanaqa/support/unittest-squad.json', predict_file_path={'dev-v1.1': '/Users/asisto/dev/Searchable/Research/KitanaQA/src/kitanaqa/support/unittest-squad.json'}, model_type='distilbert', aug_file_path=None, tokenizer_name_or_path=None, cache_dir='./example_outputs', do_lower_case=True, version_2_with_negative=False, null_score_diff_threshold=0.0, n_best_size=20, verbose_logging=False, overwrite_cache=False, max_seq_length=512, max_query_length=64, max_answer_length=30, doc_stride=128, freeze_embeds=False, do_aug=False, do_adv_eval=False, do_alum=True, eta=1e-05, alpha=5, alpha_final=None, alpha_schedule=None, eps=0.0001, sigma=0.001, K=1, data_dir='./example_outputs', eval_all_checkpoints=False), 'training_args': TrainingArguments(output_dir='./example_outputs', overwrite_output_dir=False, do_train=False, do_eval=True, do_predict=False, evaluate_during_training=False, prediction_loss_only=True, per_device_train_batch_size=2, per_device_eval_batch_size=1, per_gpu_train_batch_size=None, per_gpu_eval_batch_size=None, gradient_accumulation_steps=2, learning_rate=5e-05, weight_decay=0.0, adam_beta1=0.9, adam_beta2=0.999, adam_epsilon=1e-08, max_grad_norm=1.0, num_train_epochs=1, max_steps=2, warmup_steps=0, logging_dir='runs/Nov12_18-21-23_Aarons-MBP-7.attlocal.net', logging_first_step=False, logging_steps=500, save_steps=1, save_total_limit=None, no_cuda=False, seed=512, fp16=False, fp16_opt_level='O1', local_rank=-1, tpu_num_cores=None, tpu_metrics_debug=False, debug=False, dataloader_drop_last=False, eval_steps=1000, past_index=-1, run_name=None, disable_tqdm=False, remove_unused_columns=True), 'eval': OrderedDict([('exact', 0.0), ('f1', 9.023809523809524), ('total', 10), ('HasAns_exact', 0.0), ('HasAns_f1', 9.023809523809524), ('HasAns_total', 10), ('best_exact', 0.0), ('best_exact_thresh', 0.0), ('best_f1', 9.023809523809524), ('best_f1_thresh', 0.0)])}}}\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[2020-11-13 02:22:44] INFO - prefect.TaskRunner | Task 'eval': finished task run for task with final state: 'Success'\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "11/12/2020 18:22:44 - INFO - prefect.TaskRunner -   Task 'eval': finished task run for task with final state: 'Success'\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[2020-11-13 02:22:44] INFO - prefect.FlowRunner | Flow run SUCCESS: all reference tasks succeeded\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "11/12/2020 18:22:44 - INFO - prefect.FlowRunner -   Flow run SUCCESS: all reference tasks succeeded\n"
     ]
    }
   ],
   "source": [
    "# Construct a Flow pipeline consisting of a parent step (alum training) conditionally followed by a child step (evaluation)\n",
    "f = build_flow(\n",
    "            (model_args, training_args),\n",
    "            model=model,\n",
    "            tokenizer=tokenizer,\n",
    "            train_dataset=train_dataset)\n",
    "if f:\n",
    "    f.run()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "f.result"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}