12_Organic19_Experiment_Single.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import math\n",
    "import time\n",
    "import logging\n",
    "from data.data_loader import Dataset\n",
    "from misc.preferences import PREFERENCES\n",
    "from misc.run_configuration import get_default_params, OutputLayerType, LearningSchedulerType, OptimizerType, good_organic_hp_params\n",
    "from misc import utils\n",
    "\n",
    "from optimizer import get_optimizer\n",
    "from criterion import NllLoss, LossCombiner\n",
    "\n",
    "from models.transformer.encoder import TransformerEncoder\n",
    "from models.jointAspectTagger import JointAspectTagger\n",
    "from trainer.train import Trainer\n",
    "import pprint\n",
    "from data.organic2019 import organic_dataset as dsl\n",
    "from data.organic2019 import ORGANIC_TASK_ALL, ORGANIC_TASK_ENTITIES, ORGANIC_TASK_ATTRIBUTES, ORGANIC_TASK_ENTITIES_COMBINE, ORGANIC_TASK_COARSE"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Loading Functions\n",
    "\n",
    "These functions will load the dataset and the model. The run configuration will determine the architecture and hyper parameters."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def load_model(dataset, rc, experiment_name):\n",
    "    loss = LossCombiner(4, dataset.class_weights, NllLoss)\n",
    "    transformer = TransformerEncoder(dataset.source_embedding,\n",
    "                                     hyperparameters=rc)\n",
    "    model = JointAspectTagger(transformer, rc, 4, 20, dataset.target_names)\n",
    "    optimizer = get_optimizer(model, rc)\n",
    "    trainer = Trainer(\n",
    "                        model,\n",
    "                        loss,\n",
    "                        optimizer,\n",
    "                        rc,\n",
    "                        dataset,\n",
    "                        experiment_name,\n",
    "                        enable_tensorboard=False,\n",
    "                        verbose=False)\n",
    "    return trainer\n",
    "\n",
    "def load_dataset(rc, logger, task):\n",
    "    dataset = Dataset(\n",
    "        task,\n",
    "        logger,\n",
    "        rc,\n",
    "        source_index=0,\n",
    "        target_vocab_index=1,\n",
    "        data_path=PREFERENCES.data_root,\n",
    "        train_file=PREFERENCES.data_train,\n",
    "        valid_file=PREFERENCES.data_validation,\n",
    "        test_file=PREFERENCES.data_test,\n",
    "        file_format='.tsv',\n",
    "        init_token=None,\n",
    "        eos_token=None\n",
    "    )\n",
    "    dataset.load_data(dsl, verbose=False)\n",
    "    return dataset"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Preamble - Definitions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "PREFERENCES.defaults(\n",
    "    data_root='./data/data/organic2019',\n",
    "    data_train='train.csv',    \n",
    "    data_validation='validation.csv',\n",
    "    data_test='test.csv',\n",
    "    early_stopping='highest_5_F1'\n",
    ")\n",
    "main_experiment_name = 'Organic19_Experiments'\n",
    "use_cuda = True\n",
    "STATUS_FAIL = 'fail'\n",
    "STATUS_OK = 'ok'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    " # Definition of experiments\n",
    " \n",
    " - SpellChecker On\n",
    " - Fasttext\n",
    " - Single Sentence\n",
    " - Combined Sentence\n",
    " - Stop Words"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{ 'att_d_k': 150,\n",
      "  'att_d_v': 150,\n",
      "  'batch_size': 64,\n",
      "  'clip_comments_to': 169,\n",
      "  'dropout_rate': 0.2827482,\n",
      "  'early_stopping': 5,\n",
      "  'embedding_dim': 300,\n",
      "  'embedding_name': '6B',\n",
      "  'embedding_type': 'glove',\n",
      "  'language': 'en',\n",
      "  'learning_rate_scheduler': { 'noam_learning_rate_factor': 2.528475,\n",
      "                               'noam_learning_rate_warmup': 5499},\n",
      "  'learning_rate_scheduler_type': <LearningSchedulerType.Noam: 1>,\n",
      "  'log_every_xth_iteration': -1,\n",
      "  'model_size': 300,\n",
      "  'num_encoder_blocks': 2,\n",
      "  'num_epochs': 35,\n",
      "  'num_heads': 2,\n",
      "  'optimizer': { 'adam_beta1': 0.9206513739,\n",
      "                 'adam_beta2': 0.9392212929,\n",
      "                 'adam_eps': 1.317166484e-05,\n",
      "                 'adam_weight_decay': 0.0001,\n",
      "                 'learning_rate': 0.0097},\n",
      "  'optimizer_type': <OptimizerType.Adam: 1>,\n",
      "  'output_dropout_rate': 0.39743837844,\n",
      "  'output_layer_type': <OutputLayerType.LinearSum: 1>,\n",
      "  'pointwise_layer_size': 129,\n",
      "  'task': 'entities',\n",
      "  'use_bias': True,\n",
      "  'use_spell_checkers': False}\n"
     ]
    }
   ],
   "source": [
    "baseline = good_organic_hp_params\n",
    "print(pprint.pformat(baseline, indent=2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "experiments = [\n",
    "    {\n",
    "        'name': 'Baseline',\n",
    "        'description': 'Baseline. Uses good_organic_hp_params without any changes',\n",
    "        'loss': 1000,\n",
    "        'f1': -1,\n",
    "        'rc': {'task': ORGANIC_TASK_COARSE}\n",
    "    }\n",
    "]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Current commit"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Current commit: b'03827f7'\n"
     ]
    }
   ],
   "source": [
    "utils.get_current_git_commit()\n",
    "print('Current commit: ' + utils.get_current_git_commit())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def objective(rc, experiment):\n",
    "    run_time = time.time()\n",
    "    \n",
    "    # reset loggers\n",
    "    utils.reset_loggers()\n",
    "    experiment_name = utils.create_loggers(experiment_name=main_experiment_name)\n",
    "    logger = logging.getLogger(__name__)\n",
    "    dataset_logger = logging.getLogger('data_loader')\n",
    "    \n",
    "    logger.info('Experiment: ' + experiment['name'])\n",
    "    logger.info('Description: ' + experiment['description'])\n",
    "    \n",
    "    logger.info('Parameters')\n",
    "    logger.info(rc)\n",
    "    print('\\n\\n#########################################################################')\n",
    "    print('Name: ' + experiment['name'])\n",
    "    print('Description: ' + experiment['description'])\n",
    "    print('#########################################################################\\n\\n')\n",
    "    print(rc)\n",
    "\n",
    "    logger.debug('Load dataset')\n",
    "    try:\n",
    "        dataset = load_dataset(rc, dataset_logger, rc.task)\n",
    "    except Exception as err:\n",
    "        print('Could load dataset: ' + str(err))\n",
    "        logger.exception(\"Could not load dataset\")\n",
    "        return {\n",
    "            'status': STATUS_FAIL,\n",
    "            'eval_time': time.time() - run_time\n",
    "        }\n",
    "    logger.debug('dataset loaded')\n",
    "    logger.debug('Load model')\n",
    "\n",
    "    try:\n",
    "        trainer = load_model(dataset, rc, experiment_name)\n",
    "    except Exception as err:\n",
    "        print('Could not load model: ' + str(err))\n",
    "        logger.exception(\"Could not load model\")\n",
    "        return {\n",
    "            'status': STATUS_FAIL,\n",
    "            'eval_time': time.time() - run_time\n",
    "        }\n",
    "\n",
    "    logger.debug('model loaded')\n",
    "\n",
    "    logger.debug('Begin training')\n",
    "    model = None\n",
    "    try:\n",
    "        result = trainer.train(use_cuda=rc.use_cuda, perform_evaluation=False)\n",
    "        model = result['model']\n",
    "    except Exception as err:\n",
    "        print('Exception while training: ' + str(err))\n",
    "        logger.exception(\"Could not complete iteration\")\n",
    "        return {\n",
    "            'status': STATUS_FAIL,\n",
    "            'eval_time': time.time() - run_time,\n",
    "            'best_loss': trainer.get_best_loss(),\n",
    "            'best_f1': trainer.get_best_f1()\n",
    "        }\n",
    "\n",
    "    if math.isnan(trainer.get_best_loss()):\n",
    "        print('Loss is nan')\n",
    "        return {\n",
    "            'status': STATUS_FAIL,\n",
    "            'eval_time': time.time() - run_time,\n",
    "            'best_loss': trainer.get_best_loss(),\n",
    "            'best_f1': trainer.get_best_f1()\n",
    "        }\n",
    "\n",
    "    # perform evaluation and log results\n",
    "    result = None\n",
    "    try:\n",
    "        result = trainer.perform_final_evaluation(use_test_set=True, verbose=False)\n",
    "    except Exception as err:\n",
    "        logger.exception(\"Could not complete iteration evaluation.\")\n",
    "        print('Could not complete iteration evaluation: ' + str(err))\n",
    "        return {\n",
    "            'status': STATUS_FAIL,\n",
    "            'eval_time': time.time() - run_time,\n",
    "            'best_loss': trainer.get_best_loss(),\n",
    "            'best_f1': trainer.get_best_f1()\n",
    "        }\n",
    "    print(f'VAL f1\\t{trainer.get_best_f1()} - ({result[1][1]})')\n",
    "    print(f'VAL loss\\t{trainer.get_best_loss()}')\n",
    "    return {\n",
    "            'loss': result[1][0],\n",
    "            'status': STATUS_OK,\n",
    "            'eval_time': time.time() - run_time,\n",
    "            'best_loss': trainer.get_best_loss(),\n",
    "            'best_f1': trainer.get_best_f1(),\n",
    "            'sample_iterations': trainer.get_num_samples_seen(),\n",
    "            'iterations': trainer.get_num_iterations(),\n",
    "            'rc': rc,\n",
    "            'results': {\n",
    "                'train': {\n",
    "                    'loss': result[0][0],\n",
    "                    'f1': result[0][1]\n",
    "                },\n",
    "                'validation': {\n",
    "                    'loss': result[1][0],\n",
    "                    'f1': result[1][1]\n",
    "                },\n",
    "                'test': {\n",
    "                    'loss': result[2][0],\n",
    "                    'f1': result[2][1]\n",
    "                }\n",
    "            }\n",
    "        }"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Perform experiments"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "#########################################################################\n",
      "\n",
      "Experiment Name: Baseline\n",
      "\n",
      "#########################################################################\n",
      "\n",
      "\n",
      "Log path is  C:\\Users\\felix\\OneDrive\\Studium\\Studium\\6. Semester\\MA\\Project\\ABSA-Transformer\\logs\\Organic19_Experiments\\20190401\\3\n",
      "\n",
      "\n",
      "#########################################################################\n",
      "Name: Baseline\n",
      "Description: Baseline. Uses good_organic_hp_params without any changes\n",
      "#########################################################################\n",
      "\n",
      "\n",
      "+----------------------------------------------------------------------------------+\n",
      "|                                 Hyperparameters                                  |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|          Parameter           |                       Value                       |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|            kwargs            | {'embedding_type': 'glove', 'learning_ra[...]se'} |\n",
      "|          model_size          |                        300                        |\n",
      "|        early_stopping        |                         5                         |\n",
      "|           use_cuda           |                        True                       |\n",
      "|             task             |                       coarse                      |\n",
      "|          batch_size          |                         64                        |\n",
      "| learning_rate_scheduler_type |             LearningSchedulerType.Noam            |\n",
      "|      output_layer_type       |             OutputLayerType.LinearSum             |\n",
      "|        optimizer_type        |                 OptimizerType.Adam                |\n",
      "|        learning_rate         |                       0.0097                      |\n",
      "|  noam_learning_rate_warmup   |                        5499                       |\n",
      "|  noam_learning_rate_factor   |                      2.528475                     |\n",
      "|          adam_beta1          |                    0.9206513739                   |\n",
      "|          adam_beta2          |                    0.9392212929                   |\n",
      "|           adam_eps           |                  1.317166484e-05                  |\n",
      "|      adam_weight_decay       |                       0.0001                      |\n",
      "|         adam_amsgrad         |                       False                       |\n",
      "|           use_bias           |                       False                       |\n",
      "|         n_enc_blocks         |                         2                         |\n",
      "|           n_heads            |                         2                         |\n",
      "|             d_k              |                        150                        |\n",
      "|             d_v              |                        150                        |\n",
      "|         dropout_rate         |                     0.2827482                     |\n",
      "|     pointwise_layer_size     |                        129                        |\n",
      "|      last_layer_dropout      |                   0.39743837844                   |\n",
      "|   log_every_xth_iteration    |                         -1                        |\n",
      "|          num_epochs          |                         35                        |\n",
      "|        embedding_type        |                       glove                       |\n",
      "|        embedding_name        |                         6B                        |\n",
      "|        embedding_dim         |                        300                        |\n",
      "|       clip_comments_to       |                        169                        |\n",
      "|      finetune_embedding      |                        True                       |\n",
      "|           language           |                         en                        |\n",
      "|        use_stop_words        |                        True                       |\n",
      "|         use_stemming         |                       False                       |\n",
      "|        harmonize_bahn        |                       False                       |\n",
      "|      use_spell_checkers      |                       False                       |\n",
      "|      replace_url_tokens      |                        True                       |\n",
      "|       use_text_cleaner       |                       False                       |\n",
      "|     contraction_removal      |                       False                       |\n",
      "|    organic_text_cleaning     |                        True                       |\n",
      "|             seed             |                         42                        |\n",
      "+------------------------------+---------------------------------------------------+\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c14a6ca17d6f43deb0db1c737fc37600",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "# EP\t# IT\ttr loss\t\tval loss\tf1\t\tacc\t\tduration / total time\n",
      "1\t8k\t0.32\t\t0.19\t\t0.027\t\t0.951\t\t0.46m - 0.5m / 0.0m\n",
      "2\t17k\t0.24\t\t0.18\t\t0.111\t\t0.935\t\t0.44m - 1.0m / 16.3m\n",
      "3\t25k\t0.22\t\t0.17\t\t0.132\t\t0.913\t\t0.44m - 1.5m / 15.6m\n",
      "4\t34k\t0.19\t\t0.16\t\t0.164\t\t0.911\t\t0.45m - 2.0m / 15.7m\n",
      "5\t42k\t0.18\t\t0.18\t\t0.120\t\t0.855\t\t0.45m - 2.5m / 15.9m\n",
      "6\t51k\t0.16\t\t0.17\t\t0.170\t\t0.894\t\t0.44m - 3.0m / 15.9m\n",
      "7\t59k\t0.15\t\t0.17\t\t0.141\t\t0.885\t\t0.44m - 3.5m / 15.9m\n",
      "8\t68k\t0.14\t\t0.18\t\t0.133\t\t0.878\t\t0.45m - 4.0m / 16.0m\n",
      "9\t76k\t0.13\t\t0.21\t\t0.115\t\t0.843\t\t0.44m - 4.5m / 16.1m\n",
      "10\t84k\t0.12\t\t0.19\t\t0.147\t\t0.888\t\t0.45m - 5.0m / 16.1m\n",
      "11\t93k\t0.11\t\t0.22\t\t0.136\t\t0.888\t\t0.44m - 5.5m / 16.3m\n",
      "VAL f1\t0.17002724795640328 - (0.17002724795640328)\n",
      "VAL loss\t0.16096799044559398\n",
      "       .---.\n",
      "          /     \\\n",
      "          \\.@-@./\tExperiment: Baseline\n",
      "          /`\\_/`\\\tStatus: ok\n",
      "         //  _  \\\\\tLoss: 0.16096799044559398\n",
      "        | \\     )|_\tf1: 0.17002724795640328\n",
      "       /`\\_`>  <_/ \\\n",
      "       \\__/'---'\\__/\n",
      "\n"
     ]
    }
   ],
   "source": [
    "for e in experiments:\n",
    "    name = e['name']\n",
    "    print(f'#########################################################################\\n\\nExperiment Name: {name}\\n')\n",
    "    print('#########################################################################\\n\\n')\n",
    "    \n",
    "    # generate rc\n",
    "    rc = get_default_params(use_cuda=True, overwrite=e['rc'], from_default=baseline)\n",
    "    result = objective(rc, e)\n",
    "    \n",
    "    if result['status'] == STATUS_OK:\n",
    "        print(f\"       .---.\\n \\\n",
    "         /     \\\\\\n\\\n",
    "          \\\\.@-@./\\tExperiment: {e['name']}\\n\\\n",
    "          /`\\\\_/`\\\\\\tStatus: {result['status']}\\n\\\n",
    "         //  _  \\\\\\\\\\tLoss: {result['best_loss']}\\n\\\n",
    "        | \\\\     )|_\\tf1: {result['best_f1']}\\n\\\n",
    "       /`\\\\_`>  <_/ \\\\\\n\\\n",
    "       \\\\__/'---'\\\\__/\\n\")\n",
    "    else:\n",
    "        print(f\"       .---.\\n \\\n",
    "         /     \\\\\\n\\\n",
    "          \\\\.@-@./\\tExperiment: {e['name']} (FAIL)\\n\\\n",
    "          /`\\\\_/`\\\\\\n\\\n",
    "         //  _  \\\\\\\\\\\\n\\\n",
    "        | \\\\     )|_\\n\\\n",
    "       /`\\\\_`>  <_/ \\\\\\n\\\n",
    "       \\\\__/'---'\\\\__/\\n\")\n",
    "    \n",
    "\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}