look_at_babi_plus.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from operator import itemgetter\n",
    "from os import path\n",
    "import re\n",
    "\n",
    "from config import CONFIG\n",
    "from lib.babi import read_task"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def count_hesitations(in_turns):\n",
    "    hesitate_starts = map(lambda template: template.split()[0], CONFIG['action_templates']['hesitate'])\n",
    "    user_turns = filter(lambda x: x['agent'] == 'user', in_turns)\n",
    "    hesitation_triggers = 0\n",
    "    hesitation_turns = 0\n",
    "    hesitation_cases = set([])\n",
    "    for turn in user_turns:\n",
    "        tokens = turn['text'].strip().split()\n",
    "        hesitation_triggered = False\n",
    "        for token_i, token_j in zip(tokens[:], tokens[1:]):\n",
    "            if token_i in hesitate_starts and token_j not in ['sorry', 'yeah']:\n",
    "                hesitation_cases.add(' '.join([token_i, token_j]))\n",
    "                hesitation_triggers += 1\n",
    "                if not hesitation_triggered:\n",
    "                    hesitation_triggered = True\n",
    "                    hesitation_turns += 1\n",
    "    # print '\\n'.join(hesitation_cases)\n",
    "    return hesitation_triggers, hesitation_turns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def count_restarts(in_turns):\n",
    "    user_turns = filter(lambda x: x['agent'] == 'user', in_turns)\n",
    "    restart_triggers = 0\n",
    "    restart_turns = 0\n",
    "    restart_cases = set([])\n",
    "    for turn in user_turns:\n",
    "        tokens = turn['text'].strip().split()\n",
    "        restart_triggered = False\n",
    "        for token_i, token_j, token_k in zip(tokens[:], tokens[1:], tokens[2:]):\n",
    "            if token_i in ['uhm', 'sorry'] and token_j in ['yeah']:\n",
    "                restart_cases.add(' '.join([token_i, token_j, token_k]))\n",
    "                restart_triggers += 1\n",
    "                if not restart_triggered:\n",
    "                    restart_triggered = True\n",
    "                    restart_turns += 1\n",
    "    # print '\\n'.join(restart_cases)\n",
    "    return restart_triggers, restart_turns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def count_corrections(in_turns, in_slot_values):\n",
    "    correction_starts = [\n",
    "        filter(lambda token: not token.startswith('$'), phrase.split())\n",
    "        for phrase in CONFIG['action_templates']['correct']\n",
    "    ]\n",
    "    correction_cases = set([])\n",
    "    user_turns = filter(lambda x: x['agent'] == 'user', in_turns)\n",
    "    correction_triggers = 0\n",
    "    correction_turns = 0\n",
    "    for turn in user_turns:\n",
    "        tokens = turn['text'].strip().split()\n",
    "        correction_triggered = False\n",
    "        for token_i, token_j, token_k in zip(tokens[:], tokens[1:], tokens[2:]):\n",
    "            if ([token_i] in correction_starts and token_j in in_slot_values) or ([token_i, token_j] in correction_starts and token_k in in_slot_values) :\n",
    "                correction_cases.add(' '.join([token_i, token_j, token_k]))\n",
    "                correction_triggers += 1\n",
    "                if not correction_triggered:\n",
    "                    correction_triggered = True\n",
    "                    correction_turns += 1\n",
    "    # print '\\n'.join(correction_cases)\n",
    "    return correction_triggers, correction_turns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def count_long_distance_corrections(in_turns, in_slot_values):\n",
    "    correction_starts = [\n",
    "        filter(lambda token: not token.startswith('$'), phrase.split())\n",
    "        for phrase in CONFIG['action_templates']['correct_long_distance']\n",
    "    ]\n",
    "    correction_cases = set([])\n",
    "    user_turns = filter(lambda x: x['agent'] == 'user', in_turns)\n",
    "    correction_triggers = 0\n",
    "    correction_turns = 0\n",
    "    prepositions = ['with', 'for', 'in', 'a']\n",
    "    for turn in user_turns:\n",
    "        tokens = turn['text'].strip().split()\n",
    "        correction_triggered = False\n",
    "        for token_i, token_j, token_k in zip(tokens[:], tokens[1:], tokens[2:]):\n",
    "            if ([token_i] in correction_starts and token_j in prepositions and token_k in in_slot_values) or ([token_i, token_j] in correction_starts and token_k in prepositions):\n",
    "                correction_cases.add(' '.join([token_i, token_j, token_k]))\n",
    "                correction_triggers += 1\n",
    "                if not correction_triggered:\n",
    "                    correction_triggered = True\n",
    "                    correction_turns += 1\n",
    "    # print '\\n'.join(correction_cases)\n",
    "    return correction_triggers, correction_turns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def collect_corpus_stats(in_corpus_root):\n",
    "    task1_train = read_task(path.join(in_corpus_root, 'dialog-babi-task1-API-calls-trn.txt'))\n",
    "    task1_dev = read_task(path.join(in_corpus_root, 'dialog-babi-task1-API-calls-dev.txt'))\n",
    "    task1_test = read_task(path.join(in_corpus_root, 'dialog-babi-task1-API-calls-tst.txt'))\n",
    "    task1_oov_test = read_task(path.join(in_corpus_root, 'dialog-babi-task1-API-calls-tst-OOV.txt'))\n",
    "\n",
    "    data_all = task1_train + task1_dev + task1_test + task1_oov_test\n",
    "    turns_all = []\n",
    "    for dialogue in data_all:\n",
    "        turns_all += dialogue[1]\n",
    "\n",
    "    slot_values = set([])\n",
    "    for turn in filter(lambda x: x['text'].startswith('api_call'), turns_all):\n",
    "        slot_values.update(turn['text'].split()[1:])\n",
    "    result = {}\n",
    "    result['hesitate'] = count_hesitations(turns_all)\n",
    "    result['restart'] = count_restarts(turns_all)\n",
    "    result['correct'] = count_corrections(turns_all, slot_values)\n",
    "    result['correct_long_distance'] = count_long_distance_corrections(turns_all, slot_values)\n",
    "\n",
    "    print 'Overall number of turns:\\t{}'.format(len(turns_all))\n",
    "    user_turns = filter(lambda x: x['agent'] == 'user', turns_all)\n",
    "    print 'Among them user\\'s turns:\\t{}'.format(len(user_turns))\n",
    "\n",
    "    print 'Number of hesitations triggered:\\t{}'.format(result['hesitate'][0])\n",
    "    print 'User turns containing hesitations:\\t{0} ({1:.2f}%)'.format(result['hesitate'][1], 100. * result['hesitate'][1] / len(user_turns))\n",
    "\n",
    "    print 'Number of restarts triggered:\\t{}'.format(result['restart'][0])\n",
    "    print 'User turns containing restarts:\\t{0} ({1:.2f}%)'.format(result['restart'][1], 100. * result['restart'][1] / len(user_turns))\n",
    "\n",
    "    print 'Number of corrections triggered:\\t{}'.format(result['correct'][0])\n",
    "    print 'User turns containing corrections:\\t{0} ({1:.2f}%)'.format(result['correct'][1], 100. * result['correct'][1] / len(user_turns))\n",
    "\n",
    "    print 'Number of long-distance corrections triggered:\\t{}'.format(result['correct_long_distance'][0])\n",
    "    print 'User turns containing long-distance corrections:\\t{0} ({1:.2f}%)'.format(result['correct_long_distance'][1], 100. * result['correct_long_distance'][1] / len(user_turns))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "bAbI+\n",
    "=="
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Overall number of turns:\t47990\n",
      "Among them user's turns:\t23995\n",
      "Number of hesitations triggered:\t1194\n",
      "User turns containing hesitations:\t1167 (4.86%)\n",
      "Number of restarts triggered:\t9579\n",
      "User turns containing restarts:\t9579 (39.92%)\n",
      "Number of corrections triggered:\t4128\n",
      "User turns containing corrections:\t2556 (10.65%)\n",
      "Number of long-distance corrections triggered:\t4014\n",
      "User turns containing long-distance corrections:\t2476 (10.32%)\n"
     ]
    }
   ],
   "source": [
    "collect_corpus_stats('babi_plus')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "bAbI+ 2x\n",
    "=="
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Overall number of turns:\t95980\n",
      "Among them user's turns:\t47990\n",
      "Number of hesitations triggered:\t2402\n",
      "User turns containing hesitations:\t2334 (4.86%)\n",
      "Number of restarts triggered:\t19254\n",
      "User turns containing restarts:\t19254 (40.12%)\n",
      "Number of corrections triggered:\t8357\n",
      "User turns containing corrections:\t5190 (10.81%)\n",
      "Number of long-distance corrections triggered:\t7961\n",
      "User turns containing long-distance corrections:\t4890 (10.19%)\n"
     ]
    }
   ],
   "source": [
    "collect_corpus_stats('babi_plus_2x')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "bAbI+ 5x\n",
    "=="
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Overall number of turns:\t239950\n",
      "Among them user's turns:\t119975\n",
      "Number of hesitations triggered:\t6180\n",
      "User turns containing hesitations:\t6018 (5.02%)\n",
      "Number of restarts triggered:\t47894\n",
      "User turns containing restarts:\t47894 (39.92%)\n",
      "Number of corrections triggered:\t20328\n",
      "User turns containing corrections:\t12789 (10.66%)\n",
      "Number of long-distance corrections triggered:\t19810\n",
      "User turns containing long-distance corrections:\t12214 (10.18%)\n"
     ]
    }
   ],
   "source": [
    "collect_corpus_stats('babi_plus_5x')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "bAbI+ 10x\n",
    "=="
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Overall number of turns:\t479900\n",
      "Among them user's turns:\t239950\n",
      "Number of hesitations triggered:\t11953\n",
      "User turns containing hesitations:\t11664 (4.86%)\n",
      "Number of restarts triggered:\t96129\n",
      "User turns containing restarts:\t96129 (40.06%)\n",
      "Number of corrections triggered:\t41286\n",
      "User turns containing corrections:\t25920 (10.80%)\n",
      "Number of long-distance corrections triggered:\t39608\n",
      "User turns containing long-distance corrections:\t24465 (10.20%)\n"
     ]
    }
   ],
   "source": [
    "collect_corpus_stats('babi_plus_10x')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "bAbI+ 50x\n",
    "=="
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Overall number of turns:\t2399500\n",
      "Among them user's turns:\t1199750\n",
      "Number of hesitations triggered:\t60100\n",
      "User turns containing hesitations:\t58483 (4.87%)\n",
      "Number of restarts triggered:\t479221\n",
      "User turns containing restarts:\t479221 (39.94%)\n",
      "Number of corrections triggered:\t205220\n",
      "User turns containing corrections:\t128796 (10.74%)\n",
      "Number of long-distance corrections triggered:\t199215\n",
      "User turns containing long-distance corrections:\t123417 (10.29%)\n"
     ]
    }
   ],
   "source": [
    "collect_corpus_stats('babi_plus_50x')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "bAbI+ 100x\n",
    "=="
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Overall number of turns:\t4799000\n",
      "Among them user's turns:\t2399500\n",
      "Number of hesitations triggered:\t119308\n",
      "User turns containing hesitations:\t116042 (4.84%)\n",
      "Number of restarts triggered:\t959610\n",
      "User turns containing restarts:\t959610 (39.99%)\n",
      "Number of corrections triggered:\t412214\n",
      "User turns containing corrections:\t258775 (10.78%)\n",
      "Number of long-distance corrections triggered:\t398451\n",
      "User turns containing long-distance corrections:\t246918 (10.29%)\n"
     ]
    }
   ],
   "source": [
    "collect_corpus_stats('babi_plus_100x')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}