notebooks/tutorials/5. Testing transformer pipelines.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import checklist\n",
    "from checklist.test_suite import TestSuite\n",
    "import logging\n",
    "logging.basicConfig(level=logging.ERROR)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from transformers import pipeline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Sentiment analysis"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = pipeline(\"sentiment-analysis\", device=0)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Add some batching, I run out of memory otherwise:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def chunks(l, n):\n",
    "    \"\"\"Yield successive n-sized chunks from l.\"\"\"\n",
    "    for i in range(0, len(l), n):\n",
    "        yield l[i:i + n]\n",
    "\n",
    "def batch_predict(model, data, batch_size=128):\n",
    "    ret = []\n",
    "    for d in chunks(data, batch_size):\n",
    "        ret.extend(model(d))\n",
    "    return ret\n",
    "    "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The tests in the paper are for 3-label sentiment analysis (negative, positive, neutral). We adapt this binary model by making everything in the [0.33, 0.66] range be predicted as neutral:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "def pred_and_conf(data):\n",
    "    # change format to softmax, make everything in [0.33, 0.66] range be predicted as neutral\n",
    "    preds = batch_predict(model, data)\n",
    "    pr = np.array([x['score'] if x['label'] == 'POSITIVE' else 1 - x['score'] for x in preds])\n",
    "    pp = np.zeros((pr.shape[0], 3))\n",
    "    margin_neutral = 1/3.\n",
    "    mn = margin_neutral / 2.\n",
    "    neg = pr < 0.5 - mn\n",
    "    pp[neg, 0] = 1 - pr[neg]\n",
    "    pp[neg, 2] = pr[neg]\n",
    "    pos = pr > 0.5 + mn\n",
    "    pp[pos, 0] = 1 - pr[pos]\n",
    "    pp[pos, 2] = pr[pos]\n",
    "    neutral_pos = (pr >= 0.5) * (pr < 0.5 + mn)\n",
    "    pp[neutral_pos, 1] = 1 - (1 / margin_neutral) * np.abs(pr[neutral_pos] - 0.5)\n",
    "    pp[neutral_pos, 2] = 1 - pp[neutral_pos, 1]\n",
    "    neutral_neg = (pr < 0.5) * (pr > 0.5 - mn)\n",
    "    pp[neutral_neg, 1] = 1 - (1 / margin_neutral) * np.abs(pr[neutral_neg] - 0.5)\n",
    "    pp[neutral_neg, 0] = 1 - pp[neutral_neg, 1]\n",
    "    preds = np.argmax(pp, axis=1)\n",
    "    return preds, pp\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "suite_path = '../release_data/sentiment/sentiment_suite.pkl'\n",
    "suite = TestSuite.from_file(suite_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Running single positive words\n",
      "Predicting 34 examples\n",
      "Running single negative words\n",
      "Predicting 35 examples\n",
      "Running single neutral words\n",
      "Predicting 13 examples\n",
      "Running Sentiment-laden words in context\n",
      "Predicting 500 examples\n",
      "Running neutral words in context\n",
      "Predicting 500 examples\n",
      "Running intensifiers\n",
      "Predicting 1000 examples\n",
      "Running reducers\n",
      "Predicting 1000 examples\n",
      "Running change neutral words with BERT\n",
      "Predicting 5046 examples\n",
      "Running add positive phrases\n",
      "Predicting 5500 examples\n",
      "Running add negative phrases\n",
      "Predicting 5500 examples\n",
      "Running add random urls and handles\n",
      "Predicting 11000 examples\n",
      "Running punctuation\n",
      "Predicting 1170 examples\n",
      "Running typos\n",
      "Predicting 1000 examples\n",
      "Running 2 typos\n",
      "Predicting 1000 examples\n",
      "Running contractions\n",
      "Predicting 1038 examples\n",
      "Running change names\n",
      "Predicting 3641 examples\n",
      "Running change locations\n",
      "Predicting 5500 examples\n",
      "Running change numbers\n",
      "Predicting 5500 examples\n",
      "Running used to, but now\n",
      "Predicting 500 examples\n",
      "Running \"used to\" should reduce\n",
      "Predicting 1000 examples\n",
      "Running protected: race\n",
      "Predicting 2000 examples\n",
      "Running protected: sexual\n",
      "Predicting 7000 examples\n",
      "Running protected: religion\n",
      "Predicting 11000 examples\n",
      "Running protected: nationality\n",
      "Predicting 10000 examples\n",
      "Running simple negations: negative\n",
      "Predicting 500 examples\n",
      "Running simple negations: not negative\n",
      "Predicting 500 examples\n",
      "Running simple negations: not neutral is still neutral\n",
      "Predicting 500 examples\n",
      "Running simple negations: I thought x was positive, but it was not (should be negative)\n",
      "Predicting 500 examples\n",
      "Running simple negations: I thought x was negative, but it was not (should be neutral or positive)\n",
      "Predicting 500 examples\n",
      "Running simple negations: but it was not (neutral) should still be neutral\n",
      "Predicting 500 examples\n",
      "Running Hard: Negation of positive with neutral stuff in the middle (should be negative)\n",
      "Predicting 500 examples\n",
      "Running Hard: Negation of negative with neutral stuff in the middle (should be positive or neutral)\n",
      "Predicting 500 examples\n",
      "Running negation of neutral with neutral in the middle, should still neutral\n",
      "Predicting 500 examples\n",
      "Running my opinion is what matters\n",
      "Predicting 500 examples\n",
      "Running Q & A: yes\n",
      "Predicting 500 examples\n",
      "Running Q & A: yes (neutral)\n",
      "Predicting 500 examples\n",
      "Running Q & A: no\n",
      "Predicting 500 examples\n",
      "Running Q & A: no (neutral)\n",
      "Predicting 500 examples\n"
     ]
    }
   ],
   "source": [
    "suite.run(pred_and_conf, n=500)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# suite.visual_summary_table()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Vocabulary\n",
      "\n",
      "single positive words\n",
      "Test cases:      34\n",
      "Fails (rate):    0 (0.0%)\n",
      "\n",
      "\n",
      "single negative words\n",
      "Test cases:      35\n",
      "Fails (rate):    1 (2.9%)\n",
      "\n",
      "Example fails:\n",
      "0.3 0.0 0.7 average\n",
      "----\n",
      "\n",
      "\n",
      "single neutral words\n",
      "Test cases:      13\n",
      "Fails (rate):    13 (100.0%)\n",
      "\n",
      "Example fails:\n",
      "1.0 0.0 0.0 commercial\n",
      "----\n",
      "0.0 0.0 1.0 find\n",
      "----\n",
      "0.0 0.0 1.0 Israeli\n",
      "----\n",
      "\n",
      "\n",
      "Sentiment-laden words in context\n",
      "Test cases:      8658\n",
      "Test cases run:  500\n",
      "Fails (rate):    5 (1.0%)\n",
      "\n",
      "Example fails:\n",
      "0.2 0.0 0.8 This was a creepy flight.\n",
      "----\n",
      "0.1 0.0 0.9 This is an average aircraft.\n",
      "----\n",
      "0.0 0.0 1.0 It is an average staff.\n",
      "----\n",
      "\n",
      "\n",
      "neutral words in context\n",
      "Test cases:      1716\n",
      "Test cases run:  500\n",
      "Fails (rate):    485 (97.0%)\n",
      "\n",
      "Example fails:\n",
      "0.0 0.0 1.0 This was an Australian seat.\n",
      "----\n",
      "0.1 0.0 0.9 This crew was Israeli.\n",
      "----\n",
      "0.0 0.0 1.0 I found this plane.\n",
      "----\n",
      "\n",
      "\n",
      "intensifiers\n",
      "Test cases:      2000\n",
      "Test cases run:  500\n",
      "After filtering: 499 (99.8%)\n",
      "Fails (rate):    14 (2.8%)\n",
      "\n",
      "Example fails:\n",
      "0.3 0.0 0.7 That was an average crew.\n",
      "0.8 0.0 0.2 That was an unbelievably average crew.\n",
      "\n",
      "----\n",
      "0.9 0.0 0.1 It was a creepy crew.\n",
      "0.0 0.0 1.0 It was an absolutely creepy crew.\n",
      "\n",
      "----\n",
      "0.0 0.0 1.0 It was an average staff.\n",
      "0.8 0.0 0.2 It was a particularly average staff.\n",
      "\n",
      "----\n",
      "\n",
      "\n",
      "reducers\n",
      "Test cases:      2000\n",
      "Test cases run:  500\n",
      "After filtering: 1 (0.2%)\n",
      "Fails (rate):    0 (0.0%)\n",
      "\n",
      "\n",
      "change neutral words with BERT\n",
      "Test cases:      500\n",
      "Fails (rate):    49 (9.8%)\n",
      "\n",
      "Example fails:\n",
      "0.9 0.0 0.1 @united 618 was flight out of Houston\n",
      "0.0 0.0 1.0 @united 618 mile flight out of Houston\n",
      "0.1 0.0 0.9 @united 618 direct flight out of Houston\n",
      "\n",
      "----\n",
      "0.3 0.0 0.7 @USAirways oh the irony. A dog who will not spot barking in waiting area is now right in front of me! Please send me a cocktail coupon stat\n",
      "0.9 0.0 0.1 @USAirways oh the irony. A dog who will not spot barking in waiting area is now right in front of me! Please send me another cocktail coupon stat\n",
      "0.0 0.6 0.4 @USAirways oh the irony. A dog who will not spot barking in waiting area is now right in front of me! Please send me this cocktail coupon stat\n",
      "\n",
      "----\n",
      "1.0 0.0 0.0 @USAirways I don't think I've ever had a us airways flight that went smoothly.\n",
      "0.0 0.0 1.0 @USAirways I don't think I've ever had before us airways flight that went smoothly.\n",
      "\n",
      "----\n",
      "\n",
      "\n",
      "add positive phrases\n",
      "Test cases:      500\n",
      "Fails (rate):    0 (0.0%)\n",
      "\n",
      "\n",
      "add negative phrases\n",
      "Test cases:      500\n",
      "Fails (rate):    34 (6.8%)\n",
      "\n",
      "Example fails:\n",
      "1.0 0.0 0.0 @united any updates with the DOT looking at the united fares from London to the US from last week?\n",
      "0.9 0.0 0.1 @united any updates with the DOT looking at the united fares from London to the US from last week. I abhor you.\n",
      "\n",
      "----\n",
      "0.3 0.0 0.7 @united Thanks! LOL! #UA6259 will wait for us. Per @flightaware, same tail number as #UA5525 :)\n",
      "0.0 0.0 1.0 @united Thanks! LOL! #UA6259 will wait for us. Per @flightaware, same tail number as #UA5525. You are average.\n",
      "\n",
      "----\n",
      "0.1 0.0 0.9 @USAirways Chairman Preferred with 518,758 lifetime miles. What is my US Air reward, no #firstclass upgrades on @AmericanAir as usual.\n",
      "0.0 0.0 1.0 @USAirways Chairman Preferred with 518,758 lifetime miles. What is my US Air reward, no #firstclass upgrades on @AmericanAir as usual. You are average.\n",
      "\n",
      "----\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "Robustness\n",
      "\n",
      "add random urls and handles\n",
      "Test cases:      500\n",
      "Fails (rate):    77 (15.4%)\n",
      "\n",
      "Example fails:\n",
      "0.2 0.0 0.8 @JetBlue spoken to 2 reps. Once I'm allowed to check my bag and through the TSA checkpoint, I guarantee I will be talking to someone.\n",
      "0.3 0.7 0.0 @JetBlue spoken to 2 reps. Once I'm allowed to check my bag and through the TSA checkpoint, I guarantee I will be talking to someone. @8LFNLE\n",
      "0.4 0.6 0.0 https://t.co/HaY1Sj @JetBlue spoken to 2 reps. Once I'm allowed to check my bag and through the TSA checkpoint, I guarantee I will be talking to someone.\n",
      "\n",
      "----\n",
      "0.0 0.0 1.0 @JetBlue thank you. Is there a possibility it will change?\n",
      "0.1 0.9 0.0 @JetBlue thank you. Is there a possibility it will change? https://t.co/uEOI3u\n",
      "0.5 0.5 0.0 @JetBlue thank you. Is there a possibility it will change? https://t.co/7cpJkP\n",
      "\n",
      "----\n",
      "0.1 0.0 0.9 @USAirways finally spoke with someone. I have to email the refund dept. I love you guys, but 6 hours in line to be told no dice sucks.\n",
      "0.4 0.6 0.0 @airline  @USAirways finally spoke with someone. I have to email the refund dept. I love you guys, but 6 hours in line to be told no dice sucks.\n",
      "0.0 0.9 0.1 @ApivRX @USAirways finally spoke with someone. I have to email the refund dept. I love you guys, but 6 hours in line to be told no dice sucks.\n",
      "\n",
      "----\n",
      "\n",
      "\n",
      "punctuation\n",
      "Test cases:      500\n",
      "Fails (rate):    27 (5.4%)\n",
      "\n",
      "Example fails:\n",
      "1.0 0.0 0.0 @united flights taking off from IAD this afternoon?\n",
      "0.1 0.0 0.9 @united flights taking off from IAD this afternoon\n",
      "0.1 0.0 0.9 @united flights taking off from IAD this afternoon.\n",
      "\n",
      "----\n",
      "0.0 0.7 0.3 @SouthwestAir sent\n",
      "0.9 0.0 0.1 @SouthwestAir sent.\n",
      "\n",
      "----\n",
      "1.0 0.0 0.0 @JetBlue do you think snow in boston on 2/24 will effect my flight?\n",
      "0.0 0.0 1.0 @JetBlue do you think snow in boston on 2/24 will effect my flight\n",
      "0.1 0.0 0.9 @JetBlue do you think snow in boston on 2/24 will effect my flight.\n",
      "\n",
      "----\n",
      "\n",
      "\n",
      "typos\n",
      "Test cases:      500\n",
      "Fails (rate):    33 (6.6%)\n",
      "\n",
      "Example fails:\n",
      "0.1 0.0 0.9 @united need assistance. Line is out the door and travelling with 3 kids.\n",
      "0.9 0.0 0.1 @united need assistance. Line is out the door and travellnig with 3 kids.\n",
      "\n",
      "----\n",
      "1.0 0.0 0.0 @JetBlue Haha. I figured that. I was meaning there's no return flights out of Charlotte. It's like N/A for a week plus\n",
      "0.0 1.0 0.0 @JetBlue Haha. I figured that. I was meaning there's n oreturn flights out of Charlotte. It's like N/A for a week plus\n",
      "\n",
      "----\n",
      "0.9 0.0 0.1 @united @PGATOUR @NTrustOpen Next thing you know, United will believe they are above the DOT and take them to court. United is anti-consumer\n",
      "0.0 0.7 0.3 @united @PGATOUR @NTrustOpen Next thing you know, United will believe they are above the DOT and take them to court. United is anti-consmuer\n",
      "\n",
      "----\n",
      "\n",
      "\n",
      "2 typos\n",
      "Test cases:      500\n",
      "Fails (rate):    58 (11.6%)\n",
      "\n",
      "Example fails:\n",
      "0.0 0.0 1.0 @JetBlue Tomorrow wouldn’t have been soon enough! Thank you for the info!\n",
      "0.8 0.0 0.2 @JetBlue Tomorrow wouldn’t have been soon enough! Tahnky ou for the info!\n",
      "\n",
      "----\n",
      "0.0 0.0 1.0 @JetBlue Thank you ! What about Paris ? Could we arrange something from there ?\n",
      "0.0 0.9 0.1 @eJtBlue Thank you ! Waht about Paris ? Could we arrange something from there ?\n",
      "\n",
      "----\n",
      "0.0 0.0 1.0 @united if I miss the #Oscars I'll never fly united again\n",
      "1.0 0.0 0.0 @unitedi f I miss hte #Oscars I'll never fly united again\n",
      "\n",
      "----\n",
      "\n",
      "\n",
      "contractions\n",
      "Test cases:      1000\n",
      "Test cases run:  500\n",
      "Fails (rate):    17 (3.4%)\n",
      "\n",
      "Example fails:\n",
      "0.9 0.0 0.1 @AmericanAir yea no worries. I'm just going home to Denver. Not your fault, weather sucks bad.\n",
      "0.0 0.5 0.5 @AmericanAir yea no worries. I am just going home to Denver. Not your fault, weather sucks bad.\n",
      "\n",
      "----\n",
      "0.2 0.0 0.8 @united CDG-LAS 42 hours. And my luggage is in SFO. I've been wearing the same clothes for 42 hours. Thank you. #flythefriendlyskies\n",
      "0.0 0.8 0.2 @united CDG-LAS 42 hours. And my luggage is in SFO. I have been wearing the same clothes for 42 hours. Thank you. #flythefriendlyskies\n",
      "\n",
      "----\n",
      "0.2 0.0 0.8 @USAirways They don't stand alone for me, I bought tix based on USAir confirmed flight. What good is confirmation if you change it anyway?\n",
      "0.7 0.0 0.3 @USAirways They do not stand alone for me, I bought tix based on USAir confirmed flight. What good is confirmation if you change it anyway?\n",
      "\n",
      "----\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "NER\n",
      "\n",
      "change names\n",
      "Test cases:      331\n",
      "Fails (rate):    17 (5.1%)\n",
      "\n",
      "Example fails:\n",
      "0.3 0.7 0.0 @united yes, David Allan send an email with this number (KMM24999563V99860L0KM) and case#8719519\n",
      "0.7 0.0 0.3 @united yes, Michael Baker send an email with this number (KMM24999563V99860L0KM) and case#8719519\n",
      "0.7 0.0 0.3 @united yes, David White send an email with this number (KMM24999563V99860L0KM) and case#8719519\n",
      "\n",
      "----\n",
      "0.0 0.6 0.4 @SouthwestAir my friends from Boston stuck in Denver. Her name Jane. @RnCahill  Please contact her.\n",
      "0.2 0.0 0.8 @SouthwestAir my friends from Boston stuck in Denver. Her name Catherine. @RnCahill  Please contact her.\n",
      "\n",
      "----\n",
      "0.7 0.0 0.3 @SouthwestAir still haven't left @BWI. Maybe by the time I'm suppose to fly back to Austin on Tuesday we'll have moved.\n",
      "0.4 0.6 0.0 @SouthwestAir still haven't left @BWI. Maybe by the time I'm suppose to fly back to Shawn on Tuesday we'll have moved.\n",
      "0.5 0.5 0.0 @SouthwestAir still haven't left @BWI. Maybe by the time I'm suppose to fly back to Christian on Tuesday we'll have moved.\n",
      "\n",
      "----\n",
      "\n",
      "\n",
      "change locations\n",
      "Test cases:      909\n",
      "Test cases run:  500\n",
      "Fails (rate):    43 (8.6%)\n",
      "\n",
      "Example fails:\n",
      "0.8 0.0 0.2 @VirginAmerica to start 5xweekly #A319 flights from to #Dallas @DallasLoveField #Austin on 28APR #avgeek\n",
      "0.5 0.5 0.0 @VirginAmerica to start 5xweekly #A319 flights from to #Pine Bluff @DallasLoveField #Austin on 28APR #avgeek\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.5 0.5 0.0 @VirginAmerica to start 5xweekly #A319 flights from to #Modesto @DallasLoveField #Austin on 28APR #avgeek\n",
      "\n",
      "----\n",
      "0.2 0.0 0.8 @JetBlue ETA for flight 802 into Orlando this morning\n",
      "0.2 0.8 0.0 @JetBlue ETA for flight 802 into Yakima this morning\n",
      "0.1 0.9 0.0 @JetBlue ETA for flight 802 into Springfield this morning\n",
      "\n",
      "----\n",
      "0.4 0.6 0.0 @united The Opal Dragon book The Dragon (ALI) has woven his murdering ways from the Philippines to Australia http://t.co/N2fvElcYgz\n",
      "0.7 0.0 0.3 @united The Opal Dragon book The Dragon (ALI) has woven his murdering ways from the Mauritania to Australia http://t.co/N2fvElcYgz\n",
      "0.7 0.0 0.3 @united The Opal Dragon book The Dragon (ALI) has woven his murdering ways from the Philippines to Senegal http://t.co/N2fvElcYgz\n",
      "\n",
      "----\n",
      "\n",
      "\n",
      "change numbers\n",
      "Test cases:      1000\n",
      "Test cases run:  500\n",
      "Fails (rate):    15 (3.0%)\n",
      "\n",
      "Example fails:\n",
      "0.2 0.0 0.8 @USAirways strikes again--Late Flight crew #3/4 for the trip and maintenance #2/4.  Worth the extra $200/trip for less hassle and fewer delays\n",
      "0.0 0.6 0.4 @USAirways strikes again--Late Flight crew #3/4 for the trip and maintenance #2/4.  Worth the extra $229/trip for less hassle and fewer delays\n",
      "0.0 0.5 0.5 @USAirways strikes again--Late Flight crew #3/4 for the trip and maintenance #2/4.  Worth the extra $228/trip for less hassle and fewer delays\n",
      "\n",
      "----\n",
      "1.0 0.0 0.0 @USAirways You are 0 for 3 so far in on-time flights on this vacation.\n",
      "0.2 0.0 0.8 @USAirways You are 1 for 3 so far in on-time flights on this vacation.\n",
      "0.2 0.0 0.8 @USAirways You are 1 for 3 so far in on-time flights on this vacation.\n",
      "\n",
      "----\n",
      "0.0 0.5 0.5 @USAirways 3 hrs on hold. Husband has decided we should sleep in shifts so we don't miss the phone rep. to reschedule\n",
      "0.3 0.0 0.7 @USAirways 4 hrs on hold. Husband has decided we should sleep in shifts so we don't miss the phone rep. to reschedule\n",
      "0.3 0.0 0.7 @USAirways 4 hrs on hold. Husband has decided we should sleep in shifts so we don't miss the phone rep. to reschedule\n",
      "\n",
      "----\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "Fairness\n",
      "\n",
      "protected: race\n",
      "Test cases:      600\n",
      "Test cases run:  500\n",
      "Fails (rate):    342 (68.4%)\n",
      "\n",
      "Example fails:\n",
      "0.9 0.0 0.1 Ashley is a black speaker.\n",
      "0.0 0.0 1.0 Ashley is an asian speaker.\n",
      "0.0 0.0 1.0 Ashley is a hispanic speaker.\n",
      "\n",
      "----\n",
      "1.0 0.0 0.0 Christine is a black identity.\n",
      "0.0 0.0 1.0 Christine is an asian identity.\n",
      "0.0 0.0 1.0 Christine is a hispanic identity.\n",
      "\n",
      "----\n",
      "0.0 0.8 0.2 Samantha is a black Republican.\n",
      "0.0 0.0 1.0 Samantha is a hispanic Republican.\n",
      "0.1 0.0 0.9 Samantha is a white Republican.\n",
      "\n",
      "----\n",
      "\n",
      "\n",
      "protected: sexual\n",
      "Test cases:      600\n",
      "Test cases run:  500\n",
      "Fails (rate):    433 (86.6%)\n",
      "\n",
      "Example fails:\n",
      "0.0 0.0 1.0 Sean is an asexual perspective.\n",
      "0.8 0.0 0.2 Sean is a gay perspective.\n",
      "0.8 0.0 0.2 Sean is a homosexual perspective.\n",
      "\n",
      "----\n",
      "0.0 0.0 1.0 Samuel is an asexual history.\n",
      "0.9 0.0 0.1 Samuel is a gay history.\n",
      "0.7 0.0 0.3 Samuel is a homosexual history.\n",
      "\n",
      "----\n",
      "0.0 0.0 1.0 Chloe is an asexual student.\n",
      "1.0 0.0 0.0 Chloe is a bisexual student.\n",
      "1.0 0.0 0.0 Chloe is a gay student.\n",
      "\n",
      "----\n",
      "\n",
      "\n",
      "protected: religion\n",
      "Test cases:      600\n",
      "Test cases run:  500\n",
      "Fails (rate):    482 (96.4%)\n",
      "\n",
      "Example fails:\n",
      "0.0 0.0 1.0 Diana is a Christian sect.\n",
      "0.3 0.7 0.0 Diana is a Shi'a sect.\n",
      "1.0 0.0 0.0 Diana is an Atheist sect.\n",
      "\n",
      "----\n",
      "0.0 0.0 1.0 Benjamin is a Christian icon.\n",
      "0.9 0.0 0.1 Benjamin is an Atheist icon.\n",
      "\n",
      "----\n",
      "0.0 0.0 1.0 Megan is a Christian philosopher.\n",
      "0.9 0.0 0.1 Megan is an Atheist philosopher.\n",
      "\n",
      "----\n",
      "\n",
      "\n",
      "protected: nationality\n",
      "Test cases:      600\n",
      "Test cases run:  500\n",
      "Fails (rate):    62 (12.4%)\n",
      "\n",
      "Example fails:\n",
      "0.0 0.0 1.0 Benjamin is a Chinese nationalist.\n",
      "0.0 0.8 0.2 Benjamin is a German nationalist.\n",
      "\n",
      "----\n",
      "0.0 0.0 1.0 Jacob is a Chinese state.\n",
      "0.8 0.0 0.2 Jacob is a German state.\n",
      "0.0 0.8 0.2 Jacob is a Bangladeshi state.\n",
      "\n",
      "----\n",
      "0.2 0.0 0.8 Diana is a Chinese migrant.\n",
      "0.2 0.8 0.0 Diana is a Japanese migrant.\n",
      "0.2 0.8 0.0 Diana is a Nigerian migrant.\n",
      "\n",
      "----\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "Temporal\n",
      "\n",
      "used to, but now\n",
      "Test cases:      8000\n",
      "Test cases run:  500\n",
      "Fails (rate):    163 (32.6%)\n",
      "\n",
      "Example fails:\n",
      "0.0 0.0 1.0 I hate this airline, but in the past I would admire it.\n",
      "----\n",
      "1.0 0.0 0.0 I think this airline is wonderful, but I used to think it was poor.\n",
      "----\n",
      "1.0 0.0 0.0 I think this airline is excellent, but I used to think it was lousy.\n",
      "----\n",
      "\n",
      "\n",
      "\"used to\" should reduce\n",
      "Test cases:      4368\n",
      "Test cases run:  500\n",
      "After filtering: 6 (1.2%)\n",
      "Fails (rate):    0 (0.0%)\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "Negation\n",
      "\n",
      "simple negations: negative\n",
      "Test cases:      6318\n",
      "Test cases run:  500\n",
      "Fails (rate):    26 (5.2%)\n",
      "\n",
      "Example fails:\n",
      "0.0 0.0 1.0 I would never say I admire that crew.\n",
      "----\n",
      "0.0 0.0 1.0 I would never say I love the food.\n",
      "----\n",
      "0.0 0.0 1.0 I would never say I enjoy the company.\n",
      "----\n",
      "\n",
      "\n",
      "simple negations: not negative\n",
      "Test cases:      6786\n",
      "Test cases run:  500\n",
      "Fails (rate):    51 (10.2%)\n",
      "\n",
      "Example fails:\n",
      "0.9 0.0 0.1 It wasn't a creepy food.\n",
      "----\n",
      "0.8 0.0 0.2 It wasn't a lousy plane.\n",
      "----\n",
      "0.8 0.0 0.2 No one hates this crew.\n",
      "----\n",
      "\n",
      "\n",
      "simple negations: not neutral is still neutral\n",
      "Test cases:      2496\n",
      "Test cases run:  500\n",
      "Fails (rate):    493 (98.6%)\n",
      "\n",
      "Example fails:\n",
      "0.9 0.0 0.1 That wasn't a private airline.\n",
      "----\n",
      "1.0 0.0 0.0 It isn't an American aircraft.\n",
      "----\n",
      "1.0 0.0 0.0 That is not an American airline.\n",
      "----\n",
      "\n",
      "\n",
      "simple negations: I thought x was positive, but it was not (should be negative)\n",
      "Test cases:      1992\n",
      "Test cases run:  500\n",
      "Fails (rate):    0 (0.0%)\n",
      "\n",
      "\n",
      "simple negations: I thought x was negative, but it was not (should be neutral or positive)\n",
      "Test cases:      2124\n",
      "Test cases run:  500\n",
      "Fails (rate):    500 (100.0%)\n",
      "\n",
      "Example fails:\n",
      "1.0 0.0 0.0 I thought that flight would be unhappy, but it wasn't.\n",
      "----\n",
      "1.0 0.0 0.0 I thought that seat would be horrible, but it was not.\n",
      "----\n",
      "1.0 0.0 0.0 I thought the plane would be dreadful, but it wasn't.\n",
      "----\n",
      "\n",
      "\n",
      "simple negations: but it was not (neutral) should still be neutral\n",
      "Test cases:      804\n",
      "Test cases run:  500\n",
      "Fails (rate):    500 (100.0%)\n",
      "\n",
      "Example fails:\n",
      "1.0 0.0 0.0 I thought that airline would be Israeli, but it was not.\n",
      "----\n",
      "1.0 0.0 0.0 I thought the service would be Indian, but it wasn't.\n",
      "----\n",
      "1.0 0.0 0.0 I thought that plane would be commercial, but it wasn't.\n",
      "----\n",
      "\n",
      "\n",
      "Hard: Negation of positive with neutral stuff in the middle (should be negative)\n",
      "Test cases:      1000\n",
      "Test cases run:  500\n",
      "Fails (rate):    426 (85.2%)\n",
      "\n",
      "Example fails:\n",
      "0.0 0.0 1.0 I wouldn't say, given the time that I've been flying, that the customer service was wonderful.\n",
      "----\n",
      "0.0 0.0 1.0 I wouldn't say, given that I am from Brazil, that the was an incredible airline.\n",
      "----\n",
      "0.0 0.0 1.0 I don't think, given all that I've seen over the years, that this is a perfect food.\n",
      "----\n",
      "\n",
      "\n",
      "Hard: Negation of negative with neutral stuff in the middle (should be positive or neutral)\n",
      "Test cases:      1000\n",
      "Test cases run:  500\n",
      "Fails (rate):    498 (99.6%)\n",
      "\n",
      "Example fails:\n",
      "1.0 0.0 0.0 i don't think, given that I am from Brazil, that I regret this crew.\n",
      "----\n",
      "1.0 0.0 0.0 i wouldn't say, given that I am from Brazil, that that is an unhappy customer service.\n",
      "----\n",
      "1.0 0.0 0.0 I can't say, given the time that I've been flying, that the cabin crew was ugly.\n",
      "----\n",
      "\n",
      "\n",
      "negation of neutral with neutral in the middle, should still neutral\n",
      "Test cases:      1000\n",
      "Test cases run:  500\n",
      "Fails (rate):    480 (96.0%)\n",
      "\n",
      "Example fails:\n",
      "0.0 0.0 1.0 I wouldn't say, given that I am from Brazil, that the company was international.\n",
      "----\n",
      "1.0 0.0 0.0 I wouldn't say, given it's a Tuesday, that that airline was American.\n",
      "----\n",
      "1.0 0.0 0.0 I don't think, given all that I've seen over the years, that this is a private staff.\n",
      "----\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "SRL\n",
      "\n",
      "my opinion is what matters\n",
      "Test cases:      8528\n",
      "Test cases run:  500\n",
      "Fails (rate):    205 (41.0%)\n",
      "\n",
      "Example fails:\n",
      "1.0 0.0 0.0 I think you are nice, but some people think you are nasty.\n",
      "----\n",
      "0.0 0.0 1.0 I think you are lousy, but some people think you are extraordinary.\n",
      "----\n",
      "0.0 0.0 1.0 I despise you, but some people enjoy you.\n",
      "----\n",
      "\n",
      "\n",
      "Q & A: yes\n",
      "Test cases:      7644\n",
      "Test cases run:  500\n",
      "Fails (rate):    0 (0.0%)\n",
      "\n",
      "\n",
      "Q & A: yes (neutral)\n",
      "Test cases:      1560\n",
      "Test cases run:  500\n",
      "Fails (rate):    483 (96.6%)\n",
      "\n",
      "Example fails:\n",
      "0.9 0.0 0.1 Do I think this food is American? Yes\n",
      "----\n",
      "1.0 0.0 0.0 Do I think the airline was Indian? Yes\n",
      "----\n",
      "1.0 0.0 0.0 Do I think that customer service is American? Yes\n",
      "----\n",
      "\n",
      "\n",
      "Q & A: no\n",
      "Test cases:      7644\n",
      "Test cases run:  500\n",
      "Fails (rate):    416 (83.2%)\n",
      "\n",
      "Example fails:\n",
      "1.0 0.0 0.0 Do I think it is a bad cabin crew? No\n",
      "----\n",
      "1.0 0.0 0.0 Do I think this was an annoying company? No\n",
      "----\n",
      "0.0 0.0 1.0 Do I think that was an incredible company? No\n",
      "----\n",
      "\n",
      "\n",
      "Q & A: no (neutral)\n",
      "Test cases:      1560\n",
      "Test cases run:  500\n",
      "Fails (rate):    500 (100.0%)\n",
      "\n",
      "Example fails:\n",
      "1.0 0.0 0.0 Do I think the staff is private? No\n",
      "----\n",
      "1.0 0.0 0.0 Do I think that was an Australian customer service? No\n",
      "----\n",
      "1.0 0.0 0.0 Do I think the plane was international? No\n",
      "----\n",
      "\n",
      "\n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "suite.summary()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## QQP "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Transformers doesn't have a model pretrained on QQP, so we'll use one trained on MRPC for this demo"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "from transformers import AutoTokenizer, AutoModelForSequenceClassification\n",
    "import torch\n",
    "tokenizer = AutoTokenizer.from_pretrained(\"bert-base-cased-finetuned-mrpc\")\n",
    "model = AutoModelForSequenceClassification.from_pretrained(\"bert-base-cased-finetuned-mrpc\")\n",
    "model.to('cuda');\n",
    "model.eval();"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "def batch_qqp(data, batch_size=128):\n",
    "    ret = []\n",
    "    for d in chunks(data, batch_size):\n",
    "        t = tokenizer([a[0] for a in d], [a[1] for a in d], return_tensors='pt', padding=True).to('cuda')\n",
    "        with torch.no_grad():\n",
    "            logits = torch.softmax(model(**t)[0], dim=1).cpu().numpy()\n",
    "        ret.append(logits)\n",
    "    return np.vstack(ret)\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "from checklist.pred_wrapper import PredictorWrapper\n",
    "# wrapped_pp returns a tuple with (predictions, softmax confidences)\n",
    "wrapped_pp = PredictorWrapper.wrap_softmax(batch_qqp)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0.94038326, 0.05961675],\n",
       "       [0.09536299, 0.904637  ]], dtype=float32)"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "s0 = \"The company HuggingFace is based in New York City\"\n",
    "s1 = \"Apples are especially bad for your health\"\n",
    "s2 = \"HuggingFace's headquarters are situated in Manhattan\"\n",
    "batch_qqp([(s0, s1), (s0, s2)])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "suite_path = '../release_data/qqp/qqp_suite.pkl'\n",
    "suite = TestSuite.from_file(suite_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Running Modifier: adj\n",
      "Predicting 500 examples\n",
      "Running different adjectives\n",
      "Predicting 500 examples\n",
      "Running Different animals\n",
      "Predicting 500 examples\n",
      "Running Irrelevant modifiers - animals\n",
      "Predicting 500 examples\n",
      "Running Irrelevant modifiers - people\n",
      "Predicting 500 examples\n",
      "Running Irrelevant preamble with different examples.\n",
      "Predicting 500 examples\n",
      "Running Preamble is relevant (different injuries)\n",
      "Predicting 500 examples\n",
      "Running How can I become more {synonym}?\n",
      "Predicting 500 examples\n",
      "Running (question, f(question)) where f(question) replaces synonyms?\n",
      "Predicting 326 examples\n",
      "Running Replace synonyms in real pairs\n",
      "Predicting 684 examples\n",
      "Running How can I become more X != How can I become less X\n",
      "Predicting 500 examples\n",
      "Running How can I become more X = How can I become less antonym(X)\n",
      "Predicting 500 examples\n",
      "Running add one typo\n",
      "Predicting 1500 examples\n",
      "Running contrations\n",
      "Predicting 1427 examples\n",
      "Running (q, paraphrase(q))\n",
      "Predicting 18944 examples\n",
      "Running Product of paraphrases(q1) * paraphrases(q2)\n",
      "Predicting 9756 examples\n",
      "Running same adjectives, different people\n",
      "Predicting 500 examples\n",
      "Running same adjectives, different people v2\n",
      "Predicting 500 examples\n",
      "Running same adjectives, different people v3\n",
      "Predicting 500 examples\n",
      "Running Change same name in both questions\n",
      "Predicting 5435 examples\n",
      "Running Change same location in both questions\n",
      "Predicting 5145 examples\n",
      "Running Change same number in both questions\n",
      "Predicting 4907 examples\n",
      "Running Change first name in one of the questions\n",
      "Predicting 9967 examples\n",
      "Running Change first and last name in one of the questions\n",
      "Predicting 9630 examples\n",
      "Running Change location in one of the questions\n",
      "Predicting 10200 examples\n",
      "Running Change numbers in one of the questions\n",
      "Predicting 9693 examples\n",
      "Running Keep entitites, fill in with gibberish\n",
      "Predicting 4649 examples\n",
      "Running Is person X != Did person use to be X\n",
      "Predicting 500 examples\n",
      "Running Is person X != Is person becoming X\n",
      "Predicting 500 examples\n",
      "Running What was person's life before becoming X != What was person's life after becoming X\n",
      "Predicting 500 examples\n",
      "Running Do you have to X your dog before Y it != Do you have to X your dog after Y it.\n",
      "Predicting 500 examples\n",
      "Running Is it {ok, dangerous, ...} to {smoke, rest, ...} after != before\n",
      "Predicting 500 examples\n",
      "Running How can I become a X person != How can I become a person who is not X\n",
      "Predicting 500 examples\n",
      "Running Is it {ok, dangerous, ...} to {smoke, rest, ...} in country != Is it {ok, dangerous, ...} not to {smoke, rest, ...} in country\n",
      "Predicting 500 examples\n",
      "Running What are things a {noun} should worry about != should not worry about.\n",
      "Predicting 500 examples\n",
      "Running How can I become a X person == How can I become a person who is not antonym(X)\n",
      "Predicting 500 examples\n",
      "Running Simple coref: he and she\n",
      "Predicting 500 examples\n",
      "Running Simple coref: his and her\n",
      "Predicting 500 examples\n",
      "Running Who do X think - Who is the ... according to X\n",
      "Predicting 500 examples\n",
      "Running Order does not matter for comparison\n",
      "Predicting 1500 examples\n",
      "Running Order does not matter for symmetric relations\n",
      "Predicting 500 examples\n",
      "Running Order does matter for asymmetric relations\n",
      "Predicting 500 examples\n",
      "Running traditional SRL: active / passive swap\n",
      "Predicting 500 examples\n",
      "Running traditional SRL: wrong active / passive swap\n",
      "Predicting 500 examples\n",
      "Running traditional SRL: active / passive swap with people\n",
      "Predicting 500 examples\n",
      "Running traditional SRL: wrong active / passive swap with people\n",
      "Predicting 500 examples\n",
      "Running A or B is not the same as C and D\n",
      "Predicting 500 examples\n",
      "Running A or B is not the same as A and B\n",
      "Predicting 500 examples\n",
      "Running A and / or B is the same as B and / or A\n",
      "Predicting 500 examples\n",
      "Running a {nationality} {profession} = a {profession} and {nationality}\n",
      "Predicting 500 examples\n",
      "Running Reflexivity: (q, q) should be duplicate\n",
      "Predicting 500 examples\n",
      "Running Symmetry: f(a, b) = f(b, a)\n",
      "Predicting 1000 examples\n",
      "Running Testing implications\n",
      "Predicting 1500 examples\n"
     ]
    }
   ],
   "source": [
    "suite.run(wrapped_pp, n=500)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Vocabulary\n",
      "\n",
      "Modifier: adj\n",
      "Test cases:      1000\n",
      "Test cases run:  500\n",
      "Fails (rate):    499 (99.8%)\n",
      "\n",
      "Example fails:\n",
      "0.9 ('Is Timothy Cox an editor?', 'Is Timothy Cox an effective editor?')\n",
      "----\n",
      "0.9 ('Is Rebecca Stewart an accountant?', 'Is Rebecca Stewart an English accountant?')\n",
      "----\n",
      "0.9 ('Is Jeffrey Moore an administrator?', 'Is Jeffrey Moore an actual administrator?')\n",
      "----\n",
      "\n",
      "\n",
      "different adjectives\n",
      "Test cases:      954\n",
      "Test cases run:  500\n",
      "Fails (rate):    438 (87.6%)\n",
      "\n",
      "Example fails:\n",
      "0.9 ('Is Andrea Diaz racist?', 'Is Andrea Diaz Jewish?')\n",
      "----\n",
      "0.9 ('Is Amanda Long racist?', 'Is Amanda Long an anarchist?')\n",
      "----\n",
      "0.9 ('Is Madison Brown English?', 'Is Madison Brown Armenian?')\n",
      "----\n",
      "\n",
      "\n",
      "Different animals\n",
      "Test cases:      928\n",
      "Test cases run:  500\n",
      "Fails (rate):    500 (100.0%)\n",
      "\n",
      "Example fails:\n",
      "0.9 ('Can I feed my snail seeds?', 'Can I feed my turtle seeds?')\n",
      "----\n",
      "0.9 ('Can I feed my cat butter?', 'Can I feed my snail butter?')\n",
      "----\n",
      "0.9 ('Can I feed my lobster milk?', 'Can I feed my snail milk?')\n",
      "----\n",
      "\n",
      "\n",
      "Irrelevant modifiers - animals\n",
      "Test cases:      1000\n",
      "Test cases run:  500\n",
      "Fails (rate):    0 (0.0%)\n",
      "\n",
      "\n",
      "Irrelevant modifiers - people\n",
      "Test cases:      987\n",
      "Test cases run:  500\n",
      "Fails (rate):    0 (0.0%)\n",
      "\n",
      "\n",
      "Irrelevant preamble with different examples.\n",
      "Test cases:      938\n",
      "Test cases run:  500\n",
      "Fails (rate):    0 (0.0%)\n",
      "\n",
      "\n",
      "Preamble is relevant (different injuries)\n",
      "Test cases:      975\n",
      "Test cases run:  500\n",
      "Fails (rate):    500 (100.0%)\n",
      "\n",
      "Example fails:\n",
      "0.9 ('I hurt my heel last time I played soccer. Is it normal to hurt this part of the body?', 'I hurt my heart last time I played soccer. Is it normal to hurt this part of the body?')\n",
      "----\n",
      "0.9 ('I hurt my feet last time I played golf. Should I never play again?', 'I hurt my forearm last time I played golf. Should I never play again?')\n",
      "----\n",
      "0.9 ('I hurt my heart last time I played tennis. Is this a common injury?', 'I hurt my spine last time I played tennis. Is this a common injury?')\n",
      "----\n",
      "\n",
      "\n",
      "How can I become more X != How can I become less X\n",
      "Test cases:      2000\n",
      "Test cases run:  500\n",
      "Fails (rate):    500 (100.0%)\n",
      "\n",
      "Example fails:\n",
      "0.9 ('How can I become more irresponsible?', 'How can I become less irresponsible?')\n",
      "----\n",
      "0.9 ('How can I become more unhappy?', 'How can I become less unhappy?')\n",
      "----\n",
      "0.9 ('How can I become more humble?', 'How can I become less humble?')\n",
      "----\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "Taxonomy\n",
      "\n",
      "How can I become more {synonym}?\n",
      "Test cases:      6000\n",
      "Test cases run:  500\n",
      "Fails (rate):    0 (0.0%)\n",
      "\n",
      "\n",
      "(question, f(question)) where f(question) replaces synonyms?\n",
      "Test cases:      326\n",
      "Fails (rate):    0 (0.0%)\n",
      "\n",
      "\n",
      "Replace synonyms in real pairs\n",
      "Test cases:      251\n",
      "Fails (rate):    12 (4.8%)\n",
      "\n",
      "Example fails:\n",
      "0.4 ('Do you have to be intelligent to be intelligent?', 'What can I do to become smarter?')\n",
      "0.5 ('Do you have to be smart to be smart?', 'What can I do to become smarter?')\n",
      "\n",
      "----\n",
      "0.7 ('How can long distance relationships be successful?', 'What are the best ways to keep a long distance love relationship happy?')\n",
      "0.2 ('How can long distance relationships be successful?', 'What are the best ways to keep a long distance love relationship joyful?')\n",
      "\n",
      "----\n",
      "0.5 ('How can we say that we are happy?', '\"How do you say \"\"happy\"\" in French?\"')\n",
      "0.3 ('How can we say that we are joyful?', '\"How do you say \"\"happy\"\" in French?\"')\n",
      "0.4 ('How can we say that we are joyful?', '\"How do you say \"\"joyful\"\" in French?\"')\n",
      "\n",
      "----\n",
      "\n",
      "\n",
      "How can I become more X = How can I become less antonym(X)\n",
      "Test cases:      2000\n",
      "Test cases run:  500\n",
      "Fails (rate):    0 (0.0%)\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "Robustness\n",
      "\n",
      "add one typo\n",
      "Test cases:      500\n",
      "Fails (rate):    66 (13.2%)\n",
      "\n",
      "Example fails:\n",
      "0.8 ('What artificial intelligence can do?', 'What all does Artificial Intelligence include?')\n",
      "0.3 ('What artificial intellgience can do?', 'What all does Artificial Intelligence include?')\n",
      "\n",
      "----\n",
      "0.6 ('You have just been employed as an extension supervisor in an organisation, what are the steps you will take to ensure stability and effective performance of your task?', 'What different tasks will you perform at each step of the system development life cycle of Library Management System?')\n",
      "0.4 ('You have just been employed as an extension supervisor in an organisation, what are hte steps you will take to ensure stability and effective performance of your task?', 'What different tasks will you perform at each step of the system development life cycle of Library Management System?')\n",
      "\n",
      "----\n",
      "0.6 ('What would be the effect on the Indian economy after banning 500 and 1,000 notes?', 'What is the use of banning 500 and 1000 rupee notes and introducing new 500 and 2000 rupee notes?')\n",
      "0.2 ('What would be the effect on th eIndian economy after banning 500 and 1,000 notes?', 'What is the use of banning 500 and 1000 rupee notes and introducing new 500 and 2000 rupee notes?')\n",
      "\n",
      "----\n",
      "\n",
      "\n",
      "contrations\n",
      "Test cases:      500\n",
      "Fails (rate):    21 (4.2%)\n",
      "\n",
      "Example fails:\n",
      "0.5 ('\"I smoked a bowl of crystal meth on Friday. I\\'m 5\\'4\"\" and 187lbs. Will I be able to pass a urine drug test on Wednesday for drug court?\"', 'If I smoke meth on Fri 12pm will I pass a urine test Wednesday morning?')\n",
      "0.6 ('\"I smoked a bowl of crystal meth on Friday. I am 5\\'4\"\" and 187lbs. Will I be able to pass a urine drug test on Wednesday for drug court?\"', 'If I smoke meth on Fri 12pm will I pass a urine test Wednesday morning?')\n",
      "\n",
      "----\n",
      "0.5 (\"Why doesn't India buys SU-34 Fullback fighter bomber?\", \"Why can't India make indigenous fighter jets and weapons?\")\n",
      "0.3 (\"Why doesn't India buys SU-34 Fullback fighter bomber?\", 'Why cannot India make indigenous fighter jets and weapons?')\n",
      "\n",
      "----\n",
      "0.6 ('What is purpose of life?', 'What is the purpose of life, if not money?')\n",
      "0.4 (\"What's purpose of life?\", \"What's the purpose of life, if not money?\")\n",
      "0.4 (\"What's purpose of life?\", 'What is the purpose of life, if not money?')\n",
      "\n",
      "----\n",
      "\n",
      "\n",
      "(q, paraphrase(q))\n",
      "Test cases:      200\n",
      "Fails (rate):    78 (39.0%)\n",
      "\n",
      "Example fails:\n",
      "0.9 ('How do I retrieve and recover deleted items from my Gmail after mistakenly deleting and clearing trash?', 'How do I retrieve and recover deleted items from my Gmail after mistakenly deleting and clearing trash?')\n",
      "0.4 ('How can you retrieve and recover deleted items from your Gmail after mistakenly deleting and clearing trash?', 'If I want to retrieve and recover deleted items from my Gmail after mistakenly deleting and clearing trash, what should I do?')\n",
      "0.4 ('How can you retrieve and recover deleted items from your Gmail after mistakenly deleting and clearing trash?', 'In order to retrieve and recover deleted items from my Gmail after mistakenly deleting and clearing trash, what should I do?')\n",
      "\n",
      "----\n",
      "0.9 ('How do I know my not so known crush likes me?', 'How do I know my not so known crush likes me?')\n",
      "0.4 ('How do you know your not so known crush likes me?', 'If I want to know my not so known crush likes me, what should I do?')\n",
      "0.4 ('How can you know your not so known crush likes me?', 'If I want to know my not so known crush likes me, what should I do?')\n",
      "\n",
      "----\n",
      "0.9 ('How do I get quality backlinks?', 'How do I get quality backlinks?')\n",
      "0.3 ('How can you get quality backlinks?', 'If I want to get quality backlinks, what should I do?')\n",
      "0.3 ('If I want to get quality backlinks, what should I do?', 'How can you get quality backlinks?')\n",
      "\n",
      "----\n",
      "\n",
      "\n",
      "Product of paraphrases(q1) * paraphrases(q2)\n",
      "Test cases:      100\n",
      "Fails (rate):    81 (81.0%)\n",
      "\n",
      "Example fails:\n",
      "0.8 ('How can I watch any clip4sale video without paying?', 'Can I use an old game that is now free in a video without paying royalties?')\n",
      "0.1 ('If I want to watch any clip4sale video without paying, what should I do?', 'Do you think you can use an old game that is now free in a video without paying royalties?')\n",
      "0.1 ('If I want to watch any clip4sale video without paying, what should I do?', 'Do you think you can use an old game that is now free in a video without paying royalties?')\n",
      "\n",
      "----\n",
      "0.9 ('How can I becomec the next APJ Abdul Kalam?', 'How can I become the next APJ Abdul Kalam?')\n",
      "0.2 ('If I want to becomec the next APJ Abdul Kalam, what should I do?', 'How can you become the next APJ Abdul Kalam?')\n",
      "0.2 ('How can you becomec the next APJ Abdul Kalam?', 'If I want to become the next APJ Abdul Kalam, what should I do?')\n",
      "\n",
      "----\n",
      "0.9 ('How can I become a porn star?', 'How do I become an adult pornstar?')\n",
      "0.4 ('If I want to become a porn star, what should I do?', 'How can you become an adult pornstar?')\n",
      "0.4 ('If you want to become a porn star, what should you do?', 'How can I become an adult pornstar?')\n",
      "\n",
      "----\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "NER\n",
      "\n",
      "same adjectives, different people\n",
      "Test cases:      972\n",
      "Test cases run:  500\n",
      "Fails (rate):    500 (100.0%)\n",
      "\n",
      "Example fails:\n",
      "0.9 ('Is Kevin Lewis gay?', 'Is Mark Cruz gay?')\n",
      "----\n",
      "0.9 ('Is Richard Cruz an immigrant?', 'Is Sarah Torres an immigrant?')\n",
      "----\n",
      "0.9 ('Is Scott Flores Australian?', 'Is Taylor Price Australian?')\n",
      "----\n",
      "\n",
      "\n",
      "same adjectives, different people v2\n",
      "Test cases:      984\n",
      "Test cases run:  500\n",
      "Fails (rate):    500 (100.0%)\n",
      "\n",
      "Example fails:\n",
      "0.9 ('Is Christopher Gray gay?', 'Is Ethan Gray gay?')\n",
      "----\n",
      "0.9 ('Is Brian Taylor black?', 'Is Jacob Taylor black?')\n",
      "----\n",
      "0.9 ('Is Taylor Morris Armenian?', 'Is Shannon Morris Armenian?')\n",
      "----\n",
      "\n",
      "\n",
      "same adjectives, different people v3\n",
      "Test cases:      990\n",
      "Test cases run:  500\n",
      "Fails (rate):    500 (100.0%)\n",
      "\n",
      "Example fails:\n",
      "0.9 ('Is Kimberly Ward Muslim?', 'Is Kimberly Cook Muslim?')\n",
      "----\n",
      "0.9 ('Is Sophia Parker Christian?', 'Is Sophia Green Christian?')\n",
      "----\n",
      "0.9 ('Is Heather Mitchell an astronaut?', 'Is Heather Cooper an astronaut?')\n",
      "----\n",
      "\n",
      "\n",
      "Change same name in both questions\n",
      "Test cases:      500\n",
      "Fails (rate):    48 (9.6%)\n",
      "\n",
      "Example fails:\n",
      "0.6 ('What is Ruby famous for?', '\"What does \"\"&\"\" do in Ruby?\"')\n",
      "0.3 ('What is Brittany famous for?', '\"What does \"\"&\"\" do in Brittany?\"')\n",
      "0.4 ('What is Sara famous for?', '\"What does \"\"&\"\" do in Sara?\"')\n",
      "\n",
      "----\n",
      "0.6 ('Would Donald Trump make a good U.S president? Why or why not?', 'Donald Trump: Would you be a good president?')\n",
      "0.4 ('Would Joseph Ward make a good U.S president? Why or why not?', 'Joseph Ward: Would you be a good president?')\n",
      "0.4 ('Would Daniel Nelson make a good U.S president? Why or why not?', 'Daniel Nelson: Would you be a good president?')\n",
      "\n",
      "----\n",
      "0.6 ('Does Emma Watson have a boyfriend?', 'Who is Emma Watson?')\n",
      "0.4 ('Does Melissa Moore have a boyfriend?', 'Who is Melissa Moore?')\n",
      "0.4 ('Does Nicole Nelson have a boyfriend?', 'Who is Nicole Nelson?')\n",
      "\n",
      "----\n",
      "\n",
      "\n",
      "Change same location in both questions\n",
      "Test cases:      500\n",
      "Fails (rate):    26 (5.2%)\n",
      "\n",
      "Example fails:\n",
      "0.6 ('Can I live on $85,000 a year in Manhattan?', 'How much should someone make to live comfortably in Manhattan?')\n",
      "0.3 ('Can I live on $85,000 a year in Waukegan?', 'How much should someone make to live comfortably in Waukegan?')\n",
      "0.4 ('Can I live on $85,000 a year in Walnut Creek?', 'How much should someone make to live comfortably in Walnut Creek?')\n",
      "\n",
      "----\n",
      "0.6 ('Who is the most unpopular President of India and why?', 'Who is the president of India 2016?')\n",
      "0.3 ('Who is the most unpopular President of Grenada and why?', 'Who is the president of Grenada 2016?')\n",
      "0.5 ('Who is the most unpopular President of Liechtenstein and why?', 'Who is the president of Liechtenstein 2016?')\n",
      "\n",
      "----\n",
      "0.6 ('How can the education system in India be improved? Kindly mention point wise.', 'How can we Improve education system in India by e-Education?')\n",
      "0.4 (\"How can the education system in Côte d'Ivoire be improved? Kindly mention point wise.\", \"How can we Improve education system in Côte d'Ivoire by e-Education?\")\n",
      "0.5 ('How can the education system in Virgin Islands (U.S.) be improved? Kindly mention point wise.', 'How can we Improve education system in Virgin Islands (U.S.) by e-Education?')\n",
      "\n",
      "----\n",
      "\n",
      "\n",
      "Change same number in both questions\n",
      "Test cases:      500\n",
      "Fails (rate):    19 (3.8%)\n",
      "\n",
      "Example fails:\n",
      "0.5 ('How does banning 500 and 1000 rupee notes help to control black money?', 'Will the ban on 500 & 1000 rupee notes really work against corruption?')\n",
      "0.6 ('How does banning 500 and 931 rupee notes help to control black money?', 'Will the ban on 500 & 931 rupee notes really work against corruption?')\n",
      "0.6 ('How does banning 483 and 1000 rupee notes help to control black money?', 'Will the ban on 483 & 1000 rupee notes really work against corruption?')\n",
      "\n",
      "----\n",
      "0.7 ('Is there a good chance Bernie Sanders will run for President in 2020?', 'Do you see Bernie sanders re running in 2020?')\n",
      "0.2 ('Is there a good chance Bernie Sanders will run for President in 1812?', 'Do you see Bernie sanders re running in 1812?')\n",
      "0.3 ('Is there a good chance Bernie Sanders will run for President in 2228?', 'Do you see Bernie sanders re running in 2228?')\n",
      "\n",
      "----\n",
      "0.8 (\"Will India be able to achieve it's vision 2020 or will it take some more years for APJ Abdul Kalam’s Dream to come true?\", \"Can we achieve Abdul Kalam's vision of India by 2020?\")\n",
      "0.5 (\"Will India be able to achieve it's vision 1782 or will it take some more years for APJ Abdul Kalam’s Dream to come true?\", \"Can we achieve Abdul Kalam's vision of India by 1782?\")\n",
      "0.5 (\"Will India be able to achieve it's vision 1908 or will it take some more years for APJ Abdul Kalam’s Dream to come true?\", \"Can we achieve Abdul Kalam's vision of India by 1908?\")\n",
      "\n",
      "----\n",
      "\n",
      "\n",
      "Change first name in one of the questions\n",
      "Test cases:      500\n",
      "After filtering: 351 (70.2%)\n",
      "Fails (rate):    335 (95.4%)\n",
      "\n",
      "Example fails:\n",
      "0.9 ('Where does Hillary Clinton stand on the Palestinian-Israeli conflict?', 'What will Hillary do to resolve the Israeli-Palestinian Conflict?')\n",
      "0.9 ('Where does Lindsey Clinton stand on the Palestinian-Israeli conflict?', 'What will Hillary do to resolve the Israeli-Palestinian Conflict?')\n",
      "0.9 ('Where does Megan Clinton stand on the Palestinian-Israeli conflict?', 'What will Hillary do to resolve the Israeli-Palestinian Conflict?')\n",
      "\n",
      "----\n",
      "0.9 (\"What does Bill O'Reilly think of Donald Trump?\", \"What does Bill O'Reilly think of Donald Trump as far as anyone knows?\")\n",
      "0.9 (\"What does Bill O'Reilly think of Donald Trump?\", \"What does Bill O'Reilly think of Eric Trump as far as anyone knows?\")\n",
      "0.9 (\"What does Bill O'Reilly think of Donald Trump?\", \"What does Bill O'Reilly think of Joseph Trump as far as anyone knows?\")\n",
      "\n",
      "----\n",
      "0.9 (\"Why shouldn't I vote for Hillary Clinton?\", \"Why shouldn't you vote for Hillary Clinton?\")\n",
      "0.9 (\"Why shouldn't I vote for Alexandra Clinton?\", \"Why shouldn't you vote for Hillary Clinton?\")\n",
      "0.9 (\"Why shouldn't I vote for Hillary Clinton?\", \"Why shouldn't you vote for Stephanie Clinton?\")\n",
      "\n",
      "----\n",
      "\n",
      "\n",
      "Change first and last name in one of the questions\n",
      "Test cases:      682\n",
      "Test cases run:  500\n",
      "After filtering: 363 (72.6%)\n",
      "Fails (rate):    300 (82.6%)\n",
      "\n",
      "Example fails:\n",
      "0.9 ('How can you determine the Lewis structure for NA2O?', 'How can you determine the Lewis structure for formaldehyde?')\n",
      "0.9 ('How can you determine the Lewis structure for NA2O?', 'How can you determine the Jordan structure for formaldehyde?')\n",
      "0.9 ('How can you determine the Sean structure for NA2O?', 'How can you determine the Lewis structure for formaldehyde?')\n",
      "\n",
      "----\n",
      "0.9 ('What would Hillary Clinton do now that the election is over?', 'What will Hillary Clinton do now?')\n",
      "0.9 ('What would Ashley Jones do now that the election is over?', 'What will Hillary Clinton do now?')\n",
      "0.9 ('What would Jennifer Butler do now that the election is over?', 'What will Hillary Clinton do now?')\n",
      "\n",
      "----\n",
      "0.9 ('Why is Clinton better than Trump?', 'Why is Hillary Clinton a better choice than Donald Trump?')\n",
      "0.9 ('Why is Jeremiah better than Trump?', 'Why is Hillary Clinton a better choice than Donald Trump?')\n",
      "0.9 ('Why is Brandon better than Trump?', 'Why is Hillary Clinton a better choice than Donald Trump?')\n",
      "\n",
      "----\n",
      "\n",
      "\n",
      "Change location in one of the questions\n",
      "Test cases:      1386\n",
      "Test cases run:  500\n",
      "After filtering: 407 (81.4%)\n",
      "Fails (rate):    376 (92.4%)\n",
      "\n",
      "Example fails:\n",
      "0.9 ('How do I get an investor for my startup in India?', 'How can I fund a startup in India?')\n",
      "0.9 ('How do I get an investor for my startup in India?', 'How can I fund a startup in Sudan?')\n",
      "0.9 ('How do I get an investor for my startup in Sudan?', 'How can I fund a startup in India?')\n",
      "\n",
      "----\n",
      "0.9 ('Why was China never under the British rule?', 'Why was China never ruled by British or any other colonial powers?')\n",
      "0.9 ('Why was China never under the British rule?', 'Why was Virgin Islands (U.S.) never ruled by British or any other colonial powers?')\n",
      "0.9 ('Why was China never under the British rule?', 'Why was Timor-Leste never ruled by British or any other colonial powers?')\n",
      "\n",
      "----\n",
      "0.9 ('What is the valid reason for India only getting a few medals in the Olympics?', 'Why India fails to get medals in Olympics?')\n",
      "0.9 ('What is the valid reason for India only getting a few medals in the Olympics?', 'Why United Kingdom fails to get medals in Olympics?')\n",
      "0.8 ('What is the valid reason for United Kingdom only getting a few medals in the Olympics?', 'Why India fails to get medals in Olympics?')\n",
      "\n",
      "----\n",
      "\n",
      "\n",
      "Change numbers in one of the questions\n",
      "Test cases:      1500\n",
      "Test cases run:  500\n",
      "After filtering: 365 (73.0%)\n",
      "Fails (rate):    351 (96.2%)\n",
      "\n",
      "Example fails:\n",
      "0.8 ('\"What\\'s the meaning of the ending of season 1 of \"\"The Man in the High Castle\"\"? What actually happens at the end?\"', '\"What does the final phone call at the end of Season 1 Episode 1 in \"\"The Man in the High Castle\"\" mean?\"')\n",
      "0.8 ('\"What\\'s the meaning of the ending of season 1 of \"\"The Man in the High Castle\"\"? What actually happens at the end?\"', '\"What does the final phone call at the end of Season 2 Episode 2 in \"\"The Man in the High Castle\"\" mean?\"')\n",
      "0.8 ('\"What\\'s the meaning of the ending of season 1 of \"\"The Man in the High Castle\"\"? What actually happens at the end?\"', '\"What does the final phone call at the end of Season 2 Episode 2 in \"\"The Man in the High Castle\"\" mean?\"')\n",
      "\n",
      "----\n",
      "0.9 ('How will the ban of old 500 and 1000 rs notes help in bringing out the black money?', 'Replacing 500 and 1000 notes- how will this move reduce black money?')\n",
      "0.9 ('How will the ban of old 500 and 1005 rs notes help in bringing out the black money?', 'Replacing 500 and 1000 notes- how will this move reduce black money?')\n",
      "0.9 ('How will the ban of old 500 and 901 rs notes help in bringing out the black money?', 'Replacing 500 and 1000 notes- how will this move reduce black money?')\n",
      "\n",
      "----\n",
      "0.9 ('Does South Dakota State University plays division 1 in football?', 'Does South Dakota State University plays division 1 in baseball?')\n",
      "0.9 ('Does South Dakota State University plays division 2 in football?', 'Does South Dakota State University plays division 1 in baseball?')\n",
      "0.9 ('Does South Dakota State University plays division 2 in football?', 'Does South Dakota State University plays division 1 in baseball?')\n",
      "\n",
      "----\n",
      "\n",
      "\n",
      "Keep entitites, fill in with gibberish\n",
      "Test cases:      500\n",
      "Fails (rate):    419 (83.8%)\n",
      "\n",
      "Example fails:\n",
      "0.9 ('Has anyone received an admission offer from HKUST for 2017?', 'Has anyone received acceptance for MPhil/PhD 2016 (Engineering) from HKUST?')\n",
      "0.9 ('Has anyone received acceptance for MPhil/PhD 2016 (Engineering) from HKUST?', 'Has received MPhil/PhD 2016 from HKUST?')\n",
      "0.9 ('Has anyone received an admission offer from HKUST for 2017?', 'Has for 2017?')\n",
      "\n",
      "----\n",
      "0.0 ('What is the legality of vehicles, which are stopped by protesters in a public street, moving forward to break the human barricade?', 'If the car in front of you stops after the stop line and you stop at the stop line, do you legally have to stop a second time?')\n",
      "0.9 ('If the car in front of you stops after the stop line and you stop at the stop line, do you legally have to stop a second time?', 'If a second?')\n",
      "0.8 ('If the car in front of you stops after the stop line and you stop at the stop line, do you legally have to stop a second time?', 'If the second?')\n",
      "\n",
      "----\n",
      "0.9 ('What are the pros and cons of expert systems?', 'What are the pros and cons of AI research?')\n",
      "0.9 ('What are the pros and cons of AI research?', 'What is AI?')\n",
      "0.9 ('What are the pros and cons of AI research?', 'What about AI?')\n",
      "\n",
      "----\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "Temporal\n",
      "\n",
      "Is person X != Did person use to be X\n",
      "Test cases:      999\n",
      "Test cases run:  500\n",
      "Fails (rate):    500 (100.0%)\n",
      "\n",
      "Example fails:\n",
      "0.9 ('Is Alexander Sanders an interpreter?', 'Did Alexander Sanders use to be an interpreter?')\n",
      "----\n",
      "0.9 ('Is Amber Murphy a nurse?', 'Did Amber Murphy use to be a nurse?')\n",
      "----\n",
      "0.9 ('Is Anthony Carter an accountant?', 'Did Anthony Carter use to be an accountant?')\n",
      "----\n",
      "\n",
      "\n",
      "Is person X != Is person becoming X\n",
      "Test cases:      1000\n",
      "Test cases run:  500\n",
      "Fails (rate):    500 (100.0%)\n",
      "\n",
      "Example fails:\n",
      "0.9 ('Is Brandon Bennett an economist?', 'Is Brandon Bennett becoming an economist?')\n",
      "----\n",
      "0.9 ('Is Jennifer Rivera a person?', 'Is Jennifer Rivera becoming a person?')\n",
      "----\n",
      "0.9 ('Is Patrick Wood a candidate?', 'Is Patrick Wood becoming a candidate?')\n",
      "----\n",
      "\n",
      "\n",
      "What was person's life before becoming X != What was person's life after becoming X\n",
      "Test cases:      1000\n",
      "Test cases run:  500\n",
      "Fails (rate):    500 (100.0%)\n",
      "\n",
      "Example fails:\n",
      "0.9 (\"What was Justin Carter's life before becoming an adviser?\", \"What was Justin Carter's life after becoming an adviser?\")\n",
      "----\n",
      "0.9 (\"What was Melissa Stewart's life before becoming an executive?\", \"What was Melissa Stewart's life after becoming an executive?\")\n",
      "----\n",
      "0.9 (\"What was Erin Butler's life before becoming an intern?\", \"What was Erin Butler's life after becoming an intern?\")\n",
      "----\n",
      "\n",
      "\n",
      "Do you have to X your dog before Y it != Do you have to X your dog after Y it.\n",
      "Test cases:      1000\n",
      "Test cases run:  500\n",
      "Fails (rate):    500 (100.0%)\n",
      "\n",
      "Example fails:\n",
      "0.9 ('Do you have to weigh your dog before purchasing it?', 'Do you have to weigh your dog after purchasing it?')\n",
      "----\n",
      "0.9 ('Do you have to weigh your cat before getting it?', 'Do you have to weigh your cat after getting it?')\n",
      "----\n",
      "0.9 ('Do you have to weigh your dog before releasing it?', 'Do you have to weigh your dog after releasing it?')\n",
      "----\n",
      "\n",
      "\n",
      "Is it {ok, dangerous, ...} to {smoke, rest, ...} after != before\n",
      "Test cases:      1000\n",
      "Test cases run:  500\n",
      "Fails (rate):    500 (100.0%)\n",
      "\n",
      "Example fails:\n",
      "0.9 ('Is it reasonable to pray before 3am?', 'Is it reasonable to pray after 3am?')\n",
      "----\n",
      "0.9 ('Is it dangerous to pee before 8pm?', 'Is it dangerous to pee after 8pm?')\n",
      "----\n",
      "0.9 ('Is it reasonable to drink before 9am?', 'Is it reasonable to drink after 9am?')\n",
      "----\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "Negation\n",
      "\n",
      "How can I become a X person != How can I become a person who is not X\n",
      "Test cases:      1000\n",
      "Test cases run:  500\n",
      "Fails (rate):    500 (100.0%)\n",
      "\n",
      "Example fails:\n",
      "0.9 ('How can I become a cynical person?', 'How can I become a person who is not cynical?')\n",
      "----\n",
      "0.9 ('How can I become an alone person?', 'How can I become a person who is not alone?')\n",
      "----\n",
      "0.9 ('How can I become an ambitious person?', 'How can I become a person who is not ambitious?')\n",
      "----\n",
      "\n",
      "\n",
      "Is it {ok, dangerous, ...} to {smoke, rest, ...} in country != Is it {ok, dangerous, ...} not to {smoke, rest, ...} in country\n",
      "Test cases:      1000\n",
      "Test cases run:  500\n",
      "Fails (rate):    500 (100.0%)\n",
      "\n",
      "Example fails:\n",
      "0.9 ('Is it reasonable to discriminate in Djibouti?', 'Is it reasonable not to discriminate in Djibouti?')\n",
      "----\n",
      "0.9 ('Is it socially acceptable to surf in Palau?', 'Is it socially acceptable not to surf in Palau?')\n",
      "----\n",
      "0.9 ('Is it dangerous to vote in Senegal?', 'Is it dangerous not to vote in Senegal?')\n",
      "----\n",
      "\n",
      "\n",
      "What are things a {noun} should worry about != should not worry about.\n",
      "Test cases:      1000\n",
      "Test cases run:  500\n",
      "Fails (rate):    500 (100.0%)\n",
      "\n",
      "Example fails:\n",
      "0.9 ('What are things an interpreter should worry about?', 'What are things an interpreter should not worry about?')\n",
      "----\n",
      "0.9 ('What are things an accountant should worry about?', 'What are things an accountant should not worry about?')\n",
      "----\n",
      "0.9 ('What are things a kid should worry about?', 'What are things a kid should not worry about?')\n",
      "----\n",
      "\n",
      "\n",
      "How can I become a X person == How can I become a person who is not antonym(X)\n",
      "Test cases:      2000\n",
      "Test cases run:  500\n",
      "Fails (rate):    38 (7.6%)\n",
      "\n",
      "Example fails:\n",
      "0.4 ('How can I become an optimistic person?', 'How can I become a person who is not pessimistic?')\n",
      "----\n",
      "0.4 ('How can I become an optimistic person?', 'How can I become a person who is not pessimistic?')\n",
      "----\n",
      "0.4 ('How can I become an optimistic person?', 'How can I become a person who is not pessimistic?')\n",
      "----\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "Coref\n",
      "\n",
      "Simple coref: he and she\n",
      "Test cases:      2000\n",
      "Test cases run:  500\n",
      "Fails (rate):    500 (100.0%)\n",
      "\n",
      "Example fails:\n",
      "0.9 ('If Victoria and Jeffrey were alone, do you think he would reject her?', 'If Victoria and Jeffrey were alone, do you think she would reject him?')\n",
      "----\n",
      "0.9 ('If Samuel and Erin were alone, do you think he would reject her?', 'If Samuel and Erin were alone, do you think she would reject him?')\n",
      "----\n",
      "0.9 ('If Kimberly and Robert were alone, do you think he would reject her?', 'If Kimberly and Robert were alone, do you think she would reject him?')\n",
      "----\n",
      "\n",
      "\n",
      "Simple coref: his and her\n",
      "Test cases:      2000\n",
      "Test cases run:  500\n",
      "Fails (rate):    500 (100.0%)\n",
      "\n",
      "Example fails:\n",
      "0.9 ('If Jack and Allison were married, would his family be happy?', \"If Jack and Allison were married, would Allison's family be happy?\")\n",
      "----\n",
      "0.9 ('If Donald and Kathryn were married, would her family be happy?', \"If Donald and Kathryn were married, would Donald's family be happy?\")\n",
      "----\n",
      "0.9 ('If Shawn and Madison were married, would his family be happy?', \"If Shawn and Madison were married, would Madison's family be happy?\")\n",
      "----\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "SRL\n",
      "\n",
      "Who do X think - Who is the ... according to X\n",
      "Test cases:      1000\n",
      "Test cases run:  500\n",
      "Fails (rate):    0 (0.0%)\n",
      "\n",
      "\n",
      "Order does not matter for comparison\n",
      "Test cases:      990\n",
      "Test cases run:  500\n",
      "Fails (rate):    0 (0.0%)\n",
      "\n",
      "\n",
      "Order does not matter for symmetric relations\n",
      "Test cases:      990\n",
      "Test cases run:  500\n",
      "Fails (rate):    0 (0.0%)\n",
      "\n",
      "\n",
      "Order does matter for asymmetric relations\n",
      "Test cases:      988\n",
      "Test cases run:  500\n",
      "Fails (rate):    0 (0.0%)\n",
      "\n",
      "\n",
      "traditional SRL: active / passive swap\n",
      "Test cases:      1000\n",
      "Test cases run:  500\n",
      "Fails (rate):    0 (0.0%)\n",
      "\n",
      "\n",
      "traditional SRL: wrong active / passive swap\n",
      "Test cases:      1000\n",
      "Test cases run:  500\n",
      "Fails (rate):    500 (100.0%)\n",
      "\n",
      "Example fails:\n",
      "0.9 ('Did Olivia move the ticket?', 'Was Olivia moved by the ticket?')\n",
      "----\n",
      "0.9 ('Did Madison take the factory?', 'Was Madison taken by the factory?')\n",
      "----\n",
      "0.9 ('Did Brittany want the horse?', 'Was Brittany wanted by the horse?')\n",
      "----\n",
      "\n",
      "\n",
      "traditional SRL: active / passive swap with people\n",
      "Test cases:      990\n",
      "Test cases run:  500\n",
      "Fails (rate):    0 (0.0%)\n",
      "\n",
      "\n",
      "traditional SRL: wrong active / passive swap with people\n",
      "Test cases:      989\n",
      "Test cases run:  500\n",
      "Fails (rate):    500 (100.0%)\n",
      "\n",
      "Example fails:\n",
      "0.9 ('Does Rebecca notice Nicholas?', 'Is Rebecca noticed by Nicholas?')\n",
      "----\n",
      "0.9 ('Does Jacob dislike Jessica?', 'Is Jacob disliked by Jessica?')\n",
      "----\n",
      "0.9 ('Does Eric trust Mark?', 'Is Eric trusted by Mark?')\n",
      "----\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "Logic\n",
      "\n",
      "A or B is not the same as C and D\n",
      "Test cases:      828\n",
      "Test cases run:  500\n",
      "Fails (rate):    496 (99.2%)\n",
      "\n",
      "Example fails:\n",
      "0.9 ('Is Olivia Williams an attorney or a historian?', 'Is Olivia Williams simultaneously an activist and an entrepreneur?')\n",
      "----\n",
      "0.9 ('Is Tiffany Williams an educator or an architect?', 'Is Tiffany Williams simultaneously an executive and an assistant?')\n",
      "----\n",
      "0.9 ('Is Mary Morales an engineer or an educator?', 'Is Mary Morales simultaneously an accountant and an academic?')\n",
      "----\n",
      "\n",
      "\n",
      "A or B is not the same as A and B\n",
      "Test cases:      971\n",
      "Test cases run:  500\n",
      "Fails (rate):    500 (100.0%)\n",
      "\n",
      "Example fails:\n",
      "0.9 ('Is Zachary Murphy an advisor or an organizer?', 'Is Zachary Murphy simultaneously an advisor and an organizer?')\n",
      "----\n",
      "0.9 ('Is Sara Lewis an author or an executive?', 'Is Sara Lewis simultaneously an author and an executive?')\n",
      "----\n",
      "0.9 ('Is Michelle Jackson an investor or a producer?', 'Is Michelle Jackson simultaneously an investor and a producer?')\n",
      "----\n",
      "\n",
      "\n",
      "A and / or B is the same as B and / or A\n",
      "Test cases:      970\n",
      "Test cases run:  500\n",
      "Fails (rate):    0 (0.0%)\n",
      "\n",
      "\n",
      "a {nationality} {profession} = a {profession} and {nationality}\n",
      "Test cases:      1000\n",
      "Test cases run:  500\n",
      "Fails (rate):    0 (0.0%)\n",
      "\n",
      "\n",
      "Reflexivity: (q, q) should be duplicate\n",
      "Test cases:      1000\n",
      "Test cases run:  500\n",
      "Fails (rate):    0 (0.0%)\n",
      "\n",
      "\n",
      "Symmetry: f(a, b) = f(b, a)\n",
      "Test cases:      500\n",
      "Fails (rate):    29 (5.8%)\n",
      "\n",
      "Example fails:\n",
      "0.9 ('How and why has the United States become so divided politically and socially?', 'When and Why did America become a world power?')\n",
      "0.1 ('When and Why did America become a world power?', 'How and why has the United States become so divided politically and socially?')\n",
      "\n",
      "----\n",
      "0.1 ('How much money will Bill Ackman lose on the Herbalife short?', \"How much money can Bill Ackman earn in his short position on Herbalife, considering the best case scenario? If he isn't successful, how much can he lose?\")\n",
      "0.6 (\"How much money can Bill Ackman earn in his short position on Herbalife, considering the best case scenario? If he isn't successful, how much can he lose?\", 'How much money will Bill Ackman lose on the Herbalife short?')\n",
      "\n",
      "----\n",
      "0.3 ('How do I learn cpt and IPCC accounts journals entries in short term?', 'What are some good resources to learn journal entry (accounting)?')\n",
      "0.7 ('What are some good resources to learn journal entry (accounting)?', 'How do I learn cpt and IPCC accounts journals entries in short term?')\n",
      "\n",
      "----\n",
      "\n",
      "\n",
      "Testing implications\n",
      "Test cases:      8328\n",
      "Test cases run:  500\n",
      "After filtering: 439 (87.8%)\n",
      "Fails (rate):    86 (19.6%)\n",
      "\n",
      "Example fails:\n",
      "0.3 ('How will our economy will be affected by demonetizing Rs 500 and Rs 1000 notes?', 'What would be the effect on the Indian economy after banning 500 and 1,000 notes?')\n",
      "0.5 ('How will our economy will be affected by demonetizing Rs 500 and Rs 1000 notes?', 'Will the value of Indian rupee increase after the ban of 500 and 1000 rupee notes?')\n",
      "0.9 ('What would be the effect on the Indian economy after banning 500 and 1,000 notes?', 'Will the value of Indian rupee increase after the ban of 500 and 1000 rupee notes?')\n",
      "\n",
      "----\n",
      "0.9 (\"What are some tips to get over writer's block?\", 'How do you get over a writers block?')\n",
      "0.8 (\"What are some tips to get over writer's block?\", \"What should a writer do for inspiration when they're experiencing writer's block?\")\n",
      "0.3 ('How do you get over a writers block?', \"What should a writer do for inspiration when they're experiencing writer's block?\")\n",
      "\n",
      "----\n",
      "0.9 ('How do I impress girls?', 'How do I impress a girl on chat?')\n",
      "0.8 ('How do I impress girls?', 'What should be done to impress a girl?')\n",
      "0.3 ('How do I impress a girl on chat?', 'What should be done to impress a girl?')\n",
      "\n",
      "----\n",
      "\n",
      "\n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "suite.summary()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## SQuAD"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = pipeline(\"question-answering\", device=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "suite_path = '../release_data/squad/squad_suite.pkl'\n",
    "suite = TestSuite.from_file(suite_path)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Our suite has data as lists of (context, question) tuples. We'll adapt to this prediction format below:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "def predconfs(context_question_pairs):\n",
    "    preds = []\n",
    "    confs = []\n",
    "    for c, q in context_question_pairs:\n",
    "        try:\n",
    "            p = model(question=q, context=c, truncation=True, )\n",
    "        except:\n",
    "            print('Failed', q)\n",
    "            preds.append(' ')\n",
    "            confs.append(1)\n",
    "        preds.append(p['answer'])\n",
    "        confs.append(p['score'])\n",
    "    return preds, np.array(confs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "# suite.tests['Question typo'].run(predconfs,n=100, overwrite=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Running A is COMP than B. Who is more / less COMP?\n",
      "Predicting 200 examples\n",
      "Running Intensifiers (very, super, extremely) and reducers (somewhat, kinda, etc)?\n",
      "Predicting 1200 examples\n",
      "Running size, shape, age, color\n",
      "Predicting 400 examples\n",
      "Running Profession vs nationality\n",
      "Predicting 1000 examples\n",
      "Running Animal vs Vehicle\n",
      "Predicting 400 examples\n",
      "Running Animal vs Vehicle v2\n",
      "Predicting 400 examples\n",
      "Running Synonyms\n",
      "Predicting 400 examples\n",
      "Running A is COMP than B. Who is antonym(COMP)? B\n",
      "Predicting 400 examples\n",
      "Running A is more X than B. Who is more antonym(X)? B. Who is less X? B. Who is more X? A. Who is less antonym(X)? A.\n",
      "Predicting 1600 examples\n",
      "Running Question typo\n",
      "Predicting 200 examples\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "ERROR:transformers.tokenization_utils:We need to remove 115 to truncate the inputbut the first sequence has a length 21. Please select another truncation strategy than TruncationStrategy.ONLY_FIRST, for instance 'longest_first' or 'only_second'.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Failed What do the three richest people in the world posses more of than the lowest 48 nations together?\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "ERROR:transformers.tokenization_utils:We need to remove 116 to truncate the inputbut the first sequence has a length 22. Please select another truncation strategy than TruncationStrategy.ONLY_FIRST, for instance 'longest_first' or 'only_second'.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Failed What do the three richest people in hte world posses more of than the lowest 48 nations together?\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "ERROR:transformers.tokenization_utils:We need to remove 193 to truncate the inputbut the first sequence has a length 19. Please select another truncation strategy than TruncationStrategy.ONLY_FIRST, for instance 'longest_first' or 'only_second'.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Failed Which newspaper's parent company could not evade tax by shifting its residence to the Netherlands?\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "ERROR:transformers.tokenization_utils:We need to remove 195 to truncate the inputbut the first sequence has a length 21. Please select another truncation strategy than TruncationStrategy.ONLY_FIRST, for instance 'longest_first' or 'only_second'.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Failed Which enwspaper's parent company could not evade tax by shifting its residence to the Netherlands?\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "ERROR:transformers.tokenization_utils:We need to remove 12 to truncate the inputbut the first sequence has a length 10. Please select another truncation strategy than TruncationStrategy.ONLY_FIRST, for instance 'longest_first' or 'only_second'.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Failed How many men were in Robert's army?\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "ERROR:transformers.tokenization_utils:We need to remove 14 to truncate the inputbut the first sequence has a length 12. Please select another truncation strategy than TruncationStrategy.ONLY_FIRST, for instance 'longest_first' or 'only_second'.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Failed How many men were in Robetr's army?\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "ERROR:transformers.tokenization_utils:We need to remove 449 to truncate the inputbut the first sequence has a length 27. Please select another truncation strategy than TruncationStrategy.ONLY_FIRST, for instance 'longest_first' or 'only_second'.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Failed What did the Dutch health authorities regard as unnecessary in Geraets-Smits v Stichting Ziekenfonds?\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "ERROR:transformers.tokenization_utils:We need to remove 451 to truncate the inputbut the first sequence has a length 29. Please select another truncation strategy than TruncationStrategy.ONLY_FIRST, for instance 'longest_first' or 'only_second'.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Failed What did the Dutch health authoritiesr egard as unnecessary in Geraets-Smits v Stichting Ziekenfonds?\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "ERROR:transformers.tokenization_utils:We need to remove 112 to truncate the inputbut the first sequence has a length 12. Please select another truncation strategy than TruncationStrategy.ONLY_FIRST, for instance 'longest_first' or 'only_second'.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Failed What type of company is Van Gend en Loos?\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "ERROR:transformers.tokenization_utils:We need to remove 114 to truncate the inputbut the first sequence has a length 14. Please select another truncation strategy than TruncationStrategy.ONLY_FIRST, for instance 'longest_first' or 'only_second'.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Failed What type of company is aVn Gend en Loos?\n",
      "Running Question contractions\n",
      "Predicting 201 examples\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "ERROR:transformers.tokenization_utils:We need to remove 41 to truncate the inputbut the first sequence has a length 23. Please select another truncation strategy than TruncationStrategy.ONLY_FIRST, for instance 'longest_first' or 'only_second'.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Failed What happens to the jellyfish nematocysts when they are eaten by the haeckelia?\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "ERROR:transformers.tokenization_utils:We need to remove 42 to truncate the inputbut the first sequence has a length 24. Please select another truncation strategy than TruncationStrategy.ONLY_FIRST, for instance 'longest_first' or 'only_second'.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Failed What happens to the jellyfish nematocysts when they're eaten by the haeckelia?\n",
      "Running Add random sentence to context\n",
      "Predicting 300 examples\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "ERROR:transformers.tokenization_utils:We need to remove 28 to truncate the inputbut the first sequence has a length 10. Please select another truncation strategy than TruncationStrategy.ONLY_FIRST, for instance 'longest_first' or 'only_second'.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Failed When were the Mongols defeated by the Tran?\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "ERROR:transformers.tokenization_utils:We need to remove 42 to truncate the inputbut the first sequence has a length 10. Please select another truncation strategy than TruncationStrategy.ONLY_FIRST, for instance 'longest_first' or 'only_second'.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Failed When were the Mongols defeated by the Tran?\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "ERROR:transformers.tokenization_utils:We need to remove 42 to truncate the inputbut the first sequence has a length 10. Please select another truncation strategy than TruncationStrategy.ONLY_FIRST, for instance 'longest_first' or 'only_second'.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Failed When were the Mongols defeated by the Tran?\n",
      "Running Change name everywhere\n",
      "Predicting 1100 examples\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "ERROR:transformers.tokenization_utils:We need to remove 102 to truncate the inputbut the first sequence has a length 15. Please select another truncation strategy than TruncationStrategy.ONLY_FIRST, for instance 'longest_first' or 'only_second'.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Failed What is the largest item from Italy that is part of the sculpture collection?\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "ERROR:transformers.tokenization_utils:We need to remove 100 to truncate the inputbut the first sequence has a length 15. Please select another truncation strategy than TruncationStrategy.ONLY_FIRST, for instance 'longest_first' or 'only_second'.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Failed What is the largest item from Italy that is part of the sculpture collection?\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "ERROR:transformers.tokenization_utils:We need to remove 102 to truncate the inputbut the first sequence has a length 15. Please select another truncation strategy than TruncationStrategy.ONLY_FIRST, for instance 'longest_first' or 'only_second'.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Failed What is the largest item from Italy that is part of the sculpture collection?\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "ERROR:transformers.tokenization_utils:We need to remove 102 to truncate the inputbut the first sequence has a length 15. Please select another truncation strategy than TruncationStrategy.ONLY_FIRST, for instance 'longest_first' or 'only_second'.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Failed What is the largest item from Italy that is part of the sculpture collection?\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "ERROR:transformers.tokenization_utils:We need to remove 102 to truncate the inputbut the first sequence has a length 15. Please select another truncation strategy than TruncationStrategy.ONLY_FIRST, for instance 'longest_first' or 'only_second'.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Failed What is the largest item from Italy that is part of the sculpture collection?\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "ERROR:transformers.tokenization_utils:We need to remove 101 to truncate the inputbut the first sequence has a length 15. Please select another truncation strategy than TruncationStrategy.ONLY_FIRST, for instance 'longest_first' or 'only_second'.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Failed What is the largest item from Italy that is part of the sculpture collection?\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "ERROR:transformers.tokenization_utils:We need to remove 102 to truncate the inputbut the first sequence has a length 15. Please select another truncation strategy than TruncationStrategy.ONLY_FIRST, for instance 'longest_first' or 'only_second'.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Failed What is the largest item from Italy that is part of the sculpture collection?\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "ERROR:transformers.tokenization_utils:We need to remove 102 to truncate the inputbut the first sequence has a length 15. Please select another truncation strategy than TruncationStrategy.ONLY_FIRST, for instance 'longest_first' or 'only_second'.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Failed What is the largest item from Italy that is part of the sculpture collection?\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "ERROR:transformers.tokenization_utils:We need to remove 100 to truncate the inputbut the first sequence has a length 15. Please select another truncation strategy than TruncationStrategy.ONLY_FIRST, for instance 'longest_first' or 'only_second'.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Failed What is the largest item from Italy that is part of the sculpture collection?\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "ERROR:transformers.tokenization_utils:We need to remove 101 to truncate the inputbut the first sequence has a length 15. Please select another truncation strategy than TruncationStrategy.ONLY_FIRST, for instance 'longest_first' or 'only_second'.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Failed What is the largest item from Italy that is part of the sculpture collection?\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "ERROR:transformers.tokenization_utils:We need to remove 102 to truncate the inputbut the first sequence has a length 15. Please select another truncation strategy than TruncationStrategy.ONLY_FIRST, for instance 'longest_first' or 'only_second'.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Failed What is the largest item from Italy that is part of the sculpture collection?\n",
      "Running Change location everywhere\n",
      "Predicting 1100 examples\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "ERROR:transformers.tokenization_utils:We need to remove 18 to truncate the inputbut the first sequence has a length 16. Please select another truncation strategy than TruncationStrategy.ONLY_FIRST, for instance 'longest_first' or 'only_second'.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Failed What are some examples of territories where a member state is responsible for external relations?\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "ERROR:transformers.tokenization_utils:We need to remove 18 to truncate the inputbut the first sequence has a length 16. Please select another truncation strategy than TruncationStrategy.ONLY_FIRST, for instance 'longest_first' or 'only_second'.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Failed What are some examples of territories where a member state is responsible for external relations?\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "ERROR:transformers.tokenization_utils:We need to remove 18 to truncate the inputbut the first sequence has a length 16. Please select another truncation strategy than TruncationStrategy.ONLY_FIRST, for instance 'longest_first' or 'only_second'.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Failed What are some examples of territories where a member state is responsible for external relations?\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "ERROR:transformers.tokenization_utils:We need to remove 16 to truncate the inputbut the first sequence has a length 16. Please select another truncation strategy than TruncationStrategy.ONLY_FIRST, for instance 'longest_first' or 'only_second'.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Failed What are some examples of territories where a member state is responsible for external relations?\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "ERROR:transformers.tokenization_utils:We need to remove 449 to truncate the inputbut the first sequence has a length 27. Please select another truncation strategy than TruncationStrategy.ONLY_FIRST, for instance 'longest_first' or 'only_second'.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Failed What did the Dutch health authorities regard as unnecessary in Geraets-Smits v Stichting Ziekenfonds?\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "ERROR:transformers.tokenization_utils:We need to remove 450 to truncate the inputbut the first sequence has a length 27. Please select another truncation strategy than TruncationStrategy.ONLY_FIRST, for instance 'longest_first' or 'only_second'.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Failed What did the Dutch health authorities regard as unnecessary in Geraets-Smits v Stichting Ziekenfonds?\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "ERROR:transformers.tokenization_utils:We need to remove 453 to truncate the inputbut the first sequence has a length 27. Please select another truncation strategy than TruncationStrategy.ONLY_FIRST, for instance 'longest_first' or 'only_second'.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Failed What did the Dutch health authorities regard as unnecessary in Geraets-Smits v Stichting Ziekenfonds?\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "ERROR:transformers.tokenization_utils:We need to remove 449 to truncate the inputbut the first sequence has a length 27. Please select another truncation strategy than TruncationStrategy.ONLY_FIRST, for instance 'longest_first' or 'only_second'.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Failed What did the Dutch health authorities regard as unnecessary in Geraets-Smits v Stichting Ziekenfonds?\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "ERROR:transformers.tokenization_utils:We need to remove 449 to truncate the inputbut the first sequence has a length 27. Please select another truncation strategy than TruncationStrategy.ONLY_FIRST, for instance 'longest_first' or 'only_second'.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Failed What did the Dutch health authorities regard as unnecessary in Geraets-Smits v Stichting Ziekenfonds?\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "ERROR:transformers.tokenization_utils:We need to remove 449 to truncate the inputbut the first sequence has a length 27. Please select another truncation strategy than TruncationStrategy.ONLY_FIRST, for instance 'longest_first' or 'only_second'.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Failed What did the Dutch health authorities regard as unnecessary in Geraets-Smits v Stichting Ziekenfonds?\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "ERROR:transformers.tokenization_utils:We need to remove 451 to truncate the inputbut the first sequence has a length 27. Please select another truncation strategy than TruncationStrategy.ONLY_FIRST, for instance 'longest_first' or 'only_second'.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Failed What did the Dutch health authorities regard as unnecessary in Geraets-Smits v Stichting Ziekenfonds?\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "ERROR:transformers.tokenization_utils:We need to remove 453 to truncate the inputbut the first sequence has a length 27. Please select another truncation strategy than TruncationStrategy.ONLY_FIRST, for instance 'longest_first' or 'only_second'.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Failed What did the Dutch health authorities regard as unnecessary in Geraets-Smits v Stichting Ziekenfonds?\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "ERROR:transformers.tokenization_utils:We need to remove 449 to truncate the inputbut the first sequence has a length 27. Please select another truncation strategy than TruncationStrategy.ONLY_FIRST, for instance 'longest_first' or 'only_second'.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Failed What did the Dutch health authorities regard as unnecessary in Geraets-Smits v Stichting Ziekenfonds?\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "ERROR:transformers.tokenization_utils:We need to remove 453 to truncate the inputbut the first sequence has a length 27. Please select another truncation strategy than TruncationStrategy.ONLY_FIRST, for instance 'longest_first' or 'only_second'.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Failed What did the Dutch health authorities regard as unnecessary in Geraets-Smits v Stichting Ziekenfonds?\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "ERROR:transformers.tokenization_utils:We need to remove 449 to truncate the inputbut the first sequence has a length 27. Please select another truncation strategy than TruncationStrategy.ONLY_FIRST, for instance 'longest_first' or 'only_second'.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Failed What did the Dutch health authorities regard as unnecessary in Geraets-Smits v Stichting Ziekenfonds?\n",
      "Running There was a change in profession\n",
      "Predicting 200 examples\n",
      "Running Understanding before / after -> first / last.\n",
      "Predicting 400 examples\n",
      "Running Negation in context, may or may not be in question\n",
      "Predicting 400 examples\n",
      "Running Negation in question only.\n",
      "Predicting 800 examples\n",
      "Running M/F failure rates should be similar for different professions\n",
      "Predicting 100 examples\n",
      "Running Basic coref, he / she\n",
      "Predicting 800 examples\n",
      "Running Basic coref, his / her\n",
      "Predicting 200 examples\n",
      "Running Former / Latter\n",
      "Predicting 400 examples\n",
      "Running Agent / object distinction\n",
      "Predicting 400 examples\n",
      "Running Agent / object distinction with 3 agents\n",
      "Predicting 1600 examples\n"
     ]
    }
   ],
   "source": [
    "suite.run(predconfs, n=100, overwrite=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "def format_squad_with_context(x, pred, conf, label=None, *args, **kwargs):\n",
    "    c, q = x\n",
    "    ret = 'C: %s\\nQ: %s\\n' % (c, q)\n",
    "    if label is not None:\n",
    "        ret += 'A: %s\\n' % label\n",
    "    ret += 'P: %s\\n' % pred\n",
    "    return ret"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Vocabulary\n",
      "\n",
      "A is COMP than B. Who is more / less COMP?\n",
      "Test cases:      494\n",
      "Test cases run:  100\n",
      "Fails (rate):    100 (100.0%)\n",
      "\n",
      "Example fails:\n",
      "C: Timothy is cleaner than Kelly.\n",
      "Q: Who is less clean?\n",
      "A: Kelly\n",
      "P: Timothy\n",
      "\n",
      "\n",
      "----\n",
      "C: David is older than Samantha.\n",
      "Q: Who is less old?\n",
      "A: Samantha\n",
      "P: David\n",
      "\n",
      "\n",
      "----\n",
      "C: Michael is faster than Jose.\n",
      "Q: Who is less fast?\n",
      "A: Jose\n",
      "P: Michael\n",
      "\n",
      "\n",
      "----\n",
      "\n",
      "\n",
      "Intensifiers (very, super, extremely) and reducers (somewhat, kinda, etc)?\n",
      "Test cases:      497\n",
      "Test cases run:  100\n",
      "Fails (rate):    100 (100.0%)\n",
      "\n",
      "Example fails:\n",
      "C: Maria is open about the project. Michelle is a little open about the project.\n",
      "Q: Who is least open about the project?\n",
      "A: Michelle\n",
      "P: Maria\n",
      "\n",
      "C: Michelle is a little open about the project. Maria is open about the project.\n",
      "Q: Who is least open about the project?\n",
      "A: Michelle\n",
      "P: Maria\n",
      "\n",
      "C: Maria is very open about the project. Michelle is a little open about the project.\n",
      "Q: Who is least open about the project?\n",
      "A: Michelle\n",
      "P: Maria\n",
      "\n",
      "\n",
      "----\n",
      "C: Victoria is excited about the project. Christina is particularly excited about the project.\n",
      "Q: Who is least excited about the project?\n",
      "A: Victoria\n",
      "P: Christina\n",
      "\n",
      "C: Victoria is somewhat excited about the project. Christina is particularly excited about the project.\n",
      "Q: Who is least excited about the project?\n",
      "A: Victoria\n",
      "P: Christina\n",
      "\n",
      "C: Christina is particularly excited about the project. Victoria is excited about the project.\n",
      "Q: Who is least excited about the project?\n",
      "A: Victoria\n",
      "P: Christina\n",
      "\n",
      "\n",
      "----\n",
      "C: Anna is particular about the project. Isabella is somewhat particular about the project.\n",
      "Q: Who is most particular about the project?\n",
      "A: Anna\n",
      "P: Isabella\n",
      "\n",
      "C: Anna is really particular about the project. Isabella is particular about the project.\n",
      "Q: Who is most particular about the project?\n",
      "A: Anna\n",
      "P: Isabella\n",
      "\n",
      "C: Anna is really particular about the project. Isabella is somewhat particular about the project.\n",
      "Q: Who is most particular about the project?\n",
      "A: Anna\n",
      "P: Isabella\n",
      "\n",
      "\n",
      "----\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "Taxonomy\n",
      "\n",
      "size, shape, age, color\n",
      "Test cases:      500\n",
      "Test cases run:  100\n",
      "Fails (rate):    100 (100.0%)\n",
      "\n",
      "Example fails:\n",
      "C: There is a small oval table in the room.\n",
      "Q: What size is the table?\n",
      "A: small\n",
      "P: oval\n",
      "\n",
      "C: There is a table in the room. The table is small and oval.\n",
      "Q: What shape is the table?\n",
      "A: oval\n",
      "P: small and oval.\n",
      "\n",
      "C: There is a table in the room. The table is small and oval.\n",
      "Q: What size is the table?\n",
      "A: small\n",
      "P: small and oval.\n",
      "\n",
      "\n",
      "----\n",
      "C: There is a thing in the room. The thing is big and yellow.\n",
      "Q: What size is the thing?\n",
      "A: big\n",
      "P: big and yellow.\n",
      "\n",
      "C: There is a thing in the room. The thing is big and yellow.\n",
      "Q: What color is the thing?\n",
      "A: yellow\n",
      "P: yellow.\n",
      "\n",
      "C: There is a big yellow thing in the room.\n",
      "Q: What size is the thing?\n",
      "A: big\n",
      "P: yellow\n",
      "\n",
      "\n",
      "----\n",
      "C: There is a new black sculpture in the room.\n",
      "Q: What age is the sculpture?\n",
      "A: new\n",
      "P: black\n",
      "\n",
      "C: There is a sculpture in the room. The sculpture is new and black.\n",
      "Q: What color is the sculpture?\n",
      "A: black\n",
      "P: black.\n",
      "\n",
      "C: There is a sculpture in the room. The sculpture is new and black.\n",
      "Q: What age is the sculpture?\n",
      "A: new\n",
      "P: The sculpture is new and black.\n",
      "\n",
      "\n",
      "----\n",
      "\n",
      "\n",
      "Profession vs nationality\n",
      "Test cases:      500\n",
      "Test cases run:  100\n",
      "Fails (rate):    18 (18.0%)\n",
      "\n",
      "Example fails:\n",
      "C: Austin is a Nigerian executive.\n",
      "Q: What is Austin's job?\n",
      "A: executive\n",
      "P: Nigerian executive.\n",
      "\n",
      "\n",
      "----\n",
      "C: Katherine is an American assistant.\n",
      "Q: What is Katherine's job?\n",
      "A: assistant\n",
      "P: American assistant.\n",
      "\n",
      "\n",
      "----\n",
      "C: Kimberly is a Russian intern.\n",
      "Q: What is Kimberly's job?\n",
      "A: intern\n",
      "P: Russian intern.\n",
      "\n",
      "C: Kimberly is an intern and Russian.\n",
      "Q: What is Kimberly's job?\n",
      "A: intern\n",
      "P: intern and Russian.\n",
      "\n",
      "\n",
      "----\n",
      "\n",
      "\n",
      "Animal vs Vehicle\n",
      "Test cases:      500\n",
      "Test cases run:  100\n",
      "Fails (rate):    55 (55.0%)\n",
      "\n",
      "Example fails:\n",
      "C: Kelly has a cow and a truck.\n",
      "Q: What vehicle does Kelly have?\n",
      "A: truck\n",
      "P: a cow and a truck.\n",
      "\n",
      "\n",
      "----\n",
      "C: Maria has a hamster and a bike.\n",
      "Q: What vehicle does Maria have?\n",
      "A: bike\n",
      "P: hamster and a bike.\n",
      "\n",
      "C: Maria has a bike and a hamster.\n",
      "Q: What vehicle does Maria have?\n",
      "A: bike\n",
      "P: bike and a hamster.\n",
      "\n",
      "\n",
      "----\n",
      "C: Scott has a snake and a firetruck.\n",
      "Q: What vehicle does Scott have?\n",
      "A: firetruck\n",
      "P: a snake and a firetruck.\n",
      "\n",
      "C: Scott has a firetruck and a snake.\n",
      "Q: What vehicle does Scott have?\n",
      "A: firetruck\n",
      "P: firetruck and a snake.\n",
      "\n",
      "\n",
      "----\n",
      "\n",
      "\n",
      "Animal vs Vehicle v2\n",
      "Test cases:      496\n",
      "Test cases run:  100\n",
      "Fails (rate):    61 (61.0%)\n",
      "\n",
      "Example fails:\n",
      "C: Jeremy bought a fish. Charles bought a SUV.\n",
      "Q: Who bought an animal?\n",
      "A: Jeremy\n",
      "P: Charles\n",
      "\n",
      "C: Charles bought a SUV. Jeremy bought a fish.\n",
      "Q: Who bought an animal?\n",
      "A: Jeremy\n",
      "P: Charles\n",
      "\n",
      "\n",
      "----\n",
      "C: Joseph bought a serpent. Brittany bought a train.\n",
      "Q: Who bought a vehicle?\n",
      "A: Brittany\n",
      "P: Joseph\n",
      "\n",
      "C: Brittany bought a train. Joseph bought a serpent.\n",
      "Q: Who bought a vehicle?\n",
      "A: Brittany\n",
      "P: Joseph\n",
      "\n",
      "\n",
      "----\n",
      "C: Kimberly bought a guinea pig. Madison bought a firetruck.\n",
      "Q: Who bought a vehicle?\n",
      "A: Madison\n",
      "P: Kimberly bought a guinea pig. Madison\n",
      "\n",
      "\n",
      "----\n",
      "\n",
      "\n",
      "Synonyms\n",
      "Test cases:      447\n",
      "Test cases run:  100\n",
      "Fails (rate):    15 (15.0%)\n",
      "\n",
      "Example fails:\n",
      "C: Lisa is very happy. Jason is very grateful.\n",
      "Q: Who is joyful?\n",
      "A: Lisa\n",
      "P: Jason\n",
      "\n",
      "\n",
      "----\n",
      "C: Jordan is very vocal. Taylor is very intelligent.\n",
      "Q: Who is outspoken?\n",
      "A: Jordan\n",
      "P: Taylor\n",
      "\n",
      "\n",
      "----\n",
      "C: Jessica is very vocal. Robert is very happy.\n",
      "Q: Who is outspoken?\n",
      "A: Jessica\n",
      "P: Robert\n",
      "\n",
      "\n",
      "----\n",
      "\n",
      "\n",
      "A is COMP than B. Who is antonym(COMP)? B\n",
      "Test cases:      496\n",
      "Test cases run:  100\n",
      "Fails (rate):    100 (100.0%)\n",
      "\n",
      "Example fails:\n",
      "C: Timothy is darker than Amber.\n",
      "Q: Who is lighter?\n",
      "A: Amber\n",
      "P: Timothy\n",
      "\n",
      "C: Amber is lighter than Timothy.\n",
      "Q: Who is darker?\n",
      "A: Timothy\n",
      "P: Amber is lighter than Timothy.\n",
      "\n",
      "\n",
      "----\n",
      "C: Jason is slower than Stephanie.\n",
      "Q: Who is faster?\n",
      "A: Stephanie\n",
      "P: Jason\n",
      "\n",
      "C: Stephanie is faster than Jason.\n",
      "Q: Who is slower?\n",
      "A: Jason\n",
      "P: Stephanie is faster than Jason.\n",
      "\n",
      "C: Stephanie is faster than Jason.\n",
      "Q: Who is faster?\n",
      "A: Stephanie\n",
      "P: Stephanie is faster than Jason.\n",
      "\n",
      "\n",
      "----\n",
      "C: Christian is warmer than Daniel.\n",
      "Q: Who is colder?\n",
      "A: Daniel\n",
      "P: Christian\n",
      "\n",
      "C: Daniel is colder than Christian.\n",
      "Q: Who is warmer?\n",
      "A: Christian\n",
      "P: Daniel\n",
      "\n",
      "\n",
      "----\n",
      "\n",
      "\n",
      "A is more X than B. Who is more antonym(X)? B. Who is less X? B. Who is more X? A. Who is less antonym(X)? A.\n",
      "Test cases:      491\n",
      "Test cases run:  100\n",
      "Fails (rate):    100 (100.0%)\n",
      "\n",
      "Example fails:\n",
      "C: Jeremy is more unhappy than Christopher.\n",
      "Q: Who is less unhappy?\n",
      "A: Christopher\n",
      "P: Jeremy\n",
      "\n",
      "C: Jeremy is less happy than Christopher.\n",
      "Q: Who is less unhappy?\n",
      "A: Christopher\n",
      "P: Jeremy\n",
      "\n",
      "C: Jeremy is less happy than Christopher.\n",
      "Q: Who is more happy?\n",
      "A: Christopher\n",
      "P: Jeremy\n",
      "\n",
      "\n",
      "----\n",
      "C: Ashley is more dependent than Jessica.\n",
      "Q: Who is less dependent?\n",
      "A: Jessica\n",
      "P: Ashley\n",
      "\n",
      "C: Ashley is more dependent than Jessica.\n",
      "Q: Who is more independent?\n",
      "A: Jessica\n",
      "P: Ashley\n",
      "\n",
      "C: Ashley is less independent than Jessica.\n",
      "Q: Who is more independent?\n",
      "A: Jessica\n",
      "P: Ashley\n",
      "\n",
      "\n",
      "----\n",
      "C: Zachary is more visible than Timothy.\n",
      "Q: Who is more invisible?\n",
      "A: Timothy\n",
      "P: Zachary\n",
      "\n",
      "C: Zachary is more visible than Timothy.\n",
      "Q: Who is less visible?\n",
      "A: Timothy\n",
      "P: Zachary\n",
      "\n",
      "C: Timothy is more invisible than Zachary.\n",
      "Q: Who is more visible?\n",
      "A: Zachary\n",
      "P: Timothy\n",
      "\n",
      "\n",
      "----\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "Robustness\n",
      "\n",
      "Question typo\n",
      "Test cases:      500\n",
      "Test cases run:  100\n",
      "Fails (rate):    23 (23.0%)\n",
      "\n",
      "Example fails:\n",
      "C: The \"freedom to provide services\" under TFEU article 56 applies to people who give services \"for remuneration\", especially commercial or professional activity. For example, in Van Binsbergen v Bestuur van de Bedrijfvereniging voor de Metaalnijverheid a Dutch lawyer moved to Belgium while advising a client in a social security case, and was told he could not continue because Dutch law said only people established in the Netherlands could give legal advice. The Court of Justice held that the freedom to provide services applied, it was directly effective, and the rule was probably unjustified: having an address in the member state would be enough to pursue the legitimate aim of good administration of justice. The Court of Justice has held that secondary education falls outside the scope of article 56, because usually the state funds it, though higher education does not. Health care generally counts as a service. In Geraets-Smits v Stichting Ziekenfonds Mrs Geraets-Smits claimed she should be reimbursed by Dutch social insurance for costs of receiving treatment in Germany. The Dutch health authorities regarded the treatment unnecessary, so she argued this restricted the freedom (of the German health clinic) to provide services. Several governments submitted that hospital services should not be regarded as economic, and should not fall within article 56. But the Court of Justice held health was a \"service\" even though the government (rather than the service recipient) paid for the service. National authorities could be justified in refusing to reimburse patients for medical services abroad if the health care received at home was without undue delay, and it followed \"international medical science\" on which treatments counted as normal and necessary. The Court requires that the individual circumstances of a patient justify waiting lists, and this is also true in the context of the UK's National Health Service. Aside from public services, another sensitive field of services are those classified as illegal. Josemans v Burgemeester van Maastricht held that the Netherlands' regulation of cannabis consumption, including the prohibitions by some municipalities on tourists (but not Dutch nationals) going to coffee shops, fell outside article 56 altogether. The Court of Justice reasoned that narcotic drugs were controlled in all member states, and so this differed from other cases where prostitution or other quasi-legal activity was subject to restriction. If an activity does fall within article 56, a restriction can be justified under article 52 or overriding requirements developed by the Court of Justice. In Alpine Investments BV v Minister van Financiën a business that sold commodities futures (with Merrill Lynch and another banking firms) attempted to challenge a Dutch law that prohibiting cold calling customers. The Court of Justice held the Dutch prohibition pursued a legitimate aim to prevent \"undesirable developments in securities trading\" including protecting the consumer from aggressive sales tactics, thus maintaining confidence in the Dutch markets. In Omega Spielhallen GmbH v Bonn a \"laserdrome\" business was banned by the Bonn council. It bought fake laser gun services from a UK firm called Pulsar Ltd, but residents had protested against \"playing at killing\" entertainment. The Court of Justice held that the German constitutional value of human dignity, which underpinned the ban, did count as a justified restriction on freedom to provide services. In Liga Portuguesa de Futebol v Santa Casa da Misericórdia de Lisboa the Court of Justice also held that the state monopoly on gambling, and a penalty for a Gibraltar firm that had sold internet gambling services, was justified to prevent fraud and gambling where people's views were highly divergent. The ban was proportionate as this was an appropriate and necessary way to tackle the serious problems of fraud that arise over the internet. In the Services Directive a group of justifications were codified in article 16 that the case law has developed.\n",
      "Q: What did the Dutch health authorities regard as unnecessary in Geraets-Smits v Stichting Ziekenfonds?\n",
      "P: the lost chloroplast's existence.\n",
      "\n",
      "C: The \"freedom to provide services\" under TFEU article 56 applies to people who give services \"for remuneration\", especially commercial or professional activity. For example, in Van Binsbergen v Bestuur van de Bedrijfvereniging voor de Metaalnijverheid a Dutch lawyer moved to Belgium while advising a client in a social security case, and was told he could not continue because Dutch law said only people established in the Netherlands could give legal advice. The Court of Justice held that the freedom to provide services applied, it was directly effective, and the rule was probably unjustified: having an address in the member state would be enough to pursue the legitimate aim of good administration of justice. The Court of Justice has held that secondary education falls outside the scope of article 56, because usually the state funds it, though higher education does not. Health care generally counts as a service. In Geraets-Smits v Stichting Ziekenfonds Mrs Geraets-Smits claimed she should be reimbursed by Dutch social insurance for costs of receiving treatment in Germany. The Dutch health authorities regarded the treatment unnecessary, so she argued this restricted the freedom (of the German health clinic) to provide services. Several governments submitted that hospital services should not be regarded as economic, and should not fall within article 56. But the Court of Justice held health was a \"service\" even though the government (rather than the service recipient) paid for the service. National authorities could be justified in refusing to reimburse patients for medical services abroad if the health care received at home was without undue delay, and it followed \"international medical science\" on which treatments counted as normal and necessary. The Court requires that the individual circumstances of a patient justify waiting lists, and this is also true in the context of the UK's National Health Service. Aside from public services, another sensitive field of services are those classified as illegal. Josemans v Burgemeester van Maastricht held that the Netherlands' regulation of cannabis consumption, including the prohibitions by some municipalities on tourists (but not Dutch nationals) going to coffee shops, fell outside article 56 altogether. The Court of Justice reasoned that narcotic drugs were controlled in all member states, and so this differed from other cases where prostitution or other quasi-legal activity was subject to restriction. If an activity does fall within article 56, a restriction can be justified under article 52 or overriding requirements developed by the Court of Justice. In Alpine Investments BV v Minister van Financiën a business that sold commodities futures (with Merrill Lynch and another banking firms) attempted to challenge a Dutch law that prohibiting cold calling customers. The Court of Justice held the Dutch prohibition pursued a legitimate aim to prevent \"undesirable developments in securities trading\" including protecting the consumer from aggressive sales tactics, thus maintaining confidence in the Dutch markets. In Omega Spielhallen GmbH v Bonn a \"laserdrome\" business was banned by the Bonn council. It bought fake laser gun services from a UK firm called Pulsar Ltd, but residents had protested against \"playing at killing\" entertainment. The Court of Justice held that the German constitutional value of human dignity, which underpinned the ban, did count as a justified restriction on freedom to provide services. In Liga Portuguesa de Futebol v Santa Casa da Misericórdia de Lisboa the Court of Justice also held that the state monopoly on gambling, and a penalty for a Gibraltar firm that had sold internet gambling services, was justified to prevent fraud and gambling where people's views were highly divergent. The ban was proportionate as this was an appropriate and necessary way to tackle the serious problems of fraud that arise over the internet. In the Services Directive a group of justifications were codified in article 16 that the case law has developed.\n",
      "Q: What did the Dutch health authoritiesr egard as unnecessary in Geraets-Smits v Stichting Ziekenfonds?\n",
      "P: lost chloroplast's existence.\n",
      "\n",
      "\n",
      "----\n",
      "C: Today, Warsaw has some of the best medical facilities in Poland and East-Central Europe. The city is home to the Children's Memorial Health Institute (CMHI), the highest-reference hospital in all of Poland, as well as an active research and education center. While the Maria Skłodowska-Curie Institute of Oncology it is one of the largest and most modern oncological institutions in Europe. The clinical section is located in a 10-floor building with 700 beds, 10 operating theatres, an intensive care unit, several diagnostic departments as well as an outpatient clinic. The infrastructure has developed a lot over the past years.\n",
      "Q: What is one of the largest and most modern oncological institutions in Europe?\n",
      "P:  \n",
      "\n",
      "C: Today, Warsaw has some of the best medical facilities in Poland and East-Central Europe. The city is home to the Children's Memorial Health Institute (CMHI), the highest-reference hospital in all of Poland, as well as an active research and education center. While the Maria Skłodowska-Curie Institute of Oncology it is one of the largest and most modern oncological institutions in Europe. The clinical section is located in a 10-floor building with 700 beds, 10 operating theatres, an intensive care unit, several diagnostic departments as well as an outpatient clinic. The infrastructure has developed a lot over the past years.\n",
      "Q: What is one of the largest and most modern oncological isntitutions in Europe?\n",
      "P: Peter Howell\n",
      "\n",
      "\n",
      "----\n",
      "C: The French and Indian War (1754–1763) was the North American theater of the worldwide Seven Years' War. The war was fought between the colonies of British America and New France, with both sides supported by military units from their parent countries of Great Britain and France, as well as Native American allies. At the start of the war, the French North American colonies had a population of roughly 60,000 European settlers, compared with 2 million in the British North American colonies. The outnumbered French particularly depended on the Indians. Long in conflict, the metropole nations declared war on each other in 1756, escalating the war from a regional affair into an intercontinental conflict.\n",
      "Q: When was the French and Indian War?\n",
      "P: 1933\n",
      "\n",
      "C: The French and Indian War (1754–1763) was the North American theater of the worldwide Seven Years' War. The war was fought between the colonies of British America and New France, with both sides supported by military units from their parent countries of Great Britain and France, as well as Native American allies. At the start of the war, the French North American colonies had a population of roughly 60,000 European settlers, compared with 2 million in the British North American colonies. The outnumbered French particularly depended on the Indians. Long in conflict, the metropole nations declared war on each other in 1756, escalating the war from a regional affair into an intercontinental conflict.\n",
      "Q: When was the French an dIndian War?\n",
      "P: since\n",
      "\n",
      "\n",
      "----\n",
      "\n",
      "\n",
      "Question contractions\n",
      "Test cases:      500\n",
      "Test cases run:  100\n",
      "Fails (rate):    7 (7.0%)\n",
      "\n",
      "Example fails:\n",
      "C: Almost all ctenophores are predators – there are no vegetarians and only one genus that is partly parasitic. If food is plentiful, they can eat 10 times their own weight per day. While Beroe preys mainly on other ctenophores, other surface-water species prey on zooplankton (planktonic animals) ranging in size from the microscopic, including mollusc and fish larvae, to small adult crustaceans such as copepods, amphipods, and even krill. Members of the genus Haeckelia prey on jellyfish and incorporate their prey's nematocysts (stinging cells) into their own tentacles instead of colloblasts. Ctenophores have been compared to spiders in their wide range of techniques from capturing prey – some hang motionless in the water using their tentacles as \"webs\", some are ambush predators like Salticid jumping spiders, and some dangle a sticky droplet at the end of a fine thread, as bolas spiders do. This variety explains the wide range of body forms in a phylum with rather few species. The two-tentacled \"cydippid\" Lampea feeds exclusively on salps, close relatives of sea-squirts that form large chain-like floating colonies, and juveniles of Lampea attach themselves like parasites to salps that are too large for them to swallow. Members of the cydippid genus Pleurobrachia and the lobate Bolinopsis often reach high population densities at the same place and time because they specialize in different types of prey: Pleurobrachia's long tentacles mainly capture relatively strong swimmers such as adult copepods, while Bolinopsis generally feeds on smaller, weaker swimmers such as rotifers and mollusc and crustacean larvae.\n",
      "Q: What happens to the jellyfish nematocysts when they are eaten by the haeckelia?\n",
      "P:  \n",
      "\n",
      "C: Almost all ctenophores are predators – there are no vegetarians and only one genus that is partly parasitic. If food is plentiful, they can eat 10 times their own weight per day. While Beroe preys mainly on other ctenophores, other surface-water species prey on zooplankton (planktonic animals) ranging in size from the microscopic, including mollusc and fish larvae, to small adult crustaceans such as copepods, amphipods, and even krill. Members of the genus Haeckelia prey on jellyfish and incorporate their prey's nematocysts (stinging cells) into their own tentacles instead of colloblasts. Ctenophores have been compared to spiders in their wide range of techniques from capturing prey – some hang motionless in the water using their tentacles as \"webs\", some are ambush predators like Salticid jumping spiders, and some dangle a sticky droplet at the end of a fine thread, as bolas spiders do. This variety explains the wide range of body forms in a phylum with rather few species. The two-tentacled \"cydippid\" Lampea feeds exclusively on salps, close relatives of sea-squirts that form large chain-like floating colonies, and juveniles of Lampea attach themselves like parasites to salps that are too large for them to swallow. Members of the cydippid genus Pleurobrachia and the lobate Bolinopsis often reach high population densities at the same place and time because they specialize in different types of prey: Pleurobrachia's long tentacles mainly capture relatively strong swimmers such as adult copepods, while Bolinopsis generally feeds on smaller, weaker swimmers such as rotifers and mollusc and crustacean larvae.\n",
      "Q: What happens to the jellyfish nematocysts when they're eaten by the haeckelia?\n",
      "P: ctDNA,\n",
      "\n",
      "\n",
      "----\n",
      "C: One of the main functions of the chloroplast is its role in photosynthesis, the process by which light is transformed into chemical energy, to subsequently produce food in the form of sugars. Water (H2O) and carbon dioxide (CO2) are used in photosynthesis, and sugar and oxygen (O2) is made, using light energy. Photosynthesis is divided into two stages—the light reactions, where water is split to produce oxygen, and the dark reactions, or Calvin cycle, which builds sugar molecules from carbon dioxide. The two phases are linked by the energy carriers adenosine triphosphate (ATP) and nicotinamide adenine dinucleotide phosphate (NADP+).\n",
      "Q: What is the most important role of chloroplasts?\n",
      "P: to counteract the constant flooding and strong sedimentation in the western Rhine Delta.\n",
      "\n",
      "C: One of the main functions of the chloroplast is its role in photosynthesis, the process by which light is transformed into chemical energy, to subsequently produce food in the form of sugars. Water (H2O) and carbon dioxide (CO2) are used in photosynthesis, and sugar and oxygen (O2) is made, using light energy. Photosynthesis is divided into two stages—the light reactions, where water is split to produce oxygen, and the dark reactions, or Calvin cycle, which builds sugar molecules from carbon dioxide. The two phases are linked by the energy carriers adenosine triphosphate (ATP) and nicotinamide adenine dinucleotide phosphate (NADP+).\n",
      "Q: What's the most important role of chloroplasts?\n",
      "P: Fußach,\n",
      "\n",
      "\n",
      "----\n",
      "C: Other important complexity classes include BPP, ZPP and RP, which are defined using probabilistic Turing machines; AC and NC, which are defined using Boolean circuits; and BQP and QMA, which are defined using quantum Turing machines. #P is an important complexity class of counting problems (not decision problems). Classes like IP and AM are defined using Interactive proof systems. ALL is the class of all decision problems.\n",
      "Q: What are three examples of complexity classes associated with definitions established by probabilistic Turing machines?\n",
      "P: a fee per unit of connection time,\n",
      "\n",
      "C: Other important complexity classes include BPP, ZPP and RP, which are defined using probabilistic Turing machines; AC and NC, which are defined using Boolean circuits; and BQP and QMA, which are defined using quantum Turing machines. #P is an important complexity class of counting problems (not decision problems). Classes like IP and AM are defined using Interactive proof systems. ALL is the class of all decision problems.\n",
      "Q: What're three examples of complexity classes associated with definitions established by probabilistic Turing machines?\n",
      "P: cellular communication services,\n",
      "\n",
      "\n",
      "----\n",
      "\n",
      "\n",
      "Add random sentence to context\n",
      "Test cases:      500\n",
      "Test cases run:  100\n",
      "Fails (rate):    24 (24.0%)\n",
      "\n",
      "Example fails:\n",
      "C: The United Methodist Church is one tradition within the Christian Church. The United Methodist Church is active in ecumenical relations with other Christian groups and denominations. It is a member of the National Council of Churches, the World Council of Churches, Churches Uniting in Christ, and Christian Churches Together. In addition, it voted to seek observer status in the National Association of Evangelicals and in the World Evangelical Fellowship. However, there are some in The United Methodist Church who feel that false ecumenism might result in the \"blurring of theological and confessional differences in the interests of unity.\"\n",
      "Q: Some in the UMC feel that false ecumenism might result in what?\n",
      "P: \"blurring of theological and confessional differences in the interests of unity.\"\n",
      "\n",
      "C: Many scholars arguing for the Y. pestis as the major agent of the pandemic suggest that its extent and symptoms can be explained by a combination of bubonic plague with other diseases, including typhus, smallpox and respiratory infections. The United Methodist Church is one tradition within the Christian Church. The United Methodist Church is active in ecumenical relations with other Christian groups and denominations. It is a member of the National Council of Churches, the World Council of Churches, Churches Uniting in Christ, and Christian Churches Together. In addition, it voted to seek observer status in the National Association of Evangelicals and in the World Evangelical Fellowship. However, there are some in The United Methodist Church who feel that false ecumenism might result in the \"blurring of theological and confessional differences in the interests of unity.\"\n",
      "Q: Some in the UMC feel that false ecumenism might result in what?\n",
      "P: the \"blurring of theological and confessional differences in the interests of unity.\"\n",
      "\n",
      "C: The United Methodist Church is one tradition within the Christian Church. The United Methodist Church is active in ecumenical relations with other Christian groups and denominations. It is a member of the National Council of Churches, the World Council of Churches, Churches Uniting in Christ, and Christian Churches Together. In addition, it voted to seek observer status in the National Association of Evangelicals and in the World Evangelical Fellowship. However, there are some in The United Methodist Church who feel that false ecumenism might result in the \"blurring of theological and confessional differences in the interests of unity.\"Many scholars arguing for the Y. pestis as the major agent of the pandemic suggest that its extent and symptoms can be explained by a combination of bubonic plague with other diseases, including typhus, smallpox and respiratory infections. \n",
      "Q: Some in the UMC feel that false ecumenism might result in what?\n",
      "P: \"blurring of theological and confessional differences in the interests of unity.\"Many\n",
      "\n",
      "\n",
      "----\n",
      "C: An increase in imported cars into North America forced General Motors, Ford and Chrysler to introduce smaller and fuel-efficient models for domestic sales. The Dodge Omni / Plymouth Horizon from Chrysler, the Ford Fiesta and the Chevrolet Chevette all had four-cylinder engines and room for at least four passengers by the late 1970s. By 1985, the average American vehicle moved 17.4 miles per gallon, compared to 13.5 in 1970. The improvements stayed even though the price of a barrel of oil remained constant at $12 from 1974 to 1979. Sales of large sedans for most makes (except Chrysler products) recovered within two model years of the 1973 crisis. The Cadillac DeVille and Fleetwood, Buick Electra, Oldsmobile 98, Lincoln Continental, Mercury Marquis, and various other luxury oriented sedans became popular again in the mid-1970s. The only full-size models that did not recover were lower price models such as the Chevrolet Bel Air, and Ford Galaxie 500. Slightly smaller, mid-size models such as the Oldsmobile Cutlass, Chevrolet Monte Carlo, Ford Thunderbird and various other models sold well.\n",
      "Q: Why did GM, Ford and Chrysler introduced fuel-efficient and small cars to the US market?\n",
      "P: An increase in imported cars\n",
      "\n",
      "C: Former \"Model C\" schools are not private schools, as they are state-controlled. An increase in imported cars into North America forced General Motors, Ford and Chrysler to introduce smaller and fuel-efficient models for domestic sales. The Dodge Omni / Plymouth Horizon from Chrysler, the Ford Fiesta and the Chevrolet Chevette all had four-cylinder engines and room for at least four passengers by the late 1970s. By 1985, the average American vehicle moved 17.4 miles per gallon, compared to 13.5 in 1970. The improvements stayed even though the price of a barrel of oil remained constant at $12 from 1974 to 1979. Sales of large sedans for most makes (except Chrysler products) recovered within two model years of the 1973 crisis. The Cadillac DeVille and Fleetwood, Buick Electra, Oldsmobile 98, Lincoln Continental, Mercury Marquis, and various other luxury oriented sedans became popular again in the mid-1970s. The only full-size models that did not recover were lower price models such as the Chevrolet Bel Air, and Ford Galaxie 500. Slightly smaller, mid-size models such as the Oldsmobile Cutlass, Chevrolet Monte Carlo, Ford Thunderbird and various other models sold well.\n",
      "Q: Why did GM, Ford and Chrysler introduced fuel-efficient and small cars to the US market?\n",
      "P: An increase in imported cars into North America\n",
      "\n",
      "\n",
      "----\n",
      "C: In front of the Presiding Officers' desk is the parliamentary mace, which is made from silver and inlaid with gold panned from Scottish rivers and inscribed with the words: Wisdom, Compassion, Justice and Integrity. The words There shall be a Scottish Parliament, which are the first words of the Scotland Act, are inscribed around the head of the mace, which has a formal ceremonial role in the meetings of Parliament, reinforcing the authority of the Parliament in its ability to make laws. Presented to the Scottish Parliament by the Queen upon its official opening in July 1999, the mace is displayed in a glass case suspended from the lid. At the beginning of each sitting in the chamber, the lid of the case is rotated so that the mace is above the glass, to symbolise that a full meeting of the Parliament is taking place.\n",
      "Q: What is the parliamentary mace made of?\n",
      "P: silver\n",
      "\n",
      "C: Many well-known designers of stained glass are represented in the collection including, from the 19th century:. In front of the Presiding Officers' desk is the parliamentary mace, which is made from silver and inlaid with gold panned from Scottish rivers and inscribed with the words: Wisdom, Compassion, Justice and Integrity. The words There shall be a Scottish Parliament, which are the first words of the Scotland Act, are inscribed around the head of the mace, which has a formal ceremonial role in the meetings of Parliament, reinforcing the authority of the Parliament in its ability to make laws. Presented to the Scottish Parliament by the Queen upon its official opening in July 1999, the mace is displayed in a glass case suspended from the lid. At the beginning of each sitting in the chamber, the lid of the case is rotated so that the mace is above the glass, to symbolise that a full meeting of the Parliament is taking place.\n",
      "Q: What is the parliamentary mace made of?\n",
      "P: silver and inlaid with gold panned from Scottish rivers\n",
      "\n",
      "\n",
      "----\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "NER\n",
      "\n",
      "Change name everywhere\n",
      "Test cases:      500\n",
      "Test cases run:  100\n",
      "Fails (rate):    11 (11.0%)\n",
      "\n",
      "Example fails:\n",
      "C: Some disagree with such double or triple non-French linguistic origins, arguing that for the word to have spread into common use in France, it must have originated in the French language. The \"Hugues hypothesis\" argues that the name was derived by association with Hugues Capet, king of France, who reigned long before the Reformation. He was regarded by the Gallicans and Protestants as a noble man who respected people's dignity and lives. Janet Gray and other supporters of the hypothesis suggest that the name huguenote would be roughly equivalent to little Hugos, or those who want Hugo.\n",
      "Q: Other theories of the word's origin can be generally classed as what?\n",
      "P: October 12, 1943\n",
      "\n",
      "C: Some disagree with such double or triple non-French linguistic origins, arguing that for the word to have spread into common use in France, it must have originated in the French language. The \"Hugues hypothesis\" argues that the name was derived by association with Hugues Capet, king of France, who reigned long before the Reformation. He was regarded by the Gallicans and Protestants as a noble man who respected people's dignity and lives. Jennifer Gonzalez and other supporters of the hypothesis suggest that the name huguenote would be roughly equivalent to little Hugos, or those who want Hugo.\n",
      "Q: Other theories of the word's origin can be generally classed as what?\n",
      "P:  \n",
      "\n",
      "C: Some disagree with such double or triple non-French linguistic origins, arguing that for the word to have spread into common use in France, it must have originated in the French language. The \"Hugues hypothesis\" argues that the name was derived by association with Hugues Capet, king of France, who reigned long before the Reformation. He was regarded by the Gallicans and Protestants as a noble man who respected people's dignity and lives. Ashley Rogers and other supporters of the hypothesis suggest that the name huguenote would be roughly equivalent to little Hugos, or those who want Hugo.\n",
      "Q: Other theories of the word's origin can be generally classed as what?\n",
      "P:  \n",
      "\n",
      "\n",
      "----\n",
      "C: 2013 Economics Nobel prize winner Robert J. Shiller said that rising inequality in the United States and elsewhere is the most important problem. Increasing inequality harms economic growth. High and persistent unemployment, in which inequality increases, has a negative effect on subsequent long-run economic growth. Unemployment can harm growth not only because it is a waste of resources, but also because it generates redistributive pressures and subsequent distortions, drives people to poverty, constrains liquidity limiting labor mobility, and erodes self-esteem promoting social dislocation, unrest and conflict. Policies aiming at controlling unemployment and in particular at reducing its inequality-associated effects support economic growth.\n",
      "Q: What year did Robert J. Shiller win an Economics Nobel prize?\n",
      "P: Colin Baker and Sylvester McCoy\n",
      "\n",
      "C: 2013 Economics Nobel prize winner Michael Murphy said that rising inequality in the United States and elsewhere is the most important problem. Increasing inequality harms economic growth. High and persistent unemployment, in which inequality increases, has a negative effect on subsequent long-run economic growth. Unemployment can harm growth not only because it is a waste of resources, but also because it generates redistributive pressures and subsequent distortions, drives people to poverty, constrains liquidity limiting labor mobility, and erodes self-esteem promoting social dislocation, unrest and conflict. Policies aiming at controlling unemployment and in particular at reducing its inequality-associated effects support economic growth.\n",
      "Q: What year did Michael Murphy win an Economics Nobel prize?\n",
      "P: David Watson and Sylvester McCoy\n",
      "\n",
      "C: 2013 Economics Nobel prize winner Christopher Peterson said that rising inequality in the United States and elsewhere is the most important problem. Increasing inequality harms economic growth. High and persistent unemployment, in which inequality increases, has a negative effect on subsequent long-run economic growth. Unemployment can harm growth not only because it is a waste of resources, but also because it generates redistributive pressures and subsequent distortions, drives people to poverty, constrains liquidity limiting labor mobility, and erodes self-esteem promoting social dislocation, unrest and conflict. Policies aiming at controlling unemployment and in particular at reducing its inequality-associated effects support economic growth.\n",
      "Q: What year did Christopher Peterson win an Economics Nobel prize?\n",
      "P: Michael Evans and Sylvester McCoy\n",
      "\n",
      "\n",
      "----\n",
      "C: Islamists have asked the question, \"If Islam is a way of life, how can we say that those who want to live by its principles in legal, social, political, economic, and political spheres of life are not Muslims, but Islamists and believe in Islamism, not [just] Islam?\" Similarly, a writer for the International Crisis Group maintains that \"the conception of 'political Islam'\" is a creation of Americans to explain the Iranian Islamic Revolution and apolitical Islam was a historical fluke of the \"short-lived era of the heyday of secular Arab nationalism between 1945 and 1970\", and it is quietist/non-political Islam, not Islamism, that requires explanation.\n",
      "Q: When was the heyday of secular Arab nationalism?\n",
      "P: BAFTA nomination for the series, getting a Best Supporting Actress\n",
      "\n",
      "C: Islamists have asked the question, \"If Ryan is a way of life, how can we say that those who want to live by its principles in legal, social, political, economic, and political spheres of life are not Muslims, but Islamists and believe in Islamism, not [just] Ryan?\" Similarly, a writer for the International Crisis Group maintains that \"the conception of 'political Ryan'\" is a creation of Americans to explain the Iranian Islamic Revolution and apolitical Ryan was a historical fluke of the \"short-lived era of the heyday of secular Arab nationalism between 1945 and 1970\", and it is quietist/non-political Ryan, not Islamism, that requires explanation.\n",
      "Q: When was the heyday of secular Arab nationalism?\n",
      "P: Best Supporting Actress\n",
      "\n",
      "C: Islamists have asked the question, \"If Samuel is a way of life, how can we say that those who want to live by its principles in legal, social, political, economic, and political spheres of life are not Muslims, but Islamists and believe in Islamism, not [just] Samuel?\" Similarly, a writer for the International Crisis Group maintains that \"the conception of 'political Samuel'\" is a creation of Americans to explain the Iranian Islamic Revolution and apolitical Samuel was a historical fluke of the \"short-lived era of the heyday of secular Arab nationalism between 1945 and 1970\", and it is quietist/non-political Samuel, not Islamism, that requires explanation.\n",
      "Q: When was the heyday of secular Arab nationalism?\n",
      "P: Best Supporting Actress\n",
      "\n",
      "\n",
      "----\n",
      "\n",
      "\n",
      "Change location everywhere\n",
      "Test cases:      500\n",
      "Test cases run:  100\n",
      "Fails (rate):    62 (62.0%)\n",
      "\n",
      "Example fails:\n",
      "C: The crisis had a major impact on international relations and created a rift within NATO. Some European nations and Japan sought to disassociate themselves from United States foreign policy in the Middle East to avoid being targeted by the boycott. Arab oil producers linked any future policy changes to peace between the belligerents. To address this, the Nixon Administration began multilateral negotiations with the combatants. They arranged for Israel to pull back from the Sinai Peninsula and the Golan Heights. By January 18, 1974, US Secretary of State Henry Kissinger had negotiated an Israeli troop withdrawal from parts of the Sinai Peninsula. The promise of a negotiated settlement between Israel and Syria was enough to convince Arab oil producers to lift the embargo in March 1974.\n",
      "Q: On what date did Henry Kissinger negotiate an Israeli troop withdrawal from the Sinai Peninsula?\n",
      "P: Golden Gate Bridge.\n",
      "\n",
      "C: The crisis had a major impact on international relations and created a rift within NATO. Some European nations and Japan sought to disassociate themselves from United States foreign policy in the Middle East to avoid being targeted by the boycott. Arab oil producers linked any future policy changes to peace between the belligerents. To address this, the Nixon Administration began multilateral negotiations with the combatants. They arranged for Slovak Republic to pull back from the Sinai Peninsula and the Golan Heights. By January 18, 1974, US Secretary of State Henry Kissinger had negotiated an Israeli troop withdrawal from parts of the Sinai Peninsula. The promise of a negotiated settlement between Slovak Republic and Syria was enough to convince Arab oil producers to lift the embargo in March 1974.\n",
      "Q: On what date did Henry Kissinger negotiate an Israeli troop withdrawal from the Sinai Peninsula?\n",
      "P: January 18, 1974,\n",
      "\n",
      "C: The crisis had a major impact on international relations and created a rift within NATO. Some European nations and Japan sought to disassociate themselves from United States foreign policy in the Middle East to avoid being targeted by the boycott. Arab oil producers linked any future policy changes to peace between the belligerents. To address this, the Nixon Administration began multilateral negotiations with the combatants. They arranged for Czech Republic to pull back from the Sinai Peninsula and the Golan Heights. By January 18, 1974, US Secretary of State Henry Kissinger had negotiated an Israeli troop withdrawal from parts of the Sinai Peninsula. The promise of a negotiated settlement between Czech Republic and Syria was enough to convince Arab oil producers to lift the embargo in March 1974.\n",
      "Q: On what date did Henry Kissinger negotiate an Israeli troop withdrawal from the Sinai Peninsula?\n",
      "P: January 18, 1974,\n",
      "\n",
      "\n",
      "----\n",
      "C: The first historical reference to Warsaw dates back to the year 1313, at a time when Kraków served as the Polish capital city. Due to its central location between the Polish–Lithuanian Commonwealth's capitals of Kraków and Vilnius, Warsaw became the capital of the Commonwealth and of the Crown of the Kingdom of Poland when King Sigismund III Vasa moved his court from Kraków to Warsaw in 1596. After the Third Partition of Poland in 1795, Warsaw was incorporated into the Kingdom of Prussia. In 1806 during the Napoleonic Wars, the city became the official capital of the Grand Duchy of Warsaw, a puppet state of the First French Empire established by Napoleon Bonaparte. In accordance with the decisions of the Congress of Vienna, the Russian Empire annexed Warsaw in 1815 and it became part of the \"Congress Kingdom\". Only in 1918 did it regain independence from the foreign rule and emerge as a new capital of the independent Republic of Poland. The German invasion in 1939, the massacre of the Jewish population and deportations to concentration camps led to the uprising in the Warsaw ghetto in 1943 and to the major and devastating Warsaw Uprising between August and October 1944. Warsaw gained the title of the \"Phoenix City\" because it has survived many wars, conflicts and invasions throughout its long history. Most notably, the city required painstaking rebuilding after the extensive damage it suffered in World War II, which destroyed 85% of its buildings. On 9 November 1940, the city was awarded Poland's highest military decoration for heroism, the Virtuti Militari, during the Siege of Warsaw (1939).\n",
      "Q: When is the first reference in history to Warsaw?\n",
      "P: 50%\n",
      "\n",
      "C: The first historical reference to Warsaw dates back to the year 1313, at a time when Kraków served as the Polish capital city. Due to its central location between the Polish–Lithuanian Commonwealth's capitals of Kraków and Vilnius, Warsaw became the capital of the Commonwealth and of the Crown of the Kingdom of Afghanistan when King Sigismund III Vasa moved his court from Kraków to Warsaw in 1596. After the Third Partition of Afghanistan in 1795, Warsaw was incorporated into the Kingdom of Prussia. In 1806 during the Napoleonic Wars, the city became the official capital of the Grand Duchy of Warsaw, a puppet state of the First French Empire established by Napoleon Bonaparte. In accordance with the decisions of the Congress of Vienna, the Russian Empire annexed Warsaw in 1815 and it became part of the \"Congress Kingdom\". Only in 1918 did it regain independence from the foreign rule and emerge as a new capital of the independent Republic of Afghanistan. The German invasion in 1939, the massacre of the Jewish population and deportations to concentration camps led to the uprising in the Warsaw ghetto in 1943 and to the major and devastating Warsaw Uprising between August and October 1944. Warsaw gained the title of the \"Phoenix City\" because it has survived many wars, conflicts and invasions throughout its long history. Most notably, the city required painstaking rebuilding after the extensive damage it suffered in World War II, which destroyed 85% of its buildings. On 9 November 1940, the city was awarded Afghanistan's highest military decoration for heroism, the Virtuti Militari, during the Siege of Warsaw (1939).\n",
      "Q: When is the first reference in history to Warsaw?\n",
      "P: 1313,\n",
      "\n",
      "C: The first historical reference to Warsaw dates back to the year 1313, at a time when Kraków served as the Polish capital city. Due to its central location between the Polish–Lithuanian Commonwealth's capitals of Kraków and Vilnius, Warsaw became the capital of the Commonwealth and of the Crown of the Kingdom of Serbia when King Sigismund III Vasa moved his court from Kraków to Warsaw in 1596. After the Third Partition of Serbia in 1795, Warsaw was incorporated into the Kingdom of Prussia. In 1806 during the Napoleonic Wars, the city became the official capital of the Grand Duchy of Warsaw, a puppet state of the First French Empire established by Napoleon Bonaparte. In accordance with the decisions of the Congress of Vienna, the Russian Empire annexed Warsaw in 1815 and it became part of the \"Congress Kingdom\". Only in 1918 did it regain independence from the foreign rule and emerge as a new capital of the independent Republic of Serbia. The German invasion in 1939, the massacre of the Jewish population and deportations to concentration camps led to the uprising in the Warsaw ghetto in 1943 and to the major and devastating Warsaw Uprising between August and October 1944. Warsaw gained the title of the \"Phoenix City\" because it has survived many wars, conflicts and invasions throughout its long history. Most notably, the city required painstaking rebuilding after the extensive damage it suffered in World War II, which destroyed 85% of its buildings. On 9 November 1940, the city was awarded Serbia's highest military decoration for heroism, the Virtuti Militari, during the Siege of Warsaw (1939).\n",
      "Q: When is the first reference in history to Warsaw?\n",
      "P: 1313,\n",
      "\n",
      "\n",
      "----\n",
      "C: In October 2010, the open-access scientific journal PLoS Pathogens published a paper by a multinational team who undertook a new investigation into the role of Yersinia pestis in the Black Death following the disputed identification by Drancourt and Raoult in 1998. They assessed the presence of DNA/RNA with Polymerase Chain Reaction (PCR) techniques for Y. pestis from the tooth sockets in human skeletons from mass graves in northern, central and southern Europe that were associated archaeologically with the Black Death and subsequent resurgences. The authors concluded that this new research, together with prior analyses from the south of France and Germany, \". . . ends the debate about the etiology of the Black Death, and unambiguously demonstrates that Y. pestis was the causative agent of the epidemic plague that devastated Europe during the Middle Ages\".\n",
      "Q: When did the Plos Pathogens paper come out?\n",
      "P: 1313,\n",
      "\n",
      "C: In October 2010, the open-access scientific journal PLoS Pathogens published a paper by a multinational team who undertook a new investigation into the role of Yersinia pestis in the Black Death following the disputed identification by Drancourt and Raoult in 1998. They assessed the presence of DNA/RNA with Polymerase Chain Reaction (PCR) techniques for Y. pestis from the tooth sockets in human skeletons from mass graves in northern, central and southern Europe that were associated archaeologically with the Black Death and subsequent resurgences. The authors concluded that this new research, together with prior analyses from the south of France and Tonga, \". . . ends the debate about the etiology of the Black Death, and unambiguously demonstrates that Y. pestis was the causative agent of the epidemic plague that devastated Europe during the Middle Ages\".\n",
      "Q: When did the Plos Pathogens paper come out?\n",
      "P: October 2010,\n",
      "\n",
      "C: In October 2010, the open-access scientific journal PLoS Pathogens published a paper by a multinational team who undertook a new investigation into the role of Yersinia pestis in the Black Death following the disputed identification by Drancourt and Raoult in 1998. They assessed the presence of DNA/RNA with Polymerase Chain Reaction (PCR) techniques for Y. pestis from the tooth sockets in human skeletons from mass graves in northern, central and southern Europe that were associated archaeologically with the Black Death and subsequent resurgences. The authors concluded that this new research, together with prior analyses from the south of Brazil and Germany, \". . . ends the debate about the etiology of the Black Death, and unambiguously demonstrates that Y. pestis was the causative agent of the epidemic plague that devastated Europe during the Middle Ages\".\n",
      "Q: When did the Plos Pathogens paper come out?\n",
      "P: October 2010,\n",
      "\n",
      "\n",
      "----\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "Fairness\n",
      "\n",
      "M/F failure rates should be similar for different professions\n",
      "Test cases:      2000\n",
      "Test cases run:  100\n",
      "Fails (rate):    93 (93.0%)\n",
      "\n",
      "Example fails:\n",
      "C: Luis is not a nurse, Tracy is.\n",
      "Q: Who is a nurse?\n",
      "A: Tracy\n",
      "P: Luis\n",
      "\n",
      "----\n",
      "C: Tiffany is not a doctor, John is.\n",
      "Q: Who is a doctor?\n",
      "A: John\n",
      "P: Tiffany\n",
      "\n",
      "----\n",
      "C: Erin is not a nurse, Michael is.\n",
      "Q: Who is a nurse?\n",
      "A: Michael\n",
      "P: Erin\n",
      "\n",
      "----\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "Temporal\n",
      "\n",
      "There was a change in profession\n",
      "Test cases:      482\n",
      "Test cases run:  100\n",
      "Fails (rate):    100 (100.0%)\n",
      "\n",
      "Example fails:\n",
      "C: Both Rachel and Lauren were escorts, but there was a change in Rachel, who is now an auditor.\n",
      "Q: Who is an auditor?\n",
      "A: Rachel\n",
      "P: Rachel,\n",
      "\n",
      "C: Both Lauren and Rachel were escorts, but there was a change in Rachel, who is now an auditor.\n",
      "Q: Who is an auditor?\n",
      "A: Rachel\n",
      "P: Rachel,\n",
      "\n",
      "\n",
      "----\n",
      "C: Both Natalie and Madison were producers, but there was a change in Natalie, who is now an executive.\n",
      "Q: Who is an executive?\n",
      "A: Natalie\n",
      "P: Natalie,\n",
      "\n",
      "C: Both Madison and Natalie were producers, but there was a change in Natalie, who is now an executive.\n",
      "Q: Who is an executive?\n",
      "A: Natalie\n",
      "P: Natalie,\n",
      "\n",
      "\n",
      "----\n",
      "C: Both Ethan and Amanda were interpreters, but there was a change in Ethan, who is now an attorney.\n",
      "Q: Who is an attorney?\n",
      "A: Ethan\n",
      "P: Ethan,\n",
      "\n",
      "C: Both Amanda and Ethan were interpreters, but there was a change in Ethan, who is now an attorney.\n",
      "Q: Who is an attorney?\n",
      "A: Ethan\n",
      "P: Ethan,\n",
      "\n",
      "\n",
      "----\n",
      "\n",
      "\n",
      "Understanding before / after -> first / last.\n",
      "Test cases:      496\n",
      "Test cases run:  100\n",
      "Fails (rate):    100 (100.0%)\n",
      "\n",
      "Example fails:\n",
      "C: Noah became a author before Jacob did.\n",
      "Q: Who became a author last?\n",
      "A: Jacob\n",
      "P: Noah\n",
      "\n",
      "C: Jacob became a author after Noah did.\n",
      "Q: Who became a author first?\n",
      "A: Noah\n",
      "P: Jacob\n",
      "\n",
      "C: Jacob became a author after Noah did.\n",
      "Q: Who became a author last?\n",
      "A: Jacob\n",
      "P: Noah\n",
      "\n",
      "\n",
      "----\n",
      "C: Nicole became a investor before Joseph did.\n",
      "Q: Who became a investor last?\n",
      "A: Joseph\n",
      "P: Nicole\n",
      "\n",
      "C: Joseph became a investor after Nicole did.\n",
      "Q: Who became a investor first?\n",
      "A: Nicole\n",
      "P: Joseph\n",
      "\n",
      "\n",
      "----\n",
      "C: Eric became a administrator before Abigail did.\n",
      "Q: Who became a administrator last?\n",
      "A: Abigail\n",
      "P: Eric\n",
      "\n",
      "C: Abigail became a administrator after Eric did.\n",
      "Q: Who became a administrator first?\n",
      "A: Eric\n",
      "P: Abigail\n",
      "\n",
      "\n",
      "----\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "Negation\n",
      "\n",
      "Negation in context, may or may not be in question\n",
      "Test cases:      499\n",
      "Test cases run:  100\n",
      "Fails (rate):    92 (92.0%)\n",
      "\n",
      "Example fails:\n",
      "C: Christian is not an accountant. Emily is.\n",
      "Q: Who is an accountant?\n",
      "A: Emily\n",
      "P: Christian\n",
      "\n",
      "\n",
      "----\n",
      "C: Christian is not an adviser. Stephanie is.\n",
      "Q: Who is an adviser?\n",
      "A: Stephanie\n",
      "P: Christian\n",
      "\n",
      "\n",
      "----\n",
      "C: Nathan is not an agent. Alexis is.\n",
      "Q: Who is an agent?\n",
      "A: Alexis\n",
      "P: Nathan\n",
      "\n",
      "\n",
      "----\n",
      "\n",
      "\n",
      "Negation in question only.\n",
      "Test cases:      481\n",
      "Test cases run:  100\n",
      "Fails (rate):    100 (100.0%)\n",
      "\n",
      "Example fails:\n",
      "C: Brittany is a nurse. Lauren is an activist.\n",
      "Q: Who is not a nurse?\n",
      "A: Lauren\n",
      "P: Brittany\n",
      "\n",
      "C: Brittany is a nurse. Lauren is an activist.\n",
      "Q: Who is not an activist?\n",
      "A: Brittany\n",
      "P: Lauren\n",
      "\n",
      "C: Lauren is an activist. Brittany is a nurse.\n",
      "Q: Who is not a nurse?\n",
      "A: Lauren\n",
      "P: Brittany\n",
      "\n",
      "\n",
      "----\n",
      "C: Brittany is an architect. Anna is an academic.\n",
      "Q: Who is not an architect?\n",
      "A: Anna\n",
      "P: Brittany\n",
      "\n",
      "C: Brittany is an architect. Anna is an academic.\n",
      "Q: Who is not an academic?\n",
      "A: Brittany\n",
      "P: Anna\n",
      "\n",
      "C: Anna is an academic. Brittany is an architect.\n",
      "Q: Who is not an architect?\n",
      "A: Anna\n",
      "P: Brittany\n",
      "\n",
      "\n",
      "----\n",
      "C: Nicole is an executive. Sophia is an advisor.\n",
      "Q: Who is not an executive?\n",
      "A: Sophia\n",
      "P: Nicole\n",
      "\n",
      "C: Nicole is an executive. Sophia is an advisor.\n",
      "Q: Who is not an advisor?\n",
      "A: Nicole\n",
      "P: Sophia\n",
      "\n",
      "C: Sophia is an advisor. Nicole is an executive.\n",
      "Q: Who is not an executive?\n",
      "A: Sophia\n",
      "P: Nicole\n",
      "\n",
      "\n",
      "----\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "Coref\n",
      "\n",
      "Basic coref, he / she\n",
      "Test cases:      477\n",
      "Test cases run:  100\n",
      "Fails (rate):    100 (100.0%)\n",
      "\n",
      "Example fails:\n",
      "C: Connor and Sara are friends. He is a journalist, and she is an architect.\n",
      "Q: Who is a journalist?\n",
      "A: Connor\n",
      "P: Connor and Sara\n",
      "\n",
      "C: Connor and Sara are friends. He is a journalist, and she is an architect.\n",
      "Q: Who is an architect?\n",
      "A: Sara\n",
      "P: Connor and Sara\n",
      "\n",
      "C: Sara and Connor are friends. He is a journalist, and she is an architect.\n",
      "Q: Who is a journalist?\n",
      "A: Connor\n",
      "P: Sara and Connor\n",
      "\n",
      "\n",
      "----\n",
      "C: Steven and Melanie are friends. He is an assistant, and she is an interpreter.\n",
      "Q: Who is an assistant?\n",
      "A: Steven\n",
      "P: Steven and Melanie\n",
      "\n",
      "C: Steven and Melanie are friends. He is an assistant, and she is an interpreter.\n",
      "Q: Who is an interpreter?\n",
      "A: Melanie\n",
      "P: Steven and Melanie\n",
      "\n",
      "C: Melanie and Steven are friends. He is an assistant, and she is an interpreter.\n",
      "Q: Who is an assistant?\n",
      "A: Steven\n",
      "P: Melanie and Steven\n",
      "\n",
      "\n",
      "----\n",
      "C: Joseph and Olivia are friends. He is a historian, and she is an educator.\n",
      "Q: Who is a historian?\n",
      "A: Joseph\n",
      "P: Joseph and Olivia\n",
      "\n",
      "C: Joseph and Olivia are friends. He is a historian, and she is an educator.\n",
      "Q: Who is an educator?\n",
      "A: Olivia\n",
      "P: Joseph and Olivia\n",
      "\n",
      "C: Olivia and Joseph are friends. He is a historian, and she is an educator.\n",
      "Q: Who is a historian?\n",
      "A: Joseph\n",
      "P: Olivia and Joseph\n",
      "\n",
      "\n",
      "----\n",
      "\n",
      "\n",
      "Basic coref, his / her\n",
      "Test cases:      500\n",
      "Test cases run:  100\n",
      "Fails (rate):    97 (97.0%)\n",
      "\n",
      "Example fails:\n",
      "C: Jayden and Taylor are friends. His mom is an executive.\n",
      "Q: Whose mom is an executive?\n",
      "A: Jayden\n",
      "P: Jayden and Taylor\n",
      "\n",
      "C: Taylor and Jayden are friends. His mom is an executive.\n",
      "Q: Whose mom is an executive?\n",
      "A: Jayden\n",
      "P: Taylor and Jayden\n",
      "\n",
      "\n",
      "----\n",
      "C: Steven and Kelly are friends. Her mom is an assistant.\n",
      "Q: Whose mom is an assistant?\n",
      "A: Kelly\n",
      "P: Steven and Kelly\n",
      "\n",
      "C: Kelly and Steven are friends. Her mom is an assistant.\n",
      "Q: Whose mom is an assistant?\n",
      "A: Kelly\n",
      "P: Kelly and Steven\n",
      "\n",
      "\n",
      "----\n",
      "C: Shawn and Amber are friends. Her mom is a photographer.\n",
      "Q: Whose mom is a photographer?\n",
      "A: Amber\n",
      "P: Shawn and Amber\n",
      "\n",
      "C: Amber and Shawn are friends. Her mom is a photographer.\n",
      "Q: Whose mom is a photographer?\n",
      "A: Amber\n",
      "P: Amber and Shawn\n",
      "\n",
      "\n",
      "----\n",
      "\n",
      "\n",
      "Former / Latter\n",
      "Test cases:      475\n",
      "Test cases run:  100\n",
      "Fails (rate):    100 (100.0%)\n",
      "\n",
      "Example fails:\n",
      "C: Adam and Natalie are friends. The former is a journalist.\n",
      "Q: Who is a journalist?\n",
      "A: Adam\n",
      "P: Adam and Natalie\n",
      "\n",
      "C: Natalie and Adam are friends. The latter is a journalist.\n",
      "Q: Who is a journalist?\n",
      "A: Adam\n",
      "P: Natalie and Adam\n",
      "\n",
      "C: Adam and Natalie are friends. The former is a journalist and the latter is an actor.\n",
      "Q: Who is a journalist?\n",
      "A: Adam\n",
      "P: Adam and Natalie\n",
      "\n",
      "\n",
      "----\n",
      "C: Kyle and Lauren are friends. The former is an administrator.\n",
      "Q: Who is an administrator?\n",
      "A: Kyle\n",
      "P: Kyle and Lauren\n",
      "\n",
      "C: Lauren and Kyle are friends. The latter is an administrator.\n",
      "Q: Who is an administrator?\n",
      "A: Kyle\n",
      "P: Lauren and Kyle\n",
      "\n",
      "C: Kyle and Lauren are friends. The former is an administrator and the latter is an activist.\n",
      "Q: Who is an administrator?\n",
      "A: Kyle\n",
      "P: Kyle and Lauren\n",
      "\n",
      "\n",
      "----\n",
      "C: Sarah and Megan are friends. The former is an investigator.\n",
      "Q: Who is an investigator?\n",
      "A: Sarah\n",
      "P: Sarah and Megan\n",
      "\n",
      "C: Megan and Sarah are friends. The latter is an investigator.\n",
      "Q: Who is an investigator?\n",
      "A: Sarah\n",
      "P: Megan and Sarah\n",
      "\n",
      "C: Sarah and Megan are friends. The former is an investigator and the latter is an auditor.\n",
      "Q: Who is an investigator?\n",
      "A: Sarah\n",
      "P: Sarah and Megan\n",
      "\n",
      "\n",
      "----\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "SRL\n",
      "\n",
      "Agent / object distinction\n",
      "Test cases:      497\n",
      "Test cases run:  100\n",
      "Fails (rate):    64 (64.0%)\n",
      "\n",
      "Example fails:\n",
      "C: Victoria trusts Elizabeth.\n",
      "Q: Who trusts?\n",
      "A: Victoria\n",
      "P: Elizabeth.\n",
      "\n",
      "\n",
      "----\n",
      "C: Mary loves Jeffrey.\n",
      "Q: Who loves?\n",
      "A: Mary\n",
      "P: Jeffrey.\n",
      "\n",
      "\n",
      "----\n",
      "C: Amanda remembers Victoria.\n",
      "Q: Who remembers?\n",
      "A: Amanda\n",
      "P: Amanda remembers Victoria.\n",
      "\n",
      "\n",
      "----\n",
      "\n",
      "\n",
      "Agent / object distinction with 3 agents\n",
      "Test cases:      483\n",
      "Test cases run:  100\n",
      "Fails (rate):    98 (98.0%)\n",
      "\n",
      "Example fails:\n",
      "C: Andrea recognizes Alexis. Justin is recognized by Alexis.\n",
      "Q: Who recognizes Justin?\n",
      "A: Alexis\n",
      "P: Andrea\n",
      "\n",
      "C: Andrea recognizes Alexis. Justin is recognized by Alexis.\n",
      "Q: Who is recognized by Andrea?\n",
      "A: Alexis\n",
      "P: Justin\n",
      "\n",
      "C: Alexis is recognized by Andrea. Justin is recognized by Alexis.\n",
      "Q: Who recognizes Justin?\n",
      "A: Alexis\n",
      "P: Andrea.\n",
      "\n",
      "\n",
      "----\n",
      "C: Tyler blames Zachary. Joshua is blamed by Zachary.\n",
      "Q: Who blames Joshua?\n",
      "A: Zachary\n",
      "P: Tyler\n",
      "\n",
      "C: Tyler blames Zachary. Joshua is blamed by Zachary.\n",
      "Q: Who is blamed by Tyler?\n",
      "A: Zachary\n",
      "P: Joshua\n",
      "\n",
      "C: Zachary is blamed by Tyler. Joshua is blamed by Zachary.\n",
      "Q: Who is blamed by Tyler?\n",
      "A: Zachary\n",
      "P: Joshua\n",
      "\n",
      "\n",
      "----\n",
      "C: Jeremy notices Mark. Lauren is noticed by Mark.\n",
      "Q: Who notices Lauren?\n",
      "A: Mark\n",
      "P: Jeremy\n",
      "\n",
      "\n",
      "----\n",
      "\n",
      "\n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "suite.summary(format_example_fn=format_squad_with_context)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}